libcxgbi.c 78 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098
  1. /*
  2. * libcxgbi.c: Chelsio common library for T3/T4 iSCSI driver.
  3. *
  4. * Copyright (c) 2010-2015 Chelsio Communications, Inc.
  5. *
  6. * This program is free software; you can redistribute it and/or modify
  7. * it under the terms of the GNU General Public License as published by
  8. * the Free Software Foundation.
  9. *
  10. * Written by: Karen Xie (kxie@chelsio.com)
  11. * Written by: Rakesh Ranjan (rranjan@chelsio.com)
  12. */
  13. #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
  14. #include <linux/skbuff.h>
  15. #include <linux/crypto.h>
  16. #include <linux/scatterlist.h>
  17. #include <linux/pci.h>
  18. #include <scsi/scsi.h>
  19. #include <scsi/scsi_cmnd.h>
  20. #include <scsi/scsi_host.h>
  21. #include <linux/if_vlan.h>
  22. #include <linux/inet.h>
  23. #include <net/dst.h>
  24. #include <net/route.h>
  25. #include <net/ipv6.h>
  26. #include <net/ip6_route.h>
  27. #include <net/addrconf.h>
  28. #include <linux/inetdevice.h> /* ip_dev_find */
  29. #include <linux/module.h>
  30. #include <net/tcp.h>
  31. static unsigned int dbg_level;
  32. #include "libcxgbi.h"
  33. #define DRV_MODULE_NAME "libcxgbi"
  34. #define DRV_MODULE_DESC "Chelsio iSCSI driver library"
  35. #define DRV_MODULE_VERSION "0.9.1-ko"
  36. #define DRV_MODULE_RELDATE "Apr. 2015"
  37. static char version[] =
  38. DRV_MODULE_DESC " " DRV_MODULE_NAME
  39. " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
  40. MODULE_AUTHOR("Chelsio Communications, Inc.");
  41. MODULE_DESCRIPTION(DRV_MODULE_DESC);
  42. MODULE_VERSION(DRV_MODULE_VERSION);
  43. MODULE_LICENSE("GPL");
  44. module_param(dbg_level, uint, 0644);
  45. MODULE_PARM_DESC(dbg_level, "libiscsi debug level (default=0)");
  46. /*
  47. * cxgbi device management
  48. * maintains a list of the cxgbi devices
  49. */
  50. static LIST_HEAD(cdev_list);
  51. static DEFINE_MUTEX(cdev_mutex);
  52. static LIST_HEAD(cdev_rcu_list);
  53. static DEFINE_SPINLOCK(cdev_rcu_lock);
  54. static inline void cxgbi_decode_sw_tag(u32 sw_tag, int *idx, int *age)
  55. {
  56. if (age)
  57. *age = sw_tag & 0x7FFF;
  58. if (idx)
  59. *idx = (sw_tag >> 16) & 0x7FFF;
  60. }
  61. int cxgbi_device_portmap_create(struct cxgbi_device *cdev, unsigned int base,
  62. unsigned int max_conn)
  63. {
  64. struct cxgbi_ports_map *pmap = &cdev->pmap;
  65. pmap->port_csk = kvzalloc(array_size(max_conn,
  66. sizeof(struct cxgbi_sock *)),
  67. GFP_KERNEL | __GFP_NOWARN);
  68. if (!pmap->port_csk) {
  69. pr_warn("cdev 0x%p, portmap OOM %u.\n", cdev, max_conn);
  70. return -ENOMEM;
  71. }
  72. pmap->max_connect = max_conn;
  73. pmap->sport_base = base;
  74. spin_lock_init(&pmap->lock);
  75. return 0;
  76. }
  77. EXPORT_SYMBOL_GPL(cxgbi_device_portmap_create);
  78. void cxgbi_device_portmap_cleanup(struct cxgbi_device *cdev)
  79. {
  80. struct cxgbi_ports_map *pmap = &cdev->pmap;
  81. struct cxgbi_sock *csk;
  82. int i;
  83. for (i = 0; i < pmap->max_connect; i++) {
  84. if (pmap->port_csk[i]) {
  85. csk = pmap->port_csk[i];
  86. pmap->port_csk[i] = NULL;
  87. log_debug(1 << CXGBI_DBG_SOCK,
  88. "csk 0x%p, cdev 0x%p, offload down.\n",
  89. csk, cdev);
  90. spin_lock_bh(&csk->lock);
  91. cxgbi_sock_set_flag(csk, CTPF_OFFLOAD_DOWN);
  92. cxgbi_sock_closed(csk);
  93. spin_unlock_bh(&csk->lock);
  94. cxgbi_sock_put(csk);
  95. }
  96. }
  97. }
  98. EXPORT_SYMBOL_GPL(cxgbi_device_portmap_cleanup);
  99. static inline void cxgbi_device_destroy(struct cxgbi_device *cdev)
  100. {
  101. log_debug(1 << CXGBI_DBG_DEV,
  102. "cdev 0x%p, p# %u.\n", cdev, cdev->nports);
  103. cxgbi_hbas_remove(cdev);
  104. cxgbi_device_portmap_cleanup(cdev);
  105. if (cdev->cdev2ppm)
  106. cxgbi_ppm_release(cdev->cdev2ppm(cdev));
  107. if (cdev->pmap.max_connect)
  108. kvfree(cdev->pmap.port_csk);
  109. kfree(cdev);
  110. }
  111. struct cxgbi_device *cxgbi_device_register(unsigned int extra,
  112. unsigned int nports)
  113. {
  114. struct cxgbi_device *cdev;
  115. cdev = kzalloc(sizeof(*cdev) + extra + nports *
  116. (sizeof(struct cxgbi_hba *) +
  117. sizeof(struct net_device *)),
  118. GFP_KERNEL);
  119. if (!cdev) {
  120. pr_warn("nport %d, OOM.\n", nports);
  121. return NULL;
  122. }
  123. cdev->ports = (struct net_device **)(cdev + 1);
  124. cdev->hbas = (struct cxgbi_hba **)(((char*)cdev->ports) + nports *
  125. sizeof(struct net_device *));
  126. if (extra)
  127. cdev->dd_data = ((char *)cdev->hbas) +
  128. nports * sizeof(struct cxgbi_hba *);
  129. spin_lock_init(&cdev->pmap.lock);
  130. mutex_lock(&cdev_mutex);
  131. list_add_tail(&cdev->list_head, &cdev_list);
  132. mutex_unlock(&cdev_mutex);
  133. spin_lock(&cdev_rcu_lock);
  134. list_add_tail_rcu(&cdev->rcu_node, &cdev_rcu_list);
  135. spin_unlock(&cdev_rcu_lock);
  136. log_debug(1 << CXGBI_DBG_DEV,
  137. "cdev 0x%p, p# %u.\n", cdev, nports);
  138. return cdev;
  139. }
  140. EXPORT_SYMBOL_GPL(cxgbi_device_register);
  141. void cxgbi_device_unregister(struct cxgbi_device *cdev)
  142. {
  143. log_debug(1 << CXGBI_DBG_DEV,
  144. "cdev 0x%p, p# %u,%s.\n",
  145. cdev, cdev->nports, cdev->nports ? cdev->ports[0]->name : "");
  146. mutex_lock(&cdev_mutex);
  147. list_del(&cdev->list_head);
  148. mutex_unlock(&cdev_mutex);
  149. spin_lock(&cdev_rcu_lock);
  150. list_del_rcu(&cdev->rcu_node);
  151. spin_unlock(&cdev_rcu_lock);
  152. synchronize_rcu();
  153. cxgbi_device_destroy(cdev);
  154. }
  155. EXPORT_SYMBOL_GPL(cxgbi_device_unregister);
  156. void cxgbi_device_unregister_all(unsigned int flag)
  157. {
  158. struct cxgbi_device *cdev, *tmp;
  159. mutex_lock(&cdev_mutex);
  160. list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) {
  161. if ((cdev->flags & flag) == flag) {
  162. mutex_unlock(&cdev_mutex);
  163. cxgbi_device_unregister(cdev);
  164. mutex_lock(&cdev_mutex);
  165. }
  166. }
  167. mutex_unlock(&cdev_mutex);
  168. }
  169. EXPORT_SYMBOL_GPL(cxgbi_device_unregister_all);
  170. struct cxgbi_device *cxgbi_device_find_by_lldev(void *lldev)
  171. {
  172. struct cxgbi_device *cdev, *tmp;
  173. mutex_lock(&cdev_mutex);
  174. list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) {
  175. if (cdev->lldev == lldev) {
  176. mutex_unlock(&cdev_mutex);
  177. return cdev;
  178. }
  179. }
  180. mutex_unlock(&cdev_mutex);
  181. log_debug(1 << CXGBI_DBG_DEV,
  182. "lldev 0x%p, NO match found.\n", lldev);
  183. return NULL;
  184. }
  185. EXPORT_SYMBOL_GPL(cxgbi_device_find_by_lldev);
  186. struct cxgbi_device *cxgbi_device_find_by_netdev(struct net_device *ndev,
  187. int *port)
  188. {
  189. struct net_device *vdev = NULL;
  190. struct cxgbi_device *cdev, *tmp;
  191. int i;
  192. if (is_vlan_dev(ndev)) {
  193. vdev = ndev;
  194. ndev = vlan_dev_real_dev(ndev);
  195. log_debug(1 << CXGBI_DBG_DEV,
  196. "vlan dev %s -> %s.\n", vdev->name, ndev->name);
  197. }
  198. mutex_lock(&cdev_mutex);
  199. list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) {
  200. for (i = 0; i < cdev->nports; i++) {
  201. if (ndev == cdev->ports[i]) {
  202. cdev->hbas[i]->vdev = vdev;
  203. mutex_unlock(&cdev_mutex);
  204. if (port)
  205. *port = i;
  206. return cdev;
  207. }
  208. }
  209. }
  210. mutex_unlock(&cdev_mutex);
  211. log_debug(1 << CXGBI_DBG_DEV,
  212. "ndev 0x%p, %s, NO match found.\n", ndev, ndev->name);
  213. return NULL;
  214. }
  215. EXPORT_SYMBOL_GPL(cxgbi_device_find_by_netdev);
  216. struct cxgbi_device *cxgbi_device_find_by_netdev_rcu(struct net_device *ndev,
  217. int *port)
  218. {
  219. struct net_device *vdev = NULL;
  220. struct cxgbi_device *cdev;
  221. int i;
  222. if (is_vlan_dev(ndev)) {
  223. vdev = ndev;
  224. ndev = vlan_dev_real_dev(ndev);
  225. pr_info("vlan dev %s -> %s.\n", vdev->name, ndev->name);
  226. }
  227. rcu_read_lock();
  228. list_for_each_entry_rcu(cdev, &cdev_rcu_list, rcu_node) {
  229. for (i = 0; i < cdev->nports; i++) {
  230. if (ndev == cdev->ports[i]) {
  231. cdev->hbas[i]->vdev = vdev;
  232. rcu_read_unlock();
  233. if (port)
  234. *port = i;
  235. return cdev;
  236. }
  237. }
  238. }
  239. rcu_read_unlock();
  240. log_debug(1 << CXGBI_DBG_DEV,
  241. "ndev 0x%p, %s, NO match found.\n", ndev, ndev->name);
  242. return NULL;
  243. }
  244. EXPORT_SYMBOL_GPL(cxgbi_device_find_by_netdev_rcu);
  245. static struct cxgbi_device *cxgbi_device_find_by_mac(struct net_device *ndev,
  246. int *port)
  247. {
  248. struct net_device *vdev = NULL;
  249. struct cxgbi_device *cdev, *tmp;
  250. int i;
  251. if (is_vlan_dev(ndev)) {
  252. vdev = ndev;
  253. ndev = vlan_dev_real_dev(ndev);
  254. pr_info("vlan dev %s -> %s.\n", vdev->name, ndev->name);
  255. }
  256. mutex_lock(&cdev_mutex);
  257. list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) {
  258. for (i = 0; i < cdev->nports; i++) {
  259. if (!memcmp(ndev->dev_addr, cdev->ports[i]->dev_addr,
  260. MAX_ADDR_LEN)) {
  261. cdev->hbas[i]->vdev = vdev;
  262. mutex_unlock(&cdev_mutex);
  263. if (port)
  264. *port = i;
  265. return cdev;
  266. }
  267. }
  268. }
  269. mutex_unlock(&cdev_mutex);
  270. log_debug(1 << CXGBI_DBG_DEV,
  271. "ndev 0x%p, %s, NO match mac found.\n",
  272. ndev, ndev->name);
  273. return NULL;
  274. }
  275. void cxgbi_hbas_remove(struct cxgbi_device *cdev)
  276. {
  277. int i;
  278. struct cxgbi_hba *chba;
  279. log_debug(1 << CXGBI_DBG_DEV,
  280. "cdev 0x%p, p#%u.\n", cdev, cdev->nports);
  281. for (i = 0; i < cdev->nports; i++) {
  282. chba = cdev->hbas[i];
  283. if (chba) {
  284. cdev->hbas[i] = NULL;
  285. iscsi_host_remove(chba->shost);
  286. pci_dev_put(cdev->pdev);
  287. iscsi_host_free(chba->shost);
  288. }
  289. }
  290. }
  291. EXPORT_SYMBOL_GPL(cxgbi_hbas_remove);
  292. int cxgbi_hbas_add(struct cxgbi_device *cdev, u64 max_lun,
  293. unsigned int max_conns, struct scsi_host_template *sht,
  294. struct scsi_transport_template *stt)
  295. {
  296. struct cxgbi_hba *chba;
  297. struct Scsi_Host *shost;
  298. int i, err;
  299. log_debug(1 << CXGBI_DBG_DEV, "cdev 0x%p, p#%u.\n", cdev, cdev->nports);
  300. for (i = 0; i < cdev->nports; i++) {
  301. shost = iscsi_host_alloc(sht, sizeof(*chba), 1);
  302. if (!shost) {
  303. pr_info("0x%p, p%d, %s, host alloc failed.\n",
  304. cdev, i, cdev->ports[i]->name);
  305. err = -ENOMEM;
  306. goto err_out;
  307. }
  308. shost->transportt = stt;
  309. shost->max_lun = max_lun;
  310. shost->max_id = max_conns - 1;
  311. shost->max_channel = 0;
  312. shost->max_cmd_len = SCSI_MAX_VARLEN_CDB_SIZE;
  313. chba = iscsi_host_priv(shost);
  314. chba->cdev = cdev;
  315. chba->ndev = cdev->ports[i];
  316. chba->shost = shost;
  317. shost->can_queue = sht->can_queue - ISCSI_MGMT_CMDS_MAX;
  318. log_debug(1 << CXGBI_DBG_DEV,
  319. "cdev 0x%p, p#%d %s: chba 0x%p.\n",
  320. cdev, i, cdev->ports[i]->name, chba);
  321. pci_dev_get(cdev->pdev);
  322. err = iscsi_host_add(shost, &cdev->pdev->dev);
  323. if (err) {
  324. pr_info("cdev 0x%p, p#%d %s, host add failed.\n",
  325. cdev, i, cdev->ports[i]->name);
  326. pci_dev_put(cdev->pdev);
  327. scsi_host_put(shost);
  328. goto err_out;
  329. }
  330. cdev->hbas[i] = chba;
  331. }
  332. return 0;
  333. err_out:
  334. cxgbi_hbas_remove(cdev);
  335. return err;
  336. }
  337. EXPORT_SYMBOL_GPL(cxgbi_hbas_add);
  338. /*
  339. * iSCSI offload
  340. *
  341. * - source port management
  342. * To find a free source port in the port allocation map we use a very simple
  343. * rotor scheme to look for the next free port.
  344. *
  345. * If a source port has been specified make sure that it doesn't collide with
  346. * our normal source port allocation map. If it's outside the range of our
  347. * allocation/deallocation scheme just let them use it.
  348. *
  349. * If the source port is outside our allocation range, the caller is
  350. * responsible for keeping track of their port usage.
  351. */
  352. static struct cxgbi_sock *find_sock_on_port(struct cxgbi_device *cdev,
  353. unsigned char port_id)
  354. {
  355. struct cxgbi_ports_map *pmap = &cdev->pmap;
  356. unsigned int i;
  357. unsigned int used;
  358. if (!pmap->max_connect || !pmap->used)
  359. return NULL;
  360. spin_lock_bh(&pmap->lock);
  361. used = pmap->used;
  362. for (i = 0; used && i < pmap->max_connect; i++) {
  363. struct cxgbi_sock *csk = pmap->port_csk[i];
  364. if (csk) {
  365. if (csk->port_id == port_id) {
  366. spin_unlock_bh(&pmap->lock);
  367. return csk;
  368. }
  369. used--;
  370. }
  371. }
  372. spin_unlock_bh(&pmap->lock);
  373. return NULL;
  374. }
  375. static int sock_get_port(struct cxgbi_sock *csk)
  376. {
  377. struct cxgbi_device *cdev = csk->cdev;
  378. struct cxgbi_ports_map *pmap = &cdev->pmap;
  379. unsigned int start;
  380. int idx;
  381. __be16 *port;
  382. if (!pmap->max_connect) {
  383. pr_err("cdev 0x%p, p#%u %s, NO port map.\n",
  384. cdev, csk->port_id, cdev->ports[csk->port_id]->name);
  385. return -EADDRNOTAVAIL;
  386. }
  387. if (csk->csk_family == AF_INET)
  388. port = &csk->saddr.sin_port;
  389. else /* ipv6 */
  390. port = &csk->saddr6.sin6_port;
  391. if (*port) {
  392. pr_err("source port NON-ZERO %u.\n",
  393. ntohs(*port));
  394. return -EADDRINUSE;
  395. }
  396. spin_lock_bh(&pmap->lock);
  397. if (pmap->used >= pmap->max_connect) {
  398. spin_unlock_bh(&pmap->lock);
  399. pr_info("cdev 0x%p, p#%u %s, ALL ports used.\n",
  400. cdev, csk->port_id, cdev->ports[csk->port_id]->name);
  401. return -EADDRNOTAVAIL;
  402. }
  403. start = idx = pmap->next;
  404. do {
  405. if (++idx >= pmap->max_connect)
  406. idx = 0;
  407. if (!pmap->port_csk[idx]) {
  408. pmap->used++;
  409. *port = htons(pmap->sport_base + idx);
  410. pmap->next = idx;
  411. pmap->port_csk[idx] = csk;
  412. spin_unlock_bh(&pmap->lock);
  413. cxgbi_sock_get(csk);
  414. log_debug(1 << CXGBI_DBG_SOCK,
  415. "cdev 0x%p, p#%u %s, p %u, %u.\n",
  416. cdev, csk->port_id,
  417. cdev->ports[csk->port_id]->name,
  418. pmap->sport_base + idx, pmap->next);
  419. return 0;
  420. }
  421. } while (idx != start);
  422. spin_unlock_bh(&pmap->lock);
  423. /* should not happen */
  424. pr_warn("cdev 0x%p, p#%u %s, next %u?\n",
  425. cdev, csk->port_id, cdev->ports[csk->port_id]->name,
  426. pmap->next);
  427. return -EADDRNOTAVAIL;
  428. }
  429. static void sock_put_port(struct cxgbi_sock *csk)
  430. {
  431. struct cxgbi_device *cdev = csk->cdev;
  432. struct cxgbi_ports_map *pmap = &cdev->pmap;
  433. __be16 *port;
  434. if (csk->csk_family == AF_INET)
  435. port = &csk->saddr.sin_port;
  436. else /* ipv6 */
  437. port = &csk->saddr6.sin6_port;
  438. if (*port) {
  439. int idx = ntohs(*port) - pmap->sport_base;
  440. *port = 0;
  441. if (idx < 0 || idx >= pmap->max_connect) {
  442. pr_err("cdev 0x%p, p#%u %s, port %u OOR.\n",
  443. cdev, csk->port_id,
  444. cdev->ports[csk->port_id]->name,
  445. ntohs(*port));
  446. return;
  447. }
  448. spin_lock_bh(&pmap->lock);
  449. pmap->port_csk[idx] = NULL;
  450. pmap->used--;
  451. spin_unlock_bh(&pmap->lock);
  452. log_debug(1 << CXGBI_DBG_SOCK,
  453. "cdev 0x%p, p#%u %s, release %u.\n",
  454. cdev, csk->port_id, cdev->ports[csk->port_id]->name,
  455. pmap->sport_base + idx);
  456. cxgbi_sock_put(csk);
  457. }
  458. }
  459. /*
  460. * iscsi tcp connection
  461. */
  462. void cxgbi_sock_free_cpl_skbs(struct cxgbi_sock *csk)
  463. {
  464. if (csk->cpl_close) {
  465. kfree_skb(csk->cpl_close);
  466. csk->cpl_close = NULL;
  467. }
  468. if (csk->cpl_abort_req) {
  469. kfree_skb(csk->cpl_abort_req);
  470. csk->cpl_abort_req = NULL;
  471. }
  472. if (csk->cpl_abort_rpl) {
  473. kfree_skb(csk->cpl_abort_rpl);
  474. csk->cpl_abort_rpl = NULL;
  475. }
  476. }
  477. EXPORT_SYMBOL_GPL(cxgbi_sock_free_cpl_skbs);
  478. static struct cxgbi_sock *cxgbi_sock_create(struct cxgbi_device *cdev)
  479. {
  480. struct cxgbi_sock *csk = kzalloc(sizeof(*csk), GFP_NOIO);
  481. if (!csk) {
  482. pr_info("alloc csk %zu failed.\n", sizeof(*csk));
  483. return NULL;
  484. }
  485. if (cdev->csk_alloc_cpls(csk) < 0) {
  486. pr_info("csk 0x%p, alloc cpls failed.\n", csk);
  487. kfree(csk);
  488. return NULL;
  489. }
  490. spin_lock_init(&csk->lock);
  491. kref_init(&csk->refcnt);
  492. skb_queue_head_init(&csk->receive_queue);
  493. skb_queue_head_init(&csk->write_queue);
  494. timer_setup(&csk->retry_timer, NULL, 0);
  495. init_completion(&csk->cmpl);
  496. rwlock_init(&csk->callback_lock);
  497. csk->cdev = cdev;
  498. csk->flags = 0;
  499. cxgbi_sock_set_state(csk, CTP_CLOSED);
  500. log_debug(1 << CXGBI_DBG_SOCK, "cdev 0x%p, new csk 0x%p.\n", cdev, csk);
  501. return csk;
  502. }
  503. static struct rtable *find_route_ipv4(struct flowi4 *fl4,
  504. __be32 saddr, __be32 daddr,
  505. __be16 sport, __be16 dport, u8 tos,
  506. int ifindex)
  507. {
  508. struct rtable *rt;
  509. rt = ip_route_output_ports(&init_net, fl4, NULL, daddr, saddr,
  510. dport, sport, IPPROTO_TCP, tos, ifindex);
  511. if (IS_ERR(rt))
  512. return NULL;
  513. return rt;
  514. }
  515. static struct cxgbi_sock *
  516. cxgbi_check_route(struct sockaddr *dst_addr, int ifindex)
  517. {
  518. struct sockaddr_in *daddr = (struct sockaddr_in *)dst_addr;
  519. struct dst_entry *dst;
  520. struct net_device *ndev;
  521. struct cxgbi_device *cdev;
  522. struct rtable *rt = NULL;
  523. struct neighbour *n;
  524. struct flowi4 fl4;
  525. struct cxgbi_sock *csk = NULL;
  526. unsigned int mtu = 0;
  527. int port = 0xFFFF;
  528. int err = 0;
  529. rt = find_route_ipv4(&fl4, 0, daddr->sin_addr.s_addr, 0,
  530. daddr->sin_port, 0, ifindex);
  531. if (!rt) {
  532. pr_info("no route to ipv4 0x%x, port %u.\n",
  533. be32_to_cpu(daddr->sin_addr.s_addr),
  534. be16_to_cpu(daddr->sin_port));
  535. err = -ENETUNREACH;
  536. goto err_out;
  537. }
  538. dst = &rt->dst;
  539. n = dst_neigh_lookup(dst, &daddr->sin_addr.s_addr);
  540. if (!n) {
  541. err = -ENODEV;
  542. goto rel_rt;
  543. }
  544. ndev = n->dev;
  545. if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
  546. pr_info("multi-cast route %pI4, port %u, dev %s.\n",
  547. &daddr->sin_addr.s_addr, ntohs(daddr->sin_port),
  548. ndev->name);
  549. err = -ENETUNREACH;
  550. goto rel_neigh;
  551. }
  552. if (ndev->flags & IFF_LOOPBACK) {
  553. ndev = ip_dev_find(&init_net, daddr->sin_addr.s_addr);
  554. if (!ndev) {
  555. err = -ENETUNREACH;
  556. goto rel_neigh;
  557. }
  558. mtu = ndev->mtu;
  559. pr_info("rt dev %s, loopback -> %s, mtu %u.\n",
  560. n->dev->name, ndev->name, mtu);
  561. }
  562. if (!(ndev->flags & IFF_UP) || !netif_carrier_ok(ndev)) {
  563. pr_info("%s interface not up.\n", ndev->name);
  564. err = -ENETDOWN;
  565. goto rel_neigh;
  566. }
  567. cdev = cxgbi_device_find_by_netdev(ndev, &port);
  568. if (!cdev)
  569. cdev = cxgbi_device_find_by_mac(ndev, &port);
  570. if (!cdev) {
  571. pr_info("dst %pI4, %s, NOT cxgbi device.\n",
  572. &daddr->sin_addr.s_addr, ndev->name);
  573. err = -ENETUNREACH;
  574. goto rel_neigh;
  575. }
  576. log_debug(1 << CXGBI_DBG_SOCK,
  577. "route to %pI4 :%u, ndev p#%d,%s, cdev 0x%p.\n",
  578. &daddr->sin_addr.s_addr, ntohs(daddr->sin_port),
  579. port, ndev->name, cdev);
  580. csk = cxgbi_sock_create(cdev);
  581. if (!csk) {
  582. err = -ENOMEM;
  583. goto rel_neigh;
  584. }
  585. csk->cdev = cdev;
  586. csk->port_id = port;
  587. csk->mtu = mtu;
  588. csk->dst = dst;
  589. csk->csk_family = AF_INET;
  590. csk->daddr.sin_addr.s_addr = daddr->sin_addr.s_addr;
  591. csk->daddr.sin_port = daddr->sin_port;
  592. csk->daddr.sin_family = daddr->sin_family;
  593. csk->saddr.sin_family = daddr->sin_family;
  594. csk->saddr.sin_addr.s_addr = fl4.saddr;
  595. neigh_release(n);
  596. return csk;
  597. rel_neigh:
  598. neigh_release(n);
  599. rel_rt:
  600. ip_rt_put(rt);
  601. err_out:
  602. return ERR_PTR(err);
  603. }
  604. #if IS_ENABLED(CONFIG_IPV6)
  605. static struct rt6_info *find_route_ipv6(const struct in6_addr *saddr,
  606. const struct in6_addr *daddr,
  607. int ifindex)
  608. {
  609. struct flowi6 fl;
  610. memset(&fl, 0, sizeof(fl));
  611. fl.flowi6_oif = ifindex;
  612. if (saddr)
  613. memcpy(&fl.saddr, saddr, sizeof(struct in6_addr));
  614. if (daddr)
  615. memcpy(&fl.daddr, daddr, sizeof(struct in6_addr));
  616. return (struct rt6_info *)ip6_route_output(&init_net, NULL, &fl);
  617. }
  618. static struct cxgbi_sock *
  619. cxgbi_check_route6(struct sockaddr *dst_addr, int ifindex)
  620. {
  621. struct sockaddr_in6 *daddr6 = (struct sockaddr_in6 *)dst_addr;
  622. struct dst_entry *dst;
  623. struct net_device *ndev;
  624. struct cxgbi_device *cdev;
  625. struct rt6_info *rt = NULL;
  626. struct neighbour *n;
  627. struct in6_addr pref_saddr;
  628. struct cxgbi_sock *csk = NULL;
  629. unsigned int mtu = 0;
  630. int port = 0xFFFF;
  631. int err = 0;
  632. rt = find_route_ipv6(NULL, &daddr6->sin6_addr, ifindex);
  633. if (!rt) {
  634. pr_info("no route to ipv6 %pI6 port %u\n",
  635. daddr6->sin6_addr.s6_addr,
  636. be16_to_cpu(daddr6->sin6_port));
  637. err = -ENETUNREACH;
  638. goto err_out;
  639. }
  640. dst = &rt->dst;
  641. n = dst_neigh_lookup(dst, &daddr6->sin6_addr);
  642. if (!n) {
  643. pr_info("%pI6, port %u, dst no neighbour.\n",
  644. daddr6->sin6_addr.s6_addr,
  645. be16_to_cpu(daddr6->sin6_port));
  646. err = -ENETUNREACH;
  647. goto rel_rt;
  648. }
  649. ndev = n->dev;
  650. if (!(ndev->flags & IFF_UP) || !netif_carrier_ok(ndev)) {
  651. pr_info("%s interface not up.\n", ndev->name);
  652. err = -ENETDOWN;
  653. goto rel_rt;
  654. }
  655. if (ipv6_addr_is_multicast(&daddr6->sin6_addr)) {
  656. pr_info("multi-cast route %pI6 port %u, dev %s.\n",
  657. daddr6->sin6_addr.s6_addr,
  658. ntohs(daddr6->sin6_port), ndev->name);
  659. err = -ENETUNREACH;
  660. goto rel_rt;
  661. }
  662. cdev = cxgbi_device_find_by_netdev(ndev, &port);
  663. if (!cdev)
  664. cdev = cxgbi_device_find_by_mac(ndev, &port);
  665. if (!cdev) {
  666. pr_info("dst %pI6 %s, NOT cxgbi device.\n",
  667. daddr6->sin6_addr.s6_addr, ndev->name);
  668. err = -ENETUNREACH;
  669. goto rel_rt;
  670. }
  671. log_debug(1 << CXGBI_DBG_SOCK,
  672. "route to %pI6 :%u, ndev p#%d,%s, cdev 0x%p.\n",
  673. daddr6->sin6_addr.s6_addr, ntohs(daddr6->sin6_port), port,
  674. ndev->name, cdev);
  675. csk = cxgbi_sock_create(cdev);
  676. if (!csk) {
  677. err = -ENOMEM;
  678. goto rel_rt;
  679. }
  680. csk->cdev = cdev;
  681. csk->port_id = port;
  682. csk->mtu = mtu;
  683. csk->dst = dst;
  684. rt6_get_prefsrc(rt, &pref_saddr);
  685. if (ipv6_addr_any(&pref_saddr)) {
  686. struct inet6_dev *idev = ip6_dst_idev((struct dst_entry *)rt);
  687. err = ipv6_dev_get_saddr(&init_net, idev ? idev->dev : NULL,
  688. &daddr6->sin6_addr, 0, &pref_saddr);
  689. if (err) {
  690. pr_info("failed to get source address to reach %pI6\n",
  691. &daddr6->sin6_addr);
  692. goto rel_rt;
  693. }
  694. }
  695. csk->csk_family = AF_INET6;
  696. csk->daddr6.sin6_addr = daddr6->sin6_addr;
  697. csk->daddr6.sin6_port = daddr6->sin6_port;
  698. csk->daddr6.sin6_family = daddr6->sin6_family;
  699. csk->saddr6.sin6_family = daddr6->sin6_family;
  700. csk->saddr6.sin6_addr = pref_saddr;
  701. neigh_release(n);
  702. return csk;
  703. rel_rt:
  704. if (n)
  705. neigh_release(n);
  706. ip6_rt_put(rt);
  707. if (csk)
  708. cxgbi_sock_closed(csk);
  709. err_out:
  710. return ERR_PTR(err);
  711. }
  712. #endif /* IS_ENABLED(CONFIG_IPV6) */
  713. void cxgbi_sock_established(struct cxgbi_sock *csk, unsigned int snd_isn,
  714. unsigned int opt)
  715. {
  716. csk->write_seq = csk->snd_nxt = csk->snd_una = snd_isn;
  717. dst_confirm(csk->dst);
  718. smp_mb();
  719. cxgbi_sock_set_state(csk, CTP_ESTABLISHED);
  720. }
  721. EXPORT_SYMBOL_GPL(cxgbi_sock_established);
  722. static void cxgbi_inform_iscsi_conn_closing(struct cxgbi_sock *csk)
  723. {
  724. log_debug(1 << CXGBI_DBG_SOCK,
  725. "csk 0x%p, state %u, flags 0x%lx, conn 0x%p.\n",
  726. csk, csk->state, csk->flags, csk->user_data);
  727. if (csk->state != CTP_ESTABLISHED) {
  728. read_lock_bh(&csk->callback_lock);
  729. if (csk->user_data)
  730. iscsi_conn_failure(csk->user_data,
  731. ISCSI_ERR_TCP_CONN_CLOSE);
  732. read_unlock_bh(&csk->callback_lock);
  733. }
  734. }
  735. void cxgbi_sock_closed(struct cxgbi_sock *csk)
  736. {
  737. log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
  738. csk, (csk)->state, (csk)->flags, (csk)->tid);
  739. cxgbi_sock_set_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED);
  740. if (csk->state == CTP_ACTIVE_OPEN || csk->state == CTP_CLOSED)
  741. return;
  742. if (csk->saddr.sin_port)
  743. sock_put_port(csk);
  744. if (csk->dst)
  745. dst_release(csk->dst);
  746. csk->cdev->csk_release_offload_resources(csk);
  747. cxgbi_sock_set_state(csk, CTP_CLOSED);
  748. cxgbi_inform_iscsi_conn_closing(csk);
  749. cxgbi_sock_put(csk);
  750. }
  751. EXPORT_SYMBOL_GPL(cxgbi_sock_closed);
  752. static void need_active_close(struct cxgbi_sock *csk)
  753. {
  754. int data_lost;
  755. int close_req = 0;
  756. log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
  757. csk, (csk)->state, (csk)->flags, (csk)->tid);
  758. spin_lock_bh(&csk->lock);
  759. if (csk->dst)
  760. dst_confirm(csk->dst);
  761. data_lost = skb_queue_len(&csk->receive_queue);
  762. __skb_queue_purge(&csk->receive_queue);
  763. if (csk->state == CTP_ACTIVE_OPEN)
  764. cxgbi_sock_set_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED);
  765. else if (csk->state == CTP_ESTABLISHED) {
  766. close_req = 1;
  767. cxgbi_sock_set_state(csk, CTP_ACTIVE_CLOSE);
  768. } else if (csk->state == CTP_PASSIVE_CLOSE) {
  769. close_req = 1;
  770. cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_2);
  771. }
  772. if (close_req) {
  773. if (!cxgbi_sock_flag(csk, CTPF_LOGOUT_RSP_RCVD) ||
  774. data_lost)
  775. csk->cdev->csk_send_abort_req(csk);
  776. else
  777. csk->cdev->csk_send_close_req(csk);
  778. }
  779. spin_unlock_bh(&csk->lock);
  780. }
  781. void cxgbi_sock_fail_act_open(struct cxgbi_sock *csk, int errno)
  782. {
  783. pr_info("csk 0x%p,%u,%lx, %pI4:%u-%pI4:%u, err %d.\n",
  784. csk, csk->state, csk->flags,
  785. &csk->saddr.sin_addr.s_addr, csk->saddr.sin_port,
  786. &csk->daddr.sin_addr.s_addr, csk->daddr.sin_port,
  787. errno);
  788. cxgbi_sock_set_state(csk, CTP_CONNECTING);
  789. csk->err = errno;
  790. cxgbi_sock_closed(csk);
  791. }
  792. EXPORT_SYMBOL_GPL(cxgbi_sock_fail_act_open);
  793. void cxgbi_sock_act_open_req_arp_failure(void *handle, struct sk_buff *skb)
  794. {
  795. struct cxgbi_sock *csk = (struct cxgbi_sock *)skb->sk;
  796. struct module *owner = csk->cdev->owner;
  797. log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
  798. csk, (csk)->state, (csk)->flags, (csk)->tid);
  799. cxgbi_sock_get(csk);
  800. spin_lock_bh(&csk->lock);
  801. if (csk->state == CTP_ACTIVE_OPEN)
  802. cxgbi_sock_fail_act_open(csk, -EHOSTUNREACH);
  803. spin_unlock_bh(&csk->lock);
  804. cxgbi_sock_put(csk);
  805. __kfree_skb(skb);
  806. module_put(owner);
  807. }
  808. EXPORT_SYMBOL_GPL(cxgbi_sock_act_open_req_arp_failure);
  809. void cxgbi_sock_rcv_abort_rpl(struct cxgbi_sock *csk)
  810. {
  811. cxgbi_sock_get(csk);
  812. spin_lock_bh(&csk->lock);
  813. cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_RCVD);
  814. if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) {
  815. cxgbi_sock_clear_flag(csk, CTPF_ABORT_RPL_PENDING);
  816. if (cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD))
  817. pr_err("csk 0x%p,%u,0x%lx,%u,ABT_RPL_RSS.\n",
  818. csk, csk->state, csk->flags, csk->tid);
  819. cxgbi_sock_closed(csk);
  820. }
  821. spin_unlock_bh(&csk->lock);
  822. cxgbi_sock_put(csk);
  823. }
  824. EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_abort_rpl);
  825. void cxgbi_sock_rcv_peer_close(struct cxgbi_sock *csk)
  826. {
  827. log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
  828. csk, (csk)->state, (csk)->flags, (csk)->tid);
  829. cxgbi_sock_get(csk);
  830. spin_lock_bh(&csk->lock);
  831. if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING))
  832. goto done;
  833. switch (csk->state) {
  834. case CTP_ESTABLISHED:
  835. cxgbi_sock_set_state(csk, CTP_PASSIVE_CLOSE);
  836. break;
  837. case CTP_ACTIVE_CLOSE:
  838. cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_2);
  839. break;
  840. case CTP_CLOSE_WAIT_1:
  841. cxgbi_sock_closed(csk);
  842. break;
  843. case CTP_ABORTING:
  844. break;
  845. default:
  846. pr_err("csk 0x%p,%u,0x%lx,%u, bad state.\n",
  847. csk, csk->state, csk->flags, csk->tid);
  848. }
  849. cxgbi_inform_iscsi_conn_closing(csk);
  850. done:
  851. spin_unlock_bh(&csk->lock);
  852. cxgbi_sock_put(csk);
  853. }
  854. EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_peer_close);
  855. void cxgbi_sock_rcv_close_conn_rpl(struct cxgbi_sock *csk, u32 snd_nxt)
  856. {
  857. log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
  858. csk, (csk)->state, (csk)->flags, (csk)->tid);
  859. cxgbi_sock_get(csk);
  860. spin_lock_bh(&csk->lock);
  861. csk->snd_una = snd_nxt - 1;
  862. if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING))
  863. goto done;
  864. switch (csk->state) {
  865. case CTP_ACTIVE_CLOSE:
  866. cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_1);
  867. break;
  868. case CTP_CLOSE_WAIT_1:
  869. case CTP_CLOSE_WAIT_2:
  870. cxgbi_sock_closed(csk);
  871. break;
  872. case CTP_ABORTING:
  873. break;
  874. default:
  875. pr_err("csk 0x%p,%u,0x%lx,%u, bad state.\n",
  876. csk, csk->state, csk->flags, csk->tid);
  877. }
  878. done:
  879. spin_unlock_bh(&csk->lock);
  880. cxgbi_sock_put(csk);
  881. }
  882. EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_close_conn_rpl);
  883. void cxgbi_sock_rcv_wr_ack(struct cxgbi_sock *csk, unsigned int credits,
  884. unsigned int snd_una, int seq_chk)
  885. {
  886. log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
  887. "csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, snd_una %u,%d.\n",
  888. csk, csk->state, csk->flags, csk->tid, credits,
  889. csk->wr_cred, csk->wr_una_cred, snd_una, seq_chk);
  890. spin_lock_bh(&csk->lock);
  891. csk->wr_cred += credits;
  892. if (csk->wr_una_cred > csk->wr_max_cred - csk->wr_cred)
  893. csk->wr_una_cred = csk->wr_max_cred - csk->wr_cred;
  894. while (credits) {
  895. struct sk_buff *p = cxgbi_sock_peek_wr(csk);
  896. if (unlikely(!p)) {
  897. pr_err("csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, empty.\n",
  898. csk, csk->state, csk->flags, csk->tid, credits,
  899. csk->wr_cred, csk->wr_una_cred);
  900. break;
  901. }
  902. if (unlikely(credits < p->csum)) {
  903. pr_warn("csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, < %u.\n",
  904. csk, csk->state, csk->flags, csk->tid,
  905. credits, csk->wr_cred, csk->wr_una_cred,
  906. p->csum);
  907. p->csum -= credits;
  908. break;
  909. } else {
  910. cxgbi_sock_dequeue_wr(csk);
  911. credits -= p->csum;
  912. kfree_skb(p);
  913. }
  914. }
  915. cxgbi_sock_check_wr_invariants(csk);
  916. if (seq_chk) {
  917. if (unlikely(before(snd_una, csk->snd_una))) {
  918. pr_warn("csk 0x%p,%u,0x%lx,%u, snd_una %u/%u.",
  919. csk, csk->state, csk->flags, csk->tid, snd_una,
  920. csk->snd_una);
  921. goto done;
  922. }
  923. if (csk->snd_una != snd_una) {
  924. csk->snd_una = snd_una;
  925. dst_confirm(csk->dst);
  926. }
  927. }
  928. if (skb_queue_len(&csk->write_queue)) {
  929. if (csk->cdev->csk_push_tx_frames(csk, 0))
  930. cxgbi_conn_tx_open(csk);
  931. } else
  932. cxgbi_conn_tx_open(csk);
  933. done:
  934. spin_unlock_bh(&csk->lock);
  935. }
  936. EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_wr_ack);
  937. static unsigned int cxgbi_sock_find_best_mtu(struct cxgbi_sock *csk,
  938. unsigned short mtu)
  939. {
  940. int i = 0;
  941. while (i < csk->cdev->nmtus - 1 && csk->cdev->mtus[i + 1] <= mtu)
  942. ++i;
  943. return i;
  944. }
  945. unsigned int cxgbi_sock_select_mss(struct cxgbi_sock *csk, unsigned int pmtu)
  946. {
  947. unsigned int idx;
  948. struct dst_entry *dst = csk->dst;
  949. csk->advmss = dst_metric_advmss(dst);
  950. if (csk->advmss > pmtu - 40)
  951. csk->advmss = pmtu - 40;
  952. if (csk->advmss < csk->cdev->mtus[0] - 40)
  953. csk->advmss = csk->cdev->mtus[0] - 40;
  954. idx = cxgbi_sock_find_best_mtu(csk, csk->advmss + 40);
  955. return idx;
  956. }
  957. EXPORT_SYMBOL_GPL(cxgbi_sock_select_mss);
  958. void cxgbi_sock_skb_entail(struct cxgbi_sock *csk, struct sk_buff *skb)
  959. {
  960. cxgbi_skcb_tcp_seq(skb) = csk->write_seq;
  961. __skb_queue_tail(&csk->write_queue, skb);
  962. }
  963. EXPORT_SYMBOL_GPL(cxgbi_sock_skb_entail);
  964. void cxgbi_sock_purge_wr_queue(struct cxgbi_sock *csk)
  965. {
  966. struct sk_buff *skb;
  967. while ((skb = cxgbi_sock_dequeue_wr(csk)) != NULL)
  968. kfree_skb(skb);
  969. }
  970. EXPORT_SYMBOL_GPL(cxgbi_sock_purge_wr_queue);
  971. void cxgbi_sock_check_wr_invariants(const struct cxgbi_sock *csk)
  972. {
  973. int pending = cxgbi_sock_count_pending_wrs(csk);
  974. if (unlikely(csk->wr_cred + pending != csk->wr_max_cred))
  975. pr_err("csk 0x%p, tid %u, credit %u + %u != %u.\n",
  976. csk, csk->tid, csk->wr_cred, pending, csk->wr_max_cred);
  977. }
  978. EXPORT_SYMBOL_GPL(cxgbi_sock_check_wr_invariants);
  979. static inline void
  980. scmd_get_params(struct scsi_cmnd *sc, struct scatterlist **sgl,
  981. unsigned int *sgcnt, unsigned int *dlen,
  982. unsigned int prot)
  983. {
  984. struct scsi_data_buffer *sdb = prot ? scsi_prot(sc) : &sc->sdb;
  985. *sgl = sdb->table.sgl;
  986. *sgcnt = sdb->table.nents;
  987. *dlen = sdb->length;
  988. /* Caution: for protection sdb, sdb->length is invalid */
  989. }
  990. void cxgbi_ddp_set_one_ppod(struct cxgbi_pagepod *ppod,
  991. struct cxgbi_task_tag_info *ttinfo,
  992. struct scatterlist **sg_pp, unsigned int *sg_off)
  993. {
  994. struct scatterlist *sg = sg_pp ? *sg_pp : NULL;
  995. unsigned int offset = sg_off ? *sg_off : 0;
  996. dma_addr_t addr = 0UL;
  997. unsigned int len = 0;
  998. int i;
  999. memcpy(ppod, &ttinfo->hdr, sizeof(struct cxgbi_pagepod_hdr));
  1000. if (sg) {
  1001. addr = sg_dma_address(sg);
  1002. len = sg_dma_len(sg);
  1003. }
  1004. for (i = 0; i < PPOD_PAGES_MAX; i++) {
  1005. if (sg) {
  1006. ppod->addr[i] = cpu_to_be64(addr + offset);
  1007. offset += PAGE_SIZE;
  1008. if (offset == (len + sg->offset)) {
  1009. offset = 0;
  1010. sg = sg_next(sg);
  1011. if (sg) {
  1012. addr = sg_dma_address(sg);
  1013. len = sg_dma_len(sg);
  1014. }
  1015. }
  1016. } else {
  1017. ppod->addr[i] = 0ULL;
  1018. }
  1019. }
  1020. /*
  1021. * the fifth address needs to be repeated in the next ppod, so do
  1022. * not move sg
  1023. */
  1024. if (sg_pp) {
  1025. *sg_pp = sg;
  1026. *sg_off = offset;
  1027. }
  1028. if (offset == len) {
  1029. offset = 0;
  1030. sg = sg_next(sg);
  1031. if (sg) {
  1032. addr = sg_dma_address(sg);
  1033. len = sg_dma_len(sg);
  1034. }
  1035. }
  1036. ppod->addr[i] = sg ? cpu_to_be64(addr + offset) : 0ULL;
  1037. }
  1038. EXPORT_SYMBOL_GPL(cxgbi_ddp_set_one_ppod);
  1039. /*
  1040. * APIs interacting with open-iscsi libraries
  1041. */
  1042. int cxgbi_ddp_ppm_setup(void **ppm_pp, struct cxgbi_device *cdev,
  1043. struct cxgbi_tag_format *tformat,
  1044. unsigned int iscsi_size, unsigned int llimit,
  1045. unsigned int start, unsigned int rsvd_factor,
  1046. unsigned int edram_start, unsigned int edram_size)
  1047. {
  1048. int err = cxgbi_ppm_init(ppm_pp, cdev->ports[0], cdev->pdev,
  1049. cdev->lldev, tformat, iscsi_size, llimit, start,
  1050. rsvd_factor, edram_start, edram_size);
  1051. if (err >= 0) {
  1052. struct cxgbi_ppm *ppm = (struct cxgbi_ppm *)(*ppm_pp);
  1053. if (ppm->ppmax < 1024 ||
  1054. ppm->tformat.pgsz_idx_dflt >= DDP_PGIDX_MAX)
  1055. cdev->flags |= CXGBI_FLAG_DDP_OFF;
  1056. err = 0;
  1057. } else {
  1058. cdev->flags |= CXGBI_FLAG_DDP_OFF;
  1059. }
  1060. return err;
  1061. }
  1062. EXPORT_SYMBOL_GPL(cxgbi_ddp_ppm_setup);
  1063. static int cxgbi_ddp_sgl_check(struct scatterlist *sgl, int nents)
  1064. {
  1065. int i;
  1066. int last_sgidx = nents - 1;
  1067. struct scatterlist *sg = sgl;
  1068. for (i = 0; i < nents; i++, sg = sg_next(sg)) {
  1069. unsigned int len = sg->length + sg->offset;
  1070. if ((sg->offset & 0x3) || (i && sg->offset) ||
  1071. ((i != last_sgidx) && len != PAGE_SIZE)) {
  1072. log_debug(1 << CXGBI_DBG_DDP,
  1073. "sg %u/%u, %u,%u, not aligned.\n",
  1074. i, nents, sg->offset, sg->length);
  1075. goto err_out;
  1076. }
  1077. }
  1078. return 0;
  1079. err_out:
  1080. return -EINVAL;
  1081. }
  1082. static int cxgbi_ddp_reserve(struct cxgbi_conn *cconn,
  1083. struct cxgbi_task_data *tdata, u32 sw_tag,
  1084. unsigned int xferlen)
  1085. {
  1086. struct cxgbi_sock *csk = cconn->cep->csk;
  1087. struct cxgbi_device *cdev = csk->cdev;
  1088. struct cxgbi_ppm *ppm = cdev->cdev2ppm(cdev);
  1089. struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo;
  1090. struct scatterlist *sgl = ttinfo->sgl;
  1091. unsigned int sgcnt = ttinfo->nents;
  1092. unsigned int sg_offset = sgl->offset;
  1093. int err;
  1094. if (cdev->flags & CXGBI_FLAG_DDP_OFF) {
  1095. log_debug(1 << CXGBI_DBG_DDP,
  1096. "cdev 0x%p DDP off.\n", cdev);
  1097. return -EINVAL;
  1098. }
  1099. if (!ppm || xferlen < DDP_THRESHOLD || !sgcnt ||
  1100. ppm->tformat.pgsz_idx_dflt >= DDP_PGIDX_MAX) {
  1101. log_debug(1 << CXGBI_DBG_DDP,
  1102. "ppm 0x%p, pgidx %u, xfer %u, sgcnt %u, NO ddp.\n",
  1103. ppm, ppm ? ppm->tformat.pgsz_idx_dflt : DDP_PGIDX_MAX,
  1104. xferlen, ttinfo->nents);
  1105. return -EINVAL;
  1106. }
  1107. /* make sure the buffer is suitable for ddp */
  1108. if (cxgbi_ddp_sgl_check(sgl, sgcnt) < 0)
  1109. return -EINVAL;
  1110. ttinfo->nr_pages = (xferlen + sgl->offset + (1 << PAGE_SHIFT) - 1) >>
  1111. PAGE_SHIFT;
  1112. /*
  1113. * the ddp tag will be used for the itt in the outgoing pdu,
  1114. * the itt genrated by libiscsi is saved in the ppm and can be
  1115. * retrieved via the ddp tag
  1116. */
  1117. err = cxgbi_ppm_ppods_reserve(ppm, ttinfo->nr_pages, 0, &ttinfo->idx,
  1118. &ttinfo->tag, (unsigned long)sw_tag);
  1119. if (err < 0) {
  1120. cconn->ddp_full++;
  1121. return err;
  1122. }
  1123. ttinfo->npods = err;
  1124. /* setup dma from scsi command sgl */
  1125. sgl->offset = 0;
  1126. err = dma_map_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE);
  1127. sgl->offset = sg_offset;
  1128. if (err == 0) {
  1129. pr_info("%s: 0x%x, xfer %u, sgl %u dma mapping err.\n",
  1130. __func__, sw_tag, xferlen, sgcnt);
  1131. goto rel_ppods;
  1132. }
  1133. if (err != ttinfo->nr_pages) {
  1134. log_debug(1 << CXGBI_DBG_DDP,
  1135. "%s: sw tag 0x%x, xfer %u, sgl %u, dma count %d.\n",
  1136. __func__, sw_tag, xferlen, sgcnt, err);
  1137. }
  1138. ttinfo->flags |= CXGBI_PPOD_INFO_FLAG_MAPPED;
  1139. ttinfo->cid = csk->port_id;
  1140. cxgbi_ppm_make_ppod_hdr(ppm, ttinfo->tag, csk->tid, sgl->offset,
  1141. xferlen, &ttinfo->hdr);
  1142. if (cdev->flags & CXGBI_FLAG_USE_PPOD_OFLDQ) {
  1143. /* write ppod from xmit_pdu (of iscsi_scsi_command pdu) */
  1144. ttinfo->flags |= CXGBI_PPOD_INFO_FLAG_VALID;
  1145. } else {
  1146. /* write ppod from control queue now */
  1147. err = cdev->csk_ddp_set_map(ppm, csk, ttinfo);
  1148. if (err < 0)
  1149. goto rel_ppods;
  1150. }
  1151. return 0;
  1152. rel_ppods:
  1153. cxgbi_ppm_ppod_release(ppm, ttinfo->idx);
  1154. if (ttinfo->flags & CXGBI_PPOD_INFO_FLAG_MAPPED) {
  1155. ttinfo->flags &= ~CXGBI_PPOD_INFO_FLAG_MAPPED;
  1156. dma_unmap_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE);
  1157. }
  1158. return -EINVAL;
  1159. }
  1160. static void task_release_itt(struct iscsi_task *task, itt_t hdr_itt)
  1161. {
  1162. struct scsi_cmnd *sc = task->sc;
  1163. struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data;
  1164. struct cxgbi_conn *cconn = tcp_conn->dd_data;
  1165. struct cxgbi_device *cdev = cconn->chba->cdev;
  1166. struct cxgbi_ppm *ppm = cdev->cdev2ppm(cdev);
  1167. u32 tag = ntohl((__force u32)hdr_itt);
  1168. log_debug(1 << CXGBI_DBG_DDP,
  1169. "cdev 0x%p, task 0x%p, release tag 0x%x.\n",
  1170. cdev, task, tag);
  1171. if (sc && sc->sc_data_direction == DMA_FROM_DEVICE &&
  1172. cxgbi_ppm_is_ddp_tag(ppm, tag)) {
  1173. struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
  1174. struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo;
  1175. if (!(cdev->flags & CXGBI_FLAG_USE_PPOD_OFLDQ))
  1176. cdev->csk_ddp_clear_map(cdev, ppm, ttinfo);
  1177. cxgbi_ppm_ppod_release(ppm, ttinfo->idx);
  1178. dma_unmap_sg(&ppm->pdev->dev, ttinfo->sgl, ttinfo->nents,
  1179. DMA_FROM_DEVICE);
  1180. }
  1181. }
  1182. static inline u32 cxgbi_build_sw_tag(u32 idx, u32 age)
  1183. {
  1184. /* assume idx and age both are < 0x7FFF (32767) */
  1185. return (idx << 16) | age;
  1186. }
  1187. static int task_reserve_itt(struct iscsi_task *task, itt_t *hdr_itt)
  1188. {
  1189. struct scsi_cmnd *sc = task->sc;
  1190. struct iscsi_conn *conn = task->conn;
  1191. struct iscsi_session *sess = conn->session;
  1192. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  1193. struct cxgbi_conn *cconn = tcp_conn->dd_data;
  1194. struct cxgbi_device *cdev = cconn->chba->cdev;
  1195. struct cxgbi_ppm *ppm = cdev->cdev2ppm(cdev);
  1196. u32 sw_tag = cxgbi_build_sw_tag(task->itt, sess->age);
  1197. u32 tag = 0;
  1198. int err = -EINVAL;
  1199. if (sc && sc->sc_data_direction == DMA_FROM_DEVICE) {
  1200. struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
  1201. struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo;
  1202. scmd_get_params(sc, &ttinfo->sgl, &ttinfo->nents,
  1203. &tdata->dlen, 0);
  1204. err = cxgbi_ddp_reserve(cconn, tdata, sw_tag, tdata->dlen);
  1205. if (!err)
  1206. tag = ttinfo->tag;
  1207. else
  1208. log_debug(1 << CXGBI_DBG_DDP,
  1209. "csk 0x%p, R task 0x%p, %u,%u, no ddp.\n",
  1210. cconn->cep->csk, task, tdata->dlen,
  1211. ttinfo->nents);
  1212. }
  1213. if (err < 0) {
  1214. err = cxgbi_ppm_make_non_ddp_tag(ppm, sw_tag, &tag);
  1215. if (err < 0)
  1216. return err;
  1217. }
  1218. /* the itt need to sent in big-endian order */
  1219. *hdr_itt = (__force itt_t)htonl(tag);
  1220. log_debug(1 << CXGBI_DBG_DDP,
  1221. "cdev 0x%p, task 0x%p, 0x%x(0x%x,0x%x)->0x%x/0x%x.\n",
  1222. cdev, task, sw_tag, task->itt, sess->age, tag, *hdr_itt);
  1223. return 0;
  1224. }
  1225. void cxgbi_parse_pdu_itt(struct iscsi_conn *conn, itt_t itt, int *idx, int *age)
  1226. {
  1227. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  1228. struct cxgbi_conn *cconn = tcp_conn->dd_data;
  1229. struct cxgbi_device *cdev = cconn->chba->cdev;
  1230. struct cxgbi_ppm *ppm = cdev->cdev2ppm(cdev);
  1231. u32 tag = ntohl((__force u32)itt);
  1232. u32 sw_bits;
  1233. if (ppm) {
  1234. if (cxgbi_ppm_is_ddp_tag(ppm, tag))
  1235. sw_bits = cxgbi_ppm_get_tag_caller_data(ppm, tag);
  1236. else
  1237. sw_bits = cxgbi_ppm_decode_non_ddp_tag(ppm, tag);
  1238. } else {
  1239. sw_bits = tag;
  1240. }
  1241. cxgbi_decode_sw_tag(sw_bits, idx, age);
  1242. log_debug(1 << CXGBI_DBG_DDP,
  1243. "cdev 0x%p, tag 0x%x/0x%x, -> 0x%x(0x%x,0x%x).\n",
  1244. cdev, tag, itt, sw_bits, idx ? *idx : 0xFFFFF,
  1245. age ? *age : 0xFF);
  1246. }
  1247. EXPORT_SYMBOL_GPL(cxgbi_parse_pdu_itt);
  1248. void cxgbi_conn_tx_open(struct cxgbi_sock *csk)
  1249. {
  1250. struct iscsi_conn *conn = csk->user_data;
  1251. if (conn) {
  1252. log_debug(1 << CXGBI_DBG_SOCK,
  1253. "csk 0x%p, cid %d.\n", csk, conn->id);
  1254. iscsi_conn_queue_work(conn);
  1255. }
  1256. }
  1257. EXPORT_SYMBOL_GPL(cxgbi_conn_tx_open);
  1258. /*
  1259. * pdu receive, interact with libiscsi_tcp
  1260. */
  1261. static inline int read_pdu_skb(struct iscsi_conn *conn,
  1262. struct sk_buff *skb,
  1263. unsigned int offset,
  1264. int offloaded)
  1265. {
  1266. int status = 0;
  1267. int bytes_read;
  1268. bytes_read = iscsi_tcp_recv_skb(conn, skb, offset, offloaded, &status);
  1269. switch (status) {
  1270. case ISCSI_TCP_CONN_ERR:
  1271. pr_info("skb 0x%p, off %u, %d, TCP_ERR.\n",
  1272. skb, offset, offloaded);
  1273. return -EIO;
  1274. case ISCSI_TCP_SUSPENDED:
  1275. log_debug(1 << CXGBI_DBG_PDU_RX,
  1276. "skb 0x%p, off %u, %d, TCP_SUSPEND, rc %d.\n",
  1277. skb, offset, offloaded, bytes_read);
  1278. /* no transfer - just have caller flush queue */
  1279. return bytes_read;
  1280. case ISCSI_TCP_SKB_DONE:
  1281. pr_info("skb 0x%p, off %u, %d, TCP_SKB_DONE.\n",
  1282. skb, offset, offloaded);
  1283. /*
  1284. * pdus should always fit in the skb and we should get
  1285. * segment done notifcation.
  1286. */
  1287. iscsi_conn_printk(KERN_ERR, conn, "Invalid pdu or skb.");
  1288. return -EFAULT;
  1289. case ISCSI_TCP_SEGMENT_DONE:
  1290. log_debug(1 << CXGBI_DBG_PDU_RX,
  1291. "skb 0x%p, off %u, %d, TCP_SEG_DONE, rc %d.\n",
  1292. skb, offset, offloaded, bytes_read);
  1293. return bytes_read;
  1294. default:
  1295. pr_info("skb 0x%p, off %u, %d, invalid status %d.\n",
  1296. skb, offset, offloaded, status);
  1297. return -EINVAL;
  1298. }
  1299. }
  1300. static int
  1301. skb_read_pdu_bhs(struct cxgbi_sock *csk, struct iscsi_conn *conn,
  1302. struct sk_buff *skb)
  1303. {
  1304. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  1305. int err;
  1306. log_debug(1 << CXGBI_DBG_PDU_RX,
  1307. "conn 0x%p, skb 0x%p, len %u, flag 0x%lx.\n",
  1308. conn, skb, skb->len, cxgbi_skcb_flags(skb));
  1309. if (!iscsi_tcp_recv_segment_is_hdr(tcp_conn)) {
  1310. pr_info("conn 0x%p, skb 0x%p, not hdr.\n", conn, skb);
  1311. iscsi_conn_failure(conn, ISCSI_ERR_PROTO);
  1312. return -EIO;
  1313. }
  1314. if (conn->hdrdgst_en &&
  1315. cxgbi_skcb_test_flag(skb, SKCBF_RX_HCRC_ERR)) {
  1316. pr_info("conn 0x%p, skb 0x%p, hcrc.\n", conn, skb);
  1317. iscsi_conn_failure(conn, ISCSI_ERR_HDR_DGST);
  1318. return -EIO;
  1319. }
  1320. if (cxgbi_skcb_test_flag(skb, SKCBF_RX_ISCSI_COMPL) &&
  1321. cxgbi_skcb_test_flag(skb, SKCBF_RX_DATA_DDPD)) {
  1322. /* If completion flag is set and data is directly
  1323. * placed in to the host memory then update
  1324. * task->exp_datasn to the datasn in completion
  1325. * iSCSI hdr as T6 adapter generates completion only
  1326. * for the last pdu of a sequence.
  1327. */
  1328. itt_t itt = ((struct iscsi_data *)skb->data)->itt;
  1329. struct iscsi_task *task = iscsi_itt_to_ctask(conn, itt);
  1330. u32 data_sn = be32_to_cpu(((struct iscsi_data *)
  1331. skb->data)->datasn);
  1332. if (task && task->sc) {
  1333. struct iscsi_tcp_task *tcp_task = task->dd_data;
  1334. tcp_task->exp_datasn = data_sn;
  1335. }
  1336. }
  1337. err = read_pdu_skb(conn, skb, 0, 0);
  1338. if (likely(err >= 0)) {
  1339. struct iscsi_hdr *hdr = (struct iscsi_hdr *)skb->data;
  1340. u8 opcode = hdr->opcode & ISCSI_OPCODE_MASK;
  1341. if (unlikely(opcode == ISCSI_OP_LOGOUT_RSP))
  1342. cxgbi_sock_set_flag(csk, CTPF_LOGOUT_RSP_RCVD);
  1343. }
  1344. return err;
  1345. }
  1346. static int skb_read_pdu_data(struct iscsi_conn *conn, struct sk_buff *lskb,
  1347. struct sk_buff *skb, unsigned int offset)
  1348. {
  1349. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  1350. bool offloaded = 0;
  1351. int opcode = tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK;
  1352. log_debug(1 << CXGBI_DBG_PDU_RX,
  1353. "conn 0x%p, skb 0x%p, len %u, flag 0x%lx.\n",
  1354. conn, skb, skb->len, cxgbi_skcb_flags(skb));
  1355. if (conn->datadgst_en &&
  1356. cxgbi_skcb_test_flag(lskb, SKCBF_RX_DCRC_ERR)) {
  1357. pr_info("conn 0x%p, skb 0x%p, dcrc 0x%lx.\n",
  1358. conn, lskb, cxgbi_skcb_flags(lskb));
  1359. iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
  1360. return -EIO;
  1361. }
  1362. if (iscsi_tcp_recv_segment_is_hdr(tcp_conn))
  1363. return 0;
  1364. /* coalesced, add header digest length */
  1365. if (lskb == skb && conn->hdrdgst_en)
  1366. offset += ISCSI_DIGEST_SIZE;
  1367. if (cxgbi_skcb_test_flag(lskb, SKCBF_RX_DATA_DDPD))
  1368. offloaded = 1;
  1369. if (opcode == ISCSI_OP_SCSI_DATA_IN)
  1370. log_debug(1 << CXGBI_DBG_PDU_RX,
  1371. "skb 0x%p, op 0x%x, itt 0x%x, %u %s ddp'ed.\n",
  1372. skb, opcode, ntohl(tcp_conn->in.hdr->itt),
  1373. tcp_conn->in.datalen, offloaded ? "is" : "not");
  1374. return read_pdu_skb(conn, skb, offset, offloaded);
  1375. }
  1376. static void csk_return_rx_credits(struct cxgbi_sock *csk, int copied)
  1377. {
  1378. struct cxgbi_device *cdev = csk->cdev;
  1379. int must_send;
  1380. u32 credits;
  1381. log_debug(1 << CXGBI_DBG_PDU_RX,
  1382. "csk 0x%p,%u,0x%lx,%u, seq %u, wup %u, thre %u, %u.\n",
  1383. csk, csk->state, csk->flags, csk->tid, csk->copied_seq,
  1384. csk->rcv_wup, cdev->rx_credit_thres,
  1385. csk->rcv_win);
  1386. if (!cdev->rx_credit_thres)
  1387. return;
  1388. if (csk->state != CTP_ESTABLISHED)
  1389. return;
  1390. credits = csk->copied_seq - csk->rcv_wup;
  1391. if (unlikely(!credits))
  1392. return;
  1393. must_send = credits + 16384 >= csk->rcv_win;
  1394. if (must_send || credits >= cdev->rx_credit_thres)
  1395. csk->rcv_wup += cdev->csk_send_rx_credits(csk, credits);
  1396. }
  1397. void cxgbi_conn_pdu_ready(struct cxgbi_sock *csk)
  1398. {
  1399. struct cxgbi_device *cdev = csk->cdev;
  1400. struct iscsi_conn *conn = csk->user_data;
  1401. struct sk_buff *skb;
  1402. unsigned int read = 0;
  1403. int err = 0;
  1404. log_debug(1 << CXGBI_DBG_PDU_RX,
  1405. "csk 0x%p, conn 0x%p.\n", csk, conn);
  1406. if (unlikely(!conn || conn->suspend_rx)) {
  1407. log_debug(1 << CXGBI_DBG_PDU_RX,
  1408. "csk 0x%p, conn 0x%p, id %d, suspend_rx %lu!\n",
  1409. csk, conn, conn ? conn->id : 0xFF,
  1410. conn ? conn->suspend_rx : 0xFF);
  1411. return;
  1412. }
  1413. while (!err) {
  1414. skb = skb_peek(&csk->receive_queue);
  1415. if (!skb ||
  1416. !(cxgbi_skcb_test_flag(skb, SKCBF_RX_STATUS))) {
  1417. if (skb)
  1418. log_debug(1 << CXGBI_DBG_PDU_RX,
  1419. "skb 0x%p, NOT ready 0x%lx.\n",
  1420. skb, cxgbi_skcb_flags(skb));
  1421. break;
  1422. }
  1423. __skb_unlink(skb, &csk->receive_queue);
  1424. read += cxgbi_skcb_rx_pdulen(skb);
  1425. log_debug(1 << CXGBI_DBG_PDU_RX,
  1426. "csk 0x%p, skb 0x%p,%u,f 0x%lx, pdu len %u.\n",
  1427. csk, skb, skb->len, cxgbi_skcb_flags(skb),
  1428. cxgbi_skcb_rx_pdulen(skb));
  1429. if (cxgbi_skcb_test_flag(skb, SKCBF_RX_COALESCED)) {
  1430. err = skb_read_pdu_bhs(csk, conn, skb);
  1431. if (err < 0) {
  1432. pr_err("coalesced bhs, csk 0x%p, skb 0x%p,%u, "
  1433. "f 0x%lx, plen %u.\n",
  1434. csk, skb, skb->len,
  1435. cxgbi_skcb_flags(skb),
  1436. cxgbi_skcb_rx_pdulen(skb));
  1437. goto skb_done;
  1438. }
  1439. err = skb_read_pdu_data(conn, skb, skb,
  1440. err + cdev->skb_rx_extra);
  1441. if (err < 0)
  1442. pr_err("coalesced data, csk 0x%p, skb 0x%p,%u, "
  1443. "f 0x%lx, plen %u.\n",
  1444. csk, skb, skb->len,
  1445. cxgbi_skcb_flags(skb),
  1446. cxgbi_skcb_rx_pdulen(skb));
  1447. } else {
  1448. err = skb_read_pdu_bhs(csk, conn, skb);
  1449. if (err < 0) {
  1450. pr_err("bhs, csk 0x%p, skb 0x%p,%u, "
  1451. "f 0x%lx, plen %u.\n",
  1452. csk, skb, skb->len,
  1453. cxgbi_skcb_flags(skb),
  1454. cxgbi_skcb_rx_pdulen(skb));
  1455. goto skb_done;
  1456. }
  1457. if (cxgbi_skcb_test_flag(skb, SKCBF_RX_DATA)) {
  1458. struct sk_buff *dskb;
  1459. dskb = skb_peek(&csk->receive_queue);
  1460. if (!dskb) {
  1461. pr_err("csk 0x%p, skb 0x%p,%u, f 0x%lx,"
  1462. " plen %u, NO data.\n",
  1463. csk, skb, skb->len,
  1464. cxgbi_skcb_flags(skb),
  1465. cxgbi_skcb_rx_pdulen(skb));
  1466. err = -EIO;
  1467. goto skb_done;
  1468. }
  1469. __skb_unlink(dskb, &csk->receive_queue);
  1470. err = skb_read_pdu_data(conn, skb, dskb, 0);
  1471. if (err < 0)
  1472. pr_err("data, csk 0x%p, skb 0x%p,%u, "
  1473. "f 0x%lx, plen %u, dskb 0x%p,"
  1474. "%u.\n",
  1475. csk, skb, skb->len,
  1476. cxgbi_skcb_flags(skb),
  1477. cxgbi_skcb_rx_pdulen(skb),
  1478. dskb, dskb->len);
  1479. __kfree_skb(dskb);
  1480. } else
  1481. err = skb_read_pdu_data(conn, skb, skb, 0);
  1482. }
  1483. skb_done:
  1484. __kfree_skb(skb);
  1485. if (err < 0)
  1486. break;
  1487. }
  1488. log_debug(1 << CXGBI_DBG_PDU_RX, "csk 0x%p, read %u.\n", csk, read);
  1489. if (read) {
  1490. csk->copied_seq += read;
  1491. csk_return_rx_credits(csk, read);
  1492. conn->rxdata_octets += read;
  1493. }
  1494. if (err < 0) {
  1495. pr_info("csk 0x%p, 0x%p, rx failed %d, read %u.\n",
  1496. csk, conn, err, read);
  1497. iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
  1498. }
  1499. }
  1500. EXPORT_SYMBOL_GPL(cxgbi_conn_pdu_ready);
  1501. static int sgl_seek_offset(struct scatterlist *sgl, unsigned int sgcnt,
  1502. unsigned int offset, unsigned int *off,
  1503. struct scatterlist **sgp)
  1504. {
  1505. int i;
  1506. struct scatterlist *sg;
  1507. for_each_sg(sgl, sg, sgcnt, i) {
  1508. if (offset < sg->length) {
  1509. *off = offset;
  1510. *sgp = sg;
  1511. return 0;
  1512. }
  1513. offset -= sg->length;
  1514. }
  1515. return -EFAULT;
  1516. }
  1517. static int
  1518. sgl_read_to_frags(struct scatterlist *sg, unsigned int sgoffset,
  1519. unsigned int dlen, struct page_frag *frags,
  1520. int frag_max, u32 *dlimit)
  1521. {
  1522. unsigned int datalen = dlen;
  1523. unsigned int sglen = sg->length - sgoffset;
  1524. struct page *page = sg_page(sg);
  1525. int i;
  1526. i = 0;
  1527. do {
  1528. unsigned int copy;
  1529. if (!sglen) {
  1530. sg = sg_next(sg);
  1531. if (!sg) {
  1532. pr_warn("sg %d NULL, len %u/%u.\n",
  1533. i, datalen, dlen);
  1534. return -EINVAL;
  1535. }
  1536. sgoffset = 0;
  1537. sglen = sg->length;
  1538. page = sg_page(sg);
  1539. }
  1540. copy = min(datalen, sglen);
  1541. if (i && page == frags[i - 1].page &&
  1542. sgoffset + sg->offset ==
  1543. frags[i - 1].offset + frags[i - 1].size) {
  1544. frags[i - 1].size += copy;
  1545. } else {
  1546. if (i >= frag_max) {
  1547. pr_warn("too many pages %u, dlen %u.\n",
  1548. frag_max, dlen);
  1549. *dlimit = dlen - datalen;
  1550. return -EINVAL;
  1551. }
  1552. frags[i].page = page;
  1553. frags[i].offset = sg->offset + sgoffset;
  1554. frags[i].size = copy;
  1555. i++;
  1556. }
  1557. datalen -= copy;
  1558. sgoffset += copy;
  1559. sglen -= copy;
  1560. } while (datalen);
  1561. return i;
  1562. }
  1563. static void cxgbi_task_data_sgl_check(struct iscsi_task *task)
  1564. {
  1565. struct scsi_cmnd *sc = task->sc;
  1566. struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
  1567. struct scatterlist *sg, *sgl = NULL;
  1568. u32 sgcnt = 0;
  1569. int i;
  1570. tdata->flags = CXGBI_TASK_SGL_CHECKED;
  1571. if (!sc)
  1572. return;
  1573. scmd_get_params(sc, &sgl, &sgcnt, &tdata->dlen, 0);
  1574. if (!sgl || !sgcnt) {
  1575. tdata->flags |= CXGBI_TASK_SGL_COPY;
  1576. return;
  1577. }
  1578. for_each_sg(sgl, sg, sgcnt, i) {
  1579. if (page_count(sg_page(sg)) < 1) {
  1580. tdata->flags |= CXGBI_TASK_SGL_COPY;
  1581. return;
  1582. }
  1583. }
  1584. }
  1585. static int
  1586. cxgbi_task_data_sgl_read(struct iscsi_task *task, u32 offset, u32 count,
  1587. u32 *dlimit)
  1588. {
  1589. struct scsi_cmnd *sc = task->sc;
  1590. struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
  1591. struct scatterlist *sgl = NULL;
  1592. struct scatterlist *sg;
  1593. u32 dlen = 0;
  1594. u32 sgcnt;
  1595. int err;
  1596. if (!sc)
  1597. return 0;
  1598. scmd_get_params(sc, &sgl, &sgcnt, &dlen, 0);
  1599. if (!sgl || !sgcnt)
  1600. return 0;
  1601. err = sgl_seek_offset(sgl, sgcnt, offset, &tdata->sgoffset, &sg);
  1602. if (err < 0) {
  1603. pr_warn("tpdu max, sgl %u, bad offset %u/%u.\n",
  1604. sgcnt, offset, tdata->dlen);
  1605. return err;
  1606. }
  1607. err = sgl_read_to_frags(sg, tdata->sgoffset, count,
  1608. tdata->frags, MAX_SKB_FRAGS, dlimit);
  1609. if (err < 0) {
  1610. log_debug(1 << CXGBI_DBG_ISCSI,
  1611. "sgl max limit, sgl %u, offset %u, %u/%u, dlimit %u.\n",
  1612. sgcnt, offset, count, tdata->dlen, *dlimit);
  1613. return err;
  1614. }
  1615. tdata->offset = offset;
  1616. tdata->count = count;
  1617. tdata->nr_frags = err;
  1618. tdata->total_count = count;
  1619. tdata->total_offset = offset;
  1620. log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
  1621. "%s: offset %u, count %u,\n"
  1622. "err %u, total_count %u, total_offset %u\n",
  1623. __func__, offset, count, err, tdata->total_count, tdata->total_offset);
  1624. return 0;
  1625. }
  1626. int cxgbi_conn_alloc_pdu(struct iscsi_task *task, u8 op)
  1627. {
  1628. struct iscsi_conn *conn = task->conn;
  1629. struct iscsi_session *session = task->conn->session;
  1630. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  1631. struct cxgbi_conn *cconn = tcp_conn->dd_data;
  1632. struct cxgbi_device *cdev = cconn->chba->cdev;
  1633. struct cxgbi_sock *csk = cconn->cep ? cconn->cep->csk : NULL;
  1634. struct iscsi_tcp_task *tcp_task = task->dd_data;
  1635. struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
  1636. struct scsi_cmnd *sc = task->sc;
  1637. u32 headroom = SKB_TX_ISCSI_PDU_HEADER_MAX;
  1638. u32 max_txdata_len = conn->max_xmit_dlength;
  1639. u32 iso_tx_rsvd = 0, local_iso_info = 0;
  1640. u32 last_tdata_offset, last_tdata_count;
  1641. int err = 0;
  1642. if (!tcp_task) {
  1643. pr_err("task 0x%p, tcp_task 0x%p, tdata 0x%p.\n",
  1644. task, tcp_task, tdata);
  1645. return -ENOMEM;
  1646. }
  1647. if (!csk) {
  1648. pr_err("task 0x%p, csk gone.\n", task);
  1649. return -EPIPE;
  1650. }
  1651. op &= ISCSI_OPCODE_MASK;
  1652. tcp_task->dd_data = tdata;
  1653. task->hdr = NULL;
  1654. last_tdata_count = tdata->count;
  1655. last_tdata_offset = tdata->offset;
  1656. if ((op == ISCSI_OP_SCSI_DATA_OUT) ||
  1657. ((op == ISCSI_OP_SCSI_CMD) &&
  1658. (sc->sc_data_direction == DMA_TO_DEVICE))) {
  1659. u32 remaining_data_tosend, dlimit = 0;
  1660. u32 max_pdu_size, max_num_pdu, num_pdu;
  1661. u32 count;
  1662. /* Preserve conn->max_xmit_dlength because it can get updated to
  1663. * ISO data size.
  1664. */
  1665. if (task->state == ISCSI_TASK_PENDING)
  1666. tdata->max_xmit_dlength = conn->max_xmit_dlength;
  1667. if (!tdata->offset)
  1668. cxgbi_task_data_sgl_check(task);
  1669. remaining_data_tosend =
  1670. tdata->dlen - tdata->offset - tdata->count;
  1671. recalculate_sgl:
  1672. max_txdata_len = tdata->max_xmit_dlength;
  1673. log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
  1674. "tdata->dlen %u, remaining to send %u "
  1675. "conn->max_xmit_dlength %u, "
  1676. "tdata->max_xmit_dlength %u\n",
  1677. tdata->dlen, remaining_data_tosend,
  1678. conn->max_xmit_dlength, tdata->max_xmit_dlength);
  1679. if (cdev->skb_iso_txhdr && !csk->disable_iso &&
  1680. (remaining_data_tosend > tdata->max_xmit_dlength) &&
  1681. !(remaining_data_tosend % 4)) {
  1682. u32 max_iso_data;
  1683. if ((op == ISCSI_OP_SCSI_CMD) &&
  1684. session->initial_r2t_en)
  1685. goto no_iso;
  1686. max_pdu_size = tdata->max_xmit_dlength +
  1687. ISCSI_PDU_NONPAYLOAD_LEN;
  1688. max_iso_data = rounddown(CXGBI_MAX_ISO_DATA_IN_SKB,
  1689. csk->advmss);
  1690. max_num_pdu = max_iso_data / max_pdu_size;
  1691. num_pdu = (remaining_data_tosend +
  1692. tdata->max_xmit_dlength - 1) /
  1693. tdata->max_xmit_dlength;
  1694. if (num_pdu > max_num_pdu)
  1695. num_pdu = max_num_pdu;
  1696. conn->max_xmit_dlength = tdata->max_xmit_dlength * num_pdu;
  1697. max_txdata_len = conn->max_xmit_dlength;
  1698. iso_tx_rsvd = cdev->skb_iso_txhdr;
  1699. local_iso_info = sizeof(struct cxgbi_iso_info);
  1700. log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
  1701. "max_pdu_size %u, max_num_pdu %u, "
  1702. "max_txdata %u, num_pdu %u\n",
  1703. max_pdu_size, max_num_pdu,
  1704. max_txdata_len, num_pdu);
  1705. }
  1706. no_iso:
  1707. count = min_t(u32, max_txdata_len, remaining_data_tosend);
  1708. err = cxgbi_task_data_sgl_read(task,
  1709. tdata->offset + tdata->count,
  1710. count, &dlimit);
  1711. if (unlikely(err < 0)) {
  1712. log_debug(1 << CXGBI_DBG_ISCSI,
  1713. "task 0x%p, tcp_task 0x%p, tdata 0x%p, "
  1714. "sgl err %d, count %u, dlimit %u\n",
  1715. task, tcp_task, tdata, err, count, dlimit);
  1716. if (dlimit) {
  1717. remaining_data_tosend =
  1718. rounddown(dlimit,
  1719. tdata->max_xmit_dlength);
  1720. if (!remaining_data_tosend)
  1721. remaining_data_tosend = dlimit;
  1722. dlimit = 0;
  1723. conn->max_xmit_dlength = remaining_data_tosend;
  1724. goto recalculate_sgl;
  1725. }
  1726. pr_err("task 0x%p, tcp_task 0x%p, tdata 0x%p, "
  1727. "sgl err %d\n",
  1728. task, tcp_task, tdata, err);
  1729. goto ret_err;
  1730. }
  1731. if ((tdata->flags & CXGBI_TASK_SGL_COPY) ||
  1732. (tdata->nr_frags > MAX_SKB_FRAGS))
  1733. headroom += conn->max_xmit_dlength;
  1734. }
  1735. tdata->skb = alloc_skb(local_iso_info + cdev->skb_tx_rsvd +
  1736. iso_tx_rsvd + headroom, GFP_ATOMIC);
  1737. if (!tdata->skb) {
  1738. tdata->count = last_tdata_count;
  1739. tdata->offset = last_tdata_offset;
  1740. err = -ENOMEM;
  1741. goto ret_err;
  1742. }
  1743. skb_reserve(tdata->skb, local_iso_info + cdev->skb_tx_rsvd +
  1744. iso_tx_rsvd);
  1745. if (task->sc) {
  1746. task->hdr = (struct iscsi_hdr *)tdata->skb->data;
  1747. } else {
  1748. task->hdr = kzalloc(SKB_TX_ISCSI_PDU_HEADER_MAX, GFP_ATOMIC);
  1749. if (!task->hdr) {
  1750. __kfree_skb(tdata->skb);
  1751. tdata->skb = NULL;
  1752. return -ENOMEM;
  1753. }
  1754. }
  1755. task->hdr_max = SKB_TX_ISCSI_PDU_HEADER_MAX;
  1756. if (iso_tx_rsvd)
  1757. cxgbi_skcb_set_flag(tdata->skb, SKCBF_TX_ISO);
  1758. /* data_out uses scsi_cmd's itt */
  1759. if (op != ISCSI_OP_SCSI_DATA_OUT)
  1760. task_reserve_itt(task, &task->hdr->itt);
  1761. log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
  1762. "task 0x%p, op 0x%x, skb 0x%p,%u+%u/%u, itt 0x%x.\n",
  1763. task, op, tdata->skb, cdev->skb_tx_rsvd, headroom,
  1764. conn->max_xmit_dlength, be32_to_cpu(task->hdr->itt));
  1765. return 0;
  1766. ret_err:
  1767. conn->max_xmit_dlength = tdata->max_xmit_dlength;
  1768. return err;
  1769. }
  1770. EXPORT_SYMBOL_GPL(cxgbi_conn_alloc_pdu);
  1771. static int
  1772. cxgbi_prep_iso_info(struct iscsi_task *task, struct sk_buff *skb,
  1773. u32 count)
  1774. {
  1775. struct cxgbi_iso_info *iso_info = (struct cxgbi_iso_info *)skb->head;
  1776. struct iscsi_r2t_info *r2t;
  1777. struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
  1778. struct iscsi_conn *conn = task->conn;
  1779. struct iscsi_session *session = conn->session;
  1780. struct iscsi_tcp_task *tcp_task = task->dd_data;
  1781. u32 burst_size = 0, r2t_dlength = 0, dlength;
  1782. u32 max_pdu_len = tdata->max_xmit_dlength;
  1783. u32 segment_offset = 0;
  1784. u32 num_pdu;
  1785. if (unlikely(!cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO)))
  1786. return 0;
  1787. memset(iso_info, 0, sizeof(struct cxgbi_iso_info));
  1788. if (task->hdr->opcode == ISCSI_OP_SCSI_CMD && session->imm_data_en) {
  1789. iso_info->flags |= CXGBI_ISO_INFO_IMM_ENABLE;
  1790. burst_size = count;
  1791. }
  1792. dlength = ntoh24(task->hdr->dlength);
  1793. dlength = min(dlength, max_pdu_len);
  1794. hton24(task->hdr->dlength, dlength);
  1795. num_pdu = (count + max_pdu_len - 1) / max_pdu_len;
  1796. if (iscsi_task_has_unsol_data(task))
  1797. r2t = &task->unsol_r2t;
  1798. else
  1799. r2t = tcp_task->r2t;
  1800. if (r2t) {
  1801. log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
  1802. "count %u, tdata->count %u, num_pdu %u,"
  1803. "task->hdr_len %u, r2t->data_length %u, r2t->sent %u\n",
  1804. count, tdata->count, num_pdu, task->hdr_len,
  1805. r2t->data_length, r2t->sent);
  1806. r2t_dlength = r2t->data_length - r2t->sent;
  1807. segment_offset = r2t->sent;
  1808. r2t->datasn += num_pdu - 1;
  1809. }
  1810. if (!r2t || !r2t->sent)
  1811. iso_info->flags |= CXGBI_ISO_INFO_FSLICE;
  1812. if (task->hdr->flags & ISCSI_FLAG_CMD_FINAL)
  1813. iso_info->flags |= CXGBI_ISO_INFO_LSLICE;
  1814. task->hdr->flags &= ~ISCSI_FLAG_CMD_FINAL;
  1815. iso_info->op = task->hdr->opcode;
  1816. iso_info->ahs = task->hdr->hlength;
  1817. iso_info->num_pdu = num_pdu;
  1818. iso_info->mpdu = max_pdu_len;
  1819. iso_info->burst_size = (burst_size + r2t_dlength) >> 2;
  1820. iso_info->len = count + task->hdr_len;
  1821. iso_info->segment_offset = segment_offset;
  1822. cxgbi_skcb_tx_iscsi_hdrlen(skb) = task->hdr_len;
  1823. return 0;
  1824. }
  1825. static inline void tx_skb_setmode(struct sk_buff *skb, int hcrc, int dcrc)
  1826. {
  1827. if (hcrc || dcrc) {
  1828. u8 submode = 0;
  1829. if (hcrc)
  1830. submode |= 1;
  1831. if (dcrc)
  1832. submode |= 2;
  1833. cxgbi_skcb_tx_ulp_mode(skb) = (ULP2_MODE_ISCSI << 4) | submode;
  1834. } else
  1835. cxgbi_skcb_tx_ulp_mode(skb) = 0;
  1836. }
  1837. static struct page *rsvd_page;
  1838. int cxgbi_conn_init_pdu(struct iscsi_task *task, unsigned int offset,
  1839. unsigned int count)
  1840. {
  1841. struct iscsi_conn *conn = task->conn;
  1842. struct iscsi_tcp_task *tcp_task = task->dd_data;
  1843. struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
  1844. struct sk_buff *skb;
  1845. struct scsi_cmnd *sc = task->sc;
  1846. u32 expected_count, expected_offset;
  1847. u32 datalen = count, dlimit = 0;
  1848. u32 i, padlen = iscsi_padding(count);
  1849. struct page *pg;
  1850. int err;
  1851. if (!tcp_task || (tcp_task->dd_data != tdata)) {
  1852. pr_err("task 0x%p,0x%p, tcp_task 0x%p, tdata 0x%p/0x%p.\n",
  1853. task, task->sc, tcp_task,
  1854. tcp_task ? tcp_task->dd_data : NULL, tdata);
  1855. return -EINVAL;
  1856. }
  1857. skb = tdata->skb;
  1858. log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
  1859. "task 0x%p,0x%p, skb 0x%p, 0x%x,0x%x,0x%x, %u+%u.\n",
  1860. task, task->sc, skb, (*skb->data) & ISCSI_OPCODE_MASK,
  1861. be32_to_cpu(task->cmdsn), be32_to_cpu(task->hdr->itt), offset, count);
  1862. skb_put(skb, task->hdr_len);
  1863. tx_skb_setmode(skb, conn->hdrdgst_en, datalen ? conn->datadgst_en : 0);
  1864. if (!count) {
  1865. tdata->count = count;
  1866. tdata->offset = offset;
  1867. tdata->nr_frags = 0;
  1868. tdata->total_offset = 0;
  1869. tdata->total_count = 0;
  1870. if (tdata->max_xmit_dlength)
  1871. conn->max_xmit_dlength = tdata->max_xmit_dlength;
  1872. cxgbi_skcb_clear_flag(skb, SKCBF_TX_ISO);
  1873. return 0;
  1874. }
  1875. log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
  1876. "data->total_count %u, tdata->total_offset %u\n",
  1877. tdata->total_count, tdata->total_offset);
  1878. expected_count = tdata->total_count;
  1879. expected_offset = tdata->total_offset;
  1880. if ((count != expected_count) ||
  1881. (offset != expected_offset)) {
  1882. err = cxgbi_task_data_sgl_read(task, offset, count, &dlimit);
  1883. if (err < 0) {
  1884. pr_err("task 0x%p,0x%p, tcp_task 0x%p, tdata 0x%p/0x%p "
  1885. "dlimit %u, sgl err %d.\n", task, task->sc,
  1886. tcp_task, tcp_task ? tcp_task->dd_data : NULL,
  1887. tdata, dlimit, err);
  1888. return err;
  1889. }
  1890. }
  1891. /* Restore original value of conn->max_xmit_dlength because
  1892. * it can get updated to ISO data size.
  1893. */
  1894. conn->max_xmit_dlength = tdata->max_xmit_dlength;
  1895. if (sc) {
  1896. struct page_frag *frag = tdata->frags;
  1897. if ((tdata->flags & CXGBI_TASK_SGL_COPY) ||
  1898. (tdata->nr_frags > MAX_SKB_FRAGS) ||
  1899. (padlen && (tdata->nr_frags ==
  1900. MAX_SKB_FRAGS))) {
  1901. char *dst = skb->data + task->hdr_len;
  1902. /* data fits in the skb's headroom */
  1903. for (i = 0; i < tdata->nr_frags; i++, frag++) {
  1904. char *src = kmap_atomic(frag->page);
  1905. memcpy(dst, src + frag->offset, frag->size);
  1906. dst += frag->size;
  1907. kunmap_atomic(src);
  1908. }
  1909. if (padlen) {
  1910. memset(dst, 0, padlen);
  1911. padlen = 0;
  1912. }
  1913. skb_put(skb, count + padlen);
  1914. } else {
  1915. for (i = 0; i < tdata->nr_frags; i++, frag++) {
  1916. get_page(frag->page);
  1917. skb_fill_page_desc(skb, i, frag->page,
  1918. frag->offset, frag->size);
  1919. }
  1920. skb->len += count;
  1921. skb->data_len += count;
  1922. skb->truesize += count;
  1923. }
  1924. } else {
  1925. pg = virt_to_head_page(task->data);
  1926. get_page(pg);
  1927. skb_fill_page_desc(skb, 0, pg,
  1928. task->data - (char *)page_address(pg),
  1929. count);
  1930. skb->len += count;
  1931. skb->data_len += count;
  1932. skb->truesize += count;
  1933. }
  1934. if (padlen) {
  1935. get_page(rsvd_page);
  1936. skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
  1937. rsvd_page, 0, padlen);
  1938. skb->data_len += padlen;
  1939. skb->truesize += padlen;
  1940. skb->len += padlen;
  1941. }
  1942. if (likely(count > tdata->max_xmit_dlength))
  1943. cxgbi_prep_iso_info(task, skb, count);
  1944. else
  1945. cxgbi_skcb_clear_flag(skb, SKCBF_TX_ISO);
  1946. return 0;
  1947. }
  1948. EXPORT_SYMBOL_GPL(cxgbi_conn_init_pdu);
  1949. static int cxgbi_sock_tx_queue_up(struct cxgbi_sock *csk, struct sk_buff *skb)
  1950. {
  1951. struct cxgbi_device *cdev = csk->cdev;
  1952. struct cxgbi_iso_info *iso_cpl;
  1953. u32 frags = skb_shinfo(skb)->nr_frags;
  1954. u32 extra_len, num_pdu, hdr_len;
  1955. u32 iso_tx_rsvd = 0;
  1956. if (csk->state != CTP_ESTABLISHED) {
  1957. log_debug(1 << CXGBI_DBG_PDU_TX,
  1958. "csk 0x%p,%u,0x%lx,%u, EAGAIN.\n",
  1959. csk, csk->state, csk->flags, csk->tid);
  1960. return -EPIPE;
  1961. }
  1962. if (csk->err) {
  1963. log_debug(1 << CXGBI_DBG_PDU_TX,
  1964. "csk 0x%p,%u,0x%lx,%u, EPIPE %d.\n",
  1965. csk, csk->state, csk->flags, csk->tid, csk->err);
  1966. return -EPIPE;
  1967. }
  1968. if ((cdev->flags & CXGBI_FLAG_DEV_T3) &&
  1969. before((csk->snd_win + csk->snd_una), csk->write_seq)) {
  1970. log_debug(1 << CXGBI_DBG_PDU_TX,
  1971. "csk 0x%p,%u,0x%lx,%u, FULL %u-%u >= %u.\n",
  1972. csk, csk->state, csk->flags, csk->tid, csk->write_seq,
  1973. csk->snd_una, csk->snd_win);
  1974. return -ENOBUFS;
  1975. }
  1976. if (cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO))
  1977. iso_tx_rsvd = cdev->skb_iso_txhdr;
  1978. if (unlikely(skb_headroom(skb) < (cdev->skb_tx_rsvd + iso_tx_rsvd))) {
  1979. pr_err("csk 0x%p, skb head %u < %u.\n",
  1980. csk, skb_headroom(skb), cdev->skb_tx_rsvd);
  1981. return -EINVAL;
  1982. }
  1983. if (skb->len != skb->data_len)
  1984. frags++;
  1985. if (frags >= SKB_WR_LIST_SIZE) {
  1986. pr_err("csk 0x%p, frags %u, %u,%u >%lu.\n",
  1987. csk, skb_shinfo(skb)->nr_frags, skb->len,
  1988. skb->data_len, SKB_WR_LIST_SIZE);
  1989. return -EINVAL;
  1990. }
  1991. cxgbi_skcb_set_flag(skb, SKCBF_TX_NEED_HDR);
  1992. skb_reset_transport_header(skb);
  1993. cxgbi_sock_skb_entail(csk, skb);
  1994. extra_len = cxgbi_ulp_extra_len(cxgbi_skcb_tx_ulp_mode(skb));
  1995. if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO))) {
  1996. iso_cpl = (struct cxgbi_iso_info *)skb->head;
  1997. num_pdu = iso_cpl->num_pdu;
  1998. hdr_len = cxgbi_skcb_tx_iscsi_hdrlen(skb);
  1999. extra_len = (cxgbi_ulp_extra_len(cxgbi_skcb_tx_ulp_mode(skb)) *
  2000. num_pdu) + (hdr_len * (num_pdu - 1));
  2001. }
  2002. csk->write_seq += (skb->len + extra_len);
  2003. return 0;
  2004. }
  2005. static int cxgbi_sock_send_skb(struct cxgbi_sock *csk, struct sk_buff *skb)
  2006. {
  2007. struct cxgbi_device *cdev = csk->cdev;
  2008. int len = skb->len;
  2009. int err;
  2010. spin_lock_bh(&csk->lock);
  2011. err = cxgbi_sock_tx_queue_up(csk, skb);
  2012. if (err < 0) {
  2013. spin_unlock_bh(&csk->lock);
  2014. return err;
  2015. }
  2016. if (likely(skb_queue_len(&csk->write_queue)))
  2017. cdev->csk_push_tx_frames(csk, 0);
  2018. spin_unlock_bh(&csk->lock);
  2019. return len;
  2020. }
  2021. int cxgbi_conn_xmit_pdu(struct iscsi_task *task)
  2022. {
  2023. struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data;
  2024. struct cxgbi_conn *cconn = tcp_conn->dd_data;
  2025. struct iscsi_tcp_task *tcp_task = task->dd_data;
  2026. struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
  2027. struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo;
  2028. struct sk_buff *skb;
  2029. struct cxgbi_sock *csk = NULL;
  2030. u32 pdulen = 0;
  2031. u32 datalen;
  2032. int err;
  2033. if (!tcp_task || (tcp_task->dd_data != tdata)) {
  2034. pr_err("task 0x%p,0x%p, tcp_task 0x%p, tdata 0x%p/0x%p.\n",
  2035. task, task->sc, tcp_task,
  2036. tcp_task ? tcp_task->dd_data : NULL, tdata);
  2037. return -EINVAL;
  2038. }
  2039. skb = tdata->skb;
  2040. if (!skb) {
  2041. log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
  2042. "task 0x%p, skb NULL.\n", task);
  2043. return 0;
  2044. }
  2045. if (cconn && cconn->cep)
  2046. csk = cconn->cep->csk;
  2047. if (!csk) {
  2048. log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
  2049. "task 0x%p, csk gone.\n", task);
  2050. return -EPIPE;
  2051. }
  2052. tdata->skb = NULL;
  2053. datalen = skb->data_len;
  2054. /* write ppod first if using ofldq to write ppod */
  2055. if (ttinfo->flags & CXGBI_PPOD_INFO_FLAG_VALID) {
  2056. struct cxgbi_ppm *ppm = csk->cdev->cdev2ppm(csk->cdev);
  2057. ttinfo->flags &= ~CXGBI_PPOD_INFO_FLAG_VALID;
  2058. if (csk->cdev->csk_ddp_set_map(ppm, csk, ttinfo) < 0)
  2059. pr_err("task 0x%p, ppod writing using ofldq failed.\n",
  2060. task);
  2061. /* continue. Let fl get the data */
  2062. }
  2063. if (!task->sc)
  2064. memcpy(skb->data, task->hdr, SKB_TX_ISCSI_PDU_HEADER_MAX);
  2065. err = cxgbi_sock_send_skb(csk, skb);
  2066. if (err > 0) {
  2067. pdulen += err;
  2068. log_debug(1 << CXGBI_DBG_PDU_TX, "task 0x%p,0x%p, rv %d.\n",
  2069. task, task->sc, err);
  2070. if (task->conn->hdrdgst_en)
  2071. pdulen += ISCSI_DIGEST_SIZE;
  2072. if (datalen && task->conn->datadgst_en)
  2073. pdulen += ISCSI_DIGEST_SIZE;
  2074. task->conn->txdata_octets += pdulen;
  2075. if (unlikely(cxgbi_is_iso_config(csk) && cxgbi_is_iso_disabled(csk))) {
  2076. if (time_after(jiffies, csk->prev_iso_ts + HZ)) {
  2077. csk->disable_iso = false;
  2078. csk->prev_iso_ts = 0;
  2079. log_debug(1 << CXGBI_DBG_PDU_TX,
  2080. "enable iso: csk 0x%p\n", csk);
  2081. }
  2082. }
  2083. return 0;
  2084. }
  2085. if (err == -EAGAIN || err == -ENOBUFS) {
  2086. log_debug(1 << CXGBI_DBG_PDU_TX,
  2087. "task 0x%p, skb 0x%p, len %u/%u, %d EAGAIN.\n",
  2088. task, skb, skb->len, skb->data_len, err);
  2089. /* reset skb to send when we are called again */
  2090. tdata->skb = skb;
  2091. if (cxgbi_is_iso_config(csk) && !cxgbi_is_iso_disabled(csk) &&
  2092. (csk->no_tx_credits++ >= 2)) {
  2093. csk->disable_iso = true;
  2094. csk->prev_iso_ts = jiffies;
  2095. log_debug(1 << CXGBI_DBG_PDU_TX,
  2096. "disable iso:csk 0x%p, ts:%lu\n",
  2097. csk, csk->prev_iso_ts);
  2098. }
  2099. return err;
  2100. }
  2101. log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
  2102. "itt 0x%x, skb 0x%p, len %u/%u, xmit err %d.\n",
  2103. task->itt, skb, skb->len, skb->data_len, err);
  2104. __kfree_skb(skb);
  2105. iscsi_conn_printk(KERN_ERR, task->conn, "xmit err %d.\n", err);
  2106. iscsi_conn_failure(task->conn, ISCSI_ERR_XMIT_FAILED);
  2107. return err;
  2108. }
  2109. EXPORT_SYMBOL_GPL(cxgbi_conn_xmit_pdu);
  2110. void cxgbi_cleanup_task(struct iscsi_task *task)
  2111. {
  2112. struct iscsi_tcp_task *tcp_task = task->dd_data;
  2113. struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
  2114. if (!tcp_task || (tcp_task->dd_data != tdata)) {
  2115. pr_info("task 0x%p,0x%p, tcp_task 0x%p, tdata 0x%p/0x%p.\n",
  2116. task, task->sc, tcp_task,
  2117. tcp_task ? tcp_task->dd_data : NULL, tdata);
  2118. return;
  2119. }
  2120. log_debug(1 << CXGBI_DBG_ISCSI,
  2121. "task 0x%p, skb 0x%p, itt 0x%x.\n",
  2122. task, tdata->skb, task->hdr_itt);
  2123. tcp_task->dd_data = NULL;
  2124. if (!task->sc)
  2125. kfree(task->hdr);
  2126. task->hdr = NULL;
  2127. /* never reached the xmit task callout */
  2128. if (tdata->skb) {
  2129. __kfree_skb(tdata->skb);
  2130. tdata->skb = NULL;
  2131. }
  2132. task_release_itt(task, task->hdr_itt);
  2133. memset(tdata, 0, sizeof(*tdata));
  2134. iscsi_tcp_cleanup_task(task);
  2135. }
  2136. EXPORT_SYMBOL_GPL(cxgbi_cleanup_task);
  2137. void cxgbi_get_conn_stats(struct iscsi_cls_conn *cls_conn,
  2138. struct iscsi_stats *stats)
  2139. {
  2140. struct iscsi_conn *conn = cls_conn->dd_data;
  2141. stats->txdata_octets = conn->txdata_octets;
  2142. stats->rxdata_octets = conn->rxdata_octets;
  2143. stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
  2144. stats->dataout_pdus = conn->dataout_pdus_cnt;
  2145. stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
  2146. stats->datain_pdus = conn->datain_pdus_cnt;
  2147. stats->r2t_pdus = conn->r2t_pdus_cnt;
  2148. stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
  2149. stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
  2150. stats->digest_err = 0;
  2151. stats->timeout_err = 0;
  2152. stats->custom_length = 1;
  2153. strcpy(stats->custom[0].desc, "eh_abort_cnt");
  2154. stats->custom[0].value = conn->eh_abort_cnt;
  2155. }
  2156. EXPORT_SYMBOL_GPL(cxgbi_get_conn_stats);
  2157. static int cxgbi_conn_max_xmit_dlength(struct iscsi_conn *conn)
  2158. {
  2159. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  2160. struct cxgbi_conn *cconn = tcp_conn->dd_data;
  2161. struct cxgbi_device *cdev = cconn->chba->cdev;
  2162. unsigned int headroom = SKB_MAX_HEAD(cdev->skb_tx_rsvd);
  2163. unsigned int max_def = 512 * MAX_SKB_FRAGS;
  2164. unsigned int max = max(max_def, headroom);
  2165. max = min(cconn->chba->cdev->tx_max_size, max);
  2166. if (conn->max_xmit_dlength)
  2167. conn->max_xmit_dlength = min(conn->max_xmit_dlength, max);
  2168. else
  2169. conn->max_xmit_dlength = max;
  2170. cxgbi_align_pdu_size(conn->max_xmit_dlength);
  2171. return 0;
  2172. }
  2173. static int cxgbi_conn_max_recv_dlength(struct iscsi_conn *conn)
  2174. {
  2175. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  2176. struct cxgbi_conn *cconn = tcp_conn->dd_data;
  2177. unsigned int max = cconn->chba->cdev->rx_max_size;
  2178. cxgbi_align_pdu_size(max);
  2179. if (conn->max_recv_dlength) {
  2180. if (conn->max_recv_dlength > max) {
  2181. pr_err("MaxRecvDataSegmentLength %u > %u.\n",
  2182. conn->max_recv_dlength, max);
  2183. return -EINVAL;
  2184. }
  2185. conn->max_recv_dlength = min(conn->max_recv_dlength, max);
  2186. cxgbi_align_pdu_size(conn->max_recv_dlength);
  2187. } else
  2188. conn->max_recv_dlength = max;
  2189. return 0;
  2190. }
  2191. int cxgbi_set_conn_param(struct iscsi_cls_conn *cls_conn,
  2192. enum iscsi_param param, char *buf, int buflen)
  2193. {
  2194. struct iscsi_conn *conn = cls_conn->dd_data;
  2195. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  2196. struct cxgbi_conn *cconn = tcp_conn->dd_data;
  2197. struct cxgbi_sock *csk = cconn->cep->csk;
  2198. int err;
  2199. log_debug(1 << CXGBI_DBG_ISCSI,
  2200. "cls_conn 0x%p, param %d, buf(%d) %s.\n",
  2201. cls_conn, param, buflen, buf);
  2202. switch (param) {
  2203. case ISCSI_PARAM_HDRDGST_EN:
  2204. err = iscsi_set_param(cls_conn, param, buf, buflen);
  2205. if (!err && conn->hdrdgst_en)
  2206. err = csk->cdev->csk_ddp_setup_digest(csk, csk->tid,
  2207. conn->hdrdgst_en,
  2208. conn->datadgst_en);
  2209. break;
  2210. case ISCSI_PARAM_DATADGST_EN:
  2211. err = iscsi_set_param(cls_conn, param, buf, buflen);
  2212. if (!err && conn->datadgst_en)
  2213. err = csk->cdev->csk_ddp_setup_digest(csk, csk->tid,
  2214. conn->hdrdgst_en,
  2215. conn->datadgst_en);
  2216. break;
  2217. case ISCSI_PARAM_MAX_R2T:
  2218. return iscsi_tcp_set_max_r2t(conn, buf);
  2219. case ISCSI_PARAM_MAX_RECV_DLENGTH:
  2220. err = iscsi_set_param(cls_conn, param, buf, buflen);
  2221. if (!err)
  2222. err = cxgbi_conn_max_recv_dlength(conn);
  2223. break;
  2224. case ISCSI_PARAM_MAX_XMIT_DLENGTH:
  2225. err = iscsi_set_param(cls_conn, param, buf, buflen);
  2226. if (!err)
  2227. err = cxgbi_conn_max_xmit_dlength(conn);
  2228. break;
  2229. default:
  2230. return iscsi_set_param(cls_conn, param, buf, buflen);
  2231. }
  2232. return err;
  2233. }
  2234. EXPORT_SYMBOL_GPL(cxgbi_set_conn_param);
  2235. int cxgbi_get_ep_param(struct iscsi_endpoint *ep, enum iscsi_param param,
  2236. char *buf)
  2237. {
  2238. struct cxgbi_endpoint *cep = ep->dd_data;
  2239. struct cxgbi_sock *csk;
  2240. log_debug(1 << CXGBI_DBG_ISCSI,
  2241. "cls_conn 0x%p, param %d.\n", ep, param);
  2242. switch (param) {
  2243. case ISCSI_PARAM_CONN_PORT:
  2244. case ISCSI_PARAM_CONN_ADDRESS:
  2245. if (!cep)
  2246. return -ENOTCONN;
  2247. csk = cep->csk;
  2248. if (!csk)
  2249. return -ENOTCONN;
  2250. return iscsi_conn_get_addr_param((struct sockaddr_storage *)
  2251. &csk->daddr, param, buf);
  2252. default:
  2253. break;
  2254. }
  2255. return -ENOSYS;
  2256. }
  2257. EXPORT_SYMBOL_GPL(cxgbi_get_ep_param);
  2258. struct iscsi_cls_conn *
  2259. cxgbi_create_conn(struct iscsi_cls_session *cls_session, u32 cid)
  2260. {
  2261. struct iscsi_cls_conn *cls_conn;
  2262. struct iscsi_conn *conn;
  2263. struct iscsi_tcp_conn *tcp_conn;
  2264. struct cxgbi_conn *cconn;
  2265. cls_conn = iscsi_tcp_conn_setup(cls_session, sizeof(*cconn), cid);
  2266. if (!cls_conn)
  2267. return NULL;
  2268. conn = cls_conn->dd_data;
  2269. tcp_conn = conn->dd_data;
  2270. cconn = tcp_conn->dd_data;
  2271. cconn->iconn = conn;
  2272. log_debug(1 << CXGBI_DBG_ISCSI,
  2273. "cid %u(0x%x), cls 0x%p,0x%p, conn 0x%p,0x%p,0x%p.\n",
  2274. cid, cid, cls_session, cls_conn, conn, tcp_conn, cconn);
  2275. return cls_conn;
  2276. }
  2277. EXPORT_SYMBOL_GPL(cxgbi_create_conn);
  2278. int cxgbi_bind_conn(struct iscsi_cls_session *cls_session,
  2279. struct iscsi_cls_conn *cls_conn,
  2280. u64 transport_eph, int is_leading)
  2281. {
  2282. struct iscsi_conn *conn = cls_conn->dd_data;
  2283. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  2284. struct cxgbi_conn *cconn = tcp_conn->dd_data;
  2285. struct cxgbi_ppm *ppm;
  2286. struct iscsi_endpoint *ep;
  2287. struct cxgbi_endpoint *cep;
  2288. struct cxgbi_sock *csk;
  2289. int err;
  2290. ep = iscsi_lookup_endpoint(transport_eph);
  2291. if (!ep)
  2292. return -EINVAL;
  2293. /* setup ddp pagesize */
  2294. cep = ep->dd_data;
  2295. csk = cep->csk;
  2296. ppm = csk->cdev->cdev2ppm(csk->cdev);
  2297. err = csk->cdev->csk_ddp_setup_pgidx(csk, csk->tid,
  2298. ppm->tformat.pgsz_idx_dflt);
  2299. if (err < 0)
  2300. goto put_ep;
  2301. err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
  2302. if (err) {
  2303. err = -EINVAL;
  2304. goto put_ep;
  2305. }
  2306. /* calculate the tag idx bits needed for this conn based on cmds_max */
  2307. cconn->task_idx_bits = (__ilog2_u32(conn->session->cmds_max - 1)) + 1;
  2308. write_lock_bh(&csk->callback_lock);
  2309. csk->user_data = conn;
  2310. cconn->chba = cep->chba;
  2311. cconn->cep = cep;
  2312. cep->cconn = cconn;
  2313. write_unlock_bh(&csk->callback_lock);
  2314. cxgbi_conn_max_xmit_dlength(conn);
  2315. cxgbi_conn_max_recv_dlength(conn);
  2316. log_debug(1 << CXGBI_DBG_ISCSI,
  2317. "cls 0x%p,0x%p, ep 0x%p, cconn 0x%p, csk 0x%p.\n",
  2318. cls_session, cls_conn, ep, cconn, csk);
  2319. /* init recv engine */
  2320. iscsi_tcp_hdr_recv_prep(tcp_conn);
  2321. put_ep:
  2322. iscsi_put_endpoint(ep);
  2323. return err;
  2324. }
  2325. EXPORT_SYMBOL_GPL(cxgbi_bind_conn);
  2326. struct iscsi_cls_session *cxgbi_create_session(struct iscsi_endpoint *ep,
  2327. u16 cmds_max, u16 qdepth,
  2328. u32 initial_cmdsn)
  2329. {
  2330. struct cxgbi_endpoint *cep;
  2331. struct cxgbi_hba *chba;
  2332. struct Scsi_Host *shost;
  2333. struct iscsi_cls_session *cls_session;
  2334. struct iscsi_session *session;
  2335. if (!ep) {
  2336. pr_err("missing endpoint.\n");
  2337. return NULL;
  2338. }
  2339. cep = ep->dd_data;
  2340. chba = cep->chba;
  2341. shost = chba->shost;
  2342. BUG_ON(chba != iscsi_host_priv(shost));
  2343. cls_session = iscsi_session_setup(chba->cdev->itp, shost,
  2344. cmds_max, 0,
  2345. sizeof(struct iscsi_tcp_task) +
  2346. sizeof(struct cxgbi_task_data),
  2347. initial_cmdsn, ISCSI_MAX_TARGET);
  2348. if (!cls_session)
  2349. return NULL;
  2350. session = cls_session->dd_data;
  2351. if (iscsi_tcp_r2tpool_alloc(session))
  2352. goto remove_session;
  2353. log_debug(1 << CXGBI_DBG_ISCSI,
  2354. "ep 0x%p, cls sess 0x%p.\n", ep, cls_session);
  2355. return cls_session;
  2356. remove_session:
  2357. iscsi_session_teardown(cls_session);
  2358. return NULL;
  2359. }
  2360. EXPORT_SYMBOL_GPL(cxgbi_create_session);
  2361. void cxgbi_destroy_session(struct iscsi_cls_session *cls_session)
  2362. {
  2363. log_debug(1 << CXGBI_DBG_ISCSI,
  2364. "cls sess 0x%p.\n", cls_session);
  2365. iscsi_tcp_r2tpool_free(cls_session->dd_data);
  2366. iscsi_session_teardown(cls_session);
  2367. }
  2368. EXPORT_SYMBOL_GPL(cxgbi_destroy_session);
  2369. int cxgbi_set_host_param(struct Scsi_Host *shost, enum iscsi_host_param param,
  2370. char *buf, int buflen)
  2371. {
  2372. struct cxgbi_hba *chba = iscsi_host_priv(shost);
  2373. if (!chba->ndev) {
  2374. shost_printk(KERN_ERR, shost, "Could not get host param. "
  2375. "netdev for host not set.\n");
  2376. return -ENODEV;
  2377. }
  2378. log_debug(1 << CXGBI_DBG_ISCSI,
  2379. "shost 0x%p, hba 0x%p,%s, param %d, buf(%d) %s.\n",
  2380. shost, chba, chba->ndev->name, param, buflen, buf);
  2381. switch (param) {
  2382. case ISCSI_HOST_PARAM_IPADDRESS:
  2383. {
  2384. __be32 addr = in_aton(buf);
  2385. log_debug(1 << CXGBI_DBG_ISCSI,
  2386. "hba %s, req. ipv4 %pI4.\n", chba->ndev->name, &addr);
  2387. cxgbi_set_iscsi_ipv4(chba, addr);
  2388. return 0;
  2389. }
  2390. case ISCSI_HOST_PARAM_HWADDRESS:
  2391. case ISCSI_HOST_PARAM_NETDEV_NAME:
  2392. return 0;
  2393. default:
  2394. return iscsi_host_set_param(shost, param, buf, buflen);
  2395. }
  2396. }
  2397. EXPORT_SYMBOL_GPL(cxgbi_set_host_param);
  2398. int cxgbi_get_host_param(struct Scsi_Host *shost, enum iscsi_host_param param,
  2399. char *buf)
  2400. {
  2401. struct cxgbi_hba *chba = iscsi_host_priv(shost);
  2402. int len = 0;
  2403. if (!chba->ndev) {
  2404. shost_printk(KERN_ERR, shost, "Could not get host param. "
  2405. "netdev for host not set.\n");
  2406. return -ENODEV;
  2407. }
  2408. log_debug(1 << CXGBI_DBG_ISCSI,
  2409. "shost 0x%p, hba 0x%p,%s, param %d.\n",
  2410. shost, chba, chba->ndev->name, param);
  2411. switch (param) {
  2412. case ISCSI_HOST_PARAM_HWADDRESS:
  2413. len = sysfs_format_mac(buf, chba->ndev->dev_addr, 6);
  2414. break;
  2415. case ISCSI_HOST_PARAM_NETDEV_NAME:
  2416. len = sprintf(buf, "%s\n", chba->ndev->name);
  2417. break;
  2418. case ISCSI_HOST_PARAM_IPADDRESS:
  2419. {
  2420. struct cxgbi_sock *csk = find_sock_on_port(chba->cdev,
  2421. chba->port_id);
  2422. if (csk) {
  2423. len = sprintf(buf, "%pIS",
  2424. (struct sockaddr *)&csk->saddr);
  2425. }
  2426. log_debug(1 << CXGBI_DBG_ISCSI,
  2427. "hba %s, addr %s.\n", chba->ndev->name, buf);
  2428. break;
  2429. }
  2430. default:
  2431. return iscsi_host_get_param(shost, param, buf);
  2432. }
  2433. return len;
  2434. }
  2435. EXPORT_SYMBOL_GPL(cxgbi_get_host_param);
  2436. struct iscsi_endpoint *cxgbi_ep_connect(struct Scsi_Host *shost,
  2437. struct sockaddr *dst_addr,
  2438. int non_blocking)
  2439. {
  2440. struct iscsi_endpoint *ep;
  2441. struct cxgbi_endpoint *cep;
  2442. struct cxgbi_hba *hba = NULL;
  2443. struct cxgbi_sock *csk;
  2444. int ifindex = 0;
  2445. int err = -EINVAL;
  2446. log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK,
  2447. "shost 0x%p, non_blocking %d, dst_addr 0x%p.\n",
  2448. shost, non_blocking, dst_addr);
  2449. if (shost) {
  2450. hba = iscsi_host_priv(shost);
  2451. if (!hba) {
  2452. pr_info("shost 0x%p, priv NULL.\n", shost);
  2453. goto err_out;
  2454. }
  2455. }
  2456. check_route:
  2457. if (dst_addr->sa_family == AF_INET) {
  2458. csk = cxgbi_check_route(dst_addr, ifindex);
  2459. #if IS_ENABLED(CONFIG_IPV6)
  2460. } else if (dst_addr->sa_family == AF_INET6) {
  2461. csk = cxgbi_check_route6(dst_addr, ifindex);
  2462. #endif
  2463. } else {
  2464. pr_info("address family 0x%x NOT supported.\n",
  2465. dst_addr->sa_family);
  2466. err = -EAFNOSUPPORT;
  2467. return (struct iscsi_endpoint *)ERR_PTR(err);
  2468. }
  2469. if (IS_ERR(csk))
  2470. return (struct iscsi_endpoint *)csk;
  2471. cxgbi_sock_get(csk);
  2472. if (!hba)
  2473. hba = csk->cdev->hbas[csk->port_id];
  2474. else if (hba != csk->cdev->hbas[csk->port_id]) {
  2475. if (ifindex != hba->ndev->ifindex) {
  2476. cxgbi_sock_put(csk);
  2477. cxgbi_sock_closed(csk);
  2478. ifindex = hba->ndev->ifindex;
  2479. goto check_route;
  2480. }
  2481. pr_info("Could not connect through requested host %u"
  2482. "hba 0x%p != 0x%p (%u).\n",
  2483. shost->host_no, hba,
  2484. csk->cdev->hbas[csk->port_id], csk->port_id);
  2485. err = -ENOSPC;
  2486. goto release_conn;
  2487. }
  2488. err = sock_get_port(csk);
  2489. if (err)
  2490. goto release_conn;
  2491. cxgbi_sock_set_state(csk, CTP_CONNECTING);
  2492. err = csk->cdev->csk_init_act_open(csk);
  2493. if (err)
  2494. goto release_conn;
  2495. if (cxgbi_sock_is_closing(csk)) {
  2496. err = -ENOSPC;
  2497. pr_info("csk 0x%p is closing.\n", csk);
  2498. goto release_conn;
  2499. }
  2500. ep = iscsi_create_endpoint(sizeof(*cep));
  2501. if (!ep) {
  2502. err = -ENOMEM;
  2503. pr_info("iscsi alloc ep, OOM.\n");
  2504. goto release_conn;
  2505. }
  2506. cep = ep->dd_data;
  2507. cep->csk = csk;
  2508. cep->chba = hba;
  2509. log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK,
  2510. "ep 0x%p, cep 0x%p, csk 0x%p, hba 0x%p,%s.\n",
  2511. ep, cep, csk, hba, hba->ndev->name);
  2512. return ep;
  2513. release_conn:
  2514. cxgbi_sock_put(csk);
  2515. cxgbi_sock_closed(csk);
  2516. err_out:
  2517. return ERR_PTR(err);
  2518. }
  2519. EXPORT_SYMBOL_GPL(cxgbi_ep_connect);
  2520. int cxgbi_ep_poll(struct iscsi_endpoint *ep, int timeout_ms)
  2521. {
  2522. struct cxgbi_endpoint *cep = ep->dd_data;
  2523. struct cxgbi_sock *csk = cep->csk;
  2524. if (!cxgbi_sock_is_established(csk))
  2525. return 0;
  2526. return 1;
  2527. }
  2528. EXPORT_SYMBOL_GPL(cxgbi_ep_poll);
  2529. void cxgbi_ep_disconnect(struct iscsi_endpoint *ep)
  2530. {
  2531. struct cxgbi_endpoint *cep = ep->dd_data;
  2532. struct cxgbi_conn *cconn = cep->cconn;
  2533. struct cxgbi_sock *csk = cep->csk;
  2534. log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK,
  2535. "ep 0x%p, cep 0x%p, cconn 0x%p, csk 0x%p,%u,0x%lx.\n",
  2536. ep, cep, cconn, csk, csk->state, csk->flags);
  2537. if (cconn && cconn->iconn) {
  2538. iscsi_suspend_tx(cconn->iconn);
  2539. write_lock_bh(&csk->callback_lock);
  2540. cep->csk->user_data = NULL;
  2541. cconn->cep = NULL;
  2542. write_unlock_bh(&csk->callback_lock);
  2543. }
  2544. iscsi_destroy_endpoint(ep);
  2545. if (likely(csk->state >= CTP_ESTABLISHED))
  2546. need_active_close(csk);
  2547. else
  2548. cxgbi_sock_closed(csk);
  2549. cxgbi_sock_put(csk);
  2550. }
  2551. EXPORT_SYMBOL_GPL(cxgbi_ep_disconnect);
  2552. int cxgbi_iscsi_init(struct iscsi_transport *itp,
  2553. struct scsi_transport_template **stt)
  2554. {
  2555. *stt = iscsi_register_transport(itp);
  2556. if (*stt == NULL) {
  2557. pr_err("unable to register %s transport 0x%p.\n",
  2558. itp->name, itp);
  2559. return -ENODEV;
  2560. }
  2561. log_debug(1 << CXGBI_DBG_ISCSI,
  2562. "%s, registered iscsi transport 0x%p.\n",
  2563. itp->name, stt);
  2564. return 0;
  2565. }
  2566. EXPORT_SYMBOL_GPL(cxgbi_iscsi_init);
  2567. void cxgbi_iscsi_cleanup(struct iscsi_transport *itp,
  2568. struct scsi_transport_template **stt)
  2569. {
  2570. if (*stt) {
  2571. log_debug(1 << CXGBI_DBG_ISCSI,
  2572. "de-register transport 0x%p, %s, stt 0x%p.\n",
  2573. itp, itp->name, *stt);
  2574. *stt = NULL;
  2575. iscsi_unregister_transport(itp);
  2576. }
  2577. }
  2578. EXPORT_SYMBOL_GPL(cxgbi_iscsi_cleanup);
  2579. umode_t cxgbi_attr_is_visible(int param_type, int param)
  2580. {
  2581. switch (param_type) {
  2582. case ISCSI_HOST_PARAM:
  2583. switch (param) {
  2584. case ISCSI_HOST_PARAM_NETDEV_NAME:
  2585. case ISCSI_HOST_PARAM_HWADDRESS:
  2586. case ISCSI_HOST_PARAM_IPADDRESS:
  2587. case ISCSI_HOST_PARAM_INITIATOR_NAME:
  2588. return S_IRUGO;
  2589. default:
  2590. return 0;
  2591. }
  2592. case ISCSI_PARAM:
  2593. switch (param) {
  2594. case ISCSI_PARAM_MAX_RECV_DLENGTH:
  2595. case ISCSI_PARAM_MAX_XMIT_DLENGTH:
  2596. case ISCSI_PARAM_HDRDGST_EN:
  2597. case ISCSI_PARAM_DATADGST_EN:
  2598. case ISCSI_PARAM_CONN_ADDRESS:
  2599. case ISCSI_PARAM_CONN_PORT:
  2600. case ISCSI_PARAM_EXP_STATSN:
  2601. case ISCSI_PARAM_PERSISTENT_ADDRESS:
  2602. case ISCSI_PARAM_PERSISTENT_PORT:
  2603. case ISCSI_PARAM_PING_TMO:
  2604. case ISCSI_PARAM_RECV_TMO:
  2605. case ISCSI_PARAM_INITIAL_R2T_EN:
  2606. case ISCSI_PARAM_MAX_R2T:
  2607. case ISCSI_PARAM_IMM_DATA_EN:
  2608. case ISCSI_PARAM_FIRST_BURST:
  2609. case ISCSI_PARAM_MAX_BURST:
  2610. case ISCSI_PARAM_PDU_INORDER_EN:
  2611. case ISCSI_PARAM_DATASEQ_INORDER_EN:
  2612. case ISCSI_PARAM_ERL:
  2613. case ISCSI_PARAM_TARGET_NAME:
  2614. case ISCSI_PARAM_TPGT:
  2615. case ISCSI_PARAM_USERNAME:
  2616. case ISCSI_PARAM_PASSWORD:
  2617. case ISCSI_PARAM_USERNAME_IN:
  2618. case ISCSI_PARAM_PASSWORD_IN:
  2619. case ISCSI_PARAM_FAST_ABORT:
  2620. case ISCSI_PARAM_ABORT_TMO:
  2621. case ISCSI_PARAM_LU_RESET_TMO:
  2622. case ISCSI_PARAM_TGT_RESET_TMO:
  2623. case ISCSI_PARAM_IFACE_NAME:
  2624. case ISCSI_PARAM_INITIATOR_NAME:
  2625. return S_IRUGO;
  2626. default:
  2627. return 0;
  2628. }
  2629. }
  2630. return 0;
  2631. }
  2632. EXPORT_SYMBOL_GPL(cxgbi_attr_is_visible);
  2633. static int __init libcxgbi_init_module(void)
  2634. {
  2635. pr_info("%s", version);
  2636. BUILD_BUG_ON(sizeof_field(struct sk_buff, cb) <
  2637. sizeof(struct cxgbi_skb_cb));
  2638. rsvd_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  2639. if (!rsvd_page)
  2640. return -ENOMEM;
  2641. return 0;
  2642. }
  2643. static void __exit libcxgbi_exit_module(void)
  2644. {
  2645. cxgbi_device_unregister_all(0xFF);
  2646. put_page(rsvd_page);
  2647. return;
  2648. }
  2649. module_init(libcxgbi_init_module);
  2650. module_exit(libcxgbi_exit_module);