mtip32xx.c 108 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Driver for the Micron P320 SSD
  4. * Copyright (C) 2011 Micron Technology, Inc.
  5. *
  6. * Portions of this code were derived from works subjected to the
  7. * following copyright:
  8. * Copyright (C) 2009 Integrated Device Technology, Inc.
  9. */
  10. #include <linux/pci.h>
  11. #include <linux/interrupt.h>
  12. #include <linux/ata.h>
  13. #include <linux/delay.h>
  14. #include <linux/hdreg.h>
  15. #include <linux/uaccess.h>
  16. #include <linux/random.h>
  17. #include <linux/smp.h>
  18. #include <linux/compat.h>
  19. #include <linux/fs.h>
  20. #include <linux/module.h>
  21. #include <linux/genhd.h>
  22. #include <linux/blkdev.h>
  23. #include <linux/blk-mq.h>
  24. #include <linux/bio.h>
  25. #include <linux/dma-mapping.h>
  26. #include <linux/idr.h>
  27. #include <linux/kthread.h>
  28. #include <../drivers/ata/ahci.h>
  29. #include <linux/export.h>
  30. #include <linux/debugfs.h>
  31. #include <linux/prefetch.h>
  32. #include <linux/numa.h>
  33. #include "mtip32xx.h"
  34. #define HW_CMD_SLOT_SZ (MTIP_MAX_COMMAND_SLOTS * 32)
  35. /* DMA region containing RX Fis, Identify, RLE10, and SMART buffers */
  36. #define AHCI_RX_FIS_SZ 0x100
  37. #define AHCI_RX_FIS_OFFSET 0x0
  38. #define AHCI_IDFY_SZ ATA_SECT_SIZE
  39. #define AHCI_IDFY_OFFSET 0x400
  40. #define AHCI_SECTBUF_SZ ATA_SECT_SIZE
  41. #define AHCI_SECTBUF_OFFSET 0x800
  42. #define AHCI_SMARTBUF_SZ ATA_SECT_SIZE
  43. #define AHCI_SMARTBUF_OFFSET 0xC00
  44. /* 0x100 + 0x200 + 0x200 + 0x200 is smaller than 4k but we pad it out */
  45. #define BLOCK_DMA_ALLOC_SZ 4096
  46. /* DMA region containing command table (should be 8192 bytes) */
  47. #define AHCI_CMD_SLOT_SZ sizeof(struct mtip_cmd_hdr)
  48. #define AHCI_CMD_TBL_SZ (MTIP_MAX_COMMAND_SLOTS * AHCI_CMD_SLOT_SZ)
  49. #define AHCI_CMD_TBL_OFFSET 0x0
  50. /* DMA region per command (contains header and SGL) */
  51. #define AHCI_CMD_TBL_HDR_SZ 0x80
  52. #define AHCI_CMD_TBL_HDR_OFFSET 0x0
  53. #define AHCI_CMD_TBL_SGL_SZ (MTIP_MAX_SG * sizeof(struct mtip_cmd_sg))
  54. #define AHCI_CMD_TBL_SGL_OFFSET AHCI_CMD_TBL_HDR_SZ
  55. #define CMD_DMA_ALLOC_SZ (AHCI_CMD_TBL_SGL_SZ + AHCI_CMD_TBL_HDR_SZ)
  56. #define HOST_CAP_NZDMA (1 << 19)
  57. #define HOST_HSORG 0xFC
  58. #define HSORG_DISABLE_SLOTGRP_INTR (1<<24)
  59. #define HSORG_DISABLE_SLOTGRP_PXIS (1<<16)
  60. #define HSORG_HWREV 0xFF00
  61. #define HSORG_STYLE 0x8
  62. #define HSORG_SLOTGROUPS 0x7
  63. #define PORT_COMMAND_ISSUE 0x38
  64. #define PORT_SDBV 0x7C
  65. #define PORT_OFFSET 0x100
  66. #define PORT_MEM_SIZE 0x80
  67. #define PORT_IRQ_ERR \
  68. (PORT_IRQ_HBUS_ERR | PORT_IRQ_IF_ERR | PORT_IRQ_CONNECT | \
  69. PORT_IRQ_PHYRDY | PORT_IRQ_UNK_FIS | PORT_IRQ_BAD_PMP | \
  70. PORT_IRQ_TF_ERR | PORT_IRQ_HBUS_DATA_ERR | PORT_IRQ_IF_NONFATAL | \
  71. PORT_IRQ_OVERFLOW)
  72. #define PORT_IRQ_LEGACY \
  73. (PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS)
  74. #define PORT_IRQ_HANDLED \
  75. (PORT_IRQ_SDB_FIS | PORT_IRQ_LEGACY | \
  76. PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR | \
  77. PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)
  78. #define DEF_PORT_IRQ \
  79. (PORT_IRQ_ERR | PORT_IRQ_LEGACY | PORT_IRQ_SDB_FIS)
  80. /* product numbers */
  81. #define MTIP_PRODUCT_UNKNOWN 0x00
  82. #define MTIP_PRODUCT_ASICFPGA 0x11
  83. /* Device instance number, incremented each time a device is probed. */
  84. static int instance;
  85. static struct list_head online_list;
  86. static struct list_head removing_list;
  87. static spinlock_t dev_lock;
  88. /*
  89. * Global variable used to hold the major block device number
  90. * allocated in mtip_init().
  91. */
  92. static int mtip_major;
  93. static struct dentry *dfs_parent;
  94. static struct dentry *dfs_device_status;
  95. static u32 cpu_use[NR_CPUS];
  96. static DEFINE_IDA(rssd_index_ida);
  97. static int mtip_block_initialize(struct driver_data *dd);
  98. #ifdef CONFIG_COMPAT
  99. struct mtip_compat_ide_task_request_s {
  100. __u8 io_ports[8];
  101. __u8 hob_ports[8];
  102. ide_reg_valid_t out_flags;
  103. ide_reg_valid_t in_flags;
  104. int data_phase;
  105. int req_cmd;
  106. compat_ulong_t out_size;
  107. compat_ulong_t in_size;
  108. };
  109. #endif
  110. /*
  111. * This function check_for_surprise_removal is called
  112. * while card is removed from the system and it will
  113. * read the vendor id from the configuration space
  114. *
  115. * @pdev Pointer to the pci_dev structure.
  116. *
  117. * return value
  118. * true if device removed, else false
  119. */
  120. static bool mtip_check_surprise_removal(struct pci_dev *pdev)
  121. {
  122. u16 vendor_id = 0;
  123. struct driver_data *dd = pci_get_drvdata(pdev);
  124. if (dd->sr)
  125. return true;
  126. /* Read the vendorID from the configuration space */
  127. pci_read_config_word(pdev, 0x00, &vendor_id);
  128. if (vendor_id == 0xFFFF) {
  129. dd->sr = true;
  130. if (dd->queue)
  131. blk_queue_flag_set(QUEUE_FLAG_DEAD, dd->queue);
  132. else
  133. dev_warn(&dd->pdev->dev,
  134. "%s: dd->queue is NULL\n", __func__);
  135. return true; /* device removed */
  136. }
  137. return false; /* device present */
  138. }
  139. static struct mtip_cmd *mtip_cmd_from_tag(struct driver_data *dd,
  140. unsigned int tag)
  141. {
  142. struct blk_mq_hw_ctx *hctx = dd->queue->queue_hw_ctx[0];
  143. return blk_mq_rq_to_pdu(blk_mq_tag_to_rq(hctx->tags, tag));
  144. }
  145. /*
  146. * Reset the HBA (without sleeping)
  147. *
  148. * @dd Pointer to the driver data structure.
  149. *
  150. * return value
  151. * 0 The reset was successful.
  152. * -1 The HBA Reset bit did not clear.
  153. */
  154. static int mtip_hba_reset(struct driver_data *dd)
  155. {
  156. unsigned long timeout;
  157. /* Set the reset bit */
  158. writel(HOST_RESET, dd->mmio + HOST_CTL);
  159. /* Flush */
  160. readl(dd->mmio + HOST_CTL);
  161. /*
  162. * Spin for up to 10 seconds waiting for reset acknowledgement. Spec
  163. * is 1 sec but in LUN failure conditions, up to 10 secs are required
  164. */
  165. timeout = jiffies + msecs_to_jiffies(10000);
  166. do {
  167. mdelay(10);
  168. if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))
  169. return -1;
  170. } while ((readl(dd->mmio + HOST_CTL) & HOST_RESET)
  171. && time_before(jiffies, timeout));
  172. if (readl(dd->mmio + HOST_CTL) & HOST_RESET)
  173. return -1;
  174. return 0;
  175. }
  176. /*
  177. * Issue a command to the hardware.
  178. *
  179. * Set the appropriate bit in the s_active and Command Issue hardware
  180. * registers, causing hardware command processing to begin.
  181. *
  182. * @port Pointer to the port structure.
  183. * @tag The tag of the command to be issued.
  184. *
  185. * return value
  186. * None
  187. */
  188. static inline void mtip_issue_ncq_command(struct mtip_port *port, int tag)
  189. {
  190. int group = tag >> 5;
  191. /* guard SACT and CI registers */
  192. spin_lock(&port->cmd_issue_lock[group]);
  193. writel((1 << MTIP_TAG_BIT(tag)),
  194. port->s_active[MTIP_TAG_INDEX(tag)]);
  195. writel((1 << MTIP_TAG_BIT(tag)),
  196. port->cmd_issue[MTIP_TAG_INDEX(tag)]);
  197. spin_unlock(&port->cmd_issue_lock[group]);
  198. }
  199. /*
  200. * Enable/disable the reception of FIS
  201. *
  202. * @port Pointer to the port data structure
  203. * @enable 1 to enable, 0 to disable
  204. *
  205. * return value
  206. * Previous state: 1 enabled, 0 disabled
  207. */
  208. static int mtip_enable_fis(struct mtip_port *port, int enable)
  209. {
  210. u32 tmp;
  211. /* enable FIS reception */
  212. tmp = readl(port->mmio + PORT_CMD);
  213. if (enable)
  214. writel(tmp | PORT_CMD_FIS_RX, port->mmio + PORT_CMD);
  215. else
  216. writel(tmp & ~PORT_CMD_FIS_RX, port->mmio + PORT_CMD);
  217. /* Flush */
  218. readl(port->mmio + PORT_CMD);
  219. return (((tmp & PORT_CMD_FIS_RX) == PORT_CMD_FIS_RX));
  220. }
  221. /*
  222. * Enable/disable the DMA engine
  223. *
  224. * @port Pointer to the port data structure
  225. * @enable 1 to enable, 0 to disable
  226. *
  227. * return value
  228. * Previous state: 1 enabled, 0 disabled.
  229. */
  230. static int mtip_enable_engine(struct mtip_port *port, int enable)
  231. {
  232. u32 tmp;
  233. /* enable FIS reception */
  234. tmp = readl(port->mmio + PORT_CMD);
  235. if (enable)
  236. writel(tmp | PORT_CMD_START, port->mmio + PORT_CMD);
  237. else
  238. writel(tmp & ~PORT_CMD_START, port->mmio + PORT_CMD);
  239. readl(port->mmio + PORT_CMD);
  240. return (((tmp & PORT_CMD_START) == PORT_CMD_START));
  241. }
  242. /*
  243. * Enables the port DMA engine and FIS reception.
  244. *
  245. * return value
  246. * None
  247. */
  248. static inline void mtip_start_port(struct mtip_port *port)
  249. {
  250. /* Enable FIS reception */
  251. mtip_enable_fis(port, 1);
  252. /* Enable the DMA engine */
  253. mtip_enable_engine(port, 1);
  254. }
  255. /*
  256. * Deinitialize a port by disabling port interrupts, the DMA engine,
  257. * and FIS reception.
  258. *
  259. * @port Pointer to the port structure
  260. *
  261. * return value
  262. * None
  263. */
  264. static inline void mtip_deinit_port(struct mtip_port *port)
  265. {
  266. /* Disable interrupts on this port */
  267. writel(0, port->mmio + PORT_IRQ_MASK);
  268. /* Disable the DMA engine */
  269. mtip_enable_engine(port, 0);
  270. /* Disable FIS reception */
  271. mtip_enable_fis(port, 0);
  272. }
  273. /*
  274. * Initialize a port.
  275. *
  276. * This function deinitializes the port by calling mtip_deinit_port() and
  277. * then initializes it by setting the command header and RX FIS addresses,
  278. * clearing the SError register and any pending port interrupts before
  279. * re-enabling the default set of port interrupts.
  280. *
  281. * @port Pointer to the port structure.
  282. *
  283. * return value
  284. * None
  285. */
  286. static void mtip_init_port(struct mtip_port *port)
  287. {
  288. int i;
  289. mtip_deinit_port(port);
  290. /* Program the command list base and FIS base addresses */
  291. if (readl(port->dd->mmio + HOST_CAP) & HOST_CAP_64) {
  292. writel((port->command_list_dma >> 16) >> 16,
  293. port->mmio + PORT_LST_ADDR_HI);
  294. writel((port->rxfis_dma >> 16) >> 16,
  295. port->mmio + PORT_FIS_ADDR_HI);
  296. set_bit(MTIP_PF_HOST_CAP_64, &port->flags);
  297. }
  298. writel(port->command_list_dma & 0xFFFFFFFF,
  299. port->mmio + PORT_LST_ADDR);
  300. writel(port->rxfis_dma & 0xFFFFFFFF, port->mmio + PORT_FIS_ADDR);
  301. /* Clear SError */
  302. writel(readl(port->mmio + PORT_SCR_ERR), port->mmio + PORT_SCR_ERR);
  303. /* reset the completed registers.*/
  304. for (i = 0; i < port->dd->slot_groups; i++)
  305. writel(0xFFFFFFFF, port->completed[i]);
  306. /* Clear any pending interrupts for this port */
  307. writel(readl(port->mmio + PORT_IRQ_STAT), port->mmio + PORT_IRQ_STAT);
  308. /* Clear any pending interrupts on the HBA. */
  309. writel(readl(port->dd->mmio + HOST_IRQ_STAT),
  310. port->dd->mmio + HOST_IRQ_STAT);
  311. /* Enable port interrupts */
  312. writel(DEF_PORT_IRQ, port->mmio + PORT_IRQ_MASK);
  313. }
  314. /*
  315. * Restart a port
  316. *
  317. * @port Pointer to the port data structure.
  318. *
  319. * return value
  320. * None
  321. */
  322. static void mtip_restart_port(struct mtip_port *port)
  323. {
  324. unsigned long timeout;
  325. /* Disable the DMA engine */
  326. mtip_enable_engine(port, 0);
  327. /* Chip quirk: wait up to 500ms for PxCMD.CR == 0 */
  328. timeout = jiffies + msecs_to_jiffies(500);
  329. while ((readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON)
  330. && time_before(jiffies, timeout))
  331. ;
  332. if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag))
  333. return;
  334. /*
  335. * Chip quirk: escalate to hba reset if
  336. * PxCMD.CR not clear after 500 ms
  337. */
  338. if (readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) {
  339. dev_warn(&port->dd->pdev->dev,
  340. "PxCMD.CR not clear, escalating reset\n");
  341. if (mtip_hba_reset(port->dd))
  342. dev_err(&port->dd->pdev->dev,
  343. "HBA reset escalation failed.\n");
  344. /* 30 ms delay before com reset to quiesce chip */
  345. mdelay(30);
  346. }
  347. dev_warn(&port->dd->pdev->dev, "Issuing COM reset\n");
  348. /* Set PxSCTL.DET */
  349. writel(readl(port->mmio + PORT_SCR_CTL) |
  350. 1, port->mmio + PORT_SCR_CTL);
  351. readl(port->mmio + PORT_SCR_CTL);
  352. /* Wait 1 ms to quiesce chip function */
  353. timeout = jiffies + msecs_to_jiffies(1);
  354. while (time_before(jiffies, timeout))
  355. ;
  356. if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag))
  357. return;
  358. /* Clear PxSCTL.DET */
  359. writel(readl(port->mmio + PORT_SCR_CTL) & ~1,
  360. port->mmio + PORT_SCR_CTL);
  361. readl(port->mmio + PORT_SCR_CTL);
  362. /* Wait 500 ms for bit 0 of PORT_SCR_STS to be set */
  363. timeout = jiffies + msecs_to_jiffies(500);
  364. while (((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0)
  365. && time_before(jiffies, timeout))
  366. ;
  367. if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag))
  368. return;
  369. if ((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0)
  370. dev_warn(&port->dd->pdev->dev,
  371. "COM reset failed\n");
  372. mtip_init_port(port);
  373. mtip_start_port(port);
  374. }
  375. static int mtip_device_reset(struct driver_data *dd)
  376. {
  377. int rv = 0;
  378. if (mtip_check_surprise_removal(dd->pdev))
  379. return 0;
  380. if (mtip_hba_reset(dd) < 0)
  381. rv = -EFAULT;
  382. mdelay(1);
  383. mtip_init_port(dd->port);
  384. mtip_start_port(dd->port);
  385. /* Enable interrupts on the HBA. */
  386. writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN,
  387. dd->mmio + HOST_CTL);
  388. return rv;
  389. }
  390. /*
  391. * Helper function for tag logging
  392. */
  393. static void print_tags(struct driver_data *dd,
  394. char *msg,
  395. unsigned long *tagbits,
  396. int cnt)
  397. {
  398. unsigned char tagmap[128];
  399. int group, tagmap_len = 0;
  400. memset(tagmap, 0, sizeof(tagmap));
  401. for (group = SLOTBITS_IN_LONGS; group > 0; group--)
  402. tagmap_len += sprintf(tagmap + tagmap_len, "%016lX ",
  403. tagbits[group-1]);
  404. dev_warn(&dd->pdev->dev,
  405. "%d command(s) %s: tagmap [%s]", cnt, msg, tagmap);
  406. }
  407. static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer,
  408. dma_addr_t buffer_dma, unsigned int sectors);
  409. static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id,
  410. struct smart_attr *attrib);
  411. static void mtip_complete_command(struct mtip_cmd *cmd, blk_status_t status)
  412. {
  413. struct request *req = blk_mq_rq_from_pdu(cmd);
  414. cmd->status = status;
  415. if (likely(!blk_should_fake_timeout(req->q)))
  416. blk_mq_complete_request(req);
  417. }
  418. /*
  419. * Handle an error.
  420. *
  421. * @dd Pointer to the DRIVER_DATA structure.
  422. *
  423. * return value
  424. * None
  425. */
  426. static void mtip_handle_tfe(struct driver_data *dd)
  427. {
  428. int group, tag, bit, reissue, rv;
  429. struct mtip_port *port;
  430. struct mtip_cmd *cmd;
  431. u32 completed;
  432. struct host_to_dev_fis *fis;
  433. unsigned long tagaccum[SLOTBITS_IN_LONGS];
  434. unsigned int cmd_cnt = 0;
  435. unsigned char *buf;
  436. char *fail_reason = NULL;
  437. int fail_all_ncq_write = 0, fail_all_ncq_cmds = 0;
  438. dev_warn(&dd->pdev->dev, "Taskfile error\n");
  439. port = dd->port;
  440. if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags)) {
  441. cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL);
  442. dbg_printk(MTIP_DRV_NAME " TFE for the internal command\n");
  443. mtip_complete_command(cmd, BLK_STS_IOERR);
  444. return;
  445. }
  446. /* clear the tag accumulator */
  447. memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long));
  448. /* Loop through all the groups */
  449. for (group = 0; group < dd->slot_groups; group++) {
  450. completed = readl(port->completed[group]);
  451. dev_warn(&dd->pdev->dev, "g=%u, comp=%x\n", group, completed);
  452. /* clear completed status register in the hardware.*/
  453. writel(completed, port->completed[group]);
  454. /* Process successfully completed commands */
  455. for (bit = 0; bit < 32 && completed; bit++) {
  456. if (!(completed & (1<<bit)))
  457. continue;
  458. tag = (group << 5) + bit;
  459. /* Skip the internal command slot */
  460. if (tag == MTIP_TAG_INTERNAL)
  461. continue;
  462. cmd = mtip_cmd_from_tag(dd, tag);
  463. mtip_complete_command(cmd, 0);
  464. set_bit(tag, tagaccum);
  465. cmd_cnt++;
  466. }
  467. }
  468. print_tags(dd, "completed (TFE)", tagaccum, cmd_cnt);
  469. /* Restart the port */
  470. mdelay(20);
  471. mtip_restart_port(port);
  472. /* Trying to determine the cause of the error */
  473. rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ,
  474. dd->port->log_buf,
  475. dd->port->log_buf_dma, 1);
  476. if (rv) {
  477. dev_warn(&dd->pdev->dev,
  478. "Error in READ LOG EXT (10h) command\n");
  479. /* non-critical error, don't fail the load */
  480. } else {
  481. buf = (unsigned char *)dd->port->log_buf;
  482. if (buf[259] & 0x1) {
  483. dev_info(&dd->pdev->dev,
  484. "Write protect bit is set.\n");
  485. set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag);
  486. fail_all_ncq_write = 1;
  487. fail_reason = "write protect";
  488. }
  489. if (buf[288] == 0xF7) {
  490. dev_info(&dd->pdev->dev,
  491. "Exceeded Tmax, drive in thermal shutdown.\n");
  492. set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag);
  493. fail_all_ncq_cmds = 1;
  494. fail_reason = "thermal shutdown";
  495. }
  496. if (buf[288] == 0xBF) {
  497. set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag);
  498. dev_info(&dd->pdev->dev,
  499. "Drive indicates rebuild has failed. Secure erase required.\n");
  500. fail_all_ncq_cmds = 1;
  501. fail_reason = "rebuild failed";
  502. }
  503. }
  504. /* clear the tag accumulator */
  505. memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long));
  506. /* Loop through all the groups */
  507. for (group = 0; group < dd->slot_groups; group++) {
  508. for (bit = 0; bit < 32; bit++) {
  509. reissue = 1;
  510. tag = (group << 5) + bit;
  511. cmd = mtip_cmd_from_tag(dd, tag);
  512. fis = (struct host_to_dev_fis *)cmd->command;
  513. /* Should re-issue? */
  514. if (tag == MTIP_TAG_INTERNAL ||
  515. fis->command == ATA_CMD_SET_FEATURES)
  516. reissue = 0;
  517. else {
  518. if (fail_all_ncq_cmds ||
  519. (fail_all_ncq_write &&
  520. fis->command == ATA_CMD_FPDMA_WRITE)) {
  521. dev_warn(&dd->pdev->dev,
  522. " Fail: %s w/tag %d [%s].\n",
  523. fis->command == ATA_CMD_FPDMA_WRITE ?
  524. "write" : "read",
  525. tag,
  526. fail_reason != NULL ?
  527. fail_reason : "unknown");
  528. mtip_complete_command(cmd, BLK_STS_MEDIUM);
  529. continue;
  530. }
  531. }
  532. /*
  533. * First check if this command has
  534. * exceeded its retries.
  535. */
  536. if (reissue && (cmd->retries-- > 0)) {
  537. set_bit(tag, tagaccum);
  538. /* Re-issue the command. */
  539. mtip_issue_ncq_command(port, tag);
  540. continue;
  541. }
  542. /* Retire a command that will not be reissued */
  543. dev_warn(&port->dd->pdev->dev,
  544. "retiring tag %d\n", tag);
  545. mtip_complete_command(cmd, BLK_STS_IOERR);
  546. }
  547. }
  548. print_tags(dd, "reissued (TFE)", tagaccum, cmd_cnt);
  549. }
  550. /*
  551. * Handle a set device bits interrupt
  552. */
  553. static inline void mtip_workq_sdbfx(struct mtip_port *port, int group,
  554. u32 completed)
  555. {
  556. struct driver_data *dd = port->dd;
  557. int tag, bit;
  558. struct mtip_cmd *command;
  559. if (!completed) {
  560. WARN_ON_ONCE(!completed);
  561. return;
  562. }
  563. /* clear completed status register in the hardware.*/
  564. writel(completed, port->completed[group]);
  565. /* Process completed commands. */
  566. for (bit = 0; (bit < 32) && completed; bit++) {
  567. if (completed & 0x01) {
  568. tag = (group << 5) | bit;
  569. /* skip internal command slot. */
  570. if (unlikely(tag == MTIP_TAG_INTERNAL))
  571. continue;
  572. command = mtip_cmd_from_tag(dd, tag);
  573. mtip_complete_command(command, 0);
  574. }
  575. completed >>= 1;
  576. }
  577. /* If last, re-enable interrupts */
  578. if (atomic_dec_return(&dd->irq_workers_active) == 0)
  579. writel(0xffffffff, dd->mmio + HOST_IRQ_STAT);
  580. }
  581. /*
  582. * Process legacy pio and d2h interrupts
  583. */
  584. static inline void mtip_process_legacy(struct driver_data *dd, u32 port_stat)
  585. {
  586. struct mtip_port *port = dd->port;
  587. struct mtip_cmd *cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL);
  588. if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags) && cmd) {
  589. int group = MTIP_TAG_INDEX(MTIP_TAG_INTERNAL);
  590. int status = readl(port->cmd_issue[group]);
  591. if (!(status & (1 << MTIP_TAG_BIT(MTIP_TAG_INTERNAL))))
  592. mtip_complete_command(cmd, 0);
  593. }
  594. }
  595. /*
  596. * Demux and handle errors
  597. */
  598. static inline void mtip_process_errors(struct driver_data *dd, u32 port_stat)
  599. {
  600. if (unlikely(port_stat & PORT_IRQ_CONNECT)) {
  601. dev_warn(&dd->pdev->dev,
  602. "Clearing PxSERR.DIAG.x\n");
  603. writel((1 << 26), dd->port->mmio + PORT_SCR_ERR);
  604. }
  605. if (unlikely(port_stat & PORT_IRQ_PHYRDY)) {
  606. dev_warn(&dd->pdev->dev,
  607. "Clearing PxSERR.DIAG.n\n");
  608. writel((1 << 16), dd->port->mmio + PORT_SCR_ERR);
  609. }
  610. if (unlikely(port_stat & ~PORT_IRQ_HANDLED)) {
  611. dev_warn(&dd->pdev->dev,
  612. "Port stat errors %x unhandled\n",
  613. (port_stat & ~PORT_IRQ_HANDLED));
  614. if (mtip_check_surprise_removal(dd->pdev))
  615. return;
  616. }
  617. if (likely(port_stat & (PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR))) {
  618. set_bit(MTIP_PF_EH_ACTIVE_BIT, &dd->port->flags);
  619. wake_up_interruptible(&dd->port->svc_wait);
  620. }
  621. }
  622. static inline irqreturn_t mtip_handle_irq(struct driver_data *data)
  623. {
  624. struct driver_data *dd = (struct driver_data *) data;
  625. struct mtip_port *port = dd->port;
  626. u32 hba_stat, port_stat;
  627. int rv = IRQ_NONE;
  628. int do_irq_enable = 1, i, workers;
  629. struct mtip_work *twork;
  630. hba_stat = readl(dd->mmio + HOST_IRQ_STAT);
  631. if (hba_stat) {
  632. rv = IRQ_HANDLED;
  633. /* Acknowledge the interrupt status on the port.*/
  634. port_stat = readl(port->mmio + PORT_IRQ_STAT);
  635. if (unlikely(port_stat == 0xFFFFFFFF)) {
  636. mtip_check_surprise_removal(dd->pdev);
  637. return IRQ_HANDLED;
  638. }
  639. writel(port_stat, port->mmio + PORT_IRQ_STAT);
  640. /* Demux port status */
  641. if (likely(port_stat & PORT_IRQ_SDB_FIS)) {
  642. do_irq_enable = 0;
  643. WARN_ON_ONCE(atomic_read(&dd->irq_workers_active) != 0);
  644. /* Start at 1: group zero is always local? */
  645. for (i = 0, workers = 0; i < MTIP_MAX_SLOT_GROUPS;
  646. i++) {
  647. twork = &dd->work[i];
  648. twork->completed = readl(port->completed[i]);
  649. if (twork->completed)
  650. workers++;
  651. }
  652. atomic_set(&dd->irq_workers_active, workers);
  653. if (workers) {
  654. for (i = 1; i < MTIP_MAX_SLOT_GROUPS; i++) {
  655. twork = &dd->work[i];
  656. if (twork->completed)
  657. queue_work_on(
  658. twork->cpu_binding,
  659. dd->isr_workq,
  660. &twork->work);
  661. }
  662. if (likely(dd->work[0].completed))
  663. mtip_workq_sdbfx(port, 0,
  664. dd->work[0].completed);
  665. } else {
  666. /*
  667. * Chip quirk: SDB interrupt but nothing
  668. * to complete
  669. */
  670. do_irq_enable = 1;
  671. }
  672. }
  673. if (unlikely(port_stat & PORT_IRQ_ERR)) {
  674. if (unlikely(mtip_check_surprise_removal(dd->pdev))) {
  675. /* don't proceed further */
  676. return IRQ_HANDLED;
  677. }
  678. if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
  679. &dd->dd_flag))
  680. return rv;
  681. mtip_process_errors(dd, port_stat & PORT_IRQ_ERR);
  682. }
  683. if (unlikely(port_stat & PORT_IRQ_LEGACY))
  684. mtip_process_legacy(dd, port_stat & PORT_IRQ_LEGACY);
  685. }
  686. /* acknowledge interrupt */
  687. if (unlikely(do_irq_enable))
  688. writel(hba_stat, dd->mmio + HOST_IRQ_STAT);
  689. return rv;
  690. }
  691. /*
  692. * HBA interrupt subroutine.
  693. *
  694. * @irq IRQ number.
  695. * @instance Pointer to the driver data structure.
  696. *
  697. * return value
  698. * IRQ_HANDLED A HBA interrupt was pending and handled.
  699. * IRQ_NONE This interrupt was not for the HBA.
  700. */
  701. static irqreturn_t mtip_irq_handler(int irq, void *instance)
  702. {
  703. struct driver_data *dd = instance;
  704. return mtip_handle_irq(dd);
  705. }
  706. static void mtip_issue_non_ncq_command(struct mtip_port *port, int tag)
  707. {
  708. writel(1 << MTIP_TAG_BIT(tag), port->cmd_issue[MTIP_TAG_INDEX(tag)]);
  709. }
  710. static bool mtip_pause_ncq(struct mtip_port *port,
  711. struct host_to_dev_fis *fis)
  712. {
  713. unsigned long task_file_data;
  714. task_file_data = readl(port->mmio+PORT_TFDATA);
  715. if ((task_file_data & 1))
  716. return false;
  717. if (fis->command == ATA_CMD_SEC_ERASE_PREP) {
  718. port->ic_pause_timer = jiffies;
  719. return true;
  720. } else if ((fis->command == ATA_CMD_DOWNLOAD_MICRO) &&
  721. (fis->features == 0x03)) {
  722. set_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags);
  723. port->ic_pause_timer = jiffies;
  724. return true;
  725. } else if ((fis->command == ATA_CMD_SEC_ERASE_UNIT) ||
  726. ((fis->command == 0xFC) &&
  727. (fis->features == 0x27 || fis->features == 0x72 ||
  728. fis->features == 0x62 || fis->features == 0x26))) {
  729. clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag);
  730. clear_bit(MTIP_DDF_REBUILD_FAILED_BIT, &port->dd->dd_flag);
  731. /* Com reset after secure erase or lowlevel format */
  732. mtip_restart_port(port);
  733. clear_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags);
  734. return false;
  735. }
  736. return false;
  737. }
  738. static bool mtip_commands_active(struct mtip_port *port)
  739. {
  740. unsigned int active;
  741. unsigned int n;
  742. /*
  743. * Ignore s_active bit 0 of array element 0.
  744. * This bit will always be set
  745. */
  746. active = readl(port->s_active[0]) & 0xFFFFFFFE;
  747. for (n = 1; n < port->dd->slot_groups; n++)
  748. active |= readl(port->s_active[n]);
  749. return active != 0;
  750. }
  751. /*
  752. * Wait for port to quiesce
  753. *
  754. * @port Pointer to port data structure
  755. * @timeout Max duration to wait (ms)
  756. *
  757. * return value
  758. * 0 Success
  759. * -EBUSY Commands still active
  760. */
  761. static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout)
  762. {
  763. unsigned long to;
  764. bool active = true;
  765. blk_mq_quiesce_queue(port->dd->queue);
  766. to = jiffies + msecs_to_jiffies(timeout);
  767. do {
  768. if (test_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags) &&
  769. test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) {
  770. msleep(20);
  771. continue; /* svc thd is actively issuing commands */
  772. }
  773. msleep(100);
  774. if (mtip_check_surprise_removal(port->dd->pdev))
  775. goto err_fault;
  776. active = mtip_commands_active(port);
  777. if (!active)
  778. break;
  779. } while (time_before(jiffies, to));
  780. blk_mq_unquiesce_queue(port->dd->queue);
  781. return active ? -EBUSY : 0;
  782. err_fault:
  783. blk_mq_unquiesce_queue(port->dd->queue);
  784. return -EFAULT;
  785. }
  786. struct mtip_int_cmd {
  787. int fis_len;
  788. dma_addr_t buffer;
  789. int buf_len;
  790. u32 opts;
  791. };
  792. /*
  793. * Execute an internal command and wait for the completion.
  794. *
  795. * @port Pointer to the port data structure.
  796. * @fis Pointer to the FIS that describes the command.
  797. * @fis_len Length in WORDS of the FIS.
  798. * @buffer DMA accessible for command data.
  799. * @buf_len Length, in bytes, of the data buffer.
  800. * @opts Command header options, excluding the FIS length
  801. * and the number of PRD entries.
  802. * @timeout Time in ms to wait for the command to complete.
  803. *
  804. * return value
  805. * 0 Command completed successfully.
  806. * -EFAULT The buffer address is not correctly aligned.
  807. * -EBUSY Internal command or other IO in progress.
  808. * -EAGAIN Time out waiting for command to complete.
  809. */
  810. static int mtip_exec_internal_command(struct mtip_port *port,
  811. struct host_to_dev_fis *fis,
  812. int fis_len,
  813. dma_addr_t buffer,
  814. int buf_len,
  815. u32 opts,
  816. unsigned long timeout)
  817. {
  818. struct mtip_cmd *int_cmd;
  819. struct driver_data *dd = port->dd;
  820. struct request *rq;
  821. struct mtip_int_cmd icmd = {
  822. .fis_len = fis_len,
  823. .buffer = buffer,
  824. .buf_len = buf_len,
  825. .opts = opts
  826. };
  827. int rv = 0;
  828. /* Make sure the buffer is 8 byte aligned. This is asic specific. */
  829. if (buffer & 0x00000007) {
  830. dev_err(&dd->pdev->dev, "SG buffer is not 8 byte aligned\n");
  831. return -EFAULT;
  832. }
  833. if (mtip_check_surprise_removal(dd->pdev))
  834. return -EFAULT;
  835. rq = blk_mq_alloc_request(dd->queue, REQ_OP_DRV_IN, BLK_MQ_REQ_RESERVED);
  836. if (IS_ERR(rq)) {
  837. dbg_printk(MTIP_DRV_NAME "Unable to allocate tag for PIO cmd\n");
  838. return -EFAULT;
  839. }
  840. set_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags);
  841. if (fis->command == ATA_CMD_SEC_ERASE_PREP)
  842. set_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags);
  843. clear_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags);
  844. if (fis->command != ATA_CMD_STANDBYNOW1) {
  845. /* wait for io to complete if non atomic */
  846. if (mtip_quiesce_io(port, MTIP_QUIESCE_IO_TIMEOUT_MS) < 0) {
  847. dev_warn(&dd->pdev->dev, "Failed to quiesce IO\n");
  848. blk_mq_free_request(rq);
  849. clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags);
  850. wake_up_interruptible(&port->svc_wait);
  851. return -EBUSY;
  852. }
  853. }
  854. /* Copy the command to the command table */
  855. int_cmd = blk_mq_rq_to_pdu(rq);
  856. int_cmd->icmd = &icmd;
  857. memcpy(int_cmd->command, fis, fis_len*4);
  858. rq->timeout = timeout;
  859. /* insert request and run queue */
  860. blk_execute_rq(rq->q, NULL, rq, true);
  861. if (int_cmd->status) {
  862. dev_err(&dd->pdev->dev, "Internal command [%02X] failed %d\n",
  863. fis->command, int_cmd->status);
  864. rv = -EIO;
  865. if (mtip_check_surprise_removal(dd->pdev) ||
  866. test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
  867. &dd->dd_flag)) {
  868. dev_err(&dd->pdev->dev,
  869. "Internal command [%02X] wait returned due to SR\n",
  870. fis->command);
  871. rv = -ENXIO;
  872. goto exec_ic_exit;
  873. }
  874. mtip_device_reset(dd); /* recover from timeout issue */
  875. rv = -EAGAIN;
  876. goto exec_ic_exit;
  877. }
  878. if (readl(port->cmd_issue[MTIP_TAG_INDEX(MTIP_TAG_INTERNAL)])
  879. & (1 << MTIP_TAG_BIT(MTIP_TAG_INTERNAL))) {
  880. rv = -ENXIO;
  881. if (!test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) {
  882. mtip_device_reset(dd);
  883. rv = -EAGAIN;
  884. }
  885. }
  886. exec_ic_exit:
  887. /* Clear the allocated and active bits for the internal command. */
  888. blk_mq_free_request(rq);
  889. clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags);
  890. if (rv >= 0 && mtip_pause_ncq(port, fis)) {
  891. /* NCQ paused */
  892. return rv;
  893. }
  894. wake_up_interruptible(&port->svc_wait);
  895. return rv;
  896. }
  897. /*
  898. * Byte-swap ATA ID strings.
  899. *
  900. * ATA identify data contains strings in byte-swapped 16-bit words.
  901. * They must be swapped (on all architectures) to be usable as C strings.
  902. * This function swaps bytes in-place.
  903. *
  904. * @buf The buffer location of the string
  905. * @len The number of bytes to swap
  906. *
  907. * return value
  908. * None
  909. */
  910. static inline void ata_swap_string(u16 *buf, unsigned int len)
  911. {
  912. int i;
  913. for (i = 0; i < (len/2); i++)
  914. be16_to_cpus(&buf[i]);
  915. }
  916. static void mtip_set_timeout(struct driver_data *dd,
  917. struct host_to_dev_fis *fis,
  918. unsigned int *timeout, u8 erasemode)
  919. {
  920. switch (fis->command) {
  921. case ATA_CMD_DOWNLOAD_MICRO:
  922. *timeout = 120000; /* 2 minutes */
  923. break;
  924. case ATA_CMD_SEC_ERASE_UNIT:
  925. case 0xFC:
  926. if (erasemode)
  927. *timeout = ((*(dd->port->identify + 90) * 2) * 60000);
  928. else
  929. *timeout = ((*(dd->port->identify + 89) * 2) * 60000);
  930. break;
  931. case ATA_CMD_STANDBYNOW1:
  932. *timeout = 120000; /* 2 minutes */
  933. break;
  934. case 0xF7:
  935. case 0xFA:
  936. *timeout = 60000; /* 60 seconds */
  937. break;
  938. case ATA_CMD_SMART:
  939. *timeout = 15000; /* 15 seconds */
  940. break;
  941. default:
  942. *timeout = MTIP_IOCTL_CMD_TIMEOUT_MS;
  943. break;
  944. }
  945. }
  946. /*
  947. * Request the device identity information.
  948. *
  949. * If a user space buffer is not specified, i.e. is NULL, the
  950. * identify information is still read from the drive and placed
  951. * into the identify data buffer (@e port->identify) in the
  952. * port data structure.
  953. * When the identify buffer contains valid identify information @e
  954. * port->identify_valid is non-zero.
  955. *
  956. * @port Pointer to the port structure.
  957. * @user_buffer A user space buffer where the identify data should be
  958. * copied.
  959. *
  960. * return value
  961. * 0 Command completed successfully.
  962. * -EFAULT An error occurred while coping data to the user buffer.
  963. * -1 Command failed.
  964. */
  965. static int mtip_get_identify(struct mtip_port *port, void __user *user_buffer)
  966. {
  967. int rv = 0;
  968. struct host_to_dev_fis fis;
  969. if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag))
  970. return -EFAULT;
  971. /* Build the FIS. */
  972. memset(&fis, 0, sizeof(struct host_to_dev_fis));
  973. fis.type = 0x27;
  974. fis.opts = 1 << 7;
  975. fis.command = ATA_CMD_ID_ATA;
  976. /* Set the identify information as invalid. */
  977. port->identify_valid = 0;
  978. /* Clear the identify information. */
  979. memset(port->identify, 0, sizeof(u16) * ATA_ID_WORDS);
  980. /* Execute the command. */
  981. if (mtip_exec_internal_command(port,
  982. &fis,
  983. 5,
  984. port->identify_dma,
  985. sizeof(u16) * ATA_ID_WORDS,
  986. 0,
  987. MTIP_INT_CMD_TIMEOUT_MS)
  988. < 0) {
  989. rv = -1;
  990. goto out;
  991. }
  992. /*
  993. * Perform any necessary byte-swapping. Yes, the kernel does in fact
  994. * perform field-sensitive swapping on the string fields.
  995. * See the kernel use of ata_id_string() for proof of this.
  996. */
  997. #ifdef __LITTLE_ENDIAN
  998. ata_swap_string(port->identify + 27, 40); /* model string*/
  999. ata_swap_string(port->identify + 23, 8); /* firmware string*/
  1000. ata_swap_string(port->identify + 10, 20); /* serial# string*/
  1001. #else
  1002. {
  1003. int i;
  1004. for (i = 0; i < ATA_ID_WORDS; i++)
  1005. port->identify[i] = le16_to_cpu(port->identify[i]);
  1006. }
  1007. #endif
  1008. /* Check security locked state */
  1009. if (port->identify[128] & 0x4)
  1010. set_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag);
  1011. else
  1012. clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag);
  1013. /* Set the identify buffer as valid. */
  1014. port->identify_valid = 1;
  1015. if (user_buffer) {
  1016. if (copy_to_user(
  1017. user_buffer,
  1018. port->identify,
  1019. ATA_ID_WORDS * sizeof(u16))) {
  1020. rv = -EFAULT;
  1021. goto out;
  1022. }
  1023. }
  1024. out:
  1025. return rv;
  1026. }
  1027. /*
  1028. * Issue a standby immediate command to the device.
  1029. *
  1030. * @port Pointer to the port structure.
  1031. *
  1032. * return value
  1033. * 0 Command was executed successfully.
  1034. * -1 An error occurred while executing the command.
  1035. */
  1036. static int mtip_standby_immediate(struct mtip_port *port)
  1037. {
  1038. int rv;
  1039. struct host_to_dev_fis fis;
  1040. unsigned long start;
  1041. unsigned int timeout;
  1042. /* Build the FIS. */
  1043. memset(&fis, 0, sizeof(struct host_to_dev_fis));
  1044. fis.type = 0x27;
  1045. fis.opts = 1 << 7;
  1046. fis.command = ATA_CMD_STANDBYNOW1;
  1047. mtip_set_timeout(port->dd, &fis, &timeout, 0);
  1048. start = jiffies;
  1049. rv = mtip_exec_internal_command(port,
  1050. &fis,
  1051. 5,
  1052. 0,
  1053. 0,
  1054. 0,
  1055. timeout);
  1056. dbg_printk(MTIP_DRV_NAME "Time taken to complete standby cmd: %d ms\n",
  1057. jiffies_to_msecs(jiffies - start));
  1058. if (rv)
  1059. dev_warn(&port->dd->pdev->dev,
  1060. "STANDBY IMMEDIATE command failed.\n");
  1061. return rv;
  1062. }
  1063. /*
  1064. * Issue a READ LOG EXT command to the device.
  1065. *
  1066. * @port pointer to the port structure.
  1067. * @page page number to fetch
  1068. * @buffer pointer to buffer
  1069. * @buffer_dma dma address corresponding to @buffer
  1070. * @sectors page length to fetch, in sectors
  1071. *
  1072. * return value
  1073. * @rv return value from mtip_exec_internal_command()
  1074. */
  1075. static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer,
  1076. dma_addr_t buffer_dma, unsigned int sectors)
  1077. {
  1078. struct host_to_dev_fis fis;
  1079. memset(&fis, 0, sizeof(struct host_to_dev_fis));
  1080. fis.type = 0x27;
  1081. fis.opts = 1 << 7;
  1082. fis.command = ATA_CMD_READ_LOG_EXT;
  1083. fis.sect_count = sectors & 0xFF;
  1084. fis.sect_cnt_ex = (sectors >> 8) & 0xFF;
  1085. fis.lba_low = page;
  1086. fis.lba_mid = 0;
  1087. fis.device = ATA_DEVICE_OBS;
  1088. memset(buffer, 0, sectors * ATA_SECT_SIZE);
  1089. return mtip_exec_internal_command(port,
  1090. &fis,
  1091. 5,
  1092. buffer_dma,
  1093. sectors * ATA_SECT_SIZE,
  1094. 0,
  1095. MTIP_INT_CMD_TIMEOUT_MS);
  1096. }
  1097. /*
  1098. * Issue a SMART READ DATA command to the device.
  1099. *
  1100. * @port pointer to the port structure.
  1101. * @buffer pointer to buffer
  1102. * @buffer_dma dma address corresponding to @buffer
  1103. *
  1104. * return value
  1105. * @rv return value from mtip_exec_internal_command()
  1106. */
  1107. static int mtip_get_smart_data(struct mtip_port *port, u8 *buffer,
  1108. dma_addr_t buffer_dma)
  1109. {
  1110. struct host_to_dev_fis fis;
  1111. memset(&fis, 0, sizeof(struct host_to_dev_fis));
  1112. fis.type = 0x27;
  1113. fis.opts = 1 << 7;
  1114. fis.command = ATA_CMD_SMART;
  1115. fis.features = 0xD0;
  1116. fis.sect_count = 1;
  1117. fis.lba_mid = 0x4F;
  1118. fis.lba_hi = 0xC2;
  1119. fis.device = ATA_DEVICE_OBS;
  1120. return mtip_exec_internal_command(port,
  1121. &fis,
  1122. 5,
  1123. buffer_dma,
  1124. ATA_SECT_SIZE,
  1125. 0,
  1126. 15000);
  1127. }
  1128. /*
  1129. * Get the value of a smart attribute
  1130. *
  1131. * @port pointer to the port structure
  1132. * @id attribute number
  1133. * @attrib pointer to return attrib information corresponding to @id
  1134. *
  1135. * return value
  1136. * -EINVAL NULL buffer passed or unsupported attribute @id.
  1137. * -EPERM Identify data not valid, SMART not supported or not enabled
  1138. */
  1139. static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id,
  1140. struct smart_attr *attrib)
  1141. {
  1142. int rv, i;
  1143. struct smart_attr *pattr;
  1144. if (!attrib)
  1145. return -EINVAL;
  1146. if (!port->identify_valid) {
  1147. dev_warn(&port->dd->pdev->dev, "IDENTIFY DATA not valid\n");
  1148. return -EPERM;
  1149. }
  1150. if (!(port->identify[82] & 0x1)) {
  1151. dev_warn(&port->dd->pdev->dev, "SMART not supported\n");
  1152. return -EPERM;
  1153. }
  1154. if (!(port->identify[85] & 0x1)) {
  1155. dev_warn(&port->dd->pdev->dev, "SMART not enabled\n");
  1156. return -EPERM;
  1157. }
  1158. memset(port->smart_buf, 0, ATA_SECT_SIZE);
  1159. rv = mtip_get_smart_data(port, port->smart_buf, port->smart_buf_dma);
  1160. if (rv) {
  1161. dev_warn(&port->dd->pdev->dev, "Failed to ge SMART data\n");
  1162. return rv;
  1163. }
  1164. pattr = (struct smart_attr *)(port->smart_buf + 2);
  1165. for (i = 0; i < 29; i++, pattr++)
  1166. if (pattr->attr_id == id) {
  1167. memcpy(attrib, pattr, sizeof(struct smart_attr));
  1168. break;
  1169. }
  1170. if (i == 29) {
  1171. dev_warn(&port->dd->pdev->dev,
  1172. "Query for invalid SMART attribute ID\n");
  1173. rv = -EINVAL;
  1174. }
  1175. return rv;
  1176. }
  1177. /*
  1178. * Get the drive capacity.
  1179. *
  1180. * @dd Pointer to the device data structure.
  1181. * @sectors Pointer to the variable that will receive the sector count.
  1182. *
  1183. * return value
  1184. * 1 Capacity was returned successfully.
  1185. * 0 The identify information is invalid.
  1186. */
  1187. static bool mtip_hw_get_capacity(struct driver_data *dd, sector_t *sectors)
  1188. {
  1189. struct mtip_port *port = dd->port;
  1190. u64 total, raw0, raw1, raw2, raw3;
  1191. raw0 = port->identify[100];
  1192. raw1 = port->identify[101];
  1193. raw2 = port->identify[102];
  1194. raw3 = port->identify[103];
  1195. total = raw0 | raw1<<16 | raw2<<32 | raw3<<48;
  1196. *sectors = total;
  1197. return (bool) !!port->identify_valid;
  1198. }
  1199. /*
  1200. * Display the identify command data.
  1201. *
  1202. * @port Pointer to the port data structure.
  1203. *
  1204. * return value
  1205. * None
  1206. */
  1207. static void mtip_dump_identify(struct mtip_port *port)
  1208. {
  1209. sector_t sectors;
  1210. unsigned short revid;
  1211. char cbuf[42];
  1212. if (!port->identify_valid)
  1213. return;
  1214. strlcpy(cbuf, (char *)(port->identify+10), 21);
  1215. dev_info(&port->dd->pdev->dev,
  1216. "Serial No.: %s\n", cbuf);
  1217. strlcpy(cbuf, (char *)(port->identify+23), 9);
  1218. dev_info(&port->dd->pdev->dev,
  1219. "Firmware Ver.: %s\n", cbuf);
  1220. strlcpy(cbuf, (char *)(port->identify+27), 41);
  1221. dev_info(&port->dd->pdev->dev, "Model: %s\n", cbuf);
  1222. dev_info(&port->dd->pdev->dev, "Security: %04x %s\n",
  1223. port->identify[128],
  1224. port->identify[128] & 0x4 ? "(LOCKED)" : "");
  1225. if (mtip_hw_get_capacity(port->dd, &sectors))
  1226. dev_info(&port->dd->pdev->dev,
  1227. "Capacity: %llu sectors (%llu MB)\n",
  1228. (u64)sectors,
  1229. ((u64)sectors) * ATA_SECT_SIZE >> 20);
  1230. pci_read_config_word(port->dd->pdev, PCI_REVISION_ID, &revid);
  1231. switch (revid & 0xFF) {
  1232. case 0x1:
  1233. strlcpy(cbuf, "A0", 3);
  1234. break;
  1235. case 0x3:
  1236. strlcpy(cbuf, "A2", 3);
  1237. break;
  1238. default:
  1239. strlcpy(cbuf, "?", 2);
  1240. break;
  1241. }
  1242. dev_info(&port->dd->pdev->dev,
  1243. "Card Type: %s\n", cbuf);
  1244. }
  1245. /*
  1246. * Map the commands scatter list into the command table.
  1247. *
  1248. * @command Pointer to the command.
  1249. * @nents Number of scatter list entries.
  1250. *
  1251. * return value
  1252. * None
  1253. */
  1254. static inline void fill_command_sg(struct driver_data *dd,
  1255. struct mtip_cmd *command,
  1256. int nents)
  1257. {
  1258. int n;
  1259. unsigned int dma_len;
  1260. struct mtip_cmd_sg *command_sg;
  1261. struct scatterlist *sg;
  1262. command_sg = command->command + AHCI_CMD_TBL_HDR_SZ;
  1263. for_each_sg(command->sg, sg, nents, n) {
  1264. dma_len = sg_dma_len(sg);
  1265. if (dma_len > 0x400000)
  1266. dev_err(&dd->pdev->dev,
  1267. "DMA segment length truncated\n");
  1268. command_sg->info = cpu_to_le32((dma_len-1) & 0x3FFFFF);
  1269. command_sg->dba = cpu_to_le32(sg_dma_address(sg));
  1270. command_sg->dba_upper =
  1271. cpu_to_le32((sg_dma_address(sg) >> 16) >> 16);
  1272. command_sg++;
  1273. }
  1274. }
  1275. /*
  1276. * @brief Execute a drive command.
  1277. *
  1278. * return value 0 The command completed successfully.
  1279. * return value -1 An error occurred while executing the command.
  1280. */
  1281. static int exec_drive_task(struct mtip_port *port, u8 *command)
  1282. {
  1283. struct host_to_dev_fis fis;
  1284. struct host_to_dev_fis *reply = (port->rxfis + RX_FIS_D2H_REG);
  1285. unsigned int to;
  1286. /* Build the FIS. */
  1287. memset(&fis, 0, sizeof(struct host_to_dev_fis));
  1288. fis.type = 0x27;
  1289. fis.opts = 1 << 7;
  1290. fis.command = command[0];
  1291. fis.features = command[1];
  1292. fis.sect_count = command[2];
  1293. fis.sector = command[3];
  1294. fis.cyl_low = command[4];
  1295. fis.cyl_hi = command[5];
  1296. fis.device = command[6] & ~0x10; /* Clear the dev bit*/
  1297. mtip_set_timeout(port->dd, &fis, &to, 0);
  1298. dbg_printk(MTIP_DRV_NAME " %s: User Command: cmd %x, feat %x, nsect %x, sect %x, lcyl %x, hcyl %x, sel %x\n",
  1299. __func__,
  1300. command[0],
  1301. command[1],
  1302. command[2],
  1303. command[3],
  1304. command[4],
  1305. command[5],
  1306. command[6]);
  1307. /* Execute the command. */
  1308. if (mtip_exec_internal_command(port,
  1309. &fis,
  1310. 5,
  1311. 0,
  1312. 0,
  1313. 0,
  1314. to) < 0) {
  1315. return -1;
  1316. }
  1317. command[0] = reply->command; /* Status*/
  1318. command[1] = reply->features; /* Error*/
  1319. command[4] = reply->cyl_low;
  1320. command[5] = reply->cyl_hi;
  1321. dbg_printk(MTIP_DRV_NAME " %s: Completion Status: stat %x, err %x , cyl_lo %x cyl_hi %x\n",
  1322. __func__,
  1323. command[0],
  1324. command[1],
  1325. command[4],
  1326. command[5]);
  1327. return 0;
  1328. }
  1329. /*
  1330. * @brief Execute a drive command.
  1331. *
  1332. * @param port Pointer to the port data structure.
  1333. * @param command Pointer to the user specified command parameters.
  1334. * @param user_buffer Pointer to the user space buffer where read sector
  1335. * data should be copied.
  1336. *
  1337. * return value 0 The command completed successfully.
  1338. * return value -EFAULT An error occurred while copying the completion
  1339. * data to the user space buffer.
  1340. * return value -1 An error occurred while executing the command.
  1341. */
  1342. static int exec_drive_command(struct mtip_port *port, u8 *command,
  1343. void __user *user_buffer)
  1344. {
  1345. struct host_to_dev_fis fis;
  1346. struct host_to_dev_fis *reply;
  1347. u8 *buf = NULL;
  1348. dma_addr_t dma_addr = 0;
  1349. int rv = 0, xfer_sz = command[3];
  1350. unsigned int to;
  1351. if (xfer_sz) {
  1352. if (!user_buffer)
  1353. return -EFAULT;
  1354. buf = dma_alloc_coherent(&port->dd->pdev->dev,
  1355. ATA_SECT_SIZE * xfer_sz,
  1356. &dma_addr,
  1357. GFP_KERNEL);
  1358. if (!buf) {
  1359. dev_err(&port->dd->pdev->dev,
  1360. "Memory allocation failed (%d bytes)\n",
  1361. ATA_SECT_SIZE * xfer_sz);
  1362. return -ENOMEM;
  1363. }
  1364. }
  1365. /* Build the FIS. */
  1366. memset(&fis, 0, sizeof(struct host_to_dev_fis));
  1367. fis.type = 0x27;
  1368. fis.opts = 1 << 7;
  1369. fis.command = command[0];
  1370. fis.features = command[2];
  1371. fis.sect_count = command[3];
  1372. if (fis.command == ATA_CMD_SMART) {
  1373. fis.sector = command[1];
  1374. fis.cyl_low = 0x4F;
  1375. fis.cyl_hi = 0xC2;
  1376. }
  1377. mtip_set_timeout(port->dd, &fis, &to, 0);
  1378. if (xfer_sz)
  1379. reply = (port->rxfis + RX_FIS_PIO_SETUP);
  1380. else
  1381. reply = (port->rxfis + RX_FIS_D2H_REG);
  1382. dbg_printk(MTIP_DRV_NAME
  1383. " %s: User Command: cmd %x, sect %x, "
  1384. "feat %x, sectcnt %x\n",
  1385. __func__,
  1386. command[0],
  1387. command[1],
  1388. command[2],
  1389. command[3]);
  1390. /* Execute the command. */
  1391. if (mtip_exec_internal_command(port,
  1392. &fis,
  1393. 5,
  1394. (xfer_sz ? dma_addr : 0),
  1395. (xfer_sz ? ATA_SECT_SIZE * xfer_sz : 0),
  1396. 0,
  1397. to)
  1398. < 0) {
  1399. rv = -EFAULT;
  1400. goto exit_drive_command;
  1401. }
  1402. /* Collect the completion status. */
  1403. command[0] = reply->command; /* Status*/
  1404. command[1] = reply->features; /* Error*/
  1405. command[2] = reply->sect_count;
  1406. dbg_printk(MTIP_DRV_NAME
  1407. " %s: Completion Status: stat %x, "
  1408. "err %x, nsect %x\n",
  1409. __func__,
  1410. command[0],
  1411. command[1],
  1412. command[2]);
  1413. if (xfer_sz) {
  1414. if (copy_to_user(user_buffer,
  1415. buf,
  1416. ATA_SECT_SIZE * command[3])) {
  1417. rv = -EFAULT;
  1418. goto exit_drive_command;
  1419. }
  1420. }
  1421. exit_drive_command:
  1422. if (buf)
  1423. dma_free_coherent(&port->dd->pdev->dev,
  1424. ATA_SECT_SIZE * xfer_sz, buf, dma_addr);
  1425. return rv;
  1426. }
  1427. /*
  1428. * Indicates whether a command has a single sector payload.
  1429. *
  1430. * @command passed to the device to perform the certain event.
  1431. * @features passed to the device to perform the certain event.
  1432. *
  1433. * return value
  1434. * 1 command is one that always has a single sector payload,
  1435. * regardless of the value in the Sector Count field.
  1436. * 0 otherwise
  1437. *
  1438. */
  1439. static unsigned int implicit_sector(unsigned char command,
  1440. unsigned char features)
  1441. {
  1442. unsigned int rv = 0;
  1443. /* list of commands that have an implicit sector count of 1 */
  1444. switch (command) {
  1445. case ATA_CMD_SEC_SET_PASS:
  1446. case ATA_CMD_SEC_UNLOCK:
  1447. case ATA_CMD_SEC_ERASE_PREP:
  1448. case ATA_CMD_SEC_ERASE_UNIT:
  1449. case ATA_CMD_SEC_FREEZE_LOCK:
  1450. case ATA_CMD_SEC_DISABLE_PASS:
  1451. case ATA_CMD_PMP_READ:
  1452. case ATA_CMD_PMP_WRITE:
  1453. rv = 1;
  1454. break;
  1455. case ATA_CMD_SET_MAX:
  1456. if (features == ATA_SET_MAX_UNLOCK)
  1457. rv = 1;
  1458. break;
  1459. case ATA_CMD_SMART:
  1460. if ((features == ATA_SMART_READ_VALUES) ||
  1461. (features == ATA_SMART_READ_THRESHOLDS))
  1462. rv = 1;
  1463. break;
  1464. case ATA_CMD_CONF_OVERLAY:
  1465. if ((features == ATA_DCO_IDENTIFY) ||
  1466. (features == ATA_DCO_SET))
  1467. rv = 1;
  1468. break;
  1469. }
  1470. return rv;
  1471. }
  1472. /*
  1473. * Executes a taskfile
  1474. * See ide_taskfile_ioctl() for derivation
  1475. */
  1476. static int exec_drive_taskfile(struct driver_data *dd,
  1477. void __user *buf,
  1478. ide_task_request_t *req_task,
  1479. int outtotal)
  1480. {
  1481. struct host_to_dev_fis fis;
  1482. struct host_to_dev_fis *reply;
  1483. u8 *outbuf = NULL;
  1484. u8 *inbuf = NULL;
  1485. dma_addr_t outbuf_dma = 0;
  1486. dma_addr_t inbuf_dma = 0;
  1487. dma_addr_t dma_buffer = 0;
  1488. int err = 0;
  1489. unsigned int taskin = 0;
  1490. unsigned int taskout = 0;
  1491. u8 nsect = 0;
  1492. unsigned int timeout;
  1493. unsigned int force_single_sector;
  1494. unsigned int transfer_size;
  1495. unsigned long task_file_data;
  1496. int intotal = outtotal + req_task->out_size;
  1497. int erasemode = 0;
  1498. taskout = req_task->out_size;
  1499. taskin = req_task->in_size;
  1500. /* 130560 = 512 * 0xFF*/
  1501. if (taskin > 130560 || taskout > 130560)
  1502. return -EINVAL;
  1503. if (taskout) {
  1504. outbuf = memdup_user(buf + outtotal, taskout);
  1505. if (IS_ERR(outbuf))
  1506. return PTR_ERR(outbuf);
  1507. outbuf_dma = dma_map_single(&dd->pdev->dev, outbuf,
  1508. taskout, DMA_TO_DEVICE);
  1509. if (dma_mapping_error(&dd->pdev->dev, outbuf_dma)) {
  1510. err = -ENOMEM;
  1511. goto abort;
  1512. }
  1513. dma_buffer = outbuf_dma;
  1514. }
  1515. if (taskin) {
  1516. inbuf = memdup_user(buf + intotal, taskin);
  1517. if (IS_ERR(inbuf)) {
  1518. err = PTR_ERR(inbuf);
  1519. inbuf = NULL;
  1520. goto abort;
  1521. }
  1522. inbuf_dma = dma_map_single(&dd->pdev->dev, inbuf,
  1523. taskin, DMA_FROM_DEVICE);
  1524. if (dma_mapping_error(&dd->pdev->dev, inbuf_dma)) {
  1525. err = -ENOMEM;
  1526. goto abort;
  1527. }
  1528. dma_buffer = inbuf_dma;
  1529. }
  1530. /* only supports PIO and non-data commands from this ioctl. */
  1531. switch (req_task->data_phase) {
  1532. case TASKFILE_OUT:
  1533. nsect = taskout / ATA_SECT_SIZE;
  1534. reply = (dd->port->rxfis + RX_FIS_PIO_SETUP);
  1535. break;
  1536. case TASKFILE_IN:
  1537. reply = (dd->port->rxfis + RX_FIS_PIO_SETUP);
  1538. break;
  1539. case TASKFILE_NO_DATA:
  1540. reply = (dd->port->rxfis + RX_FIS_D2H_REG);
  1541. break;
  1542. default:
  1543. err = -EINVAL;
  1544. goto abort;
  1545. }
  1546. /* Build the FIS. */
  1547. memset(&fis, 0, sizeof(struct host_to_dev_fis));
  1548. fis.type = 0x27;
  1549. fis.opts = 1 << 7;
  1550. fis.command = req_task->io_ports[7];
  1551. fis.features = req_task->io_ports[1];
  1552. fis.sect_count = req_task->io_ports[2];
  1553. fis.lba_low = req_task->io_ports[3];
  1554. fis.lba_mid = req_task->io_ports[4];
  1555. fis.lba_hi = req_task->io_ports[5];
  1556. /* Clear the dev bit*/
  1557. fis.device = req_task->io_ports[6] & ~0x10;
  1558. if ((req_task->in_flags.all == 0) && (req_task->out_flags.all & 1)) {
  1559. req_task->in_flags.all =
  1560. IDE_TASKFILE_STD_IN_FLAGS |
  1561. (IDE_HOB_STD_IN_FLAGS << 8);
  1562. fis.lba_low_ex = req_task->hob_ports[3];
  1563. fis.lba_mid_ex = req_task->hob_ports[4];
  1564. fis.lba_hi_ex = req_task->hob_ports[5];
  1565. fis.features_ex = req_task->hob_ports[1];
  1566. fis.sect_cnt_ex = req_task->hob_ports[2];
  1567. } else {
  1568. req_task->in_flags.all = IDE_TASKFILE_STD_IN_FLAGS;
  1569. }
  1570. force_single_sector = implicit_sector(fis.command, fis.features);
  1571. if ((taskin || taskout) && (!fis.sect_count)) {
  1572. if (nsect)
  1573. fis.sect_count = nsect;
  1574. else {
  1575. if (!force_single_sector) {
  1576. dev_warn(&dd->pdev->dev,
  1577. "data movement but "
  1578. "sect_count is 0\n");
  1579. err = -EINVAL;
  1580. goto abort;
  1581. }
  1582. }
  1583. }
  1584. dbg_printk(MTIP_DRV_NAME
  1585. " %s: cmd %x, feat %x, nsect %x,"
  1586. " sect/lbal %x, lcyl/lbam %x, hcyl/lbah %x,"
  1587. " head/dev %x\n",
  1588. __func__,
  1589. fis.command,
  1590. fis.features,
  1591. fis.sect_count,
  1592. fis.lba_low,
  1593. fis.lba_mid,
  1594. fis.lba_hi,
  1595. fis.device);
  1596. /* check for erase mode support during secure erase.*/
  1597. if ((fis.command == ATA_CMD_SEC_ERASE_UNIT) && outbuf &&
  1598. (outbuf[0] & MTIP_SEC_ERASE_MODE)) {
  1599. erasemode = 1;
  1600. }
  1601. mtip_set_timeout(dd, &fis, &timeout, erasemode);
  1602. /* Determine the correct transfer size.*/
  1603. if (force_single_sector)
  1604. transfer_size = ATA_SECT_SIZE;
  1605. else
  1606. transfer_size = ATA_SECT_SIZE * fis.sect_count;
  1607. /* Execute the command.*/
  1608. if (mtip_exec_internal_command(dd->port,
  1609. &fis,
  1610. 5,
  1611. dma_buffer,
  1612. transfer_size,
  1613. 0,
  1614. timeout) < 0) {
  1615. err = -EIO;
  1616. goto abort;
  1617. }
  1618. task_file_data = readl(dd->port->mmio+PORT_TFDATA);
  1619. if ((req_task->data_phase == TASKFILE_IN) && !(task_file_data & 1)) {
  1620. reply = dd->port->rxfis + RX_FIS_PIO_SETUP;
  1621. req_task->io_ports[7] = reply->control;
  1622. } else {
  1623. reply = dd->port->rxfis + RX_FIS_D2H_REG;
  1624. req_task->io_ports[7] = reply->command;
  1625. }
  1626. /* reclaim the DMA buffers.*/
  1627. if (inbuf_dma)
  1628. dma_unmap_single(&dd->pdev->dev, inbuf_dma, taskin,
  1629. DMA_FROM_DEVICE);
  1630. if (outbuf_dma)
  1631. dma_unmap_single(&dd->pdev->dev, outbuf_dma, taskout,
  1632. DMA_TO_DEVICE);
  1633. inbuf_dma = 0;
  1634. outbuf_dma = 0;
  1635. /* return the ATA registers to the caller.*/
  1636. req_task->io_ports[1] = reply->features;
  1637. req_task->io_ports[2] = reply->sect_count;
  1638. req_task->io_ports[3] = reply->lba_low;
  1639. req_task->io_ports[4] = reply->lba_mid;
  1640. req_task->io_ports[5] = reply->lba_hi;
  1641. req_task->io_ports[6] = reply->device;
  1642. if (req_task->out_flags.all & 1) {
  1643. req_task->hob_ports[3] = reply->lba_low_ex;
  1644. req_task->hob_ports[4] = reply->lba_mid_ex;
  1645. req_task->hob_ports[5] = reply->lba_hi_ex;
  1646. req_task->hob_ports[1] = reply->features_ex;
  1647. req_task->hob_ports[2] = reply->sect_cnt_ex;
  1648. }
  1649. dbg_printk(MTIP_DRV_NAME
  1650. " %s: Completion: stat %x,"
  1651. "err %x, sect_cnt %x, lbalo %x,"
  1652. "lbamid %x, lbahi %x, dev %x\n",
  1653. __func__,
  1654. req_task->io_ports[7],
  1655. req_task->io_ports[1],
  1656. req_task->io_ports[2],
  1657. req_task->io_ports[3],
  1658. req_task->io_ports[4],
  1659. req_task->io_ports[5],
  1660. req_task->io_ports[6]);
  1661. if (taskout) {
  1662. if (copy_to_user(buf + outtotal, outbuf, taskout)) {
  1663. err = -EFAULT;
  1664. goto abort;
  1665. }
  1666. }
  1667. if (taskin) {
  1668. if (copy_to_user(buf + intotal, inbuf, taskin)) {
  1669. err = -EFAULT;
  1670. goto abort;
  1671. }
  1672. }
  1673. abort:
  1674. if (inbuf_dma)
  1675. dma_unmap_single(&dd->pdev->dev, inbuf_dma, taskin,
  1676. DMA_FROM_DEVICE);
  1677. if (outbuf_dma)
  1678. dma_unmap_single(&dd->pdev->dev, outbuf_dma, taskout,
  1679. DMA_TO_DEVICE);
  1680. kfree(outbuf);
  1681. kfree(inbuf);
  1682. return err;
  1683. }
  1684. /*
  1685. * Handle IOCTL calls from the Block Layer.
  1686. *
  1687. * This function is called by the Block Layer when it receives an IOCTL
  1688. * command that it does not understand. If the IOCTL command is not supported
  1689. * this function returns -ENOTTY.
  1690. *
  1691. * @dd Pointer to the driver data structure.
  1692. * @cmd IOCTL command passed from the Block Layer.
  1693. * @arg IOCTL argument passed from the Block Layer.
  1694. *
  1695. * return value
  1696. * 0 The IOCTL completed successfully.
  1697. * -ENOTTY The specified command is not supported.
  1698. * -EFAULT An error occurred copying data to a user space buffer.
  1699. * -EIO An error occurred while executing the command.
  1700. */
  1701. static int mtip_hw_ioctl(struct driver_data *dd, unsigned int cmd,
  1702. unsigned long arg)
  1703. {
  1704. switch (cmd) {
  1705. case HDIO_GET_IDENTITY:
  1706. {
  1707. if (copy_to_user((void __user *)arg, dd->port->identify,
  1708. sizeof(u16) * ATA_ID_WORDS))
  1709. return -EFAULT;
  1710. break;
  1711. }
  1712. case HDIO_DRIVE_CMD:
  1713. {
  1714. u8 drive_command[4];
  1715. /* Copy the user command info to our buffer. */
  1716. if (copy_from_user(drive_command,
  1717. (void __user *) arg,
  1718. sizeof(drive_command)))
  1719. return -EFAULT;
  1720. /* Execute the drive command. */
  1721. if (exec_drive_command(dd->port,
  1722. drive_command,
  1723. (void __user *) (arg+4)))
  1724. return -EIO;
  1725. /* Copy the status back to the users buffer. */
  1726. if (copy_to_user((void __user *) arg,
  1727. drive_command,
  1728. sizeof(drive_command)))
  1729. return -EFAULT;
  1730. break;
  1731. }
  1732. case HDIO_DRIVE_TASK:
  1733. {
  1734. u8 drive_command[7];
  1735. /* Copy the user command info to our buffer. */
  1736. if (copy_from_user(drive_command,
  1737. (void __user *) arg,
  1738. sizeof(drive_command)))
  1739. return -EFAULT;
  1740. /* Execute the drive command. */
  1741. if (exec_drive_task(dd->port, drive_command))
  1742. return -EIO;
  1743. /* Copy the status back to the users buffer. */
  1744. if (copy_to_user((void __user *) arg,
  1745. drive_command,
  1746. sizeof(drive_command)))
  1747. return -EFAULT;
  1748. break;
  1749. }
  1750. case HDIO_DRIVE_TASKFILE: {
  1751. ide_task_request_t req_task;
  1752. int ret, outtotal;
  1753. if (copy_from_user(&req_task, (void __user *) arg,
  1754. sizeof(req_task)))
  1755. return -EFAULT;
  1756. outtotal = sizeof(req_task);
  1757. ret = exec_drive_taskfile(dd, (void __user *) arg,
  1758. &req_task, outtotal);
  1759. if (copy_to_user((void __user *) arg, &req_task,
  1760. sizeof(req_task)))
  1761. return -EFAULT;
  1762. return ret;
  1763. }
  1764. default:
  1765. return -EINVAL;
  1766. }
  1767. return 0;
  1768. }
  1769. /*
  1770. * Submit an IO to the hw
  1771. *
  1772. * This function is called by the block layer to issue an io
  1773. * to the device. Upon completion, the callback function will
  1774. * be called with the data parameter passed as the callback data.
  1775. *
  1776. * @dd Pointer to the driver data structure.
  1777. * @start First sector to read.
  1778. * @nsect Number of sectors to read.
  1779. * @tag The tag of this read command.
  1780. * @callback Pointer to the function that should be called
  1781. * when the read completes.
  1782. * @data Callback data passed to the callback function
  1783. * when the read completes.
  1784. * @dir Direction (read or write)
  1785. *
  1786. * return value
  1787. * None
  1788. */
  1789. static void mtip_hw_submit_io(struct driver_data *dd, struct request *rq,
  1790. struct mtip_cmd *command,
  1791. struct blk_mq_hw_ctx *hctx)
  1792. {
  1793. struct mtip_cmd_hdr *hdr =
  1794. dd->port->command_list + sizeof(struct mtip_cmd_hdr) * rq->tag;
  1795. struct host_to_dev_fis *fis;
  1796. struct mtip_port *port = dd->port;
  1797. int dma_dir = rq_data_dir(rq) == READ ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
  1798. u64 start = blk_rq_pos(rq);
  1799. unsigned int nsect = blk_rq_sectors(rq);
  1800. unsigned int nents;
  1801. /* Map the scatter list for DMA access */
  1802. nents = blk_rq_map_sg(hctx->queue, rq, command->sg);
  1803. nents = dma_map_sg(&dd->pdev->dev, command->sg, nents, dma_dir);
  1804. prefetch(&port->flags);
  1805. command->scatter_ents = nents;
  1806. /*
  1807. * The number of retries for this command before it is
  1808. * reported as a failure to the upper layers.
  1809. */
  1810. command->retries = MTIP_MAX_RETRIES;
  1811. /* Fill out fis */
  1812. fis = command->command;
  1813. fis->type = 0x27;
  1814. fis->opts = 1 << 7;
  1815. if (dma_dir == DMA_FROM_DEVICE)
  1816. fis->command = ATA_CMD_FPDMA_READ;
  1817. else
  1818. fis->command = ATA_CMD_FPDMA_WRITE;
  1819. fis->lba_low = start & 0xFF;
  1820. fis->lba_mid = (start >> 8) & 0xFF;
  1821. fis->lba_hi = (start >> 16) & 0xFF;
  1822. fis->lba_low_ex = (start >> 24) & 0xFF;
  1823. fis->lba_mid_ex = (start >> 32) & 0xFF;
  1824. fis->lba_hi_ex = (start >> 40) & 0xFF;
  1825. fis->device = 1 << 6;
  1826. fis->features = nsect & 0xFF;
  1827. fis->features_ex = (nsect >> 8) & 0xFF;
  1828. fis->sect_count = ((rq->tag << 3) | (rq->tag >> 5));
  1829. fis->sect_cnt_ex = 0;
  1830. fis->control = 0;
  1831. fis->res2 = 0;
  1832. fis->res3 = 0;
  1833. fill_command_sg(dd, command, nents);
  1834. if (unlikely(command->unaligned))
  1835. fis->device |= 1 << 7;
  1836. /* Populate the command header */
  1837. hdr->ctba = cpu_to_le32(command->command_dma & 0xFFFFFFFF);
  1838. if (test_bit(MTIP_PF_HOST_CAP_64, &dd->port->flags))
  1839. hdr->ctbau = cpu_to_le32((command->command_dma >> 16) >> 16);
  1840. hdr->opts = cpu_to_le32((nents << 16) | 5 | AHCI_CMD_PREFETCH);
  1841. hdr->byte_count = 0;
  1842. command->direction = dma_dir;
  1843. /*
  1844. * To prevent this command from being issued
  1845. * if an internal command is in progress or error handling is active.
  1846. */
  1847. if (unlikely(port->flags & MTIP_PF_PAUSE_IO)) {
  1848. set_bit(rq->tag, port->cmds_to_issue);
  1849. set_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags);
  1850. return;
  1851. }
  1852. /* Issue the command to the hardware */
  1853. mtip_issue_ncq_command(port, rq->tag);
  1854. }
  1855. /*
  1856. * Sysfs status dump.
  1857. *
  1858. * @dev Pointer to the device structure, passed by the kernrel.
  1859. * @attr Pointer to the device_attribute structure passed by the kernel.
  1860. * @buf Pointer to the char buffer that will receive the stats info.
  1861. *
  1862. * return value
  1863. * The size, in bytes, of the data copied into buf.
  1864. */
  1865. static ssize_t mtip_hw_show_status(struct device *dev,
  1866. struct device_attribute *attr,
  1867. char *buf)
  1868. {
  1869. struct driver_data *dd = dev_to_disk(dev)->private_data;
  1870. int size = 0;
  1871. if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag))
  1872. size += sprintf(buf, "%s", "thermal_shutdown\n");
  1873. else if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag))
  1874. size += sprintf(buf, "%s", "write_protect\n");
  1875. else
  1876. size += sprintf(buf, "%s", "online\n");
  1877. return size;
  1878. }
  1879. static DEVICE_ATTR(status, 0444, mtip_hw_show_status, NULL);
  1880. /* debugsfs entries */
  1881. static ssize_t show_device_status(struct device_driver *drv, char *buf)
  1882. {
  1883. int size = 0;
  1884. struct driver_data *dd, *tmp;
  1885. unsigned long flags;
  1886. char id_buf[42];
  1887. u16 status = 0;
  1888. spin_lock_irqsave(&dev_lock, flags);
  1889. size += sprintf(&buf[size], "Devices Present:\n");
  1890. list_for_each_entry_safe(dd, tmp, &online_list, online_list) {
  1891. if (dd->pdev) {
  1892. if (dd->port &&
  1893. dd->port->identify &&
  1894. dd->port->identify_valid) {
  1895. strlcpy(id_buf,
  1896. (char *) (dd->port->identify + 10), 21);
  1897. status = *(dd->port->identify + 141);
  1898. } else {
  1899. memset(id_buf, 0, 42);
  1900. status = 0;
  1901. }
  1902. if (dd->port &&
  1903. test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) {
  1904. size += sprintf(&buf[size],
  1905. " device %s %s (ftl rebuild %d %%)\n",
  1906. dev_name(&dd->pdev->dev),
  1907. id_buf,
  1908. status);
  1909. } else {
  1910. size += sprintf(&buf[size],
  1911. " device %s %s\n",
  1912. dev_name(&dd->pdev->dev),
  1913. id_buf);
  1914. }
  1915. }
  1916. }
  1917. size += sprintf(&buf[size], "Devices Being Removed:\n");
  1918. list_for_each_entry_safe(dd, tmp, &removing_list, remove_list) {
  1919. if (dd->pdev) {
  1920. if (dd->port &&
  1921. dd->port->identify &&
  1922. dd->port->identify_valid) {
  1923. strlcpy(id_buf,
  1924. (char *) (dd->port->identify+10), 21);
  1925. status = *(dd->port->identify + 141);
  1926. } else {
  1927. memset(id_buf, 0, 42);
  1928. status = 0;
  1929. }
  1930. if (dd->port &&
  1931. test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) {
  1932. size += sprintf(&buf[size],
  1933. " device %s %s (ftl rebuild %d %%)\n",
  1934. dev_name(&dd->pdev->dev),
  1935. id_buf,
  1936. status);
  1937. } else {
  1938. size += sprintf(&buf[size],
  1939. " device %s %s\n",
  1940. dev_name(&dd->pdev->dev),
  1941. id_buf);
  1942. }
  1943. }
  1944. }
  1945. spin_unlock_irqrestore(&dev_lock, flags);
  1946. return size;
  1947. }
  1948. static ssize_t mtip_hw_read_device_status(struct file *f, char __user *ubuf,
  1949. size_t len, loff_t *offset)
  1950. {
  1951. struct driver_data *dd = (struct driver_data *)f->private_data;
  1952. int size = *offset;
  1953. char *buf;
  1954. int rv = 0;
  1955. if (!len || *offset)
  1956. return 0;
  1957. buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL);
  1958. if (!buf) {
  1959. dev_err(&dd->pdev->dev,
  1960. "Memory allocation: status buffer\n");
  1961. return -ENOMEM;
  1962. }
  1963. size += show_device_status(NULL, buf);
  1964. *offset = size <= len ? size : len;
  1965. size = copy_to_user(ubuf, buf, *offset);
  1966. if (size)
  1967. rv = -EFAULT;
  1968. kfree(buf);
  1969. return rv ? rv : *offset;
  1970. }
  1971. static ssize_t mtip_hw_read_registers(struct file *f, char __user *ubuf,
  1972. size_t len, loff_t *offset)
  1973. {
  1974. struct driver_data *dd = (struct driver_data *)f->private_data;
  1975. char *buf;
  1976. u32 group_allocated;
  1977. int size = *offset;
  1978. int n, rv = 0;
  1979. if (!len || size)
  1980. return 0;
  1981. buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL);
  1982. if (!buf) {
  1983. dev_err(&dd->pdev->dev,
  1984. "Memory allocation: register buffer\n");
  1985. return -ENOMEM;
  1986. }
  1987. size += sprintf(&buf[size], "H/ S ACTive : [ 0x");
  1988. for (n = dd->slot_groups-1; n >= 0; n--)
  1989. size += sprintf(&buf[size], "%08X ",
  1990. readl(dd->port->s_active[n]));
  1991. size += sprintf(&buf[size], "]\n");
  1992. size += sprintf(&buf[size], "H/ Command Issue : [ 0x");
  1993. for (n = dd->slot_groups-1; n >= 0; n--)
  1994. size += sprintf(&buf[size], "%08X ",
  1995. readl(dd->port->cmd_issue[n]));
  1996. size += sprintf(&buf[size], "]\n");
  1997. size += sprintf(&buf[size], "H/ Completed : [ 0x");
  1998. for (n = dd->slot_groups-1; n >= 0; n--)
  1999. size += sprintf(&buf[size], "%08X ",
  2000. readl(dd->port->completed[n]));
  2001. size += sprintf(&buf[size], "]\n");
  2002. size += sprintf(&buf[size], "H/ PORT IRQ STAT : [ 0x%08X ]\n",
  2003. readl(dd->port->mmio + PORT_IRQ_STAT));
  2004. size += sprintf(&buf[size], "H/ HOST IRQ STAT : [ 0x%08X ]\n",
  2005. readl(dd->mmio + HOST_IRQ_STAT));
  2006. size += sprintf(&buf[size], "\n");
  2007. size += sprintf(&buf[size], "L/ Commands in Q : [ 0x");
  2008. for (n = dd->slot_groups-1; n >= 0; n--) {
  2009. if (sizeof(long) > sizeof(u32))
  2010. group_allocated =
  2011. dd->port->cmds_to_issue[n/2] >> (32*(n&1));
  2012. else
  2013. group_allocated = dd->port->cmds_to_issue[n];
  2014. size += sprintf(&buf[size], "%08X ", group_allocated);
  2015. }
  2016. size += sprintf(&buf[size], "]\n");
  2017. *offset = size <= len ? size : len;
  2018. size = copy_to_user(ubuf, buf, *offset);
  2019. if (size)
  2020. rv = -EFAULT;
  2021. kfree(buf);
  2022. return rv ? rv : *offset;
  2023. }
  2024. static ssize_t mtip_hw_read_flags(struct file *f, char __user *ubuf,
  2025. size_t len, loff_t *offset)
  2026. {
  2027. struct driver_data *dd = (struct driver_data *)f->private_data;
  2028. char *buf;
  2029. int size = *offset;
  2030. int rv = 0;
  2031. if (!len || size)
  2032. return 0;
  2033. buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL);
  2034. if (!buf) {
  2035. dev_err(&dd->pdev->dev,
  2036. "Memory allocation: flag buffer\n");
  2037. return -ENOMEM;
  2038. }
  2039. size += sprintf(&buf[size], "Flag-port : [ %08lX ]\n",
  2040. dd->port->flags);
  2041. size += sprintf(&buf[size], "Flag-dd : [ %08lX ]\n",
  2042. dd->dd_flag);
  2043. *offset = size <= len ? size : len;
  2044. size = copy_to_user(ubuf, buf, *offset);
  2045. if (size)
  2046. rv = -EFAULT;
  2047. kfree(buf);
  2048. return rv ? rv : *offset;
  2049. }
  2050. static const struct file_operations mtip_device_status_fops = {
  2051. .owner = THIS_MODULE,
  2052. .open = simple_open,
  2053. .read = mtip_hw_read_device_status,
  2054. .llseek = no_llseek,
  2055. };
  2056. static const struct file_operations mtip_regs_fops = {
  2057. .owner = THIS_MODULE,
  2058. .open = simple_open,
  2059. .read = mtip_hw_read_registers,
  2060. .llseek = no_llseek,
  2061. };
  2062. static const struct file_operations mtip_flags_fops = {
  2063. .owner = THIS_MODULE,
  2064. .open = simple_open,
  2065. .read = mtip_hw_read_flags,
  2066. .llseek = no_llseek,
  2067. };
  2068. /*
  2069. * Create the sysfs related attributes.
  2070. *
  2071. * @dd Pointer to the driver data structure.
  2072. * @kobj Pointer to the kobj for the block device.
  2073. *
  2074. * return value
  2075. * 0 Operation completed successfully.
  2076. * -EINVAL Invalid parameter.
  2077. */
  2078. static int mtip_hw_sysfs_init(struct driver_data *dd, struct kobject *kobj)
  2079. {
  2080. if (!kobj || !dd)
  2081. return -EINVAL;
  2082. if (sysfs_create_file(kobj, &dev_attr_status.attr))
  2083. dev_warn(&dd->pdev->dev,
  2084. "Error creating 'status' sysfs entry\n");
  2085. return 0;
  2086. }
  2087. /*
  2088. * Remove the sysfs related attributes.
  2089. *
  2090. * @dd Pointer to the driver data structure.
  2091. * @kobj Pointer to the kobj for the block device.
  2092. *
  2093. * return value
  2094. * 0 Operation completed successfully.
  2095. * -EINVAL Invalid parameter.
  2096. */
  2097. static int mtip_hw_sysfs_exit(struct driver_data *dd, struct kobject *kobj)
  2098. {
  2099. if (!kobj || !dd)
  2100. return -EINVAL;
  2101. sysfs_remove_file(kobj, &dev_attr_status.attr);
  2102. return 0;
  2103. }
  2104. static int mtip_hw_debugfs_init(struct driver_data *dd)
  2105. {
  2106. if (!dfs_parent)
  2107. return -1;
  2108. dd->dfs_node = debugfs_create_dir(dd->disk->disk_name, dfs_parent);
  2109. if (IS_ERR_OR_NULL(dd->dfs_node)) {
  2110. dev_warn(&dd->pdev->dev,
  2111. "Error creating node %s under debugfs\n",
  2112. dd->disk->disk_name);
  2113. dd->dfs_node = NULL;
  2114. return -1;
  2115. }
  2116. debugfs_create_file("flags", 0444, dd->dfs_node, dd, &mtip_flags_fops);
  2117. debugfs_create_file("registers", 0444, dd->dfs_node, dd,
  2118. &mtip_regs_fops);
  2119. return 0;
  2120. }
  2121. static void mtip_hw_debugfs_exit(struct driver_data *dd)
  2122. {
  2123. debugfs_remove_recursive(dd->dfs_node);
  2124. }
  2125. /*
  2126. * Perform any init/resume time hardware setup
  2127. *
  2128. * @dd Pointer to the driver data structure.
  2129. *
  2130. * return value
  2131. * None
  2132. */
  2133. static inline void hba_setup(struct driver_data *dd)
  2134. {
  2135. u32 hwdata;
  2136. hwdata = readl(dd->mmio + HOST_HSORG);
  2137. /* interrupt bug workaround: use only 1 IS bit.*/
  2138. writel(hwdata |
  2139. HSORG_DISABLE_SLOTGRP_INTR |
  2140. HSORG_DISABLE_SLOTGRP_PXIS,
  2141. dd->mmio + HOST_HSORG);
  2142. }
  2143. static int mtip_device_unaligned_constrained(struct driver_data *dd)
  2144. {
  2145. return (dd->pdev->device == P420M_DEVICE_ID ? 1 : 0);
  2146. }
  2147. /*
  2148. * Detect the details of the product, and store anything needed
  2149. * into the driver data structure. This includes product type and
  2150. * version and number of slot groups.
  2151. *
  2152. * @dd Pointer to the driver data structure.
  2153. *
  2154. * return value
  2155. * None
  2156. */
  2157. static void mtip_detect_product(struct driver_data *dd)
  2158. {
  2159. u32 hwdata;
  2160. unsigned int rev, slotgroups;
  2161. /*
  2162. * HBA base + 0xFC [15:0] - vendor-specific hardware interface
  2163. * info register:
  2164. * [15:8] hardware/software interface rev#
  2165. * [ 3] asic-style interface
  2166. * [ 2:0] number of slot groups, minus 1 (only valid for asic-style).
  2167. */
  2168. hwdata = readl(dd->mmio + HOST_HSORG);
  2169. dd->product_type = MTIP_PRODUCT_UNKNOWN;
  2170. dd->slot_groups = 1;
  2171. if (hwdata & 0x8) {
  2172. dd->product_type = MTIP_PRODUCT_ASICFPGA;
  2173. rev = (hwdata & HSORG_HWREV) >> 8;
  2174. slotgroups = (hwdata & HSORG_SLOTGROUPS) + 1;
  2175. dev_info(&dd->pdev->dev,
  2176. "ASIC-FPGA design, HS rev 0x%x, "
  2177. "%i slot groups [%i slots]\n",
  2178. rev,
  2179. slotgroups,
  2180. slotgroups * 32);
  2181. if (slotgroups > MTIP_MAX_SLOT_GROUPS) {
  2182. dev_warn(&dd->pdev->dev,
  2183. "Warning: driver only supports "
  2184. "%i slot groups.\n", MTIP_MAX_SLOT_GROUPS);
  2185. slotgroups = MTIP_MAX_SLOT_GROUPS;
  2186. }
  2187. dd->slot_groups = slotgroups;
  2188. return;
  2189. }
  2190. dev_warn(&dd->pdev->dev, "Unrecognized product id\n");
  2191. }
  2192. /*
  2193. * Blocking wait for FTL rebuild to complete
  2194. *
  2195. * @dd Pointer to the DRIVER_DATA structure.
  2196. *
  2197. * return value
  2198. * 0 FTL rebuild completed successfully
  2199. * -EFAULT FTL rebuild error/timeout/interruption
  2200. */
  2201. static int mtip_ftl_rebuild_poll(struct driver_data *dd)
  2202. {
  2203. unsigned long timeout, cnt = 0, start;
  2204. dev_warn(&dd->pdev->dev,
  2205. "FTL rebuild in progress. Polling for completion.\n");
  2206. start = jiffies;
  2207. timeout = jiffies + msecs_to_jiffies(MTIP_FTL_REBUILD_TIMEOUT_MS);
  2208. do {
  2209. if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
  2210. &dd->dd_flag)))
  2211. return -EFAULT;
  2212. if (mtip_check_surprise_removal(dd->pdev))
  2213. return -EFAULT;
  2214. if (mtip_get_identify(dd->port, NULL) < 0)
  2215. return -EFAULT;
  2216. if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) ==
  2217. MTIP_FTL_REBUILD_MAGIC) {
  2218. ssleep(1);
  2219. /* Print message every 3 minutes */
  2220. if (cnt++ >= 180) {
  2221. dev_warn(&dd->pdev->dev,
  2222. "FTL rebuild in progress (%d secs).\n",
  2223. jiffies_to_msecs(jiffies - start) / 1000);
  2224. cnt = 0;
  2225. }
  2226. } else {
  2227. dev_warn(&dd->pdev->dev,
  2228. "FTL rebuild complete (%d secs).\n",
  2229. jiffies_to_msecs(jiffies - start) / 1000);
  2230. mtip_block_initialize(dd);
  2231. return 0;
  2232. }
  2233. } while (time_before(jiffies, timeout));
  2234. /* Check for timeout */
  2235. dev_err(&dd->pdev->dev,
  2236. "Timed out waiting for FTL rebuild to complete (%d secs).\n",
  2237. jiffies_to_msecs(jiffies - start) / 1000);
  2238. return -EFAULT;
  2239. }
  2240. static void mtip_softirq_done_fn(struct request *rq)
  2241. {
  2242. struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
  2243. struct driver_data *dd = rq->q->queuedata;
  2244. /* Unmap the DMA scatter list entries */
  2245. dma_unmap_sg(&dd->pdev->dev, cmd->sg, cmd->scatter_ents,
  2246. cmd->direction);
  2247. if (unlikely(cmd->unaligned))
  2248. atomic_inc(&dd->port->cmd_slot_unal);
  2249. blk_mq_end_request(rq, cmd->status);
  2250. }
  2251. static bool mtip_abort_cmd(struct request *req, void *data, bool reserved)
  2252. {
  2253. struct mtip_cmd *cmd = blk_mq_rq_to_pdu(req);
  2254. struct driver_data *dd = data;
  2255. dbg_printk(MTIP_DRV_NAME " Aborting request, tag = %d\n", req->tag);
  2256. clear_bit(req->tag, dd->port->cmds_to_issue);
  2257. cmd->status = BLK_STS_IOERR;
  2258. mtip_softirq_done_fn(req);
  2259. return true;
  2260. }
  2261. static bool mtip_queue_cmd(struct request *req, void *data, bool reserved)
  2262. {
  2263. struct driver_data *dd = data;
  2264. set_bit(req->tag, dd->port->cmds_to_issue);
  2265. blk_abort_request(req);
  2266. return true;
  2267. }
  2268. /*
  2269. * service thread to issue queued commands
  2270. *
  2271. * @data Pointer to the driver data structure.
  2272. *
  2273. * return value
  2274. * 0
  2275. */
  2276. static int mtip_service_thread(void *data)
  2277. {
  2278. struct driver_data *dd = (struct driver_data *)data;
  2279. unsigned long slot, slot_start, slot_wrap, to;
  2280. unsigned int num_cmd_slots = dd->slot_groups * 32;
  2281. struct mtip_port *port = dd->port;
  2282. while (1) {
  2283. if (kthread_should_stop() ||
  2284. test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags))
  2285. goto st_out;
  2286. clear_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags);
  2287. /*
  2288. * the condition is to check neither an internal command is
  2289. * is in progress nor error handling is active
  2290. */
  2291. wait_event_interruptible(port->svc_wait, (port->flags) &&
  2292. (port->flags & MTIP_PF_SVC_THD_WORK));
  2293. if (kthread_should_stop() ||
  2294. test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags))
  2295. goto st_out;
  2296. if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
  2297. &dd->dd_flag)))
  2298. goto st_out;
  2299. set_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags);
  2300. restart_eh:
  2301. /* Demux bits: start with error handling */
  2302. if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) {
  2303. mtip_handle_tfe(dd);
  2304. clear_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags);
  2305. }
  2306. if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags))
  2307. goto restart_eh;
  2308. if (test_bit(MTIP_PF_TO_ACTIVE_BIT, &port->flags)) {
  2309. to = jiffies + msecs_to_jiffies(5000);
  2310. do {
  2311. mdelay(100);
  2312. } while (atomic_read(&dd->irq_workers_active) != 0 &&
  2313. time_before(jiffies, to));
  2314. if (atomic_read(&dd->irq_workers_active) != 0)
  2315. dev_warn(&dd->pdev->dev,
  2316. "Completion workers still active!");
  2317. blk_mq_quiesce_queue(dd->queue);
  2318. blk_mq_tagset_busy_iter(&dd->tags, mtip_queue_cmd, dd);
  2319. set_bit(MTIP_PF_ISSUE_CMDS_BIT, &dd->port->flags);
  2320. if (mtip_device_reset(dd))
  2321. blk_mq_tagset_busy_iter(&dd->tags,
  2322. mtip_abort_cmd, dd);
  2323. clear_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags);
  2324. blk_mq_unquiesce_queue(dd->queue);
  2325. }
  2326. if (test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) {
  2327. slot = 1;
  2328. /* used to restrict the loop to one iteration */
  2329. slot_start = num_cmd_slots;
  2330. slot_wrap = 0;
  2331. while (1) {
  2332. slot = find_next_bit(port->cmds_to_issue,
  2333. num_cmd_slots, slot);
  2334. if (slot_wrap == 1) {
  2335. if ((slot_start >= slot) ||
  2336. (slot >= num_cmd_slots))
  2337. break;
  2338. }
  2339. if (unlikely(slot_start == num_cmd_slots))
  2340. slot_start = slot;
  2341. if (unlikely(slot == num_cmd_slots)) {
  2342. slot = 1;
  2343. slot_wrap = 1;
  2344. continue;
  2345. }
  2346. /* Issue the command to the hardware */
  2347. mtip_issue_ncq_command(port, slot);
  2348. clear_bit(slot, port->cmds_to_issue);
  2349. }
  2350. clear_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags);
  2351. }
  2352. if (test_bit(MTIP_PF_REBUILD_BIT, &port->flags)) {
  2353. if (mtip_ftl_rebuild_poll(dd) == 0)
  2354. clear_bit(MTIP_PF_REBUILD_BIT, &port->flags);
  2355. }
  2356. }
  2357. st_out:
  2358. return 0;
  2359. }
  2360. /*
  2361. * DMA region teardown
  2362. *
  2363. * @dd Pointer to driver_data structure
  2364. *
  2365. * return value
  2366. * None
  2367. */
  2368. static void mtip_dma_free(struct driver_data *dd)
  2369. {
  2370. struct mtip_port *port = dd->port;
  2371. if (port->block1)
  2372. dma_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ,
  2373. port->block1, port->block1_dma);
  2374. if (port->command_list) {
  2375. dma_free_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ,
  2376. port->command_list, port->command_list_dma);
  2377. }
  2378. }
  2379. /*
  2380. * DMA region setup
  2381. *
  2382. * @dd Pointer to driver_data structure
  2383. *
  2384. * return value
  2385. * -ENOMEM Not enough free DMA region space to initialize driver
  2386. */
  2387. static int mtip_dma_alloc(struct driver_data *dd)
  2388. {
  2389. struct mtip_port *port = dd->port;
  2390. /* Allocate dma memory for RX Fis, Identify, and Sector Bufffer */
  2391. port->block1 =
  2392. dma_alloc_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ,
  2393. &port->block1_dma, GFP_KERNEL);
  2394. if (!port->block1)
  2395. return -ENOMEM;
  2396. /* Allocate dma memory for command list */
  2397. port->command_list =
  2398. dma_alloc_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ,
  2399. &port->command_list_dma, GFP_KERNEL);
  2400. if (!port->command_list) {
  2401. dma_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ,
  2402. port->block1, port->block1_dma);
  2403. port->block1 = NULL;
  2404. port->block1_dma = 0;
  2405. return -ENOMEM;
  2406. }
  2407. /* Setup all pointers into first DMA region */
  2408. port->rxfis = port->block1 + AHCI_RX_FIS_OFFSET;
  2409. port->rxfis_dma = port->block1_dma + AHCI_RX_FIS_OFFSET;
  2410. port->identify = port->block1 + AHCI_IDFY_OFFSET;
  2411. port->identify_dma = port->block1_dma + AHCI_IDFY_OFFSET;
  2412. port->log_buf = port->block1 + AHCI_SECTBUF_OFFSET;
  2413. port->log_buf_dma = port->block1_dma + AHCI_SECTBUF_OFFSET;
  2414. port->smart_buf = port->block1 + AHCI_SMARTBUF_OFFSET;
  2415. port->smart_buf_dma = port->block1_dma + AHCI_SMARTBUF_OFFSET;
  2416. return 0;
  2417. }
  2418. static int mtip_hw_get_identify(struct driver_data *dd)
  2419. {
  2420. struct smart_attr attr242;
  2421. unsigned char *buf;
  2422. int rv;
  2423. if (mtip_get_identify(dd->port, NULL) < 0)
  2424. return -EFAULT;
  2425. if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) ==
  2426. MTIP_FTL_REBUILD_MAGIC) {
  2427. set_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags);
  2428. return MTIP_FTL_REBUILD_MAGIC;
  2429. }
  2430. mtip_dump_identify(dd->port);
  2431. /* check write protect, over temp and rebuild statuses */
  2432. rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ,
  2433. dd->port->log_buf,
  2434. dd->port->log_buf_dma, 1);
  2435. if (rv) {
  2436. dev_warn(&dd->pdev->dev,
  2437. "Error in READ LOG EXT (10h) command\n");
  2438. /* non-critical error, don't fail the load */
  2439. } else {
  2440. buf = (unsigned char *)dd->port->log_buf;
  2441. if (buf[259] & 0x1) {
  2442. dev_info(&dd->pdev->dev,
  2443. "Write protect bit is set.\n");
  2444. set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag);
  2445. }
  2446. if (buf[288] == 0xF7) {
  2447. dev_info(&dd->pdev->dev,
  2448. "Exceeded Tmax, drive in thermal shutdown.\n");
  2449. set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag);
  2450. }
  2451. if (buf[288] == 0xBF) {
  2452. dev_info(&dd->pdev->dev,
  2453. "Drive indicates rebuild has failed.\n");
  2454. set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag);
  2455. }
  2456. }
  2457. /* get write protect progess */
  2458. memset(&attr242, 0, sizeof(struct smart_attr));
  2459. if (mtip_get_smart_attr(dd->port, 242, &attr242))
  2460. dev_warn(&dd->pdev->dev,
  2461. "Unable to check write protect progress\n");
  2462. else
  2463. dev_info(&dd->pdev->dev,
  2464. "Write protect progress: %u%% (%u blocks)\n",
  2465. attr242.cur, le32_to_cpu(attr242.data));
  2466. return rv;
  2467. }
  2468. /*
  2469. * Called once for each card.
  2470. *
  2471. * @dd Pointer to the driver data structure.
  2472. *
  2473. * return value
  2474. * 0 on success, else an error code.
  2475. */
  2476. static int mtip_hw_init(struct driver_data *dd)
  2477. {
  2478. int i;
  2479. int rv;
  2480. unsigned long timeout, timetaken;
  2481. dd->mmio = pcim_iomap_table(dd->pdev)[MTIP_ABAR];
  2482. mtip_detect_product(dd);
  2483. if (dd->product_type == MTIP_PRODUCT_UNKNOWN) {
  2484. rv = -EIO;
  2485. goto out1;
  2486. }
  2487. hba_setup(dd);
  2488. dd->port = kzalloc_node(sizeof(struct mtip_port), GFP_KERNEL,
  2489. dd->numa_node);
  2490. if (!dd->port) {
  2491. dev_err(&dd->pdev->dev,
  2492. "Memory allocation: port structure\n");
  2493. return -ENOMEM;
  2494. }
  2495. /* Continue workqueue setup */
  2496. for (i = 0; i < MTIP_MAX_SLOT_GROUPS; i++)
  2497. dd->work[i].port = dd->port;
  2498. /* Enable unaligned IO constraints for some devices */
  2499. if (mtip_device_unaligned_constrained(dd))
  2500. dd->unal_qdepth = MTIP_MAX_UNALIGNED_SLOTS;
  2501. else
  2502. dd->unal_qdepth = 0;
  2503. atomic_set(&dd->port->cmd_slot_unal, dd->unal_qdepth);
  2504. /* Spinlock to prevent concurrent issue */
  2505. for (i = 0; i < MTIP_MAX_SLOT_GROUPS; i++)
  2506. spin_lock_init(&dd->port->cmd_issue_lock[i]);
  2507. /* Set the port mmio base address. */
  2508. dd->port->mmio = dd->mmio + PORT_OFFSET;
  2509. dd->port->dd = dd;
  2510. /* DMA allocations */
  2511. rv = mtip_dma_alloc(dd);
  2512. if (rv < 0)
  2513. goto out1;
  2514. /* Setup the pointers to the extended s_active and CI registers. */
  2515. for (i = 0; i < dd->slot_groups; i++) {
  2516. dd->port->s_active[i] =
  2517. dd->port->mmio + i*0x80 + PORT_SCR_ACT;
  2518. dd->port->cmd_issue[i] =
  2519. dd->port->mmio + i*0x80 + PORT_COMMAND_ISSUE;
  2520. dd->port->completed[i] =
  2521. dd->port->mmio + i*0x80 + PORT_SDBV;
  2522. }
  2523. timetaken = jiffies;
  2524. timeout = jiffies + msecs_to_jiffies(30000);
  2525. while (((readl(dd->port->mmio + PORT_SCR_STAT) & 0x0F) != 0x03) &&
  2526. time_before(jiffies, timeout)) {
  2527. mdelay(100);
  2528. }
  2529. if (unlikely(mtip_check_surprise_removal(dd->pdev))) {
  2530. timetaken = jiffies - timetaken;
  2531. dev_warn(&dd->pdev->dev,
  2532. "Surprise removal detected at %u ms\n",
  2533. jiffies_to_msecs(timetaken));
  2534. rv = -ENODEV;
  2535. goto out2 ;
  2536. }
  2537. if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) {
  2538. timetaken = jiffies - timetaken;
  2539. dev_warn(&dd->pdev->dev,
  2540. "Removal detected at %u ms\n",
  2541. jiffies_to_msecs(timetaken));
  2542. rv = -EFAULT;
  2543. goto out2;
  2544. }
  2545. /* Conditionally reset the HBA. */
  2546. if (!(readl(dd->mmio + HOST_CAP) & HOST_CAP_NZDMA)) {
  2547. if (mtip_hba_reset(dd) < 0) {
  2548. dev_err(&dd->pdev->dev,
  2549. "Card did not reset within timeout\n");
  2550. rv = -EIO;
  2551. goto out2;
  2552. }
  2553. } else {
  2554. /* Clear any pending interrupts on the HBA */
  2555. writel(readl(dd->mmio + HOST_IRQ_STAT),
  2556. dd->mmio + HOST_IRQ_STAT);
  2557. }
  2558. mtip_init_port(dd->port);
  2559. mtip_start_port(dd->port);
  2560. /* Setup the ISR and enable interrupts. */
  2561. rv = request_irq(dd->pdev->irq, mtip_irq_handler, IRQF_SHARED,
  2562. dev_driver_string(&dd->pdev->dev), dd);
  2563. if (rv) {
  2564. dev_err(&dd->pdev->dev,
  2565. "Unable to allocate IRQ %d\n", dd->pdev->irq);
  2566. goto out2;
  2567. }
  2568. irq_set_affinity_hint(dd->pdev->irq, get_cpu_mask(dd->isr_binding));
  2569. /* Enable interrupts on the HBA. */
  2570. writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN,
  2571. dd->mmio + HOST_CTL);
  2572. init_waitqueue_head(&dd->port->svc_wait);
  2573. if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) {
  2574. rv = -EFAULT;
  2575. goto out3;
  2576. }
  2577. return rv;
  2578. out3:
  2579. /* Disable interrupts on the HBA. */
  2580. writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN,
  2581. dd->mmio + HOST_CTL);
  2582. /* Release the IRQ. */
  2583. irq_set_affinity_hint(dd->pdev->irq, NULL);
  2584. free_irq(dd->pdev->irq, dd);
  2585. out2:
  2586. mtip_deinit_port(dd->port);
  2587. mtip_dma_free(dd);
  2588. out1:
  2589. /* Free the memory allocated for the for structure. */
  2590. kfree(dd->port);
  2591. return rv;
  2592. }
  2593. static int mtip_standby_drive(struct driver_data *dd)
  2594. {
  2595. int rv = 0;
  2596. if (dd->sr || !dd->port)
  2597. return -ENODEV;
  2598. /*
  2599. * Send standby immediate (E0h) to the drive so that it
  2600. * saves its state.
  2601. */
  2602. if (!test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags) &&
  2603. !test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag) &&
  2604. !test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag)) {
  2605. rv = mtip_standby_immediate(dd->port);
  2606. if (rv)
  2607. dev_warn(&dd->pdev->dev,
  2608. "STANDBY IMMEDIATE failed\n");
  2609. }
  2610. return rv;
  2611. }
  2612. /*
  2613. * Called to deinitialize an interface.
  2614. *
  2615. * @dd Pointer to the driver data structure.
  2616. *
  2617. * return value
  2618. * 0
  2619. */
  2620. static int mtip_hw_exit(struct driver_data *dd)
  2621. {
  2622. if (!dd->sr) {
  2623. /* de-initialize the port. */
  2624. mtip_deinit_port(dd->port);
  2625. /* Disable interrupts on the HBA. */
  2626. writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN,
  2627. dd->mmio + HOST_CTL);
  2628. }
  2629. /* Release the IRQ. */
  2630. irq_set_affinity_hint(dd->pdev->irq, NULL);
  2631. free_irq(dd->pdev->irq, dd);
  2632. msleep(1000);
  2633. /* Free dma regions */
  2634. mtip_dma_free(dd);
  2635. /* Free the memory allocated for the for structure. */
  2636. kfree(dd->port);
  2637. dd->port = NULL;
  2638. return 0;
  2639. }
  2640. /*
  2641. * Issue a Standby Immediate command to the device.
  2642. *
  2643. * This function is called by the Block Layer just before the
  2644. * system powers off during a shutdown.
  2645. *
  2646. * @dd Pointer to the driver data structure.
  2647. *
  2648. * return value
  2649. * 0
  2650. */
  2651. static int mtip_hw_shutdown(struct driver_data *dd)
  2652. {
  2653. /*
  2654. * Send standby immediate (E0h) to the drive so that it
  2655. * saves its state.
  2656. */
  2657. mtip_standby_drive(dd);
  2658. return 0;
  2659. }
  2660. /*
  2661. * Suspend function
  2662. *
  2663. * This function is called by the Block Layer just before the
  2664. * system hibernates.
  2665. *
  2666. * @dd Pointer to the driver data structure.
  2667. *
  2668. * return value
  2669. * 0 Suspend was successful
  2670. * -EFAULT Suspend was not successful
  2671. */
  2672. static int mtip_hw_suspend(struct driver_data *dd)
  2673. {
  2674. /*
  2675. * Send standby immediate (E0h) to the drive
  2676. * so that it saves its state.
  2677. */
  2678. if (mtip_standby_drive(dd) != 0) {
  2679. dev_err(&dd->pdev->dev,
  2680. "Failed standby-immediate command\n");
  2681. return -EFAULT;
  2682. }
  2683. /* Disable interrupts on the HBA.*/
  2684. writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN,
  2685. dd->mmio + HOST_CTL);
  2686. mtip_deinit_port(dd->port);
  2687. return 0;
  2688. }
  2689. /*
  2690. * Resume function
  2691. *
  2692. * This function is called by the Block Layer as the
  2693. * system resumes.
  2694. *
  2695. * @dd Pointer to the driver data structure.
  2696. *
  2697. * return value
  2698. * 0 Resume was successful
  2699. * -EFAULT Resume was not successful
  2700. */
  2701. static int mtip_hw_resume(struct driver_data *dd)
  2702. {
  2703. /* Perform any needed hardware setup steps */
  2704. hba_setup(dd);
  2705. /* Reset the HBA */
  2706. if (mtip_hba_reset(dd) != 0) {
  2707. dev_err(&dd->pdev->dev,
  2708. "Unable to reset the HBA\n");
  2709. return -EFAULT;
  2710. }
  2711. /*
  2712. * Enable the port, DMA engine, and FIS reception specific
  2713. * h/w in controller.
  2714. */
  2715. mtip_init_port(dd->port);
  2716. mtip_start_port(dd->port);
  2717. /* Enable interrupts on the HBA.*/
  2718. writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN,
  2719. dd->mmio + HOST_CTL);
  2720. return 0;
  2721. }
  2722. /*
  2723. * Helper function for reusing disk name
  2724. * upon hot insertion.
  2725. */
  2726. static int rssd_disk_name_format(char *prefix,
  2727. int index,
  2728. char *buf,
  2729. int buflen)
  2730. {
  2731. const int base = 'z' - 'a' + 1;
  2732. char *begin = buf + strlen(prefix);
  2733. char *end = buf + buflen;
  2734. char *p;
  2735. int unit;
  2736. p = end - 1;
  2737. *p = '\0';
  2738. unit = base;
  2739. do {
  2740. if (p == begin)
  2741. return -EINVAL;
  2742. *--p = 'a' + (index % unit);
  2743. index = (index / unit) - 1;
  2744. } while (index >= 0);
  2745. memmove(begin, p, end - p);
  2746. memcpy(buf, prefix, strlen(prefix));
  2747. return 0;
  2748. }
  2749. /*
  2750. * Block layer IOCTL handler.
  2751. *
  2752. * @dev Pointer to the block_device structure.
  2753. * @mode ignored
  2754. * @cmd IOCTL command passed from the user application.
  2755. * @arg Argument passed from the user application.
  2756. *
  2757. * return value
  2758. * 0 IOCTL completed successfully.
  2759. * -ENOTTY IOCTL not supported or invalid driver data
  2760. * structure pointer.
  2761. */
  2762. static int mtip_block_ioctl(struct block_device *dev,
  2763. fmode_t mode,
  2764. unsigned cmd,
  2765. unsigned long arg)
  2766. {
  2767. struct driver_data *dd = dev->bd_disk->private_data;
  2768. if (!capable(CAP_SYS_ADMIN))
  2769. return -EACCES;
  2770. if (!dd)
  2771. return -ENOTTY;
  2772. if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)))
  2773. return -ENOTTY;
  2774. switch (cmd) {
  2775. case BLKFLSBUF:
  2776. return -ENOTTY;
  2777. default:
  2778. return mtip_hw_ioctl(dd, cmd, arg);
  2779. }
  2780. }
  2781. #ifdef CONFIG_COMPAT
  2782. /*
  2783. * Block layer compat IOCTL handler.
  2784. *
  2785. * @dev Pointer to the block_device structure.
  2786. * @mode ignored
  2787. * @cmd IOCTL command passed from the user application.
  2788. * @arg Argument passed from the user application.
  2789. *
  2790. * return value
  2791. * 0 IOCTL completed successfully.
  2792. * -ENOTTY IOCTL not supported or invalid driver data
  2793. * structure pointer.
  2794. */
  2795. static int mtip_block_compat_ioctl(struct block_device *dev,
  2796. fmode_t mode,
  2797. unsigned cmd,
  2798. unsigned long arg)
  2799. {
  2800. struct driver_data *dd = dev->bd_disk->private_data;
  2801. if (!capable(CAP_SYS_ADMIN))
  2802. return -EACCES;
  2803. if (!dd)
  2804. return -ENOTTY;
  2805. if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)))
  2806. return -ENOTTY;
  2807. switch (cmd) {
  2808. case BLKFLSBUF:
  2809. return -ENOTTY;
  2810. case HDIO_DRIVE_TASKFILE: {
  2811. struct mtip_compat_ide_task_request_s __user *compat_req_task;
  2812. ide_task_request_t req_task;
  2813. int compat_tasksize, outtotal, ret;
  2814. compat_tasksize =
  2815. sizeof(struct mtip_compat_ide_task_request_s);
  2816. compat_req_task =
  2817. (struct mtip_compat_ide_task_request_s __user *) arg;
  2818. if (copy_from_user(&req_task, (void __user *) arg,
  2819. compat_tasksize - (2 * sizeof(compat_long_t))))
  2820. return -EFAULT;
  2821. if (get_user(req_task.out_size, &compat_req_task->out_size))
  2822. return -EFAULT;
  2823. if (get_user(req_task.in_size, &compat_req_task->in_size))
  2824. return -EFAULT;
  2825. outtotal = sizeof(struct mtip_compat_ide_task_request_s);
  2826. ret = exec_drive_taskfile(dd, (void __user *) arg,
  2827. &req_task, outtotal);
  2828. if (copy_to_user((void __user *) arg, &req_task,
  2829. compat_tasksize -
  2830. (2 * sizeof(compat_long_t))))
  2831. return -EFAULT;
  2832. if (put_user(req_task.out_size, &compat_req_task->out_size))
  2833. return -EFAULT;
  2834. if (put_user(req_task.in_size, &compat_req_task->in_size))
  2835. return -EFAULT;
  2836. return ret;
  2837. }
  2838. default:
  2839. return mtip_hw_ioctl(dd, cmd, arg);
  2840. }
  2841. }
  2842. #endif
  2843. /*
  2844. * Obtain the geometry of the device.
  2845. *
  2846. * You may think that this function is obsolete, but some applications,
  2847. * fdisk for example still used CHS values. This function describes the
  2848. * device as having 224 heads and 56 sectors per cylinder. These values are
  2849. * chosen so that each cylinder is aligned on a 4KB boundary. Since a
  2850. * partition is described in terms of a start and end cylinder this means
  2851. * that each partition is also 4KB aligned. Non-aligned partitions adversely
  2852. * affects performance.
  2853. *
  2854. * @dev Pointer to the block_device strucutre.
  2855. * @geo Pointer to a hd_geometry structure.
  2856. *
  2857. * return value
  2858. * 0 Operation completed successfully.
  2859. * -ENOTTY An error occurred while reading the drive capacity.
  2860. */
  2861. static int mtip_block_getgeo(struct block_device *dev,
  2862. struct hd_geometry *geo)
  2863. {
  2864. struct driver_data *dd = dev->bd_disk->private_data;
  2865. sector_t capacity;
  2866. if (!dd)
  2867. return -ENOTTY;
  2868. if (!(mtip_hw_get_capacity(dd, &capacity))) {
  2869. dev_warn(&dd->pdev->dev,
  2870. "Could not get drive capacity.\n");
  2871. return -ENOTTY;
  2872. }
  2873. geo->heads = 224;
  2874. geo->sectors = 56;
  2875. sector_div(capacity, (geo->heads * geo->sectors));
  2876. geo->cylinders = capacity;
  2877. return 0;
  2878. }
  2879. static int mtip_block_open(struct block_device *dev, fmode_t mode)
  2880. {
  2881. struct driver_data *dd;
  2882. if (dev && dev->bd_disk) {
  2883. dd = (struct driver_data *) dev->bd_disk->private_data;
  2884. if (dd) {
  2885. if (test_bit(MTIP_DDF_REMOVAL_BIT,
  2886. &dd->dd_flag)) {
  2887. return -ENODEV;
  2888. }
  2889. return 0;
  2890. }
  2891. }
  2892. return -ENODEV;
  2893. }
  2894. static void mtip_block_release(struct gendisk *disk, fmode_t mode)
  2895. {
  2896. }
  2897. /*
  2898. * Block device operation function.
  2899. *
  2900. * This structure contains pointers to the functions required by the block
  2901. * layer.
  2902. */
  2903. static const struct block_device_operations mtip_block_ops = {
  2904. .open = mtip_block_open,
  2905. .release = mtip_block_release,
  2906. .ioctl = mtip_block_ioctl,
  2907. #ifdef CONFIG_COMPAT
  2908. .compat_ioctl = mtip_block_compat_ioctl,
  2909. #endif
  2910. .getgeo = mtip_block_getgeo,
  2911. .owner = THIS_MODULE
  2912. };
  2913. static inline bool is_se_active(struct driver_data *dd)
  2914. {
  2915. if (unlikely(test_bit(MTIP_PF_SE_ACTIVE_BIT, &dd->port->flags))) {
  2916. if (dd->port->ic_pause_timer) {
  2917. unsigned long to = dd->port->ic_pause_timer +
  2918. msecs_to_jiffies(1000);
  2919. if (time_after(jiffies, to)) {
  2920. clear_bit(MTIP_PF_SE_ACTIVE_BIT,
  2921. &dd->port->flags);
  2922. clear_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag);
  2923. dd->port->ic_pause_timer = 0;
  2924. wake_up_interruptible(&dd->port->svc_wait);
  2925. return false;
  2926. }
  2927. }
  2928. return true;
  2929. }
  2930. return false;
  2931. }
  2932. static inline bool is_stopped(struct driver_data *dd, struct request *rq)
  2933. {
  2934. if (likely(!(dd->dd_flag & MTIP_DDF_STOP_IO)))
  2935. return false;
  2936. if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))
  2937. return true;
  2938. if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag))
  2939. return true;
  2940. if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag) &&
  2941. rq_data_dir(rq))
  2942. return true;
  2943. if (test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag))
  2944. return true;
  2945. if (test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag))
  2946. return true;
  2947. return false;
  2948. }
  2949. static bool mtip_check_unal_depth(struct blk_mq_hw_ctx *hctx,
  2950. struct request *rq)
  2951. {
  2952. struct driver_data *dd = hctx->queue->queuedata;
  2953. struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
  2954. if (rq_data_dir(rq) == READ || !dd->unal_qdepth)
  2955. return false;
  2956. /*
  2957. * If unaligned depth must be limited on this controller, mark it
  2958. * as unaligned if the IO isn't on a 4k boundary (start of length).
  2959. */
  2960. if (blk_rq_sectors(rq) <= 64) {
  2961. if ((blk_rq_pos(rq) & 7) || (blk_rq_sectors(rq) & 7))
  2962. cmd->unaligned = 1;
  2963. }
  2964. if (cmd->unaligned && atomic_dec_if_positive(&dd->port->cmd_slot_unal) >= 0)
  2965. return true;
  2966. return false;
  2967. }
  2968. static blk_status_t mtip_issue_reserved_cmd(struct blk_mq_hw_ctx *hctx,
  2969. struct request *rq)
  2970. {
  2971. struct driver_data *dd = hctx->queue->queuedata;
  2972. struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
  2973. struct mtip_int_cmd *icmd = cmd->icmd;
  2974. struct mtip_cmd_hdr *hdr =
  2975. dd->port->command_list + sizeof(struct mtip_cmd_hdr) * rq->tag;
  2976. struct mtip_cmd_sg *command_sg;
  2977. if (mtip_commands_active(dd->port))
  2978. return BLK_STS_DEV_RESOURCE;
  2979. hdr->ctba = cpu_to_le32(cmd->command_dma & 0xFFFFFFFF);
  2980. if (test_bit(MTIP_PF_HOST_CAP_64, &dd->port->flags))
  2981. hdr->ctbau = cpu_to_le32((cmd->command_dma >> 16) >> 16);
  2982. /* Populate the SG list */
  2983. hdr->opts = cpu_to_le32(icmd->opts | icmd->fis_len);
  2984. if (icmd->buf_len) {
  2985. command_sg = cmd->command + AHCI_CMD_TBL_HDR_SZ;
  2986. command_sg->info = cpu_to_le32((icmd->buf_len-1) & 0x3FFFFF);
  2987. command_sg->dba = cpu_to_le32(icmd->buffer & 0xFFFFFFFF);
  2988. command_sg->dba_upper =
  2989. cpu_to_le32((icmd->buffer >> 16) >> 16);
  2990. hdr->opts |= cpu_to_le32((1 << 16));
  2991. }
  2992. /* Populate the command header */
  2993. hdr->byte_count = 0;
  2994. blk_mq_start_request(rq);
  2995. mtip_issue_non_ncq_command(dd->port, rq->tag);
  2996. return 0;
  2997. }
  2998. static blk_status_t mtip_queue_rq(struct blk_mq_hw_ctx *hctx,
  2999. const struct blk_mq_queue_data *bd)
  3000. {
  3001. struct driver_data *dd = hctx->queue->queuedata;
  3002. struct request *rq = bd->rq;
  3003. struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
  3004. if (blk_rq_is_passthrough(rq))
  3005. return mtip_issue_reserved_cmd(hctx, rq);
  3006. if (unlikely(mtip_check_unal_depth(hctx, rq)))
  3007. return BLK_STS_DEV_RESOURCE;
  3008. if (is_se_active(dd) || is_stopped(dd, rq))
  3009. return BLK_STS_IOERR;
  3010. blk_mq_start_request(rq);
  3011. mtip_hw_submit_io(dd, rq, cmd, hctx);
  3012. return BLK_STS_OK;
  3013. }
  3014. static void mtip_free_cmd(struct blk_mq_tag_set *set, struct request *rq,
  3015. unsigned int hctx_idx)
  3016. {
  3017. struct driver_data *dd = set->driver_data;
  3018. struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
  3019. if (!cmd->command)
  3020. return;
  3021. dma_free_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, cmd->command,
  3022. cmd->command_dma);
  3023. }
  3024. static int mtip_init_cmd(struct blk_mq_tag_set *set, struct request *rq,
  3025. unsigned int hctx_idx, unsigned int numa_node)
  3026. {
  3027. struct driver_data *dd = set->driver_data;
  3028. struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
  3029. cmd->command = dma_alloc_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ,
  3030. &cmd->command_dma, GFP_KERNEL);
  3031. if (!cmd->command)
  3032. return -ENOMEM;
  3033. sg_init_table(cmd->sg, MTIP_MAX_SG);
  3034. return 0;
  3035. }
  3036. static enum blk_eh_timer_return mtip_cmd_timeout(struct request *req,
  3037. bool reserved)
  3038. {
  3039. struct driver_data *dd = req->q->queuedata;
  3040. if (reserved) {
  3041. struct mtip_cmd *cmd = blk_mq_rq_to_pdu(req);
  3042. cmd->status = BLK_STS_TIMEOUT;
  3043. blk_mq_complete_request(req);
  3044. return BLK_EH_DONE;
  3045. }
  3046. if (test_bit(req->tag, dd->port->cmds_to_issue))
  3047. goto exit_handler;
  3048. if (test_and_set_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags))
  3049. goto exit_handler;
  3050. wake_up_interruptible(&dd->port->svc_wait);
  3051. exit_handler:
  3052. return BLK_EH_RESET_TIMER;
  3053. }
  3054. static const struct blk_mq_ops mtip_mq_ops = {
  3055. .queue_rq = mtip_queue_rq,
  3056. .init_request = mtip_init_cmd,
  3057. .exit_request = mtip_free_cmd,
  3058. .complete = mtip_softirq_done_fn,
  3059. .timeout = mtip_cmd_timeout,
  3060. };
  3061. /*
  3062. * Block layer initialization function.
  3063. *
  3064. * This function is called once by the PCI layer for each P320
  3065. * device that is connected to the system.
  3066. *
  3067. * @dd Pointer to the driver data structure.
  3068. *
  3069. * return value
  3070. * 0 on success else an error code.
  3071. */
  3072. static int mtip_block_initialize(struct driver_data *dd)
  3073. {
  3074. int rv = 0, wait_for_rebuild = 0;
  3075. sector_t capacity;
  3076. unsigned int index = 0;
  3077. struct kobject *kobj;
  3078. if (dd->disk)
  3079. goto skip_create_disk; /* hw init done, before rebuild */
  3080. if (mtip_hw_init(dd)) {
  3081. rv = -EINVAL;
  3082. goto protocol_init_error;
  3083. }
  3084. dd->disk = alloc_disk_node(MTIP_MAX_MINORS, dd->numa_node);
  3085. if (dd->disk == NULL) {
  3086. dev_err(&dd->pdev->dev,
  3087. "Unable to allocate gendisk structure\n");
  3088. rv = -EINVAL;
  3089. goto alloc_disk_error;
  3090. }
  3091. rv = ida_alloc(&rssd_index_ida, GFP_KERNEL);
  3092. if (rv < 0)
  3093. goto ida_get_error;
  3094. index = rv;
  3095. rv = rssd_disk_name_format("rssd",
  3096. index,
  3097. dd->disk->disk_name,
  3098. DISK_NAME_LEN);
  3099. if (rv)
  3100. goto disk_index_error;
  3101. dd->disk->major = dd->major;
  3102. dd->disk->first_minor = index * MTIP_MAX_MINORS;
  3103. dd->disk->minors = MTIP_MAX_MINORS;
  3104. dd->disk->fops = &mtip_block_ops;
  3105. dd->disk->private_data = dd;
  3106. dd->index = index;
  3107. mtip_hw_debugfs_init(dd);
  3108. memset(&dd->tags, 0, sizeof(dd->tags));
  3109. dd->tags.ops = &mtip_mq_ops;
  3110. dd->tags.nr_hw_queues = 1;
  3111. dd->tags.queue_depth = MTIP_MAX_COMMAND_SLOTS;
  3112. dd->tags.reserved_tags = 1;
  3113. dd->tags.cmd_size = sizeof(struct mtip_cmd);
  3114. dd->tags.numa_node = dd->numa_node;
  3115. dd->tags.flags = BLK_MQ_F_SHOULD_MERGE;
  3116. dd->tags.driver_data = dd;
  3117. dd->tags.timeout = MTIP_NCQ_CMD_TIMEOUT_MS;
  3118. rv = blk_mq_alloc_tag_set(&dd->tags);
  3119. if (rv) {
  3120. dev_err(&dd->pdev->dev,
  3121. "Unable to allocate request queue\n");
  3122. goto block_queue_alloc_tag_error;
  3123. }
  3124. /* Allocate the request queue. */
  3125. dd->queue = blk_mq_init_queue(&dd->tags);
  3126. if (IS_ERR(dd->queue)) {
  3127. dev_err(&dd->pdev->dev,
  3128. "Unable to allocate request queue\n");
  3129. rv = -ENOMEM;
  3130. goto block_queue_alloc_init_error;
  3131. }
  3132. dd->disk->queue = dd->queue;
  3133. dd->queue->queuedata = dd;
  3134. skip_create_disk:
  3135. /* Initialize the protocol layer. */
  3136. wait_for_rebuild = mtip_hw_get_identify(dd);
  3137. if (wait_for_rebuild < 0) {
  3138. dev_err(&dd->pdev->dev,
  3139. "Protocol layer initialization failed\n");
  3140. rv = -EINVAL;
  3141. goto init_hw_cmds_error;
  3142. }
  3143. /*
  3144. * if rebuild pending, start the service thread, and delay the block
  3145. * queue creation and device_add_disk()
  3146. */
  3147. if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC)
  3148. goto start_service_thread;
  3149. /* Set device limits. */
  3150. blk_queue_flag_set(QUEUE_FLAG_NONROT, dd->queue);
  3151. blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, dd->queue);
  3152. blk_queue_max_segments(dd->queue, MTIP_MAX_SG);
  3153. blk_queue_physical_block_size(dd->queue, 4096);
  3154. blk_queue_max_hw_sectors(dd->queue, 0xffff);
  3155. blk_queue_max_segment_size(dd->queue, 0x400000);
  3156. dma_set_max_seg_size(&dd->pdev->dev, 0x400000);
  3157. blk_queue_io_min(dd->queue, 4096);
  3158. /* Set the capacity of the device in 512 byte sectors. */
  3159. if (!(mtip_hw_get_capacity(dd, &capacity))) {
  3160. dev_warn(&dd->pdev->dev,
  3161. "Could not read drive capacity\n");
  3162. rv = -EIO;
  3163. goto read_capacity_error;
  3164. }
  3165. set_capacity(dd->disk, capacity);
  3166. /* Enable the block device and add it to /dev */
  3167. device_add_disk(&dd->pdev->dev, dd->disk, NULL);
  3168. dd->bdev = bdget_disk(dd->disk, 0);
  3169. /*
  3170. * Now that the disk is active, initialize any sysfs attributes
  3171. * managed by the protocol layer.
  3172. */
  3173. kobj = kobject_get(&disk_to_dev(dd->disk)->kobj);
  3174. if (kobj) {
  3175. mtip_hw_sysfs_init(dd, kobj);
  3176. kobject_put(kobj);
  3177. }
  3178. if (dd->mtip_svc_handler) {
  3179. set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag);
  3180. return rv; /* service thread created for handling rebuild */
  3181. }
  3182. start_service_thread:
  3183. dd->mtip_svc_handler = kthread_create_on_node(mtip_service_thread,
  3184. dd, dd->numa_node,
  3185. "mtip_svc_thd_%02d", index);
  3186. if (IS_ERR(dd->mtip_svc_handler)) {
  3187. dev_err(&dd->pdev->dev, "service thread failed to start\n");
  3188. dd->mtip_svc_handler = NULL;
  3189. rv = -EFAULT;
  3190. goto kthread_run_error;
  3191. }
  3192. wake_up_process(dd->mtip_svc_handler);
  3193. if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC)
  3194. rv = wait_for_rebuild;
  3195. return rv;
  3196. kthread_run_error:
  3197. bdput(dd->bdev);
  3198. dd->bdev = NULL;
  3199. /* Delete our gendisk. This also removes the device from /dev */
  3200. del_gendisk(dd->disk);
  3201. read_capacity_error:
  3202. init_hw_cmds_error:
  3203. blk_cleanup_queue(dd->queue);
  3204. block_queue_alloc_init_error:
  3205. blk_mq_free_tag_set(&dd->tags);
  3206. block_queue_alloc_tag_error:
  3207. mtip_hw_debugfs_exit(dd);
  3208. disk_index_error:
  3209. ida_free(&rssd_index_ida, index);
  3210. ida_get_error:
  3211. put_disk(dd->disk);
  3212. alloc_disk_error:
  3213. mtip_hw_exit(dd); /* De-initialize the protocol layer. */
  3214. protocol_init_error:
  3215. return rv;
  3216. }
  3217. static bool mtip_no_dev_cleanup(struct request *rq, void *data, bool reserv)
  3218. {
  3219. struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
  3220. cmd->status = BLK_STS_IOERR;
  3221. blk_mq_complete_request(rq);
  3222. return true;
  3223. }
  3224. /*
  3225. * Block layer deinitialization function.
  3226. *
  3227. * Called by the PCI layer as each P320 device is removed.
  3228. *
  3229. * @dd Pointer to the driver data structure.
  3230. *
  3231. * return value
  3232. * 0
  3233. */
  3234. static int mtip_block_remove(struct driver_data *dd)
  3235. {
  3236. struct kobject *kobj;
  3237. mtip_hw_debugfs_exit(dd);
  3238. if (dd->mtip_svc_handler) {
  3239. set_bit(MTIP_PF_SVC_THD_STOP_BIT, &dd->port->flags);
  3240. wake_up_interruptible(&dd->port->svc_wait);
  3241. kthread_stop(dd->mtip_svc_handler);
  3242. }
  3243. /* Clean up the sysfs attributes, if created */
  3244. if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) {
  3245. kobj = kobject_get(&disk_to_dev(dd->disk)->kobj);
  3246. if (kobj) {
  3247. mtip_hw_sysfs_exit(dd, kobj);
  3248. kobject_put(kobj);
  3249. }
  3250. }
  3251. if (!dd->sr) {
  3252. /*
  3253. * Explicitly wait here for IOs to quiesce,
  3254. * as mtip_standby_drive usually won't wait for IOs.
  3255. */
  3256. if (!mtip_quiesce_io(dd->port, MTIP_QUIESCE_IO_TIMEOUT_MS))
  3257. mtip_standby_drive(dd);
  3258. }
  3259. else
  3260. dev_info(&dd->pdev->dev, "device %s surprise removal\n",
  3261. dd->disk->disk_name);
  3262. blk_freeze_queue_start(dd->queue);
  3263. blk_mq_quiesce_queue(dd->queue);
  3264. blk_mq_tagset_busy_iter(&dd->tags, mtip_no_dev_cleanup, dd);
  3265. blk_mq_unquiesce_queue(dd->queue);
  3266. /*
  3267. * Delete our gendisk structure. This also removes the device
  3268. * from /dev
  3269. */
  3270. if (dd->bdev) {
  3271. bdput(dd->bdev);
  3272. dd->bdev = NULL;
  3273. }
  3274. if (dd->disk) {
  3275. if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag))
  3276. del_gendisk(dd->disk);
  3277. if (dd->disk->queue) {
  3278. blk_cleanup_queue(dd->queue);
  3279. blk_mq_free_tag_set(&dd->tags);
  3280. dd->queue = NULL;
  3281. }
  3282. put_disk(dd->disk);
  3283. }
  3284. dd->disk = NULL;
  3285. ida_free(&rssd_index_ida, dd->index);
  3286. /* De-initialize the protocol layer. */
  3287. mtip_hw_exit(dd);
  3288. return 0;
  3289. }
  3290. /*
  3291. * Function called by the PCI layer when just before the
  3292. * machine shuts down.
  3293. *
  3294. * If a protocol layer shutdown function is present it will be called
  3295. * by this function.
  3296. *
  3297. * @dd Pointer to the driver data structure.
  3298. *
  3299. * return value
  3300. * 0
  3301. */
  3302. static int mtip_block_shutdown(struct driver_data *dd)
  3303. {
  3304. mtip_hw_shutdown(dd);
  3305. /* Delete our gendisk structure, and cleanup the blk queue. */
  3306. if (dd->disk) {
  3307. dev_info(&dd->pdev->dev,
  3308. "Shutting down %s ...\n", dd->disk->disk_name);
  3309. if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag))
  3310. del_gendisk(dd->disk);
  3311. if (dd->disk->queue) {
  3312. blk_cleanup_queue(dd->queue);
  3313. blk_mq_free_tag_set(&dd->tags);
  3314. }
  3315. put_disk(dd->disk);
  3316. dd->disk = NULL;
  3317. dd->queue = NULL;
  3318. }
  3319. ida_free(&rssd_index_ida, dd->index);
  3320. return 0;
  3321. }
  3322. static int mtip_block_suspend(struct driver_data *dd)
  3323. {
  3324. dev_info(&dd->pdev->dev,
  3325. "Suspending %s ...\n", dd->disk->disk_name);
  3326. mtip_hw_suspend(dd);
  3327. return 0;
  3328. }
  3329. static int mtip_block_resume(struct driver_data *dd)
  3330. {
  3331. dev_info(&dd->pdev->dev, "Resuming %s ...\n",
  3332. dd->disk->disk_name);
  3333. mtip_hw_resume(dd);
  3334. return 0;
  3335. }
  3336. static void drop_cpu(int cpu)
  3337. {
  3338. cpu_use[cpu]--;
  3339. }
  3340. static int get_least_used_cpu_on_node(int node)
  3341. {
  3342. int cpu, least_used_cpu, least_cnt;
  3343. const struct cpumask *node_mask;
  3344. node_mask = cpumask_of_node(node);
  3345. least_used_cpu = cpumask_first(node_mask);
  3346. least_cnt = cpu_use[least_used_cpu];
  3347. cpu = least_used_cpu;
  3348. for_each_cpu(cpu, node_mask) {
  3349. if (cpu_use[cpu] < least_cnt) {
  3350. least_used_cpu = cpu;
  3351. least_cnt = cpu_use[cpu];
  3352. }
  3353. }
  3354. cpu_use[least_used_cpu]++;
  3355. return least_used_cpu;
  3356. }
  3357. /* Helper for selecting a node in round robin mode */
  3358. static inline int mtip_get_next_rr_node(void)
  3359. {
  3360. static int next_node = NUMA_NO_NODE;
  3361. if (next_node == NUMA_NO_NODE) {
  3362. next_node = first_online_node;
  3363. return next_node;
  3364. }
  3365. next_node = next_online_node(next_node);
  3366. if (next_node == MAX_NUMNODES)
  3367. next_node = first_online_node;
  3368. return next_node;
  3369. }
  3370. static DEFINE_HANDLER(0);
  3371. static DEFINE_HANDLER(1);
  3372. static DEFINE_HANDLER(2);
  3373. static DEFINE_HANDLER(3);
  3374. static DEFINE_HANDLER(4);
  3375. static DEFINE_HANDLER(5);
  3376. static DEFINE_HANDLER(6);
  3377. static DEFINE_HANDLER(7);
  3378. static void mtip_disable_link_opts(struct driver_data *dd, struct pci_dev *pdev)
  3379. {
  3380. int pos;
  3381. unsigned short pcie_dev_ctrl;
  3382. pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
  3383. if (pos) {
  3384. pci_read_config_word(pdev,
  3385. pos + PCI_EXP_DEVCTL,
  3386. &pcie_dev_ctrl);
  3387. if (pcie_dev_ctrl & (1 << 11) ||
  3388. pcie_dev_ctrl & (1 << 4)) {
  3389. dev_info(&dd->pdev->dev,
  3390. "Disabling ERO/No-Snoop on bridge device %04x:%04x\n",
  3391. pdev->vendor, pdev->device);
  3392. pcie_dev_ctrl &= ~(PCI_EXP_DEVCTL_NOSNOOP_EN |
  3393. PCI_EXP_DEVCTL_RELAX_EN);
  3394. pci_write_config_word(pdev,
  3395. pos + PCI_EXP_DEVCTL,
  3396. pcie_dev_ctrl);
  3397. }
  3398. }
  3399. }
  3400. static void mtip_fix_ero_nosnoop(struct driver_data *dd, struct pci_dev *pdev)
  3401. {
  3402. /*
  3403. * This workaround is specific to AMD/ATI chipset with a PCI upstream
  3404. * device with device id 0x5aXX
  3405. */
  3406. if (pdev->bus && pdev->bus->self) {
  3407. if (pdev->bus->self->vendor == PCI_VENDOR_ID_ATI &&
  3408. ((pdev->bus->self->device & 0xff00) == 0x5a00)) {
  3409. mtip_disable_link_opts(dd, pdev->bus->self);
  3410. } else {
  3411. /* Check further up the topology */
  3412. struct pci_dev *parent_dev = pdev->bus->self;
  3413. if (parent_dev->bus &&
  3414. parent_dev->bus->parent &&
  3415. parent_dev->bus->parent->self &&
  3416. parent_dev->bus->parent->self->vendor ==
  3417. PCI_VENDOR_ID_ATI &&
  3418. (parent_dev->bus->parent->self->device &
  3419. 0xff00) == 0x5a00) {
  3420. mtip_disable_link_opts(dd,
  3421. parent_dev->bus->parent->self);
  3422. }
  3423. }
  3424. }
  3425. }
  3426. /*
  3427. * Called for each supported PCI device detected.
  3428. *
  3429. * This function allocates the private data structure, enables the
  3430. * PCI device and then calls the block layer initialization function.
  3431. *
  3432. * return value
  3433. * 0 on success else an error code.
  3434. */
  3435. static int mtip_pci_probe(struct pci_dev *pdev,
  3436. const struct pci_device_id *ent)
  3437. {
  3438. int rv = 0;
  3439. struct driver_data *dd = NULL;
  3440. char cpu_list[256];
  3441. const struct cpumask *node_mask;
  3442. int cpu, i = 0, j = 0;
  3443. int my_node = NUMA_NO_NODE;
  3444. unsigned long flags;
  3445. /* Allocate memory for this devices private data. */
  3446. my_node = pcibus_to_node(pdev->bus);
  3447. if (my_node != NUMA_NO_NODE) {
  3448. if (!node_online(my_node))
  3449. my_node = mtip_get_next_rr_node();
  3450. } else {
  3451. dev_info(&pdev->dev, "Kernel not reporting proximity, choosing a node\n");
  3452. my_node = mtip_get_next_rr_node();
  3453. }
  3454. dev_info(&pdev->dev, "NUMA node %d (closest: %d,%d, probe on %d:%d)\n",
  3455. my_node, pcibus_to_node(pdev->bus), dev_to_node(&pdev->dev),
  3456. cpu_to_node(raw_smp_processor_id()), raw_smp_processor_id());
  3457. dd = kzalloc_node(sizeof(struct driver_data), GFP_KERNEL, my_node);
  3458. if (dd == NULL) {
  3459. dev_err(&pdev->dev,
  3460. "Unable to allocate memory for driver data\n");
  3461. return -ENOMEM;
  3462. }
  3463. /* Attach the private data to this PCI device. */
  3464. pci_set_drvdata(pdev, dd);
  3465. rv = pcim_enable_device(pdev);
  3466. if (rv < 0) {
  3467. dev_err(&pdev->dev, "Unable to enable device\n");
  3468. goto iomap_err;
  3469. }
  3470. /* Map BAR5 to memory. */
  3471. rv = pcim_iomap_regions(pdev, 1 << MTIP_ABAR, MTIP_DRV_NAME);
  3472. if (rv < 0) {
  3473. dev_err(&pdev->dev, "Unable to map regions\n");
  3474. goto iomap_err;
  3475. }
  3476. rv = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
  3477. if (rv) {
  3478. dev_warn(&pdev->dev, "64-bit DMA enable failed\n");
  3479. goto setmask_err;
  3480. }
  3481. /* Copy the info we may need later into the private data structure. */
  3482. dd->major = mtip_major;
  3483. dd->instance = instance;
  3484. dd->pdev = pdev;
  3485. dd->numa_node = my_node;
  3486. INIT_LIST_HEAD(&dd->online_list);
  3487. INIT_LIST_HEAD(&dd->remove_list);
  3488. memset(dd->workq_name, 0, 32);
  3489. snprintf(dd->workq_name, 31, "mtipq%d", dd->instance);
  3490. dd->isr_workq = create_workqueue(dd->workq_name);
  3491. if (!dd->isr_workq) {
  3492. dev_warn(&pdev->dev, "Can't create wq %d\n", dd->instance);
  3493. rv = -ENOMEM;
  3494. goto setmask_err;
  3495. }
  3496. memset(cpu_list, 0, sizeof(cpu_list));
  3497. node_mask = cpumask_of_node(dd->numa_node);
  3498. if (!cpumask_empty(node_mask)) {
  3499. for_each_cpu(cpu, node_mask)
  3500. {
  3501. snprintf(&cpu_list[j], 256 - j, "%d ", cpu);
  3502. j = strlen(cpu_list);
  3503. }
  3504. dev_info(&pdev->dev, "Node %d on package %d has %d cpu(s): %s\n",
  3505. dd->numa_node,
  3506. topology_physical_package_id(cpumask_first(node_mask)),
  3507. nr_cpus_node(dd->numa_node),
  3508. cpu_list);
  3509. } else
  3510. dev_dbg(&pdev->dev, "mtip32xx: node_mask empty\n");
  3511. dd->isr_binding = get_least_used_cpu_on_node(dd->numa_node);
  3512. dev_info(&pdev->dev, "Initial IRQ binding node:cpu %d:%d\n",
  3513. cpu_to_node(dd->isr_binding), dd->isr_binding);
  3514. /* first worker context always runs in ISR */
  3515. dd->work[0].cpu_binding = dd->isr_binding;
  3516. dd->work[1].cpu_binding = get_least_used_cpu_on_node(dd->numa_node);
  3517. dd->work[2].cpu_binding = get_least_used_cpu_on_node(dd->numa_node);
  3518. dd->work[3].cpu_binding = dd->work[0].cpu_binding;
  3519. dd->work[4].cpu_binding = dd->work[1].cpu_binding;
  3520. dd->work[5].cpu_binding = dd->work[2].cpu_binding;
  3521. dd->work[6].cpu_binding = dd->work[2].cpu_binding;
  3522. dd->work[7].cpu_binding = dd->work[1].cpu_binding;
  3523. /* Log the bindings */
  3524. for_each_present_cpu(cpu) {
  3525. memset(cpu_list, 0, sizeof(cpu_list));
  3526. for (i = 0, j = 0; i < MTIP_MAX_SLOT_GROUPS; i++) {
  3527. if (dd->work[i].cpu_binding == cpu) {
  3528. snprintf(&cpu_list[j], 256 - j, "%d ", i);
  3529. j = strlen(cpu_list);
  3530. }
  3531. }
  3532. if (j)
  3533. dev_info(&pdev->dev, "CPU %d: WQs %s\n", cpu, cpu_list);
  3534. }
  3535. INIT_WORK(&dd->work[0].work, mtip_workq_sdbf0);
  3536. INIT_WORK(&dd->work[1].work, mtip_workq_sdbf1);
  3537. INIT_WORK(&dd->work[2].work, mtip_workq_sdbf2);
  3538. INIT_WORK(&dd->work[3].work, mtip_workq_sdbf3);
  3539. INIT_WORK(&dd->work[4].work, mtip_workq_sdbf4);
  3540. INIT_WORK(&dd->work[5].work, mtip_workq_sdbf5);
  3541. INIT_WORK(&dd->work[6].work, mtip_workq_sdbf6);
  3542. INIT_WORK(&dd->work[7].work, mtip_workq_sdbf7);
  3543. pci_set_master(pdev);
  3544. rv = pci_enable_msi(pdev);
  3545. if (rv) {
  3546. dev_warn(&pdev->dev,
  3547. "Unable to enable MSI interrupt.\n");
  3548. goto msi_initialize_err;
  3549. }
  3550. mtip_fix_ero_nosnoop(dd, pdev);
  3551. /* Initialize the block layer. */
  3552. rv = mtip_block_initialize(dd);
  3553. if (rv < 0) {
  3554. dev_err(&pdev->dev,
  3555. "Unable to initialize block layer\n");
  3556. goto block_initialize_err;
  3557. }
  3558. /*
  3559. * Increment the instance count so that each device has a unique
  3560. * instance number.
  3561. */
  3562. instance++;
  3563. if (rv != MTIP_FTL_REBUILD_MAGIC)
  3564. set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag);
  3565. else
  3566. rv = 0; /* device in rebuild state, return 0 from probe */
  3567. /* Add to online list even if in ftl rebuild */
  3568. spin_lock_irqsave(&dev_lock, flags);
  3569. list_add(&dd->online_list, &online_list);
  3570. spin_unlock_irqrestore(&dev_lock, flags);
  3571. goto done;
  3572. block_initialize_err:
  3573. pci_disable_msi(pdev);
  3574. msi_initialize_err:
  3575. if (dd->isr_workq) {
  3576. flush_workqueue(dd->isr_workq);
  3577. destroy_workqueue(dd->isr_workq);
  3578. drop_cpu(dd->work[0].cpu_binding);
  3579. drop_cpu(dd->work[1].cpu_binding);
  3580. drop_cpu(dd->work[2].cpu_binding);
  3581. }
  3582. setmask_err:
  3583. pcim_iounmap_regions(pdev, 1 << MTIP_ABAR);
  3584. iomap_err:
  3585. kfree(dd);
  3586. pci_set_drvdata(pdev, NULL);
  3587. return rv;
  3588. done:
  3589. return rv;
  3590. }
  3591. /*
  3592. * Called for each probed device when the device is removed or the
  3593. * driver is unloaded.
  3594. *
  3595. * return value
  3596. * None
  3597. */
  3598. static void mtip_pci_remove(struct pci_dev *pdev)
  3599. {
  3600. struct driver_data *dd = pci_get_drvdata(pdev);
  3601. unsigned long flags, to;
  3602. set_bit(MTIP_DDF_REMOVAL_BIT, &dd->dd_flag);
  3603. spin_lock_irqsave(&dev_lock, flags);
  3604. list_del_init(&dd->online_list);
  3605. list_add(&dd->remove_list, &removing_list);
  3606. spin_unlock_irqrestore(&dev_lock, flags);
  3607. mtip_check_surprise_removal(pdev);
  3608. synchronize_irq(dd->pdev->irq);
  3609. /* Spin until workers are done */
  3610. to = jiffies + msecs_to_jiffies(4000);
  3611. do {
  3612. msleep(20);
  3613. } while (atomic_read(&dd->irq_workers_active) != 0 &&
  3614. time_before(jiffies, to));
  3615. if (!dd->sr)
  3616. fsync_bdev(dd->bdev);
  3617. if (atomic_read(&dd->irq_workers_active) != 0) {
  3618. dev_warn(&dd->pdev->dev,
  3619. "Completion workers still active!\n");
  3620. }
  3621. blk_set_queue_dying(dd->queue);
  3622. set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag);
  3623. /* Clean up the block layer. */
  3624. mtip_block_remove(dd);
  3625. if (dd->isr_workq) {
  3626. flush_workqueue(dd->isr_workq);
  3627. destroy_workqueue(dd->isr_workq);
  3628. drop_cpu(dd->work[0].cpu_binding);
  3629. drop_cpu(dd->work[1].cpu_binding);
  3630. drop_cpu(dd->work[2].cpu_binding);
  3631. }
  3632. pci_disable_msi(pdev);
  3633. spin_lock_irqsave(&dev_lock, flags);
  3634. list_del_init(&dd->remove_list);
  3635. spin_unlock_irqrestore(&dev_lock, flags);
  3636. kfree(dd);
  3637. pcim_iounmap_regions(pdev, 1 << MTIP_ABAR);
  3638. pci_set_drvdata(pdev, NULL);
  3639. }
  3640. /*
  3641. * Called for each probed device when the device is suspended.
  3642. *
  3643. * return value
  3644. * 0 Success
  3645. * <0 Error
  3646. */
  3647. static int mtip_pci_suspend(struct pci_dev *pdev, pm_message_t mesg)
  3648. {
  3649. int rv = 0;
  3650. struct driver_data *dd = pci_get_drvdata(pdev);
  3651. if (!dd) {
  3652. dev_err(&pdev->dev,
  3653. "Driver private datastructure is NULL\n");
  3654. return -EFAULT;
  3655. }
  3656. set_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag);
  3657. /* Disable ports & interrupts then send standby immediate */
  3658. rv = mtip_block_suspend(dd);
  3659. if (rv < 0) {
  3660. dev_err(&pdev->dev,
  3661. "Failed to suspend controller\n");
  3662. return rv;
  3663. }
  3664. /*
  3665. * Save the pci config space to pdev structure &
  3666. * disable the device
  3667. */
  3668. pci_save_state(pdev);
  3669. pci_disable_device(pdev);
  3670. /* Move to Low power state*/
  3671. pci_set_power_state(pdev, PCI_D3hot);
  3672. return rv;
  3673. }
  3674. /*
  3675. * Called for each probed device when the device is resumed.
  3676. *
  3677. * return value
  3678. * 0 Success
  3679. * <0 Error
  3680. */
  3681. static int mtip_pci_resume(struct pci_dev *pdev)
  3682. {
  3683. int rv = 0;
  3684. struct driver_data *dd;
  3685. dd = pci_get_drvdata(pdev);
  3686. if (!dd) {
  3687. dev_err(&pdev->dev,
  3688. "Driver private datastructure is NULL\n");
  3689. return -EFAULT;
  3690. }
  3691. /* Move the device to active State */
  3692. pci_set_power_state(pdev, PCI_D0);
  3693. /* Restore PCI configuration space */
  3694. pci_restore_state(pdev);
  3695. /* Enable the PCI device*/
  3696. rv = pcim_enable_device(pdev);
  3697. if (rv < 0) {
  3698. dev_err(&pdev->dev,
  3699. "Failed to enable card during resume\n");
  3700. goto err;
  3701. }
  3702. pci_set_master(pdev);
  3703. /*
  3704. * Calls hbaReset, initPort, & startPort function
  3705. * then enables interrupts
  3706. */
  3707. rv = mtip_block_resume(dd);
  3708. if (rv < 0)
  3709. dev_err(&pdev->dev, "Unable to resume\n");
  3710. err:
  3711. clear_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag);
  3712. return rv;
  3713. }
  3714. /*
  3715. * Shutdown routine
  3716. *
  3717. * return value
  3718. * None
  3719. */
  3720. static void mtip_pci_shutdown(struct pci_dev *pdev)
  3721. {
  3722. struct driver_data *dd = pci_get_drvdata(pdev);
  3723. if (dd)
  3724. mtip_block_shutdown(dd);
  3725. }
  3726. /* Table of device ids supported by this driver. */
  3727. static const struct pci_device_id mtip_pci_tbl[] = {
  3728. { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320H_DEVICE_ID) },
  3729. { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320M_DEVICE_ID) },
  3730. { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320S_DEVICE_ID) },
  3731. { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P325M_DEVICE_ID) },
  3732. { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P420H_DEVICE_ID) },
  3733. { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P420M_DEVICE_ID) },
  3734. { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P425M_DEVICE_ID) },
  3735. { 0 }
  3736. };
  3737. /* Structure that describes the PCI driver functions. */
  3738. static struct pci_driver mtip_pci_driver = {
  3739. .name = MTIP_DRV_NAME,
  3740. .id_table = mtip_pci_tbl,
  3741. .probe = mtip_pci_probe,
  3742. .remove = mtip_pci_remove,
  3743. .suspend = mtip_pci_suspend,
  3744. .resume = mtip_pci_resume,
  3745. .shutdown = mtip_pci_shutdown,
  3746. };
  3747. MODULE_DEVICE_TABLE(pci, mtip_pci_tbl);
  3748. /*
  3749. * Module initialization function.
  3750. *
  3751. * Called once when the module is loaded. This function allocates a major
  3752. * block device number to the Cyclone devices and registers the PCI layer
  3753. * of the driver.
  3754. *
  3755. * Return value
  3756. * 0 on success else error code.
  3757. */
  3758. static int __init mtip_init(void)
  3759. {
  3760. int error;
  3761. pr_info(MTIP_DRV_NAME " Version " MTIP_DRV_VERSION "\n");
  3762. spin_lock_init(&dev_lock);
  3763. INIT_LIST_HEAD(&online_list);
  3764. INIT_LIST_HEAD(&removing_list);
  3765. /* Allocate a major block device number to use with this driver. */
  3766. error = register_blkdev(0, MTIP_DRV_NAME);
  3767. if (error <= 0) {
  3768. pr_err("Unable to register block device (%d)\n",
  3769. error);
  3770. return -EBUSY;
  3771. }
  3772. mtip_major = error;
  3773. dfs_parent = debugfs_create_dir("rssd", NULL);
  3774. if (IS_ERR_OR_NULL(dfs_parent)) {
  3775. pr_warn("Error creating debugfs parent\n");
  3776. dfs_parent = NULL;
  3777. }
  3778. if (dfs_parent) {
  3779. dfs_device_status = debugfs_create_file("device_status",
  3780. 0444, dfs_parent, NULL,
  3781. &mtip_device_status_fops);
  3782. if (IS_ERR_OR_NULL(dfs_device_status)) {
  3783. pr_err("Error creating device_status node\n");
  3784. dfs_device_status = NULL;
  3785. }
  3786. }
  3787. /* Register our PCI operations. */
  3788. error = pci_register_driver(&mtip_pci_driver);
  3789. if (error) {
  3790. debugfs_remove(dfs_parent);
  3791. unregister_blkdev(mtip_major, MTIP_DRV_NAME);
  3792. }
  3793. return error;
  3794. }
  3795. /*
  3796. * Module de-initialization function.
  3797. *
  3798. * Called once when the module is unloaded. This function deallocates
  3799. * the major block device number allocated by mtip_init() and
  3800. * unregisters the PCI layer of the driver.
  3801. *
  3802. * Return value
  3803. * none
  3804. */
  3805. static void __exit mtip_exit(void)
  3806. {
  3807. /* Release the allocated major block device number. */
  3808. unregister_blkdev(mtip_major, MTIP_DRV_NAME);
  3809. /* Unregister the PCI driver. */
  3810. pci_unregister_driver(&mtip_pci_driver);
  3811. debugfs_remove_recursive(dfs_parent);
  3812. }
  3813. MODULE_AUTHOR("Micron Technology, Inc");
  3814. MODULE_DESCRIPTION("Micron RealSSD PCIe Block Driver");
  3815. MODULE_LICENSE("GPL");
  3816. MODULE_VERSION(MTIP_DRV_VERSION);
  3817. module_init(mtip_init);
  3818. module_exit(mtip_exit);