sequencer.c 112 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998
  1. // SPDX-License-Identifier: BSD-3-Clause
  2. /*
  3. * Copyright Altera Corporation (C) 2012-2015
  4. */
  5. #include <common.h>
  6. #include <log.h>
  7. #include <asm/io.h>
  8. #include <asm/arch/sdram.h>
  9. #include <errno.h>
  10. #include <hang.h>
  11. #include "sequencer.h"
  12. static const struct socfpga_sdr_rw_load_manager *sdr_rw_load_mgr_regs =
  13. (struct socfpga_sdr_rw_load_manager *)
  14. (SDR_PHYGRP_RWMGRGRP_ADDRESS | 0x800);
  15. static const struct socfpga_sdr_rw_load_jump_manager *sdr_rw_load_jump_mgr_regs
  16. = (struct socfpga_sdr_rw_load_jump_manager *)
  17. (SDR_PHYGRP_RWMGRGRP_ADDRESS | 0xC00);
  18. static const struct socfpga_sdr_reg_file *sdr_reg_file =
  19. (struct socfpga_sdr_reg_file *)SDR_PHYGRP_REGFILEGRP_ADDRESS;
  20. static const struct socfpga_sdr_scc_mgr *sdr_scc_mgr =
  21. (struct socfpga_sdr_scc_mgr *)
  22. (SDR_PHYGRP_SCCGRP_ADDRESS | 0xe00);
  23. static const struct socfpga_phy_mgr_cmd *phy_mgr_cmd =
  24. (struct socfpga_phy_mgr_cmd *)SDR_PHYGRP_PHYMGRGRP_ADDRESS;
  25. static const struct socfpga_phy_mgr_cfg *phy_mgr_cfg =
  26. (struct socfpga_phy_mgr_cfg *)
  27. (SDR_PHYGRP_PHYMGRGRP_ADDRESS | 0x40);
  28. static const struct socfpga_data_mgr *data_mgr =
  29. (struct socfpga_data_mgr *)SDR_PHYGRP_DATAMGRGRP_ADDRESS;
  30. static const struct socfpga_sdr_ctrl *sdr_ctrl =
  31. (struct socfpga_sdr_ctrl *)SDR_CTRLGRP_ADDRESS;
  32. #define DELTA_D 1
  33. /*
  34. * In order to reduce ROM size, most of the selectable calibration steps are
  35. * decided at compile time based on the user's calibration mode selection,
  36. * as captured by the STATIC_CALIB_STEPS selection below.
  37. *
  38. * However, to support simulation-time selection of fast simulation mode, where
  39. * we skip everything except the bare minimum, we need a few of the steps to
  40. * be dynamic. In those cases, we either use the DYNAMIC_CALIB_STEPS for the
  41. * check, which is based on the rtl-supplied value, or we dynamically compute
  42. * the value to use based on the dynamically-chosen calibration mode
  43. */
  44. #define DLEVEL 0
  45. #define STATIC_IN_RTL_SIM 0
  46. #define STATIC_SKIP_DELAY_LOOPS 0
  47. #define STATIC_CALIB_STEPS (STATIC_IN_RTL_SIM | CALIB_SKIP_FULL_TEST | \
  48. STATIC_SKIP_DELAY_LOOPS)
  49. #define SKIP_DELAY_LOOP_VALUE_OR_ZERO(non_skip_value) \
  50. ((non_skip_value) & seq->skip_delay_mask)
  51. bool dram_is_ddr(const u8 ddr)
  52. {
  53. const struct socfpga_sdram_config *cfg = socfpga_get_sdram_config();
  54. const u8 type = (cfg->ctrl_cfg >> SDR_CTRLGRP_CTRLCFG_MEMTYPE_LSB) &
  55. SDR_CTRLGRP_CTRLCFG_MEMTYPE_MASK;
  56. if (ddr == 2 && type == 1) /* DDR2 */
  57. return true;
  58. if (ddr == 3 && type == 2) /* DDR3 */
  59. return true;
  60. return false;
  61. }
  62. static void set_failing_group_stage(struct socfpga_sdrseq *seq,
  63. u32 group, u32 stage, u32 substage)
  64. {
  65. /*
  66. * Only set the global stage if there was not been any other
  67. * failing group
  68. */
  69. if (seq->gbl.error_stage == CAL_STAGE_NIL) {
  70. seq->gbl.error_substage = substage;
  71. seq->gbl.error_stage = stage;
  72. seq->gbl.error_group = group;
  73. }
  74. }
  75. static void reg_file_set_group(u16 set_group)
  76. {
  77. clrsetbits_le32(&sdr_reg_file->cur_stage, 0xffff0000, set_group << 16);
  78. }
  79. static void reg_file_set_stage(u8 set_stage)
  80. {
  81. clrsetbits_le32(&sdr_reg_file->cur_stage, 0xffff, set_stage & 0xff);
  82. }
  83. static void reg_file_set_sub_stage(u8 set_sub_stage)
  84. {
  85. set_sub_stage &= 0xff;
  86. clrsetbits_le32(&sdr_reg_file->cur_stage, 0xff00, set_sub_stage << 8);
  87. }
  88. /**
  89. * phy_mgr_initialize() - Initialize PHY Manager
  90. *
  91. * Initialize PHY Manager.
  92. */
  93. static void phy_mgr_initialize(struct socfpga_sdrseq *seq)
  94. {
  95. u32 ratio;
  96. debug("%s:%d\n", __func__, __LINE__);
  97. /* Calibration has control over path to memory */
  98. /*
  99. * In Hard PHY this is a 2-bit control:
  100. * 0: AFI Mux Select
  101. * 1: DDIO Mux Select
  102. */
  103. writel(0x3, &phy_mgr_cfg->mux_sel);
  104. /* USER memory clock is not stable we begin initialization */
  105. writel(0, &phy_mgr_cfg->reset_mem_stbl);
  106. /* USER calibration status all set to zero */
  107. writel(0, &phy_mgr_cfg->cal_status);
  108. writel(0, &phy_mgr_cfg->cal_debug_info);
  109. /* Init params only if we do NOT skip calibration. */
  110. if ((seq->dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL)
  111. return;
  112. ratio = seq->rwcfg->mem_dq_per_read_dqs /
  113. seq->rwcfg->mem_virtual_groups_per_read_dqs;
  114. seq->param.read_correct_mask_vg = (1 << ratio) - 1;
  115. seq->param.write_correct_mask_vg = (1 << ratio) - 1;
  116. seq->param.read_correct_mask = (1 << seq->rwcfg->mem_dq_per_read_dqs)
  117. - 1;
  118. seq->param.write_correct_mask = (1 << seq->rwcfg->mem_dq_per_write_dqs)
  119. - 1;
  120. }
  121. /**
  122. * set_rank_and_odt_mask() - Set Rank and ODT mask
  123. * @rank: Rank mask
  124. * @odt_mode: ODT mode, OFF or READ_WRITE
  125. *
  126. * Set Rank and ODT mask (On-Die Termination).
  127. */
  128. static void set_rank_and_odt_mask(struct socfpga_sdrseq *seq,
  129. const u32 rank, const u32 odt_mode)
  130. {
  131. u32 odt_mask_0 = 0;
  132. u32 odt_mask_1 = 0;
  133. u32 cs_and_odt_mask;
  134. if (odt_mode == RW_MGR_ODT_MODE_OFF) {
  135. odt_mask_0 = 0x0;
  136. odt_mask_1 = 0x0;
  137. } else { /* RW_MGR_ODT_MODE_READ_WRITE */
  138. switch (seq->rwcfg->mem_number_of_ranks) {
  139. case 1: /* 1 Rank */
  140. /* Read: ODT = 0 ; Write: ODT = 1 */
  141. odt_mask_0 = 0x0;
  142. odt_mask_1 = 0x1;
  143. break;
  144. case 2: /* 2 Ranks */
  145. if (seq->rwcfg->mem_number_of_cs_per_dimm == 1) {
  146. /*
  147. * - Dual-Slot , Single-Rank (1 CS per DIMM)
  148. * OR
  149. * - RDIMM, 4 total CS (2 CS per DIMM, 2 DIMM)
  150. *
  151. * Since MEM_NUMBER_OF_RANKS is 2, they
  152. * are both single rank with 2 CS each
  153. * (special for RDIMM).
  154. *
  155. * Read: Turn on ODT on the opposite rank
  156. * Write: Turn on ODT on all ranks
  157. */
  158. odt_mask_0 = 0x3 & ~(1 << rank);
  159. odt_mask_1 = 0x3;
  160. if (dram_is_ddr(2))
  161. odt_mask_1 &= ~(1 << rank);
  162. } else {
  163. /*
  164. * - Single-Slot , Dual-Rank (2 CS per DIMM)
  165. *
  166. * Read: Turn on ODT off on all ranks
  167. * Write: Turn on ODT on active rank
  168. */
  169. odt_mask_0 = 0x0;
  170. odt_mask_1 = 0x3 & (1 << rank);
  171. }
  172. break;
  173. case 4: /* 4 Ranks */
  174. /*
  175. * DDR3 Read, DDR2 Read/Write:
  176. * ----------+-----------------------+
  177. * | ODT |
  178. * +-----------------------+
  179. * Rank | 3 | 2 | 1 | 0 |
  180. * ----------+-----+-----+-----+-----+
  181. * 0 | 0 | 1 | 0 | 0 |
  182. * 1 | 1 | 0 | 0 | 0 |
  183. * 2 | 0 | 0 | 0 | 1 |
  184. * 3 | 0 | 0 | 1 | 0 |
  185. * ----------+-----+-----+-----+-----+
  186. *
  187. * DDR3 Write:
  188. * ----------+-----------------------+
  189. * | ODT |
  190. * Write To +-----------------------+
  191. * Rank | 3 | 2 | 1 | 0 |
  192. * ----------+-----+-----+-----+-----+
  193. * 0 | 0 | 1 | 0 | 1 |
  194. * 1 | 1 | 0 | 1 | 0 |
  195. * 2 | 0 | 1 | 0 | 1 |
  196. * 3 | 1 | 0 | 1 | 0 |
  197. * ----------+-----+-----+-----+-----+
  198. */
  199. switch (rank) {
  200. case 0:
  201. odt_mask_0 = 0x4;
  202. if (dram_is_ddr(2))
  203. odt_mask_1 = 0x4;
  204. else if (dram_is_ddr(3))
  205. odt_mask_1 = 0x5;
  206. break;
  207. case 1:
  208. odt_mask_0 = 0x8;
  209. if (dram_is_ddr(2))
  210. odt_mask_1 = 0x8;
  211. else if (dram_is_ddr(3))
  212. odt_mask_1 = 0xA;
  213. break;
  214. case 2:
  215. odt_mask_0 = 0x1;
  216. if (dram_is_ddr(2))
  217. odt_mask_1 = 0x1;
  218. else if (dram_is_ddr(3))
  219. odt_mask_1 = 0x5;
  220. break;
  221. case 3:
  222. odt_mask_0 = 0x2;
  223. if (dram_is_ddr(2))
  224. odt_mask_1 = 0x2;
  225. else if (dram_is_ddr(3))
  226. odt_mask_1 = 0xA;
  227. break;
  228. }
  229. break;
  230. }
  231. }
  232. cs_and_odt_mask = (0xFF & ~(1 << rank)) |
  233. ((0xFF & odt_mask_0) << 8) |
  234. ((0xFF & odt_mask_1) << 16);
  235. writel(cs_and_odt_mask, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  236. RW_MGR_SET_CS_AND_ODT_MASK_OFFSET);
  237. }
  238. /**
  239. * scc_mgr_set() - Set SCC Manager register
  240. * @off: Base offset in SCC Manager space
  241. * @grp: Read/Write group
  242. * @val: Value to be set
  243. *
  244. * This function sets the SCC Manager (Scan Chain Control Manager) register.
  245. */
  246. static void scc_mgr_set(u32 off, u32 grp, u32 val)
  247. {
  248. writel(val, SDR_PHYGRP_SCCGRP_ADDRESS | off | (grp << 2));
  249. }
  250. /**
  251. * scc_mgr_initialize() - Initialize SCC Manager registers
  252. *
  253. * Initialize SCC Manager registers.
  254. */
  255. static void scc_mgr_initialize(void)
  256. {
  257. /*
  258. * Clear register file for HPS. 16 (2^4) is the size of the
  259. * full register file in the scc mgr:
  260. * RFILE_DEPTH = 1 + log2(MEM_DQ_PER_DQS + 1 + MEM_DM_PER_DQS +
  261. * MEM_IF_READ_DQS_WIDTH - 1);
  262. */
  263. int i;
  264. for (i = 0; i < 16; i++) {
  265. debug_cond(DLEVEL >= 1, "%s:%d: Clearing SCC RFILE index %u\n",
  266. __func__, __LINE__, i);
  267. scc_mgr_set(SCC_MGR_HHP_RFILE_OFFSET, i, 0);
  268. }
  269. }
  270. static void scc_mgr_set_dqdqs_output_phase(u32 write_group, u32 phase)
  271. {
  272. scc_mgr_set(SCC_MGR_DQDQS_OUT_PHASE_OFFSET, write_group, phase);
  273. }
  274. static void scc_mgr_set_dqs_bus_in_delay(u32 read_group, u32 delay)
  275. {
  276. scc_mgr_set(SCC_MGR_DQS_IN_DELAY_OFFSET, read_group, delay);
  277. }
  278. static void scc_mgr_set_dqs_en_phase(u32 read_group, u32 phase)
  279. {
  280. scc_mgr_set(SCC_MGR_DQS_EN_PHASE_OFFSET, read_group, phase);
  281. }
  282. static void scc_mgr_set_dqs_en_delay(u32 read_group, u32 delay)
  283. {
  284. scc_mgr_set(SCC_MGR_DQS_EN_DELAY_OFFSET, read_group, delay);
  285. }
  286. static void scc_mgr_set_dq_in_delay(u32 dq_in_group, u32 delay)
  287. {
  288. scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET, dq_in_group, delay);
  289. }
  290. static void scc_mgr_set_dqs_io_in_delay(struct socfpga_sdrseq *seq,
  291. u32 delay)
  292. {
  293. scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET,
  294. seq->rwcfg->mem_dq_per_write_dqs, delay);
  295. }
  296. static void scc_mgr_set_dm_in_delay(struct socfpga_sdrseq *seq, u32 dm,
  297. u32 delay)
  298. {
  299. scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET,
  300. seq->rwcfg->mem_dq_per_write_dqs + 1 + dm,
  301. delay);
  302. }
  303. static void scc_mgr_set_dq_out1_delay(u32 dq_in_group, u32 delay)
  304. {
  305. scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET, dq_in_group, delay);
  306. }
  307. static void scc_mgr_set_dqs_out1_delay(struct socfpga_sdrseq *seq,
  308. u32 delay)
  309. {
  310. scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET,
  311. seq->rwcfg->mem_dq_per_write_dqs, delay);
  312. }
  313. static void scc_mgr_set_dm_out1_delay(struct socfpga_sdrseq *seq, u32 dm,
  314. u32 delay)
  315. {
  316. scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET,
  317. seq->rwcfg->mem_dq_per_write_dqs + 1 + dm,
  318. delay);
  319. }
  320. /* load up dqs config settings */
  321. static void scc_mgr_load_dqs(u32 dqs)
  322. {
  323. writel(dqs, &sdr_scc_mgr->dqs_ena);
  324. }
  325. /* load up dqs io config settings */
  326. static void scc_mgr_load_dqs_io(void)
  327. {
  328. writel(0, &sdr_scc_mgr->dqs_io_ena);
  329. }
  330. /* load up dq config settings */
  331. static void scc_mgr_load_dq(u32 dq_in_group)
  332. {
  333. writel(dq_in_group, &sdr_scc_mgr->dq_ena);
  334. }
  335. /* load up dm config settings */
  336. static void scc_mgr_load_dm(u32 dm)
  337. {
  338. writel(dm, &sdr_scc_mgr->dm_ena);
  339. }
  340. /**
  341. * scc_mgr_set_all_ranks() - Set SCC Manager register for all ranks
  342. * @off: Base offset in SCC Manager space
  343. * @grp: Read/Write group
  344. * @val: Value to be set
  345. * @update: If non-zero, trigger SCC Manager update for all ranks
  346. *
  347. * This function sets the SCC Manager (Scan Chain Control Manager) register
  348. * and optionally triggers the SCC update for all ranks.
  349. */
  350. static void scc_mgr_set_all_ranks(struct socfpga_sdrseq *seq,
  351. const u32 off, const u32 grp, const u32 val,
  352. const int update)
  353. {
  354. u32 r;
  355. for (r = 0; r < seq->rwcfg->mem_number_of_ranks;
  356. r += NUM_RANKS_PER_SHADOW_REG) {
  357. scc_mgr_set(off, grp, val);
  358. if (update || (r == 0)) {
  359. writel(grp, &sdr_scc_mgr->dqs_ena);
  360. writel(0, &sdr_scc_mgr->update);
  361. }
  362. }
  363. }
  364. static void scc_mgr_set_dqs_en_phase_all_ranks(struct socfpga_sdrseq *seq,
  365. u32 read_group, u32 phase)
  366. {
  367. /*
  368. * USER although the h/w doesn't support different phases per
  369. * shadow register, for simplicity our scc manager modeling
  370. * keeps different phase settings per shadow reg, and it's
  371. * important for us to keep them in sync to match h/w.
  372. * for efficiency, the scan chain update should occur only
  373. * once to sr0.
  374. */
  375. scc_mgr_set_all_ranks(seq, SCC_MGR_DQS_EN_PHASE_OFFSET,
  376. read_group, phase, 0);
  377. }
  378. static void scc_mgr_set_dqdqs_output_phase_all_ranks(struct socfpga_sdrseq *seq,
  379. u32 write_group, u32 phase)
  380. {
  381. /*
  382. * USER although the h/w doesn't support different phases per
  383. * shadow register, for simplicity our scc manager modeling
  384. * keeps different phase settings per shadow reg, and it's
  385. * important for us to keep them in sync to match h/w.
  386. * for efficiency, the scan chain update should occur only
  387. * once to sr0.
  388. */
  389. scc_mgr_set_all_ranks(seq, SCC_MGR_DQDQS_OUT_PHASE_OFFSET,
  390. write_group, phase, 0);
  391. }
  392. static void scc_mgr_set_dqs_en_delay_all_ranks(struct socfpga_sdrseq *seq,
  393. u32 read_group, u32 delay)
  394. {
  395. /*
  396. * In shadow register mode, the T11 settings are stored in
  397. * registers in the core, which are updated by the DQS_ENA
  398. * signals. Not issuing the SCC_MGR_UPD command allows us to
  399. * save lots of rank switching overhead, by calling
  400. * select_shadow_regs_for_update with update_scan_chains
  401. * set to 0.
  402. */
  403. scc_mgr_set_all_ranks(seq, SCC_MGR_DQS_EN_DELAY_OFFSET,
  404. read_group, delay, 1);
  405. }
  406. /**
  407. * scc_mgr_set_oct_out1_delay() - Set OCT output delay
  408. * @write_group: Write group
  409. * @delay: Delay value
  410. *
  411. * This function sets the OCT output delay in SCC manager.
  412. */
  413. static void scc_mgr_set_oct_out1_delay(struct socfpga_sdrseq *seq,
  414. const u32 write_group, const u32 delay)
  415. {
  416. const int ratio = seq->rwcfg->mem_if_read_dqs_width /
  417. seq->rwcfg->mem_if_write_dqs_width;
  418. const int base = write_group * ratio;
  419. int i;
  420. /*
  421. * Load the setting in the SCC manager
  422. * Although OCT affects only write data, the OCT delay is controlled
  423. * by the DQS logic block which is instantiated once per read group.
  424. * For protocols where a write group consists of multiple read groups,
  425. * the setting must be set multiple times.
  426. */
  427. for (i = 0; i < ratio; i++)
  428. scc_mgr_set(SCC_MGR_OCT_OUT1_DELAY_OFFSET, base + i, delay);
  429. }
  430. /**
  431. * scc_mgr_set_hhp_extras() - Set HHP extras.
  432. *
  433. * Load the fixed setting in the SCC manager HHP extras.
  434. */
  435. static void scc_mgr_set_hhp_extras(void)
  436. {
  437. /*
  438. * Load the fixed setting in the SCC manager
  439. * bits: 0:0 = 1'b1 - DQS bypass
  440. * bits: 1:1 = 1'b1 - DQ bypass
  441. * bits: 4:2 = 3'b001 - rfifo_mode
  442. * bits: 6:5 = 2'b01 - rfifo clock_select
  443. * bits: 7:7 = 1'b0 - separate gating from ungating setting
  444. * bits: 8:8 = 1'b0 - separate OE from Output delay setting
  445. */
  446. const u32 value = (0 << 8) | (0 << 7) | (1 << 5) |
  447. (1 << 2) | (1 << 1) | (1 << 0);
  448. const u32 addr = SDR_PHYGRP_SCCGRP_ADDRESS |
  449. SCC_MGR_HHP_GLOBALS_OFFSET |
  450. SCC_MGR_HHP_EXTRAS_OFFSET;
  451. debug_cond(DLEVEL >= 1, "%s:%d Setting HHP Extras\n",
  452. __func__, __LINE__);
  453. writel(value, addr);
  454. debug_cond(DLEVEL >= 1, "%s:%d Done Setting HHP Extras\n",
  455. __func__, __LINE__);
  456. }
  457. /**
  458. * scc_mgr_zero_all() - Zero all DQS config
  459. *
  460. * Zero all DQS config.
  461. */
  462. static void scc_mgr_zero_all(struct socfpga_sdrseq *seq)
  463. {
  464. int i, r;
  465. /*
  466. * USER Zero all DQS config settings, across all groups and all
  467. * shadow registers
  468. */
  469. for (r = 0; r < seq->rwcfg->mem_number_of_ranks;
  470. r += NUM_RANKS_PER_SHADOW_REG) {
  471. for (i = 0; i < seq->rwcfg->mem_if_read_dqs_width; i++) {
  472. /*
  473. * The phases actually don't exist on a per-rank basis,
  474. * but there's no harm updating them several times, so
  475. * let's keep the code simple.
  476. */
  477. scc_mgr_set_dqs_bus_in_delay(i,
  478. seq->iocfg->dqs_in_reserve
  479. );
  480. scc_mgr_set_dqs_en_phase(i, 0);
  481. scc_mgr_set_dqs_en_delay(i, 0);
  482. }
  483. for (i = 0; i < seq->rwcfg->mem_if_write_dqs_width; i++) {
  484. scc_mgr_set_dqdqs_output_phase(i, 0);
  485. /* Arria V/Cyclone V don't have out2. */
  486. scc_mgr_set_oct_out1_delay(seq, i,
  487. seq->iocfg->dqs_out_reserve);
  488. }
  489. }
  490. /* Multicast to all DQS group enables. */
  491. writel(0xff, &sdr_scc_mgr->dqs_ena);
  492. writel(0, &sdr_scc_mgr->update);
  493. }
  494. /**
  495. * scc_set_bypass_mode() - Set bypass mode and trigger SCC update
  496. * @write_group: Write group
  497. *
  498. * Set bypass mode and trigger SCC update.
  499. */
  500. static void scc_set_bypass_mode(const u32 write_group)
  501. {
  502. /* Multicast to all DQ enables. */
  503. writel(0xff, &sdr_scc_mgr->dq_ena);
  504. writel(0xff, &sdr_scc_mgr->dm_ena);
  505. /* Update current DQS IO enable. */
  506. writel(0, &sdr_scc_mgr->dqs_io_ena);
  507. /* Update the DQS logic. */
  508. writel(write_group, &sdr_scc_mgr->dqs_ena);
  509. /* Hit update. */
  510. writel(0, &sdr_scc_mgr->update);
  511. }
  512. /**
  513. * scc_mgr_load_dqs_for_write_group() - Load DQS settings for Write Group
  514. * @write_group: Write group
  515. *
  516. * Load DQS settings for Write Group, do not trigger SCC update.
  517. */
  518. static void scc_mgr_load_dqs_for_write_group(struct socfpga_sdrseq *seq,
  519. const u32 write_group)
  520. {
  521. const int ratio = seq->rwcfg->mem_if_read_dqs_width /
  522. seq->rwcfg->mem_if_write_dqs_width;
  523. const int base = write_group * ratio;
  524. int i;
  525. /*
  526. * Load the setting in the SCC manager
  527. * Although OCT affects only write data, the OCT delay is controlled
  528. * by the DQS logic block which is instantiated once per read group.
  529. * For protocols where a write group consists of multiple read groups,
  530. * the setting must be set multiple times.
  531. */
  532. for (i = 0; i < ratio; i++)
  533. writel(base + i, &sdr_scc_mgr->dqs_ena);
  534. }
  535. /**
  536. * scc_mgr_zero_group() - Zero all configs for a group
  537. *
  538. * Zero DQ, DM, DQS and OCT configs for a group.
  539. */
  540. static void scc_mgr_zero_group(struct socfpga_sdrseq *seq,
  541. const u32 write_group, const int out_only)
  542. {
  543. int i, r;
  544. for (r = 0; r < seq->rwcfg->mem_number_of_ranks;
  545. r += NUM_RANKS_PER_SHADOW_REG) {
  546. /* Zero all DQ config settings. */
  547. for (i = 0; i < seq->rwcfg->mem_dq_per_write_dqs; i++) {
  548. scc_mgr_set_dq_out1_delay(i, 0);
  549. if (!out_only)
  550. scc_mgr_set_dq_in_delay(i, 0);
  551. }
  552. /* Multicast to all DQ enables. */
  553. writel(0xff, &sdr_scc_mgr->dq_ena);
  554. /* Zero all DM config settings. */
  555. for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
  556. if (!out_only)
  557. scc_mgr_set_dm_in_delay(seq, i, 0);
  558. scc_mgr_set_dm_out1_delay(seq, i, 0);
  559. }
  560. /* Multicast to all DM enables. */
  561. writel(0xff, &sdr_scc_mgr->dm_ena);
  562. /* Zero all DQS IO settings. */
  563. if (!out_only)
  564. scc_mgr_set_dqs_io_in_delay(seq, 0);
  565. /* Arria V/Cyclone V don't have out2. */
  566. scc_mgr_set_dqs_out1_delay(seq, seq->iocfg->dqs_out_reserve);
  567. scc_mgr_set_oct_out1_delay(seq, write_group,
  568. seq->iocfg->dqs_out_reserve);
  569. scc_mgr_load_dqs_for_write_group(seq, write_group);
  570. /* Multicast to all DQS IO enables (only 1 in total). */
  571. writel(0, &sdr_scc_mgr->dqs_io_ena);
  572. /* Hit update to zero everything. */
  573. writel(0, &sdr_scc_mgr->update);
  574. }
  575. }
  576. /*
  577. * apply and load a particular input delay for the DQ pins in a group
  578. * group_bgn is the index of the first dq pin (in the write group)
  579. */
  580. static void scc_mgr_apply_group_dq_in_delay(struct socfpga_sdrseq *seq,
  581. u32 group_bgn, u32 delay)
  582. {
  583. u32 i, p;
  584. for (i = 0, p = group_bgn; i < seq->rwcfg->mem_dq_per_read_dqs;
  585. i++, p++) {
  586. scc_mgr_set_dq_in_delay(p, delay);
  587. scc_mgr_load_dq(p);
  588. }
  589. }
  590. /**
  591. * scc_mgr_apply_group_dq_out1_delay() - Apply and load an output delay for the
  592. * DQ pins in a group
  593. * @delay: Delay value
  594. *
  595. * Apply and load a particular output delay for the DQ pins in a group.
  596. */
  597. static void scc_mgr_apply_group_dq_out1_delay(struct socfpga_sdrseq *seq,
  598. const u32 delay)
  599. {
  600. int i;
  601. for (i = 0; i < seq->rwcfg->mem_dq_per_write_dqs; i++) {
  602. scc_mgr_set_dq_out1_delay(i, delay);
  603. scc_mgr_load_dq(i);
  604. }
  605. }
  606. /* apply and load a particular output delay for the DM pins in a group */
  607. static void scc_mgr_apply_group_dm_out1_delay(struct socfpga_sdrseq *seq,
  608. u32 delay1)
  609. {
  610. u32 i;
  611. for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
  612. scc_mgr_set_dm_out1_delay(seq, i, delay1);
  613. scc_mgr_load_dm(i);
  614. }
  615. }
  616. /* apply and load delay on both DQS and OCT out1 */
  617. static void scc_mgr_apply_group_dqs_io_and_oct_out1(struct socfpga_sdrseq *seq,
  618. u32 write_group, u32 delay)
  619. {
  620. scc_mgr_set_dqs_out1_delay(seq, delay);
  621. scc_mgr_load_dqs_io();
  622. scc_mgr_set_oct_out1_delay(seq, write_group, delay);
  623. scc_mgr_load_dqs_for_write_group(seq, write_group);
  624. }
  625. /**
  626. * scc_mgr_apply_group_all_out_delay_add() - Apply a delay to the entire output
  627. * side: DQ, DM, DQS, OCT
  628. * @write_group: Write group
  629. * @delay: Delay value
  630. *
  631. * Apply a delay to the entire output side: DQ, DM, DQS, OCT.
  632. */
  633. static void scc_mgr_apply_group_all_out_delay_add(struct socfpga_sdrseq *seq,
  634. const u32 write_group,
  635. const u32 delay)
  636. {
  637. u32 i, new_delay;
  638. /* DQ shift */
  639. for (i = 0; i < seq->rwcfg->mem_dq_per_write_dqs; i++)
  640. scc_mgr_load_dq(i);
  641. /* DM shift */
  642. for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++)
  643. scc_mgr_load_dm(i);
  644. /* DQS shift */
  645. new_delay = READ_SCC_DQS_IO_OUT2_DELAY + delay;
  646. if (new_delay > seq->iocfg->io_out2_delay_max) {
  647. debug_cond(DLEVEL >= 1,
  648. "%s:%d (%u, %u) DQS: %u > %d; adding %u to OUT1\n",
  649. __func__, __LINE__, write_group, delay, new_delay,
  650. seq->iocfg->io_out2_delay_max,
  651. new_delay - seq->iocfg->io_out2_delay_max);
  652. new_delay -= seq->iocfg->io_out2_delay_max;
  653. scc_mgr_set_dqs_out1_delay(seq, new_delay);
  654. }
  655. scc_mgr_load_dqs_io();
  656. /* OCT shift */
  657. new_delay = READ_SCC_OCT_OUT2_DELAY + delay;
  658. if (new_delay > seq->iocfg->io_out2_delay_max) {
  659. debug_cond(DLEVEL >= 1,
  660. "%s:%d (%u, %u) DQS: %u > %d; adding %u to OUT1\n",
  661. __func__, __LINE__, write_group, delay,
  662. new_delay, seq->iocfg->io_out2_delay_max,
  663. new_delay - seq->iocfg->io_out2_delay_max);
  664. new_delay -= seq->iocfg->io_out2_delay_max;
  665. scc_mgr_set_oct_out1_delay(seq, write_group, new_delay);
  666. }
  667. scc_mgr_load_dqs_for_write_group(seq, write_group);
  668. }
  669. /**
  670. * scc_mgr_apply_group_all_out_delay_add() - Apply a delay to the entire output
  671. * side to all ranks
  672. * @write_group: Write group
  673. * @delay: Delay value
  674. *
  675. * Apply a delay to the entire output side (DQ, DM, DQS, OCT) to all ranks.
  676. */
  677. static void
  678. scc_mgr_apply_group_all_out_delay_add_all_ranks(struct socfpga_sdrseq *seq,
  679. const u32 write_group,
  680. const u32 delay)
  681. {
  682. int r;
  683. for (r = 0; r < seq->rwcfg->mem_number_of_ranks;
  684. r += NUM_RANKS_PER_SHADOW_REG) {
  685. scc_mgr_apply_group_all_out_delay_add(seq, write_group, delay);
  686. writel(0, &sdr_scc_mgr->update);
  687. }
  688. }
  689. /**
  690. * set_jump_as_return() - Return instruction optimization
  691. *
  692. * Optimization used to recover some slots in ddr3 inst_rom could be
  693. * applied to other protocols if we wanted to
  694. */
  695. static void set_jump_as_return(struct socfpga_sdrseq *seq)
  696. {
  697. /*
  698. * To save space, we replace return with jump to special shared
  699. * RETURN instruction so we set the counter to large value so that
  700. * we always jump.
  701. */
  702. writel(0xff, &sdr_rw_load_mgr_regs->load_cntr0);
  703. writel(seq->rwcfg->rreturn, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  704. }
  705. /**
  706. * delay_for_n_mem_clocks() - Delay for N memory clocks
  707. * @clocks: Length of the delay
  708. *
  709. * Delay for N memory clocks.
  710. */
  711. static void delay_for_n_mem_clocks(struct socfpga_sdrseq *seq,
  712. const u32 clocks)
  713. {
  714. u32 afi_clocks;
  715. u16 c_loop;
  716. u8 inner;
  717. u8 outer;
  718. debug("%s:%d: clocks=%u ... start\n", __func__, __LINE__, clocks);
  719. /* Scale (rounding up) to get afi clocks. */
  720. afi_clocks = DIV_ROUND_UP(clocks, seq->misccfg->afi_rate_ratio);
  721. if (afi_clocks) /* Temporary underflow protection */
  722. afi_clocks--;
  723. /*
  724. * Note, we don't bother accounting for being off a little
  725. * bit because of a few extra instructions in outer loops.
  726. * Note, the loops have a test at the end, and do the test
  727. * before the decrement, and so always perform the loop
  728. * 1 time more than the counter value
  729. */
  730. c_loop = afi_clocks >> 16;
  731. outer = c_loop ? 0xff : (afi_clocks >> 8);
  732. inner = outer ? 0xff : afi_clocks;
  733. /*
  734. * rom instructions are structured as follows:
  735. *
  736. * IDLE_LOOP2: jnz cntr0, TARGET_A
  737. * IDLE_LOOP1: jnz cntr1, TARGET_B
  738. * return
  739. *
  740. * so, when doing nested loops, TARGET_A is set to IDLE_LOOP2, and
  741. * TARGET_B is set to IDLE_LOOP2 as well
  742. *
  743. * if we have no outer loop, though, then we can use IDLE_LOOP1 only,
  744. * and set TARGET_B to IDLE_LOOP1 and we skip IDLE_LOOP2 entirely
  745. *
  746. * a little confusing, but it helps save precious space in the inst_rom
  747. * and sequencer rom and keeps the delays more accurate and reduces
  748. * overhead
  749. */
  750. if (afi_clocks < 0x100) {
  751. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner),
  752. &sdr_rw_load_mgr_regs->load_cntr1);
  753. writel(seq->rwcfg->idle_loop1,
  754. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  755. writel(seq->rwcfg->idle_loop1, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  756. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  757. } else {
  758. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner),
  759. &sdr_rw_load_mgr_regs->load_cntr0);
  760. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(outer),
  761. &sdr_rw_load_mgr_regs->load_cntr1);
  762. writel(seq->rwcfg->idle_loop2,
  763. &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  764. writel(seq->rwcfg->idle_loop2,
  765. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  766. do {
  767. writel(seq->rwcfg->idle_loop2,
  768. SDR_PHYGRP_RWMGRGRP_ADDRESS |
  769. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  770. } while (c_loop-- != 0);
  771. }
  772. debug("%s:%d clocks=%u ... end\n", __func__, __LINE__, clocks);
  773. }
  774. static void delay_for_n_ns(struct socfpga_sdrseq *seq, const u32 ns)
  775. {
  776. delay_for_n_mem_clocks(seq, (ns * seq->misccfg->afi_clk_freq *
  777. seq->misccfg->afi_rate_ratio) / 1000);
  778. }
  779. /**
  780. * rw_mgr_mem_init_load_regs() - Load instruction registers
  781. * @cntr0: Counter 0 value
  782. * @cntr1: Counter 1 value
  783. * @cntr2: Counter 2 value
  784. * @jump: Jump instruction value
  785. *
  786. * Load instruction registers.
  787. */
  788. static void rw_mgr_mem_init_load_regs(struct socfpga_sdrseq *seq,
  789. u32 cntr0, u32 cntr1, u32 cntr2, u32 jump)
  790. {
  791. u32 grpaddr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
  792. RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  793. /* Load counters */
  794. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr0),
  795. &sdr_rw_load_mgr_regs->load_cntr0);
  796. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr1),
  797. &sdr_rw_load_mgr_regs->load_cntr1);
  798. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr2),
  799. &sdr_rw_load_mgr_regs->load_cntr2);
  800. /* Load jump address */
  801. writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  802. writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  803. writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  804. /* Execute count instruction */
  805. writel(jump, grpaddr);
  806. }
  807. /**
  808. * rw_mgr_mem_load_user_ddr2() - Load user calibration values for DDR2
  809. * @handoff: Indicate whether this is initialization or handoff phase
  810. *
  811. * Load user calibration values and optionally precharge the banks.
  812. */
  813. static void rw_mgr_mem_load_user_ddr2(struct socfpga_sdrseq *seq,
  814. const int handoff)
  815. {
  816. u32 grpaddr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
  817. RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  818. u32 r;
  819. for (r = 0; r < seq->rwcfg->mem_number_of_ranks; r++) {
  820. /* set rank */
  821. set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_OFF);
  822. /* precharge all banks ... */
  823. writel(seq->rwcfg->precharge_all, grpaddr);
  824. writel(seq->rwcfg->emr2, grpaddr);
  825. writel(seq->rwcfg->emr3, grpaddr);
  826. writel(seq->rwcfg->emr, grpaddr);
  827. if (handoff) {
  828. writel(seq->rwcfg->mr_user, grpaddr);
  829. continue;
  830. }
  831. writel(seq->rwcfg->mr_dll_reset, grpaddr);
  832. writel(seq->rwcfg->precharge_all, grpaddr);
  833. writel(seq->rwcfg->refresh, grpaddr);
  834. delay_for_n_ns(seq, 200);
  835. writel(seq->rwcfg->refresh, grpaddr);
  836. delay_for_n_ns(seq, 200);
  837. writel(seq->rwcfg->mr_calib, grpaddr);
  838. writel(/*seq->rwcfg->*/0x0b, grpaddr); // EMR_OCD_ENABLE
  839. writel(seq->rwcfg->emr, grpaddr);
  840. delay_for_n_mem_clocks(seq, 200);
  841. }
  842. }
  843. /**
  844. * rw_mgr_mem_load_user_ddr3() - Load user calibration values
  845. * @fin1: Final instruction 1
  846. * @fin2: Final instruction 2
  847. * @precharge: If 1, precharge the banks at the end
  848. *
  849. * Load user calibration values and optionally precharge the banks.
  850. */
  851. static void rw_mgr_mem_load_user_ddr3(struct socfpga_sdrseq *seq,
  852. const u32 fin1, const u32 fin2,
  853. const int precharge)
  854. {
  855. u32 grpaddr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
  856. RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  857. u32 r;
  858. for (r = 0; r < seq->rwcfg->mem_number_of_ranks; r++) {
  859. /* set rank */
  860. set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_OFF);
  861. /* precharge all banks ... */
  862. if (precharge)
  863. writel(seq->rwcfg->precharge_all, grpaddr);
  864. /*
  865. * USER Use Mirror-ed commands for odd ranks if address
  866. * mirrorring is on
  867. */
  868. if ((seq->rwcfg->mem_address_mirroring >> r) & 0x1) {
  869. set_jump_as_return(seq);
  870. writel(seq->rwcfg->mrs2_mirr, grpaddr);
  871. delay_for_n_mem_clocks(seq, 4);
  872. set_jump_as_return(seq);
  873. writel(seq->rwcfg->mrs3_mirr, grpaddr);
  874. delay_for_n_mem_clocks(seq, 4);
  875. set_jump_as_return(seq);
  876. writel(seq->rwcfg->mrs1_mirr, grpaddr);
  877. delay_for_n_mem_clocks(seq, 4);
  878. set_jump_as_return(seq);
  879. writel(fin1, grpaddr);
  880. } else {
  881. set_jump_as_return(seq);
  882. writel(seq->rwcfg->mrs2, grpaddr);
  883. delay_for_n_mem_clocks(seq, 4);
  884. set_jump_as_return(seq);
  885. writel(seq->rwcfg->mrs3, grpaddr);
  886. delay_for_n_mem_clocks(seq, 4);
  887. set_jump_as_return(seq);
  888. writel(seq->rwcfg->mrs1, grpaddr);
  889. set_jump_as_return(seq);
  890. writel(fin2, grpaddr);
  891. }
  892. if (precharge)
  893. continue;
  894. set_jump_as_return(seq);
  895. writel(seq->rwcfg->zqcl, grpaddr);
  896. /* tZQinit = tDLLK = 512 ck cycles */
  897. delay_for_n_mem_clocks(seq, 512);
  898. }
  899. }
  900. /**
  901. * rw_mgr_mem_load_user() - Load user calibration values
  902. * @fin1: Final instruction 1
  903. * @fin2: Final instruction 2
  904. * @precharge: If 1, precharge the banks at the end
  905. *
  906. * Load user calibration values and optionally precharge the banks.
  907. */
  908. static void rw_mgr_mem_load_user(struct socfpga_sdrseq *seq,
  909. const u32 fin1, const u32 fin2,
  910. const int precharge)
  911. {
  912. if (dram_is_ddr(2))
  913. rw_mgr_mem_load_user_ddr2(seq, precharge);
  914. else if (dram_is_ddr(3))
  915. rw_mgr_mem_load_user_ddr3(seq, fin1, fin2, precharge);
  916. else
  917. hang();
  918. }
  919. /**
  920. * rw_mgr_mem_initialize() - Initialize RW Manager
  921. *
  922. * Initialize RW Manager.
  923. */
  924. static void rw_mgr_mem_initialize(struct socfpga_sdrseq *seq)
  925. {
  926. debug("%s:%d\n", __func__, __LINE__);
  927. /* The reset / cke part of initialization is broadcasted to all ranks */
  928. if (dram_is_ddr(3)) {
  929. writel(RW_MGR_RANK_ALL, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  930. RW_MGR_SET_CS_AND_ODT_MASK_OFFSET);
  931. }
  932. /*
  933. * Here's how you load register for a loop
  934. * Counters are located @ 0x800
  935. * Jump address are located @ 0xC00
  936. * For both, registers 0 to 3 are selected using bits 3 and 2, like
  937. * in 0x800, 0x804, 0x808, 0x80C and 0xC00, 0xC04, 0xC08, 0xC0C
  938. * I know this ain't pretty, but Avalon bus throws away the 2 least
  939. * significant bits
  940. */
  941. /* Start with memory RESET activated */
  942. /* tINIT = 200us */
  943. /*
  944. * 200us @ 266MHz (3.75 ns) ~ 54000 clock cycles
  945. * If a and b are the number of iteration in 2 nested loops
  946. * it takes the following number of cycles to complete the operation:
  947. * number_of_cycles = ((2 + n) * a + 2) * b
  948. * where n is the number of instruction in the inner loop
  949. * One possible solution is n = 0 , a = 256 , b = 106 => a = FF,
  950. * b = 6A
  951. */
  952. rw_mgr_mem_init_load_regs(seq, seq->misccfg->tinit_cntr0_val,
  953. seq->misccfg->tinit_cntr1_val,
  954. seq->misccfg->tinit_cntr2_val,
  955. seq->rwcfg->init_reset_0_cke_0);
  956. /* Indicate that memory is stable. */
  957. writel(1, &phy_mgr_cfg->reset_mem_stbl);
  958. if (dram_is_ddr(2)) {
  959. writel(seq->rwcfg->nop, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  960. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  961. /* Bring up clock enable. */
  962. /* tXRP < 400 ck cycles */
  963. delay_for_n_ns(seq, 400);
  964. } else if (dram_is_ddr(3)) {
  965. /*
  966. * transition the RESET to high
  967. * Wait for 500us
  968. */
  969. /*
  970. * 500us @ 266MHz (3.75 ns) ~ 134000 clock cycles
  971. * If a and b are the number of iteration in 2 nested loops
  972. * it takes the following number of cycles to complete the
  973. * operation number_of_cycles = ((2 + n) * a + 2) * b
  974. * where n is the number of instruction in the inner loop
  975. * One possible solution is
  976. * n = 2 , a = 131 , b = 256 => a = 83, b = FF
  977. */
  978. rw_mgr_mem_init_load_regs(seq, seq->misccfg->treset_cntr0_val,
  979. seq->misccfg->treset_cntr1_val,
  980. seq->misccfg->treset_cntr2_val,
  981. seq->rwcfg->init_reset_1_cke_0);
  982. /* Bring up clock enable. */
  983. /* tXRP < 250 ck cycles */
  984. delay_for_n_mem_clocks(seq, 250);
  985. }
  986. rw_mgr_mem_load_user(seq, seq->rwcfg->mrs0_dll_reset_mirr,
  987. seq->rwcfg->mrs0_dll_reset, 0);
  988. }
  989. /**
  990. * rw_mgr_mem_handoff() - Hand off the memory to user
  991. *
  992. * At the end of calibration we have to program the user settings in
  993. * and hand off the memory to the user.
  994. */
  995. static void rw_mgr_mem_handoff(struct socfpga_sdrseq *seq)
  996. {
  997. rw_mgr_mem_load_user(seq, seq->rwcfg->mrs0_user_mirr,
  998. seq->rwcfg->mrs0_user, 1);
  999. /*
  1000. * Need to wait tMOD (12CK or 15ns) time before issuing other
  1001. * commands, but we will have plenty of NIOS cycles before actual
  1002. * handoff so its okay.
  1003. */
  1004. }
  1005. /**
  1006. * rw_mgr_mem_calibrate_write_test_issue() - Issue write test command
  1007. * @group: Write Group
  1008. * @use_dm: Use DM
  1009. *
  1010. * Issue write test command. Two variants are provided, one that just tests
  1011. * a write pattern and another that tests datamask functionality.
  1012. */
  1013. static void rw_mgr_mem_calibrate_write_test_issue(struct socfpga_sdrseq *seq,
  1014. u32 group, u32 test_dm)
  1015. {
  1016. const u32 quick_write_mode =
  1017. (STATIC_CALIB_STEPS & CALIB_SKIP_WRITES) &&
  1018. seq->misccfg->enable_super_quick_calibration;
  1019. u32 mcc_instruction;
  1020. u32 rw_wl_nop_cycles;
  1021. /*
  1022. * Set counter and jump addresses for the right
  1023. * number of NOP cycles.
  1024. * The number of supported NOP cycles can range from -1 to infinity
  1025. * Three different cases are handled:
  1026. *
  1027. * 1. For a number of NOP cycles greater than 0, the RW Mgr looping
  1028. * mechanism will be used to insert the right number of NOPs
  1029. *
  1030. * 2. For a number of NOP cycles equals to 0, the micro-instruction
  1031. * issuing the write command will jump straight to the
  1032. * micro-instruction that turns on DQS (for DDRx), or outputs write
  1033. * data (for RLD), skipping
  1034. * the NOP micro-instruction all together
  1035. *
  1036. * 3. A number of NOP cycles equal to -1 indicates that DQS must be
  1037. * turned on in the same micro-instruction that issues the write
  1038. * command. Then we need
  1039. * to directly jump to the micro-instruction that sends out the data
  1040. *
  1041. * NOTE: Implementing this mechanism uses 2 RW Mgr jump-counters
  1042. * (2 and 3). One jump-counter (0) is used to perform multiple
  1043. * write-read operations.
  1044. * one counter left to issue this command in "multiple-group" mode
  1045. */
  1046. rw_wl_nop_cycles = seq->gbl.rw_wl_nop_cycles;
  1047. if (rw_wl_nop_cycles == -1) {
  1048. /*
  1049. * CNTR 2 - We want to execute the special write operation that
  1050. * turns on DQS right away and then skip directly to the
  1051. * instruction that sends out the data. We set the counter to a
  1052. * large number so that the jump is always taken.
  1053. */
  1054. writel(0xFF, &sdr_rw_load_mgr_regs->load_cntr2);
  1055. /* CNTR 3 - Not used */
  1056. if (test_dm) {
  1057. mcc_instruction = seq->rwcfg->lfsr_wr_rd_dm_bank_0_wl_1;
  1058. writel(seq->rwcfg->lfsr_wr_rd_dm_bank_0_data,
  1059. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  1060. writel(seq->rwcfg->lfsr_wr_rd_dm_bank_0_nop,
  1061. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  1062. } else {
  1063. mcc_instruction = seq->rwcfg->lfsr_wr_rd_bank_0_wl_1;
  1064. writel(seq->rwcfg->lfsr_wr_rd_bank_0_data,
  1065. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  1066. writel(seq->rwcfg->lfsr_wr_rd_bank_0_nop,
  1067. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  1068. }
  1069. } else if (rw_wl_nop_cycles == 0) {
  1070. /*
  1071. * CNTR 2 - We want to skip the NOP operation and go straight
  1072. * to the DQS enable instruction. We set the counter to a large
  1073. * number so that the jump is always taken.
  1074. */
  1075. writel(0xFF, &sdr_rw_load_mgr_regs->load_cntr2);
  1076. /* CNTR 3 - Not used */
  1077. if (test_dm) {
  1078. mcc_instruction = seq->rwcfg->lfsr_wr_rd_dm_bank_0;
  1079. writel(seq->rwcfg->lfsr_wr_rd_dm_bank_0_dqs,
  1080. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  1081. } else {
  1082. mcc_instruction = seq->rwcfg->lfsr_wr_rd_bank_0;
  1083. writel(seq->rwcfg->lfsr_wr_rd_bank_0_dqs,
  1084. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  1085. }
  1086. } else {
  1087. /*
  1088. * CNTR 2 - In this case we want to execute the next instruction
  1089. * and NOT take the jump. So we set the counter to 0. The jump
  1090. * address doesn't count.
  1091. */
  1092. writel(0x0, &sdr_rw_load_mgr_regs->load_cntr2);
  1093. writel(0x0, &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  1094. /*
  1095. * CNTR 3 - Set the nop counter to the number of cycles we
  1096. * need to loop for, minus 1.
  1097. */
  1098. writel(rw_wl_nop_cycles - 1, &sdr_rw_load_mgr_regs->load_cntr3);
  1099. if (test_dm) {
  1100. mcc_instruction = seq->rwcfg->lfsr_wr_rd_dm_bank_0;
  1101. writel(seq->rwcfg->lfsr_wr_rd_dm_bank_0_nop,
  1102. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  1103. } else {
  1104. mcc_instruction = seq->rwcfg->lfsr_wr_rd_bank_0;
  1105. writel(seq->rwcfg->lfsr_wr_rd_bank_0_nop,
  1106. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  1107. }
  1108. }
  1109. writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  1110. RW_MGR_RESET_READ_DATAPATH_OFFSET);
  1111. if (quick_write_mode)
  1112. writel(0x08, &sdr_rw_load_mgr_regs->load_cntr0);
  1113. else
  1114. writel(0x40, &sdr_rw_load_mgr_regs->load_cntr0);
  1115. writel(mcc_instruction, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  1116. /*
  1117. * CNTR 1 - This is used to ensure enough time elapses
  1118. * for read data to come back.
  1119. */
  1120. writel(0x30, &sdr_rw_load_mgr_regs->load_cntr1);
  1121. if (test_dm) {
  1122. writel(seq->rwcfg->lfsr_wr_rd_dm_bank_0_wait,
  1123. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  1124. } else {
  1125. writel(seq->rwcfg->lfsr_wr_rd_bank_0_wait,
  1126. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  1127. }
  1128. writel(mcc_instruction, (SDR_PHYGRP_RWMGRGRP_ADDRESS |
  1129. RW_MGR_RUN_SINGLE_GROUP_OFFSET) +
  1130. (group << 2));
  1131. }
  1132. /**
  1133. * rw_mgr_mem_calibrate_write_test() - Test writes, check for single/multiple
  1134. * pass
  1135. * @rank_bgn: Rank number
  1136. * @write_group: Write Group
  1137. * @use_dm: Use DM
  1138. * @all_correct: All bits must be correct in the mask
  1139. * @bit_chk: Resulting bit mask after the test
  1140. * @all_ranks: Test all ranks
  1141. *
  1142. * Test writes, can check for a single bit pass or multiple bit pass.
  1143. */
  1144. static int
  1145. rw_mgr_mem_calibrate_write_test(struct socfpga_sdrseq *seq,
  1146. const u32 rank_bgn, const u32 write_group,
  1147. const u32 use_dm, const u32 all_correct,
  1148. u32 *bit_chk, const u32 all_ranks)
  1149. {
  1150. const u32 rank_end = all_ranks ?
  1151. seq->rwcfg->mem_number_of_ranks :
  1152. (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
  1153. const u32 shift_ratio = seq->rwcfg->mem_dq_per_write_dqs /
  1154. seq->rwcfg->mem_virtual_groups_per_write_dqs;
  1155. const u32 correct_mask_vg = seq->param.write_correct_mask_vg;
  1156. u32 tmp_bit_chk, base_rw_mgr, group;
  1157. int vg, r;
  1158. *bit_chk = seq->param.write_correct_mask;
  1159. for (r = rank_bgn; r < rank_end; r++) {
  1160. /* Set rank */
  1161. set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_READ_WRITE);
  1162. tmp_bit_chk = 0;
  1163. for (vg = seq->rwcfg->mem_virtual_groups_per_write_dqs - 1;
  1164. vg >= 0; vg--) {
  1165. /* Reset the FIFOs to get pointers to known state. */
  1166. writel(0, &phy_mgr_cmd->fifo_reset);
  1167. group = write_group *
  1168. seq->rwcfg->mem_virtual_groups_per_write_dqs
  1169. + vg;
  1170. rw_mgr_mem_calibrate_write_test_issue(seq, group,
  1171. use_dm);
  1172. base_rw_mgr = readl(SDR_PHYGRP_RWMGRGRP_ADDRESS);
  1173. tmp_bit_chk <<= shift_ratio;
  1174. tmp_bit_chk |= (correct_mask_vg & ~(base_rw_mgr));
  1175. }
  1176. *bit_chk &= tmp_bit_chk;
  1177. }
  1178. set_rank_and_odt_mask(seq, 0, RW_MGR_ODT_MODE_OFF);
  1179. if (all_correct) {
  1180. debug_cond(DLEVEL >= 2,
  1181. "write_test(%u,%u,ALL) : %u == %u => %i\n",
  1182. write_group, use_dm, *bit_chk,
  1183. seq->param.write_correct_mask,
  1184. *bit_chk == seq->param.write_correct_mask);
  1185. return *bit_chk == seq->param.write_correct_mask;
  1186. } else {
  1187. debug_cond(DLEVEL >= 2,
  1188. "write_test(%u,%u,ONE) : %u != %i => %i\n",
  1189. write_group, use_dm, *bit_chk, 0, *bit_chk != 0);
  1190. return *bit_chk != 0x00;
  1191. }
  1192. }
  1193. /**
  1194. * rw_mgr_mem_calibrate_read_test_patterns() - Read back test patterns
  1195. * @rank_bgn: Rank number
  1196. * @group: Read/Write Group
  1197. * @all_ranks: Test all ranks
  1198. *
  1199. * Performs a guaranteed read on the patterns we are going to use during a
  1200. * read test to ensure memory works.
  1201. */
  1202. static int
  1203. rw_mgr_mem_calibrate_read_test_patterns(struct socfpga_sdrseq *seq,
  1204. const u32 rank_bgn, const u32 group,
  1205. const u32 all_ranks)
  1206. {
  1207. const u32 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
  1208. RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  1209. const u32 addr_offset =
  1210. (group * seq->rwcfg->mem_virtual_groups_per_read_dqs)
  1211. << 2;
  1212. const u32 rank_end = all_ranks ?
  1213. seq->rwcfg->mem_number_of_ranks :
  1214. (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
  1215. const u32 shift_ratio = seq->rwcfg->mem_dq_per_read_dqs /
  1216. seq->rwcfg->mem_virtual_groups_per_read_dqs;
  1217. const u32 correct_mask_vg = seq->param.read_correct_mask_vg;
  1218. u32 tmp_bit_chk, base_rw_mgr, bit_chk;
  1219. int vg, r;
  1220. int ret = 0;
  1221. bit_chk = seq->param.read_correct_mask;
  1222. for (r = rank_bgn; r < rank_end; r++) {
  1223. /* Set rank */
  1224. set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_READ_WRITE);
  1225. /* Load up a constant bursts of read commands */
  1226. writel(0x20, &sdr_rw_load_mgr_regs->load_cntr0);
  1227. writel(seq->rwcfg->guaranteed_read,
  1228. &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  1229. writel(0x20, &sdr_rw_load_mgr_regs->load_cntr1);
  1230. writel(seq->rwcfg->guaranteed_read_cont,
  1231. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  1232. tmp_bit_chk = 0;
  1233. for (vg = seq->rwcfg->mem_virtual_groups_per_read_dqs - 1;
  1234. vg >= 0; vg--) {
  1235. /* Reset the FIFOs to get pointers to known state. */
  1236. writel(0, &phy_mgr_cmd->fifo_reset);
  1237. writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  1238. RW_MGR_RESET_READ_DATAPATH_OFFSET);
  1239. writel(seq->rwcfg->guaranteed_read,
  1240. addr + addr_offset + (vg << 2));
  1241. base_rw_mgr = readl(SDR_PHYGRP_RWMGRGRP_ADDRESS);
  1242. tmp_bit_chk <<= shift_ratio;
  1243. tmp_bit_chk |= correct_mask_vg & ~base_rw_mgr;
  1244. }
  1245. bit_chk &= tmp_bit_chk;
  1246. }
  1247. writel(seq->rwcfg->clear_dqs_enable, addr + (group << 2));
  1248. set_rank_and_odt_mask(seq, 0, RW_MGR_ODT_MODE_OFF);
  1249. if (bit_chk != seq->param.read_correct_mask)
  1250. ret = -EIO;
  1251. debug_cond(DLEVEL >= 1,
  1252. "%s:%d test_load_patterns(%u,ALL) => (%u == %u) => %i\n",
  1253. __func__, __LINE__, group, bit_chk,
  1254. seq->param.read_correct_mask, ret);
  1255. return ret;
  1256. }
  1257. /**
  1258. * rw_mgr_mem_calibrate_read_load_patterns() - Load up the patterns for read
  1259. * test
  1260. * @rank_bgn: Rank number
  1261. * @all_ranks: Test all ranks
  1262. *
  1263. * Load up the patterns we are going to use during a read test.
  1264. */
  1265. static void rw_mgr_mem_calibrate_read_load_patterns(struct socfpga_sdrseq *seq,
  1266. const u32 rank_bgn,
  1267. const int all_ranks)
  1268. {
  1269. const u32 rank_end = all_ranks ?
  1270. seq->rwcfg->mem_number_of_ranks :
  1271. (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
  1272. u32 r;
  1273. debug("%s:%d\n", __func__, __LINE__);
  1274. for (r = rank_bgn; r < rank_end; r++) {
  1275. /* set rank */
  1276. set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_READ_WRITE);
  1277. /* Load up a constant bursts */
  1278. writel(0x20, &sdr_rw_load_mgr_regs->load_cntr0);
  1279. writel(seq->rwcfg->guaranteed_write_wait0,
  1280. &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  1281. writel(0x20, &sdr_rw_load_mgr_regs->load_cntr1);
  1282. writel(seq->rwcfg->guaranteed_write_wait1,
  1283. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  1284. writel(0x04, &sdr_rw_load_mgr_regs->load_cntr2);
  1285. writel(seq->rwcfg->guaranteed_write_wait2,
  1286. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  1287. writel(0x04, &sdr_rw_load_mgr_regs->load_cntr3);
  1288. writel(seq->rwcfg->guaranteed_write_wait3,
  1289. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  1290. writel(seq->rwcfg->guaranteed_write,
  1291. SDR_PHYGRP_RWMGRGRP_ADDRESS |
  1292. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  1293. }
  1294. set_rank_and_odt_mask(seq, 0, RW_MGR_ODT_MODE_OFF);
  1295. }
  1296. /**
  1297. * rw_mgr_mem_calibrate_read_test() - Perform READ test on single rank
  1298. * @rank_bgn: Rank number
  1299. * @group: Read/Write group
  1300. * @num_tries: Number of retries of the test
  1301. * @all_correct: All bits must be correct in the mask
  1302. * @bit_chk: Resulting bit mask after the test
  1303. * @all_groups: Test all R/W groups
  1304. * @all_ranks: Test all ranks
  1305. *
  1306. * Try a read and see if it returns correct data back. Test has dummy reads
  1307. * inserted into the mix used to align DQS enable. Test has more thorough
  1308. * checks than the regular read test.
  1309. */
  1310. static int
  1311. rw_mgr_mem_calibrate_read_test(struct socfpga_sdrseq *seq,
  1312. const u32 rank_bgn, const u32 group,
  1313. const u32 num_tries, const u32 all_correct,
  1314. u32 *bit_chk,
  1315. const u32 all_groups, const u32 all_ranks)
  1316. {
  1317. const u32 rank_end = all_ranks ? seq->rwcfg->mem_number_of_ranks :
  1318. (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
  1319. const u32 quick_read_mode =
  1320. ((STATIC_CALIB_STEPS & CALIB_SKIP_DELAY_SWEEPS) &&
  1321. seq->misccfg->enable_super_quick_calibration);
  1322. u32 correct_mask_vg = seq->param.read_correct_mask_vg;
  1323. u32 tmp_bit_chk;
  1324. u32 base_rw_mgr;
  1325. u32 addr;
  1326. int r, vg, ret;
  1327. *bit_chk = seq->param.read_correct_mask;
  1328. for (r = rank_bgn; r < rank_end; r++) {
  1329. /* set rank */
  1330. set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_READ_WRITE);
  1331. writel(0x10, &sdr_rw_load_mgr_regs->load_cntr1);
  1332. writel(seq->rwcfg->read_b2b_wait1,
  1333. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  1334. writel(0x10, &sdr_rw_load_mgr_regs->load_cntr2);
  1335. writel(seq->rwcfg->read_b2b_wait2,
  1336. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  1337. if (quick_read_mode)
  1338. writel(0x1, &sdr_rw_load_mgr_regs->load_cntr0);
  1339. /* need at least two (1+1) reads to capture failures */
  1340. else if (all_groups)
  1341. writel(0x06, &sdr_rw_load_mgr_regs->load_cntr0);
  1342. else
  1343. writel(0x32, &sdr_rw_load_mgr_regs->load_cntr0);
  1344. writel(seq->rwcfg->read_b2b,
  1345. &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  1346. if (all_groups)
  1347. writel(seq->rwcfg->mem_if_read_dqs_width *
  1348. seq->rwcfg->mem_virtual_groups_per_read_dqs - 1,
  1349. &sdr_rw_load_mgr_regs->load_cntr3);
  1350. else
  1351. writel(0x0, &sdr_rw_load_mgr_regs->load_cntr3);
  1352. writel(seq->rwcfg->read_b2b,
  1353. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  1354. tmp_bit_chk = 0;
  1355. for (vg = seq->rwcfg->mem_virtual_groups_per_read_dqs - 1;
  1356. vg >= 0; vg--) {
  1357. /* Reset the FIFOs to get pointers to known state. */
  1358. writel(0, &phy_mgr_cmd->fifo_reset);
  1359. writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  1360. RW_MGR_RESET_READ_DATAPATH_OFFSET);
  1361. if (all_groups) {
  1362. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
  1363. RW_MGR_RUN_ALL_GROUPS_OFFSET;
  1364. } else {
  1365. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
  1366. RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  1367. }
  1368. writel(seq->rwcfg->read_b2b, addr +
  1369. ((group *
  1370. seq->rwcfg->mem_virtual_groups_per_read_dqs +
  1371. vg) << 2));
  1372. base_rw_mgr = readl(SDR_PHYGRP_RWMGRGRP_ADDRESS);
  1373. tmp_bit_chk <<=
  1374. seq->rwcfg->mem_dq_per_read_dqs /
  1375. seq->rwcfg->mem_virtual_groups_per_read_dqs;
  1376. tmp_bit_chk |= correct_mask_vg & ~(base_rw_mgr);
  1377. }
  1378. *bit_chk &= tmp_bit_chk;
  1379. }
  1380. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  1381. writel(seq->rwcfg->clear_dqs_enable, addr + (group << 2));
  1382. set_rank_and_odt_mask(seq, 0, RW_MGR_ODT_MODE_OFF);
  1383. if (all_correct) {
  1384. ret = (*bit_chk == seq->param.read_correct_mask);
  1385. debug_cond(DLEVEL >= 2,
  1386. "%s:%d read_test(%u,ALL,%u) => (%u == %u) => %i\n",
  1387. __func__, __LINE__, group, all_groups, *bit_chk,
  1388. seq->param.read_correct_mask, ret);
  1389. } else {
  1390. ret = (*bit_chk != 0x00);
  1391. debug_cond(DLEVEL >= 2,
  1392. "%s:%d read_test(%u,ONE,%u) => (%u != %u) => %i\n",
  1393. __func__, __LINE__, group, all_groups, *bit_chk,
  1394. 0, ret);
  1395. }
  1396. return ret;
  1397. }
  1398. /**
  1399. * rw_mgr_mem_calibrate_read_test_all_ranks() - Perform READ test on all ranks
  1400. * @grp: Read/Write group
  1401. * @num_tries: Number of retries of the test
  1402. * @all_correct: All bits must be correct in the mask
  1403. * @all_groups: Test all R/W groups
  1404. *
  1405. * Perform a READ test across all memory ranks.
  1406. */
  1407. static int
  1408. rw_mgr_mem_calibrate_read_test_all_ranks(struct socfpga_sdrseq *seq,
  1409. const u32 grp, const u32 num_tries,
  1410. const u32 all_correct,
  1411. const u32 all_groups)
  1412. {
  1413. u32 bit_chk;
  1414. return rw_mgr_mem_calibrate_read_test(seq, 0, grp, num_tries,
  1415. all_correct, &bit_chk, all_groups,
  1416. 1);
  1417. }
  1418. /**
  1419. * rw_mgr_incr_vfifo() - Increase VFIFO value
  1420. * @grp: Read/Write group
  1421. *
  1422. * Increase VFIFO value.
  1423. */
  1424. static void rw_mgr_incr_vfifo(const u32 grp)
  1425. {
  1426. writel(grp, &phy_mgr_cmd->inc_vfifo_hard_phy);
  1427. }
  1428. /**
  1429. * rw_mgr_decr_vfifo() - Decrease VFIFO value
  1430. * @grp: Read/Write group
  1431. *
  1432. * Decrease VFIFO value.
  1433. */
  1434. static void rw_mgr_decr_vfifo(struct socfpga_sdrseq *seq, const u32 grp)
  1435. {
  1436. u32 i;
  1437. for (i = 0; i < seq->misccfg->read_valid_fifo_size - 1; i++)
  1438. rw_mgr_incr_vfifo(grp);
  1439. }
  1440. /**
  1441. * find_vfifo_failing_read() - Push VFIFO to get a failing read
  1442. * @grp: Read/Write group
  1443. *
  1444. * Push VFIFO until a failing read happens.
  1445. */
  1446. static int find_vfifo_failing_read(struct socfpga_sdrseq *seq,
  1447. const u32 grp)
  1448. {
  1449. u32 v, ret, fail_cnt = 0;
  1450. for (v = 0; v < seq->misccfg->read_valid_fifo_size; v++) {
  1451. debug_cond(DLEVEL >= 2, "%s:%d: vfifo %u\n",
  1452. __func__, __LINE__, v);
  1453. ret = rw_mgr_mem_calibrate_read_test_all_ranks(seq, grp, 1,
  1454. PASS_ONE_BIT, 0);
  1455. if (!ret) {
  1456. fail_cnt++;
  1457. if (fail_cnt == 2)
  1458. return v;
  1459. }
  1460. /* Fiddle with FIFO. */
  1461. rw_mgr_incr_vfifo(grp);
  1462. }
  1463. /* No failing read found! Something must have gone wrong. */
  1464. debug_cond(DLEVEL >= 2, "%s:%d: vfifo failed\n", __func__, __LINE__);
  1465. return 0;
  1466. }
  1467. /**
  1468. * sdr_find_phase_delay() - Find DQS enable phase or delay
  1469. * @working: If 1, look for working phase/delay, if 0, look for non-working
  1470. * @delay: If 1, look for delay, if 0, look for phase
  1471. * @grp: Read/Write group
  1472. * @work: Working window position
  1473. * @work_inc: Working window increment
  1474. * @pd: DQS Phase/Delay Iterator
  1475. *
  1476. * Find working or non-working DQS enable phase setting.
  1477. */
  1478. static int sdr_find_phase_delay(struct socfpga_sdrseq *seq, int working,
  1479. int delay, const u32 grp, u32 *work,
  1480. const u32 work_inc, u32 *pd)
  1481. {
  1482. const u32 max = delay ? seq->iocfg->dqs_en_delay_max :
  1483. seq->iocfg->dqs_en_phase_max;
  1484. u32 ret;
  1485. for (; *pd <= max; (*pd)++) {
  1486. if (delay)
  1487. scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, *pd);
  1488. else
  1489. scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, *pd);
  1490. ret = rw_mgr_mem_calibrate_read_test_all_ranks(seq, grp, 1,
  1491. PASS_ONE_BIT, 0);
  1492. if (!working)
  1493. ret = !ret;
  1494. if (ret)
  1495. return 0;
  1496. if (work)
  1497. *work += work_inc;
  1498. }
  1499. return -EINVAL;
  1500. }
  1501. /**
  1502. * sdr_find_phase() - Find DQS enable phase
  1503. * @working: If 1, look for working phase, if 0, look for non-working phase
  1504. * @grp: Read/Write group
  1505. * @work: Working window position
  1506. * @i: Iterator
  1507. * @p: DQS Phase Iterator
  1508. *
  1509. * Find working or non-working DQS enable phase setting.
  1510. */
  1511. static int sdr_find_phase(struct socfpga_sdrseq *seq, int working,
  1512. const u32 grp, u32 *work, u32 *i, u32 *p)
  1513. {
  1514. const u32 end = seq->misccfg->read_valid_fifo_size + (working ? 0 : 1);
  1515. int ret;
  1516. for (; *i < end; (*i)++) {
  1517. if (working)
  1518. *p = 0;
  1519. ret = sdr_find_phase_delay(seq, working, 0, grp, work,
  1520. seq->iocfg->delay_per_opa_tap, p);
  1521. if (!ret)
  1522. return 0;
  1523. if (*p > seq->iocfg->dqs_en_phase_max) {
  1524. /* Fiddle with FIFO. */
  1525. rw_mgr_incr_vfifo(grp);
  1526. if (!working)
  1527. *p = 0;
  1528. }
  1529. }
  1530. return -EINVAL;
  1531. }
  1532. /**
  1533. * sdr_working_phase() - Find working DQS enable phase
  1534. * @grp: Read/Write group
  1535. * @work_bgn: Working window start position
  1536. * @d: dtaps output value
  1537. * @p: DQS Phase Iterator
  1538. * @i: Iterator
  1539. *
  1540. * Find working DQS enable phase setting.
  1541. */
  1542. static int sdr_working_phase(struct socfpga_sdrseq *seq, const u32 grp,
  1543. u32 *work_bgn, u32 *d, u32 *p, u32 *i)
  1544. {
  1545. const u32 dtaps_per_ptap = seq->iocfg->delay_per_opa_tap /
  1546. seq->iocfg->delay_per_dqs_en_dchain_tap;
  1547. int ret;
  1548. *work_bgn = 0;
  1549. for (*d = 0; *d <= dtaps_per_ptap; (*d)++) {
  1550. *i = 0;
  1551. scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, *d);
  1552. ret = sdr_find_phase(seq, 1, grp, work_bgn, i, p);
  1553. if (!ret)
  1554. return 0;
  1555. *work_bgn += seq->iocfg->delay_per_dqs_en_dchain_tap;
  1556. }
  1557. /* Cannot find working solution */
  1558. debug_cond(DLEVEL >= 2, "%s:%d find_dqs_en_phase: no vfifo/ptap/dtap\n",
  1559. __func__, __LINE__);
  1560. return -EINVAL;
  1561. }
  1562. /**
  1563. * sdr_backup_phase() - Find DQS enable backup phase
  1564. * @grp: Read/Write group
  1565. * @work_bgn: Working window start position
  1566. * @p: DQS Phase Iterator
  1567. *
  1568. * Find DQS enable backup phase setting.
  1569. */
  1570. static void sdr_backup_phase(struct socfpga_sdrseq *seq, const u32 grp,
  1571. u32 *work_bgn, u32 *p)
  1572. {
  1573. u32 tmp_delay, d;
  1574. int ret;
  1575. /* Special case code for backing up a phase */
  1576. if (*p == 0) {
  1577. *p = seq->iocfg->dqs_en_phase_max;
  1578. rw_mgr_decr_vfifo(seq, grp);
  1579. } else {
  1580. (*p)--;
  1581. }
  1582. tmp_delay = *work_bgn - seq->iocfg->delay_per_opa_tap;
  1583. scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, *p);
  1584. for (d = 0; d <= seq->iocfg->dqs_en_delay_max && tmp_delay < *work_bgn;
  1585. d++) {
  1586. scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, d);
  1587. ret = rw_mgr_mem_calibrate_read_test_all_ranks(seq, grp, 1,
  1588. PASS_ONE_BIT, 0);
  1589. if (ret) {
  1590. *work_bgn = tmp_delay;
  1591. break;
  1592. }
  1593. tmp_delay += seq->iocfg->delay_per_dqs_en_dchain_tap;
  1594. }
  1595. /* Restore VFIFO to old state before we decremented it (if needed). */
  1596. (*p)++;
  1597. if (*p > seq->iocfg->dqs_en_phase_max) {
  1598. *p = 0;
  1599. rw_mgr_incr_vfifo(grp);
  1600. }
  1601. scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, 0);
  1602. }
  1603. /**
  1604. * sdr_nonworking_phase() - Find non-working DQS enable phase
  1605. * @grp: Read/Write group
  1606. * @work_end: Working window end position
  1607. * @p: DQS Phase Iterator
  1608. * @i: Iterator
  1609. *
  1610. * Find non-working DQS enable phase setting.
  1611. */
  1612. static int sdr_nonworking_phase(struct socfpga_sdrseq *seq,
  1613. const u32 grp, u32 *work_end, u32 *p, u32 *i)
  1614. {
  1615. int ret;
  1616. (*p)++;
  1617. *work_end += seq->iocfg->delay_per_opa_tap;
  1618. if (*p > seq->iocfg->dqs_en_phase_max) {
  1619. /* Fiddle with FIFO. */
  1620. *p = 0;
  1621. rw_mgr_incr_vfifo(grp);
  1622. }
  1623. ret = sdr_find_phase(seq, 0, grp, work_end, i, p);
  1624. if (ret) {
  1625. /* Cannot see edge of failing read. */
  1626. debug_cond(DLEVEL >= 2, "%s:%d: end: failed\n",
  1627. __func__, __LINE__);
  1628. }
  1629. return ret;
  1630. }
  1631. /**
  1632. * sdr_find_window_center() - Find center of the working DQS window.
  1633. * @grp: Read/Write group
  1634. * @work_bgn: First working settings
  1635. * @work_end: Last working settings
  1636. *
  1637. * Find center of the working DQS enable window.
  1638. */
  1639. static int sdr_find_window_center(struct socfpga_sdrseq *seq,
  1640. const u32 grp, const u32 work_bgn,
  1641. const u32 work_end)
  1642. {
  1643. u32 work_mid;
  1644. int tmp_delay = 0;
  1645. int i, p, d;
  1646. work_mid = (work_bgn + work_end) / 2;
  1647. debug_cond(DLEVEL >= 2, "work_bgn=%d work_end=%d work_mid=%d\n",
  1648. work_bgn, work_end, work_mid);
  1649. /* Get the middle delay to be less than a VFIFO delay */
  1650. tmp_delay = (seq->iocfg->dqs_en_phase_max + 1)
  1651. * seq->iocfg->delay_per_opa_tap;
  1652. debug_cond(DLEVEL >= 2, "vfifo ptap delay %d\n", tmp_delay);
  1653. work_mid %= tmp_delay;
  1654. debug_cond(DLEVEL >= 2, "new work_mid %d\n", work_mid);
  1655. tmp_delay = rounddown(work_mid, seq->iocfg->delay_per_opa_tap);
  1656. if (tmp_delay > seq->iocfg->dqs_en_phase_max
  1657. * seq->iocfg->delay_per_opa_tap) {
  1658. tmp_delay = seq->iocfg->dqs_en_phase_max
  1659. * seq->iocfg->delay_per_opa_tap;
  1660. }
  1661. p = tmp_delay / seq->iocfg->delay_per_opa_tap;
  1662. debug_cond(DLEVEL >= 2, "new p %d, tmp_delay=%d\n", p, tmp_delay);
  1663. d = DIV_ROUND_UP(work_mid - tmp_delay,
  1664. seq->iocfg->delay_per_dqs_en_dchain_tap);
  1665. if (d > seq->iocfg->dqs_en_delay_max)
  1666. d = seq->iocfg->dqs_en_delay_max;
  1667. tmp_delay += d * seq->iocfg->delay_per_dqs_en_dchain_tap;
  1668. debug_cond(DLEVEL >= 2, "new d %d, tmp_delay=%d\n", d, tmp_delay);
  1669. scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, p);
  1670. scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, d);
  1671. /*
  1672. * push vfifo until we can successfully calibrate. We can do this
  1673. * because the largest possible margin in 1 VFIFO cycle.
  1674. */
  1675. for (i = 0; i < seq->misccfg->read_valid_fifo_size; i++) {
  1676. debug_cond(DLEVEL >= 2, "find_dqs_en_phase: center\n");
  1677. if (rw_mgr_mem_calibrate_read_test_all_ranks(seq, grp, 1,
  1678. PASS_ONE_BIT,
  1679. 0)) {
  1680. debug_cond(DLEVEL >= 2,
  1681. "%s:%d center: found: ptap=%u dtap=%u\n",
  1682. __func__, __LINE__, p, d);
  1683. return 0;
  1684. }
  1685. /* Fiddle with FIFO. */
  1686. rw_mgr_incr_vfifo(grp);
  1687. }
  1688. debug_cond(DLEVEL >= 2, "%s:%d center: failed.\n",
  1689. __func__, __LINE__);
  1690. return -EINVAL;
  1691. }
  1692. /**
  1693. * rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase() - Find a good DQS enable to
  1694. * use
  1695. * @grp: Read/Write Group
  1696. *
  1697. * Find a good DQS enable to use.
  1698. */
  1699. static int
  1700. rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(struct socfpga_sdrseq *seq,
  1701. const u32 grp)
  1702. {
  1703. u32 d, p, i;
  1704. u32 dtaps_per_ptap;
  1705. u32 work_bgn, work_end;
  1706. u32 found_passing_read, found_failing_read = 0, initial_failing_dtap;
  1707. int ret;
  1708. debug("%s:%d %u\n", __func__, __LINE__, grp);
  1709. reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
  1710. scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, 0);
  1711. scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, 0);
  1712. /* Step 0: Determine number of delay taps for each phase tap. */
  1713. dtaps_per_ptap = seq->iocfg->delay_per_opa_tap /
  1714. seq->iocfg->delay_per_dqs_en_dchain_tap;
  1715. /* Step 1: First push vfifo until we get a failing read. */
  1716. find_vfifo_failing_read(seq, grp);
  1717. /* Step 2: Find first working phase, increment in ptaps. */
  1718. work_bgn = 0;
  1719. ret = sdr_working_phase(seq, grp, &work_bgn, &d, &p, &i);
  1720. if (ret)
  1721. return ret;
  1722. work_end = work_bgn;
  1723. /*
  1724. * If d is 0 then the working window covers a phase tap and we can
  1725. * follow the old procedure. Otherwise, we've found the beginning
  1726. * and we need to increment the dtaps until we find the end.
  1727. */
  1728. if (d == 0) {
  1729. /*
  1730. * Step 3a: If we have room, back off by one and
  1731. * increment in dtaps.
  1732. */
  1733. sdr_backup_phase(seq, grp, &work_bgn, &p);
  1734. /*
  1735. * Step 4a: go forward from working phase to non working
  1736. * phase, increment in ptaps.
  1737. */
  1738. ret = sdr_nonworking_phase(seq, grp, &work_end, &p, &i);
  1739. if (ret)
  1740. return ret;
  1741. /* Step 5a: Back off one from last, increment in dtaps. */
  1742. /* Special case code for backing up a phase */
  1743. if (p == 0) {
  1744. p = seq->iocfg->dqs_en_phase_max;
  1745. rw_mgr_decr_vfifo(seq, grp);
  1746. } else {
  1747. p = p - 1;
  1748. }
  1749. work_end -= seq->iocfg->delay_per_opa_tap;
  1750. scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, p);
  1751. d = 0;
  1752. debug_cond(DLEVEL >= 2, "%s:%d p: ptap=%u\n",
  1753. __func__, __LINE__, p);
  1754. }
  1755. /* The dtap increment to find the failing edge is done here. */
  1756. sdr_find_phase_delay(seq, 0, 1, grp, &work_end,
  1757. seq->iocfg->delay_per_dqs_en_dchain_tap, &d);
  1758. /* Go back to working dtap */
  1759. if (d != 0)
  1760. work_end -= seq->iocfg->delay_per_dqs_en_dchain_tap;
  1761. debug_cond(DLEVEL >= 2,
  1762. "%s:%d p/d: ptap=%u dtap=%u end=%u\n",
  1763. __func__, __LINE__, p, d - 1, work_end);
  1764. if (work_end < work_bgn) {
  1765. /* nil range */
  1766. debug_cond(DLEVEL >= 2, "%s:%d end-2: failed\n",
  1767. __func__, __LINE__);
  1768. return -EINVAL;
  1769. }
  1770. debug_cond(DLEVEL >= 2, "%s:%d found range [%u,%u]\n",
  1771. __func__, __LINE__, work_bgn, work_end);
  1772. /*
  1773. * We need to calculate the number of dtaps that equal a ptap.
  1774. * To do that we'll back up a ptap and re-find the edge of the
  1775. * window using dtaps
  1776. */
  1777. debug_cond(DLEVEL >= 2, "%s:%d calculate dtaps_per_ptap for tracking\n",
  1778. __func__, __LINE__);
  1779. /* Special case code for backing up a phase */
  1780. if (p == 0) {
  1781. p = seq->iocfg->dqs_en_phase_max;
  1782. rw_mgr_decr_vfifo(seq, grp);
  1783. debug_cond(DLEVEL >= 2, "%s:%d backedup cycle/phase: p=%u\n",
  1784. __func__, __LINE__, p);
  1785. } else {
  1786. p = p - 1;
  1787. debug_cond(DLEVEL >= 2, "%s:%d backedup phase only: p=%u",
  1788. __func__, __LINE__, p);
  1789. }
  1790. scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, p);
  1791. /*
  1792. * Increase dtap until we first see a passing read (in case the
  1793. * window is smaller than a ptap), and then a failing read to
  1794. * mark the edge of the window again.
  1795. */
  1796. /* Find a passing read. */
  1797. debug_cond(DLEVEL >= 2, "%s:%d find passing read\n",
  1798. __func__, __LINE__);
  1799. initial_failing_dtap = d;
  1800. found_passing_read = !sdr_find_phase_delay(seq, 1, 1, grp, NULL, 0, &d);
  1801. if (found_passing_read) {
  1802. /* Find a failing read. */
  1803. debug_cond(DLEVEL >= 2, "%s:%d find failing read\n",
  1804. __func__, __LINE__);
  1805. d++;
  1806. found_failing_read = !sdr_find_phase_delay(seq, 0, 1, grp, NULL,
  1807. 0, &d);
  1808. } else {
  1809. debug_cond(DLEVEL >= 1,
  1810. "%s:%d failed to calculate dtaps per ptap. Fall back on static value\n",
  1811. __func__, __LINE__);
  1812. }
  1813. /*
  1814. * The dynamically calculated dtaps_per_ptap is only valid if we
  1815. * found a passing/failing read. If we didn't, it means d hit the max
  1816. * (seq->iocfg->dqs_en_delay_max). Otherwise, dtaps_per_ptap retains its
  1817. * statically calculated value.
  1818. */
  1819. if (found_passing_read && found_failing_read)
  1820. dtaps_per_ptap = d - initial_failing_dtap;
  1821. writel(dtaps_per_ptap, &sdr_reg_file->dtaps_per_ptap);
  1822. debug_cond(DLEVEL >= 2, "%s:%d dtaps_per_ptap=%u - %u = %u",
  1823. __func__, __LINE__, d, initial_failing_dtap, dtaps_per_ptap);
  1824. /* Step 6: Find the centre of the window. */
  1825. ret = sdr_find_window_center(seq, grp, work_bgn, work_end);
  1826. return ret;
  1827. }
  1828. /**
  1829. * search_stop_check() - Check if the detected edge is valid
  1830. * @write: Perform read (Stage 2) or write (Stage 3) calibration
  1831. * @d: DQS delay
  1832. * @rank_bgn: Rank number
  1833. * @write_group: Write Group
  1834. * @read_group: Read Group
  1835. * @bit_chk: Resulting bit mask after the test
  1836. * @sticky_bit_chk: Resulting sticky bit mask after the test
  1837. * @use_read_test: Perform read test
  1838. *
  1839. * Test if the found edge is valid.
  1840. */
  1841. static u32 search_stop_check(struct socfpga_sdrseq *seq, const int write,
  1842. const int d, const int rank_bgn,
  1843. const u32 write_group, const u32 read_group,
  1844. u32 *bit_chk, u32 *sticky_bit_chk,
  1845. const u32 use_read_test)
  1846. {
  1847. const u32 ratio = seq->rwcfg->mem_if_read_dqs_width /
  1848. seq->rwcfg->mem_if_write_dqs_width;
  1849. const u32 correct_mask = write ? seq->param.write_correct_mask :
  1850. seq->param.read_correct_mask;
  1851. const u32 per_dqs = write ? seq->rwcfg->mem_dq_per_write_dqs :
  1852. seq->rwcfg->mem_dq_per_read_dqs;
  1853. u32 ret;
  1854. /*
  1855. * Stop searching when the read test doesn't pass AND when
  1856. * we've seen a passing read on every bit.
  1857. */
  1858. if (write) { /* WRITE-ONLY */
  1859. ret = !rw_mgr_mem_calibrate_write_test(seq, rank_bgn,
  1860. write_group, 0,
  1861. PASS_ONE_BIT, bit_chk,
  1862. 0);
  1863. } else if (use_read_test) { /* READ-ONLY */
  1864. ret = !rw_mgr_mem_calibrate_read_test(seq, rank_bgn, read_group,
  1865. NUM_READ_PB_TESTS,
  1866. PASS_ONE_BIT, bit_chk,
  1867. 0, 0);
  1868. } else { /* READ-ONLY */
  1869. rw_mgr_mem_calibrate_write_test(seq, rank_bgn, write_group, 0,
  1870. PASS_ONE_BIT, bit_chk, 0);
  1871. *bit_chk = *bit_chk >> (per_dqs *
  1872. (read_group - (write_group * ratio)));
  1873. ret = (*bit_chk == 0);
  1874. }
  1875. *sticky_bit_chk = *sticky_bit_chk | *bit_chk;
  1876. ret = ret && (*sticky_bit_chk == correct_mask);
  1877. debug_cond(DLEVEL >= 2,
  1878. "%s:%d center(left): dtap=%u => %u == %u && %u",
  1879. __func__, __LINE__, d,
  1880. *sticky_bit_chk, correct_mask, ret);
  1881. return ret;
  1882. }
  1883. /**
  1884. * search_left_edge() - Find left edge of DQ/DQS working phase
  1885. * @write: Perform read (Stage 2) or write (Stage 3) calibration
  1886. * @rank_bgn: Rank number
  1887. * @write_group: Write Group
  1888. * @read_group: Read Group
  1889. * @test_bgn: Rank number to begin the test
  1890. * @sticky_bit_chk: Resulting sticky bit mask after the test
  1891. * @left_edge: Left edge of the DQ/DQS phase
  1892. * @right_edge: Right edge of the DQ/DQS phase
  1893. * @use_read_test: Perform read test
  1894. *
  1895. * Find left edge of DQ/DQS working phase.
  1896. */
  1897. static void search_left_edge(struct socfpga_sdrseq *seq, const int write,
  1898. const int rank_bgn, const u32 write_group,
  1899. const u32 read_group, const u32 test_bgn,
  1900. u32 *sticky_bit_chk, int *left_edge,
  1901. int *right_edge, const u32 use_read_test)
  1902. {
  1903. const u32 delay_max = write ? seq->iocfg->io_out1_delay_max :
  1904. seq->iocfg->io_in_delay_max;
  1905. const u32 dqs_max = write ? seq->iocfg->io_out1_delay_max :
  1906. seq->iocfg->dqs_in_delay_max;
  1907. const u32 per_dqs = write ? seq->rwcfg->mem_dq_per_write_dqs :
  1908. seq->rwcfg->mem_dq_per_read_dqs;
  1909. u32 stop, bit_chk;
  1910. int i, d;
  1911. for (d = 0; d <= dqs_max; d++) {
  1912. if (write)
  1913. scc_mgr_apply_group_dq_out1_delay(seq, d);
  1914. else
  1915. scc_mgr_apply_group_dq_in_delay(seq, test_bgn, d);
  1916. writel(0, &sdr_scc_mgr->update);
  1917. stop = search_stop_check(seq, write, d, rank_bgn, write_group,
  1918. read_group, &bit_chk, sticky_bit_chk,
  1919. use_read_test);
  1920. if (stop == 1)
  1921. break;
  1922. /* stop != 1 */
  1923. for (i = 0; i < per_dqs; i++) {
  1924. if (bit_chk & 1) {
  1925. /*
  1926. * Remember a passing test as
  1927. * the left_edge.
  1928. */
  1929. left_edge[i] = d;
  1930. } else {
  1931. /*
  1932. * If a left edge has not been seen
  1933. * yet, then a future passing test
  1934. * will mark this edge as the right
  1935. * edge.
  1936. */
  1937. if (left_edge[i] == delay_max + 1)
  1938. right_edge[i] = -(d + 1);
  1939. }
  1940. bit_chk >>= 1;
  1941. }
  1942. }
  1943. /* Reset DQ delay chains to 0 */
  1944. if (write)
  1945. scc_mgr_apply_group_dq_out1_delay(seq, 0);
  1946. else
  1947. scc_mgr_apply_group_dq_in_delay(seq, test_bgn, 0);
  1948. *sticky_bit_chk = 0;
  1949. for (i = per_dqs - 1; i >= 0; i--) {
  1950. debug_cond(DLEVEL >= 2,
  1951. "%s:%d vfifo_center: left_edge[%u]: %d right_edge[%u]: %d\n",
  1952. __func__, __LINE__, i, left_edge[i],
  1953. i, right_edge[i]);
  1954. /*
  1955. * Check for cases where we haven't found the left edge,
  1956. * which makes our assignment of the the right edge invalid.
  1957. * Reset it to the illegal value.
  1958. */
  1959. if ((left_edge[i] == delay_max + 1) &&
  1960. (right_edge[i] != delay_max + 1)) {
  1961. right_edge[i] = delay_max + 1;
  1962. debug_cond(DLEVEL >= 2,
  1963. "%s:%d vfifo_center: reset right_edge[%u]: %d\n",
  1964. __func__, __LINE__, i, right_edge[i]);
  1965. }
  1966. /*
  1967. * Reset sticky bit
  1968. * READ: except for bits where we have seen both
  1969. * the left and right edge.
  1970. * WRITE: except for bits where we have seen the
  1971. * left edge.
  1972. */
  1973. *sticky_bit_chk <<= 1;
  1974. if (write) {
  1975. if (left_edge[i] != delay_max + 1)
  1976. *sticky_bit_chk |= 1;
  1977. } else {
  1978. if ((left_edge[i] != delay_max + 1) &&
  1979. (right_edge[i] != delay_max + 1))
  1980. *sticky_bit_chk |= 1;
  1981. }
  1982. }
  1983. }
  1984. /**
  1985. * search_right_edge() - Find right edge of DQ/DQS working phase
  1986. * @write: Perform read (Stage 2) or write (Stage 3) calibration
  1987. * @rank_bgn: Rank number
  1988. * @write_group: Write Group
  1989. * @read_group: Read Group
  1990. * @start_dqs: DQS start phase
  1991. * @start_dqs_en: DQS enable start phase
  1992. * @sticky_bit_chk: Resulting sticky bit mask after the test
  1993. * @left_edge: Left edge of the DQ/DQS phase
  1994. * @right_edge: Right edge of the DQ/DQS phase
  1995. * @use_read_test: Perform read test
  1996. *
  1997. * Find right edge of DQ/DQS working phase.
  1998. */
  1999. static int search_right_edge(struct socfpga_sdrseq *seq, const int write,
  2000. const int rank_bgn, const u32 write_group,
  2001. const u32 read_group, const int start_dqs,
  2002. const int start_dqs_en, u32 *sticky_bit_chk,
  2003. int *left_edge, int *right_edge,
  2004. const u32 use_read_test)
  2005. {
  2006. const u32 delay_max = write ? seq->iocfg->io_out1_delay_max :
  2007. seq->iocfg->io_in_delay_max;
  2008. const u32 dqs_max = write ? seq->iocfg->io_out1_delay_max :
  2009. seq->iocfg->dqs_in_delay_max;
  2010. const u32 per_dqs = write ? seq->rwcfg->mem_dq_per_write_dqs :
  2011. seq->rwcfg->mem_dq_per_read_dqs;
  2012. u32 stop, bit_chk;
  2013. int i, d;
  2014. for (d = 0; d <= dqs_max - start_dqs; d++) {
  2015. if (write) { /* WRITE-ONLY */
  2016. scc_mgr_apply_group_dqs_io_and_oct_out1(seq,
  2017. write_group,
  2018. d + start_dqs);
  2019. } else { /* READ-ONLY */
  2020. scc_mgr_set_dqs_bus_in_delay(read_group, d + start_dqs);
  2021. if (seq->iocfg->shift_dqs_en_when_shift_dqs) {
  2022. u32 delay = d + start_dqs_en;
  2023. if (delay > seq->iocfg->dqs_en_delay_max)
  2024. delay = seq->iocfg->dqs_en_delay_max;
  2025. scc_mgr_set_dqs_en_delay(read_group, delay);
  2026. }
  2027. scc_mgr_load_dqs(read_group);
  2028. }
  2029. writel(0, &sdr_scc_mgr->update);
  2030. stop = search_stop_check(seq, write, d, rank_bgn, write_group,
  2031. read_group, &bit_chk, sticky_bit_chk,
  2032. use_read_test);
  2033. if (stop == 1) {
  2034. if (write && (d == 0)) { /* WRITE-ONLY */
  2035. for (i = 0;
  2036. i < seq->rwcfg->mem_dq_per_write_dqs;
  2037. i++) {
  2038. /*
  2039. * d = 0 failed, but it passed when
  2040. * testing the left edge, so it must be
  2041. * marginal, set it to -1
  2042. */
  2043. if (right_edge[i] == delay_max + 1 &&
  2044. left_edge[i] != delay_max + 1)
  2045. right_edge[i] = -1;
  2046. }
  2047. }
  2048. break;
  2049. }
  2050. /* stop != 1 */
  2051. for (i = 0; i < per_dqs; i++) {
  2052. if (bit_chk & 1) {
  2053. /*
  2054. * Remember a passing test as
  2055. * the right_edge.
  2056. */
  2057. right_edge[i] = d;
  2058. } else {
  2059. if (d != 0) {
  2060. /*
  2061. * If a right edge has not
  2062. * been seen yet, then a future
  2063. * passing test will mark this
  2064. * edge as the left edge.
  2065. */
  2066. if (right_edge[i] == delay_max + 1)
  2067. left_edge[i] = -(d + 1);
  2068. } else {
  2069. /*
  2070. * d = 0 failed, but it passed
  2071. * when testing the left edge,
  2072. * so it must be marginal, set
  2073. * it to -1
  2074. */
  2075. if (right_edge[i] == delay_max + 1 &&
  2076. left_edge[i] != delay_max + 1)
  2077. right_edge[i] = -1;
  2078. /*
  2079. * If a right edge has not been
  2080. * seen yet, then a future
  2081. * passing test will mark this
  2082. * edge as the left edge.
  2083. */
  2084. else if (right_edge[i] == delay_max + 1)
  2085. left_edge[i] = -(d + 1);
  2086. }
  2087. }
  2088. debug_cond(DLEVEL >= 2, "%s:%d center[r,d=%u]: ",
  2089. __func__, __LINE__, d);
  2090. debug_cond(DLEVEL >= 2,
  2091. "bit_chk_test=%i left_edge[%u]: %d ",
  2092. bit_chk & 1, i, left_edge[i]);
  2093. debug_cond(DLEVEL >= 2, "right_edge[%u]: %d\n", i,
  2094. right_edge[i]);
  2095. bit_chk >>= 1;
  2096. }
  2097. }
  2098. /* Check that all bits have a window */
  2099. for (i = 0; i < per_dqs; i++) {
  2100. debug_cond(DLEVEL >= 2,
  2101. "%s:%d write_center: left_edge[%u]: %d right_edge[%u]: %d",
  2102. __func__, __LINE__, i, left_edge[i],
  2103. i, right_edge[i]);
  2104. if ((left_edge[i] == dqs_max + 1) ||
  2105. (right_edge[i] == dqs_max + 1))
  2106. return i + 1; /* FIXME: If we fail, retval > 0 */
  2107. }
  2108. return 0;
  2109. }
  2110. /**
  2111. * get_window_mid_index() - Find the best middle setting of DQ/DQS phase
  2112. * @write: Perform read (Stage 2) or write (Stage 3) calibration
  2113. * @left_edge: Left edge of the DQ/DQS phase
  2114. * @right_edge: Right edge of the DQ/DQS phase
  2115. * @mid_min: Best DQ/DQS phase middle setting
  2116. *
  2117. * Find index and value of the middle of the DQ/DQS working phase.
  2118. */
  2119. static int get_window_mid_index(struct socfpga_sdrseq *seq,
  2120. const int write, int *left_edge,
  2121. int *right_edge, int *mid_min)
  2122. {
  2123. const u32 per_dqs = write ? seq->rwcfg->mem_dq_per_write_dqs :
  2124. seq->rwcfg->mem_dq_per_read_dqs;
  2125. int i, mid, min_index;
  2126. /* Find middle of window for each DQ bit */
  2127. *mid_min = left_edge[0] - right_edge[0];
  2128. min_index = 0;
  2129. for (i = 1; i < per_dqs; i++) {
  2130. mid = left_edge[i] - right_edge[i];
  2131. if (mid < *mid_min) {
  2132. *mid_min = mid;
  2133. min_index = i;
  2134. }
  2135. }
  2136. /*
  2137. * -mid_min/2 represents the amount that we need to move DQS.
  2138. * If mid_min is odd and positive we'll need to add one to make
  2139. * sure the rounding in further calculations is correct (always
  2140. * bias to the right), so just add 1 for all positive values.
  2141. */
  2142. if (*mid_min > 0)
  2143. (*mid_min)++;
  2144. *mid_min = *mid_min / 2;
  2145. debug_cond(DLEVEL >= 1, "%s:%d vfifo_center: *mid_min=%d (index=%u)\n",
  2146. __func__, __LINE__, *mid_min, min_index);
  2147. return min_index;
  2148. }
  2149. /**
  2150. * center_dq_windows() - Center the DQ/DQS windows
  2151. * @write: Perform read (Stage 2) or write (Stage 3) calibration
  2152. * @left_edge: Left edge of the DQ/DQS phase
  2153. * @right_edge: Right edge of the DQ/DQS phase
  2154. * @mid_min: Adjusted DQ/DQS phase middle setting
  2155. * @orig_mid_min: Original DQ/DQS phase middle setting
  2156. * @min_index: DQ/DQS phase middle setting index
  2157. * @test_bgn: Rank number to begin the test
  2158. * @dq_margin: Amount of shift for the DQ
  2159. * @dqs_margin: Amount of shift for the DQS
  2160. *
  2161. * Align the DQ/DQS windows in each group.
  2162. */
  2163. static void center_dq_windows(struct socfpga_sdrseq *seq,
  2164. const int write, int *left_edge, int *right_edge,
  2165. const int mid_min, const int orig_mid_min,
  2166. const int min_index, const int test_bgn,
  2167. int *dq_margin, int *dqs_margin)
  2168. {
  2169. const s32 delay_max = write ? seq->iocfg->io_out1_delay_max :
  2170. seq->iocfg->io_in_delay_max;
  2171. const s32 per_dqs = write ? seq->rwcfg->mem_dq_per_write_dqs :
  2172. seq->rwcfg->mem_dq_per_read_dqs;
  2173. const s32 delay_off = write ? SCC_MGR_IO_OUT1_DELAY_OFFSET :
  2174. SCC_MGR_IO_IN_DELAY_OFFSET;
  2175. const s32 addr = SDR_PHYGRP_SCCGRP_ADDRESS | delay_off;
  2176. s32 temp_dq_io_delay1;
  2177. int shift_dq, i, p;
  2178. /* Initialize data for export structures */
  2179. *dqs_margin = delay_max + 1;
  2180. *dq_margin = delay_max + 1;
  2181. /* add delay to bring centre of all DQ windows to the same "level" */
  2182. for (i = 0, p = test_bgn; i < per_dqs; i++, p++) {
  2183. /* Use values before divide by 2 to reduce round off error */
  2184. shift_dq = (left_edge[i] - right_edge[i] -
  2185. (left_edge[min_index] - right_edge[min_index]))/2 +
  2186. (orig_mid_min - mid_min);
  2187. debug_cond(DLEVEL >= 2,
  2188. "vfifo_center: before: shift_dq[%u]=%d\n",
  2189. i, shift_dq);
  2190. temp_dq_io_delay1 = readl(addr + (i << 2));
  2191. if (shift_dq + temp_dq_io_delay1 > delay_max)
  2192. shift_dq = delay_max - temp_dq_io_delay1;
  2193. else if (shift_dq + temp_dq_io_delay1 < 0)
  2194. shift_dq = -temp_dq_io_delay1;
  2195. debug_cond(DLEVEL >= 2,
  2196. "vfifo_center: after: shift_dq[%u]=%d\n",
  2197. i, shift_dq);
  2198. if (write)
  2199. scc_mgr_set_dq_out1_delay(i,
  2200. temp_dq_io_delay1 + shift_dq);
  2201. else
  2202. scc_mgr_set_dq_in_delay(p,
  2203. temp_dq_io_delay1 + shift_dq);
  2204. scc_mgr_load_dq(p);
  2205. debug_cond(DLEVEL >= 2,
  2206. "vfifo_center: margin[%u]=[%d,%d]\n", i,
  2207. left_edge[i] - shift_dq + (-mid_min),
  2208. right_edge[i] + shift_dq - (-mid_min));
  2209. /* To determine values for export structures */
  2210. if (left_edge[i] - shift_dq + (-mid_min) < *dq_margin)
  2211. *dq_margin = left_edge[i] - shift_dq + (-mid_min);
  2212. if (right_edge[i] + shift_dq - (-mid_min) < *dqs_margin)
  2213. *dqs_margin = right_edge[i] + shift_dq - (-mid_min);
  2214. }
  2215. }
  2216. /**
  2217. * rw_mgr_mem_calibrate_vfifo_center() - Per-bit deskew DQ and centering
  2218. * @rank_bgn: Rank number
  2219. * @rw_group: Read/Write Group
  2220. * @test_bgn: Rank at which the test begins
  2221. * @use_read_test: Perform a read test
  2222. * @update_fom: Update FOM
  2223. *
  2224. * Per-bit deskew DQ and centering.
  2225. */
  2226. static int rw_mgr_mem_calibrate_vfifo_center(struct socfpga_sdrseq *seq,
  2227. const u32 rank_bgn,
  2228. const u32 rw_group,
  2229. const u32 test_bgn,
  2230. const int use_read_test,
  2231. const int update_fom)
  2232. {
  2233. const u32 addr =
  2234. SDR_PHYGRP_SCCGRP_ADDRESS + SCC_MGR_DQS_IN_DELAY_OFFSET +
  2235. (rw_group << 2);
  2236. /*
  2237. * Store these as signed since there are comparisons with
  2238. * signed numbers.
  2239. */
  2240. u32 sticky_bit_chk;
  2241. s32 left_edge[seq->rwcfg->mem_dq_per_read_dqs];
  2242. s32 right_edge[seq->rwcfg->mem_dq_per_read_dqs];
  2243. s32 orig_mid_min, mid_min;
  2244. s32 new_dqs, start_dqs, start_dqs_en = 0, final_dqs_en;
  2245. s32 dq_margin, dqs_margin;
  2246. int i, min_index;
  2247. int ret;
  2248. debug("%s:%d: %u %u", __func__, __LINE__, rw_group, test_bgn);
  2249. start_dqs = readl(addr);
  2250. if (seq->iocfg->shift_dqs_en_when_shift_dqs)
  2251. start_dqs_en = readl(addr - seq->iocfg->dqs_en_delay_offset);
  2252. /* set the left and right edge of each bit to an illegal value */
  2253. /* use (seq->iocfg->io_in_delay_max + 1) as an illegal value */
  2254. sticky_bit_chk = 0;
  2255. for (i = 0; i < seq->rwcfg->mem_dq_per_read_dqs; i++) {
  2256. left_edge[i] = seq->iocfg->io_in_delay_max + 1;
  2257. right_edge[i] = seq->iocfg->io_in_delay_max + 1;
  2258. }
  2259. /* Search for the left edge of the window for each bit */
  2260. search_left_edge(seq, 0, rank_bgn, rw_group, rw_group, test_bgn,
  2261. &sticky_bit_chk,
  2262. left_edge, right_edge, use_read_test);
  2263. /* Search for the right edge of the window for each bit */
  2264. ret = search_right_edge(seq, 0, rank_bgn, rw_group, rw_group,
  2265. start_dqs, start_dqs_en,
  2266. &sticky_bit_chk,
  2267. left_edge, right_edge, use_read_test);
  2268. if (ret) {
  2269. /*
  2270. * Restore delay chain settings before letting the loop
  2271. * in rw_mgr_mem_calibrate_vfifo to retry different
  2272. * dqs/ck relationships.
  2273. */
  2274. scc_mgr_set_dqs_bus_in_delay(rw_group, start_dqs);
  2275. if (seq->iocfg->shift_dqs_en_when_shift_dqs)
  2276. scc_mgr_set_dqs_en_delay(rw_group, start_dqs_en);
  2277. scc_mgr_load_dqs(rw_group);
  2278. writel(0, &sdr_scc_mgr->update);
  2279. debug_cond(DLEVEL >= 1,
  2280. "%s:%d vfifo_center: failed to find edge [%u]: %d %d",
  2281. __func__, __LINE__, i, left_edge[i], right_edge[i]);
  2282. if (use_read_test) {
  2283. set_failing_group_stage(seq, rw_group *
  2284. seq->rwcfg->mem_dq_per_read_dqs + i,
  2285. CAL_STAGE_VFIFO,
  2286. CAL_SUBSTAGE_VFIFO_CENTER);
  2287. } else {
  2288. set_failing_group_stage(seq, rw_group *
  2289. seq->rwcfg->mem_dq_per_read_dqs + i,
  2290. CAL_STAGE_VFIFO_AFTER_WRITES,
  2291. CAL_SUBSTAGE_VFIFO_CENTER);
  2292. }
  2293. return -EIO;
  2294. }
  2295. min_index = get_window_mid_index(seq, 0, left_edge, right_edge,
  2296. &mid_min);
  2297. /* Determine the amount we can change DQS (which is -mid_min) */
  2298. orig_mid_min = mid_min;
  2299. new_dqs = start_dqs - mid_min;
  2300. if (new_dqs > seq->iocfg->dqs_in_delay_max)
  2301. new_dqs = seq->iocfg->dqs_in_delay_max;
  2302. else if (new_dqs < 0)
  2303. new_dqs = 0;
  2304. mid_min = start_dqs - new_dqs;
  2305. debug_cond(DLEVEL >= 1, "vfifo_center: new mid_min=%d new_dqs=%d\n",
  2306. mid_min, new_dqs);
  2307. if (seq->iocfg->shift_dqs_en_when_shift_dqs) {
  2308. if (start_dqs_en - mid_min > seq->iocfg->dqs_en_delay_max)
  2309. mid_min += start_dqs_en - mid_min -
  2310. seq->iocfg->dqs_en_delay_max;
  2311. else if (start_dqs_en - mid_min < 0)
  2312. mid_min += start_dqs_en - mid_min;
  2313. }
  2314. new_dqs = start_dqs - mid_min;
  2315. debug_cond(DLEVEL >= 1,
  2316. "vfifo_center: start_dqs=%d start_dqs_en=%d new_dqs=%d mid_min=%d\n",
  2317. start_dqs,
  2318. seq->iocfg->shift_dqs_en_when_shift_dqs ? start_dqs_en : -1,
  2319. new_dqs, mid_min);
  2320. /* Add delay to bring centre of all DQ windows to the same "level". */
  2321. center_dq_windows(seq, 0, left_edge, right_edge, mid_min, orig_mid_min,
  2322. min_index, test_bgn, &dq_margin, &dqs_margin);
  2323. /* Move DQS-en */
  2324. if (seq->iocfg->shift_dqs_en_when_shift_dqs) {
  2325. final_dqs_en = start_dqs_en - mid_min;
  2326. scc_mgr_set_dqs_en_delay(rw_group, final_dqs_en);
  2327. scc_mgr_load_dqs(rw_group);
  2328. }
  2329. /* Move DQS */
  2330. scc_mgr_set_dqs_bus_in_delay(rw_group, new_dqs);
  2331. scc_mgr_load_dqs(rw_group);
  2332. debug_cond(DLEVEL >= 2,
  2333. "%s:%d vfifo_center: dq_margin=%d dqs_margin=%d",
  2334. __func__, __LINE__, dq_margin, dqs_margin);
  2335. /*
  2336. * Do not remove this line as it makes sure all of our decisions
  2337. * have been applied. Apply the update bit.
  2338. */
  2339. writel(0, &sdr_scc_mgr->update);
  2340. if ((dq_margin < 0) || (dqs_margin < 0))
  2341. return -EINVAL;
  2342. return 0;
  2343. }
  2344. /**
  2345. * rw_mgr_mem_calibrate_guaranteed_write() - Perform guaranteed write into the
  2346. * device
  2347. * @rw_group: Read/Write Group
  2348. * @phase: DQ/DQS phase
  2349. *
  2350. * Because initially no communication ca be reliably performed with the memory
  2351. * device, the sequencer uses a guaranteed write mechanism to write data into
  2352. * the memory device.
  2353. */
  2354. static int rw_mgr_mem_calibrate_guaranteed_write(struct socfpga_sdrseq *seq,
  2355. const u32 rw_group,
  2356. const u32 phase)
  2357. {
  2358. int ret;
  2359. /* Set a particular DQ/DQS phase. */
  2360. scc_mgr_set_dqdqs_output_phase_all_ranks(seq, rw_group, phase);
  2361. debug_cond(DLEVEL >= 1, "%s:%d guaranteed write: g=%u p=%u\n",
  2362. __func__, __LINE__, rw_group, phase);
  2363. /*
  2364. * Altera EMI_RM 2015.05.04 :: Figure 1-25
  2365. * Load up the patterns used by read calibration using the
  2366. * current DQDQS phase.
  2367. */
  2368. rw_mgr_mem_calibrate_read_load_patterns(seq, 0, 1);
  2369. if (seq->gbl.phy_debug_mode_flags & PHY_DEBUG_DISABLE_GUARANTEED_READ)
  2370. return 0;
  2371. /*
  2372. * Altera EMI_RM 2015.05.04 :: Figure 1-26
  2373. * Back-to-Back reads of the patterns used for calibration.
  2374. */
  2375. ret = rw_mgr_mem_calibrate_read_test_patterns(seq, 0, rw_group, 1);
  2376. if (ret)
  2377. debug_cond(DLEVEL >= 1,
  2378. "%s:%d Guaranteed read test failed: g=%u p=%u\n",
  2379. __func__, __LINE__, rw_group, phase);
  2380. return ret;
  2381. }
  2382. /**
  2383. * rw_mgr_mem_calibrate_dqs_enable_calibration() - DQS Enable Calibration
  2384. * @rw_group: Read/Write Group
  2385. * @test_bgn: Rank at which the test begins
  2386. *
  2387. * DQS enable calibration ensures reliable capture of the DQ signal without
  2388. * glitches on the DQS line.
  2389. */
  2390. static int
  2391. rw_mgr_mem_calibrate_dqs_enable_calibration(struct socfpga_sdrseq *seq,
  2392. const u32 rw_group,
  2393. const u32 test_bgn)
  2394. {
  2395. /*
  2396. * Altera EMI_RM 2015.05.04 :: Figure 1-27
  2397. * DQS and DQS Eanble Signal Relationships.
  2398. */
  2399. /* We start at zero, so have one less dq to devide among */
  2400. const u32 delay_step = seq->iocfg->io_in_delay_max /
  2401. (seq->rwcfg->mem_dq_per_read_dqs - 1);
  2402. int ret;
  2403. u32 i, p, d, r;
  2404. debug("%s:%d (%u,%u)\n", __func__, __LINE__, rw_group, test_bgn);
  2405. /* Try different dq_in_delays since the DQ path is shorter than DQS. */
  2406. for (r = 0; r < seq->rwcfg->mem_number_of_ranks;
  2407. r += NUM_RANKS_PER_SHADOW_REG) {
  2408. for (i = 0, p = test_bgn, d = 0;
  2409. i < seq->rwcfg->mem_dq_per_read_dqs;
  2410. i++, p++, d += delay_step) {
  2411. debug_cond(DLEVEL >= 1,
  2412. "%s:%d: g=%u r=%u i=%u p=%u d=%u\n",
  2413. __func__, __LINE__, rw_group, r, i, p, d);
  2414. scc_mgr_set_dq_in_delay(p, d);
  2415. scc_mgr_load_dq(p);
  2416. }
  2417. writel(0, &sdr_scc_mgr->update);
  2418. }
  2419. /*
  2420. * Try rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase across different
  2421. * dq_in_delay values
  2422. */
  2423. ret = rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(seq, rw_group);
  2424. debug_cond(DLEVEL >= 1,
  2425. "%s:%d: g=%u found=%u; Reseting delay chain to zero\n",
  2426. __func__, __LINE__, rw_group, !ret);
  2427. for (r = 0; r < seq->rwcfg->mem_number_of_ranks;
  2428. r += NUM_RANKS_PER_SHADOW_REG) {
  2429. scc_mgr_apply_group_dq_in_delay(seq, test_bgn, 0);
  2430. writel(0, &sdr_scc_mgr->update);
  2431. }
  2432. return ret;
  2433. }
  2434. /**
  2435. * rw_mgr_mem_calibrate_dq_dqs_centering() - Centering DQ/DQS
  2436. * @rw_group: Read/Write Group
  2437. * @test_bgn: Rank at which the test begins
  2438. * @use_read_test: Perform a read test
  2439. * @update_fom: Update FOM
  2440. *
  2441. * The centerin DQ/DQS stage attempts to align DQ and DQS signals on reads
  2442. * within a group.
  2443. */
  2444. static int
  2445. rw_mgr_mem_calibrate_dq_dqs_centering(struct socfpga_sdrseq *seq,
  2446. const u32 rw_group, const u32 test_bgn,
  2447. const int use_read_test,
  2448. const int update_fom)
  2449. {
  2450. int ret, grp_calibrated;
  2451. u32 rank_bgn, sr;
  2452. /*
  2453. * Altera EMI_RM 2015.05.04 :: Figure 1-28
  2454. * Read per-bit deskew can be done on a per shadow register basis.
  2455. */
  2456. grp_calibrated = 1;
  2457. for (rank_bgn = 0, sr = 0;
  2458. rank_bgn < seq->rwcfg->mem_number_of_ranks;
  2459. rank_bgn += NUM_RANKS_PER_SHADOW_REG, sr++) {
  2460. ret = rw_mgr_mem_calibrate_vfifo_center(seq, rank_bgn, rw_group,
  2461. test_bgn,
  2462. use_read_test,
  2463. update_fom);
  2464. if (!ret)
  2465. continue;
  2466. grp_calibrated = 0;
  2467. }
  2468. if (!grp_calibrated)
  2469. return -EIO;
  2470. return 0;
  2471. }
  2472. /**
  2473. * rw_mgr_mem_calibrate_vfifo() - Calibrate the read valid prediction FIFO
  2474. * @rw_group: Read/Write Group
  2475. * @test_bgn: Rank at which the test begins
  2476. *
  2477. * Stage 1: Calibrate the read valid prediction FIFO.
  2478. *
  2479. * This function implements UniPHY calibration Stage 1, as explained in
  2480. * detail in Altera EMI_RM 2015.05.04 , "UniPHY Calibration Stages".
  2481. *
  2482. * - read valid prediction will consist of finding:
  2483. * - DQS enable phase and DQS enable delay (DQS Enable Calibration)
  2484. * - DQS input phase and DQS input delay (DQ/DQS Centering)
  2485. * - we also do a per-bit deskew on the DQ lines.
  2486. */
  2487. static int rw_mgr_mem_calibrate_vfifo(struct socfpga_sdrseq *seq,
  2488. const u32 rw_group, const u32 test_bgn)
  2489. {
  2490. u32 p, d;
  2491. u32 dtaps_per_ptap;
  2492. u32 failed_substage;
  2493. int ret;
  2494. debug("%s:%d: %u %u\n", __func__, __LINE__, rw_group, test_bgn);
  2495. /* Update info for sims */
  2496. reg_file_set_group(rw_group);
  2497. reg_file_set_stage(CAL_STAGE_VFIFO);
  2498. reg_file_set_sub_stage(CAL_SUBSTAGE_GUARANTEED_READ);
  2499. failed_substage = CAL_SUBSTAGE_GUARANTEED_READ;
  2500. /* USER Determine number of delay taps for each phase tap. */
  2501. dtaps_per_ptap = DIV_ROUND_UP(seq->iocfg->delay_per_opa_tap,
  2502. seq->iocfg->delay_per_dqs_en_dchain_tap)
  2503. - 1;
  2504. for (d = 0; d <= dtaps_per_ptap; d += 2) {
  2505. /*
  2506. * In RLDRAMX we may be messing the delay of pins in
  2507. * the same write rw_group but outside of the current read
  2508. * the rw_group, but that's ok because we haven't calibrated
  2509. * output side yet.
  2510. */
  2511. if (d > 0) {
  2512. scc_mgr_apply_group_all_out_delay_add_all_ranks(seq,
  2513. rw_group,
  2514. d);
  2515. }
  2516. for (p = 0; p <= seq->iocfg->dqdqs_out_phase_max; p++) {
  2517. /* 1) Guaranteed Write */
  2518. ret = rw_mgr_mem_calibrate_guaranteed_write(seq,
  2519. rw_group,
  2520. p);
  2521. if (ret)
  2522. break;
  2523. /* 2) DQS Enable Calibration */
  2524. ret = rw_mgr_mem_calibrate_dqs_enable_calibration(seq,
  2525. rw_group,
  2526. test_bgn);
  2527. if (ret) {
  2528. failed_substage = CAL_SUBSTAGE_DQS_EN_PHASE;
  2529. continue;
  2530. }
  2531. /* 3) Centering DQ/DQS */
  2532. /*
  2533. * If doing read after write calibration, do not update
  2534. * FOM now. Do it then.
  2535. */
  2536. ret = rw_mgr_mem_calibrate_dq_dqs_centering(seq,
  2537. rw_group,
  2538. test_bgn,
  2539. 1, 0);
  2540. if (ret) {
  2541. failed_substage = CAL_SUBSTAGE_VFIFO_CENTER;
  2542. continue;
  2543. }
  2544. /* All done. */
  2545. goto cal_done_ok;
  2546. }
  2547. }
  2548. /* Calibration Stage 1 failed. */
  2549. set_failing_group_stage(seq, rw_group, CAL_STAGE_VFIFO,
  2550. failed_substage);
  2551. return 0;
  2552. /* Calibration Stage 1 completed OK. */
  2553. cal_done_ok:
  2554. /*
  2555. * Reset the delay chains back to zero if they have moved > 1
  2556. * (check for > 1 because loop will increase d even when pass in
  2557. * first case).
  2558. */
  2559. if (d > 2)
  2560. scc_mgr_zero_group(seq, rw_group, 1);
  2561. return 1;
  2562. }
  2563. /**
  2564. * rw_mgr_mem_calibrate_vfifo_end() - DQ/DQS Centering.
  2565. * @rw_group: Read/Write Group
  2566. * @test_bgn: Rank at which the test begins
  2567. *
  2568. * Stage 3: DQ/DQS Centering.
  2569. *
  2570. * This function implements UniPHY calibration Stage 3, as explained in
  2571. * detail in Altera EMI_RM 2015.05.04 , "UniPHY Calibration Stages".
  2572. */
  2573. static int rw_mgr_mem_calibrate_vfifo_end(struct socfpga_sdrseq *seq,
  2574. const u32 rw_group,
  2575. const u32 test_bgn)
  2576. {
  2577. int ret;
  2578. debug("%s:%d %u %u", __func__, __LINE__, rw_group, test_bgn);
  2579. /* Update info for sims. */
  2580. reg_file_set_group(rw_group);
  2581. reg_file_set_stage(CAL_STAGE_VFIFO_AFTER_WRITES);
  2582. reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
  2583. ret = rw_mgr_mem_calibrate_dq_dqs_centering(seq, rw_group, test_bgn, 0,
  2584. 1);
  2585. if (ret)
  2586. set_failing_group_stage(seq, rw_group,
  2587. CAL_STAGE_VFIFO_AFTER_WRITES,
  2588. CAL_SUBSTAGE_VFIFO_CENTER);
  2589. return ret;
  2590. }
  2591. /**
  2592. * rw_mgr_mem_calibrate_lfifo() - Minimize latency
  2593. *
  2594. * Stage 4: Minimize latency.
  2595. *
  2596. * This function implements UniPHY calibration Stage 4, as explained in
  2597. * detail in Altera EMI_RM 2015.05.04 , "UniPHY Calibration Stages".
  2598. * Calibrate LFIFO to find smallest read latency.
  2599. */
  2600. static u32 rw_mgr_mem_calibrate_lfifo(struct socfpga_sdrseq *seq)
  2601. {
  2602. int found_one = 0;
  2603. debug("%s:%d\n", __func__, __LINE__);
  2604. /* Update info for sims. */
  2605. reg_file_set_stage(CAL_STAGE_LFIFO);
  2606. reg_file_set_sub_stage(CAL_SUBSTAGE_READ_LATENCY);
  2607. /* Load up the patterns used by read calibration for all ranks */
  2608. rw_mgr_mem_calibrate_read_load_patterns(seq, 0, 1);
  2609. do {
  2610. writel(seq->gbl.curr_read_lat, &phy_mgr_cfg->phy_rlat);
  2611. debug_cond(DLEVEL >= 2, "%s:%d lfifo: read_lat=%u",
  2612. __func__, __LINE__, seq->gbl.curr_read_lat);
  2613. if (!rw_mgr_mem_calibrate_read_test_all_ranks(seq, 0,
  2614. NUM_READ_TESTS,
  2615. PASS_ALL_BITS, 1))
  2616. break;
  2617. found_one = 1;
  2618. /*
  2619. * Reduce read latency and see if things are
  2620. * working correctly.
  2621. */
  2622. seq->gbl.curr_read_lat--;
  2623. } while (seq->gbl.curr_read_lat > 0);
  2624. /* Reset the fifos to get pointers to known state. */
  2625. writel(0, &phy_mgr_cmd->fifo_reset);
  2626. if (found_one) {
  2627. /* Add a fudge factor to the read latency that was determined */
  2628. seq->gbl.curr_read_lat += 2;
  2629. writel(seq->gbl.curr_read_lat, &phy_mgr_cfg->phy_rlat);
  2630. debug_cond(DLEVEL >= 2,
  2631. "%s:%d lfifo: success: using read_lat=%u\n",
  2632. __func__, __LINE__, seq->gbl.curr_read_lat);
  2633. } else {
  2634. set_failing_group_stage(seq, 0xff, CAL_STAGE_LFIFO,
  2635. CAL_SUBSTAGE_READ_LATENCY);
  2636. debug_cond(DLEVEL >= 2,
  2637. "%s:%d lfifo: failed at initial read_lat=%u\n",
  2638. __func__, __LINE__, seq->gbl.curr_read_lat);
  2639. }
  2640. return found_one;
  2641. }
  2642. /**
  2643. * search_window() - Search for the/part of the window with DM/DQS shift
  2644. * @search_dm: If 1, search for the DM shift, if 0, search for DQS
  2645. * shift
  2646. * @rank_bgn: Rank number
  2647. * @write_group: Write Group
  2648. * @bgn_curr: Current window begin
  2649. * @end_curr: Current window end
  2650. * @bgn_best: Current best window begin
  2651. * @end_best: Current best window end
  2652. * @win_best: Size of the best window
  2653. * @new_dqs: New DQS value (only applicable if search_dm = 0).
  2654. *
  2655. * Search for the/part of the window with DM/DQS shift.
  2656. */
  2657. static void search_window(struct socfpga_sdrseq *seq,
  2658. const int search_dm, const u32 rank_bgn,
  2659. const u32 write_group, int *bgn_curr, int *end_curr,
  2660. int *bgn_best, int *end_best, int *win_best,
  2661. int new_dqs)
  2662. {
  2663. u32 bit_chk;
  2664. const int max = seq->iocfg->io_out1_delay_max - new_dqs;
  2665. int d, di;
  2666. /* Search for the/part of the window with DM/DQS shift. */
  2667. for (di = max; di >= 0; di -= DELTA_D) {
  2668. if (search_dm) {
  2669. d = di;
  2670. scc_mgr_apply_group_dm_out1_delay(seq, d);
  2671. } else {
  2672. /* For DQS, we go from 0...max */
  2673. d = max - di;
  2674. /*
  2675. * Note: This only shifts DQS, so are we limiting
  2676. * ourselves to width of DQ unnecessarily.
  2677. */
  2678. scc_mgr_apply_group_dqs_io_and_oct_out1(seq,
  2679. write_group,
  2680. d + new_dqs);
  2681. }
  2682. writel(0, &sdr_scc_mgr->update);
  2683. if (rw_mgr_mem_calibrate_write_test(seq, rank_bgn, write_group,
  2684. 1, PASS_ALL_BITS, &bit_chk,
  2685. 0)) {
  2686. /* Set current end of the window. */
  2687. *end_curr = search_dm ? -d : d;
  2688. /*
  2689. * If a starting edge of our window has not been seen
  2690. * this is our current start of the DM window.
  2691. */
  2692. if (*bgn_curr == seq->iocfg->io_out1_delay_max + 1)
  2693. *bgn_curr = search_dm ? -d : d;
  2694. /*
  2695. * If current window is bigger than best seen.
  2696. * Set best seen to be current window.
  2697. */
  2698. if ((*end_curr - *bgn_curr + 1) > *win_best) {
  2699. *win_best = *end_curr - *bgn_curr + 1;
  2700. *bgn_best = *bgn_curr;
  2701. *end_best = *end_curr;
  2702. }
  2703. } else {
  2704. /* We just saw a failing test. Reset temp edge. */
  2705. *bgn_curr = seq->iocfg->io_out1_delay_max + 1;
  2706. *end_curr = seq->iocfg->io_out1_delay_max + 1;
  2707. /* Early exit is only applicable to DQS. */
  2708. if (search_dm)
  2709. continue;
  2710. /*
  2711. * Early exit optimization: if the remaining delay
  2712. * chain space is less than already seen largest
  2713. * window we can exit.
  2714. */
  2715. if (*win_best - 1 > seq->iocfg->io_out1_delay_max
  2716. - new_dqs - d)
  2717. break;
  2718. }
  2719. }
  2720. }
  2721. /*
  2722. * rw_mgr_mem_calibrate_writes_center() - Center all windows
  2723. * @rank_bgn: Rank number
  2724. * @write_group: Write group
  2725. * @test_bgn: Rank at which the test begins
  2726. *
  2727. * Center all windows. Do per-bit-deskew to possibly increase size of
  2728. * certain windows.
  2729. */
  2730. static int
  2731. rw_mgr_mem_calibrate_writes_center(struct socfpga_sdrseq *seq,
  2732. const u32 rank_bgn, const u32 write_group,
  2733. const u32 test_bgn)
  2734. {
  2735. int i;
  2736. u32 sticky_bit_chk;
  2737. u32 min_index;
  2738. int left_edge[seq->rwcfg->mem_dq_per_write_dqs];
  2739. int right_edge[seq->rwcfg->mem_dq_per_write_dqs];
  2740. int mid;
  2741. int mid_min, orig_mid_min;
  2742. int new_dqs, start_dqs;
  2743. int dq_margin, dqs_margin, dm_margin;
  2744. int bgn_curr = seq->iocfg->io_out1_delay_max + 1;
  2745. int end_curr = seq->iocfg->io_out1_delay_max + 1;
  2746. int bgn_best = seq->iocfg->io_out1_delay_max + 1;
  2747. int end_best = seq->iocfg->io_out1_delay_max + 1;
  2748. int win_best = 0;
  2749. int ret;
  2750. debug("%s:%d %u %u", __func__, __LINE__, write_group, test_bgn);
  2751. dm_margin = 0;
  2752. start_dqs = readl((SDR_PHYGRP_SCCGRP_ADDRESS |
  2753. SCC_MGR_IO_OUT1_DELAY_OFFSET) +
  2754. (seq->rwcfg->mem_dq_per_write_dqs << 2));
  2755. /* Per-bit deskew. */
  2756. /*
  2757. * Set the left and right edge of each bit to an illegal value.
  2758. * Use (seq->iocfg->io_out1_delay_max + 1) as an illegal value.
  2759. */
  2760. sticky_bit_chk = 0;
  2761. for (i = 0; i < seq->rwcfg->mem_dq_per_write_dqs; i++) {
  2762. left_edge[i] = seq->iocfg->io_out1_delay_max + 1;
  2763. right_edge[i] = seq->iocfg->io_out1_delay_max + 1;
  2764. }
  2765. /* Search for the left edge of the window for each bit. */
  2766. search_left_edge(seq, 1, rank_bgn, write_group, 0, test_bgn,
  2767. &sticky_bit_chk,
  2768. left_edge, right_edge, 0);
  2769. /* Search for the right edge of the window for each bit. */
  2770. ret = search_right_edge(seq, 1, rank_bgn, write_group, 0,
  2771. start_dqs, 0,
  2772. &sticky_bit_chk,
  2773. left_edge, right_edge, 0);
  2774. if (ret) {
  2775. set_failing_group_stage(seq, test_bgn + ret - 1,
  2776. CAL_STAGE_WRITES,
  2777. CAL_SUBSTAGE_WRITES_CENTER);
  2778. return -EINVAL;
  2779. }
  2780. min_index = get_window_mid_index(seq, 1, left_edge, right_edge,
  2781. &mid_min);
  2782. /* Determine the amount we can change DQS (which is -mid_min). */
  2783. orig_mid_min = mid_min;
  2784. new_dqs = start_dqs;
  2785. mid_min = 0;
  2786. debug_cond(DLEVEL >= 1,
  2787. "%s:%d write_center: start_dqs=%d new_dqs=%d mid_min=%d\n",
  2788. __func__, __LINE__, start_dqs, new_dqs, mid_min);
  2789. /* Add delay to bring centre of all DQ windows to the same "level". */
  2790. center_dq_windows(seq, 1, left_edge, right_edge, mid_min, orig_mid_min,
  2791. min_index, 0, &dq_margin, &dqs_margin);
  2792. /* Move DQS */
  2793. scc_mgr_apply_group_dqs_io_and_oct_out1(seq, write_group, new_dqs);
  2794. writel(0, &sdr_scc_mgr->update);
  2795. /* Centre DM */
  2796. debug_cond(DLEVEL >= 2, "%s:%d write_center: DM\n", __func__, __LINE__);
  2797. /*
  2798. * Set the left and right edge of each bit to an illegal value.
  2799. * Use (seq->iocfg->io_out1_delay_max + 1) as an illegal value.
  2800. */
  2801. left_edge[0] = seq->iocfg->io_out1_delay_max + 1;
  2802. right_edge[0] = seq->iocfg->io_out1_delay_max + 1;
  2803. /* Search for the/part of the window with DM shift. */
  2804. search_window(seq, 1, rank_bgn, write_group, &bgn_curr, &end_curr,
  2805. &bgn_best, &end_best, &win_best, 0);
  2806. /* Reset DM delay chains to 0. */
  2807. scc_mgr_apply_group_dm_out1_delay(seq, 0);
  2808. /*
  2809. * Check to see if the current window nudges up aganist 0 delay.
  2810. * If so we need to continue the search by shifting DQS otherwise DQS
  2811. * search begins as a new search.
  2812. */
  2813. if (end_curr != 0) {
  2814. bgn_curr = seq->iocfg->io_out1_delay_max + 1;
  2815. end_curr = seq->iocfg->io_out1_delay_max + 1;
  2816. }
  2817. /* Search for the/part of the window with DQS shifts. */
  2818. search_window(seq, 0, rank_bgn, write_group, &bgn_curr, &end_curr,
  2819. &bgn_best, &end_best, &win_best, new_dqs);
  2820. /* Assign left and right edge for cal and reporting. */
  2821. left_edge[0] = -1 * bgn_best;
  2822. right_edge[0] = end_best;
  2823. debug_cond(DLEVEL >= 2, "%s:%d dm_calib: left=%d right=%d\n",
  2824. __func__, __LINE__, left_edge[0], right_edge[0]);
  2825. /* Move DQS (back to orig). */
  2826. scc_mgr_apply_group_dqs_io_and_oct_out1(seq, write_group, new_dqs);
  2827. /* Move DM */
  2828. /* Find middle of window for the DM bit. */
  2829. mid = (left_edge[0] - right_edge[0]) / 2;
  2830. /* Only move right, since we are not moving DQS/DQ. */
  2831. if (mid < 0)
  2832. mid = 0;
  2833. /* dm_marign should fail if we never find a window. */
  2834. if (win_best == 0)
  2835. dm_margin = -1;
  2836. else
  2837. dm_margin = left_edge[0] - mid;
  2838. scc_mgr_apply_group_dm_out1_delay(seq, mid);
  2839. writel(0, &sdr_scc_mgr->update);
  2840. debug_cond(DLEVEL >= 2,
  2841. "%s:%d dm_calib: left=%d right=%d mid=%d dm_margin=%d\n",
  2842. __func__, __LINE__, left_edge[0], right_edge[0],
  2843. mid, dm_margin);
  2844. /* Export values. */
  2845. seq->gbl.fom_out += dq_margin + dqs_margin;
  2846. debug_cond(DLEVEL >= 2,
  2847. "%s:%d write_center: dq_margin=%d dqs_margin=%d dm_margin=%d\n",
  2848. __func__, __LINE__, dq_margin, dqs_margin, dm_margin);
  2849. /*
  2850. * Do not remove this line as it makes sure all of our
  2851. * decisions have been applied.
  2852. */
  2853. writel(0, &sdr_scc_mgr->update);
  2854. if ((dq_margin < 0) || (dqs_margin < 0) || (dm_margin < 0))
  2855. return -EINVAL;
  2856. return 0;
  2857. }
  2858. /**
  2859. * rw_mgr_mem_calibrate_writes() - Write Calibration Part One
  2860. * @rank_bgn: Rank number
  2861. * @group: Read/Write Group
  2862. * @test_bgn: Rank at which the test begins
  2863. *
  2864. * Stage 2: Write Calibration Part One.
  2865. *
  2866. * This function implements UniPHY calibration Stage 2, as explained in
  2867. * detail in Altera EMI_RM 2015.05.04 , "UniPHY Calibration Stages".
  2868. */
  2869. static int rw_mgr_mem_calibrate_writes(struct socfpga_sdrseq *seq,
  2870. const u32 rank_bgn, const u32 group,
  2871. const u32 test_bgn)
  2872. {
  2873. int ret;
  2874. /* Update info for sims */
  2875. debug("%s:%d %u %u\n", __func__, __LINE__, group, test_bgn);
  2876. reg_file_set_group(group);
  2877. reg_file_set_stage(CAL_STAGE_WRITES);
  2878. reg_file_set_sub_stage(CAL_SUBSTAGE_WRITES_CENTER);
  2879. ret = rw_mgr_mem_calibrate_writes_center(seq, rank_bgn, group,
  2880. test_bgn);
  2881. if (ret)
  2882. set_failing_group_stage(seq, group, CAL_STAGE_WRITES,
  2883. CAL_SUBSTAGE_WRITES_CENTER);
  2884. return ret;
  2885. }
  2886. /**
  2887. * mem_precharge_and_activate() - Precharge all banks and activate
  2888. *
  2889. * Precharge all banks and activate row 0 in bank "000..." and bank "111...".
  2890. */
  2891. static void mem_precharge_and_activate(struct socfpga_sdrseq *seq)
  2892. {
  2893. int r;
  2894. for (r = 0; r < seq->rwcfg->mem_number_of_ranks; r++) {
  2895. /* Set rank. */
  2896. set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_OFF);
  2897. /* Precharge all banks. */
  2898. writel(seq->rwcfg->precharge_all, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  2899. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  2900. writel(0x0F, &sdr_rw_load_mgr_regs->load_cntr0);
  2901. writel(seq->rwcfg->activate_0_and_1_wait1,
  2902. &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  2903. writel(0x0F, &sdr_rw_load_mgr_regs->load_cntr1);
  2904. writel(seq->rwcfg->activate_0_and_1_wait2,
  2905. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  2906. /* Activate rows. */
  2907. writel(seq->rwcfg->activate_0_and_1,
  2908. SDR_PHYGRP_RWMGRGRP_ADDRESS |
  2909. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  2910. }
  2911. }
  2912. /**
  2913. * mem_init_latency() - Configure memory RLAT and WLAT settings
  2914. *
  2915. * Configure memory RLAT and WLAT parameters.
  2916. */
  2917. static void mem_init_latency(struct socfpga_sdrseq *seq)
  2918. {
  2919. /*
  2920. * For AV/CV, LFIFO is hardened and always runs at full rate
  2921. * so max latency in AFI clocks, used here, is correspondingly
  2922. * smaller.
  2923. */
  2924. const u32 max_latency = (1 << seq->misccfg->max_latency_count_width)
  2925. - 1;
  2926. u32 rlat, wlat;
  2927. debug("%s:%d\n", __func__, __LINE__);
  2928. /*
  2929. * Read in write latency.
  2930. * WL for Hard PHY does not include additive latency.
  2931. */
  2932. wlat = readl(&data_mgr->t_wl_add);
  2933. wlat += readl(&data_mgr->mem_t_add);
  2934. seq->gbl.rw_wl_nop_cycles = wlat - 1;
  2935. /* Read in readl latency. */
  2936. rlat = readl(&data_mgr->t_rl_add);
  2937. /* Set a pretty high read latency initially. */
  2938. seq->gbl.curr_read_lat = rlat + 16;
  2939. if (seq->gbl.curr_read_lat > max_latency)
  2940. seq->gbl.curr_read_lat = max_latency;
  2941. writel(seq->gbl.curr_read_lat, &phy_mgr_cfg->phy_rlat);
  2942. /* Advertise write latency. */
  2943. writel(wlat, &phy_mgr_cfg->afi_wlat);
  2944. }
  2945. /**
  2946. * @mem_skip_calibrate() - Set VFIFO and LFIFO to instant-on settings
  2947. *
  2948. * Set VFIFO and LFIFO to instant-on settings in skip calibration mode.
  2949. */
  2950. static void mem_skip_calibrate(struct socfpga_sdrseq *seq)
  2951. {
  2952. u32 vfifo_offset;
  2953. u32 i, j, r;
  2954. debug("%s:%d\n", __func__, __LINE__);
  2955. /* Need to update every shadow register set used by the interface */
  2956. for (r = 0; r < seq->rwcfg->mem_number_of_ranks;
  2957. r += NUM_RANKS_PER_SHADOW_REG) {
  2958. /*
  2959. * Set output phase alignment settings appropriate for
  2960. * skip calibration.
  2961. */
  2962. for (i = 0; i < seq->rwcfg->mem_if_read_dqs_width; i++) {
  2963. scc_mgr_set_dqs_en_phase(i, 0);
  2964. if (seq->iocfg->dll_chain_length == 6)
  2965. scc_mgr_set_dqdqs_output_phase(i, 6);
  2966. else
  2967. scc_mgr_set_dqdqs_output_phase(i, 7);
  2968. /*
  2969. * Case:33398
  2970. *
  2971. * Write data arrives to the I/O two cycles before write
  2972. * latency is reached (720 deg).
  2973. * -> due to bit-slip in a/c bus
  2974. * -> to allow board skew where dqs is longer than ck
  2975. * -> how often can this happen!?
  2976. * -> can claim back some ptaps for high freq
  2977. * support if we can relax this, but i digress...
  2978. *
  2979. * The write_clk leads mem_ck by 90 deg
  2980. * The minimum ptap of the OPA is 180 deg
  2981. * Each ptap has (360 / IO_DLL_CHAIN_LENGH) deg of delay
  2982. * The write_clk is always delayed by 2 ptaps
  2983. *
  2984. * Hence, to make DQS aligned to CK, we need to delay
  2985. * DQS by:
  2986. * (720 - 90 - 180 - 2) *
  2987. * (360 / seq->iocfg->dll_chain_length)
  2988. *
  2989. * Dividing the above by
  2990. (360 / seq->iocfg->dll_chain_length)
  2991. * gives us the number of ptaps, which simplies to:
  2992. *
  2993. * (1.25 * seq->iocfg->dll_chain_length - 2)
  2994. */
  2995. scc_mgr_set_dqdqs_output_phase(i,
  2996. ((125 * seq->iocfg->dll_chain_length)
  2997. / 100) - 2);
  2998. }
  2999. writel(0xff, &sdr_scc_mgr->dqs_ena);
  3000. writel(0xff, &sdr_scc_mgr->dqs_io_ena);
  3001. for (i = 0; i < seq->rwcfg->mem_if_write_dqs_width; i++) {
  3002. writel(i, SDR_PHYGRP_SCCGRP_ADDRESS |
  3003. SCC_MGR_GROUP_COUNTER_OFFSET);
  3004. }
  3005. writel(0xff, &sdr_scc_mgr->dq_ena);
  3006. writel(0xff, &sdr_scc_mgr->dm_ena);
  3007. writel(0, &sdr_scc_mgr->update);
  3008. }
  3009. /* Compensate for simulation model behaviour */
  3010. for (i = 0; i < seq->rwcfg->mem_if_read_dqs_width; i++) {
  3011. scc_mgr_set_dqs_bus_in_delay(i, 10);
  3012. scc_mgr_load_dqs(i);
  3013. }
  3014. writel(0, &sdr_scc_mgr->update);
  3015. /*
  3016. * ArriaV has hard FIFOs that can only be initialized by incrementing
  3017. * in sequencer.
  3018. */
  3019. vfifo_offset = seq->misccfg->calib_vfifo_offset;
  3020. for (j = 0; j < vfifo_offset; j++)
  3021. writel(0xff, &phy_mgr_cmd->inc_vfifo_hard_phy);
  3022. writel(0, &phy_mgr_cmd->fifo_reset);
  3023. /*
  3024. * For Arria V and Cyclone V with hard LFIFO, we get the skip-cal
  3025. * setting from generation-time constant.
  3026. */
  3027. seq->gbl.curr_read_lat = seq->misccfg->calib_lfifo_offset;
  3028. writel(seq->gbl.curr_read_lat, &phy_mgr_cfg->phy_rlat);
  3029. }
  3030. /**
  3031. * mem_calibrate() - Memory calibration entry point.
  3032. *
  3033. * Perform memory calibration.
  3034. */
  3035. static u32 mem_calibrate(struct socfpga_sdrseq *seq)
  3036. {
  3037. u32 i;
  3038. u32 rank_bgn, sr;
  3039. u32 write_group, write_test_bgn;
  3040. u32 read_group, read_test_bgn;
  3041. u32 run_groups, current_run;
  3042. u32 failing_groups = 0;
  3043. u32 group_failed = 0;
  3044. const u32 rwdqs_ratio = seq->rwcfg->mem_if_read_dqs_width /
  3045. seq->rwcfg->mem_if_write_dqs_width;
  3046. debug("%s:%d\n", __func__, __LINE__);
  3047. /* Initialize the data settings */
  3048. seq->gbl.error_substage = CAL_SUBSTAGE_NIL;
  3049. seq->gbl.error_stage = CAL_STAGE_NIL;
  3050. seq->gbl.error_group = 0xff;
  3051. seq->gbl.fom_in = 0;
  3052. seq->gbl.fom_out = 0;
  3053. /* Initialize WLAT and RLAT. */
  3054. mem_init_latency(seq);
  3055. /* Initialize bit slips. */
  3056. mem_precharge_and_activate(seq);
  3057. for (i = 0; i < seq->rwcfg->mem_if_read_dqs_width; i++) {
  3058. writel(i, SDR_PHYGRP_SCCGRP_ADDRESS |
  3059. SCC_MGR_GROUP_COUNTER_OFFSET);
  3060. /* Only needed once to set all groups, pins, DQ, DQS, DM. */
  3061. if (i == 0)
  3062. scc_mgr_set_hhp_extras();
  3063. scc_set_bypass_mode(i);
  3064. }
  3065. /* Calibration is skipped. */
  3066. if ((seq->dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL) {
  3067. /*
  3068. * Set VFIFO and LFIFO to instant-on settings in skip
  3069. * calibration mode.
  3070. */
  3071. mem_skip_calibrate(seq);
  3072. /*
  3073. * Do not remove this line as it makes sure all of our
  3074. * decisions have been applied.
  3075. */
  3076. writel(0, &sdr_scc_mgr->update);
  3077. return 1;
  3078. }
  3079. /* Calibration is not skipped. */
  3080. for (i = 0; i < NUM_CALIB_REPEAT; i++) {
  3081. /*
  3082. * Zero all delay chain/phase settings for all
  3083. * groups and all shadow register sets.
  3084. */
  3085. scc_mgr_zero_all(seq);
  3086. run_groups = ~0;
  3087. for (write_group = 0, write_test_bgn = 0; write_group
  3088. < seq->rwcfg->mem_if_write_dqs_width; write_group++,
  3089. write_test_bgn += seq->rwcfg->mem_dq_per_write_dqs) {
  3090. /* Initialize the group failure */
  3091. group_failed = 0;
  3092. current_run = run_groups & ((1 <<
  3093. RW_MGR_NUM_DQS_PER_WRITE_GROUP) - 1);
  3094. run_groups = run_groups >>
  3095. RW_MGR_NUM_DQS_PER_WRITE_GROUP;
  3096. if (current_run == 0)
  3097. continue;
  3098. writel(write_group, SDR_PHYGRP_SCCGRP_ADDRESS |
  3099. SCC_MGR_GROUP_COUNTER_OFFSET);
  3100. scc_mgr_zero_group(seq, write_group, 0);
  3101. for (read_group = write_group * rwdqs_ratio,
  3102. read_test_bgn = 0;
  3103. read_group < (write_group + 1) * rwdqs_ratio;
  3104. read_group++,
  3105. read_test_bgn += seq->rwcfg->mem_dq_per_read_dqs) {
  3106. if (STATIC_CALIB_STEPS & CALIB_SKIP_VFIFO)
  3107. continue;
  3108. /* Calibrate the VFIFO */
  3109. if (rw_mgr_mem_calibrate_vfifo(seq, read_group,
  3110. read_test_bgn))
  3111. continue;
  3112. if (!(seq->gbl.phy_debug_mode_flags &
  3113. PHY_DEBUG_SWEEP_ALL_GROUPS))
  3114. return 0;
  3115. /* The group failed, we're done. */
  3116. goto grp_failed;
  3117. }
  3118. /* Calibrate the output side */
  3119. for (rank_bgn = 0, sr = 0;
  3120. rank_bgn < seq->rwcfg->mem_number_of_ranks;
  3121. rank_bgn += NUM_RANKS_PER_SHADOW_REG, sr++) {
  3122. if (STATIC_CALIB_STEPS & CALIB_SKIP_WRITES)
  3123. continue;
  3124. /* Not needed in quick mode! */
  3125. if (STATIC_CALIB_STEPS &
  3126. CALIB_SKIP_DELAY_SWEEPS)
  3127. continue;
  3128. /* Calibrate WRITEs */
  3129. if (!rw_mgr_mem_calibrate_writes(seq, rank_bgn,
  3130. write_group,
  3131. write_test_bgn))
  3132. continue;
  3133. group_failed = 1;
  3134. if (!(seq->gbl.phy_debug_mode_flags &
  3135. PHY_DEBUG_SWEEP_ALL_GROUPS))
  3136. return 0;
  3137. }
  3138. /* Some group failed, we're done. */
  3139. if (group_failed)
  3140. goto grp_failed;
  3141. for (read_group = write_group * rwdqs_ratio,
  3142. read_test_bgn = 0;
  3143. read_group < (write_group + 1) * rwdqs_ratio;
  3144. read_group++,
  3145. read_test_bgn += seq->rwcfg->mem_dq_per_read_dqs) {
  3146. if (STATIC_CALIB_STEPS & CALIB_SKIP_WRITES)
  3147. continue;
  3148. if (!rw_mgr_mem_calibrate_vfifo_end(seq,
  3149. read_group,
  3150. read_test_bgn))
  3151. continue;
  3152. if (!(seq->gbl.phy_debug_mode_flags &
  3153. PHY_DEBUG_SWEEP_ALL_GROUPS))
  3154. return 0;
  3155. /* The group failed, we're done. */
  3156. goto grp_failed;
  3157. }
  3158. /* No group failed, continue as usual. */
  3159. continue;
  3160. grp_failed: /* A group failed, increment the counter. */
  3161. failing_groups++;
  3162. }
  3163. /*
  3164. * USER If there are any failing groups then report
  3165. * the failure.
  3166. */
  3167. if (failing_groups != 0)
  3168. return 0;
  3169. if (STATIC_CALIB_STEPS & CALIB_SKIP_LFIFO)
  3170. continue;
  3171. /* Calibrate the LFIFO */
  3172. if (!rw_mgr_mem_calibrate_lfifo(seq))
  3173. return 0;
  3174. }
  3175. /*
  3176. * Do not remove this line as it makes sure all of our decisions
  3177. * have been applied.
  3178. */
  3179. writel(0, &sdr_scc_mgr->update);
  3180. return 1;
  3181. }
  3182. /**
  3183. * run_mem_calibrate() - Perform memory calibration
  3184. *
  3185. * This function triggers the entire memory calibration procedure.
  3186. */
  3187. static int run_mem_calibrate(struct socfpga_sdrseq *seq)
  3188. {
  3189. int pass;
  3190. u32 ctrl_cfg;
  3191. debug("%s:%d\n", __func__, __LINE__);
  3192. /* Reset pass/fail status shown on afi_cal_success/fail */
  3193. writel(PHY_MGR_CAL_RESET, &phy_mgr_cfg->cal_status);
  3194. /* Stop tracking manager. */
  3195. ctrl_cfg = readl(&sdr_ctrl->ctrl_cfg);
  3196. writel(ctrl_cfg & ~SDR_CTRLGRP_CTRLCFG_DQSTRKEN_MASK,
  3197. &sdr_ctrl->ctrl_cfg);
  3198. phy_mgr_initialize(seq);
  3199. rw_mgr_mem_initialize(seq);
  3200. /* Perform the actual memory calibration. */
  3201. pass = mem_calibrate(seq);
  3202. mem_precharge_and_activate(seq);
  3203. writel(0, &phy_mgr_cmd->fifo_reset);
  3204. /* Handoff. */
  3205. rw_mgr_mem_handoff(seq);
  3206. /*
  3207. * In Hard PHY this is a 2-bit control:
  3208. * 0: AFI Mux Select
  3209. * 1: DDIO Mux Select
  3210. */
  3211. writel(0x2, &phy_mgr_cfg->mux_sel);
  3212. /* Start tracking manager. */
  3213. writel(ctrl_cfg, &sdr_ctrl->ctrl_cfg);
  3214. return pass;
  3215. }
  3216. /**
  3217. * debug_mem_calibrate() - Report result of memory calibration
  3218. * @pass: Value indicating whether calibration passed or failed
  3219. *
  3220. * This function reports the results of the memory calibration
  3221. * and writes debug information into the register file.
  3222. */
  3223. static void debug_mem_calibrate(struct socfpga_sdrseq *seq, int pass)
  3224. {
  3225. u32 debug_info;
  3226. if (pass) {
  3227. debug("%s: CALIBRATION PASSED\n", __FILE__);
  3228. seq->gbl.fom_in /= 2;
  3229. seq->gbl.fom_out /= 2;
  3230. if (seq->gbl.fom_in > 0xff)
  3231. seq->gbl.fom_in = 0xff;
  3232. if (seq->gbl.fom_out > 0xff)
  3233. seq->gbl.fom_out = 0xff;
  3234. /* Update the FOM in the register file */
  3235. debug_info = seq->gbl.fom_in;
  3236. debug_info |= seq->gbl.fom_out << 8;
  3237. writel(debug_info, &sdr_reg_file->fom);
  3238. writel(debug_info, &phy_mgr_cfg->cal_debug_info);
  3239. writel(PHY_MGR_CAL_SUCCESS, &phy_mgr_cfg->cal_status);
  3240. } else {
  3241. debug("%s: CALIBRATION FAILED\n", __FILE__);
  3242. debug_info = seq->gbl.error_stage;
  3243. debug_info |= seq->gbl.error_substage << 8;
  3244. debug_info |= seq->gbl.error_group << 16;
  3245. writel(debug_info, &sdr_reg_file->failing_stage);
  3246. writel(debug_info, &phy_mgr_cfg->cal_debug_info);
  3247. writel(PHY_MGR_CAL_FAIL, &phy_mgr_cfg->cal_status);
  3248. /* Update the failing group/stage in the register file */
  3249. debug_info = seq->gbl.error_stage;
  3250. debug_info |= seq->gbl.error_substage << 8;
  3251. debug_info |= seq->gbl.error_group << 16;
  3252. writel(debug_info, &sdr_reg_file->failing_stage);
  3253. }
  3254. debug("%s: Calibration complete\n", __FILE__);
  3255. }
  3256. /**
  3257. * hc_initialize_rom_data() - Initialize ROM data
  3258. *
  3259. * Initialize ROM data.
  3260. */
  3261. static void hc_initialize_rom_data(void)
  3262. {
  3263. unsigned int nelem = 0;
  3264. const u32 *rom_init;
  3265. u32 i, addr;
  3266. socfpga_get_seq_inst_init(&rom_init, &nelem);
  3267. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_INST_ROM_WRITE_OFFSET;
  3268. for (i = 0; i < nelem; i++)
  3269. writel(rom_init[i], addr + (i << 2));
  3270. socfpga_get_seq_ac_init(&rom_init, &nelem);
  3271. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_AC_ROM_WRITE_OFFSET;
  3272. for (i = 0; i < nelem; i++)
  3273. writel(rom_init[i], addr + (i << 2));
  3274. }
  3275. /**
  3276. * initialize_reg_file() - Initialize SDR register file
  3277. *
  3278. * Initialize SDR register file.
  3279. */
  3280. static void initialize_reg_file(struct socfpga_sdrseq *seq)
  3281. {
  3282. /* Initialize the register file with the correct data */
  3283. writel(seq->misccfg->reg_file_init_seq_signature,
  3284. &sdr_reg_file->signature);
  3285. writel(0, &sdr_reg_file->debug_data_addr);
  3286. writel(0, &sdr_reg_file->cur_stage);
  3287. writel(0, &sdr_reg_file->fom);
  3288. writel(0, &sdr_reg_file->failing_stage);
  3289. writel(0, &sdr_reg_file->debug1);
  3290. writel(0, &sdr_reg_file->debug2);
  3291. }
  3292. /**
  3293. * initialize_hps_phy() - Initialize HPS PHY
  3294. *
  3295. * Initialize HPS PHY.
  3296. */
  3297. static void initialize_hps_phy(void)
  3298. {
  3299. u32 reg;
  3300. /*
  3301. * Tracking also gets configured here because it's in the
  3302. * same register.
  3303. */
  3304. u32 trk_sample_count = 7500;
  3305. u32 trk_long_idle_sample_count = (10 << 16) | 100;
  3306. /*
  3307. * Format is number of outer loops in the 16 MSB, sample
  3308. * count in 16 LSB.
  3309. */
  3310. reg = 0;
  3311. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ACDELAYEN_SET(2);
  3312. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQDELAYEN_SET(1);
  3313. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSDELAYEN_SET(1);
  3314. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSLOGICDELAYEN_SET(1);
  3315. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_RESETDELAYEN_SET(0);
  3316. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_LPDDRDIS_SET(1);
  3317. /*
  3318. * This field selects the intrinsic latency to RDATA_EN/FULL path.
  3319. * 00-bypass, 01- add 5 cycles, 10- add 10 cycles, 11- add 15 cycles.
  3320. */
  3321. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ADDLATSEL_SET(0);
  3322. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_SET(
  3323. trk_sample_count);
  3324. writel(reg, &sdr_ctrl->phy_ctrl0);
  3325. reg = 0;
  3326. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_SAMPLECOUNT_31_20_SET(
  3327. trk_sample_count >>
  3328. SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_WIDTH);
  3329. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_SET(
  3330. trk_long_idle_sample_count);
  3331. writel(reg, &sdr_ctrl->phy_ctrl1);
  3332. reg = 0;
  3333. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_2_LONGIDLESAMPLECOUNT_31_20_SET(
  3334. trk_long_idle_sample_count >>
  3335. SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_WIDTH);
  3336. writel(reg, &sdr_ctrl->phy_ctrl2);
  3337. }
  3338. /**
  3339. * initialize_tracking() - Initialize tracking
  3340. *
  3341. * Initialize the register file with usable initial data.
  3342. */
  3343. static void initialize_tracking(struct socfpga_sdrseq *seq)
  3344. {
  3345. /*
  3346. * Initialize the register file with the correct data.
  3347. * Compute usable version of value in case we skip full
  3348. * computation later.
  3349. */
  3350. writel(DIV_ROUND_UP(seq->iocfg->delay_per_opa_tap,
  3351. seq->iocfg->delay_per_dchain_tap) - 1,
  3352. &sdr_reg_file->dtaps_per_ptap);
  3353. /* trk_sample_count */
  3354. writel(7500, &sdr_reg_file->trk_sample_count);
  3355. /* longidle outer loop [15:0] */
  3356. writel((10 << 16) | (100 << 0), &sdr_reg_file->trk_longidle);
  3357. /*
  3358. * longidle sample count [31:24]
  3359. * trfc, worst case of 933Mhz 4Gb [23:16]
  3360. * trcd, worst case [15:8]
  3361. * vfifo wait [7:0]
  3362. */
  3363. writel((243 << 24) | (14 << 16) | (10 << 8) | (4 << 0),
  3364. &sdr_reg_file->delays);
  3365. /* mux delay */
  3366. if (dram_is_ddr(2)) {
  3367. writel(0, &sdr_reg_file->trk_rw_mgr_addr);
  3368. } else if (dram_is_ddr(3)) {
  3369. writel((seq->rwcfg->idle << 24) |
  3370. (seq->rwcfg->activate_1 << 16) |
  3371. (seq->rwcfg->sgle_read << 8) |
  3372. (seq->rwcfg->precharge_all << 0),
  3373. &sdr_reg_file->trk_rw_mgr_addr);
  3374. }
  3375. writel(seq->rwcfg->mem_if_read_dqs_width,
  3376. &sdr_reg_file->trk_read_dqs_width);
  3377. /* trefi [7:0] */
  3378. if (dram_is_ddr(2)) {
  3379. writel(1000 << 0, &sdr_reg_file->trk_rfsh);
  3380. } else if (dram_is_ddr(3)) {
  3381. writel((seq->rwcfg->refresh_all << 24) | (1000 << 0),
  3382. &sdr_reg_file->trk_rfsh);
  3383. }
  3384. }
  3385. int sdram_calibration_full(struct socfpga_sdr *sdr)
  3386. {
  3387. u32 pass;
  3388. struct socfpga_sdrseq seq;
  3389. /*
  3390. * For size reasons, this file uses hard coded addresses.
  3391. * Check if we are called with the correct address.
  3392. */
  3393. if (sdr != (struct socfpga_sdr *)SOCFPGA_SDR_ADDRESS)
  3394. return -ENODEV;
  3395. memset(&seq, 0, sizeof(seq));
  3396. seq.rwcfg = socfpga_get_sdram_rwmgr_config();
  3397. seq.iocfg = socfpga_get_sdram_io_config();
  3398. seq.misccfg = socfpga_get_sdram_misc_config();
  3399. /* Set the calibration enabled by default */
  3400. seq.gbl.phy_debug_mode_flags |= PHY_DEBUG_ENABLE_CAL_RPT;
  3401. /*
  3402. * Only sweep all groups (regardless of fail state) by default
  3403. * Set enabled read test by default.
  3404. */
  3405. #if DISABLE_GUARANTEED_READ
  3406. seq.gbl.phy_debug_mode_flags |= PHY_DEBUG_DISABLE_GUARANTEED_READ;
  3407. #endif
  3408. /* Initialize the register file */
  3409. initialize_reg_file(&seq);
  3410. /* Initialize any PHY CSR */
  3411. initialize_hps_phy();
  3412. scc_mgr_initialize();
  3413. initialize_tracking(&seq);
  3414. debug("%s: Preparing to start memory calibration\n", __FILE__);
  3415. debug("%s:%d\n", __func__, __LINE__);
  3416. debug_cond(DLEVEL >= 1,
  3417. "DDR3 FULL_RATE ranks=%u cs/dimm=%u dq/dqs=%u,%u vg/dqs=%u,%u ",
  3418. seq.rwcfg->mem_number_of_ranks,
  3419. seq.rwcfg->mem_number_of_cs_per_dimm,
  3420. seq.rwcfg->mem_dq_per_read_dqs,
  3421. seq.rwcfg->mem_dq_per_write_dqs,
  3422. seq.rwcfg->mem_virtual_groups_per_read_dqs,
  3423. seq.rwcfg->mem_virtual_groups_per_write_dqs);
  3424. debug_cond(DLEVEL >= 1,
  3425. "dqs=%u,%u dq=%u dm=%u ptap_delay=%u dtap_delay=%u ",
  3426. seq.rwcfg->mem_if_read_dqs_width,
  3427. seq.rwcfg->mem_if_write_dqs_width,
  3428. seq.rwcfg->mem_data_width, seq.rwcfg->mem_data_mask_width,
  3429. seq.iocfg->delay_per_opa_tap,
  3430. seq.iocfg->delay_per_dchain_tap);
  3431. debug_cond(DLEVEL >= 1, "dtap_dqsen_delay=%u, dll=%u",
  3432. seq.iocfg->delay_per_dqs_en_dchain_tap,
  3433. seq.iocfg->dll_chain_length);
  3434. debug_cond(DLEVEL >= 1,
  3435. "max values: en_p=%u dqdqs_p=%u en_d=%u dqs_in_d=%u ",
  3436. seq.iocfg->dqs_en_phase_max, seq.iocfg->dqdqs_out_phase_max,
  3437. seq.iocfg->dqs_en_delay_max, seq.iocfg->dqs_in_delay_max);
  3438. debug_cond(DLEVEL >= 1, "io_in_d=%u io_out1_d=%u io_out2_d=%u ",
  3439. seq.iocfg->io_in_delay_max, seq.iocfg->io_out1_delay_max,
  3440. seq.iocfg->io_out2_delay_max);
  3441. debug_cond(DLEVEL >= 1, "dqs_in_reserve=%u dqs_out_reserve=%u\n",
  3442. seq.iocfg->dqs_in_reserve, seq.iocfg->dqs_out_reserve);
  3443. hc_initialize_rom_data();
  3444. /* update info for sims */
  3445. reg_file_set_stage(CAL_STAGE_NIL);
  3446. reg_file_set_group(0);
  3447. /*
  3448. * Load global needed for those actions that require
  3449. * some dynamic calibration support.
  3450. */
  3451. seq.dyn_calib_steps = STATIC_CALIB_STEPS;
  3452. /*
  3453. * Load global to allow dynamic selection of delay loop settings
  3454. * based on calibration mode.
  3455. */
  3456. if (!(seq.dyn_calib_steps & CALIB_SKIP_DELAY_LOOPS))
  3457. seq.skip_delay_mask = 0xff;
  3458. else
  3459. seq.skip_delay_mask = 0x0;
  3460. pass = run_mem_calibrate(&seq);
  3461. debug_mem_calibrate(&seq, pass);
  3462. return pass;
  3463. }