sequencer.c 112 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991
  1. // SPDX-License-Identifier: BSD-3-Clause
  2. /*
  3. * Copyright Altera Corporation (C) 2012-2015
  4. */
  5. #include <common.h>
  6. #include <log.h>
  7. #include <asm/io.h>
  8. #include <asm/arch/sdram.h>
  9. #include <errno.h>
  10. #include <hang.h>
  11. #include "sequencer.h"
  12. static const struct socfpga_sdr_rw_load_manager *sdr_rw_load_mgr_regs =
  13. (struct socfpga_sdr_rw_load_manager *)
  14. (SDR_PHYGRP_RWMGRGRP_ADDRESS | 0x800);
  15. static const struct socfpga_sdr_rw_load_jump_manager *sdr_rw_load_jump_mgr_regs
  16. = (struct socfpga_sdr_rw_load_jump_manager *)
  17. (SDR_PHYGRP_RWMGRGRP_ADDRESS | 0xC00);
  18. static const struct socfpga_sdr_reg_file *sdr_reg_file =
  19. (struct socfpga_sdr_reg_file *)SDR_PHYGRP_REGFILEGRP_ADDRESS;
  20. static const struct socfpga_sdr_scc_mgr *sdr_scc_mgr =
  21. (struct socfpga_sdr_scc_mgr *)
  22. (SDR_PHYGRP_SCCGRP_ADDRESS | 0xe00);
  23. static const struct socfpga_phy_mgr_cmd *phy_mgr_cmd =
  24. (struct socfpga_phy_mgr_cmd *)SDR_PHYGRP_PHYMGRGRP_ADDRESS;
  25. static const struct socfpga_phy_mgr_cfg *phy_mgr_cfg =
  26. (struct socfpga_phy_mgr_cfg *)
  27. (SDR_PHYGRP_PHYMGRGRP_ADDRESS | 0x40);
  28. static const struct socfpga_data_mgr *data_mgr =
  29. (struct socfpga_data_mgr *)SDR_PHYGRP_DATAMGRGRP_ADDRESS;
  30. static const struct socfpga_sdr_ctrl *sdr_ctrl =
  31. (struct socfpga_sdr_ctrl *)SDR_CTRLGRP_ADDRESS;
  32. #define DELTA_D 1
  33. /*
  34. * In order to reduce ROM size, most of the selectable calibration steps are
  35. * decided at compile time based on the user's calibration mode selection,
  36. * as captured by the STATIC_CALIB_STEPS selection below.
  37. *
  38. * However, to support simulation-time selection of fast simulation mode, where
  39. * we skip everything except the bare minimum, we need a few of the steps to
  40. * be dynamic. In those cases, we either use the DYNAMIC_CALIB_STEPS for the
  41. * check, which is based on the rtl-supplied value, or we dynamically compute
  42. * the value to use based on the dynamically-chosen calibration mode
  43. */
  44. #define DLEVEL 0
  45. #define STATIC_IN_RTL_SIM 0
  46. #define STATIC_SKIP_DELAY_LOOPS 0
  47. #define STATIC_CALIB_STEPS (STATIC_IN_RTL_SIM | CALIB_SKIP_FULL_TEST | \
  48. STATIC_SKIP_DELAY_LOOPS)
  49. #define SKIP_DELAY_LOOP_VALUE_OR_ZERO(non_skip_value) \
  50. ((non_skip_value) & seq->skip_delay_mask)
  51. bool dram_is_ddr(const u8 ddr)
  52. {
  53. const struct socfpga_sdram_config *cfg = socfpga_get_sdram_config();
  54. const u8 type = (cfg->ctrl_cfg >> SDR_CTRLGRP_CTRLCFG_MEMTYPE_LSB) &
  55. SDR_CTRLGRP_CTRLCFG_MEMTYPE_MASK;
  56. if (ddr == 2 && type == 1) /* DDR2 */
  57. return true;
  58. if (ddr == 3 && type == 2) /* DDR3 */
  59. return true;
  60. return false;
  61. }
  62. static void set_failing_group_stage(struct socfpga_sdrseq *seq,
  63. u32 group, u32 stage, u32 substage)
  64. {
  65. /*
  66. * Only set the global stage if there was not been any other
  67. * failing group
  68. */
  69. if (seq->gbl.error_stage == CAL_STAGE_NIL) {
  70. seq->gbl.error_substage = substage;
  71. seq->gbl.error_stage = stage;
  72. seq->gbl.error_group = group;
  73. }
  74. }
  75. static void reg_file_set_group(u16 set_group)
  76. {
  77. clrsetbits_le32(&sdr_reg_file->cur_stage, 0xffff0000, set_group << 16);
  78. }
  79. static void reg_file_set_stage(u8 set_stage)
  80. {
  81. clrsetbits_le32(&sdr_reg_file->cur_stage, 0xffff, set_stage & 0xff);
  82. }
  83. static void reg_file_set_sub_stage(u8 set_sub_stage)
  84. {
  85. set_sub_stage &= 0xff;
  86. clrsetbits_le32(&sdr_reg_file->cur_stage, 0xff00, set_sub_stage << 8);
  87. }
  88. /**
  89. * phy_mgr_initialize() - Initialize PHY Manager
  90. *
  91. * Initialize PHY Manager.
  92. */
  93. static void phy_mgr_initialize(struct socfpga_sdrseq *seq)
  94. {
  95. u32 ratio;
  96. debug("%s:%d\n", __func__, __LINE__);
  97. /* Calibration has control over path to memory */
  98. /*
  99. * In Hard PHY this is a 2-bit control:
  100. * 0: AFI Mux Select
  101. * 1: DDIO Mux Select
  102. */
  103. writel(0x3, &phy_mgr_cfg->mux_sel);
  104. /* USER memory clock is not stable we begin initialization */
  105. writel(0, &phy_mgr_cfg->reset_mem_stbl);
  106. /* USER calibration status all set to zero */
  107. writel(0, &phy_mgr_cfg->cal_status);
  108. writel(0, &phy_mgr_cfg->cal_debug_info);
  109. /* Init params only if we do NOT skip calibration. */
  110. if ((seq->dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL)
  111. return;
  112. ratio = seq->rwcfg->mem_dq_per_read_dqs /
  113. seq->rwcfg->mem_virtual_groups_per_read_dqs;
  114. seq->param.read_correct_mask_vg = (1 << ratio) - 1;
  115. seq->param.write_correct_mask_vg = (1 << ratio) - 1;
  116. seq->param.read_correct_mask = (1 << seq->rwcfg->mem_dq_per_read_dqs)
  117. - 1;
  118. seq->param.write_correct_mask = (1 << seq->rwcfg->mem_dq_per_write_dqs)
  119. - 1;
  120. }
  121. /**
  122. * set_rank_and_odt_mask() - Set Rank and ODT mask
  123. * @rank: Rank mask
  124. * @odt_mode: ODT mode, OFF or READ_WRITE
  125. *
  126. * Set Rank and ODT mask (On-Die Termination).
  127. */
  128. static void set_rank_and_odt_mask(struct socfpga_sdrseq *seq,
  129. const u32 rank, const u32 odt_mode)
  130. {
  131. u32 odt_mask_0 = 0;
  132. u32 odt_mask_1 = 0;
  133. u32 cs_and_odt_mask;
  134. if (odt_mode == RW_MGR_ODT_MODE_OFF) {
  135. odt_mask_0 = 0x0;
  136. odt_mask_1 = 0x0;
  137. } else { /* RW_MGR_ODT_MODE_READ_WRITE */
  138. switch (seq->rwcfg->mem_number_of_ranks) {
  139. case 1: /* 1 Rank */
  140. /* Read: ODT = 0 ; Write: ODT = 1 */
  141. odt_mask_0 = 0x0;
  142. odt_mask_1 = 0x1;
  143. break;
  144. case 2: /* 2 Ranks */
  145. if (seq->rwcfg->mem_number_of_cs_per_dimm == 1) {
  146. /*
  147. * - Dual-Slot , Single-Rank (1 CS per DIMM)
  148. * OR
  149. * - RDIMM, 4 total CS (2 CS per DIMM, 2 DIMM)
  150. *
  151. * Since MEM_NUMBER_OF_RANKS is 2, they
  152. * are both single rank with 2 CS each
  153. * (special for RDIMM).
  154. *
  155. * Read: Turn on ODT on the opposite rank
  156. * Write: Turn on ODT on all ranks
  157. */
  158. odt_mask_0 = 0x3 & ~(1 << rank);
  159. odt_mask_1 = 0x3;
  160. if (dram_is_ddr(2))
  161. odt_mask_1 &= ~(1 << rank);
  162. } else {
  163. /*
  164. * - Single-Slot , Dual-Rank (2 CS per DIMM)
  165. *
  166. * Read: Turn on ODT off on all ranks
  167. * Write: Turn on ODT on active rank
  168. */
  169. odt_mask_0 = 0x0;
  170. odt_mask_1 = 0x3 & (1 << rank);
  171. }
  172. break;
  173. case 4: /* 4 Ranks */
  174. /*
  175. * DDR3 Read, DDR2 Read/Write:
  176. * ----------+-----------------------+
  177. * | ODT |
  178. * +-----------------------+
  179. * Rank | 3 | 2 | 1 | 0 |
  180. * ----------+-----+-----+-----+-----+
  181. * 0 | 0 | 1 | 0 | 0 |
  182. * 1 | 1 | 0 | 0 | 0 |
  183. * 2 | 0 | 0 | 0 | 1 |
  184. * 3 | 0 | 0 | 1 | 0 |
  185. * ----------+-----+-----+-----+-----+
  186. *
  187. * DDR3 Write:
  188. * ----------+-----------------------+
  189. * | ODT |
  190. * Write To +-----------------------+
  191. * Rank | 3 | 2 | 1 | 0 |
  192. * ----------+-----+-----+-----+-----+
  193. * 0 | 0 | 1 | 0 | 1 |
  194. * 1 | 1 | 0 | 1 | 0 |
  195. * 2 | 0 | 1 | 0 | 1 |
  196. * 3 | 1 | 0 | 1 | 0 |
  197. * ----------+-----+-----+-----+-----+
  198. */
  199. switch (rank) {
  200. case 0:
  201. odt_mask_0 = 0x4;
  202. if (dram_is_ddr(2))
  203. odt_mask_1 = 0x4;
  204. else if (dram_is_ddr(3))
  205. odt_mask_1 = 0x5;
  206. break;
  207. case 1:
  208. odt_mask_0 = 0x8;
  209. if (dram_is_ddr(2))
  210. odt_mask_1 = 0x8;
  211. else if (dram_is_ddr(3))
  212. odt_mask_1 = 0xA;
  213. break;
  214. case 2:
  215. odt_mask_0 = 0x1;
  216. if (dram_is_ddr(2))
  217. odt_mask_1 = 0x1;
  218. else if (dram_is_ddr(3))
  219. odt_mask_1 = 0x5;
  220. break;
  221. case 3:
  222. odt_mask_0 = 0x2;
  223. if (dram_is_ddr(2))
  224. odt_mask_1 = 0x2;
  225. else if (dram_is_ddr(3))
  226. odt_mask_1 = 0xA;
  227. break;
  228. }
  229. break;
  230. }
  231. }
  232. cs_and_odt_mask = (0xFF & ~(1 << rank)) |
  233. ((0xFF & odt_mask_0) << 8) |
  234. ((0xFF & odt_mask_1) << 16);
  235. writel(cs_and_odt_mask, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  236. RW_MGR_SET_CS_AND_ODT_MASK_OFFSET);
  237. }
  238. /**
  239. * scc_mgr_set() - Set SCC Manager register
  240. * @off: Base offset in SCC Manager space
  241. * @grp: Read/Write group
  242. * @val: Value to be set
  243. *
  244. * This function sets the SCC Manager (Scan Chain Control Manager) register.
  245. */
  246. static void scc_mgr_set(u32 off, u32 grp, u32 val)
  247. {
  248. writel(val, SDR_PHYGRP_SCCGRP_ADDRESS | off | (grp << 2));
  249. }
  250. /**
  251. * scc_mgr_initialize() - Initialize SCC Manager registers
  252. *
  253. * Initialize SCC Manager registers.
  254. */
  255. static void scc_mgr_initialize(void)
  256. {
  257. /*
  258. * Clear register file for HPS. 16 (2^4) is the size of the
  259. * full register file in the scc mgr:
  260. * RFILE_DEPTH = 1 + log2(MEM_DQ_PER_DQS + 1 + MEM_DM_PER_DQS +
  261. * MEM_IF_READ_DQS_WIDTH - 1);
  262. */
  263. int i;
  264. for (i = 0; i < 16; i++) {
  265. debug_cond(DLEVEL >= 1, "%s:%d: Clearing SCC RFILE index %u\n",
  266. __func__, __LINE__, i);
  267. scc_mgr_set(SCC_MGR_HHP_RFILE_OFFSET, i, 0);
  268. }
  269. }
  270. static void scc_mgr_set_dqdqs_output_phase(u32 write_group, u32 phase)
  271. {
  272. scc_mgr_set(SCC_MGR_DQDQS_OUT_PHASE_OFFSET, write_group, phase);
  273. }
  274. static void scc_mgr_set_dqs_bus_in_delay(u32 read_group, u32 delay)
  275. {
  276. scc_mgr_set(SCC_MGR_DQS_IN_DELAY_OFFSET, read_group, delay);
  277. }
  278. static void scc_mgr_set_dqs_en_phase(u32 read_group, u32 phase)
  279. {
  280. scc_mgr_set(SCC_MGR_DQS_EN_PHASE_OFFSET, read_group, phase);
  281. }
  282. static void scc_mgr_set_dqs_en_delay(u32 read_group, u32 delay)
  283. {
  284. scc_mgr_set(SCC_MGR_DQS_EN_DELAY_OFFSET, read_group, delay);
  285. }
  286. static void scc_mgr_set_dq_in_delay(u32 dq_in_group, u32 delay)
  287. {
  288. scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET, dq_in_group, delay);
  289. }
  290. static void scc_mgr_set_dqs_io_in_delay(struct socfpga_sdrseq *seq,
  291. u32 delay)
  292. {
  293. scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET,
  294. seq->rwcfg->mem_dq_per_write_dqs, delay);
  295. }
  296. static void scc_mgr_set_dm_in_delay(struct socfpga_sdrseq *seq, u32 dm,
  297. u32 delay)
  298. {
  299. scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET,
  300. seq->rwcfg->mem_dq_per_write_dqs + 1 + dm,
  301. delay);
  302. }
  303. static void scc_mgr_set_dq_out1_delay(u32 dq_in_group, u32 delay)
  304. {
  305. scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET, dq_in_group, delay);
  306. }
  307. static void scc_mgr_set_dqs_out1_delay(struct socfpga_sdrseq *seq,
  308. u32 delay)
  309. {
  310. scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET,
  311. seq->rwcfg->mem_dq_per_write_dqs, delay);
  312. }
  313. static void scc_mgr_set_dm_out1_delay(struct socfpga_sdrseq *seq, u32 dm,
  314. u32 delay)
  315. {
  316. scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET,
  317. seq->rwcfg->mem_dq_per_write_dqs + 1 + dm,
  318. delay);
  319. }
  320. /* load up dqs config settings */
  321. static void scc_mgr_load_dqs(u32 dqs)
  322. {
  323. writel(dqs, &sdr_scc_mgr->dqs_ena);
  324. }
  325. /* load up dqs io config settings */
  326. static void scc_mgr_load_dqs_io(void)
  327. {
  328. writel(0, &sdr_scc_mgr->dqs_io_ena);
  329. }
  330. /* load up dq config settings */
  331. static void scc_mgr_load_dq(u32 dq_in_group)
  332. {
  333. writel(dq_in_group, &sdr_scc_mgr->dq_ena);
  334. }
  335. /* load up dm config settings */
  336. static void scc_mgr_load_dm(u32 dm)
  337. {
  338. writel(dm, &sdr_scc_mgr->dm_ena);
  339. }
  340. /**
  341. * scc_mgr_set_all_ranks() - Set SCC Manager register for all ranks
  342. * @off: Base offset in SCC Manager space
  343. * @grp: Read/Write group
  344. * @val: Value to be set
  345. * @update: If non-zero, trigger SCC Manager update for all ranks
  346. *
  347. * This function sets the SCC Manager (Scan Chain Control Manager) register
  348. * and optionally triggers the SCC update for all ranks.
  349. */
  350. static void scc_mgr_set_all_ranks(struct socfpga_sdrseq *seq,
  351. const u32 off, const u32 grp, const u32 val,
  352. const int update)
  353. {
  354. u32 r;
  355. for (r = 0; r < seq->rwcfg->mem_number_of_ranks;
  356. r += NUM_RANKS_PER_SHADOW_REG) {
  357. scc_mgr_set(off, grp, val);
  358. if (update || (r == 0)) {
  359. writel(grp, &sdr_scc_mgr->dqs_ena);
  360. writel(0, &sdr_scc_mgr->update);
  361. }
  362. }
  363. }
  364. static void scc_mgr_set_dqs_en_phase_all_ranks(struct socfpga_sdrseq *seq,
  365. u32 read_group, u32 phase)
  366. {
  367. /*
  368. * USER although the h/w doesn't support different phases per
  369. * shadow register, for simplicity our scc manager modeling
  370. * keeps different phase settings per shadow reg, and it's
  371. * important for us to keep them in sync to match h/w.
  372. * for efficiency, the scan chain update should occur only
  373. * once to sr0.
  374. */
  375. scc_mgr_set_all_ranks(seq, SCC_MGR_DQS_EN_PHASE_OFFSET,
  376. read_group, phase, 0);
  377. }
  378. static void scc_mgr_set_dqdqs_output_phase_all_ranks(struct socfpga_sdrseq *seq,
  379. u32 write_group, u32 phase)
  380. {
  381. /*
  382. * USER although the h/w doesn't support different phases per
  383. * shadow register, for simplicity our scc manager modeling
  384. * keeps different phase settings per shadow reg, and it's
  385. * important for us to keep them in sync to match h/w.
  386. * for efficiency, the scan chain update should occur only
  387. * once to sr0.
  388. */
  389. scc_mgr_set_all_ranks(seq, SCC_MGR_DQDQS_OUT_PHASE_OFFSET,
  390. write_group, phase, 0);
  391. }
  392. static void scc_mgr_set_dqs_en_delay_all_ranks(struct socfpga_sdrseq *seq,
  393. u32 read_group, u32 delay)
  394. {
  395. /*
  396. * In shadow register mode, the T11 settings are stored in
  397. * registers in the core, which are updated by the DQS_ENA
  398. * signals. Not issuing the SCC_MGR_UPD command allows us to
  399. * save lots of rank switching overhead, by calling
  400. * select_shadow_regs_for_update with update_scan_chains
  401. * set to 0.
  402. */
  403. scc_mgr_set_all_ranks(seq, SCC_MGR_DQS_EN_DELAY_OFFSET,
  404. read_group, delay, 1);
  405. }
  406. /**
  407. * scc_mgr_set_oct_out1_delay() - Set OCT output delay
  408. * @write_group: Write group
  409. * @delay: Delay value
  410. *
  411. * This function sets the OCT output delay in SCC manager.
  412. */
  413. static void scc_mgr_set_oct_out1_delay(struct socfpga_sdrseq *seq,
  414. const u32 write_group, const u32 delay)
  415. {
  416. const int ratio = seq->rwcfg->mem_if_read_dqs_width /
  417. seq->rwcfg->mem_if_write_dqs_width;
  418. const int base = write_group * ratio;
  419. int i;
  420. /*
  421. * Load the setting in the SCC manager
  422. * Although OCT affects only write data, the OCT delay is controlled
  423. * by the DQS logic block which is instantiated once per read group.
  424. * For protocols where a write group consists of multiple read groups,
  425. * the setting must be set multiple times.
  426. */
  427. for (i = 0; i < ratio; i++)
  428. scc_mgr_set(SCC_MGR_OCT_OUT1_DELAY_OFFSET, base + i, delay);
  429. }
  430. /**
  431. * scc_mgr_set_hhp_extras() - Set HHP extras.
  432. *
  433. * Load the fixed setting in the SCC manager HHP extras.
  434. */
  435. static void scc_mgr_set_hhp_extras(void)
  436. {
  437. /*
  438. * Load the fixed setting in the SCC manager
  439. * bits: 0:0 = 1'b1 - DQS bypass
  440. * bits: 1:1 = 1'b1 - DQ bypass
  441. * bits: 4:2 = 3'b001 - rfifo_mode
  442. * bits: 6:5 = 2'b01 - rfifo clock_select
  443. * bits: 7:7 = 1'b0 - separate gating from ungating setting
  444. * bits: 8:8 = 1'b0 - separate OE from Output delay setting
  445. */
  446. const u32 value = (0 << 8) | (0 << 7) | (1 << 5) |
  447. (1 << 2) | (1 << 1) | (1 << 0);
  448. const u32 addr = SDR_PHYGRP_SCCGRP_ADDRESS |
  449. SCC_MGR_HHP_GLOBALS_OFFSET |
  450. SCC_MGR_HHP_EXTRAS_OFFSET;
  451. debug_cond(DLEVEL >= 1, "%s:%d Setting HHP Extras\n",
  452. __func__, __LINE__);
  453. writel(value, addr);
  454. debug_cond(DLEVEL >= 1, "%s:%d Done Setting HHP Extras\n",
  455. __func__, __LINE__);
  456. }
  457. /**
  458. * scc_mgr_zero_all() - Zero all DQS config
  459. *
  460. * Zero all DQS config.
  461. */
  462. static void scc_mgr_zero_all(struct socfpga_sdrseq *seq)
  463. {
  464. int i, r;
  465. /*
  466. * USER Zero all DQS config settings, across all groups and all
  467. * shadow registers
  468. */
  469. for (r = 0; r < seq->rwcfg->mem_number_of_ranks;
  470. r += NUM_RANKS_PER_SHADOW_REG) {
  471. for (i = 0; i < seq->rwcfg->mem_if_read_dqs_width; i++) {
  472. /*
  473. * The phases actually don't exist on a per-rank basis,
  474. * but there's no harm updating them several times, so
  475. * let's keep the code simple.
  476. */
  477. scc_mgr_set_dqs_bus_in_delay(i,
  478. seq->iocfg->dqs_in_reserve
  479. );
  480. scc_mgr_set_dqs_en_phase(i, 0);
  481. scc_mgr_set_dqs_en_delay(i, 0);
  482. }
  483. for (i = 0; i < seq->rwcfg->mem_if_write_dqs_width; i++) {
  484. scc_mgr_set_dqdqs_output_phase(i, 0);
  485. /* Arria V/Cyclone V don't have out2. */
  486. scc_mgr_set_oct_out1_delay(seq, i,
  487. seq->iocfg->dqs_out_reserve);
  488. }
  489. }
  490. /* Multicast to all DQS group enables. */
  491. writel(0xff, &sdr_scc_mgr->dqs_ena);
  492. writel(0, &sdr_scc_mgr->update);
  493. }
  494. /**
  495. * scc_set_bypass_mode() - Set bypass mode and trigger SCC update
  496. * @write_group: Write group
  497. *
  498. * Set bypass mode and trigger SCC update.
  499. */
  500. static void scc_set_bypass_mode(const u32 write_group)
  501. {
  502. /* Multicast to all DQ enables. */
  503. writel(0xff, &sdr_scc_mgr->dq_ena);
  504. writel(0xff, &sdr_scc_mgr->dm_ena);
  505. /* Update current DQS IO enable. */
  506. writel(0, &sdr_scc_mgr->dqs_io_ena);
  507. /* Update the DQS logic. */
  508. writel(write_group, &sdr_scc_mgr->dqs_ena);
  509. /* Hit update. */
  510. writel(0, &sdr_scc_mgr->update);
  511. }
  512. /**
  513. * scc_mgr_load_dqs_for_write_group() - Load DQS settings for Write Group
  514. * @write_group: Write group
  515. *
  516. * Load DQS settings for Write Group, do not trigger SCC update.
  517. */
  518. static void scc_mgr_load_dqs_for_write_group(struct socfpga_sdrseq *seq,
  519. const u32 write_group)
  520. {
  521. const int ratio = seq->rwcfg->mem_if_read_dqs_width /
  522. seq->rwcfg->mem_if_write_dqs_width;
  523. const int base = write_group * ratio;
  524. int i;
  525. /*
  526. * Load the setting in the SCC manager
  527. * Although OCT affects only write data, the OCT delay is controlled
  528. * by the DQS logic block which is instantiated once per read group.
  529. * For protocols where a write group consists of multiple read groups,
  530. * the setting must be set multiple times.
  531. */
  532. for (i = 0; i < ratio; i++)
  533. writel(base + i, &sdr_scc_mgr->dqs_ena);
  534. }
  535. /**
  536. * scc_mgr_zero_group() - Zero all configs for a group
  537. *
  538. * Zero DQ, DM, DQS and OCT configs for a group.
  539. */
  540. static void scc_mgr_zero_group(struct socfpga_sdrseq *seq,
  541. const u32 write_group, const int out_only)
  542. {
  543. int i, r;
  544. for (r = 0; r < seq->rwcfg->mem_number_of_ranks;
  545. r += NUM_RANKS_PER_SHADOW_REG) {
  546. /* Zero all DQ config settings. */
  547. for (i = 0; i < seq->rwcfg->mem_dq_per_write_dqs; i++) {
  548. scc_mgr_set_dq_out1_delay(i, 0);
  549. if (!out_only)
  550. scc_mgr_set_dq_in_delay(i, 0);
  551. }
  552. /* Multicast to all DQ enables. */
  553. writel(0xff, &sdr_scc_mgr->dq_ena);
  554. /* Zero all DM config settings. */
  555. for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
  556. if (!out_only)
  557. scc_mgr_set_dm_in_delay(seq, i, 0);
  558. scc_mgr_set_dm_out1_delay(seq, i, 0);
  559. }
  560. /* Multicast to all DM enables. */
  561. writel(0xff, &sdr_scc_mgr->dm_ena);
  562. /* Zero all DQS IO settings. */
  563. if (!out_only)
  564. scc_mgr_set_dqs_io_in_delay(seq, 0);
  565. /* Arria V/Cyclone V don't have out2. */
  566. scc_mgr_set_dqs_out1_delay(seq, seq->iocfg->dqs_out_reserve);
  567. scc_mgr_set_oct_out1_delay(seq, write_group,
  568. seq->iocfg->dqs_out_reserve);
  569. scc_mgr_load_dqs_for_write_group(seq, write_group);
  570. /* Multicast to all DQS IO enables (only 1 in total). */
  571. writel(0, &sdr_scc_mgr->dqs_io_ena);
  572. /* Hit update to zero everything. */
  573. writel(0, &sdr_scc_mgr->update);
  574. }
  575. }
  576. /*
  577. * apply and load a particular input delay for the DQ pins in a group
  578. * group_bgn is the index of the first dq pin (in the write group)
  579. */
  580. static void scc_mgr_apply_group_dq_in_delay(struct socfpga_sdrseq *seq,
  581. u32 group_bgn, u32 delay)
  582. {
  583. u32 i, p;
  584. for (i = 0, p = group_bgn; i < seq->rwcfg->mem_dq_per_read_dqs;
  585. i++, p++) {
  586. scc_mgr_set_dq_in_delay(p, delay);
  587. scc_mgr_load_dq(p);
  588. }
  589. }
  590. /**
  591. * scc_mgr_apply_group_dq_out1_delay() - Apply and load an output delay for the
  592. * DQ pins in a group
  593. * @delay: Delay value
  594. *
  595. * Apply and load a particular output delay for the DQ pins in a group.
  596. */
  597. static void scc_mgr_apply_group_dq_out1_delay(struct socfpga_sdrseq *seq,
  598. const u32 delay)
  599. {
  600. int i;
  601. for (i = 0; i < seq->rwcfg->mem_dq_per_write_dqs; i++) {
  602. scc_mgr_set_dq_out1_delay(i, delay);
  603. scc_mgr_load_dq(i);
  604. }
  605. }
  606. /* apply and load a particular output delay for the DM pins in a group */
  607. static void scc_mgr_apply_group_dm_out1_delay(struct socfpga_sdrseq *seq,
  608. u32 delay1)
  609. {
  610. u32 i;
  611. for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
  612. scc_mgr_set_dm_out1_delay(seq, i, delay1);
  613. scc_mgr_load_dm(i);
  614. }
  615. }
  616. /* apply and load delay on both DQS and OCT out1 */
  617. static void scc_mgr_apply_group_dqs_io_and_oct_out1(struct socfpga_sdrseq *seq,
  618. u32 write_group, u32 delay)
  619. {
  620. scc_mgr_set_dqs_out1_delay(seq, delay);
  621. scc_mgr_load_dqs_io();
  622. scc_mgr_set_oct_out1_delay(seq, write_group, delay);
  623. scc_mgr_load_dqs_for_write_group(seq, write_group);
  624. }
  625. /**
  626. * scc_mgr_apply_group_all_out_delay_add() - Apply a delay to the entire output
  627. * side: DQ, DM, DQS, OCT
  628. * @write_group: Write group
  629. * @delay: Delay value
  630. *
  631. * Apply a delay to the entire output side: DQ, DM, DQS, OCT.
  632. */
  633. static void scc_mgr_apply_group_all_out_delay_add(struct socfpga_sdrseq *seq,
  634. const u32 write_group,
  635. const u32 delay)
  636. {
  637. u32 i, new_delay;
  638. /* DQ shift */
  639. for (i = 0; i < seq->rwcfg->mem_dq_per_write_dqs; i++)
  640. scc_mgr_load_dq(i);
  641. /* DM shift */
  642. for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++)
  643. scc_mgr_load_dm(i);
  644. /* DQS shift */
  645. new_delay = READ_SCC_DQS_IO_OUT2_DELAY + delay;
  646. if (new_delay > seq->iocfg->io_out2_delay_max) {
  647. debug_cond(DLEVEL >= 1,
  648. "%s:%d (%u, %u) DQS: %u > %d; adding %u to OUT1\n",
  649. __func__, __LINE__, write_group, delay, new_delay,
  650. seq->iocfg->io_out2_delay_max,
  651. new_delay - seq->iocfg->io_out2_delay_max);
  652. new_delay -= seq->iocfg->io_out2_delay_max;
  653. scc_mgr_set_dqs_out1_delay(seq, new_delay);
  654. }
  655. scc_mgr_load_dqs_io();
  656. /* OCT shift */
  657. new_delay = READ_SCC_OCT_OUT2_DELAY + delay;
  658. if (new_delay > seq->iocfg->io_out2_delay_max) {
  659. debug_cond(DLEVEL >= 1,
  660. "%s:%d (%u, %u) DQS: %u > %d; adding %u to OUT1\n",
  661. __func__, __LINE__, write_group, delay,
  662. new_delay, seq->iocfg->io_out2_delay_max,
  663. new_delay - seq->iocfg->io_out2_delay_max);
  664. new_delay -= seq->iocfg->io_out2_delay_max;
  665. scc_mgr_set_oct_out1_delay(seq, write_group, new_delay);
  666. }
  667. scc_mgr_load_dqs_for_write_group(seq, write_group);
  668. }
  669. /**
  670. * scc_mgr_apply_group_all_out_delay_add() - Apply a delay to the entire output
  671. * side to all ranks
  672. * @write_group: Write group
  673. * @delay: Delay value
  674. *
  675. * Apply a delay to the entire output side (DQ, DM, DQS, OCT) to all ranks.
  676. */
  677. static void
  678. scc_mgr_apply_group_all_out_delay_add_all_ranks(struct socfpga_sdrseq *seq,
  679. const u32 write_group,
  680. const u32 delay)
  681. {
  682. int r;
  683. for (r = 0; r < seq->rwcfg->mem_number_of_ranks;
  684. r += NUM_RANKS_PER_SHADOW_REG) {
  685. scc_mgr_apply_group_all_out_delay_add(seq, write_group, delay);
  686. writel(0, &sdr_scc_mgr->update);
  687. }
  688. }
  689. /**
  690. * set_jump_as_return() - Return instruction optimization
  691. *
  692. * Optimization used to recover some slots in ddr3 inst_rom could be
  693. * applied to other protocols if we wanted to
  694. */
  695. static void set_jump_as_return(struct socfpga_sdrseq *seq)
  696. {
  697. /*
  698. * To save space, we replace return with jump to special shared
  699. * RETURN instruction so we set the counter to large value so that
  700. * we always jump.
  701. */
  702. writel(0xff, &sdr_rw_load_mgr_regs->load_cntr0);
  703. writel(seq->rwcfg->rreturn, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  704. }
  705. /**
  706. * delay_for_n_mem_clocks() - Delay for N memory clocks
  707. * @clocks: Length of the delay
  708. *
  709. * Delay for N memory clocks.
  710. */
  711. static void delay_for_n_mem_clocks(struct socfpga_sdrseq *seq,
  712. const u32 clocks)
  713. {
  714. u32 afi_clocks;
  715. u16 c_loop;
  716. u8 inner;
  717. u8 outer;
  718. debug("%s:%d: clocks=%u ... start\n", __func__, __LINE__, clocks);
  719. /* Scale (rounding up) to get afi clocks. */
  720. afi_clocks = DIV_ROUND_UP(clocks, seq->misccfg->afi_rate_ratio);
  721. if (afi_clocks) /* Temporary underflow protection */
  722. afi_clocks--;
  723. /*
  724. * Note, we don't bother accounting for being off a little
  725. * bit because of a few extra instructions in outer loops.
  726. * Note, the loops have a test at the end, and do the test
  727. * before the decrement, and so always perform the loop
  728. * 1 time more than the counter value
  729. */
  730. c_loop = afi_clocks >> 16;
  731. outer = c_loop ? 0xff : (afi_clocks >> 8);
  732. inner = outer ? 0xff : afi_clocks;
  733. /*
  734. * rom instructions are structured as follows:
  735. *
  736. * IDLE_LOOP2: jnz cntr0, TARGET_A
  737. * IDLE_LOOP1: jnz cntr1, TARGET_B
  738. * return
  739. *
  740. * so, when doing nested loops, TARGET_A is set to IDLE_LOOP2, and
  741. * TARGET_B is set to IDLE_LOOP2 as well
  742. *
  743. * if we have no outer loop, though, then we can use IDLE_LOOP1 only,
  744. * and set TARGET_B to IDLE_LOOP1 and we skip IDLE_LOOP2 entirely
  745. *
  746. * a little confusing, but it helps save precious space in the inst_rom
  747. * and sequencer rom and keeps the delays more accurate and reduces
  748. * overhead
  749. */
  750. if (afi_clocks < 0x100) {
  751. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner),
  752. &sdr_rw_load_mgr_regs->load_cntr1);
  753. writel(seq->rwcfg->idle_loop1,
  754. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  755. writel(seq->rwcfg->idle_loop1, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  756. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  757. } else {
  758. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner),
  759. &sdr_rw_load_mgr_regs->load_cntr0);
  760. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(outer),
  761. &sdr_rw_load_mgr_regs->load_cntr1);
  762. writel(seq->rwcfg->idle_loop2,
  763. &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  764. writel(seq->rwcfg->idle_loop2,
  765. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  766. do {
  767. writel(seq->rwcfg->idle_loop2,
  768. SDR_PHYGRP_RWMGRGRP_ADDRESS |
  769. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  770. } while (c_loop-- != 0);
  771. }
  772. debug("%s:%d clocks=%u ... end\n", __func__, __LINE__, clocks);
  773. }
  774. static void delay_for_n_ns(struct socfpga_sdrseq *seq, const u32 ns)
  775. {
  776. delay_for_n_mem_clocks(seq, (ns * seq->misccfg->afi_clk_freq *
  777. seq->misccfg->afi_rate_ratio) / 1000);
  778. }
  779. /**
  780. * rw_mgr_mem_init_load_regs() - Load instruction registers
  781. * @cntr0: Counter 0 value
  782. * @cntr1: Counter 1 value
  783. * @cntr2: Counter 2 value
  784. * @jump: Jump instruction value
  785. *
  786. * Load instruction registers.
  787. */
  788. static void rw_mgr_mem_init_load_regs(struct socfpga_sdrseq *seq,
  789. u32 cntr0, u32 cntr1, u32 cntr2, u32 jump)
  790. {
  791. u32 grpaddr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
  792. RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  793. /* Load counters */
  794. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr0),
  795. &sdr_rw_load_mgr_regs->load_cntr0);
  796. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr1),
  797. &sdr_rw_load_mgr_regs->load_cntr1);
  798. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr2),
  799. &sdr_rw_load_mgr_regs->load_cntr2);
  800. /* Load jump address */
  801. writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  802. writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  803. writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  804. /* Execute count instruction */
  805. writel(jump, grpaddr);
  806. }
  807. /**
  808. * rw_mgr_mem_load_user_ddr2() - Load user calibration values for DDR2
  809. * @handoff: Indicate whether this is initialization or handoff phase
  810. *
  811. * Load user calibration values and optionally precharge the banks.
  812. */
  813. static void rw_mgr_mem_load_user_ddr2(struct socfpga_sdrseq *seq,
  814. const int handoff)
  815. {
  816. u32 grpaddr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
  817. RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  818. u32 r;
  819. for (r = 0; r < seq->rwcfg->mem_number_of_ranks; r++) {
  820. /* set rank */
  821. set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_OFF);
  822. /* precharge all banks ... */
  823. writel(seq->rwcfg->precharge_all, grpaddr);
  824. writel(seq->rwcfg->emr2, grpaddr);
  825. writel(seq->rwcfg->emr3, grpaddr);
  826. writel(seq->rwcfg->emr, grpaddr);
  827. if (handoff) {
  828. writel(seq->rwcfg->mr_user, grpaddr);
  829. continue;
  830. }
  831. writel(seq->rwcfg->mr_dll_reset, grpaddr);
  832. writel(seq->rwcfg->precharge_all, grpaddr);
  833. writel(seq->rwcfg->refresh, grpaddr);
  834. delay_for_n_ns(seq, 200);
  835. writel(seq->rwcfg->refresh, grpaddr);
  836. delay_for_n_ns(seq, 200);
  837. writel(seq->rwcfg->mr_calib, grpaddr);
  838. writel(/*seq->rwcfg->*/0x0b, grpaddr); // EMR_OCD_ENABLE
  839. writel(seq->rwcfg->emr, grpaddr);
  840. delay_for_n_mem_clocks(seq, 200);
  841. }
  842. }
  843. /**
  844. * rw_mgr_mem_load_user_ddr3() - Load user calibration values
  845. * @fin1: Final instruction 1
  846. * @fin2: Final instruction 2
  847. * @precharge: If 1, precharge the banks at the end
  848. *
  849. * Load user calibration values and optionally precharge the banks.
  850. */
  851. static void rw_mgr_mem_load_user_ddr3(struct socfpga_sdrseq *seq,
  852. const u32 fin1, const u32 fin2,
  853. const int precharge)
  854. {
  855. u32 grpaddr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
  856. RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  857. u32 r;
  858. for (r = 0; r < seq->rwcfg->mem_number_of_ranks; r++) {
  859. /* set rank */
  860. set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_OFF);
  861. /* precharge all banks ... */
  862. if (precharge)
  863. writel(seq->rwcfg->precharge_all, grpaddr);
  864. /*
  865. * USER Use Mirror-ed commands for odd ranks if address
  866. * mirrorring is on
  867. */
  868. if ((seq->rwcfg->mem_address_mirroring >> r) & 0x1) {
  869. set_jump_as_return(seq);
  870. writel(seq->rwcfg->mrs2_mirr, grpaddr);
  871. delay_for_n_mem_clocks(seq, 4);
  872. set_jump_as_return(seq);
  873. writel(seq->rwcfg->mrs3_mirr, grpaddr);
  874. delay_for_n_mem_clocks(seq, 4);
  875. set_jump_as_return(seq);
  876. writel(seq->rwcfg->mrs1_mirr, grpaddr);
  877. delay_for_n_mem_clocks(seq, 4);
  878. set_jump_as_return(seq);
  879. writel(fin1, grpaddr);
  880. } else {
  881. set_jump_as_return(seq);
  882. writel(seq->rwcfg->mrs2, grpaddr);
  883. delay_for_n_mem_clocks(seq, 4);
  884. set_jump_as_return(seq);
  885. writel(seq->rwcfg->mrs3, grpaddr);
  886. delay_for_n_mem_clocks(seq, 4);
  887. set_jump_as_return(seq);
  888. writel(seq->rwcfg->mrs1, grpaddr);
  889. set_jump_as_return(seq);
  890. writel(fin2, grpaddr);
  891. }
  892. if (precharge)
  893. continue;
  894. set_jump_as_return(seq);
  895. writel(seq->rwcfg->zqcl, grpaddr);
  896. /* tZQinit = tDLLK = 512 ck cycles */
  897. delay_for_n_mem_clocks(seq, 512);
  898. }
  899. }
  900. /**
  901. * rw_mgr_mem_load_user() - Load user calibration values
  902. * @fin1: Final instruction 1
  903. * @fin2: Final instruction 2
  904. * @precharge: If 1, precharge the banks at the end
  905. *
  906. * Load user calibration values and optionally precharge the banks.
  907. */
  908. static void rw_mgr_mem_load_user(struct socfpga_sdrseq *seq,
  909. const u32 fin1, const u32 fin2,
  910. const int precharge)
  911. {
  912. if (dram_is_ddr(2))
  913. rw_mgr_mem_load_user_ddr2(seq, precharge);
  914. else if (dram_is_ddr(3))
  915. rw_mgr_mem_load_user_ddr3(seq, fin1, fin2, precharge);
  916. else
  917. hang();
  918. }
  919. /**
  920. * rw_mgr_mem_initialize() - Initialize RW Manager
  921. *
  922. * Initialize RW Manager.
  923. */
  924. static void rw_mgr_mem_initialize(struct socfpga_sdrseq *seq)
  925. {
  926. debug("%s:%d\n", __func__, __LINE__);
  927. /* The reset / cke part of initialization is broadcasted to all ranks */
  928. if (dram_is_ddr(3)) {
  929. writel(RW_MGR_RANK_ALL, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  930. RW_MGR_SET_CS_AND_ODT_MASK_OFFSET);
  931. }
  932. /*
  933. * Here's how you load register for a loop
  934. * Counters are located @ 0x800
  935. * Jump address are located @ 0xC00
  936. * For both, registers 0 to 3 are selected using bits 3 and 2, like
  937. * in 0x800, 0x804, 0x808, 0x80C and 0xC00, 0xC04, 0xC08, 0xC0C
  938. * I know this ain't pretty, but Avalon bus throws away the 2 least
  939. * significant bits
  940. */
  941. /* Start with memory RESET activated */
  942. /* tINIT = 200us */
  943. /*
  944. * 200us @ 266MHz (3.75 ns) ~ 54000 clock cycles
  945. * If a and b are the number of iteration in 2 nested loops
  946. * it takes the following number of cycles to complete the operation:
  947. * number_of_cycles = ((2 + n) * a + 2) * b
  948. * where n is the number of instruction in the inner loop
  949. * One possible solution is n = 0 , a = 256 , b = 106 => a = FF,
  950. * b = 6A
  951. */
  952. rw_mgr_mem_init_load_regs(seq, seq->misccfg->tinit_cntr0_val,
  953. seq->misccfg->tinit_cntr1_val,
  954. seq->misccfg->tinit_cntr2_val,
  955. seq->rwcfg->init_reset_0_cke_0);
  956. /* Indicate that memory is stable. */
  957. writel(1, &phy_mgr_cfg->reset_mem_stbl);
  958. if (dram_is_ddr(2)) {
  959. writel(seq->rwcfg->nop, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  960. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  961. /* Bring up clock enable. */
  962. /* tXRP < 400 ck cycles */
  963. delay_for_n_ns(seq, 400);
  964. } else if (dram_is_ddr(3)) {
  965. /*
  966. * transition the RESET to high
  967. * Wait for 500us
  968. */
  969. /*
  970. * 500us @ 266MHz (3.75 ns) ~ 134000 clock cycles
  971. * If a and b are the number of iteration in 2 nested loops
  972. * it takes the following number of cycles to complete the
  973. * operation number_of_cycles = ((2 + n) * a + 2) * b
  974. * where n is the number of instruction in the inner loop
  975. * One possible solution is
  976. * n = 2 , a = 131 , b = 256 => a = 83, b = FF
  977. */
  978. rw_mgr_mem_init_load_regs(seq, seq->misccfg->treset_cntr0_val,
  979. seq->misccfg->treset_cntr1_val,
  980. seq->misccfg->treset_cntr2_val,
  981. seq->rwcfg->init_reset_1_cke_0);
  982. /* Bring up clock enable. */
  983. /* tXRP < 250 ck cycles */
  984. delay_for_n_mem_clocks(seq, 250);
  985. }
  986. rw_mgr_mem_load_user(seq, seq->rwcfg->mrs0_dll_reset_mirr,
  987. seq->rwcfg->mrs0_dll_reset, 0);
  988. }
  989. /**
  990. * rw_mgr_mem_handoff() - Hand off the memory to user
  991. *
  992. * At the end of calibration we have to program the user settings in
  993. * and hand off the memory to the user.
  994. */
  995. static void rw_mgr_mem_handoff(struct socfpga_sdrseq *seq)
  996. {
  997. rw_mgr_mem_load_user(seq, seq->rwcfg->mrs0_user_mirr,
  998. seq->rwcfg->mrs0_user, 1);
  999. /*
  1000. * Need to wait tMOD (12CK or 15ns) time before issuing other
  1001. * commands, but we will have plenty of NIOS cycles before actual
  1002. * handoff so its okay.
  1003. */
  1004. }
  1005. /**
  1006. * rw_mgr_mem_calibrate_write_test_issue() - Issue write test command
  1007. * @group: Write Group
  1008. * @use_dm: Use DM
  1009. *
  1010. * Issue write test command. Two variants are provided, one that just tests
  1011. * a write pattern and another that tests datamask functionality.
  1012. */
  1013. static void rw_mgr_mem_calibrate_write_test_issue(struct socfpga_sdrseq *seq,
  1014. u32 group, u32 test_dm)
  1015. {
  1016. const u32 quick_write_mode =
  1017. (STATIC_CALIB_STEPS & CALIB_SKIP_WRITES) &&
  1018. seq->misccfg->enable_super_quick_calibration;
  1019. u32 mcc_instruction;
  1020. u32 rw_wl_nop_cycles;
  1021. /*
  1022. * Set counter and jump addresses for the right
  1023. * number of NOP cycles.
  1024. * The number of supported NOP cycles can range from -1 to infinity
  1025. * Three different cases are handled:
  1026. *
  1027. * 1. For a number of NOP cycles greater than 0, the RW Mgr looping
  1028. * mechanism will be used to insert the right number of NOPs
  1029. *
  1030. * 2. For a number of NOP cycles equals to 0, the micro-instruction
  1031. * issuing the write command will jump straight to the
  1032. * micro-instruction that turns on DQS (for DDRx), or outputs write
  1033. * data (for RLD), skipping
  1034. * the NOP micro-instruction all together
  1035. *
  1036. * 3. A number of NOP cycles equal to -1 indicates that DQS must be
  1037. * turned on in the same micro-instruction that issues the write
  1038. * command. Then we need
  1039. * to directly jump to the micro-instruction that sends out the data
  1040. *
  1041. * NOTE: Implementing this mechanism uses 2 RW Mgr jump-counters
  1042. * (2 and 3). One jump-counter (0) is used to perform multiple
  1043. * write-read operations.
  1044. * one counter left to issue this command in "multiple-group" mode
  1045. */
  1046. rw_wl_nop_cycles = seq->gbl.rw_wl_nop_cycles;
  1047. if (rw_wl_nop_cycles == -1) {
  1048. /*
  1049. * CNTR 2 - We want to execute the special write operation that
  1050. * turns on DQS right away and then skip directly to the
  1051. * instruction that sends out the data. We set the counter to a
  1052. * large number so that the jump is always taken.
  1053. */
  1054. writel(0xFF, &sdr_rw_load_mgr_regs->load_cntr2);
  1055. /* CNTR 3 - Not used */
  1056. if (test_dm) {
  1057. mcc_instruction = seq->rwcfg->lfsr_wr_rd_dm_bank_0_wl_1;
  1058. writel(seq->rwcfg->lfsr_wr_rd_dm_bank_0_data,
  1059. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  1060. writel(seq->rwcfg->lfsr_wr_rd_dm_bank_0_nop,
  1061. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  1062. } else {
  1063. mcc_instruction = seq->rwcfg->lfsr_wr_rd_bank_0_wl_1;
  1064. writel(seq->rwcfg->lfsr_wr_rd_bank_0_data,
  1065. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  1066. writel(seq->rwcfg->lfsr_wr_rd_bank_0_nop,
  1067. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  1068. }
  1069. } else if (rw_wl_nop_cycles == 0) {
  1070. /*
  1071. * CNTR 2 - We want to skip the NOP operation and go straight
  1072. * to the DQS enable instruction. We set the counter to a large
  1073. * number so that the jump is always taken.
  1074. */
  1075. writel(0xFF, &sdr_rw_load_mgr_regs->load_cntr2);
  1076. /* CNTR 3 - Not used */
  1077. if (test_dm) {
  1078. mcc_instruction = seq->rwcfg->lfsr_wr_rd_dm_bank_0;
  1079. writel(seq->rwcfg->lfsr_wr_rd_dm_bank_0_dqs,
  1080. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  1081. } else {
  1082. mcc_instruction = seq->rwcfg->lfsr_wr_rd_bank_0;
  1083. writel(seq->rwcfg->lfsr_wr_rd_bank_0_dqs,
  1084. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  1085. }
  1086. } else {
  1087. /*
  1088. * CNTR 2 - In this case we want to execute the next instruction
  1089. * and NOT take the jump. So we set the counter to 0. The jump
  1090. * address doesn't count.
  1091. */
  1092. writel(0x0, &sdr_rw_load_mgr_regs->load_cntr2);
  1093. writel(0x0, &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  1094. /*
  1095. * CNTR 3 - Set the nop counter to the number of cycles we
  1096. * need to loop for, minus 1.
  1097. */
  1098. writel(rw_wl_nop_cycles - 1, &sdr_rw_load_mgr_regs->load_cntr3);
  1099. if (test_dm) {
  1100. mcc_instruction = seq->rwcfg->lfsr_wr_rd_dm_bank_0;
  1101. writel(seq->rwcfg->lfsr_wr_rd_dm_bank_0_nop,
  1102. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  1103. } else {
  1104. mcc_instruction = seq->rwcfg->lfsr_wr_rd_bank_0;
  1105. writel(seq->rwcfg->lfsr_wr_rd_bank_0_nop,
  1106. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  1107. }
  1108. }
  1109. writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  1110. RW_MGR_RESET_READ_DATAPATH_OFFSET);
  1111. if (quick_write_mode)
  1112. writel(0x08, &sdr_rw_load_mgr_regs->load_cntr0);
  1113. else
  1114. writel(0x40, &sdr_rw_load_mgr_regs->load_cntr0);
  1115. writel(mcc_instruction, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  1116. /*
  1117. * CNTR 1 - This is used to ensure enough time elapses
  1118. * for read data to come back.
  1119. */
  1120. writel(0x30, &sdr_rw_load_mgr_regs->load_cntr1);
  1121. if (test_dm) {
  1122. writel(seq->rwcfg->lfsr_wr_rd_dm_bank_0_wait,
  1123. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  1124. } else {
  1125. writel(seq->rwcfg->lfsr_wr_rd_bank_0_wait,
  1126. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  1127. }
  1128. writel(mcc_instruction, (SDR_PHYGRP_RWMGRGRP_ADDRESS |
  1129. RW_MGR_RUN_SINGLE_GROUP_OFFSET) +
  1130. (group << 2));
  1131. }
  1132. /**
  1133. * rw_mgr_mem_calibrate_write_test() - Test writes, check for single/multiple
  1134. * pass
  1135. * @rank_bgn: Rank number
  1136. * @write_group: Write Group
  1137. * @use_dm: Use DM
  1138. * @all_correct: All bits must be correct in the mask
  1139. * @bit_chk: Resulting bit mask after the test
  1140. * @all_ranks: Test all ranks
  1141. *
  1142. * Test writes, can check for a single bit pass or multiple bit pass.
  1143. */
  1144. static int
  1145. rw_mgr_mem_calibrate_write_test(struct socfpga_sdrseq *seq,
  1146. const u32 rank_bgn, const u32 write_group,
  1147. const u32 use_dm, const u32 all_correct,
  1148. u32 *bit_chk, const u32 all_ranks)
  1149. {
  1150. const u32 rank_end = all_ranks ?
  1151. seq->rwcfg->mem_number_of_ranks :
  1152. (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
  1153. const u32 shift_ratio = seq->rwcfg->mem_dq_per_write_dqs /
  1154. seq->rwcfg->mem_virtual_groups_per_write_dqs;
  1155. const u32 correct_mask_vg = seq->param.write_correct_mask_vg;
  1156. u32 tmp_bit_chk, base_rw_mgr, group;
  1157. int vg, r;
  1158. *bit_chk = seq->param.write_correct_mask;
  1159. for (r = rank_bgn; r < rank_end; r++) {
  1160. /* Set rank */
  1161. set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_READ_WRITE);
  1162. tmp_bit_chk = 0;
  1163. for (vg = seq->rwcfg->mem_virtual_groups_per_write_dqs - 1;
  1164. vg >= 0; vg--) {
  1165. /* Reset the FIFOs to get pointers to known state. */
  1166. writel(0, &phy_mgr_cmd->fifo_reset);
  1167. group = write_group *
  1168. seq->rwcfg->mem_virtual_groups_per_write_dqs
  1169. + vg;
  1170. rw_mgr_mem_calibrate_write_test_issue(seq, group,
  1171. use_dm);
  1172. base_rw_mgr = readl(SDR_PHYGRP_RWMGRGRP_ADDRESS);
  1173. tmp_bit_chk <<= shift_ratio;
  1174. tmp_bit_chk |= (correct_mask_vg & ~(base_rw_mgr));
  1175. }
  1176. *bit_chk &= tmp_bit_chk;
  1177. }
  1178. set_rank_and_odt_mask(seq, 0, RW_MGR_ODT_MODE_OFF);
  1179. if (all_correct) {
  1180. debug_cond(DLEVEL >= 2,
  1181. "write_test(%u,%u,ALL) : %u == %u => %i\n",
  1182. write_group, use_dm, *bit_chk,
  1183. seq->param.write_correct_mask,
  1184. *bit_chk == seq->param.write_correct_mask);
  1185. return *bit_chk == seq->param.write_correct_mask;
  1186. } else {
  1187. debug_cond(DLEVEL >= 2,
  1188. "write_test(%u,%u,ONE) : %u != %i => %i\n",
  1189. write_group, use_dm, *bit_chk, 0, *bit_chk != 0);
  1190. return *bit_chk != 0x00;
  1191. }
  1192. }
  1193. /**
  1194. * rw_mgr_mem_calibrate_read_test_patterns() - Read back test patterns
  1195. * @rank_bgn: Rank number
  1196. * @group: Read/Write Group
  1197. * @all_ranks: Test all ranks
  1198. *
  1199. * Performs a guaranteed read on the patterns we are going to use during a
  1200. * read test to ensure memory works.
  1201. */
  1202. static int
  1203. rw_mgr_mem_calibrate_read_test_patterns(struct socfpga_sdrseq *seq,
  1204. const u32 rank_bgn, const u32 group,
  1205. const u32 all_ranks)
  1206. {
  1207. const u32 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
  1208. RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  1209. const u32 addr_offset =
  1210. (group * seq->rwcfg->mem_virtual_groups_per_read_dqs)
  1211. << 2;
  1212. const u32 rank_end = all_ranks ?
  1213. seq->rwcfg->mem_number_of_ranks :
  1214. (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
  1215. const u32 shift_ratio = seq->rwcfg->mem_dq_per_read_dqs /
  1216. seq->rwcfg->mem_virtual_groups_per_read_dqs;
  1217. const u32 correct_mask_vg = seq->param.read_correct_mask_vg;
  1218. u32 tmp_bit_chk, base_rw_mgr, bit_chk;
  1219. int vg, r;
  1220. int ret = 0;
  1221. bit_chk = seq->param.read_correct_mask;
  1222. for (r = rank_bgn; r < rank_end; r++) {
  1223. /* Set rank */
  1224. set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_READ_WRITE);
  1225. /* Load up a constant bursts of read commands */
  1226. writel(0x20, &sdr_rw_load_mgr_regs->load_cntr0);
  1227. writel(seq->rwcfg->guaranteed_read,
  1228. &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  1229. writel(0x20, &sdr_rw_load_mgr_regs->load_cntr1);
  1230. writel(seq->rwcfg->guaranteed_read_cont,
  1231. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  1232. tmp_bit_chk = 0;
  1233. for (vg = seq->rwcfg->mem_virtual_groups_per_read_dqs - 1;
  1234. vg >= 0; vg--) {
  1235. /* Reset the FIFOs to get pointers to known state. */
  1236. writel(0, &phy_mgr_cmd->fifo_reset);
  1237. writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  1238. RW_MGR_RESET_READ_DATAPATH_OFFSET);
  1239. writel(seq->rwcfg->guaranteed_read,
  1240. addr + addr_offset + (vg << 2));
  1241. base_rw_mgr = readl(SDR_PHYGRP_RWMGRGRP_ADDRESS);
  1242. tmp_bit_chk <<= shift_ratio;
  1243. tmp_bit_chk |= correct_mask_vg & ~base_rw_mgr;
  1244. }
  1245. bit_chk &= tmp_bit_chk;
  1246. }
  1247. writel(seq->rwcfg->clear_dqs_enable, addr + (group << 2));
  1248. set_rank_and_odt_mask(seq, 0, RW_MGR_ODT_MODE_OFF);
  1249. if (bit_chk != seq->param.read_correct_mask)
  1250. ret = -EIO;
  1251. debug_cond(DLEVEL >= 1,
  1252. "%s:%d test_load_patterns(%u,ALL) => (%u == %u) => %i\n",
  1253. __func__, __LINE__, group, bit_chk,
  1254. seq->param.read_correct_mask, ret);
  1255. return ret;
  1256. }
  1257. /**
  1258. * rw_mgr_mem_calibrate_read_load_patterns() - Load up the patterns for read
  1259. * test
  1260. * @rank_bgn: Rank number
  1261. * @all_ranks: Test all ranks
  1262. *
  1263. * Load up the patterns we are going to use during a read test.
  1264. */
  1265. static void rw_mgr_mem_calibrate_read_load_patterns(struct socfpga_sdrseq *seq,
  1266. const u32 rank_bgn,
  1267. const int all_ranks)
  1268. {
  1269. const u32 rank_end = all_ranks ?
  1270. seq->rwcfg->mem_number_of_ranks :
  1271. (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
  1272. u32 r;
  1273. debug("%s:%d\n", __func__, __LINE__);
  1274. for (r = rank_bgn; r < rank_end; r++) {
  1275. /* set rank */
  1276. set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_READ_WRITE);
  1277. /* Load up a constant bursts */
  1278. writel(0x20, &sdr_rw_load_mgr_regs->load_cntr0);
  1279. writel(seq->rwcfg->guaranteed_write_wait0,
  1280. &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  1281. writel(0x20, &sdr_rw_load_mgr_regs->load_cntr1);
  1282. writel(seq->rwcfg->guaranteed_write_wait1,
  1283. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  1284. writel(0x04, &sdr_rw_load_mgr_regs->load_cntr2);
  1285. writel(seq->rwcfg->guaranteed_write_wait2,
  1286. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  1287. writel(0x04, &sdr_rw_load_mgr_regs->load_cntr3);
  1288. writel(seq->rwcfg->guaranteed_write_wait3,
  1289. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  1290. writel(seq->rwcfg->guaranteed_write,
  1291. SDR_PHYGRP_RWMGRGRP_ADDRESS |
  1292. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  1293. }
  1294. set_rank_and_odt_mask(seq, 0, RW_MGR_ODT_MODE_OFF);
  1295. }
  1296. /**
  1297. * rw_mgr_mem_calibrate_read_test() - Perform READ test on single rank
  1298. * @rank_bgn: Rank number
  1299. * @group: Read/Write group
  1300. * @num_tries: Number of retries of the test
  1301. * @all_correct: All bits must be correct in the mask
  1302. * @bit_chk: Resulting bit mask after the test
  1303. * @all_groups: Test all R/W groups
  1304. * @all_ranks: Test all ranks
  1305. *
  1306. * Try a read and see if it returns correct data back. Test has dummy reads
  1307. * inserted into the mix used to align DQS enable. Test has more thorough
  1308. * checks than the regular read test.
  1309. */
  1310. static int
  1311. rw_mgr_mem_calibrate_read_test(struct socfpga_sdrseq *seq,
  1312. const u32 rank_bgn, const u32 group,
  1313. const u32 num_tries, const u32 all_correct,
  1314. u32 *bit_chk,
  1315. const u32 all_groups, const u32 all_ranks)
  1316. {
  1317. const u32 rank_end = all_ranks ? seq->rwcfg->mem_number_of_ranks :
  1318. (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
  1319. const u32 quick_read_mode =
  1320. ((STATIC_CALIB_STEPS & CALIB_SKIP_DELAY_SWEEPS) &&
  1321. seq->misccfg->enable_super_quick_calibration);
  1322. u32 correct_mask_vg = seq->param.read_correct_mask_vg;
  1323. u32 tmp_bit_chk;
  1324. u32 base_rw_mgr;
  1325. u32 addr;
  1326. int r, vg, ret;
  1327. *bit_chk = seq->param.read_correct_mask;
  1328. for (r = rank_bgn; r < rank_end; r++) {
  1329. /* set rank */
  1330. set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_READ_WRITE);
  1331. writel(0x10, &sdr_rw_load_mgr_regs->load_cntr1);
  1332. writel(seq->rwcfg->read_b2b_wait1,
  1333. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  1334. writel(0x10, &sdr_rw_load_mgr_regs->load_cntr2);
  1335. writel(seq->rwcfg->read_b2b_wait2,
  1336. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  1337. if (quick_read_mode)
  1338. writel(0x1, &sdr_rw_load_mgr_regs->load_cntr0);
  1339. /* need at least two (1+1) reads to capture failures */
  1340. else if (all_groups)
  1341. writel(0x06, &sdr_rw_load_mgr_regs->load_cntr0);
  1342. else
  1343. writel(0x32, &sdr_rw_load_mgr_regs->load_cntr0);
  1344. writel(seq->rwcfg->read_b2b,
  1345. &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  1346. if (all_groups)
  1347. writel(seq->rwcfg->mem_if_read_dqs_width *
  1348. seq->rwcfg->mem_virtual_groups_per_read_dqs - 1,
  1349. &sdr_rw_load_mgr_regs->load_cntr3);
  1350. else
  1351. writel(0x0, &sdr_rw_load_mgr_regs->load_cntr3);
  1352. writel(seq->rwcfg->read_b2b,
  1353. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  1354. tmp_bit_chk = 0;
  1355. for (vg = seq->rwcfg->mem_virtual_groups_per_read_dqs - 1;
  1356. vg >= 0; vg--) {
  1357. /* Reset the FIFOs to get pointers to known state. */
  1358. writel(0, &phy_mgr_cmd->fifo_reset);
  1359. writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  1360. RW_MGR_RESET_READ_DATAPATH_OFFSET);
  1361. if (all_groups) {
  1362. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
  1363. RW_MGR_RUN_ALL_GROUPS_OFFSET;
  1364. } else {
  1365. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
  1366. RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  1367. }
  1368. writel(seq->rwcfg->read_b2b, addr +
  1369. ((group *
  1370. seq->rwcfg->mem_virtual_groups_per_read_dqs +
  1371. vg) << 2));
  1372. base_rw_mgr = readl(SDR_PHYGRP_RWMGRGRP_ADDRESS);
  1373. tmp_bit_chk <<=
  1374. seq->rwcfg->mem_dq_per_read_dqs /
  1375. seq->rwcfg->mem_virtual_groups_per_read_dqs;
  1376. tmp_bit_chk |= correct_mask_vg & ~(base_rw_mgr);
  1377. }
  1378. *bit_chk &= tmp_bit_chk;
  1379. }
  1380. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  1381. writel(seq->rwcfg->clear_dqs_enable, addr + (group << 2));
  1382. set_rank_and_odt_mask(seq, 0, RW_MGR_ODT_MODE_OFF);
  1383. if (all_correct) {
  1384. ret = (*bit_chk == seq->param.read_correct_mask);
  1385. debug_cond(DLEVEL >= 2,
  1386. "%s:%d read_test(%u,ALL,%u) => (%u == %u) => %i\n",
  1387. __func__, __LINE__, group, all_groups, *bit_chk,
  1388. seq->param.read_correct_mask, ret);
  1389. } else {
  1390. ret = (*bit_chk != 0x00);
  1391. debug_cond(DLEVEL >= 2,
  1392. "%s:%d read_test(%u,ONE,%u) => (%u != %u) => %i\n",
  1393. __func__, __LINE__, group, all_groups, *bit_chk,
  1394. 0, ret);
  1395. }
  1396. return ret;
  1397. }
  1398. /**
  1399. * rw_mgr_mem_calibrate_read_test_all_ranks() - Perform READ test on all ranks
  1400. * @grp: Read/Write group
  1401. * @num_tries: Number of retries of the test
  1402. * @all_correct: All bits must be correct in the mask
  1403. * @all_groups: Test all R/W groups
  1404. *
  1405. * Perform a READ test across all memory ranks.
  1406. */
  1407. static int
  1408. rw_mgr_mem_calibrate_read_test_all_ranks(struct socfpga_sdrseq *seq,
  1409. const u32 grp, const u32 num_tries,
  1410. const u32 all_correct,
  1411. const u32 all_groups)
  1412. {
  1413. u32 bit_chk;
  1414. return rw_mgr_mem_calibrate_read_test(seq, 0, grp, num_tries,
  1415. all_correct, &bit_chk, all_groups,
  1416. 1);
  1417. }
  1418. /**
  1419. * rw_mgr_incr_vfifo() - Increase VFIFO value
  1420. * @grp: Read/Write group
  1421. *
  1422. * Increase VFIFO value.
  1423. */
  1424. static void rw_mgr_incr_vfifo(const u32 grp)
  1425. {
  1426. writel(grp, &phy_mgr_cmd->inc_vfifo_hard_phy);
  1427. }
  1428. /**
  1429. * rw_mgr_decr_vfifo() - Decrease VFIFO value
  1430. * @grp: Read/Write group
  1431. *
  1432. * Decrease VFIFO value.
  1433. */
  1434. static void rw_mgr_decr_vfifo(struct socfpga_sdrseq *seq, const u32 grp)
  1435. {
  1436. u32 i;
  1437. for (i = 0; i < seq->misccfg->read_valid_fifo_size - 1; i++)
  1438. rw_mgr_incr_vfifo(grp);
  1439. }
  1440. /**
  1441. * find_vfifo_failing_read() - Push VFIFO to get a failing read
  1442. * @grp: Read/Write group
  1443. *
  1444. * Push VFIFO until a failing read happens.
  1445. */
  1446. static int find_vfifo_failing_read(struct socfpga_sdrseq *seq,
  1447. const u32 grp)
  1448. {
  1449. u32 v, ret, fail_cnt = 0;
  1450. for (v = 0; v < seq->misccfg->read_valid_fifo_size; v++) {
  1451. debug_cond(DLEVEL >= 2, "%s:%d: vfifo %u\n",
  1452. __func__, __LINE__, v);
  1453. ret = rw_mgr_mem_calibrate_read_test_all_ranks(seq, grp, 1,
  1454. PASS_ONE_BIT, 0);
  1455. if (!ret) {
  1456. fail_cnt++;
  1457. if (fail_cnt == 2)
  1458. return v;
  1459. }
  1460. /* Fiddle with FIFO. */
  1461. rw_mgr_incr_vfifo(grp);
  1462. }
  1463. /* No failing read found! Something must have gone wrong. */
  1464. debug_cond(DLEVEL >= 2, "%s:%d: vfifo failed\n", __func__, __LINE__);
  1465. return 0;
  1466. }
  1467. /**
  1468. * sdr_find_phase_delay() - Find DQS enable phase or delay
  1469. * @working: If 1, look for working phase/delay, if 0, look for non-working
  1470. * @delay: If 1, look for delay, if 0, look for phase
  1471. * @grp: Read/Write group
  1472. * @work: Working window position
  1473. * @work_inc: Working window increment
  1474. * @pd: DQS Phase/Delay Iterator
  1475. *
  1476. * Find working or non-working DQS enable phase setting.
  1477. */
  1478. static int sdr_find_phase_delay(struct socfpga_sdrseq *seq, int working,
  1479. int delay, const u32 grp, u32 *work,
  1480. const u32 work_inc, u32 *pd)
  1481. {
  1482. const u32 max = delay ? seq->iocfg->dqs_en_delay_max :
  1483. seq->iocfg->dqs_en_phase_max;
  1484. u32 ret;
  1485. for (; *pd <= max; (*pd)++) {
  1486. if (delay)
  1487. scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, *pd);
  1488. else
  1489. scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, *pd);
  1490. ret = rw_mgr_mem_calibrate_read_test_all_ranks(seq, grp, 1,
  1491. PASS_ONE_BIT, 0);
  1492. if (!working)
  1493. ret = !ret;
  1494. if (ret)
  1495. return 0;
  1496. if (work)
  1497. *work += work_inc;
  1498. }
  1499. return -EINVAL;
  1500. }
  1501. /**
  1502. * sdr_find_phase() - Find DQS enable phase
  1503. * @working: If 1, look for working phase, if 0, look for non-working phase
  1504. * @grp: Read/Write group
  1505. * @work: Working window position
  1506. * @i: Iterator
  1507. * @p: DQS Phase Iterator
  1508. *
  1509. * Find working or non-working DQS enable phase setting.
  1510. */
  1511. static int sdr_find_phase(struct socfpga_sdrseq *seq, int working,
  1512. const u32 grp, u32 *work, u32 *i, u32 *p)
  1513. {
  1514. const u32 end = seq->misccfg->read_valid_fifo_size + (working ? 0 : 1);
  1515. int ret;
  1516. for (; *i < end; (*i)++) {
  1517. if (working)
  1518. *p = 0;
  1519. ret = sdr_find_phase_delay(seq, working, 0, grp, work,
  1520. seq->iocfg->delay_per_opa_tap, p);
  1521. if (!ret)
  1522. return 0;
  1523. if (*p > seq->iocfg->dqs_en_phase_max) {
  1524. /* Fiddle with FIFO. */
  1525. rw_mgr_incr_vfifo(grp);
  1526. if (!working)
  1527. *p = 0;
  1528. }
  1529. }
  1530. return -EINVAL;
  1531. }
  1532. /**
  1533. * sdr_working_phase() - Find working DQS enable phase
  1534. * @grp: Read/Write group
  1535. * @work_bgn: Working window start position
  1536. * @d: dtaps output value
  1537. * @p: DQS Phase Iterator
  1538. * @i: Iterator
  1539. *
  1540. * Find working DQS enable phase setting.
  1541. */
  1542. static int sdr_working_phase(struct socfpga_sdrseq *seq, const u32 grp,
  1543. u32 *work_bgn, u32 *d, u32 *p, u32 *i)
  1544. {
  1545. const u32 dtaps_per_ptap = seq->iocfg->delay_per_opa_tap /
  1546. seq->iocfg->delay_per_dqs_en_dchain_tap;
  1547. int ret;
  1548. *work_bgn = 0;
  1549. for (*d = 0; *d <= dtaps_per_ptap; (*d)++) {
  1550. *i = 0;
  1551. scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, *d);
  1552. ret = sdr_find_phase(seq, 1, grp, work_bgn, i, p);
  1553. if (!ret)
  1554. return 0;
  1555. *work_bgn += seq->iocfg->delay_per_dqs_en_dchain_tap;
  1556. }
  1557. /* Cannot find working solution */
  1558. debug_cond(DLEVEL >= 2, "%s:%d find_dqs_en_phase: no vfifo/ptap/dtap\n",
  1559. __func__, __LINE__);
  1560. return -EINVAL;
  1561. }
  1562. /**
  1563. * sdr_backup_phase() - Find DQS enable backup phase
  1564. * @grp: Read/Write group
  1565. * @work_bgn: Working window start position
  1566. * @p: DQS Phase Iterator
  1567. *
  1568. * Find DQS enable backup phase setting.
  1569. */
  1570. static void sdr_backup_phase(struct socfpga_sdrseq *seq, const u32 grp,
  1571. u32 *work_bgn, u32 *p)
  1572. {
  1573. u32 tmp_delay, d;
  1574. int ret;
  1575. /* Special case code for backing up a phase */
  1576. if (*p == 0) {
  1577. *p = seq->iocfg->dqs_en_phase_max;
  1578. rw_mgr_decr_vfifo(seq, grp);
  1579. } else {
  1580. (*p)--;
  1581. }
  1582. tmp_delay = *work_bgn - seq->iocfg->delay_per_opa_tap;
  1583. scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, *p);
  1584. for (d = 0; d <= seq->iocfg->dqs_en_delay_max && tmp_delay < *work_bgn;
  1585. d++) {
  1586. scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, d);
  1587. ret = rw_mgr_mem_calibrate_read_test_all_ranks(seq, grp, 1,
  1588. PASS_ONE_BIT, 0);
  1589. if (ret) {
  1590. *work_bgn = tmp_delay;
  1591. break;
  1592. }
  1593. tmp_delay += seq->iocfg->delay_per_dqs_en_dchain_tap;
  1594. }
  1595. /* Restore VFIFO to old state before we decremented it (if needed). */
  1596. (*p)++;
  1597. if (*p > seq->iocfg->dqs_en_phase_max) {
  1598. *p = 0;
  1599. rw_mgr_incr_vfifo(grp);
  1600. }
  1601. scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, 0);
  1602. }
  1603. /**
  1604. * sdr_nonworking_phase() - Find non-working DQS enable phase
  1605. * @grp: Read/Write group
  1606. * @work_end: Working window end position
  1607. * @p: DQS Phase Iterator
  1608. * @i: Iterator
  1609. *
  1610. * Find non-working DQS enable phase setting.
  1611. */
  1612. static int sdr_nonworking_phase(struct socfpga_sdrseq *seq,
  1613. const u32 grp, u32 *work_end, u32 *p, u32 *i)
  1614. {
  1615. int ret;
  1616. (*p)++;
  1617. *work_end += seq->iocfg->delay_per_opa_tap;
  1618. if (*p > seq->iocfg->dqs_en_phase_max) {
  1619. /* Fiddle with FIFO. */
  1620. *p = 0;
  1621. rw_mgr_incr_vfifo(grp);
  1622. }
  1623. ret = sdr_find_phase(seq, 0, grp, work_end, i, p);
  1624. if (ret) {
  1625. /* Cannot see edge of failing read. */
  1626. debug_cond(DLEVEL >= 2, "%s:%d: end: failed\n",
  1627. __func__, __LINE__);
  1628. }
  1629. return ret;
  1630. }
  1631. /**
  1632. * sdr_find_window_center() - Find center of the working DQS window.
  1633. * @grp: Read/Write group
  1634. * @work_bgn: First working settings
  1635. * @work_end: Last working settings
  1636. *
  1637. * Find center of the working DQS enable window.
  1638. */
  1639. static int sdr_find_window_center(struct socfpga_sdrseq *seq,
  1640. const u32 grp, const u32 work_bgn,
  1641. const u32 work_end)
  1642. {
  1643. u32 work_mid;
  1644. int tmp_delay = 0;
  1645. int i, p, d;
  1646. work_mid = (work_bgn + work_end) / 2;
  1647. debug_cond(DLEVEL >= 2, "work_bgn=%d work_end=%d work_mid=%d\n",
  1648. work_bgn, work_end, work_mid);
  1649. /* Get the middle delay to be less than a VFIFO delay */
  1650. tmp_delay = (seq->iocfg->dqs_en_phase_max + 1)
  1651. * seq->iocfg->delay_per_opa_tap;
  1652. debug_cond(DLEVEL >= 2, "vfifo ptap delay %d\n", tmp_delay);
  1653. work_mid %= tmp_delay;
  1654. debug_cond(DLEVEL >= 2, "new work_mid %d\n", work_mid);
  1655. tmp_delay = rounddown(work_mid, seq->iocfg->delay_per_opa_tap);
  1656. if (tmp_delay > seq->iocfg->dqs_en_phase_max
  1657. * seq->iocfg->delay_per_opa_tap) {
  1658. tmp_delay = seq->iocfg->dqs_en_phase_max
  1659. * seq->iocfg->delay_per_opa_tap;
  1660. }
  1661. p = tmp_delay / seq->iocfg->delay_per_opa_tap;
  1662. debug_cond(DLEVEL >= 2, "new p %d, tmp_delay=%d\n", p, tmp_delay);
  1663. d = DIV_ROUND_UP(work_mid - tmp_delay,
  1664. seq->iocfg->delay_per_dqs_en_dchain_tap);
  1665. if (d > seq->iocfg->dqs_en_delay_max)
  1666. d = seq->iocfg->dqs_en_delay_max;
  1667. tmp_delay += d * seq->iocfg->delay_per_dqs_en_dchain_tap;
  1668. debug_cond(DLEVEL >= 2, "new d %d, tmp_delay=%d\n", d, tmp_delay);
  1669. scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, p);
  1670. scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, d);
  1671. /*
  1672. * push vfifo until we can successfully calibrate. We can do this
  1673. * because the largest possible margin in 1 VFIFO cycle.
  1674. */
  1675. for (i = 0; i < seq->misccfg->read_valid_fifo_size; i++) {
  1676. debug_cond(DLEVEL >= 2, "find_dqs_en_phase: center\n");
  1677. if (rw_mgr_mem_calibrate_read_test_all_ranks(seq, grp, 1,
  1678. PASS_ONE_BIT,
  1679. 0)) {
  1680. debug_cond(DLEVEL >= 2,
  1681. "%s:%d center: found: ptap=%u dtap=%u\n",
  1682. __func__, __LINE__, p, d);
  1683. return 0;
  1684. }
  1685. /* Fiddle with FIFO. */
  1686. rw_mgr_incr_vfifo(grp);
  1687. }
  1688. debug_cond(DLEVEL >= 2, "%s:%d center: failed.\n",
  1689. __func__, __LINE__);
  1690. return -EINVAL;
  1691. }
  1692. /**
  1693. * rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase() - Find a good DQS enable to
  1694. * use
  1695. * @grp: Read/Write Group
  1696. *
  1697. * Find a good DQS enable to use.
  1698. */
  1699. static int
  1700. rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(struct socfpga_sdrseq *seq,
  1701. const u32 grp)
  1702. {
  1703. u32 d, p, i;
  1704. u32 dtaps_per_ptap;
  1705. u32 work_bgn, work_end;
  1706. u32 found_passing_read, found_failing_read = 0, initial_failing_dtap;
  1707. int ret;
  1708. debug("%s:%d %u\n", __func__, __LINE__, grp);
  1709. reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
  1710. scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, 0);
  1711. scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, 0);
  1712. /* Step 0: Determine number of delay taps for each phase tap. */
  1713. dtaps_per_ptap = seq->iocfg->delay_per_opa_tap /
  1714. seq->iocfg->delay_per_dqs_en_dchain_tap;
  1715. /* Step 1: First push vfifo until we get a failing read. */
  1716. find_vfifo_failing_read(seq, grp);
  1717. /* Step 2: Find first working phase, increment in ptaps. */
  1718. work_bgn = 0;
  1719. ret = sdr_working_phase(seq, grp, &work_bgn, &d, &p, &i);
  1720. if (ret)
  1721. return ret;
  1722. work_end = work_bgn;
  1723. /*
  1724. * If d is 0 then the working window covers a phase tap and we can
  1725. * follow the old procedure. Otherwise, we've found the beginning
  1726. * and we need to increment the dtaps until we find the end.
  1727. */
  1728. if (d == 0) {
  1729. /*
  1730. * Step 3a: If we have room, back off by one and
  1731. * increment in dtaps.
  1732. */
  1733. sdr_backup_phase(seq, grp, &work_bgn, &p);
  1734. /*
  1735. * Step 4a: go forward from working phase to non working
  1736. * phase, increment in ptaps.
  1737. */
  1738. ret = sdr_nonworking_phase(seq, grp, &work_end, &p, &i);
  1739. if (ret)
  1740. return ret;
  1741. /* Step 5a: Back off one from last, increment in dtaps. */
  1742. /* Special case code for backing up a phase */
  1743. if (p == 0) {
  1744. p = seq->iocfg->dqs_en_phase_max;
  1745. rw_mgr_decr_vfifo(seq, grp);
  1746. } else {
  1747. p = p - 1;
  1748. }
  1749. work_end -= seq->iocfg->delay_per_opa_tap;
  1750. scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, p);
  1751. d = 0;
  1752. debug_cond(DLEVEL >= 2, "%s:%d p: ptap=%u\n",
  1753. __func__, __LINE__, p);
  1754. }
  1755. /* The dtap increment to find the failing edge is done here. */
  1756. sdr_find_phase_delay(seq, 0, 1, grp, &work_end,
  1757. seq->iocfg->delay_per_dqs_en_dchain_tap, &d);
  1758. /* Go back to working dtap */
  1759. if (d != 0)
  1760. work_end -= seq->iocfg->delay_per_dqs_en_dchain_tap;
  1761. debug_cond(DLEVEL >= 2,
  1762. "%s:%d p/d: ptap=%u dtap=%u end=%u\n",
  1763. __func__, __LINE__, p, d - 1, work_end);
  1764. if (work_end < work_bgn) {
  1765. /* nil range */
  1766. debug_cond(DLEVEL >= 2, "%s:%d end-2: failed\n",
  1767. __func__, __LINE__);
  1768. return -EINVAL;
  1769. }
  1770. debug_cond(DLEVEL >= 2, "%s:%d found range [%u,%u]\n",
  1771. __func__, __LINE__, work_bgn, work_end);
  1772. /*
  1773. * We need to calculate the number of dtaps that equal a ptap.
  1774. * To do that we'll back up a ptap and re-find the edge of the
  1775. * window using dtaps
  1776. */
  1777. debug_cond(DLEVEL >= 2, "%s:%d calculate dtaps_per_ptap for tracking\n",
  1778. __func__, __LINE__);
  1779. /* Special case code for backing up a phase */
  1780. if (p == 0) {
  1781. p = seq->iocfg->dqs_en_phase_max;
  1782. rw_mgr_decr_vfifo(seq, grp);
  1783. debug_cond(DLEVEL >= 2, "%s:%d backedup cycle/phase: p=%u\n",
  1784. __func__, __LINE__, p);
  1785. } else {
  1786. p = p - 1;
  1787. debug_cond(DLEVEL >= 2, "%s:%d backedup phase only: p=%u",
  1788. __func__, __LINE__, p);
  1789. }
  1790. scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, p);
  1791. /*
  1792. * Increase dtap until we first see a passing read (in case the
  1793. * window is smaller than a ptap), and then a failing read to
  1794. * mark the edge of the window again.
  1795. */
  1796. /* Find a passing read. */
  1797. debug_cond(DLEVEL >= 2, "%s:%d find passing read\n",
  1798. __func__, __LINE__);
  1799. initial_failing_dtap = d;
  1800. found_passing_read = !sdr_find_phase_delay(seq, 1, 1, grp, NULL, 0, &d);
  1801. if (found_passing_read) {
  1802. /* Find a failing read. */
  1803. debug_cond(DLEVEL >= 2, "%s:%d find failing read\n",
  1804. __func__, __LINE__);
  1805. d++;
  1806. found_failing_read = !sdr_find_phase_delay(seq, 0, 1, grp, NULL,
  1807. 0, &d);
  1808. } else {
  1809. debug_cond(DLEVEL >= 1,
  1810. "%s:%d failed to calculate dtaps per ptap. Fall back on static value\n",
  1811. __func__, __LINE__);
  1812. }
  1813. /*
  1814. * The dynamically calculated dtaps_per_ptap is only valid if we
  1815. * found a passing/failing read. If we didn't, it means d hit the max
  1816. * (seq->iocfg->dqs_en_delay_max). Otherwise, dtaps_per_ptap retains its
  1817. * statically calculated value.
  1818. */
  1819. if (found_passing_read && found_failing_read)
  1820. dtaps_per_ptap = d - initial_failing_dtap;
  1821. writel(dtaps_per_ptap, &sdr_reg_file->dtaps_per_ptap);
  1822. debug_cond(DLEVEL >= 2, "%s:%d dtaps_per_ptap=%u - %u = %u",
  1823. __func__, __LINE__, d, initial_failing_dtap, dtaps_per_ptap);
  1824. /* Step 6: Find the centre of the window. */
  1825. ret = sdr_find_window_center(seq, grp, work_bgn, work_end);
  1826. return ret;
  1827. }
  1828. /**
  1829. * search_stop_check() - Check if the detected edge is valid
  1830. * @write: Perform read (Stage 2) or write (Stage 3) calibration
  1831. * @d: DQS delay
  1832. * @rank_bgn: Rank number
  1833. * @write_group: Write Group
  1834. * @read_group: Read Group
  1835. * @bit_chk: Resulting bit mask after the test
  1836. * @sticky_bit_chk: Resulting sticky bit mask after the test
  1837. * @use_read_test: Perform read test
  1838. *
  1839. * Test if the found edge is valid.
  1840. */
  1841. static u32 search_stop_check(struct socfpga_sdrseq *seq, const int write,
  1842. const int d, const int rank_bgn,
  1843. const u32 write_group, const u32 read_group,
  1844. u32 *bit_chk, u32 *sticky_bit_chk,
  1845. const u32 use_read_test)
  1846. {
  1847. const u32 ratio = seq->rwcfg->mem_if_read_dqs_width /
  1848. seq->rwcfg->mem_if_write_dqs_width;
  1849. const u32 correct_mask = write ? seq->param.write_correct_mask :
  1850. seq->param.read_correct_mask;
  1851. const u32 per_dqs = write ? seq->rwcfg->mem_dq_per_write_dqs :
  1852. seq->rwcfg->mem_dq_per_read_dqs;
  1853. u32 ret;
  1854. /*
  1855. * Stop searching when the read test doesn't pass AND when
  1856. * we've seen a passing read on every bit.
  1857. */
  1858. if (write) { /* WRITE-ONLY */
  1859. ret = !rw_mgr_mem_calibrate_write_test(seq, rank_bgn,
  1860. write_group, 0,
  1861. PASS_ONE_BIT, bit_chk,
  1862. 0);
  1863. } else if (use_read_test) { /* READ-ONLY */
  1864. ret = !rw_mgr_mem_calibrate_read_test(seq, rank_bgn, read_group,
  1865. NUM_READ_PB_TESTS,
  1866. PASS_ONE_BIT, bit_chk,
  1867. 0, 0);
  1868. } else { /* READ-ONLY */
  1869. rw_mgr_mem_calibrate_write_test(seq, rank_bgn, write_group, 0,
  1870. PASS_ONE_BIT, bit_chk, 0);
  1871. *bit_chk = *bit_chk >> (per_dqs *
  1872. (read_group - (write_group * ratio)));
  1873. ret = (*bit_chk == 0);
  1874. }
  1875. *sticky_bit_chk = *sticky_bit_chk | *bit_chk;
  1876. ret = ret && (*sticky_bit_chk == correct_mask);
  1877. debug_cond(DLEVEL >= 2,
  1878. "%s:%d center(left): dtap=%u => %u == %u && %u",
  1879. __func__, __LINE__, d,
  1880. *sticky_bit_chk, correct_mask, ret);
  1881. return ret;
  1882. }
  1883. /**
  1884. * search_left_edge() - Find left edge of DQ/DQS working phase
  1885. * @write: Perform read (Stage 2) or write (Stage 3) calibration
  1886. * @rank_bgn: Rank number
  1887. * @write_group: Write Group
  1888. * @read_group: Read Group
  1889. * @test_bgn: Rank number to begin the test
  1890. * @sticky_bit_chk: Resulting sticky bit mask after the test
  1891. * @left_edge: Left edge of the DQ/DQS phase
  1892. * @right_edge: Right edge of the DQ/DQS phase
  1893. * @use_read_test: Perform read test
  1894. *
  1895. * Find left edge of DQ/DQS working phase.
  1896. */
  1897. static void search_left_edge(struct socfpga_sdrseq *seq, const int write,
  1898. const int rank_bgn, const u32 write_group,
  1899. const u32 read_group, const u32 test_bgn,
  1900. u32 *sticky_bit_chk, int *left_edge,
  1901. int *right_edge, const u32 use_read_test)
  1902. {
  1903. const u32 delay_max = write ? seq->iocfg->io_out1_delay_max :
  1904. seq->iocfg->io_in_delay_max;
  1905. const u32 dqs_max = write ? seq->iocfg->io_out1_delay_max :
  1906. seq->iocfg->dqs_in_delay_max;
  1907. const u32 per_dqs = write ? seq->rwcfg->mem_dq_per_write_dqs :
  1908. seq->rwcfg->mem_dq_per_read_dqs;
  1909. u32 stop, bit_chk;
  1910. int i, d;
  1911. for (d = 0; d <= dqs_max; d++) {
  1912. if (write)
  1913. scc_mgr_apply_group_dq_out1_delay(seq, d);
  1914. else
  1915. scc_mgr_apply_group_dq_in_delay(seq, test_bgn, d);
  1916. writel(0, &sdr_scc_mgr->update);
  1917. stop = search_stop_check(seq, write, d, rank_bgn, write_group,
  1918. read_group, &bit_chk, sticky_bit_chk,
  1919. use_read_test);
  1920. if (stop == 1)
  1921. break;
  1922. /* stop != 1 */
  1923. for (i = 0; i < per_dqs; i++) {
  1924. if (bit_chk & 1) {
  1925. /*
  1926. * Remember a passing test as
  1927. * the left_edge.
  1928. */
  1929. left_edge[i] = d;
  1930. } else {
  1931. /*
  1932. * If a left edge has not been seen
  1933. * yet, then a future passing test
  1934. * will mark this edge as the right
  1935. * edge.
  1936. */
  1937. if (left_edge[i] == delay_max + 1)
  1938. right_edge[i] = -(d + 1);
  1939. }
  1940. bit_chk >>= 1;
  1941. }
  1942. }
  1943. /* Reset DQ delay chains to 0 */
  1944. if (write)
  1945. scc_mgr_apply_group_dq_out1_delay(seq, 0);
  1946. else
  1947. scc_mgr_apply_group_dq_in_delay(seq, test_bgn, 0);
  1948. *sticky_bit_chk = 0;
  1949. for (i = per_dqs - 1; i >= 0; i--) {
  1950. debug_cond(DLEVEL >= 2,
  1951. "%s:%d vfifo_center: left_edge[%u]: %d right_edge[%u]: %d\n",
  1952. __func__, __LINE__, i, left_edge[i],
  1953. i, right_edge[i]);
  1954. /*
  1955. * Check for cases where we haven't found the left edge,
  1956. * which makes our assignment of the the right edge invalid.
  1957. * Reset it to the illegal value.
  1958. */
  1959. if ((left_edge[i] == delay_max + 1) &&
  1960. (right_edge[i] != delay_max + 1)) {
  1961. right_edge[i] = delay_max + 1;
  1962. debug_cond(DLEVEL >= 2,
  1963. "%s:%d vfifo_center: reset right_edge[%u]: %d\n",
  1964. __func__, __LINE__, i, right_edge[i]);
  1965. }
  1966. /*
  1967. * Reset sticky bit
  1968. * READ: except for bits where we have seen both
  1969. * the left and right edge.
  1970. * WRITE: except for bits where we have seen the
  1971. * left edge.
  1972. */
  1973. *sticky_bit_chk <<= 1;
  1974. if (write) {
  1975. if (left_edge[i] != delay_max + 1)
  1976. *sticky_bit_chk |= 1;
  1977. } else {
  1978. if ((left_edge[i] != delay_max + 1) &&
  1979. (right_edge[i] != delay_max + 1))
  1980. *sticky_bit_chk |= 1;
  1981. }
  1982. }
  1983. }
  1984. /**
  1985. * search_right_edge() - Find right edge of DQ/DQS working phase
  1986. * @write: Perform read (Stage 2) or write (Stage 3) calibration
  1987. * @rank_bgn: Rank number
  1988. * @write_group: Write Group
  1989. * @read_group: Read Group
  1990. * @start_dqs: DQS start phase
  1991. * @start_dqs_en: DQS enable start phase
  1992. * @sticky_bit_chk: Resulting sticky bit mask after the test
  1993. * @left_edge: Left edge of the DQ/DQS phase
  1994. * @right_edge: Right edge of the DQ/DQS phase
  1995. * @use_read_test: Perform read test
  1996. *
  1997. * Find right edge of DQ/DQS working phase.
  1998. */
  1999. static int search_right_edge(struct socfpga_sdrseq *seq, const int write,
  2000. const int rank_bgn, const u32 write_group,
  2001. const u32 read_group, const int start_dqs,
  2002. const int start_dqs_en, u32 *sticky_bit_chk,
  2003. int *left_edge, int *right_edge,
  2004. const u32 use_read_test)
  2005. {
  2006. const u32 delay_max = write ? seq->iocfg->io_out1_delay_max :
  2007. seq->iocfg->io_in_delay_max;
  2008. const u32 dqs_max = write ? seq->iocfg->io_out1_delay_max :
  2009. seq->iocfg->dqs_in_delay_max;
  2010. const u32 per_dqs = write ? seq->rwcfg->mem_dq_per_write_dqs :
  2011. seq->rwcfg->mem_dq_per_read_dqs;
  2012. u32 stop, bit_chk;
  2013. int i, d;
  2014. for (d = 0; d <= dqs_max - start_dqs; d++) {
  2015. if (write) { /* WRITE-ONLY */
  2016. scc_mgr_apply_group_dqs_io_and_oct_out1(seq,
  2017. write_group,
  2018. d + start_dqs);
  2019. } else { /* READ-ONLY */
  2020. scc_mgr_set_dqs_bus_in_delay(read_group, d + start_dqs);
  2021. if (seq->iocfg->shift_dqs_en_when_shift_dqs) {
  2022. u32 delay = d + start_dqs_en;
  2023. if (delay > seq->iocfg->dqs_en_delay_max)
  2024. delay = seq->iocfg->dqs_en_delay_max;
  2025. scc_mgr_set_dqs_en_delay(read_group, delay);
  2026. }
  2027. scc_mgr_load_dqs(read_group);
  2028. }
  2029. writel(0, &sdr_scc_mgr->update);
  2030. stop = search_stop_check(seq, write, d, rank_bgn, write_group,
  2031. read_group, &bit_chk, sticky_bit_chk,
  2032. use_read_test);
  2033. if (stop == 1) {
  2034. if (write && (d == 0)) { /* WRITE-ONLY */
  2035. for (i = 0;
  2036. i < seq->rwcfg->mem_dq_per_write_dqs;
  2037. i++) {
  2038. /*
  2039. * d = 0 failed, but it passed when
  2040. * testing the left edge, so it must be
  2041. * marginal, set it to -1
  2042. */
  2043. if (right_edge[i] == delay_max + 1 &&
  2044. left_edge[i] != delay_max + 1)
  2045. right_edge[i] = -1;
  2046. }
  2047. }
  2048. break;
  2049. }
  2050. /* stop != 1 */
  2051. for (i = 0; i < per_dqs; i++) {
  2052. if (bit_chk & 1) {
  2053. /*
  2054. * Remember a passing test as
  2055. * the right_edge.
  2056. */
  2057. right_edge[i] = d;
  2058. } else {
  2059. if (d != 0) {
  2060. /*
  2061. * If a right edge has not
  2062. * been seen yet, then a future
  2063. * passing test will mark this
  2064. * edge as the left edge.
  2065. */
  2066. if (right_edge[i] == delay_max + 1)
  2067. left_edge[i] = -(d + 1);
  2068. } else {
  2069. /*
  2070. * d = 0 failed, but it passed
  2071. * when testing the left edge,
  2072. * so it must be marginal, set
  2073. * it to -1
  2074. */
  2075. if (right_edge[i] == delay_max + 1 &&
  2076. left_edge[i] != delay_max + 1)
  2077. right_edge[i] = -1;
  2078. /*
  2079. * If a right edge has not been
  2080. * seen yet, then a future
  2081. * passing test will mark this
  2082. * edge as the left edge.
  2083. */
  2084. else if (right_edge[i] == delay_max + 1)
  2085. left_edge[i] = -(d + 1);
  2086. }
  2087. }
  2088. debug_cond(DLEVEL >= 2, "%s:%d center[r,d=%u]: ",
  2089. __func__, __LINE__, d);
  2090. debug_cond(DLEVEL >= 2,
  2091. "bit_chk_test=%i left_edge[%u]: %d ",
  2092. bit_chk & 1, i, left_edge[i]);
  2093. debug_cond(DLEVEL >= 2, "right_edge[%u]: %d\n", i,
  2094. right_edge[i]);
  2095. bit_chk >>= 1;
  2096. }
  2097. }
  2098. /* Check that all bits have a window */
  2099. for (i = 0; i < per_dqs; i++) {
  2100. debug_cond(DLEVEL >= 2,
  2101. "%s:%d write_center: left_edge[%u]: %d right_edge[%u]: %d",
  2102. __func__, __LINE__, i, left_edge[i],
  2103. i, right_edge[i]);
  2104. if ((left_edge[i] == dqs_max + 1) ||
  2105. (right_edge[i] == dqs_max + 1))
  2106. return i + 1; /* FIXME: If we fail, retval > 0 */
  2107. }
  2108. return 0;
  2109. }
  2110. /**
  2111. * get_window_mid_index() - Find the best middle setting of DQ/DQS phase
  2112. * @write: Perform read (Stage 2) or write (Stage 3) calibration
  2113. * @left_edge: Left edge of the DQ/DQS phase
  2114. * @right_edge: Right edge of the DQ/DQS phase
  2115. * @mid_min: Best DQ/DQS phase middle setting
  2116. *
  2117. * Find index and value of the middle of the DQ/DQS working phase.
  2118. */
  2119. static int get_window_mid_index(struct socfpga_sdrseq *seq,
  2120. const int write, int *left_edge,
  2121. int *right_edge, int *mid_min)
  2122. {
  2123. const u32 per_dqs = write ? seq->rwcfg->mem_dq_per_write_dqs :
  2124. seq->rwcfg->mem_dq_per_read_dqs;
  2125. int i, mid, min_index;
  2126. /* Find middle of window for each DQ bit */
  2127. *mid_min = left_edge[0] - right_edge[0];
  2128. min_index = 0;
  2129. for (i = 1; i < per_dqs; i++) {
  2130. mid = left_edge[i] - right_edge[i];
  2131. if (mid < *mid_min) {
  2132. *mid_min = mid;
  2133. min_index = i;
  2134. }
  2135. }
  2136. /*
  2137. * -mid_min/2 represents the amount that we need to move DQS.
  2138. * If mid_min is odd and positive we'll need to add one to make
  2139. * sure the rounding in further calculations is correct (always
  2140. * bias to the right), so just add 1 for all positive values.
  2141. */
  2142. if (*mid_min > 0)
  2143. (*mid_min)++;
  2144. *mid_min = *mid_min / 2;
  2145. debug_cond(DLEVEL >= 1, "%s:%d vfifo_center: *mid_min=%d (index=%u)\n",
  2146. __func__, __LINE__, *mid_min, min_index);
  2147. return min_index;
  2148. }
  2149. /**
  2150. * center_dq_windows() - Center the DQ/DQS windows
  2151. * @write: Perform read (Stage 2) or write (Stage 3) calibration
  2152. * @left_edge: Left edge of the DQ/DQS phase
  2153. * @right_edge: Right edge of the DQ/DQS phase
  2154. * @mid_min: Adjusted DQ/DQS phase middle setting
  2155. * @orig_mid_min: Original DQ/DQS phase middle setting
  2156. * @min_index: DQ/DQS phase middle setting index
  2157. * @test_bgn: Rank number to begin the test
  2158. * @dq_margin: Amount of shift for the DQ
  2159. * @dqs_margin: Amount of shift for the DQS
  2160. *
  2161. * Align the DQ/DQS windows in each group.
  2162. */
  2163. static void center_dq_windows(struct socfpga_sdrseq *seq,
  2164. const int write, int *left_edge, int *right_edge,
  2165. const int mid_min, const int orig_mid_min,
  2166. const int min_index, const int test_bgn,
  2167. int *dq_margin, int *dqs_margin)
  2168. {
  2169. const s32 delay_max = write ? seq->iocfg->io_out1_delay_max :
  2170. seq->iocfg->io_in_delay_max;
  2171. const s32 per_dqs = write ? seq->rwcfg->mem_dq_per_write_dqs :
  2172. seq->rwcfg->mem_dq_per_read_dqs;
  2173. const s32 delay_off = write ? SCC_MGR_IO_OUT1_DELAY_OFFSET :
  2174. SCC_MGR_IO_IN_DELAY_OFFSET;
  2175. const s32 addr = SDR_PHYGRP_SCCGRP_ADDRESS | delay_off;
  2176. s32 temp_dq_io_delay1;
  2177. int shift_dq, i, p;
  2178. /* Initialize data for export structures */
  2179. *dqs_margin = delay_max + 1;
  2180. *dq_margin = delay_max + 1;
  2181. /* add delay to bring centre of all DQ windows to the same "level" */
  2182. for (i = 0, p = test_bgn; i < per_dqs; i++, p++) {
  2183. /* Use values before divide by 2 to reduce round off error */
  2184. shift_dq = (left_edge[i] - right_edge[i] -
  2185. (left_edge[min_index] - right_edge[min_index]))/2 +
  2186. (orig_mid_min - mid_min);
  2187. debug_cond(DLEVEL >= 2,
  2188. "vfifo_center: before: shift_dq[%u]=%d\n",
  2189. i, shift_dq);
  2190. temp_dq_io_delay1 = readl(addr + (i << 2));
  2191. if (shift_dq + temp_dq_io_delay1 > delay_max)
  2192. shift_dq = delay_max - temp_dq_io_delay1;
  2193. else if (shift_dq + temp_dq_io_delay1 < 0)
  2194. shift_dq = -temp_dq_io_delay1;
  2195. debug_cond(DLEVEL >= 2,
  2196. "vfifo_center: after: shift_dq[%u]=%d\n",
  2197. i, shift_dq);
  2198. if (write)
  2199. scc_mgr_set_dq_out1_delay(i,
  2200. temp_dq_io_delay1 + shift_dq);
  2201. else
  2202. scc_mgr_set_dq_in_delay(p,
  2203. temp_dq_io_delay1 + shift_dq);
  2204. scc_mgr_load_dq(p);
  2205. debug_cond(DLEVEL >= 2,
  2206. "vfifo_center: margin[%u]=[%d,%d]\n", i,
  2207. left_edge[i] - shift_dq + (-mid_min),
  2208. right_edge[i] + shift_dq - (-mid_min));
  2209. /* To determine values for export structures */
  2210. if (left_edge[i] - shift_dq + (-mid_min) < *dq_margin)
  2211. *dq_margin = left_edge[i] - shift_dq + (-mid_min);
  2212. if (right_edge[i] + shift_dq - (-mid_min) < *dqs_margin)
  2213. *dqs_margin = right_edge[i] + shift_dq - (-mid_min);
  2214. }
  2215. }
  2216. /**
  2217. * rw_mgr_mem_calibrate_vfifo_center() - Per-bit deskew DQ and centering
  2218. * @rank_bgn: Rank number
  2219. * @rw_group: Read/Write Group
  2220. * @test_bgn: Rank at which the test begins
  2221. * @use_read_test: Perform a read test
  2222. * @update_fom: Update FOM
  2223. *
  2224. * Per-bit deskew DQ and centering.
  2225. */
  2226. static int rw_mgr_mem_calibrate_vfifo_center(struct socfpga_sdrseq *seq,
  2227. const u32 rank_bgn,
  2228. const u32 rw_group,
  2229. const u32 test_bgn,
  2230. const int use_read_test,
  2231. const int update_fom)
  2232. {
  2233. const u32 addr =
  2234. SDR_PHYGRP_SCCGRP_ADDRESS + SCC_MGR_DQS_IN_DELAY_OFFSET +
  2235. (rw_group << 2);
  2236. /*
  2237. * Store these as signed since there are comparisons with
  2238. * signed numbers.
  2239. */
  2240. u32 sticky_bit_chk;
  2241. s32 left_edge[seq->rwcfg->mem_dq_per_read_dqs];
  2242. s32 right_edge[seq->rwcfg->mem_dq_per_read_dqs];
  2243. s32 orig_mid_min, mid_min;
  2244. s32 new_dqs, start_dqs, start_dqs_en = 0, final_dqs_en;
  2245. s32 dq_margin, dqs_margin;
  2246. int i, min_index;
  2247. int ret;
  2248. debug("%s:%d: %u %u", __func__, __LINE__, rw_group, test_bgn);
  2249. start_dqs = readl(addr);
  2250. if (seq->iocfg->shift_dqs_en_when_shift_dqs)
  2251. start_dqs_en = readl(addr - seq->iocfg->dqs_en_delay_offset);
  2252. /* set the left and right edge of each bit to an illegal value */
  2253. /* use (seq->iocfg->io_in_delay_max + 1) as an illegal value */
  2254. sticky_bit_chk = 0;
  2255. for (i = 0; i < seq->rwcfg->mem_dq_per_read_dqs; i++) {
  2256. left_edge[i] = seq->iocfg->io_in_delay_max + 1;
  2257. right_edge[i] = seq->iocfg->io_in_delay_max + 1;
  2258. }
  2259. /* Search for the left edge of the window for each bit */
  2260. search_left_edge(seq, 0, rank_bgn, rw_group, rw_group, test_bgn,
  2261. &sticky_bit_chk,
  2262. left_edge, right_edge, use_read_test);
  2263. /* Search for the right edge of the window for each bit */
  2264. ret = search_right_edge(seq, 0, rank_bgn, rw_group, rw_group,
  2265. start_dqs, start_dqs_en,
  2266. &sticky_bit_chk,
  2267. left_edge, right_edge, use_read_test);
  2268. if (ret) {
  2269. /*
  2270. * Restore delay chain settings before letting the loop
  2271. * in rw_mgr_mem_calibrate_vfifo to retry different
  2272. * dqs/ck relationships.
  2273. */
  2274. scc_mgr_set_dqs_bus_in_delay(rw_group, start_dqs);
  2275. if (seq->iocfg->shift_dqs_en_when_shift_dqs)
  2276. scc_mgr_set_dqs_en_delay(rw_group, start_dqs_en);
  2277. scc_mgr_load_dqs(rw_group);
  2278. writel(0, &sdr_scc_mgr->update);
  2279. debug_cond(DLEVEL >= 1,
  2280. "%s:%d vfifo_center: failed to find edge [%u]: %d %d",
  2281. __func__, __LINE__, i, left_edge[i], right_edge[i]);
  2282. if (use_read_test) {
  2283. set_failing_group_stage(seq, rw_group *
  2284. seq->rwcfg->mem_dq_per_read_dqs + i,
  2285. CAL_STAGE_VFIFO,
  2286. CAL_SUBSTAGE_VFIFO_CENTER);
  2287. } else {
  2288. set_failing_group_stage(seq, rw_group *
  2289. seq->rwcfg->mem_dq_per_read_dqs + i,
  2290. CAL_STAGE_VFIFO_AFTER_WRITES,
  2291. CAL_SUBSTAGE_VFIFO_CENTER);
  2292. }
  2293. return -EIO;
  2294. }
  2295. min_index = get_window_mid_index(seq, 0, left_edge, right_edge,
  2296. &mid_min);
  2297. /* Determine the amount we can change DQS (which is -mid_min) */
  2298. orig_mid_min = mid_min;
  2299. new_dqs = start_dqs - mid_min;
  2300. if (new_dqs > seq->iocfg->dqs_in_delay_max)
  2301. new_dqs = seq->iocfg->dqs_in_delay_max;
  2302. else if (new_dqs < 0)
  2303. new_dqs = 0;
  2304. mid_min = start_dqs - new_dqs;
  2305. debug_cond(DLEVEL >= 1, "vfifo_center: new mid_min=%d new_dqs=%d\n",
  2306. mid_min, new_dqs);
  2307. if (seq->iocfg->shift_dqs_en_when_shift_dqs) {
  2308. if (start_dqs_en - mid_min > seq->iocfg->dqs_en_delay_max)
  2309. mid_min += start_dqs_en - mid_min -
  2310. seq->iocfg->dqs_en_delay_max;
  2311. else if (start_dqs_en - mid_min < 0)
  2312. mid_min += start_dqs_en - mid_min;
  2313. }
  2314. new_dqs = start_dqs - mid_min;
  2315. debug_cond(DLEVEL >= 1,
  2316. "vfifo_center: start_dqs=%d start_dqs_en=%d new_dqs=%d mid_min=%d\n",
  2317. start_dqs,
  2318. seq->iocfg->shift_dqs_en_when_shift_dqs ? start_dqs_en : -1,
  2319. new_dqs, mid_min);
  2320. /* Add delay to bring centre of all DQ windows to the same "level". */
  2321. center_dq_windows(seq, 0, left_edge, right_edge, mid_min, orig_mid_min,
  2322. min_index, test_bgn, &dq_margin, &dqs_margin);
  2323. /* Move DQS-en */
  2324. if (seq->iocfg->shift_dqs_en_when_shift_dqs) {
  2325. final_dqs_en = start_dqs_en - mid_min;
  2326. scc_mgr_set_dqs_en_delay(rw_group, final_dqs_en);
  2327. scc_mgr_load_dqs(rw_group);
  2328. }
  2329. /* Move DQS */
  2330. scc_mgr_set_dqs_bus_in_delay(rw_group, new_dqs);
  2331. scc_mgr_load_dqs(rw_group);
  2332. debug_cond(DLEVEL >= 2,
  2333. "%s:%d vfifo_center: dq_margin=%d dqs_margin=%d",
  2334. __func__, __LINE__, dq_margin, dqs_margin);
  2335. /*
  2336. * Do not remove this line as it makes sure all of our decisions
  2337. * have been applied. Apply the update bit.
  2338. */
  2339. writel(0, &sdr_scc_mgr->update);
  2340. if ((dq_margin < 0) || (dqs_margin < 0))
  2341. return -EINVAL;
  2342. return 0;
  2343. }
  2344. /**
  2345. * rw_mgr_mem_calibrate_guaranteed_write() - Perform guaranteed write into the
  2346. * device
  2347. * @rw_group: Read/Write Group
  2348. * @phase: DQ/DQS phase
  2349. *
  2350. * Because initially no communication ca be reliably performed with the memory
  2351. * device, the sequencer uses a guaranteed write mechanism to write data into
  2352. * the memory device.
  2353. */
  2354. static int rw_mgr_mem_calibrate_guaranteed_write(struct socfpga_sdrseq *seq,
  2355. const u32 rw_group,
  2356. const u32 phase)
  2357. {
  2358. int ret;
  2359. /* Set a particular DQ/DQS phase. */
  2360. scc_mgr_set_dqdqs_output_phase_all_ranks(seq, rw_group, phase);
  2361. debug_cond(DLEVEL >= 1, "%s:%d guaranteed write: g=%u p=%u\n",
  2362. __func__, __LINE__, rw_group, phase);
  2363. /*
  2364. * Altera EMI_RM 2015.05.04 :: Figure 1-25
  2365. * Load up the patterns used by read calibration using the
  2366. * current DQDQS phase.
  2367. */
  2368. rw_mgr_mem_calibrate_read_load_patterns(seq, 0, 1);
  2369. if (seq->gbl.phy_debug_mode_flags & PHY_DEBUG_DISABLE_GUARANTEED_READ)
  2370. return 0;
  2371. /*
  2372. * Altera EMI_RM 2015.05.04 :: Figure 1-26
  2373. * Back-to-Back reads of the patterns used for calibration.
  2374. */
  2375. ret = rw_mgr_mem_calibrate_read_test_patterns(seq, 0, rw_group, 1);
  2376. if (ret)
  2377. debug_cond(DLEVEL >= 1,
  2378. "%s:%d Guaranteed read test failed: g=%u p=%u\n",
  2379. __func__, __LINE__, rw_group, phase);
  2380. return ret;
  2381. }
  2382. /**
  2383. * rw_mgr_mem_calibrate_dqs_enable_calibration() - DQS Enable Calibration
  2384. * @rw_group: Read/Write Group
  2385. * @test_bgn: Rank at which the test begins
  2386. *
  2387. * DQS enable calibration ensures reliable capture of the DQ signal without
  2388. * glitches on the DQS line.
  2389. */
  2390. static int
  2391. rw_mgr_mem_calibrate_dqs_enable_calibration(struct socfpga_sdrseq *seq,
  2392. const u32 rw_group,
  2393. const u32 test_bgn)
  2394. {
  2395. /*
  2396. * Altera EMI_RM 2015.05.04 :: Figure 1-27
  2397. * DQS and DQS Eanble Signal Relationships.
  2398. */
  2399. /* We start at zero, so have one less dq to devide among */
  2400. const u32 delay_step = seq->iocfg->io_in_delay_max /
  2401. (seq->rwcfg->mem_dq_per_read_dqs - 1);
  2402. int ret;
  2403. u32 i, p, d, r;
  2404. debug("%s:%d (%u,%u)\n", __func__, __LINE__, rw_group, test_bgn);
  2405. /* Try different dq_in_delays since the DQ path is shorter than DQS. */
  2406. for (r = 0; r < seq->rwcfg->mem_number_of_ranks;
  2407. r += NUM_RANKS_PER_SHADOW_REG) {
  2408. for (i = 0, p = test_bgn, d = 0;
  2409. i < seq->rwcfg->mem_dq_per_read_dqs;
  2410. i++, p++, d += delay_step) {
  2411. debug_cond(DLEVEL >= 1,
  2412. "%s:%d: g=%u r=%u i=%u p=%u d=%u\n",
  2413. __func__, __LINE__, rw_group, r, i, p, d);
  2414. scc_mgr_set_dq_in_delay(p, d);
  2415. scc_mgr_load_dq(p);
  2416. }
  2417. writel(0, &sdr_scc_mgr->update);
  2418. }
  2419. /*
  2420. * Try rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase across different
  2421. * dq_in_delay values
  2422. */
  2423. ret = rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(seq, rw_group);
  2424. debug_cond(DLEVEL >= 1,
  2425. "%s:%d: g=%u found=%u; Reseting delay chain to zero\n",
  2426. __func__, __LINE__, rw_group, !ret);
  2427. for (r = 0; r < seq->rwcfg->mem_number_of_ranks;
  2428. r += NUM_RANKS_PER_SHADOW_REG) {
  2429. scc_mgr_apply_group_dq_in_delay(seq, test_bgn, 0);
  2430. writel(0, &sdr_scc_mgr->update);
  2431. }
  2432. return ret;
  2433. }
  2434. /**
  2435. * rw_mgr_mem_calibrate_dq_dqs_centering() - Centering DQ/DQS
  2436. * @rw_group: Read/Write Group
  2437. * @test_bgn: Rank at which the test begins
  2438. * @use_read_test: Perform a read test
  2439. * @update_fom: Update FOM
  2440. *
  2441. * The centerin DQ/DQS stage attempts to align DQ and DQS signals on reads
  2442. * within a group.
  2443. */
  2444. static int
  2445. rw_mgr_mem_calibrate_dq_dqs_centering(struct socfpga_sdrseq *seq,
  2446. const u32 rw_group, const u32 test_bgn,
  2447. const int use_read_test,
  2448. const int update_fom)
  2449. {
  2450. int ret, grp_calibrated;
  2451. u32 rank_bgn, sr;
  2452. /*
  2453. * Altera EMI_RM 2015.05.04 :: Figure 1-28
  2454. * Read per-bit deskew can be done on a per shadow register basis.
  2455. */
  2456. grp_calibrated = 1;
  2457. for (rank_bgn = 0, sr = 0;
  2458. rank_bgn < seq->rwcfg->mem_number_of_ranks;
  2459. rank_bgn += NUM_RANKS_PER_SHADOW_REG, sr++) {
  2460. ret = rw_mgr_mem_calibrate_vfifo_center(seq, rank_bgn, rw_group,
  2461. test_bgn,
  2462. use_read_test,
  2463. update_fom);
  2464. if (!ret)
  2465. continue;
  2466. grp_calibrated = 0;
  2467. }
  2468. if (!grp_calibrated)
  2469. return -EIO;
  2470. return 0;
  2471. }
  2472. /**
  2473. * rw_mgr_mem_calibrate_vfifo() - Calibrate the read valid prediction FIFO
  2474. * @rw_group: Read/Write Group
  2475. * @test_bgn: Rank at which the test begins
  2476. *
  2477. * Stage 1: Calibrate the read valid prediction FIFO.
  2478. *
  2479. * This function implements UniPHY calibration Stage 1, as explained in
  2480. * detail in Altera EMI_RM 2015.05.04 , "UniPHY Calibration Stages".
  2481. *
  2482. * - read valid prediction will consist of finding:
  2483. * - DQS enable phase and DQS enable delay (DQS Enable Calibration)
  2484. * - DQS input phase and DQS input delay (DQ/DQS Centering)
  2485. * - we also do a per-bit deskew on the DQ lines.
  2486. */
  2487. static int rw_mgr_mem_calibrate_vfifo(struct socfpga_sdrseq *seq,
  2488. const u32 rw_group, const u32 test_bgn)
  2489. {
  2490. u32 p, d;
  2491. u32 dtaps_per_ptap;
  2492. u32 failed_substage;
  2493. int ret;
  2494. debug("%s:%d: %u %u\n", __func__, __LINE__, rw_group, test_bgn);
  2495. /* Update info for sims */
  2496. reg_file_set_group(rw_group);
  2497. reg_file_set_stage(CAL_STAGE_VFIFO);
  2498. reg_file_set_sub_stage(CAL_SUBSTAGE_GUARANTEED_READ);
  2499. failed_substage = CAL_SUBSTAGE_GUARANTEED_READ;
  2500. /* USER Determine number of delay taps for each phase tap. */
  2501. dtaps_per_ptap = DIV_ROUND_UP(seq->iocfg->delay_per_opa_tap,
  2502. seq->iocfg->delay_per_dqs_en_dchain_tap)
  2503. - 1;
  2504. for (d = 0; d <= dtaps_per_ptap; d += 2) {
  2505. /*
  2506. * In RLDRAMX we may be messing the delay of pins in
  2507. * the same write rw_group but outside of the current read
  2508. * the rw_group, but that's ok because we haven't calibrated
  2509. * output side yet.
  2510. */
  2511. if (d > 0) {
  2512. scc_mgr_apply_group_all_out_delay_add_all_ranks(seq,
  2513. rw_group,
  2514. d);
  2515. }
  2516. for (p = 0; p <= seq->iocfg->dqdqs_out_phase_max; p++) {
  2517. /* 1) Guaranteed Write */
  2518. ret = rw_mgr_mem_calibrate_guaranteed_write(seq,
  2519. rw_group,
  2520. p);
  2521. if (ret)
  2522. break;
  2523. /* 2) DQS Enable Calibration */
  2524. ret = rw_mgr_mem_calibrate_dqs_enable_calibration(seq,
  2525. rw_group,
  2526. test_bgn);
  2527. if (ret) {
  2528. failed_substage = CAL_SUBSTAGE_DQS_EN_PHASE;
  2529. continue;
  2530. }
  2531. /* 3) Centering DQ/DQS */
  2532. /*
  2533. * If doing read after write calibration, do not update
  2534. * FOM now. Do it then.
  2535. */
  2536. ret = rw_mgr_mem_calibrate_dq_dqs_centering(seq,
  2537. rw_group,
  2538. test_bgn,
  2539. 1, 0);
  2540. if (ret) {
  2541. failed_substage = CAL_SUBSTAGE_VFIFO_CENTER;
  2542. continue;
  2543. }
  2544. /* All done. */
  2545. goto cal_done_ok;
  2546. }
  2547. }
  2548. /* Calibration Stage 1 failed. */
  2549. set_failing_group_stage(seq, rw_group, CAL_STAGE_VFIFO,
  2550. failed_substage);
  2551. return 0;
  2552. /* Calibration Stage 1 completed OK. */
  2553. cal_done_ok:
  2554. /*
  2555. * Reset the delay chains back to zero if they have moved > 1
  2556. * (check for > 1 because loop will increase d even when pass in
  2557. * first case).
  2558. */
  2559. if (d > 2)
  2560. scc_mgr_zero_group(seq, rw_group, 1);
  2561. return 1;
  2562. }
  2563. /**
  2564. * rw_mgr_mem_calibrate_vfifo_end() - DQ/DQS Centering.
  2565. * @rw_group: Read/Write Group
  2566. * @test_bgn: Rank at which the test begins
  2567. *
  2568. * Stage 3: DQ/DQS Centering.
  2569. *
  2570. * This function implements UniPHY calibration Stage 3, as explained in
  2571. * detail in Altera EMI_RM 2015.05.04 , "UniPHY Calibration Stages".
  2572. */
  2573. static int rw_mgr_mem_calibrate_vfifo_end(struct socfpga_sdrseq *seq,
  2574. const u32 rw_group,
  2575. const u32 test_bgn)
  2576. {
  2577. int ret;
  2578. debug("%s:%d %u %u", __func__, __LINE__, rw_group, test_bgn);
  2579. /* Update info for sims. */
  2580. reg_file_set_group(rw_group);
  2581. reg_file_set_stage(CAL_STAGE_VFIFO_AFTER_WRITES);
  2582. reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
  2583. ret = rw_mgr_mem_calibrate_dq_dqs_centering(seq, rw_group, test_bgn, 0,
  2584. 1);
  2585. if (ret)
  2586. set_failing_group_stage(seq, rw_group,
  2587. CAL_STAGE_VFIFO_AFTER_WRITES,
  2588. CAL_SUBSTAGE_VFIFO_CENTER);
  2589. return ret;
  2590. }
  2591. /**
  2592. * rw_mgr_mem_calibrate_lfifo() - Minimize latency
  2593. *
  2594. * Stage 4: Minimize latency.
  2595. *
  2596. * This function implements UniPHY calibration Stage 4, as explained in
  2597. * detail in Altera EMI_RM 2015.05.04 , "UniPHY Calibration Stages".
  2598. * Calibrate LFIFO to find smallest read latency.
  2599. */
  2600. static u32 rw_mgr_mem_calibrate_lfifo(struct socfpga_sdrseq *seq)
  2601. {
  2602. int found_one = 0;
  2603. debug("%s:%d\n", __func__, __LINE__);
  2604. /* Update info for sims. */
  2605. reg_file_set_stage(CAL_STAGE_LFIFO);
  2606. reg_file_set_sub_stage(CAL_SUBSTAGE_READ_LATENCY);
  2607. /* Load up the patterns used by read calibration for all ranks */
  2608. rw_mgr_mem_calibrate_read_load_patterns(seq, 0, 1);
  2609. do {
  2610. writel(seq->gbl.curr_read_lat, &phy_mgr_cfg->phy_rlat);
  2611. debug_cond(DLEVEL >= 2, "%s:%d lfifo: read_lat=%u",
  2612. __func__, __LINE__, seq->gbl.curr_read_lat);
  2613. if (!rw_mgr_mem_calibrate_read_test_all_ranks(seq, 0,
  2614. NUM_READ_TESTS,
  2615. PASS_ALL_BITS, 1))
  2616. break;
  2617. found_one = 1;
  2618. /*
  2619. * Reduce read latency and see if things are
  2620. * working correctly.
  2621. */
  2622. seq->gbl.curr_read_lat--;
  2623. } while (seq->gbl.curr_read_lat > 0);
  2624. /* Reset the fifos to get pointers to known state. */
  2625. writel(0, &phy_mgr_cmd->fifo_reset);
  2626. if (found_one) {
  2627. /* Add a fudge factor to the read latency that was determined */
  2628. seq->gbl.curr_read_lat += 2;
  2629. writel(seq->gbl.curr_read_lat, &phy_mgr_cfg->phy_rlat);
  2630. debug_cond(DLEVEL >= 2,
  2631. "%s:%d lfifo: success: using read_lat=%u\n",
  2632. __func__, __LINE__, seq->gbl.curr_read_lat);
  2633. } else {
  2634. set_failing_group_stage(seq, 0xff, CAL_STAGE_LFIFO,
  2635. CAL_SUBSTAGE_READ_LATENCY);
  2636. debug_cond(DLEVEL >= 2,
  2637. "%s:%d lfifo: failed at initial read_lat=%u\n",
  2638. __func__, __LINE__, seq->gbl.curr_read_lat);
  2639. }
  2640. return found_one;
  2641. }
  2642. /**
  2643. * search_window() - Search for the/part of the window with DM/DQS shift
  2644. * @search_dm: If 1, search for the DM shift, if 0, search for DQS
  2645. * shift
  2646. * @rank_bgn: Rank number
  2647. * @write_group: Write Group
  2648. * @bgn_curr: Current window begin
  2649. * @end_curr: Current window end
  2650. * @bgn_best: Current best window begin
  2651. * @end_best: Current best window end
  2652. * @win_best: Size of the best window
  2653. * @new_dqs: New DQS value (only applicable if search_dm = 0).
  2654. *
  2655. * Search for the/part of the window with DM/DQS shift.
  2656. */
  2657. static void search_window(struct socfpga_sdrseq *seq,
  2658. const int search_dm, const u32 rank_bgn,
  2659. const u32 write_group, int *bgn_curr, int *end_curr,
  2660. int *bgn_best, int *end_best, int *win_best,
  2661. int new_dqs)
  2662. {
  2663. u32 bit_chk;
  2664. const int max = seq->iocfg->io_out1_delay_max - new_dqs;
  2665. int d, di;
  2666. /* Search for the/part of the window with DM/DQS shift. */
  2667. for (di = max; di >= 0; di -= DELTA_D) {
  2668. if (search_dm) {
  2669. d = di;
  2670. scc_mgr_apply_group_dm_out1_delay(seq, d);
  2671. } else {
  2672. /* For DQS, we go from 0...max */
  2673. d = max - di;
  2674. /*
  2675. * Note: This only shifts DQS, so are we limiting
  2676. * ourselves to width of DQ unnecessarily.
  2677. */
  2678. scc_mgr_apply_group_dqs_io_and_oct_out1(seq,
  2679. write_group,
  2680. d + new_dqs);
  2681. }
  2682. writel(0, &sdr_scc_mgr->update);
  2683. if (rw_mgr_mem_calibrate_write_test(seq, rank_bgn, write_group,
  2684. 1, PASS_ALL_BITS, &bit_chk,
  2685. 0)) {
  2686. /* Set current end of the window. */
  2687. *end_curr = search_dm ? -d : d;
  2688. /*
  2689. * If a starting edge of our window has not been seen
  2690. * this is our current start of the DM window.
  2691. */
  2692. if (*bgn_curr == seq->iocfg->io_out1_delay_max + 1)
  2693. *bgn_curr = search_dm ? -d : d;
  2694. /*
  2695. * If current window is bigger than best seen.
  2696. * Set best seen to be current window.
  2697. */
  2698. if ((*end_curr - *bgn_curr + 1) > *win_best) {
  2699. *win_best = *end_curr - *bgn_curr + 1;
  2700. *bgn_best = *bgn_curr;
  2701. *end_best = *end_curr;
  2702. }
  2703. } else {
  2704. /* We just saw a failing test. Reset temp edge. */
  2705. *bgn_curr = seq->iocfg->io_out1_delay_max + 1;
  2706. *end_curr = seq->iocfg->io_out1_delay_max + 1;
  2707. /* Early exit is only applicable to DQS. */
  2708. if (search_dm)
  2709. continue;
  2710. /*
  2711. * Early exit optimization: if the remaining delay
  2712. * chain space is less than already seen largest
  2713. * window we can exit.
  2714. */
  2715. if (*win_best - 1 > seq->iocfg->io_out1_delay_max
  2716. - new_dqs - d)
  2717. break;
  2718. }
  2719. }
  2720. }
  2721. /*
  2722. * rw_mgr_mem_calibrate_writes_center() - Center all windows
  2723. * @rank_bgn: Rank number
  2724. * @write_group: Write group
  2725. * @test_bgn: Rank at which the test begins
  2726. *
  2727. * Center all windows. Do per-bit-deskew to possibly increase size of
  2728. * certain windows.
  2729. */
  2730. static int
  2731. rw_mgr_mem_calibrate_writes_center(struct socfpga_sdrseq *seq,
  2732. const u32 rank_bgn, const u32 write_group,
  2733. const u32 test_bgn)
  2734. {
  2735. int i;
  2736. u32 sticky_bit_chk;
  2737. u32 min_index;
  2738. int left_edge[seq->rwcfg->mem_dq_per_write_dqs];
  2739. int right_edge[seq->rwcfg->mem_dq_per_write_dqs];
  2740. int mid;
  2741. int mid_min, orig_mid_min;
  2742. int new_dqs, start_dqs;
  2743. int dq_margin, dqs_margin, dm_margin;
  2744. int bgn_curr = seq->iocfg->io_out1_delay_max + 1;
  2745. int end_curr = seq->iocfg->io_out1_delay_max + 1;
  2746. int bgn_best = seq->iocfg->io_out1_delay_max + 1;
  2747. int end_best = seq->iocfg->io_out1_delay_max + 1;
  2748. int win_best = 0;
  2749. int ret;
  2750. debug("%s:%d %u %u", __func__, __LINE__, write_group, test_bgn);
  2751. dm_margin = 0;
  2752. start_dqs = readl((SDR_PHYGRP_SCCGRP_ADDRESS |
  2753. SCC_MGR_IO_OUT1_DELAY_OFFSET) +
  2754. (seq->rwcfg->mem_dq_per_write_dqs << 2));
  2755. /* Per-bit deskew. */
  2756. /*
  2757. * Set the left and right edge of each bit to an illegal value.
  2758. * Use (seq->iocfg->io_out1_delay_max + 1) as an illegal value.
  2759. */
  2760. sticky_bit_chk = 0;
  2761. for (i = 0; i < seq->rwcfg->mem_dq_per_write_dqs; i++) {
  2762. left_edge[i] = seq->iocfg->io_out1_delay_max + 1;
  2763. right_edge[i] = seq->iocfg->io_out1_delay_max + 1;
  2764. }
  2765. /* Search for the left edge of the window for each bit. */
  2766. search_left_edge(seq, 1, rank_bgn, write_group, 0, test_bgn,
  2767. &sticky_bit_chk,
  2768. left_edge, right_edge, 0);
  2769. /* Search for the right edge of the window for each bit. */
  2770. ret = search_right_edge(seq, 1, rank_bgn, write_group, 0,
  2771. start_dqs, 0,
  2772. &sticky_bit_chk,
  2773. left_edge, right_edge, 0);
  2774. if (ret) {
  2775. set_failing_group_stage(seq, test_bgn + ret - 1,
  2776. CAL_STAGE_WRITES,
  2777. CAL_SUBSTAGE_WRITES_CENTER);
  2778. return -EINVAL;
  2779. }
  2780. min_index = get_window_mid_index(seq, 1, left_edge, right_edge,
  2781. &mid_min);
  2782. /* Determine the amount we can change DQS (which is -mid_min). */
  2783. orig_mid_min = mid_min;
  2784. new_dqs = start_dqs;
  2785. mid_min = 0;
  2786. debug_cond(DLEVEL >= 1,
  2787. "%s:%d write_center: start_dqs=%d new_dqs=%d mid_min=%d\n",
  2788. __func__, __LINE__, start_dqs, new_dqs, mid_min);
  2789. /* Add delay to bring centre of all DQ windows to the same "level". */
  2790. center_dq_windows(seq, 1, left_edge, right_edge, mid_min, orig_mid_min,
  2791. min_index, 0, &dq_margin, &dqs_margin);
  2792. /* Move DQS */
  2793. scc_mgr_apply_group_dqs_io_and_oct_out1(seq, write_group, new_dqs);
  2794. writel(0, &sdr_scc_mgr->update);
  2795. /* Centre DM */
  2796. debug_cond(DLEVEL >= 2, "%s:%d write_center: DM\n", __func__, __LINE__);
  2797. /* Search for the/part of the window with DM shift. */
  2798. search_window(seq, 1, rank_bgn, write_group, &bgn_curr, &end_curr,
  2799. &bgn_best, &end_best, &win_best, 0);
  2800. /* Reset DM delay chains to 0. */
  2801. scc_mgr_apply_group_dm_out1_delay(seq, 0);
  2802. /*
  2803. * Check to see if the current window nudges up aganist 0 delay.
  2804. * If so we need to continue the search by shifting DQS otherwise DQS
  2805. * search begins as a new search.
  2806. */
  2807. if (end_curr != 0) {
  2808. bgn_curr = seq->iocfg->io_out1_delay_max + 1;
  2809. end_curr = seq->iocfg->io_out1_delay_max + 1;
  2810. }
  2811. /* Search for the/part of the window with DQS shifts. */
  2812. search_window(seq, 0, rank_bgn, write_group, &bgn_curr, &end_curr,
  2813. &bgn_best, &end_best, &win_best, new_dqs);
  2814. /* Assign left and right edge for cal and reporting. */
  2815. left_edge[0] = -1 * bgn_best;
  2816. right_edge[0] = end_best;
  2817. debug_cond(DLEVEL >= 2, "%s:%d dm_calib: left=%d right=%d\n",
  2818. __func__, __LINE__, left_edge[0], right_edge[0]);
  2819. /* Move DQS (back to orig). */
  2820. scc_mgr_apply_group_dqs_io_and_oct_out1(seq, write_group, new_dqs);
  2821. /* Move DM */
  2822. /* Find middle of window for the DM bit. */
  2823. mid = (left_edge[0] - right_edge[0]) / 2;
  2824. /* Only move right, since we are not moving DQS/DQ. */
  2825. if (mid < 0)
  2826. mid = 0;
  2827. /* dm_marign should fail if we never find a window. */
  2828. if (win_best == 0)
  2829. dm_margin = -1;
  2830. else
  2831. dm_margin = left_edge[0] - mid;
  2832. scc_mgr_apply_group_dm_out1_delay(seq, mid);
  2833. writel(0, &sdr_scc_mgr->update);
  2834. debug_cond(DLEVEL >= 2,
  2835. "%s:%d dm_calib: left=%d right=%d mid=%d dm_margin=%d\n",
  2836. __func__, __LINE__, left_edge[0], right_edge[0],
  2837. mid, dm_margin);
  2838. /* Export values. */
  2839. seq->gbl.fom_out += dq_margin + dqs_margin;
  2840. debug_cond(DLEVEL >= 2,
  2841. "%s:%d write_center: dq_margin=%d dqs_margin=%d dm_margin=%d\n",
  2842. __func__, __LINE__, dq_margin, dqs_margin, dm_margin);
  2843. /*
  2844. * Do not remove this line as it makes sure all of our
  2845. * decisions have been applied.
  2846. */
  2847. writel(0, &sdr_scc_mgr->update);
  2848. if ((dq_margin < 0) || (dqs_margin < 0) || (dm_margin < 0))
  2849. return -EINVAL;
  2850. return 0;
  2851. }
  2852. /**
  2853. * rw_mgr_mem_calibrate_writes() - Write Calibration Part One
  2854. * @rank_bgn: Rank number
  2855. * @group: Read/Write Group
  2856. * @test_bgn: Rank at which the test begins
  2857. *
  2858. * Stage 2: Write Calibration Part One.
  2859. *
  2860. * This function implements UniPHY calibration Stage 2, as explained in
  2861. * detail in Altera EMI_RM 2015.05.04 , "UniPHY Calibration Stages".
  2862. */
  2863. static int rw_mgr_mem_calibrate_writes(struct socfpga_sdrseq *seq,
  2864. const u32 rank_bgn, const u32 group,
  2865. const u32 test_bgn)
  2866. {
  2867. int ret;
  2868. /* Update info for sims */
  2869. debug("%s:%d %u %u\n", __func__, __LINE__, group, test_bgn);
  2870. reg_file_set_group(group);
  2871. reg_file_set_stage(CAL_STAGE_WRITES);
  2872. reg_file_set_sub_stage(CAL_SUBSTAGE_WRITES_CENTER);
  2873. ret = rw_mgr_mem_calibrate_writes_center(seq, rank_bgn, group,
  2874. test_bgn);
  2875. if (ret)
  2876. set_failing_group_stage(seq, group, CAL_STAGE_WRITES,
  2877. CAL_SUBSTAGE_WRITES_CENTER);
  2878. return ret;
  2879. }
  2880. /**
  2881. * mem_precharge_and_activate() - Precharge all banks and activate
  2882. *
  2883. * Precharge all banks and activate row 0 in bank "000..." and bank "111...".
  2884. */
  2885. static void mem_precharge_and_activate(struct socfpga_sdrseq *seq)
  2886. {
  2887. int r;
  2888. for (r = 0; r < seq->rwcfg->mem_number_of_ranks; r++) {
  2889. /* Set rank. */
  2890. set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_OFF);
  2891. /* Precharge all banks. */
  2892. writel(seq->rwcfg->precharge_all, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  2893. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  2894. writel(0x0F, &sdr_rw_load_mgr_regs->load_cntr0);
  2895. writel(seq->rwcfg->activate_0_and_1_wait1,
  2896. &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  2897. writel(0x0F, &sdr_rw_load_mgr_regs->load_cntr1);
  2898. writel(seq->rwcfg->activate_0_and_1_wait2,
  2899. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  2900. /* Activate rows. */
  2901. writel(seq->rwcfg->activate_0_and_1,
  2902. SDR_PHYGRP_RWMGRGRP_ADDRESS |
  2903. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  2904. }
  2905. }
  2906. /**
  2907. * mem_init_latency() - Configure memory RLAT and WLAT settings
  2908. *
  2909. * Configure memory RLAT and WLAT parameters.
  2910. */
  2911. static void mem_init_latency(struct socfpga_sdrseq *seq)
  2912. {
  2913. /*
  2914. * For AV/CV, LFIFO is hardened and always runs at full rate
  2915. * so max latency in AFI clocks, used here, is correspondingly
  2916. * smaller.
  2917. */
  2918. const u32 max_latency = (1 << seq->misccfg->max_latency_count_width)
  2919. - 1;
  2920. u32 rlat, wlat;
  2921. debug("%s:%d\n", __func__, __LINE__);
  2922. /*
  2923. * Read in write latency.
  2924. * WL for Hard PHY does not include additive latency.
  2925. */
  2926. wlat = readl(&data_mgr->t_wl_add);
  2927. wlat += readl(&data_mgr->mem_t_add);
  2928. seq->gbl.rw_wl_nop_cycles = wlat - 1;
  2929. /* Read in readl latency. */
  2930. rlat = readl(&data_mgr->t_rl_add);
  2931. /* Set a pretty high read latency initially. */
  2932. seq->gbl.curr_read_lat = rlat + 16;
  2933. if (seq->gbl.curr_read_lat > max_latency)
  2934. seq->gbl.curr_read_lat = max_latency;
  2935. writel(seq->gbl.curr_read_lat, &phy_mgr_cfg->phy_rlat);
  2936. /* Advertise write latency. */
  2937. writel(wlat, &phy_mgr_cfg->afi_wlat);
  2938. }
  2939. /**
  2940. * @mem_skip_calibrate() - Set VFIFO and LFIFO to instant-on settings
  2941. *
  2942. * Set VFIFO and LFIFO to instant-on settings in skip calibration mode.
  2943. */
  2944. static void mem_skip_calibrate(struct socfpga_sdrseq *seq)
  2945. {
  2946. u32 vfifo_offset;
  2947. u32 i, j, r;
  2948. debug("%s:%d\n", __func__, __LINE__);
  2949. /* Need to update every shadow register set used by the interface */
  2950. for (r = 0; r < seq->rwcfg->mem_number_of_ranks;
  2951. r += NUM_RANKS_PER_SHADOW_REG) {
  2952. /*
  2953. * Set output phase alignment settings appropriate for
  2954. * skip calibration.
  2955. */
  2956. for (i = 0; i < seq->rwcfg->mem_if_read_dqs_width; i++) {
  2957. scc_mgr_set_dqs_en_phase(i, 0);
  2958. if (seq->iocfg->dll_chain_length == 6)
  2959. scc_mgr_set_dqdqs_output_phase(i, 6);
  2960. else
  2961. scc_mgr_set_dqdqs_output_phase(i, 7);
  2962. /*
  2963. * Case:33398
  2964. *
  2965. * Write data arrives to the I/O two cycles before write
  2966. * latency is reached (720 deg).
  2967. * -> due to bit-slip in a/c bus
  2968. * -> to allow board skew where dqs is longer than ck
  2969. * -> how often can this happen!?
  2970. * -> can claim back some ptaps for high freq
  2971. * support if we can relax this, but i digress...
  2972. *
  2973. * The write_clk leads mem_ck by 90 deg
  2974. * The minimum ptap of the OPA is 180 deg
  2975. * Each ptap has (360 / IO_DLL_CHAIN_LENGH) deg of delay
  2976. * The write_clk is always delayed by 2 ptaps
  2977. *
  2978. * Hence, to make DQS aligned to CK, we need to delay
  2979. * DQS by:
  2980. * (720 - 90 - 180 - 2) *
  2981. * (360 / seq->iocfg->dll_chain_length)
  2982. *
  2983. * Dividing the above by
  2984. (360 / seq->iocfg->dll_chain_length)
  2985. * gives us the number of ptaps, which simplies to:
  2986. *
  2987. * (1.25 * seq->iocfg->dll_chain_length - 2)
  2988. */
  2989. scc_mgr_set_dqdqs_output_phase(i,
  2990. ((125 * seq->iocfg->dll_chain_length)
  2991. / 100) - 2);
  2992. }
  2993. writel(0xff, &sdr_scc_mgr->dqs_ena);
  2994. writel(0xff, &sdr_scc_mgr->dqs_io_ena);
  2995. for (i = 0; i < seq->rwcfg->mem_if_write_dqs_width; i++) {
  2996. writel(i, SDR_PHYGRP_SCCGRP_ADDRESS |
  2997. SCC_MGR_GROUP_COUNTER_OFFSET);
  2998. }
  2999. writel(0xff, &sdr_scc_mgr->dq_ena);
  3000. writel(0xff, &sdr_scc_mgr->dm_ena);
  3001. writel(0, &sdr_scc_mgr->update);
  3002. }
  3003. /* Compensate for simulation model behaviour */
  3004. for (i = 0; i < seq->rwcfg->mem_if_read_dqs_width; i++) {
  3005. scc_mgr_set_dqs_bus_in_delay(i, 10);
  3006. scc_mgr_load_dqs(i);
  3007. }
  3008. writel(0, &sdr_scc_mgr->update);
  3009. /*
  3010. * ArriaV has hard FIFOs that can only be initialized by incrementing
  3011. * in sequencer.
  3012. */
  3013. vfifo_offset = seq->misccfg->calib_vfifo_offset;
  3014. for (j = 0; j < vfifo_offset; j++)
  3015. writel(0xff, &phy_mgr_cmd->inc_vfifo_hard_phy);
  3016. writel(0, &phy_mgr_cmd->fifo_reset);
  3017. /*
  3018. * For Arria V and Cyclone V with hard LFIFO, we get the skip-cal
  3019. * setting from generation-time constant.
  3020. */
  3021. seq->gbl.curr_read_lat = seq->misccfg->calib_lfifo_offset;
  3022. writel(seq->gbl.curr_read_lat, &phy_mgr_cfg->phy_rlat);
  3023. }
  3024. /**
  3025. * mem_calibrate() - Memory calibration entry point.
  3026. *
  3027. * Perform memory calibration.
  3028. */
  3029. static u32 mem_calibrate(struct socfpga_sdrseq *seq)
  3030. {
  3031. u32 i;
  3032. u32 rank_bgn, sr;
  3033. u32 write_group, write_test_bgn;
  3034. u32 read_group, read_test_bgn;
  3035. u32 run_groups, current_run;
  3036. u32 failing_groups = 0;
  3037. u32 group_failed = 0;
  3038. const u32 rwdqs_ratio = seq->rwcfg->mem_if_read_dqs_width /
  3039. seq->rwcfg->mem_if_write_dqs_width;
  3040. debug("%s:%d\n", __func__, __LINE__);
  3041. /* Initialize the data settings */
  3042. seq->gbl.error_substage = CAL_SUBSTAGE_NIL;
  3043. seq->gbl.error_stage = CAL_STAGE_NIL;
  3044. seq->gbl.error_group = 0xff;
  3045. seq->gbl.fom_in = 0;
  3046. seq->gbl.fom_out = 0;
  3047. /* Initialize WLAT and RLAT. */
  3048. mem_init_latency(seq);
  3049. /* Initialize bit slips. */
  3050. mem_precharge_and_activate(seq);
  3051. for (i = 0; i < seq->rwcfg->mem_if_read_dqs_width; i++) {
  3052. writel(i, SDR_PHYGRP_SCCGRP_ADDRESS |
  3053. SCC_MGR_GROUP_COUNTER_OFFSET);
  3054. /* Only needed once to set all groups, pins, DQ, DQS, DM. */
  3055. if (i == 0)
  3056. scc_mgr_set_hhp_extras();
  3057. scc_set_bypass_mode(i);
  3058. }
  3059. /* Calibration is skipped. */
  3060. if ((seq->dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL) {
  3061. /*
  3062. * Set VFIFO and LFIFO to instant-on settings in skip
  3063. * calibration mode.
  3064. */
  3065. mem_skip_calibrate(seq);
  3066. /*
  3067. * Do not remove this line as it makes sure all of our
  3068. * decisions have been applied.
  3069. */
  3070. writel(0, &sdr_scc_mgr->update);
  3071. return 1;
  3072. }
  3073. /* Calibration is not skipped. */
  3074. for (i = 0; i < NUM_CALIB_REPEAT; i++) {
  3075. /*
  3076. * Zero all delay chain/phase settings for all
  3077. * groups and all shadow register sets.
  3078. */
  3079. scc_mgr_zero_all(seq);
  3080. run_groups = ~0;
  3081. for (write_group = 0, write_test_bgn = 0; write_group
  3082. < seq->rwcfg->mem_if_write_dqs_width; write_group++,
  3083. write_test_bgn += seq->rwcfg->mem_dq_per_write_dqs) {
  3084. /* Initialize the group failure */
  3085. group_failed = 0;
  3086. current_run = run_groups & ((1 <<
  3087. RW_MGR_NUM_DQS_PER_WRITE_GROUP) - 1);
  3088. run_groups = run_groups >>
  3089. RW_MGR_NUM_DQS_PER_WRITE_GROUP;
  3090. if (current_run == 0)
  3091. continue;
  3092. writel(write_group, SDR_PHYGRP_SCCGRP_ADDRESS |
  3093. SCC_MGR_GROUP_COUNTER_OFFSET);
  3094. scc_mgr_zero_group(seq, write_group, 0);
  3095. for (read_group = write_group * rwdqs_ratio,
  3096. read_test_bgn = 0;
  3097. read_group < (write_group + 1) * rwdqs_ratio;
  3098. read_group++,
  3099. read_test_bgn += seq->rwcfg->mem_dq_per_read_dqs) {
  3100. if (STATIC_CALIB_STEPS & CALIB_SKIP_VFIFO)
  3101. continue;
  3102. /* Calibrate the VFIFO */
  3103. if (rw_mgr_mem_calibrate_vfifo(seq, read_group,
  3104. read_test_bgn))
  3105. continue;
  3106. if (!(seq->gbl.phy_debug_mode_flags &
  3107. PHY_DEBUG_SWEEP_ALL_GROUPS))
  3108. return 0;
  3109. /* The group failed, we're done. */
  3110. goto grp_failed;
  3111. }
  3112. /* Calibrate the output side */
  3113. for (rank_bgn = 0, sr = 0;
  3114. rank_bgn < seq->rwcfg->mem_number_of_ranks;
  3115. rank_bgn += NUM_RANKS_PER_SHADOW_REG, sr++) {
  3116. if (STATIC_CALIB_STEPS & CALIB_SKIP_WRITES)
  3117. continue;
  3118. /* Not needed in quick mode! */
  3119. if (STATIC_CALIB_STEPS &
  3120. CALIB_SKIP_DELAY_SWEEPS)
  3121. continue;
  3122. /* Calibrate WRITEs */
  3123. if (!rw_mgr_mem_calibrate_writes(seq, rank_bgn,
  3124. write_group,
  3125. write_test_bgn))
  3126. continue;
  3127. group_failed = 1;
  3128. if (!(seq->gbl.phy_debug_mode_flags &
  3129. PHY_DEBUG_SWEEP_ALL_GROUPS))
  3130. return 0;
  3131. }
  3132. /* Some group failed, we're done. */
  3133. if (group_failed)
  3134. goto grp_failed;
  3135. for (read_group = write_group * rwdqs_ratio,
  3136. read_test_bgn = 0;
  3137. read_group < (write_group + 1) * rwdqs_ratio;
  3138. read_group++,
  3139. read_test_bgn += seq->rwcfg->mem_dq_per_read_dqs) {
  3140. if (STATIC_CALIB_STEPS & CALIB_SKIP_WRITES)
  3141. continue;
  3142. if (!rw_mgr_mem_calibrate_vfifo_end(seq,
  3143. read_group,
  3144. read_test_bgn))
  3145. continue;
  3146. if (!(seq->gbl.phy_debug_mode_flags &
  3147. PHY_DEBUG_SWEEP_ALL_GROUPS))
  3148. return 0;
  3149. /* The group failed, we're done. */
  3150. goto grp_failed;
  3151. }
  3152. /* No group failed, continue as usual. */
  3153. continue;
  3154. grp_failed: /* A group failed, increment the counter. */
  3155. failing_groups++;
  3156. }
  3157. /*
  3158. * USER If there are any failing groups then report
  3159. * the failure.
  3160. */
  3161. if (failing_groups != 0)
  3162. return 0;
  3163. if (STATIC_CALIB_STEPS & CALIB_SKIP_LFIFO)
  3164. continue;
  3165. /* Calibrate the LFIFO */
  3166. if (!rw_mgr_mem_calibrate_lfifo(seq))
  3167. return 0;
  3168. }
  3169. /*
  3170. * Do not remove this line as it makes sure all of our decisions
  3171. * have been applied.
  3172. */
  3173. writel(0, &sdr_scc_mgr->update);
  3174. return 1;
  3175. }
  3176. /**
  3177. * run_mem_calibrate() - Perform memory calibration
  3178. *
  3179. * This function triggers the entire memory calibration procedure.
  3180. */
  3181. static int run_mem_calibrate(struct socfpga_sdrseq *seq)
  3182. {
  3183. int pass;
  3184. u32 ctrl_cfg;
  3185. debug("%s:%d\n", __func__, __LINE__);
  3186. /* Reset pass/fail status shown on afi_cal_success/fail */
  3187. writel(PHY_MGR_CAL_RESET, &phy_mgr_cfg->cal_status);
  3188. /* Stop tracking manager. */
  3189. ctrl_cfg = readl(&sdr_ctrl->ctrl_cfg);
  3190. writel(ctrl_cfg & ~SDR_CTRLGRP_CTRLCFG_DQSTRKEN_MASK,
  3191. &sdr_ctrl->ctrl_cfg);
  3192. phy_mgr_initialize(seq);
  3193. rw_mgr_mem_initialize(seq);
  3194. /* Perform the actual memory calibration. */
  3195. pass = mem_calibrate(seq);
  3196. mem_precharge_and_activate(seq);
  3197. writel(0, &phy_mgr_cmd->fifo_reset);
  3198. /* Handoff. */
  3199. rw_mgr_mem_handoff(seq);
  3200. /*
  3201. * In Hard PHY this is a 2-bit control:
  3202. * 0: AFI Mux Select
  3203. * 1: DDIO Mux Select
  3204. */
  3205. writel(0x2, &phy_mgr_cfg->mux_sel);
  3206. /* Start tracking manager. */
  3207. writel(ctrl_cfg, &sdr_ctrl->ctrl_cfg);
  3208. return pass;
  3209. }
  3210. /**
  3211. * debug_mem_calibrate() - Report result of memory calibration
  3212. * @pass: Value indicating whether calibration passed or failed
  3213. *
  3214. * This function reports the results of the memory calibration
  3215. * and writes debug information into the register file.
  3216. */
  3217. static void debug_mem_calibrate(struct socfpga_sdrseq *seq, int pass)
  3218. {
  3219. u32 debug_info;
  3220. if (pass) {
  3221. debug(KBUILD_BASENAME ": CALIBRATION PASSED\n");
  3222. seq->gbl.fom_in /= 2;
  3223. seq->gbl.fom_out /= 2;
  3224. if (seq->gbl.fom_in > 0xff)
  3225. seq->gbl.fom_in = 0xff;
  3226. if (seq->gbl.fom_out > 0xff)
  3227. seq->gbl.fom_out = 0xff;
  3228. /* Update the FOM in the register file */
  3229. debug_info = seq->gbl.fom_in;
  3230. debug_info |= seq->gbl.fom_out << 8;
  3231. writel(debug_info, &sdr_reg_file->fom);
  3232. writel(debug_info, &phy_mgr_cfg->cal_debug_info);
  3233. writel(PHY_MGR_CAL_SUCCESS, &phy_mgr_cfg->cal_status);
  3234. } else {
  3235. debug(KBUILD_BASENAME ": CALIBRATION FAILED\n");
  3236. debug_info = seq->gbl.error_stage;
  3237. debug_info |= seq->gbl.error_substage << 8;
  3238. debug_info |= seq->gbl.error_group << 16;
  3239. writel(debug_info, &sdr_reg_file->failing_stage);
  3240. writel(debug_info, &phy_mgr_cfg->cal_debug_info);
  3241. writel(PHY_MGR_CAL_FAIL, &phy_mgr_cfg->cal_status);
  3242. /* Update the failing group/stage in the register file */
  3243. debug_info = seq->gbl.error_stage;
  3244. debug_info |= seq->gbl.error_substage << 8;
  3245. debug_info |= seq->gbl.error_group << 16;
  3246. writel(debug_info, &sdr_reg_file->failing_stage);
  3247. }
  3248. debug(KBUILD_BASENAME ": Calibration complete\n");
  3249. }
  3250. /**
  3251. * hc_initialize_rom_data() - Initialize ROM data
  3252. *
  3253. * Initialize ROM data.
  3254. */
  3255. static void hc_initialize_rom_data(void)
  3256. {
  3257. unsigned int nelem = 0;
  3258. const u32 *rom_init;
  3259. u32 i, addr;
  3260. socfpga_get_seq_inst_init(&rom_init, &nelem);
  3261. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_INST_ROM_WRITE_OFFSET;
  3262. for (i = 0; i < nelem; i++)
  3263. writel(rom_init[i], addr + (i << 2));
  3264. socfpga_get_seq_ac_init(&rom_init, &nelem);
  3265. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_AC_ROM_WRITE_OFFSET;
  3266. for (i = 0; i < nelem; i++)
  3267. writel(rom_init[i], addr + (i << 2));
  3268. }
  3269. /**
  3270. * initialize_reg_file() - Initialize SDR register file
  3271. *
  3272. * Initialize SDR register file.
  3273. */
  3274. static void initialize_reg_file(struct socfpga_sdrseq *seq)
  3275. {
  3276. /* Initialize the register file with the correct data */
  3277. writel(seq->misccfg->reg_file_init_seq_signature,
  3278. &sdr_reg_file->signature);
  3279. writel(0, &sdr_reg_file->debug_data_addr);
  3280. writel(0, &sdr_reg_file->cur_stage);
  3281. writel(0, &sdr_reg_file->fom);
  3282. writel(0, &sdr_reg_file->failing_stage);
  3283. writel(0, &sdr_reg_file->debug1);
  3284. writel(0, &sdr_reg_file->debug2);
  3285. }
  3286. /**
  3287. * initialize_hps_phy() - Initialize HPS PHY
  3288. *
  3289. * Initialize HPS PHY.
  3290. */
  3291. static void initialize_hps_phy(void)
  3292. {
  3293. u32 reg;
  3294. /*
  3295. * Tracking also gets configured here because it's in the
  3296. * same register.
  3297. */
  3298. u32 trk_sample_count = 7500;
  3299. u32 trk_long_idle_sample_count = (10 << 16) | 100;
  3300. /*
  3301. * Format is number of outer loops in the 16 MSB, sample
  3302. * count in 16 LSB.
  3303. */
  3304. reg = 0;
  3305. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ACDELAYEN_SET(2);
  3306. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQDELAYEN_SET(1);
  3307. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSDELAYEN_SET(1);
  3308. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSLOGICDELAYEN_SET(1);
  3309. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_RESETDELAYEN_SET(0);
  3310. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_LPDDRDIS_SET(1);
  3311. /*
  3312. * This field selects the intrinsic latency to RDATA_EN/FULL path.
  3313. * 00-bypass, 01- add 5 cycles, 10- add 10 cycles, 11- add 15 cycles.
  3314. */
  3315. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ADDLATSEL_SET(0);
  3316. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_SET(
  3317. trk_sample_count);
  3318. writel(reg, &sdr_ctrl->phy_ctrl0);
  3319. reg = 0;
  3320. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_SAMPLECOUNT_31_20_SET(
  3321. trk_sample_count >>
  3322. SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_WIDTH);
  3323. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_SET(
  3324. trk_long_idle_sample_count);
  3325. writel(reg, &sdr_ctrl->phy_ctrl1);
  3326. reg = 0;
  3327. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_2_LONGIDLESAMPLECOUNT_31_20_SET(
  3328. trk_long_idle_sample_count >>
  3329. SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_WIDTH);
  3330. writel(reg, &sdr_ctrl->phy_ctrl2);
  3331. }
  3332. /**
  3333. * initialize_tracking() - Initialize tracking
  3334. *
  3335. * Initialize the register file with usable initial data.
  3336. */
  3337. static void initialize_tracking(struct socfpga_sdrseq *seq)
  3338. {
  3339. /*
  3340. * Initialize the register file with the correct data.
  3341. * Compute usable version of value in case we skip full
  3342. * computation later.
  3343. */
  3344. writel(DIV_ROUND_UP(seq->iocfg->delay_per_opa_tap,
  3345. seq->iocfg->delay_per_dchain_tap) - 1,
  3346. &sdr_reg_file->dtaps_per_ptap);
  3347. /* trk_sample_count */
  3348. writel(7500, &sdr_reg_file->trk_sample_count);
  3349. /* longidle outer loop [15:0] */
  3350. writel((10 << 16) | (100 << 0), &sdr_reg_file->trk_longidle);
  3351. /*
  3352. * longidle sample count [31:24]
  3353. * trfc, worst case of 933Mhz 4Gb [23:16]
  3354. * trcd, worst case [15:8]
  3355. * vfifo wait [7:0]
  3356. */
  3357. writel((243 << 24) | (14 << 16) | (10 << 8) | (4 << 0),
  3358. &sdr_reg_file->delays);
  3359. /* mux delay */
  3360. if (dram_is_ddr(2)) {
  3361. writel(0, &sdr_reg_file->trk_rw_mgr_addr);
  3362. } else if (dram_is_ddr(3)) {
  3363. writel((seq->rwcfg->idle << 24) |
  3364. (seq->rwcfg->activate_1 << 16) |
  3365. (seq->rwcfg->sgle_read << 8) |
  3366. (seq->rwcfg->precharge_all << 0),
  3367. &sdr_reg_file->trk_rw_mgr_addr);
  3368. }
  3369. writel(seq->rwcfg->mem_if_read_dqs_width,
  3370. &sdr_reg_file->trk_read_dqs_width);
  3371. /* trefi [7:0] */
  3372. if (dram_is_ddr(2)) {
  3373. writel(1000 << 0, &sdr_reg_file->trk_rfsh);
  3374. } else if (dram_is_ddr(3)) {
  3375. writel((seq->rwcfg->refresh_all << 24) | (1000 << 0),
  3376. &sdr_reg_file->trk_rfsh);
  3377. }
  3378. }
  3379. int sdram_calibration_full(struct socfpga_sdr *sdr)
  3380. {
  3381. u32 pass;
  3382. struct socfpga_sdrseq seq;
  3383. /*
  3384. * For size reasons, this file uses hard coded addresses.
  3385. * Check if we are called with the correct address.
  3386. */
  3387. if (sdr != (struct socfpga_sdr *)SOCFPGA_SDR_ADDRESS)
  3388. return -ENODEV;
  3389. memset(&seq, 0, sizeof(seq));
  3390. seq.rwcfg = socfpga_get_sdram_rwmgr_config();
  3391. seq.iocfg = socfpga_get_sdram_io_config();
  3392. seq.misccfg = socfpga_get_sdram_misc_config();
  3393. /* Set the calibration enabled by default */
  3394. seq.gbl.phy_debug_mode_flags |= PHY_DEBUG_ENABLE_CAL_RPT;
  3395. /*
  3396. * Only sweep all groups (regardless of fail state) by default
  3397. * Set enabled read test by default.
  3398. */
  3399. #if DISABLE_GUARANTEED_READ
  3400. seq.gbl.phy_debug_mode_flags |= PHY_DEBUG_DISABLE_GUARANTEED_READ;
  3401. #endif
  3402. /* Initialize the register file */
  3403. initialize_reg_file(&seq);
  3404. /* Initialize any PHY CSR */
  3405. initialize_hps_phy();
  3406. scc_mgr_initialize();
  3407. initialize_tracking(&seq);
  3408. debug(KBUILD_BASENAME ": Preparing to start memory calibration\n");
  3409. debug("%s:%d\n", __func__, __LINE__);
  3410. debug_cond(DLEVEL >= 1,
  3411. "DDR3 FULL_RATE ranks=%u cs/dimm=%u dq/dqs=%u,%u vg/dqs=%u,%u ",
  3412. seq.rwcfg->mem_number_of_ranks,
  3413. seq.rwcfg->mem_number_of_cs_per_dimm,
  3414. seq.rwcfg->mem_dq_per_read_dqs,
  3415. seq.rwcfg->mem_dq_per_write_dqs,
  3416. seq.rwcfg->mem_virtual_groups_per_read_dqs,
  3417. seq.rwcfg->mem_virtual_groups_per_write_dqs);
  3418. debug_cond(DLEVEL >= 1,
  3419. "dqs=%u,%u dq=%u dm=%u ptap_delay=%u dtap_delay=%u ",
  3420. seq.rwcfg->mem_if_read_dqs_width,
  3421. seq.rwcfg->mem_if_write_dqs_width,
  3422. seq.rwcfg->mem_data_width, seq.rwcfg->mem_data_mask_width,
  3423. seq.iocfg->delay_per_opa_tap,
  3424. seq.iocfg->delay_per_dchain_tap);
  3425. debug_cond(DLEVEL >= 1, "dtap_dqsen_delay=%u, dll=%u",
  3426. seq.iocfg->delay_per_dqs_en_dchain_tap,
  3427. seq.iocfg->dll_chain_length);
  3428. debug_cond(DLEVEL >= 1,
  3429. "max values: en_p=%u dqdqs_p=%u en_d=%u dqs_in_d=%u ",
  3430. seq.iocfg->dqs_en_phase_max, seq.iocfg->dqdqs_out_phase_max,
  3431. seq.iocfg->dqs_en_delay_max, seq.iocfg->dqs_in_delay_max);
  3432. debug_cond(DLEVEL >= 1, "io_in_d=%u io_out1_d=%u io_out2_d=%u ",
  3433. seq.iocfg->io_in_delay_max, seq.iocfg->io_out1_delay_max,
  3434. seq.iocfg->io_out2_delay_max);
  3435. debug_cond(DLEVEL >= 1, "dqs_in_reserve=%u dqs_out_reserve=%u\n",
  3436. seq.iocfg->dqs_in_reserve, seq.iocfg->dqs_out_reserve);
  3437. hc_initialize_rom_data();
  3438. /* update info for sims */
  3439. reg_file_set_stage(CAL_STAGE_NIL);
  3440. reg_file_set_group(0);
  3441. /*
  3442. * Load global needed for those actions that require
  3443. * some dynamic calibration support.
  3444. */
  3445. seq.dyn_calib_steps = STATIC_CALIB_STEPS;
  3446. /*
  3447. * Load global to allow dynamic selection of delay loop settings
  3448. * based on calibration mode.
  3449. */
  3450. if (!(seq.dyn_calib_steps & CALIB_SKIP_DELAY_LOOPS))
  3451. seq.skip_delay_mask = 0xff;
  3452. else
  3453. seq.skip_delay_mask = 0x0;
  3454. pass = run_mem_calibrate(&seq);
  3455. debug_mem_calibrate(&seq, pass);
  3456. return pass;
  3457. }