ddr3_write_leveling.c 39 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366
  1. /*
  2. * Copyright (C) Marvell International Ltd. and its affiliates
  3. *
  4. * SPDX-License-Identifier: GPL-2.0
  5. */
  6. #include <common.h>
  7. #include <i2c.h>
  8. #include <spl.h>
  9. #include <asm/io.h>
  10. #include <asm/arch/cpu.h>
  11. #include <asm/arch/soc.h>
  12. #include "ddr3_hw_training.h"
  13. /*
  14. * Debug
  15. */
  16. #define DEBUG_WL_C(s, d, l) \
  17. DEBUG_WL_S(s); DEBUG_WL_D(d, l); DEBUG_WL_S("\n")
  18. #define DEBUG_WL_FULL_C(s, d, l) \
  19. DEBUG_WL_FULL_S(s); DEBUG_WL_FULL_D(d, l); DEBUG_WL_FULL_S("\n")
  20. #ifdef MV_DEBUG_WL
  21. #define DEBUG_WL_S(s) puts(s)
  22. #define DEBUG_WL_D(d, l) printf("%x", d)
  23. #define DEBUG_RL_S(s) \
  24. debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%s", s)
  25. #define DEBUG_RL_D(d, l) \
  26. debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%x", d)
  27. #else
  28. #define DEBUG_WL_S(s)
  29. #define DEBUG_WL_D(d, l)
  30. #endif
  31. #ifdef MV_DEBUG_WL_FULL
  32. #define DEBUG_WL_FULL_S(s) puts(s)
  33. #define DEBUG_WL_FULL_D(d, l) printf("%x", d)
  34. #else
  35. #define DEBUG_WL_FULL_S(s)
  36. #define DEBUG_WL_FULL_D(d, l)
  37. #endif
  38. #define WL_SUP_EXPECTED_DATA 0x21
  39. #define WL_SUP_READ_DRAM_ENTRY 0x8
  40. static int ddr3_write_leveling_single_cs(u32 cs, u32 freq, int ratio_2to1,
  41. u32 *result,
  42. MV_DRAM_INFO *dram_info);
  43. static void ddr3_write_ctrl_pup_reg(int bc_acc, u32 pup, u32 reg_addr,
  44. u32 data);
  45. extern u16 odt_static[ODT_OPT][MAX_CS];
  46. extern u16 odt_dynamic[ODT_OPT][MAX_CS];
  47. extern u32 wl_sup_pattern[LEN_WL_SUP_PATTERN];
  48. /*
  49. * Name: ddr3_write_leveling_hw
  50. * Desc: Execute Write leveling phase by HW
  51. * Args: freq - current sequence frequency
  52. * dram_info - main struct
  53. * Notes:
  54. * Returns: MV_OK if success, MV_FAIL if fail.
  55. */
  56. int ddr3_write_leveling_hw(u32 freq, MV_DRAM_INFO *dram_info)
  57. {
  58. u32 reg, phase, delay, cs, pup;
  59. #ifdef MV88F67XX
  60. int dpde_flag = 0;
  61. #endif
  62. /* Debug message - Start Read leveling procedure */
  63. DEBUG_WL_S("DDR3 - Write Leveling - Starting HW WL procedure\n");
  64. #ifdef MV88F67XX
  65. /* Dynamic pad issue (BTS669) during WL */
  66. reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
  67. if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
  68. dpde_flag = 1;
  69. reg_write(REG_DUNIT_CTRL_LOW_ADDR,
  70. reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
  71. }
  72. #endif
  73. reg = 1 << REG_DRAM_TRAINING_WL_OFFS;
  74. /* Config the retest number */
  75. reg |= (COUNT_HW_WL << REG_DRAM_TRAINING_RETEST_OFFS);
  76. reg |= (dram_info->cs_ena << (REG_DRAM_TRAINING_CS_OFFS));
  77. reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
  78. reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) |
  79. (1 << REG_DRAM_TRAINING_AUTO_OFFS);
  80. reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg);
  81. /* Wait */
  82. do {
  83. reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
  84. (1 << REG_DRAM_TRAINING_AUTO_OFFS);
  85. } while (reg); /* Wait for '0' */
  86. reg = reg_read(REG_DRAM_TRAINING_ADDR);
  87. /* Check if Successful */
  88. if (reg & (1 << REG_DRAM_TRAINING_ERROR_OFFS)) {
  89. /*
  90. * Read results to arrays - Results are required for WL
  91. * High freq Supplement and DQS Centralization
  92. */
  93. for (cs = 0; cs < MAX_CS; cs++) {
  94. if (dram_info->cs_ena & (1 << cs)) {
  95. for (pup = 0;
  96. pup < dram_info->num_of_total_pups;
  97. pup++) {
  98. if (pup == dram_info->num_of_std_pups
  99. && dram_info->ecc_ena)
  100. pup = ECC_PUP;
  101. reg =
  102. ddr3_read_pup_reg(PUP_WL_MODE, cs,
  103. pup);
  104. phase =
  105. (reg >> REG_PHY_PHASE_OFFS) &
  106. PUP_PHASE_MASK;
  107. delay = reg & PUP_DELAY_MASK;
  108. dram_info->wl_val[cs][pup][P] = phase;
  109. dram_info->wl_val[cs][pup][D] = delay;
  110. dram_info->wl_val[cs][pup][S] =
  111. WL_HI_FREQ_STATE - 1;
  112. reg =
  113. ddr3_read_pup_reg(PUP_WL_MODE + 0x1,
  114. cs, pup);
  115. dram_info->wl_val[cs][pup][DQS] =
  116. (reg & 0x3F);
  117. }
  118. #ifdef MV_DEBUG_WL
  119. /* Debug message - Print res for cs[i]: cs,PUP,Phase,Delay */
  120. DEBUG_WL_S("DDR3 - Write Leveling - Write Leveling Cs - ");
  121. DEBUG_WL_D((u32) cs, 1);
  122. DEBUG_WL_S(" Results:\n");
  123. for (pup = 0;
  124. pup < dram_info->num_of_total_pups;
  125. pup++) {
  126. if (pup == dram_info->num_of_std_pups
  127. && dram_info->ecc_ena)
  128. pup = ECC_PUP;
  129. DEBUG_WL_S("DDR3 - Write Leveling - PUP: ");
  130. DEBUG_WL_D((u32) pup, 1);
  131. DEBUG_WL_S(", Phase: ");
  132. DEBUG_WL_D((u32)
  133. dram_info->wl_val[cs][pup]
  134. [P], 1);
  135. DEBUG_WL_S(", Delay: ");
  136. DEBUG_WL_D((u32)
  137. dram_info->wl_val[cs][pup]
  138. [D], 2);
  139. DEBUG_WL_S("\n");
  140. }
  141. #endif
  142. }
  143. }
  144. /* Dynamic pad issue (BTS669) during WL */
  145. #ifdef MV88F67XX
  146. if (dpde_flag) {
  147. reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
  148. (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
  149. reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
  150. }
  151. #endif
  152. DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
  153. return MV_OK;
  154. } else {
  155. DEBUG_WL_S("DDR3 - Write Leveling - HW WL Error\n");
  156. return MV_FAIL;
  157. }
  158. }
  159. /*
  160. * Name: ddr3_wl_supplement
  161. * Desc: Write Leveling Supplement
  162. * Args: dram_info - main struct
  163. * Notes:
  164. * Returns: MV_OK if success, MV_FAIL if fail.
  165. */
  166. int ddr3_wl_supplement(MV_DRAM_INFO *dram_info)
  167. {
  168. u32 cs, cnt, pup_num, sum, phase, delay, max_pup_num, pup, sdram_offset;
  169. u32 tmp_count, ecc, reg;
  170. u32 ddr_width, tmp_pup, idx;
  171. u32 sdram_pup_val, uj;
  172. u32 one_clk_err = 0, align_err = 0, no_err = 0, err = 0, err_n = 0;
  173. u32 sdram_data[LEN_WL_SUP_PATTERN] __aligned(32) = { 0 };
  174. ddr_width = dram_info->ddr_width;
  175. no_err = 0;
  176. DEBUG_WL_S("DDR3 - Write Leveling Hi-Freq Supplement - Starting\n");
  177. switch (ddr_width) {
  178. /* Data error from pos-adge to pos-adge */
  179. case 16:
  180. one_clk_err = 4;
  181. align_err = 4;
  182. break;
  183. case 32:
  184. one_clk_err = 8;
  185. align_err = 8;
  186. break;
  187. case 64:
  188. one_clk_err = 0x10;
  189. align_err = 0x10;
  190. break;
  191. default:
  192. DEBUG_WL_S("Error - bus width!!!\n");
  193. return MV_FAIL;
  194. }
  195. /* Enable SW override */
  196. reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
  197. (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
  198. /* [0] = 1 - Enable SW override */
  199. /* 0x15B8 - Training SW 2 Register */
  200. reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
  201. DEBUG_WL_S("DDR3 - Write Leveling Hi-Freq Supplement - SW Override Enabled\n");
  202. reg = (1 << REG_DRAM_TRAINING_AUTO_OFFS);
  203. reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
  204. tmp_count = 0;
  205. for (cs = 0; cs < MAX_CS; cs++) {
  206. if (dram_info->cs_ena & (1 << cs)) {
  207. sum = 0;
  208. /*
  209. * 2 iterations loop: 1)actual WL results 2) fix WL
  210. * if needed
  211. */
  212. for (cnt = 0; cnt < COUNT_WL_HI_FREQ; cnt++) {
  213. DEBUG_WL_C("COUNT = ", cnt, 1);
  214. for (ecc = 0; ecc < (dram_info->ecc_ena + 1);
  215. ecc++) {
  216. if (ecc) {
  217. DEBUG_WL_S("ECC PUP:\n");
  218. } else {
  219. DEBUG_WL_S("DATA PUP:\n");
  220. }
  221. max_pup_num =
  222. dram_info->num_of_std_pups * (1 -
  223. ecc) +
  224. ecc;
  225. /* ECC Support - Switch ECC Mux on ecc=1 */
  226. reg =
  227. (reg_read(REG_DRAM_TRAINING_2_ADDR)
  228. & ~(1 <<
  229. REG_DRAM_TRAINING_2_ECC_MUX_OFFS));
  230. reg |=
  231. (dram_info->ecc_ena *
  232. ecc <<
  233. REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
  234. reg_write(REG_DRAM_TRAINING_2_ADDR,
  235. reg);
  236. ddr3_reset_phy_read_fifo();
  237. /* Write to memory */
  238. sdram_offset =
  239. tmp_count * (SDRAM_CS_SIZE + 1) +
  240. 0x200;
  241. if (MV_OK != ddr3_dram_sram_burst((u32)
  242. wl_sup_pattern,
  243. sdram_offset,
  244. LEN_WL_SUP_PATTERN))
  245. return MV_FAIL;
  246. /* Read from memory */
  247. if (MV_OK !=
  248. ddr3_dram_sram_burst(sdram_offset,
  249. (u32)
  250. sdram_data,
  251. LEN_WL_SUP_PATTERN))
  252. return MV_FAIL;
  253. /* Print the buffer */
  254. for (uj = 0; uj < LEN_WL_SUP_PATTERN;
  255. uj++) {
  256. if ((uj % 4 == 0) && (uj != 0)) {
  257. DEBUG_WL_S("\n");
  258. }
  259. DEBUG_WL_D(sdram_data[uj],
  260. 8);
  261. DEBUG_WL_S(" ");
  262. }
  263. /* Check pup which DQS/DATA is error */
  264. for (pup = 0; pup < max_pup_num; pup++) {
  265. /* ECC support - bit 8 */
  266. pup_num = (ecc) ? ECC_PUP : pup;
  267. if (pup < 4) { /* lower 32 bit */
  268. tmp_pup = pup;
  269. idx =
  270. WL_SUP_READ_DRAM_ENTRY;
  271. } else { /* higher 32 bit */
  272. tmp_pup = pup - 4;
  273. idx =
  274. WL_SUP_READ_DRAM_ENTRY
  275. + 1;
  276. }
  277. DEBUG_WL_S("\nCS: ");
  278. DEBUG_WL_D((u32) cs, 1);
  279. DEBUG_WL_S(" PUP: ");
  280. DEBUG_WL_D((u32) pup_num, 1);
  281. DEBUG_WL_S("\n");
  282. sdram_pup_val =
  283. ((sdram_data[idx] >>
  284. ((tmp_pup) * 8)) & 0xFF);
  285. DEBUG_WL_C("Actual Data = ",
  286. sdram_pup_val, 2);
  287. DEBUG_WL_C("Expected Data = ",
  288. (WL_SUP_EXPECTED_DATA
  289. + pup), 2);
  290. /*
  291. * ALINGHMENT: calculate
  292. * expected data vs actual data
  293. */
  294. err =
  295. (WL_SUP_EXPECTED_DATA +
  296. pup) - sdram_pup_val;
  297. /*
  298. * CLOCK LONG: calculate
  299. * expected data vs actual data
  300. */
  301. err_n =
  302. sdram_pup_val -
  303. (WL_SUP_EXPECTED_DATA +
  304. pup);
  305. DEBUG_WL_C("err = ", err, 2);
  306. DEBUG_WL_C("err_n = ", err_n,
  307. 2);
  308. if (err == no_err) {
  309. /* PUP is correct - increment State */
  310. dram_info->wl_val[cs]
  311. [pup_num]
  312. [S] = 1;
  313. } else if (err_n == one_clk_err) {
  314. /* clock is longer than DQS */
  315. phase =
  316. ((dram_info->wl_val
  317. [cs]
  318. [pup_num][P] +
  319. WL_HI_FREQ_SHIFT)
  320. % MAX_PHASE_2TO1);
  321. dram_info->wl_val[cs]
  322. [pup_num]
  323. [P] = phase;
  324. delay =
  325. dram_info->wl_val
  326. [cs][pup_num]
  327. [D];
  328. DEBUG_WL_S("#### Clock is longer than DQS more than one clk cycle ####\n");
  329. ddr3_write_pup_reg
  330. (PUP_WL_MODE, cs,
  331. pup * (1 - ecc) +
  332. ECC_PUP * ecc,
  333. phase, delay);
  334. } else if (err == align_err) {
  335. /* clock is align to DQS */
  336. phase =
  337. dram_info->wl_val
  338. [cs][pup_num]
  339. [P];
  340. delay =
  341. dram_info->wl_val
  342. [cs][pup_num]
  343. [D];
  344. DEBUG_WL_S("#### Alignment PUPS problem ####\n");
  345. if ((phase == 0)
  346. || ((phase == 1)
  347. && (delay <=
  348. 0x10))) {
  349. DEBUG_WL_S("#### Warning - Possible Layout Violation (DQS is longer than CLK)####\n");
  350. }
  351. phase = 0x0;
  352. delay = 0x0;
  353. dram_info->wl_val[cs]
  354. [pup_num]
  355. [P] = phase;
  356. dram_info->wl_val[cs]
  357. [pup_num]
  358. [D] = delay;
  359. ddr3_write_pup_reg
  360. (PUP_WL_MODE, cs,
  361. pup * (1 - ecc) +
  362. ECC_PUP * ecc,
  363. phase, delay);
  364. }
  365. /* Stop condition for ECC phase */
  366. pup = (ecc) ? max_pup_num : pup;
  367. }
  368. /* ECC Support - Disable ECC MUX */
  369. reg =
  370. (reg_read(REG_DRAM_TRAINING_2_ADDR)
  371. & ~(1 <<
  372. REG_DRAM_TRAINING_2_ECC_MUX_OFFS));
  373. reg_write(REG_DRAM_TRAINING_2_ADDR,
  374. reg);
  375. }
  376. }
  377. for (pup = 0; pup < dram_info->num_of_std_pups; pup++)
  378. sum += dram_info->wl_val[cs][pup][S];
  379. if (dram_info->ecc_ena)
  380. sum += dram_info->wl_val[cs][ECC_PUP][S];
  381. /* Checks if any pup is not locked after the change */
  382. if (sum < (WL_HI_FREQ_STATE * (dram_info->num_of_total_pups))) {
  383. DEBUG_WL_C("DDR3 - Write Leveling Hi-Freq Supplement - didn't work for Cs - ",
  384. (u32) cs, 1);
  385. return MV_FAIL;
  386. }
  387. tmp_count++;
  388. }
  389. }
  390. dram_info->wl_max_phase = 0;
  391. dram_info->wl_min_phase = 10;
  392. /*
  393. * Read results to arrays - Results are required for DQS Centralization
  394. */
  395. for (cs = 0; cs < MAX_CS; cs++) {
  396. if (dram_info->cs_ena & (1 << cs)) {
  397. for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
  398. if (pup == dram_info->num_of_std_pups
  399. && dram_info->ecc_ena)
  400. pup = ECC_PUP;
  401. reg = ddr3_read_pup_reg(PUP_WL_MODE, cs, pup);
  402. phase =
  403. (reg >> REG_PHY_PHASE_OFFS) &
  404. PUP_PHASE_MASK;
  405. if (phase > dram_info->wl_max_phase)
  406. dram_info->wl_max_phase = phase;
  407. if (phase < dram_info->wl_min_phase)
  408. dram_info->wl_min_phase = phase;
  409. }
  410. }
  411. }
  412. /* Disable SW override - Must be in a different stage */
  413. /* [0]=0 - Enable SW override */
  414. reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
  415. reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
  416. /* 0x15B8 - Training SW 2 Register */
  417. reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
  418. reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
  419. (1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
  420. reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
  421. DEBUG_WL_S("DDR3 - Write Leveling Hi-Freq Supplement - Ended Successfully\n");
  422. return MV_OK;
  423. }
  424. /*
  425. * Name: ddr3_write_leveling_hw_reg_dimm
  426. * Desc: Execute Write leveling phase by HW
  427. * Args: freq - current sequence frequency
  428. * dram_info - main struct
  429. * Notes:
  430. * Returns: MV_OK if success, MV_FAIL if fail.
  431. */
  432. int ddr3_write_leveling_hw_reg_dimm(u32 freq, MV_DRAM_INFO *dram_info)
  433. {
  434. u32 reg, phase, delay, cs, pup, pup_num;
  435. __maybe_unused int dpde_flag = 0;
  436. /* Debug message - Start Read leveling procedure */
  437. DEBUG_WL_S("DDR3 - Write Leveling - Starting HW WL procedure\n");
  438. if (dram_info->num_cs > 2) {
  439. DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
  440. return MV_NO_CHANGE;
  441. }
  442. /* If target freq = 400 move clock start point */
  443. /* Write to control PUP to Control Deskew Regs */
  444. if (freq <= DDR_400) {
  445. for (pup = 0; pup <= dram_info->num_of_total_pups; pup++) {
  446. /* PUP_DELAY_MASK 0x1F */
  447. /* reg = 0x0C10001F + (uj << 16); */
  448. ddr3_write_ctrl_pup_reg(1, pup, CNTRL_PUP_DESKEW + pup,
  449. 0x1F);
  450. }
  451. }
  452. #ifdef MV88F67XX
  453. /* Dynamic pad issue (BTS669) during WL */
  454. reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
  455. if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
  456. dpde_flag = 1;
  457. reg_write(REG_DUNIT_CTRL_LOW_ADDR,
  458. reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
  459. }
  460. #endif
  461. reg = (1 << REG_DRAM_TRAINING_WL_OFFS);
  462. /* Config the retest number */
  463. reg |= (COUNT_HW_WL << REG_DRAM_TRAINING_RETEST_OFFS);
  464. reg |= (dram_info->cs_ena << (REG_DRAM_TRAINING_CS_OFFS));
  465. reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
  466. reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) |
  467. (1 << REG_DRAM_TRAINING_AUTO_OFFS);
  468. reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg);
  469. /* Wait */
  470. do {
  471. reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
  472. (1 << REG_DRAM_TRAINING_AUTO_OFFS);
  473. } while (reg); /* Wait for '0' */
  474. reg = reg_read(REG_DRAM_TRAINING_ADDR);
  475. /* Check if Successful */
  476. if (reg & (1 << REG_DRAM_TRAINING_ERROR_OFFS)) {
  477. /*
  478. * Read results to arrays - Results are required for WL High
  479. * freq Supplement and DQS Centralization
  480. */
  481. for (cs = 0; cs < MAX_CS; cs++) {
  482. if (dram_info->cs_ena & (1 << cs)) {
  483. for (pup = 0;
  484. pup < dram_info->num_of_total_pups;
  485. pup++) {
  486. if (pup == dram_info->num_of_std_pups
  487. && dram_info->ecc_ena)
  488. pup = ECC_BIT;
  489. reg =
  490. ddr3_read_pup_reg(PUP_WL_MODE, cs,
  491. pup);
  492. phase =
  493. (reg >> REG_PHY_PHASE_OFFS) &
  494. PUP_PHASE_MASK;
  495. delay = reg & PUP_DELAY_MASK;
  496. dram_info->wl_val[cs][pup][P] = phase;
  497. dram_info->wl_val[cs][pup][D] = delay;
  498. if ((phase == 1) && (delay >= 0x1D)) {
  499. /*
  500. * Need to do it here for
  501. * uncorrect WL values
  502. */
  503. ddr3_write_pup_reg(PUP_WL_MODE,
  504. cs, pup, 0,
  505. 0);
  506. dram_info->wl_val[cs][pup][P] =
  507. 0;
  508. dram_info->wl_val[cs][pup][D] =
  509. 0;
  510. }
  511. dram_info->wl_val[cs][pup][S] =
  512. WL_HI_FREQ_STATE - 1;
  513. reg =
  514. ddr3_read_pup_reg(PUP_WL_MODE + 0x1,
  515. cs, pup);
  516. dram_info->wl_val[cs][pup][DQS] =
  517. (reg & 0x3F);
  518. }
  519. #ifdef MV_DEBUG_WL
  520. /*
  521. * Debug message - Print res for cs[i]:
  522. * cs,PUP,Phase,Delay
  523. */
  524. DEBUG_WL_S("DDR3 - Write Leveling - Write Leveling Cs - ");
  525. DEBUG_WL_D((u32) cs, 1);
  526. DEBUG_WL_S(" Results:\n");
  527. for (pup = 0;
  528. pup < dram_info->num_of_total_pups;
  529. pup++) {
  530. DEBUG_WL_S
  531. ("DDR3 - Write Leveling - PUP: ");
  532. DEBUG_WL_D((u32) pup, 1);
  533. DEBUG_WL_S(", Phase: ");
  534. DEBUG_WL_D((u32)
  535. dram_info->wl_val[cs][pup]
  536. [P], 1);
  537. DEBUG_WL_S(", Delay: ");
  538. DEBUG_WL_D((u32)
  539. dram_info->wl_val[cs][pup]
  540. [D], 2);
  541. DEBUG_WL_S("\n");
  542. }
  543. #endif
  544. }
  545. }
  546. #ifdef MV88F67XX
  547. /* Dynamic pad issue (BTS669) during WL */
  548. if (dpde_flag) {
  549. reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
  550. (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
  551. reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
  552. }
  553. #endif
  554. DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
  555. /* If target freq = 400 move clock back */
  556. /* Write to control PUP to Control Deskew Regs */
  557. if (freq <= DDR_400) {
  558. for (pup = 0; pup <= dram_info->num_of_total_pups;
  559. pup++) {
  560. ddr3_write_ctrl_pup_reg(1, pup,
  561. CNTRL_PUP_DESKEW + pup, 0);
  562. }
  563. }
  564. return MV_OK;
  565. } else {
  566. /* Configure Each PUP with locked leveling settings */
  567. for (cs = 0; cs < MAX_CS; cs++) {
  568. if (dram_info->cs_ena & (1 << cs)) {
  569. for (pup = 0;
  570. pup < dram_info->num_of_total_pups;
  571. pup++) {
  572. /* ECC support - bit 8 */
  573. pup_num = (pup == dram_info->num_of_std_pups) ?
  574. ECC_BIT : pup;
  575. ddr3_write_pup_reg(PUP_WL_MODE, cs,
  576. pup_num, 0, 0);
  577. }
  578. }
  579. }
  580. reg_write(REG_DRAM_TRAINING_ADDR, 0);
  581. /* If target freq = 400 move clock back */
  582. /* Write to control PUP to Control Deskew Regs */
  583. if (freq <= DDR_400) {
  584. for (pup = 0; pup <= dram_info->num_of_total_pups;
  585. pup++) {
  586. ddr3_write_ctrl_pup_reg(1, pup,
  587. CNTRL_PUP_DESKEW + pup, 0);
  588. }
  589. }
  590. DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
  591. return MV_NO_CHANGE;
  592. }
  593. }
  594. /*
  595. * Name: ddr3_write_leveling_sw
  596. * Desc: Execute Write leveling phase by SW
  597. * Args: freq - current sequence frequency
  598. * dram_info - main struct
  599. * Notes:
  600. * Returns: MV_OK if success, MV_FAIL if fail.
  601. */
  602. int ddr3_write_leveling_sw(u32 freq, int ratio_2to1, MV_DRAM_INFO *dram_info)
  603. {
  604. u32 reg, cs, cnt, pup, max_pup_num;
  605. u32 res[MAX_CS];
  606. max_pup_num = dram_info->num_of_total_pups;
  607. __maybe_unused int dpde_flag = 0;
  608. /* Debug message - Start Write leveling procedure */
  609. DEBUG_WL_S("DDR3 - Write Leveling - Starting SW WL procedure\n");
  610. #ifdef MV88F67XX
  611. /* Dynamic pad issue (BTS669) during WL */
  612. reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
  613. if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
  614. dpde_flag = 1;
  615. reg_write(REG_DUNIT_CTRL_LOW_ADDR,
  616. reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
  617. }
  618. #endif
  619. /* Set Output buffer-off to all CS and correct ODT values */
  620. for (cs = 0; cs < MAX_CS; cs++) {
  621. if (dram_info->cs_ena & (1 << cs)) {
  622. reg = reg_read(REG_DDR3_MR1_ADDR) &
  623. REG_DDR3_MR1_ODT_MASK;
  624. reg |= odt_static[dram_info->cs_ena][cs];
  625. reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
  626. /* 0x15D0 - DDR3 MR0 Register */
  627. reg_write(REG_DDR3_MR1_ADDR, reg);
  628. /* Issue MRS Command to current cs */
  629. reg = REG_SDRAM_OPERATION_CMD_MR1 &
  630. ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
  631. /*
  632. * [3-0] = 0x4 - MR1 Command, [11-8] -
  633. * enable current cs
  634. */
  635. /* 0x1418 - SDRAM Operation Register */
  636. reg_write(REG_SDRAM_OPERATION_ADDR, reg);
  637. udelay(MRS_DELAY);
  638. }
  639. }
  640. DEBUG_WL_FULL_S("DDR3 - Write Leveling - Qoff and RTT Values are set for all Cs\n");
  641. /* Enable SW override */
  642. reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
  643. (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
  644. /* [0] = 1 - Enable SW override */
  645. /* 0x15B8 - Training SW 2 Register */
  646. reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
  647. DEBUG_WL_FULL_S("DDR3 - Write Leveling - SW Override Enabled\n");
  648. /* Enable PHY write leveling mode */
  649. reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
  650. ~(1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
  651. /* [2] = 0 - TrnWLMode - Enable */
  652. /* 0x15B8 - Training SW 2 Register */
  653. reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
  654. /* Reset WL results arry */
  655. memset(dram_info->wl_val, 0, sizeof(u32) * MAX_CS * MAX_PUP_NUM * 7);
  656. /* Loop for each cs */
  657. for (cs = 0; cs < MAX_CS; cs++) {
  658. if (dram_info->cs_ena & (1 << cs)) {
  659. DEBUG_WL_FULL_C("DDR3 - Write Leveling - Starting working with Cs - ",
  660. (u32) cs, 1);
  661. /* Refresh X9 current cs */
  662. DEBUG_WL_FULL_S("DDR3 - Write Leveling - Refresh X9\n");
  663. for (cnt = 0; cnt < COUNT_WL_RFRS; cnt++) {
  664. reg =
  665. REG_SDRAM_OPERATION_CMD_RFRS & ~(1 <<
  666. (REG_SDRAM_OPERATION_CS_OFFS
  667. + cs));
  668. /* [3-0] = 0x2 - refresh, [11-8] - enable current cs */
  669. reg_write(REG_SDRAM_OPERATION_ADDR, reg); /* 0x1418 - SDRAM Operation Register */
  670. do {
  671. reg =
  672. ((reg_read
  673. (REG_SDRAM_OPERATION_ADDR)) &
  674. REG_SDRAM_OPERATION_CMD_RFRS_DONE);
  675. } while (reg); /* Wait for '0' */
  676. }
  677. /* Configure MR1 in Cs[CsNum] - write leveling on, output buffer on */
  678. DEBUG_WL_FULL_S("DDR3 - Write Leveling - Configure MR1 for current Cs: WL-on,OB-on\n");
  679. reg = reg_read(REG_DDR3_MR1_ADDR) &
  680. REG_DDR3_MR1_OUTBUF_WL_MASK;
  681. /* Set ODT Values */
  682. reg &= REG_DDR3_MR1_ODT_MASK;
  683. reg |= odt_static[dram_info->cs_ena][cs];
  684. /* Enable WL MODE */
  685. reg |= (1 << REG_DDR3_MR1_WL_ENA_OFFS);
  686. /* [7]=1, [12]=0 - Output Buffer and write leveling enabled */
  687. reg_write(REG_DDR3_MR1_ADDR, reg); /* 0x15D4 - DDR3 MR1 Register */
  688. /* Issue MRS Command to current cs */
  689. reg = REG_SDRAM_OPERATION_CMD_MR1 &
  690. ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
  691. /*
  692. * [3-0] = 0x4 - MR1 Command, [11-8] -
  693. * enable current cs
  694. */
  695. /* 0x1418 - SDRAM Operation Register */
  696. reg_write(REG_SDRAM_OPERATION_ADDR, reg);
  697. udelay(MRS_DELAY);
  698. /* Write leveling cs[cs] */
  699. if (MV_OK !=
  700. ddr3_write_leveling_single_cs(cs, freq, ratio_2to1,
  701. (u32 *)(res + cs),
  702. dram_info)) {
  703. DEBUG_WL_FULL_C("DDR3 - Write Leveling single Cs - FAILED - Cs - ",
  704. (u32) cs, 1);
  705. for (pup = 0; pup < max_pup_num; pup++) {
  706. if (((res[cs] >> pup) & 0x1) == 0) {
  707. DEBUG_WL_C("Failed Byte : ",
  708. pup, 1);
  709. }
  710. }
  711. return MV_FAIL;
  712. }
  713. /* Set TrnWLDeUpd - After each CS is done */
  714. reg = reg_read(REG_TRAINING_WL_ADDR) |
  715. (1 << REG_TRAINING_WL_CS_DONE_OFFS);
  716. /* 0x16AC - Training Write leveling register */
  717. reg_write(REG_TRAINING_WL_ADDR, reg);
  718. /*
  719. * Debug message - Finished Write leveling cs[cs] -
  720. * each PUP Fail/Success
  721. */
  722. DEBUG_WL_FULL_C("DDR3 - Write Leveling - Finished Cs - ", (u32) cs,
  723. 1);
  724. DEBUG_WL_FULL_C("DDR3 - Write Leveling - The Results: 1-PUP locked, 0-PUP failed -",
  725. (u32) res[cs], 3);
  726. /*
  727. * Configure MR1 in cs[cs] - write leveling off (0),
  728. * output buffer off (1)
  729. */
  730. reg = reg_read(REG_DDR3_MR1_ADDR) &
  731. REG_DDR3_MR1_OUTBUF_WL_MASK;
  732. reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
  733. /* No need to sort ODT since it is same CS */
  734. /* 0x15D4 - DDR3 MR1 Register */
  735. reg_write(REG_DDR3_MR1_ADDR, reg);
  736. /* Issue MRS Command to current cs */
  737. reg = REG_SDRAM_OPERATION_CMD_MR1 &
  738. ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
  739. /*
  740. * [3-0] = 0x4 - MR1 Command, [11-8] -
  741. * enable current cs
  742. */
  743. /* 0x1418 - SDRAM Operation Register */
  744. reg_write(REG_SDRAM_OPERATION_ADDR, reg);
  745. udelay(MRS_DELAY);
  746. }
  747. }
  748. /* Disable WL Mode */
  749. /* [2]=1 - TrnWLMode - Disable */
  750. reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
  751. reg |= (1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
  752. /* 0x15B8 - Training SW 2 Register */
  753. reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
  754. /* Disable SW override - Must be in a different stage */
  755. /* [0]=0 - Enable SW override */
  756. reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
  757. reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
  758. /* 0x15B8 - Training SW 2 Register */
  759. reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
  760. /* Set Output buffer-on to all CS and correct ODT values */
  761. for (cs = 0; cs < MAX_CS; cs++) {
  762. if (dram_info->cs_ena & (1 << cs)) {
  763. reg = reg_read(REG_DDR3_MR1_ADDR) &
  764. REG_DDR3_MR1_ODT_MASK;
  765. reg &= REG_DDR3_MR1_OUTBUF_WL_MASK;
  766. reg |= odt_static[dram_info->cs_ena][cs];
  767. /* 0x15D0 - DDR3 MR1 Register */
  768. reg_write(REG_DDR3_MR1_ADDR, reg);
  769. /* Issue MRS Command to current cs */
  770. reg = REG_SDRAM_OPERATION_CMD_MR1 &
  771. ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
  772. /*
  773. * [3-0] = 0x4 - MR1 Command, [11-8] -
  774. * enable current cs
  775. */
  776. /* 0x1418 - SDRAM Operation Register */
  777. reg_write(REG_SDRAM_OPERATION_ADDR, reg);
  778. udelay(MRS_DELAY);
  779. }
  780. }
  781. #ifdef MV88F67XX
  782. /* Dynamic pad issue (BTS669) during WL */
  783. if (dpde_flag) {
  784. reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
  785. (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
  786. reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
  787. }
  788. #endif
  789. DEBUG_WL_FULL_S("DDR3 - Write Leveling - Finished WL procedure for all Cs\n");
  790. return MV_OK;
  791. }
  792. #if !defined(MV88F672X)
  793. /*
  794. * Name: ddr3_write_leveling_sw
  795. * Desc: Execute Write leveling phase by SW
  796. * Args: freq - current sequence frequency
  797. * dram_info - main struct
  798. * Notes:
  799. * Returns: MV_OK if success, MV_FAIL if fail.
  800. */
  801. int ddr3_write_leveling_sw_reg_dimm(u32 freq, int ratio_2to1,
  802. MV_DRAM_INFO *dram_info)
  803. {
  804. u32 reg, cs, cnt, pup;
  805. u32 res[MAX_CS];
  806. __maybe_unused int dpde_flag = 0;
  807. /* Debug message - Start Write leveling procedure */
  808. DEBUG_WL_S("DDR3 - Write Leveling - Starting SW WL procedure\n");
  809. #ifdef MV88F67XX
  810. /* Dynamic pad issue (BTS669) during WL */
  811. reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
  812. if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
  813. dpde_flag = 1;
  814. reg_write(REG_DUNIT_CTRL_LOW_ADDR,
  815. reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
  816. }
  817. #endif
  818. /* If target freq = 400 move clock start point */
  819. /* Write to control PUP to Control Deskew Regs */
  820. if (freq <= DDR_400) {
  821. for (pup = 0; pup <= dram_info->num_of_total_pups; pup++) {
  822. /* PUP_DELAY_MASK 0x1F */
  823. /* reg = 0x0C10001F + (uj << 16); */
  824. ddr3_write_ctrl_pup_reg(1, pup, CNTRL_PUP_DESKEW + pup,
  825. 0x1F);
  826. }
  827. }
  828. /* Set Output buffer-off to all CS and correct ODT values */
  829. for (cs = 0; cs < MAX_CS; cs++) {
  830. if (dram_info->cs_ena & (1 << cs)) {
  831. reg = reg_read(REG_DDR3_MR1_ADDR) &
  832. REG_DDR3_MR1_ODT_MASK;
  833. reg |= odt_static[dram_info->cs_ena][cs];
  834. reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
  835. /* 0x15D0 - DDR3 MR0 Register */
  836. reg_write(REG_DDR3_MR1_ADDR, reg);
  837. /* Issue MRS Command to current cs */
  838. reg = REG_SDRAM_OPERATION_CMD_MR1 &
  839. ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
  840. /*
  841. * [3-0] = 0x4 - MR1 Command, [11-8] -
  842. * enable current cs
  843. */
  844. /* 0x1418 - SDRAM Operation Register */
  845. reg_write(REG_SDRAM_OPERATION_ADDR, reg);
  846. udelay(MRS_DELAY);
  847. }
  848. }
  849. DEBUG_WL_FULL_S("DDR3 - Write Leveling - Qoff and RTT Values are set for all Cs\n");
  850. /* Enable SW override */
  851. reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
  852. (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
  853. /* [0] = 1 - Enable SW override */
  854. /* 0x15B8 - Training SW 2 Register */
  855. reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
  856. DEBUG_WL_FULL_S("DDR3 - Write Leveling - SW Override Enabled\n");
  857. /* Enable PHY write leveling mode */
  858. reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
  859. ~(1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
  860. /* [2] = 0 - TrnWLMode - Enable */
  861. /* 0x15B8 - Training SW 2 Register */
  862. reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
  863. /* Loop for each cs */
  864. for (cs = 0; cs < MAX_CS; cs++) {
  865. if (dram_info->cs_ena & (1 << cs)) {
  866. DEBUG_WL_FULL_C("DDR3 - Write Leveling - Starting working with Cs - ",
  867. (u32) cs, 1);
  868. /* Refresh X9 current cs */
  869. DEBUG_WL_FULL_S("DDR3 - Write Leveling - Refresh X9\n");
  870. for (cnt = 0; cnt < COUNT_WL_RFRS; cnt++) {
  871. reg =
  872. REG_SDRAM_OPERATION_CMD_RFRS & ~(1 <<
  873. (REG_SDRAM_OPERATION_CS_OFFS
  874. + cs));
  875. /* [3-0] = 0x2 - refresh, [11-8] - enable current cs */
  876. reg_write(REG_SDRAM_OPERATION_ADDR, reg); /* 0x1418 - SDRAM Operation Register */
  877. do {
  878. reg =
  879. ((reg_read
  880. (REG_SDRAM_OPERATION_ADDR)) &
  881. REG_SDRAM_OPERATION_CMD_RFRS_DONE);
  882. } while (reg); /* Wait for '0' */
  883. }
  884. /*
  885. * Configure MR1 in Cs[CsNum] - write leveling on,
  886. * output buffer on
  887. */
  888. DEBUG_WL_FULL_S("DDR3 - Write Leveling - Configure MR1 for current Cs: WL-on,OB-on\n");
  889. reg = reg_read(REG_DDR3_MR1_ADDR) &
  890. REG_DDR3_MR1_OUTBUF_WL_MASK;
  891. /* Set ODT Values */
  892. reg &= REG_DDR3_MR1_ODT_MASK;
  893. reg |= odt_static[dram_info->cs_ena][cs];
  894. /* Enable WL MODE */
  895. reg |= (1 << REG_DDR3_MR1_WL_ENA_OFFS);
  896. /*
  897. * [7]=1, [12]=0 - Output Buffer and write leveling
  898. * enabled
  899. */
  900. /* 0x15D4 - DDR3 MR1 Register */
  901. reg_write(REG_DDR3_MR1_ADDR, reg);
  902. /* Issue MRS Command to current cs */
  903. reg = REG_SDRAM_OPERATION_CMD_MR1 &
  904. ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
  905. /*
  906. * [3-0] = 0x4 - MR1 Command, [11-8] -
  907. * enable current cs
  908. */
  909. /* 0x1418 - SDRAM Operation Register */
  910. reg_write(REG_SDRAM_OPERATION_ADDR, reg);
  911. udelay(MRS_DELAY);
  912. /* Write leveling cs[cs] */
  913. if (MV_OK !=
  914. ddr3_write_leveling_single_cs(cs, freq, ratio_2to1,
  915. (u32 *)(res + cs),
  916. dram_info)) {
  917. DEBUG_WL_FULL_C("DDR3 - Write Leveling single Cs - FAILED - Cs - ",
  918. (u32) cs, 1);
  919. return MV_FAIL;
  920. }
  921. /* Set TrnWLDeUpd - After each CS is done */
  922. reg = reg_read(REG_TRAINING_WL_ADDR) |
  923. (1 << REG_TRAINING_WL_CS_DONE_OFFS);
  924. /* 0x16AC - Training Write leveling register */
  925. reg_write(REG_TRAINING_WL_ADDR, reg);
  926. /*
  927. * Debug message - Finished Write leveling cs[cs] -
  928. * each PUP Fail/Success
  929. */
  930. DEBUG_WL_FULL_C("DDR3 - Write Leveling - Finished Cs - ", (u32) cs,
  931. 1);
  932. DEBUG_WL_FULL_C("DDR3 - Write Leveling - The Results: 1-PUP locked, 0-PUP failed -",
  933. (u32) res[cs], 3);
  934. /* Configure MR1 in cs[cs] - write leveling off (0), output buffer off (1) */
  935. reg = reg_read(REG_DDR3_MR1_ADDR) &
  936. REG_DDR3_MR1_OUTBUF_WL_MASK;
  937. reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
  938. /* No need to sort ODT since it is same CS */
  939. /* 0x15D4 - DDR3 MR1 Register */
  940. reg_write(REG_DDR3_MR1_ADDR, reg);
  941. /* Issue MRS Command to current cs */
  942. reg = REG_SDRAM_OPERATION_CMD_MR1 &
  943. ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
  944. /*
  945. * [3-0] = 0x4 - MR1 Command, [11-8] -
  946. * enable current cs
  947. */
  948. /* 0x1418 - SDRAM Operation Register */
  949. reg_write(REG_SDRAM_OPERATION_ADDR, reg);
  950. udelay(MRS_DELAY);
  951. }
  952. }
  953. /* Disable WL Mode */
  954. /* [2]=1 - TrnWLMode - Disable */
  955. reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
  956. reg |= (1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
  957. /* 0x15B8 - Training SW 2 Register */
  958. reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
  959. /* Disable SW override - Must be in a different stage */
  960. /* [0]=0 - Enable SW override */
  961. reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
  962. reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
  963. /* 0x15B8 - Training SW 2 Register */
  964. reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
  965. /* Set Output buffer-on to all CS and correct ODT values */
  966. for (cs = 0; cs < MAX_CS; cs++) {
  967. if (dram_info->cs_ena & (1 << cs)) {
  968. reg = reg_read(REG_DDR3_MR1_ADDR) &
  969. REG_DDR3_MR1_ODT_MASK;
  970. reg &= REG_DDR3_MR1_OUTBUF_WL_MASK;
  971. reg |= odt_static[dram_info->cs_ena][cs];
  972. /* 0x15D0 - DDR3 MR1 Register */
  973. reg_write(REG_DDR3_MR1_ADDR, reg);
  974. /* Issue MRS Command to current cs */
  975. reg = REG_SDRAM_OPERATION_CMD_MR1 &
  976. ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
  977. /*
  978. * [3-0] = 0x4 - MR1 Command, [11-8] -
  979. * enable current cs
  980. */
  981. /* 0x1418 - SDRAM Operation Register */
  982. reg_write(REG_SDRAM_OPERATION_ADDR, reg);
  983. udelay(MRS_DELAY);
  984. }
  985. }
  986. #ifdef MV88F67XX
  987. /* Dynamic pad issue (BTS669) during WL */
  988. if (dpde_flag) {
  989. reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
  990. (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
  991. reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
  992. }
  993. #endif
  994. /* If target freq = 400 move clock back */
  995. /* Write to control PUP to Control Deskew Regs */
  996. if (freq <= DDR_400) {
  997. for (pup = 0; pup <= dram_info->num_of_total_pups; pup++) {
  998. ddr3_write_ctrl_pup_reg(1, pup, CNTRL_PUP_DESKEW + pup,
  999. 0);
  1000. }
  1001. }
  1002. DEBUG_WL_FULL_S("DDR3 - Write Leveling - Finished WL procedure for all Cs\n");
  1003. return MV_OK;
  1004. }
  1005. #endif
  1006. /*
  1007. * Name: ddr3_write_leveling_single_cs
  1008. * Desc: Execute Write leveling for single Chip select
  1009. * Args: cs - current chip select
  1010. * freq - current sequence frequency
  1011. * result - res array
  1012. * dram_info - main struct
  1013. * Notes:
  1014. * Returns: MV_OK if success, MV_FAIL if fail.
  1015. */
  1016. static int ddr3_write_leveling_single_cs(u32 cs, u32 freq, int ratio_2to1,
  1017. u32 *result, MV_DRAM_INFO *dram_info)
  1018. {
  1019. u32 reg, pup_num, delay, phase, phaseMax, max_pup_num, pup,
  1020. max_pup_mask;
  1021. max_pup_num = dram_info->num_of_total_pups;
  1022. *result = 0;
  1023. u32 flag[MAX_PUP_NUM] = { 0 };
  1024. DEBUG_WL_FULL_C("DDR3 - Write Leveling Single Cs - WL for Cs - ",
  1025. (u32) cs, 1);
  1026. switch (max_pup_num) {
  1027. case 2:
  1028. max_pup_mask = 0x3;
  1029. break;
  1030. case 4:
  1031. max_pup_mask = 0xf;
  1032. DEBUG_WL_C("max_pup_mask = ", max_pup_mask, 3);
  1033. break;
  1034. case 5:
  1035. max_pup_mask = 0x1f;
  1036. DEBUG_WL_C("max_pup_mask = ", max_pup_mask, 3);
  1037. break;
  1038. case 8:
  1039. max_pup_mask = 0xff;
  1040. DEBUG_WL_C("max_pup_mask = ", max_pup_mask, 3);
  1041. break;
  1042. case 9:
  1043. max_pup_mask = 0x1ff;
  1044. DEBUG_WL_C("max_pup_mask = ", max_pup_mask, 3);
  1045. break;
  1046. default:
  1047. DEBUG_WL_C("ddr3_write_leveling_single_cs wrong max_pup_num = ",
  1048. max_pup_num, 3);
  1049. return MV_FAIL;
  1050. }
  1051. /* CS ODT Override */
  1052. reg = reg_read(REG_SDRAM_ODT_CTRL_HIGH_ADDR) &
  1053. REG_SDRAM_ODT_CTRL_HIGH_OVRD_MASK;
  1054. reg |= (REG_SDRAM_ODT_CTRL_HIGH_OVRD_ENA << (2 * cs));
  1055. /* Set 0x3 - Enable ODT on the curent cs and disable on other cs */
  1056. /* 0x1498 - SDRAM ODT Control high */
  1057. reg_write(REG_SDRAM_ODT_CTRL_HIGH_ADDR, reg);
  1058. DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - ODT Asserted for current Cs\n");
  1059. /* tWLMRD Delay */
  1060. /* Delay of minimum 40 Dram clock cycles - 20 Tclk cycles */
  1061. udelay(1);
  1062. /* [1:0] - current cs number */
  1063. reg = (reg_read(REG_TRAINING_WL_ADDR) & REG_TRAINING_WL_CS_MASK) | cs;
  1064. reg |= (1 << REG_TRAINING_WL_UPD_OFFS); /* [2] - trnWLCsUpd */
  1065. /* 0x16AC - Training Write leveling register */
  1066. reg_write(REG_TRAINING_WL_ADDR, reg);
  1067. /* Broadcast to all PUPs: Reset DQS phase, reset leveling delay */
  1068. ddr3_write_pup_reg(PUP_WL_MODE, cs, PUP_BC, 0, 0);
  1069. /* Seek Edge */
  1070. DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge - Current Cs\n");
  1071. /* Drive DQS high for one cycle - All data PUPs */
  1072. DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge - Driving DQS high for one cycle\n");
  1073. if (!ratio_2to1) {
  1074. reg = (reg_read(REG_TRAINING_WL_ADDR) &
  1075. REG_TRAINING_WL_RATIO_MASK) | REG_TRAINING_WL_1TO1;
  1076. } else {
  1077. reg = (reg_read(REG_TRAINING_WL_ADDR) &
  1078. REG_TRAINING_WL_RATIO_MASK) | REG_TRAINING_WL_2TO1;
  1079. }
  1080. /* 0x16AC - Training Write leveling register */
  1081. reg_write(REG_TRAINING_WL_ADDR, reg);
  1082. /* Wait tWLdelay */
  1083. do {
  1084. /* [29] - trnWLDelayExp */
  1085. reg = (reg_read(REG_TRAINING_WL_ADDR)) &
  1086. REG_TRAINING_WL_DELAYEXP_MASK;
  1087. } while (reg == 0x0); /* Wait for '1' */
  1088. /* Read WL res */
  1089. reg = (reg_read(REG_TRAINING_WL_ADDR) >> REG_TRAINING_WL_RESULTS_OFFS) &
  1090. REG_TRAINING_WL_RESULTS_MASK;
  1091. /* [28:20] - TrnWLResult */
  1092. if (!ratio_2to1) /* Different phase options for 2:1 or 1:1 modes */
  1093. phaseMax = MAX_PHASE_1TO1;
  1094. else
  1095. phaseMax = MAX_PHASE_2TO1;
  1096. DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge - Shift DQS + Octet Leveling\n");
  1097. /* Shift DQS + Octet leveling */
  1098. for (phase = 0; phase < phaseMax; phase++) {
  1099. for (delay = 0; delay < MAX_DELAY; delay++) {
  1100. /* Broadcast to all PUPs: DQS phase,leveling delay */
  1101. ddr3_write_pup_reg(PUP_WL_MODE, cs, PUP_BC, phase,
  1102. delay);
  1103. udelay(1); /* Delay of 3 Tclk cycles */
  1104. DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge: Phase = ");
  1105. DEBUG_WL_FULL_D((u32) phase, 1);
  1106. DEBUG_WL_FULL_S(", Delay = ");
  1107. DEBUG_WL_FULL_D((u32) delay, 1);
  1108. DEBUG_WL_FULL_S("\n");
  1109. /* Drive DQS high for one cycle - All data PUPs */
  1110. if (!ratio_2to1) {
  1111. reg = (reg_read(REG_TRAINING_WL_ADDR) &
  1112. REG_TRAINING_WL_RATIO_MASK) |
  1113. REG_TRAINING_WL_1TO1;
  1114. } else {
  1115. reg = (reg_read(REG_TRAINING_WL_ADDR) &
  1116. REG_TRAINING_WL_RATIO_MASK) |
  1117. REG_TRAINING_WL_2TO1;
  1118. }
  1119. reg_write(REG_TRAINING_WL_ADDR, reg); /* 0x16AC */
  1120. /* Wait tWLdelay */
  1121. do {
  1122. reg = (reg_read(REG_TRAINING_WL_ADDR)) &
  1123. REG_TRAINING_WL_DELAYEXP_MASK;
  1124. } while (reg == 0x0); /* [29] Wait for '1' */
  1125. /* Read WL res */
  1126. reg = reg_read(REG_TRAINING_WL_ADDR);
  1127. reg = (reg >> REG_TRAINING_WL_RESULTS_OFFS) &
  1128. REG_TRAINING_WL_RESULTS_MASK; /* [28:20] */
  1129. DEBUG_WL_FULL_C("DDR3 - Write Leveling Single Cs - Seek Edge: Results = ",
  1130. (u32) reg, 3);
  1131. /* Update State machine */
  1132. for (pup = 0; pup < (max_pup_num); pup++) {
  1133. /* ECC support - bit 8 */
  1134. pup_num = (pup == dram_info->num_of_std_pups) ?
  1135. ECC_BIT : pup;
  1136. if (dram_info->wl_val[cs][pup][S] == 0) {
  1137. /* Update phase to PUP */
  1138. dram_info->wl_val[cs][pup][P] = phase;
  1139. /* Update delay to PUP */
  1140. dram_info->wl_val[cs][pup][D] = delay;
  1141. }
  1142. if (((reg >> pup_num) & 0x1) == 0)
  1143. flag[pup_num] = 1;
  1144. if (((reg >> pup_num) & 0x1)
  1145. && (flag[pup_num] == 1)
  1146. && (dram_info->wl_val[cs][pup][S] == 0)) {
  1147. /*
  1148. * If the PUP is locked now and in last
  1149. * counter states
  1150. */
  1151. /* Go to next state */
  1152. dram_info->wl_val[cs][pup][S] = 1;
  1153. /* Set res */
  1154. *result = *result | (1 << pup_num);
  1155. }
  1156. }
  1157. /* If all locked - Break the loops - Finished */
  1158. if (*result == max_pup_mask) {
  1159. phase = phaseMax;
  1160. delay = MAX_DELAY;
  1161. DEBUG_WL_S("DDR3 - Write Leveling Single Cs - Seek Edge: All Locked\n");
  1162. }
  1163. }
  1164. }
  1165. /* Debug message - Print res for cs[i]: cs,PUP,Phase,Delay */
  1166. DEBUG_WL_C("DDR3 - Write Leveling - Results for CS - ", (u32) cs, 1);
  1167. for (pup = 0; pup < (max_pup_num); pup++) {
  1168. DEBUG_WL_S("DDR3 - Write Leveling - PUP: ");
  1169. DEBUG_WL_D((u32) pup, 1);
  1170. DEBUG_WL_S(", Phase: ");
  1171. DEBUG_WL_D((u32) dram_info->wl_val[cs][pup][P], 1);
  1172. DEBUG_WL_S(", Delay: ");
  1173. DEBUG_WL_D((u32) dram_info->wl_val[cs][pup][D], 2);
  1174. DEBUG_WL_S("\n");
  1175. }
  1176. /* Check if some not locked and return error */
  1177. if (*result != max_pup_mask) {
  1178. DEBUG_WL_S("DDR3 - Write Leveling - ERROR - not all PUPS were locked\n");
  1179. return MV_FAIL;
  1180. }
  1181. /* Configure Each PUP with locked leveling settings */
  1182. for (pup = 0; pup < (max_pup_num); pup++) {
  1183. /* ECC support - bit 8 */
  1184. pup_num = (pup == dram_info->num_of_std_pups) ? ECC_BIT : pup;
  1185. phase = dram_info->wl_val[cs][pup][P];
  1186. delay = dram_info->wl_val[cs][pup][D];
  1187. ddr3_write_pup_reg(PUP_WL_MODE, cs, pup_num, phase, delay);
  1188. }
  1189. /* CS ODT Override */
  1190. reg = reg_read(REG_SDRAM_ODT_CTRL_HIGH_ADDR) &
  1191. REG_SDRAM_ODT_CTRL_HIGH_OVRD_MASK;
  1192. /* 0x1498 - SDRAM ODT Control high */
  1193. reg_write(REG_SDRAM_ODT_CTRL_HIGH_ADDR, reg);
  1194. return MV_OK;
  1195. }
  1196. /*
  1197. * Perform DDR3 Control PUP Indirect Write
  1198. */
  1199. static void ddr3_write_ctrl_pup_reg(int bc_acc, u32 pup, u32 reg_addr, u32 data)
  1200. {
  1201. u32 reg = 0;
  1202. /* Store value for write */
  1203. reg = (data & 0xFFFF);
  1204. /* Set bit 26 for control PHY access */
  1205. reg |= (1 << REG_PHY_CNTRL_OFFS);
  1206. /* Configure BC or UC access to PHYs */
  1207. if (bc_acc == 1)
  1208. reg |= (1 << REG_PHY_BC_OFFS);
  1209. else
  1210. reg |= (pup << REG_PHY_PUP_OFFS);
  1211. /* Set PHY register address to write to */
  1212. reg |= (reg_addr << REG_PHY_CS_OFFS);
  1213. reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg); /* 0x16A0 */
  1214. reg |= REG_PHY_REGISTRY_FILE_ACCESS_OP_WR;
  1215. reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg); /* 0x16A0 */
  1216. do {
  1217. reg = (reg_read(REG_PHY_REGISTRY_FILE_ACCESS_ADDR)) &
  1218. REG_PHY_REGISTRY_FILE_ACCESS_OP_DONE;
  1219. } while (reg); /* Wait for '0' to mark the end of the transaction */
  1220. }