octeontx_nand.c 56 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 2018 Marvell International Ltd.
  4. */
  5. #include <dm.h>
  6. #include <dm/device-internal.h>
  7. #include <dm/devres.h>
  8. #include <dm/of_access.h>
  9. #include <malloc.h>
  10. #include <memalign.h>
  11. #include <nand.h>
  12. #include <pci.h>
  13. #include <time.h>
  14. #include <linux/bitfield.h>
  15. #include <linux/ctype.h>
  16. #include <linux/dma-mapping.h>
  17. #include <linux/delay.h>
  18. #include <linux/errno.h>
  19. #include <linux/err.h>
  20. #include <linux/ioport.h>
  21. #include <linux/libfdt.h>
  22. #include <linux/mtd/mtd.h>
  23. #include <linux/mtd/nand_bch.h>
  24. #include <linux/mtd/nand_ecc.h>
  25. #include <asm/io.h>
  26. #include <asm/types.h>
  27. #include <asm/dma-mapping.h>
  28. #include <asm/arch/clock.h>
  29. #include "octeontx_bch.h"
  30. #ifdef DEBUG
  31. # undef CONFIG_LOGLEVEL
  32. # define CONFIG_LOGLEVEL 8
  33. #endif
  34. /*
  35. * The NDF_CMD queue takes commands between 16 - 128 bit.
  36. * All commands must be 16 bit aligned and are little endian.
  37. * WAIT_STATUS commands must be 64 bit aligned.
  38. * Commands are selected by the 4 bit opcode.
  39. *
  40. * Available Commands:
  41. *
  42. * 16 Bit:
  43. * NOP
  44. * WAIT
  45. * BUS_ACQ, BUS_REL
  46. * CHIP_EN, CHIP_DIS
  47. *
  48. * 32 Bit:
  49. * CLE_CMD
  50. * RD_CMD, RD_EDO_CMD
  51. * WR_CMD
  52. *
  53. * 64 Bit:
  54. * SET_TM_PAR
  55. *
  56. * 96 Bit:
  57. * ALE_CMD
  58. *
  59. * 128 Bit:
  60. * WAIT_STATUS, WAIT_STATUS_ALE
  61. */
  62. /* NDF Register offsets */
  63. #define NDF_CMD 0x0
  64. #define NDF_MISC 0x8
  65. #define NDF_ECC_CNT 0x10
  66. #define NDF_DRBELL 0x30
  67. #define NDF_ST_REG 0x38 /* status */
  68. #define NDF_INT 0x40
  69. #define NDF_INT_W1S 0x48
  70. #define NDF_DMA_CFG 0x50
  71. #define NDF_DMA_ADR 0x58
  72. #define NDF_INT_ENA_W1C 0x60
  73. #define NDF_INT_ENA_W1S 0x68
  74. /* NDF command opcodes */
  75. #define NDF_OP_NOP 0x0
  76. #define NDF_OP_SET_TM_PAR 0x1
  77. #define NDF_OP_WAIT 0x2
  78. #define NDF_OP_CHIP_EN_DIS 0x3
  79. #define NDF_OP_CLE_CMD 0x4
  80. #define NDF_OP_ALE_CMD 0x5
  81. #define NDF_OP_WR_CMD 0x8
  82. #define NDF_OP_RD_CMD 0x9
  83. #define NDF_OP_RD_EDO_CMD 0xa
  84. #define NDF_OP_WAIT_STATUS 0xb /* same opcode for WAIT_STATUS_ALE */
  85. #define NDF_OP_BUS_ACQ_REL 0xf
  86. #define NDF_BUS_ACQUIRE 1
  87. #define NDF_BUS_RELEASE 0
  88. #define DBGX_EDSCR(X) (0x87A008000088 + (X) * 0x80000)
  89. struct ndf_nop_cmd {
  90. u16 opcode: 4;
  91. u16 nop: 12;
  92. };
  93. struct ndf_wait_cmd {
  94. u16 opcode:4;
  95. u16 r_b:1; /* wait for one cycle or PBUS_WAIT deassert */
  96. u16:3;
  97. u16 wlen:3; /* timing parameter select */
  98. u16:5;
  99. };
  100. struct ndf_bus_cmd {
  101. u16 opcode:4;
  102. u16 direction:4; /* 1 = acquire, 0 = release */
  103. u16:8;
  104. };
  105. struct ndf_chip_cmd {
  106. u16 opcode:4;
  107. u16 chip:3; /* select chip, 0 = disable */
  108. u16 enable:1; /* 1 = enable, 0 = disable */
  109. u16 bus_width:2; /* 10 = 16 bit, 01 = 8 bit */
  110. u16:6;
  111. };
  112. struct ndf_cle_cmd {
  113. u32 opcode:4;
  114. u32:4;
  115. u32 cmd_data:8; /* command sent to the PBUS AD pins */
  116. u32 clen1:3; /* time between PBUS CLE and WE asserts */
  117. u32 clen2:3; /* time WE remains asserted */
  118. u32 clen3:3; /* time between WE deassert and CLE */
  119. u32:7;
  120. };
  121. /* RD_EDO_CMD uses the same layout as RD_CMD */
  122. struct ndf_rd_cmd {
  123. u32 opcode:4;
  124. u32 data:16; /* data bytes */
  125. u32 rlen1:3;
  126. u32 rlen2:3;
  127. u32 rlen3:3;
  128. u32 rlen4:3;
  129. };
  130. struct ndf_wr_cmd {
  131. u32 opcode:4;
  132. u32 data:16; /* data bytes */
  133. u32:4;
  134. u32 wlen1:3;
  135. u32 wlen2:3;
  136. u32:3;
  137. };
  138. struct ndf_set_tm_par_cmd {
  139. u64 opcode:4;
  140. u64 tim_mult:4; /* multiplier for the seven parameters */
  141. u64 tm_par1:8; /* --> Following are the 7 timing parameters that */
  142. u64 tm_par2:8; /* specify the number of coprocessor cycles. */
  143. u64 tm_par3:8; /* A value of zero means one cycle. */
  144. u64 tm_par4:8; /* All values are scaled by tim_mult */
  145. u64 tm_par5:8; /* using tim_par * (2 ^ tim_mult). */
  146. u64 tm_par6:8;
  147. u64 tm_par7:8;
  148. };
  149. struct ndf_ale_cmd {
  150. u32 opcode:4;
  151. u32:4;
  152. u32 adr_byte_num:4; /* number of address bytes to be sent */
  153. u32:4;
  154. u32 alen1:3;
  155. u32 alen2:3;
  156. u32 alen3:3;
  157. u32 alen4:3;
  158. u32:4;
  159. u8 adr_byt1;
  160. u8 adr_byt2;
  161. u8 adr_byt3;
  162. u8 adr_byt4;
  163. u8 adr_byt5;
  164. u8 adr_byt6;
  165. u8 adr_byt7;
  166. u8 adr_byt8;
  167. };
  168. struct ndf_wait_status_cmd {
  169. u32 opcode:4;
  170. u32:4;
  171. u32 data:8; /** data */
  172. u32 clen1:3;
  173. u32 clen2:3;
  174. u32 clen3:3;
  175. u32:8;
  176. /** set to 5 to select WAIT_STATUS_ALE command */
  177. u32 ale_ind:8;
  178. /** ALE only: number of address bytes to be sent */
  179. u32 adr_byte_num:4;
  180. u32:4;
  181. u32 alen1:3; /* ALE only */
  182. u32 alen2:3; /* ALE only */
  183. u32 alen3:3; /* ALE only */
  184. u32 alen4:3; /* ALE only */
  185. u32:4;
  186. u8 adr_byt[4]; /* ALE only */
  187. u32 nine:4; /* set to 9 */
  188. u32 and_mask:8;
  189. u32 comp_byte:8;
  190. u32 rlen1:3;
  191. u32 rlen2:3;
  192. u32 rlen3:3;
  193. u32 rlen4:3;
  194. };
  195. union ndf_cmd {
  196. u64 val[2];
  197. union {
  198. struct ndf_nop_cmd nop;
  199. struct ndf_wait_cmd wait;
  200. struct ndf_bus_cmd bus_acq_rel;
  201. struct ndf_chip_cmd chip_en_dis;
  202. struct ndf_cle_cmd cle_cmd;
  203. struct ndf_rd_cmd rd_cmd;
  204. struct ndf_wr_cmd wr_cmd;
  205. struct ndf_set_tm_par_cmd set_tm_par;
  206. struct ndf_ale_cmd ale_cmd;
  207. struct ndf_wait_status_cmd wait_status;
  208. } u;
  209. };
  210. /** Disable multi-bit error hangs */
  211. #define NDF_MISC_MB_DIS BIT_ULL(27)
  212. /** High watermark for NBR FIFO or load/store operations */
  213. #define NDF_MISC_NBR_HWM GENMASK_ULL(26, 24)
  214. /** Wait input filter count */
  215. #define NDF_MISC_WAIT_CNT GENMASK_ULL(23, 18)
  216. /** Unfilled NFD_CMD queue bytes */
  217. #define NDF_MISC_FR_BYTE GENMASK_ULL(17, 7)
  218. /** Set by HW when it reads the last 8 bytes of NDF_CMD */
  219. #define NDF_MISC_RD_DONE BIT_ULL(6)
  220. /** Set by HW when it reads. SW read of NDF_CMD clears it */
  221. #define NDF_MISC_RD_VAL BIT_ULL(5)
  222. /** Let HW read NDF_CMD queue. Cleared on SW NDF_CMD write */
  223. #define NDF_MISC_RD_CMD BIT_ULL(4)
  224. /** Boot disable */
  225. #define NDF_MISC_BT_DIS BIT_ULL(2)
  226. /** Stop command execution after completing command queue */
  227. #define NDF_MISC_EX_DIS BIT_ULL(1)
  228. /** Reset fifo */
  229. #define NDF_MISC_RST_FF BIT_ULL(0)
  230. /** DMA engine enable */
  231. #define NDF_DMA_CFG_EN BIT_ULL(63)
  232. /** Read or write */
  233. #define NDF_DMA_CFG_RW BIT_ULL(62)
  234. /** Terminates DMA and clears enable bit */
  235. #define NDF_DMA_CFG_CLR BIT_ULL(61)
  236. /** 32-bit swap enable */
  237. #define NDF_DMA_CFG_SWAP32 BIT_ULL(59)
  238. /** 16-bit swap enable */
  239. #define NDF_DMA_CFG_SWAP16 BIT_ULL(58)
  240. /** 8-bit swap enable */
  241. #define NDF_DMA_CFG_SWAP8 BIT_ULL(57)
  242. /** Endian mode */
  243. #define NDF_DMA_CFG_CMD_BE BIT_ULL(56)
  244. /** Number of 64 bit transfers */
  245. #define NDF_DMA_CFG_SIZE GENMASK_ULL(55, 36)
  246. /** Command execution status idle */
  247. #define NDF_ST_REG_EXE_IDLE BIT_ULL(15)
  248. /** Command execution SM states */
  249. #define NDF_ST_REG_EXE_SM GENMASK_ULL(14, 11)
  250. /** DMA and load SM states */
  251. #define NDF_ST_REG_BT_SM GENMASK_ULL(10, 7)
  252. /** Queue read-back SM bad state */
  253. #define NDF_ST_REG_RD_FF_BAD BIT_ULL(6)
  254. /** Queue read-back SM states */
  255. #define NDF_ST_REG_RD_FF GENMASK_ULL(5, 4)
  256. /** Main SM is in a bad state */
  257. #define NDF_ST_REG_MAIN_BAD BIT_ULL(3)
  258. /** Main SM states */
  259. #define NDF_ST_REG_MAIN_SM GENMASK_ULL(2, 0)
  260. #define MAX_NAND_NAME_LEN 64
  261. #if (defined(NAND_MAX_PAGESIZE) && (NAND_MAX_PAGESIZE > 4096)) || \
  262. !defined(NAND_MAX_PAGESIZE)
  263. # undef NAND_MAX_PAGESIZE
  264. # define NAND_MAX_PAGESIZE 4096
  265. #endif
  266. #if (defined(NAND_MAX_OOBSIZE) && (NAND_MAX_OOBSIZE > 256)) || \
  267. !defined(NAND_MAX_OOBSIZE)
  268. # undef NAND_MAX_OOBSIZE
  269. # define NAND_MAX_OOBSIZE 256
  270. #endif
  271. #define OCTEONTX_NAND_DRIVER_NAME "octeontx_nand"
  272. #define NDF_TIMEOUT 1000 /** Timeout in ms */
  273. #define USEC_PER_SEC 1000000 /** Linux compatibility */
  274. #ifndef NAND_MAX_CHIPS
  275. # define NAND_MAX_CHIPS 8 /** Linux compatibility */
  276. #endif
  277. struct octeontx_nand_chip {
  278. struct list_head node;
  279. struct nand_chip nand;
  280. struct ndf_set_tm_par_cmd timings;
  281. int cs;
  282. int selected_page;
  283. int iface_mode;
  284. int row_bytes;
  285. int col_bytes;
  286. bool oob_only;
  287. bool iface_set;
  288. };
  289. struct octeontx_nand_buf {
  290. u8 *dmabuf;
  291. dma_addr_t dmaaddr;
  292. int dmabuflen;
  293. int data_len;
  294. int data_index;
  295. };
  296. /** NAND flash controller (NDF) related information */
  297. struct octeontx_nfc {
  298. struct nand_hw_control controller;
  299. struct udevice *dev;
  300. void __iomem *base;
  301. struct list_head chips;
  302. int selected_chip; /* Currently selected NAND chip number */
  303. /*
  304. * Status is separate from octeontx_nand_buf because
  305. * it can be used in parallel and during init.
  306. */
  307. u8 *stat;
  308. dma_addr_t stat_addr;
  309. bool use_status;
  310. struct octeontx_nand_buf buf;
  311. union bch_resp *bch_resp;
  312. dma_addr_t bch_rhandle;
  313. /* BCH of all-0xff, so erased pages read as error-free */
  314. unsigned char *eccmask;
  315. };
  316. /* settable timings - 0..7 select timing of alen1..4/clen1..3/etc */
  317. enum tm_idx {
  318. t0, /* fixed at 4<<mult cycles */
  319. t1, t2, t3, t4, t5, t6, t7, /* settable per ONFI-timing mode */
  320. };
  321. struct octeontx_probe_device {
  322. struct list_head list;
  323. struct udevice *dev;
  324. };
  325. static struct bch_vf *bch_vf;
  326. /** Deferred devices due to BCH not being ready */
  327. LIST_HEAD(octeontx_pci_nand_deferred_devices);
  328. /** default parameters used for probing chips */
  329. #define MAX_ONFI_MODE 5
  330. static int default_onfi_timing;
  331. static int slew_ns = 2; /* default timing padding */
  332. static int def_ecc_size = 512; /* 1024 best for sw_bch, <= 4095 for hw_bch */
  333. static int default_width = 1; /* 8 bit */
  334. static int default_page_size = 2048;
  335. static struct ndf_set_tm_par_cmd default_timing_parms;
  336. /** Port from Linux */
  337. #define readq_poll_timeout(addr, val, cond, delay_us, timeout_us) \
  338. ({ \
  339. ulong __start = get_timer(0); \
  340. void *__addr = (addr); \
  341. const ulong __timeout_ms = timeout_us / 1000; \
  342. do { \
  343. (val) = readq(__addr); \
  344. if (cond) \
  345. break; \
  346. if (timeout_us && get_timer(__start) > __timeout_ms) { \
  347. (val) = readq(__addr); \
  348. break; \
  349. } \
  350. if (delay_us) \
  351. udelay(delay_us); \
  352. } while (1); \
  353. (cond) ? 0 : -ETIMEDOUT; \
  354. })
  355. /** Ported from Linux 4.9.0 include/linux/of.h for compatibility */
  356. static inline int of_get_child_count(const ofnode node)
  357. {
  358. return fdtdec_get_child_count(gd->fdt_blob, ofnode_to_offset(node));
  359. }
  360. /**
  361. * Linux compatibility from Linux 4.9.0 drivers/mtd/nand/nand_base.c
  362. */
  363. static int nand_ooblayout_ecc_lp(struct mtd_info *mtd, int section,
  364. struct mtd_oob_region *oobregion)
  365. {
  366. struct nand_chip *chip = mtd_to_nand(mtd);
  367. struct nand_ecc_ctrl *ecc = &chip->ecc;
  368. if (section || !ecc->total)
  369. return -ERANGE;
  370. oobregion->length = ecc->total;
  371. oobregion->offset = mtd->oobsize - oobregion->length;
  372. return 0;
  373. }
  374. /**
  375. * Linux compatibility from Linux 4.9.0 drivers/mtd/nand/nand_base.c
  376. */
  377. static int nand_ooblayout_free_lp(struct mtd_info *mtd, int section,
  378. struct mtd_oob_region *oobregion)
  379. {
  380. struct nand_chip *chip = mtd_to_nand(mtd);
  381. struct nand_ecc_ctrl *ecc = &chip->ecc;
  382. if (section)
  383. return -ERANGE;
  384. oobregion->length = mtd->oobsize - ecc->total - 2;
  385. oobregion->offset = 2;
  386. return 0;
  387. }
  388. static const struct mtd_ooblayout_ops nand_ooblayout_lp_ops = {
  389. .ecc = nand_ooblayout_ecc_lp,
  390. .rfree = nand_ooblayout_free_lp,
  391. };
  392. static inline struct octeontx_nand_chip *to_otx_nand(struct nand_chip *nand)
  393. {
  394. return container_of(nand, struct octeontx_nand_chip, nand);
  395. }
  396. static inline struct octeontx_nfc *to_otx_nfc(struct nand_hw_control *ctrl)
  397. {
  398. return container_of(ctrl, struct octeontx_nfc, controller);
  399. }
  400. static int octeontx_nand_calc_ecc_layout(struct nand_chip *nand)
  401. {
  402. struct nand_ecclayout *layout = nand->ecc.layout;
  403. struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
  404. struct mtd_info *mtd = &nand->mtd;
  405. int oobsize = mtd->oobsize;
  406. int i;
  407. bool layout_alloc = false;
  408. if (!layout) {
  409. layout = devm_kzalloc(tn->dev, sizeof(*layout), GFP_KERNEL);
  410. if (!layout)
  411. return -ENOMEM;
  412. nand->ecc.layout = layout;
  413. layout_alloc = true;
  414. }
  415. layout->eccbytes = nand->ecc.steps * nand->ecc.bytes;
  416. /* Reserve 2 bytes for bad block marker */
  417. if (layout->eccbytes + 2 > oobsize) {
  418. pr_err("No suitable oob scheme available for oobsize %d eccbytes %u\n",
  419. oobsize, layout->eccbytes);
  420. goto fail;
  421. }
  422. /* put ecc bytes at oob tail */
  423. for (i = 0; i < layout->eccbytes; i++)
  424. layout->eccpos[i] = oobsize - layout->eccbytes + i;
  425. layout->oobfree[0].offset = 2;
  426. layout->oobfree[0].length = oobsize - 2 - layout->eccbytes;
  427. nand->ecc.layout = layout;
  428. return 0;
  429. fail:
  430. if (layout_alloc)
  431. kfree(layout);
  432. return -1;
  433. }
  434. /*
  435. * Read a single byte from the temporary buffer. Used after READID
  436. * to get the NAND information and for STATUS.
  437. */
  438. static u8 octeontx_nand_read_byte(struct mtd_info *mtd)
  439. {
  440. struct nand_chip *nand = mtd_to_nand(mtd);
  441. struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
  442. if (tn->use_status) {
  443. tn->use_status = false;
  444. return *tn->stat;
  445. }
  446. if (tn->buf.data_index < tn->buf.data_len)
  447. return tn->buf.dmabuf[tn->buf.data_index++];
  448. dev_err(tn->dev, "No data to read, idx: 0x%x, len: 0x%x\n",
  449. tn->buf.data_index, tn->buf.data_len);
  450. return 0xff;
  451. }
  452. /*
  453. * Read a number of pending bytes from the temporary buffer. Used
  454. * to get page and OOB data.
  455. */
  456. static void octeontx_nand_read_buf(struct mtd_info *mtd, u8 *buf, int len)
  457. {
  458. struct nand_chip *nand = mtd_to_nand(mtd);
  459. struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
  460. if (len > tn->buf.data_len - tn->buf.data_index) {
  461. dev_err(tn->dev, "Not enough data for read of %d bytes\n", len);
  462. return;
  463. }
  464. memcpy(buf, tn->buf.dmabuf + tn->buf.data_index, len);
  465. tn->buf.data_index += len;
  466. }
  467. static void octeontx_nand_write_buf(struct mtd_info *mtd,
  468. const u8 *buf, int len)
  469. {
  470. struct nand_chip *nand = mtd_to_nand(mtd);
  471. struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
  472. memcpy(tn->buf.dmabuf + tn->buf.data_len, buf, len);
  473. tn->buf.data_len += len;
  474. }
  475. /* Overwrite default function to avoid sync abort on chip = -1. */
  476. static void octeontx_nand_select_chip(struct mtd_info *mtd, int chip)
  477. {
  478. }
  479. static inline int timing_to_cycle(u32 psec, unsigned long clock)
  480. {
  481. unsigned int ns;
  482. int ticks;
  483. ns = DIV_ROUND_UP(psec, 1000);
  484. ns += slew_ns;
  485. /* no rounding needed since clock is multiple of 1MHz */
  486. clock /= 1000000;
  487. ns *= clock;
  488. ticks = DIV_ROUND_UP(ns, 1000);
  489. /* actual delay is (tm_parX+1)<<tim_mult */
  490. if (ticks)
  491. ticks--;
  492. return ticks;
  493. }
  494. static void set_timings(struct octeontx_nand_chip *chip,
  495. struct ndf_set_tm_par_cmd *tp,
  496. const struct nand_sdr_timings *timings,
  497. unsigned long sclk)
  498. {
  499. /* scaled coprocessor-cycle values */
  500. u32 s_wh, s_cls, s_clh, s_rp, s_wb, s_wc;
  501. tp->tim_mult = 0;
  502. s_wh = timing_to_cycle(timings->tWH_min, sclk);
  503. s_cls = timing_to_cycle(timings->tCLS_min, sclk);
  504. s_clh = timing_to_cycle(timings->tCLH_min, sclk);
  505. s_rp = timing_to_cycle(timings->tRP_min, sclk);
  506. s_wb = timing_to_cycle(timings->tWB_max, sclk);
  507. s_wc = timing_to_cycle(timings->tWC_min, sclk);
  508. tp->tm_par1 = s_wh;
  509. tp->tm_par2 = s_clh;
  510. tp->tm_par3 = s_rp + 1;
  511. tp->tm_par4 = s_cls - s_wh;
  512. tp->tm_par5 = s_wc - s_wh + 1;
  513. tp->tm_par6 = s_wb;
  514. tp->tm_par7 = 0;
  515. tp->tim_mult++; /* overcompensate for bad math */
  516. /* TODO: comment parameter re-use */
  517. pr_debug("%s: tim_par: mult: %d p1: %d p2: %d p3: %d\n",
  518. __func__, tp->tim_mult, tp->tm_par1, tp->tm_par2, tp->tm_par3);
  519. pr_debug(" p4: %d p5: %d p6: %d p7: %d\n",
  520. tp->tm_par4, tp->tm_par5, tp->tm_par6, tp->tm_par7);
  521. }
  522. static int set_default_timings(struct octeontx_nfc *tn,
  523. const struct nand_sdr_timings *timings)
  524. {
  525. unsigned long sclk = octeontx_get_io_clock();
  526. set_timings(NULL, &default_timing_parms, timings, sclk);
  527. return 0;
  528. }
  529. static int octeontx_nfc_chip_set_timings(struct octeontx_nand_chip *chip,
  530. const struct nand_sdr_timings *timings)
  531. {
  532. /*struct octeontx_nfc *tn = to_otx_nfc(chip->nand.controller);*/
  533. unsigned long sclk = octeontx_get_io_clock();
  534. set_timings(chip, &chip->timings, timings, sclk);
  535. return 0;
  536. }
  537. /* How many bytes are free in the NFD_CMD queue? */
  538. static int ndf_cmd_queue_free(struct octeontx_nfc *tn)
  539. {
  540. u64 ndf_misc;
  541. ndf_misc = readq(tn->base + NDF_MISC);
  542. return FIELD_GET(NDF_MISC_FR_BYTE, ndf_misc);
  543. }
  544. /* Submit a command to the NAND command queue. */
  545. static int ndf_submit(struct octeontx_nfc *tn, union ndf_cmd *cmd)
  546. {
  547. int opcode = cmd->val[0] & 0xf;
  548. switch (opcode) {
  549. /* All these commands fit in one 64bit word */
  550. case NDF_OP_NOP:
  551. case NDF_OP_SET_TM_PAR:
  552. case NDF_OP_WAIT:
  553. case NDF_OP_CHIP_EN_DIS:
  554. case NDF_OP_CLE_CMD:
  555. case NDF_OP_WR_CMD:
  556. case NDF_OP_RD_CMD:
  557. case NDF_OP_RD_EDO_CMD:
  558. case NDF_OP_BUS_ACQ_REL:
  559. if (ndf_cmd_queue_free(tn) < 8)
  560. goto full;
  561. writeq(cmd->val[0], tn->base + NDF_CMD);
  562. break;
  563. case NDF_OP_ALE_CMD:
  564. /* ALE commands take either one or two 64bit words */
  565. if (cmd->u.ale_cmd.adr_byte_num < 5) {
  566. if (ndf_cmd_queue_free(tn) < 8)
  567. goto full;
  568. writeq(cmd->val[0], tn->base + NDF_CMD);
  569. } else {
  570. if (ndf_cmd_queue_free(tn) < 16)
  571. goto full;
  572. writeq(cmd->val[0], tn->base + NDF_CMD);
  573. writeq(cmd->val[1], tn->base + NDF_CMD);
  574. }
  575. break;
  576. case NDF_OP_WAIT_STATUS: /* Wait status commands take two 64bit words */
  577. if (ndf_cmd_queue_free(tn) < 16)
  578. goto full;
  579. writeq(cmd->val[0], tn->base + NDF_CMD);
  580. writeq(cmd->val[1], tn->base + NDF_CMD);
  581. break;
  582. default:
  583. dev_err(tn->dev, "%s: unknown command: %u\n", __func__, opcode);
  584. return -EINVAL;
  585. }
  586. return 0;
  587. full:
  588. dev_err(tn->dev, "%s: no space left in command queue\n", __func__);
  589. return -ENOMEM;
  590. }
  591. /**
  592. * Wait for the ready/busy signal. First wait for busy to be valid,
  593. * then wait for busy to de-assert.
  594. */
  595. static int ndf_build_wait_busy(struct octeontx_nfc *tn)
  596. {
  597. union ndf_cmd cmd;
  598. memset(&cmd, 0, sizeof(cmd));
  599. cmd.u.wait.opcode = NDF_OP_WAIT;
  600. cmd.u.wait.r_b = 1;
  601. cmd.u.wait.wlen = t6;
  602. if (ndf_submit(tn, &cmd))
  603. return -ENOMEM;
  604. return 0;
  605. }
  606. static bool ndf_dma_done(struct octeontx_nfc *tn)
  607. {
  608. u64 dma_cfg;
  609. /* Enable bit should be clear after a transfer */
  610. dma_cfg = readq(tn->base + NDF_DMA_CFG);
  611. if (!(dma_cfg & NDF_DMA_CFG_EN))
  612. return true;
  613. return false;
  614. }
  615. static int ndf_wait(struct octeontx_nfc *tn)
  616. {
  617. ulong start = get_timer(0);
  618. bool done;
  619. while (!(done = ndf_dma_done(tn)) && get_timer(start) < NDF_TIMEOUT)
  620. ;
  621. if (!done) {
  622. dev_err(tn->dev, "%s: timeout error\n", __func__);
  623. return -ETIMEDOUT;
  624. }
  625. return 0;
  626. }
  627. static int ndf_wait_idle(struct octeontx_nfc *tn)
  628. {
  629. u64 val;
  630. u64 dval = 0;
  631. int rc;
  632. int pause = 100;
  633. u64 tot_us = USEC_PER_SEC / 10;
  634. rc = readq_poll_timeout(tn->base + NDF_ST_REG,
  635. val, val & NDF_ST_REG_EXE_IDLE, pause, tot_us);
  636. if (!rc)
  637. rc = readq_poll_timeout(tn->base + NDF_DMA_CFG,
  638. dval, !(dval & NDF_DMA_CFG_EN),
  639. pause, tot_us);
  640. return rc;
  641. }
  642. /** Issue set timing parameters */
  643. static int ndf_queue_cmd_timing(struct octeontx_nfc *tn,
  644. struct ndf_set_tm_par_cmd *timings)
  645. {
  646. union ndf_cmd cmd;
  647. memset(&cmd, 0, sizeof(cmd));
  648. cmd.u.set_tm_par.opcode = NDF_OP_SET_TM_PAR;
  649. cmd.u.set_tm_par.tim_mult = timings->tim_mult;
  650. cmd.u.set_tm_par.tm_par1 = timings->tm_par1;
  651. cmd.u.set_tm_par.tm_par2 = timings->tm_par2;
  652. cmd.u.set_tm_par.tm_par3 = timings->tm_par3;
  653. cmd.u.set_tm_par.tm_par4 = timings->tm_par4;
  654. cmd.u.set_tm_par.tm_par5 = timings->tm_par5;
  655. cmd.u.set_tm_par.tm_par6 = timings->tm_par6;
  656. cmd.u.set_tm_par.tm_par7 = timings->tm_par7;
  657. return ndf_submit(tn, &cmd);
  658. }
  659. /** Issue bus acquire or release */
  660. static int ndf_queue_cmd_bus(struct octeontx_nfc *tn, int direction)
  661. {
  662. union ndf_cmd cmd;
  663. memset(&cmd, 0, sizeof(cmd));
  664. cmd.u.bus_acq_rel.opcode = NDF_OP_BUS_ACQ_REL;
  665. cmd.u.bus_acq_rel.direction = direction;
  666. return ndf_submit(tn, &cmd);
  667. }
  668. /* Issue chip select or deselect */
  669. static int ndf_queue_cmd_chip(struct octeontx_nfc *tn, int enable, int chip,
  670. int width)
  671. {
  672. union ndf_cmd cmd;
  673. memset(&cmd, 0, sizeof(cmd));
  674. cmd.u.chip_en_dis.opcode = NDF_OP_CHIP_EN_DIS;
  675. cmd.u.chip_en_dis.chip = chip;
  676. cmd.u.chip_en_dis.enable = enable;
  677. cmd.u.chip_en_dis.bus_width = width;
  678. return ndf_submit(tn, &cmd);
  679. }
  680. static int ndf_queue_cmd_wait(struct octeontx_nfc *tn, int t_delay)
  681. {
  682. union ndf_cmd cmd;
  683. memset(&cmd, 0, sizeof(cmd));
  684. cmd.u.wait.opcode = NDF_OP_WAIT;
  685. cmd.u.wait.wlen = t_delay;
  686. return ndf_submit(tn, &cmd);
  687. }
  688. static int ndf_queue_cmd_cle(struct octeontx_nfc *tn, int command)
  689. {
  690. union ndf_cmd cmd;
  691. memset(&cmd, 0, sizeof(cmd));
  692. cmd.u.cle_cmd.opcode = NDF_OP_CLE_CMD;
  693. cmd.u.cle_cmd.cmd_data = command;
  694. cmd.u.cle_cmd.clen1 = t4;
  695. cmd.u.cle_cmd.clen2 = t1;
  696. cmd.u.cle_cmd.clen3 = t2;
  697. return ndf_submit(tn, &cmd);
  698. }
  699. static int ndf_queue_cmd_ale(struct octeontx_nfc *tn, int addr_bytes,
  700. struct nand_chip *nand, u64 page,
  701. u32 col, int page_size)
  702. {
  703. struct octeontx_nand_chip *octeontx_nand = (nand) ?
  704. to_otx_nand(nand) : NULL;
  705. union ndf_cmd cmd;
  706. memset(&cmd, 0, sizeof(cmd));
  707. cmd.u.ale_cmd.opcode = NDF_OP_ALE_CMD;
  708. cmd.u.ale_cmd.adr_byte_num = addr_bytes;
  709. /* set column bit for OOB area, assume OOB follows page */
  710. if (octeontx_nand && octeontx_nand->oob_only)
  711. col += page_size;
  712. /* page is u64 for this generality, even if cmdfunc() passes int */
  713. switch (addr_bytes) {
  714. /* 4-8 bytes: page, then 2-byte col */
  715. case 8:
  716. cmd.u.ale_cmd.adr_byt8 = (page >> 40) & 0xff;
  717. fallthrough;
  718. case 7:
  719. cmd.u.ale_cmd.adr_byt7 = (page >> 32) & 0xff;
  720. fallthrough;
  721. case 6:
  722. cmd.u.ale_cmd.adr_byt6 = (page >> 24) & 0xff;
  723. fallthrough;
  724. case 5:
  725. cmd.u.ale_cmd.adr_byt5 = (page >> 16) & 0xff;
  726. fallthrough;
  727. case 4:
  728. cmd.u.ale_cmd.adr_byt4 = (page >> 8) & 0xff;
  729. cmd.u.ale_cmd.adr_byt3 = page & 0xff;
  730. cmd.u.ale_cmd.adr_byt2 = (col >> 8) & 0xff;
  731. cmd.u.ale_cmd.adr_byt1 = col & 0xff;
  732. break;
  733. /* 1-3 bytes: just the page address */
  734. case 3:
  735. cmd.u.ale_cmd.adr_byt3 = (page >> 16) & 0xff;
  736. fallthrough;
  737. case 2:
  738. cmd.u.ale_cmd.adr_byt2 = (page >> 8) & 0xff;
  739. fallthrough;
  740. case 1:
  741. cmd.u.ale_cmd.adr_byt1 = page & 0xff;
  742. break;
  743. default:
  744. break;
  745. }
  746. cmd.u.ale_cmd.alen1 = t3;
  747. cmd.u.ale_cmd.alen2 = t1;
  748. cmd.u.ale_cmd.alen3 = t5;
  749. cmd.u.ale_cmd.alen4 = t2;
  750. return ndf_submit(tn, &cmd);
  751. }
  752. static int ndf_queue_cmd_write(struct octeontx_nfc *tn, int len)
  753. {
  754. union ndf_cmd cmd;
  755. memset(&cmd, 0, sizeof(cmd));
  756. cmd.u.wr_cmd.opcode = NDF_OP_WR_CMD;
  757. cmd.u.wr_cmd.data = len;
  758. cmd.u.wr_cmd.wlen1 = t3;
  759. cmd.u.wr_cmd.wlen2 = t1;
  760. return ndf_submit(tn, &cmd);
  761. }
  762. static int ndf_build_pre_cmd(struct octeontx_nfc *tn, int cmd1,
  763. int addr_bytes, u64 page, u32 col, int cmd2)
  764. {
  765. struct nand_chip *nand = tn->controller.active;
  766. struct octeontx_nand_chip *octeontx_nand;
  767. struct ndf_set_tm_par_cmd *timings;
  768. int width, page_size, rc;
  769. /* Also called before chip probing is finished */
  770. if (!nand) {
  771. timings = &default_timing_parms;
  772. page_size = default_page_size;
  773. width = default_width;
  774. } else {
  775. octeontx_nand = to_otx_nand(nand);
  776. timings = &octeontx_nand->timings;
  777. page_size = nand->mtd.writesize;
  778. if (nand->options & NAND_BUSWIDTH_16)
  779. width = 2;
  780. else
  781. width = 1;
  782. }
  783. rc = ndf_queue_cmd_timing(tn, timings);
  784. if (rc)
  785. return rc;
  786. rc = ndf_queue_cmd_bus(tn, NDF_BUS_ACQUIRE);
  787. if (rc)
  788. return rc;
  789. rc = ndf_queue_cmd_chip(tn, 1, tn->selected_chip, width);
  790. if (rc)
  791. return rc;
  792. rc = ndf_queue_cmd_wait(tn, t1);
  793. if (rc)
  794. return rc;
  795. rc = ndf_queue_cmd_cle(tn, cmd1);
  796. if (rc)
  797. return rc;
  798. if (addr_bytes) {
  799. rc = ndf_build_wait_busy(tn);
  800. if (rc)
  801. return rc;
  802. rc = ndf_queue_cmd_ale(tn, addr_bytes, nand,
  803. page, col, page_size);
  804. if (rc)
  805. return rc;
  806. }
  807. /* CLE 2 */
  808. if (cmd2) {
  809. rc = ndf_build_wait_busy(tn);
  810. if (rc)
  811. return rc;
  812. rc = ndf_queue_cmd_cle(tn, cmd2);
  813. if (rc)
  814. return rc;
  815. }
  816. return 0;
  817. }
  818. static int ndf_build_post_cmd(struct octeontx_nfc *tn, int hold_time)
  819. {
  820. int rc;
  821. /* Deselect chip */
  822. rc = ndf_queue_cmd_chip(tn, 0, 0, 0);
  823. if (rc)
  824. return rc;
  825. rc = ndf_queue_cmd_wait(tn, t2);
  826. if (rc)
  827. return rc;
  828. /* Release bus */
  829. rc = ndf_queue_cmd_bus(tn, 0);
  830. if (rc)
  831. return rc;
  832. rc = ndf_queue_cmd_wait(tn, hold_time);
  833. if (rc)
  834. return rc;
  835. /*
  836. * Last action is ringing the doorbell with number of bus
  837. * acquire-releases cycles (currently 1).
  838. */
  839. writeq(1, tn->base + NDF_DRBELL);
  840. return 0;
  841. }
  842. /* Setup the NAND DMA engine for a transfer. */
  843. static void ndf_setup_dma(struct octeontx_nfc *tn, int is_write,
  844. dma_addr_t bus_addr, int len)
  845. {
  846. u64 dma_cfg;
  847. dma_cfg = FIELD_PREP(NDF_DMA_CFG_RW, is_write) |
  848. FIELD_PREP(NDF_DMA_CFG_SIZE, (len >> 3) - 1);
  849. dma_cfg |= NDF_DMA_CFG_EN;
  850. writeq(bus_addr, tn->base + NDF_DMA_ADR);
  851. writeq(dma_cfg, tn->base + NDF_DMA_CFG);
  852. }
  853. static int octeontx_nand_reset(struct octeontx_nfc *tn)
  854. {
  855. int rc;
  856. rc = ndf_build_pre_cmd(tn, NAND_CMD_RESET, 0, 0, 0, 0);
  857. if (rc)
  858. return rc;
  859. rc = ndf_build_wait_busy(tn);
  860. if (rc)
  861. return rc;
  862. rc = ndf_build_post_cmd(tn, t2);
  863. if (rc)
  864. return rc;
  865. return 0;
  866. }
  867. static int ndf_read(struct octeontx_nfc *tn, int cmd1, int addr_bytes,
  868. u64 page, u32 col, int cmd2, int len)
  869. {
  870. dma_addr_t bus_addr = tn->use_status ? tn->stat_addr : tn->buf.dmaaddr;
  871. struct nand_chip *nand = tn->controller.active;
  872. int timing_mode, bytes, rc;
  873. union ndf_cmd cmd;
  874. u64 start, end;
  875. pr_debug("%s(%p, 0x%x, 0x%x, 0x%llx, 0x%x, 0x%x, 0x%x)\n", __func__,
  876. tn, cmd1, addr_bytes, page, col, cmd2, len);
  877. if (!nand)
  878. timing_mode = default_onfi_timing;
  879. else
  880. timing_mode = nand->onfi_timing_mode_default;
  881. /* Build the command and address cycles */
  882. rc = ndf_build_pre_cmd(tn, cmd1, addr_bytes, page, col, cmd2);
  883. if (rc) {
  884. dev_err(tn->dev, "Build pre command failed\n");
  885. return rc;
  886. }
  887. /* This waits for some time, then waits for busy to be de-asserted. */
  888. rc = ndf_build_wait_busy(tn);
  889. if (rc) {
  890. dev_err(tn->dev, "Wait timeout\n");
  891. return rc;
  892. }
  893. memset(&cmd, 0, sizeof(cmd));
  894. if (timing_mode < 4)
  895. cmd.u.rd_cmd.opcode = NDF_OP_RD_CMD;
  896. else
  897. cmd.u.rd_cmd.opcode = NDF_OP_RD_EDO_CMD;
  898. cmd.u.rd_cmd.data = len;
  899. cmd.u.rd_cmd.rlen1 = t7;
  900. cmd.u.rd_cmd.rlen2 = t3;
  901. cmd.u.rd_cmd.rlen3 = t1;
  902. cmd.u.rd_cmd.rlen4 = t7;
  903. rc = ndf_submit(tn, &cmd);
  904. if (rc) {
  905. dev_err(tn->dev, "Error submitting command\n");
  906. return rc;
  907. }
  908. start = (u64)bus_addr;
  909. ndf_setup_dma(tn, 0, bus_addr, len);
  910. rc = ndf_build_post_cmd(tn, t2);
  911. if (rc) {
  912. dev_err(tn->dev, "Build post command failed\n");
  913. return rc;
  914. }
  915. /* Wait for the DMA to complete */
  916. rc = ndf_wait(tn);
  917. if (rc) {
  918. dev_err(tn->dev, "DMA timed out\n");
  919. return rc;
  920. }
  921. end = readq(tn->base + NDF_DMA_ADR);
  922. bytes = end - start;
  923. /* Make sure NDF is really done */
  924. rc = ndf_wait_idle(tn);
  925. if (rc) {
  926. dev_err(tn->dev, "poll idle failed\n");
  927. return rc;
  928. }
  929. pr_debug("%s: Read %d bytes\n", __func__, bytes);
  930. return bytes;
  931. }
  932. static int octeontx_nand_get_features(struct mtd_info *mtd,
  933. struct nand_chip *chip, int feature_addr,
  934. u8 *subfeature_para)
  935. {
  936. struct nand_chip *nand = chip;
  937. struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
  938. int len = 8;
  939. int rc;
  940. pr_debug("%s: feature addr: 0x%x\n", __func__, feature_addr);
  941. memset(tn->buf.dmabuf, 0xff, len);
  942. tn->buf.data_index = 0;
  943. tn->buf.data_len = 0;
  944. rc = ndf_read(tn, NAND_CMD_GET_FEATURES, 1, feature_addr, 0, 0, len);
  945. if (rc)
  946. return rc;
  947. memcpy(subfeature_para, tn->buf.dmabuf, ONFI_SUBFEATURE_PARAM_LEN);
  948. return 0;
  949. }
  950. static int octeontx_nand_set_features(struct mtd_info *mtd,
  951. struct nand_chip *chip, int feature_addr,
  952. u8 *subfeature_para)
  953. {
  954. struct nand_chip *nand = chip;
  955. struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
  956. const int len = ONFI_SUBFEATURE_PARAM_LEN;
  957. int rc;
  958. rc = ndf_build_pre_cmd(tn, NAND_CMD_SET_FEATURES,
  959. 1, feature_addr, 0, 0);
  960. if (rc)
  961. return rc;
  962. memcpy(tn->buf.dmabuf, subfeature_para, len);
  963. memset(tn->buf.dmabuf + len, 0, 8 - len);
  964. ndf_setup_dma(tn, 1, tn->buf.dmaaddr, 8);
  965. rc = ndf_queue_cmd_write(tn, 8);
  966. if (rc)
  967. return rc;
  968. rc = ndf_build_wait_busy(tn);
  969. if (rc)
  970. return rc;
  971. rc = ndf_build_post_cmd(tn, t2);
  972. if (rc)
  973. return rc;
  974. return 0;
  975. }
  976. /*
  977. * Read a page from NAND. If the buffer has room, the out of band
  978. * data will be included.
  979. */
  980. static int ndf_page_read(struct octeontx_nfc *tn, u64 page, int col, int len)
  981. {
  982. debug("%s(%p, 0x%llx, 0x%x, 0x%x) active: %p\n", __func__,
  983. tn, page, col, len, tn->controller.active);
  984. struct nand_chip *nand = tn->controller.active;
  985. struct octeontx_nand_chip *chip = to_otx_nand(nand);
  986. int addr_bytes = chip->row_bytes + chip->col_bytes;
  987. memset(tn->buf.dmabuf, 0xff, len);
  988. return ndf_read(tn, NAND_CMD_READ0, addr_bytes,
  989. page, col, NAND_CMD_READSTART, len);
  990. }
  991. /* Erase a NAND block */
  992. static int ndf_block_erase(struct octeontx_nfc *tn, u64 page_addr)
  993. {
  994. struct nand_chip *nand = tn->controller.active;
  995. struct octeontx_nand_chip *chip = to_otx_nand(nand);
  996. int addr_bytes = chip->row_bytes;
  997. int rc;
  998. rc = ndf_build_pre_cmd(tn, NAND_CMD_ERASE1, addr_bytes,
  999. page_addr, 0, NAND_CMD_ERASE2);
  1000. if (rc)
  1001. return rc;
  1002. /* Wait for R_B to signal erase is complete */
  1003. rc = ndf_build_wait_busy(tn);
  1004. if (rc)
  1005. return rc;
  1006. rc = ndf_build_post_cmd(tn, t2);
  1007. if (rc)
  1008. return rc;
  1009. /* Wait until the command queue is idle */
  1010. return ndf_wait_idle(tn);
  1011. }
  1012. /*
  1013. * Write a page (or less) to NAND.
  1014. */
  1015. static int ndf_page_write(struct octeontx_nfc *tn, int page)
  1016. {
  1017. int len, rc;
  1018. struct nand_chip *nand = tn->controller.active;
  1019. struct octeontx_nand_chip *chip = to_otx_nand(nand);
  1020. int addr_bytes = chip->row_bytes + chip->col_bytes;
  1021. len = tn->buf.data_len - tn->buf.data_index;
  1022. chip->oob_only = (tn->buf.data_index >= nand->mtd.writesize);
  1023. WARN_ON_ONCE(len & 0x7);
  1024. ndf_setup_dma(tn, 1, tn->buf.dmaaddr + tn->buf.data_index, len);
  1025. rc = ndf_build_pre_cmd(tn, NAND_CMD_SEQIN, addr_bytes, page, 0, 0);
  1026. if (rc)
  1027. return rc;
  1028. rc = ndf_queue_cmd_write(tn, len);
  1029. if (rc)
  1030. return rc;
  1031. rc = ndf_queue_cmd_cle(tn, NAND_CMD_PAGEPROG);
  1032. if (rc)
  1033. return rc;
  1034. /* Wait for R_B to signal program is complete */
  1035. rc = ndf_build_wait_busy(tn);
  1036. if (rc)
  1037. return rc;
  1038. rc = ndf_build_post_cmd(tn, t2);
  1039. if (rc)
  1040. return rc;
  1041. /* Wait for the DMA to complete */
  1042. rc = ndf_wait(tn);
  1043. if (rc)
  1044. return rc;
  1045. /* Data transfer is done but NDF is not, it is waiting for R/B# */
  1046. return ndf_wait_idle(tn);
  1047. }
  1048. static void octeontx_nand_cmdfunc(struct mtd_info *mtd, unsigned int command,
  1049. int column, int page_addr)
  1050. {
  1051. struct nand_chip *nand = mtd_to_nand(mtd);
  1052. struct octeontx_nand_chip *octeontx_nand = to_otx_nand(nand);
  1053. struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
  1054. int rc;
  1055. tn->selected_chip = octeontx_nand->cs;
  1056. if (tn->selected_chip < 0 || tn->selected_chip >= NAND_MAX_CHIPS) {
  1057. dev_err(tn->dev, "invalid chip select\n");
  1058. return;
  1059. }
  1060. tn->use_status = false;
  1061. pr_debug("%s(%p, 0x%x, 0x%x, 0x%x) cs: %d\n", __func__, mtd, command,
  1062. column, page_addr, tn->selected_chip);
  1063. switch (command) {
  1064. case NAND_CMD_READID:
  1065. tn->buf.data_index = 0;
  1066. octeontx_nand->oob_only = false;
  1067. rc = ndf_read(tn, command, 1, column, 0, 0, 8);
  1068. if (rc < 0)
  1069. dev_err(tn->dev, "READID failed with %d\n", rc);
  1070. else
  1071. tn->buf.data_len = rc;
  1072. break;
  1073. case NAND_CMD_READOOB:
  1074. octeontx_nand->oob_only = true;
  1075. tn->buf.data_index = 0;
  1076. tn->buf.data_len = 0;
  1077. rc = ndf_page_read(tn, page_addr, column, mtd->oobsize);
  1078. if (rc < mtd->oobsize)
  1079. dev_err(tn->dev, "READOOB failed with %d\n",
  1080. tn->buf.data_len);
  1081. else
  1082. tn->buf.data_len = rc;
  1083. break;
  1084. case NAND_CMD_READ0:
  1085. octeontx_nand->oob_only = false;
  1086. tn->buf.data_index = 0;
  1087. tn->buf.data_len = 0;
  1088. rc = ndf_page_read(tn, page_addr, column,
  1089. mtd->writesize + mtd->oobsize);
  1090. if (rc < mtd->writesize + mtd->oobsize)
  1091. dev_err(tn->dev, "READ0 failed with %d\n", rc);
  1092. else
  1093. tn->buf.data_len = rc;
  1094. break;
  1095. case NAND_CMD_STATUS:
  1096. /* used in oob/not states */
  1097. tn->use_status = true;
  1098. rc = ndf_read(tn, command, 0, 0, 0, 0, 8);
  1099. if (rc < 0)
  1100. dev_err(tn->dev, "STATUS failed with %d\n", rc);
  1101. break;
  1102. case NAND_CMD_RESET:
  1103. /* used in oob/not states */
  1104. rc = octeontx_nand_reset(tn);
  1105. if (rc < 0)
  1106. dev_err(tn->dev, "RESET failed with %d\n", rc);
  1107. break;
  1108. case NAND_CMD_PARAM:
  1109. octeontx_nand->oob_only = false;
  1110. tn->buf.data_index = 0;
  1111. rc = ndf_read(tn, command, 1, 0, 0, 0,
  1112. min(tn->buf.dmabuflen, 3 * 512));
  1113. if (rc < 0)
  1114. dev_err(tn->dev, "PARAM failed with %d\n", rc);
  1115. else
  1116. tn->buf.data_len = rc;
  1117. break;
  1118. case NAND_CMD_RNDOUT:
  1119. tn->buf.data_index = column;
  1120. break;
  1121. case NAND_CMD_ERASE1:
  1122. if (ndf_block_erase(tn, page_addr))
  1123. dev_err(tn->dev, "ERASE1 failed\n");
  1124. break;
  1125. case NAND_CMD_ERASE2:
  1126. /* We do all erase processing in the first command, so ignore
  1127. * this one.
  1128. */
  1129. break;
  1130. case NAND_CMD_SEQIN:
  1131. octeontx_nand->oob_only = (column >= mtd->writesize);
  1132. tn->buf.data_index = column;
  1133. tn->buf.data_len = column;
  1134. octeontx_nand->selected_page = page_addr;
  1135. break;
  1136. case NAND_CMD_PAGEPROG:
  1137. rc = ndf_page_write(tn, octeontx_nand->selected_page);
  1138. if (rc)
  1139. dev_err(tn->dev, "PAGEPROG failed with %d\n", rc);
  1140. break;
  1141. case NAND_CMD_SET_FEATURES:
  1142. octeontx_nand->oob_only = false;
  1143. /* assume tn->buf.data_len == 4 of data has been set there */
  1144. rc = octeontx_nand_set_features(mtd, nand,
  1145. page_addr, tn->buf.dmabuf);
  1146. if (rc)
  1147. dev_err(tn->dev, "SET_FEATURES failed with %d\n", rc);
  1148. break;
  1149. case NAND_CMD_GET_FEATURES:
  1150. octeontx_nand->oob_only = false;
  1151. rc = octeontx_nand_get_features(mtd, nand,
  1152. page_addr, tn->buf.dmabuf);
  1153. if (!rc) {
  1154. tn->buf.data_index = 0;
  1155. tn->buf.data_len = 4;
  1156. } else {
  1157. dev_err(tn->dev, "GET_FEATURES failed with %d\n", rc);
  1158. }
  1159. break;
  1160. default:
  1161. WARN_ON_ONCE(1);
  1162. dev_err(tn->dev, "unhandled nand cmd: %x\n", command);
  1163. }
  1164. }
  1165. static int octeontx_nand_waitfunc(struct mtd_info *mtd, struct nand_chip *chip)
  1166. {
  1167. struct octeontx_nfc *tn = to_otx_nfc(chip->controller);
  1168. int ret;
  1169. ret = ndf_wait_idle(tn);
  1170. return (ret < 0) ? -EIO : 0;
  1171. }
  1172. /* check compatibility with ONFI timing mode#N, and optionally apply */
  1173. /* TODO: Implement chipnr support? */
  1174. static int octeontx_nand_setup_dat_intf(struct mtd_info *mtd, int chipnr,
  1175. const struct nand_data_interface *conf)
  1176. {
  1177. static const bool check_only;
  1178. struct nand_chip *nand = mtd_to_nand(mtd);
  1179. struct octeontx_nand_chip *chip = to_otx_nand(nand);
  1180. static u64 t_wc_n[MAX_ONFI_MODE + 2]; /* cache a mode signature */
  1181. int mode; /* deduced mode number, for reporting and restricting */
  1182. int rc;
  1183. /*
  1184. * Cache timing modes for reporting, and reducing needless change.
  1185. *
  1186. * Challenge: caller does not pass ONFI mode#, but reporting the mode
  1187. * and restricting to a maximum, or a list, are useful for diagnosing
  1188. * new hardware. So use tWC_min, distinct and monotonic across modes,
  1189. * to discover the requested/accepted mode number
  1190. */
  1191. for (mode = MAX_ONFI_MODE; mode >= 0 && !t_wc_n[0]; mode--) {
  1192. const struct nand_sdr_timings *t;
  1193. t = onfi_async_timing_mode_to_sdr_timings(mode);
  1194. if (!t)
  1195. continue;
  1196. t_wc_n[mode] = t->tWC_min;
  1197. }
  1198. if (!conf) {
  1199. rc = -EINVAL;
  1200. } else if (check_only) {
  1201. rc = 0;
  1202. } else if (nand->data_interface &&
  1203. chip->iface_set && chip->iface_mode == mode) {
  1204. /*
  1205. * Cases:
  1206. * - called from nand_reset, which clears DDR timing
  1207. * mode back to SDR. BUT if we're already in SDR,
  1208. * timing mode persists over resets.
  1209. * While mtd/nand layer only supports SDR,
  1210. * this is always safe. And this driver only supports SDR.
  1211. *
  1212. * - called from post-power-event nand_reset (maybe
  1213. * NFC+flash power down, or system hibernate.
  1214. * Address this when CONFIG_PM support added
  1215. */
  1216. rc = 0;
  1217. } else {
  1218. rc = octeontx_nfc_chip_set_timings(chip, &conf->timings.sdr);
  1219. if (!rc) {
  1220. chip->iface_mode = mode;
  1221. chip->iface_set = true;
  1222. }
  1223. }
  1224. return rc;
  1225. }
  1226. static void octeontx_bch_reset(void)
  1227. {
  1228. }
  1229. /*
  1230. * Given a page, calculate the ECC code
  1231. *
  1232. * chip: Pointer to NAND chip data structure
  1233. * buf: Buffer to calculate ECC on
  1234. * code: Buffer to hold ECC data
  1235. *
  1236. * Return 0 on success or -1 on failure
  1237. */
  1238. static int octeontx_nand_bch_calculate_ecc_internal(struct mtd_info *mtd,
  1239. dma_addr_t ihandle,
  1240. u8 *code)
  1241. {
  1242. struct nand_chip *nand = mtd_to_nand(mtd);
  1243. struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
  1244. int rc;
  1245. int i;
  1246. static u8 *ecc_buffer;
  1247. static int ecc_size;
  1248. static unsigned long ecc_handle;
  1249. union bch_resp *r = tn->bch_resp;
  1250. if (!ecc_buffer || ecc_size < nand->ecc.size) {
  1251. ecc_size = nand->ecc.size;
  1252. ecc_buffer = dma_alloc_coherent(ecc_size,
  1253. (unsigned long *)&ecc_handle);
  1254. }
  1255. memset(ecc_buffer, 0, nand->ecc.bytes);
  1256. r->u16 = 0;
  1257. __iowmb(); /* flush done=0 before making request */
  1258. rc = octeontx_bch_encode(bch_vf, ihandle, nand->ecc.size,
  1259. nand->ecc.strength,
  1260. (dma_addr_t)ecc_handle, tn->bch_rhandle);
  1261. if (!rc) {
  1262. octeontx_bch_wait(bch_vf, r, tn->bch_rhandle);
  1263. } else {
  1264. dev_err(tn->dev, "octeontx_bch_encode failed\n");
  1265. return -1;
  1266. }
  1267. if (!r->s.done || r->s.uncorrectable) {
  1268. dev_err(tn->dev,
  1269. "%s timeout, done:%d uncorr:%d corr:%d erased:%d\n",
  1270. __func__, r->s.done, r->s.uncorrectable,
  1271. r->s.num_errors, r->s.erased);
  1272. octeontx_bch_reset();
  1273. return -1;
  1274. }
  1275. memcpy(code, ecc_buffer, nand->ecc.bytes);
  1276. for (i = 0; i < nand->ecc.bytes; i++)
  1277. code[i] ^= tn->eccmask[i];
  1278. return tn->bch_resp->s.num_errors;
  1279. }
  1280. /*
  1281. * Given a page, calculate the ECC code
  1282. *
  1283. * mtd: MTD block structure
  1284. * dat: raw data (unused)
  1285. * ecc_code: buffer for ECC
  1286. */
  1287. static int octeontx_nand_bch_calculate(struct mtd_info *mtd,
  1288. const u8 *dat, u8 *ecc_code)
  1289. {
  1290. struct nand_chip *nand = mtd_to_nand(mtd);
  1291. dma_addr_t handle = dma_map_single((u8 *)dat,
  1292. nand->ecc.size, DMA_TO_DEVICE);
  1293. int ret;
  1294. ret = octeontx_nand_bch_calculate_ecc_internal(mtd, handle,
  1295. (void *)ecc_code);
  1296. return ret;
  1297. }
  1298. /*
  1299. * Detect and correct multi-bit ECC for a page
  1300. *
  1301. * mtd: MTD block structure
  1302. * dat: raw data read from the chip
  1303. * read_ecc: ECC from the chip (unused)
  1304. * isnull: unused
  1305. *
  1306. * Returns number of bits corrected or -1 if unrecoverable
  1307. */
  1308. static int octeontx_nand_bch_correct(struct mtd_info *mtd, u_char *dat,
  1309. u_char *read_ecc, u_char *isnull)
  1310. {
  1311. struct nand_chip *nand = mtd_to_nand(mtd);
  1312. struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
  1313. int i = nand->ecc.size + nand->ecc.bytes;
  1314. static u8 *data_buffer;
  1315. static dma_addr_t ihandle;
  1316. static int buffer_size;
  1317. dma_addr_t ohandle;
  1318. union bch_resp *r = tn->bch_resp;
  1319. int rc;
  1320. if (i > buffer_size) {
  1321. if (buffer_size)
  1322. free(data_buffer);
  1323. data_buffer = dma_alloc_coherent(i,
  1324. (unsigned long *)&ihandle);
  1325. if (!data_buffer) {
  1326. dev_err(tn->dev,
  1327. "%s: Could not allocate %d bytes for buffer\n",
  1328. __func__, i);
  1329. goto error;
  1330. }
  1331. buffer_size = i;
  1332. }
  1333. memcpy(data_buffer, dat, nand->ecc.size);
  1334. memcpy(data_buffer + nand->ecc.size, read_ecc, nand->ecc.bytes);
  1335. for (i = 0; i < nand->ecc.bytes; i++)
  1336. data_buffer[nand->ecc.size + i] ^= tn->eccmask[i];
  1337. r->u16 = 0;
  1338. __iowmb(); /* flush done=0 before making request */
  1339. ohandle = dma_map_single(dat, nand->ecc.size, DMA_FROM_DEVICE);
  1340. rc = octeontx_bch_decode(bch_vf, ihandle, nand->ecc.size,
  1341. nand->ecc.strength, ohandle, tn->bch_rhandle);
  1342. if (!rc)
  1343. octeontx_bch_wait(bch_vf, r, tn->bch_rhandle);
  1344. if (rc) {
  1345. dev_err(tn->dev, "octeontx_bch_decode failed\n");
  1346. goto error;
  1347. }
  1348. if (!r->s.done) {
  1349. dev_err(tn->dev, "Error: BCH engine timeout\n");
  1350. octeontx_bch_reset();
  1351. goto error;
  1352. }
  1353. if (r->s.erased) {
  1354. debug("Info: BCH block is erased\n");
  1355. return 0;
  1356. }
  1357. if (r->s.uncorrectable) {
  1358. debug("Cannot correct NAND block, response: 0x%x\n",
  1359. r->u16);
  1360. goto error;
  1361. }
  1362. return r->s.num_errors;
  1363. error:
  1364. debug("Error performing bch correction\n");
  1365. return -1;
  1366. }
  1367. void octeontx_nand_bch_hwctl(struct mtd_info *mtd, int mode)
  1368. {
  1369. /* Do nothing. */
  1370. }
  1371. static int octeontx_nand_hw_bch_read_page(struct mtd_info *mtd,
  1372. struct nand_chip *chip, u8 *buf,
  1373. int oob_required, int page)
  1374. {
  1375. struct nand_chip *nand = mtd_to_nand(mtd);
  1376. struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
  1377. int i, eccsize = chip->ecc.size, ret;
  1378. int eccbytes = chip->ecc.bytes;
  1379. int eccsteps = chip->ecc.steps;
  1380. u8 *p;
  1381. u8 *ecc_code = chip->buffers->ecccode;
  1382. unsigned int max_bitflips = 0;
  1383. /* chip->read_buf() insists on sequential order, we do OOB first */
  1384. memcpy(chip->oob_poi, tn->buf.dmabuf + mtd->writesize, mtd->oobsize);
  1385. /* Use private buffer as input for ECC correction */
  1386. p = tn->buf.dmabuf;
  1387. ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
  1388. chip->ecc.total);
  1389. if (ret)
  1390. return ret;
  1391. for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
  1392. int stat;
  1393. debug("Correcting block offset %lx, ecc offset %x\n",
  1394. p - buf, i);
  1395. stat = chip->ecc.correct(mtd, p, &ecc_code[i], NULL);
  1396. if (stat < 0) {
  1397. mtd->ecc_stats.failed++;
  1398. debug("Cannot correct NAND page %d\n", page);
  1399. } else {
  1400. mtd->ecc_stats.corrected += stat;
  1401. max_bitflips = max_t(unsigned int, max_bitflips, stat);
  1402. }
  1403. }
  1404. /* Copy corrected data to caller's buffer now */
  1405. memcpy(buf, tn->buf.dmabuf, mtd->writesize);
  1406. return max_bitflips;
  1407. }
  1408. static int octeontx_nand_hw_bch_write_page(struct mtd_info *mtd,
  1409. struct nand_chip *chip,
  1410. const u8 *buf, int oob_required,
  1411. int page)
  1412. {
  1413. struct octeontx_nfc *tn = to_otx_nfc(chip->controller);
  1414. int i, eccsize = chip->ecc.size, ret;
  1415. int eccbytes = chip->ecc.bytes;
  1416. int eccsteps = chip->ecc.steps;
  1417. const u8 *p;
  1418. u8 *ecc_calc = chip->buffers->ecccalc;
  1419. debug("%s(buf?%p, oob%d p%x)\n",
  1420. __func__, buf, oob_required, page);
  1421. for (i = 0; i < chip->ecc.total; i++)
  1422. ecc_calc[i] = 0xFF;
  1423. /* Copy the page data from caller's buffers to private buffer */
  1424. chip->write_buf(mtd, buf, mtd->writesize);
  1425. /* Use private date as source for ECC calculation */
  1426. p = tn->buf.dmabuf;
  1427. /* Hardware ECC calculation */
  1428. for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
  1429. int ret;
  1430. ret = chip->ecc.calculate(mtd, p, &ecc_calc[i]);
  1431. if (ret < 0)
  1432. debug("calculate(mtd, p?%p, &ecc_calc[%d]?%p) returned %d\n",
  1433. p, i, &ecc_calc[i], ret);
  1434. debug("block offset %lx, ecc offset %x\n", p - buf, i);
  1435. }
  1436. ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0,
  1437. chip->ecc.total);
  1438. if (ret)
  1439. return ret;
  1440. /* Store resulting OOB into private buffer, will be sent to HW */
  1441. chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
  1442. return 0;
  1443. }
  1444. /**
  1445. * nand_write_page_raw - [INTERN] raw page write function
  1446. * @mtd: mtd info structure
  1447. * @chip: nand chip info structure
  1448. * @buf: data buffer
  1449. * @oob_required: must write chip->oob_poi to OOB
  1450. * @page: page number to write
  1451. *
  1452. * Not for syndrome calculating ECC controllers, which use a special oob layout.
  1453. */
  1454. static int octeontx_nand_write_page_raw(struct mtd_info *mtd,
  1455. struct nand_chip *chip,
  1456. const u8 *buf, int oob_required,
  1457. int page)
  1458. {
  1459. chip->write_buf(mtd, buf, mtd->writesize);
  1460. if (oob_required)
  1461. chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
  1462. return 0;
  1463. }
  1464. /**
  1465. * octeontx_nand_write_oob_std - [REPLACEABLE] the most common OOB data write
  1466. * function
  1467. * @mtd: mtd info structure
  1468. * @chip: nand chip info structure
  1469. * @page: page number to write
  1470. */
  1471. static int octeontx_nand_write_oob_std(struct mtd_info *mtd,
  1472. struct nand_chip *chip,
  1473. int page)
  1474. {
  1475. int status = 0;
  1476. const u8 *buf = chip->oob_poi;
  1477. int length = mtd->oobsize;
  1478. chip->cmdfunc(mtd, NAND_CMD_SEQIN, mtd->writesize, page);
  1479. chip->write_buf(mtd, buf, length);
  1480. /* Send command to program the OOB data */
  1481. chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
  1482. status = chip->waitfunc(mtd, chip);
  1483. return status & NAND_STATUS_FAIL ? -EIO : 0;
  1484. }
  1485. /**
  1486. * octeontx_nand_read_page_raw - [INTERN] read raw page data without ecc
  1487. * @mtd: mtd info structure
  1488. * @chip: nand chip info structure
  1489. * @buf: buffer to store read data
  1490. * @oob_required: caller requires OOB data read to chip->oob_poi
  1491. * @page: page number to read
  1492. *
  1493. * Not for syndrome calculating ECC controllers, which use a special oob layout.
  1494. */
  1495. static int octeontx_nand_read_page_raw(struct mtd_info *mtd,
  1496. struct nand_chip *chip,
  1497. u8 *buf, int oob_required, int page)
  1498. {
  1499. chip->read_buf(mtd, buf, mtd->writesize);
  1500. if (oob_required)
  1501. chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
  1502. return 0;
  1503. }
  1504. static int octeontx_nand_read_oob_std(struct mtd_info *mtd,
  1505. struct nand_chip *chip,
  1506. int page)
  1507. {
  1508. chip->cmdfunc(mtd, NAND_CMD_READOOB, 0, page);
  1509. chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
  1510. return 0;
  1511. }
  1512. static int octeontx_nand_calc_bch_ecc_strength(struct nand_chip *nand)
  1513. {
  1514. struct mtd_info *mtd = nand_to_mtd(nand);
  1515. struct nand_ecc_ctrl *ecc = &nand->ecc;
  1516. struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
  1517. int nsteps = mtd->writesize / ecc->size;
  1518. int oobchunk = mtd->oobsize / nsteps;
  1519. /* ecc->strength determines ecc_level and OOB's ecc_bytes. */
  1520. const u8 strengths[] = {4, 8, 16, 24, 32, 40, 48, 56, 60, 64};
  1521. /* first set the desired ecc_level to match strengths[] */
  1522. int index = ARRAY_SIZE(strengths) - 1;
  1523. int need;
  1524. while (index > 0 && !(ecc->options & NAND_ECC_MAXIMIZE) &&
  1525. strengths[index - 1] >= ecc->strength)
  1526. index--;
  1527. do {
  1528. need = DIV_ROUND_UP(15 * strengths[index], 8);
  1529. if (need <= oobchunk - 2)
  1530. break;
  1531. } while (index > 0);
  1532. debug("%s: steps ds: %d, strength ds: %d\n", __func__,
  1533. nand->ecc_step_ds, nand->ecc_strength_ds);
  1534. ecc->strength = strengths[index];
  1535. ecc->bytes = need;
  1536. debug("%s: strength: %d, bytes: %d\n", __func__, ecc->strength,
  1537. ecc->bytes);
  1538. if (!tn->eccmask)
  1539. tn->eccmask = devm_kzalloc(tn->dev, ecc->bytes, GFP_KERNEL);
  1540. if (!tn->eccmask)
  1541. return -ENOMEM;
  1542. return 0;
  1543. }
  1544. /* sample the BCH signature of an erased (all 0xff) page,
  1545. * to XOR into all page traffic, so erased pages have no ECC errors
  1546. */
  1547. static int octeontx_bch_save_empty_eccmask(struct nand_chip *nand)
  1548. {
  1549. struct mtd_info *mtd = nand_to_mtd(nand);
  1550. struct octeontx_nfc *tn = to_otx_nfc(nand->controller);
  1551. unsigned int eccsize = nand->ecc.size;
  1552. unsigned int eccbytes = nand->ecc.bytes;
  1553. u8 erased_ecc[eccbytes];
  1554. unsigned long erased_handle;
  1555. unsigned char *erased_page = dma_alloc_coherent(eccsize,
  1556. &erased_handle);
  1557. int i;
  1558. int rc = 0;
  1559. if (!erased_page)
  1560. return -ENOMEM;
  1561. memset(erased_page, 0xff, eccsize);
  1562. memset(erased_ecc, 0, eccbytes);
  1563. rc = octeontx_nand_bch_calculate_ecc_internal(mtd,
  1564. (dma_addr_t)erased_handle,
  1565. erased_ecc);
  1566. free(erased_page);
  1567. for (i = 0; i < eccbytes; i++)
  1568. tn->eccmask[i] = erased_ecc[i] ^ 0xff;
  1569. return rc;
  1570. }
  1571. static void octeontx_nfc_chip_sizing(struct nand_chip *nand)
  1572. {
  1573. struct octeontx_nand_chip *chip = to_otx_nand(nand);
  1574. struct mtd_info *mtd = nand_to_mtd(nand);
  1575. struct nand_ecc_ctrl *ecc = &nand->ecc;
  1576. chip->row_bytes = nand->onfi_params.addr_cycles & 0xf;
  1577. chip->col_bytes = nand->onfi_params.addr_cycles >> 4;
  1578. debug("%s(%p) row bytes: %d, col bytes: %d, ecc mode: %d\n",
  1579. __func__, nand, chip->row_bytes, chip->col_bytes, ecc->mode);
  1580. /*
  1581. * HW_BCH using OcteonTX BCH engine, or SOFT_BCH laid out in
  1582. * HW_BCH-compatible fashion, depending on devtree advice
  1583. * and kernel config.
  1584. * BCH/NFC hardware capable of subpage ops, not implemented.
  1585. */
  1586. mtd_set_ooblayout(mtd, &nand_ooblayout_lp_ops);
  1587. nand->options |= NAND_NO_SUBPAGE_WRITE;
  1588. debug("%s: start steps: %d, size: %d, bytes: %d\n",
  1589. __func__, ecc->steps, ecc->size, ecc->bytes);
  1590. debug("%s: step ds: %d, strength ds: %d\n", __func__,
  1591. nand->ecc_step_ds, nand->ecc_strength_ds);
  1592. if (ecc->mode != NAND_ECC_NONE) {
  1593. int nsteps = ecc->steps ? ecc->steps : 1;
  1594. if (ecc->size && ecc->size != mtd->writesize)
  1595. nsteps = mtd->writesize / ecc->size;
  1596. else if (mtd->writesize > def_ecc_size &&
  1597. !(mtd->writesize & (def_ecc_size - 1)))
  1598. nsteps = mtd->writesize / def_ecc_size;
  1599. ecc->steps = nsteps;
  1600. ecc->size = mtd->writesize / nsteps;
  1601. ecc->bytes = mtd->oobsize / nsteps;
  1602. if (nand->ecc_strength_ds)
  1603. ecc->strength = nand->ecc_strength_ds;
  1604. if (nand->ecc_step_ds)
  1605. ecc->size = nand->ecc_step_ds;
  1606. /*
  1607. * no subpage ops, but set subpage-shift to match ecc->steps
  1608. * so mtd_nandbiterrs tests appropriate boundaries
  1609. */
  1610. if (!mtd->subpage_sft && !(ecc->steps & (ecc->steps - 1)))
  1611. mtd->subpage_sft = fls(ecc->steps) - 1;
  1612. if (IS_ENABLED(CONFIG_NAND_OCTEONTX_HW_ECC)) {
  1613. debug("%s: ecc mode: %d\n", __func__, ecc->mode);
  1614. if (ecc->mode != NAND_ECC_SOFT &&
  1615. !octeontx_nand_calc_bch_ecc_strength(nand)) {
  1616. struct octeontx_nfc *tn =
  1617. to_otx_nfc(nand->controller);
  1618. debug("Using hardware BCH engine support\n");
  1619. ecc->mode = NAND_ECC_HW_SYNDROME;
  1620. ecc->read_page = octeontx_nand_hw_bch_read_page;
  1621. ecc->write_page =
  1622. octeontx_nand_hw_bch_write_page;
  1623. ecc->read_page_raw =
  1624. octeontx_nand_read_page_raw;
  1625. ecc->write_page_raw =
  1626. octeontx_nand_write_page_raw;
  1627. ecc->read_oob = octeontx_nand_read_oob_std;
  1628. ecc->write_oob = octeontx_nand_write_oob_std;
  1629. ecc->calculate = octeontx_nand_bch_calculate;
  1630. ecc->correct = octeontx_nand_bch_correct;
  1631. ecc->hwctl = octeontx_nand_bch_hwctl;
  1632. debug("NAND chip %d using hw_bch\n",
  1633. tn->selected_chip);
  1634. debug(" %d bytes ECC per %d byte block\n",
  1635. ecc->bytes, ecc->size);
  1636. debug(" for %d bits of correction per block.",
  1637. ecc->strength);
  1638. octeontx_nand_calc_ecc_layout(nand);
  1639. octeontx_bch_save_empty_eccmask(nand);
  1640. }
  1641. }
  1642. }
  1643. }
  1644. static int octeontx_nfc_chip_init(struct octeontx_nfc *tn, struct udevice *dev,
  1645. ofnode node)
  1646. {
  1647. struct octeontx_nand_chip *chip;
  1648. struct nand_chip *nand;
  1649. struct mtd_info *mtd;
  1650. int ret;
  1651. chip = devm_kzalloc(dev, sizeof(*chip), GFP_KERNEL);
  1652. if (!chip)
  1653. return -ENOMEM;
  1654. debug("%s: Getting chip select\n", __func__);
  1655. ret = ofnode_read_s32(node, "reg", &chip->cs);
  1656. if (ret) {
  1657. dev_err(dev, "could not retrieve reg property: %d\n", ret);
  1658. return ret;
  1659. }
  1660. if (chip->cs >= NAND_MAX_CHIPS) {
  1661. dev_err(dev, "invalid reg value: %u (max CS = 7)\n", chip->cs);
  1662. return -EINVAL;
  1663. }
  1664. debug("%s: chip select: %d\n", __func__, chip->cs);
  1665. nand = &chip->nand;
  1666. nand->controller = &tn->controller;
  1667. if (!tn->controller.active)
  1668. tn->controller.active = nand;
  1669. debug("%s: Setting flash node\n", __func__);
  1670. nand_set_flash_node(nand, node);
  1671. nand->options = 0;
  1672. nand->select_chip = octeontx_nand_select_chip;
  1673. nand->cmdfunc = octeontx_nand_cmdfunc;
  1674. nand->waitfunc = octeontx_nand_waitfunc;
  1675. nand->read_byte = octeontx_nand_read_byte;
  1676. nand->read_buf = octeontx_nand_read_buf;
  1677. nand->write_buf = octeontx_nand_write_buf;
  1678. nand->onfi_set_features = octeontx_nand_set_features;
  1679. nand->onfi_get_features = octeontx_nand_get_features;
  1680. nand->setup_data_interface = octeontx_nand_setup_dat_intf;
  1681. mtd = nand_to_mtd(nand);
  1682. debug("%s: mtd: %p\n", __func__, mtd);
  1683. mtd->dev->parent = dev;
  1684. debug("%s: NDF_MISC: 0x%llx\n", __func__,
  1685. readq(tn->base + NDF_MISC));
  1686. /* TODO: support more then 1 chip */
  1687. debug("%s: Scanning identification\n", __func__);
  1688. ret = nand_scan_ident(mtd, 1, NULL);
  1689. if (ret)
  1690. return ret;
  1691. debug("%s: Sizing chip\n", __func__);
  1692. octeontx_nfc_chip_sizing(nand);
  1693. debug("%s: Scanning tail\n", __func__);
  1694. ret = nand_scan_tail(mtd);
  1695. if (ret) {
  1696. dev_err(dev, "nand_scan_tail failed: %d\n", ret);
  1697. return ret;
  1698. }
  1699. debug("%s: Registering mtd\n", __func__);
  1700. ret = nand_register(0, mtd);
  1701. debug("%s: Adding tail\n", __func__);
  1702. list_add_tail(&chip->node, &tn->chips);
  1703. return 0;
  1704. }
  1705. static int octeontx_nfc_chips_init(struct octeontx_nfc *tn)
  1706. {
  1707. struct udevice *dev = tn->dev;
  1708. ofnode node = dev_ofnode(dev);
  1709. ofnode nand_node;
  1710. int nr_chips = of_get_child_count(node);
  1711. int ret;
  1712. debug("%s: node: %s\n", __func__, ofnode_get_name(node));
  1713. debug("%s: %d chips\n", __func__, nr_chips);
  1714. if (nr_chips > NAND_MAX_CHIPS) {
  1715. dev_err(dev, "too many NAND chips: %d\n", nr_chips);
  1716. return -EINVAL;
  1717. }
  1718. if (!nr_chips) {
  1719. debug("no DT NAND chips found\n");
  1720. return -ENODEV;
  1721. }
  1722. pr_info("%s: scanning %d chips DTs\n", __func__, nr_chips);
  1723. ofnode_for_each_subnode(nand_node, node) {
  1724. debug("%s: Calling octeontx_nfc_chip_init(%p, %s, %ld)\n",
  1725. __func__, tn, dev->name, nand_node.of_offset);
  1726. ret = octeontx_nfc_chip_init(tn, dev, nand_node);
  1727. if (ret)
  1728. return ret;
  1729. }
  1730. return 0;
  1731. }
  1732. /* Reset NFC and initialize registers. */
  1733. static int octeontx_nfc_init(struct octeontx_nfc *tn)
  1734. {
  1735. const struct nand_sdr_timings *timings;
  1736. u64 ndf_misc;
  1737. int rc;
  1738. /* Initialize values and reset the fifo */
  1739. ndf_misc = readq(tn->base + NDF_MISC);
  1740. ndf_misc &= ~NDF_MISC_EX_DIS;
  1741. ndf_misc |= (NDF_MISC_BT_DIS | NDF_MISC_RST_FF);
  1742. writeq(ndf_misc, tn->base + NDF_MISC);
  1743. debug("%s: NDF_MISC: 0x%llx\n", __func__, readq(tn->base + NDF_MISC));
  1744. /* Bring the fifo out of reset */
  1745. ndf_misc &= ~(NDF_MISC_RST_FF);
  1746. /* Maximum of co-processor cycles for glitch filtering */
  1747. ndf_misc |= FIELD_PREP(NDF_MISC_WAIT_CNT, 0x3f);
  1748. writeq(ndf_misc, tn->base + NDF_MISC);
  1749. /* Set timing parameters to onfi mode 0 for probing */
  1750. timings = onfi_async_timing_mode_to_sdr_timings(0);
  1751. if (IS_ERR(timings))
  1752. return PTR_ERR(timings);
  1753. rc = set_default_timings(tn, timings);
  1754. if (rc)
  1755. return rc;
  1756. return 0;
  1757. }
  1758. static int octeontx_pci_nand_probe(struct udevice *dev)
  1759. {
  1760. struct octeontx_nfc *tn = dev_get_priv(dev);
  1761. int ret;
  1762. static bool probe_done;
  1763. debug("%s(%s) tn: %p\n", __func__, dev->name, tn);
  1764. if (probe_done)
  1765. return 0;
  1766. if (IS_ENABLED(CONFIG_NAND_OCTEONTX_HW_ECC)) {
  1767. bch_vf = octeontx_bch_getv();
  1768. if (!bch_vf) {
  1769. struct octeontx_probe_device *probe_dev;
  1770. debug("%s: bch not yet initialized\n", __func__);
  1771. probe_dev = calloc(sizeof(*probe_dev), 1);
  1772. if (!probe_dev) {
  1773. printf("%s: Out of memory\n", __func__);
  1774. return -ENOMEM;
  1775. }
  1776. probe_dev->dev = dev;
  1777. INIT_LIST_HEAD(&probe_dev->list);
  1778. list_add_tail(&probe_dev->list,
  1779. &octeontx_pci_nand_deferred_devices);
  1780. debug("%s: Defering probe until after BCH initialization\n",
  1781. __func__);
  1782. return 0;
  1783. }
  1784. }
  1785. tn->dev = dev;
  1786. INIT_LIST_HEAD(&tn->chips);
  1787. tn->base = dm_pci_map_bar(dev, PCI_BASE_ADDRESS_0, PCI_REGION_MEM);
  1788. if (!tn->base) {
  1789. ret = -EINVAL;
  1790. goto release;
  1791. }
  1792. debug("%s: bar at %p\n", __func__, tn->base);
  1793. tn->buf.dmabuflen = NAND_MAX_PAGESIZE + NAND_MAX_OOBSIZE;
  1794. tn->buf.dmabuf = dma_alloc_coherent(tn->buf.dmabuflen,
  1795. (unsigned long *)&tn->buf.dmaaddr);
  1796. if (!tn->buf.dmabuf) {
  1797. ret = -ENOMEM;
  1798. debug("%s: Could not allocate DMA buffer\n", __func__);
  1799. goto unclk;
  1800. }
  1801. /* one hw-bch response, for one outstanding transaction */
  1802. tn->bch_resp = dma_alloc_coherent(sizeof(*tn->bch_resp),
  1803. (unsigned long *)&tn->bch_rhandle);
  1804. tn->stat = dma_alloc_coherent(8, (unsigned long *)&tn->stat_addr);
  1805. if (!tn->stat || !tn->bch_resp) {
  1806. debug("%s: Could not allocate bch status or response\n",
  1807. __func__);
  1808. ret = -ENOMEM;
  1809. goto unclk;
  1810. }
  1811. debug("%s: Calling octeontx_nfc_init()\n", __func__);
  1812. octeontx_nfc_init(tn);
  1813. debug("%s: Initializing chips\n", __func__);
  1814. ret = octeontx_nfc_chips_init(tn);
  1815. debug("%s: init chips ret: %d\n", __func__, ret);
  1816. if (ret) {
  1817. if (ret != -ENODEV)
  1818. dev_err(dev, "failed to init nand chips\n");
  1819. goto unclk;
  1820. }
  1821. dev_info(dev, "probed\n");
  1822. return 0;
  1823. unclk:
  1824. release:
  1825. return ret;
  1826. }
  1827. int octeontx_pci_nand_disable(struct udevice *dev)
  1828. {
  1829. struct octeontx_nfc *tn = dev_get_priv(dev);
  1830. u64 dma_cfg;
  1831. u64 ndf_misc;
  1832. debug("%s: Disabling NAND device %s\n", __func__, dev->name);
  1833. dma_cfg = readq(tn->base + NDF_DMA_CFG);
  1834. dma_cfg &= ~NDF_DMA_CFG_EN;
  1835. dma_cfg |= NDF_DMA_CFG_CLR;
  1836. writeq(dma_cfg, tn->base + NDF_DMA_CFG);
  1837. /* Disable execution and put FIFO in reset mode */
  1838. ndf_misc = readq(tn->base + NDF_MISC);
  1839. ndf_misc |= NDF_MISC_EX_DIS | NDF_MISC_RST_FF;
  1840. writeq(ndf_misc, tn->base + NDF_MISC);
  1841. ndf_misc &= ~NDF_MISC_RST_FF;
  1842. writeq(ndf_misc, tn->base + NDF_MISC);
  1843. #ifdef DEBUG
  1844. printf("%s: NDF_MISC: 0x%llx\n", __func__, readq(tn->base + NDF_MISC));
  1845. #endif
  1846. /* Clear any interrupts and enable bits */
  1847. writeq(~0ull, tn->base + NDF_INT_ENA_W1C);
  1848. writeq(~0ull, tn->base + NDF_INT);
  1849. debug("%s: NDF_ST_REG: 0x%llx\n", __func__,
  1850. readq(tn->base + NDF_ST_REG));
  1851. return 0;
  1852. }
  1853. /**
  1854. * Since it's possible (and even likely) that the NAND device will be probed
  1855. * before the BCH device has been probed, we may need to defer the probing.
  1856. *
  1857. * In this case, the initial probe returns success but the actual probing
  1858. * is deferred until the BCH VF has been probed.
  1859. *
  1860. * @return 0 for success, otherwise error
  1861. */
  1862. int octeontx_pci_nand_deferred_probe(void)
  1863. {
  1864. int rc = 0;
  1865. struct octeontx_probe_device *pdev;
  1866. debug("%s: Performing deferred probing\n", __func__);
  1867. list_for_each_entry(pdev, &octeontx_pci_nand_deferred_devices, list) {
  1868. debug("%s: Probing %s\n", __func__, pdev->dev->name);
  1869. dev_get_flags(pdev->dev) &= ~DM_FLAG_ACTIVATED;
  1870. rc = device_probe(pdev->dev);
  1871. if (rc && rc != -ENODEV) {
  1872. printf("%s: Error %d with deferred probe of %s\n",
  1873. __func__, rc, pdev->dev->name);
  1874. break;
  1875. }
  1876. }
  1877. return rc;
  1878. }
  1879. static const struct pci_device_id octeontx_nfc_pci_id_table[] = {
  1880. { PCI_VDEVICE(CAVIUM, 0xA04F) },
  1881. {}
  1882. };
  1883. static int octeontx_nand_of_to_plat(struct udevice *dev)
  1884. {
  1885. return 0;
  1886. }
  1887. static const struct udevice_id octeontx_nand_ids[] = {
  1888. { .compatible = "cavium,cn8130-nand" },
  1889. { },
  1890. };
  1891. U_BOOT_DRIVER(octeontx_pci_nand) = {
  1892. .name = OCTEONTX_NAND_DRIVER_NAME,
  1893. .id = UCLASS_MTD,
  1894. .of_match = of_match_ptr(octeontx_nand_ids),
  1895. .of_to_plat = octeontx_nand_of_to_plat,
  1896. .probe = octeontx_pci_nand_probe,
  1897. .priv_auto = sizeof(struct octeontx_nfc),
  1898. .remove = octeontx_pci_nand_disable,
  1899. .flags = DM_FLAG_OS_PREPARE,
  1900. };
  1901. U_BOOT_PCI_DEVICE(octeontx_pci_nand, octeontx_nfc_pci_id_table);
  1902. void board_nand_init(void)
  1903. {
  1904. struct udevice *dev;
  1905. int ret;
  1906. if (IS_ENABLED(CONFIG_NAND_OCTEONTX_HW_ECC)) {
  1907. ret = uclass_get_device_by_driver(UCLASS_MISC,
  1908. DM_GET_DRIVER(octeontx_pci_bchpf),
  1909. &dev);
  1910. if (ret && ret != -ENODEV) {
  1911. pr_err("Failed to initialize OcteonTX BCH PF controller. (error %d)\n",
  1912. ret);
  1913. }
  1914. ret = uclass_get_device_by_driver(UCLASS_MISC,
  1915. DM_GET_DRIVER(octeontx_pci_bchvf),
  1916. &dev);
  1917. if (ret && ret != -ENODEV) {
  1918. pr_err("Failed to initialize OcteonTX BCH VF controller. (error %d)\n",
  1919. ret);
  1920. }
  1921. }
  1922. ret = uclass_get_device_by_driver(UCLASS_MTD,
  1923. DM_GET_DRIVER(octeontx_pci_nand),
  1924. &dev);
  1925. if (ret && ret != -ENODEV)
  1926. pr_err("Failed to initialize OcteonTX NAND controller. (error %d)\n",
  1927. ret);
  1928. }