nvme.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /*
  3. * Copyright (C) 2017 NXP Semiconductors
  4. * Copyright (C) 2017 Bin Meng <bmeng.cn@gmail.com>
  5. */
  6. #include <common.h>
  7. #include <blk.h>
  8. #include <cpu_func.h>
  9. #include <dm.h>
  10. #include <errno.h>
  11. #include <malloc.h>
  12. #include <memalign.h>
  13. #include <pci.h>
  14. #include <time.h>
  15. #include <dm/device-internal.h>
  16. #include <linux/compat.h>
  17. #include "nvme.h"
  18. #define NVME_Q_DEPTH 2
  19. #define NVME_AQ_DEPTH 2
  20. #define NVME_SQ_SIZE(depth) (depth * sizeof(struct nvme_command))
  21. #define NVME_CQ_SIZE(depth) (depth * sizeof(struct nvme_completion))
  22. #define ADMIN_TIMEOUT 60
  23. #define IO_TIMEOUT 30
  24. #define MAX_PRP_POOL 512
  25. enum nvme_queue_id {
  26. NVME_ADMIN_Q,
  27. NVME_IO_Q,
  28. NVME_Q_NUM,
  29. };
  30. /*
  31. * An NVM Express queue. Each device has at least two (one for admin
  32. * commands and one for I/O commands).
  33. */
  34. struct nvme_queue {
  35. struct nvme_dev *dev;
  36. struct nvme_command *sq_cmds;
  37. struct nvme_completion *cqes;
  38. wait_queue_head_t sq_full;
  39. u32 __iomem *q_db;
  40. u16 q_depth;
  41. s16 cq_vector;
  42. u16 sq_head;
  43. u16 sq_tail;
  44. u16 cq_head;
  45. u16 qid;
  46. u8 cq_phase;
  47. u8 cqe_seen;
  48. unsigned long cmdid_data[];
  49. };
  50. static int nvme_wait_ready(struct nvme_dev *dev, bool enabled)
  51. {
  52. u32 bit = enabled ? NVME_CSTS_RDY : 0;
  53. int timeout;
  54. ulong start;
  55. /* Timeout field in the CAP register is in 500 millisecond units */
  56. timeout = NVME_CAP_TIMEOUT(dev->cap) * 500;
  57. start = get_timer(0);
  58. while (get_timer(start) < timeout) {
  59. if ((readl(&dev->bar->csts) & NVME_CSTS_RDY) == bit)
  60. return 0;
  61. }
  62. return -ETIME;
  63. }
  64. static int nvme_setup_prps(struct nvme_dev *dev, u64 *prp2,
  65. int total_len, u64 dma_addr)
  66. {
  67. u32 page_size = dev->page_size;
  68. int offset = dma_addr & (page_size - 1);
  69. u64 *prp_pool;
  70. int length = total_len;
  71. int i, nprps;
  72. u32 prps_per_page = (page_size >> 3) - 1;
  73. u32 num_pages;
  74. length -= (page_size - offset);
  75. if (length <= 0) {
  76. *prp2 = 0;
  77. return 0;
  78. }
  79. if (length)
  80. dma_addr += (page_size - offset);
  81. if (length <= page_size) {
  82. *prp2 = dma_addr;
  83. return 0;
  84. }
  85. nprps = DIV_ROUND_UP(length, page_size);
  86. num_pages = DIV_ROUND_UP(nprps, prps_per_page);
  87. if (nprps > dev->prp_entry_num) {
  88. free(dev->prp_pool);
  89. /*
  90. * Always increase in increments of pages. It doesn't waste
  91. * much memory and reduces the number of allocations.
  92. */
  93. dev->prp_pool = memalign(page_size, num_pages * page_size);
  94. if (!dev->prp_pool) {
  95. printf("Error: malloc prp_pool fail\n");
  96. return -ENOMEM;
  97. }
  98. dev->prp_entry_num = prps_per_page * num_pages;
  99. }
  100. prp_pool = dev->prp_pool;
  101. i = 0;
  102. while (nprps) {
  103. if (i == ((page_size >> 3) - 1)) {
  104. *(prp_pool + i) = cpu_to_le64((ulong)prp_pool +
  105. page_size);
  106. i = 0;
  107. prp_pool += page_size;
  108. }
  109. *(prp_pool + i++) = cpu_to_le64(dma_addr);
  110. dma_addr += page_size;
  111. nprps--;
  112. }
  113. *prp2 = (ulong)dev->prp_pool;
  114. flush_dcache_range((ulong)dev->prp_pool, (ulong)dev->prp_pool +
  115. dev->prp_entry_num * sizeof(u64));
  116. return 0;
  117. }
  118. static __le16 nvme_get_cmd_id(void)
  119. {
  120. static unsigned short cmdid;
  121. return cpu_to_le16((cmdid < USHRT_MAX) ? cmdid++ : 0);
  122. }
  123. static u16 nvme_read_completion_status(struct nvme_queue *nvmeq, u16 index)
  124. {
  125. u64 start = (ulong)&nvmeq->cqes[index];
  126. u64 stop = start + sizeof(struct nvme_completion);
  127. invalidate_dcache_range(start, stop);
  128. return le16_to_cpu(readw(&(nvmeq->cqes[index].status)));
  129. }
  130. /**
  131. * nvme_submit_cmd() - copy a command into a queue and ring the doorbell
  132. *
  133. * @nvmeq: The queue to use
  134. * @cmd: The command to send
  135. */
  136. static void nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
  137. {
  138. u16 tail = nvmeq->sq_tail;
  139. memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));
  140. flush_dcache_range((ulong)&nvmeq->sq_cmds[tail],
  141. (ulong)&nvmeq->sq_cmds[tail] + sizeof(*cmd));
  142. if (++tail == nvmeq->q_depth)
  143. tail = 0;
  144. writel(tail, nvmeq->q_db);
  145. nvmeq->sq_tail = tail;
  146. }
  147. static int nvme_submit_sync_cmd(struct nvme_queue *nvmeq,
  148. struct nvme_command *cmd,
  149. u32 *result, unsigned timeout)
  150. {
  151. u16 head = nvmeq->cq_head;
  152. u16 phase = nvmeq->cq_phase;
  153. u16 status;
  154. ulong start_time;
  155. ulong timeout_us = timeout * 100000;
  156. cmd->common.command_id = nvme_get_cmd_id();
  157. nvme_submit_cmd(nvmeq, cmd);
  158. start_time = timer_get_us();
  159. for (;;) {
  160. status = nvme_read_completion_status(nvmeq, head);
  161. if ((status & 0x01) == phase)
  162. break;
  163. if (timeout_us > 0 && (timer_get_us() - start_time)
  164. >= timeout_us)
  165. return -ETIMEDOUT;
  166. }
  167. status >>= 1;
  168. if (status) {
  169. printf("ERROR: status = %x, phase = %d, head = %d\n",
  170. status, phase, head);
  171. status = 0;
  172. if (++head == nvmeq->q_depth) {
  173. head = 0;
  174. phase = !phase;
  175. }
  176. writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
  177. nvmeq->cq_head = head;
  178. nvmeq->cq_phase = phase;
  179. return -EIO;
  180. }
  181. if (result)
  182. *result = le32_to_cpu(readl(&(nvmeq->cqes[head].result)));
  183. if (++head == nvmeq->q_depth) {
  184. head = 0;
  185. phase = !phase;
  186. }
  187. writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
  188. nvmeq->cq_head = head;
  189. nvmeq->cq_phase = phase;
  190. return status;
  191. }
  192. static int nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd,
  193. u32 *result)
  194. {
  195. return nvme_submit_sync_cmd(dev->queues[NVME_ADMIN_Q], cmd,
  196. result, ADMIN_TIMEOUT);
  197. }
  198. static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev,
  199. int qid, int depth)
  200. {
  201. struct nvme_queue *nvmeq = malloc(sizeof(*nvmeq));
  202. if (!nvmeq)
  203. return NULL;
  204. memset(nvmeq, 0, sizeof(*nvmeq));
  205. nvmeq->cqes = (void *)memalign(4096, NVME_CQ_SIZE(depth));
  206. if (!nvmeq->cqes)
  207. goto free_nvmeq;
  208. memset((void *)nvmeq->cqes, 0, NVME_CQ_SIZE(depth));
  209. nvmeq->sq_cmds = (void *)memalign(4096, NVME_SQ_SIZE(depth));
  210. if (!nvmeq->sq_cmds)
  211. goto free_queue;
  212. memset((void *)nvmeq->sq_cmds, 0, NVME_SQ_SIZE(depth));
  213. nvmeq->dev = dev;
  214. nvmeq->cq_head = 0;
  215. nvmeq->cq_phase = 1;
  216. nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
  217. nvmeq->q_depth = depth;
  218. nvmeq->qid = qid;
  219. dev->queue_count++;
  220. dev->queues[qid] = nvmeq;
  221. return nvmeq;
  222. free_queue:
  223. free((void *)nvmeq->cqes);
  224. free_nvmeq:
  225. free(nvmeq);
  226. return NULL;
  227. }
  228. static int nvme_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
  229. {
  230. struct nvme_command c;
  231. memset(&c, 0, sizeof(c));
  232. c.delete_queue.opcode = opcode;
  233. c.delete_queue.qid = cpu_to_le16(id);
  234. return nvme_submit_admin_cmd(dev, &c, NULL);
  235. }
  236. static int nvme_delete_sq(struct nvme_dev *dev, u16 sqid)
  237. {
  238. return nvme_delete_queue(dev, nvme_admin_delete_sq, sqid);
  239. }
  240. static int nvme_delete_cq(struct nvme_dev *dev, u16 cqid)
  241. {
  242. return nvme_delete_queue(dev, nvme_admin_delete_cq, cqid);
  243. }
  244. static int nvme_enable_ctrl(struct nvme_dev *dev)
  245. {
  246. dev->ctrl_config &= ~NVME_CC_SHN_MASK;
  247. dev->ctrl_config |= NVME_CC_ENABLE;
  248. writel(cpu_to_le32(dev->ctrl_config), &dev->bar->cc);
  249. return nvme_wait_ready(dev, true);
  250. }
  251. static int nvme_disable_ctrl(struct nvme_dev *dev)
  252. {
  253. dev->ctrl_config &= ~NVME_CC_SHN_MASK;
  254. dev->ctrl_config &= ~NVME_CC_ENABLE;
  255. writel(cpu_to_le32(dev->ctrl_config), &dev->bar->cc);
  256. return nvme_wait_ready(dev, false);
  257. }
  258. static void nvme_free_queue(struct nvme_queue *nvmeq)
  259. {
  260. free((void *)nvmeq->cqes);
  261. free(nvmeq->sq_cmds);
  262. free(nvmeq);
  263. }
  264. static void nvme_free_queues(struct nvme_dev *dev, int lowest)
  265. {
  266. int i;
  267. for (i = dev->queue_count - 1; i >= lowest; i--) {
  268. struct nvme_queue *nvmeq = dev->queues[i];
  269. dev->queue_count--;
  270. dev->queues[i] = NULL;
  271. nvme_free_queue(nvmeq);
  272. }
  273. }
  274. static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
  275. {
  276. struct nvme_dev *dev = nvmeq->dev;
  277. nvmeq->sq_tail = 0;
  278. nvmeq->cq_head = 0;
  279. nvmeq->cq_phase = 1;
  280. nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
  281. memset((void *)nvmeq->cqes, 0, NVME_CQ_SIZE(nvmeq->q_depth));
  282. flush_dcache_range((ulong)nvmeq->cqes,
  283. (ulong)nvmeq->cqes + NVME_CQ_SIZE(nvmeq->q_depth));
  284. dev->online_queues++;
  285. }
  286. static int nvme_configure_admin_queue(struct nvme_dev *dev)
  287. {
  288. int result;
  289. u32 aqa;
  290. u64 cap = dev->cap;
  291. struct nvme_queue *nvmeq;
  292. /* most architectures use 4KB as the page size */
  293. unsigned page_shift = 12;
  294. unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12;
  295. unsigned dev_page_max = NVME_CAP_MPSMAX(cap) + 12;
  296. if (page_shift < dev_page_min) {
  297. debug("Device minimum page size (%u) too large for host (%u)\n",
  298. 1 << dev_page_min, 1 << page_shift);
  299. return -ENODEV;
  300. }
  301. if (page_shift > dev_page_max) {
  302. debug("Device maximum page size (%u) smaller than host (%u)\n",
  303. 1 << dev_page_max, 1 << page_shift);
  304. page_shift = dev_page_max;
  305. }
  306. result = nvme_disable_ctrl(dev);
  307. if (result < 0)
  308. return result;
  309. nvmeq = dev->queues[NVME_ADMIN_Q];
  310. if (!nvmeq) {
  311. nvmeq = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
  312. if (!nvmeq)
  313. return -ENOMEM;
  314. }
  315. aqa = nvmeq->q_depth - 1;
  316. aqa |= aqa << 16;
  317. aqa |= aqa << 16;
  318. dev->page_size = 1 << page_shift;
  319. dev->ctrl_config = NVME_CC_CSS_NVM;
  320. dev->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
  321. dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
  322. dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
  323. writel(aqa, &dev->bar->aqa);
  324. nvme_writeq((ulong)nvmeq->sq_cmds, &dev->bar->asq);
  325. nvme_writeq((ulong)nvmeq->cqes, &dev->bar->acq);
  326. result = nvme_enable_ctrl(dev);
  327. if (result)
  328. goto free_nvmeq;
  329. nvmeq->cq_vector = 0;
  330. nvme_init_queue(dev->queues[NVME_ADMIN_Q], 0);
  331. return result;
  332. free_nvmeq:
  333. nvme_free_queues(dev, 0);
  334. return result;
  335. }
  336. static int nvme_alloc_cq(struct nvme_dev *dev, u16 qid,
  337. struct nvme_queue *nvmeq)
  338. {
  339. struct nvme_command c;
  340. int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;
  341. memset(&c, 0, sizeof(c));
  342. c.create_cq.opcode = nvme_admin_create_cq;
  343. c.create_cq.prp1 = cpu_to_le64((ulong)nvmeq->cqes);
  344. c.create_cq.cqid = cpu_to_le16(qid);
  345. c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
  346. c.create_cq.cq_flags = cpu_to_le16(flags);
  347. c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);
  348. return nvme_submit_admin_cmd(dev, &c, NULL);
  349. }
  350. static int nvme_alloc_sq(struct nvme_dev *dev, u16 qid,
  351. struct nvme_queue *nvmeq)
  352. {
  353. struct nvme_command c;
  354. int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM;
  355. memset(&c, 0, sizeof(c));
  356. c.create_sq.opcode = nvme_admin_create_sq;
  357. c.create_sq.prp1 = cpu_to_le64((ulong)nvmeq->sq_cmds);
  358. c.create_sq.sqid = cpu_to_le16(qid);
  359. c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
  360. c.create_sq.sq_flags = cpu_to_le16(flags);
  361. c.create_sq.cqid = cpu_to_le16(qid);
  362. return nvme_submit_admin_cmd(dev, &c, NULL);
  363. }
  364. int nvme_identify(struct nvme_dev *dev, unsigned nsid,
  365. unsigned cns, dma_addr_t dma_addr)
  366. {
  367. struct nvme_command c;
  368. u32 page_size = dev->page_size;
  369. int offset = dma_addr & (page_size - 1);
  370. int length = sizeof(struct nvme_id_ctrl);
  371. int ret;
  372. memset(&c, 0, sizeof(c));
  373. c.identify.opcode = nvme_admin_identify;
  374. c.identify.nsid = cpu_to_le32(nsid);
  375. c.identify.prp1 = cpu_to_le64(dma_addr);
  376. length -= (page_size - offset);
  377. if (length <= 0) {
  378. c.identify.prp2 = 0;
  379. } else {
  380. dma_addr += (page_size - offset);
  381. c.identify.prp2 = cpu_to_le64(dma_addr);
  382. }
  383. c.identify.cns = cpu_to_le32(cns);
  384. ret = nvme_submit_admin_cmd(dev, &c, NULL);
  385. if (!ret)
  386. invalidate_dcache_range(dma_addr,
  387. dma_addr + sizeof(struct nvme_id_ctrl));
  388. return ret;
  389. }
  390. int nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid,
  391. dma_addr_t dma_addr, u32 *result)
  392. {
  393. struct nvme_command c;
  394. memset(&c, 0, sizeof(c));
  395. c.features.opcode = nvme_admin_get_features;
  396. c.features.nsid = cpu_to_le32(nsid);
  397. c.features.prp1 = cpu_to_le64(dma_addr);
  398. c.features.fid = cpu_to_le32(fid);
  399. /*
  400. * TODO: add cache invalidate operation when the size of
  401. * the DMA buffer is known
  402. */
  403. return nvme_submit_admin_cmd(dev, &c, result);
  404. }
  405. int nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11,
  406. dma_addr_t dma_addr, u32 *result)
  407. {
  408. struct nvme_command c;
  409. memset(&c, 0, sizeof(c));
  410. c.features.opcode = nvme_admin_set_features;
  411. c.features.prp1 = cpu_to_le64(dma_addr);
  412. c.features.fid = cpu_to_le32(fid);
  413. c.features.dword11 = cpu_to_le32(dword11);
  414. /*
  415. * TODO: add cache flush operation when the size of
  416. * the DMA buffer is known
  417. */
  418. return nvme_submit_admin_cmd(dev, &c, result);
  419. }
  420. static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
  421. {
  422. struct nvme_dev *dev = nvmeq->dev;
  423. int result;
  424. nvmeq->cq_vector = qid - 1;
  425. result = nvme_alloc_cq(dev, qid, nvmeq);
  426. if (result < 0)
  427. goto release_cq;
  428. result = nvme_alloc_sq(dev, qid, nvmeq);
  429. if (result < 0)
  430. goto release_sq;
  431. nvme_init_queue(nvmeq, qid);
  432. return result;
  433. release_sq:
  434. nvme_delete_sq(dev, qid);
  435. release_cq:
  436. nvme_delete_cq(dev, qid);
  437. return result;
  438. }
  439. static int nvme_set_queue_count(struct nvme_dev *dev, int count)
  440. {
  441. int status;
  442. u32 result;
  443. u32 q_count = (count - 1) | ((count - 1) << 16);
  444. status = nvme_set_features(dev, NVME_FEAT_NUM_QUEUES,
  445. q_count, 0, &result);
  446. if (status < 0)
  447. return status;
  448. if (status > 1)
  449. return 0;
  450. return min(result & 0xffff, result >> 16) + 1;
  451. }
  452. static void nvme_create_io_queues(struct nvme_dev *dev)
  453. {
  454. unsigned int i;
  455. for (i = dev->queue_count; i <= dev->max_qid; i++)
  456. if (!nvme_alloc_queue(dev, i, dev->q_depth))
  457. break;
  458. for (i = dev->online_queues; i <= dev->queue_count - 1; i++)
  459. if (nvme_create_queue(dev->queues[i], i))
  460. break;
  461. }
  462. static int nvme_setup_io_queues(struct nvme_dev *dev)
  463. {
  464. int nr_io_queues;
  465. int result;
  466. nr_io_queues = 1;
  467. result = nvme_set_queue_count(dev, nr_io_queues);
  468. if (result <= 0)
  469. return result;
  470. dev->max_qid = nr_io_queues;
  471. /* Free previously allocated queues */
  472. nvme_free_queues(dev, nr_io_queues + 1);
  473. nvme_create_io_queues(dev);
  474. return 0;
  475. }
  476. static int nvme_get_info_from_identify(struct nvme_dev *dev)
  477. {
  478. struct nvme_id_ctrl *ctrl;
  479. int ret;
  480. int shift = NVME_CAP_MPSMIN(dev->cap) + 12;
  481. ctrl = memalign(dev->page_size, sizeof(struct nvme_id_ctrl));
  482. if (!ctrl)
  483. return -ENOMEM;
  484. ret = nvme_identify(dev, 0, 1, (dma_addr_t)(long)ctrl);
  485. if (ret) {
  486. free(ctrl);
  487. return -EIO;
  488. }
  489. dev->nn = le32_to_cpu(ctrl->nn);
  490. dev->vwc = ctrl->vwc;
  491. memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn));
  492. memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn));
  493. memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr));
  494. if (ctrl->mdts)
  495. dev->max_transfer_shift = (ctrl->mdts + shift);
  496. else {
  497. /*
  498. * Maximum Data Transfer Size (MDTS) field indicates the maximum
  499. * data transfer size between the host and the controller. The
  500. * host should not submit a command that exceeds this transfer
  501. * size. The value is in units of the minimum memory page size
  502. * and is reported as a power of two (2^n).
  503. *
  504. * The spec also says: a value of 0h indicates no restrictions
  505. * on transfer size. But in nvme_blk_read/write() below we have
  506. * the following algorithm for maximum number of logic blocks
  507. * per transfer:
  508. *
  509. * u16 lbas = 1 << (dev->max_transfer_shift - ns->lba_shift);
  510. *
  511. * In order for lbas not to overflow, the maximum number is 15
  512. * which means dev->max_transfer_shift = 15 + 9 (ns->lba_shift).
  513. * Let's use 20 which provides 1MB size.
  514. */
  515. dev->max_transfer_shift = 20;
  516. }
  517. free(ctrl);
  518. return 0;
  519. }
  520. int nvme_get_namespace_id(struct udevice *udev, u32 *ns_id, u8 *eui64)
  521. {
  522. struct nvme_ns *ns = dev_get_priv(udev);
  523. if (ns_id)
  524. *ns_id = ns->ns_id;
  525. if (eui64)
  526. memcpy(eui64, ns->eui64, sizeof(ns->eui64));
  527. return 0;
  528. }
  529. int nvme_scan_namespace(void)
  530. {
  531. struct uclass *uc;
  532. struct udevice *dev;
  533. int ret;
  534. ret = uclass_get(UCLASS_NVME, &uc);
  535. if (ret)
  536. return ret;
  537. uclass_foreach_dev(dev, uc) {
  538. ret = device_probe(dev);
  539. if (ret)
  540. return ret;
  541. }
  542. return 0;
  543. }
  544. static int nvme_blk_probe(struct udevice *udev)
  545. {
  546. struct nvme_dev *ndev = dev_get_priv(udev->parent);
  547. struct blk_desc *desc = dev_get_uclass_platdata(udev);
  548. struct nvme_ns *ns = dev_get_priv(udev);
  549. u8 flbas;
  550. struct pci_child_platdata *pplat;
  551. struct nvme_id_ns *id;
  552. id = memalign(ndev->page_size, sizeof(struct nvme_id_ns));
  553. if (!id)
  554. return -ENOMEM;
  555. memset(ns, 0, sizeof(*ns));
  556. ns->dev = ndev;
  557. /* extract the namespace id from the block device name */
  558. ns->ns_id = trailing_strtol(udev->name) + 1;
  559. if (nvme_identify(ndev, ns->ns_id, 0, (dma_addr_t)(long)id)) {
  560. free(id);
  561. return -EIO;
  562. }
  563. memcpy(&ns->eui64, &id->eui64, sizeof(id->eui64));
  564. flbas = id->flbas & NVME_NS_FLBAS_LBA_MASK;
  565. ns->flbas = flbas;
  566. ns->lba_shift = id->lbaf[flbas].ds;
  567. ns->mode_select_num_blocks = le64_to_cpu(id->nsze);
  568. ns->mode_select_block_len = 1 << ns->lba_shift;
  569. list_add(&ns->list, &ndev->namespaces);
  570. desc->lba = ns->mode_select_num_blocks;
  571. desc->log2blksz = ns->lba_shift;
  572. desc->blksz = 1 << ns->lba_shift;
  573. desc->bdev = udev;
  574. pplat = dev_get_parent_platdata(udev->parent);
  575. sprintf(desc->vendor, "0x%.4x", pplat->vendor);
  576. memcpy(desc->product, ndev->serial, sizeof(ndev->serial));
  577. memcpy(desc->revision, ndev->firmware_rev, sizeof(ndev->firmware_rev));
  578. free(id);
  579. return 0;
  580. }
  581. static ulong nvme_blk_rw(struct udevice *udev, lbaint_t blknr,
  582. lbaint_t blkcnt, void *buffer, bool read)
  583. {
  584. struct nvme_ns *ns = dev_get_priv(udev);
  585. struct nvme_dev *dev = ns->dev;
  586. struct nvme_command c;
  587. struct blk_desc *desc = dev_get_uclass_platdata(udev);
  588. int status;
  589. u64 prp2;
  590. u64 total_len = blkcnt << desc->log2blksz;
  591. u64 temp_len = total_len;
  592. u64 slba = blknr;
  593. u16 lbas = 1 << (dev->max_transfer_shift - ns->lba_shift);
  594. u64 total_lbas = blkcnt;
  595. flush_dcache_range((unsigned long)buffer,
  596. (unsigned long)buffer + total_len);
  597. c.rw.opcode = read ? nvme_cmd_read : nvme_cmd_write;
  598. c.rw.flags = 0;
  599. c.rw.nsid = cpu_to_le32(ns->ns_id);
  600. c.rw.control = 0;
  601. c.rw.dsmgmt = 0;
  602. c.rw.reftag = 0;
  603. c.rw.apptag = 0;
  604. c.rw.appmask = 0;
  605. c.rw.metadata = 0;
  606. while (total_lbas) {
  607. if (total_lbas < lbas) {
  608. lbas = (u16)total_lbas;
  609. total_lbas = 0;
  610. } else {
  611. total_lbas -= lbas;
  612. }
  613. if (nvme_setup_prps(dev, &prp2,
  614. lbas << ns->lba_shift, (ulong)buffer))
  615. return -EIO;
  616. c.rw.slba = cpu_to_le64(slba);
  617. slba += lbas;
  618. c.rw.length = cpu_to_le16(lbas - 1);
  619. c.rw.prp1 = cpu_to_le64((ulong)buffer);
  620. c.rw.prp2 = cpu_to_le64(prp2);
  621. status = nvme_submit_sync_cmd(dev->queues[NVME_IO_Q],
  622. &c, NULL, IO_TIMEOUT);
  623. if (status)
  624. break;
  625. temp_len -= (u32)lbas << ns->lba_shift;
  626. buffer += lbas << ns->lba_shift;
  627. }
  628. if (read)
  629. invalidate_dcache_range((unsigned long)buffer,
  630. (unsigned long)buffer + total_len);
  631. return (total_len - temp_len) >> desc->log2blksz;
  632. }
  633. static ulong nvme_blk_read(struct udevice *udev, lbaint_t blknr,
  634. lbaint_t blkcnt, void *buffer)
  635. {
  636. return nvme_blk_rw(udev, blknr, blkcnt, buffer, true);
  637. }
  638. static ulong nvme_blk_write(struct udevice *udev, lbaint_t blknr,
  639. lbaint_t blkcnt, const void *buffer)
  640. {
  641. return nvme_blk_rw(udev, blknr, blkcnt, (void *)buffer, false);
  642. }
  643. static const struct blk_ops nvme_blk_ops = {
  644. .read = nvme_blk_read,
  645. .write = nvme_blk_write,
  646. };
  647. U_BOOT_DRIVER(nvme_blk) = {
  648. .name = "nvme-blk",
  649. .id = UCLASS_BLK,
  650. .probe = nvme_blk_probe,
  651. .ops = &nvme_blk_ops,
  652. .priv_auto_alloc_size = sizeof(struct nvme_ns),
  653. };
  654. static int nvme_bind(struct udevice *udev)
  655. {
  656. static int ndev_num;
  657. char name[20];
  658. sprintf(name, "nvme#%d", ndev_num++);
  659. return device_set_name(udev, name);
  660. }
  661. static int nvme_probe(struct udevice *udev)
  662. {
  663. int ret;
  664. struct nvme_dev *ndev = dev_get_priv(udev);
  665. ndev->instance = trailing_strtol(udev->name);
  666. INIT_LIST_HEAD(&ndev->namespaces);
  667. ndev->bar = dm_pci_map_bar(udev, PCI_BASE_ADDRESS_0,
  668. PCI_REGION_MEM);
  669. if (readl(&ndev->bar->csts) == -1) {
  670. ret = -ENODEV;
  671. printf("Error: %s: Out of memory!\n", udev->name);
  672. goto free_nvme;
  673. }
  674. ndev->queues = malloc(NVME_Q_NUM * sizeof(struct nvme_queue *));
  675. if (!ndev->queues) {
  676. ret = -ENOMEM;
  677. printf("Error: %s: Out of memory!\n", udev->name);
  678. goto free_nvme;
  679. }
  680. memset(ndev->queues, 0, NVME_Q_NUM * sizeof(struct nvme_queue *));
  681. ndev->cap = nvme_readq(&ndev->bar->cap);
  682. ndev->q_depth = min_t(int, NVME_CAP_MQES(ndev->cap) + 1, NVME_Q_DEPTH);
  683. ndev->db_stride = 1 << NVME_CAP_STRIDE(ndev->cap);
  684. ndev->dbs = ((void __iomem *)ndev->bar) + 4096;
  685. ret = nvme_configure_admin_queue(ndev);
  686. if (ret)
  687. goto free_queue;
  688. /* Allocate after the page size is known */
  689. ndev->prp_pool = memalign(ndev->page_size, MAX_PRP_POOL);
  690. if (!ndev->prp_pool) {
  691. ret = -ENOMEM;
  692. printf("Error: %s: Out of memory!\n", udev->name);
  693. goto free_nvme;
  694. }
  695. ndev->prp_entry_num = MAX_PRP_POOL >> 3;
  696. ret = nvme_setup_io_queues(ndev);
  697. if (ret)
  698. goto free_queue;
  699. nvme_get_info_from_identify(ndev);
  700. return 0;
  701. free_queue:
  702. free((void *)ndev->queues);
  703. free_nvme:
  704. return ret;
  705. }
  706. U_BOOT_DRIVER(nvme) = {
  707. .name = "nvme",
  708. .id = UCLASS_NVME,
  709. .bind = nvme_bind,
  710. .probe = nvme_probe,
  711. .priv_auto_alloc_size = sizeof(struct nvme_dev),
  712. };
  713. struct pci_device_id nvme_supported[] = {
  714. { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, ~0) },
  715. {}
  716. };
  717. U_BOOT_PCI_DEVICE(nvme, nvme_supported);