nvme.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /*
  3. * Copyright (C) 2017 NXP Semiconductors
  4. * Copyright (C) 2017 Bin Meng <bmeng.cn@gmail.com>
  5. */
  6. #include <common.h>
  7. #include <dm.h>
  8. #include <errno.h>
  9. #include <memalign.h>
  10. #include <pci.h>
  11. #include <dm/device-internal.h>
  12. #include "nvme.h"
  13. #define NVME_Q_DEPTH 2
  14. #define NVME_AQ_DEPTH 2
  15. #define NVME_SQ_SIZE(depth) (depth * sizeof(struct nvme_command))
  16. #define NVME_CQ_SIZE(depth) (depth * sizeof(struct nvme_completion))
  17. #define ADMIN_TIMEOUT 60
  18. #define IO_TIMEOUT 30
  19. #define MAX_PRP_POOL 512
  20. enum nvme_queue_id {
  21. NVME_ADMIN_Q,
  22. NVME_IO_Q,
  23. NVME_Q_NUM,
  24. };
  25. /*
  26. * An NVM Express queue. Each device has at least two (one for admin
  27. * commands and one for I/O commands).
  28. */
  29. struct nvme_queue {
  30. struct nvme_dev *dev;
  31. struct nvme_command *sq_cmds;
  32. struct nvme_completion *cqes;
  33. wait_queue_head_t sq_full;
  34. u32 __iomem *q_db;
  35. u16 q_depth;
  36. s16 cq_vector;
  37. u16 sq_head;
  38. u16 sq_tail;
  39. u16 cq_head;
  40. u16 qid;
  41. u8 cq_phase;
  42. u8 cqe_seen;
  43. unsigned long cmdid_data[];
  44. };
  45. static int nvme_wait_ready(struct nvme_dev *dev, bool enabled)
  46. {
  47. u32 bit = enabled ? NVME_CSTS_RDY : 0;
  48. int timeout;
  49. ulong start;
  50. /* Timeout field in the CAP register is in 500 millisecond units */
  51. timeout = NVME_CAP_TIMEOUT(dev->cap) * 500;
  52. start = get_timer(0);
  53. while (get_timer(start) < timeout) {
  54. if ((readl(&dev->bar->csts) & NVME_CSTS_RDY) == bit)
  55. return 0;
  56. }
  57. return -ETIME;
  58. }
  59. static int nvme_setup_prps(struct nvme_dev *dev, u64 *prp2,
  60. int total_len, u64 dma_addr)
  61. {
  62. u32 page_size = dev->page_size;
  63. int offset = dma_addr & (page_size - 1);
  64. u64 *prp_pool;
  65. int length = total_len;
  66. int i, nprps;
  67. length -= (page_size - offset);
  68. if (length <= 0) {
  69. *prp2 = 0;
  70. return 0;
  71. }
  72. if (length)
  73. dma_addr += (page_size - offset);
  74. if (length <= page_size) {
  75. *prp2 = dma_addr;
  76. return 0;
  77. }
  78. nprps = DIV_ROUND_UP(length, page_size);
  79. if (nprps > dev->prp_entry_num) {
  80. free(dev->prp_pool);
  81. dev->prp_pool = malloc(nprps << 3);
  82. if (!dev->prp_pool) {
  83. printf("Error: malloc prp_pool fail\n");
  84. return -ENOMEM;
  85. }
  86. dev->prp_entry_num = nprps;
  87. }
  88. prp_pool = dev->prp_pool;
  89. i = 0;
  90. while (nprps) {
  91. if (i == ((page_size >> 3) - 1)) {
  92. *(prp_pool + i) = cpu_to_le64((ulong)prp_pool +
  93. page_size);
  94. i = 0;
  95. prp_pool += page_size;
  96. }
  97. *(prp_pool + i++) = cpu_to_le64(dma_addr);
  98. dma_addr += page_size;
  99. nprps--;
  100. }
  101. *prp2 = (ulong)dev->prp_pool;
  102. return 0;
  103. }
  104. static __le16 nvme_get_cmd_id(void)
  105. {
  106. static unsigned short cmdid;
  107. return cpu_to_le16((cmdid < USHRT_MAX) ? cmdid++ : 0);
  108. }
  109. static u16 nvme_read_completion_status(struct nvme_queue *nvmeq, u16 index)
  110. {
  111. u64 start = (ulong)&nvmeq->cqes[index];
  112. u64 stop = start + sizeof(struct nvme_completion);
  113. invalidate_dcache_range(start, stop);
  114. return le16_to_cpu(readw(&(nvmeq->cqes[index].status)));
  115. }
  116. /**
  117. * nvme_submit_cmd() - copy a command into a queue and ring the doorbell
  118. *
  119. * @nvmeq: The queue to use
  120. * @cmd: The command to send
  121. */
  122. static void nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
  123. {
  124. u16 tail = nvmeq->sq_tail;
  125. memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));
  126. flush_dcache_range((ulong)&nvmeq->sq_cmds[tail],
  127. (ulong)&nvmeq->sq_cmds[tail] + sizeof(*cmd));
  128. if (++tail == nvmeq->q_depth)
  129. tail = 0;
  130. writel(tail, nvmeq->q_db);
  131. nvmeq->sq_tail = tail;
  132. }
  133. static int nvme_submit_sync_cmd(struct nvme_queue *nvmeq,
  134. struct nvme_command *cmd,
  135. u32 *result, unsigned timeout)
  136. {
  137. u16 head = nvmeq->cq_head;
  138. u16 phase = nvmeq->cq_phase;
  139. u16 status;
  140. ulong start_time;
  141. ulong timeout_us = timeout * 100000;
  142. cmd->common.command_id = nvme_get_cmd_id();
  143. nvme_submit_cmd(nvmeq, cmd);
  144. start_time = timer_get_us();
  145. for (;;) {
  146. status = nvme_read_completion_status(nvmeq, head);
  147. if ((status & 0x01) == phase)
  148. break;
  149. if (timeout_us > 0 && (timer_get_us() - start_time)
  150. >= timeout_us)
  151. return -ETIMEDOUT;
  152. }
  153. status >>= 1;
  154. if (status) {
  155. printf("ERROR: status = %x, phase = %d, head = %d\n",
  156. status, phase, head);
  157. status = 0;
  158. if (++head == nvmeq->q_depth) {
  159. head = 0;
  160. phase = !phase;
  161. }
  162. writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
  163. nvmeq->cq_head = head;
  164. nvmeq->cq_phase = phase;
  165. return -EIO;
  166. }
  167. if (result)
  168. *result = le32_to_cpu(readl(&(nvmeq->cqes[head].result)));
  169. if (++head == nvmeq->q_depth) {
  170. head = 0;
  171. phase = !phase;
  172. }
  173. writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
  174. nvmeq->cq_head = head;
  175. nvmeq->cq_phase = phase;
  176. return status;
  177. }
  178. static int nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd,
  179. u32 *result)
  180. {
  181. return nvme_submit_sync_cmd(dev->queues[NVME_ADMIN_Q], cmd,
  182. result, ADMIN_TIMEOUT);
  183. }
  184. static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev,
  185. int qid, int depth)
  186. {
  187. struct nvme_queue *nvmeq = malloc(sizeof(*nvmeq));
  188. if (!nvmeq)
  189. return NULL;
  190. memset(nvmeq, 0, sizeof(*nvmeq));
  191. nvmeq->cqes = (void *)memalign(4096, NVME_CQ_SIZE(depth));
  192. if (!nvmeq->cqes)
  193. goto free_nvmeq;
  194. memset((void *)nvmeq->cqes, 0, NVME_CQ_SIZE(depth));
  195. nvmeq->sq_cmds = (void *)memalign(4096, NVME_SQ_SIZE(depth));
  196. if (!nvmeq->sq_cmds)
  197. goto free_queue;
  198. memset((void *)nvmeq->sq_cmds, 0, NVME_SQ_SIZE(depth));
  199. nvmeq->dev = dev;
  200. nvmeq->cq_head = 0;
  201. nvmeq->cq_phase = 1;
  202. nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
  203. nvmeq->q_depth = depth;
  204. nvmeq->qid = qid;
  205. dev->queue_count++;
  206. dev->queues[qid] = nvmeq;
  207. return nvmeq;
  208. free_queue:
  209. free((void *)nvmeq->cqes);
  210. free_nvmeq:
  211. free(nvmeq);
  212. return NULL;
  213. }
  214. static int nvme_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
  215. {
  216. struct nvme_command c;
  217. memset(&c, 0, sizeof(c));
  218. c.delete_queue.opcode = opcode;
  219. c.delete_queue.qid = cpu_to_le16(id);
  220. return nvme_submit_admin_cmd(dev, &c, NULL);
  221. }
  222. static int nvme_delete_sq(struct nvme_dev *dev, u16 sqid)
  223. {
  224. return nvme_delete_queue(dev, nvme_admin_delete_sq, sqid);
  225. }
  226. static int nvme_delete_cq(struct nvme_dev *dev, u16 cqid)
  227. {
  228. return nvme_delete_queue(dev, nvme_admin_delete_cq, cqid);
  229. }
  230. static int nvme_enable_ctrl(struct nvme_dev *dev)
  231. {
  232. dev->ctrl_config &= ~NVME_CC_SHN_MASK;
  233. dev->ctrl_config |= NVME_CC_ENABLE;
  234. writel(cpu_to_le32(dev->ctrl_config), &dev->bar->cc);
  235. return nvme_wait_ready(dev, true);
  236. }
  237. static int nvme_disable_ctrl(struct nvme_dev *dev)
  238. {
  239. dev->ctrl_config &= ~NVME_CC_SHN_MASK;
  240. dev->ctrl_config &= ~NVME_CC_ENABLE;
  241. writel(cpu_to_le32(dev->ctrl_config), &dev->bar->cc);
  242. return nvme_wait_ready(dev, false);
  243. }
  244. static void nvme_free_queue(struct nvme_queue *nvmeq)
  245. {
  246. free((void *)nvmeq->cqes);
  247. free(nvmeq->sq_cmds);
  248. free(nvmeq);
  249. }
  250. static void nvme_free_queues(struct nvme_dev *dev, int lowest)
  251. {
  252. int i;
  253. for (i = dev->queue_count - 1; i >= lowest; i--) {
  254. struct nvme_queue *nvmeq = dev->queues[i];
  255. dev->queue_count--;
  256. dev->queues[i] = NULL;
  257. nvme_free_queue(nvmeq);
  258. }
  259. }
  260. static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
  261. {
  262. struct nvme_dev *dev = nvmeq->dev;
  263. nvmeq->sq_tail = 0;
  264. nvmeq->cq_head = 0;
  265. nvmeq->cq_phase = 1;
  266. nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
  267. memset((void *)nvmeq->cqes, 0, NVME_CQ_SIZE(nvmeq->q_depth));
  268. flush_dcache_range((ulong)nvmeq->cqes,
  269. (ulong)nvmeq->cqes + NVME_CQ_SIZE(nvmeq->q_depth));
  270. dev->online_queues++;
  271. }
  272. static int nvme_configure_admin_queue(struct nvme_dev *dev)
  273. {
  274. int result;
  275. u32 aqa;
  276. u64 cap = dev->cap;
  277. struct nvme_queue *nvmeq;
  278. /* most architectures use 4KB as the page size */
  279. unsigned page_shift = 12;
  280. unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12;
  281. unsigned dev_page_max = NVME_CAP_MPSMAX(cap) + 12;
  282. if (page_shift < dev_page_min) {
  283. debug("Device minimum page size (%u) too large for host (%u)\n",
  284. 1 << dev_page_min, 1 << page_shift);
  285. return -ENODEV;
  286. }
  287. if (page_shift > dev_page_max) {
  288. debug("Device maximum page size (%u) smaller than host (%u)\n",
  289. 1 << dev_page_max, 1 << page_shift);
  290. page_shift = dev_page_max;
  291. }
  292. result = nvme_disable_ctrl(dev);
  293. if (result < 0)
  294. return result;
  295. nvmeq = dev->queues[NVME_ADMIN_Q];
  296. if (!nvmeq) {
  297. nvmeq = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
  298. if (!nvmeq)
  299. return -ENOMEM;
  300. }
  301. aqa = nvmeq->q_depth - 1;
  302. aqa |= aqa << 16;
  303. aqa |= aqa << 16;
  304. dev->page_size = 1 << page_shift;
  305. dev->ctrl_config = NVME_CC_CSS_NVM;
  306. dev->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
  307. dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
  308. dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
  309. writel(aqa, &dev->bar->aqa);
  310. nvme_writeq((ulong)nvmeq->sq_cmds, &dev->bar->asq);
  311. nvme_writeq((ulong)nvmeq->cqes, &dev->bar->acq);
  312. result = nvme_enable_ctrl(dev);
  313. if (result)
  314. goto free_nvmeq;
  315. nvmeq->cq_vector = 0;
  316. nvme_init_queue(dev->queues[NVME_ADMIN_Q], 0);
  317. return result;
  318. free_nvmeq:
  319. nvme_free_queues(dev, 0);
  320. return result;
  321. }
  322. static int nvme_alloc_cq(struct nvme_dev *dev, u16 qid,
  323. struct nvme_queue *nvmeq)
  324. {
  325. struct nvme_command c;
  326. int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;
  327. memset(&c, 0, sizeof(c));
  328. c.create_cq.opcode = nvme_admin_create_cq;
  329. c.create_cq.prp1 = cpu_to_le64((ulong)nvmeq->cqes);
  330. c.create_cq.cqid = cpu_to_le16(qid);
  331. c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
  332. c.create_cq.cq_flags = cpu_to_le16(flags);
  333. c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);
  334. return nvme_submit_admin_cmd(dev, &c, NULL);
  335. }
  336. static int nvme_alloc_sq(struct nvme_dev *dev, u16 qid,
  337. struct nvme_queue *nvmeq)
  338. {
  339. struct nvme_command c;
  340. int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM;
  341. memset(&c, 0, sizeof(c));
  342. c.create_sq.opcode = nvme_admin_create_sq;
  343. c.create_sq.prp1 = cpu_to_le64((ulong)nvmeq->sq_cmds);
  344. c.create_sq.sqid = cpu_to_le16(qid);
  345. c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
  346. c.create_sq.sq_flags = cpu_to_le16(flags);
  347. c.create_sq.cqid = cpu_to_le16(qid);
  348. return nvme_submit_admin_cmd(dev, &c, NULL);
  349. }
  350. int nvme_identify(struct nvme_dev *dev, unsigned nsid,
  351. unsigned cns, dma_addr_t dma_addr)
  352. {
  353. struct nvme_command c;
  354. u32 page_size = dev->page_size;
  355. int offset = dma_addr & (page_size - 1);
  356. int length = sizeof(struct nvme_id_ctrl);
  357. int ret;
  358. memset(&c, 0, sizeof(c));
  359. c.identify.opcode = nvme_admin_identify;
  360. c.identify.nsid = cpu_to_le32(nsid);
  361. c.identify.prp1 = cpu_to_le64(dma_addr);
  362. length -= (page_size - offset);
  363. if (length <= 0) {
  364. c.identify.prp2 = 0;
  365. } else {
  366. dma_addr += (page_size - offset);
  367. c.identify.prp2 = cpu_to_le64(dma_addr);
  368. }
  369. c.identify.cns = cpu_to_le32(cns);
  370. ret = nvme_submit_admin_cmd(dev, &c, NULL);
  371. if (!ret)
  372. invalidate_dcache_range(dma_addr,
  373. dma_addr + sizeof(struct nvme_id_ctrl));
  374. return ret;
  375. }
  376. int nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid,
  377. dma_addr_t dma_addr, u32 *result)
  378. {
  379. struct nvme_command c;
  380. memset(&c, 0, sizeof(c));
  381. c.features.opcode = nvme_admin_get_features;
  382. c.features.nsid = cpu_to_le32(nsid);
  383. c.features.prp1 = cpu_to_le64(dma_addr);
  384. c.features.fid = cpu_to_le32(fid);
  385. /*
  386. * TODO: add cache invalidate operation when the size of
  387. * the DMA buffer is known
  388. */
  389. return nvme_submit_admin_cmd(dev, &c, result);
  390. }
  391. int nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11,
  392. dma_addr_t dma_addr, u32 *result)
  393. {
  394. struct nvme_command c;
  395. memset(&c, 0, sizeof(c));
  396. c.features.opcode = nvme_admin_set_features;
  397. c.features.prp1 = cpu_to_le64(dma_addr);
  398. c.features.fid = cpu_to_le32(fid);
  399. c.features.dword11 = cpu_to_le32(dword11);
  400. /*
  401. * TODO: add cache flush operation when the size of
  402. * the DMA buffer is known
  403. */
  404. return nvme_submit_admin_cmd(dev, &c, result);
  405. }
  406. static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
  407. {
  408. struct nvme_dev *dev = nvmeq->dev;
  409. int result;
  410. nvmeq->cq_vector = qid - 1;
  411. result = nvme_alloc_cq(dev, qid, nvmeq);
  412. if (result < 0)
  413. goto release_cq;
  414. result = nvme_alloc_sq(dev, qid, nvmeq);
  415. if (result < 0)
  416. goto release_sq;
  417. nvme_init_queue(nvmeq, qid);
  418. return result;
  419. release_sq:
  420. nvme_delete_sq(dev, qid);
  421. release_cq:
  422. nvme_delete_cq(dev, qid);
  423. return result;
  424. }
  425. static int nvme_set_queue_count(struct nvme_dev *dev, int count)
  426. {
  427. int status;
  428. u32 result;
  429. u32 q_count = (count - 1) | ((count - 1) << 16);
  430. status = nvme_set_features(dev, NVME_FEAT_NUM_QUEUES,
  431. q_count, 0, &result);
  432. if (status < 0)
  433. return status;
  434. if (status > 1)
  435. return 0;
  436. return min(result & 0xffff, result >> 16) + 1;
  437. }
  438. static void nvme_create_io_queues(struct nvme_dev *dev)
  439. {
  440. unsigned int i;
  441. for (i = dev->queue_count; i <= dev->max_qid; i++)
  442. if (!nvme_alloc_queue(dev, i, dev->q_depth))
  443. break;
  444. for (i = dev->online_queues; i <= dev->queue_count - 1; i++)
  445. if (nvme_create_queue(dev->queues[i], i))
  446. break;
  447. }
  448. static int nvme_setup_io_queues(struct nvme_dev *dev)
  449. {
  450. int nr_io_queues;
  451. int result;
  452. nr_io_queues = 1;
  453. result = nvme_set_queue_count(dev, nr_io_queues);
  454. if (result <= 0)
  455. return result;
  456. dev->max_qid = nr_io_queues;
  457. /* Free previously allocated queues */
  458. nvme_free_queues(dev, nr_io_queues + 1);
  459. nvme_create_io_queues(dev);
  460. return 0;
  461. }
  462. static int nvme_get_info_from_identify(struct nvme_dev *dev)
  463. {
  464. ALLOC_CACHE_ALIGN_BUFFER(char, buf, sizeof(struct nvme_id_ctrl));
  465. struct nvme_id_ctrl *ctrl = (struct nvme_id_ctrl *)buf;
  466. int ret;
  467. int shift = NVME_CAP_MPSMIN(dev->cap) + 12;
  468. ret = nvme_identify(dev, 0, 1, (dma_addr_t)(long)ctrl);
  469. if (ret)
  470. return -EIO;
  471. dev->nn = le32_to_cpu(ctrl->nn);
  472. dev->vwc = ctrl->vwc;
  473. memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn));
  474. memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn));
  475. memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr));
  476. if (ctrl->mdts)
  477. dev->max_transfer_shift = (ctrl->mdts + shift);
  478. else {
  479. /*
  480. * Maximum Data Transfer Size (MDTS) field indicates the maximum
  481. * data transfer size between the host and the controller. The
  482. * host should not submit a command that exceeds this transfer
  483. * size. The value is in units of the minimum memory page size
  484. * and is reported as a power of two (2^n).
  485. *
  486. * The spec also says: a value of 0h indicates no restrictions
  487. * on transfer size. But in nvme_blk_read/write() below we have
  488. * the following algorithm for maximum number of logic blocks
  489. * per transfer:
  490. *
  491. * u16 lbas = 1 << (dev->max_transfer_shift - ns->lba_shift);
  492. *
  493. * In order for lbas not to overflow, the maximum number is 15
  494. * which means dev->max_transfer_shift = 15 + 9 (ns->lba_shift).
  495. * Let's use 20 which provides 1MB size.
  496. */
  497. dev->max_transfer_shift = 20;
  498. }
  499. return 0;
  500. }
  501. int nvme_scan_namespace(void)
  502. {
  503. struct uclass *uc;
  504. struct udevice *dev;
  505. int ret;
  506. ret = uclass_get(UCLASS_NVME, &uc);
  507. if (ret)
  508. return ret;
  509. uclass_foreach_dev(dev, uc) {
  510. ret = device_probe(dev);
  511. if (ret)
  512. return ret;
  513. }
  514. return 0;
  515. }
  516. static int nvme_blk_probe(struct udevice *udev)
  517. {
  518. struct nvme_dev *ndev = dev_get_priv(udev->parent);
  519. struct blk_desc *desc = dev_get_uclass_platdata(udev);
  520. struct nvme_ns *ns = dev_get_priv(udev);
  521. u8 flbas;
  522. ALLOC_CACHE_ALIGN_BUFFER(char, buf, sizeof(struct nvme_id_ns));
  523. struct nvme_id_ns *id = (struct nvme_id_ns *)buf;
  524. struct pci_child_platdata *pplat;
  525. memset(ns, 0, sizeof(*ns));
  526. ns->dev = ndev;
  527. /* extract the namespace id from the block device name */
  528. ns->ns_id = trailing_strtol(udev->name) + 1;
  529. if (nvme_identify(ndev, ns->ns_id, 0, (dma_addr_t)(long)id))
  530. return -EIO;
  531. flbas = id->flbas & NVME_NS_FLBAS_LBA_MASK;
  532. ns->flbas = flbas;
  533. ns->lba_shift = id->lbaf[flbas].ds;
  534. ns->mode_select_num_blocks = le64_to_cpu(id->nsze);
  535. ns->mode_select_block_len = 1 << ns->lba_shift;
  536. list_add(&ns->list, &ndev->namespaces);
  537. desc->lba = ns->mode_select_num_blocks;
  538. desc->log2blksz = ns->lba_shift;
  539. desc->blksz = 1 << ns->lba_shift;
  540. desc->bdev = udev;
  541. pplat = dev_get_parent_platdata(udev->parent);
  542. sprintf(desc->vendor, "0x%.4x", pplat->vendor);
  543. memcpy(desc->product, ndev->serial, sizeof(ndev->serial));
  544. memcpy(desc->revision, ndev->firmware_rev, sizeof(ndev->firmware_rev));
  545. return 0;
  546. }
  547. static ulong nvme_blk_rw(struct udevice *udev, lbaint_t blknr,
  548. lbaint_t blkcnt, void *buffer, bool read)
  549. {
  550. struct nvme_ns *ns = dev_get_priv(udev);
  551. struct nvme_dev *dev = ns->dev;
  552. struct nvme_command c;
  553. struct blk_desc *desc = dev_get_uclass_platdata(udev);
  554. int status;
  555. u64 prp2;
  556. u64 total_len = blkcnt << desc->log2blksz;
  557. u64 temp_len = total_len;
  558. u64 slba = blknr;
  559. u16 lbas = 1 << (dev->max_transfer_shift - ns->lba_shift);
  560. u64 total_lbas = blkcnt;
  561. if (!read)
  562. flush_dcache_range((unsigned long)buffer,
  563. (unsigned long)buffer + total_len);
  564. c.rw.opcode = read ? nvme_cmd_read : nvme_cmd_write;
  565. c.rw.flags = 0;
  566. c.rw.nsid = cpu_to_le32(ns->ns_id);
  567. c.rw.control = 0;
  568. c.rw.dsmgmt = 0;
  569. c.rw.reftag = 0;
  570. c.rw.apptag = 0;
  571. c.rw.appmask = 0;
  572. c.rw.metadata = 0;
  573. while (total_lbas) {
  574. if (total_lbas < lbas) {
  575. lbas = (u16)total_lbas;
  576. total_lbas = 0;
  577. } else {
  578. total_lbas -= lbas;
  579. }
  580. if (nvme_setup_prps(dev, &prp2,
  581. lbas << ns->lba_shift, (ulong)buffer))
  582. return -EIO;
  583. c.rw.slba = cpu_to_le64(slba);
  584. slba += lbas;
  585. c.rw.length = cpu_to_le16(lbas - 1);
  586. c.rw.prp1 = cpu_to_le64((ulong)buffer);
  587. c.rw.prp2 = cpu_to_le64(prp2);
  588. status = nvme_submit_sync_cmd(dev->queues[NVME_IO_Q],
  589. &c, NULL, IO_TIMEOUT);
  590. if (status)
  591. break;
  592. temp_len -= (u32)lbas << ns->lba_shift;
  593. buffer += lbas << ns->lba_shift;
  594. }
  595. if (read)
  596. invalidate_dcache_range((unsigned long)buffer,
  597. (unsigned long)buffer + total_len);
  598. return (total_len - temp_len) >> desc->log2blksz;
  599. }
  600. static ulong nvme_blk_read(struct udevice *udev, lbaint_t blknr,
  601. lbaint_t blkcnt, void *buffer)
  602. {
  603. return nvme_blk_rw(udev, blknr, blkcnt, buffer, true);
  604. }
  605. static ulong nvme_blk_write(struct udevice *udev, lbaint_t blknr,
  606. lbaint_t blkcnt, const void *buffer)
  607. {
  608. return nvme_blk_rw(udev, blknr, blkcnt, (void *)buffer, false);
  609. }
  610. static const struct blk_ops nvme_blk_ops = {
  611. .read = nvme_blk_read,
  612. .write = nvme_blk_write,
  613. };
  614. U_BOOT_DRIVER(nvme_blk) = {
  615. .name = "nvme-blk",
  616. .id = UCLASS_BLK,
  617. .probe = nvme_blk_probe,
  618. .ops = &nvme_blk_ops,
  619. .priv_auto_alloc_size = sizeof(struct nvme_ns),
  620. };
  621. static int nvme_bind(struct udevice *udev)
  622. {
  623. static int ndev_num;
  624. char name[20];
  625. sprintf(name, "nvme#%d", ndev_num++);
  626. return device_set_name(udev, name);
  627. }
  628. static int nvme_probe(struct udevice *udev)
  629. {
  630. int ret;
  631. struct nvme_dev *ndev = dev_get_priv(udev);
  632. ndev->instance = trailing_strtol(udev->name);
  633. INIT_LIST_HEAD(&ndev->namespaces);
  634. ndev->bar = dm_pci_map_bar(udev, PCI_BASE_ADDRESS_0,
  635. PCI_REGION_MEM);
  636. if (readl(&ndev->bar->csts) == -1) {
  637. ret = -ENODEV;
  638. printf("Error: %s: Out of memory!\n", udev->name);
  639. goto free_nvme;
  640. }
  641. ndev->queues = malloc(NVME_Q_NUM * sizeof(struct nvme_queue *));
  642. if (!ndev->queues) {
  643. ret = -ENOMEM;
  644. printf("Error: %s: Out of memory!\n", udev->name);
  645. goto free_nvme;
  646. }
  647. memset(ndev->queues, 0, NVME_Q_NUM * sizeof(struct nvme_queue *));
  648. ndev->prp_pool = malloc(MAX_PRP_POOL);
  649. if (!ndev->prp_pool) {
  650. ret = -ENOMEM;
  651. printf("Error: %s: Out of memory!\n", udev->name);
  652. goto free_nvme;
  653. }
  654. ndev->prp_entry_num = MAX_PRP_POOL >> 3;
  655. ndev->cap = nvme_readq(&ndev->bar->cap);
  656. ndev->q_depth = min_t(int, NVME_CAP_MQES(ndev->cap) + 1, NVME_Q_DEPTH);
  657. ndev->db_stride = 1 << NVME_CAP_STRIDE(ndev->cap);
  658. ndev->dbs = ((void __iomem *)ndev->bar) + 4096;
  659. ret = nvme_configure_admin_queue(ndev);
  660. if (ret)
  661. goto free_queue;
  662. ret = nvme_setup_io_queues(ndev);
  663. if (ret)
  664. goto free_queue;
  665. nvme_get_info_from_identify(ndev);
  666. return 0;
  667. free_queue:
  668. free((void *)ndev->queues);
  669. free_nvme:
  670. return ret;
  671. }
  672. U_BOOT_DRIVER(nvme) = {
  673. .name = "nvme",
  674. .id = UCLASS_NVME,
  675. .bind = nvme_bind,
  676. .probe = nvme_probe,
  677. .priv_auto_alloc_size = sizeof(struct nvme_dev),
  678. };
  679. struct pci_device_id nvme_supported[] = {
  680. { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, ~0) },
  681. {}
  682. };
  683. U_BOOT_PCI_DEVICE(nvme, nvme_supported);