nvme.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /*
  3. * Copyright (C) 2017 NXP Semiconductors
  4. * Copyright (C) 2017 Bin Meng <bmeng.cn@gmail.com>
  5. */
  6. #include <common.h>
  7. #include <dm.h>
  8. #include <errno.h>
  9. #include <memalign.h>
  10. #include <pci.h>
  11. #include <dm/device-internal.h>
  12. #include "nvme.h"
  13. #define NVME_Q_DEPTH 2
  14. #define NVME_AQ_DEPTH 2
  15. #define NVME_SQ_SIZE(depth) (depth * sizeof(struct nvme_command))
  16. #define NVME_CQ_SIZE(depth) (depth * sizeof(struct nvme_completion))
  17. #define ADMIN_TIMEOUT 60
  18. #define IO_TIMEOUT 30
  19. #define MAX_PRP_POOL 512
  20. enum nvme_queue_id {
  21. NVME_ADMIN_Q,
  22. NVME_IO_Q,
  23. NVME_Q_NUM,
  24. };
  25. /*
  26. * An NVM Express queue. Each device has at least two (one for admin
  27. * commands and one for I/O commands).
  28. */
  29. struct nvme_queue {
  30. struct nvme_dev *dev;
  31. struct nvme_command *sq_cmds;
  32. struct nvme_completion *cqes;
  33. wait_queue_head_t sq_full;
  34. u32 __iomem *q_db;
  35. u16 q_depth;
  36. s16 cq_vector;
  37. u16 sq_head;
  38. u16 sq_tail;
  39. u16 cq_head;
  40. u16 qid;
  41. u8 cq_phase;
  42. u8 cqe_seen;
  43. unsigned long cmdid_data[];
  44. };
  45. static int nvme_wait_ready(struct nvme_dev *dev, bool enabled)
  46. {
  47. u32 bit = enabled ? NVME_CSTS_RDY : 0;
  48. int timeout;
  49. ulong start;
  50. /* Timeout field in the CAP register is in 500 millisecond units */
  51. timeout = NVME_CAP_TIMEOUT(dev->cap) * 500;
  52. start = get_timer(0);
  53. while (get_timer(start) < timeout) {
  54. if ((readl(&dev->bar->csts) & NVME_CSTS_RDY) == bit)
  55. return 0;
  56. }
  57. return -ETIME;
  58. }
  59. static int nvme_setup_prps(struct nvme_dev *dev, u64 *prp2,
  60. int total_len, u64 dma_addr)
  61. {
  62. u32 page_size = dev->page_size;
  63. int offset = dma_addr & (page_size - 1);
  64. u64 *prp_pool;
  65. int length = total_len;
  66. int i, nprps;
  67. u32 prps_per_page = (page_size >> 3) - 1;
  68. u32 num_pages;
  69. length -= (page_size - offset);
  70. if (length <= 0) {
  71. *prp2 = 0;
  72. return 0;
  73. }
  74. if (length)
  75. dma_addr += (page_size - offset);
  76. if (length <= page_size) {
  77. *prp2 = dma_addr;
  78. return 0;
  79. }
  80. nprps = DIV_ROUND_UP(length, page_size);
  81. num_pages = DIV_ROUND_UP(nprps, prps_per_page);
  82. if (nprps > dev->prp_entry_num) {
  83. free(dev->prp_pool);
  84. /*
  85. * Always increase in increments of pages. It doesn't waste
  86. * much memory and reduces the number of allocations.
  87. */
  88. dev->prp_pool = memalign(page_size, num_pages * page_size);
  89. if (!dev->prp_pool) {
  90. printf("Error: malloc prp_pool fail\n");
  91. return -ENOMEM;
  92. }
  93. dev->prp_entry_num = prps_per_page * num_pages;
  94. }
  95. prp_pool = dev->prp_pool;
  96. i = 0;
  97. while (nprps) {
  98. if (i == ((page_size >> 3) - 1)) {
  99. *(prp_pool + i) = cpu_to_le64((ulong)prp_pool +
  100. page_size);
  101. i = 0;
  102. prp_pool += page_size;
  103. }
  104. *(prp_pool + i++) = cpu_to_le64(dma_addr);
  105. dma_addr += page_size;
  106. nprps--;
  107. }
  108. *prp2 = (ulong)dev->prp_pool;
  109. return 0;
  110. }
  111. static __le16 nvme_get_cmd_id(void)
  112. {
  113. static unsigned short cmdid;
  114. return cpu_to_le16((cmdid < USHRT_MAX) ? cmdid++ : 0);
  115. }
  116. static u16 nvme_read_completion_status(struct nvme_queue *nvmeq, u16 index)
  117. {
  118. u64 start = (ulong)&nvmeq->cqes[index];
  119. u64 stop = start + sizeof(struct nvme_completion);
  120. invalidate_dcache_range(start, stop);
  121. return le16_to_cpu(readw(&(nvmeq->cqes[index].status)));
  122. }
  123. /**
  124. * nvme_submit_cmd() - copy a command into a queue and ring the doorbell
  125. *
  126. * @nvmeq: The queue to use
  127. * @cmd: The command to send
  128. */
  129. static void nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
  130. {
  131. u16 tail = nvmeq->sq_tail;
  132. memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));
  133. flush_dcache_range((ulong)&nvmeq->sq_cmds[tail],
  134. (ulong)&nvmeq->sq_cmds[tail] + sizeof(*cmd));
  135. if (++tail == nvmeq->q_depth)
  136. tail = 0;
  137. writel(tail, nvmeq->q_db);
  138. nvmeq->sq_tail = tail;
  139. }
  140. static int nvme_submit_sync_cmd(struct nvme_queue *nvmeq,
  141. struct nvme_command *cmd,
  142. u32 *result, unsigned timeout)
  143. {
  144. u16 head = nvmeq->cq_head;
  145. u16 phase = nvmeq->cq_phase;
  146. u16 status;
  147. ulong start_time;
  148. ulong timeout_us = timeout * 100000;
  149. cmd->common.command_id = nvme_get_cmd_id();
  150. nvme_submit_cmd(nvmeq, cmd);
  151. start_time = timer_get_us();
  152. for (;;) {
  153. status = nvme_read_completion_status(nvmeq, head);
  154. if ((status & 0x01) == phase)
  155. break;
  156. if (timeout_us > 0 && (timer_get_us() - start_time)
  157. >= timeout_us)
  158. return -ETIMEDOUT;
  159. }
  160. status >>= 1;
  161. if (status) {
  162. printf("ERROR: status = %x, phase = %d, head = %d\n",
  163. status, phase, head);
  164. status = 0;
  165. if (++head == nvmeq->q_depth) {
  166. head = 0;
  167. phase = !phase;
  168. }
  169. writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
  170. nvmeq->cq_head = head;
  171. nvmeq->cq_phase = phase;
  172. return -EIO;
  173. }
  174. if (result)
  175. *result = le32_to_cpu(readl(&(nvmeq->cqes[head].result)));
  176. if (++head == nvmeq->q_depth) {
  177. head = 0;
  178. phase = !phase;
  179. }
  180. writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
  181. nvmeq->cq_head = head;
  182. nvmeq->cq_phase = phase;
  183. return status;
  184. }
  185. static int nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd,
  186. u32 *result)
  187. {
  188. return nvme_submit_sync_cmd(dev->queues[NVME_ADMIN_Q], cmd,
  189. result, ADMIN_TIMEOUT);
  190. }
  191. static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev,
  192. int qid, int depth)
  193. {
  194. struct nvme_queue *nvmeq = malloc(sizeof(*nvmeq));
  195. if (!nvmeq)
  196. return NULL;
  197. memset(nvmeq, 0, sizeof(*nvmeq));
  198. nvmeq->cqes = (void *)memalign(4096, NVME_CQ_SIZE(depth));
  199. if (!nvmeq->cqes)
  200. goto free_nvmeq;
  201. memset((void *)nvmeq->cqes, 0, NVME_CQ_SIZE(depth));
  202. nvmeq->sq_cmds = (void *)memalign(4096, NVME_SQ_SIZE(depth));
  203. if (!nvmeq->sq_cmds)
  204. goto free_queue;
  205. memset((void *)nvmeq->sq_cmds, 0, NVME_SQ_SIZE(depth));
  206. nvmeq->dev = dev;
  207. nvmeq->cq_head = 0;
  208. nvmeq->cq_phase = 1;
  209. nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
  210. nvmeq->q_depth = depth;
  211. nvmeq->qid = qid;
  212. dev->queue_count++;
  213. dev->queues[qid] = nvmeq;
  214. return nvmeq;
  215. free_queue:
  216. free((void *)nvmeq->cqes);
  217. free_nvmeq:
  218. free(nvmeq);
  219. return NULL;
  220. }
  221. static int nvme_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
  222. {
  223. struct nvme_command c;
  224. memset(&c, 0, sizeof(c));
  225. c.delete_queue.opcode = opcode;
  226. c.delete_queue.qid = cpu_to_le16(id);
  227. return nvme_submit_admin_cmd(dev, &c, NULL);
  228. }
  229. static int nvme_delete_sq(struct nvme_dev *dev, u16 sqid)
  230. {
  231. return nvme_delete_queue(dev, nvme_admin_delete_sq, sqid);
  232. }
  233. static int nvme_delete_cq(struct nvme_dev *dev, u16 cqid)
  234. {
  235. return nvme_delete_queue(dev, nvme_admin_delete_cq, cqid);
  236. }
  237. static int nvme_enable_ctrl(struct nvme_dev *dev)
  238. {
  239. dev->ctrl_config &= ~NVME_CC_SHN_MASK;
  240. dev->ctrl_config |= NVME_CC_ENABLE;
  241. writel(cpu_to_le32(dev->ctrl_config), &dev->bar->cc);
  242. return nvme_wait_ready(dev, true);
  243. }
  244. static int nvme_disable_ctrl(struct nvme_dev *dev)
  245. {
  246. dev->ctrl_config &= ~NVME_CC_SHN_MASK;
  247. dev->ctrl_config &= ~NVME_CC_ENABLE;
  248. writel(cpu_to_le32(dev->ctrl_config), &dev->bar->cc);
  249. return nvme_wait_ready(dev, false);
  250. }
  251. static void nvme_free_queue(struct nvme_queue *nvmeq)
  252. {
  253. free((void *)nvmeq->cqes);
  254. free(nvmeq->sq_cmds);
  255. free(nvmeq);
  256. }
  257. static void nvme_free_queues(struct nvme_dev *dev, int lowest)
  258. {
  259. int i;
  260. for (i = dev->queue_count - 1; i >= lowest; i--) {
  261. struct nvme_queue *nvmeq = dev->queues[i];
  262. dev->queue_count--;
  263. dev->queues[i] = NULL;
  264. nvme_free_queue(nvmeq);
  265. }
  266. }
  267. static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
  268. {
  269. struct nvme_dev *dev = nvmeq->dev;
  270. nvmeq->sq_tail = 0;
  271. nvmeq->cq_head = 0;
  272. nvmeq->cq_phase = 1;
  273. nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
  274. memset((void *)nvmeq->cqes, 0, NVME_CQ_SIZE(nvmeq->q_depth));
  275. flush_dcache_range((ulong)nvmeq->cqes,
  276. (ulong)nvmeq->cqes + NVME_CQ_SIZE(nvmeq->q_depth));
  277. dev->online_queues++;
  278. }
  279. static int nvme_configure_admin_queue(struct nvme_dev *dev)
  280. {
  281. int result;
  282. u32 aqa;
  283. u64 cap = dev->cap;
  284. struct nvme_queue *nvmeq;
  285. /* most architectures use 4KB as the page size */
  286. unsigned page_shift = 12;
  287. unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12;
  288. unsigned dev_page_max = NVME_CAP_MPSMAX(cap) + 12;
  289. if (page_shift < dev_page_min) {
  290. debug("Device minimum page size (%u) too large for host (%u)\n",
  291. 1 << dev_page_min, 1 << page_shift);
  292. return -ENODEV;
  293. }
  294. if (page_shift > dev_page_max) {
  295. debug("Device maximum page size (%u) smaller than host (%u)\n",
  296. 1 << dev_page_max, 1 << page_shift);
  297. page_shift = dev_page_max;
  298. }
  299. result = nvme_disable_ctrl(dev);
  300. if (result < 0)
  301. return result;
  302. nvmeq = dev->queues[NVME_ADMIN_Q];
  303. if (!nvmeq) {
  304. nvmeq = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
  305. if (!nvmeq)
  306. return -ENOMEM;
  307. }
  308. aqa = nvmeq->q_depth - 1;
  309. aqa |= aqa << 16;
  310. aqa |= aqa << 16;
  311. dev->page_size = 1 << page_shift;
  312. dev->ctrl_config = NVME_CC_CSS_NVM;
  313. dev->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
  314. dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
  315. dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
  316. writel(aqa, &dev->bar->aqa);
  317. nvme_writeq((ulong)nvmeq->sq_cmds, &dev->bar->asq);
  318. nvme_writeq((ulong)nvmeq->cqes, &dev->bar->acq);
  319. result = nvme_enable_ctrl(dev);
  320. if (result)
  321. goto free_nvmeq;
  322. nvmeq->cq_vector = 0;
  323. nvme_init_queue(dev->queues[NVME_ADMIN_Q], 0);
  324. return result;
  325. free_nvmeq:
  326. nvme_free_queues(dev, 0);
  327. return result;
  328. }
  329. static int nvme_alloc_cq(struct nvme_dev *dev, u16 qid,
  330. struct nvme_queue *nvmeq)
  331. {
  332. struct nvme_command c;
  333. int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;
  334. memset(&c, 0, sizeof(c));
  335. c.create_cq.opcode = nvme_admin_create_cq;
  336. c.create_cq.prp1 = cpu_to_le64((ulong)nvmeq->cqes);
  337. c.create_cq.cqid = cpu_to_le16(qid);
  338. c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
  339. c.create_cq.cq_flags = cpu_to_le16(flags);
  340. c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);
  341. return nvme_submit_admin_cmd(dev, &c, NULL);
  342. }
  343. static int nvme_alloc_sq(struct nvme_dev *dev, u16 qid,
  344. struct nvme_queue *nvmeq)
  345. {
  346. struct nvme_command c;
  347. int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM;
  348. memset(&c, 0, sizeof(c));
  349. c.create_sq.opcode = nvme_admin_create_sq;
  350. c.create_sq.prp1 = cpu_to_le64((ulong)nvmeq->sq_cmds);
  351. c.create_sq.sqid = cpu_to_le16(qid);
  352. c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
  353. c.create_sq.sq_flags = cpu_to_le16(flags);
  354. c.create_sq.cqid = cpu_to_le16(qid);
  355. return nvme_submit_admin_cmd(dev, &c, NULL);
  356. }
  357. int nvme_identify(struct nvme_dev *dev, unsigned nsid,
  358. unsigned cns, dma_addr_t dma_addr)
  359. {
  360. struct nvme_command c;
  361. u32 page_size = dev->page_size;
  362. int offset = dma_addr & (page_size - 1);
  363. int length = sizeof(struct nvme_id_ctrl);
  364. int ret;
  365. memset(&c, 0, sizeof(c));
  366. c.identify.opcode = nvme_admin_identify;
  367. c.identify.nsid = cpu_to_le32(nsid);
  368. c.identify.prp1 = cpu_to_le64(dma_addr);
  369. length -= (page_size - offset);
  370. if (length <= 0) {
  371. c.identify.prp2 = 0;
  372. } else {
  373. dma_addr += (page_size - offset);
  374. c.identify.prp2 = cpu_to_le64(dma_addr);
  375. }
  376. c.identify.cns = cpu_to_le32(cns);
  377. ret = nvme_submit_admin_cmd(dev, &c, NULL);
  378. if (!ret)
  379. invalidate_dcache_range(dma_addr,
  380. dma_addr + sizeof(struct nvme_id_ctrl));
  381. return ret;
  382. }
  383. int nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid,
  384. dma_addr_t dma_addr, u32 *result)
  385. {
  386. struct nvme_command c;
  387. memset(&c, 0, sizeof(c));
  388. c.features.opcode = nvme_admin_get_features;
  389. c.features.nsid = cpu_to_le32(nsid);
  390. c.features.prp1 = cpu_to_le64(dma_addr);
  391. c.features.fid = cpu_to_le32(fid);
  392. /*
  393. * TODO: add cache invalidate operation when the size of
  394. * the DMA buffer is known
  395. */
  396. return nvme_submit_admin_cmd(dev, &c, result);
  397. }
  398. int nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11,
  399. dma_addr_t dma_addr, u32 *result)
  400. {
  401. struct nvme_command c;
  402. memset(&c, 0, sizeof(c));
  403. c.features.opcode = nvme_admin_set_features;
  404. c.features.prp1 = cpu_to_le64(dma_addr);
  405. c.features.fid = cpu_to_le32(fid);
  406. c.features.dword11 = cpu_to_le32(dword11);
  407. /*
  408. * TODO: add cache flush operation when the size of
  409. * the DMA buffer is known
  410. */
  411. return nvme_submit_admin_cmd(dev, &c, result);
  412. }
  413. static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
  414. {
  415. struct nvme_dev *dev = nvmeq->dev;
  416. int result;
  417. nvmeq->cq_vector = qid - 1;
  418. result = nvme_alloc_cq(dev, qid, nvmeq);
  419. if (result < 0)
  420. goto release_cq;
  421. result = nvme_alloc_sq(dev, qid, nvmeq);
  422. if (result < 0)
  423. goto release_sq;
  424. nvme_init_queue(nvmeq, qid);
  425. return result;
  426. release_sq:
  427. nvme_delete_sq(dev, qid);
  428. release_cq:
  429. nvme_delete_cq(dev, qid);
  430. return result;
  431. }
  432. static int nvme_set_queue_count(struct nvme_dev *dev, int count)
  433. {
  434. int status;
  435. u32 result;
  436. u32 q_count = (count - 1) | ((count - 1) << 16);
  437. status = nvme_set_features(dev, NVME_FEAT_NUM_QUEUES,
  438. q_count, 0, &result);
  439. if (status < 0)
  440. return status;
  441. if (status > 1)
  442. return 0;
  443. return min(result & 0xffff, result >> 16) + 1;
  444. }
  445. static void nvme_create_io_queues(struct nvme_dev *dev)
  446. {
  447. unsigned int i;
  448. for (i = dev->queue_count; i <= dev->max_qid; i++)
  449. if (!nvme_alloc_queue(dev, i, dev->q_depth))
  450. break;
  451. for (i = dev->online_queues; i <= dev->queue_count - 1; i++)
  452. if (nvme_create_queue(dev->queues[i], i))
  453. break;
  454. }
  455. static int nvme_setup_io_queues(struct nvme_dev *dev)
  456. {
  457. int nr_io_queues;
  458. int result;
  459. nr_io_queues = 1;
  460. result = nvme_set_queue_count(dev, nr_io_queues);
  461. if (result <= 0)
  462. return result;
  463. dev->max_qid = nr_io_queues;
  464. /* Free previously allocated queues */
  465. nvme_free_queues(dev, nr_io_queues + 1);
  466. nvme_create_io_queues(dev);
  467. return 0;
  468. }
  469. static int nvme_get_info_from_identify(struct nvme_dev *dev)
  470. {
  471. ALLOC_CACHE_ALIGN_BUFFER(char, buf, sizeof(struct nvme_id_ctrl));
  472. struct nvme_id_ctrl *ctrl = (struct nvme_id_ctrl *)buf;
  473. int ret;
  474. int shift = NVME_CAP_MPSMIN(dev->cap) + 12;
  475. ret = nvme_identify(dev, 0, 1, (dma_addr_t)(long)ctrl);
  476. if (ret)
  477. return -EIO;
  478. dev->nn = le32_to_cpu(ctrl->nn);
  479. dev->vwc = ctrl->vwc;
  480. memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn));
  481. memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn));
  482. memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr));
  483. if (ctrl->mdts)
  484. dev->max_transfer_shift = (ctrl->mdts + shift);
  485. else {
  486. /*
  487. * Maximum Data Transfer Size (MDTS) field indicates the maximum
  488. * data transfer size between the host and the controller. The
  489. * host should not submit a command that exceeds this transfer
  490. * size. The value is in units of the minimum memory page size
  491. * and is reported as a power of two (2^n).
  492. *
  493. * The spec also says: a value of 0h indicates no restrictions
  494. * on transfer size. But in nvme_blk_read/write() below we have
  495. * the following algorithm for maximum number of logic blocks
  496. * per transfer:
  497. *
  498. * u16 lbas = 1 << (dev->max_transfer_shift - ns->lba_shift);
  499. *
  500. * In order for lbas not to overflow, the maximum number is 15
  501. * which means dev->max_transfer_shift = 15 + 9 (ns->lba_shift).
  502. * Let's use 20 which provides 1MB size.
  503. */
  504. dev->max_transfer_shift = 20;
  505. }
  506. return 0;
  507. }
  508. int nvme_get_namespace_id(struct udevice *udev, u32 *ns_id, u8 *eui64)
  509. {
  510. struct nvme_ns *ns = dev_get_priv(udev);
  511. if (ns_id)
  512. *ns_id = ns->ns_id;
  513. if (eui64)
  514. memcpy(eui64, ns->eui64, sizeof(ns->eui64));
  515. return 0;
  516. }
  517. int nvme_scan_namespace(void)
  518. {
  519. struct uclass *uc;
  520. struct udevice *dev;
  521. int ret;
  522. ret = uclass_get(UCLASS_NVME, &uc);
  523. if (ret)
  524. return ret;
  525. uclass_foreach_dev(dev, uc) {
  526. ret = device_probe(dev);
  527. if (ret)
  528. return ret;
  529. }
  530. return 0;
  531. }
  532. static int nvme_blk_probe(struct udevice *udev)
  533. {
  534. struct nvme_dev *ndev = dev_get_priv(udev->parent);
  535. struct blk_desc *desc = dev_get_uclass_platdata(udev);
  536. struct nvme_ns *ns = dev_get_priv(udev);
  537. u8 flbas;
  538. ALLOC_CACHE_ALIGN_BUFFER(char, buf, sizeof(struct nvme_id_ns));
  539. struct nvme_id_ns *id = (struct nvme_id_ns *)buf;
  540. struct pci_child_platdata *pplat;
  541. memset(ns, 0, sizeof(*ns));
  542. ns->dev = ndev;
  543. /* extract the namespace id from the block device name */
  544. ns->ns_id = trailing_strtol(udev->name) + 1;
  545. if (nvme_identify(ndev, ns->ns_id, 0, (dma_addr_t)(long)id))
  546. return -EIO;
  547. memcpy(&ns->eui64, &id->eui64, sizeof(id->eui64));
  548. flbas = id->flbas & NVME_NS_FLBAS_LBA_MASK;
  549. ns->flbas = flbas;
  550. ns->lba_shift = id->lbaf[flbas].ds;
  551. ns->mode_select_num_blocks = le64_to_cpu(id->nsze);
  552. ns->mode_select_block_len = 1 << ns->lba_shift;
  553. list_add(&ns->list, &ndev->namespaces);
  554. desc->lba = ns->mode_select_num_blocks;
  555. desc->log2blksz = ns->lba_shift;
  556. desc->blksz = 1 << ns->lba_shift;
  557. desc->bdev = udev;
  558. pplat = dev_get_parent_platdata(udev->parent);
  559. sprintf(desc->vendor, "0x%.4x", pplat->vendor);
  560. memcpy(desc->product, ndev->serial, sizeof(ndev->serial));
  561. memcpy(desc->revision, ndev->firmware_rev, sizeof(ndev->firmware_rev));
  562. return 0;
  563. }
  564. static ulong nvme_blk_rw(struct udevice *udev, lbaint_t blknr,
  565. lbaint_t blkcnt, void *buffer, bool read)
  566. {
  567. struct nvme_ns *ns = dev_get_priv(udev);
  568. struct nvme_dev *dev = ns->dev;
  569. struct nvme_command c;
  570. struct blk_desc *desc = dev_get_uclass_platdata(udev);
  571. int status;
  572. u64 prp2;
  573. u64 total_len = blkcnt << desc->log2blksz;
  574. u64 temp_len = total_len;
  575. u64 slba = blknr;
  576. u16 lbas = 1 << (dev->max_transfer_shift - ns->lba_shift);
  577. u64 total_lbas = blkcnt;
  578. if (!read)
  579. flush_dcache_range((unsigned long)buffer,
  580. (unsigned long)buffer + total_len);
  581. c.rw.opcode = read ? nvme_cmd_read : nvme_cmd_write;
  582. c.rw.flags = 0;
  583. c.rw.nsid = cpu_to_le32(ns->ns_id);
  584. c.rw.control = 0;
  585. c.rw.dsmgmt = 0;
  586. c.rw.reftag = 0;
  587. c.rw.apptag = 0;
  588. c.rw.appmask = 0;
  589. c.rw.metadata = 0;
  590. while (total_lbas) {
  591. if (total_lbas < lbas) {
  592. lbas = (u16)total_lbas;
  593. total_lbas = 0;
  594. } else {
  595. total_lbas -= lbas;
  596. }
  597. if (nvme_setup_prps(dev, &prp2,
  598. lbas << ns->lba_shift, (ulong)buffer))
  599. return -EIO;
  600. c.rw.slba = cpu_to_le64(slba);
  601. slba += lbas;
  602. c.rw.length = cpu_to_le16(lbas - 1);
  603. c.rw.prp1 = cpu_to_le64((ulong)buffer);
  604. c.rw.prp2 = cpu_to_le64(prp2);
  605. status = nvme_submit_sync_cmd(dev->queues[NVME_IO_Q],
  606. &c, NULL, IO_TIMEOUT);
  607. if (status)
  608. break;
  609. temp_len -= (u32)lbas << ns->lba_shift;
  610. buffer += lbas << ns->lba_shift;
  611. }
  612. if (read)
  613. invalidate_dcache_range((unsigned long)buffer,
  614. (unsigned long)buffer + total_len);
  615. return (total_len - temp_len) >> desc->log2blksz;
  616. }
  617. static ulong nvme_blk_read(struct udevice *udev, lbaint_t blknr,
  618. lbaint_t blkcnt, void *buffer)
  619. {
  620. return nvme_blk_rw(udev, blknr, blkcnt, buffer, true);
  621. }
  622. static ulong nvme_blk_write(struct udevice *udev, lbaint_t blknr,
  623. lbaint_t blkcnt, const void *buffer)
  624. {
  625. return nvme_blk_rw(udev, blknr, blkcnt, (void *)buffer, false);
  626. }
  627. static const struct blk_ops nvme_blk_ops = {
  628. .read = nvme_blk_read,
  629. .write = nvme_blk_write,
  630. };
  631. U_BOOT_DRIVER(nvme_blk) = {
  632. .name = "nvme-blk",
  633. .id = UCLASS_BLK,
  634. .probe = nvme_blk_probe,
  635. .ops = &nvme_blk_ops,
  636. .priv_auto_alloc_size = sizeof(struct nvme_ns),
  637. };
  638. static int nvme_bind(struct udevice *udev)
  639. {
  640. static int ndev_num;
  641. char name[20];
  642. sprintf(name, "nvme#%d", ndev_num++);
  643. return device_set_name(udev, name);
  644. }
  645. static int nvme_probe(struct udevice *udev)
  646. {
  647. int ret;
  648. struct nvme_dev *ndev = dev_get_priv(udev);
  649. ndev->instance = trailing_strtol(udev->name);
  650. INIT_LIST_HEAD(&ndev->namespaces);
  651. ndev->bar = dm_pci_map_bar(udev, PCI_BASE_ADDRESS_0,
  652. PCI_REGION_MEM);
  653. if (readl(&ndev->bar->csts) == -1) {
  654. ret = -ENODEV;
  655. printf("Error: %s: Out of memory!\n", udev->name);
  656. goto free_nvme;
  657. }
  658. ndev->queues = malloc(NVME_Q_NUM * sizeof(struct nvme_queue *));
  659. if (!ndev->queues) {
  660. ret = -ENOMEM;
  661. printf("Error: %s: Out of memory!\n", udev->name);
  662. goto free_nvme;
  663. }
  664. memset(ndev->queues, 0, NVME_Q_NUM * sizeof(struct nvme_queue *));
  665. ndev->cap = nvme_readq(&ndev->bar->cap);
  666. ndev->q_depth = min_t(int, NVME_CAP_MQES(ndev->cap) + 1, NVME_Q_DEPTH);
  667. ndev->db_stride = 1 << NVME_CAP_STRIDE(ndev->cap);
  668. ndev->dbs = ((void __iomem *)ndev->bar) + 4096;
  669. ret = nvme_configure_admin_queue(ndev);
  670. if (ret)
  671. goto free_queue;
  672. /* Allocate after the page size is known */
  673. ndev->prp_pool = memalign(ndev->page_size, MAX_PRP_POOL);
  674. if (!ndev->prp_pool) {
  675. ret = -ENOMEM;
  676. printf("Error: %s: Out of memory!\n", udev->name);
  677. goto free_nvme;
  678. }
  679. ndev->prp_entry_num = MAX_PRP_POOL >> 3;
  680. ret = nvme_setup_io_queues(ndev);
  681. if (ret)
  682. goto free_queue;
  683. nvme_get_info_from_identify(ndev);
  684. return 0;
  685. free_queue:
  686. free((void *)ndev->queues);
  687. free_nvme:
  688. return ret;
  689. }
  690. U_BOOT_DRIVER(nvme) = {
  691. .name = "nvme",
  692. .id = UCLASS_NVME,
  693. .bind = nvme_bind,
  694. .probe = nvme_probe,
  695. .priv_auto_alloc_size = sizeof(struct nvme_dev),
  696. };
  697. struct pci_device_id nvme_supported[] = {
  698. { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, ~0) },
  699. {}
  700. };
  701. U_BOOT_PCI_DEVICE(nvme, nvme_supported);