dma.h 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408
  1. /* SPDX-License-Identifier: GPL-2.0-or-later */
  2. /*
  3. * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
  4. */
  5. #ifndef IOATDMA_H
  6. #define IOATDMA_H
  7. #include <linux/dmaengine.h>
  8. #include <linux/init.h>
  9. #include <linux/dmapool.h>
  10. #include <linux/cache.h>
  11. #include <linux/pci_ids.h>
  12. #include <linux/circ_buf.h>
  13. #include <linux/interrupt.h>
  14. #include "registers.h"
  15. #include "hw.h"
  16. #define IOAT_DMA_VERSION "5.00"
  17. #define IOAT_DMA_DCA_ANY_CPU ~0
  18. #define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, dma_dev)
  19. #define to_dev(ioat_chan) (&(ioat_chan)->ioat_dma->pdev->dev)
  20. #define to_pdev(ioat_chan) ((ioat_chan)->ioat_dma->pdev)
  21. #define chan_num(ch) ((int)((ch)->reg_base - (ch)->ioat_dma->reg_base) / 0x80)
  22. /* ioat hardware assumes at least two sources for raid operations */
  23. #define src_cnt_to_sw(x) ((x) + 2)
  24. #define src_cnt_to_hw(x) ((x) - 2)
  25. #define ndest_to_sw(x) ((x) + 1)
  26. #define ndest_to_hw(x) ((x) - 1)
  27. #define src16_cnt_to_sw(x) ((x) + 9)
  28. #define src16_cnt_to_hw(x) ((x) - 9)
  29. /*
  30. * workaround for IOAT ver.3.0 null descriptor issue
  31. * (channel returns error when size is 0)
  32. */
  33. #define NULL_DESC_BUFFER_SIZE 1
  34. enum ioat_irq_mode {
  35. IOAT_NOIRQ = 0,
  36. IOAT_MSIX,
  37. IOAT_MSI,
  38. IOAT_INTX
  39. };
  40. /**
  41. * struct ioatdma_device - internal representation of a IOAT device
  42. * @pdev: PCI-Express device
  43. * @reg_base: MMIO register space base address
  44. * @completion_pool: DMA buffers for completion ops
  45. * @sed_hw_pool: DMA super descriptor pools
  46. * @dma_dev: embedded struct dma_device
  47. * @version: version of ioatdma device
  48. * @msix_entries: irq handlers
  49. * @idx: per channel data
  50. * @dca: direct cache access context
  51. * @irq_mode: interrupt mode (INTX, MSI, MSIX)
  52. * @cap: read DMA capabilities register
  53. */
  54. struct ioatdma_device {
  55. struct pci_dev *pdev;
  56. void __iomem *reg_base;
  57. struct dma_pool *completion_pool;
  58. #define MAX_SED_POOLS 5
  59. struct dma_pool *sed_hw_pool[MAX_SED_POOLS];
  60. struct dma_device dma_dev;
  61. u8 version;
  62. #define IOAT_MAX_CHANS 4
  63. struct msix_entry msix_entries[IOAT_MAX_CHANS];
  64. struct ioatdma_chan *idx[IOAT_MAX_CHANS];
  65. struct dca_provider *dca;
  66. enum ioat_irq_mode irq_mode;
  67. u32 cap;
  68. /* shadow version for CB3.3 chan reset errata workaround */
  69. u64 msixtba0;
  70. u64 msixdata0;
  71. u32 msixpba;
  72. };
  73. #define IOAT_MAX_ORDER 16
  74. #define IOAT_MAX_DESCS (1 << IOAT_MAX_ORDER)
  75. #define IOAT_CHUNK_SIZE (SZ_512K)
  76. #define IOAT_DESCS_PER_CHUNK (IOAT_CHUNK_SIZE / IOAT_DESC_SZ)
  77. struct ioat_descs {
  78. void *virt;
  79. dma_addr_t hw;
  80. };
  81. struct ioatdma_chan {
  82. struct dma_chan dma_chan;
  83. void __iomem *reg_base;
  84. dma_addr_t last_completion;
  85. spinlock_t cleanup_lock;
  86. unsigned long state;
  87. #define IOAT_CHAN_DOWN 0
  88. #define IOAT_COMPLETION_ACK 1
  89. #define IOAT_RESET_PENDING 2
  90. #define IOAT_KOBJ_INIT_FAIL 3
  91. #define IOAT_RUN 5
  92. #define IOAT_CHAN_ACTIVE 6
  93. struct timer_list timer;
  94. #define RESET_DELAY msecs_to_jiffies(100)
  95. struct ioatdma_device *ioat_dma;
  96. dma_addr_t completion_dma;
  97. u64 *completion;
  98. struct tasklet_struct cleanup_task;
  99. struct kobject kobj;
  100. /* ioat v2 / v3 channel attributes
  101. * @xfercap_log; log2 of channel max transfer length (for fast division)
  102. * @head: allocated index
  103. * @issued: hardware notification point
  104. * @tail: cleanup index
  105. * @dmacount: identical to 'head' except for occasionally resetting to zero
  106. * @alloc_order: log2 of the number of allocated descriptors
  107. * @produce: number of descriptors to produce at submit time
  108. * @ring: software ring buffer implementation of hardware ring
  109. * @prep_lock: serializes descriptor preparation (producers)
  110. */
  111. size_t xfercap_log;
  112. u16 head;
  113. u16 issued;
  114. u16 tail;
  115. u16 dmacount;
  116. u16 alloc_order;
  117. u16 produce;
  118. struct ioat_ring_ent **ring;
  119. spinlock_t prep_lock;
  120. struct ioat_descs descs[IOAT_MAX_DESCS / IOAT_DESCS_PER_CHUNK];
  121. int desc_chunks;
  122. int intr_coalesce;
  123. int prev_intr_coalesce;
  124. };
  125. struct ioat_sysfs_entry {
  126. struct attribute attr;
  127. ssize_t (*show)(struct dma_chan *, char *);
  128. ssize_t (*store)(struct dma_chan *, const char *, size_t);
  129. };
  130. /**
  131. * struct ioat_sed_ent - wrapper around super extended hardware descriptor
  132. * @hw: hardware SED
  133. * @dma: dma address for the SED
  134. * @parent: point to the dma descriptor that's the parent
  135. * @hw_pool: descriptor pool index
  136. */
  137. struct ioat_sed_ent {
  138. struct ioat_sed_raw_descriptor *hw;
  139. dma_addr_t dma;
  140. struct ioat_ring_ent *parent;
  141. unsigned int hw_pool;
  142. };
  143. /**
  144. * struct ioat_ring_ent - wrapper around hardware descriptor
  145. * @hw: hardware DMA descriptor (for memcpy)
  146. * @xor: hardware xor descriptor
  147. * @xor_ex: hardware xor extension descriptor
  148. * @pq: hardware pq descriptor
  149. * @pq_ex: hardware pq extension descriptor
  150. * @pqu: hardware pq update descriptor
  151. * @raw: hardware raw (un-typed) descriptor
  152. * @txd: the generic software descriptor for all engines
  153. * @len: total transaction length for unmap
  154. * @result: asynchronous result of validate operations
  155. * @id: identifier for debug
  156. * @sed: pointer to super extended descriptor sw desc
  157. */
  158. struct ioat_ring_ent {
  159. union {
  160. struct ioat_dma_descriptor *hw;
  161. struct ioat_xor_descriptor *xor;
  162. struct ioat_xor_ext_descriptor *xor_ex;
  163. struct ioat_pq_descriptor *pq;
  164. struct ioat_pq_ext_descriptor *pq_ex;
  165. struct ioat_pq_update_descriptor *pqu;
  166. struct ioat_raw_descriptor *raw;
  167. };
  168. size_t len;
  169. struct dma_async_tx_descriptor txd;
  170. enum sum_check_flags *result;
  171. #ifdef DEBUG
  172. int id;
  173. #endif
  174. struct ioat_sed_ent *sed;
  175. };
  176. extern const struct sysfs_ops ioat_sysfs_ops;
  177. extern struct ioat_sysfs_entry ioat_version_attr;
  178. extern struct ioat_sysfs_entry ioat_cap_attr;
  179. extern int ioat_pending_level;
  180. extern int ioat_ring_alloc_order;
  181. extern struct kobj_type ioat_ktype;
  182. extern struct kmem_cache *ioat_cache;
  183. extern int ioat_ring_max_alloc_order;
  184. extern struct kmem_cache *ioat_sed_cache;
  185. static inline struct ioatdma_chan *to_ioat_chan(struct dma_chan *c)
  186. {
  187. return container_of(c, struct ioatdma_chan, dma_chan);
  188. }
  189. /* wrapper around hardware descriptor format + additional software fields */
  190. #ifdef DEBUG
  191. #define set_desc_id(desc, i) ((desc)->id = (i))
  192. #define desc_id(desc) ((desc)->id)
  193. #else
  194. #define set_desc_id(desc, i)
  195. #define desc_id(desc) (0)
  196. #endif
  197. static inline void
  198. __dump_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_dma_descriptor *hw,
  199. struct dma_async_tx_descriptor *tx, int id)
  200. {
  201. struct device *dev = to_dev(ioat_chan);
  202. dev_dbg(dev, "desc[%d]: (%#llx->%#llx) cookie: %d flags: %#x"
  203. " ctl: %#10.8x (op: %#x int_en: %d compl: %d)\n", id,
  204. (unsigned long long) tx->phys,
  205. (unsigned long long) hw->next, tx->cookie, tx->flags,
  206. hw->ctl, hw->ctl_f.op, hw->ctl_f.int_en, hw->ctl_f.compl_write);
  207. }
  208. #define dump_desc_dbg(c, d) \
  209. ({ if (d) __dump_desc_dbg(c, d->hw, &d->txd, desc_id(d)); 0; })
  210. static inline struct ioatdma_chan *
  211. ioat_chan_by_index(struct ioatdma_device *ioat_dma, int index)
  212. {
  213. return ioat_dma->idx[index];
  214. }
  215. static inline u64 ioat_chansts(struct ioatdma_chan *ioat_chan)
  216. {
  217. return readq(ioat_chan->reg_base + IOAT_CHANSTS_OFFSET);
  218. }
  219. static inline u64 ioat_chansts_to_addr(u64 status)
  220. {
  221. return status & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
  222. }
  223. static inline u32 ioat_chanerr(struct ioatdma_chan *ioat_chan)
  224. {
  225. return readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
  226. }
  227. static inline void ioat_suspend(struct ioatdma_chan *ioat_chan)
  228. {
  229. u8 ver = ioat_chan->ioat_dma->version;
  230. writeb(IOAT_CHANCMD_SUSPEND,
  231. ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
  232. }
  233. static inline void ioat_reset(struct ioatdma_chan *ioat_chan)
  234. {
  235. u8 ver = ioat_chan->ioat_dma->version;
  236. writeb(IOAT_CHANCMD_RESET,
  237. ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
  238. }
  239. static inline bool ioat_reset_pending(struct ioatdma_chan *ioat_chan)
  240. {
  241. u8 ver = ioat_chan->ioat_dma->version;
  242. u8 cmd;
  243. cmd = readb(ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
  244. return (cmd & IOAT_CHANCMD_RESET) == IOAT_CHANCMD_RESET;
  245. }
  246. static inline bool is_ioat_active(unsigned long status)
  247. {
  248. return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_ACTIVE);
  249. }
  250. static inline bool is_ioat_idle(unsigned long status)
  251. {
  252. return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_DONE);
  253. }
  254. static inline bool is_ioat_halted(unsigned long status)
  255. {
  256. return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_HALTED);
  257. }
  258. static inline bool is_ioat_suspended(unsigned long status)
  259. {
  260. return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_SUSPENDED);
  261. }
  262. /* channel was fatally programmed */
  263. static inline bool is_ioat_bug(unsigned long err)
  264. {
  265. return !!err;
  266. }
  267. static inline u32 ioat_ring_size(struct ioatdma_chan *ioat_chan)
  268. {
  269. return 1 << ioat_chan->alloc_order;
  270. }
  271. /* count of descriptors in flight with the engine */
  272. static inline u16 ioat_ring_active(struct ioatdma_chan *ioat_chan)
  273. {
  274. return CIRC_CNT(ioat_chan->head, ioat_chan->tail,
  275. ioat_ring_size(ioat_chan));
  276. }
  277. /* count of descriptors pending submission to hardware */
  278. static inline u16 ioat_ring_pending(struct ioatdma_chan *ioat_chan)
  279. {
  280. return CIRC_CNT(ioat_chan->head, ioat_chan->issued,
  281. ioat_ring_size(ioat_chan));
  282. }
  283. static inline u32 ioat_ring_space(struct ioatdma_chan *ioat_chan)
  284. {
  285. return ioat_ring_size(ioat_chan) - ioat_ring_active(ioat_chan);
  286. }
  287. static inline u16
  288. ioat_xferlen_to_descs(struct ioatdma_chan *ioat_chan, size_t len)
  289. {
  290. u16 num_descs = len >> ioat_chan->xfercap_log;
  291. num_descs += !!(len & ((1 << ioat_chan->xfercap_log) - 1));
  292. return num_descs;
  293. }
  294. static inline struct ioat_ring_ent *
  295. ioat_get_ring_ent(struct ioatdma_chan *ioat_chan, u16 idx)
  296. {
  297. return ioat_chan->ring[idx & (ioat_ring_size(ioat_chan) - 1)];
  298. }
  299. static inline void
  300. ioat_set_chainaddr(struct ioatdma_chan *ioat_chan, u64 addr)
  301. {
  302. writel(addr & 0x00000000FFFFFFFF,
  303. ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
  304. writel(addr >> 32,
  305. ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
  306. }
  307. /* IOAT Prep functions */
  308. struct dma_async_tx_descriptor *
  309. ioat_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
  310. dma_addr_t dma_src, size_t len, unsigned long flags);
  311. struct dma_async_tx_descriptor *
  312. ioat_prep_interrupt_lock(struct dma_chan *c, unsigned long flags);
  313. struct dma_async_tx_descriptor *
  314. ioat_prep_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
  315. unsigned int src_cnt, size_t len, unsigned long flags);
  316. struct dma_async_tx_descriptor *
  317. ioat_prep_xor_val(struct dma_chan *chan, dma_addr_t *src,
  318. unsigned int src_cnt, size_t len,
  319. enum sum_check_flags *result, unsigned long flags);
  320. struct dma_async_tx_descriptor *
  321. ioat_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
  322. unsigned int src_cnt, const unsigned char *scf, size_t len,
  323. unsigned long flags);
  324. struct dma_async_tx_descriptor *
  325. ioat_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
  326. unsigned int src_cnt, const unsigned char *scf, size_t len,
  327. enum sum_check_flags *pqres, unsigned long flags);
  328. struct dma_async_tx_descriptor *
  329. ioat_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src,
  330. unsigned int src_cnt, size_t len, unsigned long flags);
  331. struct dma_async_tx_descriptor *
  332. ioat_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src,
  333. unsigned int src_cnt, size_t len,
  334. enum sum_check_flags *result, unsigned long flags);
  335. /* IOAT Operation functions */
  336. irqreturn_t ioat_dma_do_interrupt(int irq, void *data);
  337. irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data);
  338. struct ioat_ring_ent **
  339. ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags);
  340. void ioat_start_null_desc(struct ioatdma_chan *ioat_chan);
  341. void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan);
  342. int ioat_reset_hw(struct ioatdma_chan *ioat_chan);
  343. enum dma_status
  344. ioat_tx_status(struct dma_chan *c, dma_cookie_t cookie,
  345. struct dma_tx_state *txstate);
  346. void ioat_cleanup_event(struct tasklet_struct *t);
  347. void ioat_timer_event(struct timer_list *t);
  348. int ioat_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs);
  349. void ioat_issue_pending(struct dma_chan *chan);
  350. /* IOAT Init functions */
  351. bool is_bwd_ioat(struct pci_dev *pdev);
  352. struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase);
  353. void ioat_kobject_add(struct ioatdma_device *ioat_dma, struct kobj_type *type);
  354. void ioat_kobject_del(struct ioatdma_device *ioat_dma);
  355. int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma);
  356. void ioat_stop(struct ioatdma_chan *ioat_chan);
  357. #endif /* IOATDMA_H */