ioatdma.c 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841
  1. /*
  2. * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
  3. *
  4. * This program is free software; you can redistribute it and/or modify it
  5. * under the terms of the GNU General Public License as published by the Free
  6. * Software Foundation; either version 2 of the License, or (at your option)
  7. * any later version.
  8. *
  9. * This program is distributed in the hope that it will be useful, but WITHOUT
  10. * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11. * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  12. * more details.
  13. *
  14. * You should have received a copy of the GNU General Public License along with
  15. * this program; if not, write to the Free Software Foundation, Inc., 59
  16. * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17. *
  18. * The full GNU General Public License is included in this distribution in the
  19. * file called COPYING.
  20. */
  21. /*
  22. * This driver supports an Intel I/OAT DMA engine, which does asynchronous
  23. * copy operations.
  24. */
  25. #include <linux/init.h>
  26. #include <linux/module.h>
  27. #include <linux/pci.h>
  28. #include <linux/interrupt.h>
  29. #include <linux/dmaengine.h>
  30. #include <linux/delay.h>
  31. #include <linux/dma-mapping.h>
  32. #include "ioatdma.h"
  33. #include "ioatdma_io.h"
  34. #include "ioatdma_registers.h"
  35. #include "ioatdma_hw.h"
  36. #define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common)
  37. #define to_ioat_device(dev) container_of(dev, struct ioat_device, common)
  38. #define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
  39. /* internal functions */
  40. static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
  41. static void __devexit ioat_remove(struct pci_dev *pdev);
  42. static int enumerate_dma_channels(struct ioat_device *device)
  43. {
  44. u8 xfercap_scale;
  45. u32 xfercap;
  46. int i;
  47. struct ioat_dma_chan *ioat_chan;
  48. device->common.chancnt = ioatdma_read8(device, IOAT_CHANCNT_OFFSET);
  49. xfercap_scale = ioatdma_read8(device, IOAT_XFERCAP_OFFSET);
  50. xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
  51. for (i = 0; i < device->common.chancnt; i++) {
  52. ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL);
  53. if (!ioat_chan) {
  54. device->common.chancnt = i;
  55. break;
  56. }
  57. ioat_chan->device = device;
  58. ioat_chan->reg_base = device->reg_base + (0x80 * (i + 1));
  59. ioat_chan->xfercap = xfercap;
  60. spin_lock_init(&ioat_chan->cleanup_lock);
  61. spin_lock_init(&ioat_chan->desc_lock);
  62. INIT_LIST_HEAD(&ioat_chan->free_desc);
  63. INIT_LIST_HEAD(&ioat_chan->used_desc);
  64. /* This should be made common somewhere in dmaengine.c */
  65. ioat_chan->common.device = &device->common;
  66. ioat_chan->common.client = NULL;
  67. list_add_tail(&ioat_chan->common.device_node,
  68. &device->common.channels);
  69. }
  70. return device->common.chancnt;
  71. }
  72. static struct ioat_desc_sw *ioat_dma_alloc_descriptor(
  73. struct ioat_dma_chan *ioat_chan,
  74. gfp_t flags)
  75. {
  76. struct ioat_dma_descriptor *desc;
  77. struct ioat_desc_sw *desc_sw;
  78. struct ioat_device *ioat_device;
  79. dma_addr_t phys;
  80. ioat_device = to_ioat_device(ioat_chan->common.device);
  81. desc = pci_pool_alloc(ioat_device->dma_pool, flags, &phys);
  82. if (unlikely(!desc))
  83. return NULL;
  84. desc_sw = kzalloc(sizeof(*desc_sw), flags);
  85. if (unlikely(!desc_sw)) {
  86. pci_pool_free(ioat_device->dma_pool, desc, phys);
  87. return NULL;
  88. }
  89. memset(desc, 0, sizeof(*desc));
  90. desc_sw->hw = desc;
  91. desc_sw->phys = phys;
  92. return desc_sw;
  93. }
  94. #define INITIAL_IOAT_DESC_COUNT 128
  95. static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan);
  96. /* returns the actual number of allocated descriptors */
  97. static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)
  98. {
  99. struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
  100. struct ioat_desc_sw *desc = NULL;
  101. u16 chanctrl;
  102. u32 chanerr;
  103. int i;
  104. LIST_HEAD(tmp_list);
  105. /*
  106. * In-use bit automatically set by reading chanctrl
  107. * If 0, we got it, if 1, someone else did
  108. */
  109. chanctrl = ioatdma_chan_read16(ioat_chan, IOAT_CHANCTRL_OFFSET);
  110. if (chanctrl & IOAT_CHANCTRL_CHANNEL_IN_USE)
  111. return -EBUSY;
  112. /* Setup register to interrupt and write completion status on error */
  113. chanctrl = IOAT_CHANCTRL_CHANNEL_IN_USE |
  114. IOAT_CHANCTRL_ERR_INT_EN |
  115. IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
  116. IOAT_CHANCTRL_ERR_COMPLETION_EN;
  117. ioatdma_chan_write16(ioat_chan, IOAT_CHANCTRL_OFFSET, chanctrl);
  118. chanerr = ioatdma_chan_read32(ioat_chan, IOAT_CHANERR_OFFSET);
  119. if (chanerr) {
  120. printk("IOAT: CHANERR = %x, clearing\n", chanerr);
  121. ioatdma_chan_write32(ioat_chan, IOAT_CHANERR_OFFSET, chanerr);
  122. }
  123. /* Allocate descriptors */
  124. for (i = 0; i < INITIAL_IOAT_DESC_COUNT; i++) {
  125. desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);
  126. if (!desc) {
  127. printk(KERN_ERR "IOAT: Only %d initial descriptors\n", i);
  128. break;
  129. }
  130. list_add_tail(&desc->node, &tmp_list);
  131. }
  132. spin_lock_bh(&ioat_chan->desc_lock);
  133. list_splice(&tmp_list, &ioat_chan->free_desc);
  134. spin_unlock_bh(&ioat_chan->desc_lock);
  135. /* allocate a completion writeback area */
  136. /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
  137. ioat_chan->completion_virt =
  138. pci_pool_alloc(ioat_chan->device->completion_pool,
  139. GFP_KERNEL,
  140. &ioat_chan->completion_addr);
  141. memset(ioat_chan->completion_virt, 0,
  142. sizeof(*ioat_chan->completion_virt));
  143. ioatdma_chan_write32(ioat_chan, IOAT_CHANCMP_OFFSET_LOW,
  144. ((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF);
  145. ioatdma_chan_write32(ioat_chan, IOAT_CHANCMP_OFFSET_HIGH,
  146. ((u64) ioat_chan->completion_addr) >> 32);
  147. ioat_start_null_desc(ioat_chan);
  148. return i;
  149. }
  150. static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan);
  151. static void ioat_dma_free_chan_resources(struct dma_chan *chan)
  152. {
  153. struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
  154. struct ioat_device *ioat_device = to_ioat_device(chan->device);
  155. struct ioat_desc_sw *desc, *_desc;
  156. u16 chanctrl;
  157. int in_use_descs = 0;
  158. ioat_dma_memcpy_cleanup(ioat_chan);
  159. ioatdma_chan_write8(ioat_chan, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_RESET);
  160. spin_lock_bh(&ioat_chan->desc_lock);
  161. list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) {
  162. in_use_descs++;
  163. list_del(&desc->node);
  164. pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys);
  165. kfree(desc);
  166. }
  167. list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) {
  168. list_del(&desc->node);
  169. pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys);
  170. kfree(desc);
  171. }
  172. spin_unlock_bh(&ioat_chan->desc_lock);
  173. pci_pool_free(ioat_device->completion_pool,
  174. ioat_chan->completion_virt,
  175. ioat_chan->completion_addr);
  176. /* one is ok since we left it on there on purpose */
  177. if (in_use_descs > 1)
  178. printk(KERN_ERR "IOAT: Freeing %d in use descriptors!\n",
  179. in_use_descs - 1);
  180. ioat_chan->last_completion = ioat_chan->completion_addr = 0;
  181. /* Tell hw the chan is free */
  182. chanctrl = ioatdma_chan_read16(ioat_chan, IOAT_CHANCTRL_OFFSET);
  183. chanctrl &= ~IOAT_CHANCTRL_CHANNEL_IN_USE;
  184. ioatdma_chan_write16(ioat_chan, IOAT_CHANCTRL_OFFSET, chanctrl);
  185. }
  186. /**
  187. * do_ioat_dma_memcpy - actual function that initiates a IOAT DMA transaction
  188. * @ioat_chan: IOAT DMA channel handle
  189. * @dest: DMA destination address
  190. * @src: DMA source address
  191. * @len: transaction length in bytes
  192. */
  193. static dma_cookie_t do_ioat_dma_memcpy(struct ioat_dma_chan *ioat_chan,
  194. dma_addr_t dest,
  195. dma_addr_t src,
  196. size_t len)
  197. {
  198. struct ioat_desc_sw *first;
  199. struct ioat_desc_sw *prev;
  200. struct ioat_desc_sw *new;
  201. dma_cookie_t cookie;
  202. LIST_HEAD(new_chain);
  203. u32 copy;
  204. size_t orig_len;
  205. dma_addr_t orig_src, orig_dst;
  206. unsigned int desc_count = 0;
  207. unsigned int append = 0;
  208. if (!ioat_chan || !dest || !src)
  209. return -EFAULT;
  210. if (!len)
  211. return ioat_chan->common.cookie;
  212. orig_len = len;
  213. orig_src = src;
  214. orig_dst = dest;
  215. first = NULL;
  216. prev = NULL;
  217. spin_lock_bh(&ioat_chan->desc_lock);
  218. while (len) {
  219. if (!list_empty(&ioat_chan->free_desc)) {
  220. new = to_ioat_desc(ioat_chan->free_desc.next);
  221. list_del(&new->node);
  222. } else {
  223. /* try to get another desc */
  224. new = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC);
  225. /* will this ever happen? */
  226. /* TODO add upper limit on these */
  227. BUG_ON(!new);
  228. }
  229. copy = min((u32) len, ioat_chan->xfercap);
  230. new->hw->size = copy;
  231. new->hw->ctl = 0;
  232. new->hw->src_addr = src;
  233. new->hw->dst_addr = dest;
  234. new->cookie = 0;
  235. /* chain together the physical address list for the HW */
  236. if (!first)
  237. first = new;
  238. else
  239. prev->hw->next = (u64) new->phys;
  240. prev = new;
  241. len -= copy;
  242. dest += copy;
  243. src += copy;
  244. list_add_tail(&new->node, &new_chain);
  245. desc_count++;
  246. }
  247. new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
  248. new->hw->next = 0;
  249. /* cookie incr and addition to used_list must be atomic */
  250. cookie = ioat_chan->common.cookie;
  251. cookie++;
  252. if (cookie < 0)
  253. cookie = 1;
  254. ioat_chan->common.cookie = new->cookie = cookie;
  255. pci_unmap_addr_set(new, src, orig_src);
  256. pci_unmap_addr_set(new, dst, orig_dst);
  257. pci_unmap_len_set(new, src_len, orig_len);
  258. pci_unmap_len_set(new, dst_len, orig_len);
  259. /* write address into NextDescriptor field of last desc in chain */
  260. to_ioat_desc(ioat_chan->used_desc.prev)->hw->next = first->phys;
  261. list_splice_init(&new_chain, ioat_chan->used_desc.prev);
  262. ioat_chan->pending += desc_count;
  263. if (ioat_chan->pending >= 20) {
  264. append = 1;
  265. ioat_chan->pending = 0;
  266. }
  267. spin_unlock_bh(&ioat_chan->desc_lock);
  268. if (append)
  269. ioatdma_chan_write8(ioat_chan,
  270. IOAT_CHANCMD_OFFSET,
  271. IOAT_CHANCMD_APPEND);
  272. return cookie;
  273. }
  274. /**
  275. * ioat_dma_memcpy_buf_to_buf - wrapper that takes src & dest bufs
  276. * @chan: IOAT DMA channel handle
  277. * @dest: DMA destination address
  278. * @src: DMA source address
  279. * @len: transaction length in bytes
  280. */
  281. static dma_cookie_t ioat_dma_memcpy_buf_to_buf(struct dma_chan *chan,
  282. void *dest,
  283. void *src,
  284. size_t len)
  285. {
  286. dma_addr_t dest_addr;
  287. dma_addr_t src_addr;
  288. struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
  289. dest_addr = pci_map_single(ioat_chan->device->pdev,
  290. dest, len, PCI_DMA_FROMDEVICE);
  291. src_addr = pci_map_single(ioat_chan->device->pdev,
  292. src, len, PCI_DMA_TODEVICE);
  293. return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
  294. }
  295. /**
  296. * ioat_dma_memcpy_buf_to_pg - wrapper, copying from a buf to a page
  297. * @chan: IOAT DMA channel handle
  298. * @page: pointer to the page to copy to
  299. * @offset: offset into that page
  300. * @src: DMA source address
  301. * @len: transaction length in bytes
  302. */
  303. static dma_cookie_t ioat_dma_memcpy_buf_to_pg(struct dma_chan *chan,
  304. struct page *page,
  305. unsigned int offset,
  306. void *src,
  307. size_t len)
  308. {
  309. dma_addr_t dest_addr;
  310. dma_addr_t src_addr;
  311. struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
  312. dest_addr = pci_map_page(ioat_chan->device->pdev,
  313. page, offset, len, PCI_DMA_FROMDEVICE);
  314. src_addr = pci_map_single(ioat_chan->device->pdev,
  315. src, len, PCI_DMA_TODEVICE);
  316. return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
  317. }
  318. /**
  319. * ioat_dma_memcpy_pg_to_pg - wrapper, copying between two pages
  320. * @chan: IOAT DMA channel handle
  321. * @dest_pg: pointer to the page to copy to
  322. * @dest_off: offset into that page
  323. * @src_pg: pointer to the page to copy from
  324. * @src_off: offset into that page
  325. * @len: transaction length in bytes. This is guaranteed not to make a copy
  326. * across a page boundary.
  327. */
  328. static dma_cookie_t ioat_dma_memcpy_pg_to_pg(struct dma_chan *chan,
  329. struct page *dest_pg,
  330. unsigned int dest_off,
  331. struct page *src_pg,
  332. unsigned int src_off,
  333. size_t len)
  334. {
  335. dma_addr_t dest_addr;
  336. dma_addr_t src_addr;
  337. struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
  338. dest_addr = pci_map_page(ioat_chan->device->pdev,
  339. dest_pg, dest_off, len, PCI_DMA_FROMDEVICE);
  340. src_addr = pci_map_page(ioat_chan->device->pdev,
  341. src_pg, src_off, len, PCI_DMA_TODEVICE);
  342. return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
  343. }
  344. /**
  345. * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended descriptors to hw
  346. * @chan: DMA channel handle
  347. */
  348. static void ioat_dma_memcpy_issue_pending(struct dma_chan *chan)
  349. {
  350. struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
  351. if (ioat_chan->pending != 0) {
  352. ioat_chan->pending = 0;
  353. ioatdma_chan_write8(ioat_chan,
  354. IOAT_CHANCMD_OFFSET,
  355. IOAT_CHANCMD_APPEND);
  356. }
  357. }
  358. static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan)
  359. {
  360. unsigned long phys_complete;
  361. struct ioat_desc_sw *desc, *_desc;
  362. dma_cookie_t cookie = 0;
  363. prefetch(chan->completion_virt);
  364. if (!spin_trylock(&chan->cleanup_lock))
  365. return;
  366. /* The completion writeback can happen at any time,
  367. so reads by the driver need to be atomic operations
  368. The descriptor physical addresses are limited to 32-bits
  369. when the CPU can only do a 32-bit mov */
  370. #if (BITS_PER_LONG == 64)
  371. phys_complete =
  372. chan->completion_virt->full & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
  373. #else
  374. phys_complete = chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK;
  375. #endif
  376. if ((chan->completion_virt->full & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
  377. IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
  378. printk("IOAT: Channel halted, chanerr = %x\n",
  379. ioatdma_chan_read32(chan, IOAT_CHANERR_OFFSET));
  380. /* TODO do something to salvage the situation */
  381. }
  382. if (phys_complete == chan->last_completion) {
  383. spin_unlock(&chan->cleanup_lock);
  384. return;
  385. }
  386. spin_lock_bh(&chan->desc_lock);
  387. list_for_each_entry_safe(desc, _desc, &chan->used_desc, node) {
  388. /*
  389. * Incoming DMA requests may use multiple descriptors, due to
  390. * exceeding xfercap, perhaps. If so, only the last one will
  391. * have a cookie, and require unmapping.
  392. */
  393. if (desc->cookie) {
  394. cookie = desc->cookie;
  395. /* yes we are unmapping both _page and _single alloc'd
  396. regions with unmap_page. Is this *really* that bad?
  397. */
  398. pci_unmap_page(chan->device->pdev,
  399. pci_unmap_addr(desc, dst),
  400. pci_unmap_len(desc, dst_len),
  401. PCI_DMA_FROMDEVICE);
  402. pci_unmap_page(chan->device->pdev,
  403. pci_unmap_addr(desc, src),
  404. pci_unmap_len(desc, src_len),
  405. PCI_DMA_TODEVICE);
  406. }
  407. if (desc->phys != phys_complete) {
  408. /* a completed entry, but not the last, so cleanup */
  409. list_del(&desc->node);
  410. list_add_tail(&desc->node, &chan->free_desc);
  411. } else {
  412. /* last used desc. Do not remove, so we can append from
  413. it, but don't look at it next time, either */
  414. desc->cookie = 0;
  415. /* TODO check status bits? */
  416. break;
  417. }
  418. }
  419. spin_unlock_bh(&chan->desc_lock);
  420. chan->last_completion = phys_complete;
  421. if (cookie != 0)
  422. chan->completed_cookie = cookie;
  423. spin_unlock(&chan->cleanup_lock);
  424. }
  425. /**
  426. * ioat_dma_is_complete - poll the status of a IOAT DMA transaction
  427. * @chan: IOAT DMA channel handle
  428. * @cookie: DMA transaction identifier
  429. * @done: if not %NULL, updated with last completed transaction
  430. * @used: if not %NULL, updated with last used transaction
  431. */
  432. static enum dma_status ioat_dma_is_complete(struct dma_chan *chan,
  433. dma_cookie_t cookie,
  434. dma_cookie_t *done,
  435. dma_cookie_t *used)
  436. {
  437. struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
  438. dma_cookie_t last_used;
  439. dma_cookie_t last_complete;
  440. enum dma_status ret;
  441. last_used = chan->cookie;
  442. last_complete = ioat_chan->completed_cookie;
  443. if (done)
  444. *done= last_complete;
  445. if (used)
  446. *used = last_used;
  447. ret = dma_async_is_complete(cookie, last_complete, last_used);
  448. if (ret == DMA_SUCCESS)
  449. return ret;
  450. ioat_dma_memcpy_cleanup(ioat_chan);
  451. last_used = chan->cookie;
  452. last_complete = ioat_chan->completed_cookie;
  453. if (done)
  454. *done= last_complete;
  455. if (used)
  456. *used = last_used;
  457. return dma_async_is_complete(cookie, last_complete, last_used);
  458. }
  459. /* PCI API */
  460. static struct pci_device_id ioat_pci_tbl[] = {
  461. { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) },
  462. { 0, }
  463. };
  464. static struct pci_driver ioat_pci_drv = {
  465. .name = "ioatdma",
  466. .id_table = ioat_pci_tbl,
  467. .probe = ioat_probe,
  468. .remove = __devexit_p(ioat_remove),
  469. };
  470. static irqreturn_t ioat_do_interrupt(int irq, void *data)
  471. {
  472. struct ioat_device *instance = data;
  473. unsigned long attnstatus;
  474. u8 intrctrl;
  475. intrctrl = ioatdma_read8(instance, IOAT_INTRCTRL_OFFSET);
  476. if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
  477. return IRQ_NONE;
  478. if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
  479. ioatdma_write8(instance, IOAT_INTRCTRL_OFFSET, intrctrl);
  480. return IRQ_NONE;
  481. }
  482. attnstatus = ioatdma_read32(instance, IOAT_ATTNSTATUS_OFFSET);
  483. printk(KERN_ERR "ioatdma error: interrupt! status %lx\n", attnstatus);
  484. ioatdma_write8(instance, IOAT_INTRCTRL_OFFSET, intrctrl);
  485. return IRQ_HANDLED;
  486. }
  487. static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan)
  488. {
  489. struct ioat_desc_sw *desc;
  490. spin_lock_bh(&ioat_chan->desc_lock);
  491. if (!list_empty(&ioat_chan->free_desc)) {
  492. desc = to_ioat_desc(ioat_chan->free_desc.next);
  493. list_del(&desc->node);
  494. } else {
  495. /* try to get another desc */
  496. spin_unlock_bh(&ioat_chan->desc_lock);
  497. desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);
  498. spin_lock_bh(&ioat_chan->desc_lock);
  499. /* will this ever happen? */
  500. BUG_ON(!desc);
  501. }
  502. desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL;
  503. desc->hw->next = 0;
  504. list_add_tail(&desc->node, &ioat_chan->used_desc);
  505. spin_unlock_bh(&ioat_chan->desc_lock);
  506. #if (BITS_PER_LONG == 64)
  507. ioatdma_chan_write64(ioat_chan, IOAT_CHAINADDR_OFFSET, desc->phys);
  508. #else
  509. ioatdma_chan_write32(ioat_chan,
  510. IOAT_CHAINADDR_OFFSET_LOW,
  511. (u32) desc->phys);
  512. ioatdma_chan_write32(ioat_chan, IOAT_CHAINADDR_OFFSET_HIGH, 0);
  513. #endif
  514. ioatdma_chan_write8(ioat_chan, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_START);
  515. }
  516. /*
  517. * Perform a IOAT transaction to verify the HW works.
  518. */
  519. #define IOAT_TEST_SIZE 2000
  520. static int ioat_self_test(struct ioat_device *device)
  521. {
  522. int i;
  523. u8 *src;
  524. u8 *dest;
  525. struct dma_chan *dma_chan;
  526. dma_cookie_t cookie;
  527. int err = 0;
  528. src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
  529. if (!src)
  530. return -ENOMEM;
  531. dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
  532. if (!dest) {
  533. kfree(src);
  534. return -ENOMEM;
  535. }
  536. /* Fill in src buffer */
  537. for (i = 0; i < IOAT_TEST_SIZE; i++)
  538. src[i] = (u8)i;
  539. /* Start copy, using first DMA channel */
  540. dma_chan = container_of(device->common.channels.next,
  541. struct dma_chan,
  542. device_node);
  543. if (ioat_dma_alloc_chan_resources(dma_chan) < 1) {
  544. err = -ENODEV;
  545. goto out;
  546. }
  547. cookie = ioat_dma_memcpy_buf_to_buf(dma_chan, dest, src, IOAT_TEST_SIZE);
  548. ioat_dma_memcpy_issue_pending(dma_chan);
  549. msleep(1);
  550. if (ioat_dma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
  551. printk(KERN_ERR "ioatdma: Self-test copy timed out, disabling\n");
  552. err = -ENODEV;
  553. goto free_resources;
  554. }
  555. if (memcmp(src, dest, IOAT_TEST_SIZE)) {
  556. printk(KERN_ERR "ioatdma: Self-test copy failed compare, disabling\n");
  557. err = -ENODEV;
  558. goto free_resources;
  559. }
  560. free_resources:
  561. ioat_dma_free_chan_resources(dma_chan);
  562. out:
  563. kfree(src);
  564. kfree(dest);
  565. return err;
  566. }
  567. static int __devinit ioat_probe(struct pci_dev *pdev,
  568. const struct pci_device_id *ent)
  569. {
  570. int err;
  571. unsigned long mmio_start, mmio_len;
  572. void __iomem *reg_base;
  573. struct ioat_device *device;
  574. err = pci_enable_device(pdev);
  575. if (err)
  576. goto err_enable_device;
  577. err = pci_set_dma_mask(pdev, DMA_64BIT_MASK);
  578. if (err)
  579. err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
  580. if (err)
  581. goto err_set_dma_mask;
  582. err = pci_request_regions(pdev, ioat_pci_drv.name);
  583. if (err)
  584. goto err_request_regions;
  585. mmio_start = pci_resource_start(pdev, 0);
  586. mmio_len = pci_resource_len(pdev, 0);
  587. reg_base = ioremap(mmio_start, mmio_len);
  588. if (!reg_base) {
  589. err = -ENOMEM;
  590. goto err_ioremap;
  591. }
  592. device = kzalloc(sizeof(*device), GFP_KERNEL);
  593. if (!device) {
  594. err = -ENOMEM;
  595. goto err_kzalloc;
  596. }
  597. /* DMA coherent memory pool for DMA descriptor allocations */
  598. device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
  599. sizeof(struct ioat_dma_descriptor), 64, 0);
  600. if (!device->dma_pool) {
  601. err = -ENOMEM;
  602. goto err_dma_pool;
  603. }
  604. device->completion_pool = pci_pool_create("completion_pool", pdev, sizeof(u64), SMP_CACHE_BYTES, SMP_CACHE_BYTES);
  605. if (!device->completion_pool) {
  606. err = -ENOMEM;
  607. goto err_completion_pool;
  608. }
  609. device->pdev = pdev;
  610. pci_set_drvdata(pdev, device);
  611. #ifdef CONFIG_PCI_MSI
  612. if (pci_enable_msi(pdev) == 0) {
  613. device->msi = 1;
  614. } else {
  615. device->msi = 0;
  616. }
  617. #endif
  618. err = request_irq(pdev->irq, &ioat_do_interrupt, IRQF_SHARED, "ioat",
  619. device);
  620. if (err)
  621. goto err_irq;
  622. device->reg_base = reg_base;
  623. ioatdma_write8(device, IOAT_INTRCTRL_OFFSET, IOAT_INTRCTRL_MASTER_INT_EN);
  624. pci_set_master(pdev);
  625. INIT_LIST_HEAD(&device->common.channels);
  626. enumerate_dma_channels(device);
  627. device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources;
  628. device->common.device_free_chan_resources = ioat_dma_free_chan_resources;
  629. device->common.device_memcpy_buf_to_buf = ioat_dma_memcpy_buf_to_buf;
  630. device->common.device_memcpy_buf_to_pg = ioat_dma_memcpy_buf_to_pg;
  631. device->common.device_memcpy_pg_to_pg = ioat_dma_memcpy_pg_to_pg;
  632. device->common.device_memcpy_complete = ioat_dma_is_complete;
  633. device->common.device_memcpy_issue_pending = ioat_dma_memcpy_issue_pending;
  634. printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n",
  635. device->common.chancnt);
  636. err = ioat_self_test(device);
  637. if (err)
  638. goto err_self_test;
  639. dma_async_device_register(&device->common);
  640. return 0;
  641. err_self_test:
  642. err_irq:
  643. pci_pool_destroy(device->completion_pool);
  644. err_completion_pool:
  645. pci_pool_destroy(device->dma_pool);
  646. err_dma_pool:
  647. kfree(device);
  648. err_kzalloc:
  649. iounmap(reg_base);
  650. err_ioremap:
  651. pci_release_regions(pdev);
  652. err_request_regions:
  653. err_set_dma_mask:
  654. pci_disable_device(pdev);
  655. err_enable_device:
  656. return err;
  657. }
  658. static void __devexit ioat_remove(struct pci_dev *pdev)
  659. {
  660. struct ioat_device *device;
  661. struct dma_chan *chan, *_chan;
  662. struct ioat_dma_chan *ioat_chan;
  663. device = pci_get_drvdata(pdev);
  664. dma_async_device_unregister(&device->common);
  665. free_irq(device->pdev->irq, device);
  666. #ifdef CONFIG_PCI_MSI
  667. if (device->msi)
  668. pci_disable_msi(device->pdev);
  669. #endif
  670. pci_pool_destroy(device->dma_pool);
  671. pci_pool_destroy(device->completion_pool);
  672. iounmap(device->reg_base);
  673. pci_release_regions(pdev);
  674. pci_disable_device(pdev);
  675. list_for_each_entry_safe(chan, _chan, &device->common.channels, device_node) {
  676. ioat_chan = to_ioat_chan(chan);
  677. list_del(&chan->device_node);
  678. kfree(ioat_chan);
  679. }
  680. kfree(device);
  681. }
  682. /* MODULE API */
  683. MODULE_VERSION("1.7");
  684. MODULE_LICENSE("GPL");
  685. MODULE_AUTHOR("Intel Corporation");
  686. static int __init ioat_init_module(void)
  687. {
  688. /* it's currently unsafe to unload this module */
  689. /* if forced, worst case is that rmmod hangs */
  690. __unsafe(THIS_MODULE);
  691. return pci_register_driver(&ioat_pci_drv);
  692. }
  693. module_init(ioat_init_module);
  694. static void __exit ioat_exit_module(void)
  695. {
  696. pci_unregister_driver(&ioat_pci_drv);
  697. }
  698. module_exit(ioat_exit_module);