amd_gart_64.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Dynamic DMA mapping support for AMD Hammer.
  4. *
  5. * Use the integrated AGP GART in the Hammer northbridge as an IOMMU for PCI.
  6. * This allows to use PCI devices that only support 32bit addresses on systems
  7. * with more than 4GB.
  8. *
  9. * See Documentation/core-api/dma-api-howto.rst for the interface specification.
  10. *
  11. * Copyright 2002 Andi Kleen, SuSE Labs.
  12. */
  13. #include <linux/types.h>
  14. #include <linux/ctype.h>
  15. #include <linux/agp_backend.h>
  16. #include <linux/init.h>
  17. #include <linux/mm.h>
  18. #include <linux/sched.h>
  19. #include <linux/sched/debug.h>
  20. #include <linux/string.h>
  21. #include <linux/spinlock.h>
  22. #include <linux/pci.h>
  23. #include <linux/topology.h>
  24. #include <linux/interrupt.h>
  25. #include <linux/bitmap.h>
  26. #include <linux/kdebug.h>
  27. #include <linux/scatterlist.h>
  28. #include <linux/iommu-helper.h>
  29. #include <linux/syscore_ops.h>
  30. #include <linux/io.h>
  31. #include <linux/gfp.h>
  32. #include <linux/atomic.h>
  33. #include <linux/dma-direct.h>
  34. #include <linux/dma-map-ops.h>
  35. #include <asm/mtrr.h>
  36. #include <asm/proto.h>
  37. #include <asm/iommu.h>
  38. #include <asm/gart.h>
  39. #include <asm/set_memory.h>
  40. #include <asm/swiotlb.h>
  41. #include <asm/dma.h>
  42. #include <asm/amd_nb.h>
  43. #include <asm/x86_init.h>
  44. #include <asm/iommu_table.h>
  45. static unsigned long iommu_bus_base; /* GART remapping area (physical) */
  46. static unsigned long iommu_size; /* size of remapping area bytes */
  47. static unsigned long iommu_pages; /* .. and in pages */
  48. static u32 *iommu_gatt_base; /* Remapping table */
  49. /*
  50. * If this is disabled the IOMMU will use an optimized flushing strategy
  51. * of only flushing when an mapping is reused. With it true the GART is
  52. * flushed for every mapping. Problem is that doing the lazy flush seems
  53. * to trigger bugs with some popular PCI cards, in particular 3ware (but
  54. * has been also also seen with Qlogic at least).
  55. */
  56. static int iommu_fullflush = 1;
  57. /* Allocation bitmap for the remapping area: */
  58. static DEFINE_SPINLOCK(iommu_bitmap_lock);
  59. /* Guarded by iommu_bitmap_lock: */
  60. static unsigned long *iommu_gart_bitmap;
  61. static u32 gart_unmapped_entry;
  62. #define GPTE_VALID 1
  63. #define GPTE_COHERENT 2
  64. #define GPTE_ENCODE(x) \
  65. (((x) & 0xfffff000) | (((x) >> 32) << 4) | GPTE_VALID | GPTE_COHERENT)
  66. #define GPTE_DECODE(x) (((x) & 0xfffff000) | (((u64)(x) & 0xff0) << 28))
  67. #ifdef CONFIG_AGP
  68. #define AGPEXTERN extern
  69. #else
  70. #define AGPEXTERN
  71. #endif
  72. /* GART can only remap to physical addresses < 1TB */
  73. #define GART_MAX_PHYS_ADDR (1ULL << 40)
  74. /* backdoor interface to AGP driver */
  75. AGPEXTERN int agp_memory_reserved;
  76. AGPEXTERN __u32 *agp_gatt_table;
  77. static unsigned long next_bit; /* protected by iommu_bitmap_lock */
  78. static bool need_flush; /* global flush state. set for each gart wrap */
  79. static unsigned long alloc_iommu(struct device *dev, int size,
  80. unsigned long align_mask)
  81. {
  82. unsigned long offset, flags;
  83. unsigned long boundary_size;
  84. unsigned long base_index;
  85. base_index = ALIGN(iommu_bus_base & dma_get_seg_boundary(dev),
  86. PAGE_SIZE) >> PAGE_SHIFT;
  87. boundary_size = dma_get_seg_boundary_nr_pages(dev, PAGE_SHIFT);
  88. spin_lock_irqsave(&iommu_bitmap_lock, flags);
  89. offset = iommu_area_alloc(iommu_gart_bitmap, iommu_pages, next_bit,
  90. size, base_index, boundary_size, align_mask);
  91. if (offset == -1) {
  92. need_flush = true;
  93. offset = iommu_area_alloc(iommu_gart_bitmap, iommu_pages, 0,
  94. size, base_index, boundary_size,
  95. align_mask);
  96. }
  97. if (offset != -1) {
  98. next_bit = offset+size;
  99. if (next_bit >= iommu_pages) {
  100. next_bit = 0;
  101. need_flush = true;
  102. }
  103. }
  104. if (iommu_fullflush)
  105. need_flush = true;
  106. spin_unlock_irqrestore(&iommu_bitmap_lock, flags);
  107. return offset;
  108. }
  109. static void free_iommu(unsigned long offset, int size)
  110. {
  111. unsigned long flags;
  112. spin_lock_irqsave(&iommu_bitmap_lock, flags);
  113. bitmap_clear(iommu_gart_bitmap, offset, size);
  114. if (offset >= next_bit)
  115. next_bit = offset + size;
  116. spin_unlock_irqrestore(&iommu_bitmap_lock, flags);
  117. }
  118. /*
  119. * Use global flush state to avoid races with multiple flushers.
  120. */
  121. static void flush_gart(void)
  122. {
  123. unsigned long flags;
  124. spin_lock_irqsave(&iommu_bitmap_lock, flags);
  125. if (need_flush) {
  126. amd_flush_garts();
  127. need_flush = false;
  128. }
  129. spin_unlock_irqrestore(&iommu_bitmap_lock, flags);
  130. }
  131. #ifdef CONFIG_IOMMU_LEAK
  132. /* Debugging aid for drivers that don't free their IOMMU tables */
  133. static void dump_leak(void)
  134. {
  135. static int dump;
  136. if (dump)
  137. return;
  138. dump = 1;
  139. show_stack(NULL, NULL, KERN_ERR);
  140. debug_dma_dump_mappings(NULL);
  141. }
  142. #endif
  143. static void iommu_full(struct device *dev, size_t size, int dir)
  144. {
  145. /*
  146. * Ran out of IOMMU space for this operation. This is very bad.
  147. * Unfortunately the drivers cannot handle this operation properly.
  148. * Return some non mapped prereserved space in the aperture and
  149. * let the Northbridge deal with it. This will result in garbage
  150. * in the IO operation. When the size exceeds the prereserved space
  151. * memory corruption will occur or random memory will be DMAed
  152. * out. Hopefully no network devices use single mappings that big.
  153. */
  154. dev_err(dev, "PCI-DMA: Out of IOMMU space for %lu bytes\n", size);
  155. #ifdef CONFIG_IOMMU_LEAK
  156. dump_leak();
  157. #endif
  158. }
  159. static inline int
  160. need_iommu(struct device *dev, unsigned long addr, size_t size)
  161. {
  162. return force_iommu || !dma_capable(dev, addr, size, true);
  163. }
  164. static inline int
  165. nonforced_iommu(struct device *dev, unsigned long addr, size_t size)
  166. {
  167. return !dma_capable(dev, addr, size, true);
  168. }
  169. /* Map a single continuous physical area into the IOMMU.
  170. * Caller needs to check if the iommu is needed and flush.
  171. */
  172. static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem,
  173. size_t size, int dir, unsigned long align_mask)
  174. {
  175. unsigned long npages = iommu_num_pages(phys_mem, size, PAGE_SIZE);
  176. unsigned long iommu_page;
  177. int i;
  178. if (unlikely(phys_mem + size > GART_MAX_PHYS_ADDR))
  179. return DMA_MAPPING_ERROR;
  180. iommu_page = alloc_iommu(dev, npages, align_mask);
  181. if (iommu_page == -1) {
  182. if (!nonforced_iommu(dev, phys_mem, size))
  183. return phys_mem;
  184. if (panic_on_overflow)
  185. panic("dma_map_area overflow %lu bytes\n", size);
  186. iommu_full(dev, size, dir);
  187. return DMA_MAPPING_ERROR;
  188. }
  189. for (i = 0; i < npages; i++) {
  190. iommu_gatt_base[iommu_page + i] = GPTE_ENCODE(phys_mem);
  191. phys_mem += PAGE_SIZE;
  192. }
  193. return iommu_bus_base + iommu_page*PAGE_SIZE + (phys_mem & ~PAGE_MASK);
  194. }
  195. /* Map a single area into the IOMMU */
  196. static dma_addr_t gart_map_page(struct device *dev, struct page *page,
  197. unsigned long offset, size_t size,
  198. enum dma_data_direction dir,
  199. unsigned long attrs)
  200. {
  201. unsigned long bus;
  202. phys_addr_t paddr = page_to_phys(page) + offset;
  203. if (!need_iommu(dev, paddr, size))
  204. return paddr;
  205. bus = dma_map_area(dev, paddr, size, dir, 0);
  206. flush_gart();
  207. return bus;
  208. }
  209. /*
  210. * Free a DMA mapping.
  211. */
  212. static void gart_unmap_page(struct device *dev, dma_addr_t dma_addr,
  213. size_t size, enum dma_data_direction dir,
  214. unsigned long attrs)
  215. {
  216. unsigned long iommu_page;
  217. int npages;
  218. int i;
  219. if (WARN_ON_ONCE(dma_addr == DMA_MAPPING_ERROR))
  220. return;
  221. /*
  222. * This driver will not always use a GART mapping, but might have
  223. * created a direct mapping instead. If that is the case there is
  224. * nothing to unmap here.
  225. */
  226. if (dma_addr < iommu_bus_base ||
  227. dma_addr >= iommu_bus_base + iommu_size)
  228. return;
  229. iommu_page = (dma_addr - iommu_bus_base)>>PAGE_SHIFT;
  230. npages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
  231. for (i = 0; i < npages; i++) {
  232. iommu_gatt_base[iommu_page + i] = gart_unmapped_entry;
  233. }
  234. free_iommu(iommu_page, npages);
  235. }
  236. /*
  237. * Wrapper for pci_unmap_single working with scatterlists.
  238. */
  239. static void gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
  240. enum dma_data_direction dir, unsigned long attrs)
  241. {
  242. struct scatterlist *s;
  243. int i;
  244. for_each_sg(sg, s, nents, i) {
  245. if (!s->dma_length || !s->length)
  246. break;
  247. gart_unmap_page(dev, s->dma_address, s->dma_length, dir, 0);
  248. }
  249. }
  250. /* Fallback for dma_map_sg in case of overflow */
  251. static int dma_map_sg_nonforce(struct device *dev, struct scatterlist *sg,
  252. int nents, int dir)
  253. {
  254. struct scatterlist *s;
  255. int i;
  256. #ifdef CONFIG_IOMMU_DEBUG
  257. pr_debug("dma_map_sg overflow\n");
  258. #endif
  259. for_each_sg(sg, s, nents, i) {
  260. unsigned long addr = sg_phys(s);
  261. if (nonforced_iommu(dev, addr, s->length)) {
  262. addr = dma_map_area(dev, addr, s->length, dir, 0);
  263. if (addr == DMA_MAPPING_ERROR) {
  264. if (i > 0)
  265. gart_unmap_sg(dev, sg, i, dir, 0);
  266. nents = 0;
  267. sg[0].dma_length = 0;
  268. break;
  269. }
  270. }
  271. s->dma_address = addr;
  272. s->dma_length = s->length;
  273. }
  274. flush_gart();
  275. return nents;
  276. }
  277. /* Map multiple scatterlist entries continuous into the first. */
  278. static int __dma_map_cont(struct device *dev, struct scatterlist *start,
  279. int nelems, struct scatterlist *sout,
  280. unsigned long pages)
  281. {
  282. unsigned long iommu_start = alloc_iommu(dev, pages, 0);
  283. unsigned long iommu_page = iommu_start;
  284. struct scatterlist *s;
  285. int i;
  286. if (iommu_start == -1)
  287. return -1;
  288. for_each_sg(start, s, nelems, i) {
  289. unsigned long pages, addr;
  290. unsigned long phys_addr = s->dma_address;
  291. BUG_ON(s != start && s->offset);
  292. if (s == start) {
  293. sout->dma_address = iommu_bus_base;
  294. sout->dma_address += iommu_page*PAGE_SIZE + s->offset;
  295. sout->dma_length = s->length;
  296. } else {
  297. sout->dma_length += s->length;
  298. }
  299. addr = phys_addr;
  300. pages = iommu_num_pages(s->offset, s->length, PAGE_SIZE);
  301. while (pages--) {
  302. iommu_gatt_base[iommu_page] = GPTE_ENCODE(addr);
  303. addr += PAGE_SIZE;
  304. iommu_page++;
  305. }
  306. }
  307. BUG_ON(iommu_page - iommu_start != pages);
  308. return 0;
  309. }
  310. static inline int
  311. dma_map_cont(struct device *dev, struct scatterlist *start, int nelems,
  312. struct scatterlist *sout, unsigned long pages, int need)
  313. {
  314. if (!need) {
  315. BUG_ON(nelems != 1);
  316. sout->dma_address = start->dma_address;
  317. sout->dma_length = start->length;
  318. return 0;
  319. }
  320. return __dma_map_cont(dev, start, nelems, sout, pages);
  321. }
  322. /*
  323. * DMA map all entries in a scatterlist.
  324. * Merge chunks that have page aligned sizes into a continuous mapping.
  325. */
  326. static int gart_map_sg(struct device *dev, struct scatterlist *sg, int nents,
  327. enum dma_data_direction dir, unsigned long attrs)
  328. {
  329. struct scatterlist *s, *ps, *start_sg, *sgmap;
  330. int need = 0, nextneed, i, out, start;
  331. unsigned long pages = 0;
  332. unsigned int seg_size;
  333. unsigned int max_seg_size;
  334. if (nents == 0)
  335. return 0;
  336. out = 0;
  337. start = 0;
  338. start_sg = sg;
  339. sgmap = sg;
  340. seg_size = 0;
  341. max_seg_size = dma_get_max_seg_size(dev);
  342. ps = NULL; /* shut up gcc */
  343. for_each_sg(sg, s, nents, i) {
  344. dma_addr_t addr = sg_phys(s);
  345. s->dma_address = addr;
  346. BUG_ON(s->length == 0);
  347. nextneed = need_iommu(dev, addr, s->length);
  348. /* Handle the previous not yet processed entries */
  349. if (i > start) {
  350. /*
  351. * Can only merge when the last chunk ends on a
  352. * page boundary and the new one doesn't have an
  353. * offset.
  354. */
  355. if (!iommu_merge || !nextneed || !need || s->offset ||
  356. (s->length + seg_size > max_seg_size) ||
  357. (ps->offset + ps->length) % PAGE_SIZE) {
  358. if (dma_map_cont(dev, start_sg, i - start,
  359. sgmap, pages, need) < 0)
  360. goto error;
  361. out++;
  362. seg_size = 0;
  363. sgmap = sg_next(sgmap);
  364. pages = 0;
  365. start = i;
  366. start_sg = s;
  367. }
  368. }
  369. seg_size += s->length;
  370. need = nextneed;
  371. pages += iommu_num_pages(s->offset, s->length, PAGE_SIZE);
  372. ps = s;
  373. }
  374. if (dma_map_cont(dev, start_sg, i - start, sgmap, pages, need) < 0)
  375. goto error;
  376. out++;
  377. flush_gart();
  378. if (out < nents) {
  379. sgmap = sg_next(sgmap);
  380. sgmap->dma_length = 0;
  381. }
  382. return out;
  383. error:
  384. flush_gart();
  385. gart_unmap_sg(dev, sg, out, dir, 0);
  386. /* When it was forced or merged try again in a dumb way */
  387. if (force_iommu || iommu_merge) {
  388. out = dma_map_sg_nonforce(dev, sg, nents, dir);
  389. if (out > 0)
  390. return out;
  391. }
  392. if (panic_on_overflow)
  393. panic("dma_map_sg: overflow on %lu pages\n", pages);
  394. iommu_full(dev, pages << PAGE_SHIFT, dir);
  395. for_each_sg(sg, s, nents, i)
  396. s->dma_address = DMA_MAPPING_ERROR;
  397. return 0;
  398. }
  399. /* allocate and map a coherent mapping */
  400. static void *
  401. gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr,
  402. gfp_t flag, unsigned long attrs)
  403. {
  404. void *vaddr;
  405. vaddr = dma_direct_alloc(dev, size, dma_addr, flag, attrs);
  406. if (!vaddr ||
  407. !force_iommu || dev->coherent_dma_mask <= DMA_BIT_MASK(24))
  408. return vaddr;
  409. *dma_addr = dma_map_area(dev, virt_to_phys(vaddr), size,
  410. DMA_BIDIRECTIONAL, (1UL << get_order(size)) - 1);
  411. flush_gart();
  412. if (unlikely(*dma_addr == DMA_MAPPING_ERROR))
  413. goto out_free;
  414. return vaddr;
  415. out_free:
  416. dma_direct_free(dev, size, vaddr, *dma_addr, attrs);
  417. return NULL;
  418. }
  419. /* free a coherent mapping */
  420. static void
  421. gart_free_coherent(struct device *dev, size_t size, void *vaddr,
  422. dma_addr_t dma_addr, unsigned long attrs)
  423. {
  424. gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, 0);
  425. dma_direct_free(dev, size, vaddr, dma_addr, attrs);
  426. }
  427. static int no_agp;
  428. static __init unsigned long check_iommu_size(unsigned long aper, u64 aper_size)
  429. {
  430. unsigned long a;
  431. if (!iommu_size) {
  432. iommu_size = aper_size;
  433. if (!no_agp)
  434. iommu_size /= 2;
  435. }
  436. a = aper + iommu_size;
  437. iommu_size -= round_up(a, PMD_PAGE_SIZE) - a;
  438. if (iommu_size < 64*1024*1024) {
  439. pr_warn("PCI-DMA: Warning: Small IOMMU %luMB."
  440. " Consider increasing the AGP aperture in BIOS\n",
  441. iommu_size >> 20);
  442. }
  443. return iommu_size;
  444. }
  445. static __init unsigned read_aperture(struct pci_dev *dev, u32 *size)
  446. {
  447. unsigned aper_size = 0, aper_base_32, aper_order;
  448. u64 aper_base;
  449. pci_read_config_dword(dev, AMD64_GARTAPERTUREBASE, &aper_base_32);
  450. pci_read_config_dword(dev, AMD64_GARTAPERTURECTL, &aper_order);
  451. aper_order = (aper_order >> 1) & 7;
  452. aper_base = aper_base_32 & 0x7fff;
  453. aper_base <<= 25;
  454. aper_size = (32 * 1024 * 1024) << aper_order;
  455. if (aper_base + aper_size > 0x100000000UL || !aper_size)
  456. aper_base = 0;
  457. *size = aper_size;
  458. return aper_base;
  459. }
  460. static void enable_gart_translations(void)
  461. {
  462. int i;
  463. if (!amd_nb_has_feature(AMD_NB_GART))
  464. return;
  465. for (i = 0; i < amd_nb_num(); i++) {
  466. struct pci_dev *dev = node_to_amd_nb(i)->misc;
  467. enable_gart_translation(dev, __pa(agp_gatt_table));
  468. }
  469. /* Flush the GART-TLB to remove stale entries */
  470. amd_flush_garts();
  471. }
  472. /*
  473. * If fix_up_north_bridges is set, the north bridges have to be fixed up on
  474. * resume in the same way as they are handled in gart_iommu_hole_init().
  475. */
  476. static bool fix_up_north_bridges;
  477. static u32 aperture_order;
  478. static u32 aperture_alloc;
  479. void set_up_gart_resume(u32 aper_order, u32 aper_alloc)
  480. {
  481. fix_up_north_bridges = true;
  482. aperture_order = aper_order;
  483. aperture_alloc = aper_alloc;
  484. }
  485. static void gart_fixup_northbridges(void)
  486. {
  487. int i;
  488. if (!fix_up_north_bridges)
  489. return;
  490. if (!amd_nb_has_feature(AMD_NB_GART))
  491. return;
  492. pr_info("PCI-DMA: Restoring GART aperture settings\n");
  493. for (i = 0; i < amd_nb_num(); i++) {
  494. struct pci_dev *dev = node_to_amd_nb(i)->misc;
  495. /*
  496. * Don't enable translations just yet. That is the next
  497. * step. Restore the pre-suspend aperture settings.
  498. */
  499. gart_set_size_and_enable(dev, aperture_order);
  500. pci_write_config_dword(dev, AMD64_GARTAPERTUREBASE, aperture_alloc >> 25);
  501. }
  502. }
  503. static void gart_resume(void)
  504. {
  505. pr_info("PCI-DMA: Resuming GART IOMMU\n");
  506. gart_fixup_northbridges();
  507. enable_gart_translations();
  508. }
  509. static struct syscore_ops gart_syscore_ops = {
  510. .resume = gart_resume,
  511. };
  512. /*
  513. * Private Northbridge GATT initialization in case we cannot use the
  514. * AGP driver for some reason.
  515. */
  516. static __init int init_amd_gatt(struct agp_kern_info *info)
  517. {
  518. unsigned aper_size, gatt_size, new_aper_size;
  519. unsigned aper_base, new_aper_base;
  520. struct pci_dev *dev;
  521. void *gatt;
  522. int i;
  523. pr_info("PCI-DMA: Disabling AGP.\n");
  524. aper_size = aper_base = info->aper_size = 0;
  525. dev = NULL;
  526. for (i = 0; i < amd_nb_num(); i++) {
  527. dev = node_to_amd_nb(i)->misc;
  528. new_aper_base = read_aperture(dev, &new_aper_size);
  529. if (!new_aper_base)
  530. goto nommu;
  531. if (!aper_base) {
  532. aper_size = new_aper_size;
  533. aper_base = new_aper_base;
  534. }
  535. if (aper_size != new_aper_size || aper_base != new_aper_base)
  536. goto nommu;
  537. }
  538. if (!aper_base)
  539. goto nommu;
  540. info->aper_base = aper_base;
  541. info->aper_size = aper_size >> 20;
  542. gatt_size = (aper_size >> PAGE_SHIFT) * sizeof(u32);
  543. gatt = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
  544. get_order(gatt_size));
  545. if (!gatt)
  546. panic("Cannot allocate GATT table");
  547. if (set_memory_uc((unsigned long)gatt, gatt_size >> PAGE_SHIFT))
  548. panic("Could not set GART PTEs to uncacheable pages");
  549. agp_gatt_table = gatt;
  550. register_syscore_ops(&gart_syscore_ops);
  551. flush_gart();
  552. pr_info("PCI-DMA: aperture base @ %x size %u KB\n",
  553. aper_base, aper_size>>10);
  554. return 0;
  555. nommu:
  556. /* Should not happen anymore */
  557. pr_warn("PCI-DMA: More than 4GB of RAM and no IOMMU - falling back to iommu=soft.\n");
  558. return -1;
  559. }
  560. static const struct dma_map_ops gart_dma_ops = {
  561. .map_sg = gart_map_sg,
  562. .unmap_sg = gart_unmap_sg,
  563. .map_page = gart_map_page,
  564. .unmap_page = gart_unmap_page,
  565. .alloc = gart_alloc_coherent,
  566. .free = gart_free_coherent,
  567. .mmap = dma_common_mmap,
  568. .get_sgtable = dma_common_get_sgtable,
  569. .dma_supported = dma_direct_supported,
  570. .get_required_mask = dma_direct_get_required_mask,
  571. .alloc_pages = dma_direct_alloc_pages,
  572. .free_pages = dma_direct_free_pages,
  573. };
  574. static void gart_iommu_shutdown(void)
  575. {
  576. struct pci_dev *dev;
  577. int i;
  578. /* don't shutdown it if there is AGP installed */
  579. if (!no_agp)
  580. return;
  581. if (!amd_nb_has_feature(AMD_NB_GART))
  582. return;
  583. for (i = 0; i < amd_nb_num(); i++) {
  584. u32 ctl;
  585. dev = node_to_amd_nb(i)->misc;
  586. pci_read_config_dword(dev, AMD64_GARTAPERTURECTL, &ctl);
  587. ctl &= ~GARTEN;
  588. pci_write_config_dword(dev, AMD64_GARTAPERTURECTL, ctl);
  589. }
  590. }
  591. int __init gart_iommu_init(void)
  592. {
  593. struct agp_kern_info info;
  594. unsigned long iommu_start;
  595. unsigned long aper_base, aper_size;
  596. unsigned long start_pfn, end_pfn;
  597. unsigned long scratch;
  598. if (!amd_nb_has_feature(AMD_NB_GART))
  599. return 0;
  600. #ifndef CONFIG_AGP_AMD64
  601. no_agp = 1;
  602. #else
  603. /* Makefile puts PCI initialization via subsys_initcall first. */
  604. /* Add other AMD AGP bridge drivers here */
  605. no_agp = no_agp ||
  606. (agp_amd64_init() < 0) ||
  607. (agp_copy_info(agp_bridge, &info) < 0);
  608. #endif
  609. if (no_iommu ||
  610. (!force_iommu && max_pfn <= MAX_DMA32_PFN) ||
  611. !gart_iommu_aperture ||
  612. (no_agp && init_amd_gatt(&info) < 0)) {
  613. if (max_pfn > MAX_DMA32_PFN) {
  614. pr_warn("More than 4GB of memory but GART IOMMU not available.\n");
  615. pr_warn("falling back to iommu=soft.\n");
  616. }
  617. return 0;
  618. }
  619. /* need to map that range */
  620. aper_size = info.aper_size << 20;
  621. aper_base = info.aper_base;
  622. end_pfn = (aper_base>>PAGE_SHIFT) + (aper_size>>PAGE_SHIFT);
  623. start_pfn = PFN_DOWN(aper_base);
  624. if (!pfn_range_is_mapped(start_pfn, end_pfn))
  625. init_memory_mapping(start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT,
  626. PAGE_KERNEL);
  627. pr_info("PCI-DMA: using GART IOMMU.\n");
  628. iommu_size = check_iommu_size(info.aper_base, aper_size);
  629. iommu_pages = iommu_size >> PAGE_SHIFT;
  630. iommu_gart_bitmap = (void *) __get_free_pages(GFP_KERNEL | __GFP_ZERO,
  631. get_order(iommu_pages/8));
  632. if (!iommu_gart_bitmap)
  633. panic("Cannot allocate iommu bitmap\n");
  634. pr_info("PCI-DMA: Reserving %luMB of IOMMU area in the AGP aperture\n",
  635. iommu_size >> 20);
  636. agp_memory_reserved = iommu_size;
  637. iommu_start = aper_size - iommu_size;
  638. iommu_bus_base = info.aper_base + iommu_start;
  639. iommu_gatt_base = agp_gatt_table + (iommu_start>>PAGE_SHIFT);
  640. /*
  641. * Unmap the IOMMU part of the GART. The alias of the page is
  642. * always mapped with cache enabled and there is no full cache
  643. * coherency across the GART remapping. The unmapping avoids
  644. * automatic prefetches from the CPU allocating cache lines in
  645. * there. All CPU accesses are done via the direct mapping to
  646. * the backing memory. The GART address is only used by PCI
  647. * devices.
  648. */
  649. set_memory_np((unsigned long)__va(iommu_bus_base),
  650. iommu_size >> PAGE_SHIFT);
  651. /*
  652. * Tricky. The GART table remaps the physical memory range,
  653. * so the CPU wont notice potential aliases and if the memory
  654. * is remapped to UC later on, we might surprise the PCI devices
  655. * with a stray writeout of a cacheline. So play it sure and
  656. * do an explicit, full-scale wbinvd() _after_ having marked all
  657. * the pages as Not-Present:
  658. */
  659. wbinvd();
  660. /*
  661. * Now all caches are flushed and we can safely enable
  662. * GART hardware. Doing it early leaves the possibility
  663. * of stale cache entries that can lead to GART PTE
  664. * errors.
  665. */
  666. enable_gart_translations();
  667. /*
  668. * Try to workaround a bug (thanks to BenH):
  669. * Set unmapped entries to a scratch page instead of 0.
  670. * Any prefetches that hit unmapped entries won't get an bus abort
  671. * then. (P2P bridge may be prefetching on DMA reads).
  672. */
  673. scratch = get_zeroed_page(GFP_KERNEL);
  674. if (!scratch)
  675. panic("Cannot allocate iommu scratch page");
  676. gart_unmapped_entry = GPTE_ENCODE(__pa(scratch));
  677. flush_gart();
  678. dma_ops = &gart_dma_ops;
  679. x86_platform.iommu_shutdown = gart_iommu_shutdown;
  680. swiotlb = 0;
  681. return 0;
  682. }
  683. void __init gart_parse_options(char *p)
  684. {
  685. int arg;
  686. if (isdigit(*p) && get_option(&p, &arg))
  687. iommu_size = arg;
  688. if (!strncmp(p, "fullflush", 9))
  689. iommu_fullflush = 1;
  690. if (!strncmp(p, "nofullflush", 11))
  691. iommu_fullflush = 0;
  692. if (!strncmp(p, "noagp", 5))
  693. no_agp = 1;
  694. if (!strncmp(p, "noaperture", 10))
  695. fix_aperture = 0;
  696. /* duplicated from pci-dma.c */
  697. if (!strncmp(p, "force", 5))
  698. gart_iommu_aperture_allowed = 1;
  699. if (!strncmp(p, "allowed", 7))
  700. gart_iommu_aperture_allowed = 1;
  701. if (!strncmp(p, "memaper", 7)) {
  702. fallback_aper_force = 1;
  703. p += 7;
  704. if (*p == '=') {
  705. ++p;
  706. if (get_option(&p, &arg))
  707. fallback_aper_order = arg;
  708. }
  709. }
  710. }
  711. IOMMU_INIT_POST(gart_iommu_hole_init);