fsl_pamu_domain.c 27 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. *
  4. * Copyright (C) 2013 Freescale Semiconductor, Inc.
  5. * Author: Varun Sethi <varun.sethi@freescale.com>
  6. */
  7. #define pr_fmt(fmt) "fsl-pamu-domain: %s: " fmt, __func__
  8. #include "fsl_pamu_domain.h"
  9. #include <sysdev/fsl_pci.h>
  10. /*
  11. * Global spinlock that needs to be held while
  12. * configuring PAMU.
  13. */
  14. static DEFINE_SPINLOCK(iommu_lock);
  15. static struct kmem_cache *fsl_pamu_domain_cache;
  16. static struct kmem_cache *iommu_devinfo_cache;
  17. static DEFINE_SPINLOCK(device_domain_lock);
  18. struct iommu_device pamu_iommu; /* IOMMU core code handle */
  19. static struct fsl_dma_domain *to_fsl_dma_domain(struct iommu_domain *dom)
  20. {
  21. return container_of(dom, struct fsl_dma_domain, iommu_domain);
  22. }
  23. static int __init iommu_init_mempool(void)
  24. {
  25. fsl_pamu_domain_cache = kmem_cache_create("fsl_pamu_domain",
  26. sizeof(struct fsl_dma_domain),
  27. 0,
  28. SLAB_HWCACHE_ALIGN,
  29. NULL);
  30. if (!fsl_pamu_domain_cache) {
  31. pr_debug("Couldn't create fsl iommu_domain cache\n");
  32. return -ENOMEM;
  33. }
  34. iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
  35. sizeof(struct device_domain_info),
  36. 0,
  37. SLAB_HWCACHE_ALIGN,
  38. NULL);
  39. if (!iommu_devinfo_cache) {
  40. pr_debug("Couldn't create devinfo cache\n");
  41. kmem_cache_destroy(fsl_pamu_domain_cache);
  42. return -ENOMEM;
  43. }
  44. return 0;
  45. }
  46. static phys_addr_t get_phys_addr(struct fsl_dma_domain *dma_domain, dma_addr_t iova)
  47. {
  48. u32 win_cnt = dma_domain->win_cnt;
  49. struct dma_window *win_ptr = &dma_domain->win_arr[0];
  50. struct iommu_domain_geometry *geom;
  51. geom = &dma_domain->iommu_domain.geometry;
  52. if (!win_cnt || !dma_domain->geom_size) {
  53. pr_debug("Number of windows/geometry not configured for the domain\n");
  54. return 0;
  55. }
  56. if (win_cnt > 1) {
  57. u64 subwin_size;
  58. dma_addr_t subwin_iova;
  59. u32 wnd;
  60. subwin_size = dma_domain->geom_size >> ilog2(win_cnt);
  61. subwin_iova = iova & ~(subwin_size - 1);
  62. wnd = (subwin_iova - geom->aperture_start) >> ilog2(subwin_size);
  63. win_ptr = &dma_domain->win_arr[wnd];
  64. }
  65. if (win_ptr->valid)
  66. return win_ptr->paddr + (iova & (win_ptr->size - 1));
  67. return 0;
  68. }
  69. static int map_subwins(int liodn, struct fsl_dma_domain *dma_domain)
  70. {
  71. struct dma_window *sub_win_ptr = &dma_domain->win_arr[0];
  72. int i, ret;
  73. unsigned long rpn, flags;
  74. for (i = 0; i < dma_domain->win_cnt; i++) {
  75. if (sub_win_ptr[i].valid) {
  76. rpn = sub_win_ptr[i].paddr >> PAMU_PAGE_SHIFT;
  77. spin_lock_irqsave(&iommu_lock, flags);
  78. ret = pamu_config_spaace(liodn, dma_domain->win_cnt, i,
  79. sub_win_ptr[i].size,
  80. ~(u32)0,
  81. rpn,
  82. dma_domain->snoop_id,
  83. dma_domain->stash_id,
  84. (i > 0) ? 1 : 0,
  85. sub_win_ptr[i].prot);
  86. spin_unlock_irqrestore(&iommu_lock, flags);
  87. if (ret) {
  88. pr_debug("SPAACE configuration failed for liodn %d\n",
  89. liodn);
  90. return ret;
  91. }
  92. }
  93. }
  94. return ret;
  95. }
  96. static int map_win(int liodn, struct fsl_dma_domain *dma_domain)
  97. {
  98. int ret;
  99. struct dma_window *wnd = &dma_domain->win_arr[0];
  100. phys_addr_t wnd_addr = dma_domain->iommu_domain.geometry.aperture_start;
  101. unsigned long flags;
  102. spin_lock_irqsave(&iommu_lock, flags);
  103. ret = pamu_config_ppaace(liodn, wnd_addr,
  104. wnd->size,
  105. ~(u32)0,
  106. wnd->paddr >> PAMU_PAGE_SHIFT,
  107. dma_domain->snoop_id, dma_domain->stash_id,
  108. 0, wnd->prot);
  109. spin_unlock_irqrestore(&iommu_lock, flags);
  110. if (ret)
  111. pr_debug("PAACE configuration failed for liodn %d\n", liodn);
  112. return ret;
  113. }
  114. /* Map the DMA window corresponding to the LIODN */
  115. static int map_liodn(int liodn, struct fsl_dma_domain *dma_domain)
  116. {
  117. if (dma_domain->win_cnt > 1)
  118. return map_subwins(liodn, dma_domain);
  119. else
  120. return map_win(liodn, dma_domain);
  121. }
  122. /* Update window/subwindow mapping for the LIODN */
  123. static int update_liodn(int liodn, struct fsl_dma_domain *dma_domain, u32 wnd_nr)
  124. {
  125. int ret;
  126. struct dma_window *wnd = &dma_domain->win_arr[wnd_nr];
  127. unsigned long flags;
  128. spin_lock_irqsave(&iommu_lock, flags);
  129. if (dma_domain->win_cnt > 1) {
  130. ret = pamu_config_spaace(liodn, dma_domain->win_cnt, wnd_nr,
  131. wnd->size,
  132. ~(u32)0,
  133. wnd->paddr >> PAMU_PAGE_SHIFT,
  134. dma_domain->snoop_id,
  135. dma_domain->stash_id,
  136. (wnd_nr > 0) ? 1 : 0,
  137. wnd->prot);
  138. if (ret)
  139. pr_debug("Subwindow reconfiguration failed for liodn %d\n",
  140. liodn);
  141. } else {
  142. phys_addr_t wnd_addr;
  143. wnd_addr = dma_domain->iommu_domain.geometry.aperture_start;
  144. ret = pamu_config_ppaace(liodn, wnd_addr,
  145. wnd->size,
  146. ~(u32)0,
  147. wnd->paddr >> PAMU_PAGE_SHIFT,
  148. dma_domain->snoop_id, dma_domain->stash_id,
  149. 0, wnd->prot);
  150. if (ret)
  151. pr_debug("Window reconfiguration failed for liodn %d\n",
  152. liodn);
  153. }
  154. spin_unlock_irqrestore(&iommu_lock, flags);
  155. return ret;
  156. }
  157. static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain,
  158. u32 val)
  159. {
  160. int ret = 0, i;
  161. unsigned long flags;
  162. spin_lock_irqsave(&iommu_lock, flags);
  163. if (!dma_domain->win_arr) {
  164. pr_debug("Windows not configured, stash destination update failed for liodn %d\n",
  165. liodn);
  166. spin_unlock_irqrestore(&iommu_lock, flags);
  167. return -EINVAL;
  168. }
  169. for (i = 0; i < dma_domain->win_cnt; i++) {
  170. ret = pamu_update_paace_stash(liodn, i, val);
  171. if (ret) {
  172. pr_debug("Failed to update SPAACE %d field for liodn %d\n ",
  173. i, liodn);
  174. spin_unlock_irqrestore(&iommu_lock, flags);
  175. return ret;
  176. }
  177. }
  178. spin_unlock_irqrestore(&iommu_lock, flags);
  179. return ret;
  180. }
  181. /* Set the geometry parameters for a LIODN */
  182. static int pamu_set_liodn(int liodn, struct device *dev,
  183. struct fsl_dma_domain *dma_domain,
  184. struct iommu_domain_geometry *geom_attr,
  185. u32 win_cnt)
  186. {
  187. phys_addr_t window_addr, window_size;
  188. phys_addr_t subwin_size;
  189. int ret = 0, i;
  190. u32 omi_index = ~(u32)0;
  191. unsigned long flags;
  192. /*
  193. * Configure the omi_index at the geometry setup time.
  194. * This is a static value which depends on the type of
  195. * device and would not change thereafter.
  196. */
  197. get_ome_index(&omi_index, dev);
  198. window_addr = geom_attr->aperture_start;
  199. window_size = dma_domain->geom_size;
  200. spin_lock_irqsave(&iommu_lock, flags);
  201. ret = pamu_disable_liodn(liodn);
  202. if (!ret)
  203. ret = pamu_config_ppaace(liodn, window_addr, window_size, omi_index,
  204. 0, dma_domain->snoop_id,
  205. dma_domain->stash_id, win_cnt, 0);
  206. spin_unlock_irqrestore(&iommu_lock, flags);
  207. if (ret) {
  208. pr_debug("PAACE configuration failed for liodn %d, win_cnt =%d\n",
  209. liodn, win_cnt);
  210. return ret;
  211. }
  212. if (win_cnt > 1) {
  213. subwin_size = window_size >> ilog2(win_cnt);
  214. for (i = 0; i < win_cnt; i++) {
  215. spin_lock_irqsave(&iommu_lock, flags);
  216. ret = pamu_disable_spaace(liodn, i);
  217. if (!ret)
  218. ret = pamu_config_spaace(liodn, win_cnt, i,
  219. subwin_size, omi_index,
  220. 0, dma_domain->snoop_id,
  221. dma_domain->stash_id,
  222. 0, 0);
  223. spin_unlock_irqrestore(&iommu_lock, flags);
  224. if (ret) {
  225. pr_debug("SPAACE configuration failed for liodn %d\n",
  226. liodn);
  227. return ret;
  228. }
  229. }
  230. }
  231. return ret;
  232. }
  233. static int check_size(u64 size, dma_addr_t iova)
  234. {
  235. /*
  236. * Size must be a power of two and at least be equal
  237. * to PAMU page size.
  238. */
  239. if ((size & (size - 1)) || size < PAMU_PAGE_SIZE) {
  240. pr_debug("Size too small or not a power of two\n");
  241. return -EINVAL;
  242. }
  243. /* iova must be page size aligned */
  244. if (iova & (size - 1)) {
  245. pr_debug("Address is not aligned with window size\n");
  246. return -EINVAL;
  247. }
  248. return 0;
  249. }
  250. static struct fsl_dma_domain *iommu_alloc_dma_domain(void)
  251. {
  252. struct fsl_dma_domain *domain;
  253. domain = kmem_cache_zalloc(fsl_pamu_domain_cache, GFP_KERNEL);
  254. if (!domain)
  255. return NULL;
  256. domain->stash_id = ~(u32)0;
  257. domain->snoop_id = ~(u32)0;
  258. domain->win_cnt = pamu_get_max_subwin_cnt();
  259. domain->geom_size = 0;
  260. INIT_LIST_HEAD(&domain->devices);
  261. spin_lock_init(&domain->domain_lock);
  262. return domain;
  263. }
  264. static void remove_device_ref(struct device_domain_info *info, u32 win_cnt)
  265. {
  266. unsigned long flags;
  267. list_del(&info->link);
  268. spin_lock_irqsave(&iommu_lock, flags);
  269. if (win_cnt > 1)
  270. pamu_free_subwins(info->liodn);
  271. pamu_disable_liodn(info->liodn);
  272. spin_unlock_irqrestore(&iommu_lock, flags);
  273. spin_lock_irqsave(&device_domain_lock, flags);
  274. dev_iommu_priv_set(info->dev, NULL);
  275. kmem_cache_free(iommu_devinfo_cache, info);
  276. spin_unlock_irqrestore(&device_domain_lock, flags);
  277. }
  278. static void detach_device(struct device *dev, struct fsl_dma_domain *dma_domain)
  279. {
  280. struct device_domain_info *info, *tmp;
  281. unsigned long flags;
  282. spin_lock_irqsave(&dma_domain->domain_lock, flags);
  283. /* Remove the device from the domain device list */
  284. list_for_each_entry_safe(info, tmp, &dma_domain->devices, link) {
  285. if (!dev || (info->dev == dev))
  286. remove_device_ref(info, dma_domain->win_cnt);
  287. }
  288. spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
  289. }
  290. static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct device *dev)
  291. {
  292. struct device_domain_info *info, *old_domain_info;
  293. unsigned long flags;
  294. spin_lock_irqsave(&device_domain_lock, flags);
  295. /*
  296. * Check here if the device is already attached to domain or not.
  297. * If the device is already attached to a domain detach it.
  298. */
  299. old_domain_info = dev_iommu_priv_get(dev);
  300. if (old_domain_info && old_domain_info->domain != dma_domain) {
  301. spin_unlock_irqrestore(&device_domain_lock, flags);
  302. detach_device(dev, old_domain_info->domain);
  303. spin_lock_irqsave(&device_domain_lock, flags);
  304. }
  305. info = kmem_cache_zalloc(iommu_devinfo_cache, GFP_ATOMIC);
  306. info->dev = dev;
  307. info->liodn = liodn;
  308. info->domain = dma_domain;
  309. list_add(&info->link, &dma_domain->devices);
  310. /*
  311. * In case of devices with multiple LIODNs just store
  312. * the info for the first LIODN as all
  313. * LIODNs share the same domain
  314. */
  315. if (!dev_iommu_priv_get(dev))
  316. dev_iommu_priv_set(dev, info);
  317. spin_unlock_irqrestore(&device_domain_lock, flags);
  318. }
  319. static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain,
  320. dma_addr_t iova)
  321. {
  322. struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
  323. if (iova < domain->geometry.aperture_start ||
  324. iova > domain->geometry.aperture_end)
  325. return 0;
  326. return get_phys_addr(dma_domain, iova);
  327. }
  328. static bool fsl_pamu_capable(enum iommu_cap cap)
  329. {
  330. return cap == IOMMU_CAP_CACHE_COHERENCY;
  331. }
  332. static void fsl_pamu_domain_free(struct iommu_domain *domain)
  333. {
  334. struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
  335. /* remove all the devices from the device list */
  336. detach_device(NULL, dma_domain);
  337. dma_domain->enabled = 0;
  338. dma_domain->mapped = 0;
  339. kmem_cache_free(fsl_pamu_domain_cache, dma_domain);
  340. }
  341. static struct iommu_domain *fsl_pamu_domain_alloc(unsigned type)
  342. {
  343. struct fsl_dma_domain *dma_domain;
  344. if (type != IOMMU_DOMAIN_UNMANAGED)
  345. return NULL;
  346. dma_domain = iommu_alloc_dma_domain();
  347. if (!dma_domain) {
  348. pr_debug("dma_domain allocation failed\n");
  349. return NULL;
  350. }
  351. /* defaul geometry 64 GB i.e. maximum system address */
  352. dma_domain->iommu_domain. geometry.aperture_start = 0;
  353. dma_domain->iommu_domain.geometry.aperture_end = (1ULL << 36) - 1;
  354. dma_domain->iommu_domain.geometry.force_aperture = true;
  355. return &dma_domain->iommu_domain;
  356. }
  357. /* Configure geometry settings for all LIODNs associated with domain */
  358. static int pamu_set_domain_geometry(struct fsl_dma_domain *dma_domain,
  359. struct iommu_domain_geometry *geom_attr,
  360. u32 win_cnt)
  361. {
  362. struct device_domain_info *info;
  363. int ret = 0;
  364. list_for_each_entry(info, &dma_domain->devices, link) {
  365. ret = pamu_set_liodn(info->liodn, info->dev, dma_domain,
  366. geom_attr, win_cnt);
  367. if (ret)
  368. break;
  369. }
  370. return ret;
  371. }
  372. /* Update stash destination for all LIODNs associated with the domain */
  373. static int update_domain_stash(struct fsl_dma_domain *dma_domain, u32 val)
  374. {
  375. struct device_domain_info *info;
  376. int ret = 0;
  377. list_for_each_entry(info, &dma_domain->devices, link) {
  378. ret = update_liodn_stash(info->liodn, dma_domain, val);
  379. if (ret)
  380. break;
  381. }
  382. return ret;
  383. }
  384. /* Update domain mappings for all LIODNs associated with the domain */
  385. static int update_domain_mapping(struct fsl_dma_domain *dma_domain, u32 wnd_nr)
  386. {
  387. struct device_domain_info *info;
  388. int ret = 0;
  389. list_for_each_entry(info, &dma_domain->devices, link) {
  390. ret = update_liodn(info->liodn, dma_domain, wnd_nr);
  391. if (ret)
  392. break;
  393. }
  394. return ret;
  395. }
  396. static int disable_domain_win(struct fsl_dma_domain *dma_domain, u32 wnd_nr)
  397. {
  398. struct device_domain_info *info;
  399. int ret = 0;
  400. list_for_each_entry(info, &dma_domain->devices, link) {
  401. if (dma_domain->win_cnt == 1 && dma_domain->enabled) {
  402. ret = pamu_disable_liodn(info->liodn);
  403. if (!ret)
  404. dma_domain->enabled = 0;
  405. } else {
  406. ret = pamu_disable_spaace(info->liodn, wnd_nr);
  407. }
  408. }
  409. return ret;
  410. }
  411. static void fsl_pamu_window_disable(struct iommu_domain *domain, u32 wnd_nr)
  412. {
  413. struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
  414. unsigned long flags;
  415. int ret;
  416. spin_lock_irqsave(&dma_domain->domain_lock, flags);
  417. if (!dma_domain->win_arr) {
  418. pr_debug("Number of windows not configured\n");
  419. spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
  420. return;
  421. }
  422. if (wnd_nr >= dma_domain->win_cnt) {
  423. pr_debug("Invalid window index\n");
  424. spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
  425. return;
  426. }
  427. if (dma_domain->win_arr[wnd_nr].valid) {
  428. ret = disable_domain_win(dma_domain, wnd_nr);
  429. if (!ret) {
  430. dma_domain->win_arr[wnd_nr].valid = 0;
  431. dma_domain->mapped--;
  432. }
  433. }
  434. spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
  435. }
  436. static int fsl_pamu_window_enable(struct iommu_domain *domain, u32 wnd_nr,
  437. phys_addr_t paddr, u64 size, int prot)
  438. {
  439. struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
  440. struct dma_window *wnd;
  441. int pamu_prot = 0;
  442. int ret;
  443. unsigned long flags;
  444. u64 win_size;
  445. if (prot & IOMMU_READ)
  446. pamu_prot |= PAACE_AP_PERMS_QUERY;
  447. if (prot & IOMMU_WRITE)
  448. pamu_prot |= PAACE_AP_PERMS_UPDATE;
  449. spin_lock_irqsave(&dma_domain->domain_lock, flags);
  450. if (!dma_domain->win_arr) {
  451. pr_debug("Number of windows not configured\n");
  452. spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
  453. return -ENODEV;
  454. }
  455. if (wnd_nr >= dma_domain->win_cnt) {
  456. pr_debug("Invalid window index\n");
  457. spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
  458. return -EINVAL;
  459. }
  460. win_size = dma_domain->geom_size >> ilog2(dma_domain->win_cnt);
  461. if (size > win_size) {
  462. pr_debug("Invalid window size\n");
  463. spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
  464. return -EINVAL;
  465. }
  466. if (dma_domain->win_cnt == 1) {
  467. if (dma_domain->enabled) {
  468. pr_debug("Disable the window before updating the mapping\n");
  469. spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
  470. return -EBUSY;
  471. }
  472. ret = check_size(size, domain->geometry.aperture_start);
  473. if (ret) {
  474. pr_debug("Aperture start not aligned to the size\n");
  475. spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
  476. return -EINVAL;
  477. }
  478. }
  479. wnd = &dma_domain->win_arr[wnd_nr];
  480. if (!wnd->valid) {
  481. wnd->paddr = paddr;
  482. wnd->size = size;
  483. wnd->prot = pamu_prot;
  484. ret = update_domain_mapping(dma_domain, wnd_nr);
  485. if (!ret) {
  486. wnd->valid = 1;
  487. dma_domain->mapped++;
  488. }
  489. } else {
  490. pr_debug("Disable the window before updating the mapping\n");
  491. ret = -EBUSY;
  492. }
  493. spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
  494. return ret;
  495. }
  496. /*
  497. * Attach the LIODN to the DMA domain and configure the geometry
  498. * and window mappings.
  499. */
  500. static int handle_attach_device(struct fsl_dma_domain *dma_domain,
  501. struct device *dev, const u32 *liodn,
  502. int num)
  503. {
  504. unsigned long flags;
  505. struct iommu_domain *domain = &dma_domain->iommu_domain;
  506. int ret = 0;
  507. int i;
  508. spin_lock_irqsave(&dma_domain->domain_lock, flags);
  509. for (i = 0; i < num; i++) {
  510. /* Ensure that LIODN value is valid */
  511. if (liodn[i] >= PAACE_NUMBER_ENTRIES) {
  512. pr_debug("Invalid liodn %d, attach device failed for %pOF\n",
  513. liodn[i], dev->of_node);
  514. ret = -EINVAL;
  515. break;
  516. }
  517. attach_device(dma_domain, liodn[i], dev);
  518. /*
  519. * Check if geometry has already been configured
  520. * for the domain. If yes, set the geometry for
  521. * the LIODN.
  522. */
  523. if (dma_domain->win_arr) {
  524. u32 win_cnt = dma_domain->win_cnt > 1 ? dma_domain->win_cnt : 0;
  525. ret = pamu_set_liodn(liodn[i], dev, dma_domain,
  526. &domain->geometry, win_cnt);
  527. if (ret)
  528. break;
  529. if (dma_domain->mapped) {
  530. /*
  531. * Create window/subwindow mapping for
  532. * the LIODN.
  533. */
  534. ret = map_liodn(liodn[i], dma_domain);
  535. if (ret)
  536. break;
  537. }
  538. }
  539. }
  540. spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
  541. return ret;
  542. }
  543. static int fsl_pamu_attach_device(struct iommu_domain *domain,
  544. struct device *dev)
  545. {
  546. struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
  547. const u32 *liodn;
  548. u32 liodn_cnt;
  549. int len, ret = 0;
  550. struct pci_dev *pdev = NULL;
  551. struct pci_controller *pci_ctl;
  552. /*
  553. * Use LIODN of the PCI controller while attaching a
  554. * PCI device.
  555. */
  556. if (dev_is_pci(dev)) {
  557. pdev = to_pci_dev(dev);
  558. pci_ctl = pci_bus_to_host(pdev->bus);
  559. /*
  560. * make dev point to pci controller device
  561. * so we can get the LIODN programmed by
  562. * u-boot.
  563. */
  564. dev = pci_ctl->parent;
  565. }
  566. liodn = of_get_property(dev->of_node, "fsl,liodn", &len);
  567. if (liodn) {
  568. liodn_cnt = len / sizeof(u32);
  569. ret = handle_attach_device(dma_domain, dev, liodn, liodn_cnt);
  570. } else {
  571. pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
  572. ret = -EINVAL;
  573. }
  574. return ret;
  575. }
  576. static void fsl_pamu_detach_device(struct iommu_domain *domain,
  577. struct device *dev)
  578. {
  579. struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
  580. const u32 *prop;
  581. int len;
  582. struct pci_dev *pdev = NULL;
  583. struct pci_controller *pci_ctl;
  584. /*
  585. * Use LIODN of the PCI controller while detaching a
  586. * PCI device.
  587. */
  588. if (dev_is_pci(dev)) {
  589. pdev = to_pci_dev(dev);
  590. pci_ctl = pci_bus_to_host(pdev->bus);
  591. /*
  592. * make dev point to pci controller device
  593. * so we can get the LIODN programmed by
  594. * u-boot.
  595. */
  596. dev = pci_ctl->parent;
  597. }
  598. prop = of_get_property(dev->of_node, "fsl,liodn", &len);
  599. if (prop)
  600. detach_device(dev, dma_domain);
  601. else
  602. pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
  603. }
  604. static int configure_domain_geometry(struct iommu_domain *domain, void *data)
  605. {
  606. struct iommu_domain_geometry *geom_attr = data;
  607. struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
  608. dma_addr_t geom_size;
  609. unsigned long flags;
  610. geom_size = geom_attr->aperture_end - geom_attr->aperture_start + 1;
  611. /*
  612. * Sanity check the geometry size. Also, we do not support
  613. * DMA outside of the geometry.
  614. */
  615. if (check_size(geom_size, geom_attr->aperture_start) ||
  616. !geom_attr->force_aperture) {
  617. pr_debug("Invalid PAMU geometry attributes\n");
  618. return -EINVAL;
  619. }
  620. spin_lock_irqsave(&dma_domain->domain_lock, flags);
  621. if (dma_domain->enabled) {
  622. pr_debug("Can't set geometry attributes as domain is active\n");
  623. spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
  624. return -EBUSY;
  625. }
  626. /* Copy the domain geometry information */
  627. memcpy(&domain->geometry, geom_attr,
  628. sizeof(struct iommu_domain_geometry));
  629. dma_domain->geom_size = geom_size;
  630. spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
  631. return 0;
  632. }
  633. /* Set the domain stash attribute */
  634. static int configure_domain_stash(struct fsl_dma_domain *dma_domain, void *data)
  635. {
  636. struct pamu_stash_attribute *stash_attr = data;
  637. unsigned long flags;
  638. int ret;
  639. spin_lock_irqsave(&dma_domain->domain_lock, flags);
  640. memcpy(&dma_domain->dma_stash, stash_attr,
  641. sizeof(struct pamu_stash_attribute));
  642. dma_domain->stash_id = get_stash_id(stash_attr->cache,
  643. stash_attr->cpu);
  644. if (dma_domain->stash_id == ~(u32)0) {
  645. pr_debug("Invalid stash attributes\n");
  646. spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
  647. return -EINVAL;
  648. }
  649. ret = update_domain_stash(dma_domain, dma_domain->stash_id);
  650. spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
  651. return ret;
  652. }
  653. /* Configure domain dma state i.e. enable/disable DMA */
  654. static int configure_domain_dma_state(struct fsl_dma_domain *dma_domain, bool enable)
  655. {
  656. struct device_domain_info *info;
  657. unsigned long flags;
  658. int ret;
  659. spin_lock_irqsave(&dma_domain->domain_lock, flags);
  660. if (enable && !dma_domain->mapped) {
  661. pr_debug("Can't enable DMA domain without valid mapping\n");
  662. spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
  663. return -ENODEV;
  664. }
  665. dma_domain->enabled = enable;
  666. list_for_each_entry(info, &dma_domain->devices, link) {
  667. ret = (enable) ? pamu_enable_liodn(info->liodn) :
  668. pamu_disable_liodn(info->liodn);
  669. if (ret)
  670. pr_debug("Unable to set dma state for liodn %d",
  671. info->liodn);
  672. }
  673. spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
  674. return 0;
  675. }
  676. static int fsl_pamu_set_windows(struct iommu_domain *domain, u32 w_count)
  677. {
  678. struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
  679. unsigned long flags;
  680. int ret;
  681. spin_lock_irqsave(&dma_domain->domain_lock, flags);
  682. /* Ensure domain is inactive i.e. DMA should be disabled for the domain */
  683. if (dma_domain->enabled) {
  684. pr_debug("Can't set geometry attributes as domain is active\n");
  685. spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
  686. return -EBUSY;
  687. }
  688. /* Ensure that the geometry has been set for the domain */
  689. if (!dma_domain->geom_size) {
  690. pr_debug("Please configure geometry before setting the number of windows\n");
  691. spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
  692. return -EINVAL;
  693. }
  694. /*
  695. * Ensure we have valid window count i.e. it should be less than
  696. * maximum permissible limit and should be a power of two.
  697. */
  698. if (w_count > pamu_get_max_subwin_cnt() || !is_power_of_2(w_count)) {
  699. pr_debug("Invalid window count\n");
  700. spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
  701. return -EINVAL;
  702. }
  703. ret = pamu_set_domain_geometry(dma_domain, &domain->geometry,
  704. w_count > 1 ? w_count : 0);
  705. if (!ret) {
  706. kfree(dma_domain->win_arr);
  707. dma_domain->win_arr = kcalloc(w_count,
  708. sizeof(*dma_domain->win_arr),
  709. GFP_ATOMIC);
  710. if (!dma_domain->win_arr) {
  711. spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
  712. return -ENOMEM;
  713. }
  714. dma_domain->win_cnt = w_count;
  715. }
  716. spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
  717. return ret;
  718. }
  719. static int fsl_pamu_set_domain_attr(struct iommu_domain *domain,
  720. enum iommu_attr attr_type, void *data)
  721. {
  722. struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
  723. int ret = 0;
  724. switch (attr_type) {
  725. case DOMAIN_ATTR_GEOMETRY:
  726. ret = configure_domain_geometry(domain, data);
  727. break;
  728. case DOMAIN_ATTR_FSL_PAMU_STASH:
  729. ret = configure_domain_stash(dma_domain, data);
  730. break;
  731. case DOMAIN_ATTR_FSL_PAMU_ENABLE:
  732. ret = configure_domain_dma_state(dma_domain, *(int *)data);
  733. break;
  734. case DOMAIN_ATTR_WINDOWS:
  735. ret = fsl_pamu_set_windows(domain, *(u32 *)data);
  736. break;
  737. default:
  738. pr_debug("Unsupported attribute type\n");
  739. ret = -EINVAL;
  740. break;
  741. }
  742. return ret;
  743. }
  744. static int fsl_pamu_get_domain_attr(struct iommu_domain *domain,
  745. enum iommu_attr attr_type, void *data)
  746. {
  747. struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
  748. int ret = 0;
  749. switch (attr_type) {
  750. case DOMAIN_ATTR_FSL_PAMU_STASH:
  751. memcpy(data, &dma_domain->dma_stash,
  752. sizeof(struct pamu_stash_attribute));
  753. break;
  754. case DOMAIN_ATTR_FSL_PAMU_ENABLE:
  755. *(int *)data = dma_domain->enabled;
  756. break;
  757. case DOMAIN_ATTR_FSL_PAMUV1:
  758. *(int *)data = DOMAIN_ATTR_FSL_PAMUV1;
  759. break;
  760. case DOMAIN_ATTR_WINDOWS:
  761. *(u32 *)data = dma_domain->win_cnt;
  762. break;
  763. default:
  764. pr_debug("Unsupported attribute type\n");
  765. ret = -EINVAL;
  766. break;
  767. }
  768. return ret;
  769. }
  770. static struct iommu_group *get_device_iommu_group(struct device *dev)
  771. {
  772. struct iommu_group *group;
  773. group = iommu_group_get(dev);
  774. if (!group)
  775. group = iommu_group_alloc();
  776. return group;
  777. }
  778. static bool check_pci_ctl_endpt_part(struct pci_controller *pci_ctl)
  779. {
  780. u32 version;
  781. /* Check the PCI controller version number by readding BRR1 register */
  782. version = in_be32(pci_ctl->cfg_addr + (PCI_FSL_BRR1 >> 2));
  783. version &= PCI_FSL_BRR1_VER;
  784. /* If PCI controller version is >= 0x204 we can partition endpoints */
  785. return version >= 0x204;
  786. }
  787. /* Get iommu group information from peer devices or devices on the parent bus */
  788. static struct iommu_group *get_shared_pci_device_group(struct pci_dev *pdev)
  789. {
  790. struct pci_dev *tmp;
  791. struct iommu_group *group;
  792. struct pci_bus *bus = pdev->bus;
  793. /*
  794. * Traverese the pci bus device list to get
  795. * the shared iommu group.
  796. */
  797. while (bus) {
  798. list_for_each_entry(tmp, &bus->devices, bus_list) {
  799. if (tmp == pdev)
  800. continue;
  801. group = iommu_group_get(&tmp->dev);
  802. if (group)
  803. return group;
  804. }
  805. bus = bus->parent;
  806. }
  807. return NULL;
  808. }
  809. static struct iommu_group *get_pci_device_group(struct pci_dev *pdev)
  810. {
  811. struct pci_controller *pci_ctl;
  812. bool pci_endpt_partitioning;
  813. struct iommu_group *group = NULL;
  814. pci_ctl = pci_bus_to_host(pdev->bus);
  815. pci_endpt_partitioning = check_pci_ctl_endpt_part(pci_ctl);
  816. /* We can partition PCIe devices so assign device group to the device */
  817. if (pci_endpt_partitioning) {
  818. group = pci_device_group(&pdev->dev);
  819. /*
  820. * PCIe controller is not a paritionable entity
  821. * free the controller device iommu_group.
  822. */
  823. if (pci_ctl->parent->iommu_group)
  824. iommu_group_remove_device(pci_ctl->parent);
  825. } else {
  826. /*
  827. * All devices connected to the controller will share the
  828. * PCI controllers device group. If this is the first
  829. * device to be probed for the pci controller, copy the
  830. * device group information from the PCI controller device
  831. * node and remove the PCI controller iommu group.
  832. * For subsequent devices, the iommu group information can
  833. * be obtained from sibling devices (i.e. from the bus_devices
  834. * link list).
  835. */
  836. if (pci_ctl->parent->iommu_group) {
  837. group = get_device_iommu_group(pci_ctl->parent);
  838. iommu_group_remove_device(pci_ctl->parent);
  839. } else {
  840. group = get_shared_pci_device_group(pdev);
  841. }
  842. }
  843. if (!group)
  844. group = ERR_PTR(-ENODEV);
  845. return group;
  846. }
  847. static struct iommu_group *fsl_pamu_device_group(struct device *dev)
  848. {
  849. struct iommu_group *group = ERR_PTR(-ENODEV);
  850. int len;
  851. /*
  852. * For platform devices we allocate a separate group for
  853. * each of the devices.
  854. */
  855. if (dev_is_pci(dev))
  856. group = get_pci_device_group(to_pci_dev(dev));
  857. else if (of_get_property(dev->of_node, "fsl,liodn", &len))
  858. group = get_device_iommu_group(dev);
  859. return group;
  860. }
  861. static struct iommu_device *fsl_pamu_probe_device(struct device *dev)
  862. {
  863. return &pamu_iommu;
  864. }
  865. static void fsl_pamu_release_device(struct device *dev)
  866. {
  867. }
  868. static const struct iommu_ops fsl_pamu_ops = {
  869. .capable = fsl_pamu_capable,
  870. .domain_alloc = fsl_pamu_domain_alloc,
  871. .domain_free = fsl_pamu_domain_free,
  872. .attach_dev = fsl_pamu_attach_device,
  873. .detach_dev = fsl_pamu_detach_device,
  874. .domain_window_enable = fsl_pamu_window_enable,
  875. .domain_window_disable = fsl_pamu_window_disable,
  876. .iova_to_phys = fsl_pamu_iova_to_phys,
  877. .domain_set_attr = fsl_pamu_set_domain_attr,
  878. .domain_get_attr = fsl_pamu_get_domain_attr,
  879. .probe_device = fsl_pamu_probe_device,
  880. .release_device = fsl_pamu_release_device,
  881. .device_group = fsl_pamu_device_group,
  882. };
  883. int __init pamu_domain_init(void)
  884. {
  885. int ret = 0;
  886. ret = iommu_init_mempool();
  887. if (ret)
  888. return ret;
  889. ret = iommu_device_sysfs_add(&pamu_iommu, NULL, NULL, "iommu0");
  890. if (ret)
  891. return ret;
  892. iommu_device_set_ops(&pamu_iommu, &fsl_pamu_ops);
  893. ret = iommu_device_register(&pamu_iommu);
  894. if (ret) {
  895. iommu_device_sysfs_remove(&pamu_iommu);
  896. pr_err("Can't register iommu device\n");
  897. return ret;
  898. }
  899. bus_set_iommu(&platform_bus_type, &fsl_pamu_ops);
  900. bus_set_iommu(&pci_bus_type, &fsl_pamu_ops);
  901. return ret;
  902. }