123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182 |
- // SPDX-License-Identifier: GPL-2.0-only
- /*
- * Copyright (C) 2011-2014 NVIDIA CORPORATION. All rights reserved.
- */
- #include <linux/bitops.h>
- #include <linux/debugfs.h>
- #include <linux/err.h>
- #include <linux/iommu.h>
- #include <linux/kernel.h>
- #include <linux/of.h>
- #include <linux/of_device.h>
- #include <linux/platform_device.h>
- #include <linux/slab.h>
- #include <linux/spinlock.h>
- #include <linux/dma-mapping.h>
- #include <soc/tegra/ahb.h>
- #include <soc/tegra/mc.h>
- struct tegra_smmu_group {
- struct list_head list;
- struct tegra_smmu *smmu;
- const struct tegra_smmu_group_soc *soc;
- struct iommu_group *group;
- unsigned int swgroup;
- };
- struct tegra_smmu {
- void __iomem *regs;
- struct device *dev;
- struct tegra_mc *mc;
- const struct tegra_smmu_soc *soc;
- struct list_head groups;
- unsigned long pfn_mask;
- unsigned long tlb_mask;
- unsigned long *asids;
- struct mutex lock;
- struct list_head list;
- struct dentry *debugfs;
- struct iommu_device iommu; /* IOMMU Core code handle */
- };
- struct tegra_smmu_as {
- struct iommu_domain domain;
- struct tegra_smmu *smmu;
- unsigned int use_count;
- spinlock_t lock;
- u32 *count;
- struct page **pts;
- struct page *pd;
- dma_addr_t pd_dma;
- unsigned id;
- u32 attr;
- };
- static struct tegra_smmu_as *to_smmu_as(struct iommu_domain *dom)
- {
- return container_of(dom, struct tegra_smmu_as, domain);
- }
- static inline void smmu_writel(struct tegra_smmu *smmu, u32 value,
- unsigned long offset)
- {
- writel(value, smmu->regs + offset);
- }
- static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset)
- {
- return readl(smmu->regs + offset);
- }
- #define SMMU_CONFIG 0x010
- #define SMMU_CONFIG_ENABLE (1 << 0)
- #define SMMU_TLB_CONFIG 0x14
- #define SMMU_TLB_CONFIG_HIT_UNDER_MISS (1 << 29)
- #define SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION (1 << 28)
- #define SMMU_TLB_CONFIG_ACTIVE_LINES(smmu) \
- ((smmu)->soc->num_tlb_lines & (smmu)->tlb_mask)
- #define SMMU_PTC_CONFIG 0x18
- #define SMMU_PTC_CONFIG_ENABLE (1 << 29)
- #define SMMU_PTC_CONFIG_REQ_LIMIT(x) (((x) & 0x0f) << 24)
- #define SMMU_PTC_CONFIG_INDEX_MAP(x) ((x) & 0x3f)
- #define SMMU_PTB_ASID 0x01c
- #define SMMU_PTB_ASID_VALUE(x) ((x) & 0x7f)
- #define SMMU_PTB_DATA 0x020
- #define SMMU_PTB_DATA_VALUE(dma, attr) ((dma) >> 12 | (attr))
- #define SMMU_MK_PDE(dma, attr) ((dma) >> SMMU_PTE_SHIFT | (attr))
- #define SMMU_TLB_FLUSH 0x030
- #define SMMU_TLB_FLUSH_VA_MATCH_ALL (0 << 0)
- #define SMMU_TLB_FLUSH_VA_MATCH_SECTION (2 << 0)
- #define SMMU_TLB_FLUSH_VA_MATCH_GROUP (3 << 0)
- #define SMMU_TLB_FLUSH_VA_SECTION(addr) ((((addr) & 0xffc00000) >> 12) | \
- SMMU_TLB_FLUSH_VA_MATCH_SECTION)
- #define SMMU_TLB_FLUSH_VA_GROUP(addr) ((((addr) & 0xffffc000) >> 12) | \
- SMMU_TLB_FLUSH_VA_MATCH_GROUP)
- #define SMMU_TLB_FLUSH_ASID_MATCH (1 << 31)
- #define SMMU_PTC_FLUSH 0x034
- #define SMMU_PTC_FLUSH_TYPE_ALL (0 << 0)
- #define SMMU_PTC_FLUSH_TYPE_ADR (1 << 0)
- #define SMMU_PTC_FLUSH_HI 0x9b8
- #define SMMU_PTC_FLUSH_HI_MASK 0x3
- /* per-SWGROUP SMMU_*_ASID register */
- #define SMMU_ASID_ENABLE (1 << 31)
- #define SMMU_ASID_MASK 0x7f
- #define SMMU_ASID_VALUE(x) ((x) & SMMU_ASID_MASK)
- /* page table definitions */
- #define SMMU_NUM_PDE 1024
- #define SMMU_NUM_PTE 1024
- #define SMMU_SIZE_PD (SMMU_NUM_PDE * 4)
- #define SMMU_SIZE_PT (SMMU_NUM_PTE * 4)
- #define SMMU_PDE_SHIFT 22
- #define SMMU_PTE_SHIFT 12
- #define SMMU_PAGE_MASK (~(SMMU_SIZE_PT-1))
- #define SMMU_OFFSET_IN_PAGE(x) ((unsigned long)(x) & ~SMMU_PAGE_MASK)
- #define SMMU_PFN_PHYS(x) ((phys_addr_t)(x) << SMMU_PTE_SHIFT)
- #define SMMU_PHYS_PFN(x) ((unsigned long)((x) >> SMMU_PTE_SHIFT))
- #define SMMU_PD_READABLE (1 << 31)
- #define SMMU_PD_WRITABLE (1 << 30)
- #define SMMU_PD_NONSECURE (1 << 29)
- #define SMMU_PDE_READABLE (1 << 31)
- #define SMMU_PDE_WRITABLE (1 << 30)
- #define SMMU_PDE_NONSECURE (1 << 29)
- #define SMMU_PDE_NEXT (1 << 28)
- #define SMMU_PTE_READABLE (1 << 31)
- #define SMMU_PTE_WRITABLE (1 << 30)
- #define SMMU_PTE_NONSECURE (1 << 29)
- #define SMMU_PDE_ATTR (SMMU_PDE_READABLE | SMMU_PDE_WRITABLE | \
- SMMU_PDE_NONSECURE)
- static unsigned int iova_pd_index(unsigned long iova)
- {
- return (iova >> SMMU_PDE_SHIFT) & (SMMU_NUM_PDE - 1);
- }
- static unsigned int iova_pt_index(unsigned long iova)
- {
- return (iova >> SMMU_PTE_SHIFT) & (SMMU_NUM_PTE - 1);
- }
- static bool smmu_dma_addr_valid(struct tegra_smmu *smmu, dma_addr_t addr)
- {
- addr >>= 12;
- return (addr & smmu->pfn_mask) == addr;
- }
- static dma_addr_t smmu_pde_to_dma(struct tegra_smmu *smmu, u32 pde)
- {
- return (dma_addr_t)(pde & smmu->pfn_mask) << 12;
- }
- static void smmu_flush_ptc_all(struct tegra_smmu *smmu)
- {
- smmu_writel(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH);
- }
- static inline void smmu_flush_ptc(struct tegra_smmu *smmu, dma_addr_t dma,
- unsigned long offset)
- {
- u32 value;
- offset &= ~(smmu->mc->soc->atom_size - 1);
- if (smmu->mc->soc->num_address_bits > 32) {
- #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
- value = (dma >> 32) & SMMU_PTC_FLUSH_HI_MASK;
- #else
- value = 0;
- #endif
- smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI);
- }
- value = (dma + offset) | SMMU_PTC_FLUSH_TYPE_ADR;
- smmu_writel(smmu, value, SMMU_PTC_FLUSH);
- }
- static inline void smmu_flush_tlb(struct tegra_smmu *smmu)
- {
- smmu_writel(smmu, SMMU_TLB_FLUSH_VA_MATCH_ALL, SMMU_TLB_FLUSH);
- }
- static inline void smmu_flush_tlb_asid(struct tegra_smmu *smmu,
- unsigned long asid)
- {
- u32 value;
- if (smmu->soc->num_asids == 4)
- value = (asid & 0x3) << 29;
- else
- value = (asid & 0x7f) << 24;
- value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_MATCH_ALL;
- smmu_writel(smmu, value, SMMU_TLB_FLUSH);
- }
- static inline void smmu_flush_tlb_section(struct tegra_smmu *smmu,
- unsigned long asid,
- unsigned long iova)
- {
- u32 value;
- if (smmu->soc->num_asids == 4)
- value = (asid & 0x3) << 29;
- else
- value = (asid & 0x7f) << 24;
- value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_SECTION(iova);
- smmu_writel(smmu, value, SMMU_TLB_FLUSH);
- }
- static inline void smmu_flush_tlb_group(struct tegra_smmu *smmu,
- unsigned long asid,
- unsigned long iova)
- {
- u32 value;
- if (smmu->soc->num_asids == 4)
- value = (asid & 0x3) << 29;
- else
- value = (asid & 0x7f) << 24;
- value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_GROUP(iova);
- smmu_writel(smmu, value, SMMU_TLB_FLUSH);
- }
- static inline void smmu_flush(struct tegra_smmu *smmu)
- {
- smmu_readl(smmu, SMMU_PTB_ASID);
- }
- static int tegra_smmu_alloc_asid(struct tegra_smmu *smmu, unsigned int *idp)
- {
- unsigned long id;
- mutex_lock(&smmu->lock);
- id = find_first_zero_bit(smmu->asids, smmu->soc->num_asids);
- if (id >= smmu->soc->num_asids) {
- mutex_unlock(&smmu->lock);
- return -ENOSPC;
- }
- set_bit(id, smmu->asids);
- *idp = id;
- mutex_unlock(&smmu->lock);
- return 0;
- }
- static void tegra_smmu_free_asid(struct tegra_smmu *smmu, unsigned int id)
- {
- mutex_lock(&smmu->lock);
- clear_bit(id, smmu->asids);
- mutex_unlock(&smmu->lock);
- }
- static bool tegra_smmu_capable(enum iommu_cap cap)
- {
- return false;
- }
- static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type)
- {
- struct tegra_smmu_as *as;
- if (type != IOMMU_DOMAIN_UNMANAGED)
- return NULL;
- as = kzalloc(sizeof(*as), GFP_KERNEL);
- if (!as)
- return NULL;
- as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE;
- as->pd = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO);
- if (!as->pd) {
- kfree(as);
- return NULL;
- }
- as->count = kcalloc(SMMU_NUM_PDE, sizeof(u32), GFP_KERNEL);
- if (!as->count) {
- __free_page(as->pd);
- kfree(as);
- return NULL;
- }
- as->pts = kcalloc(SMMU_NUM_PDE, sizeof(*as->pts), GFP_KERNEL);
- if (!as->pts) {
- kfree(as->count);
- __free_page(as->pd);
- kfree(as);
- return NULL;
- }
- spin_lock_init(&as->lock);
- /* setup aperture */
- as->domain.geometry.aperture_start = 0;
- as->domain.geometry.aperture_end = 0xffffffff;
- as->domain.geometry.force_aperture = true;
- return &as->domain;
- }
- static void tegra_smmu_domain_free(struct iommu_domain *domain)
- {
- struct tegra_smmu_as *as = to_smmu_as(domain);
- /* TODO: free page directory and page tables */
- WARN_ON_ONCE(as->use_count);
- kfree(as->count);
- kfree(as->pts);
- kfree(as);
- }
- static const struct tegra_smmu_swgroup *
- tegra_smmu_find_swgroup(struct tegra_smmu *smmu, unsigned int swgroup)
- {
- const struct tegra_smmu_swgroup *group = NULL;
- unsigned int i;
- for (i = 0; i < smmu->soc->num_swgroups; i++) {
- if (smmu->soc->swgroups[i].swgroup == swgroup) {
- group = &smmu->soc->swgroups[i];
- break;
- }
- }
- return group;
- }
- static void tegra_smmu_enable(struct tegra_smmu *smmu, unsigned int swgroup,
- unsigned int asid)
- {
- const struct tegra_smmu_swgroup *group;
- unsigned int i;
- u32 value;
- group = tegra_smmu_find_swgroup(smmu, swgroup);
- if (group) {
- value = smmu_readl(smmu, group->reg);
- value &= ~SMMU_ASID_MASK;
- value |= SMMU_ASID_VALUE(asid);
- value |= SMMU_ASID_ENABLE;
- smmu_writel(smmu, value, group->reg);
- } else {
- pr_warn("%s group from swgroup %u not found\n", __func__,
- swgroup);
- /* No point moving ahead if group was not found */
- return;
- }
- for (i = 0; i < smmu->soc->num_clients; i++) {
- const struct tegra_mc_client *client = &smmu->soc->clients[i];
- if (client->swgroup != swgroup)
- continue;
- value = smmu_readl(smmu, client->smmu.reg);
- value |= BIT(client->smmu.bit);
- smmu_writel(smmu, value, client->smmu.reg);
- }
- }
- static void tegra_smmu_disable(struct tegra_smmu *smmu, unsigned int swgroup,
- unsigned int asid)
- {
- const struct tegra_smmu_swgroup *group;
- unsigned int i;
- u32 value;
- group = tegra_smmu_find_swgroup(smmu, swgroup);
- if (group) {
- value = smmu_readl(smmu, group->reg);
- value &= ~SMMU_ASID_MASK;
- value |= SMMU_ASID_VALUE(asid);
- value &= ~SMMU_ASID_ENABLE;
- smmu_writel(smmu, value, group->reg);
- }
- for (i = 0; i < smmu->soc->num_clients; i++) {
- const struct tegra_mc_client *client = &smmu->soc->clients[i];
- if (client->swgroup != swgroup)
- continue;
- value = smmu_readl(smmu, client->smmu.reg);
- value &= ~BIT(client->smmu.bit);
- smmu_writel(smmu, value, client->smmu.reg);
- }
- }
- static int tegra_smmu_as_prepare(struct tegra_smmu *smmu,
- struct tegra_smmu_as *as)
- {
- u32 value;
- int err;
- if (as->use_count > 0) {
- as->use_count++;
- return 0;
- }
- as->pd_dma = dma_map_page(smmu->dev, as->pd, 0, SMMU_SIZE_PD,
- DMA_TO_DEVICE);
- if (dma_mapping_error(smmu->dev, as->pd_dma))
- return -ENOMEM;
- /* We can't handle 64-bit DMA addresses */
- if (!smmu_dma_addr_valid(smmu, as->pd_dma)) {
- err = -ENOMEM;
- goto err_unmap;
- }
- err = tegra_smmu_alloc_asid(smmu, &as->id);
- if (err < 0)
- goto err_unmap;
- smmu_flush_ptc(smmu, as->pd_dma, 0);
- smmu_flush_tlb_asid(smmu, as->id);
- smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID);
- value = SMMU_PTB_DATA_VALUE(as->pd_dma, as->attr);
- smmu_writel(smmu, value, SMMU_PTB_DATA);
- smmu_flush(smmu);
- as->smmu = smmu;
- as->use_count++;
- return 0;
- err_unmap:
- dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);
- return err;
- }
- static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu,
- struct tegra_smmu_as *as)
- {
- if (--as->use_count > 0)
- return;
- tegra_smmu_free_asid(smmu, as->id);
- dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);
- as->smmu = NULL;
- }
- static int tegra_smmu_attach_dev(struct iommu_domain *domain,
- struct device *dev)
- {
- struct tegra_smmu *smmu = dev_iommu_priv_get(dev);
- struct tegra_smmu_as *as = to_smmu_as(domain);
- struct device_node *np = dev->of_node;
- struct of_phandle_args args;
- unsigned int index = 0;
- int err = 0;
- while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index,
- &args)) {
- unsigned int swgroup = args.args[0];
- if (args.np != smmu->dev->of_node) {
- of_node_put(args.np);
- continue;
- }
- of_node_put(args.np);
- err = tegra_smmu_as_prepare(smmu, as);
- if (err < 0)
- return err;
- tegra_smmu_enable(smmu, swgroup, as->id);
- index++;
- }
- if (index == 0)
- return -ENODEV;
- return 0;
- }
- static void tegra_smmu_detach_dev(struct iommu_domain *domain, struct device *dev)
- {
- struct tegra_smmu_as *as = to_smmu_as(domain);
- struct device_node *np = dev->of_node;
- struct tegra_smmu *smmu = as->smmu;
- struct of_phandle_args args;
- unsigned int index = 0;
- while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index,
- &args)) {
- unsigned int swgroup = args.args[0];
- if (args.np != smmu->dev->of_node) {
- of_node_put(args.np);
- continue;
- }
- of_node_put(args.np);
- tegra_smmu_disable(smmu, swgroup, as->id);
- tegra_smmu_as_unprepare(smmu, as);
- index++;
- }
- }
- static void tegra_smmu_set_pde(struct tegra_smmu_as *as, unsigned long iova,
- u32 value)
- {
- unsigned int pd_index = iova_pd_index(iova);
- struct tegra_smmu *smmu = as->smmu;
- u32 *pd = page_address(as->pd);
- unsigned long offset = pd_index * sizeof(*pd);
- /* Set the page directory entry first */
- pd[pd_index] = value;
- /* The flush the page directory entry from caches */
- dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset,
- sizeof(*pd), DMA_TO_DEVICE);
- /* And flush the iommu */
- smmu_flush_ptc(smmu, as->pd_dma, offset);
- smmu_flush_tlb_section(smmu, as->id, iova);
- smmu_flush(smmu);
- }
- static u32 *tegra_smmu_pte_offset(struct page *pt_page, unsigned long iova)
- {
- u32 *pt = page_address(pt_page);
- return pt + iova_pt_index(iova);
- }
- static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova,
- dma_addr_t *dmap)
- {
- unsigned int pd_index = iova_pd_index(iova);
- struct tegra_smmu *smmu = as->smmu;
- struct page *pt_page;
- u32 *pd;
- pt_page = as->pts[pd_index];
- if (!pt_page)
- return NULL;
- pd = page_address(as->pd);
- *dmap = smmu_pde_to_dma(smmu, pd[pd_index]);
- return tegra_smmu_pte_offset(pt_page, iova);
- }
- static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova,
- dma_addr_t *dmap, struct page *page)
- {
- unsigned int pde = iova_pd_index(iova);
- struct tegra_smmu *smmu = as->smmu;
- if (!as->pts[pde]) {
- dma_addr_t dma;
- dma = dma_map_page(smmu->dev, page, 0, SMMU_SIZE_PT,
- DMA_TO_DEVICE);
- if (dma_mapping_error(smmu->dev, dma)) {
- __free_page(page);
- return NULL;
- }
- if (!smmu_dma_addr_valid(smmu, dma)) {
- dma_unmap_page(smmu->dev, dma, SMMU_SIZE_PT,
- DMA_TO_DEVICE);
- __free_page(page);
- return NULL;
- }
- as->pts[pde] = page;
- tegra_smmu_set_pde(as, iova, SMMU_MK_PDE(dma, SMMU_PDE_ATTR |
- SMMU_PDE_NEXT));
- *dmap = dma;
- } else {
- u32 *pd = page_address(as->pd);
- *dmap = smmu_pde_to_dma(smmu, pd[pde]);
- }
- return tegra_smmu_pte_offset(as->pts[pde], iova);
- }
- static void tegra_smmu_pte_get_use(struct tegra_smmu_as *as, unsigned long iova)
- {
- unsigned int pd_index = iova_pd_index(iova);
- as->count[pd_index]++;
- }
- static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova)
- {
- unsigned int pde = iova_pd_index(iova);
- struct page *page = as->pts[pde];
- /*
- * When no entries in this page table are used anymore, return the
- * memory page to the system.
- */
- if (--as->count[pde] == 0) {
- struct tegra_smmu *smmu = as->smmu;
- u32 *pd = page_address(as->pd);
- dma_addr_t pte_dma = smmu_pde_to_dma(smmu, pd[pde]);
- tegra_smmu_set_pde(as, iova, 0);
- dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE);
- __free_page(page);
- as->pts[pde] = NULL;
- }
- }
- static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova,
- u32 *pte, dma_addr_t pte_dma, u32 val)
- {
- struct tegra_smmu *smmu = as->smmu;
- unsigned long offset = SMMU_OFFSET_IN_PAGE(pte);
- *pte = val;
- dma_sync_single_range_for_device(smmu->dev, pte_dma, offset,
- 4, DMA_TO_DEVICE);
- smmu_flush_ptc(smmu, pte_dma, offset);
- smmu_flush_tlb_group(smmu, as->id, iova);
- smmu_flush(smmu);
- }
- static struct page *as_get_pde_page(struct tegra_smmu_as *as,
- unsigned long iova, gfp_t gfp,
- unsigned long *flags)
- {
- unsigned int pde = iova_pd_index(iova);
- struct page *page = as->pts[pde];
- /* at first check whether allocation needs to be done at all */
- if (page)
- return page;
- /*
- * In order to prevent exhaustion of the atomic memory pool, we
- * allocate page in a sleeping context if GFP flags permit. Hence
- * spinlock needs to be unlocked and re-locked after allocation.
- */
- if (!(gfp & __GFP_ATOMIC))
- spin_unlock_irqrestore(&as->lock, *flags);
- page = alloc_page(gfp | __GFP_DMA | __GFP_ZERO);
- if (!(gfp & __GFP_ATOMIC))
- spin_lock_irqsave(&as->lock, *flags);
- /*
- * In a case of blocking allocation, a concurrent mapping may win
- * the PDE allocation. In this case the allocated page isn't needed
- * if allocation succeeded and the allocation failure isn't fatal.
- */
- if (as->pts[pde]) {
- if (page)
- __free_page(page);
- page = as->pts[pde];
- }
- return page;
- }
- static int
- __tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
- phys_addr_t paddr, size_t size, int prot, gfp_t gfp,
- unsigned long *flags)
- {
- struct tegra_smmu_as *as = to_smmu_as(domain);
- dma_addr_t pte_dma;
- struct page *page;
- u32 pte_attrs;
- u32 *pte;
- page = as_get_pde_page(as, iova, gfp, flags);
- if (!page)
- return -ENOMEM;
- pte = as_get_pte(as, iova, &pte_dma, page);
- if (!pte)
- return -ENOMEM;
- /* If we aren't overwriting a pre-existing entry, increment use */
- if (*pte == 0)
- tegra_smmu_pte_get_use(as, iova);
- pte_attrs = SMMU_PTE_NONSECURE;
- if (prot & IOMMU_READ)
- pte_attrs |= SMMU_PTE_READABLE;
- if (prot & IOMMU_WRITE)
- pte_attrs |= SMMU_PTE_WRITABLE;
- tegra_smmu_set_pte(as, iova, pte, pte_dma,
- SMMU_PHYS_PFN(paddr) | pte_attrs);
- return 0;
- }
- static size_t
- __tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
- size_t size, struct iommu_iotlb_gather *gather)
- {
- struct tegra_smmu_as *as = to_smmu_as(domain);
- dma_addr_t pte_dma;
- u32 *pte;
- pte = tegra_smmu_pte_lookup(as, iova, &pte_dma);
- if (!pte || !*pte)
- return 0;
- tegra_smmu_set_pte(as, iova, pte, pte_dma, 0);
- tegra_smmu_pte_put_use(as, iova);
- return size;
- }
- static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
- phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
- {
- struct tegra_smmu_as *as = to_smmu_as(domain);
- unsigned long flags;
- int ret;
- spin_lock_irqsave(&as->lock, flags);
- ret = __tegra_smmu_map(domain, iova, paddr, size, prot, gfp, &flags);
- spin_unlock_irqrestore(&as->lock, flags);
- return ret;
- }
- static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
- size_t size, struct iommu_iotlb_gather *gather)
- {
- struct tegra_smmu_as *as = to_smmu_as(domain);
- unsigned long flags;
- spin_lock_irqsave(&as->lock, flags);
- size = __tegra_smmu_unmap(domain, iova, size, gather);
- spin_unlock_irqrestore(&as->lock, flags);
- return size;
- }
- static phys_addr_t tegra_smmu_iova_to_phys(struct iommu_domain *domain,
- dma_addr_t iova)
- {
- struct tegra_smmu_as *as = to_smmu_as(domain);
- unsigned long pfn;
- dma_addr_t pte_dma;
- u32 *pte;
- pte = tegra_smmu_pte_lookup(as, iova, &pte_dma);
- if (!pte || !*pte)
- return 0;
- pfn = *pte & as->smmu->pfn_mask;
- return SMMU_PFN_PHYS(pfn) + SMMU_OFFSET_IN_PAGE(iova);
- }
- static struct tegra_smmu *tegra_smmu_find(struct device_node *np)
- {
- struct platform_device *pdev;
- struct tegra_mc *mc;
- pdev = of_find_device_by_node(np);
- if (!pdev)
- return NULL;
- mc = platform_get_drvdata(pdev);
- if (!mc)
- return NULL;
- return mc->smmu;
- }
- static int tegra_smmu_configure(struct tegra_smmu *smmu, struct device *dev,
- struct of_phandle_args *args)
- {
- const struct iommu_ops *ops = smmu->iommu.ops;
- int err;
- err = iommu_fwspec_init(dev, &dev->of_node->fwnode, ops);
- if (err < 0) {
- dev_err(dev, "failed to initialize fwspec: %d\n", err);
- return err;
- }
- err = ops->of_xlate(dev, args);
- if (err < 0) {
- dev_err(dev, "failed to parse SW group ID: %d\n", err);
- iommu_fwspec_free(dev);
- return err;
- }
- return 0;
- }
- static struct iommu_device *tegra_smmu_probe_device(struct device *dev)
- {
- struct device_node *np = dev->of_node;
- struct tegra_smmu *smmu = NULL;
- struct of_phandle_args args;
- unsigned int index = 0;
- int err;
- while (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index,
- &args) == 0) {
- smmu = tegra_smmu_find(args.np);
- if (smmu) {
- err = tegra_smmu_configure(smmu, dev, &args);
- of_node_put(args.np);
- if (err < 0)
- return ERR_PTR(err);
- /*
- * Only a single IOMMU master interface is currently
- * supported by the Linux kernel, so abort after the
- * first match.
- */
- dev_iommu_priv_set(dev, smmu);
- break;
- }
- of_node_put(args.np);
- index++;
- }
- if (!smmu)
- return ERR_PTR(-ENODEV);
- return &smmu->iommu;
- }
- static void tegra_smmu_release_device(struct device *dev)
- {
- dev_iommu_priv_set(dev, NULL);
- }
- static const struct tegra_smmu_group_soc *
- tegra_smmu_find_group(struct tegra_smmu *smmu, unsigned int swgroup)
- {
- unsigned int i, j;
- for (i = 0; i < smmu->soc->num_groups; i++)
- for (j = 0; j < smmu->soc->groups[i].num_swgroups; j++)
- if (smmu->soc->groups[i].swgroups[j] == swgroup)
- return &smmu->soc->groups[i];
- return NULL;
- }
- static void tegra_smmu_group_release(void *iommu_data)
- {
- struct tegra_smmu_group *group = iommu_data;
- struct tegra_smmu *smmu = group->smmu;
- mutex_lock(&smmu->lock);
- list_del(&group->list);
- mutex_unlock(&smmu->lock);
- }
- static struct iommu_group *tegra_smmu_group_get(struct tegra_smmu *smmu,
- unsigned int swgroup)
- {
- const struct tegra_smmu_group_soc *soc;
- struct tegra_smmu_group *group;
- struct iommu_group *grp;
- /* Find group_soc associating with swgroup */
- soc = tegra_smmu_find_group(smmu, swgroup);
- mutex_lock(&smmu->lock);
- /* Find existing iommu_group associating with swgroup or group_soc */
- list_for_each_entry(group, &smmu->groups, list)
- if ((group->swgroup == swgroup) || (soc && group->soc == soc)) {
- grp = iommu_group_ref_get(group->group);
- mutex_unlock(&smmu->lock);
- return grp;
- }
- group = devm_kzalloc(smmu->dev, sizeof(*group), GFP_KERNEL);
- if (!group) {
- mutex_unlock(&smmu->lock);
- return NULL;
- }
- INIT_LIST_HEAD(&group->list);
- group->swgroup = swgroup;
- group->smmu = smmu;
- group->soc = soc;
- group->group = iommu_group_alloc();
- if (IS_ERR(group->group)) {
- devm_kfree(smmu->dev, group);
- mutex_unlock(&smmu->lock);
- return NULL;
- }
- iommu_group_set_iommudata(group->group, group, tegra_smmu_group_release);
- if (soc)
- iommu_group_set_name(group->group, soc->name);
- list_add_tail(&group->list, &smmu->groups);
- mutex_unlock(&smmu->lock);
- return group->group;
- }
- static struct iommu_group *tegra_smmu_device_group(struct device *dev)
- {
- struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
- struct tegra_smmu *smmu = dev_iommu_priv_get(dev);
- struct iommu_group *group;
- group = tegra_smmu_group_get(smmu, fwspec->ids[0]);
- if (!group)
- group = generic_device_group(dev);
- return group;
- }
- static int tegra_smmu_of_xlate(struct device *dev,
- struct of_phandle_args *args)
- {
- u32 id = args->args[0];
- return iommu_fwspec_add_ids(dev, &id, 1);
- }
- static const struct iommu_ops tegra_smmu_ops = {
- .capable = tegra_smmu_capable,
- .domain_alloc = tegra_smmu_domain_alloc,
- .domain_free = tegra_smmu_domain_free,
- .attach_dev = tegra_smmu_attach_dev,
- .detach_dev = tegra_smmu_detach_dev,
- .probe_device = tegra_smmu_probe_device,
- .release_device = tegra_smmu_release_device,
- .device_group = tegra_smmu_device_group,
- .map = tegra_smmu_map,
- .unmap = tegra_smmu_unmap,
- .iova_to_phys = tegra_smmu_iova_to_phys,
- .of_xlate = tegra_smmu_of_xlate,
- .pgsize_bitmap = SZ_4K,
- };
- static void tegra_smmu_ahb_enable(void)
- {
- static const struct of_device_id ahb_match[] = {
- { .compatible = "nvidia,tegra30-ahb", },
- { }
- };
- struct device_node *ahb;
- ahb = of_find_matching_node(NULL, ahb_match);
- if (ahb) {
- tegra_ahb_enable_smmu(ahb);
- of_node_put(ahb);
- }
- }
- static int tegra_smmu_swgroups_show(struct seq_file *s, void *data)
- {
- struct tegra_smmu *smmu = s->private;
- unsigned int i;
- u32 value;
- seq_printf(s, "swgroup enabled ASID\n");
- seq_printf(s, "------------------------\n");
- for (i = 0; i < smmu->soc->num_swgroups; i++) {
- const struct tegra_smmu_swgroup *group = &smmu->soc->swgroups[i];
- const char *status;
- unsigned int asid;
- value = smmu_readl(smmu, group->reg);
- if (value & SMMU_ASID_ENABLE)
- status = "yes";
- else
- status = "no";
- asid = value & SMMU_ASID_MASK;
- seq_printf(s, "%-9s %-7s %#04x\n", group->name, status,
- asid);
- }
- return 0;
- }
- DEFINE_SHOW_ATTRIBUTE(tegra_smmu_swgroups);
- static int tegra_smmu_clients_show(struct seq_file *s, void *data)
- {
- struct tegra_smmu *smmu = s->private;
- unsigned int i;
- u32 value;
- seq_printf(s, "client enabled\n");
- seq_printf(s, "--------------------\n");
- for (i = 0; i < smmu->soc->num_clients; i++) {
- const struct tegra_mc_client *client = &smmu->soc->clients[i];
- const char *status;
- value = smmu_readl(smmu, client->smmu.reg);
- if (value & BIT(client->smmu.bit))
- status = "yes";
- else
- status = "no";
- seq_printf(s, "%-12s %s\n", client->name, status);
- }
- return 0;
- }
- DEFINE_SHOW_ATTRIBUTE(tegra_smmu_clients);
- static void tegra_smmu_debugfs_init(struct tegra_smmu *smmu)
- {
- smmu->debugfs = debugfs_create_dir("smmu", NULL);
- if (!smmu->debugfs)
- return;
- debugfs_create_file("swgroups", S_IRUGO, smmu->debugfs, smmu,
- &tegra_smmu_swgroups_fops);
- debugfs_create_file("clients", S_IRUGO, smmu->debugfs, smmu,
- &tegra_smmu_clients_fops);
- }
- static void tegra_smmu_debugfs_exit(struct tegra_smmu *smmu)
- {
- debugfs_remove_recursive(smmu->debugfs);
- }
- struct tegra_smmu *tegra_smmu_probe(struct device *dev,
- const struct tegra_smmu_soc *soc,
- struct tegra_mc *mc)
- {
- struct tegra_smmu *smmu;
- size_t size;
- u32 value;
- int err;
- smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
- if (!smmu)
- return ERR_PTR(-ENOMEM);
- /*
- * This is a bit of a hack. Ideally we'd want to simply return this
- * value. However the IOMMU registration process will attempt to add
- * all devices to the IOMMU when bus_set_iommu() is called. In order
- * not to rely on global variables to track the IOMMU instance, we
- * set it here so that it can be looked up from the .probe_device()
- * callback via the IOMMU device's .drvdata field.
- */
- mc->smmu = smmu;
- size = BITS_TO_LONGS(soc->num_asids) * sizeof(long);
- smmu->asids = devm_kzalloc(dev, size, GFP_KERNEL);
- if (!smmu->asids)
- return ERR_PTR(-ENOMEM);
- INIT_LIST_HEAD(&smmu->groups);
- mutex_init(&smmu->lock);
- smmu->regs = mc->regs;
- smmu->soc = soc;
- smmu->dev = dev;
- smmu->mc = mc;
- smmu->pfn_mask =
- BIT_MASK(mc->soc->num_address_bits - SMMU_PTE_SHIFT) - 1;
- dev_dbg(dev, "address bits: %u, PFN mask: %#lx\n",
- mc->soc->num_address_bits, smmu->pfn_mask);
- smmu->tlb_mask = (1 << fls(smmu->soc->num_tlb_lines)) - 1;
- dev_dbg(dev, "TLB lines: %u, mask: %#lx\n", smmu->soc->num_tlb_lines,
- smmu->tlb_mask);
- value = SMMU_PTC_CONFIG_ENABLE | SMMU_PTC_CONFIG_INDEX_MAP(0x3f);
- if (soc->supports_request_limit)
- value |= SMMU_PTC_CONFIG_REQ_LIMIT(8);
- smmu_writel(smmu, value, SMMU_PTC_CONFIG);
- value = SMMU_TLB_CONFIG_HIT_UNDER_MISS |
- SMMU_TLB_CONFIG_ACTIVE_LINES(smmu);
- if (soc->supports_round_robin_arbitration)
- value |= SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION;
- smmu_writel(smmu, value, SMMU_TLB_CONFIG);
- smmu_flush_ptc_all(smmu);
- smmu_flush_tlb(smmu);
- smmu_writel(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG);
- smmu_flush(smmu);
- tegra_smmu_ahb_enable();
- err = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, dev_name(dev));
- if (err)
- return ERR_PTR(err);
- iommu_device_set_ops(&smmu->iommu, &tegra_smmu_ops);
- iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
- err = iommu_device_register(&smmu->iommu);
- if (err) {
- iommu_device_sysfs_remove(&smmu->iommu);
- return ERR_PTR(err);
- }
- err = bus_set_iommu(&platform_bus_type, &tegra_smmu_ops);
- if (err < 0) {
- iommu_device_unregister(&smmu->iommu);
- iommu_device_sysfs_remove(&smmu->iommu);
- return ERR_PTR(err);
- }
- if (IS_ENABLED(CONFIG_DEBUG_FS))
- tegra_smmu_debugfs_init(smmu);
- return smmu;
- }
- void tegra_smmu_remove(struct tegra_smmu *smmu)
- {
- iommu_device_unregister(&smmu->iommu);
- iommu_device_sysfs_remove(&smmu->iommu);
- if (IS_ENABLED(CONFIG_DEBUG_FS))
- tegra_smmu_debugfs_exit(smmu);
- }
|