diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index d9a9eaf95db1..a87d4ceab106 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -23,7 +23,8 @@ config IOMMU_IO_PGTABLE config IOMMU_IO_PGTABLE_LPAE bool "ARMv7/v8 Long Descriptor Format" select IOMMU_IO_PGTABLE - depends on HAS_DMA && (ARM || ARM64 || COMPILE_TEST) + # SWIOTLB guarantees a dma_to_phys() implementation + depends on ARM || ARM64 || (COMPILE_TEST && SWIOTLB) help Enable support for the ARM long descriptor pagetable format. This allocator supports 4K/2M/1G, 16K/32M and 64K/512M page diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 77b269fa31f3..e72307fd005f 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -276,6 +276,71 @@ static void iopte_tblcnt_add(arm_lpae_iopte *table_ptep, int cnt) static bool suppress_map_failures; +static dma_addr_t __arm_lpae_dma_addr(struct device *dev, void *pages) +{ + return phys_to_dma(dev, virt_to_phys(pages)); +} + +static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, + struct io_pgtable_cfg *cfg, + void *cookie) +{ + struct device *dev = cfg->iommu_dev; + dma_addr_t dma; + void *pages = io_pgtable_alloc_pages_exact(cfg, cookie, + size, gfp | __GFP_ZERO); + + if (!pages) + return NULL; + + if (dev) { + dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma)) + goto out_free; + /* + * We depend on the IOMMU being able to work with any physical + * address directly, so if the DMA layer suggests it can't by + * giving us back some translation, that bodes very badly... + */ + if (dma != __arm_lpae_dma_addr(dev, pages)) + goto out_unmap; + } + + return pages; + +out_unmap: + dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n"); + dma_unmap_single(dev, dma, size, DMA_TO_DEVICE); +out_free: + io_pgtable_free_pages_exact(cfg, cookie, pages, size); + return NULL; +} + +static void __arm_lpae_free_pages(void *pages, size_t size, + struct io_pgtable_cfg *cfg, void *cookie) +{ + struct device *dev = cfg->iommu_dev; + + if (dev) + dma_unmap_single(dev, __arm_lpae_dma_addr(dev, pages), + size, DMA_TO_DEVICE); + io_pgtable_free_pages_exact(cfg, cookie, pages, size); +} + +static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte, + struct io_pgtable_cfg *cfg, void *cookie) +{ + struct device *dev = cfg->iommu_dev; + + *ptep = pte; + + if (dev) + dma_sync_single_for_device(dev, __arm_lpae_dma_addr(dev, ptep), + sizeof(pte), DMA_TO_DEVICE); + else if (cfg->tlb->flush_pgtable) + cfg->tlb->flush_pgtable(ptep, sizeof(pte), cookie); +} + static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, unsigned long iova, phys_addr_t paddr, arm_lpae_iopte prot, int lvl, @@ -283,6 +348,7 @@ static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, bool flush) { arm_lpae_iopte pte = prot; + struct io_pgtable_cfg *cfg = &data->iop.cfg; /* We require an unmap first */ if (*ptep & ARM_LPAE_PTE_VALID) { @@ -290,7 +356,7 @@ static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, return -EEXIST; } - if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) pte |= ARM_LPAE_PTE_NS; if (lvl == ARM_LPAE_MAX_LEVELS - 1) @@ -304,9 +370,7 @@ static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, *ptep = pte; if (flush) - data->iop.cfg.tlb->flush_pgtable(ptep, sizeof(*ptep), - data->iop.cookie); - + __arm_lpae_set_pte(ptep, pte, cfg, data->iop.cookie); if (prev_ptep) iopte_tblcnt_add(prev_ptep, 1); @@ -334,12 +398,13 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, void *cookie = data->iop.cookie; size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); arm_lpae_iopte *pgtable = ptep; + struct io_pgtable_cfg *cfg = &data->iop.cfg; /* Find our entry at the current level */ ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); /* If we can install a leaf entry at this level, then do so */ - if (size == block_size && (size & data->iop.cfg.pgsize_bitmap)) { + if (size == block_size && (size & cfg->pgsize_bitmap)) { if (!ms) return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep, prev_ptep, true); @@ -383,19 +448,15 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, /* Grab a pointer to the next level */ pte = *ptep; if (!pte) { - cptep = io_pgtable_alloc_pages_exact(&data->iop.cfg, cookie, - 1UL << data->pg_shift, - GFP_ATOMIC | __GFP_ZERO); + cptep = __arm_lpae_alloc_pages(1UL << data->pg_shift, + GFP_ATOMIC, cfg, cookie); if (!cptep) return -ENOMEM; - data->iop.cfg.tlb->flush_pgtable(cptep, 1UL << data->pg_shift, - cookie); pte = __pa(cptep) | ARM_LPAE_PTE_TYPE_TABLE; - if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) pte |= ARM_LPAE_PTE_NSTABLE; - *ptep = pte; - data->iop.cfg.tlb->flush_pgtable(ptep, sizeof(*ptep), cookie); + __arm_lpae_set_pte(ptep, pte, cfg, cookie); } else { cptep = iopte_deref(pte, data); } @@ -570,8 +631,8 @@ static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, } end: - io_pgtable_free_pages_exact(&data->iop.cfg, data->iop.cookie, - start, table_size); + __arm_lpae_free_pages(start, table_size, &data->iop.cfg, + data->iop.cookie); } static void arm_lpae_free_pgtable(struct io_pgtable *iop) @@ -591,8 +652,8 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, unsigned long blk_start, blk_end; phys_addr_t blk_paddr; arm_lpae_iopte table = 0; + struct io_pgtable_cfg *cfg = &data->iop.cfg; void *cookie = data->iop.cookie; - const struct iommu_gather_ops *tlb = data->iop.cfg.tlb; blk_start = iova & ~(blk_size - 1); blk_end = blk_start + blk_size; @@ -619,8 +680,7 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, } } - *ptep = table; - tlb->flush_pgtable(ptep, sizeof(*ptep), cookie); + __arm_lpae_set_pte(ptep, table, cfg, cookie); return size; } @@ -642,8 +702,7 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, /* If the size matches this level, we're in the right place */ if (size == blk_size) { - *ptep = 0; - tlb->flush_pgtable(ptep, sizeof(*ptep), cookie); + __arm_lpae_set_pte(ptep, 0, &data->iop.cfg, cookie); if (!iopte_leaf(pte, lvl)) { /* Also flush any partial walks */ @@ -677,8 +736,7 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, iopte_tblcnt_sub(ptep, entries); if (!iopte_tblcnt(*ptep)) { /* no valid mappings left under this table. free it. */ - *ptep = 0; - tlb->flush_pgtable(ptep, sizeof(*ptep), cookie); + __arm_lpae_set_pte(ptep, 0, cfg, data->iop.cookie); io_pgtable_free_pages_exact( &data->iop.cfg, cookie, table_base, max_entries * sizeof(*table_base)); @@ -910,12 +968,13 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) cfg->arm_lpae_s1_cfg.mair[1] = 0; /* Looking good; allocate a pgd */ - data->pgd = io_pgtable_alloc_pages_exact(cfg, cookie, data->pgd_size, - GFP_KERNEL | __GFP_ZERO); + data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg, cookie); if (!data->pgd) goto out_free_data; - cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie); + if (cfg->tlb->flush_pgtable) + cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie); + /* TTBRs */ cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd); cfg->arm_lpae_s1_cfg.ttbr[1] = 0; @@ -998,12 +1057,13 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) cfg->arm_lpae_s2_cfg.vtcr = reg; /* Allocate pgd pages */ - data->pgd = io_pgtable_alloc_pages_exact(cfg, cookie, data->pgd_size, - GFP_KERNEL | __GFP_ZERO); + data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg, cookie); if (!data->pgd) goto out_free_data; - cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie); + if (cfg->tlb->flush_pgtable) + cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie); + /* VTTBR */ cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd); return &data->iop; diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h index 5d3848aeba07..c8fbc7e0d4c8 100644 --- a/drivers/iommu/io-pgtable.h +++ b/drivers/iommu/io-pgtable.h @@ -49,6 +49,8 @@ struct iommu_gather_ops { * @ias: Input address (iova) size, in bits. * @oas: Output address (paddr) size, in bits. * @tlb: TLB management callbacks for this set of tables. + * @iommu_dev: The device representing the DMA configuration for the + * page table walker. */ struct io_pgtable_cfg { #define IO_PGTABLE_QUIRK_ARM_NS (1 << 0) /* Set NS bit in PTEs */ @@ -57,6 +59,7 @@ struct io_pgtable_cfg { unsigned int ias; unsigned int oas; const struct iommu_gather_ops *tlb; + struct device *iommu_dev; /* Low-level data specific to the table format */ union {