diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index e72307fd005f..257dad949df1 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -27,6 +27,8 @@ #include #include +#include + #include "io-pgtable.h" #define ARM_LPAE_MAX_ADDR_BITS 48 @@ -293,7 +295,7 @@ static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, if (!pages) return NULL; - if (dev) { + if (!selftest_running) { dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE); if (dma_mapping_error(dev, dma)) goto out_free; @@ -321,24 +323,22 @@ static void __arm_lpae_free_pages(void *pages, size_t size, { struct device *dev = cfg->iommu_dev; - if (dev) + if (!selftest_running) dma_unmap_single(dev, __arm_lpae_dma_addr(dev, pages), size, DMA_TO_DEVICE); io_pgtable_free_pages_exact(cfg, cookie, pages, size); } static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte, - struct io_pgtable_cfg *cfg, void *cookie) + struct io_pgtable_cfg *cfg) { struct device *dev = cfg->iommu_dev; *ptep = pte; - if (dev) + if (!selftest_running) dma_sync_single_for_device(dev, __arm_lpae_dma_addr(dev, ptep), sizeof(pte), DMA_TO_DEVICE); - else if (cfg->tlb->flush_pgtable) - cfg->tlb->flush_pgtable(ptep, sizeof(pte), cookie); } static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, @@ -370,7 +370,7 @@ static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, *ptep = pte; if (flush) - __arm_lpae_set_pte(ptep, pte, cfg, data->iop.cookie); + __arm_lpae_set_pte(ptep, pte, cfg); if (prev_ptep) iopte_tblcnt_add(prev_ptep, 1); @@ -411,10 +411,10 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, if (lvl == MAP_STATE_LVL) { if (ms->pgtable) - data->iop.cfg.tlb->flush_pgtable( - ms->pte_start, + dma_sync_single_for_device(cfg->iommu_dev, + __arm_lpae_dma_addr(ms->pte_start), ms->num_pte * sizeof(*ptep), - data->iop.cookie); + DMA_TO_DEVICE); ms->iova_end = round_down(iova, SZ_2M) + SZ_2M; ms->pgtable = pgtable; @@ -429,10 +429,10 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, * mapping. Flush out the previous page mappings. */ if (ms->pgtable) - data->iop.cfg.tlb->flush_pgtable( - ms->pte_start, + dma_sync_single_for_device(cfg->iommu_dev, + __arm_lpae_dma_addr(ms->pte_start), ms->num_pte * sizeof(*ptep), - data->iop.cookie); + DMA_TO_DEVICE); memset(ms, 0, sizeof(*ms)); ms = NULL; } @@ -456,7 +456,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, pte = __pa(cptep) | ARM_LPAE_PTE_TYPE_TABLE; if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) pte |= ARM_LPAE_PTE_NSTABLE; - __arm_lpae_set_pte(ptep, pte, cfg, cookie); + __arm_lpae_set_pte(ptep, pte, cfg); } else { cptep = iopte_deref(pte, data); } @@ -515,7 +515,7 @@ static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova, { struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); arm_lpae_iopte *ptep = data->pgd; - int lvl = ARM_LPAE_START_LVL(data); + int ret, lvl = ARM_LPAE_START_LVL(data); arm_lpae_iopte prot; /* If no access, then nothing to do */ @@ -523,8 +523,15 @@ static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova, return 0; prot = arm_lpae_prot_to_pte(data, iommu_prot); - return __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep, NULL, - NULL); + ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep, NULL, + NULL); + /* + * Synchronise all PTE updates for the new mapping before there's + * a chance for anything to kick off a table walk for the new iova. + */ + wmb(); + + return ret; } static int arm_lpae_map_sg(struct io_pgtable_ops *ops, unsigned long iova, @@ -532,6 +539,7 @@ static int arm_lpae_map_sg(struct io_pgtable_ops *ops, unsigned long iova, int iommu_prot, size_t *size) { struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); + struct io_pgtable_cfg *cfg = &data->iop.cfg; arm_lpae_iopte *ptep = data->pgd; int lvl = ARM_LPAE_START_LVL(data); arm_lpae_iopte prot; @@ -591,9 +599,15 @@ static int arm_lpae_map_sg(struct io_pgtable_ops *ops, unsigned long iova, } if (ms.pgtable) - data->iop.cfg.tlb->flush_pgtable( - ms.pte_start, ms.num_pte * sizeof(*ms.pte_start), - data->iop.cookie); + dma_sync_single_for_device(cfg->iommu_dev, + __arm_lpae_dma_addr(ms.pte_start), + ms.num_pte * sizeof(*ms.pte_start), + DMA_TO_DEVICE); + /* + * Synchronise all PTE updates for the new mapping before there's + * a chance for anything to kick off a table walk for the new iova. + */ + wmb(); return mapped; @@ -653,7 +667,6 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, phys_addr_t blk_paddr; arm_lpae_iopte table = 0; struct io_pgtable_cfg *cfg = &data->iop.cfg; - void *cookie = data->iop.cookie; blk_start = iova & ~(blk_size - 1); blk_end = blk_start + blk_size; @@ -680,7 +693,7 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, } } - __arm_lpae_set_pte(ptep, table, cfg, cookie); + __arm_lpae_set_pte(ptep, table, cfg); return size; } @@ -689,7 +702,7 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, arm_lpae_iopte *ptep, arm_lpae_iopte *prev_ptep) { arm_lpae_iopte pte; - const struct iommu_gather_ops *tlb = data->iop.cfg.tlb; + struct io_pgtable_cfg *cfg = &data->iop.cfg; void *cookie = data->iop.cookie; size_t blk_size = ARM_LPAE_BLOCK_SIZE(lvl, data); @@ -702,7 +715,7 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, /* If the size matches this level, we're in the right place */ if (size == blk_size) { - __arm_lpae_set_pte(ptep, 0, &data->iop.cfg, cookie); + __arm_lpae_set_pte(ptep, 0, &data->iop.cfg); if (!iopte_leaf(pte, lvl)) { /* Also flush any partial walks */ @@ -731,12 +744,14 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, table += tl_offset; memset(table, 0, table_len); - tlb->flush_pgtable(table, table_len, cookie); + dma_sync_single_for_device(cfg->iommu_dev, + __arm_lpae_dma_addr(table), + table_len, DMA_TO_DEVICE); iopte_tblcnt_sub(ptep, entries); if (!iopte_tblcnt(*ptep)) { /* no valid mappings left under this table. free it. */ - __arm_lpae_set_pte(ptep, 0, cfg, data->iop.cookie); + __arm_lpae_set_pte(ptep, 0, cfg); io_pgtable_free_pages_exact( &data->iop.cfg, cookie, table_base, max_entries * sizeof(*table_base)); @@ -972,8 +987,8 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) if (!data->pgd) goto out_free_data; - if (cfg->tlb->flush_pgtable) - cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie); + /* Ensure the empty pgd is visible before any actual TTBR write */ + wmb(); /* TTBRs */ cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd); @@ -1061,8 +1076,8 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) if (!data->pgd) goto out_free_data; - if (cfg->tlb->flush_pgtable) - cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie); + /* Ensure the empty pgd is visible before any actual TTBR write */ + wmb(); /* VTTBR */ cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd); diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h index c8fbc7e0d4c8..4501f6e68363 100644 --- a/drivers/iommu/io-pgtable.h +++ b/drivers/iommu/io-pgtable.h @@ -19,7 +19,9 @@ enum io_pgtable_fmt { * * @tlb_flush_all: Synchronously invalidate the entire TLB context. * @tlb_add_flush: Queue up a TLB invalidation for a virtual address range. - * @tlb_sync: Ensure any queue TLB invalidation has taken effect. + * @tlb_sync: Ensure any queued TLB invalidation has taken effect, and + * any corresponding page table updates are visible to the + * IOMMU. * @flush_pgtable: Ensure page table updates are visible to the IOMMU. * @alloc_pages_exact: Allocate page table memory (optional, defaults to * alloc_pages_exact)