diff --git a/drivers/iommu/dma-mapping-fast.c b/drivers/iommu/dma-mapping-fast.c index 2acb9242bcf8..8c6364f03eac 100644 --- a/drivers/iommu/dma-mapping-fast.c +++ b/drivers/iommu/dma-mapping-fast.c @@ -25,6 +25,13 @@ #define FAST_PAGE_SIZE (1UL << FAST_PAGE_SHIFT) #define FAST_PAGE_MASK (~(PAGE_SIZE - 1)) #define FAST_PTE_ADDR_MASK ((av8l_fast_iopte)0xfffffffff000) +#define FAST_MAIR_ATTR_IDX_CACHE 1 +#define FAST_PTE_ATTRINDX_SHIFT 2 +#define FAST_PTE_ATTRINDX_MASK 0x7 +#define FAST_PTE_SH_SHIFT 8 +#define FAST_PTE_SH_MASK (((av8l_fast_iopte)0x3) << FAST_PTE_SH_SHIFT) +#define FAST_PTE_SH_OS (((av8l_fast_iopte)2) << FAST_PTE_SH_SHIFT) +#define FAST_PTE_SH_IS (((av8l_fast_iopte)3) << FAST_PTE_SH_SHIFT) static pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot, bool coherent) @@ -56,6 +63,36 @@ static void fast_dmac_clean_range(struct dma_fast_smmu_mapping *mapping, dmac_clean_range(start, end); } +static bool __fast_is_pte_coherent(av8l_fast_iopte *ptep) +{ + int attr_idx = (*ptep & (FAST_PTE_ATTRINDX_MASK << + FAST_PTE_ATTRINDX_SHIFT)) >> + FAST_PTE_ATTRINDX_SHIFT; + + if ((attr_idx == FAST_MAIR_ATTR_IDX_CACHE) && + (((*ptep & FAST_PTE_SH_MASK) == FAST_PTE_SH_IS) || + (*ptep & FAST_PTE_SH_MASK) == FAST_PTE_SH_OS)) + return true; + + return false; +} + +static bool is_dma_coherent(struct device *dev, struct dma_attrs *attrs) +{ + bool is_coherent; + + if (dma_get_attr(DMA_ATTR_FORCE_COHERENT, attrs)) + is_coherent = true; + else if (dma_get_attr(DMA_ATTR_FORCE_NON_COHERENT, attrs)) + is_coherent = false; + else if (is_device_dma_coherent(dev)) + is_coherent = true; + else + is_coherent = false; + + return is_coherent; +} + /* * Checks if the allocated range (ending at @end) covered the upcoming * stale bit. We don't need to know exactly where the range starts since @@ -315,7 +352,7 @@ static dma_addr_t fast_smmu_map_page(struct device *dev, struct page *page, int nptes = len >> FAST_PAGE_SHIFT; bool skip_sync = dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs); int prot = __fast_dma_direction_to_prot(dir); - bool is_coherent = is_device_dma_coherent(dev); + bool is_coherent = is_dma_coherent(dev, attrs); prot = __get_iommu_pgprot(attrs, prot, is_coherent); @@ -360,7 +397,7 @@ static void fast_smmu_unmap_page(struct device *dev, dma_addr_t iova, int nptes = len >> FAST_PAGE_SHIFT; struct page *page = phys_to_page((*pmd & FAST_PTE_ADDR_MASK)); bool skip_sync = dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs); - bool is_coherent = is_device_dma_coherent(dev); + bool is_coherent = is_dma_coherent(dev, attrs); if (!skip_sync && !is_coherent) __fast_dma_page_dev_to_cpu(page, offset, size, dir); @@ -381,7 +418,7 @@ static void fast_smmu_sync_single_for_cpu(struct device *dev, unsigned long offset = iova & ~FAST_PAGE_MASK; struct page *page = phys_to_page((*pmd & FAST_PTE_ADDR_MASK)); - if (!is_device_dma_coherent(dev)) + if (!__fast_is_pte_coherent(pmd)) __fast_dma_page_dev_to_cpu(page, offset, size, dir); } @@ -394,7 +431,7 @@ static void fast_smmu_sync_single_for_device(struct device *dev, unsigned long offset = iova & ~FAST_PAGE_MASK; struct page *page = phys_to_page((*pmd & FAST_PTE_ADDR_MASK)); - if (!is_device_dma_coherent(dev)) + if (!__fast_is_pte_coherent(pmd)) __fast_dma_page_cpu_to_dev(page, offset, size, dir); } @@ -472,7 +509,7 @@ static void *fast_smmu_alloc(struct device *dev, size_t size, struct sg_mapping_iter miter; unsigned int count = ALIGN(size, SZ_4K) >> PAGE_SHIFT; int prot = IOMMU_READ | IOMMU_WRITE; /* TODO: extract from attrs */ - bool is_coherent = is_device_dma_coherent(dev); + bool is_coherent = is_dma_coherent(dev, attrs); pgprot_t remap_prot = __get_dma_pgprot(attrs, PAGE_KERNEL, is_coherent); struct page **pages; @@ -596,7 +633,7 @@ static int fast_smmu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, unsigned long uaddr = vma->vm_start; struct page **pages; int i, nr_pages, ret = 0; - bool coherent = is_device_dma_coherent(dev); + bool coherent = is_dma_coherent(dev, attrs); vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot, coherent);