Merge "dma-mapping: add i/o coherency support for 32bit"

This commit is contained in:
Linux Build Service Account 2017-04-12 16:10:12 -07:00 committed by Gerrit - the friendly Code Review server
commit e56d3280a5

View file

@ -67,7 +67,8 @@ __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
static void __dma_free_remap(void *cpu_addr, size_t size, bool no_warn);
static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot);
static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot,
bool coherent);
static void *arm_dma_remap(struct device *dev, void *cpu_addr,
dma_addr_t handle, size_t size,
@ -76,6 +77,33 @@ static void *arm_dma_remap(struct device *dev, void *cpu_addr,
static void arm_dma_unremap(struct device *dev, void *remapped_addr,
size_t size);
static bool is_dma_coherent(struct device *dev, struct dma_attrs *attrs)
{
bool is_coherent;
if (dma_get_attr(DMA_ATTR_FORCE_COHERENT, attrs))
is_coherent = true;
else if (dma_get_attr(DMA_ATTR_FORCE_NON_COHERENT, attrs))
is_coherent = false;
else if (is_device_dma_coherent(dev))
is_coherent = true;
else
is_coherent = false;
return is_coherent;
}
static int __get_iommu_pgprot(struct dma_attrs *attrs, int prot,
bool coherent)
{
if (!dma_get_attr(DMA_ATTR_EXEC_MAPPING, attrs))
prot |= IOMMU_NOEXEC;
if (coherent)
prot |= IOMMU_CACHE;
return prot;
}
/**
* arm_dma_map_page - map a portion of a page for streaming DMA
* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
@ -245,7 +273,7 @@ static u64 get_coherent_dma_mask(struct device *dev)
}
static void __dma_clear_buffer(struct page *page, size_t size,
struct dma_attrs *attrs)
struct dma_attrs *attrs, bool is_coherent)
{
/*
* Ensure that the allocated pages are zeroed, and that any data
@ -258,18 +286,22 @@ static void __dma_clear_buffer(struct page *page, size_t size,
void *ptr = kmap_atomic(page);
if (!dma_get_attr(DMA_ATTR_SKIP_ZEROING, attrs))
memset(ptr, 0, PAGE_SIZE);
dmac_flush_range(ptr, ptr + PAGE_SIZE);
if (!is_coherent)
dmac_flush_range(ptr, ptr + PAGE_SIZE);
kunmap_atomic(ptr);
page++;
size -= PAGE_SIZE;
}
outer_flush_range(base, end);
if (!is_coherent)
outer_flush_range(base, end);
} else {
void *ptr = page_address(page);
if (!dma_get_attr(DMA_ATTR_SKIP_ZEROING, attrs))
memset(ptr, 0, size);
dmac_flush_range(ptr, ptr + size);
outer_flush_range(__pa(ptr), __pa(ptr) + size);
if (!is_coherent) {
dmac_flush_range(ptr, ptr + size);
outer_flush_range(__pa(ptr), __pa(ptr) + size);
}
}
}
@ -277,7 +309,8 @@ static void __dma_clear_buffer(struct page *page, size_t size,
* Allocate a DMA buffer for 'dev' of size 'size' using the
* specified gfp mask. Note that 'size' must be page aligned.
*/
static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
static struct page *__dma_alloc_buffer(struct device *dev, size_t size,
gfp_t gfp, bool coherent)
{
unsigned long order = get_order(size);
struct page *page, *p, *e;
@ -293,7 +326,7 @@ static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gf
for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++)
__free_page(p);
__dma_clear_buffer(page, size, NULL);
__dma_clear_buffer(page, size, NULL, coherent);
return page;
}
@ -527,7 +560,7 @@ static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
{
struct page *page;
void *ptr = NULL;
page = __dma_alloc_buffer(dev, size, gfp);
page = __dma_alloc_buffer(dev, size, gfp, false);
if (!page)
return NULL;
if (!want_vaddr)
@ -602,7 +635,7 @@ static void *__alloc_from_contiguous(struct device *dev, size_t size,
*/
if (!(dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs) &&
dma_get_attr(DMA_ATTR_SKIP_ZEROING, attrs)))
__dma_clear_buffer(page, size, attrs);
__dma_clear_buffer(page, size, attrs, false);
if (PageHighMem(page)) {
if (!want_vaddr) {
@ -643,15 +676,13 @@ static void __free_from_contiguous(struct device *dev, struct page *page,
dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);
}
static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot)
static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot,
bool coherent)
{
if (dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs))
prot = pgprot_writecombine(prot);
else if (dma_get_attr(DMA_ATTR_STRONGLY_ORDERED, attrs))
if (dma_get_attr(DMA_ATTR_STRONGLY_ORDERED, attrs))
prot = pgprot_stronglyordered(prot);
/* if non-consistent just pass back what was given */
else if (!dma_get_attr(DMA_ATTR_NON_CONSISTENT, attrs))
prot = pgprot_dmacoherent(prot);
else if (!coherent || dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs))
prot = pgprot_writecombine(prot);
return prot;
}
@ -673,10 +704,10 @@ static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot)
#endif /* CONFIG_MMU */
static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp,
struct page **ret_page)
struct page **ret_page, bool coherent)
{
struct page *page;
page = __dma_alloc_buffer(dev, size, gfp);
page = __dma_alloc_buffer(dev, size, gfp, coherent);
if (!page)
return NULL;
@ -724,12 +755,14 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs);
if (nommu())
addr = __alloc_simple_buffer(dev, size, gfp, &page);
addr = __alloc_simple_buffer(dev, size, gfp, &page,
is_coherent);
else if (dev_get_cma_area(dev) && (gfp & __GFP_DIRECT_RECLAIM))
addr = __alloc_from_contiguous(dev, size, prot, &page,
caller, attrs);
else if (is_coherent)
addr = __alloc_simple_buffer(dev, size, gfp, &page);
addr = __alloc_simple_buffer(dev, size, gfp, &page,
is_coherent);
else if (!gfpflags_allow_blocking(gfp))
addr = __alloc_from_pool(size, &page);
else
@ -749,7 +782,7 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
gfp_t gfp, struct dma_attrs *attrs)
{
pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, false);
return __dma_alloc(dev, size, handle, gfp, prot, false,
attrs, __builtin_return_address(0));
@ -792,8 +825,9 @@ static void *arm_dma_remap(struct device *dev, void *cpu_addr,
struct dma_attrs *attrs)
{
void *ptr;
bool is_coherent = is_dma_coherent(dev, attrs);
struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, is_coherent);
unsigned long offset = handle & ~PAGE_MASK;
size = PAGE_ALIGN(size + offset);
@ -837,7 +871,8 @@ int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
struct dma_attrs *attrs)
{
#ifdef CONFIG_MMU
vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
false);
#endif /* CONFIG_MMU */
return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
}
@ -1238,6 +1273,7 @@ static struct page **__iommu_alloc_buffer(struct device *dev, size_t size,
size_t count = size >> PAGE_SHIFT;
size_t array_size = count * sizeof(struct page *);
int i = 0;
bool is_coherent = is_dma_coherent(dev, attrs);
if (array_size <= PAGE_SIZE)
pages = kzalloc(array_size, GFP_KERNEL);
@ -1255,7 +1291,7 @@ static struct page **__iommu_alloc_buffer(struct device *dev, size_t size,
if (!page)
goto error;
__dma_clear_buffer(page, size, NULL);
__dma_clear_buffer(page, size, attrs, is_coherent);
for (i = 0; i < count; i++)
pages[i] = page + i;
@ -1299,7 +1335,8 @@ static struct page **__iommu_alloc_buffer(struct device *dev, size_t size,
pages[i + j] = pages[i] + j;
}
__dma_clear_buffer(pages[i], PAGE_SIZE << order, NULL);
__dma_clear_buffer(pages[i], PAGE_SIZE << order, attrs,
is_coherent);
i += 1 << order;
count -= 1 << order;
}
@ -1353,16 +1390,20 @@ __iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot,
* Create a mapping in device IO address space for specified pages
*/
static dma_addr_t
__iommu_create_mapping(struct device *dev, struct page **pages, size_t size)
__iommu_create_mapping(struct device *dev, struct page **pages, size_t size,
struct dma_attrs *attrs)
{
struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
dma_addr_t dma_addr, iova;
int i;
int prot = IOMMU_READ | IOMMU_WRITE;
dma_addr = __alloc_iova(mapping, size);
if (dma_addr == DMA_ERROR_CODE)
return dma_addr;
prot = __get_iommu_pgprot(attrs, prot,
is_dma_coherent(dev, attrs));
iova = dma_addr;
for (i = 0; i < count; ) {
@ -1377,8 +1418,7 @@ __iommu_create_mapping(struct device *dev, struct page **pages, size_t size)
break;
len = (j - i) << PAGE_SHIFT;
ret = iommu_map(mapping->domain, iova, phys, len,
IOMMU_READ|IOMMU_WRITE);
ret = iommu_map(mapping->domain, iova, phys, len, prot);
if (ret < 0)
goto fail;
iova += len;
@ -1435,23 +1475,52 @@ static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs)
}
static void *__iommu_alloc_atomic(struct device *dev, size_t size,
dma_addr_t *handle)
dma_addr_t *handle, gfp_t gfp,
struct dma_attrs *attrs)
{
struct page *page;
struct page **pages;
size_t count = size >> PAGE_SHIFT;
size_t array_size = count * sizeof(struct page *);
void *addr;
int i;
bool coherent = is_dma_coherent(dev, attrs);
addr = __alloc_from_pool(size, &page);
if (!addr)
if (array_size <= PAGE_SIZE)
pages = kzalloc(array_size, gfp);
else
pages = vzalloc(array_size);
if (!pages)
return NULL;
*handle = __iommu_create_mapping(dev, &page, size);
if (coherent) {
page = alloc_pages(gfp, get_order(size));
addr = page ? page_address(page) : NULL;
} else {
addr = __alloc_from_pool(size, &page);
}
if (!addr)
goto err_free;
for (i = 0; i < count ; i++)
pages[i] = page + i;
*handle = __iommu_create_mapping(dev, pages, size, attrs);
if (*handle == DMA_ERROR_CODE)
goto err_mapping;
kvfree(pages);
return addr;
err_mapping:
__free_from_pool(addr, size);
if (coherent)
__free_pages(page, get_order(size));
else
__free_from_pool(addr, size);
err_free:
kvfree(pages);
return NULL;
}
@ -1465,7 +1534,9 @@ static void __iommu_free_atomic(struct device *dev, void *cpu_addr,
static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
{
pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
bool coherent = is_dma_coherent(dev, attrs);
pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, coherent);
struct page **pages;
void *addr = NULL;
@ -1473,7 +1544,7 @@ static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
size = PAGE_ALIGN(size);
if (!gfpflags_allow_blocking(gfp))
return __iommu_alloc_atomic(dev, size, handle);
return __iommu_alloc_atomic(dev, size, handle, gfp, attrs);
/*
* Following is a work-around (a.k.a. hack) to prevent pages
@ -1488,7 +1559,7 @@ static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
if (!pages)
return NULL;
*handle = __iommu_create_mapping(dev, pages, size);
*handle = __iommu_create_mapping(dev, pages, size, attrs);
if (*handle == DMA_ERROR_CODE)
goto err_buffer;
@ -1518,8 +1589,10 @@ static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
struct page **pages = __iommu_get_pages(cpu_addr, attrs);
unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
unsigned long off = vma->vm_pgoff;
bool coherent = is_dma_coherent(dev, attrs);
vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
coherent);
if (!pages)
return -ENXIO;
@ -1750,6 +1823,8 @@ int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg,
dev_err(dev, "Couldn't allocate iova for sg %p\n", sg);
return 0;
}
prot = __get_iommu_pgprot(attrs, prot,
is_dma_coherent(dev, attrs));
ret = iommu_map_sg(mapping->domain, iova, sg, nents, prot);
if (ret != total_length) {
@ -1836,6 +1911,13 @@ void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
{
struct scatterlist *s;
int i;
struct dma_iommu_mapping *mapping = dev->archdata.mapping;
dma_addr_t iova = sg_dma_address(sg);
bool iova_coherent = iommu_is_iova_coherent(mapping->domain, iova);
if (iova_coherent)
return;
for_each_sg(sg, s, nents, i)
__dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);
@ -1854,6 +1936,12 @@ void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
{
struct scatterlist *s;
int i;
struct dma_iommu_mapping *mapping = dev->archdata.mapping;
dma_addr_t iova = sg_dma_address(sg);
bool iova_coherent = iommu_is_iova_coherent(mapping->domain, iova);
if (iova_coherent)
return;
for_each_sg(sg, s, nents, i)
__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
@ -1887,6 +1975,8 @@ static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *p
return dma_addr;
prot = __dma_direction_to_prot(dir);
prot = __get_iommu_pgprot(attrs, prot,
is_dma_coherent(dev, attrs));
ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page) +
start_offset, len, prot);
@ -1913,7 +2003,8 @@ static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,
unsigned long offset, size_t size, enum dma_data_direction dir,
struct dma_attrs *attrs)
{
if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
if (!is_dma_coherent(dev, attrs) &&
!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
__dma_page_cpu_to_dev(page, offset, size, dir);
return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs);
@ -1960,7 +2051,8 @@ static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
int offset = handle & ~PAGE_MASK;
int len = PAGE_ALIGN(size + offset);
if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
if (!(is_dma_coherent(dev, attrs) ||
dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)))
__dma_page_dev_to_cpu(page, offset, size, dir);
iommu_unmap(mapping->domain, iova, len);
@ -1974,8 +2066,10 @@ static void arm_iommu_sync_single_for_cpu(struct device *dev,
dma_addr_t iova = handle & PAGE_MASK;
struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
unsigned int offset = handle & ~PAGE_MASK;
bool iova_coherent = iommu_is_iova_coherent(mapping->domain, handle);
__dma_page_dev_to_cpu(page, offset, size, dir);
if (!iova_coherent)
__dma_page_dev_to_cpu(page, offset, size, dir);
}
static void arm_iommu_sync_single_for_device(struct device *dev,
@ -1985,8 +2079,10 @@ static void arm_iommu_sync_single_for_device(struct device *dev,
dma_addr_t iova = handle & PAGE_MASK;
struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
unsigned int offset = handle & ~PAGE_MASK;
bool iova_coherent = iommu_is_iova_coherent(mapping->domain, handle);
__dma_page_cpu_to_dev(page, offset, size, dir);
if (!iova_coherent)
__dma_page_cpu_to_dev(page, offset, size, dir);
}
const struct dma_map_ops iommu_ops = {