1377 lines
40 KiB
C
1377 lines
40 KiB
C
/*******************************************************************************
|
|
* Copyright (C) 2016-2018, Allwinner Technology CO., LTD.
|
|
* Author: fanqinghua <fanqinghua@allwinnertech.com>
|
|
*
|
|
* This file is provided under a dual BSD/GPL license. When using or
|
|
* redistributing this file, you may do so under either license.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
******************************************************************************/
|
|
/* #define DEBUG */
|
|
#include <linux/interrupt.h>
|
|
#include <linux/platform_device.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/pm_runtime.h>
|
|
#include <linux/clk.h>
|
|
#include <linux/err.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/iommu.h>
|
|
#include <linux/errno.h>
|
|
#include <linux/list.h>
|
|
#include <linux/memblock.h>
|
|
#include <linux/export.h>
|
|
#include <linux/of.h>
|
|
#include <linux/of_irq.h>
|
|
#include <linux/of_iommu.h>
|
|
#include <linux/of_platform.h>
|
|
#include <linux/module.h>
|
|
#include <linux/io.h>
|
|
#include <linux/fs.h>
|
|
#include <linux/dma-mapping.h>
|
|
#include <linux/delay.h>
|
|
#include <linux/kernel.h>
|
|
#include <asm/cacheflush.h>
|
|
#include <asm/pgtable.h>
|
|
#include <asm/dma-iommu.h>
|
|
#include <linux/uaccess.h>
|
|
#include "sunxi-iommu.h"
|
|
|
|
#define _max(x, y) (((u64)(x) > (u64)(y)) ? x : y)
|
|
|
|
static struct kmem_cache *iopte_cache;
|
|
static struct sunxi_iommu *global_iommu_dev;
|
|
struct sunxi_iommu_domain *global_sunxi_domain;
|
|
|
|
#ifdef CONFIG_ARCH_SUN50IW6
|
|
static const char *master[10] = {"DE", "VE_R", "DI", "VE", "CSI", "VP9"};
|
|
#endif
|
|
#ifdef CONFIG_ARCH_SUN50IW3
|
|
static const char *master[10] = {"DE0", "DE1", "VE_R", "VE", "CSI", "VP9"};
|
|
#endif
|
|
/**
|
|
* iopde_offset() - find out the address of page directory entry
|
|
* @iopd: the base address of Page Directory
|
|
* @iova: virtual I/O address
|
|
**/
|
|
static inline u32 *iopde_offset(u32 *iopd, unsigned int iova)
|
|
{
|
|
return iopd + IOPDE_INDEX(iova);
|
|
}
|
|
|
|
/**
|
|
* iopte_offset() - find out the address of page table entry
|
|
* @ent: the entry of page directory
|
|
* @iova: virtual I/O address
|
|
**/
|
|
static inline u32 *iopte_offset(u32 *ent, unsigned int iova)
|
|
{
|
|
return (u32 *)__va(IOPTE_BASE(*ent)) + IOPTE_INDEX(iova);
|
|
}
|
|
|
|
/* TODO: set cache flags per prot IOMMU_CACHE */
|
|
static inline u32 sunxi_mk_pte(u32 page, int prot)
|
|
{
|
|
u32 flags = 0;
|
|
|
|
flags |= (prot & IOMMU_READ) ? SUNXI_PTE_PAGE_READABLE : 0;
|
|
flags |= (prot & IOMMU_WRITE) ? SUNXI_PTE_PAGE_WRITABLE : 0;
|
|
page &= IOMMU_PT_MASK;
|
|
return page | flags | SUNXI_PTE_PAGE_VALID;
|
|
}
|
|
|
|
static inline u32 sunxi_iommu_read(struct sunxi_iommu *iommu, u32 offset)
|
|
{
|
|
return readl(iommu->base + offset);
|
|
}
|
|
|
|
static inline void sunxi_iommu_write(struct sunxi_iommu *iommu,
|
|
u32 offset, u32 value)
|
|
{
|
|
writel(value, iommu->base + offset);
|
|
}
|
|
|
|
static int sunxi_tlb_flush(struct sunxi_iommu *iommu)
|
|
{
|
|
int ret;
|
|
|
|
sunxi_iommu_write(iommu, IOMMU_TLB_FLUSH_ENABLE_REG, 0x0003003f);
|
|
ret = sunxi_wait_when(
|
|
(sunxi_iommu_read(iommu, IOMMU_TLB_FLUSH_ENABLE_REG)), 2);
|
|
if (ret)
|
|
dev_err(iommu->dev, "Enable flush all request timed out\n");
|
|
return ret;
|
|
}
|
|
|
|
static int sunxi_tlb_invalid(dma_addr_t iova, u32 mask)
|
|
{
|
|
struct sunxi_iommu *iommu = global_iommu_dev;
|
|
int ret = 0;
|
|
unsigned long mflag;
|
|
|
|
spin_lock_irqsave(&iommu->iommu_lock, mflag);
|
|
sunxi_iommu_write(iommu, IOMMU_AUTO_GATING_REG, 0x0);
|
|
sunxi_iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_REG, iova);
|
|
sunxi_iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_MASK_REG, mask);
|
|
sunxi_iommu_write(iommu, IOMMU_TLB_IVLD_ENABLE_REG, 0x1);
|
|
ret = sunxi_wait_when(
|
|
(sunxi_iommu_read(iommu, IOMMU_TLB_IVLD_ENABLE_REG)&0x1), 2);
|
|
if (ret) {
|
|
dev_err(iommu->dev, "TLB Invalid timed out\n");
|
|
goto out;
|
|
}
|
|
|
|
out:
|
|
sunxi_iommu_write(iommu, IOMMU_AUTO_GATING_REG, 0x1);
|
|
spin_unlock_irqrestore(&iommu->iommu_lock, mflag);
|
|
return ret;
|
|
}
|
|
|
|
static int sunxi_ptw_cache_invalid(dma_addr_t iova)
|
|
{
|
|
struct sunxi_iommu *iommu = global_iommu_dev;
|
|
int ret = 0;
|
|
unsigned long mflag;
|
|
|
|
spin_lock_irqsave(&iommu->iommu_lock, mflag);
|
|
sunxi_iommu_write(iommu, IOMMU_AUTO_GATING_REG, 0x0);
|
|
sunxi_iommu_write(iommu, IOMMU_PC_IVLD_ADDR_REG, iova);
|
|
sunxi_iommu_write(iommu, IOMMU_PC_IVLD_ENABLE_REG, 0x1);
|
|
ret = sunxi_wait_when(
|
|
(sunxi_iommu_read(iommu, IOMMU_PC_IVLD_ENABLE_REG)&0x1), 2);
|
|
if (ret) {
|
|
dev_err(iommu->dev, "PTW cache invalid timed out\n");
|
|
goto out;
|
|
}
|
|
|
|
out:
|
|
sunxi_iommu_write(iommu, IOMMU_AUTO_GATING_REG, 0x1);
|
|
spin_unlock_irqrestore(&iommu->iommu_lock, mflag);
|
|
return ret;
|
|
}
|
|
|
|
void sunxi_enable_device_iommu(unsigned int mastor_id, bool flag)
|
|
{
|
|
struct sunxi_iommu *iommu = global_iommu_dev;
|
|
unsigned long mflag;
|
|
|
|
BUG_ON(mastor_id > 5);
|
|
spin_lock_irqsave(&iommu->iommu_lock, mflag);
|
|
if (flag)
|
|
iommu->bypass &= ~(mastor_id_bitmap[mastor_id]);
|
|
else
|
|
iommu->bypass |= mastor_id_bitmap[mastor_id];
|
|
sunxi_iommu_write(iommu, IOMMU_BYPASS_REG, iommu->bypass);
|
|
spin_unlock_irqrestore(&iommu->iommu_lock, mflag);
|
|
}
|
|
EXPORT_SYMBOL(sunxi_enable_device_iommu);
|
|
|
|
static int sunxi_tlb_init(struct sunxi_iommu_domain *sunxi_domain)
|
|
{
|
|
struct sunxi_iommu *iommu = global_iommu_dev;
|
|
phys_addr_t dte_addr;
|
|
unsigned long mflag;
|
|
int ret = 0;
|
|
|
|
/* iommu init */
|
|
spin_lock_irqsave(&iommu->iommu_lock, mflag);
|
|
dte_addr = __pa(sunxi_domain->pgtable);
|
|
sunxi_iommu_write(iommu, IOMMU_TTB_REG, dte_addr);
|
|
sunxi_iommu_write(iommu, IOMMU_TLB_PREFETCH_REG, 0x3f);
|
|
sunxi_iommu_write(iommu, IOMMU_INT_ENABLE_REG, 0xffffffff);
|
|
sunxi_iommu_write(iommu, IOMMU_BYPASS_REG, iommu->bypass);
|
|
ret = sunxi_tlb_flush(iommu);
|
|
if (ret) {
|
|
dev_err(iommu->dev, "Enable flush all request timed out\n");
|
|
goto out;
|
|
}
|
|
sunxi_iommu_write(iommu, IOMMU_AUTO_GATING_REG, 0x1);
|
|
sunxi_iommu_write(iommu, IOMMU_ENABLE_REG, IOMMU_ENABLE);
|
|
out:
|
|
spin_unlock_irqrestore(&iommu->iommu_lock, mflag);
|
|
return ret;
|
|
|
|
}
|
|
|
|
static int sunxi_iova_invalid_helper(unsigned long iova)
|
|
{
|
|
struct sunxi_iommu_domain *sunxi_domain = global_sunxi_domain;
|
|
u32 *pte_addr, *dte_addr;
|
|
|
|
dte_addr = iopde_offset(sunxi_domain->pgtable, iova);
|
|
if ((*dte_addr & 0x3) != 0x1) {
|
|
pr_err("0x%lx is not mapped!\n", iova);
|
|
return 1;
|
|
}
|
|
pte_addr = iopte_offset(dte_addr, iova);
|
|
if ((*pte_addr & 0x2) == 0) {
|
|
pr_err("0x%lx is not mapped!\n", iova);
|
|
return 1;
|
|
}
|
|
pr_err("0x%lx is mapped!\n", iova);
|
|
return 0;
|
|
}
|
|
|
|
static irqreturn_t sunxi_iommu_irq(int irq, void *dev_id)
|
|
{
|
|
u32 inter_status_reg, addr_reg, int_masterid_bitmap,
|
|
data_reg, l1_pgint_reg, l2_pgint_reg, master_id;
|
|
struct sunxi_iommu *iommu = dev_id;
|
|
unsigned long mflag;
|
|
|
|
spin_lock_irqsave(&iommu->iommu_lock, mflag);
|
|
inter_status_reg = sunxi_iommu_read(iommu, IOMMU_INT_STA_REG) & 0x3ffff;
|
|
l1_pgint_reg = sunxi_iommu_read(iommu, IOMMU_L1PG_INT_REG);
|
|
l2_pgint_reg = sunxi_iommu_read(iommu, IOMMU_L2PG_INT_REG);
|
|
|
|
if (inter_status_reg & MICRO_TLB0_INVALID_INTER_MASK) {
|
|
addr_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG0);
|
|
data_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_DATA_REG0);
|
|
} else if (inter_status_reg & MICRO_TLB1_INVALID_INTER_MASK) {
|
|
addr_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG1);
|
|
data_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_DATA_REG1);
|
|
} else if (inter_status_reg & MICRO_TLB2_INVALID_INTER_MASK) {
|
|
addr_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG2);
|
|
data_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_DATA_REG2);
|
|
} else if (inter_status_reg & MICRO_TLB3_INVALID_INTER_MASK) {
|
|
addr_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG3);
|
|
data_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_DATA_REG3);
|
|
} else if (inter_status_reg & MICRO_TLB4_INVALID_INTER_MASK) {
|
|
addr_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG4);
|
|
data_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_DATA_REG4);
|
|
} else if (inter_status_reg & MICRO_TLB5_INVALID_INTER_MASK) {
|
|
addr_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG5);
|
|
data_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_DATA_REG5);
|
|
} else if (inter_status_reg & L1_PAGETABLE_INVALID_INTER_MASK) {
|
|
addr_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG6);
|
|
data_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_DATA_REG6);
|
|
} else if (inter_status_reg & L2_PAGETABLE_INVALID_INTER_MASK) {
|
|
addr_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG7);
|
|
data_reg = sunxi_iommu_read(iommu, IOMMU_INT_ERR_DATA_REG7);
|
|
} else {
|
|
pr_err("sunxi iommu int error!!!\n");
|
|
}
|
|
if (sunxi_iova_invalid_helper(addr_reg)) {
|
|
int_masterid_bitmap =
|
|
inter_status_reg | l1_pgint_reg | l2_pgint_reg;
|
|
int_masterid_bitmap &= 0xffff;
|
|
master_id = __ffs(int_masterid_bitmap);
|
|
BUG_ON(master_id > 5);
|
|
pr_err("%s invalid address: 0x%x, data:0x%x, id:0x%x\n",
|
|
master[master_id], addr_reg, data_reg,
|
|
int_masterid_bitmap);
|
|
}
|
|
sunxi_iommu_write(iommu, IOMMU_AUTO_GATING_REG, 0x0);
|
|
sunxi_iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_REG, addr_reg);
|
|
sunxi_iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_MASK_REG,
|
|
(u32)IOMMU_PT_MASK);
|
|
sunxi_iommu_write(iommu, IOMMU_TLB_IVLD_ENABLE_REG, 0x1);
|
|
while (sunxi_iommu_read(iommu, IOMMU_TLB_IVLD_ENABLE_REG) & 0x1)
|
|
;
|
|
sunxi_iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_REG,
|
|
addr_reg + 0x2000);
|
|
sunxi_iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_MASK_REG,
|
|
(u32)IOMMU_PT_MASK);
|
|
sunxi_iommu_write(iommu, IOMMU_TLB_IVLD_ENABLE_REG, 0x1);
|
|
while (sunxi_iommu_read(iommu, IOMMU_TLB_IVLD_ENABLE_REG) & 0x1)
|
|
;
|
|
sunxi_iommu_write(iommu, IOMMU_PC_IVLD_ADDR_REG, addr_reg);
|
|
sunxi_iommu_write(iommu, IOMMU_PC_IVLD_ENABLE_REG, 0x1);
|
|
while (sunxi_iommu_read(iommu, IOMMU_PC_IVLD_ENABLE_REG) & 0x1)
|
|
;
|
|
sunxi_iommu_write(iommu, IOMMU_PC_IVLD_ADDR_REG,
|
|
addr_reg + 0x200000);
|
|
sunxi_iommu_write(iommu, IOMMU_PC_IVLD_ENABLE_REG, 0x1);
|
|
while (sunxi_iommu_read(iommu, IOMMU_PC_IVLD_ENABLE_REG) & 0x1)
|
|
;
|
|
sunxi_iommu_write(iommu, IOMMU_AUTO_GATING_REG, 0x1);
|
|
|
|
sunxi_iommu_write(iommu, IOMMU_INT_CLR_REG, inter_status_reg);
|
|
inter_status_reg |= (l1_pgint_reg | l2_pgint_reg);
|
|
inter_status_reg &= 0xffff;
|
|
sunxi_iommu_write(iommu, IOMMU_RESET_REG, ~inter_status_reg);
|
|
sunxi_iommu_write(iommu, IOMMU_RESET_REG, 0xffffffff);
|
|
spin_unlock_irqrestore(&iommu->iommu_lock, mflag);
|
|
return IRQ_HANDLED;
|
|
}
|
|
|
|
static int sunxi_alloc_iopte(u32 *sent, int prot)
|
|
{
|
|
u32 *pent;
|
|
u32 flags = 0;
|
|
|
|
flags |= (prot & IOMMU_READ) ? DENT_READABLE : 0;
|
|
flags |= (prot & IOMMU_WRITE) ? DENT_WRITABLE : 0;
|
|
|
|
pent = kmem_cache_zalloc(iopte_cache, GFP_ATOMIC);
|
|
BUG_ON((unsigned long)pent & (PT_SIZE - 1));
|
|
if (!pent) {
|
|
pr_err("%s, %d, malloc failed!\n", __func__, __LINE__);
|
|
return 0;
|
|
}
|
|
*sent = __pa(pent) | DENT_VALID;
|
|
__dma_map_area(sent, sizeof(*sent), DMA_TO_DEVICE);
|
|
|
|
return 1;
|
|
}
|
|
|
|
static void sunxi_free_iopte(u32 *pent)
|
|
{
|
|
kmem_cache_free(iopte_cache, pent);
|
|
}
|
|
|
|
void sunxi_zap_tlb(unsigned long iova, size_t size)
|
|
{
|
|
sunxi_tlb_invalid(iova, (u32)IOMMU_PT_MASK);
|
|
sunxi_tlb_invalid(iova + SPAGE_SIZE, (u32)IOMMU_PT_MASK);
|
|
sunxi_tlb_invalid(iova + size, (u32)IOMMU_PT_MASK);
|
|
sunxi_tlb_invalid(iova + size + SPAGE_SIZE, (u32)IOMMU_PT_MASK);
|
|
sunxi_ptw_cache_invalid(iova);
|
|
sunxi_ptw_cache_invalid(iova + SPD_SIZE);
|
|
sunxi_ptw_cache_invalid(iova + size);
|
|
sunxi_ptw_cache_invalid(iova + size + SPD_SIZE);
|
|
}
|
|
|
|
static void sunxi_iommu_domain_destroy(struct iommu_domain *domain)
|
|
{
|
|
struct sunxi_iommu_domain *sdm = domain->priv;
|
|
struct sunxi_mdev *mdev, *tmp;
|
|
u32 *dent;
|
|
u32 *pent;
|
|
u32 iova;
|
|
int i;
|
|
|
|
WARN_ON(!list_empty(&sdm->mdevs));
|
|
spin_lock(&sdm->lock);
|
|
list_for_each_entry_safe(mdev, tmp, &sdm->mdevs, node) {
|
|
list_del(&mdev->node);
|
|
kfree(mdev);
|
|
}
|
|
spin_unlock(&sdm->lock);
|
|
|
|
mutex_lock(&sdm->dt_lock);
|
|
for (i = 0; i < NUM_ENTRIES_PDE; i++) {
|
|
dent = (sdm->pgtable + i);
|
|
iova = i << IOMMU_PD_SHIFT;
|
|
if (IS_VAILD(*dent)) {
|
|
pent = iopte_offset(dent, iova);
|
|
memset(pent, 0, PT_SIZE);
|
|
__dma_map_area(pent, PT_SIZE, DMA_TO_DEVICE);
|
|
sunxi_tlb_invalid(iova, (u32)IOMMU_PD_MASK);
|
|
*dent = 0;
|
|
__dma_map_area(dent, sizeof(*dent), DMA_TO_DEVICE);
|
|
sunxi_ptw_cache_invalid(iova);
|
|
sunxi_free_iopte(pent);
|
|
}
|
|
}
|
|
mutex_unlock(&sdm->dt_lock);
|
|
arm_iommu_release_mapping(sdm->mapping);
|
|
free_pages((unsigned long)sdm->pgtable, get_order(PD_SIZE));
|
|
free_pages((unsigned long)sdm->sg_buffer,
|
|
get_order(MAX_SG_TABLE_SIZE));
|
|
kfree(domain->priv);
|
|
domain->priv = NULL;
|
|
global_sunxi_domain = NULL;
|
|
}
|
|
|
|
static int sunxi_iommu_domain_init(struct iommu_domain *domain)
|
|
{
|
|
struct sunxi_iommu_domain *sunxi_domain;
|
|
|
|
sunxi_domain = kzalloc(sizeof(*sunxi_domain), GFP_KERNEL);
|
|
if (!sunxi_domain)
|
|
return -ENOMEM;
|
|
|
|
/*
|
|
* The page table size is 16K, and must be aligned 16K.
|
|
*/
|
|
sunxi_domain->pgtable = (unsigned int *)__get_free_pages(
|
|
GFP_KERNEL, get_order(PD_SIZE));
|
|
if (!sunxi_domain->pgtable)
|
|
goto err;
|
|
|
|
sunxi_domain->sg_buffer = (unsigned int *)__get_free_pages(
|
|
GFP_KERNEL, get_order(MAX_SG_TABLE_SIZE));
|
|
if (!sunxi_domain->sg_buffer)
|
|
goto err1;
|
|
|
|
memset(sunxi_domain->pgtable, 0, PD_SIZE);
|
|
mutex_init(&sunxi_domain->dt_lock);
|
|
spin_lock_init(&sunxi_domain->lock);
|
|
INIT_LIST_HEAD(&sunxi_domain->mdevs);
|
|
global_sunxi_domain = sunxi_domain;
|
|
domain->geometry.aperture_start = 0;
|
|
domain->geometry.aperture_end = (1ULL << 32) - 1;
|
|
domain->geometry.force_aperture = true;
|
|
domain->priv = sunxi_domain;
|
|
return 0;
|
|
|
|
err1:
|
|
free_pages((unsigned long)sunxi_domain->pgtable,
|
|
get_order(PD_SIZE));
|
|
err:
|
|
kfree(sunxi_domain);
|
|
return -ENOMEM;
|
|
}
|
|
|
|
/*
|
|
* Notice: the size may bigger than SPAGE, so we must consider it.
|
|
* We should process three conditions:
|
|
* 1. start pfn and end pfn in a same PDE
|
|
* 2. start pfn in the pre PDE and end pfn in the next PDE
|
|
* 3. end pfn is bigger than start pfn plus 1
|
|
*/
|
|
static int sunxi_iommu_map(struct iommu_domain *domain, unsigned long iova,
|
|
phys_addr_t paddr, size_t size, int prot)
|
|
{
|
|
struct sunxi_iommu_domain *priv = domain->priv;
|
|
u32 *dent, *pent;
|
|
size_t iova_start, iova_end, s_iova_start, paddr_start;
|
|
int i, j;
|
|
int ret = 0;
|
|
|
|
BUG_ON(priv->pgtable == NULL);
|
|
size = SPAGE_ALIGN(size);
|
|
iova_start = iova & IOMMU_PT_MASK;
|
|
paddr_start = paddr & IOMMU_PT_MASK;
|
|
iova_end = SPAGE_ALIGN(iova + size) - SPAGE_SIZE;
|
|
mutex_lock(&priv->dt_lock);
|
|
if (IOPDE_INDEX(iova_start) == IOPDE_INDEX(iova_end)) {
|
|
/*map in the same pde.*/
|
|
s_iova_start = iova_start;
|
|
dent = iopde_offset(priv->pgtable, iova_start);
|
|
if (!IS_VAILD(*dent))
|
|
sunxi_alloc_iopte(dent, prot);
|
|
for (; iova_start <= iova_end;
|
|
iova_start += SPAGE_SIZE, paddr_start += SPAGE_SIZE) {
|
|
pent = iopte_offset(dent, iova_start);
|
|
BUG_ON(*pent);
|
|
*pent = sunxi_mk_pte(paddr_start, prot);
|
|
}
|
|
__dma_map_area(iopte_offset(dent, s_iova_start),
|
|
(size / SPAGE_SIZE) << 2, DMA_TO_DEVICE);
|
|
goto out;
|
|
} else {
|
|
/*map to the bottom of this pde.*/
|
|
dent = iopde_offset(priv->pgtable, iova_start);
|
|
if (!IS_VAILD(*dent))
|
|
sunxi_alloc_iopte(dent, prot);
|
|
s_iova_start = iova_start;
|
|
for (; iova_start < SPDE_ALIGN(s_iova_start+1);
|
|
iova_start += SPAGE_SIZE, paddr_start += SPAGE_SIZE) {
|
|
pent = iopte_offset(dent, iova_start);
|
|
BUG_ON(*pent);
|
|
*pent = sunxi_mk_pte(paddr_start, prot);
|
|
}
|
|
__dma_map_area(iopte_offset(dent, s_iova_start),
|
|
((SPDE_ALIGN(s_iova_start+1) - s_iova_start) / SPAGE_SIZE) << 2,
|
|
DMA_TO_DEVICE);
|
|
}
|
|
|
|
if (IOPDE_INDEX(iova_start) < IOPDE_INDEX(iova_end)) {
|
|
/*map the number of pde*/
|
|
for (i = IOPDE_INDEX(iova_start); i < IOPDE_INDEX(iova_end);
|
|
i++, iova_start += SPD_SIZE, paddr_start += SPD_SIZE) {
|
|
dent = iopde_offset(priv->pgtable, iova_start);
|
|
if (!IS_VAILD(*dent))
|
|
sunxi_alloc_iopte(dent, prot);
|
|
pent = iopte_offset(dent, iova_start);
|
|
for (j = 0; j < NUM_ENTRIES_PTE; j++)
|
|
*(pent + j) =
|
|
sunxi_mk_pte(paddr_start + (j * SPAGE_SIZE), prot);
|
|
__dma_map_area(pent, PT_SIZE, DMA_TO_DEVICE);
|
|
}
|
|
}
|
|
|
|
/*map the last pde*/
|
|
dent = iopde_offset(priv->pgtable, iova_start);
|
|
if (!IS_VAILD(*dent))
|
|
sunxi_alloc_iopte(dent, prot);
|
|
s_iova_start = iova_start;
|
|
for (; iova_start <= iova_end;
|
|
iova_start += SPAGE_SIZE, paddr_start += SPAGE_SIZE) {
|
|
pent = iopte_offset(dent, iova_start);
|
|
*pent = sunxi_mk_pte(paddr_start, prot);
|
|
}
|
|
__dma_map_area(iopte_offset(dent, s_iova_start),
|
|
((iova_end + SPAGE_SIZE - s_iova_start) / SPAGE_SIZE) << 2,
|
|
DMA_TO_DEVICE);
|
|
|
|
out:
|
|
sunxi_zap_tlb(iova, size);
|
|
mutex_unlock(&priv->dt_lock);
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* Notice: the size may bigger than SPAGE, so we must consider it.
|
|
* We should process three conditions:
|
|
* 1. start pfn and end pfn in a same PDE
|
|
* 2. start pfn in the pre PDE and end pfn in the next PDE
|
|
* 3. end pfn is bigger than start pfn plus 1
|
|
*/
|
|
static size_t sunxi_iommu_unmap(struct iommu_domain *domain,
|
|
unsigned long iova, size_t size)
|
|
{
|
|
struct sunxi_iommu_domain *priv = domain->priv;
|
|
u32 *dent;
|
|
u32 *pent;
|
|
int i = 0;
|
|
size_t iova_start, iova_end, s_iova_start;
|
|
|
|
BUG_ON(priv->pgtable == NULL);
|
|
iova_start = iova & IOMMU_PT_MASK;
|
|
iova_end = SPAGE_ALIGN(iova + size) - SPAGE_SIZE;
|
|
mutex_lock(&priv->dt_lock);
|
|
if (IOPDE_INDEX(iova_start) == IOPDE_INDEX(iova_end)) {
|
|
/*unmap in the same pde.*/
|
|
/*need consider recycling the pte.*/
|
|
dent = iopde_offset(priv->pgtable, iova_start);
|
|
if (IS_VAILD(*dent)) {
|
|
for (; iova_start <= iova_end;
|
|
iova_start += SPAGE_SIZE) {
|
|
pent = iopte_offset(dent, iova_start);
|
|
*pent = 0;
|
|
__dma_map_area(pent, sizeof(*pent),
|
|
DMA_TO_DEVICE);
|
|
sunxi_tlb_invalid(iova_start, (u32)IOMMU_PT_MASK);
|
|
}
|
|
}
|
|
goto out;
|
|
} else {
|
|
/*unmap to the bottom of this pde.*/
|
|
dent = iopde_offset(priv->pgtable, iova_start);
|
|
if (IS_VAILD(*dent)) {
|
|
s_iova_start = iova_start;
|
|
for (; iova_start < SPDE_ALIGN(s_iova_start+1);
|
|
iova_start += SPAGE_SIZE) {
|
|
pent = iopte_offset(dent, iova_start);
|
|
*pent = 0;
|
|
__dma_map_area(pent, sizeof(*pent),
|
|
DMA_TO_DEVICE);
|
|
sunxi_tlb_invalid(iova_start, (u32)IOMMU_PT_MASK);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (IOPDE_INDEX(iova_start) < IOPDE_INDEX(iova_end)) {
|
|
/*unmap the number of pde*/
|
|
for (i = IOPDE_INDEX(iova_start); i < IOPDE_INDEX(iova_end);
|
|
i++, iova_start += SPD_SIZE) {
|
|
dent = iopde_offset(priv->pgtable, iova_start);
|
|
if (IS_VAILD(*dent)) {
|
|
pent = iopte_offset(dent, iova_start);
|
|
memset(pent, 0, PT_SIZE);
|
|
__dma_map_area(pent, PT_SIZE, DMA_TO_DEVICE);
|
|
sunxi_tlb_invalid(iova_start, (u32)IOMMU_PD_MASK);
|
|
*dent = 0;
|
|
__dma_map_area(dent, sizeof(*dent),
|
|
DMA_TO_DEVICE);
|
|
sunxi_ptw_cache_invalid(iova_start);
|
|
sunxi_free_iopte(pent);
|
|
}
|
|
}
|
|
}
|
|
|
|
/*unmap the last pde*/
|
|
/*need consider recycling the pte.*/
|
|
dent = iopde_offset(priv->pgtable, iova_start);
|
|
if (IS_VAILD(*dent)) {
|
|
for (; iova_start <= iova_end; iova_start += SPAGE_SIZE) {
|
|
pent = iopte_offset(dent, iova_start);
|
|
*pent = 0;
|
|
__dma_map_area(pent, sizeof(*pent), DMA_TO_DEVICE);
|
|
sunxi_tlb_invalid(iova_start, (u32)IOMMU_PT_MASK);
|
|
}
|
|
}
|
|
|
|
out:
|
|
sunxi_zap_tlb(iova, size);
|
|
mutex_unlock(&priv->dt_lock);
|
|
return size;
|
|
}
|
|
|
|
static size_t sunxi_convert_sg_to_table(struct scatterlist *sg, size_t size)
|
|
{
|
|
struct scatterlist *cur = sg;
|
|
u32 *p = global_sunxi_domain->sg_buffer;
|
|
int i;
|
|
size_t total_length = 0;
|
|
|
|
BUG_ON(size > MAX_SG_SIZE);
|
|
while (total_length < size) {
|
|
u32 phys = page_to_phys(sg_page(cur));
|
|
unsigned int length = PAGE_ALIGN(cur->offset + cur->length);
|
|
|
|
for (i = 0; i < length / SPAGE_SIZE; i++, p++)
|
|
*p = phys + (i * SPAGE_SIZE);
|
|
cur = sg_next(cur);
|
|
total_length += length;
|
|
}
|
|
return total_length;
|
|
}
|
|
|
|
static size_t sunxi_iommu_map_sg(struct iommu_domain *domain,
|
|
unsigned long iova, struct scatterlist *sg, size_t size, int prot)
|
|
{
|
|
struct sunxi_iommu_domain *priv = domain->priv;
|
|
u32 *dent, *pent;
|
|
size_t iova_start, iova_end, s_iova_start;
|
|
u32 *paddr_start;
|
|
int i, j;
|
|
size_t total_size = 0;
|
|
|
|
BUG_ON(priv->pgtable == NULL);
|
|
size = SPAGE_ALIGN(size);
|
|
iova_start = iova & IOMMU_PT_MASK;
|
|
paddr_start = global_sunxi_domain->sg_buffer;
|
|
iova_end = SPAGE_ALIGN(iova + size) - SPAGE_SIZE;
|
|
mutex_lock(&priv->dt_lock);
|
|
total_size = sunxi_convert_sg_to_table(sg, size);
|
|
if (IOPDE_INDEX(iova_start) == IOPDE_INDEX(iova_end)) {
|
|
/*map in the same pde.*/
|
|
s_iova_start = iova_start;
|
|
dent = iopde_offset(priv->pgtable, iova_start);
|
|
if (!IS_VAILD(*dent))
|
|
sunxi_alloc_iopte(dent, prot);
|
|
for (; iova_start <= iova_end;
|
|
iova_start += SPAGE_SIZE, paddr_start++) {
|
|
pent = iopte_offset(dent, iova_start);
|
|
BUG_ON(*pent);
|
|
*pent = sunxi_mk_pte(*paddr_start, prot);
|
|
}
|
|
__dma_map_area(iopte_offset(dent, s_iova_start),
|
|
(size / SPAGE_SIZE) << 2, DMA_TO_DEVICE);
|
|
goto out;
|
|
} else {
|
|
/*map to the bottom of this pde.*/
|
|
dent = iopde_offset(priv->pgtable, iova_start);
|
|
if (!IS_VAILD(*dent))
|
|
sunxi_alloc_iopte(dent, prot);
|
|
s_iova_start = iova_start;
|
|
for (; iova_start < SPDE_ALIGN(s_iova_start+1);
|
|
iova_start += SPAGE_SIZE, paddr_start++) {
|
|
pent = iopte_offset(dent, iova_start);
|
|
BUG_ON(*pent);
|
|
*pent = sunxi_mk_pte(*paddr_start, prot);
|
|
}
|
|
__dma_map_area(iopte_offset(dent, s_iova_start),
|
|
((SPDE_ALIGN(s_iova_start+1) - s_iova_start) / SPAGE_SIZE) << 2,
|
|
DMA_TO_DEVICE);
|
|
}
|
|
|
|
if (IOPDE_INDEX(iova_start) < IOPDE_INDEX(iova_end)) {
|
|
/*map the number of pde*/
|
|
for (i = IOPDE_INDEX(iova_start); i < IOPDE_INDEX(iova_end);
|
|
i++, iova_start += SPD_SIZE) {
|
|
dent = iopde_offset(priv->pgtable, iova_start);
|
|
if (!IS_VAILD(*dent))
|
|
sunxi_alloc_iopte(dent, prot);
|
|
pent = iopte_offset(dent, iova_start);
|
|
for (j = 0; j < NUM_ENTRIES_PTE; j++)
|
|
*(pent + j) =
|
|
sunxi_mk_pte(*(paddr_start++), prot);
|
|
__dma_map_area(pent, PT_SIZE, DMA_TO_DEVICE);
|
|
}
|
|
}
|
|
|
|
/*map the last pde*/
|
|
dent = iopde_offset(priv->pgtable, iova_start);
|
|
if (!IS_VAILD(*dent))
|
|
sunxi_alloc_iopte(dent, prot);
|
|
s_iova_start = iova_start;
|
|
for (; iova_start <= iova_end;
|
|
iova_start += SPAGE_SIZE, paddr_start++) {
|
|
pent = iopte_offset(dent, iova_start);
|
|
*pent = sunxi_mk_pte(*paddr_start, prot);
|
|
}
|
|
__dma_map_area(iopte_offset(dent, s_iova_start),
|
|
((iova_end + SPAGE_SIZE - s_iova_start) / SPAGE_SIZE) << 2,
|
|
DMA_TO_DEVICE);
|
|
|
|
out:
|
|
sunxi_zap_tlb(iova, size);
|
|
mutex_unlock(&priv->dt_lock);
|
|
return total_size;
|
|
}
|
|
|
|
static phys_addr_t sunxi_iommu_iova_to_phys(struct iommu_domain *domain,
|
|
dma_addr_t iova)
|
|
{
|
|
struct sunxi_iommu_domain *priv = domain->priv;
|
|
u32 *entry;
|
|
phys_addr_t phys = 0;
|
|
|
|
mutex_lock(&priv->dt_lock);
|
|
entry = iopde_offset(priv->pgtable, iova);
|
|
|
|
if (IS_VAILD(*entry)) {
|
|
entry = iopte_offset(entry, iova);
|
|
phys = IOPTE_TO_PFN(entry) + IOVA_PAGE_OFT(iova);
|
|
}
|
|
mutex_unlock(&priv->dt_lock);
|
|
return phys;
|
|
}
|
|
|
|
static int sunxi_iommu_attach_device(struct iommu_domain *domain,
|
|
struct device *dev)
|
|
{
|
|
int ret = 0;
|
|
|
|
pr_err("Attached to iommu domain\n");
|
|
return ret;
|
|
}
|
|
|
|
static void sunxi_iommu_detach_device(struct iommu_domain *domain,
|
|
struct device *dev)
|
|
{
|
|
pr_err("Detached to iommu domain\n");
|
|
}
|
|
|
|
static int sunxi_iommu_add_device(struct device *dev)
|
|
{
|
|
struct sunxi_mdev *mdev;
|
|
struct of_phandle_args args;
|
|
struct dma_iommu_mapping *mapping;
|
|
struct sunxi_iommu_domain *sunxi_domain = global_sunxi_domain;
|
|
struct sunxi_iommu *iommu = global_iommu_dev;
|
|
int ret = 0;
|
|
|
|
/* parse device tree to get master id and pdev */
|
|
ret = of_parse_phandle_with_args(dev->of_node, "iommus",
|
|
"#iommu-cells", 0, &args);
|
|
if (ret < 0) {
|
|
ret = -EINVAL;
|
|
goto out;
|
|
}
|
|
mdev = kzalloc(sizeof(*mdev), GFP_KERNEL);
|
|
if (!mdev) {
|
|
ret = -ENOMEM;
|
|
goto out_put_node;
|
|
}
|
|
|
|
mdev->dev = dev;
|
|
mdev->tlbid = args.args[0];
|
|
mdev->flag = args.args[1];
|
|
|
|
if (!sunxi_domain) {
|
|
mapping = arm_iommu_create_mapping(&platform_bus_type, 0,
|
|
SZ_1G * 4UL);
|
|
if (IS_ERR(mapping)) {
|
|
ret = PTR_ERR(mapping);
|
|
goto out_free_mdev;
|
|
}
|
|
sunxi_domain = global_sunxi_domain;
|
|
sunxi_domain->mapping = mapping;
|
|
sunxi_tlb_init(sunxi_domain);
|
|
} else {
|
|
mapping = sunxi_domain->mapping;
|
|
}
|
|
if (arm_iommu_attach_device(dev, mapping)) {
|
|
pr_err("arm_iommu_attach_device failed\n");
|
|
ret = -EINVAL;
|
|
goto out_free_mdev;
|
|
}
|
|
sunxi_enable_device_iommu(mdev->tlbid, mdev->flag);
|
|
pr_debug("mastor id:%d, bypass:0x%x\n",
|
|
mdev->tlbid, sunxi_iommu_read(iommu, IOMMU_BYPASS_REG));
|
|
spin_lock(&sunxi_domain->lock);
|
|
list_add_tail(&mdev->node, &sunxi_domain->mdevs);
|
|
spin_unlock(&sunxi_domain->lock);
|
|
of_node_put(args.np);
|
|
return ret;
|
|
|
|
out_free_mdev:
|
|
kfree(mdev);
|
|
out_put_node:
|
|
of_node_put(args.np);
|
|
out:
|
|
return ret;
|
|
}
|
|
|
|
static void sunxi_iommu_remove_device(struct device *dev)
|
|
{
|
|
struct sunxi_mdev *mdev, *tmp;
|
|
struct sunxi_iommu_domain *sunxi_domain = global_sunxi_domain;
|
|
|
|
if (!dev->of_node) {
|
|
pr_warn("%s: Unable to get the device tree node\n", __func__);
|
|
return;
|
|
}
|
|
|
|
spin_lock(&sunxi_domain->lock);
|
|
list_for_each_entry_safe(mdev, tmp, &sunxi_domain->mdevs, node) {
|
|
if (mdev->dev == dev) {
|
|
arm_iommu_detach_device(dev);
|
|
sunxi_enable_device_iommu(mdev->tlbid, false);
|
|
list_del(&mdev->node);
|
|
kfree(mdev);
|
|
goto out;
|
|
}
|
|
}
|
|
out:
|
|
spin_unlock(&sunxi_domain->lock);
|
|
}
|
|
|
|
static unsigned long sunxi_iommu_reg_read(struct iommu_domain *domain,
|
|
unsigned long offset)
|
|
{
|
|
struct sunxi_iommu *iommu = global_iommu_dev;
|
|
u32 retval;
|
|
|
|
if ((offset >= SZ_1K) || (offset & 0x3)) {
|
|
pr_err("Invalid offset: 0x%lx\n", offset);
|
|
return 0;
|
|
}
|
|
spin_lock(&iommu->iommu_lock);
|
|
retval = sunxi_iommu_read(iommu, offset);
|
|
spin_unlock(&iommu->iommu_lock);
|
|
return retval;
|
|
}
|
|
|
|
static void sunxi_iommu_reg_write(struct iommu_domain *domain,
|
|
unsigned long offset, unsigned long val)
|
|
{
|
|
struct sunxi_iommu *iommu = global_iommu_dev;
|
|
|
|
if ((offset >= SZ_1K) || (offset & 0x3)) {
|
|
pr_err("Invalid offset: 0x%lx\n", offset);
|
|
return;
|
|
}
|
|
spin_lock(&iommu->iommu_lock);
|
|
sunxi_iommu_write(iommu, offset, val);
|
|
spin_unlock(&iommu->iommu_lock);
|
|
}
|
|
|
|
int sunxi_iova_test_write(dma_addr_t iova, u32 val)
|
|
{
|
|
struct sunxi_iommu *iommu = global_iommu_dev;
|
|
int retval;
|
|
|
|
spin_lock(&iommu->iommu_lock);
|
|
sunxi_iommu_write(iommu, IOMMU_VA_REG, iova);
|
|
sunxi_iommu_write(iommu, IOMMU_VA_DATA_REG, val);
|
|
sunxi_iommu_write(iommu,
|
|
IOMMU_VA_CONFIG_REG, 0x00000100);
|
|
sunxi_iommu_write(iommu,
|
|
IOMMU_VA_CONFIG_REG, 0x00000101);
|
|
retval = sunxi_wait_when((sunxi_iommu_read(iommu,
|
|
IOMMU_VA_CONFIG_REG) & 0x1), 1);
|
|
if (retval)
|
|
dev_err(iommu->dev,
|
|
"write VA address request timed out\n");
|
|
spin_unlock(&iommu->iommu_lock);
|
|
return retval;
|
|
}
|
|
|
|
unsigned long sunxi_iova_test_read(dma_addr_t iova)
|
|
{
|
|
struct sunxi_iommu *iommu = global_iommu_dev;
|
|
unsigned long retval;
|
|
|
|
spin_lock(&iommu->iommu_lock);
|
|
sunxi_iommu_write(iommu, IOMMU_VA_REG, iova);
|
|
sunxi_iommu_write(iommu,
|
|
IOMMU_VA_CONFIG_REG, 0x00000000);
|
|
sunxi_iommu_write(iommu,
|
|
IOMMU_VA_CONFIG_REG, 0x00000001);
|
|
retval = sunxi_wait_when((sunxi_iommu_read(iommu,
|
|
IOMMU_VA_CONFIG_REG) & 0x1), 1);
|
|
if (retval) {
|
|
dev_err(iommu->dev,
|
|
"read VA address request timed out\n");
|
|
retval = 0;
|
|
goto out;
|
|
}
|
|
retval = sunxi_iommu_read(iommu,
|
|
IOMMU_VA_DATA_REG);
|
|
out:
|
|
spin_unlock(&iommu->iommu_lock);
|
|
return retval;
|
|
}
|
|
|
|
static struct iommu_ops sunxi_iommu_ops = {
|
|
.domain_init = &sunxi_iommu_domain_init,
|
|
.domain_destroy = &sunxi_iommu_domain_destroy,
|
|
.attach_dev = &sunxi_iommu_attach_device,
|
|
.detach_dev = &sunxi_iommu_detach_device,
|
|
.map = &sunxi_iommu_map,
|
|
.unmap = &sunxi_iommu_unmap,
|
|
.map_sg = sunxi_iommu_map_sg,
|
|
.iova_to_phys = &sunxi_iommu_iova_to_phys,
|
|
.pgsize_bitmap = SZ_4K,
|
|
|
|
.domain_has_cap = NULL,
|
|
.add_device = sunxi_iommu_add_device,
|
|
.remove_device = sunxi_iommu_remove_device,
|
|
.device_group = NULL,
|
|
.domain_get_attr = NULL,
|
|
.domain_set_attr = NULL,
|
|
.domain_window_enable = NULL,
|
|
.domain_window_disable = NULL,
|
|
.reg_read = sunxi_iommu_reg_read,
|
|
.reg_write = sunxi_iommu_reg_write,
|
|
};
|
|
|
|
static ssize_t sunxi_iommu_enable_show(struct device *dev,
|
|
struct device_attribute *attr, char *buf)
|
|
{
|
|
struct sunxi_iommu *iommu = global_iommu_dev;
|
|
u32 data;
|
|
|
|
spin_lock(&iommu->iommu_lock);
|
|
data = sunxi_iommu_read(iommu, IOMMU_PMU_ENABLE_REG);
|
|
spin_unlock(&iommu->iommu_lock);
|
|
return snprintf(buf, PAGE_SIZE,
|
|
"enable = %d\n", data & 0x1 ? 1 : 0);
|
|
}
|
|
|
|
static ssize_t sunxi_iommu_enable_store(struct device *dev,
|
|
struct device_attribute *attr,
|
|
const char *buf, size_t count)
|
|
{
|
|
struct sunxi_iommu *iommu = global_iommu_dev;
|
|
unsigned long val;
|
|
u32 data;
|
|
int retval;
|
|
|
|
if (kstrtoul(buf, 0, &val))
|
|
return -EINVAL;
|
|
|
|
if (val) {
|
|
spin_lock(&iommu->iommu_lock);
|
|
data = sunxi_iommu_read(iommu, IOMMU_PMU_ENABLE_REG);
|
|
sunxi_iommu_write(iommu, IOMMU_PMU_ENABLE_REG, data | 0x1);
|
|
data = sunxi_iommu_read(iommu, IOMMU_PMU_CLR_REG);
|
|
sunxi_iommu_write(iommu, IOMMU_PMU_CLR_REG, data | 0x1);
|
|
retval = sunxi_wait_when((sunxi_iommu_read(iommu,
|
|
IOMMU_PMU_CLR_REG) & 0x1), 1);
|
|
if (retval)
|
|
dev_err(iommu->dev, "Clear PMU Count timed out\n");
|
|
spin_unlock(&iommu->iommu_lock);
|
|
} else {
|
|
spin_lock(&iommu->iommu_lock);
|
|
data = sunxi_iommu_read(iommu, IOMMU_PMU_CLR_REG);
|
|
sunxi_iommu_write(iommu, IOMMU_PMU_CLR_REG, data | 0x1);
|
|
retval = sunxi_wait_when((sunxi_iommu_read(iommu,
|
|
IOMMU_PMU_CLR_REG) & 0x1), 1);
|
|
if (retval)
|
|
dev_err(iommu->dev, "Clear PMU Count timed out\n");
|
|
data = sunxi_iommu_read(iommu, IOMMU_PMU_ENABLE_REG);
|
|
sunxi_iommu_write(iommu, IOMMU_PMU_ENABLE_REG, data & ~0x1);
|
|
spin_unlock(&iommu->iommu_lock);
|
|
}
|
|
|
|
return count;
|
|
}
|
|
|
|
static ssize_t sunxi_iommu_profilling_show(struct device *dev,
|
|
struct device_attribute *attr, char *buf)
|
|
{
|
|
struct sunxi_iommu *iommu = global_iommu_dev;
|
|
u64 micro_tlb0_access_count;
|
|
u64 micro_tlb0_hit_count;
|
|
u16 micro_tlb0_hitrate;
|
|
u16 macro_tlb0_hitrate;
|
|
u64 micro_tlb1_access_count;
|
|
u64 micro_tlb1_hit_count;
|
|
u16 micro_tlb1_hitrate;
|
|
u16 macro_tlb1_hitrate;
|
|
u64 micro_tlb2_access_count;
|
|
u64 micro_tlb2_hit_count;
|
|
u16 micro_tlb2_hitrate;
|
|
u16 macro_tlb2_hitrate;
|
|
u64 micro_tlb3_access_count;
|
|
u64 micro_tlb3_hit_count;
|
|
u16 micro_tlb3_hitrate;
|
|
u16 macro_tlb3_hitrate;
|
|
u64 micro_tlb4_access_count;
|
|
u64 micro_tlb4_hit_count;
|
|
u16 micro_tlb4_hitrate;
|
|
u16 macro_tlb4_hitrate;
|
|
u64 micro_tlb5_access_count;
|
|
u64 micro_tlb5_hit_count;
|
|
u16 micro_tlb5_hitrate;
|
|
u16 macro_tlb5_hitrate;
|
|
u64 macrotlb_access_count;
|
|
u64 macrotlb_hit_count;
|
|
u16 macrotlb_hitrate;
|
|
u64 ptwcache_access_count;
|
|
u64 ptwcache_hit_count;
|
|
u16 ptwcache_hitrate;
|
|
u64 micro_tlb0_latency;
|
|
u64 micro_tlb1_latency;
|
|
u64 micro_tlb2_latency;
|
|
u64 micro_tlb3_latency;
|
|
u64 micro_tlb4_latency;
|
|
u64 micro_tlb5_latency;
|
|
|
|
spin_lock(&iommu->iommu_lock);
|
|
macrotlb_access_count =
|
|
((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_HIGH_REG6) &&
|
|
0x7ff) << 32) |
|
|
sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_LOW_REG6);
|
|
macrotlb_hit_count =
|
|
((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_HIT_HIGH_REG6) &&
|
|
0x7ff) << 32) | sunxi_iommu_read(iommu, IOMMU_PMU_HIT_LOW_REG6);
|
|
macrotlb_hitrate =
|
|
(macrotlb_hit_count * 1000) /
|
|
_max(1, macrotlb_access_count);
|
|
|
|
micro_tlb0_access_count =
|
|
((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_HIGH_REG0) &&
|
|
0x7ff) << 32) |
|
|
sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_LOW_REG0);
|
|
micro_tlb0_hit_count =
|
|
((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_HIT_HIGH_REG0) &&
|
|
0x7ff) << 32) | sunxi_iommu_read(iommu, IOMMU_PMU_HIT_LOW_REG0);
|
|
micro_tlb0_hitrate =
|
|
(micro_tlb0_hit_count * 1000) /
|
|
_max(1, micro_tlb0_access_count);
|
|
macro_tlb0_hitrate = micro_tlb0_access_count == 0 ? 0 :
|
|
((1000 - micro_tlb0_hitrate) * macrotlb_hitrate) / 1000;
|
|
|
|
micro_tlb1_access_count =
|
|
((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_HIGH_REG1) &&
|
|
0x7ff) << 32) |
|
|
sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_LOW_REG1);
|
|
micro_tlb1_hit_count =
|
|
((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_HIT_HIGH_REG1) &&
|
|
0x7ff) << 32) | sunxi_iommu_read(iommu, IOMMU_PMU_HIT_LOW_REG1);
|
|
micro_tlb1_hitrate =
|
|
(micro_tlb1_hit_count * 1000) /
|
|
_max(1, micro_tlb1_access_count);
|
|
macro_tlb1_hitrate = micro_tlb1_access_count == 0 ? 0 :
|
|
((1000 - micro_tlb1_hitrate) * macrotlb_hitrate) / 1000;
|
|
|
|
micro_tlb2_access_count =
|
|
((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_HIGH_REG2) &&
|
|
0x7ff) << 32) |
|
|
sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_LOW_REG2);
|
|
micro_tlb2_hit_count =
|
|
((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_HIT_HIGH_REG2) &&
|
|
0x7ff) << 32) | sunxi_iommu_read(iommu, IOMMU_PMU_HIT_LOW_REG2);
|
|
micro_tlb2_hitrate =
|
|
(micro_tlb2_hit_count * 1000) /
|
|
_max(1, micro_tlb2_access_count);
|
|
macro_tlb2_hitrate = micro_tlb2_access_count == 0 ? 0 :
|
|
((1000 - micro_tlb2_hitrate) * macrotlb_hitrate) / 1000;
|
|
|
|
micro_tlb3_access_count =
|
|
((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_HIGH_REG3) &&
|
|
0x7ff) << 32) |
|
|
sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_LOW_REG3);
|
|
micro_tlb3_hit_count =
|
|
((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_HIT_HIGH_REG3) &&
|
|
0x7ff) << 32) | sunxi_iommu_read(iommu, IOMMU_PMU_HIT_LOW_REG3);
|
|
micro_tlb3_hitrate =
|
|
(micro_tlb3_hit_count * 1000) /
|
|
_max(1, micro_tlb3_access_count);
|
|
macro_tlb3_hitrate = micro_tlb3_access_count == 0 ? 0 :
|
|
((1000 - micro_tlb3_hitrate) * macrotlb_hitrate) / 1000;
|
|
|
|
micro_tlb4_access_count =
|
|
((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_HIGH_REG4) &&
|
|
0x7ff) << 32) |
|
|
sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_LOW_REG4);
|
|
micro_tlb4_hit_count =
|
|
((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_HIT_HIGH_REG4) &&
|
|
0x7ff) << 32) | sunxi_iommu_read(iommu, IOMMU_PMU_HIT_LOW_REG4);
|
|
micro_tlb4_hitrate =
|
|
(micro_tlb4_hit_count * 1000) /
|
|
_max(1, micro_tlb4_access_count);
|
|
macro_tlb4_hitrate = micro_tlb4_access_count == 0 ? 0 :
|
|
((1000 - micro_tlb4_hitrate) * macrotlb_hitrate) / 1000;
|
|
|
|
micro_tlb5_access_count =
|
|
((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_HIGH_REG5) &&
|
|
0x7ff) << 32) |
|
|
sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_LOW_REG5);
|
|
micro_tlb5_hit_count =
|
|
((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_HIT_HIGH_REG5) &&
|
|
0x7ff) << 32) | sunxi_iommu_read(iommu, IOMMU_PMU_HIT_LOW_REG5);
|
|
micro_tlb5_hitrate =
|
|
(micro_tlb5_hit_count * 1000) /
|
|
_max(1, micro_tlb5_access_count);
|
|
macro_tlb5_hitrate = micro_tlb5_access_count == 0 ? 0 :
|
|
((1000 - micro_tlb5_hitrate) * macrotlb_hitrate) / 1000;
|
|
|
|
ptwcache_access_count =
|
|
((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_HIGH_REG7) &&
|
|
0x7ff) << 32) |
|
|
sunxi_iommu_read(iommu, IOMMU_PMU_ACCESS_LOW_REG7);
|
|
ptwcache_hit_count =
|
|
((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_HIT_HIGH_REG7) &&
|
|
0x7ff) << 32) | sunxi_iommu_read(iommu, IOMMU_PMU_HIT_LOW_REG7);
|
|
ptwcache_hitrate =
|
|
(ptwcache_hit_count * 1000) /
|
|
_max(1, ptwcache_access_count);
|
|
|
|
micro_tlb0_latency =
|
|
((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_TL_HIGH_REG0) &&
|
|
0x3ffff) << 32) |
|
|
sunxi_iommu_read(iommu, IOMMU_PMU_TL_LOW_REG0);
|
|
micro_tlb1_latency =
|
|
((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_TL_HIGH_REG1) &&
|
|
0x3ffff) << 32) |
|
|
sunxi_iommu_read(iommu, IOMMU_PMU_TL_LOW_REG1);
|
|
micro_tlb2_latency =
|
|
((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_TL_HIGH_REG2) &&
|
|
0x3ffff) << 32) |
|
|
sunxi_iommu_read(iommu, IOMMU_PMU_TL_LOW_REG2);
|
|
micro_tlb3_latency =
|
|
((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_TL_HIGH_REG3) &&
|
|
0x3ffff) << 32) |
|
|
sunxi_iommu_read(iommu, IOMMU_PMU_TL_LOW_REG3);
|
|
micro_tlb4_latency =
|
|
((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_TL_HIGH_REG4) &&
|
|
0x3ffff) << 32) |
|
|
sunxi_iommu_read(iommu, IOMMU_PMU_TL_LOW_REG4);
|
|
micro_tlb5_latency =
|
|
((u64)(sunxi_iommu_read(iommu, IOMMU_PMU_TL_HIGH_REG5) &&
|
|
0x3ffff) << 32) |
|
|
sunxi_iommu_read(iommu, IOMMU_PMU_TL_LOW_REG5);
|
|
spin_unlock(&iommu->iommu_lock);
|
|
|
|
return snprintf(buf, PAGE_SIZE,
|
|
"%s_access_count = 0x%llx\n"
|
|
"%s_hit_count = 0x%llx\n"
|
|
"%s_hitrate = %d\n"
|
|
"%s_access_count = 0x%llx\n"
|
|
"%s_hit_count = 0x%llx\n"
|
|
"%s_hitrate = %d\n"
|
|
"%s_access_count = 0x%llx\n"
|
|
"%s_hit_count = 0x%llx\n"
|
|
"%s_hitrate = %d\n"
|
|
"%s_access_count = 0x%llx\n"
|
|
"%s_hit_count = 0x%llx\n"
|
|
"%s_hitrate = %d\n"
|
|
"%s_access_count = 0x%llu\n"
|
|
"%s_hit_count = 0x%llu\n"
|
|
"%s_hitrate = %d\n"
|
|
"%s_access_count = 0x%llx\n"
|
|
"%s_hit_count = 0x%llx\n"
|
|
"%s_hitrate = %d\n"
|
|
"macrotlb_access_count = 0x%llx\n"
|
|
"macrotlb_hit_count = 0x%llx\n"
|
|
"macrotlb_hitrate = %d\n"
|
|
"ptwcache_access_count = 0x%llx\n"
|
|
"ptwcache_hit_count = 0x%llx\n"
|
|
"ptwcache_hitrate = %d\n"
|
|
"%s_latency = 0x%llx\n"
|
|
"%s_latency = 0x%llx\n"
|
|
"%s_latency = 0x%llx\n"
|
|
"%s_latency = 0x%llx\n"
|
|
"%s_latency = 0x%llx\n"
|
|
"%s_latency = 0x%llx\n",
|
|
master[0], micro_tlb0_access_count,
|
|
master[0], micro_tlb0_hit_count,
|
|
master[0], micro_tlb0_hitrate + macro_tlb0_hitrate,
|
|
master[1], micro_tlb1_access_count,
|
|
master[1], micro_tlb1_hit_count,
|
|
master[1], micro_tlb1_hitrate + macro_tlb1_hitrate,
|
|
master[2], micro_tlb2_access_count,
|
|
master[2], micro_tlb2_hit_count,
|
|
master[2], micro_tlb2_hitrate + macro_tlb2_hitrate,
|
|
master[3], micro_tlb3_access_count,
|
|
master[3], micro_tlb3_hit_count,
|
|
master[3], micro_tlb3_hitrate + macro_tlb3_hitrate,
|
|
master[4], micro_tlb4_access_count,
|
|
master[4], micro_tlb4_hit_count,
|
|
master[4], micro_tlb4_hitrate + macro_tlb4_hitrate,
|
|
master[5], micro_tlb5_access_count,
|
|
master[5], micro_tlb5_hit_count,
|
|
master[5], micro_tlb5_hitrate + macro_tlb5_hitrate,
|
|
macrotlb_access_count,
|
|
macrotlb_hit_count,
|
|
macrotlb_hitrate,
|
|
ptwcache_access_count,
|
|
ptwcache_hit_count,
|
|
ptwcache_hitrate,
|
|
master[0], micro_tlb0_latency,
|
|
master[1], micro_tlb1_latency,
|
|
master[2], micro_tlb2_latency,
|
|
master[3], micro_tlb3_latency,
|
|
master[4], micro_tlb4_latency,
|
|
master[5], micro_tlb5_latency);
|
|
}
|
|
|
|
static struct device_attribute sunxi_iommu_enable_attr =
|
|
__ATTR(enable, S_IWUSR | S_IRUGO, sunxi_iommu_enable_show,
|
|
sunxi_iommu_enable_store);
|
|
static struct device_attribute sunxi_iommu_profilling_attr =
|
|
__ATTR(profilling, S_IRUGO, sunxi_iommu_profilling_show, NULL);
|
|
|
|
static void sunxi_iommu_sysfs_create(struct platform_device *_pdev)
|
|
{
|
|
device_create_file(&_pdev->dev, &sunxi_iommu_enable_attr);
|
|
device_create_file(&_pdev->dev, &sunxi_iommu_profilling_attr);
|
|
}
|
|
|
|
static void sunxi_iommu_sysfs_remove(struct platform_device *_pdev)
|
|
{
|
|
device_remove_file(&_pdev->dev, &sunxi_iommu_enable_attr);
|
|
device_remove_file(&_pdev->dev, &sunxi_iommu_profilling_attr);
|
|
}
|
|
|
|
static int sunxi_iommu_probe(struct platform_device *pdev)
|
|
{
|
|
int ret, irq;
|
|
struct device *dev = &pdev->dev;
|
|
struct sunxi_iommu *iommu_dev;
|
|
struct resource *res;
|
|
|
|
iommu_dev = kzalloc(sizeof(*iommu_dev), GFP_KERNEL);
|
|
if (!iommu_dev) {
|
|
dev_dbg(dev, "Not enough memory\n");
|
|
return -ENOMEM;
|
|
}
|
|
|
|
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
|
|
if (!res) {
|
|
dev_dbg(dev, "Unable to find resource region\n");
|
|
ret = -ENOENT;
|
|
goto err_res;
|
|
}
|
|
|
|
iommu_dev->base = devm_ioremap_resource(&pdev->dev, res);
|
|
if (!iommu_dev->base) {
|
|
dev_dbg(dev, "Unable to map IOMEM @ PA:%#x\n",
|
|
(unsigned int)res->start);
|
|
ret = -ENOENT;
|
|
goto err_res;
|
|
}
|
|
|
|
iommu_dev->bypass = 0x3f;
|
|
irq = platform_get_irq(pdev, 0);
|
|
if (irq <= 0) {
|
|
dev_dbg(dev, "Unable to find IRQ resource\n");
|
|
goto err_irq;
|
|
}
|
|
pr_info(" irq = %d\n", irq);
|
|
|
|
ret = devm_request_irq(dev, irq, sunxi_iommu_irq, IRQF_DISABLED,
|
|
dev_name(dev), (void *)iommu_dev);
|
|
if (ret < 0) {
|
|
dev_dbg(dev, "Unabled to register interrupt handler\n");
|
|
goto err_irq;
|
|
}
|
|
iommu_dev->irq = irq;
|
|
iommu_dev->clk = of_clk_get_by_name(dev->of_node, "iommu");
|
|
if (IS_ERR(iommu_dev->clk)) {
|
|
iommu_dev->clk = NULL;
|
|
dev_dbg(dev, "Unable to find clock\n");
|
|
goto err_clk;
|
|
}
|
|
clk_prepare_enable(iommu_dev->clk);
|
|
/*
|
|
* TODO: initialize hardware configuration, such as clock, bypass bit.
|
|
*/
|
|
|
|
platform_set_drvdata(pdev, iommu_dev);
|
|
iommu_dev->dev = dev;
|
|
spin_lock_init(&iommu_dev->iommu_lock);
|
|
global_iommu_dev = iommu_dev;
|
|
if (dev->parent)
|
|
pm_runtime_enable(dev);
|
|
bus_set_iommu(&platform_bus_type, &sunxi_iommu_ops);
|
|
sunxi_iommu_sysfs_create(pdev);
|
|
return 0;
|
|
|
|
err_clk:
|
|
free_irq(irq, iommu_dev);
|
|
err_irq:
|
|
devm_iounmap(dev, iommu_dev->base);
|
|
err_res:
|
|
kfree(iommu_dev);
|
|
dev_err(dev, "Failed to initialize\n");
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int sunxi_iommu_remove(struct platform_device *pdev)
|
|
{
|
|
struct sunxi_iommu *mmu_dev = platform_get_drvdata(pdev);
|
|
|
|
sunxi_iommu_sysfs_remove(pdev);
|
|
free_irq(mmu_dev->irq, mmu_dev);
|
|
devm_iounmap(mmu_dev->dev, mmu_dev->base);
|
|
kfree(mmu_dev);
|
|
global_iommu_dev = NULL;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int sunxi_iommu_suspend(struct device *dev)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static int sunxi_iommu_resume(struct device *dev)
|
|
{
|
|
struct sunxi_iommu_domain *sunxi_domain = global_sunxi_domain;
|
|
int err;
|
|
|
|
err = sunxi_tlb_init(sunxi_domain);
|
|
return err;
|
|
}
|
|
|
|
const struct dev_pm_ops sunxi_iommu_pm_ops = {
|
|
.suspend = sunxi_iommu_suspend,
|
|
.resume = sunxi_iommu_resume,
|
|
};
|
|
|
|
static const struct of_device_id sunxi_iommu_dt[] = {
|
|
{ .compatible = "allwinner,sunxi-iommu", },
|
|
};
|
|
MODULE_DEVICE_TABLE(of, sunxi_iommu_dt);
|
|
|
|
static struct platform_driver sunxi_iommu_driver = {
|
|
.probe = sunxi_iommu_probe,
|
|
.remove = sunxi_iommu_remove,
|
|
.driver = {
|
|
.owner = THIS_MODULE,
|
|
.name = "sunxi-iommu",
|
|
.pm = &sunxi_iommu_pm_ops,
|
|
.of_match_table = sunxi_iommu_dt,
|
|
}
|
|
};
|
|
|
|
static int __init sunxi_iommu_init(void)
|
|
{
|
|
struct device_node *np;
|
|
int ret;
|
|
|
|
np = of_find_matching_node(NULL, sunxi_iommu_dt);
|
|
if (!np)
|
|
return 0;
|
|
|
|
of_node_put(np);
|
|
|
|
iopte_cache = kmem_cache_create("sunxi-iopte-cache", PT_SIZE,
|
|
PT_SIZE, SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
|
|
if (!iopte_cache)
|
|
return -ENOMEM;
|
|
|
|
ret = platform_driver_register(&sunxi_iommu_driver);
|
|
if (ret)
|
|
kmem_cache_destroy(iopte_cache);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static void __exit sunxi_iommu_exit(void)
|
|
{
|
|
kmem_cache_destroy(iopte_cache);
|
|
platform_driver_unregister(&sunxi_iommu_driver);
|
|
}
|
|
|
|
subsys_initcall(sunxi_iommu_init);
|
|
module_exit(sunxi_iommu_exit);
|
|
|
|
MODULE_DESCRIPTION("IOMMU Driver for Allwinner");
|
|
MODULE_AUTHOR("fanqinghua <fanqinghua@allwinnertech.com>");
|
|
MODULE_ALIAS("platform:allwinner");
|
|
MODULE_LICENSE("GPL v2");
|