// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2015-2016 MediaTek Inc. * Author: Yong Wu */ #define pr_fmt(fmt) "mtk_iommu: " fmt #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if IS_ENABLED(CONFIG_MTK_SMI) #include <../misc/mediatek/smi/mtk-smi-dbg.h> #endif #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_DBG) #include <../misc/mediatek/iommu/iommu_debug.h> #endif #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_SECURE) #include <../misc/mediatek/iommu/iommu_secure.h> #endif #include "mtk_iommu.h" #define REG_MMU_PT_BASE_ADDR 0x000 #define MMU_PT_ADDR_MASK GENMASK(31, 7) #define REG_MMU_STA 0x008 #define REG_MMU_INVALIDATE 0x020 #define F_ALL_INVLD 0x2 #define F_MMU_INV_RANGE 0x1 #define REG_MMU_INVLD_START_A 0x024 #define REG_MMU_INVLD_END_A 0x028 #define REG_MMU_INV_SEL_GEN2 0x02c #define REG_MMU_INV_SEL_GEN1 0x038 #define F_INVLD_EN0 BIT(0) #define F_INVLD_EN1 BIT(1) #define REG_MMU_DUMMY 0x044 #define REG_MMU_MISC_CTRL 0x048 #define F_MMU_IN_ORDER_WR_EN_MASK (BIT(1) | BIT(17)) #define F_MMU_STANDARD_AXI_MODE_MASK (BIT(3) | BIT(19)) #define REG_MMU_DCM_DIS 0x050 #define REG_MMU_WR_LEN_CTRL 0x054 #define F_MMU_WR_THROT_DIS_MASK (BIT(5) | BIT(21)) #define REG_MMU_DBG(index) (0x060 + index * 4) #define REG_MMU_TBW_ID 0xa0 #define REG_MMU_CTRL_REG 0x110 #define F_MMU_MON_EN BIT(1) #define F_MMU_SYNC_INVLDT_EN BIT(3) #define F_MMU_TF_PROT_TO_PROGRAM_ADDR (2 << 4) #define F_MMU_PREFETCH_RT_REPLACE_MOD BIT(4) #define F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173 (2 << 5) #define REG_MMU_IVRP_PADDR 0x114 #define REG_MMU_VLD_PA_RNG 0x118 #define F_MMU_VLD_PA_RNG(EA, SA) (((EA) << 8) | (SA)) #define REG_MMU_INT_CONTROL0 0x120 #define F_L2_MULIT_HIT_EN BIT(0) #define F_TABLE_WALK_FAULT_INT_EN BIT(1) #define F_PREETCH_FIFO_OVERFLOW_INT_EN BIT(2) #define F_MISS_FIFO_OVERFLOW_INT_EN BIT(3) #define F_TLB_INVALID_DONE_EN BIT(4) #define F_PREFETCH_FIFO_ERR_INT_EN BIT(5) #define F_MISS_FIFO_ERR_INT_EN BIT(6) #define F_INT_CLR_BIT BIT(12) #define REG_MMU_INT_MAIN_CONTROL 0x124 /* mmu0 | mmu1 */ #define F_INT_TRANSLATION_FAULT (BIT(0) | BIT(7)) #define F_INT_MAIN_MULTI_HIT_FAULT (BIT(1) | BIT(8)) #define F_INT_INVALID_PA_FAULT (BIT(2) | BIT(9)) #define F_INT_ENTRY_REPLACEMENT_FAULT (BIT(3) | BIT(10)) #define F_INT_TLB_MISS_FAULT (BIT(4) | BIT(11)) #define F_INT_MISS_TRANSACTION_FIFO_FAULT (BIT(5) | BIT(12)) #define F_INT_PRETETCH_TRANSATION_FIFO_FAULT (BIT(6) | BIT(13)) #define F_INT_MAIN_MAU_INT_EN(MAUCNT) \ (F_REG_MMU_MAU_INT_MASK(0, MAUCNT) | F_REG_MMU_MAU_INT_MASK(1, MAUCNT)) #define F_INT_MMU_MAU_INT_EN(MMU, MAU, MAUCNT) BIT(14+(MMU)*(MAUCNT)+(MAU)) #define REG_MMU_CPE_DONE 0x12C #define REG_MMU_FAULT_ST0 0x130 #define F_INT_MHIT_ERR BIT(0) #define F_INT_TBW_FAULT BIT(1) #define F_INT_PFQ_FIFO_FULL BIT(2) #define F_INT_MQ_FIFO_FULL BIT(3) #define F_INT_INVLDT_DONE BIT(4) #define F_INT_PFQ_OUT_FIFO_ERR BIT(5) #define F_INT_PFQ_IN_FIFO_ERR BIT(6) #define F_INT_MQ_OUT_FIFO_ERR BIT(7) #define F_INT_MQ_IN_FIFO_ERR BIT(8) #define F_INT_CDB_SLICE_ERR BIT(9) #define F_REG_MMU_FAULT_ST0_MASK GENMASK(9, 0) #define REG_MMU_FAULT_ST1 0x134 #define F_INT_MMU_TF_ERR(MMU) BIT(0+(MMU)*7) #define F_INT_MMU_MHIT_ERR(MMU) BIT(1+(MMU)*7) #define F_INT_MMU_INV_PA_ERR(MMU) BIT(2+(MMU)*7) #define F_INT_MMU_ENTR_REP_ERR(MMU) BIT(3+(MMU)*7) #define F_INT_MMU_TLBM_ERR(MMU) BIT(4+(MMU)*7) #define F_INT_MMU_MQ_OVF_ERR(MMU) BIT(5+(MMU)*7) #define F_INT_MMU_PFQ_OVF_ERR(MMU) BIT(6+(MMU)*7) #define F_INT_MMU_MAU_INT_STA(MMU, MAU, MAUCNT) BIT(14+(MMU)*(MAUCNT)+(MAU)) #define F_REG_MMU0_INV_PA_MASK BIT(0) #define F_REG_MMU0_TF_MASK BIT(2) #define F_REG_MMU0_MAU_INT_MASK GENMASK(10, 7) #define F_REG_MMU0_FAULT_MASK GENMASK(6, 0) #define F_REG_MMU1_FAULT_MASK GENMASK(13, 7) #define F_REG_MMU_MAU_INT_MASK(MMU, MAUCNT) \ GENMASK((14+(MAUCNT)*((MMU)+1)-1), (14+(MAUCNT)*(MMU))) #define REG_MMU_TBWALK_FAULT_VA 0x138 #define F_L2_TBWALK_FAULT_VA_31_12_MASK GENMASK(31, 12) #define F_L2_TBWALK_FAULT_VA_33_32_MASK GENMASK(10, 9) #define F_L2_TBWALK_FAULT_PAGE_LAYER_BIT BIT(0) #define REG_MMU0_FAULT_VA 0x13c #define F_MMU_INVAL_VA_31_12_MASK GENMASK(31, 12) #define F_MMU_INVAL_VA_34_32_MASK GENMASK(11, 9) #define F_MMU_INVAL_PA_34_32_MASK GENMASK(8, 6) #define F_MMU_FAULT_VA_WRITE_BIT BIT(1) #define F_MMU_FAULT_VA_LAYER_BIT BIT(0) #define REG_MMU0_INVLD_PA 0x140 #define REG_MMU1_FAULT_VA 0x144 #define REG_MMU1_INVLD_PA 0x148 #define REG_MMU0_INT_ID 0x150 #define REG_MMU1_INT_ID 0x154 #define F_MMU_INT_ID_COMM_ID(a) (((a) >> 9) & 0xf) #define F_MMU_INT_ID_SUB_COMM_ID(a) (((a) >> 7) & 0x3) #define F_MMU_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7) #define F_MMU_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f) #define REG_MMU_RS_VA(MMU, RS) (0x380 + MMU * 0x300 + RS * 0x10) /* MAU set register */ #define REG_MMU_MAU_SA(MMU, MAU) (0x900+(MMU)*0x100+(MAU)*0x24) #define REG_MMU_MAU_SA_EXT(MMU, MAU) (0x904+(MMU)*0x100+(MAU)*0x24) #define REG_MMU_MAU_EA(MMU, MAU) (0x908+(MMU)*0x100+(MAU)*0x24) #define REG_MMU_MAU_EA_EXT(MMU, MAU) (0x90C+(MMU)*0x100+(MAU)*0x24) #define REG_MMU_MAU_LARB_EN(MMU, MAU) (0x910+(MMU)*0x100+(MAU)*0x24) #define REG_MMU_MAU_PORT_EN(MMU, MAU) (0x914+(MMU)*0x100+(MAU)*0x24) #define REG_MMU_MAU_ASRT_ID(MMU, MAU) (0x918+(MMU)*0x100+(MAU)*0x24) #define REG_MMU_MAU_AA(MMU, MAU) (0x91C+(MMU)*0x100+(MAU)*0x24) #define REG_MMU_MAU_AA_EXT(MMU, MAU) (0x920+(MMU)*0x100+(MAU)*0x24) #define REG_MMU_MAU_CLR(MMU, CNT) (0x900+(MMU)*0x100+(CNT)*0x24) #define REG_MMU_MAU_IO(MMU, CNT) (0x904+(MMU)*0x100+(CNT)*0x24) #define REG_MMU_MAU_RW(MMU, CNT) (0x908+(MMU)*0x100+(CNT)*0x24) #define REG_MMU_MAU_VA(MMU, CNT) (0x90C+(MMU)*0x100+(CNT)*0x24) #define REG_MMU_MAU_ASRT_STA(MMU, CNT) (0x910+(MMU)*0x100+(CNT)*0x24) #define REG_MMU_MAU_C4K(MMU, CNT) (0x914+(MMU)*0x100+(CNT)*0x24) #define REG_MMU_MAU_ASSERT_INFO(MMU, CNT) (0x918+(MMU)*0x100+(CNT)*0x24) #define F_MMU_MAU_BIT_VAL(VAL, MAU) ((!!(VAL))<<(MAU)) #define F_MMU_MAU_ASRT_ID_VAL GENMASK(7, 0) /* iommu hw register define */ #define MTK_IOMMU_DEBUG_REG_NR (7) #define MTK_IOMMU_MMU_COUNT (2) #define MTK_IOMMU_RS_COUNT (16) #define MTK_PROTECT_PA_ALIGN 256 #define HAS_4GB_MODE BIT(0) /* HW will use the EMI clock if there isn't the "bclk". */ #define HAS_BCLK BIT(1) #define HAS_VLD_PA_RNG BIT(2) #define RESET_AXI BIT(3) #define OUT_ORDER_WR_EN BIT(4) #define HAS_SUB_COMM BIT(5) #define WR_THROT_EN BIT(6) #define HAS_LEGACY_IVRP_PADDR BIT(7) #define IOVA_34_EN BIT(8) #define NOT_STD_AXI_MODE BIT(9) #define SET_TBW_ID BIT(10) #define LINK_WITH_APU BIT(11) #define TLB_SYNC_EN BIT(12) #define IOMMU_SEC_EN BIT(13) #define SKIP_CFG_PORT BIT(14) /* For IOMMU EP/bring up phase: CLK AO */ #define IOMMU_CLK_AO_EN BIT(15) /* For IOMMU EP/bring up phase: smi not ready */ #define IOMMU_EN_PRE BIT(16) /* Debug: Skip register IRQ */ #define IOMMU_NO_IRQ BIT(17) #define GET_DOM_ID_LEGACY BIT(18) #define HAS_SMI_SUB_COMM BIT(19) #define SAME_SUBSYS BIT(20) #define IOMMU_MAU_EN BIT(21) #define PM_OPS_SKIP BIT(22) #define SHARE_PGTABLE BIT(23) #define PGTABLE_PA_35_EN BIT(24) #define POWER_ON_STA 1 #define POWER_OFF_STA 0 #define MTK_IOMMU_HAS_FLAG(pdata, _x) \ ((((pdata)->flags) & (_x)) == (_x)) #define MTK_IOMMU_ISR_COUNT_MAX 2 #define MTK_IOMMU_ISR_DISABLE_TIME 10 #define MTK_IOMMU_TF_IOVA_DUMP_NUM 5 #define MMU_MAU_REG_BACKUP_SIZE (100 * sizeof(unsigned int)) /* hyp-pmm fastcalls */ #define HYP_PMM_SHARE_IOVA (0XBB00FFA2) #define HYP_PMM_UNSHARE_IOVA (0XBB00FFA3) #define HYP_PMM_GET_HYPMMU_TYPE2_EN (0XBB00FFA4) #define HYP_PMM_REG_HYPMMU_SHARE_REGION (0XBB00FFA5) #define HYP_PMM_IOVA_TO_PHYS (0XBB00FFA6) #define HYP_PMM_HYPMMU_TYPE2_INV (0XBB00FFA7) struct mtk_iommu_domain { int tab_id; struct io_pgtable_cfg cfg; struct io_pgtable_ops *iop; u32 ttbr; struct mtk_iommu_data *data; struct iommu_domain domain; }; static const struct iommu_ops mtk_iommu_ops; static bool pd_sta[MM_IOMMU_NUM]; static spinlock_t *tlb_locks[MM_IOMMU_NUM]; static struct notifier_block mtk_pd_notifiers[MM_IOMMU_NUM]; static bool hypmmu_type2_en; static struct mutex init_mutexs[PGTBALE_NUM]; static struct mutex group_mutexs[MTK_IOMMU_GROUP_MAX]; static atomic_t init_once_flag = ATOMIC_INIT(0); static int mtk_iommu_hw_init(const struct mtk_iommu_data *data); #define MTK_IOMMU_TLB_ADDR(iova) ({ \ dma_addr_t _addr = iova; \ ((lower_32_bits(_addr) & GENMASK(31, 12)) | upper_32_bits(_addr));\ }) #define MTK_IOMMU_ID_FLAG(type, id) BIT((2*(type)) + (id)) /* * In M4U 4GB mode, the physical address is remapped as below: * * CPU Physical address: * ==================== * * 0 1G 2G 3G 4G 5G * |---A---|---B---|---C---|---D---|---E---| * +--I/O--+------------Memory-------------+ * * IOMMU output physical address: * ============================= * * 4G 5G 6G 7G 8G * |---E---|---B---|---C---|---D---| * +------------Memory-------------+ * * The Region 'A'(I/O) can NOT be mapped by M4U; For Region 'B'/'C'/'D', the * bit32 of the CPU physical address always is needed to set, and for Region * 'E', the CPU physical address keep as is. * Additionally, The iommu consumers always use the CPU phyiscal address. */ #define MTK_IOMMU_4GB_MODE_REMAP_BASE 0x140000000UL static LIST_HEAD(m4ulist); /* List all the M4U HWs */ static LIST_HEAD(mm_iommu_list); /* List apu iommu HWs */ static LIST_HEAD(apu_iommu_list); /* List mm iommu HWs */ static bool share_pgtable; #define for_each_m4u(data, head) list_for_each_entry(data, head, list) enum iova_type { NORMAL, PROTECTED, SECURE, IOVA_TYPE_NUM }; struct mtk_iommu_iova_region { dma_addr_t iova_base; unsigned long long size; enum iova_type type; }; static const struct mtk_iommu_iova_region single_domain[] = { {.iova_base = 0, .size = SZ_4G}, }; static const struct mtk_iommu_iova_region single_domain_ext[] = { {.iova_base = 0, .size = SZ_4G * 4}, }; static const struct mtk_iommu_iova_region mt8192_multi_dom[] = { { .iova_base = 0x0, .size = SZ_4G}, /* disp: 0 ~ 4G */ #if IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) { .iova_base = SZ_4G, .size = SZ_4G}, /* vdec: 4G ~ 8G */ { .iova_base = SZ_4G * 2, .size = SZ_4G}, /* CAM/MDP: 8G ~ 12G */ { .iova_base = 0x240000000ULL, .size = 0x4000000}, /* CCU0 */ { .iova_base = 0x244000000ULL, .size = 0x4000000}, /* CCU1 */ #endif }; /* * 0,NORMAL: 0x2_0000_0000~0x3_FFFF_FFFF(8GB) * 1.CAM_MDP: 0x0_0000_1000~0x0_FBFF_FFFF(4G-4KB-64MB) * 2.AIE_RESV: 0x0_FC00_0000~0x0_FFFF_FFFF(64MB) * 3.LK_RESV: 0x1_0600_0000~0x1_07FF_FFFF(32MB) * 4,VDEC_VENC: 0x1_0000_0000~0x1_9FFF_FFFF(2.5GB-32MB) * 5.VDO_UP_512MB_1: 0x1_A000_0000~0x1_BFFF_FFFF(512MB) * 6.VDO_UP_512MB_2: 0x1_C000_0000~0x1_DFFF_FFFF(512MB) * 7.VDO_UP_256MB_1: 0x1_E000_0000~0x1_EFFF_FFFF(256MB) * 8.VDO_UP_256MB_1: 0x1_F000_0000~0x1_FFFF_FFFF(256MB) */ static struct mtk_iommu_iova_region mt6835_multi_dom[] __maybe_unused = { { .iova_base = SZ_4G * 2, .size = (SZ_4G * 2), .type = NORMAL}, /*0, NORMAL */ { .iova_base = SZ_4K, .size = (SZ_4G * 1 - SZ_4K - SZ_64M), .type = NORMAL}, /* 1,CAM_MDP */ { .iova_base = 0xFC000000ULL, .size = (SZ_64M), .type = PROTECTED}, /* 2,AIE_RESV_64M */ { .iova_base = 0x106000000ULL, .size = SZ_32M, .type = NORMAL}, /* 3,LK_RESV:32MB */ { .iova_base = 0x100000000ULL, .size = 0xA0000000, .type = NORMAL}, /* 4,VDEC_VENC,JPEGENC*/ { .iova_base = 0x1A0000000ULL, .size = SZ_512M, .type = PROTECTED}, /* 5,VDO_UP_512MB_1 */ { .iova_base = 0x1C0000000ULL, .size = SZ_512M, .type = PROTECTED}, /* 6,VDO_UP_512MB_2 */ { .iova_base = 0x1E0000000ULL, .size = SZ_256M, .type = PROTECTED}, /* 7,VDO_UP_256MB_1 */ { .iova_base = 0x1F0000000ULL, .size = SZ_256M, .type = PROTECTED}, /* 8,VDO_UP_256MB_1 */ }; /* * 0,NORMAL: 0x0000_4000~0x1_05ff_ffff * 1_0800_0000 ~ 0x1_0fff_ffff * 0x2_0000_0000~0x3_FFFF_FFFF * 1.LK_RESV: 0x1_0600_0000~0x1_07FF_FFFF(32MB) * 2.VDO_UP_512MB_1: 0x1_1000_0000~0x1_2FFF_FFFF(512MB) * 3.VDO_UP_512MB_2: 0x1_3000_0000~0x1_4FFF_FFFF(512MB) * 4.VDO_UP_256MB_1: 0x1_5000_0000~0x1_5FFF_FFFF(256MB) * 5.VDO_UP_256MB_1: 0x1_6000_0000~0x1_6FFF_FFFF(256MB) * 6,VDEC: 0x1_7000_0000~0x1_FFFF_FFFF(2GB+256MB) */ static const struct mtk_iommu_iova_region mt6855_multi_dom[] = { { .iova_base = SZ_4G, .size = (SZ_4G * 3), .type = NORMAL}, /*0, NORMAL */ { .iova_base = SZ_4K, .size = (SZ_4G * 1 - SZ_4K), .type = NORMAL}, /* 1,CAM_MDP_4G-4K */ { .iova_base = 0x106000000ULL, .size = SZ_32M, .type = NORMAL}, /* 2,LK_RESV:32MB */ { .iova_base = 0x110000000ULL, .size = SZ_512M, .type = PROTECTED}, /* 3,VDO_UP_512MB_1 */ { .iova_base = 0x130000000ULL, .size = SZ_512M, .type = PROTECTED}, /* 4,VDO_UP_512MB_2 */ { .iova_base = 0x150000000ULL, .size = SZ_256M, .type = PROTECTED}, /* 5,VDO_UP_256MB_1 */ { .iova_base = 0x160000000ULL, .size = SZ_256M, .type = PROTECTED}, /* 6,VDO_UP_256MB_1 */ { .iova_base = 0x170000000ULL, .size = 0x90000000, .type = NORMAL}, /* 7,VDEC */ }; static const struct mtk_iommu_iova_region mt6873_multi_dom[] = { { .iova_base = 0x0, .size = SZ_4G}, /* disp : 0 ~ 4G */ { .iova_base = SZ_4G, .size = SZ_4G}, /* vdec : 4G ~ 8G */ { .iova_base = SZ_4G * 2, .size = SZ_4G}, /* CAM/MDP: 8G ~ 12G */ { .iova_base = 0x240000000ULL, .size = 0x4000000}, /* CCU0 */ { .iova_base = 0x244000000ULL, .size = 0x4000000}, /* CCU1 */ { .iova_base = SZ_4G * 3, .size = SZ_4G}, /* APU DATA */ { .iova_base = 0x304000000ULL, .size = 0x4000000}, /* APU VLM */ { .iova_base = 0x310000000ULL, .size = 0x10000000}, /* APU VPU */ { .iova_base = 0x370000000ULL, .size = 0x12600000}, /* APU REG */ }; /* * 0,NORMAL: total: 12GB + 96MB * 0x1000~0x1_05FF_FFFF(4GB + 96MB) * 0x2_0000_0000~0x3_FFFF_FFFF (8GB) * 1,LK_RESV: 0x1_0600_0000~0x1_07FF_FFFF * 2,CCU0: 0x1_0800_0000~0x1_0BFF_FFFF * 3,CCU1: 0x1_0C00_0000~0x1_0FFF_FFFF * 4,VDO_UP_512MB_1: 0x1_1000_0000~0x1_2FFF_FFFF * 5,VDO_UP_512MB_2: 0x1_3000_0000~0x1_4FFF_FFFF * 6,VDO_UP_256MB_1: 0x1_5000_0000~0x1_5FFF_FFFF * 7,VDO_UP_256MB_1: 0x1_6000_0000~0x1_6FFF_FFFF * 8,VDEC: 0x1_7000_0000~0x1_FFFF_FFFF */ static const struct mtk_iommu_iova_region mt6879_multi_dom_mm[] = { { .iova_base = SZ_4K, .size = (SZ_4G * 4 - SZ_4K), .type = NORMAL}, /*0. NORMAL */ { .iova_base = 0x106000000ULL, .size = SZ_32M, .type = PROTECTED}, /* 1,LK_RESV:32MB */ { .iova_base = 0x108000000ULL, .size = SZ_64M, .type = PROTECTED}, /* 2,CCU0:64MB */ { .iova_base = 0x10C000000ULL, .size = SZ_64M, .type = PROTECTED}, /* 3,CCU1:64MB */ { .iova_base = 0x110000000ULL, .size = SZ_512M, .type = PROTECTED}, /* 4,VDO_UP_512MB_1 */ { .iova_base = 0x130000000ULL, .size = SZ_512M, .type = PROTECTED}, /* 5,VDO_UP_512MB_2 */ { .iova_base = 0x150000000ULL, .size = SZ_256M, .type = PROTECTED}, /* 6,VDO_UP_256MB_1 */ { .iova_base = 0x160000000ULL, .size = SZ_256M, .type = PROTECTED}, /* 7,VDO_UP_256MB_1 */ { .iova_base = 0x170000000ULL, .size = 0x90000000, .type = NORMAL}, /* 8,VDEC */ }; /* * 0,APU_DATA(NORMAL): 12.375GB * 0x2000_0000~0x3FFF_FFFF(512MB) * 0x1_0000_0000~0x1_05FF_FFFF(96MB) * 0x1_0800_0000~0x3_FFFF_FFFF(11.875GB) * 1,APU_SECURE: 0x1000~0x1FFF_FFFF(512MB) * 2.APU_CODE: 0x4000_0000~0xFFFF_FFFF(3GB) * 3.LK_RESV: 0x1_0600_0000~0x1_07FF_FFFF(32MB) */ static const struct mtk_iommu_iova_region mt6879_multi_dom_apu[] = { { .iova_base = SZ_4K, .size = (SZ_4G * 4 - SZ_4K), .type = NORMAL}, /*0. APU_DATA */ { .iova_base = SZ_4K, .size = (SZ_512M - SZ_4K), .type = SECURE}, /* 1,APU_SECURE:512M */ { .iova_base = SZ_1G, .size = (SZ_1G * 3ULL), .type = NORMAL}, /* 2,APU_CODE:3GB */ { .iova_base = 0x106000000ULL, .size = SZ_32M, .type = NORMAL}, /* 3,LK_RESV:32MB */ }; /* * 0,NORMAL: total: 12GB + 64MB * 0x1000~0x1_03FF_FFFF(4GB + 64MB) * 0x2_0000_0000~0x3_FFFF_FFFF (8GB) * 1,GCE: 0x1_0400_0000~0x1_05FF_FFFF * 2,LK_RESV: 0x1_0600_0000~0x1_07FF_FFFF * 3,CCU0: 0x1_0800_0000~0x1_0BFF_FFFF * 4,CCU1: 0x1_0C00_0000~0x1_0FFF_FFFF * 5,VDO_UP: 0x1_1000_0000~0x1_6FFF_FFFF * 6,VDEC: 0x1_7000_0000~0x1_FFFF_FFFF */ static const struct mtk_iommu_iova_region mt6886_multi_dom_mm[] = { { .iova_base = SZ_4K, .size = (SZ_4G * 4 - SZ_4K), .type = NORMAL}, /*0. NORMAL */ { .iova_base = 0x104000000ULL, .size = SZ_32M, .type = NORMAL}, /* 1,GCE:32MB */ { .iova_base = 0x106000000ULL, .size = SZ_32M, .type = NORMAL}, /* 2,LK_RESV:32MB */ { .iova_base = 0x108000000ULL, .size = SZ_64M, .type = PROTECTED}, /* 3,CCU0:64MB */ { .iova_base = 0x10C000000ULL, .size = SZ_64M, .type = PROTECTED}, /* 4,CCU1:64MB */ { .iova_base = 0x110000000ULL, .size = 0x60000000, .type = PROTECTED}, /* 5,VDO_UP:1536M */ { .iova_base = 0x170000000ULL, .size = 0x90000000, .type = NORMAL}, /* 6,VDEC */ }; /* * 0,APU_DATA(NORMAL): 12.375GB * 0x2000_0000~0x3FFF_FFFF(512MB) * 0x1_0000_0000~0x1_05FF_FFFF(96MB) * 0x1_0800_0000~0x3_FFFF_FFFF(11.875GB) * 1,APU_SECURE: 0x1000~0x1FFF_FFFF(512MB) * 2.APU_CODE: 0x4000_0000~0xFFFF_FFFF(3GB) * 3.LK_RESV: 0x1_0600_0000~0x1_07FF_FFFF(32MB) */ static const struct mtk_iommu_iova_region mt6886_multi_dom_apu[] = { { .iova_base = SZ_4K, .size = (SZ_4G * 4 - SZ_4K), .type = NORMAL}, /*0. APU_DATA */ { .iova_base = SZ_4K, .size = (SZ_512M - SZ_4K), .type = SECURE}, /* 1,APU_SECURE:512M */ { .iova_base = SZ_1G, .size = (SZ_1G * 3ULL), .type = NORMAL}, /* 2,APU_CODE:3GB */ { .iova_base = 0x106000000ULL, .size = SZ_32M, .type = NORMAL}, /* 3,LK_RESV:32MB */ }; /* * 0.NORMAL: total: 14.04GB * -NORMAL: 0x1000~0x1FFF_FFFF(512MB) * -VDEC * -NORMAL: 0x32C0_0000~0xFFFF_FFFF(3.2GB) * -NORMAL: 0x1_0000_0000~0x1_05FF_FFFF(96MB) * -LK + CCU + Video uP * -NORMAL: 0x1_7000_0000~0x3_FFFF_FFFF(10.25GB) * 1.VDEC: 0x2000_0000~0x32BF_FFFF(300MB) * 2.LK_RESV: 0x1_0600_0000~0x1_07FF_FFFF(32MB) * 3.CCU0: 0x1_0800_0000~0x1_0BFF_FFFF(64MB) * 4.CCU1: 0x1_0C00_0000~0x1_0FFF_FFFF(64MB) * 5.VDO_UP_512MB_1: 0x1_1000_0000~0x1_2FFF_FFFF(512MB) * 6.VDO_UP_512MB_2: 0x1_3000_0000~0x1_4FFF_FFFF(512MB) * 7.VDO_UP_256MB_1: 0x1_5000_0000~0x1_5FFF_FFFF(256MB) * 8.VDO_UP_256MB_1: 0x1_6000_0000~0x1_6FFF_FFFF(256MB) */ static const struct mtk_iommu_iova_region mt6895_multi_dom_mm[] = { { .iova_base = SZ_4K, .size = (SZ_4G * 4 - SZ_4K), .type = NORMAL}, /*0. NORMAL */ { .iova_base = 0x20000000ULL, .size = 0x12c00000, .type = NORMAL}, /* 1,VDEC 300MB */ { .iova_base = 0x106000000ULL, .size = SZ_32M, .type = NORMAL}, /* 2,LK_RESV:32MB */ { .iova_base = 0x108000000ULL, .size = SZ_64M, .type = PROTECTED}, /* 3,CCU0:64MB */ { .iova_base = 0x10C000000ULL, .size = SZ_64M, .type = PROTECTED}, /* 4,CCU1:64MB */ { .iova_base = 0x110000000ULL, .size = SZ_512M, .type = PROTECTED}, /* 5,VDO_UP_512MB_1 */ { .iova_base = 0x130000000ULL, .size = SZ_512M, .type = PROTECTED}, /* 6,VDO_UP_512MB_2 */ { .iova_base = 0x150000000ULL, .size = SZ_256M, .type = PROTECTED}, /* 7,VDO_UP_256MB_1 */ { .iova_base = 0x160000000ULL, .size = SZ_256M, .type = PROTECTED}, /* 8,VDO_UP_256MB_1 */ }; /* * 0,APU_DATA(NORMAL): 12.375GB * 0x2000_0000~0x3FFF_FFFF(512MB) * 0x1_0000_0000~0x1_05FF_FFFF(96MB) * 0x1_0800_0000~0x3_FFFF_FFFF(11.875GB) * 1,APU_SECURE: 0x1000~0x1FFF_FFFF(512MB) * 2.APU_CODE: 0x4000_0000~0xFFFF_FFFF(3GB) * 3.LK_RESV: 0x1_0600_0000~0x1_07FF_FFFF(32MB) */ static const struct mtk_iommu_iova_region mt6895_multi_dom_apu[] = { { .iova_base = SZ_4K, .size = (SZ_4G * 4 - SZ_4K), .type = NORMAL},/*0. APU_DATA */ { .iova_base = SZ_4K, .size = (SZ_512M - SZ_4K), .type = SECURE}, /* 1,APU_SECURE:512M */ { .iova_base = SZ_1G, .size = (SZ_1G * 3ULL), .type = NORMAL}, /* 2,APU_CODE:3GB */ { .iova_base = 0x106000000ULL, .size = SZ_32M, .type = NORMAL}, /* 3,LK_RESV:32MB */ }; /* * 0.NORMAL: total: 14.04GB * 0x1000~0x1FFF_FFFF(512MB) * VDEC * 0x32C0_0000~0xFFFF_FFFF(3.2GB) * 0x1_0000_0000~0x1_05FF_FFFF(96MB) * CCU + Video uP * 0x1_7000_0000~0x3_FFFF_FFFF(10.25GB) * 1.VDEC: 0x2000_0000~0x32BF_FFFF(300MB) * 2.LK_RESV: 0x1_0600_0000~0x1_07FF_FFFF(32MB) * 3.CCU0: 0x1_0800_0000~0x1_0BFF_FFFF(64MB) * 4.CCU1: 0x1_0C00_0000~0x1_0FFF_FFFF(64MB) * 5.VDO_UP_512MB_1: 0x1_1000_0000~0x1_2FFF_FFFF(512MB) * 6.VDO_UP_512MB_2: 0x1_3000_0000~0x1_4FFF_FFFF(512MB) * 7.VDO_UP_256MB_1: 0x1_5000_0000~0x1_5FFF_FFFF(256MB) * 8.VDO_UP_256MB_1: 0x1_6000_0000~0x1_6FFF_FFFF(256MB) */ static const struct mtk_iommu_iova_region mt6983_multi_dom_mm[] = { { .iova_base = SZ_4K, .size = (SZ_4G * 4 - SZ_4K), .type = NORMAL}, /*0. NORMAL */ { .iova_base = 0x20000000ULL, .size = 0x12c00000, .type = NORMAL}, /* 1,VDEC 300MB */ { .iova_base = 0x106000000ULL, .size = SZ_32M, .type = NORMAL}, /* 2,LK_RESV:32MB */ { .iova_base = 0x108000000ULL, .size = SZ_64M, .type = PROTECTED}, /* 3,CCU0:64MB */ { .iova_base = 0x10C000000ULL, .size = SZ_64M, .type = PROTECTED}, /* 4,CCU1:64MB */ { .iova_base = 0x110000000ULL, .size = SZ_512M, .type = PROTECTED}, /* 5,VDO_UP_512MB_1 */ { .iova_base = 0x130000000ULL, .size = SZ_512M, .type = PROTECTED}, /* 6,VDO_UP_512MB_2 */ { .iova_base = 0x150000000ULL, .size = SZ_256M, .type = PROTECTED}, /* 7,VDO_UP_256MB_1 */ { .iova_base = 0x160000000ULL, .size = SZ_256M, .type = PROTECTED}, /* 8,VDO_UP_256MB_1 */ }; /* * 0,APU_DATA(NORMAL): 12.375GB * 0x2000_0000~0x3FFF_FFFF(512MB) * 0x1_0000_0000~0x1_05FF_FFFF(96MB) * 0x1_0800_0000~0x3_FFFF_FFFF(11.875GB) * 1,APU_SECURE: 0x1000~0x1FFF_FFFF(512MB) * 2.APU_CODE: 0x4000_0000~0xFFFF_FFFF(3GB) * 3.LK_RESV: 0x1_0600_0000~0x1_07FF_FFFF(32MB) */ static const struct mtk_iommu_iova_region mt6983_multi_dom_apu[] = { { .iova_base = SZ_4K, .size = (SZ_4G * 4 - SZ_4K), .type = NORMAL},/*0. APU_DATA */ { .iova_base = SZ_4K, .size = (SZ_512M - SZ_4K), .type = SECURE}, /* 1,APU_SECURE:512M */ { .iova_base = SZ_1G, .size = (SZ_1G * 3ULL), .type = NORMAL}, /* 2,APU_CODE:3GB */ { .iova_base = 0x106000000ULL, .size = SZ_32M, .type = NORMAL}, /* 3,LK_RESV:32MB */ }; /* * 0.NORMAL: total: 13.2GB + 96M * -NORMAL: 0x1000~0x1FFF_FFFF(512MB) * -VDEC * -NORMAL: 0x32C0_0000~0xFFFF_FFFF(3.2GB) * -NORMAL: 0x1_0000_0000~0x1_05FF_FFFF(96MB) * -LK + CCU + Video uP * -NORMAL: 0x1_7000_0000~0x3_FFFF_FFFF(10.25GB) * 1.VDEC: 0x2000_0000~0x32BF_FFFF(300MB) * 2.LK_RESV: 0x1_0600_0000~0x1_07FF_FFFF(32MB) * 3.CCU0: 0x1_0800_0000~0x1_0BFF_FFFF(64MB) * 4.CCU1: 0x1_0C00_0000~0x1_0FFF_FFFF(64MB) * 5.VDO_UP_768MB: 0x1_1000_0000~0x1_3FFF_FFFF(768M) * 6.VDO_UP_1.5G: 0x1_4000_0000~0x1_9FFF_FFFF(1.5G) */ static const struct mtk_iommu_iova_region mt6985_multi_dom_mm[] = { { .iova_base = SZ_4K, .size = (SZ_4G * 4 - SZ_4K), .type = NORMAL}, /*0. NORMAL */ { .iova_base = 0x20000000ULL, .size = 0x12c00000, .type = NORMAL}, /* 1,VDEC 300MB */ { .iova_base = 0x106000000ULL, .size = SZ_32M, .type = NORMAL}, /* 2,LK_RESV:32MB */ { .iova_base = 0x108000000ULL, .size = SZ_64M, .type = PROTECTED}, /* 3,CCU0:64MB */ { .iova_base = 0x10C000000ULL, .size = SZ_64M, .type = PROTECTED}, /* 4,CCU1:64MB */ { .iova_base = 0x110000000ULL, .size = SZ_128M * 6, .type = PROTECTED}, /* 4,VDO_UP_768MB */ { .iova_base = 0x140000000ULL, .size = 0x60000000, .type = NORMAL}, /* 5,VDO_UP_1.5G */ }; /* * 0,APU_DATA(NORMAL): 12.375GB * 0x2000_0000~0x3FFF_FFFF(512MB) * 0x1_0000_0000~0x1_05FF_FFFF(96MB) * 0x1_0800_0000~0x3_FFFF_FFFF(11.875GB) * 1,APU_SECURE: 0x1000~0x1FFF_FFFF(512MB) * 2.APU_CODE: 0x4000_0000~0xFFFF_FFFF(3GB) * 3.LK_RESV: 0x1_0600_0000~0x1_07FF_FFFF(32MB) */ static const struct mtk_iommu_iova_region mt6985_multi_dom_apu[] = { { .iova_base = SZ_4K, .size = (SZ_4G * 4 - SZ_4K), .type = NORMAL},/*0. APU_DATA */ { .iova_base = SZ_4K, .size = (SZ_512M - SZ_4K), .type = SECURE}, /* 1,APU_SECURE:512M */ { .iova_base = SZ_1G, .size = (SZ_1G * 3ULL), .type = NORMAL}, /* 2,APU_CODE:3GB */ { .iova_base = 0x106000000ULL, .size = SZ_32M, .type = NORMAL}, /* 3,LK_RESV:32MB */ }; static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova); uint64_t mtee_iova_to_phys(unsigned long iova, u32 tab_id, u32 *sr_info, u64 *pa, u32 *type, u32 *lvl) { struct arm_smccc_res smc_res; uint32_t ns_res, prot_res; arm_smccc_smc(HYP_PMM_IOVA_TO_PHYS, lower_32_bits(iova), upper_32_bits(iova), tab_id, 0, 0, 0, 0, &smc_res); /* * ns_table: a0[31:0], prot_table: a0[63:32] * smc_res.a0: include PA + SR_INFO + TYPE * smc_res.a0[2:0] = PA[34:32] * smc_res.a0[7:3] = SR_INFO * smc_res.a0[9:8] = TYPE * smc_res.a0[11:10] = lvl */ ns_res = smc_res.a0 & 0xffffffff; prot_res = (smc_res.a0 >> 32) & 0xffffffff; sr_info[NS_TAB] = FIELD_GET(GENMASK(7, 3), ns_res); pa[NS_TAB] = (FIELD_GET(GENMASK(2, 0), ns_res) << 32) | (ns_res & (~(PAGE_SIZE - 1))); type[NS_TAB] = FIELD_GET(GENMASK(9, 8), ns_res); lvl[NS_TAB] = FIELD_GET(GENMASK(11, 10), ns_res); sr_info[PROT_TAB] = FIELD_GET(GENMASK(7, 3), prot_res); pa[PROT_TAB] = (FIELD_GET(GENMASK(2, 0), prot_res) << 32) | (prot_res & (~(PAGE_SIZE - 1))); type[PROT_TAB] = FIELD_GET(GENMASK(9, 8), prot_res); lvl[PROT_TAB] = FIELD_GET(GENMASK(11, 10), prot_res); return pa[NS_TAB]; } EXPORT_SYMBOL_GPL(mtee_iova_to_phys); static int mtee_share_iova(uint64_t iova_start, uint32_t size) { struct arm_smccc_res smc_res; arm_smccc_smc(HYP_PMM_SHARE_IOVA, lower_32_bits(iova_start), upper_32_bits(iova_start), size, 0, 0, 0, 0, &smc_res); if (smc_res.a0) return -EINVAL; return 0; } static int mtee_unshare_iova(uint64_t iova_start, uint32_t size) { struct arm_smccc_res smc_res; arm_smccc_smc(HYP_PMM_UNSHARE_IOVA, lower_32_bits(iova_start), upper_32_bits(iova_start), size, 0, 0, 0, 0, &smc_res); if (smc_res.a0) return -EINVAL; return 0; } static bool mtee_hypmmu_type2_enabled(void) { struct arm_smccc_res smc_res; arm_smccc_smc(HYP_PMM_GET_HYPMMU_TYPE2_EN, 0, 0, 0, 0, 0, 0, 0, &smc_res); pr_info("%s, hyp-mmu ret:%lu\n", __func__, smc_res.a0); if (smc_res.a0 == 1) { hypmmu_type2_en = true; return true; } return false; } static int mtee_hypmmu_type2_inv(unsigned long iova, size_t size, int tab_id) { struct arm_smccc_res smc_res; u32 sa, ea; sa = MTK_IOMMU_TLB_ADDR(iova); ea = MTK_IOMMU_TLB_ADDR(iova + size - 1); arm_smccc_smc(HYP_PMM_HYPMMU_TYPE2_INV, sa, ea, tab_id, 0, 0, 0, 0, &smc_res); if (smc_res.a0) { pr_err("%s err ret:%lu", __func__, smc_res.a0); return -EINVAL; } return 0; } static int mtee_hypmmu_reg_share_region(u32 base_page_no, u32 size_in_pages, u32 tab_id) { struct arm_smccc_res smc_res; arm_smccc_smc(HYP_PMM_REG_HYPMMU_SHARE_REGION, base_page_no, size_in_pages, tab_id, 0, 0, 0, 0, &smc_res); if (smc_res.a0) { pr_err("%s err ret:%lu", __func__, smc_res.a0); WARN_ON_ONCE(smc_res.a0 != 0); return -EINVAL; } return 0; } static int iova_is_secure(struct mtk_iommu_data *data, unsigned long iova, size_t size) { int i; const struct mtk_iommu_iova_region *region; unsigned long iova_end = iova + size - 1; if (iova > iova_end) return 0; for (i = 0; i < data->plat_data->iova_region_nr; i++) { region = &data->plat_data->iova_region[i]; if (iova >= region->iova_base && iova_end < (region->iova_base + region->size) && region->type == PROTECTED) return 1; } return 0; } static int iova_secure_map(struct mtk_iommu_data *data, unsigned long iova, size_t size, bool mapped) { int ret; if (!MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_SEC_EN)) return 0; ret = iova_is_secure(data, iova, size); if (!ret) return 0; if (mapped) ret = mtee_share_iova(iova, size); else ret = mtee_unshare_iova(iova, size); if (ret) { pr_err("%s err, share_iova failed, iova:0x%lx ~ 0x%lx, mapped:%d\n", __func__, iova, (iova + size - 1), mapped); return -EINVAL; } pr_info("%s done, iova:0x%lx ~ 0x%lx, mapped:%d\n", __func__, iova, (iova + size - 1), mapped); return 0; } static int iova_secure_inv(unsigned long iova, size_t size, int tab_id) { int ret; ret = mtee_hypmmu_type2_inv(iova, size, tab_id); if (ret) { pr_err("%s err, type2_inv failed, iova:0x%lx ~ 0x%lx\n", __func__, iova, (iova + size - 1)); return -EINVAL; } return 0; } /* If 2 M4U share a domain(use the same hwlist), Put the corresponding info in first data.*/ static struct mtk_iommu_data *mtk_iommu_get_first_data(struct list_head *hwlist) { return list_first_entry(hwlist, struct mtk_iommu_data, list); } static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) { return container_of(dom, struct mtk_iommu_domain, domain); } /** * mtk_iommu_power_get - Get iommu power status, * conditionally call pm_runtime_get_if_in_use. * @data: iommu data of target device * @see #mtk_iommu_power_put * @return true if power on * * Notice: This function must be called pairs with mtk_iommu_power_put. */ static __maybe_unused bool mtk_iommu_power_get(struct mtk_iommu_data *data) { bool has_pm = !!data->dev->pm_domain; if (has_pm && !MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_CLK_AO_EN)) { if ((data->plat_data->iommu_type == MM_IOMMU && pd_sta[data->plat_data->iommu_id] == POWER_OFF_STA) || (data->plat_data->iommu_type != MM_IOMMU && pm_runtime_get_if_in_use(data->dev) <= 0)) { return false; } } return true; } /** * mtk_iommu_power_put - Put iommu power status, * conditionally call pm_runtime_put. * @data: iommu data of target device * @see #mtk_iommu_power_get * * Notice: This function must be called pairs with mtk_iommu_power_get. */ static __maybe_unused void mtk_iommu_power_put(struct mtk_iommu_data *data) { bool has_pm = !!data->dev->pm_domain; if (has_pm && !MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_CLK_AO_EN) && data->plat_data->iommu_type != MM_IOMMU) pm_runtime_put(data->dev); } static void mtk_iommu_bk0_intr_en(const struct mtk_iommu_data *data, unsigned long enable) { u32 regval; void __iomem *base = data->base; if (enable) { regval = F_L2_MULIT_HIT_EN | F_TABLE_WALK_FAULT_INT_EN | F_PREFETCH_FIFO_ERR_INT_EN | F_MISS_FIFO_ERR_INT_EN; writel_relaxed(regval, base + REG_MMU_INT_CONTROL0); regval = F_INT_TRANSLATION_FAULT | F_INT_MAIN_MULTI_HIT_FAULT | F_INT_INVALID_PA_FAULT | F_INT_ENTRY_REPLACEMENT_FAULT | F_INT_TLB_MISS_FAULT | F_INT_MISS_TRANSACTION_FIFO_FAULT; if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_MAU_EN)) regval |= F_INT_MAIN_MAU_INT_EN(data->plat_data->mau_count); writel_relaxed(regval, base + REG_MMU_INT_MAIN_CONTROL); } else { writel_relaxed(0, base + REG_MMU_INT_CONTROL0); writel_relaxed(0, base + REG_MMU_INT_MAIN_CONTROL); } } static inline void mtk_iommu_isr_setup(struct mtk_iommu_data *data, unsigned long enable) { bool has_pm = !!data->dev->pm_domain; pr_info("%s, iommu:(%d,%d), enable:%lu\n", __func__, data->plat_data->iommu_type, data->plat_data->iommu_id, enable); if (has_pm && !MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_CLK_AO_EN)) { if ((data->plat_data->iommu_type == MM_IOMMU && pd_sta[data->plat_data->iommu_id] == POWER_OFF_STA) || (data->plat_data->iommu_type != MM_IOMMU && pm_runtime_get_if_in_use(data->dev) <= 0)) { pr_info("%s, power off:%s\n", __func__, dev_name(data->dev)); return; } } mtk_iommu_bk0_intr_en(data, enable); #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_SECURE) if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_SEC_EN)) mtk_iommu_sec_bk_irq_en_by_atf(data->plat_data->iommu_type, data->plat_data->iommu_id, enable); #endif if (has_pm && !MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_CLK_AO_EN) && data->plat_data->iommu_type != MM_IOMMU) pm_runtime_put(data->dev); } static void mtk_iommu_isr_restart(struct timer_list *t) { struct mtk_iommu_data *data = from_timer(data, t, iommu_isr_pause_timer); mtk_iommu_isr_setup(data, 1); #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_DBG) mtk_iommu_debug_reset(); #endif } static int mtk_iommu_isr_pause_timer_init(struct mtk_iommu_data *data) { timer_setup(&data->iommu_isr_pause_timer, mtk_iommu_isr_restart, 0); return 0; } static int mtk_iommu_isr_pause(struct mtk_iommu_data *data, int delay) { /* disable all intr */ mtk_iommu_isr_setup(data, 0); /* delay seconds */ data->iommu_isr_pause_timer.expires = jiffies + delay * HZ; if (!timer_pending(&data->iommu_isr_pause_timer)) add_timer(&data->iommu_isr_pause_timer); return 0; } static void mtk_iommu_isr_record(struct mtk_iommu_data *data) { /* we allow one irq in 1s, or we will disable them after isr_cnt s. */ if (!data->isr_cnt || time_after(jiffies, data->first_jiffies + data->isr_cnt * HZ)) { data->isr_cnt = 1; data->first_jiffies = jiffies; } else { data->isr_cnt++; if (data->isr_cnt >= MTK_IOMMU_ISR_COUNT_MAX) { /* irq too many! disable irq for a while, to avoid HWT timeout*/ mtk_iommu_isr_pause(data, MTK_IOMMU_ISR_DISABLE_TIME); data->isr_cnt = 0; } } } static void mtk_dump_reg_for_hang_issue(struct mtk_iommu_data *data); static void mtk_iommu_tlb_flush_check(struct mtk_iommu_data *data, bool range) { u32 tlb_en = readl_relaxed(data->base + REG_MMU_INVALIDATE); u32 cpe_done = readl_relaxed(data->base + REG_MMU_CPE_DONE); pr_info("%s in, range:%d, tlb_en:0x%x, cpe_done:%d\n", __func__, range, tlb_en, cpe_done); /* check tlb inv enable bit */ if (range && (tlb_en & F_MMU_INV_RANGE)) { pr_warn("%s, TLB flush Range timed out, need to extend time!!(%d, %d)\n", __func__, data->plat_data->iommu_type, data->plat_data->iommu_id); mtk_dump_reg_for_hang_issue(data); mtk_smi_dbg_hang_detect("iommu"); pr_warn("%s, dump: 0x20:0x%x, 0x12c:0x%x\n", __func__, readl_relaxed(data->base + REG_MMU_INVALIDATE), readl_relaxed(data->base + REG_MMU_CPE_DONE)); return; } else if (!range && (tlb_en & F_ALL_INVLD)) { pr_warn("%s, TLB flush All timed out, need to extend time!!(%d, %d)\n", __func__, data->plat_data->iommu_type, data->plat_data->iommu_id); mtk_dump_reg_for_hang_issue(data); mtk_smi_dbg_hang_detect("iommu"); pr_warn("%s, dump: 0x20:0x%x, 0x12c:0x%x\n", __func__, readl_relaxed(data->base + REG_MMU_INVALIDATE), readl_relaxed(data->base + REG_MMU_CPE_DONE)); return; } /* check tlb inv done bit */ if (!cpe_done) { pr_warn("%s, subsys power status is Off!!\n", __func__); return; } pr_info("%s, TLB flush already done\n", __func__); } /* Notice!!: Before use it, must be ensure mtcmos is on */ static void mtk_iommu_tlb_flush(struct mtk_iommu_data *data) { bool has_pm = !!data->dev->pm_domain; unsigned long flags; int iommu_ids; spin_lock_irqsave(&data->tlb_lock, flags); if (has_pm && !MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_CLK_AO_EN)) { if ((data->plat_data->iommu_type == MM_IOMMU && pd_sta[data->plat_data->iommu_id] == POWER_OFF_STA) || (data->plat_data->iommu_type != MM_IOMMU && pm_runtime_get_if_in_use(data->dev) <= 0)) { spin_unlock_irqrestore(&data->tlb_lock, flags); return; } } iommu_ids = MTK_IOMMU_ID_FLAG(data->plat_data->iommu_type, data->plat_data->iommu_id); writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, data->base + data->plat_data->inv_sel_reg); writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE); wmb(); /* Make sure the tlb flush all done */ if (MTK_IOMMU_HAS_FLAG(data->plat_data, TLB_SYNC_EN)) { int ret; u32 tmp; u32 ctrl_reg = readl_relaxed(data->base + REG_MMU_CTRL_REG); u32 sync_en = ctrl_reg & F_MMU_SYNC_INVLDT_EN; if (!sync_en) { pr_info("skip flush all polling, 0x%x, (%d, %d)\n", ctrl_reg, data->plat_data->iommu_type, data->plat_data->iommu_id); goto skip_polling; } /* tlb sync */ ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE, tmp, tmp != 0, 10, 1000); if (ret) { pr_warn("TLB flush All timed out, (%d, %d)\n", data->plat_data->iommu_type, data->plat_data->iommu_id); mtk_iommu_tlb_flush_check(data, false); } /* Clear the CPE status */ writel_relaxed(0, data->base + REG_MMU_CPE_DONE); } skip_polling: if (has_pm && !MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_CLK_AO_EN) && data->plat_data->iommu_type != MM_IOMMU) pm_runtime_put(data->dev); spin_unlock_irqrestore(&data->tlb_lock, flags); #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_DBG) mtk_iommu_tlb_sync_trace(0x0, 0x1, iommu_ids); #endif } static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data *data) { struct list_head *head = data->hw_list; for_each_m4u(data, head) { mtk_iommu_tlb_flush(data); } } static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, size_t granule, struct mtk_iommu_data *data) { struct list_head *head = data->hw_list; struct mtk_iommu_data *orig_data = data; bool need_sync_all = false; unsigned long flags; int iommu_ids = 0; int ret; u32 tmp; for_each_m4u(data, head) { bool has_pm = !!data->dev->pm_domain; spin_lock_irqsave(&data->tlb_lock, flags); if (has_pm && !MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_CLK_AO_EN)) { if ((data->plat_data->iommu_type == MM_IOMMU && pd_sta[data->plat_data->iommu_id] == POWER_OFF_STA) || (data->plat_data->iommu_type != MM_IOMMU && pm_runtime_get_if_in_use(data->dev) <= 0)) { spin_unlock_irqrestore(&data->tlb_lock, flags); continue; } } iommu_ids |= MTK_IOMMU_ID_FLAG(data->plat_data->iommu_type, data->plat_data->iommu_id); writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, data->base + data->plat_data->inv_sel_reg); writel_relaxed(MTK_IOMMU_TLB_ADDR(iova), data->base + REG_MMU_INVLD_START_A); writel_relaxed(MTK_IOMMU_TLB_ADDR(iova + size - 1), data->base + REG_MMU_INVLD_END_A); writel_relaxed(F_MMU_INV_RANGE, data->base + REG_MMU_INVALIDATE); /* tlb sync */ ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE, tmp, tmp != 0, 10, 1000); if (ret) { pr_warn("Partial TLB flush timed out, (%d, %d), iova:0x%llx,0x%zx\n", data->plat_data->iommu_type, data->plat_data->iommu_id, iova, size); if (MTK_IOMMU_HAS_FLAG(data->plat_data, TLB_SYNC_EN)) mtk_iommu_tlb_flush_check(data, true); else need_sync_all = true; } /* Clear the CPE status */ writel_relaxed(0, data->base + REG_MMU_CPE_DONE); spin_unlock_irqrestore(&data->tlb_lock, flags); if (has_pm && !MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_CLK_AO_EN) && data->plat_data->iommu_type != MM_IOMMU) pm_runtime_put(data->dev); } #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_DBG) mtk_iommu_tlb_sync_trace(iova, size, iommu_ids); #endif if (need_sync_all) mtk_iommu_tlb_flush_all(orig_data); } static void mtk_iommu_dump_iova(struct mtk_iommu_data *data, enum iommu_bank bank, u64 fault_iova) { u64 iova_tmp, hw_pa[TAB_TYPE_NUM] = { 0 }; phys_addr_t fake_pa; u32 sr_info[TAB_TYPE_NUM] = { 0 }; u32 pg_type[TAB_TYPE_NUM] = { 0 }; u32 lvl[TAB_TYPE_NUM] = { 0 }; int i; for (i = 0, iova_tmp = fault_iova; i < MTK_IOMMU_TF_IOVA_DUMP_NUM; i++) { if (i > 0) iova_tmp -= SZ_4K; fake_pa = mtk_iommu_iova_to_phys(&data->m4u_dom->domain, iova_tmp); if (hypmmu_type2_en == true) hw_pa[NS_TAB] = mtee_iova_to_phys(iova_tmp, data->plat_data->tab_id, sr_info, hw_pa, pg_type, lvl); pr_err("error, type2_en:%d, index:%d, lvl:%u, pg_type:0x%x, falut_iova:0x%lx, fault_pa:0x%llx ~ 0x%llx\n", hypmmu_type2_en, i, lvl[NS_TAB], pg_type[NS_TAB], iova_tmp, (u64)fake_pa, hw_pa[NS_TAB]); if (!fake_pa && i > 0) break; } #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_DBG) /* skip dump when fault iova = 0 */ if (fault_iova) mtk_iova_map_dump(fault_iova, data->plat_data->tab_id); #endif } #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_SECURE) static irqreturn_t mtk_iommu_dump_sec_bank(struct mtk_iommu_data *data, uint32_t bank) { u32 regval; int ret; int id = data->plat_data->iommu_id; enum mtk_iommu_type type = data->plat_data->iommu_type; struct device *dev = data->bk_dev[bank]; u32 va34_32, pa34_32, fault_iova_32 = 0, fault_pa_32 = 0; u64 fault_iova, fault_pa; bool layer, write; if (!MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_SEC_EN)) { pr_warn("%s error, bank support is disabled, type:%d, id:%d, bank:%u\n", __func__, type, id, bank); return IRQ_HANDLED; } ret = mtk_iommu_secure_bk_tf_dump(type, id, bank, &fault_iova_32, &fault_pa_32, ®val); if (ret) { pr_warn("%s call to TF-A fail, type:%d, id:%d, bank:%u\n", __func__, type, id, bank); return IRQ_HANDLED; } if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOVA_34_EN)) { va34_32 = FIELD_GET(F_MMU_INVAL_VA_34_32_MASK, fault_iova_32); fault_iova = (u64)(fault_iova_32 & F_MMU_INVAL_VA_31_12_MASK); fault_iova |= (u64)va34_32 << 32; } else { fault_iova = (u64)fault_iova_32; } pa34_32 = FIELD_GET(F_MMU_INVAL_PA_34_32_MASK, fault_iova_32); fault_pa = (u64)fault_pa_32; fault_pa |= (u64)pa34_32 << 32; layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT; write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT; pr_info("%s fault iova=0x%llx pa=0x%llx layer=%d %s\n", dev_name(dev), fault_iova, fault_pa, layer, write ? "write" : "read"); mtk_iommu_dump_iova(data, bank, fault_iova); report_custom_iommu_fault(fault_iova, fault_pa, regval, type, id); mtk_iommu_tlb_flush_all(data); mtk_iommu_isr_record(data); return IRQ_HANDLED; } static irqreturn_t mtk_iommu_isr_sec(int irq, struct mtk_iommu_data *data) { uint32_t bk; for (bk = IOMMU_BK1; bk < IOMMU_BK_NUM; bk++) { if (data->bk_irq[bk] == irq) { pr_info("%s start, type:%d, id:%d, bank:%u\n", __func__, data->plat_data->iommu_type, data->plat_data->iommu_id, bk); return mtk_iommu_dump_sec_bank(data, bk); } } return IRQ_NONE; } #endif #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_SECURE) static void mtk_iommu_mau_init(struct mtk_iommu_data *data); static int mtk_iommu_mau_dump_status(struct mtk_iommu_data *data, int slave, int mau, bool aee_dump); #endif static void mtk_iommu_isr_other(struct mtk_iommu_data *data, u32 int_state0, u32 int_state1) { enum mtk_iommu_type type = data->plat_data->iommu_type; int id = data->plat_data->iommu_id; void __iomem *base = data->base; struct device *dev = data->dev; unsigned int mau_count; u32 va_33_32; u64 fault_iova; unsigned int layer; int slave_id, mau, i; if (!int_state0 && !int_state1) { pr_info("%s, iommu:(%d,%d) no error\n", __func__, type, id); return; } /* MMU Interrupt Status0 For L2 Related Interrupt Status */ if (int_state0 & F_REG_MMU_FAULT_ST0_MASK) pr_notice("%s, iommu:(%d,%d)_0 int_state0:0x%x happens\n", __func__, type, id, int_state0); /* L2 table walk fault */ if (int_state0 & F_INT_TBW_FAULT) { fault_iova = readl_relaxed(base + REG_MMU_TBWALK_FAULT_VA); layer = fault_iova & F_L2_TBWALK_FAULT_PAGE_LAYER_BIT; if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOVA_34_EN)) { va_33_32 = FIELD_GET(F_L2_TBWALK_FAULT_VA_33_32_MASK, fault_iova); fault_iova = fault_iova & F_L2_TBWALK_FAULT_VA_31_12_MASK; fault_iova |= (u64)va_33_32 << 32; } dev_warn(dev, "L2 table walk fault: iova=0x%lx, layer=%d\n", fault_iova, layer); } /* MMU Interrupt Status1 for MTLB Related Interrupt Status */ if ((int_state1 & F_INT_TRANSLATION_FAULT) == 0) { if ((int_state1 & F_REG_MMU0_FAULT_MASK) || (int_state1 & F_REG_MMU1_FAULT_MASK)) { slave_id = (int_state1 & F_REG_MMU0_FAULT_MASK) ? 0 : 1; pr_notice("%s, iommu:(%d,%d)_%d int_state1:0x%x happens\n", __func__, type, id, slave_id, int_state1); } } /* MMU Interrupt Status1 for MAU Related Interrupt Status */ if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_MAU_EN)) { mau_count = data->plat_data->mau_count; for (i = 0, slave_id = 0; i < MTK_IOMMU_MMU_COUNT; i++) { if (int_state1 & F_REG_MMU_MAU_INT_MASK(i, mau_count)) { slave_id = i; break; } } if (int_state1 & F_REG_MMU_MAU_INT_MASK(slave_id, mau_count)) { pr_notice("%s, iommu:(%d,%d)_%d int_state1:0x%x mau int happens\n", __func__, type, id, slave_id, int_state1); for (mau = 0; mau < mau_count; mau++) { if (int_state1 & F_INT_MMU_MAU_INT_STA(slave_id, mau, mau_count)) { #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_SECURE) mtk_iommu_mau_dump_status(data, slave_id, mau, false); #endif } } } } } static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) { struct mtk_iommu_data *data = dev_id; struct mtk_iommu_domain *dom = data->m4u_dom; void __iomem *base = data->base; /* bank0 base */ struct device *dev = data->dev; /* bank0 dev */ u32 int_state0, int_state1, regval, va34_32, pa34_32, table_base; u64 fault_iova, fault_pa; bool layer, write; #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_SECURE) int ret; #endif #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_DBG) int id = data->plat_data->iommu_id; enum mtk_iommu_type type = data->plat_data->iommu_type; #else unsigned int fault_larb, fault_port, sub_comm = 0; #endif #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_SECURE) if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_SEC_EN)) { if (mtk_iommu_isr_sec(irq, data) == IRQ_HANDLED) return IRQ_HANDLED; /* switch to secure debug */ ret = ao_secure_dbg_switch_by_atf(type, id, 1); if (ret) pr_notice("%s, iommu:(%d,%d) failed to enable secure debug:%d\n", __func__, type, id, ret); } #endif table_base = readl_relaxed(base + REG_MMU_PT_BASE_ADDR); /* Read error info from registers */ int_state0 = readl_relaxed(base + REG_MMU_FAULT_ST0); int_state1 = readl_relaxed(base + REG_MMU_FAULT_ST1); if (int_state1 & F_INT_TRANSLATION_FAULT) { if (int_state1 & F_INT_MMU_TF_ERR(0)) { regval = readl_relaxed(base + REG_MMU0_INT_ID); fault_iova = readl_relaxed(base + REG_MMU0_FAULT_VA); fault_pa = readl_relaxed(base + REG_MMU0_INVLD_PA); } else { regval = readl_relaxed(base + REG_MMU1_INT_ID); fault_iova = readl_relaxed(base + REG_MMU1_FAULT_VA); fault_pa = readl_relaxed(base + REG_MMU1_INVLD_PA); } layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT; write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT; pr_info("%s, iommu:(%d,%d) reg_raw_data: int_status:0x%x,0x%x, int_id:0x%x, int_va:0x%llx, int_pa:0x%llx\n", __func__, type, id, int_state0, int_state1, regval, fault_iova, fault_pa); if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOVA_34_EN)) { va34_32 = FIELD_GET(F_MMU_INVAL_VA_34_32_MASK, fault_iova); fault_iova = fault_iova & F_MMU_INVAL_VA_31_12_MASK; fault_iova |= (u64)va34_32 << 32; } pa34_32 = FIELD_GET(F_MMU_INVAL_PA_34_32_MASK, fault_iova); fault_pa |= (u64)pa34_32 << 32; #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_DBG) pr_info("%s base:0x%x fault type=0x%x iova=0x%llx pa=0x%llx layer=%d %s, pgtable=0x%x, tfrp_pa=0x%llx, 0x114=0x%x\n", dev_name(dev), table_base, int_state1, fault_iova, fault_pa, layer, (write ? "write" : "read"), dom->cfg.arm_v7s_cfg.ttbr, data->protect_base, readl_relaxed(base + REG_MMU_IVRP_PADDR)); mtk_iommu_dump_iova(data, IOMMU_BK0, fault_iova); report_custom_iommu_fault(fault_iova, fault_pa, regval, type, id); #else fault_port = F_MMU_INT_ID_PORT_ID(regval); if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_SUB_COMM)) { fault_larb = F_MMU_INT_ID_COMM_ID(regval); sub_comm = F_MMU_INT_ID_SUB_COMM_ID(regval); } else { fault_larb = F_MMU_INT_ID_LARB_ID(regval); } fault_larb = data->plat_data->larbid_remap[fault_larb][sub_comm]; if (report_iommu_fault(&dom->domain, data->dev, fault_iova, write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) { dev_err_ratelimited( dev, "fault type=0x%x iova=0x%llx pa=0x%llx larb=%d port=%d layer=%d %s\n", int_state1, fault_iova, fault_pa, fault_larb, fault_port, layer, write ? "write" : "read"); } #endif } mtk_iommu_isr_other(data, int_state0, int_state1); /* Interrupt clear */ regval = readl_relaxed(base + REG_MMU_INT_CONTROL0); regval |= F_INT_CLR_BIT; writel_relaxed(regval, base + REG_MMU_INT_CONTROL0); mtk_iommu_tlb_flush_all(data); mtk_iommu_isr_record(data); #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_SECURE) if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_SEC_EN)) { ret = ao_secure_dbg_switch_by_atf(type, id, 0); if (ret) pr_notice("%s, iommu:(%d,%d) failed to disable secure debug:%d\n", __func__, type, id, ret); } #endif return IRQ_HANDLED; } int dev_is_normal_region(struct device *dev) { struct mtk_iommu_data *data; struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); int domid; if (!fwspec) { pr_err("%s err, dev(%s) is not iommu-dev\n", __func__, dev_name(dev)); return 0; } data = dev_iommu_priv_get(dev); domid = MTK_M4U_TO_DOM(fwspec->ids[0]); pr_info("%s, domid:%d -- %u\n", __func__, domid, data->plat_data->normal_dom); return domid == data->plat_data->normal_dom; } EXPORT_SYMBOL_GPL(dev_is_normal_region); static int mtk_iommu_get_domain_id(struct device *dev, const struct mtk_iommu_plat_data *plat_data) { const struct mtk_iommu_iova_region *rgn = plat_data->iova_region; const struct bus_dma_region *dma_rgn = dev->dma_range_map; struct mtk_iommu_data *data = dev_iommu_priv_get(dev); int i, candidate = -1; dma_addr_t dma_end; if (!data) { dev_info(dev, "Get iommu data fail\n"); return -EINVAL; } if (MTK_IOMMU_HAS_FLAG(data->plat_data, GET_DOM_ID_LEGACY)) { int domid; struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); domid = MTK_M4U_TO_DOM(fwspec->ids[0]); if (domid >= plat_data->iova_region_nr) { dev_err(dev, "iommu domain id(%d/%d) is error.\n", domid, plat_data->iova_region_nr); return -EINVAL; } return domid; } if (!dma_rgn || plat_data->iova_region_nr == 1) return 0; dma_end = dma_rgn->dma_start + dma_rgn->size - 1; for (i = 0; i < plat_data->iova_region_nr; i++, rgn++) { /* Best fit. */ if (dma_rgn->dma_start == rgn->iova_base && dma_end == rgn->iova_base + rgn->size - 1) return i; /* ok if it is inside this region. */ if (dma_rgn->dma_start >= rgn->iova_base && dma_end < rgn->iova_base + rgn->size) candidate = i; } if (candidate >= 0) return candidate; dev_err(dev, "Can NOT find the iommu domain id(%pad 0x%llx).\n", &dma_rgn->dma_start, dma_rgn->size); return -EINVAL; } static void mtk_iommu_config(struct mtk_iommu_data *data, struct device *dev, bool enable, unsigned int domid) { struct mtk_smi_larb_iommu *larb_mmu; unsigned int larbid, portid; struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); const struct mtk_iommu_iova_region *region; int i; if (!fwspec) { dev_info(dev, "Get iommu fwspec fail\n"); return; } if (data->plat_data->iommu_type != MM_IOMMU || MTK_IOMMU_HAS_FLAG(data->plat_data, SKIP_CFG_PORT)) return; for (i = 0; i < fwspec->num_ids; ++i) { larbid = MTK_M4U_TO_LARB(fwspec->ids[i]); portid = MTK_M4U_TO_PORT(fwspec->ids[i]); larb_mmu = &data->larb_imu[larbid]; region = data->plat_data->iova_region + domid; larb_mmu->bank[portid] = upper_32_bits(region->iova_base); dev_dbg(dev, "%s iommu for larb(%s) port %d dom %d bank %d.\n", enable ? "enable" : "disable", dev_name(larb_mmu->dev), portid, domid, larb_mmu->bank[portid]); if (enable) larb_mmu->mmu |= MTK_SMI_MMU_EN(portid); else larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid); } } static void register_share_region(const struct mtk_iommu_plat_data *plat_data) { int i; const struct mtk_iommu_iova_region *region; u32 base_page_no; u32 size_in_pages; for (i = 0; i < plat_data->iova_region_nr; i++) { region = &plat_data->iova_region[i]; if (region->type == PROTECTED) { base_page_no = region->iova_base >> PAGE_SHIFT; size_in_pages = region->size >> PAGE_SHIFT; pr_info("%s: tab_id:%d, reg base=0x%x size=0x%x\n", __func__, plat_data->tab_id, base_page_no, size_in_pages); mtee_hypmmu_reg_share_region(base_page_no, size_in_pages, plat_data->tab_id); } } } static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom, struct mtk_iommu_data *data, unsigned int domid) { const struct mtk_iommu_iova_region *region; struct list_head *head = data->hw_list; struct mtk_iommu_data *data_temp; for_each_m4u(data_temp, head) { /* Use the exist domain as there is only one pgtable here. */ if (data_temp->m4u_dom) { dom->iop = data_temp->m4u_dom->iop; dom->cfg = data_temp->m4u_dom->cfg; dom->ttbr = data_temp->m4u_dom->ttbr; dom->domain.pgsize_bitmap = data_temp->m4u_dom->cfg.pgsize_bitmap; goto update_iova_region; } } dom->cfg = (struct io_pgtable_cfg) { .quirks = IO_PGTABLE_QUIRK_ARM_NS | IO_PGTABLE_QUIRK_NO_PERMS | IO_PGTABLE_QUIRK_ARM_MTK_EXT, .pgsize_bitmap = mtk_iommu_ops.pgsize_bitmap, .ias = MTK_IOMMU_HAS_FLAG(data->plat_data, IOVA_34_EN) ? 34 : 32, .iommu_dev = data->dev, }; if (MTK_IOMMU_HAS_FLAG(data->plat_data, PGTABLE_PA_35_EN)) dom->cfg.quirks |= IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT; if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_4GB_MODE)) dom->cfg.oas = data->enable_4GB ? 33 : 32; else dom->cfg.oas = 35; if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_SEC_EN) && mtee_hypmmu_type2_enabled()) { pr_info("hyp-mmu type2 enabled. turn on coherent_walk\n"); dom->cfg.coherent_walk = true; register_share_region(data->plat_data); } pr_info("%s, create table data:(%d,%d)\n", __func__, data->plat_data->iommu_type, data->plat_data->iommu_id); dom->iop = alloc_io_pgtable_ops(ARM_V7S, &dom->cfg, data); if (!dom->iop) { dev_err(data->dev, "Failed to alloc io pgtable\n"); return -EINVAL; } dom->ttbr = dom->cfg.quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT ? dom->cfg.arm_v7s_cfg.ttbr : dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK; /* Update our support page sizes bitmap */ dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap; update_iova_region: /* Update the iova region for this domain */ region = data->plat_data->iova_region + domid; dom->domain.geometry.aperture_start = region->iova_base; dom->domain.geometry.aperture_end = region->iova_base + region->size - 1; dom->domain.geometry.force_aperture = true; pr_info("%s, dom:%u, start:%pa, end:%pa, table:0x%x\n", __func__, domid, &dom->domain.geometry.aperture_start, &dom->domain.geometry.aperture_end, dom->cfg.arm_v7s_cfg.ttbr); return 0; } static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type) { struct mtk_iommu_domain *dom; if (type != IOMMU_DOMAIN_DMA) return NULL; dom = kzalloc(sizeof(*dom), GFP_KERNEL); if (!dom) return NULL; return &dom->domain; } static void mtk_iommu_domain_free(struct iommu_domain *domain) { kfree(to_mtk_domain(domain)); } static int mtk_iommu_set_dev_dma(struct device *dev) { int ret = 0; if (!dev) return -EINVAL; if (!dev->dma_parms) { dev->dma_parms = devm_kzalloc(dev, sizeof(*dev->dma_parms), GFP_KERNEL); if (!dev->dma_parms) return -ENOMEM; } ret = dma_set_max_seg_size(dev, (unsigned int)DMA_BIT_MASK(34)); if (ret) { dev_info(dev, "Failed to set DMA segment size\n"); return ret; } return 0; } static int mtk_iommu_attach_device(struct iommu_domain *domain, struct device *dev) { struct mtk_iommu_data *data = dev_iommu_priv_get(dev); struct mtk_iommu_domain *dom = to_mtk_domain(domain); struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct device *m4udev; int tab_id, domid, ret = 0; if (!data || !dom || !fwspec) { dev_info(dev, "Get iommu datas fail\n"); return -ENODEV; } m4udev = data->dev; tab_id = MTK_M4U_TO_TAB(fwspec->ids[0]); domid = mtk_iommu_get_domain_id(dev, data->plat_data); if (domid < 0) return domid; mutex_lock(&init_mutexs[tab_id]); if (!dom->data) { if (mtk_iommu_domain_finalise(dom, data, domid)) { ret = -ENODEV; goto out_unlock; } dom->data = data; dom->tab_id = MTK_M4U_TO_TAB(fwspec->ids[0]); pr_info("%s, set mtk_iommu_domain, data:(%d,%d), tab_id:%d\n", __func__, data->plat_data->iommu_type, data->plat_data->iommu_id, dom->tab_id); } if (!data->m4u_dom) { /* Initialize the M4U HW */ ret = pm_runtime_resume_and_get(m4udev); if (ret < 0) { dev_err(data->dev, "%s, PM fail:%d, dom:%d, iommu_dev:(%d,%d), user_dev:%s\n", __func__, ret, domid, data->plat_data->iommu_type, data->plat_data->iommu_id, dev_name(dev)); goto out_unlock; } /* * Because m4u_dom is used by mtk_iommu_isr, we must set it before * enable all banks irq to avoid m4u_dom is NULL. * ex: apu_iommu irq test. */ data->m4u_dom = dom; ret = mtk_iommu_hw_init(data); if (ret) { dev_err(data->dev, "HW init fail %d in attach\n", ret); pm_runtime_put(m4udev); goto out_unlock; } writel(data->m4u_dom->ttbr, data->base + REG_MMU_PT_BASE_ADDR); pr_info("%s, iommu_dev:%s(%d,%d), user_dev:%s, pgtable:0x%lx -- 0x%x -- 0x%x, tab_id:%d\n", __func__, dev_name(data->dev), data->plat_data->iommu_type, data->plat_data->iommu_id, dev_name(dev), (unsigned long)dom->cfg.arm_v7s_cfg.ttbr, dom->cfg.arm_v7s_cfg.ttbr, readl_relaxed(data->base + REG_MMU_PT_BASE_ADDR), dom->tab_id); #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_SECURE) if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_MAU_EN) && MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_SEC_EN)) mtk_iommu_mau_init(data); #endif pm_runtime_put(m4udev); } mtk_iommu_config(data, dev, true, domid); mtk_iommu_set_dev_dma(dev); out_unlock: mutex_unlock(&init_mutexs[tab_id]); return ret; } static void mtk_iommu_detach_device(struct iommu_domain *domain, struct device *dev) { struct mtk_iommu_data *data = dev_iommu_priv_get(dev); if (!data) { dev_info(dev, "Get iommu data fail\n"); return; } mtk_iommu_config(data, dev, false, 0); } static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot, gfp_t gfp) { int ret; struct mtk_iommu_domain *dom = to_mtk_domain(domain); int retry_count = 0; /* The "4GB mode" M4U physically can not use the lower remap of Dram. */ if (dom->data->enable_4GB) paddr |= BIT_ULL(32); /* Synchronize with the tlb_lock */ ret = dom->iop->map(dom->iop, iova, paddr, size, prot, gfp); /* retry if atomic alloc memory fail, most wait 4ms at atomic or 64ms at normal. */ while (ret == -ENOMEM && (gfp & GFP_ATOMIC) != 0 && retry_count < 8) { pr_info("%s, retry map alloc memory %d\n", __func__, retry_count + 1); if (in_atomic() || irqs_disabled() || in_interrupt()) { ret = dom->iop->map(dom->iop, iova, paddr, size, prot, gfp); } else { /* if not in atomic ctx, wait memory reclaim. */ gfp_t ignore_atomic = (gfp & ~GFP_ATOMIC) | GFP_KERNEL; ret = dom->iop->map(dom->iop, iova, paddr, size, prot, ignore_atomic); } if (ret == -ENOMEM) { retry_count++; if (in_atomic() || irqs_disabled() || in_interrupt()) { udelay(500); } else { usleep_range(8000, 10*1000); } } } if (ret) { pr_info("%s fail:%d, iommu:(%d,%d) tab_id:%d, iova:0x%lx end:0x%lx pa:%pa size:0x%zx prot:0x%x gfp:0x%x\n", __func__, ret, dom->data->plat_data->iommu_type, dom->data->plat_data->iommu_id, dom->tab_id, iova, (iova + size - 1), &paddr, size, prot, gfp); if (ret != -ENOMEM) mtk_iommu_dump_iova(dom->data, IOMMU_BK0, iova); } return ret; } static size_t mtk_iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size, struct iommu_iotlb_gather *gather) { size_t ret; struct mtk_iommu_domain *dom = to_mtk_domain(domain); iommu_iotlb_gather_add_range(gather, iova, size); ret = dom->iop->unmap(dom->iop, iova, size, gather); if (!ret) { pr_info("%s fail:%d, iommu:(%d,%d) tab_id:%d, iova:0x%lx end:0x%lx size:0x%zx\n", __func__, ret, dom->data->plat_data->iommu_type, dom->data->plat_data->iommu_id, dom->tab_id, iova, (iova + size - 1), size); mtk_iommu_dump_iova(dom->data, IOMMU_BK0, iova); } return ret; } static void mtk_iommu_flush_iotlb_all(struct iommu_domain *domain) { struct mtk_iommu_domain *dom = to_mtk_domain(domain); mtk_iommu_tlb_flush_all(dom->data); } static void mtk_iommu_iotlb_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) { struct mtk_iommu_domain *dom = to_mtk_domain(domain); size_t length = gather->end - gather->start + 1; int ret; if (gather->start > gather->end) { pr_err("unmap invalid iova range : 0x%lx ~ 0x%lx\n", gather->start, gather->end); return; } #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_DBG) if (gather->start > 0 && gather->start != ULONG_MAX) mtk_iova_unmap(dom->tab_id, gather->start, length); #endif if (!hypmmu_type2_en) { /* clean bank1 */ ret = iova_secure_map(dom->data, gather->start, length, false); if (ret) pr_warn("%s failed\n", __func__); } else { ret = iova_secure_inv(gather->start, length, dom->tab_id); if (ret) pr_warn("%s failed\n", __func__); } mtk_iommu_tlb_flush_range_sync(gather->start, length, gather->pgsize, dom->data); } static void mtk_iommu_sync_map(struct iommu_domain *domain, unsigned long iova, size_t size) { struct mtk_iommu_domain *dom = to_mtk_domain(domain); int ret; if (iova > (iova + size)) { pr_err("map invalid iova range : 0x%lx ~ 0x%lx\n", iova, iova + size); return; } #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_DBG) if (iova > 0 && iova != ULONG_MAX) mtk_iova_map(dom->tab_id, iova, size); #endif if (!hypmmu_type2_en) { ret = iova_secure_map(dom->data, iova, size, true); if (ret) pr_warn("%s failed\n", __func__); } else { ret = iova_secure_inv(iova, size, dom->tab_id); if (ret) pr_warn("%s failed\n", __func__); } mtk_iommu_tlb_flush_range_sync(iova, size, size, dom->data); } static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { struct mtk_iommu_domain *dom = to_mtk_domain(domain); phys_addr_t pa; pa = dom->iop->iova_to_phys(dom->iop, iova); if (IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT) && dom->data->enable_4GB && pa >= MTK_IOMMU_4GB_MODE_REMAP_BASE) pa &= ~BIT_ULL(32); return pa; } static struct iommu_device *mtk_iommu_probe_device(struct device *dev) { struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct mtk_iommu_data *data; unsigned int larbid, larbidx, i; if (!fwspec || fwspec->ops != &mtk_iommu_ops) return ERR_PTR(-ENODEV); /* Not a iommu client device */ data = dev_iommu_priv_get(dev); /* * The device that connects with each a larb is a independent HW. * All the ports in each a device should be in the same larbs. */ larbid = MTK_M4U_TO_LARB(fwspec->ids[0]); for (i = 1; i < fwspec->num_ids; i++) { larbidx = MTK_M4U_TO_LARB(fwspec->ids[i]); if (larbid != larbidx) { dev_err(dev, "Can only use one larb. Fail@larb%d-%d.\n", larbid, larbidx); return ERR_PTR(-EINVAL); } } return &data->iommu; } static void mtk_iommu_release_device(struct device *dev) { struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); if (!fwspec || fwspec->ops != &mtk_iommu_ops) return; iommu_fwspec_free(dev); } static struct iommu_group *mtk_iommu_device_group(struct device *dev) { struct mtk_iommu_data *c_data, *data; struct list_head *hw_list; struct iommu_group *group; int domid; c_data = dev_iommu_priv_get(dev); if (!c_data) { dev_info(dev, "Get iommu data fail\n"); return ERR_PTR(-ENODEV); } hw_list = c_data->hw_list; data = mtk_iommu_get_first_data(hw_list); if (!data) return ERR_PTR(-ENODEV); domid = mtk_iommu_get_domain_id(dev, data->plat_data); if (domid < 0) return ERR_PTR(domid); mutex_lock(&group_mutexs[domid]); group = data->m4u_group[domid]; if (!group) { pr_info("%s create group, data:(%d,%d)-->(%d,%d), dev:%s, domid:%d\n", __func__, c_data->plat_data->iommu_type, c_data->plat_data->iommu_id, data->plat_data->iommu_type, data->plat_data->iommu_id, dev_name(dev), domid); group = iommu_group_alloc(); if (!IS_ERR(group)) data->m4u_group[domid] = group; else dev_err(dev, "Failed to allocate M4U IOMMU group\n"); } else { iommu_group_ref_get(group); } mutex_unlock(&group_mutexs[domid]); return group; } static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) { struct platform_device *m4updev; if (args->args_count != 1) { dev_err(dev, "invalid #iommu-cells(%d) property for IOMMU\n", args->args_count); return -EINVAL; } if (!dev_iommu_priv_get(dev)) { /* Get the m4u device */ m4updev = of_find_device_by_node(args->np); if (WARN_ON(!m4updev)) return -EINVAL; dev_iommu_priv_set(dev, platform_get_drvdata(m4updev)); } return iommu_fwspec_add_ids(dev, args->args, 1); } static void mtk_iommu_get_resv_regions(struct device *dev, struct list_head *head) { struct mtk_iommu_data *data = dev_iommu_priv_get(dev); const struct mtk_iommu_iova_region *resv, *curdom; struct iommu_resv_region *region; int prot = IOMMU_WRITE | IOMMU_READ; unsigned int domid, i; if (!data) { dev_info(dev, "Get iommu data fail\n"); return; } domid = mtk_iommu_get_domain_id(dev, data->plat_data); if ((int)domid < 0) return; curdom = data->plat_data->iova_region + domid; for (i = 0; i < data->plat_data->iova_region_nr; i++) { resv = data->plat_data->iova_region + i; /* Only reserve when the region is inside the current domain */ if (resv->iova_base <= curdom->iova_base || resv->iova_base + resv->size >= curdom->iova_base + curdom->size) continue; region = iommu_alloc_resv_region(resv->iova_base, resv->size, prot, IOMMU_RESV_RESERVED); if (!region) return; list_add_tail(®ion->list, head); } } static const struct iommu_ops mtk_iommu_ops = { .domain_alloc = mtk_iommu_domain_alloc, .domain_free = mtk_iommu_domain_free, .attach_dev = mtk_iommu_attach_device, .detach_dev = mtk_iommu_detach_device, .map = mtk_iommu_map, .unmap = mtk_iommu_unmap, .flush_iotlb_all = mtk_iommu_flush_iotlb_all, .iotlb_sync = mtk_iommu_iotlb_sync, .iotlb_sync_map = mtk_iommu_sync_map, .iova_to_phys = mtk_iommu_iova_to_phys, .probe_device = mtk_iommu_probe_device, .release_device = mtk_iommu_release_device, .device_group = mtk_iommu_device_group, .of_xlate = mtk_iommu_of_xlate, .get_resv_regions = mtk_iommu_get_resv_regions, .put_resv_regions = generic_iommu_put_resv_regions, .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M, .owner = THIS_MODULE, }; static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) { u32 regval; int i; if (data->plat_data->m4u_plat == M4U_MT8173) { regval = F_MMU_PREFETCH_RT_REPLACE_MOD | F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173; } else { regval = readl_relaxed(data->base + REG_MMU_CTRL_REG); if (MTK_IOMMU_HAS_FLAG(data->plat_data, TLB_SYNC_EN)) regval |= F_MMU_SYNC_INVLDT_EN; else regval |= F_MMU_TF_PROT_TO_PROGRAM_ADDR; regval |= F_MMU_MON_EN; } writel_relaxed(regval, data->base + REG_MMU_CTRL_REG); if (data->enable_4GB && MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_VLD_PA_RNG)) { /* * If 4GB mode is enabled, the validate PA range is from * 0x1_0000_0000 to 0x1_ffff_ffff. here record bit[32:30]. */ regval = F_MMU_VLD_PA_RNG(7, 4); writel_relaxed(regval, data->base + REG_MMU_VLD_PA_RNG); } writel_relaxed(0, data->base + REG_MMU_DCM_DIS); if (MTK_IOMMU_HAS_FLAG(data->plat_data, WR_THROT_EN)) { /* write command throttling mode */ regval = readl_relaxed(data->base + REG_MMU_WR_LEN_CTRL); regval &= ~F_MMU_WR_THROT_DIS_MASK; writel_relaxed(regval, data->base + REG_MMU_WR_LEN_CTRL); } if (MTK_IOMMU_HAS_FLAG(data->plat_data, RESET_AXI)) { /* The register is called STANDARD_AXI_MODE in this case */ regval = 0; } else { regval = readl_relaxed(data->base + REG_MMU_MISC_CTRL); if (MTK_IOMMU_HAS_FLAG(data->plat_data, NOT_STD_AXI_MODE)) regval &= ~F_MMU_STANDARD_AXI_MODE_MASK; if (MTK_IOMMU_HAS_FLAG(data->plat_data, OUT_ORDER_WR_EN)) regval &= ~F_MMU_IN_ORDER_WR_EN_MASK; } writel_relaxed(regval, data->base + REG_MMU_MISC_CTRL); if (MTK_IOMMU_HAS_FLAG(data->plat_data, SET_TBW_ID)) writel_relaxed(data->plat_data->tbw_reg_val, data->base + REG_MMU_TBW_ID); mtk_iommu_bk0_intr_en(data, 1); if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_LEGACY_IVRP_PADDR)) regval = (data->protect_base >> 1) | (data->enable_4GB << 31); else regval = lower_32_bits(data->protect_base) | upper_32_bits(data->protect_base); writel_relaxed(regval, data->base + REG_MMU_IVRP_PADDR); #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_SECURE) if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_SEC_EN)) mtk_iommu_sec_bk_init_by_atf(data->plat_data->iommu_type, data->plat_data->iommu_id); #endif for (i = IOMMU_BK0; i < IOMMU_BK_NUM; i++) { struct device *dev; unsigned int irq; if (i == IOMMU_BK0) { dev = data->dev; irq = data->irq; } else { if (!MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_SEC_EN)) break; dev = data->bk_dev[i]; irq = data->bk_irq[i]; } if (!irq) { pr_err("%s error, irq is 0(%d,%d,%d)\n", __func__, data->plat_data->iommu_type, data->plat_data->iommu_id, i); continue; } if (devm_request_irq(dev, irq, mtk_iommu_isr, 0, dev_name(dev), (void *)data)) { if (i == IOMMU_BK0) writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR); dev_err(dev, "Failed @ IRQ-%d Request\n", irq); return -ENODEV; } pr_info("%s, register irq done, (%d,%d,%d) %d\n", __func__, data->plat_data->iommu_type, data->plat_data->iommu_id, i, irq); } pr_info("%s done, (%d,%d), dump reg: 0x48:0x%x, 0x50:0x%x, 0x54:0x%x, 0xa0:0x%x, 0x110:0x%x, 0x114:0x%x, 0x120:0x%x, 0x124:0x%x\n", __func__, data->plat_data->iommu_type, data->plat_data->iommu_id, readl_relaxed(data->base + REG_MMU_MISC_CTRL), readl_relaxed(data->base + REG_MMU_DCM_DIS), readl_relaxed(data->base + REG_MMU_WR_LEN_CTRL), readl_relaxed(data->base + REG_MMU_TBW_ID), readl_relaxed(data->base + REG_MMU_CTRL_REG), readl_relaxed(data->base + REG_MMU_IVRP_PADDR), readl_relaxed(data->base + REG_MMU_INT_CONTROL0), readl_relaxed(data->base + REG_MMU_INT_MAIN_CONTROL)); return 0; } static const struct component_master_ops mtk_iommu_com_ops = { .bind = mtk_iommu_bind, .unbind = mtk_iommu_unbind, }; static int mtk_iommu_pd_callback(struct notifier_block *nb, unsigned long flags, void *data) { unsigned long lock_flags; if (nb->priority < 0) return NOTIFY_DONE; spin_lock_irqsave(tlb_locks[nb->priority], lock_flags); if (flags == GENPD_NOTIFY_ON) pd_sta[nb->priority] = POWER_ON_STA; else if (flags == GENPD_NOTIFY_PRE_OFF) pd_sta[nb->priority] = POWER_OFF_STA; spin_unlock_irqrestore(tlb_locks[nb->priority], lock_flags); #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_DBG) if (flags == GENPD_NOTIFY_ON || flags == GENPD_NOTIFY_PRE_OFF) { mtk_iommu_pm_trace(((flags == GENPD_NOTIFY_ON) ? IOMMU_POWER_ON : IOMMU_POWER_OFF), nb->priority, pd_sta[nb->priority], flags, NULL); } #endif return NOTIFY_OK; } static int mtk_iommu_dbg_hang_cb(struct notifier_block *nb, unsigned long action, void *data) { if (!IS_ERR_OR_NULL(data) && strcmp((char *) data, "iommu") == 0) return NOTIFY_DONE; mtk_iommu_dbg_hang_detect(MM_IOMMU, DISP_IOMMU); mtk_iommu_dbg_hang_detect(MM_IOMMU, MDP_IOMMU); return NOTIFY_OK; } static int register_dbg_notifier; static struct notifier_block mtk_iommu_dbg_hang_nb = { .notifier_call = mtk_iommu_dbg_hang_cb, }; /********** mtk iommu MAU start **********/ static inline void iommu_set_field_by_mask(void __iomem *m4u_base, unsigned int reg, unsigned long mask, unsigned int val) { unsigned int regval; regval = readl_relaxed(m4u_base + reg); regval = (regval & (~mask)) | val; writel_relaxed(regval, m4u_base + reg); } static inline unsigned int iommu_get_field_by_mask(void __iomem *m4u_base, unsigned int reg, unsigned int mask) { return readl_relaxed(m4u_base + reg) & mask; } #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_SECURE) static int mtk_iommu_mau_start_monitor(struct mtk_iommu_data *data, unsigned int slave, unsigned int mau, const struct mau_config_info *mau_cfg) { unsigned int mau_count = data->plat_data->mau_count; enum mtk_iommu_type type = data->plat_data->iommu_type; int id = data->plat_data->iommu_id; unsigned long flags; void __iomem *base; int ret; if (slave >= MTK_IOMMU_MMU_COUNT || mau >= mau_count) { pr_notice("%s, invalid iommu:(%d,%d), slave:%d, mau:%d\n", __func__, type, id, slave, mau); return -1; } spin_lock_irqsave(&data->tlb_lock, flags); if (!mtk_iommu_power_get(data)) { pr_notice("%s, iommu:(%d,%d) power off dev:%s\n", __func__, type, id, dev_name(data->dev)); spin_unlock_irqrestore(&data->tlb_lock, flags); return 0; } /* switch to secure debug */ ret = ao_secure_dbg_switch_by_atf(type, id, 1); if (ret) { mtk_iommu_power_put(data); spin_unlock_irqrestore(&data->tlb_lock, flags); pr_notice("%s, iommu:(%d,%d) failed to enable secure debug:%d\n", __func__, type, id, ret); return ret; } base = data->base; /*enable interrupt*/ iommu_set_field_by_mask(base, REG_MMU_INT_MAIN_CONTROL, F_INT_MMU_MAU_INT_EN(slave, mau, mau_count), F_INT_MMU_MAU_INT_EN(slave, mau, mau_count)); /*config start addr*/ writel_relaxed(mau_cfg->start, base + REG_MMU_MAU_SA(slave, mau)); writel_relaxed(mau_cfg->start_bit32, base + REG_MMU_MAU_SA_EXT(slave, mau)); /*config end addr*/ writel_relaxed(mau_cfg->end, base + REG_MMU_MAU_EA(slave, mau)); writel_relaxed(mau_cfg->end_bit32, base + REG_MMU_MAU_EA_EXT(slave, mau)); /*config larb id*/ writel_relaxed(mau_cfg->larb_mask, base + REG_MMU_MAU_LARB_EN(slave, mau)); /*config port id*/ writel_relaxed(mau_cfg->port_mask, base + REG_MMU_MAU_PORT_EN(slave, mau)); /*config input/output*/ iommu_set_field_by_mask(base, REG_MMU_MAU_IO(slave, mau_count), F_MMU_MAU_BIT_VAL(1, mau), F_MMU_MAU_BIT_VAL(mau_cfg->io, mau)); /*config read/write*/ iommu_set_field_by_mask(base, REG_MMU_MAU_RW(slave, mau_count), F_MMU_MAU_BIT_VAL(1, mau), F_MMU_MAU_BIT_VAL(mau_cfg->wr, mau)); /*config PA/VA*/ iommu_set_field_by_mask(base, REG_MMU_MAU_VA(slave, mau_count), F_MMU_MAU_BIT_VAL(1, mau), F_MMU_MAU_BIT_VAL(mau_cfg->virt, mau)); wmb(); /*make sure the MAU ops has been triggered*/ pr_notice("%s, iommu:(%d,%d), slave:%d, mau:%d, mau_reg: start=0x%x(0x%x), end=0x%x(0x%x), wr:0x%x, virt:0x%x, io:0x%x, larb:0x%x, port:0x%x\n", __func__, type, id, slave, mau, readl_relaxed(base + REG_MMU_MAU_SA(slave, mau)), readl_relaxed(base + REG_MMU_MAU_SA_EXT(slave, mau)), readl_relaxed(base + REG_MMU_MAU_EA(slave, mau)), readl_relaxed(base + REG_MMU_MAU_EA_EXT(slave, mau)), readl_relaxed(base + REG_MMU_MAU_RW(slave, mau_count)), readl_relaxed(base + REG_MMU_MAU_VA(slave, mau_count)), readl_relaxed(base + REG_MMU_MAU_IO(slave, mau_count)), readl_relaxed(base + REG_MMU_MAU_LARB_EN(slave, mau)), readl_relaxed(base + REG_MMU_MAU_PORT_EN(slave, mau))); ret = ao_secure_dbg_switch_by_atf(type, id, 0); if (ret) pr_notice("%s, iommu:(%d,%d) failed to disable secure debug:%d\n", __func__, type, id, ret); mtk_iommu_power_put(data); spin_unlock_irqrestore(&data->tlb_lock, flags); return 0; } /* Notice: must fill cfg->slave/mau before call this func. */ static int mtk_iommu_mau_get_config(struct mtk_iommu_data *data, struct mau_config_info *mau_cfg) { unsigned int mau_count = data->plat_data->mau_count; enum mtk_iommu_type type = data->plat_data->iommu_type; int id = data->plat_data->iommu_id; int slave = mau_cfg->slave; int mau = mau_cfg->mau; void __iomem *base; if (slave >= MTK_IOMMU_MMU_COUNT || mau >= mau_count) { pr_notice("%s, invalid iommu:(%d,%d), slave:%d, mau:%d\n", __func__, type, id, slave, mau); return -1; } base = data->base; mau_cfg->start = readl_relaxed(base + REG_MMU_MAU_SA(slave, mau)); mau_cfg->end = readl_relaxed(base + REG_MMU_MAU_EA(slave, mau)); mau_cfg->start_bit32 = readl_relaxed(base + REG_MMU_MAU_SA_EXT(slave, mau)); mau_cfg->end_bit32 = readl_relaxed(base + REG_MMU_MAU_EA_EXT(slave, mau)); mau_cfg->port_mask = readl_relaxed(base + REG_MMU_MAU_PORT_EN(slave, mau)); mau_cfg->larb_mask = readl_relaxed(base + REG_MMU_MAU_LARB_EN(slave, mau)); mau_cfg->io = !!(iommu_get_field_by_mask(base, REG_MMU_MAU_IO(slave, mau_count), F_MMU_MAU_BIT_VAL(1, mau))); mau_cfg->wr = !!(iommu_get_field_by_mask(base, REG_MMU_MAU_RW(slave, mau_count), F_MMU_MAU_BIT_VAL(1, mau))); mau_cfg->virt = !!(iommu_get_field_by_mask(base, REG_MMU_MAU_VA(slave, mau_count), F_MMU_MAU_BIT_VAL(1, mau))); return 0; } static int mtk_iommu_mau_dump_status(struct mtk_iommu_data *data, int slave, int mau, bool aee_dump) { unsigned int mau_count = data->plat_data->mau_count; enum mtk_iommu_type type = data->plat_data->iommu_type; int id = data->plat_data->iommu_id; void __iomem *base; unsigned long flags; unsigned int status; unsigned int falut_id, assert_id, assert_address, assert_b32; struct mau_config_info mau_cfg; char *port_name = NULL; int ret; if (slave >= MTK_IOMMU_MMU_COUNT || mau >= mau_count) { pr_notice("%s, invalid iommu:(%d,%d), slave:%d, mau:%d\n", __func__, type, id, slave, mau); return -1; } spin_lock_irqsave(&data->tlb_lock, flags); if (!mtk_iommu_power_get(data)) { pr_notice("%s, iommu:(%d,%d), slave:%d, mau:%d, power off dev:%s\n", __func__, type, id, slave, mau, dev_name(data->dev)); spin_unlock_irqrestore(&data->tlb_lock, flags); return 0; } base = data->base; status = readl_relaxed(base + REG_MMU_MAU_ASRT_STA(slave, mau_count)); if (status & (1 << mau)) { pr_notice("%s: iommu:(%d,%d), slave:%d, mau_assert:%d, status:0x%x\n", __func__, type, id, slave, mau, status); assert_id = readl_relaxed(base + REG_MMU_MAU_ASRT_ID(slave, mau)); assert_address = readl_relaxed(base + REG_MMU_MAU_AA(slave, mau)); assert_b32 = readl_relaxed(base + REG_MMU_MAU_AA_EXT(slave, mau)); falut_id = (assert_id & F_MMU_MAU_ASRT_ID_VAL) << 2; #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_DBG) port_name = mtk_iommu_get_port_name(type, id, falut_id); if (aee_dump) report_iommu_mau_fault(assert_id, falut_id, (port_name ? port_name : "port_unknown"), assert_address, assert_b32); #endif pr_notice("%s: ASRT_ID=0x%x, FALUT_ID=0x%x(%s), AA=0x%x, AA_EXT=0x%x\n", __func__, assert_id, falut_id, (port_name ? port_name : "port_unknown"), assert_address, assert_b32); /* Clear bits for MAU set */ writel_relaxed((1 << mau), base + REG_MMU_MAU_CLR(slave, mau_count)); writel_relaxed(0, base + REG_MMU_MAU_CLR(slave, mau_count)); wmb(); /*make sure the MAU data is cleared*/ mau_cfg.slave = slave; mau_cfg.mau = mau; ret = mtk_iommu_mau_get_config(data, &mau_cfg); if (!ret) { pr_info("%s, iommu:(%d,%d), slave:%d, mau:%d, mau_cfg: start=0x%x(0x%x), end=0x%x(0x%x), wr:0x%x, virt:0x%x, io:0x%x, larb:0x%x, port:0x%x\n", __func__, type, id, slave, mau, mau_cfg.start, mau_cfg.start_bit32, mau_cfg.end, mau_cfg.end_bit32, mau_cfg.wr, mau_cfg.virt, mau_cfg.io, mau_cfg.larb_mask, mau_cfg.port_mask); } } else pr_notice("%s: mau no assert in MAU set %d, status:0x%x\n", __func__, mau, status); mtk_iommu_power_put(data); spin_unlock_irqrestore(&data->tlb_lock, flags); return 0; } static int mtk_iommu_mau_reg_backup(struct mtk_iommu_data *data) { unsigned int mau_count = data->plat_data->mau_count; enum mtk_iommu_type type = data->plat_data->iommu_type; struct mtk_iommu_suspend_reg *reg = &data->reg; int id = data->plat_data->iommu_id; unsigned int *p_reg, *p_reg_base; void __iomem *base = data->base; unsigned int real_size; int slave, mau, ret; if (!reg->mau) { pr_notice("%s, iommu:(%d,%d) no memory for backup\n", __func__, type, id); return -1; } /* switch to secure debug */ ret = ao_secure_dbg_switch_by_atf(type, id, 1); if (ret) { pr_notice("%s, iommu:(%d,%d) failed to enable secure debug:%d\n", __func__, type, id, ret); return ret; } p_reg_base = reg->mau; p_reg = reg->mau; for (slave = 0; slave < MTK_IOMMU_MMU_COUNT; slave++) { for (mau = 0; mau < mau_count; mau++) { *(p_reg++) = readl_relaxed(base + REG_MMU_MAU_SA(slave, mau)); *(p_reg++) = readl_relaxed(base + REG_MMU_MAU_SA_EXT(slave, mau)); *(p_reg++) = readl_relaxed(base + REG_MMU_MAU_EA(slave, mau)); *(p_reg++) = readl_relaxed(base + REG_MMU_MAU_EA_EXT(slave, mau)); *(p_reg++) = readl_relaxed(base + REG_MMU_MAU_LARB_EN(slave, mau)); *(p_reg++) = readl_relaxed(base + REG_MMU_MAU_PORT_EN(slave, mau)); } *(p_reg++) = readl_relaxed(base + REG_MMU_MAU_IO(slave, mau_count)); *(p_reg++) = readl_relaxed(base + REG_MMU_MAU_RW(slave, mau_count)); *(p_reg++) = readl_relaxed(base + REG_MMU_MAU_VA(slave, mau_count)); } /* disable secure debug */ ret = ao_secure_dbg_switch_by_atf(type, id, 0); if (ret) pr_notice("%s, iommu:(%d,%d) failed to disable secure debug:%d\n", __func__, type, id, ret); /* check register size (to prevent overflow) */ real_size = (p_reg - p_reg_base) * sizeof(unsigned int); if (real_size > MMU_MAU_REG_BACKUP_SIZE) pr_info("%s error: overflow! %d>%d\n", __func__, real_size, (int)MMU_MAU_REG_BACKUP_SIZE); reg->mau_real_size = real_size; return 0; } static int mtk_iommu_mau_reg_restore(struct mtk_iommu_data *data) { unsigned int mau_count = data->plat_data->mau_count; enum mtk_iommu_type type = data->plat_data->iommu_type; int id = data->plat_data->iommu_id; struct mtk_iommu_suspend_reg *reg = &data->reg; unsigned int *p_reg, *p_reg_base; void __iomem *base = data->base; unsigned int real_size; int slave, mau, ret; if (!reg->mau) { pr_notice("%s failed, iommu:(%d,%d) no memory for restore\n", __func__, type, id); return -1; } /* switch to secure debug */ ret = ao_secure_dbg_switch_by_atf(type, id, 1); if (ret) { pr_notice("%s, iommu:(%d,%d) failed to enable secure debug:%d\n", __func__, type, id, ret); return ret; } p_reg_base = reg->mau; p_reg = reg->mau; for (slave = 0; slave < MTK_IOMMU_MMU_COUNT; slave++) { for (mau = 0; mau < mau_count; mau++) { writel_relaxed(*(p_reg++), base + REG_MMU_MAU_SA(slave, mau)); writel_relaxed(*(p_reg++), base + REG_MMU_MAU_SA_EXT(slave, mau)); writel_relaxed(*(p_reg++), base + REG_MMU_MAU_EA(slave, mau)); writel_relaxed(*(p_reg++), base + REG_MMU_MAU_EA_EXT(slave, mau)); writel_relaxed(*(p_reg++), base + REG_MMU_MAU_LARB_EN(slave, mau)); writel_relaxed(*(p_reg++), base + REG_MMU_MAU_PORT_EN(slave, mau)); } writel_relaxed(*(p_reg++), base + REG_MMU_MAU_IO(slave, mau_count)); writel_relaxed(*(p_reg++), base + REG_MMU_MAU_RW(slave, mau_count)); writel_relaxed(*(p_reg++), base + REG_MMU_MAU_VA(slave, mau_count)); } wmb(); /*make sure the MVA data is restored*/ /* disable secure debug */ ret = ao_secure_dbg_switch_by_atf(type, id, 0); if (ret) pr_notice("%s, iommu:(%d,%d) failed to disable secure debug:%d\n", __func__, type, id, ret); /* check register size (to prevent overflow) */ real_size = (p_reg - p_reg_base) * sizeof(unsigned int); if (real_size != reg->mau_real_size) pr_notice("%s error: %d!=%d\n", __func__, real_size, reg->mau_real_size); return 0; } static void mtk_iommu_mau_init(struct mtk_iommu_data *data) { enum mtk_iommu_type type = data->plat_data->iommu_type; int id = data->plat_data->iommu_id; const struct mau_config_info *mau_config; unsigned int slave, mau; pr_info("%s, iommu dev:%s\n", __func__, dev_name(data->dev)); #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_DBG) for (slave = 0; slave < MTK_IOMMU_MMU_COUNT; slave++) { for (mau = 0; mau < data->plat_data->mau_count; mau++) { mau_config = mtk_iommu_get_mau_config(type, id, slave, mau); if (mau_config != NULL) mtk_iommu_mau_start_monitor(data, slave, mau, mau_config); } } #endif } #endif /**********mtk iommu mau end**********/ static int mtk_iommu_probe(struct platform_device *pdev) { struct mtk_iommu_data *data; struct device *dev = &pdev->dev; struct device_node *larbnode, *smicomm_node; struct platform_device *plarbdev; struct device_link *link; struct resource *res; resource_size_t ioaddr; struct component_match *match = NULL; struct regmap *infracfg; void *protect; int i, larb_nr, ret; u32 val; char *p; bool disp_power_on = 0; pr_info("%s start dev:%s\n", __func__, dev_name(dev)); data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; data->dev = dev; data->plat_data = of_device_get_match_data(dev); if (!atomic_cmpxchg(&init_once_flag, 0, 1)) { for (i = 0; i < PGTBALE_NUM; i++) mutex_init(&init_mutexs[i]); for (i = 0; i < MTK_IOMMU_GROUP_MAX; i++) mutex_init(&group_mutexs[i]); } /* Protect memory. HW will access here while translation fault.*/ protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, GFP_KERNEL); if (!protect) return -ENOMEM; data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN); if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_4GB_MODE)) { switch (data->plat_data->m4u_plat) { case M4U_MT2712: p = "mediatek,mt2712-infracfg"; break; case M4U_MT8173: p = "mediatek,mt8173-infracfg"; break; default: p = NULL; } infracfg = syscon_regmap_lookup_by_compatible(p); if (IS_ERR(infracfg)) return PTR_ERR(infracfg); ret = regmap_read(infracfg, REG_INFRA_MISC, &val); if (ret) return ret; data->enable_4GB = !!(val & F_DDR_4GB_SUPPORT_EN); } if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_MAU_EN)) { struct mtk_iommu_suspend_reg *reg = &data->reg; reg->mau = kmalloc(MMU_MAU_REG_BACKUP_SIZE, GFP_KERNEL | __GFP_ZERO); if (!reg->mau) return -ENOMEM; } res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!res) return -EINVAL; data->base = devm_ioremap_resource(dev, res); if (IS_ERR(data->base)) return PTR_ERR(data->base); ioaddr = res->start; if (!MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_NO_IRQ)) { data->irq = platform_get_irq(pdev, 0); if (data->irq < 0) { pr_err("%s, %s(%d,%d) can not irq!!\n", __func__, dev_name(dev), data->plat_data->iommu_type, data->plat_data->iommu_id); return data->irq; } } /* * Note: we must be find iommu bank from bank1; * And if iommu upstream, we need to merged with bank0. */ if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_SEC_EN)) { int bk_nr = of_count_phandle_with_args(dev->of_node, "mediatek,iommu_banks", NULL); if (bk_nr >= IOMMU_BK_NUM || bk_nr < 0) { pr_info("%s, get bank nr fail, %d\n", __func__, bk_nr); goto out; } pr_info("%s, get bank nr:%d\n", __func__, bk_nr); for (i = 0; i < bk_nr; i++) { u32 bk_id; resource_size_t bk_pa; struct resource *bk_res[IOMMU_BK_NUM]; struct device_node *bk_node; struct platform_device *bk_dev; bk_node = of_parse_phandle(dev->of_node, "mediatek,iommu_banks", i); if (!bk_node) { dev_warn(dev, "Find iommu_bank:%d node fail\n", i); continue; } ret = of_property_read_u32(bk_node, "mediatek,bank-id", &bk_id); if (ret) { dev_warn(dev, "Get mediatek,bank-id fail\n"); continue; } bk_dev = of_find_device_by_node(bk_node); if (!bk_dev) { of_node_put(bk_node); dev_warn(dev, "Find iommu_bank:%d dev fail\n", bk_id); continue; } if (!MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_NO_IRQ)) { data->bk_irq[bk_id] = platform_get_irq(bk_dev, 0); if (data->bk_irq[bk_id] < 0) { dev_err(dev, "Get iommu_bank:%d irq fail\n", bk_id); return data->bk_irq[bk_id]; } } data->bk_dev[bk_id] = &bk_dev->dev; bk_res[bk_id] = platform_get_resource(bk_dev, IORESOURCE_MEM, 0); bk_pa = bk_res[bk_id]->start; dev_info(data->bk_dev[bk_id], "Get iommu bank:%u(%s) irq:%d ,pa:%pa, success\n", bk_id, dev_name(data->bk_dev[bk_id]), data->bk_irq[bk_id], &bk_pa); } } out: if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_BCLK)) { data->bclk = devm_clk_get(dev, "bclk"); if (IS_ERR(data->bclk)) { dev_err(dev, "%s,get clk failed\n", __func__); return PTR_ERR(data->bclk); } } if (MTK_IOMMU_HAS_FLAG(data->plat_data, PGTABLE_PA_35_EN)) { ret = dma_set_mask(dev, DMA_BIT_MASK(35)); if (ret) { dev_err(dev, "Failed to set dma_mask 35.\n"); return ret; } } /* mt6873, mt6893, mt6983, mt6879 */ if (data->plat_data->iommu_type == APU_IOMMU && MTK_IOMMU_HAS_FLAG(data->plat_data, LINK_WITH_APU)) { struct device_node *apunode; struct platform_device *apudev; struct device_link *link; apunode = of_parse_phandle(dev->of_node, "mediatek,apu_power", 0); if (!apunode) { dev_warn(dev, "Can't find apu power node!\n"); return -EINVAL; } apudev = of_find_device_by_node(apunode); if (!apudev) { of_node_put(apunode); dev_warn(dev, "Find apudev fail!\n"); return -ENODEV; } link = device_link_add(&apudev->dev, dev, DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); if (!link) dev_err(dev, "Unable link %s.\n", apudev->name); goto skip_smi; } if (data->plat_data->iommu_type != MM_IOMMU || MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_EN_PRE)) { dev_info(dev, "skip smi\n"); goto skip_smi; } larb_nr = of_count_phandle_with_args(dev->of_node, "mediatek,larbs", NULL); if (larb_nr < 0) { dev_err(dev, "%s, can't find mediatek,larbs !\n", __func__); return larb_nr; } for (i = 0; i < larb_nr; i++) { u32 id; larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i); if (!larbnode) { dev_err(dev, "%s, can't find larbnode:%d !\n", __func__, i); return -EINVAL; } if (!of_device_is_available(larbnode)) { of_node_put(larbnode); continue; } ret = of_property_read_u32(larbnode, "mediatek,larb-id", &id); if (ret)/* The id is consecutive if there is no this property */ id = i; disp_power_on |= of_property_read_bool(larbnode, "init-power-on"); plarbdev = of_find_device_by_node(larbnode); if (!plarbdev) { dev_err(dev, "%s, can't find larb dev:%d !\n", __func__, i); of_node_put(larbnode); return -EPROBE_DEFER; } data->larb_imu[id].dev = &plarbdev->dev; component_match_add_release(dev, &match, release_of, compare_of, larbnode); } /* Get smi-common dev from the last larb. */ smicomm_node = of_parse_phandle(larbnode, "mediatek,smi", 0); if (!smicomm_node) { dev_err(dev, "%s, can't find smicomm_node phase1\n", __func__); return -EINVAL; } if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_SMI_SUB_COMM)) { int string_nr, ret; const char *compat_name = NULL; repeat: pr_info("%s, check smi commom node start, dev:%s\n", __func__, dev_name(dev)); if (!smicomm_node) { dev_err(dev, "%s, can't find smicomm_node phase2\n", __func__); return -EINVAL; } string_nr = of_property_count_strings(smicomm_node, "compatible"); if (string_nr < 0) { dev_err(dev, "%s err, find compatible fail, nr:%d\n", __func__, string_nr); return -EINVAL; } for (i = 0; i < string_nr; i++) { ret = of_property_read_string_index(smicomm_node, "compatible", i, &compat_name); if (ret) { dev_err(dev, "%s err, find compatible name fail, ret:%d\n", __func__, ret); return -EINVAL; } pr_info("%s, compatible_name:%s, i:%d\n", __func__, compat_name, i); if (strstr(compat_name, "sub-common")) break; /* it is sub-common and goto find next node */ } if (i < string_nr) { pr_info("%s, find next level node, dev:%s\n", __func__, dev_name(dev)); smicomm_node = of_parse_phandle(smicomm_node, "mediatek,smi", 0); goto repeat; } } plarbdev = of_find_device_by_node(smicomm_node); of_node_put(smicomm_node); data->smicomm_dev = &plarbdev->dev; pr_info("%s, smi_common:%s, iommu_dev:%s\n", __func__, dev_name(&plarbdev->dev), dev_name(dev)); skip_smi: pm_runtime_enable(dev); if (data->smicomm_dev) { link = device_link_add(data->smicomm_dev, dev, DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); if (!link) { dev_err(dev, "Unable to link %s.\n", dev_name(data->smicomm_dev)); ret = -EINVAL; goto out_runtime_disable; } } platform_set_drvdata(pdev, data); ret = iommu_device_sysfs_add(&data->iommu, dev, NULL, "mtk-iommu.%pa", &ioaddr); if (ret) goto out_link_remove; ret = iommu_device_register(&data->iommu, &mtk_iommu_ops, dev); if (ret) goto out_sysfs_remove; spin_lock_init(&data->tlb_lock); if (MTK_IOMMU_HAS_FLAG(data->plat_data, SHARE_PGTABLE)) { list_add_tail(&data->list, &m4ulist); data->hw_list = &m4ulist; share_pgtable = true; } else { list_add_tail(&data->list, data->plat_data->hw_list); data->hw_list = data->plat_data->hw_list; } if (!iommu_present(&platform_bus_type)) { ret = bus_set_iommu(&platform_bus_type, &mtk_iommu_ops); if (ret) goto out_list_del; } if (data->plat_data->iommu_type == MM_IOMMU && !MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_EN_PRE)) { ret = component_master_add_with_match(dev, &mtk_iommu_com_ops, match); if (ret) goto out_bus_set_null; } /* register the notifier for power domain just for mm_iommu */ if (data->plat_data->iommu_type == MM_IOMMU) { int r, iommu_id = data->plat_data->iommu_id; mtk_pd_notifiers[iommu_id].notifier_call = mtk_iommu_pd_callback; mtk_pd_notifiers[iommu_id].priority = iommu_id; /* defaut power on,if larbs has the node "init-power-on" */ if (disp_power_on) { if (MTK_IOMMU_HAS_FLAG(data->plat_data, SAME_SUBSYS)) { pd_sta[DISP_IOMMU] = POWER_ON_STA; pd_sta[MDP_IOMMU] = POWER_ON_STA; pr_info("%s, config power on, (%d,%d)\n", __func__, data->plat_data->iommu_type, data->plat_data->iommu_id); } else { pd_sta[iommu_id] = POWER_ON_STA; } } r = dev_pm_genpd_add_notifier(dev, &mtk_pd_notifiers[iommu_id]); tlb_locks[iommu_id] = &data->tlb_lock; pr_info("%s add_notifier dev:%s, disp_power_on:%d, iommu:%d\n", __func__, dev_name(dev), disp_power_on, iommu_id); if (r) pr_info("%s notifier err, dev:%s\n", __func__, dev_name(dev)); } #if IS_ENABLED(CONFIG_MTK_SMI) if (data->plat_data->iommu_type == MM_IOMMU) { if (register_dbg_notifier != 1) { mtk_smi_dbg_register_notifier(&mtk_iommu_dbg_hang_nb); register_dbg_notifier = 1; } } #endif mtk_iommu_isr_pause_timer_init(data); pr_info("%s done dev:%s, head:%lx\n", __func__, dev_name(dev), (unsigned long)data->hw_list); return ret; out_bus_set_null: bus_set_iommu(&platform_bus_type, NULL); out_list_del: list_del(&data->list); iommu_device_unregister(&data->iommu); out_sysfs_remove: iommu_device_sysfs_remove(&data->iommu); out_link_remove: device_link_remove(data->smicomm_dev, dev); out_runtime_disable: pm_runtime_disable(dev); return ret; } static int mtk_iommu_remove(struct platform_device *pdev) { struct mtk_iommu_data *data = platform_get_drvdata(pdev); iommu_device_sysfs_remove(&data->iommu); iommu_device_unregister(&data->iommu); if (iommu_present(&platform_bus_type)) bus_set_iommu(&platform_bus_type, NULL); if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_BCLK)) clk_disable_unprepare(data->bclk); device_link_remove(data->smicomm_dev, &pdev->dev); pm_runtime_disable(&pdev->dev); if (!MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_NO_IRQ)) devm_free_irq(&pdev->dev, data->irq, data); component_master_del(&pdev->dev, &mtk_iommu_com_ops); return 0; } static int __maybe_unused mtk_iommu_runtime_suspend(struct device *dev) { struct mtk_iommu_data *data = dev_get_drvdata(dev); struct mtk_iommu_suspend_reg *reg = &data->reg; void __iomem *base = data->base; if (MTK_IOMMU_HAS_FLAG(data->plat_data, PM_OPS_SKIP)) return 0; /* skip read register when hw_init is not finish. */ if (!data->m4u_dom) { if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_BCLK)) clk_disable_unprepare(data->bclk); return 0; } reg->wr_len_ctrl = readl_relaxed(base + REG_MMU_WR_LEN_CTRL); reg->misc_ctrl = readl_relaxed(base + REG_MMU_MISC_CTRL); reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM_DIS); reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG); reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL0); reg->int_main_control = readl_relaxed(base + REG_MMU_INT_MAIN_CONTROL); reg->ivrp_paddr = readl_relaxed(base + REG_MMU_IVRP_PADDR); reg->vld_pa_rng = readl_relaxed(base + REG_MMU_VLD_PA_RNG); reg->tbw_id = readl_relaxed(base + REG_MMU_TBW_ID); #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_SECURE) if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_SEC_EN)) { mtk_iommu_secure_bk_backup_by_atf(data->plat_data->iommu_type, data->plat_data->iommu_id); if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_MAU_EN)) mtk_iommu_mau_reg_backup(data); } #endif #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_DBG) mtk_iommu_pm_trace(IOMMU_SUSPEND, data->plat_data->iommu_id, pd_sta[data->plat_data->iommu_id], 0, dev); #endif if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_BCLK)) clk_disable_unprepare(data->bclk); return 0; } static int __maybe_unused mtk_iommu_runtime_resume(struct device *dev) { struct mtk_iommu_data *data = dev_get_drvdata(dev); struct mtk_iommu_suspend_reg *reg = &data->reg; struct mtk_iommu_domain *m4u_dom = data->m4u_dom; void __iomem *base = data->base; int ret; if (MTK_IOMMU_HAS_FLAG(data->plat_data, PM_OPS_SKIP)) { if (data->plat_data->iommu_type == APU_IOMMU) mtk_iommu_tlb_flush(data); return 0; } if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_BCLK)) { ret = clk_prepare_enable(data->bclk); if (ret) { dev_err(data->dev, "Failed to enable clk(%d) in resume\n", ret); return ret; } } /* * Uppon first resume, only enable the clk and return, since the values of the * registers are not yet set. */ if (!m4u_dom) return 0; writel_relaxed(reg->tbw_id, base + REG_MMU_TBW_ID); writel_relaxed(reg->wr_len_ctrl, base + REG_MMU_WR_LEN_CTRL); writel_relaxed(reg->misc_ctrl, base + REG_MMU_MISC_CTRL); writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS); writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG); writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL0); writel_relaxed(reg->int_main_control, base + REG_MMU_INT_MAIN_CONTROL); writel_relaxed(reg->ivrp_paddr, base + REG_MMU_IVRP_PADDR); writel_relaxed(reg->vld_pa_rng, base + REG_MMU_VLD_PA_RNG); writel(m4u_dom->ttbr, base + REG_MMU_PT_BASE_ADDR); #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_SECURE) if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_SEC_EN)) { mtk_iommu_secure_bk_restore_by_atf(data->plat_data->iommu_type, data->plat_data->iommu_id); if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_MAU_EN)) mtk_iommu_mau_reg_restore(data); } #endif /* * Users may allocate dma buffer when HW is runtime suspended, * in which case it will lack the necessary tlb flush. * Thus, make sure to update the tlb after each PM resume. */ mtk_iommu_tlb_flush(data); #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_DBG) mtk_iommu_pm_trace(IOMMU_RESUME, data->plat_data->iommu_id, pd_sta[data->plat_data->iommu_id], 0, dev); #endif return 0; } #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_SECURE) static int mtk_dump_reg(const struct mtk_iommu_data *data, unsigned int start, unsigned int length) { int i = 0; void __iomem *base = data->base; for (i = 0; i < length; i += 4) { if (length - i == 1) pr_info("0x%x=0x%x\n", start + 4 * i, readl_relaxed(base + start + 4 * i)); else if (length - i == 2) pr_info("0x%x=0x%x, 0x%x=0x%x\n", start + 4 * i, readl_relaxed(base + start + 4 * i), start + 4 * (i + 1), readl_relaxed(base + start + 4 * (i + 1))); else if (length - i == 3) pr_info("0x%x=0x%x, 0x%x=0x%x, 0x%x=0x%x\n", start + 4 * i, readl_relaxed(base + start + 4 * i), start + 4 * (i + 1), readl_relaxed(base + start + 4 * (i + 1)), start + 4 * (i + 2), readl_relaxed(base + start + 4 * (i + 2))); else if (length - i >= 4) pr_info("0x%x=0x%x, 0x%x=0x%x, 0x%x=0x%x, 0x%x=0x%x\n", start + 4 * i, readl_relaxed(base + start + 4 * i), start + 4 * (i + 1), readl_relaxed(base + start + 4 * (i + 1)), start + 4 * (i + 2), readl_relaxed(base + start + 4 * (i + 2)), start + 4 * (i + 3), readl_relaxed(base + start + 4 * (i + 3))); } return 0; } static int mtk_dump_debug_reg_info(const struct mtk_iommu_data *data) { pr_info("---- iommu(%d, %d) debug register ----\n", data->plat_data->iommu_type, data->plat_data->iommu_id); return mtk_dump_reg(data, REG_MMU_DBG(0), MTK_IOMMU_DEBUG_REG_NR); } static int mtk_dump_rs_sta_info(const struct mtk_iommu_data *data, int mmu) { pr_info("---- iommu(%d, %d) mmu%d: RS status register ----\n", data->plat_data->iommu_type, data->plat_data->iommu_id, mmu); pr_info("--<0x0>iova/bank --<0x4>descriptor --<0x8>2nd-base --<0xc>status\n"); return mtk_dump_reg(data, REG_MMU_RS_VA(mmu, 0), MTK_IOMMU_RS_COUNT * 4); } #endif static void mtk_dump_reg_for_hang_issue(struct mtk_iommu_data *data) { #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_SECURE) int cnt, ret, i, dump_count = 1; #endif void __iomem *base = data->base; if (data->base == NULL) { pr_err("%s, base is NULL\n", __func__); return; } base = data->base; pr_info("%s start, (%d, %d)\n", __func__, data->plat_data->iommu_type, data->plat_data->iommu_id); /* control register */ pr_info("REG_MMU_PT_BASE_ADDR(0x000) = 0x%x\n", readl_relaxed(base + REG_MMU_PT_BASE_ADDR)); pr_info("REG_MMU_IVRP_PADDR(0x114) = 0x%x\n", readl_relaxed(base + REG_MMU_IVRP_PADDR)); pr_info("REG_MMU_DUMMY(0x044) = 0x%x\n", readl_relaxed(base + REG_MMU_DUMMY)); pr_info("REG_MMU_MISC_CTRL(0x048) = 0x%x\n", readl_relaxed(base + REG_MMU_MISC_CTRL)); pr_info("REG_MMU_DCM_DIS(0x050) = 0x%x\n", readl_relaxed(base + REG_MMU_DCM_DIS)); pr_info("REG_MMU_WR_LEN_CTRL(0x054) = 0x%x\n", readl_relaxed(base + REG_MMU_WR_LEN_CTRL)); pr_info("REG_MMU_TBW_ID(0x0A0) = 0x%x\n", readl_relaxed(base + REG_MMU_TBW_ID)); pr_info("REG_MMU_CTRL_REG(0x110) = 0x%x\n", readl_relaxed(base + REG_MMU_CTRL_REG)); #if IS_ENABLED(CONFIG_MTK_IOMMU_MISC_SECURE) if (!MTK_IOMMU_HAS_FLAG(data->plat_data, IOMMU_SEC_EN)) { pr_notice("%s abort, (%d, %d), secure not ready\n", __func__, data->plat_data->iommu_type, data->plat_data->iommu_id); return; } ret = ao_secure_dbg_switch_by_atf(data->plat_data->iommu_type, data->plat_data->iommu_id, 1); if (ret) { pr_err("%s, failed to enable secure debug\n", __func__); return; } for (cnt = 0; cnt < dump_count; cnt++) { pr_info("===== the %d time: REG_MMU_STA(0x008) = 0x%x =====\n", cnt, readl_relaxed(base + REG_MMU_STA)); mtk_dump_debug_reg_info(data); for (i = 0; i < MTK_IOMMU_MMU_COUNT; i++) mtk_dump_rs_sta_info(data, i); } pr_info("===== dump hang reg end =====\n"); ret = ao_secure_dbg_switch_by_atf(data->plat_data->iommu_type, data->plat_data->iommu_id, 0); if (ret) pr_err("%s, failed to disable secure debug\n", __func__); #endif pr_info("%s done, (%d, %d)\n", __func__, data->plat_data->iommu_type, data->plat_data->iommu_id); } void mtk_iommu_dbg_hang_detect(enum mtk_iommu_type type, int id) { struct list_head *hw_list; struct mtk_iommu_data *data; unsigned long flags; if (!share_pgtable) { switch (type) { case MM_IOMMU: hw_list = &mm_iommu_list; break; case APU_IOMMU: hw_list = &apu_iommu_list; break; default: pr_err("%s failed, type is invalid, %d\n", __func__, type); return; } } else { hw_list = &m4ulist; } for_each_m4u(data, hw_list) { if (data->plat_data->iommu_type == type && data->plat_data->iommu_id == id) { spin_lock_irqsave(&data->tlb_lock, flags); if (!mtk_iommu_power_get(data)) { pr_notice("%s, iommu:(%d,%d) power off dev:%s\n", __func__, type, id, dev_name(data->dev)); spin_unlock_irqrestore(&data->tlb_lock, flags); return; } mtk_dump_reg_for_hang_issue(data); mtk_iommu_power_put(data); spin_unlock_irqrestore(&data->tlb_lock, flags); return; } } pr_info("%s, (%d,%d) no dump\n", __func__, type, id); } EXPORT_SYMBOL_GPL(mtk_iommu_dbg_hang_detect); static const struct dev_pm_ops mtk_iommu_pm_ops = { SET_RUNTIME_PM_OPS(mtk_iommu_runtime_suspend, mtk_iommu_runtime_resume, NULL) SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) }; static const struct mtk_iommu_plat_data mt2712_data = { .m4u_plat = M4U_MT2712, .flags = HAS_4GB_MODE | HAS_BCLK | HAS_VLD_PA_RNG | NOT_STD_AXI_MODE | SHARE_PGTABLE, .inv_sel_reg = REG_MMU_INV_SEL_GEN1, .iova_region = single_domain, .iova_region_nr = ARRAY_SIZE(single_domain), .iommu_id = DISP_IOMMU, .iommu_type = MM_IOMMU, .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}}, }; static const struct mtk_iommu_plat_data mt6779_data = { .m4u_plat = M4U_MT6779, .flags = HAS_SUB_COMM | OUT_ORDER_WR_EN | WR_THROT_EN | NOT_STD_AXI_MODE | SHARE_PGTABLE, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iova_region = single_domain, .iova_region_nr = ARRAY_SIZE(single_domain), .iommu_id = DISP_IOMMU, .iommu_type = MM_IOMMU, .larbid_remap = {{0}, {1}, {2}, {3}, {5}, {7, 8}, {10}, {9}}, }; static const struct mtk_iommu_plat_data mt6835_data_disp = { .m4u_plat = M4U_MT6835, .flags = OUT_ORDER_WR_EN | GET_DOM_ID_LEGACY | NOT_STD_AXI_MODE | TLB_SYNC_EN | SHARE_PGTABLE | IOMMU_SEC_EN | IOMMU_MAU_EN | SKIP_CFG_PORT | IOVA_34_EN | HAS_SMI_SUB_COMM, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = DISP_IOMMU, .iommu_type = MM_IOMMU, .normal_dom = 0, .iova_region = mt6835_multi_dom, .iova_region_nr = ARRAY_SIZE(mt6835_multi_dom), .mau_count = 4, }; /* copy the mtk_iommu_plat_data form the mt6873 because * the mt6853 almost uset the same data as mt6873. * but only add the IOVA_34_EN flag. */ static const struct mtk_iommu_plat_data mt6853_data = { .m4u_plat = M4U_MT6853, .flags = HAS_SUB_COMM | OUT_ORDER_WR_EN | WR_THROT_EN | HAS_BCLK | NOT_STD_AXI_MODE | IOVA_34_EN | SHARE_PGTABLE, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = DISP_IOMMU, .iommu_type = MM_IOMMU, /* use the same data as mt6873 */ .iova_region = mt6873_multi_dom, .iova_region_nr = ARRAY_SIZE(mt6873_multi_dom), .larbid_remap = {{0}, {1}, {4, 5}, {7}, {2}, {9, 11, 19, 20}, {0, 14, 16}, {0, 13, 18, 17}}, }; static const struct mtk_iommu_plat_data mt6855_data_disp = { .m4u_plat = M4U_MT6855, .flags = HAS_SUB_COMM | OUT_ORDER_WR_EN | GET_DOM_ID_LEGACY | NOT_STD_AXI_MODE | TLB_SYNC_EN | SHARE_PGTABLE | IOMMU_SEC_EN | SKIP_CFG_PORT | IOVA_34_EN | HAS_BCLK | HAS_SMI_SUB_COMM, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = DISP_IOMMU, .iommu_type = MM_IOMMU, .normal_dom = 0, .iova_region = mt6855_multi_dom, .iova_region_nr = ARRAY_SIZE(mt6855_multi_dom), /* not use larbid_remap */ }; static const struct mtk_iommu_plat_data mt6873_data = { .m4u_plat = M4U_MT6873, .flags = HAS_SUB_COMM | OUT_ORDER_WR_EN | WR_THROT_EN | HAS_BCLK | NOT_STD_AXI_MODE | SHARE_PGTABLE, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = DISP_IOMMU, .iommu_type = MM_IOMMU, .iova_region = mt6873_multi_dom, .iova_region_nr = ARRAY_SIZE(mt6873_multi_dom), /* not use larbid_remap */ .larbid_remap = {{0}, {1}, {4, 5}, {7}, {2}, {9, 11, 19, 20}, {0, 14, 16}, {0, 13, 18, 17}}, }; static const struct mtk_iommu_plat_data mt6873_data_apu = { .m4u_plat = M4U_MT6873, .flags = LINK_WITH_APU | SHARE_PGTABLE, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = APU_IOMMU0, .iommu_type = APU_IOMMU, .iova_region = mt6873_multi_dom, /* not use larbid_remap */ .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}}, .iova_region_nr = ARRAY_SIZE(mt6873_multi_dom), }; static const struct mtk_iommu_plat_data mt6879_data_disp = { .m4u_plat = M4U_MT6879, .flags = HAS_SUB_COMM | OUT_ORDER_WR_EN | GET_DOM_ID_LEGACY | NOT_STD_AXI_MODE | TLB_SYNC_EN | IOMMU_SEC_EN | SKIP_CFG_PORT | IOVA_34_EN | HAS_BCLK | HAS_SMI_SUB_COMM | IOMMU_MAU_EN, .hw_list = &mm_iommu_list, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = DISP_IOMMU, .iommu_type = MM_IOMMU, .tab_id = MM_TABLE, .normal_dom = 0, .iova_region = mt6879_multi_dom_mm, .iova_region_nr = ARRAY_SIZE(mt6879_multi_dom_mm), .mau_count = 4, /* not use larbid_remap */ }; static const struct mtk_iommu_plat_data mt6879_data_apu0 = { .m4u_plat = M4U_MT6879, .flags = HAS_SUB_COMM | TLB_SYNC_EN | IOMMU_SEC_EN | GET_DOM_ID_LEGACY | IOVA_34_EN | LINK_WITH_APU | PM_OPS_SKIP | IOMMU_MAU_EN, .hw_list = &apu_iommu_list, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = APU_IOMMU0, .iommu_type = APU_IOMMU, .tab_id = APU_TABLE, .normal_dom = 0, .iova_region = mt6879_multi_dom_apu, .iova_region_nr = ARRAY_SIZE(mt6879_multi_dom_apu), .mau_count = 4, /* not use larbid_remap */ }; static const struct mtk_iommu_plat_data mt6886_data_disp = { .m4u_plat = M4U_MT6886, .flags = OUT_ORDER_WR_EN | GET_DOM_ID_LEGACY | SKIP_CFG_PORT | NOT_STD_AXI_MODE | TLB_SYNC_EN | IOMMU_SEC_EN | IOVA_34_EN | HAS_SMI_SUB_COMM | PGTABLE_PA_35_EN | IOMMU_MAU_EN, .hw_list = &mm_iommu_list, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = DISP_IOMMU, .iommu_type = MM_IOMMU, .tab_id = MM_TABLE, .normal_dom = 0, .iova_region = mt6886_multi_dom_mm, .iova_region_nr = ARRAY_SIZE(mt6886_multi_dom_mm), .mau_count = 4, }; static const struct mtk_iommu_plat_data mt6886_data_apu0 = { .m4u_plat = M4U_MT6886, .flags = TLB_SYNC_EN | GET_DOM_ID_LEGACY | IOMMU_SEC_EN | IOVA_34_EN | LINK_WITH_APU | IOMMU_MAU_EN | PGTABLE_PA_35_EN | PM_OPS_SKIP, .hw_list = &apu_iommu_list, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = APU_IOMMU0, .iommu_type = APU_IOMMU, .tab_id = APU_TABLE, .normal_dom = 0, .iova_region = mt6886_multi_dom_apu, .iova_region_nr = ARRAY_SIZE(mt6886_multi_dom_apu), .mau_count = 4, }; static const struct mtk_iommu_plat_data mt6893_data_iommu0 = { .m4u_plat = M4U_MT6893, .flags = NOT_STD_AXI_MODE | HAS_SUB_COMM | OUT_ORDER_WR_EN | WR_THROT_EN | HAS_BCLK | IOVA_34_EN | GET_DOM_ID_LEGACY | SHARE_PGTABLE, /* not use larbid_remap */ .larbid_remap = {{0}, {1}, {4, 5}, {7}, {2}, {9, 11, 19, 20}, {0, 14, 16}, {0, 13, 18, 17}}, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = DISP_IOMMU, .iommu_type = MM_IOMMU, .normal_dom = 0, .iova_region = mt6873_multi_dom, .iova_region_nr = ARRAY_SIZE(mt6873_multi_dom), }; static const struct mtk_iommu_plat_data mt6893_data_iommu1 = { .m4u_plat = M4U_MT6893, .flags = NOT_STD_AXI_MODE | HAS_SUB_COMM | OUT_ORDER_WR_EN | WR_THROT_EN | HAS_BCLK | IOVA_34_EN | GET_DOM_ID_LEGACY | SHARE_PGTABLE, /* not use larbid_remap */ .larbid_remap = {{0}, {1}, {4, 5}, {7}, {2}, {9, 11, 19, 20}, {0, 14, 16}, {0, 13, 18, 17}}, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = MDP_IOMMU, .iommu_type = MM_IOMMU, .normal_dom = 0, .iova_region = mt6873_multi_dom, .iova_region_nr = ARRAY_SIZE(mt6873_multi_dom), }; static const struct mtk_iommu_plat_data mt6893_data_iommu2 = { .m4u_plat = M4U_MT6893, .flags = LINK_WITH_APU | IOVA_34_EN | GET_DOM_ID_LEGACY | SHARE_PGTABLE, .iommu_id = APU_IOMMU0, .iommu_type = APU_IOMMU, .normal_dom = 0, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iova_region = mt6873_multi_dom, /* not use larbid_remap */ .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}}, .iova_region_nr = ARRAY_SIZE(mt6873_multi_dom), }; static const struct mtk_iommu_plat_data mt6893_data_iommu3 = { .m4u_plat = M4U_MT6893, .flags = LINK_WITH_APU | SHARE_PGTABLE, .iommu_id = APU_IOMMU1, .iommu_type = APU_IOMMU, .normal_dom = 0, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iova_region = mt6873_multi_dom, /* not use larbid_remap */ .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}}, .iova_region_nr = ARRAY_SIZE(mt6873_multi_dom), }; static const struct mtk_iommu_plat_data mt6895_data_disp = { .m4u_plat = M4U_MT6895, .flags = HAS_SUB_COMM | OUT_ORDER_WR_EN | GET_DOM_ID_LEGACY | NOT_STD_AXI_MODE | TLB_SYNC_EN | IOMMU_SEC_EN | SKIP_CFG_PORT | IOVA_34_EN | IOMMU_MAU_EN | HAS_SMI_SUB_COMM | SAME_SUBSYS | HAS_BCLK, .hw_list = &mm_iommu_list, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = DISP_IOMMU, .iommu_type = MM_IOMMU, .tab_id = MM_TABLE, .normal_dom = 0, .iova_region = mt6895_multi_dom_mm, .iova_region_nr = ARRAY_SIZE(mt6895_multi_dom_mm), .mau_count = 4, /* not use larbid_remap */ }; static const struct mtk_iommu_plat_data mt6895_data_mdp = { .m4u_plat = M4U_MT6895, .flags = HAS_SUB_COMM | OUT_ORDER_WR_EN | GET_DOM_ID_LEGACY | NOT_STD_AXI_MODE | TLB_SYNC_EN | IOMMU_SEC_EN | SKIP_CFG_PORT | IOVA_34_EN | IOMMU_MAU_EN | HAS_SMI_SUB_COMM | SAME_SUBSYS | HAS_BCLK, .hw_list = &mm_iommu_list, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = MDP_IOMMU, .iommu_type = MM_IOMMU, .tab_id = MM_TABLE, .normal_dom = 0, .iova_region = mt6895_multi_dom_mm, .iova_region_nr = ARRAY_SIZE(mt6895_multi_dom_mm), .mau_count = 4, /* not use larbid_remap */ }; static const struct mtk_iommu_plat_data mt6895_data_apu0 = { .m4u_plat = M4U_MT6895, .flags = HAS_SUB_COMM | TLB_SYNC_EN | IOMMU_SEC_EN | GET_DOM_ID_LEGACY | IOVA_34_EN | LINK_WITH_APU | IOMMU_MAU_EN | PM_OPS_SKIP, .hw_list = &apu_iommu_list, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = APU_IOMMU0, .iommu_type = APU_IOMMU, .tab_id = APU_TABLE, .normal_dom = 0, .iova_region = mt6895_multi_dom_apu, .iova_region_nr = ARRAY_SIZE(mt6895_multi_dom_apu), .mau_count = 4, /* not use larbid_remap */ }; static const struct mtk_iommu_plat_data mt6895_data_apu1 = { .m4u_plat = M4U_MT6895, .flags = HAS_SUB_COMM | TLB_SYNC_EN | IOMMU_SEC_EN | GET_DOM_ID_LEGACY | IOVA_34_EN | LINK_WITH_APU | IOMMU_MAU_EN | PM_OPS_SKIP, .hw_list = &apu_iommu_list, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = APU_IOMMU1, .iommu_type = APU_IOMMU, .tab_id = APU_TABLE, .normal_dom = 0, .iova_region = mt6895_multi_dom_apu, .iova_region_nr = ARRAY_SIZE(mt6895_multi_dom_apu), .mau_count = 4, /* not use larbid_remap */ }; static const struct mtk_iommu_plat_data mt6983_data_disp = { .m4u_plat = M4U_MT6983, .flags = HAS_SUB_COMM | OUT_ORDER_WR_EN | GET_DOM_ID_LEGACY | NOT_STD_AXI_MODE | TLB_SYNC_EN | IOMMU_SEC_EN | SKIP_CFG_PORT | IOVA_34_EN | PGTABLE_PA_35_EN | HAS_BCLK | HAS_SMI_SUB_COMM | SAME_SUBSYS | IOMMU_MAU_EN, .hw_list = &mm_iommu_list, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = DISP_IOMMU, .iommu_type = MM_IOMMU, .tab_id = MM_TABLE, .normal_dom = 0, .iova_region = mt6983_multi_dom_mm, .iova_region_nr = ARRAY_SIZE(mt6983_multi_dom_mm), .mau_count = 4, /* not use larbid_remap */ }; static const struct mtk_iommu_plat_data mt6983_data_mdp = { .m4u_plat = M4U_MT6983, .flags = HAS_SUB_COMM | OUT_ORDER_WR_EN | GET_DOM_ID_LEGACY | NOT_STD_AXI_MODE | TLB_SYNC_EN | IOMMU_SEC_EN | SKIP_CFG_PORT | IOVA_34_EN | PGTABLE_PA_35_EN | HAS_BCLK | HAS_SMI_SUB_COMM | SAME_SUBSYS | IOMMU_MAU_EN, .hw_list = &mm_iommu_list, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = MDP_IOMMU, .iommu_type = MM_IOMMU, .tab_id = MM_TABLE, .normal_dom = 0, .iova_region = mt6983_multi_dom_mm, .iova_region_nr = ARRAY_SIZE(mt6983_multi_dom_mm), .mau_count = 4, /* not use larbid_remap */ }; static const struct mtk_iommu_plat_data mt6983_data_apu0 = { .m4u_plat = M4U_MT6983, .flags = HAS_SUB_COMM | TLB_SYNC_EN | IOMMU_SEC_EN | GET_DOM_ID_LEGACY | IOVA_34_EN | LINK_WITH_APU | IOMMU_MAU_EN | PM_OPS_SKIP | PGTABLE_PA_35_EN, .hw_list = &apu_iommu_list, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = APU_IOMMU0, .iommu_type = APU_IOMMU, .tab_id = APU_TABLE, .normal_dom = 0, .iova_region = mt6983_multi_dom_apu, .iova_region_nr = ARRAY_SIZE(mt6983_multi_dom_apu), .mau_count = 4, /* not use larbid_remap */ }; static const struct mtk_iommu_plat_data mt6983_data_apu1 = { .m4u_plat = M4U_MT6983, .flags = HAS_SUB_COMM | TLB_SYNC_EN | IOMMU_SEC_EN | GET_DOM_ID_LEGACY | IOVA_34_EN | LINK_WITH_APU | IOMMU_MAU_EN | PM_OPS_SKIP | PGTABLE_PA_35_EN, .hw_list = &apu_iommu_list, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = APU_IOMMU1, .iommu_type = APU_IOMMU, .tab_id = APU_TABLE, .normal_dom = 0, .iova_region = mt6983_multi_dom_apu, .iova_region_nr = ARRAY_SIZE(mt6983_multi_dom_apu), .mau_count = 4, /* not use larbid_remap */ }; static const struct mtk_iommu_plat_data mt6985_data_disp = { .m4u_plat = M4U_MT6985, .flags = OUT_ORDER_WR_EN | GET_DOM_ID_LEGACY | NOT_STD_AXI_MODE | TLB_SYNC_EN | IOMMU_SEC_EN | SKIP_CFG_PORT | IOVA_34_EN | PGTABLE_PA_35_EN | HAS_BCLK | HAS_SMI_SUB_COMM | SAME_SUBSYS, .hw_list = &mm_iommu_list, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = DISP_IOMMU, .iommu_type = MM_IOMMU, .tab_id = MM_TABLE, .normal_dom = 0, .iova_region = mt6985_multi_dom_mm, .iova_region_nr = ARRAY_SIZE(mt6985_multi_dom_mm), .mau_count = 4, }; static const struct mtk_iommu_plat_data mt6985_data_mdp = { .m4u_plat = M4U_MT6985, .flags = OUT_ORDER_WR_EN | GET_DOM_ID_LEGACY | NOT_STD_AXI_MODE | TLB_SYNC_EN | IOMMU_SEC_EN | SKIP_CFG_PORT | IOVA_34_EN | PGTABLE_PA_35_EN | HAS_BCLK | HAS_SMI_SUB_COMM | SAME_SUBSYS, .hw_list = &mm_iommu_list, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = MDP_IOMMU, .iommu_type = MM_IOMMU, .tab_id = MM_TABLE, .normal_dom = 0, .iova_region = mt6985_multi_dom_mm, .iova_region_nr = ARRAY_SIZE(mt6985_multi_dom_mm), .mau_count = 4, }; static const struct mtk_iommu_plat_data mt6985_data_apu0 = { .m4u_plat = M4U_MT6985, .flags = TLB_SYNC_EN | IOMMU_SEC_EN | PGTABLE_PA_35_EN | GET_DOM_ID_LEGACY | IOVA_34_EN | LINK_WITH_APU | PM_OPS_SKIP, .hw_list = &apu_iommu_list, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = APU_IOMMU0, .iommu_type = APU_IOMMU, .tab_id = APU_TABLE, .normal_dom = 0, .iova_region = mt6985_multi_dom_apu, .iova_region_nr = ARRAY_SIZE(mt6985_multi_dom_apu), .mau_count = 4, }; static const struct mtk_iommu_plat_data mt6985_data_apu1 = { .m4u_plat = M4U_MT6985, .flags = TLB_SYNC_EN | IOMMU_SEC_EN | PGTABLE_PA_35_EN | GET_DOM_ID_LEGACY | IOVA_34_EN | LINK_WITH_APU | PM_OPS_SKIP, .hw_list = &apu_iommu_list, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iommu_id = APU_IOMMU1, .iommu_type = APU_IOMMU, .tab_id = APU_TABLE, .normal_dom = 0, .iova_region = mt6985_multi_dom_apu, .iova_region_nr = ARRAY_SIZE(mt6985_multi_dom_apu), .mau_count = 4, }; static const struct mtk_iommu_plat_data mt8167_data = { .m4u_plat = M4U_MT8167, .flags = RESET_AXI | HAS_LEGACY_IVRP_PADDR | SHARE_PGTABLE, .inv_sel_reg = REG_MMU_INV_SEL_GEN1, .iommu_id = DISP_IOMMU, .iommu_type = MM_IOMMU, .iova_region = single_domain, .iova_region_nr = ARRAY_SIZE(single_domain), .larbid_remap = {{0}, {1}, {2}}, /* Linear mapping. */ }; static const struct mtk_iommu_plat_data mt8173_data = { .m4u_plat = M4U_MT8173, .flags = HAS_4GB_MODE | HAS_BCLK | RESET_AXI | HAS_LEGACY_IVRP_PADDR | SHARE_PGTABLE, .inv_sel_reg = REG_MMU_INV_SEL_GEN1, .iova_region = single_domain, .iova_region_nr = ARRAY_SIZE(single_domain), .iommu_id = DISP_IOMMU, .iommu_type = MM_IOMMU, .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}}, /* Linear mapping. */ }; static const struct mtk_iommu_plat_data mt8183_data = { .m4u_plat = M4U_MT8183, .flags = RESET_AXI | SHARE_PGTABLE, .inv_sel_reg = REG_MMU_INV_SEL_GEN1, .iova_region = single_domain, .iova_region_nr = ARRAY_SIZE(single_domain), .iommu_id = DISP_IOMMU, .iommu_type = MM_IOMMU, .larbid_remap = {{0}, {4}, {5}, {6}, {7}, {2}, {3}, {1}}, }; static const struct mtk_iommu_plat_data mt8192_data = { .m4u_plat = M4U_MT8192, .flags = HAS_BCLK | HAS_SUB_COMM | OUT_ORDER_WR_EN | WR_THROT_EN | IOVA_34_EN | SHARE_PGTABLE, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iova_region = mt8192_multi_dom, .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), .iommu_id = DISP_IOMMU, .iommu_type = MM_IOMMU, .larbid_remap = {{0}, {1}, {4, 5}, {7}, {2}, {9, 11, 19, 20}, {0, 14, 16}, {0, 13, 18, 17}}, }; static const struct of_device_id mtk_iommu_of_ids[] = { { .compatible = "mediatek,mt2712-m4u", .data = &mt2712_data}, { .compatible = "mediatek,mt6779-m4u", .data = &mt6779_data}, { .compatible = "mediatek,mt6835-disp-iommu", .data = &mt6835_data_disp}, { .compatible = "mediatek,mt6853-m4u", .data = &mt6853_data}, { .compatible = "mediatek,mt6855-disp-iommu", .data = &mt6855_data_disp}, { .compatible = "mediatek,mt6873-m4u", .data = &mt6873_data}, { .compatible = "mediatek,mt6873-apu-iommu", .data = &mt6873_data_apu}, { .compatible = "mediatek,mt6879-apu-iommu0", .data = &mt6879_data_apu0}, { .compatible = "mediatek,mt6879-disp-iommu", .data = &mt6879_data_disp}, { .compatible = "mediatek,mt6886-apu-iommu0", .data = &mt6886_data_apu0}, { .compatible = "mediatek,mt6886-disp-iommu", .data = &mt6886_data_disp}, { .compatible = "mediatek,mt6893-iommu0", .data = &mt6893_data_iommu0}, { .compatible = "mediatek,mt6893-iommu1", .data = &mt6893_data_iommu1}, { .compatible = "mediatek,mt6893-iommu2", .data = &mt6893_data_iommu2}, { .compatible = "mediatek,mt6893-iommu3", .data = &mt6893_data_iommu3}, { .compatible = "mediatek,mt6895-disp-iommu", .data = &mt6895_data_disp}, { .compatible = "mediatek,mt6895-mdp-iommu", .data = &mt6895_data_mdp}, { .compatible = "mediatek,mt6895-apu-iommu0", .data = &mt6895_data_apu0}, { .compatible = "mediatek,mt6895-apu-iommu1", .data = &mt6895_data_apu1}, { .compatible = "mediatek,mt6983-apu-iommu0", .data = &mt6983_data_apu0}, { .compatible = "mediatek,mt6983-apu-iommu1", .data = &mt6983_data_apu1}, { .compatible = "mediatek,mt6983-disp-iommu", .data = &mt6983_data_disp}, { .compatible = "mediatek,mt6983-mdp-iommu", .data = &mt6983_data_mdp}, { .compatible = "mediatek,mt6985-apu-iommu0", .data = &mt6985_data_apu0}, { .compatible = "mediatek,mt6985-apu-iommu1", .data = &mt6985_data_apu1}, { .compatible = "mediatek,mt6985-disp-iommu", .data = &mt6985_data_disp}, { .compatible = "mediatek,mt6985-mdp-iommu", .data = &mt6985_data_mdp}, { .compatible = "mediatek,mt8167-m4u", .data = &mt8167_data}, { .compatible = "mediatek,mt8173-m4u", .data = &mt8173_data}, { .compatible = "mediatek,mt8183-m4u", .data = &mt8183_data}, { .compatible = "mediatek,mt8192-m4u", .data = &mt8192_data}, {} }; static struct platform_driver mtk_iommu_driver = { .probe = mtk_iommu_probe, .remove = mtk_iommu_remove, .driver = { .name = "mtk-iommu", .of_match_table = mtk_iommu_of_ids, .pm = &mtk_iommu_pm_ops, } }; module_platform_driver(mtk_iommu_driver); MODULE_DESCRIPTION("IOMMU API for MediaTek M4U implementations"); MODULE_LICENSE("GPL v2");