iommu/arm-smmu-v3: Batch ATC invalidation commands
PATCH v2 1/1 iommu/arm-smmu-v3: Batch ATC invalidation commands
Similar to commit 2af2e72b18b4 ("iommu/arm-smmu-v3: Defer TLB invalidation until ->iotlb_sync()"), build up a list of ATC invalidation commands and submit them all at once to the command queue instead of one-by-one.
As there is only one caller of arm_smmu_atc_inv_master() left, we can simplify it and avoid passing in struct arm_smmu_cmdq_ent.
---
v2:
- Simplify arm_smmu_atc_inv_master()
- Rebase on v5.6-rc1
drivers/iommu/arm-smmu-v3.c | 38 ++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-)
drivers/iommu/arm-smmu-v3.c +25 -13
@@ -2132,17 +2132,16 @@ arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size, cmd->atc.size = log2_span; } -static int arm_smmu_atc_inv_master (struct arm_smmu_master *master, -struct arm_smmu_cmdq_ent *cmd) +static int arm_smmu_atc_inv_master (struct arm_smmu_master *master) {int i; +struct arm_smmu_cmdq_ent cmd; -if (!master->ats_enabled) -return 0 ; +arm_smmu_atc_inv_to_cmd (0 ,0 ,0 , &cmd);for (i =0 ; i < master->num_sids; i++) { - cmd->atc.sid = master->sids[i]; -arm_smmu_cmdq_issue_cmd (master->smmu, cmd); + cmd.atc.sid = master->sids[i]; +arm_smmu_cmdq_issue_cmd (master->smmu, &cmd); }return arm_smmu_cmdq_issue_sync (master->smmu);
@@ -2151,10 +2150,11 @@ static int arm_smmu_atc_inv_master(struct arm_smmu_master *master,static int arm_smmu_atc_inv_domain (struct arm_smmu_domain *smmu_domain,int ssid,unsigned long iova,size_t size) { -int ret =0 ; +int i, cmdn =0 ;unsigned long flags;struct arm_smmu_cmdq_ent cmd;struct arm_smmu_master *master; +u64 cmds[CMDQ_BATCH_ENTRIES *CMDQ_ENT_DWORDS ];if (!(smmu_domain->smmu->features &ARM_SMMU_FEAT_ATS ))return 0 ;
@@ -2179,11 +2179,25 @@ static int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,arm_smmu_atc_inv_to_cmd (ssid, iova, size, &cmd);spin_lock_irqsave (&smmu_domain->devices_lock, flags); -list_for_each_entry (master, &smmu_domain->devices, domain_head) - ret |=arm_smmu_atc_inv_master (master, &cmd); +list_for_each_entry (master, &smmu_domain->devices, domain_head) { +if (!master->ats_enabled) +continue ; + +for (i =0 ; i < master->num_sids; i++) { +if (cmdn ==CMDQ_BATCH_ENTRIES ) { +arm_smmu_cmdq_issue_cmdlist (smmu_domain->smmu, + cmds, cmdn, false); + cmdn =0 ; + } + + cmd.atc.sid = master->sids[i]; +arm_smmu_cmdq_build_cmd (&cmds[cmdn *CMDQ_ENT_DWORDS ], &cmd); + cmdn++; + } + }spin_unlock_irqrestore (&smmu_domain->devices_lock, flags); -return ret ? -ETIMEDOUT :0 ; +return arm_smmu_cmdq_issue_cmdlist (smmu_domain->smmu, cmds, cmdn, true); }/* IO_PGTABLE API */
@@ -2611,7 +2625,6 @@ static void arm_smmu_enable_ats(struct arm_smmu_master *master)static void arm_smmu_disable_ats (struct arm_smmu_master *master) { -struct arm_smmu_cmdq_ent cmd;struct arm_smmu_domain *smmu_domain = master->domain;if (!master->ats_enabled)
@@ -2623,8 +2636,7 @@ static void arm_smmu_disable_ats(struct arm_smmu_master *master)* ATC invalidation via the SMMU. */ wmb (); -arm_smmu_atc_inv_to_cmd (0 ,0 ,0 , &cmd); -arm_smmu_atc_inv_master (master, &cmd); +arm_smmu_atc_inv_master (master);atomic_dec (&smmu_domain->nr_ats_masters); }
Signature · 6 lines
2.20.1 _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
Quoted 5 lines · Rob Herring
Similar to commit
2af2e72b18b4("iommu/arm-smmu-v3: Defer TLB invalidation until ->iotlb_sync()"), build up a list of ATC invalidation commands and submit them all at once to the command queue instead of one-by-one.
Quoted 8 lines
As there is only one caller of
arm_smmu_atc_inv_master()left, we can simplify it and avoid passing in structarm_smmu_cmdq_ent.Cc Jean-Philippe Brucker <[email protected]>Cc Will Deacon <[email protected]>Cc Robin Murphy <[email protected]>Cc Joerg Roedel <[email protected]>Signed-off-by Rob Herring <[email protected]>
Since I'm adding a third user of cmdq batching [1], I had a go at factoring them. I can send the attached patch with my next version, if it looks OK.
Thanks,
Jean
[1] https://lore.kernel.org/linux-iommu/[email protected]/
From b304f322e6293be4ec8b5a01e2ef67e8fa34143c Mon Sep 17 00:00:00 2001
From: Jean-Philippe Brucker <[email protected]>
Date: Mon, 17 Feb 2020 17:42:54 +0100
Subject: [PATCH] iommu/arm-smmu-v3: Factor command queue batching
Factor the code for command queue batching, which is now repeated three times for TLB, ATC and CFG invalidations.
--- drivers/iommu/arm-smmu-v3.c | 66 +++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 32 deletions(-)
drivers/iommu/arm-smmu-v3.c +34 -32
@@ -548,6 +548,11 @@ struct arm_smmu_cmdq {atomic_t lock; }; +struct arm_smmu_cmdq_batch { +u64 cmds[CMDQ_BATCH_ENTRIES *CMDQ_ENT_DWORDS ]; +int num; +}; +struct arm_smmu_evtq {struct arm_smmu_queue q;u32 max_stalls;
@@ -1482,15 +1487,33 @@ static int arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)return arm_smmu_cmdq_issue_cmdlist (smmu,NULL ,0 , true); } +static void arm_smmu_cmdq_batch_add (struct arm_smmu_device *smmu, +struct arm_smmu_cmdq_batch *cmds, +struct arm_smmu_cmdq_ent *cmd) +{ +if (cmds->num ==CMDQ_BATCH_ENTRIES ) { +arm_smmu_cmdq_issue_cmdlist (smmu, cmds->cmds, cmds->num, false); + cmds->num =0 ; + } +arm_smmu_cmdq_build_cmd (&cmds->cmds[cmds->num *CMDQ_ENT_DWORDS ], cmd); + cmds->num++; +} + +static int arm_smmu_cmdq_batch_submit (struct arm_smmu_device *smmu, +struct arm_smmu_cmdq_batch *cmds) +{ +return arm_smmu_cmdq_issue_cmdlist (smmu, cmds->cmds, cmds->num, true); +} + +/* Context descriptor manipulation functions */ static void arm_smmu_sync_cd (struct arm_smmu_domain *smmu_domain,int ssid,bool leaf) {size_t i; -int cmdn =0 ;unsigned long flags;struct arm_smmu_master *master; -u64 cmds[CMDQ_BATCH_ENTRIES *CMDQ_ENT_DWORDS ]; +struct arm_smmu_cmdq_batch cmds = {};struct arm_smmu_device *smmu = smmu_domain->smmu;struct arm_smmu_cmdq_ent cmd = { .opcode =CMDQ_OP_CFGI_CD ,
@@ -1503,19 +1526,13 @@ static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain,spin_lock_irqsave (&smmu_domain->devices_lock, flags);list_for_each_entry (master, &smmu_domain->devices, domain_head) {for (i =0 ; i < master->num_sids; i++) { -if (cmdn ==CMDQ_BATCH_ENTRIES ) { -arm_smmu_cmdq_issue_cmdlist (smmu, cmds, cmdn, false); - cmdn =0 ; - } - cmd.cfgi.sid = master->sids[i]; -arm_smmu_cmdq_build_cmd (&cmds[cmdn *CMDQ_ENT_DWORDS ], &cmd); - cmdn++; +arm_smmu_cmdq_batch_add (smmu, &cmds, &cmd); } }spin_unlock_irqrestore (&smmu_domain->devices_lock, flags); -arm_smmu_cmdq_issue_cmdlist (smmu, cmds, cmdn, true); +arm_smmu_cmdq_batch_submit (smmu, &cmds); } static int arm_smmu_alloc_cd_leaf_table(struct arm_smmu_device *smmu,
@@ -2160,11 +2177,11 @@ static int arm_smmu_atc_inv_master(struct arm_smmu_master *master)static int arm_smmu_atc_inv_domain (struct arm_smmu_domain *smmu_domain,int ssid,unsigned long iova,size_t size) { -int i, cmdn =0 ; +int i;unsigned long flags;struct arm_smmu_cmdq_ent cmd;struct arm_smmu_master *master; -u64 cmds[CMDQ_BATCH_ENTRIES *CMDQ_ENT_DWORDS ]; +struct arm_smmu_cmdq_batch cmds = {};if (!(smmu_domain->smmu->features &ARM_SMMU_FEAT_ATS ))return 0 ;
@@ -2194,20 +2211,13 @@ static int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,continue ;for (i =0 ; i < master->num_sids; i++) { -if (cmdn ==CMDQ_BATCH_ENTRIES ) { -arm_smmu_cmdq_issue_cmdlist (smmu_domain->smmu, - cmds, cmdn, false); - cmdn =0 ; - } - cmd.atc.sid = master->sids[i]; -arm_smmu_cmdq_build_cmd (&cmds[cmdn *CMDQ_ENT_DWORDS ], &cmd); - cmdn++; +arm_smmu_cmdq_batch_add (smmu_domain->smmu, &cmds, &cmd); } }spin_unlock_irqrestore (&smmu_domain->devices_lock, flags); -return arm_smmu_cmdq_issue_cmdlist (smmu_domain->smmu, cmds, cmdn, true); +return arm_smmu_cmdq_batch_submit (smmu_domain->smmu, &cmds); }/* IO_PGTABLE API */
@@ -2242,10 +2252,9 @@ static void arm_smmu_tlb_inv_range(unsigned long iova, size_t size, size_t granule, bool leaf, struct arm_smmu_domain *smmu_domain) { - u64 cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS]; struct arm_smmu_device *smmu = smmu_domain->smmu;unsigned long start = iova, end = iova + size; -int i =0 ; +struct arm_smmu_cmdq_batch cmds = {}; struct arm_smmu_cmdq_ent cmd = { .tlbi = { .leaf = leaf,
@@ -2264,18 +2273,11 @@ static void arm_smmu_tlb_inv_range(unsigned long iova, size_t size, }while (iova < end) { -if (i ==CMDQ_BATCH_ENTRIES ) { -arm_smmu_cmdq_issue_cmdlist (smmu, cmds, i, false); - i =0 ; - } - cmd.tlbi.addr = iova; -arm_smmu_cmdq_build_cmd (&cmds[i *CMDQ_ENT_DWORDS ], &cmd); +arm_smmu_cmdq_batch_add (smmu, &cmds, &cmd); iova += granule; - i++; } - -arm_smmu_cmdq_issue_cmdlist (smmu, cmds, i, true); +arm_smmu_cmdq_batch_submit (smmu, &cmds);/* * Unfortunately, this can't be leaf-only since we may have
Signature · 1 line
2.25.0
_______________________________________________
iommu mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/iommu
Will Deacon
maintainerQuoted 15 lines · Jean-Philippe Brucker
Quoted 5 lines · Rob Herring
Similar to commit
2af2e72b18b4("iommu/arm-smmu-v3: Defer TLB invalidation until ->iotlb_sync()"), build up a list of ATC invalidation commands and submit them all at once to the command queue instead of one-by-one.Quoted 8 lines
As there is only one caller of
arm_smmu_atc_inv_master()left, we can simplify it and avoid passing in structarm_smmu_cmdq_ent.Cc Jean-Philippe Brucker <[email protected]>Cc Will Deacon <[email protected]>Cc Robin Murphy <[email protected]>Cc Joerg Roedel <[email protected]>Signed-off-by Rob Herring <[email protected]>
Quoted 6 lines
Reviewed-by Jean-Philippe Brucker <[email protected]>Since I'm adding a third user of cmdq batching [1], I had a go at factoring them. I can send the attached patch with my next version, if it looks OK.
Yes, the idea looks good to me. Feel free to pick this stuff into a series so that I can review it for 5.7.
Will
_______________________________________________
iommu mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/iommu