hw/riscv: iommu: Support automatic PTE A/D updates
cover hw/riscv: iommu: Support automatic PTE A/D updates
This series adds support for automatic updates of page-table accessed
(A) and dirty (D) bits in the RISC-V IOMMU, as specified by the RISC-V IOMMU architecture. The feature is advertised through the AMO_HWAD capability and controlled by the SADE and GADE bits in device contexts.
The first two patches fix implicit G-stage page-table access handling and IOATC permission checks. The third patch implements atomic A/D bit updates using CAS, with page-table walk retries on concurrent modifications and G-stage permission checks for implicit writes during nested translation.
The implementation has been validated with QTests, Linux DMA tests under KVM and TCG, and concurrency tests covering CAS retries, PTE modifications, and G-stage mapping changes.
Tan Chi (3):
hw/riscv: iommu: Fix implicit G-stage page-table accesses hw/riscv: iommu: Check permissions on IOATC hits hw/riscv: iommu: Support automatic PTE A/D updates
hw/riscv/riscv-iommu.c | 256 +++++++++++++++++++++++++++++++++++++---- hw/riscv/riscv-iommu.h | 1 + 2 files changed, 232 insertions(+), 25 deletions(-)
Signature · 1 line
2.53.0
PATCH 1/3 hw/riscv: iommu: Fix implicit G-stage page-table accesses
When translating a DMA write, G-stage accesses to first-stage page tables are implicit reads. The current implementation checks permissions using the original DMA access type, which may incorrectly require write permission for these reads.
Use read permission for implicit page-table accesses while preserving the original DMA access type for the final translation and fault reporting.
Also track the fault address separately from the translation result and set the implicit-access flag in iotval2 for guest-page faults. Route address-range failures through the common page-fault path to report the correct fault address.
Signed-off-by Tan Chi <[email protected]>
--- hw/riscv/riscv-iommu.c | 60 ++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 22 deletions(-)
hw/riscv/riscv-iommu.c +38 -22
@@ -273,13 +273,15 @@ static hwaddr riscv_iommu_napot_page_mask(hwaddr ppn, hwaddr addr, hwaddr *out)* Keeping them separate for now, as error reporting and flow specifics are * sufficiently different for separate implementation. * -* @s : IOMMU Device State -* @ctx : Translation context for device id and process address space id. -* @iotlb : translation data: physical address and access mode. -* @return : success or fault cause code. +* @s : IOMMU Device State +* @ctx : Translation context for device id and process +* address space id. +* @iotlb : translation data: physical address and access mode. +* @fault_iotval2 : Fault address and implicit-access flags for iotval2. +* @return : success or fault cause code. */ static int riscv_iommu_spa_fetch (RISCVIOMMUState *s,RISCVIOMMUContext *ctx, -IOMMUTLBEntry *iotlb) +IOMMUTLBEntry *iotlb,uint64_t *fault_iotval2) {IOMMUAccessFlags trans_perm =IOMMU_NONE ;dma_addr_t addr, base;
@@ -299,10 +301,10 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,MemTxResult ret;bool pv = !!ctx->process_id;/* -* Keep the request permission separate from iotlb->perm. G-stage +* Keep the request permission separate from iotlb->perm. G-stage * walks translate S-stage PTE addresses before the real leaf is * reached, but permission checks and fault types must still use the -* original request. A successful walk leaves iotlb->perm with the +* original request. A successful walk leaves iotlb->perm with the * effective leaf permission for the translation cache. */ const IOMMUAccessFlags req_perm = iotlb->perm;
@@ -424,6 +426,14 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,const dma_addr_t pte_addr = base + idx * sc[pass].ptesize;const bool ade = ctx->tc & (pass ?RISCV_IOMMU_DC_TC_GADE :RISCV_IOMMU_DC_TC_SADE ); +/* +* G-stage accesses to S-stage page tables are implicit reads. +* The original DMA permission is used for the final data access. +*/ +const bool implicit_read = + pass ==G_STAGE && en_s && sc[S_STAGE ].step != sc[S_STAGE ].levels; +const IOMMUAccessFlags acc_perm = + implicit_read ?IOMMU_RO : req_perm;if (ade && !(s->cap &RISCV_IOMMU_CAP_AMO_HWAD )) {/* GADE/SADE are reserved bits if AMO_HWAD is cleared. */
@@ -442,15 +452,11 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx, masked_msbs = (addr >> (va_len -1 )) & mask;if (masked_msbs !=0 && masked_msbs != mask) { -return (req_perm &IOMMU_WO ) ? -RISCV_IOMMU_FQ_CAUSE_WR_FAULT_S : -RISCV_IOMMU_FQ_CAUSE_RD_FAULT_S ; + goto page_fault; } }else {if ((addr & va_mask) != addr) { -return (req_perm &IOMMU_WO ) ? -RISCV_IOMMU_FQ_CAUSE_WR_FAULT_VS : -RISCV_IOMMU_FQ_CAUSE_RD_FAULT_VS ; + goto page_fault; } } }
@@ -499,13 +505,13 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,* non-user mode leaf PTE and !pv we need to fault. */ break ; - }else if ((req_perm &IOMMU_RO ) && !(pte &PTE_R )) { + }else if ((acc_perm &IOMMU_RO ) && !(pte &PTE_R )) {break ;/* Read access check failed */ - }else if ((req_perm &IOMMU_WO ) && !(pte &PTE_W )) { + }else if ((acc_perm &IOMMU_WO ) && !(pte &PTE_W )) {break ;/* Write access check failed */ }else if (!ade && !(pte &PTE_A )) {break ;/* Access bit not set */ - }else if ((req_perm &IOMMU_WO ) && !ade && !(pte &PTE_D )) { + }else if ((acc_perm &IOMMU_WO ) && !ade && !(pte &PTE_D )) {break ;/* Dirty bit not set */ }else if (pass ==G_STAGE && !(pte &PTE_U )) {/*
@@ -587,13 +593,22 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx, } }while (1 ); +page_fault:/* * riscv_iommu_translate() will receive a fault and then call -* riscv_iommu_report_fault() using iotlb->translated_addr -* as iotval2. Update translated_addr it with the latest -* translated addr we have. +* riscv_iommu_report_fault() using fault_iotval2 as iotval2. +* Update fault_iotval2 with the latest translated address. +* For guest-page faults, set the implicit first-stage +* page-table access flag in the low bit. */ - iotlb->translated_addr = addr; + *fault_iotval2 = addr; + +if (pass ==G_STAGE ) { +bool implicit_read = + en_s && sc[S_STAGE ].step != sc[S_STAGE ].levels; + *fault_iotval2 = (addr & ~3ULL ) | + (implicit_read ?1 :0 ); + } return (req_perm & IOMMU_WO) ? (pass ? RISCV_IOMMU_FQ_CAUSE_WR_FAULT_VS :
@@ -1740,6 +1755,7 @@ static int riscv_iommu_translate(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,bool enable_pri;hwaddr phys =0 ;int fault; +uint64_t fault_iotval2 =0 ;riscv_iommu_hpm_incr_ctr (s, ctx,RISCV_IOMMU_HPMEVENT_URQ );
@@ -1776,7 +1792,7 @@ static int riscv_iommu_translate(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,riscv_iommu_hpm_incr_ctr (s, ctx,RISCV_IOMMU_HPMEVENT_TLB_MISS );/* Translate using device directory / page table information. */ - fault =riscv_iommu_spa_fetch (s, ctx, iotlb); + fault =riscv_iommu_spa_fetch (s, ctx, iotlb, &fault_iotval2);if (!fault && iotlb->target_as == &s->trap_as) {/* Do not cache trapped MSI translations */
@@ -1827,7 +1843,7 @@ done: }riscv_iommu_report_fault (s, ctx, ttype, fault, enable_pid, - iotlb->iova, iotlb->translated_addr); + iotlb->iova, fault_iotval2);return fault; }
Signature · 1 line
2.53.0
Re: [PATCH 1/3] hw/riscv: iommu: Fix implicit G-stage page-table accesses
Quoted 114 lines · Tan Chi
When translating a DMA write, G-stage accesses to first-stage page tables are implicit reads. The current implementation checks permissions using the original DMA access type, which may incorrectly require write permission for these reads.
Use read permission for implicit page-table accesses while preserving the original DMA access type for the final translation and fault reporting.
Also track the fault address separately from the translation result and set the implicit-access flag in iotval2 for guest-page faults. Route address-range failures through the common page-fault path to report the correct fault address.
Signed-off-by Tan Chi <[email protected]>--- hw/riscv/riscv-iommu.c | 60 ++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 22 deletions(-)hw/riscv/riscv-iommu.c +23
-16@@ -273,13 +273,15 @@ static hwaddr riscv_iommu_napot_page_mask(hwaddr ppn, hwaddr addr, hwaddr *out)* Keeping them separate for now, as error reporting and flow specifics are * sufficiently different for separate implementation. * -* @s : IOMMU Device State -* @ctx : Translation context for device id and process address space id. -* @iotlb : translation data: physical address and access mode. -* @return : success or fault cause code. +* @s : IOMMU Device State +* @ctx : Translation context for device id and process +* address space id. +* @iotlb : translation data: physical address and access mode. +* @fault_iotval2 : Fault address and implicit-access flags for iotval2. +* @return : success or fault cause code. */ static int riscv_iommu_spa_fetch (RISCVIOMMUState *s,RISCVIOMMUContext *ctx, -IOMMUTLBEntry *iotlb) +IOMMUTLBEntry *iotlb,uint64_t *fault_iotval2) {IOMMUAccessFlags trans_perm =IOMMU_NONE ;dma_addr_t addr, base;@@ -299,10 +301,10 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,MemTxResult ret;bool pv = !!ctx->process_id;/* -* Keep the request permission separate from iotlb->perm. G-stage +* Keep the request permission separate from iotlb->perm. G-stage * walks translate S-stage PTE addresses before the real leaf is * reached, but permission checks and fault types must still use the -* original request. A successful walk leaves iotlb->perm with the +* original request. A successful walk leaves iotlb->perm with the * effective leaf permission for the translation cache. */ const IOMMUAccessFlags req_perm = iotlb->perm;@@ -424,6 +426,14 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,const dma_addr_t pte_addr = base + idx * sc[pass].ptesize;const bool ade = ctx->tc & (pass ?RISCV_IOMMU_DC_TC_GADE :RISCV_IOMMU_DC_TC_SADE ); +/* +* G-stage accesses to S-stage page tables are implicit reads. +* The original DMA permission is used for the final data access. +*/ +const bool implicit_read = + pass ==G_STAGE && en_s && sc[S_STAGE ].step != sc[S_STAGE ].levels; +const IOMMUAccessFlags acc_perm = + implicit_read ?IOMMU_RO : req_perm;if (ade && !(s->cap &RISCV_IOMMU_CAP_AMO_HWAD )) {/* GADE/SADE are reserved bits if AMO_HWAD is cleared. */ @@ -442,15 +452,11 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx, masked_msbs = (addr >> (va_len -1 )) & mask;if (masked_msbs !=0 && masked_msbs != mask) { -return (req_perm &IOMMU_WO ) ? -RISCV_IOMMU_FQ_CAUSE_WR_FAULT_S : -RISCV_IOMMU_FQ_CAUSE_RD_FAULT_S ; + goto page_fault; } }else {if ((addr & va_mask) != addr) { -return (req_perm &IOMMU_WO ) ? -RISCV_IOMMU_FQ_CAUSE_WR_FAULT_VS : -RISCV_IOMMU_FQ_CAUSE_RD_FAULT_VS ; + goto page_fault; } } }@@ -499,13 +505,13 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,* non-user mode leaf PTE and !pv we need to fault. */ break ; - }else if ((req_perm &IOMMU_RO ) && !(pte &PTE_R )) { + }else if ((acc_perm &IOMMU_RO ) && !(pte &PTE_R )) {break ;/* Read access check failed */ - }else if ((req_perm &IOMMU_WO ) && !(pte &PTE_W )) { + }else if ((acc_perm &IOMMU_WO ) && !(pte &PTE_W )) {break ;/* Write access check failed */ }else if (!ade && !(pte &PTE_A )) {break ;/* Access bit not set */ - }else if ((req_perm &IOMMU_WO ) && !ade && !(pte &PTE_D )) { + }else if ((acc_perm &IOMMU_WO ) && !ade && !(pte &PTE_D )) {break ;/* Dirty bit not set */ }else if (pass ==G_STAGE && !(pte &PTE_U )) {/* @@ -587,13 +593,22 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx, } }while (1 ); +page_fault:/* * riscv_iommu_translate() will receive a fault and then call
hw/riscv/riscv-iommu.c static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx, · line 604
Quoted 10 lines
hw/riscv/riscv-iommu.c +5
-4-* riscv_iommu_report_fault() using iotlb->translated_addr -* as iotval2. Update translated_addr it with the latest -* translated addr we have. +* riscv_iommu_report_fault() using fault_iotval2 as iotval2. +* Update fault_iotval2 with the latest translated address. +* For guest-page faults, set the implicit first-stage +* page-table access flag in the low bit. */ - iotlb->translated_addr = addr; + *fault_iotval2 = addr;
The previous logic was added as a failsafe to not leave iotval2 zeroed during a fault (Gitlab #3559). Now that you're adding a logic to calculate fault_iotval2 based on G_STAGE I think we can change the existing logic to closely match the spec, i.e. iotval2 is supposed to be 0 for S_STAGE faults.
You can set:
*fault_iotval2 = 0;
At this point and then we'll calculate the right value if pass == G_STAGE down below. I tested this change with Gitlab #3559 and it's still a pass. You'll probably want to re-run your own testcases again to be sure but I'm fairly certain this is ok to do.
All the other changes LGTM. Thanks,
Daniel
Quoted 36 lines
hw/riscv/riscv-iommu.c +10
-2+ +if (pass ==G_STAGE ) { +bool implicit_read = + en_s && sc[S_STAGE ].step != sc[S_STAGE ].levels; + *fault_iotval2 = (addr & ~3ULL ) | + (implicit_read ?1 :0 ); + } return (req_perm & IOMMU_WO) ? (pass ? RISCV_IOMMU_FQ_CAUSE_WR_FAULT_VS :@@ -1740,6 +1755,7 @@ static int riscv_iommu_translate(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,bool enable_pri;hwaddr phys =0 ;int fault; +uint64_t fault_iotval2 =0 ;riscv_iommu_hpm_incr_ctr (s, ctx,RISCV_IOMMU_HPMEVENT_URQ );@@ -1776,7 +1792,7 @@ static int riscv_iommu_translate(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,riscv_iommu_hpm_incr_ctr (s, ctx,RISCV_IOMMU_HPMEVENT_TLB_MISS );/* Translate using device directory / page table information. */ - fault =riscv_iommu_spa_fetch (s, ctx, iotlb); + fault =riscv_iommu_spa_fetch (s, ctx, iotlb, &fault_iotval2);if (!fault && iotlb->target_as == &s->trap_as) {/* Do not cache trapped MSI translations */ @@ -1827,7 +1843,7 @@ done: }riscv_iommu_report_fault (s, ctx, ttype, fault, enable_pid, - iotlb->iova, iotlb->translated_addr); + iotlb->iova, fault_iotval2);return fault; }
Re: [PATCH 1/3] hw/riscv: iommu: Fix implicit G-stage page-table accesses
In reply to Tan ChiQuoted 118 lines · Tan Chi
When translating a DMA write, G-stage accesses to first-stage page tables are implicit reads. The current implementation checks permissions using the original DMA access type, which may incorrectly require write permission for these reads.
Use read permission for implicit page-table accesses while preserving the original DMA access type for the final translation and fault reporting.
Also track the fault address separately from the translation result and set the implicit-access flag in iotval2 for guest-page faults. Route address-range failures through the common page-fault path to report the correct fault address.
Signed-off-by Tan Chi <[email protected]>--- hw/riscv/riscv-iommu.c | 60 ++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 22 deletions(-)hw/riscv/riscv-iommu.c +24
-19@@ -273,13 +273,15 @@ static hwaddr riscv_iommu_napot_page_mask(hwaddr ppn, hwaddr addr, hwaddr *out)* Keeping them separate for now, as error reporting and flow specifics are * sufficiently different for separate implementation. * -* @s : IOMMU Device State -* @ctx : Translation context for device id and process address space id. -* @iotlb : translation data: physical address and access mode. -* @return : success or fault cause code. +* @s : IOMMU Device State +* @ctx : Translation context for device id and process +* address space id. +* @iotlb : translation data: physical address and access mode. +* @fault_iotval2 : Fault address and implicit-access flags for iotval2. +* @return : success or fault cause code. */ static int riscv_iommu_spa_fetch (RISCVIOMMUState *s,RISCVIOMMUContext *ctx, -IOMMUTLBEntry *iotlb) +IOMMUTLBEntry *iotlb,uint64_t *fault_iotval2) {IOMMUAccessFlags trans_perm =IOMMU_NONE ;dma_addr_t addr, base;@@ -299,10 +301,10 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,MemTxResult ret;bool pv = !!ctx->process_id;/* -* Keep the request permission separate from iotlb->perm. G-stage +* Keep the request permission separate from iotlb->perm. G-stage * walks translate S-stage PTE addresses before the real leaf is * reached, but permission checks and fault types must still use the -* original request. A successful walk leaves iotlb->perm with the +* original request. A successful walk leaves iotlb->perm with the * effective leaf permission for the translation cache. */ const IOMMUAccessFlags req_perm = iotlb->perm;@@ -424,6 +426,14 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,const dma_addr_t pte_addr = base + idx * sc[pass].ptesize;const bool ade = ctx->tc & (pass ?RISCV_IOMMU_DC_TC_GADE :RISCV_IOMMU_DC_TC_SADE ); +/* +* G-stage accesses to S-stage page tables are implicit reads. +* The original DMA permission is used for the final data access. +*/ +const bool implicit_read = + pass ==G_STAGE && en_s && sc[S_STAGE ].step != sc[S_STAGE ].levels; +const IOMMUAccessFlags acc_perm = + implicit_read ?IOMMU_RO : req_perm;if (ade && !(s->cap &RISCV_IOMMU_CAP_AMO_HWAD )) {/* GADE/SADE are reserved bits if AMO_HWAD is cleared. */ @@ -442,15 +452,11 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx, masked_msbs = (addr >> (va_len -1 )) & mask;if (masked_msbs !=0 && masked_msbs != mask) { -return (req_perm &IOMMU_WO ) ? -RISCV_IOMMU_FQ_CAUSE_WR_FAULT_S : -RISCV_IOMMU_FQ_CAUSE_RD_FAULT_S ; + goto page_fault; } }else {if ((addr & va_mask) != addr) { -return (req_perm &IOMMU_WO ) ? -RISCV_IOMMU_FQ_CAUSE_WR_FAULT_VS : -RISCV_IOMMU_FQ_CAUSE_RD_FAULT_VS ; + goto page_fault; } } }@@ -499,13 +505,13 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,* non-user mode leaf PTE and !pv we need to fault. */ break ; - }else if ((req_perm &IOMMU_RO ) && !(pte &PTE_R )) { + }else if ((acc_perm &IOMMU_RO ) && !(pte &PTE_R )) {break ;/* Read access check failed */ - }else if ((req_perm &IOMMU_WO ) && !(pte &PTE_W )) { + }else if ((acc_perm &IOMMU_WO ) && !(pte &PTE_W )) {break ;/* Write access check failed */ }else if (!ade && !(pte &PTE_A )) {break ;/* Access bit not set */ - }else if ((req_perm &IOMMU_WO ) && !ade && !(pte &PTE_D )) { + }else if ((acc_perm &IOMMU_WO ) && !ade && !(pte &PTE_D )) {break ;/* Dirty bit not set */ }else if (pass ==G_STAGE && !(pte &PTE_U )) {/* @@ -587,13 +593,22 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx, } }while (1 ); +page_fault:/* * riscv_iommu_translate() will receive a fault and then call -* riscv_iommu_report_fault() using iotlb->translated_addr -* as iotval2. Update translated_addr it with the latest -* translated addr we have. +* riscv_iommu_report_fault() using fault_iotval2 as iotval2.
hw/riscv/riscv-iommu.c static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx, · line 608
Quoted 10 lines
hw/riscv/riscv-iommu.c +8
-1+* Update fault_iotval2 with the latest translated address. +* For guest-page faults, set the implicit first-stage +* page-table access flag in the low bit. */ - iotlb->translated_addr = addr; + *fault_iotval2 = addr; + +if (pass ==G_STAGE ) { +bool implicit_read = + en_s && sc[S_STAGE ].step != sc[S_STAGE ].levels;
One nit pick:
Duplicated implicit_read declaration and computation.
Suggest to declare implicit_read within function scope. Like:
const IOMMUAccessFlags req_perm = iotlb->perm;
+ bool implicit_read;
...
do {
...
+ implicit_read =
+ pass == G_STAGE && en_s &&
+ sc[S_STAGE].step != sc[S_STAGE].levels;
...
} while (1);
page_fault:
*fault_iotval2 = addr;
if (pass == G_STAGE) {
*fault_iotval2 = (addr & ~3ULL) | (implicit_read ? 1 : 0); }
Otherwise,
Reviewed-by Nutty Liu <[email protected]>
Thanks,
Nutty
Quoted 32 lines
hw/riscv/riscv-iommu.c +6
-2+ *fault_iotval2 = (addr & ~3ULL ) | + (implicit_read ?1 :0 ); + } return (req_perm & IOMMU_WO) ? (pass ? RISCV_IOMMU_FQ_CAUSE_WR_FAULT_VS :@@ -1740,6 +1755,7 @@ static int riscv_iommu_translate(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,bool enable_pri;hwaddr phys =0 ;int fault; +uint64_t fault_iotval2 =0 ;riscv_iommu_hpm_incr_ctr (s, ctx,RISCV_IOMMU_HPMEVENT_URQ );@@ -1776,7 +1792,7 @@ static int riscv_iommu_translate(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,riscv_iommu_hpm_incr_ctr (s, ctx,RISCV_IOMMU_HPMEVENT_TLB_MISS );/* Translate using device directory / page table information. */ - fault =riscv_iommu_spa_fetch (s, ctx, iotlb); + fault =riscv_iommu_spa_fetch (s, ctx, iotlb, &fault_iotval2);if (!fault && iotlb->target_as == &s->trap_as) {/* Do not cache trapped MSI translations */ @@ -1827,7 +1843,7 @@ done: }riscv_iommu_report_fault (s, ctx, ttype, fault, enable_pid, - iotlb->iova, iotlb->translated_addr); + iotlb->iova, fault_iotval2);return fault; }
Re: [PATCH 1/3] hw/riscv: iommu: Fix implicit G-stage page-table accesses
In reply to Tan ChiQuoted 118 lines · Tan Chi
When translating a DMA write, G-stage accesses to first-stage page tables are implicit reads. The current implementation checks permissions using the original DMA access type, which may incorrectly require write permission for these reads.
Use read permission for implicit page-table accesses while preserving the original DMA access type for the final translation and fault reporting.
Also track the fault address separately from the translation result and set the implicit-access flag in iotval2 for guest-page faults. Route address-range failures through the common page-fault path to report the correct fault address.
Signed-off-by Tan Chi <[email protected]>--- hw/riscv/riscv-iommu.c | 60 ++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 22 deletions(-)hw/riscv/riscv-iommu.c +24
-19@@ -273,13 +273,15 @@ static hwaddr riscv_iommu_napot_page_mask(hwaddr ppn, hwaddr addr, hwaddr *out)* Keeping them separate for now, as error reporting and flow specifics are * sufficiently different for separate implementation. * -* @s : IOMMU Device State -* @ctx : Translation context for device id and process address space id. -* @iotlb : translation data: physical address and access mode. -* @return : success or fault cause code. +* @s : IOMMU Device State +* @ctx : Translation context for device id and process +* address space id. +* @iotlb : translation data: physical address and access mode. +* @fault_iotval2 : Fault address and implicit-access flags for iotval2. +* @return : success or fault cause code. */ static int riscv_iommu_spa_fetch (RISCVIOMMUState *s,RISCVIOMMUContext *ctx, -IOMMUTLBEntry *iotlb) +IOMMUTLBEntry *iotlb,uint64_t *fault_iotval2) {IOMMUAccessFlags trans_perm =IOMMU_NONE ;dma_addr_t addr, base;@@ -299,10 +301,10 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,MemTxResult ret;bool pv = !!ctx->process_id;/* -* Keep the request permission separate from iotlb->perm. G-stage +* Keep the request permission separate from iotlb->perm. G-stage * walks translate S-stage PTE addresses before the real leaf is * reached, but permission checks and fault types must still use the -* original request. A successful walk leaves iotlb->perm with the +* original request. A successful walk leaves iotlb->perm with the * effective leaf permission for the translation cache. */ const IOMMUAccessFlags req_perm = iotlb->perm;@@ -424,6 +426,14 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,const dma_addr_t pte_addr = base + idx * sc[pass].ptesize;const bool ade = ctx->tc & (pass ?RISCV_IOMMU_DC_TC_GADE :RISCV_IOMMU_DC_TC_SADE ); +/* +* G-stage accesses to S-stage page tables are implicit reads. +* The original DMA permission is used for the final data access. +*/ +const bool implicit_read = + pass ==G_STAGE && en_s && sc[S_STAGE ].step != sc[S_STAGE ].levels; +const IOMMUAccessFlags acc_perm = + implicit_read ?IOMMU_RO : req_perm;if (ade && !(s->cap &RISCV_IOMMU_CAP_AMO_HWAD )) {/* GADE/SADE are reserved bits if AMO_HWAD is cleared. */ @@ -442,15 +452,11 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx, masked_msbs = (addr >> (va_len -1 )) & mask;if (masked_msbs !=0 && masked_msbs != mask) { -return (req_perm &IOMMU_WO ) ? -RISCV_IOMMU_FQ_CAUSE_WR_FAULT_S : -RISCV_IOMMU_FQ_CAUSE_RD_FAULT_S ; + goto page_fault; } }else {if ((addr & va_mask) != addr) { -return (req_perm &IOMMU_WO ) ? -RISCV_IOMMU_FQ_CAUSE_WR_FAULT_VS : -RISCV_IOMMU_FQ_CAUSE_RD_FAULT_VS ; + goto page_fault; } } }@@ -499,13 +505,13 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,* non-user mode leaf PTE and !pv we need to fault. */ break ; - }else if ((req_perm &IOMMU_RO ) && !(pte &PTE_R )) { + }else if ((acc_perm &IOMMU_RO ) && !(pte &PTE_R )) {break ;/* Read access check failed */ - }else if ((req_perm &IOMMU_WO ) && !(pte &PTE_W )) { + }else if ((acc_perm &IOMMU_WO ) && !(pte &PTE_W )) {break ;/* Write access check failed */ }else if (!ade && !(pte &PTE_A )) {break ;/* Access bit not set */ - }else if ((req_perm &IOMMU_WO ) && !ade && !(pte &PTE_D )) { + }else if ((acc_perm &IOMMU_WO ) && !ade && !(pte &PTE_D )) {break ;/* Dirty bit not set */ }else if (pass ==G_STAGE && !(pte &PTE_U )) {/* @@ -587,13 +593,22 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx, } }while (1 ); +page_fault:/* * riscv_iommu_translate() will receive a fault and then call -* riscv_iommu_report_fault() using iotlb->translated_addr -* as iotval2. Update translated_addr it with the latest -* translated addr we have. +* riscv_iommu_report_fault() using fault_iotval2 as iotval2.
hw/riscv/riscv-iommu.c static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx, · line 608
Quoted 10 lines
hw/riscv/riscv-iommu.c +8
-1+* Update fault_iotval2 with the latest translated address. +* For guest-page faults, set the implicit first-stage +* page-table access flag in the low bit. */ - iotlb->translated_addr = addr; + *fault_iotval2 = addr; + +if (pass ==G_STAGE ) { +bool implicit_read = + en_s && sc[S_STAGE ].step != sc[S_STAGE ].levels;
One nit pick:
Duplicated implicit_read declaration and computation.
Suggest to declare implicit_read within function scope. Like:
const IOMMUAccessFlags req_perm = iotlb->perm; + bool implicit_read;
...
do {
...
+ implicit_read =
+ pass == G_STAGE && en_s &&
+ sc[S_STAGE].step != sc[S_STAGE].levels;
...
} while (1);
page_fault:
*fault_iotval2 = addr;
if (pass == G_STAGE) {
*fault_iotval2 = (addr & ~3ULL) | (implicit_read ? 1 : 0); }
Otherwise,
Reviewed-by Nutty Liu <[email protected]>
Thanks,
Nutty
Quoted 32 lines
hw/riscv/riscv-iommu.c +6
-2+ *fault_iotval2 = (addr & ~3ULL ) | + (implicit_read ?1 :0 ); + } return (req_perm & IOMMU_WO) ? (pass ? RISCV_IOMMU_FQ_CAUSE_WR_FAULT_VS :@@ -1740,6 +1755,7 @@ static int riscv_iommu_translate(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,bool enable_pri;hwaddr phys =0 ;int fault; +uint64_t fault_iotval2 =0 ;riscv_iommu_hpm_incr_ctr (s, ctx,RISCV_IOMMU_HPMEVENT_URQ );@@ -1776,7 +1792,7 @@ static int riscv_iommu_translate(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,riscv_iommu_hpm_incr_ctr (s, ctx,RISCV_IOMMU_HPMEVENT_TLB_MISS );/* Translate using device directory / page table information. */ - fault =riscv_iommu_spa_fetch (s, ctx, iotlb); + fault =riscv_iommu_spa_fetch (s, ctx, iotlb, &fault_iotval2);if (!fault && iotlb->target_as == &s->trap_as) {/* Do not cache trapped MSI translations */ @@ -1827,7 +1843,7 @@ done: }riscv_iommu_report_fault (s, ctx, ttype, fault, enable_pid, - iotlb->iova, iotlb->translated_addr); + iotlb->iova, fault_iotval2);return fault; }
PATCH 2/3 hw/riscv: iommu: Check permissions on IOATC hits
In reply to Tan ChiThe current implementation reuses cached translations without checking whether their permissions satisfy the requested access.
Only reuse an IOATC entry when its permissions cover the current request. Otherwise, perform a page-table walk to check the requested access. Preserve the existing behavior for requests with IOMMU_NONE.
This is also needed for hardware A/D updates, where mappings with a clear D bit must not cache write permission until the bit has been set.
Signed-off-by Tan Chi <[email protected]>
--- hw/riscv/riscv-iommu.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
hw/riscv/riscv-iommu.c +3 -1
@@ -1781,7 +1781,9 @@ static int riscv_iommu_translate(RISCVIOMMUState *s, RISCVIOMMUContext *ctx, phys = iot->phys; }qemu_mutex_unlock (&s->iot_lock); -if (perm !=IOMMU_NONE ) { +if (perm !=IOMMU_NONE && + (iotlb->perm ==IOMMU_NONE || + (perm & iotlb->perm) == iotlb->perm)) { iotlb->translated_addr =PPN_PHYS (phys); iotlb->addr_mask = ~TARGET_PAGE_MASK ; iotlb->perm = perm;
Signature · 1 line
2.53.0
Re: [PATCH 2/3] hw/riscv: iommu: Check permissions on IOATC hits
Quoted 7 lines · Tan Chi
The current implementation reuses cached translations without checking whether their permissions satisfy the requested access.
Only reuse an IOATC entry when its permissions cover the current request. Otherwise, perform a page-table walk to check the requested access. Preserve the existing behavior for requests with
IOMMU_NONE.
Quoted 5 lines
This is also needed for hardware A/D updates, where mappings with a clear D bit must not cache write permission until the bit has been set.
Signed-off-by Tan Chi <[email protected]>---
Reviewed-by Daniel Henrique Barboza <[email protected]>
Quoted 18 lines
hw/riscv/riscv-iommu.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)hw/riscv/riscv-iommu.c +3
-1@@ -1781,7 +1781,9 @@ static int riscv_iommu_translate(RISCVIOMMUState *s, RISCVIOMMUContext *ctx, phys = iot->phys; }qemu_mutex_unlock (&s->iot_lock); -if (perm !=IOMMU_NONE ) { +if (perm !=IOMMU_NONE && + (iotlb->perm ==IOMMU_NONE || + (perm & iotlb->perm) == iotlb->perm)) { iotlb->translated_addr =PPN_PHYS (phys); iotlb->addr_mask = ~TARGET_PAGE_MASK ; iotlb->perm = perm;
Re: [PATCH 2/3] hw/riscv: iommu: Check permissions on IOATC hits
In reply to Tan ChiQuoted 11 lines · Tan Chi
The current implementation reuses cached translations without checking whether their permissions satisfy the requested access.
Only reuse an IOATC entry when its permissions cover the current request. Otherwise, perform a page-table walk to check the requested access. Preserve the existing behavior for requests with
IOMMU_NONE.This is also needed for hardware A/D updates, where mappings with a clear D bit must not cache write permission until the bit has been set.
Signed-off-by Tan Chi <[email protected]>
Reviewed-by Nutty Liu <[email protected]>
Thanks,
Nutty
Quoted 19 lines
--- hw/riscv/riscv-iommu.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)hw/riscv/riscv-iommu.c +3
-1@@ -1781,7 +1781,9 @@ static int riscv_iommu_translate(RISCVIOMMUState *s, RISCVIOMMUContext *ctx, phys = iot->phys; }qemu_mutex_unlock (&s->iot_lock); -if (perm !=IOMMU_NONE ) { +if (perm !=IOMMU_NONE && + (iotlb->perm ==IOMMU_NONE || + (perm & iotlb->perm) == iotlb->perm)) { iotlb->translated_addr =PPN_PHYS (phys); iotlb->addr_mask = ~TARGET_PAGE_MASK ; iotlb->perm = perm;
PATCH 3/3 hw/riscv: iommu: Support automatic PTE A/D updates
In reply to Tan ChiAdd support for the AMO_HWAD capability to automatically update the accessed and dirty bits of leaf PTEs when SADE or GADE is enabled.
Use atomic compare-and-exchange to update PTEs, restarting the page-table walk if a concurrent modification is detected.
For nested translation, check G-stage write permissions before updating an S-stage PTE. Restart the walk if the write translation resolves to a different physical address.
Avoid caching write permission when the PTE's dirty bit is clear, so subsequent writes can trigger the required update.
Expose the capability through the new "amo-hwad" device property, disabled by default.
Signed-off-by Tan Chi <[email protected]>
--- hw/riscv/riscv-iommu.c | 196 ++++++++++++++++++++++++++++++++++++++++- hw/riscv/riscv-iommu.h | 1 + 2 files changed, 193 insertions(+), 4 deletions(-)
hw/riscv/riscv-iommu.c +192 -4
@@ -265,6 +265,128 @@ static hwaddr riscv_iommu_napot_page_mask(hwaddr ppn, hwaddr addr, hwaddr *out)return page_mask; } +typedef enum RISCVIOMMUADResult { +RISCV_IOMMU_AD_SUCCESS , +RISCV_IOMMU_AD_RETRY , +RISCV_IOMMU_AD_FAULT , +}RISCVIOMMUADResult ; + +/* +* Atomically update the A/D bits of a leaf PTE. +* Retry the page-table walk if the PTE has changed. +*/ +static RISCVIOMMUADResult +riscv_iommu_ad_update (RISCVIOMMUState *s,hwaddr pte_addr, +unsigned int pte_size,uint64_t pte,uint64_t updated_pte) +{ +MemoryRegion *mr; +hwaddr offset, len = pte_size; +uint64_t old_pte; +void *pte_ptr; + +if (pte_addr & (pte_size -1 )) { +return RISCV_IOMMU_AD_FAULT ; + } + +RCU_READ_LOCK_GUARD (); + + mr =address_space_translate (s->target_as, pte_addr, &offset, + &len, true,MEMTXATTRS_UNSPECIFIED ); + +if (len < pte_size || !memory_region_is_ram (mr)) { +/* +* The entire PTE must be backed by a single RAM region +* for an atomic update. +*/ +return RISCV_IOMMU_AD_FAULT ; + } + + pte_ptr =qemu_map_ram_ptr (mr->ram_block, offset); + +if (pte_size ==4 ) { +uint32_t expected =cpu_to_le32 ((uint32_t )pte); +uint32_t desired =cpu_to_le32 ((uint32_t )updated_pte); + + old_pte =qatomic_cmpxchg ((uint32_t *)pte_ptr, expected, desired); + old_pte =le32_to_cpu (old_pte); + }else { +uint64_t expected =cpu_to_le64 (pte); +uint64_t desired =cpu_to_le64 (updated_pte); + + old_pte =qatomic_cmpxchg ((uint64_t *)pte_ptr, expected, desired); + old_pte =le64_to_cpu (old_pte); + } + +if (old_pte != pte) { +return RISCV_IOMMU_AD_RETRY ; + } + +return RISCV_IOMMU_AD_SUCCESS ; +} + +static int riscv_iommu_spa_fetch (RISCVIOMMUState *s,RISCVIOMMUContext *ctx, +IOMMUTLBEntry *iotlb,bool implicit_write,uint64_t *fault_iotval2); + +/* +* Updating a first-stage leaf PTE requires an implicit write. +* Translate its GPA through the G-stage, checking write permission +* and updating the G-stage's A/D bits as needed. +*/ +static int riscv_iommu_check_pte_write (RISCVIOMMUState *s, +RISCVIOMMUContext *ctx, +dma_addr_t pte_gpa, +IOMMUAccessFlags req_perm, +hwaddr *pte_spa, +uint64_t *fault_iotval2) +{ +RISCVIOMMUContext gctx = *ctx; +uint64_t iotval2 =0 ; +int fault; + +IOMMUTLBEntry iotlb = { + .iova = pte_gpa, + .target_as = s->target_as, + .addr_mask = ~0ULL , + .perm =IOMMU_WO , + }; + + gctx.satp =RISCV_IOMMU_DC_FSC_MODE_BARE ; + + fault =riscv_iommu_spa_fetch (s, &gctx, &iotlb, true, &iotval2); + +if (fault) { + *fault_iotval2 = iotval2; + +if (fault ==RISCV_IOMMU_FQ_CAUSE_RD_FAULT_VS || + fault ==RISCV_IOMMU_FQ_CAUSE_WR_FAULT_VS ) { +/* +* Per the RISC-V IOMMU Fault/Event Queue format, iotval2[0] +* indicates an implicit first-stage access and iotval2[1] +* indicates an implicit write. Report the fault for the +* original DMA access rather than the implicit write. +*/ + *fault_iotval2 = (pte_gpa & ~3ULL ) |3 ; + + fault = (req_perm &IOMMU_WO ) ? +RISCV_IOMMU_FQ_CAUSE_WR_FAULT_VS : +RISCV_IOMMU_FQ_CAUSE_RD_FAULT_VS ; + }else if (fault ==RISCV_IOMMU_FQ_CAUSE_RD_FAULT || + fault ==RISCV_IOMMU_FQ_CAUSE_WR_FAULT ) { +/* +* A physical access fault also uses the original +* DMA access type. +*/ + fault = (req_perm &IOMMU_WO ) ? +RISCV_IOMMU_FQ_CAUSE_WR_FAULT : +RISCV_IOMMU_FQ_CAUSE_RD_FAULT ; + } +return fault; + } + + *pte_spa = iotlb.translated_addr; +return 0 ; +} +/* * RISCV IOMMU Address Translation Lookup - Page Table Walk *
@@ -277,14 +399,15 @@ static hwaddr riscv_iommu_napot_page_mask(hwaddr ppn, hwaddr addr, hwaddr *out)* @ctx : Translation context for device id and process * address space id. * @iotlb : translation data: physical address and access mode. +* @implicit_write : Whether this is an implicit write to a first-stage PTE. * @fault_iotval2 : Fault address and implicit-access flags for iotval2. * @return : success or fault cause code. */ static int riscv_iommu_spa_fetch (RISCVIOMMUState *s,RISCVIOMMUContext *ctx, -IOMMUTLBEntry *iotlb,uint64_t *fault_iotval2) +IOMMUTLBEntry *iotlb,bool implicit_write,uint64_t *fault_iotval2) {IOMMUAccessFlags trans_perm =IOMMU_NONE ; -dma_addr_t addr, base; +dma_addr_t addr, base, s_base_gpa =0 ;uint64_t satp, gatp, pte;bool en_s, en_g;struct {
@@ -308,6 +431,8 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,* effective leaf permission for the translation cache. */ const IOMMUAccessFlags req_perm = iotlb->perm; +const hwaddr addr_mask = iotlb->addr_mask; +const hwaddr translated_addr = iotlb->translated_addr; satp =get_field (ctx->satp,RISCV_IOMMU_ATP_MODE_FIELD ); gatp =get_field (ctx->gatp,RISCV_IOMMU_ATP_MODE_FIELD );
@@ -326,7 +451,7 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,* means we can't do an early MSI check unless we have * strictly !en_s. */ -if (!en_s && (req_perm &IOMMU_WO ) && +if (!implicit_write && !en_s && (req_perm &IOMMU_WO ) &&riscv_iommu_msi_check (s, ctx, iotlb->iova)) { iotlb->target_as = &s->trap_as; iotlb->translated_addr = iotlb->iova;
@@ -413,6 +538,17 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,/* S/G stages translation tables root pointers */ gatp =PPN_PHYS (get_field (ctx->gatp,RISCV_IOMMU_ATP_PPN_FIELD )); satp =PPN_PHYS (get_field (ctx->satp,RISCV_IOMMU_ATP_PPN_FIELD )); + +restart: + sc[S_STAGE ].step =0 ; + sc[G_STAGE ].step =0 ; + trans_perm =IOMMU_NONE ; + iotlb->addr_mask = addr_mask; + iotlb->translated_addr = translated_addr; + iotlb->perm = req_perm; + s_base_gpa =0 ; + *fault_iotval2 =0 ; + addr = (en_s && en_g) ? satp : iotlb->iova; base = en_g ? gatp : satp; pass = en_g ?G_STAGE :S_STAGE ;
@@ -540,8 +676,51 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx, iotlb->addr_mask &= (1ULL << va_skip) -1 ; } +/* Automatically update the leaf PTE's A/D bits when enabled. */ +uint64_t updated_pte = pte; +if (ade) { + updated_pte |=PTE_A ; + +if (acc_perm &IOMMU_WO ) { + updated_pte |=PTE_D ; + } + } +if (updated_pte != pte) { +if (pass ==S_STAGE && en_g) { +const dma_addr_t s_pte_gpa = + s_base_gpa + idx * sc[pass].ptesize; +hwaddr write_spa; + +int fault =riscv_iommu_check_pte_write ( + s, ctx, s_pte_gpa, req_perm, &write_spa, + fault_iotval2); +if (fault) { +return fault; + } + +if (write_spa != pte_addr) { + goto restart; + } + } + +RISCVIOMMUADResult result =riscv_iommu_ad_update ( + s, pte_addr, sc[pass].ptesize, pte, updated_pte); + +if (result ==RISCV_IOMMU_AD_RETRY ) { + goto restart; + } +if (result ==RISCV_IOMMU_AD_FAULT ) { +return (req_perm &IOMMU_WO ) ? +RISCV_IOMMU_FQ_CAUSE_WR_FAULT : +RISCV_IOMMU_FQ_CAUSE_RD_FAULT ; + } + pte = updated_pte; + } +/* Continue with S-Stage translation? */ if (pass && sc[0 ].step != sc[0 ].levels) { +/* Save the S-stage page-table GPA for A/D updates. */ + s_base_gpa = addr; pass =S_STAGE ; addr = iotlb->iova;continue ;
@@ -552,6 +731,11 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx, ((pte & PTE_R) ? IOMMU_RW : IOMMU_WO) :IOMMU_RO ; +/* Do not cache write permission until the D bit is set. */ +if (!(pte &PTE_D )) { + leaf_perm &= ~IOMMU_WO ; + } + trans_perm = trans_perm ==IOMMU_NONE ? leaf_perm : trans_perm & leaf_perm;
@@ -1794,7 +1978,7 @@ static int riscv_iommu_translate(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,riscv_iommu_hpm_incr_ctr (s, ctx,RISCV_IOMMU_HPMEVENT_TLB_MISS );/* Translate using device directory / page table information. */ - fault =riscv_iommu_spa_fetch (s, ctx, iotlb, &fault_iotval2); + fault =riscv_iommu_spa_fetch (s, ctx, iotlb, false, &fault_iotval2);if (!fault && iotlb->target_as == &s->trap_as) {/* Do not cache trapped MSI translations */
@@ -2732,6 +2916,9 @@ static void riscv_iommu_realize(DeviceState *dev, Error **errp)if (s->enable_ats) { s->cap |=RISCV_IOMMU_CAP_ATS ; } +if (s->enable_amo_hwad) { + s->cap |=RISCV_IOMMU_CAP_AMO_HWAD ; + }if (s->enable_s_stage) { s->cap |=RISCV_IOMMU_CAP_SV32 |RISCV_IOMMU_CAP_SV39 |RISCV_IOMMU_CAP_SV48 |RISCV_IOMMU_CAP_SV57 ;
@@ -2891,6 +3078,7 @@ static const Property riscv_iommu_properties[] = { LIMIT_CACHE_IOT),DEFINE_PROP_BOOL ("intremap" , RISCVIOMMUState, enable_msi, TRUE),DEFINE_PROP_BOOL ("ats" , RISCVIOMMUState, enable_ats, TRUE), +DEFINE_PROP_BOOL ("amo-hwad" , RISCVIOMMUState, enable_amo_hwad, false),DEFINE_PROP_BOOL ("off" , RISCVIOMMUState, enable_off, TRUE),DEFINE_PROP_BOOL ("s-stage" , RISCVIOMMUState, enable_s_stage, TRUE), DEFINE_PROP_BOOL("g-stage", RISCVIOMMUState, enable_g_stage, TRUE),
hw/riscv/riscv-iommu.h +1 -0
@@ -44,6 +44,7 @@ struct RISCVIOMMUState {bool enable_off;/* Enable out-of-reset OFF mode (DMA disabled) */ bool enable_msi;/* Enable MSI remapping */ bool enable_ats;/* Enable ATS support */ +bool enable_amo_hwad;/* Enable hardware A/D updates */ bool enable_s_stage;/* Enable S/VS-Stage translation */ bool enable_g_stage;/* Enable G-Stage translation */
Signature · 1 line
2.53.0
Quoted 10 lines · Tan Chi
Add support for the
AMO_HWADcapability to automatically update the accessed and dirty bits of leaf PTEs when SADE or GADE is enabled.Use atomic compare-and-exchange to update PTEs, restarting the page-table walk if a concurrent modification is detected.
For nested translation, check G-stage write permissions before updating an S-stage PTE. Restart the walk if the write translation resolves to a different physical address.
Quoted 8 lines
Avoid caching write permission when the PTE's dirty bit is clear, so subsequent writes can trigger the required update.
Expose the capability through the new "amo-hwad" device property, disabled by default.
Signed-off-by Tan Chi <[email protected]>---
Reviewed-by Daniel Henrique Barboza <[email protected]>
Quoted 294 lines
hw/riscv/riscv-iommu.c | 196 ++++++++++++++++++++++++++++++++++++++++- hw/riscv/riscv-iommu.h | 1 + 2 files changed, 193 insertions(+), 4 deletions(-)hw/riscv/riscv-iommu.c +192
-4@@ -265,6 +265,128 @@ static hwaddr riscv_iommu_napot_page_mask(hwaddr ppn, hwaddr addr, hwaddr *out)return page_mask; } +typedef enum RISCVIOMMUADResult { +RISCV_IOMMU_AD_SUCCESS , +RISCV_IOMMU_AD_RETRY , +RISCV_IOMMU_AD_FAULT , +}RISCVIOMMUADResult ; + +/* +* Atomically update the A/D bits of a leaf PTE. +* Retry the page-table walk if the PTE has changed. +*/ +static RISCVIOMMUADResult +riscv_iommu_ad_update (RISCVIOMMUState *s,hwaddr pte_addr, +unsigned int pte_size,uint64_t pte,uint64_t updated_pte) +{ +MemoryRegion *mr; +hwaddr offset, len = pte_size; +uint64_t old_pte; +void *pte_ptr; + +if (pte_addr & (pte_size -1 )) { +return RISCV_IOMMU_AD_FAULT ; + } + +RCU_READ_LOCK_GUARD (); + + mr =address_space_translate (s->target_as, pte_addr, &offset, + &len, true,MEMTXATTRS_UNSPECIFIED ); + +if (len < pte_size || !memory_region_is_ram (mr)) { +/* +* The entire PTE must be backed by a single RAM region +* for an atomic update. +*/ +return RISCV_IOMMU_AD_FAULT ; + } + + pte_ptr =qemu_map_ram_ptr (mr->ram_block, offset); + +if (pte_size ==4 ) { +uint32_t expected =cpu_to_le32 ((uint32_t )pte); +uint32_t desired =cpu_to_le32 ((uint32_t )updated_pte); + + old_pte =qatomic_cmpxchg ((uint32_t *)pte_ptr, expected, desired); + old_pte =le32_to_cpu (old_pte); + }else { +uint64_t expected =cpu_to_le64 (pte); +uint64_t desired =cpu_to_le64 (updated_pte); + + old_pte =qatomic_cmpxchg ((uint64_t *)pte_ptr, expected, desired); + old_pte =le64_to_cpu (old_pte); + } + +if (old_pte != pte) { +return RISCV_IOMMU_AD_RETRY ; + } + +return RISCV_IOMMU_AD_SUCCESS ; +} + +static int riscv_iommu_spa_fetch (RISCVIOMMUState *s,RISCVIOMMUContext *ctx, +IOMMUTLBEntry *iotlb,bool implicit_write,uint64_t *fault_iotval2); + +/* +* Updating a first-stage leaf PTE requires an implicit write. +* Translate its GPA through the G-stage, checking write permission +* and updating the G-stage's A/D bits as needed. +*/ +static int riscv_iommu_check_pte_write (RISCVIOMMUState *s, +RISCVIOMMUContext *ctx, +dma_addr_t pte_gpa, +IOMMUAccessFlags req_perm, +hwaddr *pte_spa, +uint64_t *fault_iotval2) +{ +RISCVIOMMUContext gctx = *ctx; +uint64_t iotval2 =0 ; +int fault; + +IOMMUTLBEntry iotlb = { + .iova = pte_gpa, + .target_as = s->target_as, + .addr_mask = ~0ULL , + .perm =IOMMU_WO , + }; + + gctx.satp =RISCV_IOMMU_DC_FSC_MODE_BARE ; + + fault =riscv_iommu_spa_fetch (s, &gctx, &iotlb, true, &iotval2); + +if (fault) { + *fault_iotval2 = iotval2; + +if (fault ==RISCV_IOMMU_FQ_CAUSE_RD_FAULT_VS || + fault ==RISCV_IOMMU_FQ_CAUSE_WR_FAULT_VS ) { +/* +* Per the RISC-V IOMMU Fault/Event Queue format, iotval2[0] +* indicates an implicit first-stage access and iotval2[1] +* indicates an implicit write. Report the fault for the +* original DMA access rather than the implicit write. +*/ + *fault_iotval2 = (pte_gpa & ~3ULL ) |3 ; + + fault = (req_perm &IOMMU_WO ) ? +RISCV_IOMMU_FQ_CAUSE_WR_FAULT_VS : +RISCV_IOMMU_FQ_CAUSE_RD_FAULT_VS ; + }else if (fault ==RISCV_IOMMU_FQ_CAUSE_RD_FAULT || + fault ==RISCV_IOMMU_FQ_CAUSE_WR_FAULT ) { +/* +* A physical access fault also uses the original +* DMA access type. +*/ + fault = (req_perm &IOMMU_WO ) ? +RISCV_IOMMU_FQ_CAUSE_WR_FAULT : +RISCV_IOMMU_FQ_CAUSE_RD_FAULT ; + } +return fault; + } + + *pte_spa = iotlb.translated_addr; +return 0 ; +} +/* * RISCV IOMMU Address Translation Lookup - Page Table Walk * @@ -277,14 +399,15 @@ static hwaddr riscv_iommu_napot_page_mask(hwaddr ppn, hwaddr addr, hwaddr *out)* @ctx : Translation context for device id and process * address space id. * @iotlb : translation data: physical address and access mode. +* @implicit_write : Whether this is an implicit write to a first-stage PTE. * @fault_iotval2 : Fault address and implicit-access flags for iotval2. * @return : success or fault cause code. */ static int riscv_iommu_spa_fetch (RISCVIOMMUState *s,RISCVIOMMUContext *ctx, -IOMMUTLBEntry *iotlb,uint64_t *fault_iotval2) +IOMMUTLBEntry *iotlb,bool implicit_write,uint64_t *fault_iotval2) {IOMMUAccessFlags trans_perm =IOMMU_NONE ; -dma_addr_t addr, base; +dma_addr_t addr, base, s_base_gpa =0 ;uint64_t satp, gatp, pte;bool en_s, en_g;struct {@@ -308,6 +431,8 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,* effective leaf permission for the translation cache. */ const IOMMUAccessFlags req_perm = iotlb->perm; +const hwaddr addr_mask = iotlb->addr_mask; +const hwaddr translated_addr = iotlb->translated_addr; satp =get_field (ctx->satp,RISCV_IOMMU_ATP_MODE_FIELD ); gatp =get_field (ctx->gatp,RISCV_IOMMU_ATP_MODE_FIELD );@@ -326,7 +451,7 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,* means we can't do an early MSI check unless we have * strictly !en_s. */ -if (!en_s && (req_perm &IOMMU_WO ) && +if (!implicit_write && !en_s && (req_perm &IOMMU_WO ) &&riscv_iommu_msi_check (s, ctx, iotlb->iova)) { iotlb->target_as = &s->trap_as; iotlb->translated_addr = iotlb->iova;@@ -413,6 +538,17 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,/* S/G stages translation tables root pointers */ gatp =PPN_PHYS (get_field (ctx->gatp,RISCV_IOMMU_ATP_PPN_FIELD )); satp =PPN_PHYS (get_field (ctx->satp,RISCV_IOMMU_ATP_PPN_FIELD )); + +restart: + sc[S_STAGE ].step =0 ; + sc[G_STAGE ].step =0 ; + trans_perm =IOMMU_NONE ; + iotlb->addr_mask = addr_mask; + iotlb->translated_addr = translated_addr; + iotlb->perm = req_perm; + s_base_gpa =0 ; + *fault_iotval2 =0 ; + addr = (en_s && en_g) ? satp : iotlb->iova; base = en_g ? gatp : satp; pass = en_g ?G_STAGE :S_STAGE ;@@ -540,8 +676,51 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx, iotlb->addr_mask &= (1ULL << va_skip) -1 ; } +/* Automatically update the leaf PTE's A/D bits when enabled. */ +uint64_t updated_pte = pte; +if (ade) { + updated_pte |=PTE_A ; + +if (acc_perm &IOMMU_WO ) { + updated_pte |=PTE_D ; + } + } +if (updated_pte != pte) { +if (pass ==S_STAGE && en_g) { +const dma_addr_t s_pte_gpa = + s_base_gpa + idx * sc[pass].ptesize; +hwaddr write_spa; + +int fault =riscv_iommu_check_pte_write ( + s, ctx, s_pte_gpa, req_perm, &write_spa, + fault_iotval2); +if (fault) { +return fault; + } + +if (write_spa != pte_addr) { + goto restart; + } + } + +RISCVIOMMUADResult result =riscv_iommu_ad_update ( + s, pte_addr, sc[pass].ptesize, pte, updated_pte); + +if (result ==RISCV_IOMMU_AD_RETRY ) { + goto restart; + } +if (result ==RISCV_IOMMU_AD_FAULT ) { +return (req_perm &IOMMU_WO ) ? +RISCV_IOMMU_FQ_CAUSE_WR_FAULT : +RISCV_IOMMU_FQ_CAUSE_RD_FAULT ; + } + pte = updated_pte; + } +/* Continue with S-Stage translation? */ if (pass && sc[0 ].step != sc[0 ].levels) { +/* Save the S-stage page-table GPA for A/D updates. */ + s_base_gpa = addr; pass =S_STAGE ; addr = iotlb->iova;continue ;@@ -552,6 +731,11 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx, ((pte & PTE_R) ? IOMMU_RW : IOMMU_WO) :IOMMU_RO ; +/* Do not cache write permission until the D bit is set. */ +if (!(pte &PTE_D )) { + leaf_perm &= ~IOMMU_WO ; + } + trans_perm = trans_perm ==IOMMU_NONE ? leaf_perm : trans_perm & leaf_perm;@@ -1794,7 +1978,7 @@ static int riscv_iommu_translate(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,riscv_iommu_hpm_incr_ctr (s, ctx,RISCV_IOMMU_HPMEVENT_TLB_MISS );/* Translate using device directory / page table information. */ - fault =riscv_iommu_spa_fetch (s, ctx, iotlb, &fault_iotval2); + fault =riscv_iommu_spa_fetch (s, ctx, iotlb, false, &fault_iotval2);if (!fault && iotlb->target_as == &s->trap_as) {/* Do not cache trapped MSI translations */ @@ -2732,6 +2916,9 @@ static void riscv_iommu_realize(DeviceState *dev, Error **errp)if (s->enable_ats) { s->cap |=RISCV_IOMMU_CAP_ATS ; } +if (s->enable_amo_hwad) { + s->cap |=RISCV_IOMMU_CAP_AMO_HWAD ; + }if (s->enable_s_stage) { s->cap |=RISCV_IOMMU_CAP_SV32 |RISCV_IOMMU_CAP_SV39 |RISCV_IOMMU_CAP_SV48 |RISCV_IOMMU_CAP_SV57 ;@@ -2891,6 +3078,7 @@ static const Property riscv_iommu_properties[] = { LIMIT_CACHE_IOT),DEFINE_PROP_BOOL ("intremap" , RISCVIOMMUState, enable_msi, TRUE),DEFINE_PROP_BOOL ("ats" , RISCVIOMMUState, enable_ats, TRUE), +DEFINE_PROP_BOOL ("amo-hwad" , RISCVIOMMUState, enable_amo_hwad, false),DEFINE_PROP_BOOL ("off" , RISCVIOMMUState, enable_off, TRUE),DEFINE_PROP_BOOL ("s-stage" , RISCVIOMMUState, enable_s_stage, TRUE), DEFINE_PROP_BOOL("g-stage", RISCVIOMMUState, enable_g_stage, TRUE),hw/riscv/riscv-iommu.h +1
-0@@ -44,6 +44,7 @@ struct RISCVIOMMUState {bool enable_off;/* Enable out-of-reset OFF mode (DMA disabled) */ bool enable_msi;/* Enable MSI remapping */ bool enable_ats;/* Enable ATS support */ +bool enable_amo_hwad;/* Enable hardware A/D updates */ bool enable_s_stage;/* Enable S/VS-Stage translation */ bool enable_g_stage;/* Enable G-Stage translation */
Quoted 10 lines · Tan Chi
Add support for the
AMO_HWADcapability to automatically update the accessed and dirty bits of leaf PTEs when SADE or GADE is enabled.Use atomic compare-and-exchange to update PTEs, restarting the page-table walk if a concurrent modification is detected.
For nested translation, check G-stage write permissions before updating an S-stage PTE. Restart the walk if the write translation resolves to a different physical address.
Quoted 7 lines
Avoid caching write permission when the PTE's dirty bit is clear, so subsequent writes can trigger the required update.
Expose the capability through the new "amo-hwad" device property, disabled by default.
Signed-off-by Tan Chi <[email protected]>
Reviewed-by Nutty Liu <[email protected]>
Thanks,
Nutty
Quoted 295 lines
--- hw/riscv/riscv-iommu.c | 196 ++++++++++++++++++++++++++++++++++++++++- hw/riscv/riscv-iommu.h | 1 + 2 files changed, 193 insertions(+), 4 deletions(-)hw/riscv/riscv-iommu.c +192
-4@@ -265,6 +265,128 @@ static hwaddr riscv_iommu_napot_page_mask(hwaddr ppn, hwaddr addr, hwaddr *out)return page_mask; } +typedef enum RISCVIOMMUADResult { +RISCV_IOMMU_AD_SUCCESS , +RISCV_IOMMU_AD_RETRY , +RISCV_IOMMU_AD_FAULT , +}RISCVIOMMUADResult ; + +/* +* Atomically update the A/D bits of a leaf PTE. +* Retry the page-table walk if the PTE has changed. +*/ +static RISCVIOMMUADResult +riscv_iommu_ad_update (RISCVIOMMUState *s,hwaddr pte_addr, +unsigned int pte_size,uint64_t pte,uint64_t updated_pte) +{ +MemoryRegion *mr; +hwaddr offset, len = pte_size; +uint64_t old_pte; +void *pte_ptr; + +if (pte_addr & (pte_size -1 )) { +return RISCV_IOMMU_AD_FAULT ; + } + +RCU_READ_LOCK_GUARD (); + + mr =address_space_translate (s->target_as, pte_addr, &offset, + &len, true,MEMTXATTRS_UNSPECIFIED ); + +if (len < pte_size || !memory_region_is_ram (mr)) { +/* +* The entire PTE must be backed by a single RAM region +* for an atomic update. +*/ +return RISCV_IOMMU_AD_FAULT ; + } + + pte_ptr =qemu_map_ram_ptr (mr->ram_block, offset); + +if (pte_size ==4 ) { +uint32_t expected =cpu_to_le32 ((uint32_t )pte); +uint32_t desired =cpu_to_le32 ((uint32_t )updated_pte); + + old_pte =qatomic_cmpxchg ((uint32_t *)pte_ptr, expected, desired); + old_pte =le32_to_cpu (old_pte); + }else { +uint64_t expected =cpu_to_le64 (pte); +uint64_t desired =cpu_to_le64 (updated_pte); + + old_pte =qatomic_cmpxchg ((uint64_t *)pte_ptr, expected, desired); + old_pte =le64_to_cpu (old_pte); + } + +if (old_pte != pte) { +return RISCV_IOMMU_AD_RETRY ; + } + +return RISCV_IOMMU_AD_SUCCESS ; +} + +static int riscv_iommu_spa_fetch (RISCVIOMMUState *s,RISCVIOMMUContext *ctx, +IOMMUTLBEntry *iotlb,bool implicit_write,uint64_t *fault_iotval2); + +/* +* Updating a first-stage leaf PTE requires an implicit write. +* Translate its GPA through the G-stage, checking write permission +* and updating the G-stage's A/D bits as needed. +*/ +static int riscv_iommu_check_pte_write (RISCVIOMMUState *s, +RISCVIOMMUContext *ctx, +dma_addr_t pte_gpa, +IOMMUAccessFlags req_perm, +hwaddr *pte_spa, +uint64_t *fault_iotval2) +{ +RISCVIOMMUContext gctx = *ctx; +uint64_t iotval2 =0 ; +int fault; + +IOMMUTLBEntry iotlb = { + .iova = pte_gpa, + .target_as = s->target_as, + .addr_mask = ~0ULL , + .perm =IOMMU_WO , + }; + + gctx.satp =RISCV_IOMMU_DC_FSC_MODE_BARE ; + + fault =riscv_iommu_spa_fetch (s, &gctx, &iotlb, true, &iotval2); + +if (fault) { + *fault_iotval2 = iotval2; + +if (fault ==RISCV_IOMMU_FQ_CAUSE_RD_FAULT_VS || + fault ==RISCV_IOMMU_FQ_CAUSE_WR_FAULT_VS ) { +/* +* Per the RISC-V IOMMU Fault/Event Queue format, iotval2[0] +* indicates an implicit first-stage access and iotval2[1] +* indicates an implicit write. Report the fault for the +* original DMA access rather than the implicit write. +*/ + *fault_iotval2 = (pte_gpa & ~3ULL ) |3 ; + + fault = (req_perm &IOMMU_WO ) ? +RISCV_IOMMU_FQ_CAUSE_WR_FAULT_VS : +RISCV_IOMMU_FQ_CAUSE_RD_FAULT_VS ; + }else if (fault ==RISCV_IOMMU_FQ_CAUSE_RD_FAULT || + fault ==RISCV_IOMMU_FQ_CAUSE_WR_FAULT ) { +/* +* A physical access fault also uses the original +* DMA access type. +*/ + fault = (req_perm &IOMMU_WO ) ? +RISCV_IOMMU_FQ_CAUSE_WR_FAULT : +RISCV_IOMMU_FQ_CAUSE_RD_FAULT ; + } +return fault; + } + + *pte_spa = iotlb.translated_addr; +return 0 ; +} +/* * RISCV IOMMU Address Translation Lookup - Page Table Walk * @@ -277,14 +399,15 @@ static hwaddr riscv_iommu_napot_page_mask(hwaddr ppn, hwaddr addr, hwaddr *out)* @ctx : Translation context for device id and process * address space id. * @iotlb : translation data: physical address and access mode. +* @implicit_write : Whether this is an implicit write to a first-stage PTE. * @fault_iotval2 : Fault address and implicit-access flags for iotval2. * @return : success or fault cause code. */ static int riscv_iommu_spa_fetch (RISCVIOMMUState *s,RISCVIOMMUContext *ctx, -IOMMUTLBEntry *iotlb,uint64_t *fault_iotval2) +IOMMUTLBEntry *iotlb,bool implicit_write,uint64_t *fault_iotval2) {IOMMUAccessFlags trans_perm =IOMMU_NONE ; -dma_addr_t addr, base; +dma_addr_t addr, base, s_base_gpa =0 ;uint64_t satp, gatp, pte;bool en_s, en_g;struct {@@ -308,6 +431,8 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,* effective leaf permission for the translation cache. */ const IOMMUAccessFlags req_perm = iotlb->perm; +const hwaddr addr_mask = iotlb->addr_mask; +const hwaddr translated_addr = iotlb->translated_addr; satp =get_field (ctx->satp,RISCV_IOMMU_ATP_MODE_FIELD ); gatp =get_field (ctx->gatp,RISCV_IOMMU_ATP_MODE_FIELD );@@ -326,7 +451,7 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,* means we can't do an early MSI check unless we have * strictly !en_s. */ -if (!en_s && (req_perm &IOMMU_WO ) && +if (!implicit_write && !en_s && (req_perm &IOMMU_WO ) &&riscv_iommu_msi_check (s, ctx, iotlb->iova)) { iotlb->target_as = &s->trap_as; iotlb->translated_addr = iotlb->iova;@@ -413,6 +538,17 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,/* S/G stages translation tables root pointers */ gatp =PPN_PHYS (get_field (ctx->gatp,RISCV_IOMMU_ATP_PPN_FIELD )); satp =PPN_PHYS (get_field (ctx->satp,RISCV_IOMMU_ATP_PPN_FIELD )); + +restart: + sc[S_STAGE ].step =0 ; + sc[G_STAGE ].step =0 ; + trans_perm =IOMMU_NONE ; + iotlb->addr_mask = addr_mask; + iotlb->translated_addr = translated_addr; + iotlb->perm = req_perm; + s_base_gpa =0 ; + *fault_iotval2 =0 ; + addr = (en_s && en_g) ? satp : iotlb->iova; base = en_g ? gatp : satp; pass = en_g ?G_STAGE :S_STAGE ;@@ -540,8 +676,51 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx, iotlb->addr_mask &= (1ULL << va_skip) -1 ; } +/* Automatically update the leaf PTE's A/D bits when enabled. */ +uint64_t updated_pte = pte; +if (ade) { + updated_pte |=PTE_A ; + +if (acc_perm &IOMMU_WO ) { + updated_pte |=PTE_D ; + } + } +if (updated_pte != pte) { +if (pass ==S_STAGE && en_g) { +const dma_addr_t s_pte_gpa = + s_base_gpa + idx * sc[pass].ptesize; +hwaddr write_spa; + +int fault =riscv_iommu_check_pte_write ( + s, ctx, s_pte_gpa, req_perm, &write_spa, + fault_iotval2); +if (fault) { +return fault; + } + +if (write_spa != pte_addr) { + goto restart; + } + } + +RISCVIOMMUADResult result =riscv_iommu_ad_update ( + s, pte_addr, sc[pass].ptesize, pte, updated_pte); + +if (result ==RISCV_IOMMU_AD_RETRY ) { + goto restart; + } +if (result ==RISCV_IOMMU_AD_FAULT ) { +return (req_perm &IOMMU_WO ) ? +RISCV_IOMMU_FQ_CAUSE_WR_FAULT : +RISCV_IOMMU_FQ_CAUSE_RD_FAULT ; + } + pte = updated_pte; + } +/* Continue with S-Stage translation? */ if (pass && sc[0 ].step != sc[0 ].levels) { +/* Save the S-stage page-table GPA for A/D updates. */ + s_base_gpa = addr; pass =S_STAGE ; addr = iotlb->iova;continue ;@@ -552,6 +731,11 @@ static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx, ((pte & PTE_R) ? IOMMU_RW : IOMMU_WO) :IOMMU_RO ; +/* Do not cache write permission until the D bit is set. */ +if (!(pte &PTE_D )) { + leaf_perm &= ~IOMMU_WO ; + } + trans_perm = trans_perm ==IOMMU_NONE ? leaf_perm : trans_perm & leaf_perm;@@ -1794,7 +1978,7 @@ static int riscv_iommu_translate(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,riscv_iommu_hpm_incr_ctr (s, ctx,RISCV_IOMMU_HPMEVENT_TLB_MISS );/* Translate using device directory / page table information. */ - fault =riscv_iommu_spa_fetch (s, ctx, iotlb, &fault_iotval2); + fault =riscv_iommu_spa_fetch (s, ctx, iotlb, false, &fault_iotval2);if (!fault && iotlb->target_as == &s->trap_as) {/* Do not cache trapped MSI translations */ @@ -2732,6 +2916,9 @@ static void riscv_iommu_realize(DeviceState *dev, Error **errp)if (s->enable_ats) { s->cap |=RISCV_IOMMU_CAP_ATS ; } +if (s->enable_amo_hwad) { + s->cap |=RISCV_IOMMU_CAP_AMO_HWAD ; + }if (s->enable_s_stage) { s->cap |=RISCV_IOMMU_CAP_SV32 |RISCV_IOMMU_CAP_SV39 |RISCV_IOMMU_CAP_SV48 |RISCV_IOMMU_CAP_SV57 ;@@ -2891,6 +3078,7 @@ static const Property riscv_iommu_properties[] = { LIMIT_CACHE_IOT),DEFINE_PROP_BOOL ("intremap" , RISCVIOMMUState, enable_msi, TRUE),DEFINE_PROP_BOOL ("ats" , RISCVIOMMUState, enable_ats, TRUE), +DEFINE_PROP_BOOL ("amo-hwad" , RISCVIOMMUState, enable_amo_hwad, false),DEFINE_PROP_BOOL ("off" , RISCVIOMMUState, enable_off, TRUE),DEFINE_PROP_BOOL ("s-stage" , RISCVIOMMUState, enable_s_stage, TRUE), DEFINE_PROP_BOOL("g-stage", RISCVIOMMUState, enable_g_stage, TRUE),hw/riscv/riscv-iommu.h +1
-0@@ -44,6 +44,7 @@ struct RISCVIOMMUState {bool enable_off;/* Enable out-of-reset OFF mode (DMA disabled) */ bool enable_msi;/* Enable MSI remapping */ bool enable_ats;/* Enable ATS support */ +bool enable_amo_hwad;/* Enable hardware A/D updates */ bool enable_s_stage;/* Enable S/VS-Stage translation */ bool enable_g_stage;/* Enable G-Stage translation */