iommu/arm-smmu-v3: Add PRI support

index 4d00d796f0783..2176ee8bec767 100644
@@ -2843,6 +2843,12 @@ static int arm_smmu_enable_iopf(struct arm_smmu_master *master,
 	if (master->iopf_refcount) {
 		master->iopf_refcount++;
 		master_domain->using_iopf = true;
+		/*
+		 * If the device is already on the IOPF queue (domain replace),
+		 * drain in-flight fault handlers so nothing will hold the old
+		 * domain when the core switches the attach handle.
+		 */
+		iopf_queue_flush_dev(master->dev);
 		return 0;
 	}
 

@@ -2866,8 +2872,11 @@ static void arm_smmu_disable_iopf(struct arm_smmu_master *master,
 		return;
 
 	master->iopf_refcount--;
-	if (master->iopf_refcount == 0)
+	if (master->iopf_refcount == 0) {
+		/* Drain in-flight fault handlers before removing device */
+		iopf_queue_flush_dev(master->dev);
 		iopf_queue_remove_device(master->smmu->evtq.iopf, master->dev);
+	}
 }
 
 static void arm_smmu_remove_master_domain(struct arm_smmu_master *master,

index 2176ee8bec767..055dde5367131 100644
@@ -992,29 +992,56 @@ static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu,
 					   cmds->num, true);
 }
 
-static void arm_smmu_page_response(struct device *dev, struct iopf_fault *unused,
+static void arm_smmu_page_response(struct device *dev, struct iopf_fault *evt,
 				   struct iommu_page_response *resp)
 {
 	struct arm_smmu_cmdq_ent cmd = {0};
 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
 	int sid = master->streams[0].id;
 
-	if (WARN_ON(!master->stall_enabled))
+	/* Handle PRI responses */
+	if (evt && evt->fault.type == IOMMU_FAULT_PAGE_REQ) {
+		cmd.opcode = CMDQ_OP_PRI_RESP;
+		cmd.pri.sid = sid;
+		cmd.pri.grpid = resp->grpid;
+		if (evt->fault.prm.flags &
+		    IOMMU_FAULT_PAGE_REQUEST_PASID_VALID) {
+			cmd.substream_valid = true;
+			cmd.pri.ssid = resp->pasid;
+		}
+		switch (resp->code) {
+		case IOMMU_PAGE_RESP_SUCCESS:
+			cmd.pri.resp = PRI_RESP_SUCC;
+			break;
+		case IOMMU_PAGE_RESP_FAILURE:
+			cmd.pri.resp = PRI_RESP_FAIL;
+			break;
+		case IOMMU_PAGE_RESP_INVALID:
+			cmd.pri.resp = PRI_RESP_DENY;
+			break;
+		default:
+			WARN_ON(true);
+			return;
+		}
+	} else if (master->stall_enabled) {
+		cmd.opcode = CMDQ_OP_RESUME;
+		cmd.resume.sid = sid;
+		cmd.resume.stag = resp->grpid;
+		switch (resp->code) {
+		case IOMMU_PAGE_RESP_INVALID:
+		case IOMMU_PAGE_RESP_FAILURE:
+			cmd.resume.resp = CMDQ_RESUME_0_RESP_ABORT;
+			break;
+		case IOMMU_PAGE_RESP_SUCCESS:
+			cmd.resume.resp = CMDQ_RESUME_0_RESP_RETRY;
+			break;
+		default:
+			WARN_ON(true);
+			return;
+		}
+	} else {
+		WARN_ON(true);
 		return;
-
-	cmd.opcode		= CMDQ_OP_RESUME;
-	cmd.resume.sid		= sid;
-	cmd.resume.stag		= resp->grpid;
-	switch (resp->code) {
-	case IOMMU_PAGE_RESP_INVALID:
-	case IOMMU_PAGE_RESP_FAILURE:
-		cmd.resume.resp = CMDQ_RESUME_0_RESP_ABORT;
-		break;
-	case IOMMU_PAGE_RESP_SUCCESS:
-		cmd.resume.resp = CMDQ_RESUME_0_RESP_RETRY;
-		break;
-	default:
-		break;
 	}
 
 	arm_smmu_cmdq_issue_cmd(master->smmu, &cmd);

index 055dde5367131..277d753c7675c 100644
@@ -2061,40 +2061,41 @@ static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
 
 static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
 {
-	u32 sid, ssid;
-	u16 grpid;
-	bool ssv, last;
-
-	sid = FIELD_GET(PRIQ_0_SID, evt[0]);
-	ssv = FIELD_GET(PRIQ_0_SSID_V, evt[0]);
-	ssid = ssv ? FIELD_GET(PRIQ_0_SSID, evt[0]) : IOMMU_NO_PASID;
-	last = FIELD_GET(PRIQ_0_PRG_LAST, evt[0]);
-	grpid = FIELD_GET(PRIQ_1_PRG_IDX, evt[1]);
-
-	dev_info(smmu->dev, "unexpected PRI request received:\n");
-	dev_info(smmu->dev,
-		 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
-		 sid, ssid, grpid, last ? "L" : "",
-		 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
-		 evt[0] & PRIQ_0_PERM_READ ? "R" : "",
-		 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
-		 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
-		 evt[1] & PRIQ_1_ADDR_MASK);
-
-	if (last) {
-		struct arm_smmu_cmdq_ent cmd = {
-			.opcode			= CMDQ_OP_PRI_RESP,
-			.substream_valid	= ssv,
-			.pri			= {
-				.sid	= sid,
-				.ssid	= ssid,
-				.grpid	= grpid,
-				.resp	= PRI_RESP_DENY,
-			},
-		};
+	u32 sid = FIELD_GET(PRIQ_0_SID, evt[0]);
+	struct iopf_fault iopf_fault = {0};
+	struct iommu_fault *fault = &iopf_fault.fault;
+	struct arm_smmu_master *master;
+
+	INIT_LIST_HEAD(&iopf_fault.list);
 
-		arm_smmu_cmdq_issue_cmd(smmu, &cmd);
+	fault->type = IOMMU_FAULT_PAGE_REQ;
+
+	if (FIELD_GET(PRIQ_0_PRG_LAST, evt[0]))
+		fault->prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE;
+	if (FIELD_GET(PRIQ_0_SSID_V, evt[0])) {
+		fault->prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
+		fault->prm.pasid = FIELD_GET(PRIQ_0_SSID, evt[0]);
 	}
+
+	fault->prm.grpid = FIELD_GET(PRIQ_1_PRG_IDX, evt[1]);
+
+	if (evt[0] & PRIQ_0_PERM_READ)
+		fault->prm.perm |= IOMMU_FAULT_PERM_READ;
+	if (evt[0] & PRIQ_0_PERM_WRITE)
+		fault->prm.perm |= IOMMU_FAULT_PERM_WRITE;
+	if (evt[0] & PRIQ_0_PERM_EXEC)
+		fault->prm.perm |= IOMMU_FAULT_PERM_EXEC;
+	if (evt[0] & PRIQ_0_PERM_PRIV)
+		fault->prm.perm |= IOMMU_FAULT_PERM_PRIV;
+
+	fault->prm.addr = FIELD_GET(PRIQ_1_ADDR_MASK, evt[1]) << 12;
+
+	guard(mutex)(&smmu->streams_mutex);
+	master = arm_smmu_find_master(smmu, sid);
+	if (master)
+		iommu_report_device_fault(master->dev, &iopf_fault);
+	else
+		dev_warn(smmu->dev, "PPR: unrecognized StreamID 0x%x\n", sid);
 }
 
 static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)