iommu: add qcom_iommu for early "B" family devices (v3)

Cc [email protected]

Signed-off-by Rob Clark <[email protected]>

---
 .../devicetree/bindings/iommu/qcom,iommu.txt       | 121 +++++++++++++++++++++
 1 file changed, 121 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/iommu/qcom,iommu.txt

An iommu driver for Qualcomm "B" family devices which do not completely implement the ARM SMMU spec. These devices have context-bank register layout that is similar to ARM SMMU, but no global register space (or at least not one that is accessible).

Signed-off-by Rob Clark <[email protected]>

Signed-off-by Stanimir Varbanov <[email protected]>

---
 drivers/iommu/Kconfig      |  10 +
 drivers/iommu/Makefile     |   1 +
 drivers/iommu/qcom_iommu.c | 825 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 836 insertions(+)
 create mode 100644 drivers/iommu/qcom_iommu.c

drivers/iommu/Kconfig +0 -0

index 37e204f..400a404 100644

drivers/iommu/Kconfig +7 -0

+       depends on ARCH_QCOM || COMPILE_TEST
+       select IOMMU_API
+       select IOMMU_IO_PGTABLE_LPAE
+       select ARM_DMA_USE_IOMMU
+       help
+         Support for IOMMU on certain Qualcomm SoCs.
+
 endif # IOMMU_SUPPORT

drivers/iommu/Makefile +1 -0

index 195f7b9..b910aea 100644
@@ -27,3 +27,4 @@ obj-$(CONFIG_TEGRA_IOMMU_SMMU) += tegra-smmu.o
 obj-$(CONFIG_EXYNOS_IOMMU) += exynos-iommu.o
 obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o
 obj-$(CONFIG_S390_IOMMU) += s390-iommu.o
+obj-$(CONFIG_QCOM_IOMMU) += qcom_iommu.o

drivers/iommu/qcom_iommu.c new file +5 -0

new file mode 100644
index 0000000..1cf7c8e
@@ -0,0 +1,825 @@
+/*
+ * IOMMU API for QCOM secure IOMMUs.  Somewhat based on arm-smmu.c
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as

drivers/iommu/qcom_iommu.c new file +91 -0

+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/of_iommu.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/qcom_scm.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+#include "io-pgtable.h"
+#include "arm-smmu-regs.h"
+
+#define SMMU_INTR_SEL_NS     0x2000
+
+struct qcom_iommu_dev {
+       /* IOMMU core code handle */
+       struct iommu_device      iommu;
+       struct device           *dev;
+       struct clk              *iface_clk;
+       struct clk              *bus_clk;
+       void __iomem            *local_base;
+       u32                      sec_id;
+       struct list_head         context_list;   /* list of qcom_iommu_context */
+};
+
+struct qcom_iommu_ctx {
+       struct device           *dev;
+       void __iomem            *base;
+       unsigned int             irq;
+       bool                     secure_init;
+       u32                      asid;      /* asid and ctx bank # are 1:1 */
+       struct iommu_group      *group;
+       struct list_head         node;      /* head in qcom_iommu_device::context_list */
+};
+
+struct qcom_iommu_domain {
+       struct io_pgtable_ops   *pgtbl_ops;
+       spinlock_t               pgtbl_lock;
+       struct mutex             init_mutex; /* Protects iommu pointer */
+       struct iommu_domain      domain;
+       struct qcom_iommu_dev   *iommu;
+};
+
+static struct qcom_iommu_domain *to_qcom_iommu_domain(struct iommu_domain *dom)
+{
+       return container_of(dom, struct qcom_iommu_domain, domain);
+}
+
+static const struct iommu_ops qcom_iommu_ops;
+
+static struct qcom_iommu_dev * __to_iommu(struct iommu_fwspec *fwspec)
+{
+       if (!fwspec || fwspec->ops != &qcom_iommu_ops)
+               return NULL;
+       return fwspec->iommu_priv;
+}
+
+static struct qcom_iommu_dev * to_iommu(struct iommu_fwspec *fwspec)
+{
+       struct qcom_iommu_dev *qcom_iommu = __to_iommu(fwspec);
+       WARN_ON(!qcom_iommu);
+       return qcom_iommu;
+}
+
+static struct qcom_iommu_ctx * to_ctx(struct iommu_fwspec *fwspec, unsigned asid)
+{
+       struct qcom_iommu_dev *qcom_iommu = to_iommu(fwspec);
+       struct qcom_iommu_ctx *ctx;
+
+       if (!qcom_iommu)
+               return NULL;
+
+       list_for_each_entry(ctx, &qcom_iommu->context_list, node)
+               if (ctx->asid == asid)
+                       return ctx;
+
+       WARN(1, "no ctx for asid %u\n", asid);
+       return NULL;
+}
+
+static inline void
+iommu_writel(struct qcom_iommu_ctx *ctx, unsigned reg, u32 val)
+{
+       writel_relaxed(val, ctx->base + reg);
+}
+
+static inline void
+iommu_writeq(struct qcom_iommu_ctx *ctx, unsigned reg, u64 val)
+{

drivers/iommu/qcom_iommu.c new file +153 -0

+iommu_readq(struct qcom_iommu_ctx *ctx, unsigned reg)
+{
+       return readq_relaxed(ctx->base + reg);
+}
+
+static void __sync_tlb(struct qcom_iommu_ctx *ctx)
+{
+       unsigned int val;
+       unsigned int ret;
+
+       iommu_writel(ctx, ARM_SMMU_CB_TLBSYNC, 0);
+
+       ret = readl_poll_timeout(ctx->base + ARM_SMMU_CB_TLBSTATUS, val,
+                                (val & 0x1) == 0, 0, 5000000);
+       if (ret)
+               dev_err(ctx->dev, "timeout waiting for TLB SYNC\n");
+}
+
+static void qcom_iommu_tlb_sync(void *cookie)
+{
+       struct iommu_fwspec *fwspec = cookie;
+       unsigned i;
+
+       for (i = 0; i < fwspec->num_ids; i++)
+               __sync_tlb(to_ctx(fwspec, fwspec->ids[i]));
+}
+
+static void qcom_iommu_tlb_inv_context(void *cookie)
+{
+       struct iommu_fwspec *fwspec = cookie;
+       unsigned i;
+
+       for (i = 0; i < fwspec->num_ids; i++) {
+               struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+
+               iommu_writel(ctx, ARM_SMMU_CB_S1_TLBIASID, ctx->asid);
+               __sync_tlb(ctx);
+       }
+}
+
+static void qcom_iommu_tlb_inv_range_nosync(unsigned long iova, size_t size,
+                                           size_t granule, bool leaf, void *cookie)
+{
+       struct iommu_fwspec *fwspec = cookie;
+       unsigned i, reg;
+
+       reg = leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
+
+       for (i = 0; i < fwspec->num_ids; i++) {
+               struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+               size_t s = size;
+
+               iova &= ~12UL;
+               iova |= ctx->asid;
+               do {
+                       iommu_writel(ctx, reg, iova);
+                       iova += granule;
+               } while (s -= granule);
+       }
+}
+
+static const struct iommu_gather_ops qcom_gather_ops = {
+       .tlb_flush_all  = qcom_iommu_tlb_inv_context,
+       .tlb_add_flush  = qcom_iommu_tlb_inv_range_nosync,
+       .tlb_sync       = qcom_iommu_tlb_sync,
+};
+
+static irqreturn_t qcom_iommu_fault(int irq, void *dev)
+{
+       struct qcom_iommu_ctx *ctx = dev;
+       u32 fsr, fsynr;
+       unsigned long iova;
+
+       fsr = iommu_readl(ctx, ARM_SMMU_CB_FSR);
+
+       if (!(fsr & FSR_FAULT))
+               return IRQ_NONE;
+
+       fsynr = iommu_readl(ctx, ARM_SMMU_CB_FSYNR0);
+       iova = iommu_readq(ctx, ARM_SMMU_CB_FAR);
+
+       dev_err_ratelimited(ctx->dev,
+                           "Unhandled context fault: fsr=0x%x, "
+                           "iova=0x%08lx, fsynr=0x%x, cb=%d\n",
+                           fsr, iova, fsynr, ctx->asid);
+
+       iommu_writel(ctx, ARM_SMMU_CB_FSR, fsr);
+
+       return IRQ_HANDLED;
+}
+
+static int qcom_iommu_init_domain(struct iommu_domain *domain,
+                                 struct qcom_iommu_dev *qcom_iommu,
+                                 struct iommu_fwspec *fwspec)
+{
+       struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+       struct io_pgtable_ops *pgtbl_ops;
+       struct io_pgtable_cfg pgtbl_cfg;
+       int i, ret = 0;
+       u32 reg;
+
+       mutex_lock(&qcom_domain->init_mutex);
+       if (qcom_domain->iommu)
+               goto out_unlock;
+
+       pgtbl_cfg = (struct io_pgtable_cfg) {
+               .pgsize_bitmap  = qcom_iommu_ops.pgsize_bitmap,
+               .ias            = 32,
+               .oas            = 40,
+               .tlb            = &qcom_gather_ops,
+               .iommu_dev      = qcom_iommu->dev,
+       };
+
+       qcom_domain->iommu = qcom_iommu;
+       pgtbl_ops = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &pgtbl_cfg, fwspec);
+       if (!pgtbl_ops) {
+               dev_err(qcom_iommu->dev, "failed to allocate pagetable ops\n");
+               ret = -ENOMEM;
+               goto out_clear_iommu;
+       }
+
+       /* Update the domain's page sizes to reflect the page table format */
+       domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
+       domain->geometry.aperture_end = (1ULL << pgtbl_cfg.ias) - 1;
+       domain->geometry.force_aperture = true;
+
+       for (i = 0; i < fwspec->num_ids; i++) {
+               struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+
+               if (!ctx->secure_init) {
+                       ret = qcom_scm_restore_sec_cfg(qcom_iommu->sec_id, ctx->asid);
+                       if (ret) {
+                               dev_err(qcom_iommu->dev, "secure init failed: %d\n", ret);
+                               goto out_clear_iommu;
+                       }
+                       ctx->secure_init = true;
+               }
+
+               /* TTBRs */
+               iommu_writeq(ctx, ARM_SMMU_CB_TTBR0,
+                               pgtbl_cfg.arm_lpae_s1_cfg.ttbr[0] |
+                               ((u64)ctx->asid << TTBRn_ASID_SHIFT));
+               iommu_writeq(ctx, ARM_SMMU_CB_TTBR1,
+                               pgtbl_cfg.arm_lpae_s1_cfg.ttbr[1] |
+                               ((u64)ctx->asid << TTBRn_ASID_SHIFT));
+
+               /* TTBCR */
+               iommu_writel(ctx, ARM_SMMU_CB_TTBCR2,
+                               (pgtbl_cfg.arm_lpae_s1_cfg.tcr >> 32) |
+                               TTBCR2_SEP_UPSTREAM);
+               iommu_writel(ctx, ARM_SMMU_CB_TTBCR,
+                               pgtbl_cfg.arm_lpae_s1_cfg.tcr);
+

drivers/iommu/qcom_iommu.c new file +466 -0

+               reg |= SCTLR_E;
+#endif
+               iommu_writel(ctx, ARM_SMMU_CB_SCTLR, reg);
+       }
+
+       mutex_unlock(&qcom_domain->init_mutex);
+
+       /* Publish page table ops for map/unmap */
+       qcom_domain->pgtbl_ops = pgtbl_ops;
+
+       return 0;
+
+out_clear_iommu:
+       qcom_domain->iommu = NULL;
+out_unlock:
+       mutex_unlock(&qcom_domain->init_mutex);
+       return ret;
+}
+
+static struct iommu_domain *qcom_iommu_domain_alloc(unsigned type)
+{
+       struct qcom_iommu_domain *qcom_domain;
+
+       if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
+               return NULL;
+       /*
+        * Allocate the domain and initialise some of its data structures.
+        * We can't really do anything meaningful until we've added a
+        * master.
+        */
+       qcom_domain = kzalloc(sizeof(*qcom_domain), GFP_KERNEL);
+       if (!qcom_domain)
+               return NULL;
+
+       if (type == IOMMU_DOMAIN_DMA &&
+           iommu_get_dma_cookie(&qcom_domain->domain)) {
+               kfree(qcom_domain);
+               return NULL;
+       }
+
+       mutex_init(&qcom_domain->init_mutex);
+       spin_lock_init(&qcom_domain->pgtbl_lock);
+
+       return &qcom_domain->domain;
+}
+
+static void qcom_iommu_domain_free(struct iommu_domain *domain)
+{
+       struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+
+       if (WARN_ON(qcom_domain->iommu))    /* forgot to detach? */
+               return;
+
+       iommu_put_dma_cookie(domain);
+
+       free_io_pgtable_ops(qcom_domain->pgtbl_ops);
+
+       kfree(qcom_domain);
+}
+
+static int qcom_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
+{
+       struct qcom_iommu_dev *qcom_iommu = to_iommu(dev->iommu_fwspec);
+       struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+       int ret;
+
+       if (!qcom_iommu) {
+               dev_err(dev, "cannot attach to IOMMU, is it on the same bus?\n");
+               return -ENXIO;
+       }
+
+       /* Ensure that the domain is finalized */
+       pm_runtime_get_sync(qcom_iommu->dev);
+       ret = qcom_iommu_init_domain(domain, qcom_iommu, dev->iommu_fwspec);
+       pm_runtime_put_sync(qcom_iommu->dev);
+       if (ret < 0)
+               return ret;
+
+       /*
+        * Sanity check the domain. We don't support domains across
+        * different IOMMUs.
+        */
+       if (qcom_domain->iommu != qcom_iommu) {
+               dev_err(dev, "cannot attach to IOMMU %s while already "
+                       "attached to domain on IOMMU %s\n",
+                       dev_name(qcom_domain->iommu->dev),
+                       dev_name(qcom_iommu->dev));
+               return -EINVAL;
+       }
+
+       return 0;
+}
+
+static void qcom_iommu_detach_dev(struct iommu_domain *domain, struct device *dev)
+{
+       struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+       struct qcom_iommu_dev *qcom_iommu = to_iommu(fwspec);
+       struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+       unsigned i;
+
+       if (!qcom_domain->iommu)
+               return;
+
+       pm_runtime_get_sync(qcom_iommu->dev);
+       for (i = 0; i < fwspec->num_ids; i++) {
+               struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+
+               /* Disable the context bank: */
+               iommu_writel(ctx, ARM_SMMU_CB_SCTLR, 0);
+       }
+       pm_runtime_put_sync(qcom_iommu->dev);
+
+       qcom_domain->iommu = NULL;
+}
+
+static int qcom_iommu_map(struct iommu_domain *domain, unsigned long iova,
+                         phys_addr_t paddr, size_t size, int prot)
+{
+       int ret;
+       unsigned long flags;
+       struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+       struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
+
+       if (!ops)
+               return -ENODEV;
+
+       spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
+       ret = ops->map(ops, iova, paddr, size, prot);
+       spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
+       return ret;
+}
+
+static size_t qcom_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
+                              size_t size)
+{
+       size_t ret;
+       unsigned long flags;
+       struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+       struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
+
+       if (!ops)
+               return 0;
+
+       spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
+       ret = ops->unmap(ops, iova, size);
+       spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
+       return ret;
+}
+
+static phys_addr_t qcom_iommu_iova_to_phys(struct iommu_domain *domain,
+                                          dma_addr_t iova)
+{
+       phys_addr_t ret;
+       unsigned long flags;
+       struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+       struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
+
+       if (!ops)
+               return 0;
+
+       spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
+       ret = ops->iova_to_phys(ops, iova);
+       spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
+
+       return ret;
+}
+
+static bool qcom_iommu_capable(enum iommu_cap cap)
+{
+       switch (cap) {
+       case IOMMU_CAP_CACHE_COHERENCY:
+               /*
+                * Return true here as the SMMU can always send out coherent
+                * requests.
+                */
+               return true;
+       case IOMMU_CAP_NOEXEC:
+               return true;
+       default:
+               return false;
+       }
+}
+
+static int qcom_iommu_add_device(struct device *dev)
+{
+       struct qcom_iommu_dev *qcom_iommu = __to_iommu(dev->iommu_fwspec);
+       struct iommu_group *group;
+       struct device_link *link;
+
+       if (!qcom_iommu)
+               return -ENODEV;
+
+       /*
+        * Establish the link between iommu and master, so that the
+        * iommu gets runtime enabled/disabled as per the master's
+        * needs.
+        */
+       link = device_link_add(dev, qcom_iommu->dev, DL_FLAG_PM_RUNTIME);
+       if (!link) {
+               dev_err(qcom_iommu->dev, "Unable to create device link between %s and %s\n",
+                       dev_name(qcom_iommu->dev), dev_name(dev));
+               return -ENODEV;
+       }
+
+       group = iommu_group_get_for_dev(dev);
+       if (IS_ERR_OR_NULL(group))
+               return PTR_ERR_OR_ZERO(group);
+
+       iommu_group_put(group);
+       iommu_device_link(&qcom_iommu->iommu, dev);
+
+       return 0;
+}
+
+static void qcom_iommu_remove_device(struct device *dev)
+{
+       struct qcom_iommu_dev *qcom_iommu = to_iommu(dev->iommu_fwspec);
+
+       if (!qcom_iommu)
+               return;
+
+       iommu_group_remove_device(dev);
+       iommu_device_unlink(&qcom_iommu->iommu, dev);
+       iommu_fwspec_free(dev);
+}
+
+static struct iommu_group *qcom_iommu_device_group(struct device *dev)
+{
+       struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+       struct iommu_group *group = NULL;
+       unsigned i;
+
+       for (i = 0; i < fwspec->num_ids; i++) {
+               struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+
+               if (group && ctx->group && group != ctx->group)
+                       return ERR_PTR(-EINVAL);
+
+               group = ctx->group;
+       }
+
+       if (group)
+               return iommu_group_ref_get(group);
+
+       group = generic_device_group(dev);
+
+       for (i = 0; i < fwspec->num_ids; i++) {
+               struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+               ctx->group = iommu_group_ref_get(group);
+       }
+
+       return group;
+}
+
+static int qcom_iommu_of_xlate(struct device *dev, struct of_phandle_args *args)
+{
+       struct platform_device *iommu_pdev;
+
+       if (args->args_count != 1) {
+               dev_err(dev, "incorrect number of iommu params found for %s "
+                       "(found %d, expected 1)\n",
+                       args->np->full_name, args->args_count);
+               return -EINVAL;
+       }
+
+       if (!dev->iommu_fwspec->iommu_priv) {
+               iommu_pdev = of_find_device_by_node(args->np);
+               if (WARN_ON(!iommu_pdev))
+                       return -EINVAL;
+
+               dev->iommu_fwspec->iommu_priv = platform_get_drvdata(iommu_pdev);
+       }
+
+       return iommu_fwspec_add_ids(dev, &args->args[0], 1);
+}
+
+static const struct iommu_ops qcom_iommu_ops = {
+       .capable        = qcom_iommu_capable,
+       .domain_alloc   = qcom_iommu_domain_alloc,
+       .domain_free    = qcom_iommu_domain_free,
+       .attach_dev     = qcom_iommu_attach_dev,
+       .detach_dev     = qcom_iommu_detach_dev,
+       .map            = qcom_iommu_map,
+       .unmap          = qcom_iommu_unmap,
+       .map_sg         = default_iommu_map_sg,
+       .iova_to_phys   = qcom_iommu_iova_to_phys,
+       .add_device     = qcom_iommu_add_device,
+       .remove_device  = qcom_iommu_remove_device,
+       .device_group   = qcom_iommu_device_group,
+       .of_xlate       = qcom_iommu_of_xlate,
+       .pgsize_bitmap  = SZ_4K | SZ_64K | SZ_1M | SZ_16M,
+};
+
+static int qcom_iommu_enable_clocks(struct qcom_iommu_dev *qcom_iommu)
+{
+       int ret;
+
+       ret = clk_prepare_enable(qcom_iommu->iface_clk);
+       if (ret) {
+               dev_err(qcom_iommu->dev, "Couldn't enable iface_clk\n");
+               return ret;
+       }
+
+       ret = clk_prepare_enable(qcom_iommu->bus_clk);
+       if (ret) {
+               dev_err(qcom_iommu->dev, "Couldn't enable bus_clk\n");
+               clk_disable_unprepare(qcom_iommu->iface_clk);
+               return ret;
+       }
+
+       return 0;
+}
+
+static void qcom_iommu_disable_clocks(struct qcom_iommu_dev *qcom_iommu)
+{
+       clk_disable_unprepare(qcom_iommu->bus_clk);
+       clk_disable_unprepare(qcom_iommu->iface_clk);
+}
+
+static int qcom_iommu_ctx_probe(struct platform_device *pdev)
+{
+       struct qcom_iommu_ctx *ctx;
+       struct device *dev = &pdev->dev;
+       struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(dev->parent);
+       struct resource *res;
+       int ret;
+       u32 reg;
+
+       ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
+       if (!ctx)
+               return -ENOMEM;
+
+       ctx->dev = dev;
+       platform_set_drvdata(pdev, ctx);
+
+       res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+       ctx->base = devm_ioremap_resource(dev, res);
+       if (IS_ERR(ctx->base))
+               return PTR_ERR(ctx->base);
+
+       ctx->irq = platform_get_irq(pdev, 0);
+       if (ctx->irq < 0) {
+               dev_err(dev, "failed to get irq\n");
+               return -ENODEV;
+       }
+
+       ret = devm_request_irq(dev, ctx->irq,
+                              qcom_iommu_fault,
+                              IRQF_SHARED,
+                              "qcom-iommu-fault",
+                              ctx);
+       if (ret) {
+               dev_err(dev, "failed to request IRQ %u\n", ctx->irq);
+               return ret;
+       }
+
+       /* read the "reg" property directly to get the relative address
+        * of the context bank, and calculate the asid from that:
+        */
+       if (of_property_read_u32_index(dev->of_node, "reg", 0, &reg)) {
+               dev_err(dev, "missing reg property\n");
+               return -ENODEV;
+       }
+
+       ctx->asid = reg / 0x1000;      /* context banks are 0x1000 apart */
+
+       dev_dbg(dev, "found asid %u\n", ctx->asid);
+
+       list_add_tail(&ctx->node, &qcom_iommu->context_list);
+
+       return 0;
+}
+
+static int qcom_iommu_ctx_remove(struct platform_device *pdev)
+{
+       struct qcom_iommu_ctx *ctx = platform_get_drvdata(pdev);
+
+       iommu_group_put(ctx->group);
+       platform_set_drvdata(pdev, NULL);
+
+       return 0;
+}
+
+static const struct of_device_id ctx_of_match[] = {
+       { .compatible = "qcom,msm-iommu-v1-ns" },
+       { .compatible = "qcom,msm-iommu-v1-sec" },
+       { /* sentinel */ }
+};
+
+static struct platform_driver qcom_iommu_ctx_driver = {
+       .driver = {
+               .name           = "qcom-iommu-ctx",
+               .of_match_table = of_match_ptr(ctx_of_match),
+       },
+       .probe  = qcom_iommu_ctx_probe,
+       .remove = qcom_iommu_ctx_remove,
+};
+module_platform_driver(qcom_iommu_ctx_driver);
+
+static int qcom_iommu_device_probe(struct platform_device *pdev)
+{
+       struct qcom_iommu_dev *qcom_iommu;
+       struct device *dev = &pdev->dev;
+       struct resource *res;
+       int ret;
+
+       qcom_iommu = devm_kzalloc(dev, sizeof(*qcom_iommu), GFP_KERNEL);
+       if (!qcom_iommu)
+               return -ENOMEM;
+       qcom_iommu->dev = dev;
+
+       INIT_LIST_HEAD(&qcom_iommu->context_list);
+
+       res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+       if (res)
+               qcom_iommu->local_base = devm_ioremap_resource(dev, res);
+
+       qcom_iommu->iface_clk = devm_clk_get(dev, "iface");
+       if (IS_ERR(qcom_iommu->iface_clk)) {
+               dev_err(dev, "failed to get iface clock\n");
+               return PTR_ERR(qcom_iommu->iface_clk);
+       }
+
+       qcom_iommu->bus_clk = devm_clk_get(dev, "bus");
+       if (IS_ERR(qcom_iommu->bus_clk)) {
+               dev_err(dev, "failed to get bus clock\n");
+               return PTR_ERR(qcom_iommu->bus_clk);
+       }
+
+       if (of_property_read_u32(dev->of_node, "qcom,iommu-secure-id",
+                                &qcom_iommu->sec_id)) {
+               dev_err(dev, "missing qcom,iommu-secure-id property\n");
+               return -ENODEV;
+       }
+
+       platform_set_drvdata(pdev, qcom_iommu);
+
+       /* register context bank devices, which are child nodes: */
+       ret = of_platform_populate(dev->of_node, ctx_of_match, NULL, dev);
+       if (ret) {
+               dev_err(dev, "Failed to populate iommu contexts\n");
+               return ret;
+       }
+
+       ret = iommu_device_sysfs_add(&qcom_iommu->iommu, dev, NULL,
+                                    "smmu.%pa", &res->start);
+       if (ret) {
+               dev_err(dev, "Failed to register iommu in sysfs\n");
+               return ret;
+       }
+
+       iommu_device_set_ops(&qcom_iommu->iommu, &qcom_iommu_ops);
+       iommu_device_set_fwnode(&qcom_iommu->iommu, dev->fwnode);
+
+       ret = iommu_device_register(&qcom_iommu->iommu);
+       if (ret) {
+               dev_err(dev, "Failed to register iommu\n");
+               return ret;
+       }
+
+       pm_runtime_enable(dev);
+       bus_set_iommu(&platform_bus_type, &qcom_iommu_ops);
+
+       if (qcom_iommu->local_base) {
+               pm_runtime_get_sync(dev);
+               writel_relaxed(0xffffffff, qcom_iommu->local_base + SMMU_INTR_SEL_NS);

drivers/iommu/qcom_iommu.c new file +38 -0

+
+       return 0;
+}
+
+#ifdef CONFIG_PM
+static int qcom_iommu_resume(struct device *dev)
+{
+       struct platform_device *pdev = to_platform_device(dev);
+       struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
+
+       return qcom_iommu_enable_clocks(qcom_iommu);
+}
+
+static int qcom_iommu_suspend(struct device *dev)
+{
+       struct platform_device *pdev = to_platform_device(dev);
+       struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
+
+       qcom_iommu_disable_clocks(qcom_iommu);
+
+       return 0;
+}
+#endif
+
+static const struct dev_pm_ops qcom_iommu_pm_ops = {
+       SET_RUNTIME_PM_OPS(qcom_iommu_suspend, qcom_iommu_resume, NULL)
+       SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+                               pm_runtime_force_resume)
+};
+
+static const struct of_device_id qcom_iommu_of_match[] = {
+       { .compatible = "qcom,msm-iommu-v1" },
+       { /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, qcom_iommu_of_match);
+
+static struct platform_driver qcom_iommu_driver = {
+       .driver = {

drivers/iommu/qcom_iommu.c new file +3 -1

+
+MODULE_DESCRIPTION("IOMMU API for QCOM IOMMU v1 implementations");
+MODULE_LICENSE("GPL v2");
--

2.9.3

Quoted 7 lines

drivers/iommu/qcom_iommu.c new file +7 -0

+
+static struct platform_driver qcom_iommu_driver = {
+       .driver = {
+               .name           = "qcom-iommu",
+               .of_match_table = of_match_ptr(qcom_iommu_of_match),
+               .pm             = &qcom_iommu_pm_ops,
+       },

drivers/iommu/qcom_iommu.c line 822

Quoted 6 lines

drivers/iommu/qcom_iommu.c new file +6 -0

+       .probe  = qcom_iommu_device_probe,
+       .remove = qcom_iommu_device_remove,
+};
+module_platform_driver(qcom_iommu_driver);
+
+IOMMU_OF_DECLARE(qcom_iommu_dev, "qcom,msm-iommu-v1", NULL);

Is this needed any more with deferred probe now?

Quoted 7 lines

drivers/iommu/qcom_iommu.c new file +3 -1

+
+MODULE_DESCRIPTION("IOMMU API for QCOM IOMMU v1 implementations");
+MODULE_LICENSE("GPL v2");
--

2.9.3

An iommu driver for Qualcomm "B" family devices which do not completely implement the ARM SMMU spec. These devices have context-bank register layout that is similar to ARM SMMU, but no global register space (or at least not one that is accessible).

Signed-off-by Rob Clark <[email protected]>

---
v1: original
v2: bindings cleanups and kconfig issues that kbuild robot pointed out
v4: fix issues pointed out by Rob H. and actually make device removal
work
v3: fix WARN_ON() splats reported by Archit

 drivers/iommu/Kconfig      |   9 +
 drivers/iommu/Makefile     |   1 +
 drivers/iommu/qcom_iommu.c | 833 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 843 insertions(+)
 create mode 100644 drivers/iommu/qcom_iommu.c

drivers/iommu/Kconfig +0 -0

index 37e204f..f8f79a4 100644

Quoted 24 lines · Rob Clark

An iommu driver for Qualcomm "B" family devices which do not completely implement the ARM SMMU spec. These devices have context-bank register layout that is similar to ARM SMMU, but no global register space (or at least not one that is accessible).

Signed-off-by Rob Clark <[email protected]>

---
v1: original
v2: bindings cleanups and kconfig issues that kbuild robot pointed out
v4: fix issues pointed out by Rob H. and actually make device removal
work
v3: fix WARN_ON() splats reported by Archit

 drivers/iommu/Kconfig      |   9 +
 drivers/iommu/Makefile     |   1 +
 drivers/iommu/qcom_iommu.c | 833 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 843 insertions(+)
 create mode 100644 drivers/iommu/qcom_iommu.c

drivers/iommu/Kconfig +0 -0

index 37e204f..f8f79a4 100644

drivers/iommu/Kconfig config MTK_IOMMU_V1 · line 363

Quoted 5 lines

drivers/iommu/Kconfig +2 -0

@@ -359,4 +359,13 @@ config MTK_IOMMU_V1
 
          if unsure, say N here.
 
+config QCOM_IOMMU
+       bool "Qualcomm IOMMU Support"

Either this needs to be tristate or...

[...]

Quoted 1 line

+#include <linux/module.h>

this include and the things that need it should go. Or some explanation like "once X happens, then we can enable as module" and leave it all for now.

See this[1] for some background.

Rob

[1] https://lwn.net/Articles/643854/

Quoted 33 lines · Rob Herring

Quoted 24 lines · Rob Clark

An iommu driver for Qualcomm "B" family devices which do not completely implement the ARM SMMU spec. These devices have context-bank register layout that is similar to ARM SMMU, but no global register space (or at least not one that is accessible).

Signed-off-by Rob Clark <[email protected]>

---
v1: original
v2: bindings cleanups and kconfig issues that kbuild robot pointed out
v4: fix issues pointed out by Rob H. and actually make device removal
work
v3: fix WARN_ON() splats reported by Archit

 drivers/iommu/Kconfig      |   9 +
 drivers/iommu/Makefile     |   1 +
 drivers/iommu/qcom_iommu.c | 833 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 843 insertions(+)
 create mode 100644 drivers/iommu/qcom_iommu.c

drivers/iommu/Kconfig +0 -0

index 37e204f..f8f79a4 100644

drivers/iommu/Kconfig config MTK_IOMMU_V1 · line 363

Quoted 5 lines

drivers/iommu/Kconfig +2 -0

@@ -359,4 +359,13 @@ config MTK_IOMMU_V1
 
          if unsure, say N here.
 
+config QCOM_IOMMU
+       bool "Qualcomm IOMMU Support"

Either this needs to be tristate or...

Quoted 6 lines

[...]

Quoted 1 line

+#include <linux/module.h>

this include and the things that need it should go. Or some explanation like "once X happens, then we can enable as module" and leave it all for now.

tbh, I'm not sure what the issue is for modules (other than potentially that you'd want the iommu driver fairly early in boot if you didn't have an initrd). I just saw that the other iommu drivers are all bool. (Sorry, I don't really follow iommu-devel so not familiar with the history.) With my distro hat on, I would prefer them to be modules eventually.

Quoted 2 lines

See this[1] for some background.

it mentions there are some downsides, but I can't see where those downsides are listed ;-)

I would kinda prefer to leave the MODULE_*() stuff in place unless modular iommu drivers are never going to happen.

Quoted 41 lines · Rob Clark

Quoted 33 lines · Rob Herring

Quoted 24 lines · Rob Clark

An iommu driver for Qualcomm "B" family devices which do not completely implement the ARM SMMU spec. These devices have context-bank register layout that is similar to ARM SMMU, but no global register space (or at least not one that is accessible).

Signed-off-by Rob Clark <[email protected]>

---
v1: original
v2: bindings cleanups and kconfig issues that kbuild robot pointed out
v4: fix issues pointed out by Rob H. and actually make device removal
work
v3: fix WARN_ON() splats reported by Archit

 drivers/iommu/Kconfig      |   9 +
 drivers/iommu/Makefile     |   1 +
 drivers/iommu/qcom_iommu.c | 833 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 843 insertions(+)
 create mode 100644 drivers/iommu/qcom_iommu.c

drivers/iommu/Kconfig +0 -0

index 37e204f..f8f79a4 100644

drivers/iommu/Kconfig config MTK_IOMMU_V1 · line 363

Quoted 5 lines

drivers/iommu/Kconfig +2 -0

@@ -359,4 +359,13 @@ config MTK_IOMMU_V1
 
          if unsure, say N here.
 
+config QCOM_IOMMU
+       bool "Qualcomm IOMMU Support"

Either this needs to be tristate or...

Quoted 6 lines

[...]

Quoted 1 line

+#include <linux/module.h>

this include and the things that need it should go. Or some explanation like "once X happens, then we can enable as module" and leave it all for now.

Quoted 6 lines

tbh, I'm not sure what the issue is for modules (other than potentially that you'd want the iommu driver fairly early in boot if you didn't have an initrd). I just saw that the other iommu drivers are all bool. (Sorry, I don't really follow iommu-devel so not familiar with the history.) With my distro hat on, I would prefer them to be modules eventually.

For starters, does it even build as a module if you allow that? It might not work because of some run-time ordering, but that's good enough for this discussion.

Quoted 8 lines

Quoted 2 lines

See this[1] for some background.

it mentions there are some downsides, but I can't see where those downsides are listed ;-)

I would kinda prefer to leave the MODULE_*() stuff in place unless modular iommu drivers are never going to happen.

Maybe a note, so the module police don't fix it.

Rob

drivers/iommu/qcom_iommu.c new file +10 -0

+static irqreturn_t qcom_iommu_fault(int irq, void *dev)
+{
+	struct qcom_iommu_ctx *ctx = dev;
+	u32 fsr, fsynr;
+	unsigned long iova;
+
+	fsr = iommu_readl(ctx, ARM_SMMU_CB_FSR);
+
+	if (!(fsr & FSR_FAULT))
+		return IRQ_NONE;

drivers/iommu/qcom_iommu.c new file +634 -0

+
+	return IRQ_HANDLED;
+}
+
+static int qcom_iommu_init_domain(struct iommu_domain *domain,
+				  struct qcom_iommu_dev *qcom_iommu,
+				  struct iommu_fwspec *fwspec)
+{
+	struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+	struct io_pgtable_ops *pgtbl_ops;
+	struct io_pgtable_cfg pgtbl_cfg;
+	int i, ret = 0;
+	u32 reg;
+
+	mutex_lock(&qcom_domain->init_mutex);
+	if (qcom_domain->iommu)
+		goto out_unlock;
+
+	pgtbl_cfg = (struct io_pgtable_cfg) {
+		.pgsize_bitmap	= qcom_iommu_ops.pgsize_bitmap,
+		.ias		= 32,
+		.oas		= 40,
+		.tlb		= &qcom_gather_ops,
+		.iommu_dev	= qcom_iommu->dev,
+	};
+
+	qcom_domain->iommu = qcom_iommu;
+	pgtbl_ops = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &pgtbl_cfg, fwspec);
+	if (!pgtbl_ops) {
+		dev_err(qcom_iommu->dev, "failed to allocate pagetable ops\n");
+		ret = -ENOMEM;
+		goto out_clear_iommu;
+	}
+
+	/* Update the domain's page sizes to reflect the page table format */
+	domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
+	domain->geometry.aperture_end = (1ULL << pgtbl_cfg.ias) - 1;
+	domain->geometry.force_aperture = true;
+
+	for (i = 0; i < fwspec->num_ids; i++) {
+		struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+
+		if (!ctx->secure_init) {
+			ret = qcom_scm_restore_sec_cfg(qcom_iommu->sec_id, ctx->asid);
+			if (ret) {
+				dev_err(qcom_iommu->dev, "secure init failed: %d\n", ret);
+				goto out_clear_iommu;
+			}
+			ctx->secure_init = true;
+		}
+
+		/* TTBRs */
+		iommu_writeq(ctx, ARM_SMMU_CB_TTBR0,
+				pgtbl_cfg.arm_lpae_s1_cfg.ttbr[0] |
+				((u64)ctx->asid << TTBRn_ASID_SHIFT));
+		iommu_writeq(ctx, ARM_SMMU_CB_TTBR1,
+				pgtbl_cfg.arm_lpae_s1_cfg.ttbr[1] |
+				((u64)ctx->asid << TTBRn_ASID_SHIFT));
+
+		/* TTBCR */
+		iommu_writel(ctx, ARM_SMMU_CB_TTBCR2,
+				(pgtbl_cfg.arm_lpae_s1_cfg.tcr >> 32) |
+				TTBCR2_SEP_UPSTREAM);
+		iommu_writel(ctx, ARM_SMMU_CB_TTBCR,
+				pgtbl_cfg.arm_lpae_s1_cfg.tcr);
+
+		/* MAIRs (stage-1 only) */
+		iommu_writel(ctx, ARM_SMMU_CB_S1_MAIR0,
+				pgtbl_cfg.arm_lpae_s1_cfg.mair[0]);
+		iommu_writel(ctx, ARM_SMMU_CB_S1_MAIR1,
+				pgtbl_cfg.arm_lpae_s1_cfg.mair[1]);
+
+		/* SCTLR */
+		reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE |
+			SCTLR_M | SCTLR_S1_ASIDPNE;
+
+		if (IS_ENABLED(CONFIG_BIG_ENDIAN))
+			reg |= SCTLR_E;
+
+		iommu_writel(ctx, ARM_SMMU_CB_SCTLR, reg);
+	}
+
+	mutex_unlock(&qcom_domain->init_mutex);
+
+	/* Publish page table ops for map/unmap */
+	qcom_domain->pgtbl_ops = pgtbl_ops;
+
+	return 0;
+
+out_clear_iommu:
+	qcom_domain->iommu = NULL;
+out_unlock:
+	mutex_unlock(&qcom_domain->init_mutex);
+	return ret;
+}
+
+static struct iommu_domain *qcom_iommu_domain_alloc(unsigned type)
+{
+	struct qcom_iommu_domain *qcom_domain;
+
+	if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
+		return NULL;
+	/*
+	 * Allocate the domain and initialise some of its data structures.
+	 * We can't really do anything meaningful until we've added a
+	 * master.
+	 */
+	qcom_domain = kzalloc(sizeof(*qcom_domain), GFP_KERNEL);
+	if (!qcom_domain)
+		return NULL;
+
+	if (type == IOMMU_DOMAIN_DMA &&
+	    iommu_get_dma_cookie(&qcom_domain->domain)) {
+		kfree(qcom_domain);
+		return NULL;
+	}
+
+	mutex_init(&qcom_domain->init_mutex);
+	spin_lock_init(&qcom_domain->pgtbl_lock);
+
+	return &qcom_domain->domain;
+}
+
+static void qcom_iommu_domain_free(struct iommu_domain *domain)
+{
+	struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+
+	if (WARN_ON(qcom_domain->iommu))    /* forgot to detach? */
+		return;
+
+	iommu_put_dma_cookie(domain);
+
+	free_io_pgtable_ops(qcom_domain->pgtbl_ops);
+
+	kfree(qcom_domain);
+}
+
+static int qcom_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
+{
+	struct qcom_iommu_dev *qcom_iommu = to_iommu(dev->iommu_fwspec);
+	struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+	int ret;
+
+	if (!qcom_iommu) {
+		dev_err(dev, "cannot attach to IOMMU, is it on the same bus?\n");
+		return -ENXIO;
+	}
+
+	/* Ensure that the domain is finalized */
+	pm_runtime_get_sync(qcom_iommu->dev);
+	ret = qcom_iommu_init_domain(domain, qcom_iommu, dev->iommu_fwspec);
+	pm_runtime_put_sync(qcom_iommu->dev);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * Sanity check the domain. We don't support domains across
+	 * different IOMMUs.
+	 */
+	if (qcom_domain->iommu != qcom_iommu) {
+		dev_err(dev, "cannot attach to IOMMU %s while already "
+			"attached to domain on IOMMU %s\n",
+			dev_name(qcom_domain->iommu->dev),
+			dev_name(qcom_iommu->dev));
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void qcom_iommu_detach_dev(struct iommu_domain *domain, struct device *dev)
+{
+	struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+	struct qcom_iommu_dev *qcom_iommu = to_iommu(fwspec);
+	struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+	unsigned i;
+
+	if (!qcom_domain->iommu)
+		return;
+
+	pm_runtime_get_sync(qcom_iommu->dev);
+	for (i = 0; i < fwspec->num_ids; i++) {
+		struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+
+		/* Disable the context bank: */
+		iommu_writel(ctx, ARM_SMMU_CB_SCTLR, 0);
+	}
+	pm_runtime_put_sync(qcom_iommu->dev);
+
+	qcom_domain->iommu = NULL;
+}
+
+static int qcom_iommu_map(struct iommu_domain *domain, unsigned long iova,
+			  phys_addr_t paddr, size_t size, int prot)
+{
+	int ret;
+	unsigned long flags;
+	struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+	struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
+
+	if (!ops)
+		return -ENODEV;
+
+	spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
+	ret = ops->map(ops, iova, paddr, size, prot);
+	spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
+	return ret;
+}
+
+static size_t qcom_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
+			       size_t size)
+{
+	size_t ret;
+	unsigned long flags;
+	struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+	struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
+
+	if (!ops)
+		return 0;
+
+	spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
+	ret = ops->unmap(ops, iova, size);
+	spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
+	return ret;
+}
+
+static phys_addr_t qcom_iommu_iova_to_phys(struct iommu_domain *domain,
+					   dma_addr_t iova)
+{
+	phys_addr_t ret;
+	unsigned long flags;
+	struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+	struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
+
+	if (!ops)
+		return 0;
+
+	spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
+	ret = ops->iova_to_phys(ops, iova);
+	spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
+
+	return ret;
+}
+
+static bool qcom_iommu_capable(enum iommu_cap cap)
+{
+	switch (cap) {
+	case IOMMU_CAP_CACHE_COHERENCY:
+		/*
+		 * Return true here as the SMMU can always send out coherent
+		 * requests.
+		 */
+		return true;
+	case IOMMU_CAP_NOEXEC:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static int qcom_iommu_add_device(struct device *dev)
+{
+	struct qcom_iommu_dev *qcom_iommu = __to_iommu(dev->iommu_fwspec);
+	struct iommu_group *group;
+	struct device_link *link;
+
+	if (!qcom_iommu)
+		return -ENODEV;
+
+	/*
+	 * Establish the link between iommu and master, so that the
+	 * iommu gets runtime enabled/disabled as per the master's
+	 * needs.
+	 */
+	link = device_link_add(dev, qcom_iommu->dev, DL_FLAG_PM_RUNTIME);
+	if (!link) {
+		dev_err(qcom_iommu->dev, "Unable to create device link between %s and %s\n",
+			dev_name(qcom_iommu->dev), dev_name(dev));
+		return -ENODEV;
+	}
+
+	group = iommu_group_get_for_dev(dev);
+	if (IS_ERR_OR_NULL(group))
+		return PTR_ERR_OR_ZERO(group);
+
+	iommu_group_put(group);
+	iommu_device_link(&qcom_iommu->iommu, dev);
+
+	return 0;
+}
+
+static void qcom_iommu_remove_device(struct device *dev)
+{
+	struct qcom_iommu_dev *qcom_iommu = to_iommu(dev->iommu_fwspec);
+
+	if (!qcom_iommu)
+		return;
+
+	iommu_device_unlink(&qcom_iommu->iommu, dev);
+	iommu_group_remove_device(dev);
+	iommu_fwspec_free(dev);
+}
+
+static struct iommu_group *qcom_iommu_device_group(struct device *dev)
+{
+	struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+	struct iommu_group *group = NULL;
+	unsigned i;
+
+	for (i = 0; i < fwspec->num_ids; i++) {
+		struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+
+		if (group && ctx->group && group != ctx->group)
+			return ERR_PTR(-EINVAL);
+
+		group = ctx->group;
+	}
+
+	if (group)
+		return iommu_group_ref_get(group);
+
+	group = generic_device_group(dev);
+
+	for (i = 0; i < fwspec->num_ids; i++) {
+		struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+		ctx->group = iommu_group_ref_get(group);
+	}
+
+	return group;
+}
+
+static int qcom_iommu_of_xlate(struct device *dev, struct of_phandle_args *args)
+{
+	struct platform_device *iommu_pdev;
+
+	if (args->args_count != 1) {
+		dev_err(dev, "incorrect number of iommu params found for %s "
+			"(found %d, expected 1)\n",
+			args->np->full_name, args->args_count);
+		return -EINVAL;
+	}
+
+	if (!dev->iommu_fwspec->iommu_priv) {
+		iommu_pdev = of_find_device_by_node(args->np);
+		if (WARN_ON(!iommu_pdev))
+			return -EINVAL;
+
+		dev->iommu_fwspec->iommu_priv = platform_get_drvdata(iommu_pdev);
+	}
+
+	return iommu_fwspec_add_ids(dev, &args->args[0], 1);
+}
+
+static const struct iommu_ops qcom_iommu_ops = {
+	.capable	= qcom_iommu_capable,
+	.domain_alloc	= qcom_iommu_domain_alloc,
+	.domain_free	= qcom_iommu_domain_free,
+	.attach_dev	= qcom_iommu_attach_dev,
+	.detach_dev	= qcom_iommu_detach_dev,
+	.map		= qcom_iommu_map,
+	.unmap		= qcom_iommu_unmap,
+	.map_sg		= default_iommu_map_sg,
+	.iova_to_phys	= qcom_iommu_iova_to_phys,
+	.add_device	= qcom_iommu_add_device,
+	.remove_device	= qcom_iommu_remove_device,
+	.device_group	= qcom_iommu_device_group,
+	.of_xlate	= qcom_iommu_of_xlate,
+	.pgsize_bitmap	= SZ_4K | SZ_64K | SZ_1M | SZ_16M,
+};
+
+static int qcom_iommu_enable_clocks(struct qcom_iommu_dev *qcom_iommu)
+{
+	int ret;
+
+	ret = clk_prepare_enable(qcom_iommu->iface_clk);
+	if (ret) {
+		dev_err(qcom_iommu->dev, "Couldn't enable iface_clk\n");
+		return ret;
+	}
+
+	ret = clk_prepare_enable(qcom_iommu->bus_clk);
+	if (ret) {
+		dev_err(qcom_iommu->dev, "Couldn't enable bus_clk\n");
+		clk_disable_unprepare(qcom_iommu->iface_clk);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void qcom_iommu_disable_clocks(struct qcom_iommu_dev *qcom_iommu)
+{
+	clk_disable_unprepare(qcom_iommu->bus_clk);
+	clk_disable_unprepare(qcom_iommu->iface_clk);
+}
+
+static int qcom_iommu_ctx_probe(struct platform_device *pdev)
+{
+	struct qcom_iommu_ctx *ctx;
+	struct device *dev = &pdev->dev;
+	struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(dev->parent);
+	struct resource *res;
+	int ret;
+	u32 reg;
+
+	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	ctx->dev = dev;
+	platform_set_drvdata(pdev, ctx);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	ctx->base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(ctx->base))
+		return PTR_ERR(ctx->base);
+
+	ctx->irq = platform_get_irq(pdev, 0);
+	if (ctx->irq < 0) {
+		dev_err(dev, "failed to get irq\n");
+		return -ENODEV;
+	}
+
+	ret = devm_request_irq(dev, ctx->irq,
+			       qcom_iommu_fault,
+			       IRQF_SHARED,
+			       "qcom-iommu-fault",
+			       ctx);
+	if (ret) {
+		dev_err(dev, "failed to request IRQ %u\n", ctx->irq);
+		return ret;
+	}
+
+	/* read the "reg" property directly to get the relative address
+	 * of the context bank, and calculate the asid from that:
+	 */
+	if (of_property_read_u32_index(dev->of_node, "reg", 0, &reg)) {
+		dev_err(dev, "missing reg property\n");
+		return -ENODEV;
+	}
+
+	ctx->asid = reg / 0x1000;      /* context banks are 0x1000 apart */
+
+	dev_dbg(dev, "found asid %u\n", ctx->asid);
+
+	list_add_tail(&ctx->node, &qcom_iommu->context_list);
+
+	return 0;
+}
+
+static int qcom_iommu_ctx_remove(struct platform_device *pdev)
+{
+	struct qcom_iommu_ctx *ctx = platform_get_drvdata(pdev);
+
+	iommu_group_put(ctx->group);
+	platform_set_drvdata(pdev, NULL);
+
+	list_del(&ctx->node);
+
+	return 0;
+}
+
+static const struct of_device_id ctx_of_match[] = {
+	{ .compatible = "qcom,msm-iommu-v1-ns" },
+	{ .compatible = "qcom,msm-iommu-v1-sec" },
+	{ /* sentinel */ }
+};
+
+static struct platform_driver qcom_iommu_ctx_driver = {
+	.driver	= {
+		.name		= "qcom-iommu-ctx",
+		.of_match_table	= of_match_ptr(ctx_of_match),
+	},
+	.probe	= qcom_iommu_ctx_probe,
+	.remove = qcom_iommu_ctx_remove,
+};
+
+static int qcom_iommu_device_probe(struct platform_device *pdev)
+{
+	struct qcom_iommu_dev *qcom_iommu;
+	struct device *dev = &pdev->dev;
+	struct resource *res;
+	int ret;
+
+	qcom_iommu = devm_kzalloc(dev, sizeof(*qcom_iommu), GFP_KERNEL);
+	if (!qcom_iommu)
+		return -ENOMEM;
+	qcom_iommu->dev = dev;
+
+	INIT_LIST_HEAD(&qcom_iommu->context_list);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (res)
+		qcom_iommu->local_base = devm_ioremap_resource(dev, res);
+
+	qcom_iommu->iface_clk = devm_clk_get(dev, "iface");
+	if (IS_ERR(qcom_iommu->iface_clk)) {
+		dev_err(dev, "failed to get iface clock\n");
+		return PTR_ERR(qcom_iommu->iface_clk);
+	}
+
+	qcom_iommu->bus_clk = devm_clk_get(dev, "bus");
+	if (IS_ERR(qcom_iommu->bus_clk)) {
+		dev_err(dev, "failed to get bus clock\n");
+		return PTR_ERR(qcom_iommu->bus_clk);
+	}
+
+	if (of_property_read_u32(dev->of_node, "qcom,iommu-secure-id",
+				 &qcom_iommu->sec_id)) {
+		dev_err(dev, "missing qcom,iommu-secure-id property\n");
+		return -ENODEV;
+	}
+
+	platform_set_drvdata(pdev, qcom_iommu);
+
+	/* register context bank devices, which are child nodes: */
+	ret = devm_of_platform_populate(dev);
+	if (ret) {
+		dev_err(dev, "Failed to populate iommu contexts\n");
+		return ret;
+	}
+
+	ret = iommu_device_sysfs_add(&qcom_iommu->iommu, dev, NULL,
+				     dev_name(dev));
+	if (ret) {
+		dev_err(dev, "Failed to register iommu in sysfs\n");
+		return ret;
+	}
+
+	iommu_device_set_ops(&qcom_iommu->iommu, &qcom_iommu_ops);
+	iommu_device_set_fwnode(&qcom_iommu->iommu, dev->fwnode);
+
+	ret = iommu_device_register(&qcom_iommu->iommu);
+	if (ret) {
+		dev_err(dev, "Failed to register iommu\n");
+		return ret;
+	}
+
+	pm_runtime_enable(dev);
+	bus_set_iommu(&platform_bus_type, &qcom_iommu_ops);
+
+	if (qcom_iommu->local_base) {
+		pm_runtime_get_sync(dev);
+		writel_relaxed(0xffffffff, qcom_iommu->local_base + SMMU_INTR_SEL_NS);
+		pm_runtime_put_sync(dev);
+	}
+
+	return 0;
+}
+
+static int qcom_iommu_device_remove(struct platform_device *pdev)
+{
+	struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
+
+	bus_set_iommu(&platform_bus_type, NULL);
+
+	pm_runtime_force_suspend(&pdev->dev);
+	platform_set_drvdata(pdev, NULL);
+	iommu_device_sysfs_remove(&qcom_iommu->iommu);
+	iommu_device_unregister(&qcom_iommu->iommu);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM
+static int qcom_iommu_resume(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
+
+	return qcom_iommu_enable_clocks(qcom_iommu);
+}
+
+static int qcom_iommu_suspend(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
+
+	qcom_iommu_disable_clocks(qcom_iommu);
+
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops qcom_iommu_pm_ops = {
+	SET_RUNTIME_PM_OPS(qcom_iommu_suspend, qcom_iommu_resume, NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
+};
+
+static const struct of_device_id qcom_iommu_of_match[] = {
+	{ .compatible = "qcom,msm-iommu-v1" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, qcom_iommu_of_match);
+
+static struct platform_driver qcom_iommu_driver = {
+	.driver	= {
+		.name		= "qcom-iommu",
+		.of_match_table	= of_match_ptr(qcom_iommu_of_match),
+		.pm		= &qcom_iommu_pm_ops,
+	},
+	.probe	= qcom_iommu_device_probe,
+	.remove	= qcom_iommu_device_remove,
+};
+
+static int __init qcom_iommu_init(void)
+{
+	int ret;
+
+	ret = platform_driver_register(&qcom_iommu_ctx_driver);
+	if (ret)
+		return ret;
+
+	ret = platform_driver_register(&qcom_iommu_driver);
+	if (ret)
+		platform_driver_unregister(&qcom_iommu_ctx_driver);
+
+	return ret;
+}
+
+static void __exit qcom_iommu_exit(void)
+{
+	platform_driver_unregister(&qcom_iommu_driver);
+	platform_driver_unregister(&qcom_iommu_ctx_driver);
+}
+
+module_init(qcom_iommu_init);
+module_exit(qcom_iommu_exit);
+
+IOMMU_OF_DECLARE(qcom_iommu_dev, "qcom,msm-iommu-v1", NULL);
+
+MODULE_DESCRIPTION("IOMMU API for QCOM IOMMU v1 implementations");
+MODULE_LICENSE("GPL v2");

Hi Rob,

<snip..>

Quoted 10 lines

drivers/iommu/qcom_iommu.c new file +10 -0

+static irqreturn_t qcom_iommu_fault(int irq, void *dev)
+{
+     struct qcom_iommu_ctx *ctx = dev;
+     u32 fsr, fsynr;
+     unsigned long iova;
+
+     fsr = iommu_readl(ctx, ARM_SMMU_CB_FSR);
+
+     if (!(fsr & FSR_FAULT))
+             return IRQ_NONE;

drivers/iommu/qcom_iommu.c line 221

Quoted 10 lines

drivers/iommu/qcom_iommu.c new file +10 -0

+
+     fsynr = iommu_readl(ctx, ARM_SMMU_CB_FSYNR0);
+     iova = iommu_readq(ctx, ARM_SMMU_CB_FAR);
+
+     dev_err_ratelimited(ctx->dev,
+                         "Unhandled context fault: fsr=0x%x, "
+                         "iova=0x%08lx, fsynr=0x%x, cb=%d\n",
+                         fsr, iova, fsynr, ctx->asid);
+
+     iommu_writel(ctx, ARM_SMMU_CB_FSR, fsr);

Just thinking if the clocks should be enabled in the fault handler for handling cases that would happen out of the master context. While global faults are one case, that is anyways is handled in secure world for this case. Something like bootloader used the iommu and not handled the fault, and getting the fault in kernel the moment we enable the ctx. Atleast downstream seems to enable the clocks in the fault handler explicitly.

Regards,
Sricharan

Quoted 625 lines

drivers/iommu/qcom_iommu.c new file +624 -0

+
+     return IRQ_HANDLED;
+}
+
+static int qcom_iommu_init_domain(struct iommu_domain *domain,
+                               struct qcom_iommu_dev *qcom_iommu,
+                               struct iommu_fwspec *fwspec)
+{
+     struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+     struct io_pgtable_ops *pgtbl_ops;
+     struct io_pgtable_cfg pgtbl_cfg;
+     int i, ret = 0;
+     u32 reg;
+
+     mutex_lock(&qcom_domain->init_mutex);
+     if (qcom_domain->iommu)
+             goto out_unlock;
+
+     pgtbl_cfg = (struct io_pgtable_cfg) {
+             .pgsize_bitmap  = qcom_iommu_ops.pgsize_bitmap,
+             .ias            = 32,
+             .oas            = 40,
+             .tlb            = &qcom_gather_ops,
+             .iommu_dev      = qcom_iommu->dev,
+     };
+
+     qcom_domain->iommu = qcom_iommu;
+     pgtbl_ops = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &pgtbl_cfg, fwspec);
+     if (!pgtbl_ops) {
+             dev_err(qcom_iommu->dev, "failed to allocate pagetable ops\n");
+             ret = -ENOMEM;
+             goto out_clear_iommu;
+     }
+
+     /* Update the domain's page sizes to reflect the page table format */
+     domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
+     domain->geometry.aperture_end = (1ULL << pgtbl_cfg.ias) - 1;
+     domain->geometry.force_aperture = true;
+
+     for (i = 0; i < fwspec->num_ids; i++) {
+             struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+
+             if (!ctx->secure_init) {
+                     ret = qcom_scm_restore_sec_cfg(qcom_iommu->sec_id, ctx->asid);
+                     if (ret) {
+                             dev_err(qcom_iommu->dev, "secure init failed: %d\n", ret);
+                             goto out_clear_iommu;
+                     }
+                     ctx->secure_init = true;
+             }
+
+             /* TTBRs */
+             iommu_writeq(ctx, ARM_SMMU_CB_TTBR0,
+                             pgtbl_cfg.arm_lpae_s1_cfg.ttbr[0] |
+                             ((u64)ctx->asid << TTBRn_ASID_SHIFT));
+             iommu_writeq(ctx, ARM_SMMU_CB_TTBR1,
+                             pgtbl_cfg.arm_lpae_s1_cfg.ttbr[1] |
+                             ((u64)ctx->asid << TTBRn_ASID_SHIFT));
+
+             /* TTBCR */
+             iommu_writel(ctx, ARM_SMMU_CB_TTBCR2,
+                             (pgtbl_cfg.arm_lpae_s1_cfg.tcr >> 32) |
+                             TTBCR2_SEP_UPSTREAM);
+             iommu_writel(ctx, ARM_SMMU_CB_TTBCR,
+                             pgtbl_cfg.arm_lpae_s1_cfg.tcr);
+
+             /* MAIRs (stage-1 only) */
+             iommu_writel(ctx, ARM_SMMU_CB_S1_MAIR0,
+                             pgtbl_cfg.arm_lpae_s1_cfg.mair[0]);
+             iommu_writel(ctx, ARM_SMMU_CB_S1_MAIR1,
+                             pgtbl_cfg.arm_lpae_s1_cfg.mair[1]);
+
+             /* SCTLR */
+             reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE |
+                     SCTLR_M | SCTLR_S1_ASIDPNE;
+
+             if (IS_ENABLED(CONFIG_BIG_ENDIAN))
+                     reg |= SCTLR_E;
+
+             iommu_writel(ctx, ARM_SMMU_CB_SCTLR, reg);
+     }
+
+     mutex_unlock(&qcom_domain->init_mutex);
+
+     /* Publish page table ops for map/unmap */
+     qcom_domain->pgtbl_ops = pgtbl_ops;
+
+     return 0;
+
+out_clear_iommu:
+     qcom_domain->iommu = NULL;
+out_unlock:
+     mutex_unlock(&qcom_domain->init_mutex);
+     return ret;
+}
+
+static struct iommu_domain *qcom_iommu_domain_alloc(unsigned type)
+{
+     struct qcom_iommu_domain *qcom_domain;
+
+     if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
+             return NULL;
+     /*
+      * Allocate the domain and initialise some of its data structures.
+      * We can't really do anything meaningful until we've added a
+      * master.
+      */
+     qcom_domain = kzalloc(sizeof(*qcom_domain), GFP_KERNEL);
+     if (!qcom_domain)
+             return NULL;
+
+     if (type == IOMMU_DOMAIN_DMA &&
+         iommu_get_dma_cookie(&qcom_domain->domain)) {
+             kfree(qcom_domain);
+             return NULL;
+     }
+
+     mutex_init(&qcom_domain->init_mutex);
+     spin_lock_init(&qcom_domain->pgtbl_lock);
+
+     return &qcom_domain->domain;
+}
+
+static void qcom_iommu_domain_free(struct iommu_domain *domain)
+{
+     struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+
+     if (WARN_ON(qcom_domain->iommu))    /* forgot to detach? */
+             return;
+
+     iommu_put_dma_cookie(domain);
+
+     free_io_pgtable_ops(qcom_domain->pgtbl_ops);
+
+     kfree(qcom_domain);
+}
+
+static int qcom_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
+{
+     struct qcom_iommu_dev *qcom_iommu = to_iommu(dev->iommu_fwspec);
+     struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+     int ret;
+
+     if (!qcom_iommu) {
+             dev_err(dev, "cannot attach to IOMMU, is it on the same bus?\n");
+             return -ENXIO;
+     }
+
+     /* Ensure that the domain is finalized */
+     pm_runtime_get_sync(qcom_iommu->dev);
+     ret = qcom_iommu_init_domain(domain, qcom_iommu, dev->iommu_fwspec);
+     pm_runtime_put_sync(qcom_iommu->dev);
+     if (ret < 0)
+             return ret;
+
+     /*
+      * Sanity check the domain. We don't support domains across
+      * different IOMMUs.
+      */
+     if (qcom_domain->iommu != qcom_iommu) {
+             dev_err(dev, "cannot attach to IOMMU %s while already "
+                     "attached to domain on IOMMU %s\n",
+                     dev_name(qcom_domain->iommu->dev),
+                     dev_name(qcom_iommu->dev));
+             return -EINVAL;
+     }
+
+     return 0;
+}
+
+static void qcom_iommu_detach_dev(struct iommu_domain *domain, struct device *dev)
+{
+     struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+     struct qcom_iommu_dev *qcom_iommu = to_iommu(fwspec);
+     struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+     unsigned i;
+
+     if (!qcom_domain->iommu)
+             return;
+
+     pm_runtime_get_sync(qcom_iommu->dev);
+     for (i = 0; i < fwspec->num_ids; i++) {
+             struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+
+             /* Disable the context bank: */
+             iommu_writel(ctx, ARM_SMMU_CB_SCTLR, 0);
+     }
+     pm_runtime_put_sync(qcom_iommu->dev);
+
+     qcom_domain->iommu = NULL;
+}
+
+static int qcom_iommu_map(struct iommu_domain *domain, unsigned long iova,
+                       phys_addr_t paddr, size_t size, int prot)
+{
+     int ret;
+     unsigned long flags;
+     struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+     struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
+
+     if (!ops)
+             return -ENODEV;
+
+     spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
+     ret = ops->map(ops, iova, paddr, size, prot);
+     spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
+     return ret;
+}
+
+static size_t qcom_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
+                            size_t size)
+{
+     size_t ret;
+     unsigned long flags;
+     struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+     struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
+
+     if (!ops)
+             return 0;
+
+     spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
+     ret = ops->unmap(ops, iova, size);
+     spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
+     return ret;
+}
+
+static phys_addr_t qcom_iommu_iova_to_phys(struct iommu_domain *domain,
+                                        dma_addr_t iova)
+{
+     phys_addr_t ret;
+     unsigned long flags;
+     struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+     struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
+
+     if (!ops)
+             return 0;
+
+     spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
+     ret = ops->iova_to_phys(ops, iova);
+     spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
+
+     return ret;
+}
+
+static bool qcom_iommu_capable(enum iommu_cap cap)
+{
+     switch (cap) {
+     case IOMMU_CAP_CACHE_COHERENCY:
+             /*
+              * Return true here as the SMMU can always send out coherent
+              * requests.
+              */
+             return true;
+     case IOMMU_CAP_NOEXEC:
+             return true;
+     default:
+             return false;
+     }
+}
+
+static int qcom_iommu_add_device(struct device *dev)
+{
+     struct qcom_iommu_dev *qcom_iommu = __to_iommu(dev->iommu_fwspec);
+     struct iommu_group *group;
+     struct device_link *link;
+
+     if (!qcom_iommu)
+             return -ENODEV;
+
+     /*
+      * Establish the link between iommu and master, so that the
+      * iommu gets runtime enabled/disabled as per the master's
+      * needs.
+      */
+     link = device_link_add(dev, qcom_iommu->dev, DL_FLAG_PM_RUNTIME);
+     if (!link) {
+             dev_err(qcom_iommu->dev, "Unable to create device link between %s and %s\n",
+                     dev_name(qcom_iommu->dev), dev_name(dev));
+             return -ENODEV;
+     }
+
+     group = iommu_group_get_for_dev(dev);
+     if (IS_ERR_OR_NULL(group))
+             return PTR_ERR_OR_ZERO(group);
+
+     iommu_group_put(group);
+     iommu_device_link(&qcom_iommu->iommu, dev);
+
+     return 0;
+}
+
+static void qcom_iommu_remove_device(struct device *dev)
+{
+     struct qcom_iommu_dev *qcom_iommu = to_iommu(dev->iommu_fwspec);
+
+     if (!qcom_iommu)
+             return;
+
+     iommu_device_unlink(&qcom_iommu->iommu, dev);
+     iommu_group_remove_device(dev);
+     iommu_fwspec_free(dev);
+}
+
+static struct iommu_group *qcom_iommu_device_group(struct device *dev)
+{
+     struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+     struct iommu_group *group = NULL;
+     unsigned i;
+
+     for (i = 0; i < fwspec->num_ids; i++) {
+             struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+
+             if (group && ctx->group && group != ctx->group)
+                     return ERR_PTR(-EINVAL);
+
+             group = ctx->group;
+     }
+
+     if (group)
+             return iommu_group_ref_get(group);
+
+     group = generic_device_group(dev);
+
+     for (i = 0; i < fwspec->num_ids; i++) {
+             struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+             ctx->group = iommu_group_ref_get(group);
+     }
+
+     return group;
+}
+
+static int qcom_iommu_of_xlate(struct device *dev, struct of_phandle_args *args)
+{
+     struct platform_device *iommu_pdev;
+
+     if (args->args_count != 1) {
+             dev_err(dev, "incorrect number of iommu params found for %s "
+                     "(found %d, expected 1)\n",
+                     args->np->full_name, args->args_count);
+             return -EINVAL;
+     }
+
+     if (!dev->iommu_fwspec->iommu_priv) {
+             iommu_pdev = of_find_device_by_node(args->np);
+             if (WARN_ON(!iommu_pdev))
+                     return -EINVAL;
+
+             dev->iommu_fwspec->iommu_priv = platform_get_drvdata(iommu_pdev);
+     }
+
+     return iommu_fwspec_add_ids(dev, &args->args[0], 1);
+}
+
+static const struct iommu_ops qcom_iommu_ops = {
+     .capable        = qcom_iommu_capable,
+     .domain_alloc   = qcom_iommu_domain_alloc,
+     .domain_free    = qcom_iommu_domain_free,
+     .attach_dev     = qcom_iommu_attach_dev,
+     .detach_dev     = qcom_iommu_detach_dev,
+     .map            = qcom_iommu_map,
+     .unmap          = qcom_iommu_unmap,
+     .map_sg         = default_iommu_map_sg,
+     .iova_to_phys   = qcom_iommu_iova_to_phys,
+     .add_device     = qcom_iommu_add_device,
+     .remove_device  = qcom_iommu_remove_device,
+     .device_group   = qcom_iommu_device_group,
+     .of_xlate       = qcom_iommu_of_xlate,
+     .pgsize_bitmap  = SZ_4K | SZ_64K | SZ_1M | SZ_16M,
+};
+
+static int qcom_iommu_enable_clocks(struct qcom_iommu_dev *qcom_iommu)
+{
+     int ret;
+
+     ret = clk_prepare_enable(qcom_iommu->iface_clk);
+     if (ret) {
+             dev_err(qcom_iommu->dev, "Couldn't enable iface_clk\n");
+             return ret;
+     }
+
+     ret = clk_prepare_enable(qcom_iommu->bus_clk);
+     if (ret) {
+             dev_err(qcom_iommu->dev, "Couldn't enable bus_clk\n");
+             clk_disable_unprepare(qcom_iommu->iface_clk);
+             return ret;
+     }
+
+     return 0;
+}
+
+static void qcom_iommu_disable_clocks(struct qcom_iommu_dev *qcom_iommu)
+{
+     clk_disable_unprepare(qcom_iommu->bus_clk);
+     clk_disable_unprepare(qcom_iommu->iface_clk);
+}
+
+static int qcom_iommu_ctx_probe(struct platform_device *pdev)
+{
+     struct qcom_iommu_ctx *ctx;
+     struct device *dev = &pdev->dev;
+     struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(dev->parent);
+     struct resource *res;
+     int ret;
+     u32 reg;
+
+     ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
+     if (!ctx)
+             return -ENOMEM;
+
+     ctx->dev = dev;
+     platform_set_drvdata(pdev, ctx);
+
+     res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+     ctx->base = devm_ioremap_resource(dev, res);
+     if (IS_ERR(ctx->base))
+             return PTR_ERR(ctx->base);
+
+     ctx->irq = platform_get_irq(pdev, 0);
+     if (ctx->irq < 0) {
+             dev_err(dev, "failed to get irq\n");
+             return -ENODEV;
+     }
+
+     ret = devm_request_irq(dev, ctx->irq,
+                            qcom_iommu_fault,
+                            IRQF_SHARED,
+                            "qcom-iommu-fault",
+                            ctx);
+     if (ret) {
+             dev_err(dev, "failed to request IRQ %u\n", ctx->irq);
+             return ret;
+     }
+
+     /* read the "reg" property directly to get the relative address
+      * of the context bank, and calculate the asid from that:
+      */
+     if (of_property_read_u32_index(dev->of_node, "reg", 0, &reg)) {
+             dev_err(dev, "missing reg property\n");
+             return -ENODEV;
+     }
+
+     ctx->asid = reg / 0x1000;      /* context banks are 0x1000 apart */
+
+     dev_dbg(dev, "found asid %u\n", ctx->asid);
+
+     list_add_tail(&ctx->node, &qcom_iommu->context_list);
+
+     return 0;
+}
+
+static int qcom_iommu_ctx_remove(struct platform_device *pdev)
+{
+     struct qcom_iommu_ctx *ctx = platform_get_drvdata(pdev);
+
+     iommu_group_put(ctx->group);
+     platform_set_drvdata(pdev, NULL);
+
+     list_del(&ctx->node);
+
+     return 0;
+}
+
+static const struct of_device_id ctx_of_match[] = {
+     { .compatible = "qcom,msm-iommu-v1-ns" },
+     { .compatible = "qcom,msm-iommu-v1-sec" },
+     { /* sentinel */ }
+};
+
+static struct platform_driver qcom_iommu_ctx_driver = {
+     .driver = {
+             .name           = "qcom-iommu-ctx",
+             .of_match_table = of_match_ptr(ctx_of_match),
+     },
+     .probe  = qcom_iommu_ctx_probe,
+     .remove = qcom_iommu_ctx_remove,
+};
+
+static int qcom_iommu_device_probe(struct platform_device *pdev)
+{
+     struct qcom_iommu_dev *qcom_iommu;
+     struct device *dev = &pdev->dev;
+     struct resource *res;
+     int ret;
+
+     qcom_iommu = devm_kzalloc(dev, sizeof(*qcom_iommu), GFP_KERNEL);
+     if (!qcom_iommu)
+             return -ENOMEM;
+     qcom_iommu->dev = dev;
+
+     INIT_LIST_HEAD(&qcom_iommu->context_list);
+
+     res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+     if (res)
+             qcom_iommu->local_base = devm_ioremap_resource(dev, res);
+
+     qcom_iommu->iface_clk = devm_clk_get(dev, "iface");
+     if (IS_ERR(qcom_iommu->iface_clk)) {
+             dev_err(dev, "failed to get iface clock\n");
+             return PTR_ERR(qcom_iommu->iface_clk);
+     }
+
+     qcom_iommu->bus_clk = devm_clk_get(dev, "bus");
+     if (IS_ERR(qcom_iommu->bus_clk)) {
+             dev_err(dev, "failed to get bus clock\n");
+             return PTR_ERR(qcom_iommu->bus_clk);
+     }
+
+     if (of_property_read_u32(dev->of_node, "qcom,iommu-secure-id",
+                              &qcom_iommu->sec_id)) {
+             dev_err(dev, "missing qcom,iommu-secure-id property\n");
+             return -ENODEV;
+     }
+
+     platform_set_drvdata(pdev, qcom_iommu);
+
+     /* register context bank devices, which are child nodes: */
+     ret = devm_of_platform_populate(dev);
+     if (ret) {
+             dev_err(dev, "Failed to populate iommu contexts\n");
+             return ret;
+     }
+
+     ret = iommu_device_sysfs_add(&qcom_iommu->iommu, dev, NULL,
+                                  dev_name(dev));
+     if (ret) {
+             dev_err(dev, "Failed to register iommu in sysfs\n");
+             return ret;
+     }
+
+     iommu_device_set_ops(&qcom_iommu->iommu, &qcom_iommu_ops);
+     iommu_device_set_fwnode(&qcom_iommu->iommu, dev->fwnode);
+
+     ret = iommu_device_register(&qcom_iommu->iommu);
+     if (ret) {
+             dev_err(dev, "Failed to register iommu\n");
+             return ret;
+     }
+
+     pm_runtime_enable(dev);
+     bus_set_iommu(&platform_bus_type, &qcom_iommu_ops);
+
+     if (qcom_iommu->local_base) {
+             pm_runtime_get_sync(dev);
+             writel_relaxed(0xffffffff, qcom_iommu->local_base + SMMU_INTR_SEL_NS);
+             pm_runtime_put_sync(dev);
+     }
+
+     return 0;
+}
+
+static int qcom_iommu_device_remove(struct platform_device *pdev)
+{
+     struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
+
+     bus_set_iommu(&platform_bus_type, NULL);
+
+     pm_runtime_force_suspend(&pdev->dev);
+     platform_set_drvdata(pdev, NULL);
+     iommu_device_sysfs_remove(&qcom_iommu->iommu);
+     iommu_device_unregister(&qcom_iommu->iommu);
+
+     return 0;
+}
+
+#ifdef CONFIG_PM
+static int qcom_iommu_resume(struct device *dev)
+{
+     struct platform_device *pdev = to_platform_device(dev);
+     struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
+
+     return qcom_iommu_enable_clocks(qcom_iommu);
+}
+
+static int qcom_iommu_suspend(struct device *dev)
+{
+     struct platform_device *pdev = to_platform_device(dev);
+     struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
+
+     qcom_iommu_disable_clocks(qcom_iommu);
+
+     return 0;
+}
+#endif
+
+static const struct dev_pm_ops qcom_iommu_pm_ops = {
+     SET_RUNTIME_PM_OPS(qcom_iommu_suspend, qcom_iommu_resume, NULL)
+     SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+                             pm_runtime_force_resume)
+};
+
+static const struct of_device_id qcom_iommu_of_match[] = {
+     { .compatible = "qcom,msm-iommu-v1" },
+     { /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, qcom_iommu_of_match);
+
+static struct platform_driver qcom_iommu_driver = {
+     .driver = {
+             .name           = "qcom-iommu",
+             .of_match_table = of_match_ptr(qcom_iommu_of_match),
+             .pm             = &qcom_iommu_pm_ops,
+     },
+     .probe  = qcom_iommu_device_probe,
+     .remove = qcom_iommu_device_remove,
+};
+
+static int __init qcom_iommu_init(void)
+{
+     int ret;
+
+     ret = platform_driver_register(&qcom_iommu_ctx_driver);
+     if (ret)
+             return ret;
+
+     ret = platform_driver_register(&qcom_iommu_driver);
+     if (ret)
+             platform_driver_unregister(&qcom_iommu_ctx_driver);
+
+     return ret;
+}
+
+static void __exit qcom_iommu_exit(void)
+{
+     platform_driver_unregister(&qcom_iommu_driver);

drivers/iommu/qcom_iommu.c line 855

Quoted 10 lines

drivers/iommu/qcom_iommu.c new file +10 -0

+     platform_driver_unregister(&qcom_iommu_ctx_driver);
+}
+
+module_init(qcom_iommu_init);
+module_exit(qcom_iommu_exit);
+
+IOMMU_OF_DECLARE(qcom_iommu_dev, "qcom,msm-iommu-v1", NULL);
+
+MODULE_DESCRIPTION("IOMMU API for QCOM IOMMU v1 implementations");
+MODULE_LICENSE("GPL v2");

--
"QUALCOMM INDIA, on behalf of Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum, hosted by The Linux Foundation

Quoted 25 lines · Sricharan R

Hi Rob,

<snip..>

Quoted 10 lines

drivers/iommu/qcom_iommu.c new file +10 -0

+static irqreturn_t qcom_iommu_fault(int irq, void *dev)
+{
+     struct qcom_iommu_ctx *ctx = dev;
+     u32 fsr, fsynr;
+     unsigned long iova;
+
+     fsr = iommu_readl(ctx, ARM_SMMU_CB_FSR);
+
+     if (!(fsr & FSR_FAULT))
+             return IRQ_NONE;

drivers/iommu/qcom_iommu.c line 221

Quoted 10 lines

drivers/iommu/qcom_iommu.c new file +10 -0

+
+     fsynr = iommu_readl(ctx, ARM_SMMU_CB_FSYNR0);
+     iova = iommu_readq(ctx, ARM_SMMU_CB_FAR);
+
+     dev_err_ratelimited(ctx->dev,
+                         "Unhandled context fault: fsr=0x%x, "
+                         "iova=0x%08lx, fsynr=0x%x, cb=%d\n",
+                         fsr, iova, fsynr, ctx->asid);
+
+     iommu_writel(ctx, ARM_SMMU_CB_FSR, fsr);

Quoted 7 lines

Just thinking if the clocks should be enabled in the fault handler for handling cases that would happen out of the master context. While global faults are one case, that is anyways is handled in secure world for this case. Something like bootloader used the iommu and not handled the fault, and getting the fault in kernel the moment we enable the ctx. Atleast downstream seems to enable the clocks in the fault handler explicitly.

hmm, I wonder if we should instead do something to clear interrupts when we initialize the context?

I guess we probably don't want to get fault irq's from the bootloader..

BR,
-R

Quoted 642 lines

Regards,
Sricharan

Quoted 625 lines

drivers/iommu/qcom_iommu.c new file +624 -0

+
+     return IRQ_HANDLED;
+}
+
+static int qcom_iommu_init_domain(struct iommu_domain *domain,
+                               struct qcom_iommu_dev *qcom_iommu,
+                               struct iommu_fwspec *fwspec)
+{
+     struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+     struct io_pgtable_ops *pgtbl_ops;
+     struct io_pgtable_cfg pgtbl_cfg;
+     int i, ret = 0;
+     u32 reg;
+
+     mutex_lock(&qcom_domain->init_mutex);
+     if (qcom_domain->iommu)
+             goto out_unlock;
+
+     pgtbl_cfg = (struct io_pgtable_cfg) {
+             .pgsize_bitmap  = qcom_iommu_ops.pgsize_bitmap,
+             .ias            = 32,
+             .oas            = 40,
+             .tlb            = &qcom_gather_ops,
+             .iommu_dev      = qcom_iommu->dev,
+     };
+
+     qcom_domain->iommu = qcom_iommu;
+     pgtbl_ops = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &pgtbl_cfg, fwspec);
+     if (!pgtbl_ops) {
+             dev_err(qcom_iommu->dev, "failed to allocate pagetable ops\n");
+             ret = -ENOMEM;
+             goto out_clear_iommu;
+     }
+
+     /* Update the domain's page sizes to reflect the page table format */
+     domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
+     domain->geometry.aperture_end = (1ULL << pgtbl_cfg.ias) - 1;
+     domain->geometry.force_aperture = true;
+
+     for (i = 0; i < fwspec->num_ids; i++) {
+             struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+
+             if (!ctx->secure_init) {
+                     ret = qcom_scm_restore_sec_cfg(qcom_iommu->sec_id, ctx->asid);
+                     if (ret) {
+                             dev_err(qcom_iommu->dev, "secure init failed: %d\n", ret);
+                             goto out_clear_iommu;
+                     }
+                     ctx->secure_init = true;
+             }
+
+             /* TTBRs */
+             iommu_writeq(ctx, ARM_SMMU_CB_TTBR0,
+                             pgtbl_cfg.arm_lpae_s1_cfg.ttbr[0] |
+                             ((u64)ctx->asid << TTBRn_ASID_SHIFT));
+             iommu_writeq(ctx, ARM_SMMU_CB_TTBR1,
+                             pgtbl_cfg.arm_lpae_s1_cfg.ttbr[1] |
+                             ((u64)ctx->asid << TTBRn_ASID_SHIFT));
+
+             /* TTBCR */
+             iommu_writel(ctx, ARM_SMMU_CB_TTBCR2,
+                             (pgtbl_cfg.arm_lpae_s1_cfg.tcr >> 32) |
+                             TTBCR2_SEP_UPSTREAM);
+             iommu_writel(ctx, ARM_SMMU_CB_TTBCR,
+                             pgtbl_cfg.arm_lpae_s1_cfg.tcr);
+
+             /* MAIRs (stage-1 only) */
+             iommu_writel(ctx, ARM_SMMU_CB_S1_MAIR0,
+                             pgtbl_cfg.arm_lpae_s1_cfg.mair[0]);
+             iommu_writel(ctx, ARM_SMMU_CB_S1_MAIR1,
+                             pgtbl_cfg.arm_lpae_s1_cfg.mair[1]);
+
+             /* SCTLR */
+             reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE |
+                     SCTLR_M | SCTLR_S1_ASIDPNE;
+
+             if (IS_ENABLED(CONFIG_BIG_ENDIAN))
+                     reg |= SCTLR_E;
+
+             iommu_writel(ctx, ARM_SMMU_CB_SCTLR, reg);
+     }
+
+     mutex_unlock(&qcom_domain->init_mutex);
+
+     /* Publish page table ops for map/unmap */
+     qcom_domain->pgtbl_ops = pgtbl_ops;
+
+     return 0;
+
+out_clear_iommu:
+     qcom_domain->iommu = NULL;
+out_unlock:
+     mutex_unlock(&qcom_domain->init_mutex);
+     return ret;
+}
+
+static struct iommu_domain *qcom_iommu_domain_alloc(unsigned type)
+{
+     struct qcom_iommu_domain *qcom_domain;
+
+     if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
+             return NULL;
+     /*
+      * Allocate the domain and initialise some of its data structures.
+      * We can't really do anything meaningful until we've added a
+      * master.
+      */
+     qcom_domain = kzalloc(sizeof(*qcom_domain), GFP_KERNEL);
+     if (!qcom_domain)
+             return NULL;
+
+     if (type == IOMMU_DOMAIN_DMA &&
+         iommu_get_dma_cookie(&qcom_domain->domain)) {
+             kfree(qcom_domain);
+             return NULL;
+     }
+
+     mutex_init(&qcom_domain->init_mutex);
+     spin_lock_init(&qcom_domain->pgtbl_lock);
+
+     return &qcom_domain->domain;
+}
+
+static void qcom_iommu_domain_free(struct iommu_domain *domain)
+{
+     struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+
+     if (WARN_ON(qcom_domain->iommu))    /* forgot to detach? */
+             return;
+
+     iommu_put_dma_cookie(domain);
+
+     free_io_pgtable_ops(qcom_domain->pgtbl_ops);
+
+     kfree(qcom_domain);
+}
+
+static int qcom_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
+{
+     struct qcom_iommu_dev *qcom_iommu = to_iommu(dev->iommu_fwspec);
+     struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+     int ret;
+
+     if (!qcom_iommu) {
+             dev_err(dev, "cannot attach to IOMMU, is it on the same bus?\n");
+             return -ENXIO;
+     }
+
+     /* Ensure that the domain is finalized */
+     pm_runtime_get_sync(qcom_iommu->dev);
+     ret = qcom_iommu_init_domain(domain, qcom_iommu, dev->iommu_fwspec);
+     pm_runtime_put_sync(qcom_iommu->dev);
+     if (ret < 0)
+             return ret;
+
+     /*
+      * Sanity check the domain. We don't support domains across
+      * different IOMMUs.
+      */
+     if (qcom_domain->iommu != qcom_iommu) {
+             dev_err(dev, "cannot attach to IOMMU %s while already "
+                     "attached to domain on IOMMU %s\n",
+                     dev_name(qcom_domain->iommu->dev),
+                     dev_name(qcom_iommu->dev));
+             return -EINVAL;
+     }
+
+     return 0;
+}
+
+static void qcom_iommu_detach_dev(struct iommu_domain *domain, struct device *dev)
+{
+     struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+     struct qcom_iommu_dev *qcom_iommu = to_iommu(fwspec);
+     struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+     unsigned i;
+
+     if (!qcom_domain->iommu)
+             return;
+
+     pm_runtime_get_sync(qcom_iommu->dev);
+     for (i = 0; i < fwspec->num_ids; i++) {
+             struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+
+             /* Disable the context bank: */
+             iommu_writel(ctx, ARM_SMMU_CB_SCTLR, 0);
+     }
+     pm_runtime_put_sync(qcom_iommu->dev);
+
+     qcom_domain->iommu = NULL;
+}
+
+static int qcom_iommu_map(struct iommu_domain *domain, unsigned long iova,
+                       phys_addr_t paddr, size_t size, int prot)
+{
+     int ret;
+     unsigned long flags;
+     struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+     struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
+
+     if (!ops)
+             return -ENODEV;
+
+     spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
+     ret = ops->map(ops, iova, paddr, size, prot);
+     spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
+     return ret;
+}
+
+static size_t qcom_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
+                            size_t size)
+{
+     size_t ret;
+     unsigned long flags;
+     struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+     struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
+
+     if (!ops)
+             return 0;
+
+     spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
+     ret = ops->unmap(ops, iova, size);
+     spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
+     return ret;
+}
+
+static phys_addr_t qcom_iommu_iova_to_phys(struct iommu_domain *domain,
+                                        dma_addr_t iova)
+{
+     phys_addr_t ret;
+     unsigned long flags;
+     struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+     struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
+
+     if (!ops)
+             return 0;
+
+     spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
+     ret = ops->iova_to_phys(ops, iova);
+     spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
+
+     return ret;
+}
+
+static bool qcom_iommu_capable(enum iommu_cap cap)
+{
+     switch (cap) {
+     case IOMMU_CAP_CACHE_COHERENCY:
+             /*
+              * Return true here as the SMMU can always send out coherent
+              * requests.
+              */
+             return true;
+     case IOMMU_CAP_NOEXEC:
+             return true;
+     default:
+             return false;
+     }
+}
+
+static int qcom_iommu_add_device(struct device *dev)
+{
+     struct qcom_iommu_dev *qcom_iommu = __to_iommu(dev->iommu_fwspec);
+     struct iommu_group *group;
+     struct device_link *link;
+
+     if (!qcom_iommu)
+             return -ENODEV;
+
+     /*
+      * Establish the link between iommu and master, so that the
+      * iommu gets runtime enabled/disabled as per the master's
+      * needs.
+      */
+     link = device_link_add(dev, qcom_iommu->dev, DL_FLAG_PM_RUNTIME);
+     if (!link) {
+             dev_err(qcom_iommu->dev, "Unable to create device link between %s and %s\n",
+                     dev_name(qcom_iommu->dev), dev_name(dev));
+             return -ENODEV;
+     }
+
+     group = iommu_group_get_for_dev(dev);
+     if (IS_ERR_OR_NULL(group))
+             return PTR_ERR_OR_ZERO(group);
+
+     iommu_group_put(group);
+     iommu_device_link(&qcom_iommu->iommu, dev);
+
+     return 0;
+}
+
+static void qcom_iommu_remove_device(struct device *dev)
+{
+     struct qcom_iommu_dev *qcom_iommu = to_iommu(dev->iommu_fwspec);
+
+     if (!qcom_iommu)
+             return;
+
+     iommu_device_unlink(&qcom_iommu->iommu, dev);
+     iommu_group_remove_device(dev);
+     iommu_fwspec_free(dev);
+}
+
+static struct iommu_group *qcom_iommu_device_group(struct device *dev)
+{
+     struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+     struct iommu_group *group = NULL;
+     unsigned i;
+
+     for (i = 0; i < fwspec->num_ids; i++) {
+             struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+
+             if (group && ctx->group && group != ctx->group)
+                     return ERR_PTR(-EINVAL);
+
+             group = ctx->group;
+     }
+
+     if (group)
+             return iommu_group_ref_get(group);
+
+     group = generic_device_group(dev);
+
+     for (i = 0; i < fwspec->num_ids; i++) {
+             struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+             ctx->group = iommu_group_ref_get(group);
+     }
+
+     return group;
+}
+
+static int qcom_iommu_of_xlate(struct device *dev, struct of_phandle_args *args)
+{
+     struct platform_device *iommu_pdev;
+
+     if (args->args_count != 1) {
+             dev_err(dev, "incorrect number of iommu params found for %s "
+                     "(found %d, expected 1)\n",
+                     args->np->full_name, args->args_count);
+             return -EINVAL;
+     }
+
+     if (!dev->iommu_fwspec->iommu_priv) {
+             iommu_pdev = of_find_device_by_node(args->np);
+             if (WARN_ON(!iommu_pdev))
+                     return -EINVAL;
+
+             dev->iommu_fwspec->iommu_priv = platform_get_drvdata(iommu_pdev);
+     }
+
+     return iommu_fwspec_add_ids(dev, &args->args[0], 1);
+}
+
+static const struct iommu_ops qcom_iommu_ops = {
+     .capable        = qcom_iommu_capable,
+     .domain_alloc   = qcom_iommu_domain_alloc,
+     .domain_free    = qcom_iommu_domain_free,
+     .attach_dev     = qcom_iommu_attach_dev,
+     .detach_dev     = qcom_iommu_detach_dev,
+     .map            = qcom_iommu_map,
+     .unmap          = qcom_iommu_unmap,
+     .map_sg         = default_iommu_map_sg,
+     .iova_to_phys   = qcom_iommu_iova_to_phys,
+     .add_device     = qcom_iommu_add_device,
+     .remove_device  = qcom_iommu_remove_device,
+     .device_group   = qcom_iommu_device_group,
+     .of_xlate       = qcom_iommu_of_xlate,
+     .pgsize_bitmap  = SZ_4K | SZ_64K | SZ_1M | SZ_16M,
+};
+
+static int qcom_iommu_enable_clocks(struct qcom_iommu_dev *qcom_iommu)
+{
+     int ret;
+
+     ret = clk_prepare_enable(qcom_iommu->iface_clk);
+     if (ret) {
+             dev_err(qcom_iommu->dev, "Couldn't enable iface_clk\n");
+             return ret;
+     }
+
+     ret = clk_prepare_enable(qcom_iommu->bus_clk);
+     if (ret) {
+             dev_err(qcom_iommu->dev, "Couldn't enable bus_clk\n");
+             clk_disable_unprepare(qcom_iommu->iface_clk);
+             return ret;
+     }
+
+     return 0;
+}
+
+static void qcom_iommu_disable_clocks(struct qcom_iommu_dev *qcom_iommu)
+{
+     clk_disable_unprepare(qcom_iommu->bus_clk);
+     clk_disable_unprepare(qcom_iommu->iface_clk);
+}
+
+static int qcom_iommu_ctx_probe(struct platform_device *pdev)
+{
+     struct qcom_iommu_ctx *ctx;
+     struct device *dev = &pdev->dev;
+     struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(dev->parent);
+     struct resource *res;
+     int ret;
+     u32 reg;
+
+     ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
+     if (!ctx)
+             return -ENOMEM;
+
+     ctx->dev = dev;
+     platform_set_drvdata(pdev, ctx);
+
+     res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+     ctx->base = devm_ioremap_resource(dev, res);
+     if (IS_ERR(ctx->base))
+             return PTR_ERR(ctx->base);
+
+     ctx->irq = platform_get_irq(pdev, 0);
+     if (ctx->irq < 0) {
+             dev_err(dev, "failed to get irq\n");
+             return -ENODEV;
+     }
+
+     ret = devm_request_irq(dev, ctx->irq,
+                            qcom_iommu_fault,
+                            IRQF_SHARED,
+                            "qcom-iommu-fault",
+                            ctx);
+     if (ret) {
+             dev_err(dev, "failed to request IRQ %u\n", ctx->irq);
+             return ret;
+     }
+
+     /* read the "reg" property directly to get the relative address
+      * of the context bank, and calculate the asid from that:
+      */
+     if (of_property_read_u32_index(dev->of_node, "reg", 0, &reg)) {
+             dev_err(dev, "missing reg property\n");
+             return -ENODEV;
+     }
+
+     ctx->asid = reg / 0x1000;      /* context banks are 0x1000 apart */
+
+     dev_dbg(dev, "found asid %u\n", ctx->asid);
+
+     list_add_tail(&ctx->node, &qcom_iommu->context_list);
+
+     return 0;
+}
+
+static int qcom_iommu_ctx_remove(struct platform_device *pdev)
+{
+     struct qcom_iommu_ctx *ctx = platform_get_drvdata(pdev);
+
+     iommu_group_put(ctx->group);
+     platform_set_drvdata(pdev, NULL);
+
+     list_del(&ctx->node);
+
+     return 0;
+}
+
+static const struct of_device_id ctx_of_match[] = {
+     { .compatible = "qcom,msm-iommu-v1-ns" },
+     { .compatible = "qcom,msm-iommu-v1-sec" },
+     { /* sentinel */ }
+};
+
+static struct platform_driver qcom_iommu_ctx_driver = {
+     .driver = {
+             .name           = "qcom-iommu-ctx",
+             .of_match_table = of_match_ptr(ctx_of_match),
+     },
+     .probe  = qcom_iommu_ctx_probe,
+     .remove = qcom_iommu_ctx_remove,
+};
+
+static int qcom_iommu_device_probe(struct platform_device *pdev)
+{
+     struct qcom_iommu_dev *qcom_iommu;
+     struct device *dev = &pdev->dev;
+     struct resource *res;
+     int ret;
+
+     qcom_iommu = devm_kzalloc(dev, sizeof(*qcom_iommu), GFP_KERNEL);
+     if (!qcom_iommu)
+             return -ENOMEM;
+     qcom_iommu->dev = dev;
+
+     INIT_LIST_HEAD(&qcom_iommu->context_list);
+
+     res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+     if (res)
+             qcom_iommu->local_base = devm_ioremap_resource(dev, res);
+
+     qcom_iommu->iface_clk = devm_clk_get(dev, "iface");
+     if (IS_ERR(qcom_iommu->iface_clk)) {
+             dev_err(dev, "failed to get iface clock\n");
+             return PTR_ERR(qcom_iommu->iface_clk);
+     }
+
+     qcom_iommu->bus_clk = devm_clk_get(dev, "bus");
+     if (IS_ERR(qcom_iommu->bus_clk)) {
+             dev_err(dev, "failed to get bus clock\n");
+             return PTR_ERR(qcom_iommu->bus_clk);
+     }
+
+     if (of_property_read_u32(dev->of_node, "qcom,iommu-secure-id",
+                              &qcom_iommu->sec_id)) {
+             dev_err(dev, "missing qcom,iommu-secure-id property\n");
+             return -ENODEV;
+     }
+
+     platform_set_drvdata(pdev, qcom_iommu);
+
+     /* register context bank devices, which are child nodes: */
+     ret = devm_of_platform_populate(dev);
+     if (ret) {
+             dev_err(dev, "Failed to populate iommu contexts\n");
+             return ret;
+     }
+
+     ret = iommu_device_sysfs_add(&qcom_iommu->iommu, dev, NULL,
+                                  dev_name(dev));
+     if (ret) {
+             dev_err(dev, "Failed to register iommu in sysfs\n");
+             return ret;
+     }
+
+     iommu_device_set_ops(&qcom_iommu->iommu, &qcom_iommu_ops);
+     iommu_device_set_fwnode(&qcom_iommu->iommu, dev->fwnode);
+
+     ret = iommu_device_register(&qcom_iommu->iommu);
+     if (ret) {
+             dev_err(dev, "Failed to register iommu\n");
+             return ret;
+     }
+
+     pm_runtime_enable(dev);
+     bus_set_iommu(&platform_bus_type, &qcom_iommu_ops);
+
+     if (qcom_iommu->local_base) {
+             pm_runtime_get_sync(dev);
+             writel_relaxed(0xffffffff, qcom_iommu->local_base + SMMU_INTR_SEL_NS);
+             pm_runtime_put_sync(dev);
+     }
+
+     return 0;
+}
+
+static int qcom_iommu_device_remove(struct platform_device *pdev)
+{
+     struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
+
+     bus_set_iommu(&platform_bus_type, NULL);
+
+     pm_runtime_force_suspend(&pdev->dev);
+     platform_set_drvdata(pdev, NULL);
+     iommu_device_sysfs_remove(&qcom_iommu->iommu);
+     iommu_device_unregister(&qcom_iommu->iommu);
+
+     return 0;
+}
+
+#ifdef CONFIG_PM
+static int qcom_iommu_resume(struct device *dev)
+{
+     struct platform_device *pdev = to_platform_device(dev);
+     struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
+
+     return qcom_iommu_enable_clocks(qcom_iommu);
+}
+
+static int qcom_iommu_suspend(struct device *dev)
+{
+     struct platform_device *pdev = to_platform_device(dev);
+     struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
+
+     qcom_iommu_disable_clocks(qcom_iommu);
+
+     return 0;
+}
+#endif
+
+static const struct dev_pm_ops qcom_iommu_pm_ops = {
+     SET_RUNTIME_PM_OPS(qcom_iommu_suspend, qcom_iommu_resume, NULL)
+     SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+                             pm_runtime_force_resume)
+};
+
+static const struct of_device_id qcom_iommu_of_match[] = {
+     { .compatible = "qcom,msm-iommu-v1" },
+     { /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, qcom_iommu_of_match);
+
+static struct platform_driver qcom_iommu_driver = {
+     .driver = {
+             .name           = "qcom-iommu",
+             .of_match_table = of_match_ptr(qcom_iommu_of_match),
+             .pm             = &qcom_iommu_pm_ops,
+     },
+     .probe  = qcom_iommu_device_probe,
+     .remove = qcom_iommu_device_remove,
+};
+
+static int __init qcom_iommu_init(void)
+{
+     int ret;
+
+     ret = platform_driver_register(&qcom_iommu_ctx_driver);
+     if (ret)
+             return ret;
+
+     ret = platform_driver_register(&qcom_iommu_driver);
+     if (ret)
+             platform_driver_unregister(&qcom_iommu_ctx_driver);
+
+     return ret;
+}
+
+static void __exit qcom_iommu_exit(void)
+{
+     platform_driver_unregister(&qcom_iommu_driver);

drivers/iommu/qcom_iommu.c line 855

Quoted 10 lines

drivers/iommu/qcom_iommu.c new file +10 -0

+     platform_driver_unregister(&qcom_iommu_ctx_driver);
+}
+
+module_init(qcom_iommu_init);
+module_exit(qcom_iommu_exit);
+
+IOMMU_OF_DECLARE(qcom_iommu_dev, "qcom,msm-iommu-v1", NULL);
+
+MODULE_DESCRIPTION("IOMMU API for QCOM IOMMU v1 implementations");
+MODULE_LICENSE("GPL v2");

--
"QUALCOMM INDIA, on behalf of Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum, hosted by The Linux Foundation