DMA-API/PCI map_peer_resource support for peer-to-peer

index 5f4a2e0..6c2c82a 100644
@@ -8,25 +8,10 @@
 
 #include "pci.h"
 
-static struct pci_bus *find_pci_root_bus(struct pci_bus *bus)
-{
-	while (bus->parent)
-		bus = bus->parent;
-
-	return bus;
-}
-
-struct pci_host_bridge *pci_find_host_bridge(struct pci_bus *bus)
-{
-	struct pci_bus *root_bus = find_pci_root_bus(bus);
-
-	return to_pci_host_bridge(root_bus->bridge);
-}
-
 struct device *pci_get_host_bridge_device(struct pci_dev *dev)
 {
-	struct pci_bus *root_bus = find_pci_root_bus(dev->bus);
-	struct device *bridge = root_bus->bridge;
+	struct pci_host_bridge *host_bridge = pci_find_host_bridge(dev->bus);
+	struct device *bridge = &host_bridge->dev;
 
 	kobject_get(&bridge->kobj);
 	return bridge;

index 8a0321a..d58063e 100644
@@ -512,6 +512,13 @@ static inline struct pci_dev *pci_upstream_bridge(struct pci_dev *dev)
 	return dev->bus->self;
 }
 
+static inline struct pci_host_bridge *pci_find_host_bridge(struct pci_bus *bus)
+{
+	while (bus->parent)
+		bus = bus->parent;
+	return to_pci_host_bridge(bus->bridge);
+}
+
 struct device *pci_get_host_bridge_device(struct pci_dev *dev);
 void pci_put_host_bridge_device(struct device *dev);
 

index 0008c95..b8ba0f0 100644
@@ -25,6 +25,7 @@
 #include <linux/device.h>
 #include <linux/pm_runtime.h>
 #include <linux/pci_hotplug.h>
+#include <linux/iommu.h>
 #include <asm-generic/pci-bridge.h>
 #include <asm/setup.h>
 #include "pci.h"

@@ -4302,6 +4303,104 @@ void pci_ignore_hotplug(struct pci_dev *dev)
 }
 EXPORT_SYMBOL_GPL(pci_ignore_hotplug);
 
+bool pci_peer_traffic_supported(struct pci_dev *dev, struct pci_dev *peer)
+{
+	struct pci_host_bridge *dev_host_bridge;
+	struct pci_host_bridge *peer_host_bridge;
+
+	/*
+	 * Disallow the peer-to-peer traffic if the devices do not share a
+	 * host bridge. The PCI specifications does not make any guarantees
+	 * about P2P capabilities between devices under separate domains.
+	 *
+	 * PCI Local Bus Specification Revision 3.0, section 3.10:
+	 *    "Peer-to-peer transactions crossing multiple host bridges
+	 *     PCI host bridges may, but are not required to, support PCI
+	 *     peer-to-peer transactions that traverse multiple PCI host
+	 *     bridges."
+	 */
+	dev_host_bridge = pci_find_host_bridge(dev->bus);
+	peer_host_bridge = pci_find_host_bridge(peer->bus);
+	if (dev_host_bridge != peer_host_bridge)
+		return false;
+
+	if (pci_is_pcie(dev) && pci_is_pcie(peer)) {
+		/*
+		 * Access Control Services (ACS) Checks
+		 *
+		 * ACS has a capability bit for P2P Request Redirects (RR),
+		 * but unfortunately it doesn't tell us much about the real
+		 * capabilities of the hardware.
+		 *
+		 * PCI Express Base Specification Revision 3.0, section
+		 * 6.12.1.1:
+		 *    "ACS P2P Request Redirect: must be implemented by Root
+		 *     Ports that support peer-to-peer traffic with other
+		 *     Root Ports; [80]"
+		 * but
+		 *    "[80] Root Port indication of ACS P2P Request Redirect
+		 *     or ACS P2P Completion Redirect support does not imply
+		 *     any particular level of peer-to-peer support by the
+		 *     Root Complex, or that peer-to-peer traffic is
+		 *     supported at all"
+		 */
+		struct pci_dev *rpdev = dev->bus->self;
+		struct pci_dev *rppeer = peer->bus->self;
+		struct pci_dev *common_upstream;
+		int pos;
+		u16 cap;
+
+		while ((rpdev) && (pci_is_pcie(rpdev)) &&
+		       (pci_pcie_type(rpdev) != PCI_EXP_TYPE_ROOT_PORT))
+			rpdev = rpdev->bus->self;
+
+		while ((rppeer) && (pci_is_pcie(rppeer)) &&
+		       (pci_pcie_type(rppeer) != PCI_EXP_TYPE_ROOT_PORT))
+			rppeer = rppeer->bus->self;
+
+		common_upstream = pci_find_common_upstream_dev(dev, peer);
+
+		/*
+		 * If ACS is not implemented, we have no idea about P2P
+		 * support. Optimistically allow this if there is a common
+		 * upstream device.
+		 */
+		pos = pci_find_ext_capability(rpdev, PCI_EXT_CAP_ID_ACS);
+		if (!pos)
+			return common_upstream != NULL;
+
+		/*
+		 * If the devices are under the same root port and have a common
+		 * upstream device, allow if the root port is further upstream
+		 * from the common upstream device and the common upstream
+		 * device has Upstream Forwarding disabled, or if the root port
+		 * is the common upstream device and ACS is not implemented.
+		 */
+		pci_read_config_word(rpdev, pos + PCI_ACS_CAP, &cap);
+		if ((rpdev == rppeer && common_upstream) &&
+		    (((common_upstream != rpdev) &&
+		      !pci_acs_enabled(common_upstream, PCI_ACS_UF)) ||
+		     ((common_upstream == rpdev) && ((cap & PCI_ACS_RR) == 0))))
+			return true;
+
+		/*
+		 * If ACS RR is implemented and disabled, allow only if the
+		 * devices are under the same root port.
+		 */
+		if (cap & PCI_ACS_RR && !pci_acs_enabled(rpdev, PCI_ACS_RR))
+			return rpdev == rppeer;
+
+		/*
+		 * If ACS RR is not implemented, or is implemented and enabled,
+		 * only allow if there's a translation agent enabled to do the
+		 * redirect.
+		 */
+		return iommu_present(&pci_bus_type);
+	}
+
+	return false;
+}
+
 #define RESOURCE_ALIGNMENT_PARAM_SIZE COMMAND_LINE_SIZE
 static char resource_alignment_param[RESOURCE_ALIGNMENT_PARAM_SIZE] = {0};
 static DEFINE_SPINLOCK(resource_alignment_lock);

index 8262b9e..46939f7 100644
@@ -1914,4 +1914,7 @@ static inline bool pci_ari_enabled(struct pci_bus *bus)
 {
 	return bus->self && bus->self->ari_enabled;
 }
+
+bool pci_peer_traffic_supported(struct pci_dev *dev, struct pci_dev *peer);
+
 #endif /* LINUX_PCI_H */

index a57e9b7..13a47f283 100644
@@ -471,6 +471,10 @@ DECLARE_STATS_COUNTER(cnt_map_single);
 DECLARE_STATS_COUNTER(cnt_unmap_single);
 DECLARE_STATS_COUNTER(cnt_map_sg);
 DECLARE_STATS_COUNTER(cnt_unmap_sg);
+#ifdef CONFIG_HAS_DMA_P2P
+DECLARE_STATS_COUNTER(cnt_map_peer_resource);
+DECLARE_STATS_COUNTER(cnt_unmap_peer_resource);
+#endif
 DECLARE_STATS_COUNTER(cnt_alloc_coherent);
 DECLARE_STATS_COUNTER(cnt_free_coherent);
 DECLARE_STATS_COUNTER(cross_page);

@@ -509,6 +513,10 @@ static void amd_iommu_stats_init(void)
 	amd_iommu_stats_add(&cnt_unmap_single);
 	amd_iommu_stats_add(&cnt_map_sg);
 	amd_iommu_stats_add(&cnt_unmap_sg);
+#ifdef CONFIG_HAS_DMA_P2P
+	amd_iommu_stats_add(&cnt_map_peer_resource);
+	amd_iommu_stats_add(&cnt_unmap_peer_resource);
+#endif
 	amd_iommu_stats_add(&cnt_alloc_coherent);
 	amd_iommu_stats_add(&cnt_free_coherent);
 	amd_iommu_stats_add(&cross_page);

@@ -2585,20 +2593,16 @@ static void __unmap_single(struct dma_ops_domain *dma_dom,
 }
 
 /*
- * The exported map_single function for dma_ops.
+ * Wrapper function that contains code common to mapping a physical address
+ * range from a page or a resource.
  */
-static dma_addr_t map_page(struct device *dev, struct page *page,
-			   unsigned long offset, size_t size,
-			   enum dma_data_direction dir,
-			   struct dma_attrs *attrs)
+static dma_addr_t __map_phys(struct device *dev, phys_addr_t paddr,
+			     size_t size, enum dma_data_direction dir)
 {
 	unsigned long flags;
 	struct protection_domain *domain;
 	dma_addr_t addr;
 	u64 dma_mask;
-	phys_addr_t paddr = page_to_phys(page) + offset;
-
-	INC_STATS_COUNTER(cnt_map_single);
 
 	domain = get_domain(dev);
 	if (PTR_ERR(domain) == -EINVAL)

@@ -2624,16 +2628,15 @@ out:
 }
 
 /*
- * The exported unmap_single function for dma_ops.
+ * Wrapper function that contains code common to unmapping a physical address
+ * range from a page or a resource.
  */
-static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
-		       enum dma_data_direction dir, struct dma_attrs *attrs)
+static void __unmap_phys(struct device *dev, dma_addr_t dma_addr, size_t size,
+			 enum dma_data_direction dir)
 {
 	unsigned long flags;
 	struct protection_domain *domain;
 
-	INC_STATS_COUNTER(cnt_unmap_single);
-
 	domain = get_domain(dev);
 	if (IS_ERR(domain))
 		return;

@@ -2707,6 +2710,72 @@ unmap:
 }
 
 /*
+ * The exported map_single function for dma_ops.
+ */
+static dma_addr_t map_page(struct device *dev, struct page *page,
+			   unsigned long offset, size_t size,
+			   enum dma_data_direction dir,
+			   struct dma_attrs *attrs)
+{
+	INC_STATS_COUNTER(cnt_map_single);
+
+	return __map_phys(dev, page_to_phys(page) + offset, size, dir);
+}
+
+/*
+ * The exported unmap_single function for dma_ops.
+ */
+static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
+		       enum dma_data_direction dir, struct dma_attrs *attrs)
+{
+	INC_STATS_COUNTER(cnt_unmap_single);
+
+	__unmap_phys(dev, dma_addr, size, dir);
+}
+
+#ifdef CONFIG_HAS_DMA_P2P
+/*
+ * The exported map_peer_resource function for dma_ops.
+ */
+static dma_peer_addr_t map_peer_resource(struct device *dev,
+					 struct device *peer,
+					 struct resource *res,
+					 unsigned long offset,
+					 size_t size,
+					 enum dma_data_direction dir,
+					 struct dma_attrs *attrs)
+{
+	struct pci_dev *pdev;
+	struct pci_dev *ppeer;
+
+	INC_STATS_COUNTER(cnt_map_peer_resource);
+
+	if (!dev_is_pci(dev) || !dev_is_pci(peer))
+		return DMA_ERROR_CODE;
+
+	pdev = to_pci_dev(dev);
+	ppeer = to_pci_dev(peer);
+
+	if (!pci_peer_traffic_supported(pdev, ppeer))
+		return DMA_ERROR_CODE;
+
+	return __map_phys(dev, res->start + offset, size, dir);
+}
+
+/*
+ * The exported unmap_peer_resource function for dma_ops.
+ */
+static void unmap_peer_resource(struct device *dev, dma_peer_addr_t dma_addr,
+				size_t size, enum dma_data_direction dir,
+				struct dma_attrs *attrs)
+{
+	INC_STATS_COUNTER(cnt_unmap_peer_resource);
+
+	__unmap_phys(dev, dma_addr, size, dir);
+}
+#endif
+
+/*
  * The exported map_sg function for dma_ops (handles scatter-gather
  * lists).
  */

@@ -2852,6 +2921,10 @@ static struct dma_map_ops amd_iommu_dma_ops = {
 	.unmap_page = unmap_page,
 	.map_sg = map_sg,
 	.unmap_sg = unmap_sg,
+#ifdef CONFIG_HAS_DMA_P2P
+	.map_peer_resource = map_peer_resource,
+	.unmap_peer_resource = unmap_peer_resource,
+#endif
 	.dma_supported = amd_iommu_dma_supported,
 };
 

index 8715287..dd85173 100644
@@ -140,13 +140,13 @@ typedef unsigned long blkcnt_t;
 #define pgoff_t unsigned long
 
 /*
- * A dma_addr_t can hold any valid DMA address, i.e., any address returned
+ * A dma_addr_t can hold any valid DMA address, i.e., most addresses returned
  * by the DMA API.
  *
  * If the DMA API only uses 32-bit addresses, dma_addr_t need only be 32
  * bits wide.  Bus addresses, e.g., PCI BARs, may be wider than 32 bits,
  * but drivers do memory-mapped I/O to ioremapped kernel virtual addresses,
- * so they don't care about the size of the actual bus addresses.
+ * so they don't care about the size of the actual bus addresses in most cases.
  */
 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
 typedef u64 dma_addr_t;

@@ -154,6 +154,15 @@ typedef u64 dma_addr_t;
 typedef u32 dma_addr_t;
 #endif
 
+/*
+ * Unlike dma_addr_t, a dma_peer_addr_t can hold any bus address for the
+ * platform. Drivers should only need this for peer-to-peer use cases, where
+ * the DMA target is also on the bus.
+ */
+#ifdef CONFIG_HAS_DMA_P2P
+typedef u64 dma_peer_addr_t;
+#endif
+
 typedef unsigned __bitwise__ gfp_t;
 typedef unsigned __bitwise__ fmode_t;
 typedef unsigned __bitwise__ oom_flags_t;

index fe8cb61..4ff8364 100644
@@ -43,7 +43,21 @@ extern void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr);
 
 extern void debug_dma_unmap_page(struct device *dev, dma_addr_t addr,
 				 size_t size, int direction, bool map_single);
-
+#ifdef CONFIG_HAS_DMA_P2P
+extern void debug_dma_map_peer_resource(struct device *dev,
+					struct device *peer,
+					struct resource *resource,
+					size_t offset, size_t size,
+					int direction,
+					dma_peer_addr_t dma_addr);
+
+extern void debug_dma_peer_mapping_error(struct device *dev,
+					 dma_peer_addr_t dma_addr);
+
+extern void debug_dma_unmap_peer_resource(struct device *dev,
+					  dma_peer_addr_t addr,
+					  size_t size, int direction);
+#endif
 extern void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
 			     int nents, int mapped_ents, int direction);
 

@@ -120,6 +134,29 @@ static inline void debug_dma_unmap_page(struct device *dev, dma_addr_t addr,
 {
 }
 
+#ifdef CONFIG_HAS_DMA_P2P
+static inline void debug_dma_map_peer_resource(struct device *dev,
+					       struct device *peer,
+					       struct resource *resource,
+					       size_t offset, size_t size,
+					       int direction,
+					       dma_peer_addr_t dma_addr)
+{
+}
+
+static inline void debug_dma_peer_mapping_error(struct device *dev,
+						dma_peer_addr_t dma_addr)
+{
+}
+
+static inline void debug_dma_unmap_peer_resource(struct device *dev,
+						 dma_peer_addr_t addr,
+						 size_t size,
+						 int direction)
+{
+}
+#endif
+
 static inline void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
 				    int nents, int mapped_ents, int direction)
 {

128 lines not on this page · raw message

index 940d5ec..45eec17 100644
@@ -73,6 +73,42 @@ static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg
 		ops->unmap_sg(dev, sg, nents, dir, attrs);
 }
 
+#ifdef CONFIG_HAS_DMA_P2P
+static inline dma_peer_addr_t dma_map_peer_resource_attrs(struct device *dev,
+							  struct device *peer,
+							  struct resource *res,
+							  size_t offset,
+							  size_t size,
+							  enum dma_data_direction dir,
+							  struct dma_attrs *attrs)
+{
+	struct dma_map_ops *ops = get_dma_ops(dev);
+	dma_peer_addr_t addr;
+
+	BUG_ON(!valid_dma_direction(dir));
+	BUG_ON(ops->map_peer_resource == NULL);
+	addr = ops->map_peer_resource(dev, peer, res, offset, size, dir,
+				      attrs);
+	debug_dma_map_peer_resource(dev, peer, res, offset, size, dir, addr);
+
+	return addr;
+}
+
+static inline void dma_unmap_peer_resource_attrs(struct device *dev,
+						 dma_peer_addr_t addr,
+						 size_t size,
+						 enum dma_data_direction dir,
+						 struct dma_attrs *attrs)
+{
+	struct dma_map_ops *ops = get_dma_ops(dev);
+
+	BUG_ON(!valid_dma_direction(dir));
+	if (ops->unmap_peer_resource)
+		ops->unmap_peer_resource(dev, addr, size, dir, attrs);
+	debug_dma_unmap_peer_resource(dev, addr, size, dir);
+}
+#endif
+
 static inline dma_addr_t dma_map_page(struct device *dev, struct page *page,
 				      size_t offset, size_t size,
 				      enum dma_data_direction dir)

@@ -181,6 +217,13 @@ dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
 #define dma_map_sg(d, s, n, r) dma_map_sg_attrs(d, s, n, r, NULL)
 #define dma_unmap_sg(d, s, n, r) dma_unmap_sg_attrs(d, s, n, r, NULL)
 
+#ifdef CONFIG_HAS_DMA_P2P
+#define dma_map_peer_resource(d, p, e, o, s, r) \
+	dma_map_peer_resource_attrs(d, p, e, o, s, r, NULL)
+#define dma_unmap_peer_resource(d, a, s, r) \
+	dma_unmap_peer_resource_attrs(d, a, s, r, NULL)
+#endif
+
 extern int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
 			   void *cpu_addr, dma_addr_t dma_addr, size_t size);
 

index ac07ff0..7b8fddc 100644
@@ -34,6 +34,18 @@ struct dma_map_ops {
 	void (*unmap_page)(struct device *dev, dma_addr_t dma_handle,
 			   size_t size, enum dma_data_direction dir,
 			   struct dma_attrs *attrs);
+#ifdef CONFIG_HAS_DMA_P2P
+	dma_peer_addr_t (*map_peer_resource)(struct device *dev,
+					     struct device *peer,
+					     struct resource *res,
+					     unsigned long offset, size_t size,
+					     enum dma_data_direction dir,
+					     struct dma_attrs *attrs);
+	void (*unmap_peer_resource)(struct device *dev,
+				    dma_peer_addr_t dma_handle,
+				    size_t size, enum dma_data_direction dir,
+				    struct dma_attrs *attrs);
+#endif
 	/*
 	 * map_sg returns 0 on error and a value > 0 on success.
 	 * It should never return a value < 0.

index c110843..7baf09e 100644
@@ -61,6 +61,29 @@ pci_unmap_page(struct pci_dev *hwdev, dma_addr_t dma_address,
 	dma_unmap_page(hwdev == NULL ? NULL : &hwdev->dev, dma_address, size, (enum dma_data_direction)direction);
 }
 
+#ifdef CONFIG_HAS_DMA_P2P
+static inline dma_peer_addr_t
+pci_map_peer_resource(struct pci_dev *hwdev, struct pci_dev *peer,
+		      int res, unsigned long offset, size_t size,
+		      int direction)
+{
+	return dma_map_peer_resource(hwdev == NULL ? NULL : &hwdev->dev, peer == NULL ? NULL : &peer->dev, peer == NULL ? NULL : &peer->resource[res], offset, size, (enum dma_data_direction)direction);
+}
+
+static inline int
+pci_dma_peer_mapping_error(struct pci_dev *pdev, dma_peer_addr_t dma_addr)
+{
+	return dma_peer_mapping_error(&pdev->dev, dma_addr);
+}
+
+static inline void
+pci_unmap_peer_resource(struct pci_dev *hwdev, dma_peer_addr_t dma_address,
+			size_t size, int direction)
+{
+	dma_unmap_peer_resource(hwdev == NULL ? NULL : &hwdev->dev, dma_address, size, (enum dma_data_direction)direction);
+}
+#endif
+
 static inline int
 pci_map_sg(struct pci_dev *hwdev, struct scatterlist *sg,
 	   int nents, int direction)

index a20ce7d..b6ed554 100644
@@ -384,3 +384,28 @@ int pci_dev_present(const struct pci_device_id *ids)
 	return 0;
 }
 EXPORT_SYMBOL(pci_dev_present);
+
+/**
+ * pci_find_common_upstream_dev - Returns the first common upstream device
+ * @dev: the PCI device from which the bus hierarchy walk will start
+ * @other: the PCI device whose bus number will be used for the search
+ *
+ * Walks up the bus hierarchy from the @dev device, looking for the first bus
+ * which the @other device is downstream of. Returns %NULL if the devices do
+ * not share any upstream devices.
+ */
+struct pci_dev *pci_find_common_upstream_dev(struct pci_dev *dev,
+					     struct pci_dev *other)
+{
+	struct pci_dev *pdev = dev;
+
+	while (pdev != NULL) {
+		if ((other->bus->number >= pdev->bus->busn_res.start) &&
+		    (other->bus->number <= pdev->bus->busn_res.end))
+			return pdev;
+		pdev = pci_upstream_bridge(pdev);
+	}
+
+	return NULL;
+}
+EXPORT_SYMBOL(pci_find_common_upstream_dev);

index d58063e..8262b9e 100644
@@ -860,6 +860,8 @@ static inline struct pci_dev *pci_get_bus_and_slot(unsigned int bus,
 }
 struct pci_dev *pci_get_class(unsigned int class, struct pci_dev *from);
 int pci_dev_present(const struct pci_device_id *ids);
+struct pci_dev *pci_find_common_upstream_dev(struct pci_dev *from,
+					     struct pci_dev *to);
 
 int pci_bus_read_config_byte(struct pci_bus *bus, unsigned int devfn,
 			     int where, u8 *val);

index a98a7b2..0d3746f 100644
@@ -3544,6 +3544,40 @@ static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
 	intel_unmap(dev, dev_addr);
 }
 
+#ifdef CONFIG_HAS_DMA_P2P
+static dma_peer_addr_t intel_map_peer_resource(struct device *dev,
+					       struct device *peer,
+					       struct resource *res,
+					       unsigned long offset,
+					       size_t size,
+					       enum dma_data_direction dir,
+					       struct dma_attrs *attrs)
+{
+	struct pci_dev *pdev;
+	struct pci_dev *ppeer;
+
+	if (!dev_is_pci(dev) || !dev_is_pci(peer))
+		return 0;
+
+	pdev = to_pci_dev(dev);
+	ppeer = to_pci_dev(peer);
+
+	if (!pci_peer_traffic_supported(pdev, ppeer))
+		return 0;
+
+	return __intel_map_single(dev, res->start + offset, size,
+				  dir, *dev->dma_mask);
+}
+
+static void intel_unmap_peer_resource(struct device *dev,
+				      dma_peer_addr_t dev_addr,
+				      size_t size, enum dma_data_direction dir,
+				      struct dma_attrs *attrs)
+{
+	intel_unmap(dev, dev_addr);
+}
+#endif
+
 static void *intel_alloc_coherent(struct device *dev, size_t size,
 				  dma_addr_t *dma_handle, gfp_t flags,
 				  struct dma_attrs *attrs)

@@ -3700,6 +3734,10 @@ struct dma_map_ops intel_dma_ops = {
 	.unmap_sg = intel_unmap_sg,
 	.map_page = intel_map_page,
 	.unmap_page = intel_unmap_page,
+#ifdef CONFIG_HAS_DMA_P2P
+	.map_peer_resource = intel_map_peer_resource,
+	.unmap_peer_resource = intel_unmap_peer_resource,
+#endif
 	.mapping_error = intel_mapping_error,
 };
 

index da15918..4b45cc6 100644
@@ -38,6 +38,39 @@ static dma_addr_t nommu_map_page(struct device *dev, struct page *page,
 	return bus;
 }
 
+#ifdef CONFIG_HAS_DMA_P2P
+static dma_peer_addr_t nommu_map_peer_resource(struct device *dev,
+					       struct device *peer,
+					       struct resource *res,
+					       unsigned long offset,
+					       size_t size,
+					       enum dma_data_direction dir,
+					       struct dma_attrs *attrs)
+{
+	struct pci_dev *pdev;
+	struct pci_dev *ppeer;
+	struct pci_bus_region region;
+	dma_peer_addr_t dma_address;
+
+	if (!dev_is_pci(dev) || !dev_is_pci(peer))
+		return DMA_ERROR_CODE;
+
+	pdev = to_pci_dev(dev);
+	ppeer = to_pci_dev(peer);
+
+	if (!pci_peer_traffic_supported(pdev, ppeer))
+		return DMA_ERROR_CODE;
+
+	pcibios_resource_to_bus(pdev->bus, &region, res);
+	dma_address = region.start + offset;
+	WARN_ON(size == 0);
+	if (!check_addr("map_peer_resource", dev, dma_address, size))
+		return DMA_ERROR_CODE;
+	flush_write_buffers();
+	return dma_address;
+}
+#endif
+
 /* Map a set of buffers described by scatterlist in streaming
  * mode for DMA.  This is the scatter-gather version of the
  * above pci_map_single interface.  Here the scatter gather list

@@ -93,6 +126,9 @@ struct dma_map_ops nommu_dma_ops = {
 	.free			= dma_generic_free_coherent,
 	.map_sg			= nommu_map_sg,
 	.map_page		= nommu_map_page,
+#ifdef CONFIG_HAS_DMA_P2P
+	.map_peer_resource	= nommu_map_peer_resource,
+#endif
 	.sync_single_for_device = nommu_sync_single_for_device,
 	.sync_sg_for_device	= nommu_sync_sg_for_device,
 	.is_phys		= 1,

index 1f5b728..64472d8 100644
@@ -44,16 +44,30 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev)
 #include <asm-generic/dma-mapping-common.h>
 
 /* Make sure we keep the same behaviour */
-static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
+static inline int dma_mapping_error_common(struct device *dev, u64 dma_addr)
 {
 	struct dma_map_ops *ops = get_dma_ops(dev);
-	debug_dma_mapping_error(dev, dma_addr);
 	if (ops->mapping_error)
 		return ops->mapping_error(dev, dma_addr);
 
 	return (dma_addr == DMA_ERROR_CODE);
 }
 
+static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
+{
+	debug_dma_mapping_error(dev, dma_addr);
+	return dma_mapping_error_common(dev, (u64)dma_addr);
+}
+
+#ifdef CONFIG_HAS_DMA_P2P
+static inline int dma_peer_mapping_error(struct device *dev,
+					 dma_peer_addr_t dma_addr)
+{
+	debug_dma_peer_mapping_error(dev, dma_addr);
+	return dma_mapping_error_common(dev, (u64)dma_addr);
+}
+#endif
+
 #define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f)
 #define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h)