gpudev: pin GPU memory
PATCH 1/1 gpudev: pin GPU memory
From: Elena Agostini <[email protected]>
Enable the possibility to make a GPU memory area accessible from the CPU.
GPU memory has to be allocated via rte_gpu_mem_alloc().
This patch allows the gpudev library to pin, through the GPU driver, a chunk of GPU memory and to return a memory pointer usable by the CPU to access the GPU memory area.
--- lib/gpudev/gpudev.c | 47 +++++++++++++++++++++++++++++++++++ lib/gpudev/gpudev_driver.h | 6 +++++ lib/gpudev/rte_gpudev.h | 50 ++++++++++++++++++++++++++++++++++++++ lib/gpudev/version.map | 2 ++ 4 files changed, 105 insertions(+)
lib/gpudev/gpudev.c +47 -0
@@ -634,6 +634,53 @@ rte_gpu_mem_unregister(int16_t dev_id, void *ptr)return GPU_DRV_RET (dev->ops.mem_unregister (dev, ptr)); } +int +rte_gpu_mem_pin (int16_t dev_id,size_t size,void *ptr) +{ +struct rte_gpu *dev; + + dev =gpu_get_by_id (dev_id); +if (dev ==NULL ) { +GPU_LOG (ERR ,"pin mem for invalid device ID %d" , dev_id); + rte_errno =ENODEV ; +return -rte_errno; + } + +if (dev->ops.mem_pin ==NULL ) { +GPU_LOG (ERR ,"mem pinning not supported" ); + rte_errno =ENOTSUP ; +return -rte_errno; + } + +if (ptr ==NULL || size ==0 )/* dry-run */ +return 0 ; + +return GPU_DRV_RET (dev->ops.mem_pin (dev, size, ptr)); +} + +int +rte_gpu_mem_unpin (int16_t dev_id,void *ptr) +{ +struct rte_gpu *dev; + + dev =gpu_get_by_id (dev_id); +if (dev ==NULL ) { +GPU_LOG (ERR ,"unpin mem for invalid device ID %d" , dev_id); + rte_errno =ENODEV ; +return -rte_errno; + } + +if (dev->ops.mem_unpin ==NULL ) { + rte_errno =ENOTSUP ; +return -rte_errno; + } + +if (ptr ==NULL )/* dry-run */ +return 0 ; + +return GPU_DRV_RET (dev->ops.mem_unpin (dev, ptr)); +} +int rte_gpu_wmb (int16_t dev_id) {
lib/gpudev/gpudev_driver.h +6 -0
@@ -31,6 +31,8 @@ typedef int (rte_gpu_mem_alloc_t)(struct rte_gpu *dev, size_t size, void **ptr);typedef int (rte_gpu_mem_free_t )(struct rte_gpu *dev,void *ptr);typedef int (rte_gpu_mem_register_t )(struct rte_gpu *dev,size_t size,void *ptr);typedef int (rte_gpu_mem_unregister_t )(struct rte_gpu *dev,void *ptr); +typedef int (rte_gpu_mem_pin_t )(struct rte_gpu *dev,size_t size,void *ptr); +typedef int (rte_gpu_mem_unpin_t )(struct rte_gpu *dev,void *ptr);typedef int (rte_gpu_wmb_t )(struct rte_gpu *dev);struct rte_gpu_ops {
@@ -46,6 +48,10 @@ struct rte_gpu_ops {rte_gpu_mem_register_t *mem_register;/* Unregister CPU memory from device. */ rte_gpu_mem_unregister_t *mem_unregister; +/* Pin GPU memory. */ +rte_gpu_mem_pin_t *mem_pin; +/* Unpin GPU memory. */ +rte_gpu_mem_unpin_t *mem_unpin;/* Enforce GPU write memory barrier. */ rte_gpu_wmb_t *wmb; };
lib/gpudev/rte_gpudev.h +50 -0
56 lines not on this page · raw message
lib/gpudev/version.map +2 -0
@@ -21,7 +21,9 @@ EXPERIMENTAL { rte_gpu_is_valid; rte_gpu_mem_alloc; rte_gpu_mem_free; + rte_gpu_mem_pin; rte_gpu_mem_register; + rte_gpu_mem_unpin; rte_gpu_mem_unregister; rte_gpu_wmb; };
Signature · 1 line
2.17.1
PATCH v2 1/1 gpudev: pin GPU memory
From: Elena Agostini <[email protected]>
Enable the possibility to make a GPU memory area accessible from the CPU.
GPU memory has to be allocated via rte_gpu_mem_alloc().
This patch allows the gpudev library to pin, through the GPU driver, a chunk of GPU memory and to return a memory pointer usable by the CPU to access the GPU memory area.
--- lib/gpudev/gpudev.c | 47 +++++++++++++++++++++++++++++++++++ lib/gpudev/gpudev_driver.h | 6 +++++ lib/gpudev/rte_gpudev.h | 50 ++++++++++++++++++++++++++++++++++++++ lib/gpudev/version.map | 2 ++ 4 files changed, 105 insertions(+)
lib/gpudev/gpudev.c +47 -0
@@ -634,6 +634,53 @@ rte_gpu_mem_unregister(int16_t dev_id, void *ptr)return GPU_DRV_RET (dev->ops.mem_unregister (dev, ptr)); } +int +rte_gpu_mem_pin (int16_t dev_id,size_t size,void *ptr) +{ +struct rte_gpu *dev; + + dev =gpu_get_by_id (dev_id); +if (dev ==NULL ) { +GPU_LOG (ERR ,"pin mem for invalid device ID %d" , dev_id); + rte_errno =ENODEV ; +return -rte_errno; + } + +if (dev->ops.mem_pin ==NULL ) { +GPU_LOG (ERR ,"mem pinning not supported" ); + rte_errno =ENOTSUP ; +return -rte_errno; + } + +if (ptr ==NULL || size ==0 )/* dry-run */ +return 0 ; + +return GPU_DRV_RET (dev->ops.mem_pin (dev, size, ptr)); +} + +int +rte_gpu_mem_unpin (int16_t dev_id,void *ptr) +{ +struct rte_gpu *dev; + + dev =gpu_get_by_id (dev_id); +if (dev ==NULL ) { +GPU_LOG (ERR ,"unpin mem for invalid device ID %d" , dev_id); + rte_errno =ENODEV ; +return -rte_errno; + } + +if (dev->ops.mem_unpin ==NULL ) { + rte_errno =ENOTSUP ; +return -rte_errno; + } + +if (ptr ==NULL )/* dry-run */ +return 0 ; + +return GPU_DRV_RET (dev->ops.mem_unpin (dev, ptr)); +} +int rte_gpu_wmb (int16_t dev_id) {
lib/gpudev/gpudev_driver.h +6 -0
@@ -31,6 +31,8 @@ typedef int (rte_gpu_mem_alloc_t)(struct rte_gpu *dev, size_t size, void **ptr);typedef int (rte_gpu_mem_free_t )(struct rte_gpu *dev,void *ptr);typedef int (rte_gpu_mem_register_t )(struct rte_gpu *dev,size_t size,void *ptr);typedef int (rte_gpu_mem_unregister_t )(struct rte_gpu *dev,void *ptr); +typedef int (rte_gpu_mem_pin_t )(struct rte_gpu *dev,size_t size,void *ptr); +typedef int (rte_gpu_mem_unpin_t )(struct rte_gpu *dev,void *ptr);typedef int (rte_gpu_wmb_t )(struct rte_gpu *dev);struct rte_gpu_ops {
@@ -46,6 +48,10 @@ struct rte_gpu_ops {rte_gpu_mem_register_t *mem_register;/* Unregister CPU memory from device. */ rte_gpu_mem_unregister_t *mem_unregister; +/* Pin GPU memory. */ +rte_gpu_mem_pin_t *mem_pin; +/* Unpin GPU memory. */ +rte_gpu_mem_unpin_t *mem_unpin;/* Enforce GPU write memory barrier. */ rte_gpu_wmb_t *wmb; };
lib/gpudev/rte_gpudev.h +50 -0
56 lines not on this page · raw message
lib/gpudev/version.map +2 -0
@@ -21,7 +21,9 @@ EXPERIMENTAL { rte_gpu_is_valid; rte_gpu_mem_alloc; rte_gpu_mem_free; + rte_gpu_mem_pin; rte_gpu_mem_register; + rte_gpu_mem_unpin; rte_gpu_mem_unregister; rte_gpu_wmb; };
Signature · 1 line
2.17.1
04/01/2022 03:41, [email protected]:
Quoted 12 lines
From: Elena Agostini <[email protected]>
Enable the possibility to make a GPU memory area accessible from the CPU.
GPU memory has to be allocated via
rte_gpu_mem_alloc().This patch allows the gpudev library to pin, through the GPU driver, a chunk of GPU memory and to return a memory pointer usable by the CPU to access the GPU memory area.
Signed-off-by Elena Agostini <[email protected]>
[...]
Quoted 5 lines
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Pin a chunk of GPU memory to make it accessible from the CPU
You should define what means "pin" exactly. Which properties should we expect?
Quoted 1 line
+ * using the memory pointer returned by the function.
Which function should return the pointer? rte_gpu_mem_pin is returning an int.
Quoted 1 line
+ * GPU memory has to be allocated via
rte_gpu_mem_alloc().
Why pinning is not done by rte_gpu_mem_alloc()? Should it be a flag?
Quoted 11 lines
+ *
+ * @paramdev_id
+ * Device ID requiring pinned memory.
+ * @param size
+ * Number of bytes to pin.
+ * Requesting 0 will do nothing.
+ * @param ptr
+ * Pointer to the GPU memory area to be pinned.
+ * NULL is a no-op accepted value.
+
+ * @return
Quoted 3 lines
+ * A pointer to the pinned GPU memory usable by the CPU, otherwise NULL and rte_errno is set: + * - ENODEV if invalid dev_id + * - EINVAL if reserved flags
Which reserved flags?
Quoted 3 lines
+ * - ENOTSUP if operation not supported by the driver + * - E2BIG if size is higher than limit + * - ENOMEM if out of space
Is out of space relevant for pinning?
Quoted 4 lines
+ * - EPERM if driver error
+ */lib/gpudev/rte_gpudev.h +2
-0+__rte_experimental +int rte_gpu_mem_pin(int16_t dev_id, size_t size, void *ptr);
Quoted 14 lines
04/01/2022 03:41, [email protected]:
Quoted 12 lines
From: Elena Agostini <[email protected]>
Enable the possibility to make a GPU memory area accessible from the CPU.
GPU memory has to be allocated via
rte_gpu_mem_alloc().This patch allows the gpudev library to pin, through the GPU driver, a chunk of GPU memory and to return a memory pointer usable by the CPU to access the GPU memory area.
Signed-off-by Elena Agostini <[email protected]>[...]
Quoted 9 lines
Quoted 5 lines
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Pin a chunk of GPU memory to make it accessible from the CPUYou should define what means "pin" exactly. Which properties should we expect?
Thanks for reviewing, this is the kind of discussion I wanted to have. Maybe "pin" is too GDRCopy specific oriented.
Here I want to make a GPU memory buffer visible from the CPU. In case of NVIDIA, this means the GPU memory address has to be pinned (virtual address doesn't change) and dma-mapped.
Maybe the name should be more like rte_gpu_mem_to_cpu() that's more explicative and generic.
Quoted 4 lines
Quoted 1 line
+ * using the memory pointer returned by the function.
Which function should return the pointer?
rte_gpu_mem_pinis returning an int.
Oversight, will fix it.
Quoted 6 lines
Quoted 3 lines
+ * GPU memory has to be allocated via
rte_gpu_mem_alloc().Why pinning is not done by
rte_gpu_mem_alloc()? Should it be a flag?
rte_gpu_mem_alloc() allocate virtual memory on the GPU that doesn't have to be necessarily shared (pinned) to make it visible from CPU.
Quoted 16 lines
Quoted 12 lines
+ *
+ * @paramdev_id
+ * Device ID requiring pinned memory.
+ * @param size
+ * Number of bytes to pin.
+ * Requesting 0 will do nothing.
+ * @param ptr
+ * Pointer to the GPU memory area to be pinned.
+ * NULL is a no-op accepted value.
+
+ * @returnQuoted 3 lines
+ * A pointer to the pinned GPU memory usable by the CPU, otherwise NULL and rte_errno is set: + * - ENODEV if invalid dev_id + * - EINVAL if reserved flags
Quoted 7 lines
Which reserved flags?
Quoted 3 lines
+ * - ENOTSUP if operation not supported by the driver + * - E2BIG if size is higher than limit + * - ENOMEM if out of spaceIs out of space relevant for pinning?
Yes, let me add it
Quoted 5 lines
Quoted 5 lines
+ * - EPERM if driver error
+ */lib/gpudev/rte_gpudev.h +2
-0+__rte_experimental +int rte_gpu_mem_pin(int16_t dev_id, size_t size, void *ptr);
What happens when the Nvidia GPU driver kernel callback occurs to invalidate the pinned GPU memory region? Doesn't the NIC need to cease all DMA transfers to/from that region before the kernel callback can complete?
From: Elena Agostini <[email protected]>
Sent: 04 January 2022 13:55
To: NBU-Contact-Thomas Monjalon (EXTERNAL) <[email protected]>
Cc: [email protected]
Subject: Re: [PATCH v2] gpudev: pin GPU memory
CAUTION: This email originated from outside of the organization. Do not click links or open attachments unless you recognize the sender and know the content is safe.
Quoted 14 lines
04/01/2022 03:41, [email protected]<mailto:[email protected]>:
Quoted 12 lines
From: Elena Agostini <[email protected]<mailto:[email protected]>>
Enable the possibility to make a GPU memory area accessible from the CPU.
GPU memory has to be allocated via
rte_gpu_mem_alloc().This patch allows the gpudev library to pin, through the GPU driver, a chunk of GPU memory and to return a memory pointer usable by the CPU to access the GPU memory area.
[...]
Quoted 9 lines
Quoted 5 lines
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Pin a chunk of GPU memory to make it accessible from the CPUYou should define what means "pin" exactly. Which properties should we expect?
Thanks for reviewing, this is the kind of discussion I wanted to have. Maybe "pin" is too GDRCopy specific oriented.
Here I want to make a GPU memory buffer visible from the CPU. In case of NVIDIA, this means the GPU memory address has to be pinned (virtual address doesn't change) and dma-mapped.
Maybe the name should be more like rte_gpu_mem_to_cpu() that's more explicative and generic.
Quoted 4 lines
Quoted 1 line
+ * using the memory pointer returned by the function.
Which function should return the pointer?
rte_gpu_mem_pinis returning an int.
Oversight, will fix it.
Quoted 6 lines
Quoted 3 lines
+ * GPU memory has to be allocated via
rte_gpu_mem_alloc().Why pinning is not done by
rte_gpu_mem_alloc()? Should it be a flag?
rte_gpu_mem_alloc() allocate virtual memory on the GPU that doesn't have to be necessarily shared (pinned) to make it visible from CPU.
Quoted 16 lines
Quoted 12 lines
+ *
+ * @paramdev_id
+ * Device ID requiring pinned memory.
+ * @param size
+ * Number of bytes to pin.
+ * Requesting 0 will do nothing.
+ * @param ptr
+ * Pointer to the GPU memory area to be pinned.
+ * NULL is a no-op accepted value.
+
+ * @returnQuoted 3 lines
+ * A pointer to the pinned GPU memory usable by the CPU, otherwise NULL and rte_errno is set: + * - ENODEV if invalid dev_id + * - EINVAL if reserved flags
Quoted 7 lines
Which reserved flags?
Quoted 3 lines
+ * - ENOTSUP if operation not supported by the driver + * - E2BIG if size is higher than limit + * - ENOMEM if out of spaceIs out of space relevant for pinning?
Yes, let me add it
Quoted 5 lines
Quoted 5 lines
+ * - EPERM if driver error
+ */lib/gpudev/rte_gpudev.h +2
-0+__rte_experimental +int rte_gpu_mem_pin(int16_t dev_id, size_t size, void *ptr);
This specific feature is not about NIC activity, it’s about make GPU memory visible and accessible to the CPU so there isn’t an asynchronous hidden invalidation procedure. That’s the reason for my comment to change the name from rte_gpu_mem_pin() to rte_gpu_mem_to_cpu() (or similar).
The scenario you are proposing can happen but in a different context, this is not the case.
Thanks
EA
From: John Alexander <[email protected]>
Date: Tuesday, 4 January 2022 at 18:31
To: Elena Agostini <[email protected]>, NBU-Contact-Thomas Monjalon (EXTERNAL) <[email protected]>
Cc: [email protected] <[email protected]>
Subject: RE: [PATCH v2] gpudev: pin GPU memory
External email: Use caution opening links or attachments
What happens when the Nvidia GPU driver kernel callback occurs to invalidate the pinned GPU memory region? Doesn’t the NIC need to cease all DMA transfers to/from that region before the kernel callback can complete?
John Alexander
Senior Software Engineer [cid:[email protected]]<https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fwww.datapath.co.uk%2F&data=04%7C01%7Ceagostini%40nvidia.com%7Ce2fbb83b86ac4dc3571308d9cfa7b653%7C43083d15727340c1b7db39efd9ccc17a%7C0%7C0%7C637769142848844684%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&sdata=9b3%2FEpEiezz%2B0nLDmm2pE5ZcLkEzNbW%2F%2BJ1o61q2J3k%3D&reserved=0> Bemrose House, Bemrose Park, Wayzgoose Drive
,
Derby
,
DE21 6XQ
[cid:[email protected]]
+44 (0)1332 294 441<tel:+44%20(0)1332%20294%20441>
|
www.datapath.co.uk<https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fwww.datapath.co.uk%2F&data=04%7C01%7Ceagostini%40nvidia.com%7Ce2fbb83b86ac4dc3571308d9cfa7b653%7C43083d15727340c1b7db39efd9ccc17a%7C0%7C0%7C637769142848844684%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&sdata=9b3%2FEpEiezz%2B0nLDmm2pE5ZcLkEzNbW%2F%2BJ1o61q2J3k%3D&reserved=0> [LinkedIn]<https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fwww.linkedin.com%2Fcompany%2Fdatapath-ltd&data=04%7C01%7Ceagostini%40nvidia.com%7Ce2fbb83b86ac4dc3571308d9cfa7b653%7C43083d15727340c1b7db39efd9ccc17a%7C0%7C0%7C637769142848844684%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&sdata=0PTscJhiHEK27PKDjq2nNgH5oRe499xubLh4S705u4A%3D&reserved=0> [Twitter]<https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fwww.twitter.com%2Fdatapathltd&data=04%7C01%7Ceagostini%40nvidia.com%7Ce2fbb83b86ac4dc3571308d9cfa7b653%7C43083d15727340c1b7db39efd9ccc17a%7C0%7C0%7C637769142848844684%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&sdata=CVia5Sq%2Fi2yOAGx5d8PSsF7wJsA2oVM%2BgySwg5E7p2U%3D&reserved=0> [YouTube]<https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fwww.youtube.com%2Fdatapathderby&data=04%7C01%7Ceagostini%40nvidia.com%7Ce2fbb83b86ac4dc3571308d9cfa7b653%7C43083d15727340c1b7db39efd9ccc17a%7C0%7C0%7C637769142848844684%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&sdata=Obz00eF3OgTdp3YI5ux2YFRdx6Q65keqlMXVY6QNXCM%3D&reserved=0> [cid:[email protected]]<https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fwww.datapath.co.uk%2Fevents-and-webinars%2F&data=04%7C01%7Ceagostini%40nvidia.com%7Ce2fbb83b86ac4dc3571308d9cfa7b653%7C43083d15727340c1b7db39efd9ccc17a%7C0%7C0%7C637769142848844684%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&sdata=MO6M5IE0mAymLgzIZwGqPPxqftcmzH7Gy43ZOGYHwAs%3D&reserved=0> [Vote for Datapath]<https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fwww.datapath.co.uk%2Fdatapath-shortlisted-for-coveted-industry-award%2F&data=04%7C01%7Ceagostini%40nvidia.com%7Ce2fbb83b86ac4dc3571308d9cfa7b653%7C43083d15727340c1b7db39efd9ccc17a%7C0%7C0%7C637769142848844684%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&sdata=D0wKDJVR%2B9kSzyIvtxt5yOdKZPCb%2Fd48biairPlYqt4%3D&reserved=0> Datapath Ltd. Registered Number: 1609392. Registered in England at Bemrose House, Bemrose Park, Wayzgoose Drive, Derby. DE21 6XQ.
From: Elena Agostini <[email protected]>
Sent: 04 January 2022 13:55
To: NBU-Contact-Thomas Monjalon (EXTERNAL) <[email protected]>
Cc: [email protected]
Subject: Re: [PATCH v2] gpudev: pin GPU memory
CAUTION: This email originated from outside of the organization. Do not click links or open attachments unless you recognize the sender and know the content is safe.
Quoted 14 lines
04/01/2022 03:41, [email protected]<mailto:[email protected]>:
Quoted 12 lines
From: Elena Agostini <[email protected]<mailto:[email protected]>>
Enable the possibility to make a GPU memory area accessible from the CPU.
GPU memory has to be allocated via
rte_gpu_mem_alloc().This patch allows the gpudev library to pin, through the GPU driver, a chunk of GPU memory and to return a memory pointer usable by the CPU to access the GPU memory area.
[...]
Quoted 9 lines
Quoted 5 lines
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Pin a chunk of GPU memory to make it accessible from the CPUYou should define what means "pin" exactly. Which properties should we expect?
Thanks for reviewing, this is the kind of discussion I wanted to have. Maybe "pin" is too GDRCopy specific oriented.
Here I want to make a GPU memory buffer visible from the CPU. In case of NVIDIA, this means the GPU memory address has to be pinned (virtual address doesn't change) and dma-mapped.
Maybe the name should be more like rte_gpu_mem_to_cpu() that's more explicative and generic.
Quoted 4 lines
Quoted 1 line
+ * using the memory pointer returned by the function.
Which function should return the pointer?
rte_gpu_mem_pinis returning an int.
Oversight, will fix it.
Quoted 6 lines
Quoted 3 lines
+ * GPU memory has to be allocated via
rte_gpu_mem_alloc().Why pinning is not done by
rte_gpu_mem_alloc()? Should it be a flag?
rte_gpu_mem_alloc() allocate virtual memory on the GPU that doesn't have to be necessarily shared (pinned) to make it visible from CPU.
Quoted 16 lines
Quoted 12 lines
+ *
+ * @paramdev_id
+ * Device ID requiring pinned memory.
+ * @param size
+ * Number of bytes to pin.
+ * Requesting 0 will do nothing.
+ * @param ptr
+ * Pointer to the GPU memory area to be pinned.
+ * NULL is a no-op accepted value.
+
+ * @returnQuoted 3 lines
+ * A pointer to the pinned GPU memory usable by the CPU, otherwise NULL and rte_errno is set: + * - ENODEV if invalid dev_id + * - EINVAL if reserved flags
Quoted 7 lines
Which reserved flags?
Quoted 3 lines
+ * - ENOTSUP if operation not supported by the driver + * - E2BIG if size is higher than limit + * - ENOMEM if out of spaceIs out of space relevant for pinning?
Yes, let me add it
Quoted 5 lines
Quoted 5 lines
+ * - EPERM if driver error
+ */lib/gpudev/rte_gpudev.h +2
-0+__rte_experimental +int rte_gpu_mem_pin(int16_t dev_id, size_t size, void *ptr);
PATCH v3 1/1 gpudev: expose GPU memory
In reply to [email protected]From: Elena Agostini <[email protected]>
Enable the possibility to expose a GPU memory area and make it accessible from the CPU.
GPU memory has to be allocated via rte_gpu_mem_alloc().
This patch allows the gpudev library to expose (and unexpose), through the GPU driver, a chunk of GPU memory and to return a memory pointer usable by the CPU to access the GPU memory area.
Changelog:
- Move to pin/unpin naming to expose/unexpose
- Minor changes to function definitions
--- lib/gpudev/gpudev.c | 61 ++++++++++++++++++++++++++++++++++++++ lib/gpudev/gpudev_driver.h | 6 ++++ lib/gpudev/rte_gpudev.h | 49 ++++++++++++++++++++++++++++++ lib/gpudev/version.map | 2 ++ 4 files changed, 118 insertions(+)
lib/gpudev/gpudev.c +61 -0
@@ -634,6 +634,67 @@ rte_gpu_mem_unregister(int16_t dev_id, void *ptr)return GPU_DRV_RET (dev->ops.mem_unregister (dev, ptr)); } +void * +rte_gpu_mem_expose (int16_t dev_id,size_t size,void *ptr) +{ +struct rte_gpu *dev; +void *ptr_out; +int ret; + + dev =gpu_get_by_id (dev_id); +if (dev ==NULL ) { +GPU_LOG (ERR ,"expose mem for invalid device ID %d" , dev_id); + rte_errno =ENODEV ; +return NULL ; + } + +if (dev->ops.mem_expose ==NULL ) { +GPU_LOG (ERR ,"mem exposure not supported" ); + rte_errno =ENOTSUP ; +return NULL ; + } + +if (ptr ==NULL || size ==0 )/* dry-run */ +return NULL ; + + ret =GPU_DRV_RET (dev->ops.mem_expose (dev, size, ptr, &ptr_out)); + +switch (ret) { +case 0 : +return ptr_out; +case -ENOMEM : +case -E2BIG : + rte_errno = -ret; +return NULL ; +default : + rte_errno = -EPERM ; +return NULL ; + } +} + +int +rte_gpu_mem_unexpose (int16_t dev_id,void *ptr) +{ +struct rte_gpu *dev; + + dev =gpu_get_by_id (dev_id); +if (dev ==NULL ) { +GPU_LOG (ERR ,"unexpose mem for invalid device ID %d" , dev_id); + rte_errno =ENODEV ; +return -rte_errno; + } + +if (dev->ops.mem_unexpose ==NULL ) { + rte_errno =ENOTSUP ; +return -rte_errno; + } + +if (ptr ==NULL )/* dry-run */ +return 0 ; + +return GPU_DRV_RET (dev->ops.mem_unexpose (dev, ptr)); +} +int rte_gpu_wmb (int16_t dev_id) {
lib/gpudev/gpudev_driver.h +6 -0
@@ -31,6 +31,8 @@ typedef int (rte_gpu_mem_alloc_t)(struct rte_gpu *dev, size_t size, void **ptr);typedef int (rte_gpu_mem_free_t )(struct rte_gpu *dev,void *ptr);typedef int (rte_gpu_mem_register_t )(struct rte_gpu *dev,size_t size,void *ptr);typedef int (rte_gpu_mem_unregister_t )(struct rte_gpu *dev,void *ptr); +typedef int (rte_gpu_mem_expose_t )(struct rte_gpu *dev,size_t size,void *ptr_in,void **ptr_out); +typedef int (rte_gpu_mem_unexpose_t )(struct rte_gpu *dev,void *ptr);typedef int (rte_gpu_wmb_t )(struct rte_gpu *dev);struct rte_gpu_ops {
@@ -46,6 +48,10 @@ struct rte_gpu_ops {rte_gpu_mem_register_t *mem_register;/* Unregister CPU memory from device. */ rte_gpu_mem_unregister_t *mem_unregister; +/* Pin GPU memory. */ +rte_gpu_mem_expose_t *mem_expose; +/* Unpin GPU memory. */ +rte_gpu_mem_unexpose_t *mem_unexpose;/* Enforce GPU write memory barrier. */ rte_gpu_wmb_t *wmb; };
lib/gpudev/rte_gpudev.h +49 -0
55 lines not on this page · raw message
lib/gpudev/version.map +2 -0
@@ -20,8 +20,10 @@ EXPERIMENTAL { rte_gpu_init; rte_gpu_is_valid; rte_gpu_mem_alloc; + rte_gpu_mem_expose; rte_gpu_mem_free; rte_gpu_mem_register; + rte_gpu_mem_unexpose; rte_gpu_mem_unregister; rte_gpu_wmb; };
Signature · 1 line
2.17.1
PATCH v4 1/2 gpudev: expose GPU memory to CPU
In reply to [email protected]From: Elena Agostini <[email protected]>
Enable the possibility to expose a GPU memory area and make it accessible from the CPU.
GPU memory has to be allocated via rte_gpu_mem_alloc().
This patch allows the gpudev library to map (and unmap), through the GPU driver, a chunk of GPU memory and to return a memory pointer usable by the CPU to access the GPU memory area.
--- doc/guides/prog_guide/gpudev.rst | 9 +++++ drivers/gpu/cuda/cuda.c | 2 ++ lib/gpudev/gpudev.c | 61 ++++++++++++++++++++++++++++++++ lib/gpudev/gpudev_driver.h | 6 ++++ lib/gpudev/rte_gpudev.h | 49 +++++++++++++++++++++++++ lib/gpudev/version.map | 2 ++ 6 files changed, 129 insertions(+)
doc/guides/prog_guide/gpudev.rst +9 -0
@@ -73,6 +73,15 @@ Later, it's also possible to unregister that memory with gpudev. CPU memory registered outside of the gpudev library (e.g. with GPU specific library) cannot be unregistered by the gpudev library. +CPU mapping +~~~~~~~~~~~~~~~~~~~ + +gpudev can map into the CPU address space a GPU memory address allocated with gpudev. +gpudev returns a pointer the CPU can use to access (ready or write) GPU memory. +Later, it's also possible to unmap that memory with gpudev. +GPU memory CPU mapped outside of the gpudev library (e.g. with GPU specific library) +cannot be unmapped by the gpudev library. + Memory Barrier ~~~~~~~~~~~~~~
drivers/gpu/cuda/cuda.c +2 -0
@@ -1177,6 +1177,8 @@ cuda_gpu_probe(__rte_unused struct rte_pci_driver *pci_drv, struct rte_pci_devic dev->ops.mem_free = cuda_mem_free; dev->ops.mem_register = cuda_mem_register; dev->ops.mem_unregister = cuda_mem_unregister; + dev->ops.mem_cpu_map =NULL ; + dev->ops.mem_cpu_unmap =NULL ; dev->ops.wmb = cuda_wmb;rte_gpu_complete_new (dev);
lib/gpudev/gpudev.c +61 -0
67 lines not on this page · raw message
lib/gpudev/gpudev_driver.h +6 -0
@@ -31,6 +31,8 @@ typedef int (rte_gpu_mem_alloc_t)(struct rte_gpu *dev, size_t size, unsigned inttypedef int (rte_gpu_mem_free_t )(struct rte_gpu *dev,void *ptr);typedef int (rte_gpu_mem_register_t )(struct rte_gpu *dev,size_t size,void *ptr);typedef int (rte_gpu_mem_unregister_t )(struct rte_gpu *dev,void *ptr); +typedef int (rte_gpu_mem_cpu_map_t )(struct rte_gpu *dev,size_t size,void *ptr_in,void **ptr_out); +typedef int (rte_gpu_mem_cpu_unmap_t )(struct rte_gpu *dev,void *ptr);typedef int (rte_gpu_wmb_t )(struct rte_gpu *dev);struct rte_gpu_ops {
@@ -46,6 +48,10 @@ struct rte_gpu_ops {rte_gpu_mem_register_t *mem_register;/* Unregister CPU memory from device. */ rte_gpu_mem_unregister_t *mem_unregister; +/* Map GPU memory for CPU visibility. */ +rte_gpu_mem_cpu_map_t *mem_cpu_map; +/* Unmap GPU memory for CPU visibility. */ +rte_gpu_mem_cpu_unmap_t *mem_cpu_unmap;/* Enforce GPU write memory barrier. */ rte_gpu_wmb_t *wmb; };
lib/gpudev/rte_gpudev.h +49 -0
55 lines not on this page · raw message
lib/gpudev/version.map +2 -0
@@ -20,8 +20,10 @@ EXPERIMENTAL { rte_gpu_init; rte_gpu_is_valid; rte_gpu_mem_alloc; + rte_gpu_mem_cpu_map; rte_gpu_mem_free; rte_gpu_mem_register; + rte_gpu_mem_cpu_unmap; rte_gpu_mem_unregister; rte_gpu_wmb; };
Signature · 1 line
2.17.1
PATCH v4 2/2 app/test-gpudev: test cpu_map/cpu_unmap functions
From: Elena Agostini <[email protected]>
New test case added to test the gpudev cpu_map/cpu_unmap functions.
--- app/test-gpudev/main.c | 63 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+)
app/test-gpudev/main.c +63 -0
@@ -185,6 +185,68 @@ register_cpu_memory(uint16_t gpu_id)return -1 ; } +static int +gpu_mem_cpu_map (uint16_t gpu_id) +{ +void *ptr_gpu =NULL ; +void *ptr_cpu =NULL ; +size_t buf_bytes =1024 ; +unsigned int align =4096 ; +int ret; + +printf ("\n=======> TEST: Map GPU memory for CPU visibility\n\n" ); + +/* Alloc memory on GPU 0 with 4kB alignment */ + ptr_gpu =rte_gpu_mem_alloc (gpu_id, buf_bytes, align); +if (ptr_gpu ==NULL ) { +fprintf (stderr,"rte_gpu_mem_alloc GPU memory returned error\n" ); + goto error; + } +printf ("GPU memory allocated at 0x%p size is %zd bytes\n" , + ptr_gpu, buf_bytes); + + ptr_cpu =rte_gpu_mem_cpu_map (gpu_id, buf_bytes, ptr_gpu); +if (ptr_cpu ==NULL ) { +fprintf (stderr,"rte_gpu_mem_cpu_map returned error\n" ); + goto error; + } +printf ("GPU memory mapped for CPU access at 0x%p\n" , ptr_cpu); + + ((uint8_t *)ptr_cpu)[0 ] =0x4 ; + ((uint8_t *)ptr_cpu)[1 ] =0x5 ; + ((uint8_t *)ptr_cpu)[2 ] =0x6 ; + +printf ("GPU memory first 3 bytes set from CPU: %x %x %x\n" , + ((uint8_t *)ptr_cpu)[0 ], + ((uint8_t *)ptr_cpu)[1 ], + ((uint8_t *)ptr_cpu)[2 ]); + + ret =rte_gpu_mem_cpu_unmap (gpu_id, ptr_cpu); +if (ret <0 ) { +fprintf (stderr,"rte_gpu_mem_cpu_unmap returned error %d\n" , ret); + goto error; + } +printf ("GPU memory mapped for CPU access at 0x%p\n" , ptr_cpu); + + ret =rte_gpu_mem_free (gpu_id, ptr_gpu); +if (ret <0 ) { +fprintf (stderr,"rte_gpu_mem_free returned error %d\n" , ret); + goto error; + } +printf ("GPU memory 0x%p freed\n" , ptr_gpu); + +printf ("\n=======> TEST: PASSED\n" ); +return 0 ; + +error: + +rte_gpu_mem_cpu_unmap (gpu_id, ptr_cpu); +rte_gpu_mem_free (gpu_id, ptr_gpu); + +printf ("\n=======> TEST: FAILED\n" ); +return -1 ; +} +static int create_update_comm_flag (uint16_t gpu_id) {
@@ -402,6 +464,7 @@ main(int argc, char **argv)*/ alloc_gpu_memory (gpu_id);register_cpu_memory (gpu_id); +gpu_mem_cpu_map (gpu_id);/** * Communication items test
Signature · 1 line
2.17.1
RE: [PATCH v4 1/2] gpudev: expose GPU memory to CPU
In reply to [email protected]Quoted 30 lines
-----Original Message-----
From: [email protected] <[email protected]>
Sent: Thursday, January 27, 2022 11:47
To: [email protected]
Cc: Elena Agostini <[email protected]>
Subject: [PATCH v4 1/2] gpudev: expose GPU memory to CPUFrom: Elena Agostini <[email protected]>
Enable the possibility to expose a GPU memory area and make it accessible from the CPU.
GPU memory has to be allocated via
rte_gpu_mem_alloc().This patch allows the gpudev library to map (and unmap), through the GPU driver, a chunk of GPU memory and to return a memory pointer usable by the CPU to access the GPU memory area.
Signed-off-by Elena Agostini <[email protected]>--- doc/guides/prog_guide/gpudev.rst | 9 +++++ drivers/gpu/cuda/cuda.c | 2 ++ lib/gpudev/gpudev.c | 61 ++++++++++++++++++++++++++++++++ lib/gpudev/gpudev_driver.h | 6 ++++ lib/gpudev/rte_gpudev.h | 49 +++++++++++++++++++++++++ lib/gpudev/version.map | 2 ++ 6 files changed, 129 insertions(+)
lib/gpudev/rte_gpudev.h int rte_gpu_mem_register(int16_t dev_id, size_t size, void *ptr); · line 481
Quoted 2 lines
lib/gpudev/rte_gpudev.h +2
-0+__rte_experimental +void *rte_gpu_mem_cpu_map(int16_t dev_id, size_t size, void *ptr);
How about add some direction words like "to/from" to make it straightforward ?
For this: rte_gpu_mem_map_to_cpu ?
lib/gpudev/rte_gpudev.h int rte_gpu_mem_register(int16_t dev_id, size_t size, void *ptr); · line 503
Quoted 3 lines
lib/gpudev/rte_gpudev.h +3
-0+__rte_experimental +int rte_gpu_mem_cpu_unmap(int16_t dev_id, void *ptr); +
And rte_gpu_mem_unmap_to_cpu ?
Quoted 3 lines
};
--
2.17.1
Re: [PATCH v4 1/2] gpudev: expose GPU memory to CPU
Quoted 42 lines
From: Wang, Haiyue <[email protected]>
Date: Thursday, 27 January 2022 at 07:56
To: Elena Agostini <[email protected]>, [email protected] <[email protected]>
Subject: RE: [PATCH v4 1/2] gpudev: expose GPU memory to CPU External email: Use caution opening links or attachmentsQuoted 30 lines
-----Original Message-----
From: [email protected] <[email protected]>
Sent: Thursday, January 27, 2022 11:47
To: [email protected]
Cc: Elena Agostini <[email protected]>
Subject: [PATCH v4 1/2] gpudev: expose GPU memory to CPUFrom: Elena Agostini <[email protected]>
Enable the possibility to expose a GPU memory area and make it accessible from the CPU.
GPU memory has to be allocated via
rte_gpu_mem_alloc().This patch allows the gpudev library to map (and unmap), through the GPU driver, a chunk of GPU memory and to return a memory pointer usable by the CPU to access the GPU memory area.
Signed-off-by Elena Agostini <[email protected]>--- doc/guides/prog_guide/gpudev.rst | 9 +++++ drivers/gpu/cuda/cuda.c | 2 ++ lib/gpudev/gpudev.c | 61 ++++++++++++++++++++++++++++++++ lib/gpudev/gpudev_driver.h | 6 ++++ lib/gpudev/rte_gpudev.h | 49 +++++++++++++++++++++++++ lib/gpudev/version.map | 2 ++ 6 files changed, 129 insertions(+)lib/gpudev/rte_gpudev.h int rte_gpu_mem_register(int16_t dev_id, size_t size, void *ptr); · line 481
Quoted 2 lines
lib/gpudev/rte_gpudev.h +2
-0+__rte_experimental +void *rte_gpu_mem_cpu_map(int16_t dev_id, size_t size, void *ptr);How about add some direction words like "to/from" to make it straightforward ?
Quoted 11 lines
For this:
rte_gpu_mem_map_to_cpu?lib/gpudev/rte_gpudev.h int rte_gpu_mem_register(int16_t dev_id, size_t size, void *ptr); · line 503
Quoted 3 lines
lib/gpudev/rte_gpudev.h +3
-0+__rte_experimental +int rte_gpu_mem_cpu_unmap(int16_t dev_id, void *ptr); +And
rte_gpu_mem_unmap_to_cpu?Quoted 3 lines
};
--
2.17.1
Motivation behind the name is to be compliant with rte_*_dma_map/unmap() functions where the entity comes first (DMA) and the action later (map/unmap).
I think your proposal is more readable but it's also too verbose for an API (map_to_cpu/unmap_from_cpu).
RE: [PATCH v4 1/2] gpudev: expose GPU memory to CPU
From: Elena Agostini <[email protected]>
Sent: Thursday, February 10, 2022 18:38
To: Wang, Haiyue <[email protected]>; [email protected]
Subject: Re: [PATCH v4 1/2] gpudev: expose GPU memory to CPU
Quoted 54 lines
From: Wang, Haiyue <mailto:[email protected]>
Date: Thursday, 27 January 2022 at 07:56
To: Elena Agostini <mailto:[email protected]>, mailto:[email protected] <mailto:[email protected]>
Subject: RE: [PATCH v4 1/2] gpudev: expose GPU memory to CPU
External email: Use caution opening links or attachmentsQuoted 30 lines
-----Original Message-----
From: mailto:[email protected] <mailto:[email protected]>
Sent: Thursday, January 27, 2022 11:47
To: mailto:[email protected]
Cc: Elena Agostini <mailto:[email protected]>
Subject: [PATCH v4 1/2] gpudev: expose GPU memory to CPUFrom: Elena Agostini <mailto:[email protected]>
Enable the possibility to expose a GPU memory area and make it accessible from the CPU.
GPU memory has to be allocated via
rte_gpu_mem_alloc().This patch allows the gpudev library to map (and unmap), through the GPU driver, a chunk of GPU memory and to return a memory pointer usable by the CPU to access the GPU memory area.
Signed-off-by Elena Agostini <mailto:[email protected]>---
doc/guides/prog_guide/gpudev.rst| 9 +++++ drivers/gpu/cuda/cuda.c | 2 ++ lib/gpudev/gpudev.c | 61 ++++++++++++++++++++++++++++++++ lib/gpudev/gpudev_driver.h | 6 ++++ lib/gpudev/rte_gpudev.h | 49 +++++++++++++++++++++++++ lib/gpudev/version.map | 2 ++6 files changed, 129 insertions(+)lib/gpudev/rte_gpudev.h int rte_gpu_mem_register(int16_t dev_id, size_t size, void *ptr); · line 481
Quoted 2 lines
lib/gpudev/rte_gpudev.h +2
-0+__rte_experimental +void *rte_gpu_mem_cpu_map(int16_t dev_id, size_t size, void *ptr);How about add some direction words like "to/from" to make it straightforward ?
For this:
rte_gpu_mem_map_to_cpu?lib/gpudev/rte_gpudev.h int rte_gpu_mem_register(int16_t dev_id, size_t size, void *ptr); · line 503
Quoted 3 lines
lib/gpudev/rte_gpudev.h +3
-0+__rte_experimental +int rte_gpu_mem_cpu_unmap(int16_t dev_id, void *ptr); +And
rte_gpu_mem_unmap_to_cpu?Quoted 3 lines
};
--
2.17.1
Quoted 7 lines
Motivation behind the name is to be compliant with rte_*_dma_map/unmap() functions where the entity comes first (DMA) and the action later (map/unmap).
I think your proposal is more readable but it's also too verbose for an API (map_to_cpu/unmap_from_cpu).
Then just ' rte_gpu_mem_map' & ' rte_gpu_mem_umap', since DPDK is running on CPU. ;-)
PATCH v5 1/2 gpudev: expose GPU memory to CPU
In reply to [email protected]From: Elena Agostini <[email protected]>
Enable the possibility to expose a GPU memory area and make it accessible from the CPU.
GPU memory has to be allocated via rte_gpu_mem_alloc().
This patch allows the gpudev library to map (and unmap), through the GPU driver, a chunk of GPU memory and to return a memory pointer usable by the CPU to access the GPU memory area.
--- doc/guides/prog_guide/gpudev.rst | 9 +++++ drivers/gpu/cuda/cuda.c | 2 ++ lib/gpudev/gpudev.c | 61 ++++++++++++++++++++++++++++++++ lib/gpudev/gpudev_driver.h | 6 ++++ lib/gpudev/rte_gpudev.h | 49 +++++++++++++++++++++++++ lib/gpudev/version.map | 2 ++ 6 files changed, 129 insertions(+)
doc/guides/prog_guide/gpudev.rst +9 -0
@@ -73,6 +73,15 @@ Later, it's also possible to unregister that memory with gpudev. CPU memory registered outside of the gpudev library (e.g. with GPU specific library) cannot be unregistered by the gpudev library. +CPU mapping +~~~~~~~~~~~~~~~~~~~ + +gpudev can map into the CPU address space a GPU memory address allocated with gpudev. +gpudev returns a pointer the CPU can use to access (ready or write) GPU memory. +Later, it's also possible to unmap that memory with gpudev. +GPU memory CPU mapped outside of the gpudev library (e.g. with GPU specific library) +cannot be unmapped by the gpudev library. + Memory Barrier ~~~~~~~~~~~~~~
drivers/gpu/cuda/cuda.c +2 -0
@@ -1177,6 +1177,8 @@ cuda_gpu_probe(__rte_unused struct rte_pci_driver *pci_drv, struct rte_pci_devic dev->ops.mem_free = cuda_mem_free; dev->ops.mem_register = cuda_mem_register; dev->ops.mem_unregister = cuda_mem_unregister; + dev->ops.mem_cpu_map =NULL ; + dev->ops.mem_cpu_unmap =NULL ; dev->ops.wmb = cuda_wmb;rte_gpu_complete_new (dev);
lib/gpudev/gpudev.c +61 -0
67 lines not on this page · raw message
lib/gpudev/gpudev_driver.h +6 -0
@@ -31,6 +31,8 @@ typedef int (rte_gpu_mem_alloc_t)(struct rte_gpu *dev, size_t size, unsigned inttypedef int (rte_gpu_mem_free_t )(struct rte_gpu *dev,void *ptr);typedef int (rte_gpu_mem_register_t )(struct rte_gpu *dev,size_t size,void *ptr);typedef int (rte_gpu_mem_unregister_t )(struct rte_gpu *dev,void *ptr); +typedef int (rte_gpu_mem_cpu_map_t )(struct rte_gpu *dev,size_t size,void *ptr_in,void **ptr_out); +typedef int (rte_gpu_mem_cpu_unmap_t )(struct rte_gpu *dev,void *ptr);typedef int (rte_gpu_wmb_t )(struct rte_gpu *dev);struct rte_gpu_ops {
@@ -46,6 +48,10 @@ struct rte_gpu_ops {rte_gpu_mem_register_t *mem_register;/* Unregister CPU memory from device. */ rte_gpu_mem_unregister_t *mem_unregister; +/* Map GPU memory for CPU visibility. */ +rte_gpu_mem_cpu_map_t *mem_cpu_map; +/* Unmap GPU memory for CPU visibility. */ +rte_gpu_mem_cpu_unmap_t *mem_cpu_unmap;/* Enforce GPU write memory barrier. */ rte_gpu_wmb_t *wmb; };
lib/gpudev/rte_gpudev.h +49 -0
55 lines not on this page · raw message
lib/gpudev/version.map +2 -0
@@ -20,8 +20,10 @@ EXPERIMENTAL { rte_gpu_init; rte_gpu_is_valid; rte_gpu_mem_alloc; + rte_gpu_mem_cpu_map; rte_gpu_mem_free; rte_gpu_mem_register; + rte_gpu_mem_cpu_unmap; rte_gpu_mem_unregister; rte_gpu_wmb; };
Signature · 1 line
2.17.1
PATCH v5 2/2 app/test-gpudev: test cpu_map/cpu_unmap functions
From: Elena Agostini <[email protected]>
New test case added to test the gpudev cpu_map/cpu_unmap functions.
--- app/test-gpudev/main.c | 63 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+)
app/test-gpudev/main.c +63 -0
@@ -185,6 +185,68 @@ register_cpu_memory(uint16_t gpu_id)return -1 ; } +static int +gpu_mem_cpu_map (uint16_t gpu_id) +{ +void *ptr_gpu =NULL ; +void *ptr_cpu =NULL ; +size_t buf_bytes =1024 ; +unsigned int align =4096 ; +int ret; + +printf ("\n=======> TEST: Map GPU memory for CPU visibility\n\n" ); + +/* Alloc memory on GPU 0 with 4kB alignment */ + ptr_gpu =rte_gpu_mem_alloc (gpu_id, buf_bytes, align); +if (ptr_gpu ==NULL ) { +fprintf (stderr,"rte_gpu_mem_alloc GPU memory returned error\n" ); + goto error; + } +printf ("GPU memory allocated at 0x%p size is %zd bytes\n" , + ptr_gpu, buf_bytes); + + ptr_cpu =rte_gpu_mem_cpu_map (gpu_id, buf_bytes, ptr_gpu); +if (ptr_cpu ==NULL ) { +fprintf (stderr,"rte_gpu_mem_cpu_map returned error\n" ); + goto error; + } +printf ("GPU memory mapped for CPU access at 0x%p\n" , ptr_cpu); + + ((uint8_t *)ptr_cpu)[0 ] =0x4 ; + ((uint8_t *)ptr_cpu)[1 ] =0x5 ; + ((uint8_t *)ptr_cpu)[2 ] =0x6 ; + +printf ("GPU memory first 3 bytes set from CPU: %x %x %x\n" , + ((uint8_t *)ptr_cpu)[0 ], + ((uint8_t *)ptr_cpu)[1 ], + ((uint8_t *)ptr_cpu)[2 ]); + + ret =rte_gpu_mem_cpu_unmap (gpu_id, ptr_cpu); +if (ret <0 ) { +fprintf (stderr,"rte_gpu_mem_cpu_unmap returned error %d\n" , ret); + goto error; + } +printf ("GPU memory mapped for CPU access at 0x%p\n" , ptr_cpu); + + ret =rte_gpu_mem_free (gpu_id, ptr_gpu); +if (ret <0 ) { +fprintf (stderr,"rte_gpu_mem_free returned error %d\n" , ret); + goto error; + } +printf ("GPU memory 0x%p freed\n" , ptr_gpu); + +printf ("\n=======> TEST: PASSED\n" ); +return 0 ; + +error: + +rte_gpu_mem_cpu_unmap (gpu_id, ptr_cpu); +rte_gpu_mem_free (gpu_id, ptr_gpu); + +printf ("\n=======> TEST: FAILED\n" ); +return -1 ; +} +static int create_update_comm_flag (uint16_t gpu_id) {
@@ -402,6 +464,7 @@ main(int argc, char **argv)*/ alloc_gpu_memory (gpu_id);register_cpu_memory (gpu_id); +gpu_mem_cpu_map (gpu_id);/** * Communication items test
Signature · 1 line
2.17.1
Re: [PATCH v5 1/2] gpudev: expose GPU memory to CPU
In reply to [email protected]27/01/2022 04:50, [email protected]:
Quoted 5 lines
From: Elena Agostini <[email protected]>
Enable the possibility to expose a GPU memory area and make it accessible from the CPU.
Quoted 7 lines
GPU memory has to be allocated via
rte_gpu_mem_alloc().This patch allows the gpudev library to map (and unmap), through the GPU driver, a chunk of GPU memory and to return a memory pointer usable by the CPU to access the GPU memory area.
Signed-off-by Elena Agostini <[email protected]>
Squashed and applied, thanks.