gpu/cuda: introduce CUDA driver

RFC PATCH Superseded [email protected] 4 participants 28 messages Oct 5, 2021 – Nov 16, 2021

Subsystems (2)

DRM drivers Arm/Amlogic Meson SoC support

lore

Share & tools

lore

[email protected]

author

2021-10-05 14:39 UTC #0 lore raw

RFC PATCH 1/1 gpu/cuda: introduce CUDA driver

From: Elena Agostini <[email protected]>

This is the CUDA implementation of the gpudev library. Funcitonalities implemented through CUDA Driver API are:

- Device probe and remove
- Manage device memory allocations
- Register/unregister external CPU memory in the device memory area

Signed-off-by Elena Agostini <[email protected]>

---
 drivers/gpu/cuda/cuda.c      | 751 +++++++++++++++++++++++++++++++++++
 drivers/gpu/cuda/meson.build |  30 ++
 drivers/gpu/cuda/version.map |   3 +
 drivers/gpu/meson.build      |   2 +-
 4 files changed, 785 insertions(+), 1 deletion(-)
 create mode 100644 drivers/gpu/cuda/cuda.c
 create mode 100644 drivers/gpu/cuda/meson.build
 create mode 100644 drivers/gpu/cuda/version.map

drivers/gpu/cuda/cuda.c new file +751 -0

new file mode 100644
index 0000000000..202f0a0c0c
@@ -0,0 +1,751 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+*/
+
+#include <rte_common.h>
+#include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_errno.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+#include <rte_byteorder.h>
+#include <rte_dev.h>
+
+#include <gpudev_driver.h>
+#include <cuda.h>
+
+/* NVIDIA GPU vendor */
+#define NVIDIA_GPU_VENDOR_ID (0x10de)
+
+/* NVIDIA GPU device IDs */
+#define NVIDIA_GPU_A100_40GB_DEVICE_ID (0x20f1)
+#define NVIDIA_GPU_V100_32GB_DEVICE_ID (0x1db6)
+
+#define CUDA_MAX_ALLOCATION_NUM 512
+
+#define GPU_PAGE_SHIFT 16
+#define GPU_PAGE_SIZE (1UL << GPU_PAGE_SHIFT)
+
+RTE_LOG_REGISTER_DEFAULT(gpu_logtype, NOTICE);
+
+/** Helper macro for logging */
+#define rte_gpu_log(level, fmt, ...) \
+	rte_log(RTE_LOG_ ## level, gpu_logtype, fmt "\n", ##__VA_ARGS__)
+
+#define rte_gpu_log_debug(fmt, ...) \
+	rte_gpu_log(DEBUG, RTE_STR(__LINE__) ":%s() " fmt, __func__, \
+		##__VA_ARGS__)
+
+/* NVIDIA GPU address map */
+static struct rte_pci_id pci_id_cuda_map[] = {
+	{
+		RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
+				NVIDIA_GPU_A100_40GB_DEVICE_ID)
+	},
+	{
+		RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
+				NVIDIA_GPU_V100_32GB_DEVICE_ID)
+	},
+	/* {.device_id = 0}, ?? */
+};
+
+/* Device private info */
+struct cuda_info {
+    char gpu_name[RTE_DEV_NAME_MAX_LEN];
+    CUdevice cu_dev;
+};
+
+/* Type of memory allocated by CUDA driver */
+enum mem_type {
+	GPU_MEM = 0,
+	CPU_REGISTERED,
+	GPU_REGISTERED /* Not used yet */
+};
+
+/* key associated to a memory address */
+typedef uintptr_t ptr_key;
+
+/* Single entry of the memory list */
+struct mem_entry {
+    CUdeviceptr ptr_d;
+	void * ptr_h;
+    size_t size;
+	struct rte_gpu *dev;
+	CUcontext ctx;
+	ptr_key pkey;
+	enum mem_type mtype;
+	struct mem_entry * prev;
+	struct mem_entry * next;
+};
+
+struct mem_entry * mem_alloc_list_head = NULL;
+struct mem_entry * mem_alloc_list_tail = NULL;
+uint32_t mem_alloc_list_last_elem = 0;
+
+/* Generate a key from a memory pointer */
+static ptr_key
+get_hash_from_ptr(void * ptr)
+{
+	return (uintptr_t) ptr;
+}
+
+static uint32_t
+mem_list_count_item(void)
+{
+	return mem_alloc_list_last_elem;
+}
+
+/* Initiate list of memory allocations if not done yet */
+static struct mem_entry *
+mem_list_add_item(void)
+{
+	/* Initiate list of memory allocations if not done yet */
+	if(mem_alloc_list_head == NULL)
+	{
+		mem_alloc_list_head = rte_zmalloc(NULL, sizeof(struct mem_entry), RTE_CACHE_LINE_SIZE);
+		if (mem_alloc_list_head == NULL) {
+			rte_gpu_log(ERR, "Failed to allocate memory for memory list.\n");
+			return NULL;
+		}
+
+		mem_alloc_list_head->next = NULL;
+		mem_alloc_list_head->prev = NULL;
+		mem_alloc_list_tail = mem_alloc_list_head;
+	}
+	else
+	{
+		struct mem_entry * mem_alloc_list_cur = rte_zmalloc(NULL, sizeof(struct mem_entry), RTE_CACHE_LINE_SIZE);
+		if (mem_alloc_list_cur == NULL) {
+			rte_gpu_log(ERR, "Failed to allocate memory for memory list.\n");
+			return NULL;
+		}
+
+		mem_alloc_list_tail->next = mem_alloc_list_cur;
+		mem_alloc_list_cur->prev = mem_alloc_list_tail;
+		mem_alloc_list_tail = mem_alloc_list_tail->next;
+		mem_alloc_list_tail->next = NULL;
+	}
+
+	mem_alloc_list_last_elem++;
+
+	return mem_alloc_list_tail;
+}
+
+static struct mem_entry *
+mem_list_find_item(ptr_key pk)
+{
+	struct mem_entry * mem_alloc_list_cur = NULL;
+
+	if( mem_alloc_list_head == NULL )
+	{
+		rte_gpu_log(ERR, "Memory list doesn't exist\n");
+		return NULL;
+	}
+
+	if(mem_list_count_item() == 0)
+	{
+		rte_gpu_log(ERR, "No items in memory list\n");
+		return NULL;
+	}
+
+	mem_alloc_list_cur = mem_alloc_list_head;
+
+	while(mem_alloc_list_cur != NULL)
+	{
+		if(mem_alloc_list_cur->pkey == pk)
+			return mem_alloc_list_cur;
+		mem_alloc_list_cur = mem_alloc_list_cur->next;
+	}
+
+	return mem_alloc_list_cur;
+}
+
+static int
+mem_list_del_item(ptr_key pk)
+{
+	struct mem_entry * mem_alloc_list_cur = NULL;
+
+	mem_alloc_list_cur = mem_list_find_item(pk);
+	if(mem_alloc_list_cur == NULL)
+		return -EINVAL;
+
+	/* if key is in head */
+	if(mem_alloc_list_cur->prev == NULL)
+		mem_alloc_list_head = mem_alloc_list_cur->next;
+	else
+	{
+		mem_alloc_list_cur->prev->next = mem_alloc_list_cur->next;
+		if(mem_alloc_list_cur->next != NULL)
+			mem_alloc_list_cur->next->prev = mem_alloc_list_cur->prev;
+	}
+
+	rte_free(mem_alloc_list_cur);
+
+	mem_alloc_list_last_elem--;
+
+	return 0;
+}
+
+static int
+cuda_dev_info_get(struct rte_gpu *dev, struct rte_gpu_info *info)
+{
+	int ret = 0;
+	CUresult res;
+	struct rte_gpu_info parent_info;
+	CUexecAffinityParam affinityPrm;
+	const char * err_string;
+	struct cuda_info * private;
+	CUcontext current_ctx;
+	CUcontext input_ctx;
+
+	if(dev == NULL)
+		return -EINVAL;
+
+	/* Child initialization time probably called by rte_gpu_add_child() */
+	if(
+		dev->mpshared->info.parent != RTE_GPU_ID_NONE &&
+		dev->mpshared->dev_private == NULL
+	)
+	{
+		/* Store current ctx */
+		res = cuCtxGetCurrent(&current_ctx);
+		if(CUDA_SUCCESS != res)
+		{
+			cuGetErrorString(res, &(err_string));
+			rte_gpu_log(ERR, "cuCtxGetCurrent failed with %s.\n", err_string);
+
+			return -1;
+		}
+
+		/* Set child ctx as current ctx */
+		input_ctx = (CUcontext)dev->mpshared->info.context;
+		res = cuCtxSetCurrent(input_ctx);
+		if(CUDA_SUCCESS != res)
+		{
+			cuGetErrorString(res, &(err_string));
+			rte_gpu_log(ERR, "cuda_dev_info_get cuCtxSetCurrent input failed with %s.\n", err_string);
+
+			return -1;
+		}
+
+		/*
+		* Ctx capacity info
+		*/
+
+		/* MPS compatible */
+		res = cuCtxGetExecAffinity(&affinityPrm, CU_EXEC_AFFINITY_TYPE_SM_COUNT);
+		if(CUDA_SUCCESS != res)
+		{
+			cuGetErrorString(res, &(err_string));
+			rte_gpu_log(ERR, "cuCtxGetExecAffinity failed with %s.\n", err_string);
+		}
+		dev->mpshared->info.processor_count = (uint32_t)affinityPrm.param.smCount.val;
+
+		ret = rte_gpu_info_get(dev->mpshared->info.parent, &parent_info);
+		if (ret)
+			return -ENODEV;
+		dev->mpshared->info.total_memory = parent_info.total_memory;
+
+		/*
+		* GPU Device private info
+		*/
+		dev->mpshared->dev_private = rte_zmalloc(NULL, sizeof(struct cuda_info), RTE_CACHE_LINE_SIZE);
+		if (dev->mpshared->dev_private == NULL) {
+			rte_gpu_log(ERR, "Failed to allocate memory for GPU process private.\n");
+
+			return -1;
+		}
+
+		private = (struct cuda_info *)dev->mpshared->dev_private;
+
+		res = cuCtxGetDevice(&(private->cu_dev));
+		if(CUDA_SUCCESS != res)
+		{
+			cuGetErrorString(res, &(err_string));
+			rte_gpu_log(ERR, "cuCtxGetDevice failed with %s.\n", err_string);
+
+			return -1;
+		}
+
+		res = cuDeviceGetName(private->gpu_name, RTE_DEV_NAME_MAX_LEN, private->cu_dev);
+		if(CUDA_SUCCESS != res)
+		{
+			cuGetErrorString(res, &(err_string));
+			rte_gpu_log(ERR, "cuDeviceGetName failed with %s.\n", err_string);
+
+			return -1;
+		}
+
+		/* Restore original ctx as current ctx */
+		res = cuCtxSetCurrent(current_ctx);
+		if(CUDA_SUCCESS != res)
+		{
+			cuGetErrorString(res, &(err_string));
+			rte_gpu_log(ERR, "cuda_dev_info_get cuCtxSetCurrent current failed with %s.\n", err_string);
+
+			return -1;
+		}
+	}
+
+	*info = dev->mpshared->info;
+
+	return 0;
+}
+
+/*
+ * GPU Memory
+ */
+
+static int
+cuda_mem_alloc(struct rte_gpu * dev, size_t size, void ** ptr)
+{
+	CUresult res;
+	const char * err_string;
+	CUcontext current_ctx;
+	CUcontext input_ctx;
+	unsigned int flag = 1;
+
+	if(dev == NULL || size == 0)
+		return -EINVAL;
+
+	/* Store current ctx */
+	res = cuCtxGetCurrent(&current_ctx);
+	if(CUDA_SUCCESS != res)
+	{
+		cuGetErrorString(res, &(err_string));
+		rte_gpu_log(ERR, "cuCtxGetCurrent failed with %s.\n", err_string);
+
+		return -1;
+	}
+
+	/* Set child ctx as current ctx */
+	input_ctx = (CUcontext)dev->mpshared->info.context;
+	res = cuCtxSetCurrent(input_ctx);
+	if(CUDA_SUCCESS != res)
+	{
+		cuGetErrorString(res, &(err_string));
+		rte_gpu_log(ERR, "cuda_mem_alloc cuCtxSetCurrent input failed with %s.\n", err_string);
+
+		return -1;
+	}
+
+	/* Get next memory list item */
+	mem_alloc_list_tail = mem_list_add_item();
+	if(mem_alloc_list_tail == NULL)
+		return -ENOMEM;
+
+	/* Allocate memory */
+	mem_alloc_list_tail->size = size;
+	res = cuMemAlloc(&(mem_alloc_list_tail->ptr_d), mem_alloc_list_tail->size);
+	if (CUDA_SUCCESS != res) {
+		cuGetErrorString(res, &(err_string));
+		rte_gpu_log(ERR, "cuda_mem_alloc cuCtxSetCurrent current failed with %s.\n", err_string);
+
+		return -1;
+	}
+
+	/* GPUDirect RDMA attribute required */
+	res = cuPointerSetAttribute(&flag, CU_POINTER_ATTRIBUTE_SYNC_MEMOPS, mem_alloc_list_tail->ptr_d);
+	if (CUDA_SUCCESS != res) {
+		rte_gpu_log(ERR, "Could not set SYNC MEMOP attribute for GPU memory at %llx , err %d\n", mem_alloc_list_tail->ptr_d, res);
+		return -1;
+	}
+
+	mem_alloc_list_tail->pkey = get_hash_from_ptr((void *) mem_alloc_list_tail->ptr_d);
+	mem_alloc_list_tail->ptr_h = NULL;
+	mem_alloc_list_tail->size = size;
+	mem_alloc_list_tail->dev = dev;
+	mem_alloc_list_tail->ctx = (CUcontext)dev->mpshared->info.context;
+	mem_alloc_list_tail->mtype = GPU_MEM;
+
+	/* Restore original ctx as current ctx */
+	res = cuCtxSetCurrent(current_ctx);
+	if(CUDA_SUCCESS != res)
+	{
+		cuGetErrorString(res, &(err_string));
+		rte_gpu_log(ERR, "cuda_mem_alloc cuCtxSetCurrent current failed with %s.\n", err_string);
+
+		return -1;
+	}
+
+	*ptr = (void*) mem_alloc_list_tail->ptr_d;
+
+	return 0;
+}
+
+static int
+cuda_mem_register(struct rte_gpu * dev, size_t size, void * ptr)
+{
+	CUresult res;
+	const char * err_string;
+	CUcontext current_ctx;
+	CUcontext input_ctx;
+	unsigned int flag = 1;
+	int use_ptr_h = 0;
+
+	if(dev == NULL || size == 0 || ptr == NULL)
+		return -EINVAL;
+
+	/* Store current ctx */
+	res = cuCtxGetCurrent(&current_ctx);
+	if(CUDA_SUCCESS != res)
+	{
+		cuGetErrorString(res, &(err_string));
+		rte_gpu_log(ERR, "cuCtxGetCurrent failed with %s.\n", err_string);
+
+		return -1;
+	}
+
+	/* Set child ctx as current ctx */
+	input_ctx = (CUcontext)dev->mpshared->info.context;
+	res = cuCtxSetCurrent(input_ctx);
+	if(CUDA_SUCCESS != res)
+	{
+		cuGetErrorString(res, &(err_string));
+		rte_gpu_log(ERR, "cuda_mem_register cuCtxSetCurrent input failed with %s.\n", err_string);
+
+		return -1;
+	}
+
+	/* Get next memory list item */
+	mem_alloc_list_tail = mem_list_add_item();
+	if(mem_alloc_list_tail == NULL)
+		return -ENOMEM;
+
+	/* Allocate memory */
+	mem_alloc_list_tail->size = size;
+	mem_alloc_list_tail->ptr_h = ptr;
+
+	res = cuMemHostRegister(mem_alloc_list_tail->ptr_h, mem_alloc_list_tail->size, CU_MEMHOSTREGISTER_PORTABLE | CU_MEMHOSTREGISTER_DEVICEMAP);
+	if (CUDA_SUCCESS != res) {
+		cuGetErrorString(res, &(err_string));
+		rte_gpu_log(ERR, "cuda_mem_register cuMemHostRegister failed with %s ptr %p size %zd.\n",
+						err_string, mem_alloc_list_tail->ptr_h, mem_alloc_list_tail->size
+					);
+
+		return -1;
+	}
+
+	res = cuDeviceGetAttribute(&(use_ptr_h),
+									CU_DEVICE_ATTRIBUTE_CAN_USE_HOST_POINTER_FOR_REGISTERED_MEM,
+									((struct cuda_info *)(dev->mpshared->dev_private))->cu_dev
+								);
+	if(CUDA_SUCCESS != res)
+	{
+		cuGetErrorString(res, &(err_string));
+		rte_gpu_log(ERR, "cuDeviceGetAttribute failed with %s.\n",
+					err_string
+			);
+
+		return -1;
+	}
+
+	if(use_ptr_h == 0)
+	{
+		res = cuMemHostGetDevicePointer(&(mem_alloc_list_tail->ptr_d), mem_alloc_list_tail->ptr_h, 0);
+		if (CUDA_SUCCESS != res) {
+			cuGetErrorString(res, &(err_string));
+			rte_gpu_log(ERR, "cuMemHostGetDevicePointer failed with %s.\n", err_string);
+
+			return -1;
+		}
+
+		if((uintptr_t) mem_alloc_list_tail->ptr_d != (uintptr_t) mem_alloc_list_tail->ptr_h)
+		{
+			rte_gpu_log(ERR, "Host input pointer is different wrt GPU registered pointer\n");
+			return -1;
+		}
+	}
+	else
+		mem_alloc_list_tail->ptr_d = (CUdeviceptr) mem_alloc_list_tail->ptr_h;
+
+	/* GPUDirect RDMA attribute required */
+	res = cuPointerSetAttribute(&flag, CU_POINTER_ATTRIBUTE_SYNC_MEMOPS, mem_alloc_list_tail->ptr_d);
+	if (CUDA_SUCCESS != res) {
+		rte_gpu_log(ERR, "Could not set SYNC MEMOP attribute for GPU memory at %llx , err %d\n", mem_alloc_list_tail->ptr_d, res);
+		return -1;
+	}
+
+	mem_alloc_list_tail->pkey = get_hash_from_ptr((void *) mem_alloc_list_tail->ptr_h);
+	mem_alloc_list_tail->size = size;
+	mem_alloc_list_tail->dev = dev;
+	mem_alloc_list_tail->ctx = (CUcontext)dev->mpshared->info.context;
+	mem_alloc_list_tail->mtype = CPU_REGISTERED;
+
+	/* Restore original ctx as current ctx */
+	res = cuCtxSetCurrent(current_ctx);
+	if(CUDA_SUCCESS != res)
+	{
+		cuGetErrorString(res, &(err_string));
+		rte_gpu_log(ERR, "cuda_mem_register cuCtxSetCurrent current failed with %s.\n", err_string);
+
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+cuda_mem_free(struct rte_gpu * dev, void * ptr)
+{
+	CUresult res;
+	struct mem_entry * mem_item;
+	const char * err_string;
+	ptr_key hk;
+
+	if(dev == NULL || ptr == NULL)
+		return -EINVAL;
+
+	hk = get_hash_from_ptr((void *) ptr);
+
+	mem_item = mem_list_find_item(hk);
+	if(mem_item == NULL)
+	{
+		rte_gpu_log(ERR, "Memory address 0x%p not found in driver memory\n", ptr);
+		return -1;
+	}
+
+	if(mem_item->mtype == GPU_MEM)
+	{
+		res = cuMemFree(mem_item->ptr_d);
+		if(CUDA_SUCCESS != res)
+		{
+			cuGetErrorString(res, &(err_string));
+			rte_gpu_log(ERR, "cuMemFree current failed with %s.\n", err_string);
+
+			return -1;
+		}
+
+		return mem_list_del_item(hk);
+
+	}
+	else
+	{
+		rte_gpu_log(ERR, "Memory type %d not supported\n", mem_item->mtype);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+cuda_mem_unregister(struct rte_gpu * dev, void * ptr)
+{
+	CUresult res;
+	struct mem_entry * mem_item;
+	const char * err_string;
+	ptr_key hk;
+
+	if(dev == NULL || ptr == NULL)
+		return -EINVAL;
+
+	hk = get_hash_from_ptr((void *) ptr);
+
+	mem_item = mem_list_find_item(hk);
+	if(mem_item == NULL)
+	{
+		rte_gpu_log(ERR, "Memory address 0x%p not nd in driver memory\n", ptr);
+		return -1;
+	}
+
+	if(mem_item->mtype == CPU_REGISTERED)
+	{
+		res = cuMemHostUnregister(ptr);
+		if(CUDA_SUCCESS != res)
+		{
+			cuGetErrorString(res, &(err_string));
+			rte_gpu_log(ERR, "cuMemHostUnregister current failed with %s.\n", err_string);
+
+			return -1;
+		}
+
+		return mem_list_del_item(hk);
+	}
+	else
+	{
+		rte_gpu_log(ERR, "Memory type %d not supported\n", mem_item->mtype);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+cuda_dev_close(struct rte_gpu * dev)
+{
+	if (dev == NULL)
+		return -EINVAL;
+
+	rte_free(dev->mpshared->dev_private);
+
+	return 0;
+}
+
+static int
+cuda_gpu_probe(__rte_unused struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
+{
+	struct rte_gpu *dev = NULL;
+	CUresult res;
+	CUdevice cu_dev_id;
+	CUcontext pctx;
+	char dev_name[RTE_DEV_NAME_MAX_LEN];
+	const char * err_string;
+	int processor_count = 0;
+	struct cuda_info * private;
+
+	if (pci_dev == NULL) {
+		rte_gpu_log(ERR, "NULL PCI device");
+		return -EINVAL;
+	}
+
+	rte_pci_device_name(&pci_dev->addr, dev_name, sizeof(dev_name));
+
+	/* Allocate memory to be used privately by drivers */
+	dev = rte_gpu_allocate(pci_dev->device.name);
+	if (dev == NULL)
+		return -ENODEV;
+
+	/* Fill HW specific part of device structure */
+	dev->device = &pci_dev->device;
+	dev->mpshared->info.numa_node = pci_dev->device.numa_node;
+
+	/*
+	 * GPU Device init
+	 */
+
+	/*
+	 * Required to initialize the CUDA Driver.
+	 * Multiple calls of cuInit() will return immediately
+	 * without making any relevant change
+	 */
+	cuInit(0);
+
+	/* Get NVIDIA GPU Device descriptor */
+	res = cuDeviceGetByPCIBusId(&cu_dev_id, dev->device->name);
+	if(CUDA_SUCCESS != res)
+	{
+		cuGetErrorString(res, &(err_string));
+		rte_gpu_log(ERR, "cuDeviceGetByPCIBusId name %s failed with %d: %s.\n",
+					dev->device->name, res, err_string
+			);
+
+		return -1;
+	}
+
+	res = cuDevicePrimaryCtxRetain(&pctx, cu_dev_id);
+	if(CUDA_SUCCESS != res)
+	{
+		cuGetErrorString(res, &(err_string));
+		rte_gpu_log(ERR, "cuDevicePrimaryCtxRetain name %s failed with %d: %s.\n",
+					dev->device->name, res, err_string
+			);
+
+		return -1;
+	}
+
+	dev->mpshared->info.context = (uint64_t) pctx;
+
+	/*
+	 * GPU Device generic info
+	 */
+
+	/* Processor count */
+	res = cuDeviceGetAttribute(&(processor_count), CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT, cu_dev_id);
+	if(CUDA_SUCCESS != res)
+	{
+		cuGetErrorString(res, &(err_string));
+		rte_gpu_log(ERR, "cuDeviceGetAttribute failed with %s.\n",
+					err_string
+			);
+
+		return -1;
+	}
+	dev->mpshared->info.processor_count = (uint32_t)processor_count;
+
+	/* Total memory */
+	res = cuDeviceTotalMem(&dev->mpshared->info.total_memory, cu_dev_id);
+	if(CUDA_SUCCESS != res)
+	{
+		cuGetErrorString(res, &(err_string));
+		rte_gpu_log(ERR, "cuDeviceTotalMem failed with %s.\n",
+					err_string
+			);
+
+		return -1;
+	}
+
+	/*
+	 * GPU Device private info
+	 */
+	dev->mpshared->dev_private = rte_zmalloc(NULL, sizeof(struct cuda_info), RTE_CACHE_LINE_SIZE);
+	if (dev->mpshared->dev_private == NULL) {
+		rte_gpu_log(ERR, "Failed to allocate memory for GPU process private.\n");
+
+		return -1;
+	}
+
+	private = (struct cuda_info *)dev->mpshared->dev_private;
+	private->cu_dev = cu_dev_id;
+	res = cuDeviceGetName(private->gpu_name, RTE_DEV_NAME_MAX_LEN, cu_dev_id);
+	if(CUDA_SUCCESS != res)
+	{
+		cuGetErrorString(res, &(err_string));
+		rte_gpu_log(ERR, "cuDeviceGetName failed with %s.\n", err_string);
+
+		return -1;
+	}
+
+	dev->ops.mem_alloc = cuda_mem_alloc;
+	dev->ops.mem_free = cuda_mem_free;
+	dev->ops.mem_register = cuda_mem_register;
+	dev->ops.mem_unregister = cuda_mem_unregister;
+	dev->ops.dev_info_get = cuda_dev_info_get;
+	dev->ops.dev_close = cuda_dev_close;
+
+	rte_gpu_complete_new(dev);
+
+	rte_gpu_log_debug("dev id = %u name = %s\n", dev->mpshared->info.dev_id, private->gpu_name);
+
+	return 0;
+}
+
+static int
+cuda_gpu_remove(struct rte_pci_device *pci_dev)
+{
+	struct rte_gpu *dev;
+	int ret;
+	uint8_t gpu_id;
+
+	if (pci_dev == NULL)
+		return -EINVAL;
+
+	dev = rte_gpu_get_by_name(pci_dev->device.name);
+	if (dev == NULL) {
+		rte_gpu_log(ERR,
+				"Couldn't find HW dev \"%s\" to uninitialise it",
+				pci_dev->device.name);
+		return -ENODEV;
+	}
+	gpu_id = dev->mpshared->info.dev_id;
+
+	/* release dev from library */
+	ret = rte_gpu_release(dev);
+	if (ret)
+		rte_gpu_log(ERR, "Device %i failed to uninit: %i", gpu_id, ret);
+
+	rte_gpu_log_debug("Destroyed dev = %u", gpu_id);
+
+	return 0;
+}
+
+static struct rte_pci_driver rte_cuda_driver = {
+	.id_table = pci_id_cuda_map,
+	.drv_flags = RTE_PCI_DRV_WC_ACTIVATE,
+	.probe = cuda_gpu_probe,
+	.remove = cuda_gpu_remove,
+};
+
+RTE_PMD_REGISTER_PCI(gpu_cuda, rte_cuda_driver);
+RTE_PMD_REGISTER_PCI_TABLE(gpu_cuda, pci_id_cuda_map);
+RTE_PMD_REGISTER_KMOD_DEP(gpu_cuda, "* nvidia & (nv_peer_mem | nvpeer_mem)");
+

drivers/gpu/cuda/meson.build new file +30 -0

new file mode 100644
index 0000000000..53e40e6832
@@ -0,0 +1,30 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+if not is_linux
+        build = false
+        reason = 'only supported on Linux'
+endif
+
+# cuda_dep = dependency('cuda-11.1', required: true, version : '>=11.1', method: 'pkg-config')
+# if not cuda_dep.found()
+#         build = false
+#         reason = 'missing dependency, "CUDA"'
+#         subdir_done()
+# endif
+# ext_deps += cuda_dep
+
+cuda_dep = dependency('cuda', version : '>=11', modules: ['cuda'])
+ext_deps += cuda_dep
+
+# cudart_dep = dependency('cudart-11.1', required: true, version : '>=11.1', method: 'pkg-config')
+# if not cudart_dep.found()
+#         build = false
+#         reason = 'missing dependency, "CUDA RT"'
+#         subdir_done()
+# endif
+# ext_deps += cudart_dep
+
+deps += ['gpudev','pci','bus_pci', 'hash']
+sources = files('cuda.c')
+# headers = files('header.h')

drivers/gpu/cuda/version.map new file +3 -0

new file mode 100644
index 0000000000..4a76d1d52d
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};

drivers/gpu/meson.build +1 -1

index e51ad3381b..601bedcd61 100644
@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright (c) 2021 NVIDIA Corporation & Affiliates
 
-drivers = []
+drivers = [ 'cuda' ]

Signature · 1 line

2.17.1

[email protected]

author

2021-11-03 17:51 UTC #1 lore raw

v2 cover gpu/cuda: introduce CUDA driver

From: Elena Agostini <[email protected]>

This is the CUDA implementation of the gpudev library. Funcitonalities implemented through CUDA Driver API are:

- Device probe and remove
- Manage device memory allocations
- Register/unregister external CPU memory in the device memory area

Changelog:

- Checkpatch validation
- GPU CUDA driver docs

Elena Agostini (1):
gpu/cuda: introduce CUDA driver

 doc/guides/gpus/cuda.rst     |  91 +++++
 doc/guides/gpus/index.rst    |   1 +
 drivers/gpu/cuda/cuda.c      | 716 +++++++++++++++++++++++++++++++++++
 drivers/gpu/cuda/meson.build |  30 ++
 drivers/gpu/cuda/version.map |   3 +
 drivers/gpu/meson.build      |   2 +-
 6 files changed, 842 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/gpus/cuda.rst
 create mode 100644 drivers/gpu/cuda/cuda.c
 create mode 100644 drivers/gpu/cuda/meson.build
 create mode 100644 drivers/gpu/cuda/version.map

Signature · 1 line

2.17.1

[email protected]

author

2021-11-03 17:51 UTC #2 lore raw

PATCH v2 1/1 gpu/cuda: introduce CUDA driver

From: Elena Agostini <[email protected]>

This is the CUDA implementation of the gpudev library. Funcitonalities implemented through CUDA Driver API are:

- Device probe and remove
- Manage device memory allocations
- Register/unregister external CPU memory in the device memory area

Signed-off-by Elena Agostini <[email protected]>

---
 doc/guides/gpus/cuda.rst     |  91 +++++
 doc/guides/gpus/index.rst    |   1 +
 drivers/gpu/cuda/cuda.c      | 716 +++++++++++++++++++++++++++++++++++
 drivers/gpu/cuda/meson.build |  30 ++
 drivers/gpu/cuda/version.map |   3 +
 drivers/gpu/meson.build      |   2 +-
 6 files changed, 842 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/gpus/cuda.rst
 create mode 100644 drivers/gpu/cuda/cuda.c
 create mode 100644 drivers/gpu/cuda/meson.build
 create mode 100644 drivers/gpu/cuda/version.map

doc/guides/gpus/cuda.rst new file +91 -0

new file mode 100644
index 0000000000..dec2f92a3e
@@ -0,0 +1,91 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+   Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+CUDA GPU driver
+===============
+
+The CUDA GPU driver library (**librte_gpu_cuda**) provides support for NVIDIA GPUs.
+Information and documentation about these devices can be found on the
+`NVIDIA website <http://www.nvidia.com>`__. Help is also provided by the
+`NVIDIA CUDA Toolkit developer zone <https://docs.nvidia.com/cuda>`__.
+
+Design
+------
+
+**librte_gpu_cuda** relies on CUDA Driver API (no need for CUDA Runtime API).
+
+Goal of this driver library is not to provide a wrapper for the whole CUDA Driver API.
+Instead, the scope is to implement the generic features of gpudev API.
+For a CUDA application, integrating the gpudev library functions using the CUDA driver library
+is quite straighforward and doesn't create any compatibility problem.
+
+Initialization
+~~~~~~~~~~~~~~
+
+During initialization, CUDA driver library detects NVIDIA physical GPUs on the
+system or specified via EAL device options (e.g. ``-a b6:00.0``).
+The driver initializes the CUDA driver environment through ``cuInit(0)`` function.
+For this reason, it's required to set any CUDA environment configuration before
+calling ``rte_eal_init`` function in the DPDK application.
+
+If the CUDA driver enviroment has been already initialized, the ``cuInit(0)``
+in CUDA driver library has no effect.
+
+CUDA Driver sub-contexts
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+After initialization, a CUDA application can create multiple sub-contexts on GPU
+physical devices. Through gpudev library, is possible to register these sub-contexts
+in the CUDA driver library as child devices having as parent a GPU physical device.
+
+CUDA driver library also supports `MPS <https://docs.nvidia.com/deploy/pdf/CUDA_Multi_Process_Service_Overview.pdf>`__.
+
+GPU memory management
+~~~~~~~~~~~~~~~~~~~~~
+
+The CUDA driver library maintains a table of GPU memory addresses allocated
+and CPU memory addresses registered associated to the input CUDA context.
+Whenever the application tried to deallocate or deregister a memory address,
+if the address is not in the table the CUDA driver library will return an error.
+
+Features
+--------
+
+- Register new child devices aka new CUDA Driver contexts
+- Allocate memory on the GPU
+- Register CPU memory to make it visible from GPU
+
+Minimal requirements
+--------------------
+
+Minimal requirements to enable the CUDA driver library are:
+
+- NVIDIA GPU Ampere or Volta
+- CUDA 11.4 Driver API or newer
+
+`GPUDirect RDMA Technology <https://docs.nvidia.com/cuda/gpudirect-rdma/index.html>`__
+allows compatible network cards (e.g. Mellanox) to directly send and receive packets
+using GPU memory instead of additional memory copies through the CPU system memory.
+To enable this technology, system requirements are:
+
+- `nvidia-peermem <https://docs.nvidia.com/cuda/gpudirect-rdma/index.html#nvidia-peermem>`__ module running on the system
+- Mellanox Network card ConnectX-5 or newer (BlueField models included)
+- DPDK mlx5 PMD enabled
+- To reach the best performance, a PCIe switch between GPU and NIC is recommended
+
+Limitations
+-----------
+
+Supported only on Linux.
+
+Supported GPUs
+--------------
+
+The following NVIDIA GPU devices are supported by this CUDA driver:
+
+- NVIDIA A100 80GB PCIe
+- NVIDIA A100 40GB PCIe
+- NVIDIA A30 24GB
+- NVIDIA A10 24GB
+- NVIDIA V100 32GB PCIe
+- NVIDIA V100 16GB PCIe

doc/guides/gpus/index.rst +1 -0

index 1878423239..4b7a420556 100644
@@ -9,3 +9,4 @@ General-Purpose Graphics Processing Unit Drivers
    :numbered:
 
    overview
+   cuda

drivers/gpu/cuda/cuda.c new file +716 -0

716 lines not on this page · raw message

drivers/gpu/cuda/meson.build new file +30 -0

new file mode 100644
index 0000000000..53e40e6832
@@ -0,0 +1,30 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+if not is_linux
+        build = false
+        reason = 'only supported on Linux'
+endif
+
+# cuda_dep = dependency('cuda-11.1', required: true, version : '>=11.1', method: 'pkg-config')
+# if not cuda_dep.found()
+#         build = false
+#         reason = 'missing dependency, "CUDA"'
+#         subdir_done()
+# endif
+# ext_deps += cuda_dep
+
+cuda_dep = dependency('cuda', version : '>=11', modules: ['cuda'])
+ext_deps += cuda_dep
+
+# cudart_dep = dependency('cudart-11.1', required: true, version : '>=11.1', method: 'pkg-config')
+# if not cudart_dep.found()
+#         build = false
+#         reason = 'missing dependency, "CUDA RT"'
+#         subdir_done()
+# endif
+# ext_deps += cudart_dep
+
+deps += ['gpudev','pci','bus_pci', 'hash']
+sources = files('cuda.c')
+# headers = files('header.h')

drivers/gpu/cuda/version.map new file +3 -0

new file mode 100644
index 0000000000..4a76d1d52d
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};

drivers/gpu/meson.build +1 -1

index e51ad3381b..601bedcd61 100644
@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright (c) 2021 NVIDIA Corporation & Affiliates
 
-drivers = []
+drivers = [ 'cuda' ]

Signature · 1 line

2.17.1

Stephen Hemminger

2021-11-03 18:15 UTC #3 lore raw

drivers/gpu/cuda/cuda.c line 35

Quoted 7 lines · On Thu, 4 Nov 2021 02:01:28 +0000 <[email protected]> wrote:

drivers/gpu/cuda/cuda.c new file +7 -0

+
+#define CUDA_MAX_ALLOCATION_NUM 512
+
+#define GPU_PAGE_SHIFT 16
+#define GPU_PAGE_SIZE (1UL << GPU_PAGE_SHIFT)
+
+RTE_LOG_REGISTER_DEFAULT(gpu_logtype, NOTICE);

Use static before this RTE_LOG_REGISTER_DEFAULT. You don't need to export gpu_log_type as global do you?

drivers/gpu/cuda/cuda.c line 89

Quoted 3 lines

drivers/gpu/cuda/cuda.c new file +3 -0

+struct mem_entry *mem_alloc_list_head = NULL;
+struct mem_entry *mem_alloc_list_tail = NULL;
+uint32_t mem_alloc_list_last_elem = 0;

These should be static since specific to this driver.

Stephen Hemminger

2021-11-08 18:35 UTC #4 lore raw

In reply to [email protected]

drivers/gpu/cuda/cuda.c line 80

Quoted 7 lines · On Thu, 4 Nov 2021 02:01:28 +0000 <[email protected]> wrote:

drivers/gpu/cuda/cuda.c new file +7 -0

+/* Single entry of the memory list */
+struct mem_entry {
+	CUdeviceptr ptr_d;
+	void *ptr_h;
+	size_t size;
+	struct rte_gpu *dev;
+	CUcontext ctx;

Not sure where these types CUdeviceptr and CUcontext are coming from, but the code looks like Windows style not DPDK or Linux.

Please don't introduce CamelCase typedef's and never typedefs for pointers.

Elena Agostini

author

2021-11-08 18:39 UTC #5 lore raw

Quoted 9 lines

From: Stephen Hemminger <[email protected]>
Date: Monday, 8 November 2021 at 19:35
To: Elena Agostini <[email protected]>
Cc: [email protected] <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v2 1/1] gpu/cuda: introduce CUDA driver External email: Use caution opening links or attachments>
On Thu, 4 Nov 2021 02:01:28 +0000
<[email protected]> wrote:>

Quoted 11 lines

drivers/gpu/cuda/cuda.c
Quoted 7 lines
drivers/gpu/cuda/cuda.c new file +7 -0
+/* Single entry of the memory list */
+struct mem_entry {
+     CUdeviceptr ptr_d;
+     void *ptr_h;
+     size_t size;
+     struct rte_gpu *dev;
+     CUcontext ctx;>
Not sure where these types CUdeviceptr and CUcontext are coming from, but the code looks like Windows style not DPDK or Linux.> Please don't introduce CamelCase typedef's and never typedefs for pointers.

These are CUDA Driver API specific types, I can’t change them.

Stephen Hemminger

2021-11-08 18:59 UTC #6 lore raw

Quoted 21 lines · On Mon, 8 Nov 2021 18:39:36 +0000 Elena Agostini <[email protected]> wrote:

Quoted 9 lines
From: Stephen Hemminger <[email protected]>
Date: Monday, 8 November 2021 at 19:35
To: Elena Agostini <[email protected]>
Cc: [email protected] <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v2 1/1] gpu/cuda: introduce CUDA driver External email: Use caution opening links or attachments>
On Thu, 4 Nov 2021 02:01:28 +0000
<[email protected]> wrote:>
Quoted 11 lines
drivers/gpu/cuda/cuda.c
Quoted 7 lines
drivers/gpu/cuda/cuda.c new file +7 -0
+/* Single entry of the memory list */
+struct mem_entry {
+     CUdeviceptr ptr_d;
+     void *ptr_h;
+     size_t size;
+     struct rte_gpu *dev;
+     CUcontext ctx;>  
Not sure where these types CUdeviceptr and CUcontext are coming from, but the code looks like Windows style not DPDK or Linux.> Please don't introduce CamelCase typedef's and never typedefs for pointers.

Quoted 1 line

These are CUDA Driver API specific types, I can’t change them.

Could you at least avoid using the pointer typedefs. When a pointer is typedef'd it leads to confusion to readers and some dumb static analyzers.

Elena Agostini

author

2021-11-08 19:07 UTC #7 lore raw

Quoted 28 lines

From: Stephen Hemminger <[email protected]>
Date: Monday, 8 November 2021 at 19:59
To: Elena Agostini <[email protected]>
Cc: [email protected] <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v2 1/1] gpu/cuda: introduce CUDA driver> On Mon, 8 Nov 2021 18:39:36 +0000
Elena Agostini <[email protected]> wrote:>
Quoted 21 lines
Quoted 20 lines
From: Stephen Hemminger <[email protected]>
Date: Monday, 8 November 2021 at 19:35
To: Elena Agostini <[email protected]>
Cc: [email protected] <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v2 1/1] gpu/cuda: introduce CUDA driver External email: Use caution opening links or attachments>
On Thu, 4 Nov 2021 02:01:28 +0000
<[email protected]> wrote:>
drivers/gpu/cuda/cuda.c
Quoted 7 lines
drivers/gpu/cuda/cuda.c new file +7 -0
+/* Single entry of the memory list */
+struct mem_entry {
+     CUdeviceptr ptr_d;
+     void *ptr_h;
+     size_t size;
+     struct rte_gpu *dev;
+     CUcontext ctx;>
Not sure where these types CUdeviceptr and CUcontext are coming from, but the code looks like Windows style not DPDK or Linux.> Please don't introduce CamelCase typedef's and never typedefs for pointers.

Quoted 4 lines

Quoted 1 line
These are CUDA Driver API specific types, I can’t change them.>
Could you at least avoid using the pointer typedefs. When a pointer is typedef'd it leads to confusion to readers and some dumb static analyzers.

There is not pointer typedef here.
The typedef is about uintptr_t

[email protected]

author

2021-11-08 18:17 UTC #8 lore raw

v3 cover gpu/cuda: introduce CUDA driver

In reply to [email protected]

From: Elena Agostini <[email protected]>

This is the CUDA implementation of the gpudev library. Funcitonalities implemented through CUDA Driver API are:

- Device probe and remove
- Manage device memory allocations
- Register/unregister external CPU memory in the device memory area

Changelog:
- CUDA driver implementation of the GPU write memory barrier
- Fixed styling reported by checkpatch

Elena Agostini (1):
gpu/cuda: introduce CUDA driver

 doc/guides/gpus/cuda.rst     | 110 +++++
 doc/guides/gpus/index.rst    |   1 +
 drivers/gpu/cuda/cuda.c      | 801 +++++++++++++++++++++++++++++++++++
 drivers/gpu/cuda/meson.build |  13 +
 drivers/gpu/cuda/version.map |   3 +
 drivers/gpu/meson.build      |   2 +-
 6 files changed, 929 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/gpus/cuda.rst
 create mode 100644 drivers/gpu/cuda/cuda.c
 create mode 100644 drivers/gpu/cuda/meson.build
 create mode 100644 drivers/gpu/cuda/version.map

Signature · 1 line

2.17.1

[email protected]

author

2021-11-08 18:17 UTC #9 lore raw

PATCH v3 1/1 gpu/cuda: introduce CUDA driver

From: Elena Agostini <[email protected]>

This is the CUDA implementation of the gpudev library. Funcitonalities implemented through CUDA Driver API are:

- Device probe and remove
- Manage device memory allocations
- Register/unregister external CPU memory in the device memory area

Signed-off-by Elena Agostini <[email protected]>

---
 doc/guides/gpus/cuda.rst     | 110 +++++
 doc/guides/gpus/index.rst    |   1 +
 drivers/gpu/cuda/cuda.c      | 801 +++++++++++++++++++++++++++++++++++
 drivers/gpu/cuda/meson.build |  13 +
 drivers/gpu/cuda/version.map |   3 +
 drivers/gpu/meson.build      |   2 +-
 6 files changed, 929 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/gpus/cuda.rst
 create mode 100644 drivers/gpu/cuda/cuda.c
 create mode 100644 drivers/gpu/cuda/meson.build
 create mode 100644 drivers/gpu/cuda/version.map

doc/guides/gpus/cuda.rst new file +110 -0

new file mode 100644
index 0000000000..64a78bf1e1
@@ -0,0 +1,110 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+   Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+CUDA GPU driver
+===============
+
+The CUDA GPU driver library (**librte_gpu_cuda**) provides support for NVIDIA GPUs.
+Information and documentation about these devices can be found on the
+`NVIDIA website <http://www.nvidia.com>`__. Help is also provided by the
+`NVIDIA CUDA Toolkit developer zone <https://docs.nvidia.com/cuda>`__.
+
+Design
+------
+
+**librte_gpu_cuda** relies on CUDA Driver API (no need for CUDA Runtime API).
+
+Goal of this driver library is not to provide a wrapper for the whole CUDA Driver API.
+Instead, the scope is to implement the generic features of gpudev API.
+For a CUDA application, integrating the gpudev library functions using the CUDA driver library
+is quite straightforward and doesn't create any compatibility problem.
+
+Initialization
+~~~~~~~~~~~~~~
+
+During initialization, CUDA driver library detects NVIDIA physical GPUs on the
+system or specified via EAL device options (e.g. ``-a b6:00.0``).
+The driver initializes the CUDA driver environment through ``cuInit(0)`` function.
+For this reason, it's required to set any CUDA environment configuration before
+calling ``rte_eal_init`` function in the DPDK application.
+
+If the CUDA driver environment has been already initialized, the ``cuInit(0)``
+in CUDA driver library has no effect.
+
+CUDA Driver sub-contexts
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+After initialization, a CUDA application can create multiple sub-contexts on GPU
+physical devices. Through gpudev library, is possible to register these sub-contexts
+in the CUDA driver library as child devices having as parent a GPU physical device.
+
+CUDA driver library also supports `MPS <https://docs.nvidia.com/deploy/pdf/CUDA_Multi_Process_Service_Overview.pdf>`__.
+
+GPU memory management
+~~~~~~~~~~~~~~~~~~~~~
+
+The CUDA driver library maintains a table of GPU memory addresses allocated
+and CPU memory addresses registered associated to the input CUDA context.
+Whenever the application tried to deallocate or deregister a memory address,
+if the address is not in the table the CUDA driver library will return an error.
+
+Features
+--------
+
+- Register new child devices aka new CUDA Driver contexts
+- Allocate memory on the GPU
+- Register CPU memory to make it visible from GPU
+
+Minimal requirements
+--------------------
+
+Minimal requirements to enable the CUDA driver library are:
+
+- NVIDIA GPU Ampere or Volta
+- CUDA 11.4 Driver API or newer
+
+`GPUDirect RDMA Technology <https://docs.nvidia.com/cuda/gpudirect-rdma/index.html>`__
+allows compatible network cards (e.g. Mellanox) to directly send and receive packets
+using GPU memory instead of additional memory copies through the CPU system memory.
+To enable this technology, system requirements are:
+
+- `nvidia-peermem <https://docs.nvidia.com/cuda/gpudirect-rdma/index.html#nvidia-peermem>`__ module running on the system
+- Mellanox Network card ConnectX-5 or newer (BlueField models included)
+- DPDK mlx5 PMD enabled
+- To reach the best performance, a PCIe switch between GPU and NIC is recommended
+
+Limitations
+-----------
+
+Supported only on Linux.
+
+Supported GPUs
+--------------
+
+The following NVIDIA GPU devices are supported by this CUDA driver:
+
+- NVIDIA A100 80GB PCIe
+- NVIDIA A100 40GB PCIe
+- NVIDIA A30 24GB
+- NVIDIA A10 24GB
+- NVIDIA V100 32GB PCIe
+- NVIDIA V100 16GB PCIe
+
+External references
+-------------------
+
+A good example of how to use the GPU CUDA driver through the gpudev library
+is the l2fwd-nv application that can be found `here <https://github.com/NVIDIA/l2fwd-nv>`__.
+
+The application is based on vanilla DPDK example l2fwd and it's enhanced with GPU memory
+managed through gpudev library and CUDA to launch the swap of packets' MAC addresses workload
+on the GPU.
+
+l2fwd-nv is not intended to be used for performance (testpmd is the good candidate for this).
+The goal is to show different use-cases about how a CUDA application can use DPDK to:
+
+- allocate memory on GPU device using gpudev library
+- use that memory to create an external GPU memory mempool
+- receive packets directly in GPU memory
+- coordinate the workload on the GPU with the network and CPU activity to receive packets
+- send modified packets directly from the GPU memory

doc/guides/gpus/index.rst +1 -0

index 1878423239..4b7a420556 100644
@@ -9,3 +9,4 @@ General-Purpose Graphics Processing Unit Drivers
    :numbered:
 
    overview
+   cuda

drivers/gpu/cuda/cuda.c new file +801 -0

801 lines not on this page · raw message

drivers/gpu/cuda/meson.build new file +13 -0

new file mode 100644
index 0000000000..92b30c35b4
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+if not is_linux
+        build = false
+        reason = 'only supported on Linux'
+endif
+
+cuda_dep = dependency('cuda', version : '>=11', modules: ['cuda'])
+ext_deps += cuda_dep
+
+deps += ['gpudev','pci','bus_pci', 'hash']
+sources = files('cuda.c')

drivers/gpu/cuda/version.map new file +3 -0

new file mode 100644
index 0000000000..4a76d1d52d
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};

drivers/gpu/meson.build +1 -1

index e51ad3381b..601bedcd61 100644
@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright (c) 2021 NVIDIA Corporation & Affiliates
 
-drivers = []
+drivers = [ 'cuda' ]

Signature · 1 line

2.17.1

David Marchand

2021-11-08 19:52 UTC #10 lore raw

Quoted 6 lines · On Mon, Nov 8, 2021 at 7:17 PM <[email protected]> wrote:

drivers/gpu/cuda/meson.build new file +0 -0
new file mode 100644
index 0000000000..92b30c35b4

drivers/gpu/cuda/meson.build line 9

Quoted 9 lines

drivers/gpu/cuda/meson.build new file +9 -0

@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+if not is_linux
+        build = false
+        reason = 'only supported on Linux'
+endif
+
+cuda_dep = dependency('cuda', version : '>=11', modules: ['cuda'])

Compiler for C supports arguments -D_DEFAULT_SOURCE: YES
Compiler for C supports arguments -D_XOPEN_SOURCE=600: YES Compiler for C supports arguments -flax-vector-conversions: YES Compiler for C supports arguments -Wno-strict-aliasing -Wstrict-aliasing: YES Compiler for C supports arguments -Wno-format-nonliteral -Wformat-nonliteral: YES
Library libturbo found: NO
Library libldpc_decoder_5gnr found: NO
Dependency cuda found: NO (tried pkgconfig and cmake)

drivers/gpu/cuda/meson.build:9:0: ERROR: Dependency "cuda" not found, tried pkgconfig and cmake

Signature · 1 line

David Marchand

Stephen Hemminger

2021-11-08 19:02 UTC #11 lore raw

In reply to [email protected]

Quoted 11 lines · On Tue, 5 Oct 2021 22:49:05 +0000 <[email protected]> wrote:

From: Elena Agostini <[email protected]>
This is the CUDA implementation of the gpudev library. Funcitonalities implemented through CUDA Driver API are:
- Device probe and remove
- Manage device memory allocations
- Register/unregister external CPU memory in the device memory area
Signed-off-by Elena Agostini <[email protected]>
---

What is the license of the CUDA Driver?

Elena Agostini

author

2021-11-08 21:20 UTC #12 lore raw

Quoted 16 lines

From: Stephen Hemminger <[email protected]>
Date: Monday, 8 November 2021 at 20:02
To: Elena Agostini <[email protected]>
Cc: [email protected] <[email protected]>
Subject: Re: [dpdk-dev] [RFC PATCH] gpu/cuda: introduce CUDA driver External email: Use caution opening links or attachments
Quoted 5 lines · On Tue, 5 Oct 2021 22:49:05 +0000 <[email protected]> wrote:
From: Elena Agostini <[email protected]>
This is the CUDA implementation of the gpudev library. Funcitonalities implemented through CUDA Driver API are:

Quoted 8 lines

Quoted 6 lines
- Device probe and remove
- Manage device memory allocations
- Register/unregister external CPU memory in the device memory area
Signed-off-by Elena Agostini <[email protected]>
---
What is the license of the CUDA Driver?

As you can see in the code:
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright (c) 2021 NVIDIA Corporation & Affiliates */

Stephen Hemminger

2021-11-08 22:07 UTC #13 lore raw

Quoted 16 lines · On Mon, 8 Nov 2021 21:20:31 +0000 Elena Agostini <[email protected]> wrote:

Quoted 16 lines
From: Stephen Hemminger <[email protected]>
Date: Monday, 8 November 2021 at 20:02
To: Elena Agostini <[email protected]>
Cc: [email protected] <[email protected]>
Subject: Re: [dpdk-dev] [RFC PATCH] gpu/cuda: introduce CUDA driver External email: Use caution opening links or attachments
Quoted 5 lines · On Tue, 5 Oct 2021 22:49:05 +0000 <[email protected]> wrote:
From: Elena Agostini <[email protected]>
This is the CUDA implementation of the gpudev library. Funcitonalities implemented through CUDA Driver API are:

Quoted 9 lines

Quoted 8 lines
Quoted 6 lines
- Device probe and remove
- Manage device memory allocations
- Register/unregister external CPU memory in the device memory area
Signed-off-by Elena Agostini <[email protected]>
---
What is the license of the CUDA Driver?

I meant the CUDA driver API?
The DPDK policy is that driver can not be a wrapper around a close source API.

Stephen Hemminger

2021-11-08 23:15 UTC #14 lore raw

Quoted 26 lines · On Mon, 8 Nov 2021 14:07:47 -0800 Stephen Hemminger…

Quoted 23 lines · On Mon, 8 Nov 2021 21:20:31 +0000 Elena Agostini <[email protected]> wrote:
Quoted 23 lines
From: Stephen Hemminger <[email protected]>
Date: Monday, 8 November 2021 at 20:02
To: Elena Agostini <[email protected]>
Cc: [email protected] <[email protected]>
Subject: Re: [dpdk-dev] [RFC PATCH] gpu/cuda: introduce CUDA driver External email: Use caution opening links or attachments
Quoted 11 lines · On Tue, 5 Oct 2021 22:49:05 +0000 <[email protected]> wrote:
From: Elena Agostini <[email protected]>
This is the CUDA implementation of the gpudev library. Funcitonalities implemented through CUDA Driver API are:
- Device probe and remove
- Manage device memory allocations
- Register/unregister external CPU memory in the device memory area
Signed-off-by Elena Agostini <[email protected]>
---

Quoted 6 lines

Quoted 2 lines
Quoted 1 line
What is the license of the CUDA Driver?
I meant the CUDA driver API?
The DPDK policy is that driver can not be a wrapper around a close source API.

If it is this license agreement from Nvidia
https://docs.nvidia.com/cuda/eula/index.html

Then it is clearly not open source and I would have to recommend against allowing this driver in DPDK.

Corollary: without a open-source GPU driver, I would also recommend against including the GPU driver subsystem in DPDK.

Note: these views are my own as part of the open-source community and do not represent those of my employer.

[email protected]

author

2021-11-08 21:39 UTC #15 lore raw

v4 cover gpu/cuda: introduce CUDA driver

In reply to [email protected]

From: Elena Agostini <[email protected]>

This is the CUDA implementation of the gpudev library. Funcitonalities implemented through CUDA Driver API are:

- Device probe and remove
- Manage device memory allocations
- Register/unregister external CPU memory in the device memory area

Changelog:
- CUDA driver implementation of the GPU write memory barrier
- Fixed styling reported by checkpatch
- CUDA driver is optional: not built if CUDA >= 11.4 not found

Elena Agostini (1):
gpu/cuda: introduce CUDA driver

 doc/guides/gpus/cuda.rst     | 110 +++++
 doc/guides/gpus/index.rst    |   1 +
 drivers/gpu/cuda/cuda.c      | 813 +++++++++++++++++++++++++++++++++++
 drivers/gpu/cuda/meson.build |  20 +
 drivers/gpu/cuda/version.map |   3 +
 drivers/gpu/meson.build      |   2 +-
 6 files changed, 948 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/gpus/cuda.rst
 create mode 100644 drivers/gpu/cuda/cuda.c
 create mode 100644 drivers/gpu/cuda/meson.build
 create mode 100644 drivers/gpu/cuda/version.map

Signature · 1 line

2.17.1

[email protected]

author

2021-11-08 21:39 UTC #16 lore raw

PATCH v4 1/1 gpu/cuda: introduce CUDA driver

From: Elena Agostini <[email protected]>

This is the CUDA implementation of the gpudev library. Funcitonalities implemented through CUDA Driver API are:

- Device probe and remove
- Manage device memory allocations
- Register/unregister external CPU memory in the device memory area

Signed-off-by Elena Agostini <[email protected]>

---
 doc/guides/gpus/cuda.rst     | 110 +++++
 doc/guides/gpus/index.rst    |   1 +
 drivers/gpu/cuda/cuda.c      | 813 +++++++++++++++++++++++++++++++++++
 drivers/gpu/cuda/meson.build |  20 +
 drivers/gpu/cuda/version.map |   3 +
 drivers/gpu/meson.build      |   2 +-
 6 files changed, 948 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/gpus/cuda.rst
 create mode 100644 drivers/gpu/cuda/cuda.c
 create mode 100644 drivers/gpu/cuda/meson.build
 create mode 100644 drivers/gpu/cuda/version.map

doc/guides/gpus/cuda.rst new file +110 -0

new file mode 100644
index 0000000000..64a78bf1e1
@@ -0,0 +1,110 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+   Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+CUDA GPU driver
+===============
+
+The CUDA GPU driver library (**librte_gpu_cuda**) provides support for NVIDIA GPUs.
+Information and documentation about these devices can be found on the
+`NVIDIA website <http://www.nvidia.com>`__. Help is also provided by the
+`NVIDIA CUDA Toolkit developer zone <https://docs.nvidia.com/cuda>`__.
+
+Design
+------
+
+**librte_gpu_cuda** relies on CUDA Driver API (no need for CUDA Runtime API).
+
+Goal of this driver library is not to provide a wrapper for the whole CUDA Driver API.
+Instead, the scope is to implement the generic features of gpudev API.
+For a CUDA application, integrating the gpudev library functions using the CUDA driver library
+is quite straightforward and doesn't create any compatibility problem.
+
+Initialization
+~~~~~~~~~~~~~~
+
+During initialization, CUDA driver library detects NVIDIA physical GPUs on the
+system or specified via EAL device options (e.g. ``-a b6:00.0``).
+The driver initializes the CUDA driver environment through ``cuInit(0)`` function.
+For this reason, it's required to set any CUDA environment configuration before
+calling ``rte_eal_init`` function in the DPDK application.
+
+If the CUDA driver environment has been already initialized, the ``cuInit(0)``
+in CUDA driver library has no effect.
+
+CUDA Driver sub-contexts
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+After initialization, a CUDA application can create multiple sub-contexts on GPU
+physical devices. Through gpudev library, is possible to register these sub-contexts
+in the CUDA driver library as child devices having as parent a GPU physical device.
+
+CUDA driver library also supports `MPS <https://docs.nvidia.com/deploy/pdf/CUDA_Multi_Process_Service_Overview.pdf>`__.
+
+GPU memory management
+~~~~~~~~~~~~~~~~~~~~~
+
+The CUDA driver library maintains a table of GPU memory addresses allocated
+and CPU memory addresses registered associated to the input CUDA context.
+Whenever the application tried to deallocate or deregister a memory address,
+if the address is not in the table the CUDA driver library will return an error.
+
+Features
+--------
+
+- Register new child devices aka new CUDA Driver contexts
+- Allocate memory on the GPU
+- Register CPU memory to make it visible from GPU
+
+Minimal requirements
+--------------------
+
+Minimal requirements to enable the CUDA driver library are:
+
+- NVIDIA GPU Ampere or Volta
+- CUDA 11.4 Driver API or newer
+
+`GPUDirect RDMA Technology <https://docs.nvidia.com/cuda/gpudirect-rdma/index.html>`__
+allows compatible network cards (e.g. Mellanox) to directly send and receive packets
+using GPU memory instead of additional memory copies through the CPU system memory.
+To enable this technology, system requirements are:
+
+- `nvidia-peermem <https://docs.nvidia.com/cuda/gpudirect-rdma/index.html#nvidia-peermem>`__ module running on the system
+- Mellanox Network card ConnectX-5 or newer (BlueField models included)
+- DPDK mlx5 PMD enabled
+- To reach the best performance, a PCIe switch between GPU and NIC is recommended
+
+Limitations
+-----------
+
+Supported only on Linux.
+
+Supported GPUs
+--------------
+
+The following NVIDIA GPU devices are supported by this CUDA driver:
+
+- NVIDIA A100 80GB PCIe
+- NVIDIA A100 40GB PCIe
+- NVIDIA A30 24GB
+- NVIDIA A10 24GB
+- NVIDIA V100 32GB PCIe
+- NVIDIA V100 16GB PCIe
+
+External references
+-------------------
+
+A good example of how to use the GPU CUDA driver through the gpudev library
+is the l2fwd-nv application that can be found `here <https://github.com/NVIDIA/l2fwd-nv>`__.
+
+The application is based on vanilla DPDK example l2fwd and it's enhanced with GPU memory
+managed through gpudev library and CUDA to launch the swap of packets' MAC addresses workload
+on the GPU.
+
+l2fwd-nv is not intended to be used for performance (testpmd is the good candidate for this).
+The goal is to show different use-cases about how a CUDA application can use DPDK to:
+
+- allocate memory on GPU device using gpudev library
+- use that memory to create an external GPU memory mempool
+- receive packets directly in GPU memory
+- coordinate the workload on the GPU with the network and CPU activity to receive packets
+- send modified packets directly from the GPU memory

doc/guides/gpus/index.rst +1 -0

index 1878423239..4b7a420556 100644
@@ -9,3 +9,4 @@ General-Purpose Graphics Processing Unit Drivers
    :numbered:
 
    overview
+   cuda

drivers/gpu/cuda/cuda.c new file +813 -0

813 lines not on this page · raw message

drivers/gpu/cuda/meson.build new file +20 -0

new file mode 100644
index 0000000000..f084bbcf83
@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+if not is_linux
+        build = false
+        reason = 'only supported on Linux'
+endif
+
+cuda_dep = dependency('cuda', version : '>=11.4', modules: ['cuda'], required: false)
+
+if not cuda_dep.found()
+         build = false
+         reason = 'missing dependency, "cuda >= 11.4"'
+         subdir_done()
+endif
+
+ext_deps += cuda_dep
+
+deps += ['gpudev','pci','bus_pci', 'hash']
+sources = files('cuda.c')

drivers/gpu/cuda/version.map new file +3 -0

new file mode 100644
index 0000000000..4a76d1d52d
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};

drivers/gpu/meson.build +1 -1

index e51ad3381b..601bedcd61 100644
@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright (c) 2021 NVIDIA Corporation & Affiliates
 
-drivers = []
+drivers = [ 'cuda' ]

Signature · 1 line

2.17.1

[email protected]

author

2021-11-15 14:25 UTC #17 lore raw

v5 cover gpu/cuda: introduce CUDA driver

In reply to [email protected]

From: Elena Agostini <[email protected]>

This is the CUDA implementation of the gpudev library. Funcitonalities implemented through CUDA Driver API are:

- Device probe and remove
- Manage device memory allocations
- Register/unregister external CPU memory in the device memory area

Changelog:
- CUDA driver implementation of the GPU write memory barrier
- Fixed styling reported by checkpatch
- CUDA driver library not required at build time
- CUDA driver library is loaded at runtime through dlopen
- Documentation updated

Elena Agostini (1):
gpu/cuda: introduce CUDA driver

 doc/guides/gpus/cuda.rst               |  127 +++
 doc/guides/gpus/index.rst              |    1 +
 doc/guides/rel_notes/release_21_11.rst |    2 +
 drivers/gpu/cuda/cuda.c                | 1132 ++++++++++++++++++++++++
 drivers/gpu/cuda/cuda_loader.h         |  301 +++++++
 drivers/gpu/cuda/meson.build           |   10 +
 drivers/gpu/cuda/version.map           |    3 +
 drivers/gpu/meson.build                |    2 +-
 8 files changed, 1577 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/gpus/cuda.rst
 create mode 100644 drivers/gpu/cuda/cuda.c
 create mode 100644 drivers/gpu/cuda/cuda_loader.h
 create mode 100644 drivers/gpu/cuda/meson.build
 create mode 100644 drivers/gpu/cuda/version.map

Signature · 1 line

2.17.1

[email protected]

author

2021-11-15 14:25 UTC #18 lore raw

PATCH v5 1/1 gpu/cuda: introduce CUDA driver

From: Elena Agostini <[email protected]>

This is the CUDA implementation of the gpudev library. Funcitonalities implemented through CUDA Driver API are:

- Device probe and remove
- Manage device memory allocations
- Register/unregister external CPU memory in the device memory area

Signed-off-by Elena Agostini <[email protected]>

---
 doc/guides/gpus/cuda.rst               |  127 +++
 doc/guides/gpus/index.rst              |    1 +
 doc/guides/rel_notes/release_21_11.rst |    2 +
 drivers/gpu/cuda/cuda.c                | 1132 ++++++++++++++++++++++++
 drivers/gpu/cuda/cuda_loader.h         |  301 +++++++
 drivers/gpu/cuda/meson.build           |   10 +
 drivers/gpu/cuda/version.map           |    3 +
 drivers/gpu/meson.build                |    2 +-
 8 files changed, 1577 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/gpus/cuda.rst
 create mode 100644 drivers/gpu/cuda/cuda.c
 create mode 100644 drivers/gpu/cuda/cuda_loader.h
 create mode 100644 drivers/gpu/cuda/meson.build
 create mode 100644 drivers/gpu/cuda/version.map

doc/guides/gpus/cuda.rst new file +127 -0

new file mode 100644
index 0000000000..313fcfeffc
@@ -0,0 +1,127 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+   Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+CUDA GPU driver
+===============
+
+The CUDA GPU driver library (**librte_gpu_cuda**) provides support for NVIDIA GPUs.
+Information and documentation about these devices can be found on the
+`NVIDIA website <http://www.nvidia.com>`__. Help is also provided by the
+`NVIDIA CUDA Toolkit developer zone <https://docs.nvidia.com/cuda>`__.
+
+CUDA Shared Library
+-------------------
+
+To avoid any system configuration issue, the CUDA API **libcuda.so** shared library
+is not linked at building time because of a Meson's bug that looks
+for `cudart` module even if the `meson.build` file only requires default `cuda` module.
+
+**libcuda.so** is loaded at runtime in the ``cuda_gpu_probe`` function through ``dlopen``
+when the very first GPU is detected.
+If your CUDA installation resides in a custom directory you need to set
+the environment variable ``CUDA_PATH`` to specify where ``dlopen``
+can look for your **libcuda.so**.
+
+All CUDA API symbols are loaded at runtime as well.
+For this reason, to build the CUDA driver library
+you don't need to have the CUDA library installed on your system.
+
+Design
+------
+
+**librte_gpu_cuda** relies on CUDA Driver API (no need for CUDA Runtime API).
+
+Goal of this driver library is not to provide a wrapper for the whole CUDA Driver API.
+Instead, the scope is to implement the generic features of gpudev API.
+For a CUDA application, integrating the gpudev library functions using the CUDA driver library
+is quite straightforward and doesn't create any compatibility problem.
+
+Initialization
+~~~~~~~~~~~~~~
+
+During initialization, CUDA driver library detects NVIDIA physical GPUs on the
+system or specified via EAL device options (e.g. ``-a b6:00.0``).
+The driver initializes the CUDA driver environment through ``cuInit(0)`` function.
+For this reason, it's required to set any CUDA environment configuration before
+calling ``rte_eal_init`` function in the DPDK application.
+
+If the CUDA driver environment has been already initialized, the ``cuInit(0)``
+in CUDA driver library has no effect.
+
+CUDA Driver sub-contexts
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+After initialization, a CUDA application can create multiple sub-contexts on GPU
+physical devices. Through gpudev library, is possible to register these sub-contexts
+in the CUDA driver library as child devices having as parent a GPU physical device.
+
+CUDA driver library also supports `MPS <https://docs.nvidia.com/deploy/pdf/CUDA_Multi_Process_Service_Overview.pdf>`__.
+
+GPU memory management
+~~~~~~~~~~~~~~~~~~~~~
+
+The CUDA driver library maintains a table of GPU memory addresses allocated
+and CPU memory addresses registered associated to the input CUDA context.
+Whenever the application tried to deallocate or deregister a memory address,
+if the address is not in the table the CUDA driver library will return an error.
+
+Features
+--------
+
+- Register new child devices aka new CUDA Driver contexts
+- Allocate memory on the GPU
+- Register CPU memory to make it visible from GPU
+
+Minimal requirements
+--------------------
+
+Minimal requirements to enable the CUDA driver library are:
+
+- NVIDIA GPU Ampere or Volta
+- CUDA 11.4 Driver API or newer
+
+`GPUDirect RDMA Technology <https://docs.nvidia.com/cuda/gpudirect-rdma/index.html>`__
+allows compatible network cards (e.g. Mellanox) to directly send and receive packets
+using GPU memory instead of additional memory copies through the CPU system memory.
+To enable this technology, system requirements are:
+
+- `nvidia-peermem <https://docs.nvidia.com/cuda/gpudirect-rdma/index.html#nvidia-peermem>`__ module running on the system
+- Mellanox Network card ConnectX-5 or newer (BlueField models included)
+- DPDK mlx5 PMD enabled
+- To reach the best performance, an additional PCIe switch between GPU and NIC is recommended
+
+Limitations
+-----------
+
+Supported only on Linux.
+
+Supported GPUs
+--------------
+
+The following NVIDIA GPU devices are supported by this CUDA driver library:
+
+- NVIDIA A100 80GB PCIe
+- NVIDIA A100 40GB PCIe
+- NVIDIA A30 24GB
+- NVIDIA A10 24GB
+- NVIDIA V100 32GB PCIe
+- NVIDIA V100 16GB PCIe
+
+External references
+-------------------
+
+A good example of how to use the GPU CUDA driver library through the gpudev library
+is the l2fwd-nv application that can be found `here <https://github.com/NVIDIA/l2fwd-nv>`__.
+
+The application is based on vanilla DPDK example l2fwd and it's enhanced with GPU memory
+managed through gpudev library and CUDA to launch the swap of packets' MAC addresses workload
+on the GPU.
+
+l2fwd-nv is not intended to be used for performance (testpmd is the good candidate for this).
+The goal is to show different use-cases about how a CUDA application can use DPDK to:
+
+- allocate memory on GPU device using gpudev library
+- use that memory to create an external GPU memory mempool
+- receive packets directly in GPU memory
+- coordinate the workload on the GPU with the network and CPU activity to receive packets
+- send modified packets directly from the GPU memory

doc/guides/gpus/index.rst +1 -0

index 1878423239..4b7a420556 100644
@@ -9,3 +9,4 @@ General-Purpose Graphics Processing Unit Drivers
    :numbered:
 
    overview
+   cuda

doc/guides/rel_notes/release_21_11.rst +2 -0

index 7d60b554d8..c628deaeea 100644
@@ -111,6 +111,8 @@ New Features
   * Memory management
   * Communication flag & list
 
+* **Added NVIDIA GPU driver implemented with CUDA library.**
+
 * **Added new RSS offload types for IPv4/L4 checksum in RSS flow.**
 
   Added macros ETH_RSS_IPV4_CHKSUM and ETH_RSS_L4_CHKSUM, now IPv4 and

drivers/gpu/cuda/cuda.c new file +1132 -0

1,132 lines not on this page · raw message

drivers/gpu/cuda/cuda_loader.h new file +301 -0

301 lines not on this page · raw message

drivers/gpu/cuda/meson.build new file +10 -0

new file mode 100644
index 0000000000..f2a3095d8d
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+if not is_linux
+        build = false
+        reason = 'only supported on Linux'
+endif
+
+deps += ['gpudev','pci','bus_pci']
+sources = files('cuda.c')

drivers/gpu/cuda/version.map new file +3 -0

new file mode 100644
index 0000000000..4a76d1d52d
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};

drivers/gpu/meson.build +1 -1

index e51ad3381b..601bedcd61 100644
@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright (c) 2021 NVIDIA Corporation & Affiliates
 
-drivers = []
+drivers = [ 'cuda' ]

Signature · 1 line

2.17.1

[email protected]

author

2021-11-16 12:37 UTC #19 lore raw

v6 cover gpu/cuda: introduce CUDA driver

In reply to [email protected]

From: Elena Agostini <[email protected]>

This is the CUDA implementation of the gpudev library. Funcitonalities implemented through CUDA Driver API are:

- Device probe and remove
- Manage device memory allocations
- Register/unregister external CPU memory in the device memory area

Changelog:
- CUDA driver implementation of the GPU write memory barrier
- Fixed styling reported by checkpatch
- CUDA driver shared library libcuda.so is not required at build time
- CUDA driver shared library libcuda.so is loaded at runtime
- CUDA driver headers are required at build time
- Documentation updated

Elena Agostini (1):
gpu/cuda: introduce CUDA driver

 doc/guides/gpus/cuda.rst               |  151 ++++
 doc/guides/gpus/index.rst              |    1 +
 doc/guides/rel_notes/release_21_11.rst |    2 +
 drivers/gpu/cuda/cuda.c                | 1145 ++++++++++++++++++++++++
 drivers/gpu/cuda/meson.build           |   18 +
 drivers/gpu/cuda/version.map           |    3 +
 drivers/gpu/meson.build                |    2 +-
 7 files changed, 1321 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/gpus/cuda.rst
 create mode 100644 drivers/gpu/cuda/cuda.c
 create mode 100644 drivers/gpu/cuda/meson.build
 create mode 100644 drivers/gpu/cuda/version.map

Signature · 1 line

2.17.1

[email protected]

author

2021-11-16 12:37 UTC #20 lore raw

PATCH v6 1/1 gpu/cuda: introduce CUDA driver

From: Elena Agostini <[email protected]>

This is the CUDA implementation of the gpudev library. Functionalities implemented through CUDA Driver API are:
- Device probe and remove
- Manage device memory allocations
- Register/unregister external CPU memory in the device memory area

Signed-off-by Elena Agostini <[email protected]>

---
 doc/guides/gpus/cuda.rst               |  151 ++++
 doc/guides/gpus/index.rst              |    1 +
 doc/guides/rel_notes/release_21_11.rst |    2 +
 drivers/gpu/cuda/cuda.c                | 1145 ++++++++++++++++++++++++
 drivers/gpu/cuda/meson.build           |   18 +
 drivers/gpu/cuda/version.map           |    3 +
 drivers/gpu/meson.build                |    2 +-
 7 files changed, 1321 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/gpus/cuda.rst
 create mode 100644 drivers/gpu/cuda/cuda.c
 create mode 100644 drivers/gpu/cuda/meson.build
 create mode 100644 drivers/gpu/cuda/version.map

doc/guides/gpus/cuda.rst new file +151 -0

new file mode 100644
index 0000000000..9897d52d06
@@ -0,0 +1,151 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+   Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+CUDA GPU driver
+===============
+
+The CUDA GPU driver library (**librte_gpu_cuda**) provides support for NVIDIA GPUs.
+Information and documentation about these devices can be found on the
+`NVIDIA website <http://www.nvidia.com>`_. Help is also provided by the
+`NVIDIA CUDA Toolkit developer zone <https://docs.nvidia.com/cuda>`_.
+
+Build dependencies
+------------------
+
+The CUDA GPU driver library has an header-only dependency on ``cuda.h`` and ``cudaTypedefs.h``.
+To get these headers there are two options:
+
+- Install `CUDA Toolkit <https://developer.nvidia.com/cuda-toolkit>`_ 
+  (either regular or stubs installation).
+- Download these two headers from this `CUDA headers
+  <https://gitlab.com/nvidia/headers/cuda-individual/cudart>`_ public repo.
+
+You need to indicate to meson where CUDA headers files are through the CFLAGS variable.
+Two ways:
+
+- Set ``export CFLAGS=-I/usr/local/cuda/include`` before building
+- Add CFLAGS in the meson command line ``CFLAGS=-I/usr/local/cuda/include meson build``
+
+If headers are not found, the CUDA GPU driver library is not built.
+
+CUDA Shared Library
+-------------------
+
+To avoid any system configuration issue, the CUDA API **libcuda.so** shared library
+is not linked at building time because of a Meson bug that looks
+for `cudart` module even if the `meson.build` file only requires default `cuda` module.
+
+**libcuda.so** is loaded at runtime in the ``cuda_gpu_probe`` function through ``dlopen``
+when the very first GPU is detected.
+If CUDA installation resides in a custom directory,
+the environment variable ``CUDA_PATH_L`` should specify where ``dlopen``
+can look for **libcuda.so**.
+
+All CUDA API symbols are loaded at runtime as well.
+For this reason, to build the CUDA driver library,
+no need to install the CUDA library.
+
+Design
+------
+
+**librte_gpu_cuda** relies on CUDA Driver API (no need for CUDA Runtime API).
+
+Goal of this driver library is not to provide a wrapper for the whole CUDA Driver API.
+Instead, the scope is to implement the generic features of gpudev API.
+For a CUDA application, integrating the gpudev library functions
+using the CUDA driver library is quite straightforward
+and doesn't create any compatibility problem.
+
+Initialization
+~~~~~~~~~~~~~~
+
+During initialization, CUDA driver library detects NVIDIA physical GPUs
+on the system or specified via EAL device options (e.g. ``-a b6:00.0``).
+The driver initializes the CUDA driver environment through ``cuInit(0)`` function.
+For this reason, it's required to set any CUDA environment configuration before
+calling ``rte_eal_init`` function in the DPDK application.
+
+If the CUDA driver environment has been already initialized, the ``cuInit(0)``
+in CUDA driver library has no effect.
+
+CUDA Driver sub-contexts
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+After initialization, a CUDA application can create multiple sub-contexts
+on GPU physical devices.
+Through gpudev library, is possible to register these sub-contexts
+in the CUDA driver library as child devices having as parent a GPU physical device.
+
+CUDA driver library also supports `MPS
+<https://docs.nvidia.com/deploy/pdf/CUDA_Multi_Process_Service_Overview.pdf>`__.
+
+GPU memory management
+~~~~~~~~~~~~~~~~~~~~~
+
+The CUDA driver library maintains a table of GPU memory addresses allocated
+and CPU memory addresses registered associated to the input CUDA context.
+Whenever the application tried to deallocate or deregister a memory address,
+if the address is not in the table the CUDA driver library will return an error.
+
+Features
+--------
+
+- Register new child devices aka new CUDA Driver contexts.
+- Allocate memory on the GPU.
+- Register CPU memory to make it visible from GPU.
+
+Minimal requirements
+--------------------
+
+Minimal requirements to enable the CUDA driver library are:
+
+- NVIDIA GPU Ampere or Volta
+- CUDA 11.4 Driver API or newer
+
+`GPUDirect RDMA Technology <https://docs.nvidia.com/cuda/gpudirect-rdma/index.html>`_
+allows compatible network cards (e.g. Mellanox) to directly send and receive packets
+using GPU memory instead of additional memory copies through the CPU system memory.
+To enable this technology, system requirements are:
+
+- `nvidia-peermem <https://docs.nvidia.com/cuda/gpudirect-rdma/index.html#nvidia-peermem>`_
+  module running on the system;
+- Mellanox network card ConnectX-5 or newer (BlueField models included);
+- DPDK mlx5 PMD enabled;
+- To reach the best performance, an additional PCIe switch between GPU and NIC is recommended.
+
+Limitations
+-----------
+
+Supported only on Linux.
+
+Supported GPUs
+--------------
+
+The following NVIDIA GPU devices are supported by this CUDA driver library:
+
+- NVIDIA A100 80GB PCIe
+- NVIDIA A100 40GB PCIe
+- NVIDIA A30 24GB
+- NVIDIA A10 24GB
+- NVIDIA V100 32GB PCIe
+- NVIDIA V100 16GB PCIe
+
+External references
+-------------------
+
+A good example of how to use the GPU CUDA driver library through the gpudev library
+is the l2fwd-nv application that can be found `here <https://github.com/NVIDIA/l2fwd-nv>`_.
+
+The application is based on vanilla DPDK example l2fwd
+and is enhanced with GPU memory managed through gpudev library
+and CUDA to launch the swap of packets MAC addresses workload on the GPU.
+
+l2fwd-nv is not intended to be used for performance
+(testpmd is the good candidate for this).
+The goal is to show different use-cases about how a CUDA application can use DPDK to:
+
+- Allocate memory on GPU device using gpudev library.
+- Use that memory to create an external GPU memory mempool.
+- Receive packets directly in GPU memory.
+- Coordinate the workload on the GPU with the network and CPU activity to receive packets.
+- Send modified packets directly from the GPU memory.

doc/guides/gpus/index.rst +1 -0

index 1878423239..4b7a420556 100644
@@ -9,3 +9,4 @@ General-Purpose Graphics Processing Unit Drivers
    :numbered:
 
    overview
+   cuda

doc/guides/rel_notes/release_21_11.rst +2 -0

index cd4dcd0077..d76bba2fe3 100644
@@ -111,6 +111,8 @@ New Features
   * Memory management
   * Communication flag & list
 
+* **Added NVIDIA GPU driver implemented with CUDA library.**
+
 * **Added new RSS offload types for IPv4/L4 checksum in RSS flow.**
 
   Added macros ETH_RSS_IPV4_CHKSUM and ETH_RSS_L4_CHKSUM, now IPv4 and

drivers/gpu/cuda/cuda.c new file +1145 -0

1,145 lines not on this page · raw message

drivers/gpu/cuda/meson.build new file +18 -0

new file mode 100644
index 0000000000..4bb3e14cd2
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+if not is_linux
+        build = false
+        reason = 'only supported on Linux'
+endif
+
+cuda_h_found = cc.has_header('cuda.h')
+cuda_typeh_found = cc.has_header('cudaTypedefs.h')
+
+if not cuda_h_found or not cuda_typeh_found
+        build = false
+        reason = 'missing dependency cuda headers cuda.h and cudaTypedefs.h'
+endif
+
+deps += ['gpudev','pci','bus_pci']
+sources = files('cuda.c')

drivers/gpu/cuda/version.map new file +3 -0

new file mode 100644
index 0000000000..4a76d1d52d
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};

drivers/gpu/meson.build +1 -1

index e51ad3381b..601bedcd61 100644
@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright (c) 2021 NVIDIA Corporation & Affiliates
 
-drivers = []
+drivers = [ 'cuda' ]

Signature · 1 line

2.17.1

[email protected]

author

2021-11-16 14:39 UTC #21 lore raw

v7 cover gpu/cuda: introduce CUDA driver

In reply to [email protected]

From: Elena Agostini <[email protected]>

This is the CUDA implementation of the gpudev library. Funcitonalities implemented through CUDA Driver API are:

- Device probe and remove
- Manage device memory allocations
- Register/unregister external CPU memory in the device memory area

Elena Agostini (1):
gpu/cuda: introduce CUDA driver

 doc/guides/gpus/cuda.rst               |  152 ++++
 doc/guides/gpus/index.rst              |    1 +
 doc/guides/rel_notes/release_21_11.rst |    2 +
 drivers/gpu/cuda/cuda.c                | 1150 ++++++++++++++++++++++++
 drivers/gpu/cuda/meson.build           |   18 +
 drivers/gpu/cuda/version.map           |    3 +
 drivers/gpu/meson.build                |    2 +-
 7 files changed, 1327 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/gpus/cuda.rst
 create mode 100644 drivers/gpu/cuda/cuda.c
 create mode 100644 drivers/gpu/cuda/meson.build
 create mode 100644 drivers/gpu/cuda/version.map

Signature · 1 line

2.17.1

[email protected]

author

2021-11-16 14:39 UTC #22 lore raw

PATCH v7 1/1 gpu/cuda: introduce CUDA driver

From: Elena Agostini <[email protected]>

Signed-off-by Elena Agostini <[email protected]>

---
 doc/guides/gpus/cuda.rst               |  152 ++++
 doc/guides/gpus/index.rst              |    1 +
 doc/guides/rel_notes/release_21_11.rst |    2 +
 drivers/gpu/cuda/cuda.c                | 1150 ++++++++++++++++++++++++
 drivers/gpu/cuda/meson.build           |   18 +
 drivers/gpu/cuda/version.map           |    3 +
 drivers/gpu/meson.build                |    2 +-
 7 files changed, 1327 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/gpus/cuda.rst
 create mode 100644 drivers/gpu/cuda/cuda.c
 create mode 100644 drivers/gpu/cuda/meson.build
 create mode 100644 drivers/gpu/cuda/version.map

doc/guides/gpus/cuda.rst new file +152 -0

new file mode 100644
index 0000000000..d007c9ff2c
@@ -0,0 +1,152 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+   Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+CUDA GPU driver
+===============
+
+The CUDA GPU driver library (**librte_gpu_cuda**) provides support for NVIDIA GPUs.
+Information and documentation about these devices can be found on the
+`NVIDIA website <http://www.nvidia.com>`_. Help is also provided by the
+`NVIDIA CUDA Toolkit developer zone <https://docs.nvidia.com/cuda>`_.
+
+Build dependencies
+------------------
+
+The CUDA GPU driver library has an header-only dependency on ``cuda.h`` and ``cudaTypedefs.h``.
+To get these headers there are two options:
+
+- Install `CUDA Toolkit <https://developer.nvidia.com/cuda-toolkit>`_ 
+  (either regular or stubs installation).
+- Download these two headers from this `CUDA headers
+  <https://gitlab.com/nvidia/headers/cuda-individual/cudart>`_ public repo.
+
+You need to indicate to meson where CUDA headers files are through the CFLAGS variable.
+Three ways:
+
+- Set ``export CFLAGS=-I/usr/local/cuda/include`` before building
+- Add CFLAGS in the meson command line ``CFLAGS=-I/usr/local/cuda/include meson build``
+- Add the ``-Dc_args`` in meson command line ``meson build -Dc_args=-I/usr/local/cuda/include``
+
+If headers are not found, the CUDA GPU driver library is not built.
+
+CUDA Shared Library
+-------------------
+
+To avoid any system configuration issue, the CUDA API **libcuda.so** shared library
+is not linked at building time because of a Meson bug that looks
+for `cudart` module even if the `meson.build` file only requires default `cuda` module.
+
+**libcuda.so** is loaded at runtime in the ``cuda_gpu_probe`` function through ``dlopen``
+when the very first GPU is detected.
+If CUDA installation resides in a custom directory,
+the environment variable ``CUDA_PATH_L`` should specify where ``dlopen``
+can look for **libcuda.so**.
+
+All CUDA API symbols are loaded at runtime as well.
+For this reason, to build the CUDA driver library,
+no need to install the CUDA library.
+
+Design
+------
+
+**librte_gpu_cuda** relies on CUDA Driver API (no need for CUDA Runtime API).
+
+Goal of this driver library is not to provide a wrapper for the whole CUDA Driver API.
+Instead, the scope is to implement the generic features of gpudev API.
+For a CUDA application, integrating the gpudev library functions
+using the CUDA driver library is quite straightforward
+and doesn't create any compatibility problem.
+
+Initialization
+~~~~~~~~~~~~~~
+
+During initialization, CUDA driver library detects NVIDIA physical GPUs
+on the system or specified via EAL device options (e.g. ``-a b6:00.0``).
+The driver initializes the CUDA driver environment through ``cuInit(0)`` function.
+For this reason, it's required to set any CUDA environment configuration before
+calling ``rte_eal_init`` function in the DPDK application.
+
+If the CUDA driver environment has been already initialized, the ``cuInit(0)``
+in CUDA driver library has no effect.
+
+CUDA Driver sub-contexts
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+After initialization, a CUDA application can create multiple sub-contexts
+on GPU physical devices.
+Through gpudev library, is possible to register these sub-contexts
+in the CUDA driver library as child devices having as parent a GPU physical device.
+
+CUDA driver library also supports `MPS
+<https://docs.nvidia.com/deploy/pdf/CUDA_Multi_Process_Service_Overview.pdf>`__.
+
+GPU memory management
+~~~~~~~~~~~~~~~~~~~~~
+
+The CUDA driver library maintains a table of GPU memory addresses allocated
+and CPU memory addresses registered associated to the input CUDA context.
+Whenever the application tried to deallocate or deregister a memory address,
+if the address is not in the table the CUDA driver library will return an error.
+
+Features
+--------
+
+- Register new child devices aka new CUDA Driver contexts.
+- Allocate memory on the GPU.
+- Register CPU memory to make it visible from GPU.
+
+Minimal requirements
+--------------------
+
+Minimal requirements to enable the CUDA driver library are:
+
+- NVIDIA GPU Ampere or Volta
+- CUDA 11.4 Driver API or newer
+
+`GPUDirect RDMA Technology <https://docs.nvidia.com/cuda/gpudirect-rdma/index.html>`_
+allows compatible network cards (e.g. Mellanox) to directly send and receive packets
+using GPU memory instead of additional memory copies through the CPU system memory.
+To enable this technology, system requirements are:
+
+- `nvidia-peermem <https://docs.nvidia.com/cuda/gpudirect-rdma/index.html#nvidia-peermem>`_
+  module running on the system;
+- Mellanox network card ConnectX-5 or newer (BlueField models included);
+- DPDK mlx5 PMD enabled;
+- To reach the best performance, an additional PCIe switch between GPU and NIC is recommended.
+
+Limitations
+-----------
+
+Supported only on Linux.
+
+Supported GPUs
+--------------
+
+The following NVIDIA GPU devices are supported by this CUDA driver library:
+
+- NVIDIA A100 80GB PCIe
+- NVIDIA A100 40GB PCIe
+- NVIDIA A30 24GB
+- NVIDIA A10 24GB
+- NVIDIA V100 32GB PCIe
+- NVIDIA V100 16GB PCIe
+
+External references
+-------------------
+
+A good example of how to use the GPU CUDA driver library through the gpudev library
+is the l2fwd-nv application that can be found `here <https://github.com/NVIDIA/l2fwd-nv>`_.
+
+The application is based on vanilla DPDK example l2fwd
+and is enhanced with GPU memory managed through gpudev library
+and CUDA to launch the swap of packets MAC addresses workload on the GPU.
+
+l2fwd-nv is not intended to be used for performance
+(testpmd is the good candidate for this).
+The goal is to show different use-cases about how a CUDA application can use DPDK to:
+
+- Allocate memory on GPU device using gpudev library.
+- Use that memory to create an external GPU memory mempool.
+- Receive packets directly in GPU memory.
+- Coordinate the workload on the GPU with the network and CPU activity to receive packets.
+- Send modified packets directly from the GPU memory.

doc/guides/gpus/index.rst +1 -0

index 1878423239..4b7a420556 100644
@@ -9,3 +9,4 @@ General-Purpose Graphics Processing Unit Drivers
    :numbered:
 
    overview
+   cuda

doc/guides/rel_notes/release_21_11.rst +2 -0

index cd4dcd0077..d76bba2fe3 100644
@@ -111,6 +111,8 @@ New Features
   * Memory management
   * Communication flag & list
 
+* **Added NVIDIA GPU driver implemented with CUDA library.**
+
 * **Added new RSS offload types for IPv4/L4 checksum in RSS flow.**
 
   Added macros ETH_RSS_IPV4_CHKSUM and ETH_RSS_L4_CHKSUM, now IPv4 and

drivers/gpu/cuda/cuda.c new file +1150 -0

1,150 lines not on this page · raw message

drivers/gpu/cuda/meson.build new file +18 -0

new file mode 100644
index 0000000000..4bb3e14cd2
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+if not is_linux
+        build = false
+        reason = 'only supported on Linux'
+endif
+
+cuda_h_found = cc.has_header('cuda.h')
+cuda_typeh_found = cc.has_header('cudaTypedefs.h')
+
+if not cuda_h_found or not cuda_typeh_found
+        build = false
+        reason = 'missing dependency cuda headers cuda.h and cudaTypedefs.h'
+endif
+
+deps += ['gpudev','pci','bus_pci']
+sources = files('cuda.c')

drivers/gpu/cuda/version.map new file +3 -0

new file mode 100644
index 0000000000..4a76d1d52d
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};

drivers/gpu/meson.build +1 -1

index e51ad3381b..601bedcd61 100644
@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright (c) 2021 NVIDIA Corporation & Affiliates
 
-drivers = []
+drivers = [ 'cuda' ]

Signature · 1 line

2.17.1

Stephen Hemminger

2021-11-16 15:58 UTC #23 lore raw

Minor comments, overall looks fine.

+/* CUDA Driver functions loaded with dlsym() */
+CUresult CUDAAPI (*sym_cuInit)(unsigned int flags) = NULL; +CUresult CUDAAPI (*sym_cuDriverGetVersion)(int *driverVersion) = NULL; +CUresult CUDAAPI (*sym_cuGetProcAddress)(const char *symbol,
+ void **pfn, int cudaVersion, uint64_t flags) = NULL;
+

Can these be local (static) since not prefixed with driver specific name. Also global variables are always loaded as zero so NULL initialization is unnecessary. For other types checkpatch would complain.

drivers/gpu/cuda/cuda.c line 86

Quoted 2 lines

drivers/gpu/cuda/cuda.c new file +2 -0

+/* NVIDIA GPU address map */
+static struct rte_pci_id pci_id_cuda_map[] = {

Can this be const?

Thomas Monjalon

2021-11-16 16:35 UTC #24 lore raw

16/11/2021 16:58, Stephen Hemminger:

Quoted 15 lines

Minor comments, overall looks fine.
drivers/gpu/cuda/cuda.c new file +6 -0
+/* CUDA Driver functions loaded with dlsym() */
+CUresult CUDAAPI (*sym_cuInit)(unsigned int flags) = NULL;
+CUresult CUDAAPI (*sym_cuDriverGetVersion)(int *driverVersion) = NULL;
+CUresult CUDAAPI (*sym_cuGetProcAddress)(const char *symbol,
+		void **pfn, int cudaVersion, uint64_t flags) = NULL;
+
Can these be local (static) since not prefixed with driver specific name. Also global variables are always loaded as zero so NULL initialization is unnecessary. For other types checkpatch would complain.

Quoted 4 lines

drivers/gpu/cuda/cuda.c line 86
Quoted 2 lines
drivers/gpu/cuda/cuda.c new file +2 -0
+/* NVIDIA GPU address map */
+static struct rte_pci_id pci_id_cuda_map[] = {
Can this be const?

I didn't really push yet, so I can do this fixups now.

Thomas Monjalon

2021-11-16 16:40 UTC #25 lore raw

In reply to Stephen Hemminger

16/11/2021 16:58, Stephen Hemminger:

Quoted 15 lines

Minor comments, overall looks fine.
drivers/gpu/cuda/cuda.c new file +6 -0
+/* CUDA Driver functions loaded with dlsym() */
+CUresult CUDAAPI (*sym_cuInit)(unsigned int flags) = NULL;
+CUresult CUDAAPI (*sym_cuDriverGetVersion)(int *driverVersion) = NULL;
+CUresult CUDAAPI (*sym_cuGetProcAddress)(const char *symbol,
+		void **pfn, int cudaVersion, uint64_t flags) = NULL;
+
Can these be local (static) since not prefixed with driver specific name. Also global variables are always loaded as zero so NULL initialization is unnecessary. For other types checkpatch would complain.

Quoted 4 lines

drivers/gpu/cuda/cuda.c line 86
Quoted 2 lines
drivers/gpu/cuda/cuda.c new file +2 -0
+/* NVIDIA GPU address map */
+static struct rte_pci_id pci_id_cuda_map[] = {
Can this be const?

On a separate note, Elena did what you suggested for the logtype variable:

	static RTE_LOG_REGISTER_DEFAULT(cuda_logtype, NOTICE);

So cuda_logtype is static.
I think we should do the same for all DPDK logtypes. Volunteer for a patch?

Thomas Monjalon

2021-11-16 16:30 UTC #26 lore raw

In reply to [email protected]

16/11/2021 23:50, [email protected]:

Quoted 13 lines

From: Elena Agostini <[email protected]>
This is the CUDA implementation of the gpudev library. Functionalities implemented through CUDA Driver API are:
- Device probe and remove
- Manage device memory allocations
- Register/unregister external CPU memory in the device memory area
Signed-off-by Elena Agostini <[email protected]>
---

drivers/gpu/cuda/version.map line 3

Quoted 3 lines

drivers/gpu/cuda/version.map new file +3 -0
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};

Should be DPDK_22

Applied with some minor fixes, thanks.

Thomas Monjalon

2021-11-16 16:44 UTC #27 lore raw

16/11/2021 17:30, Thomas Monjalon:

Quoted 20 lines

16/11/2021 23:50, [email protected]:
Quoted 13 lines
From: Elena Agostini <[email protected]>
This is the CUDA implementation of the gpudev library. Functionalities implemented through CUDA Driver API are:
- Device probe and remove
- Manage device memory allocations
- Register/unregister external CPU memory in the device memory area
Signed-off-by Elena Agostini <[email protected]>
---
drivers/gpu/cuda/version.map line 3
Quoted 3 lines
drivers/gpu/cuda/version.map new file +3 -0
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
Should be DPDK_22

Quoted 1 line

Applied with some minor fixes, thanks.

Did the static/const fixups suggested by Stephen before pushing to the repo.