ntb/ae4dma: added Amd userspace DMA and NTB drivers
Subsystems (2)
PATCH 1/1 ntb/ae4dma: added Amd userspace DMA and NTB drivers
Added a user-space driver with support for the
AMD EPYC 4th Generation DMA (AE4DMA) offload engine. AMD Non Transparent Bridge driver.
- Implementation of new user-space driver supporting NTB and DMA memory copy offload on AMD EYPC 9004 & 8004 systems (Genoa and Siena processors).
- Updated the NTB_FWD application to support AMD NTB driver.
--- app/test-dma-perf/benchmark.c | 29 +- app/test-dma-perf/config.ini | 134 +++-- app/test-dma-perf/main.c | 7 +- app/test/test_dmadev.c | 47 +- drivers/dma/ae4dma/ae4dma_dmadev.c | 704 +++++++++++++++++++++++++ drivers/dma/ae4dma/ae4dma_hw_defs.h | 223 ++++++++ drivers/dma/ae4dma/ae4dma_internal.h | 138 +++++ drivers/dma/ae4dma/meson.build | 7 + drivers/dma/meson.build | 1 + drivers/raw/ntb/meson.build | 1 + drivers/raw/ntb/ntb.c | 371 +++++++++----- drivers/raw/ntb/ntb.h | 15 + drivers/raw/ntb/ntb_hw_amd.c | 738 +++++++++++++++++++++++++++ drivers/raw/ntb/ntb_hw_amd.h | 123 +++++ drivers/raw/ntb/ntb_hw_intel.c | 21 + drivers/raw/ntb/rte_pmd_ntb.h | 2 + examples/ntb/commands.list | 2 +- examples/ntb/ntb_fwd.c | 362 +++++++++++-- usertools/dpdk-devbind.py | 15 +- 19 files changed, 2698 insertions(+), 242 deletions(-) create mode 100644 drivers/dma/ae4dma/ae4dma_dmadev.c create mode 100644 drivers/dma/ae4dma/ae4dma_hw_defs.h create mode 100644 drivers/dma/ae4dma/ae4dma_internal.h create mode 100644 drivers/dma/ae4dma/meson.build create mode 100644 drivers/raw/ntb/ntb_hw_amd.c create mode 100644 drivers/raw/ntb/ntb_hw_amd.h
app/test-dma-perf/benchmark.c +27 -2
@@ -266,6 +266,27 @@ error_exit(int dev_id)rte_exit (EXIT_FAILURE ,"DMA error\n" ); } +static void +await_hw (int16_t dev_id,uint16_t vchan) +{ +enum rte_dma_vchan_status st; + +if (rte_dma_vchan_status (dev_id, vchan, &st) <0 ) { +/* for drivers that don't support this op, just sleep for 1 us */ +rte_delay_us_sleep (1 ); +return ; + } + +/* for those that do, *max* end time is one second from now, but all should be faster */ +const uint64_t end_cycles =rte_get_timer_cycles () +rte_get_timer_hz (); + +while (st ==RTE_DMA_VCHAN_ACTIVE &&rte_get_timer_cycles () < end_cycles) { +rte_pause (); +rte_dma_vchan_status (dev_id, vchan, &st); + } +} + +static inline void do_dma_submit_and_poll (uint16_t dev_id,uint64_t *async_cnt,volatile struct worker_info *worker_info)
@@ -276,7 +297,6 @@ do_dma_submit_and_poll(uint16_t dev_id, uint64_t *async_cnt, ret =rte_dma_submit (dev_id,0 );if (ret <0 )error_exit (dev_id); - nr_cpl =rte_dma_completed (dev_id,0 ,MAX_DMA_CPL_NB ,NULL ,NULL ); *async_cnt -= nr_cpl; worker_info->total_cpl += nr_cpl;
@@ -309,14 +329,16 @@ do_dma_plain_mem_copy(void *p)for (i =0 ; i < nr_buf; i++) { dma_copy: ret =rte_dma_copy (dev_id,0 ,rte_mbuf_data_iova (srcs[i]), -rte_mbuf_data_iova (dsts[i]), buf_size,0 ); +rte_mbuf_data_iova (dsts[i]), buf_size,0 );// if (unlikely (ret <0 )) { +await_hw (dev_id,0 );if (ret == -ENOSPC ) {do_dma_submit_and_poll (dev_id, &async_cnt, worker_info); goto dma_copy; }else error_exit (dev_id); } + async_cnt++;if ((async_cnt % kick_batch) ==0 )
@@ -756,6 +778,7 @@ mem_copy_benchmark(struct test_configure *cfg)while (1 ) {bool ready = true; +for (i =0 ; i < nb_workers; i++) {if (lcores[i]->worker_info.ready_flag == false) { ready =0 ;
@@ -786,6 +809,7 @@ mem_copy_benchmark(struct test_configure *cfg)for (k =0 ; k < nb_workers; k++) {struct rte_mbuf **src_buf =NULL , **dst_buf =NULL ;uint32_t nr_buf_pt = nr_buf / nb_workers; + vchan_dev = &cfg->dma_config[k].vchan_dev; offset = nr_buf / nb_workers * k; src_buf = srcs + offset;
@@ -871,6 +895,7 @@ mem_copy_benchmark(struct test_configure *cfg)for (k =0 ; k < nb_workers; k++) {struct rte_mbuf **sbuf =NULL , **dbuf =NULL ; + vchan_dev = &cfg->dma_config[k].vchan_dev; offset = nr_buf / nb_workers * k; m =NULL ;
app/test-dma-perf/config.ini +86 -48
143 lines not on this page · raw message
app/test-dma-perf/main.c +5 -2
@@ -192,6 +192,7 @@ parse_lcore(struct test_configure *test_case, const char *value)strlcpy (input, value, len +1 );char *token =strtok (input,", " ); +while (token !=NULL ) { lcore_dma_map = &(test_case->dma_config[test_case->num_worker++].lcore_dma_map);memset (lcore_dma_map,0 ,sizeof (struct lcore_dma_map_t ));
@@ -201,6 +202,7 @@ parse_lcore(struct test_configure *test_case, const char *value) }uint16_t lcore_id =atoi (token); + lcore_dma_map->lcore = lcore_id; token =strtok (NULL ,", " );
@@ -428,6 +430,7 @@ load_configs(const char *path)char lc_dma[RTE_DEV_NAME_MAX_LEN ];int i =0 ; +while (1 ) {snprintf (lc_dma,RTE_DEV_NAME_MAX_LEN ,"lcore_dma%d" , i); lcore_dma =rte_cfgfile_get_entry (cfgfile, section_name, lc_dma);
@@ -460,6 +463,7 @@ load_configs(const char *path) }else { lcore_dma =rte_cfgfile_get_entry (cfgfile, section_name,"lcore" );int lcore_ret =parse_lcore (test_case, lcore_dma); +if (lcore_ret <0 ) {printf ("parse lcore error in case %d.\n" , i +1 ); test_case->is_valid = false;
@@ -551,6 +555,7 @@ main(int argc, char *argv[])if (rst_path_ptr ==NULL ) {strlcpy (rst_path, cfg_path_ptr,PATH_MAX );char *token =strtok (basename (rst_path),"." ); +if (token ==NULL ) {printf ("Config file error.\n" );return -1 ;
@@ -566,7 +571,6 @@ main(int argc, char *argv[])return -1 ; }fclose (fd); -printf ("Running cases...\n" );for (i =0 ; i < case_nb; i++) {if (test_cases[i].is_skip) {
@@ -644,7 +648,6 @@ main(int argc, char *argv[])printf ("Case process unknown terminated.\n\n" ); } } -printf ("Bye...\n" );return 0 ; }
app/test/test_dmadev.c +34 -13
191 lines not on this page · raw message
drivers/dma/ae4dma/ae4dma_dmadev.c new file +704 -0
704 lines not on this page · raw message
drivers/dma/ae4dma/ae4dma_hw_defs.h new file +223 -0
223 lines not on this page · raw message
drivers/dma/ae4dma/ae4dma_internal.h new file +138 -0
138 lines not on this page · raw message
drivers/dma/ae4dma/meson.build new file +7 -0
@@ -0,0 +1,7 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2024 Advanced Micro Devices, Inc. All rights reserved. + +build = dpdk_conf.has('RTE_ARCH_X86') +reason = 'only supported on x86' +sources = files('ae4dma_dmadev.c') +deps += ['bus_pci', 'dmadev']
drivers/dma/meson.build +1 -0
@@ -9,6 +9,7 @@ drivers = [ 'idxd', 'ioat', 'odm', + 'ae4dma', 'skeleton', ] std_deps = ['dmadev']
drivers/raw/ntb/meson.build +1 -0
@@ -3,5 +3,6 @@ deps += ['rawdev', 'mbuf', 'mempool', 'pci', 'bus_pci'] sources = files('ntb.c', + 'ntb_hw_amd.c', 'ntb_hw_intel.c') headers = files('rte_pmd_ntb.h')
drivers/raw/ntb/ntb.c +243 -128
504 lines not on this page · raw message
drivers/raw/ntb/ntb.h +15 -0
@@ -15,11 +15,14 @@ extern int ntb_logtype;/* Vendor ID */ #define NTB_INTEL_VENDOR_ID 0x8086 +#define NTB_AMD_VENDOR_ID 0x1022/* Device IDs */ #define NTB_INTEL_DEV_ID_B2B_SKX 0x201C#define NTB_INTEL_DEV_ID_B2B_ICX 0x347E#define NTB_INTEL_DEV_ID_B2B_SPR 0x347E +#define NTB_AMD_DEV_ID_PRI 0x14C0 +#define NTB_AMD_DEV_ID_SEC 0x14C3/* Reserved to app to use. */ #define NTB_SPAD_USER "spad_user_"
@@ -40,6 +43,8 @@ enum ntb_xstats_idx {enum ntb_topo {NTB_TOPO_NONE =0 , +NTB_TOPO_PRI , +NTB_TOPO_SEC ,NTB_TOPO_B2B_USD ,NTB_TOPO_B2B_DSD , };
@@ -88,6 +93,7 @@ enum ntb_spad_idx {* NTB device operations * @ntb_dev_init: Init ntb dev. * @get_peer_mw_addr: To get the addr of peer mw[mw_idx]. +* @get_peer_mw_addr_phys: To get the physical addr of peer mw[mw_idx]. * @mw_set_trans: Set translation of internal memory that remote can access. * @ioremap: Translate the remote host address to bar address. * @get_link_status: get link status, link speed and link width.
@@ -104,6 +110,7 @@ enum ntb_spad_idx {struct ntb_dev_ops {int (*ntb_dev_init)(const struct rte_rawdev *dev);void *(*get_peer_mw_addr)(const struct rte_rawdev *dev,int mw_idx); +uint64_t (*get_peer_mw_addr_phys)(const struct rte_rawdev *dev,int mw_idx);int (*mw_set_trans)(const struct rte_rawdev *dev,int mw_idx,uint64_t addr,uint64_t size);void *(*ioremap)(const struct rte_rawdev *dev,uint64_t addr);
@@ -119,6 +126,7 @@ struct ntb_dev_ops {int (*peer_db_set)(const struct rte_rawdev *dev,uint8_t db_bit);int (*vector_bind)(const struct rte_rawdev *dev,uint8_t intr,uint8_t msix); +int (*interrupt_handler)(void *param); };struct ntb_desc {
@@ -209,6 +217,8 @@ struct ntb_hw {const struct ntb_dev_ops *ntb_ops;struct rte_pci_device *pci_dev; +/**< PMD-specific private data */ +rte_rawdev_obj_t pmd_private;char *hw_addr;uint8_t peer_dev_up;
@@ -216,6 +226,9 @@ struct ntb_hw {/* remote mem base addr */ uint64_t *peer_mw_base; +void *base_addr; +uint64_t base_addr_phys; +uint16_t queue_pairs;uint16_t queue_size;uint32_t hdr_size_per_queue;
@@ -236,4 +249,6 @@ struct ntb_hw { int spad_user_list[NTB_SPAD_USER_MAX_NUM]; }; +extern const struct ntb_dev_ops amd_ntb_ops; +#endif /* _NTB_H_ */
drivers/raw/ntb/ntb_hw_amd.c new file +738 -0
738 lines not on this page · raw message
drivers/raw/ntb/ntb_hw_amd.h new file +123 -0
123 lines not on this page · raw message
drivers/raw/ntb/ntb_hw_intel.c +21 -0
@@ -613,6 +613,26 @@ intel_ntb_vector_bind(const struct rte_rawdev *dev, uint8_t intr, uint8_t msix)return 0 ; } +static void +intel_ntb_dev_interrupt_handler (void *param) +{ +struct rte_rawdev *dev = (struct rte_rawdev *)param; +struct ntb_hw *hw = dev->dev_private; +uint32_t val_h, val_l; +uint64_t peer_mw_size; +uint64_t db_bits =0 ; +uint8_t peer_mw_cnt; +int i =0 ; + +if (hw->ntb_ops->db_read ==NULL || + hw->ntb_ops->db_clear ==NULL || + hw->ntb_ops->peer_db_set ==NULL ) { +NTB_LOG (ERR ,"Doorbell is not supported." ); +return ; + } +} + +/* operations for primary side of local ntb */ const struct ntb_dev_ops intel_ntb_ops = { .ntb_dev_init = intel_ntb_dev_init,
@@ -628,4 +648,5 @@ const struct ntb_dev_ops intel_ntb_ops = { .db_set_mask = intel_ntb_db_set_mask, .peer_db_set = intel_ntb_peer_db_set, .vector_bind = intel_ntb_vector_bind,
Lines of this file are not shown. View the raw message.
examples/ntb/ntb_fwd.c +317 -45
362 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Signature · 1 line
2.34.1
Part of the diff is not shown here. View the raw message.
05/07/2025 16:10, Bhagyada Modali:
Quoted 10 lines
Added a user-space driver with support for the
AMD EPYC 4th Generation DMA (AE4DMA) offload engine. AMD Non Transparent Bridge driver.- Implementation of new user-space driver supporting NTB and DMA memory copy offload on AMD EYPC 9004 & 8004 systems (Genoa and Siena processors).
- Updated theNTB_FWDapplication to support AMD NTB driver.Signed-off-by Bhagyada Modali <[email protected]>
It looks to be a draft.
There are so many problems in this patch that I was expecting a follow-up, but looks like no.
Please check the contributors guide for the tools you should run before submitting.
Then make sure your patch pass the CI in patchwork.
Thank you