enable FIFO for NTB
cover enable FIFO for NTB
Enable FIFO for NTB rawdev driver to support packet based processing. And an example is provided to support txonly, rxonly, iofwd between NTB device and ethdev, and file transmission.
Xiaoyun Li (4):
raw/ntb: setup ntb queue
raw/ntb: add xstats support
raw/ntb: add enqueue and dequeue functions
examples/ntb: support more functions for NTB
doc/guides/rawdevs/ntb.rst | 67 +- doc/guides/rel_notes/release_19_11.rst | 4 + doc/guides/sample_app_ug/ntb.rst | 59 +- drivers/raw/ntb/Makefile | 3 + drivers/raw/ntb/meson.build | 1 + drivers/raw/ntb/ntb.c | 1078 +++++++++++++++----- drivers/raw/ntb/ntb.h | 162 ++- drivers/raw/ntb/ntb_hw_intel.c | 48 +- drivers/raw/ntb/rte_pmd_ntb.h | 43 + examples/ntb/meson.build | 3 + examples/ntb/ntb_fwd.c | 1297 +++++++++++++++++++++--- 11 files changed, 2349 insertions(+), 416 deletions(-) create mode 100644 drivers/raw/ntb/rte_pmd_ntb.h
Signature · 1 line
2.17.1
PATCH 1/4 raw/ntb: setup ntb queue
Setup and init ntb txq and rxq. And negotiate queue information with the peer. If queue size and number of queues are not consistent on both sides, return error.
--- doc/guides/rawdevs/ntb.rst | 39 +- doc/guides/rel_notes/release_19_11.rst | 4 + drivers/raw/ntb/Makefile | 3 + drivers/raw/ntb/meson.build | 1 + drivers/raw/ntb/ntb.c | 705 ++++++++++++++++++------- drivers/raw/ntb/ntb.h | 151 ++++-- drivers/raw/ntb/ntb_hw_intel.c | 26 +- drivers/raw/ntb/rte_pmd_ntb.h | 43 ++ 8 files changed, 718 insertions(+), 254 deletions(-) create mode 100644 drivers/raw/ntb/rte_pmd_ntb.h
doc/guides/rawdevs/ntb.rst +38 -1
@@ -45,8 +45,45 @@ to use, i.e. igb_uio, vfio. The ``dpdk-devbind.py`` script can be used to show devices status and to bind them to a suitable kernel driver. They will appear under the category of "Misc (rawdev) devices". +Ring Layout +----------- + +Since read/write remote system's memory are through PCI bus, remote read +is much more expensive than remote write. Thus, the enqueue and dequeue +based on ntb ring should avoid remote read. The ring layout for ntb is +like the following: +- Ring Format: + desc_ring: + 0 16 64 + +---------------------------------------------------------------+ + | buffer address | + +---------------+-----------------------------------------------+ + | buffer length | resv | + +---------------+-----------------------------------------------+ + used_ring: + 0 16 32 + +---------------+---------------+ + | packet length | flags | + +---------------+---------------+ +- Ring Layout + +------------------------+ +------------------------+ + | used_ring | | desc_ring | + | +---+ | | +---+ | + | | | | | | | | + | +---+ +--------+ | | +---+ | + | | | ---> | buffer | <+---+-| | | + | +---+ +--------+ | | +---+ | + | | | | | | | | + | +---+ | | +---+ | + | ... | | ... | + | | | | + | +---------+ | | +---------+ | + | | tx_tail | | | | rx_tail | | + | System A +---------+ | | System B +---------+ | + +------------------------+ +------------------------+ + <---------traffic--------- + Limitation ---------- -- The FIFO hasn't been introduced and will come in 19.11 release. - This PMD only supports Intel Skylake platform.
doc/guides/rel_notes/release_19_11.rst +4 -0
@@ -56,6 +56,10 @@ New Features Also, make sure to start the actual text at the margin. ========================================================= + * **Introduced FIFO for NTB PMD.** + + Introduced FIFO for NTB (Non-transparent Bridge) PMD to support + packet based processing. Removed Items -------------
drivers/raw/ntb/Makefile +3 -0
@@ -25,4 +25,7 @@ LIBABIVER := 1 SRCS-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV) += ntb.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV) += ntb_hw_intel.c +# install this header file +SYMLINK-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV)-include := rte_pmd_ntb.h + include $(RTE_SDK)/mk/rte.lib.mk
drivers/raw/ntb/meson.build +1 -0
@@ -5,4 +5,5 @@ deps += ['rawdev', 'mbuf', 'mempool', 'pci', 'bus_pci'] sources = files('ntb.c', 'ntb_hw_intel.c') +install_headers('rte_pmd_ntb.h') allow_experimental_apis = true
drivers/raw/ntb/ntb.c +511 -194
943 lines not on this page · raw message
drivers/raw/ntb/ntb.h +105 -46
205 lines not on this page · raw message
drivers/raw/ntb/ntb_hw_intel.c +13 -13
@@ -26,7 +26,7 @@ static enum xeon_ntb_bar intel_ntb_bar[] = { };static int -intel_ntb_dev_init (struct rte_rawdev *dev) +intel_ntb_dev_init (const struct rte_rawdev *dev) {struct ntb_hw *hw = dev->dev_private;uint8_t reg_val, bar;
@@ -77,7 +77,7 @@ intel_ntb_dev_init(struct rte_rawdev *dev) hw->db_cnt =XEON_DB_COUNT ; hw->spad_cnt =XEON_SPAD_COUNT ; - hw->mw_size =rte_zmalloc ("uint64_t" , + hw->mw_size =rte_zmalloc ("ntb_mw_size" , hw->mw_cnt *sizeof (uint64_t ),0 );for (i =0 ; i < hw->mw_cnt; i++) { bar = intel_ntb_bar[i];
@@ -94,7 +94,7 @@ intel_ntb_dev_init(struct rte_rawdev *dev) }static void * -intel_ntb_get_peer_mw_addr (struct rte_rawdev *dev,int mw_idx) +intel_ntb_get_peer_mw_addr (const struct rte_rawdev *dev,int mw_idx) {struct ntb_hw *hw = dev->dev_private;uint8_t bar;
@@ -116,7 +116,7 @@ intel_ntb_get_peer_mw_addr(struct rte_rawdev *dev, int mw_idx) }static int -intel_ntb_mw_set_trans (struct rte_rawdev *dev,int mw_idx, +intel_ntb_mw_set_trans (const struct rte_rawdev *dev,int mw_idx,uint64_t addr,uint64_t size) {struct ntb_hw *hw = dev->dev_private;
@@ -163,7 +163,7 @@ intel_ntb_mw_set_trans(struct rte_rawdev *dev, int mw_idx, }static int -intel_ntb_get_link_status (struct rte_rawdev *dev) +intel_ntb_get_link_status (const struct rte_rawdev *dev) {struct ntb_hw *hw = dev->dev_private;uint16_t reg_val;
@@ -195,7 +195,7 @@ intel_ntb_get_link_status(struct rte_rawdev *dev) }static int -intel_ntb_set_link (struct rte_rawdev *dev,bool up) +intel_ntb_set_link (const struct rte_rawdev *dev,bool up) {struct ntb_hw *hw = dev->dev_private;uint32_t ntb_ctrl, reg_off;
@@ -221,7 +221,7 @@ intel_ntb_set_link(struct rte_rawdev *dev, bool up) }static uint32_t -intel_ntb_spad_read (struct rte_rawdev *dev,int spad,bool peer) +intel_ntb_spad_read (const struct rte_rawdev *dev,int spad,bool peer) {struct ntb_hw *hw = dev->dev_private;uint32_t spad_v, reg_off;
@@ -241,7 +241,7 @@ intel_ntb_spad_read(struct rte_rawdev *dev, int spad, bool peer) }static int -intel_ntb_spad_write (struct rte_rawdev *dev,int spad, +intel_ntb_spad_write (const struct rte_rawdev *dev,int spad,bool peer,uint32_t spad_v) {struct ntb_hw *hw = dev->dev_private;
@@ -263,7 +263,7 @@ intel_ntb_spad_write(struct rte_rawdev *dev, int spad, }static uint64_t -intel_ntb_db_read (struct rte_rawdev *dev) +intel_ntb_db_read (const struct rte_rawdev *dev) {struct ntb_hw *hw = dev->dev_private;uint64_t db_off, db_bits;
@@ -278,7 +278,7 @@ intel_ntb_db_read(struct rte_rawdev *dev) }static int -intel_ntb_db_clear (struct rte_rawdev *dev,uint64_t db_bits) +intel_ntb_db_clear (const struct rte_rawdev *dev,uint64_t db_bits) {struct ntb_hw *hw = dev->dev_private;uint64_t db_off;
@@ -293,7 +293,7 @@ intel_ntb_db_clear(struct rte_rawdev *dev, uint64_t db_bits) }static int -intel_ntb_db_set_mask (struct rte_rawdev *dev,uint64_t db_mask) +intel_ntb_db_set_mask (const struct rte_rawdev *dev,uint64_t db_mask) {struct ntb_hw *hw = dev->dev_private;uint64_t db_m_off;
@@ -312,7 +312,7 @@ intel_ntb_db_set_mask(struct rte_rawdev *dev, uint64_t db_mask) }static int -intel_ntb_peer_db_set (struct rte_rawdev *dev,uint8_t db_idx) +intel_ntb_peer_db_set (const struct rte_rawdev *dev,uint8_t db_idx) {struct ntb_hw *hw = dev->dev_private;uint32_t db_off;
@@ -332,7 +332,7 @@ intel_ntb_peer_db_set(struct rte_rawdev *dev, uint8_t db_idx) }static int -intel_ntb_vector_bind (struct rte_rawdev *dev,uint8_t intr,uint8_t msix) +intel_ntb_vector_bind (const struct rte_rawdev *dev,uint8_t intr,uint8_t msix) {struct ntb_hw *hw = dev->dev_private;uint8_t reg_off;
drivers/raw/ntb/rte_pmd_ntb.h new file +43 -0
43 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH 2/4 raw/ntb: add xstats support
In reply to Xiaoyun LiAdd xstats support for ntb rawdev.
Support tx-packets, tx-bytes, tx-errors and
rx-packets, rx-bytes, rx-missed.
--- drivers/raw/ntb/ntb.c | 135 ++++++++++++++++++++++++++++++++++++------ drivers/raw/ntb/ntb.h | 11 ++++ 2 files changed, 128 insertions(+), 18 deletions(-)
drivers/raw/ntb/ntb.c +117 -18
@@ -30,6 +30,17 @@ static const struct rte_pci_id pci_id_ntb_map[] = { { .vendor_id = 0, /* sentinel */ }, }; +/* Align with enum ntb_xstats_idx */ +static struct rte_rawdev_xstats_name ntb_xstats_names[] = { + {"Tx-packtes" }, + {"Tx-bytes" }, + {"Tx-errors" }, + {"Rx-packets" }, + {"Rx-bytes" }, + {"Rx-missed" }, +}; +#define NTB_XSTATS_NUM RTE_DIM(ntb_xstats_names) +static inline void ntb_link_cleanup (struct rte_rawdev *dev) {
@@ -538,6 +549,10 @@ ntb_queue_init(struct rte_rawdev *dev, uint16_t qp_id) txq->last_avail =0 ; txq->nb_tx_free = txq->nb_tx_desc -1 ; +/* Set per queue stats. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) + hw->ntb_xstats[i +NTB_XSTATS_NUM * (qp_id +1 )] =0 ; +return 0 ; }
@@ -614,6 +629,7 @@ ntb_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config) {struct ntb_dev_config *conf = config;struct ntb_hw *hw = dev->dev_private; +uint32_t xstats_num;int ret; hw->queue_pairs = conf->num_queues;
@@ -624,6 +640,10 @@ ntb_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config) sizeof(struct ntb_rx_queue *) * hw->queue_pairs, 0); hw->tx_queues =rte_zmalloc ("ntb_tx_queues" ,sizeof (struct ntb_tx_queue *) * hw->queue_pairs,0 ); +/* First total stats, then per queue stats. */ + xstats_num = (hw->queue_pairs +1 ) *NTB_XSTATS_NUM ; + hw->ntb_xstats =rte_zmalloc ("ntb_xstats" , xstats_num * +sizeof (uint64_t ),0 );/* Start handshake with the peer. */ ret =ntb_handshake_work (dev);
@@ -645,6 +665,10 @@ ntb_dev_start(struct rte_rawdev *dev)if (!hw->link_status || !hw->peer_dev_up)return -EINVAL ; +/* Set total stats. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) + hw->ntb_xstats[i] =0 ; +for (i =0 ; i < hw->queue_pairs; i++) { ret =ntb_queue_init (dev, i);if (ret) {
@@ -909,38 +933,113 @@ ntb_attr_get(struct rte_rawdev *dev, const char *attr_name, }static int -ntb_xstats_get(const struct rte_rawdev *dev __rte_unused, - const unsigned int ids[] __rte_unused, - uint64_t values[] __rte_unused, - unsigned int n __rte_unused) +ntb_xstats_get (const struct rte_rawdev *dev, +const unsigned int ids[], +uint64_t values[], +unsigned int n) { -return 0 ; +struct ntb_hw *hw = dev->dev_private; +uint32_t i, j, off, xstats_num; + +/* Calculate total stats of all queues. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) { +for (j =0 ; j < hw->queue_pairs; j++) { + off =NTB_XSTATS_NUM * (j +1 ) + i; + hw->ntb_xstats[i] += hw->ntb_xstats[off]; + } + } + + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); +for (i =0 ; i < n && ids[i] < xstats_num; i++) + values[i] = hw->ntb_xstats[ids[i]]; + +return i; }static int -ntb_xstats_get_names(const struct rte_rawdev *dev __rte_unused, - struct rte_rawdev_xstats_name *xstats_names __rte_unused, - unsigned int size __rte_unused) +ntb_xstats_get_names (const struct rte_rawdev *dev, +struct rte_rawdev_xstats_name *xstats_names, +unsigned int size) { -return 0 ; +struct ntb_hw *hw = dev->dev_private; +uint32_t xstats_num, i, j, off; + + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); +if (xstats_names ==NULL || size < xstats_num) +return xstats_num; + +/* Total stats names */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) { +strncpy (xstats_names[i].name, ntb_xstats_names[i].name, +sizeof (xstats_names[0 ].name)); + } + +/* Queue stats names */ +for (i =0 ; i < hw->queue_pairs; i++) { +for (j =0 ; j <NTB_XSTATS_NUM ; j++) { + off = j + (i +1 ) *NTB_XSTATS_NUM ; +snprintf (xstats_names[off].name, +sizeof (xstats_names[0 ].name), +"%s_q%u" , ntb_xstats_names[j].name, i); + } + } + +return xstats_num; }static uint64_t -ntb_xstats_get_by_name(const struct rte_rawdev *dev __rte_unused, - const char *name __rte_unused, - unsigned int *id __rte_unused) +ntb_xstats_get_by_name (const struct rte_rawdev *dev, +const char *name,unsigned int *id) { -return 0 ; +struct rte_rawdev_xstats_name *xstats_names; +struct ntb_hw *hw = dev->dev_private; +uint32_t xstats_num, i, j, off; + +if (name ==NULL ) +return -EINVAL ; + + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); + xstats_names =rte_zmalloc ("ntb_stats_name" , +sizeof (struct rte_rawdev_xstats_name ) * + xstats_num,0 ); +ntb_xstats_get_names (dev, xstats_names, xstats_num); + +/* Calculate total stats of all queues. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) { +for (j =0 ; j < hw->queue_pairs; j++) { + off =NTB_XSTATS_NUM * (j +1 ) + i; + hw->ntb_xstats[i] += hw->ntb_xstats[off]; + } + } + +for (i =0 ; i < xstats_num; i++) { +if (!strncmp (name, xstats_names[i].name, +RTE_RAW_DEV_XSTATS_NAME_SIZE )) { + *id = i; +rte_free (xstats_names); +return hw->ntb_xstats[i]; + } + } + +NTB_LOG (ERR ,"Cannot find the xstats name." ); + +return -EINVAL ; }static int -ntb_xstats_reset(struct rte_rawdev *dev __rte_unused, - const uint32_t ids[] __rte_unused, - uint32_t nb_ids __rte_unused) +ntb_xstats_reset (struct rte_rawdev *dev, +const uint32_t ids[], +uint32_t nb_ids) { -return 0 ; -} +struct ntb_hw *hw = dev->dev_private; +uint32_t i, xstats_num; + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); +for (i =0 ; i < nb_ids && ids[i] < xstats_num; i++) + hw->ntb_xstats[ids[i]] =0 ; + +return i; +}static const struct rte_rawdev_ops ntb_ops = { .dev_info_get = ntb_dev_info_get,
drivers/raw/ntb/ntb.h +11 -0
@@ -27,6 +27,15 @@ extern int ntb_logtype;#define NTB_DFLT_TX_FREE_THRESH 256 +enum ntb_xstats_idx { +NTB_TX_PKTS_ID =0 , +NTB_TX_BYTES_ID , +NTB_TX_ERRS_ID , +NTB_RX_PKTS_ID , +NTB_RX_BYTES_ID , +NTB_RX_MISS_ID , +}; +enum ntb_topo {NTB_TOPO_NONE =0 ,NTB_TOPO_B2B_USD ,
@@ -216,6 +225,8 @@ struct ntb_hw {uint8_t peer_used_mws; +uint64_t *ntb_xstats; +/* Reserve several spad for app to use. */ int spad_user_list[NTB_SPAD_USER_MAX_NUM ]; };
Signature · 1 line
2.17.1
PATCH 3/4 raw/ntb: add enqueue and dequeue functions
In reply to Xiaoyun LiIntroduce enqueue and dequeue functions to support packet based processing. And enable write-combining for ntb driver since it can improve the performance a lot.
--- doc/guides/rawdevs/ntb.rst | 28 ++++ drivers/raw/ntb/ntb.c | 242 ++++++++++++++++++++++++++++++--- drivers/raw/ntb/ntb.h | 2 + drivers/raw/ntb/ntb_hw_intel.c | 22 +++ 4 files changed, 275 insertions(+), 19 deletions(-)
doc/guides/rawdevs/ntb.rst +28 -0
@@ -45,6 +45,24 @@ to use, i.e. igb_uio, vfio. The ``dpdk-devbind.py`` script can be used to show devices status and to bind them to a suitable kernel driver. They will appear under the category of "Misc (rawdev) devices". +Prerequisites +------------- +NTB PMD needs kernel PCI driver to support write combining (WC) to get +better performance. The difference will be more than 10 times. +To enable WC, there are 2 ways. +- Insert igb_uio with ``wc_active=1`` flag if use igb_uio driver. + insmod igb_uio.ko wc_active=1 +- Enable WC for NTB device's Bar 2 and Bar 4 (Mapped memory) manually. + Get bar base address using ``lspci -vvv -s ae:00.0 | grep Region``. + Region 0: Memory at 39bfe0000000 (64-bit, prefetchable) [size=64K] + Region 2: Memory at 39bfa0000000 (64-bit, prefetchable) [size=512M] + Region 4: Memory at 39bfc0000000 (64-bit, prefetchable) [size=512M] + Using the following command to enable WC. + echo "base=0x39bfa0000000 size=0x400000 type=write-combining" >> /proc/mtrr + echo "base=0x39bfc0000000 size=0x400000 type=write-combining" >> /proc/mtrr + To disable WC for these regions, using the following. + echo "disable=1" >> /proc/mtrr + Ring Layout -----------
@@ -83,6 +101,16 @@ like the following: +------------------------+ +------------------------+ <---------traffic--------- +- Enqueue and Dequeue + Based on this ring layout, enqueue reads rx_tail to get how many free + buffers and writes used_ring and tx_tail to tell the peer which buffers + are filled with data. + And dequeue reads tx_tail to get how many packets are arrived, and + writes desc_ring and rx_tail to tell the peer about the new allocated + buffers. + So in this way, only remote write happens and remote read can be avoid + to get better performance. + Limitation ----------
drivers/raw/ntb/ntb.c +223 -19
272 lines not on this page · raw message
drivers/raw/ntb/ntb.h +2 -0
@@ -87,6 +87,7 @@ enum ntb_spad_idx {* @ntb_dev_init: Init ntb dev. * @get_peer_mw_addr: To get the addr of peer mw[mw_idx]. * @mw_set_trans: Set translation of internal memory that remote can access. +* @ioremap: Translate the remote host address to bar address. * @get_link_status: get link status, link speed and link width. * @set_link: Set local side up/down. * @spad_read: Read local/peer spad register val.
@@ -103,6 +104,7 @@ struct ntb_dev_ops {void *(*get_peer_mw_addr)(const struct rte_rawdev *dev,int mw_idx);int (*mw_set_trans)(const struct rte_rawdev *dev,int mw_idx,uint64_t addr,uint64_t size); +void *(*ioremap)(const struct rte_rawdev *dev,uint64_t addr);int (*get_link_status)(const struct rte_rawdev *dev);int (*set_link)(const struct rte_rawdev *dev,bool up); uint32_t (*spad_read)(const struct rte_rawdev *dev, int spad,
drivers/raw/ntb/ntb_hw_intel.c +22 -0
@@ -162,6 +162,27 @@ intel_ntb_mw_set_trans(const struct rte_rawdev *dev, int mw_idx,return 0 ; } +static void * +intel_ntb_ioremap (const struct rte_rawdev *dev,uint64_t addr) +{ +struct ntb_hw *hw = dev->dev_private; +void *mapped =NULL ; +void *base; +int i; + +for (i =0 ; i < hw->peer_used_mws; i++) { +if (addr >= hw->peer_mw_base[i] && + addr <= hw->peer_mw_base[i] + hw->mw_size[i]) { + base =intel_ntb_get_peer_mw_addr (dev, i); + mapped = (void *)(addr - hw->peer_mw_base[i] + + (uint64_t )base); +break ; + } + } + +return mapped; +} +static int intel_ntb_get_link_status (const struct rte_rawdev *dev) {
@@ -357,6 +378,7 @@ const struct ntb_dev_ops intel_ntb_ops = { .ntb_dev_init = intel_ntb_dev_init, .get_peer_mw_addr = intel_ntb_get_peer_mw_addr, .mw_set_trans = intel_ntb_mw_set_trans, + .ioremap = intel_ntb_ioremap, .get_link_status = intel_ntb_get_link_status, .set_link = intel_ntb_set_link, .spad_read = intel_ntb_spad_read,
Signature · 1 line
2.17.1
PATCH 4/4 examples/ntb: support more functions for NTB
In reply to Xiaoyun LiSupport to transmit files between two systems. Support iofwd between one ethdev and NTB device. Support rxonly and txonly for NTB device. Support to set forwarding mode as file-trans, txonly, rxonly or iofwd.
Support to show/clear port stats and throughput.
--- doc/guides/sample_app_ug/ntb.rst | 59 +- examples/ntb/meson.build | 3 + examples/ntb/ntb_fwd.c | 1297 +++++++++++++++++++++++++++--- 3 files changed, 1231 insertions(+), 128 deletions(-)
doc/guides/sample_app_ug/ntb.rst +53 -6
@@ -5,8 +5,17 @@ NTB Sample Application ====================== The ntb sample application shows how to use ntb rawdev driver. -This sample provides interactive mode to transmit file between -two hosts. +This sample provides interactive mode to do packet based processing +between two systems. + +This sample supports 4 types of packet forwarding mode. + +* ``file-trans``: transmit files between two systems. The sample will + be polling to receive files from the peer and save the file as + ``ntb_recv_file[N]``, [N] represents the number of received file. +* ``rxonly``: NTB receives packets but doesn't transmit them. +* ``txonly``: NTB generates and transmits packets without receiving any. +* ``iofwd``: iofwd between NTB device and ethdev. Compiling the Application -------------------------
@@ -29,6 +38,40 @@ Refer to the *DPDK Getting Started Guide* for general information on running applications and the Environment Abstraction Layer (EAL) options. +Command-line Options +-------------------- + +The application supports the following command-line options. + +* ``--buf-size=N`` + + Set the data size of the mbufs used to N bytes, where N < 65536. + The default value is 2048. + +* ``--fwd-mode=mode`` + + Set the packet forwarding mode as ``file-trans``, ``txonly``, + ``rxonly`` or ``iofwd``. + +* ``--nb-desc=N`` + + Set number of descriptors of queue as N, namely queue size, + where 64 <= N <= 1024. The default value is 1024. + +* ``--txfreet=N`` + + Set the transmit free threshold of TX rings to N, where 0 <= N <= + the value of ``--nb-desc``. The default value is 256. + +* ``--burst=N`` + + Set the number of packets per burst to N, where 1 <= N <= 32. + The default value is 32. + +* ``--qp=N`` + + Set the number of queues as N, where qp > 0. + Using the application ---------------------
@@ -41,7 +84,11 @@ The application is console-driven using the cmdline DPDK interface: From this interface the available commands and descriptions of what they do as as follows: -* ``send [filepath]``: Send file to the peer host. -* ``receive [filepath]``: Receive file to [filepath]. Need the peer - to send file successfully first. -* ``quit``: Exit program +* ``send [filepath]``: Send file to the peer host. Need to be in + file-trans forwarding mode first. +* ``start``: Start transmission. +* ``stop``: Stop transmission. +* ``show/clear port stats``: Show/Clear port stats and throughput. +* ``set fwd file-trans/rxonly/txonly/iofwd``: Set packet forwarding + mode. +* ``quit``: Exit program.
examples/ntb/meson.build +3 -0
@@ -14,3 +14,6 @@ cflags += ['-D_FILE_OFFSET_BITS=64'] sources = files( 'ntb_fwd.c' ) +if dpdk_conf.has('RTE_LIBRTE_PMD_NTB_RAWDEV') + deps += 'rawdev_ntb' +endif
examples/ntb/ntb_fwd.c +1175 -122
1,447 lines not on this page · raw message
Signature · 1 line
2.17.1
-----Original Message-----
From: Li, Xiaoyun
Sent: Wednesday, September 4, 2019 10:39 PM
To: Wu, Jingjing <[email protected]>; Wiles, Keith <[email protected]>; Maslekar, Omkar <[email protected]>; Liang, Cunming <[email protected]>
Cc: [email protected]; Li, Xiaoyun <[email protected]>
Subject: [PATCH 0/4] enable FIFO for NTB
Enable FIFO for NTB rawdev driver to support packet based processing. And an example is provided to support txonly, rxonly, iofwd between NTB device and ethdev, and file transmission.
Xiaoyun Li (4):
raw/ntb: setup ntb queue
raw/ntb: add xstats support
raw/ntb: add enqueue and dequeue functions
examples/ntb: support more functions for NTB
doc/guides/rawdevs/ntb.rst | 67 +- doc/guides/rel_notes/release_19_11.rst | 4 + doc/guides/sample_app_ug/ntb.rst | 59 +- drivers/raw/ntb/Makefile | 3 + drivers/raw/ntb/meson.build | 1 + drivers/raw/ntb/ntb.c | 1078 +++++++++++++++----- drivers/raw/ntb/ntb.h | 162 ++- drivers/raw/ntb/ntb_hw_intel.c | 48 +- drivers/raw/ntb/rte_pmd_ntb.h | 43 + examples/ntb/meson.build | 3 + examples/ntb/ntb_fwd.c | 1297 +++++++++++++++++++++--- 11 files changed, 2349 insertions(+), 416 deletions(-) create mode 100644 drivers/raw/ntb/rte_pmd_ntb.h
--
2.17.1
v2 cover enable FIFO for NTB
In reply to Xiaoyun LiEnable FIFO for NTB rawdev driver to support packet based processing. And an example is provided to support txonly, rxonly, iofwd between NTB device and ethdev, and file transmission.
Xiaoyun Li (4):
raw/ntb: setup ntb queue
raw/ntb: add xstats support
raw/ntb: add enqueue and dequeue functions
examples/ntb: support more functions for NTB
---
v2:
* Fixed compile issues with 32-bit machine and lack of including file.
* Fixed a typo.
doc/guides/rawdevs/ntb.rst | 67 +- doc/guides/rel_notes/release_19_11.rst | 4 + doc/guides/sample_app_ug/ntb.rst | 59 +- drivers/raw/ntb/Makefile | 3 + drivers/raw/ntb/meson.build | 1 + drivers/raw/ntb/ntb.c | 1078 +++++++++++++++----- drivers/raw/ntb/ntb.h | 162 ++- drivers/raw/ntb/ntb_hw_intel.c | 48 +- drivers/raw/ntb/rte_pmd_ntb.h | 43 + examples/ntb/meson.build | 3 + examples/ntb/ntb_fwd.c | 1298 +++++++++++++++++++++--- 11 files changed, 2350 insertions(+), 416 deletions(-) create mode 100644 drivers/raw/ntb/rte_pmd_ntb.h
Signature · 1 line
2.17.1
PATCH v2 1/4 raw/ntb: setup ntb queue
Setup and init ntb txq and rxq. And negotiate queue information with the peer. If queue size and number of queues are not consistent on both sides, return error.
--- doc/guides/rawdevs/ntb.rst | 39 +- doc/guides/rel_notes/release_19_11.rst | 4 + drivers/raw/ntb/Makefile | 3 + drivers/raw/ntb/meson.build | 1 + drivers/raw/ntb/ntb.c | 705 ++++++++++++++++++------- drivers/raw/ntb/ntb.h | 151 ++++-- drivers/raw/ntb/ntb_hw_intel.c | 26 +- drivers/raw/ntb/rte_pmd_ntb.h | 43 ++ 8 files changed, 718 insertions(+), 254 deletions(-) create mode 100644 drivers/raw/ntb/rte_pmd_ntb.h
doc/guides/rawdevs/ntb.rst +38 -1
@@ -45,8 +45,45 @@ to use, i.e. igb_uio, vfio. The ``dpdk-devbind.py`` script can be used to show devices status and to bind them to a suitable kernel driver. They will appear under the category of "Misc (rawdev) devices". +Ring Layout +----------- + +Since read/write remote system's memory are through PCI bus, remote read +is much more expensive than remote write. Thus, the enqueue and dequeue +based on ntb ring should avoid remote read. The ring layout for ntb is +like the following: +- Ring Format: + desc_ring: + 0 16 64 + +---------------------------------------------------------------+ + | buffer address | + +---------------+-----------------------------------------------+ + | buffer length | resv | + +---------------+-----------------------------------------------+ + used_ring: + 0 16 32 + +---------------+---------------+ + | packet length | flags | + +---------------+---------------+ +- Ring Layout + +------------------------+ +------------------------+ + | used_ring | | desc_ring | + | +---+ | | +---+ | + | | | | | | | | + | +---+ +--------+ | | +---+ | + | | | ---> | buffer | <+---+-| | | + | +---+ +--------+ | | +---+ | + | | | | | | | | + | +---+ | | +---+ | + | ... | | ... | + | | | | + | +---------+ | | +---------+ | + | | tx_tail | | | | rx_tail | | + | System A +---------+ | | System B +---------+ | + +------------------------+ +------------------------+ + <---------traffic--------- + Limitation ---------- -- The FIFO hasn't been introduced and will come in 19.11 release. - This PMD only supports Intel Skylake platform.
doc/guides/rel_notes/release_19_11.rst +4 -0
@@ -56,6 +56,10 @@ New Features Also, make sure to start the actual text at the margin. ========================================================= + * **Introduced FIFO for NTB PMD.** + + Introduced FIFO for NTB (Non-transparent Bridge) PMD to support + packet based processing. Removed Items -------------
drivers/raw/ntb/Makefile +3 -0
@@ -25,4 +25,7 @@ LIBABIVER := 1 SRCS-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV) += ntb.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV) += ntb_hw_intel.c +# install this header file +SYMLINK-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV)-include := rte_pmd_ntb.h + include $(RTE_SDK)/mk/rte.lib.mk
drivers/raw/ntb/meson.build +1 -0
@@ -5,4 +5,5 @@ deps += ['rawdev', 'mbuf', 'mempool', 'pci', 'bus_pci'] sources = files('ntb.c', 'ntb_hw_intel.c') +install_headers('rte_pmd_ntb.h') allow_experimental_apis = true
drivers/raw/ntb/ntb.c +511 -194
943 lines not on this page · raw message
drivers/raw/ntb/ntb.h +105 -46
205 lines not on this page · raw message
drivers/raw/ntb/ntb_hw_intel.c +13 -13
@@ -26,7 +26,7 @@ static enum xeon_ntb_bar intel_ntb_bar[] = { };static int -intel_ntb_dev_init (struct rte_rawdev *dev) +intel_ntb_dev_init (const struct rte_rawdev *dev) {struct ntb_hw *hw = dev->dev_private;uint8_t reg_val, bar;
@@ -77,7 +77,7 @@ intel_ntb_dev_init(struct rte_rawdev *dev) hw->db_cnt =XEON_DB_COUNT ; hw->spad_cnt =XEON_SPAD_COUNT ; - hw->mw_size =rte_zmalloc ("uint64_t" , + hw->mw_size =rte_zmalloc ("ntb_mw_size" , hw->mw_cnt *sizeof (uint64_t ),0 );for (i =0 ; i < hw->mw_cnt; i++) { bar = intel_ntb_bar[i];
@@ -94,7 +94,7 @@ intel_ntb_dev_init(struct rte_rawdev *dev) }static void * -intel_ntb_get_peer_mw_addr (struct rte_rawdev *dev,int mw_idx) +intel_ntb_get_peer_mw_addr (const struct rte_rawdev *dev,int mw_idx) {struct ntb_hw *hw = dev->dev_private;uint8_t bar;
@@ -116,7 +116,7 @@ intel_ntb_get_peer_mw_addr(struct rte_rawdev *dev, int mw_idx) }static int -intel_ntb_mw_set_trans (struct rte_rawdev *dev,int mw_idx, +intel_ntb_mw_set_trans (const struct rte_rawdev *dev,int mw_idx,uint64_t addr,uint64_t size) {struct ntb_hw *hw = dev->dev_private;
@@ -163,7 +163,7 @@ intel_ntb_mw_set_trans(struct rte_rawdev *dev, int mw_idx, }static int -intel_ntb_get_link_status (struct rte_rawdev *dev) +intel_ntb_get_link_status (const struct rte_rawdev *dev) {struct ntb_hw *hw = dev->dev_private;uint16_t reg_val;
@@ -195,7 +195,7 @@ intel_ntb_get_link_status(struct rte_rawdev *dev) }static int -intel_ntb_set_link (struct rte_rawdev *dev,bool up) +intel_ntb_set_link (const struct rte_rawdev *dev,bool up) {struct ntb_hw *hw = dev->dev_private;uint32_t ntb_ctrl, reg_off;
@@ -221,7 +221,7 @@ intel_ntb_set_link(struct rte_rawdev *dev, bool up) }static uint32_t -intel_ntb_spad_read (struct rte_rawdev *dev,int spad,bool peer) +intel_ntb_spad_read (const struct rte_rawdev *dev,int spad,bool peer) {struct ntb_hw *hw = dev->dev_private;uint32_t spad_v, reg_off;
@@ -241,7 +241,7 @@ intel_ntb_spad_read(struct rte_rawdev *dev, int spad, bool peer) }static int -intel_ntb_spad_write (struct rte_rawdev *dev,int spad, +intel_ntb_spad_write (const struct rte_rawdev *dev,int spad,bool peer,uint32_t spad_v) {struct ntb_hw *hw = dev->dev_private;
@@ -263,7 +263,7 @@ intel_ntb_spad_write(struct rte_rawdev *dev, int spad, }static uint64_t -intel_ntb_db_read (struct rte_rawdev *dev) +intel_ntb_db_read (const struct rte_rawdev *dev) {struct ntb_hw *hw = dev->dev_private;uint64_t db_off, db_bits;
@@ -278,7 +278,7 @@ intel_ntb_db_read(struct rte_rawdev *dev) }static int -intel_ntb_db_clear (struct rte_rawdev *dev,uint64_t db_bits) +intel_ntb_db_clear (const struct rte_rawdev *dev,uint64_t db_bits) {struct ntb_hw *hw = dev->dev_private;uint64_t db_off;
@@ -293,7 +293,7 @@ intel_ntb_db_clear(struct rte_rawdev *dev, uint64_t db_bits) }static int -intel_ntb_db_set_mask (struct rte_rawdev *dev,uint64_t db_mask) +intel_ntb_db_set_mask (const struct rte_rawdev *dev,uint64_t db_mask) {struct ntb_hw *hw = dev->dev_private;uint64_t db_m_off;
@@ -312,7 +312,7 @@ intel_ntb_db_set_mask(struct rte_rawdev *dev, uint64_t db_mask) }static int -intel_ntb_peer_db_set (struct rte_rawdev *dev,uint8_t db_idx) +intel_ntb_peer_db_set (const struct rte_rawdev *dev,uint8_t db_idx) {struct ntb_hw *hw = dev->dev_private;uint32_t db_off;
@@ -332,7 +332,7 @@ intel_ntb_peer_db_set(struct rte_rawdev *dev, uint8_t db_idx) }static int -intel_ntb_vector_bind (struct rte_rawdev *dev,uint8_t intr,uint8_t msix) +intel_ntb_vector_bind (const struct rte_rawdev *dev,uint8_t intr,uint8_t msix) {struct ntb_hw *hw = dev->dev_private;uint8_t reg_off;
drivers/raw/ntb/rte_pmd_ntb.h new file +43 -0
43 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v2 2/4 raw/ntb: add xstats support
In reply to Xiaoyun LiAdd xstats support for ntb rawdev.
Support tx-packets, tx-bytes, tx-errors and
rx-packets, rx-bytes, rx-missed.
--- drivers/raw/ntb/ntb.c | 135 ++++++++++++++++++++++++++++++++++++------ drivers/raw/ntb/ntb.h | 11 ++++ 2 files changed, 128 insertions(+), 18 deletions(-)
drivers/raw/ntb/ntb.c +117 -18
@@ -30,6 +30,17 @@ static const struct rte_pci_id pci_id_ntb_map[] = { { .vendor_id = 0, /* sentinel */ }, }; +/* Align with enum ntb_xstats_idx */ +static struct rte_rawdev_xstats_name ntb_xstats_names[] = { + {"Tx-packets" }, + {"Tx-bytes" }, + {"Tx-errors" }, + {"Rx-packets" }, + {"Rx-bytes" }, + {"Rx-missed" }, +}; +#define NTB_XSTATS_NUM RTE_DIM(ntb_xstats_names) +static inline void ntb_link_cleanup (struct rte_rawdev *dev) {
@@ -538,6 +549,10 @@ ntb_queue_init(struct rte_rawdev *dev, uint16_t qp_id) txq->last_avail =0 ; txq->nb_tx_free = txq->nb_tx_desc -1 ; +/* Set per queue stats. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) + hw->ntb_xstats[i +NTB_XSTATS_NUM * (qp_id +1 )] =0 ; +return 0 ; }
@@ -614,6 +629,7 @@ ntb_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config) {struct ntb_dev_config *conf = config;struct ntb_hw *hw = dev->dev_private; +uint32_t xstats_num;int ret; hw->queue_pairs = conf->num_queues;
@@ -624,6 +640,10 @@ ntb_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config) sizeof(struct ntb_rx_queue *) * hw->queue_pairs, 0); hw->tx_queues =rte_zmalloc ("ntb_tx_queues" ,sizeof (struct ntb_tx_queue *) * hw->queue_pairs,0 ); +/* First total stats, then per queue stats. */ + xstats_num = (hw->queue_pairs +1 ) *NTB_XSTATS_NUM ; + hw->ntb_xstats =rte_zmalloc ("ntb_xstats" , xstats_num * +sizeof (uint64_t ),0 );/* Start handshake with the peer. */ ret =ntb_handshake_work (dev);
@@ -645,6 +665,10 @@ ntb_dev_start(struct rte_rawdev *dev)if (!hw->link_status || !hw->peer_dev_up)return -EINVAL ; +/* Set total stats. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) + hw->ntb_xstats[i] =0 ; +for (i =0 ; i < hw->queue_pairs; i++) { ret =ntb_queue_init (dev, i);if (ret) {
@@ -909,38 +933,113 @@ ntb_attr_get(struct rte_rawdev *dev, const char *attr_name, }static int -ntb_xstats_get(const struct rte_rawdev *dev __rte_unused, - const unsigned int ids[] __rte_unused, - uint64_t values[] __rte_unused, - unsigned int n __rte_unused) +ntb_xstats_get (const struct rte_rawdev *dev, +const unsigned int ids[], +uint64_t values[], +unsigned int n) { -return 0 ; +struct ntb_hw *hw = dev->dev_private; +uint32_t i, j, off, xstats_num; + +/* Calculate total stats of all queues. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) { +for (j =0 ; j < hw->queue_pairs; j++) { + off =NTB_XSTATS_NUM * (j +1 ) + i; + hw->ntb_xstats[i] += hw->ntb_xstats[off]; + } + } + + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); +for (i =0 ; i < n && ids[i] < xstats_num; i++) + values[i] = hw->ntb_xstats[ids[i]]; + +return i; }static int -ntb_xstats_get_names(const struct rte_rawdev *dev __rte_unused, - struct rte_rawdev_xstats_name *xstats_names __rte_unused, - unsigned int size __rte_unused) +ntb_xstats_get_names (const struct rte_rawdev *dev, +struct rte_rawdev_xstats_name *xstats_names, +unsigned int size) { -return 0 ; +struct ntb_hw *hw = dev->dev_private; +uint32_t xstats_num, i, j, off; + + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); +if (xstats_names ==NULL || size < xstats_num) +return xstats_num; + +/* Total stats names */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) { +strncpy (xstats_names[i].name, ntb_xstats_names[i].name, +RTE_RAW_DEV_XSTATS_NAME_SIZE ); + } + +/* Queue stats names */ +for (i =0 ; i < hw->queue_pairs; i++) { +for (j =0 ; j <NTB_XSTATS_NUM ; j++) { + off = j + (i +1 ) *NTB_XSTATS_NUM ; +snprintf (xstats_names[off].name, +sizeof (xstats_names[0 ].name), +"%s_q%u" , ntb_xstats_names[j].name, i); + } + } + +return xstats_num; }static uint64_t -ntb_xstats_get_by_name(const struct rte_rawdev *dev __rte_unused, - const char *name __rte_unused, - unsigned int *id __rte_unused) +ntb_xstats_get_by_name (const struct rte_rawdev *dev, +const char *name,unsigned int *id) { -return 0 ; +struct rte_rawdev_xstats_name *xstats_names; +struct ntb_hw *hw = dev->dev_private; +uint32_t xstats_num, i, j, off; + +if (name ==NULL ) +return -EINVAL ; + + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); + xstats_names =rte_zmalloc ("ntb_stats_name" , +sizeof (struct rte_rawdev_xstats_name ) * + xstats_num,0 ); +ntb_xstats_get_names (dev, xstats_names, xstats_num); + +/* Calculate total stats of all queues. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) { +for (j =0 ; j < hw->queue_pairs; j++) { + off =NTB_XSTATS_NUM * (j +1 ) + i; + hw->ntb_xstats[i] += hw->ntb_xstats[off]; + } + } + +for (i =0 ; i < xstats_num; i++) { +if (!strncmp (name, xstats_names[i].name, +RTE_RAW_DEV_XSTATS_NAME_SIZE )) { + *id = i; +rte_free (xstats_names); +return hw->ntb_xstats[i]; + } + } + +NTB_LOG (ERR ,"Cannot find the xstats name." ); + +return -EINVAL ; }static int -ntb_xstats_reset(struct rte_rawdev *dev __rte_unused, - const uint32_t ids[] __rte_unused, - uint32_t nb_ids __rte_unused) +ntb_xstats_reset (struct rte_rawdev *dev, +const uint32_t ids[], +uint32_t nb_ids) { -return 0 ; -} +struct ntb_hw *hw = dev->dev_private; +uint32_t i, xstats_num; + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); +for (i =0 ; i < nb_ids && ids[i] < xstats_num; i++) + hw->ntb_xstats[ids[i]] =0 ; + +return i; +}static const struct rte_rawdev_ops ntb_ops = { .dev_info_get = ntb_dev_info_get,
drivers/raw/ntb/ntb.h +11 -0
@@ -27,6 +27,15 @@ extern int ntb_logtype;#define NTB_DFLT_TX_FREE_THRESH 256 +enum ntb_xstats_idx { +NTB_TX_PKTS_ID =0 , +NTB_TX_BYTES_ID , +NTB_TX_ERRS_ID , +NTB_RX_PKTS_ID , +NTB_RX_BYTES_ID , +NTB_RX_MISS_ID , +}; +enum ntb_topo {NTB_TOPO_NONE =0 ,NTB_TOPO_B2B_USD ,
@@ -216,6 +225,8 @@ struct ntb_hw {uint8_t peer_used_mws; +uint64_t *ntb_xstats; +/* Reserve several spad for app to use. */ int spad_user_list[NTB_SPAD_USER_MAX_NUM ]; };
Signature · 1 line
2.17.1
PATCH v2 3/4 raw/ntb: add enqueue and dequeue functions
In reply to Xiaoyun LiIntroduce enqueue and dequeue functions to support packet based processing. And enable write-combining for ntb driver since it can improve the performance a lot.
--- doc/guides/rawdevs/ntb.rst | 28 ++++ drivers/raw/ntb/ntb.c | 242 ++++++++++++++++++++++++++++++--- drivers/raw/ntb/ntb.h | 2 + drivers/raw/ntb/ntb_hw_intel.c | 22 +++ 4 files changed, 275 insertions(+), 19 deletions(-)
doc/guides/rawdevs/ntb.rst +28 -0
@@ -45,6 +45,24 @@ to use, i.e. igb_uio, vfio. The ``dpdk-devbind.py`` script can be used to show devices status and to bind them to a suitable kernel driver. They will appear under the category of "Misc (rawdev) devices". +Prerequisites +------------- +NTB PMD needs kernel PCI driver to support write combining (WC) to get +better performance. The difference will be more than 10 times. +To enable WC, there are 2 ways. +- Insert igb_uio with ``wc_active=1`` flag if use igb_uio driver. + insmod igb_uio.ko wc_active=1 +- Enable WC for NTB device's Bar 2 and Bar 4 (Mapped memory) manually. + Get bar base address using ``lspci -vvv -s ae:00.0 | grep Region``. + Region 0: Memory at 39bfe0000000 (64-bit, prefetchable) [size=64K] + Region 2: Memory at 39bfa0000000 (64-bit, prefetchable) [size=512M] + Region 4: Memory at 39bfc0000000 (64-bit, prefetchable) [size=512M] + Using the following command to enable WC. + echo "base=0x39bfa0000000 size=0x400000 type=write-combining" >> /proc/mtrr + echo "base=0x39bfc0000000 size=0x400000 type=write-combining" >> /proc/mtrr + To disable WC for these regions, using the following. + echo "disable=1" >> /proc/mtrr + Ring Layout -----------
@@ -83,6 +101,16 @@ like the following: +------------------------+ +------------------------+ <---------traffic--------- +- Enqueue and Dequeue + Based on this ring layout, enqueue reads rx_tail to get how many free + buffers and writes used_ring and tx_tail to tell the peer which buffers + are filled with data. + And dequeue reads tx_tail to get how many packets are arrived, and + writes desc_ring and rx_tail to tell the peer about the new allocated + buffers. + So in this way, only remote write happens and remote read can be avoid + to get better performance. + Limitation ----------
drivers/raw/ntb/ntb.c +223 -19
272 lines not on this page · raw message
drivers/raw/ntb/ntb.h +2 -0
@@ -87,6 +87,7 @@ enum ntb_spad_idx {* @ntb_dev_init: Init ntb dev. * @get_peer_mw_addr: To get the addr of peer mw[mw_idx]. * @mw_set_trans: Set translation of internal memory that remote can access. +* @ioremap: Translate the remote host address to bar address. * @get_link_status: get link status, link speed and link width. * @set_link: Set local side up/down. * @spad_read: Read local/peer spad register val.
@@ -103,6 +104,7 @@ struct ntb_dev_ops {void *(*get_peer_mw_addr)(const struct rte_rawdev *dev,int mw_idx);int (*mw_set_trans)(const struct rte_rawdev *dev,int mw_idx,uint64_t addr,uint64_t size); +void *(*ioremap)(const struct rte_rawdev *dev,uint64_t addr);int (*get_link_status)(const struct rte_rawdev *dev);int (*set_link)(const struct rte_rawdev *dev,bool up); uint32_t (*spad_read)(const struct rte_rawdev *dev, int spad,
drivers/raw/ntb/ntb_hw_intel.c +22 -0
@@ -162,6 +162,27 @@ intel_ntb_mw_set_trans(const struct rte_rawdev *dev, int mw_idx,return 0 ; } +static void * +intel_ntb_ioremap (const struct rte_rawdev *dev,uint64_t addr) +{ +struct ntb_hw *hw = dev->dev_private; +void *mapped =NULL ; +void *base; +int i; + +for (i =0 ; i < hw->peer_used_mws; i++) { +if (addr >= hw->peer_mw_base[i] && + addr <= hw->peer_mw_base[i] + hw->mw_size[i]) { + base =intel_ntb_get_peer_mw_addr (dev, i); + mapped = (void *)(size_t )(addr - hw->peer_mw_base[i] + + (size_t )base); +break ; + } + } + +return mapped; +} +static int intel_ntb_get_link_status (const struct rte_rawdev *dev) {
@@ -357,6 +378,7 @@ const struct ntb_dev_ops intel_ntb_ops = { .ntb_dev_init = intel_ntb_dev_init, .get_peer_mw_addr = intel_ntb_get_peer_mw_addr, .mw_set_trans = intel_ntb_mw_set_trans, + .ioremap = intel_ntb_ioremap, .get_link_status = intel_ntb_get_link_status, .set_link = intel_ntb_set_link, .spad_read = intel_ntb_spad_read,
Signature · 1 line
2.17.1
PATCH v2 4/4 examples/ntb: support more functions for NTB
In reply to Xiaoyun LiSupport to transmit files between two systems. Support iofwd between one ethdev and NTB device. Support rxonly and txonly for NTB device. Support to set forwarding mode as file-trans, txonly, rxonly or iofwd.
Support to show/clear port stats and throughput.
--- doc/guides/sample_app_ug/ntb.rst | 59 +- examples/ntb/meson.build | 3 + examples/ntb/ntb_fwd.c | 1298 +++++++++++++++++++++++++++--- 3 files changed, 1232 insertions(+), 128 deletions(-)
doc/guides/sample_app_ug/ntb.rst +53 -6
@@ -5,8 +5,17 @@ NTB Sample Application ====================== The ntb sample application shows how to use ntb rawdev driver. -This sample provides interactive mode to transmit file between -two hosts. +This sample provides interactive mode to do packet based processing +between two systems. + +This sample supports 4 types of packet forwarding mode. + +* ``file-trans``: transmit files between two systems. The sample will + be polling to receive files from the peer and save the file as + ``ntb_recv_file[N]``, [N] represents the number of received file. +* ``rxonly``: NTB receives packets but doesn't transmit them. +* ``txonly``: NTB generates and transmits packets without receiving any. +* ``iofwd``: iofwd between NTB device and ethdev. Compiling the Application -------------------------
@@ -29,6 +38,40 @@ Refer to the *DPDK Getting Started Guide* for general information on running applications and the Environment Abstraction Layer (EAL) options. +Command-line Options +-------------------- + +The application supports the following command-line options. + +* ``--buf-size=N`` + + Set the data size of the mbufs used to N bytes, where N < 65536. + The default value is 2048. + +* ``--fwd-mode=mode`` + + Set the packet forwarding mode as ``file-trans``, ``txonly``, + ``rxonly`` or ``iofwd``. + +* ``--nb-desc=N`` + + Set number of descriptors of queue as N, namely queue size, + where 64 <= N <= 1024. The default value is 1024. + +* ``--txfreet=N`` + + Set the transmit free threshold of TX rings to N, where 0 <= N <= + the value of ``--nb-desc``. The default value is 256. + +* ``--burst=N`` + + Set the number of packets per burst to N, where 1 <= N <= 32. + The default value is 32. + +* ``--qp=N`` + + Set the number of queues as N, where qp > 0. + Using the application ---------------------
@@ -41,7 +84,11 @@ The application is console-driven using the cmdline DPDK interface: From this interface the available commands and descriptions of what they do as as follows: -* ``send [filepath]``: Send file to the peer host. -* ``receive [filepath]``: Receive file to [filepath]. Need the peer - to send file successfully first. -* ``quit``: Exit program +* ``send [filepath]``: Send file to the peer host. Need to be in + file-trans forwarding mode first. +* ``start``: Start transmission. +* ``stop``: Stop transmission. +* ``show/clear port stats``: Show/Clear port stats and throughput. +* ``set fwd file-trans/rxonly/txonly/iofwd``: Set packet forwarding + mode. +* ``quit``: Exit program.
examples/ntb/meson.build +3 -0
@@ -14,3 +14,6 @@ cflags += ['-D_FILE_OFFSET_BITS=64'] sources = files( 'ntb_fwd.c' ) +if dpdk_conf.has('RTE_LIBRTE_PMD_NTB_RAWDEV') + deps += 'rawdev_ntb' +endif
examples/ntb/ntb_fwd.c +1176 -122
1,448 lines not on this page · raw message
Signature · 1 line
2.17.1
v3 cover enable FIFO for NTB
In reply to Xiaoyun LiEnable FIFO for NTB rawdev driver to support packet based processing. And an example is provided to support txonly, rxonly, iofwd between NTB device and ethdev, and file transmission.
---
v3:
* Replace strncpy with memcpy to avoid gcc-9 compile issue.
v2:
* Fixed compile issues with 32-bit machine and lack of including file.
* Fixed a typo.
Xiaoyun Li (4):
raw/ntb: setup ntb queue
raw/ntb: add xstats support
raw/ntb: add enqueue and dequeue functions
examples/ntb: support more functions for NTB
doc/guides/rawdevs/ntb.rst | 67 +- doc/guides/rel_notes/release_19_11.rst | 4 + doc/guides/sample_app_ug/ntb.rst | 59 +- drivers/raw/ntb/Makefile | 3 + drivers/raw/ntb/meson.build | 1 + drivers/raw/ntb/ntb.c | 1075 +++++++++++++++----- drivers/raw/ntb/ntb.h | 162 ++- drivers/raw/ntb/ntb_hw_intel.c | 48 +- drivers/raw/ntb/rte_pmd_ntb.h | 43 + examples/ntb/meson.build | 3 + examples/ntb/ntb_fwd.c | 1298 +++++++++++++++++++++--- 11 files changed, 2347 insertions(+), 416 deletions(-) create mode 100644 drivers/raw/ntb/rte_pmd_ntb.h
Signature · 1 line
2.17.1
PATCH v3 1/4 raw/ntb: setup ntb queue
Setup and init ntb txq and rxq. And negotiate queue information with the peer. If queue size and number of queues are not consistent on both sides, return error.
--- doc/guides/rawdevs/ntb.rst | 39 +- doc/guides/rel_notes/release_19_11.rst | 4 + drivers/raw/ntb/Makefile | 3 + drivers/raw/ntb/meson.build | 1 + drivers/raw/ntb/ntb.c | 705 ++++++++++++++++++------- drivers/raw/ntb/ntb.h | 151 ++++-- drivers/raw/ntb/ntb_hw_intel.c | 26 +- drivers/raw/ntb/rte_pmd_ntb.h | 43 ++ 8 files changed, 718 insertions(+), 254 deletions(-) create mode 100644 drivers/raw/ntb/rte_pmd_ntb.h
doc/guides/rawdevs/ntb.rst +38 -1
@@ -45,8 +45,45 @@ to use, i.e. igb_uio, vfio. The ``dpdk-devbind.py`` script can be used to show devices status and to bind them to a suitable kernel driver. They will appear under the category of "Misc (rawdev) devices". +Ring Layout +----------- + +Since read/write remote system's memory are through PCI bus, remote read +is much more expensive than remote write. Thus, the enqueue and dequeue +based on ntb ring should avoid remote read. The ring layout for ntb is +like the following: +- Ring Format: + desc_ring: + 0 16 64 + +---------------------------------------------------------------+ + | buffer address | + +---------------+-----------------------------------------------+ + | buffer length | resv | + +---------------+-----------------------------------------------+ + used_ring: + 0 16 32 + +---------------+---------------+ + | packet length | flags | + +---------------+---------------+ +- Ring Layout + +------------------------+ +------------------------+ + | used_ring | | desc_ring | + | +---+ | | +---+ | + | | | | | | | | + | +---+ +--------+ | | +---+ | + | | | ---> | buffer | <+---+-| | | + | +---+ +--------+ | | +---+ | + | | | | | | | | + | +---+ | | +---+ | + | ... | | ... | + | | | | + | +---------+ | | +---------+ | + | | tx_tail | | | | rx_tail | | + | System A +---------+ | | System B +---------+ | + +------------------------+ +------------------------+ + <---------traffic--------- + Limitation ---------- -- The FIFO hasn't been introduced and will come in 19.11 release. - This PMD only supports Intel Skylake platform.
doc/guides/rel_notes/release_19_11.rst +4 -0
@@ -56,6 +56,10 @@ New Features Also, make sure to start the actual text at the margin. ========================================================= + * **Introduced FIFO for NTB PMD.** + + Introduced FIFO for NTB (Non-transparent Bridge) PMD to support + packet based processing. Removed Items -------------
drivers/raw/ntb/Makefile +3 -0
@@ -25,4 +25,7 @@ LIBABIVER := 1 SRCS-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV) += ntb.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV) += ntb_hw_intel.c +# install this header file +SYMLINK-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV)-include := rte_pmd_ntb.h + include $(RTE_SDK)/mk/rte.lib.mk
drivers/raw/ntb/meson.build +1 -0
@@ -5,4 +5,5 @@ deps += ['rawdev', 'mbuf', 'mempool', 'pci', 'bus_pci'] sources = files('ntb.c', 'ntb_hw_intel.c') +install_headers('rte_pmd_ntb.h') allow_experimental_apis = true
drivers/raw/ntb/ntb.c +511 -194
943 lines not on this page · raw message
drivers/raw/ntb/ntb.h +105 -46
205 lines not on this page · raw message
drivers/raw/ntb/ntb_hw_intel.c +13 -13
@@ -26,7 +26,7 @@ static enum xeon_ntb_bar intel_ntb_bar[] = { };static int -intel_ntb_dev_init (struct rte_rawdev *dev) +intel_ntb_dev_init (const struct rte_rawdev *dev) {struct ntb_hw *hw = dev->dev_private;uint8_t reg_val, bar;
@@ -77,7 +77,7 @@ intel_ntb_dev_init(struct rte_rawdev *dev) hw->db_cnt =XEON_DB_COUNT ; hw->spad_cnt =XEON_SPAD_COUNT ; - hw->mw_size =rte_zmalloc ("uint64_t" , + hw->mw_size =rte_zmalloc ("ntb_mw_size" , hw->mw_cnt *sizeof (uint64_t ),0 );for (i =0 ; i < hw->mw_cnt; i++) { bar = intel_ntb_bar[i];
@@ -94,7 +94,7 @@ intel_ntb_dev_init(struct rte_rawdev *dev) }static void * -intel_ntb_get_peer_mw_addr (struct rte_rawdev *dev,int mw_idx) +intel_ntb_get_peer_mw_addr (const struct rte_rawdev *dev,int mw_idx) {struct ntb_hw *hw = dev->dev_private;uint8_t bar;
@@ -116,7 +116,7 @@ intel_ntb_get_peer_mw_addr(struct rte_rawdev *dev, int mw_idx) }static int -intel_ntb_mw_set_trans (struct rte_rawdev *dev,int mw_idx, +intel_ntb_mw_set_trans (const struct rte_rawdev *dev,int mw_idx,uint64_t addr,uint64_t size) {struct ntb_hw *hw = dev->dev_private;
@@ -163,7 +163,7 @@ intel_ntb_mw_set_trans(struct rte_rawdev *dev, int mw_idx, }static int -intel_ntb_get_link_status (struct rte_rawdev *dev) +intel_ntb_get_link_status (const struct rte_rawdev *dev) {struct ntb_hw *hw = dev->dev_private;uint16_t reg_val;
@@ -195,7 +195,7 @@ intel_ntb_get_link_status(struct rte_rawdev *dev) }static int -intel_ntb_set_link (struct rte_rawdev *dev,bool up) +intel_ntb_set_link (const struct rte_rawdev *dev,bool up) {struct ntb_hw *hw = dev->dev_private;uint32_t ntb_ctrl, reg_off;
@@ -221,7 +221,7 @@ intel_ntb_set_link(struct rte_rawdev *dev, bool up) }static uint32_t -intel_ntb_spad_read (struct rte_rawdev *dev,int spad,bool peer) +intel_ntb_spad_read (const struct rte_rawdev *dev,int spad,bool peer) {struct ntb_hw *hw = dev->dev_private;uint32_t spad_v, reg_off;
@@ -241,7 +241,7 @@ intel_ntb_spad_read(struct rte_rawdev *dev, int spad, bool peer) }static int -intel_ntb_spad_write (struct rte_rawdev *dev,int spad, +intel_ntb_spad_write (const struct rte_rawdev *dev,int spad,bool peer,uint32_t spad_v) {struct ntb_hw *hw = dev->dev_private;
@@ -263,7 +263,7 @@ intel_ntb_spad_write(struct rte_rawdev *dev, int spad, }static uint64_t -intel_ntb_db_read (struct rte_rawdev *dev) +intel_ntb_db_read (const struct rte_rawdev *dev) {struct ntb_hw *hw = dev->dev_private;uint64_t db_off, db_bits;
@@ -278,7 +278,7 @@ intel_ntb_db_read(struct rte_rawdev *dev) }static int -intel_ntb_db_clear (struct rte_rawdev *dev,uint64_t db_bits) +intel_ntb_db_clear (const struct rte_rawdev *dev,uint64_t db_bits) {struct ntb_hw *hw = dev->dev_private;uint64_t db_off;
@@ -293,7 +293,7 @@ intel_ntb_db_clear(struct rte_rawdev *dev, uint64_t db_bits) }static int -intel_ntb_db_set_mask (struct rte_rawdev *dev,uint64_t db_mask) +intel_ntb_db_set_mask (const struct rte_rawdev *dev,uint64_t db_mask) {struct ntb_hw *hw = dev->dev_private;uint64_t db_m_off;
@@ -312,7 +312,7 @@ intel_ntb_db_set_mask(struct rte_rawdev *dev, uint64_t db_mask) }static int -intel_ntb_peer_db_set (struct rte_rawdev *dev,uint8_t db_idx) +intel_ntb_peer_db_set (const struct rte_rawdev *dev,uint8_t db_idx) {struct ntb_hw *hw = dev->dev_private;uint32_t db_off;
@@ -332,7 +332,7 @@ intel_ntb_peer_db_set(struct rte_rawdev *dev, uint8_t db_idx) }static int -intel_ntb_vector_bind (struct rte_rawdev *dev,uint8_t intr,uint8_t msix) +intel_ntb_vector_bind (const struct rte_rawdev *dev,uint8_t intr,uint8_t msix) {struct ntb_hw *hw = dev->dev_private;uint8_t reg_off;
drivers/raw/ntb/rte_pmd_ntb.h new file +43 -0
43 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v3 2/4 raw/ntb: add xstats support
In reply to Xiaoyun LiAdd xstats support for ntb rawdev.
Support tx-packets, tx-bytes, tx-errors and
rx-packets, rx-bytes, rx-missed.
--- drivers/raw/ntb/ntb.c | 132 ++++++++++++++++++++++++++++++++++++------ drivers/raw/ntb/ntb.h | 11 ++++ 2 files changed, 125 insertions(+), 18 deletions(-)
drivers/raw/ntb/ntb.c +114 -18
@@ -30,6 +30,17 @@ static const struct rte_pci_id pci_id_ntb_map[] = { { .vendor_id = 0, /* sentinel */ }, }; +/* Align with enum ntb_xstats_idx */ +static struct rte_rawdev_xstats_name ntb_xstats_names[] = { + {"Tx-packets" }, + {"Tx-bytes" }, + {"Tx-errors" }, + {"Rx-packets" }, + {"Rx-bytes" }, + {"Rx-missed" }, +}; +#define NTB_XSTATS_NUM RTE_DIM(ntb_xstats_names) +static inline void ntb_link_cleanup (struct rte_rawdev *dev) {
@@ -538,6 +549,10 @@ ntb_queue_init(struct rte_rawdev *dev, uint16_t qp_id) txq->last_avail =0 ; txq->nb_tx_free = txq->nb_tx_desc -1 ; +/* Set per queue stats. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) + hw->ntb_xstats[i +NTB_XSTATS_NUM * (qp_id +1 )] =0 ; +return 0 ; }
@@ -614,6 +629,7 @@ ntb_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config) {struct ntb_dev_config *conf = config;struct ntb_hw *hw = dev->dev_private; +uint32_t xstats_num;int ret; hw->queue_pairs = conf->num_queues;
@@ -624,6 +640,10 @@ ntb_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config) sizeof(struct ntb_rx_queue *) * hw->queue_pairs, 0); hw->tx_queues =rte_zmalloc ("ntb_tx_queues" ,sizeof (struct ntb_tx_queue *) * hw->queue_pairs,0 ); +/* First total stats, then per queue stats. */ + xstats_num = (hw->queue_pairs +1 ) *NTB_XSTATS_NUM ; + hw->ntb_xstats =rte_zmalloc ("ntb_xstats" , xstats_num * +sizeof (uint64_t ),0 );/* Start handshake with the peer. */ ret =ntb_handshake_work (dev);
@@ -645,6 +665,10 @@ ntb_dev_start(struct rte_rawdev *dev)if (!hw->link_status || !hw->peer_dev_up)return -EINVAL ; +/* Set total stats. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) + hw->ntb_xstats[i] =0 ; +for (i =0 ; i < hw->queue_pairs; i++) { ret =ntb_queue_init (dev, i);if (ret) {
@@ -909,38 +933,110 @@ ntb_attr_get(struct rte_rawdev *dev, const char *attr_name, }static int -ntb_xstats_get(const struct rte_rawdev *dev __rte_unused, - const unsigned int ids[] __rte_unused, - uint64_t values[] __rte_unused, - unsigned int n __rte_unused) +ntb_xstats_get (const struct rte_rawdev *dev, +const unsigned int ids[], +uint64_t values[], +unsigned int n) { -return 0 ; +struct ntb_hw *hw = dev->dev_private; +uint32_t i, j, off, xstats_num; + +/* Calculate total stats of all queues. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) { +for (j =0 ; j < hw->queue_pairs; j++) { + off =NTB_XSTATS_NUM * (j +1 ) + i; + hw->ntb_xstats[i] += hw->ntb_xstats[off]; + } + } + + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); +for (i =0 ; i < n && ids[i] < xstats_num; i++) + values[i] = hw->ntb_xstats[ids[i]]; + +return i; }static int -ntb_xstats_get_names(const struct rte_rawdev *dev __rte_unused, - struct rte_rawdev_xstats_name *xstats_names __rte_unused, - unsigned int size __rte_unused) +ntb_xstats_get_names (const struct rte_rawdev *dev, +struct rte_rawdev_xstats_name *xstats_names, +unsigned int size) { -return 0 ; +struct ntb_hw *hw = dev->dev_private; +uint32_t xstats_num, i, j, off; + + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); +if (xstats_names ==NULL || size < xstats_num) +return xstats_num; + +/* Total stats names */ +memcpy (xstats_names, ntb_xstats_names,sizeof (ntb_xstats_names)); + +/* Queue stats names */ +for (i =0 ; i < hw->queue_pairs; i++) { +for (j =0 ; j <NTB_XSTATS_NUM ; j++) { + off = j + (i +1 ) *NTB_XSTATS_NUM ; +snprintf (xstats_names[off].name, +sizeof (xstats_names[0 ].name), +"%s_q%u" , ntb_xstats_names[j].name, i); + } + } + +return xstats_num; }static uint64_t -ntb_xstats_get_by_name(const struct rte_rawdev *dev __rte_unused, - const char *name __rte_unused, - unsigned int *id __rte_unused) +ntb_xstats_get_by_name (const struct rte_rawdev *dev, +const char *name,unsigned int *id) { -return 0 ; +struct rte_rawdev_xstats_name *xstats_names; +struct ntb_hw *hw = dev->dev_private; +uint32_t xstats_num, i, j, off; + +if (name ==NULL ) +return -EINVAL ; + + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); + xstats_names =rte_zmalloc ("ntb_stats_name" , +sizeof (struct rte_rawdev_xstats_name ) * + xstats_num,0 ); +ntb_xstats_get_names (dev, xstats_names, xstats_num); + +/* Calculate total stats of all queues. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) { +for (j =0 ; j < hw->queue_pairs; j++) { + off =NTB_XSTATS_NUM * (j +1 ) + i; + hw->ntb_xstats[i] += hw->ntb_xstats[off]; + } + } + +for (i =0 ; i < xstats_num; i++) { +if (!strncmp (name, xstats_names[i].name, +RTE_RAW_DEV_XSTATS_NAME_SIZE )) { + *id = i; +rte_free (xstats_names); +return hw->ntb_xstats[i]; + } + } + +NTB_LOG (ERR ,"Cannot find the xstats name." ); + +return -EINVAL ; }static int -ntb_xstats_reset(struct rte_rawdev *dev __rte_unused, - const uint32_t ids[] __rte_unused, - uint32_t nb_ids __rte_unused) +ntb_xstats_reset (struct rte_rawdev *dev, +const uint32_t ids[], +uint32_t nb_ids) { -return 0 ; -} +struct ntb_hw *hw = dev->dev_private; +uint32_t i, xstats_num; + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); +for (i =0 ; i < nb_ids && ids[i] < xstats_num; i++) + hw->ntb_xstats[ids[i]] =0 ; + +return i; +}static const struct rte_rawdev_ops ntb_ops = { .dev_info_get = ntb_dev_info_get,
drivers/raw/ntb/ntb.h +11 -0
@@ -27,6 +27,15 @@ extern int ntb_logtype;#define NTB_DFLT_TX_FREE_THRESH 256 +enum ntb_xstats_idx { +NTB_TX_PKTS_ID =0 , +NTB_TX_BYTES_ID , +NTB_TX_ERRS_ID , +NTB_RX_PKTS_ID , +NTB_RX_BYTES_ID , +NTB_RX_MISS_ID , +}; +enum ntb_topo {NTB_TOPO_NONE =0 ,NTB_TOPO_B2B_USD ,
@@ -216,6 +225,8 @@ struct ntb_hw {uint8_t peer_used_mws; +uint64_t *ntb_xstats; +/* Reserve several spad for app to use. */ int spad_user_list[NTB_SPAD_USER_MAX_NUM ]; };
Signature · 1 line
2.17.1
PATCH v3 3/4 raw/ntb: add enqueue and dequeue functions
In reply to Xiaoyun LiIntroduce enqueue and dequeue functions to support packet based processing. And enable write-combining for ntb driver since it can improve the performance a lot.
--- doc/guides/rawdevs/ntb.rst | 28 ++++ drivers/raw/ntb/ntb.c | 242 ++++++++++++++++++++++++++++++--- drivers/raw/ntb/ntb.h | 2 + drivers/raw/ntb/ntb_hw_intel.c | 22 +++ 4 files changed, 275 insertions(+), 19 deletions(-)
doc/guides/rawdevs/ntb.rst +28 -0
@@ -45,6 +45,24 @@ to use, i.e. igb_uio, vfio. The ``dpdk-devbind.py`` script can be used to show devices status and to bind them to a suitable kernel driver. They will appear under the category of "Misc (rawdev) devices". +Prerequisites +------------- +NTB PMD needs kernel PCI driver to support write combining (WC) to get +better performance. The difference will be more than 10 times. +To enable WC, there are 2 ways. +- Insert igb_uio with ``wc_active=1`` flag if use igb_uio driver. + insmod igb_uio.ko wc_active=1 +- Enable WC for NTB device's Bar 2 and Bar 4 (Mapped memory) manually. + Get bar base address using ``lspci -vvv -s ae:00.0 | grep Region``. + Region 0: Memory at 39bfe0000000 (64-bit, prefetchable) [size=64K] + Region 2: Memory at 39bfa0000000 (64-bit, prefetchable) [size=512M] + Region 4: Memory at 39bfc0000000 (64-bit, prefetchable) [size=512M] + Using the following command to enable WC. + echo "base=0x39bfa0000000 size=0x400000 type=write-combining" >> /proc/mtrr + echo "base=0x39bfc0000000 size=0x400000 type=write-combining" >> /proc/mtrr + To disable WC for these regions, using the following. + echo "disable=1" >> /proc/mtrr + Ring Layout -----------
@@ -83,6 +101,16 @@ like the following: +------------------------+ +------------------------+ <---------traffic--------- +- Enqueue and Dequeue + Based on this ring layout, enqueue reads rx_tail to get how many free + buffers and writes used_ring and tx_tail to tell the peer which buffers + are filled with data. + And dequeue reads tx_tail to get how many packets are arrived, and + writes desc_ring and rx_tail to tell the peer about the new allocated + buffers. + So in this way, only remote write happens and remote read can be avoid + to get better performance. + Limitation ----------
drivers/raw/ntb/ntb.c +223 -19
272 lines not on this page · raw message
drivers/raw/ntb/ntb.h +2 -0
@@ -87,6 +87,7 @@ enum ntb_spad_idx {* @ntb_dev_init: Init ntb dev. * @get_peer_mw_addr: To get the addr of peer mw[mw_idx]. * @mw_set_trans: Set translation of internal memory that remote can access. +* @ioremap: Translate the remote host address to bar address. * @get_link_status: get link status, link speed and link width. * @set_link: Set local side up/down. * @spad_read: Read local/peer spad register val.
@@ -103,6 +104,7 @@ struct ntb_dev_ops {void *(*get_peer_mw_addr)(const struct rte_rawdev *dev,int mw_idx);int (*mw_set_trans)(const struct rte_rawdev *dev,int mw_idx,uint64_t addr,uint64_t size); +void *(*ioremap)(const struct rte_rawdev *dev,uint64_t addr);int (*get_link_status)(const struct rte_rawdev *dev);int (*set_link)(const struct rte_rawdev *dev,bool up); uint32_t (*spad_read)(const struct rte_rawdev *dev, int spad,
drivers/raw/ntb/ntb_hw_intel.c +22 -0
@@ -162,6 +162,27 @@ intel_ntb_mw_set_trans(const struct rte_rawdev *dev, int mw_idx,return 0 ; } +static void * +intel_ntb_ioremap (const struct rte_rawdev *dev,uint64_t addr) +{ +struct ntb_hw *hw = dev->dev_private; +void *mapped =NULL ; +void *base; +int i; + +for (i =0 ; i < hw->peer_used_mws; i++) { +if (addr >= hw->peer_mw_base[i] && + addr <= hw->peer_mw_base[i] + hw->mw_size[i]) { + base =intel_ntb_get_peer_mw_addr (dev, i); + mapped = (void *)(size_t )(addr - hw->peer_mw_base[i] + + (size_t )base); +break ; + } + } + +return mapped; +} +static int intel_ntb_get_link_status (const struct rte_rawdev *dev) {
@@ -357,6 +378,7 @@ const struct ntb_dev_ops intel_ntb_ops = { .ntb_dev_init = intel_ntb_dev_init, .get_peer_mw_addr = intel_ntb_get_peer_mw_addr, .mw_set_trans = intel_ntb_mw_set_trans, + .ioremap = intel_ntb_ioremap, .get_link_status = intel_ntb_get_link_status, .set_link = intel_ntb_set_link, .spad_read = intel_ntb_spad_read,
Signature · 1 line
2.17.1
PATCH v3 4/4 examples/ntb: support more functions for NTB
In reply to Xiaoyun LiSupport to transmit files between two systems. Support iofwd between one ethdev and NTB device. Support rxonly and txonly for NTB device. Support to set forwarding mode as file-trans, txonly, rxonly or iofwd.
Support to show/clear port stats and throughput.
--- doc/guides/sample_app_ug/ntb.rst | 59 +- examples/ntb/meson.build | 3 + examples/ntb/ntb_fwd.c | 1298 +++++++++++++++++++++++++++--- 3 files changed, 1232 insertions(+), 128 deletions(-)
doc/guides/sample_app_ug/ntb.rst +53 -6
@@ -5,8 +5,17 @@ NTB Sample Application ====================== The ntb sample application shows how to use ntb rawdev driver. -This sample provides interactive mode to transmit file between -two hosts. +This sample provides interactive mode to do packet based processing +between two systems. + +This sample supports 4 types of packet forwarding mode. + +* ``file-trans``: transmit files between two systems. The sample will + be polling to receive files from the peer and save the file as + ``ntb_recv_file[N]``, [N] represents the number of received file. +* ``rxonly``: NTB receives packets but doesn't transmit them. +* ``txonly``: NTB generates and transmits packets without receiving any. +* ``iofwd``: iofwd between NTB device and ethdev. Compiling the Application -------------------------
@@ -29,6 +38,40 @@ Refer to the *DPDK Getting Started Guide* for general information on running applications and the Environment Abstraction Layer (EAL) options. +Command-line Options +-------------------- + +The application supports the following command-line options. + +* ``--buf-size=N`` + + Set the data size of the mbufs used to N bytes, where N < 65536. + The default value is 2048. + +* ``--fwd-mode=mode`` + + Set the packet forwarding mode as ``file-trans``, ``txonly``, + ``rxonly`` or ``iofwd``. + +* ``--nb-desc=N`` + + Set number of descriptors of queue as N, namely queue size, + where 64 <= N <= 1024. The default value is 1024. + +* ``--txfreet=N`` + + Set the transmit free threshold of TX rings to N, where 0 <= N <= + the value of ``--nb-desc``. The default value is 256. + +* ``--burst=N`` + + Set the number of packets per burst to N, where 1 <= N <= 32. + The default value is 32. + +* ``--qp=N`` + + Set the number of queues as N, where qp > 0. + Using the application ---------------------
@@ -41,7 +84,11 @@ The application is console-driven using the cmdline DPDK interface: From this interface the available commands and descriptions of what they do as as follows: -* ``send [filepath]``: Send file to the peer host. -* ``receive [filepath]``: Receive file to [filepath]. Need the peer - to send file successfully first. -* ``quit``: Exit program +* ``send [filepath]``: Send file to the peer host. Need to be in + file-trans forwarding mode first. +* ``start``: Start transmission. +* ``stop``: Stop transmission. +* ``show/clear port stats``: Show/Clear port stats and throughput. +* ``set fwd file-trans/rxonly/txonly/iofwd``: Set packet forwarding + mode. +* ``quit``: Exit program.
examples/ntb/meson.build +3 -0
@@ -14,3 +14,6 @@ cflags += ['-D_FILE_OFFSET_BITS=64'] sources = files( 'ntb_fwd.c' ) +if dpdk_conf.has('RTE_LIBRTE_PMD_NTB_RAWDEV') + deps += 'rawdev_ntb' +endif
examples/ntb/ntb_fwd.c +1176 -122
1,448 lines not on this page · raw message
Signature · 1 line
2.17.1
v4 cover enable FIFO for NTB
In reply to Xiaoyun LiEnable FIFO for NTB rawdev driver to support packet based processing. And an example is provided to support txonly, rxonly, iofwd between NTB device and ethdev, and file transmission.
---
v4:
* Fixed compile issues with 32-bit machine.
* Fixed total xstats issue.
v3:
* Replace strncpy with memcpy to avoid gcc-9 compile issue.
v2:
* Fixed compile issues with 32-bit machine and lack of including file.
* Fixed a typo.
Xiaoyun Li (4):
raw/ntb: setup ntb queue
raw/ntb: add xstats support
raw/ntb: add enqueue and dequeue functions
examples/ntb: support more functions for NTB
doc/guides/rawdevs/ntb.rst | 67 +- doc/guides/rel_notes/release_19_11.rst | 4 + doc/guides/sample_app_ug/ntb.rst | 59 +- drivers/raw/ntb/Makefile | 3 + drivers/raw/ntb/meson.build | 1 + drivers/raw/ntb/ntb.c | 1076 +++++++++++++++----- drivers/raw/ntb/ntb.h | 162 ++- drivers/raw/ntb/ntb_hw_intel.c | 48 +- drivers/raw/ntb/rte_pmd_ntb.h | 43 + examples/ntb/meson.build | 3 + examples/ntb/ntb_fwd.c | 1298 +++++++++++++++++++++--- 11 files changed, 2348 insertions(+), 416 deletions(-) create mode 100644 drivers/raw/ntb/rte_pmd_ntb.h
Signature · 1 line
2.17.1
PATCH v4 1/4 raw/ntb: setup ntb queue
Setup and init ntb txq and rxq. And negotiate queue information with the peer. If queue size and number of queues are not consistent on both sides, return error.
--- doc/guides/rawdevs/ntb.rst | 39 +- doc/guides/rel_notes/release_19_11.rst | 4 + drivers/raw/ntb/Makefile | 3 + drivers/raw/ntb/meson.build | 1 + drivers/raw/ntb/ntb.c | 705 ++++++++++++++++++------- drivers/raw/ntb/ntb.h | 151 ++++-- drivers/raw/ntb/ntb_hw_intel.c | 26 +- drivers/raw/ntb/rte_pmd_ntb.h | 43 ++ 8 files changed, 718 insertions(+), 254 deletions(-) create mode 100644 drivers/raw/ntb/rte_pmd_ntb.h
doc/guides/rawdevs/ntb.rst +38 -1
@@ -45,8 +45,45 @@ to use, i.e. igb_uio, vfio. The ``dpdk-devbind.py`` script can be used to show devices status and to bind them to a suitable kernel driver. They will appear under the category of "Misc (rawdev) devices". +Ring Layout +----------- + +Since read/write remote system's memory are through PCI bus, remote read +is much more expensive than remote write. Thus, the enqueue and dequeue +based on ntb ring should avoid remote read. The ring layout for ntb is +like the following: +- Ring Format: + desc_ring: + 0 16 64 + +---------------------------------------------------------------+ + | buffer address | + +---------------+-----------------------------------------------+ + | buffer length | resv | + +---------------+-----------------------------------------------+ + used_ring: + 0 16 32 + +---------------+---------------+ + | packet length | flags | + +---------------+---------------+ +- Ring Layout + +------------------------+ +------------------------+ + | used_ring | | desc_ring | + | +---+ | | +---+ | + | | | | | | | | + | +---+ +--------+ | | +---+ | + | | | ---> | buffer | <+---+-| | | + | +---+ +--------+ | | +---+ | + | | | | | | | | + | +---+ | | +---+ | + | ... | | ... | + | | | | + | +---------+ | | +---------+ | + | | tx_tail | | | | rx_tail | | + | System A +---------+ | | System B +---------+ | + +------------------------+ +------------------------+ + <---------traffic--------- + Limitation ---------- -- The FIFO hasn't been introduced and will come in 19.11 release. - This PMD only supports Intel Skylake platform.
doc/guides/rel_notes/release_19_11.rst +4 -0
@@ -56,6 +56,10 @@ New Features Also, make sure to start the actual text at the margin. ========================================================= + * **Introduced FIFO for NTB PMD.** + + Introduced FIFO for NTB (Non-transparent Bridge) PMD to support + packet based processing. Removed Items -------------
drivers/raw/ntb/Makefile +3 -0
@@ -25,4 +25,7 @@ LIBABIVER := 1 SRCS-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV) += ntb.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV) += ntb_hw_intel.c +# install this header file +SYMLINK-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV)-include := rte_pmd_ntb.h + include $(RTE_SDK)/mk/rte.lib.mk
drivers/raw/ntb/meson.build +1 -0
@@ -5,4 +5,5 @@ deps += ['rawdev', 'mbuf', 'mempool', 'pci', 'bus_pci'] sources = files('ntb.c', 'ntb_hw_intel.c') +install_headers('rte_pmd_ntb.h') allow_experimental_apis = true
drivers/raw/ntb/ntb.c +511 -194
943 lines not on this page · raw message
drivers/raw/ntb/ntb.h +105 -46
205 lines not on this page · raw message
drivers/raw/ntb/ntb_hw_intel.c +13 -13
@@ -26,7 +26,7 @@ static enum xeon_ntb_bar intel_ntb_bar[] = { };static int -intel_ntb_dev_init (struct rte_rawdev *dev) +intel_ntb_dev_init (const struct rte_rawdev *dev) {struct ntb_hw *hw = dev->dev_private;uint8_t reg_val, bar;
@@ -77,7 +77,7 @@ intel_ntb_dev_init(struct rte_rawdev *dev) hw->db_cnt =XEON_DB_COUNT ; hw->spad_cnt =XEON_SPAD_COUNT ; - hw->mw_size =rte_zmalloc ("uint64_t" , + hw->mw_size =rte_zmalloc ("ntb_mw_size" , hw->mw_cnt *sizeof (uint64_t ),0 );for (i =0 ; i < hw->mw_cnt; i++) { bar = intel_ntb_bar[i];
@@ -94,7 +94,7 @@ intel_ntb_dev_init(struct rte_rawdev *dev) }static void * -intel_ntb_get_peer_mw_addr (struct rte_rawdev *dev,int mw_idx) +intel_ntb_get_peer_mw_addr (const struct rte_rawdev *dev,int mw_idx) {struct ntb_hw *hw = dev->dev_private;uint8_t bar;
@@ -116,7 +116,7 @@ intel_ntb_get_peer_mw_addr(struct rte_rawdev *dev, int mw_idx) }static int -intel_ntb_mw_set_trans (struct rte_rawdev *dev,int mw_idx, +intel_ntb_mw_set_trans (const struct rte_rawdev *dev,int mw_idx,uint64_t addr,uint64_t size) {struct ntb_hw *hw = dev->dev_private;
@@ -163,7 +163,7 @@ intel_ntb_mw_set_trans(struct rte_rawdev *dev, int mw_idx, }static int -intel_ntb_get_link_status (struct rte_rawdev *dev) +intel_ntb_get_link_status (const struct rte_rawdev *dev) {struct ntb_hw *hw = dev->dev_private;uint16_t reg_val;
@@ -195,7 +195,7 @@ intel_ntb_get_link_status(struct rte_rawdev *dev) }static int -intel_ntb_set_link (struct rte_rawdev *dev,bool up) +intel_ntb_set_link (const struct rte_rawdev *dev,bool up) {struct ntb_hw *hw = dev->dev_private;uint32_t ntb_ctrl, reg_off;
@@ -221,7 +221,7 @@ intel_ntb_set_link(struct rte_rawdev *dev, bool up) }static uint32_t -intel_ntb_spad_read (struct rte_rawdev *dev,int spad,bool peer) +intel_ntb_spad_read (const struct rte_rawdev *dev,int spad,bool peer) {struct ntb_hw *hw = dev->dev_private;uint32_t spad_v, reg_off;
@@ -241,7 +241,7 @@ intel_ntb_spad_read(struct rte_rawdev *dev, int spad, bool peer) }static int -intel_ntb_spad_write (struct rte_rawdev *dev,int spad, +intel_ntb_spad_write (const struct rte_rawdev *dev,int spad,bool peer,uint32_t spad_v) {struct ntb_hw *hw = dev->dev_private;
@@ -263,7 +263,7 @@ intel_ntb_spad_write(struct rte_rawdev *dev, int spad, }static uint64_t -intel_ntb_db_read (struct rte_rawdev *dev) +intel_ntb_db_read (const struct rte_rawdev *dev) {struct ntb_hw *hw = dev->dev_private;uint64_t db_off, db_bits;
@@ -278,7 +278,7 @@ intel_ntb_db_read(struct rte_rawdev *dev) }static int -intel_ntb_db_clear (struct rte_rawdev *dev,uint64_t db_bits) +intel_ntb_db_clear (const struct rte_rawdev *dev,uint64_t db_bits) {struct ntb_hw *hw = dev->dev_private;uint64_t db_off;
@@ -293,7 +293,7 @@ intel_ntb_db_clear(struct rte_rawdev *dev, uint64_t db_bits) }static int -intel_ntb_db_set_mask (struct rte_rawdev *dev,uint64_t db_mask) +intel_ntb_db_set_mask (const struct rte_rawdev *dev,uint64_t db_mask) {struct ntb_hw *hw = dev->dev_private;uint64_t db_m_off;
@@ -312,7 +312,7 @@ intel_ntb_db_set_mask(struct rte_rawdev *dev, uint64_t db_mask) }static int -intel_ntb_peer_db_set (struct rte_rawdev *dev,uint8_t db_idx) +intel_ntb_peer_db_set (const struct rte_rawdev *dev,uint8_t db_idx) {struct ntb_hw *hw = dev->dev_private;uint32_t db_off;
@@ -332,7 +332,7 @@ intel_ntb_peer_db_set(struct rte_rawdev *dev, uint8_t db_idx) }static int -intel_ntb_vector_bind (struct rte_rawdev *dev,uint8_t intr,uint8_t msix) +intel_ntb_vector_bind (const struct rte_rawdev *dev,uint8_t intr,uint8_t msix) {struct ntb_hw *hw = dev->dev_private;uint8_t reg_off;
drivers/raw/ntb/rte_pmd_ntb.h new file +43 -0
43 lines not on this page · raw message
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v4 1/4] raw/ntb: setup ntb queue
<...>
drivers/raw/ntb/ntb.c ntb_dev_intr_handler(void *param) · line 282
Quoted 12 lines
drivers/raw/ntb/ntb.c +12
-0+static void +ntb_rxq_release (struct ntb_rx_queue *rxq) +{ +if (!rxq) { +NTB_LOG (ERR ,"Pointer to rxq is NULL" ); +return ; + } + +ntb_rxq_release_mbufs (rxq); + +rte_free (rxq->sw_ring); +rte_free (rxq);
It' better to free rxq out of this function, as the point of param "rxq" cannot be set to NULL in this func.
Quoted 10 lines
drivers/raw/ntb/ntb.c +10
-0+} + +static int +ntb_rxq_setup (struct rte_rawdev *dev, +uint16_t qp_id, +rte_rawdev_obj_t queue_conf) +{ +struct ntb_queue_conf *rxq_conf = queue_conf; +struct ntb_hw *hw = dev->dev_private; +struct ntb_rx_queue *rxq;
drivers/raw/ntb/ntb.c ntb_dev_intr_handler(void *param) · line 302
Quoted 10 lines
drivers/raw/ntb/ntb.c +10
-0+ +/* Allocate the rx queue data structure */ + rxq =rte_zmalloc_socket ("ntb rx queue" , +sizeof (struct ntb_rx_queue ), +RTE_CACHE_LINE_SIZE , + dev->socket_id); +if (!rxq) { +NTB_LOG (ERR ,"Failed to allocate memory for " +"rx queue data structure." ); +return -ENOMEM ;
Need to free rxq here.
<...>
drivers/raw/ntb/ntb.c ntb_dev_intr_handler(void *param) · line 362
Quoted 12 lines
drivers/raw/ntb/ntb.c +11
-0+static void +ntb_txq_release (struct ntb_tx_queue *txq) { +if (!txq) { +NTB_LOG (ERR ,"Pointer to txq is NULL" ); +return ; + } + +ntb_txq_release_mbufs (txq); + +rte_free (txq->sw_ring); +rte_free (txq);
The same as above "ntb_rxq_release".
<...>
drivers/raw/ntb/ntb.c ntb_dev_intr_handler(void *param) · line 436
Quoted 9 lines
drivers/raw/ntb/ntb.c +9
-0+static int +ntb_queue_setup (struct rte_rawdev *dev, +uint16_t queue_id, +rte_rawdev_obj_t queue_conf) +{ +struct ntb_hw *hw = dev->dev_private; +int ret; + +if (queue_id > hw->queue_pairs)
Should be ">=" ?
Quoted 11 lines
drivers/raw/ntb/ntb.c +11
-0+return -EINVAL ; + + ret =ntb_txq_setup (dev, queue_id, queue_conf); +if (ret <0 ) +return ret; + + ret =ntb_rxq_setup (dev, queue_id, queue_conf); + +return ret; +} +
drivers/raw/ntb/ntb.c ntb_dev_intr_handler(void *param) · line 455
Quoted 10 lines
drivers/raw/ntb/ntb.c +6
-2static int -ntb_queue_release(struct rte_rawdev *dev __rte_unused, - uint16_t queue_id __rte_unused) +ntb_queue_release (struct rte_rawdev *dev,uint16_t queue_id) { +struct ntb_hw *hw = dev->dev_private; +struct ntb_tx_queue *txq; +struct ntb_rx_queue *rxq; + +if (queue_id > hw->queue_pairs)
Should be ">=" ?
Quoted 19 lines
drivers/raw/ntb/ntb.c +12
-0+return -EINVAL ; + + txq = hw->tx_queues[queue_id]; + rxq = hw->rx_queues[queue_id]; +ntb_txq_release (txq); +ntb_rxq_release (rxq); +return 0 ; }@@ -234,6 +470,77 @@ ntb_queue_count(struct rte_rawdev *dev)return hw->queue_pairs; } +static int +ntb_queue_init (struct rte_rawdev *dev,uint16_t qp_id) +{ +struct ntb_hw *hw = dev->dev_private; +struct ntb_rx_queue *rxq = hw->rx_queues[qp_id];
drivers/raw/ntb/ntb.c ntb_queue_count(struct rte_rawdev *dev) · line 487
Quoted 10 lines
drivers/raw/ntb/ntb.c +10
-0+struct ntb_tx_queue *txq = hw->tx_queues[qp_id]; +volatile struct ntb_header *local_hdr; +struct ntb_header *remote_hdr; +uint16_t q_size = hw->queue_size; +uint32_t hdr_offset; +void *bar_addr; +uint16_t i; + +if (hw->ntb_ops->get_peer_mw_addr ==NULL ) { +NTB_LOG (ERR ,"Failed to get mapped peer addr." );
Would it be better to log as "XX ops is not supported" to keep consistent as others?
Quoted 15 lines
drivers/raw/ntb/ntb.c +15
-0+return -EINVAL ; + } + +/* Put queue info into the start of shared memory. */ + hdr_offset = hw->hdr_size_per_queue * qp_id; + local_hdr = (volatile struct ntb_header *) + ((size_t )hw->mz[0 ]->addr + hdr_offset); + bar_addr = (*hw->ntb_ops->get_peer_mw_addr)(dev,0 ); +if (bar_addr ==NULL ) +return -EINVAL ; + remote_hdr = (struct ntb_header *) + ((size_t )bar_addr + hdr_offset); + +/* rxq init. */ + rxq->rx_desc_ring = (struct ntb_desc *)
drivers/raw/ntb/ntb.c ntb_queue_count(struct rte_rawdev *dev) · line 512
Quoted 10 lines
drivers/raw/ntb/ntb.c +10
-0+ (&remote_hdr->desc_ring); + rxq->rx_used_ring = (volatile struct ntb_used *) + (&local_hdr->desc_ring[q_size]); + rxq->avail_cnt = &remote_hdr->avail_cnt; + rxq->used_cnt = &local_hdr->used_cnt; + +for (i =0 ; i < rxq->nb_rx_desc -1 ; i++) { +struct rte_mbuf *mbuf =rte_mbuf_raw_alloc (rxq->mpool); +if (unlikely (!mbuf)) { +NTB_LOG (ERR ,"Failed to allocate mbuf for RX" );
Need release mbufs allocated here or in " ntb_dev_start".
<...>
drivers/raw/ntb/ntb.c static void · line 608
Quoted 4 lines
drivers/raw/ntb/ntb.c +4
-0+ hw->hdr_size_per_queue =RTE_ALIGN (sizeof (struct ntb_header ) + + hw->queue_size *sizeof (struct ntb_desc ) + + hw->queue_size *sizeof (struct ntb_used ), +RTE_CACHE_LINE_SIZE );
hw->hdr_size_per_queue is internal information, why put the assignment in ntb_dev_info_get?
Quoted 14 lines
drivers/raw/ntb/ntb.c +8
-2+ info->ntb_hdr_size = hw->hdr_size_per_queue * hw->queue_pairs; }static int -ntb_dev_configure(const struct rte_rawdev *dev __rte_unused, - rte_rawdev_obj_t config __rte_unused) +ntb_dev_configure (const struct rte_rawdev *dev,rte_rawdev_obj_t config) { +struct ntb_dev_config *conf = config; +struct ntb_hw *hw = dev->dev_private; +int ret; + + hw->queue_pairs = conf->num_queues; + hw->queue_size = conf->queue_size;
drivers/raw/ntb/ntb.c static void · line 630
Quoted 10 lines
drivers/raw/ntb/ntb.c +10
-0+ hw->used_mw_num = conf->mz_num; + hw->mz = conf->mz_list; + hw->rx_queues =rte_zmalloc ("ntb_rx_queues" , +sizeof (struct ntb_rx_queue *) * hw->queue_pairs,0 ); + hw->tx_queues =rte_zmalloc ("ntb_tx_queues" , +sizeof (struct ntb_tx_queue *) * hw->queue_pairs,0 ); + +/* Start handshake with the peer. */ + ret =ntb_handshake_work (dev); +if (ret <0 )
Free?
Quoted 19 lines
drivers/raw/ntb/ntb.c +8
-2+return ret; +return 0 ; }@@ -337,21 +637,52 @@ static int ntb_dev_start(struct rte_rawdev *dev) {struct ntb_hw *hw = dev->dev_private; -int ret, i; +uint32_t peer_base_l, peer_val; +uint64_t peer_base_h; +uint32_t i; +int ret; -/* TODO: init queues and start queues. */ +if (!hw->link_status || !hw->peer_dev_up) +return -EINVAL ;
drivers/raw/ntb/ntb.c static int · line 651
Quoted 10 lines
drivers/raw/ntb/ntb.c +3
-6-/* Map memory of bar_size to remote. */ - hw->mz =rte_zmalloc ("struct rte_memzone *" , - hw->mw_cnt *sizeof (struct rte_memzone *),0 ); -for (i =0 ; i < hw->mw_cnt; i++) { - ret =ntb_set_mw (dev, i, hw->mw_size[i]); +for (i =0 ; i < hw->queue_pairs; i++) { + ret =ntb_queue_init (dev, i);if (ret) { -NTB_LOG (ERR ,"Fail to set mw." ); +NTB_LOG (ERR ,"Failed to init queue." );
Free when error.
<...>
drivers/raw/ntb/ntb.h enum ntb_spad_idx { · line 120
Quoted 3 lines
drivers/raw/ntb/ntb.h +3
-0+struct ntb_used { +uint16_t len;/* buffer length */ +#define NTB_FLAG_EOP 1/* end of packet */
Better to
Quoted 9 lines
drivers/raw/ntb/ntb.h +9
-0+uint16_t flags;/* flags */ +}; + +struct ntb_rx_entry { +struct rte_mbuf *mbuf; +}; + +struct ntb_rx_queue { +struct ntb_desc *rx_desc_ring;
drivers/raw/ntb/ntb.h enum ntb_spad_idx { · line 139
Quoted 10 lines
drivers/raw/ntb/ntb.h +10
-0+volatile struct ntb_used *rx_used_ring; +uint16_t *avail_cnt; +volatile uint16_t *used_cnt; +uint16_t last_avail; +uint16_t last_used; +uint16_t nb_rx_desc; + +uint16_t rx_free_thresh; + +struct rte_mempool *mpool;/**< mempool for mbuf allocation */
Generally comments: comments in internal header doesn't need to be wrapped with "/**< */", keep consistent in one file would be nice.
Re: [dpdk-dev] [PATCH v4 1/4] raw/ntb: setup ntb queue
Hi
Quoted 10 lines
-----Original Message-----
From: Wu, Jingjing
Sent: Monday, September 23, 2019 10:50
To: Li, Xiaoyun <[email protected]>; Wiles, Keith <[email protected]>; Maslekar, Omkar <[email protected]>; Liang, Cunming <[email protected]>
Cc: [email protected]
Subject: RE: [PATCH v4 1/4] raw/ntb: setup ntb queue<...>
Quoted 13 lines
drivers/raw/ntb/ntb.c ntb_dev_intr_handler(void *param) · line 282
Quoted 11 lines
drivers/raw/ntb/ntb.c +11
-0+static void +ntb_rxq_release (struct ntb_rx_queue *rxq) { +if (!rxq) { +NTB_LOG (ERR ,"Pointer to rxq is NULL" ); +return ; + } + +ntb_rxq_release_mbufs (rxq); + +rte_free (rxq->sw_ring); +rte_free (rxq);It' better to free rxq out of this function, as the point of param "rxq" cannot be set to NULL in this func.
OK.
Quoted 14 lines
Quoted 14 lines
drivers/raw/ntb/ntb.c +13
-0+} + +static int +ntb_rxq_setup (struct rte_rawdev *dev, +uint16_t qp_id, +rte_rawdev_obj_t queue_conf) +{ +struct ntb_queue_conf *rxq_conf = queue_conf; +struct ntb_hw *hw = dev->dev_private; +struct ntb_rx_queue *rxq; + +/* Allocate the rx queue data structure */ + rxq = rte_zmalloc_socket("ntb rx queue",
Quoted 8 lines
drivers/raw/ntb/ntb.c ntb_dev_intr_handler(void *param) · line 302
Quoted 7 lines
drivers/raw/ntb/ntb.c +7
-0+sizeof (struct ntb_rx_queue ), +RTE_CACHE_LINE_SIZE , + dev->socket_id); +if (!rxq) { +NTB_LOG (ERR ,"Failed to allocate memory for " +"rx queue data structure." ); +return -ENOMEM ;Need to free rxq here.
It only allocates memory. And this error means allocate failure and rxq == NULL. So no need to free here.
Quoted 15 lines
<...>
drivers/raw/ntb/ntb.c ntb_dev_intr_handler(void *param) · line 362
Quoted 12 lines
drivers/raw/ntb/ntb.c +11
-0+static void +ntb_txq_release (struct ntb_tx_queue *txq) { +if (!txq) { +NTB_LOG (ERR ,"Pointer to txq is NULL" ); +return ; + } + +ntb_txq_release_mbufs (txq); + +rte_free (txq->sw_ring); +rte_free (txq);The same as above "
ntb_rxq_release".
OK.
Quoted 13 lines
<...>
drivers/raw/ntb/ntb.c ntb_dev_intr_handler(void *param) · line 436
Quoted 9 lines
drivers/raw/ntb/ntb.c +9
-0+static int +ntb_queue_setup (struct rte_rawdev *dev, +uint16_t queue_id, +rte_rawdev_obj_t queue_conf) +{ +struct ntb_hw *hw = dev->dev_private; +int ret; + +if (queue_id > hw->queue_pairs)Should be ">=" ?
Yes.
Quoted 38 lines
Quoted 12 lines
drivers/raw/ntb/ntb.c +11
-0+return -EINVAL ; + + ret =ntb_txq_setup (dev, queue_id, queue_conf); +if (ret <0 ) +return ret; + + ret =ntb_rxq_setup (dev, queue_id, queue_conf); + +return ret; +} +drivers/raw/ntb/ntb.c ntb_dev_intr_handler(void *param) · line 455
Quoted 10 lines
drivers/raw/ntb/ntb.c +6
-2static int -ntb_queue_release(struct rte_rawdev *dev __rte_unused, - uint16_t queue_id __rte_unused) +ntb_queue_release (struct rte_rawdev *dev,uint16_t queue_id) { +struct ntb_hw *hw = dev->dev_private; +struct ntb_tx_queue *txq; +struct ntb_rx_queue *rxq; + +if (queue_id > hw->queue_pairs)Should be ">=" ?
Quoted 14 lines
drivers/raw/ntb/ntb.c +7
-0+return -EINVAL ; + + txq = hw->tx_queues[queue_id]; + rxq = hw->rx_queues[queue_id]; +ntb_txq_release (txq); +ntb_rxq_release (rxq); +return 0 ; }@@ -234,6 +470,77 @@ ntb_queue_count(struct rte_rawdev *dev)return hw->queue_pairs; }
Quoted 16 lines
Quoted 4 lines
drivers/raw/ntb/ntb.c +4
-0+static int +ntb_queue_init (struct rte_rawdev *dev,uint16_t qp_id) { +struct ntb_hw *hw = dev->dev_private; +struct ntb_rx_queue *rxq = hw->rx_queues[qp_id];drivers/raw/ntb/ntb.c ntb_queue_count(struct rte_rawdev *dev) · line 487
Quoted 10 lines
drivers/raw/ntb/ntb.c +10
-0+struct ntb_tx_queue *txq = hw->tx_queues[qp_id]; +volatile struct ntb_header *local_hdr; +struct ntb_header *remote_hdr; +uint16_t q_size = hw->queue_size; +uint32_t hdr_offset; +void *bar_addr; +uint16_t i; + +if (hw->ntb_ops->get_peer_mw_addr ==NULL ) { +NTB_LOG (ERR ,"Failed to get mapped peer addr." );Would it be better to log as "XX ops is not supported" to keep consistent as others?
OK.
Quoted 19 lines
Quoted 19 lines
drivers/raw/ntb/ntb.c +18
-0+return -EINVAL ; + } + +/* Put queue info into the start of shared memory. */ + hdr_offset = hw->hdr_size_per_queue * qp_id; + local_hdr = (volatile struct ntb_header *) + ((size_t )hw->mz[0 ]->addr + hdr_offset); + bar_addr = (*hw->ntb_ops->get_peer_mw_addr)(dev,0 ); +if (bar_addr ==NULL ) +return -EINVAL ; + remote_hdr = (struct ntb_header *) + ((size_t )bar_addr + hdr_offset); + +/* rxq init. */ + rxq->rx_desc_ring = (struct ntb_desc *) + (&remote_hdr->desc_ring); + rxq->rx_used_ring = (volatile struct ntb_used *) + (&local_hdr->desc_ring[q_size]);
Quoted 8 lines
drivers/raw/ntb/ntb.c ntb_queue_count(struct rte_rawdev *dev) · line 512
Quoted 7 lines
drivers/raw/ntb/ntb.c +7
-0+ rxq->avail_cnt = &remote_hdr->avail_cnt; + rxq->used_cnt = &local_hdr->used_cnt; + +for (i =0 ; i < rxq->nb_rx_desc -1 ; i++) { +struct rte_mbuf *mbuf =rte_mbuf_raw_alloc (rxq->mpool); +if (unlikely (!mbuf)) { +NTB_LOG (ERR ,"Failed to allocate mbuf for RX" );Need release mbufs allocated here or in "
ntb_dev_start".
OK.
Quoted 9 lines
<...>
drivers/raw/ntb/ntb.c static void · line 608
Quoted 4 lines
drivers/raw/ntb/ntb.c +4
-0+ hw->hdr_size_per_queue =RTE_ALIGN (sizeof (struct ntb_header ) + + hw->queue_size *sizeof (struct ntb_desc ) + + hw->queue_size *sizeof (struct ntb_used ), +RTE_CACHE_LINE_SIZE );
hw->hdr_size_per_queueis internal information, why put the assignment inntb_dev_info_get?
Because the calculation needs the app to pass params (queue size and queue number). And the app needs the result to populate mempool and then configure to driver. There is no else place where can do the calculation.
Quoted 19 lines
Quoted 19 lines
drivers/raw/ntb/ntb.c +12
-2+ info->ntb_hdr_size = hw->hdr_size_per_queue * hw->queue_pairs; }static int -ntb_dev_configure(const struct rte_rawdev *dev __rte_unused, - rte_rawdev_obj_t config __rte_unused) +ntb_dev_configure (const struct rte_rawdev *dev,rte_rawdev_obj_t +config) { +struct ntb_dev_config *conf = config; +struct ntb_hw *hw = dev->dev_private; +int ret; + + hw->queue_pairs = conf->num_queues; + hw->queue_size = conf->queue_size; + hw->used_mw_num = conf->mz_num; + hw->mz = conf->mz_list; + hw->rx_queues = rte_zmalloc("ntb_rx_queues",
Quoted 8 lines
drivers/raw/ntb/ntb.c static void · line 630
Quoted 7 lines
drivers/raw/ntb/ntb.c +7
-0+ sizeof(struct ntb_rx_queue *) * hw->queue_pairs, 0); + hw->tx_queues =rte_zmalloc ("ntb_tx_queues" , +sizeof (struct ntb_tx_queue *) * hw->queue_pairs,0 ); + +/* Start handshake with the peer. */ + ret =ntb_handshake_work (dev); +if (ret <0 )Free?
OK.
Quoted 19 lines
Quoted 19 lines
drivers/raw/ntb/ntb.c +8
-2+return ret; +return 0 ; }@@ -337,21 +637,52 @@ static int ntb_dev_start(struct rte_rawdev *dev) {struct ntb_hw *hw = dev->dev_private; -int ret, i; +uint32_t peer_base_l, peer_val; +uint64_t peer_base_h; +uint32_t i; +int ret; -/* TODO: init queues and start queues. */ +if (!hw->link_status || !hw->peer_dev_up) +return -EINVAL ;
Quoted 11 lines
drivers/raw/ntb/ntb.c static int · line 651
Quoted 10 lines
drivers/raw/ntb/ntb.c +3
-6-/* Map memory of bar_size to remote. */ - hw->mz =rte_zmalloc ("struct rte_memzone *" , - hw->mw_cnt *sizeof (struct rte_memzone *),0 ); -for (i =0 ; i < hw->mw_cnt; i++) { - ret =ntb_set_mw (dev, i, hw->mw_size[i]); +for (i =0 ; i < hw->queue_pairs; i++) { + ret =ntb_queue_init (dev, i);if (ret) { -NTB_LOG (ERR ,"Fail to set mw." ); +NTB_LOG (ERR ,"Failed to init queue." );Free when error.
OK.
Quoted 20 lines
<...>
drivers/raw/ntb/ntb.h enum ntb_spad_idx { · line 120
Quoted 3 lines
drivers/raw/ntb/ntb.h +3
-0+struct ntb_used { +uint16_t len;/* buffer length */ +#define NTB_FLAG_EOP 1/* end of packet */ Better to
Quoted 13 lines
drivers/raw/ntb/ntb.h +13
-0+uint16_t flags;/* flags */ +}; + +struct ntb_rx_entry { +struct rte_mbuf *mbuf; +}; + +struct ntb_rx_queue { +struct ntb_desc *rx_desc_ring; +volatile struct ntb_used *rx_used_ring; +uint16_t *avail_cnt; +volatile uint16_t *used_cnt; +uint16_t last_avail;
Quoted 8 lines
drivers/raw/ntb/ntb.h enum ntb_spad_idx { · line 139
Quoted 6 lines
drivers/raw/ntb/ntb.h +6
-0+uint16_t last_used; +uint16_t nb_rx_desc; + +uint16_t rx_free_thresh; + +struct rte_mempool *mpool;/**< mempool for mbuf allocation */ Generally comments: comments in internal header doesn't need to be wrapped with "/**< */", keep consistent in one file would be nice.
OK.
PATCH v4 2/4 raw/ntb: add xstats support
In reply to Xiaoyun LiAdd xstats support for ntb rawdev.
Support tx-packets, tx-bytes, tx-errors and
rx-packets, rx-bytes, rx-missed.
--- drivers/raw/ntb/ntb.c | 133 ++++++++++++++++++++++++++++++++++++------ drivers/raw/ntb/ntb.h | 11 ++++ 2 files changed, 126 insertions(+), 18 deletions(-)
drivers/raw/ntb/ntb.c +115 -18
@@ -30,6 +30,17 @@ static const struct rte_pci_id pci_id_ntb_map[] = { { .vendor_id = 0, /* sentinel */ }, }; +/* Align with enum ntb_xstats_idx */ +static struct rte_rawdev_xstats_name ntb_xstats_names[] = { + {"Tx-packets" }, + {"Tx-bytes" }, + {"Tx-errors" }, + {"Rx-packets" }, + {"Rx-bytes" }, + {"Rx-missed" }, +}; +#define NTB_XSTATS_NUM RTE_DIM(ntb_xstats_names) +static inline void ntb_link_cleanup (struct rte_rawdev *dev) {
@@ -538,6 +549,10 @@ ntb_queue_init(struct rte_rawdev *dev, uint16_t qp_id) txq->last_avail =0 ; txq->nb_tx_free = txq->nb_tx_desc -1 ; +/* Set per queue stats. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) + hw->ntb_xstats[i +NTB_XSTATS_NUM * (qp_id +1 )] =0 ; +return 0 ; }
@@ -614,6 +629,7 @@ ntb_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config) {struct ntb_dev_config *conf = config;struct ntb_hw *hw = dev->dev_private; +uint32_t xstats_num;int ret; hw->queue_pairs = conf->num_queues;
@@ -624,6 +640,10 @@ ntb_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config) sizeof(struct ntb_rx_queue *) * hw->queue_pairs, 0); hw->tx_queues =rte_zmalloc ("ntb_tx_queues" ,sizeof (struct ntb_tx_queue *) * hw->queue_pairs,0 ); +/* First total stats, then per queue stats. */ + xstats_num = (hw->queue_pairs +1 ) *NTB_XSTATS_NUM ; + hw->ntb_xstats =rte_zmalloc ("ntb_xstats" , xstats_num * +sizeof (uint64_t ),0 );/* Start handshake with the peer. */ ret =ntb_handshake_work (dev);
@@ -645,6 +665,10 @@ ntb_dev_start(struct rte_rawdev *dev)if (!hw->link_status || !hw->peer_dev_up)return -EINVAL ; +/* Set total stats. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) + hw->ntb_xstats[i] =0 ; +for (i =0 ; i < hw->queue_pairs; i++) { ret =ntb_queue_init (dev, i);if (ret) {
@@ -909,38 +933,111 @@ ntb_attr_get(struct rte_rawdev *dev, const char *attr_name, }static int -ntb_xstats_get(const struct rte_rawdev *dev __rte_unused, - const unsigned int ids[] __rte_unused, - uint64_t values[] __rte_unused, - unsigned int n __rte_unused) +ntb_xstats_get (const struct rte_rawdev *dev, +const unsigned int ids[], +uint64_t values[], +unsigned int n) { -return 0 ; +struct ntb_hw *hw = dev->dev_private; +uint32_t i, j, off, xstats_num; + +/* Calculate total stats of all queues. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) { + hw->ntb_xstats[i] =0 ; +for (j =0 ; j < hw->queue_pairs; j++) { + off =NTB_XSTATS_NUM * (j +1 ) + i; + hw->ntb_xstats[i] += hw->ntb_xstats[off]; + } + } + + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); +for (i =0 ; i < n && ids[i] < xstats_num; i++) + values[i] = hw->ntb_xstats[ids[i]]; + +return i; }static int -ntb_xstats_get_names(const struct rte_rawdev *dev __rte_unused, - struct rte_rawdev_xstats_name *xstats_names __rte_unused, - unsigned int size __rte_unused) +ntb_xstats_get_names (const struct rte_rawdev *dev, +struct rte_rawdev_xstats_name *xstats_names, +unsigned int size) { -return 0 ; +struct ntb_hw *hw = dev->dev_private; +uint32_t xstats_num, i, j, off; + + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); +if (xstats_names ==NULL || size < xstats_num) +return xstats_num; + +/* Total stats names */ +memcpy (xstats_names, ntb_xstats_names,sizeof (ntb_xstats_names)); + +/* Queue stats names */ +for (i =0 ; i < hw->queue_pairs; i++) { +for (j =0 ; j <NTB_XSTATS_NUM ; j++) { + off = j + (i +1 ) *NTB_XSTATS_NUM ; +snprintf (xstats_names[off].name, +sizeof (xstats_names[0 ].name), +"%s_q%u" , ntb_xstats_names[j].name, i); + } + } + +return xstats_num; }static uint64_t -ntb_xstats_get_by_name(const struct rte_rawdev *dev __rte_unused, - const char *name __rte_unused, - unsigned int *id __rte_unused) +ntb_xstats_get_by_name (const struct rte_rawdev *dev, +const char *name,unsigned int *id) { -return 0 ; +struct rte_rawdev_xstats_name *xstats_names; +struct ntb_hw *hw = dev->dev_private; +uint32_t xstats_num, i, j, off; + +if (name ==NULL ) +return -EINVAL ; + + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); + xstats_names =rte_zmalloc ("ntb_stats_name" , +sizeof (struct rte_rawdev_xstats_name ) * + xstats_num,0 ); +ntb_xstats_get_names (dev, xstats_names, xstats_num); + +/* Calculate total stats of all queues. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) { +for (j =0 ; j < hw->queue_pairs; j++) { + off =NTB_XSTATS_NUM * (j +1 ) + i; + hw->ntb_xstats[i] += hw->ntb_xstats[off]; + } + } + +for (i =0 ; i < xstats_num; i++) { +if (!strncmp (name, xstats_names[i].name, +RTE_RAW_DEV_XSTATS_NAME_SIZE )) { + *id = i; +rte_free (xstats_names); +return hw->ntb_xstats[i]; + } + } + +NTB_LOG (ERR ,"Cannot find the xstats name." ); + +return -EINVAL ; }static int -ntb_xstats_reset(struct rte_rawdev *dev __rte_unused, - const uint32_t ids[] __rte_unused, - uint32_t nb_ids __rte_unused) +ntb_xstats_reset (struct rte_rawdev *dev, +const uint32_t ids[], +uint32_t nb_ids) { -return 0 ; -} +struct ntb_hw *hw = dev->dev_private; +uint32_t i, xstats_num; + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); +for (i =0 ; i < nb_ids && ids[i] < xstats_num; i++) + hw->ntb_xstats[ids[i]] =0 ; + +return i; +}static const struct rte_rawdev_ops ntb_ops = { .dev_info_get = ntb_dev_info_get,
drivers/raw/ntb/ntb.h +11 -0
@@ -27,6 +27,15 @@ extern int ntb_logtype;#define NTB_DFLT_TX_FREE_THRESH 256 +enum ntb_xstats_idx { +NTB_TX_PKTS_ID =0 , +NTB_TX_BYTES_ID , +NTB_TX_ERRS_ID , +NTB_RX_PKTS_ID , +NTB_RX_BYTES_ID , +NTB_RX_MISS_ID , +}; +enum ntb_topo {NTB_TOPO_NONE =0 ,NTB_TOPO_B2B_USD ,
@@ -216,6 +225,8 @@ struct ntb_hw {uint8_t peer_used_mws; +uint64_t *ntb_xstats; +/* Reserve several spad for app to use. */ int spad_user_list[NTB_SPAD_USER_MAX_NUM ]; };
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v4 2/4] raw/ntb: add xstats support
Quoted 7 lines
static int
drivers/raw/ntb/ntb.c +3
-3-ntb_xstats_reset(struct rte_rawdev *dev __rte_unused, - const uint32_t ids[] __rte_unused, - uint32_t nb_ids __rte_unused) +ntb_xstats_reset(struct rte_rawdev *dev, + const uint32_t ids[], + uint32_t nb_ids)
drivers/raw/ntb/ntb.c ntb_attr_get(struct rte_rawdev *dev, const char *attr_name, · line 1038
Quoted 10 lines
drivers/raw/ntb/ntb.c +6
-2{ -return 0 ; -} +struct ntb_hw *hw = dev->dev_private; +uint32_t i, xstats_num; + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); +for (i =0 ; i < nb_ids && ids[i] < xstats_num; i++) + hw->ntb_xstats[ids[i]] =0 ; +
As there is no lock for the xstats, the enqueue and dequeuer thread are updating the value. It will cause competition. Suggest to save the ntx_xstats, and update the value when enqueue and dequeuer are updating.
Thanks
Jingjing
PATCH v4 3/4 raw/ntb: add enqueue and dequeue functions
In reply to Xiaoyun LiIntroduce enqueue and dequeue functions to support packet based processing. And enable write-combining for ntb driver since it can improve the performance a lot.
--- doc/guides/rawdevs/ntb.rst | 28 ++++ drivers/raw/ntb/ntb.c | 242 ++++++++++++++++++++++++++++++--- drivers/raw/ntb/ntb.h | 2 + drivers/raw/ntb/ntb_hw_intel.c | 22 +++ 4 files changed, 275 insertions(+), 19 deletions(-)
doc/guides/rawdevs/ntb.rst +28 -0
@@ -45,6 +45,24 @@ to use, i.e. igb_uio, vfio. The ``dpdk-devbind.py`` script can be used to show devices status and to bind them to a suitable kernel driver. They will appear under the category of "Misc (rawdev) devices". +Prerequisites +------------- +NTB PMD needs kernel PCI driver to support write combining (WC) to get +better performance. The difference will be more than 10 times. +To enable WC, there are 2 ways. +- Insert igb_uio with ``wc_active=1`` flag if use igb_uio driver. + insmod igb_uio.ko wc_active=1 +- Enable WC for NTB device's Bar 2 and Bar 4 (Mapped memory) manually. + Get bar base address using ``lspci -vvv -s ae:00.0 | grep Region``. + Region 0: Memory at 39bfe0000000 (64-bit, prefetchable) [size=64K] + Region 2: Memory at 39bfa0000000 (64-bit, prefetchable) [size=512M] + Region 4: Memory at 39bfc0000000 (64-bit, prefetchable) [size=512M] + Using the following command to enable WC. + echo "base=0x39bfa0000000 size=0x400000 type=write-combining" >> /proc/mtrr + echo "base=0x39bfc0000000 size=0x400000 type=write-combining" >> /proc/mtrr + To disable WC for these regions, using the following. + echo "disable=1" >> /proc/mtrr + Ring Layout -----------
@@ -83,6 +101,16 @@ like the following: +------------------------+ +------------------------+ <---------traffic--------- +- Enqueue and Dequeue + Based on this ring layout, enqueue reads rx_tail to get how many free + buffers and writes used_ring and tx_tail to tell the peer which buffers + are filled with data. + And dequeue reads tx_tail to get how many packets are arrived, and + writes desc_ring and rx_tail to tell the peer about the new allocated + buffers. + So in this way, only remote write happens and remote read can be avoid + to get better performance. + Limitation ----------
drivers/raw/ntb/ntb.c +223 -19
272 lines not on this page · raw message
drivers/raw/ntb/ntb.h +2 -0
@@ -87,6 +87,7 @@ enum ntb_spad_idx {* @ntb_dev_init: Init ntb dev. * @get_peer_mw_addr: To get the addr of peer mw[mw_idx]. * @mw_set_trans: Set translation of internal memory that remote can access. +* @ioremap: Translate the remote host address to bar address. * @get_link_status: get link status, link speed and link width. * @set_link: Set local side up/down. * @spad_read: Read local/peer spad register val.
@@ -103,6 +104,7 @@ struct ntb_dev_ops {void *(*get_peer_mw_addr)(const struct rte_rawdev *dev,int mw_idx);int (*mw_set_trans)(const struct rte_rawdev *dev,int mw_idx,uint64_t addr,uint64_t size); +void *(*ioremap)(const struct rte_rawdev *dev,uint64_t addr);int (*get_link_status)(const struct rte_rawdev *dev);int (*set_link)(const struct rte_rawdev *dev,bool up); uint32_t (*spad_read)(const struct rte_rawdev *dev, int spad,
drivers/raw/ntb/ntb_hw_intel.c +22 -0
@@ -162,6 +162,27 @@ intel_ntb_mw_set_trans(const struct rte_rawdev *dev, int mw_idx,return 0 ; } +static void * +intel_ntb_ioremap (const struct rte_rawdev *dev,uint64_t addr) +{ +struct ntb_hw *hw = dev->dev_private; +void *mapped =NULL ; +void *base; +int i; + +for (i =0 ; i < hw->peer_used_mws; i++) { +if (addr >= hw->peer_mw_base[i] && + addr <= hw->peer_mw_base[i] + hw->mw_size[i]) { + base =intel_ntb_get_peer_mw_addr (dev, i); + mapped = (void *)(size_t )(addr - hw->peer_mw_base[i] + + (size_t )base); +break ; + } + } + +return mapped; +} +static int intel_ntb_get_link_status (const struct rte_rawdev *dev) {
@@ -357,6 +378,7 @@ const struct ntb_dev_ops intel_ntb_ops = { .ntb_dev_init = intel_ntb_dev_init, .get_peer_mw_addr = intel_ntb_get_peer_mw_addr, .mw_set_trans = intel_ntb_mw_set_trans, + .ioremap = intel_ntb_ioremap, .get_link_status = intel_ntb_get_link_status, .set_link = intel_ntb_set_link, .spad_read = intel_ntb_spad_read,
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v4 3/4] raw/ntb: add enqueue and dequeue functions
Quoted 10 lines
-----Original Message-----
From: Li, Xiaoyun
Sent: Monday, September 9, 2019 11:27 AM
To: Wu, Jingjing <[email protected]>; Wiles, Keith <[email protected]>; Maslekar, Omkar <[email protected]>; Liang, Cunming <[email protected]>
Cc: [email protected]; Li, Xiaoyun <[email protected]>
Subject: [PATCH v4 3/4] raw/ntb: add enqueue and dequeue functions
Quoted 5 lines
Introduce enqueue and dequeue functions to support packet based processing. And enable write-combining for ntb driver since it can improve the performance a lot.
Signed-off-by Xiaoyun Li <[email protected]>
PATCH v4 4/4 examples/ntb: support more functions for NTB
In reply to Xiaoyun LiSupport to transmit files between two systems. Support iofwd between one ethdev and NTB device. Support rxonly and txonly for NTB device. Support to set forwarding mode as file-trans, txonly, rxonly or iofwd.
Support to show/clear port stats and throughput.
--- doc/guides/sample_app_ug/ntb.rst | 59 +- examples/ntb/meson.build | 3 + examples/ntb/ntb_fwd.c | 1298 +++++++++++++++++++++++++++--- 3 files changed, 1232 insertions(+), 128 deletions(-)
doc/guides/sample_app_ug/ntb.rst +53 -6
@@ -5,8 +5,17 @@ NTB Sample Application ====================== The ntb sample application shows how to use ntb rawdev driver. -This sample provides interactive mode to transmit file between -two hosts. +This sample provides interactive mode to do packet based processing +between two systems. + +This sample supports 4 types of packet forwarding mode. + +* ``file-trans``: transmit files between two systems. The sample will + be polling to receive files from the peer and save the file as + ``ntb_recv_file[N]``, [N] represents the number of received file. +* ``rxonly``: NTB receives packets but doesn't transmit them. +* ``txonly``: NTB generates and transmits packets without receiving any. +* ``iofwd``: iofwd between NTB device and ethdev. Compiling the Application -------------------------
@@ -29,6 +38,40 @@ Refer to the *DPDK Getting Started Guide* for general information on running applications and the Environment Abstraction Layer (EAL) options. +Command-line Options +-------------------- + +The application supports the following command-line options. + +* ``--buf-size=N`` + + Set the data size of the mbufs used to N bytes, where N < 65536. + The default value is 2048. + +* ``--fwd-mode=mode`` + + Set the packet forwarding mode as ``file-trans``, ``txonly``, + ``rxonly`` or ``iofwd``. + +* ``--nb-desc=N`` + + Set number of descriptors of queue as N, namely queue size, + where 64 <= N <= 1024. The default value is 1024. + +* ``--txfreet=N`` + + Set the transmit free threshold of TX rings to N, where 0 <= N <= + the value of ``--nb-desc``. The default value is 256. + +* ``--burst=N`` + + Set the number of packets per burst to N, where 1 <= N <= 32. + The default value is 32. + +* ``--qp=N`` + + Set the number of queues as N, where qp > 0. + Using the application ---------------------
@@ -41,7 +84,11 @@ The application is console-driven using the cmdline DPDK interface: From this interface the available commands and descriptions of what they do as as follows: -* ``send [filepath]``: Send file to the peer host. -* ``receive [filepath]``: Receive file to [filepath]. Need the peer - to send file successfully first. -* ``quit``: Exit program +* ``send [filepath]``: Send file to the peer host. Need to be in + file-trans forwarding mode first. +* ``start``: Start transmission. +* ``stop``: Stop transmission. +* ``show/clear port stats``: Show/Clear port stats and throughput. +* ``set fwd file-trans/rxonly/txonly/iofwd``: Set packet forwarding + mode. +* ``quit``: Exit program.
examples/ntb/meson.build +3 -0
@@ -14,3 +14,6 @@ cflags += ['-D_FILE_OFFSET_BITS=64'] sources = files( 'ntb_fwd.c' ) +if dpdk_conf.has('RTE_LIBRTE_PMD_NTB_RAWDEV') + deps += 'rawdev_ntb' +endif
examples/ntb/ntb_fwd.c +1176 -122
1,448 lines not on this page · raw message
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v4 4/4] examples/ntb: support more functions for NTB
<...>
doc/guides/sample_app_ug/ntb.rst Refer to the *DPDK Getting Started Guide* for general information on · line 73
Quoted 3 lines
doc/guides/sample_app_ug/ntb.rst +3
-0+* ``--qp=N`` + + Set the number of queues as N, where qp > 0.
The default value is 1?
<...>
examples/ntb/ntb_fwd.c main(int argc, char **argv) · line 1424
Quoted 7 lines
examples/ntb/ntb_fwd.c +7
-0+ +/* Set default fwd mode if user doesn't set it. */ +if (fwd_mode ==MAX_FWD_MODE && eth_port_id <RTE_MAX_ETHPORTS ) { +printf ("Set default fwd mode as iofwd.\n" ); + fwd_mode =IOFWD ; + } +if (fwd_mode ==MAX_FWD_MODE ) {
use "else if"? because (fwd_mode == MAX_FWD_MODE) including (fwd_mode == MAX_FWD_MODE && eth_port_id < RTE_MAX_ETHPORTS)
Thanks
Jingjing
Re: [dpdk-dev] [PATCH v4 4/4] examples/ntb: support more functions for NTB
Hi
Quoted 9 lines
-----Original Message-----
From: Wu, Jingjing
Sent: Monday, September 23, 2019 15:19
To: Li, Xiaoyun <[email protected]>; Wiles, Keith <[email protected]>; Maslekar, Omkar <[email protected]>; Liang, Cunming <[email protected]>
Cc: [email protected]
Subject: RE: [PATCH v4 4/4] examples/ntb: support more functions for NTB
Quoted 6 lines
<...>
doc/guides/sample_app_ug/ntb.rst Refer to the *DPDK Getting Started Guide* for general information on · line 73
Quoted 3 lines
doc/guides/sample_app_ug/ntb.rst +3
-0+* ``--qp=N`` + + Set the number of queues as N, where qp > 0.The default value is 1?
Yes. Will add respective comment and guide.
Quoted 7 lines
<...>
examples/ntb/ntb_fwd.c main(int argc, char **argv)
Quoted 3 lines
examples/ntb/ntb_fwd.c +3
-0+ +/* Set default fwd mode if user doesn't set it. */ + if (fwd_mode == MAX_FWD_MODE && eth_port_id <
RTE_MAX_ETHPORTS) {
Quoted 7 lines
examples/ntb/ntb_fwd.c main(int argc, char **argv) · line 1424
Quoted 4 lines
examples/ntb/ntb_fwd.c +4
-0+printf ("Set default fwd mode as iofwd.\n" ); + fwd_mode =IOFWD ; + } +if (fwd_mode ==MAX_FWD_MODE ) {use "else if"? because (
fwd_mode==MAX_FWD_MODE) including (fwd_mode==MAX_FWD_MODE&ð_port_id<RTE_MAX_ETHPORTS)
Yes. Thanks.
Quoted 3 lines
Thanks
Jingjing
Re: [dpdk-dev] [PATCH v4 4/4] examples/ntb: support more functions for NTB
In reply to Wu, JingjingHi
Quoted 9 lines
-----Original Message-----
From: Wu, Jingjing
Sent: Monday, September 23, 2019 15:19
To: Li, Xiaoyun <[email protected]>; Wiles, Keith <[email protected]>; Maslekar, Omkar <[email protected]>; Liang, Cunming <[email protected]>
Cc: [email protected]
Subject: RE: [PATCH v4 4/4] examples/ntb: support more functions for NTB
Quoted 6 lines
<...>
doc/guides/sample_app_ug/ntb.rst Refer to the *DPDK Getting Started Guide* for general information on · line 73
Quoted 3 lines
doc/guides/sample_app_ug/ntb.rst +3
-0+* ``--qp=N`` + + Set the number of queues as N, where qp > 0.The default value is 1?
Yes. Will add missing info in doc.
Quoted 7 lines
<...>
examples/ntb/ntb_fwd.c main(int argc, char **argv)
Quoted 3 lines
examples/ntb/ntb_fwd.c +3
-0+ +/* Set default fwd mode if user doesn't set it. */ + if (fwd_mode == MAX_FWD_MODE && eth_port_id <
RTE_MAX_ETHPORTS) {
Quoted 7 lines
examples/ntb/ntb_fwd.c main(int argc, char **argv) · line 1424
Quoted 4 lines
examples/ntb/ntb_fwd.c +4
-0+printf ("Set default fwd mode as iofwd.\n" ); + fwd_mode =IOFWD ; + } +if (fwd_mode ==MAX_FWD_MODE ) {use "else if"? because (
fwd_mode==MAX_FWD_MODE) including (fwd_mode==MAX_FWD_MODE&ð_port_id<RTE_MAX_ETHPORTS)
No. If it is that case, the fwd_mode will be set as IOFWD in the first if and won't hit this one.
Quoted 3 lines
Thanks
Jingjing
v5 cover enable FIFO for NTB
In reply to Xiaoyun LiEnable FIFO for NTB rawdev driver to support packet based processing. And an example is provided to support txonly, rxonly, iofwd between NTB device and ethdev, and file transmission.
---
v5:
* Added missing free function when error happens.
* Reworked on xstats reset and get to avoid competition of reset and
* en/dequeue.
* Added missing info in doc.
v4:
* Fixed compile issues with 32-bit machine.
* Fixed total xstats issue.
v3:
* Replace strncpy with memcpy to avoid gcc-9 compile issue.
v2:
* Fixed compile issues with 32-bit machine and lack of including file.
* Fixed a typo.
Xiaoyun Li (4):
raw/ntb: setup ntb queue
raw/ntb: add xstats support
raw/ntb: add enqueue and dequeue functions
examples/ntb: support more functions for NTB
doc/guides/rawdevs/ntb.rst | 67 +- doc/guides/rel_notes/release_19_11.rst | 4 + doc/guides/sample_app_ug/ntb.rst | 59 +- drivers/raw/ntb/Makefile | 3 + drivers/raw/ntb/meson.build | 1 + drivers/raw/ntb/ntb.c | 1134 ++++++++++++++++----- drivers/raw/ntb/ntb.h | 163 ++- drivers/raw/ntb/ntb_hw_intel.c | 48 +- drivers/raw/ntb/rte_pmd_ntb.h | 43 + examples/ntb/meson.build | 3 + examples/ntb/ntb_fwd.c | 1298 +++++++++++++++++++++--- 11 files changed, 2406 insertions(+), 417 deletions(-) create mode 100644 drivers/raw/ntb/rte_pmd_ntb.h
Signature · 1 line
2.17.1
PATCH v5 1/4 raw/ntb: setup ntb queue
Setup and init ntb txq and rxq. And negotiate queue information with the peer. If queue size and number of queues are not consistent on both sides, return error.
--- doc/guides/rawdevs/ntb.rst | 39 +- doc/guides/rel_notes/release_19_11.rst | 4 + drivers/raw/ntb/Makefile | 3 + drivers/raw/ntb/meson.build | 1 + drivers/raw/ntb/ntb.c | 726 ++++++++++++++++++------- drivers/raw/ntb/ntb.h | 151 +++-- drivers/raw/ntb/ntb_hw_intel.c | 26 +- drivers/raw/ntb/rte_pmd_ntb.h | 43 ++ 8 files changed, 738 insertions(+), 255 deletions(-) create mode 100644 drivers/raw/ntb/rte_pmd_ntb.h
doc/guides/rawdevs/ntb.rst +38 -1
@@ -45,8 +45,45 @@ to use, i.e. igb_uio, vfio. The ``dpdk-devbind.py`` script can be used to show devices status and to bind them to a suitable kernel driver. They will appear under the category of "Misc (rawdev) devices". +Ring Layout +----------- + +Since read/write remote system's memory are through PCI bus, remote read +is much more expensive than remote write. Thus, the enqueue and dequeue +based on ntb ring should avoid remote read. The ring layout for ntb is +like the following: +- Ring Format: + desc_ring: + 0 16 64 + +---------------------------------------------------------------+ + | buffer address | + +---------------+-----------------------------------------------+ + | buffer length | resv | + +---------------+-----------------------------------------------+ + used_ring: + 0 16 32 + +---------------+---------------+ + | packet length | flags | + +---------------+---------------+ +- Ring Layout + +------------------------+ +------------------------+ + | used_ring | | desc_ring | + | +---+ | | +---+ | + | | | | | | | | + | +---+ +--------+ | | +---+ | + | | | ---> | buffer | <+---+-| | | + | +---+ +--------+ | | +---+ | + | | | | | | | | + | +---+ | | +---+ | + | ... | | ... | + | | | | + | +---------+ | | +---------+ | + | | tx_tail | | | | rx_tail | | + | System A +---------+ | | System B +---------+ | + +------------------------+ +------------------------+ + <---------traffic--------- + Limitation ---------- -- The FIFO hasn't been introduced and will come in 19.11 release. - This PMD only supports Intel Skylake platform.
doc/guides/rel_notes/release_19_11.rst +4 -0
@@ -56,6 +56,10 @@ New Features Also, make sure to start the actual text at the margin. ========================================================= + * **Introduced FIFO for NTB PMD.** + + Introduced FIFO for NTB (Non-transparent Bridge) PMD to support + packet based processing. Removed Items -------------
drivers/raw/ntb/Makefile +3 -0
@@ -25,4 +25,7 @@ LIBABIVER := 1 SRCS-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV) += ntb.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV) += ntb_hw_intel.c +# install this header file +SYMLINK-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV)-include := rte_pmd_ntb.h + include $(RTE_SDK)/mk/rte.lib.mk
drivers/raw/ntb/meson.build +1 -0
@@ -5,4 +5,5 @@ deps += ['rawdev', 'mbuf', 'mempool', 'pci', 'bus_pci'] sources = files('ntb.c', 'ntb_hw_intel.c') +install_headers('rte_pmd_ntb.h') allow_experimental_apis = true
drivers/raw/ntb/ntb.c +531 -195
966 lines not on this page · raw message
drivers/raw/ntb/ntb.h +105 -46
205 lines not on this page · raw message
drivers/raw/ntb/ntb_hw_intel.c +13 -13
@@ -26,7 +26,7 @@ static enum xeon_ntb_bar intel_ntb_bar[] = { };static int -intel_ntb_dev_init (struct rte_rawdev *dev) +intel_ntb_dev_init (const struct rte_rawdev *dev) {struct ntb_hw *hw = dev->dev_private;uint8_t reg_val, bar;
@@ -77,7 +77,7 @@ intel_ntb_dev_init(struct rte_rawdev *dev) hw->db_cnt =XEON_DB_COUNT ; hw->spad_cnt =XEON_SPAD_COUNT ; - hw->mw_size =rte_zmalloc ("uint64_t" , + hw->mw_size =rte_zmalloc ("ntb_mw_size" , hw->mw_cnt *sizeof (uint64_t ),0 );for (i =0 ; i < hw->mw_cnt; i++) { bar = intel_ntb_bar[i];
@@ -94,7 +94,7 @@ intel_ntb_dev_init(struct rte_rawdev *dev) }static void * -intel_ntb_get_peer_mw_addr (struct rte_rawdev *dev,int mw_idx) +intel_ntb_get_peer_mw_addr (const struct rte_rawdev *dev,int mw_idx) {struct ntb_hw *hw = dev->dev_private;uint8_t bar;
@@ -116,7 +116,7 @@ intel_ntb_get_peer_mw_addr(struct rte_rawdev *dev, int mw_idx) }static int -intel_ntb_mw_set_trans (struct rte_rawdev *dev,int mw_idx, +intel_ntb_mw_set_trans (const struct rte_rawdev *dev,int mw_idx,uint64_t addr,uint64_t size) {struct ntb_hw *hw = dev->dev_private;
@@ -163,7 +163,7 @@ intel_ntb_mw_set_trans(struct rte_rawdev *dev, int mw_idx, }static int -intel_ntb_get_link_status (struct rte_rawdev *dev) +intel_ntb_get_link_status (const struct rte_rawdev *dev) {struct ntb_hw *hw = dev->dev_private;uint16_t reg_val;
@@ -195,7 +195,7 @@ intel_ntb_get_link_status(struct rte_rawdev *dev) }static int -intel_ntb_set_link (struct rte_rawdev *dev,bool up) +intel_ntb_set_link (const struct rte_rawdev *dev,bool up) {struct ntb_hw *hw = dev->dev_private;uint32_t ntb_ctrl, reg_off;
@@ -221,7 +221,7 @@ intel_ntb_set_link(struct rte_rawdev *dev, bool up) }static uint32_t -intel_ntb_spad_read (struct rte_rawdev *dev,int spad,bool peer) +intel_ntb_spad_read (const struct rte_rawdev *dev,int spad,bool peer) {struct ntb_hw *hw = dev->dev_private;uint32_t spad_v, reg_off;
@@ -241,7 +241,7 @@ intel_ntb_spad_read(struct rte_rawdev *dev, int spad, bool peer) }static int -intel_ntb_spad_write (struct rte_rawdev *dev,int spad, +intel_ntb_spad_write (const struct rte_rawdev *dev,int spad,bool peer,uint32_t spad_v) {struct ntb_hw *hw = dev->dev_private;
@@ -263,7 +263,7 @@ intel_ntb_spad_write(struct rte_rawdev *dev, int spad, }static uint64_t -intel_ntb_db_read (struct rte_rawdev *dev) +intel_ntb_db_read (const struct rte_rawdev *dev) {struct ntb_hw *hw = dev->dev_private;uint64_t db_off, db_bits;
@@ -278,7 +278,7 @@ intel_ntb_db_read(struct rte_rawdev *dev) }static int -intel_ntb_db_clear (struct rte_rawdev *dev,uint64_t db_bits) +intel_ntb_db_clear (const struct rte_rawdev *dev,uint64_t db_bits) {struct ntb_hw *hw = dev->dev_private;uint64_t db_off;
@@ -293,7 +293,7 @@ intel_ntb_db_clear(struct rte_rawdev *dev, uint64_t db_bits) }static int -intel_ntb_db_set_mask (struct rte_rawdev *dev,uint64_t db_mask) +intel_ntb_db_set_mask (const struct rte_rawdev *dev,uint64_t db_mask) {struct ntb_hw *hw = dev->dev_private;uint64_t db_m_off;
@@ -312,7 +312,7 @@ intel_ntb_db_set_mask(struct rte_rawdev *dev, uint64_t db_mask) }static int -intel_ntb_peer_db_set (struct rte_rawdev *dev,uint8_t db_idx) +intel_ntb_peer_db_set (const struct rte_rawdev *dev,uint8_t db_idx) {struct ntb_hw *hw = dev->dev_private;uint32_t db_off;
@@ -332,7 +332,7 @@ intel_ntb_peer_db_set(struct rte_rawdev *dev, uint8_t db_idx) }static int -intel_ntb_vector_bind (struct rte_rawdev *dev,uint8_t intr,uint8_t msix) +intel_ntb_vector_bind (const struct rte_rawdev *dev,uint8_t intr,uint8_t msix) {struct ntb_hw *hw = dev->dev_private;uint8_t reg_off;
drivers/raw/ntb/rte_pmd_ntb.h new file +43 -0
43 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v5 2/4 raw/ntb: add xstats support
In reply to Xiaoyun LiAdd xstats support for ntb rawdev.
Support tx-packets, tx-bytes, tx-errors and
rx-packets, rx-bytes, rx-missed.
--- drivers/raw/ntb/ntb.c | 170 +++++++++++++++++++++++++++++++++++++----- drivers/raw/ntb/ntb.h | 12 +++ 2 files changed, 164 insertions(+), 18 deletions(-)
drivers/raw/ntb/ntb.c +152 -18
@@ -30,6 +30,17 @@ static const struct rte_pci_id pci_id_ntb_map[] = { { .vendor_id = 0, /* sentinel */ }, }; +/* Align with enum ntb_xstats_idx */ +static struct rte_rawdev_xstats_name ntb_xstats_names[] = { + {"Tx-packets" }, + {"Tx-bytes" }, + {"Tx-errors" }, + {"Rx-packets" }, + {"Rx-bytes" }, + {"Rx-missed" }, +}; +#define NTB_XSTATS_NUM RTE_DIM(ntb_xstats_names) +static inline void ntb_link_cleanup (struct rte_rawdev *dev) {
@@ -542,6 +553,12 @@ ntb_queue_init(struct rte_rawdev *dev, uint16_t qp_id) txq->last_avail =0 ; txq->nb_tx_free = txq->nb_tx_desc -1 ; +/* Set per queue stats. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) { + hw->ntb_xstats[i +NTB_XSTATS_NUM * (qp_id +1 )] =0 ; + hw->ntb_xstats_off[i +NTB_XSTATS_NUM * (qp_id +1 )] =0 ; + } +return 0 ; }
@@ -618,6 +635,7 @@ ntb_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config) {struct ntb_dev_config *conf = config;struct ntb_hw *hw = dev->dev_private; +uint32_t xstats_num;int ret; hw->queue_pairs = conf->num_queues;
@@ -628,6 +646,12 @@ ntb_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config) sizeof(struct ntb_rx_queue *) * hw->queue_pairs, 0); hw->tx_queues =rte_zmalloc ("ntb_tx_queues" ,sizeof (struct ntb_tx_queue *) * hw->queue_pairs,0 ); +/* First total stats, then per queue stats. */ + xstats_num = (hw->queue_pairs +1 ) *NTB_XSTATS_NUM ; + hw->ntb_xstats =rte_zmalloc ("ntb_xstats" , xstats_num * +sizeof (uint64_t ),0 ); + hw->ntb_xstats_off =rte_zmalloc ("ntb_xstats_off" , xstats_num * +sizeof (uint64_t ),0 );/* Start handshake with the peer. */ ret =ntb_handshake_work (dev);
@@ -654,6 +678,12 @@ ntb_dev_start(struct rte_rawdev *dev)if (!hw->link_status || !hw->peer_dev_up)return -EINVAL ; +/* Set total stats. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) { + hw->ntb_xstats[i] =0 ; + hw->ntb_xstats_off[i] =0 ; + } +for (i =0 ; i < hw->queue_pairs; i++) { ret =ntb_queue_init (dev, i);if (ret) {
@@ -927,39 +957,143 @@ ntb_attr_get(struct rte_rawdev *dev, const char *attr_name,return -EINVAL ; } +static inline uint64_t +ntb_stats_update (uint64_t offset,uint64_t stat) +{ +if (stat >= offset) +return (stat - offset); +else +return (uint64_t )(((uint64_t )-1 ) - offset + stat +1 ); +} +static int -ntb_xstats_get(const struct rte_rawdev *dev __rte_unused, - const unsigned int ids[] __rte_unused, - uint64_t values[] __rte_unused, - unsigned int n __rte_unused) +ntb_xstats_get (const struct rte_rawdev *dev, +const unsigned int ids[], +uint64_t values[], +unsigned int n) { -return 0 ; +struct ntb_hw *hw = dev->dev_private; +uint32_t i, j, off, xstats_num; + +/* Calculate total stats of all queues. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) { + hw->ntb_xstats[i] =0 ; +for (j =0 ; j < hw->queue_pairs; j++) { + off =NTB_XSTATS_NUM * (j +1 ) + i; + hw->ntb_xstats[i] += +ntb_stats_update (hw->ntb_xstats_off[off], + hw->ntb_xstats[off]); + } + } + + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); +for (i =0 ; i < n && ids[i] < xstats_num; i++) { +if (ids[i] <NTB_XSTATS_NUM ) + values[i] = hw->ntb_xstats[ids[i]]; +else + values[i] = +ntb_stats_update (hw->ntb_xstats_off[ids[i]], + hw->ntb_xstats[ids[i]]); + } + +return i; }static int -ntb_xstats_get_names(const struct rte_rawdev *dev __rte_unused, - struct rte_rawdev_xstats_name *xstats_names __rte_unused, - unsigned int size __rte_unused) +ntb_xstats_get_names (const struct rte_rawdev *dev, +struct rte_rawdev_xstats_name *xstats_names, +unsigned int size) { -return 0 ; +struct ntb_hw *hw = dev->dev_private; +uint32_t xstats_num, i, j, off; + + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); +if (xstats_names ==NULL || size < xstats_num) +return xstats_num; + +/* Total stats names */ +memcpy (xstats_names, ntb_xstats_names,sizeof (ntb_xstats_names)); + +/* Queue stats names */ +for (i =0 ; i < hw->queue_pairs; i++) { +for (j =0 ; j <NTB_XSTATS_NUM ; j++) { + off = j + (i +1 ) *NTB_XSTATS_NUM ; +snprintf (xstats_names[off].name, +sizeof (xstats_names[0 ].name), +"%s_q%u" , ntb_xstats_names[j].name, i); + } + } + +return xstats_num; }static uint64_t -ntb_xstats_get_by_name(const struct rte_rawdev *dev __rte_unused, - const char *name __rte_unused, - unsigned int *id __rte_unused) +ntb_xstats_get_by_name (const struct rte_rawdev *dev, +const char *name,unsigned int *id) { -return 0 ; +struct rte_rawdev_xstats_name *xstats_names; +struct ntb_hw *hw = dev->dev_private; +uint32_t xstats_num, i, j, off; + +if (name ==NULL ) +return -EINVAL ; + + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); + xstats_names =rte_zmalloc ("ntb_stats_name" , +sizeof (struct rte_rawdev_xstats_name ) * + xstats_num,0 ); +ntb_xstats_get_names (dev, xstats_names, xstats_num); + +/* Calculate total stats of all queues. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) { +for (j =0 ; j < hw->queue_pairs; j++) { + off =NTB_XSTATS_NUM * (j +1 ) + i; + hw->ntb_xstats[i] += +ntb_stats_update (hw->ntb_xstats_off[off], + hw->ntb_xstats[off]); + } + } + +for (i =0 ; i < xstats_num; i++) { +if (!strncmp (name, xstats_names[i].name, +RTE_RAW_DEV_XSTATS_NAME_SIZE )) { + *id = i; +rte_free (xstats_names); +if (i <NTB_XSTATS_NUM ) +return hw->ntb_xstats[i]; +else +return ntb_stats_update (hw->ntb_xstats_off[i], + hw->ntb_xstats[i]); + } + } + +NTB_LOG (ERR ,"Cannot find the xstats name." ); + +return -EINVAL ; }static int -ntb_xstats_reset(struct rte_rawdev *dev __rte_unused, - const uint32_t ids[] __rte_unused, - uint32_t nb_ids __rte_unused) +ntb_xstats_reset (struct rte_rawdev *dev, +const uint32_t ids[], +uint32_t nb_ids) { -return 0 ; -} +struct ntb_hw *hw = dev->dev_private; +uint32_t i, j, off, xstats_num; + + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); +for (i =0 ; i < nb_ids && ids[i] < xstats_num; i++) { +if (ids[i] <NTB_XSTATS_NUM ) { +for (j =0 ; j < hw->queue_pairs; j++) { + off =NTB_XSTATS_NUM * (j +1 ) + i; + hw->ntb_xstats_off[off] = hw->ntb_xstats[off]; + } + }else { + hw->ntb_xstats_off[ids[i]] = hw->ntb_xstats[ids[i]]; + } + } +return i; +}static const struct rte_rawdev_ops ntb_ops = { .dev_info_get = ntb_dev_info_get,
drivers/raw/ntb/ntb.h +12 -0
@@ -27,6 +27,15 @@ extern int ntb_logtype;#define NTB_DFLT_TX_FREE_THRESH 256 +enum ntb_xstats_idx { +NTB_TX_PKTS_ID =0 , +NTB_TX_BYTES_ID , +NTB_TX_ERRS_ID , +NTB_RX_PKTS_ID , +NTB_RX_BYTES_ID , +NTB_RX_MISS_ID , +}; +enum ntb_topo {NTB_TOPO_NONE =0 ,NTB_TOPO_B2B_USD ,
@@ -216,6 +225,9 @@ struct ntb_hw {uint8_t peer_used_mws; +uint64_t *ntb_xstats; +uint64_t *ntb_xstats_off; +/* Reserve several spad for app to use. */ int spad_user_list[NTB_SPAD_USER_MAX_NUM ]; };
Signature · 1 line
2.17.1
PATCH v5 3/4 raw/ntb: add enqueue and dequeue functions
In reply to Xiaoyun LiIntroduce enqueue and dequeue functions to support packet based processing. And enable write-combining for ntb driver since it can improve the performance a lot.
--- doc/guides/rawdevs/ntb.rst | 28 ++++ drivers/raw/ntb/ntb.c | 242 ++++++++++++++++++++++++++++++--- drivers/raw/ntb/ntb.h | 2 + drivers/raw/ntb/ntb_hw_intel.c | 22 +++ 4 files changed, 275 insertions(+), 19 deletions(-)
doc/guides/rawdevs/ntb.rst +28 -0
@@ -45,6 +45,24 @@ to use, i.e. igb_uio, vfio. The ``dpdk-devbind.py`` script can be used to show devices status and to bind them to a suitable kernel driver. They will appear under the category of "Misc (rawdev) devices". +Prerequisites +------------- +NTB PMD needs kernel PCI driver to support write combining (WC) to get +better performance. The difference will be more than 10 times. +To enable WC, there are 2 ways. +- Insert igb_uio with ``wc_active=1`` flag if use igb_uio driver. + insmod igb_uio.ko wc_active=1 +- Enable WC for NTB device's Bar 2 and Bar 4 (Mapped memory) manually. + Get bar base address using ``lspci -vvv -s ae:00.0 | grep Region``. + Region 0: Memory at 39bfe0000000 (64-bit, prefetchable) [size=64K] + Region 2: Memory at 39bfa0000000 (64-bit, prefetchable) [size=512M] + Region 4: Memory at 39bfc0000000 (64-bit, prefetchable) [size=512M] + Using the following command to enable WC. + echo "base=0x39bfa0000000 size=0x400000 type=write-combining" >> /proc/mtrr + echo "base=0x39bfc0000000 size=0x400000 type=write-combining" >> /proc/mtrr + To disable WC for these regions, using the following. + echo "disable=1" >> /proc/mtrr + Ring Layout -----------
@@ -83,6 +101,16 @@ like the following: +------------------------+ +------------------------+ <---------traffic--------- +- Enqueue and Dequeue + Based on this ring layout, enqueue reads rx_tail to get how many free + buffers and writes used_ring and tx_tail to tell the peer which buffers + are filled with data. + And dequeue reads tx_tail to get how many packets are arrived, and + writes desc_ring and rx_tail to tell the peer about the new allocated + buffers. + So in this way, only remote write happens and remote read can be avoid + to get better performance. + Limitation ----------
drivers/raw/ntb/ntb.c +223 -19
272 lines not on this page · raw message
drivers/raw/ntb/ntb.h +2 -0
@@ -87,6 +87,7 @@ enum ntb_spad_idx {* @ntb_dev_init: Init ntb dev. * @get_peer_mw_addr: To get the addr of peer mw[mw_idx]. * @mw_set_trans: Set translation of internal memory that remote can access. +* @ioremap: Translate the remote host address to bar address. * @get_link_status: get link status, link speed and link width. * @set_link: Set local side up/down. * @spad_read: Read local/peer spad register val.
@@ -103,6 +104,7 @@ struct ntb_dev_ops {void *(*get_peer_mw_addr)(const struct rte_rawdev *dev,int mw_idx);int (*mw_set_trans)(const struct rte_rawdev *dev,int mw_idx,uint64_t addr,uint64_t size); +void *(*ioremap)(const struct rte_rawdev *dev,uint64_t addr);int (*get_link_status)(const struct rte_rawdev *dev);int (*set_link)(const struct rte_rawdev *dev,bool up); uint32_t (*spad_read)(const struct rte_rawdev *dev, int spad,
drivers/raw/ntb/ntb_hw_intel.c +22 -0
@@ -162,6 +162,27 @@ intel_ntb_mw_set_trans(const struct rte_rawdev *dev, int mw_idx,return 0 ; } +static void * +intel_ntb_ioremap (const struct rte_rawdev *dev,uint64_t addr) +{ +struct ntb_hw *hw = dev->dev_private; +void *mapped =NULL ; +void *base; +int i; + +for (i =0 ; i < hw->peer_used_mws; i++) { +if (addr >= hw->peer_mw_base[i] && + addr <= hw->peer_mw_base[i] + hw->mw_size[i]) { + base =intel_ntb_get_peer_mw_addr (dev, i); + mapped = (void *)(size_t )(addr - hw->peer_mw_base[i] + + (size_t )base); +break ; + } + } + +return mapped; +} +static int intel_ntb_get_link_status (const struct rte_rawdev *dev) {
@@ -357,6 +378,7 @@ const struct ntb_dev_ops intel_ntb_ops = { .ntb_dev_init = intel_ntb_dev_init, .get_peer_mw_addr = intel_ntb_get_peer_mw_addr, .mw_set_trans = intel_ntb_mw_set_trans, + .ioremap = intel_ntb_ioremap, .get_link_status = intel_ntb_get_link_status, .set_link = intel_ntb_set_link, .spad_read = intel_ntb_spad_read,
Signature · 1 line
2.17.1
PATCH v5 4/4 examples/ntb: support more functions for NTB
In reply to Xiaoyun LiSupport to transmit files between two systems. Support iofwd between one ethdev and NTB device. Support rxonly and txonly for NTB device. Support to set forwarding mode as file-trans, txonly, rxonly or iofwd.
Support to show/clear port stats and throughput.
--- doc/guides/sample_app_ug/ntb.rst | 59 +- examples/ntb/meson.build | 3 + examples/ntb/ntb_fwd.c | 1298 +++++++++++++++++++++++++++--- 3 files changed, 1232 insertions(+), 128 deletions(-)
doc/guides/sample_app_ug/ntb.rst +53 -6
@@ -5,8 +5,17 @@ NTB Sample Application ====================== The ntb sample application shows how to use ntb rawdev driver. -This sample provides interactive mode to transmit file between -two hosts. +This sample provides interactive mode to do packet based processing +between two systems. + +This sample supports 4 types of packet forwarding mode. + +* ``file-trans``: transmit files between two systems. The sample will + be polling to receive files from the peer and save the file as + ``ntb_recv_file[N]``, [N] represents the number of received file. +* ``rxonly``: NTB receives packets but doesn't transmit them. +* ``txonly``: NTB generates and transmits packets without receiving any. +* ``iofwd``: iofwd between NTB device and ethdev. Compiling the Application -------------------------
@@ -29,6 +38,40 @@ Refer to the *DPDK Getting Started Guide* for general information on running applications and the Environment Abstraction Layer (EAL) options. +Command-line Options +-------------------- + +The application supports the following command-line options. + +* ``--buf-size=N`` + + Set the data size of the mbufs used to N bytes, where N < 65536. + The default value is 2048. + +* ``--fwd-mode=mode`` + + Set the packet forwarding mode as ``file-trans``, ``txonly``, + ``rxonly`` or ``iofwd``. + +* ``--nb-desc=N`` + + Set number of descriptors of queue as N, namely queue size, + where 64 <= N <= 1024. The default value is 1024. + +* ``--txfreet=N`` + + Set the transmit free threshold of TX rings to N, where 0 <= N <= + the value of ``--nb-desc``. The default value is 256. + +* ``--burst=N`` + + Set the number of packets per burst to N, where 1 <= N <= 32. + The default value is 32. + +* ``--qp=N`` + + Set the number of queues as N, where qp > 0. The default value is 1. + Using the application ---------------------
@@ -41,7 +84,11 @@ The application is console-driven using the cmdline DPDK interface: From this interface the available commands and descriptions of what they do as as follows: -* ``send [filepath]``: Send file to the peer host. -* ``receive [filepath]``: Receive file to [filepath]. Need the peer - to send file successfully first. -* ``quit``: Exit program +* ``send [filepath]``: Send file to the peer host. Need to be in + file-trans forwarding mode first. +* ``start``: Start transmission. +* ``stop``: Stop transmission. +* ``show/clear port stats``: Show/Clear port stats and throughput. +* ``set fwd file-trans/rxonly/txonly/iofwd``: Set packet forwarding + mode. +* ``quit``: Exit program.
examples/ntb/meson.build +3 -0
@@ -14,3 +14,6 @@ cflags += ['-D_FILE_OFFSET_BITS=64'] sources = files( 'ntb_fwd.c' ) +if dpdk_conf.has('RTE_LIBRTE_PMD_NTB_RAWDEV') + deps += 'rawdev_ntb' +endif
examples/ntb/ntb_fwd.c +1176 -122
1,448 lines not on this page · raw message
Signature · 1 line
2.17.1
v6 cover enable FIFO for NTB
In reply to Xiaoyun LiEnable FIFO for NTB rawdev driver to support packet based processing. And an example is provided to support txonly, rxonly, iofwd between NTB device and ethdev, and file transmission.
---
v6:
* Fixed xstats reset typo.
* Fixed free functions when error happens.
* Added missing info in doc.
v5:
* Added missing free function when error happens.
* Reworked on xstats reset and get to avoid competition of reset and
* en/dequeue.
* Added missing info in doc.
v4:
* Fixed compile issues with 32-bit machine.
* Fixed total xstats issue.
v3:
* Replace strncpy with memcpy to avoid gcc-9 compile issue.
v2:
* Fixed compile issues with 32-bit machine and lack of including file.
* Fixed a typo.
Xiaoyun Li (4):
raw/ntb: setup ntb queue
raw/ntb: add xstats support
raw/ntb: add enqueue and dequeue functions
examples/ntb: support more functions for NTB
doc/guides/rawdevs/ntb.rst | 93 +- doc/guides/rel_notes/release_19_11.rst | 4 + doc/guides/sample_app_ug/ntb.rst | 59 +- drivers/raw/ntb/Makefile | 3 + drivers/raw/ntb/meson.build | 1 + drivers/raw/ntb/ntb.c | 1130 ++++++++++++++++----- drivers/raw/ntb/ntb.h | 163 ++- drivers/raw/ntb/ntb_hw_intel.c | 48 +- drivers/raw/ntb/rte_pmd_ntb.h | 43 + examples/ntb/meson.build | 3 + examples/ntb/ntb_fwd.c | 1298 +++++++++++++++++++++--- 11 files changed, 2428 insertions(+), 417 deletions(-) create mode 100644 drivers/raw/ntb/rte_pmd_ntb.h
Signature · 1 line
2.17.1
PATCH v6 1/4 raw/ntb: setup ntb queue
Setup and init ntb txq and rxq. And negotiate queue information with the peer. If queue size and number of queues are not consistent on both sides, return error.
--- doc/guides/rawdevs/ntb.rst | 39 +- doc/guides/rel_notes/release_19_11.rst | 4 + drivers/raw/ntb/Makefile | 3 + drivers/raw/ntb/meson.build | 1 + drivers/raw/ntb/ntb.c | 722 ++++++++++++++++++------- drivers/raw/ntb/ntb.h | 151 ++++-- drivers/raw/ntb/ntb_hw_intel.c | 26 +- drivers/raw/ntb/rte_pmd_ntb.h | 43 ++ 8 files changed, 734 insertions(+), 255 deletions(-) create mode 100644 drivers/raw/ntb/rte_pmd_ntb.h
doc/guides/rawdevs/ntb.rst +38 -1
@@ -45,8 +45,45 @@ to use, i.e. igb_uio, vfio. The ``dpdk-devbind.py`` script can be used to show devices status and to bind them to a suitable kernel driver. They will appear under the category of "Misc (rawdev) devices". +Ring Layout +----------- + +Since read/write remote system's memory are through PCI bus, remote read +is much more expensive than remote write. Thus, the enqueue and dequeue +based on ntb ring should avoid remote read. The ring layout for ntb is +like the following: +- Ring Format: + desc_ring: + 0 16 64 + +---------------------------------------------------------------+ + | buffer address | + +---------------+-----------------------------------------------+ + | buffer length | resv | + +---------------+-----------------------------------------------+ + used_ring: + 0 16 32 + +---------------+---------------+ + | packet length | flags | + +---------------+---------------+ +- Ring Layout + +------------------------+ +------------------------+ + | used_ring | | desc_ring | + | +---+ | | +---+ | + | | | | | | | | + | +---+ +--------+ | | +---+ | + | | | ---> | buffer | <+---+-| | | + | +---+ +--------+ | | +---+ | + | | | | | | | | + | +---+ | | +---+ | + | ... | | ... | + | | | | + | +---------+ | | +---------+ | + | | tx_tail | | | | rx_tail | | + | System A +---------+ | | System B +---------+ | + +------------------------+ +------------------------+ + <---------traffic--------- + Limitation ---------- -- The FIFO hasn't been introduced and will come in 19.11 release. - This PMD only supports Intel Skylake platform.
doc/guides/rel_notes/release_19_11.rst +4 -0
@@ -56,6 +56,10 @@ New Features Also, make sure to start the actual text at the margin. ========================================================= + * **Introduced FIFO for NTB PMD.** + + Introduced FIFO for NTB (Non-transparent Bridge) PMD to support + packet based processing. Removed Items -------------
drivers/raw/ntb/Makefile +3 -0
@@ -25,4 +25,7 @@ LIBABIVER := 1 SRCS-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV) += ntb.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV) += ntb_hw_intel.c +# install this header file +SYMLINK-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV)-include := rte_pmd_ntb.h + include $(RTE_SDK)/mk/rte.lib.mk
drivers/raw/ntb/meson.build +1 -0
@@ -5,4 +5,5 @@ deps += ['rawdev', 'mbuf', 'mempool', 'pci', 'bus_pci'] sources = files('ntb.c', 'ntb_hw_intel.c') +install_headers('rte_pmd_ntb.h') allow_experimental_apis = true
drivers/raw/ntb/ntb.c +527 -195
961 lines not on this page · raw message
drivers/raw/ntb/ntb.h +105 -46
205 lines not on this page · raw message
drivers/raw/ntb/ntb_hw_intel.c +13 -13
@@ -26,7 +26,7 @@ static enum xeon_ntb_bar intel_ntb_bar[] = { };static int -intel_ntb_dev_init (struct rte_rawdev *dev) +intel_ntb_dev_init (const struct rte_rawdev *dev) {struct ntb_hw *hw = dev->dev_private;uint8_t reg_val, bar;
@@ -77,7 +77,7 @@ intel_ntb_dev_init(struct rte_rawdev *dev) hw->db_cnt =XEON_DB_COUNT ; hw->spad_cnt =XEON_SPAD_COUNT ; - hw->mw_size =rte_zmalloc ("uint64_t" , + hw->mw_size =rte_zmalloc ("ntb_mw_size" , hw->mw_cnt *sizeof (uint64_t ),0 );for (i =0 ; i < hw->mw_cnt; i++) { bar = intel_ntb_bar[i];
@@ -94,7 +94,7 @@ intel_ntb_dev_init(struct rte_rawdev *dev) }static void * -intel_ntb_get_peer_mw_addr (struct rte_rawdev *dev,int mw_idx) +intel_ntb_get_peer_mw_addr (const struct rte_rawdev *dev,int mw_idx) {struct ntb_hw *hw = dev->dev_private;uint8_t bar;
@@ -116,7 +116,7 @@ intel_ntb_get_peer_mw_addr(struct rte_rawdev *dev, int mw_idx) }static int -intel_ntb_mw_set_trans (struct rte_rawdev *dev,int mw_idx, +intel_ntb_mw_set_trans (const struct rte_rawdev *dev,int mw_idx,uint64_t addr,uint64_t size) {struct ntb_hw *hw = dev->dev_private;
@@ -163,7 +163,7 @@ intel_ntb_mw_set_trans(struct rte_rawdev *dev, int mw_idx, }static int -intel_ntb_get_link_status (struct rte_rawdev *dev) +intel_ntb_get_link_status (const struct rte_rawdev *dev) {struct ntb_hw *hw = dev->dev_private;uint16_t reg_val;
@@ -195,7 +195,7 @@ intel_ntb_get_link_status(struct rte_rawdev *dev) }static int -intel_ntb_set_link (struct rte_rawdev *dev,bool up) +intel_ntb_set_link (const struct rte_rawdev *dev,bool up) {struct ntb_hw *hw = dev->dev_private;uint32_t ntb_ctrl, reg_off;
@@ -221,7 +221,7 @@ intel_ntb_set_link(struct rte_rawdev *dev, bool up) }static uint32_t -intel_ntb_spad_read (struct rte_rawdev *dev,int spad,bool peer) +intel_ntb_spad_read (const struct rte_rawdev *dev,int spad,bool peer) {struct ntb_hw *hw = dev->dev_private;uint32_t spad_v, reg_off;
@@ -241,7 +241,7 @@ intel_ntb_spad_read(struct rte_rawdev *dev, int spad, bool peer) }static int -intel_ntb_spad_write (struct rte_rawdev *dev,int spad, +intel_ntb_spad_write (const struct rte_rawdev *dev,int spad,bool peer,uint32_t spad_v) {struct ntb_hw *hw = dev->dev_private;
@@ -263,7 +263,7 @@ intel_ntb_spad_write(struct rte_rawdev *dev, int spad, }static uint64_t -intel_ntb_db_read (struct rte_rawdev *dev) +intel_ntb_db_read (const struct rte_rawdev *dev) {struct ntb_hw *hw = dev->dev_private;uint64_t db_off, db_bits;
@@ -278,7 +278,7 @@ intel_ntb_db_read(struct rte_rawdev *dev) }static int -intel_ntb_db_clear (struct rte_rawdev *dev,uint64_t db_bits) +intel_ntb_db_clear (const struct rte_rawdev *dev,uint64_t db_bits) {struct ntb_hw *hw = dev->dev_private;uint64_t db_off;
@@ -293,7 +293,7 @@ intel_ntb_db_clear(struct rte_rawdev *dev, uint64_t db_bits) }static int -intel_ntb_db_set_mask (struct rte_rawdev *dev,uint64_t db_mask) +intel_ntb_db_set_mask (const struct rte_rawdev *dev,uint64_t db_mask) {struct ntb_hw *hw = dev->dev_private;uint64_t db_m_off;
@@ -312,7 +312,7 @@ intel_ntb_db_set_mask(struct rte_rawdev *dev, uint64_t db_mask) }static int -intel_ntb_peer_db_set (struct rte_rawdev *dev,uint8_t db_idx) +intel_ntb_peer_db_set (const struct rte_rawdev *dev,uint8_t db_idx) {struct ntb_hw *hw = dev->dev_private;uint32_t db_off;
@@ -332,7 +332,7 @@ intel_ntb_peer_db_set(struct rte_rawdev *dev, uint8_t db_idx) }static int -intel_ntb_vector_bind (struct rte_rawdev *dev,uint8_t intr,uint8_t msix) +intel_ntb_vector_bind (const struct rte_rawdev *dev,uint8_t intr,uint8_t msix) {struct ntb_hw *hw = dev->dev_private;uint8_t reg_off;
drivers/raw/ntb/rte_pmd_ntb.h new file +43 -0
43 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v6 2/4 raw/ntb: add xstats support
In reply to Xiaoyun LiAdd xstats support for ntb rawdev.
Support tx-packets, tx-bytes, tx-errors and
rx-packets, rx-bytes, rx-missed.
--- drivers/raw/ntb/ntb.c | 170 +++++++++++++++++++++++++++++++++++++----- drivers/raw/ntb/ntb.h | 12 +++ 2 files changed, 164 insertions(+), 18 deletions(-)
drivers/raw/ntb/ntb.c +152 -18
@@ -30,6 +30,17 @@ static const struct rte_pci_id pci_id_ntb_map[] = { { .vendor_id = 0, /* sentinel */ }, }; +/* Align with enum ntb_xstats_idx */ +static struct rte_rawdev_xstats_name ntb_xstats_names[] = { + {"Tx-packets" }, + {"Tx-bytes" }, + {"Tx-errors" }, + {"Rx-packets" }, + {"Rx-bytes" }, + {"Rx-missed" }, +}; +#define NTB_XSTATS_NUM RTE_DIM(ntb_xstats_names) +static inline void ntb_link_cleanup (struct rte_rawdev *dev) {
@@ -538,6 +549,12 @@ ntb_queue_init(struct rte_rawdev *dev, uint16_t qp_id) txq->last_avail =0 ; txq->nb_tx_free = txq->nb_tx_desc -1 ; +/* Set per queue stats. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) { + hw->ntb_xstats[i +NTB_XSTATS_NUM * (qp_id +1 )] =0 ; + hw->ntb_xstats_off[i +NTB_XSTATS_NUM * (qp_id +1 )] =0 ; + } +return 0 ; }
@@ -614,6 +631,7 @@ ntb_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config) {struct ntb_dev_config *conf = config;struct ntb_hw *hw = dev->dev_private; +uint32_t xstats_num;int ret; hw->queue_pairs = conf->num_queues;
@@ -624,6 +642,12 @@ ntb_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config) sizeof(struct ntb_rx_queue *) * hw->queue_pairs, 0); hw->tx_queues =rte_zmalloc ("ntb_tx_queues" ,sizeof (struct ntb_tx_queue *) * hw->queue_pairs,0 ); +/* First total stats, then per queue stats. */ + xstats_num = (hw->queue_pairs +1 ) *NTB_XSTATS_NUM ; + hw->ntb_xstats =rte_zmalloc ("ntb_xstats" , xstats_num * +sizeof (uint64_t ),0 ); + hw->ntb_xstats_off =rte_zmalloc ("ntb_xstats_off" , xstats_num * +sizeof (uint64_t ),0 );/* Start handshake with the peer. */ ret =ntb_handshake_work (dev);
@@ -650,6 +674,12 @@ ntb_dev_start(struct rte_rawdev *dev)if (!hw->link_status || !hw->peer_dev_up)return -EINVAL ; +/* Set total stats. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) { + hw->ntb_xstats[i] =0 ; + hw->ntb_xstats_off[i] =0 ; + } +for (i =0 ; i < hw->queue_pairs; i++) { ret =ntb_queue_init (dev, i);if (ret) {
@@ -923,39 +953,143 @@ ntb_attr_get(struct rte_rawdev *dev, const char *attr_name,return -EINVAL ; } +static inline uint64_t +ntb_stats_update (uint64_t offset,uint64_t stat) +{ +if (stat >= offset) +return (stat - offset); +else +return (uint64_t )(((uint64_t )-1 ) - offset + stat +1 ); +} +static int -ntb_xstats_get(const struct rte_rawdev *dev __rte_unused, - const unsigned int ids[] __rte_unused, - uint64_t values[] __rte_unused, - unsigned int n __rte_unused) +ntb_xstats_get (const struct rte_rawdev *dev, +const unsigned int ids[], +uint64_t values[], +unsigned int n) { -return 0 ; +struct ntb_hw *hw = dev->dev_private; +uint32_t i, j, off, xstats_num; + +/* Calculate total stats of all queues. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) { + hw->ntb_xstats[i] =0 ; +for (j =0 ; j < hw->queue_pairs; j++) { + off =NTB_XSTATS_NUM * (j +1 ) + i; + hw->ntb_xstats[i] += +ntb_stats_update (hw->ntb_xstats_off[off], + hw->ntb_xstats[off]); + } + } + + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); +for (i =0 ; i < n && ids[i] < xstats_num; i++) { +if (ids[i] <NTB_XSTATS_NUM ) + values[i] = hw->ntb_xstats[ids[i]]; +else + values[i] = +ntb_stats_update (hw->ntb_xstats_off[ids[i]], + hw->ntb_xstats[ids[i]]); + } + +return i; }static int -ntb_xstats_get_names(const struct rte_rawdev *dev __rte_unused, - struct rte_rawdev_xstats_name *xstats_names __rte_unused, - unsigned int size __rte_unused) +ntb_xstats_get_names (const struct rte_rawdev *dev, +struct rte_rawdev_xstats_name *xstats_names, +unsigned int size) { -return 0 ; +struct ntb_hw *hw = dev->dev_private; +uint32_t xstats_num, i, j, off; + + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); +if (xstats_names ==NULL || size < xstats_num) +return xstats_num; + +/* Total stats names */ +memcpy (xstats_names, ntb_xstats_names,sizeof (ntb_xstats_names)); + +/* Queue stats names */ +for (i =0 ; i < hw->queue_pairs; i++) { +for (j =0 ; j <NTB_XSTATS_NUM ; j++) { + off = j + (i +1 ) *NTB_XSTATS_NUM ; +snprintf (xstats_names[off].name, +sizeof (xstats_names[0 ].name), +"%s_q%u" , ntb_xstats_names[j].name, i); + } + } + +return xstats_num; }static uint64_t -ntb_xstats_get_by_name(const struct rte_rawdev *dev __rte_unused, - const char *name __rte_unused, - unsigned int *id __rte_unused) +ntb_xstats_get_by_name (const struct rte_rawdev *dev, +const char *name,unsigned int *id) { -return 0 ; +struct rte_rawdev_xstats_name *xstats_names; +struct ntb_hw *hw = dev->dev_private; +uint32_t xstats_num, i, j, off; + +if (name ==NULL ) +return -EINVAL ; + + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); + xstats_names =rte_zmalloc ("ntb_stats_name" , +sizeof (struct rte_rawdev_xstats_name ) * + xstats_num,0 ); +ntb_xstats_get_names (dev, xstats_names, xstats_num); + +/* Calculate total stats of all queues. */ +for (i =0 ; i <NTB_XSTATS_NUM ; i++) { +for (j =0 ; j < hw->queue_pairs; j++) { + off =NTB_XSTATS_NUM * (j +1 ) + i; + hw->ntb_xstats[i] += +ntb_stats_update (hw->ntb_xstats_off[off], + hw->ntb_xstats[off]); + } + } + +for (i =0 ; i < xstats_num; i++) { +if (!strncmp (name, xstats_names[i].name, +RTE_RAW_DEV_XSTATS_NAME_SIZE )) { + *id = i; +rte_free (xstats_names); +if (i <NTB_XSTATS_NUM ) +return hw->ntb_xstats[i]; +else +return ntb_stats_update (hw->ntb_xstats_off[i], + hw->ntb_xstats[i]); + } + } + +NTB_LOG (ERR ,"Cannot find the xstats name." ); + +return -EINVAL ; }static int -ntb_xstats_reset(struct rte_rawdev *dev __rte_unused, - const uint32_t ids[] __rte_unused, - uint32_t nb_ids __rte_unused) +ntb_xstats_reset (struct rte_rawdev *dev, +const uint32_t ids[], +uint32_t nb_ids) { -return 0 ; -} +struct ntb_hw *hw = dev->dev_private; +uint32_t i, j, off, xstats_num; + + xstats_num =NTB_XSTATS_NUM * (hw->queue_pairs +1 ); +for (i =0 ; i < nb_ids && ids[i] < xstats_num; i++) { +if (ids[i] <NTB_XSTATS_NUM ) { +for (j =0 ; j < hw->queue_pairs; j++) { + off =NTB_XSTATS_NUM * (j +1 ) + ids[i]; + hw->ntb_xstats_off[off] = hw->ntb_xstats[off]; + } + }else { + hw->ntb_xstats_off[ids[i]] = hw->ntb_xstats[ids[i]]; + } + } +return i; +}static const struct rte_rawdev_ops ntb_ops = { .dev_info_get = ntb_dev_info_get,
drivers/raw/ntb/ntb.h +12 -0
@@ -27,6 +27,15 @@ extern int ntb_logtype;#define NTB_DFLT_TX_FREE_THRESH 256 +enum ntb_xstats_idx { +NTB_TX_PKTS_ID =0 , +NTB_TX_BYTES_ID , +NTB_TX_ERRS_ID , +NTB_RX_PKTS_ID , +NTB_RX_BYTES_ID , +NTB_RX_MISS_ID , +}; +enum ntb_topo {NTB_TOPO_NONE =0 ,NTB_TOPO_B2B_USD ,
@@ -216,6 +225,9 @@ struct ntb_hw {uint8_t peer_used_mws; +uint64_t *ntb_xstats; +uint64_t *ntb_xstats_off; +/* Reserve several spad for app to use. */ int spad_user_list[NTB_SPAD_USER_MAX_NUM ]; };
Signature · 1 line
2.17.1
PATCH v6 3/4 raw/ntb: add enqueue and dequeue functions
In reply to Xiaoyun LiIntroduce enqueue and dequeue functions to support packet based processing. And enable write-combining for ntb driver since it can improve the performance a lot.
--- doc/guides/rawdevs/ntb.rst | 54 ++++++++ drivers/raw/ntb/ntb.c | 242 ++++++++++++++++++++++++++++++--- drivers/raw/ntb/ntb.h | 2 + drivers/raw/ntb/ntb_hw_intel.c | 22 +++ 4 files changed, 301 insertions(+), 19 deletions(-)
doc/guides/rawdevs/ntb.rst +54 -0
@@ -45,6 +45,50 @@ to use, i.e. igb_uio, vfio. The ``dpdk-devbind.py`` script can be used to show devices status and to bind them to a suitable kernel driver. They will appear under the category of "Misc (rawdev) devices". +Prerequisites +------------- +NTB PMD needs kernel PCI driver to support write combining (WC) to get +better performance. The difference will be more than 10 times. +To enable WC, there are 2 ways. +- Insert igb_uio with ``wc_active=1`` flag if use igb_uio driver. + +.. code-block:: console + insmod igb_uio.ko wc_active=1 + +- Enable WC for NTB device's Bar 2 and Bar 4 (Mapped memory) manually. +The reference is https://www.kernel.org/doc/html/latest/x86/mtrr.html +Get bar base address using ``lspci -vvv -s ae:00.0 | grep Region``. + +.. code-block:: console + + # lspci -vvv -s ae:00.0 | grep Region + Region 0: Memory at 39bfe0000000 (64-bit, prefetchable) [size=64K] + Region 2: Memory at 39bfa0000000 (64-bit, prefetchable) [size=512M] + Region 4: Memory at 39bfc0000000 (64-bit, prefetchable) [size=512M] + +Using the following command to enable WC. + +.. code-block:: console + + echo "base=0x39bfa0000000 size=0x20000000 type=write-combining" >> /proc/mtrr + echo "base=0x39bfc0000000 size=0x20000000 type=write-combining" >> /proc/mtrr + +And the results: + +.. code-block:: console + + # cat /proc/mtrr + reg00: base=0x000000000 ( 0MB), size= 2048MB, count=1: write-back + reg01: base=0x07f000000 ( 2032MB), size= 16MB, count=1: uncachable + reg02: base=0x39bfa0000000 (60553728MB), size= 512MB, count=1: write-combining + reg03: base=0x39bfc0000000 (60554240MB), size= 512MB, count=1: write-combining + +To disable WC for these regions, using the following. + +.. code-block:: console + echo "disable=2" >> /proc/mtrr + echo "disable=3" >> /proc/mtrr + Ring Layout -----------
@@ -83,6 +127,16 @@ like the following: +------------------------+ +------------------------+ <---------traffic--------- +- Enqueue and Dequeue + Based on this ring layout, enqueue reads rx_tail to get how many free + buffers and writes used_ring and tx_tail to tell the peer which buffers + are filled with data. + And dequeue reads tx_tail to get how many packets are arrived, and + writes desc_ring and rx_tail to tell the peer about the new allocated + buffers. + So in this way, only remote write happens and remote read can be avoid + to get better performance. + Limitation ----------
drivers/raw/ntb/ntb.c +223 -19
272 lines not on this page · raw message
drivers/raw/ntb/ntb.h +2 -0
@@ -87,6 +87,7 @@ enum ntb_spad_idx {* @ntb_dev_init: Init ntb dev. * @get_peer_mw_addr: To get the addr of peer mw[mw_idx]. * @mw_set_trans: Set translation of internal memory that remote can access. +* @ioremap: Translate the remote host address to bar address. * @get_link_status: get link status, link speed and link width. * @set_link: Set local side up/down. * @spad_read: Read local/peer spad register val.
@@ -103,6 +104,7 @@ struct ntb_dev_ops {void *(*get_peer_mw_addr)(const struct rte_rawdev *dev,int mw_idx);int (*mw_set_trans)(const struct rte_rawdev *dev,int mw_idx,uint64_t addr,uint64_t size); +void *(*ioremap)(const struct rte_rawdev *dev,uint64_t addr);int (*get_link_status)(const struct rte_rawdev *dev);int (*set_link)(const struct rte_rawdev *dev,bool up); uint32_t (*spad_read)(const struct rte_rawdev *dev, int spad,
drivers/raw/ntb/ntb_hw_intel.c +22 -0
@@ -162,6 +162,27 @@ intel_ntb_mw_set_trans(const struct rte_rawdev *dev, int mw_idx,return 0 ; } +static void * +intel_ntb_ioremap (const struct rte_rawdev *dev,uint64_t addr) +{ +struct ntb_hw *hw = dev->dev_private; +void *mapped =NULL ; +void *base; +int i; + +for (i =0 ; i < hw->peer_used_mws; i++) { +if (addr >= hw->peer_mw_base[i] && + addr <= hw->peer_mw_base[i] + hw->mw_size[i]) { + base =intel_ntb_get_peer_mw_addr (dev, i); + mapped = (void *)(size_t )(addr - hw->peer_mw_base[i] + + (size_t )base); +break ; + } + } + +return mapped; +} +static int intel_ntb_get_link_status (const struct rte_rawdev *dev) {
@@ -357,6 +378,7 @@ const struct ntb_dev_ops intel_ntb_ops = { .ntb_dev_init = intel_ntb_dev_init, .get_peer_mw_addr = intel_ntb_get_peer_mw_addr, .mw_set_trans = intel_ntb_mw_set_trans, + .ioremap = intel_ntb_ioremap, .get_link_status = intel_ntb_get_link_status, .set_link = intel_ntb_set_link, .spad_read = intel_ntb_spad_read,
Signature · 1 line
2.17.1
PATCH v6 4/4 examples/ntb: support more functions for NTB
In reply to Xiaoyun LiSupport to transmit files between two systems. Support iofwd between one ethdev and NTB device. Support rxonly and txonly for NTB device. Support to set forwarding mode as file-trans, txonly, rxonly or iofwd.
Support to show/clear port stats and throughput.
--- doc/guides/sample_app_ug/ntb.rst | 59 +- examples/ntb/meson.build | 3 + examples/ntb/ntb_fwd.c | 1298 +++++++++++++++++++++++++++--- 3 files changed, 1232 insertions(+), 128 deletions(-)
doc/guides/sample_app_ug/ntb.rst +53 -6
@@ -5,8 +5,17 @@ NTB Sample Application ====================== The ntb sample application shows how to use ntb rawdev driver. -This sample provides interactive mode to transmit file between -two hosts. +This sample provides interactive mode to do packet based processing +between two systems. + +This sample supports 4 types of packet forwarding mode. + +* ``file-trans``: transmit files between two systems. The sample will + be polling to receive files from the peer and save the file as + ``ntb_recv_file[N]``, [N] represents the number of received file. +* ``rxonly``: NTB receives packets but doesn't transmit them. +* ``txonly``: NTB generates and transmits packets without receiving any. +* ``iofwd``: iofwd between NTB device and ethdev. Compiling the Application -------------------------
@@ -29,6 +38,40 @@ Refer to the *DPDK Getting Started Guide* for general information on running applications and the Environment Abstraction Layer (EAL) options. +Command-line Options +-------------------- + +The application supports the following command-line options. + +* ``--buf-size=N`` + + Set the data size of the mbufs used to N bytes, where N < 65536. + The default value is 2048. + +* ``--fwd-mode=mode`` + + Set the packet forwarding mode as ``file-trans``, ``txonly``, + ``rxonly`` or ``iofwd``. + +* ``--nb-desc=N`` + + Set number of descriptors of queue as N, namely queue size, + where 64 <= N <= 1024. The default value is 1024. + +* ``--txfreet=N`` + + Set the transmit free threshold of TX rings to N, where 0 <= N <= + the value of ``--nb-desc``. The default value is 256. + +* ``--burst=N`` + + Set the number of packets per burst to N, where 1 <= N <= 32. + The default value is 32. + +* ``--qp=N`` + + Set the number of queues as N, where qp > 0. The default value is 1. + Using the application ---------------------
@@ -41,7 +84,11 @@ The application is console-driven using the cmdline DPDK interface: From this interface the available commands and descriptions of what they do as as follows: -* ``send [filepath]``: Send file to the peer host. -* ``receive [filepath]``: Receive file to [filepath]. Need the peer - to send file successfully first. -* ``quit``: Exit program +* ``send [filepath]``: Send file to the peer host. Need to be in + file-trans forwarding mode first. +* ``start``: Start transmission. +* ``stop``: Stop transmission. +* ``show/clear port stats``: Show/Clear port stats and throughput. +* ``set fwd file-trans/rxonly/txonly/iofwd``: Set packet forwarding + mode. +* ``quit``: Exit program.
examples/ntb/meson.build +3 -0
@@ -14,3 +14,6 @@ cflags += ['-D_FILE_OFFSET_BITS=64'] sources = files( 'ntb_fwd.c' ) +if dpdk_conf.has('RTE_LIBRTE_PMD_NTB_RAWDEV') + deps += 'rawdev_ntb' +endif
examples/ntb/ntb_fwd.c +1176 -122
1,448 lines not on this page · raw message
Signature · 1 line
2.17.1
Quoted 10 lines
-----Original Message-----
From: Li, Xiaoyun
Sent: Thursday, September 26, 2019 11:20 AM
To: Wu, Jingjing <[email protected]>; Wiles, Keith <[email protected]>; Maslekar, Omkar <[email protected]>; Liang, Cunming <[email protected]>
Cc: [email protected]; Li, Xiaoyun <[email protected]>
Subject: [PATCH v6 0/4] enable FIFO for NTB
Quoted 7 lines
Enable FIFO for NTB rawdev driver to support packet based processing. And an example is provided to support txonly, rxonly, iofwd between NTB device and ethdev, and file transmission.
Acked-by Omkar Maslekar <[email protected]>
Series Acked-by: Jingjing Wu <[email protected]>
Thanks
Jingjing
Thomas Monjalon
Applied26/09/2019 06:04, Wu, Jingjing:
Quoted 9 lines
From: Li, Xiaoyun
Quoted 7 lines
Enable FIFO for NTB rawdev driver to support packet based processing. And an example is provided to support txonly, rxonly, iofwd between NTB device and ethdev, and file transmission.
Acked-by Omkar Maslekar <[email protected]>Series Acked-by: Jingjing Wu <[email protected]>
Applied, thanks
Quoted 8 lines · Thomas Monjalon
26/09/2019 06:04, Wu, Jingjing:
Quoted 6 lines
From: Li, Xiaoyun
Quoted 5 lines
Enable FIFO for NTB rawdev driver to support packet based processing. And an example is provided to support txonly, rxonly, iofwd between NTB device and ethdev, and file transmission.
Quoted 6 lines
Quoted 3 lines
Quoted 2 lines
Acked-by Omkar Maslekar <[email protected]>Series Acked-by: Jingjing Wu <[email protected]>
Applied, thanks
Pushed a fix on the last patch to fix master compilation, please check other variables:
https://git.dpdk.org/dpdk/commit/?id=fe56fe635b1d0933344d60e95d9359034dc52f75
Thanks.
Signature · 1 line
David Marchand
Thanks!
Quoted 36 lines
-----Original Message-----
From: David Marchand [mailto:[email protected]]
Sent: Monday, October 21, 2019 23:55
To: Li, Xiaoyun <[email protected]>
Cc: dev <[email protected]>; Wu, Jingjing <[email protected]>; Wiles, Keith <[email protected]>; Maslekar, Omkar <[email protected]>; Liang, Cunming <[email protected]>; Thomas Monjalon <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v6 0/4] enable FIFO for NTBQuoted 7 lines · Thomas Monjalon
26/09/2019 06:04, Wu, Jingjing:
Quoted 5 lines
From: Li, Xiaoyun
Quoted 4 lines
Enable FIFO for NTB rawdev driver to support packet based processing. And an example is provided to support txonly, rxonly, iofwd between NTB device and ethdev, and file transmission.
Quoted 6 lines
Quoted 3 lines
Quoted 2 lines
Acked-by Omkar Maslekar <[email protected]>Series Acked-by: Jingjing Wu <[email protected]>
Applied, thanks
Pushed a fix on the last patch to fix master compilation, please check other variables:
https://git.dpdk.org/dpdk/commit/?id=fe56fe635b1d0933344d60e95d9359034d
c52f75Thanks.
--
David Marchand