net/af_xdp: parse numa node id from sysfs
PATCH 1/1 net/af_xdp: parse numa node id from sysfs
Get from /sys/class/net/{if}/device/numa_node.
--- drivers/net/af_xdp/rte_eth_af_xdp.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-)
drivers/net/af_xdp/rte_eth_af_xdp.c +14 -3
@@ -39,6 +39,7 @@#include <rte_power_intrinsics.h> #include "compat.h" +#include "eal_filesystem.h" #ifndef SO_PREFER_BUSY_POLL #define SO_PREFER_BUSY_POLL 69
@@ -2038,9 +2039,6 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)return -EINVAL ; } -if (dev->device.numa_node ==SOCKET_ID_ANY ) - dev->device.numa_node =rte_socket_id (); -if (parse_parameters (kvlist, if_name, &xsk_start_queue_idx, &xsk_queue_cnt, &shared_umem, prog_path, &busy_budget, &force_copy) <0 ) {
@@ -2053,6 +2051,19 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)return -EINVAL ; } +/* get numa node id from net sysfs */ +if (dev->device.numa_node ==SOCKET_ID_ANY ) { +unsigned long numa =0 ; +char numa_path[PATH_MAX ]; + +snprintf (numa_path,sizeof (numa_path),"/sys/class/net/%s/device/numa_node" , + if_name); +if (eal_parse_sysfs_value (numa_path, &numa) !=0 ) + dev->device.numa_node =rte_socket_id (); +else + dev->device.numa_node = numa; + } + busy_budget = busy_budget ==-1 ?ETH_AF_XDP_DFLT_BUSY_BUDGET : busy_budget;
Signature · 1 line
2.34.1
Quoted 39 lines · Frank Du
Get from
/sys/class/net/{if}/device/numa_node.Signed-off-by Frank Du <[email protected]>--- drivers/net/af_xdp/rte_eth_af_xdp.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-)drivers/net/af_xdp/rte_eth_af_xdp.c +7
-3@@ -39,6 +39,7 @@#include <rte_power_intrinsics.h> #include "compat.h" +#include "eal_filesystem.h" #ifndef SO_PREFER_BUSY_POLL #define SO_PREFER_BUSY_POLL 69@@ -2038,9 +2039,6 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)return -EINVAL ; } -if (dev->device.numa_node ==SOCKET_ID_ANY ) - dev->device.numa_node =rte_socket_id (); -if (parse_parameters (kvlist, if_name, &xsk_start_queue_idx, &xsk_queue_cnt, &shared_umem, prog_path, &busy_budget, &force_copy) <0 ) {@@ -2053,6 +2051,19 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)return -EINVAL ; } +/* get numa node id from net sysfs */ +if (dev->device.numa_node ==SOCKET_ID_ANY ) { +unsigned long numa =0 ; +char numa_path[PATH_MAX ]; + + snprintf(numa_path, sizeof(numa_path), "/sys/class/net/%s/device/numa_node",
drivers/net/af_xdp/rte_eth_af_xdp.c rte_pmd_af_xdp_probe(struct rte_vdev_device *dev) · line 2069
Quoted 10 lines
drivers/net/af_xdp/rte_eth_af_xdp.c +7
-0+ if_name); +if (eal_parse_sysfs_value (numa_path, &numa) !=0 ) + dev->device.numa_node =rte_socket_id (); +else + dev->device.numa_node = numa; + } + busy_budget = busy_budget ==-1 ?ETH_AF_XDP_DFLT_BUSY_BUDGET : busy_budget;
Hi Frank,
It looks reasonable to set virtual DPDK af_xdp device socket to actual underlying device socket. And as I checked quickly, it works as expected.
But what is the impact and motivation of the patch? In other words why you are doing this patch and what output you are expecting as a result? Did you able to do any performance testing, and are you observing any difference before and after this test?
Thanks,
ferruh
Hi ferruh,
Our application use rte_eth_dev_socket_id to query the socket that a NIC port connected, then allocate lcore/memory according to this affinity.
The remote memory access is really slow compared to local.
Thanks,
Frank
-----Original Message-----
From: Ferruh Yigit, <[email protected]>
Sent: Monday, January 16, 2023 9:15 PM
To: Du, Frank <[email protected]>; Loftus, Ciara <[email protected]>
Cc: [email protected]
Subject: Re: [PATCH] net/af_xdp: parse numa node id from sysfs
Quoted 40 lines · Frank Du
Get from
/sys/class/net/{if}/device/numa_node.Signed-off-by Frank Du <[email protected]>--- drivers/net/af_xdp/rte_eth_af_xdp.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-)drivers/net/af_xdp/rte_eth_af_xdp.c +1
-3@@ -39,6 +39,7 @@#include <rte_power_intrinsics.h> #include "compat.h" +#include "eal_filesystem.h" #ifndef SO_PREFER_BUSY_POLL #define SO_PREFER_BUSY_POLL 69@@ -2038,9 +2039,6 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)return -EINVAL ; } -if (dev->device.numa_node ==SOCKET_ID_ANY ) - dev->device.numa_node =rte_socket_id (); -if (parse_parameters (kvlist, if_name, &xsk_start_queue_idx, &xsk_queue_cnt, &shared_umem, prog_path, &busy_budget, &force_copy) < 0) { @@ -2053,6 +2051,19 @@
rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)drivers/net/af_xdp/rte_eth_af_xdp.c +5
-0return -EINVAL ; } +/* get numa node id from net sysfs */ +if (dev->device.numa_node ==SOCKET_ID_ANY ) { +unsigned long numa =0 ; +char numa_path[PATH_MAX ]; +
drivers/net/af_xdp/rte_eth_af_xdp.c rte_pmd_af_xdp_probe(struct rte_vdev_device *dev) · line 2068
Quoted 10 lines
drivers/net/af_xdp/rte_eth_af_xdp.c +8
-0+snprintf (numa_path,sizeof (numa_path),"/sys/class/net/%s/device/numa_node" , + if_name); +if (eal_parse_sysfs_value (numa_path, &numa) !=0 ) + dev->device.numa_node =rte_socket_id (); +else + dev->device.numa_node = numa; + } + busy_budget = busy_budget ==-1 ?ETH_AF_XDP_DFLT_BUSY_BUDGET : busy_budget;
Hi Frank,
It looks reasonable to set virtual DPDK af_xdp device socket to actual underlying device socket. And as I checked quickly, it works as expected.
But what is the impact and motivation of the patch? In other words why you are doing this patch and what output you are expecting as a result? Did you able to do any performance testing, and are you observing any difference before and after this test?
Thanks,
ferruh
On 1/17/2023 1:35 AM, Du, Frank wrote:
Moved down, please don't top post.
Quoted 68 lines
Quoted 62 lines
-----Original Message-----
From: Ferruh Yigit, <[email protected]>
Sent: Monday, January 16, 2023 9:15 PM
To: Du, Frank <[email protected]>; Loftus, Ciara <[email protected]>
Cc: [email protected]
Subject: Re: [PATCH]net/af_xdp: parse numa node id from sysfsQuoted 40 lines · Frank Du
Get from
/sys/class/net/{if}/device/numa_node.Signed-off-by Frank Du <[email protected]>--- drivers/net/af_xdp/rte_eth_af_xdp.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-)drivers/net/af_xdp/rte_eth_af_xdp.c +1
-3@@ -39,6 +39,7 @@#include <rte_power_intrinsics.h> #include "compat.h" +#include "eal_filesystem.h" #ifndef SO_PREFER_BUSY_POLL #define SO_PREFER_BUSY_POLL 69@@ -2038,9 +2039,6 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)return -EINVAL ; } -if (dev->device.numa_node ==SOCKET_ID_ANY ) - dev->device.numa_node =rte_socket_id (); -if (parse_parameters (kvlist, if_name, &xsk_start_queue_idx, &xsk_queue_cnt, &shared_umem, prog_path, &busy_budget, &force_copy) < 0) { @@ -2053,6 +2051,19 @@
rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)drivers/net/af_xdp/rte_eth_af_xdp.c +5
-0return -EINVAL ; } +/* get numa node id from net sysfs */ +if (dev->device.numa_node ==SOCKET_ID_ANY ) { +unsigned long numa =0 ; +char numa_path[PATH_MAX ]; +drivers/net/af_xdp/rte_eth_af_xdp.c rte_pmd_af_xdp_probe(struct rte_vdev_device *dev) · line 2068
Quoted 10 lines
drivers/net/af_xdp/rte_eth_af_xdp.c +8
-0+snprintf (numa_path,sizeof (numa_path),"/sys/class/net/%s/device/numa_node" , + if_name); +if (eal_parse_sysfs_value (numa_path, &numa) !=0 ) + dev->device.numa_node =rte_socket_id (); +else + dev->device.numa_node = numa; + } + busy_budget = busy_budget ==-1 ?ETH_AF_XDP_DFLT_BUSY_BUDGET : busy_budget;Hi Frank,
Quoted 5 lines
It looks reasonable to set virtual DPDK
af_xdpdevice socket to actual underlying device socket. And as I checked quickly, it works as expected.But what is the impact and motivation of the patch? In other words why you are doing this patch and what output you are expecting as a result? Did you able to do any performance testing, and are you observing any difference before and after this test?
Quoted 6 lines
Hi ferruh,
Our application use
rte_eth_dev_socket_idto query the socket that a NIC port connected, then allocate lcore/memory according to this affinity.The remote memory access is really slow compared to local.
As you observing any performance gain after change? If so, how much?
Quoted 89 lines
-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Tuesday, January 17, 2023 5:14 PM
To: Du, Frank <[email protected]>; Loftus, Ciara <[email protected]>
Cc: [email protected]
Subject: Re: [PATCH]net/af_xdp: parse numa node id from sysfsOn 1/17/2023 1:35 AM, Du, Frank wrote:
Moved down, please don't top post.
Quoted 25 lines
Quoted 25 lines
-----Original Message-----
From: Ferruh Yigit, <[email protected]>
Sent: Monday, January 16, 2023 9:15 PM
To: Du, Frank <[email protected]>; Loftus, Ciara <[email protected]>
Cc: [email protected]
Subject: Re: [PATCH]net/af_xdp: parse numa node id from sysfsQuoted 15 lines · Frank Du
Get from
/sys/class/net/{if}/device/numa_node.Signed-off-by Frank Du <[email protected]>--- drivers/net/af_xdp/rte_eth_af_xdp.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-)drivers/net/af_xdp/rte_eth_af_xdp.c +0
-0@@ -39,6 +39,7 @@#include <rte_power_intrinsics.h> Quoted 6 lines
Quoted 6 lines
drivers/net/af_xdp/rte_eth_af_xdp.c rte_pmd_af_xdp_probe(struct rte_vdev_device
Quoted 6 lines
drivers/net/af_xdp/rte_eth_af_xdp.c +1
-0#include "compat.h" +#include "eal_filesystem.h" #ifndef SO_PREFER_BUSY_POLL #define SO_PREFER_BUSY_POLL 69@@ -2038,9 +2039,6 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device*dev)
Quoted 9 lines
Quoted 9 lines
drivers/net/af_xdp/rte_eth_af_xdp.c rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)
Quoted 9 lines
drivers/net/af_xdp/rte_eth_af_xdp.c +0
-3return -EINVAL ; } -if (dev->device.numa_node ==SOCKET_ID_ANY ) - dev->device.numa_node =rte_socket_id (); -if (parse_parameters (kvlist, if_name, &xsk_start_queue_idx, &xsk_queue_cnt, &shared_umem, prog_path, &busy_budget, &force_copy) < 0) { @@ -2053,6+2051,19 @@
Quoted 10 lines
Quoted 10 lines
drivers/net/af_xdp/rte_eth_af_xdp.c rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)
Quoted 10 lines
rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)drivers/net/af_xdp/rte_eth_af_xdp.c +6
-0return -EINVAL ; } +/* get numa node id from net sysfs */ +if (dev->device.numa_node ==SOCKET_ID_ANY ) { +unsigned long numa =0 ; +char numa_path[PATH_MAX ]; + + snprintf(numa_path, sizeof(numa_path),"/sys/class/net/%s/device/numa_node",
Quoted 13 lines
Quoted 13 lines
drivers/net/af_xdp/rte_eth_af_xdp.c rte_pmd_af_xdp_probe(struct rte_vdev_device *dev) · line 2068
Quoted 10 lines
drivers/net/af_xdp/rte_eth_af_xdp.c +7
-0+ if_name); +if (eal_parse_sysfs_value (numa_path, &numa) !=0 ) + dev->device.numa_node =rte_socket_id (); +else + dev->device.numa_node = numa; + } + busy_budget = busy_budget ==-1 ?ETH_AF_XDP_DFLT_BUSY_BUDGET : busy_budget;Hi Frank,
Quoted 1 line
Quoted 1 line
It looks reasonable to set virtual DPDK
af_xdpdevice socket to actualunderlying device socket. And as I checked quickly, it works as expected.
Quoted 2 lines
Quoted 2 lines
But what is the impact and motivation of the patch? In other words why you
are doing this patch and what output you are expecting as a result?
Quoted 1 line
Quoted 1 line
Did you able to do any performance testing, and are you observing any
difference before and after this test?
Quoted 4 lines
Quoted 1 line
Hi ferruh,
Quoted 7 lines
Quoted 1 line
Our application use
rte_eth_dev_socket_idto query the socket that a NIC portconnected, then allocate lcore/memory according to this affinity.
Quoted 3 lines
The remote memory access is really slow compared to local.
As you observing any performance gain after change? If so, how much?
Hi Ferruh,
The NIC in our setup is on the socket 1, if our workload(both memory and cpu) are running on socket 0, it can get max 12g/s throughput on a single core. With this patch, the workload is running on the correct socket 1 cpus, it can get up to 16g/s on a single core.
Quoted 92 lines · Frank
Quoted 92 lines
-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Tuesday, January 17, 2023 5:14 PM
To: Du, Frank <[email protected]>; Loftus, Ciara <[email protected]>
Cc: [email protected]
Subject: Re: [PATCH]net/af_xdp: parse numa node id from sysfsOn 1/17/2023 1:35 AM, Du, Frank wrote:
Moved down, please don't top post.
Quoted 25 lines
Quoted 25 lines
-----Original Message-----
From: Ferruh Yigit, <[email protected]>
Sent: Monday, January 16, 2023 9:15 PM
To: Du, Frank <[email protected]>; Loftus, Ciara <[email protected]>
Cc: [email protected]
Subject: Re: [PATCH]net/af_xdp: parse numa node id from sysfsQuoted 15 lines · Frank Du
Get from
/sys/class/net/{if}/device/numa_node.Signed-off-by Frank Du <[email protected]>--- drivers/net/af_xdp/rte_eth_af_xdp.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-)drivers/net/af_xdp/rte_eth_af_xdp.c +0
-0@@ -39,6 +39,7 @@#include <rte_power_intrinsics.h> Quoted 6 lines
Quoted 6 lines
drivers/net/af_xdp/rte_eth_af_xdp.c rte_pmd_af_xdp_probe(struct rte_vdev_device
Quoted 6 lines
drivers/net/af_xdp/rte_eth_af_xdp.c +1
-0#include "compat.h" +#include "eal_filesystem.h" #ifndef SO_PREFER_BUSY_POLL #define SO_PREFER_BUSY_POLL 69@@ -2038,9 +2039,6 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device*dev)
Quoted 9 lines
Quoted 9 lines
drivers/net/af_xdp/rte_eth_af_xdp.c rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)
Quoted 9 lines
drivers/net/af_xdp/rte_eth_af_xdp.c +0
-3return -EINVAL ; } -if (dev->device.numa_node ==SOCKET_ID_ANY ) - dev->device.numa_node =rte_socket_id (); -if (parse_parameters (kvlist, if_name, &xsk_start_queue_idx, &xsk_queue_cnt, &shared_umem, prog_path, &busy_budget, &force_copy) < 0) { @@ -2053,6+2051,19 @@
Quoted 10 lines
Quoted 10 lines
drivers/net/af_xdp/rte_eth_af_xdp.c rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)
Quoted 10 lines
rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)drivers/net/af_xdp/rte_eth_af_xdp.c +6
-0return -EINVAL ; } +/* get numa node id from net sysfs */ +if (dev->device.numa_node ==SOCKET_ID_ANY ) { +unsigned long numa =0 ; +char numa_path[PATH_MAX ]; + + snprintf(numa_path, sizeof(numa_path),"/sys/class/net/%s/device/numa_node",
Quoted 13 lines
Quoted 13 lines
drivers/net/af_xdp/rte_eth_af_xdp.c rte_pmd_af_xdp_probe(struct rte_vdev_device *dev) · line 2068
Quoted 10 lines
drivers/net/af_xdp/rte_eth_af_xdp.c +7
-0+ if_name); +if (eal_parse_sysfs_value (numa_path, &numa) !=0 ) + dev->device.numa_node =rte_socket_id (); +else + dev->device.numa_node = numa; + } + busy_budget = busy_budget ==-1 ?ETH_AF_XDP_DFLT_BUSY_BUDGET : busy_budget;Hi Frank,
Quoted 1 line
Quoted 1 line
It looks reasonable to set virtual DPDK
af_xdpdevice socket to actualunderlying device socket. And as I checked quickly, it works as expected.
Quoted 2 lines
Quoted 2 lines
But what is the impact and motivation of the patch? In other words why you
are doing this patch and what output you are expecting as a result?
Quoted 1 line
Quoted 1 line
Did you able to do any performance testing, and are you observing any
difference before and after this test?
Quoted 5 lines
Quoted 1 line
Hi ferruh,
Our application use
rte_eth_dev_socket_idto query the socket that a NIC portconnected, then allocate lcore/memory according to this affinity.
Quoted 1 line
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines
The remote memory access is really slow compared to local.
As you observing any performance gain after change? If so, how much?
Hi Ferruh,
The NIC in our setup is on the socket 1, if our workload(both memory and cpu) are running on socket 0, it can get max 12g/s throughput on a single core. With this patch, the workload is running on the correct socket 1 cpus, it can get up to 16g/s on a single core.
Thanks for clarification.
Quoted 3 lines · Frank Du
Get from
/sys/class/net/{if}/device/numa_node.Signed-off-by Frank Du <[email protected]>
Applied to dpdk-next-net/main, thanks.