Skip to contentSkip to conversation

net/af_xdp: parse numa node id from sysfs

PATCH Applied Frank Du 2 participants 7 messages Dec 12, 2022 – Jan 18, 2023 Network drivers

lore
Share & tools
lore

Frank Du

author
#0 lore raw

PATCH 1/1 net/af_xdp: parse numa node id from sysfs

Get from /sys/class/net/{if}/device/numa_node.

---
 drivers/net/af_xdp/rte_eth_af_xdp.c | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)
drivers/net/af_xdp/rte_eth_af_xdp.c +14 -3
index b6ec9bf490..38b9d36ab5 100644
@@ -39,6 +39,7 @@
 #include <rte_power_intrinsics.h>
 
 #include "compat.h"
+#include "eal_filesystem.h"
 
 #ifndef SO_PREFER_BUSY_POLL
 #define SO_PREFER_BUSY_POLL 69
@@ -2038,9 +2039,6 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)
 		return -EINVAL;
 	}
 
-	if (dev->device.numa_node == SOCKET_ID_ANY)
-		dev->device.numa_node = rte_socket_id();
-
 	if (parse_parameters(kvlist, if_name, &xsk_start_queue_idx,
 			     &xsk_queue_cnt, &shared_umem, prog_path,
 			     &busy_budget, &force_copy) < 0) {
@@ -2053,6 +2051,19 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)
 		return -EINVAL;
 	}
 
+	/* get numa node id from net sysfs */
+	if (dev->device.numa_node == SOCKET_ID_ANY) {
+		unsigned long numa = 0;
+		char numa_path[PATH_MAX];
+
+		snprintf(numa_path, sizeof(numa_path), "/sys/class/net/%s/device/numa_node",
+			 if_name);
+		if (eal_parse_sysfs_value(numa_path, &numa) != 0)
+			dev->device.numa_node = rte_socket_id();
+		else
+			dev->device.numa_node = numa;
+	}
+
 	busy_budget = busy_budget == -1 ? ETH_AF_XDP_DFLT_BUSY_BUDGET :
 					busy_budget;
 
Signature · 1 line
2.34.1
#1 lore raw
Quoted 39 lines · Frank Du

Get from /sys/class/net/{if}/device/numa_node.

---
 drivers/net/af_xdp/rte_eth_af_xdp.c | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)
drivers/net/af_xdp/rte_eth_af_xdp.c +7 -3
index b6ec9bf490..38b9d36ab5 100644
@@ -39,6 +39,7 @@
 #include <rte_power_intrinsics.h>
 
 #include "compat.h"
+#include "eal_filesystem.h"
 
 #ifndef SO_PREFER_BUSY_POLL
 #define SO_PREFER_BUSY_POLL 69
@@ -2038,9 +2039,6 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)
 		return -EINVAL;
 	}
 
-	if (dev->device.numa_node == SOCKET_ID_ANY)
-		dev->device.numa_node = rte_socket_id();
-
 	if (parse_parameters(kvlist, if_name, &xsk_start_queue_idx,
 			     &xsk_queue_cnt, &shared_umem, prog_path,
 			     &busy_budget, &force_copy) < 0) {
@@ -2053,6 +2051,19 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)
 		return -EINVAL;
 	}
 
+	/* get numa node id from net sysfs */
+	if (dev->device.numa_node == SOCKET_ID_ANY) {
+		unsigned long numa = 0;
+		char numa_path[PATH_MAX];
+
+		snprintf(numa_path, sizeof(numa_path), "/sys/class/net/%s/device/numa_node",

drivers/net/af_xdp/rte_eth_af_xdp.c rte_pmd_af_xdp_probe(struct rte_vdev_device *dev) · line 2069

Quoted 10 lines
drivers/net/af_xdp/rte_eth_af_xdp.c +7 -0
+			 if_name);
+		if (eal_parse_sysfs_value(numa_path, &numa) != 0)
+			dev->device.numa_node = rte_socket_id();
+		else
+			dev->device.numa_node = numa;
+	}
+
 	busy_budget = busy_budget == -1 ? ETH_AF_XDP_DFLT_BUSY_BUDGET :
 					busy_budget;
 

Hi Frank,

It looks reasonable to set virtual DPDK af_xdp device socket to actual underlying device socket. And as I checked quickly, it works as expected.

But what is the impact and motivation of the patch? In other words why you are doing this patch and what output you are expecting as a result? Did you able to do any performance testing, and are you observing any difference before and after this test?

Thanks,
ferruh

author
#2 lore raw

Hi ferruh,

Our application use rte_eth_dev_socket_id to query the socket that a NIC port connected, then allocate lcore/memory according to this affinity.

The remote memory access is really slow compared to local.

Thanks,
Frank

-----Original Message-----
From: Ferruh Yigit, <[email protected]>
Sent: Monday, January 16, 2023 9:15 PM
To: Du, Frank <[email protected]>; Loftus, Ciara <[email protected]>
Cc: [email protected]
Subject: Re: [PATCH] net/af_xdp: parse numa node id from sysfs

Quoted 40 lines · Frank Du

Get from /sys/class/net/{if}/device/numa_node.

---
 drivers/net/af_xdp/rte_eth_af_xdp.c | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)
drivers/net/af_xdp/rte_eth_af_xdp.c +1 -3
index b6ec9bf490..38b9d36ab5 100644
@@ -39,6 +39,7 @@
 #include <rte_power_intrinsics.h>
 
 #include "compat.h"
+#include "eal_filesystem.h"
 
 #ifndef SO_PREFER_BUSY_POLL
 #define SO_PREFER_BUSY_POLL 69
@@ -2038,9 +2039,6 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)
 		return -EINVAL;
 	}
 
-	if (dev->device.numa_node == SOCKET_ID_ANY)
-		dev->device.numa_node = rte_socket_id();
-
 	if (parse_parameters(kvlist, if_name, &xsk_start_queue_idx,
 			     &xsk_queue_cnt, &shared_umem, prog_path,
 			     &busy_budget, &force_copy) < 0) { @@ -2053,6 +2051,19 @@ 

rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)

drivers/net/af_xdp/rte_eth_af_xdp.c +5 -0
 		return -EINVAL;
 	}
 
+	/* get numa node id from net sysfs */
+	if (dev->device.numa_node == SOCKET_ID_ANY) {
+		unsigned long numa = 0;
+		char numa_path[PATH_MAX];
+

drivers/net/af_xdp/rte_eth_af_xdp.c rte_pmd_af_xdp_probe(struct rte_vdev_device *dev) · line 2068

Quoted 10 lines
drivers/net/af_xdp/rte_eth_af_xdp.c +8 -0
+		snprintf(numa_path, sizeof(numa_path), "/sys/class/net/%s/device/numa_node",
+			 if_name);
+		if (eal_parse_sysfs_value(numa_path, &numa) != 0)
+			dev->device.numa_node = rte_socket_id();
+		else
+			dev->device.numa_node = numa;
+	}
+
 	busy_budget = busy_budget == -1 ? ETH_AF_XDP_DFLT_BUSY_BUDGET :
 					busy_budget;

Hi Frank,

It looks reasonable to set virtual DPDK af_xdp device socket to actual underlying device socket. And as I checked quickly, it works as expected.

But what is the impact and motivation of the patch? In other words why you are doing this patch and what output you are expecting as a result? Did you able to do any performance testing, and are you observing any difference before and after this test?

Thanks,
ferruh

#3 lore raw

On 1/17/2023 1:35 AM, Du, Frank wrote:

Moved down, please don't top post.

Quoted 68 lines
Quoted 62 lines

-----Original Message-----
From: Ferruh Yigit, <[email protected]>
Sent: Monday, January 16, 2023 9:15 PM
To: Du, Frank <[email protected]>; Loftus, Ciara <[email protected]>
Cc: [email protected]
Subject: Re: [PATCH] net/af_xdp: parse numa node id from sysfs

Quoted 40 lines · Frank Du

Get from /sys/class/net/{if}/device/numa_node.

---
 drivers/net/af_xdp/rte_eth_af_xdp.c | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)
drivers/net/af_xdp/rte_eth_af_xdp.c +1 -3
index b6ec9bf490..38b9d36ab5 100644
@@ -39,6 +39,7 @@
 #include <rte_power_intrinsics.h>
 
 #include "compat.h"
+#include "eal_filesystem.h"
 
 #ifndef SO_PREFER_BUSY_POLL
 #define SO_PREFER_BUSY_POLL 69
@@ -2038,9 +2039,6 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)
 		return -EINVAL;
 	}
 
-	if (dev->device.numa_node == SOCKET_ID_ANY)
-		dev->device.numa_node = rte_socket_id();
-
 	if (parse_parameters(kvlist, if_name, &xsk_start_queue_idx,
 			     &xsk_queue_cnt, &shared_umem, prog_path,
 			     &busy_budget, &force_copy) < 0) { @@ -2053,6 +2051,19 @@ 

rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)

drivers/net/af_xdp/rte_eth_af_xdp.c +5 -0
 		return -EINVAL;
 	}
 
+	/* get numa node id from net sysfs */
+	if (dev->device.numa_node == SOCKET_ID_ANY) {
+		unsigned long numa = 0;
+		char numa_path[PATH_MAX];
+

drivers/net/af_xdp/rte_eth_af_xdp.c rte_pmd_af_xdp_probe(struct rte_vdev_device *dev) · line 2068

Quoted 10 lines
drivers/net/af_xdp/rte_eth_af_xdp.c +8 -0
+		snprintf(numa_path, sizeof(numa_path), "/sys/class/net/%s/device/numa_node",
+			 if_name);
+		if (eal_parse_sysfs_value(numa_path, &numa) != 0)
+			dev->device.numa_node = rte_socket_id();
+		else
+			dev->device.numa_node = numa;
+	}
+
 	busy_budget = busy_budget == -1 ? ETH_AF_XDP_DFLT_BUSY_BUDGET :
 					busy_budget;

Hi Frank,

Quoted 5 lines

It looks reasonable to set virtual DPDK af_xdp device socket to actual underlying device socket. And as I checked quickly, it works as expected.

But what is the impact and motivation of the patch? In other words why you are doing this patch and what output you are expecting as a result? Did you able to do any performance testing, and are you observing any difference before and after this test?

Quoted 6 lines

Hi ferruh,

Our application use rte_eth_dev_socket_id to query the socket that a NIC port connected, then allocate lcore/memory according to this affinity.

The remote memory access is really slow compared to local.

As you observing any performance gain after change? If so, how much?

author
#4 lore raw
Quoted 89 lines

-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Tuesday, January 17, 2023 5:14 PM
To: Du, Frank <[email protected]>; Loftus, Ciara <[email protected]>
Cc: [email protected]
Subject: Re: [PATCH] net/af_xdp: parse numa node id from sysfs

On 1/17/2023 1:35 AM, Du, Frank wrote:

Moved down, please don't top post.

Quoted 25 lines
Quoted 25 lines

-----Original Message-----
From: Ferruh Yigit, <[email protected]>
Sent: Monday, January 16, 2023 9:15 PM
To: Du, Frank <[email protected]>; Loftus, Ciara <[email protected]>
Cc: [email protected]
Subject: Re: [PATCH] net/af_xdp: parse numa node id from sysfs

Quoted 15 lines · Frank Du

Get from /sys/class/net/{if}/device/numa_node.

---
 drivers/net/af_xdp/rte_eth_af_xdp.c | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)
drivers/net/af_xdp/rte_eth_af_xdp.c +0 -0
index b6ec9bf490..38b9d36ab5 100644
@@ -39,6 +39,7 @@
 #include <rte_power_intrinsics.h>
 
Quoted 6 lines
Quoted 6 lines

drivers/net/af_xdp/rte_eth_af_xdp.c rte_pmd_af_xdp_probe(struct rte_vdev_device

Quoted 6 lines
drivers/net/af_xdp/rte_eth_af_xdp.c +1 -0
 #include "compat.h"
+#include "eal_filesystem.h"
 
 #ifndef SO_PREFER_BUSY_POLL
 #define SO_PREFER_BUSY_POLL 69
@@ -2038,9 +2039,6 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device

*dev)

Quoted 9 lines
Quoted 9 lines

drivers/net/af_xdp/rte_eth_af_xdp.c rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)

Quoted 9 lines
drivers/net/af_xdp/rte_eth_af_xdp.c +0 -3
 		return -EINVAL;
 	}
 
-	if (dev->device.numa_node == SOCKET_ID_ANY)
-		dev->device.numa_node = rte_socket_id();
-
 	if (parse_parameters(kvlist, if_name, &xsk_start_queue_idx,
 			     &xsk_queue_cnt, &shared_umem, prog_path,
 			     &busy_budget, &force_copy) < 0) { @@ -2053,6

+2051,19 @@

Quoted 10 lines
Quoted 10 lines

drivers/net/af_xdp/rte_eth_af_xdp.c rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)

Quoted 10 lines

rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)

drivers/net/af_xdp/rte_eth_af_xdp.c +6 -0
 		return -EINVAL;
 	}
 
+	/* get numa node id from net sysfs */
+	if (dev->device.numa_node == SOCKET_ID_ANY) {
+		unsigned long numa = 0;
+		char numa_path[PATH_MAX];
+
+		snprintf(numa_path, sizeof(numa_path),

"/sys/class/net/%s/device/numa_node",

Quoted 13 lines
Quoted 13 lines

drivers/net/af_xdp/rte_eth_af_xdp.c rte_pmd_af_xdp_probe(struct rte_vdev_device *dev) · line 2068

Quoted 10 lines
drivers/net/af_xdp/rte_eth_af_xdp.c +7 -0
+			 if_name);
+		if (eal_parse_sysfs_value(numa_path, &numa) != 0)
+			dev->device.numa_node = rte_socket_id();
+		else
+			dev->device.numa_node = numa;
+	}
+
 	busy_budget = busy_budget == -1 ? ETH_AF_XDP_DFLT_BUSY_BUDGET :
 					busy_budget;

Hi Frank,

Quoted 1 line
Quoted 1 line

It looks reasonable to set virtual DPDK af_xdp device socket to actual

underlying device socket. And as I checked quickly, it works as expected.

Quoted 2 lines
Quoted 2 lines

But what is the impact and motivation of the patch? In other words why you

are doing this patch and what output you are expecting as a result?

Quoted 1 line
Quoted 1 line

Did you able to do any performance testing, and are you observing any

difference before and after this test?

Quoted 4 lines
Quoted 1 line

Hi ferruh,

Quoted 7 lines
Quoted 1 line

Our application use rte_eth_dev_socket_id to query the socket that a NIC port

connected, then allocate lcore/memory according to this affinity.

Quoted 3 lines

The remote memory access is really slow compared to local.

As you observing any performance gain after change? If so, how much?

Hi Ferruh,

The NIC in our setup is on the socket 1, if our workload(both memory and cpu) are running on socket 0, it can get max 12g/s throughput on a single core. With this patch, the workload is running on the correct socket 1 cpus, it can get up to 16g/s on a single core.

#5 lore raw
Quoted 92 lines · Frank
Quoted 92 lines

-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Tuesday, January 17, 2023 5:14 PM
To: Du, Frank <[email protected]>; Loftus, Ciara <[email protected]>
Cc: [email protected]
Subject: Re: [PATCH] net/af_xdp: parse numa node id from sysfs

On 1/17/2023 1:35 AM, Du, Frank wrote:

Moved down, please don't top post.

Quoted 25 lines
Quoted 25 lines

-----Original Message-----
From: Ferruh Yigit, <[email protected]>
Sent: Monday, January 16, 2023 9:15 PM
To: Du, Frank <[email protected]>; Loftus, Ciara <[email protected]>
Cc: [email protected]
Subject: Re: [PATCH] net/af_xdp: parse numa node id from sysfs

Quoted 15 lines · Frank Du

Get from /sys/class/net/{if}/device/numa_node.

---
 drivers/net/af_xdp/rte_eth_af_xdp.c | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)
drivers/net/af_xdp/rte_eth_af_xdp.c +0 -0
index b6ec9bf490..38b9d36ab5 100644
@@ -39,6 +39,7 @@
 #include <rte_power_intrinsics.h>
 
Quoted 6 lines
Quoted 6 lines

drivers/net/af_xdp/rte_eth_af_xdp.c rte_pmd_af_xdp_probe(struct rte_vdev_device

Quoted 6 lines
drivers/net/af_xdp/rte_eth_af_xdp.c +1 -0
 #include "compat.h"
+#include "eal_filesystem.h"
 
 #ifndef SO_PREFER_BUSY_POLL
 #define SO_PREFER_BUSY_POLL 69
@@ -2038,9 +2039,6 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device

*dev)

Quoted 9 lines
Quoted 9 lines

drivers/net/af_xdp/rte_eth_af_xdp.c rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)

Quoted 9 lines
drivers/net/af_xdp/rte_eth_af_xdp.c +0 -3
 		return -EINVAL;
 	}
 
-	if (dev->device.numa_node == SOCKET_ID_ANY)
-		dev->device.numa_node = rte_socket_id();
-
 	if (parse_parameters(kvlist, if_name, &xsk_start_queue_idx,
 			     &xsk_queue_cnt, &shared_umem, prog_path,
 			     &busy_budget, &force_copy) < 0) { @@ -2053,6

+2051,19 @@

Quoted 10 lines
Quoted 10 lines

drivers/net/af_xdp/rte_eth_af_xdp.c rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)

Quoted 10 lines

rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)

drivers/net/af_xdp/rte_eth_af_xdp.c +6 -0
 		return -EINVAL;
 	}
 
+	/* get numa node id from net sysfs */
+	if (dev->device.numa_node == SOCKET_ID_ANY) {
+		unsigned long numa = 0;
+		char numa_path[PATH_MAX];
+
+		snprintf(numa_path, sizeof(numa_path),

"/sys/class/net/%s/device/numa_node",

Quoted 13 lines
Quoted 13 lines

drivers/net/af_xdp/rte_eth_af_xdp.c rte_pmd_af_xdp_probe(struct rte_vdev_device *dev) · line 2068

Quoted 10 lines
drivers/net/af_xdp/rte_eth_af_xdp.c +7 -0
+			 if_name);
+		if (eal_parse_sysfs_value(numa_path, &numa) != 0)
+			dev->device.numa_node = rte_socket_id();
+		else
+			dev->device.numa_node = numa;
+	}
+
 	busy_budget = busy_budget == -1 ? ETH_AF_XDP_DFLT_BUSY_BUDGET :
 					busy_budget;

Hi Frank,

Quoted 1 line
Quoted 1 line

It looks reasonable to set virtual DPDK af_xdp device socket to actual

underlying device socket. And as I checked quickly, it works as expected.

Quoted 2 lines
Quoted 2 lines

But what is the impact and motivation of the patch? In other words why you

are doing this patch and what output you are expecting as a result?

Quoted 1 line
Quoted 1 line

Did you able to do any performance testing, and are you observing any

difference before and after this test?

Quoted 5 lines
Quoted 1 line

Hi ferruh,

Our application use rte_eth_dev_socket_id to query the socket that a NIC port

connected, then allocate lcore/memory according to this affinity.

Quoted 1 line
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines

The remote memory access is really slow compared to local.

As you observing any performance gain after change? If so, how much?

Hi Ferruh,

The NIC in our setup is on the socket 1, if our workload(both memory and cpu) are running on socket 0, it can get max 12g/s throughput on a single core. With this patch, the workload is running on the correct socket 1 cpus, it can get up to 16g/s on a single core.

Thanks for clarification.