Skip to contentSkip to conversation

tidbits: net-udp: solicit new client for server mode

PATCH Superseded Hannes Diethelm 2 participants 24 messages May 28 – Jul 27 3 messages not in the archive

lore
Share & tools
lore
#0 lore raw

cover tidbits: net-udp: solicit new client for server mode

As discussed in [PATCH] tidbits: net-udp: add server and client mode, evl_net_solicit() is also needed for each client in server mode.

I went for option 2, it is not to lazy and I did not find any way to ask evl if an ARP is already done.

While testing, I also discovered that EINPROGRESS can appear after some time. I think this is due the kernel garbage-collects the ARP entry.

With this patch, this issue is also gone.

Hannes Diethelm (1):
tidbits: net-udp: solicit new client for server mode

 tidbits/oob-net-udp.c | 58 +++++++++++++++++++++++++++++++++++++------
 1 file changed, 50 insertions(+), 8 deletions(-)
Signature · 1 line
2.47.3
#1 lore raw

PATCH 1/1 tidbits: net-udp: solicit new client for server mode

This fixes random EINPROGRESS at init or during runtime.

Also correct whitespaces and make stdout more consistent.

---
 tidbits/oob-net-udp.c | 58 +++++++++++++++++++++++++++++++++++++------
 1 file changed, 50 insertions(+), 8 deletions(-)
tidbits/oob-net-udp.c +50 -8
index 11b8459..7af834f 100644
@@ -50,12 +50,12 @@ static void usage(void)
 }
 
 static void print_addr(char* text, struct sockaddr_in *addr){
-    char ip_str[INET_ADDRSTRLEN+1];
-    inet_ntop(AF_INET, &(addr->sin_addr), ip_str, sizeof(ip_str));
-    evl_printf("%s--------\n", text);
-    evl_printf("IP-Address: %s\n", ip_str);
-    evl_printf("Port:       %d\n", ntohs(addr->sin_port));
-    evl_printf("Family:     %d\n", addr->sin_family);
+	char ip_str[INET_ADDRSTRLEN+1];
+	inet_ntop(AF_INET, &(addr->sin_addr), ip_str, sizeof(ip_str));
+	evl_printf("== %s\n", text);
+	evl_printf("   ip-address: %s\n", ip_str);
+	evl_printf("   port:       %d\n", ntohs(addr->sin_port));
+	evl_printf("   family:     %d\n", addr->sin_family);
 }
 
 static void sender(int s, const char *text, int mcount,
@@ -226,6 +226,8 @@ static void client(int s, const char *text, int mcount,
 	free(tbuf);
 }
 
+#define SERVER_ADDR_LIST_SIZE 64
+
 static void server(int s, const char *text, int mcount,
 		struct sockaddr_in *addr, int iter)
 {
@@ -236,6 +238,9 @@ static void server(int s, const char *text, int mcount,
 	ssize_t ret;
 	char *tbuf;
 	char rbuf[16384];
+	in_addr_t addr_list[SERVER_ADDR_LIST_SIZE]={};
+	size_t addr_list_fill=0;
+	bool solicit_done;
 
 	tlen = (strlen(text) + 1) * mcount;
 	tbuf = malloc(tlen);
@@ -276,6 +281,39 @@ static void server(int s, const char *text, int mcount,
 			evl_printf(" (TRUNCATED)");
 		evl_printf(": %.*s\n", (int)ret, rbuf);
 
+		/*
+		 * We need to call evl_net_solicit for each new
+		 * client once before sending data. This will break
+		 * realtime for the first response.
+		 * If this is not done and the ARP address is not
+		 * yet in cache or garbage-collected, oob_sendmsg
+		 * will return EINPROGRESS on start or during runtime.
+		 */
+		solicit_done = false;
+		for(size_t i = 0; i < addr_list_fill && !solicit_done; i++){
+			if(addr_list[i] == _addr.sin_addr.s_addr){
+				solicit_done = true;
+			}
+		}
+		if(!solicit_done){
+			if(verbosity){
+				char ip_str[INET_ADDRSTRLEN+1];
+				inet_ntop(AF_INET, &(_addr.sin_addr), ip_str, sizeof(ip_str));
+				evl_printf("== client %s first seen: evl_net_solicit\n", ip_str);
+			}
+			ret = evl_net_solicit(s, (const struct sockaddr *)&_addr,
+					EVL_NEIGH_PERMANENT);
+			if (ret)
+				error(1, -ret, "evl_net_solicit()");
+
+			if(addr_list_fill < SERVER_ADDR_LIST_SIZE){
+				addr_list[addr_list_fill] = _addr.sin_addr.s_addr;
+				addr_list_fill ++;
+			}else{
+				error(1, EPERM, "address list full");
+			}
+		}
+
 		iov.iov_base = tbuf;
 		iov.iov_len = tlen;
 		msghdr.msg_iov = &iov;
@@ -433,7 +471,7 @@ int main(int argc, char *argv[])
 		receiver(s, &addr, iter);
 	} else if (mode == CLIENT) {
 		if (verbosity)
-			printf("== client mode (<= %s:%d)\n", ip, port);
+			printf("== client mode (<=> %s:%d)\n", ip, port);
 
 		/*
 		 * Guarantee a mere oob path from the first packet
@@ -449,8 +487,12 @@ int main(int argc, char *argv[])
 		client(s, text, mcount, &addr, iter, delay);
 	} else if (mode == SERVER) {
 		if (verbosity)
-			printf("== server mode (<= %s:%d)\n", ip, port);
+			printf("== server mode (<=> %s:%d)\n", ip, port);
 
+		/*
+		 * The server calls evl_net_solicit() internally
+		 * for each new ip address.
+		 */
 		server(s, text, mcount, &addr, iter);
 	} else {
 		error(1, 0, "Mode not implemented");
Signature · 1 line
2.47.3
#2 lore raw
Quoted 42 lines · Hannes Diethelm

This fixes random EINPROGRESS at init or during runtime.

Also correct whitespaces and make stdout more consistent.

---
 tidbits/oob-net-udp.c | 58 +++++++++++++++++++++++++++++++++++++------
 1 file changed, 50 insertions(+), 8 deletions(-)
tidbits/oob-net-udp.c +8 -6
index 11b8459..7af834f 100644
@@ -50,12 +50,12 @@ static void usage(void)
 }
 
 static void print_addr(char* text, struct sockaddr_in *addr){
-    char ip_str[INET_ADDRSTRLEN+1];
-    inet_ntop(AF_INET, &(addr->sin_addr), ip_str, sizeof(ip_str));
-    evl_printf("%s--------\n", text);
-    evl_printf("IP-Address: %s\n", ip_str);
-    evl_printf("Port:       %d\n", ntohs(addr->sin_port));
-    evl_printf("Family:     %d\n", addr->sin_family);
+	char ip_str[INET_ADDRSTRLEN+1];
+	inet_ntop(AF_INET, &(addr->sin_addr), ip_str, sizeof(ip_str));
+	evl_printf("== %s\n", text);
+	evl_printf("   ip-address: %s\n", ip_str);
+	evl_printf("   port:       %d\n", ntohs(addr->sin_port));
+	evl_printf("   family:     %d\n", addr->sin_family);
 }
 
 static void sender(int s, const char *text, int mcount,
@@ -226,6 +226,8 @@ static void client(int s, const char *text, int mcount,
 	free(tbuf);
 }
 
+#define SERVER_ADDR_LIST_SIZE 64
+
 static void server(int s, const char *text, int mcount,
 		struct sockaddr_in *addr, int iter)
 {

tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 241

Quoted 4 lines
tidbits/oob-net-udp.c +1 -0
@@ -236,6 +238,9 @@ static void server(int s, const char *text, int mcount,
 	ssize_t ret;
 	char *tbuf;
 	char rbuf[16384];
+	in_addr_t addr_list[SERVER_ADDR_LIST_SIZE]={};

^ missing whitespaces

Quoted 7 lines
tidbits/oob-net-udp.c +2 -0
+	size_t addr_list_fill=0;
+	bool solicit_done;
 
 	tlen = (strlen(text) + 1) * mcount;
 	tbuf = malloc(tlen);
@@ -276,6 +281,39 @@ static void server(int s, const char *text, int mcount,
 			evl_printf(" (TRUNCATED)");

tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 291

Quoted 10 lines
tidbits/oob-net-udp.c +8 -0
 		evl_printf(": %.*s\n", (int)ret, rbuf);
 
+		/*
+		 * We need to call evl_net_solicit for each new
+		 * client once before sending data. This will break
+		 * realtime for the first response.
+		 * If this is not done and the ARP address is not
+		 * yet in cache or garbage-collected, oob_sendmsg
+		 * will return EINPROGRESS on start or during runtime.
+		 */

We can happily send redundant solicit requests to the core, no need to filter out cached addresses, evl_net_solicit() will do the right thing.

tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 293

Quoted 2 lines
tidbits/oob-net-udp.c +2 -0
+		solicit_done = false;
+		for(size_t i = 0; i < addr_list_fill && !solicit_done; i++){

^ missing whitespace

tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 294

Quoted 1 line
tidbits/oob-net-udp.c +1 -0
+			if(addr_list[i] == _addr.sin_addr.s_addr){

^ missing whitespace

Quoted 7 lines
tidbits/oob-net-udp.c +7 -0
+				solicit_done = true;
+			}
+		}
+		if(!solicit_done){
+			if(verbosity){
+				char ip_str[INET_ADDRSTRLEN+1];
+				inet_ntop(AF_INET, &(_addr.sin_addr), ip_str, sizeof(ip_str));

tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 311

Quoted 10 lines
tidbits/oob-net-udp.c +10 -0
+				evl_printf("== client %s first seen: evl_net_solicit\n", ip_str);
+			}
+			ret = evl_net_solicit(s, (const struct sockaddr *)&_addr,
+					EVL_NEIGH_PERMANENT);
+			if (ret)
+				error(1, -ret, "evl_net_solicit()");
+
+			if(addr_list_fill < SERVER_ADDR_LIST_SIZE){
+				addr_list[addr_list_fill] = _addr.sin_addr.s_addr;
+				addr_list_fill ++;

^ extraneous whitespace

Quoted 31 lines
tidbits/oob-net-udp.c +11 -2
+			}else{
+				error(1, EPERM, "address list full");
+			}
+		}
+
 		iov.iov_base = tbuf;
 		iov.iov_len = tlen;
 		msghdr.msg_iov = &iov;
@@ -433,7 +471,7 @@ int main(int argc, char *argv[])
 		receiver(s, &addr, iter);
 	} else if (mode == CLIENT) {
 		if (verbosity)
-			printf("== client mode (<= %s:%d)\n", ip, port);
+			printf("== client mode (<=> %s:%d)\n", ip, port);
 
 		/*
 		 * Guarantee a mere oob path from the first packet
@@ -449,8 +487,12 @@ int main(int argc, char *argv[])
 		client(s, text, mcount, &addr, iter, delay);
 	} else if (mode == SERVER) {
 		if (verbosity)
-			printf("== server mode (<= %s:%d)\n", ip, port);
+			printf("== server mode (<=> %s:%d)\n", ip, port);
 
+		/*
+		 * The server calls evl_net_solicit() internally
+		 * for each new ip address.
+		 */
 		server(s, text, mcount, &addr, iter);
 	} else {
 		error(1, 0, "Mode not implemented");
Signature · 1 line
Philippe.
#3 lore raw
Quoted 58 lines · Philippe Gerum
Quoted 42 lines · Hannes Diethelm

This fixes random EINPROGRESS at init or during runtime.

Also correct whitespaces and make stdout more consistent.

---
 tidbits/oob-net-udp.c | 58 +++++++++++++++++++++++++++++++++++++------
 1 file changed, 50 insertions(+), 8 deletions(-)
tidbits/oob-net-udp.c +8 -6
index 11b8459..7af834f 100644
@@ -50,12 +50,12 @@ static void usage(void)
 }
 
 static void print_addr(char* text, struct sockaddr_in *addr){
-    char ip_str[INET_ADDRSTRLEN+1];
-    inet_ntop(AF_INET, &(addr->sin_addr), ip_str, sizeof(ip_str));
-    evl_printf("%s--------\n", text);
-    evl_printf("IP-Address: %s\n", ip_str);
-    evl_printf("Port:       %d\n", ntohs(addr->sin_port));
-    evl_printf("Family:     %d\n", addr->sin_family);
+	char ip_str[INET_ADDRSTRLEN+1];
+	inet_ntop(AF_INET, &(addr->sin_addr), ip_str, sizeof(ip_str));
+	evl_printf("== %s\n", text);
+	evl_printf("   ip-address: %s\n", ip_str);
+	evl_printf("   port:       %d\n", ntohs(addr->sin_port));
+	evl_printf("   family:     %d\n", addr->sin_family);
 }
 
 static void sender(int s, const char *text, int mcount,
@@ -226,6 +226,8 @@ static void client(int s, const char *text, int mcount,
 	free(tbuf);
 }
 
+#define SERVER_ADDR_LIST_SIZE 64
+
 static void server(int s, const char *text, int mcount,
 		struct sockaddr_in *addr, int iter)
 {

tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 241

Quoted 4 lines
tidbits/oob-net-udp.c +1 -0
@@ -236,6 +238,9 @@ static void server(int s, const char *text, int mcount,
 	ssize_t ret;
 	char *tbuf;
 	char rbuf[16384];
+	in_addr_t addr_list[SERVER_ADDR_LIST_SIZE]={};

^ missing whitespaces

Quoted 9 lines
tidbits/oob-net-udp.c +2 -0
+	size_t addr_list_fill=0;
+	bool solicit_done;
 
 	tlen = (strlen(text) + 1) * mcount;
 	tbuf = malloc(tlen);
@@ -276,6 +281,39 @@ static void server(int s, const char *text, int mcount,
 			evl_printf(" (TRUNCATED)");
 		evl_printf(": %.*s\n", (int)ret, rbuf);
 
Quoted 12 lines

tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 291

Quoted 8 lines
tidbits/oob-net-udp.c +8 -0
+		/*
+		 * We need to call evl_net_solicit for each new
+		 * client once before sending data. This will break
+		 * realtime for the first response.
+		 * If this is not done and the ARP address is not
+		 * yet in cache or garbage-collected, oob_sendmsg
+		 * will return EINPROGRESS on start or during runtime.
+		 */

We can happily send redundant solicit requests to the core, no need to filter out cached addresses, evl_net_solicit() will do the right thing.

Except that doing so would always demote the caller to the in-band stage, which may not be what you want. The fact that we'd need to maintain a cache in apps in order to figure out whether solicitation should be done shows a shortcoming in the core, users should not have to do this dance.

We should have an oob call for probing the route+arp caches with ipv4. I'll look into this asap.

Signature · 1 line
Philippe.
#4 lore raw

Am 01.06.26 um 11:10 schrieb Philippe Gerum:

Quoted 73 lines
Quoted 58 lines · Philippe Gerum
Quoted 42 lines · Hannes Diethelm

This fixes random EINPROGRESS at init or during runtime.

Also correct whitespaces and make stdout more consistent.

---
  tidbits/oob-net-udp.c | 58 +++++++++++++++++++++++++++++++++++++------
  1 file changed, 50 insertions(+), 8 deletions(-)
tidbits/oob-net-udp.c +8 -6
index 11b8459..7af834f 100644
@@ -50,12 +50,12 @@ static void usage(void)
  }
 
  static void print_addr(char* text, struct sockaddr_in *addr){
-    char ip_str[INET_ADDRSTRLEN+1];
-    inet_ntop(AF_INET, &(addr->sin_addr), ip_str, sizeof(ip_str));
-    evl_printf("%s--------\n", text);
-    evl_printf("IP-Address: %s\n", ip_str);
-    evl_printf("Port:       %d\n", ntohs(addr->sin_port));
-    evl_printf("Family:     %d\n", addr->sin_family);
+	char ip_str[INET_ADDRSTRLEN+1];
+	inet_ntop(AF_INET, &(addr->sin_addr), ip_str, sizeof(ip_str));
+	evl_printf("== %s\n", text);
+	evl_printf("   ip-address: %s\n", ip_str);
+	evl_printf("   port:       %d\n", ntohs(addr->sin_port));
+	evl_printf("   family:     %d\n", addr->sin_family);
  }
 
  static void sender(int s, const char *text, int mcount,
@@ -226,6 +226,8 @@ static void client(int s, const char *text, int mcount,
  	free(tbuf);
  }
 
+#define SERVER_ADDR_LIST_SIZE 64
+
  static void server(int s, const char *text, int mcount,
  		struct sockaddr_in *addr, int iter)
  {

tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 241

Quoted 4 lines
tidbits/oob-net-udp.c +1 -0
@@ -236,6 +238,9 @@ static void server(int s, const char *text, int mcount,
  	ssize_t ret;
  	char *tbuf;
  	char rbuf[16384];
+	in_addr_t addr_list[SERVER_ADDR_LIST_SIZE]={};

^ missing whitespaces

Quoted 9 lines
tidbits/oob-net-udp.c +2 -0
+	size_t addr_list_fill=0;
+	bool solicit_done;
 
  	tlen = (strlen(text) + 1) * mcount;
  	tbuf = malloc(tlen);
@@ -276,6 +281,39 @@ static void server(int s, const char *text, int mcount,
  			evl_printf(" (TRUNCATED)");
  		evl_printf(": %.*s\n", (int)ret, rbuf);
 
Quoted 12 lines

tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 291

Quoted 8 lines
tidbits/oob-net-udp.c +8 -0
+		/*
+		 * We need to call evl_net_solicit for each new
+		 * client once before sending data. This will break
+		 * realtime for the first response.
+		 * If this is not done and the ARP address is not
+		 * yet in cache or garbage-collected, oob_sendmsg
+		 * will return EINPROGRESS on start or during runtime.
+		 */

We can happily send redundant solicit requests to the core, no need to filter out cached addresses, evl_net_solicit() will do the right thing.

Quoted 9 lines

Except that doing so would always demote the caller to the in-band stage, which may not be what you want. The fact that we'd need to maintain a cache in apps in order to figure out whether solicitation should be done shows a shortcoming in the core, users should not have to do this dance.

We should have an oob call for probing the route+arp caches with ipv4. I'll look into this asap.

Yes, this is the reason I do it this way and print out a message if this happens.

I think server mode with multiple clients in real time is not something you can just easily do without careful considerations what happens when multiple clients send a message at the same time or TDMA behind. My intent in this example is mostly to show that it is possible.

A real application would need one of these options:
- A thread only for solicitation so the main thread doesn't get blocked
- evl_net_solicit() with a flag like MSG_DONTWAIT and then poll in the main loop if the client is ready
- Know the clients in advance
- A warm up phase during clients can connect
- ...

However, a way for probing and reading the arp would shurely help. It would also be nice to have an "evl net" command to list the entry's. I tried the normal arp command but it doesn't behave nicely with evl.

No hurry from my side, I have linuxcnc already running well with evl oob networking, I only need client mode and i know the IP in advance, so all good there.

Sorry about the whitespace issues, switching between projects... I sent a patch v2. Feel free to merge this already and I can create a separate patch when probing ARP is here or wait and I create a patch v3.

#5 lore raw
Quoted 93 lines · Hannes Diethelm

Am 01.06.26 um 11:10 schrieb Philippe Gerum:

Quoted 67 lines
Quoted 56 lines · Philippe Gerum
Quoted 39 lines · Hannes Diethelm

This fixes random EINPROGRESS at init or during runtime.

Also correct whitespaces and make stdout more consistent.

---
  tidbits/oob-net-udp.c | 58 +++++++++++++++++++++++++++++++++++++------
  1 file changed, 50 insertions(+), 8 deletions(-)
tidbits/oob-net-udp.c +7 -6
index 11b8459..7af834f 100644
@@ -50,12 +50,12 @@ static void usage(void)
  }
    static void print_addr(char* text, struct sockaddr_in *addr){
-    char ip_str[INET_ADDRSTRLEN+1];
-    inet_ntop(AF_INET, &(addr->sin_addr), ip_str, sizeof(ip_str));
-    evl_printf("%s--------\n", text);
-    evl_printf("IP-Address: %s\n", ip_str);
-    evl_printf("Port:       %d\n", ntohs(addr->sin_port));
-    evl_printf("Family:     %d\n", addr->sin_family);
+	char ip_str[INET_ADDRSTRLEN+1];
+	inet_ntop(AF_INET, &(addr->sin_addr), ip_str, sizeof(ip_str));
+	evl_printf("== %s\n", text);
+	evl_printf("   ip-address: %s\n", ip_str);
+	evl_printf("   port:       %d\n", ntohs(addr->sin_port));
+	evl_printf("   family:     %d\n", addr->sin_family);
  }
    static void sender(int s, const char *text, int mcount,
@@ -226,6 +226,8 @@ static void client(int s, const char *text, int mcount,
  	free(tbuf);
  }
  +#define SERVER_ADDR_LIST_SIZE 64
+
  static void server(int s, const char *text, int mcount,
  		struct sockaddr_in *addr, int iter)
  {

tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 241

Quoted 4 lines
tidbits/oob-net-udp.c +1 -0
@@ -236,6 +238,9 @@ static void server(int s, const char *text, int mcount,
  	ssize_t ret;
  	char *tbuf;
  	char rbuf[16384];
+	in_addr_t addr_list[SERVER_ADDR_LIST_SIZE]={};

^ missing whitespaces

Quoted 10 lines
tidbits/oob-net-udp.c +4 -0
+	size_t addr_list_fill=0;
+	bool solicit_done;
    	tlen = (strlen(text) + 1) * mcount;
  	tbuf = malloc(tlen);
@@ -276,6 +281,39 @@ static void server(int s, const char *text, int mcount,
  			evl_printf(" (TRUNCATED)");
  		evl_printf(": %.*s\n", (int)ret, rbuf);
  +		/*
+		 * We need to call evl_net_solicit for each new
+		 * client once before sending data. This will break
Quoted 9 lines

tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 291

Quoted 5 lines
tidbits/oob-net-udp.c +5 -0
+		 * realtime for the first response.
+		 * If this is not done and the ARP address is not
+		 * yet in cache or garbage-collected, oob_sendmsg
+		 * will return EINPROGRESS on start or during runtime.
+		 */

We can happily send redundant solicit requests to the core, no need to filter out cached addresses, evl_net_solicit() will do the right thing.

Quoted 8 lines

Except that doing so would always demote the caller to the in-band stage, which may not be what you want. The fact that we'd need to maintain a cache in apps in order to figure out whether solicitation should be done shows a shortcoming in the core, users should not have to do this dance.
We should have an oob call for probing the route+arp caches with ipv4. I'll look into this asap.

Yes, this is the reason I do it this way and print out a message if this happens.

I think server mode with multiple clients in real time is not something you can just easily do without careful considerations what happens when multiple clients send a message at the same time or TDMA behind. My intent in this example is mostly to show that it is possible.

A real application would need one of these options:
- A thread only for solicitation so the main thread doesn't get blocked
- evl_net_solicit() with a flag like MSG_DONTWAIT and then poll in the main loop if the client is ready
- Know the clients in advance
- A warm up phase during clients can connect
- ...

Quoted 1 line

However, a way for probing and reading the arp would shurely help.

Here is a general proposal to deal with the issue of sending UDP packets to unplanned destinations, which is basically what the server mode has to do, when receiving unsolicited messages.

We have a single invariant to cope with: the evl network stack wants to use the routing+arp information produced by the regular in-band stack, so that we can benefit from potentially complex routing logic without reinventing that wheel. Therefore, the route to any new/unsolicited destination must be resolved by the in-band stack once so that we can send UDP packets to that peer from the oob stage next.

From the point above, and as you pointed out already, an oob sender must either plan for reaching a particular destination (e.g. soliciting it prior to entering time-critical code), or accept an initial delay for the route to be resolved if the destination is not yet known on first transmit (i.e. out of the oob caches).

So, I would say that evl could meet the requirements you stated above by implementing the following:

- Honor MSG_PROBE for oob_sendmsg(), so that only the general call sanity and route resolution to the destination host is performed when set in the request flags, without actually sending any data. On success of such call, we would know that the routing information is readily available from the oob caches, no offload to in-band would have happened if we had not given this flag. The absence of routing information to the destination from some oob cache would yield a specific error, so that the caller may decide what to do next.

- Extend the effect of receiving MSG_DONTWAIT (and more generally O_NONBLOCK on fildes) to what we would do upon missing routing
information: if present, return with a specific error code _without_ relaying the packet to the in-band stack. The caller may then decide to handle the case locally. Otherwise, proceed as usual (i.e. relay to the in-band stack, then notify the caller with -EINPROGRESS).

- Provide a way to synchronize with the route resolution process in evl, i.e. a syscall that would block until a given destination is available from the oob cache. This call already exists, evl_net_solicit() can be used for that purpose (if a resolution request is already in flight, subsequent ones to the same destination won't cause any harm).

Points 1 and 2 are implemented in [1]. Feedback greatly appreciated and important when you have time. Usability for real-world applications is key.

Quoted 3 lines

It would also be nice to have an "evl net" command to list the entry's. I tried the normal arp command but it doesn't behave nicely with evl.

Could you please elaborate on this issue? Currently, updates to the in-band cache are propagated to the oob cache (by registering a hook into the relevant notifier chain in the kernel). So I would expect all destinations of interest to oob which are visible from the in-band arp cache to be available from the evl map as well.

This said, I agree that we need a way to inspect the oob cache specifically. Working on it.

[1] https://gitlab.com/Xenomai/xenomai4/linux-evl/-/tree/wip/net-solicit?ref_type=heads

Signature · 1 line
Philippe.
#6 lore raw

Am 20.06.26 um 19:28 schrieb Philippe Gerum:

Quoted 117 lines
Quoted 93 lines · Hannes Diethelm

Am 01.06.26 um 11:10 schrieb Philippe Gerum:

Quoted 67 lines
Quoted 56 lines · Philippe Gerum
Quoted 39 lines · Hannes Diethelm

This fixes random EINPROGRESS at init or during runtime.

Also correct whitespaces and make stdout more consistent.

---
   tidbits/oob-net-udp.c | 58 +++++++++++++++++++++++++++++++++++++------
   1 file changed, 50 insertions(+), 8 deletions(-)
tidbits/oob-net-udp.c +7 -6
index 11b8459..7af834f 100644
@@ -50,12 +50,12 @@ static void usage(void)
   }
     static void print_addr(char* text, struct sockaddr_in *addr){
-    char ip_str[INET_ADDRSTRLEN+1];
-    inet_ntop(AF_INET, &(addr->sin_addr), ip_str, sizeof(ip_str));
-    evl_printf("%s--------\n", text);
-    evl_printf("IP-Address: %s\n", ip_str);
-    evl_printf("Port:       %d\n", ntohs(addr->sin_port));
-    evl_printf("Family:     %d\n", addr->sin_family);
+	char ip_str[INET_ADDRSTRLEN+1];
+	inet_ntop(AF_INET, &(addr->sin_addr), ip_str, sizeof(ip_str));
+	evl_printf("== %s\n", text);
+	evl_printf("   ip-address: %s\n", ip_str);
+	evl_printf("   port:       %d\n", ntohs(addr->sin_port));
+	evl_printf("   family:     %d\n", addr->sin_family);
   }
     static void sender(int s, const char *text, int mcount,
@@ -226,6 +226,8 @@ static void client(int s, const char *text, int mcount,
   	free(tbuf);
   }
   +#define SERVER_ADDR_LIST_SIZE 64
+
   static void server(int s, const char *text, int mcount,
   		struct sockaddr_in *addr, int iter)
   {

tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 241

Quoted 4 lines
tidbits/oob-net-udp.c +1 -0
@@ -236,6 +238,9 @@ static void server(int s, const char *text, int mcount,
   	ssize_t ret;
   	char *tbuf;
   	char rbuf[16384];
+	in_addr_t addr_list[SERVER_ADDR_LIST_SIZE]={};

^ missing whitespaces

Quoted 10 lines
tidbits/oob-net-udp.c +4 -0
+	size_t addr_list_fill=0;
+	bool solicit_done;
     	tlen = (strlen(text) + 1) * mcount;
   	tbuf = malloc(tlen);
@@ -276,6 +281,39 @@ static void server(int s, const char *text, int mcount,
   			evl_printf(" (TRUNCATED)");
   		evl_printf(": %.*s\n", (int)ret, rbuf);
   +		/*
+		 * We need to call evl_net_solicit for each new
+		 * client once before sending data. This will break
Quoted 9 lines

tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 291

Quoted 5 lines
tidbits/oob-net-udp.c +5 -0
+		 * realtime for the first response.
+		 * If this is not done and the ARP address is not
+		 * yet in cache or garbage-collected, oob_sendmsg
+		 * will return EINPROGRESS on start or during runtime.
+		 */

We can happily send redundant solicit requests to the core, no need to filter out cached addresses, evl_net_solicit() will do the right thing.

Quoted 8 lines

Except that doing so would always demote the caller to the in-band stage, which may not be what you want. The fact that we'd need to maintain a cache in apps in order to figure out whether solicitation should be done shows a shortcoming in the core, users should not have to do this dance.
We should have an oob call for probing the route+arp caches with ipv4. I'll look into this asap.

Yes, this is the reason I do it this way and print out a message if this happens.

I think server mode with multiple clients in real time is not something you can just easily do without careful considerations what happens when multiple clients send a message at the same time or TDMA behind. My intent in this example is mostly to show that it is possible.

A real application would need one of these options:
- A thread only for solicitation so the main thread doesn't get blocked
- evl_net_solicit() with a flag like MSG_DONTWAIT and then poll in the main loop if the client is ready
- Know the clients in advance
- A warm up phase during clients can connect
- ...

Quoted 1 line

However, a way for probing and reading the arp would shurely help.

Here is a general proposal to deal with the issue of sending UDP packets to unplanned destinations, which is basically what the server mode has to do, when receiving unsolicited messages.

We have a single invariant to cope with: the evl network stack wants to use the routing+arp information produced by the regular in-band stack, so that we can benefit from potentially complex routing logic without reinventing that wheel. Therefore, the route to any new/unsolicited destination must be resolved by the in-band stack once so that we can send UDP packets to that peer from the oob stage next.

From the point above, and as you pointed out already, an oob sender must
either plan for reaching a particular destination (e.g. soliciting it prior to entering time-critical code), or accept an initial delay for the route to be resolved if the destination is not yet known on first transmit (i.e. out of the oob caches).

So, I would say that evl could meet the requirements you stated above by implementing the following:

Quoted 8 lines

- Honor MSG_PROBE for oob_sendmsg(), so that only the general call sanity and route resolution to the destination host is performed when set in the request flags, without actually sending any data. On success of such call, we would know that the routing information is readily available from the oob caches, no offload to in-band would have happened if we had not given this flag. The absence of routing information to the destination from some oob cache would yield a specific error, so that the caller may decide what to do next.

It seams the flag MSG_PROBE is kernel only? I did not find any occurrence in /usr/include or in libevl.

Quoted 7 lines

- Extend the effect of receiving MSG_DONTWAIT (and more generally O_NONBLOCK on fildes) to what we would do upon missing routing
information: if present, return with a specific error code _without_ relaying the packet to the in-band stack. The caller may then decide to handle the case locally. Otherwise, proceed as usual (i.e. relay to the in-band stack, then notify the caller with -EINPROGRESS).

So the oob-net-udp server code can be changed to use MSG_DONTWAIT and if the return value is EWOULDBLOCK, call evl_net_solicit() and try again instead of holding a list of IP's right?

Quoted 10 lines

- Provide a way to synchronize with the route resolution process in evl, i.e. a syscall that would block until a given destination is available from the oob cache. This call already exists, evl_net_solicit() can be used for that purpose (if a resolution request is already in flight, subsequent ones to the same destination won't cause any harm).

Points 1 and 2 are implemented in [1]. Feedback greatly appreciated and important when you have time. Usability for real-world applications is key.

I am a bit busy right now, but I would like to test it. However, It can take a week or two.

Quoted 11 lines
Quoted 4 lines

It would also be nice to have an "evl net" command to list the entry's. I tried the normal arp command but it doesn't behave nicely with evl.

Could you please elaborate on this issue? Currently, updates to the in-band cache are propagated to the oob cache (by registering a hook into the relevant notifier chain in the kernel). So I would expect all destinations of interest to oob which are visible from the in-band arp cache to be available from the evl map as well.

The issue was on my side, I need to run "arp -n" so it does not resolve host names. Otherwise, it takes quite some time. But this makes sense, with OOB active, it will never receive the answer on the request, so it times out.

By the way: Since some time, all incoming network traffic goes to the OOB stage when it is enabled. However, all outgoing traffic from the in-band stage is still sent. Would it make sense to block all outgoing traffic after "evl net -ei" except might be ARP requests if they are initialized by evl_net_solicit()?

At the moment, this is done using nftables for linuxcnc, this works fine for preemt_rt and xenomai, so not really needed.

Quoted 5 lines

This said, I agree that we need a way to inspect the oob cache specifically. Working on it.

[1] https://gitlab.com/Xenomai/xenomai4/linux-evl/-/tree/wip/net-solicit?ref_type=heads

#7 lore raw
Quoted 10 lines · Hannes Diethelm

Am 20.06.26 um 19:28 schrieb Philippe Gerum:

Quoted 8 lines

- Honor MSG_PROBE for oob_sendmsg(), so that only the general call sanity and route resolution to the destination host is performed when set in the request flags, without actually sending any data. On success of such call, we would know that the routing information is readily available from the oob caches, no offload to in-band would have happened if we had not given this flag. The absence of routing information to the destination from some oob cache would yield a specific error, so that the caller may decide what to do next.

Quoted 3 lines

It seams the flag MSG_PROBE is kernel only? I did not find any occurrence in /usr/include or in libevl.

Yep, my bad. Using MSG_PROBE is not the right way, since that would conflict with MSG_PROXY in userland which has a totally different meaning. I have revisited the implementation, simplifying it actually:
since the evl netstack already accepts zero-sized messages, sending such a datagram to the UDP layer now amounts to returning early with the address resolution status, short-circuiting the logic before the actual transmission happens.

IOW, passing a NULL or empty iov into the msghdr struct does what MSG_PROBE was intended to do.

Quoted 11 lines
Quoted 6 lines

- Extend the effect of receiving MSG_DONTWAIT (and more generally O_NONBLOCK on fildes) to what we would do upon missing routing
information: if present, return with a specific error code _without_ relaying the packet to the in-band stack. The caller may then decide to handle the case locally. Otherwise, proceed as usual (i.e. relay to the in-band stack, then notify the caller with -EINPROGRESS).

So the oob-net-udp server code can be changed to use MSG_DONTWAIT and if the return value is EWOULDBLOCK, call evl_net_solicit() and try again instead of holding a list of IP's right?

Yep.

Quoted 11 lines
Quoted 10 lines

- Provide a way to synchronize with the route resolution process in evl,
i.e. a syscall that would block until a given destination is available from the oob cache. This call already exists, evl_net_solicit() can be used for that purpose (if a resolution request is already in flight, subsequent ones to the same destination won't cause any harm).
Points 1 and 2 are implemented in [1]. Feedback greatly appreciated and
important when you have time. Usability for real-world applications is key.

Quoted 3 lines

I am a bit busy right now, but I would like to test it. However, It can take a week or two.

Np. Your contribution has been valuable, so it's worth waiting for it.

Quoted 11 lines
Quoted 10 lines
Quoted 4 lines

It would also be nice to have an "evl net" command to list the entry's. I tried the normal arp command but it doesn't behave nicely with evl.

Could you please elaborate on this issue? Currently, updates to the in-band cache are propagated to the oob cache (by registering a hook into the relevant notifier chain in the kernel). So I would expect all destinations of interest to oob which are visible from the in-band arp cache to be available from the evl map as well.

Quoted 4 lines

The issue was on my side, I need to run "arp -n" so it does not resolve host names. Otherwise, it takes quite some time. But this makes sense, with OOB active, it will never receive the answer on the request, so it times out.

Ok, this is not user-friendly. I'll have a look.

Quoted 5 lines

By the way: Since some time, all incoming network traffic goes to the OOB stage when it is enabled. However, all outgoing traffic from the in-band stage is still sent. Would it make sense to block all outgoing traffic after "evl net -ei" except might be ARP requests if they are initialized by evl_net_solicit()?

I believe this may break some use cases I'm aware of. Some people do currently use such asymmetry between ingress-oob and egress-inband on the same interface on purpose.

Quoted 3 lines

At the moment, this is done using nftables for linuxcnc, this works fine for preemt_rt and xenomai, so not really needed.

Currently, it is assumed that any outgoing inband traffic has a purpose, even on an oob-enabled interface. What kind of egress inband traffic could/should be dropped safely when flowing through an oob interface?

Quoted 5 lines
Quoted 5 lines

This said, I agree that we need a way to inspect the oob cache specifically. Working on it.
[1]
https://gitlab.com/Xenomai/xenomai4/linux-evl/-/tree/wip/net-solicit?ref_type=heads

Signature · 1 line
Philippe.
#8 lore raw

Am 04.07.26 um 19:52 schrieb Philippe Gerum:

Quoted 24 lines
Quoted 10 lines · Hannes Diethelm

Am 20.06.26 um 19:28 schrieb Philippe Gerum:

Quoted 8 lines

- Honor MSG_PROBE for oob_sendmsg(), so that only the general call sanity and route resolution to the destination host is performed when set in the request flags, without actually sending any data. On success of such call, we would know that the routing information is readily available from the oob caches, no offload to in-band would have happened if we had not given this flag. The absence of routing information to the destination from some oob cache would yield a specific error, so that the caller may decide what to do next.

Quoted 3 lines

It seams the flag MSG_PROBE is kernel only? I did not find any occurrence in /usr/include or in libevl.

Yep, my bad. Using MSG_PROBE is not the right way, since that would conflict with MSG_PROXY in userland which has a totally different meaning. I have revisited the implementation, simplifying it actually:
since the evl netstack already accepts zero-sized messages, sending such a datagram to the UDP layer now amounts to returning early with the address resolution status, short-circuiting the logic before the actual transmission happens.

Quoted 2 lines

IOW, passing a NULL or empty iov into the msghdr struct does what MSG_PROBE was intended to do.

I tested this variant. It works. But I wonder:
If I use:
ret = oob_sendmsg(s, &msghdr, NULL, 0); errno is set to EHOSTUNREACH If I use:
ret = oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT); errno is set to EWOULDBLOCK

Is this intended? EHOSTUNREACH is like halve correct. Yes, the host can not be reached but only due to no ARP request is sent.

For oob-net-udp server mode, this variant is a bit wastefull due to oob_sendmsg() would either always be called twice or I would have to keep the list of clients.

Quoted 8 lines
Quoted 8 lines
Quoted 7 lines

- Extend the effect of receiving MSG_DONTWAIT (and more generally O_NONBLOCK on fildes) to what we would do upon missing routing
information: if present, return with a specific error code _without_ relaying the packet to the in-band stack. The caller may then decide to handle the case locally. Otherwise, proceed as usual (i.e. relay to the in-band stack, then notify the caller with -EINPROGRESS).

Quoted 6 lines
Quoted 4 lines

So the oob-net-udp server code can be changed to use MSG_DONTWAIT and if the return value is EWOULDBLOCK, call evl_net_solicit() and try again instead of holding a list of IP's right?

Yep.

This works nicely, I will send a patch changing oob-net-udp server mode to use this.

The only disadvantage is that if for what ever reason, the client is already in the ARP cache but not permanent, it is cleared after a timeout, so evl_net_solicit() can happen later than expected.

Quoted 42 lines
Quoted 12 lines
Quoted 11 lines

- Provide a way to synchronize with the route resolution process in evl,
i.e. a syscall that would block until a given destination is available from the oob cache. This call already exists, evl_net_solicit() can be used for that purpose (if a resolution request is already in flight, subsequent ones to the same destination won't cause any harm).
Points 1 and 2 are implemented in [1]. Feedback greatly appreciated and
important when you have time. Usability for real-world applications is key.

Quoted 3 lines

I am a bit busy right now, but I would like to test it. However, It can take a week or two.

Np. Your contribution has been valuable, so it's worth waiting for it.

Quoted 11 lines
Quoted 10 lines
Quoted 4 lines

It would also be nice to have an "evl net" command to list the entry's. I tried the normal arp command but it doesn't behave nicely with evl.

Could you please elaborate on this issue? Currently, updates to the in-band cache are propagated to the oob cache (by registering a hook into the relevant notifier chain in the kernel). So I would expect all destinations of interest to oob which are visible from the in-band arp cache to be available from the evl map as well.

Quoted 4 lines

The issue was on my side, I need to run "arp -n" so it does not resolve host names. Otherwise, it takes quite some time. But this makes sense, with OOB active, it will never receive the answer on the request, so it times out.

Ok, this is not user-friendly. I'll have a look.

Quoted 5 lines

By the way: Since some time, all incoming network traffic goes to the OOB stage when it is enabled. However, all outgoing traffic from the in-band stage is still sent. Would it make sense to block all outgoing traffic after "evl net -ei" except might be ARP requests if they are initialized by evl_net_solicit()?

Quoted 3 lines

I believe this may break some use cases I'm aware of. Some people do currently use such asymmetry between ingress-oob and egress-inband on the same interface on purpose.

In this case, it is fine.

Quoted 8 lines
Quoted 4 lines

At the moment, this is done using nftables for linuxcnc, this works fine for preemt_rt and xenomai, so not really needed.

Currently, it is assumed that any outgoing inband traffic has a purpose, even on an oob-enabled interface. What kind of egress inband traffic could/should be dropped safely when flowing through an oob interface?

Mostly unintended traffic by either user error or some running services sending discovery messages to all interfaces. Using nftables works nicely so there is no real need to do anything else.

Quoted 7 lines
Quoted 6 lines
Quoted 6 lines

This said, I agree that we need a way to inspect the oob cache specifically. Working on it.
[1]
https://gitlab.com/Xenomai/xenomai4/linux-evl/-/tree/wip/net-solicit?ref_type=heads

I've seen this commits are already on v6.12.y-cip-evl-rebase? I did these tests on v6.12.y-cip-evl-rebase.

#9 lore raw
Quoted 26 lines · Hannes Diethelm

Am 04.07.26 um 19:52 schrieb Philippe Gerum:

Quoted 15 lines
Quoted 10 lines · Hannes Diethelm

Am 20.06.26 um 19:28 schrieb Philippe Gerum:

Quoted 8 lines

- Honor MSG_PROBE for oob_sendmsg(), so that only the general call sanity and route resolution to the destination host is performed when set in the request flags, without actually sending any data. On success of such call, we would know that the routing information is readily available from the oob caches, no offload to in-band would have happened if we had not given this flag. The absence of routing information to the destination from some oob cache would yield a specific error, so that the caller may decide what to do next.

Quoted 3 lines

It seams the flag MSG_PROBE is kernel only? I did not find any occurrence in /usr/include or in libevl.

Quoted 9 lines

Yep, my bad. Using MSG_PROBE is not the right way, since that would conflict with MSG_PROXY in userland which has a totally different meaning. I have revisited the implementation, simplifying it actually:
since the evl netstack already accepts zero-sized messages, sending such a datagram to the UDP layer now amounts to returning early with the address resolution status, short-circuiting the logic before the actual transmission happens.
IOW, passing a NULL or empty iov into the msghdr struct does what MSG_PROBE was intended to do.

Quoted 9 lines

I tested this variant. It works. But I wonder:
If I use:
ret = oob_sendmsg(s, &msghdr, NULL, 0); errno is set to EHOSTUNREACH If I use:
ret = oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT); errno is set to EWOULDBLOCK

Is this intended? EHOSTUNREACH is like halve correct. Yes, the host can not be reached but only due to no ARP request is sent.

EHOSTUNREACH was intended as a way to distinguish from EWOULDBLOCK wrt lack of buffer space for the outgoing message, this code was the only option close enough to the idea to be conveyed available from the errno list that would not conflict with other situations. Now, since such probing mode needs no message space in the first place, this is guarding against the impossible, which does not make sense. Returning -EWOULDBLOCK in both cases above would still be
practical. e.g. something along these lines:

kernel/evl/net/ipv4/udp.c +10 -10
index d819616c3b5b..5912a1fbcfb5 100644
@@ -417,22 +417,22 @@ static ssize_t send_udp(struct evl_socket *esk,
 	 * address.
 	 */
 	ret = find_egress_path(esk, daddr, &ert, &earp, &pseudo_earp, msg_flags);
-	if (ret == -EMULTIHOP)
-		return ret;	/* MSG_DONTROUTE cannot be honored. */
-
 	if (ret) {
+		if (ret != -EHOSTUNREACH)
+			return ret;
+
+		if (datalen == 0)
+			return -EWOULDBLOCK; /* Address probe failed. */
+
 		/*
-		 * No route known from the front cache - bummer. We
-		 * may have to offload the transmit operation to the
-		 * in-band stack, unless only probing or MSG_DONTWAIT
-		 * is set.
+		 * We have a message to send but no route was found in
+		 * the front cache - bummer. We may have to offload
+		 * the transmit operation to the in-band stack, unless
+		 * only probing or MSG_DONTWAIT is set.
 		 */
 		if (msg_flags & MSG_DONTWAIT)
 			return -EWOULDBLOCK;
 
-		if (datalen == 0)
-			return ret;
-
 		/*
 		 * We always charge the socket even when offloading to
 		 * the in-band stack although we won't consume any
Quoted 3 lines

For oob-net-udp server mode, this variant is a bit wastefull due to oob_sendmsg() would either always be called twice or I would have to keep the list of clients.

Which brings back the option of some MSG_xxx operation flag so that we could pass a valid buffer _and_ a probing flag, but then we'd need Dovetail to add one to the standard list (in userland) because I don't see any standard one to piggyback off of.

Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
Quoted 7 lines

- Extend the effect of receiving MSG_DONTWAIT (and more generally O_NONBLOCK on fildes) to what we would do upon missing routing
information: if present, return with a specific error code _without_ relaying the packet to the in-band stack. The caller may then decide to handle the case locally. Otherwise, proceed as usual (i.e. relay to the in-band stack, then notify the caller with -EINPROGRESS).

So the oob-net-udp server code can be changed to use MSG_DONTWAIT and if the return value is EWOULDBLOCK, call evl_net_solicit() and try again instead of holding a list of IP's right?

Quoted 8 lines
Quoted 1 line

Yep.

This works nicely, I will send a patch changing oob-net-udp server mode to use this.

The only disadvantage is that if for what ever reason, the client is already in the ARP cache but not permanent, it is cleared after a timeout, so evl_net_solicit() can happen later than expected.

Yep, because at the moment, the core mirrors to the front cache all insertions and deletions happening into the inband cache unconditionally. We could force a permanent state for any entry we are about to insert into the front cache in order to prevent what you described, but I'm wary about unwanted side-effects.

Quoted 50 lines
Quoted 35 lines
Quoted 12 lines
Quoted 11 lines

- Provide a way to synchronize with the route resolution process in evl,
i.e. a syscall that would block until a given destination is available from the oob cache. This call already exists, evl_net_solicit() can be used for that purpose (if a resolution request is already in flight, subsequent ones to the same destination won't cause any harm).
Points 1 and 2 are implemented in [1]. Feedback greatly appreciated and
important when you have time. Usability for real-world applications is key.

Quoted 3 lines

I am a bit busy right now, but I would like to test it. However, It can take a week or two.

Np. Your contribution has been valuable, so it's worth waiting for it.

Quoted 11 lines
Quoted 10 lines
Quoted 4 lines

It would also be nice to have an "evl net" command to list the entry's. I tried the normal arp command but it doesn't behave nicely with evl.

Could you please elaborate on this issue? Currently, updates to the in-band cache are propagated to the oob cache (by registering a hook into the relevant notifier chain in the kernel). So I would expect all destinations of interest to oob which are visible from the in-band arp cache to be available from the evl map as well.

Quoted 4 lines

The issue was on my side, I need to run "arp -n" so it does not resolve host names. Otherwise, it takes quite some time. But this makes sense, with OOB active, it will never receive the answer on the request, so it times out.

Ok, this is not user-friendly. I'll have a look.

Quoted 8 lines
Quoted 5 lines

By the way: Since some time, all incoming network traffic goes to the OOB stage when it is enabled. However, all outgoing traffic from the in-band stage is still sent. Would it make sense to block all outgoing traffic after "evl net -ei" except might be ARP requests if they are initialized by evl_net_solicit()?

I believe this may break some use cases I'm aware of. Some people do currently use such asymmetry between ingress-oob and egress-inband on the same interface on purpose.

In this case, it is fine.

Quoted 4 lines
Quoted 4 lines

At the moment, this is done using nftables for linuxcnc, this works fine for preemt_rt and xenomai, so not really needed.

Quoted 9 lines
Quoted 4 lines

Currently, it is assumed that any outgoing inband traffic has a purpose,
even on an oob-enabled interface. What kind of egress inband traffic could/should be dropped safely when flowing through an oob interface?

Mostly unintended traffic by either user error or some running services sending discovery messages to all interfaces. Using nftables works nicely so there is no real need to do anything else.

Ok.

Quoted 9 lines
Quoted 7 lines
Quoted 6 lines
Quoted 6 lines

This said, I agree that we need a way to inspect the oob cache specifically. Working on it.
[1]
https://gitlab.com/Xenomai/xenomai4/linux-evl/-/tree/wip/net-solicit?ref_type=heads

I've seen this commits are already on v6.12.y-cip-evl-rebase? I did these tests on v6.12.y-cip-evl-rebase.

Yep, because this probing mode is here to stay, although I'm considering another way to provide this using a dedicated MSG_xx flag. Problem ATM is that I'd like not to pollute the namespace uselessly by creating non-std flags. This said, it may be legitimate to extend the existing set in this case.

Signature · 1 line
Philippe.
#10 lore raw

Am 20.07.26 um 16:27 schrieb Philippe Gerum:

Quoted 75 lines
Quoted 26 lines · Hannes Diethelm

Am 04.07.26 um 19:52 schrieb Philippe Gerum:

Quoted 15 lines
Quoted 10 lines · Hannes Diethelm

Am 20.06.26 um 19:28 schrieb Philippe Gerum:

Quoted 8 lines

- Honor MSG_PROBE for oob_sendmsg(), so that only the general call sanity and route resolution to the destination host is performed when set in the request flags, without actually sending any data. On success of such call, we would know that the routing information is readily available from the oob caches, no offload to in-band would have happened if we had not given this flag. The absence of routing information to the destination from some oob cache would yield a specific error, so that the caller may decide what to do next.

Quoted 3 lines

It seams the flag MSG_PROBE is kernel only? I did not find any occurrence in /usr/include or in libevl.

Quoted 9 lines

Yep, my bad. Using MSG_PROBE is not the right way, since that would conflict with MSG_PROXY in userland which has a totally different meaning. I have revisited the implementation, simplifying it actually:
since the evl netstack already accepts zero-sized messages, sending such a datagram to the UDP layer now amounts to returning early with the address resolution status, short-circuiting the logic before the actual transmission happens.
IOW, passing a NULL or empty iov into the msghdr struct does what MSG_PROBE was intended to do.

Quoted 9 lines

I tested this variant. It works. But I wonder:
If I use:
ret = oob_sendmsg(s, &msghdr, NULL, 0); errno is set to EHOSTUNREACH If I use:
ret = oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT); errno is set to EWOULDBLOCK

Is this intended? EHOSTUNREACH is like halve correct. Yes, the host can not be reached but only due to no ARP request is sent.

EHOSTUNREACH was intended as a way to distinguish from EWOULDBLOCK wrt lack of buffer space for the outgoing message, this code was the only option close enough to the idea to be conveyed available from the errno list that would not conflict with other situations. Now, since such probing mode needs no message space in the first place, this is guarding against the impossible, which does not make sense. Returning -EWOULDBLOCK in both cases above would still be
practical. e.g. something along these lines:

kernel/evl/net/ipv4/udp.c +10 -7
index d819616c3b5b..5912a1fbcfb5 100644
@@ -417,22 +417,22 @@ static ssize_t send_udp(struct evl_socket *esk,
  	 * address.
  	 */
  	ret = find_egress_path(esk, daddr, &ert, &earp, &pseudo_earp, msg_flags);
-	if (ret == -EMULTIHOP)
-		return ret;	/* MSG_DONTROUTE cannot be honored. */
-
  	if (ret) {
+		if (ret != -EHOSTUNREACH)
+			return ret;
+
+		if (datalen == 0)
+			return -EWOULDBLOCK; /* Address probe failed. */
+
  		/*
-		 * No route known from the front cache - bummer. We
-		 * may have to offload the transmit operation to the
-		 * in-band stack, unless only probing or MSG_DONTWAIT
-		 * is set.
+		 * We have a message to send but no route was found in
+		 * the front cache - bummer. We may have to offload
+		 * the transmit operation to the in-band stack, unless
+		 * only probing or MSG_DONTWAIT is set.

kernel/evl/net/ipv4/udp.c static ssize_t send_udp(struct evl_socket *esk, · line 438

Quoted 10 lines
kernel/evl/net/ipv4/udp.c +0 -3
  		 */
  		if (msg_flags & MSG_DONTWAIT)
  			return -EWOULDBLOCK;
 
-		if (datalen == 0)
-			return ret;
-
  		/*
  		 * We always charge the socket even when offloading to
  		 * the in-band stack although we won't consume any

I think in this case, it is fine as it is. You also won't expect EWOULDBLOCK or EAGAIN as long as you don't set MSG_DONTWAIT.

But now there are two ways of probing. Either with or withouth MSG_DONTWAIT that behave slightly different. Might be just support MSG_DONTWAIT -> EWOULDBLOCK and drop the other variant? Or is there a reason for the variant withouth MSG_DONTWAIT?

Quoted 9 lines
Quoted 3 lines

For oob-net-udp server mode, this variant is a bit wastefull due to oob_sendmsg() would either always be called twice or I would have to keep the list of clients.

Which brings back the option of some MSG_xxx operation flag so that we could pass a valid buffer _and_ a probing flag, but then we'd need Dovetail to add one to the standard list (in userland) because I don't see any standard one to piggyback off of.

Is there a reason to pass a valid buffer to only probe? Either you probe with a NULL buffer / size 0 or you pass a buffer and set MSG_DONTWAIT so the message is sent if possible. If not, evl_net_solicit() and retry.

BTW: NULL buffer / size 1 fails. But it also makes no sense.

Quoted 21 lines
Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
Quoted 7 lines

- Extend the effect of receiving MSG_DONTWAIT (and more generally O_NONBLOCK on fildes) to what we would do upon missing routing
information: if present, return with a specific error code _without_ relaying the packet to the in-band stack. The caller may then decide to handle the case locally. Otherwise, proceed as usual (i.e. relay to the in-band stack, then notify the caller with -EINPROGRESS).

So the oob-net-udp server code can be changed to use MSG_DONTWAIT and if the return value is EWOULDBLOCK, call evl_net_solicit() and try again instead of holding a list of IP's right?

Quoted 8 lines
Quoted 1 line

Yep.

This works nicely, I will send a patch changing oob-net-udp server mode to use this.

The only disadvantage is that if for what ever reason, the client is already in the ARP cache but not permanent, it is cleared after a timeout, so evl_net_solicit() can happen later than expected.

Quoted 5 lines

Yep, because at the moment, the core mirrors to the front cache all insertions and deletions happening into the inband cache unconditionally. We could force a permanent state for any entry we are about to insert into the front cache in order to prevent what you described, but I'm wary about unwanted side-effects.

Right now you can use evl_net_solicit(..., EVL_NEIGH_PERMANENT) to make shure it is permanent but this is an in band call according to the doc. So if you want to stay out of band, you need to probe.

Now the probing doesn't show if the arp entry is permanent or going to disapear soon. So it can pass at the first try and fail later. Right now, you are better off doing evl_net_solicit() for every client if you want to be shure.

With ioctl(s, SIOCGARP, &arp_request), you can check if the entry is permanent (arp_request.arp_flags & ATF_PERM) and then use evl_net_solicit(..., EVL_NEIGH_PERMANENT) if not. But the ioctl is also in band.

An option would be either to support of oob_ioctl() for SIOCGARP. Or create something like evl_net_routeinfo(s, addr) returning flags would make probing obsolete.

Flags could be:
EVL_ARP_COM Lookup complete (If this is not set, no ARP entry)
EVL_ARP_PERM Permanent entry
EVL_ROUTE_MULTIHOP More than one hop away (Will fail with MSG_DONTROUTE)

Quoted 71 lines
Quoted 51 lines
Quoted 36 lines
Quoted 13 lines
Quoted 12 lines

- Provide a way to synchronize with the route resolution process in evl,
i.e. a syscall that would block until a given destination is available from the oob cache. This call already exists, evl_net_solicit() can be used for that purpose (if a resolution request is already in flight, subsequent ones to the same destination won't cause any harm).
Points 1 and 2 are implemented in [1]. Feedback greatly appreciated and
important when you have time. Usability for real-world applications is key.

Quoted 3 lines

I am a bit busy right now, but I would like to test it. However, It can take a week or two.

Np. Your contribution has been valuable, so it's worth waiting for it.

Quoted 11 lines
Quoted 10 lines
Quoted 4 lines

It would also be nice to have an "evl net" command to list the entry's. I tried the normal arp command but it doesn't behave nicely with evl.

Could you please elaborate on this issue? Currently, updates to the in-band cache are propagated to the oob cache (by registering a hook into the relevant notifier chain in the kernel). So I would expect all destinations of interest to oob which are visible from the in-band arp cache to be available from the evl map as well.

Quoted 4 lines

The issue was on my side, I need to run "arp -n" so it does not resolve host names. Otherwise, it takes quite some time. But this makes sense, with OOB active, it will never receive the answer on the request, so it times out.

Ok, this is not user-friendly. I'll have a look.

Quoted 8 lines
Quoted 5 lines

By the way: Since some time, all incoming network traffic goes to the OOB stage when it is enabled. However, all outgoing traffic from the in-band stage is still sent. Would it make sense to block all outgoing traffic after "evl net -ei" except might be ARP requests if they are initialized by evl_net_solicit()?

I believe this may break some use cases I'm aware of. Some people do currently use such asymmetry between ingress-oob and egress-inband on the same interface on purpose.

In this case, it is fine.

Quoted 4 lines
Quoted 4 lines

At the moment, this is done using nftables for linuxcnc, this works fine for preemt_rt and xenomai, so not really needed.

Quoted 9 lines
Quoted 4 lines

Currently, it is assumed that any outgoing inband traffic has a purpose,
even on an oob-enabled interface. What kind of egress inband traffic could/should be dropped safely when flowing through an oob interface?

Mostly unintended traffic by either user error or some running services sending discovery messages to all interfaces. Using nftables works nicely so there is no real need to do anything else.

Ok.

Quoted 8 lines
Quoted 7 lines
Quoted 6 lines
Quoted 6 lines

This said, I agree that we need a way to inspect the oob cache specifically. Working on it.
[1]
https://gitlab.com/Xenomai/xenomai4/linux-evl/-/tree/wip/net-solicit?ref_type=heads

Quoted 8 lines
Quoted 1 line

I've seen this commits are already on v6.12.y-cip-evl-rebase? I did these tests on v6.12.y-cip-evl-rebase.

Yep, because this probing mode is here to stay, although I'm considering another way to provide this using a dedicated MSG_xx flag. Problem ATM is that I'd like not to pollute the namespace uselessly by creating non-std flags. This said, it may be legitimate to extend the existing set in this case.

Fine for me, so I did not have to compile yet another kernel.

#11 lore raw
Quoted 88 lines · Hannes Diethelm

Am 20.07.26 um 16:27 schrieb Philippe Gerum:

Quoted 73 lines
Quoted 26 lines · Hannes Diethelm

Am 04.07.26 um 19:52 schrieb Philippe Gerum:

Quoted 15 lines
Quoted 10 lines · Hannes Diethelm

Am 20.06.26 um 19:28 schrieb Philippe Gerum:

Quoted 8 lines

- Honor MSG_PROBE for oob_sendmsg(), so that only the general call sanity and route resolution to the destination host is performed when set in the request flags, without actually sending any data. On success of such call, we would know that the routing information is readily available from the oob caches, no offload to in-band would have happened if we had not given this flag. The absence of routing information to the destination from some oob cache would yield a specific error, so that the caller may decide what to do next.

Quoted 3 lines

It seams the flag MSG_PROBE is kernel only? I did not find any occurrence in /usr/include or in libevl.

Quoted 9 lines

Yep, my bad. Using MSG_PROBE is not the right way, since that would conflict with MSG_PROXY in userland which has a totally different meaning. I have revisited the implementation, simplifying it actually:
since the evl netstack already accepts zero-sized messages, sending such a datagram to the UDP layer now amounts to returning early with the address resolution status, short-circuiting the logic before the actual transmission happens.
IOW, passing a NULL or empty iov into the msghdr struct does what MSG_PROBE was intended to do.

Quoted 9 lines

I tested this variant. It works. But I wonder:
If I use:
ret = oob_sendmsg(s, &msghdr, NULL, 0); errno is set to EHOSTUNREACH If I use:
ret = oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT); errno is set to EWOULDBLOCK

Is this intended? EHOSTUNREACH is like halve correct. Yes, the host can not be reached but only due to no ARP request is sent.

EHOSTUNREACH was intended as a way to distinguish from EWOULDBLOCK wrt
lack of buffer space for the outgoing message, this code was the only option close enough to the idea to be conveyed available from the errno list that would not conflict with other situations. Now, since such probing mode needs no message space in the first place, this is guarding against the impossible, which does not make sense. Returning -EWOULDBLOCK in both cases above would still be
practical. e.g. something along these lines:

kernel/evl/net/ipv4/udp.c +9 -7
index d819616c3b5b..5912a1fbcfb5 100644
@@ -417,22 +417,22 @@ static ssize_t send_udp(struct evl_socket *esk,
  	 * address.
  	 */
  	ret = find_egress_path(esk, daddr, &ert, &earp, &pseudo_earp, msg_flags);
-	if (ret == -EMULTIHOP)
-		return ret;	/* MSG_DONTROUTE cannot be honored. */
-
  	if (ret) {
+		if (ret != -EHOSTUNREACH)
+			return ret;
+
+		if (datalen == 0)
+			return -EWOULDBLOCK; /* Address probe failed. */
+
  		/*
-		 * No route known from the front cache - bummer. We
-		 * may have to offload the transmit operation to the
-		 * in-band stack, unless only probing or MSG_DONTWAIT
-		 * is set.
+		 * We have a message to send but no route was found in
+		 * the front cache - bummer. We may have to offload
+		 * the transmit operation to the in-band stack, unless

kernel/evl/net/ipv4/udp.c static ssize_t send_udp(struct evl_socket *esk, · line 438

Quoted 10 lines
kernel/evl/net/ipv4/udp.c +1 -2
+		 * only probing or MSG_DONTWAIT is set.
  		 */
  		if (msg_flags & MSG_DONTWAIT)
  			return -EWOULDBLOCK;
  -		if (datalen == 0)
-			return ret;
-
  		/*
  		 * We always charge the socket even when offloading to
  		 * the in-band stack although we won't consume any

I think in this case, it is fine as it is. You also won't expect EWOULDBLOCK or EAGAIN as long as you don't set MSG_DONTWAIT.

Quoted 3 lines

But now there are two ways of probing. Either with or withouth MSG_DONTWAIT that behave slightly different. Might be just support MSG_DONTWAIT -> EWOULDBLOCK and drop the other
variant?

You mean detect a probing request when receiving MSG_DONTWAIT and a zero-sized buffer? That is an option. Another option would be to always return EHOSTUNREACH/??? on failed probe regardless of whether MSG_DONTWAIT is set. I would preferably go for the second option iff we can settle on a unconfusing, unambiguous error status.

Quoted 3 lines

Or is there a reason
for the variant withouth MSG_DONTWAIT?

In fact, the original intent was to assume that receiving a zero-sized message should be considered as a probing request. Then the effect of detecting MSG_DONTWAIT too in this particular code path was overlooked, which unexpectedly introduced another variant, which is indeed one too many.

Quoted 10 lines
Quoted 9 lines
Quoted 3 lines

For oob-net-udp server mode, this variant is a bit wastefull due to oob_sendmsg() would either always be called twice or I would have to keep the list of clients.

Which brings back the option of some MSG_xxx operation flag so that we
could pass a valid buffer _and_ a probing flag, but then we'd need Dovetail to add one to the standard list (in userland) because I don't see any standard one to piggyback off of.

Quoted 6 lines

Is there a reason to pass a valid buffer to only probe? Either you probe with a NULL buffer / size 0 or you pass a buffer and set MSG_DONTWAIT so the message is sent if possible. If not, evl_net_solicit() and retry.

BTW: NULL buffer / size 1 fails. But it also makes no sense.

Yep, in this case, -EFAULT is a reasonable outcome since the key argument is the message size. If the core is told that at least one byte is valid, then a valid pointer to at least one byte should be given.

Quoted 38 lines
Quoted 20 lines
Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
Quoted 7 lines

- Extend the effect of receiving MSG_DONTWAIT (and more generally O_NONBLOCK on fildes) to what we would do upon missing routing
information: if present, return with a specific error code _without_ relaying the packet to the in-band stack. The caller may then decide to handle the case locally. Otherwise, proceed as usual (i.e. relay to the in-band stack, then notify the caller with -EINPROGRESS).

So the oob-net-udp server code can be changed to use MSG_DONTWAIT and if the return value is EWOULDBLOCK, call evl_net_solicit() and try again instead of holding a list of IP's right?

Quoted 8 lines
Quoted 1 line

Yep.

This works nicely, I will send a patch changing oob-net-udp server mode to use this.

The only disadvantage is that if for what ever reason, the client is already in the ARP cache but not permanent, it is cleared after a timeout, so evl_net_solicit() can happen later than expected.

Quoted 5 lines

Yep, because at the moment, the core mirrors to the front cache all insertions and deletions happening into the inband cache unconditionally. We could force a permanent state for any entry we are about to insert into the front cache in order to prevent what you described, but I'm wary about unwanted side-effects.

Right now you can use evl_net_solicit(..., EVL_NEIGH_PERMANENT) to make shure it is permanent but this is an in band call according to the doc. So if you want to stay out of band, you need to probe.

Now the probing doesn't show if the arp entry is permanent or going to disapear soon. So it can pass at the first try and fail later. Right now, you are better off doing evl_net_solicit() for every client if you want to be shure.

With ioctl(s, SIOCGARP, &arp_request), you can check if the entry is permanent (arp_request.arp_flags & ATF_PERM) and then use evl_net_solicit(..., EVL_NEIGH_PERMANENT) if not. But the ioctl is also in band.

Quoted 8 lines

An option would be either to support of oob_ioctl() for SIOCGARP. Or create something like evl_net_routeinfo(s, addr) returning flags would make probing obsolete.

Flags could be:
EVL_ARP_COM Lookup complete (If this is not set, no ARP entry)
EVL_ARP_PERM Permanent entry
EVL_ROUTE_MULTIHOP More than one hop away (Will fail with MSG_DONTROUTE)

Having oob_ioctl(SIOCGARP) which would perform a lookup into the oob front cache managed by the core is perfectly doable, then we could provide evl_net_routeinfo() as syntactic sugar building on it.

I would consider reusing the ATF_* flags directly instead of adding yet another set of EVL_* flags only to alias to the former, since both caches hold the very same routing entries, therefore do provide the same information. In that sense, EVL_NEIGH_PERMANENT seems redundant since it fundamentally means ATF_PERM. Documenting ATF_PERM in the API docs while aliasing EVL_NEIGH_PERMANENT to ATF_PERM for backward compat may be a reasonable trade-off, until we can drop the former in order to further reduce the namespace pollution.

Signature · 1 line
Philippe.
#12 lore raw
Quoted 94 lines · Philippe Gerum
Quoted 88 lines · Hannes Diethelm

Am 20.07.26 um 16:27 schrieb Philippe Gerum:

Quoted 73 lines
Quoted 26 lines · Hannes Diethelm

Am 04.07.26 um 19:52 schrieb Philippe Gerum:

Quoted 15 lines
Quoted 10 lines · Hannes Diethelm

Am 20.06.26 um 19:28 schrieb Philippe Gerum:

Quoted 8 lines

- Honor MSG_PROBE for oob_sendmsg(), so that only the general call sanity and route resolution to the destination host is performed when set in the request flags, without actually sending any data. On success of such call, we would know that the routing information is readily available from the oob caches, no offload to in-band would have happened if we had not given this flag. The absence of routing information to the destination from some oob cache would yield a specific error, so that the caller may decide what to do next.

Quoted 3 lines

It seams the flag MSG_PROBE is kernel only? I did not find any occurrence in /usr/include or in libevl.

Quoted 9 lines

Yep, my bad. Using MSG_PROBE is not the right way, since that would conflict with MSG_PROXY in userland which has a totally different meaning. I have revisited the implementation, simplifying it actually:
since the evl netstack already accepts zero-sized messages, sending such a datagram to the UDP layer now amounts to returning early with the address resolution status, short-circuiting the logic before the actual transmission happens.
IOW, passing a NULL or empty iov into the msghdr struct does what MSG_PROBE was intended to do.

Quoted 9 lines

I tested this variant. It works. But I wonder:
If I use:
ret = oob_sendmsg(s, &msghdr, NULL, 0); errno is set to EHOSTUNREACH If I use:
ret = oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT); errno is set to EWOULDBLOCK

Is this intended? EHOSTUNREACH is like halve correct. Yes, the host can not be reached but only due to no ARP request is sent.

EHOSTUNREACH was intended as a way to distinguish from EWOULDBLOCK wrt
lack of buffer space for the outgoing message, this code was the only option close enough to the idea to be conveyed available from the errno list that would not conflict with other situations. Now, since such probing mode needs no message space in the first place, this is guarding against the impossible, which does not make sense. Returning -EWOULDBLOCK in both cases above would still be
practical. e.g. something along these lines:

kernel/evl/net/ipv4/udp.c +9 -7
index d819616c3b5b..5912a1fbcfb5 100644
@@ -417,22 +417,22 @@ static ssize_t send_udp(struct evl_socket *esk,
  	 * address.
  	 */
  	ret = find_egress_path(esk, daddr, &ert, &earp, &pseudo_earp, msg_flags);
-	if (ret == -EMULTIHOP)
-		return ret;	/* MSG_DONTROUTE cannot be honored. */
-
  	if (ret) {
+		if (ret != -EHOSTUNREACH)
+			return ret;
+
+		if (datalen == 0)
+			return -EWOULDBLOCK; /* Address probe failed. */
+
  		/*
-		 * No route known from the front cache - bummer. We
-		 * may have to offload the transmit operation to the
-		 * in-band stack, unless only probing or MSG_DONTWAIT
-		 * is set.
+		 * We have a message to send but no route was found in
+		 * the front cache - bummer. We may have to offload
+		 * the transmit operation to the in-band stack, unless

kernel/evl/net/ipv4/udp.c static ssize_t send_udp(struct evl_socket *esk, · line 438

Quoted 10 lines
kernel/evl/net/ipv4/udp.c +1 -2
+		 * only probing or MSG_DONTWAIT is set.
  		 */
  		if (msg_flags & MSG_DONTWAIT)
  			return -EWOULDBLOCK;
  -		if (datalen == 0)
-			return ret;
-
  		/*
  		 * We always charge the socket even when offloading to
  		 * the in-band stack although we won't consume any

I think in this case, it is fine as it is. You also won't expect EWOULDBLOCK or EAGAIN as long as you don't set MSG_DONTWAIT.

Quoted 3 lines

But now there are two ways of probing. Either with or withouth MSG_DONTWAIT that behave slightly different. Might be just support MSG_DONTWAIT -> EWOULDBLOCK and drop the other
variant?

Quoted 6 lines

You mean detect a probing request when receiving MSG_DONTWAIT and a zero-sized buffer? That is an option. Another option would be to always return EHOSTUNREACH/??? on failed probe regardless of whether MSG_DONTWAIT is set. I would preferably go for the second option iff we can settle on a unconfusing, unambiguous error status.

Ok, I believe that the best option is to go back to an explicit operation flag for probing eventually, because all other options seem confusing. Therefore MSG_PROBE handling was resurrected [1] in the udp layer, and EHOSTUNREACH is unambiguously used to denote a failed probe if MSG_PROBE is set.

As you pointed out, the MSG_PROBE definition is missing from the common socket.h bits in user-space, but the same flag value is defined as MSG_PROXY in *libc headers, which looks like obsolete. Anyway, to address this, MSG_PROBE was added to the evl/uapi bits with enough guards to prevent conflicts or mismatches. It defines the constant expected by the kernel, which is very unlikely to change since this belongs to the part of the ABI which is written in stone.

The commit log states the possible outcomes depending on the operation flags and/or size of the message passed to oob_sendmsg().

[1]
https://gitlab.com/Xenomai/xenomai4/linux-evl/-/commit/566f032dc3bc4c3e945e94887b7f57cd6e57dfb6

Signature · 1 line
Philippe.
#13 lore raw
In reply to Hannes Diethelm
Quoted 5 lines · Hannes Diethelm

Am 20.07.26 um 16:27 schrieb Philippe Gerum:

An option would be either to support of oob_ioctl() for SIOCGARP. Or create something like evl_net_routeinfo(s, addr) returning flags would make probing obsolete.

Not entirely. The issue with solely having SIOCGARP or any probe-only explicit request is that you would have to pair two syscalls at each transmit at least, one to probe for the sender address before possibly soliciting that peer if absent from the oob cache, another one for sending the message eventually. i.e., for every packet:

    ioctl(SIOCGARP)
         !ATF_COM? -> evl_net_solicit()
    oob_sendmsg(..., 0)

OTOH, with the latest attempt to address this issue, we can send a probe-only request by passing MSG_PROBE (EHOSTUNREACH), or a request that does not attempt to defer transmit to the inband stage on probe failure by passing MSG_DONTWAIT (EWOULDBLOCK). i.e., for every packet

redo:
oob_sendmsg(..., MSG_DONTWAIT)
EWOULDBLOCK?
oob_sendmsg(..., MSG_PROBE)
EHOSTUNREACH?
evl_net_solicit()
goto redo
otherwise assume ENOMEM
otherwise all done

IOW, using a proper combination of MSG_DONTWAIT and MSG_PROBE in the right sequence would either succeed to send the packet immediately on the first oob_sendmsg() call, otherwise fail on memory shortage or missing route from the oob cache. In the latter case, which could only happen once for each new peer under normal circumstances, we could disambiguate the EWOULDBLOCK status using an explicit probe.

A better way to do this in a single step unambiguously would require the addition of another operation flag, like MSG_DONTDEFER, preventing the deferral to inband on failed probe and causing oob_sendmsg() to return with a specific error code. e.g. something as simple as the following would cover all requirements:

     oob_sendmsg(..., MSG_DONTDEFER) -> EADDRNOTAVAIL on failed probe.

Now, we might also piggyback off of MSG_OOB instead of defining yet another operation flag, as a way to say "oob only, don't relay to inband", but I'm still pondering whether this would be nicely witty or utterly confusing..

Signature · 1 line
Philippe.
#14 lore raw

Am 21.07.26 um 22:15 schrieb Philippe Gerum:

Quoted 14 lines
Quoted 5 lines · Hannes Diethelm

Am 20.07.26 um 16:27 schrieb Philippe Gerum:

An option would be either to support of oob_ioctl() for SIOCGARP. Or create something like evl_net_routeinfo(s, addr) returning flags would make probing obsolete.

Not entirely. The issue with solely having SIOCGARP or any probe-only explicit request is that you would have to pair two syscalls at each transmit at least, one to probe for the sender address before possibly soliciting that peer if absent from the oob cache, another one for sending the message eventually. i.e., for every packet:

Quoted 3 lines
     ioctl(SIOCGARP)
          !ATF_COM? -> evl_net_solicit()
     oob_sendmsg(..., 0)

You are right, that is an unneeded syscall as long as you don't keep a list of clients as before. This can remove the need from calling evl_net_solicit() in the case the client is already in ARP and permanent but this will be the only change and won't help that much.

Quoted 29 lines

OTOH, with the latest attempt to address this issue, we can send a probe-only request by passing MSG_PROBE (EHOSTUNREACH), or a request that does not attempt to defer transmit to the inband stage on probe failure by passing MSG_DONTWAIT (EWOULDBLOCK). i.e., for every packet

redo:
oob_sendmsg(..., MSG_DONTWAIT)
EWOULDBLOCK?
oob_sendmsg(..., MSG_PROBE)
EHOSTUNREACH?
evl_net_solicit()
goto redo
otherwise assume ENOMEM
otherwise all done

IOW, using a proper combination of MSG_DONTWAIT and MSG_PROBE in the right sequence would either succeed to send the packet immediately on the first oob_sendmsg() call, otherwise fail on memory shortage or missing route from the oob cache. In the latter case, which could only happen once for each new peer under normal circumstances, we could disambiguate the EWOULDBLOCK status using an explicit probe.

A better way to do this in a single step unambiguously would require the addition of another operation flag, like MSG_DONTDEFER, preventing the deferral to inband on failed probe and causing oob_sendmsg() to return with a specific error code. e.g. something as simple as the following would cover all requirements:

Quoted 7 lines
      oob_sendmsg(..., MSG_DONTDEFER) -> EADDRNOTAVAIL on failed probe.

Now, we might also piggyback off of MSG_OOB instead of defining yet another operation flag, as a way to say "oob only, don't relay to inband", but I'm still pondering whether this would be nicely witty or utterly confusing..

Yes, this is a variant. I was not aware that MSG_DONTWAIT -> EWOULDBLOCK can also mean ENOMEM.

Now after considering all options, i start to prefer the variant before this patch, keeping a list of already solicit'ed clients. It is straight forward, simple and fast. And you know exactly when to expect an in band call. Might be we should just drop this patch instead of adding unnecessary complexity for example code?

With MSG_DONTWAIT/MSG_PROBE you are not sure if the ARP entry is permanent. That means if you are unlucky, a few packages are sent to the client before EHOSTUNREACH is returned which could be bad for real time.
If you have a real time server and clients, I imagine clients would expect the first response to be delayed due to initialization / ARP but every following response to be in time.

With ioctl(SIOCGARP) or evl_net_routeinfo(), a list of checked client's would also be needed due to it would be wasteful to call the kernel each time before sending a package.

MSG_DONTWAIT / MSG_PROBE could still be useful in certain applications, for example when you have a separate thread that handles evl_net_solicit(), so you can just call oob_sendmsg() until it succeeds without needing to signal back.

This would be something like this:

server_thread:

    if(!is_in_list(addr))
        write_solicit_queue(addr)
        add_to_list(addr)
    oob_sendmsg(..., MSG_DONTWAIT)
         EWOULDBLOCK?
               oob_sendmsg(..., MSG_PROBE)
                  EHOSTUNREACH?
                      skip or mark for retry depending on the application
                  otherwise assume ENOMEM -> generate error
         otherwise all done

solicit_thread:
while(true)
addr=read_solicit_queue()
evl_net_solicit(addr)

I also prefer using an explicit MSG_PROBE instead of an implicit zero size message.

For ioctl(SIOCGARP) or evl_net_routeinfo(), I don't see a good use-case after all. You can just use evl_net_solicit(), it probably won't hurt if you do it once to much and you have to implement the case when evl_net_solicit() is needed anyway and take the timing for this into account.

A message that is not in the archive (sent off-list or not yet received): [email protected]

A message that is not in the archive (sent off-list or not yet received): [email protected]

A message that is not in the archive (sent off-list or not yet received): [email protected]

#18 lore raw

Am 22.07.26 um 20:36 schrieb Philippe Gerum:

Quoted 119 lines
Quoted 80 lines · Philippe Gerum
Quoted 73 lines · Hannes Diethelm

Am 22.07.26 um 00:36 schrieb Hannes Diethelm:

Quoted 62 lines

Am 21.07.26 um 22:15 schrieb Philippe Gerum:

Quoted 15 lines
Quoted 6 lines · Hannes Diethelm

Am 20.07.26 um 16:27 schrieb Philippe Gerum:

An option would be either to support of oob_ioctl() for SIOCGARP. Or create something like
evl_net_routeinfo(s, addr) returning flags would make probing obsolete.

Not entirely. The issue with solely having SIOCGARP or any probe-only explicit request is that you would have to pair two syscalls at each transmit at least, one to probe for the sender address before possibly soliciting that peer if absent from the oob cache, another one for sending the message eventually. i.e., for every packet:

Quoted 3 lines

     ioctl(SIOCGARP)
          !ATF_COM? -> evl_net_solicit()
     oob_sendmsg(..., 0)

You are right, that is an unneeded syscall as long as you don't keep a
list of clients as before. This can remove the need from calling evl_net_solicit() in the case the client is already in ARP and permanent but
this will be the only change and won't help that much.

Quoted 29 lines

OTOH, with the latest attempt to address this issue, we can send a probe-only request by passing MSG_PROBE (EHOSTUNREACH), or a request that does not attempt to defer transmit to the inband stage on probe failure by passing MSG_DONTWAIT (EWOULDBLOCK). i.e., for every packet

redo:
     oob_sendmsg(..., MSG_DONTWAIT)
          EWOULDBLOCK?
                oob_sendmsg(..., MSG_PROBE)
                   EHOSTUNREACH?
                       evl_net_solicit()
                       goto redo
                   otherwise assume ENOMEM
          otherwise all done

IOW, using a proper combination of MSG_DONTWAIT and MSG_PROBE in the right sequence would either succeed to send the packet immediately on the first oob_sendmsg() call, otherwise fail on memory shortage or missing route from the oob cache. In the latter case, which could only happen once for each new peer under normal circumstances, we could disambiguate the EWOULDBLOCK status using an explicit probe.

A better way to do this in a single step unambiguously would require the addition of another operation flag, like MSG_DONTDEFER, preventing the deferral to inband on failed probe and causing oob_sendmsg() to return with a specific error code. e.g. something as simple as the following would cover all requirements:

Quoted 7 lines

      oob_sendmsg(..., MSG_DONTDEFER) -> EADDRNOTAVAIL on failed probe.

Now, we might also piggyback off of MSG_OOB instead of defining yet another operation flag, as a way to say "oob only, don't relay to inband", but I'm still pondering whether this would be nicely witty or utterly confusing..

Quoted 9 lines

Yes, this is a variant. I was not aware that MSG_DONTWAIT -> EWOULDBLOCK
can also mean ENOMEM.
Now after considering all options, i start to prefer the variant before
this patch, keeping a list of already solicit'ed clients. It is straight forward, simple and fast. And you know exactly when to expect an in band call. Might be we should just drop this patch instead of adding unnecessary complexity for example code?

Quoted 4 lines

BTW: With drop this patch I don't mean the one already in but this one:
[PATCH] tidbits: net-udp: use MSG_DONTWAIT for server mode
https://lore.kernel.org/xenomai/[email protected]/T/#u

Quoted 4 lines

Got it, it may be worth waiting for the dust to settle in the core regarding the probing issue before adapting this tidbit anyway. I'm going to consolidate the points we have discussed so far, in order to come up with an implementation that satisfies all the requirements.

Here is a new proposal that should cover all the cases we discussed so far (next/v6.12.y-cip-evl-rebase branch). A new flag is introduced, namely MSG_STEADY, which tells the core to only accept cache lookups that deliver permanent ARP records. This flag can be combined with MSG_PROBE and MSG_DONTWAIT for extended semantics. Meanwhile, a zero-sized message is still accepted in order to request a mere address probe without having to pass any meaningful data. This goes like this:

Function: probe-only

	      oob_sendmsg(s, &msghdr, NULL, MSG_PROBE)
	          -> EADDRNOTAVAIL if no peer address found

Function: probe-only, permanent peer address required

oob_sendmsg(s, &msghdr, NULL, MSG_PROBE|MSG_STEADY)

	          -> EADDRNOTAVAIL if no permanent peer address found
                      (also upon match of aging ARP record)

Function: send, no deferral (to inband) allowed

	      oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT)
	          -> EADDRNOTAVAIL if no peer address found
	          -> EWOULDBLOCK on memory shortage

Function: send, no deferral (to inband) allowed, permanent peer address required

	      oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT|MSG_STEADY)
	          -> EADDRNOTAVAIL if no permanent peer address found
                      (also upon match of aging ARP record)
	          -> EWOULDBLOCK on memory shortage
Quoted 6 lines

Function: send, permanent peer address required

oob_sendmsg(s, &msghdr, NULL, MSG_STEADY)
-> EADDRNOTAVAIL if no permanent peer address found
(also upon match of aging ARP record)

Looks promising. This also means after oob_sendmsg(..., MSG_DONTWAIT|MSG_STEADY)
no second call oob_sendmsg(..., MSG_PROBE) is needed to handle -ENOMEM right?

oob_sendmsg(s, &msghdr, NULL, 0) is the only case with deferral to inband as it looks in udp.c:421...423?
if (ret != -EADDRNOTAVAIL ||

	msg_flags & (MSG_PROBE|MSG_DONTWAIT|MSG_STEADY))
		return ret;

I have to test it and will send an update to the oob_net_udp patch soon.

BTW: CC [email protected] got lost in my last email. I added it again.

#19 lore raw
Quoted 118 lines · Hannes Diethelm

Am 22.07.26 um 20:36 schrieb Philippe Gerum:

Quoted 111 lines
Quoted 80 lines · Philippe Gerum
Quoted 73 lines · Hannes Diethelm

Am 22.07.26 um 00:36 schrieb Hannes Diethelm:

Quoted 62 lines

Am 21.07.26 um 22:15 schrieb Philippe Gerum:

Quoted 15 lines
Quoted 6 lines · Hannes Diethelm

Am 20.07.26 um 16:27 schrieb Philippe Gerum:

An option would be either to support of oob_ioctl() for SIOCGARP. Or create something like
evl_net_routeinfo(s, addr) returning flags would make probing obsolete.

Not entirely. The issue with solely having SIOCGARP or any probe-only explicit request is that you would have to pair two syscalls at each transmit at least, one to probe for the sender address before possibly soliciting that peer if absent from the oob cache, another one for sending the message eventually. i.e., for every packet:

Quoted 3 lines

     ioctl(SIOCGARP)
          !ATF_COM? -> evl_net_solicit()
     oob_sendmsg(..., 0)

You are right, that is an unneeded syscall as long as you don't keep a
list of clients as before. This can remove the need from calling evl_net_solicit() in the case the client is already in ARP and permanent but
this will be the only change and won't help that much.

Quoted 29 lines

OTOH, with the latest attempt to address this issue, we can send a probe-only request by passing MSG_PROBE (EHOSTUNREACH), or a request that does not attempt to defer transmit to the inband stage on probe failure by passing MSG_DONTWAIT (EWOULDBLOCK). i.e., for every packet

redo:
     oob_sendmsg(..., MSG_DONTWAIT)
          EWOULDBLOCK?
                oob_sendmsg(..., MSG_PROBE)
                   EHOSTUNREACH?
                       evl_net_solicit()
                       goto redo
                   otherwise assume ENOMEM
          otherwise all done

IOW, using a proper combination of MSG_DONTWAIT and MSG_PROBE in the right sequence would either succeed to send the packet immediately on the first oob_sendmsg() call, otherwise fail on memory shortage or missing route from the oob cache. In the latter case, which could only happen once for each new peer under normal circumstances, we could disambiguate the EWOULDBLOCK status using an explicit probe.

A better way to do this in a single step unambiguously would require the addition of another operation flag, like MSG_DONTDEFER, preventing the deferral to inband on failed probe and causing oob_sendmsg() to return with a specific error code. e.g. something as simple as the following would cover all requirements:

Quoted 7 lines

      oob_sendmsg(..., MSG_DONTDEFER) -> EADDRNOTAVAIL on failed probe.

Now, we might also piggyback off of MSG_OOB instead of defining yet another operation flag, as a way to say "oob only, don't relay to inband", but I'm still pondering whether this would be nicely witty or utterly confusing..

Quoted 9 lines

Yes, this is a variant. I was not aware that MSG_DONTWAIT -> EWOULDBLOCK
can also mean ENOMEM.
Now after considering all options, i start to prefer the variant before
this patch, keeping a list of already solicit'ed clients. It is straight forward, simple and fast. And you know exactly when to expect an in band call. Might be we should just drop this patch instead of adding unnecessary complexity for example code?

Quoted 4 lines

BTW: With drop this patch I don't mean the one already in but this one:
[PATCH] tidbits: net-udp: use MSG_DONTWAIT for server mode
https://lore.kernel.org/xenomai/[email protected]/T/#u

Quoted 4 lines

Got it, it may be worth waiting for the dust to settle in the core regarding the probing issue before adapting this tidbit anyway. I'm going to consolidate the points we have discussed so far, in order to come up with an implementation that satisfies all the requirements.

Here is a new proposal that should cover all the cases we discussed so
far (next/v6.12.y-cip-evl-rebase branch). A new flag is introduced, namely MSG_STEADY, which tells the core to only accept cache lookups that deliver permanent ARP records. This flag can be combined with MSG_PROBE and MSG_DONTWAIT for extended semantics. Meanwhile, a zero-sized message is still accepted in order to request a mere address probe without having to pass any meaningful data. This goes like this:
Function: probe-only

	      oob_sendmsg(s, &msghdr, NULL, MSG_PROBE)
	          -> EADDRNOTAVAIL if no peer address found
Function: probe-only, permanent peer address required
   	      oob_sendmsg(s, &msghdr, NULL, MSG_PROBE|MSG_STEADY)
	          -> EADDRNOTAVAIL if no permanent peer address found
                      (also upon match of aging ARP record)
Function: send, no deferral (to inband) allowed
	      oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT)
	          -> EADDRNOTAVAIL if no peer address found
	          -> EWOULDBLOCK on memory shortage
Function: send, no deferral (to inband) allowed, permanent peer
address required
	      oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT|MSG_STEADY)
	          -> EADDRNOTAVAIL if no permanent peer address found
                      (also upon match of aging ARP record)
	          -> EWOULDBLOCK on memory shortage
Quoted 5 lines

Function: send, permanent peer address required
oob_sendmsg(s, &msghdr, NULL, MSG_STEADY)
-> EADDRNOTAVAIL if no permanent peer address found
(also upon match of aging ARP record)

Quoted 4 lines

Looks promising. This also means after oob_sendmsg(..., MSG_DONTWAIT|MSG_STEADY)
no second call oob_sendmsg(..., MSG_PROBE) is needed to handle -ENOMEM right?

Correct.

Quoted 8 lines

oob_sendmsg(s, &msghdr, NULL, 0) is the only case with deferral to inband as it looks in udp.c:421...423?
if (ret != -EADDRNOTAVAIL ||

	msg_flags & (MSG_PROBE|MSG_DONTWAIT|MSG_STEADY))
		return ret;

Open question: does it still make sense to have MSG_DONTWAIT prevent the deferral to the inband stage, now that MSG_STEADY can be used for this? i.e.:

Quoted 4 lines
Quoted 4 lines

Function: send, no deferral (to inband) allowed

	      oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT)
	          -> EADDRNOTAVAIL if no peer address found
	          -> EWOULDBLOCK on memory shortage

Not considering MSG_DONTWAIT (alone) anymore as a way to prevent deferral would make MSG_STEADY the only option for this, tightening the requirement from "any cached destination address" to "permanent destination address". This being said:

- If $user wants a sure shot wrt to oob transmission, then a permanent address is most likely desired, in which case MSG_STEADY should be given. Otherwise, what would be the point of sending to a peer using an aging ARP record _and_ being concerned about timeliness and deterministic behavior in general (therefore denying deferral)?

- As it is, the implementation somewhat abuses the dont-wait semantics. As a matter of fact, inband deferral won't make the caller wait at all, only the packet will be delayed until the regular netstack picks it. Meanwhile, EWOULDBLOCK is conventionally paired with MSG_DONTWAIT, not EADDRNOTAVAIL.

Quoted 2 lines

I have to test it and will send an update to the oob_net_udp patch soon.

Thanks.

Quoted 2 lines

BTW: CC [email protected] got lost in my last email. I added it again.

Signature · 1 line
Philippe.
#20 lore raw

Am 23.07.26 um 09:47 schrieb Philippe Gerum:

Quoted 156 lines
Quoted 118 lines · Hannes Diethelm

Am 22.07.26 um 20:36 schrieb Philippe Gerum:

Quoted 111 lines
Quoted 80 lines · Philippe Gerum
Quoted 73 lines · Hannes Diethelm

Am 22.07.26 um 00:36 schrieb Hannes Diethelm:

Quoted 62 lines

Am 21.07.26 um 22:15 schrieb Philippe Gerum:

Quoted 15 lines
Quoted 6 lines · Hannes Diethelm

Am 20.07.26 um 16:27 schrieb Philippe Gerum:

An option would be either to support of oob_ioctl() for SIOCGARP. Or create something like
evl_net_routeinfo(s, addr) returning flags would make probing obsolete.

Not entirely. The issue with solely having SIOCGARP or any probe-only explicit request is that you would have to pair two syscalls at each transmit at least, one to probe for the sender address before possibly soliciting that peer if absent from the oob cache, another one for sending the message eventually. i.e., for every packet:

Quoted 3 lines

     ioctl(SIOCGARP)
          !ATF_COM? -> evl_net_solicit()
     oob_sendmsg(..., 0)

You are right, that is an unneeded syscall as long as you don't keep a
list of clients as before. This can remove the need from calling evl_net_solicit() in the case the client is already in ARP and permanent but
this will be the only change and won't help that much.

Quoted 29 lines

OTOH, with the latest attempt to address this issue, we can send a probe-only request by passing MSG_PROBE (EHOSTUNREACH), or a request that does not attempt to defer transmit to the inband stage on probe failure by passing MSG_DONTWAIT (EWOULDBLOCK). i.e., for every packet

redo:
     oob_sendmsg(..., MSG_DONTWAIT)
          EWOULDBLOCK?
                oob_sendmsg(..., MSG_PROBE)
                   EHOSTUNREACH?
                       evl_net_solicit()
                       goto redo
                   otherwise assume ENOMEM
          otherwise all done

IOW, using a proper combination of MSG_DONTWAIT and MSG_PROBE in the right sequence would either succeed to send the packet immediately on the first oob_sendmsg() call, otherwise fail on memory shortage or missing route from the oob cache. In the latter case, which could only happen once for each new peer under normal circumstances, we could disambiguate the EWOULDBLOCK status using an explicit probe.

A better way to do this in a single step unambiguously would require the addition of another operation flag, like MSG_DONTDEFER, preventing the deferral to inband on failed probe and causing oob_sendmsg() to return with a specific error code. e.g. something as simple as the following would cover all requirements:

Quoted 7 lines

      oob_sendmsg(..., MSG_DONTDEFER) -> EADDRNOTAVAIL on failed probe.

Now, we might also piggyback off of MSG_OOB instead of defining yet another operation flag, as a way to say "oob only, don't relay to inband", but I'm still pondering whether this would be nicely witty or utterly confusing..

Quoted 9 lines

Yes, this is a variant. I was not aware that MSG_DONTWAIT -> EWOULDBLOCK
can also mean ENOMEM.
Now after considering all options, i start to prefer the variant before
this patch, keeping a list of already solicit'ed clients. It is straight forward, simple and fast. And you know exactly when to expect an in band call. Might be we should just drop this patch instead of adding unnecessary complexity for example code?

Quoted 4 lines

BTW: With drop this patch I don't mean the one already in but this one:
[PATCH] tidbits: net-udp: use MSG_DONTWAIT for server mode
https://lore.kernel.org/xenomai/[email protected]/T/#u

Quoted 4 lines

Got it, it may be worth waiting for the dust to settle in the core regarding the probing issue before adapting this tidbit anyway. I'm going to consolidate the points we have discussed so far, in order to come up with an implementation that satisfies all the requirements.

Here is a new proposal that should cover all the cases we discussed so
far (next/v6.12.y-cip-evl-rebase branch). A new flag is introduced, namely MSG_STEADY, which tells the core to only accept cache lookups that deliver permanent ARP records. This flag can be combined with MSG_PROBE and MSG_DONTWAIT for extended semantics. Meanwhile, a zero-sized message is still accepted in order to request a mere address probe without having to pass any meaningful data. This goes like this:
Function: probe-only

	      oob_sendmsg(s, &msghdr, NULL, MSG_PROBE)
	          -> EADDRNOTAVAIL if no peer address found

Function: probe-only, permanent peer address required
oob_sendmsg(s, &msghdr, NULL, MSG_PROBE|MSG_STEADY)
-> EADDRNOTAVAIL if no permanent peer address found
(also upon match of aging ARP record)
Function: send, no deferral (to inband) allowed

	      oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT)
	          -> EADDRNOTAVAIL if no peer address found
	          -> EWOULDBLOCK on memory shortage
Function: send, no deferral (to inband) allowed, permanent peer
address required
	      oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT|MSG_STEADY)
	          -> EADDRNOTAVAIL if no permanent peer address found
                       (also upon match of aging ARP record)
	          -> EWOULDBLOCK on memory shortage
Quoted 5 lines

Function: send, permanent peer address required
oob_sendmsg(s, &msghdr, NULL, MSG_STEADY)
-> EADDRNOTAVAIL if no permanent peer address found
(also upon match of aging ARP record)

Quoted 4 lines

Looks promising. This also means after oob_sendmsg(..., MSG_DONTWAIT|MSG_STEADY)
no second call oob_sendmsg(..., MSG_PROBE) is needed to handle -ENOMEM right?

Correct.

Quoted 8 lines

oob_sendmsg(s, &msghdr, NULL, 0) is the only case with deferral to inband as it looks in udp.c:421...423?
if (ret != -EADDRNOTAVAIL ||

	msg_flags & (MSG_PROBE|MSG_DONTWAIT|MSG_STEADY))
		return ret;

Open question: does it still make sense to have MSG_DONTWAIT prevent the deferral to the inband stage, now that MSG_STEADY can be used for this? i.e.:

Quoted 4 lines
Quoted 4 lines

Function: send, no deferral (to inband) allowed

	      oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT)
	          -> EADDRNOTAVAIL if no peer address found
	          -> EWOULDBLOCK on memory shortage

Not considering MSG_DONTWAIT (alone) anymore as a way to prevent deferral would make MSG_STEADY the only option for this, tightening the requirement from "any cached destination address" to "permanent destination address". This being said:

- If $user wants a sure shot wrt to oob transmission, then a permanent address is most likely desired, in which case MSG_STEADY should be given. Otherwise, what would be the point of sending to a peer using an aging ARP record _and_ being concerned about timeliness and deterministic behavior in general (therefore denying deferral)?

Quoted 6 lines

- As it is, the implementation somewhat abuses the dont-wait semantics. As a matter of fact, inband deferral won't make the caller wait at all, only the packet will be delayed until the regular netstack picks it. Meanwhile, EWOULDBLOCK is conventionally paired with MSG_DONTWAIT, not EADDRNOTAVAIL.

MSG_DONTWAIT denying inband was only introduced in 644d55e78e8e, right?

In this case I agree. Either you'd like to stay out of band now and in the future -> MSG_STEADY or you don't care -> 0.

Otherwise, it might break already existing applications.

Quoted 8 lines
Quoted 2 lines

I have to test it and will send an update to the oob_net_udp patch soon.

Thanks.

Quoted 2 lines

BTW: CC [email protected] got lost in my last email. I added it again.

#21 lore raw

PATCH 1/1 tidbits: net-udp: use MSG_STABLE for server mode

MSG_STABLE is now supported, so it is used to check if a client is already in ARP cache and permanent. If not, use evl_net_solicit() and try again.

---
 tidbits/oob-net-udp.c | 72 +++++++++++++++++++------------------------
 1 file changed, 32 insertions(+), 40 deletions(-)
tidbits/oob-net-udp.c +32 -40
index 658b463..931c4dc 100644
@@ -238,9 +238,6 @@ static void server(int s, const char *text, int mcount,
 	ssize_t ret;
 	char *tbuf;
 	char rbuf[16384];
-	in_addr_t addr_list[SERVER_ADDR_LIST_SIZE] = {};
-	size_t addr_list_fill = 0;
-	bool solicit_done;
 
 	tlen = (strlen(text) + 1) * mcount;
 	tbuf = malloc(tlen);
@@ -281,39 +278,6 @@ static void server(int s, const char *text, int mcount,
 			evl_printf(" (TRUNCATED)");
 		evl_printf(": %.*s\n", (int)ret, rbuf);
 
-		/*
-		 * We need to call evl_net_solicit for each new client
-		 * once before sending data. This demotes the caller to the
-		 * in-band stage for the first response.
-		 * If this is not done and the ARP address is not
-		 * yet in cache or garbage-collected, oob_sendmsg
-		 * will return EINPROGRESS on start or during runtime.
-		 */
-		solicit_done = false;
-		for (size_t i = 0; i < addr_list_fill && !solicit_done; i++) {
-			if (addr_list[i] == _addr.sin_addr.s_addr) {
-				solicit_done = true;
-			}
-		}
-		if (!solicit_done) {
-			if (verbosity) {
-				char ip_str[INET_ADDRSTRLEN+1];
-				inet_ntop(AF_INET, &(_addr.sin_addr), ip_str, sizeof(ip_str));
-				evl_printf("== client %s first seen: evl_net_solicit\n", ip_str);
-			}
-			ret = evl_net_solicit(s, (const struct sockaddr *)&_addr,
-					EVL_NEIGH_PERMANENT);
-			if (ret)
-				error(1, -ret, "evl_net_solicit()");
-
-			if (addr_list_fill < SERVER_ADDR_LIST_SIZE) {
-				addr_list[addr_list_fill] = _addr.sin_addr.s_addr;
-				addr_list_fill++;
-			} else {
-				error(1, EPERM, "address list full");
-			}
-		}
-
 		iov.iov_base = tbuf;
 		iov.iov_len = tlen;
 		msghdr.msg_iov = &iov;
@@ -323,9 +287,37 @@ static void server(int s, const char *text, int mcount,
 		msghdr.msg_name = &_addr;
 		msghdr.msg_namelen = sizeof(_addr);
 		msghdr.msg_flags = 0;
-		ret = oob_sendmsg(s, &msghdr, NULL, 0);
-		if (ret < 0)
-			error(1, errno, "oob_sendmsg() failed");
+		ret = oob_sendmsg(s, &msghdr, NULL, MSG_STEADY);
+		if (ret < 0) {
+			if (errno == EADDRNOTAVAIL) {
+				/*
+				 * oob_sendmsg with MSG_STEADY returns EADDRNOTAVAIL if
+				 * the MAC address is not yet in ARP cache or not permanent.
+				 * We need to call evl_net_solicit in this case.
+				 * This demotes the caller to the in-band stage for the first response.
+				 * To probe if the MAC address is in the ARP cache without sending
+				 * any data, oob_sendmsg() with msghdr.msg_iov = NULL
+				 * and msghdr.msg_iovlen = 0 can be used.
+				 */
+				if (verbosity) {
+					char ip_str[INET_ADDRSTRLEN + 1];
+					inet_ntop(AF_INET, &(_addr.sin_addr), ip_str, sizeof(ip_str));
+					evl_printf("== client %s not in ARP cache: evl_net_solicit\n",
+						ip_str);
+				}
+				ret = evl_net_solicit(s, (const struct sockaddr *)&_addr,
+					EVL_NEIGH_PERMANENT);
+				if (ret)
+					error(1, -ret, "evl_net_solicit()");
+
+				/* oob_sendmsg again, now it should succeed */
+				ret = oob_sendmsg(s, &msghdr, NULL, MSG_STEADY);
+				if (ret < 0)
+					error(1, errno, "oob_sendmsg() failed");
+			} else {
+				error(1, errno, "oob_sendmsg() failed");
+			}
+		}
 		if (verbosity > 1)
 			print_addr("sent to", &_addr);
 	}
@@ -350,7 +342,7 @@ int main(int argc, char *argv[])
 	struct sockaddr_in addr;
 	const char *ip = NULL;
 	ssize_t ret;
-	useconds_t delay=1000000;
+	useconds_t delay = 1000000;
 
 	while ((c = getopt(argc, argv, "a:m:n:i:w:I:p:dsTRCSb")) != EOF) {
 		switch (c) {
Signature · 1 line
2.47.3
#22 lore raw

PATCH v2 1/1 tidbits: net-udp: use MSG_STEADY for server mode

MSG_STEADY is now supported, so it is used to check if a client is already in ARP cache and permanent. If not, use evl_net_solicit() and try again.

---
 tidbits/oob-net-udp.c | 71 +++++++++++++++++++------------------------
 1 file changed, 31 insertions(+), 40 deletions(-)
tidbits/oob-net-udp.c +31 -40
index 658b463..475e26a 100644
@@ -238,9 +238,6 @@ static void server(int s, const char *text, int mcount,
 	ssize_t ret;
 	char *tbuf;
 	char rbuf[16384];
-	in_addr_t addr_list[SERVER_ADDR_LIST_SIZE] = {};
-	size_t addr_list_fill = 0;
-	bool solicit_done;
 
 	tlen = (strlen(text) + 1) * mcount;
 	tbuf = malloc(tlen);
@@ -281,39 +278,6 @@ static void server(int s, const char *text, int mcount,
 			evl_printf(" (TRUNCATED)");
 		evl_printf(": %.*s\n", (int)ret, rbuf);
 
-		/*
-		 * We need to call evl_net_solicit for each new client
-		 * once before sending data. This demotes the caller to the
-		 * in-band stage for the first response.
-		 * If this is not done and the ARP address is not
-		 * yet in cache or garbage-collected, oob_sendmsg
-		 * will return EINPROGRESS on start or during runtime.
-		 */
-		solicit_done = false;
-		for (size_t i = 0; i < addr_list_fill && !solicit_done; i++) {
-			if (addr_list[i] == _addr.sin_addr.s_addr) {
-				solicit_done = true;
-			}
-		}
-		if (!solicit_done) {
-			if (verbosity) {
-				char ip_str[INET_ADDRSTRLEN+1];
-				inet_ntop(AF_INET, &(_addr.sin_addr), ip_str, sizeof(ip_str));
-				evl_printf("== client %s first seen: evl_net_solicit\n", ip_str);
-			}
-			ret = evl_net_solicit(s, (const struct sockaddr *)&_addr,
-					EVL_NEIGH_PERMANENT);
-			if (ret)
-				error(1, -ret, "evl_net_solicit()");
-
-			if (addr_list_fill < SERVER_ADDR_LIST_SIZE) {
-				addr_list[addr_list_fill] = _addr.sin_addr.s_addr;
-				addr_list_fill++;
-			} else {
-				error(1, EPERM, "address list full");
-			}
-		}
-
 		iov.iov_base = tbuf;
 		iov.iov_len = tlen;
 		msghdr.msg_iov = &iov;
@@ -323,9 +287,36 @@ static void server(int s, const char *text, int mcount,
 		msghdr.msg_name = &_addr;
 		msghdr.msg_namelen = sizeof(_addr);
 		msghdr.msg_flags = 0;
-		ret = oob_sendmsg(s, &msghdr, NULL, 0);
-		if (ret < 0)
-			error(1, errno, "oob_sendmsg() failed");
+		ret = oob_sendmsg(s, &msghdr, NULL, MSG_STEADY);
+		if (ret < 0) {
+			if (errno == EADDRNOTAVAIL) {
+				/*
+				 * oob_sendmsg with MSG_STEADY returns EADDRNOTAVAIL if
+				 * the MAC address is not yet in ARP cache or not permanent.
+				 * We need to call evl_net_solicit in this case.
+				 * This demotes the caller to the in-band stage for the first response.
+				 * To probe if the MAC address is in the ARP cache without sending
+				 * any data, oob_sendmsg(..., MSG_STEADY|MSG_PROBE) can be used.
+				 */
+				if (verbosity) {
+					char ip_str[INET_ADDRSTRLEN + 1];
+					inet_ntop(AF_INET, &(_addr.sin_addr), ip_str, sizeof(ip_str));
+					evl_printf("== client %s not in ARP cache: evl_net_solicit\n",
+						ip_str);
+				}
+				ret = evl_net_solicit(s, (const struct sockaddr *)&_addr,
+					EVL_NEIGH_PERMANENT);
+				if (ret)
+					error(1, -ret, "evl_net_solicit()");
+
+				/* oob_sendmsg again, now it should succeed */
+				ret = oob_sendmsg(s, &msghdr, NULL, MSG_STEADY);
+				if (ret < 0)
+					error(1, errno, "oob_sendmsg() failed");
+			} else {
+				error(1, errno, "oob_sendmsg() failed");
+			}
+		}
 		if (verbosity > 1)
 			print_addr("sent to", &_addr);
 	}
@@ -350,7 +341,7 @@ int main(int argc, char *argv[])
 	struct sockaddr_in addr;
 	const char *ip = NULL;
 	ssize_t ret;
-	useconds_t delay=1000000;
+	useconds_t delay = 1000000;
 
 	while ((c = getopt(argc, argv, "a:m:n:i:w:I:p:dsTRCSb")) != EOF) {
 		switch (c) {
Signature · 1 line
2.47.3
#23 lore raw
In reply to Hannes Diethelm
Quoted 161 lines · Hannes Diethelm

Am 23.07.26 um 09:47 schrieb Philippe Gerum:

Quoted 143 lines
Quoted 118 lines · Hannes Diethelm

Am 22.07.26 um 20:36 schrieb Philippe Gerum:

Quoted 111 lines
Quoted 80 lines · Philippe Gerum
Quoted 73 lines · Hannes Diethelm

Am 22.07.26 um 00:36 schrieb Hannes Diethelm:

Quoted 62 lines

Am 21.07.26 um 22:15 schrieb Philippe Gerum:

Quoted 15 lines
Quoted 6 lines · Hannes Diethelm

Am 20.07.26 um 16:27 schrieb Philippe Gerum:

An option would be either to support of oob_ioctl() for SIOCGARP. Or create something like
evl_net_routeinfo(s, addr) returning flags would make probing obsolete.

Not entirely. The issue with solely having SIOCGARP or any probe-only explicit request is that you would have to pair two syscalls at each transmit at least, one to probe for the sender address before possibly soliciting that peer if absent from the oob cache, another one for sending the message eventually. i.e., for every packet:

Quoted 3 lines

     ioctl(SIOCGARP)
          !ATF_COM? -> evl_net_solicit()
     oob_sendmsg(..., 0)

You are right, that is an unneeded syscall as long as you don't keep a
list of clients as before. This can remove the need from calling evl_net_solicit() in the case the client is already in ARP and permanent but
this will be the only change and won't help that much.

Quoted 29 lines

OTOH, with the latest attempt to address this issue, we can send a probe-only request by passing MSG_PROBE (EHOSTUNREACH), or a request that does not attempt to defer transmit to the inband stage on probe failure by passing MSG_DONTWAIT (EWOULDBLOCK). i.e., for every packet

redo:
     oob_sendmsg(..., MSG_DONTWAIT)
          EWOULDBLOCK?
                oob_sendmsg(..., MSG_PROBE)
                   EHOSTUNREACH?
                       evl_net_solicit()
                       goto redo
                   otherwise assume ENOMEM
          otherwise all done

IOW, using a proper combination of MSG_DONTWAIT and MSG_PROBE in the right sequence would either succeed to send the packet immediately on the first oob_sendmsg() call, otherwise fail on memory shortage or missing route from the oob cache. In the latter case, which could only happen once for each new peer under normal circumstances, we could disambiguate the EWOULDBLOCK status using an explicit probe.

A better way to do this in a single step unambiguously would require the addition of another operation flag, like MSG_DONTDEFER, preventing the deferral to inband on failed probe and causing oob_sendmsg() to return with a specific error code. e.g. something as simple as the following would cover all requirements:

Quoted 7 lines

      oob_sendmsg(..., MSG_DONTDEFER) -> EADDRNOTAVAIL on failed probe.

Now, we might also piggyback off of MSG_OOB instead of defining yet another operation flag, as a way to say "oob only, don't relay to inband", but I'm still pondering whether this would be nicely witty or utterly confusing..

Quoted 9 lines

Yes, this is a variant. I was not aware that MSG_DONTWAIT -> EWOULDBLOCK
can also mean ENOMEM.
Now after considering all options, i start to prefer the variant before
this patch, keeping a list of already solicit'ed clients. It is straight forward, simple and fast. And you know exactly when to expect an in band call. Might be we should just drop this patch instead of adding unnecessary complexity for example code?

Quoted 4 lines

BTW: With drop this patch I don't mean the one already in but this one:
[PATCH] tidbits: net-udp: use MSG_DONTWAIT for server mode
https://lore.kernel.org/xenomai/[email protected]/T/#u

Quoted 4 lines

Got it, it may be worth waiting for the dust to settle in the core regarding the probing issue before adapting this tidbit anyway. I'm going to consolidate the points we have discussed so far, in order to come up with an implementation that satisfies all the requirements.

Here is a new proposal that should cover all the cases we discussed so
far (next/v6.12.y-cip-evl-rebase branch). A new flag is introduced, namely MSG_STEADY, which tells the core to only accept cache lookups that deliver permanent ARP records. This flag can be combined with MSG_PROBE and MSG_DONTWAIT for extended semantics. Meanwhile, a zero-sized message is still accepted in order to request a mere address probe without having to pass any meaningful data. This goes like this:
Function: probe-only

	      oob_sendmsg(s, &msghdr, NULL, MSG_PROBE)
	          -> EADDRNOTAVAIL if no peer address found

Function: probe-only, permanent peer address required
oob_sendmsg(s, &msghdr, NULL, MSG_PROBE|MSG_STEADY)
-> EADDRNOTAVAIL if no permanent peer address found
(also upon match of aging ARP record)
Function: send, no deferral (to inband) allowed

	      oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT)
	          -> EADDRNOTAVAIL if no peer address found
	          -> EWOULDBLOCK on memory shortage
Function: send, no deferral (to inband) allowed, permanent peer
address required
	      oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT|MSG_STEADY)
	          -> EADDRNOTAVAIL if no permanent peer address found
                       (also upon match of aging ARP record)
	          -> EWOULDBLOCK on memory shortage
Quoted 5 lines

Function: send, permanent peer address required
oob_sendmsg(s, &msghdr, NULL, MSG_STEADY)
-> EADDRNOTAVAIL if no permanent peer address found
(also upon match of aging ARP record)

Quoted 4 lines

Looks promising. This also means after oob_sendmsg(..., MSG_DONTWAIT|MSG_STEADY)
no second call oob_sendmsg(..., MSG_PROBE) is needed to handle -ENOMEM right?

Correct.

Quoted 8 lines

oob_sendmsg(s, &msghdr, NULL, 0) is the only case with deferral to inband as it looks in udp.c:421...423?
if (ret != -EADDRNOTAVAIL ||

	msg_flags & (MSG_PROBE|MSG_DONTWAIT|MSG_STEADY))
		return ret;

Open question: does it still make sense to have MSG_DONTWAIT prevent the
deferral to the inband stage, now that MSG_STEADY can be used for this? i.e.:

Quoted 4 lines
Quoted 4 lines

Function: send, no deferral (to inband) allowed

	      oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT)
	          -> EADDRNOTAVAIL if no peer address found
	          -> EWOULDBLOCK on memory shortage
Quoted 16 lines

Not considering MSG_DONTWAIT (alone) anymore as a way to prevent deferral would make MSG_STEADY the only option for this, tightening the requirement from "any cached destination address" to "permanent destination address". This being said:
- If $user wants a sure shot wrt to oob transmission, then a permanent
address is most likely desired, in which case MSG_STEADY should be given. Otherwise, what would be the point of sending to a peer using an aging ARP record _and_ being concerned about timeliness and deterministic behavior in general (therefore denying deferral)?
- As it is, the implementation somewhat abuses the dont-wait semantics. As a matter of fact, inband deferral won't make the caller wait at all, only the packet will be delayed until the regular netstack picks it. Meanwhile, EWOULDBLOCK is conventionally paired with MSG_DONTWAIT, not EADDRNOTAVAIL.

Quoted 2 lines

MSG_DONTWAIT denying inband was only introduced in 644d55e78e8e, right?

Yes, this can still be considered as a tentative, undocumented so far.

Quoted 4 lines

In this case I agree. Either you'd like to stay out of band now and in the future -> MSG_STEADY or you don't care -> 0.

Otherwise, it might break already existing applications.

Signature · 1 line
Philippe.
#24 lore raw
In reply to Philippe Gerum

Am 23.07.26 um 09:47 schrieb Philippe Gerum:

Quoted 5 lines
Quoted 2 lines

I have to test it and will send an update to the oob_net_udp patch soon.

Thanks.

So, I sent the patch:
[PATCH] tidbits: net-udp: use MSG_STABLE for server mode

However, something is not working as I would expect:
The second oob_sendmsg() line 314 also returns EADDRNOTAVAIL after evl_net_solicit().

If I leave MSG_STEADY away for the second oob_sendmsg(), it works. However, I see 2x:
== client 192.168.122.1 not in ARP cache: evl_net_solicit From then on, it works forever, also if I restart oob-net-udp.

Looks like the NUD_PERMANENT is not synchronized immediately but only after the second evl_net_solicit(), shown by this test:

arp -i enp7s0 -d 192.168.122.1

oob-net-udp -d -i enp7s0 -a 0.0.0.0 -S -p 5201 -m "Server"
...
== client 192.168.122.1 not in ARP cache: evl_net_solicit ./oob-net-udp: oob_sendmsg() failed: Cannot assign requested address

oob-net-udp -d -i enp7s0 -a 0.0.0.0 -S -p 5201 -m "Server"
...
== client 192.168.122.1 not in ARP cache: evl_net_solicit ... now it works

oob-net-udp -d -i enp7s0 -a 0.0.0.0 -S -p 5201 -m "Server"
...no evl_net_solicit() any more, still works

linux-evl: 9ad9bf1c
libevl:    115d45af
#25 lore raw

PATCH v2 1/1 tidbits: net-udp: solicit new client for server mode

In reply to Philippe Gerum

This fixes random EINPROGRESS at init or during runtime.

Also correct whitespaces and make stdout more consistent.

---
 tidbits/oob-net-udp.c | 58 +++++++++++++++++++++++++++++++++++++------
 1 file changed, 50 insertions(+), 8 deletions(-)
tidbits/oob-net-udp.c +50 -8
index 11b8459..658b463 100644
@@ -50,12 +50,12 @@ static void usage(void)
 }
 
 static void print_addr(char* text, struct sockaddr_in *addr){
-    char ip_str[INET_ADDRSTRLEN+1];
-    inet_ntop(AF_INET, &(addr->sin_addr), ip_str, sizeof(ip_str));
-    evl_printf("%s--------\n", text);
-    evl_printf("IP-Address: %s\n", ip_str);
-    evl_printf("Port:       %d\n", ntohs(addr->sin_port));
-    evl_printf("Family:     %d\n", addr->sin_family);
+	char ip_str[INET_ADDRSTRLEN+1];
+	inet_ntop(AF_INET, &(addr->sin_addr), ip_str, sizeof(ip_str));
+	evl_printf("== %s\n", text);
+	evl_printf("   ip-address: %s\n", ip_str);
+	evl_printf("   port:       %d\n", ntohs(addr->sin_port));
+	evl_printf("   family:     %d\n", addr->sin_family);
 }
 
 static void sender(int s, const char *text, int mcount,
@@ -226,6 +226,8 @@ static void client(int s, const char *text, int mcount,
 	free(tbuf);
 }
 
+#define SERVER_ADDR_LIST_SIZE 64
+
 static void server(int s, const char *text, int mcount,
 		struct sockaddr_in *addr, int iter)
 {
@@ -236,6 +238,9 @@ static void server(int s, const char *text, int mcount,
 	ssize_t ret;
 	char *tbuf;
 	char rbuf[16384];
+	in_addr_t addr_list[SERVER_ADDR_LIST_SIZE] = {};
+	size_t addr_list_fill = 0;
+	bool solicit_done;
 
 	tlen = (strlen(text) + 1) * mcount;
 	tbuf = malloc(tlen);
@@ -276,6 +281,39 @@ static void server(int s, const char *text, int mcount,
 			evl_printf(" (TRUNCATED)");
 		evl_printf(": %.*s\n", (int)ret, rbuf);
 
+		/*
+		 * We need to call evl_net_solicit for each new client
+		 * once before sending data. This demotes the caller to the
+		 * in-band stage for the first response.
+		 * If this is not done and the ARP address is not
+		 * yet in cache or garbage-collected, oob_sendmsg
+		 * will return EINPROGRESS on start or during runtime.
+		 */
+		solicit_done = false;
+		for (size_t i = 0; i < addr_list_fill && !solicit_done; i++) {
+			if (addr_list[i] == _addr.sin_addr.s_addr) {
+				solicit_done = true;
+			}
+		}
+		if (!solicit_done) {
+			if (verbosity) {
+				char ip_str[INET_ADDRSTRLEN+1];
+				inet_ntop(AF_INET, &(_addr.sin_addr), ip_str, sizeof(ip_str));
+				evl_printf("== client %s first seen: evl_net_solicit\n", ip_str);
+			}
+			ret = evl_net_solicit(s, (const struct sockaddr *)&_addr,
+					EVL_NEIGH_PERMANENT);
+			if (ret)
+				error(1, -ret, "evl_net_solicit()");
+
+			if (addr_list_fill < SERVER_ADDR_LIST_SIZE) {
+				addr_list[addr_list_fill] = _addr.sin_addr.s_addr;
+				addr_list_fill++;
+			} else {
+				error(1, EPERM, "address list full");
+			}
+		}
+
 		iov.iov_base = tbuf;
 		iov.iov_len = tlen;
 		msghdr.msg_iov = &iov;
@@ -433,7 +471,7 @@ int main(int argc, char *argv[])
 		receiver(s, &addr, iter);
 	} else if (mode == CLIENT) {
 		if (verbosity)
-			printf("== client mode (<= %s:%d)\n", ip, port);
+			printf("== client mode (<=> %s:%d)\n", ip, port);
 
 		/*
 		 * Guarantee a mere oob path from the first packet
@@ -449,8 +487,12 @@ int main(int argc, char *argv[])
 		client(s, text, mcount, &addr, iter, delay);
 	} else if (mode == SERVER) {
 		if (verbosity)
-			printf("== server mode (<= %s:%d)\n", ip, port);
+			printf("== server mode (<=> %s:%d)\n", ip, port);
 
+		/*
+		 * The server calls evl_net_solicit() internally
+		 * for each new ip address.
+		 */
 		server(s, text, mcount, &addr, iter);
 	} else {
 		error(1, 0, "Mode not implemented");
Signature · 1 line
2.47.3
#26 lore raw
In reply to Hannes Diethelm
Quoted 8 lines · Hannes Diethelm

As discussed in [PATCH] tidbits: net-udp: add server and client mode, evl_net_solicit() is also needed for each client in server mode.

I went for option 2, it is not to lazy and I did not find any way to ask evl if an ARP is already done.

While testing, I also discovered that EINPROGRESS can appear after some time. I think this is due the kernel garbage-collects the ARP entry.

Correct, there is a timeout on ARP entries. You can make an entry permanent and therefore not considered for garbage collection by passing the EVL_NEIGH_PERMANENT flag to evl_net_solicit()

From the command line, the standard 'arp' command can be used to achieve what you would do with evl_net_solicit() as well, since in-band updates to the neighbour table are forwarded to evl.

Signature · 1 line
Philippe.