tidbits: net-udp: solicit new client for server mode
cover tidbits: net-udp: solicit new client for server mode
As discussed in [PATCH] tidbits: net-udp: add server and client mode, evl_net_solicit() is also needed for each client in server mode.
I went for option 2, it is not to lazy and I did not find any way to ask evl if an ARP is already done.
While testing, I also discovered that EINPROGRESS can appear after some time. I think this is due the kernel garbage-collects the ARP entry.
With this patch, this issue is also gone.
Hannes Diethelm (1):
tidbits: net-udp: solicit new client for server mode
tidbits/oob-net-udp.c | 58 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 8 deletions(-)
Signature · 1 line
2.47.3
PATCH 1/1 tidbits: net-udp: solicit new client for server mode
This fixes random EINPROGRESS at init or during runtime.
Also correct whitespaces and make stdout more consistent.
--- tidbits/oob-net-udp.c | 58 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 8 deletions(-)
tidbits/oob-net-udp.c +50 -8
@@ -50,12 +50,12 @@ static void usage(void) }static void print_addr (char * text,struct sockaddr_in *addr){ -char ip_str[INET_ADDRSTRLEN +1 ]; -inet_ntop (AF_INET , &(addr->sin_addr), ip_str,sizeof (ip_str)); -evl_printf ("%s--------\n" , text); -evl_printf ("IP-Address: %s\n" , ip_str); -evl_printf ("Port: %d\n" ,ntohs (addr->sin_port)); -evl_printf ("Family: %d\n" , addr->sin_family); +char ip_str[INET_ADDRSTRLEN +1 ]; +inet_ntop (AF_INET , &(addr->sin_addr), ip_str,sizeof (ip_str)); +evl_printf ("== %s\n" , text); +evl_printf (" ip-address: %s\n" , ip_str); +evl_printf (" port: %d\n" ,ntohs (addr->sin_port)); +evl_printf (" family: %d\n" , addr->sin_family); } static void sender(int s, const char *text, int mcount,
@@ -226,6 +226,8 @@ static void client(int s, const char *text, int mcount,free (tbuf); } +#define SERVER_ADDR_LIST_SIZE 64 +static void server (int s,const char *text,int mcount,struct sockaddr_in *addr,int iter) {
@@ -236,6 +238,9 @@ static void server(int s, const char *text, int mcount,ssize_t ret;char *tbuf;char rbuf[16384 ]; +in_addr_t addr_list[SERVER_ADDR_LIST_SIZE ]={}; +size_t addr_list_fill=0 ; +bool solicit_done; tlen = (strlen (text) +1 ) * mcount; tbuf =malloc (tlen);
@@ -276,6 +281,39 @@ static void server(int s, const char *text, int mcount,evl_printf (" (TRUNCATED)" );evl_printf (": %.*s\n" , (int )ret, rbuf); +/* +* We need to call evl_net_solicit for each new +* client once before sending data. This will break +* realtime for the first response. +* If this is not done and the ARP address is not +* yet in cache or garbage-collected, oob_sendmsg +* will return EINPROGRESS on start or during runtime. +*/ + solicit_done = false; +for (size_t i =0 ; i < addr_list_fill && !solicit_done; i++){ +if (addr_list[i] == _addr.sin_addr.s_addr){ + solicit_done = true; + } + } +if (!solicit_done){ +if (verbosity){ +char ip_str[INET_ADDRSTRLEN +1 ]; +inet_ntop (AF_INET , &(_addr.sin_addr), ip_str,sizeof (ip_str)); +evl_printf ("== client %s first seen: evl_net_solicit\n" , ip_str); + } + ret =evl_net_solicit (s, (const struct sockaddr *)&_addr, +EVL_NEIGH_PERMANENT ); +if (ret) +error (1 , -ret,"evl_net_solicit()" ); + +if (addr_list_fill <SERVER_ADDR_LIST_SIZE ){ + addr_list[addr_list_fill] = _addr.sin_addr.s_addr; + addr_list_fill ++; + }else { +error (1 ,EPERM ,"address list full" ); + } + } + iov.iov_base = tbuf; iov.iov_len = tlen; msghdr.msg_iov = &iov;
@@ -433,7 +471,7 @@ int main(int argc, char *argv[])receiver (s, &addr, iter); }else if (mode ==CLIENT ) {if (verbosity) -printf ("== client mode (<= %s:%d)\n" , ip, port); +printf ("== client mode (<=> %s:%d)\n" , ip, port);/* * Guarantee a mere oob path from the first packet
@@ -449,8 +487,12 @@ int main(int argc, char *argv[])client (s, text, mcount, &addr, iter, delay); }else if (mode ==SERVER ) {if (verbosity) -printf ("== server mode (<= %s:%d)\n" , ip, port); +printf ("== server mode (<=> %s:%d)\n" , ip, port); +/* +* The server calls evl_net_solicit() internally +* for each new ip address. +*/ server (s, text, mcount, &addr, iter); }else {error (1 ,0 ,"Mode not implemented" );
Signature · 1 line
2.47.3
Quoted 42 lines · Hannes Diethelm
This fixes random EINPROGRESS at init or during runtime.
Also correct whitespaces and make stdout more consistent.
Signed-off-by Hannes Diethelm <[email protected]>--- tidbits/oob-net-udp.c | 58 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 8 deletions(-)tidbits/oob-net-udp.c +8
-6@@ -50,12 +50,12 @@ static void usage(void) }static void print_addr (char * text,struct sockaddr_in *addr){ -char ip_str[INET_ADDRSTRLEN +1 ]; -inet_ntop (AF_INET , &(addr->sin_addr), ip_str,sizeof (ip_str)); -evl_printf ("%s--------\n" , text); -evl_printf ("IP-Address: %s\n" , ip_str); -evl_printf ("Port: %d\n" ,ntohs (addr->sin_port)); -evl_printf ("Family: %d\n" , addr->sin_family); +char ip_str[INET_ADDRSTRLEN +1 ]; +inet_ntop (AF_INET , &(addr->sin_addr), ip_str,sizeof (ip_str)); +evl_printf ("== %s\n" , text); +evl_printf (" ip-address: %s\n" , ip_str); +evl_printf (" port: %d\n" ,ntohs (addr->sin_port)); +evl_printf (" family: %d\n" , addr->sin_family); } static void sender(int s, const char *text, int mcount,@@ -226,6 +226,8 @@ static void client(int s, const char *text, int mcount,free (tbuf); } +#define SERVER_ADDR_LIST_SIZE 64 +static void server (int s,const char *text,int mcount,struct sockaddr_in *addr,int iter) {
tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 241
Quoted 4 lines
tidbits/oob-net-udp.c +1
-0@@ -236,6 +238,9 @@ static void server(int s, const char *text, int mcount,ssize_t ret;char *tbuf;char rbuf[16384 ]; +in_addr_t addr_list[SERVER_ADDR_LIST_SIZE ]={};
^ missing whitespaces
Quoted 7 lines
tidbits/oob-net-udp.c +2
-0+size_t addr_list_fill=0 ; +bool solicit_done; tlen = (strlen (text) +1 ) * mcount; tbuf =malloc (tlen);@@ -276,6 +281,39 @@ static void server(int s, const char *text, int mcount,evl_printf (" (TRUNCATED)" );
tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 291
Quoted 10 lines
tidbits/oob-net-udp.c +8
-0evl_printf (": %.*s\n" , (int )ret, rbuf); +/* +* We need to call evl_net_solicit for each new +* client once before sending data. This will break +* realtime for the first response. +* If this is not done and the ARP address is not +* yet in cache or garbage-collected, oob_sendmsg +* will return EINPROGRESS on start or during runtime. +*/
We can happily send redundant solicit requests to the core, no need to filter out cached addresses, evl_net_solicit() will do the right thing.
tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 293
Quoted 2 lines
tidbits/oob-net-udp.c +2
-0+ solicit_done = false; +for (size_t i =0 ; i < addr_list_fill && !solicit_done; i++){
^ missing whitespace
tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 294
Quoted 1 line
tidbits/oob-net-udp.c +1
-0+if (addr_list[i] == _addr.sin_addr.s_addr){
^ missing whitespace
Quoted 7 lines
tidbits/oob-net-udp.c +7
-0+ solicit_done = true; + } + } +if (!solicit_done){ +if (verbosity){ +char ip_str[INET_ADDRSTRLEN +1 ]; +inet_ntop (AF_INET , &(_addr.sin_addr), ip_str,sizeof (ip_str));
tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 311
Quoted 10 lines
tidbits/oob-net-udp.c +10
-0+evl_printf ("== client %s first seen: evl_net_solicit\n" , ip_str); + } + ret =evl_net_solicit (s, (const struct sockaddr *)&_addr, +EVL_NEIGH_PERMANENT ); +if (ret) +error (1 , -ret,"evl_net_solicit()" ); + +if (addr_list_fill <SERVER_ADDR_LIST_SIZE ){ + addr_list[addr_list_fill] = _addr.sin_addr.s_addr; + addr_list_fill ++;
^ extraneous whitespace
Quoted 31 lines
tidbits/oob-net-udp.c +11
-2+ }else { +error (1 ,EPERM ,"address list full" ); + } + } + iov.iov_base = tbuf; iov.iov_len = tlen; msghdr.msg_iov = &iov;@@ -433,7 +471,7 @@ int main(int argc, char *argv[])receiver (s, &addr, iter); }else if (mode ==CLIENT ) {if (verbosity) -printf ("== client mode (<= %s:%d)\n" , ip, port); +printf ("== client mode (<=> %s:%d)\n" , ip, port);/* * Guarantee a mere oob path from the first packet @@ -449,8 +487,12 @@ int main(int argc, char *argv[])client (s, text, mcount, &addr, iter, delay); }else if (mode ==SERVER ) {if (verbosity) -printf ("== server mode (<= %s:%d)\n" , ip, port); +printf ("== server mode (<=> %s:%d)\n" , ip, port); +/* +* The server calls evl_net_solicit() internally +* for each new ip address. +*/ server (s, text, mcount, &addr, iter); }else {error (1 ,0 ,"Mode not implemented" );
Signature · 1 line
Philippe.
Quoted 58 lines · Philippe Gerum
Quoted 42 lines · Hannes Diethelm
This fixes random EINPROGRESS at init or during runtime.
Also correct whitespaces and make stdout more consistent.
Signed-off-by Hannes Diethelm <[email protected]>--- tidbits/oob-net-udp.c | 58 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 8 deletions(-)tidbits/oob-net-udp.c +8
-6@@ -50,12 +50,12 @@ static void usage(void) }static void print_addr (char * text,struct sockaddr_in *addr){ -char ip_str[INET_ADDRSTRLEN +1 ]; -inet_ntop (AF_INET , &(addr->sin_addr), ip_str,sizeof (ip_str)); -evl_printf ("%s--------\n" , text); -evl_printf ("IP-Address: %s\n" , ip_str); -evl_printf ("Port: %d\n" ,ntohs (addr->sin_port)); -evl_printf ("Family: %d\n" , addr->sin_family); +char ip_str[INET_ADDRSTRLEN +1 ]; +inet_ntop (AF_INET , &(addr->sin_addr), ip_str,sizeof (ip_str)); +evl_printf ("== %s\n" , text); +evl_printf (" ip-address: %s\n" , ip_str); +evl_printf (" port: %d\n" ,ntohs (addr->sin_port)); +evl_printf (" family: %d\n" , addr->sin_family); } static void sender(int s, const char *text, int mcount,@@ -226,6 +226,8 @@ static void client(int s, const char *text, int mcount,free (tbuf); } +#define SERVER_ADDR_LIST_SIZE 64 +static void server (int s,const char *text,int mcount,struct sockaddr_in *addr,int iter) {tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 241
Quoted 4 lines
tidbits/oob-net-udp.c +1
-0@@ -236,6 +238,9 @@ static void server(int s, const char *text, int mcount,ssize_t ret;char *tbuf;char rbuf[16384 ]; +in_addr_t addr_list[SERVER_ADDR_LIST_SIZE ]={};^ missing whitespaces
Quoted 9 lines
tidbits/oob-net-udp.c +2
-0+size_t addr_list_fill=0 ; +bool solicit_done; tlen = (strlen (text) +1 ) * mcount; tbuf =malloc (tlen);@@ -276,6 +281,39 @@ static void server(int s, const char *text, int mcount,evl_printf (" (TRUNCATED)" );evl_printf (": %.*s\n" , (int )ret, rbuf);
Quoted 12 lines
tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 291
Quoted 8 lines
tidbits/oob-net-udp.c +8
-0+/* +* We need to call evl_net_solicit for each new +* client once before sending data. This will break +* realtime for the first response. +* If this is not done and the ARP address is not +* yet in cache or garbage-collected, oob_sendmsg +* will return EINPROGRESS on start or during runtime. +*/ We can happily send redundant solicit requests to the core, no need to filter out cached addresses,
evl_net_solicit()will do the right thing.
Except that doing so would always demote the caller to the in-band stage, which may not be what you want. The fact that we'd need to maintain a cache in apps in order to figure out whether solicitation should be done shows a shortcoming in the core, users should not have to do this dance.
We should have an oob call for probing the route+arp caches with ipv4. I'll look into this asap.
Signature · 1 line
Philippe.
Am 01.06.26 um 11:10 schrieb Philippe Gerum:
Quoted 73 lines
Quoted 58 lines · Philippe Gerum
Quoted 42 lines · Hannes Diethelm
This fixes random EINPROGRESS at init or during runtime.
Also correct whitespaces and make stdout more consistent.
Signed-off-by Hannes Diethelm <[email protected]>--- tidbits/oob-net-udp.c | 58 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 8 deletions(-)tidbits/oob-net-udp.c +8
-6@@ -50,12 +50,12 @@ static void usage(void) }static void print_addr (char * text,struct sockaddr_in *addr){ -char ip_str[INET_ADDRSTRLEN +1 ]; -inet_ntop (AF_INET , &(addr->sin_addr), ip_str,sizeof (ip_str)); -evl_printf ("%s--------\n" , text); -evl_printf ("IP-Address: %s\n" , ip_str); -evl_printf ("Port: %d\n" ,ntohs (addr->sin_port)); -evl_printf ("Family: %d\n" , addr->sin_family); +char ip_str[INET_ADDRSTRLEN +1 ]; +inet_ntop (AF_INET , &(addr->sin_addr), ip_str,sizeof (ip_str)); +evl_printf ("== %s\n" , text); +evl_printf (" ip-address: %s\n" , ip_str); +evl_printf (" port: %d\n" ,ntohs (addr->sin_port)); +evl_printf (" family: %d\n" , addr->sin_family); } static void sender(int s, const char *text, int mcount,@@ -226,6 +226,8 @@ static void client(int s, const char *text, int mcount,free (tbuf); } +#define SERVER_ADDR_LIST_SIZE 64 +static void server (int s,const char *text,int mcount,struct sockaddr_in *addr,int iter) {tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 241
Quoted 4 lines
tidbits/oob-net-udp.c +1
-0@@ -236,6 +238,9 @@ static void server(int s, const char *text, int mcount,ssize_t ret;char *tbuf;char rbuf[16384 ]; +in_addr_t addr_list[SERVER_ADDR_LIST_SIZE ]={};^ missing whitespaces
Quoted 9 lines
tidbits/oob-net-udp.c +2
-0+size_t addr_list_fill=0 ; +bool solicit_done; tlen = (strlen (text) +1 ) * mcount; tbuf =malloc (tlen);@@ -276,6 +281,39 @@ static void server(int s, const char *text, int mcount,evl_printf (" (TRUNCATED)" );evl_printf (": %.*s\n" , (int )ret, rbuf);Quoted 12 lines
tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 291
Quoted 8 lines
tidbits/oob-net-udp.c +8
-0+/* +* We need to call evl_net_solicit for each new +* client once before sending data. This will break +* realtime for the first response. +* If this is not done and the ARP address is not +* yet in cache or garbage-collected, oob_sendmsg +* will return EINPROGRESS on start or during runtime. +*/ We can happily send redundant solicit requests to the core, no need to filter out cached addresses,
evl_net_solicit()will do the right thing.
Quoted 9 lines
Except that doing so would always demote the caller to the in-band stage, which may not be what you want. The fact that we'd need to maintain a cache in apps in order to figure out whether solicitation should be done shows a shortcoming in the core, users should not have to do this dance.
We should have an oob call for probing the route+arp caches with ipv4. I'll look into this asap.
Yes, this is the reason I do it this way and print out a message if this happens.
I think server mode with multiple clients in real time is not something you can just easily do without careful considerations what happens when multiple clients send a message at the same time or TDMA behind. My intent in this example is mostly to show that it is possible.
A real application would need one of these options:
- A thread only for solicitation so the main thread doesn't get blocked
- evl_net_solicit() with a flag like MSG_DONTWAIT and then poll in the main loop if the client is ready
- Know the clients in advance
- A warm up phase during clients can connect
- ...
However, a way for probing and reading the arp would shurely help. It would also be nice to have an "evl net" command to list the entry's. I tried the normal arp command but it doesn't behave nicely with evl.
No hurry from my side, I have linuxcnc already running well with evl oob networking, I only need client mode and i know the IP in advance, so all good there.
Sorry about the whitespace issues, switching between projects... I sent a patch v2. Feel free to merge this already and I can create a separate patch when probing ARP is here or wait and I create a patch v3.
Quoted 93 lines · Hannes Diethelm
Am 01.06.26 um 11:10 schrieb Philippe Gerum:
Quoted 67 lines
Quoted 56 lines · Philippe Gerum
Quoted 39 lines · Hannes Diethelm
This fixes random EINPROGRESS at init or during runtime.
Also correct whitespaces and make stdout more consistent.
Signed-off-by Hannes Diethelm <[email protected]>--- tidbits/oob-net-udp.c | 58 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 8 deletions(-)tidbits/oob-net-udp.c +7
-6@@ -50,12 +50,12 @@ static void usage(void) }static void print_addr (char * text,struct sockaddr_in *addr){ -char ip_str[INET_ADDRSTRLEN +1 ]; -inet_ntop (AF_INET , &(addr->sin_addr), ip_str,sizeof (ip_str)); -evl_printf ("%s--------\n" , text); -evl_printf ("IP-Address: %s\n" , ip_str); -evl_printf ("Port: %d\n" ,ntohs (addr->sin_port)); -evl_printf ("Family: %d\n" , addr->sin_family); +char ip_str[INET_ADDRSTRLEN +1 ]; +inet_ntop (AF_INET , &(addr->sin_addr), ip_str,sizeof (ip_str)); +evl_printf ("== %s\n" , text); +evl_printf (" ip-address: %s\n" , ip_str); +evl_printf (" port: %d\n" ,ntohs (addr->sin_port)); +evl_printf (" family: %d\n" , addr->sin_family); } static void sender(int s, const char *text, int mcount,@@ -226,6 +226,8 @@ static void client(int s, const char *text, int mcount,free (tbuf); } +#define SERVER_ADDR_LIST_SIZE 64 +static void server (int s,const char *text,int mcount,struct sockaddr_in *addr,int iter) {tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 241
Quoted 4 lines
tidbits/oob-net-udp.c +1
-0@@ -236,6 +238,9 @@ static void server(int s, const char *text, int mcount,ssize_t ret;char *tbuf;char rbuf[16384 ]; +in_addr_t addr_list[SERVER_ADDR_LIST_SIZE ]={};^ missing whitespaces
Quoted 10 lines
tidbits/oob-net-udp.c +4
-0+size_t addr_list_fill=0 ; +bool solicit_done; tlen = (strlen (text) +1 ) * mcount; tbuf =malloc (tlen);@@ -276,6 +281,39 @@ static void server(int s, const char *text, int mcount,evl_printf (" (TRUNCATED)" );evl_printf (": %.*s\n" , (int )ret, rbuf); + /* +* We need to call evl_net_solicit for each new +* client once before sending data. This will break Quoted 9 lines
tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 291
Quoted 5 lines
tidbits/oob-net-udp.c +5
-0+* realtime for the first response. +* If this is not done and the ARP address is not +* yet in cache or garbage-collected, oob_sendmsg +* will return EINPROGRESS on start or during runtime. +*/ We can happily send redundant solicit requests to the core, no need to filter out cached addresses,
evl_net_solicit()will do the right thing.Quoted 8 lines
Except that doing so would always demote the caller to the in-band stage, which may not be what you want. The fact that we'd need to maintain a cache in apps in order to figure out whether solicitation should be done shows a shortcoming in the core, users should not have to do this dance.
We should have an oob call for probing the route+arp caches with ipv4. I'll look into this asap.Yes, this is the reason I do it this way and print out a message if this happens.
I think server mode with multiple clients in real time is not something you can just easily do without careful considerations what happens when multiple clients send a message at the same time or TDMA behind. My intent in this example is mostly to show that it is possible.
A real application would need one of these options:
- A thread only for solicitation so the main thread doesn't get blocked
-evl_net_solicit()with a flag likeMSG_DONTWAITand then poll in the main loop if the client is ready
- Know the clients in advance
- A warm up phase during clients can connect
- ...
Quoted 1 line
However, a way for probing and reading the arp would shurely help.
Here is a general proposal to deal with the issue of sending UDP packets to unplanned destinations, which is basically what the server mode has to do, when receiving unsolicited messages.
We have a single invariant to cope with: the evl network stack wants to use the routing+arp information produced by the regular in-band stack, so that we can benefit from potentially complex routing logic without reinventing that wheel. Therefore, the route to any new/unsolicited destination must be resolved by the in-band stack once so that we can send UDP packets to that peer from the oob stage next.
From the point above, and as you pointed out already, an oob sender must either plan for reaching a particular destination (e.g. soliciting it prior to entering time-critical code), or accept an initial delay for the route to be resolved if the destination is not yet known on first transmit (i.e. out of the oob caches).
So, I would say that evl could meet the requirements you stated above by implementing the following:
- Honor MSG_PROBE for oob_sendmsg(), so that only the general call sanity and route resolution to the destination host is performed when set in the request flags, without actually sending any data. On success of such call, we would know that the routing information is readily available from the oob caches, no offload to in-band would have happened if we had not given this flag. The absence of routing information to the destination from some oob cache would yield a specific error, so that the caller may decide what to do next.
- Extend the effect of receiving MSG_DONTWAIT (and more generally O_NONBLOCK on fildes) to what we would do upon missing routing
information: if present, return with a specific error code _without_ relaying the packet to the in-band stack. The caller may then decide to handle the case locally. Otherwise, proceed as usual (i.e. relay to the in-band stack, then notify the caller with -EINPROGRESS).
- Provide a way to synchronize with the route resolution process in evl, i.e. a syscall that would block until a given destination is available from the oob cache. This call already exists, evl_net_solicit() can be used for that purpose (if a resolution request is already in flight, subsequent ones to the same destination won't cause any harm).
Points 1 and 2 are implemented in [1]. Feedback greatly appreciated and important when you have time. Usability for real-world applications is key.
Quoted 3 lines
It would also be nice to have an "evl net" command to list the entry's. I tried the normal arp command but it doesn't behave nicely with evl.
Could you please elaborate on this issue? Currently, updates to the in-band cache are propagated to the oob cache (by registering a hook into the relevant notifier chain in the kernel). So I would expect all destinations of interest to oob which are visible from the in-band arp cache to be available from the evl map as well.
This said, I agree that we need a way to inspect the oob cache specifically. Working on it.
[1] https://gitlab.com/Xenomai/xenomai4/linux-evl/-/tree/wip/net-solicit?ref_type=heads
Signature · 1 line
Philippe.
Am 20.06.26 um 19:28 schrieb Philippe Gerum:
Quoted 117 lines
Quoted 93 lines · Hannes Diethelm
Am 01.06.26 um 11:10 schrieb Philippe Gerum:
Quoted 67 lines
Quoted 56 lines · Philippe Gerum
Quoted 39 lines · Hannes Diethelm
This fixes random EINPROGRESS at init or during runtime.
Also correct whitespaces and make stdout more consistent.
Signed-off-by Hannes Diethelm <[email protected]>--- tidbits/oob-net-udp.c | 58 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 8 deletions(-)tidbits/oob-net-udp.c +7
-6@@ -50,12 +50,12 @@ static void usage(void) }static void print_addr (char * text,struct sockaddr_in *addr){ -char ip_str[INET_ADDRSTRLEN +1 ]; -inet_ntop (AF_INET , &(addr->sin_addr), ip_str,sizeof (ip_str)); -evl_printf ("%s--------\n" , text); -evl_printf ("IP-Address: %s\n" , ip_str); -evl_printf ("Port: %d\n" ,ntohs (addr->sin_port)); -evl_printf ("Family: %d\n" , addr->sin_family); +char ip_str[INET_ADDRSTRLEN +1 ]; +inet_ntop (AF_INET , &(addr->sin_addr), ip_str,sizeof (ip_str)); +evl_printf ("== %s\n" , text); +evl_printf (" ip-address: %s\n" , ip_str); +evl_printf (" port: %d\n" ,ntohs (addr->sin_port)); +evl_printf (" family: %d\n" , addr->sin_family); } static void sender(int s, const char *text, int mcount,@@ -226,6 +226,8 @@ static void client(int s, const char *text, int mcount,free (tbuf); } +#define SERVER_ADDR_LIST_SIZE 64 +static void server (int s,const char *text,int mcount,struct sockaddr_in *addr,int iter) {tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 241
Quoted 4 lines
tidbits/oob-net-udp.c +1
-0@@ -236,6 +238,9 @@ static void server(int s, const char *text, int mcount,ssize_t ret;char *tbuf;char rbuf[16384 ]; +in_addr_t addr_list[SERVER_ADDR_LIST_SIZE ]={};^ missing whitespaces
Quoted 10 lines
tidbits/oob-net-udp.c +4
-0+size_t addr_list_fill=0 ; +bool solicit_done; tlen = (strlen (text) +1 ) * mcount; tbuf =malloc (tlen);@@ -276,6 +281,39 @@ static void server(int s, const char *text, int mcount,evl_printf (" (TRUNCATED)" );evl_printf (": %.*s\n" , (int )ret, rbuf); + /* +* We need to call evl_net_solicit for each new +* client once before sending data. This will break Quoted 9 lines
tidbits/oob-net-udp.c static void server(int s, const char *text, int mcount, · line 291
Quoted 5 lines
tidbits/oob-net-udp.c +5
-0+* realtime for the first response. +* If this is not done and the ARP address is not +* yet in cache or garbage-collected, oob_sendmsg +* will return EINPROGRESS on start or during runtime. +*/ We can happily send redundant solicit requests to the core, no need to filter out cached addresses,
evl_net_solicit()will do the right thing.Quoted 8 lines
Except that doing so would always demote the caller to the in-band stage, which may not be what you want. The fact that we'd need to maintain a cache in apps in order to figure out whether solicitation should be done shows a shortcoming in the core, users should not have to do this dance.
We should have an oob call for probing the route+arp caches with ipv4. I'll look into this asap.Yes, this is the reason I do it this way and print out a message if this happens.
I think server mode with multiple clients in real time is not something you can just easily do without careful considerations what happens when multiple clients send a message at the same time or TDMA behind. My intent in this example is mostly to show that it is possible.
A real application would need one of these options:
- A thread only for solicitation so the main thread doesn't get blocked
-evl_net_solicit()with a flag likeMSG_DONTWAITand then poll in the main loop if the client is ready
- Know the clients in advance
- A warm up phase during clients can connect
- ...Quoted 1 line
However, a way for probing and reading the arp would shurely help.
Here is a general proposal to deal with the issue of sending UDP packets to unplanned destinations, which is basically what the server mode has to do, when receiving unsolicited messages.
We have a single invariant to cope with: the evl network stack wants to use the routing+arp information produced by the regular in-band stack, so that we can benefit from potentially complex routing logic without reinventing that wheel. Therefore, the route to any new/unsolicited destination must be resolved by the in-band stack once so that we can send UDP packets to that peer from the oob stage next.
From the point above, and as you pointed out already, an oob sender must
either plan for reaching a particular destination (e.g. soliciting it prior to entering time-critical code), or accept an initial delay for the route to be resolved if the destination is not yet known on first transmit (i.e. out of the oob caches).So, I would say that evl could meet the requirements you stated above by implementing the following:
Quoted 8 lines
- Honor
MSG_PROBEforoob_sendmsg(), so that only the general call sanity and route resolution to the destination host is performed when set in the request flags, without actually sending any data. On success of such call, we would know that the routing information is readily available from the oob caches, no offload to in-band would have happened if we had not given this flag. The absence of routing information to the destination from some oob cache would yield a specific error, so that the caller may decide what to do next.
It seams the flag MSG_PROBE is kernel only? I did not find any occurrence in /usr/include or in libevl.
Quoted 7 lines
- Extend the effect of receiving
MSG_DONTWAIT(and more generallyO_NONBLOCKon fildes) to what we would do upon missing routing
information: if present, return with a specific error code _without_ relaying the packet to the in-band stack. The caller may then decide to handle the case locally. Otherwise, proceed as usual (i.e. relay to the in-band stack, then notify the caller with-EINPROGRESS).
So the oob-net-udp server code can be changed to use MSG_DONTWAIT and if the return value is EWOULDBLOCK, call evl_net_solicit() and try again instead of holding a list of IP's right?
Quoted 10 lines
- Provide a way to synchronize with the route resolution process in evl, i.e. a syscall that would block until a given destination is available from the oob cache. This call already exists,
evl_net_solicit()can be used for that purpose (if a resolution request is already in flight, subsequent ones to the same destination won't cause any harm).Points 1 and 2 are implemented in [1]. Feedback greatly appreciated and important when you have time. Usability for real-world applications is key.
I am a bit busy right now, but I would like to test it. However, It can take a week or two.
Quoted 11 lines
Quoted 4 lines
It would also be nice to have an "evl net" command to list the entry's. I tried the normal arp command but it doesn't behave nicely with evl.
Could you please elaborate on this issue? Currently, updates to the in-band cache are propagated to the oob cache (by registering a hook into the relevant notifier chain in the kernel). So I would expect all destinations of interest to oob which are visible from the in-band arp cache to be available from the evl map as well.
The issue was on my side, I need to run "arp -n" so it does not resolve host names. Otherwise, it takes quite some time. But this makes sense, with OOB active, it will never receive the answer on the request, so it times out.
By the way: Since some time, all incoming network traffic goes to the OOB stage when it is enabled. However, all outgoing traffic from the in-band stage is still sent. Would it make sense to block all outgoing traffic after "evl net -ei" except might be ARP requests if they are initialized by evl_net_solicit()?
At the moment, this is done using nftables for linuxcnc, this works fine for preemt_rt and xenomai, so not really needed.
Quoted 5 lines
This said, I agree that we need a way to inspect the oob cache specifically. Working on it.
[1] https://gitlab.com/Xenomai/xenomai4/linux-evl/-/tree/wip/net-solicit?ref_type=heads
Quoted 10 lines · Hannes Diethelm
Am 20.06.26 um 19:28 schrieb Philippe Gerum:
Quoted 8 lines
- Honor
MSG_PROBEforoob_sendmsg(), so that only the general call sanity and route resolution to the destination host is performed when set in the request flags, without actually sending any data. On success of such call, we would know that the routing information is readily available from the oob caches, no offload to in-band would have happened if we had not given this flag. The absence of routing information to the destination from some oob cache would yield a specific error, so that the caller may decide what to do next.
Quoted 3 lines
It seams the flag
MSG_PROBEis kernel only? I did not find any occurrence in/usr/includeor in libevl.
Yep, my bad. Using MSG_PROBE is not the right way, since that would conflict with MSG_PROXY in userland which has a totally different meaning. I have revisited the implementation, simplifying it actually:
since the evl netstack already accepts zero-sized messages, sending such a datagram to the UDP layer now amounts to returning early with the address resolution status, short-circuiting the logic before the actual transmission happens.
IOW, passing a NULL or empty iov into the msghdr struct does what MSG_PROBE was intended to do.
Quoted 11 lines
Quoted 6 lines
- Extend the effect of receiving
MSG_DONTWAIT(and more generallyO_NONBLOCKon fildes) to what we would do upon missing routing
information: if present, return with a specific error code _without_ relaying the packet to the in-band stack. The caller may then decide to handle the case locally. Otherwise, proceed as usual (i.e. relay to the in-band stack, then notify the caller with-EINPROGRESS).So the oob-net-udp server code can be changed to use
MSG_DONTWAITand if the return value is EWOULDBLOCK, callevl_net_solicit()and try again instead of holding a list of IP's right?
Yep.
Quoted 11 lines
Quoted 10 lines
- Provide a way to synchronize with the route resolution process in evl,
i.e. a syscall that would block until a given destination is available from the oob cache. This call already exists,evl_net_solicit()can be used for that purpose (if a resolution request is already in flight, subsequent ones to the same destination won't cause any harm).
Points 1 and 2 are implemented in [1]. Feedback greatly appreciated and
important when you have time. Usability for real-world applications is key.
Quoted 3 lines
I am a bit busy right now, but I would like to test it. However, It can take a week or two.
Np. Your contribution has been valuable, so it's worth waiting for it.
Quoted 11 lines
Quoted 10 lines
Quoted 4 lines
It would also be nice to have an "evl net" command to list the entry's. I tried the normal arp command but it doesn't behave nicely with evl.
Could you please elaborate on this issue? Currently, updates to the in-band cache are propagated to the oob cache (by registering a hook into the relevant notifier chain in the kernel). So I would expect all destinations of interest to oob which are visible from the in-band arp cache to be available from the evl map as well.
Quoted 4 lines
The issue was on my side, I need to run "arp -n" so it does not resolve host names. Otherwise, it takes quite some time. But this makes sense, with OOB active, it will never receive the answer on the request, so it times out.
Ok, this is not user-friendly. I'll have a look.
Quoted 5 lines
By the way: Since some time, all incoming network traffic goes to the OOB stage when it is enabled. However, all outgoing traffic from the in-band stage is still sent. Would it make sense to block all outgoing traffic after "evl net -ei" except might be ARP requests if they are initialized by
evl_net_solicit()?
I believe this may break some use cases I'm aware of. Some people do currently use such asymmetry between ingress-oob and egress-inband on the same interface on purpose.
Quoted 3 lines
At the moment, this is done using nftables for linuxcnc, this works fine for
preemt_rtand xenomai, so not really needed.
Currently, it is assumed that any outgoing inband traffic has a purpose, even on an oob-enabled interface. What kind of egress inband traffic could/should be dropped safely when flowing through an oob interface?
Quoted 5 lines
Quoted 5 lines
This said, I agree that we need a way to inspect the oob cache specifically. Working on it.
[1]
https://gitlab.com/Xenomai/xenomai4/linux-evl/-/tree/wip/net-solicit?ref_type=heads
Signature · 1 line
Philippe.
Am 04.07.26 um 19:52 schrieb Philippe Gerum:
Quoted 24 lines
Quoted 10 lines · Hannes Diethelm
Am 20.06.26 um 19:28 schrieb Philippe Gerum:
Quoted 8 lines
- Honor
MSG_PROBEforoob_sendmsg(), so that only the general call sanity and route resolution to the destination host is performed when set in the request flags, without actually sending any data. On success of such call, we would know that the routing information is readily available from the oob caches, no offload to in-band would have happened if we had not given this flag. The absence of routing information to the destination from some oob cache would yield a specific error, so that the caller may decide what to do next.Quoted 3 lines
It seams the flag
MSG_PROBEis kernel only? I did not find any occurrence in/usr/includeor in libevl.Yep, my bad. Using
MSG_PROBEis not the right way, since that would conflict withMSG_PROXYin userland which has a totally different meaning. I have revisited the implementation, simplifying it actually:
since the evl netstack already accepts zero-sized messages, sending such a datagram to the UDP layer now amounts to returning early with the address resolution status, short-circuiting the logic before the actual transmission happens.
Quoted 2 lines
IOW, passing a NULL or empty iov into the msghdr struct does what
MSG_PROBEwas intended to do.
I tested this variant. It works. But I wonder:
If I use:
ret = oob_sendmsg(s, &msghdr, NULL, 0); errno is set to EHOSTUNREACH If I use:
ret = oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT); errno is set to EWOULDBLOCK
Is this intended? EHOSTUNREACH is like halve correct. Yes, the host can not be reached but only due to no ARP request is sent.
For oob-net-udp server mode, this variant is a bit wastefull due to oob_sendmsg() would either always be called twice or I would have to keep the list of clients.
Quoted 8 lines
Quoted 8 lines
Quoted 7 lines
- Extend the effect of receiving
MSG_DONTWAIT(and more generallyO_NONBLOCKon fildes) to what we would do upon missing routing
information: if present, return with a specific error code _without_ relaying the packet to the in-band stack. The caller may then decide to handle the case locally. Otherwise, proceed as usual (i.e. relay to the in-band stack, then notify the caller with-EINPROGRESS).
Quoted 6 lines
Quoted 4 lines
So the oob-net-udp server code can be changed to use
MSG_DONTWAITand if the return value is EWOULDBLOCK, callevl_net_solicit()and try again instead of holding a list of IP's right?Yep.
This works nicely, I will send a patch changing oob-net-udp server mode to use this.
The only disadvantage is that if for what ever reason, the client is already in the ARP cache but not permanent, it is cleared after a timeout, so evl_net_solicit() can happen later than expected.
Quoted 42 lines
Quoted 12 lines
Quoted 11 lines
- Provide a way to synchronize with the route resolution process in evl,
i.e. a syscall that would block until a given destination is available from the oob cache. This call already exists,evl_net_solicit()can be used for that purpose (if a resolution request is already in flight, subsequent ones to the same destination won't cause any harm).
Points 1 and 2 are implemented in [1]. Feedback greatly appreciated and
important when you have time. Usability for real-world applications is key.Quoted 3 lines
I am a bit busy right now, but I would like to test it. However, It can take a week or two.
Np. Your contribution has been valuable, so it's worth waiting for it.
Quoted 11 lines
Quoted 10 lines
Quoted 4 lines
It would also be nice to have an "evl net" command to list the entry's. I tried the normal arp command but it doesn't behave nicely with evl.
Could you please elaborate on this issue? Currently, updates to the in-band cache are propagated to the oob cache (by registering a hook into the relevant notifier chain in the kernel). So I would expect all destinations of interest to oob which are visible from the in-band arp cache to be available from the evl map as well.
Quoted 4 lines
The issue was on my side, I need to run "arp -n" so it does not resolve host names. Otherwise, it takes quite some time. But this makes sense, with OOB active, it will never receive the answer on the request, so it times out.
Ok, this is not user-friendly. I'll have a look.
Quoted 5 lines
By the way: Since some time, all incoming network traffic goes to the OOB stage when it is enabled. However, all outgoing traffic from the in-band stage is still sent. Would it make sense to block all outgoing traffic after "evl net -ei" except might be ARP requests if they are initialized by
evl_net_solicit()?
Quoted 3 lines
I believe this may break some use cases I'm aware of. Some people do currently use such asymmetry between ingress-oob and egress-inband on the same interface on purpose.
In this case, it is fine.
Quoted 8 lines
Quoted 4 lines
At the moment, this is done using nftables for linuxcnc, this works fine for
preemt_rtand xenomai, so not really needed.Currently, it is assumed that any outgoing inband traffic has a purpose, even on an oob-enabled interface. What kind of egress inband traffic could/should be dropped safely when flowing through an oob interface?
Mostly unintended traffic by either user error or some running services sending discovery messages to all interfaces. Using nftables works nicely so there is no real need to do anything else.
Quoted 7 lines
Quoted 6 lines
Quoted 6 lines
This said, I agree that we need a way to inspect the oob cache specifically. Working on it.
[1]
https://gitlab.com/Xenomai/xenomai4/linux-evl/-/tree/wip/net-solicit?ref_type=heads
I've seen this commits are already on v6.12.y-cip-evl-rebase? I did these tests on v6.12.y-cip-evl-rebase.
Quoted 26 lines · Hannes Diethelm
Am 04.07.26 um 19:52 schrieb Philippe Gerum:
Quoted 15 lines
Quoted 10 lines · Hannes Diethelm
Am 20.06.26 um 19:28 schrieb Philippe Gerum:
Quoted 8 lines
- Honor
MSG_PROBEforoob_sendmsg(), so that only the general call sanity and route resolution to the destination host is performed when set in the request flags, without actually sending any data. On success of such call, we would know that the routing information is readily available from the oob caches, no offload to in-band would have happened if we had not given this flag. The absence of routing information to the destination from some oob cache would yield a specific error, so that the caller may decide what to do next.Quoted 3 lines
It seams the flag
MSG_PROBEis kernel only? I did not find any occurrence in/usr/includeor in libevl.Quoted 9 lines
Yep, my bad. Using
MSG_PROBEis not the right way, since that would conflict withMSG_PROXYin userland which has a totally different meaning. I have revisited the implementation, simplifying it actually:
since the evl netstack already accepts zero-sized messages, sending such a datagram to the UDP layer now amounts to returning early with the address resolution status, short-circuiting the logic before the actual transmission happens.
IOW, passing a NULL or empty iov into the msghdr struct does whatMSG_PROBEwas intended to do.
Quoted 9 lines
I tested this variant. It works. But I wonder:
If I use:
ret =oob_sendmsg(s, &msghdr, NULL, 0); errno is set to EHOSTUNREACH If I use:
ret =oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT); errno is set to EWOULDBLOCKIs this intended? EHOSTUNREACH is like halve correct. Yes, the host can not be reached but only due to no ARP request is sent.
EHOSTUNREACH was intended as a way to distinguish from EWOULDBLOCK wrt lack of buffer space for the outgoing message, this code was the only option close enough to the idea to be conveyed available from the errno list that would not conflict with other situations. Now, since such probing mode needs no message space in the first place, this is guarding against the impossible, which does not make sense. Returning -EWOULDBLOCK in both cases above would still be
practical. e.g. something along these lines:
kernel/evl/net/ipv4/udp.c +10 -10
@@ -417,22 +417,22 @@ static ssize_t send_udp(struct evl_socket *esk,* address. */ ret =find_egress_path (esk, daddr, &ert, &earp, &pseudo_earp, msg_flags); -if (ret == -EMULTIHOP ) -return ret;/* MSG_DONTROUTE cannot be honored. */ -if (ret) { +if (ret != -EHOSTUNREACH ) +return ret; + +if (datalen ==0 ) +return -EWOULDBLOCK ;/* Address probe failed. */ +/* -* No route known from the front cache - bummer. We -* may have to offload the transmit operation to the -* in-band stack, unless only probing or MSG_DONTWAIT -* is set. +* We have a message to send but no route was found in +* the front cache - bummer. We may have to offload +* the transmit operation to the in-band stack, unless +* only probing or MSG_DONTWAIT is set. */ if (msg_flags &MSG_DONTWAIT )return -EWOULDBLOCK ; -if (datalen ==0 ) -return ret; -/* * We always charge the socket even when offloading to * the in-band stack although we won't consume any
Quoted 3 lines
For oob-net-udp server mode, this variant is a bit wastefull due to
oob_sendmsg()would either always be called twice or I would have to keep the list of clients.
Which brings back the option of some MSG_xxx operation flag so that we could pass a valid buffer _and_ a probing flag, but then we'd need Dovetail to add one to the standard list (in userland) because I don't see any standard one to piggyback off of.
Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
Quoted 7 lines
- Extend the effect of receiving
MSG_DONTWAIT(and more generallyO_NONBLOCKon fildes) to what we would do upon missing routing
information: if present, return with a specific error code _without_ relaying the packet to the in-band stack. The caller may then decide to handle the case locally. Otherwise, proceed as usual (i.e. relay to the in-band stack, then notify the caller with-EINPROGRESS).So the oob-net-udp server code can be changed to use
MSG_DONTWAITand if the return value is EWOULDBLOCK, callevl_net_solicit()and try again instead of holding a list of IP's right?
Quoted 8 lines
Quoted 1 line
Yep.
This works nicely, I will send a patch changing oob-net-udp server mode to use this.
The only disadvantage is that if for what ever reason, the client is already in the ARP cache but not permanent, it is cleared after a timeout, so
evl_net_solicit()can happen later than expected.
Yep, because at the moment, the core mirrors to the front cache all insertions and deletions happening into the inband cache unconditionally. We could force a permanent state for any entry we are about to insert into the front cache in order to prevent what you described, but I'm wary about unwanted side-effects.
Quoted 50 lines
Quoted 35 lines
Quoted 12 lines
Quoted 11 lines
- Provide a way to synchronize with the route resolution process in evl,
i.e. a syscall that would block until a given destination is available from the oob cache. This call already exists,evl_net_solicit()can be used for that purpose (if a resolution request is already in flight, subsequent ones to the same destination won't cause any harm).
Points 1 and 2 are implemented in [1]. Feedback greatly appreciated and
important when you have time. Usability for real-world applications is key.Quoted 3 lines
I am a bit busy right now, but I would like to test it. However, It can take a week or two.
Np. Your contribution has been valuable, so it's worth waiting for it.
Quoted 11 lines
Quoted 10 lines
Quoted 4 lines
It would also be nice to have an "evl net" command to list the entry's. I tried the normal arp command but it doesn't behave nicely with evl.
Could you please elaborate on this issue? Currently, updates to the in-band cache are propagated to the oob cache (by registering a hook into the relevant notifier chain in the kernel). So I would expect all destinations of interest to oob which are visible from the in-band arp cache to be available from the evl map as well.
Quoted 4 lines
The issue was on my side, I need to run "arp -n" so it does not resolve host names. Otherwise, it takes quite some time. But this makes sense, with OOB active, it will never receive the answer on the request, so it times out.
Ok, this is not user-friendly. I'll have a look.
Quoted 8 lines
Quoted 5 lines
By the way: Since some time, all incoming network traffic goes to the OOB stage when it is enabled. However, all outgoing traffic from the in-band stage is still sent. Would it make sense to block all outgoing traffic after "evl net -ei" except might be ARP requests if they are initialized by
evl_net_solicit()?I believe this may break some use cases I'm aware of. Some people do currently use such asymmetry between ingress-oob and egress-inband on the same interface on purpose.
In this case, it is fine.
Quoted 4 lines
Quoted 4 lines
At the moment, this is done using nftables for linuxcnc, this works fine for
preemt_rtand xenomai, so not really needed.
Quoted 9 lines
Quoted 4 lines
Currently, it is assumed that any outgoing inband traffic has a purpose,
even on an oob-enabled interface. What kind of egress inband traffic could/should be dropped safely when flowing through an oob interface?Mostly unintended traffic by either user error or some running services sending discovery messages to all interfaces. Using nftables works nicely so there is no real need to do anything else.
Ok.
Quoted 9 lines
Quoted 7 lines
Quoted 6 lines
Quoted 6 lines
This said, I agree that we need a way to inspect the oob cache specifically. Working on it.
[1]
https://gitlab.com/Xenomai/xenomai4/linux-evl/-/tree/wip/net-solicit?ref_type=headsI've seen this commits are already on v6.12.y-cip-evl-rebase? I did these tests on v6.12.y-cip-evl-rebase.
Yep, because this probing mode is here to stay, although I'm considering another way to provide this using a dedicated MSG_xx flag. Problem ATM is that I'd like not to pollute the namespace uselessly by creating non-std flags. This said, it may be legitimate to extend the existing set in this case.
Signature · 1 line
Philippe.
Am 20.07.26 um 16:27 schrieb Philippe Gerum:
Quoted 75 lines
Quoted 26 lines · Hannes Diethelm
Am 04.07.26 um 19:52 schrieb Philippe Gerum:
Quoted 15 lines
Quoted 10 lines · Hannes Diethelm
Am 20.06.26 um 19:28 schrieb Philippe Gerum:
Quoted 8 lines
- Honor
MSG_PROBEforoob_sendmsg(), so that only the general call sanity and route resolution to the destination host is performed when set in the request flags, without actually sending any data. On success of such call, we would know that the routing information is readily available from the oob caches, no offload to in-band would have happened if we had not given this flag. The absence of routing information to the destination from some oob cache would yield a specific error, so that the caller may decide what to do next.Quoted 3 lines
It seams the flag
MSG_PROBEis kernel only? I did not find any occurrence in/usr/includeor in libevl.Quoted 9 lines
Yep, my bad. Using
MSG_PROBEis not the right way, since that would conflict withMSG_PROXYin userland which has a totally different meaning. I have revisited the implementation, simplifying it actually:
since the evl netstack already accepts zero-sized messages, sending such a datagram to the UDP layer now amounts to returning early with the address resolution status, short-circuiting the logic before the actual transmission happens.
IOW, passing a NULL or empty iov into the msghdr struct does whatMSG_PROBEwas intended to do.Quoted 9 lines
I tested this variant. It works. But I wonder:
If I use:
ret =oob_sendmsg(s, &msghdr, NULL, 0); errno is set to EHOSTUNREACH If I use:
ret =oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT); errno is set to EWOULDBLOCKIs this intended? EHOSTUNREACH is like halve correct. Yes, the host can not be reached but only due to no ARP request is sent.
EHOSTUNREACH was intended as a way to distinguish from EWOULDBLOCK wrt lack of buffer space for the outgoing message, this code was the only option close enough to the idea to be conveyed available from the errno list that would not conflict with other situations. Now, since such probing mode needs no message space in the first place, this is guarding against the impossible, which does not make sense. Returning
-EWOULDBLOCKin both cases above would still be
practical. e.g. something along these lines:kernel/evl/net/ipv4/udp.c +10
-7@@ -417,22 +417,22 @@ static ssize_t send_udp(struct evl_socket *esk,* address. */ ret =find_egress_path (esk, daddr, &ert, &earp, &pseudo_earp, msg_flags); -if (ret == -EMULTIHOP ) -return ret;/* MSG_DONTROUTE cannot be honored. */ -if (ret) { +if (ret != -EHOSTUNREACH ) +return ret; + +if (datalen ==0 ) +return -EWOULDBLOCK ;/* Address probe failed. */ +/* -* No route known from the front cache - bummer. We -* may have to offload the transmit operation to the -* in-band stack, unless only probing or MSG_DONTWAIT -* is set. +* We have a message to send but no route was found in +* the front cache - bummer. We may have to offload +* the transmit operation to the in-band stack, unless +* only probing or MSG_DONTWAIT is set.
kernel/evl/net/ipv4/udp.c static ssize_t send_udp(struct evl_socket *esk, · line 438
Quoted 10 lines
kernel/evl/net/ipv4/udp.c +0
-3*/ if (msg_flags &MSG_DONTWAIT )return -EWOULDBLOCK ; -if (datalen ==0 ) -return ret; -/* * We always charge the socket even when offloading to * the in-band stack although we won't consume any
I think in this case, it is fine as it is. You also won't expect EWOULDBLOCK or EAGAIN as long as you don't set MSG_DONTWAIT.
But now there are two ways of probing. Either with or withouth MSG_DONTWAIT that behave slightly different. Might be just support MSG_DONTWAIT -> EWOULDBLOCK and drop the other variant? Or is there a reason for the variant withouth MSG_DONTWAIT?
Quoted 9 lines
Quoted 3 lines
For oob-net-udp server mode, this variant is a bit wastefull due to
oob_sendmsg()would either always be called twice or I would have to keep the list of clients.Which brings back the option of some
MSG_xxxoperation flag so that we could pass a valid buffer _and_ a probing flag, but then we'd need Dovetail to add one to the standard list (in userland) because I don't see any standard one to piggyback off of.
Is there a reason to pass a valid buffer to only probe? Either you probe with a NULL buffer / size 0 or you pass a buffer and set MSG_DONTWAIT so the message is sent if possible. If not, evl_net_solicit() and retry.
BTW: NULL buffer / size 1 fails. But it also makes no sense.
Quoted 21 lines
Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
Quoted 7 lines
- Extend the effect of receiving
MSG_DONTWAIT(and more generallyO_NONBLOCKon fildes) to what we would do upon missing routing
information: if present, return with a specific error code _without_ relaying the packet to the in-band stack. The caller may then decide to handle the case locally. Otherwise, proceed as usual (i.e. relay to the in-band stack, then notify the caller with-EINPROGRESS).So the oob-net-udp server code can be changed to use
MSG_DONTWAITand if the return value is EWOULDBLOCK, callevl_net_solicit()and try again instead of holding a list of IP's right?Quoted 8 lines
Quoted 1 line
Yep.
This works nicely, I will send a patch changing oob-net-udp server mode to use this.
The only disadvantage is that if for what ever reason, the client is already in the ARP cache but not permanent, it is cleared after a timeout, so
evl_net_solicit()can happen later than expected.
Quoted 5 lines
Yep, because at the moment, the core mirrors to the front cache all insertions and deletions happening into the inband cache unconditionally. We could force a permanent state for any entry we are about to insert into the front cache in order to prevent what you described, but I'm wary about unwanted side-effects.
Right now you can use evl_net_solicit(..., EVL_NEIGH_PERMANENT) to make shure it is permanent but this is an in band call according to the doc. So if you want to stay out of band, you need to probe.
Now the probing doesn't show if the arp entry is permanent or going to disapear soon. So it can pass at the first try and fail later. Right now, you are better off doing evl_net_solicit() for every client if you want to be shure.
With ioctl(s, SIOCGARP, &arp_request), you can check if the entry is permanent (arp_request.arp_flags & ATF_PERM) and then use evl_net_solicit(..., EVL_NEIGH_PERMANENT) if not. But the ioctl is also in band.
An option would be either to support of oob_ioctl() for SIOCGARP. Or create something like evl_net_routeinfo(s, addr) returning flags would make probing obsolete.
Flags could be:EVL_ARP_COM Lookup complete (If this is not set, no ARP entry)EVL_ARP_PERM Permanent entryEVL_ROUTE_MULTIHOP More than one hop away (Will fail with MSG_DONTROUTE)
Quoted 71 lines
Quoted 51 lines
Quoted 36 lines
Quoted 13 lines
Quoted 12 lines
- Provide a way to synchronize with the route resolution process in evl,
i.e. a syscall that would block until a given destination is available from the oob cache. This call already exists,evl_net_solicit()can be used for that purpose (if a resolution request is already in flight, subsequent ones to the same destination won't cause any harm).
Points 1 and 2 are implemented in [1]. Feedback greatly appreciated and
important when you have time. Usability for real-world applications is key.Quoted 3 lines
I am a bit busy right now, but I would like to test it. However, It can take a week or two.
Np. Your contribution has been valuable, so it's worth waiting for it.
Quoted 11 lines
Quoted 10 lines
Quoted 4 lines
It would also be nice to have an "evl net" command to list the entry's. I tried the normal arp command but it doesn't behave nicely with evl.
Could you please elaborate on this issue? Currently, updates to the in-band cache are propagated to the oob cache (by registering a hook into the relevant notifier chain in the kernel). So I would expect all destinations of interest to oob which are visible from the in-band arp cache to be available from the evl map as well.
Quoted 4 lines
The issue was on my side, I need to run "arp -n" so it does not resolve host names. Otherwise, it takes quite some time. But this makes sense, with OOB active, it will never receive the answer on the request, so it times out.
Ok, this is not user-friendly. I'll have a look.
Quoted 8 lines
Quoted 5 lines
By the way: Since some time, all incoming network traffic goes to the OOB stage when it is enabled. However, all outgoing traffic from the in-band stage is still sent. Would it make sense to block all outgoing traffic after "evl net -ei" except might be ARP requests if they are initialized by
evl_net_solicit()?I believe this may break some use cases I'm aware of. Some people do currently use such asymmetry between ingress-oob and egress-inband on the same interface on purpose.
In this case, it is fine.
Quoted 4 lines
Quoted 4 lines
At the moment, this is done using nftables for linuxcnc, this works fine for
preemt_rtand xenomai, so not really needed.Quoted 9 lines
Quoted 4 lines
Currently, it is assumed that any outgoing inband traffic has a purpose,
even on an oob-enabled interface. What kind of egress inband traffic could/should be dropped safely when flowing through an oob interface?Mostly unintended traffic by either user error or some running services sending discovery messages to all interfaces. Using nftables works nicely so there is no real need to do anything else.
Ok.
Quoted 8 lines
Quoted 7 lines
Quoted 6 lines
Quoted 6 lines
This said, I agree that we need a way to inspect the oob cache specifically. Working on it.
[1]
https://gitlab.com/Xenomai/xenomai4/linux-evl/-/tree/wip/net-solicit?ref_type=heads
Quoted 8 lines
Quoted 1 line
I've seen this commits are already on v6.12.y-cip-evl-rebase? I did these tests on v6.12.y-cip-evl-rebase.
Yep, because this probing mode is here to stay, although I'm considering another way to provide this using a dedicated
MSG_xxflag. Problem ATM is that I'd like not to pollute the namespace uselessly by creating non-std flags. This said, it may be legitimate to extend the existing set in this case.
Fine for me, so I did not have to compile yet another kernel.
Quoted 88 lines · Hannes Diethelm
Am 20.07.26 um 16:27 schrieb Philippe Gerum:
Quoted 73 lines
Quoted 26 lines · Hannes Diethelm
Am 04.07.26 um 19:52 schrieb Philippe Gerum:
Quoted 15 lines
Quoted 10 lines · Hannes Diethelm
Am 20.06.26 um 19:28 schrieb Philippe Gerum:
Quoted 8 lines
- Honor
MSG_PROBEforoob_sendmsg(), so that only the general call sanity and route resolution to the destination host is performed when set in the request flags, without actually sending any data. On success of such call, we would know that the routing information is readily available from the oob caches, no offload to in-band would have happened if we had not given this flag. The absence of routing information to the destination from some oob cache would yield a specific error, so that the caller may decide what to do next.Quoted 3 lines
It seams the flag
MSG_PROBEis kernel only? I did not find any occurrence in/usr/includeor in libevl.Quoted 9 lines
Yep, my bad. Using
MSG_PROBEis not the right way, since that would conflict withMSG_PROXYin userland which has a totally different meaning. I have revisited the implementation, simplifying it actually:
since the evl netstack already accepts zero-sized messages, sending such a datagram to the UDP layer now amounts to returning early with the address resolution status, short-circuiting the logic before the actual transmission happens.
IOW, passing a NULL or empty iov into the msghdr struct does whatMSG_PROBEwas intended to do.Quoted 9 lines
I tested this variant. It works. But I wonder:
If I use:
ret =oob_sendmsg(s, &msghdr, NULL, 0); errno is set to EHOSTUNREACH If I use:
ret =oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT); errno is set to EWOULDBLOCKIs this intended? EHOSTUNREACH is like halve correct. Yes, the host can not be reached but only due to no ARP request is sent.
EHOSTUNREACH was intended as a way to distinguish from EWOULDBLOCK wrt
lack of buffer space for the outgoing message, this code was the only option close enough to the idea to be conveyed available from the errno list that would not conflict with other situations. Now, since such probing mode needs no message space in the first place, this is guarding against the impossible, which does not make sense. Returning-EWOULDBLOCKin both cases above would still be
practical. e.g. something along these lines:kernel/evl/net/ipv4/udp.c +9
-7@@ -417,22 +417,22 @@ static ssize_t send_udp(struct evl_socket *esk,* address. */ ret =find_egress_path (esk, daddr, &ert, &earp, &pseudo_earp, msg_flags); -if (ret == -EMULTIHOP ) -return ret;/* MSG_DONTROUTE cannot be honored. */ -if (ret) { +if (ret != -EHOSTUNREACH ) +return ret; + +if (datalen ==0 ) +return -EWOULDBLOCK ;/* Address probe failed. */ +/* -* No route known from the front cache - bummer. We -* may have to offload the transmit operation to the -* in-band stack, unless only probing or MSG_DONTWAIT -* is set. +* We have a message to send but no route was found in +* the front cache - bummer. We may have to offload +* the transmit operation to the in-band stack, unless kernel/evl/net/ipv4/udp.c static ssize_t send_udp(struct evl_socket *esk, · line 438
Quoted 10 lines
kernel/evl/net/ipv4/udp.c +1
-2+* only probing or MSG_DONTWAIT is set. */ if (msg_flags &MSG_DONTWAIT )return -EWOULDBLOCK ; - if (datalen == 0) -return ret; -/* * We always charge the socket even when offloading to * the in-band stack although we won't consume any I think in this case, it is fine as it is. You also won't expect EWOULDBLOCK or EAGAIN as long as you don't set
MSG_DONTWAIT.
Quoted 3 lines
But now there are two ways of probing. Either with or withouth
MSG_DONTWAITthat behave slightly different. Might be just supportMSG_DONTWAIT-> EWOULDBLOCK and drop the other
variant?
You mean detect a probing request when receiving MSG_DONTWAIT and a zero-sized buffer? That is an option. Another option would be to always return EHOSTUNREACH/??? on failed probe regardless of whether MSG_DONTWAIT is set. I would preferably go for the second option iff we can settle on a unconfusing, unambiguous error status.
Quoted 3 lines
Or is there a reason
for the variant withouthMSG_DONTWAIT?
In fact, the original intent was to assume that receiving a zero-sized message should be considered as a probing request. Then the effect of detecting MSG_DONTWAIT too in this particular code path was overlooked, which unexpectedly introduced another variant, which is indeed one too many.
Quoted 10 lines
Quoted 9 lines
Quoted 3 lines
For oob-net-udp server mode, this variant is a bit wastefull due to
oob_sendmsg()would either always be called twice or I would have to keep the list of clients.Which brings back the option of some
MSG_xxxoperation flag so that we
could pass a valid buffer _and_ a probing flag, but then we'd need Dovetail to add one to the standard list (in userland) because I don't see any standard one to piggyback off of.
Quoted 6 lines
Is there a reason to pass a valid buffer to only probe? Either you probe with a NULL buffer / size 0 or you pass a buffer and set
MSG_DONTWAITso the message is sent if possible. If not,evl_net_solicit()and retry.BTW: NULL buffer / size 1 fails. But it also makes no sense.
Yep, in this case, -EFAULT is a reasonable outcome since the key argument is the message size. If the core is told that at least one byte is valid, then a valid pointer to at least one byte should be given.
Quoted 38 lines
Quoted 20 lines
Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
Quoted 7 lines
- Extend the effect of receiving
MSG_DONTWAIT(and more generallyO_NONBLOCKon fildes) to what we would do upon missing routing
information: if present, return with a specific error code _without_ relaying the packet to the in-band stack. The caller may then decide to handle the case locally. Otherwise, proceed as usual (i.e. relay to the in-band stack, then notify the caller with-EINPROGRESS).So the oob-net-udp server code can be changed to use
MSG_DONTWAITand if the return value is EWOULDBLOCK, callevl_net_solicit()and try again instead of holding a list of IP's right?Quoted 8 lines
Quoted 1 line
Yep.
This works nicely, I will send a patch changing oob-net-udp server mode to use this.
The only disadvantage is that if for what ever reason, the client is already in the ARP cache but not permanent, it is cleared after a timeout, so
evl_net_solicit()can happen later than expected.Quoted 5 lines
Yep, because at the moment, the core mirrors to the front cache all insertions and deletions happening into the inband cache unconditionally. We could force a permanent state for any entry we are about to insert into the front cache in order to prevent what you described, but I'm wary about unwanted side-effects.
Right now you can use
evl_net_solicit(..., EVL_NEIGH_PERMANENT)to make shure it is permanent but this is an in band call according to the doc. So if you want to stay out of band, you need to probe.Now the probing doesn't show if the arp entry is permanent or going to disapear soon. So it can pass at the first try and fail later. Right now, you are better off doing
evl_net_solicit()for every client if you want to be shure.With
ioctl(s, SIOCGARP, &arp_request), you can check if the entry is permanent (arp_request.arp_flags &ATF_PERM) and then useevl_net_solicit(..., EVL_NEIGH_PERMANENT)if not. But the ioctl is also in band.
Quoted 8 lines
An option would be either to support of
oob_ioctl()for SIOCGARP. Or create something likeevl_net_routeinfo(s, addr)returning flags would make probing obsolete.Flags could be:
EVL_ARP_COMLookup complete (If this is not set, no ARP entry)EVL_ARP_PERMPermanent entryEVL_ROUTE_MULTIHOPMore than one hop away (Will fail withMSG_DONTROUTE)
Having oob_ioctl(SIOCGARP) which would perform a lookup into the oob front cache managed by the core is perfectly doable, then we could provide evl_net_routeinfo() as syntactic sugar building on it.
I would consider reusing the ATF_* flags directly instead of adding yet another set of EVL_* flags only to alias to the former, since both caches hold the very same routing entries, therefore do provide the same information. In that sense, EVL_NEIGH_PERMANENT seems redundant since it fundamentally means ATF_PERM. Documenting ATF_PERM in the API docs while aliasing EVL_NEIGH_PERMANENT to ATF_PERM for backward compat may be a reasonable trade-off, until we can drop the former in order to further reduce the namespace pollution.
Signature · 1 line
Philippe.
Quoted 94 lines · Philippe Gerum
Quoted 88 lines · Hannes Diethelm
Am 20.07.26 um 16:27 schrieb Philippe Gerum:
Quoted 73 lines
Quoted 26 lines · Hannes Diethelm
Am 04.07.26 um 19:52 schrieb Philippe Gerum:
Quoted 15 lines
Quoted 10 lines · Hannes Diethelm
Am 20.06.26 um 19:28 schrieb Philippe Gerum:
Quoted 8 lines
- Honor
MSG_PROBEforoob_sendmsg(), so that only the general call sanity and route resolution to the destination host is performed when set in the request flags, without actually sending any data. On success of such call, we would know that the routing information is readily available from the oob caches, no offload to in-band would have happened if we had not given this flag. The absence of routing information to the destination from some oob cache would yield a specific error, so that the caller may decide what to do next.Quoted 3 lines
It seams the flag
MSG_PROBEis kernel only? I did not find any occurrence in/usr/includeor in libevl.Quoted 9 lines
Yep, my bad. Using
MSG_PROBEis not the right way, since that would conflict withMSG_PROXYin userland which has a totally different meaning. I have revisited the implementation, simplifying it actually:
since the evl netstack already accepts zero-sized messages, sending such a datagram to the UDP layer now amounts to returning early with the address resolution status, short-circuiting the logic before the actual transmission happens.
IOW, passing a NULL or empty iov into the msghdr struct does whatMSG_PROBEwas intended to do.Quoted 9 lines
I tested this variant. It works. But I wonder:
If I use:
ret =oob_sendmsg(s, &msghdr, NULL, 0); errno is set to EHOSTUNREACH If I use:
ret =oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT); errno is set to EWOULDBLOCKIs this intended? EHOSTUNREACH is like halve correct. Yes, the host can not be reached but only due to no ARP request is sent.
EHOSTUNREACH was intended as a way to distinguish from EWOULDBLOCK wrt
lack of buffer space for the outgoing message, this code was the only option close enough to the idea to be conveyed available from the errno list that would not conflict with other situations. Now, since such probing mode needs no message space in the first place, this is guarding against the impossible, which does not make sense. Returning-EWOULDBLOCKin both cases above would still be
practical. e.g. something along these lines:kernel/evl/net/ipv4/udp.c +9
-7@@ -417,22 +417,22 @@ static ssize_t send_udp(struct evl_socket *esk,* address. */ ret =find_egress_path (esk, daddr, &ert, &earp, &pseudo_earp, msg_flags); -if (ret == -EMULTIHOP ) -return ret;/* MSG_DONTROUTE cannot be honored. */ -if (ret) { +if (ret != -EHOSTUNREACH ) +return ret; + +if (datalen ==0 ) +return -EWOULDBLOCK ;/* Address probe failed. */ +/* -* No route known from the front cache - bummer. We -* may have to offload the transmit operation to the -* in-band stack, unless only probing or MSG_DONTWAIT -* is set. +* We have a message to send but no route was found in +* the front cache - bummer. We may have to offload +* the transmit operation to the in-band stack, unless kernel/evl/net/ipv4/udp.c static ssize_t send_udp(struct evl_socket *esk, · line 438
Quoted 10 lines
kernel/evl/net/ipv4/udp.c +1
-2+* only probing or MSG_DONTWAIT is set. */ if (msg_flags &MSG_DONTWAIT )return -EWOULDBLOCK ; - if (datalen == 0) -return ret; -/* * We always charge the socket even when offloading to * the in-band stack although we won't consume any I think in this case, it is fine as it is. You also won't expect EWOULDBLOCK or EAGAIN as long as you don't set
MSG_DONTWAIT.Quoted 3 lines
But now there are two ways of probing. Either with or withouth
MSG_DONTWAITthat behave slightly different. Might be just supportMSG_DONTWAIT-> EWOULDBLOCK and drop the other
variant?
Quoted 6 lines
You mean detect a probing request when receiving
MSG_DONTWAITand a zero-sized buffer? That is an option. Another option would be to always return EHOSTUNREACH/??? on failed probe regardless of whetherMSG_DONTWAITis set. I would preferably go for the second option iff we can settle on a unconfusing, unambiguous error status.
Ok, I believe that the best option is to go back to an explicit operation flag for probing eventually, because all other options seem confusing. Therefore MSG_PROBE handling was resurrected [1] in the udp layer, and EHOSTUNREACH is unambiguously used to denote a failed probe if MSG_PROBE is set.
As you pointed out, the MSG_PROBE definition is missing from the common socket.h bits in user-space, but the same flag value is defined as MSG_PROXY in *libc headers, which looks like obsolete. Anyway, to address this, MSG_PROBE was added to the evl/uapi bits with enough guards to prevent conflicts or mismatches. It defines the constant expected by the kernel, which is very unlikely to change since this belongs to the part of the ABI which is written in stone.
The commit log states the possible outcomes depending on the operation flags and/or size of the message passed to oob_sendmsg().
[1]
https://gitlab.com/Xenomai/xenomai4/linux-evl/-/commit/566f032dc3bc4c3e945e94887b7f57cd6e57dfb6
Signature · 1 line
Philippe.
Quoted 5 lines · Hannes Diethelm
Am 20.07.26 um 16:27 schrieb Philippe Gerum:
An option would be either to support of
oob_ioctl()for SIOCGARP. Or create something likeevl_net_routeinfo(s, addr)returning flags would make probing obsolete.
Not entirely. The issue with solely having SIOCGARP or any probe-only explicit request is that you would have to pair two syscalls at each transmit at least, one to probe for the sender address before possibly soliciting that peer if absent from the oob cache, another one for sending the message eventually. i.e., for every packet:
ioctl(SIOCGARP)
!ATF_COM? -> evl_net_solicit()
oob_sendmsg(..., 0)OTOH, with the latest attempt to address this issue, we can send a probe-only request by passing MSG_PROBE (EHOSTUNREACH), or a request that does not attempt to defer transmit to the inband stage on probe failure by passing MSG_DONTWAIT (EWOULDBLOCK). i.e., for every packet
redo:
oob_sendmsg(..., MSG_DONTWAIT)
EWOULDBLOCK?
oob_sendmsg(..., MSG_PROBE)
EHOSTUNREACH?
evl_net_solicit()
goto redo
otherwise assume ENOMEM
otherwise all done
IOW, using a proper combination of MSG_DONTWAIT and MSG_PROBE in the right sequence would either succeed to send the packet immediately on the first oob_sendmsg() call, otherwise fail on memory shortage or missing route from the oob cache. In the latter case, which could only happen once for each new peer under normal circumstances, we could disambiguate the EWOULDBLOCK status using an explicit probe.
A better way to do this in a single step unambiguously would require the addition of another operation flag, like MSG_DONTDEFER, preventing the deferral to inband on failed probe and causing oob_sendmsg() to return with a specific error code. e.g. something as simple as the following would cover all requirements:
oob_sendmsg(..., MSG_DONTDEFER) -> EADDRNOTAVAIL on failed probe.
Now, we might also piggyback off of MSG_OOB instead of defining yet another operation flag, as a way to say "oob only, don't relay to inband", but I'm still pondering whether this would be nicely witty or utterly confusing..
Signature · 1 line
Philippe.
Am 21.07.26 um 22:15 schrieb Philippe Gerum:
Quoted 14 lines
Quoted 5 lines · Hannes Diethelm
Am 20.07.26 um 16:27 schrieb Philippe Gerum:
An option would be either to support of
oob_ioctl()for SIOCGARP. Or create something likeevl_net_routeinfo(s, addr)returning flags would make probing obsolete.Not entirely. The issue with solely having SIOCGARP or any probe-only explicit request is that you would have to pair two syscalls at each transmit at least, one to probe for the sender address before possibly soliciting that peer if absent from the oob cache, another one for sending the message eventually. i.e., for every packet:
Quoted 3 lines
ioctl(SIOCGARP)
!ATF_COM? -> evl_net_solicit()
oob_sendmsg(..., 0)You are right, that is an unneeded syscall as long as you don't keep a list of clients as before. This can remove the need from calling evl_net_solicit() in the case the client is already in ARP and permanent but this will be the only change and won't help that much.
Quoted 29 lines
OTOH, with the latest attempt to address this issue, we can send a probe-only request by passing
MSG_PROBE(EHOSTUNREACH), or a request that does not attempt to defer transmit to the inband stage on probe failure by passingMSG_DONTWAIT(EWOULDBLOCK). i.e., for every packetredo:
oob_sendmsg(..., MSG_DONTWAIT)
EWOULDBLOCK?
oob_sendmsg(..., MSG_PROBE)
EHOSTUNREACH?
evl_net_solicit()
goto redo
otherwise assume ENOMEM
otherwise all doneIOW, using a proper combination of
MSG_DONTWAITandMSG_PROBEin the right sequence would either succeed to send the packet immediately on the firstoob_sendmsg()call, otherwise fail on memory shortage or missing route from the oob cache. In the latter case, which could only happen once for each new peer under normal circumstances, we could disambiguate the EWOULDBLOCK status using an explicit probe.A better way to do this in a single step unambiguously would require the addition of another operation flag, like
MSG_DONTDEFER, preventing the deferral to inband on failed probe and causingoob_sendmsg()to return with a specific error code. e.g. something as simple as the following would cover all requirements:
Quoted 7 lines
oob_sendmsg(..., MSG_DONTDEFER) -> EADDRNOTAVAIL on failed probe.Now, we might also piggyback off of
MSG_OOBinstead of defining yet another operation flag, as a way to say "oob only, don't relay to inband", but I'm still pondering whether this would be nicely witty or utterly confusing..
Yes, this is a variant. I was not aware that MSG_DONTWAIT -> EWOULDBLOCK can also mean ENOMEM.
Now after considering all options, i start to prefer the variant before this patch, keeping a list of already solicit'ed clients. It is straight forward, simple and fast. And you know exactly when to expect an in band call. Might be we should just drop this patch instead of adding unnecessary complexity for example code?
With MSG_DONTWAIT/MSG_PROBE you are not sure if the ARP entry is permanent. That means if you are unlucky, a few packages are sent to the client before EHOSTUNREACH is returned which could be bad for real time.
If you have a real time server and clients, I imagine clients would expect the first response to be delayed due to initialization / ARP but every following response to be in time.
With ioctl(SIOCGARP) or evl_net_routeinfo(), a list of checked client's would also be needed due to it would be wasteful to call the kernel each time before sending a package.
MSG_DONTWAIT / MSG_PROBE could still be useful in certain applications, for example when you have a separate thread that handles evl_net_solicit(), so you can just call oob_sendmsg() until it succeeds without needing to signal back.
This would be something like this:
server_thread:
if(!is_in_list(addr))
write_solicit_queue(addr)
add_to_list(addr)
oob_sendmsg(..., MSG_DONTWAIT)
EWOULDBLOCK?
oob_sendmsg(..., MSG_PROBE)
EHOSTUNREACH?
skip or mark for retry depending on the application
otherwise assume ENOMEM -> generate error
otherwise all donesolicit_thread:
while(true)
addr=read_solicit_queue()
evl_net_solicit(addr)
I also prefer using an explicit MSG_PROBE instead of an implicit zero size message.
For ioctl(SIOCGARP) or evl_net_routeinfo(), I don't see a good use-case after all. You can just use evl_net_solicit(), it probably won't hurt if you do it once to much and you have to implement the case when evl_net_solicit() is needed anyway and take the timing for this into account.
A message that is not in the archive (sent off-list or not yet received): [email protected]
A message that is not in the archive (sent off-list or not yet received): [email protected]
A message that is not in the archive (sent off-list or not yet received): [email protected]
Am 22.07.26 um 20:36 schrieb Philippe Gerum:
Quoted 119 lines
Quoted 80 lines · Philippe Gerum
Quoted 73 lines · Hannes Diethelm
Am 22.07.26 um 00:36 schrieb Hannes Diethelm:
Quoted 62 lines
Am 21.07.26 um 22:15 schrieb Philippe Gerum:
Quoted 15 lines
Quoted 6 lines · Hannes Diethelm
Am 20.07.26 um 16:27 schrieb Philippe Gerum:
An option would be either to support of
oob_ioctl()for SIOCGARP. Or create something likeevl_net_routeinfo(s, addr)returning flags would make probing obsolete.Not entirely. The issue with solely having SIOCGARP or any probe-only explicit request is that you would have to pair two syscalls at each transmit at least, one to probe for the sender address before possibly soliciting that peer if absent from the oob cache, another one for sending the message eventually. i.e., for every packet:
Quoted 3 lines
ioctl(SIOCGARP)
!ATF_COM? ->evl_net_solicit()
oob_sendmsg(..., 0)You are right, that is an unneeded syscall as long as you don't keep a
list of clients as before. This can remove the need from callingevl_net_solicit()in the case the client is already in ARP and permanent but
this will be the only change and won't help that much.Quoted 29 lines
OTOH, with the latest attempt to address this issue, we can send a probe-only request by passing
MSG_PROBE(EHOSTUNREACH), or a request that does not attempt to defer transmit to the inband stage on probe failure by passingMSG_DONTWAIT(EWOULDBLOCK). i.e., for every packetredo:
oob_sendmsg(..., MSG_DONTWAIT)
EWOULDBLOCK?
oob_sendmsg(..., MSG_PROBE)
EHOSTUNREACH?
evl_net_solicit()
goto redo
otherwise assume ENOMEM
otherwise all doneIOW, using a proper combination of
MSG_DONTWAITandMSG_PROBEin the right sequence would either succeed to send the packet immediately on the firstoob_sendmsg()call, otherwise fail on memory shortage or missing route from the oob cache. In the latter case, which could only happen once for each new peer under normal circumstances, we could disambiguate the EWOULDBLOCK status using an explicit probe.A better way to do this in a single step unambiguously would require the addition of another operation flag, like
MSG_DONTDEFER, preventing the deferral to inband on failed probe and causingoob_sendmsg()to return with a specific error code. e.g. something as simple as the following would cover all requirements:Quoted 7 lines
oob_sendmsg(..., MSG_DONTDEFER)-> EADDRNOTAVAIL on failed probe.Now, we might also piggyback off of
MSG_OOBinstead of defining yet another operation flag, as a way to say "oob only, don't relay to inband", but I'm still pondering whether this would be nicely witty or utterly confusing..Quoted 9 lines
Yes, this is a variant. I was not aware that
MSG_DONTWAIT-> EWOULDBLOCK
can also mean ENOMEM.
Now after considering all options, i start to prefer the variant before
this patch, keeping a list of already solicit'ed clients. It is straight forward, simple and fast. And you know exactly when to expect an in band call. Might be we should just drop this patch instead of adding unnecessary complexity for example code?Quoted 4 lines
BTW: With drop this patch I don't mean the one already in but this one:
[PATCH] tidbits: net-udp: useMSG_DONTWAITfor server mode
https://lore.kernel.org/xenomai/[email protected]/T/#uQuoted 4 lines
Got it, it may be worth waiting for the dust to settle in the core regarding the probing issue before adapting this tidbit anyway. I'm going to consolidate the points we have discussed so far, in order to come up with an implementation that satisfies all the requirements.
Here is a new proposal that should cover all the cases we discussed so far (next/v6.12.y-cip-evl-rebase branch). A new flag is introduced, namely
MSG_STEADY, which tells the core to only accept cache lookups that deliver permanent ARP records. This flag can be combined withMSG_PROBEandMSG_DONTWAITfor extended semantics. Meanwhile, a zero-sized message is still accepted in order to request a mere address probe without having to pass any meaningful data. This goes like this:Function: probe-only
oob_sendmsg(s, &msghdr, NULL, MSG_PROBE) -> EADDRNOTAVAIL if no peer address foundFunction: probe-only, permanent peer address required
oob_sendmsg(s, &msghdr, NULL, MSG_PROBE|MSG_STEADY)-> EADDRNOTAVAIL if no permanent peer address found (also upon match of aging ARP record)Function: send, no deferral (to inband) allowed
oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT) -> EADDRNOTAVAIL if no peer address found -> EWOULDBLOCK on memory shortageFunction: send, no deferral (to inband) allowed, permanent peer address required
oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT|MSG_STEADY) -> EADDRNOTAVAIL if no permanent peer address found (also upon match of aging ARP record) -> EWOULDBLOCK on memory shortage
Quoted 6 lines
Function: send, permanent peer address required
oob_sendmsg(s, &msghdr, NULL, MSG_STEADY)
-> EADDRNOTAVAIL if no permanent peer address found
(also upon match of aging ARP record)
Looks promising. This also means after oob_sendmsg(..., MSG_DONTWAIT|MSG_STEADY)
no second call oob_sendmsg(..., MSG_PROBE) is needed to handle -ENOMEM right?
oob_sendmsg(s, &msghdr, NULL, 0) is the only case with deferral to inband as it looks in udp.c:421...423?
if (ret != -EADDRNOTAVAIL ||
msg_flags & (MSG_PROBE|MSG_DONTWAIT|MSG_STEADY)) return ret;
I have to test it and will send an update to the oob_net_udp patch soon.
BTW: CC [email protected] got lost in my last email. I added it again.
Quoted 118 lines · Hannes Diethelm
Am 22.07.26 um 20:36 schrieb Philippe Gerum:
Quoted 111 lines
Quoted 80 lines · Philippe Gerum
Quoted 73 lines · Hannes Diethelm
Am 22.07.26 um 00:36 schrieb Hannes Diethelm:
Quoted 62 lines
Am 21.07.26 um 22:15 schrieb Philippe Gerum:
Quoted 15 lines
Quoted 6 lines · Hannes Diethelm
Am 20.07.26 um 16:27 schrieb Philippe Gerum:
An option would be either to support of
oob_ioctl()for SIOCGARP. Or create something likeevl_net_routeinfo(s, addr)returning flags would make probing obsolete.Not entirely. The issue with solely having SIOCGARP or any probe-only explicit request is that you would have to pair two syscalls at each transmit at least, one to probe for the sender address before possibly soliciting that peer if absent from the oob cache, another one for sending the message eventually. i.e., for every packet:
Quoted 3 lines
ioctl(SIOCGARP)
!ATF_COM? ->evl_net_solicit()
oob_sendmsg(..., 0)You are right, that is an unneeded syscall as long as you don't keep a
list of clients as before. This can remove the need from callingevl_net_solicit()in the case the client is already in ARP and permanent but
this will be the only change and won't help that much.Quoted 29 lines
OTOH, with the latest attempt to address this issue, we can send a probe-only request by passing
MSG_PROBE(EHOSTUNREACH), or a request that does not attempt to defer transmit to the inband stage on probe failure by passingMSG_DONTWAIT(EWOULDBLOCK). i.e., for every packetredo:
oob_sendmsg(..., MSG_DONTWAIT)
EWOULDBLOCK?
oob_sendmsg(..., MSG_PROBE)
EHOSTUNREACH?
evl_net_solicit()
goto redo
otherwise assume ENOMEM
otherwise all doneIOW, using a proper combination of
MSG_DONTWAITandMSG_PROBEin the right sequence would either succeed to send the packet immediately on the firstoob_sendmsg()call, otherwise fail on memory shortage or missing route from the oob cache. In the latter case, which could only happen once for each new peer under normal circumstances, we could disambiguate the EWOULDBLOCK status using an explicit probe.A better way to do this in a single step unambiguously would require the addition of another operation flag, like
MSG_DONTDEFER, preventing the deferral to inband on failed probe and causingoob_sendmsg()to return with a specific error code. e.g. something as simple as the following would cover all requirements:Quoted 7 lines
oob_sendmsg(..., MSG_DONTDEFER)-> EADDRNOTAVAIL on failed probe.Now, we might also piggyback off of
MSG_OOBinstead of defining yet another operation flag, as a way to say "oob only, don't relay to inband", but I'm still pondering whether this would be nicely witty or utterly confusing..Quoted 9 lines
Yes, this is a variant. I was not aware that
MSG_DONTWAIT-> EWOULDBLOCK
can also mean ENOMEM.
Now after considering all options, i start to prefer the variant before
this patch, keeping a list of already solicit'ed clients. It is straight forward, simple and fast. And you know exactly when to expect an in band call. Might be we should just drop this patch instead of adding unnecessary complexity for example code?Quoted 4 lines
BTW: With drop this patch I don't mean the one already in but this one:
[PATCH] tidbits: net-udp: useMSG_DONTWAITfor server mode
https://lore.kernel.org/xenomai/[email protected]/T/#uQuoted 4 lines
Got it, it may be worth waiting for the dust to settle in the core regarding the probing issue before adapting this tidbit anyway. I'm going to consolidate the points we have discussed so far, in order to come up with an implementation that satisfies all the requirements.
Here is a new proposal that should cover all the cases we discussed so
far (next/v6.12.y-cip-evl-rebase branch). A new flag is introduced, namelyMSG_STEADY, which tells the core to only accept cache lookups that deliver permanent ARP records. This flag can be combined withMSG_PROBEandMSG_DONTWAITfor extended semantics. Meanwhile, a zero-sized message is still accepted in order to request a mere address probe without having to pass any meaningful data. This goes like this:
Function: probe-onlyoob_sendmsg(s, &msghdr, NULL, MSG_PROBE) -> EADDRNOTAVAIL if no peer address found Function: probe-only, permanent peer address required oob_sendmsg(s, &msghdr, NULL, MSG_PROBE|MSG_STEADY) -> EADDRNOTAVAIL if no permanent peer address found (also upon match of aging ARP record) Function: send, no deferral (to inband) allowed oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT) -> EADDRNOTAVAIL if no peer address found -> EWOULDBLOCK on memory shortage Function: send, no deferral (to inband) allowed, permanent peer address required oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT|MSG_STEADY) -> EADDRNOTAVAIL if no permanent peer address found (also upon match of aging ARP record) -> EWOULDBLOCK on memory shortageQuoted 5 lines
Function: send, permanent peer address required
oob_sendmsg(s, &msghdr, NULL, MSG_STEADY)
-> EADDRNOTAVAIL if no permanent peer address found
(also upon match of aging ARP record)
Quoted 4 lines
Looks promising. This also means after
oob_sendmsg(..., MSG_DONTWAIT|MSG_STEADY)
no second calloob_sendmsg(..., MSG_PROBE)is needed to handle-ENOMEMright?
Correct.
Quoted 8 lines
oob_sendmsg(s, &msghdr, NULL, 0)is the only case with deferral to inband as it looks in udp.c:421...423?
if (ret !=-EADDRNOTAVAIL||msg_flags & (MSG_PROBE|MSG_DONTWAIT|MSG_STEADY)) return ret;
Open question: does it still make sense to have MSG_DONTWAIT prevent the deferral to the inband stage, now that MSG_STEADY can be used for this? i.e.:
Quoted 4 lines
Quoted 4 lines
Function: send, no deferral (to inband) allowed
oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT) -> EADDRNOTAVAIL if no peer address found -> EWOULDBLOCK on memory shortage
Not considering MSG_DONTWAIT (alone) anymore as a way to prevent deferral would make MSG_STEADY the only option for this, tightening the requirement from "any cached destination address" to "permanent destination address". This being said:
- If $user wants a sure shot wrt to oob transmission, then a permanent address is most likely desired, in which case MSG_STEADY should be given. Otherwise, what would be the point of sending to a peer using an aging ARP record _and_ being concerned about timeliness and deterministic behavior in general (therefore denying deferral)?
- As it is, the implementation somewhat abuses the dont-wait semantics. As a matter of fact, inband deferral won't make the caller wait at all, only the packet will be delayed until the regular netstack picks it. Meanwhile, EWOULDBLOCK is conventionally paired with MSG_DONTWAIT, not EADDRNOTAVAIL.
Quoted 2 lines
I have to test it and will send an update to the
oob_net_udppatch soon.
Thanks.
Quoted 2 lines
BTW: CC [email protected] got lost in my last email. I added it again.
Signature · 1 line
Philippe.
Am 23.07.26 um 09:47 schrieb Philippe Gerum:
Quoted 156 lines
Quoted 118 lines · Hannes Diethelm
Am 22.07.26 um 20:36 schrieb Philippe Gerum:
Quoted 111 lines
Quoted 80 lines · Philippe Gerum
Quoted 73 lines · Hannes Diethelm
Am 22.07.26 um 00:36 schrieb Hannes Diethelm:
Quoted 62 lines
Am 21.07.26 um 22:15 schrieb Philippe Gerum:
Quoted 15 lines
Quoted 6 lines · Hannes Diethelm
Am 20.07.26 um 16:27 schrieb Philippe Gerum:
An option would be either to support of
oob_ioctl()for SIOCGARP. Or create something likeevl_net_routeinfo(s, addr)returning flags would make probing obsolete.Not entirely. The issue with solely having SIOCGARP or any probe-only explicit request is that you would have to pair two syscalls at each transmit at least, one to probe for the sender address before possibly soliciting that peer if absent from the oob cache, another one for sending the message eventually. i.e., for every packet:
Quoted 3 lines
ioctl(SIOCGARP)
!ATF_COM? ->evl_net_solicit()
oob_sendmsg(..., 0)You are right, that is an unneeded syscall as long as you don't keep a
list of clients as before. This can remove the need from callingevl_net_solicit()in the case the client is already in ARP and permanent but
this will be the only change and won't help that much.Quoted 29 lines
OTOH, with the latest attempt to address this issue, we can send a probe-only request by passing
MSG_PROBE(EHOSTUNREACH), or a request that does not attempt to defer transmit to the inband stage on probe failure by passingMSG_DONTWAIT(EWOULDBLOCK). i.e., for every packetredo:
oob_sendmsg(..., MSG_DONTWAIT)
EWOULDBLOCK?
oob_sendmsg(..., MSG_PROBE)
EHOSTUNREACH?
evl_net_solicit()
goto redo
otherwise assume ENOMEM
otherwise all doneIOW, using a proper combination of
MSG_DONTWAITandMSG_PROBEin the right sequence would either succeed to send the packet immediately on the firstoob_sendmsg()call, otherwise fail on memory shortage or missing route from the oob cache. In the latter case, which could only happen once for each new peer under normal circumstances, we could disambiguate the EWOULDBLOCK status using an explicit probe.A better way to do this in a single step unambiguously would require the addition of another operation flag, like
MSG_DONTDEFER, preventing the deferral to inband on failed probe and causingoob_sendmsg()to return with a specific error code. e.g. something as simple as the following would cover all requirements:Quoted 7 lines
oob_sendmsg(..., MSG_DONTDEFER)-> EADDRNOTAVAIL on failed probe.Now, we might also piggyback off of
MSG_OOBinstead of defining yet another operation flag, as a way to say "oob only, don't relay to inband", but I'm still pondering whether this would be nicely witty or utterly confusing..Quoted 9 lines
Yes, this is a variant. I was not aware that
MSG_DONTWAIT-> EWOULDBLOCK
can also mean ENOMEM.
Now after considering all options, i start to prefer the variant before
this patch, keeping a list of already solicit'ed clients. It is straight forward, simple and fast. And you know exactly when to expect an in band call. Might be we should just drop this patch instead of adding unnecessary complexity for example code?Quoted 4 lines
BTW: With drop this patch I don't mean the one already in but this one:
[PATCH] tidbits: net-udp: useMSG_DONTWAITfor server mode
https://lore.kernel.org/xenomai/[email protected]/T/#uQuoted 4 lines
Got it, it may be worth waiting for the dust to settle in the core regarding the probing issue before adapting this tidbit anyway. I'm going to consolidate the points we have discussed so far, in order to come up with an implementation that satisfies all the requirements.
Here is a new proposal that should cover all the cases we discussed so
far (next/v6.12.y-cip-evl-rebase branch). A new flag is introduced, namelyMSG_STEADY, which tells the core to only accept cache lookups that deliver permanent ARP records. This flag can be combined withMSG_PROBEandMSG_DONTWAITfor extended semantics. Meanwhile, a zero-sized message is still accepted in order to request a mere address probe without having to pass any meaningful data. This goes like this:
Function: probe-onlyoob_sendmsg(s, &msghdr, NULL, MSG_PROBE) -> EADDRNOTAVAIL if no peer address foundFunction: probe-only, permanent peer address required
oob_sendmsg(s, &msghdr, NULL, MSG_PROBE|MSG_STEADY)
-> EADDRNOTAVAIL if no permanent peer address found
(also upon match of aging ARP record)
Function: send, no deferral (to inband) allowedoob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT) -> EADDRNOTAVAIL if no peer address found -> EWOULDBLOCK on memory shortage Function: send, no deferral (to inband) allowed, permanent peer address required oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT|MSG_STEADY) -> EADDRNOTAVAIL if no permanent peer address found (also upon match of aging ARP record) -> EWOULDBLOCK on memory shortageQuoted 5 lines
Function: send, permanent peer address required
oob_sendmsg(s, &msghdr, NULL, MSG_STEADY)
-> EADDRNOTAVAIL if no permanent peer address found
(also upon match of aging ARP record)Quoted 4 lines
Looks promising. This also means after
oob_sendmsg(..., MSG_DONTWAIT|MSG_STEADY)
no second calloob_sendmsg(..., MSG_PROBE)is needed to handle-ENOMEMright?Correct.
Quoted 8 lines
oob_sendmsg(s, &msghdr, NULL, 0)is the only case with deferral to inband as it looks in udp.c:421...423?
if (ret !=-EADDRNOTAVAIL||msg_flags & (MSG_PROBE|MSG_DONTWAIT|MSG_STEADY)) return ret;Open question: does it still make sense to have
MSG_DONTWAITprevent the deferral to the inband stage, now thatMSG_STEADYcan be used for this? i.e.:Quoted 4 lines
Quoted 4 lines
Function: send, no deferral (to inband) allowed
oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT) -> EADDRNOTAVAIL if no peer address found -> EWOULDBLOCK on memory shortageNot considering
MSG_DONTWAIT(alone) anymore as a way to prevent deferral would makeMSG_STEADYthe only option for this, tightening the requirement from "any cached destination address" to "permanent destination address". This being said:- If $user wants a sure shot wrt to oob transmission, then a permanent address is most likely desired, in which case
MSG_STEADYshould be given. Otherwise, what would be the point of sending to a peer using an aging ARP record _and_ being concerned about timeliness and deterministic behavior in general (therefore denying deferral)?
Quoted 6 lines
- As it is, the implementation somewhat abuses the dont-wait semantics. As a matter of fact, inband deferral won't make the caller wait at all, only the packet will be delayed until the regular netstack picks it. Meanwhile, EWOULDBLOCK is conventionally paired with
MSG_DONTWAIT, not EADDRNOTAVAIL.
MSG_DONTWAIT denying inband was only introduced in 644d55e78e8e, right?
In this case I agree. Either you'd like to stay out of band now and in the future -> MSG_STEADY or you don't care -> 0.
Otherwise, it might break already existing applications.
Quoted 8 lines
Quoted 2 lines
I have to test it and will send an update to the
oob_net_udppatch soon.Thanks.
Quoted 2 lines
BTW: CC [email protected] got lost in my last email. I added it again.
PATCH 1/1 tidbits: net-udp: use MSG_STABLE for server mode
MSG_STABLE is now supported, so it is used to check if a client is already in ARP cache and permanent. If not, use evl_net_solicit() and try again.
--- tidbits/oob-net-udp.c | 72 +++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 40 deletions(-)
tidbits/oob-net-udp.c +32 -40
@@ -238,9 +238,6 @@ static void server(int s, const char *text, int mcount,ssize_t ret;char *tbuf;char rbuf[16384 ]; -in_addr_t addr_list[SERVER_ADDR_LIST_SIZE ] = {}; -size_t addr_list_fill =0 ; -bool solicit_done; tlen = (strlen (text) +1 ) * mcount; tbuf =malloc (tlen);
@@ -281,39 +278,6 @@ static void server(int s, const char *text, int mcount,evl_printf (" (TRUNCATED)" );evl_printf (": %.*s\n" , (int )ret, rbuf); -/* -* We need to call evl_net_solicit for each new client -* once before sending data. This demotes the caller to the -* in-band stage for the first response. -* If this is not done and the ARP address is not -* yet in cache or garbage-collected, oob_sendmsg -* will return EINPROGRESS on start or during runtime. -*/ - solicit_done = false; -for (size_t i =0 ; i < addr_list_fill && !solicit_done; i++) { -if (addr_list[i] == _addr.sin_addr.s_addr) { - solicit_done = true; - } - } -if (!solicit_done) { -if (verbosity) { -char ip_str[INET_ADDRSTRLEN +1 ]; -inet_ntop (AF_INET , &(_addr.sin_addr), ip_str,sizeof (ip_str)); -evl_printf ("== client %s first seen: evl_net_solicit\n" , ip_str); - } - ret =evl_net_solicit (s, (const struct sockaddr *)&_addr, -EVL_NEIGH_PERMANENT ); -if (ret) -error (1 , -ret,"evl_net_solicit()" ); - -if (addr_list_fill <SERVER_ADDR_LIST_SIZE ) { - addr_list[addr_list_fill] = _addr.sin_addr.s_addr; - addr_list_fill++; - }else { -error (1 ,EPERM ,"address list full" ); - } - } - iov.iov_base = tbuf; iov.iov_len = tlen; msghdr.msg_iov = &iov;
@@ -323,9 +287,37 @@ static void server(int s, const char *text, int mcount, msghdr.msg_name = &_addr; msghdr.msg_namelen =sizeof (_addr); msghdr.msg_flags =0 ; - ret =oob_sendmsg (s, &msghdr,NULL ,0 ); -if (ret <0 ) -error (1 , errno,"oob_sendmsg() failed" ); + ret =oob_sendmsg (s, &msghdr,NULL ,MSG_STEADY ); +if (ret <0 ) { +if (errno ==EADDRNOTAVAIL ) { +/* +* oob_sendmsg with MSG_STEADY returns EADDRNOTAVAIL if +* the MAC address is not yet in ARP cache or not permanent. +* We need to call evl_net_solicit in this case. +* This demotes the caller to the in-band stage for the first response. +* To probe if the MAC address is in the ARP cache without sending +* any data, oob_sendmsg() with msghdr.msg_iov = NULL +* and msghdr.msg_iovlen = 0 can be used. +*/ +if (verbosity) { +char ip_str[INET_ADDRSTRLEN +1 ]; +inet_ntop (AF_INET , &(_addr.sin_addr), ip_str,sizeof (ip_str)); +evl_printf ("== client %s not in ARP cache: evl_net_solicit\n" , + ip_str); + } + ret =evl_net_solicit (s, (const struct sockaddr *)&_addr, +EVL_NEIGH_PERMANENT ); +if (ret) +error (1 , -ret,"evl_net_solicit()" ); + +/* oob_sendmsg again, now it should succeed */ + ret =oob_sendmsg (s, &msghdr,NULL ,MSG_STEADY ); +if (ret <0 ) +error (1 , errno,"oob_sendmsg() failed" ); + }else { +error (1 , errno,"oob_sendmsg() failed" ); + } + }if (verbosity >1 )print_addr ("sent to" , &_addr); }
@@ -350,7 +342,7 @@ int main(int argc, char *argv[])struct sockaddr_in addr;const char *ip =NULL ;ssize_t ret; -useconds_t delay=1000000 ; +useconds_t delay =1000000 ;while ((c =getopt (argc, argv,"a:m:n:i:w:I:p:dsTRCSb" )) !=EOF ) {switch (c) {
Signature · 1 line
2.47.3
PATCH v2 1/1 tidbits: net-udp: use MSG_STEADY for server mode
MSG_STEADY is now supported, so it is used to check if a client is already in ARP cache and permanent. If not, use evl_net_solicit() and try again.
--- tidbits/oob-net-udp.c | 71 +++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 40 deletions(-)
tidbits/oob-net-udp.c +31 -40
@@ -238,9 +238,6 @@ static void server(int s, const char *text, int mcount,ssize_t ret;char *tbuf;char rbuf[16384 ]; -in_addr_t addr_list[SERVER_ADDR_LIST_SIZE ] = {}; -size_t addr_list_fill =0 ; -bool solicit_done; tlen = (strlen (text) +1 ) * mcount; tbuf =malloc (tlen);
@@ -281,39 +278,6 @@ static void server(int s, const char *text, int mcount,evl_printf (" (TRUNCATED)" );evl_printf (": %.*s\n" , (int )ret, rbuf); -/* -* We need to call evl_net_solicit for each new client -* once before sending data. This demotes the caller to the -* in-band stage for the first response. -* If this is not done and the ARP address is not -* yet in cache or garbage-collected, oob_sendmsg -* will return EINPROGRESS on start or during runtime. -*/ - solicit_done = false; -for (size_t i =0 ; i < addr_list_fill && !solicit_done; i++) { -if (addr_list[i] == _addr.sin_addr.s_addr) { - solicit_done = true; - } - } -if (!solicit_done) { -if (verbosity) { -char ip_str[INET_ADDRSTRLEN +1 ]; -inet_ntop (AF_INET , &(_addr.sin_addr), ip_str,sizeof (ip_str)); -evl_printf ("== client %s first seen: evl_net_solicit\n" , ip_str); - } - ret =evl_net_solicit (s, (const struct sockaddr *)&_addr, -EVL_NEIGH_PERMANENT ); -if (ret) -error (1 , -ret,"evl_net_solicit()" ); - -if (addr_list_fill <SERVER_ADDR_LIST_SIZE ) { - addr_list[addr_list_fill] = _addr.sin_addr.s_addr; - addr_list_fill++; - }else { -error (1 ,EPERM ,"address list full" ); - } - } - iov.iov_base = tbuf; iov.iov_len = tlen; msghdr.msg_iov = &iov;
@@ -323,9 +287,36 @@ static void server(int s, const char *text, int mcount, msghdr.msg_name = &_addr; msghdr.msg_namelen =sizeof (_addr); msghdr.msg_flags =0 ; - ret =oob_sendmsg (s, &msghdr,NULL ,0 ); -if (ret <0 ) -error (1 , errno,"oob_sendmsg() failed" ); + ret =oob_sendmsg (s, &msghdr,NULL ,MSG_STEADY ); +if (ret <0 ) { +if (errno ==EADDRNOTAVAIL ) { +/* +* oob_sendmsg with MSG_STEADY returns EADDRNOTAVAIL if +* the MAC address is not yet in ARP cache or not permanent. +* We need to call evl_net_solicit in this case. +* This demotes the caller to the in-band stage for the first response. +* To probe if the MAC address is in the ARP cache without sending +* any data, oob_sendmsg(..., MSG_STEADY|MSG_PROBE) can be used. +*/ +if (verbosity) { +char ip_str[INET_ADDRSTRLEN +1 ]; +inet_ntop (AF_INET , &(_addr.sin_addr), ip_str,sizeof (ip_str)); +evl_printf ("== client %s not in ARP cache: evl_net_solicit\n" , + ip_str); + } + ret =evl_net_solicit (s, (const struct sockaddr *)&_addr, +EVL_NEIGH_PERMANENT ); +if (ret) +error (1 , -ret,"evl_net_solicit()" ); + +/* oob_sendmsg again, now it should succeed */ + ret =oob_sendmsg (s, &msghdr,NULL ,MSG_STEADY ); +if (ret <0 ) +error (1 , errno,"oob_sendmsg() failed" ); + }else { +error (1 , errno,"oob_sendmsg() failed" ); + } + }if (verbosity >1 )print_addr ("sent to" , &_addr); }
@@ -350,7 +341,7 @@ int main(int argc, char *argv[])struct sockaddr_in addr;const char *ip =NULL ;ssize_t ret; -useconds_t delay=1000000 ; +useconds_t delay =1000000 ;while ((c =getopt (argc, argv,"a:m:n:i:w:I:p:dsTRCSb" )) !=EOF ) {switch (c) {
Signature · 1 line
2.47.3
Quoted 161 lines · Hannes Diethelm
Am 23.07.26 um 09:47 schrieb Philippe Gerum:
Quoted 143 lines
Quoted 118 lines · Hannes Diethelm
Am 22.07.26 um 20:36 schrieb Philippe Gerum:
Quoted 111 lines
Quoted 80 lines · Philippe Gerum
Quoted 73 lines · Hannes Diethelm
Am 22.07.26 um 00:36 schrieb Hannes Diethelm:
Quoted 62 lines
Am 21.07.26 um 22:15 schrieb Philippe Gerum:
Quoted 15 lines
Quoted 6 lines · Hannes Diethelm
Am 20.07.26 um 16:27 schrieb Philippe Gerum:
An option would be either to support of
oob_ioctl()for SIOCGARP. Or create something likeevl_net_routeinfo(s, addr)returning flags would make probing obsolete.Not entirely. The issue with solely having SIOCGARP or any probe-only explicit request is that you would have to pair two syscalls at each transmit at least, one to probe for the sender address before possibly soliciting that peer if absent from the oob cache, another one for sending the message eventually. i.e., for every packet:
Quoted 3 lines
ioctl(SIOCGARP)
!ATF_COM? ->evl_net_solicit()
oob_sendmsg(..., 0)You are right, that is an unneeded syscall as long as you don't keep a
list of clients as before. This can remove the need from callingevl_net_solicit()in the case the client is already in ARP and permanent but
this will be the only change and won't help that much.Quoted 29 lines
OTOH, with the latest attempt to address this issue, we can send a probe-only request by passing
MSG_PROBE(EHOSTUNREACH), or a request that does not attempt to defer transmit to the inband stage on probe failure by passingMSG_DONTWAIT(EWOULDBLOCK). i.e., for every packetredo:
oob_sendmsg(..., MSG_DONTWAIT)
EWOULDBLOCK?
oob_sendmsg(..., MSG_PROBE)
EHOSTUNREACH?
evl_net_solicit()
goto redo
otherwise assume ENOMEM
otherwise all doneIOW, using a proper combination of
MSG_DONTWAITandMSG_PROBEin the right sequence would either succeed to send the packet immediately on the firstoob_sendmsg()call, otherwise fail on memory shortage or missing route from the oob cache. In the latter case, which could only happen once for each new peer under normal circumstances, we could disambiguate the EWOULDBLOCK status using an explicit probe.A better way to do this in a single step unambiguously would require the addition of another operation flag, like
MSG_DONTDEFER, preventing the deferral to inband on failed probe and causingoob_sendmsg()to return with a specific error code. e.g. something as simple as the following would cover all requirements:Quoted 7 lines
oob_sendmsg(..., MSG_DONTDEFER)-> EADDRNOTAVAIL on failed probe.Now, we might also piggyback off of
MSG_OOBinstead of defining yet another operation flag, as a way to say "oob only, don't relay to inband", but I'm still pondering whether this would be nicely witty or utterly confusing..Quoted 9 lines
Yes, this is a variant. I was not aware that
MSG_DONTWAIT-> EWOULDBLOCK
can also mean ENOMEM.
Now after considering all options, i start to prefer the variant before
this patch, keeping a list of already solicit'ed clients. It is straight forward, simple and fast. And you know exactly when to expect an in band call. Might be we should just drop this patch instead of adding unnecessary complexity for example code?Quoted 4 lines
BTW: With drop this patch I don't mean the one already in but this one:
[PATCH] tidbits: net-udp: useMSG_DONTWAITfor server mode
https://lore.kernel.org/xenomai/[email protected]/T/#uQuoted 4 lines
Got it, it may be worth waiting for the dust to settle in the core regarding the probing issue before adapting this tidbit anyway. I'm going to consolidate the points we have discussed so far, in order to come up with an implementation that satisfies all the requirements.
Here is a new proposal that should cover all the cases we discussed so
far (next/v6.12.y-cip-evl-rebase branch). A new flag is introduced, namelyMSG_STEADY, which tells the core to only accept cache lookups that deliver permanent ARP records. This flag can be combined withMSG_PROBEandMSG_DONTWAITfor extended semantics. Meanwhile, a zero-sized message is still accepted in order to request a mere address probe without having to pass any meaningful data. This goes like this:
Function: probe-onlyoob_sendmsg(s, &msghdr, NULL, MSG_PROBE) -> EADDRNOTAVAIL if no peer address foundFunction: probe-only, permanent peer address required
oob_sendmsg(s, &msghdr, NULL, MSG_PROBE|MSG_STEADY)
-> EADDRNOTAVAIL if no permanent peer address found
(also upon match of aging ARP record)
Function: send, no deferral (to inband) allowedoob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT) -> EADDRNOTAVAIL if no peer address found -> EWOULDBLOCK on memory shortage Function: send, no deferral (to inband) allowed, permanent peer address required oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT|MSG_STEADY) -> EADDRNOTAVAIL if no permanent peer address found (also upon match of aging ARP record) -> EWOULDBLOCK on memory shortageQuoted 5 lines
Function: send, permanent peer address required
oob_sendmsg(s, &msghdr, NULL, MSG_STEADY)
-> EADDRNOTAVAIL if no permanent peer address found
(also upon match of aging ARP record)Quoted 4 lines
Looks promising. This also means after
oob_sendmsg(..., MSG_DONTWAIT|MSG_STEADY)
no second calloob_sendmsg(..., MSG_PROBE)is needed to handle-ENOMEMright?Correct.
Quoted 8 lines
oob_sendmsg(s, &msghdr, NULL, 0)is the only case with deferral to inband as it looks in udp.c:421...423?
if (ret !=-EADDRNOTAVAIL||msg_flags & (MSG_PROBE|MSG_DONTWAIT|MSG_STEADY)) return ret;Open question: does it still make sense to have
MSG_DONTWAITprevent the
deferral to the inband stage, now thatMSG_STEADYcan be used for this? i.e.:Quoted 4 lines
Quoted 4 lines
Function: send, no deferral (to inband) allowed
oob_sendmsg(s, &msghdr, NULL, MSG_DONTWAIT) -> EADDRNOTAVAIL if no peer address found -> EWOULDBLOCK on memory shortageQuoted 16 lines
Not considering
MSG_DONTWAIT(alone) anymore as a way to prevent deferral would makeMSG_STEADYthe only option for this, tightening the requirement from "any cached destination address" to "permanent destination address". This being said:
- If $user wants a sure shot wrt to oob transmission, then a permanent
address is most likely desired, in which caseMSG_STEADYshould be given. Otherwise, what would be the point of sending to a peer using an aging ARP record _and_ being concerned about timeliness and deterministic behavior in general (therefore denying deferral)?
- As it is, the implementation somewhat abuses the dont-wait semantics. As a matter of fact, inband deferral won't make the caller wait at all, only the packet will be delayed until the regular netstack picks it. Meanwhile, EWOULDBLOCK is conventionally paired withMSG_DONTWAIT, not EADDRNOTAVAIL.
Quoted 2 lines
MSG_DONTWAITdenying inband was only introduced in644d55e78e8e, right?
Yes, this can still be considered as a tentative, undocumented so far.
Quoted 4 lines
In this case I agree. Either you'd like to stay out of band now and in the future ->
MSG_STEADYor you don't care -> 0.Otherwise, it might break already existing applications.
Signature · 1 line
Philippe.
Am 23.07.26 um 09:47 schrieb Philippe Gerum:
Quoted 5 lines
Quoted 2 lines
I have to test it and will send an update to the
oob_net_udppatch soon.Thanks.
So, I sent the patch:
[PATCH] tidbits: net-udp: use MSG_STABLE for server mode
However, something is not working as I would expect:
The second oob_sendmsg() line 314 also returns EADDRNOTAVAIL after evl_net_solicit().
If I leave MSG_STEADY away for the second oob_sendmsg(), it works. However, I see 2x:
== client 192.168.122.1 not in ARP cache: evl_net_solicit From then on, it works forever, also if I restart oob-net-udp.
Looks like the NUD_PERMANENT is not synchronized immediately but only after the second evl_net_solicit(), shown by this test:
arp -i enp7s0 -d 192.168.122.1
oob-net-udp -d -i enp7s0 -a 0.0.0.0 -S -p 5201 -m "Server"
...
== client 192.168.122.1 not in ARP cache: evl_net_solicit ./oob-net-udp: oob_sendmsg() failed: Cannot assign requested address
oob-net-udp -d -i enp7s0 -a 0.0.0.0 -S -p 5201 -m "Server"
...
== client 192.168.122.1 not in ARP cache: evl_net_solicit ... now it works
oob-net-udp -d -i enp7s0 -a 0.0.0.0 -S -p 5201 -m "Server"
...no evl_net_solicit() any more, still works
linux-evl: 9ad9bf1c libevl: 115d45af
PATCH v2 1/1 tidbits: net-udp: solicit new client for server mode
In reply to Philippe GerumThis fixes random EINPROGRESS at init or during runtime.
Also correct whitespaces and make stdout more consistent.
--- tidbits/oob-net-udp.c | 58 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 8 deletions(-)
tidbits/oob-net-udp.c +50 -8
@@ -50,12 +50,12 @@ static void usage(void) }static void print_addr (char * text,struct sockaddr_in *addr){ -char ip_str[INET_ADDRSTRLEN +1 ]; -inet_ntop (AF_INET , &(addr->sin_addr), ip_str,sizeof (ip_str)); -evl_printf ("%s--------\n" , text); -evl_printf ("IP-Address: %s\n" , ip_str); -evl_printf ("Port: %d\n" ,ntohs (addr->sin_port)); -evl_printf ("Family: %d\n" , addr->sin_family); +char ip_str[INET_ADDRSTRLEN +1 ]; +inet_ntop (AF_INET , &(addr->sin_addr), ip_str,sizeof (ip_str)); +evl_printf ("== %s\n" , text); +evl_printf (" ip-address: %s\n" , ip_str); +evl_printf (" port: %d\n" ,ntohs (addr->sin_port)); +evl_printf (" family: %d\n" , addr->sin_family); } static void sender(int s, const char *text, int mcount,
@@ -226,6 +226,8 @@ static void client(int s, const char *text, int mcount,free (tbuf); } +#define SERVER_ADDR_LIST_SIZE 64 +static void server (int s,const char *text,int mcount,struct sockaddr_in *addr,int iter) {
@@ -236,6 +238,9 @@ static void server(int s, const char *text, int mcount,ssize_t ret;char *tbuf;char rbuf[16384 ]; +in_addr_t addr_list[SERVER_ADDR_LIST_SIZE ] = {}; +size_t addr_list_fill =0 ; +bool solicit_done; tlen = (strlen (text) +1 ) * mcount; tbuf =malloc (tlen);
@@ -276,6 +281,39 @@ static void server(int s, const char *text, int mcount,evl_printf (" (TRUNCATED)" );evl_printf (": %.*s\n" , (int )ret, rbuf); +/* +* We need to call evl_net_solicit for each new client +* once before sending data. This demotes the caller to the +* in-band stage for the first response. +* If this is not done and the ARP address is not +* yet in cache or garbage-collected, oob_sendmsg +* will return EINPROGRESS on start or during runtime. +*/ + solicit_done = false; +for (size_t i =0 ; i < addr_list_fill && !solicit_done; i++) { +if (addr_list[i] == _addr.sin_addr.s_addr) { + solicit_done = true; + } + } +if (!solicit_done) { +if (verbosity) { +char ip_str[INET_ADDRSTRLEN +1 ]; +inet_ntop (AF_INET , &(_addr.sin_addr), ip_str,sizeof (ip_str)); +evl_printf ("== client %s first seen: evl_net_solicit\n" , ip_str); + } + ret =evl_net_solicit (s, (const struct sockaddr *)&_addr, +EVL_NEIGH_PERMANENT ); +if (ret) +error (1 , -ret,"evl_net_solicit()" ); + +if (addr_list_fill <SERVER_ADDR_LIST_SIZE ) { + addr_list[addr_list_fill] = _addr.sin_addr.s_addr; + addr_list_fill++; + }else { +error (1 ,EPERM ,"address list full" ); + } + } + iov.iov_base = tbuf; iov.iov_len = tlen; msghdr.msg_iov = &iov;
@@ -433,7 +471,7 @@ int main(int argc, char *argv[])receiver (s, &addr, iter); }else if (mode ==CLIENT ) {if (verbosity) -printf ("== client mode (<= %s:%d)\n" , ip, port); +printf ("== client mode (<=> %s:%d)\n" , ip, port);/* * Guarantee a mere oob path from the first packet
@@ -449,8 +487,12 @@ int main(int argc, char *argv[])client (s, text, mcount, &addr, iter, delay); }else if (mode ==SERVER ) {if (verbosity) -printf ("== server mode (<= %s:%d)\n" , ip, port); +printf ("== server mode (<=> %s:%d)\n" , ip, port); +/* +* The server calls evl_net_solicit() internally +* for each new ip address. +*/ server (s, text, mcount, &addr, iter); }else {error (1 ,0 ,"Mode not implemented" );
Signature · 1 line
2.47.3
Quoted 8 lines · Hannes Diethelm
As discussed in [PATCH] tidbits: net-udp: add server and client mode,
evl_net_solicit()is also needed for each client in server mode.I went for option 2, it is not to lazy and I did not find any way to ask evl if an ARP is already done.
While testing, I also discovered that EINPROGRESS can appear after some time. I think this is due the kernel garbage-collects the ARP entry.
Correct, there is a timeout on ARP entries. You can make an entry permanent and therefore not considered for garbage collection by passing the EVL_NEIGH_PERMANENT flag to evl_net_solicit()
From the command line, the standard 'arp' command can be used to achieve what you would do with evl_net_solicit() as well, since in-band updates to the neighbour table are forwarded to evl.
Signature · 1 line
Philippe.