support multi-pthread per core
A message that is not in the archive (sent off-list or not yet received): [email protected]
A message that is not in the archive (sent off-list or not yet received): 1417589628-43666-1-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
cover support multi-pthread per core
The patch series contain the enhancements of EAL and fixes for libraries
to run multi-pthreads(either EAL or non-EAL thread) per physical core.
Two major changes list as below:
- Extend the core affinity of each EAL thread to 1:n.
Each lcore stands for a EAL thread rather than a logical core.
The change adds new EAL option to allow static lcore to cpuset assginment. Then a lcore(EAL thread) affinity to a cpuset, original 1:1 mapping is the special case.
- Fix the libraries to allow running on any non-EAL thread.
It fix the gaps running libraries in non-EAL thread(dynamic created by user). Each fix libraries take care the case of rte_lcore_id() >= RTE_MAX_LCORE.
Thanks a million for the comments from Konstantin, Bruce, Mirek and Stephen in RFC review.
*** BLURB HERE ***
Cunming Liang (15):
eal: add cpuset into per EAL thread lcore_config
eal: new eal option '--lcores' for cpu assignment
eal: add support parsing socket_id from cpuset
eal: new TLS definition and API declaration
eal: add eal_common_thread.c for common thread API
eal: add rte_gettid() to acquire unique system tid
eal: apply affinity of EAL thread by assigned cpuset
enic: fix re-define freebsd compile complain
malloc: fix the issue of SOCKET_ID_ANY
log: fix the gap to support non-EAL thread
eal: set _lcore_id and _socket_id to (-1) by default
eal: fix recursive spinlock in non-EAL thraed
mempool: add support to non-EAL thread
ring: add support to non-EAL thread
timer: add support to non-EAL thread
lib/librte_eal/bsdapp/eal/Makefile | 1 + lib/librte_eal/bsdapp/eal/eal.c | 13 +- lib/librte_eal/bsdapp/eal/eal_lcore.c | 14 ++ lib/librte_eal/bsdapp/eal/eal_memory.c | 2 + lib/librte_eal/bsdapp/eal/eal_thread.c | 76 +++--- lib/librte_eal/common/eal_common_launch.c | 1 - lib/librte_eal/common/eal_common_log.c | 17 +- lib/librte_eal/common/eal_common_options.c | 262 ++++++++++++++++++++- lib/librte_eal/common/eal_common_thread.c | 142 +++++++++++ lib/librte_eal/common/eal_options.h | 2 + lib/librte_eal/common/eal_thread.h | 66 ++++++ .../common/include/generic/rte_spinlock.h | 4 +- lib/librte_eal/common/include/rte_eal.h | 27 +++ lib/librte_eal/common/include/rte_lcore.h | 37 ++- lib/librte_eal/common/include/rte_log.h | 5 + lib/librte_eal/linuxapp/eal/Makefile | 4 + lib/librte_eal/linuxapp/eal/eal.c | 7 +- lib/librte_eal/linuxapp/eal/eal_lcore.c | 15 ++ lib/librte_eal/linuxapp/eal/eal_thread.c | 78 +++--- lib/librte_malloc/malloc_heap.h | 7 +- lib/librte_mempool/rte_mempool.h | 18 +- lib/librte_pmd_enic/enic.h | 1 + lib/librte_pmd_enic/enic_compat.h | 1 + lib/librte_ring/rte_ring.h | 10 +- lib/librte_timer/rte_timer.c | 40 +++- lib/librte_timer/rte_timer.h | 2 +- 26 files changed, 721 insertions(+), 131 deletions(-) create mode 100644 lib/librte_eal/common/eal_common_thread.c
Signature · 1 line
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 1421914598-2747-1-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
PATCH 1/15 eal: add cpuset into per EAL thread lcore_config
The patch adds 'cpuset' into per-lcore configure 'lcore_config[]', as the lcore no longer always 1:1 pinning with physical cpu. The lcore now stands for a EAL thread rather than a logical cpu.
It doesn't change the default behavior of 1:1 mapping, but allows to affinity the EAL thread to multiple cpus.
--- lib/librte_eal/bsdapp/eal/eal_lcore.c | 7 +++++++ lib/librte_eal/bsdapp/eal/eal_memory.c | 2 ++ lib/librte_eal/common/include/rte_lcore.h | 8 ++++++++ lib/librte_eal/linuxapp/eal/Makefile | 1 + lib/librte_eal/linuxapp/eal/eal_lcore.c | 8 ++++++++ 5 files changed, 26 insertions(+)
lib/librte_eal/bsdapp/eal/eal_lcore.c +7 -0
@@ -76,11 +76,18 @@ rte_eal_cpu_init(void)* ones and enable them by default. */ for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +/* init cpuset for per lcore config */ +CPU_ZERO (&lcore_config[lcore_id].cpuset); + lcore_config[lcore_id].detected = (lcore_id < ncpus);if (lcore_config[lcore_id].detected ==0 ) { config->lcore_role[lcore_id] =ROLE_OFF ;continue ; } + +/* By default, lcore 1:1 map to cpu id */ +CPU_SET (lcore_id, &lcore_config[lcore_id].cpuset); +/* By default, each detected core is enabled */ config->lcore_role[lcore_id] =ROLE_RTE ; lcore_config[lcore_id].core_id =cpu_core_id (lcore_id);
lib/librte_eal/bsdapp/eal/eal_memory.c +2 -0
@@ -45,6 +45,8 @@#include "eal_internal_cfg.h" #include "eal_filesystem.h" +/* avoid re-defined against with freebsd header */ +#undef PAGE_SIZE#define PAGE_SIZE (sysconf(_SC_PAGESIZE))/*
lib/librte_eal/common/include/rte_lcore.h +8 -0
@@ -50,6 +50,13 @@ extern "C" {#define LCORE_ID_ANY -1/**< Any lcore. */ +#if defined(__linux__) +typedef cpu_set_t rte_cpuset_t ; +#elif defined(__FreeBSD__) +#include <pthread_np.h> +typedef cpuset_t rte_cpuset_t ; +#endif +/** * Structure storing internal configuration (per-lcore) */
@@ -65,6 +72,7 @@ struct lcore_config {unsigned socket_id;/**< physical socket id for this lcore */ unsigned core_id;/**< core number on socket for this lcore */ int core_index;/**< relative index, starting from 0 */ +rte_cpuset_t cpuset;/**< cpu set which the lcore affinity to */ };/**
lib/librte_eal/linuxapp/eal/Makefile +1 -0
@@ -87,6 +87,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_dev.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_options.c CFLAGS_eal.o := -D_GNU_SOURCE +CFLAGS_eal_lcore.o := -D_GNU_SOURCE CFLAGS_eal_thread.o := -D_GNU_SOURCE CFLAGS_eal_log.o := -D_GNU_SOURCE CFLAGS_eal_common_log.o := -D_GNU_SOURCE
lib/librte_eal/linuxapp/eal/eal_lcore.c +8 -0
@@ -158,11 +158,19 @@ rte_eal_cpu_init(void)* ones and enable them by default. */ for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +/* init cpuset for per lcore config */ +CPU_ZERO (&lcore_config[lcore_id].cpuset); + +/* in 1:1 mapping, record related cpu detected state */ lcore_config[lcore_id].detected =cpu_detected (lcore_id);if (lcore_config[lcore_id].detected ==0 ) { config->lcore_role[lcore_id] =ROLE_OFF ;continue ; } + +/* By default, lcore 1:1 map to cpu id */ +CPU_SET (lcore_id, &lcore_config[lcore_id].cpuset); +/* By default, each detected core is enabled */ config->lcore_role[lcore_id] =ROLE_RTE ; lcore_config[lcore_id].core_id =cpu_core_id (lcore_id);
Signature · 1 line
1.8.1.4
PATCH 2/15 eal: new eal option '--lcores' for cpu assignment
In reply to a message not in the archiveIt supports one new eal long option '--lcores' for EAL thread cpuset assignment.
The format pattern:
--lcores='lcores[@cpus]<,lcores[@cpus]>'
lcores, cpus could be a single digit or a group.
'(' and ')' are necessary if it's a group.
If not supply '@cpus', the value of cpus uses the same as lcores.
e.g. '1,2@(5-7),(3-5)@(0,2),(0,6)' means starting 7 EAL thread as below
lcore 0 runs on cpuset 0x41 (cpu 0,6)
lcore 1 runs on cpuset 0x2 (cpu 1)
lcore 2 runs on cpuset 0xe0 (cpu 5,6,7)
lcore 3,4,5 runs on cpuset 0x5 (cpu 0,2)
lcore 6 runs on cpuset 0x41 (cpu 0,6)
--- lib/librte_eal/common/eal_common_launch.c | 1 - lib/librte_eal/common/eal_common_options.c | 262 ++++++++++++++++++++++++++++- lib/librte_eal/common/eal_options.h | 2 + lib/librte_eal/linuxapp/eal/Makefile | 1 + 4 files changed, 261 insertions(+), 5 deletions(-)
lib/librte_eal/common/eal_common_launch.c +0 -1
@@ -117,4 +117,3 @@ rte_eal_mp_wait_lcore(void)rte_eal_wait_lcore (lcore_id); } } -
lib/librte_eal/common/eal_common_options.c +258 -4
304 lines not on this page · raw message
lib/librte_eal/common/eal_options.h +2 -0
@@ -77,6 +77,8 @@ enum {OPT_CREATE_UIO_DEV_NUM , #define OPT_VFIO_INTR "vfio-intr"OPT_VFIO_INTR_NUM , +#define OPT_LCORES "lcores" +OPT_LCORES_NUM , OPT_LONG_MAX_NUM };
lib/librte_eal/linuxapp/eal/Makefile +1 -0
@@ -95,6 +95,7 @@ CFLAGS_eal_hugepage_info.o := -D_GNU_SOURCE CFLAGS_eal_pci.o := -D_GNU_SOURCE CFLAGS_eal_pci_vfio.o := -D_GNU_SOURCE CFLAGS_eal_common_whitelist.o := -D_GNU_SOURCE +CFLAGS_eal_common_options.o := -D_GNU_SOURCE # workaround for a gcc bug with noreturn attribute # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603
Signature · 1 line
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 1421914598-2747-3-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
Re: [PATCH v1 02/15] eal: new eal option '--lcores' for cpu assignment
Quoted 8 lines · Cunming Liang
It supports one new eal long option '--lcores' for EAL thread cpuset assignment.
The format pattern:
--lcores='lcores[@cpus]<,lcores[@cpus]>'lcores, cpus could be a single digit or a group.
'(' and ')' are necessary if it's a group.
If not supply '@cpus', the value of cpus uses the same as lcores.
Quoted 7 lines
e.g. '1,2@(5-7),(3-5)@(0,2),(0,6)' means starting 7 EAL thread as below
lcore 0 runs on cpuset0x41(cpu 0,6)
lcore 1 runs on cpuset0x2(cpu 1)
lcore 2 runs on cpuset0xe0(cpu 5,6,7)
lcore 3,4,5 runs on cpuset0x5(cpu 0,2)
lcore 6 runs on cpuset0x41(cpu 0,6)
This strikes me as very confusing, though a couple of tweaks might help with readability. The lcore 0 at the end is especially confusing. Perhaps we can limit the allowed formats here,
* require the lcore_id to be specified - the lack of an lcore id for the last part makes having it as lcore 0 surprising.
* only allow one lcore id to be given for each set of cores.
I think it may still be readable if we allow the core set to be omitted if its to be the same as the lcore_id.
It's probably still not going to be very tidy, but I think we can improve things.
/Bruce
Quoted 360 lines
Signed-off-by Cunming Liang <[email protected]>--- lib/librte_eal/common/eal_common_launch.c | 1 - lib/librte_eal/common/eal_common_options.c | 262 ++++++++++++++++++++++++++++- lib/librte_eal/common/eal_options.h | 2 + lib/librte_eal/linuxapp/eal/Makefile | 1 + 4 files changed, 261 insertions(+), 5 deletions(-)lib/librte_eal/common/eal_common_launch.c +0
-1@@ -117,4 +117,3 @@ rte_eal_mp_wait_lcore(void)rte_eal_wait_lcore (lcore_id); } } -lib/librte_eal/common/eal_common_options.c +258
-4@@ -45,6 +45,7 @@#include <rte_lcore.h> #include <rte_version.h> #include <rte_devargs.h> +#include <rte_memcpy.h> #include "eal_internal_cfg.h" #include "eal_options.h" @@ -85,6 +86,7 @@ eal_long_options[] = { {OPT_XEN_DOM0, 0, 0, OPT_XEN_DOM0_NUM}, {OPT_CREATE_UIO_DEV, 1, NULL, OPT_CREATE_UIO_DEV_NUM}, {OPT_VFIO_INTR, 1, NULL, OPT_VFIO_INTR_NUM}, + {OPT_LCORES, 1, 0, OPT_LCORES_NUM}, {0, 0, 0, 0} };@@ -255,9 +257,11 @@ eal_parse_corelist(const char *corelist)if (min ==RTE_MAX_LCORE ) min = idx;for (idx = min; idx <= max; idx++) { - cfg->lcore_role[idx] =ROLE_RTE ; - lcore_config[idx].core_index = count; - count++; +if (cfg->lcore_role[idx] !=ROLE_RTE ) { + cfg->lcore_role[idx] =ROLE_RTE ; + lcore_config[idx].core_index = count; + count++; + } } min =RTE_MAX_LCORE ; }else @@ -289,6 +293,241 @@ eal_parse_master_lcore(const char *arg)return 0 ; } +/* +* Parse elem, the elem could be single number or '(' ')' group +* Within group elem, '-' used for a range seperator; +* ',' used for a single number. +*/ +static int +eal_parse_set (const char *input,uint16_t set[],unsigned num) +{ +unsigned idx; +const char *str = input; +char *end =NULL ; +unsigned min, max; + +memset (set,0 , num *sizeof (uint16_t )); + +while (isblank (*str)) + str++; + +/* only digit or left bracket is qulify for start point */ +if ((!isdigit (*str) && *str !='(' ) || *str =='\0' ) +return -1 ; + +/* process single number */ +if (*str !='(' ) { + errno =0 ; + idx =strtoul (str, &end,10 ); +if (errno || end ==NULL || idx >= num) +return -1 ; +else { +while (isblank (*end)) + end++; + +if (*end !=',' && *end !='\0' && + *end !='@' ) +return -1 ; + + set[idx] =1 ; +return end - input; + } + } + +/* process set within bracket */ + str++; +while (isblank (*str)) + str++; +if (*str =='\0' ) +return -1 ; + + min =RTE_MAX_LCORE ; +do { + +/* go ahead to the first digit */ +while (isblank (*str)) + str++; +if (!isdigit (*str)) +return -1 ; + +/* get the digit value */ + errno =0 ; + idx =strtoul (str, &end,10 ); +if (errno || end ==NULL || idx >= num) +return -1 ; + +/* go ahead to separator '-',',' and ')' */ +while (isblank (*end)) + end++; +if (*end =='-' ) { +if (min ==RTE_MAX_LCORE ) + min = idx; +else /* avoid continuous '-' */ +return -1 ; + }else if ((*end ==',' ) || (*end ==')' )) { + max = idx; +if (min ==RTE_MAX_LCORE ) + min = idx; +for (idx =RTE_MIN (min, max); + idx <=RTE_MAX (min, max); idx++) + set[idx] =1 ; + + min =RTE_MAX_LCORE ; + }else +return -1 ; + + str = end +1 ; + }while (*end !='\0' && *end !=')' ); + +return str - input; +} + +/* convert from set array to cpuset bitmap */ +static inline int +convert_to_cpuset (rte_cpuset_t *cpusetp, +uint16_t *set,unsigned num) +{ +unsigned idx; + +CPU_ZERO (cpusetp); + +for (idx =0 ; idx < num; idx++) { +if (!set[idx]) +continue ; + +if (!lcore_config[idx].detected) { +RTE_LOG (ERR ,EAL ,"core %u " +"unavailable\n" , idx); +return -1 ; + } + +CPU_SET (idx, cpusetp); + } + +return 0 ; +} + +/* +* The format pattern: --lcores='lcores[@cpus]<,lcores[@cpus]>' +* lcores, cpus could be a single digit or a group. +* '(' and ')' are necessary if it's a group. +* If not supply '@cpus', the value of cpus uses the same as lcores. +* e.g. '1,2@(5-7),(3-5)@(0,2),(0,6)' means start 7 EAL thread as below +* lcore 0 runs on cpuset 0x41 (cpu 0,6) +* lcore 1 runs on cpuset 0x2 (cpu 1) +* lcore 2 runs on cpuset 0xe0 (cpu 5,6,7) +* lcore 3,4,5 runs on cpuset 0x5 (cpu 0,2) +* lcore 6 runs on cpuset 0x41 (cpu 0,6) +*/ +static int +eal_parse_lcores (const char *lcores) +{ +struct rte_config *cfg =rte_eal_get_configuration (); +static uint16_t set[RTE_MAX_LCORE ]; +unsigned idx =0 ; +int i; +unsigned count =0 ; +const char *lcore_start =NULL ; +const char *end =NULL ; +int offset; +rte_cpuset_t cpuset; +int ret =-1 ; + +if (lcores ==NULL ) +return -1 ; + +/* Remove all blank characters ahead and after */ +while (isblank (*lcores)) + lcores++; + i =strnlen (lcores,sysconf (_SC_ARG_MAX)); +while ((i >0 ) &&isblank (lcores[i -1 ])) + i--; + +CPU_ZERO (&cpuset); + +/* Reset lcore config */ +for (idx =0 ; idx <RTE_MAX_LCORE ; idx++) { + cfg->lcore_role[idx] =ROLE_OFF ; + lcore_config[idx].core_index =-1 ; +CPU_ZERO (&lcore_config[idx].cpuset); + } + +/* Get list of cores */ +do { +while (isblank (*lcores)) + lcores++; +if (*lcores =='\0' ) + goto err; + +/* record lcore_set start point */ + lcore_start = lcores; + +/* go across a complete bracket */ +if (*lcore_start =='(' ) { + lcores +=strcspn (lcores,")" ); +if (*lcores++ =='\0' ) + goto err; + } + +/* scan the separator '@', ','(next) or '\0'(finish) */ + lcores +=strcspn (lcores,"@," ); + +if (*lcores =='@' ) { +/* explict assign cpu_set */ + offset =eal_parse_set (lcores +1 , set,RTE_DIM (set)); +if (offset <0 ) + goto err; + +/* prepare cpu_set and update the end cursor */ +if (0 >convert_to_cpuset (&cpuset, + set,RTE_DIM (set))) + goto err; + end = lcores +1 + offset; + }else /* ',' or '\0' */ +/* haven't given cpu_set, current loop done */ + end = lcores; + +if (*end !=',' && *end !='\0' ) + goto err; + +/* parse lcore_set from start point */ +if (0 >eal_parse_set (lcore_start, set,RTE_DIM (set))) + goto err; + +/* without '@', by default using lcore_set as cpu_set */ +if (*lcores !='@' && +0 >convert_to_cpuset (&cpuset, set,RTE_DIM (set))) + goto err; + +/* start to update lcore_set */ +for (idx =0 ; idx <RTE_MAX_LCORE ; idx++) { +if (!set[idx]) +continue ; + +if (cfg->lcore_role[idx] !=ROLE_RTE ) { + lcore_config[idx].core_index = count; + cfg->lcore_role[idx] =ROLE_RTE ; + count++; + } +rte_memcpy (&lcore_config[idx].cpuset, &cpuset, +sizeof (rte_cpuset_t)); + } + + lcores = end +1 ; + }while (*end !='\0' ); + +if (count ==0 ) + goto err; + + cfg->lcore_count = count; + lcores_parsed =1 ; + ret =0 ; + +err: + +return ret; +} +static int eal_parse_syslog (const char *facility,struct internal_config *conf) {@@ -489,6 +728,13 @@ eal_parse_common_option(int opt, const char *optarg, conf->log_level = log;break ; } +case OPT_LCORES_NUM : +if (eal_parse_lcores (optarg) <0 ) { +RTE_LOG (ERR ,EAL ,"invalid parameter for --" +OPT_LCORES "\n" ); +return -1 ; + } +break ;/* don't know what to do, leave this to caller */ default :@@ -527,7 +773,7 @@ eal_check_common_options(struct internal_config *internal_cfg)if (!lcores_parsed) {RTE_LOG (ERR ,EAL ,"CPU cores must be enabled with options " -"-c or -l\n" ); +"-c, -l or --lcores\n" );return -1 ; }if (cfg->lcore_role[cfg->master_lcore] !=ROLE_RTE ) {@@ -583,6 +829,14 @@ eal_common_usage(void)" The argument format is <c1>[-c2][,c3[-c4],...]\n" " where c1, c2, etc are core indexes between 0 and %d\n" " --" OPT_MASTER_LCORE " ID: Core ID that is used as master\n" +" --" OPT_LCORES " MAP: maps between lcore_set to phys_cpu_set\n" +" The argument format is\n" +" 'lcores[@cpus]<,lcores[@cpus],...>'\n" +" lcores and cpus list are grouped by '(' and ')'\n" +" Within the group, '-' is used for range separator,\n" +" ',' is used for single number separator.\n" +" '( )' can be omitted for single element group, '@' \n" +" can be omitted if cpus and lcores has the same value\n" " -n NUM : Number of memory channels\n" " -v : Display version information on startup\n" " -m MB : memory to allocate (see also --" OPT_SOCKET_MEM ")\n" lib/librte_eal/common/eal_options.h +2
-0@@ -77,6 +77,8 @@ enum {OPT_CREATE_UIO_DEV_NUM , #define OPT_VFIO_INTR "vfio-intr"OPT_VFIO_INTR_NUM , +#define OPT_LCORES "lcores" +OPT_LCORES_NUM , OPT_LONG_MAX_NUM };lib/librte_eal/linuxapp/eal/Makefile +1
-0@@ -95,6 +95,7 @@ CFLAGS_eal_hugepage_info.o := -D_GNU_SOURCE CFLAGS_eal_pci.o := -D_GNU_SOURCE CFLAGS_eal_pci_vfio.o := -D_GNU_SOURCE CFLAGS_eal_common_whitelist.o := -D_GNU_SOURCE +CFLAGS_eal_common_options.o := -D_GNU_SOURCE # workaround for a gcc bug with noreturn attribute # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603Signature · 1 line
1.8.1.4
Re: [PATCH v1 02/15] eal: new eal option '--lcores' for cpu assignment
Hi Bruce,
Quoted 24 lines
-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Bruce Richardson
Sent: Thursday, January 22, 2015 12:19 PM
To: Liang, Cunming
Cc: [email protected]
Subject: Re: [dpdk-dev] [PATCH v1 02/15] eal: new eal option '--lcores' for cpu assignmentQuoted 8 lines · Cunming Liang
It supports one new eal long option '--lcores' for EAL thread cpuset assignment.
The format pattern:
--lcores='lcores[@cpus]<,lcores[@cpus]>'lcores, cpus could be a single digit or a group.
'(' and ')' are necessary if it's a group.
If not supply '@cpus', the value of cpus uses the same as lcores.Quoted 7 lines
e.g. '1,2@(5-7),(3-5)@(0,2),(0,6)' means starting 7 EAL thread as below
lcore 0 runs on cpuset0x41(cpu 0,6)
lcore 1 runs on cpuset0x2(cpu 1)
lcore 2 runs on cpuset0xe0(cpu 5,6,7)
lcore 3,4,5 runs on cpuset0x5(cpu 0,2)
lcore 6 runs on cpuset0x41(cpu 0,6)
Quoted 2 lines
This strikes me as very confusing, though a couple of tweaks might help with readability. The lcore 0 at the end is especially confusing.
Didn't get you here: do you find (0,6) confusing, right? Because braces implicitly specifies affinity for group of en-braced lcores?
Quoted 4 lines
Perhaps we can
limit the allowed formats here,
* require thelcore_idto be specified - the lack of an lcore id for the last part makes having it as lcore 0 surprising.
Again, not sure I understand you properly: lcore_id(s) are always specified explicitly. Physical cpus part might be omitted.
Quoted 1 line
* only allow one lcore id to be given for each set of cores.
So you mean for '(3-5)@(0,2)' user would have to: '3@(0,2),4@(0,2),5@(0,2)'?
I don't see big difference here, but imagine you'd like to create a pool of 32 EAL-threads running on same cpu set. With current syntax it is just something like: '(32-63)@(0-7)'.
With what you proposing it will be a very long list.
Quoted 3 lines
I think it may still be readable if we allow the core set to be omitted if its to be the same as the
lcore_id.
I think that is supported.
See lcore_id=1 in Steve's example above.
As I understand: --lcores='0,2,3-5' is equal to '-l 0,2,3-5' and to '-c 0x3d'.
Konstantin
Quoted 365 lines
It's probably still not going to be very tidy, but I think we can improve things.
/Bruce
Quoted 360 lines
Signed-off-by Cunming Liang <[email protected]>--- lib/librte_eal/common/eal_common_launch.c | 1 - lib/librte_eal/common/eal_common_options.c | 262 ++++++++++++++++++++++++++++- lib/librte_eal/common/eal_options.h | 2 + lib/librte_eal/linuxapp/eal/Makefile | 1 + 4 files changed, 261 insertions(+), 5 deletions(-)lib/librte_eal/common/eal_common_launch.c +0
-1@@ -117,4 +117,3 @@ rte_eal_mp_wait_lcore(void)rte_eal_wait_lcore (lcore_id); } } -lib/librte_eal/common/eal_common_options.c +258
-4@@ -45,6 +45,7 @@#include <rte_lcore.h> #include <rte_version.h> #include <rte_devargs.h> +#include <rte_memcpy.h> #include "eal_internal_cfg.h" #include "eal_options.h" @@ -85,6 +86,7 @@ eal_long_options[] = { {OPT_XEN_DOM0, 0, 0, OPT_XEN_DOM0_NUM}, {OPT_CREATE_UIO_DEV, 1, NULL, OPT_CREATE_UIO_DEV_NUM}, {OPT_VFIO_INTR, 1, NULL, OPT_VFIO_INTR_NUM}, + {OPT_LCORES, 1, 0, OPT_LCORES_NUM}, {0, 0, 0, 0} };@@ -255,9 +257,11 @@ eal_parse_corelist(const char *corelist)if (min ==RTE_MAX_LCORE ) min = idx;for (idx = min; idx <= max; idx++) { - cfg->lcore_role[idx] =ROLE_RTE ; - lcore_config[idx].core_index = count; - count++; +if (cfg->lcore_role[idx] !=ROLE_RTE ) { + cfg->lcore_role[idx] =ROLE_RTE ; + lcore_config[idx].core_index = count; + count++; + } } min =RTE_MAX_LCORE ; }else @@ -289,6 +293,241 @@ eal_parse_master_lcore(const char *arg)return 0 ; } +/* +* Parse elem, the elem could be single number or '(' ')' group +* Within group elem, '-' used for a range seperator; +* ',' used for a single number. +*/ +static int +eal_parse_set (const char *input,uint16_t set[],unsigned num) +{ +unsigned idx; +const char *str = input; +char *end =NULL ; +unsigned min, max; + +memset (set,0 , num *sizeof (uint16_t )); + +while (isblank (*str)) + str++; + +/* only digit or left bracket is qulify for start point */ +if ((!isdigit (*str) && *str !='(' ) || *str =='\0' ) +return -1 ; + +/* process single number */ +if (*str !='(' ) { + errno =0 ; + idx =strtoul (str, &end,10 ); +if (errno || end ==NULL || idx >= num) +return -1 ; +else { +while (isblank (*end)) + end++; + +if (*end !=',' && *end !='\0' && + *end !='@' ) +return -1 ; + + set[idx] =1 ; +return end - input; + } + } + +/* process set within bracket */ + str++; +while (isblank (*str)) + str++; +if (*str =='\0' ) +return -1 ; + + min =RTE_MAX_LCORE ; +do { + +/* go ahead to the first digit */ +while (isblank (*str)) + str++; +if (!isdigit (*str)) +return -1 ; + +/* get the digit value */ + errno =0 ; + idx =strtoul (str, &end,10 ); +if (errno || end ==NULL || idx >= num) +return -1 ; + +/* go ahead to separator '-',',' and ')' */ +while (isblank (*end)) + end++; +if (*end =='-' ) { +if (min ==RTE_MAX_LCORE ) + min = idx; +else /* avoid continuous '-' */ +return -1 ; + }else if ((*end ==',' ) || (*end ==')' )) { + max = idx; +if (min ==RTE_MAX_LCORE ) + min = idx; +for (idx =RTE_MIN (min, max); + idx <=RTE_MAX (min, max); idx++) + set[idx] =1 ; + + min =RTE_MAX_LCORE ; + }else +return -1 ; + + str = end +1 ; + }while (*end !='\0' && *end !=')' ); + +return str - input; +} + +/* convert from set array to cpuset bitmap */ +static inline int +convert_to_cpuset (rte_cpuset_t *cpusetp, +uint16_t *set,unsigned num) +{ +unsigned idx; + +CPU_ZERO (cpusetp); + +for (idx =0 ; idx < num; idx++) { +if (!set[idx]) +continue ; + +if (!lcore_config[idx].detected) { +RTE_LOG (ERR ,EAL ,"core %u " +"unavailable\n" , idx); +return -1 ; + } + +CPU_SET (idx, cpusetp); + } + +return 0 ; +} + +/* +* The format pattern: --lcores='lcores[@cpus]<,lcores[@cpus]>' +* lcores, cpus could be a single digit or a group. +* '(' and ')' are necessary if it's a group. +* If not supply '@cpus', the value of cpus uses the same as lcores. +* e.g. '1,2@(5-7),(3-5)@(0,2),(0,6)' means start 7 EAL thread as below +* lcore 0 runs on cpuset 0x41 (cpu 0,6) +* lcore 1 runs on cpuset 0x2 (cpu 1) +* lcore 2 runs on cpuset 0xe0 (cpu 5,6,7) +* lcore 3,4,5 runs on cpuset 0x5 (cpu 0,2) +* lcore 6 runs on cpuset 0x41 (cpu 0,6) +*/ +static int +eal_parse_lcores (const char *lcores) +{ +struct rte_config *cfg =rte_eal_get_configuration (); +static uint16_t set[RTE_MAX_LCORE ]; +unsigned idx =0 ; +int i; +unsigned count =0 ; +const char *lcore_start =NULL ; +const char *end =NULL ; +int offset; +rte_cpuset_t cpuset; +int ret =-1 ; + +if (lcores ==NULL ) +return -1 ; + +/* Remove all blank characters ahead and after */ +while (isblank (*lcores)) + lcores++; + i =strnlen (lcores,sysconf (_SC_ARG_MAX)); +while ((i >0 ) &&isblank (lcores[i -1 ])) + i--; + +CPU_ZERO (&cpuset); + +/* Reset lcore config */ +for (idx =0 ; idx <RTE_MAX_LCORE ; idx++) { + cfg->lcore_role[idx] =ROLE_OFF ; + lcore_config[idx].core_index =-1 ; +CPU_ZERO (&lcore_config[idx].cpuset); + } + +/* Get list of cores */ +do { +while (isblank (*lcores)) + lcores++; +if (*lcores =='\0' ) + goto err; + +/* record lcore_set start point */ + lcore_start = lcores; + +/* go across a complete bracket */ +if (*lcore_start =='(' ) { + lcores +=strcspn (lcores,")" ); +if (*lcores++ =='\0' ) + goto err; + } + +/* scan the separator '@', ','(next) or '\0'(finish) */ + lcores +=strcspn (lcores,"@," ); + +if (*lcores =='@' ) { +/* explict assign cpu_set */ + offset =eal_parse_set (lcores +1 , set,RTE_DIM (set)); +if (offset <0 ) + goto err; + +/* prepare cpu_set and update the end cursor */ +if (0 >convert_to_cpuset (&cpuset, + set,RTE_DIM (set))) + goto err; + end = lcores +1 + offset; + }else /* ',' or '\0' */ +/* haven't given cpu_set, current loop done */ + end = lcores; + +if (*end !=',' && *end !='\0' ) + goto err; + +/* parse lcore_set from start point */ +if (0 >eal_parse_set (lcore_start, set,RTE_DIM (set))) + goto err; + +/* without '@', by default using lcore_set as cpu_set */ +if (*lcores !='@' && +0 >convert_to_cpuset (&cpuset, set,RTE_DIM (set))) + goto err; + +/* start to update lcore_set */ +for (idx =0 ; idx <RTE_MAX_LCORE ; idx++) { +if (!set[idx]) +continue ; + +if (cfg->lcore_role[idx] !=ROLE_RTE ) { + lcore_config[idx].core_index = count; + cfg->lcore_role[idx] =ROLE_RTE ; + count++; + } +rte_memcpy (&lcore_config[idx].cpuset, &cpuset, +sizeof (rte_cpuset_t)); + } + + lcores = end +1 ; + }while (*end !='\0' ); + +if (count ==0 ) + goto err; + + cfg->lcore_count = count; + lcores_parsed =1 ; + ret =0 ; + +err: + +return ret; +} +static int eal_parse_syslog (const char *facility,struct internal_config *conf) {@@ -489,6 +728,13 @@ eal_parse_common_option(int opt, const char *optarg, conf->log_level = log;break ; } +case OPT_LCORES_NUM : +if (eal_parse_lcores (optarg) <0 ) { +RTE_LOG (ERR ,EAL ,"invalid parameter for --" +OPT_LCORES "\n" ); +return -1 ; + } +break ;/* don't know what to do, leave this to caller */ default :@@ -527,7 +773,7 @@ eal_check_common_options(struct internal_config *internal_cfg)if (!lcores_parsed) {RTE_LOG (ERR ,EAL ,"CPU cores must be enabled with options " -"-c or -l\n" ); +"-c, -l or --lcores\n" );return -1 ; }if (cfg->lcore_role[cfg->master_lcore] !=ROLE_RTE ) {@@ -583,6 +829,14 @@ eal_common_usage(void)" The argument format is <c1>[-c2][,c3[-c4],...]\n" " where c1, c2, etc are core indexes between 0 and %d\n" " --" OPT_MASTER_LCORE " ID: Core ID that is used as master\n" +" --" OPT_LCORES " MAP: maps between lcore_set to phys_cpu_set\n" +" The argument format is\n" +" 'lcores[@cpus]<,lcores[@cpus],...>'\n" +" lcores and cpus list are grouped by '(' and ')'\n" +" Within the group, '-' is used for range separator,\n" +" ',' is used for single number separator.\n" +" '( )' can be omitted for single element group, '@' \n" +" can be omitted if cpus and lcores has the same value\n" " -n NUM : Number of memory channels\n" " -v : Display version information on startup\n" " -m MB : memory to allocate (see also --" OPT_SOCKET_MEM ")\n" lib/librte_eal/common/eal_options.h +2
-0@@ -77,6 +77,8 @@ enum {OPT_CREATE_UIO_DEV_NUM , #define OPT_VFIO_INTR "vfio-intr"OPT_VFIO_INTR_NUM , +#define OPT_LCORES "lcores" +OPT_LCORES_NUM , OPT_LONG_MAX_NUM };lib/librte_eal/linuxapp/eal/Makefile +1
-1@@ -95,6 +95,7 @@ CFLAGS_eal_hugepage_info.o := -D_GNU_SOURCE CFLAGS_eal_pci.o := -D_GNU_SOURCE CFLAGS_eal_pci_vfio.o := -D_GNU_SOURCE CFLAGS_eal_common_whitelist.o := -D_GNU_SOURCE +CFLAGS_eal_common_options.o := -D_GNU_SOURCE # workaround for a gcc bug with noreturn attribute # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603 --1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 2601191342CEEE43887BDE71AB977258213DEF7B-pww93C2UFcwu0RiL9chJVbfspsVTdybXVpNB7YpNyf8@public.gmane.org
Re: [PATCH v1 02/15] eal: new eal option '--lcores' for cpu assignment
Hi,
I want to mention that similar but for me much more readable syntax have Pktgen-DPDK for defining core - port mapping. Maybe we can adopt this syntax for new '--lcores' parameter.
See '-m' parameter syntax on Pktgen readme.
https://github.com/pktgen/Pktgen-DPDK/blob/master/dpdk/examples/pktgen/README.md
Quoted 443 lines
-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Ananyev, Konstantin
Sent: Thursday, January 22, 2015 3:34 PM
To: Richardson, Bruce; Liang, Cunming
Cc: [email protected]
Subject: Re: [dpdk-dev] [PATCH v1 02/15] eal: new eal option '--lcores' for cpu assignmentHi Bruce,
Quoted 6 lines
-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Bruce Richardson
Sent: Thursday, January 22, 2015 12:19 PM
To: Liang, Cunming
Cc: [email protected]
Subject: Re: [dpdk-dev] [PATCH v1 02/15] eal: new eal option '--lcores' for cpuassignment
Quoted 3 lines · Cunming Liang
Quoted 1 line
It supports one new eal long option '--lcores' for EAL thread cpuset
assignment.
Quoted 15 lines
Quoted 7 lines
The format pattern:
--lcores='lcores[@cpus]<,lcores[@cpus]>'lcores, cpus could be a single digit or a group.
'(' and ')' are necessary if it's a group.
If not supply '@cpus', the value of cpus uses the same as lcores.Quoted 7 lines
e.g. '1,2@(5-7),(3-5)@(0,2),(0,6)' means starting 7 EAL thread as below
lcore 0 runs on cpuset0x41(cpu 0,6)
lcore 1 runs on cpuset0x2(cpu 1)
lcore 2 runs on cpuset0xe0(cpu 5,6,7)
lcore 3,4,5 runs on cpuset0x5(cpu 0,2)
lcore 6 runs on cpuset0x41(cpu 0,6)Quoted 2 lines
This strikes me as very confusing, though a couple of tweaks might help with readability. The lcore 0 at the end is especially confusing.
Didn't get you here: do you find (0,6) confusing, right? Because braces implicitly specifies affinity for group of en-braced lcores?
Quoted 4 lines
Perhaps we can
limit the allowed formats here,
* require thelcore_idto be specified - the lack of an lcore id for the last part makes having it as lcore 0 surprising.Again, not sure I understand you properly:
lcore_id(s)are always specified explicitly.
Physical cpus part might be omitted.Quoted 1 line
* only allow one lcore id to be given for each set of cores.
So you mean for '(3-5)@(0,2)' user would have to: '3@(0,2),4@(0,2),5@(0,2)'? I don't see big difference here, but imagine you'd like to create a pool of 32 EAL- threads running on same cpu set.
With current syntax it is just something like: '(32-63)@(0-7)'.
With what you proposing it will be a very long list.Quoted 3 lines
I think it may still be readable if we allow the core set to be omitted if its to be the same as the
lcore_id.I think that is supported.
See lcore_id=1 in Steve's example above.
As I understand: --lcores='0,2,3-5' is equal to '-l 0,2,3-5' and to '-c0x3d'.Konstantin
Quoted 9 lines
It's probably still not going to be very tidy, but I think we can improve things.
/Bruce
Quoted 4 lines
Signed-off-by Cunming Liang <[email protected]>--- lib/librte_eal/common/eal_common_launch.c | 1 - lib/librte_eal/common/eal_common_options.c | 262++++++++++++++++++++++++++++-
Quoted 5 lines
Review
Quoted 5 lines
lib/librte_eal/common/eal_options.h | 2 + lib/librte_eal/linuxapp/eal/Makefile | 1 + 4 files changed, 261 insertions(+), 5 deletions(-)(quoted diff) +0
-0
b/lib/librte_eal/common/eal_common_launch.cQuoted 9 lines
Review
Quoted 9 lines
index 599f83b..2d732b1 100644
lib/librte_eal/common/eal_common_launch.c +0
-1@@ -117,4 +117,3 @@ rte_eal_mp_wait_lcore(void)rte_eal_wait_lcore (lcore_id); } } -(quoted diff) +0
-0
b/lib/librte_eal/common/eal_common_options.cQuoted 269 lines
Quoted 269 lines
index e2810ab..fc47588 100644
lib/librte_eal/common/eal_common_options.c +238
-3@@ -45,6 +45,7 @@#include <rte_lcore.h> #include <rte_version.h> #include <rte_devargs.h> +#include <rte_memcpy.h> #include "eal_internal_cfg.h" #include "eal_options.h" @@ -85,6 +86,7 @@ eal_long_options[] = { {OPT_XEN_DOM0, 0, 0, OPT_XEN_DOM0_NUM}, {OPT_CREATE_UIO_DEV, 1, NULL, OPT_CREATE_UIO_DEV_NUM}, {OPT_VFIO_INTR, 1, NULL, OPT_VFIO_INTR_NUM}, + {OPT_LCORES, 1, 0, OPT_LCORES_NUM}, {0, 0, 0, 0} };@@ -255,9 +257,11 @@ eal_parse_corelist(const char *corelist)if (min ==RTE_MAX_LCORE ) min = idx;for (idx = min; idx <= max; idx++) { - cfg->lcore_role[idx] =ROLE_RTE ; - lcore_config[idx].core_index = count; - count++; +if (cfg->lcore_role[idx] !=ROLE_RTE ) { + cfg->lcore_role[idx] =ROLE_RTE ; + lcore_config[idx].core_index = count; + count++; + } } min =RTE_MAX_LCORE ; }else @@ -289,6 +293,241 @@ eal_parse_master_lcore(const char *arg)return 0 ; } +/* +* Parse elem, the elem could be single number or '(' ')' group +* Within group elem, '-' used for a range seperator; +* ',' used for a single number. +*/ +static int +eal_parse_set (const char *input,uint16_t set[],unsigned num) +{ +unsigned idx; +const char *str = input; +char *end =NULL ; +unsigned min, max; + +memset (set,0 , num *sizeof (uint16_t )); + +while (isblank (*str)) + str++; + +/* only digit or left bracket is qulify for start point */ +if ((!isdigit (*str) && *str !='(' ) || *str =='\0' ) +return -1 ; + +/* process single number */ +if (*str !='(' ) { + errno =0 ; + idx =strtoul (str, &end,10 ); +if (errno || end ==NULL || idx >= num) +return -1 ; +else { +while (isblank (*end)) + end++; + +if (*end !=',' && *end !='\0' && + *end !='@' ) +return -1 ; + + set[idx] =1 ; +return end - input; + } + } + +/* process set within bracket */ + str++; +while (isblank (*str)) + str++; +if (*str =='\0' ) +return -1 ; + + min =RTE_MAX_LCORE ; +do { + +/* go ahead to the first digit */ +while (isblank (*str)) + str++; +if (!isdigit (*str)) +return -1 ; + +/* get the digit value */ + errno =0 ; + idx =strtoul (str, &end,10 ); +if (errno || end ==NULL || idx >= num) +return -1 ; + +/* go ahead to separator '-',',' and ')' */ +while (isblank (*end)) + end++; +if (*end =='-' ) { +if (min ==RTE_MAX_LCORE ) + min = idx; +else /* avoid continuous '-' */ +return -1 ; + }else if ((*end ==',' ) || (*end ==')' )) { + max = idx; +if (min ==RTE_MAX_LCORE ) + min = idx; +for (idx =RTE_MIN (min, max); + idx <=RTE_MAX (min, max); idx++) + set[idx] =1 ; + + min =RTE_MAX_LCORE ; + }else +return -1 ; + + str = end +1 ; + }while (*end !='\0' && *end !=')' ); + +return str - input; +} + +/* convert from set array to cpuset bitmap */ +static inline int +convert_to_cpuset (rte_cpuset_t *cpusetp, +uint16_t *set,unsigned num) +{ +unsigned idx; + +CPU_ZERO (cpusetp); + +for (idx =0 ; idx < num; idx++) { +if (!set[idx]) +continue ; + +if (!lcore_config[idx].detected) { +RTE_LOG (ERR ,EAL ,"core %u " +"unavailable\n" , idx); +return -1 ; + } + +CPU_SET (idx, cpusetp); + } + +return 0 ; +} + +/* +* The format pattern: --lcores='lcores[@cpus]<,lcores[@cpus]>' +* lcores, cpus could be a single digit or a group. +* '(' and ')' are necessary if it's a group. +* If not supply '@cpus', the value of cpus uses the same as lcores. +* e.g. '1,2@(5-7),(3-5)@(0,2),(0,6)' means start 7 EAL thread as below +* lcore 0 runs on cpuset 0x41 (cpu 0,6) +* lcore 1 runs on cpuset 0x2 (cpu 1) +* lcore 2 runs on cpuset 0xe0 (cpu 5,6,7) +* lcore 3,4,5 runs on cpuset 0x5 (cpu 0,2) +* lcore 6 runs on cpuset 0x41 (cpu 0,6) +*/ +static int +eal_parse_lcores (const char *lcores) +{ +struct rte_config *cfg =rte_eal_get_configuration (); +static uint16_t set[RTE_MAX_LCORE ]; +unsigned idx =0 ; +int i; +unsigned count =0 ; +const char *lcore_start =NULL ; +const char *end =NULL ; +int offset; +rte_cpuset_t cpuset; +int ret =-1 ; + +if (lcores ==NULL ) +return -1 ; + +/* Remove all blank characters ahead and after */ +while (isblank (*lcores)) + lcores++; + i =strnlen (lcores,sysconf (_SC_ARG_MAX)); +while ((i >0 ) &&isblank (lcores[i -1 ])) + i--; + +CPU_ZERO (&cpuset); + +/* Reset lcore config */ +for (idx =0 ; idx <RTE_MAX_LCORE ; idx++) { + cfg->lcore_role[idx] =ROLE_OFF ; + lcore_config[idx].core_index =-1 ; +CPU_ZERO (&lcore_config[idx].cpuset); + } + +/* Get list of cores */ +do { +while (isblank (*lcores)) + lcores++; +if (*lcores =='\0' ) + goto err; + +/* record lcore_set start point */ + lcore_start = lcores; + +/* go across a complete bracket */ +if (*lcore_start =='(' ) { + lcores +=strcspn (lcores,")" ); +if (*lcores++ =='\0' ) + goto err; + } + +/* scan the separator '@', ','(next) or '\0'(finish) */ + lcores +=strcspn (lcores,"@," ); + +if (*lcores =='@' ) { +/* explict assign cpu_set */ + offset =eal_parse_set (lcores +1 , set,RTE_DIM (set)); +if (offset <0 ) + goto err; + +/* prepare cpu_set and update the end cursor */ +if (0 >convert_to_cpuset (&cpuset, + set,RTE_DIM (set))) + goto err; + end = lcores +1 + offset; + }else /* ',' or '\0' */ +/* haven't given cpu_set, current loop done */ + end = lcores; + +if (*end !=',' && *end !='\0' ) + goto err; + +/* parse lcore_set from start point */ +if (0 >eal_parse_set (lcore_start, set,RTE_DIM (set))) + goto err; + +/* without '@', by default using lcore_set as cpu_set */ +if (*lcores !='@' && +0 >convert_to_cpuset (&cpuset, set,RTE_DIM (set))) + goto err; + +/* start to update lcore_set */ +for (idx =0 ; idx <RTE_MAX_LCORE ; idx++) { +if (!set[idx]) +continue ; + +if (cfg->lcore_role[idx] !=ROLE_RTE ) { + lcore_config[idx].core_index = count; + cfg->lcore_role[idx] =ROLE_RTE ; + count++; + } +rte_memcpy (&lcore_config[idx].cpuset, &cpuset, +sizeof (rte_cpuset_t)); + } + + lcores = end +1 ; + }while (*end !='\0' ); + +if (count ==0 ) + goto err; + + cfg->lcore_count = count; + lcores_parsed =1 ; + ret =0 ; + +err:Quoted 8 lines
lib/librte_eal/common/eal_common_options.c eal_parse_common_option(int opt, const char
Quoted 8 lines
lib/librte_eal/common/eal_common_options.c +4
-0+ +return ret; +} +static int eal_parse_syslog (const char *facility,struct internal_config *conf) {@@ -489,6 +728,13 @@ eal_parse_common_option(int opt, const char*optarg,
Quoted 11 lines
Quoted 11 lines
lib/librte_eal/common/eal_common_options.c +7
-0conf->log_level = log;break ; } +case OPT_LCORES_NUM : +if (eal_parse_lcores (optarg) <0 ) { +RTE_LOG (ERR ,EAL ,"invalid parameter for --" +OPT_LCORES "\n" ); +return -1 ; + } +break ;Quoted 3 lines
lib/librte_eal/common/eal_common_options.c eal_check_common_options(struct internal_config
Quoted 3 lines
lib/librte_eal/common/eal_common_options.c +0
-0/* don't know what to do, leave this to caller */ default :@@ -527,7 +773,7 @@ eal_check_common_options(struct internal_config
*internal_cfg)Quoted 13 lines
lib/librte_eal/common/eal_common_options.c eal_common_usage(void) · line 832
Quoted 13 lines
lib/librte_eal/common/eal_common_options.c +2
-1if (!lcores_parsed) {RTE_LOG (ERR ,EAL ,"CPU cores must be enabled with options " -"-c or -l\n" ); +"-c, -l or --lcores\n" );return -1 ; }if (cfg->lcore_role[cfg->master_lcore] !=ROLE_RTE ) {@@ -583,6 +829,14 @@ eal_common_usage(void)" The argument format is <c1>[-c2][,c3[-c4],...]\n" " where c1, c2, etc are core indexes between 0 and %d\n" " --" OPT_MASTER_LCORE " ID: Core ID that is used as master\n" + " --"OPT_LCORES" MAP: maps between lcore_set tophys_cpu_set\n"
Quoted 10 lines
lib/librte_eal/common/eal_common_options.c eal_common_usage(void)
Quoted 10 lines
lib/librte_eal/common/eal_common_options.c +7
-0+" The argument format is\n" +" 'lcores[@cpus]<,lcores[@cpus],...>'\n" +" lcores and cpus list are grouped by '(' and ')'\n" +" Within the group, '-' is used for range separator,\n" +" ',' is used for single number separator.\n" +" '( )' can be omitted for single element group, '@' \n" +" can be omitted if cpus and lcores has the same value\n" " -n NUM : Number of memory channels\n" " -v : Display version information on startup\n" " -m MB : memory to allocate (see also --"OPT_SOCKET_MEM")\n"
Quoted 1 line
Review
Quoted 1 line
(quoted diff) +0
-0
b/lib/librte_eal/common/eal_options.hQuoted 12 lines
Quoted 12 lines
index e476f8d..a1cc59f 100644
lib/librte_eal/common/eal_options.h +2
-0@@ -77,6 +77,8 @@ enum {OPT_CREATE_UIO_DEV_NUM , #define OPT_VFIO_INTR "vfio-intr"OPT_VFIO_INTR_NUM , +#define OPT_LCORES "lcores" +OPT_LCORES_NUM , OPT_LONG_MAX_NUM };Quoted 1 line
Review
Quoted 1 line
(quoted diff) +0
-0b/lib/librte_eal/linuxapp/eal/Makefile
Quoted 14 lines
Quoted 14 lines
index 0e9c447..025d836 100644
lib/librte_eal/linuxapp/eal/Makefile +1
-1@@ -95,6 +95,7 @@ CFLAGS_eal_hugepage_info.o := -D_GNU_SOURCE CFLAGS_eal_pci.o := -D_GNU_SOURCE CFLAGS_eal_pci_vfio.o := -D_GNU_SOURCE CFLAGS_eal_common_whitelist.o := -D_GNU_SOURCE +CFLAGS_eal_common_options.o := -D_GNU_SOURCE # workaround for a gcc bug with noreturn attribute # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603 --1.8.1.4
A message that is not in the archive (sent off-list or not yet received): F6F2A6264E145F47A18AB6DF8E87425D12B8C01A-kPTMFJFq+rFP9JyJpTNKArfspsVTdybXVpNB7YpNyf8@public.gmane.org
Re: [PATCH v1 02/15] eal: new eal option '--lcores' for cpu assignment
Hi Pawel,
I don't see much different there.
If replacing '@' to '.'; '()' to '[]'; and ',' to '/'; they're almost the same. Without having rx/tx case, so ':' is useless in our case. Considering the semantic, '@'(at) is more readable than '.' for core assignment.
-Liang Cunming
Quoted 468 lines
-----Original Message-----
From: Wodkowski, PawelX
Sent: Thursday, January 22, 2015 11:17 PM
To: Ananyev, Konstantin; Richardson, Bruce; Liang, Cunming
Cc: [email protected]
Subject: RE: [dpdk-dev] [PATCH v1 02/15] eal: new eal option '--lcores' for cpu assignmentHi,
I want to mention that similar but for me much more readable syntax have Pktgen-DPDK for defining core - port mapping. Maybe we can adopt this syntax for new '--lcores' parameter.See '-m' parameter syntax on Pktgen readme.
https://github.com/pktgen/Pktgen-DPDK/blob/master/dpdk/examples/pktgen/R
EADME.mdQuoted 389 lines
-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Ananyev, Konstantin
Sent: Thursday, January 22, 2015 3:34 PM
To: Richardson, Bruce; Liang, Cunming
Cc: [email protected]
Subject: Re: [dpdk-dev] [PATCH v1 02/15] eal: new eal option '--lcores' for cpu assignmentHi Bruce,
Quoted 6 lines
-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Bruce Richardson
Sent: Thursday, January 22, 2015 12:19 PM
To: Liang, Cunming
Cc: [email protected]
Subject: Re: [dpdk-dev] [PATCH v1 02/15] eal: new eal option '--lcores' for cpuassignment
Quoted 3 lines · Cunming Liang
Quoted 1 line
It supports one new eal long option '--lcores' for EAL thread cpuset
assignment.
Quoted 15 lines
Quoted 7 lines
The format pattern:
--lcores='lcores[@cpus]<,lcores[@cpus]>'lcores, cpus could be a single digit or a group.
'(' and ')' are necessary if it's a group.
If not supply '@cpus', the value of cpus uses the same as lcores.Quoted 7 lines
e.g. '1,2@(5-7),(3-5)@(0,2),(0,6)' means starting 7 EAL thread as below
lcore 0 runs on cpuset0x41(cpu 0,6)
lcore 1 runs on cpuset0x2(cpu 1)
lcore 2 runs on cpuset0xe0(cpu 5,6,7)
lcore 3,4,5 runs on cpuset0x5(cpu 0,2)
lcore 6 runs on cpuset0x41(cpu 0,6)Quoted 2 lines
This strikes me as very confusing, though a couple of tweaks might help with readability. The lcore 0 at the end is especially confusing.
Didn't get you here: do you find (0,6) confusing, right? Because braces implicitly specifies affinity for group of en-braced lcores?
Quoted 4 lines
Perhaps we can
limit the allowed formats here,
* require thelcore_idto be specified - the lack of an lcore id for the last part makes having it as lcore 0 surprising.Again, not sure I understand you properly:
lcore_id(s)are always specified explicitly.
Physical cpus part might be omitted.Quoted 1 line
* only allow one lcore id to be given for each set of cores.
So you mean for '(3-5)@(0,2)' user would have to: '3@(0,2),4@(0,2),5@(0,2)'? I don't see big difference here, but imagine you'd like to create a pool of 32 EAL- threads running on same cpu set.
With current syntax it is just something like: '(32-63)@(0-7)'.
With what you proposing it will be a very long list.Quoted 3 lines
I think it may still be readable if we allow the core set to be omitted if its to be the same as the
lcore_id.I think that is supported.
See lcore_id=1 in Steve's example above.
As I understand: --lcores='0,2,3-5' is equal to '-l 0,2,3-5' and to '-c0x3d'.Konstantin
Quoted 9 lines
It's probably still not going to be very tidy, but I think we can improve things.
/Bruce
Quoted 4 lines
Signed-off-by Cunming Liang <[email protected]>--- lib/librte_eal/common/eal_common_launch.c | 1 - lib/librte_eal/common/eal_common_options.c | 262++++++++++++++++++++++++++++-
Quoted 5 lines
Review
Quoted 5 lines
lib/librte_eal/common/eal_options.h | 2 + lib/librte_eal/linuxapp/eal/Makefile | 1 + 4 files changed, 261 insertions(+), 5 deletions(-)(quoted diff) +0
-0
b/lib/librte_eal/common/eal_common_launch.cQuoted 9 lines
Review
Quoted 9 lines
index 599f83b..2d732b1 100644
lib/librte_eal/common/eal_common_launch.c +0
-1@@ -117,4 +117,3 @@ rte_eal_mp_wait_lcore(void)rte_eal_wait_lcore (lcore_id); } } -(quoted diff) +0
-0
b/lib/librte_eal/common/eal_common_options.cQuoted 269 lines
Quoted 269 lines
index e2810ab..fc47588 100644
lib/librte_eal/common/eal_common_options.c +238
-3@@ -45,6 +45,7 @@#include <rte_lcore.h> #include <rte_version.h> #include <rte_devargs.h> +#include <rte_memcpy.h> #include "eal_internal_cfg.h" #include "eal_options.h" @@ -85,6 +86,7 @@ eal_long_options[] = { {OPT_XEN_DOM0, 0, 0, OPT_XEN_DOM0_NUM}, {OPT_CREATE_UIO_DEV, 1, NULL, OPT_CREATE_UIO_DEV_NUM}, {OPT_VFIO_INTR, 1, NULL, OPT_VFIO_INTR_NUM}, + {OPT_LCORES, 1, 0, OPT_LCORES_NUM}, {0, 0, 0, 0} };@@ -255,9 +257,11 @@ eal_parse_corelist(const char *corelist)if (min ==RTE_MAX_LCORE ) min = idx;for (idx = min; idx <= max; idx++) { - cfg->lcore_role[idx] =ROLE_RTE ; - lcore_config[idx].core_index = count; - count++; +if (cfg->lcore_role[idx] !=ROLE_RTE ) { + cfg->lcore_role[idx] =ROLE_RTE ; + lcore_config[idx].core_index = count; + count++; + } } min =RTE_MAX_LCORE ; }else @@ -289,6 +293,241 @@ eal_parse_master_lcore(const char *arg)return 0 ; } +/* +* Parse elem, the elem could be single number or '(' ')' group +* Within group elem, '-' used for a range seperator; +* ',' used for a single number. +*/ +static int +eal_parse_set (const char *input,uint16_t set[],unsigned num) +{ +unsigned idx; +const char *str = input; +char *end =NULL ; +unsigned min, max; + +memset (set,0 , num *sizeof (uint16_t )); + +while (isblank (*str)) + str++; + +/* only digit or left bracket is qulify for start point */ +if ((!isdigit (*str) && *str !='(' ) || *str =='\0' ) +return -1 ; + +/* process single number */ +if (*str !='(' ) { + errno =0 ; + idx =strtoul (str, &end,10 ); +if (errno || end ==NULL || idx >= num) +return -1 ; +else { +while (isblank (*end)) + end++; + +if (*end !=',' && *end !='\0' && + *end !='@' ) +return -1 ; + + set[idx] =1 ; +return end - input; + } + } + +/* process set within bracket */ + str++; +while (isblank (*str)) + str++; +if (*str =='\0' ) +return -1 ; + + min =RTE_MAX_LCORE ; +do { + +/* go ahead to the first digit */ +while (isblank (*str)) + str++; +if (!isdigit (*str)) +return -1 ; + +/* get the digit value */ + errno =0 ; + idx =strtoul (str, &end,10 ); +if (errno || end ==NULL || idx >= num) +return -1 ; + +/* go ahead to separator '-',',' and ')' */ +while (isblank (*end)) + end++; +if (*end =='-' ) { +if (min ==RTE_MAX_LCORE ) + min = idx; +else /* avoid continuous '-' */ +return -1 ; + }else if ((*end ==',' ) || (*end ==')' )) { + max = idx; +if (min ==RTE_MAX_LCORE ) + min = idx; +for (idx =RTE_MIN (min, max); + idx <=RTE_MAX (min, max); idx++) + set[idx] =1 ; + + min =RTE_MAX_LCORE ; + }else +return -1 ; + + str = end +1 ; + }while (*end !='\0' && *end !=')' ); + +return str - input; +} + +/* convert from set array to cpuset bitmap */ +static inline int +convert_to_cpuset (rte_cpuset_t *cpusetp, +uint16_t *set,unsigned num) +{ +unsigned idx; + +CPU_ZERO (cpusetp); + +for (idx =0 ; idx < num; idx++) { +if (!set[idx]) +continue ; + +if (!lcore_config[idx].detected) { +RTE_LOG (ERR ,EAL ,"core %u " +"unavailable\n" , idx); +return -1 ; + } + +CPU_SET (idx, cpusetp); + } + +return 0 ; +} + +/* +* The format pattern: --lcores='lcores[@cpus]<,lcores[@cpus]>' +* lcores, cpus could be a single digit or a group. +* '(' and ')' are necessary if it's a group. +* If not supply '@cpus', the value of cpus uses the same as lcores. +* e.g. '1,2@(5-7),(3-5)@(0,2),(0,6)' means start 7 EAL thread as below +* lcore 0 runs on cpuset 0x41 (cpu 0,6) +* lcore 1 runs on cpuset 0x2 (cpu 1) +* lcore 2 runs on cpuset 0xe0 (cpu 5,6,7) +* lcore 3,4,5 runs on cpuset 0x5 (cpu 0,2) +* lcore 6 runs on cpuset 0x41 (cpu 0,6) +*/ +static int +eal_parse_lcores (const char *lcores) +{ +struct rte_config *cfg =rte_eal_get_configuration (); +static uint16_t set[RTE_MAX_LCORE ]; +unsigned idx =0 ; +int i; +unsigned count =0 ; +const char *lcore_start =NULL ; +const char *end =NULL ; +int offset; +rte_cpuset_t cpuset; +int ret =-1 ; + +if (lcores ==NULL ) +return -1 ; + +/* Remove all blank characters ahead and after */ +while (isblank (*lcores)) + lcores++; + i =strnlen (lcores,sysconf (_SC_ARG_MAX)); +while ((i >0 ) &&isblank (lcores[i -1 ])) + i--; + +CPU_ZERO (&cpuset); + +/* Reset lcore config */ +for (idx =0 ; idx <RTE_MAX_LCORE ; idx++) { + cfg->lcore_role[idx] =ROLE_OFF ; + lcore_config[idx].core_index =-1 ; +CPU_ZERO (&lcore_config[idx].cpuset); + } + +/* Get list of cores */ +do { +while (isblank (*lcores)) + lcores++; +if (*lcores =='\0' ) + goto err; + +/* record lcore_set start point */ + lcore_start = lcores; + +/* go across a complete bracket */ +if (*lcore_start =='(' ) { + lcores +=strcspn (lcores,")" ); +if (*lcores++ =='\0' ) + goto err; + } + +/* scan the separator '@', ','(next) or '\0'(finish) */ + lcores +=strcspn (lcores,"@," ); + +if (*lcores =='@' ) { +/* explict assign cpu_set */ + offset =eal_parse_set (lcores +1 , set,RTE_DIM (set)); +if (offset <0 ) + goto err; + +/* prepare cpu_set and update the end cursor */ +if (0 >convert_to_cpuset (&cpuset, + set,RTE_DIM (set))) + goto err; + end = lcores +1 + offset; + }else /* ',' or '\0' */ +/* haven't given cpu_set, current loop done */ + end = lcores; + +if (*end !=',' && *end !='\0' ) + goto err; + +/* parse lcore_set from start point */ +if (0 >eal_parse_set (lcore_start, set,RTE_DIM (set))) + goto err; + +/* without '@', by default using lcore_set as cpu_set */ +if (*lcores !='@' && +0 >convert_to_cpuset (&cpuset, set,RTE_DIM (set))) + goto err; + +/* start to update lcore_set */ +for (idx =0 ; idx <RTE_MAX_LCORE ; idx++) { +if (!set[idx]) +continue ; + +if (cfg->lcore_role[idx] !=ROLE_RTE ) { + lcore_config[idx].core_index = count; + cfg->lcore_role[idx] =ROLE_RTE ; + count++; + } +rte_memcpy (&lcore_config[idx].cpuset, &cpuset, +sizeof (rte_cpuset_t)); + } + + lcores = end +1 ; + }while (*end !='\0' ); + +if (count ==0 ) + goto err; + + cfg->lcore_count = count; + lcores_parsed =1 ; + ret =0 ; + +err:Quoted 8 lines
lib/librte_eal/common/eal_common_options.c eal_parse_common_option(int opt, const char
Quoted 8 lines
lib/librte_eal/common/eal_common_options.c +4
-0+ +return ret; +} +static int eal_parse_syslog (const char *facility,struct internal_config *conf) {@@ -489,6 +728,13 @@ eal_parse_common_option(int opt, const char*optarg,
Quoted 11 lines
Quoted 11 lines
lib/librte_eal/common/eal_common_options.c +7
-0conf->log_level = log;break ; } +case OPT_LCORES_NUM : +if (eal_parse_lcores (optarg) <0 ) { +RTE_LOG (ERR ,EAL ,"invalid parameter for --" +OPT_LCORES "\n" ); +return -1 ; + } +break ;Quoted 3 lines
lib/librte_eal/common/eal_common_options.c eal_check_common_options(struct internal_config
Quoted 3 lines
lib/librte_eal/common/eal_common_options.c +0
-0/* don't know what to do, leave this to caller */ default :@@ -527,7 +773,7 @@ eal_check_common_options(struct internal_config
*internal_cfg)Quoted 1 line
Quoted 1 line
Quoted 9 lines
Quoted 9 lines
lib/librte_eal/common/eal_common_options.c eal_common_usage(void) · line 829
Quoted 9 lines
lib/librte_eal/common/eal_common_options.c +1
-1if (!lcores_parsed) {RTE_LOG (ERR ,EAL ,"CPU cores must be enabled with options " -"-c or -l\n" ); +"-c, -l or --lcores\n" );return -1 ; }if (cfg->lcore_role[cfg->master_lcore] !=ROLE_RTE ) {@@ -583,6 +829,14 @@ eal_common_usage(void) " The argument format is<c1>[-c2][,c3[-c4],...]\n"
Quoted 1 line
Quoted 1 line
lib/librte_eal/common/eal_common_options.c eal_common_usage(void) · line 830
Quoted 1 line
lib/librte_eal/common/eal_common_options.c +0
-0" where c1, c2, etc are core indexes between0 and %d\n"
Quoted 1 line
Quoted 1 line
lib/librte_eal/common/eal_common_options.c eal_common_usage(void) · line 831
Quoted 1 line
lib/librte_eal/common/eal_common_options.c +0
-0" --"OPT_MASTER_LCORE" ID: Core ID that is used asmaster\n"
Quoted 4 lines
Quoted 1 line
lib/librte_eal/common/eal_common_options.c eal_common_usage(void) · line 832
Quoted 1 line
lib/librte_eal/common/eal_common_options.c +1
-0+ " --"OPT_LCORES" MAP: maps between lcore_set tophys_cpu_set\n"
Quoted 2 lines
lib/librte_eal/common/eal_common_options.c eal_common_usage(void) · line 834
Quoted 2 lines
lib/librte_eal/common/eal_common_options.c +2
-0+" The argument format is\n" + "'lcores[@cpus]<,lcores[@cpus],...>'\n"
Quoted 1 line
Quoted 1 line
lib/librte_eal/common/eal_common_options.c eal_common_usage(void) · line 835
Quoted 1 line
lib/librte_eal/common/eal_common_options.c +1
-0+ " lcores and cpus list are grouped by '(' and')'\n"
Quoted 1 line
Quoted 1 line
lib/librte_eal/common/eal_common_options.c eal_common_usage(void) · line 836
Quoted 1 line
lib/librte_eal/common/eal_common_options.c +1
-0+ " Within the group, '-' is used for rangeseparator,\n"
Quoted 2 lines
Quoted 2 lines
lib/librte_eal/common/eal_common_options.c eal_common_usage(void) · line 838
Quoted 2 lines
lib/librte_eal/common/eal_common_options.c +2
-0+" ',' is used for single number separator.\n" + " '( )' can be omitted for single elementgroup, '@' \n"
Quoted 1 line
Quoted 1 line
lib/librte_eal/common/eal_common_options.c eal_common_usage(void) · line 839
Quoted 1 line
lib/librte_eal/common/eal_common_options.c +1
-0+ " can be omitted if cpus and lcores has thesame value\n"
Quoted 34 lines
Quoted 3 lines
lib/librte_eal/common/eal_common_options.c eal_common_usage(void) · line 842
Quoted 3 lines
lib/librte_eal/common/eal_common_options.c +0
-0" -n NUM : Number of memory channels\n" " -v : Display version information on startup\n" " -m MB : memory to allocate (see also --"OPT_SOCKET_MEM")\n"
Quoted 1 line
Review
Quoted 1 line
(quoted diff) +0
-0
b/lib/librte_eal/common/eal_options.hQuoted 12 lines
Quoted 12 lines
index e476f8d..a1cc59f 100644
lib/librte_eal/common/eal_options.h +2
-0@@ -77,6 +77,8 @@ enum {OPT_CREATE_UIO_DEV_NUM , #define OPT_VFIO_INTR "vfio-intr"OPT_VFIO_INTR_NUM , +#define OPT_LCORES "lcores" +OPT_LCORES_NUM , OPT_LONG_MAX_NUM };Quoted 1 line
Review
Quoted 1 line
(quoted diff) +0
-0b/lib/librte_eal/linuxapp/eal/Makefile
Quoted 14 lines
Quoted 14 lines
index 0e9c447..025d836 100644
lib/librte_eal/linuxapp/eal/Makefile +1
-1@@ -95,6 +95,7 @@ CFLAGS_eal_hugepage_info.o := -D_GNU_SOURCE CFLAGS_eal_pci.o := -D_GNU_SOURCE CFLAGS_eal_pci_vfio.o := -D_GNU_SOURCE CFLAGS_eal_common_whitelist.o := -D_GNU_SOURCE +CFLAGS_eal_common_options.o := -D_GNU_SOURCE # workaround for a gcc bug with noreturn attribute # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603 --1.8.1.4
Re: [PATCH v1 02/15] eal: new eal option '--lcores' for cpu assignment
In reply to a message not in the archiveQuoted 48 lines · Konstantin
Hi Bruce,
Quoted 24 lines
-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Bruce Richardson
Sent: Thursday, January 22, 2015 12:19 PM
To: Liang, Cunming
Cc: [email protected]
Subject: Re: [dpdk-dev] [PATCH v1 02/15] eal: new eal option '--lcores' for cpu assignmentQuoted 8 lines · Cunming Liang
It supports one new eal long option '--lcores' for EAL thread cpuset assignment.
The format pattern:
--lcores='lcores[@cpus]<,lcores[@cpus]>'lcores, cpus could be a single digit or a group.
'(' and ')' are necessary if it's a group.
If not supply '@cpus', the value of cpus uses the same as lcores.Quoted 7 lines
e.g. '1,2@(5-7),(3-5)@(0,2),(0,6)' means starting 7 EAL thread as below
lcore 0 runs on cpuset0x41(cpu 0,6)
lcore 1 runs on cpuset0x2(cpu 1)
lcore 2 runs on cpuset0xe0(cpu 5,6,7)
lcore 3,4,5 runs on cpuset0x5(cpu 0,2)
lcore 6 runs on cpuset0x41(cpu 0,6)Quoted 2 lines
This strikes me as very confusing, though a couple of tweaks might help with readability. The lcore 0 at the end is especially confusing.
Didn't get you here: do you find (0,6) confusing, right? Because braces implicitly specifies affinity for group of en-braced lcores?
Quoted 4 lines
Perhaps we can
limit the allowed formats here,
* require thelcore_idto be specified - the lack of an lcore id for the last part makes having it as lcore 0 surprising.Again, not sure I understand you properly:
lcore_id(s)are always specified explicitly. Physical cpus part might be omitted.Quoted 1 line
* only allow one lcore id to be given for each set of cores.
So you mean for '(3-5)@(0,2)' user would have to: '3@(0,2),4@(0,2),5@(0,2)'?
I don't see big difference here, but imagine you'd like to create a pool of 32 EAL-threads running on same cpu set. With current syntax it is just something like: '(32-63)@(0-7)'.
With what you proposing it will be a very long list.Quoted 1 line
Quoted 8 lines
Quoted 2 lines
I think it may still be readable if we allow the core set to be omitted if its to be the same as the
lcore_id.I think that is supported.
See lcore_id=1 in Steve's example above.
As I understand: --lcores='0,2,3-5' is equal to '-l 0,2,3-5' and to '-c0x3d'.Konstantin
Ok, thanks for the clarification.
/Bruce
Re: [PATCH v1 02/15] eal: new eal option '--lcores' for cpu assignment
In reply to Bruce RichardsonQuoted 27 lines
-----Original Message-----
From: Richardson, Bruce
Sent: Thursday, January 22, 2015 8:19 PM
To: Liang, Cunming
Cc: [email protected]
Subject: Re: [dpdk-dev] [PATCH v1 02/15] eal: new eal option '--lcores' for cpu assignmentQuoted 8 lines · Cunming Liang
It supports one new eal long option '--lcores' for EAL thread cpuset assignment.
The format pattern:
--lcores='lcores[@cpus]<,lcores[@cpus]>'lcores, cpus could be a single digit or a group.
'(' and ')' are necessary if it's a group.
If not supply '@cpus', the value of cpus uses the same as lcores.Quoted 7 lines
e.g. '1,2@(5-7),(3-5)@(0,2),(0,6)' means starting 7 EAL thread as below
lcore 0 runs on cpuset0x41(cpu 0,6)
lcore 1 runs on cpuset0x2(cpu 1)
lcore 2 runs on cpuset0xe0(cpu 5,6,7)
lcore 3,4,5 runs on cpuset0x5(cpu 0,2)
lcore 6 runs on cpuset0x41(cpu 0,6)
Quoted 6 lines
This strikes me as very confusing, though a couple of tweaks might help with readability. The lcore 0 at the end is especially confusing. Perhaps we can limit the allowed formats here,
* require thelcore_idto be specified - the lack of an lcore id for the last part makes having it as lcore 0 surprising.
* only allow one lcore id to be given for each set of cores.
[Liang, Cunming] The last one lcore_set (0,6) without cpuset assigned is equal to '(0,6)@(0,6)' or '0@(0,6), 6@(0,6)'. It's not a typical use case but gives an aggressive sample, it shows the simple way to explain the map.
Quoted 381 lines
I think it may still be readable if we allow the core set to be omitted if its to be the same as the
lcore_id.It's probably still not going to be very tidy, but I think we can improve things.
/Bruce
Quoted 4 lines
Signed-off-by Cunming Liang <[email protected]>--- lib/librte_eal/common/eal_common_launch.c | 1 - lib/librte_eal/common/eal_common_options.c | 262++++++++++++++++++++++++++++-
Review
Quoted 5 lines
lib/librte_eal/common/eal_options.h | 2 + lib/librte_eal/linuxapp/eal/Makefile | 1 + 4 files changed, 261 insertions(+), 5 deletions(-)(quoted diff) +0
-0
b/lib/librte_eal/common/eal_common_launch.cReview
Quoted 9 lines
index 599f83b..2d732b1 100644
lib/librte_eal/common/eal_common_launch.c +0
-1@@ -117,4 +117,3 @@ rte_eal_mp_wait_lcore(void)rte_eal_wait_lcore (lcore_id); } } -(quoted diff) +0
-0
b/lib/librte_eal/common/eal_common_options.clib/librte_eal/common/eal_common_options.c eal_parse_common_option(int opt, const char
Quoted 277 lines
index e2810ab..fc47588 100644
lib/librte_eal/common/eal_common_options.c +242
-3@@ -45,6 +45,7 @@#include <rte_lcore.h> #include <rte_version.h> #include <rte_devargs.h> +#include <rte_memcpy.h> #include "eal_internal_cfg.h" #include "eal_options.h" @@ -85,6 +86,7 @@ eal_long_options[] = { {OPT_XEN_DOM0, 0, 0, OPT_XEN_DOM0_NUM}, {OPT_CREATE_UIO_DEV, 1, NULL, OPT_CREATE_UIO_DEV_NUM}, {OPT_VFIO_INTR, 1, NULL, OPT_VFIO_INTR_NUM}, + {OPT_LCORES, 1, 0, OPT_LCORES_NUM}, {0, 0, 0, 0} };@@ -255,9 +257,11 @@ eal_parse_corelist(const char *corelist)if (min ==RTE_MAX_LCORE ) min = idx;for (idx = min; idx <= max; idx++) { - cfg->lcore_role[idx] =ROLE_RTE ; - lcore_config[idx].core_index = count; - count++; +if (cfg->lcore_role[idx] !=ROLE_RTE ) { + cfg->lcore_role[idx] =ROLE_RTE ; + lcore_config[idx].core_index = count; + count++; + } } min =RTE_MAX_LCORE ; }else @@ -289,6 +293,241 @@ eal_parse_master_lcore(const char *arg)return 0 ; } +/* +* Parse elem, the elem could be single number or '(' ')' group +* Within group elem, '-' used for a range seperator; +* ',' used for a single number. +*/ +static int +eal_parse_set (const char *input,uint16_t set[],unsigned num) +{ +unsigned idx; +const char *str = input; +char *end =NULL ; +unsigned min, max; + +memset (set,0 , num *sizeof (uint16_t )); + +while (isblank (*str)) + str++; + +/* only digit or left bracket is qulify for start point */ +if ((!isdigit (*str) && *str !='(' ) || *str =='\0' ) +return -1 ; + +/* process single number */ +if (*str !='(' ) { + errno =0 ; + idx =strtoul (str, &end,10 ); +if (errno || end ==NULL || idx >= num) +return -1 ; +else { +while (isblank (*end)) + end++; + +if (*end !=',' && *end !='\0' && + *end !='@' ) +return -1 ; + + set[idx] =1 ; +return end - input; + } + } + +/* process set within bracket */ + str++; +while (isblank (*str)) + str++; +if (*str =='\0' ) +return -1 ; + + min =RTE_MAX_LCORE ; +do { + +/* go ahead to the first digit */ +while (isblank (*str)) + str++; +if (!isdigit (*str)) +return -1 ; + +/* get the digit value */ + errno =0 ; + idx =strtoul (str, &end,10 ); +if (errno || end ==NULL || idx >= num) +return -1 ; + +/* go ahead to separator '-',',' and ')' */ +while (isblank (*end)) + end++; +if (*end =='-' ) { +if (min ==RTE_MAX_LCORE ) + min = idx; +else /* avoid continuous '-' */ +return -1 ; + }else if ((*end ==',' ) || (*end ==')' )) { + max = idx; +if (min ==RTE_MAX_LCORE ) + min = idx; +for (idx =RTE_MIN (min, max); + idx <=RTE_MAX (min, max); idx++) + set[idx] =1 ; + + min =RTE_MAX_LCORE ; + }else +return -1 ; + + str = end +1 ; + }while (*end !='\0' && *end !=')' ); + +return str - input; +} + +/* convert from set array to cpuset bitmap */ +static inline int +convert_to_cpuset (rte_cpuset_t *cpusetp, +uint16_t *set,unsigned num) +{ +unsigned idx; + +CPU_ZERO (cpusetp); + +for (idx =0 ; idx < num; idx++) { +if (!set[idx]) +continue ; + +if (!lcore_config[idx].detected) { +RTE_LOG (ERR ,EAL ,"core %u " +"unavailable\n" , idx); +return -1 ; + } + +CPU_SET (idx, cpusetp); + } + +return 0 ; +} + +/* +* The format pattern: --lcores='lcores[@cpus]<,lcores[@cpus]>' +* lcores, cpus could be a single digit or a group. +* '(' and ')' are necessary if it's a group. +* If not supply '@cpus', the value of cpus uses the same as lcores. +* e.g. '1,2@(5-7),(3-5)@(0,2),(0,6)' means start 7 EAL thread as below +* lcore 0 runs on cpuset 0x41 (cpu 0,6) +* lcore 1 runs on cpuset 0x2 (cpu 1) +* lcore 2 runs on cpuset 0xe0 (cpu 5,6,7) +* lcore 3,4,5 runs on cpuset 0x5 (cpu 0,2) +* lcore 6 runs on cpuset 0x41 (cpu 0,6) +*/ +static int +eal_parse_lcores (const char *lcores) +{ +struct rte_config *cfg =rte_eal_get_configuration (); +static uint16_t set[RTE_MAX_LCORE ]; +unsigned idx =0 ; +int i; +unsigned count =0 ; +const char *lcore_start =NULL ; +const char *end =NULL ; +int offset; +rte_cpuset_t cpuset; +int ret =-1 ; + +if (lcores ==NULL ) +return -1 ; + +/* Remove all blank characters ahead and after */ +while (isblank (*lcores)) + lcores++; + i =strnlen (lcores,sysconf (_SC_ARG_MAX)); +while ((i >0 ) &&isblank (lcores[i -1 ])) + i--; + +CPU_ZERO (&cpuset); + +/* Reset lcore config */ +for (idx =0 ; idx <RTE_MAX_LCORE ; idx++) { + cfg->lcore_role[idx] =ROLE_OFF ; + lcore_config[idx].core_index =-1 ; +CPU_ZERO (&lcore_config[idx].cpuset); + } + +/* Get list of cores */ +do { +while (isblank (*lcores)) + lcores++; +if (*lcores =='\0' ) + goto err; + +/* record lcore_set start point */ + lcore_start = lcores; + +/* go across a complete bracket */ +if (*lcore_start =='(' ) { + lcores +=strcspn (lcores,")" ); +if (*lcores++ =='\0' ) + goto err; + } + +/* scan the separator '@', ','(next) or '\0'(finish) */ + lcores +=strcspn (lcores,"@," ); + +if (*lcores =='@' ) { +/* explict assign cpu_set */ + offset =eal_parse_set (lcores +1 , set,RTE_DIM (set)); +if (offset <0 ) + goto err; + +/* prepare cpu_set and update the end cursor */ +if (0 >convert_to_cpuset (&cpuset, + set,RTE_DIM (set))) + goto err; + end = lcores +1 + offset; + }else /* ',' or '\0' */ +/* haven't given cpu_set, current loop done */ + end = lcores; + +if (*end !=',' && *end !='\0' ) + goto err; + +/* parse lcore_set from start point */ +if (0 >eal_parse_set (lcore_start, set,RTE_DIM (set))) + goto err; + +/* without '@', by default using lcore_set as cpu_set */ +if (*lcores !='@' && +0 >convert_to_cpuset (&cpuset, set,RTE_DIM (set))) + goto err; + +/* start to update lcore_set */ +for (idx =0 ; idx <RTE_MAX_LCORE ; idx++) { +if (!set[idx]) +continue ; + +if (cfg->lcore_role[idx] !=ROLE_RTE ) { + lcore_config[idx].core_index = count; + cfg->lcore_role[idx] =ROLE_RTE ; + count++; + } +rte_memcpy (&lcore_config[idx].cpuset, &cpuset, +sizeof (rte_cpuset_t)); + } + + lcores = end +1 ; + }while (*end !='\0' ); + +if (count ==0 ) + goto err; + + cfg->lcore_count = count; + lcores_parsed =1 ; + ret =0 ; + +err: + +return ret; +} +static int eal_parse_syslog (const char *facility,struct internal_config *conf) {@@ -489,6 +728,13 @@ eal_parse_common_option(int opt, const char*optarg,
lib/librte_eal/common/eal_common_options.c eal_check_common_options(struct internal_config
Quoted 14 lines
lib/librte_eal/common/eal_common_options.c +7
-0conf->log_level = log;break ; } +case OPT_LCORES_NUM : +if (eal_parse_lcores (optarg) <0 ) { +RTE_LOG (ERR ,EAL ,"invalid parameter for --" +OPT_LCORES "\n" ); +return -1 ; + } +break ;/* don't know what to do, leave this to caller */ default :@@ -527,7 +773,7 @@ eal_check_common_options(struct internal_config
*internal_cfg)lib/librte_eal/common/eal_common_options.c eal_common_usage(void) · line 830
Quoted 11 lines
lib/librte_eal/common/eal_common_options.c +1
-1if (!lcores_parsed) {RTE_LOG (ERR ,EAL ,"CPU cores must be enabled with options " -"-c or -l\n" ); +"-c, -l or --lcores\n" );return -1 ; }if (cfg->lcore_role[cfg->master_lcore] !=ROLE_RTE ) {@@ -583,6 +829,14 @@ eal_common_usage(void)" The argument format is <c1>[-c2][,c3[-c4],...]\n" " where c1, c2, etc are core indexes between 0and %d\n"
lib/librte_eal/common/eal_common_options.c eal_common_usage(void) · line 832
Quoted 2 lines
lib/librte_eal/common/eal_common_options.c +1
-0" --" OPT_MASTER_LCORE " ID: Core ID that is used as master\n" + " --"OPT_LCORES" MAP: maps between lcore_set tophys_cpu_set\n"
lib/librte_eal/common/eal_common_options.c eal_common_usage(void)
Quoted 4 lines
lib/librte_eal/common/eal_common_options.c +4
-0+" The argument format is\n" +" 'lcores[@cpus]<,lcores[@cpus],...>'\n" +" lcores and cpus list are grouped by '(' and ')'\n" + " Within the group, '-' is used for rangeseparator,\n"
lib/librte_eal/common/eal_common_options.c eal_common_usage(void) · line 838
Quoted 2 lines
lib/librte_eal/common/eal_common_options.c +2
-0+" ',' is used for single number separator.\n" + " '( )' can be omitted for single element group,'@' \n"
lib/librte_eal/common/eal_common_options.c eal_common_usage(void) · line 839
Quoted 1 line
lib/librte_eal/common/eal_common_options.c +1
-0+ " can be omitted if cpus and lcores has the samevalue\n"
lib/librte_eal/common/eal_common_options.c eal_common_usage(void)
Quoted 3 lines
lib/librte_eal/common/eal_common_options.c +0
-0" -n NUM : Number of memory channels\n" " -v : Display version information on startup\n" " -m MB : memory to allocate (see also--"OPT_SOCKET_MEM")\n"
Review
Quoted 1 line
(quoted diff) +0
-0
b/lib/librte_eal/common/eal_options.hReview
Quoted 13 lines
index e476f8d..a1cc59f 100644
lib/librte_eal/common/eal_options.h +2
-0@@ -77,6 +77,8 @@ enum {OPT_CREATE_UIO_DEV_NUM , #define OPT_VFIO_INTR "vfio-intr"OPT_VFIO_INTR_NUM , +#define OPT_LCORES "lcores" +OPT_LCORES_NUM , OPT_LONG_MAX_NUM };(quoted diff) +0
-0b/lib/librte_eal/linuxapp/eal/Makefile
Quoted 14 lines
index 0e9c447..025d836 100644
lib/librte_eal/linuxapp/eal/Makefile +1
-1@@ -95,6 +95,7 @@ CFLAGS_eal_hugepage_info.o := -D_GNU_SOURCE CFLAGS_eal_pci.o := -D_GNU_SOURCE CFLAGS_eal_pci_vfio.o := -D_GNU_SOURCE CFLAGS_eal_common_whitelist.o := -D_GNU_SOURCE +CFLAGS_eal_common_options.o := -D_GNU_SOURCE # workaround for a gcc bug with noreturn attribute # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603 --1.8.1.4
PATCH 3/15 eal: add support parsing socket_id from cpuset
In reply to a message not in the archiveIt returns the socket_id if all cpus in the cpuset belongs to the same NUMA node, otherwise it will return SOCKET_ID_ANY.
--- lib/librte_eal/bsdapp/eal/eal_lcore.c | 7 +++++ lib/librte_eal/common/eal_thread.h | 52 +++++++++++++++++++++++++++++++++ lib/librte_eal/linuxapp/eal/eal_lcore.c | 7 +++++ 3 files changed, 66 insertions(+)
lib/librte_eal/bsdapp/eal/eal_lcore.c +7 -0
@@ -41,6 +41,7 @@#include <rte_debug.h> #include "eal_private.h" +#include "eal_thread.h" /* No topology information available on FreeBSD including NUMA info */ #define cpu_core_id (X ) 0
@@ -112,3 +113,9 @@ rte_eal_cpu_init(void)return 0 ; } + +unsigned +eal_cpu_socket_id (__rte_unused unsigned cpu_id) +{ +return cpu_socket_id (cpu_id); +}
lib/librte_eal/common/eal_thread.h +52 -0
62 lines not on this page · raw message
lib/librte_eal/linuxapp/eal/eal_lcore.c +7 -0
@@ -45,6 +45,7 @@#include "eal_private.h" #include "eal_filesystem.h" +#include "eal_thread.h" #define SYS_CPU_DIR "/sys/devices/system/cpu/cpu%u"#define CORE_ID_FILE "topology/core_id"
@@ -197,3 +198,9 @@ rte_eal_cpu_init(void)return 0 ; } + +unsigned +eal_cpu_socket_id (unsigned cpu_id) +{ +return cpu_socket_id (cpu_id); +}
Signature · 1 line
1.8.1.4
PATCH 4/15 eal: new TLS definition and API declaration
In reply to a message not in the archive1. add two TLS *_socket_id* and *_cpuset*
2. add two external API rte_thread_set/get_affinity
3. add one internal API eal_thread_dump_affinity
--- lib/librte_eal/bsdapp/eal/eal_thread.c | 2 ++ lib/librte_eal/common/eal_thread.h | 14 ++++++++++++++ lib/librte_eal/common/include/rte_lcore.h | 29 +++++++++++++++++++++++++++-- lib/librte_eal/linuxapp/eal/eal_thread.c | 2 ++ 4 files changed, 45 insertions(+), 2 deletions(-)
lib/librte_eal/bsdapp/eal/eal_thread.c +2 -0
@@ -56,6 +56,8 @@#include "eal_thread.h" RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); +RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);/* * Send a message to a slave lcore identified by slave_id to call a
lib/librte_eal/common/eal_thread.h +14 -0
@@ -102,4 +102,18 @@ eal_cpuset_socket_id(rte_cpuset_t *cpusetp) return socket_id; } +/** +* Dump the current pthread cpuset. +* This function is private to EAL. +* +* @param str +* The string buffer the cpuset will dump to. +* @param size +* The string buffer size. +*/ +#define CPU_STR_LEN 256 +void +eal_thread_dump_affinity (char str[],unsigned size); + +#endif /* EAL_THREAD_H */
lib/librte_eal/common/include/rte_lcore.h +27 -2
@@ -43,6 +43,7 @@#include <rte_per_lcore.h> #include <rte_eal.h> #include <rte_launch.h> +#include <rte_memory.h> #ifdef __cplusplusextern "C" {
@@ -80,7 +81,9 @@ struct lcore_config {*/ extern struct lcore_config lcore_config[RTE_MAX_LCORE ]; -RTE_DECLARE_PER_LCORE (unsigned, _lcore_id);/**< Per core "core id". */ +RTE_DECLARE_PER_LCORE (unsigned, _lcore_id);/**< Per thread "lcore id". */ +RTE_DECLARE_PER_LCORE (unsigned, _socket_id);/**< Per thread "socket id". */ +RTE_DECLARE_PER_LCORE (rte_cpuset_t, _cpuset);/**< Per thread "cpuset". */ /** * Return the ID of the execution unit we are running on.
@@ -146,7 +149,7 @@ rte_lcore_index(int lcore_id)static inline unsigned rte_socket_id (void ) { -return lcore_config[rte_lcore_id ()].socket_id; +return RTE_PER_LCORE (_socket_id); }/**
@@ -229,6 +232,28 @@ rte_get_next_lcore(unsigned i, int skip_master, int wrap) i<RTE_MAX_LCORE ; \ i = rte_get_next_lcore(i, 1, 0)) +/** +* Set core affinity of the current thread. +* Support both EAL and none-EAL thread and update TLS. +* +* @param cpusetp +* Point to cpu_set_t for setting current thread affinity. +* @return +* On success, return 0; otherwise return -1; +*/ +int rte_thread_set_affinity (rte_cpuset_t *cpusetp); + +/** +* Get core affinity of the current thread. +* +* @param cpusetp +* Point to cpu_set_t for getting current thread cpu affinity. +* @return +* On success, return 0; otherwise return -1; +*/ +int rte_thread_get_affinity (rte_cpuset_t *cpusetp); + + #ifdef __cplusplus }#endif
lib/librte_eal/linuxapp/eal/eal_thread.c +2 -0
@@ -56,6 +56,8 @@#include "eal_thread.h" RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); +RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);/* * Send a message to a slave lcore identified by slave_id to call a
Signature · 1 line
1.8.1.4
PATCH 5/15 eal: add eal_common_thread.c for common thread API
In reply to a message not in the archiveThe API works for both EAL thread and none EAL thread. When calling rte_thread_set_affinity, the *_socket_id* and *_cpuset* of calling thread will be updated if the thread successful set the cpu affinity.
--- lib/librte_eal/bsdapp/eal/Makefile | 1 + lib/librte_eal/common/eal_common_thread.c | 142 ++++++++++++++++++++++++++++++ lib/librte_eal/linuxapp/eal/Makefile | 2 + 3 files changed, 145 insertions(+) create mode 100644 lib/librte_eal/common/eal_common_thread.c
lib/librte_eal/bsdapp/eal/Makefile +1 -0
@@ -73,6 +73,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_EAL_BSDAPP) += eal_common_hexdump.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_BSDAPP) += eal_common_devargs.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_BSDAPP) += eal_common_dev.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_BSDAPP) += eal_common_options.c +SRCS-$(CONFIG_RTE_LIBRTE_EAL_BSDAPP) += eal_common_thread.c CFLAGS_eal.o := -D_GNU_SOURCE #CFLAGS_eal_thread.o := -D_GNU_SOURCE
lib/librte_eal/common/eal_common_thread.c new file +142 -0
142 lines not on this page · raw message
lib/librte_eal/linuxapp/eal/Makefile +2 -0
@@ -85,6 +85,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_hexdump.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_devargs.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_dev.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_options.c +SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_thread.c CFLAGS_eal.o := -D_GNU_SOURCE CFLAGS_eal_lcore.o := -D_GNU_SOURCE
@@ -96,6 +97,7 @@ CFLAGS_eal_pci.o := -D_GNU_SOURCE CFLAGS_eal_pci_vfio.o := -D_GNU_SOURCE CFLAGS_eal_common_whitelist.o := -D_GNU_SOURCE CFLAGS_eal_common_options.o := -D_GNU_SOURCE +CFLAGS_eal_common_thread.o := -D_GNU_SOURCE # workaround for a gcc bug with noreturn attribute # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603
Signature · 1 line
1.8.1.4
PATCH 6/15 eal: add rte_gettid() to acquire unique system tid
In reply to a message not in the archiveThe rte_gettid() wraps the linux and freebsd syscall gettid(). It provides a persistent unique thread id for the calling thread. It will save the unique id in TLS on the first time.
--- lib/librte_eal/bsdapp/eal/eal_thread.c | 9 +++++++++ lib/librte_eal/common/include/rte_eal.h | 27 +++++++++++++++++++++++++++ lib/librte_eal/linuxapp/eal/eal_thread.c | 7 +++++++ 3 files changed, 43 insertions(+)
lib/librte_eal/bsdapp/eal/eal_thread.c +9 -0
@@ -39,6 +39,7 @@#include <sched.h> #include <pthread_np.h> #include <sys/queue.h> +#include <sys/thr.h> #include <rte_debug.h> #include <rte_atomic.h>
@@ -233,3 +234,11 @@ eal_thread_loop(__attribute__((unused)) void *arg)/* pthread_exit(NULL); */ /* return NULL; */ } + +/* require calling thread tid by gettid() */ +int rte_sys_gettid (void ) +{ +long lwpid; +thr_self (&lwpid); +return (int )lwpid; +}
lib/librte_eal/common/include/rte_eal.h +27 -0
@@ -41,6 +41,9 @@*/ #include <stdint.h> +#include <sched.h> + +#include <rte_per_lcore.h> #ifdef __cplusplusextern "C" {
@@ -262,6 +265,30 @@ rte_set_application_usage_hook( rte_usage_hook_t usage_func );*/ int rte_eal_has_hugepages (void ); +/** +* A wrap API for syscall gettid. +* +* @return +* On success, returns the thread ID of calling process. +* It always successful. +*/ +int rte_sys_gettid (void ); + +/** +* Get system unique thread id. +* +* @return +* On success, returns the thread ID of calling process. +* It always successful. +*/ +static inline int rte_gettid (void ) +{ + static RTE_DEFINE_PER_LCORE(int, _thread_id) = -1; +if (RTE_PER_LCORE (_thread_id) ==-1 ) +RTE_PER_LCORE (_thread_id) =rte_sys_gettid (); +return RTE_PER_LCORE (_thread_id); +} + #ifdef __cplusplus }#endif
lib/librte_eal/linuxapp/eal/eal_thread.c +7 -0
@@ -39,6 +39,7 @@#include <pthread.h> #include <sched.h> #include <sys/queue.h> +#include <sys/syscall.h> #include <rte_debug.h> #include <rte_atomic.h>
@@ -233,3 +234,9 @@ eal_thread_loop(__attribute__((unused)) void *arg)/* pthread_exit(NULL); */ /* return NULL; */ } + +/* require calling thread tid by gettid() */ +int rte_sys_gettid (void ) +{ +return (int )syscall (SYS_gettid); +}
Signature · 1 line
1.8.1.4
PATCH 7/15 eal: apply affinity of EAL thread by assigned cpuset
In reply to a message not in the archiveEAL threads use assigned cpuset to set core affinity during startup. It keeps 1:1 mapping, if no '--lcores' option is used.
--- lib/librte_eal/bsdapp/eal/eal.c | 13 ++++--- lib/librte_eal/bsdapp/eal/eal_thread.c | 63 +++++++++--------------------- lib/librte_eal/linuxapp/eal/eal.c | 7 +++- lib/librte_eal/linuxapp/eal/eal_thread.c | 67 +++++++++++--------------------- 4 files changed, 54 insertions(+), 96 deletions(-)
lib/librte_eal/bsdapp/eal/eal.c +8 -5
@@ -432,6 +432,7 @@ rte_eal_init(int argc, char **argv)int i, fctret, ret;pthread_t thread_id;static rte_atomic32_t run_once =RTE_ATOMIC32_INIT (0 ); +char cpuset[CPU_STR_LEN ];if (!rte_atomic32_test_and_set (&run_once))return -1 ;
@@ -502,13 +503,17 @@ rte_eal_init(int argc, char **argv)if (rte_eal_pci_init () <0 )rte_panic ("Cannot init PCI\n" ); -RTE_LOG (DEBUG ,EAL ,"Master core %u is ready (tid=%p)\n" , - rte_config.master_lcore, thread_id); -eal_check_mem_on_local_socket ();rte_eal_mcfg_complete (); +eal_thread_init_master (rte_config.master_lcore); + +eal_thread_dump_affinity (cpuset,CPU_STR_LEN ); + +RTE_LOG (DEBUG ,EAL ,"Master lcore %u is ready (tid=%p;cpuset=[%s])\n" , + rte_config.master_lcore, thread_id, cpuset); +if (rte_eal_dev_init () <0 )rte_panic ("Cannot init pmd devices\n" );
@@ -532,8 +537,6 @@ rte_eal_init(int argc, char **argv)rte_panic ("Cannot create thread\n" ); } -eal_thread_init_master (rte_config.master_lcore); -/* * Launch a dummy function on all slave lcores, so that master lcore * knows they are all ready when this function returns.
lib/librte_eal/bsdapp/eal/eal_thread.c +19 -44
95 lines not on this page · raw message
lib/librte_eal/linuxapp/eal/eal.c +5 -2
@@ -702,6 +702,7 @@ rte_eal_init(int argc, char **argv)static rte_atomic32_t run_once =RTE_ATOMIC32_INIT (0 );struct shared_driver *solib =NULL ;const char *logid; +char cpuset[CPU_STR_LEN ];if (!rte_atomic32_test_and_set (&run_once))return -1 ;
@@ -802,8 +803,10 @@ rte_eal_init(int argc, char **argv)eal_thread_init_master (rte_config.master_lcore); -RTE_LOG (DEBUG ,EAL ,"Master core %u is ready (tid=%x)\n" , - rte_config.master_lcore, (int )thread_id); +eal_thread_dump_affinity (cpuset,CPU_STR_LEN ); + +RTE_LOG (DEBUG ,EAL ,"Master lcore %u is ready (tid=%x;cpuset=[%s])\n" , + rte_config.master_lcore, (int )thread_id, cpuset);if (rte_eal_dev_init () <0 )rte_panic ("Cannot init pmd devices\n" );
lib/librte_eal/linuxapp/eal/eal_thread.c +22 -45
110 lines not on this page · raw message
Signature · 1 line
1.8.1.4
PATCH 8/15 enic: fix re-define freebsd compile complain
In reply to a message not in the archiveSome macro already been defined by freebsd 'sys/param.h'.
--- lib/librte_pmd_enic/enic.h | 1 + lib/librte_pmd_enic/enic_compat.h | 1 + 2 files changed, 2 insertions(+)
lib/librte_pmd_enic/enic.h +1 -0
@@ -66,6 +66,7 @@#define ENIC_CALC_IP_CKSUM 1#define ENIC_CALC_TCP_UDP_CKSUM 2#define ENIC_MAX_MTU 9000 +#undef PAGE_SIZE#define PAGE_SIZE 4096#define PAGE_ROUND_UP (x) \ ((((unsigned long)(x)) + PAGE_SIZE-1) & (~(PAGE_SIZE-1)))
lib/librte_pmd_enic/enic_compat.h +1 -0
@@ -67,6 +67,7 @@ #define pr_warn(y, args...) dev_warning(0, y, ##args)#define BUG () pr_err("BUG at %s:%d", __func__, __LINE__) +#undef ALIGN#define ALIGN (x, a) __ALIGN_MASK(x, (typeof(x))(a)-1)#define __ALIGN_MASK (x, mask) (((x)+(mask))&~(mask))#define udelay usleep
Signature · 1 line
1.8.1.4
PATCH 9/15 malloc: fix the issue of SOCKET_ID_ANY
In reply to a message not in the archiveAdd check for rte_socket_id(), avoid get unexpected return like (-1).
--- lib/librte_malloc/malloc_heap.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)
lib/librte_malloc/malloc_heap.h +6 -1
@@ -44,7 +44,12 @@ extern "C" {static inline unsigned malloc_get_numa_socket (void ) { -return rte_socket_id (); +unsigned socket_id =rte_socket_id (); + +if (socket_id == (unsigned )SOCKET_ID_ANY ) +return 0 ; + +return socket_id; } void *
Signature · 1 line
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 1421914598-2747-10-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
Re: [PATCH v1 09/15] malloc: fix the issue of SOCKET_ID_ANY
lib/librte_malloc/malloc_heap.h extern "C" { · line 49
Quoted 4 lines · On Thu, 22 Jan 2015 16:16:32 +0800 Cunming Liang…
lib/librte_malloc/malloc_heap.h +3
-1-return rte_socket_id (); +unsigned socket_id =rte_socket_id (); + +if (socket_id == (unsigned )SOCKET_ID_ANY )
I prefer not casting -1 to unsigned it will cause warnings. It is better to make socket_id an integer and then have the implicit cast in the return.
A message that is not in the archive (sent off-list or not yet received): 20150125230449.3dbf4306-CA4OZQ/[email protected]
Re: [PATCH v1 09/15] malloc: fix the issue of SOCKET_ID_ANY
Quoted 12 lines
-----Original Message-----
From: Stephen Hemminger [mailto:[email protected]]
Sent: Sunday, January 25, 2015 4:05 PM
To: Liang, Cunming
Cc: [email protected]
Subject: Re: [dpdk-dev] [PATCH v1 09/15] malloc: fix the issue ofSOCKET_ID_ANYOn Thu, 22 Jan 2015 16:16:32 +0800
Cunming Liang <[email protected]> wrote:
Quoted 8 lines
lib/librte_malloc/malloc_heap.h extern "C" { · line 49
Quoted 4 lines
lib/librte_malloc/malloc_heap.h +3
-1-return rte_socket_id (); +unsigned socket_id =rte_socket_id (); + +if (socket_id == (unsigned )SOCKET_ID_ANY )I prefer not casting -1 to unsigned it will cause warnings. It is better to make
socket_idan integer and then have the implicit cast in the return.
[Liang, Cunming] I didn't got warning about it, in which version of compiler complain it ?
Re: [PATCH v1 09/15] malloc: fix the issue of SOCKET_ID_ANY
In reply to a message not in the archivelib/librte_malloc/malloc_heap.h extern "C" { · line 49
Quoted 4 lines · On Thu, 22 Jan 2015 16:16:32 +0800 Cunming Liang…
lib/librte_malloc/malloc_heap.h +3
-1-return rte_socket_id (); +unsigned socket_id =rte_socket_id (); + +if (socket_id == (unsigned )SOCKET_ID_ANY )
I prefer not casting -1 to unsigned it will cause warnings. It is better to make socket_id an integer and then have the implicit cast in the return
PATCH 10/15 log: fix the gap to support non-EAL thread
In reply to a message not in the archiveFor those non-EAL thread, *_lcore_id* is invalid and probably larger than RTE_MAX_LCORE. The patch adds the check and allows only EAL thread using EAL per thread log level and log type. Others shares the global log level.
--- lib/librte_eal/common/eal_common_log.c | 17 +++++++++++++++-- lib/librte_eal/common/include/rte_log.h | 5 +++++ 2 files changed, 20 insertions(+), 2 deletions(-)
lib/librte_eal/common/eal_common_log.c +15 -2
@@ -193,11 +193,20 @@ rte_set_log_type(uint32_t type, int enable) rte_logs.type &= (~type); } +/* Get global log type */ +uint32_t +rte_get_log_type (void ) +{ +return rte_logs.type; +} +/* get the current loglevel for the message beeing processed */ int rte_log_cur_msg_loglevel (void ) {unsigned lcore_id; lcore_id =rte_lcore_id (); +if (lcore_id >=RTE_MAX_LCORE ) +return rte_get_log_level ();return log_cur_msg[lcore_id].loglevel; }
@@ -206,6 +215,8 @@ int rte_log_cur_msg_logtype(void) {unsigned lcore_id; lcore_id =rte_lcore_id (); +if (lcore_id >=RTE_MAX_LCORE ) +return rte_get_log_type ();return log_cur_msg[lcore_id].logtype; }
@@ -265,8 +276,10 @@ rte_vlog(__attribute__((unused)) uint32_t level,/* save loglevel and logtype in a global per-lcore variable */ lcore_id =rte_lcore_id (); - log_cur_msg[lcore_id].loglevel = level; - log_cur_msg[lcore_id].logtype = logtype; +if (lcore_id <RTE_MAX_LCORE ) { + log_cur_msg[lcore_id].loglevel = level; + log_cur_msg[lcore_id].logtype = logtype; + } ret =vfprintf (f, format, ap);fflush (f);
lib/librte_eal/common/include/rte_log.h +5 -0
@@ -144,6 +144,11 @@ uint32_t rte_get_log_level(void);void rte_set_log_type (uint32_t type,int enable);/** +* Get the global log type. +*/ +uint32_t rte_get_log_type (void ); + +/** * Get the current loglevel for the message being processed. * * Before calling the user-defined stream for logging, the log
Signature · 1 line
1.8.1.4
PATCH 11/15 eal: set _lcore_id and _socket_id to (-1) by default
In reply to a message not in the archiveFor those none EAL thread, *_lcore_id* shall always be LCORE_ID_ANY. The libraries using *_lcore_id* as index need to take care. *_socket_id* always be SOCKET_ID_ANY unitl the thread changes the affinity by rte_thread_set_affinity()
--- lib/librte_eal/bsdapp/eal/eal_thread.c | 4 ++-- lib/librte_eal/linuxapp/eal/eal_thread.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-)
lib/librte_eal/bsdapp/eal/eal_thread.c +2 -2
@@ -56,8 +56,8 @@#include "eal_private.h" #include "eal_thread.h" -RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); -RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (unsigned, _lcore_id) = (unsigned )LCORE_ID_ANY ; +RTE_DEFINE_PER_LCORE (unsigned, _socket_id) = (unsigned )SOCKET_ID_ANY ;RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);/*
lib/librte_eal/linuxapp/eal/eal_thread.c +2 -2
@@ -57,8 +57,8 @@#include "eal_private.h" #include "eal_thread.h" -RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); -RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (unsigned, _lcore_id) = (unsigned )LCORE_ID_ANY ; +RTE_DEFINE_PER_LCORE (unsigned, _socket_id) = (unsigned )SOCKET_ID_ANY ;RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);/*
Signature · 1 line
1.8.1.4
PATCH 12/15 eal: fix recursive spinlock in non-EAL thraed
In reply to a message not in the archiveIn non-EAL thread, lcore_id alrways be LCORE_ID_ANY. It cann't be used as unique id for recursive spinlock. Then use rte_gettid() to replace it.
--- lib/librte_eal/common/include/generic/rte_spinlock.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
lib/librte_eal/common/include/generic/rte_spinlock.h +2 -2
@@ -179,7 +179,7 @@ static inline void rte_spinlock_recursive_init(rte_spinlock_recursive_t *slr)*/ static inline void rte_spinlock_recursive_lock (rte_spinlock_recursive_t *slr) { -int id =rte_lcore_id (); +int id =rte_gettid ();if (slr->user != id) {rte_spinlock_lock (&slr->sl);
@@ -212,7 +212,7 @@ static inline void rte_spinlock_recursive_unlock(rte_spinlock_recursive_t *slr)*/ static inline int rte_spinlock_recursive_trylock (rte_spinlock_recursive_t *slr) { -int id =rte_lcore_id (); +int id =rte_gettid ();if (slr->user != id) {if (rte_spinlock_trylock (&slr->sl) ==0 )
Signature · 1 line
1.8.1.4
PATCH 13/15 mempool: add support to non-EAL thread
In reply to a message not in the archiveFor non-EAL thread, bypass per lcore cache, directly use ring pool. It allows using rte_mempool in either EAL thread or any user pthread. As in non-EAL thread, it directly rely on rte_ring and it's none preemptive. It doesn't suggest to run multi-pthread/cpu which compete the rte_mempool. It will get bad performance and has critical risk if scheduling policy is RT.
--- lib/librte_mempool/rte_mempool.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-)
lib/librte_mempool/rte_mempool.h +11 -7
@@ -198,10 +198,12 @@ struct rte_mempool {* Number to add to the object-oriented statistics. */ #ifdef RTE_LIBRTE_MEMPOOL_DEBUG -#define __MEMPOOL_STAT_ADD (mp, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - mp->stats[__lcore_id].name##_objs += n; \ - mp->stats[__lcore_id].name##_bulk += 1; \ +#define __MEMPOOL_STAT_ADD (mp, name, n) do { \ + unsigned __lcore_id = rte_lcore_id(); \ + if (__lcore_id < RTE_MAX_LCORE) { \ + mp->stats[__lcore_id].name##_objs += n; \ + mp->stats[__lcore_id].name##_bulk += 1; \ + } \ } while(0)#else #define __MEMPOOL_STAT_ADD (mp, name, n) do {} while(0)
@@ -767,8 +769,9 @@ __mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table,__MEMPOOL_STAT_ADD (mp, put, n);#if RTE_MEMPOOL_CACHE_MAX_SIZE >0 -/* cache is not enabled or single producer */ -if (unlikely (cache_size ==0 || is_mp ==0 )) +/* cache is not enabled or single producer or none EAL thread */ +if (unlikely (cache_size ==0 || is_mp ==0 || + lcore_id >=RTE_MAX_LCORE )) goto ring_enqueue;/* Go straight to ring if put would overflow mem allocated for cache */
@@ -952,7 +955,8 @@ __mempool_get_bulk(struct rte_mempool *mp, void **obj_table,uint32_t cache_size = mp->cache_size;/* cache is not enabled or single consumer */ -if (unlikely (cache_size ==0 || is_mc ==0 || n >= cache_size)) +if (unlikely (cache_size ==0 || is_mc ==0 || + n >= cache_size || lcore_id >=RTE_MAX_LCORE )) goto ring_dequeue; cache = &mp->local_cache[lcore_id];
Signature · 1 line
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 1421914598-2747-14-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
Re: [PATCH v1 13/15] mempool: add support to non-EAL thread
Quoted 44 lines
-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Cunming Liang
Sent: Thursday, January 22, 2015 9:17 AM
To: [email protected]
Subject: [dpdk-dev] [PATCH v1 13/15] mempool: add support to non-EAL threadFor non-EAL thread, bypass per lcore cache, directly use ring pool. It allows using
rte_mempoolin either EAL thread or any user pthread. As in non-EAL thread, it directly rely onrte_ringand it's none preemptive. It doesn't suggest to run multi-pthread/cpu which compete therte_mempool.
It will get bad performance and has critical risk if scheduling policy is RT.Signed-off-by Cunming Liang <[email protected]>--- lib/librte_mempool/rte_mempool.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-)lib/librte_mempool/rte_mempool.h +6
-4@@ -198,10 +198,12 @@ struct rte_mempool {* Number to add to the object-oriented statistics. */ #ifdef RTE_LIBRTE_MEMPOOL_DEBUG -#define __MEMPOOL_STAT_ADD (mp, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - mp->stats[__lcore_id].name##_objs += n; \ - mp->stats[__lcore_id].name##_bulk += 1; \ +#define __MEMPOOL_STAT_ADD (mp, name, n) do { \ + unsigned __lcore_id = rte_lcore_id(); \ + if (__lcore_id < RTE_MAX_LCORE) { \ + mp->stats[__lcore_id].name##_objs += n; \ + mp->stats[__lcore_id].name##_bulk += 1; \ + } \ } while(0)#else #define __MEMPOOL_STAT_ADD (mp, name, n) do {} while(0)
lib/librte_mempool/rte_mempool.h __mempool_put_bulk(struct rte_mempool *mp, · line 773
Quoted 7 lines
lib/librte_mempool/rte_mempool.h +1
-2@@ -767,8 +769,9 @@ __mempool_put_bulk(struct rte_mempool *mp,void *const *obj_table,__MEMPOOL_STAT_ADD (mp ,put ,n );#if RTE_MEMPOOL_CACHE_MAX_SIZE >0 -/* cache is not enabled or single producer */ -if (unlikely (cache_size ==0 || is_mp ==0 )) +/* cache is not enabled or single producer or none EAL thread */
I don't understand this limitation.
I see that the rte_membuf.h defines table per RTE_MAX_LCORE like below
#if RTE_MEMPOOL_CACHE_MAX_SIZE > 0
/** Per-lcore local cache. */
struct rte_mempool_cache local_cache[RTE_MAX_LCORE];#endif
But why we cannot extent the size of the local cache table to something like RTE_MAX_THREADS that does not exceed max value of rte_lcore_id()
Keeping this condition here is a real performance killer!!!!!!.
I saw in my test application spending more 95% of CPU time reading the atomic in M C/MP ring utilizing access to mempool.
Same comment for get operation below
Quoted 19 lines
lib/librte_mempool/rte_mempool.h +2
-0+if (unlikely (cache_size ==0 || is_mp ==0 || + lcore_id >=RTE_MAX_LCORE )) goto ring_enqueue;/* Go straight to ring if put would overflow mem allocated for cache
*/lib/librte_mempool/rte_mempool.h +2
-1@@ -952,7 +955,8 @@ __mempool_get_bulk(struct rte_mempool *mp, void **obj_table, uint32_t cache_size = mp->cache_size;/* cache is not enabled or single consumer */ -if (unlikely (cache_size ==0 || is_mc ==0 || n >= cache_size)) +if (unlikely (cache_size ==0 || is_mc ==0 || + n >= cache_size || lcore_id >=RTE_MAX_LCORE )) goto ring_dequeue;cache = &mp->local_cache[lcore_id];
--
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 7C4248CAE043B144B1CD242D275626532FE3BEE9-kPTMFJFq+rGvNW/[email protected]
Re: [PATCH v1 13/15] mempool: add support to non-EAL thread
Quoted 67 lines
-----Original Message-----
From: Walukiewicz, Miroslaw
Sent: Thursday, January 22, 2015 5:53 PM
To: Liang, Cunming; [email protected]
Subject: RE: [dpdk-dev] [PATCH v1 13/15] mempool: add support to non-EAL threadQuoted 42 lines
-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Cunming Liang
Sent: Thursday, January 22, 2015 9:17 AM
To: [email protected]
Subject: [dpdk-dev] [PATCH v1 13/15] mempool: add support to non-EAL threadFor non-EAL thread, bypass per lcore cache, directly use ring pool. It allows using
rte_mempoolin either EAL thread or any user pthread. As in non-EAL thread, it directly rely onrte_ringand it's none preemptive. It doesn't suggest to run multi-pthread/cpu which compete therte_mempool.
It will get bad performance and has critical risk if scheduling policy is RT.Signed-off-by Cunming Liang <[email protected]>--- lib/librte_mempool/rte_mempool.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-)lib/librte_mempool/rte_mempool.h +6
-4@@ -198,10 +198,12 @@ struct rte_mempool {* Number to add to the object-oriented statistics. */ #ifdef RTE_LIBRTE_MEMPOOL_DEBUG -#define __MEMPOOL_STAT_ADD (mp, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - mp->stats[__lcore_id].name##_objs += n; \ - mp->stats[__lcore_id].name##_bulk += 1; \ +#define __MEMPOOL_STAT_ADD (mp, name, n) do { \ + unsigned __lcore_id = rte_lcore_id(); \ + if (__lcore_id < RTE_MAX_LCORE) { \ + mp->stats[__lcore_id].name##_objs += n; \ + mp->stats[__lcore_id].name##_bulk += 1; \ + } \ } while(0)#else #define __MEMPOOL_STAT_ADD (mp, name, n) do {} while(0)lib/librte_mempool/rte_mempool.h __mempool_put_bulk(struct rte_mempool *mp, · line 773
Quoted 7 lines
lib/librte_mempool/rte_mempool.h +1
-2@@ -767,8 +769,9 @@ __mempool_put_bulk(struct rte_mempool *mp,void *const *obj_table,__MEMPOOL_STAT_ADD (mp ,put ,n );#if RTE_MEMPOOL_CACHE_MAX_SIZE >0 -/* cache is not enabled or single producer */ -if (unlikely (cache_size ==0 || is_mp ==0 )) +/* cache is not enabled or single producer or none EAL thread */ I don't understand this limitation.
I see that the
rte_membuf.hdefines table perRTE_MAX_LCORElike below#if RTE_MEMPOOL_CACHE_MAX_SIZE > 0 /** Per-lcore local cache. */ struct rte_mempool_cache local_cache[RTE_MAX_LCORE];
Quoted 8 lines
#endif
But why we cannot extent the size of the local cache table to something like
RTE_MAX_THREADSthat does not exceed max value ofrte_lcore_id()Keeping this condition here is a real performance killer!!!!!!. I saw in my test application spending more 95% of CPU time reading the atomic in M C/MP ring utilizing access to mempool.
[Liang, Cunming] This is the first step to make it work.
By Konstantin's comments, shall prevent to allocate unique id by ourselves. And the return value from gettid() is too large as an index.
For non-EAL thread performance gap, will think about additional fix patch here. If care about performance, still prefer to choose EAL thread now.
Quoted 22 lines
Same comment for get operation below
Quoted 19 lines
lib/librte_mempool/rte_mempool.h +2
-0+if (unlikely (cache_size ==0 || is_mp ==0 || + lcore_id >=RTE_MAX_LCORE )) goto ring_enqueue;/* Go straight to ring if put would overflow mem allocated for cache
*/lib/librte_mempool/rte_mempool.h +2
-1@@ -952,7 +955,8 @@ __mempool_get_bulk(struct rte_mempool *mp, void **obj_table, uint32_t cache_size = mp->cache_size;/* cache is not enabled or single consumer */ -if (unlikely (cache_size ==0 || is_mc ==0 || n >= cache_size)) +if (unlikely (cache_size ==0 || is_mc ==0 || + n >= cache_size || lcore_id >=RTE_MAX_LCORE )) goto ring_dequeue;cache = &mp->local_cache[lcore_id];
--
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): D0158A423229094DA7ABF71CF2FA0DA3118A975E-E2R4CRU6q/[email protected]
Re: [PATCH v1 13/15] mempool: add support to non-EAL thread
Quoted 86 lines
-----Original Message-----
From: Liang, Cunming
Sent: Thursday, January 22, 2015 1:20 PM
To: Walukiewicz, Miroslaw; [email protected]
Subject: RE: [dpdk-dev] [PATCH v1 13/15] mempool: add support to non-EAL threadQuoted 5 lines
-----Original Message-----
From: Walukiewicz, Miroslaw
Sent: Thursday, January 22, 2015 5:53 PM
To: Liang, Cunming; [email protected]
Subject: RE: [dpdk-dev] [PATCH v1 13/15] mempool: add support to non-EAL
Quoted 23 lines
thread
Quoted 19 lines
-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Cunming Liang
Sent: Thursday, January 22, 2015 9:17 AM
To: [email protected]
Subject: [dpdk-dev] [PATCH v1 13/15] mempool: add support to non-EAL threadFor non-EAL thread, bypass per lcore cache, directly use ring pool. It allows using
rte_mempoolin either EAL thread or any user pthread. As in non-EAL thread, it directly rely onrte_ringand it's none preemptive. It doesn't suggest to run multi-pthread/cpu which compete therte_mempool.
It will get bad performance and has critical risk if scheduling policy is RT.Signed-off-by Cunming Liang <[email protected]>--- lib/librte_mempool/rte_mempool.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-)Quoted 10 lines
lib/librte_mempool/rte_mempool.h struct rte_mempool { · line 201
Quoted 10 lines
lib/librte_mempool/rte_mempool.h +0
-1@@ -198,10 +198,12 @@ struct rte_mempool {* Number to add to the object-oriented statistics. */ #ifdef RTE_LIBRTE_MEMPOOL_DEBUG -#define __MEMPOOL_STAT_ADD (mp, name, n) do {\Quoted 23 lines
Quoted 13 lines
lib/librte_mempool/rte_mempool.h +6
-3-unsigned __lcore_id =rte_lcore_id (); \ - mp->stats[__lcore_id].name##_objs += n; \ - mp->stats[__lcore_id].name##_bulk += 1; \ +#define __MEMPOOL_STAT_ADD (mp, name, n) do { \ + unsigned __lcore_id = rte_lcore_id(); \ + if (__lcore_id < RTE_MAX_LCORE) { \ + mp->stats[__lcore_id].name##_objs += n; \ + mp->stats[__lcore_id].name##_bulk += 1; \ + } \ } while(0)#else #define __MEMPOOL_STAT_ADD (mp, name, n) do {} while(0)lib/librte_mempool/rte_mempool.h __mempool_put_bulk(struct rte_mempool *mp, · line 773
Quoted 7 lines
lib/librte_mempool/rte_mempool.h +1
-2@@ -767,8 +769,9 @@ __mempool_put_bulk(struct rte_mempool *mp,void *const *obj_table,__MEMPOOL_STAT_ADD (mp ,put ,n );#if RTE_MEMPOOL_CACHE_MAX_SIZE >0 -/* cache is not enabled or single producer */ -if (unlikely (cache_size ==0 || is_mp ==0 )) +/* cache is not enabled or single producer or none EAL thread */ I don't understand this limitation.
Quoted 7 lines
I see that the
rte_membuf.hdefines table perRTE_MAX_LCORElike below#if RTE_MEMPOOL_CACHE_MAX_SIZE > 0 /** Per-lcore local cache. */ struct rte_mempool_cache local_cache[RTE_MAX_LCORE];#endif
But why we cannot extent the size of the local cache table to something
like
Quoted 4 lines
RTE_MAX_THREADSthat does not exceed max value ofrte_lcore_id()Keeping this condition here is a real performance killer!!!!!!. I saw in my test application spending more 95% of CPU time reading the
Quoted 8 lines
atomic
Quoted 1 line
in M C/MP ring utilizing access to mempool.
[Liang, Cunming] This is the first step to make it work.
By Konstantin's comments, shall prevent to allocate unique id by ourselves. And the return value fromgettid()is too large as an index. For non-EAL thread performance gap, will think about additional fix patch here.
If care about performance, still prefer to choose EAL thread now.
In previous patch you had allocation of the thread id on base of unique gettid() as number not a potential pointer as we can expect from implementation getid() from Linux or BSD.
The another problem is that we compare here int with some unique thread identifier. How can you prevent that when implementation of gettid will change and unique thread identifier will be Less than RTE_MAX_LCORE and will be still unique.
I think that your assumption will work for well-known operating systems but will be very unportable.
Regarding performance the DPDK can work efficiently in different environments including pthreads. You can imagine running DPDK from pthread application where affinity will be made by application. Effectiveness depends on application thread implementation comparable to EAL threads.
I think that this is a goal for this change.
Quoted 23 lines
Quoted 10 lines
Same comment for get operation below
lib/librte_mempool/rte_mempool.h __mempool_get_bulk(struct rte_mempool *mp,
Quoted 7 lines
lib/librte_mempool/rte_mempool.h +2
-0+if (unlikely (cache_size ==0 || is_mp ==0 || + lcore_id >=RTE_MAX_LCORE )) goto ring_enqueue;/* Go straight to ring if put would overflow mem allocated for cache
*/lib/librte_mempool/rte_mempool.h +0
-0@@ -952,7 +955,8 @@ __mempool_get_bulk(struct rte_mempool *mp,void
Quoted 12 lines
Quoted 12 lines
lib/librte_mempool/rte_mempool.h +2
-1**obj_table, uint32_t cache_size = mp->cache_size;/* cache is not enabled or single consumer */ -if (unlikely (cache_size ==0 || is_mc ==0 || n >= cache_size)) +if (unlikely (cache_size ==0 || is_mc ==0 || + n >= cache_size || lcore_id >=RTE_MAX_LCORE )) goto ring_dequeue;cache = &mp->local_cache[lcore_id];
--
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 7C4248CAE043B144B1CD242D275626532FE3C144-kPTMFJFq+rGvNW/[email protected]
Re: [PATCH v1 13/15] mempool: add support to non-EAL thread
Hi Miroslaw,
Quoted 101 lines
-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Walukiewicz, Miroslaw
Sent: Thursday, January 22, 2015 12:45 PM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v1 13/15] mempool: add support to non-EAL threadQuoted 84 lines
-----Original Message-----
From: Liang, Cunming
Sent: Thursday, January 22, 2015 1:20 PM
To: Walukiewicz, Miroslaw; [email protected]
Subject: RE: [dpdk-dev] [PATCH v1 13/15] mempool: add support to non-EAL threadQuoted 5 lines
-----Original Message-----
From: Walukiewicz, Miroslaw
Sent: Thursday, January 22, 2015 5:53 PM
To: Liang, Cunming; [email protected]
Subject: RE: [dpdk-dev] [PATCH v1 13/15] mempool: add support to non-EAL
Quoted 23 lines
thread
Quoted 19 lines
-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Cunming Liang
Sent: Thursday, January 22, 2015 9:17 AM
To: [email protected]
Subject: [dpdk-dev] [PATCH v1 13/15] mempool: add support to non-EAL threadFor non-EAL thread, bypass per lcore cache, directly use ring pool. It allows using
rte_mempoolin either EAL thread or any user pthread. As in non-EAL thread, it directly rely onrte_ringand it's none preemptive. It doesn't suggest to run multi-pthread/cpu which compete therte_mempool.
It will get bad performance and has critical risk if scheduling policy is RT.Signed-off-by Cunming Liang <[email protected]>--- lib/librte_mempool/rte_mempool.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-)Quoted 10 lines
lib/librte_mempool/rte_mempool.h struct rte_mempool { · line 201
Quoted 10 lines
lib/librte_mempool/rte_mempool.h +0
-1@@ -198,10 +198,12 @@ struct rte_mempool {* Number to add to the object-oriented statistics. */ #ifdef RTE_LIBRTE_MEMPOOL_DEBUG -#define __MEMPOOL_STAT_ADD (mp, name, n) do {\Quoted 23 lines
Quoted 13 lines
lib/librte_mempool/rte_mempool.h +6
-3-unsigned __lcore_id =rte_lcore_id (); \ - mp->stats[__lcore_id].name##_objs += n; \ - mp->stats[__lcore_id].name##_bulk += 1; \ +#define __MEMPOOL_STAT_ADD (mp, name, n) do { \ + unsigned __lcore_id = rte_lcore_id(); \ + if (__lcore_id < RTE_MAX_LCORE) { \ + mp->stats[__lcore_id].name##_objs += n; \ + mp->stats[__lcore_id].name##_bulk += 1; \ + } \ } while(0)#else #define __MEMPOOL_STAT_ADD (mp, name, n) do {} while(0)lib/librte_mempool/rte_mempool.h __mempool_put_bulk(struct rte_mempool *mp, · line 773
Quoted 7 lines
lib/librte_mempool/rte_mempool.h +1
-2@@ -767,8 +769,9 @@ __mempool_put_bulk(struct rte_mempool *mp,void *const *obj_table,__MEMPOOL_STAT_ADD (mp ,put ,n );#if RTE_MEMPOOL_CACHE_MAX_SIZE >0 -/* cache is not enabled or single producer */ -if (unlikely (cache_size ==0 || is_mp ==0 )) +/* cache is not enabled or single producer or none EAL thread */ I don't understand this limitation.
Quoted 7 lines
I see that the
rte_membuf.hdefines table perRTE_MAX_LCORElike below#if RTE_MEMPOOL_CACHE_MAX_SIZE > 0 /** Per-lcore local cache. */ struct rte_mempool_cache local_cache[RTE_MAX_LCORE];#endif
But why we cannot extent the size of the local cache table to something
like
Quoted 4 lines
RTE_MAX_THREADSthat does not exceed max value ofrte_lcore_id()Keeping this condition here is a real performance killer!!!!!!. I saw in my test application spending more 95% of CPU time reading the
Quoted 8 lines
atomic
Quoted 1 line
in M C/MP ring utilizing access to mempool.
[Liang, Cunming] This is the first step to make it work.
By Konstantin's comments, shall prevent to allocate unique id by ourselves. And the return value fromgettid()is too large as an index. For non-EAL thread performance gap, will think about additional fix patch here.
If care about performance, still prefer to choose EAL thread now.
Quoted 2 lines
In previous patch you had allocation of the thread id on base of unique
gettid()as number not a potential pointer as we can expect from implementationgetid()from Linux or BSD.
I am really puzzled with your sentence above.
What ' potential pointer' you are talking about?rte_lcore_id() - returns unsigned 32bit integer (as it always did)._lcore_id for each EAL thread is assigned at rte_eal_init().
For the EAL thread _lcore_id value is in interval [0, RTE_MAX_LCORE) and
it is up to the user to make sure that each _lcore_id is unique inside DPDK MultiProcess group. That's all as it was before.
What's new with Steve's patch:
1) At startup user can select a set of physical cpu(s) on which each EAL thread (lcore) can run.
Quoted 1 line
From explanation at [PATCH v1 02/15]:
--lcores='1,2@(5-7),(3-5)@(0,2),(0,6)' means starting 7 EAL thread as below lcore 0 runs on cpuset 0x41 (cpu 0,6)
lcore 1 runs on cpuset 0x2 (cpu 1)
lcore 2 runs on cpuset 0xe0 (cpu 5,6,7)
lcore 3,4,5 runs on cpuset 0x5 (cpu 0,2)
lcore 6 runs on cpuset 0x41 (cpu 0,6)
2) Non-EAL (dynamically created threads) will have their _lcore_id set to LCORE_ID_ANY (==UINT32_MAX). That allow us to distinguish them from EAL threads (lcores).
Make changes in DPDK functions, so that they will safely work within such threads too, but there could be some performance (rte_mempool get/put) degradation comaring to EAL thread.
Quoted 2 lines
The another problem is that we compare here int with some unique thread identifier.
We are not doing that here._lcore_id is unsigned int.
Quoted 2 lines
How can you prevent that when implementation of gettid will change and unique thread identifier will be Less than
RTE_MAX_LCOREand will be still unique.
We are not using gettid() to assign _lcore_id values. And never did.
Please read the patch set carefully.
Quoted 6 lines
I think that your assumption will work for well-known operating systems but will be very unportable.
Regarding performance the DPDK can work efficiently in different environments including pthreads. You can imagine running DPDK from pthread application where affinity will be made by application. Effectiveness depends on application thread implementation comparable to EAL threads.
That was one of the goals of the patch: make all DPDK API to work with dynamically created threads. So now it is safe to do:
static void *thread_func1(void *arg)
{
rte_pktmbuf_free((struct rte_mbuf *)arg); return NULL;
}
pthread_t tid;
struct rte_mbuf *m = rte_pktmbuf_alloc(mp);
pthread_create(&tid, NULL, thread_func1, m);
As Steve pointed - if you are dependent on rte_mempool get/put performance inside your threads - better to use EAL threads.
With current patch you can have up to RTE_MAX_LCORE such threads and can assign any cpu affinity for each of them. Though, if you have an idea how to extend mempool cache to any number of dynamically created threads (make cache for each mempool sort of TLS?), then sure we can discuss it.
But I suppose it should be a metter of separate patch.
Konstantin
Quoted 26 lines
I think that this is a goal for this change.
Quoted 23 lines
Quoted 10 lines
Same comment for get operation below
lib/librte_mempool/rte_mempool.h __mempool_get_bulk(struct rte_mempool *mp,
Quoted 7 lines
lib/librte_mempool/rte_mempool.h +2
-0+if (unlikely (cache_size ==0 || is_mp ==0 || + lcore_id >=RTE_MAX_LCORE )) goto ring_enqueue;/* Go straight to ring if put would overflow mem allocated for cache
*/lib/librte_mempool/rte_mempool.h +0
-0@@ -952,7 +955,8 @@ __mempool_get_bulk(struct rte_mempool *mp,void
Quoted 12 lines
Quoted 12 lines
lib/librte_mempool/rte_mempool.h +2
-1**obj_table, uint32_t cache_size = mp->cache_size;/* cache is not enabled or single consumer */ -if (unlikely (cache_size ==0 || is_mc ==0 || n >= cache_size)) +if (unlikely (cache_size ==0 || is_mc ==0 || + n >= cache_size || lcore_id >=RTE_MAX_LCORE )) goto ring_dequeue;cache = &mp->local_cache[lcore_id];
--
1.8.1.4
PATCH 14/15 ring: add support to non-EAL thread
In reply to a message not in the archivering debug stat won't take care non-EAL thread.
--- lib/librte_ring/rte_ring.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-)
lib/librte_ring/rte_ring.h +6 -4
@@ -188,10 +188,12 @@ struct rte_ring {* The number to add to the object-oriented statistics. */ #ifdef RTE_LIBRTE_RING_DEBUG -#define __RING_STAT_ADD (r, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - r->stats[__lcore_id].name##_objs += n; \ - r->stats[__lcore_id].name##_bulk += 1; \ +#define __RING_STAT_ADD (r, name, n) do { \ + unsigned __lcore_id = rte_lcore_id(); \ + if (__lcore_id < RTE_MAX_LCORE) { \ + r->stats[__lcore_id].name##_objs += n; \ + r->stats[__lcore_id].name##_bulk += 1; \ + } \ } while(0)#else #define __RING_STAT_ADD (r, name, n) do {} while(0)
Signature · 1 line
1.8.1.4
PATCH 15/15 timer: add support to non-EAL thread
In reply to a message not in the archiveAllow to setup timers only for EAL (lcore) threads (__lcore_id < MAX_LCORE_ID). E.g. – dynamically created thread will be able to reset/stop timer for lcore thread, but it will be not allowed to setup timer for itself or another non-lcore thread. rte_timer_manage() for non-lcore thread would simply do nothing and return straightway.
--- lib/librte_timer/rte_timer.c | 40 +++++++++++++++++++++++++++++++--------- lib/librte_timer/rte_timer.h | 2 +- 2 files changed, 32 insertions(+), 10 deletions(-)
lib/librte_timer/rte_timer.c +31 -9
@@ -79,9 +79,10 @@ static struct priv_timer priv_timer[RTE_MAX_LCORE];/* when debug is enabled, store some statistics */ #ifdef RTE_LIBRTE_TIMER_DEBUG -#define __TIMER_STAT_ADD (name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - priv_timer[__lcore_id].stats.name += (n); \ +#define __TIMER_STAT_ADD (name, n) do { \ + unsigned __lcore_id = rte_lcore_id(); \ + if (__lcore_id < RTE_MAX_LCORE) \ + priv_timer[__lcore_id].stats.name += (n); \ } while(0)#else #define __TIMER_STAT_ADD (name, n) do {} while(0)
@@ -127,15 +128,26 @@ timer_set_config_state(struct rte_timer *tim,unsigned lcore_id; lcore_id =rte_lcore_id (); +if (lcore_id >=RTE_MAX_LCORE ) + lcore_id =LCORE_ID_ANY ;/* wait that the timer is in correct status before update, * and mark it as being configured */ while (success ==0 ) { prev_status.u32 = tim->status.u32; +/* +* prevent race condition of non-EAL threads +* to update the timer. When 'owner == LCORE_ID_ANY', +* it means updated by a non-EAL thread. +*/ +if (lcore_id == (unsigned )LCORE_ID_ANY && + (uint16_t )lcore_id == prev_status.owner) +return -1 ; +/* timer is running on another core, exit */ if (prev_status.state ==RTE_TIMER_RUNNING && - (unsigned )prev_status.owner != lcore_id) + prev_status.owner != (uint16_t )lcore_id)return -1 ;/* timer is being configured on another core */
@@ -366,9 +378,13 @@ __rte_timer_reset(struct rte_timer *tim, uint64_t expire,/* round robin for tim_lcore */ if (tim_lcore == (unsigned )LCORE_ID_ANY ) { - tim_lcore =rte_get_next_lcore (priv_timer[lcore_id].prev_lcore, -0 ,1 ); - priv_timer[lcore_id].prev_lcore = tim_lcore; +if (lcore_id <RTE_MAX_LCORE ) { + tim_lcore =rte_get_next_lcore ( + priv_timer[lcore_id].prev_lcore, +0 ,1 ); + priv_timer[lcore_id].prev_lcore = tim_lcore; + }else + tim_lcore =rte_get_next_lcore (LCORE_ID_ANY ,0 ,1 ); }/* wait that the timer is in correct status before update,
@@ -378,7 +394,8 @@ __rte_timer_reset(struct rte_timer *tim, uint64_t expire,return -1 ;__TIMER_STAT_ADD (reset,1 ); -if (prev_status.state ==RTE_TIMER_RUNNING ) { +if (prev_status.state ==RTE_TIMER_RUNNING && + lcore_id <RTE_MAX_LCORE ) { priv_timer[lcore_id].updated =1 ; }
@@ -455,7 +472,8 @@ rte_timer_stop(struct rte_timer *tim)return -1 ;__TIMER_STAT_ADD (stop,1 ); -if (prev_status.state ==RTE_TIMER_RUNNING ) { +if (prev_status.state ==RTE_TIMER_RUNNING && + lcore_id <RTE_MAX_LCORE ) { priv_timer[lcore_id].updated =1 ; }
@@ -499,6 +517,10 @@ void rte_timer_manage(void)uint64_t cur_time;int i, ret; +/* timer manager only runs on EAL thread */ +if (lcore_id >=RTE_MAX_LCORE ) +return ; +__TIMER_STAT_ADD (manage,1 );/* optimize for the case where per-cpu list is empty */ if (priv_timer[lcore_id].pending_head.sl_next[0 ] ==NULL )
lib/librte_timer/rte_timer.h +1 -1
@@ -76,7 +76,7 @@ extern "C" {#define RTE_TIMER_RUNNING 2/**< State: timer function is running. */ #define RTE_TIMER_CONFIG 3/**< State: timer is being configured. */ -#define RTE_TIMER_NO_OWNER -1/**< Timer has no owner. */ +#define RTE_TIMER_NO_OWNER -2/**< Timer has no owner. */ /** * Timer type: Periodic or single (one-shot).
Signature · 1 line
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 1421914598-2747-16-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
Re: [PATCH v1 15/15] timer: add support to non-EAL thread
Quoted 29 lines
-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Cunming Liang
Sent: Thursday, January 22, 2015 9:17 AM
To: [email protected]
Subject: [dpdk-dev] [PATCH v1 15/15] timer: add support to non-EAL threadAllow to setup timers only for EAL (lcore) threads (
__lcore_id<MAX_LCORE_ID).
E.g. – dynamically created thread will be able to reset/stop timer for lcore thread,
but it will be not allowed to setup timer for itself or another non-lcore thread.rte_timer_manage()for non-lcore thread would simply do nothing and return straightway.Signed-off-by Cunming Liang <[email protected]>--- lib/librte_timer/rte_timer.c | 40 +++++++++++++++++++++++++++++++---- ----- lib/librte_timer/rte_timer.h | 2 +- 2 files changed, 32 insertions(+), 10 deletions(-)lib/librte_timer/rte_timer.c +0
-0
lib/librte_timer/rte_timer.c static struct priv_timer priv_timer[RTE_MAX_LCORE]; · line 79
Quoted 1 line
lib/librte_timer/rte_timer.c +0
-0@@ -79,9 +79,10 @@ static struct priv_timer priv_timer[RTE_MAX_LCORE];
Why not extend the priv_timer size to value being in range returned by rte_lcore_id().
All timer stuff will work automatically after such change without any change in timer logic including stats.
Quoted 109 lines
lib/librte_timer/rte_timer.c +4
-3/* when debug is enabled, store some statistics */ #ifdef RTE_LIBRTE_TIMER_DEBUG -#define __TIMER_STAT_ADD (name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - priv_timer[__lcore_id].stats.name += (n); \ +#define __TIMER_STAT_ADD (name, n) do { \ +unsigned __lcore_id =rte_lcore_id (); \ +if (__lcore_id <RTE_MAX_LCORE ) \ + priv_timer[__lcore_id].stats.name += (n); \ } while(0)#else
#define__TIMER_STAT_ADD(name, n)do {} while(0)lib/librte_timer/rte_timer.c +19
-4@@ -127,15 +128,26 @@ timer_set_config_state(struct rte_timer *tim,unsigned lcore_id; lcore_id =rte_lcore_id (); +if (lcore_id >=RTE_MAX_LCORE ) + lcore_id =LCORE_ID_ANY ;/* wait that the timer is in correct status before update, * and mark it as being configured */ while (success ==0 ) { prev_status.u32 = tim->status.u32; +/* +* prevent race condition of non-EAL threads +* to update the timer. When 'owner == LCORE_ID_ANY', +* it means updated by a non-EAL thread. +*/ +if (lcore_id == (unsigned )LCORE_ID_ANY && + (uint16_t )lcore_id == prev_status.owner) +return -1 ; +/* timer is running on another core, exit */ if (prev_status.state ==RTE_TIMER_RUNNING && - (unsigned )prev_status.owner != lcore_id) + prev_status.owner != (uint16_t )lcore_id)return -1 ;/* timer is being configured on another core */ @@ -366,9 +378,13 @@ __rte_timer_reset(struct rte_timer *tim, uint64_t expire,/* round robin for tim_lcore */ if (tim_lcore == (unsigned)LCORE_ID_ANY) { - tim_lcore =rte_get_next_lcore (priv_timer[lcore_id].prev_lcore, -0 ,1 ); - priv_timer[lcore_id].prev_lcore = tim_lcore; + if (lcore_id < RTE_MAX_LCORE) { + tim_lcore =rte_get_next_lcore ( + priv_timer[lcore_id].prev_lcore, +0 ,1 ); + priv_timer[lcore_id].prev_lcore = tim_lcore; + } else + tim_lcore = rte_get_next_lcore(LCORE_ID_ANY, 0,1 );lib/librte_timer/rte_timer.c +8
-2}/* wait that the timer is in correct status before update, @@ -378,7 +394,8 @@ __rte_timer_reset(struct rte_timer *tim, uint64_t expire, return -1 ;__TIMER_STAT_ADD (reset,1 ); -if (prev_status.state ==RTE_TIMER_RUNNING ) { +if (prev_status.state ==RTE_TIMER_RUNNING && + lcore_id <RTE_MAX_LCORE ) { priv_timer[lcore_id].updated =1 ; }@@ -455,7 +472,8 @@ rte_timer_stop(struct rte_timer *tim)return -1 ;__TIMER_STAT_ADD (stop,1 ); -if (prev_status.state ==RTE_TIMER_RUNNING ) { +if (prev_status.state ==RTE_TIMER_RUNNING && + lcore_id <RTE_MAX_LCORE ) { priv_timer[lcore_id].updated =1 ; }@@ -499,6 +517,10 @@ void rte_timer_manage(void)uint64_t cur_time;int i, ret; +/* timer manager only runs on EAL thread */ +if (lcore_id >=RTE_MAX_LCORE ) +return ; +__TIMER_STAT_ADD (manage,1 );/* optimize for the case where per-cpu list is empty */ if (priv_timer[lcore_id].pending_head.sl_next[0 ] ==NULL )lib/librte_timer/rte_timer.h +1
-2@@ -76,7 +76,7 @@ extern "C" {#define RTE_TIMER_RUNNING 2/**< State: timer function is running. */ #define RTE_TIMER_CONFIG 3/**< State: timer is being configured. */ -#define RTE_TIMER_NO_OWNER -1/**< Timer has no owner. */ +#define RTE_TIMER_NO_OWNER -2/**< Timer has no owner. */ /** * Timer type: Periodic or single (one-shot). -- 1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 7C4248CAE043B144B1CD242D275626532FE3BF0A-kPTMFJFq+rGvNW/[email protected]
Re: [PATCH v1 15/15] timer: add support to non-EAL thread
Quoted 42 lines
-----Original Message-----
From: Walukiewicz, Miroslaw
Sent: Thursday, January 22, 2015 5:58 PM
To: Liang, Cunming; [email protected]
Subject: RE: [dpdk-dev] [PATCH v1 15/15] timer: add support to non-EAL threadQuoted 27 lines
-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Cunming Liang
Sent: Thursday, January 22, 2015 9:17 AM
To: [email protected]
Subject: [dpdk-dev] [PATCH v1 15/15] timer: add support to non-EAL threadAllow to setup timers only for EAL (lcore) threads (
__lcore_id<MAX_LCORE_ID).
E.g. – dynamically created thread will be able to reset/stop timer for lcore thread,
but it will be not allowed to setup timer for itself or another non-lcore thread.rte_timer_manage()for non-lcore thread would simply do nothing and return straightway.Signed-off-by Cunming Liang <[email protected]>--- lib/librte_timer/rte_timer.c | 40 +++++++++++++++++++++++++++++++---- ----- lib/librte_timer/rte_timer.h | 2 +- 2 files changed, 32 insertions(+), 10 deletions(-)lib/librte_timer/rte_timer.c +0
-0lib/librte_timer/rte_timer.c static struct priv_timer priv_timer[RTE_MAX_LCORE]; · line 79
Quoted 1 line
lib/librte_timer/rte_timer.c +0
-0@@ -79,9 +79,10 @@ static struct priv_timer priv_timer[RTE_MAX_LCORE];Why not extend the
priv_timersize to value being in range returned byrte_lcore_id().
Quoted 2 lines
All timer stuff will work automatically after such change without any change in timer logic including stats.
[Liang, Cunming] The same reason as mempool does.
It won't expect to involve dynamic unique id allocation for user thread on the first step. The failure secondary won't release the reserved id which cause potential unexpected leak. So will look for other approach to improve the libraries in the next step.
Quoted 110 lines
Quoted 110 lines
lib/librte_timer/rte_timer.c +4
-3/* when debug is enabled, store some statistics */ #ifdef RTE_LIBRTE_TIMER_DEBUG -#define __TIMER_STAT_ADD (name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - priv_timer[__lcore_id].stats.name += (n); \ +#define __TIMER_STAT_ADD (name, n) do { \ +unsigned __lcore_id =rte_lcore_id (); \ +if (__lcore_id <RTE_MAX_LCORE ) \ + priv_timer[__lcore_id].stats.name += (n); \ } while(0)#else
#define__TIMER_STAT_ADD(name, n)do {} while(0)lib/librte_timer/rte_timer.c +19
-4@@ -127,15 +128,26 @@ timer_set_config_state(struct rte_timer *tim,unsigned lcore_id; lcore_id =rte_lcore_id (); +if (lcore_id >=RTE_MAX_LCORE ) + lcore_id =LCORE_ID_ANY ;/* wait that the timer is in correct status before update, * and mark it as being configured */ while (success ==0 ) { prev_status.u32 = tim->status.u32; +/* +* prevent race condition of non-EAL threads +* to update the timer. When 'owner == LCORE_ID_ANY', +* it means updated by a non-EAL thread. +*/ +if (lcore_id == (unsigned )LCORE_ID_ANY && + (uint16_t )lcore_id == prev_status.owner) +return -1 ; +/* timer is running on another core, exit */ if (prev_status.state ==RTE_TIMER_RUNNING && - (unsigned )prev_status.owner != lcore_id) + prev_status.owner != (uint16_t )lcore_id)return -1 ;/* timer is being configured on another core */ @@ -366,9 +378,13 @@ __rte_timer_reset(struct rte_timer *tim, uint64_t expire,/* round robin for tim_lcore */ if (tim_lcore == (unsigned)LCORE_ID_ANY) { - tim_lcore =rte_get_next_lcore (priv_timer[lcore_id].prev_lcore, -0 ,1 ); - priv_timer[lcore_id].prev_lcore = tim_lcore; + if (lcore_id < RTE_MAX_LCORE) { + tim_lcore =rte_get_next_lcore ( + priv_timer[lcore_id].prev_lcore, +0 ,1 ); + priv_timer[lcore_id].prev_lcore = tim_lcore; + } else + tim_lcore = rte_get_next_lcore(LCORE_ID_ANY, 0,1 );lib/librte_timer/rte_timer.c +8
-2}/* wait that the timer is in correct status before update, @@ -378,7 +394,8 @@ __rte_timer_reset(struct rte_timer *tim, uint64_t expire, return -1 ;__TIMER_STAT_ADD (reset,1 ); -if (prev_status.state ==RTE_TIMER_RUNNING ) { +if (prev_status.state ==RTE_TIMER_RUNNING && + lcore_id <RTE_MAX_LCORE ) { priv_timer[lcore_id].updated =1 ; }@@ -455,7 +472,8 @@ rte_timer_stop(struct rte_timer *tim)return -1 ;__TIMER_STAT_ADD (stop,1 ); -if (prev_status.state ==RTE_TIMER_RUNNING ) { +if (prev_status.state ==RTE_TIMER_RUNNING && + lcore_id <RTE_MAX_LCORE ) { priv_timer[lcore_id].updated =1 ; }@@ -499,6 +517,10 @@ void rte_timer_manage(void)uint64_t cur_time;int i, ret; +/* timer manager only runs on EAL thread */ +if (lcore_id >=RTE_MAX_LCORE ) +return ; +__TIMER_STAT_ADD (manage,1 );/* optimize for the case where per-cpu list is empty */ if (priv_timer[lcore_id].pending_head.sl_next[0 ] ==NULL )lib/librte_timer/rte_timer.h +1
-2@@ -76,7 +76,7 @@ extern "C" {#define RTE_TIMER_RUNNING 2/**< State: timer function is running. */ #define RTE_TIMER_CONFIG 3/**< State: timer is being configured. */ -#define RTE_TIMER_NO_OWNER -1/**< Timer has no owner. */ +#define RTE_TIMER_NO_OWNER -2/**< Timer has no owner. */ /** * Timer type: Periodic or single (one-shot). -- 1.8.1.4
Ananyev, Konstantin
Acked-by Konstantin AnanyevQuoted 69 lines
-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Cunming Liang
Sent: Thursday, January 22, 2015 8:16 AM
To: [email protected]
Subject: [dpdk-dev] [PATCH v1 00/15] support multi-pthread per coreThe patch series contain the enhancements of EAL and fixes for libraries
to run multi-pthreads(either EAL or non-EAL thread) per physical core.
Two major changes list as below:
- Extend the core affinity of each EAL thread to 1:n.
Each lcore stands for a EAL thread rather than a logical core.
The change adds new EAL option to allow static lcore to cpuset assginment. Then a lcore(EAL thread) affinity to a cpuset, original 1:1 mapping is the special case.
- Fix the libraries to allow running on any non-EAL thread.
It fix the gaps running libraries in non-EAL thread(dynamic created by user). Each fix libraries take care the case ofrte_lcore_id()>=RTE_MAX_LCORE.Thanks a million for the comments from Konstantin, Bruce, Mirek and Stephen in RFC review.
*** BLURB HERE ***
Cunming Liang (15):
eal: add cpuset into per EAL threadlcore_config
eal: new eal option '--lcores' for cpu assignment
eal: add support parsingsocket_idfrom cpuset
eal: new TLS definition and API declaration
eal: addeal_common_thread.cfor common thread API
eal: addrte_gettid()to acquire unique system tid
eal: apply affinity of EAL thread by assigned cpuset
enic: fix re-define freebsd compile complain
malloc: fix the issue ofSOCKET_ID_ANY
log: fix the gap to support non-EAL thread
eal: set_lcore_idand_socket_idto (-1) by default
eal: fix recursive spinlock in non-EAL thraed
mempool: add support to non-EAL thread
ring: add support to non-EAL thread
timer: add support to non-EAL threadlib/librte_eal/bsdapp/eal/Makefile | 1 + lib/librte_eal/bsdapp/eal/eal.c | 13 +- lib/librte_eal/bsdapp/eal/eal_lcore.c | 14 ++ lib/librte_eal/bsdapp/eal/eal_memory.c | 2 + lib/librte_eal/bsdapp/eal/eal_thread.c | 76 +++--- lib/librte_eal/common/eal_common_launch.c | 1 - lib/librte_eal/common/eal_common_log.c | 17 +- lib/librte_eal/common/eal_common_options.c | 262 ++++++++++++++++++++- lib/librte_eal/common/eal_common_thread.c | 142 +++++++++++ lib/librte_eal/common/eal_options.h | 2 + lib/librte_eal/common/eal_thread.h | 66 ++++++ .../common/include/generic/rte_spinlock.h | 4 +- lib/librte_eal/common/include/rte_eal.h | 27 +++ lib/librte_eal/common/include/rte_lcore.h | 37 ++- lib/librte_eal/common/include/rte_log.h | 5 + lib/librte_eal/linuxapp/eal/Makefile | 4 + lib/librte_eal/linuxapp/eal/eal.c | 7 +- lib/librte_eal/linuxapp/eal/eal_lcore.c | 15 ++ lib/librte_eal/linuxapp/eal/eal_thread.c | 78 +++--- lib/librte_malloc/malloc_heap.h | 7 +- lib/librte_mempool/rte_mempool.h | 18 +- lib/librte_pmd_enic/enic.h | 1 + lib/librte_pmd_enic/enic_compat.h | 1 + lib/librte_ring/rte_ring.h | 10 +- lib/librte_timer/rte_timer.c | 40 +++- lib/librte_timer/rte_timer.h | 2 +- 26 files changed, 721 insertions(+), 131 deletions(-) create mode 100644 lib/librte_eal/common/eal_common_thread.c
Quoted 1 line
--
Quoted 1 line
1.8.1.4
v2 cover support multi-pthread per core
In reply to a message not in the archivev2 changes:
add '<number>-<number>' support for EAL option '--lcores'
The patch series contain the enhancements of EAL and fixes for libraries
to run multi-pthreads(either EAL or non-EAL thread) per physical core.
Two major changes list as below:
- Extend the core affinity of each EAL thread to 1:n.
Each lcore stands for a EAL thread rather than a logical core.
The change adds new EAL option to allow static lcore to cpuset assginment. Then a lcore(EAL thread) affinity to a cpuset, original 1:1 mapping is the special case.
- Fix the libraries to allow running on any non-EAL thread.
It fix the gaps running libraries in non-EAL thread(dynamic created by user). Each fix libraries take care the case of rte_lcore_id() >= RTE_MAX_LCORE.
Thanks a million for the comments from Konstantin, Bruce, Mirek and Stephen in RFC review.
*** BLURB HERE ***
Cunming Liang (15):
eal: add cpuset into per EAL thread lcore_config
eal: new eal option '--lcores' for cpu assignment
eal: add support parsing socket_id from cpuset
eal: new TLS definition and API declaration
eal: add eal_common_thread.c for common thread API
eal: add rte_gettid() to acquire unique system tid
eal: apply affinity of EAL thread by assigned cpuset
enic: fix re-define freebsd compile complain
malloc: fix the issue of SOCKET_ID_ANY
log: fix the gap to support non-EAL thread
eal: set _lcore_id and _socket_id to (-1) by default
eal: fix recursive spinlock in non-EAL thraed
mempool: add support to non-EAL thread
ring: add support to non-EAL thread
timer: add support to non-EAL thread
lib/librte_eal/bsdapp/eal/Makefile | 1 + lib/librte_eal/bsdapp/eal/eal.c | 13 +- lib/librte_eal/bsdapp/eal/eal_lcore.c | 14 + lib/librte_eal/bsdapp/eal/eal_memory.c | 2 + lib/librte_eal/bsdapp/eal/eal_thread.c | 76 +++--- lib/librte_eal/common/eal_common_launch.c | 1 - lib/librte_eal/common/eal_common_log.c | 17 +- lib/librte_eal/common/eal_common_options.c | 300 ++++++++++++++++++++- lib/librte_eal/common/eal_common_thread.c | 142 ++++++++++ lib/librte_eal/common/eal_options.h | 2 + lib/librte_eal/common/eal_thread.h | 66 +++++ .../common/include/generic/rte_spinlock.h | 4 +- lib/librte_eal/common/include/rte_eal.h | 27 ++ lib/librte_eal/common/include/rte_lcore.h | 37 ++- lib/librte_eal/common/include/rte_log.h | 5 + lib/librte_eal/linuxapp/eal/Makefile | 4 + lib/librte_eal/linuxapp/eal/eal.c | 7 +- lib/librte_eal/linuxapp/eal/eal_lcore.c | 15 ++ lib/librte_eal/linuxapp/eal/eal_thread.c | 78 +++--- lib/librte_malloc/malloc_heap.h | 7 +- lib/librte_mempool/rte_mempool.h | 18 +- lib/librte_pmd_enic/enic.h | 1 + lib/librte_pmd_enic/enic_compat.h | 1 + lib/librte_ring/rte_ring.h | 10 +- lib/librte_timer/rte_timer.c | 40 ++- lib/librte_timer/rte_timer.h | 2 +- 26 files changed, 759 insertions(+), 131 deletions(-) create mode 100644 lib/librte_eal/common/eal_common_thread.c
Signature · 1 line
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 1422428365-5875-1-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
PATCH v2 1/15 eal: add cpuset into per EAL thread lcore_config
The patch adds 'cpuset' into per-lcore configure 'lcore_config[]', as the lcore no longer always 1:1 pinning with physical cpu. The lcore now stands for a EAL thread rather than a logical cpu.
It doesn't change the default behavior of 1:1 mapping, but allows to affinity the EAL thread to multiple cpus.
--- lib/librte_eal/bsdapp/eal/eal_lcore.c | 7 +++++++ lib/librte_eal/bsdapp/eal/eal_memory.c | 2 ++ lib/librte_eal/common/include/rte_lcore.h | 8 ++++++++ lib/librte_eal/linuxapp/eal/Makefile | 1 + lib/librte_eal/linuxapp/eal/eal_lcore.c | 8 ++++++++ 5 files changed, 26 insertions(+)
lib/librte_eal/bsdapp/eal/eal_lcore.c +7 -0
@@ -76,11 +76,18 @@ rte_eal_cpu_init(void)* ones and enable them by default. */ for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +/* init cpuset for per lcore config */ +CPU_ZERO (&lcore_config[lcore_id].cpuset); + lcore_config[lcore_id].detected = (lcore_id < ncpus);if (lcore_config[lcore_id].detected ==0 ) { config->lcore_role[lcore_id] =ROLE_OFF ;continue ; } + +/* By default, lcore 1:1 map to cpu id */ +CPU_SET (lcore_id, &lcore_config[lcore_id].cpuset); +/* By default, each detected core is enabled */ config->lcore_role[lcore_id] =ROLE_RTE ; lcore_config[lcore_id].core_id =cpu_core_id (lcore_id);
lib/librte_eal/bsdapp/eal/eal_memory.c +2 -0
@@ -45,6 +45,8 @@#include "eal_internal_cfg.h" #include "eal_filesystem.h" +/* avoid re-defined against with freebsd header */ +#undef PAGE_SIZE#define PAGE_SIZE (sysconf(_SC_PAGESIZE))/*
lib/librte_eal/common/include/rte_lcore.h +8 -0
@@ -50,6 +50,13 @@ extern "C" {#define LCORE_ID_ANY -1/**< Any lcore. */ +#if defined(__linux__) +typedef cpu_set_t rte_cpuset_t ; +#elif defined(__FreeBSD__) +#include <pthread_np.h> +typedef cpuset_t rte_cpuset_t ; +#endif +/** * Structure storing internal configuration (per-lcore) */
@@ -65,6 +72,7 @@ struct lcore_config {unsigned socket_id;/**< physical socket id for this lcore */ unsigned core_id;/**< core number on socket for this lcore */ int core_index;/**< relative index, starting from 0 */ +rte_cpuset_t cpuset;/**< cpu set which the lcore affinity to */ };/**
lib/librte_eal/linuxapp/eal/Makefile +1 -0
@@ -87,6 +87,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_dev.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_options.c CFLAGS_eal.o := -D_GNU_SOURCE +CFLAGS_eal_lcore.o := -D_GNU_SOURCE CFLAGS_eal_thread.o := -D_GNU_SOURCE CFLAGS_eal_log.o := -D_GNU_SOURCE CFLAGS_eal_common_log.o := -D_GNU_SOURCE
lib/librte_eal/linuxapp/eal/eal_lcore.c +8 -0
@@ -158,11 +158,19 @@ rte_eal_cpu_init(void)* ones and enable them by default. */ for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +/* init cpuset for per lcore config */ +CPU_ZERO (&lcore_config[lcore_id].cpuset); + +/* in 1:1 mapping, record related cpu detected state */ lcore_config[lcore_id].detected =cpu_detected (lcore_id);if (lcore_config[lcore_id].detected ==0 ) { config->lcore_role[lcore_id] =ROLE_OFF ;continue ; } + +/* By default, lcore 1:1 map to cpu id */ +CPU_SET (lcore_id, &lcore_config[lcore_id].cpuset); +/* By default, each detected core is enabled */ config->lcore_role[lcore_id] =ROLE_RTE ; lcore_config[lcore_id].core_id =cpu_core_id (lcore_id);
Signature · 1 line
1.8.1.4
PATCH v2 2/15 eal: new eal option '--lcores' for cpu assignment
In reply to a message not in the archiveIt supports one new eal long option '--lcores' for EAL thread cpuset assignment.
The format pattern:
--lcores='lcores[@cpus]<,lcores[@cpus]>'
lcores, cpus could be a single digit/range or a group. '(' and ')' are necessary if it's a group.
If not supply '@cpus', the value of cpus uses the same as lcores.
e.g. '1,2@(5-7),(3-5)@(0,2),(0,6),7-8' means starting 9 EAL thread as below
lcore 0 runs on cpuset 0x41 (cpu 0,6)
lcore 1 runs on cpuset 0x2 (cpu 1)
lcore 2 runs on cpuset 0xe0 (cpu 5,6,7)
lcore 3,4,5 runs on cpuset 0x5 (cpu 0,2)
lcore 6 runs on cpuset 0x41 (cpu 0,6)
lcore 7 runs on cpuset 0x80 (cpu 7)
lcore 8 runs on cpuset 0x100 (cpu 8)
--- lib/librte_eal/common/eal_common_launch.c | 1 - lib/librte_eal/common/eal_common_options.c | 300 ++++++++++++++++++++++++++++- lib/librte_eal/common/eal_options.h | 2 + lib/librte_eal/linuxapp/eal/Makefile | 1 + 4 files changed, 299 insertions(+), 5 deletions(-)
lib/librte_eal/common/eal_common_launch.c +0 -1
@@ -117,4 +117,3 @@ rte_eal_mp_wait_lcore(void)rte_eal_wait_lcore (lcore_id); } } -
lib/librte_eal/common/eal_common_options.c +296 -4
342 lines not on this page · raw message
lib/librte_eal/common/eal_options.h +2 -0
@@ -77,6 +77,8 @@ enum {OPT_CREATE_UIO_DEV_NUM , #define OPT_VFIO_INTR "vfio-intr"OPT_VFIO_INTR_NUM , +#define OPT_LCORES "lcores" +OPT_LCORES_NUM , OPT_LONG_MAX_NUM };
lib/librte_eal/linuxapp/eal/Makefile +1 -0
@@ -95,6 +95,7 @@ CFLAGS_eal_hugepage_info.o := -D_GNU_SOURCE CFLAGS_eal_pci.o := -D_GNU_SOURCE CFLAGS_eal_pci_vfio.o := -D_GNU_SOURCE CFLAGS_eal_common_whitelist.o := -D_GNU_SOURCE +CFLAGS_eal_common_options.o := -D_GNU_SOURCE # workaround for a gcc bug with noreturn attribute # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603
Signature · 1 line
1.8.1.4
PATCH v2 3/15 eal: add support parsing socket_id from cpuset
In reply to a message not in the archiveIt returns the socket_id if all cpus in the cpuset belongs to the same NUMA node, otherwise it will return SOCKET_ID_ANY.
--- lib/librte_eal/bsdapp/eal/eal_lcore.c | 7 +++++ lib/librte_eal/common/eal_thread.h | 52 +++++++++++++++++++++++++++++++++ lib/librte_eal/linuxapp/eal/eal_lcore.c | 7 +++++ 3 files changed, 66 insertions(+)
lib/librte_eal/bsdapp/eal/eal_lcore.c +7 -0
@@ -41,6 +41,7 @@#include <rte_debug.h> #include "eal_private.h" +#include "eal_thread.h" /* No topology information available on FreeBSD including NUMA info */ #define cpu_core_id (X ) 0
@@ -112,3 +113,9 @@ rte_eal_cpu_init(void)return 0 ; } + +unsigned +eal_cpu_socket_id (__rte_unused unsigned cpu_id) +{ +return cpu_socket_id (cpu_id); +}
lib/librte_eal/common/eal_thread.h +52 -0
62 lines not on this page · raw message
lib/librte_eal/linuxapp/eal/eal_lcore.c +7 -0
@@ -45,6 +45,7 @@#include "eal_private.h" #include "eal_filesystem.h" +#include "eal_thread.h" #define SYS_CPU_DIR "/sys/devices/system/cpu/cpu%u"#define CORE_ID_FILE "topology/core_id"
@@ -197,3 +198,9 @@ rte_eal_cpu_init(void)return 0 ; } + +unsigned +eal_cpu_socket_id (unsigned cpu_id) +{ +return cpu_socket_id (cpu_id); +}
Signature · 1 line
1.8.1.4
PATCH v2 4/15 eal: new TLS definition and API declaration
In reply to a message not in the archive1. add two TLS *_socket_id* and *_cpuset*
2. add two external API rte_thread_set/get_affinity
3. add one internal API eal_thread_dump_affinity
--- lib/librte_eal/bsdapp/eal/eal_thread.c | 2 ++ lib/librte_eal/common/eal_thread.h | 14 ++++++++++++++ lib/librte_eal/common/include/rte_lcore.h | 29 +++++++++++++++++++++++++++-- lib/librte_eal/linuxapp/eal/eal_thread.c | 2 ++ 4 files changed, 45 insertions(+), 2 deletions(-)
lib/librte_eal/bsdapp/eal/eal_thread.c +2 -0
@@ -56,6 +56,8 @@#include "eal_thread.h" RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); +RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);/* * Send a message to a slave lcore identified by slave_id to call a
lib/librte_eal/common/eal_thread.h +14 -0
@@ -102,4 +102,18 @@ eal_cpuset_socket_id(rte_cpuset_t *cpusetp) return socket_id; } +/** +* Dump the current pthread cpuset. +* This function is private to EAL. +* +* @param str +* The string buffer the cpuset will dump to. +* @param size +* The string buffer size. +*/ +#define CPU_STR_LEN 256 +void +eal_thread_dump_affinity (char str[],unsigned size); + +#endif /* EAL_THREAD_H */
lib/librte_eal/common/include/rte_lcore.h +27 -2
@@ -43,6 +43,7 @@#include <rte_per_lcore.h> #include <rte_eal.h> #include <rte_launch.h> +#include <rte_memory.h> #ifdef __cplusplusextern "C" {
@@ -80,7 +81,9 @@ struct lcore_config {*/ extern struct lcore_config lcore_config[RTE_MAX_LCORE ]; -RTE_DECLARE_PER_LCORE (unsigned, _lcore_id);/**< Per core "core id". */ +RTE_DECLARE_PER_LCORE (unsigned, _lcore_id);/**< Per thread "lcore id". */ +RTE_DECLARE_PER_LCORE (unsigned, _socket_id);/**< Per thread "socket id". */ +RTE_DECLARE_PER_LCORE (rte_cpuset_t, _cpuset);/**< Per thread "cpuset". */ /** * Return the ID of the execution unit we are running on.
@@ -146,7 +149,7 @@ rte_lcore_index(int lcore_id)static inline unsigned rte_socket_id (void ) { -return lcore_config[rte_lcore_id ()].socket_id; +return RTE_PER_LCORE (_socket_id); }/**
@@ -229,6 +232,28 @@ rte_get_next_lcore(unsigned i, int skip_master, int wrap) i<RTE_MAX_LCORE ; \ i = rte_get_next_lcore(i, 1, 0)) +/** +* Set core affinity of the current thread. +* Support both EAL and none-EAL thread and update TLS. +* +* @param cpusetp +* Point to cpu_set_t for setting current thread affinity. +* @return +* On success, return 0; otherwise return -1; +*/ +int rte_thread_set_affinity (rte_cpuset_t *cpusetp); + +/** +* Get core affinity of the current thread. +* +* @param cpusetp +* Point to cpu_set_t for getting current thread cpu affinity. +* @return +* On success, return 0; otherwise return -1; +*/ +int rte_thread_get_affinity (rte_cpuset_t *cpusetp); + + #ifdef __cplusplus }#endif
lib/librte_eal/linuxapp/eal/eal_thread.c +2 -0
@@ -56,6 +56,8 @@#include "eal_thread.h" RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); +RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);/* * Send a message to a slave lcore identified by slave_id to call a
Signature · 1 line
1.8.1.4
PATCH v2 5/15 eal: add eal_common_thread.c for common thread API
In reply to a message not in the archiveThe API works for both EAL thread and none EAL thread. When calling rte_thread_set_affinity, the *_socket_id* and *_cpuset* of calling thread will be updated if the thread successful set the cpu affinity.
--- lib/librte_eal/bsdapp/eal/Makefile | 1 + lib/librte_eal/common/eal_common_thread.c | 142 ++++++++++++++++++++++++++++++ lib/librte_eal/linuxapp/eal/Makefile | 2 + 3 files changed, 145 insertions(+) create mode 100644 lib/librte_eal/common/eal_common_thread.c
lib/librte_eal/bsdapp/eal/Makefile +1 -0
@@ -73,6 +73,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_EAL_BSDAPP) += eal_common_hexdump.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_BSDAPP) += eal_common_devargs.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_BSDAPP) += eal_common_dev.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_BSDAPP) += eal_common_options.c +SRCS-$(CONFIG_RTE_LIBRTE_EAL_BSDAPP) += eal_common_thread.c CFLAGS_eal.o := -D_GNU_SOURCE #CFLAGS_eal_thread.o := -D_GNU_SOURCE
lib/librte_eal/common/eal_common_thread.c new file +142 -0
142 lines not on this page · raw message
lib/librte_eal/linuxapp/eal/Makefile +2 -0
@@ -85,6 +85,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_hexdump.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_devargs.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_dev.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_options.c +SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_thread.c CFLAGS_eal.o := -D_GNU_SOURCE CFLAGS_eal_lcore.o := -D_GNU_SOURCE
@@ -96,6 +97,7 @@ CFLAGS_eal_pci.o := -D_GNU_SOURCE CFLAGS_eal_pci_vfio.o := -D_GNU_SOURCE CFLAGS_eal_common_whitelist.o := -D_GNU_SOURCE CFLAGS_eal_common_options.o := -D_GNU_SOURCE +CFLAGS_eal_common_thread.o := -D_GNU_SOURCE # workaround for a gcc bug with noreturn attribute # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603
Signature · 1 line
1.8.1.4
PATCH v2 6/15 eal: add rte_gettid() to acquire unique system tid
In reply to a message not in the archiveThe rte_gettid() wraps the linux and freebsd syscall gettid(). It provides a persistent unique thread id for the calling thread. It will save the unique id in TLS on the first time.
--- lib/librte_eal/bsdapp/eal/eal_thread.c | 9 +++++++++ lib/librte_eal/common/include/rte_eal.h | 27 +++++++++++++++++++++++++++ lib/librte_eal/linuxapp/eal/eal_thread.c | 7 +++++++ 3 files changed, 43 insertions(+)
lib/librte_eal/bsdapp/eal/eal_thread.c +9 -0
@@ -39,6 +39,7 @@#include <sched.h> #include <pthread_np.h> #include <sys/queue.h> +#include <sys/thr.h> #include <rte_debug.h> #include <rte_atomic.h>
@@ -233,3 +234,11 @@ eal_thread_loop(__attribute__((unused)) void *arg)/* pthread_exit(NULL); */ /* return NULL; */ } + +/* require calling thread tid by gettid() */ +int rte_sys_gettid (void ) +{ +long lwpid; +thr_self (&lwpid); +return (int )lwpid; +}
lib/librte_eal/common/include/rte_eal.h +27 -0
@@ -41,6 +41,9 @@*/ #include <stdint.h> +#include <sched.h> + +#include <rte_per_lcore.h> #ifdef __cplusplusextern "C" {
@@ -262,6 +265,30 @@ rte_set_application_usage_hook( rte_usage_hook_t usage_func );*/ int rte_eal_has_hugepages (void ); +/** +* A wrap API for syscall gettid. +* +* @return +* On success, returns the thread ID of calling process. +* It always successful. +*/ +int rte_sys_gettid (void ); + +/** +* Get system unique thread id. +* +* @return +* On success, returns the thread ID of calling process. +* It always successful. +*/ +static inline int rte_gettid (void ) +{ + static RTE_DEFINE_PER_LCORE(int, _thread_id) = -1; +if (RTE_PER_LCORE (_thread_id) ==-1 ) +RTE_PER_LCORE (_thread_id) =rte_sys_gettid (); +return RTE_PER_LCORE (_thread_id); +} + #ifdef __cplusplus }#endif
lib/librte_eal/linuxapp/eal/eal_thread.c +7 -0
@@ -39,6 +39,7 @@#include <pthread.h> #include <sched.h> #include <sys/queue.h> +#include <sys/syscall.h> #include <rte_debug.h> #include <rte_atomic.h>
@@ -233,3 +234,9 @@ eal_thread_loop(__attribute__((unused)) void *arg)/* pthread_exit(NULL); */ /* return NULL; */ } + +/* require calling thread tid by gettid() */ +int rte_sys_gettid (void ) +{ +return (int )syscall (SYS_gettid); +}
Signature · 1 line
1.8.1.4
PATCH v2 7/15 eal: apply affinity of EAL thread by assigned cpuset
In reply to a message not in the archiveEAL threads use assigned cpuset to set core affinity during startup. It keeps 1:1 mapping, if no '--lcores' option is used.
--- lib/librte_eal/bsdapp/eal/eal.c | 13 ++++--- lib/librte_eal/bsdapp/eal/eal_thread.c | 63 +++++++++--------------------- lib/librte_eal/linuxapp/eal/eal.c | 7 +++- lib/librte_eal/linuxapp/eal/eal_thread.c | 67 +++++++++++--------------------- 4 files changed, 54 insertions(+), 96 deletions(-)
lib/librte_eal/bsdapp/eal/eal.c +8 -5
@@ -432,6 +432,7 @@ rte_eal_init(int argc, char **argv)int i, fctret, ret;pthread_t thread_id;static rte_atomic32_t run_once =RTE_ATOMIC32_INIT (0 ); +char cpuset[CPU_STR_LEN ];if (!rte_atomic32_test_and_set (&run_once))return -1 ;
@@ -502,13 +503,17 @@ rte_eal_init(int argc, char **argv)if (rte_eal_pci_init () <0 )rte_panic ("Cannot init PCI\n" ); -RTE_LOG (DEBUG ,EAL ,"Master core %u is ready (tid=%p)\n" , - rte_config.master_lcore, thread_id); -eal_check_mem_on_local_socket ();rte_eal_mcfg_complete (); +eal_thread_init_master (rte_config.master_lcore); + +eal_thread_dump_affinity (cpuset,CPU_STR_LEN ); + +RTE_LOG (DEBUG ,EAL ,"Master lcore %u is ready (tid=%p;cpuset=[%s])\n" , + rte_config.master_lcore, thread_id, cpuset); +if (rte_eal_dev_init () <0 )rte_panic ("Cannot init pmd devices\n" );
@@ -532,8 +537,6 @@ rte_eal_init(int argc, char **argv)rte_panic ("Cannot create thread\n" ); } -eal_thread_init_master (rte_config.master_lcore); -/* * Launch a dummy function on all slave lcores, so that master lcore * knows they are all ready when this function returns.
lib/librte_eal/bsdapp/eal/eal_thread.c +19 -44
95 lines not on this page · raw message
lib/librte_eal/linuxapp/eal/eal.c +5 -2
@@ -702,6 +702,7 @@ rte_eal_init(int argc, char **argv)static rte_atomic32_t run_once =RTE_ATOMIC32_INIT (0 );struct shared_driver *solib =NULL ;const char *logid; +char cpuset[CPU_STR_LEN ];if (!rte_atomic32_test_and_set (&run_once))return -1 ;
@@ -802,8 +803,10 @@ rte_eal_init(int argc, char **argv)eal_thread_init_master (rte_config.master_lcore); -RTE_LOG (DEBUG ,EAL ,"Master core %u is ready (tid=%x)\n" , - rte_config.master_lcore, (int )thread_id); +eal_thread_dump_affinity (cpuset,CPU_STR_LEN ); + +RTE_LOG (DEBUG ,EAL ,"Master lcore %u is ready (tid=%x;cpuset=[%s])\n" , + rte_config.master_lcore, (int )thread_id, cpuset);if (rte_eal_dev_init () <0 )rte_panic ("Cannot init pmd devices\n" );
lib/librte_eal/linuxapp/eal/eal_thread.c +22 -45
110 lines not on this page · raw message
Signature · 1 line
1.8.1.4
PATCH v2 8/15 enic: fix re-define freebsd compile complain
In reply to a message not in the archiveSome macro already been defined by freebsd 'sys/param.h'.
--- lib/librte_pmd_enic/enic.h | 1 + lib/librte_pmd_enic/enic_compat.h | 1 + 2 files changed, 2 insertions(+)
lib/librte_pmd_enic/enic.h +1 -0
@@ -66,6 +66,7 @@#define ENIC_CALC_IP_CKSUM 1#define ENIC_CALC_TCP_UDP_CKSUM 2#define ENIC_MAX_MTU 9000 +#undef PAGE_SIZE#define PAGE_SIZE 4096#define PAGE_ROUND_UP (x) \ ((((unsigned long)(x)) + PAGE_SIZE-1) & (~(PAGE_SIZE-1)))
lib/librte_pmd_enic/enic_compat.h +1 -0
@@ -67,6 +67,7 @@ #define pr_warn(y, args...) dev_warning(0, y, ##args)#define BUG () pr_err("BUG at %s:%d", __func__, __LINE__) +#undef ALIGN#define ALIGN (x, a) __ALIGN_MASK(x, (typeof(x))(a)-1)#define __ALIGN_MASK (x, mask) (((x)+(mask))&~(mask))#define udelay usleep
Signature · 1 line
1.8.1.4
PATCH v2 9/15 malloc: fix the issue of SOCKET_ID_ANY
In reply to a message not in the archiveAdd check for rte_socket_id(), avoid get unexpected return like (-1).
--- lib/librte_malloc/malloc_heap.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)
lib/librte_malloc/malloc_heap.h +6 -1
@@ -44,7 +44,12 @@ extern "C" {static inline unsigned malloc_get_numa_socket (void ) { -return rte_socket_id (); +unsigned socket_id =rte_socket_id (); + +if (socket_id == (unsigned )SOCKET_ID_ANY ) +return 0 ; + +return socket_id; } void *
Signature · 1 line
1.8.1.4
PATCH v2 10/15 log: fix the gap to support non-EAL thread
In reply to a message not in the archiveFor those non-EAL thread, *_lcore_id* is invalid and probably larger than RTE_MAX_LCORE. The patch adds the check and allows only EAL thread using EAL per thread log level and log type. Others shares the global log level.
--- lib/librte_eal/common/eal_common_log.c | 17 +++++++++++++++-- lib/librte_eal/common/include/rte_log.h | 5 +++++ 2 files changed, 20 insertions(+), 2 deletions(-)
lib/librte_eal/common/eal_common_log.c +15 -2
@@ -193,11 +193,20 @@ rte_set_log_type(uint32_t type, int enable) rte_logs.type &= (~type); } +/* Get global log type */ +uint32_t +rte_get_log_type (void ) +{ +return rte_logs.type; +} +/* get the current loglevel for the message beeing processed */ int rte_log_cur_msg_loglevel (void ) {unsigned lcore_id; lcore_id =rte_lcore_id (); +if (lcore_id >=RTE_MAX_LCORE ) +return rte_get_log_level ();return log_cur_msg[lcore_id].loglevel; }
@@ -206,6 +215,8 @@ int rte_log_cur_msg_logtype(void) {unsigned lcore_id; lcore_id =rte_lcore_id (); +if (lcore_id >=RTE_MAX_LCORE ) +return rte_get_log_type ();return log_cur_msg[lcore_id].logtype; }
@@ -265,8 +276,10 @@ rte_vlog(__attribute__((unused)) uint32_t level,/* save loglevel and logtype in a global per-lcore variable */ lcore_id =rte_lcore_id (); - log_cur_msg[lcore_id].loglevel = level; - log_cur_msg[lcore_id].logtype = logtype; +if (lcore_id <RTE_MAX_LCORE ) { + log_cur_msg[lcore_id].loglevel = level; + log_cur_msg[lcore_id].logtype = logtype; + } ret =vfprintf (f, format, ap);fflush (f);
lib/librte_eal/common/include/rte_log.h +5 -0
@@ -144,6 +144,11 @@ uint32_t rte_get_log_level(void);void rte_set_log_type (uint32_t type,int enable);/** +* Get the global log type. +*/ +uint32_t rte_get_log_type (void ); + +/** * Get the current loglevel for the message being processed. * * Before calling the user-defined stream for logging, the log
Signature · 1 line
1.8.1.4
PATCH v2 11/15 eal: set _lcore_id and _socket_id to (-1) by default
In reply to a message not in the archiveFor those none EAL thread, *_lcore_id* shall always be LCORE_ID_ANY. The libraries using *_lcore_id* as index need to take care. *_socket_id* always be SOCKET_ID_ANY unitl the thread changes the affinity by rte_thread_set_affinity()
--- lib/librte_eal/bsdapp/eal/eal_thread.c | 4 ++-- lib/librte_eal/linuxapp/eal/eal_thread.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-)
lib/librte_eal/bsdapp/eal/eal_thread.c +2 -2
@@ -56,8 +56,8 @@#include "eal_private.h" #include "eal_thread.h" -RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); -RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (unsigned, _lcore_id) = (unsigned )LCORE_ID_ANY ; +RTE_DEFINE_PER_LCORE (unsigned, _socket_id) = (unsigned )SOCKET_ID_ANY ;RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);/*
lib/librte_eal/linuxapp/eal/eal_thread.c +2 -2
@@ -57,8 +57,8 @@#include "eal_private.h" #include "eal_thread.h" -RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); -RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (unsigned, _lcore_id) = (unsigned )LCORE_ID_ANY ; +RTE_DEFINE_PER_LCORE (unsigned, _socket_id) = (unsigned )SOCKET_ID_ANY ;RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);/*
Signature · 1 line
1.8.1.4
PATCH v2 12/15 eal: fix recursive spinlock in non-EAL thraed
In reply to a message not in the archiveIn non-EAL thread, lcore_id alrways be LCORE_ID_ANY. It cann't be used as unique id for recursive spinlock. Then use rte_gettid() to replace it.
--- lib/librte_eal/common/include/generic/rte_spinlock.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
lib/librte_eal/common/include/generic/rte_spinlock.h +2 -2
@@ -179,7 +179,7 @@ static inline void rte_spinlock_recursive_init(rte_spinlock_recursive_t *slr)*/ static inline void rte_spinlock_recursive_lock (rte_spinlock_recursive_t *slr) { -int id =rte_lcore_id (); +int id =rte_gettid ();if (slr->user != id) {rte_spinlock_lock (&slr->sl);
@@ -212,7 +212,7 @@ static inline void rte_spinlock_recursive_unlock(rte_spinlock_recursive_t *slr)*/ static inline int rte_spinlock_recursive_trylock (rte_spinlock_recursive_t *slr) { -int id =rte_lcore_id (); +int id =rte_gettid ();if (slr->user != id) {if (rte_spinlock_trylock (&slr->sl) ==0 )
Signature · 1 line
1.8.1.4
PATCH v2 13/15 mempool: add support to non-EAL thread
In reply to a message not in the archiveFor non-EAL thread, bypass per lcore cache, directly use ring pool. It allows using rte_mempool in either EAL thread or any user pthread. As in non-EAL thread, it directly rely on rte_ring and it's none preemptive. It doesn't suggest to run multi-pthread/cpu which compete the rte_mempool. It will get bad performance and has critical risk if scheduling policy is RT.
--- lib/librte_mempool/rte_mempool.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-)
lib/librte_mempool/rte_mempool.h +11 -7
@@ -198,10 +198,12 @@ struct rte_mempool {* Number to add to the object-oriented statistics. */ #ifdef RTE_LIBRTE_MEMPOOL_DEBUG -#define __MEMPOOL_STAT_ADD (mp, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - mp->stats[__lcore_id].name##_objs += n; \ - mp->stats[__lcore_id].name##_bulk += 1; \ +#define __MEMPOOL_STAT_ADD (mp, name, n) do { \ + unsigned __lcore_id = rte_lcore_id(); \ + if (__lcore_id < RTE_MAX_LCORE) { \ + mp->stats[__lcore_id].name##_objs += n; \ + mp->stats[__lcore_id].name##_bulk += 1; \ + } \ } while(0)#else #define __MEMPOOL_STAT_ADD (mp, name, n) do {} while(0)
@@ -767,8 +769,9 @@ __mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table,__MEMPOOL_STAT_ADD (mp, put, n);#if RTE_MEMPOOL_CACHE_MAX_SIZE >0 -/* cache is not enabled or single producer */ -if (unlikely (cache_size ==0 || is_mp ==0 )) +/* cache is not enabled or single producer or none EAL thread */ +if (unlikely (cache_size ==0 || is_mp ==0 || + lcore_id >=RTE_MAX_LCORE )) goto ring_enqueue;/* Go straight to ring if put would overflow mem allocated for cache */
@@ -952,7 +955,8 @@ __mempool_get_bulk(struct rte_mempool *mp, void **obj_table,uint32_t cache_size = mp->cache_size;/* cache is not enabled or single consumer */ -if (unlikely (cache_size ==0 || is_mc ==0 || n >= cache_size)) +if (unlikely (cache_size ==0 || is_mc ==0 || + n >= cache_size || lcore_id >=RTE_MAX_LCORE )) goto ring_dequeue; cache = &mp->local_cache[lcore_id];
Signature · 1 line
1.8.1.4
PATCH v2 14/15 ring: add support to non-EAL thread
In reply to a message not in the archivering debug stat won't take care non-EAL thread.
--- lib/librte_ring/rte_ring.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-)
lib/librte_ring/rte_ring.h +6 -4
@@ -188,10 +188,12 @@ struct rte_ring {* The number to add to the object-oriented statistics. */ #ifdef RTE_LIBRTE_RING_DEBUG -#define __RING_STAT_ADD (r, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - r->stats[__lcore_id].name##_objs += n; \ - r->stats[__lcore_id].name##_bulk += 1; \ +#define __RING_STAT_ADD (r, name, n) do { \ + unsigned __lcore_id = rte_lcore_id(); \ + if (__lcore_id < RTE_MAX_LCORE) { \ + r->stats[__lcore_id].name##_objs += n; \ + r->stats[__lcore_id].name##_bulk += 1; \ + } \ } while(0)#else #define __RING_STAT_ADD (r, name, n) do {} while(0)
Signature · 1 line
1.8.1.4
PATCH v2 15/15 timer: add support to non-EAL thread
In reply to a message not in the archiveAllow to setup timers only for EAL (lcore) threads (__lcore_id < MAX_LCORE_ID). E.g. – dynamically created thread will be able to reset/stop timer for lcore thread, but it will be not allowed to setup timer for itself or another non-lcore thread. rte_timer_manage() for non-lcore thread would simply do nothing and return straightway.
--- lib/librte_timer/rte_timer.c | 40 +++++++++++++++++++++++++++++++--------- lib/librte_timer/rte_timer.h | 2 +- 2 files changed, 32 insertions(+), 10 deletions(-)
lib/librte_timer/rte_timer.c +31 -9
@@ -79,9 +79,10 @@ static struct priv_timer priv_timer[RTE_MAX_LCORE];/* when debug is enabled, store some statistics */ #ifdef RTE_LIBRTE_TIMER_DEBUG -#define __TIMER_STAT_ADD (name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - priv_timer[__lcore_id].stats.name += (n); \ +#define __TIMER_STAT_ADD (name, n) do { \ + unsigned __lcore_id = rte_lcore_id(); \ + if (__lcore_id < RTE_MAX_LCORE) \ + priv_timer[__lcore_id].stats.name += (n); \ } while(0)#else #define __TIMER_STAT_ADD (name, n) do {} while(0)
@@ -127,15 +128,26 @@ timer_set_config_state(struct rte_timer *tim,unsigned lcore_id; lcore_id =rte_lcore_id (); +if (lcore_id >=RTE_MAX_LCORE ) + lcore_id =LCORE_ID_ANY ;/* wait that the timer is in correct status before update, * and mark it as being configured */ while (success ==0 ) { prev_status.u32 = tim->status.u32; +/* +* prevent race condition of non-EAL threads +* to update the timer. When 'owner == LCORE_ID_ANY', +* it means updated by a non-EAL thread. +*/ +if (lcore_id == (unsigned )LCORE_ID_ANY && + (uint16_t )lcore_id == prev_status.owner) +return -1 ; +/* timer is running on another core, exit */ if (prev_status.state ==RTE_TIMER_RUNNING && - (unsigned )prev_status.owner != lcore_id) + prev_status.owner != (uint16_t )lcore_id)return -1 ;/* timer is being configured on another core */
@@ -366,9 +378,13 @@ __rte_timer_reset(struct rte_timer *tim, uint64_t expire,/* round robin for tim_lcore */ if (tim_lcore == (unsigned )LCORE_ID_ANY ) { - tim_lcore =rte_get_next_lcore (priv_timer[lcore_id].prev_lcore, -0 ,1 ); - priv_timer[lcore_id].prev_lcore = tim_lcore; +if (lcore_id <RTE_MAX_LCORE ) { + tim_lcore =rte_get_next_lcore ( + priv_timer[lcore_id].prev_lcore, +0 ,1 ); + priv_timer[lcore_id].prev_lcore = tim_lcore; + }else + tim_lcore =rte_get_next_lcore (LCORE_ID_ANY ,0 ,1 ); }/* wait that the timer is in correct status before update,
@@ -378,7 +394,8 @@ __rte_timer_reset(struct rte_timer *tim, uint64_t expire,return -1 ;__TIMER_STAT_ADD (reset,1 ); -if (prev_status.state ==RTE_TIMER_RUNNING ) { +if (prev_status.state ==RTE_TIMER_RUNNING && + lcore_id <RTE_MAX_LCORE ) { priv_timer[lcore_id].updated =1 ; }
@@ -455,7 +472,8 @@ rte_timer_stop(struct rte_timer *tim)return -1 ;__TIMER_STAT_ADD (stop,1 ); -if (prev_status.state ==RTE_TIMER_RUNNING ) { +if (prev_status.state ==RTE_TIMER_RUNNING && + lcore_id <RTE_MAX_LCORE ) { priv_timer[lcore_id].updated =1 ; }
@@ -499,6 +517,10 @@ void rte_timer_manage(void)uint64_t cur_time;int i, ret; +/* timer manager only runs on EAL thread */ +if (lcore_id >=RTE_MAX_LCORE ) +return ; +__TIMER_STAT_ADD (manage,1 );/* optimize for the case where per-cpu list is empty */ if (priv_timer[lcore_id].pending_head.sl_next[0 ] ==NULL )
lib/librte_timer/rte_timer.h +1 -1
@@ -76,7 +76,7 @@ extern "C" {#define RTE_TIMER_RUNNING 2/**< State: timer function is running. */ #define RTE_TIMER_CONFIG 3/**< State: timer is being configured. */ -#define RTE_TIMER_NO_OWNER -1/**< Timer has no owner. */ +#define RTE_TIMER_NO_OWNER -2/**< Timer has no owner. */ /** * Timer type: Periodic or single (one-shot).
Signature · 1 line
1.8.1.4
v3 cover support multi-pthread per core
In reply to a message not in the archivev3 changes:
add sched_yield() in rte_ring to avoid long spin [15/16]
v2 changes:
add '<number>-<number>' support for EAL option '--lcores'
The patch series contain the enhancements of EAL and fixes for libraries
to run multi-pthreads(either EAL or non-EAL thread) per physical core.
Two major changes list as below:
- Extend the core affinity of each EAL thread to 1:n.
Each lcore stands for a EAL thread rather than a logical core.
The change adds new EAL option to allow static lcore to cpuset assginment. Then a lcore(EAL thread) affinity to a cpuset, original 1:1 mapping is the special case.
- Fix the libraries to allow running on any non-EAL thread.
It fix the gaps running libraries in non-EAL thread(dynamic created by user). Each fix libraries take care the case of rte_lcore_id() >= RTE_MAX_LCORE.
Thanks a million for the comments from Konstantin, Bruce, Mirek and Stephen in RFC review.
*** BLURB HERE ***
Cunming Liang (16):
eal: add cpuset into per EAL thread lcore_config
eal: new eal option '--lcores' for cpu assignment
eal: add support parsing socket_id from cpuset
eal: new TLS definition and API declaration
eal: add eal_common_thread.c for common thread API
eal: add rte_gettid() to acquire unique system tid
eal: apply affinity of EAL thread by assigned cpuset
enic: fix re-define freebsd compile complain
malloc: fix the issue of SOCKET_ID_ANY
log: fix the gap to support non-EAL thread
eal: set _lcore_id and _socket_id to (-1) by default
eal: fix recursive spinlock in non-EAL thraed
mempool: add support to non-EAL thread
ring: add support to non-EAL thread
ring: add sched_yield to avoid spin forever
timer: add support to non-EAL thread
lib/librte_eal/bsdapp/eal/Makefile | 1 + lib/librte_eal/bsdapp/eal/eal.c | 13 +- lib/librte_eal/bsdapp/eal/eal_lcore.c | 14 + lib/librte_eal/bsdapp/eal/eal_memory.c | 2 + lib/librte_eal/bsdapp/eal/eal_thread.c | 76 +++--- lib/librte_eal/common/eal_common_launch.c | 1 - lib/librte_eal/common/eal_common_log.c | 17 +- lib/librte_eal/common/eal_common_options.c | 300 ++++++++++++++++++++- lib/librte_eal/common/eal_common_thread.c | 142 ++++++++++ lib/librte_eal/common/eal_options.h | 2 + lib/librte_eal/common/eal_thread.h | 66 +++++ .../common/include/generic/rte_spinlock.h | 4 +- lib/librte_eal/common/include/rte_eal.h | 27 ++ lib/librte_eal/common/include/rte_lcore.h | 37 ++- lib/librte_eal/common/include/rte_log.h | 5 + lib/librte_eal/linuxapp/eal/Makefile | 4 + lib/librte_eal/linuxapp/eal/eal.c | 7 +- lib/librte_eal/linuxapp/eal/eal_lcore.c | 15 ++ lib/librte_eal/linuxapp/eal/eal_thread.c | 78 +++--- lib/librte_malloc/malloc_heap.h | 7 +- lib/librte_mempool/rte_mempool.h | 18 +- lib/librte_pmd_enic/enic.h | 1 + lib/librte_pmd_enic/enic_compat.h | 1 + lib/librte_ring/rte_ring.h | 35 ++- lib/librte_timer/rte_timer.c | 40 ++- lib/librte_timer/rte_timer.h | 2 +- 26 files changed, 778 insertions(+), 137 deletions(-) create mode 100644 lib/librte_eal/common/eal_common_thread.c
Signature · 1 line
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 1422491072-5114-1-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
PATCH v3 1/16 eal: add cpuset into per EAL thread lcore_config
The patch adds 'cpuset' into per-lcore configure 'lcore_config[]', as the lcore no longer always 1:1 pinning with physical cpu. The lcore now stands for a EAL thread rather than a logical cpu.
It doesn't change the default behavior of 1:1 mapping, but allows to affinity the EAL thread to multiple cpus.
--- lib/librte_eal/bsdapp/eal/eal_lcore.c | 7 +++++++ lib/librte_eal/bsdapp/eal/eal_memory.c | 2 ++ lib/librte_eal/common/include/rte_lcore.h | 8 ++++++++ lib/librte_eal/linuxapp/eal/Makefile | 1 + lib/librte_eal/linuxapp/eal/eal_lcore.c | 8 ++++++++ 5 files changed, 26 insertions(+)
lib/librte_eal/bsdapp/eal/eal_lcore.c +7 -0
@@ -76,11 +76,18 @@ rte_eal_cpu_init(void)* ones and enable them by default. */ for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +/* init cpuset for per lcore config */ +CPU_ZERO (&lcore_config[lcore_id].cpuset); + lcore_config[lcore_id].detected = (lcore_id < ncpus);if (lcore_config[lcore_id].detected ==0 ) { config->lcore_role[lcore_id] =ROLE_OFF ;continue ; } + +/* By default, lcore 1:1 map to cpu id */ +CPU_SET (lcore_id, &lcore_config[lcore_id].cpuset); +/* By default, each detected core is enabled */ config->lcore_role[lcore_id] =ROLE_RTE ; lcore_config[lcore_id].core_id =cpu_core_id (lcore_id);
lib/librte_eal/bsdapp/eal/eal_memory.c +2 -0
@@ -45,6 +45,8 @@#include "eal_internal_cfg.h" #include "eal_filesystem.h" +/* avoid re-defined against with freebsd header */ +#undef PAGE_SIZE#define PAGE_SIZE (sysconf(_SC_PAGESIZE))/*
lib/librte_eal/common/include/rte_lcore.h +8 -0
@@ -50,6 +50,13 @@ extern "C" {#define LCORE_ID_ANY -1/**< Any lcore. */ +#if defined(__linux__) +typedef cpu_set_t rte_cpuset_t ; +#elif defined(__FreeBSD__) +#include <pthread_np.h> +typedef cpuset_t rte_cpuset_t ; +#endif +/** * Structure storing internal configuration (per-lcore) */
@@ -65,6 +72,7 @@ struct lcore_config {unsigned socket_id;/**< physical socket id for this lcore */ unsigned core_id;/**< core number on socket for this lcore */ int core_index;/**< relative index, starting from 0 */ +rte_cpuset_t cpuset;/**< cpu set which the lcore affinity to */ };/**
lib/librte_eal/linuxapp/eal/Makefile +1 -0
@@ -87,6 +87,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_dev.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_options.c CFLAGS_eal.o := -D_GNU_SOURCE +CFLAGS_eal_lcore.o := -D_GNU_SOURCE CFLAGS_eal_thread.o := -D_GNU_SOURCE CFLAGS_eal_log.o := -D_GNU_SOURCE CFLAGS_eal_common_log.o := -D_GNU_SOURCE
lib/librte_eal/linuxapp/eal/eal_lcore.c +8 -0
@@ -158,11 +158,19 @@ rte_eal_cpu_init(void)* ones and enable them by default. */ for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +/* init cpuset for per lcore config */ +CPU_ZERO (&lcore_config[lcore_id].cpuset); + +/* in 1:1 mapping, record related cpu detected state */ lcore_config[lcore_id].detected =cpu_detected (lcore_id);if (lcore_config[lcore_id].detected ==0 ) { config->lcore_role[lcore_id] =ROLE_OFF ;continue ; } + +/* By default, lcore 1:1 map to cpu id */ +CPU_SET (lcore_id, &lcore_config[lcore_id].cpuset); +/* By default, each detected core is enabled */ config->lcore_role[lcore_id] =ROLE_RTE ; lcore_config[lcore_id].core_id =cpu_core_id (lcore_id);
Signature · 1 line
1.8.1.4
PATCH v3 2/16 eal: new eal option '--lcores' for cpu assignment
In reply to a message not in the archiveIt supports one new eal long option '--lcores' for EAL thread cpuset assignment.
The format pattern:
--lcores='lcores[@cpus]<,lcores[@cpus]>'
lcores, cpus could be a single digit/range or a group. '(' and ')' are necessary if it's a group.
If not supply '@cpus', the value of cpus uses the same as lcores.
e.g. '1,2@(5-7),(3-5)@(0,2),(0,6),7-8' means starting 9 EAL thread as below
lcore 0 runs on cpuset 0x41 (cpu 0,6)
lcore 1 runs on cpuset 0x2 (cpu 1)
lcore 2 runs on cpuset 0xe0 (cpu 5,6,7)
lcore 3,4,5 runs on cpuset 0x5 (cpu 0,2)
lcore 6 runs on cpuset 0x41 (cpu 0,6)
lcore 7 runs on cpuset 0x80 (cpu 7)
lcore 8 runs on cpuset 0x100 (cpu 8)
--- lib/librte_eal/common/eal_common_launch.c | 1 - lib/librte_eal/common/eal_common_options.c | 300 ++++++++++++++++++++++++++++- lib/librte_eal/common/eal_options.h | 2 + lib/librte_eal/linuxapp/eal/Makefile | 1 + 4 files changed, 299 insertions(+), 5 deletions(-)
lib/librte_eal/common/eal_common_launch.c +0 -1
@@ -117,4 +117,3 @@ rte_eal_mp_wait_lcore(void)rte_eal_wait_lcore (lcore_id); } } -
lib/librte_eal/common/eal_common_options.c +296 -4
342 lines not on this page · raw message
lib/librte_eal/common/eal_options.h +2 -0
@@ -77,6 +77,8 @@ enum {OPT_CREATE_UIO_DEV_NUM , #define OPT_VFIO_INTR "vfio-intr"OPT_VFIO_INTR_NUM , +#define OPT_LCORES "lcores" +OPT_LCORES_NUM , OPT_LONG_MAX_NUM };
lib/librte_eal/linuxapp/eal/Makefile +1 -0
@@ -95,6 +95,7 @@ CFLAGS_eal_hugepage_info.o := -D_GNU_SOURCE CFLAGS_eal_pci.o := -D_GNU_SOURCE CFLAGS_eal_pci_vfio.o := -D_GNU_SOURCE CFLAGS_eal_common_whitelist.o := -D_GNU_SOURCE +CFLAGS_eal_common_options.o := -D_GNU_SOURCE # workaround for a gcc bug with noreturn attribute # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603
Signature · 1 line
1.8.1.4
PATCH v3 3/16 eal: add support parsing socket_id from cpuset
In reply to a message not in the archiveIt returns the socket_id if all cpus in the cpuset belongs to the same NUMA node, otherwise it will return SOCKET_ID_ANY.
--- lib/librte_eal/bsdapp/eal/eal_lcore.c | 7 +++++ lib/librte_eal/common/eal_thread.h | 52 +++++++++++++++++++++++++++++++++ lib/librte_eal/linuxapp/eal/eal_lcore.c | 7 +++++ 3 files changed, 66 insertions(+)
lib/librte_eal/bsdapp/eal/eal_lcore.c +7 -0
@@ -41,6 +41,7 @@#include <rte_debug.h> #include "eal_private.h" +#include "eal_thread.h" /* No topology information available on FreeBSD including NUMA info */ #define cpu_core_id (X ) 0
@@ -112,3 +113,9 @@ rte_eal_cpu_init(void)return 0 ; } + +unsigned +eal_cpu_socket_id (__rte_unused unsigned cpu_id) +{ +return cpu_socket_id (cpu_id); +}
lib/librte_eal/common/eal_thread.h +52 -0
62 lines not on this page · raw message
lib/librte_eal/linuxapp/eal/eal_lcore.c +7 -0
@@ -45,6 +45,7 @@#include "eal_private.h" #include "eal_filesystem.h" +#include "eal_thread.h" #define SYS_CPU_DIR "/sys/devices/system/cpu/cpu%u"#define CORE_ID_FILE "topology/core_id"
@@ -197,3 +198,9 @@ rte_eal_cpu_init(void)return 0 ; } + +unsigned +eal_cpu_socket_id (unsigned cpu_id) +{ +return cpu_socket_id (cpu_id); +}
Signature · 1 line
1.8.1.4
PATCH v3 4/16 eal: new TLS definition and API declaration
In reply to a message not in the archive1. add two TLS *_socket_id* and *_cpuset*
2. add two external API rte_thread_set/get_affinity
3. add one internal API eal_thread_dump_affinity
--- lib/librte_eal/bsdapp/eal/eal_thread.c | 2 ++ lib/librte_eal/common/eal_thread.h | 14 ++++++++++++++ lib/librte_eal/common/include/rte_lcore.h | 29 +++++++++++++++++++++++++++-- lib/librte_eal/linuxapp/eal/eal_thread.c | 2 ++ 4 files changed, 45 insertions(+), 2 deletions(-)
lib/librte_eal/bsdapp/eal/eal_thread.c +2 -0
@@ -56,6 +56,8 @@#include "eal_thread.h" RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); +RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);/* * Send a message to a slave lcore identified by slave_id to call a
lib/librte_eal/common/eal_thread.h +14 -0
@@ -102,4 +102,18 @@ eal_cpuset_socket_id(rte_cpuset_t *cpusetp) return socket_id; } +/** +* Dump the current pthread cpuset. +* This function is private to EAL. +* +* @param str +* The string buffer the cpuset will dump to. +* @param size +* The string buffer size. +*/ +#define CPU_STR_LEN 256 +void +eal_thread_dump_affinity (char str[],unsigned size); + +#endif /* EAL_THREAD_H */
lib/librte_eal/common/include/rte_lcore.h +27 -2
@@ -43,6 +43,7 @@#include <rte_per_lcore.h> #include <rte_eal.h> #include <rte_launch.h> +#include <rte_memory.h> #ifdef __cplusplusextern "C" {
@@ -80,7 +81,9 @@ struct lcore_config {*/ extern struct lcore_config lcore_config[RTE_MAX_LCORE ]; -RTE_DECLARE_PER_LCORE (unsigned, _lcore_id);/**< Per core "core id". */ +RTE_DECLARE_PER_LCORE (unsigned, _lcore_id);/**< Per thread "lcore id". */ +RTE_DECLARE_PER_LCORE (unsigned, _socket_id);/**< Per thread "socket id". */ +RTE_DECLARE_PER_LCORE (rte_cpuset_t, _cpuset);/**< Per thread "cpuset". */ /** * Return the ID of the execution unit we are running on.
@@ -146,7 +149,7 @@ rte_lcore_index(int lcore_id)static inline unsigned rte_socket_id (void ) { -return lcore_config[rte_lcore_id ()].socket_id; +return RTE_PER_LCORE (_socket_id); }/**
@@ -229,6 +232,28 @@ rte_get_next_lcore(unsigned i, int skip_master, int wrap) i<RTE_MAX_LCORE ; \ i = rte_get_next_lcore(i, 1, 0)) +/** +* Set core affinity of the current thread. +* Support both EAL and none-EAL thread and update TLS. +* +* @param cpusetp +* Point to cpu_set_t for setting current thread affinity. +* @return +* On success, return 0; otherwise return -1; +*/ +int rte_thread_set_affinity (rte_cpuset_t *cpusetp); + +/** +* Get core affinity of the current thread. +* +* @param cpusetp +* Point to cpu_set_t for getting current thread cpu affinity. +* @return +* On success, return 0; otherwise return -1; +*/ +int rte_thread_get_affinity (rte_cpuset_t *cpusetp); + + #ifdef __cplusplus }#endif
lib/librte_eal/linuxapp/eal/eal_thread.c +2 -0
@@ -56,6 +56,8 @@#include "eal_thread.h" RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); +RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);/* * Send a message to a slave lcore identified by slave_id to call a
Signature · 1 line
1.8.1.4
PATCH v3 5/16 eal: add eal_common_thread.c for common thread API
In reply to a message not in the archiveThe API works for both EAL thread and none EAL thread. When calling rte_thread_set_affinity, the *_socket_id* and *_cpuset* of calling thread will be updated if the thread successful set the cpu affinity.
--- lib/librte_eal/bsdapp/eal/Makefile | 1 + lib/librte_eal/common/eal_common_thread.c | 142 ++++++++++++++++++++++++++++++ lib/librte_eal/linuxapp/eal/Makefile | 2 + 3 files changed, 145 insertions(+) create mode 100644 lib/librte_eal/common/eal_common_thread.c
lib/librte_eal/bsdapp/eal/Makefile +1 -0
@@ -73,6 +73,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_EAL_BSDAPP) += eal_common_hexdump.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_BSDAPP) += eal_common_devargs.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_BSDAPP) += eal_common_dev.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_BSDAPP) += eal_common_options.c +SRCS-$(CONFIG_RTE_LIBRTE_EAL_BSDAPP) += eal_common_thread.c CFLAGS_eal.o := -D_GNU_SOURCE #CFLAGS_eal_thread.o := -D_GNU_SOURCE
lib/librte_eal/common/eal_common_thread.c new file +142 -0
142 lines not on this page · raw message
lib/librte_eal/linuxapp/eal/Makefile +2 -0
@@ -85,6 +85,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_hexdump.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_devargs.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_dev.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_options.c +SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_thread.c CFLAGS_eal.o := -D_GNU_SOURCE CFLAGS_eal_lcore.o := -D_GNU_SOURCE
@@ -96,6 +97,7 @@ CFLAGS_eal_pci.o := -D_GNU_SOURCE CFLAGS_eal_pci_vfio.o := -D_GNU_SOURCE CFLAGS_eal_common_whitelist.o := -D_GNU_SOURCE CFLAGS_eal_common_options.o := -D_GNU_SOURCE +CFLAGS_eal_common_thread.o := -D_GNU_SOURCE # workaround for a gcc bug with noreturn attribute # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603
Signature · 1 line
1.8.1.4
PATCH v3 6/16 eal: add rte_gettid() to acquire unique system tid
In reply to a message not in the archiveThe rte_gettid() wraps the linux and freebsd syscall gettid(). It provides a persistent unique thread id for the calling thread. It will save the unique id in TLS on the first time.
--- lib/librte_eal/bsdapp/eal/eal_thread.c | 9 +++++++++ lib/librte_eal/common/include/rte_eal.h | 27 +++++++++++++++++++++++++++ lib/librte_eal/linuxapp/eal/eal_thread.c | 7 +++++++ 3 files changed, 43 insertions(+)
lib/librte_eal/bsdapp/eal/eal_thread.c +9 -0
@@ -39,6 +39,7 @@#include <sched.h> #include <pthread_np.h> #include <sys/queue.h> +#include <sys/thr.h> #include <rte_debug.h> #include <rte_atomic.h>
@@ -233,3 +234,11 @@ eal_thread_loop(__attribute__((unused)) void *arg)/* pthread_exit(NULL); */ /* return NULL; */ } + +/* require calling thread tid by gettid() */ +int rte_sys_gettid (void ) +{ +long lwpid; +thr_self (&lwpid); +return (int )lwpid; +}
lib/librte_eal/common/include/rte_eal.h +27 -0
@@ -41,6 +41,9 @@*/ #include <stdint.h> +#include <sched.h> + +#include <rte_per_lcore.h> #ifdef __cplusplusextern "C" {
@@ -262,6 +265,30 @@ rte_set_application_usage_hook( rte_usage_hook_t usage_func );*/ int rte_eal_has_hugepages (void ); +/** +* A wrap API for syscall gettid. +* +* @return +* On success, returns the thread ID of calling process. +* It always successful. +*/ +int rte_sys_gettid (void ); + +/** +* Get system unique thread id. +* +* @return +* On success, returns the thread ID of calling process. +* It always successful. +*/ +static inline int rte_gettid (void ) +{ + static RTE_DEFINE_PER_LCORE(int, _thread_id) = -1; +if (RTE_PER_LCORE (_thread_id) ==-1 ) +RTE_PER_LCORE (_thread_id) =rte_sys_gettid (); +return RTE_PER_LCORE (_thread_id); +} + #ifdef __cplusplus }#endif
lib/librte_eal/linuxapp/eal/eal_thread.c +7 -0
@@ -39,6 +39,7 @@#include <pthread.h> #include <sched.h> #include <sys/queue.h> +#include <sys/syscall.h> #include <rte_debug.h> #include <rte_atomic.h>
@@ -233,3 +234,9 @@ eal_thread_loop(__attribute__((unused)) void *arg)/* pthread_exit(NULL); */ /* return NULL; */ } + +/* require calling thread tid by gettid() */ +int rte_sys_gettid (void ) +{ +return (int )syscall (SYS_gettid); +}
Signature · 1 line
1.8.1.4
PATCH v3 7/16 eal: apply affinity of EAL thread by assigned cpuset
In reply to a message not in the archiveEAL threads use assigned cpuset to set core affinity during startup. It keeps 1:1 mapping, if no '--lcores' option is used.
--- lib/librte_eal/bsdapp/eal/eal.c | 13 ++++--- lib/librte_eal/bsdapp/eal/eal_thread.c | 63 +++++++++--------------------- lib/librte_eal/linuxapp/eal/eal.c | 7 +++- lib/librte_eal/linuxapp/eal/eal_thread.c | 67 +++++++++++--------------------- 4 files changed, 54 insertions(+), 96 deletions(-)
lib/librte_eal/bsdapp/eal/eal.c +8 -5
@@ -432,6 +432,7 @@ rte_eal_init(int argc, char **argv)int i, fctret, ret;pthread_t thread_id;static rte_atomic32_t run_once =RTE_ATOMIC32_INIT (0 ); +char cpuset[CPU_STR_LEN ];if (!rte_atomic32_test_and_set (&run_once))return -1 ;
@@ -502,13 +503,17 @@ rte_eal_init(int argc, char **argv)if (rte_eal_pci_init () <0 )rte_panic ("Cannot init PCI\n" ); -RTE_LOG (DEBUG ,EAL ,"Master core %u is ready (tid=%p)\n" , - rte_config.master_lcore, thread_id); -eal_check_mem_on_local_socket ();rte_eal_mcfg_complete (); +eal_thread_init_master (rte_config.master_lcore); + +eal_thread_dump_affinity (cpuset,CPU_STR_LEN ); + +RTE_LOG (DEBUG ,EAL ,"Master lcore %u is ready (tid=%p;cpuset=[%s])\n" , + rte_config.master_lcore, thread_id, cpuset); +if (rte_eal_dev_init () <0 )rte_panic ("Cannot init pmd devices\n" );
@@ -532,8 +537,6 @@ rte_eal_init(int argc, char **argv)rte_panic ("Cannot create thread\n" ); } -eal_thread_init_master (rte_config.master_lcore); -/* * Launch a dummy function on all slave lcores, so that master lcore * knows they are all ready when this function returns.
lib/librte_eal/bsdapp/eal/eal_thread.c +19 -44
95 lines not on this page · raw message
lib/librte_eal/linuxapp/eal/eal.c +5 -2
@@ -702,6 +702,7 @@ rte_eal_init(int argc, char **argv)static rte_atomic32_t run_once =RTE_ATOMIC32_INIT (0 );struct shared_driver *solib =NULL ;const char *logid; +char cpuset[CPU_STR_LEN ];if (!rte_atomic32_test_and_set (&run_once))return -1 ;
@@ -802,8 +803,10 @@ rte_eal_init(int argc, char **argv)eal_thread_init_master (rte_config.master_lcore); -RTE_LOG (DEBUG ,EAL ,"Master core %u is ready (tid=%x)\n" , - rte_config.master_lcore, (int )thread_id); +eal_thread_dump_affinity (cpuset,CPU_STR_LEN ); + +RTE_LOG (DEBUG ,EAL ,"Master lcore %u is ready (tid=%x;cpuset=[%s])\n" , + rte_config.master_lcore, (int )thread_id, cpuset);if (rte_eal_dev_init () <0 )rte_panic ("Cannot init pmd devices\n" );
lib/librte_eal/linuxapp/eal/eal_thread.c +22 -45
110 lines not on this page · raw message
Signature · 1 line
1.8.1.4
PATCH v3 8/16 enic: fix re-define freebsd compile complain
In reply to a message not in the archiveSome macro already been defined by freebsd 'sys/param.h'.
--- lib/librte_pmd_enic/enic.h | 1 + lib/librte_pmd_enic/enic_compat.h | 1 + 2 files changed, 2 insertions(+)
lib/librte_pmd_enic/enic.h +1 -0
@@ -66,6 +66,7 @@#define ENIC_CALC_IP_CKSUM 1#define ENIC_CALC_TCP_UDP_CKSUM 2#define ENIC_MAX_MTU 9000 +#undef PAGE_SIZE#define PAGE_SIZE 4096#define PAGE_ROUND_UP (x) \ ((((unsigned long)(x)) + PAGE_SIZE-1) & (~(PAGE_SIZE-1)))
lib/librte_pmd_enic/enic_compat.h +1 -0
@@ -67,6 +67,7 @@ #define pr_warn(y, args...) dev_warning(0, y, ##args)#define BUG () pr_err("BUG at %s:%d", __func__, __LINE__) +#undef ALIGN#define ALIGN (x, a) __ALIGN_MASK(x, (typeof(x))(a)-1)#define __ALIGN_MASK (x, mask) (((x)+(mask))&~(mask))#define udelay usleep
Signature · 1 line
1.8.1.4
PATCH v3 9/16 malloc: fix the issue of SOCKET_ID_ANY
In reply to a message not in the archiveAdd check for rte_socket_id(), avoid get unexpected return like (-1).
--- lib/librte_malloc/malloc_heap.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)
lib/librte_malloc/malloc_heap.h +6 -1
@@ -44,7 +44,12 @@ extern "C" {static inline unsigned malloc_get_numa_socket (void ) { -return rte_socket_id (); +unsigned socket_id =rte_socket_id (); + +if (socket_id == (unsigned )SOCKET_ID_ANY ) +return 0 ; + +return socket_id; } void *
Signature · 1 line
1.8.1.4
PATCH v3 10/16 log: fix the gap to support non-EAL thread
In reply to a message not in the archiveFor those non-EAL thread, *_lcore_id* is invalid and probably larger than RTE_MAX_LCORE. The patch adds the check and allows only EAL thread using EAL per thread log level and log type. Others shares the global log level.
--- lib/librte_eal/common/eal_common_log.c | 17 +++++++++++++++-- lib/librte_eal/common/include/rte_log.h | 5 +++++ 2 files changed, 20 insertions(+), 2 deletions(-)
lib/librte_eal/common/eal_common_log.c +15 -2
@@ -193,11 +193,20 @@ rte_set_log_type(uint32_t type, int enable) rte_logs.type &= (~type); } +/* Get global log type */ +uint32_t +rte_get_log_type (void ) +{ +return rte_logs.type; +} +/* get the current loglevel for the message beeing processed */ int rte_log_cur_msg_loglevel (void ) {unsigned lcore_id; lcore_id =rte_lcore_id (); +if (lcore_id >=RTE_MAX_LCORE ) +return rte_get_log_level ();return log_cur_msg[lcore_id].loglevel; }
@@ -206,6 +215,8 @@ int rte_log_cur_msg_logtype(void) {unsigned lcore_id; lcore_id =rte_lcore_id (); +if (lcore_id >=RTE_MAX_LCORE ) +return rte_get_log_type ();return log_cur_msg[lcore_id].logtype; }
@@ -265,8 +276,10 @@ rte_vlog(__attribute__((unused)) uint32_t level,/* save loglevel and logtype in a global per-lcore variable */ lcore_id =rte_lcore_id (); - log_cur_msg[lcore_id].loglevel = level; - log_cur_msg[lcore_id].logtype = logtype; +if (lcore_id <RTE_MAX_LCORE ) { + log_cur_msg[lcore_id].loglevel = level; + log_cur_msg[lcore_id].logtype = logtype; + } ret =vfprintf (f, format, ap);fflush (f);
lib/librte_eal/common/include/rte_log.h +5 -0
@@ -144,6 +144,11 @@ uint32_t rte_get_log_level(void);void rte_set_log_type (uint32_t type,int enable);/** +* Get the global log type. +*/ +uint32_t rte_get_log_type (void ); + +/** * Get the current loglevel for the message being processed. * * Before calling the user-defined stream for logging, the log
Signature · 1 line
1.8.1.4
PATCH v3 11/16 eal: set _lcore_id and _socket_id to (-1) by default
In reply to a message not in the archiveFor those none EAL thread, *_lcore_id* shall always be LCORE_ID_ANY. The libraries using *_lcore_id* as index need to take care. *_socket_id* always be SOCKET_ID_ANY unitl the thread changes the affinity by rte_thread_set_affinity()
--- lib/librte_eal/bsdapp/eal/eal_thread.c | 4 ++-- lib/librte_eal/linuxapp/eal/eal_thread.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-)
lib/librte_eal/bsdapp/eal/eal_thread.c +2 -2
@@ -56,8 +56,8 @@#include "eal_private.h" #include "eal_thread.h" -RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); -RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (unsigned, _lcore_id) = (unsigned )LCORE_ID_ANY ; +RTE_DEFINE_PER_LCORE (unsigned, _socket_id) = (unsigned )SOCKET_ID_ANY ;RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);/*
lib/librte_eal/linuxapp/eal/eal_thread.c +2 -2
@@ -57,8 +57,8 @@#include "eal_private.h" #include "eal_thread.h" -RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); -RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (unsigned, _lcore_id) = (unsigned )LCORE_ID_ANY ; +RTE_DEFINE_PER_LCORE (unsigned, _socket_id) = (unsigned )SOCKET_ID_ANY ;RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);/*
Signature · 1 line
1.8.1.4
PATCH v3 12/16 eal: fix recursive spinlock in non-EAL thraed
In reply to a message not in the archiveIn non-EAL thread, lcore_id alrways be LCORE_ID_ANY. It cann't be used as unique id for recursive spinlock. Then use rte_gettid() to replace it.
--- lib/librte_eal/common/include/generic/rte_spinlock.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
lib/librte_eal/common/include/generic/rte_spinlock.h +2 -2
@@ -179,7 +179,7 @@ static inline void rte_spinlock_recursive_init(rte_spinlock_recursive_t *slr)*/ static inline void rte_spinlock_recursive_lock (rte_spinlock_recursive_t *slr) { -int id =rte_lcore_id (); +int id =rte_gettid ();if (slr->user != id) {rte_spinlock_lock (&slr->sl);
@@ -212,7 +212,7 @@ static inline void rte_spinlock_recursive_unlock(rte_spinlock_recursive_t *slr)*/ static inline int rte_spinlock_recursive_trylock (rte_spinlock_recursive_t *slr) { -int id =rte_lcore_id (); +int id =rte_gettid ();if (slr->user != id) {if (rte_spinlock_trylock (&slr->sl) ==0 )
Signature · 1 line
1.8.1.4
PATCH v3 13/16 mempool: add support to non-EAL thread
In reply to a message not in the archiveFor non-EAL thread, bypass per lcore cache, directly use ring pool. It allows using rte_mempool in either EAL thread or any user pthread. As in non-EAL thread, it directly rely on rte_ring and it's none preemptive. It doesn't suggest to run multi-pthread/cpu which compete the rte_mempool. It will get bad performance and has critical risk if scheduling policy is RT.
--- lib/librte_mempool/rte_mempool.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-)
lib/librte_mempool/rte_mempool.h +11 -7
@@ -198,10 +198,12 @@ struct rte_mempool {* Number to add to the object-oriented statistics. */ #ifdef RTE_LIBRTE_MEMPOOL_DEBUG -#define __MEMPOOL_STAT_ADD (mp, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - mp->stats[__lcore_id].name##_objs += n; \ - mp->stats[__lcore_id].name##_bulk += 1; \ +#define __MEMPOOL_STAT_ADD (mp, name, n) do { \ + unsigned __lcore_id = rte_lcore_id(); \ + if (__lcore_id < RTE_MAX_LCORE) { \ + mp->stats[__lcore_id].name##_objs += n; \ + mp->stats[__lcore_id].name##_bulk += 1; \ + } \ } while(0)#else #define __MEMPOOL_STAT_ADD (mp, name, n) do {} while(0)
@@ -767,8 +769,9 @@ __mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table,__MEMPOOL_STAT_ADD (mp, put, n);#if RTE_MEMPOOL_CACHE_MAX_SIZE >0 -/* cache is not enabled or single producer */ -if (unlikely (cache_size ==0 || is_mp ==0 )) +/* cache is not enabled or single producer or none EAL thread */ +if (unlikely (cache_size ==0 || is_mp ==0 || + lcore_id >=RTE_MAX_LCORE )) goto ring_enqueue;/* Go straight to ring if put would overflow mem allocated for cache */
@@ -952,7 +955,8 @@ __mempool_get_bulk(struct rte_mempool *mp, void **obj_table,uint32_t cache_size = mp->cache_size;/* cache is not enabled or single consumer */ -if (unlikely (cache_size ==0 || is_mc ==0 || n >= cache_size)) +if (unlikely (cache_size ==0 || is_mc ==0 || + n >= cache_size || lcore_id >=RTE_MAX_LCORE )) goto ring_dequeue; cache = &mp->local_cache[lcore_id];
Signature · 1 line
1.8.1.4
PATCH v3 14/16 ring: add support to non-EAL thread
In reply to a message not in the archivering debug stat won't take care non-EAL thread.
--- lib/librte_ring/rte_ring.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-)
lib/librte_ring/rte_ring.h +6 -4
@@ -188,10 +188,12 @@ struct rte_ring {* The number to add to the object-oriented statistics. */ #ifdef RTE_LIBRTE_RING_DEBUG -#define __RING_STAT_ADD (r, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - r->stats[__lcore_id].name##_objs += n; \ - r->stats[__lcore_id].name##_bulk += 1; \ +#define __RING_STAT_ADD (r, name, n) do { \ + unsigned __lcore_id = rte_lcore_id(); \ + if (__lcore_id < RTE_MAX_LCORE) { \ + r->stats[__lcore_id].name##_objs += n; \ + r->stats[__lcore_id].name##_bulk += 1; \ + } \ } while(0)#else #define __RING_STAT_ADD (r, name, n) do {} while(0)
Signature · 1 line
1.8.1.4
PATCH v3 15/16 ring: add sched_yield to avoid spin forever
In reply to a message not in the archiveIt does a gentle yield after spin for a while.
It reduces the wasting by spin when the preemption happens.
--- lib/librte_ring/rte_ring.h | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-)
lib/librte_ring/rte_ring.h +19 -6
@@ -126,6 +126,7 @@ struct rte_ring_debug_stats {#define RTE_RING_NAMESIZE 32/**< The maximum length of a ring name. */ #define RTE_RING_MZ_PREFIX "RG_" +#define RTE_RING_PAUSE_REP 0x100/**< yield after num of times pause. */ /** * An RTE ring structure.
@@ -410,7 +411,7 @@ __rte_ring_mp_do_enqueue(struct rte_ring *r, void * const *obj_table,uint32_t cons_tail, free_entries;const unsigned max = n;int success; -unsigned i; +unsigned i, rep;uint32_t mask = r->prod.mask;int ret;
@@ -468,8 +469,14 @@ __rte_ring_mp_do_enqueue(struct rte_ring *r, void * const *obj_table,* If there are other enqueues in progress that preceded us, * we need to wait for them to complete */ -while (unlikely (r->prod.tail != prod_head)) -rte_pause (); +do { +for (rep =RTE_RING_PAUSE_REP ; + rep !=0 && r->prod.tail != prod_head; rep--) +rte_pause (); + +if (rep ==0 ) +sched_yield (); + }while (rep ==0 ); r->prod.tail = prod_next;return ret;
@@ -589,7 +596,7 @@ __rte_ring_mc_do_dequeue(struct rte_ring *r, void **obj_table,uint32_t cons_next, entries;const unsigned max = n;int success; -unsigned i; +unsigned i, rep;uint32_t mask = r->prod.mask;/* move cons.head atomically */
@@ -634,8 +641,14 @@ __rte_ring_mc_do_dequeue(struct rte_ring *r, void **obj_table,* If there are other dequeues in progress that preceded us, * we need to wait for them to complete */ -while (unlikely (r->cons.tail != cons_head)) -rte_pause (); +do { +for (rep =RTE_RING_PAUSE_REP ; + rep !=0 && r->cons.tail != cons_head; rep--) +rte_pause (); + +if (rep ==0 ) +sched_yield (); + }while (rep ==0 );__RING_STAT_ADD (r, deq_success, n); r->cons.tail = cons_next;
Signature · 1 line
1.8.1.4
PATCH v3 16/16 timer: add support to non-EAL thread
In reply to a message not in the archiveAllow to setup timers only for EAL (lcore) threads (__lcore_id < MAX_LCORE_ID). E.g. – dynamically created thread will be able to reset/stop timer for lcore thread, but it will be not allowed to setup timer for itself or another non-lcore thread. rte_timer_manage() for non-lcore thread would simply do nothing and return straightway.
--- lib/librte_timer/rte_timer.c | 40 +++++++++++++++++++++++++++++++--------- lib/librte_timer/rte_timer.h | 2 +- 2 files changed, 32 insertions(+), 10 deletions(-)
lib/librte_timer/rte_timer.c +31 -9
@@ -79,9 +79,10 @@ static struct priv_timer priv_timer[RTE_MAX_LCORE];/* when debug is enabled, store some statistics */ #ifdef RTE_LIBRTE_TIMER_DEBUG -#define __TIMER_STAT_ADD (name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - priv_timer[__lcore_id].stats.name += (n); \ +#define __TIMER_STAT_ADD (name, n) do { \ + unsigned __lcore_id = rte_lcore_id(); \ + if (__lcore_id < RTE_MAX_LCORE) \ + priv_timer[__lcore_id].stats.name += (n); \ } while(0)#else #define __TIMER_STAT_ADD (name, n) do {} while(0)
@@ -127,15 +128,26 @@ timer_set_config_state(struct rte_timer *tim,unsigned lcore_id; lcore_id =rte_lcore_id (); +if (lcore_id >=RTE_MAX_LCORE ) + lcore_id =LCORE_ID_ANY ;/* wait that the timer is in correct status before update, * and mark it as being configured */ while (success ==0 ) { prev_status.u32 = tim->status.u32; +/* +* prevent race condition of non-EAL threads +* to update the timer. When 'owner == LCORE_ID_ANY', +* it means updated by a non-EAL thread. +*/ +if (lcore_id == (unsigned )LCORE_ID_ANY && + (uint16_t )lcore_id == prev_status.owner) +return -1 ; +/* timer is running on another core, exit */ if (prev_status.state ==RTE_TIMER_RUNNING && - (unsigned )prev_status.owner != lcore_id) + prev_status.owner != (uint16_t )lcore_id)return -1 ;/* timer is being configured on another core */
@@ -366,9 +378,13 @@ __rte_timer_reset(struct rte_timer *tim, uint64_t expire,/* round robin for tim_lcore */ if (tim_lcore == (unsigned )LCORE_ID_ANY ) { - tim_lcore =rte_get_next_lcore (priv_timer[lcore_id].prev_lcore, -0 ,1 ); - priv_timer[lcore_id].prev_lcore = tim_lcore; +if (lcore_id <RTE_MAX_LCORE ) { + tim_lcore =rte_get_next_lcore ( + priv_timer[lcore_id].prev_lcore, +0 ,1 ); + priv_timer[lcore_id].prev_lcore = tim_lcore; + }else + tim_lcore =rte_get_next_lcore (LCORE_ID_ANY ,0 ,1 ); }/* wait that the timer is in correct status before update,
@@ -378,7 +394,8 @@ __rte_timer_reset(struct rte_timer *tim, uint64_t expire,return -1 ;__TIMER_STAT_ADD (reset,1 ); -if (prev_status.state ==RTE_TIMER_RUNNING ) { +if (prev_status.state ==RTE_TIMER_RUNNING && + lcore_id <RTE_MAX_LCORE ) { priv_timer[lcore_id].updated =1 ; }
@@ -455,7 +472,8 @@ rte_timer_stop(struct rte_timer *tim)return -1 ;__TIMER_STAT_ADD (stop,1 ); -if (prev_status.state ==RTE_TIMER_RUNNING ) { +if (prev_status.state ==RTE_TIMER_RUNNING && + lcore_id <RTE_MAX_LCORE ) { priv_timer[lcore_id].updated =1 ; }
@@ -499,6 +517,10 @@ void rte_timer_manage(void)uint64_t cur_time;int i, ret; +/* timer manager only runs on EAL thread */ +if (lcore_id >=RTE_MAX_LCORE ) +return ; +__TIMER_STAT_ADD (manage,1 );/* optimize for the case where per-cpu list is empty */ if (priv_timer[lcore_id].pending_head.sl_next[0 ] ==NULL )
lib/librte_timer/rte_timer.h +1 -1
@@ -76,7 +76,7 @@ extern "C" {#define RTE_TIMER_RUNNING 2/**< State: timer function is running. */ #define RTE_TIMER_CONFIG 3/**< State: timer is being configured. */ -#define RTE_TIMER_NO_OWNER -1/**< Timer has no owner. */ +#define RTE_TIMER_NO_OWNER -2/**< Timer has no owner. */ /** * Timer type: Periodic or single (one-shot).
Signature · 1 line
1.8.1.4
v4 cover support multi-pthread per core
In reply to a message not in the archivev4 changes:
new patch fixing strnlen() invalid return in 32bit icc [03/17] update and add more comments on sched_yield() [16/17]
v3 changes:
new patch adding sched_yield() in rte_ring to avoid long spin [16/17]
v2 changes:
add '<number>-<number>' support for EAL option '--lcores' [02/17]
The patch series contain the enhancements of EAL and fixes for libraries
to run multi-pthreads(either EAL or non-EAL thread) per physical core.
Two major changes list as below:
- Extend the core affinity of each EAL thread to 1:n.
Each lcore stands for a EAL thread rather than a logical core.
The change adds new EAL option to allow static lcore to cpuset assginment. Then a lcore(EAL thread) affinity to a cpuset, original 1:1 mapping is the special case.
- Fix the libraries to allow running on any non-EAL thread.
It fix the gaps running libraries in non-EAL thread(dynamic created by user). Each fix libraries take care the case of rte_lcore_id() >= RTE_MAX_LCORE.
Thanks a million for the comments from Konstantin, Bruce, Mirek and Stephen in RFC review.
*** BLURB HERE ***
Cunming Liang (17):
eal: add cpuset into per EAL thread lcore_config
eal: new eal option '--lcores' for cpu assignment
eal: fix wrong strnlen() return value in 32bit icc
eal: add support parsing socket_id from cpuset
eal: new TLS definition and API declaration
eal: add eal_common_thread.c for common thread API
eal: add rte_gettid() to acquire unique system tid
eal: apply affinity of EAL thread by assigned cpuset
enic: fix re-define freebsd compile complain
malloc: fix the issue of SOCKET_ID_ANY
log: fix the gap to support non-EAL thread
eal: set _lcore_id and _socket_id to (-1) by default
eal: fix recursive spinlock in non-EAL thraed
mempool: add support to non-EAL thread
ring: add support to non-EAL thread
ring: add sched_yield to avoid spin forever
timer: add support to non-EAL thread
lib/librte_eal/bsdapp/eal/Makefile | 1 + lib/librte_eal/bsdapp/eal/eal.c | 13 +- lib/librte_eal/bsdapp/eal/eal_lcore.c | 14 + lib/librte_eal/bsdapp/eal/eal_memory.c | 2 + lib/librte_eal/bsdapp/eal/eal_thread.c | 76 +++--- lib/librte_eal/common/eal_common_launch.c | 1 - lib/librte_eal/common/eal_common_log.c | 17 +- lib/librte_eal/common/eal_common_options.c | 302 ++++++++++++++++++++- lib/librte_eal/common/eal_common_thread.c | 142 ++++++++++ lib/librte_eal/common/eal_options.h | 2 + lib/librte_eal/common/eal_thread.h | 66 +++++ .../common/include/generic/rte_spinlock.h | 4 +- lib/librte_eal/common/include/rte_eal.h | 27 ++ lib/librte_eal/common/include/rte_lcore.h | 37 ++- lib/librte_eal/common/include/rte_log.h | 5 + lib/librte_eal/linuxapp/eal/Makefile | 4 + lib/librte_eal/linuxapp/eal/eal.c | 7 +- lib/librte_eal/linuxapp/eal/eal_lcore.c | 15 + lib/librte_eal/linuxapp/eal/eal_thread.c | 78 +++--- lib/librte_malloc/malloc_heap.h | 7 +- lib/librte_mempool/rte_mempool.h | 18 +- lib/librte_pmd_enic/enic.h | 1 + lib/librte_pmd_enic/enic_compat.h | 1 + lib/librte_ring/rte_ring.h | 45 ++- lib/librte_timer/rte_timer.c | 40 ++- lib/librte_timer/rte_timer.h | 2 +- 26 files changed, 789 insertions(+), 138 deletions(-) create mode 100644 lib/librte_eal/common/eal_common_thread.c
Signature · 1 line
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 1422842559-13617-1-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
PATCH v4 1/17 eal: add cpuset into per EAL thread lcore_config
The patch adds 'cpuset' into per-lcore configure 'lcore_config[]', as the lcore no longer always 1:1 pinning with physical cpu. The lcore now stands for a EAL thread rather than a logical cpu.
It doesn't change the default behavior of 1:1 mapping, but allows to affinity the EAL thread to multiple cpus.
--- lib/librte_eal/bsdapp/eal/eal_lcore.c | 7 +++++++ lib/librte_eal/bsdapp/eal/eal_memory.c | 2 ++ lib/librte_eal/common/include/rte_lcore.h | 8 ++++++++ lib/librte_eal/linuxapp/eal/Makefile | 1 + lib/librte_eal/linuxapp/eal/eal_lcore.c | 8 ++++++++ 5 files changed, 26 insertions(+)
lib/librte_eal/bsdapp/eal/eal_lcore.c +7 -0
@@ -76,11 +76,18 @@ rte_eal_cpu_init(void)* ones and enable them by default. */ for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +/* init cpuset for per lcore config */ +CPU_ZERO (&lcore_config[lcore_id].cpuset); + lcore_config[lcore_id].detected = (lcore_id < ncpus);if (lcore_config[lcore_id].detected ==0 ) { config->lcore_role[lcore_id] =ROLE_OFF ;continue ; } + +/* By default, lcore 1:1 map to cpu id */ +CPU_SET (lcore_id, &lcore_config[lcore_id].cpuset); +/* By default, each detected core is enabled */ config->lcore_role[lcore_id] =ROLE_RTE ; lcore_config[lcore_id].core_id =cpu_core_id (lcore_id);
lib/librte_eal/bsdapp/eal/eal_memory.c +2 -0
@@ -45,6 +45,8 @@#include "eal_internal_cfg.h" #include "eal_filesystem.h" +/* avoid re-defined against with freebsd header */ +#undef PAGE_SIZE#define PAGE_SIZE (sysconf(_SC_PAGESIZE))/*
lib/librte_eal/common/include/rte_lcore.h +8 -0
@@ -50,6 +50,13 @@ extern "C" {#define LCORE_ID_ANY -1/**< Any lcore. */ +#if defined(__linux__) +typedef cpu_set_t rte_cpuset_t ; +#elif defined(__FreeBSD__) +#include <pthread_np.h> +typedef cpuset_t rte_cpuset_t ; +#endif +/** * Structure storing internal configuration (per-lcore) */
@@ -65,6 +72,7 @@ struct lcore_config {unsigned socket_id;/**< physical socket id for this lcore */ unsigned core_id;/**< core number on socket for this lcore */ int core_index;/**< relative index, starting from 0 */ +rte_cpuset_t cpuset;/**< cpu set which the lcore affinity to */ };/**
lib/librte_eal/linuxapp/eal/Makefile +1 -0
@@ -87,6 +87,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_dev.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_options.c CFLAGS_eal.o := -D_GNU_SOURCE +CFLAGS_eal_lcore.o := -D_GNU_SOURCE CFLAGS_eal_thread.o := -D_GNU_SOURCE CFLAGS_eal_log.o := -D_GNU_SOURCE CFLAGS_eal_common_log.o := -D_GNU_SOURCE
lib/librte_eal/linuxapp/eal/eal_lcore.c +8 -0
@@ -158,11 +158,19 @@ rte_eal_cpu_init(void)* ones and enable them by default. */ for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +/* init cpuset for per lcore config */ +CPU_ZERO (&lcore_config[lcore_id].cpuset); + +/* in 1:1 mapping, record related cpu detected state */ lcore_config[lcore_id].detected =cpu_detected (lcore_id);if (lcore_config[lcore_id].detected ==0 ) { config->lcore_role[lcore_id] =ROLE_OFF ;continue ; } + +/* By default, lcore 1:1 map to cpu id */ +CPU_SET (lcore_id, &lcore_config[lcore_id].cpuset); +/* By default, each detected core is enabled */ config->lcore_role[lcore_id] =ROLE_RTE ; lcore_config[lcore_id].core_id =cpu_core_id (lcore_id);
Signature · 1 line
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 1422842559-13617-2-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
Re: [PATCH v4 01/17] eal: add cpuset into per EAL thread lcore_config
Hi,
Quoted 13 lines · Cunming Liang
The patch adds 'cpuset' into per-lcore configure 'lcore_config[]', as the lcore no longer always 1:1 pinning with physical cpu. The lcore now stands for a EAL thread rather than a logical cpu.
It doesn't change the default behavior of 1:1 mapping, but allows to affinity the EAL thread to multiple cpus.
[...]
lib/librte_eal/bsdapp/eal/eal_memory.c +0
-0
lib/librte_eal/bsdapp/eal/eal_memory.c line 50
Quoted 6 lines
lib/librte_eal/bsdapp/eal/eal_memory.c +2
-0@@ -45,6 +45,8 @@#include "eal_internal_cfg.h" #include "eal_filesystem.h" +/* avoid re-defined against with freebsd header */ +#undef PAGE_SIZE#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
I don't see the link with the patch. Should this go somewhere else?
Quoted 7 lines
lib/librte_eal/bsdapp/eal/eal_memory.c +0
-0/* lib/librte_eal/common/include/rte_lcore.h +0
-0
lib/librte_eal/common/include/rte_lcore.h extern "C" { · line 59
Quoted 10 lines
lib/librte_eal/common/include/rte_lcore.h +7
-0@@ -50,6 +50,13 @@ extern "C" {#define LCORE_ID_ANY -1/**< Any lcore. */ +#if defined(__linux__) +typedef cpu_set_t rte_cpuset_t ; +#elif defined(__FreeBSD__) +#include <pthread_np.h> +typedef cpuset_t rte_cpuset_t ; +#endif +
Should we also define RTE_CPU_SETSIZE?
For linux, should <sched.h> be included?
If I understand well, after the patch series, the user of rte_thread_set_affinity() and rte_thread_get_affinity() are supposed to use the macros from sched.h to access to this cpuset parameter. So I'm wondering if it's not better to use cpu_set_t from libc instead of redefining rte_cpuset_t.
To reword my question: what is the purpose of redefining cpu_set_t in rte_cpuset_t if we still need to use all the libc API to access to it?
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH v4 01/17] eal: add cpuset into per EAL thread lcore_config
Quoted 47 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Monday, February 09, 2015 4:00 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 01/17] eal: add cpuset into per EAL threadlcore_configHi,
Review
Quoted 9 lines · Cunming Liang
The patch adds 'cpuset' into per-lcore configure 'lcore_config[]', as the lcore no longer always 1:1 pinning with physical cpu. The lcore now stands for a EAL thread rather than a logical cpu.
It doesn't change the default behavior of 1:1 mapping, but allows to affinity the EAL thread to multiple cpus.
[...]
(quoted diff) +0
-0
b/lib/librte_eal/bsdapp/eal/eal_memory.clib/librte_eal/bsdapp/eal/eal_memory.c line 50
Quoted 10 lines
index 65ee87d..a34d500 100644
lib/librte_eal/bsdapp/eal/eal_memory.c +2
-0@@ -45,6 +45,8 @@#include "eal_internal_cfg.h" #include "eal_filesystem.h" +/* avoid re-defined against with freebsd header */ +#undef PAGE_SIZE#define PAGE_SIZE (sysconf(_SC_PAGESIZE))I don't see the link with the patch. Should this go somewhere else?
Review
Quoted 3 lines
lib/librte_eal/bsdapp/eal/eal_memory.c +0
-0/* (quoted diff) +0
-0
b/lib/librte_eal/common/include/rte_lcore.hQuoted 7 lines
index 49b2c03..4c7d6bb 100644
lib/librte_eal/common/include/rte_lcore.h +0
-0@@ -50,6 +50,13 @@ extern "C" {#define LCORE_ID_ANY -1/**< Any lcore. */
Quoted 10 lines
lib/librte_eal/common/include/rte_lcore.h extern "C" { · line 59
Quoted 7 lines
lib/librte_eal/common/include/rte_lcore.h +7
-0+#if defined(__linux__) +typedef cpu_set_t rte_cpuset_t ; +#elif defined(__FreeBSD__) +#include <pthread_np.h> +typedef cpuset_t rte_cpuset_t ; +#endif +Should we also define
RTE_CPU_SETSIZE?
For linux, should <sched.h> be included?
[LCM] It uses the fix size cpuset, won't use CPU_ALLOC() to get the pointer of cpuset. The RTE_CPU_SETSIZE always equal to sizeof(rte_cpuset_t).
Quoted 10 lines
If I understand well, after the patch series, the user of
rte_thread_set_affinity()andrte_thread_get_affinity()are supposed to use the macros fromsched.hto access to this cpuset parameter. So I'm wondering if it's not better to usecpu_set_tfrom libc instead of redefiningrte_cpuset_t.To reword my question: what is the purpose of redefining
cpu_set_tinrte_cpuset_tif we still need to use all the libc API to access to it?
[LCM] In linux the type is *cpu_set_t*, but in freebsd it's *cpuset_t*.
The purpose of *rte_cpuset_t* is to make the consistent type definition in EAL, and to avoid lots of #ifdef for this diff. In either linux or freebsd, it still can use the MACRO in libc to set the rte_cpuset_t.
Quoted 4 lines
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): D0158A423229094DA7ABF71CF2FA0DA3118D7CE1-E2R4CRU6q/[email protected]
Re: [PATCH v4 01/17] eal: add cpuset into per EAL thread lcore_config
Hi,
Quoted 18 lines · Cunming
Quoted 18 lines · Cunming Liang
Review
Quoted 9 lines
The patch adds 'cpuset' into per-lcore configure 'lcore_config[]', as the lcore no longer always 1:1 pinning with physical cpu. The lcore now stands for a EAL thread rather than a logical cpu.
It doesn't change the default behavior of 1:1 mapping, but allows to affinity the EAL thread to multiple cpus.
[...]
(quoted diff) +0
-0
b/lib/librte_eal/bsdapp/eal/eal_memory.cQuoted 7 lines
index 65ee87d..a34d500 100644
lib/librte_eal/bsdapp/eal/eal_memory.c +0
-0@@ -45,6 +45,8 @@#include "eal_internal_cfg.h" #include "eal_filesystem.h"
Quoted 5 lines
Quoted 5 lines
lib/librte_eal/bsdapp/eal/eal_memory.c line 50
Quoted 3 lines
lib/librte_eal/bsdapp/eal/eal_memory.c +2
-0+/* avoid re-defined against with freebsd header */ +#undef PAGE_SIZE#define PAGE_SIZE (sysconf(_SC_PAGESIZE))I don't see the link with the patch. Should this go somewhere else?
Maybe you missed this one.
Quoted 19 lines
Quoted 9 lines
Review
Quoted 1 line
(quoted diff) +0
-0
b/lib/librte_eal/common/include/rte_lcore.hQuoted 7 lines
index 49b2c03..4c7d6bb 100644
lib/librte_eal/common/include/rte_lcore.h +0
-0@@ -50,6 +50,13 @@ extern "C" {#define LCORE_ID_ANY -1/**< Any lcore. */ Quoted 10 lines
lib/librte_eal/common/include/rte_lcore.h extern "C" { · line 59
Quoted 7 lines
lib/librte_eal/common/include/rte_lcore.h +7
-0+#if defined(__linux__) +typedef cpu_set_t rte_cpuset_t ; +#elif defined(__FreeBSD__) +#include <pthread_np.h> +typedef cpuset_t rte_cpuset_t ; +#endif +Should we also define
RTE_CPU_SETSIZE?
For linux, should <sched.h> be included?
Quoted 2 lines
[LCM] It uses the fix size cpuset, won't use
CPU_ALLOC()to get the pointer of cpuset. TheRTE_CPU_SETSIZEalways equal tosizeof(rte_cpuset_t).
The advantage of using CPU_ALLOC() is to avoid issues when the number of core will be higher than 1024. I agree it's probably a bit early to think about this, but it could happen soon :)
Quoted 6 lines
Quoted 6 lines
If I understand well, after the patch series, the user of
rte_thread_set_affinity()andrte_thread_get_affinity()are supposed to use the macros fromsched.hto access to this cpuset parameter. So I'm wondering if it's not better to usecpu_set_tfrom libc instead of redefiningrte_cpuset_t.
Quoted 6 lines
Quoted 3 lines
To reword my question: what is the purpose of redefining
cpu_set_tinrte_cpuset_tif we still need to use all the libc API to access to it?[LCM] In linux the type is *cpu_set_t*, but in freebsd it's *cpuset_t*.
The purpose of *rte_cpuset_t* is to make the consistent type definition in EAL, and to avoid lots of #ifdef for this diff. In either linux or freebsd, it still can use the MACRO in libc to set therte_cpuset_t.
OK, it makes sense then. I did not notice the difference between linux and bsd.
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH v4 01/17] eal: add cpuset into per EAL thread lcore_config
Quoted 57 lines
-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Olivier MATZ
Sent: Monday, February 09, 2015 5:07 PM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 01/17] eal: add cpuset into per EAL threadlcore_configHi,
Quoted 18 lines · Cunming
Quoted 18 lines · Cunming Liang
Review
Quoted 9 lines
The patch adds 'cpuset' into per-lcore configure 'lcore_config[]', as the lcore no longer always 1:1 pinning with physical cpu. The lcore now stands for a EAL thread rather than a logical cpu.
It doesn't change the default behavior of 1:1 mapping, but allows to affinity the EAL thread to multiple cpus.
[...]
(quoted diff) +0
-0
b/lib/librte_eal/bsdapp/eal/eal_memory.cQuoted 7 lines
index 65ee87d..a34d500 100644
lib/librte_eal/bsdapp/eal/eal_memory.c +0
-0@@ -45,6 +45,8 @@#include "eal_internal_cfg.h" #include "eal_filesystem.h" Quoted 5 lines
Quoted 5 lines
lib/librte_eal/bsdapp/eal/eal_memory.c line 50
Quoted 3 lines
lib/librte_eal/bsdapp/eal/eal_memory.c +2
-0+/* avoid re-defined against with freebsd header */ +#undef PAGE_SIZE#define PAGE_SIZE (sysconf(_SC_PAGESIZE))I don't see the link with the patch. Should this go somewhere else?
Maybe you missed this one.
Quoted 19 lines
Quoted 19 lines
Review
Quoted 1 line
(quoted diff) +0
-0
b/lib/librte_eal/common/include/rte_lcore.hQuoted 4 lines
index 49b2c03..4c7d6bb 100644
lib/librte_eal/common/include/rte_lcore.h extern "C" { · line 59
Quoted 10 lines
lib/librte_eal/common/include/rte_lcore.h +7
-0@@ -50,6 +50,13 @@ extern "C" {#define LCORE_ID_ANY -1/**< Any lcore. */ +#if defined(__linux__) +typedef cpu_set_t rte_cpuset_t ; +#elif defined(__FreeBSD__) +#include <pthread_np.h> +typedef cpuset_t rte_cpuset_t ; +#endif +Should we also define
RTE_CPU_SETSIZE?
For linux, should <sched.h> be included?
Quoted 6 lines
Quoted 2 lines
[LCM] It uses the fix size cpuset, won't use
CPU_ALLOC()to get the pointer of cpuset. TheRTE_CPU_SETSIZEalways equal tosizeof(rte_cpuset_t).The advantage of using
CPU_ALLOC()is to avoid issues when the number of core will be higher than 1024. I agree it's probably a bit early to think about this, but it could happen soon :)
I personally don't think, we'll hit 1K cpu limit anytime soon...
Quoted 1 line
From other side - fixed size cpuset allows to cleanup and simplify code quite a bit.
So, I'd suggest to stick with fixed size for now.
Konstantin
Quoted 17 lines
Quoted 8 lines
Quoted 8 lines
If I understand well, after the patch series, the user of
rte_thread_set_affinity()andrte_thread_get_affinity()are supposed to use the macros fromsched.hto access to this cpuset parameter. So I'm wondering if it's not better to usecpu_set_tfrom libc instead of redefiningrte_cpuset_t.Quoted 6 lines
Quoted 3 lines
To reword my question: what is the purpose of redefining
cpu_set_tinrte_cpuset_tif we still need to use all the libc API to access to it?[LCM] In linux the type is *cpu_set_t*, but in freebsd it's *cpuset_t*.
The purpose of *rte_cpuset_t* is to make the consistent type definition in EAL, and to avoid lots of #ifdef for this diff. In either linux or freebsd, it still can use the MACRO in libc to set therte_cpuset_t.OK, it makes sense then. I did not notice the difference between linux and bsd.
Re: [PATCH v4 01/17] eal: add cpuset into per EAL thread lcore_config
In reply to a message not in the archiveQuoted 30 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Tuesday, February 10, 2015 1:07 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 01/17] eal: add cpuset into per EAL threadlcore_configHi,
Quoted 18 lines · Cunming
Quoted 18 lines · Cunming Liang
Review
Quoted 9 lines
The patch adds 'cpuset' into per-lcore configure 'lcore_config[]', as the lcore no longer always 1:1 pinning with physical cpu. The lcore now stands for a EAL thread rather than a logical cpu.
It doesn't change the default behavior of 1:1 mapping, but allows to affinity the EAL thread to multiple cpus.
[...]
(quoted diff) +0
-0
b/lib/librte_eal/bsdapp/eal/eal_memory.cQuoted 7 lines
index 65ee87d..a34d500 100644
lib/librte_eal/bsdapp/eal/eal_memory.c +0
-0@@ -45,6 +45,8 @@#include "eal_internal_cfg.h" #include "eal_filesystem.h"
Quoted 7 lines
Quoted 5 lines
Quoted 5 lines
lib/librte_eal/bsdapp/eal/eal_memory.c line 50
Quoted 3 lines
lib/librte_eal/bsdapp/eal/eal_memory.c +2
-0+/* avoid re-defined against with freebsd header */ +#undef PAGE_SIZE#define PAGE_SIZE (sysconf(_SC_PAGESIZE))I don't see the link with the patch. Should this go somewhere else?
Maybe you missed this one.
[LCM] Yes, I missed this one. I agree to move to a separate one and remove undef but rename the PAGE_SIZE to EAL_PAGE_SIZE.
Quoted 47 lines
Quoted 21 lines
Quoted 11 lines
Review
Quoted 3 lines
(quoted diff) +0
-0
b/lib/librte_eal/common/include/rte_lcore.hQuoted 7 lines
index 49b2c03..4c7d6bb 100644
lib/librte_eal/common/include/rte_lcore.h +0
-0@@ -50,6 +50,13 @@ extern "C" {#define LCORE_ID_ANY -1/**< Any lcore. */ Quoted 10 lines
lib/librte_eal/common/include/rte_lcore.h extern "C" { · line 59
Quoted 7 lines
lib/librte_eal/common/include/rte_lcore.h +7
-0+#if defined(__linux__) +typedef cpu_set_t rte_cpuset_t ; +#elif defined(__FreeBSD__) +#include <pthread_np.h> +typedef cpuset_t rte_cpuset_t ; +#endif +Should we also define
RTE_CPU_SETSIZE?
For linux, should <sched.h> be included?Quoted 1 line
[LCM] It uses the fix size cpuset, won't use
CPU_ALLOC()to get the pointer ofcpuset.
Quoted 1 line
The
RTE_CPU_SETSIZEalways equal tosizeof(rte_cpuset_t).The advantage of using
CPU_ALLOC()is to avoid issues when the number of core will be higher than 1024. I agree it's probably a bit early to think about this, but it could happen soon :)Quoted 11 lines
Quoted 9 lines
If I understand well, after the patch series, the user of
rte_thread_set_affinity()andrte_thread_get_affinity()are supposed to use the macros fromsched.hto access to this cpuset parameter. So I'm wondering if it's not better to usecpu_set_tfrom libc instead of redefiningrte_cpuset_t.To reword my question: what is the purpose of redefining
cpu_set_tinrte_cpuset_tif we still need to use all the libc API to access to it?[LCM] In linux the type is *cpu_set_t*, but in freebsd it's *cpuset_t*. The purpose of *rte_cpuset_t* is to make the consistent type definition in EAL,
and to avoid lots of #ifdef for this diff.
Quoted 1 line
In either linux or freebsd, it still can use the MACRO in libc to set the
rte_cpuset_t.OK, it makes sense then. I did not notice the difference between linux and bsd.
PATCH v4 2/17 eal: new eal option '--lcores' for cpu assignment
In reply to a message not in the archiveIt supports one new eal long option '--lcores' for EAL thread cpuset assignment.
The format pattern:
--lcores='lcores[@cpus]<,lcores[@cpus]>'
lcores, cpus could be a single digit/range or a group. '(' and ')' are necessary if it's a group.
If not supply '@cpus', the value of cpus uses the same as lcores.
e.g. '1,2@(5-7),(3-5)@(0,2),(0,6),7-8' means starting 9 EAL thread as below
lcore 0 runs on cpuset 0x41 (cpu 0,6)
lcore 1 runs on cpuset 0x2 (cpu 1)
lcore 2 runs on cpuset 0xe0 (cpu 5,6,7)
lcore 3,4,5 runs on cpuset 0x5 (cpu 0,2)
lcore 6 runs on cpuset 0x41 (cpu 0,6)
lcore 7 runs on cpuset 0x80 (cpu 7)
lcore 8 runs on cpuset 0x100 (cpu 8)
--- lib/librte_eal/common/eal_common_launch.c | 1 - lib/librte_eal/common/eal_common_options.c | 300 ++++++++++++++++++++++++++++- lib/librte_eal/common/eal_options.h | 2 + lib/librte_eal/linuxapp/eal/Makefile | 1 + 4 files changed, 299 insertions(+), 5 deletions(-)
lib/librte_eal/common/eal_common_launch.c +0 -1
@@ -117,4 +117,3 @@ rte_eal_mp_wait_lcore(void)rte_eal_wait_lcore (lcore_id); } } -
lib/librte_eal/common/eal_common_options.c +296 -4
342 lines not on this page · raw message
lib/librte_eal/common/eal_options.h +2 -0
@@ -77,6 +77,8 @@ enum {OPT_CREATE_UIO_DEV_NUM , #define OPT_VFIO_INTR "vfio-intr"OPT_VFIO_INTR_NUM , +#define OPT_LCORES "lcores" +OPT_LCORES_NUM , OPT_LONG_MAX_NUM };
lib/librte_eal/linuxapp/eal/Makefile +1 -0
@@ -95,6 +95,7 @@ CFLAGS_eal_hugepage_info.o := -D_GNU_SOURCE CFLAGS_eal_pci.o := -D_GNU_SOURCE CFLAGS_eal_pci_vfio.o := -D_GNU_SOURCE CFLAGS_eal_common_whitelist.o := -D_GNU_SOURCE +CFLAGS_eal_common_options.o := -D_GNU_SOURCE # workaround for a gcc bug with noreturn attribute # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603
Signature · 1 line
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 1422842559-13617-3-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
Re: [PATCH v4 02/17] eal: new eal option '--lcores' for cpu assignment
Hi,
Quoted 30 lines · Cunming Liang
It supports one new eal long option '--lcores' for EAL thread cpuset assignment.
The format pattern:
--lcores='lcores[@cpus]<,lcores[@cpus]>'lcores, cpus could be a single digit/range or a group. '(' and ')' are necessary if it's a group.
If not supply '@cpus', the value of cpus uses the same as lcores.e.g. '1,2@(5-7),(3-5)@(0,2),(0,6),7-8' means starting 9 EAL thread as below
lcore 0 runs on cpuset0x41(cpu 0,6)
lcore 1 runs on cpuset0x2(cpu 1)
lcore 2 runs on cpuset0xe0(cpu 5,6,7)
lcore 3,4,5 runs on cpuset0x5(cpu 0,2)
lcore 6 runs on cpuset0x41(cpu 0,6)
lcore 7 runs on cpuset0x80(cpu 7)
lcore 8 runs on cpuset0x100(cpu 8)Signed-off-by Cunming Liang <[email protected]>--- lib/librte_eal/common/eal_common_launch.c | 1 - lib/librte_eal/common/eal_common_options.c | 300 ++++++++++++++++++++++++++++- lib/librte_eal/common/eal_options.h | 2 + lib/librte_eal/linuxapp/eal/Makefile | 1 + 4 files changed, 299 insertions(+), 5 deletions(-)lib/librte_eal/common/eal_common_launch.c +0
-0
lib/librte_eal/common/eal_common_launch.c rte_eal_mp_wait_lcore(void) · line 120
Quoted 4 lines
lib/librte_eal/common/eal_common_launch.c +0
-1@@ -117,4 +117,3 @@ rte_eal_mp_wait_lcore(void)rte_eal_wait_lcore (lcore_id); } } -
This line should be removed from the patch.
Quoted 36 lines
lib/librte_eal/common/eal_common_options.c +7
-3@@ -45,6 +45,7 @@#include <rte_lcore.h> #include <rte_version.h> #include <rte_devargs.h> +#include <rte_memcpy.h> #include "eal_internal_cfg.h" #include "eal_options.h" @@ -85,6 +86,7 @@ eal_long_options[] = { {OPT_XEN_DOM0, 0, 0, OPT_XEN_DOM0_NUM}, {OPT_CREATE_UIO_DEV, 1, NULL, OPT_CREATE_UIO_DEV_NUM}, {OPT_VFIO_INTR, 1, NULL, OPT_VFIO_INTR_NUM}, + {OPT_LCORES, 1, 0, OPT_LCORES_NUM}, {0, 0, 0, 0} };@@ -255,9 +257,11 @@ eal_parse_corelist(const char *corelist)if (min ==RTE_MAX_LCORE ) min = idx;for (idx = min; idx <= max; idx++) { - cfg->lcore_role[idx] =ROLE_RTE ; - lcore_config[idx].core_index = count; - count++; +if (cfg->lcore_role[idx] !=ROLE_RTE ) { + cfg->lcore_role[idx] =ROLE_RTE ; + lcore_config[idx].core_index = count; + count++; + } } min =RTE_MAX_LCORE ; }else
lib/librte_eal/common/eal_common_options.c eal_parse_master_lcore(const char *arg) · line 305
Quoted 10 lines
lib/librte_eal/common/eal_common_options.c +7
-0@@ -292,6 +296,279 @@ eal_parse_master_lcore(const char *arg)return 0 ; } +/* +* Parse elem, the elem could be single number/range or '(' ')' group +* Within group elem, '-' used for a range seperator; +* ',' used for a single number. +*/ +static int +eal_parse_set (const char *input,uint16_t set[],unsigned num)
It's not very clear what elem is. Maybe it could be a bit reworded. What about naming the function "eal_parse_cpuset()" instead?
Quoted 95 lines
lib/librte_eal/common/eal_common_options.c +95
-0+{ +unsigned idx; +const char *str = input; +char *end =NULL ; +unsigned min, max; + +memset (set,0 , num *sizeof (uint16_t )); + +while (isblank (*str)) + str++; + +/* only digit or left bracket is qulify for start point */ +if ((!isdigit (*str) && *str !='(' ) || *str =='\0' ) +return -1 ; + +/* process single number or single range of number */ +if (*str !='(' ) { + errno =0 ; + idx =strtoul (str, &end,10 ); +if (errno || end ==NULL || idx >= num) +return -1 ; +else { +while (isblank (*end)) + end++; + + min = idx; + max = idx; +if (*end =='-' ) { +/* proccess single <number>-<number> */ + end++; +while (isblank (*end)) + end++; +if (!isdigit (*end)) +return -1 ; + + errno =0 ; + idx =strtoul (end, &end,10 ); +if (errno || end ==NULL || idx >= num) +return -1 ; + max = idx; +while (isblank (*end)) + end++; +if (*end !=',' && *end !='\0' ) +return -1 ; + } + +if (*end !=',' && *end !='\0' && + *end !='@' ) +return -1 ; + +for (idx =RTE_MIN (min, max); + idx <=RTE_MAX (min, max); idx++) + set[idx] =1 ; + +return end - input; + } + } + +/* process set within bracket */ + str++; +while (isblank (*str)) + str++; +if (*str =='\0' ) +return -1 ; + + min =RTE_MAX_LCORE ; + do { + + /* go ahead to the first digit */ + while (isblank(*str)) + str++; + if (!isdigit(*str)) + return -1; + + /* get the digit value */ + errno = 0; + idx = strtoul(str, &end, 10); + if (errno || end == NULL || idx >= num) + return -1; + + /* go ahead to separator '-',',' and ')' */ + while (isblank(*end)) + end++; + if (*end == '-') { + if (min == RTE_MAX_LCORE) + min = idx; + else /* avoid continuous '-' */ + return -1; + } else if ((*end == ',') || (*end == ')')) { + max = idx; + if (min == RTE_MAX_LCORE) + min = idx; + for (idx = RTE_MIN(min, max); + idx <= RTE_MAX(min, max); idx++) + set[idx] = 1;
lib/librte_eal/common/eal_common_options.c eal_parse_master_lcore(const char *arg) · line 410
Quoted 10 lines
lib/librte_eal/common/eal_common_options.c +10
-0+ + min =RTE_MAX_LCORE ; + }else +return -1 ; + + str = end +1 ; + }while (*end !='\0' && *end !=')' ); + +return str - input; +}
In the function above, there are some typos in the comments
seperator -> separator
qulify -> qualify
proccess -> process
lib/librte_eal/common/eal_common_options.c eal_parse_master_lcore(const char *arg) · line 415
Quoted 5 lines
lib/librte_eal/common/eal_common_options.c +5
-0+ +/* convert from set array to cpuset bitmap */ +static inline int +convert_to_cpuset (rte_cpuset_t *cpusetp, +uint16_t *set,unsigned num)
I don't think the function should be inlined
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH v4 02/17] eal: new eal option '--lcores' for cpu assignment
Quoted 40 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Monday, February 09, 2015 4:00 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 02/17] eal: new eal option '--lcores' for cpu assignmentHi,
Quoted 17 lines · Cunming Liang
It supports one new eal long option '--lcores' for EAL thread cpuset assignment.
The format pattern:
--lcores='lcores[@cpus]<,lcores[@cpus]>'lcores, cpus could be a single digit/range or a group. '(' and ')' are necessary if it's a group.
If not supply '@cpus', the value of cpus uses the same as lcores.e.g. '1,2@(5-7),(3-5)@(0,2),(0,6),7-8' means starting 9 EAL thread as below
lcore 0 runs on cpuset0x41(cpu 0,6)
lcore 1 runs on cpuset0x2(cpu 1)
lcore 2 runs on cpuset0xe0(cpu 5,6,7)
lcore 3,4,5 runs on cpuset0x5(cpu 0,2)
lcore 6 runs on cpuset0x41(cpu 0,6)
lcore 7 runs on cpuset0x80(cpu 7)
lcore 8 runs on cpuset0x100(cpu 8)Quoted 4 lines
Signed-off-by Cunming Liang <[email protected]>--- lib/librte_eal/common/eal_common_launch.c | 1 - lib/librte_eal/common/eal_common_options.c | 300++++++++++++++++++++++++++++-
Review
Quoted 5 lines
lib/librte_eal/common/eal_options.h | 2 + lib/librte_eal/linuxapp/eal/Makefile | 1 + 4 files changed, 299 insertions(+), 5 deletions(-)(quoted diff) +0
-0
b/lib/librte_eal/common/eal_common_launch.c
Quoted 11 lines
lib/librte_eal/common/eal_common_launch.c rte_eal_mp_wait_lcore(void) · line 120
Quoted 8 lines
index 599f83b..2d732b1 100644
lib/librte_eal/common/eal_common_launch.c +0
-1@@ -117,4 +117,3 @@ rte_eal_mp_wait_lcore(void)rte_eal_wait_lcore (lcore_id); } } -This line should be removed from the patch.
[LCM] Accept.
Quoted 42 lines
Review
Quoted 3 lines
(quoted diff) +0
-0
b/lib/librte_eal/common/eal_common_options.cQuoted 38 lines
index 67e02dc..29ebb6f 100644
lib/librte_eal/common/eal_common_options.c +7
-3@@ -45,6 +45,7 @@#include <rte_lcore.h> #include <rte_version.h> #include <rte_devargs.h> +#include <rte_memcpy.h> #include "eal_internal_cfg.h" #include "eal_options.h" @@ -85,6 +86,7 @@ eal_long_options[] = { {OPT_XEN_DOM0, 0, 0, OPT_XEN_DOM0_NUM}, {OPT_CREATE_UIO_DEV, 1, NULL, OPT_CREATE_UIO_DEV_NUM}, {OPT_VFIO_INTR, 1, NULL, OPT_VFIO_INTR_NUM}, + {OPT_LCORES, 1, 0, OPT_LCORES_NUM}, {0, 0, 0, 0} };@@ -255,9 +257,11 @@ eal_parse_corelist(const char *corelist)if (min ==RTE_MAX_LCORE ) min = idx;for (idx = min; idx <= max; idx++) { - cfg->lcore_role[idx] =ROLE_RTE ; - lcore_config[idx].core_index = count; - count++; +if (cfg->lcore_role[idx] !=ROLE_RTE ) { + cfg->lcore_role[idx] =ROLE_RTE ; + lcore_config[idx].core_index = count; + count++; + } } min =RTE_MAX_LCORE ; }else @@ -292,6 +296,279 @@ eal_parse_master_lcore(const char *arg)return 0 ; }
Quoted 10 lines
lib/librte_eal/common/eal_common_options.c eal_parse_master_lcore(const char *arg) · line 305
Quoted 7 lines
lib/librte_eal/common/eal_common_options.c +7
-0+/* +* Parse elem, the elem could be single number/range or '(' ')' group +* Within group elem, '-' used for a range seperator; +* ',' used for a single number. +*/ +static int +eal_parse_set (const char *input,uint16_t set[],unsigned num)It's not very clear what elem is. Maybe it could be a bit reworded. What about naming the function "
eal_parse_cpuset()" instead?
[LCM] As it not only parse cpuset but also used for lcore set, so 'eal_parse_cpuset' is not accurate. The set/elem here identify for a single number (e.g. 1), a number range (e.g. 4-6) or a group (e.g. (3,4-8,9) ). I'll reword the comment for better understand. Thanks.
Quoted 104 lines
Quoted 104 lines
lib/librte_eal/common/eal_common_options.c +102
-0+{ +unsigned idx; +const char *str = input; +char *end =NULL ; +unsigned min, max; + +memset (set,0 , num *sizeof (uint16_t )); + +while (isblank (*str)) + str++; + +/* only digit or left bracket is qulify for start point */ +if ((!isdigit (*str) && *str !='(' ) || *str =='\0' ) +return -1 ; + +/* process single number or single range of number */ +if (*str !='(' ) { + errno =0 ; + idx =strtoul (str, &end,10 ); +if (errno || end ==NULL || idx >= num) +return -1 ; +else { +while (isblank (*end)) + end++; + + min = idx; + max = idx; +if (*end =='-' ) { +/* proccess single <number>-<number> */ + end++; +while (isblank (*end)) + end++; +if (!isdigit (*end)) +return -1 ; + + errno =0 ; + idx =strtoul (end, &end,10 ); +if (errno || end ==NULL || idx >= num) +return -1 ; + max = idx; +while (isblank (*end)) + end++; +if (*end !=',' && *end !='\0' ) +return -1 ; + } + +if (*end !=',' && *end !='\0' && + *end !='@' ) +return -1 ; + +for (idx =RTE_MIN (min, max); + idx <=RTE_MAX (min, max); idx++) + set[idx] =1 ; + +return end - input; + } + } + +/* process set within bracket */ + str++; +while (isblank (*str)) + str++; +if (*str =='\0' ) +return -1 ; + + min =RTE_MAX_LCORE ; +do { + +/* go ahead to the first digit */ +while (isblank (*str)) + str++; +if (!isdigit (*str)) +return -1 ; + +/* get the digit value */ + errno =0 ; + idx =strtoul (str, &end,10 ); +if (errno || end ==NULL || idx >= num) +return -1 ; + +/* go ahead to separator '-',',' and ')' */ +while (isblank (*end)) + end++; +if (*end =='-' ) { +if (min ==RTE_MAX_LCORE ) + min = idx; +else /* avoid continuous '-' */ +return -1 ; + }else if ((*end ==',' ) || (*end ==')' )) { + max = idx; +if (min ==RTE_MAX_LCORE ) + min = idx; +for (idx =RTE_MIN (min, max); + idx <=RTE_MAX (min, max); idx++) + set[idx] =1 ; + + min =RTE_MAX_LCORE ; + }else +return -1 ; + + str = end +1 ; + }while (*end !='\0' && *end !=')' );
Quoted 8 lines
lib/librte_eal/common/eal_common_options.c eal_parse_master_lcore(const char *arg) · line 410
Quoted 3 lines
lib/librte_eal/common/eal_common_options.c +3
-0+ +return str - input; +}In the function above, there are some typos in the comments
seperator -> separator
qulify -> qualify
proccess -> process
[LCM] Sorry for that, will fix it.
Quoted 8 lines
lib/librte_eal/common/eal_common_options.c eal_parse_master_lcore(const char *arg) · line 415
Quoted 6 lines
lib/librte_eal/common/eal_common_options.c +5
-0+ +/* convert from set array to cpuset bitmap */ +static inline int +convert_to_cpuset (rte_cpuset_t *cpusetp, +uint16_t *set,unsigned num)I don't think the function should be inlined
[LCM] accept.
Quoted 5 lines
Regards,
Olivier
PATCH v4 3/17 eal: fix wrong strnlen() return value in 32bit icc
In reply to a message not in the archiveThe problem is that strnlen() here may return invalid value with 32bit icc. (actually it returns it’s second parameter,e.g: sysconf(_SC_ARG_MAX)).
It starts to manifest hwen max_len parameter is > 2M and using icc –m32 –O2 (or above).
--- lib/librte_eal/common/eal_common_options.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
lib/librte_eal/common/eal_common_options.c +2 -2
@@ -227,7 +227,7 @@ eal_parse_corelist(const char *corelist)/* Remove all blank characters ahead and after */ while (isblank (*corelist)) corelist++; - i =strnlen (corelist,sysconf (_SC_ARG_MAX)); + i =strnlen (corelist,PATH_MAX );while ((i >0 ) &&isblank (corelist[i -1 ])) i--;
@@ -469,7 +469,7 @@ eal_parse_lcores(const char *lcores)/* Remove all blank characters ahead and after */ while (isblank (*lcores)) lcores++; - i =strnlen (lcores,sysconf (_SC_ARG_MAX)); + i =strnlen (lcores,PATH_MAX );while ((i >0 ) &&isblank (lcores[i -1 ])) i--;
Signature · 1 line
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 1422842559-13617-4-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
Re: [PATCH v4 03/17] eal: fix wrong strnlen() return value in 32bit icc
Hi,
Quoted 25 lines · Cunming Liang
The problem is that
strnlen()here may return invalid value with 32bit icc. (actually it returns it’s second parameter,e.g:sysconf(_SC_ARG_MAX)).
It starts to manifest hwenmax_lenparameter is > 2M and using icc –m32 –O2 (or above).Suggested-by Konstantin Ananyev <[email protected]>Signed-off-by Cunming Liang <[email protected]>--- lib/librte_eal/common/eal_common_options.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)lib/librte_eal/common/eal_common_options.c +1
-1@@ -227,7 +227,7 @@ eal_parse_corelist(const char *corelist)/* Remove all blank characters ahead and after */ while (isblank (*corelist)) corelist++; - i =strnlen (corelist,sysconf (_SC_ARG_MAX)); + i =strnlen (corelist,PATH_MAX );while ((i >0 ) &&isblank (corelist[i -1 ])) i--;
lib/librte_eal/common/eal_common_options.c eal_parse_lcores(const char *lcores) · line 475
Quoted 8 lines
lib/librte_eal/common/eal_common_options.c +1
-1@@ -469,7 +469,7 @@ eal_parse_lcores(const char *lcores)/* Remove all blank characters ahead and after */ while (isblank (*lcores)) lcores++; - i =strnlen (lcores,sysconf (_SC_ARG_MAX)); + i =strnlen (lcores,PATH_MAX );while ((i >0 ) &&isblank (lcores[i -1 ])) i--;
I think PATH_MAX is not equivalent to _SC_ARG_MAX.
But the main question is: why do we need to use strnlen() here instead of strlen? We can expect that argv[] pointers are always nul-terminated. Replacing them by strlen() would probably also solve the icc issue.
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH v4 03/17] eal: fix wrong strnlen() return value in 32bit icc
Quoted 50 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Monday, February 09, 2015 4:00 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 03/17] eal: fix wrongstrnlen()return value in 32bit iccHi,
Quoted 3 lines · Cunming Liang
The problem is that
strnlen()here may return invalid value with 32bit icc. (actually it returns it’s second parameter,e.g:sysconf(_SC_ARG_MAX)). It starts to manifest hwenmax_lenparameter is > 2M and using icc –m32 –O2(or above).
Review
Quoted 8 lines
Suggested-by Konstantin Ananyev <[email protected]>Signed-off-by Cunming Liang <[email protected]>--- lib/librte_eal/common/eal_common_options.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)(quoted diff) +0
-0
b/lib/librte_eal/common/eal_common_options.cQuoted 14 lines
index 29ebb6f..22d5d37 100644
lib/librte_eal/common/eal_common_options.c +1
-1@@ -227,7 +227,7 @@ eal_parse_corelist(const char *corelist)/* Remove all blank characters ahead and after */ while (isblank (*corelist)) corelist++; - i =strnlen (corelist,sysconf (_SC_ARG_MAX)); + i =strnlen (corelist,PATH_MAX );while ((i >0 ) &&isblank (corelist[i -1 ])) i--;lib/librte_eal/common/eal_common_options.c eal_parse_lcores(const char *lcores) · line 475
Quoted 8 lines
lib/librte_eal/common/eal_common_options.c +1
-1@@ -469,7 +469,7 @@ eal_parse_lcores(const char *lcores)/* Remove all blank characters ahead and after */ while (isblank (*lcores)) lcores++; - i =strnlen (lcores,sysconf (_SC_ARG_MAX)); + i =strnlen (lcores,PATH_MAX );while ((i >0 ) &&isblank (lcores[i -1 ])) i--;I think
PATH_MAXis not equivalent to_SC_ARG_MAX.
Quoted 3 lines
But the main question is: why do we need to use
strnlen()here instead of strlen? We can expect that argv[] pointers are always nul-terminated. Replacing them bystrlen()would probably also solve the icc issue.
[LCM] You're right, here strlen() also solve icc issue and no risk for argv[].
But follows practice suggestion, keeping using those with 'n' function in DPDK is not bad. There's additional two reason to keep strnlen and PATH_MAX.
1. PATH_MAX is defined as 4096 which is enough as our input. It doesn't matter to be _SC_ARG_MAX or not.
2. strnlen and PATH_MAX already used in eal_parse_coremask, to keep the style consistent in '-l' and '--lcores'.
Quoted 3 lines
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): D0158A423229094DA7ABF71CF2FA0DA3118D7D34-E2R4CRU6q/[email protected]
Re: [PATCH v4 03/17] eal: fix wrong strnlen() return value in 32bit icc
Hi,
Quoted 13 lines · Cunming
Quoted 13 lines
lib/librte_eal/common/eal_common_options.c eal_parse_lcores(const char *lcores) · line 475
Quoted 10 lines
lib/librte_eal/common/eal_common_options.c +1
-1@@ -469,7 +469,7 @@ eal_parse_lcores(const char *lcores)/* Remove all blank characters ahead and after */ while (isblank (*lcores)) lcores++; - i =strnlen (lcores,sysconf (_SC_ARG_MAX)); + i =strnlen (lcores,PATH_MAX );while ((i >0 ) &&isblank (lcores[i -1 ])) i--;I think
PATH_MAXis not equivalent to_SC_ARG_MAX.
Quoted 7 lines
Quoted 3 lines
But the main question is: why do we need to use
strnlen()here instead of strlen? We can expect that argv[] pointers are always nul-terminated. Replacing them bystrlen()would probably also solve the icc issue.[LCM] You're right, here
strlen()also solve icc issue and no risk for argv[].
But follows practice suggestion, keeping using those with 'n' function in DPDK is not bad. There's additional two reason to keep strnlen andPATH_MAX.
1.PATH_MAXis defined as 4096 which is enough as our input. It doesn't matter to be_SC_ARG_MAXor not.
PATH_MAX is 4096 but it's not related to the maximum argument length.
Quoted 1 line
2. strnlen and
PATH_MAXalready used ineal_parse_coremask, to keep the style consistent in '-l' and '--lcores'.
I don't think it's a valid argument.
What is the problem of using strlen()? It looks it solves all the issues. Using strlen on valid strings is not a security issue.
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH v4 03/17] eal: fix wrong strnlen() return value in 32bit icc
Quoted 37 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Tuesday, February 10, 2015 1:13 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 03/17] eal: fix wrongstrnlen()return value in 32bit iccHi,
Quoted 13 lines · Cunming
Quoted 13 lines
Review eal_parse_lcores(const char *lcores) · line 475
Quoted 10 lines
(quoted diff) +1
-1@@ -469,7 +469,7 @@ eal_parse_lcores(const char *lcores) /* Remove all blank characters ahead and after */ while (isblank(*lcores)) lcores++; - i = strnlen(lcores, sysconf(_SC_ARG_MAX)); + i = strnlen(lcores, PATH_MAX); while ((i > 0) && isblank(lcores[i - 1])) i--;I think
PATH_MAXis not equivalent to_SC_ARG_MAX.Quoted 5 lines
Quoted 3 lines
But the main question is: why do we need to use
strnlen()here instead of strlen? We can expect that argv[] pointers are always nul-terminated. Replacing them bystrlen()would probably also solve the icc issue.[LCM] You're right, here
strlen()also solve icc issue and no risk for argv[]. But follows practice suggestion, keeping using those with 'n' function in DPDK isnot bad.
Quoted 2 lines
There's additional two reason to keep strnlen and
PATH_MAX.
1.PATH_MAXis defined as 4096 which is enough as our input. It doesn't matterto be
_SC_ARG_MAXor not.
PATH_MAXis 4096 but it's not related to the maximum argument length.
Quoted 7 lines
Quoted 1 line
2. strnlen and
PATH_MAXalready used ineal_parse_coremask, to keep thestyle consistent in '-l' and '--lcores'.
I don't think it's a valid argument.
What is the problem of using
strlen()? It looks it solves all the issues. Using strlen on valid strings is not a security issue.
[LCM] All right, I buy in your point.
Quoted 4 lines
Regards,
Olivier
PATCH v4 4/17 eal: add support parsing socket_id from cpuset
In reply to a message not in the archiveIt returns the socket_id if all cpus in the cpuset belongs to the same NUMA node, otherwise it will return SOCKET_ID_ANY.
--- lib/librte_eal/bsdapp/eal/eal_lcore.c | 7 +++++ lib/librte_eal/common/eal_thread.h | 52 +++++++++++++++++++++++++++++++++ lib/librte_eal/linuxapp/eal/eal_lcore.c | 7 +++++ 3 files changed, 66 insertions(+)
lib/librte_eal/bsdapp/eal/eal_lcore.c +7 -0
@@ -41,6 +41,7 @@#include <rte_debug.h> #include "eal_private.h" +#include "eal_thread.h" /* No topology information available on FreeBSD including NUMA info */ #define cpu_core_id (X ) 0
@@ -112,3 +113,9 @@ rte_eal_cpu_init(void)return 0 ; } + +unsigned +eal_cpu_socket_id (__rte_unused unsigned cpu_id) +{ +return cpu_socket_id (cpu_id); +}
lib/librte_eal/common/eal_thread.h +52 -0
62 lines not on this page · raw message
lib/librte_eal/linuxapp/eal/eal_lcore.c +7 -0
@@ -45,6 +45,7 @@#include "eal_private.h" #include "eal_filesystem.h" +#include "eal_thread.h" #define SYS_CPU_DIR "/sys/devices/system/cpu/cpu%u"#define CORE_ID_FILE "topology/core_id"
@@ -197,3 +198,9 @@ rte_eal_cpu_init(void)return 0 ; } + +unsigned +eal_cpu_socket_id (unsigned cpu_id) +{ +return cpu_socket_id (cpu_id); +}
Signature · 1 line
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 1422842559-13617-5-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
Re: [PATCH v4 04/17] eal: add support parsing socket_id from cpuset
Hi,
Quoted 51 lines · Cunming Liang
It returns the
socket_idif all cpus in the cpuset belongs to the same NUMA node, otherwise it will returnSOCKET_ID_ANY.Signed-off-by Cunming Liang <[email protected]>--- lib/librte_eal/bsdapp/eal/eal_lcore.c | 7 +++++ lib/librte_eal/common/eal_thread.h | 52 +++++++++++++++++++++++++++++++++ lib/librte_eal/linuxapp/eal/eal_lcore.c | 7 +++++ 3 files changed, 66 insertions(+)lib/librte_eal/bsdapp/eal/eal_lcore.c +7
-0@@ -41,6 +41,7 @@#include <rte_debug.h> #include "eal_private.h" +#include "eal_thread.h" /* No topology information available on FreeBSD including NUMA info */ #define cpu_core_id (X ) 0@@ -112,3 +113,9 @@ rte_eal_cpu_init(void)return 0 ; } + +unsigned +eal_cpu_socket_id (__rte_unused unsigned cpu_id) +{ +return cpu_socket_id (cpu_id); +}lib/librte_eal/common/eal_thread.h +4
-0@@ -34,6 +34,10 @@#ifndef EAL_THREAD_H #define EAL_THREAD_H +#include <sched.h> + +#include <rte_debug.h> +/** * basic loop of thread, called for each thread by eal_init(). * @@ -50,4 +54,52 @@ __attribute__((noreturn)) void *eal_thread_loop(void *arg);*/ void eal_thread_init_master (unsigned lcore_id);
lib/librte_eal/common/eal_thread.h __attribute__((noreturn)) void *eal_thread_loop(void *arg); · line 66
Quoted 10 lines
lib/librte_eal/common/eal_thread.h +10
-0+/** +* Get the NUMA socket id from cpu id. +* This function is private to EAL. +* +* @param cpu_id +* The logical process id. +* @return +* socket_id or SOCKET_ID_ANY +*/ +unsigned eal_cpu_socket_id (unsigned cpu_id);
Wouldn't it be better to rename the existing function cpu_socket_id() in eal_cpu_socket_id() and export it in eal_thread.h?
In case of bsd where cpu_socket_id() is implemented using a #define, a new function should be created returning 0.
Quoted 9 lines
lib/librte_eal/common/eal_thread.h +9
-0+ +/** +* Get the NUMA socket id from cpuset. +* This function is private to EAL. +* +* @param cpusetp +* The point to a valid cpu set. +* @return +* socket_id or SOCKET_ID_ANY
lib/librte_eal/common/eal_thread.h __attribute__((noreturn)) void *eal_thread_loop(void *arg); · line 85
Quoted 10 lines
lib/librte_eal/common/eal_thread.h +10
-0+*/ +static inline int +eal_cpuset_socket_id (rte_cpuset_t *cpusetp) +{ +unsigned cpu =0 ; +int socket_id =SOCKET_ID_ANY ; +int sid; + +if (cpusetp ==NULL ) +return SOCKET_ID_ANY ;
SOCKET_ID_ANY is not defined, maybe <rte_lcore.h> should be included somewhere.
Quoted 8 lines
lib/librte_eal/common/eal_thread.h +8
-0+ + do { + if (!CPU_ISSET(cpu, cpusetp)) + continue; + + if (socket_id == SOCKET_ID_ANY) + socket_id = eal_cpu_socket_id(cpu); +
lib/librte_eal/common/eal_thread.h __attribute__((noreturn)) void *eal_thread_loop(void *arg); · line 103
Quoted 10 lines
lib/librte_eal/common/eal_thread.h +10
-0+ sid =eal_cpu_socket_id (cpu); +if (socket_id != sid) { + socket_id =SOCKET_ID_ANY ; +break ; + } + + }while (++cpu <RTE_MAX_LCORE ); + +return socket_id; +}
I don't think this function should be inlined.
As this function is not used, it could be interesting for reviewers to understand when
Quoted 25 lines
lib/librte_eal/common/eal_thread.h +1
-0+#endif /* EAL_THREAD_H */ lib/librte_eal/linuxapp/eal/eal_lcore.c +7
-0@@ -45,6 +45,7 @@#include "eal_private.h" #include "eal_filesystem.h" +#include "eal_thread.h" #define SYS_CPU_DIR "/sys/devices/system/cpu/cpu%u"#define CORE_ID_FILE "topology/core_id"@@ -197,3 +198,9 @@ rte_eal_cpu_init(void)return 0 ; } + +unsigned +eal_cpu_socket_id (unsigned cpu_id) +{ +return cpu_socket_id (cpu_id); +}
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH v4 04/17] eal: add support parsing socket_id from cpuset
Quoted 81 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Monday, February 09, 2015 4:00 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 04/17] eal: add support parsingsocket_idfrom cpusetHi,
Quoted 7 lines · Cunming Liang
It returns the
socket_idif all cpus in the cpuset belongs to the same NUMA node, otherwise it will returnSOCKET_ID_ANY.Signed-off-by Cunming Liang <[email protected]>--- lib/librte_eal/bsdapp/eal/eal_lcore.c | 7 +++++ lib/librte_eal/common/eal_thread.h | 52+++++++++++++++++++++++++++++++++
Review
Quoted 4 lines
lib/librte_eal/linuxapp/eal/eal_lcore.c | 7 +++++ 3 files changed, 66 insertions(+)(quoted diff) +0
-0
b/lib/librte_eal/bsdapp/eal/eal_lcore.cReview
Quoted 22 lines
index 72f8ac2..162fb4f 100644
lib/librte_eal/bsdapp/eal/eal_lcore.c +7
-0@@ -41,6 +41,7 @@#include <rte_debug.h> #include "eal_private.h" +#include "eal_thread.h" /* No topology information available on FreeBSD including NUMA info */ #define cpu_core_id (X ) 0@@ -112,3 +113,9 @@ rte_eal_cpu_init(void)return 0 ; } + +unsigned +eal_cpu_socket_id (__rte_unused unsigned cpu_id) +{ +return cpu_socket_id (cpu_id); +}(quoted diff) +0
-0
b/lib/librte_eal/common/eal_thread.hlib/librte_eal/common/eal_thread.h __attribute__((noreturn)) void *eal_thread_loop(void
Quoted 15 lines
index b53b84d..a25ee86 100644
lib/librte_eal/common/eal_thread.h +4
-0@@ -34,6 +34,10 @@#ifndef EAL_THREAD_H #define EAL_THREAD_H +#include <sched.h> + +#include <rte_debug.h> +/** * basic loop of thread, called for each thread by eal_init(). * @@ -50,4 +54,52 @@ __attribute__((noreturn)) void *eal_thread_loop(void*arg);
lib/librte_eal/common/eal_thread.h __attribute__((noreturn)) void *eal_thread_loop(void *arg); · line 66
Quoted 13 lines
lib/librte_eal/common/eal_thread.h +10
-0*/ void eal_thread_init_master (unsigned lcore_id); +/** +* Get the NUMA socket id from cpu id. +* This function is private to EAL. +* +* @param cpu_id +* The logical process id. +* @return +* socket_id or SOCKET_ID_ANY +*/ +unsigned eal_cpu_socket_id (unsigned cpu_id);Wouldn't it be better to rename the existing function
cpu_socket_id()ineal_cpu_socket_id()and export it ineal_thread.h?
Quoted 2 lines
In case of bsd where
cpu_socket_id()is implemented using a #define, a new function should be created returning 0.
[LCM] In eal_lcore.c, the cpu_socket_id()/cpu_core_id() defined as static and only used in rte_eal_cpu_init(). I suppose the purpose of origin design is to make the sysfs parsing only visible in the file.
No matter remove the 'static' prefix of cpu_core_id() or add a new wrap eal_cpu_socket_id(), it results in a new extern EAL API. So I prefer not change the visibility of the origin static function but have one as extern interface.
Quoted 16 lines
Quoted 16 lines
lib/librte_eal/common/eal_thread.h +14
-0+ +/** +* Get the NUMA socket id from cpuset. +* This function is private to EAL. +* +* @param cpusetp +* The point to a valid cpu set. +* @return +* socket_id or SOCKET_ID_ANY +*/ +static inline int +eal_cpuset_socket_id (rte_cpuset_t *cpusetp) +{ +unsigned cpu =0 ;
Quoted 8 lines
lib/librte_eal/common/eal_thread.h __attribute__((noreturn)) void *eal_thread_loop(void *arg); · line 85
Quoted 5 lines
lib/librte_eal/common/eal_thread.h +5
-0+int socket_id =SOCKET_ID_ANY ; +int sid; + +if (cpusetp ==NULL ) +return SOCKET_ID_ANY ;
SOCKET_ID_ANYis not defined, maybe <rte_lcore.h> should be included somewhere.
[LCM] Agree with you, eal_cpuset_socket_id() can move into eal_common_thread.c. And add rte_memory.h for SOCKET_ID_ANY reference.
Quoted 23 lines
Quoted 9 lines
lib/librte_eal/common/eal_thread.h +8
-0+ + do { + if (!CPU_ISSET(cpu, cpusetp)) + continue; + + if (socket_id == SOCKET_ID_ANY) + socket_id = eal_cpu_socket_id(cpu); +lib/librte_eal/common/eal_thread.h __attribute__((noreturn)) void *eal_thread_loop(void *arg); · line 103
Quoted 10 lines
lib/librte_eal/common/eal_thread.h +10
-0+ sid =eal_cpu_socket_id (cpu); +if (socket_id != sid) { + socket_id =SOCKET_ID_ANY ; +break ; + } + + }while (++cpu <RTE_MAX_LCORE ); + +return socket_id; +}I don't think this function should be inlined.
Quoted 2 lines
As this function is not used, it could be interesting for reviewers to understand when
[LCM] It's used in eal_thread_set_affinity() of eal_thread.c.
Quoted 27 lines
Review
Quoted 4 lines
lib/librte_eal/common/eal_thread.h +1
-0+#endif /* EAL_THREAD_H */ (quoted diff) +0
-0
b/lib/librte_eal/linuxapp/eal/eal_lcore.cQuoted 22 lines
index 29615f8..922af6d 100644
lib/librte_eal/linuxapp/eal/eal_lcore.c +7
-0@@ -45,6 +45,7 @@#include "eal_private.h" #include "eal_filesystem.h" +#include "eal_thread.h" #define SYS_CPU_DIR "/sys/devices/system/cpu/cpu%u"#define CORE_ID_FILE "topology/core_id"@@ -197,3 +198,9 @@ rte_eal_cpu_init(void)return 0 ; } + +unsigned +eal_cpu_socket_id (unsigned cpu_id) +{ +return cpu_socket_id (cpu_id); +}
A message that is not in the archive (sent off-list or not yet received): D0158A423229094DA7ABF71CF2FA0DA3118D7D72-E2R4CRU6q/[email protected]
Re: [PATCH v4 04/17] eal: add support parsing socket_id from cpuset
Hi,
Quoted 19 lines · Cunming
Quoted 19 lines
Review __attribute__((noreturn)) void *eal_thread_loop(void
Quoted 1 line
(quoted diff) +0
-0@@ -50,4 +54,52 @@ __attribute__((noreturn)) void *eal_thread_loop(void*arg);
lib/librte_eal/common/eal_thread.h __attribute__((noreturn)) void *eal_thread_loop(void *arg); · line 66
Quoted 13 lines
lib/librte_eal/common/eal_thread.h +10
-0*/ void eal_thread_init_master (unsigned lcore_id); +/** +* Get the NUMA socket id from cpu id. +* This function is private to EAL. +* +* @param cpu_id +* The logical process id. +* @return +* socket_id or SOCKET_ID_ANY +*/ +unsigned eal_cpu_socket_id (unsigned cpu_id);Wouldn't it be better to rename the existing function
cpu_socket_id()ineal_cpu_socket_id()and export it ineal_thread.h?
Quoted 6 lines
Quoted 2 lines
In case of bsd where
cpu_socket_id()is implemented using a #define, a new function should be created returning 0.[LCM] In
eal_lcore.c, the cpu_socket_id()/cpu_core_id() defined as static and only used inrte_eal_cpu_init(). I suppose the purpose of origin design is to make the sysfs parsing only visible in the file.
No matter remove the 'static' prefix ofcpu_core_id()or add a new wrapeal_cpu_socket_id(), it results in a new extern EAL API. So I prefer not change the visibility of the origin static function but have one as extern interface.
Yes, but I don't see what is the advantage of using a wrapper. If there is no advantage, I think the one with the less code is better.
Quoted 31 lines
Quoted 31 lines
Quoted 17 lines
lib/librte_eal/common/eal_thread.h +17
-0+static inline int +eal_cpuset_socket_id (rte_cpuset_t *cpusetp) +{ +unsigned cpu =0 ; +int socket_id =SOCKET_ID_ANY ; +int sid; + +if (cpusetp ==NULL ) +return SOCKET_ID_ANY ; + + do { + if (!CPU_ISSET(cpu, cpusetp)) + continue; + + if (socket_id == SOCKET_ID_ANY) + socket_id = eal_cpu_socket_id(cpu); +lib/librte_eal/common/eal_thread.h __attribute__((noreturn)) void *eal_thread_loop(void *arg); · line 103
Quoted 10 lines
lib/librte_eal/common/eal_thread.h +10
-0+ sid =eal_cpu_socket_id (cpu); +if (socket_id != sid) { + socket_id =SOCKET_ID_ANY ; +break ; + } + + }while (++cpu <RTE_MAX_LCORE ); + +return socket_id; +}I don't think this function should be inlined.
Quoted 3 lines
Quoted 2 lines
As this function is not used, it could be interesting for reviewers to understand when
[LCM] It's used in
eal_thread_set_affinity()ofeal_thread.c.
As it's not visible in the patch, could you add an explanation in the commit log?
PATCH v4 5/17 eal: new TLS definition and API declaration
In reply to a message not in the archive1. add two TLS *_socket_id* and *_cpuset*
2. add two external API rte_thread_set/get_affinity
3. add one internal API eal_thread_dump_affinity
--- lib/librte_eal/bsdapp/eal/eal_thread.c | 2 ++ lib/librte_eal/common/eal_thread.h | 14 ++++++++++++++ lib/librte_eal/common/include/rte_lcore.h | 29 +++++++++++++++++++++++++++-- lib/librte_eal/linuxapp/eal/eal_thread.c | 2 ++ 4 files changed, 45 insertions(+), 2 deletions(-)
lib/librte_eal/bsdapp/eal/eal_thread.c +2 -0
@@ -56,6 +56,8 @@#include "eal_thread.h" RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); +RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);/* * Send a message to a slave lcore identified by slave_id to call a
lib/librte_eal/common/eal_thread.h +14 -0
@@ -102,4 +102,18 @@ eal_cpuset_socket_id(rte_cpuset_t *cpusetp) return socket_id; } +/** +* Dump the current pthread cpuset. +* This function is private to EAL. +* +* @param str +* The string buffer the cpuset will dump to. +* @param size +* The string buffer size. +*/ +#define CPU_STR_LEN 256 +void +eal_thread_dump_affinity (char str[],unsigned size); + +#endif /* EAL_THREAD_H */
lib/librte_eal/common/include/rte_lcore.h +27 -2
@@ -43,6 +43,7 @@#include <rte_per_lcore.h> #include <rte_eal.h> #include <rte_launch.h> +#include <rte_memory.h> #ifdef __cplusplusextern "C" {
@@ -80,7 +81,9 @@ struct lcore_config {*/ extern struct lcore_config lcore_config[RTE_MAX_LCORE ]; -RTE_DECLARE_PER_LCORE (unsigned, _lcore_id);/**< Per core "core id". */ +RTE_DECLARE_PER_LCORE (unsigned, _lcore_id);/**< Per thread "lcore id". */ +RTE_DECLARE_PER_LCORE (unsigned, _socket_id);/**< Per thread "socket id". */ +RTE_DECLARE_PER_LCORE (rte_cpuset_t, _cpuset);/**< Per thread "cpuset". */ /** * Return the ID of the execution unit we are running on.
@@ -146,7 +149,7 @@ rte_lcore_index(int lcore_id)static inline unsigned rte_socket_id (void ) { -return lcore_config[rte_lcore_id ()].socket_id; +return RTE_PER_LCORE (_socket_id); }/**
@@ -229,6 +232,28 @@ rte_get_next_lcore(unsigned i, int skip_master, int wrap) i<RTE_MAX_LCORE ; \ i = rte_get_next_lcore(i, 1, 0)) +/** +* Set core affinity of the current thread. +* Support both EAL and none-EAL thread and update TLS. +* +* @param cpusetp +* Point to cpu_set_t for setting current thread affinity. +* @return +* On success, return 0; otherwise return -1; +*/ +int rte_thread_set_affinity (rte_cpuset_t *cpusetp); + +/** +* Get core affinity of the current thread. +* +* @param cpusetp +* Point to cpu_set_t for getting current thread cpu affinity. +* @return +* On success, return 0; otherwise return -1; +*/ +int rte_thread_get_affinity (rte_cpuset_t *cpusetp); + + #ifdef __cplusplus }#endif
lib/librte_eal/linuxapp/eal/eal_thread.c +2 -0
@@ -56,6 +56,8 @@#include "eal_thread.h" RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); +RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);/* * Send a message to a slave lcore identified by slave_id to call a
Signature · 1 line
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 1422842559-13617-6-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
Re: [PATCH v4 05/17] eal: new TLS definition and API declaration
Hi,
Quoted 3 lines · Cunming Liang
1. add two TLS *_socket_id* and *_cpuset*
2. add two external API rte_thread_set/get_affinity
3. add one internal APIeal_thread_dump_affinity
To me, it's a bit strage to add an API withtout the associated code. Maybe you have a good reason to do that, but I think in this case it should be explained in the commit log.
Quoted 32 lines
Signed-off-by Cunming Liang <[email protected]>--- lib/librte_eal/bsdapp/eal/eal_thread.c | 2 ++ lib/librte_eal/common/eal_thread.h | 14 ++++++++++++++ lib/librte_eal/common/include/rte_lcore.h | 29 +++++++++++++++++++++++++++-- lib/librte_eal/linuxapp/eal/eal_thread.c | 2 ++ 4 files changed, 45 insertions(+), 2 deletions(-)lib/librte_eal/bsdapp/eal/eal_thread.c +2
-0@@ -56,6 +56,8 @@#include "eal_thread.h" RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); +RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);/* * Send a message to a slave lcore identified by slave_id to call a lib/librte_eal/common/eal_thread.h +2
-0@@ -102,4 +102,18 @@ eal_cpuset_socket_id(rte_cpuset_t *cpusetp)return socket_id; } +/** +* Dump the current pthread cpuset.
lib/librte_eal/common/eal_thread.h eal_cpuset_socket_id(rte_cpuset_t *cpusetp) · line 116
Quoted 10 lines
lib/librte_eal/common/eal_thread.h +10
-0+* This function is private to EAL. +* +* @param str +* The string buffer the cpuset will dump to. +* @param size +* The string buffer size. +*/ +#define CPU_STR_LEN 256 +void +eal_thread_dump_affinity (char str[],unsigned size);
Although it's equivalent for function arguments, I think "char *str" is usually preferred over "char str[]". See for instance in snprintf() or fgets().
What is the purpose of CPU_STR_LEN?
What occurs if the size of the dump is greater than the size of the given buffer? Is the string truncated? Is there a \0 at the end? This should be described in the API comments. Maybe adding a return value could help the user to determine if the string was truncated.
Quoted 27 lines
lib/librte_eal/common/eal_thread.h +2
-0+ +#endif /* EAL_THREAD_H */ lib/librte_eal/common/include/rte_lcore.h +4
-1@@ -43,6 +43,7 @@#include <rte_per_lcore.h> #include <rte_eal.h> #include <rte_launch.h> +#include <rte_memory.h> #ifdef __cplusplusextern "C" {@@ -80,7 +81,9 @@ struct lcore_config {*/ extern struct lcore_config lcore_config[RTE_MAX_LCORE ]; -RTE_DECLARE_PER_LCORE (unsigned, _lcore_id);/**< Per core "core id". */ +RTE_DECLARE_PER_LCORE (unsigned, _lcore_id);/**< Per thread "lcore id". */ +RTE_DECLARE_PER_LCORE (unsigned, _socket_id);/**< Per thread "socket id". */ +RTE_DECLARE_PER_LCORE (rte_cpuset_t, _cpuset);/**< Per thread "cpuset". */ /** * Return the ID of the execution unit we are running on.
lib/librte_eal/common/include/rte_lcore.h rte_lcore_index(int lcore_id) · line 153
Quoted 6 lines
lib/librte_eal/common/include/rte_lcore.h +1
-1@@ -146,7 +149,7 @@ rte_lcore_index(int lcore_id)static inline unsigned rte_socket_id (void ) { -return lcore_config[rte_lcore_id ()].socket_id; +return RTE_PER_LCORE (_socket_id); }
I don't see where the _socket_id variable is assigned. I think there is probably an issue with the splitting of the patches.
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH v4 05/17] eal: new TLS definition and API declaration
Quoted 11 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Monday, February 09, 2015 4:00 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 05/17] eal: new TLS definition and API declarationHi,
Quoted 8 lines
Quoted 3 lines · Cunming Liang
1. add two TLS *_socket_id* and *_cpuset*
2. add two external API rte_thread_set/get_affinity
3. add one internal APIeal_thread_dump_affinityTo me, it's a bit strage to add an API withtout the associated code. Maybe you have a good reason to do that, but I think in this case it should be explained in the commit log.
[LCM] Accept.
Quoted 34 lines
Quoted 7 lines
Signed-off-by Cunming Liang <[email protected]>--- lib/librte_eal/bsdapp/eal/eal_thread.c | 2 ++ lib/librte_eal/common/eal_thread.h | 14 ++++++++++++++ lib/librte_eal/common/include/rte_lcore.h | 29+++++++++++++++++++++++++++--
Review
Quoted 4 lines
lib/librte_eal/linuxapp/eal/eal_thread.c | 2 ++ 4 files changed, 45 insertions(+), 2 deletions(-)(quoted diff) +0
-0
b/lib/librte_eal/bsdapp/eal/eal_thread.cReview
Quoted 13 lines
index ab05368..10220c7 100644
lib/librte_eal/bsdapp/eal/eal_thread.c +2
-0@@ -56,6 +56,8 @@#include "eal_thread.h" RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); +RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);/* * Send a message to a slave lcore identified by slave_id to call a (quoted diff) +0
-0
b/lib/librte_eal/common/eal_thread.hQuoted 7 lines
index a25ee86..28edf51 100644
lib/librte_eal/common/eal_thread.h +0
-0@@ -102,4 +102,18 @@ eal_cpuset_socket_id(rte_cpuset_t *cpusetp)return socket_id; }
Quoted 16 lines
lib/librte_eal/common/eal_thread.h eal_cpuset_socket_id(rte_cpuset_t *cpusetp) · line 116
Quoted 12 lines
lib/librte_eal/common/eal_thread.h +12
-0+/** +* Dump the current pthread cpuset. +* This function is private to EAL. +* +* @param str +* The string buffer the cpuset will dump to. +* @param size +* The string buffer size. +*/ +#define CPU_STR_LEN 256 +void +eal_thread_dump_affinity (char str[],unsigned size);Although it's equivalent for function arguments, I think "char *str" is usually preferred over "char str[]". See for instance in
snprintf()orfgets().
[LCM] Accept.
Quoted 2 lines
What is the purpose of
CPU_STR_LEN?
[LCM] For default quick reference for str[] definition used in dump_affinity()
Quoted 3 lines
What occurs if the size of the dump is greater than the size of the given buffer? Is the string truncated? Is there a \0 at the end?
[LCM] Yes, always have a '\0' in the end.
Quoted 1 line
This should be described in the API comments.
[LCM] Accept.
Quoted 2 lines
Maybe adding a return
value could help the user to determine if the string was truncated.
[LCM] Good idea, so the user can continue to print '...' for the truncated part.
Quoted 29 lines
Review
Quoted 5 lines
lib/librte_eal/common/eal_thread.h +2
-0+ +#endif /* EAL_THREAD_H */ (quoted diff) +0
-0
b/lib/librte_eal/common/include/rte_lcore.hQuoted 12 lines
index 4c7d6bb..facdbdc 100644
lib/librte_eal/common/include/rte_lcore.h +1
-0@@ -43,6 +43,7 @@#include <rte_per_lcore.h> #include <rte_eal.h> #include <rte_launch.h> +#include <rte_memory.h> #ifdef __cplusplusextern "C" {lib/librte_eal/common/include/rte_lcore.h struct lcore_config { · line 84
Quoted 5 lines
lib/librte_eal/common/include/rte_lcore.h +1
-1@@ -80,7 +81,9 @@ struct lcore_config {*/ extern struct lcore_config lcore_config[RTE_MAX_LCORE ]; -RTE_DECLARE_PER_LCORE (unsigned, _lcore_id);/**< Per core "core id". */ +RTE_DECLARE_PER_LCORE (unsigned, _lcore_id);/**< Per thread "lcore id". */
lib/librte_eal/common/include/rte_lcore.h struct lcore_config { · line 85
Quoted 1 line
lib/librte_eal/common/include/rte_lcore.h +1
-0+RTE_DECLARE_PER_LCORE (unsigned, _socket_id);/**< Per thread "socket id". */
lib/librte_eal/common/include/rte_lcore.h struct lcore_config { · line 86
Quoted 1 line
lib/librte_eal/common/include/rte_lcore.h +1
-0+RTE_DECLARE_PER_LCORE (rte_cpuset_t, _cpuset);/**< Per thread "cpuset". */
Quoted 1 line
Quoted 12 lines
lib/librte_eal/common/include/rte_lcore.h rte_lcore_index(int lcore_id) · line 153
Quoted 9 lines
/**
* Return the ID of the execution unit we are running on.lib/librte_eal/common/include/rte_lcore.h +1
-1@@ -146,7 +149,7 @@ rte_lcore_index(int lcore_id)static inline unsigned rte_socket_id (void ) { -return lcore_config[rte_lcore_id ()].socket_id; +return RTE_PER_LCORE (_socket_id); }I don't see where the
_socket_idvariable is assigned. I think there is probably an issue with the splitting of the patches.
[LCM] The value initializes as SOCKET_ID_ANY when RTE_DEFINE_PER_LCORE().
And updated in eal_thread_set_affinity() for EAL thread and rte_thread_set_affinity() for non-EAL thread.
Quoted 3 lines
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): D0158A423229094DA7ABF71CF2FA0DA3118D7DA9-E2R4CRU6q/[email protected]
Re: [PATCH v4 05/17] eal: new TLS definition and API declaration
Hi,
Quoted 16 lines · Cunming
Quoted 8 lines
Quoted 8 lines
+/**
+ * Dump the current pthread cpuset.
+ * This function is private to EAL.
+ *
+ * @param str
+ * The string buffer the cpuset will dump to.
+ * @param size
+ * The string buffer size.Quoted 8 lines
lib/librte_eal/common/eal_thread.h eal_cpuset_socket_id(rte_cpuset_t *cpusetp) · line 116
Quoted 4 lines
+ */
lib/librte_eal/common/eal_thread.h +3
-0+#define CPU_STR_LEN 256 +void +eal_thread_dump_affinity (char str[],unsigned size);Although it's equivalent for function arguments, I think "char *str" is usually preferred over "char str[]". See for instance in
snprintf()orfgets().
Quoted 4 lines
[LCM] Accept.
Quoted 2 lines
What is the purpose of
CPU_STR_LEN?[LCM] For default quick reference for str[] definition used in
dump_affinity()
So the API comment of the function is not placed at the right place.
A comment "Default buffer size to use with eal_thread_dump_affinity()" should be added above CPU_STR_LEN. Also, it could be renamed in RTE_CPU_STR_LEN or RTE_CPU_AFFINITY_STR_LEN.
Quoted 24 lines
Quoted 12 lines
lib/librte_eal/common/include/rte_lcore.h struct lcore_config { · line 84
Quoted 6 lines
lib/librte_eal/common/include/rte_lcore.h +1
-1@@ -80,7 +81,9 @@ struct lcore_config {*/ extern struct lcore_config lcore_config[RTE_MAX_LCORE ]; -RTE_DECLARE_PER_LCORE (unsigned, _lcore_id);/**< Per core "core id". */ +RTE_DECLARE_PER_LCORE (unsigned, _lcore_id);/**< Per thread "lcore id". */
Review struct lcore_config { · line 85
Quoted 1 line
(quoted diff) +1
-0+RTE_DECLARE_PER_LCORE(unsigned, _socket_id); /**< Per thread "socket id".*/
Review struct lcore_config { · line 86
Quoted 1 line
(quoted diff) +1
-0+RTE_DECLARE_PER_LCORE(rte_cpuset_t, _cpuset); /**< Per thread "cpuset".*/
Quoted 1 line
Quoted 12 lines
lib/librte_eal/common/include/rte_lcore.h rte_lcore_index(int lcore_id) · line 153
Quoted 9 lines
/**
* Return the ID of the execution unit we are running on.lib/librte_eal/common/include/rte_lcore.h +1
-1@@ -146,7 +149,7 @@ rte_lcore_index(int lcore_id)static inline unsigned rte_socket_id (void ) { -return lcore_config[rte_lcore_id ()].socket_id; +return RTE_PER_LCORE (_socket_id); }I don't see where the
_socket_idvariable is assigned. I think there is probably an issue with the splitting of the patches.
Quoted 2 lines
[LCM] The value initializes as
SOCKET_ID_ANYwhenRTE_DEFINE_PER_LCORE().
And updated ineal_thread_set_affinity()for EAL thread andrte_thread_set_affinity()for non-EAL thread.
This is done in a later patches:
"eal: set _lcore_id and _socket_id to (-1) by default" "eal: apply affinity of EAL thread by assigned cpuset"
That's why I said there is probably an issue with the ordering of the patches as these values are used here but initialized later in the series.
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH v4 05/17] eal: new TLS definition and API declaration
Quoted 31 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Tuesday, February 10, 2015 1:26 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 05/17] eal: new TLS definition and API declarationHi,
Quoted 19 lines · Cunming
Quoted 8 lines
Quoted 8 lines
+/**
+ * Dump the current pthread cpuset.
+ * This function is private to EAL.
+ *
+ * @param str
+ * The string buffer the cpuset will dump to.
+ * @param size
+ * The string buffer size.Quoted 8 lines
Review
Quoted 4 lines
+ */
(quoted diff) +3
-0+#define CPU_STR_LEN 256 +void +eal_thread_dump_affinity(char str[], unsigned size);Although it's equivalent for function arguments, I think "char *str" is usually preferred over "char str[]". See for instance in
snprintf()orfgets().[LCM] Accept.
Quoted 2 lines
What is the purpose of
CPU_STR_LEN?
Quoted 8 lines
Quoted 1 line
[LCM] For default quick reference for str[] definition used in
dump_affinity()So the API comment of the function is not placed at the right place.
A comment "Default buffer size to use with
eal_thread_dump_affinity()" should be added aboveCPU_STR_LEN. Also, it could be renamed inRTE_CPU_STR_LENorRTE_CPU_AFFINITY_STR_LEN.
[LCM] Got you.
Quoted 33 lines
Quoted 11 lines
Quoted 11 lines
Review struct lcore_config { · line 84
Quoted 9 lines
(quoted diff) +1
-1@@ -80,7 +81,9 @@ struct lcore_config { */ extern struct lcore_config lcore_config[RTE_MAX_LCORE]; -RTE_DECLARE_PER_LCORE(unsigned, _lcore_id); /**< Per core "core id". */ +RTE_DECLARE_PER_LCORE(unsigned, _lcore_id); /**< Per thread "lcore id".*/
Quoted 1 line
(quoted diff) +1
-0+RTE_DECLARE_PER_LCORE(unsigned, _socket_id); /**< Per thread "socketid".
Quoted 2 lines
Quoted 2 lines
*/
Quoted 1 line
(quoted diff) +1
-0+RTE_DECLARE_PER_LCORE(rte_cpuset_t, _cpuset); /**< Per thread"cpuset".
Quoted 14 lines
Quoted 14 lines
*/
Review rte_lcore_index(int lcore_id) · line 153
Quoted 10 lines
(quoted diff) +1
-1/** * Return the ID of the execution unit we are running on.@@ -146,7 +149,7 @@ rte_lcore_index(int lcore_id) static inline unsigned rte_socket_id(void) { - return lcore_config[rte_lcore_id()].socket_id; + return RTE_PER_LCORE(_socket_id); }I don't see where the
_socket_idvariable is assigned. I think there is probably an issue with the splitting of the patches.Quoted 2 lines
[LCM] The value initializes as
SOCKET_ID_ANYwhenRTE_DEFINE_PER_LCORE(). And updated ineal_thread_set_affinity()for EAL thread and
rte_thread_set_affinity()for non-EAL thread.
Quoted 8 lines
This is done in a later patches:
"eal: set
_lcore_idand_socket_idto (-1) by default" "eal: apply affinity of EAL thread by assigned cpuset"That's why I said there is probably an issue with the ordering of the patches as these values are used here but initialized later in the series.
[LCM] Will reorder them in next version.
PATCH v4 6/17 eal: add eal_common_thread.c for common thread API
In reply to a message not in the archiveThe API works for both EAL thread and none EAL thread. When calling rte_thread_set_affinity, the *_socket_id* and *_cpuset* of calling thread will be updated if the thread successful set the cpu affinity.
--- lib/librte_eal/bsdapp/eal/Makefile | 1 + lib/librte_eal/common/eal_common_thread.c | 142 ++++++++++++++++++++++++++++++ lib/librte_eal/linuxapp/eal/Makefile | 2 + 3 files changed, 145 insertions(+) create mode 100644 lib/librte_eal/common/eal_common_thread.c
lib/librte_eal/bsdapp/eal/Makefile +1 -0
@@ -73,6 +73,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_EAL_BSDAPP) += eal_common_hexdump.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_BSDAPP) += eal_common_devargs.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_BSDAPP) += eal_common_dev.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_BSDAPP) += eal_common_options.c +SRCS-$(CONFIG_RTE_LIBRTE_EAL_BSDAPP) += eal_common_thread.c CFLAGS_eal.o := -D_GNU_SOURCE #CFLAGS_eal_thread.o := -D_GNU_SOURCE
lib/librte_eal/common/eal_common_thread.c new file +142 -0
142 lines not on this page · raw message
lib/librte_eal/linuxapp/eal/Makefile +2 -0
@@ -85,6 +85,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_hexdump.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_devargs.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_dev.c SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_options.c +SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_thread.c CFLAGS_eal.o := -D_GNU_SOURCE CFLAGS_eal_lcore.o := -D_GNU_SOURCE
@@ -96,6 +97,7 @@ CFLAGS_eal_pci.o := -D_GNU_SOURCE CFLAGS_eal_pci_vfio.o := -D_GNU_SOURCE CFLAGS_eal_common_whitelist.o := -D_GNU_SOURCE CFLAGS_eal_common_options.o := -D_GNU_SOURCE +CFLAGS_eal_common_thread.o := -D_GNU_SOURCE # workaround for a gcc bug with noreturn attribute # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603
Signature · 1 line
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 1422842559-13617-7-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
Re: [PATCH v4 06/17] eal: add eal_common_thread.c for common thread API
Hi,
Quoted 6 lines · Cunming Liang
The API works for both EAL thread and none EAL thread. When calling
rte_thread_set_affinity, the *_socket_id* and *_cpuset* of calling thread will be updated if the thread successful set the cpu affinity.[...]
lib/librte_eal/common/eal_common_thread.c line 55
Quoted 9 lines
lib/librte_eal/common/eal_common_thread.c new file +9
-0+int +rte_thread_set_affinity (rte_cpuset_t *cpusetp) +{ +int s; +unsigned lcore_id; +pthread_t tid; + +if (!cpusetp) +return -1 ;
Is it really needed to test that cpusetp is not NULL?
lib/librte_eal/common/eal_common_thread.c line 58
Quoted 3 lines
lib/librte_eal/common/eal_common_thread.c new file +3
-0+ + lcore_id =rte_lcore_id (); +if (lcore_id != (unsigned )LCORE_ID_ANY ) {
This is strange to see something that cannot happen:lcore_id == LCORE_ID_ANY is only possible after your patch is 12/17 is added. Maybe it can be reordered to avoid this inconsistency?
Quoted 29 lines
lib/librte_eal/common/eal_common_thread.c new file +29
-0+/* EAL thread */ + tid = lcore_config[lcore_id].thread_id; + + s =pthread_setaffinity_np (tid,sizeof (rte_cpuset_t), cpusetp); +if (s !=0 ) { +RTE_LOG (ERR ,EAL ,"pthread_setaffinity_np failed\n" ); +return -1 ; + } + +/* store socket_id in TLS for quick access */ +RTE_PER_LCORE (_socket_id) = +eal_cpuset_socket_id (cpusetp); + +/* store cpuset in TLS for quick access */ +rte_memcpy (&RTE_PER_LCORE (_cpuset), cpusetp, +sizeof (rte_cpuset_t)); + +/* update lcore_config */ + lcore_config[lcore_id].socket_id =RTE_PER_LCORE (_socket_id); +rte_memcpy (&lcore_config[lcore_id].cpuset, cpusetp, +sizeof (rte_cpuset_t)); + }else { +/* none EAL thread */ + tid =pthread_self (); + + s =pthread_setaffinity_np (tid,sizeof (rte_cpuset_t), cpusetp); +if (s !=0 ) { +RTE_LOG (ERR ,EAL ,"pthread_setaffinity_np failed\n" ); +return -1 ;
lib/librte_eal/common/eal_common_thread.c line 97
Quoted 10 lines
lib/librte_eal/common/eal_common_thread.c new file +10
-0+ } + +/* store cpuset in TLS for quick access */ +rte_memcpy (&RTE_PER_LCORE (_cpuset), cpusetp, +sizeof (rte_cpuset_t)); + +/* store socket_id in TLS for quick access */ +RTE_PER_LCORE (_socket_id) = +eal_cpuset_socket_id (cpusetp); + }
Why not always using pthread_self() to get the tid?
I think most of the code could be factorized here. The only difference (which is hard to see as is as code is not exactly ordered in the same manner) is that the config is updated in case it's an EAL thread.
lib/librte_eal/common/eal_common_thread.c line 106
Quoted 9 lines
lib/librte_eal/common/eal_common_thread.c new file +9
-0+ +return 0 ; +} + +int +rte_thread_get_affinity (rte_cpuset_t *cpusetp) +{ +if (!cpusetp) +return -1 ;
Same here. This is the only reason why rte_thread_get_affinity() could fail. Removing this test would allow to change the API to return void instead. It will avoid a useless test below in eal_thread_dump_affinity().
Quoted 9 lines
lib/librte_eal/common/eal_common_thread.c new file +9
-0+ +rte_memcpy (cpusetp, &RTE_PER_LCORE (_cpuset), +sizeof (rte_cpuset_t)); + +return 0 ; +} + +void +eal_thread_dump_affinity (char str[],unsigned size)
lib/librte_eal/common/eal_common_thread.c line 125
Quoted 10 lines
lib/librte_eal/common/eal_common_thread.c new file +10
-0+{ +rte_cpuset_t cpuset; +unsigned cpu; +int ret; +unsigned int out =0 ; + +if (rte_thread_get_affinity (&cpuset) <0 ) { + str[0 ] ='\0' ; +return ; + }
This one could be removed it the (== NULL) test is removed.
lib/librte_eal/common/eal_common_thread.c line 134
Quoted 9 lines
lib/librte_eal/common/eal_common_thread.c new file +9
-0+ +for (cpu =0 ; cpu <RTE_MAX_LCORE ; cpu++) { +if (!CPU_ISSET (cpu, &cpuset)) +continue ; + + ret =snprintf (str + out, + size - out,"%u," , cpu); +if (ret <0 || (unsigned )ret >= size - out) +break ;
On the contrary, I think here returning an error to the user would be useful so he can knows that the dump is not complete.
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH v4 06/17] eal: add eal_common_thread.c for common thread API
Quoted 17 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Monday, February 09, 2015 4:00 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 06/17] eal: addeal_common_thread.cfor common thread APIHi,
Quoted 5 lines · Cunming Liang
The API works for both EAL thread and none EAL thread. When calling
rte_thread_set_affinity, the *_socket_id* and *_cpuset* of calling thread will be updated if the thread successful set the cpu affinity.
Quoted 12 lines
lib/librte_eal/common/eal_common_thread.c line 55
Quoted 10 lines
[...]
lib/librte_eal/common/eal_common_thread.c new file +9
-0+int +rte_thread_set_affinity (rte_cpuset_t *cpusetp) +{ +int s; +unsigned lcore_id; +pthread_t tid; + +if (!cpusetp) +return -1 ;Is it really needed to test that cpusetp is not NULL?
[LCM] Accept, we can ignore it and depend on pthread_setaffinity_np() to return failure.
Quoted 8 lines
lib/librte_eal/common/eal_common_thread.c line 58
Quoted 4 lines
lib/librte_eal/common/eal_common_thread.c new file +3
-0+ + lcore_id =rte_lcore_id (); +if (lcore_id != (unsigned )LCORE_ID_ANY ) {This is strange to see something that cannot happen:
lcore_id==LCORE_ID_ANYis only possible after your patch is 12/17 is added. Maybe it can be reordered to avoid this inconsistency?
[LCM] You're right, here do some re-order.
The point is to make everything ready before switching the default value to -1. And we can have the whole function implement in one patch.
It just won't take effect, but won't bring additional risk.
Quoted 34 lines
Quoted 34 lines
lib/librte_eal/common/eal_common_thread.c new file +33
-0+/* EAL thread */ + tid = lcore_config[lcore_id].thread_id; + + s =pthread_setaffinity_np (tid,sizeof (rte_cpuset_t), cpusetp); +if (s !=0 ) { +RTE_LOG (ERR ,EAL ,"pthread_setaffinity_np failed\n" ); +return -1 ; + } + +/* store socket_id in TLS for quick access */ +RTE_PER_LCORE (_socket_id) = +eal_cpuset_socket_id (cpusetp); + +/* store cpuset in TLS for quick access */ +rte_memcpy (&RTE_PER_LCORE (_cpuset), cpusetp, +sizeof (rte_cpuset_t)); + +/* update lcore_config */ + lcore_config[lcore_id].socket_id =RTE_PER_LCORE (_socket_id); +rte_memcpy (&lcore_config[lcore_id].cpuset, cpusetp, +sizeof (rte_cpuset_t)); + }else { +/* none EAL thread */ + tid =pthread_self (); + + s =pthread_setaffinity_np (tid,sizeof (rte_cpuset_t), cpusetp); +if (s !=0 ) { +RTE_LOG (ERR ,EAL ,"pthread_setaffinity_np failed\n" ); +return -1 ; + } + +/* store cpuset in TLS for quick access */ + rte_memcpy(&RTE_PER_LCORE(_cpuset), cpusetp,
Quoted 8 lines
lib/librte_eal/common/eal_common_thread.c line 97
Quoted 6 lines
lib/librte_eal/common/eal_common_thread.c new file +6
-0+ sizeof(rte_cpuset_t)); + +/* store socket_id in TLS for quick access */ +RTE_PER_LCORE (_socket_id) = +eal_cpuset_socket_id (cpusetp); + }Why not always using
pthread_self()to get the tid?
[LCM] Good point, I haven't notice it.
Quoted 4 lines
I think most of the code could be factorized here. The only difference (which is hard to see as is as code is not exactly ordered in the same manner) is that the config is updated in case it's an EAL thread.
[LCM] Accept.
Quoted 17 lines
lib/librte_eal/common/eal_common_thread.c line 106
Quoted 12 lines
lib/librte_eal/common/eal_common_thread.c new file +9
-0+ +return 0 ; +} + +int +rte_thread_get_affinity (rte_cpuset_t *cpusetp) +{ +if (!cpusetp) +return -1 ;Same here. This is the only reason why
rte_thread_get_affinity()could fail. Removing this test would allow to change the API to return void instead. It will avoid a useless test below ineal_thread_dump_affinity().
[LCM] The cpusetp is used as destination of memcpy and the function suppose an EAL API. I don't think it's a good idea to remove the check, do you ?
Quoted 23 lines
Quoted 10 lines
lib/librte_eal/common/eal_common_thread.c new file +9
-0+ +rte_memcpy (cpusetp, &RTE_PER_LCORE (_cpuset), +sizeof (rte_cpuset_t)); + +return 0 ; +} + +void +eal_thread_dump_affinity (char str[],unsigned size)lib/librte_eal/common/eal_common_thread.c line 125
Quoted 10 lines
lib/librte_eal/common/eal_common_thread.c new file +10
-0+{ +rte_cpuset_t cpuset; +unsigned cpu; +int ret; +unsigned int out =0 ; + +if (rte_thread_get_affinity (&cpuset) <0 ) { + str[0 ] ='\0' ; +return ; + }This one could be removed it the (== NULL) test is removed.
Quoted 12 lines
lib/librte_eal/common/eal_common_thread.c line 134
Quoted 9 lines
lib/librte_eal/common/eal_common_thread.c new file +9
-0+ +for (cpu =0 ; cpu <RTE_MAX_LCORE ; cpu++) { +if (!CPU_ISSET (cpu, &cpuset)) +continue ; + + ret =snprintf (str + out, + size - out,"%u," , cpu); +if (ret <0 || (unsigned )ret >= size - out) +break ;On the contrary, I think here returning an error to the user would be useful so he can knows that the dump is not complete.
[LCM] accept.
Quoted 4 lines
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): D0158A423229094DA7ABF71CF2FA0DA3118D7DF6-E2R4CRU6q/[email protected]
Re: [PATCH v4 06/17] eal: add eal_common_thread.c for common thread API
Hi,
Quoted 10 lines · Cunming
Quoted 10 lines
lib/librte_eal/common/eal_common_thread.c line 106
Quoted 5 lines
lib/librte_eal/common/eal_common_thread.c new file +5
-0+int +rte_thread_get_affinity (rte_cpuset_t *cpusetp) +{ +if (!cpusetp) +return -1 ;Same here. This is the only reason why
rte_thread_get_affinity()could fail. Removing this test would allow to change the API to return void instead. It will avoid a useless test below ineal_thread_dump_affinity().
Quoted 2 lines
[LCM] The cpusetp is used as destination of memcpy and the function suppose an EAL API. I don't think it's a good idea to remove the check, do you ?
I know we often have debate on this subject on the list. My personal opinion is that checking a NULL pointer in these cases is useless because the user is suppose to give a non-NULL pointer. Returning an error will result in managing an error for something that cannot happen.
On the other hand, adding an assert() (or the dpdk equivalent) would be a good idea.
Quoted 40 lines
Quoted 23 lines
Quoted 10 lines
lib/librte_eal/common/eal_common_thread.c new file +9
-0+ +rte_memcpy (cpusetp, &RTE_PER_LCORE (_cpuset), +sizeof (rte_cpuset_t)); + +return 0 ; +} + +void +eal_thread_dump_affinity (char str[],unsigned size)lib/librte_eal/common/eal_common_thread.c line 125
Quoted 10 lines
lib/librte_eal/common/eal_common_thread.c new file +10
-0+{ +rte_cpuset_t cpuset; +unsigned cpu; +int ret; +unsigned int out =0 ; + +if (rte_thread_get_affinity (&cpuset) <0 ) { + str[0 ] ='\0' ; +return ; + }This one could be removed it the (== NULL) test is removed.
Quoted 12 lines
lib/librte_eal/common/eal_common_thread.c line 134
Quoted 9 lines
lib/librte_eal/common/eal_common_thread.c new file +9
-0+ +for (cpu =0 ; cpu <RTE_MAX_LCORE ; cpu++) { +if (!CPU_ISSET (cpu, &cpuset)) +continue ; + + ret =snprintf (str + out, + size - out,"%u," , cpu); +if (ret <0 || (unsigned )ret >= size - out) +break ;On the contrary, I think here returning an error to the user would be useful so he can knows that the dump is not complete.
[LCM] accept.
Quoted 4 lines
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH v4 06/17] eal: add eal_common_thread.c for common thread API
Quoted 26 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Tuesday, February 10, 2015 1:30 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 06/17] eal: addeal_common_thread.cfor common thread APIHi,
Quoted 11 lines · Cunming
Quoted 10 lines
Review
Quoted 5 lines
(quoted diff) +5
-0+int +rte_thread_get_affinity(rte_cpuset_t *cpusetp) +{ + if (!cpusetp) + return -1;Same here. This is the only reason why
rte_thread_get_affinity()could fail. Removing this test would allow to change the API to return void instead. It will avoid a useless test below ineal_thread_dump_affinity().[LCM] The cpusetp is used as destination of memcpy and the function suppose
an EAL API.
Quoted 1 line
I don't think it's a good idea to remove the check, do you ?
Quoted 8 lines
I know we often have debate on this subject on the list. My personal opinion is that checking a NULL pointer in these cases is useless because the user is suppose to give a non-NULL pointer. Returning an error will result in managing an error for something that cannot happen.
On the other hand, adding an
assert()(or the dpdk equivalent) would be a good idea.
[LCM] Ok, I see. Will update it.
Quoted 42 lines
Quoted 42 lines
Quoted 25 lines
Quoted 12 lines
(quoted diff) +9
-0+ + rte_memcpy(cpusetp, &RTE_PER_LCORE(_cpuset), + sizeof(rte_cpuset_t)); + + return 0; +} + +void +eal_thread_dump_affinity(char str[], unsigned size)Review
Quoted 10 lines
(quoted diff) +10
-0+{ + rte_cpuset_t cpuset; + unsigned cpu; + int ret; + unsigned int out = 0; + + if (rte_thread_get_affinity(&cpuset) < 0) { + str[0] = '\0'; + return; + }This one could be removed it the (== NULL) test is removed.
Quoted 12 lines
Review
Quoted 9 lines
(quoted diff) +9
-0+ + for (cpu = 0; cpu < RTE_MAX_LCORE; cpu++) { + if (!CPU_ISSET(cpu, &cpuset)) + continue; + + ret = snprintf(str + out, + size - out, "%u,", cpu); + if (ret < 0 || (unsigned)ret >= size - out) + break;On the contrary, I think here returning an error to the user would be useful so he can knows that the dump is not complete.
[LCM] accept.
Quoted 4 lines
Regards,
Olivier
PATCH v4 7/17 eal: add rte_gettid() to acquire unique system tid
In reply to a message not in the archiveThe rte_gettid() wraps the linux and freebsd syscall gettid(). It provides a persistent unique thread id for the calling thread. It will save the unique id in TLS on the first time.
--- lib/librte_eal/bsdapp/eal/eal_thread.c | 9 +++++++++ lib/librte_eal/common/include/rte_eal.h | 27 +++++++++++++++++++++++++++ lib/librte_eal/linuxapp/eal/eal_thread.c | 7 +++++++ 3 files changed, 43 insertions(+)
lib/librte_eal/bsdapp/eal/eal_thread.c +9 -0
@@ -39,6 +39,7 @@#include <sched.h> #include <pthread_np.h> #include <sys/queue.h> +#include <sys/thr.h> #include <rte_debug.h> #include <rte_atomic.h>
@@ -233,3 +234,11 @@ eal_thread_loop(__attribute__((unused)) void *arg)/* pthread_exit(NULL); */ /* return NULL; */ } + +/* require calling thread tid by gettid() */ +int rte_sys_gettid (void ) +{ +long lwpid; +thr_self (&lwpid); +return (int )lwpid; +}
lib/librte_eal/common/include/rte_eal.h +27 -0
@@ -41,6 +41,9 @@*/ #include <stdint.h> +#include <sched.h> + +#include <rte_per_lcore.h> #ifdef __cplusplusextern "C" {
@@ -262,6 +265,30 @@ rte_set_application_usage_hook( rte_usage_hook_t usage_func );*/ int rte_eal_has_hugepages (void ); +/** +* A wrap API for syscall gettid. +* +* @return +* On success, returns the thread ID of calling process. +* It always successful. +*/ +int rte_sys_gettid (void ); + +/** +* Get system unique thread id. +* +* @return +* On success, returns the thread ID of calling process. +* It always successful. +*/ +static inline int rte_gettid (void ) +{ + static RTE_DEFINE_PER_LCORE(int, _thread_id) = -1; +if (RTE_PER_LCORE (_thread_id) ==-1 ) +RTE_PER_LCORE (_thread_id) =rte_sys_gettid (); +return RTE_PER_LCORE (_thread_id); +} + #ifdef __cplusplus }#endif
lib/librte_eal/linuxapp/eal/eal_thread.c +7 -0
@@ -39,6 +39,7 @@#include <pthread.h> #include <sched.h> #include <sys/queue.h> +#include <sys/syscall.h> #include <rte_debug.h> #include <rte_atomic.h>
@@ -233,3 +234,9 @@ eal_thread_loop(__attribute__((unused)) void *arg)/* pthread_exit(NULL); */ /* return NULL; */ } + +/* require calling thread tid by gettid() */ +int rte_sys_gettid (void ) +{ +return (int )syscall (SYS_gettid); +}
Signature · 1 line
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 1422842559-13617-8-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
Re: [PATCH v4 07/17] eal: add rte_gettid() to acquire unique system tid
Hi,
Quoted 19 lines · Cunming Liang
The
rte_gettid()wraps the linux and freebsd syscallgettid(). It provides a persistent unique thread id for the calling thread. It will save the unique id in TLS on the first time.[...]
lib/librte_eal/common/include/rte_eal.h +13
-0+/** +* A wrap API for syscall gettid. +* +* @return +* On success, returns the thread ID of calling process. +* It always successful. +*/ +int rte_sys_gettid (void ); + +/** +* Get system unique thread id. +* +* @return
lib/librte_eal/common/include/rte_eal.h rte_set_application_usage_hook( rte_usage_hook_t usage_func ); · line 290
Quoted 10 lines
lib/librte_eal/common/include/rte_eal.h +10
-0+* On success, returns the thread ID of calling process. +* It always successful. +*/ +static inline int rte_gettid (void ) +{ + static RTE_DEFINE_PER_LCORE(int, _thread_id) = -1; +if (RTE_PER_LCORE (_thread_id) ==-1 ) +RTE_PER_LCORE (_thread_id) =rte_sys_gettid (); +return RTE_PER_LCORE (_thread_id); +}
Instead of doing the test each time rte_gettid() is called, why not having 2 functions:
rte_init_tid() -> assign the per_lcore variable rte_gettid() -> return the per_lcore variable
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH v4 07/17] eal: add rte_gettid() to acquire unique system tid
Quoted 38 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Monday, February 09, 2015 4:01 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 07/17] eal: addrte_gettid()to acquire unique system tidHi,
Quoted 26 lines · Cunming Liang
The
rte_gettid()wraps the linux and freebsd syscallgettid(). It provides a persistent unique thread id for the calling thread. It will save the unique id in TLS on the first time.[...]
lib/librte_eal/common/include/rte_eal.h +20
-0+/** +* A wrap API for syscall gettid. +* +* @return +* On success, returns the thread ID of calling process. +* It always successful. +*/ +int rte_sys_gettid (void ); + +/** +* Get system unique thread id. +* +* @return +* On success, returns the thread ID of calling process. +* It always successful. +*/ +static inline int rte_gettid (void ) +{ + static RTE_DEFINE_PER_LCORE(int, _thread_id) = -1; +if (RTE_PER_LCORE (_thread_id) ==-1 )
Quoted 8 lines
lib/librte_eal/common/include/rte_eal.h rte_set_application_usage_hook( rte_usage_hook_t usage_func ); · line 290
Quoted 3 lines
lib/librte_eal/common/include/rte_eal.h +3
-0+RTE_PER_LCORE (_thread_id) =rte_sys_gettid (); +return RTE_PER_LCORE (_thread_id); +}Instead of doing the test each time
rte_gettid()is called, why not having 2 functions:
rte_init_tid()-> assign theper_lcorevariablerte_gettid()-> return theper_lcorevariable
[LCM] The rte_gettid() mainly used in recursive spinlock.
For non-EAL thread, we don't expect new user thread has to explicit call something.
The purpose to call it in EAL thread init, is to lower down the overhead of the first calling for EAL thread.
Quoted 5 lines
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): D0158A423229094DA7ABF71CF2FA0DA3118D8371-E2R4CRU6q/[email protected]
Re: [PATCH v4 07/17] eal: add rte_gettid() to acquire unique system tid
Hi,
Quoted 20 lines · Cunming
Quoted 17 lines
Quoted 4 lines
lib/librte_eal/common/include/rte_eal.h +4
-0+/** +* Get system unique thread id. +* +* @return lib/librte_eal/common/include/rte_eal.h rte_set_application_usage_hook( rte_usage_hook_t usage_func ); · line 290
Quoted 10 lines
lib/librte_eal/common/include/rte_eal.h +10
-0+* On success, returns the thread ID of calling process. +* It always successful. +*/ +static inline int rte_gettid (void ) +{ + static RTE_DEFINE_PER_LCORE(int, _thread_id) = -1; +if (RTE_PER_LCORE (_thread_id) ==-1 ) +RTE_PER_LCORE (_thread_id) =rte_sys_gettid (); +return RTE_PER_LCORE (_thread_id); +}Instead of doing the test each time
rte_gettid()is called, why not having 2 functions:Quoted 2 lines
rte_init_tid() -> assign the per_lcore variable rte_gettid() -> return the per_lcore variable
Quoted 3 lines
[LCM] The
rte_gettid()mainly used in recursive spinlock.
For non-EAL thread, we don't expect new user thread has to explicit call something.
The purpose to call it in EAL thread init, is to lower down the overhead of the first calling for EAL thread.
Got it. So that's fine like you proposed.
Olivier
PATCH v4 8/17 eal: apply affinity of EAL thread by assigned cpuset
In reply to a message not in the archiveEAL threads use assigned cpuset to set core affinity during startup. It keeps 1:1 mapping, if no '--lcores' option is used.
--- lib/librte_eal/bsdapp/eal/eal.c | 13 ++++--- lib/librte_eal/bsdapp/eal/eal_thread.c | 63 +++++++++--------------------- lib/librte_eal/linuxapp/eal/eal.c | 7 +++- lib/librte_eal/linuxapp/eal/eal_thread.c | 67 +++++++++++--------------------- 4 files changed, 54 insertions(+), 96 deletions(-)
lib/librte_eal/bsdapp/eal/eal.c +8 -5
@@ -432,6 +432,7 @@ rte_eal_init(int argc, char **argv)int i, fctret, ret;pthread_t thread_id;static rte_atomic32_t run_once =RTE_ATOMIC32_INIT (0 ); +char cpuset[CPU_STR_LEN ];if (!rte_atomic32_test_and_set (&run_once))return -1 ;
@@ -502,13 +503,17 @@ rte_eal_init(int argc, char **argv)if (rte_eal_pci_init () <0 )rte_panic ("Cannot init PCI\n" ); -RTE_LOG (DEBUG ,EAL ,"Master core %u is ready (tid=%p)\n" , - rte_config.master_lcore, thread_id); -eal_check_mem_on_local_socket ();rte_eal_mcfg_complete (); +eal_thread_init_master (rte_config.master_lcore); + +eal_thread_dump_affinity (cpuset,CPU_STR_LEN ); + +RTE_LOG (DEBUG ,EAL ,"Master lcore %u is ready (tid=%p;cpuset=[%s])\n" , + rte_config.master_lcore, thread_id, cpuset); +if (rte_eal_dev_init () <0 )rte_panic ("Cannot init pmd devices\n" );
@@ -532,8 +537,6 @@ rte_eal_init(int argc, char **argv)rte_panic ("Cannot create thread\n" ); } -eal_thread_init_master (rte_config.master_lcore); -/* * Launch a dummy function on all slave lcores, so that master lcore * knows they are all ready when this function returns.
lib/librte_eal/bsdapp/eal/eal_thread.c +19 -44
95 lines not on this page · raw message
lib/librte_eal/linuxapp/eal/eal.c +5 -2
@@ -702,6 +702,7 @@ rte_eal_init(int argc, char **argv)static rte_atomic32_t run_once =RTE_ATOMIC32_INIT (0 );struct shared_driver *solib =NULL ;const char *logid; +char cpuset[CPU_STR_LEN ];if (!rte_atomic32_test_and_set (&run_once))return -1 ;
@@ -802,8 +803,10 @@ rte_eal_init(int argc, char **argv)eal_thread_init_master (rte_config.master_lcore); -RTE_LOG (DEBUG ,EAL ,"Master core %u is ready (tid=%x)\n" , - rte_config.master_lcore, (int )thread_id); +eal_thread_dump_affinity (cpuset,CPU_STR_LEN ); + +RTE_LOG (DEBUG ,EAL ,"Master lcore %u is ready (tid=%x;cpuset=[%s])\n" , + rte_config.master_lcore, (int )thread_id, cpuset);if (rte_eal_dev_init () <0 )rte_panic ("Cannot init pmd devices\n" );
lib/librte_eal/linuxapp/eal/eal_thread.c +22 -45
110 lines not on this page · raw message
Signature · 1 line
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 1422842559-13617-9-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
Re: [PATCH v4 08/17] eal: apply affinity of EAL thread by assigned cpuset
Hi,
Quoted 45 lines · Cunming Liang
EAL threads use assigned cpuset to set core affinity during startup. It keeps 1:1 mapping, if no '--lcores' option is used.
[...]
lib/librte_eal/bsdapp/eal/eal.c | 13 ++++--- lib/librte_eal/bsdapp/eal/eal_thread.c | 63 +++++++++--------------------- lib/librte_eal/linuxapp/eal/eal.c | 7 +++- lib/librte_eal/linuxapp/eal/eal_thread.c | 67 +++++++++++--------------------- 4 files changed, 54 insertions(+), 96 deletions(-)lib/librte_eal/bsdapp/eal/eal.c +8
-3@@ -432,6 +432,7 @@ rte_eal_init(int argc, char **argv)int i, fctret, ret;pthread_t thread_id;static rte_atomic32_t run_once =RTE_ATOMIC32_INIT (0 ); +char cpuset[CPU_STR_LEN ];if (!rte_atomic32_test_and_set (&run_once))return -1 ;@@ -502,13 +503,17 @@ rte_eal_init(int argc, char **argv)if (rte_eal_pci_init () <0 )rte_panic ("Cannot init PCI\n" ); -RTE_LOG (DEBUG ,EAL ,"Master core %u is ready (tid=%p)\n" , - rte_config.master_lcore, thread_id); -eal_check_mem_on_local_socket ();rte_eal_mcfg_complete (); +eal_thread_init_master (rte_config.master_lcore); + +eal_thread_dump_affinity (cpuset,CPU_STR_LEN ); + +RTE_LOG (DEBUG ,EAL ,"Master lcore %u is ready (tid=%p;cpuset=[%s])\n" , + rte_config.master_lcore, thread_id, cpuset); +if (rte_eal_dev_init () <0 )rte_panic ("Cannot init pmd devices\n" );
lib/librte_eal/bsdapp/eal/eal.c rte_eal_init(int argc, char **argv) · line 542
Quoted 8 lines
lib/librte_eal/bsdapp/eal/eal.c +0
-2@@ -532,8 +537,6 @@ rte_eal_init(int argc, char **argv)rte_panic ("Cannot create thread\n" ); } -eal_thread_init_master (rte_config.master_lcore); -/* * Launch a dummy function on all slave lcores, so that master lcore * knows they are all ready when this function returns.
I wonder if changing this may have an impact on third-party drivers that already use a management thread. Before the patch, the init() function of the external library was called with default affinities, and now it's called with the affinity from master lcore.
I think it should at least be noticed in the commit log.
Why are you doing this change? (I don't say it's a bad change, but I don't understand why you are doing it here)
Quoted 44 lines
lib/librte_eal/bsdapp/eal/eal_thread.c +3
-29@@ -103,55 +103,27 @@ eal_thread_set_affinity(void) {int s;pthread_t thread; - -/* -* According to the section VERSIONS of the CPU_ALLOC man page: -* -* The CPU_ZERO(), CPU_SET(), CPU_CLR(), and CPU_ISSET() macros were added -* in glibc 2.3.3. -* -* CPU_COUNT() first appeared in glibc 2.6. -* -* CPU_AND(), CPU_OR(), CPU_XOR(), CPU_EQUAL(), CPU_ALLOC(), -* CPU_ALLOC_SIZE(), CPU_FREE(), CPU_ZERO_S(), CPU_SET_S(), CPU_CLR_S(), -* CPU_ISSET_S(), CPU_AND_S(), CPU_OR_S(), CPU_XOR_S(), and CPU_EQUAL_S() -* first appeared in glibc 2.7. -*/ -#if defined(CPU_ALLOC ) -size_t size; -cpu_set_t *cpusetp; - - cpusetp =CPU_ALLOC (RTE_MAX_LCORE ); -if (cpusetp ==NULL ) { -RTE_LOG (ERR ,EAL ,"CPU_ALLOC failed\n" ); -return -1 ; - } - - size =CPU_ALLOC_SIZE (RTE_MAX_LCORE ); -CPU_ZERO_S (size, cpusetp); -CPU_SET_S (rte_lcore_id (), size, cpusetp); +unsigned lcore_id =rte_lcore_id (); thread =pthread_self (); - s =pthread_setaffinity_np (thread, size, cpusetp); + s =pthread_setaffinity_np (thread,sizeof (cpuset_t), + &lcore_config[lcore_id].cpuset);if (s !=0 ) {RTE_LOG (ERR ,EAL ,"pthread_setaffinity_np failed\n" ); -CPU_FREE (cpusetp);
lib/librte_eal/bsdapp/eal/eal_thread.c eal_thread_set_affinity(void) · line 117
Quoted 10 lines
lib/librte_eal/bsdapp/eal/eal_thread.c +2
-5return -1 ; } -CPU_FREE (cpusetp); -#else /* CPU_ALLOC */ -cpuset_t cpuset; -CPU_ZERO ( &cpuset ); -CPU_SET (rte_lcore_id (), &cpuset ); +/* acquire system unique id */ +rte_gettid ();
As suggested in the previous patch, I think having rte_init_tid() would be clearer here.
Quoted 6 lines
lib/librte_eal/bsdapp/eal/eal_thread.c +6
-0+ +/* store socket_id in TLS for quick access */ +RTE_PER_LCORE (_socket_id) = +eal_cpuset_socket_id (&lcore_config[lcore_id].cpuset); + +CPU_COPY (&lcore_config[lcore_id].cpuset, &RTE_PER_LCORE (_cpuset));
lib/librte_eal/bsdapp/eal/eal_thread.c eal_thread_set_affinity(void) · line 154
Quoted 10 lines
lib/librte_eal/bsdapp/eal/eal_thread.c +2
-7+ + lcore_config[lcore_id].socket_id =RTE_PER_LCORE (_socket_id); - thread =pthread_self (); - s =pthread_setaffinity_np (thread,sizeof ( cpuset ), &cpuset); -if (s !=0 ) { -RTE_LOG (ERR ,EAL ,"pthread_setaffinity_np failed\n" ); -return -1 ; - } -#endif
You are removing a lot of code that was using CPU_ALLOC(). Are we sure that the cpuset_t type is large enough to store all the CPUs?
It looks the current value of CPU_SETSIZE is 1024 now, but I wonder if this code was written when this value was lower. Could you check if it can happen today (maybe with an old libc)? A problem can occur if the size of cpuset_t is lower that the size of RTE_MAX_LCORE.
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH v4 08/17] eal: apply affinity of EAL thread by assigned cpuset
Quoted 71 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Monday, February 09, 2015 4:01 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 08/17] eal: apply affinity of EAL thread by assigned cpusetHi,
Quoted 45 lines · Cunming Liang
EAL threads use assigned cpuset to set core affinity during startup. It keeps 1:1 mapping, if no '--lcores' option is used.
[...]
lib/librte_eal/bsdapp/eal/eal.c | 13 ++++--- lib/librte_eal/bsdapp/eal/eal_thread.c | 63 +++++++++--------------------- lib/librte_eal/linuxapp/eal/eal.c | 7 +++- lib/librte_eal/linuxapp/eal/eal_thread.c | 67 +++++++++++--------------------- 4 files changed, 54 insertions(+), 96 deletions(-)lib/librte_eal/bsdapp/eal/eal.c +8
-3@@ -432,6 +432,7 @@ rte_eal_init(int argc, char **argv)int i, fctret, ret;pthread_t thread_id;static rte_atomic32_t run_once =RTE_ATOMIC32_INIT (0 ); +char cpuset[CPU_STR_LEN ];if (!rte_atomic32_test_and_set (&run_once))return -1 ;@@ -502,13 +503,17 @@ rte_eal_init(int argc, char **argv)if (rte_eal_pci_init () <0 )rte_panic ("Cannot init PCI\n" ); -RTE_LOG (DEBUG ,EAL ,"Master core %u is ready (tid=%p)\n" , - rte_config.master_lcore, thread_id); -eal_check_mem_on_local_socket ();rte_eal_mcfg_complete (); +eal_thread_init_master (rte_config.master_lcore); + +eal_thread_dump_affinity (cpuset,CPU_STR_LEN ); + +RTE_LOG (DEBUG ,EAL ,"Master lcore %u is ready (tid=%p;cpuset=[%s])\n" , + rte_config.master_lcore, thread_id, cpuset); +if (rte_eal_dev_init () <0 )rte_panic ("Cannot init pmd devices\n" );lib/librte_eal/bsdapp/eal/eal.c rte_eal_init(int argc, char **argv) · line 542
Quoted 8 lines
lib/librte_eal/bsdapp/eal/eal.c +0
-2@@ -532,8 +537,6 @@ rte_eal_init(int argc, char **argv)rte_panic ("Cannot create thread\n" ); } -eal_thread_init_master (rte_config.master_lcore); -/* * Launch a dummy function on all slave lcores, so that master lcore * knows they are all ready when this function returns. I wonder if changing this may have an impact on third-party drivers that already use a management thread. Before the patch, the
init()function of the external library was called with default affinities, and now it's called with the affinity from master lcore.
Quoted 4 lines
I think it should at least be noticed in the commit log.
Why are you doing this change? (I don't say it's a bad change, but I don't understand why you are doing it here)
[LCM] To be honest, the main purpose is I don't found any reason to have linuxapp and freebsdapp in different init sequence. I means in linux it init_master before dev_init(), but in freebsd it reverse.
And as the default value of TLS already changes, if dev_init() first and using those TLS, the result will be not in an EAL thread. But actually they're in the EAL master thread. So I prefer to do the change follows linuxapp sequence.
Quoted 54 lines
Review
Quoted 3 lines
(quoted diff) +0
-0
b/lib/librte_eal/bsdapp/eal/eal_thread.clib/librte_eal/bsdapp/eal/eal_thread.c eal_thread_set_affinity(void) · line 110
Quoted 12 lines
index d0c077b..5b16302 100644
lib/librte_eal/bsdapp/eal/eal_thread.c +0
-5@@ -103,55 +103,27 @@ eal_thread_set_affinity(void) {int s;pthread_t thread; - -/* -* According to the section VERSIONS of the CPU_ALLOC man page: -* -* The CPU_ZERO(), CPU_SET(), CPU_CLR(), and CPU_ISSET() macros were added
lib/librte_eal/bsdapp/eal/eal_thread.c eal_thread_set_affinity(void)
Quoted 5 lines
lib/librte_eal/bsdapp/eal/eal_thread.c +0
-5-* in glibc 2.3.3. -* -* CPU_COUNT() first appeared in glibc 2.6. -* -* CPU_AND(), CPU_OR(), CPU_XOR(), CPU_EQUAL(),
CPU_ALLOC(),lib/librte_eal/bsdapp/eal/eal_thread.c eal_thread_set_affinity(void) · line 116
Quoted 1 line
lib/librte_eal/bsdapp/eal/eal_thread.c +0
-1-* CPU_ALLOC_SIZE(), CPU_FREE(), CPU_ZERO_S(), CPU_SET_S(),
CPU_CLR_S(),lib/librte_eal/bsdapp/eal/eal_thread.c eal_thread_set_affinity(void) · line 117
Quoted 1 line
lib/librte_eal/bsdapp/eal/eal_thread.c +0
-1-* CPU_ISSET_S(), CPU_AND_S(), CPU_OR_S(), CPU_XOR_S(), and
CPU_EQUAL_S()Quoted 27 lines
lib/librte_eal/bsdapp/eal/eal_thread.c +3
-17-* first appeared in glibc 2.7. -*/ -#if defined(CPU_ALLOC ) -size_t size; -cpu_set_t *cpusetp; - - cpusetp =CPU_ALLOC (RTE_MAX_LCORE ); -if (cpusetp ==NULL ) { -RTE_LOG (ERR ,EAL ,"CPU_ALLOC failed\n" ); -return -1 ; - } - - size =CPU_ALLOC_SIZE (RTE_MAX_LCORE ); -CPU_ZERO_S (size, cpusetp); -CPU_SET_S (rte_lcore_id (), size, cpusetp); +unsigned lcore_id =rte_lcore_id (); thread =pthread_self (); - s =pthread_setaffinity_np (thread, size, cpusetp); + s =pthread_setaffinity_np (thread,sizeof (cpuset_t), + &lcore_config[lcore_id].cpuset);if (s !=0 ) {RTE_LOG (ERR ,EAL ,"pthread_setaffinity_np failed\n" ); -CPU_FREE (cpusetp);return -1 ; }
Quoted 10 lines
lib/librte_eal/bsdapp/eal/eal_thread.c eal_thread_set_affinity(void) · line 117
Quoted 7 lines
lib/librte_eal/bsdapp/eal/eal_thread.c +2
-5-CPU_FREE (cpusetp); -#else /* CPU_ALLOC */ -cpuset_t cpuset; -CPU_ZERO ( &cpuset ); -CPU_SET (rte_lcore_id (), &cpuset ); +/* acquire system unique id */ +rte_gettid ();As suggested in the previous patch, I think having
rte_init_tid()would be clearer here.
[LCM] Sorry, I didn't get your [PATCH v4 07/17] comments, probably the mailbox issue. Do you suggest to have a rte_init_tid() but not do syscall on the first time ? Any benefit, rte_gettid() looks like more simple and straight forward.
Quoted 21 lines
Quoted 6 lines
lib/librte_eal/bsdapp/eal/eal_thread.c +6
-0+ +/* store socket_id in TLS for quick access */ +RTE_PER_LCORE (_socket_id) = +eal_cpuset_socket_id (&lcore_config[lcore_id].cpuset); + +CPU_COPY (&lcore_config[lcore_id].cpuset, &RTE_PER_LCORE (_cpuset));lib/librte_eal/bsdapp/eal/eal_thread.c eal_thread_set_affinity(void) · line 154
Quoted 10 lines
lib/librte_eal/bsdapp/eal/eal_thread.c +2
-7+ + lcore_config[lcore_id].socket_id =RTE_PER_LCORE (_socket_id); - thread =pthread_self (); - s =pthread_setaffinity_np (thread,sizeof ( cpuset ), &cpuset); -if (s !=0 ) { -RTE_LOG (ERR ,EAL ,"pthread_setaffinity_np failed\n" ); -return -1 ; - } -#endif You are removing a lot of code that was using
CPU_ALLOC(). Are we sure that thecpuset_ttype is large enough to store all the CPUs?
Quoted 4 lines
It looks the current value of
CPU_SETSIZEis 1024 now, but I wonder if this code was written when this value was lower. Could you check if it can happen today (maybe with an old libc)? A problem can occur if the size ofcpuset_tis lower that the size ofRTE_MAX_LCORE.
[LCM] I found actually the MACRO is not just for support CPU_ALLOC(), but for linux or freebsd.
In freebsdapp, there's no CPU_ALLOC defined, it use fixed width *cpuset_t*.
In linuxapp, there's CPU_ALLOC defined, it use cpu_set_t* and dynamic CPU_ALLOC(RTE_MAX_LCORE).
But actually RTE_MAX_LCORE < 1024(sizeof(cpu_set_t)).
After using rte_cpuset_t, there's no additional reason to use CPU_ALLOC only for linuxapp and choose a small but dynamic width.
Quoted 4 lines
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): D0158A423229094DA7ABF71CF2FA0DA3118D7E4A-E2R4CRU6q/[email protected]
Re: [PATCH v4 08/17] eal: apply affinity of EAL thread by assigned cpuset
Hi,
Quoted 69 lines · Cunming
Quoted 69 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Monday, February 09, 2015 4:01 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 08/17] eal: apply affinity of EAL thread by assigned cpusetHi,
Quoted 45 lines · Cunming Liang
EAL threads use assigned cpuset to set core affinity during startup. It keeps 1:1 mapping, if no '--lcores' option is used.
[...]
lib/librte_eal/bsdapp/eal/eal.c | 13 ++++--- lib/librte_eal/bsdapp/eal/eal_thread.c | 63 +++++++++--------------------- lib/librte_eal/linuxapp/eal/eal.c | 7 +++- lib/librte_eal/linuxapp/eal/eal_thread.c | 67 +++++++++++--------------------- 4 files changed, 54 insertions(+), 96 deletions(-)lib/librte_eal/bsdapp/eal/eal.c +8
-3@@ -432,6 +432,7 @@ rte_eal_init(int argc, char **argv)int i, fctret, ret;pthread_t thread_id;static rte_atomic32_t run_once =RTE_ATOMIC32_INIT (0 ); +char cpuset[CPU_STR_LEN ];if (!rte_atomic32_test_and_set (&run_once))return -1 ;@@ -502,13 +503,17 @@ rte_eal_init(int argc, char **argv)if (rte_eal_pci_init () <0 )rte_panic ("Cannot init PCI\n" ); -RTE_LOG (DEBUG ,EAL ,"Master core %u is ready (tid=%p)\n" , - rte_config.master_lcore, thread_id); -eal_check_mem_on_local_socket ();rte_eal_mcfg_complete (); +eal_thread_init_master (rte_config.master_lcore); + +eal_thread_dump_affinity (cpuset,CPU_STR_LEN ); + +RTE_LOG (DEBUG ,EAL ,"Master lcore %u is ready (tid=%p;cpuset=[%s])\n" , + rte_config.master_lcore, thread_id, cpuset); +if (rte_eal_dev_init () <0 )rte_panic ("Cannot init pmd devices\n" );lib/librte_eal/bsdapp/eal/eal.c rte_eal_init(int argc, char **argv) · line 542
Quoted 8 lines
lib/librte_eal/bsdapp/eal/eal.c +0
-2@@ -532,8 +537,6 @@ rte_eal_init(int argc, char **argv)rte_panic ("Cannot create thread\n" ); } -eal_thread_init_master (rte_config.master_lcore); -/* * Launch a dummy function on all slave lcores, so that master lcore * knows they are all ready when this function returns. I wonder if changing this may have an impact on third-party drivers that already use a management thread. Before the patch, the
init()function of the external library was called with default affinities, and now it's called with the affinity from master lcore.
Quoted 6 lines
Quoted 4 lines
I think it should at least be noticed in the commit log.
Why are you doing this change? (I don't say it's a bad change, but I don't understand why you are doing it here)
[LCM] To be honest, the main purpose is I don't found any reason to have linuxapp and freebsdapp in different init sequence. I means in linux it
init_masterbeforedev_init(), but in freebsd it reverse.
I agree that's something we should fix.
Quoted 2 lines
And as the default value of TLS already changes, if
dev_init()first and using those TLS, the result will be not in an EAL thread. But actually they're in the EAL master thread. So I prefer to do the change follows linuxapp sequence.
That makes sense. Is it possible to have this reordering in a separate patch? The title could be
"eal: standardize init sequence between linux and bsd"
Quoted 64 lines
Quoted 54 lines
Review
Quoted 3 lines
(quoted diff) +0
-0
b/lib/librte_eal/bsdapp/eal/eal_thread.clib/librte_eal/bsdapp/eal/eal_thread.c eal_thread_set_affinity(void) · line 110
Quoted 12 lines
index d0c077b..5b16302 100644
lib/librte_eal/bsdapp/eal/eal_thread.c +0
-5@@ -103,55 +103,27 @@ eal_thread_set_affinity(void) {int s;pthread_t thread; - -/* -* According to the section VERSIONS of the CPU_ALLOC man page: -* -* The CPU_ZERO(), CPU_SET(), CPU_CLR(), and CPU_ISSET() macros were added
lib/librte_eal/bsdapp/eal/eal_thread.c eal_thread_set_affinity(void)
Quoted 5 lines
lib/librte_eal/bsdapp/eal/eal_thread.c +0
-5-* in glibc 2.3.3. -* -* CPU_COUNT() first appeared in glibc 2.6. -* -* CPU_AND(), CPU_OR(), CPU_XOR(), CPU_EQUAL(),
CPU_ALLOC(),lib/librte_eal/bsdapp/eal/eal_thread.c eal_thread_set_affinity(void) · line 116
Quoted 1 line
lib/librte_eal/bsdapp/eal/eal_thread.c +0
-1-* CPU_ALLOC_SIZE(), CPU_FREE(), CPU_ZERO_S(), CPU_SET_S(),
CPU_CLR_S(),lib/librte_eal/bsdapp/eal/eal_thread.c eal_thread_set_affinity(void) · line 117
Quoted 1 line
lib/librte_eal/bsdapp/eal/eal_thread.c +0
-1-* CPU_ISSET_S(), CPU_AND_S(), CPU_OR_S(), CPU_XOR_S(), and
CPU_EQUAL_S()Quoted 27 lines
lib/librte_eal/bsdapp/eal/eal_thread.c +3
-17-* first appeared in glibc 2.7. -*/ -#if defined(CPU_ALLOC ) -size_t size; -cpu_set_t *cpusetp; - - cpusetp =CPU_ALLOC (RTE_MAX_LCORE ); -if (cpusetp ==NULL ) { -RTE_LOG (ERR ,EAL ,"CPU_ALLOC failed\n" ); -return -1 ; - } - - size =CPU_ALLOC_SIZE (RTE_MAX_LCORE ); -CPU_ZERO_S (size, cpusetp); -CPU_SET_S (rte_lcore_id (), size, cpusetp); +unsigned lcore_id =rte_lcore_id (); thread =pthread_self (); - s =pthread_setaffinity_np (thread, size, cpusetp); + s =pthread_setaffinity_np (thread,sizeof (cpuset_t), + &lcore_config[lcore_id].cpuset);if (s !=0 ) {RTE_LOG (ERR ,EAL ,"pthread_setaffinity_np failed\n" ); -CPU_FREE (cpusetp);return -1 ; }Quoted 10 lines
lib/librte_eal/bsdapp/eal/eal_thread.c eal_thread_set_affinity(void) · line 117
Quoted 7 lines
lib/librte_eal/bsdapp/eal/eal_thread.c +2
-5-CPU_FREE (cpusetp); -#else /* CPU_ALLOC */ -cpuset_t cpuset; -CPU_ZERO ( &cpuset ); -CPU_SET (rte_lcore_id (), &cpuset ); +/* acquire system unique id */ +rte_gettid ();As suggested in the previous patch, I think having
rte_init_tid()would be clearer here.
Quoted 3 lines
[LCM] Sorry, I didn't get your [PATCH v4 07/17] comments, probably the mailbox issue. Do you suggest to have a
rte_init_tid()but not do syscall on the first time ? Any benefit,rte_gettid()looks like more simple and straight forward.
I think the mail was properly sent, you can see it here:
http://dpdk.org/ml/archives/dev/2015-February/012556.html
Usually, "get" functions return a value and have no side effects. "init" functions return nothing (or an error code) but have a side effect which is to initialize an internal state.
Quoted 25 lines
Quoted 21 lines
Quoted 6 lines
lib/librte_eal/bsdapp/eal/eal_thread.c +6
-0+ +/* store socket_id in TLS for quick access */ +RTE_PER_LCORE (_socket_id) = +eal_cpuset_socket_id (&lcore_config[lcore_id].cpuset); + +CPU_COPY (&lcore_config[lcore_id].cpuset, &RTE_PER_LCORE (_cpuset));lib/librte_eal/bsdapp/eal/eal_thread.c eal_thread_set_affinity(void) · line 154
Quoted 10 lines
lib/librte_eal/bsdapp/eal/eal_thread.c +2
-7+ + lcore_config[lcore_id].socket_id =RTE_PER_LCORE (_socket_id); - thread =pthread_self (); - s =pthread_setaffinity_np (thread,sizeof ( cpuset ), &cpuset); -if (s !=0 ) { -RTE_LOG (ERR ,EAL ,"pthread_setaffinity_np failed\n" ); -return -1 ; - } -#endif You are removing a lot of code that was using
CPU_ALLOC(). Are we sure that thecpuset_ttype is large enough to store all the CPUs?Quoted 4 lines
It looks the current value of
CPU_SETSIZEis 1024 now, but I wonder if this code was written when this value was lower. Could you check if it can happen today (maybe with an old libc)? A problem can occur if the size ofcpuset_tis lower that the size ofRTE_MAX_LCORE.
Quoted 5 lines
[LCM] I found actually the MACRO is not just for support
CPU_ALLOC(), but for linux or freebsd.
In freebsdapp, there's noCPU_ALLOCdefined, it use fixed width *cpuset_t*.
In linuxapp, there'sCPU_ALLOCdefined, it use cpu_set_t* and dynamicCPU_ALLOC(RTE_MAX_LCORE).
But actuallyRTE_MAX_LCORE< 1024(sizeof(cpu_set_t)).
After usingrte_cpuset_t, there's no additional reason to useCPU_ALLOConly for linuxapp and choose a small but dynamic width.
I did a quick search on google, and it seems CPU_SETSIZE is 1024 for a long time. So you are right, there is probably no reason to keep CPU_ALLOC(). As I said in a previous mail, it could be useful in the future when the number of CPUs will reach 1024, but we have some time to handle this.
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH v4 08/17] eal: apply affinity of EAL thread by assigned cpuset
Quoted 94 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Tuesday, February 10, 2015 1:37 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 08/17] eal: apply affinity of EAL thread by assigned cpusetHi,
Quoted 42 lines · Cunming
Quoted 42 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Monday, February 09, 2015 4:01 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 08/17] eal: apply affinity of EAL thread by assigned cpusetHi,
Quoted 32 lines · Cunming Liang
EAL threads use assigned cpuset to set core affinity during startup. It keeps 1:1 mapping, if no '--lcores' option is used.
[...]
lib/librte_eal/bsdapp/eal/eal.c | 13 ++++--- lib/librte_eal/bsdapp/eal/eal_thread.c | 63 +++++++++--------------------- lib/librte_eal/linuxapp/eal/eal.c | 7 +++- lib/librte_eal/linuxapp/eal/eal_thread.c | 67 +++++++++++--------------------- 4 files changed, 54 insertions(+), 96 deletions(-)lib/librte_eal/bsdapp/eal/eal.c +1
-3@@ -432,6 +432,7 @@ rte_eal_init(int argc, char **argv)int i, fctret, ret;pthread_t thread_id;static rte_atomic32_t run_once =RTE_ATOMIC32_INIT (0 ); +char cpuset[CPU_STR_LEN ];if (!rte_atomic32_test_and_set (&run_once))return -1 ;@@ -502,13 +503,17 @@ rte_eal_init(int argc, char **argv)if (rte_eal_pci_init () <0 )rte_panic ("Cannot init PCI\n" ); -RTE_LOG (DEBUG ,EAL ,"Master core %u is ready (tid=%p)\n" , - rte_config.master_lcore, thread_id); -eal_check_mem_on_local_socket ();Quoted 7 lines
Quoted 7 lines
lib/librte_eal/bsdapp/eal/eal.c rte_eal_init(int argc, char **argv) · line 514
Quoted 7 lines
lib/librte_eal/bsdapp/eal/eal.c +5
-0rte_eal_mcfg_complete (); +eal_thread_init_master (rte_config.master_lcore); + +eal_thread_dump_affinity (cpuset,CPU_STR_LEN ); + + RTE_LOG(DEBUG, EAL, "Master lcore %u is ready(tid=%p;cpuset=[%s])\n",
Quoted 20 lines
Quoted 20 lines
Quoted 6 lines
lib/librte_eal/bsdapp/eal/eal.c +2
-0+ rte_config.master_lcore, thread_id, cpuset); +if (rte_eal_dev_init () <0 )rte_panic ("Cannot init pmd devices\n" );lib/librte_eal/bsdapp/eal/eal.c rte_eal_init(int argc, char **argv) · line 542
Quoted 8 lines
lib/librte_eal/bsdapp/eal/eal.c +0
-2@@ -532,8 +537,6 @@ rte_eal_init(int argc, char **argv)rte_panic ("Cannot create thread\n" ); } -eal_thread_init_master (rte_config.master_lcore); -/* * Launch a dummy function on all slave lcores, so that master lcore * knows they are all ready when this function returns. I wonder if changing this may have an impact on third-party drivers that already use a management thread. Before the patch, the
init()function of the external library was called with default affinities, and now it's called with the affinity from master lcore.Quoted 5 lines
Quoted 4 lines
I think it should at least be noticed in the commit log.
Why are you doing this change? (I don't say it's a bad change, but I don't understand why you are doing it here)
[LCM] To be honest, the main purpose is I don't found any reason to have
linuxapp and freebsdapp in different init sequence.
Quoted 1 line
I means in linux it
init_masterbeforedev_init(), but in freebsd it reverse.I agree that's something we should fix.
Quoted 8 lines
Quoted 1 line
And as the default value of TLS already changes, if
dev_init()first and usingthose TLS, the result will be not in an EAL thread.
Quoted 1 line
But actually they're in the EAL master thread. So I prefer to do the change
follows linuxapp sequence.
That makes sense. Is it possible to have this reordering in a separate patch? The title could be
"eal: standardize init sequence between linux and bsd"
[LCM] Agree.
Quoted 72 lines
Quoted 67 lines
Quoted 57 lines
Review
Quoted 6 lines
(quoted diff) +0
-0
b/lib/librte_eal/bsdapp/eal/eal_thread.clib/librte_eal/bsdapp/eal/eal_thread.c eal_thread_set_affinity(void) · line 110
Quoted 12 lines
index d0c077b..5b16302 100644
lib/librte_eal/bsdapp/eal/eal_thread.c +0
-5@@ -103,55 +103,27 @@ eal_thread_set_affinity(void) {int s;pthread_t thread; - -/* -* According to the section VERSIONS of the CPU_ALLOC man page: -* -* The CPU_ZERO(), CPU_SET(), CPU_CLR(), and CPU_ISSET() macros were added
lib/librte_eal/bsdapp/eal/eal_thread.c eal_thread_set_affinity(void) · line 115
Quoted 5 lines
lib/librte_eal/bsdapp/eal/eal_thread.c +0
-5-* in glibc 2.3.3. -* -* CPU_COUNT() first appeared in glibc 2.6. -* -* CPU_AND(), CPU_OR(), CPU_XOR(), CPU_EQUAL(),
CPU_ALLOC(),lib/librte_eal/bsdapp/eal/eal_thread.c eal_thread_set_affinity(void) · line 116
Quoted 1 line
lib/librte_eal/bsdapp/eal/eal_thread.c +0
-1-* CPU_ALLOC_SIZE(), CPU_FREE(), CPU_ZERO_S(), CPU_SET_S(),
CPU_CLR_S(),lib/librte_eal/bsdapp/eal/eal_thread.c eal_thread_set_affinity(void) · line 117
Quoted 1 line
lib/librte_eal/bsdapp/eal/eal_thread.c +0
-1-* CPU_ISSET_S(), CPU_AND_S(), CPU_OR_S(), CPU_XOR_S(), and
CPU_EQUAL_S()Quoted 27 lines
lib/librte_eal/bsdapp/eal/eal_thread.c +3
-17-* first appeared in glibc 2.7. -*/ -#if defined(CPU_ALLOC ) -size_t size; -cpu_set_t *cpusetp; - - cpusetp =CPU_ALLOC (RTE_MAX_LCORE ); -if (cpusetp ==NULL ) { -RTE_LOG (ERR ,EAL ,"CPU_ALLOC failed\n" ); -return -1 ; - } - - size =CPU_ALLOC_SIZE (RTE_MAX_LCORE ); -CPU_ZERO_S (size, cpusetp); -CPU_SET_S (rte_lcore_id (), size, cpusetp); +unsigned lcore_id =rte_lcore_id (); thread =pthread_self (); - s =pthread_setaffinity_np (thread, size, cpusetp); + s =pthread_setaffinity_np (thread,sizeof (cpuset_t), + &lcore_config[lcore_id].cpuset);if (s !=0 ) {RTE_LOG (ERR ,EAL ,"pthread_setaffinity_np failed\n" ); -CPU_FREE (cpusetp);return -1 ; }Quoted 10 lines
lib/librte_eal/bsdapp/eal/eal_thread.c eal_thread_set_affinity(void) · line 117
Quoted 7 lines
lib/librte_eal/bsdapp/eal/eal_thread.c +2
-5-CPU_FREE (cpusetp); -#else /* CPU_ALLOC */ -cpuset_t cpuset; -CPU_ZERO ( &cpuset ); -CPU_SET (rte_lcore_id (), &cpuset ); +/* acquire system unique id */ +rte_gettid ();As suggested in the previous patch, I think having
rte_init_tid()would be clearer here.Quoted 1 line
[LCM] Sorry, I didn't get your [PATCH v4 07/17] comments, probably the
mailbox issue.
Quoted 2 lines
Do you suggest to have a
rte_init_tid()but not do syscall on the first time ? Any benefit,rte_gettid()looks like more simple and straight forward.
Quoted 6 lines
I think the mail was properly sent, you can see it here:
http://dpdk.org/ml/archives/dev/2015-February/012556.htmlUsually, "get" functions return a value and have no side effects. "init" functions return nothing (or an error code) but have a side effect which is to initialize an internal state.
[LCM] Thanks, agree your point, will update on v5.
Quoted 34 lines
Quoted 23 lines
Quoted 23 lines
Quoted 8 lines
lib/librte_eal/bsdapp/eal/eal_thread.c +6
-0+ +/* store socket_id in TLS for quick access */ +RTE_PER_LCORE (_socket_id) = +eal_cpuset_socket_id (&lcore_config[lcore_id].cpuset); + +CPU_COPY (&lcore_config[lcore_id].cpuset, &RTE_PER_LCORE (_cpuset));lib/librte_eal/bsdapp/eal/eal_thread.c eal_thread_set_affinity(void) · line 154
Quoted 10 lines
lib/librte_eal/bsdapp/eal/eal_thread.c +2
-7+ + lcore_config[lcore_id].socket_id =RTE_PER_LCORE (_socket_id); - thread =pthread_self (); - s =pthread_setaffinity_np (thread,sizeof ( cpuset ), &cpuset); -if (s !=0 ) { -RTE_LOG (ERR ,EAL ,"pthread_setaffinity_np failed\n" ); -return -1 ; - } -#endif You are removing a lot of code that was using
CPU_ALLOC(). Are we sure that thecpuset_ttype is large enough to store all the CPUs?Quoted 5 lines
Quoted 4 lines
It looks the current value of
CPU_SETSIZEis 1024 now, but I wonder if this code was written when this value was lower. Could you check if it can happen today (maybe with an old libc)? A problem can occur if the size ofcpuset_tis lower that the size ofRTE_MAX_LCORE.[LCM] I found actually the MACRO is not just for support
CPU_ALLOC(), but forlinux or freebsd.
Quoted 2 lines
In freebsdapp, there's no
CPU_ALLOCdefined, it use fixed width *cpuset_t*. In linuxapp, there'sCPU_ALLOCdefined, it use cpu_set_t* and dynamic
CPU_ALLOC(RTE_MAX_LCORE).Quoted 2 lines
But actually
RTE_MAX_LCORE< 1024(sizeof(cpu_set_t)).
After usingrte_cpuset_t, there's no additional reason to useCPU_ALLOConly
Quoted 7 lines
for linuxapp and choose a small but dynamic width.
I did a quick search on google, and it seems
CPU_SETSIZEis 1024 for a long time. So you are right, there is probably no reason to keepCPU_ALLOC(). As I said in a previous mail, it could be useful in the future when the number of CPUs will reach 1024, but we have some time to handle this.
[LCM] Ok, thanks.
Quoted 3 lines
PATCH v4 9/17 enic: fix re-define freebsd compile complain
In reply to a message not in the archiveSome macro already been defined by freebsd 'sys/param.h'.
--- lib/librte_pmd_enic/enic.h | 1 + lib/librte_pmd_enic/enic_compat.h | 1 + 2 files changed, 2 insertions(+)
lib/librte_pmd_enic/enic.h +1 -0
@@ -66,6 +66,7 @@#define ENIC_CALC_IP_CKSUM 1#define ENIC_CALC_TCP_UDP_CKSUM 2#define ENIC_MAX_MTU 9000 +#undef PAGE_SIZE#define PAGE_SIZE 4096#define PAGE_ROUND_UP (x) \ ((((unsigned long)(x)) + PAGE_SIZE-1) & (~(PAGE_SIZE-1)))
lib/librte_pmd_enic/enic_compat.h +1 -0
@@ -67,6 +67,7 @@ #define pr_warn(y, args...) dev_warning(0, y, ##args)#define BUG () pr_err("BUG at %s:%d", __func__, __LINE__) +#undef ALIGN#define ALIGN (x, a) __ALIGN_MASK(x, (typeof(x))(a)-1)#define __ALIGN_MASK (x, mask) (((x)+(mask))&~(mask))#define udelay usleep
Signature · 1 line
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 1422842559-13617-10-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
Re: [PATCH v4 09/17] enic: fix re-define freebsd compile complain
Hi,
Quoted 26 lines · Cunming Liang
Some macro already been defined by freebsd '
sys/param.h'.Signed-off-by Cunming Liang <[email protected]>--- lib/librte_pmd_enic/enic.h | 1 + lib/librte_pmd_enic/enic_compat.h | 1 + 2 files changed, 2 insertions(+)lib/librte_pmd_enic/enic.h +1
-0@@ -66,6 +66,7 @@#define ENIC_CALC_IP_CKSUM 1#define ENIC_CALC_TCP_UDP_CKSUM 2#define ENIC_MAX_MTU 9000 +#undef PAGE_SIZE#define PAGE_SIZE 4096#define PAGE_ROUND_UP (x) \ ((((unsigned long)(x)) + PAGE_SIZE-1) & (~(PAGE_SIZE-1)))lib/librte_pmd_enic/enic_compat.h +0
-0
lib/librte_pmd_enic/enic_compat.h line 73
Quoted 7 lines
lib/librte_pmd_enic/enic_compat.h +1
-0@@ -67,6 +67,7 @@ #define pr_warn(y, args...) dev_warning(0, y, ##args)#define BUG () pr_err("BUG at %s:%d", __func__, __LINE__) +#undef ALIGN#define ALIGN (x, a) __ALIGN_MASK(x, (typeof(x))(a)-1)#define __ALIGN_MASK (x, mask) (((x)+(mask))&~(mask))#define udelay usleep
Is the issue caused by a change you've made previously in the patch series?
Wouldn't it be better to rename the macros in enic instead of doing #undef?
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH v4 09/17] enic: fix re-define freebsd compile complain
Quoted 41 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Monday, February 09, 2015 4:01 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 09/17] enic: fix re-define freebsd compile complainHi,
Review
Quoted 21 lines · Cunming Liang
Some macro already been defined by freebsd '
sys/param.h'.Signed-off-by Cunming Liang <[email protected]>--- lib/librte_pmd_enic/enic.h | 1 + lib/librte_pmd_enic/enic_compat.h | 1 + 2 files changed, 2 insertions(+)lib/librte_pmd_enic/enic.h +1
-0@@ -66,6 +66,7 @@#define ENIC_CALC_IP_CKSUM 1#define ENIC_CALC_TCP_UDP_CKSUM 2#define ENIC_MAX_MTU 9000 +#undef PAGE_SIZE#define PAGE_SIZE 4096#define PAGE_ROUND_UP (x) \ ((((unsigned long)(x)) + PAGE_SIZE-1) & (~(PAGE_SIZE-1)))(quoted diff) +0
-0
b/lib/librte_pmd_enic/enic_compat.hQuoted 7 lines
index b1af838..b84c766 100644
lib/librte_pmd_enic/enic_compat.h +0
-0@@ -67,6 +67,7 @@ #define pr_warn(y, args...) dev_warning(0, y, ##args)#define BUG () pr_err("BUG at %s:%d", __func__, __LINE__)
Quoted 8 lines
lib/librte_pmd_enic/enic_compat.h line 73
Quoted 5 lines
lib/librte_pmd_enic/enic_compat.h +1
-0+#undef ALIGN#define ALIGN (x, a) __ALIGN_MASK(x, (typeof(x))(a)-1)#define __ALIGN_MASK (x, mask) (((x)+(mask))&~(mask))#define udelay usleepIs the issue caused by a change you've made previously in the patch series?
[LCM] Yes, caused by [01/17] which include <pthread_np.h> in freebsdapp.
Quoted 3 lines
Wouldn't it be better to rename the macros in enic instead of doing #undef?
[LCM] Agree, will do it.
Quoted 3 lines
Regards,
Olivier
PATCH v4 10/17 malloc: fix the issue of SOCKET_ID_ANY
In reply to a message not in the archiveAdd check for rte_socket_id(), avoid get unexpected return like (-1).
--- lib/librte_malloc/malloc_heap.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)
lib/librte_malloc/malloc_heap.h +6 -1
@@ -44,7 +44,12 @@ extern "C" {static inline unsigned malloc_get_numa_socket (void ) { -return rte_socket_id (); +unsigned socket_id =rte_socket_id (); + +if (socket_id == (unsigned )SOCKET_ID_ANY ) +return 0 ; + +return socket_id; } void *
Signature · 1 line
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 1422842559-13617-11-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
Re: [PATCH v4 10/17] malloc: fix the issue of SOCKET_ID_ANY
Hi,
Quoted 16 lines · Cunming Liang
Add check for
rte_socket_id(), avoid get unexpected return like (-1).Signed-off-by Cunming Liang <[email protected]>--- lib/librte_malloc/malloc_heap.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)lib/librte_malloc/malloc_heap.h +0
-0@@ -44,7 +44,12 @@ extern "C" {static inline unsigned malloc_get_numa_socket (void ) {
lib/librte_malloc/malloc_heap.h extern "C" { · line 55
Quoted 10 lines
lib/librte_malloc/malloc_heap.h +6
-1-return rte_socket_id (); +unsigned socket_id =rte_socket_id (); + +if (socket_id == (unsigned )SOCKET_ID_ANY ) +return 0 ; + +return socket_id; } void *
The documentation off rte_malloc_socket() says:
@param socket
NUMA socket to allocate memory on. If SOCKET_ID_ANY is used, this function will behave the same as rte_malloc().
void *rte_malloc_socket(const char *type, size_t size, unsigned align, int socket);
Your patch changes the behavior of rte_malloc() without explaining why, and the documentation becomes wrong.
Can you explain why you need this change?
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH v4 10/17] malloc: fix the issue of SOCKET_ID_ANY
Quoted 49 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Monday, February 09, 2015 4:01 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 10/17] malloc: fix the issue ofSOCKET_ID_ANYHi,
Quoted 16 lines · Cunming Liang
Add check for
rte_socket_id(), avoid get unexpected return like (-1).Signed-off-by Cunming Liang <[email protected]>--- lib/librte_malloc/malloc_heap.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)lib/librte_malloc/malloc_heap.h +0
-0@@ -44,7 +44,12 @@ extern "C" {static inline unsigned malloc_get_numa_socket (void ) {lib/librte_malloc/malloc_heap.h extern "C" { · line 55
Quoted 10 lines
lib/librte_malloc/malloc_heap.h +6
-1-return rte_socket_id (); +unsigned socket_id =rte_socket_id (); + +if (socket_id == (unsigned )SOCKET_ID_ANY ) +return 0 ; + +return socket_id; } void *The documentation off
rte_malloc_socket()says:@param socket
NUMA socket to allocate memory on. IfSOCKET_ID_ANYis used, this function will behave the same asrte_malloc().void *
rte_malloc_socket(const char *type, size_t size, unsigned align, int socket);
Quoted 4 lines
Your patch changes the behavior of
rte_malloc()without explaining why, and the documentation becomes wrong.Can you explain why you need this change?
[LCM] I don't think I change the declaration of rte_malloc_socket().
If socket_arg=SOCKET_ID_ANY, the socket value expect to the return value of malloc_get_numa_socket().
The malloc_get_numa_socket() supposed to return the correct TLS _socket_id.
It works fine for normal cases. But as we change the default value of TLS _socket_id to SOCKET_ID_ANY.
And one lcore can run on multiple cpu, if all cpus in the cpuset are not belongs to one NUMA node, the _socket_id would be SOCKET_ID_ANY. When user call rte_malloc_socket(SOCKET_ID_ANY), it does provide the same behavior as rte_malloc().
They both will get socket_id from malloc_get_numa_socket(). The addition part is the exception path process.
Quoted 3 lines
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): D0158A423229094DA7ABF71CF2FA0DA3118D7E9C-E2R4CRU6q/[email protected]
Re: [PATCH v4 10/17] malloc: fix the issue of SOCKET_ID_ANY
Hi,
Quoted 52 lines · Cunming
Quoted 52 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Monday, February 09, 2015 4:01 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 10/17] malloc: fix the issue ofSOCKET_ID_ANYHi,
Quoted 16 lines · Cunming Liang
Add check for
rte_socket_id(), avoid get unexpected return like (-1).Signed-off-by Cunming Liang <[email protected]>--- lib/librte_malloc/malloc_heap.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)lib/librte_malloc/malloc_heap.h +0
-0@@ -44,7 +44,12 @@ extern "C" {static inline unsigned malloc_get_numa_socket (void ) {lib/librte_malloc/malloc_heap.h extern "C" { · line 55
Quoted 10 lines
lib/librte_malloc/malloc_heap.h +6
-1-return rte_socket_id (); +unsigned socket_id =rte_socket_id (); + +if (socket_id == (unsigned )SOCKET_ID_ANY ) +return 0 ; + +return socket_id; } void *The documentation off
rte_malloc_socket()says:@param socket
NUMA socket to allocate memory on. IfSOCKET_ID_ANYis used, this function will behave the same asrte_malloc().void *
rte_malloc_socket(const char *type, size_t size, unsigned align, int socket);Your patch changes the behavior of
rte_malloc()without explaining why, and the documentation becomes wrong.
Quoted 8 lines
Quoted 1 line
Can you explain why you need this change?
[LCM] I don't think I change the declaration of
rte_malloc_socket().
If socket_arg=SOCKET_ID_ANY, the socket value expect to the return value ofmalloc_get_numa_socket().
Themalloc_get_numa_socket()supposed to return the correct TLS_socket_id.
It works fine for normal cases. But as we change the default value of TLS_socket_idtoSOCKET_ID_ANY.
And one lcore can run on multiple cpu, if all cpus in the cpuset are not belongs to one NUMA node, the_socket_idwould beSOCKET_ID_ANY. When user callrte_malloc_socket(SOCKET_ID_ANY), it does provide the same behavior asrte_malloc().
They both will getsocket_idfrommalloc_get_numa_socket(). The addition part is the exception path process.
Sorry, I checked again, you are right.
PATCH v4 11/17 log: fix the gap to support non-EAL thread
In reply to a message not in the archiveFor those non-EAL thread, *_lcore_id* is invalid and probably larger than RTE_MAX_LCORE. The patch adds the check and allows only EAL thread using EAL per thread log level and log type. Others shares the global log level.
--- lib/librte_eal/common/eal_common_log.c | 17 +++++++++++++++-- lib/librte_eal/common/include/rte_log.h | 5 +++++ 2 files changed, 20 insertions(+), 2 deletions(-)
lib/librte_eal/common/eal_common_log.c +15 -2
@@ -193,11 +193,20 @@ rte_set_log_type(uint32_t type, int enable) rte_logs.type &= (~type); } +/* Get global log type */ +uint32_t +rte_get_log_type (void ) +{ +return rte_logs.type; +} +/* get the current loglevel for the message beeing processed */ int rte_log_cur_msg_loglevel (void ) {unsigned lcore_id; lcore_id =rte_lcore_id (); +if (lcore_id >=RTE_MAX_LCORE ) +return rte_get_log_level ();return log_cur_msg[lcore_id].loglevel; }
@@ -206,6 +215,8 @@ int rte_log_cur_msg_logtype(void) {unsigned lcore_id; lcore_id =rte_lcore_id (); +if (lcore_id >=RTE_MAX_LCORE ) +return rte_get_log_type ();return log_cur_msg[lcore_id].logtype; }
@@ -265,8 +276,10 @@ rte_vlog(__attribute__((unused)) uint32_t level,/* save loglevel and logtype in a global per-lcore variable */ lcore_id =rte_lcore_id (); - log_cur_msg[lcore_id].loglevel = level; - log_cur_msg[lcore_id].logtype = logtype; +if (lcore_id <RTE_MAX_LCORE ) { + log_cur_msg[lcore_id].loglevel = level; + log_cur_msg[lcore_id].logtype = logtype; + } ret =vfprintf (f, format, ap);fflush (f);
lib/librte_eal/common/include/rte_log.h +5 -0
@@ -144,6 +144,11 @@ uint32_t rte_get_log_level(void);void rte_set_log_type (uint32_t type,int enable);/** +* Get the global log type. +*/ +uint32_t rte_get_log_type (void ); + +/** * Get the current loglevel for the message being processed. * * Before calling the user-defined stream for logging, the log
Signature · 1 line
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 1422842559-13617-12-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
Re: [PATCH v4 11/17] log: fix the gap to support non-EAL thread
Hi,
Quoted 64 lines · Cunming Liang
For those non-EAL thread, *_lcore_id* is invalid and probably larger than
RTE_MAX_LCORE. The patch adds the check and allows only EAL thread using EAL per thread log level and log type. Others shares the global log level.Signed-off-by Cunming Liang <[email protected]>--- lib/librte_eal/common/eal_common_log.c | 17 +++++++++++++++-- lib/librte_eal/common/include/rte_log.h | 5 +++++ 2 files changed, 20 insertions(+), 2 deletions(-)lib/librte_eal/common/eal_common_log.c +15
-2@@ -193,11 +193,20 @@ rte_set_log_type(uint32_t type, int enable) rte_logs.type &= (~type); } +/* Get global log type */ +uint32_t +rte_get_log_type (void ) +{ +return rte_logs.type; +} +/* get the current loglevel for the message beeing processed */ int rte_log_cur_msg_loglevel (void ) {unsigned lcore_id; lcore_id =rte_lcore_id (); +if (lcore_id >=RTE_MAX_LCORE ) +return rte_get_log_level ();return log_cur_msg[lcore_id].loglevel; }@@ -206,6 +215,8 @@ int rte_log_cur_msg_logtype(void) {unsigned lcore_id; lcore_id =rte_lcore_id (); +if (lcore_id >=RTE_MAX_LCORE ) +return rte_get_log_type ();return log_cur_msg[lcore_id].logtype; }@@ -265,8 +276,10 @@ rte_vlog(__attribute__((unused)) uint32_t level,/* save loglevel and logtype in a global per-lcore variable */ lcore_id =rte_lcore_id (); - log_cur_msg[lcore_id].loglevel = level; - log_cur_msg[lcore_id].logtype = logtype; +if (lcore_id <RTE_MAX_LCORE ) { + log_cur_msg[lcore_id].loglevel = level; + log_cur_msg[lcore_id].logtype = logtype; + } ret =vfprintf (f, format, ap);fflush (f);lib/librte_eal/common/include/rte_log.h +0
-0@@ -144,6 +144,11 @@ uint32_t rte_get_log_level(void);void rte_set_log_type (uint32_t type,int enable);
lib/librte_eal/common/include/rte_log.h uint32_t rte_get_log_level(void); · line 154
Quoted 10 lines
lib/librte_eal/common/include/rte_log.h +5
-0/** +* Get the global log type. +*/ +uint32_t rte_get_log_type (void ); + +/** * Get the current loglevel for the message being processed. * * Before calling the user-defined stream for logging, the log
Wouldn't it be better to change the variable:
static struct log_cur_msg log_cur_msg[RTE_MAX_LCORE]; into a pthread (tls) variable?
With your patch, the log level and log type are not saved for non-EAL threads. If TLS were used, I think it would work in any case.
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH v4 11/17] log: fix the gap to support non-EAL thread
Quoted 95 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Monday, February 09, 2015 4:01 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 11/17] log: fix the gap to support non-EAL threadHi,
Quoted 1 line · Cunming Liang
For those non-EAL thread, *_lcore_id* is invalid and probably larger than
RTE_MAX_LCORE.Quoted 1 line
The patch adds the check and allows only EAL thread using EAL per thread log
level and log type.
Review
Quoted 9 lines
Others shares the global log level.
Signed-off-by Cunming Liang <[email protected]>--- lib/librte_eal/common/eal_common_log.c | 17 +++++++++++++++-- lib/librte_eal/common/include/rte_log.h | 5 +++++ 2 files changed, 20 insertions(+), 2 deletions(-)(quoted diff) +0
-0
b/lib/librte_eal/common/eal_common_log.cReview
Quoted 47 lines
index cf57619..e8dc94a 100644
lib/librte_eal/common/eal_common_log.c +15
-2@@ -193,11 +193,20 @@ rte_set_log_type(uint32_t type, int enable) rte_logs.type &= (~type); } +/* Get global log type */ +uint32_t +rte_get_log_type (void ) +{ +return rte_logs.type; +} +/* get the current loglevel for the message beeing processed */ int rte_log_cur_msg_loglevel (void ) {unsigned lcore_id; lcore_id =rte_lcore_id (); +if (lcore_id >=RTE_MAX_LCORE ) +return rte_get_log_level ();return log_cur_msg[lcore_id].loglevel; }@@ -206,6 +215,8 @@ int rte_log_cur_msg_logtype(void) {unsigned lcore_id; lcore_id =rte_lcore_id (); +if (lcore_id >=RTE_MAX_LCORE ) +return rte_get_log_type ();return log_cur_msg[lcore_id].logtype; }@@ -265,8 +276,10 @@ rte_vlog(__attribute__((unused)) uint32_t level,/* save loglevel and logtype in a global per-lcore variable */ lcore_id =rte_lcore_id (); - log_cur_msg[lcore_id].loglevel = level; - log_cur_msg[lcore_id].logtype = logtype; +if (lcore_id <RTE_MAX_LCORE ) { + log_cur_msg[lcore_id].loglevel = level; + log_cur_msg[lcore_id].logtype = logtype; + } ret =vfprintf (f, format, ap);fflush (f);(quoted diff) +0
-0
b/lib/librte_eal/common/include/rte_log.hQuoted 6 lines
index db1ea08..f83a0d9 100644
lib/librte_eal/common/include/rte_log.h +0
-0@@ -144,6 +144,11 @@ uint32_t rte_get_log_level(void);void rte_set_log_type (uint32_t type,int enable);lib/librte_eal/common/include/rte_log.h uint32_t rte_get_log_level(void); · line 154
Quoted 10 lines
lib/librte_eal/common/include/rte_log.h +5
-0/** +* Get the global log type. +*/ +uint32_t rte_get_log_type (void ); + +/** * Get the current loglevel for the message being processed. * * Before calling the user-defined stream for logging, the log Wouldn't it be better to change the variable:
static structlog_cur_msglog_cur_msg[RTE_MAX_LCORE]; into a pthread (tls) variable?
Quoted 2 lines
With your patch, the log level and log type are not saved for non-EAL threads. If TLS were used, I think it would work in any case.
[LCM] Good point. But for this patch set, still suppose not involve big impact to EAL thread. For improve non-EAL thread, we'll have a separate patch set for it.
Quoted 3 lines
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): D0158A423229094DA7ABF71CF2FA0DA3118D7ECD-E2R4CRU6q/[email protected]
Re: [PATCH v4 11/17] log: fix the gap to support non-EAL thread
Hi,
Quoted 20 lines · Cunming
Quoted 20 lines
Quoted 5 lines
lib/librte_eal/common/include/rte_log.h +0
-0@@ -144,6 +144,11 @@ uint32_t rte_get_log_level(void);void rte_set_log_type (uint32_t type,int enable);lib/librte_eal/common/include/rte_log.h uint32_t rte_get_log_level(void); · line 154
Quoted 10 lines
lib/librte_eal/common/include/rte_log.h +5
-0/** +* Get the global log type. +*/ +uint32_t rte_get_log_type (void ); + +/** * Get the current loglevel for the message being processed. * * Before calling the user-defined stream for logging, the log Wouldn't it be better to change the variable:
static structlog_cur_msglog_cur_msg[RTE_MAX_LCORE]; into a pthread (tls) variable?
Quoted 4 lines
Quoted 2 lines
With your patch, the log level and log type are not saved for non-EAL threads. If TLS were used, I think it would work in any case.
[LCM] Good point. But for this patch set, still suppose not involve big impact to EAL thread. For improve non-EAL thread, we'll have a separate patch set for it.
OK, that's fine
Will it be for 2.0 or later?
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH v4 11/17] log: fix the gap to support non-EAL thread
Quoted 34 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Tuesday, February 10, 2015 1:45 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 11/17] log: fix the gap to support non-EAL threadHi,
Quoted 22 lines · Cunming
Quoted 22 lines
Quoted 5 lines
lib/librte_eal/common/include/rte_log.h +0
-0@@ -144,6 +144,11 @@ uint32_t rte_get_log_level(void);void rte_set_log_type (uint32_t type,int enable);lib/librte_eal/common/include/rte_log.h uint32_t rte_get_log_level(void); · line 154
Quoted 10 lines
lib/librte_eal/common/include/rte_log.h +5
-0/** +* Get the global log type. +*/ +uint32_t rte_get_log_type (void ); + +/** * Get the current loglevel for the message being processed. * * Before calling the user-defined stream for logging, the log Wouldn't it be better to change the variable:
static structlog_cur_msglog_cur_msg[RTE_MAX_LCORE]; into a pthread (tls) variable?With your patch, the log level and log type are not saved for non-EAL threads. If TLS were used, I think it would work in any case.
Quoted 7 lines
Quoted 1 line
[LCM] Good point. But for this patch set, still suppose not involve big impact to
EAL thread.
Quoted 1 line
For improve non-EAL thread, we'll have a separate patch set for it.
OK, that's fine
Will it be for 2.0 or later?
[LCM] Will be in 2.1 I suppose, together with the patch for mempool cache to support non-EAL thread.
PATCH v4 12/17 eal: set _lcore_id and _socket_id to (-1) by default
In reply to a message not in the archiveFor those none EAL thread, *_lcore_id* shall always be LCORE_ID_ANY. The libraries using *_lcore_id* as index need to take care. *_socket_id* always be SOCKET_ID_ANY unitl the thread changes the affinity by rte_thread_set_affinity()
--- lib/librte_eal/bsdapp/eal/eal_thread.c | 4 ++-- lib/librte_eal/linuxapp/eal/eal_thread.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-)
lib/librte_eal/bsdapp/eal/eal_thread.c +2 -2
@@ -56,8 +56,8 @@#include "eal_private.h" #include "eal_thread.h" -RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); -RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (unsigned, _lcore_id) = (unsigned )LCORE_ID_ANY ; +RTE_DEFINE_PER_LCORE (unsigned, _socket_id) = (unsigned )SOCKET_ID_ANY ;RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);/*
lib/librte_eal/linuxapp/eal/eal_thread.c +2 -2
@@ -57,8 +57,8 @@#include "eal_private.h" #include "eal_thread.h" -RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); -RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (unsigned, _lcore_id) = (unsigned )LCORE_ID_ANY ; +RTE_DEFINE_PER_LCORE (unsigned, _socket_id) = (unsigned )SOCKET_ID_ANY ;RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);/*
Signature · 1 line
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 1422842559-13617-13-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
Re: [PATCH v4 12/17] eal: set _lcore_id and _socket_id to (-1) by default
Hi,
Quoted 3 lines · Cunming Liang
For those none EAL thread, *_lcore_id* shall always be
LCORE_ID_ANY. The libraries using *_lcore_id* as index need to take care. *_socket_id* always beSOCKET_ID_ANYunitl the thread changes the affinity
unitl -> until
Quoted 28 lines
by
rte_thread_set_affinity()Signed-off-by Cunming Liang <[email protected]>--- lib/librte_eal/bsdapp/eal/eal_thread.c | 4 ++-- lib/librte_eal/linuxapp/eal/eal_thread.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-)lib/librte_eal/bsdapp/eal/eal_thread.c +2
-2@@ -56,8 +56,8 @@#include "eal_private.h" #include "eal_thread.h" -RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); -RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (unsigned, _lcore_id) = (unsigned )LCORE_ID_ANY ; +RTE_DEFINE_PER_LCORE (unsigned, _socket_id) = (unsigned )SOCKET_ID_ANY ;RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);/* lib/librte_eal/linuxapp/eal/eal_thread.c +0
-0
lib/librte_eal/linuxapp/eal/eal_thread.c line 62
Quoted 8 lines
lib/librte_eal/linuxapp/eal/eal_thread.c +2
-2@@ -57,8 +57,8 @@#include "eal_private.h" #include "eal_thread.h" -RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); -RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (unsigned, _lcore_id) = (unsigned )LCORE_ID_ANY ; +RTE_DEFINE_PER_LCORE (unsigned, _socket_id) = (unsigned )SOCKET_ID_ANY ;RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);
As far as I understand, now a rte_lcore_id() can return LCORE_ID_ANY. This should be modified in the rte_lcore_id() API comments.
Same for rte_socket_id().
I also wonder if the API of these functions should be modified to return an int instead of an unsigned as LCORE_ID_ANY is -1.
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH v4 12/17] eal: set _lcore_id and _socket_id to (-1) by default
Quoted 9 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Monday, February 09, 2015 4:01 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 12/17] eal: set_lcore_idand_socket_idto (-1) by default
Quoted 8 lines
Hi,
Quoted 3 lines · Cunming Liang
For those none EAL thread, *_lcore_id* shall always be
LCORE_ID_ANY. The libraries using *_lcore_id* as index need to take care. *_socket_id* always beSOCKET_ID_ANYunitl the thread changes the affinityunitl -> until
[LCM] accept.
Quoted 43 lines
Review
Quoted 10 lines
by
rte_thread_set_affinity()Signed-off-by Cunming Liang <[email protected]>--- lib/librte_eal/bsdapp/eal/eal_thread.c | 4 ++-- lib/librte_eal/linuxapp/eal/eal_thread.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-)(quoted diff) +0
-0
b/lib/librte_eal/bsdapp/eal/eal_thread.cReview
Quoted 15 lines
index 5b16302..2b3c9a8 100644
lib/librte_eal/bsdapp/eal/eal_thread.c +2
-2@@ -56,8 +56,8 @@#include "eal_private.h" #include "eal_thread.h" -RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); -RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (unsigned, _lcore_id) = (unsigned )LCORE_ID_ANY ; +RTE_DEFINE_PER_LCORE (unsigned, _socket_id) = (unsigned )SOCKET_ID_ANY ;RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);/* (quoted diff) +0
-0
b/lib/librte_eal/linuxapp/eal/eal_thread.clib/librte_eal/linuxapp/eal/eal_thread.c line 62
Quoted 12 lines
index 6eb1525..ab94e20 100644
lib/librte_eal/linuxapp/eal/eal_thread.c +2
-2@@ -57,8 +57,8 @@#include "eal_private.h" #include "eal_thread.h" -RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); -RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (unsigned, _lcore_id) = (unsigned )LCORE_ID_ANY ; +RTE_DEFINE_PER_LCORE (unsigned, _socket_id) = (unsigned )SOCKET_ID_ANY ;RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);As far as I understand, now a
rte_lcore_id()can returnLCORE_ID_ANY. This should be modified in therte_lcore_id()API comments.
Quoted 1 line
Same for
rte_socket_id().
[LCM] accept.
Quoted 3 lines
I also wonder if the API of these functions should be modified to return an int instead of an unsigned as
LCORE_ID_ANYis -1.
[LCM] I prefer not change the API definition. (unsigned)LCORE_ID_ANY already used before.
Quoted 3 lines
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): D0158A423229094DA7ABF71CF2FA0DA3118D7EF2-E2R4CRU6q/[email protected]
Re: [PATCH v4 12/17] eal: set _lcore_id and _socket_id to (-1) by default
Hi,
Quoted 15 lines · Cunming
Quoted 15 lines
lib/librte_eal/linuxapp/eal/eal_thread.c line 62
Quoted 11 lines
lib/librte_eal/linuxapp/eal/eal_thread.c +2
-2@@ -57,8 +57,8 @@#include "eal_private.h" #include "eal_thread.h" -RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); -RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (unsigned, _lcore_id) = (unsigned )LCORE_ID_ANY ; +RTE_DEFINE_PER_LCORE (unsigned, _socket_id) = (unsigned )SOCKET_ID_ANY ;RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);As far as I understand, now a
rte_lcore_id()can returnLCORE_ID_ANY. This should be modified in therte_lcore_id()API comments.
Quoted 6 lines
Quoted 1 line
Same for
rte_socket_id().[LCM] accept.
Quoted 3 lines
I also wonder if the API of these functions should be modified to return an int instead of an unsigned as
LCORE_ID_ANYis -1.[LCM] I prefer not change the API definition. (unsigned)LCORE_ID_ANY already used before.
OK
And what about directly defining the following?
#define LCORE_ID_ANY ((unsigned)-1)
It would avoid the casts.
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH v4 12/17] eal: set _lcore_id and _socket_id to (-1) by default
Quoted 36 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Tuesday, February 10, 2015 1:49 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 12/17] eal: set_lcore_idand_socket_idto (-1) by defaultHi,
Quoted 10 lines · Cunming
Quoted 10 lines
lib/librte_eal/linuxapp/eal/eal_thread.c line 61
Quoted 10 lines
lib/librte_eal/linuxapp/eal/eal_thread.c +2
-2@@ -57,8 +57,8 @@#include "eal_private.h" #include "eal_thread.h" -RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); -RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (unsigned, _lcore_id) = (unsigned )LCORE_ID_ANY ; +RTE_DEFINE_PER_LCORE(unsigned, _socket_id) =(unsigned)SOCKET_ID_ANY;
Quoted 11 lines
Quoted 6 lines
lib/librte_eal/linuxapp/eal/eal_thread.c line 62
Quoted 1 line
lib/librte_eal/linuxapp/eal/eal_thread.c +0
-0RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);As far as I understand, now a
rte_lcore_id()can returnLCORE_ID_ANY. This should be modified in therte_lcore_id()API comments.Same for
rte_socket_id().[LCM] accept.
Quoted 3 lines
I also wonder if the API of these functions should be modified to return an int instead of an unsigned as
LCORE_ID_ANYis -1.[LCM] I prefer not change the API definition. (unsigned)LCORE_ID_ANY already
used before.
Quoted 8 lines
OK
And what about directly defining the following?
#define
LCORE_ID_ANY((unsigned)-1)It would avoid the casts.
[LCM] Good point, will update it.
A message that is not in the archive (sent off-list or not yet received): D0158A423229094DA7ABF71CF2FA0DA3118D81F4-E2R4CRU6q/[email protected]
Re: [PATCH v4 12/17] eal: set _lcore_id and _socket_id to (-1) by default
Quoted 46 lines
-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Liang, Cunming
Sent: Tuesday, February 10, 2015 2:54 AM
To: Olivier MATZ; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 12/17] eal: set_lcore_idand_socket_idto (-1) by defaultQuoted 36 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Tuesday, February 10, 2015 1:49 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 12/17] eal: set_lcore_idand_socket_idto (-1) by defaultHi,
Quoted 10 lines · Cunming
Quoted 10 lines
lib/librte_eal/linuxapp/eal/eal_thread.c line 61
Quoted 10 lines
lib/librte_eal/linuxapp/eal/eal_thread.c +2
-2@@ -57,8 +57,8 @@#include "eal_private.h" #include "eal_thread.h" -RTE_DEFINE_PER_LCORE (unsigned, _lcore_id); -RTE_DEFINE_PER_LCORE (unsigned, _socket_id); +RTE_DEFINE_PER_LCORE (unsigned, _lcore_id) = (unsigned )LCORE_ID_ANY ; +RTE_DEFINE_PER_LCORE(unsigned, _socket_id) =(unsigned)SOCKET_ID_ANY;
Quoted 11 lines
Quoted 6 lines
lib/librte_eal/linuxapp/eal/eal_thread.c line 62
Quoted 1 line
lib/librte_eal/linuxapp/eal/eal_thread.c +0
-0RTE_DEFINE_PER_LCORE (rte_cpuset_t, _cpuset);As far as I understand, now a
rte_lcore_id()can returnLCORE_ID_ANY. This should be modified in therte_lcore_id()API comments.Same for
rte_socket_id().[LCM] accept.
Quoted 3 lines
I also wonder if the API of these functions should be modified to return an int instead of an unsigned as
LCORE_ID_ANYis -1.[LCM] I prefer not change the API definition. (unsigned)LCORE_ID_ANY already
used before.
OK
Quoted 7 lines
Quoted 6 lines
And what about directly defining the following?
#define
LCORE_ID_ANY((unsigned)-1)It would avoid the casts.
[LCM] Good point, will update it.
UINT32_MAX ?
PATCH v4 13/17 eal: fix recursive spinlock in non-EAL thraed
In reply to a message not in the archiveIn non-EAL thread, lcore_id alrways be LCORE_ID_ANY. It cann't be used as unique id for recursive spinlock. Then use rte_gettid() to replace it.
--- lib/librte_eal/common/include/generic/rte_spinlock.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
lib/librte_eal/common/include/generic/rte_spinlock.h +2 -2
@@ -179,7 +179,7 @@ static inline void rte_spinlock_recursive_init(rte_spinlock_recursive_t *slr)*/ static inline void rte_spinlock_recursive_lock (rte_spinlock_recursive_t *slr) { -int id =rte_lcore_id (); +int id =rte_gettid ();if (slr->user != id) {rte_spinlock_lock (&slr->sl);
@@ -212,7 +212,7 @@ static inline void rte_spinlock_recursive_unlock(rte_spinlock_recursive_t *slr)*/ static inline int rte_spinlock_recursive_trylock (rte_spinlock_recursive_t *slr) { -int id =rte_lcore_id (); +int id =rte_gettid ();if (slr->user != id) {if (rte_spinlock_trylock (&slr->sl) ==0 )
Signature · 1 line
1.8.1.4
PATCH v4 14/17 mempool: add support to non-EAL thread
In reply to a message not in the archiveFor non-EAL thread, bypass per lcore cache, directly use ring pool. It allows using rte_mempool in either EAL thread or any user pthread. As in non-EAL thread, it directly rely on rte_ring and it's none preemptive. It doesn't suggest to run multi-pthread/cpu which compete the rte_mempool. It will get bad performance and has critical risk if scheduling policy is RT.
--- lib/librte_mempool/rte_mempool.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-)
lib/librte_mempool/rte_mempool.h +11 -7
@@ -198,10 +198,12 @@ struct rte_mempool {* Number to add to the object-oriented statistics. */ #ifdef RTE_LIBRTE_MEMPOOL_DEBUG -#define __MEMPOOL_STAT_ADD (mp, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - mp->stats[__lcore_id].name##_objs += n; \ - mp->stats[__lcore_id].name##_bulk += 1; \ +#define __MEMPOOL_STAT_ADD (mp, name, n) do { \ + unsigned __lcore_id = rte_lcore_id(); \ + if (__lcore_id < RTE_MAX_LCORE) { \ + mp->stats[__lcore_id].name##_objs += n; \ + mp->stats[__lcore_id].name##_bulk += 1; \ + } \ } while(0)#else #define __MEMPOOL_STAT_ADD (mp, name, n) do {} while(0)
@@ -767,8 +769,9 @@ __mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table,__MEMPOOL_STAT_ADD (mp, put, n);#if RTE_MEMPOOL_CACHE_MAX_SIZE >0 -/* cache is not enabled or single producer */ -if (unlikely (cache_size ==0 || is_mp ==0 )) +/* cache is not enabled or single producer or none EAL thread */ +if (unlikely (cache_size ==0 || is_mp ==0 || + lcore_id >=RTE_MAX_LCORE )) goto ring_enqueue;/* Go straight to ring if put would overflow mem allocated for cache */
@@ -952,7 +955,8 @@ __mempool_get_bulk(struct rte_mempool *mp, void **obj_table,uint32_t cache_size = mp->cache_size;/* cache is not enabled or single consumer */ -if (unlikely (cache_size ==0 || is_mc ==0 || n >= cache_size)) +if (unlikely (cache_size ==0 || is_mc ==0 || + n >= cache_size || lcore_id >=RTE_MAX_LCORE )) goto ring_dequeue; cache = &mp->local_cache[lcore_id];
Signature · 1 line
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 1422842559-13617-15-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
Re: [PATCH v4 14/17] mempool: add support to non-EAL thread
Hi,
Quoted 19 lines · Cunming Liang
For non-EAL thread, bypass per lcore cache, directly use ring pool. It allows using
rte_mempoolin either EAL thread or any user pthread. As in non-EAL thread, it directly rely onrte_ringand it's none preemptive. It doesn't suggest to run multi-pthread/cpu which compete therte_mempool. It will get bad performance and has critical risk if scheduling policy is RT.Signed-off-by Cunming Liang <[email protected]>--- lib/librte_mempool/rte_mempool.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-)lib/librte_mempool/rte_mempool.h +0
-0@@ -198,10 +198,12 @@ struct rte_mempool {* Number to add to the object-oriented statistics. */ #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
lib/librte_mempool/rte_mempool.h struct rte_mempool { · line 206
Quoted 10 lines
lib/librte_mempool/rte_mempool.h +6
-4-#define __MEMPOOL_STAT_ADD (mp, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - mp->stats[__lcore_id].name##_objs += n; \ - mp->stats[__lcore_id].name##_bulk += 1; \ +#define __MEMPOOL_STAT_ADD (mp, name, n) do { \ + unsigned __lcore_id = rte_lcore_id(); \ + if (__lcore_id < RTE_MAX_LCORE) { \ + mp->stats[__lcore_id].name##_objs += n; \ + mp->stats[__lcore_id].name##_bulk += 1; \ + } \
Does it mean that we have no statistics for non-EAL threads? (same question for rings and timers in the next patches)
Quoted 17 lines
lib/librte_mempool/rte_mempool.h +3
-2} while(0)#else #define __MEMPOOL_STAT_ADD (mp, name, n) do {} while(0)@@ -767,8 +769,9 @@ __mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table,__MEMPOOL_STAT_ADD (mp, put, n);#if RTE_MEMPOOL_CACHE_MAX_SIZE >0 -/* cache is not enabled or single producer */ -if (unlikely (cache_size ==0 || is_mp ==0 )) +/* cache is not enabled or single producer or none EAL thread */ +if (unlikely (cache_size ==0 || is_mp ==0 || + lcore_id >=RTE_MAX_LCORE )) goto ring_enqueue;/* Go straight to ring if put would overflow mem allocated for cache */
lib/librte_mempool/rte_mempool.h __mempool_get_bulk(struct rte_mempool *mp, void **obj_table, · line 962
Quoted 9 lines
lib/librte_mempool/rte_mempool.h +2
-1@@ -952,7 +955,8 @@ __mempool_get_bulk(struct rte_mempool *mp, void **obj_table,uint32_t cache_size = mp->cache_size;/* cache is not enabled or single consumer */ -if (unlikely (cache_size ==0 || is_mc ==0 || n >= cache_size)) +if (unlikely (cache_size ==0 || is_mc ==0 || + n >= cache_size || lcore_id >=RTE_MAX_LCORE )) goto ring_dequeue; cache = &mp->local_cache[lcore_id];
What is the performance impact of adding this test?
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH v4 14/17] mempool: add support to non-EAL thread
Quoted 37 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Monday, February 09, 2015 4:01 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 14/17] mempool: add support to non-EAL threadHi,
Quoted 6 lines · Cunming Liang
For non-EAL thread, bypass per lcore cache, directly use ring pool. It allows using
rte_mempoolin either EAL thread or any user pthread. As in non-EAL thread, it directly rely onrte_ringand it's none preemptive. It doesn't suggest to run multi-pthread/cpu which compete therte_mempool. It will get bad performance and has critical risk if scheduling policy is RT.Review
Quoted 6 lines
Signed-off-by Cunming Liang <[email protected]>--- lib/librte_mempool/rte_mempool.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-)(quoted diff) +0
-0
b/lib/librte_mempool/rte_mempool.hQuoted 12 lines
index 3314651..4845f27 100644
lib/librte_mempool/rte_mempool.h +1
-4@@ -198,10 +198,12 @@ struct rte_mempool {* Number to add to the object-oriented statistics. */ #ifdef RTE_LIBRTE_MEMPOOL_DEBUG -#define __MEMPOOL_STAT_ADD (mp, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - mp->stats[__lcore_id].name##_objs += n; \ - mp->stats[__lcore_id].name##_bulk += 1; \ +#define __MEMPOOL_STAT_ADD (mp, name, n) do { \
Quoted 8 lines
lib/librte_mempool/rte_mempool.h struct rte_mempool { · line 206
Quoted 5 lines
lib/librte_mempool/rte_mempool.h +5
-0+unsigned __lcore_id =rte_lcore_id (); \ +if (__lcore_id <RTE_MAX_LCORE ) { \ + mp->stats[__lcore_id].name##_objs += n; \ + mp->stats[__lcore_id].name##_bulk += 1; \ + } \Does it mean that we have no statistics for non-EAL threads? (same question for rings and timers in the next patches)
[LCM] Yes, it is in this patch set, mainly focus on EAL thread and make sure no running issue on non-EAL thread. For full non-EAL function, will have other patch set to enhance non-EAL thread as the 2nd step.
Quoted 28 lines
lib/librte_mempool/rte_mempool.h __mempool_put_bulk(struct rte_mempool *mp, void
Quoted 6 lines
lib/librte_mempool/rte_mempool.h +0
-0} while(0)#else #define __MEMPOOL_STAT_ADD (mp, name, n) do {} while(0)@@ -767,8 +769,9 @@ __mempool_put_bulk(struct rte_mempool *mp, void* const
*obj_table,lib/librte_mempool/rte_mempool.h __mempool_get_bulk(struct rte_mempool *mp, void
Quoted 12 lines
lib/librte_mempool/rte_mempool.h +3
-2__MEMPOOL_STAT_ADD (mp, put, n);#if RTE_MEMPOOL_CACHE_MAX_SIZE >0 -/* cache is not enabled or single producer */ -if (unlikely (cache_size ==0 || is_mp ==0 )) +/* cache is not enabled or single producer or none EAL thread */ +if (unlikely (cache_size ==0 || is_mp ==0 || + lcore_id >=RTE_MAX_LCORE )) goto ring_enqueue;/* Go straight to ring if put would overflow mem allocated for cache */ @@ -952,7 +955,8 @@ __mempool_get_bulk(struct rte_mempool *mp, void**obj_table,
Quoted 8 lines
lib/librte_mempool/rte_mempool.h +2
-1uint32_t cache_size = mp->cache_size;/* cache is not enabled or single consumer */ -if (unlikely (cache_size ==0 || is_mc ==0 || n >= cache_size)) +if (unlikely (cache_size ==0 || is_mc ==0 || + n >= cache_size || lcore_id >=RTE_MAX_LCORE )) goto ring_dequeue;
Quoted 4 lines
lib/librte_mempool/rte_mempool.h __mempool_get_bulk(struct rte_mempool *mp, void **obj_table, · line 962
Quoted 2 lines
lib/librte_mempool/rte_mempool.h +0
-0cache = &mp->local_cache[lcore_id];What is the performance impact of adding this test?
[LCM] By perf in unit test, it's almost the same. But haven't measure EAL thread and non-EAL thread share the same mempool.
Quoted 4 lines
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): D0158A423229094DA7ABF71CF2FA0DA3118D7F20-E2R4CRU6q/[email protected]
Re: [PATCH v4 14/17] mempool: add support to non-EAL thread
Hi,
Quoted 14 lines · Cunming
Quoted 6 lines
Quoted 6 lines
#ifdef
RTE_LIBRTE_MEMPOOL_DEBUGlib/librte_mempool/rte_mempool.h +1
-4-#define __MEMPOOL_STAT_ADD (mp, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - mp->stats[__lcore_id].name##_objs += n; \ - mp->stats[__lcore_id].name##_bulk += 1; \ +#define __MEMPOOL_STAT_ADD (mp, name, n) do { \Quoted 8 lines
lib/librte_mempool/rte_mempool.h struct rte_mempool { · line 206
Quoted 5 lines
lib/librte_mempool/rte_mempool.h +5
-0+unsigned __lcore_id =rte_lcore_id (); \ +if (__lcore_id <RTE_MAX_LCORE ) { \ + mp->stats[__lcore_id].name##_objs += n; \ + mp->stats[__lcore_id].name##_bulk += 1; \ + } \Does it mean that we have no statistics for non-EAL threads? (same question for rings and timers in the next patches)
Quoted 2 lines
[LCM] Yes, it is in this patch set, mainly focus on EAL thread and make sure no running issue on non-EAL thread. For full non-EAL function, will have other patch set to enhance non-EAL thread as the 2nd step.
OK
Quoted 10 lines
Quoted 10 lines
Review __mempool_get_bulk(struct rte_mempool *mp, void
Quoted 1 line
(quoted diff) +0
-0@@ -952,7 +955,8 @@ __mempool_get_bulk(struct rte_mempool *mp, void**obj_table,
Quoted 8 lines
lib/librte_mempool/rte_mempool.h +2
-1uint32_t cache_size = mp->cache_size;/* cache is not enabled or single consumer */ -if (unlikely (cache_size ==0 || is_mc ==0 || n >= cache_size)) +if (unlikely (cache_size ==0 || is_mc ==0 || + n >= cache_size || lcore_id >=RTE_MAX_LCORE )) goto ring_dequeue;
Quoted 5 lines
Quoted 4 lines
lib/librte_mempool/rte_mempool.h __mempool_get_bulk(struct rte_mempool *mp, void **obj_table, · line 962
Quoted 2 lines
lib/librte_mempool/rte_mempool.h +0
-0cache = &mp->local_cache[lcore_id];What is the performance impact of adding this test?
[LCM] By perf in unit test, it's almost the same. But haven't measure EAL thread and non-EAL thread share the same mempool.
When you say "unit test", are you talking about mempool tests from "make test"? Do you have some numbers to share?
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH v4 14/17] mempool: add support to non-EAL thread
Quoted 48 lines
-----Original Message-----
From: Olivier MATZ [mailto:[email protected]]
Sent: Tuesday, February 10, 2015 1:52 AM
To: Liang, Cunming; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4 14/17] mempool: add support to non-EAL threadHi,
Quoted 14 lines · Cunming
Quoted 6 lines
Quoted 6 lines
#ifdef
RTE_LIBRTE_MEMPOOL_DEBUG(quoted diff) +1
-4-#define __MEMPOOL_STAT_ADD(mp, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - mp->stats[__lcore_id].name##_objs += n; \ - mp->stats[__lcore_id].name##_bulk += 1; \ +#define __MEMPOOL_STAT_ADD(mp, name, n) do { \Quoted 8 lines
Review
Quoted 5 lines
(quoted diff) +5
-0+ unsigned __lcore_id = rte_lcore_id(); \ + if (__lcore_id < RTE_MAX_LCORE) { \ + mp->stats[__lcore_id].name##_objs += n; \ + mp->stats[__lcore_id].name##_bulk += 1; \ + } \Does it mean that we have no statistics for non-EAL threads? (same question for rings and timers in the next patches)
Quoted 1 line
[LCM] Yes, it is in this patch set, mainly focus on EAL thread and make sure no
running issue on non-EAL thread.
Quoted 1 line
For full non-EAL function, will have other patch set to enhance non-EAL thread
as the 2nd step.
OK
Quoted 1 line
Quoted 1 line
Review __mempool_get_bulk(struct rte_mempool *mp,
Quoted 1 line
(quoted diff) +0
-0@@ -952,7 +955,8 @@ __mempool_get_bulk(struct rte_mempool *mp,void
Quoted 13 lines
Quoted 13 lines
**obj_table,
Review __mempool_get_bulk(struct rte_mempool *mp, · line 962
Quoted 10 lines
(quoted diff) +2
-1uint32_t cache_size = mp->cache_size; /* cache is not enabled or single consumer */ - if (unlikely(cache_size == 0 || is_mc == 0 || n >= cache_size)) + if (unlikely(cache_size == 0 || is_mc == 0 || + n >= cache_size || lcore_id >= RTE_MAX_LCORE)) goto ring_dequeue; cache = &mp->local_cache[lcore_id];What is the performance impact of adding this test?
Quoted 6 lines
Quoted 1 line
[LCM] By perf in unit test, it's almost the same. But haven't measure EAL thread
and non-EAL thread share the same mempool.
When you say "unit test", are you talking about mempool tests from "make test"? Do you have some numbers to share?
[LCM] I means DPDK app/test, run mempool_perf_test. Will add numbers on v5.
PATCH v4 15/17 ring: add support to non-EAL thread
In reply to a message not in the archivering debug stat won't take care non-EAL thread.
--- lib/librte_ring/rte_ring.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-)
lib/librte_ring/rte_ring.h +6 -4
@@ -188,10 +188,12 @@ struct rte_ring {* The number to add to the object-oriented statistics. */ #ifdef RTE_LIBRTE_RING_DEBUG -#define __RING_STAT_ADD (r, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - r->stats[__lcore_id].name##_objs += n; \ - r->stats[__lcore_id].name##_bulk += 1; \ +#define __RING_STAT_ADD (r, name, n) do { \ + unsigned __lcore_id = rte_lcore_id(); \ + if (__lcore_id < RTE_MAX_LCORE) { \ + r->stats[__lcore_id].name##_objs += n; \ + r->stats[__lcore_id].name##_bulk += 1; \ + } \ } while(0)#else #define __RING_STAT_ADD (r, name, n) do {} while(0)
Signature · 1 line
1.8.1.4
PATCH v4 16/17 ring: add sched_yield to avoid spin forever
In reply to a message not in the archiveAdd a sched_yield() syscall if the thread spins for too long, waiting other thread to finish its operations on the ring. That gives pre-empted thread a chance to proceed and finish with ring enqnue/dequeue operation.
The purpose is to reduce contention on the ring.
--- lib/librte_ring/rte_ring.h | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-)
lib/librte_ring/rte_ring.h +29 -6
@@ -126,6 +126,7 @@ struct rte_ring_debug_stats {#define RTE_RING_NAMESIZE 32/**< The maximum length of a ring name. */ #define RTE_RING_MZ_PREFIX "RG_" +#define RTE_RING_PAUSE_REP 0x100/**< yield after num of times pause. */ /** * An RTE ring structure.
@@ -410,7 +411,7 @@ __rte_ring_mp_do_enqueue(struct rte_ring *r, void * const *obj_table,uint32_t cons_tail, free_entries;const unsigned max = n;int success; -unsigned i; +unsigned i, rep;uint32_t mask = r->prod.mask;int ret;
@@ -468,8 +469,19 @@ __rte_ring_mp_do_enqueue(struct rte_ring *r, void * const *obj_table,* If there are other enqueues in progress that preceded us, * we need to wait for them to complete */ -while (unlikely (r->prod.tail != prod_head)) -rte_pause (); +do { +/* avoid spin too long waiting for other thread finish */ +for (rep =RTE_RING_PAUSE_REP ; + rep !=0 && r->prod.tail != prod_head; rep--) +rte_pause (); + +/* +* It gives pre-empted thread a chance to proceed and +* finish with ring enqnue operation. +*/ +if (rep ==0 ) +sched_yield (); + }while (rep ==0 ); r->prod.tail = prod_next;return ret;
@@ -589,7 +601,7 @@ __rte_ring_mc_do_dequeue(struct rte_ring *r, void **obj_table,uint32_t cons_next, entries;const unsigned max = n;int success; -unsigned i; +unsigned i, rep;uint32_t mask = r->prod.mask;/* move cons.head atomically */
@@ -634,8 +646,19 @@ __rte_ring_mc_do_dequeue(struct rte_ring *r, void **obj_table,* If there are other dequeues in progress that preceded us, * we need to wait for them to complete */ -while (unlikely (r->cons.tail != cons_head)) -rte_pause (); +do { +/* avoid spin too long waiting for other thread finish */ +for (rep =RTE_RING_PAUSE_REP ; + rep !=0 && r->cons.tail != cons_head; rep--) +rte_pause (); + +/* +* It gives pre-empted thread a chance to proceed and +* finish with ring denqnue operation. +*/ +if (rep ==0 ) +sched_yield (); + }while (rep ==0 );__RING_STAT_ADD (r, deq_success, n); r->cons.tail = cons_next;
Signature · 1 line
1.8.1.4
A message that is not in the archive (sent off-list or not yet received): 1422842559-13617-17-git-send-email-cunming.liang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
Re: [PATCH v4 16/17] ring: add sched_yield to avoid spin forever
Hi,
Quoted 74 lines · Cunming Liang
Add a
sched_yield()syscall if the thread spins for too long, waiting other thread to finish its operations on the ring. That gives pre-empted thread a chance to proceed and finish with ring enqnue/dequeue operation.
The purpose is to reduce contention on the ring.Signed-off-by Cunming Liang <[email protected]>--- lib/librte_ring/rte_ring.h | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-)lib/librte_ring/rte_ring.h +22
-6@@ -126,6 +126,7 @@ struct rte_ring_debug_stats {#define RTE_RING_NAMESIZE 32/**< The maximum length of a ring name. */ #define RTE_RING_MZ_PREFIX "RG_" +#define RTE_RING_PAUSE_REP 0x100/**< yield after num of times pause. */ /** * An RTE ring structure. @@ -410,7 +411,7 @@ __rte_ring_mp_do_enqueue(struct rte_ring *r, void * const *obj_table,uint32_t cons_tail, free_entries;const unsigned max = n;int success; -unsigned i; +unsigned i, rep;uint32_t mask = r->prod.mask;int ret;@@ -468,8 +469,19 @@ __rte_ring_mp_do_enqueue(struct rte_ring *r, void * const *obj_table,* If there are other enqueues in progress that preceded us, * we need to wait for them to complete */ -while (unlikely (r->prod.tail != prod_head)) -rte_pause (); +do { +/* avoid spin too long waiting for other thread finish */ +for (rep =RTE_RING_PAUSE_REP ; + rep !=0 && r->prod.tail != prod_head; rep--) +rte_pause (); + +/* +* It gives pre-empted thread a chance to proceed and +* finish with ring enqnue operation. +*/ +if (rep ==0 ) +sched_yield (); + }while (rep ==0 ); r->prod.tail = prod_next;return ret;@@ -589,7 +601,7 @@ __rte_ring_mc_do_dequeue(struct rte_ring *r, void **obj_table,uint32_t cons_next, entries;const unsigned max = n;int success; -unsigned i; +unsigned i, rep;uint32_t mask = r->prod.mask;/* move cons.head atomically */ @@ -634,8 +646,19 @@ __rte_ring_mc_do_dequeue(struct rte_ring *r, void **obj_table,* If there are other dequeues in progress that preceded us, * we need to wait for them to complete */ -while (unlikely (r->cons.tail != cons_head)) -rte_pause (); + do { + /* avoid spin too long waiting for other thread finish */ + for (rep = RTE_RING_PAUSE_REP; + rep != 0 && r->cons.tail != cons_head; rep--) + rte_pause(); +
lib/librte_ring/rte_ring.h __rte_ring_mc_do_dequeue(struct rte_ring *r, void **obj_table, · line 664
Quoted 10 lines
lib/librte_ring/rte_ring.h +7
-0+/* +* It gives pre-empted thread a chance to proceed and +* finish with ring denqnue operation. +*/ +if (rep ==0 ) +sched_yield (); + }while (rep ==0 );__RING_STAT_ADD (r, deq_success, n); r->cons.tail = cons_next;
The ring library was designed with the assumption that the code is not preemptable. The code is lock-less but not wait-less. Actually, if the code is preempted at a bad moment, it can spin forever until it's unscheduled.
I wonder if adding a sched_yield() may not penalize the current implementations that only use one pthread per core? Even if there is only one pthread in the scheduler queue for this CPU, calling the scheduler code may cost thousands of cycles.
Also, where does this value "RTE_RING_PAUSE_REP 0x100" comes from? Why 0x100 is better than 42 or than 10000?
I think it could be good to check if there is a performance impact with this change, especially where there is a lot of contention on the ring. If it has an impact, what about adding a compile or runtime option?
Regards,
Olivier
A message that is not in the archive (sent off-list or not yet received): [email protected]