add new Double Word Key hash table
Subsystems (2)
cover add new Double Word Key hash table
Currently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called DWK (double word key) hash supports 8 byte values, which is enough to store a pointer.
It would also be nice to get feedback on whether to leave the old FBK and new DWK implementations, or whether to deprecate the old one?
Vladimir Medvedkin (3):
hash: add dwk hash library
test: add dwk hash autotests
test: add dwk perf tests
app/test/Makefile | 1 + app/test/meson.build | 1 + app/test/test_dwk_hash.c | 229 +++++++++++++++++++++++++++++ app/test/test_hash_perf.c | 81 +++++++++++ lib/Makefile | 2 +- lib/librte_hash/Makefile | 4 +- lib/librte_hash/meson.build | 5 +- lib/librte_hash/rte_dwk_hash.c | 271 +++++++++++++++++++++++++++++++++++ lib/librte_hash/rte_dwk_hash.h | 178 +++++++++++++++++++++++ lib/librte_hash/rte_hash_version.map | 5 + 10 files changed, 773 insertions(+), 4 deletions(-) create mode 100644 app/test/test_dwk_hash.c create mode 100644 lib/librte_hash/rte_dwk_hash.c create mode 100644 lib/librte_hash/rte_dwk_hash.h
Signature · 1 line
2.7.4
PATCH 1/3 hash: add dwk hash library
DWK hash is a Double Word Key(uint32_t) hash table. The value is uint64_t.
This table is hash function agnostic so user must provide precalculated hash signature for add/delete/lookup operations.
--- lib/Makefile | 2 +- lib/librte_hash/Makefile | 4 +- lib/librte_hash/meson.build | 5 +- lib/librte_hash/rte_dwk_hash.c | 271 +++++++++++++++++++++++++++++++++++ lib/librte_hash/rte_dwk_hash.h | 178 +++++++++++++++++++++++ lib/librte_hash/rte_hash_version.map | 5 + 6 files changed, 461 insertions(+), 4 deletions(-) create mode 100644 lib/librte_hash/rte_dwk_hash.c create mode 100644 lib/librte_hash/rte_dwk_hash.h
lib/Makefile +1 -1
@@ -48,7 +48,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_VHOST) += librte_vhost DEPDIRS-librte_vhost := librte_eal librte_mempool librte_mbuf librte_ethdev \ librte_net librte_hash librte_cryptodev DIRS-$(CONFIG_RTE_LIBRTE_HASH) += librte_hash -DEPDIRS-librte_hash := librte_eal librte_ring +DEPDIRS-librte_hash := librte_eal librte_ring librte_mempool DIRS-$(CONFIG_RTE_LIBRTE_EFD) += librte_efd DEPDIRS-librte_efd := librte_eal librte_ring librte_hash DIRS-$(CONFIG_RTE_LIBRTE_RIB) += librte_rib
lib/librte_hash/Makefile +3 -1
@@ -8,13 +8,14 @@ LIB = librte_hash.a CFLAGS += -O3 -DALLOW_EXPERIMENTAL_API CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_ring +LDLIBS += -lrte_eal -lrte_ring -lrte_mempool EXPORT_MAP := rte_hash_version.map # all source are stored in SRCS-y SRCS-$(CONFIG_RTE_LIBRTE_HASH) := rte_cuckoo_hash.c SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_fbk_hash.c +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_dwk_hash.c # install this header file SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include := rte_hash.h
@@ -27,5 +28,6 @@ endif SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_jhash.h SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_thash.h SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_fbk_hash.h +SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_dwk_hash.h include $(RTE_SDK)/mk/rte.lib.mk
lib/librte_hash/meson.build +3 -2
@@ -3,13 +3,14 @@ headers = files('rte_crc_arm64.h', 'rte_fbk_hash.h', + 'rte_dwk_hash.h', 'rte_hash_crc.h', 'rte_hash.h', 'rte_jhash.h', 'rte_thash.h') -sources = files('rte_cuckoo_hash.c', 'rte_fbk_hash.c') -deps += ['ring'] +sources = files('rte_cuckoo_hash.c', 'rte_fbk_hash.c', 'rte_dwk_hash.c') +deps += ['ring', 'mempool'] # rte ring reset is not yet part of stable API allow_experimental_apis = true
lib/librte_hash/rte_dwk_hash.c new file +271 -0
271 lines not on this page · raw message
lib/librte_hash/rte_dwk_hash.h new file +178 -0
178 lines not on this page · raw message
lib/librte_hash/rte_hash_version.map +5 -0
@@ -35,4 +35,9 @@ EXPERIMENTAL { rte_hash_free_key_with_position; rte_hash_max_key_id; + rte_dwk_hash_create; + rte_dwk_hash_find_existing; + rte_dwk_hash_free; + rte_dwk_hash_add; + rte_dwk_hash_delete; };
Signature · 1 line
2.7.4
PATCH 2/3 test: add dwk hash autotests
In reply to Vladimir MedvedkinAdd autotests for rte_dwk_hash library
--- app/test/Makefile | 1 + app/test/meson.build | 1 + app/test/test_dwk_hash.c | 229 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 231 insertions(+) create mode 100644 app/test/test_dwk_hash.c
app/test/Makefile +1 -0
@@ -73,6 +73,7 @@ SRCS-y += test_bitmap.c SRCS-y += test_reciprocal_division.c SRCS-y += test_reciprocal_division_perf.c SRCS-y += test_fbarray.c +SRCS-y += test_dwk_hash.c SRCS-y += test_external_mem.c SRCS-y += test_rand_perf.c
app/test/meson.build +1 -0
@@ -45,6 +45,7 @@ test_sources = files('commands.c', 'test_eventdev.c', 'test_external_mem.c', 'test_fbarray.c', + 'test_dwk_hash.c', 'test_fib.c', 'test_fib_perf.c', 'test_fib6.c',
app/test/test_dwk_hash.c new file +229 -0
229 lines not on this page · raw message
Signature · 1 line
2.7.4
PATCH 3/3 test: add dwk perf tests
In reply to Vladimir MedvedkinAdd performance tests for rte_dwk_hash.
--- app/test/test_hash_perf.c | 81 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+)
app/test/test_hash_perf.c +81 -0
@@ -12,8 +12,10 @@#include <rte_hash_crc.h> #include <rte_jhash.h> #include <rte_fbk_hash.h> +#include <rte_dwk_hash.h> #include <rte_random.h> #include <rte_string_fns.h> +#include <rte_hash_crc.h> #include "test.h"
@@ -29,6 +31,8 @@#define NUM_SHUFFLES 10#define BURST_SIZE 16 +#define CRC_INIT_VAL 0xdeadbeef +enum operations {ADD =0 ,LOOKUP ,
@@ -669,6 +673,80 @@ fbk_hash_perf_test(void) }static int +dwk_hash_perf_test (void ) +{ +struct rte_dwk_hash_params params = { + .name ="dwk_hash_test" , + .entries =ENTRIES , + .socket_id =rte_socket_id (), + }; +struct rte_dwk_hash_table *handle =NULL ; +uint32_t *keys =NULL ; +unsigned indexes[TEST_SIZE ]; +uint64_t tmp_val; +uint64_t lookup_time =0 ; +unsigned added =0 ; +uint32_t key; +uint16_t val; +unsigned i, j; +int ret =0 ; + + handle =rte_dwk_hash_create (¶ms); +if (handle ==NULL ) { +printf ("Error creating table\n" ); +return -1 ; + } + + keys =rte_zmalloc (NULL ,ENTRIES *sizeof (*keys),0 ); +if (keys ==NULL ) { +printf ("fbk hash: memory allocation for key store failed\n" ); +return -1 ; + } + +/* Generate random keys and values. */ +for (i =0 ; i <ENTRIES ; i++) { + key = (uint32_t )rte_rand (); + val =rte_rand (); + +if (rte_dwk_hash_add (handle, key,rte_hash_crc_4byte (key, +CRC_INIT_VAL ), val) ==0 ) { + keys[added] = key; + added++; + } + } + +for (i =0 ; i <TEST_ITERATIONS ; i++) { +uint64_t begin; +uint64_t end; + +/* Generate random indexes into keys[] array. */ +for (j =0 ; j <TEST_SIZE ; j++) + indexes[j] =rte_rand () % added; + + begin =rte_rdtsc (); +/* Do lookups */ +for (j =0 ; j <TEST_SIZE ; j++) + ret +=rte_dwk_hash_lookup (handle, + keys[indexes[j]], +rte_hash_crc_4byte (keys[indexes[j]], +CRC_INIT_VAL ), &tmp_val); + + end =rte_rdtsc (); + lookup_time += (double )(end - begin); + } + +printf ("\n\n *** DWK Hash function performance test results ***\n" ); +if (ret ==0 ) +printf ("Number of ticks per lookup = %g\n" , + (double )lookup_time / + ((double )TEST_ITERATIONS * (double )TEST_SIZE )); + +rte_dwk_hash_free (handle); + +return 0 ; +} + +static int test_hash_perf (void ) {unsigned int with_pushes, with_locks;
@@ -695,6 +773,9 @@ test_hash_perf(void)if (fbk_hash_perf_test () <0 )return -1 ; +if (dwk_hash_perf_test () <0 ) +return -1 ; +return 0 ; }
Signature · 1 line
2.7.4
Quoted 11 lines
From: dev [mailto:[email protected]] On Behalf Of Vladimir Medvedkin
Sent: Monday, March 16, 2020 2:38 PMCurrently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called DWK (double word key) hash supports 8 byte values, which is enough to store a pointer.It would also be nice to get feedback on whether to leave the old FBK and new DWK implementations, or whether to deprecate the old one?
<rant on>
Who comes up with these names?!?
FBK (Four Byte Key) and DWK (Double Word Key) is supposed to mean the same. Could you use 32 somewhere in the name instead, like in int32_t, instead of using a growing list of creative synonyms for the same thing? Pretty please, with a cherry on top!
And if the value size is fixed too, perhaps the name should also indicate the value size.
<rant off>
It's a shame we don't have C++ class templates available in DPDK...
In other news, Mellanox has sent an RFC for an "indexed memory pool" library [1] to conserve memory by using uintXX_t instead of pointers, so perhaps a variant of a 32 bit key hash library with 32 bit values (in addition to 16 bit values in FBK and 64 bit in DWK) would be nice combination with that library.
[1]: http://mails.dpdk.org/archives/dev/2019-October/147513.html
Med venlig hilsen / kind regards
- Morten Brørup
Hi Morten,
Quoted 9 lines · Morten Brørup
Quoted 9 lines
From: dev [mailto:[email protected]] On Behalf Of Vladimir Medvedkin
Sent: Monday, March 16, 2020 2:38 PMCurrently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called DWK (double word key) hash supports 8 byte values, which is enough to store a pointer.
Quoted 7 lines
Quoted 2 lines
It would also be nice to get feedback on whether to leave the old FBK and new DWK implementations, or whether to deprecate the old one?
<rant on>
Who comes up with these names?!?
FBK (Four Byte Key) and DWK (Double Word Key) is supposed to mean the same. Could you use 32 somewhere in the name instead, like in
int32_t, instead of using a growing list of creative synonyms for the same thing? Pretty please, with a cherry on top!
That's true, at first I named it as fbk2, but then it was decided to rename it "dwk", so that there was no confusion with the existing FBK library. Naming suggestions are welcome!
Quoted 15 lines
And if the value size is fixed too, perhaps the name should also indicate the value size.
<rant off>
It's a shame we don't have C++ class templates available in DPDK...
In other news, Mellanox has sent an RFC for an "indexed memory pool" library [1] to conserve memory by using
uintXX_tinstead of pointers, so perhaps a variant of a 32 bit key hash library with 32 bit values (in addition to 16 bit values in FBK and 64 bit in DWK) would be nice combination with that library.[1]: http://mails.dpdk.org/archives/dev/2019-October/147513.html
Med venlig hilsen / kind regards
- Morten Brørup
Signature · 2 lines
Regards, Vladimir
Quoted 17 lines · Vladimir
Hi Morten,
Quoted 13 lines · Morten Brørup
Quoted 11 lines
From: dev [mailto:[email protected]] On Behalf Of Vladimir Medvedkin
Sent: Monday, March 16, 2020 2:38 PMCurrently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called DWK (double word key) hash supports 8 byte values, which is enough to store a pointer.It would also be nice to get feedback on whether to leave the old FBK and new DWK implementations, or whether to deprecate the old one?
<rant on>
Quoted 24 lines
Quoted 3 lines
Who comes up with these names?!?
FBK (Four Byte Key) and DWK (Double Word Key) is supposed to mean the same. Could you use 32 somewhere in the name instead, like in
int32_t, instead of using a growing list of creative synonyms for the same thing? Pretty please, with a cherry on top!That's true, at first I named it as fbk2, but then it was decided to rename it "dwk", so that there was no confusion with the existing FBK library. Naming suggestions are welcome!
Quoted 15 lines
And if the value size is fixed too, perhaps the name should also indicate the value size.
<rant off>
It's a shame we don't have C++ class templates available in DPDK...
In other news, Mellanox has sent an RFC for an "indexed memory pool" library [1] to conserve memory by using
uintXX_tinstead of pointers, so perhaps a variant of a 32 bit key hash library with 32 bit values (in addition to 16 bit values in FBK and 64 bit in DWK) would be nice combination with that library.[1]: http://mails.dpdk.org/archives/dev/2019-October/147513.html
Med venlig hilsen / kind regards
- Morten Brørup
Why is this different (or better) than existing rte_hash. Having more flavors is not necessarily a good thing (except in Gelato)
Quoted 23 lines
From: dev [mailto:[email protected]] On Behalf Of Medvedkin, Vladimir
Sent: Monday, March 16, 2020 7:28 PMHi Morten,
Quoted 9 lines · Morten Brørup
Quoted 9 lines
From: dev [mailto:[email protected]] On Behalf Of Vladimir Medvedkin
Sent: Monday, March 16, 2020 2:38 PMCurrently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called DWK (double word key) hash supports 8 byte values, which is enough to store a pointer.Quoted 7 lines
Quoted 2 lines
It would also be nice to get feedback on whether to leave the old FBK and new DWK implementations, or whether to deprecate the old one?
<rant on>
Who comes up with these names?!?
FBK (Four Byte Key) and DWK (Double Word Key) is supposed to mean the
Quoted 9 lines
same. Could you use 32 somewhere in the name instead, like in
int32_t, instead of using a growing list of creative synonyms for the same thing? Pretty please, with a cherry on top!That's true, at first I named it as fbk2, but then it was decided to rename it "dwk", so that there was no confusion with the existing FBK library. Naming suggestions are welcome!
OK... let me suggest a prefix with both key and value sizes in the name:
Instead of rte_dwk_hash_xxx, use rte_hash_k32v64_xxx.
However, that suggests that it is a generic hash... and your hash algorithm certainly has other properties too; so perhaps a specific name, saying something about the underlying algorithm, makes more sense after all. And the documentation will show its properties anyway.
Whatever you choose, Double Word is certainly not a good name.... a word is 32 bits and a double word is 64 bits in many non-Intel CPU architectures, including some supported by DPDK.
Quoted 24 lines
Quoted 2 lines
And if the value size is fixed too, perhaps the name should also indicate
the value size.
Quoted 6 lines
<rant off>
It's a shame we don't have C++ class templates available in DPDK...
In other news, Mellanox has sent an RFC for an "indexed memory pool"
library [1] to conserve memory by using
uintXX_tinstead of pointers, so perhaps a variant of a 32 bit key hash library with 32 bit values (in addition to 16 bit values in FBK and 64 bit in DWK) would be nice combination with that library.Quoted 7 lines
[1]: http://mails.dpdk.org/archives/dev/2019-October/147513.html
Med venlig hilsen / kind regards
- Morten Brørup--
Regards,
Vladimir
Quoted 65 lines
-----Original Message-----
From: Stephen Hemminger <[email protected]>
Sent: Monday, March 16, 2020 12:33 PM
To: Medvedkin, Vladimir <[email protected]>
Cc: Morten Brørup <[email protected]>; [email protected]; Ananyev, Konstantin <[email protected]>; Wang, Yipeng1 <[email protected]>; Gobriel, Sameh <[email protected]>; Richardson, Bruce <[email protected]>; Suanming Mou <[email protected]>; Olivier Matz <[email protected]>; Xueming(Steven) Li <[email protected]>; Andrew Rybchenko <[email protected]>; Asaf Penso <[email protected]>; Ori Kam <[email protected]>
Subject: Re: [dpdk-dev] [PATCH 0/3] add new Double Word Key hash tableQuoted 16 lines · Vladimir
Hi Morten,
Quoted 12 lines · Morten Brørup
Quoted 12 lines
From: dev [mailto:[email protected]] On Behalf Of Vladimir Medvedkin
Sent: Monday, March 16, 2020 2:38 PMCurrently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called DWK (double word key) hash supports 8 byte values, which is enough to store a pointer.It would also be nice to get feedback on whether to leave the old FBK and new DWK implementations, or whether to deprecate the old
one?
Quoted 5 lines
Quoted 5 lines
<rant on>
Who comes up with these names?!?
FBK (Four Byte Key) and DWK (Double Word Key) is supposed to mean
the same. Could you use 32 somewhere in the name instead, like in
int32_t, instead of using a growing list of creative synonyms for the same thing? Pretty please, with a cherry on top!Quoted 8 lines
That's true, at first I named it as fbk2, but then it was decided to rename it "dwk", so that there was no confusion with the existing FBK library. Naming suggestions are welcome!
Quoted 2 lines
And if the value size is fixed too, perhaps the name should also indicate
the value size.
Quoted 6 lines
Quoted 6 lines
<rant off>
It's a shame we don't have C++ class templates available in DPDK...
In other news, Mellanox has sent an RFC for an "indexed memory pool"
library [1] to conserve memory by using
uintXX_tinstead of pointers, so perhaps a variant of a 32 bit key hash library with 32 bit values (in addition to 16 bit values in FBK and 64 bit in DWK) would be nice combination with that library.Quoted 4 lines
Quoted 4 lines
[1]: http://mails.dpdk.org/archives/dev/2019-October/147513.html
Quoted 6 lines
Quoted 3 lines
Quoted 3 lines
Med venlig hilsen / kind regards
- Morten BrørupWhy is this different (or better) than existing
rte_hash. Having more flavors is not necessarily a good thing (except in Gelato)
[Wang, Yipeng]
Hi, Vladimir,
As Stephen mentioned, I think it is good idea to explain the benefit of this new type of hash table more explicitly such as Specific use cases, differences with current rte_hash, and performance numbers, etc.
Hi Yipeng, Stephen, all,
Quoted 64 lines · On 17/03/2020 19:52, Wang, Yipeng1 wrote:
Quoted 64 lines
-----Original Message-----
From: Stephen Hemminger <[email protected]>
Sent: Monday, March 16, 2020 12:33 PM
To: Medvedkin, Vladimir <[email protected]>
Cc: Morten Brørup <[email protected]>; [email protected]; Ananyev, Konstantin <[email protected]>; Wang, Yipeng1 <[email protected]>; Gobriel, Sameh <[email protected]>; Richardson, Bruce <[email protected]>; Suanming Mou <[email protected]>; Olivier Matz <[email protected]>; Xueming(Steven) Li <[email protected]>; Andrew Rybchenko <[email protected]>; Asaf Penso <[email protected]>; Ori Kam <[email protected]>
Subject: Re: [dpdk-dev] [PATCH 0/3] add new Double Word Key hash tableQuoted 16 lines · Vladimir
Hi Morten,
Quoted 12 lines · Morten Brørup
Quoted 12 lines
From: dev [mailto:[email protected]] On Behalf Of Vladimir Medvedkin
Sent: Monday, March 16, 2020 2:38 PMCurrently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called DWK (double word key) hash supports 8 byte values, which is enough to store a pointer.It would also be nice to get feedback on whether to leave the old FBK and new DWK implementations, or whether to deprecate the old
one?
Quoted 5 lines
Quoted 5 lines
<rant on>
Who comes up with these names?!?
FBK (Four Byte Key) and DWK (Double Word Key) is supposed to mean
the same. Could you use 32 somewhere in the name instead, like in
int32_t, instead of using a growing list of creative synonyms for the same thing? Pretty please, with a cherry on top!Quoted 6 lines
That's true, at first I named it as fbk2, but then it was decided to rename it "dwk", so that there was no confusion with the existing FBK library. Naming suggestions are welcome!
Quoted 1 line
And if the value size is fixed too, perhaps the name should also indicate
the value size.
Quoted 5 lines
Quoted 5 lines
<rant off>
It's a shame we don't have C++ class templates available in DPDK...
In other news, Mellanox has sent an RFC for an "indexed memory pool"
library [1] to conserve memory by using
uintXX_tinstead of pointers, so perhaps a variant of a 32 bit key hash library with 32 bit values (in addition to 16 bit values in FBK and 64 bit in DWK) would be nice combination with that library.Quoted 6 lines
Quoted 6 lines
[1]: http://mails.dpdk.org/archives/dev/2019-October/147513.html
Med venlig hilsen / kind regards
- Morten Brørup
Quoted 6 lines
Quoted 2 lines
Why is this different (or better) than existing
rte_hash. Having more flavors is not necessarily a good thing (except in Gelato)[Wang, Yipeng]
Hi, Vladimir,
As Stephen mentioned, I think it is good idea to explain the benefit of this new type of hash table more explicitly such as Specific use cases, differences with currentrte_hash, and performance numbers, etc.
The main reason for this new hash library is performance. As I mentioned earlier, the current rte_fbk implementation is pretty fast but it has a number of drawbacks such as 2 byte values and limited collision resolving capabilities. On the other hand, rte_hash (cuckoo hash) doesn't have this drawbacks but at the cost of lower performance comparing to rte_fbk.
If I understand correctly, performance penalty are due to :
1. Load two buckets
2. First compare signatures
3. If signature comparison hits get a key index and find memory location with a key itself and get the key
4. Using indirect call to memcmp() to compare two uint32_t.
The new proposed 4 byte key hash table doesn't have rte_fbk drawbacks while offers the same performance as rte_fbk.
Regarding use cases, in rte_ipsec_sad we are using rte_hash with 4 byte key size. Replacing it with a new implementation gives about 30% in performance.
The main disadvantage comparing to rte_hash is some performance degradation with high average table utilization due to chain resolving for 5th and subsequent collision.
Signature · 2 lines
Regards, Vladimir
26/03/2020 18:28, Medvedkin, Vladimir:
Quoted 49 lines
Hi Yipeng, Stephen, all,
Quoted 46 lines · On 17/03/2020 19:52, Wang, Yipeng1 wrote:
From: Stephen Hemminger <[email protected]>
Quoted 45 lines
Quoted 16 lines · Vladimir
Hi Morten,
Quoted 12 lines · Morten Brørup
Quoted 12 lines
From: dev [mailto:[email protected]] On Behalf Of Vladimir Medvedkin
Sent: Monday, March 16, 2020 2:38 PMCurrently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called DWK (double word key) hash supports 8 byte values, which is enough to store a pointer.It would also be nice to get feedback on whether to leave the old FBK and new DWK implementations, or whether to deprecate the old
one?
Quoted 5 lines
Quoted 5 lines
<rant on>
Who comes up with these names?!?
FBK (Four Byte Key) and DWK (Double Word Key) is supposed to mean
the same. Could you use 32 somewhere in the name instead, like in
int32_t, instead of using a growing list of creative synonyms for the same thing? Pretty please, with a cherry on top!Quoted 6 lines
That's true, at first I named it as fbk2, but then it was decided to rename it "dwk", so that there was no confusion with the existing FBK library. Naming suggestions are welcome!
Quoted 1 line
And if the value size is fixed too, perhaps the name should also indicate
the value size.
Quoted 5 lines
Quoted 5 lines
<rant off>
It's a shame we don't have C++ class templates available in DPDK...
In other news, Mellanox has sent an RFC for an "indexed memory pool"
library [1] to conserve memory by using
uintXX_tinstead of pointers, so perhaps a variant of a 32 bit key hash library with 32 bit values (in addition to 16 bit values in FBK and 64 bit in DWK) would be nice combination with that library.Quoted 1 line
Quoted 1 line
[1]: http://mails.dpdk.org/archives/dev/2019-October/147513.html
Yes some work is in progress to propose a new memory allocator for small objects of fixed size with small memory overhead.
Quoted 28 lines
Quoted 6 lines
Quoted 2 lines
Why is this different (or better) than existing
rte_hash. Having more flavors is not necessarily a good thing (except in Gelato)[Wang, Yipeng]
Hi, Vladimir,
As Stephen mentioned, I think it is good idea to explain the benefit of this new type of hash table more explicitly such as Specific use cases, differences with currentrte_hash, and performance numbers, etc.The main reason for this new hash library is performance. As I mentioned earlier, the current
rte_fbkimplementation is pretty fast but it has a number of drawbacks such as 2 byte values and limited collision resolving capabilities. On the other hand,rte_hash(cuckoo hash) doesn't have this drawbacks but at the cost of lower performance comparing torte_fbk.If I understand correctly, performance penalty are due to :
1. Load two buckets
2. First compare signatures
3. If signature comparison hits get a key index and find memory location with a key itself and get the key
4. Using indirect call to
memcmp()to compare twouint32_t.The new proposed 4 byte key hash table doesn't have
rte_fbkdrawbacks while offers the same performance asrte_fbk.
Quoted 7 lines
Regarding use cases, in
rte_ipsec_sadwe are usingrte_hashwith 4 byte key size. Replacing it with a new implementation gives about 30% in performance.The main disadvantage comparing to
rte_hashis some performance degradation with high average table utilization due to chain resolving for 5th and subsequent collision.
Thanks for explaining.
Please, such information should added in the documentation:
doc/guides/prog_guide/hash_lib.rst
<snip>
Quoted 61 lines
26/03/2020 18:28, Medvedkin, Vladimir:
Quoted 32 lines
Hi Yipeng, Stephen, all,
Quoted 29 lines · On 17/03/2020 19:52, Wang, Yipeng1 wrote:
From: Stephen Hemminger <[email protected]>
Quoted 28 lines
Quoted 17 lines · Vladimir
Hi Morten,
Quoted 13 lines · Morten Brørup
Quoted 13 lines
From: dev [mailto:[email protected]] On Behalf Of Vladimir Medvedkin
Sent: Monday, March 16, 2020 2:38 PMCurrently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called DWK (double word key) hash supports 8 byte values, which is enough to store a pointer.It would also be nice to get feedback on whether to leave the old FBK and new DWK implementations, or whether to deprecate the old
one?
Quoted 5 lines
Quoted 5 lines
<rant on>
Who comes up with these names?!?
FBK (Four Byte Key) and DWK (Double Word Key) is supposed to mean
the same. Could you use 32 somewhere in the name instead, like in
int32_t, instead of using a growing list of creative synonyms for the samething?
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Pretty please, with a cherry on top!
Quoted 5 lines
That's true, at first I named it as fbk2, but then it was decided to rename it "dwk", so that there was no confusion with the existing FBK library. Naming suggestions are welcome!
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 2 lines
Quoted 2 lines
And if the value size is fixed too, perhaps the name should also indicate
the value size.
Quoted 5 lines
Quoted 5 lines
<rant off>
It's a shame we don't have C++ class templates available in DPDK...
In other news, Mellanox has sent an RFC for an "indexed memory
pool"
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
library [1] to conserve memory by using
uintXX_tinstead of pointers, so perhaps a variant of a 32 bit key hash library with 32 bit values (in addition to
16 bit values in FBK and 64 bit in DWK) would be nice combination with that library.Quoted 1 line
Quoted 1 line
[1]: http://mails.dpdk.org/archives/dev/2019-October/147513.html
Yes some work is in progress to propose a new memory allocator for small objects of fixed size with small memory overhead.
Quoted 37 lines
Quoted 7 lines
Quoted 7 lines
Quoted 3 lines
Why is this different (or better) than existing
rte_hash. Having more flavors is not necessarily a good thing (except in Gelato)[Wang, Yipeng]
Hi, Vladimir,
As Stephen mentioned, I think it is good idea to explain the benefit of this new type of hash table more explicitly such as Specific use cases,differences with current
rte_hash, and performance numbers, etc.Quoted 29 lines
The main reason for this new hash library is performance. As I mentioned earlier, the current
rte_fbkimplementation is pretty fast but it has a number of drawbacks such as 2 byte values and limited collision resolving capabilities. On the other hand,rte_hash(cuckoo hash) doesn't have this drawbacks but at the cost of lower performance comparing torte_fbk.If I understand correctly, performance penalty are due to :
1. Load two buckets
2. First compare signatures
3. If signature comparison hits get a key index and find memory location with a key itself and get the key
4. Using indirect call to
memcmp()to compare twouint32_t.The new proposed 4 byte key hash table doesn't have
rte_fbkdrawbacks while offers the same performance asrte_fbk.Regarding use cases, in
rte_ipsec_sadwe are usingrte_hashwith 4 byte key size. Replacing it with a new implementation gives about 30% in performance.The main disadvantage comparing to
rte_hashis some performance degradation with high average table utilization due to chain resolving for 5th and subsequent collision.
rte_hash is linearly scalable across multiple cores for lookup due to lock-free algorithm. How is the scalability for the new algorithm?
Quoted 6 lines
Thanks for explaining.
Please, such information should added in the documentation:
doc/guides/prog_guide/hash_lib.rst
Hi Thomas,
-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Tuesday, March 31, 2020 8:56 PM
To: Medvedkin, Vladimir <[email protected]>
Cc: Wang, Yipeng1 <[email protected]>; Stephen Hemminger <[email protected]>; [email protected]; Morten Brørup <[email protected]>; [email protected]; Ananyev, Konstantin <[email protected]>; Gobriel, Sameh <[email protected]>; Richardson, Bruce <[email protected]>; Suanming Mou <[email protected]>; Olivier Matz <[email protected]>; Xueming(Steven) Li <[email protected]>; Andrew Rybchenko <[email protected]>; Asaf Penso <[email protected]>; Ori Kam <[email protected]>
Subject: Re: [dpdk-dev] [PATCH 0/3] add new Double Word Key hash table
26/03/2020 18:28, Medvedkin, Vladimir:
Quoted 52 lines
Hi Yipeng, Stephen, all,
Quoted 49 lines · On 17/03/2020 19:52, Wang, Yipeng1 wrote:
From: Stephen Hemminger <[email protected]>
Quoted 48 lines
Quoted 17 lines · Vladimir
Hi Morten,
Quoted 13 lines · Morten Brørup
Quoted 13 lines
From: dev [mailto:[email protected]] On Behalf Of Vladimir Medvedkin
Sent: Monday, March 16, 2020 2:38 PMCurrently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called DWK (double word key) hash supports 8 byte values, which is enough to store a pointer.It would also be nice to get feedback on whether to leave the old FBK and new DWK implementations, or whether to deprecate the old
one?
Quoted 5 lines
Quoted 5 lines
<rant on>
Who comes up with these names?!?
FBK (Four Byte Key) and DWK (Double Word Key) is supposed to mean
the same. Could you use 32 somewhere in the name instead, like in
int32_t, instead of using a growing list of creative synonyms for the same thing? Pretty please, with a cherry on top!Quoted 7 lines
That's true, at first I named it as fbk2, but then it was decided to rename it "dwk", so that there was no confusion with the existing FBK library. Naming suggestions are welcome!
Quoted 2 lines
And if the value size is fixed too, perhaps the name should also indicate
the value size.
Quoted 5 lines
Quoted 5 lines
<rant off>
It's a shame we don't have C++ class templates available in DPDK...
In other news, Mellanox has sent an RFC for an "indexed memory pool"
library [1] to conserve memory by using
uintXX_tinstead of pointers, so perhaps a variant of a 32 bit key hash library with 32 bit values (in addition to
16 bit values in FBK and 64 bit in DWK) would be nice combination with that library.Quoted 1 line
Quoted 1 line
[1]: http://mails.dpdk.org/archives/dev/2019-October/147513.html
Yes some work is in progress to propose a new memory allocator for small objects of fixed size with small memory overhead.
Quoted 29 lines
Quoted 7 lines
Quoted 3 lines
Why is this different (or better) than existing
rte_hash. Having more flavors is not necessarily a good thing (except in Gelato)[Wang, Yipeng]
Hi, Vladimir,
As Stephen mentioned, I think it is good idea to explain the benefit
of this new type of hash table more explicitly such as Specific use cases, differences with currentrte_hash, and performance numbers, etc.The main reason for this new hash library is performance. As I mentioned earlier, the current
rte_fbkimplementation is pretty fast but it has a number of drawbacks such as 2 byte values and limited collision resolving capabilities. On the other hand,rte_hash(cuckoo hash) doesn't have this drawbacks but at the cost of lower performance comparing torte_fbk.If I understand correctly, performance penalty are due to :
1. Load two buckets
2. First compare signatures
3. If signature comparison hits get a key index and find memory location with a key itself and get the key
4. Using indirect call to
memcmp()to compare twouint32_t.The new proposed 4 byte key hash table doesn't have
rte_fbkdrawbacks while offers the same performance asrte_fbk.
Quoted 7 lines
Regarding use cases, in
rte_ipsec_sadwe are usingrte_hashwith 4 byte key size. Replacing it with a new implementation gives about 30% in performance.The main disadvantage comparing to
rte_hashis some performance degradation with high average table utilization due to chain resolving for 5th and subsequent collision.
Thanks for explaining.
Please, such information should added in the documentation:
doc/guides/prog_guide/hash_lib.rst
I'm going to submit v2 this week, will add documentation update.
Hi Honnappa,
-----Original Message-----
From: Honnappa Nagarahalli <[email protected]>
Sent: Tuesday, March 31, 2020 10:17 PM
To: [email protected]; Medvedkin, Vladimir <[email protected]>
Cc: Wang, Yipeng1 <[email protected]>; Stephen Hemminger <[email protected]>; [email protected]; Morten Brørup <[email protected]>; [email protected]; Ananyev, Konstantin <[email protected]>; Gobriel, Sameh <[email protected]>; Richardson, Bruce <[email protected]>; Suanming Mou <[email protected]>; Olivier Matz <[email protected]>; Xueming(Steven) Li <[email protected]>; Andrew Rybchenko <[email protected]>; Asaf Penso <[email protected]>; Ori Kam <[email protected]>; nd <[email protected]>; Honnappa Nagarahalli <[email protected]>; nd <[email protected]>
Subject: RE: [dpdk-dev] [PATCH 0/3] add new Double Word Key hash table
<snip>
Quoted 68 lines
26/03/2020 18:28, Medvedkin, Vladimir:
Quoted 35 lines
Hi Yipeng, Stephen, all,
Quoted 32 lines · On 17/03/2020 19:52, Wang, Yipeng1 wrote:
From: Stephen Hemminger <[email protected]>
Quoted 31 lines
Quoted 18 lines · Vladimir
Hi Morten,
Quoted 14 lines · Morten Brørup
Quoted 14 lines
From: dev [mailto:[email protected]] On Behalf Of Vladimir Medvedkin
Sent: Monday, March 16, 2020 2:38 PMCurrently DPDK has a special implementation of a hash table for
4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values. The new implementation called DWK (double word key) hash supports 8 byte values, which is enough to store a pointer.It would also be nice to get feedback on whether to leave the old FBK and new DWK implementations, or whether to deprecate the old
one?
Quoted 6 lines
Quoted 6 lines
<rant on>
Who comes up with these names?!?
FBK (Four Byte Key) and DWK (Double Word Key) is supposed to mean
the same. Could you use 32 somewhere in the name instead, like in
int32_t, instead of using a growing list of creative synonyms for the samething?
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Pretty please, with a cherry on top!
Quoted 5 lines
That's true, at first I named it as fbk2, but then it was decided to rename it "dwk", so that there was no confusion with the existing FBK library. Naming suggestions are welcome!
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 2 lines
Quoted 2 lines
And if the value size is fixed too, perhaps the name should also indicate
the value size.
Quoted 5 lines
Quoted 5 lines
<rant off>
It's a shame we don't have C++ class templates available in DPDK...
In other news, Mellanox has sent an RFC for an "indexed memory
pool"
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
library [1] to conserve memory by using
uintXX_tinstead of pointers, so perhaps a variant of a 32 bit key hash library with 32 bit values (in addition to
16 bit values in FBK and 64 bit in DWK) would be nice combination with that library.Quoted 2 lines
Quoted 2 lines
[1]:
http://mails.dpdk.org/archives/dev/2019-October/147513.htmlYes some work is in progress to propose a new memory allocator for small objects of fixed size with small memory overhead.
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Why is this different (or better) than existing
rte_hash. Having more flavors is not necessarily a good thing (except in Gelato)
Quoted 36 lines
Quoted 5 lines
Quoted 5 lines
[Wang, Yipeng]
Hi, Vladimir,
As Stephen mentioned, I think it is good idea to explain the benefit of this new type of hash table more explicitly such as Specific use cases,differences with current
rte_hash, and performance numbers, etc.Quoted 30 lines
The main reason for this new hash library is performance. As I mentioned earlier, the current
rte_fbkimplementation is pretty fast but it has a number of drawbacks such as 2 byte values and limited collision resolving capabilities. On the other hand,rte_hash(cuckoo
hash) doesn't have this drawbacks but at the cost of lower performance comparing torte_fbk.If I understand correctly, performance penalty are due to :
1. Load two buckets
2. First compare signatures
3. If signature comparison hits get a key index and find memory location with a key itself and get the key
4. Using indirect call to
memcmp()to compare twouint32_t.The new proposed 4 byte key hash table doesn't have
rte_fbkdrawbacks while offers the same performance asrte_fbk.Regarding use cases, in
rte_ipsec_sadwe are usingrte_hashwith 4 byte key size. Replacing it with a new implementation gives about 30% in performance.The main disadvantage comparing to
rte_hashis some performance degradation with high average table utilization due to chain resolving for 5th and subsequent collision.
rte_hash is linearly scalable across multiple cores for lookup due to lock-free algorithm. How is the scalability for the new algorithm?
This library is scalable as well. It uses almost the same lock-free algorithm. The only difference with cuckoo is that cuckoo in lock-free implementation uses single global "change_counter" for all the table, and the proposed implementation uses fine grained approach with "change_counter" per bucket. So it should be more scalable with frequent concurrent updates.
Quoted 6 lines
Thanks for explaining.
Please, such information should added in the documentation:
doc/guides/prog_guide/hash_lib.rst
v2 cover add new k32v64 hash table
In reply to Vladimir MedvedkinCurrently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called K32V64 hash
supports 4 byte keys and 8 byte associated values,
which is enough to store a pointer.
It would also be nice to get feedback on whether to leave the old FBK and new k32v64 implementations or deprecate the old one?
v2:
- renamed from rte_dwk to rte_k32v64 as was suggested
- reworked lookup function, added inlined subroutines
- added avx512 key comparizon routine
- added documentation
- added statistic counters for total entries and extended entries(linked list)
Vladimir Medvedkin (4):
hash: add k32v64 hash library
hash: add documentation for k32v64 hash library
test: add k32v64 hash autotests
test: add k32v64 perf tests
app/test/Makefile | 1 + app/test/autotest_data.py | 12 ++ app/test/meson.build | 3 + app/test/test_hash_perf.c | 83 +++++++++ app/test/test_k32v64_hash.c | 229 ++++++++++++++++++++++++ doc/api/doxy-api-index.md | 1 + doc/guides/prog_guide/index.rst | 1 + doc/guides/prog_guide/k32v64_hash_lib.rst | 66 +++++++ lib/Makefile | 2 +- lib/librte_hash/Makefile | 4 +- lib/librte_hash/meson.build | 5 +- lib/librte_hash/rte_hash_version.map | 6 +- lib/librte_hash/rte_k32v64_hash.c | 279 ++++++++++++++++++++++++++++++ lib/librte_hash/rte_k32v64_hash.h | 214 +++++++++++++++++++++++ 14 files changed, 901 insertions(+), 5 deletions(-) create mode 100644 app/test/test_k32v64_hash.c create mode 100644 doc/guides/prog_guide/k32v64_hash_lib.rst create mode 100644 lib/librte_hash/rte_k32v64_hash.c create mode 100644 lib/librte_hash/rte_k32v64_hash.h
Signature · 1 line
2.7.4
PATCH v2 1/4 hash: add k32v64 hash library
K32V64 hash is a hash table that supports 32 bit keys and 64 bit values. This table is hash function agnostic so user must provide precalculated hash signature for add/delete/lookup operations.
--- lib/Makefile | 2 +- lib/librte_hash/Makefile | 4 +- lib/librte_hash/meson.build | 5 +- lib/librte_hash/rte_hash_version.map | 6 +- lib/librte_hash/rte_k32v64_hash.c | 279 +++++++++++++++++++++++++++++++++++ lib/librte_hash/rte_k32v64_hash.h | 214 +++++++++++++++++++++++++++ 6 files changed, 505 insertions(+), 5 deletions(-) create mode 100644 lib/librte_hash/rte_k32v64_hash.c create mode 100644 lib/librte_hash/rte_k32v64_hash.h
lib/Makefile +1 -1
@@ -48,7 +48,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_VHOST) += librte_vhost DEPDIRS-librte_vhost := librte_eal librte_mempool librte_mbuf librte_ethdev \ librte_net librte_hash librte_cryptodev DIRS-$(CONFIG_RTE_LIBRTE_HASH) += librte_hash -DEPDIRS-librte_hash := librte_eal librte_ring +DEPDIRS-librte_hash := librte_eal librte_ring librte_mempool DIRS-$(CONFIG_RTE_LIBRTE_EFD) += librte_efd DEPDIRS-librte_efd := librte_eal librte_ring librte_hash DIRS-$(CONFIG_RTE_LIBRTE_RIB) += librte_rib
lib/librte_hash/Makefile +3 -1
@@ -8,13 +8,14 @@ LIB = librte_hash.a CFLAGS += -O3 -DALLOW_EXPERIMENTAL_API CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_ring +LDLIBS += -lrte_eal -lrte_ring -lrte_mempool EXPORT_MAP := rte_hash_version.map # all source are stored in SRCS-y SRCS-$(CONFIG_RTE_LIBRTE_HASH) := rte_cuckoo_hash.c SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_fbk_hash.c +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_k32v64_hash.c # install this header file SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include := rte_hash.h
@@ -27,5 +28,6 @@ endif SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_jhash.h SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_thash.h SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_fbk_hash.h +SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_k32v64_hash.h include $(RTE_SDK)/mk/rte.lib.mk
lib/librte_hash/meson.build +3 -2
@@ -3,13 +3,14 @@ headers = files('rte_crc_arm64.h', 'rte_fbk_hash.h', + 'rte_k32v64_hash.h', 'rte_hash_crc.h', 'rte_hash.h', 'rte_jhash.h', 'rte_thash.h') -sources = files('rte_cuckoo_hash.c', 'rte_fbk_hash.c') -deps += ['ring'] +sources = files('rte_cuckoo_hash.c', 'rte_fbk_hash.c', 'rte_k32v64_hash.c') +deps += ['ring', 'mempool'] # rte ring reset is not yet part of stable API allow_experimental_apis = true
lib/librte_hash/rte_hash_version.map +5 -1
@@ -34,5 +34,9 @@ EXPERIMENTAL { rte_hash_free_key_with_position; rte_hash_max_key_id; - + rte_k32v64_hash_create; + rte_k32v64_hash_find_existing; + rte_k32v64_hash_free; + rte_k32v64_hash_add; + rte_k32v64_hash_delete; };
lib/librte_hash/rte_k32v64_hash.c new file +279 -0
279 lines not on this page · raw message
lib/librte_hash/rte_k32v64_hash.h new file +214 -0
214 lines not on this page · raw message
Signature · 1 line
2.7.4
PATCH v2 2/4 hash: add documentation for k32v64 hash library
In reply to Vladimir MedvedkinAdd programmers guide and doxygen API for k32v64 hash library
--- doc/api/doxy-api-index.md | 1 + doc/guides/prog_guide/index.rst | 1 + doc/guides/prog_guide/k32v64_hash_lib.rst | 66 +++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 doc/guides/prog_guide/k32v64_hash_lib.rst
doc/api/doxy-api-index.md +1 -0
@@ -121,6 +121,7 @@ The public API headers are grouped by topics: [jhash] (@ref rte_jhash.h), [thash] (@ref rte_thash.h), [FBK hash] (@ref rte_fbk_hash.h), + [K32V64 hash] (@ref rte_k32v64_hash.h), [CRC hash] (@ref rte_hash_crc.h) - **classification**
doc/guides/prog_guide/index.rst +1 -0
@@ -30,6 +30,7 @@ Programmer's Guide link_bonding_poll_mode_drv_lib timer_lib hash_lib + k32v64_hash_lib efd_lib member_lib lpm_lib
doc/guides/prog_guide/k32v64_hash_lib.rst new file +66 -0
66 lines not on this page · raw message
Signature · 1 line
2.7.4
PATCH v2 3/4 test: add k32v64 hash autotests
In reply to Vladimir MedvedkinAdd autotests for rte_k32v64_hash library
--- app/test/Makefile | 1 + app/test/autotest_data.py | 12 +++ app/test/meson.build | 3 + app/test/test_k32v64_hash.c | 229 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 245 insertions(+) create mode 100644 app/test/test_k32v64_hash.c
app/test/Makefile +1 -0
@@ -73,6 +73,7 @@ SRCS-y += test_bitmap.c SRCS-y += test_reciprocal_division.c SRCS-y += test_reciprocal_division_perf.c SRCS-y += test_fbarray.c +SRCS-y += test_k32v64_hash.c SRCS-y += test_external_mem.c SRCS-y += test_rand_perf.c
app/test/autotest_data.py +12 -0
@@ -99,6 +99,18 @@ "Report": None, }, { + "Name": "K32V64 hash autotest", + "Command": "k32v64_hash_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "K32V64 hash autotest", + "Command": "k32v64_hash_slow_autotest", + "Func": default_autotest, + "Report": None, + }, + { "Name": "LPM autotest", "Command": "lpm_autotest", "Func": default_autotest,
app/test/meson.build +3 -0
@@ -45,6 +45,7 @@ test_sources = files('commands.c', 'test_eventdev.c', 'test_external_mem.c', 'test_fbarray.c', + 'test_k32v64_hash.c', 'test_fib.c', 'test_fib_perf.c', 'test_fib6.c',
@@ -185,6 +186,7 @@ fast_test_names = [ 'flow_classify_autotest', 'hash_autotest', 'interrupt_autotest', + 'k32v64_hash_autotest', 'logs_autotest', 'lpm_autotest', 'lpm6_autotest',
@@ -270,6 +272,7 @@ perf_test_names = [ 'rand_perf_autotest', 'hash_readwrite_perf_autotest', 'hash_readwrite_lf_perf_autotest', + 'k32v64_hash_slow_autotest', ] driver_test_names = [
app/test/test_k32v64_hash.c new file +229 -0
229 lines not on this page · raw message
Signature · 1 line
2.7.4
PATCH v2 4/4 test: add k32v64 perf tests
In reply to Vladimir MedvedkinAdd performance tests for rte_k32v64_hash
--- app/test/test_hash_perf.c | 83 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+)
app/test/test_hash_perf.c +83 -0
@@ -12,8 +12,10 @@#include <rte_hash_crc.h> #include <rte_jhash.h> #include <rte_fbk_hash.h> +#include <rte_k32v64_hash.h> #include <rte_random.h> #include <rte_string_fns.h> +#include <rte_hash_crc.h> #include "test.h"
@@ -29,6 +31,8 @@#define NUM_SHUFFLES 10#define BURST_SIZE 16 +#define CRC_INIT_VAL 0xdeadbeef +enum operations {ADD =0 ,LOOKUP ,
@@ -669,6 +673,82 @@ fbk_hash_perf_test(void) }static int +k32v64_hash_perf_test (void ) +{ +struct rte_k32v64_hash_params params = { + .name ="k32v64_hash_test" , + .entries =ENTRIES *2 , + .socket_id =rte_socket_id (), + }; +struct rte_k32v64_hash_table *handle =NULL ; +uint32_t *keys =NULL ; +unsigned int indexes[TEST_SIZE ]; +uint64_t tmp_val; +uint64_t lookup_time =0 ; +uint64_t begin; +uint64_t end; +unsigned int added =0 ; +uint32_t key; +uint16_t val; +unsigned int i, j; +int ret =0 ; + + handle =rte_k32v64_hash_create (¶ms); +if (handle ==NULL ) { +printf ("Error creating table\n" ); +return -1 ; + } + + keys =rte_zmalloc (NULL ,ENTRIES *sizeof (*keys),0 ); +if (keys ==NULL ) { +printf ("fbk hash: memory allocation for key store failed\n" ); +return -1 ; + } + +/* Generate random keys and values. */ +for (i =0 ; i <ENTRIES ; i++) { + key = (uint32_t )rte_rand (); + val =rte_rand (); + +if (rte_k32v64_hash_add (handle, key,rte_hash_crc_4byte (key, +CRC_INIT_VAL ), val) ==0 ) { + keys[added] = key; + added++; + } + } + +for (i =0 ; i <TEST_ITERATIONS ; i++) { + +/* Generate random indexes into keys[] array. */ +for (j =0 ; j <TEST_SIZE ; j++) + indexes[j] =rte_rand () % added; + + begin =rte_rdtsc (); +/* Do lookups */ + +for (j =0 ; j <TEST_SIZE ; j++) + ret +=rte_k32v64_hash_lookup (handle, + keys[indexes[j]], +rte_hash_crc_4byte (keys[indexes[j]], +CRC_INIT_VAL ), &tmp_val); + + + end =rte_rdtsc (); + lookup_time += (double )(end - begin); + } + +printf ("\n\n *** K32V64 Hash function performance test results ***\n" ); +if (ret ==0 ) +printf ("Number of ticks per lookup = %g\n" , + (double )lookup_time / + ((double )TEST_ITERATIONS * (double )TEST_SIZE )); + +rte_k32v64_hash_free (handle); + +return 0 ; +} + +static int test_hash_perf (void ) {unsigned int with_pushes, with_locks;
@@ -695,6 +775,9 @@ test_hash_perf(void)if (fbk_hash_perf_test () <0 )return -1 ; +if (k32v64_hash_perf_test () <0 ) +return -1 ; +return 0 ; }
Signature · 1 line
2.7.4
Re: [dpdk-dev] [PATCH v2 1/4] hash: add k32v64 hash library
In reply to Vladimir MedvedkinHi Vladimir,
I didn't look at actual implementation (yet), just some compatibility comments.
Quoted 18 lines
K32V64 hash is a hash table that supports 32 bit keys and 64 bit values. This table is hash function agnostic so user must provide precalculated hash signature for add/delete/lookup operations.
Signed-off-by Vladimir Medvedkin <[email protected]>---
lib/librte_hash/rte_k32v64_hash.h new file +6
-0@@ -0,0 +1,214 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#ifndef _RTE_K32V64_HASH_H_ +#define _RTE_K32V64_HASH_H_
lib/librte_hash/rte_k32v64_hash.h line 16
Quoted 10 lines
lib/librte_hash/rte_k32v64_hash.h new file +10
-0+ +#ifdef __cplusplus +extern "C" { +#endif + +#include <rte_compat.h> +#include <rte_atomic.h> +#include <rte_mempool.h> + +#include <immintrin.h>
How that supposed to compile on non-X86 box?
Quoted 54 lines
lib/librte_hash/rte_k32v64_hash.h new file +54
-0+ +#define RTE_K32V64_HASH_NAMESIZE 32 +#define RTE_K32V64_KEYS_PER_BUCKET 4 +#define RTE_K32V64_WRITE_IN_PROGRESS 1 + +struct rte_k32v64_hash_params { +const char *name; +uint32_t entries; +int socket_id; +}; + +struct rte_k32v64_ext_ent { +SLIST_ENTRY (rte_k32v64_ext_ent) next; +uint32_t key; +uint64_t val; +}; + +struct rte_k32v64_hash_bucket { +uint32_t key[RTE_K32V64_KEYS_PER_BUCKET ]; +uint64_t val[RTE_K32V64_KEYS_PER_BUCKET ]; +uint8_t key_mask; +rte_atomic32_t cnt; + SLIST_HEAD(rte_k32v64_list_head, rte_k32v64_ext_ent) head; +} __rte_cache_aligned; + +struct rte_k32v64_hash_table { +char name[RTE_K32V64_HASH_NAMESIZE ];/**< Name of the hash. */ +uint32_t nb_ent; +uint32_t nb_ext_ent; +uint32_t max_ent; +uint32_t bucket_msk; +struct rte_mempool *ext_ent_pool; + __extension__struct rte_k32v64_hash_bucket t[0 ]; +}; + +static inline int +cmp_keys (struct rte_k32v64_hash_bucket *bucket,uint32_t key, +uint64_t *val) +{ +int i; + +for (i =0 ; i <RTE_K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == bucket->key[i]) && + (bucket->key_mask & (1 << i))) { + *val = bucket->val[i]; +return 1 ; + } + } + +return 0 ; +} + +#ifdef __AVX512VL__ +static inline int
lib/librte_hash/rte_k32v64_hash.h line 80
Quoted 10 lines
lib/librte_hash/rte_k32v64_hash.h new file +10
-0+cmp_keys_vec(struct rte_k32v64_hash_bucket *bucket, uint32_t key, + uint64_t *val) +{ + __m128i keys, srch_key; +__mmask8 msk; + + keys =_mm_load_si128 ((void *)bucket); + srch_key =_mm_set1_epi32 (key); + + msk =_mm_mask_cmpeq_epi32_mask (bucket->key_mask, keys, srch_key);
What if you'll run it on IA cpu without avx512 support?
Think you need there some run-time selection to decide which function to use, depending on the underlying HW.
Quoted 136 lines
lib/librte_hash/rte_k32v64_hash.h new file +134
-1+if (msk) { + *val = bucket->val[__builtin_ctz (msk)]; +return 1 ; + } + +return 0 ; +} +#endif + +static inline int +rte_k32v64_hash_lookup (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *value) +{ +uint64_t val =0 ; +struct rte_k32v64_ext_ent *ent; +int32_t cnt; + int i __rte_unused, found = 0; +uint32_t bucket = hash & table->bucket_msk; + +do { +do + cnt =rte_atomic32_read (&table->t[bucket].cnt); +while (unlikely (cnt &RTE_K32V64_WRITE_IN_PROGRESS )); + +#ifdef __AVX512VL__ + found =cmp_keys_vec (&table->t[bucket], key, &val); +#else + found =cmp_keys (&table->t[bucket], key, &val); +#endif +if (unlikely ((found ==0 ) && + (!SLIST_EMPTY (&table->t[bucket].head)))) { + SLIST_FOREACH(ent, &table->t[bucket].head, next) { +if (ent->key == key) { + val = ent->val; + found =1 ; +break ; + } + } + } + + }while (unlikely (cnt !=rte_atomic32_read (&table->t[bucket].cnt))); + +if (found ==1 ) { + *value = val; +return 0 ; + }else +return -ENOENT ; +} + +/** +* Add a key to an existing hash table with hash value. +* This operation is not multi-thread safe +* and should only be called from one thread. +* +* @param ht +* Hash table to add the key to. +* @param key +* Key to add to the hash table. +* @param value +* Value to associate with key. +* @param hash +* Hash value associated with key. +* @return +* 0 if ok, or negative value on error. +*/ +__rte_experimental +int +rte_k32v64_hash_add (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t value); + +/** +* Remove a key with a given hash value from an existing hash table. +* This operation is not multi-thread +* safe and should only be called from one thread. +* +* @param ht +* Hash table to remove the key from. +* @param key +* Key to remove from the hash table. +* @param hash +* hash value associated with key. +* @return +* 0 if ok, or negative value on error. +*/ +__rte_experimental +int +rte_k32v64_hash_delete (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash); + + +/** +* Performs a lookup for an existing hash table, and returns a pointer to +* the table if found. +* +* @param name +* Name of the hash table to find +* +* @return +* pointer to hash table structure or NULL on error with rte_errno +* set appropriately. +*/ +__rte_experimental +struct rte_k32v64_hash_table * +rte_k32v64_hash_find_existing (const char *name); + +/** +* Create a new hash table for use with four byte keys. +* +* @param params +* Parameters used in creation of hash table. +* +* @return +* Pointer to hash table structure that is used in future hash table +* operations, or NULL on error with rte_errno set appropriately. +*/ +__rte_experimental +struct rte_k32v64_hash_table * +rte_k32v64_hash_create (const struct rte_k32v64_hash_params *params); + +/** +* Free all memory used by a hash table. +* +* @param table +* Hash table to deallocate. +*/ +__rte_experimental +void +rte_k32v64_hash_free (struct rte_k32v64_hash_table *table); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_K32V64_HASH_H_ */ --2.7.4
v3 cover add new k32v64 hash table
In reply to Vladimir MedvedkinCurrently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called K32V64 hash
supports 4 byte keys and 8 byte associated values,
which is enough to store a pointer.
It would also be nice to get feedback on whether to leave the old FBK and new k32v64 implementations or deprecate the old one?
v3:
- added bulk lookup
- avx512 key comparizon is removed from .h
v2:
- renamed from rte_dwk to rte_k32v64 as was suggested
- reworked lookup function, added inlined subroutines
- added avx512 key comparizon routine
- added documentation
- added statistic counters for total entries and extended entries(linked list)
Vladimir Medvedkin (4):
hash: add k32v64 hash library
hash: add documentation for k32v64 hash library
test: add k32v64 hash autotests
test: add k32v64 perf tests
app/test/Makefile | 1 + app/test/autotest_data.py | 12 ++ app/test/meson.build | 3 + app/test/test_hash_perf.c | 130 ++++++++++++ app/test/test_k32v64_hash.c | 229 ++++++++++++++++++++++ doc/api/doxy-api-index.md | 1 + doc/guides/prog_guide/index.rst | 1 + doc/guides/prog_guide/k32v64_hash_lib.rst | 66 +++++++ lib/Makefile | 2 +- lib/librte_hash/Makefile | 13 +- lib/librte_hash/k32v64_hash_avx512vl.c | 56 ++++++ lib/librte_hash/meson.build | 17 +- lib/librte_hash/rte_hash_version.map | 6 +- lib/librte_hash/rte_k32v64_hash.c | 315 ++++++++++++++++++++++++++++++ lib/librte_hash/rte_k32v64_hash.h | 211 ++++++++++++++++++++ 15 files changed, 1058 insertions(+), 5 deletions(-) create mode 100644 app/test/test_k32v64_hash.c create mode 100644 doc/guides/prog_guide/k32v64_hash_lib.rst create mode 100644 lib/librte_hash/k32v64_hash_avx512vl.c create mode 100644 lib/librte_hash/rte_k32v64_hash.c create mode 100644 lib/librte_hash/rte_k32v64_hash.h
Signature · 1 line
2.7.4
PATCH v3 1/4 hash: add k32v64 hash library
K32V64 hash is a hash table that supports 32 bit keys and 64 bit values. This table is hash function agnostic so user must provide precalculated hash signature for add/delete/lookup operations.
--- lib/Makefile | 2 +- lib/librte_hash/Makefile | 13 +- lib/librte_hash/k32v64_hash_avx512vl.c | 56 ++++++ lib/librte_hash/meson.build | 17 +- lib/librte_hash/rte_hash_version.map | 6 +- lib/librte_hash/rte_k32v64_hash.c | 315 +++++++++++++++++++++++++++++++++ lib/librte_hash/rte_k32v64_hash.h | 211 ++++++++++++++++++++++ 7 files changed, 615 insertions(+), 5 deletions(-) create mode 100644 lib/librte_hash/k32v64_hash_avx512vl.c create mode 100644 lib/librte_hash/rte_k32v64_hash.c create mode 100644 lib/librte_hash/rte_k32v64_hash.h
lib/Makefile +1 -1
@@ -48,7 +48,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_VHOST) += librte_vhost DEPDIRS-librte_vhost := librte_eal librte_mempool librte_mbuf librte_ethdev \ librte_net librte_hash librte_cryptodev DIRS-$(CONFIG_RTE_LIBRTE_HASH) += librte_hash -DEPDIRS-librte_hash := librte_eal librte_ring +DEPDIRS-librte_hash := librte_eal librte_ring librte_mempool DIRS-$(CONFIG_RTE_LIBRTE_EFD) += librte_efd DEPDIRS-librte_efd := librte_eal librte_ring librte_hash DIRS-$(CONFIG_RTE_LIBRTE_RIB) += librte_rib
lib/librte_hash/Makefile +12 -1
@@ -8,13 +8,14 @@ LIB = librte_hash.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_ring +LDLIBS += -lrte_eal -lrte_ring -lrte_mempool EXPORT_MAP := rte_hash_version.map # all source are stored in SRCS-y SRCS-$(CONFIG_RTE_LIBRTE_HASH) := rte_cuckoo_hash.c SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_fbk_hash.c +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_k32v64_hash.c # install this header file SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include := rte_hash.h
@@ -27,5 +28,15 @@ endif SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_jhash.h SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_thash.h SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_fbk_hash.h +SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_k32v64_hash.h + +CC_AVX512VL_SUPPORT=$(shell $(CC) -mavx512vl -dM -E - </dev/null 2>&1 | \ +grep -q __AVX512VL__ && echo 1) + +ifeq ($(CC_AVX512VL_SUPPORT), 1) + SRCS-$(CONFIG_RTE_LIBRTE_HASH) += k32v64_hash_avx512vl.c + CFLAGS_k32v64_hash_avx512vl.o += -mavx512vl + CFLAGS_rte_k32v64_hash.o += -DCC_AVX512VL_SUPPORT +endif include $(RTE_SDK)/mk/rte.lib.mk
lib/librte_hash/k32v64_hash_avx512vl.c new file +56 -0
56 lines not on this page · raw message
lib/librte_hash/meson.build +15 -2
@@ -3,10 +3,23 @@ headers = files('rte_crc_arm64.h', 'rte_fbk_hash.h', + 'rte_k32v64_hash.h', 'rte_hash_crc.h', 'rte_hash.h', 'rte_jhash.h', 'rte_thash.h') -sources = files('rte_cuckoo_hash.c', 'rte_fbk_hash.c') -deps += ['ring'] +sources = files('rte_cuckoo_hash.c', 'rte_fbk_hash.c', 'rte_k32v64_hash.c') +deps += ['ring', 'mempool'] + +if dpdk_conf.has('RTE_ARCH_X86') + if cc.has_argument('-mavx512vl') + avx512_tmplib = static_library('avx512_tmp', + 'k32v64_hash_avx512vl.c', + dependencies: static_rte_mempool, + c_args: cflags + ['-mavx512vl']) + objs += avx512_tmplib.extract_objects('k32v64_hash_avx512vl.c') + cflags += '-DCC_AVX512VL_SUPPORT' + + endif +endif
lib/librte_hash/rte_hash_version.map +5 -1
@@ -34,5 +34,9 @@ EXPERIMENTAL { rte_hash_free_key_with_position; rte_hash_max_key_id; - + rte_k32v64_hash_create; + rte_k32v64_hash_find_existing; + rte_k32v64_hash_free; + rte_k32v64_hash_add; + rte_k32v64_hash_delete; };
lib/librte_hash/rte_k32v64_hash.c new file +315 -0
315 lines not on this page · raw message
lib/librte_hash/rte_k32v64_hash.h new file +211 -0
211 lines not on this page · raw message
Signature · 1 line
2.7.4
PATCH v3 2/4 hash: add documentation for k32v64 hash library
In reply to Vladimir MedvedkinAdd programmers guide and doxygen API for k32v64 hash library
--- doc/api/doxy-api-index.md | 1 + doc/guides/prog_guide/index.rst | 1 + doc/guides/prog_guide/k32v64_hash_lib.rst | 66 +++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 doc/guides/prog_guide/k32v64_hash_lib.rst
doc/api/doxy-api-index.md +1 -0
@@ -121,6 +121,7 @@ The public API headers are grouped by topics: [jhash] (@ref rte_jhash.h), [thash] (@ref rte_thash.h), [FBK hash] (@ref rte_fbk_hash.h), + [K32V64 hash] (@ref rte_k32v64_hash.h), [CRC hash] (@ref rte_hash_crc.h) - **classification**
doc/guides/prog_guide/index.rst +1 -0
@@ -30,6 +30,7 @@ Programmer's Guide link_bonding_poll_mode_drv_lib timer_lib hash_lib + k32v64_hash_lib efd_lib member_lib lpm_lib
doc/guides/prog_guide/k32v64_hash_lib.rst new file +66 -0
66 lines not on this page · raw message
Signature · 1 line
2.7.4
PATCH v3 3/4 test: add k32v64 hash autotests
In reply to Vladimir MedvedkinAdd autotests for rte_k32v64_hash library
--- app/test/Makefile | 1 + app/test/autotest_data.py | 12 +++ app/test/meson.build | 3 + app/test/test_k32v64_hash.c | 229 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 245 insertions(+) create mode 100644 app/test/test_k32v64_hash.c
app/test/Makefile +1 -0
@@ -73,6 +73,7 @@ SRCS-y += test_bitmap.c SRCS-y += test_reciprocal_division.c SRCS-y += test_reciprocal_division_perf.c SRCS-y += test_fbarray.c +SRCS-y += test_k32v64_hash.c SRCS-y += test_external_mem.c SRCS-y += test_rand_perf.c
app/test/autotest_data.py +12 -0
@@ -99,6 +99,18 @@ "Report": None, }, { + "Name": "K32V64 hash autotest", + "Command": "k32v64_hash_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "K32V64 hash autotest", + "Command": "k32v64_hash_slow_autotest", + "Func": default_autotest, + "Report": None, + }, + { "Name": "LPM autotest", "Command": "lpm_autotest", "Func": default_autotest,
app/test/meson.build +3 -0
@@ -45,6 +45,7 @@ test_sources = files('commands.c', 'test_eventdev.c', 'test_external_mem.c', 'test_fbarray.c', + 'test_k32v64_hash.c', 'test_fib.c', 'test_fib_perf.c', 'test_fib6.c',
@@ -187,6 +188,7 @@ fast_tests = [ ['flow_classify_autotest', false], ['hash_autotest', true], ['interrupt_autotest', true], + ['k32v64_hash_autotest', true], ['logs_autotest', true], ['lpm_autotest', true], ['lpm6_autotest', true],
@@ -272,6 +274,7 @@ perf_test_names = [ 'rand_perf_autotest', 'hash_readwrite_perf_autotest', 'hash_readwrite_lf_perf_autotest', + 'k32v64_hash_slow_autotest', ] driver_test_names = [
app/test/test_k32v64_hash.c new file +229 -0
229 lines not on this page · raw message
Signature · 1 line
2.7.4
PATCH v3 4/4 test: add k32v64 perf tests
In reply to Vladimir MedvedkinAdd performance tests for rte_k32v64_hash
--- app/test/test_hash_perf.c | 130 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+)
app/test/test_hash_perf.c +130 -0
@@ -12,8 +12,10 @@#include <rte_hash_crc.h> #include <rte_jhash.h> #include <rte_fbk_hash.h> +#include <rte_k32v64_hash.h> #include <rte_random.h> #include <rte_string_fns.h> +#include <rte_hash_crc.h> #include "test.h"
@@ -29,6 +31,8 @@#define NUM_SHUFFLES 10#define BURST_SIZE 16 +#define CRC_INIT_VAL 0xdeadbeef +enum operations {ADD =0 ,LOOKUP ,
@@ -668,6 +672,129 @@ fbk_hash_perf_test(void)return 0 ; } +static uint32_t * +shuf_arr (uint32_t *arr,int n,int l) +{ +int i, j; +uint32_t tmp; +uint32_t *ret_arr; + + ret_arr =rte_zmalloc (NULL , l *sizeof (uint32_t ),0 ); +for (i =0 ; i < n; i++) { + j =rte_rand () % n; + tmp = arr[j]; + arr[j] = arr[i]; + arr[i] = tmp; + } +for (i =0 ; i < l; i++) + ret_arr[i] = arr[i % n]; + +return ret_arr; +} + +static int +k32v64_hash_perf_test (void ) +{ +struct rte_k32v64_hash_params params = { + .name ="k32v64_hash_test" , + .entries =ENTRIES *2 , + .socket_id =rte_socket_id (), + }; +struct rte_k32v64_hash_table *handle =NULL ; +uint32_t *keys; +uint32_t *lookup_keys; +uint64_t tmp_val; +uint64_t lookup_time =0 ; +uint64_t begin; +uint64_t end; +unsigned int added =0 ; +uint32_t key; +uint16_t val; +unsigned int i, j, k; +int ret =0 ; +uint32_t hashes[64 ]; +uint64_t vals[64 ]; + + handle =rte_k32v64_hash_create (¶ms); +if (handle ==NULL ) { +printf ("Error creating table\n" ); +return -1 ; + } + + keys =rte_zmalloc (NULL ,ENTRIES *sizeof (*keys),0 ); +if (keys ==NULL ) { +printf ("fbk hash: memory allocation for key store failed\n" ); +return -1 ; + } + +/* Generate random keys and values. */ +for (i =0 ; i <ENTRIES ; i++) { + key = (uint32_t )rte_rand (); + val =rte_rand (); + +if (rte_k32v64_hash_add (handle, key,rte_hash_crc_4byte (key, +CRC_INIT_VAL ), val) ==0 ) { + keys[added] = key; + added++; + } + } + + lookup_keys =shuf_arr (keys, added,TEST_SIZE ); + +for (i =0 ; i <TEST_ITERATIONS ; i++) { + + begin =rte_rdtsc (); +/* Do lookups */ + +for (j =0 ; j <TEST_SIZE ; j++) + ret +=rte_k32v64_hash_lookup (handle, lookup_keys[j], +rte_hash_crc_4byte (lookup_keys[j], +CRC_INIT_VAL ), &tmp_val); + + end =rte_rdtsc (); + lookup_time += (double )(end - begin); + } + +printf ("\n\n *** K32V64 Hash function performance test results ***\n" ); +if (ret ==0 ) +printf ("Number of ticks per lookup = %g\n" , + (double )lookup_time / + ((double )TEST_ITERATIONS * (double )TEST_SIZE )); + + lookup_time =0 ; +for (i =0 ; i <TEST_ITERATIONS ; i++) { + + begin =rte_rdtsc (); +/* Do lookups */ + +for (j =0 ; j <TEST_SIZE ; j +=64 ) { +for (k =0 ; k <64 ; k++) { + hashes[k] = +rte_hash_crc_4byte (lookup_keys[j + k], +CRC_INIT_VAL ); + } + + ret +=rte_k32v64_hash_bulk_lookup (handle, + &lookup_keys[j], hashes, vals,64 ); + } + + end =rte_rdtsc (); + lookup_time += (double )(end - begin); + } + +printf ("\n\n *** K32V64 Hash function performance test results ***\n" ); +if (ret !=0 ) +printf ("Number of ticks per bulk lookup = %g\n" , + (double )lookup_time / + ((double )TEST_ITERATIONS * (double )TEST_SIZE )); + +rte_free (keys); +rte_free (lookup_keys); +rte_k32v64_hash_free (handle); + +return 0 ; +} +static int test_hash_perf (void ) {
@@ -695,6 +822,9 @@ test_hash_perf(void)if (fbk_hash_perf_test () <0 )return -1 ; +if (k32v64_hash_perf_test () <0 ) +return -1 ; +return 0 ; }
Signature · 1 line
2.7.4
Re: [dpdk-dev] [PATCH v3 0/4] add new k32v64 hash table
In reply to Vladimir MedvedkinQuoted 9 lines · Vladimir Medvedkin
Currently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called K32V64 hash
supports 4 byte keys and 8 byte associated values,
which is enough to store a pointer.It would also be nice to get feedback on whether to leave the old FBK and new k32v64 implementations or deprecate the old one?
Do you think it would be feasible to support custom-sized values and remain efficient, in a similar manner to how rte_ring_elem.h does things?
Quoted 39 lines
v3:
- added bulk lookup
- avx512 key comparizon is removed from .hv2:
- renamed fromrte_dwktorte_k32v64as was suggested
- reworked lookup function, added inlined subroutines
- added avx512 key comparizon routine
- added documentation
- added statistic counters for total entries and extended entries(linked list)Vladimir Medvedkin (4):
hash: add k32v64 hash library
hash: add documentation for k32v64 hash library
test: add k32v64 hash autotests
test: add k32v64 perf testsapp/test/Makefile | 1 + app/test/autotest_data.py | 12 ++ app/test/meson.build | 3 + app/test/test_hash_perf.c | 130 ++++++++++++ app/test/test_k32v64_hash.c | 229 ++++++++++++++++++++++ doc/api/doxy-api-index.md | 1 + doc/guides/prog_guide/index.rst | 1 + doc/guides/prog_guide/k32v64_hash_lib.rst | 66 +++++++ lib/Makefile | 2 +- lib/librte_hash/Makefile | 13 +- lib/librte_hash/k32v64_hash_avx512vl.c | 56 ++++++ lib/librte_hash/meson.build | 17 +- lib/librte_hash/rte_hash_version.map | 6 +- lib/librte_hash/rte_k32v64_hash.c | 315 ++++++++++++++++++++++++++++++ lib/librte_hash/rte_k32v64_hash.h | 211 ++++++++++++++++++++ 15 files changed, 1058 insertions(+), 5 deletions(-)create mode 100644
app/test/test_k32v64_hash.ccreate mode 100644doc/guides/prog_guide/k32v64_hash_lib.rstcreate mode 100644lib/librte_hash/k32v64_hash_avx512vl.ccreate mode 100644lib/librte_hash/rte_k32v64_hash.ccreate mode 100644lib/librte_hash/rte_k32v64_hash.h
Re: [dpdk-dev] [PATCH v3 1/4] hash: add k32v64 hash library
In reply to Vladimir MedvedkinQuoted 622 lines · Vladimir Medvedkin
K32V64 hash is a hash table that supports 32 bit keys and 64 bit values. This table is hash function agnostic so user must provide precalculated hash signature for add/delete/lookup operations.
Signed-off-by Vladimir Medvedkin <[email protected]>--- lib/Makefile | 2 +- lib/librte_hash/Makefile | 13 +- lib/librte_hash/k32v64_hash_avx512vl.c | 56 ++++++ lib/librte_hash/meson.build | 17 +- lib/librte_hash/rte_hash_version.map | 6 +- lib/librte_hash/rte_k32v64_hash.c | 315 +++++++++++++++++++++++++++++++++ lib/librte_hash/rte_k32v64_hash.h | 211 ++++++++++++++++++++++ 7 files changed, 615 insertions(+), 5 deletions(-)create mode 100644
lib/librte_hash/k32v64_hash_avx512vl.ccreate mode 100644lib/librte_hash/rte_k32v64_hash.ccreate mode 100644lib/librte_hash/rte_k32v64_hash.hlib/Makefile +1
-1@@ -48,7 +48,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_VHOST) += librte_vhost DEPDIRS-librte_vhost := librte_eal librte_mempool librte_mbuf librte_ethdev \ librte_net librte_hash librte_cryptodev DIRS-$(CONFIG_RTE_LIBRTE_HASH) += librte_hash -DEPDIRS-librte_hash := librte_eal librte_ring +DEPDIRS-librte_hash := librte_eal librte_ring librte_mempool DIRS-$(CONFIG_RTE_LIBRTE_EFD) += librte_efd DEPDIRS-librte_efd := librte_eal librte_ring librte_hash DIRS-$(CONFIG_RTE_LIBRTE_RIB) += librte_riblib/librte_hash/Makefile +12
-1@@ -8,13 +8,14 @@ LIB = librte_hash.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_ring +LDLIBS += -lrte_eal -lrte_ring -lrte_mempool EXPORT_MAP := rte_hash_version.map # all source are stored in SRCS-y SRCS-$(CONFIG_RTE_LIBRTE_HASH) := rte_cuckoo_hash.c SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_fbk_hash.c +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_k32v64_hash.c # install this header file SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include := rte_hash.h@@ -27,5 +28,15 @@ endif SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_jhash.h SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_thash.h SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_fbk_hash.h +SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_k32v64_hash.h + +CC_AVX512VL_SUPPORT=$(shell $(CC) -mavx512vl -dM -E - </dev/null 2>&1 | \ +grep -q __AVX512VL__ && echo 1) + +ifeq ($(CC_AVX512VL_SUPPORT), 1) + SRCS-$(CONFIG_RTE_LIBRTE_HASH) += k32v64_hash_avx512vl.c + CFLAGS_k32v64_hash_avx512vl.o += -mavx512vl + CFLAGS_rte_k32v64_hash.o += -DCC_AVX512VL_SUPPORT +endif include $(RTE_SDK)/mk/rte.lib.mklib/librte_hash/k32v64_hash_avx512vl.c new file +56
-0@@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include <rte_k32v64_hash.h> + +int +k32v64_hash_bulk_lookup_avx512vl (struct rte_k32v64_hash_table *table, +uint32_t *keys,uint32_t *hashes,uint64_t *values,unsigned int n); + +static inline int +k32v64_cmp_keys_avx512vl (struct rte_k32v64_hash_bucket *bucket,uint32_t key, +uint64_t *val) +{ +__m128i keys, srch_key; +__mmask8 msk; + + keys =_mm_load_si128 ((void *)bucket); + srch_key =_mm_set1_epi32 (key); + + msk =_mm_mask_cmpeq_epi32_mask (bucket->key_mask, keys, srch_key); +if (msk) { + *val = bucket->val[__builtin_ctz (msk)]; +return 1 ; + } + +return 0 ; +} + +static inline int +k32v64_hash_lookup_avx512vl (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *value) +{ +return __k32v64_hash_lookup (table, key, hash, value, + k32v64_cmp_keys_avx512vl); +} + +int +k32v64_hash_bulk_lookup_avx512vl (struct rte_k32v64_hash_table *table, +uint32_t *keys,uint32_t *hashes,uint64_t *values,unsigned int n) +{ +int ret, cnt =0 ; +unsigned int i; + +if (unlikely ((table ==NULL ) || (keys ==NULL ) || (hashes ==NULL ) || + (values ==NULL ))) +return -EINVAL ; + +for (i =0 ; i < n; i++) { + ret =k32v64_hash_lookup_avx512vl (table, keys[i], hashes[i], + &values[i]); +if (ret ==0 ) + cnt++; + } +return cnt; +}lib/librte_hash/meson.build +15
-2@@ -3,10 +3,23 @@ headers = files('rte_crc_arm64.h', 'rte_fbk_hash.h', + 'rte_k32v64_hash.h', 'rte_hash_crc.h', 'rte_hash.h', 'rte_jhash.h', 'rte_thash.h') -sources = files('rte_cuckoo_hash.c', 'rte_fbk_hash.c') -deps += ['ring'] +sources = files('rte_cuckoo_hash.c', 'rte_fbk_hash.c', 'rte_k32v64_hash.c') +deps += ['ring', 'mempool'] + +if dpdk_conf.has('RTE_ARCH_X86') + if cc.has_argument('-mavx512vl') + avx512_tmplib = static_library('avx512_tmp', + 'k32v64_hash_avx512vl.c', + dependencies: static_rte_mempool, + c_args: cflags + ['-mavx512vl']) + objs += avx512_tmplib.extract_objects('k32v64_hash_avx512vl.c') + cflags += '-DCC_AVX512VL_SUPPORT' + + endif +endiflib/librte_hash/rte_hash_version.map +5
-1@@ -34,5 +34,9 @@ EXPERIMENTAL { rte_hash_free_key_with_position; rte_hash_max_key_id; - + rte_k32v64_hash_create; + rte_k32v64_hash_find_existing; + rte_k32v64_hash_free; + rte_k32v64_hash_add; + rte_k32v64_hash_delete; };lib/librte_hash/rte_k32v64_hash.c new file +315
-0@@ -0,0 +1,315 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include <string.h> + +#include <rte_eal_memconfig.h> +#include <rte_errno.h> +#include <rte_malloc.h> +#include <rte_memory.h> +#include <rte_tailq.h> + +#include <rte_k32v64_hash.h> + +TAILQ_HEAD (rte_k32v64_hash_list, rte_tailq_entry); + +static struct rte_tailq_elem rte_k32v64_hash_tailq = { + .name ="RTE_K32V64_HASH" , +}; + +EAL_REGISTER_TAILQ (rte_k32v64_hash_tailq); + +#define VALID_KEY_MSK ((1 << RTE_K32V64_KEYS_PER_BUCKET) - 1) + +#ifdef CC_AVX512VL_SUPPORT +int +k32v64_hash_bulk_lookup_avx512vl (struct rte_k32v64_hash_table *table, +uint32_t *keys,uint32_t *hashes,uint64_t *values,unsigned int n); +#endif + +static int +k32v64_hash_bulk_lookup (struct rte_k32v64_hash_table *table,uint32_t *keys, +uint32_t *hashes,uint64_t *values,unsigned int n) +{ +int ret, cnt =0 ; +unsigned int i; + +if (unlikely ((table ==NULL ) || (keys ==NULL ) || (hashes ==NULL ) || + (values ==NULL ))) +return -EINVAL ; + +for (i =0 ; i < n; i++) { + ret =rte_k32v64_hash_lookup (table, keys[i], hashes[i], + &values[i]); +if (ret ==0 ) + cnt++; + } +return cnt; +} + +static rte_k32v64_hash_bulk_lookup_t +get_lookup_bulk_fn (void ) +{ +#ifdef CC_AVX512VL_SUPPORT +if (rte_cpu_get_flag_enabled (RTE_CPUFLAG_AVX512F )) +return k32v64_hash_bulk_lookup_avx512vl; +#endif +return k32v64_hash_bulk_lookup; +} + +int +rte_k32v64_hash_add (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t value) +{ +uint32_t bucket; +int i, idx, ret; +uint8_t msk; +struct rte_k32v64_ext_ent *tmp, *ent, *prev =NULL ; + +if (table ==NULL ) +return -EINVAL ; + + bucket = hash & table->bucket_msk; +/* Search key in table. Update value if exists */ +for (i =0 ; i <RTE_K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && + (table->t[bucket].key_mask & (1 << i))) { + table->t[bucket].val[i] = value; +return 0 ; + } + } + +if (!SLIST_EMPTY (&table->t[bucket].head)) { + SLIST_FOREACH(ent, &table->t[bucket].head, next) { +if (ent->key == key) { + ent->val = value; +return 0 ; + } + } + } + + msk = ~table->t[bucket].key_mask &VALID_KEY_MSK ; +if (msk) { + idx =__builtin_ctz (msk); + table->t[bucket].key[idx] = key; + table->t[bucket].val[idx] = value; +rte_smp_wmb (); + table->t[bucket].key_mask |=1 << idx; + table->nb_ent++; +return 0 ; + } + + ret =rte_mempool_get (table->ext_ent_pool, (void **)&ent); +if (ret <0 ) +return ret; + +SLIST_NEXT (ent, next) =NULL ; + ent->key = key; + ent->val = value; +rte_smp_wmb (); +SLIST_FOREACH (tmp, &table->t[bucket].head, next) + prev = tmp; + +if (prev ==NULL ) +SLIST_INSERT_HEAD (&table->t[bucket].head, ent, next); +else +SLIST_INSERT_AFTER (prev, ent, next); + + table->nb_ent++; + table->nb_ext_ent++; +return 0 ; +} + +int +rte_k32v64_hash_delete (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash) +{ +uint32_t bucket; +int i; +struct rte_k32v64_ext_ent *ent; + +if (table ==NULL ) +return -EINVAL ; + + bucket = hash & table->bucket_msk; + +for (i =0 ; i <RTE_K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && + (table->t[bucket].key_mask & (1 << i))) { + ent =SLIST_FIRST (&table->t[bucket].head); +if (ent) { +rte_atomic32_inc (&table->t[bucket].cnt); + table->t[bucket].key[i] = ent->key; + table->t[bucket].val[i] = ent->val; +SLIST_REMOVE_HEAD (&table->t[bucket].head, next); +rte_atomic32_inc (&table->t[bucket].cnt); + table->nb_ext_ent--; + }else + table->t[bucket].key_mask &= ~(1 << i); +if (ent) +rte_mempool_put (table->ext_ent_pool, ent); + table->nb_ent--; +return 0 ; + } + } + + SLIST_FOREACH(ent, &table->t[bucket].head, next) +if (ent->key == key) +break ; + +if (ent ==NULL ) +return -ENOENT ; + +rte_atomic32_inc (&table->t[bucket].cnt); +SLIST_REMOVE (&table->t[bucket].head, ent, rte_k32v64_ext_ent, next); +rte_atomic32_inc (&table->t[bucket].cnt); +rte_mempool_put (table->ext_ent_pool, ent); + + table->nb_ext_ent--; + table->nb_ent--; + +return 0 ; +} + +struct rte_k32v64_hash_table * +rte_k32v64_hash_find_existing (const char *name) +{ +struct rte_k32v64_hash_table *h =NULL ; +struct rte_tailq_entry *te; +struct rte_k32v64_hash_list *k32v64_hash_list; + + k32v64_hash_list =RTE_TAILQ_CAST (rte_k32v64_hash_tailq.head, + rte_k32v64_hash_list); + +rte_mcfg_tailq_read_lock (); + TAILQ_FOREACH(te, k32v64_hash_list, next) { + h = (struct rte_k32v64_hash_table *) te->data; +if (strncmp (name, h->name,RTE_K32V64_HASH_NAMESIZE ) ==0 ) +break ; + } +rte_mcfg_tailq_read_unlock (); +if (te ==NULL ) { + rte_errno =ENOENT ; +return NULL ; + } +return h; +} + +struct rte_k32v64_hash_table * +rte_k32v64_hash_create (const struct rte_k32v64_hash_params *params) +{ +char hash_name[RTE_K32V64_HASH_NAMESIZE ]; +struct rte_k32v64_hash_table *ht =NULL ; +struct rte_tailq_entry *te; +struct rte_k32v64_hash_list *k32v64_hash_list; +uint32_t mem_size, nb_buckets, max_ent; +int ret; +struct rte_mempool *mp; + +if ((params ==NULL ) || (params->name ==NULL ) || + (params->entries ==0 )) { + rte_errno =EINVAL ; +return NULL ; + } + + k32v64_hash_list =RTE_TAILQ_CAST (rte_k32v64_hash_tailq.head, + rte_k32v64_hash_list); + + ret =snprintf (hash_name,sizeof (hash_name),"K32V64_%s" , params->name); +if (ret <0 || ret >=RTE_K32V64_HASH_NAMESIZE ) { + rte_errno =ENAMETOOLONG ; +return NULL ; + } + + max_ent =rte_align32pow2 (params->entries); + nb_buckets = max_ent /RTE_K32V64_KEYS_PER_BUCKET ; + mem_size =sizeof (struct rte_k32v64_hash_table ) + +sizeof (struct rte_k32v64_hash_bucket ) * nb_buckets; + + mp =rte_mempool_create (hash_name, max_ent, +sizeof (struct rte_k32v64_ext_ent ),0 ,0 ,NULL ,NULL ,NULL ,NULL , + params->socket_id,0 ); + +if (mp ==NULL ) +return NULL ; + +rte_mcfg_tailq_write_lock (); + TAILQ_FOREACH(te, k32v64_hash_list, next) { + ht = (struct rte_k32v64_hash_table *) te->data; +if (strncmp (params->name, ht->name, +RTE_K32V64_HASH_NAMESIZE ) ==0 ) +break ; + } + ht =NULL ; +if (te !=NULL ) { + rte_errno =EEXIST ; +rte_mempool_free (mp); + goto exit; + } + + te =rte_zmalloc ("K32V64_HASH_TAILQ_ENTRY" ,sizeof (*te),0 ); +if (te ==NULL ) { +RTE_LOG (ERR ,HASH ,"Failed to allocate tailq entry\n" ); +rte_mempool_free (mp); + goto exit; + } + + ht =rte_zmalloc_socket (hash_name, mem_size, +RTE_CACHE_LINE_SIZE , params->socket_id); +if (ht ==NULL ) { +RTE_LOG (ERR ,HASH ,"Failed to allocate fbk hash table\n" ); +rte_free (te); +rte_mempool_free (mp); + goto exit; + } + +memcpy (ht->name, hash_name,sizeof (ht->name)); + ht->max_ent = max_ent; + ht->bucket_msk = nb_buckets -1 ; + ht->ext_ent_pool = mp; + ht->lookup =get_lookup_bulk_fn (); + + te->data = (void *)ht; +TAILQ_INSERT_TAIL (k32v64_hash_list, te, next); + +exit: +rte_mcfg_tailq_write_unlock (); + +return ht; +} + +void +rte_k32v64_hash_free (struct rte_k32v64_hash_table *ht) +{ +struct rte_tailq_entry *te; +struct rte_k32v64_hash_list *k32v64_hash_list; + +if (ht ==NULL ) +return ; + + k32v64_hash_list =RTE_TAILQ_CAST (rte_k32v64_hash_tailq.head, + rte_k32v64_hash_list); + +rte_mcfg_tailq_write_lock (); + +/* find out tailq entry */ + TAILQ_FOREACH(te, k32v64_hash_list, next) { +if (te->data == (void *) ht) +break ; + } + + +if (te ==NULL ) { +rte_mcfg_tailq_write_unlock (); +return ; + } + +TAILQ_REMOVE (k32v64_hash_list, te, next); + +rte_mcfg_tailq_write_unlock (); + +rte_mempool_free (ht->ext_ent_pool); +rte_free (ht); +rte_free (te); +}lib/librte_hash/rte_k32v64_hash.h new file +121
-0@@ -0,0 +1,211 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#ifndef _RTE_K32V64_HASH_H_ +#define _RTE_K32V64_HASH_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <rte_compat.h> +#include <rte_atomic.h> +#include <rte_mempool.h> + +#define RTE_K32V64_HASH_NAMESIZE 32 +#define RTE_K32V64_KEYS_PER_BUCKET 4 +#define RTE_K32V64_WRITE_IN_PROGRESS 1 + +struct rte_k32v64_hash_params { + const char *name; + uint32_t entries; + int socket_id; +}; + +struct rte_k32v64_ext_ent { + SLIST_ENTRY(rte_k32v64_ext_ent) next; + uint32_t key; + uint64_t val; +}; + +struct rte_k32v64_hash_bucket { + uint32_t key[RTE_K32V64_KEYS_PER_BUCKET]; + uint64_t val[RTE_K32V64_KEYS_PER_BUCKET]; + uint8_t key_mask; + rte_atomic32_t cnt; + SLIST_HEAD(rte_k32v64_list_head, rte_k32v64_ext_ent) head; +} __rte_cache_aligned; + +struct rte_k32v64_hash_table; + +typedef int (*rte_k32v64_hash_bulk_lookup_t) +(struct rte_k32v64_hash_table *table, uint32_t *keys, uint32_t *hashes, + uint64_t *values, unsigned int n); + +struct rte_k32v64_hash_table { + char name[RTE_K32V64_HASH_NAMESIZE]; /**< Name of the hash. */ + uint32_t nb_ent; /**< Number of entities in the table*/ + uint32_t nb_ext_ent; /**< Number of extended entities */ + uint32_t max_ent; /**< Maximum number of entities */ + uint32_t bucket_msk; + struct rte_mempool *ext_ent_pool; + rte_k32v64_hash_bulk_lookup_t lookup; + __extension__ struct rte_k32v64_hash_bucket t[0]; +}; + +typedef int (*rte_k32v64_cmp_fn_t) +(struct rte_k32v64_hash_bucket *bucket, uint32_t key, uint64_t *val); + +static inline int +__k32v64_cmp_keys(struct rte_k32v64_hash_bucket *bucket, uint32_t key, + uint64_t *val) +{ + int i; + + for (i = 0; i < RTE_K32V64_KEYS_PER_BUCKET; i++) { + if ((key == bucket->key[i]) && + (bucket->key_mask & (1 << i))) { + *val = bucket->val[i]; + return 1; + } + } + + return 0; +} + +static inline int +__k32v64_hash_lookup(struct rte_k32v64_hash_table *table, uint32_t key, + uint32_t hash, uint64_t *value, rte_k32v64_cmp_fn_t cmp_f) +{ + uint64_t val = 0; + struct rte_k32v64_ext_ent *ent; + int32_t cnt; + int found = 0; + uint32_t bucket = hash & table->bucket_msk; + + do { + do + cnt = rte_atomic32_read(&table->t[bucket].cnt); + while (unlikely(cnt & RTE_K32V64_WRITE_IN_PROGRESS)); + + found = cmp_f(&table->t[bucket], key, &val); + if (unlikely((found == 0) && + (!SLIST_EMPTY(&table->t[bucket].head)))) { + SLIST_FOREACH(ent, &table->t[bucket].head, next) { + if (ent->key == key) { + val = ent->val; + found = 1; + break; + } + } + } + + } while (unlikely(cnt != rte_atomic32_read(&table->t[bucket].cnt))); + + if (found == 1) { + *value = val; + return 0; + } else + return -ENOENT; +} + +static inline int +rte_k32v64_hash_lookup(struct rte_k32v64_hash_table *table, uint32_t key, + uint32_t hash, uint64_t *value) +{ + return __k32v64_hash_lookup(table, key, hash, value, __k32v64_cmp_keys); +} + +static inline int +rte_k32v64_hash_bulk_lookup(struct rte_k32v64_hash_table *table,
lib/librte_hash/rte_k32v64_hash.h line 131
Quoted 10 lines
lib/librte_hash/rte_k32v64_hash.h new file +10
-0+ uint32_t *keys, uint32_t *hashes, uint64_t *values, unsigned int n) +{ + return table->lookup(table, keys, hashes, values, n); +} + +/** +* Add a key to an existing hash table with hash value. +* This operation is not multi-thread safe +* and should only be called from one thread. +*
Does this hash allow multiple readers, and at most one writer, for a particular instance? If that's the case, it should probably be mentioned somewhere.
Just specifying that a particular function is not MT safe doesn't say anything if it's safe to call it in parallel with other, non-MT safe functions. I assume you can't add and delete in parallel?
Quoted 80 lines
lib/librte_hash/rte_k32v64_hash.h new file +80
-0+* @param ht +* Hash table to add the key to. +* @param key +* Key to add to the hash table. +* @param value +* Value to associate with key. +* @param hash +* Hash value associated with key. +* @return +* 0 if ok, or negative value on error. +*/ +__rte_experimental +int +rte_k32v64_hash_add (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t value); + +/** +* Remove a key with a given hash value from an existing hash table. +* This operation is not multi-thread +* safe and should only be called from one thread. +* +* @param ht +* Hash table to remove the key from. +* @param key +* Key to remove from the hash table. +* @param hash +* hash value associated with key. +* @return +* 0 if ok, or negative value on error. +*/ +__rte_experimental +int +rte_k32v64_hash_delete (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash); + + +/** +* Performs a lookup for an existing hash table, and returns a pointer to +* the table if found. +* +* @param name +* Name of the hash table to find +* +* @return +* pointer to hash table structure or NULL on error with rte_errno +* set appropriately. +*/ +__rte_experimental +struct rte_k32v64_hash_table * +rte_k32v64_hash_find_existing (const char *name); + +/** +* Create a new hash table for use with four byte keys. +* +* @param params +* Parameters used in creation of hash table. +* +* @return +* Pointer to hash table structure that is used in future hash table +* operations, or NULL on error with rte_errno set appropriately. +*/ +__rte_experimental +struct rte_k32v64_hash_table * +rte_k32v64_hash_create (const struct rte_k32v64_hash_params *params); + +/** +* Free all memory used by a hash table. +* +* @param table +* Hash table to deallocate. +*/ +__rte_experimental +void +rte_k32v64_hash_free (struct rte_k32v64_hash_table *table); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_K32V64_HASH_H_ */
Re: [dpdk-dev] [PATCH v3 0/4] add new k32v64 hash table
In reply to Vladimir Medvedkin15/04/2020 20:17, Vladimir Medvedkin:
Quoted 14 lines
Currently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called K32V64 hash
supports 4 byte keys and 8 byte associated values,
which is enough to store a pointer.It would also be nice to get feedback on whether to leave the old FBK and new k32v64 implementations or deprecate the old one?
v3:
- added bulk lookup
- avx512 key comparizon is removed from .h
Quoted 6 lines
v2:
- renamed fromrte_dwktorte_k32v64as was suggested
- reworked lookup function, added inlined subroutines
- added avx512 key comparizon routine
- added documentation
- added statistic counters for total entries and extended entries(linked list)
Please use --in-reply-to so we can follow version changes in the same email thread.
Also I am changing the states in patchwork as superseded. Please remind updating status of old patches.
Re: [dpdk-dev] [PATCH v3 0/4] add new k32v64 hash table
In reply to Mattias RönnblomHi Mattias,
-----Original Message-----
From: Mattias Rönnblom <[email protected]>
Sent: Wednesday, April 15, 2020 7:52 PM
To: Medvedkin, Vladimir <[email protected]>; [email protected]
Cc: Ananyev, Konstantin <[email protected]>; Wang, Yipeng1 <[email protected]>; Gobriel, Sameh <[email protected]>; Richardson, Bruce <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v3 0/4] add new k32v64 hash table
Quoted 8 lines · Vladimir Medvedkin
Currently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called K32V64 hash supports 4 byte keys and 8 byte associated values, which is enough to store a pointer.It would also be nice to get feedback on whether to leave the old FBK and new k32v64 implementations or deprecate the old one?
Do you think it would be feasible to support custom-sized values and remain efficient, in a similar manner to how rte_ring_elem.h does things?
I'm afraid it is not feasible. For the performance reason keys and corresponding values resides in single cache line so there are no extra memory for bigger values, such as 16B.
Quoted 39 lines
v3:
- added bulk lookup
- avx512 key comparizon is removed from .hv2:
- renamed fromrte_dwktorte_k32v64as was suggested
- reworked lookup function, added inlined subroutines
- added avx512 key comparizon routine
- added documentation
- added statistic counters for total entries and extended entries(linked list)Vladimir Medvedkin (4):
hash: add k32v64 hash library
hash: add documentation for k32v64 hash library
test: add k32v64 hash autotests
test: add k32v64 perf testsapp/test/Makefile | 1 + app/test/autotest_data.py | 12 ++ app/test/meson.build | 3 + app/test/test_hash_perf.c | 130 ++++++++++++ app/test/test_k32v64_hash.c | 229 ++++++++++++++++++++++ doc/api/doxy-api-index.md | 1 + doc/guides/prog_guide/index.rst | 1 + doc/guides/prog_guide/k32v64_hash_lib.rst | 66 +++++++ lib/Makefile | 2 +- lib/librte_hash/Makefile | 13 +- lib/librte_hash/k32v64_hash_avx512vl.c | 56 ++++++ lib/librte_hash/meson.build | 17 +- lib/librte_hash/rte_hash_version.map | 6 +- lib/librte_hash/rte_k32v64_hash.c | 315 ++++++++++++++++++++++++++++++ lib/librte_hash/rte_k32v64_hash.h | 211 ++++++++++++++++++++ 15 files changed, 1058 insertions(+), 5 deletions(-)create mode 100644
app/test/test_k32v64_hash.ccreate mode 100644doc/guides/prog_guide/k32v64_hash_lib.rstcreate mode 100644lib/librte_hash/k32v64_hash_avx512vl.ccreate mode 100644lib/librte_hash/rte_k32v64_hash.ccreate mode 100644lib/librte_hash/rte_k32v64_hash.h
Re: [dpdk-dev] [PATCH v3 1/4] hash: add k32v64 hash library
In reply to Mattias RönnblomHi Mattias,
-----Original Message-----
From: Mattias Rönnblom <[email protected]>
Sent: Wednesday, April 15, 2020 8:00 PM
To: Medvedkin, Vladimir <[email protected]>; [email protected]
Cc: Ananyev, Konstantin <[email protected]>; Wang, Yipeng1 <[email protected]>; Gobriel, Sameh <[email protected]>; Richardson, Bruce <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v3 1/4] hash: add k32v64 hash library
Quoted 614 lines · Vladimir Medvedkin
K32V64 hash is a hash table that supports 32 bit keys and 64 bit values. This table is hash function agnostic so user must provide precalculated hash signature for add/delete/lookup operations.
Signed-off-by Vladimir Medvedkin <[email protected]>--- lib/Makefile | 2 +- lib/librte_hash/Makefile | 13 +- lib/librte_hash/k32v64_hash_avx512vl.c | 56 ++++++ lib/librte_hash/meson.build | 17 +- lib/librte_hash/rte_hash_version.map | 6 +- lib/librte_hash/rte_k32v64_hash.c | 315 +++++++++++++++++++++++++++++++++ lib/librte_hash/rte_k32v64_hash.h | 211 ++++++++++++++++++++++ 7 files changed, 615 insertions(+), 5 deletions(-)create mode 100644
lib/librte_hash/k32v64_hash_avx512vl.ccreate mode 100644lib/librte_hash/rte_k32v64_hash.ccreate mode 100644lib/librte_hash/rte_k32v64_hash.hlib/Makefile +0
-0@@ -48,7 +48,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_VHOST) += librte_vhost DEPDIRS-librte_vhost := librte_eal librte_mempool librte_mbuf librte_ethdev \ librte_net librte_hash librte_cryptodev DIRS-$(CONFIG_RTE_LIBRTE_HASH) += librte_hash -DEPDIRS-librte_hash:=
librte_eallibrte_ring
+DEPDIRS-librte_hash :=librte_eallibrte_ringlibrte_mempool
DIRS-$(CONFIG_RTE_LIBRTE_EFD) +=librte_efdDEPDIRS-librte_efd :=librte_eallibrte_ringlibrte_hashDIRS-$(CONFIG_RTE_LIBRTE_RIB) +=librte_ribdiff --git
a/lib/librte_hash/Makefile b/lib/librte_hash/Makefile index ec9f864..023689d 100644lib/librte_hash/Makefile +12
-1@@ -8,13 +8,14 @@ LIB = librte_hash.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_ring +LDLIBS += -lrte_eal -lrte_ring -lrte_mempool EXPORT_MAP := rte_hash_version.map # all source are stored in SRCS-y SRCS-$(CONFIG_RTE_LIBRTE_HASH) := rte_cuckoo_hash.c SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_fbk_hash.c +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_k32v64_hash.c # install this header file SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include := rte_hash.h @@ -27,5 +28,15 @@ endif SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_jhash.h SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_thash.h SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_fbk_hash.h +SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_k32v64_hash.h + +CC_AVX512VL_SUPPORT=$(shell $(CC) -mavx512vl -dM -E - </dev/null 2>&1 +| \ grep -q __AVX512VL__ && echo 1) + +ifeq ($(CC_AVX512VL_SUPPORT), 1) + SRCS-$(CONFIG_RTE_LIBRTE_HASH) += k32v64_hash_avx512vl.c + CFLAGS_k32v64_hash_avx512vl.o += -mavx512vl + CFLAGS_rte_k32v64_hash.o += -DCC_AVX512VL_SUPPORT endif include $(RTE_SDK)/mk/rte.lib.mklib/librte_hash/k32v64_hash_avx512vl.c new file +55
-0@@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation */ + +#include <rte_k32v64_hash.h> + +int +k32v64_hash_bulk_lookup_avx512vl (struct rte_k32v64_hash_table *table, +uint32_t *keys,uint32_t *hashes,uint64_t *values,unsigned int n); + +static inline int +k32v64_cmp_keys_avx512vl (struct rte_k32v64_hash_bucket *bucket,uint32_t key, +uint64_t *val) +{ +__m128i keys, srch_key; +__mmask8 msk; + + keys =_mm_load_si128 ((void *)bucket); + srch_key =_mm_set1_epi32 (key); + + msk =_mm_mask_cmpeq_epi32_mask (bucket->key_mask, keys, srch_key); +if (msk) { + *val = bucket->val[__builtin_ctz (msk)]; +return 1 ; + } + +return 0 ; +} + +static inline int +k32v64_hash_lookup_avx512vl (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *value) +{ +return __k32v64_hash_lookup (table, key, hash, value, + k32v64_cmp_keys_avx512vl); +} + +int +k32v64_hash_bulk_lookup_avx512vl (struct rte_k32v64_hash_table *table, +uint32_t *keys,uint32_t *hashes,uint64_t *values,unsigned int n) +{ +int ret, cnt =0 ; +unsigned int i; + +if (unlikely ((table ==NULL ) || (keys ==NULL ) || (hashes ==NULL ) || + (values ==NULL ))) +return -EINVAL ; + +for (i =0 ; i < n; i++) { + ret =k32v64_hash_lookup_avx512vl (table, keys[i], hashes[i], + &values[i]); +if (ret ==0 ) + cnt++; + } +return cnt; +}lib/librte_hash/meson.build +1
-1@@ -3,10 +3,23 @@ headers = files('rte_crc_arm64.h', 'rte_fbk_hash.h', + 'rte_k32v64_hash.h', 'rte_hash_crc.h', 'rte_hash.h', 'rte_jhash.h', 'rte_thash.h') -sources = files('rte_cuckoo_hash.c', 'rte_fbk_hash.c') -deps +=['ring']
lib/librte_hash/meson.build +14
-0+sources = files('rte_cuckoo_hash.c', 'rte_fbk_hash.c', +'rte_k32v64_hash.c') deps += ['ring', 'mempool'] + +if dpdk_conf.has('RTE_ARCH_X86') + if cc.has_argument('-mavx512vl') + avx512_tmplib = static_library('avx512_tmp', + 'k32v64_hash_avx512vl.c', + dependencies: static_rte_mempool, + c_args: cflags + ['-mavx512vl']) + objs += avx512_tmplib.extract_objects('k32v64_hash_avx512vl.c') + cflags += '-DCC_AVX512VL_SUPPORT' + + endif +endiflib/librte_hash/rte_hash_version.map +5
-1@@ -34,5 +34,9 @@ EXPERIMENTAL { rte_hash_free_key_with_position; rte_hash_max_key_id; - + rte_k32v64_hash_create; + rte_k32v64_hash_find_existing; + rte_k32v64_hash_free; + rte_k32v64_hash_add; + rte_k32v64_hash_delete; };lib/librte_hash/rte_k32v64_hash.c new file +309
-0@@ -0,0 +1,315 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation */ + +#include <string.h> + +#include <rte_eal_memconfig.h> +#include <rte_errno.h> +#include <rte_malloc.h> +#include <rte_memory.h> +#include <rte_tailq.h> + +#include <rte_k32v64_hash.h> + +TAILQ_HEAD (rte_k32v64_hash_list, rte_tailq_entry); + +static struct rte_tailq_elem rte_k32v64_hash_tailq = { + .name ="RTE_K32V64_HASH" , +}; + +EAL_REGISTER_TAILQ (rte_k32v64_hash_tailq); + +#define VALID_KEY_MSK ((1 << RTE_K32V64_KEYS_PER_BUCKET) - 1) + +#ifdef CC_AVX512VL_SUPPORT +int +k32v64_hash_bulk_lookup_avx512vl (struct rte_k32v64_hash_table *table, +uint32_t *keys,uint32_t *hashes,uint64_t *values,unsigned int n); +#endif + +static int +k32v64_hash_bulk_lookup (struct rte_k32v64_hash_table *table,uint32_t *keys, +uint32_t *hashes,uint64_t *values,unsigned int n) { +int ret, cnt =0 ; +unsigned int i; + +if (unlikely ((table ==NULL ) || (keys ==NULL ) || (hashes ==NULL ) || + (values ==NULL ))) +return -EINVAL ; + +for (i =0 ; i < n; i++) { + ret =rte_k32v64_hash_lookup (table, keys[i], hashes[i], + &values[i]); +if (ret ==0 ) + cnt++; + } +return cnt; +} + +static rte_k32v64_hash_bulk_lookup_t +get_lookup_bulk_fn (void ) +{ +#ifdef CC_AVX512VL_SUPPORT +if (rte_cpu_get_flag_enabled (RTE_CPUFLAG_AVX512F )) +return k32v64_hash_bulk_lookup_avx512vl;#endif +return k32v64_hash_bulk_lookup; +} + +int +rte_k32v64_hash_add (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t value) +{ +uint32_t bucket; +int i, idx, ret; +uint8_t msk; +struct rte_k32v64_ext_ent *tmp, *ent, *prev =NULL ; + +if (table ==NULL ) +return -EINVAL ; + + bucket = hash & table->bucket_msk; +/* Search key in table. Update value if exists */ +for (i =0 ; i <RTE_K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && + (table->t[bucket].key_mask & (1 << i))) { + table->t[bucket].val[i] = value; +return 0 ; + } + } + +if (!SLIST_EMPTY (&table->t[bucket].head)) { + SLIST_FOREACH(ent, &table->t[bucket].head, next) { +if (ent->key == key) { + ent->val = value; +return 0 ; + } + } + } + + msk = ~table->t[bucket].key_mask &VALID_KEY_MSK ; +if (msk) { + idx =__builtin_ctz (msk); + table->t[bucket].key[idx] = key; + table->t[bucket].val[idx] = value; +rte_smp_wmb (); + table->t[bucket].key_mask |=1 << idx; + table->nb_ent++; +return 0 ; + } + + ret =rte_mempool_get (table->ext_ent_pool, (void **)&ent); +if (ret <0 ) +return ret; + +SLIST_NEXT (ent, next) =NULL ; + ent->key = key; + ent->val = value; +rte_smp_wmb (); +SLIST_FOREACH (tmp, &table->t[bucket].head, next) + prev = tmp; + +if (prev ==NULL ) +SLIST_INSERT_HEAD (&table->t[bucket].head, ent, next); +else +SLIST_INSERT_AFTER (prev, ent, next); + + table->nb_ent++; + table->nb_ext_ent++; +return 0 ; +} + +int +rte_k32v64_hash_delete (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash) +{ +uint32_t bucket; +int i; +struct rte_k32v64_ext_ent *ent; + +if (table ==NULL ) +return -EINVAL ; + + bucket = hash & table->bucket_msk; + +for (i =0 ; i <RTE_K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && + (table->t[bucket].key_mask & (1 << i))) { + ent =SLIST_FIRST (&table->t[bucket].head); +if (ent) { +rte_atomic32_inc (&table->t[bucket].cnt); + table->t[bucket].key[i] = ent->key; + table->t[bucket].val[i] = ent->val; +SLIST_REMOVE_HEAD (&table->t[bucket].head, next); +rte_atomic32_inc (&table->t[bucket].cnt); + table->nb_ext_ent--; + }else + table->t[bucket].key_mask &= ~(1 << i); +if (ent) +rte_mempool_put (table->ext_ent_pool, ent); + table->nb_ent--; +return 0 ; + } + } + + SLIST_FOREACH(ent, &table->t[bucket].head, next) +if (ent->key == key) +break ; + +if (ent ==NULL ) +return -ENOENT ; + +rte_atomic32_inc (&table->t[bucket].cnt); +SLIST_REMOVE (&table->t[bucket].head, ent, rte_k32v64_ext_ent, next); +rte_atomic32_inc (&table->t[bucket].cnt); +rte_mempool_put (table->ext_ent_pool, ent); + + table->nb_ext_ent--; + table->nb_ent--; + +return 0 ; +} + +struct rte_k32v64_hash_table * +rte_k32v64_hash_find_existing (const char *name) { +struct rte_k32v64_hash_table *h =NULL ; +struct rte_tailq_entry *te; +struct rte_k32v64_hash_list *k32v64_hash_list; + + k32v64_hash_list =RTE_TAILQ_CAST (rte_k32v64_hash_tailq.head, + rte_k32v64_hash_list); + +rte_mcfg_tailq_read_lock (); + TAILQ_FOREACH(te, k32v64_hash_list, next) { + h = (struct rte_k32v64_hash_table *) te->data; +if (strncmp (name, h->name,RTE_K32V64_HASH_NAMESIZE ) ==0 ) +break ; + } +rte_mcfg_tailq_read_unlock (); +if (te ==NULL ) { + rte_errno =ENOENT ; +return NULL ; + } +return h; +} + +struct rte_k32v64_hash_table * +rte_k32v64_hash_create (const struct rte_k32v64_hash_params *params) { +char hash_name[RTE_K32V64_HASH_NAMESIZE ]; +struct rte_k32v64_hash_table *ht =NULL ; +struct rte_tailq_entry *te; +struct rte_k32v64_hash_list *k32v64_hash_list; +uint32_t mem_size, nb_buckets, max_ent; +int ret; +struct rte_mempool *mp; + +if ((params ==NULL ) || (params->name ==NULL ) || + (params->entries ==0 )) { + rte_errno =EINVAL ; +return NULL ; + } + + k32v64_hash_list =RTE_TAILQ_CAST (rte_k32v64_hash_tailq.head, + rte_k32v64_hash_list); + + ret =snprintf (hash_name,sizeof (hash_name),"K32V64_%s" , params->name); +if (ret <0 || ret >=RTE_K32V64_HASH_NAMESIZE ) { + rte_errno =ENAMETOOLONG ; +return NULL ; + } + + max_ent =rte_align32pow2 (params->entries); + nb_buckets = max_ent /RTE_K32V64_KEYS_PER_BUCKET ; + mem_size =sizeof (struct rte_k32v64_hash_table ) + +sizeof (struct rte_k32v64_hash_bucket ) * nb_buckets; + + mp =rte_mempool_create (hash_name, max_ent, +sizeof (struct rte_k32v64_ext_ent ),0 ,0 ,NULL ,NULL ,NULL ,NULL , + params->socket_id,0 ); + +if (mp ==NULL ) +return NULL ; + +rte_mcfg_tailq_write_lock (); + TAILQ_FOREACH(te, k32v64_hash_list, next) { + ht = (struct rte_k32v64_hash_table *) te->data; +if (strncmp (params->name, ht->name, +RTE_K32V64_HASH_NAMESIZE ) ==0 ) +break ; + } + ht =NULL ; +if (te !=NULL ) { + rte_errno =EEXIST ; +rte_mempool_free (mp); + goto exit; + } + + te =rte_zmalloc ("K32V64_HASH_TAILQ_ENTRY" ,sizeof (*te),0 ); +if (te ==NULL ) { +RTE_LOG (ERR ,HASH ,"Failed to allocate tailq entry\n" ); +rte_mempool_free (mp); + goto exit; + } + + ht =rte_zmalloc_socket (hash_name, mem_size, +RTE_CACHE_LINE_SIZE , params->socket_id); +if (ht ==NULL ) { +RTE_LOG (ERR ,HASH ,"Failed to allocate fbk hash table\n" ); +rte_free (te); +rte_mempool_free (mp); + goto exit; + } + +memcpy (ht->name, hash_name,sizeof (ht->name)); + ht->max_ent = max_ent; + ht->bucket_msk = nb_buckets -1 ; + ht->ext_ent_pool = mp; + ht->lookup =get_lookup_bulk_fn (); + + te->data = (void *)ht; +TAILQ_INSERT_TAIL (k32v64_hash_list, te, next); + +exit: +rte_mcfg_tailq_write_unlock (); + +return ht; +} + +void +rte_k32v64_hash_free (struct rte_k32v64_hash_table *ht) { +struct rte_tailq_entry *te; +struct rte_k32v64_hash_list *k32v64_hash_list; + +if (ht ==NULL ) +return ; + + k32v64_hash_list =RTE_TAILQ_CAST (rte_k32v64_hash_tailq.head, + rte_k32v64_hash_list); + +rte_mcfg_tailq_write_lock (); + +/* find out tailq entry */ + TAILQ_FOREACH(te, k32v64_hash_list, next) { +if (te->data == (void *) ht) +break ; + } + + +if (te ==NULL ) { +rte_mcfg_tailq_write_unlock (); +return ; + } + +TAILQ_REMOVE (k32v64_hash_list, te, next); + +rte_mcfg_tailq_write_unlock (); + +rte_mempool_free (ht->ext_ent_pool); +rte_free (ht); +rte_free (te); +}lib/librte_hash/rte_k32v64_hash.h new file +118
-0@@ -0,0 +1,211 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation */ + +#ifndef _RTE_K32V64_HASH_H_ +#define _RTE_K32V64_HASH_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <rte_compat.h> +#include <rte_atomic.h> +#include <rte_mempool.h> + +#define RTE_K32V64_HASH_NAMESIZE 32 +#define RTE_K32V64_KEYS_PER_BUCKET 4 +#define RTE_K32V64_WRITE_IN_PROGRESS 1 + +struct rte_k32v64_hash_params { + const char *name; + uint32_t entries; + int socket_id; +}; + +struct rte_k32v64_ext_ent { + SLIST_ENTRY(rte_k32v64_ext_ent) next; + uint32_t key; + uint64_t val; +}; + +struct rte_k32v64_hash_bucket { + uint32_t key[RTE_K32V64_KEYS_PER_BUCKET]; + uint64_t val[RTE_K32V64_KEYS_PER_BUCKET]; + uint8_t key_mask; + rte_atomic32_t cnt; + SLIST_HEAD(rte_k32v64_list_head, rte_k32v64_ext_ent) head; } +__rte_cache_aligned; + +struct rte_k32v64_hash_table; + +typedef int (*rte_k32v64_hash_bulk_lookup_t) (struct +rte_k32v64_hash_table *table, uint32_t *keys, uint32_t *hashes, + uint64_t *values, unsigned int n); + +struct rte_k32v64_hash_table { + char name[RTE_K32V64_HASH_NAMESIZE]; /**< Name of the hash. */ + uint32_t nb_ent; /**< Number of entities in the table*/ + uint32_t nb_ext_ent; /**< Number of extended entities */ + uint32_t max_ent; /**< Maximum number of entities */ + uint32_t bucket_msk; + struct rte_mempool *ext_ent_pool; + rte_k32v64_hash_bulk_lookup_t lookup; + __extension__ struct rte_k32v64_hash_bucket t[0]; +}; + +typedef int (*rte_k32v64_cmp_fn_t) +(struct rte_k32v64_hash_bucket *bucket, uint32_t key, uint64_t *val); + +static inline int +__k32v64_cmp_keys(struct rte_k32v64_hash_bucket *bucket, uint32_t key, + uint64_t *val) +{ + int i; + + for (i = 0; i < RTE_K32V64_KEYS_PER_BUCKET; i++) { + if ((key == bucket->key[i]) && + (bucket->key_mask & (1 << i))) { + *val = bucket->val[i]; + return 1; + } + } + + return 0; +} + +static inline int +__k32v64_hash_lookup(struct rte_k32v64_hash_table *table, uint32_t key, + uint32_t hash, uint64_t *value, rte_k32v64_cmp_fn_t cmp_f) { + uint64_t val = 0; + struct rte_k32v64_ext_ent *ent; + int32_t cnt; + int found = 0; + uint32_t bucket = hash & table->bucket_msk; + + do { + do + cnt = rte_atomic32_read(&table->t[bucket].cnt); + while (unlikely(cnt & RTE_K32V64_WRITE_IN_PROGRESS)); + + found = cmp_f(&table->t[bucket], key, &val); + if (unlikely((found == 0) && + (!SLIST_EMPTY(&table->t[bucket].head)))) { + SLIST_FOREACH(ent, &table->t[bucket].head, next) { + if (ent->key == key) { + val = ent->val; + found = 1; + break; + } + } + } + + } while (unlikely(cnt != rte_atomic32_read(&table->t[bucket].cnt))); + + if (found == 1) { + *value = val; + return 0; + } else + return -ENOENT; +} + +static inline int +rte_k32v64_hash_lookup(struct rte_k32v64_hash_table *table, uint32_t key, + uint32_t hash, uint64_t *value) +{ + return __k32v64_hash_lookup(table, key, hash, value, +__k32v64_cmp_keys); } + +static inline int
lib/librte_hash/rte_k32v64_hash.h line 128
Quoted 10 lines
lib/librte_hash/rte_k32v64_hash.h new file +10
-0+rte_k32v64_hash_bulk_lookup(struct rte_k32v64_hash_table *table, + uint32_t *keys, uint32_t *hashes, uint64_t *values, unsigned int n) +{ + return table->lookup(table, keys, hashes, values, n); } + +/** +* Add a key to an existing hash table with hash value. +* This operation is not multi-thread safe +* and should only be called from one thread. +*
Does this hash allow multiple readers, and at most one writer, for a particular instance? If that's the case, it should probably be mentioned somewhere.
Just specifying that a particular function is not MT safe doesn't say anything if it's safe to call it in parallel with other, non-MT safe functions. I assume you can't add and delete in parallel?
Yes, this hash allows multiple readers and on writer. Writers must be serialized. I will add this information. Thanks!
Quoted 81 lines
lib/librte_hash/rte_k32v64_hash.h new file +81
-0+* @param ht +* Hash table to add the key to. +* @param key +* Key to add to the hash table. +* @param value +* Value to associate with key. +* @param hash +* Hash value associated with key. +* @return +* 0 if ok, or negative value on error. +*/ +__rte_experimental +int +rte_k32v64_hash_add (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t value); + +/** +* Remove a key with a given hash value from an existing hash table. +* This operation is not multi-thread +* safe and should only be called from one thread. +* +* @param ht +* Hash table to remove the key from. +* @param key +* Key to remove from the hash table. +* @param hash +* hash value associated with key. +* @return +* 0 if ok, or negative value on error. +*/ +__rte_experimental +int +rte_k32v64_hash_delete (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash); + + +/** +* Performs a lookup for an existing hash table, and returns a +pointer to +* the table if found. +* +* @param name +* Name of the hash table to find +* +* @return +* pointer to hash table structure or NULL on error with rte_errno +* set appropriately. +*/ +__rte_experimental +struct rte_k32v64_hash_table * +rte_k32v64_hash_find_existing (const char *name); + +/** +* Create a new hash table for use with four byte keys. +* +* @param params +* Parameters used in creation of hash table. +* +* @return +* Pointer to hash table structure that is used in future hash table +* operations, or NULL on error with rte_errno set appropriately. +*/ +__rte_experimental +struct rte_k32v64_hash_table * +rte_k32v64_hash_create (const struct rte_k32v64_hash_params *params); + +/** +* Free all memory used by a hash table. +* +* @param table +* Hash table to deallocate. +*/ +__rte_experimental +void +rte_k32v64_hash_free (struct rte_k32v64_hash_table *table); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_K32V64_HASH_H_ */
Re: [dpdk-dev] [PATCH v3 0/4] add new k32v64 hash table
In reply to Medvedkin, VladimirQuoted 16 lines · Vladimir
Hi Mattias,
-----Original Message-----
From: Mattias Rönnblom <[email protected]>
Sent: Wednesday, April 15, 2020 7:52 PM
To: Medvedkin, Vladimir <[email protected]>; [email protected]
Cc: Ananyev, Konstantin <[email protected]>; Wang, Yipeng1 <[email protected]>; Gobriel, Sameh <[email protected]>; Richardson, Bruce <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v3 0/4] add new k32v64 hash tableQuoted 6 lines · Vladimir Medvedkin
Currently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called K32V64 hash supports 4 byte keys and 8 byte associated values, which is enough to store a pointer.
Quoted 6 lines
Quoted 2 lines
It would also be nice to get feedback on whether to leave the old FBK and new k32v64 implementations or deprecate the old one?
Do you think it would be feasible to support custom-sized values and remain efficient, in a similar manner to how
rte_ring_elem.hdoes things?I'm afraid it is not feasible. For the performance reason keys and corresponding values resides in single cache line so there are no extra memory for bigger values, such as 16B.
Well, if you have a smaller value type (or key type) you would fit into something less-than-a-cache line, and thus reduce your memory working set further.
Quoted 39 lines
Quoted 39 lines
v3:
- added bulk lookup
- avx512 key comparizon is removed from .hv2:
- renamed fromrte_dwktorte_k32v64as was suggested
- reworked lookup function, added inlined subroutines
- added avx512 key comparizon routine
- added documentation
- added statistic counters for total entries and extended entries(linked list)Vladimir Medvedkin (4):
hash: add k32v64 hash library
hash: add documentation for k32v64 hash library
test: add k32v64 hash autotests
test: add k32v64 perf testsapp/test/Makefile | 1 + app/test/autotest_data.py | 12 ++ app/test/meson.build | 3 + app/test/test_hash_perf.c | 130 ++++++++++++ app/test/test_k32v64_hash.c | 229 ++++++++++++++++++++++ doc/api/doxy-api-index.md | 1 + doc/guides/prog_guide/index.rst | 1 + doc/guides/prog_guide/k32v64_hash_lib.rst | 66 +++++++ lib/Makefile | 2 +- lib/librte_hash/Makefile | 13 +- lib/librte_hash/k32v64_hash_avx512vl.c | 56 ++++++ lib/librte_hash/meson.build | 17 +- lib/librte_hash/rte_hash_version.map | 6 +- lib/librte_hash/rte_k32v64_hash.c | 315 ++++++++++++++++++++++++++++++ lib/librte_hash/rte_k32v64_hash.h | 211 ++++++++++++++++++++ 15 files changed, 1058 insertions(+), 5 deletions(-)create mode 100644
app/test/test_k32v64_hash.ccreate mode 100644doc/guides/prog_guide/k32v64_hash_lib.rstcreate mode 100644lib/librte_hash/k32v64_hash_avx512vl.ccreate mode 100644lib/librte_hash/rte_k32v64_hash.ccreate mode 100644lib/librte_hash/rte_k32v64_hash.h
Re: [dpdk-dev] [PATCH v3 0/4] add new k32v64 hash table
In reply to Thomas MonjalonHi Thomas,
Quoted 21 lines · Thomas Monjalon
15/04/2020 20:17, Vladimir Medvedkin:
Quoted 14 lines
Currently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called K32V64 hash
supports 4 byte keys and 8 byte associated values,
which is enough to store a pointer.It would also be nice to get feedback on whether to leave the old FBK and new k32v64 implementations or deprecate the old one?
v3:
- added bulk lookup
- avx512 key comparizon is removed from .hQuoted 6 lines
v2:
- renamed fromrte_dwktorte_k32v64as was suggested
- reworked lookup function, added inlined subroutines
- added avx512 key comparizon routine
- added documentation
- added statistic counters for total entries and extended entries(linked list)
Quoted 5 lines
Please use --in-reply-to so we can follow version changes in the same email thread.
Also I am changing the states in patchwork as superseded. Please remind updating status of old patches.
Hmm, strange, I used --in-reply-to. Also in patchwork I can see
In-Reply-To: <[email protected]>
Signature · 2 lines
Regards, Vladimir
Re: [dpdk-dev] [PATCH v3 0/4] add new k32v64 hash table
16/04/2020 16:02, Medvedkin, Vladimir:
Quoted 30 lines
Hi Thomas,
Quoted 21 lines · Thomas Monjalon
15/04/2020 20:17, Vladimir Medvedkin:
Quoted 14 lines
Currently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called K32V64 hash
supports 4 byte keys and 8 byte associated values,
which is enough to store a pointer.It would also be nice to get feedback on whether to leave the old FBK and new k32v64 implementations or deprecate the old one?
v3:
- added bulk lookup
- avx512 key comparizon is removed from .hQuoted 6 lines
v2:
- renamed fromrte_dwktorte_k32v64as was suggested
- reworked lookup function, added inlined subroutines
- added avx512 key comparizon routine
- added documentation
- added statistic counters for total entries and extended entries(linked list)Quoted 5 lines
Please use --in-reply-to so we can follow version changes in the same email thread.
Also I am changing the states in patchwork as superseded. Please remind updating status of old patches.
Quoted 3 lines
Hmm, strange, I used --in-reply-to. Also in patchwork I can see
In-Reply-To: <[email protected]>
Indeed, sorry my bad.
I can see correct threading:
http://inbox.dpdk.org/dev/[email protected]/
Re: [dpdk-dev] [PATCH v3 0/4] add new k32v64 hash table
In reply to Mattias RönnblomQuoted 76 lines
-----Original Message-----
From: Mattias Rönnblom <[email protected]>
Sent: Thursday, April 16, 2020 4:41 AM
To: Medvedkin, Vladimir <[email protected]>; [email protected]
Cc: Ananyev, Konstantin <[email protected]>; Wang, Yipeng1 <[email protected]>; Gobriel, Sameh <[email protected]>; Richardson, Bruce <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v3 0/4] add new k32v64 hash tableQuoted 18 lines · Vladimir
Hi Mattias,
-----Original Message-----
From: Mattias Rönnblom <[email protected]>
Sent: Wednesday, April 15, 2020 7:52 PM
To: Medvedkin, Vladimir <[email protected]>; [email protected]
Cc: Ananyev, Konstantin <[email protected]>; Wang, Yipeng1 <[email protected]>; Gobriel, Sameh <[email protected]>; Richardson, Bruce <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v3 0/4] add new k32v64 hash tableQuoted 6 lines · Vladimir Medvedkin
Currently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called K32V64 hash supports 4 byte keys and 8 byte associated values, which is enough to store a pointer.Quoted 4 lines
Quoted 2 lines
It would also be nice to get feedback on whether to leave the old FBK and new k32v64 implementations or deprecate the old one?
Do you think it would be feasible to support custom-sized values and remain
efficient, in a similar manner to how
rte_ring_elem.hdoes things?Quoted 2 lines
I'm afraid it is not feasible. For the performance reason keys and
corresponding values resides in single cache line so there are no extra memory for bigger values, such as 16B.
Well, if you have a smaller value type (or key type) you would fit into something less-than-a-cache line, and thus reduce your memory working set further.
Quoted 18 lines
Quoted 18 lines
v3:
- added bulk lookup
- avx512 key comparizon is removed from .hv2:
- renamed fromrte_dwktorte_k32v64as was suggested
- reworked lookup function, added inlined subroutines
- added avx512 key comparizon routine
- added documentation
- added statistic counters for total entries and extended entries(linked list)Vladimir Medvedkin (4):
hash: add k32v64 hash library
hash: add documentation for k32v64 hash library
test: add k32v64 hash autotests
test: add k32v64 perf testsQuoted 14 lines
Quoted 6 lines
app/test/Makefile | 1 + app/test/autotest_data.py | 12 ++ app/test/meson.build | 3 + app/test/test_hash_perf.c | 130 ++++++++++++ app/test/test_k32v64_hash.c | 229 ++++++++++++++++++++++ doc/api/doxy-api-index.md | 1 +Quoted 8 lines
doc/guides/prog_guide/index.rst | 1 + doc/guides/prog_guide/k32v64_hash_lib.rst | 66 +++++++ lib/Makefile | 2 +- lib/librte_hash/Makefile | 13 +- lib/librte_hash/k32v64_hash_avx512vl.c | 56 ++++++ lib/librte_hash/meson.build | 17 +- lib/librte_hash/rte_hash_version.map | 6 +- lib/librte_hash/rte_k32v64_hash.c | 315
Quoted 9 lines
++++++++++++++++++++++++++++++
Quoted 8 lines
Quoted 8 lines
lib/librte_hash/rte_k32v64_hash.h | 211 ++++++++++++++++++++ 15 files changed, 1058 insertions(+), 5 deletions(-)create mode 100644
app/test/test_k32v64_hash.ccreate mode 100644doc/guides/prog_guide/k32v64_hash_lib.rstcreate mode 100644lib/librte_hash/k32v64_hash_avx512vl.ccreate mode 100644lib/librte_hash/rte_k32v64_hash.ccreate mode 100644lib/librte_hash/rte_k32v64_hash.h
[Wang, Yipeng]
Hi, Vladimir,
Thanks for responding with the use cases earlier. I discussed with Sameh offline, here are some comments.
1. Since the proposed hash table also has some similarities to rte_table library used by packet framework, have you tried it yet? Although it is mainly for packet framework, I believe you can use it independently as well. It has implementations for special key value sizes.
I added Cristian for his comment.
2. We tend to agree with Mattias that it would be better if we have a more generic API name and with the same API we can do multiple key/value size implementations.
This is to avoid adding new APIs in future to again handle different key/value
use cases. For example, we call it rte_kv_hash, and through the parameter struct we pass in a key-value size pair we want to use.
Implementation-wise, we may only provide implementations for certain popular use cases (like the one you provided). For other general use cases, people should go with the more flexible and generic cuckoo hash. Then we should also merge the FBK under the new API.
Re: [dpdk-dev] [PATCH v3 1/4] hash: add k32v64 hash library
In reply to Vladimir MedvedkinHi Vladimir,
Apologies for late review.
My comments below.
Quoted 72 lines
K32V64 hash is a hash table that supports 32 bit keys and 64 bit values. This table is hash function agnostic so user must provide precalculated hash signature for add/delete/lookup operations.
Signed-off-by Vladimir Medvedkin <[email protected]>---
lib/librte_hash/rte_k32v64_hash.c new file +62
-0@@ -0,0 +1,315 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include <string.h> + +#include <rte_eal_memconfig.h> +#include <rte_errno.h> +#include <rte_malloc.h> +#include <rte_memory.h> +#include <rte_tailq.h> + +#include <rte_k32v64_hash.h> + +TAILQ_HEAD (rte_k32v64_hash_list, rte_tailq_entry); + +static struct rte_tailq_elem rte_k32v64_hash_tailq = { + .name ="RTE_K32V64_HASH" , +}; + +EAL_REGISTER_TAILQ (rte_k32v64_hash_tailq); + +#define VALID_KEY_MSK ((1 << RTE_K32V64_KEYS_PER_BUCKET) - 1) + +#ifdef CC_AVX512VL_SUPPORT +int +k32v64_hash_bulk_lookup_avx512vl (struct rte_k32v64_hash_table *table, +uint32_t *keys,uint32_t *hashes,uint64_t *values,unsigned int n); +#endif + +static int +k32v64_hash_bulk_lookup (struct rte_k32v64_hash_table *table,uint32_t *keys, +uint32_t *hashes,uint64_t *values,unsigned int n) +{ +int ret, cnt =0 ; +unsigned int i; + +if (unlikely ((table ==NULL ) || (keys ==NULL ) || (hashes ==NULL ) || + (values ==NULL ))) +return -EINVAL ; + +for (i =0 ; i < n; i++) { + ret =rte_k32v64_hash_lookup (table, keys[i], hashes[i], + &values[i]); +if (ret ==0 ) + cnt++; + } +return cnt; +} + +static rte_k32v64_hash_bulk_lookup_t +get_lookup_bulk_fn (void ) +{ +#ifdef CC_AVX512VL_SUPPORT +if (rte_cpu_get_flag_enabled (RTE_CPUFLAG_AVX512F )) +return k32v64_hash_bulk_lookup_avx512vl; +#endif +return k32v64_hash_bulk_lookup; +} + +int +rte_k32v64_hash_add(struct rte_k32v64_hash_table *table, uint32_t key,
lib/librte_hash/rte_k32v64_hash.c line 72
Quoted 10 lines
lib/librte_hash/rte_k32v64_hash.c new file +10
-0+ uint32_t hash, uint64_t value) +{ + uint32_t bucket; +int i, idx, ret; +uint8_t msk; +struct rte_k32v64_ext_ent *tmp, *ent, *prev =NULL ; + +if (table ==NULL ) +return -EINVAL ; +
I think for add you also need to do update bucket.cnt at the start/end of updates (as you do for del).
Quoted 60 lines
lib/librte_hash/rte_k32v64_hash.c new file +60
-0+ bucket = hash & table->bucket_msk; +/* Search key in table. Update value if exists */ +for (i =0 ; i <RTE_K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && + (table->t[bucket].key_mask & (1 << i))) { + table->t[bucket].val[i] = value; +return 0 ; + } + } + +if (!SLIST_EMPTY (&table->t[bucket].head)) { + SLIST_FOREACH(ent, &table->t[bucket].head, next) { +if (ent->key == key) { + ent->val = value; +return 0 ; + } + } + } + + msk = ~table->t[bucket].key_mask &VALID_KEY_MSK ; +if (msk) { + idx =__builtin_ctz (msk); + table->t[bucket].key[idx] = key; + table->t[bucket].val[idx] = value; +rte_smp_wmb (); + table->t[bucket].key_mask |=1 << idx; + table->nb_ent++; +return 0 ; + } + + ret =rte_mempool_get (table->ext_ent_pool, (void **)&ent); +if (ret <0 ) +return ret; + +SLIST_NEXT (ent, next) =NULL ; + ent->key = key; + ent->val = value; +rte_smp_wmb (); +SLIST_FOREACH (tmp, &table->t[bucket].head, next) + prev = tmp; + +if (prev ==NULL ) +SLIST_INSERT_HEAD (&table->t[bucket].head, ent, next); +else +SLIST_INSERT_AFTER (prev, ent, next); + + table->nb_ent++; + table->nb_ext_ent++; +return 0 ; +} + +int +rte_k32v64_hash_delete (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash) +{ +uint32_t bucket; +int i; +struct rte_k32v64_ext_ent *ent; + +if (table ==NULL )
lib/librte_hash/rte_k32v64_hash.c line 142
Quoted 10 lines
lib/librte_hash/rte_k32v64_hash.c new file +10
-0+return -EINVAL ; + + bucket = hash & table->bucket_msk; + +for (i =0 ; i <RTE_K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && + (table->t[bucket].key_mask & (1 << i))) { + ent =SLIST_FIRST (&table->t[bucket].head); +if (ent) { +rte_atomic32_inc (&table->t[bucket].cnt);
I know that right now rte_atomic32 uses _sync gcc builtins underneath, so it should be safe.
But I think the proper way would be:
table->t[bucket].cnt++;rte_smp_wmb();
or as alternative probably use C11 atomic ACQUIRE/RELEASE
lib/librte_hash/rte_k32v64_hash.c line 149
Quoted 7 lines
lib/librte_hash/rte_k32v64_hash.c new file +7
-0+ table->t[bucket].key[i] = ent->key; + table->t[bucket].val[i] = ent->val; +SLIST_REMOVE_HEAD (&table->t[bucket].head, next); +rte_atomic32_inc (&table->t[bucket].cnt); + table->nb_ext_ent--; + }else + table->t[bucket].key_mask &= ~(1 << i);
I think you protect that update with bucket.cnt.
From my perspective -a s a rule of thumb any update to the bucket/list Should be within that transaction-start/transaction-end.
Quoted 266 lines
lib/librte_hash/rte_k32v64_hash.c new file +166
-0+if (ent) +rte_mempool_put (table->ext_ent_pool, ent); + table->nb_ent--; +return 0 ; + } + } + + SLIST_FOREACH(ent, &table->t[bucket].head, next) +if (ent->key == key) +break ; + +if (ent ==NULL ) +return -ENOENT ; + +rte_atomic32_inc (&table->t[bucket].cnt); +SLIST_REMOVE (&table->t[bucket].head, ent, rte_k32v64_ext_ent, next); +rte_atomic32_inc (&table->t[bucket].cnt); +rte_mempool_put (table->ext_ent_pool, ent); + + table->nb_ext_ent--; + table->nb_ent--; + +return 0 ; +} + +struct rte_k32v64_hash_table * +rte_k32v64_hash_find_existing (const char *name) +{ +struct rte_k32v64_hash_table *h =NULL ; +struct rte_tailq_entry *te; +struct rte_k32v64_hash_list *k32v64_hash_list; + + k32v64_hash_list =RTE_TAILQ_CAST (rte_k32v64_hash_tailq.head, + rte_k32v64_hash_list); + +rte_mcfg_tailq_read_lock (); + TAILQ_FOREACH(te, k32v64_hash_list, next) { + h = (struct rte_k32v64_hash_table *) te->data; +if (strncmp (name, h->name,RTE_K32V64_HASH_NAMESIZE ) ==0 ) +break ; + } +rte_mcfg_tailq_read_unlock (); +if (te ==NULL ) { + rte_errno =ENOENT ; +return NULL ; + } +return h; +} + +struct rte_k32v64_hash_table * +rte_k32v64_hash_create (const struct rte_k32v64_hash_params *params) +{ +char hash_name[RTE_K32V64_HASH_NAMESIZE ]; +struct rte_k32v64_hash_table *ht =NULL ; +struct rte_tailq_entry *te; +struct rte_k32v64_hash_list *k32v64_hash_list; +uint32_t mem_size, nb_buckets, max_ent; +int ret; +struct rte_mempool *mp; + +if ((params ==NULL ) || (params->name ==NULL ) || + (params->entries ==0 )) { + rte_errno =EINVAL ; +return NULL ; + } + + k32v64_hash_list =RTE_TAILQ_CAST (rte_k32v64_hash_tailq.head, + rte_k32v64_hash_list); + + ret =snprintf (hash_name,sizeof (hash_name),"K32V64_%s" , params->name); +if (ret <0 || ret >=RTE_K32V64_HASH_NAMESIZE ) { + rte_errno =ENAMETOOLONG ; +return NULL ; + } + + max_ent =rte_align32pow2 (params->entries); + nb_buckets = max_ent /RTE_K32V64_KEYS_PER_BUCKET ; + mem_size =sizeof (struct rte_k32v64_hash_table ) + +sizeof (struct rte_k32v64_hash_bucket ) * nb_buckets; + + mp =rte_mempool_create (hash_name, max_ent, +sizeof (struct rte_k32v64_ext_ent ),0 ,0 ,NULL ,NULL ,NULL ,NULL , + params->socket_id,0 ); + +if (mp ==NULL ) +return NULL ; + +rte_mcfg_tailq_write_lock (); + TAILQ_FOREACH(te, k32v64_hash_list, next) { + ht = (struct rte_k32v64_hash_table *) te->data; +if (strncmp (params->name, ht->name, +RTE_K32V64_HASH_NAMESIZE ) ==0 ) +break ; + } + ht =NULL ; +if (te !=NULL ) { + rte_errno =EEXIST ; +rte_mempool_free (mp); + goto exit; + } + + te =rte_zmalloc ("K32V64_HASH_TAILQ_ENTRY" ,sizeof (*te),0 ); +if (te ==NULL ) { +RTE_LOG (ERR ,HASH ,"Failed to allocate tailq entry\n" ); +rte_mempool_free (mp); + goto exit; + } + + ht =rte_zmalloc_socket (hash_name, mem_size, +RTE_CACHE_LINE_SIZE , params->socket_id); +if (ht ==NULL ) { +RTE_LOG (ERR ,HASH ,"Failed to allocate fbk hash table\n" ); +rte_free (te); +rte_mempool_free (mp); + goto exit; + } + +memcpy (ht->name, hash_name,sizeof (ht->name)); + ht->max_ent = max_ent; + ht->bucket_msk = nb_buckets -1 ; + ht->ext_ent_pool = mp; + ht->lookup =get_lookup_bulk_fn (); + + te->data = (void *)ht; +TAILQ_INSERT_TAIL (k32v64_hash_list, te, next); + +exit: +rte_mcfg_tailq_write_unlock (); + +return ht; +} + +void +rte_k32v64_hash_free (struct rte_k32v64_hash_table *ht) +{ +struct rte_tailq_entry *te; +struct rte_k32v64_hash_list *k32v64_hash_list; + +if (ht ==NULL ) +return ; + + k32v64_hash_list =RTE_TAILQ_CAST (rte_k32v64_hash_tailq.head, + rte_k32v64_hash_list); + +rte_mcfg_tailq_write_lock (); + +/* find out tailq entry */ + TAILQ_FOREACH(te, k32v64_hash_list, next) { +if (te->data == (void *) ht) +break ; + } + + +if (te ==NULL ) { +rte_mcfg_tailq_write_unlock (); +return ; + } + +TAILQ_REMOVE (k32v64_hash_list, te, next); + +rte_mcfg_tailq_write_unlock (); + +rte_mempool_free (ht->ext_ent_pool); +rte_free (ht); +rte_free (te); +}lib/librte_hash/rte_k32v64_hash.h new file +94
-0@@ -0,0 +1,211 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#ifndef _RTE_K32V64_HASH_H_ +#define _RTE_K32V64_HASH_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <rte_compat.h> +#include <rte_atomic.h> +#include <rte_mempool.h> + +#define RTE_K32V64_HASH_NAMESIZE 32 +#define RTE_K32V64_KEYS_PER_BUCKET 4 +#define RTE_K32V64_WRITE_IN_PROGRESS 1 + +struct rte_k32v64_hash_params { + const char *name; + uint32_t entries; + int socket_id; +}; + +struct rte_k32v64_ext_ent { + SLIST_ENTRY(rte_k32v64_ext_ent) next; + uint32_t key; + uint64_t val; +}; + +struct rte_k32v64_hash_bucket { + uint32_t key[RTE_K32V64_KEYS_PER_BUCKET]; + uint64_t val[RTE_K32V64_KEYS_PER_BUCKET]; + uint8_t key_mask; + rte_atomic32_t cnt; + SLIST_HEAD(rte_k32v64_list_head, rte_k32v64_ext_ent) head; +} __rte_cache_aligned; + +struct rte_k32v64_hash_table; + +typedef int (*rte_k32v64_hash_bulk_lookup_t) +(struct rte_k32v64_hash_table *table, uint32_t *keys, uint32_t *hashes, + uint64_t *values, unsigned int n); + +struct rte_k32v64_hash_table { + char name[RTE_K32V64_HASH_NAMESIZE]; /**< Name of the hash. */ + uint32_t nb_ent; /**< Number of entities in the table*/ + uint32_t nb_ext_ent; /**< Number of extended entities */ + uint32_t max_ent; /**< Maximum number of entities */ + uint32_t bucket_msk; + struct rte_mempool *ext_ent_pool; + rte_k32v64_hash_bulk_lookup_t lookup; + __extension__ struct rte_k32v64_hash_bucket t[0]; +}; + +typedef int (*rte_k32v64_cmp_fn_t) +(struct rte_k32v64_hash_bucket *bucket, uint32_t key, uint64_t *val); + +static inline int +__k32v64_cmp_keys(struct rte_k32v64_hash_bucket *bucket, uint32_t key, + uint64_t *val) +{ + int i; + + for (i = 0; i < RTE_K32V64_KEYS_PER_BUCKET; i++) { + if ((key == bucket->key[i]) && + (bucket->key_mask & (1 << i))) { + *val = bucket->val[i]; + return 1; + } + } + + return 0; +} + +static inline int +__k32v64_hash_lookup(struct rte_k32v64_hash_table *table, uint32_t key, + uint32_t hash, uint64_t *value, rte_k32v64_cmp_fn_t cmp_f) +{ + uint64_t val = 0; + struct rte_k32v64_ext_ent *ent; + int32_t cnt; + int found = 0; + uint32_t bucket = hash & table->bucket_msk; + + do { + do + cnt = rte_atomic32_read(&table->t[bucket].cnt); + while (unlikely(cnt & RTE_K32V64_WRITE_IN_PROGRESS)); + + found = cmp_f(&table->t[bucket], key, &val); + if (unlikely((found == 0) && + (!SLIST_EMPTY(&table->t[bucket].head)))) {
lib/librte_hash/rte_k32v64_hash.h line 104
Quoted 10 lines
lib/librte_hash/rte_k32v64_hash.h new file +10
-0+ SLIST_FOREACH(ent, &table->t[bucket].head, next) { +if (ent->key == key) { + val = ent->val; + found =1 ; +break ; + } + } + } + + }while (unlikely (cnt !=rte_atomic32_read (&table->t[bucket].cnt)));
AFAIK atomic32_read is just a normal read op, so it can be reordered with other ops. So this construction doesn't protect you from races.
What you probably need here:
do {
cnt1 = table->t[bucket].cnt;
rte_smp_rmb();
....
rte_smp_rmb();
cnt2 = table->t[bucket].cnt;
while (cnt1 != cnt2 || (cnt1 & RTE_K32V64_WRITE_IN_PROGRESS) != 0)Quoted 8 lines
lib/librte_hash/rte_k32v64_hash.h new file +8
-0+ +if (found ==1 ) { + *value = val; +return 0 ; + }else +return -ENOENT ; +} +
Re: [dpdk-dev] [PATCH v3 0/4] add new k32v64 hash table
In reply to Wang, Yipeng1Hi everyone,
Quoted 87 lines
Quoted 68 lines · Vladimir
Quoted 18 lines
Hi Mattias,
-----Original Message-----
From: Mattias Rönnblom <[email protected]>
Sent: Wednesday, April 15, 2020 7:52 PM
To: Medvedkin, Vladimir <[email protected]>; [email protected]
Cc: Ananyev, Konstantin <[email protected]>; Wang, Yipeng1 <[email protected]>; Gobriel, Sameh <[email protected]>; Richardson, Bruce <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v3 0/4] add new k32v64 hash tableQuoted 6 lines · Vladimir Medvedkin
Currently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called K32V64 hash supports 4 byte keys and 8 byte associated values, which is enough to store a pointer.Quoted 4 lines
Quoted 2 lines
It would also be nice to get feedback on whether to leave the old FBK and new k32v64 implementations or deprecate the old one?
Do you think it would be feasible to support custom-sized values and remain
efficient, in a similar manner to how
rte_ring_elem.hdoes things?Quoted 2 lines
I'm afraid it is not feasible. For the performance reason keys and
corresponding values resides in single cache line so there are no extra memory for bigger values, such as 16B.
Well, if you have a smaller value type (or key type) you would fit into something less-than-a-cache line, and thus reduce your memory working set further.
Quoted 18 lines
Quoted 18 lines
v3:
- added bulk lookup
- avx512 key comparizon is removed from .hv2:
- renamed fromrte_dwktorte_k32v64as was suggested
- reworked lookup function, added inlined subroutines
- added avx512 key comparizon routine
- added documentation
- added statistic counters for total entries and extended entries(linked list)Vladimir Medvedkin (4):
hash: add k32v64 hash library
hash: add documentation for k32v64 hash library
test: add k32v64 hash autotests
test: add k32v64 perf testsQuoted 14 lines
Quoted 6 lines
app/test/Makefile | 1 + app/test/autotest_data.py | 12 ++ app/test/meson.build | 3 + app/test/test_hash_perf.c | 130 ++++++++++++ app/test/test_k32v64_hash.c | 229 ++++++++++++++++++++++ doc/api/doxy-api-index.md | 1 +Quoted 8 lines
doc/guides/prog_guide/index.rst | 1 + doc/guides/prog_guide/k32v64_hash_lib.rst | 66 +++++++ lib/Makefile | 2 +- lib/librte_hash/Makefile | 13 +- lib/librte_hash/k32v64_hash_avx512vl.c | 56 ++++++ lib/librte_hash/meson.build | 17 +- lib/librte_hash/rte_hash_version.map | 6 +- lib/librte_hash/rte_k32v64_hash.c | 315Quoted 9 lines
++++++++++++++++++++++++++++++
Quoted 8 lines
Quoted 8 lines
lib/librte_hash/rte_k32v64_hash.h | 211 ++++++++++++++++++++ 15 files changed, 1058 insertions(+), 5 deletions(-)create mode 100644
app/test/test_k32v64_hash.ccreate mode 100644doc/guides/prog_guide/k32v64_hash_lib.rstcreate mode 100644lib/librte_hash/k32v64_hash_avx512vl.ccreate mode 100644lib/librte_hash/rte_k32v64_hash.ccreate mode 100644lib/librte_hash/rte_k32v64_hash.h[Wang, Yipeng]
Hi, Vladimir,
Thanks for responding with the use cases earlier. I discussed with Sameh offline, here are some comments.1. Since the proposed hash table also has some similarities to
rte_tablelibrary used by packet framework, have you tried it yet? Although it is mainly for packet framework, I believe you can use it independently as well. It has implementations for special key value sizes.
I added Cristian for his comment.
Quoted 8 lines
2. We tend to agree with Mattias that it would be better if we have a more generic API name and with the same API we can do multiple key/value size implementations.
This is to avoid adding new APIs in future to again handle different key/value
use cases. For example, we call itrte_kv_hash, and through the parameter struct we pass in a key-value size pair we want to use.
Implementation-wise, we may only provide implementations for certain popular use cases (like the one you provided). For other general use cases, people should go with the more flexible and generic cuckoo hash. Then we should also merge the FBK under the new API.
From my perspective Vladimir work is not an attempt to introduce new API - but to fix (extend) existing FBK one.
Right now there is a contradictory situation:
from one side with 4B keys hash tables are quite common, from other side because of its limitations (2B value, no mechanism to resolve collisions) fbk_hash is hardly usable in majority of real-world scenarios.
About making it more generic - we already have one generic rte_hash API, why should we introduce an new one?
My take we should fix FBK hash and make it usable.
If it is not an option by some reason, then I think we
should deprecate and remove FBK hash at all,
and concentrate on making generic one work faster for particular scenarios. Let say we can try to move what Vladimir did, under rte_hash API umbrella for special config (4B key).
Konstantin
Re: [dpdk-dev] [PATCH v3 1/4] hash: add k32v64 hash library
In reply to Vladimir MedvedkinHi Vladimir,
I am not sure which way the decision is made, but few comments inline. Please use C11 built-ins for synchronization.
Quoted 259 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Vladimir Medvedkin
Sent: Wednesday, April 15, 2020 1:17 PM
To: [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected]
Subject: [dpdk-dev] [PATCH v3 1/4] hash: add k32v64 hash libraryK32V64 hash is a hash table that supports 32 bit keys and 64 bit values. This table is hash function agnostic so user must provide precalculated hash signature for add/delete/lookup operations.
Signed-off-by Vladimir Medvedkin <[email protected]>--- lib/Makefile | 2 +- lib/librte_hash/Makefile | 13 +- lib/librte_hash/k32v64_hash_avx512vl.c | 56 ++++++ lib/librte_hash/meson.build | 17 +- lib/librte_hash/rte_hash_version.map | 6 +- lib/librte_hash/rte_k32v64_hash.c | 315 +++++++++++++++++++++++++++++++++ lib/librte_hash/rte_k32v64_hash.h | 211 ++++++++++++++++++++++ 7 files changed, 615 insertions(+), 5 deletions(-) create mode 100644 lib/librte_hash/k32v64_hash_avx512vl.c create mode 100644 lib/librte_hash/rte_k32v64_hash.c create mode100644
lib/librte_hash/rte_k32v64_hash.hlib/Makefile +0
-0@@ -48,7 +48,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_VHOST) += librte_vhostDEPDIRS-librte_vhost :=
librte_eallibrte_mempoollibrte_mbuflibrte_ethdev\
librte_netlibrte_hashlibrte_cryptodev
DIRS-$(CONFIG_RTE_LIBRTE_HASH) +=librte_hash-DEPDIRS-librte_hash :=librte_eallibrte_ring
+DEPDIRS-librte_hash :=librte_eallibrte_ringlibrte_mempool
DIRS-$(CONFIG_RTE_LIBRTE_EFD) +=librte_efdDEPDIRS-librte_efd :=librte_eallibrte_ringlibrte_hash
DIRS-$(CONFIG_RTE_LIBRTE_RIB) +=librte_ribdiff --git
a/lib/librte_hash/Makefile b/lib/librte_hash/Makefile index ec9f864..023689d 100644lib/librte_hash/Makefile +3
-1@@ -8,13 +8,14 @@ LIB = librte_hash.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_ring +LDLIBS += -lrte_eal -lrte_ring -lrte_mempool EXPORT_MAP := rte_hash_version.map # all source are stored in SRCS-y SRCS-$(CONFIG_RTE_LIBRTE_HASH) := rte_cuckoo_hash.c SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_fbk_hash.c +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_k32v64_hash.c # install this header file SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include := rte_hash.h @@ -27,5 +28,15 @@ endif SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include +=
rte_jhash.hSYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include +=rte_thash.hSYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include +=rte_fbk_hash.hlib/librte_hash/Makefile +3
-0+SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_k32v64_hash.h + +CC_AVX512VL_SUPPORT=$(shell $(CC) -mavx512vl -dM -E - </dev/null 2>&1|
+\ grep -q__AVX512VL__&& echo 1)lib/librte_hash/Makefile +5
-0+ +ifeq ($(CC_AVX512VL_SUPPORT), 1) + SRCS-$(CONFIG_RTE_LIBRTE_HASH) += k32v64_hash_avx512vl.c + CFLAGS_k32v64_hash_avx512vl.o += -mavx512vl + CFLAGS_rte_k32v64_hash.o += -DCC_AVX512VL_SUPPORT endif include $(RTE_SDK)/mk/rte.lib.mklib/librte_hash/k32v64_hash_avx512vl.c new file +12
-0@@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include <rte_k32v64_hash.h> + +int +k32v64_hash_bulk_lookup_avx512vl (struct rte_k32v64_hash_table *table, +uint32_t *keys,uint32_t *hashes,uint64_t *values,unsigned int n); + +static inline int +k32v64_cmp_keys_avx512vl(struct rte_k32v64_hash_bucket *bucket,
uint32_tkey,lib/librte_hash/k32v64_hash_avx512vl.c new file +9
-0+ uint64_t *val) +{ +__m128i keys, srch_key; +__mmask8 msk; + + keys =_mm_load_si128 ((void *)bucket); + srch_key =_mm_set1_epi32 (key); + + msk = _mm_mask_cmpeq_epi32_mask(bucket->key_mask, keys,
srch_key);lib/librte_hash/k32v64_hash_avx512vl.c new file +10
-0+if (msk) { + *val = bucket->val[__builtin_ctz (msk)]; +return 1 ; + } + +return 0 ; +} + +static inline int +k32v64_hash_lookup_avx512vl(struct rte_k32v64_hash_table *table,
uint32_tkey,lib/librte_hash/k32v64_hash_avx512vl.c new file +24
-0+ uint32_t hash, uint64_t *value) +{ + return __k32v64_hash_lookup(table, key, hash, value, +k32v64_cmp_keys_avx512vl ); +} + +int +k32v64_hash_bulk_lookup_avx512vl (struct rte_k32v64_hash_table *table, +uint32_t *keys,uint32_t *hashes,uint64_t *values,unsigned int n) { +int ret, cnt =0 ; +unsigned int i; + +if (unlikely ((table ==NULL ) || (keys ==NULL ) || (hashes ==NULL ) || + (values ==NULL ))) +return -EINVAL ; + +for (i =0 ; i < n; i++) { + ret =k32v64_hash_lookup_avx512vl (table, keys[i], hashes[i], + &values[i]); +if (ret ==0 ) + cnt++; + } +return cnt; +}lib/librte_hash/meson.build index renamed from lib/librte_hash/meson.build +0
-06ab46ae..8a37cc4 100644
lib/librte_hash/meson.build +15
-1@@ -3,10 +3,23 @@ headers = files('rte_crc_arm64.h', 'rte_fbk_hash.h', + 'rte_k32v64_hash.h', 'rte_hash_crc.h', 'rte_hash.h', 'rte_jhash.h', 'rte_thash.h') -sources = files('rte_cuckoo_hash.c', 'rte_fbk_hash.c') -deps += ['ring'] +sources = files('rte_cuckoo_hash.c', 'rte_fbk_hash.c', +'rte_k32v64_hash.c') deps += ['ring', 'mempool'] + +if dpdk_conf.has('RTE_ARCH_X86') + if cc.has_argument('-mavx512vl') + avx512_tmplib = static_library('avx512_tmp', + 'k32v64_hash_avx512vl.c', + dependencies: static_rte_mempool, + c_args: cflags + ['-mavx512vl']) + objs += avx512_tmplib.extract_objects('k32v64_hash_avx512vl.c') + cflags += '-DCC_AVX512VL_SUPPORT' + + endif +endiflib/librte_hash/rte_hash_version.map +5
-1@@ -34,5 +34,9 @@ EXPERIMENTAL { rte_hash_free_key_with_position; rte_hash_max_key_id; - + rte_k32v64_hash_create; + rte_k32v64_hash_find_existing; + rte_k32v64_hash_free; + rte_k32v64_hash_add; + rte_k32v64_hash_delete; };lib/librte_hash/rte_k32v64_hash.c new file +32
-0@@ -0,0 +1,315 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include <string.h> + +#include <rte_eal_memconfig.h> +#include <rte_errno.h> +#include <rte_malloc.h> +#include <rte_memory.h> +#include <rte_tailq.h> + +#include <rte_k32v64_hash.h> + +TAILQ_HEAD (rte_k32v64_hash_list, rte_tailq_entry); + +static struct rte_tailq_elem rte_k32v64_hash_tailq = { + .name ="RTE_K32V64_HASH" , +}; + +EAL_REGISTER_TAILQ (rte_k32v64_hash_tailq); + +#define VALID_KEY_MSK ((1 << RTE_K32V64_KEYS_PER_BUCKET) - 1) + +#ifdef CC_AVX512VL_SUPPORT +int +k32v64_hash_bulk_lookup_avx512vl (struct rte_k32v64_hash_table *table, +uint32_t *keys,uint32_t *hashes,uint64_t *values,unsigned int n); +#endif + +static int +k32v64_hash_bulk_lookup(struct rte_k32v64_hash_table *table, uint32_t*keys,
lib/librte_hash/rte_k32v64_hash.c new file +34
-0+ uint32_t *hashes, uint64_t *values, unsigned int n) { + int ret, cnt = 0; +unsigned int i; + +if (unlikely ((table ==NULL ) || (keys ==NULL ) || (hashes ==NULL ) || + (values ==NULL ))) +return -EINVAL ; + +for (i =0 ; i < n; i++) { + ret =rte_k32v64_hash_lookup (table, keys[i], hashes[i], + &values[i]); +if (ret ==0 ) + cnt++; + } +return cnt; +} + +static rte_k32v64_hash_bulk_lookup_t +get_lookup_bulk_fn (void ) +{ +#ifdef CC_AVX512VL_SUPPORT +if (rte_cpu_get_flag_enabled (RTE_CPUFLAG_AVX512F )) +return k32v64_hash_bulk_lookup_avx512vl;#endif +return k32v64_hash_bulk_lookup; +} + +int +rte_k32v64_hash_add (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t value) +{ +uint32_t bucket; +int i, idx, ret; +uint8_t msk; +struct rte_k32v64_ext_ent *tmp, *ent, *prev =NULL ;
lib/librte_hash/rte_k32v64_hash.c line 78
Quoted 10 lines
lib/librte_hash/rte_k32v64_hash.c new file +10
-0+ +if (table ==NULL ) +return -EINVAL ; + + bucket = hash & table->bucket_msk; +/* Search key in table. Update value if exists */ +for (i =0 ; i <RTE_K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && + (table->t[bucket].key_mask & (1 << i))) { + table->t[bucket].val[i] = value;
The old value of val[i] might be getting used in the reader. It needs to be returned to the caller so that it can be put on the RCU defer queue to free later. You might also want to use an atomic store to ensure the stores are not split.
lib/librte_hash/rte_k32v64_hash.c line 86
Quoted 8 lines
lib/librte_hash/rte_k32v64_hash.c new file +8
-0+return 0 ; + } + } + +if (!SLIST_EMPTY (&table->t[bucket].head)) { + SLIST_FOREACH(ent, &table->t[bucket].head, next) { +if (ent->key == key) { + ent->val = value;
Same here, need to return the old value.
lib/librte_hash/rte_k32v64_hash.c line 98
Quoted 12 lines
lib/librte_hash/rte_k32v64_hash.c new file +12
-0+return 0 ; + } + } + } + + msk = ~table->t[bucket].key_mask &VALID_KEY_MSK ; +if (msk) { + idx =__builtin_ctz (msk); + table->t[bucket].key[idx] = key; + table->t[bucket].val[idx] = value; +rte_smp_wmb (); + table->t[bucket].key_mask |=1 << idx;
The barrier and the store can be replaced with a store-release in C11.
Quoted 9 lines
lib/librte_hash/rte_k32v64_hash.c new file +9
-0+ table->nb_ent++; +return 0 ; + } + + ret =rte_mempool_get (table->ext_ent_pool, (void **)&ent); +if (ret <0 ) +return ret; + +SLIST_NEXT (ent, next) =NULL ;
lib/librte_hash/rte_k32v64_hash.c line 117
Quoted 10 lines
lib/librte_hash/rte_k32v64_hash.c new file +10
-0+ ent->key = key; + ent->val = value; +rte_smp_wmb (); +SLIST_FOREACH (tmp, &table->t[bucket].head, next) + prev = tmp; + +if (prev ==NULL ) +SLIST_INSERT_HEAD (&table->t[bucket].head, ent, next); +else +SLIST_INSERT_AFTER (prev, ent, next);
Need C11 SLIST
lib/librte_hash/rte_k32v64_hash.c line 126
Quoted 9 lines
lib/librte_hash/rte_k32v64_hash.c new file +9
-0+ + table->nb_ent++; + table->nb_ext_ent++; +return 0 ; +} + +int +rte_k32v64_hash_delete (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash)
This should return the value corresponding to the deleted key
Quoted 7 lines
lib/librte_hash/rte_k32v64_hash.c new file +7
-0+{ +uint32_t bucket; +int i; +struct rte_k32v64_ext_ent *ent; + +if (table ==NULL ) +return -EINVAL ;
lib/librte_hash/rte_k32v64_hash.c line 143
Quoted 10 lines
lib/librte_hash/rte_k32v64_hash.c new file +10
-0+ + bucket = hash & table->bucket_msk; + +for (i =0 ; i <RTE_K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && + (table->t[bucket].key_mask & (1 << i))) { + ent =SLIST_FIRST (&table->t[bucket].head); +if (ent) { +rte_atomic32_inc (&table->t[bucket].cnt); + table->t[bucket].key[i] = ent->key;
In this case, both key and value are changing and it is not atomic. There is a possibility that the lookup function will receive incorrect value of 'val[i]'. Suggest following the method described below.
lib/librte_hash/rte_k32v64_hash.c line 148
Quoted 6 lines
lib/librte_hash/rte_k32v64_hash.c new file +2
-0+ table->t[bucket].val[i] = ent->val; + SLIST_REMOVE_HEAD(&table->t[bucket].head,next);
lib/librte_hash/rte_k32v64_hash.c new file +3
-0+rte_atomic32_inc (&table->t[bucket].cnt); + table->nb_ext_ent--; + }else
Suggest changing this into a 2 step process:
1) Delete the entry from the fixed bucket (update key_mask)
2) Move the head of extended bucket to the fixed bucket
2a) Insert the key/value from the head into the deleted index and update the key_mask (this will ensure that the reader has the entry available while it is being removed from the extended bucket) 2b) increment the counter indicating the extended bucket is changing
2c) remove the head
2d) increment the counter indicating the extended bucket change is done
The above procedure will result in removing the spinning (resulting from step 2b) in the lookup function (comment is added in the lookup function), readers will not be blocked if the writer is scheduled out.
This logic is implemented in rte_hash using cuckoo hash, suggest to take a look for required memory orderings.
lib/librte_hash/rte_k32v64_hash.c line 151
Quoted 3 lines
lib/librte_hash/rte_k32v64_hash.c new file +3
-0+ table->t[bucket].key_mask &= ~(1 << i); +if (ent) +rte_mempool_put (table->ext_ent_pool, ent);
Entry cannot be put to free list immediately as the readers are still using it.
Quoted 15 lines
lib/librte_hash/rte_k32v64_hash.c new file +14
-0+ table->nb_ent--; +return 0 ; + } + } + + SLIST_FOREACH(ent, &table->t[bucket].head, next) +if (ent->key == key) +break ; + +if (ent ==NULL ) +return -ENOENT ; + +rte_atomic32_inc (&table->t[bucket].cnt); + SLIST_REMOVE(&table->t[bucket].head, ent, rte_k32v64_ext_ent,next);
lib/librte_hash/rte_k32v64_hash.c
Quoted 2 lines
lib/librte_hash/rte_k32v64_hash.c new file +2
-0+rte_atomic32_inc (&table->t[bucket].cnt); +rte_mempool_put (table->ext_ent_pool, ent);
Entry might be getting used by the readers still.
Quoted 215 lines
lib/librte_hash/rte_k32v64_hash.c new file +20
-0+ + table->nb_ext_ent--; + table->nb_ent--; + +return 0 ; +} + +struct rte_k32v64_hash_table * +rte_k32v64_hash_find_existing (const char *name) { +struct rte_k32v64_hash_table *h =NULL ; +struct rte_tailq_entry *te; +struct rte_k32v64_hash_list *k32v64_hash_list; + + k32v64_hash_list =RTE_TAILQ_CAST (rte_k32v64_hash_tailq.head, + rte_k32v64_hash_list); + +rte_mcfg_tailq_read_lock (); + TAILQ_FOREACH(te, k32v64_hash_list, next) { + h = (struct rte_k32v64_hash_table *) te->data; + if (strncmp(name, h->name, RTE_K32V64_HASH_NAMESIZE)== 0)
lib/librte_hash/rte_k32v64_hash.c new file +30
-0+break ; + } +rte_mcfg_tailq_read_unlock (); +if (te ==NULL ) { + rte_errno =ENOENT ; +return NULL ; + } +return h; +} + +struct rte_k32v64_hash_table * +rte_k32v64_hash_create (const struct rte_k32v64_hash_params *params) { +char hash_name[RTE_K32V64_HASH_NAMESIZE ]; +struct rte_k32v64_hash_table *ht =NULL ; +struct rte_tailq_entry *te; +struct rte_k32v64_hash_list *k32v64_hash_list; +uint32_t mem_size, nb_buckets, max_ent; +int ret; +struct rte_mempool *mp; + +if ((params ==NULL ) || (params->name ==NULL ) || + (params->entries ==0 )) { + rte_errno =EINVAL ; +return NULL ; + } + + k32v64_hash_list =RTE_TAILQ_CAST (rte_k32v64_hash_tailq.head, + rte_k32v64_hash_list); + + ret = snprintf(hash_name, sizeof(hash_name), "K32V64_%s", params-Quoted 1 line
name);
lib/librte_hash/rte_k32v64_hash.c new file +12
-0+if (ret <0 || ret >=RTE_K32V64_HASH_NAMESIZE ) { + rte_errno =ENAMETOOLONG ; +return NULL ; + } + + max_ent =rte_align32pow2 (params->entries); + nb_buckets = max_ent /RTE_K32V64_KEYS_PER_BUCKET ; + mem_size =sizeof (struct rte_k32v64_hash_table ) + +sizeof (struct rte_k32v64_hash_bucket ) * nb_buckets; + + mp = rte_mempool_create(hash_name, max_ent, + sizeof(struct rte_k32v64_ext_ent), 0, 0, NULL, NULL, NULL,NULL,
lib/librte_hash/rte_k32v64_hash.c new file +83
-0+ params->socket_id, 0); + +if (mp ==NULL ) +return NULL ; + +rte_mcfg_tailq_write_lock (); + TAILQ_FOREACH(te, k32v64_hash_list, next) { + ht = (struct rte_k32v64_hash_table *) te->data; +if (strncmp (params->name, ht->name, +RTE_K32V64_HASH_NAMESIZE ) ==0 ) +break ; + } + ht =NULL ; +if (te !=NULL ) { + rte_errno =EEXIST ; +rte_mempool_free (mp); + goto exit; + } + + te =rte_zmalloc ("K32V64_HASH_TAILQ_ENTRY" ,sizeof (*te),0 ); +if (te ==NULL ) { +RTE_LOG (ERR ,HASH ,"Failed to allocate tailq entry\n" ); +rte_mempool_free (mp); + goto exit; + } + + ht =rte_zmalloc_socket (hash_name, mem_size, +RTE_CACHE_LINE_SIZE , params->socket_id); +if (ht ==NULL ) { +RTE_LOG (ERR ,HASH ,"Failed to allocate fbk hash table\n" ); +rte_free (te); +rte_mempool_free (mp); + goto exit; + } + +memcpy (ht->name, hash_name,sizeof (ht->name)); + ht->max_ent = max_ent; + ht->bucket_msk = nb_buckets -1 ; + ht->ext_ent_pool = mp; + ht->lookup =get_lookup_bulk_fn (); + + te->data = (void *)ht; +TAILQ_INSERT_TAIL (k32v64_hash_list, te, next); + +exit: +rte_mcfg_tailq_write_unlock (); + +return ht; +} + +void +rte_k32v64_hash_free (struct rte_k32v64_hash_table *ht) { +struct rte_tailq_entry *te; +struct rte_k32v64_hash_list *k32v64_hash_list; + +if (ht ==NULL ) +return ; + + k32v64_hash_list =RTE_TAILQ_CAST (rte_k32v64_hash_tailq.head, + rte_k32v64_hash_list); + +rte_mcfg_tailq_write_lock (); + +/* find out tailq entry */ + TAILQ_FOREACH(te, k32v64_hash_list, next) { +if (te->data == (void *) ht) +break ; + } + + +if (te ==NULL ) { +rte_mcfg_tailq_write_unlock (); +return ; + } + +TAILQ_REMOVE (k32v64_hash_list, te, next); + +rte_mcfg_tailq_write_unlock (); + +rte_mempool_free (ht->ext_ent_pool); +rte_free (ht); +rte_free (te); +}lib/librte_hash/rte_k32v64_hash.h new file +47
-0@@ -0,0 +1,211 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#ifndef _RTE_K32V64_HASH_H_ +#define _RTE_K32V64_HASH_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <rte_compat.h> +#include <rte_atomic.h> +#include <rte_mempool.h> + +#define RTE_K32V64_HASH_NAMESIZE 32 +#define RTE_K32V64_KEYS_PER_BUCKET 4 +#define RTE_K32V64_WRITE_IN_PROGRESS 1 + +struct rte_k32v64_hash_params { + const char *name; + uint32_t entries; + int socket_id; +}; + +struct rte_k32v64_ext_ent { + SLIST_ENTRY(rte_k32v64_ext_ent) next; + uint32_t key; + uint64_t val; +}; + +struct rte_k32v64_hash_bucket { + uint32_t key[RTE_K32V64_KEYS_PER_BUCKET]; + uint64_t val[RTE_K32V64_KEYS_PER_BUCKET]; + uint8_t key_mask; + rte_atomic32_t cnt; + SLIST_HEAD(rte_k32v64_list_head, rte_k32v64_ext_ent) head; } +__rte_cache_aligned; + +struct rte_k32v64_hash_table; + +typedef int (*rte_k32v64_hash_bulk_lookup_t) (struct +rte_k32v64_hash_table *table, uint32_t *keys, uint32_t *hashes, + uint64_t *values, unsigned int n); + +struct rte_k32v64_hash_table { + char name[RTE_K32V64_HASH_NAMESIZE]; /**< Name of thehash. */
lib/librte_hash/rte_k32v64_hash.h new file +12
-0+uint32_t nb_ent;/**< Number of entities in the table*/ +uint32_t nb_ext_ent;/**< Number of extended entities */ +uint32_t max_ent;/**< Maximum number of entities */ +uint32_t bucket_msk; +struct rte_mempool *ext_ent_pool; +rte_k32v64_hash_bulk_lookup_t lookup; + __extension__struct rte_k32v64_hash_bucket t[0 ]; +}; + +typedef int (*rte_k32v64_cmp_fn_t ) +(struct rte_k32v64_hash_bucket *bucket,uint32_t key,uint64_t *val); +
lib/librte_hash/rte_k32v64_hash.h line 69
Quoted 10 lines
lib/librte_hash/rte_k32v64_hash.h new file +10
-0+static inline int +__k32v64_cmp_keys (struct rte_k32v64_hash_bucket *bucket,uint32_t key, +uint64_t *val) +{ +int i; + +for (i =0 ; i <RTE_K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == bucket->key[i]) && + (bucket->key_mask & (1 << i))) { + *val = bucket->val[i];
This load can happen speculatively.
You have to use the guard variable and payload concept here for reader-writer concurrency. On the writer:
store -> key
store -> valstore_release -> key_mask (or there will be a rte_smp_wmb before this) 'key_mask' acts as the guard variable
On the reader:load_acquire -> key_mask (or there will be a rte_smp_rmb after this) if statement etc.....
Quoted 10 lines
lib/librte_hash/rte_k32v64_hash.h new file +10
-0+return 1 ; + } + } + +return 0 ; +} + +static inline int +__k32v64_hash_lookup (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *value,rte_k32v64_cmp_fn_t cmp_f) {
lib/librte_hash/rte_k32v64_hash.h line 90
Quoted 10 lines
lib/librte_hash/rte_k32v64_hash.h new file +10
-0+uint64_t val =0 ; +struct rte_k32v64_ext_ent *ent; +int32_t cnt; +int found =0 ; +uint32_t bucket = hash & table->bucket_msk; + + do { + do + cnt = rte_atomic32_read(&table->t[bucket].cnt); + while (unlikely(cnt & RTE_K32V64_WRITE_IN_PROGRESS));
The reader can hang here if the writer increments the counter and gets scheduled out. Using the method suggested above, you can remove this code.
Quoted 4 lines
lib/librte_hash/rte_k32v64_hash.h new file +4
-0+ + found =cmp_f (&table->t[bucket], key, &val); +if (unlikely ((found ==0 ) && + (!SLIST_EMPTY (&table->t[bucket].head)))) {
lib/librte_hash/rte_k32v64_hash.h line 104
Quoted 10 lines
lib/librte_hash/rte_k32v64_hash.h new file +10
-0+ SLIST_FOREACH(ent, &table->t[bucket].head, next) { +if (ent->key == key) { + val = ent->val; + found =1 ; +break ; + } + } + } + + }while (unlikely (cnt !=rte_atomic32_read (&table->t[bucket].cnt)));
With the logic mentioned in delete function, the counter needs to be read at the beginning of the loop. Suggest looking at rte_hash-cuckoo hash for the memory ordering required for the counter.
Quoted 108 lines
lib/librte_hash/rte_k32v64_hash.h new file +106
-1+ +if (found ==1 ) { + *value = val; +return 0 ; + }else +return -ENOENT ; +} + +static inline int +rte_k32v64_hash_lookup (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *value) +{ +return __k32v64_hash_lookup (table, key, hash, value, +__k32v64_cmp_keys); } + +static inline int +rte_k32v64_hash_bulk_lookup (struct rte_k32v64_hash_table *table, +uint32_t *keys,uint32_t *hashes,uint64_t *values,unsigned int n) { +return table->lookup (table, keys, hashes, values, n); } + +/** +* Add a key to an existing hash table with hash value. +* This operation is not multi-thread safe +* and should only be called from one thread. +* +* @param ht +* Hash table to add the key to. +* @param key +* Key to add to the hash table. +* @param value +* Value to associate with key. +* @param hash +* Hash value associated with key. +* @return +* 0 if ok, or negative value on error. +*/ +__rte_experimental +int +rte_k32v64_hash_add (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t value); + +/** +* Remove a key with a given hash value from an existing hash table. +* This operation is not multi-thread +* safe and should only be called from one thread. +* +* @param ht +* Hash table to remove the key from. +* @param key +* Key to remove from the hash table. +* @param hash +* hash value associated with key. +* @return +* 0 if ok, or negative value on error. +*/ +__rte_experimental +int +rte_k32v64_hash_delete (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash); + + +/** +* Performs a lookup for an existing hash table, and returns a pointer +to +* the table if found. +* +* @param name +* Name of the hash table to find +* +* @return +* pointer to hash table structure or NULL on error with rte_errno +* set appropriately. +*/ +__rte_experimental +struct rte_k32v64_hash_table * +rte_k32v64_hash_find_existing (const char *name); + +/** +* Create a new hash table for use with four byte keys. +* +* @param params +* Parameters used in creation of hash table. +* +* @return +* Pointer to hash table structure that is used in future hash table +* operations, or NULL on error with rte_errno set appropriately. +*/ +__rte_experimental +struct rte_k32v64_hash_table * +rte_k32v64_hash_create (const struct rte_k32v64_hash_params *params); + +/** +* Free all memory used by a hash table. +* +* @param table +* Hash table to deallocate. +*/ +__rte_experimental +void +rte_k32v64_hash_free (struct rte_k32v64_hash_table *table); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_K32V64_HASH_H_ */ --2.7.4
v4 cover add new kv hash table
In reply to Vladimir MedvedkinCurrently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called KV hash
supports 4 byte keys and 8 byte associated values,
which is enough to store a pointer.
v4:
- internal implementation is hided under universal API for any key and value sizes
- add and delete API now return old value
- add transaction counter modification to _add()
- transaction counter now modifies with c11 atomics
v3:
- added bulk lookup
- avx512 key comparizon is removed from .h
v2:
- renamed from rte_dwk to rte_k32v64 as was suggested
- reworked lookup function, added inlined subroutines
- added avx512 key comparizon routine
- added documentation
- added statistic counters for total entries and extended entries(linked list)
Vladimir Medvedkin (4):
hash: add kv hash library
hash: add documentation for kv hash library
test: add kv hash autotests
test: add kv perf tests
app/test/Makefile | 1 + app/test/autotest_data.py | 12 ++ app/test/meson.build | 3 + app/test/test_hash_perf.c | 111 +++++++++++++ app/test/test_kv_hash.c | 242 ++++++++++++++++++++++++++++ doc/api/doxy-api-index.md | 1 + doc/guides/prog_guide/index.rst | 1 + doc/guides/prog_guide/kv_hash_lib.rst | 66 ++++++++ lib/Makefile | 2 +- lib/librte_hash/Makefile | 14 +- lib/librte_hash/k32v64_hash.c | 277 +++++++++++++++++++++++++++++++++ lib/librte_hash/k32v64_hash.h | 98 ++++++++++++ lib/librte_hash/k32v64_hash_avx512vl.c | 59 +++++++ lib/librte_hash/meson.build | 17 +- lib/librte_hash/rte_hash_version.map | 6 +- lib/librte_hash/rte_kv_hash.c | 184 ++++++++++++++++++++++ lib/librte_hash/rte_kv_hash.h | 169 ++++++++++++++++++++ 17 files changed, 1258 insertions(+), 5 deletions(-) create mode 100644 app/test/test_kv_hash.c create mode 100644 doc/guides/prog_guide/kv_hash_lib.rst create mode 100644 lib/librte_hash/k32v64_hash.c create mode 100644 lib/librte_hash/k32v64_hash.h create mode 100644 lib/librte_hash/k32v64_hash_avx512vl.c create mode 100644 lib/librte_hash/rte_kv_hash.c create mode 100644 lib/librte_hash/rte_kv_hash.h
Signature · 1 line
2.7.4
PATCH v4 1/4 hash: add kv hash library
KV hash is a special optimized key-value storage for fixed key and value sizes. At the moment it supports 32 bit keys and 64 bit values. This table is hash function agnostic so user must provide precalculated hash signature for add/delete/lookup operations.
--- lib/Makefile | 2 +- lib/librte_hash/Makefile | 14 +- lib/librte_hash/k32v64_hash.c | 277 +++++++++++++++++++++++++++++++++ lib/librte_hash/k32v64_hash.h | 98 ++++++++++++ lib/librte_hash/k32v64_hash_avx512vl.c | 59 +++++++ lib/librte_hash/meson.build | 17 +- lib/librte_hash/rte_hash_version.map | 6 +- lib/librte_hash/rte_kv_hash.c | 184 ++++++++++++++++++++++ lib/librte_hash/rte_kv_hash.h | 169 ++++++++++++++++++++ 9 files changed, 821 insertions(+), 5 deletions(-) create mode 100644 lib/librte_hash/k32v64_hash.c create mode 100644 lib/librte_hash/k32v64_hash.h create mode 100644 lib/librte_hash/k32v64_hash_avx512vl.c create mode 100644 lib/librte_hash/rte_kv_hash.c create mode 100644 lib/librte_hash/rte_kv_hash.h
lib/Makefile +1 -1
@@ -48,7 +48,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_VHOST) += librte_vhost DEPDIRS-librte_vhost := librte_eal librte_mempool librte_mbuf librte_ethdev \ librte_net librte_hash librte_cryptodev DIRS-$(CONFIG_RTE_LIBRTE_HASH) += librte_hash -DEPDIRS-librte_hash := librte_eal librte_ring +DEPDIRS-librte_hash := librte_eal librte_ring librte_mempool DIRS-$(CONFIG_RTE_LIBRTE_EFD) += librte_efd DEPDIRS-librte_efd := librte_eal librte_ring librte_hash DIRS-$(CONFIG_RTE_LIBRTE_RIB) += librte_rib
lib/librte_hash/Makefile +13 -1
@@ -8,13 +8,15 @@ LIB = librte_hash.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_ring +LDLIBS += -lrte_eal -lrte_ring -lrte_mempool EXPORT_MAP := rte_hash_version.map # all source are stored in SRCS-y SRCS-$(CONFIG_RTE_LIBRTE_HASH) := rte_cuckoo_hash.c SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_fbk_hash.c +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_kv_hash.c +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += k32v64_hash.c # install this header file SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include := rte_hash.h
@@ -27,5 +29,15 @@ endif SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_jhash.h SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_thash.h SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_fbk_hash.h +SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_kv_hash.h + +CC_AVX512VL_SUPPORT=$(shell $(CC) -mavx512vl -dM -E - </dev/null 2>&1 | \ +grep -q __AVX512VL__ && echo 1) + +ifeq ($(CC_AVX512VL_SUPPORT), 1) + SRCS-$(CONFIG_RTE_LIBRTE_HASH) += k32v64_hash_avx512vl.c + CFLAGS_k32v64_hash_avx512vl.o += -mavx512vl + CFLAGS_k32v64_hash.o += -DCC_AVX512VL_SUPPORT +endif include $(RTE_SDK)/mk/rte.lib.mk
lib/librte_hash/k32v64_hash.c new file +277 -0
277 lines not on this page · raw message
lib/librte_hash/k32v64_hash.h new file +98 -0
98 lines not on this page · raw message
lib/librte_hash/k32v64_hash_avx512vl.c new file +59 -0
59 lines not on this page · raw message
lib/librte_hash/meson.build +15 -2
@@ -3,10 +3,23 @@ headers = files('rte_crc_arm64.h', 'rte_fbk_hash.h', + 'rte_kv_hash.h', 'rte_hash_crc.h', 'rte_hash.h', 'rte_jhash.h', 'rte_thash.h') -sources = files('rte_cuckoo_hash.c', 'rte_fbk_hash.c') -deps += ['ring'] +sources = files('rte_cuckoo_hash.c', 'rte_fbk_hash.c', 'rte_kv_hash.c', 'k32v64_hash.c') +deps += ['ring', 'mempool'] + +if dpdk_conf.has('RTE_ARCH_X86') + if cc.has_argument('-mavx512vl') + avx512_tmplib = static_library('avx512_tmp', + 'k32v64_hash_avx512vl.c', + dependencies: static_rte_mempool, + c_args: cflags + ['-mavx512vl']) + objs += avx512_tmplib.extract_objects('k32v64_hash_avx512vl.c') + cflags += '-DCC_AVX512VL_SUPPORT' + + endif +endif
lib/librte_hash/rte_hash_version.map +5 -1
@@ -36,5 +36,9 @@ EXPERIMENTAL { rte_hash_lookup_with_hash_bulk; rte_hash_lookup_with_hash_bulk_data; rte_hash_max_key_id; - + rte_kv_hash_create; + rte_kv_hash_find_existing; + rte_kv_hash_free; + rte_kv_hash_add; + rte_kv_hash_delete; };
lib/librte_hash/rte_kv_hash.c new file +184 -0
184 lines not on this page · raw message
lib/librte_hash/rte_kv_hash.h new file +169 -0
169 lines not on this page · raw message
Signature · 1 line
2.7.4
PATCH v4 2/4 hash: add documentation for kv hash library
In reply to Vladimir MedvedkinAdd programmers guide and doxygen API for kv hash library
--- doc/api/doxy-api-index.md | 1 + doc/guides/prog_guide/index.rst | 1 + doc/guides/prog_guide/kv_hash_lib.rst | 66 +++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 doc/guides/prog_guide/kv_hash_lib.rst
doc/api/doxy-api-index.md +1 -0
@@ -121,6 +121,7 @@ The public API headers are grouped by topics: [jhash] (@ref rte_jhash.h), [thash] (@ref rte_thash.h), [FBK hash] (@ref rte_fbk_hash.h), + [KV hash] (@ref rte_kv_hash.h), [CRC hash] (@ref rte_hash_crc.h) - **classification**
doc/guides/prog_guide/index.rst +1 -0
@@ -31,6 +31,7 @@ Programmer's Guide link_bonding_poll_mode_drv_lib timer_lib hash_lib + kv_hash_lib efd_lib member_lib lpm_lib
doc/guides/prog_guide/kv_hash_lib.rst new file +66 -0
66 lines not on this page · raw message
Signature · 1 line
2.7.4
PATCH v4 3/4 test: add kv hash autotests
In reply to Vladimir MedvedkinAdd autotests for rte_kv_hash library
--- app/test/Makefile | 1 + app/test/autotest_data.py | 12 +++ app/test/meson.build | 3 + app/test/test_kv_hash.c | 242 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 258 insertions(+) create mode 100644 app/test/test_kv_hash.c
app/test/Makefile +1 -0
@@ -73,6 +73,7 @@ SRCS-y += test_bitmap.c SRCS-y += test_reciprocal_division.c SRCS-y += test_reciprocal_division_perf.c SRCS-y += test_fbarray.c +SRCS-y += test_kv_hash.c SRCS-y += test_external_mem.c SRCS-y += test_rand_perf.c
app/test/autotest_data.py +12 -0
@@ -99,6 +99,18 @@ "Report": None, }, { + "Name": "KV hash autotest", + "Command": "kv_hash_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "KV hash autotest", + "Command": "kv_hash_slow_autotest", + "Func": default_autotest, + "Report": None, + }, + { "Name": "LPM autotest", "Command": "lpm_autotest", "Func": default_autotest,
app/test/meson.build +3 -0
@@ -45,6 +45,7 @@ test_sources = files('commands.c', 'test_eventdev.c', 'test_external_mem.c', 'test_fbarray.c', + 'test_kv_hash.c', 'test_fib.c', 'test_fib_perf.c', 'test_fib6.c',
@@ -203,6 +204,7 @@ fast_tests = [ ['hash_autotest', true], ['interrupt_autotest', true], ['ipfrag_autotest', false], + ['kv_hash_autotest', true], ['logs_autotest', true], ['lpm_autotest', true], ['lpm6_autotest', true],
@@ -291,6 +293,7 @@ perf_test_names = [ 'hash_readwrite_perf_autotest', 'hash_readwrite_lf_perf_autotest', 'trace_perf_autotest', + 'kv_hash_slow_autotest', ] driver_test_names = [
app/test/test_kv_hash.c new file +242 -0
242 lines not on this page · raw message
Signature · 1 line
2.7.4
PATCH v4 4/4 test: add kv perf tests
In reply to Vladimir MedvedkinAdd performance tests for rte_kv_hash
--- app/test/test_hash_perf.c | 111 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+)
app/test/test_hash_perf.c +111 -0
@@ -12,8 +12,10 @@#include <rte_hash_crc.h> #include <rte_jhash.h> #include <rte_fbk_hash.h> +#include <rte_kv_hash.h> #include <rte_random.h> #include <rte_string_fns.h> +#include <rte_hash_crc.h> #include "test.h"
@@ -29,6 +31,8 @@#define NUM_SHUFFLES 10#define BURST_SIZE 16 +#define CRC_INIT_VAL 0xdeadbeef +enum operations {ADD =0 ,LOOKUP ,
@@ -719,6 +723,110 @@ fbk_hash_perf_test(void)return 0 ; } +static uint32_t * +shuf_arr (uint32_t *arr,int n,int l) +{ +int i, j; +uint32_t tmp; +uint32_t *ret_arr; + + ret_arr =rte_zmalloc (NULL , l *sizeof (uint32_t ),0 ); +for (i =0 ; i < n; i++) { + j =rte_rand () % n; + tmp = arr[j]; + arr[j] = arr[i]; + arr[i] = tmp; + } +for (i =0 ; i < l; i++) + ret_arr[i] = arr[i % n]; + +return ret_arr; +} + +static int +kv_hash_perf_test (void ) +{ +struct rte_kv_hash_params params = { + .name ="kv_hash_test" , + .entries =ENTRIES *2 , + .socket_id =rte_socket_id (), + .type =RTE_KV_HASH_K32V64 , + }; +struct rte_kv_hash_table *handle =NULL ; +uint32_t *keys; +uint32_t *lookup_keys; +uint64_t lookup_time =0 ; +uint64_t begin; +uint64_t end; +unsigned int added =0 ; +uint32_t key, hash_sig; +uint16_t val; +unsigned int i, j, k; +int found, ret =0 ; +uint32_t hashes[64 ]; +uint64_t vals[64 ]; + + handle =rte_kv_hash_create (¶ms); +if (handle ==NULL ) { +printf ("Error creating table\n" ); +return -1 ; + } + + keys =rte_zmalloc (NULL ,ENTRIES *sizeof (*keys),0 ); +if (keys ==NULL ) { +printf ("fbk hash: memory allocation for key store failed\n" ); +return -1 ; + } + +/* Generate random keys and values. */ +for (i =0 ; i <ENTRIES ; i++) { + key = (uint32_t )rte_rand (); + val =rte_rand (); + hash_sig =rte_hash_crc_4byte (key,CRC_INIT_VAL ); + +if (rte_kv_hash_add (handle, &key, hash_sig, &val, + &found) ==0 ) { + keys[added] = key; + added++; + } + } + + lookup_keys =shuf_arr (keys, added,TEST_SIZE ); + + lookup_time =0 ; +for (i =0 ; i <TEST_ITERATIONS ; i++) { + + begin =rte_rdtsc (); +/* Do lookups */ + +for (j =0 ; j <TEST_SIZE ; j +=64 ) { +for (k =0 ; k <64 ; k++) { + hashes[k] = +rte_hash_crc_4byte (lookup_keys[j + k], +CRC_INIT_VAL ); + } + + ret +=rte_kv_hash_bulk_lookup (handle, + &lookup_keys[j], hashes, vals,64 ); + } + + end =rte_rdtsc (); + lookup_time += (double )(end - begin); + } + +printf ("\n\n *** KV Hash function performance test results ***\n" ); +if (ret !=0 ) +printf ("Number of ticks per bulk lookup = %g\n" , + (double )lookup_time / + ((double )TEST_ITERATIONS * (double )TEST_SIZE )); + +rte_free (keys); +rte_free (lookup_keys); +rte_kv_hash_free (handle); + +return 0 ; +} +static int test_hash_perf (void ) {
@@ -746,6 +854,9 @@ test_hash_perf(void)if (fbk_hash_perf_test () <0 )return -1 ; +if (kv_hash_perf_test () <0 ) +return -1 ; +return 0 ; }
Signature · 1 line
2.7.4
Re: [dpdk-dev] [PATCH v3 0/4] add new k32v64 hash table
In reply to Wang, Yipeng1Hi Yipeng,
Sorry for late reply
Quoted 88 lines · On 17/04/2020 01:21, Wang, Yipeng1 wrote:
Quoted 74 lines
-----Original Message-----
From: Mattias Rönnblom<[email protected]>
Sent: Thursday, April 16, 2020 4:41 AM
To: Medvedkin, Vladimir<[email protected]>;[email protected]
Cc: Ananyev, Konstantin<[email protected]>; Wang, Yipeng1 <[email protected]>; Gobriel, Sameh<[email protected]>; Richardson, Bruce<[email protected]>
Subject: Re: [dpdk-dev] [PATCH v3 0/4] add new k32v64 hash tableQuoted 18 lines · Vladimir
Hi Mattias,
-----Original Message-----
From: Mattias Rönnblom<[email protected]>
Sent: Wednesday, April 15, 2020 7:52 PM
To: Medvedkin, Vladimir<[email protected]>;[email protected]
Cc: Ananyev, Konstantin<[email protected]>; Wang, Yipeng1 <[email protected]>; Gobriel, Sameh<[email protected]>; Richardson, Bruce<[email protected]>
Subject: Re: [dpdk-dev] [PATCH v3 0/4] add new k32v64 hash tableQuoted 6 lines · Vladimir Medvedkin
Currently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called K32V64 hash supports 4 byte keys and 8 byte associated values, which is enough to store a pointer.Quoted 3 lines
Quoted 2 lines
It would also be nice to get feedback on whether to leave the old FBK and new k32v64 implementations or deprecate the old one?
Do you think it would be feasible to support custom-sized values and remain
efficient, in a similar manner to how
rte_ring_elem.hdoes things?Quoted 1 line
I'm afraid it is not feasible. For the performance reason keys and
corresponding values resides in single cache line so there are no extra memory for bigger values, such as 16B.
Well, if you have a smaller value type (or key type) you would fit into something less-than-a-cache line, and thus reduce your memory working set further.
Quoted 18 lines
Quoted 18 lines
v3:
- added bulk lookup
- avx512 key comparizon is removed from .hv2:
- renamed fromrte_dwktorte_k32v64as was suggested
- reworked lookup function, added inlined subroutines
- added avx512 key comparizon routine
- added documentation
- added statistic counters for total entries and extended entries(linked list)Vladimir Medvedkin (4):
hash: add k32v64 hash library
hash: add documentation for k32v64 hash library
test: add k32v64 hash autotests
test: add k32v64 perf testsQuoted 14 lines
Quoted 6 lines
app/test/Makefile | 1 + app/test/autotest_data.py | 12 ++ app/test/meson.build | 3 + app/test/test_hash_perf.c | 130 ++++++++++++ app/test/test_k32v64_hash.c | 229 ++++++++++++++++++++++ doc/api/doxy-api-index.md | 1 +Quoted 8 lines
doc/guides/prog_guide/index.rst | 1 + doc/guides/prog_guide/k32v64_hash_lib.rst | 66 +++++++ lib/Makefile | 2 +- lib/librte_hash/Makefile | 13 +- lib/librte_hash/k32v64_hash_avx512vl.c | 56 ++++++ lib/librte_hash/meson.build | 17 +- lib/librte_hash/rte_hash_version.map | 6 +- lib/librte_hash/rte_k32v64_hash.c | 315Quoted 9 lines
++++++++++++++++++++++++++++++
Quoted 8 lines
Quoted 8 lines
lib/librte_hash/rte_k32v64_hash.h | 211 ++++++++++++++++++++ 15 files changed, 1058 insertions(+), 5 deletions(-) create mode 100644 app/test/test_k32v64_hash.c create mode 100644 doc/guides/prog_guide/k32v64_hash_lib.rst create mode 100644 lib/librte_hash/k32v64_hash_avx512vl.c create mode 100644 lib/librte_hash/rte_k32v64_hash.c create mode 100644 lib/librte_hash/rte_k32v64_hash.h[Wang, Yipeng]
Hi, Vladimir,
Thanks for responding with the use cases earlier. I discussed with Sameh offline, here are some comments.
Quoted 4 lines
1. Since the proposed hash table also has some similarities to
rte_tablelibrary used by packet framework, have you tried it yet? Although it is mainly for packet framework, I believe you can use it independently as well. It has implementations for special key value sizes.
I added Cristian for his comment.
I looked at rte_table_hash. I'm afraid it doesn't fit our requirements due to it's design.
First, it's API uses mbufs as a key container.
Second, as I can see from the source code it is not safe in multi threaded environment regarding read-write concurrency by design. Also there are some information about it https://doc.dpdk.org/guides/prog_guide/packet_framework.html#shared-data-structures
and Cristian's comment
http://mails.dpdk.org/archives/dev/2015-September/024121.html
Quoted 9 lines
2. We tend to agree with Mattias that it would be better if we have a more generic API name and with the same API we can do multiple key/value size implementations.
This is to avoid adding new APIs in future to again handle different key/value
use cases. For example, we call itrte_kv_hash, and through the parameter struct we pass in a key-value size pair we want to use.
Implementation-wise, we may only provide implementations for certain popular use cases (like the one you provided). For other general use cases, people should go with the more flexible and generic cuckoo hash. Then we should also merge the FBK under the new API.
Agree. As was discussed offline, I've made API to be more generic regarding to key and value sizes.
Signature · 2 lines
Regards, Vladimir
Re: [dpdk-dev] [PATCH v3 1/4] hash: add k32v64 hash library
In reply to Ananyev, KonstantinHi Konstantin,
Thanks for review,
Quoted 81 lines · Konstantin
Hi Vladimir,
Apologies for late review.
My comments below.Quoted 76 lines
K32V64 hash is a hash table that supports 32 bit keys and 64 bit values. This table is hash function agnostic so user must provide precalculated hash signature for add/delete/lookup operations.
Signed-off-by Vladimir Medvedkin <[email protected]>---
lib/librte_hash/rte_k32v64_hash.c new file +66
-0@@ -0,0 +1,315 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include <string.h> + +#include <rte_eal_memconfig.h> +#include <rte_errno.h> +#include <rte_malloc.h> +#include <rte_memory.h> +#include <rte_tailq.h> + +#include <rte_k32v64_hash.h> + +TAILQ_HEAD (rte_k32v64_hash_list, rte_tailq_entry); + +static struct rte_tailq_elem rte_k32v64_hash_tailq = { +.name ="RTE_K32V64_HASH" , +}; + +EAL_REGISTER_TAILQ (rte_k32v64_hash_tailq); + +#define VALID_KEY_MSK ((1 << RTE_K32V64_KEYS_PER_BUCKET) - 1) + +#ifdef CC_AVX512VL_SUPPORT +int +k32v64_hash_bulk_lookup_avx512vl (struct rte_k32v64_hash_table *table, +uint32_t *keys,uint32_t *hashes,uint64_t *values,unsigned int n); +#endif + +static int +k32v64_hash_bulk_lookup (struct rte_k32v64_hash_table *table,uint32_t *keys, +uint32_t *hashes,uint64_t *values,unsigned int n) +{ +int ret, cnt =0 ; +unsigned int i; + +if (unlikely ((table ==NULL ) || (keys ==NULL ) || (hashes ==NULL ) || +(values ==NULL ))) +return -EINVAL ; + +for (i =0 ; i < n; i++) { +ret =rte_k32v64_hash_lookup (table, keys[i], hashes[i], +&values[i]); +if (ret ==0 ) +cnt++; +} +return cnt; +} + +static rte_k32v64_hash_bulk_lookup_t +get_lookup_bulk_fn (void ) +{ +#ifdef CC_AVX512VL_SUPPORT +if (rte_cpu_get_flag_enabled (RTE_CPUFLAG_AVX512F )) +return k32v64_hash_bulk_lookup_avx512vl; +#endif +return k32v64_hash_bulk_lookup; +} + +int +rte_k32v64_hash_add (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t value) +{ +uint32_t bucket; +int i, idx, ret;
Quoted 8 lines
lib/librte_hash/rte_k32v64_hash.c line 72
Quoted 6 lines
lib/librte_hash/rte_k32v64_hash.c new file +6
-0+uint8_t msk; +struct rte_k32v64_ext_ent *tmp, *ent, *prev =NULL ; + +if (table ==NULL ) +return -EINVAL ; +I think for add you also need to do update bucket.cnt at the start/end of updates (as you do for del).
Agree. We can not guarantee atomic update for 64bit value on 32bit arch.
But I think it is better to make transaction as small as possible so I update bucket.cnt not on start/end but right before and after key/value rewrite.
Quoted 69 lines
Quoted 69 lines
lib/librte_hash/rte_k32v64_hash.c new file +68
-0+bucket = hash & table->bucket_msk; +/* Search key in table. Update value if exists */ +for (i =0 ; i <RTE_K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && +(table->t[bucket].key_mask & (1 << i))) { +table->t[bucket].val[i] = value; +return 0 ; +} +} + +if (!SLIST_EMPTY (&table->t[bucket].head)) { +SLIST_FOREACH(ent, &table->t[bucket].head, next) { +if (ent->key == key) { +ent->val = value; +return 0 ; +} +} +} + +msk = ~table->t[bucket].key_mask &VALID_KEY_MSK ; +if (msk) { +idx =__builtin_ctz (msk); +table->t[bucket].key[idx] = key; +table->t[bucket].val[idx] = value; +rte_smp_wmb (); +table->t[bucket].key_mask |=1 << idx; +table->nb_ent++; +return 0 ; +} + +ret =rte_mempool_get (table->ext_ent_pool, (void **)&ent); +if (ret <0 ) +return ret; + +SLIST_NEXT (ent, next) =NULL ; +ent->key = key; +ent->val = value; +rte_smp_wmb (); +SLIST_FOREACH (tmp, &table->t[bucket].head, next) +prev = tmp; + +if (prev ==NULL ) +SLIST_INSERT_HEAD (&table->t[bucket].head, ent, next); +else +SLIST_INSERT_AFTER (prev, ent, next); + +table->nb_ent++; +table->nb_ext_ent++; +return 0 ; +} + +int +rte_k32v64_hash_delete (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash) +{ +uint32_t bucket; +int i; +struct rte_k32v64_ext_ent *ent; + +if (table ==NULL ) +return -EINVAL ; + +bucket = hash & table->bucket_msk; + +for (i =0 ; i <RTE_K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && +(table->t[bucket].key_mask & (1 << i))) { +ent =SLIST_FIRST (&table->t[bucket].head);
Quoted 8 lines
lib/librte_hash/rte_k32v64_hash.c line 142
Quoted 2 lines
lib/librte_hash/rte_k32v64_hash.c new file +2
-0+if (ent) { +rte_atomic32_inc (&table->t[bucket].cnt);I know that right now
rte_atomic32uses_syncgcc builtins underneath, so it should be safe.
But I think the proper way would be:
table->t[bucket].cnt++;rte_smp_wmb();
or as alternative probably use C11 atomic ACQUIRE/RELEASE
Agree.
Quoted 11 lines
lib/librte_hash/rte_k32v64_hash.c line 149
Quoted 8 lines
lib/librte_hash/rte_k32v64_hash.c new file +7
-0+table->t[bucket].key[i] = ent->key; +table->t[bucket].val[i] = ent->val; +SLIST_REMOVE_HEAD (&table->t[bucket].head, next); +rte_atomic32_inc (&table->t[bucket].cnt); +table->nb_ext_ent--; +}else +table->t[bucket].key_mask &= ~(1 << i);I think you protect that update with bucket.cnt.
From my perspective -a s a rule of thumb any update to the bucket/list
Should be within that transaction-start/transaction-end.
I think it is possible to update key_mask with C11 atomic
Quoted 280 lines
Quoted 266 lines
lib/librte_hash/rte_k32v64_hash.c new file +166
-0+if (ent) +rte_mempool_put (table->ext_ent_pool, ent); +table->nb_ent--; +return 0 ; +} +} + +SLIST_FOREACH(ent, &table->t[bucket].head, next) +if (ent->key == key) +break ; + +if (ent ==NULL ) +return -ENOENT ; + +rte_atomic32_inc (&table->t[bucket].cnt); +SLIST_REMOVE (&table->t[bucket].head, ent, rte_k32v64_ext_ent, next); +rte_atomic32_inc (&table->t[bucket].cnt); +rte_mempool_put (table->ext_ent_pool, ent); + +table->nb_ext_ent--; +table->nb_ent--; + +return 0 ; +} + +struct rte_k32v64_hash_table * +rte_k32v64_hash_find_existing (const char *name) +{ +struct rte_k32v64_hash_table *h =NULL ; +struct rte_tailq_entry *te; +struct rte_k32v64_hash_list *k32v64_hash_list; + +k32v64_hash_list =RTE_TAILQ_CAST (rte_k32v64_hash_tailq.head, +rte_k32v64_hash_list); + +rte_mcfg_tailq_read_lock (); +TAILQ_FOREACH(te, k32v64_hash_list, next) { +h = (struct rte_k32v64_hash_table *) te->data; +if (strncmp (name, h->name,RTE_K32V64_HASH_NAMESIZE ) ==0 ) +break ; +} +rte_mcfg_tailq_read_unlock (); +if (te ==NULL ) { +rte_errno =ENOENT ; +return NULL ; +} +return h; +} + +struct rte_k32v64_hash_table * +rte_k32v64_hash_create (const struct rte_k32v64_hash_params *params) +{ +char hash_name[RTE_K32V64_HASH_NAMESIZE ]; +struct rte_k32v64_hash_table *ht =NULL ; +struct rte_tailq_entry *te; +struct rte_k32v64_hash_list *k32v64_hash_list; +uint32_t mem_size, nb_buckets, max_ent; +int ret; +struct rte_mempool *mp; + +if ((params ==NULL ) || (params->name ==NULL ) || +(params->entries ==0 )) { +rte_errno =EINVAL ; +return NULL ; +} + +k32v64_hash_list =RTE_TAILQ_CAST (rte_k32v64_hash_tailq.head, +rte_k32v64_hash_list); + +ret =snprintf (hash_name,sizeof (hash_name),"K32V64_%s" , params->name); +if (ret <0 || ret >=RTE_K32V64_HASH_NAMESIZE ) { +rte_errno =ENAMETOOLONG ; +return NULL ; +} + +max_ent =rte_align32pow2 (params->entries); +nb_buckets = max_ent /RTE_K32V64_KEYS_PER_BUCKET ; +mem_size =sizeof (struct rte_k32v64_hash_table ) + +sizeof (struct rte_k32v64_hash_bucket ) * nb_buckets; + +mp =rte_mempool_create (hash_name, max_ent, +sizeof (struct rte_k32v64_ext_ent ),0 ,0 ,NULL ,NULL ,NULL ,NULL , +params->socket_id,0 ); + +if (mp ==NULL ) +return NULL ; + +rte_mcfg_tailq_write_lock (); +TAILQ_FOREACH(te, k32v64_hash_list, next) { +ht = (struct rte_k32v64_hash_table *) te->data; +if (strncmp (params->name, ht->name, +RTE_K32V64_HASH_NAMESIZE ) ==0 ) +break ; +} +ht =NULL ; +if (te !=NULL ) { +rte_errno =EEXIST ; +rte_mempool_free (mp); +goto exit; +} + +te =rte_zmalloc ("K32V64_HASH_TAILQ_ENTRY" ,sizeof (*te),0 ); +if (te ==NULL ) { +RTE_LOG (ERR ,HASH ,"Failed to allocate tailq entry\n" ); +rte_mempool_free (mp); +goto exit; +} + +ht =rte_zmalloc_socket (hash_name, mem_size, +RTE_CACHE_LINE_SIZE , params->socket_id); +if (ht ==NULL ) { +RTE_LOG (ERR ,HASH ,"Failed to allocate fbk hash table\n" ); +rte_free (te); +rte_mempool_free (mp); +goto exit; +} + +memcpy (ht->name, hash_name,sizeof (ht->name)); +ht->max_ent = max_ent; +ht->bucket_msk = nb_buckets -1 ; +ht->ext_ent_pool = mp; +ht->lookup =get_lookup_bulk_fn (); + +te->data = (void *)ht; +TAILQ_INSERT_TAIL (k32v64_hash_list, te, next); + +exit: +rte_mcfg_tailq_write_unlock (); + +return ht; +} + +void +rte_k32v64_hash_free (struct rte_k32v64_hash_table *ht) +{ +struct rte_tailq_entry *te; +struct rte_k32v64_hash_list *k32v64_hash_list; + +if (ht ==NULL ) +return ; + +k32v64_hash_list =RTE_TAILQ_CAST (rte_k32v64_hash_tailq.head, +rte_k32v64_hash_list); + +rte_mcfg_tailq_write_lock (); + +/* find out tailq entry */ +TAILQ_FOREACH(te, k32v64_hash_list, next) { +if (te->data == (void *) ht) +break ; +} + + +if (te ==NULL ) { +rte_mcfg_tailq_write_unlock (); +return ; +} + +TAILQ_REMOVE (k32v64_hash_list, te, next); + +rte_mcfg_tailq_write_unlock (); + +rte_mempool_free (ht->ext_ent_pool); +rte_free (ht); +rte_free (te); +}lib/librte_hash/rte_k32v64_hash.h new file +94
-0@@ -0,0 +1,211 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#ifndef _RTE_K32V64_HASH_H_ +#define _RTE_K32V64_HASH_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <rte_compat.h> +#include <rte_atomic.h> +#include <rte_mempool.h> + +#define RTE_K32V64_HASH_NAMESIZE32 +#define RTE_K32V64_KEYS_PER_BUCKET4 +#define RTE_K32V64_WRITE_IN_PROGRESS1 + +struct rte_k32v64_hash_params { +const char *name; +uint32_t entries; +int socket_id; +}; + +struct rte_k32v64_ext_ent { +SLIST_ENTRY(rte_k32v64_ext_ent) next; +uint32_tkey; +uint64_tval; +}; + +struct rte_k32v64_hash_bucket { +uint32_tkey[RTE_K32V64_KEYS_PER_BUCKET]; +uint64_tval[RTE_K32V64_KEYS_PER_BUCKET]; +uint8_tkey_mask; +rte_atomic32_tcnt; +SLIST_HEAD(rte_k32v64_list_head, rte_k32v64_ext_ent) head; +} __rte_cache_aligned; + +struct rte_k32v64_hash_table; + +typedef int (*rte_k32v64_hash_bulk_lookup_t) +(struct rte_k32v64_hash_table *table, uint32_t *keys, uint32_t *hashes, +uint64_t *values, unsigned int n); + +struct rte_k32v64_hash_table { +char name[RTE_K32V64_HASH_NAMESIZE];/**< Name of the hash. */ +uint32_tnb_ent;/**< Number of entities in the table*/ +uint32_tnb_ext_ent;/**< Number of extended entities */ +uint32_tmax_ent;/**< Maximum number of entities */ +uint32_tbucket_msk; +struct rte_mempool*ext_ent_pool; +rte_k32v64_hash_bulk_lookup_tlookup; +__extension__ struct rte_k32v64_hash_buckett[0]; +}; + +typedef int (*rte_k32v64_cmp_fn_t) +(struct rte_k32v64_hash_bucket *bucket, uint32_t key, uint64_t *val); + +static inline int +__k32v64_cmp_keys(struct rte_k32v64_hash_bucket *bucket, uint32_t key, +uint64_t *val) +{ +int i; + +for (i = 0; i < RTE_K32V64_KEYS_PER_BUCKET; i++) { +if ((key == bucket->key[i]) && +(bucket->key_mask & (1 << i))) { +*val = bucket->val[i]; +return 1; +} +} + +return 0; +} + +static inline int +__k32v64_hash_lookup(struct rte_k32v64_hash_table *table, uint32_t key, +uint32_t hash, uint64_t *value, rte_k32v64_cmp_fn_t cmp_f) +{ +uint64_tval = 0; +struct rte_k32v64_ext_ent *ent; +int32_tcnt; +int found = 0; +uint32_t bucket = hash & table->bucket_msk; + +do { +do +cnt = rte_atomic32_read(&table->t[bucket].cnt); +while (unlikely(cnt & RTE_K32V64_WRITE_IN_PROGRESS)); + +found = cmp_f(&table->t[bucket], key, &val); +if (unlikely((found == 0) && +(!SLIST_EMPTY(&table->t[bucket].head)))) {lib/librte_hash/rte_k32v64_hash.h line 104
Quoted 10 lines
lib/librte_hash/rte_k32v64_hash.h new file +10
-0+SLIST_FOREACH(ent, &table->t[bucket].head, next) { +if (ent->key == key) { +val = ent->val; +found =1 ; +break ; +} +} +} + +}while (unlikely (cnt !=rte_atomic32_read (&table->t[bucket].cnt)));AFAIK
atomic32_readis just a normal read op, so it can be reordered with other ops. So this construction doesn't protect you from races.
What you probably need here:
Quoted 7 lines
do {
cnt1 = table->t[bucket].cnt;rte_smp_rmb();
....rte_smp_rmb();
cnt2 = table->t[bucket].cnt;
while (cnt1 != cnt2 || (cnt1 &RTE_K32V64_WRITE_IN_PROGRESS) != 0)
Agree, this reads could be reordered. Replace it with C11 atomics.
Quoted 9 lines
Quoted 9 lines
lib/librte_hash/rte_k32v64_hash.h new file +8
-0+ +if (found ==1 ) { +*value = val; +return 0 ; +}else +return -ENOENT ; +} +
Signature · 2 lines
Regards, Vladimir
Re: [dpdk-dev] [PATCH v3 1/4] hash: add k32v64 hash library
In reply to Honnappa NagarahalliHi Honnappa,
Thanks for the comments.
Quoted 266 lines · Honnappa Nagarahalli
Hi Vladimir,
I am not sure which way the decision is made, but few comments inline. Please use C11 built-ins for synchronization.Quoted 263 lines
-----Original Message-----
From: dev<[email protected]> On Behalf Of Vladimir Medvedkin
Sent: Wednesday, April 15, 2020 1:17 PM To:[email protected] Cc:[email protected];[email protected]; [email protected];[email protected]
Subject: [dpdk-dev] [PATCH v3 1/4] hash: add k32v64 hash libraryK32V64 hash is a hash table that supports 32 bit keys and 64 bit values. This table is hash function agnostic so user must provide precalculated hash signature for add/delete/lookup operations.
Signed-off-by Vladimir Medvedkin<[email protected]>--- lib/Makefile | 2 +- lib/librte_hash/Makefile | 13 +- lib/librte_hash/k32v64_hash_avx512vl.c | 56 ++++++ lib/librte_hash/meson.build | 17 +- lib/librte_hash/rte_hash_version.map | 6 +- lib/librte_hash/rte_k32v64_hash.c | 315 +++++++++++++++++++++++++++++++++ lib/librte_hash/rte_k32v64_hash.h | 211 ++++++++++++++++++++++ 7 files changed, 615 insertions(+), 5 deletions(-) create mode 100644
lib/librte_hash/k32v64_hash_avx512vl.c
create mode 100644lib/librte_hash/rte_k32v64_hash.ccreate mode
100644lib/librte_hash/rte_k32v64_hash.hlib/Makefile +0
-0@@ -48,7 +48,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_VHOST) += librte_vhostDEPDIRS-librte_vhost :=
librte_eallibrte_mempoollibrte_mbuflibrte_ethdev\
librte_netlibrte_hashlibrte_cryptodev
DIRS-$(CONFIG_RTE_LIBRTE_HASH) +=librte_hash-DEPDIRS-librte_hash :=librte_eallibrte_ring
+DEPDIRS-librte_hash :=librte_eallibrte_ringlibrte_mempool
DIRS-$(CONFIG_RTE_LIBRTE_EFD) +=librte_efdDEPDIRS-librte_efd :=librte_eallibrte_ringlibrte_hash
DIRS-$(CONFIG_RTE_LIBRTE_RIB) +=librte_ribdiff --git
a/lib/librte_hash/Makefile b/lib/librte_hash/Makefile index ec9f864..023689d 100644lib/librte_hash/Makefile +3
-1@@ -8,13 +8,14 @@ LIB = librte_hash.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_ring +LDLIBS += -lrte_eal -lrte_ring -lrte_mempool EXPORT_MAP := rte_hash_version.map # all source are stored in SRCS-y SRCS-$(CONFIG_RTE_LIBRTE_HASH) := rte_cuckoo_hash.c SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_fbk_hash.c +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_k32v64_hash.c # install this header file SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include := rte_hash.h @@ -27,5 +28,15 @@ endif SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include +=
rte_jhash.hSYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include +=rte_thash.hSYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include +=rte_fbk_hash.hlib/librte_hash/Makefile +3
-0+SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_k32v64_hash.h + +CC_AVX512VL_SUPPORT=$(shell $(CC) -mavx512vl -dM -E - </dev/null 2>&1|
+\ grep -q__AVX512VL__&& echo 1)lib/librte_hash/Makefile +5
-0+ +ifeq ($(CC_AVX512VL_SUPPORT), 1) + SRCS-$(CONFIG_RTE_LIBRTE_HASH) += k32v64_hash_avx512vl.c + CFLAGS_k32v64_hash_avx512vl.o += -mavx512vl + CFLAGS_rte_k32v64_hash.o += -DCC_AVX512VL_SUPPORT endif include $(RTE_SDK)/mk/rte.lib.mklib/librte_hash/k32v64_hash_avx512vl.c new file +12
-0@@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include <rte_k32v64_hash.h> + +int +k32v64_hash_bulk_lookup_avx512vl (struct rte_k32v64_hash_table *table, +uint32_t *keys,uint32_t *hashes,uint64_t *values,unsigned int n); + +static inline int +k32v64_cmp_keys_avx512vl(struct rte_k32v64_hash_bucket *bucket,
uint32_tkey,lib/librte_hash/k32v64_hash_avx512vl.c new file +9
-0+ uint64_t *val) +{ +__m128i keys, srch_key; +__mmask8 msk; + + keys =_mm_load_si128 ((void *)bucket); + srch_key =_mm_set1_epi32 (key); + + msk = _mm_mask_cmpeq_epi32_mask(bucket->key_mask, keys,
srch_key);lib/librte_hash/k32v64_hash_avx512vl.c new file +10
-0+if (msk) { + *val = bucket->val[__builtin_ctz (msk)]; +return 1 ; + } + +return 0 ; +} + +static inline int +k32v64_hash_lookup_avx512vl(struct rte_k32v64_hash_table *table,
uint32_tkey,lib/librte_hash/k32v64_hash_avx512vl.c new file +24
-0+ uint32_t hash, uint64_t *value) +{ + return __k32v64_hash_lookup(table, key, hash, value, +k32v64_cmp_keys_avx512vl ); +} + +int +k32v64_hash_bulk_lookup_avx512vl (struct rte_k32v64_hash_table *table, +uint32_t *keys,uint32_t *hashes,uint64_t *values,unsigned int n) { +int ret, cnt =0 ; +unsigned int i; + +if (unlikely ((table ==NULL ) || (keys ==NULL ) || (hashes ==NULL ) || + (values ==NULL ))) +return -EINVAL ; + +for (i =0 ; i < n; i++) { + ret =k32v64_hash_lookup_avx512vl (table, keys[i], hashes[i], + &values[i]); +if (ret ==0 ) + cnt++; + } +return cnt; +}lib/librte_hash/meson.build index renamed from lib/librte_hash/meson.build +0
-06ab46ae..8a37cc4 100644
lib/librte_hash/meson.build +15
-1@@ -3,10 +3,23 @@ headers = files('rte_crc_arm64.h', 'rte_fbk_hash.h', + 'rte_k32v64_hash.h', 'rte_hash_crc.h', 'rte_hash.h', 'rte_jhash.h', 'rte_thash.h') -sources = files('rte_cuckoo_hash.c', 'rte_fbk_hash.c') -deps += ['ring'] +sources = files('rte_cuckoo_hash.c', 'rte_fbk_hash.c', +'rte_k32v64_hash.c') deps += ['ring', 'mempool'] + +if dpdk_conf.has('RTE_ARCH_X86') + if cc.has_argument('-mavx512vl') + avx512_tmplib = static_library('avx512_tmp', + 'k32v64_hash_avx512vl.c', + dependencies: static_rte_mempool, + c_args: cflags + ['-mavx512vl']) + objs += avx512_tmplib.extract_objects('k32v64_hash_avx512vl.c') + cflags += '-DCC_AVX512VL_SUPPORT' + + endif +endiflib/librte_hash/rte_hash_version.map +5
-1@@ -34,5 +34,9 @@ EXPERIMENTAL { rte_hash_free_key_with_position; rte_hash_max_key_id; - + rte_k32v64_hash_create; + rte_k32v64_hash_find_existing; + rte_k32v64_hash_free; + rte_k32v64_hash_add; + rte_k32v64_hash_delete; };lib/librte_hash/rte_k32v64_hash.c new file +32
-0@@ -0,0 +1,315 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include <string.h> + +#include <rte_eal_memconfig.h> +#include <rte_errno.h> +#include <rte_malloc.h> +#include <rte_memory.h> +#include <rte_tailq.h> + +#include <rte_k32v64_hash.h> + +TAILQ_HEAD (rte_k32v64_hash_list, rte_tailq_entry); + +static struct rte_tailq_elem rte_k32v64_hash_tailq = { + .name ="RTE_K32V64_HASH" , +}; + +EAL_REGISTER_TAILQ (rte_k32v64_hash_tailq); + +#define VALID_KEY_MSK ((1 << RTE_K32V64_KEYS_PER_BUCKET) - 1) + +#ifdef CC_AVX512VL_SUPPORT +int +k32v64_hash_bulk_lookup_avx512vl (struct rte_k32v64_hash_table *table, +uint32_t *keys,uint32_t *hashes,uint64_t *values,unsigned int n); +#endif + +static int +k32v64_hash_bulk_lookup(struct rte_k32v64_hash_table *table, uint32_t*keys,
lib/librte_hash/rte_k32v64_hash.c new file +38
-0+ uint32_t *hashes, uint64_t *values, unsigned int n) { + int ret, cnt = 0; +unsigned int i; + +if (unlikely ((table ==NULL ) || (keys ==NULL ) || (hashes ==NULL ) || + (values ==NULL ))) +return -EINVAL ; + +for (i =0 ; i < n; i++) { + ret =rte_k32v64_hash_lookup (table, keys[i], hashes[i], + &values[i]); +if (ret ==0 ) + cnt++; + } +return cnt; +} + +static rte_k32v64_hash_bulk_lookup_t +get_lookup_bulk_fn (void ) +{ +#ifdef CC_AVX512VL_SUPPORT +if (rte_cpu_get_flag_enabled (RTE_CPUFLAG_AVX512F )) +return k32v64_hash_bulk_lookup_avx512vl;#endif +return k32v64_hash_bulk_lookup; +} + +int +rte_k32v64_hash_add (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t value) +{ +uint32_t bucket; +int i, idx, ret; +uint8_t msk; +struct rte_k32v64_ext_ent *tmp, *ent, *prev =NULL ; + +if (table ==NULL ) +return -EINVAL ; +
Quoted 8 lines
lib/librte_hash/rte_k32v64_hash.c line 78
Quoted 6 lines
lib/librte_hash/rte_k32v64_hash.c new file +6
-0+ bucket = hash & table->bucket_msk; +/* Search key in table. Update value if exists */ +for (i =0 ; i <RTE_K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && + (table->t[bucket].key_mask & (1 << i))) { + table->t[bucket].val[i] = value;The old value of val[i] might be getting used in the reader. It needs to be returned to the caller so that it can be put on the RCU defer queue to free later. You might also want to use an atomic store to ensure the stores are not split.
Agree, I will change API.
Quoted 10 lines
lib/librte_hash/rte_k32v64_hash.c line 86
Quoted 8 lines
lib/librte_hash/rte_k32v64_hash.c new file +8
-0+return 0 ; + } + } + +if (!SLIST_EMPTY (&table->t[bucket].head)) { + SLIST_FOREACH(ent, &table->t[bucket].head, next) { +if (ent->key == key) { + ent->val = value;Same here, need to return the old value.
Quoted 13 lines
lib/librte_hash/rte_k32v64_hash.c line 98
Quoted 12 lines
lib/librte_hash/rte_k32v64_hash.c new file +12
-0+return 0 ; + } + } + } + + msk = ~table->t[bucket].key_mask &VALID_KEY_MSK ; +if (msk) { + idx =__builtin_ctz (msk); + table->t[bucket].key[idx] = key; + table->t[bucket].val[idx] = value; +rte_smp_wmb (); + table->t[bucket].key_mask |=1 << idx;The barrier and the store can be replaced with a store-release in C11.
Agree
Quoted 13 lines
Quoted 13 lines
lib/librte_hash/rte_k32v64_hash.c new file +12
-0+ table->nb_ent++; +return 0 ; + } + + ret =rte_mempool_get (table->ext_ent_pool, (void **)&ent); +if (ret <0 ) +return ret; + +SLIST_NEXT (ent, next) =NULL ; + ent->key = key; + ent->val = value; +rte_smp_wmb ();
Quoted 8 lines
lib/librte_hash/rte_k32v64_hash.c line 117
Quoted 7 lines
lib/librte_hash/rte_k32v64_hash.c new file +7
-0+SLIST_FOREACH (tmp, &table->t[bucket].head, next) + prev = tmp; + +if (prev ==NULL ) +SLIST_INSERT_HEAD (&table->t[bucket].head, ent, next); +else +SLIST_INSERT_AFTER (prev, ent, next);Need C11 SLIST
Please clarify
Quoted 10 lines
lib/librte_hash/rte_k32v64_hash.c line 126
Quoted 9 lines
lib/librte_hash/rte_k32v64_hash.c new file +9
-0+ + table->nb_ent++; + table->nb_ext_ent++; +return 0 ; +} + +int +rte_k32v64_hash_delete (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash)This should return the value corresponding to the deleted key
Agree.
Quoted 20 lines
Quoted 8 lines
lib/librte_hash/rte_k32v64_hash.c new file +7
-0+{ +uint32_t bucket; +int i; +struct rte_k32v64_ext_ent *ent; + +if (table ==NULL ) +return -EINVAL ;lib/librte_hash/rte_k32v64_hash.c line 143
Quoted 10 lines
lib/librte_hash/rte_k32v64_hash.c new file +10
-0+ + bucket = hash & table->bucket_msk; + +for (i =0 ; i <RTE_K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && + (table->t[bucket].key_mask & (1 << i))) { + ent =SLIST_FIRST (&table->t[bucket].head); +if (ent) { +rte_atomic32_inc (&table->t[bucket].cnt); + table->t[bucket].key[i] = ent->key;In this case, both key and value are changing and it is not atomic. There is a possibility that the lookup function will receive incorrect value of 'val[i]'. Suggest following the method described below.
Quoted 7 lines
lib/librte_hash/rte_k32v64_hash.c line 148
Quoted 6 lines
lib/librte_hash/rte_k32v64_hash.c new file +2
-0+ table->t[bucket].val[i] = ent->val; + SLIST_REMOVE_HEAD(&table->t[bucket].head,next);
lib/librte_hash/rte_k32v64_hash.c new file +3
-0+rte_atomic32_inc (&table->t[bucket].cnt); + table->nb_ext_ent--; + }else Suggest changing this into a 2 step process:
Thanks, it is a good idea, but
Quoted 3 lines
1) Delete the entry from the fixed bucket (update
key_mask)
2) Move the head of extended bucket to the fixed bucket
2a) Insert the key/value from the head into the deleted index and update thekey_mask(this will ensure that the reader has the entry available while it is being removed from the extended bucket)
With reader logic you suggested for __k32v64_cmp_keys() there could be possible race:
reader writer
read bucket.cntload_acquire -> key_mask
if (key == key[i]) <- true
----interrupt----------
1) Delete the entry from the fixed bucket (update key_mask) 2a) --------interrupt-----
---wake------
get updated wrong value
cmp bucket.cnt
return with success but with invalid value
Please correct me if I'm wrong.
Quoted 5 lines
2b) increment the counter indicating the extended bucket is changing 2c) remove the head
2d) increment the counter indicating the extended bucket change is doneThe above procedure will result in removing the spinning (resulting from step 2b) in the lookup function (comment is added in the lookup function), readers will not be blocked if the writer is scheduled out.
As far as I can see we must somehow indicate write_in_progress state for a bucket, at least because we can not guarantee atomic write for uint64_t on 32bit arch.
Quoted 6 lines
This logic is implemented in
rte_hashusing cuckoo hash, suggest to take a look for required memory orderings.lib/librte_hash/rte_k32v64_hash.c line 151
Quoted 3 lines
lib/librte_hash/rte_k32v64_hash.c new file +3
-0+ table->t[bucket].key_mask &= ~(1 << i); +if (ent) +rte_mempool_put (table->ext_ent_pool, ent);Entry cannot be put to free list immediately as the readers are still using it.
With current transaction logic if reader is using ent while writer puts it in a pool I don't see any problem, because reader will detect bucket.cnt update and rerun search again.
Quoted 269 lines
Quoted 16 lines
lib/librte_hash/rte_k32v64_hash.c new file +14
-0+ table->nb_ent--; +return 0 ; + } + } + + SLIST_FOREACH(ent, &table->t[bucket].head, next) +if (ent->key == key) +break ; + +if (ent ==NULL ) +return -ENOENT ; + +rte_atomic32_inc (&table->t[bucket].cnt); + SLIST_REMOVE(&table->t[bucket].head, ent, rte_k32v64_ext_ent,next);
lib/librte_hash/rte_k32v64_hash.c
Quoted 2 lines
lib/librte_hash/rte_k32v64_hash.c new file +2
-0+rte_atomic32_inc (&table->t[bucket].cnt); +rte_mempool_put (table->ext_ent_pool, ent);Entry might be getting used by the readers still.
Quoted 215 lines
lib/librte_hash/rte_k32v64_hash.c new file +20
-0+ + table->nb_ext_ent--; + table->nb_ent--; + +return 0 ; +} + +struct rte_k32v64_hash_table * +rte_k32v64_hash_find_existing (const char *name) { +struct rte_k32v64_hash_table *h =NULL ; +struct rte_tailq_entry *te; +struct rte_k32v64_hash_list *k32v64_hash_list; + + k32v64_hash_list =RTE_TAILQ_CAST (rte_k32v64_hash_tailq.head, + rte_k32v64_hash_list); + +rte_mcfg_tailq_read_lock (); + TAILQ_FOREACH(te, k32v64_hash_list, next) { + h = (struct rte_k32v64_hash_table *) te->data; + if (strncmp(name, h->name, RTE_K32V64_HASH_NAMESIZE)== 0)
lib/librte_hash/rte_k32v64_hash.c new file +30
-0+break ; + } +rte_mcfg_tailq_read_unlock (); +if (te ==NULL ) { + rte_errno =ENOENT ; +return NULL ; + } +return h; +} + +struct rte_k32v64_hash_table * +rte_k32v64_hash_create (const struct rte_k32v64_hash_params *params) { +char hash_name[RTE_K32V64_HASH_NAMESIZE ]; +struct rte_k32v64_hash_table *ht =NULL ; +struct rte_tailq_entry *te; +struct rte_k32v64_hash_list *k32v64_hash_list; +uint32_t mem_size, nb_buckets, max_ent; +int ret; +struct rte_mempool *mp; + +if ((params ==NULL ) || (params->name ==NULL ) || + (params->entries ==0 )) { + rte_errno =EINVAL ; +return NULL ; + } + + k32v64_hash_list =RTE_TAILQ_CAST (rte_k32v64_hash_tailq.head, + rte_k32v64_hash_list); + + ret = snprintf(hash_name, sizeof(hash_name), "K32V64_%s", params-Quoted 1 line
name);
lib/librte_hash/rte_k32v64_hash.c new file +12
-0+if (ret <0 || ret >=RTE_K32V64_HASH_NAMESIZE ) { + rte_errno =ENAMETOOLONG ; +return NULL ; + } + + max_ent =rte_align32pow2 (params->entries); + nb_buckets = max_ent /RTE_K32V64_KEYS_PER_BUCKET ; + mem_size =sizeof (struct rte_k32v64_hash_table ) + +sizeof (struct rte_k32v64_hash_bucket ) * nb_buckets; + + mp = rte_mempool_create(hash_name, max_ent, + sizeof(struct rte_k32v64_ext_ent), 0, 0, NULL, NULL, NULL,NULL,
lib/librte_hash/rte_k32v64_hash.c new file +83
-0+ params->socket_id, 0); + +if (mp ==NULL ) +return NULL ; + +rte_mcfg_tailq_write_lock (); + TAILQ_FOREACH(te, k32v64_hash_list, next) { + ht = (struct rte_k32v64_hash_table *) te->data; +if (strncmp (params->name, ht->name, +RTE_K32V64_HASH_NAMESIZE ) ==0 ) +break ; + } + ht =NULL ; +if (te !=NULL ) { + rte_errno =EEXIST ; +rte_mempool_free (mp); + goto exit; + } + + te =rte_zmalloc ("K32V64_HASH_TAILQ_ENTRY" ,sizeof (*te),0 ); +if (te ==NULL ) { +RTE_LOG (ERR ,HASH ,"Failed to allocate tailq entry\n" ); +rte_mempool_free (mp); + goto exit; + } + + ht =rte_zmalloc_socket (hash_name, mem_size, +RTE_CACHE_LINE_SIZE , params->socket_id); +if (ht ==NULL ) { +RTE_LOG (ERR ,HASH ,"Failed to allocate fbk hash table\n" ); +rte_free (te); +rte_mempool_free (mp); + goto exit; + } + +memcpy (ht->name, hash_name,sizeof (ht->name)); + ht->max_ent = max_ent; + ht->bucket_msk = nb_buckets -1 ; + ht->ext_ent_pool = mp; + ht->lookup =get_lookup_bulk_fn (); + + te->data = (void *)ht; +TAILQ_INSERT_TAIL (k32v64_hash_list, te, next); + +exit: +rte_mcfg_tailq_write_unlock (); + +return ht; +} + +void +rte_k32v64_hash_free (struct rte_k32v64_hash_table *ht) { +struct rte_tailq_entry *te; +struct rte_k32v64_hash_list *k32v64_hash_list; + +if (ht ==NULL ) +return ; + + k32v64_hash_list =RTE_TAILQ_CAST (rte_k32v64_hash_tailq.head, + rte_k32v64_hash_list); + +rte_mcfg_tailq_write_lock (); + +/* find out tailq entry */ + TAILQ_FOREACH(te, k32v64_hash_list, next) { +if (te->data == (void *) ht) +break ; + } + + +if (te ==NULL ) { +rte_mcfg_tailq_write_unlock (); +return ; + } + +TAILQ_REMOVE (k32v64_hash_list, te, next); + +rte_mcfg_tailq_write_unlock (); + +rte_mempool_free (ht->ext_ent_pool); +rte_free (ht); +rte_free (te); +}lib/librte_hash/rte_k32v64_hash.h new file +47
-0@@ -0,0 +1,211 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#ifndef _RTE_K32V64_HASH_H_ +#define _RTE_K32V64_HASH_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <rte_compat.h> +#include <rte_atomic.h> +#include <rte_mempool.h> + +#define RTE_K32V64_HASH_NAMESIZE 32 +#define RTE_K32V64_KEYS_PER_BUCKET 4 +#define RTE_K32V64_WRITE_IN_PROGRESS 1 + +struct rte_k32v64_hash_params { + const char *name; + uint32_t entries; + int socket_id; +}; + +struct rte_k32v64_ext_ent { + SLIST_ENTRY(rte_k32v64_ext_ent) next; + uint32_t key; + uint64_t val; +}; + +struct rte_k32v64_hash_bucket { + uint32_t key[RTE_K32V64_KEYS_PER_BUCKET]; + uint64_t val[RTE_K32V64_KEYS_PER_BUCKET]; + uint8_t key_mask; + rte_atomic32_t cnt; + SLIST_HEAD(rte_k32v64_list_head, rte_k32v64_ext_ent) head; } +__rte_cache_aligned; + +struct rte_k32v64_hash_table; + +typedef int (*rte_k32v64_hash_bulk_lookup_t) (struct +rte_k32v64_hash_table *table, uint32_t *keys, uint32_t *hashes, + uint64_t *values, unsigned int n); + +struct rte_k32v64_hash_table { + char name[RTE_K32V64_HASH_NAMESIZE]; /**< Name of thehash. */
lib/librte_hash/rte_k32v64_hash.h new file +12
-0+uint32_t nb_ent;/**< Number of entities in the table*/ +uint32_t nb_ext_ent;/**< Number of extended entities */ +uint32_t max_ent;/**< Maximum number of entities */ +uint32_t bucket_msk; +struct rte_mempool *ext_ent_pool; +rte_k32v64_hash_bulk_lookup_t lookup; + __extension__struct rte_k32v64_hash_bucket t[0 ]; +}; + +typedef int (*rte_k32v64_cmp_fn_t ) +(struct rte_k32v64_hash_bucket *bucket,uint32_t key,uint64_t *val); +lib/librte_hash/rte_k32v64_hash.h line 69
Quoted 10 lines
lib/librte_hash/rte_k32v64_hash.h new file +10
-0+static inline int +__k32v64_cmp_keys (struct rte_k32v64_hash_bucket *bucket,uint32_t key, +uint64_t *val) +{ +int i; + +for (i =0 ; i <RTE_K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == bucket->key[i]) && + (bucket->key_mask & (1 << i))) { + *val = bucket->val[i];This load can happen speculatively.
You have to use the guard variable and payload concept here for reader-writer concurrency. On the writer:
store -> key
store -> valstore_release->key_mask(or there will be arte_smp_wmbbefore this) 'key_mask' acts as the guard variableOn the reader:
load_acquire->key_mask(or there will be arte_smp_rmbafter this) if statement etc.....Quoted 13 lines
lib/librte_hash/rte_k32v64_hash.h new file +13
-0+return 1 ; + } + } + +return 0 ; +} + +static inline int +__k32v64_hash_lookup (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *value,rte_k32v64_cmp_fn_t cmp_f) { +uint64_t val =0 ; +struct rte_k32v64_ext_ent *ent; +int32_t cnt;
Quoted 8 lines
lib/librte_hash/rte_k32v64_hash.h line 90
Quoted 7 lines
lib/librte_hash/rte_k32v64_hash.h new file +7
-0+int found =0 ; +uint32_t bucket = hash & table->bucket_msk; + + do { + do + cnt = rte_atomic32_read(&table->t[bucket].cnt); + while (unlikely(cnt & RTE_K32V64_WRITE_IN_PROGRESS));The reader can hang here if the writer increments the counter and gets scheduled out. Using the method suggested above, you can remove this code.
Agree, but we tried to make transaction critical section as small as we can so probability is very low.
Quoted 125 lines
Quoted 5 lines
lib/librte_hash/rte_k32v64_hash.h new file +4
-0+ + found =cmp_f (&table->t[bucket], key, &val); +if (unlikely ((found ==0 ) && + (!SLIST_EMPTY (&table->t[bucket].head)))) {lib/librte_hash/rte_k32v64_hash.h line 104
Quoted 10 lines
lib/librte_hash/rte_k32v64_hash.h new file +10
-0+ SLIST_FOREACH(ent, &table->t[bucket].head, next) { +if (ent->key == key) { + val = ent->val; + found =1 ; +break ; + } + } + } + + }while (unlikely (cnt !=rte_atomic32_read (&table->t[bucket].cnt)));With the logic mentioned in delete function, the counter needs to be read at the beginning of the loop. Suggest looking at rte_hash-cuckoo hash for the memory ordering required for the counter.
Quoted 108 lines
lib/librte_hash/rte_k32v64_hash.h new file +106
-1+ +if (found ==1 ) { + *value = val; +return 0 ; + }else +return -ENOENT ; +} + +static inline int +rte_k32v64_hash_lookup (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *value) +{ +return __k32v64_hash_lookup (table, key, hash, value, +__k32v64_cmp_keys); } + +static inline int +rte_k32v64_hash_bulk_lookup (struct rte_k32v64_hash_table *table, +uint32_t *keys,uint32_t *hashes,uint64_t *values,unsigned int n) { +return table->lookup (table, keys, hashes, values, n); } + +/** +* Add a key to an existing hash table with hash value. +* This operation is not multi-thread safe +* and should only be called from one thread. +* +* @param ht +* Hash table to add the key to. +* @param key +* Key to add to the hash table. +* @param value +* Value to associate with key. +* @param hash +* Hash value associated with key. +* @return +* 0 if ok, or negative value on error. +*/ +__rte_experimental +int +rte_k32v64_hash_add (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t value); + +/** +* Remove a key with a given hash value from an existing hash table. +* This operation is not multi-thread +* safe and should only be called from one thread. +* +* @param ht +* Hash table to remove the key from. +* @param key +* Key to remove from the hash table. +* @param hash +* hash value associated with key. +* @return +* 0 if ok, or negative value on error. +*/ +__rte_experimental +int +rte_k32v64_hash_delete (struct rte_k32v64_hash_table *table,uint32_t key, +uint32_t hash); + + +/** +* Performs a lookup for an existing hash table, and returns a pointer +to +* the table if found. +* +* @param name +* Name of the hash table to find +* +* @return +* pointer to hash table structure or NULL on error with rte_errno +* set appropriately. +*/ +__rte_experimental +struct rte_k32v64_hash_table * +rte_k32v64_hash_find_existing (const char *name); + +/** +* Create a new hash table for use with four byte keys. +* +* @param params +* Parameters used in creation of hash table. +* +* @return +* Pointer to hash table structure that is used in future hash table +* operations, or NULL on error with rte_errno set appropriately. +*/ +__rte_experimental +struct rte_k32v64_hash_table * +rte_k32v64_hash_create (const struct rte_k32v64_hash_params *params); + +/** +* Free all memory used by a hash table. +* +* @param table +* Hash table to deallocate. +*/ +__rte_experimental +void +rte_k32v64_hash_free (struct rte_k32v64_hash_table *table); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_K32V64_HASH_H_ */ --2.7.4
Signature · 2 lines
Regards, Vladimir
Re: [dpdk-dev] [PATCH v4 0/4] add new kv hash table
In reply to Vladimir MedvedkinWaiting for reviews please.
08/05/2020 21:58, Vladimir Medvedkin:
Quoted 29 lines
Currently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called KV hash
supports 4 byte keys and 8 byte associated values,
which is enough to store a pointer.v4:
- internal implementation is hided under universal API for any key and value sizes
- add and delete API now return old value
- add transaction counter modification to_add()
- transaction counter now modifies with c11 atomicsv3:
- added bulk lookup
- avx512 key comparizon is removed from .hv2:
- renamed fromrte_dwktorte_k32v64as was suggested
- reworked lookup function, added inlined subroutines
- added avx512 key comparizon routine
- added documentation
- added statistic counters for total entries and extended entries(linked list)Vladimir Medvedkin (4):
hash: add kv hash library
hash: add documentation for kv hash library
test: add kv hash autotests
test: add kv perf tests
Re: [dpdk-dev] [PATCH v4 1/4] hash: add kv hash library
In reply to Vladimir MedvedkinHi Vladimir,
Quoted 101 lines
lib/librte_hash/k32v64_hash.c new file +98
-0@@ -0,0 +1,277 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include <string.h> + +#include <rte_errno.h> +#include <rte_malloc.h> +#include <rte_memory.h> + +#include "k32v64_hash.h" + +static inline int +k32v64_hash_lookup (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *value) +{ +return __k32v64_hash_lookup (table, key, hash, value, __kv_cmp_keys); +} + +static int +k32v64_hash_bulk_lookup (struct rte_kv_hash_table *ht,void *keys_p, +uint32_t *hashes,void *values_p,unsigned int n) +{ +struct k32v64_hash_table *table = (struct k32v64_hash_table *)ht; +uint32_t *keys = keys_p; +uint64_t *values = values_p; +int ret, cnt =0 ; +unsigned int i; + +if (unlikely ((table ==NULL ) || (keys ==NULL ) || (hashes ==NULL ) || + (values ==NULL ))) +return -EINVAL ; + +for (i =0 ; i < n; i++) { + ret =k32v64_hash_lookup (table, keys[i], hashes[i], + &values[i]); +if (ret ==0 ) + cnt++; + } +return cnt; +} + +#ifdef CC_AVX512VL_SUPPORT +int +k32v64_hash_bulk_lookup_avx512vl (struct rte_kv_hash_table *ht, +void *keys_p,uint32_t *hashes,void *values_p,unsigned int n); +#endif + +static rte_kv_hash_bulk_lookup_t +get_lookup_bulk_fn (void ) +{ +#ifdef CC_AVX512VL_SUPPORT +if (rte_cpu_get_flag_enabled (RTE_CPUFLAG_AVX512VL )) +return k32v64_hash_bulk_lookup_avx512vl; +#endif +return k32v64_hash_bulk_lookup; +} + +static int +k32v64_hash_add (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t value,uint64_t *old_value,int *found) +{ +uint32_t bucket; +int i, idx, ret; +uint8_t msk; +struct k32v64_ext_ent *tmp, *ent, *prev =NULL ; + +if (table ==NULL ) +return -EINVAL ; + + bucket = hash & table->bucket_msk; +/* Search key in table. Update value if exists */ +for (i =0 ; i <K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && + (table->t[bucket].key_mask & (1 << i))) { + *old_value = table->t[bucket].val[i]; + *found =1 ; +__atomic_fetch_add (&table->t[bucket].cnt,1 , + __ATOMIC_ACQUIRE); + table->t[bucket].val[i] = value; +__atomic_fetch_add (&table->t[bucket].cnt,1 , + __ATOMIC_RELEASE); +return 0 ; + } + } + +if (!SLIST_EMPTY (&table->t[bucket].head)) { + SLIST_FOREACH(ent, &table->t[bucket].head, next) { +if (ent->key == key) { + *old_value = ent->val; + *found =1 ; +__atomic_fetch_add (&table->t[bucket].cnt,1 , + __ATOMIC_ACQUIRE); + ent->val = value; +__atomic_fetch_add (&table->t[bucket].cnt,1 , + __ATOMIC_RELEASE); +return 0 ; + }
lib/librte_hash/k32v64_hash.c line 108
Quoted 10 lines
lib/librte_hash/k32v64_hash.c new file +10
-0+ } + } + + msk = ~table->t[bucket].key_mask &VALID_KEY_MSK ; +if (msk) { + idx =__builtin_ctz (msk); + table->t[bucket].key[idx] = key; + table->t[bucket].val[idx] = value; +__atomic_or_fetch (&table->t[bucket].key_mask,1 << idx, + __ATOMIC_RELEASE);
I think this case also has to guarded with table->t[bucket].cnt updates.
lib/librte_hash/k32v64_hash.c line 121
Quoted 13 lines
lib/librte_hash/k32v64_hash.c new file +13
-0+ table->nb_ent++; + *found =0 ; +return 0 ; + } + + ret =rte_mempool_get (table->ext_ent_pool, (void **)&ent); +if (ret <0 ) +return ret; + +SLIST_NEXT (ent, next) =NULL ; + ent->key = key; + ent->val = value; +rte_smp_wmb ();
__atomic_thread_fence(__ATOMIC_RELEASE);
?
Quoted 34 lines
lib/librte_hash/k32v64_hash.c new file +34
-0+SLIST_FOREACH (tmp, &table->t[bucket].head, next) + prev = tmp; + +if (prev ==NULL ) +SLIST_INSERT_HEAD (&table->t[bucket].head, ent, next); +else +SLIST_INSERT_AFTER (prev, ent, next); + + table->nb_ent++; + table->nb_ext_ent++; + *found =0 ; +return 0 ; +} + +static int +k32v64_hash_delete (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *old_value) +{ +uint32_t bucket; +int i; +struct k32v64_ext_ent *ent; + +if (table ==NULL ) +return -EINVAL ; + + bucket = hash & table->bucket_msk; + +for (i =0 ; i <K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && + (table->t[bucket].key_mask & (1 << i))) { + *old_value = table->t[bucket].val[i]; + ent =SLIST_FIRST (&table->t[bucket].head); +if (ent) { + __atomic_fetch_add(&table->t[bucket].cnt, 1,
lib/librte_hash/k32v64_hash.c line 165
Quoted 10 lines
lib/librte_hash/k32v64_hash.c new file +10
-0+ __ATOMIC_ACQUIRE); + table->t[bucket].key[i] = ent->key; + table->t[bucket].val[i] = ent->val; +SLIST_REMOVE_HEAD (&table->t[bucket].head, next); +__atomic_fetch_add (&table->t[bucket].cnt,1 , + __ATOMIC_RELEASE); + table->nb_ext_ent--; + }else +__atomic_and_fetch (&table->t[bucket].key_mask, + ~(1 << i), __ATOMIC_RELEASE);
Same thought as above - safer to guard this mask update with cnt update.
lib/librte_hash/k32v64_hash.c line 167
Quoted 2 lines
lib/librte_hash/k32v64_hash.c new file +2
-0+if (ent) +rte_mempool_put (table->ext_ent_pool, ent);
Can't this 'if(ent)' be merged with previous 'if (ent) {...}' above?
Quoted 25 lines
lib/librte_hash/k32v64_hash.c new file +25
-0+ table->nb_ent--; +return 0 ; + } + } + + SLIST_FOREACH(ent, &table->t[bucket].head, next) +if (ent->key == key) +break ; + +if (ent ==NULL ) +return -ENOENT ; + + *old_value = ent->val; + +__atomic_fetch_add (&table->t[bucket].cnt,1 , __ATOMIC_ACQUIRE); +SLIST_REMOVE (&table->t[bucket].head, ent, k32v64_ext_ent, next); +__atomic_fetch_add (&table->t[bucket].cnt,1 , __ATOMIC_RELEASE); +rte_mempool_put (table->ext_ent_pool, ent); + + table->nb_ext_ent--; + table->nb_ent--; + +return 0 ; +} +
Re: [dpdk-dev] [PATCH v4 1/4] hash: add kv hash library
Quoted 30 lines
Hi Vladimir,
Quoted 27 lines
lib/librte_hash/k32v64_hash.c new file +24
-0@@ -0,0 +1,277 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include <string.h> + +#include <rte_errno.h> +#include <rte_malloc.h> +#include <rte_memory.h> + +#include "k32v64_hash.h" + +static inline int +k32v64_hash_lookup (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *value) +{ +return __k32v64_hash_lookup (table, key, hash, value, __kv_cmp_keys); +} + +static int +k32v64_hash_bulk_lookup (struct rte_kv_hash_table *ht,void *keys_p, +uint32_t *hashes,void *values_p,unsigned int n) +{ +struct k32v64_hash_table *table = (struct k32v64_hash_table *)ht;
Quoted 8 lines
lib/librte_hash/k32v64_hash.c line 32
Quoted 8 lines
lib/librte_hash/k32v64_hash.c new file +8
-0+uint32_t *keys = keys_p; +uint64_t *values = values_p; +int ret, cnt =0 ; +unsigned int i; + +if (unlikely ((table ==NULL ) || (keys ==NULL ) || (hashes ==NULL ) || + (values ==NULL ))) +return -EINVAL ;
As a nit - this formal parameter checking better to do in public function (rte_kv_hash_bulk_lookup) before de-refencing table and calling actual lookup(). Same story for modify() - formal parameter checking can be done at the top of public function. BTW, why to unite add/delete into modify(), if internally you have 2 different functions (for add/delete) anyway?
Quoted 39 lines
Quoted 39 lines
lib/librte_hash/k32v64_hash.c new file +39
-0+ +for (i =0 ; i < n; i++) { + ret =k32v64_hash_lookup (table, keys[i], hashes[i], + &values[i]); +if (ret ==0 ) + cnt++; + } +return cnt; +} + +#ifdef CC_AVX512VL_SUPPORT +int +k32v64_hash_bulk_lookup_avx512vl (struct rte_kv_hash_table *ht, +void *keys_p,uint32_t *hashes,void *values_p,unsigned int n); +#endif + +static rte_kv_hash_bulk_lookup_t +get_lookup_bulk_fn (void ) +{ +#ifdef CC_AVX512VL_SUPPORT +if (rte_cpu_get_flag_enabled (RTE_CPUFLAG_AVX512VL )) +return k32v64_hash_bulk_lookup_avx512vl; +#endif +return k32v64_hash_bulk_lookup; +} + +static int +k32v64_hash_add (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t value,uint64_t *old_value,int *found) +{ +uint32_t bucket; +int i, idx, ret; +uint8_t msk; +struct k32v64_ext_ent *tmp, *ent, *prev =NULL ; + +if (table ==NULL ) +return -EINVAL ; + + bucket = hash & table->bucket_msk;
Quoted 8 lines
lib/librte_hash/k32v64_hash.c line 79
Quoted 8 lines
lib/librte_hash/k32v64_hash.c new file +8
-0+/* Search key in table. Update value if exists */ +for (i =0 ; i <K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && + (table->t[bucket].key_mask & (1 << i))) { + *old_value = table->t[bucket].val[i]; + *found =1 ; +__atomic_fetch_add (&table->t[bucket].cnt,1 , + __ATOMIC_ACQUIRE);
After another thought - atomic add is probably an overkill here. Something like:
void update_start(struct k32v64_hash_bucket *bkt)
{
bkt->cnt++;
__atomic_thread_fence(__ATOMIC_ACQ_REL);
}void update_finish(struct k32v64_hash_bucket *bkt)
{
__atomic_thread_fence(__ATOMIC_ACQ_REL);
bkt->cnt++;
}I think should be sufficient here.
Quoted 126 lines
Quoted 19 lines
lib/librte_hash/k32v64_hash.c new file +19
-0+ table->t[bucket].val[i] = value; +__atomic_fetch_add (&table->t[bucket].cnt,1 , + __ATOMIC_RELEASE); +return 0 ; + } + } + +if (!SLIST_EMPTY (&table->t[bucket].head)) { + SLIST_FOREACH(ent, &table->t[bucket].head, next) { +if (ent->key == key) { + *old_value = ent->val; + *found =1 ; +__atomic_fetch_add (&table->t[bucket].cnt,1 , + __ATOMIC_ACQUIRE); + ent->val = value; +__atomic_fetch_add (&table->t[bucket].cnt,1 , + __ATOMIC_RELEASE); +return 0 ; + }lib/librte_hash/k32v64_hash.c line 108
Quoted 10 lines
lib/librte_hash/k32v64_hash.c new file +10
-0+ } + } + + msk = ~table->t[bucket].key_mask &VALID_KEY_MSK ; +if (msk) { + idx =__builtin_ctz (msk); + table->t[bucket].key[idx] = key; + table->t[bucket].val[idx] = value; +__atomic_or_fetch (&table->t[bucket].key_mask,1 << idx, + __ATOMIC_RELEASE);I think this case also has to guarded with table->t[bucket].cnt updates.
lib/librte_hash/k32v64_hash.c line 121
Quoted 13 lines
lib/librte_hash/k32v64_hash.c new file +13
-0+ table->nb_ent++; + *found =0 ; +return 0 ; + } + + ret =rte_mempool_get (table->ext_ent_pool, (void **)&ent); +if (ret <0 ) +return ret; + +SLIST_NEXT (ent, next) =NULL ; + ent->key = key; + ent->val = value; +rte_smp_wmb ();
__atomic_thread_fence(__ATOMIC_RELEASE);
?Quoted 34 lines
lib/librte_hash/k32v64_hash.c new file +34
-0+SLIST_FOREACH (tmp, &table->t[bucket].head, next) + prev = tmp; + +if (prev ==NULL ) +SLIST_INSERT_HEAD (&table->t[bucket].head, ent, next); +else +SLIST_INSERT_AFTER (prev, ent, next); + + table->nb_ent++; + table->nb_ext_ent++; + *found =0 ; +return 0 ; +} + +static int +k32v64_hash_delete (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *old_value) +{ +uint32_t bucket; +int i; +struct k32v64_ext_ent *ent; + +if (table ==NULL ) +return -EINVAL ; + + bucket = hash & table->bucket_msk; + +for (i =0 ; i <K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && + (table->t[bucket].key_mask & (1 << i))) { + *old_value = table->t[bucket].val[i]; + ent =SLIST_FIRST (&table->t[bucket].head); +if (ent) { + __atomic_fetch_add(&table->t[bucket].cnt, 1,lib/librte_hash/k32v64_hash.c line 165
Quoted 10 lines
lib/librte_hash/k32v64_hash.c new file +10
-0+ __ATOMIC_ACQUIRE); + table->t[bucket].key[i] = ent->key; + table->t[bucket].val[i] = ent->val; +SLIST_REMOVE_HEAD (&table->t[bucket].head, next); +__atomic_fetch_add (&table->t[bucket].cnt,1 , + __ATOMIC_RELEASE); + table->nb_ext_ent--; + }else +__atomic_and_fetch (&table->t[bucket].key_mask, + ~(1 << i), __ATOMIC_RELEASE);Same thought as above - safer to guard this mask update with cnt update.
lib/librte_hash/k32v64_hash.c line 167
Quoted 2 lines
lib/librte_hash/k32v64_hash.c new file +2
-0+if (ent) +rte_mempool_put (table->ext_ent_pool, ent);Can't this 'if(ent)' be merged with previous 'if (ent) {...}' above?
Quoted 25 lines
lib/librte_hash/k32v64_hash.c new file +25
-0+ table->nb_ent--; +return 0 ; + } + } + + SLIST_FOREACH(ent, &table->t[bucket].head, next) +if (ent->key == key) +break ; + +if (ent ==NULL ) +return -ENOENT ; + + *old_value = ent->val; + +__atomic_fetch_add (&table->t[bucket].cnt,1 , __ATOMIC_ACQUIRE); +SLIST_REMOVE (&table->t[bucket].head, ent, k32v64_ext_ent, next); +__atomic_fetch_add (&table->t[bucket].cnt,1 , __ATOMIC_RELEASE); +rte_mempool_put (table->ext_ent_pool, ent); + + table->nb_ext_ent--; + table->nb_ent--; + +return 0 ; +} +
Re: [dpdk-dev] [PATCH v4 1/4] hash: add kv hash library
In reply to Vladimir MedvedkinQuoted 112 lines
-----Original Message-----
From: Medvedkin, Vladimir <[email protected]>
Sent: Friday, May 8, 2020 12:59 PM
To: [email protected]
Cc: Ananyev, Konstantin <[email protected]>; Wang, Yipeng1 <[email protected]>; Gobriel, Sameh <[email protected]>; Richardson, Bruce <[email protected]>
Subject: [PATCH v4 1/4] hash: add kv hash libraryKV hash is a special optimized key-value storage for fixed key and value sizes. At the moment it supports 32 bit keys and 64 bit values. This table is hash function agnostic so user must provide precalculated hash signature for add/delete/lookup operations.
Signed-off-by Vladimir Medvedkin <[email protected]>--- lib/Makefile | 2 +- lib/librte_hash/Makefile | 14 +- lib/librte_hash/k32v64_hash.c | 277 +++++++++++++++++++++++++++++++++ lib/librte_hash/k32v64_hash.h | 98 ++++++++++++ lib/librte_hash/k32v64_hash_avx512vl.c | 59 +++++++ lib/librte_hash/meson.build | 17 +- lib/librte_hash/rte_hash_version.map | 6 +- lib/librte_hash/rte_kv_hash.c | 184 ++++++++++++++++++++++ lib/librte_hash/rte_kv_hash.h | 169 ++++++++++++++++++++ 9 files changed, 821 insertions(+), 5 deletions(-) create mode 100644 lib/librte_hash/k32v64_hash.c create mode 100644 lib/librte_hash/k32v64_hash.h create mode 100644 lib/librte_hash/k32v64_hash_avx512vl.c create mode 100644 lib/librte_hash/rte_kv_hash.c create mode 100644
lib/librte_hash/rte_kv_hash.hlib/Makefile +0
-0@@ -48,7 +48,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_VHOST) += librte_vhostDEPDIRS-librte_vhost :=
librte_eallibrte_mempoollibrte_mbuflibrte_ethdev\
librte_netlibrte_hashlibrte_cryptodev
DIRS-$(CONFIG_RTE_LIBRTE_HASH) +=librte_hash-DEPDIRS-librte_hash :=librte_eallibrte_ring
+DEPDIRS-librte_hash :=librte_eallibrte_ringlibrte_mempool
DIRS-$(CONFIG_RTE_LIBRTE_EFD) +=librte_efdDEPDIRS-librte_efd :=librte_eallibrte_ringlibrte_hash
DIRS-$(CONFIG_RTE_LIBRTE_RIB) +=librte_ribdiff --git
a/lib/librte_hash/Makefile b/lib/librte_hash/Makefile index ec9f864..a0cdee9 100644lib/librte_hash/Makefile +4
-1@@ -8,13 +8,15 @@ LIB = librte_hash.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_ring +LDLIBS += -lrte_eal -lrte_ring -lrte_mempool EXPORT_MAP := rte_hash_version.map # all source are stored in SRCS-y SRCS-$(CONFIG_RTE_LIBRTE_HASH) := rte_cuckoo_hash.c SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_fbk_hash.c +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_kv_hash.c +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += k32v64_hash.c # install this header file SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include := rte_hash.h @@ -27,5 +29,15 @@ endif SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include +=
rte_jhash.hSYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include +=rte_thash.hSYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include +=rte_fbk_hash.hlib/librte_hash/Makefile +3
-0+SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_kv_hash.h + +CC_AVX512VL_SUPPORT=$(shell $(CC) -mavx512vl -dM -E - </dev/null2>&1 |
+\ grep -q__AVX512VL__&& echo 1)lib/librte_hash/Makefile +5
-0+ +ifeq ($(CC_AVX512VL_SUPPORT), 1) + SRCS-$(CONFIG_RTE_LIBRTE_HASH) += k32v64_hash_avx512vl.c + CFLAGS_k32v64_hash_avx512vl.o += -mavx512vl + CFLAGS_k32v64_hash.o += -DCC_AVX512VL_SUPPORT endif include $(RTE_SDK)/mk/rte.lib.mklib/librte_hash/k32v64_hash.c new file +17
-0@@ -0,0 +1,277 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include <string.h> + +#include <rte_errno.h> +#include <rte_malloc.h> +#include <rte_memory.h> + +#include "k32v64_hash.h" + +static inline int +k32v64_hash_lookup (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *value) +{ + return __k32v64_hash_lookup(table, key, hash, value,
__kv_cmp_keys); }lib/librte_hash/k32v64_hash.c new file +7
-0+ +static int +k32v64_hash_bulk_lookup (struct rte_kv_hash_table *ht,void *keys_p, +uint32_t *hashes,void *values_p,unsigned int n) { +struct k32v64_hash_table *table = (struct k32v64_hash_table *)ht; +uint32_t *keys = keys_p; +uint64_t *values = values_p;
lib/librte_hash/k32v64_hash.c line 36
Quoted 10 lines
lib/librte_hash/k32v64_hash.c new file +10
-0+int ret, cnt =0 ; +unsigned int i; + +if (unlikely ((table ==NULL ) || (keys ==NULL ) || (hashes ==NULL ) || + (values ==NULL ))) +return -EINVAL ; + +for (i =0 ; i < n; i++) { + ret =k32v64_hash_lookup (table, keys[i], hashes[i], + &values[i]);
[Wang, Yipeng] You don't need to start a new line for values.
lib/librte_hash/k32v64_hash.c line 43
Quoted 7 lines
lib/librte_hash/k32v64_hash.c new file +7
-0+if (ret ==0 ) + cnt++; + } +return cnt; +} + +#ifdef CC_AVX512VL_SUPPORT
[Wang, Yipeng] Why not use the already provided, e.g. #if defined(RTE_MACHINE_CPUFLAG_SSE2) like rte_hash does?
Quoted 145 lines
lib/librte_hash/k32v64_hash.c new file +47
-0+int +k32v64_hash_bulk_lookup_avx512vl (struct rte_kv_hash_table *ht, +void *keys_p,uint32_t *hashes,void *values_p,unsigned int n); +#endif + +static rte_kv_hash_bulk_lookup_t +get_lookup_bulk_fn (void ) +{ +#ifdef CC_AVX512VL_SUPPORT +if (rte_cpu_get_flag_enabled (RTE_CPUFLAG_AVX512VL )) +return k32v64_hash_bulk_lookup_avx512vl;#endif +return k32v64_hash_bulk_lookup; +} + +static int +k32v64_hash_add (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t value,uint64_t *old_value,int *found) { +uint32_t bucket; +int i, idx, ret; +uint8_t msk; +struct k32v64_ext_ent *tmp, *ent, *prev =NULL ; + +if (table ==NULL ) +return -EINVAL ; + + bucket = hash & table->bucket_msk; +/* Search key in table. Update value if exists */ +for (i =0 ; i <K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && + (table->t[bucket].key_mask & (1 << i))) { + *old_value = table->t[bucket].val[i]; + *found =1 ; +__atomic_fetch_add (&table->t[bucket].cnt,1 , + __ATOMIC_ACQUIRE); + table->t[bucket].val[i] = value; +__atomic_fetch_add (&table->t[bucket].cnt,1 , + __ATOMIC_RELEASE); +return 0 ; + } + } + +if (!SLIST_EMPTY (&table->t[bucket].head)) { + SLIST_FOREACH(ent, &table->t[bucket].head, next) { +if (ent->key == key) { + *old_value = ent->val; + *found =1 ; + __atomic_fetch_add(&table->t[bucket].cnt,1,
lib/librte_hash/k32v64_hash.c new file +3
-0+ __ATOMIC_ACQUIRE); + ent->val = value; + __atomic_fetch_add(&table->t[bucket].cnt,1,
lib/librte_hash/k32v64_hash.c new file +60
-0+ __ATOMIC_RELEASE); +return 0 ; + } + } + } + + msk = ~table->t[bucket].key_mask &VALID_KEY_MSK ; +if (msk) { + idx =__builtin_ctz (msk); + table->t[bucket].key[idx] = key; + table->t[bucket].val[idx] = value; +__atomic_or_fetch (&table->t[bucket].key_mask,1 << idx, + __ATOMIC_RELEASE); + table->nb_ent++; + *found =0 ; +return 0 ; + } + + ret =rte_mempool_get (table->ext_ent_pool, (void **)&ent); +if (ret <0 ) +return ret; + +SLIST_NEXT (ent, next) =NULL ; + ent->key = key; + ent->val = value; +rte_smp_wmb (); +SLIST_FOREACH (tmp, &table->t[bucket].head, next) + prev = tmp; + +if (prev ==NULL ) +SLIST_INSERT_HEAD (&table->t[bucket].head, ent, next); +else +SLIST_INSERT_AFTER (prev, ent, next); + + table->nb_ent++; + table->nb_ext_ent++; + *found =0 ; +return 0 ; +} + +static int +k32v64_hash_delete (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *old_value) +{ +uint32_t bucket; +int i; +struct k32v64_ext_ent *ent; + +if (table ==NULL ) +return -EINVAL ; + + bucket = hash & table->bucket_msk; + +for (i =0 ; i <K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && + (table->t[bucket].key_mask & (1 << i))) { + *old_value = table->t[bucket].val[i]; + ent =SLIST_FIRST (&table->t[bucket].head); +if (ent) { + __atomic_fetch_add(&table->t[bucket].cnt,1,
lib/librte_hash/k32v64_hash.c new file +4
-0+ __ATOMIC_ACQUIRE); + table->t[bucket].key[i] = ent->key; + table->t[bucket].val[i] = ent->val; + SLIST_REMOVE_HEAD(&table-Quoted 1 line
t[bucket].head, next);
lib/librte_hash/k32v64_hash.c new file +1
-0+ __atomic_fetch_add(&table->t[bucket].cnt,1,
lib/librte_hash/k32v64_hash.c new file +4
-0+ __ATOMIC_RELEASE); + table->nb_ext_ent--; + }else + __atomic_and_fetch(&table-Quoted 1 line
t[bucket].key_mask,
lib/librte_hash/k32v64_hash.c new file +3
-0+ ~(1 << i), __ATOMIC_RELEASE); +if (ent) + rte_mempool_put(table->ext_ent_pool,ent);
lib/librte_hash/k32v64_hash.c new file +15
-0+ table->nb_ent--; +return 0 ; + } + } + + SLIST_FOREACH(ent, &table->t[bucket].head, next) +if (ent->key == key) +break ; + +if (ent ==NULL ) +return -ENOENT ; + + *old_value = ent->val; + + __atomic_fetch_add(&table->t[bucket].cnt, 1,
__ATOMIC_ACQUIRE);
lib/librte_hash/k32v64_hash.c line 186
Quoted 4 lines
lib/librte_hash/k32v64_hash.c new file +4
-0+SLIST_REMOVE (&table->t[bucket].head, ent, k32v64_ext_ent, next); +__atomic_fetch_add (&table->t[bucket].cnt,1 , __ATOMIC_RELEASE); +rte_mempool_put (table->ext_ent_pool, ent); +
[Wang, Yipeng] I am not sure if delete can be called safely with other lookup threads. The item could be recycled to be used by another add while a lookup thread traversing the linked list. This is similar to why we need the RTE_HASH_EXTRA_FLAGS_NO_FREE_ON_DEL flag and related API in rte_hash.
Quoted 25 lines
lib/librte_hash/k32v64_hash.c new file +8
-0+ table->nb_ext_ent--; + table->nb_ent--; + +return 0 ; +} + +static int +k32v64_modify(struct rte_kv_hash_table *table, void *key_p, uint32_thash,
lib/librte_hash/k32v64_hash.c new file +7
-0+ enum rte_kv_modify_op op, void *value_p, int *found) { + struct k32v64_hash_table *ht = (struct k32v64_hash_table *)table; +uint32_t *key = key_p; +uint64_t value; + + if ((ht == NULL) || (key == NULL) || (value_p == NULL) || + (found == NULL) || (op >=
RTE_KV_MODIFY_OP_MAX)) {lib/librte_hash/k32v64_hash.c new file +7
-0+return -EINVAL ; + } + + value = *(uint64_t *)value_p; +switch (op) { +case RTE_KV_MODIFY_ADD : + return k32v64_hash_add(ht, *key, hash, value, value_p,found);
lib/librte_hash/k32v64_hash.c line 214
Quoted 5 lines
lib/librte_hash/k32v64_hash.c new file +5
-0+case RTE_KV_MODIFY_DEL : +return k32v64_hash_delete (ht, *key, hash, value_p); +default : +break ; + }
[Wang, Yipeng] A question would be why put del and add inside another mod wrapper. If we don't wrap del and add like this, we could get rid of this branch.
Quoted 4 lines
lib/librte_hash/k32v64_hash.c new file +4
-0+ +return -EINVAL ; +} +
lib/librte_hash/k32v64_hash.c line 229
Quoted 10 lines
lib/librte_hash/k32v64_hash.c new file +10
-0+struct rte_kv_hash_table * +k32v64_hash_create (const struct rte_kv_hash_params *params) { +char hash_name[RTE_KV_HASH_NAMESIZE ]; +struct k32v64_hash_table *ht =NULL ; +uint32_t mem_size, nb_buckets, max_ent; +int ret; +struct rte_mempool *mp; + +if ((params ==NULL ) || (params->name ==NULL ) || + (params->entries ==0 )) {
[Wang, Yipeng] Should we check if entry count larger than keys_per_bucket as well?
lib/librte_hash/k32v64_hash.c line 241
Quoted 13 lines
lib/librte_hash/k32v64_hash.c new file +5
-0+ rte_errno =EINVAL ; +return NULL ; + } + + ret = snprintf(hash_name, sizeof(hash_name), "KV_%s", params-Quoted 1 line
name);
lib/librte_hash/k32v64_hash.c new file +7
-0+if (ret <0 || ret >=RTE_KV_HASH_NAMESIZE ) { + rte_errno =ENAMETOOLONG ; +return NULL ; + } + + max_ent =rte_align32pow2 (params->entries); + nb_buckets = max_ent /K32V64_KEYS_PER_BUCKET ;
[Wang, Yipeng] Macro to check if keys_per_bucket need to be power of 2
Quoted 12 lines
lib/librte_hash/k32v64_hash.c new file +12
-0+ mem_size =sizeof (struct k32v64_hash_table ) + +sizeof (struct k32v64_hash_bucket ) * nb_buckets; + + mp =rte_mempool_create (hash_name, max_ent, +sizeof (struct k32v64_ext_ent ),0 ,0 ,NULL ,NULL ,NULL ,NULL , + params->socket_id,0 ); + +if (mp ==NULL ) +return NULL ; + + ht =rte_zmalloc_socket (hash_name, mem_size, +RTE_CACHE_LINE_SIZE , params->socket_id);
lib/librte_hash/k32v64_hash.c line 263
Quoted 10 lines
lib/librte_hash/k32v64_hash.c new file +10
-0+if (ht ==NULL ) { +rte_mempool_free (mp); +return NULL ; + } + +memcpy (ht->pub.name, hash_name,sizeof (ht->pub.name)); + ht->max_ent = max_ent; + ht->bucket_msk = nb_buckets -1 ; + ht->ext_ent_pool = mp; + ht->pub.lookup =get_lookup_bulk_fn ();
[Wang, Yipeng] Inside the function, we also need to check the CPUID during runtime to decide if AVX can be used, not only compile time. You could refer to example from rte_hash: ... if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE2)) ...
Quoted 34 lines
lib/librte_hash/k32v64_hash.c new file +10
-0+ ht->pub.modify = k32v64_modify; + +return (struct rte_kv_hash_table *)ht; } + +void +k32v64_hash_free (struct rte_kv_hash_table *ht) { +if (ht ==NULL ) +return ; + + rte_mempool_free(((struct k32v64_hash_table *)ht)-Quoted 1 line
ext_ent_pool);lib/librte_hash/k32v64_hash.c new file +2
-0+rte_free (ht); +}lib/librte_hash/k32v64_hash.h new file +16
-0@@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include <rte_kv_hash.h> + +#define K32V64_KEYS_PER_BUCKET 4 +#define K32V64_WRITE_IN_PROGRESS 1 +#define VALID_KEY_MSK ((1 << K32V64_KEYS_PER_BUCKET) - 1) + +struct k32v64_ext_ent { +SLIST_ENTRY (k32v64_ext_ent) next; +uint32_t key; +uint64_t val; +}; +
lib/librte_hash/k32v64_hash.h line 26
Quoted 10 lines
lib/librte_hash/k32v64_hash.h new file +10
-0+struct k32v64_hash_bucket { +uint32_t key[K32V64_KEYS_PER_BUCKET ]; +uint64_t val[K32V64_KEYS_PER_BUCKET ]; +uint8_t key_mask; +uint32_t cnt; + SLIST_HEAD(k32v64_list_head, k32v64_ext_ent) head; } +__rte_cache_aligned; + +struct k32v64_hash_table { +struct rte_kv_hash_table pub;/**< Public part */
[Wang, Yipeng] Do we need to have the pub filed? Could we just init the public part in the public create function?
Quoted 195 lines
lib/librte_hash/k32v64_hash.h new file +1
-0+uint32_t nb_ent;/**< Number of entities in the table*/
lib/librte_hash/k32v64_hash.h new file +70
-0+uint32_t nb_ext_ent;/**< Number of extended entities */ +uint32_t max_ent;/**< Maximum number of entities */ +uint32_t bucket_msk; +struct rte_mempool *ext_ent_pool; + __extension__struct k32v64_hash_bucket t[0 ]; +}; + +typedef int (*k32v64_cmp_fn_t ) +(struct k32v64_hash_bucket *bucket,uint32_t key,uint64_t *val); + +static inline int +__kv_cmp_keys (struct k32v64_hash_bucket *bucket,uint32_t key, +uint64_t *val) +{ +int i; + +for (i =0 ; i <K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == bucket->key[i]) && + (bucket->key_mask & (1 << i))) { + *val = bucket->val[i]; +return 1 ; + } + } + +return 0 ; +} + +static inline int +__k32v64_hash_lookup (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *value,k32v64_cmp_fn_t cmp_f) { +uint64_t val =0 ; +struct k32v64_ext_ent *ent; +uint32_t cnt; +int found =0 ; +uint32_t bucket = hash & table->bucket_msk; + +do { + +do { + cnt =__atomic_load_n (&table->t[bucket].cnt, + __ATOMIC_ACQUIRE); + }while (unlikely (cnt &K32V64_WRITE_IN_PROGRESS )); + + found =cmp_f (&table->t[bucket], key, &val); +if (unlikely ((found ==0 ) && + (!SLIST_EMPTY (&table->t[bucket].head)))) { + SLIST_FOREACH(ent, &table->t[bucket].head, next) { +if (ent->key == key) { + val = ent->val; + found =1 ; +break ; + } + } + } +__atomic_thread_fence (__ATOMIC_RELEASE); + }while (unlikely (cnt !=__atomic_load_n (&table->t[bucket].cnt, + __ATOMIC_RELAXED))); + +if (found ==1 ) { + *value = val; +return 0 ; + }else +return -ENOENT ; +} + +struct rte_kv_hash_table * +k32v64_hash_create (const struct rte_kv_hash_params *params); + +void +k32v64_hash_free (struct rte_kv_hash_table *ht);lib/librte_hash/k32v64_hash_avx512vl.c new file +8
-0@@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include "k32v64_hash.h" + +int +k32v64_hash_bulk_lookup_avx512vl(struct rte_kv_hash_table *ht, void
*keys_p,lib/librte_hash/k32v64_hash_avx512vl.c new file +4
-0+ uint32_t *hashes, void *values_p, unsigned int n); + +static inline int +k32v64_cmp_keys_avx512vl(struct k32v64_hash_bucket *bucket, uint32_tkey,
lib/librte_hash/k32v64_hash_avx512vl.c new file +9
-0+ uint64_t *val) +{ +__m128i keys, srch_key; +__mmask8 msk; + + keys =_mm_load_si128 ((void *)bucket); + srch_key =_mm_set1_epi32 (key); + + msk = _mm_mask_cmpeq_epi32_mask(bucket->key_mask, keys,
srch_key);lib/librte_hash/k32v64_hash_avx512vl.c new file +10
-0+if (msk) { + *val = bucket->val[__builtin_ctz (msk)]; +return 1 ; + } + +return 0 ; +} + +static inline int +k32v64_hash_lookup_avx512vl(struct k32v64_hash_table *table, uint32_tkey,
lib/librte_hash/k32v64_hash_avx512vl.c new file +8
-0+ uint32_t hash, uint64_t *value) +{ + return __k32v64_hash_lookup(table, key, hash, value, +k32v64_cmp_keys_avx512vl ); +} + +int +k32v64_hash_bulk_lookup_avx512vl(struct rte_kv_hash_table *ht, void
*keys_p,lib/librte_hash/k32v64_hash_avx512vl.c new file +19
-0+ uint32_t *hashes, void *values_p, unsigned int n) { + struct k32v64_hash_table *table = (struct k32v64_hash_table *)ht; +uint32_t *keys = keys_p; +uint64_t *values = values_p; +int ret, cnt =0 ; +unsigned int i; + +if (unlikely ((table ==NULL ) || (keys ==NULL ) || (hashes ==NULL ) || + (values ==NULL ))) +return -EINVAL ; + +for (i =0 ; i < n; i++) { + ret =k32v64_hash_lookup_avx512vl (table, keys[i], hashes[i], + &values[i]); +if (ret ==0 ) + cnt++; + } +return cnt; +}lib/librte_hash/meson.build index renamed from lib/librte_hash/meson.build +0
-06ab46ae..0d014ea 100644
lib/librte_hash/meson.build +15
-1@@ -3,10 +3,23 @@ headers = files('rte_crc_arm64.h', 'rte_fbk_hash.h', + 'rte_kv_hash.h', 'rte_hash_crc.h', 'rte_hash.h', 'rte_jhash.h', 'rte_thash.h') -sources = files('rte_cuckoo_hash.c', 'rte_fbk_hash.c') -deps += ['ring'] +sources = files('rte_cuckoo_hash.c', 'rte_fbk_hash.c', 'rte_kv_hash.c', +'k32v64_hash.c') deps += ['ring', 'mempool'] + +if dpdk_conf.has('RTE_ARCH_X86') + if cc.has_argument('-mavx512vl') + avx512_tmplib = static_library('avx512_tmp', + 'k32v64_hash_avx512vl.c', + dependencies: static_rte_mempool, + c_args: cflags + ['-mavx512vl']) + objs += avx512_tmplib.extract_objects('k32v64_hash_avx512vl.c') + cflags += '-DCC_AVX512VL_SUPPORT' + + endif +endiflib/librte_hash/rte_hash_version.map +5
-1@@ -36,5 +36,9 @@ EXPERIMENTAL { rte_hash_lookup_with_hash_bulk; rte_hash_lookup_with_hash_bulk_data; rte_hash_max_key_id; - + rte_kv_hash_create; + rte_kv_hash_find_existing; + rte_kv_hash_free; + rte_kv_hash_add; + rte_kv_hash_delete; };lib/librte_hash/rte_kv_hash.c new file +3
-0@@ -0,0 +1,184 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/
lib/librte_hash/rte_kv_hash.c line 13
Quoted 10 lines
lib/librte_hash/rte_kv_hash.c new file +10
-0+ +#include <string.h> + +#include <rte_eal_memconfig.h> +#include <rte_errno.h> +#include <rte_malloc.h> +#include <rte_memory.h> +#include <rte_tailq.h> + +#include <rte_kv_hash.h>
[Wang, Yipeng] I think for this header we should use quotes ""
Quoted 4 lines
lib/librte_hash/rte_kv_hash.c new file +4
-0+#include "k32v64_hash.h" + +TAILQ_HEAD (rte_kv_hash_list, rte_tailq_entry); +
lib/librte_hash/rte_kv_hash.c line 28
Quoted 10 lines
lib/librte_hash/rte_kv_hash.c new file +10
-0+static struct rte_tailq_elem rte_kv_hash_tailq = { + .name ="RTE_KV_HASH" , +}; + +EAL_REGISTER_TAILQ (rte_kv_hash_tailq); + +int +rte_kv_hash_add (struct rte_kv_hash_table *table,void *key, +uint32_t hash,void *value,int *found) { +if (table ==NULL )
[Wang, Yipeng] To be more consistent,
I think we should either also check key/value/found here, or just not checking at all and leave it to the next functions.
Quoted 176 lines
lib/librte_hash/rte_kv_hash.c new file +61
-0+return -EINVAL ; + +return table->modify (table, key, hash,RTE_KV_MODIFY_ADD , + value, found); +} + +int +rte_kv_hash_delete (struct rte_kv_hash_table *table,void *key, +uint32_t hash,void *value) +{ +int found; + +if (table ==NULL ) +return -EINVAL ; + +return table->modify (table, key, hash,RTE_KV_MODIFY_DEL , + value, &found); +} + +struct rte_kv_hash_table * +rte_kv_hash_find_existing (const char *name) { +struct rte_kv_hash_table *h =NULL ; +struct rte_tailq_entry *te; +struct rte_kv_hash_list *kv_hash_list; + + kv_hash_list =RTE_TAILQ_CAST (rte_kv_hash_tailq.head, + rte_kv_hash_list); + +rte_mcfg_tailq_read_lock (); + TAILQ_FOREACH(te, kv_hash_list, next) { + h = (struct rte_kv_hash_table *) te->data; +if (strncmp (name, h->name,RTE_KV_HASH_NAMESIZE ) ==0 ) +break ; + } +rte_mcfg_tailq_read_unlock (); +if (te ==NULL ) { + rte_errno =ENOENT ; +return NULL ; + } +return h; +} + +struct rte_kv_hash_table * +rte_kv_hash_create (const struct rte_kv_hash_params *params) { +char hash_name[RTE_KV_HASH_NAMESIZE ]; +struct rte_kv_hash_table *ht, *tmp_ht =NULL ; +struct rte_tailq_entry *te; +struct rte_kv_hash_list *kv_hash_list; +int ret; + +if ((params ==NULL ) || (params->name ==NULL ) || + (params->entries ==0 ) || + (params->type >=RTE_KV_HASH_MAX )) { + rte_errno =EINVAL ; +return NULL ; + } + + kv_hash_list =RTE_TAILQ_CAST (rte_kv_hash_tailq.head, + rte_kv_hash_list); + + ret = snprintf(hash_name, sizeof(hash_name), "KV_%s", params-Quoted 1 line
name);
lib/librte_hash/rte_kv_hash.c new file +92
-0+if (ret <0 || ret >=RTE_KV_HASH_NAMESIZE ) { + rte_errno =ENAMETOOLONG ; +return NULL ; + } + +switch (params->type) { +case RTE_KV_HASH_K32V64 : + ht =k32v64_hash_create (params); +break ; +default : + rte_errno =EINVAL ; +return NULL ; + } +if (ht ==NULL ) +return ht; + +rte_mcfg_tailq_write_lock (); + TAILQ_FOREACH(te, kv_hash_list, next) { + tmp_ht = (struct rte_kv_hash_table *) te->data; +if (strncmp (params->name, tmp_ht->name, +RTE_KV_HASH_NAMESIZE ) ==0 ) +break ; + } +if (te !=NULL ) { + rte_errno =EEXIST ; + goto exit; + } + + te =rte_zmalloc ("KV_HASH_TAILQ_ENTRY" ,sizeof (*te),0 ); +if (te ==NULL ) { +RTE_LOG (ERR ,HASH ,"Failed to allocate tailq entry\n" ); + goto exit; + } + + ht->type = params->type; + te->data = (void *)ht; +TAILQ_INSERT_TAIL (kv_hash_list, te, next); + +rte_mcfg_tailq_write_unlock (); + +return ht; + +exit: +rte_mcfg_tailq_write_unlock (); +switch (params->type) { +case RTE_KV_HASH_K32V64 : +k32v64_hash_free (ht); +break ; +default : +break ; + } +return NULL ; +} + +void +rte_kv_hash_free (struct rte_kv_hash_table *ht) { +struct rte_tailq_entry *te; +struct rte_kv_hash_list *kv_hash_list; + +if (ht ==NULL ) +return ; + + kv_hash_list =RTE_TAILQ_CAST (rte_kv_hash_tailq.head, + rte_kv_hash_list); + +rte_mcfg_tailq_write_lock (); + +/* find out tailq entry */ + TAILQ_FOREACH(te, kv_hash_list, next) { +if (te->data == (void *) ht) +break ; + } + + +if (te ==NULL ) { +rte_mcfg_tailq_write_unlock (); +return ; + } + +TAILQ_REMOVE (kv_hash_list, te, next); + +rte_mcfg_tailq_write_unlock (); + +switch (ht->type) { +case RTE_KV_HASH_K32V64 : +k32v64_hash_free (ht); +break ; +default : +break ; + } +rte_free (te); +}lib/librte_hash/rte_kv_hash.h new file +17
-0@@ -0,0 +1,169 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#ifndef _RTE_KV_HASH_H_ +#define _RTE_KV_HASH_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <rte_compat.h> +#include <rte_atomic.h> +#include <rte_mempool.h> + +#define RTE_KV_HASH_NAMESIZE 32 +
lib/librte_hash/rte_kv_hash.h line 27
Quoted 10 lines
lib/librte_hash/rte_kv_hash.h new file +10
-0+enum rte_kv_hash_type { +RTE_KV_HASH_K32V64 , +RTE_KV_HASH_MAX +}; + +enum rte_kv_modify_op { +RTE_KV_MODIFY_ADD , +RTE_KV_MODIFY_DEL , +RTE_KV_MODIFY_OP_MAX +};
[Wang, Yipeng] Again, any particular reason that you combine add and del into an additional struct mod?
Quoted 34 lines
lib/librte_hash/rte_kv_hash.h new file +20
-0+ +struct rte_kv_hash_params { +const char *name; +uint32_t entries; +int socket_id; +enum rte_kv_hash_type type; +}; + +struct rte_kv_hash_table ; + +typedef int (*rte_kv_hash_bulk_lookup_t ) (struct rte_kv_hash_table +*table,void *keys,uint32_t *hashes, +void *values,unsigned int n); + +typedef int (*rte_kv_hash_modify_t ) +(struct rte_kv_hash_table *table,void *key,uint32_t hash, +enum rte_kv_modify_op op,void *value,int *found); + +struct rte_kv_hash_table { +char name[RTE_KV_HASH_NAMESIZE ];/**< Name of the hash. */
lib/librte_hash/rte_kv_hash.h new file +13
-0+rte_kv_hash_bulk_lookup_t lookup; +rte_kv_hash_modify_t modify; +enum rte_kv_hash_type type; +}; + +/** +* Lookup bulk of keys. +* This function is multi-thread safe with regarding to other lookup threads. +* +* @param table +* Hash table to add the key to. +* @param keys +* Pointer to array of keys
lib/librte_hash/rte_kv_hash.h line 70
Quoted 10 lines
lib/librte_hash/rte_kv_hash.h new file +10
-0+* @param hashes +* Pointer to array of hash values associated with keys. +* @param values +* Pointer to array of value corresponded to keys +* If the key was not found the corresponding value remains intact. +* @param n +* Number of keys to lookup in batch. +* @return +* -EINVAL if there's an error, otherwise number of successful lookups. +*/
[Wang, Yipeng] experimental tag
Quoted 63 lines
lib/librte_hash/rte_kv_hash.h new file +63
-0+static inline int +rte_kv_hash_bulk_lookup (struct rte_kv_hash_table *table, +void *keys,uint32_t *hashes,void *values,unsigned int n) { +return table->lookup (table, keys, hashes, values, n); } + +/** +* Add a key to an existing hash table with hash value. +* This operation is not multi-thread safe regarding to add/delete +functions +* and should only be called from one thread. +* However it is safe to call it along with lookup. +* +* @param table +* Hash table to add the key to. +* @param key +* Key to add to the hash table. +* @param value +* Value to associate with key. +* @param hash +* Hash value associated with key. +* @found +* 0 if no previously added key was found +* 1 previously added key was found, old value associated with the key +* was written to *value +* @return +* 0 if ok, or negative value on error. +*/ +__rte_experimental +int +rte_kv_hash_add (struct rte_kv_hash_table *table,void *key, +uint32_t hash,void *value,int *found); + +/** +* Remove a key with a given hash value from an existing hash table. +* This operation is not multi-thread safe regarding to add/delete +functions +* and should only be called from one thread. +* However it is safe to call it along with lookup. +* +* @param table +* Hash table to remove the key from. +* @param key +* Key to remove from the hash table. +* @param hash +* hash value associated with key. +* @param value +* pointer to memory where the old value will be written to on success +* @return +* 0 if ok, or negative value on error. +*/ +__rte_experimental +int +rte_kv_hash_delete (struct rte_kv_hash_table *table,void *key, +uint32_t hash,void *value); + +/** +* Performs a lookup for an existing hash table, and returns a pointer +to +* the table if found. +* +* @param name +* Name of the hash table to find +*
lib/librte_hash/rte_kv_hash.h line 142
Quoted 10 lines
lib/librte_hash/rte_kv_hash.h new file +10
-0+* @return +* pointer to hash table structure or NULL on error with rte_errno +* set appropriately. +*/ +__rte_experimental +struct rte_kv_hash_table * +rte_kv_hash_find_existing (const char *name); + +/** +* Create a new hash table for use with four byte keys.
[Wang, Yipeng] inaccurate comment, not four byte keys
Quoted 29 lines
+ *
+ * @param params
+ * Parameters used in creation of hash table.
+ *
+ * @return
+ * Pointer to hash table structure that is used in future hash table
+ * operations, or NULL on error withrte_errnoset appropriately.
+ */lib/librte_hash/rte_kv_hash.h new file +19
-1+__rte_experimental +struct rte_kv_hash_table * +rte_kv_hash_create (const struct rte_kv_hash_params *params); + +/** +* Free all memory used by a hash table. +* +* @param table +* Hash table to deallocate. +*/ +__rte_experimental +void +rte_kv_hash_free (struct rte_kv_hash_table *table); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_KV_HASH_H_ */ --2.7.4
Re: [dpdk-dev] [PATCH v4 1/4] hash: add kv hash library
In reply to Vladimir MedvedkinHi Vladimir,
Few comments inline.
Quoted 101 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Vladimir Medvedkin
Sent: Friday, May 8, 2020 2:59 PM
To: [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected]
Subject: [dpdk-dev] [PATCH v4 1/4] hash: add kv hash libraryKV hash is a special optimized key-value storage for fixed key and value sizes. At the moment it supports 32 bit keys and 64 bit values. This table is hash function agnostic so user must provide precalculated hash signature for add/delete/lookup operations.
Signed-off-by Vladimir Medvedkin <[email protected]>--- lib/Makefile | 2 +- lib/librte_hash/Makefile | 14 +- lib/librte_hash/k32v64_hash.c | 277 +++++++++++++++++++++++++++++++++ lib/librte_hash/k32v64_hash.h | 98 ++++++++++++ lib/librte_hash/k32v64_hash_avx512vl.c | 59 +++++++ lib/librte_hash/meson.build | 17 +- lib/librte_hash/rte_hash_version.map | 6 +- lib/librte_hash/rte_kv_hash.c | 184 ++++++++++++++++++++++ lib/librte_hash/rte_kv_hash.h | 169 ++++++++++++++++++++ 9 files changed, 821 insertions(+), 5 deletions(-) create mode 100644 lib/librte_hash/k32v64_hash.c create mode 100644 lib/librte_hash/k32v64_hash.h create mode 100644 lib/librte_hash/k32v64_hash_avx512vl.c create mode 100644 lib/librte_hash/rte_kv_hash.c create mode 100644
lib/librte_hash/rte_kv_hash.hlib/Makefile +0
-0@@ -48,7 +48,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_VHOST) += librte_vhost DEPDIRS-librte_vhost := librte_eal librte_mempool librte_mbuf librte_ethdev \ librte_net librte_hash librte_cryptodev DIRS-$(CONFIG_RTE_LIBRTE_HASH) += librte_hash -DEPDIRS-librte_hash :=
librte_eallibrte_ring
+DEPDIRS-librte_hash :=librte_eallibrte_ringlibrte_mempool
DIRS-$(CONFIG_RTE_LIBRTE_EFD) +=librte_efdDEPDIRS-librte_efd :=librte_eallibrte_ringlibrte_hash
DIRS-$(CONFIG_RTE_LIBRTE_RIB) +=librte_ribdiff --git
a/lib/librte_hash/Makefile b/lib/librte_hash/Makefile index ec9f864..a0cdee9 100644lib/librte_hash/Makefile +4
-1@@ -8,13 +8,15 @@ LIB = librte_hash.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_ring +LDLIBS += -lrte_eal -lrte_ring -lrte_mempool EXPORT_MAP := rte_hash_version.map # all source are stored in SRCS-y SRCS-$(CONFIG_RTE_LIBRTE_HASH) := rte_cuckoo_hash.c SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_fbk_hash.c +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_kv_hash.c +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += k32v64_hash.c # install this header file SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include := rte_hash.h @@ -27,5 +29,15 @@ endif SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include +=
rte_jhash.hSYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include +=rte_thash.hSYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include +=rte_fbk_hash.hlib/librte_hash/Makefile +9
-0+SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_kv_hash.h + +CC_AVX512VL_SUPPORT=$(shell $(CC) -mavx512vl -dM -E - </dev/null 2>&1 | +\ grep -q __AVX512VL__ && echo 1) + +ifeq ($(CC_AVX512VL_SUPPORT), 1) + SRCS-$(CONFIG_RTE_LIBRTE_HASH) += k32v64_hash_avx512vl.c + CFLAGS_k32v64_hash_avx512vl.o += -mavx512vl + CFLAGS_k32v64_hash.o += -DCC_AVX512VL_SUPPORT endif include $(RTE_SDK)/mk/rte.lib.mklib/librte_hash/k32v64_hash.c new file +17
-0@@ -0,0 +1,277 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include <string.h> + +#include <rte_errno.h> +#include <rte_malloc.h> +#include <rte_memory.h> + +#include "k32v64_hash.h" + +static inline int +k32v64_hash_lookup (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *value) +{ + return __k32v64_hash_lookup(table, key, hash, value,
Quoted 1 line
__kv_cmp_keys); }
Since, this is an inline function, would it be better to push this to the header file?
Quoted 49 lines
lib/librte_hash/k32v64_hash.c new file +49
-0+ +static int +k32v64_hash_bulk_lookup (struct rte_kv_hash_table *ht,void *keys_p, +uint32_t *hashes,void *values_p,unsigned int n) { +struct k32v64_hash_table *table = (struct k32v64_hash_table *)ht; +uint32_t *keys = keys_p; +uint64_t *values = values_p; +int ret, cnt =0 ; +unsigned int i; + +if (unlikely ((table ==NULL ) || (keys ==NULL ) || (hashes ==NULL ) || + (values ==NULL ))) +return -EINVAL ; + +for (i =0 ; i < n; i++) { + ret =k32v64_hash_lookup (table, keys[i], hashes[i], + &values[i]); +if (ret ==0 ) + cnt++; + } +return cnt; +} + +#ifdef CC_AVX512VL_SUPPORT +int +k32v64_hash_bulk_lookup_avx512vl (struct rte_kv_hash_table *ht, +void *keys_p,uint32_t *hashes,void *values_p,unsigned int n); +#endif + +static rte_kv_hash_bulk_lookup_t +get_lookup_bulk_fn (void ) +{ +#ifdef CC_AVX512VL_SUPPORT +if (rte_cpu_get_flag_enabled (RTE_CPUFLAG_AVX512VL )) +return k32v64_hash_bulk_lookup_avx512vl;#endif +return k32v64_hash_bulk_lookup; +} + +static int +k32v64_hash_add (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t value,uint64_t *old_value,int *found) { +uint32_t bucket; +int i, idx, ret; +uint8_t msk; +struct k32v64_ext_ent *tmp, *ent, *prev =NULL ; + +if (table ==NULL ) +return -EINVAL ; +
lib/librte_hash/k32v64_hash.c line 80
Quoted 10 lines
lib/librte_hash/k32v64_hash.c new file +10
-0+ bucket = hash & table->bucket_msk; +/* Search key in table. Update value if exists */ +for (i =0 ; i <K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && + (table->t[bucket].key_mask & (1 << i))) { + *old_value = table->t[bucket].val[i]; + *found =1 ; +__atomic_fetch_add (&table->t[bucket].cnt,1 , + __ATOMIC_ACQUIRE); + table->t[bucket].val[i] = value;
Suggest using C11 builtin to store value. As far as I know all the supported platforms in DPDK have 64b atomic store in 32b mode. With this we will be able to avoid incrementing the counter. The reader will either get old or the new value.
Quoted 4 lines
lib/librte_hash/k32v64_hash.c new file +4
-0+__atomic_fetch_add (&table->t[bucket].cnt,1 , + __ATOMIC_RELEASE); +return 0 ; + }
lib/librte_hash/k32v64_hash.c line 94
Quoted 10 lines
lib/librte_hash/k32v64_hash.c new file +10
-0+ } + +if (!SLIST_EMPTY (&table->t[bucket].head)) { + SLIST_FOREACH(ent, &table->t[bucket].head, next) { +if (ent->key == key) { + *old_value = ent->val; + *found =1 ; +__atomic_fetch_add (&table->t[bucket].cnt,1 , + __ATOMIC_ACQUIRE); + ent->val = value;
Same here. __atomic_store(&ent->val, value, __ATOMIC_RELAXED).
Quoted 5 lines
lib/librte_hash/k32v64_hash.c new file +5
-0+__atomic_fetch_add (&table->t[bucket].cnt,1 , + __ATOMIC_RELEASE); +return 0 ; + } + }
lib/librte_hash/k32v64_hash.c line 109
Quoted 10 lines
lib/librte_hash/k32v64_hash.c new file +10
-0+ } + + msk = ~table->t[bucket].key_mask &VALID_KEY_MSK ; +if (msk) { + idx =__builtin_ctz (msk); + table->t[bucket].key[idx] = key; + table->t[bucket].val[idx] = value; +__atomic_or_fetch (&table->t[bucket].key_mask,1 << idx, + __ATOMIC_RELEASE); + table->nb_ent++;
Looks like bucket counter logic is needed for this case too. Please see the comments below in k32v64_hash_delete.
lib/librte_hash/k32v64_hash.c line 121
Quoted 12 lines
lib/librte_hash/k32v64_hash.c new file +12
-0+ *found =0 ; +return 0 ; + } + + ret =rte_mempool_get (table->ext_ent_pool, (void **)&ent); +if (ret <0 ) +return ret; + +SLIST_NEXT (ent, next) =NULL ; + ent->key = key; + ent->val = value; +rte_smp_wmb ();
We need to avoid using rte_smp_* barriers as we are adopting C11 built-in atomics. See the below comment.
lib/librte_hash/k32v64_hash.c line 128
Quoted 7 lines
lib/librte_hash/k32v64_hash.c new file +7
-0+SLIST_FOREACH (tmp, &table->t[bucket].head, next) + prev = tmp; + +if (prev ==NULL ) +SLIST_INSERT_HEAD (&table->t[bucket].head, ent, next); +else +SLIST_INSERT_AFTER (prev, ent, next);
Both the inserts need to use release order when the 'ent' is inserted. I am not sure where the SLIST is being picked up from. But looking at the SLIST implementation in 'lib/librte_eal/windows/include/sys/queue.h', it is not implemented using C11. I think we could move queue.h from windows directory to a common directory and change the SLIST implementation.
Quoted 38 lines
lib/librte_hash/k32v64_hash.c new file +31
-0+ + table->nb_ent++; + table->nb_ext_ent++; + *found =0 ; +return 0 ; +} + +static int +k32v64_hash_delete (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *old_value) +{ +uint32_t bucket; +int i; +struct k32v64_ext_ent *ent; + +if (table ==NULL ) +return -EINVAL ; + + bucket = hash & table->bucket_msk; + +for (i =0 ; i <K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && + (table->t[bucket].key_mask & (1 << i))) { + *old_value = table->t[bucket].val[i]; + ent =SLIST_FIRST (&table->t[bucket].head); +if (ent) { +__atomic_fetch_add (&table->t[bucket].cnt,1 , + __ATOMIC_ACQUIRE); + table->t[bucket].key[i] = ent->key; + table->t[bucket].val[i] = ent->val; + SLIST_REMOVE_HEAD(&table->t[bucket].head,next);
lib/librte_hash/k32v64_hash.c new file +5
-0+__atomic_fetch_add (&table->t[bucket].cnt,1 , + __ATOMIC_RELEASE); + table->nb_ext_ent--; + }else + __atomic_and_fetch(&table-Quoted 1 line
t[bucket].key_mask,
lib/librte_hash/k32v64_hash.c line 165
Quoted 1 line
lib/librte_hash/k32v64_hash.c new file +1
-0+ ~(1 << i), __ATOMIC_RELEASE);
(taking note from your responses to my comments in v3)
It is possible that the reader might match the old key but get the new value.
1) Reader: successful key match
2) Writer: k32v64_hash_delete followed by k32v64_hash_add
3) Reader: reads the value
IMO, there are 2 ways to solve this issue:
1) Use the bucket count logic while adding an entry in the non-extended bucket (marked it in k32v64_hash_add).
2) Do not use the entry in the bucket till the readers indicate that they have stopped using the entry
Quoted 37 lines
lib/librte_hash/k32v64_hash.c new file +36
-0+if (ent) +rte_mempool_put (table->ext_ent_pool, ent); + table->nb_ent--; +return 0 ; + } + } + + SLIST_FOREACH(ent, &table->t[bucket].head, next) +if (ent->key == key) +break ; + +if (ent ==NULL ) +return -ENOENT ; + + *old_value = ent->val; + +__atomic_fetch_add (&table->t[bucket].cnt,1 , __ATOMIC_ACQUIRE); +SLIST_REMOVE (&table->t[bucket].head, ent, k32v64_ext_ent, next); +__atomic_fetch_add (&table->t[bucket].cnt,1 , __ATOMIC_RELEASE); +rte_mempool_put (table->ext_ent_pool, ent); + + table->nb_ext_ent--; + table->nb_ent--; + +return 0 ; +} + +static int +k32v64_modify (struct rte_kv_hash_table *table,void *key_p,uint32_t hash, +enum rte_kv_modify_op op,void *value_p,int *found) { +struct k32v64_hash_table *ht = (struct k32v64_hash_table *)table; +uint32_t *key = key_p; +uint64_t value; + +if ((ht ==NULL ) || (key ==NULL ) || (value_p ==NULL ) || + (found ==NULL ) || (op >=RTE_KV_MODIFY_OP_MAX )){
lib/librte_hash/k32v64_hash.c
Quoted 2 lines
lib/librte_hash/k32v64_hash.c new file +2
-0+return -EINVAL ; + }
Suggest doing this check in rte_kv_hash_add/rte_kv_hash_delete. Then every implementation does not have to do this.
lib/librte_hash/k32v64_hash.c
Quoted 2 lines
lib/librte_hash/k32v64_hash.c new file +2
-0+ + value = *(uint64_t *)value_p;
In the API 'rte_kv_hash_add', value_p is 'void *' which does not convey that it is a pointer to 64b data. What happens while running on 32b systems?
Quoted 5 lines
lib/librte_hash/k32v64_hash.c new file +3
-0+switch (op) { +case RTE_KV_MODIFY_ADD : + return k32v64_hash_add(ht, *key, hash, value, value_p,found);
lib/librte_hash/k32v64_hash.c new file +1
-0+case RTE_KV_MODIFY_DEL :
lib/librte_hash/k32v64_hash.c
Quoted 10 lines
lib/librte_hash/k32v64_hash.c new file +10
-0+return k32v64_hash_delete (ht, *key, hash, value_p); +default : +break ; + } + +return -EINVAL ; +} + +struct rte_kv_hash_table * +k32v64_hash_create (const struct rte_kv_hash_params *params) {
This is a private symbol, I think it needs to have '__rte' suffix?
lib/librte_hash/k32v64_hash.c line 232
Quoted 11 lines
lib/librte_hash/k32v64_hash.c new file +11
-0+char hash_name[RTE_KV_HASH_NAMESIZE ]; +struct k32v64_hash_table *ht =NULL ; +uint32_t mem_size, nb_buckets, max_ent; +int ret; +struct rte_mempool *mp; + +if ((params ==NULL ) || (params->name ==NULL ) || + (params->entries ==0 )) { + rte_errno =EINVAL ; +return NULL ; + }
nit, these checks were done already in 'rte_kv_hash_create', these checks can be skipped.
lib/librte_hash/k32v64_hash.c line 238
Quoted 7 lines
lib/librte_hash/k32v64_hash.c new file +2
-0+ + ret = snprintf(hash_name, sizeof(hash_name), "KV_%s", params-Quoted 1 line
name);
lib/librte_hash/k32v64_hash.c new file +4
-0+if (ret <0 || ret >=RTE_KV_HASH_NAMESIZE ) { + rte_errno =ENAMETOOLONG ; +return NULL ; + }
Same here, this is checked in the calling function.
Quoted 21 lines
lib/librte_hash/k32v64_hash.c new file +21
-0+ + max_ent =rte_align32pow2 (params->entries); + nb_buckets = max_ent /K32V64_KEYS_PER_BUCKET ; + mem_size =sizeof (struct k32v64_hash_table ) + +sizeof (struct k32v64_hash_bucket ) * nb_buckets; + + mp =rte_mempool_create (hash_name, max_ent, +sizeof (struct k32v64_ext_ent ),0 ,0 ,NULL ,NULL ,NULL ,NULL , + params->socket_id,0 ); + +if (mp ==NULL ) +return NULL ; + + ht =rte_zmalloc_socket (hash_name, mem_size, +RTE_CACHE_LINE_SIZE , params->socket_id); +if (ht ==NULL ) { +rte_mempool_free (mp); +return NULL ; + } + +memcpy (ht->pub.name, hash_name,sizeof (ht->pub.name));
lib/librte_hash/k32v64_hash.c
Quoted 10 lines
lib/librte_hash/k32v64_hash.c new file +10
-0+ ht->max_ent = max_ent; + ht->bucket_msk = nb_buckets -1 ; + ht->ext_ent_pool = mp; + ht->pub.lookup =get_lookup_bulk_fn (); + ht->pub.modify = k32v64_modify; + +return (struct rte_kv_hash_table *)ht; } + +void +k32v64_hash_free (struct rte_kv_hash_table *ht) {
This is a private symbol, I think it needs to have '__rte' suffix?
Quoted 41 lines
lib/librte_hash/k32v64_hash.c new file +6
-0+if (ht ==NULL ) +return ; + +rte_mempool_free (((struct k32v64_hash_table *)ht)->ext_ent_pool); +rte_free (ht); +}lib/librte_hash/k32v64_hash.h new file +30
-0@@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include <rte_kv_hash.h> + +#define K32V64_KEYS_PER_BUCKET 4 +#define K32V64_WRITE_IN_PROGRESS 1 +#define VALID_KEY_MSK ((1 << K32V64_KEYS_PER_BUCKET) - 1) + +struct k32v64_ext_ent { +SLIST_ENTRY (k32v64_ext_ent) next; +uint32_t key; +uint64_t val; +}; + +struct k32v64_hash_bucket { +uint32_t key[K32V64_KEYS_PER_BUCKET ]; +uint64_t val[K32V64_KEYS_PER_BUCKET ]; +uint8_t key_mask; +uint32_t cnt; + SLIST_HEAD(k32v64_list_head, k32v64_ext_ent) head; } +__rte_cache_aligned; + +struct k32v64_hash_table { +struct rte_kv_hash_table pub;/**< Public part */ +uint32_t nb_ent;/**< Number of entities in the table*/ +uint32_t nb_ext_ent;/**< Number of extended entities */ +uint32_t max_ent;/**< Maximum number of entities */ +uint32_t bucket_msk;
lib/librte_hash/k32v64_hash.h line 40
Quoted 10 lines
lib/librte_hash/k32v64_hash.h new file +10
-0+struct rte_mempool *ext_ent_pool; + __extension__struct k32v64_hash_bucket t[0 ]; +}; + +typedef int (*k32v64_cmp_fn_t ) +(struct k32v64_hash_bucket *bucket,uint32_t key,uint64_t *val); + +static inline int +__kv_cmp_keys (struct k32v64_hash_bucket *bucket,uint32_t key, +uint64_t *val)
Changing __kv_cmp_keys to __k32v64_cmp_keys would be consistent
lib/librte_hash/k32v64_hash.h line 50
Quoted 10 lines
lib/librte_hash/k32v64_hash.h new file +10
-0+{ +int i; + +for (i =0 ; i <K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == bucket->key[i]) && + (bucket->key_mask & (1 << i))) { + *val = bucket->val[i]; +return 1 ; + } + }
You have to load-acquire 'key_mask' (corresponding to store-release on 'key_mask' in add). Suggest changing this as follows:
__atomic_load(&bucket->key_mask, &key_mask, __ATOMIC_ACQUIRE);
for (i = 0; i < K32V64_KEYS_PER_BUCKET; i++) {
if ((key == bucket->key[i]) && (key_mask & (1 << i))) {
*val = bucket->val[i];
return 1;
}
}Quoted 9 lines
lib/librte_hash/k32v64_hash.h new file +9
-0+ +return 0 ; +} + +static inline int +__k32v64_hash_lookup (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *value,k32v64_cmp_fn_t cmp_f) { +uint64_t val =0 ; +struct k32v64_ext_ent *ent;
lib/librte_hash/k32v64_hash.h line 70
Quoted 10 lines
lib/librte_hash/k32v64_hash.h new file +10
-0+uint32_t cnt; +int found =0 ; +uint32_t bucket = hash & table->bucket_msk; + + do { + + do { + cnt = __atomic_load_n(&table->t[bucket].cnt, + __ATOMIC_ACQUIRE); + } while (unlikely(cnt & K32V64_WRITE_IN_PROGRESS));
Agree that it is a small section. But the issue can happen. This also makes the algorithm un-acceptable in many use cases. IMHO, since we have identified the issue, we should fix it.
The issue is mainly due to not following the reader-writer concurrency design principles. i.e. the data that we want to communicate from writer to reader (key and value in this case) is not being communicated atomically. For ex: in rte_hash/cuckoo hash library, you can see that the key and pData are being communicated atomically using the key store index.
I might be wrong, but, I do not think we can make this block-free (readers move forward even when the writer is not scheduled) using bucket counter.
This problem does not exist in existing rte_hash library. It might not be performing as good as this, but it is block-free.
Quoted 186 lines
lib/librte_hash/k32v64_hash.h new file +28
-0+ + found =cmp_f (&table->t[bucket], key, &val); +if (unlikely ((found ==0 ) && + (!SLIST_EMPTY (&table->t[bucket].head)))) { + SLIST_FOREACH(ent, &table->t[bucket].head, next) { +if (ent->key == key) { + val = ent->val; + found =1 ; +break ; + } + } + } +__atomic_thread_fence (__ATOMIC_RELEASE); + }while (unlikely (cnt !=__atomic_load_n (&table->t[bucket].cnt, + __ATOMIC_RELAXED))); + +if (found ==1 ) { + *value = val; +return 0 ; + }else +return -ENOENT ; +} + +struct rte_kv_hash_table * +k32v64_hash_create (const struct rte_kv_hash_params *params); + +void +k32v64_hash_free (struct rte_kv_hash_table *ht);lib/librte_hash/k32v64_hash_avx512vl.c new file +8
-0@@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include "k32v64_hash.h" + +int +k32v64_hash_bulk_lookup_avx512vl(struct rte_kv_hash_table *ht, void
*keys_p,lib/librte_hash/k32v64_hash_avx512vl.c new file +4
-0+ uint32_t *hashes, void *values_p, unsigned int n); + +static inline int +k32v64_cmp_keys_avx512vl(struct k32v64_hash_bucket *bucket, uint32_tkey,
lib/librte_hash/k32v64_hash_avx512vl.c new file +9
-0+ uint64_t *val) +{ +__m128i keys, srch_key; +__mmask8 msk; + + keys =_mm_load_si128 ((void *)bucket); + srch_key =_mm_set1_epi32 (key); + + msk = _mm_mask_cmpeq_epi32_mask(bucket->key_mask, keys,
srch_key);lib/librte_hash/k32v64_hash_avx512vl.c new file +10
-0+if (msk) { + *val = bucket->val[__builtin_ctz (msk)]; +return 1 ; + } + +return 0 ; +} + +static inline int +k32v64_hash_lookup_avx512vl(struct k32v64_hash_table *table, uint32_tkey,
lib/librte_hash/k32v64_hash_avx512vl.c new file +8
-0+ uint32_t hash, uint64_t *value) +{ + return __k32v64_hash_lookup(table, key, hash, value, +k32v64_cmp_keys_avx512vl ); +} + +int +k32v64_hash_bulk_lookup_avx512vl(struct rte_kv_hash_table *ht, void
*keys_p,lib/librte_hash/k32v64_hash_avx512vl.c new file +19
-0+ uint32_t *hashes, void *values_p, unsigned int n) { + struct k32v64_hash_table *table = (struct k32v64_hash_table *)ht; +uint32_t *keys = keys_p; +uint64_t *values = values_p; +int ret, cnt =0 ; +unsigned int i; + +if (unlikely ((table ==NULL ) || (keys ==NULL ) || (hashes ==NULL ) || + (values ==NULL ))) +return -EINVAL ; + +for (i =0 ; i < n; i++) { + ret =k32v64_hash_lookup_avx512vl (table, keys[i], hashes[i], + &values[i]); +if (ret ==0 ) + cnt++; + } +return cnt; +}lib/librte_hash/meson.build index renamed from lib/librte_hash/meson.build +0
-06ab46ae..0d014ea 100644
lib/librte_hash/meson.build +15
-1@@ -3,10 +3,23 @@ headers = files('rte_crc_arm64.h', 'rte_fbk_hash.h', + 'rte_kv_hash.h', 'rte_hash_crc.h', 'rte_hash.h', 'rte_jhash.h', 'rte_thash.h') -sources = files('rte_cuckoo_hash.c', 'rte_fbk_hash.c') -deps += ['ring'] +sources = files('rte_cuckoo_hash.c', 'rte_fbk_hash.c', 'rte_kv_hash.c', +'k32v64_hash.c') deps += ['ring', 'mempool'] + +if dpdk_conf.has('RTE_ARCH_X86') + if cc.has_argument('-mavx512vl') + avx512_tmplib = static_library('avx512_tmp', + 'k32v64_hash_avx512vl.c', + dependencies: static_rte_mempool, + c_args: cflags + ['-mavx512vl']) + objs += avx512_tmplib.extract_objects('k32v64_hash_avx512vl.c') + cflags += '-DCC_AVX512VL_SUPPORT' + + endif +endiflib/librte_hash/rte_hash_version.map +5
-1@@ -36,5 +36,9 @@ EXPERIMENTAL { rte_hash_lookup_with_hash_bulk; rte_hash_lookup_with_hash_bulk_data; rte_hash_max_key_id; - + rte_kv_hash_create; + rte_kv_hash_find_existing; + rte_kv_hash_free; + rte_kv_hash_add; + rte_kv_hash_delete; };lib/librte_hash/rte_kv_hash.c new renamed from lib/librte_hash/rte_kv_hash.c +0
-0file mode 100644 index 0000000..03df8db
lib/librte_hash/rte_kv_hash.c new file +38
-0@@ -0,0 +1,184 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include <string.h> + +#include <rte_eal_memconfig.h> +#include <rte_errno.h> +#include <rte_malloc.h> +#include <rte_memory.h> +#include <rte_tailq.h> + +#include <rte_kv_hash.h> +#include "k32v64_hash.h" + +TAILQ_HEAD (rte_kv_hash_list, rte_tailq_entry); + +static struct rte_tailq_elem rte_kv_hash_tailq = { + .name ="RTE_KV_HASH" , +}; + +EAL_REGISTER_TAILQ (rte_kv_hash_tailq); + +int +rte_kv_hash_add (struct rte_kv_hash_table *table,void *key, +uint32_t hash,void *value,int *found) { +if (table ==NULL ) +return -EINVAL ; + +return table->modify (table, key, hash,RTE_KV_MODIFY_ADD , + value, found); +} + +int +rte_kv_hash_delete (struct rte_kv_hash_table *table,void *key, +uint32_t hash,void *value) +{ +int found;
lib/librte_hash/rte_kv_hash.c line 48
Quoted 10 lines
lib/librte_hash/rte_kv_hash.c new file +10
-0+ +if (table ==NULL ) +return -EINVAL ; + +return table->modify (table, key, hash,RTE_KV_MODIFY_DEL , + value, &found); +} + +struct rte_kv_hash_table * +rte_kv_hash_find_existing (const char *name) {
I did not see a test case for this. Please add a test case for 'rte_kv_hash_find_existing'.
Quoted 40 lines
lib/librte_hash/rte_kv_hash.c new file +40
-0+struct rte_kv_hash_table *h =NULL ; +struct rte_tailq_entry *te; +struct rte_kv_hash_list *kv_hash_list; + + kv_hash_list =RTE_TAILQ_CAST (rte_kv_hash_tailq.head, + rte_kv_hash_list); + +rte_mcfg_tailq_read_lock (); + TAILQ_FOREACH(te, kv_hash_list, next) { + h = (struct rte_kv_hash_table *) te->data; +if (strncmp (name, h->name,RTE_KV_HASH_NAMESIZE ) ==0 ) +break ; + } +rte_mcfg_tailq_read_unlock (); +if (te ==NULL ) { + rte_errno =ENOENT ; +return NULL ; + } +return h; +} + +struct rte_kv_hash_table * +rte_kv_hash_create (const struct rte_kv_hash_params *params) { +char hash_name[RTE_KV_HASH_NAMESIZE ]; +struct rte_kv_hash_table *ht, *tmp_ht =NULL ; +struct rte_tailq_entry *te; +struct rte_kv_hash_list *kv_hash_list; +int ret; + +if ((params ==NULL ) || (params->name ==NULL ) || + (params->entries ==0 ) || + (params->type >=RTE_KV_HASH_MAX )) { + rte_errno =EINVAL ; +return NULL ; + } + + kv_hash_list =RTE_TAILQ_CAST (rte_kv_hash_tailq.head, + rte_kv_hash_list); + + ret = snprintf(hash_name, sizeof(hash_name), "KV_%s", params-
Quoted 1 line
Quoted 1 line
name);
RTE_KV_HASH_NAMESIZE is a public #define. It is natural for the user to use this to define the size of the hash table name. Now, we are taking 3 characters from that space. I think it is better to increase the size of 'hash_name' by 3 characters or skip adding 'KV_' to the name.
This also has an impact on 'rte_kv_hash_find_existing' where the user passed string is being used as is without adding 'KV_'. Suggest adding a test case for 'rte_kv_hash_find_existing'.
Quoted 72 lines
lib/librte_hash/rte_kv_hash.c new file +72
-0+if (ret <0 || ret >=RTE_KV_HASH_NAMESIZE ) { + rte_errno =ENAMETOOLONG ; +return NULL ; + } + +switch (params->type) { +case RTE_KV_HASH_K32V64 : + ht =k32v64_hash_create (params); +break ; +default : + rte_errno =EINVAL ; +return NULL ; + } +if (ht ==NULL ) +return ht; + +rte_mcfg_tailq_write_lock (); + TAILQ_FOREACH(te, kv_hash_list, next) { + tmp_ht = (struct rte_kv_hash_table *) te->data; +if (strncmp (params->name, tmp_ht->name, +RTE_KV_HASH_NAMESIZE ) ==0 ) +break ; + } +if (te !=NULL ) { + rte_errno =EEXIST ; + goto exit; + } + + te =rte_zmalloc ("KV_HASH_TAILQ_ENTRY" ,sizeof (*te),0 ); +if (te ==NULL ) { +RTE_LOG (ERR ,HASH ,"Failed to allocate tailq entry\n" ); + goto exit; + } + + ht->type = params->type; + te->data = (void *)ht; +TAILQ_INSERT_TAIL (kv_hash_list, te, next); + +rte_mcfg_tailq_write_unlock (); + +return ht; + +exit: +rte_mcfg_tailq_write_unlock (); +switch (params->type) { +case RTE_KV_HASH_K32V64 : +k32v64_hash_free (ht); +break ; +default : +break ; + } +return NULL ; +} + +void +rte_kv_hash_free (struct rte_kv_hash_table *ht) { +struct rte_tailq_entry *te; +struct rte_kv_hash_list *kv_hash_list; + +if (ht ==NULL ) +return ; + + kv_hash_list =RTE_TAILQ_CAST (rte_kv_hash_tailq.head, + rte_kv_hash_list); + +rte_mcfg_tailq_write_lock (); + +/* find out tailq entry */ + TAILQ_FOREACH(te, kv_hash_list, next) { +if (te->data == (void *) ht) +break ; + }
lib/librte_hash/rte_kv_hash.c line 174
Quoted 10 lines
lib/librte_hash/rte_kv_hash.c new file +10
-0+ + +if (te ==NULL ) { +rte_mcfg_tailq_write_unlock (); +return ; + } + +TAILQ_REMOVE (kv_hash_list, te, next); + +rte_mcfg_tailq_write_unlock ();
I understand that the free is not thread safe. But, it might be safer if unlock is after the call to 'k32v64_hash_free'.
Quoted 32 lines
lib/librte_hash/rte_kv_hash.c new file +10
-0+ +switch (ht->type) { +case RTE_KV_HASH_K32V64 : +k32v64_hash_free (ht); +break ; +default : +break ; + } +rte_free (te); +}lib/librte_hash/rte_kv_hash.h new renamed from lib/librte_hash/rte_kv_hash.h +0
-0file mode 100644 index 0000000..c0375d1
lib/librte_hash/rte_kv_hash.h new file +17
-0@@ -0,0 +1,169 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#ifndef _RTE_KV_HASH_H_ +#define _RTE_KV_HASH_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <rte_compat.h> +#include <rte_atomic.h> +#include <rte_mempool.h> + +#define RTE_KV_HASH_NAMESIZE 32 +
lib/librte_hash/rte_kv_hash.h line 27
Quoted 10 lines
lib/librte_hash/rte_kv_hash.h new file +10
-0+enum rte_kv_hash_type { +RTE_KV_HASH_K32V64 , +RTE_KV_HASH_MAX +}; + +enum rte_kv_modify_op { +RTE_KV_MODIFY_ADD , +RTE_KV_MODIFY_DEL , +RTE_KV_MODIFY_OP_MAX +};
This could be in a private header file.
lib/librte_hash/rte_kv_hash.h line 34
Quoted 7 lines
lib/librte_hash/rte_kv_hash.h new file +7
-0+ +struct rte_kv_hash_params { +const char *name; +uint32_t entries; +int socket_id; +enum rte_kv_hash_type type; +};
Since this is a public structure, suggest adding some reserved or flags field which are ignored now but could be used in the future for enhancements.
Quoted 5 lines
lib/librte_hash/rte_kv_hash.h new file +5
-0+ +struct rte_kv_hash_table ; + +typedef int (*rte_kv_hash_bulk_lookup_t) (struct rte_kv_hash_table +*table, void *keys, uint32_t *hashes,
lib/librte_hash/rte_kv_hash.h line 49
Quoted 10 lines
lib/librte_hash/rte_kv_hash.h new file +10
-0+ void *values, unsigned int n); + +typedef int (*rte_kv_hash_modify_t ) +(struct rte_kv_hash_table *table,void *key,uint32_t hash, +enum rte_kv_modify_op op,void *value,int *found); + +struct rte_kv_hash_table { +char name[RTE_KV_HASH_NAMESIZE ];/**< Name of the hash. */ +rte_kv_hash_bulk_lookup_t lookup; +rte_kv_hash_modify_t modify;
There are separate APIs provided for add and delete. Is there any advantage for combining add/delete into a single function pointer in the backend? If we keep 2 separate pointers, we can get rid of 'enum rte_kv_modify_op'. It is simple to understand as well.
lib/librte_hash/rte_kv_hash.h line 55
Quoted 6 lines
lib/librte_hash/rte_kv_hash.h new file +6
-0+enum rte_kv_hash_type type; +}; + +/** +* Lookup bulk of keys. +* This function is multi-thread safe with regarding to other lookup threads.
It is safe with respect to the writer as well (reader-writer concurrency), please capture this as well.
lib/librte_hash/rte_kv_hash.h line 58
Quoted 3 lines
lib/librte_hash/rte_kv_hash.h new file +3
-0+* +* @param table +* Hash table to add the key to.
nit, 'Hash table to lookup the keys'
Quoted 6 lines
lib/librte_hash/rte_kv_hash.h new file +6
-0+* @param keys +* Pointer to array of keys +* @param hashes +* Pointer to array of hash values associated with keys. +* @param values +* Pointer to array of value corresponded to keys
lib/librte_hash/rte_kv_hash.h
Quoted 10 lines
lib/librte_hash/rte_kv_hash.h new file +10
-0+* If the key was not found the corresponding value remains intact. +* @param n +* Number of keys to lookup in batch. +* @return +* -EINVAL if there's an error, otherwise number of successful lookups. +*/ +static inline int +rte_kv_hash_bulk_lookup (struct rte_kv_hash_table *table, +void *keys,uint32_t *hashes,void *values,unsigned int n) { +return table->lookup (table, keys, hashes, values, n); }
Consider making this a non-inline function. This is a bulk lookup and I think cost of calling a function should get amortized well.
This will also allow for hiding the 'struct rte_kv_hash_table' from the user which is better for ABI. You can move the definition of the 'struct rte_kv_hash_table' and function pointer declarations to a private header file.
Quoted 4 lines
lib/librte_hash/rte_kv_hash.h new file +4
-0+ +/** +* Add a key to an existing hash table with hash value. +* This operation is not multi-thread safe regarding to add/delete
lib/librte_hash/rte_kv_hash.h line 89
Quoted 10 lines
lib/librte_hash/rte_kv_hash.h new file +10
-0+functions + * and should only be called from one thread. + * However it is safe to call it along with lookup. + * + * @param table + * Hash table to add the key to. + * @param key + * Key to add to the hash table. + * @param value + * Value to associate with key.
I think it needs to be called out here that it the data is of size 64b (even in a 32b system) because of the implementation in function 'k32v64_modify'. Why not make 'value' of type 'uint64_t *'?
Quoted 84 lines
lib/librte_hash/rte_kv_hash.h new file +82
-1+* @param hash +* Hash value associated with key. +* @found +* 0 if no previously added key was found +* 1 previously added key was found, old value associated with the key +* was written to *value +* @return +* 0 if ok, or negative value on error. +*/ +__rte_experimental +int +rte_kv_hash_add (struct rte_kv_hash_table *table,void *key, +uint32_t hash,void *value,int *found); + +/** +* Remove a key with a given hash value from an existing hash table. +* This operation is not multi-thread safe regarding to add/delete +functions +* and should only be called from one thread. +* However it is safe to call it along with lookup. +* +* @param table +* Hash table to remove the key from. +* @param key +* Key to remove from the hash table. +* @param hash +* hash value associated with key. +* @param value +* pointer to memory where the old value will be written to on success +* @return +* 0 if ok, or negative value on error. +*/ +__rte_experimental +int +rte_kv_hash_delete (struct rte_kv_hash_table *table,void *key, +uint32_t hash,void *value); + +/** +* Performs a lookup for an existing hash table, and returns a pointer +to +* the table if found. +* +* @param name +* Name of the hash table to find +* +* @return +* pointer to hash table structure or NULL on error with rte_errno +* set appropriately. +*/ +__rte_experimental +struct rte_kv_hash_table * +rte_kv_hash_find_existing (const char *name); + +/** +* Create a new hash table for use with four byte keys. +* +* @param params +* Parameters used in creation of hash table. +* +* @return +* Pointer to hash table structure that is used in future hash table +* operations, or NULL on error with rte_errno set appropriately. +*/ +__rte_experimental +struct rte_kv_hash_table * +rte_kv_hash_create (const struct rte_kv_hash_params *params); + +/** +* Free all memory used by a hash table. +* +* @param table +* Hash table to deallocate. +*/ +__rte_experimental +void +rte_kv_hash_free (struct rte_kv_hash_table *table); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_KV_HASH_H_ */ --2.7.4
Re: [dpdk-dev] [PATCH v4 1/4] hash: add kv hash library
In reply to Ananyev, KonstantinHi Konstantin,
Thanks for the review. See below
Quoted 106 lines · Konstantin
Hi Vladimir,
Quoted 104 lines
lib/librte_hash/k32v64_hash.c new file +101
-0@@ -0,0 +1,277 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include <string.h> + +#include <rte_errno.h> +#include <rte_malloc.h> +#include <rte_memory.h> + +#include "k32v64_hash.h" + +static inline int +k32v64_hash_lookup (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *value) +{ +return __k32v64_hash_lookup (table, key, hash, value, __kv_cmp_keys); +} + +static int +k32v64_hash_bulk_lookup (struct rte_kv_hash_table *ht,void *keys_p, +uint32_t *hashes,void *values_p,unsigned int n) +{ +struct k32v64_hash_table *table = (struct k32v64_hash_table *)ht; +uint32_t *keys = keys_p; +uint64_t *values = values_p; +int ret, cnt =0 ; +unsigned int i; + +if (unlikely ((table ==NULL ) || (keys ==NULL ) || (hashes ==NULL ) || +(values ==NULL ))) +return -EINVAL ; + +for (i =0 ; i < n; i++) { +ret =k32v64_hash_lookup (table, keys[i], hashes[i], +&values[i]); +if (ret ==0 ) +cnt++; +} +return cnt; +} + +#ifdef CC_AVX512VL_SUPPORT +int +k32v64_hash_bulk_lookup_avx512vl (struct rte_kv_hash_table *ht, +void *keys_p,uint32_t *hashes,void *values_p,unsigned int n); +#endif + +static rte_kv_hash_bulk_lookup_t +get_lookup_bulk_fn (void ) +{ +#ifdef CC_AVX512VL_SUPPORT +if (rte_cpu_get_flag_enabled (RTE_CPUFLAG_AVX512VL )) +return k32v64_hash_bulk_lookup_avx512vl; +#endif +return k32v64_hash_bulk_lookup; +} + +static int +k32v64_hash_add (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t value,uint64_t *old_value,int *found) +{ +uint32_t bucket; +int i, idx, ret; +uint8_t msk; +struct k32v64_ext_ent *tmp, *ent, *prev =NULL ; + +if (table ==NULL ) +return -EINVAL ; + +bucket = hash & table->bucket_msk; +/* Search key in table. Update value if exists */ +for (i =0 ; i <K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && +(table->t[bucket].key_mask & (1 << i))) { +*old_value = table->t[bucket].val[i]; +*found =1 ; +__atomic_fetch_add (&table->t[bucket].cnt,1 , +__ATOMIC_ACQUIRE); +table->t[bucket].val[i] = value; +__atomic_fetch_add (&table->t[bucket].cnt,1 , +__ATOMIC_RELEASE); +return 0 ; +} +} + +if (!SLIST_EMPTY (&table->t[bucket].head)) { +SLIST_FOREACH(ent, &table->t[bucket].head, next) { +if (ent->key == key) { +*old_value = ent->val; +*found =1 ; +__atomic_fetch_add (&table->t[bucket].cnt,1 , +__ATOMIC_ACQUIRE); +ent->val = value; +__atomic_fetch_add (&table->t[bucket].cnt,1 , +__ATOMIC_RELEASE); +return 0 ; +} +} +} +
Quoted 8 lines
lib/librte_hash/k32v64_hash.c line 108
Quoted 7 lines
lib/librte_hash/k32v64_hash.c new file +7
-0+msk = ~table->t[bucket].key_mask &VALID_KEY_MSK ; +if (msk) { +idx =__builtin_ctz (msk); +table->t[bucket].key[idx] = key; +table->t[bucket].val[idx] = value; +__atomic_or_fetch (&table->t[bucket].key_mask,1 << idx, +__ATOMIC_RELEASE);I think this case also has to guarded with table->t[bucket].cnt updates.
Agree, will change it.
Quoted 16 lines
Quoted 4 lines
lib/librte_hash/k32v64_hash.c new file +3
-0+table->nb_ent++; +*found =0 ; +return 0 ;lib/librte_hash/k32v64_hash.c line 121
Quoted 10 lines
lib/librte_hash/k32v64_hash.c new file +10
-0+} + +ret =rte_mempool_get (table->ext_ent_pool, (void **)&ent); +if (ret <0 ) +return ret; + +SLIST_NEXT (ent, next) =NULL ; +ent->key = key; +ent->val = value; +rte_smp_wmb ();
__atomic_thread_fence(__ATOMIC_RELEASE);
?
Sure, looks like I missed it, will fix
Quoted 38 lines
Quoted 38 lines
lib/librte_hash/k32v64_hash.c new file +37
-0+SLIST_FOREACH (tmp, &table->t[bucket].head, next) +prev = tmp; + +if (prev ==NULL ) +SLIST_INSERT_HEAD (&table->t[bucket].head, ent, next); +else +SLIST_INSERT_AFTER (prev, ent, next); + +table->nb_ent++; +table->nb_ext_ent++; +*found =0 ; +return 0 ; +} + +static int +k32v64_hash_delete (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *old_value) +{ +uint32_t bucket; +int i; +struct k32v64_ext_ent *ent; + +if (table ==NULL ) +return -EINVAL ; + +bucket = hash & table->bucket_msk; + +for (i =0 ; i <K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && +(table->t[bucket].key_mask & (1 << i))) { +*old_value = table->t[bucket].val[i]; +ent =SLIST_FIRST (&table->t[bucket].head); +if (ent) { +__atomic_fetch_add (&table->t[bucket].cnt,1 , +__ATOMIC_ACQUIRE); +table->t[bucket].key[i] = ent->key; +table->t[bucket].val[i] = ent->val;
Quoted 8 lines
lib/librte_hash/k32v64_hash.c line 165
Quoted 7 lines
lib/librte_hash/k32v64_hash.c new file +7
-0+SLIST_REMOVE_HEAD (&table->t[bucket].head, next); +__atomic_fetch_add (&table->t[bucket].cnt,1 , +__ATOMIC_RELEASE); +table->nb_ext_ent--; +}else +__atomic_and_fetch (&table->t[bucket].key_mask, +~(1 << i), __ATOMIC_RELEASE);Same thought as above - safer to guard this mask update with cnt update.
Agree
Quoted 4 lines
lib/librte_hash/k32v64_hash.c line 167
Quoted 3 lines
lib/librte_hash/k32v64_hash.c new file +2
-0+if (ent) +rte_mempool_put (table->ext_ent_pool, ent);Can't this 'if(ent)' be merged with previous 'if (ent) {...}' above?
Yes, will merge
Quoted 26 lines
Quoted 26 lines
lib/librte_hash/k32v64_hash.c new file +25
-0+table->nb_ent--; +return 0 ; +} +} + +SLIST_FOREACH(ent, &table->t[bucket].head, next) +if (ent->key == key) +break ; + +if (ent ==NULL ) +return -ENOENT ; + +*old_value = ent->val; + +__atomic_fetch_add (&table->t[bucket].cnt,1 , __ATOMIC_ACQUIRE); +SLIST_REMOVE (&table->t[bucket].head, ent, k32v64_ext_ent, next); +__atomic_fetch_add (&table->t[bucket].cnt,1 , __ATOMIC_RELEASE); +rte_mempool_put (table->ext_ent_pool, ent); + +table->nb_ext_ent--; +table->nb_ent--; + +return 0 ; +} +
Signature · 2 lines
Regards, Vladimir
Re: [dpdk-dev] [PATCH v4 1/4] hash: add kv hash library
In reply to Ananyev, KonstantinQuoted 33 lines · Konstantin
Quoted 33 lines
Hi Vladimir,
Quoted 31 lines
lib/librte_hash/k32v64_hash.c new file +28
-0@@ -0,0 +1,277 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include <string.h> + +#include <rte_errno.h> +#include <rte_malloc.h> +#include <rte_memory.h> + +#include "k32v64_hash.h" + +static inline int +k32v64_hash_lookup (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *value) +{ +return __k32v64_hash_lookup (table, key, hash, value, __kv_cmp_keys); +} + +static int +k32v64_hash_bulk_lookup (struct rte_kv_hash_table *ht,void *keys_p, +uint32_t *hashes,void *values_p,unsigned int n) +{ +struct k32v64_hash_table *table = (struct k32v64_hash_table *)ht; +uint32_t *keys = keys_p; +uint64_t *values = values_p; +int ret, cnt =0 ; +unsigned int i;
Quoted 8 lines
Quoted 4 lines
lib/librte_hash/k32v64_hash.c line 32
Quoted 4 lines
lib/librte_hash/k32v64_hash.c new file +4
-0+ +if (unlikely ((table ==NULL ) || (keys ==NULL ) || (hashes ==NULL ) || +(values ==NULL ))) +return -EINVAL ;As a nit - this formal parameter checking better to do in public function (
rte_kv_hash_bulk_lookup) before de-refencing table and calling actuallookup(). Same story formodify()- formal parameter checking can be done at the top of public function.
Agree, will move checking in public part
Quoted 2 lines
BTW, why to unite add/delete into
modify(), if internally you have 2 different functions (for add/delete) anyway?
This was done for future extensibility, if we want to add some additional control plane features in the future, other than add or delete. if you think it's unnecessary I'll change it with add()/del() pair.
Quoted 56 lines
Quoted 40 lines
Quoted 40 lines
lib/librte_hash/k32v64_hash.c new file +39
-0+ +for (i =0 ; i < n; i++) { +ret =k32v64_hash_lookup (table, keys[i], hashes[i], +&values[i]); +if (ret ==0 ) +cnt++; +} +return cnt; +} + +#ifdef CC_AVX512VL_SUPPORT +int +k32v64_hash_bulk_lookup_avx512vl (struct rte_kv_hash_table *ht, +void *keys_p,uint32_t *hashes,void *values_p,unsigned int n); +#endif + +static rte_kv_hash_bulk_lookup_t +get_lookup_bulk_fn (void ) +{ +#ifdef CC_AVX512VL_SUPPORT +if (rte_cpu_get_flag_enabled (RTE_CPUFLAG_AVX512VL )) +return k32v64_hash_bulk_lookup_avx512vl; +#endif +return k32v64_hash_bulk_lookup; +} + +static int +k32v64_hash_add (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t value,uint64_t *old_value,int *found) +{ +uint32_t bucket; +int i, idx, ret; +uint8_t msk; +struct k32v64_ext_ent *tmp, *ent, *prev =NULL ; + +if (table ==NULL ) +return -EINVAL ; + +bucket = hash & table->bucket_msk;Quoted 8 lines
lib/librte_hash/k32v64_hash.c line 79
Quoted 8 lines
lib/librte_hash/k32v64_hash.c new file +8
-0+/* Search key in table. Update value if exists */ +for (i =0 ; i <K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && +(table->t[bucket].key_mask & (1 << i))) { +*old_value = table->t[bucket].val[i]; +*found =1 ; +__atomic_fetch_add (&table->t[bucket].cnt,1 , +__ATOMIC_ACQUIRE);After another thought - atomic add is probably an overkill here. Something like:
voidupdate_start(struct k32v64_hash_bucket *bkt)
{bkt->cnt++; __atomic_thread_fence(__ATOMIC_ACQ_REL); }
Quoted 7 lines
void
update_finish(struct k32v64_hash_bucket *bkt){__atomic_thread_fence(__ATOMIC_ACQ_REL); bkt->cnt++;
}I think should be sufficient here.
Good point
Quoted 123 lines
Quoted 123 lines
Quoted 20 lines
lib/librte_hash/k32v64_hash.c new file +19
-0+table->t[bucket].val[i] = value; +__atomic_fetch_add (&table->t[bucket].cnt,1 , +__ATOMIC_RELEASE); +return 0 ; +} +} + +if (!SLIST_EMPTY (&table->t[bucket].head)) { +SLIST_FOREACH(ent, &table->t[bucket].head, next) { +if (ent->key == key) { +*old_value = ent->val; +*found =1 ; +__atomic_fetch_add (&table->t[bucket].cnt,1 , +__ATOMIC_ACQUIRE); +ent->val = value; +__atomic_fetch_add (&table->t[bucket].cnt,1 , +__ATOMIC_RELEASE); +return 0 ; +}lib/librte_hash/k32v64_hash.c line 108
Quoted 10 lines
lib/librte_hash/k32v64_hash.c new file +10
-0+} +} + +msk = ~table->t[bucket].key_mask &VALID_KEY_MSK ; +if (msk) { +idx =__builtin_ctz (msk); +table->t[bucket].key[idx] = key; +table->t[bucket].val[idx] = value; +__atomic_or_fetch (&table->t[bucket].key_mask,1 << idx, +__ATOMIC_RELEASE);I think this case also has to guarded with table->t[bucket].cnt updates.
lib/librte_hash/k32v64_hash.c line 121
Quoted 13 lines
lib/librte_hash/k32v64_hash.c new file +13
-0+table->nb_ent++; +*found =0 ; +return 0 ; +} + +ret =rte_mempool_get (table->ext_ent_pool, (void **)&ent); +if (ret <0 ) +return ret; + +SLIST_NEXT (ent, next) =NULL ; +ent->key = key; +ent->val = value; +rte_smp_wmb ();
__atomic_thread_fence(__ATOMIC_RELEASE);
?Quoted 34 lines
lib/librte_hash/k32v64_hash.c new file +34
-0+SLIST_FOREACH (tmp, &table->t[bucket].head, next) +prev = tmp; + +if (prev ==NULL ) +SLIST_INSERT_HEAD (&table->t[bucket].head, ent, next); +else +SLIST_INSERT_AFTER (prev, ent, next); + +table->nb_ent++; +table->nb_ext_ent++; +*found =0 ; +return 0 ; +} + +static int +k32v64_hash_delete (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *old_value) +{ +uint32_t bucket; +int i; +struct k32v64_ext_ent *ent; + +if (table ==NULL ) +return -EINVAL ; + +bucket = hash & table->bucket_msk; + +for (i =0 ; i <K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && +(table->t[bucket].key_mask & (1 << i))) { +*old_value = table->t[bucket].val[i]; +ent =SLIST_FIRST (&table->t[bucket].head); +if (ent) { +__atomic_fetch_add(&table->t[bucket].cnt, 1,lib/librte_hash/k32v64_hash.c line 165
Quoted 10 lines
lib/librte_hash/k32v64_hash.c new file +10
-0+__ATOMIC_ACQUIRE); +table->t[bucket].key[i] = ent->key; +table->t[bucket].val[i] = ent->val; +SLIST_REMOVE_HEAD (&table->t[bucket].head, next); +__atomic_fetch_add (&table->t[bucket].cnt,1 , +__ATOMIC_RELEASE); +table->nb_ext_ent--; +}else +__atomic_and_fetch (&table->t[bucket].key_mask, +~(1 << i), __ATOMIC_RELEASE);Same thought as above - safer to guard this mask update with cnt update.
lib/librte_hash/k32v64_hash.c line 167
Quoted 2 lines
lib/librte_hash/k32v64_hash.c new file +2
-0+if (ent) +rte_mempool_put (table->ext_ent_pool, ent);Can't this 'if(ent)' be merged with previous 'if (ent) {...}' above?
Quoted 25 lines
lib/librte_hash/k32v64_hash.c new file +25
-0+table->nb_ent--; +return 0 ; +} +} + +SLIST_FOREACH(ent, &table->t[bucket].head, next) +if (ent->key == key) +break ; + +if (ent ==NULL ) +return -ENOENT ; + +*old_value = ent->val; + +__atomic_fetch_add (&table->t[bucket].cnt,1 , __ATOMIC_ACQUIRE); +SLIST_REMOVE (&table->t[bucket].head, ent, k32v64_ext_ent, next); +__atomic_fetch_add (&table->t[bucket].cnt,1 , __ATOMIC_RELEASE); +rte_mempool_put (table->ext_ent_pool, ent); + +table->nb_ext_ent--; +table->nb_ent--; + +return 0 ; +} +
Signature · 2 lines
Regards, Vladimir
Re: [dpdk-dev] [PATCH v4 1/4] hash: add kv hash library
In reply to Wang, Yipeng1Hi Yipeng,
Thanks for the review. See below
Quoted 115 lines · On 24/06/2020 02:19, Wang, Yipeng1 wrote:
Quoted 115 lines
-----Original Message-----
From: Medvedkin, Vladimir <[email protected]>
Sent: Friday, May 8, 2020 12:59 PM
To: [email protected]
Cc: Ananyev, Konstantin <[email protected]>; Wang, Yipeng1 <[email protected]>; Gobriel, Sameh <[email protected]>; Richardson, Bruce <[email protected]>
Subject: [PATCH v4 1/4] hash: add kv hash libraryKV hash is a special optimized key-value storage for fixed key and value sizes. At the moment it supports 32 bit keys and 64 bit values. This table is hash function agnostic so user must provide precalculated hash signature for add/delete/lookup operations.
Signed-off-by Vladimir Medvedkin <[email protected]>--- lib/Makefile | 2 +- lib/librte_hash/Makefile | 14 +- lib/librte_hash/k32v64_hash.c | 277 +++++++++++++++++++++++++++++++++ lib/librte_hash/k32v64_hash.h | 98 ++++++++++++ lib/librte_hash/k32v64_hash_avx512vl.c | 59 +++++++ lib/librte_hash/meson.build | 17 +- lib/librte_hash/rte_hash_version.map | 6 +- lib/librte_hash/rte_kv_hash.c | 184 ++++++++++++++++++++++ lib/librte_hash/rte_kv_hash.h | 169 ++++++++++++++++++++ 9 files changed, 821 insertions(+), 5 deletions(-) create mode 100644
lib/librte_hash/k32v64_hash.ccreate mode 100644lib/librte_hash/k32v64_hash.hcreate mode 100644lib/librte_hash/k32v64_hash_avx512vl.c
create mode 100644lib/librte_hash/rte_kv_hash.ccreate mode 100644lib/librte_hash/rte_kv_hash.hlib/Makefile +0
-0@@ -48,7 +48,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_VHOST) += librte_vhostDEPDIRS-librte_vhost :=
librte_eallibrte_mempoollibrte_mbuflibrte_ethdev\
librte_netlibrte_hashlibrte_cryptodevDIRS-$(CONFIG_RTE_LIBRTE_HASH) +=librte_hash-DEPDIRS-librte_hash :=librte_eallibrte_ring
+DEPDIRS-librte_hash :=librte_eallibrte_ringlibrte_mempool
DIRS-$(CONFIG_RTE_LIBRTE_EFD) +=librte_efdDEPDIRS-librte_efd :=librte_eallibrte_ringlibrte_hash
DIRS-$(CONFIG_RTE_LIBRTE_RIB) +=librte_ribdiff --git
a/lib/librte_hash/Makefile b/lib/librte_hash/Makefile index ec9f864..a0cdee9 100644lib/librte_hash/Makefile +4
-1@@ -8,13 +8,15 @@ LIB = librte_hash.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_ring +LDLIBS += -lrte_eal -lrte_ring -lrte_mempool EXPORT_MAP := rte_hash_version.map # all source are stored in SRCS-y SRCS-$(CONFIG_RTE_LIBRTE_HASH) := rte_cuckoo_hash.c SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_fbk_hash.c +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_kv_hash.c +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += k32v64_hash.c # install this header file SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include := rte_hash.h @@ -27,5 +29,15 @@ endif SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include +=
rte_jhash.hSYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include +=rte_thash.hSYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include +=rte_fbk_hash.hlib/librte_hash/Makefile +3
-0+SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_kv_hash.h + +CC_AVX512VL_SUPPORT=$(shell $(CC) -mavx512vl -dM -E - </dev/null2>&1 |
+\ grep -q__AVX512VL__&& echo 1)lib/librte_hash/Makefile +5
-0+ +ifeq ($(CC_AVX512VL_SUPPORT), 1) +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += k32v64_hash_avx512vl.c +CFLAGS_k32v64_hash_avx512vl.o += -mavx512vl +CFLAGS_k32v64_hash.o += -DCC_AVX512VL_SUPPORT endif include $(RTE_SDK)/mk/rte.lib.mklib/librte_hash/k32v64_hash.c new file +17
-0@@ -0,0 +1,277 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include <string.h> + +#include <rte_errno.h> +#include <rte_malloc.h> +#include <rte_memory.h> + +#include "k32v64_hash.h" + +static inline int +k32v64_hash_lookup (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *value) +{ +return __k32v64_hash_lookup(table, key, hash, value,
__kv_cmp_keys); }lib/librte_hash/k32v64_hash.c new file +10
-0+ +static int +k32v64_hash_bulk_lookup (struct rte_kv_hash_table *ht,void *keys_p, +uint32_t *hashes,void *values_p,unsigned int n) { +struct k32v64_hash_table *table = (struct k32v64_hash_table *)ht; +uint32_t *keys = keys_p; +uint64_t *values = values_p; +int ret, cnt =0 ; +unsigned int i; +
Quoted 8 lines
lib/librte_hash/k32v64_hash.c line 36
Quoted 7 lines
lib/librte_hash/k32v64_hash.c new file +7
-0+if (unlikely ((table ==NULL ) || (keys ==NULL ) || (hashes ==NULL ) || +(values ==NULL ))) +return -EINVAL ; + +for (i =0 ; i < n; i++) { +ret =k32v64_hash_lookup (table, keys[i], hashes[i], +&values[i]);[Wang, Yipeng] You don't need to start a new line for values.
Then line will be too long (81), checkpatch have warning here
Quoted 8 lines
lib/librte_hash/k32v64_hash.c line 43
Quoted 7 lines
lib/librte_hash/k32v64_hash.c new file +7
-0+if (ret ==0 ) +cnt++; +} +return cnt; +} + +#ifdef CC_AVX512VL_SUPPORT [Wang, Yipeng] Why not use the already provided, e.g. #if
defined(RTE_MACHINE_CPUFLAG_SSE2)likerte_hashdoes?
For AVX512 implementation we are using avx512vl intrinsics which are not supported by some compilers. In this case supporting for AVX512F is not enough (there is only RTE_MACHINE_CPUFLAG_AVX512F flag).
Quoted 144 lines
Quoted 144 lines
lib/librte_hash/k32v64_hash.c new file +47
-0+int +k32v64_hash_bulk_lookup_avx512vl (struct rte_kv_hash_table *ht, +void *keys_p,uint32_t *hashes,void *values_p,unsigned int n); +#endif + +static rte_kv_hash_bulk_lookup_t +get_lookup_bulk_fn (void ) +{ +#ifdef CC_AVX512VL_SUPPORT +if (rte_cpu_get_flag_enabled (RTE_CPUFLAG_AVX512VL )) +return k32v64_hash_bulk_lookup_avx512vl;#endif +return k32v64_hash_bulk_lookup; +} + +static int +k32v64_hash_add (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t value,uint64_t *old_value,int *found) { +uint32_t bucket; +int i, idx, ret; +uint8_t msk; +struct k32v64_ext_ent *tmp, *ent, *prev =NULL ; + +if (table ==NULL ) +return -EINVAL ; + +bucket = hash & table->bucket_msk; +/* Search key in table. Update value if exists */ +for (i =0 ; i <K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && +(table->t[bucket].key_mask & (1 << i))) { +*old_value = table->t[bucket].val[i]; +*found =1 ; +__atomic_fetch_add (&table->t[bucket].cnt,1 , +__ATOMIC_ACQUIRE); +table->t[bucket].val[i] = value; +__atomic_fetch_add (&table->t[bucket].cnt,1 , +__ATOMIC_RELEASE); +return 0 ; +} +} + +if (!SLIST_EMPTY (&table->t[bucket].head)) { +SLIST_FOREACH(ent, &table->t[bucket].head, next) { +if (ent->key == key) { +*old_value = ent->val; +*found =1 ; +__atomic_fetch_add(&table->t[bucket].cnt,1,
lib/librte_hash/k32v64_hash.c new file +3
-0+__ATOMIC_ACQUIRE); +ent->val = value; +__atomic_fetch_add(&table->t[bucket].cnt,1,
lib/librte_hash/k32v64_hash.c new file +60
-0+__ATOMIC_RELEASE); +return 0 ; +} +} +} + +msk = ~table->t[bucket].key_mask &VALID_KEY_MSK ; +if (msk) { +idx =__builtin_ctz (msk); +table->t[bucket].key[idx] = key; +table->t[bucket].val[idx] = value; +__atomic_or_fetch (&table->t[bucket].key_mask,1 << idx, +__ATOMIC_RELEASE); +table->nb_ent++; +*found =0 ; +return 0 ; +} + +ret =rte_mempool_get (table->ext_ent_pool, (void **)&ent); +if (ret <0 ) +return ret; + +SLIST_NEXT (ent, next) =NULL ; +ent->key = key; +ent->val = value; +rte_smp_wmb (); +SLIST_FOREACH (tmp, &table->t[bucket].head, next) +prev = tmp; + +if (prev ==NULL ) +SLIST_INSERT_HEAD (&table->t[bucket].head, ent, next); +else +SLIST_INSERT_AFTER (prev, ent, next); + +table->nb_ent++; +table->nb_ext_ent++; +*found =0 ; +return 0 ; +} + +static int +k32v64_hash_delete (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *old_value) +{ +uint32_t bucket; +int i; +struct k32v64_ext_ent *ent; + +if (table ==NULL ) +return -EINVAL ; + +bucket = hash & table->bucket_msk; + +for (i =0 ; i <K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == table->t[bucket].key[i]) && +(table->t[bucket].key_mask & (1 << i))) { +*old_value = table->t[bucket].val[i]; +ent =SLIST_FIRST (&table->t[bucket].head); +if (ent) { +__atomic_fetch_add(&table->t[bucket].cnt,1,
lib/librte_hash/k32v64_hash.c new file +4
-0+__ATOMIC_ACQUIRE); +table->t[bucket].key[i] = ent->key; +table->t[bucket].val[i] = ent->val; +SLIST_REMOVE_HEAD(&table-Quoted 1 line
t[bucket].head, next);
+__atomic_fetch_add(&table->t[bucket].cnt,
1,lib/librte_hash/k32v64_hash.c new file +4
-0+__ATOMIC_RELEASE); +table->nb_ext_ent--; +}else +__atomic_and_fetch(&table-Quoted 1 line
t[bucket].key_mask,
+~(1 << i),
__ATOMIC_RELEASE);lib/librte_hash/k32v64_hash.c new file +2
-0+if (ent) +rte_mempool_put(table->ext_ent_pool,ent);
lib/librte_hash/k32v64_hash.c new file +15
-0+table->nb_ent--; +return 0 ; +} +} + +SLIST_FOREACH(ent, &table->t[bucket].head, next) +if (ent->key == key) +break ; + +if (ent ==NULL ) +return -ENOENT ; + +*old_value = ent->val; + +__atomic_fetch_add(&table->t[bucket].cnt, 1,
Quoted 8 lines
lib/librte_hash/k32v64_hash.c line 186
Quoted 5 lines
__ATOMIC_ACQUIRE);lib/librte_hash/k32v64_hash.c new file +4
-0+SLIST_REMOVE (&table->t[bucket].head, ent, k32v64_ext_ent, next); +__atomic_fetch_add (&table->t[bucket].cnt,1 , __ATOMIC_RELEASE); +rte_mempool_put (table->ext_ent_pool, ent); +[Wang, Yipeng] I am not sure if delete can be called safely with other lookup threads. The item could be recycled to be used by another add while a lookup thread traversing the linked list. This is similar to why we need the
RTE_HASH_EXTRA_FLAGS_NO_FREE_ON_DELflag and related API inrte_hash.
We discussed it with Konstantin previously. In this (very rare) case lookup thread will traverse across linked list which belongs to different bucket, won't find proper key and restart lookup again (because bucket cnt was changed). So it will never face with a linked list entry with inconsistent next pointer that does not point to some other list entry.
Quoted 24 lines
Quoted 24 lines
lib/librte_hash/k32v64_hash.c new file +8
-0+table->nb_ext_ent--; +table->nb_ent--; + +return 0 ; +} + +static int +k32v64_modify(struct rte_kv_hash_table *table, void *key_p, uint32_thash,
lib/librte_hash/k32v64_hash.c new file +7
-0+enum rte_kv_modify_op op, void *value_p, int *found) { +struct k32v64_hash_table *ht = (struct k32v64_hash_table *)table; +uint32_t *key = key_p; +uint64_t value; + +if ((ht == NULL) || (key == NULL) || (value_p == NULL) || +(found == NULL) || (op >=
RTE_KV_MODIFY_OP_MAX)) {lib/librte_hash/k32v64_hash.c new file +7
-0+return -EINVAL ; +} + +value = *(uint64_t *)value_p; +switch (op) { +case RTE_KV_MODIFY_ADD : +return k32v64_hash_add(ht, *key, hash, value, value_p,
Quoted 8 lines
lib/librte_hash/k32v64_hash.c line 214
Quoted 6 lines
found);
lib/librte_hash/k32v64_hash.c new file +5
-0+case RTE_KV_MODIFY_DEL : +return k32v64_hash_delete (ht, *key, hash, value_p); +default : +break ; +}[Wang, Yipeng] A question would be why put del and add inside another mod wrapper. If we don't wrap del and add like this, we could get rid of this branch.
This was done for future extensibility, if we want to add some additional control plane features in the future, other than add or delete. It is a control plane operation so branch here doesn't make any difference. If you think it's unnecessary I'll change it with add()/del() pair.
Quoted 7 lines
Quoted 7 lines
lib/librte_hash/k32v64_hash.c new file +7
-0+ +return -EINVAL ; +} + +struct rte_kv_hash_table * +k32v64_hash_create (const struct rte_kv_hash_params *params) { +char hash_name[RTE_KV_HASH_NAMESIZE ];
Quoted 8 lines
lib/librte_hash/k32v64_hash.c line 229
Quoted 7 lines
lib/librte_hash/k32v64_hash.c new file +7
-0+struct k32v64_hash_table *ht =NULL ; +uint32_t mem_size, nb_buckets, max_ent; +int ret; +struct rte_mempool *mp; + +if ((params ==NULL ) || (params->name ==NULL ) || +(params->entries ==0 )) {[Wang, Yipeng] Should we check if entry count larger than
keys_per_bucketas well?
No, because in general we are not limited to the number of keys in a bucket. keys_per_bucket reflects number of keys in open addressing part of a bucket.
Quoted 6 lines
Quoted 6 lines
lib/librte_hash/k32v64_hash.c new file +5
-0+rte_errno =EINVAL ; +return NULL ; +} + +ret = snprintf(hash_name, sizeof(hash_name), "KV_%s", params-Quoted 1 line
name);
Quoted 8 lines
lib/librte_hash/k32v64_hash.c line 241
Quoted 7 lines
lib/librte_hash/k32v64_hash.c new file +7
-0+if (ret <0 || ret >=RTE_KV_HASH_NAMESIZE ) { +rte_errno =ENAMETOOLONG ; +return NULL ; +} + +max_ent =rte_align32pow2 (params->entries); +nb_buckets = max_ent /K32V64_KEYS_PER_BUCKET ;[Wang, Yipeng] Macro to check if
keys_per_bucketneed to be power of 2
In general it doesn't have to be a power of 2.
Quoted 16 lines
Quoted 16 lines
lib/librte_hash/k32v64_hash.c new file +16
-0+mem_size =sizeof (struct k32v64_hash_table ) + +sizeof (struct k32v64_hash_bucket ) * nb_buckets; + +mp =rte_mempool_create (hash_name, max_ent, +sizeof (struct k32v64_ext_ent ),0 ,0 ,NULL ,NULL ,NULL ,NULL , +params->socket_id,0 ); + +if (mp ==NULL ) +return NULL ; + +ht =rte_zmalloc_socket (hash_name, mem_size, +RTE_CACHE_LINE_SIZE , params->socket_id); +if (ht ==NULL ) { +rte_mempool_free (mp); +return NULL ; +}
Quoted 8 lines
lib/librte_hash/k32v64_hash.c line 263
Quoted 6 lines
lib/librte_hash/k32v64_hash.c new file +6
-0+ +memcpy (ht->pub.name, hash_name,sizeof (ht->pub.name)); +ht->max_ent = max_ent; +ht->bucket_msk = nb_buckets -1 ; +ht->ext_ent_pool = mp; +ht->pub.lookup =get_lookup_bulk_fn ();[Wang, Yipeng] Inside the function, we also need to check the CPUID during runtime to decide if AVX can be used, not only compile time. You could refer to example from
rte_hash: ... if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE2)) ...
The same is here. In get_lookup_bulk_fn():
...
if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512VL))
...
Quoted 38 lines
Quoted 38 lines
lib/librte_hash/k32v64_hash.c new file +10
-0+ht->pub.modify = k32v64_modify; + +return (struct rte_kv_hash_table *)ht; } + +void +k32v64_hash_free (struct rte_kv_hash_table *ht) { +if (ht ==NULL ) +return ; + +rte_mempool_free(((struct k32v64_hash_table *)ht)-Quoted 1 line
ext_ent_pool);lib/librte_hash/k32v64_hash.c new file +2
-0+rte_free (ht); +}lib/librte_hash/k32v64_hash.h new file +19
-0@@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include <rte_kv_hash.h> + +#define K32V64_KEYS_PER_BUCKET4 +#define K32V64_WRITE_IN_PROGRESS1 +#define VALID_KEY_MSK ((1 << K32V64_KEYS_PER_BUCKET) - 1) + +struct k32v64_ext_ent { +SLIST_ENTRY (k32v64_ext_ent) next; +uint32_tkey ; +uint64_tval ; +}; + +struct k32v64_hash_bucket { +uint32_tkey[K32V64_KEYS_PER_BUCKET]; +uint64_tval[K32V64_KEYS_PER_BUCKET];
Quoted 8 lines
lib/librte_hash/k32v64_hash.h line 26
Quoted 7 lines
lib/librte_hash/k32v64_hash.h new file +7
-0+uint8_tkey_mask; +uint32_tcnt; +SLIST_HEAD(k32v64_list_head, k32v64_ext_ent) head; } +__rte_cache_aligned; + +struct k32v64_hash_table { +struct rte_kv_hash_table pub;/**< Public part */ [Wang, Yipeng] Do we need to have the pub filed? Could we just init the public part in the public create function?
As for me it is better to init it inside algorithm specific part. Other way we need to expose our internal algo-specific lookup_bulk and modify functions to kv_hash layer.
Quoted 198 lines
Quoted 198 lines
lib/librte_hash/k32v64_hash.h new file +1
-0+uint32_tnb_ent;/**< Number of entities in the table*/
lib/librte_hash/k32v64_hash.h new file +70
-0+uint32_tnb_ext_ent;/**< Number of extended entities */ +uint32_tmax_ent;/**< Maximum number of entities */ +uint32_tbucket_msk; +struct rte_mempool *ext_ent_pool; +__extension__ struct k32v64_hash_buckett[0]; +}; + +typedef int (*k32v64_cmp_fn_t ) +(struct k32v64_hash_bucket *bucket,uint32_t key,uint64_t *val); + +static inline int +__kv_cmp_keys (struct k32v64_hash_bucket *bucket,uint32_t key, +uint64_t *val) +{ +int i; + +for (i =0 ; i <K32V64_KEYS_PER_BUCKET ; i++) { +if ((key == bucket->key[i]) && +(bucket->key_mask & (1 << i))) { +*val = bucket->val[i]; +return 1 ; +} +} + +return 0 ; +} + +static inline int +__k32v64_hash_lookup (struct k32v64_hash_table *table,uint32_t key, +uint32_t hash,uint64_t *value,k32v64_cmp_fn_t cmp_f) { +uint64_tval =0 ; +struct k32v64_ext_ent *ent; +uint32_tcnt; +int found =0 ; +uint32_t bucket = hash & table->bucket_msk; + +do { + +do { +cnt =__atomic_load_n (&table->t[bucket].cnt, +__ATOMIC_ACQUIRE); +}while (unlikely (cnt &K32V64_WRITE_IN_PROGRESS )); + +found =cmp_f (&table->t[bucket], key, &val); +if (unlikely ((found ==0 ) && +(!SLIST_EMPTY (&table->t[bucket].head)))) { +SLIST_FOREACH(ent, &table->t[bucket].head, next) { +if (ent->key == key) { +val = ent->val; +found =1 ; +break ; +} +} +} +__atomic_thread_fence (__ATOMIC_RELEASE); +}while (unlikely (cnt !=__atomic_load_n (&table->t[bucket].cnt, + __ATOMIC_RELAXED))); + +if (found ==1 ) { +*value = val; +return 0 ; +}else +return -ENOENT ; +} + +struct rte_kv_hash_table * +k32v64_hash_create (const struct rte_kv_hash_params *params); + +void +k32v64_hash_free (struct rte_kv_hash_table *ht);lib/librte_hash/k32v64_hash_avx512vl.c new file +8
-0@@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include "k32v64_hash.h" + +int +k32v64_hash_bulk_lookup_avx512vl(struct rte_kv_hash_table *ht, void
*keys_p,lib/librte_hash/k32v64_hash_avx512vl.c new file +4
-0+uint32_t *hashes, void *values_p, unsigned int n); + +static inline int +k32v64_cmp_keys_avx512vl(struct k32v64_hash_bucket *bucket, uint32_tkey,
lib/librte_hash/k32v64_hash_avx512vl.c new file +9
-0+uint64_t *val) +{ +__m128i keys, srch_key; +__mmask8 msk; + +keys =_mm_load_si128 ((void *)bucket); +srch_key =_mm_set1_epi32 (key); + +msk = _mm_mask_cmpeq_epi32_mask(bucket->key_mask, keys,
srch_key);lib/librte_hash/k32v64_hash_avx512vl.c new file +10
-0+if (msk) { +*val = bucket->val[__builtin_ctz (msk)]; +return 1 ; +} + +return 0 ; +} + +static inline int +k32v64_hash_lookup_avx512vl(struct k32v64_hash_table *table, uint32_tkey,
lib/librte_hash/k32v64_hash_avx512vl.c new file +8
-0+uint32_t hash, uint64_t *value) +{ +return __k32v64_hash_lookup(table, key, hash, value, +k32v64_cmp_keys_avx512vl ); +} + +int +k32v64_hash_bulk_lookup_avx512vl(struct rte_kv_hash_table *ht, void
*keys_p,lib/librte_hash/k32v64_hash_avx512vl.c new file +19
-0+uint32_t *hashes, void *values_p, unsigned int n) { +struct k32v64_hash_table *table = (struct k32v64_hash_table *)ht; +uint32_t *keys = keys_p; +uint64_t *values = values_p; +int ret, cnt =0 ; +unsigned int i; + +if (unlikely ((table ==NULL ) || (keys ==NULL ) || (hashes ==NULL ) || +(values ==NULL ))) +return -EINVAL ; + +for (i =0 ; i < n; i++) { +ret =k32v64_hash_lookup_avx512vl (table, keys[i], hashes[i], +&values[i]); +if (ret ==0 ) +cnt++; +} +return cnt; +}lib/librte_hash/meson.build index renamed from lib/librte_hash/meson.build +0
-06ab46ae..0d014ea 100644
lib/librte_hash/meson.build +15
-1@@ -3,10 +3,23 @@ headers = files('rte_crc_arm64.h', 'rte_fbk_hash.h', +'rte_kv_hash.h', 'rte_hash_crc.h', 'rte_hash.h', 'rte_jhash.h', 'rte_thash.h') -sources = files('rte_cuckoo_hash.c', 'rte_fbk_hash.c') -deps += ['ring'] +sources = files('rte_cuckoo_hash.c', 'rte_fbk_hash.c', 'rte_kv_hash.c', +'k32v64_hash.c') deps += ['ring', 'mempool'] + +if dpdk_conf.has('RTE_ARCH_X86') +if cc.has_argument('-mavx512vl') +avx512_tmplib = static_library('avx512_tmp', + 'k32v64_hash_avx512vl.c', + dependencies: static_rte_mempool, + c_args: cflags + ['-mavx512vl']) + objs += avx512_tmplib.extract_objects('k32v64_hash_avx512vl.c') + cflags += '-DCC_AVX512VL_SUPPORT' + +endif +endiflib/librte_hash/rte_hash_version.map +5
-1@@ -36,5 +36,9 @@ EXPERIMENTAL { rte_hash_lookup_with_hash_bulk; rte_hash_lookup_with_hash_bulk_data; rte_hash_max_key_id; - +rte_kv_hash_create; +rte_kv_hash_find_existing; +rte_kv_hash_free; +rte_kv_hash_add; +rte_kv_hash_delete; };lib/librte_hash/rte_kv_hash.c new file +6
-0@@ -0,0 +1,184 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#include <string.h> +
Quoted 8 lines
lib/librte_hash/rte_kv_hash.c line 13
Quoted 7 lines
lib/librte_hash/rte_kv_hash.c new file +7
-0+#include <rte_eal_memconfig.h> +#include <rte_errno.h> +#include <rte_malloc.h> +#include <rte_memory.h> +#include <rte_tailq.h> + +#include <rte_kv_hash.h> [Wang, Yipeng] I think for this header we should use quotes ""
rte_kv_hash.h is a public header file, we we should do that?
Quoted 8 lines
Quoted 8 lines
lib/librte_hash/rte_kv_hash.c new file +8
-0+#include "k32v64_hash.h" + +TAILQ_HEAD (rte_kv_hash_list, rte_tailq_entry); + +static struct rte_tailq_elem rte_kv_hash_tailq = { +.name ="RTE_KV_HASH" , +}; +
Quoted 8 lines
lib/librte_hash/rte_kv_hash.c line 28
Quoted 6 lines
lib/librte_hash/rte_kv_hash.c new file +6
-0+EAL_REGISTER_TAILQ (rte_kv_hash_tailq); + +int +rte_kv_hash_add (struct rte_kv_hash_table *table,void *key, +uint32_t hash,void *value,int *found) { +if (table ==NULL )[Wang, Yipeng] To be more consistent,
I think we should either also check key/value/found here, or just not checking at all and leave it to the next functions.
Agree, I will change it next version
Quoted 224 lines
Quoted 176 lines
lib/librte_hash/rte_kv_hash.c new file +61
-0+return -EINVAL ; + +return table->modify (table, key, hash,RTE_KV_MODIFY_ADD , +value, found); +} + +int +rte_kv_hash_delete (struct rte_kv_hash_table *table,void *key, +uint32_t hash,void *value) +{ +int found; + +if (table ==NULL ) +return -EINVAL ; + +return table->modify (table, key, hash,RTE_KV_MODIFY_DEL , +value, &found); +} + +struct rte_kv_hash_table * +rte_kv_hash_find_existing (const char *name) { +struct rte_kv_hash_table *h =NULL ; +struct rte_tailq_entry *te; +struct rte_kv_hash_list *kv_hash_list; + +kv_hash_list =RTE_TAILQ_CAST (rte_kv_hash_tailq.head, +rte_kv_hash_list); + +rte_mcfg_tailq_read_lock (); +TAILQ_FOREACH(te, kv_hash_list, next) { +h = (struct rte_kv_hash_table *) te->data; +if (strncmp (name, h->name,RTE_KV_HASH_NAMESIZE ) ==0 ) +break ; +} +rte_mcfg_tailq_read_unlock (); +if (te ==NULL ) { +rte_errno =ENOENT ; +return NULL ; +} +return h; +} + +struct rte_kv_hash_table * +rte_kv_hash_create (const struct rte_kv_hash_params *params) { +char hash_name[RTE_KV_HASH_NAMESIZE ]; +struct rte_kv_hash_table *ht, *tmp_ht =NULL ; +struct rte_tailq_entry *te; +struct rte_kv_hash_list *kv_hash_list; +int ret; + +if ((params ==NULL ) || (params->name ==NULL ) || +(params->entries ==0 ) || +(params->type >=RTE_KV_HASH_MAX )) { +rte_errno =EINVAL ; +return NULL ; +} + +kv_hash_list =RTE_TAILQ_CAST (rte_kv_hash_tailq.head, +rte_kv_hash_list); + +ret = snprintf(hash_name, sizeof(hash_name), "KV_%s", params-Quoted 1 line
name);
lib/librte_hash/rte_kv_hash.c new file +92
-0+if (ret <0 || ret >=RTE_KV_HASH_NAMESIZE ) { +rte_errno =ENAMETOOLONG ; +return NULL ; +} + +switch (params->type) { +case RTE_KV_HASH_K32V64 : +ht =k32v64_hash_create (params); +break ; +default : +rte_errno =EINVAL ; +return NULL ; +} +if (ht ==NULL ) +return ht; + +rte_mcfg_tailq_write_lock (); +TAILQ_FOREACH(te, kv_hash_list, next) { +tmp_ht = (struct rte_kv_hash_table *) te->data; +if (strncmp (params->name, tmp_ht->name, +RTE_KV_HASH_NAMESIZE ) ==0 ) +break ; +} +if (te !=NULL ) { +rte_errno =EEXIST ; +goto exit; +} + +te =rte_zmalloc ("KV_HASH_TAILQ_ENTRY" ,sizeof (*te),0 ); +if (te ==NULL ) { +RTE_LOG (ERR ,HASH ,"Failed to allocate tailq entry\n" ); +goto exit; +} + +ht->type = params->type; +te->data = (void *)ht; +TAILQ_INSERT_TAIL (kv_hash_list, te, next); + +rte_mcfg_tailq_write_unlock (); + +return ht; + +exit: +rte_mcfg_tailq_write_unlock (); +switch (params->type) { +case RTE_KV_HASH_K32V64 : +k32v64_hash_free (ht); +break ; +default : +break ; +} +return NULL ; +} + +void +rte_kv_hash_free (struct rte_kv_hash_table *ht) { +struct rte_tailq_entry *te; +struct rte_kv_hash_list *kv_hash_list; + +if (ht ==NULL ) +return ; + +kv_hash_list =RTE_TAILQ_CAST (rte_kv_hash_tailq.head, +rte_kv_hash_list); + +rte_mcfg_tailq_write_lock (); + +/* find out tailq entry */ +TAILQ_FOREACH(te, kv_hash_list, next) { +if (te->data == (void *) ht) +break ; +} + + +if (te ==NULL ) { +rte_mcfg_tailq_write_unlock (); +return ; +} + +TAILQ_REMOVE (kv_hash_list, te, next); + +rte_mcfg_tailq_write_unlock (); + +switch (ht->type) { +case RTE_KV_HASH_K32V64 : +k32v64_hash_free (ht); +break ; +default : +break ; +} +rte_free (te); +}lib/librte_hash/rte_kv_hash.h new file +17
-0@@ -0,0 +1,169 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Intel Corporation +*/ + +#ifndef _RTE_KV_HASH_H_ +#define _RTE_KV_HASH_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <rte_compat.h> +#include <rte_atomic.h> +#include <rte_mempool.h> + +#define RTE_KV_HASH_NAMESIZE32 +lib/librte_hash/rte_kv_hash.h line 27
Quoted 10 lines
lib/librte_hash/rte_kv_hash.h new file +10
-0+enum rte_kv_hash_type { +RTE_KV_HASH_K32V64 , +RTE_KV_HASH_MAX +}; + +enum rte_kv_modify_op { +RTE_KV_MODIFY_ADD , +RTE_KV_MODIFY_DEL , +RTE_KV_MODIFY_OP_MAX +};[Wang, Yipeng] Again, any particular reason that you combine add and del into an additional struct mod?
Quoted 37 lines
lib/librte_hash/rte_kv_hash.h new file +20
-0+ +struct rte_kv_hash_params { +const char *name; +uint32_t entries; +int socket_id; +enum rte_kv_hash_type type; +}; + +struct rte_kv_hash_table ; + +typedef int (*rte_kv_hash_bulk_lookup_t ) (struct rte_kv_hash_table +*table,void *keys,uint32_t *hashes, +void *values,unsigned int n); + +typedef int (*rte_kv_hash_modify_t ) +(struct rte_kv_hash_table *table,void *key,uint32_t hash, +enum rte_kv_modify_op op,void *value,int *found); + +struct rte_kv_hash_table { +char name[RTE_KV_HASH_NAMESIZE ];/**< Name of the hash. */
lib/librte_hash/rte_kv_hash.h new file +16
-0+rte_kv_hash_bulk_lookup_tlookup; +rte_kv_hash_modify_tmodify; +enum rte_kv_hash_typetype ; +}; + +/** +* Lookup bulk of keys. +* This function is multi-thread safe with regarding to other lookup threads. +* +* @param table +* Hash table to add the key to. +* @param keys +* Pointer to array of keys +* @param hashes +* Pointer to array of hash values associated with keys. +* @param values
Quoted 8 lines
Quoted 7 lines
+ * Pointer to array of value corresponded to keys
+ * If the key was not found the corresponding value remains intact.
+ * @param n
+ * Number of keys to lookup in batch.
+ * @return
+ *-EINVALif there's an error, otherwise number of successful lookups.
+ */[Wang, Yipeng] experimental tag
Yes, will add
Quoted 66 lines
Quoted 66 lines
lib/librte_hash/rte_kv_hash.h new file +66
-0+static inline int +rte_kv_hash_bulk_lookup (struct rte_kv_hash_table *table, +void *keys,uint32_t *hashes,void *values,unsigned int n) { +return table->lookup (table, keys, hashes, values, n); } + +/** +* Add a key to an existing hash table with hash value. +* This operation is not multi-thread safe regarding to add/delete +functions +* and should only be called from one thread. +* However it is safe to call it along with lookup. +* +* @param table +* Hash table to add the key to. +* @param key +* Key to add to the hash table. +* @param value +* Value to associate with key. +* @param hash +* Hash value associated with key. +* @found +* 0 if no previously added key was found +* 1 previously added key was found, old value associated with the key +* was written to *value +* @return +* 0 if ok, or negative value on error. +*/ +__rte_experimental +int +rte_kv_hash_add (struct rte_kv_hash_table *table,void *key, +uint32_t hash,void *value,int *found); + +/** +* Remove a key with a given hash value from an existing hash table. +* This operation is not multi-thread safe regarding to add/delete +functions +* and should only be called from one thread. +* However it is safe to call it along with lookup. +* +* @param table +* Hash table to remove the key from. +* @param key +* Key to remove from the hash table. +* @param hash +* hash value associated with key. +* @param value +* pointer to memory where the old value will be written to on success +* @return +* 0 if ok, or negative value on error. +*/ +__rte_experimental +int +rte_kv_hash_delete (struct rte_kv_hash_table *table,void *key, +uint32_t hash,void *value); + +/** +* Performs a lookup for an existing hash table, and returns a pointer +to +* the table if found. +* +* @param name +* Name of the hash table to find +* +* @return +* pointer to hash table structure or NULL on error with rte_errno +* set appropriately.
Quoted 8 lines
lib/librte_hash/rte_kv_hash.h line 142
Quoted 7 lines
+ */
lib/librte_hash/rte_kv_hash.h new file +6
-0+__rte_experimental +struct rte_kv_hash_table * +rte_kv_hash_find_existing (const char *name); + +/** +* Create a new hash table for use with four byte keys. [Wang, Yipeng] inaccurate comment, not four byte keys
Yes, artifact from previous implementation
Quoted 29 lines
Quoted 29 lines
+ *
+ * @param params
+ * Parameters used in creation of hash table.
+ *
+ * @return
+ * Pointer to hash table structure that is used in future hash table
+ * operations, or NULL on error withrte_errnoset appropriately.
+ */lib/librte_hash/rte_kv_hash.h new file +19
-1+__rte_experimental +struct rte_kv_hash_table * +rte_kv_hash_create (const struct rte_kv_hash_params *params); + +/** +* Free all memory used by a hash table. +* +* @param table +* Hash table to deallocate. +*/ +__rte_experimental +void +rte_kv_hash_free (struct rte_kv_hash_table *table); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_KV_HASH_H_ */ --2.7.4
Signature · 2 lines
Regards, Vladimir
Re: [dpdk-dev] [PATCH v4 0/4] add new kv hash table
In reply to Vladimir Medvedkin08/05/2020 21:58, Vladimir Medvedkin:
Quoted 6 lines
Currently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called KV hash
supports 4 byte keys and 8 byte associated values,
which is enough to store a pointer.
Waiting for a v5.
Is it abandoned?
Re: [dpdk-dev] [PATCH v4 0/4] add new kv hash table
Hi Thomas,
Quoted 10 lines · Thomas Monjalon
08/05/2020 21:58, Vladimir Medvedkin:
Quoted 6 lines
Currently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called KV hash
supports 4 byte keys and 8 byte associated values,
which is enough to store a pointer.Waiting for a v5.
Is it abandoned?
It is suspended till further rework.
Quoted 2 lines
Signature · 2 lines
Regards, Vladimir
Re: [dpdk-dev] [PATCH v4 0/4] add new kv hash table
Quoted 11 lines · Vladimir
Hi Thomas,
Quoted 8 lines · Thomas Monjalon
08/05/2020 21:58, Vladimir Medvedkin:
Quoted 6 lines
Currently DPDK has a special implementation of a hash table for 4 byte keys which is called FBK hash. Unfortunately its main drawback is that it only supports 2 byte values.
The new implementation called KV hash
supports 4 byte keys and 8 byte associated values,
which is enough to store a pointer.
Quoted 6 lines
Quoted 2 lines
Waiting for a v5.
Is it abandoned?It is suspended till further rework.
Quoted 1 line
Since nothing more has arrived in 2 years, marking the original patchset as Changes requested.