This patch updates AVX512 support for xbzrle
v4 cover This patch updates AVX512 support for xbzrle
This patch updates code of AVX512 support for xbzrle_encode_buffer fucntion to accelerate xbzrle encoding speed.
The runtime check code is kept in meson.build and meson_options.txt without modifying.
The updated AVX512 algorithm is provided in ram.c, xbzrle.c and xbzrle.h.
The test code is updated in test-xbzrle.c.
ling xu (2):
Update AVX512 support for xbzrle_encode_buffer function Update test code of AVX512 support for xbzrle_encode
meson.build | 16 ++ meson_options.txt | 2 + migration/ram.c | 42 +++- migration/xbzrle.c | 181 ++++++++++++++++ migration/xbzrle.h | 4 + tests/unit/test-xbzrle.c | 458 ++++++++++++++++++++++++++++++++++++++- 6 files changed, 699 insertions(+), 4 deletions(-)
Signature · 1 line
2.25.1
PATCH v4 1/2 Update AVX512 support for xbzrle_encode_buffer function
This commit update code of avx512 support for xbzrle_encode_buffer function to accelerate xbzrle encoding speed.
Compared with C version of xbzrle_encode_buffer function, avx512 version can achieve almost 60%-70% performance improvement on unit test provided by Qemu. In addition, we provide one more unit test called "test_encode_decode_random_avx512", in which dirty data are randomly located in 4K page, and this case can achieve almost 140% performance gain.
--- meson.build | 16 ++++ meson_options.txt | 2 + migration/ram.c | 42 ++++++++++- migration/xbzrle.c | 181 +++++++++++++++++++++++++++++++++++++++++++++ migration/xbzrle.h | 4 + 5 files changed, 242 insertions(+), 3 deletions(-)
meson.build +16 -0
@@ -2262,6 +2262,22 @@ config_host_data.set('CONFIG_AVX512F_OPT', get_option('avx512f') \ int main(int argc, char *argv[]) { return bar(argv[0]); } '''), error_message: 'AVX512F not available').allowed()) +config_host_data.set('CONFIG_AVX512BW_OPT', get_option('avx512bw') \ + .require(have_cpuid_h, error_message: 'cpuid.h not available, cannot enable AVX512BW') \ + .require(cc.links(''' + #pragma GCC push_options + #pragma GCC target("avx512bw") + #include <cpuid.h> + #include <immintrin.h> + static int bar(void *a) { + + __m512i x = *(__m512i *)a; + __m512i res= _mm512_abs_epi8(x); + return res[1]; + } + int main(int argc, char *argv[]) { return bar(argv[0]); } + '''), error_message: 'AVX512BW not available').allowed()) + have_pvrdma = get_option('pvrdma') \ .require(rdma.found(), error_message: 'PVRDMA requires OpenFabrics libraries') \ .require(cc.compiles(gnu_source_prefix + '''
meson_options.txt +2 -0
@@ -104,6 +104,8 @@ option('avx2', type: 'feature', value: 'auto', description: 'AVX2 optimizations') option('avx512f', type: 'feature', value: 'disabled', description: 'AVX512F optimizations') +option('avx512bw', type: 'feature', value: 'auto', + description: 'AVX512BW optimizations') option('keyring', type: 'feature', value: 'auto', description: 'Linux keyring support')
migration/ram.c +39 -3
54 lines not on this page · raw message
migration/xbzrle.c +181 -0
184 lines not on this page · raw message
migration/xbzrle.h +4 -0
@@ -18,4 +18,8 @@ int xbzrle_encode_buffer(uint8_t *old_buf, uint8_t *new_buf, int slen, uint8_t *dst, int dlen);int xbzrle_decode_buffer (uint8_t *src,int slen,uint8_t *dst,int dlen); +#if defined(CONFIG_AVX512BW_OPT ) +int xbzrle_encode_buffer_avx512 (uint8_t *old_buf,uint8_t *new_buf,int slen, +uint8_t *dst,int dlen); +#endif #endif
Signature · 1 line
2.25.1
PATCH v4 2/2 Update test code of AVX512 support for xbzrle_encode
In reply to ling xu--- tests/unit/test-xbzrle.c | 458 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 457 insertions(+), 1 deletion(-)
tests/unit/test-xbzrle.c +457 -1
@@ -16,6 +16,35 @@#define XBZRLE_PAGE_SIZE 4096 +#if defined(CONFIG_AVX512BW_OPT ) +static bool is_cpu_support_avx512bw; +#include "qemu/cpuid.h" +static void __attribute__((constructor))init_cpu_flag (void ) +{ +unsigned max =__get_cpuid_max (0 ,NULL ); +int a, b, c, d; + is_cpu_support_avx512bw = false; +if (max >=1 ) { +__cpuid (1 , a, b, c, d); +/* We must check that AVX is not just available, but usable. */ +if ((c & bit_OSXSAVE) && (c & bit_AVX) && max >=7 ) { +int bv; + __asm("xgetbv" :"=a" (bv),"=d" (d) :"c" (0 )); +__cpuid_count (7 ,0 , a, b, c, d); +/* 0xe6: +* XCR0[7:5] = 111b (OPMASK state, upper 256-bit of ZMM0-ZMM15 +* and ZMM16-ZMM31 state are enabled by OS) +* XCR0[2:1] = 11b (XMM state and YMM state are enabled by OS) +*/ +if ((bv &0xe6 ) ==0xe6 && (b & bit_AVX512BW)) { + is_cpu_support_avx512bw = true; + } + } + } +return ; +} +#endif +static void test_uleb (void ) {uint32_t i, val;
@@ -173,11 +202,438 @@ static void test_encode_decode(void) } } +static float *encode_decode_zero (void ) +{ +uint8_t *buffer =g_malloc0 (XBZRLE_PAGE_SIZE ); +uint8_t *compressed =g_malloc0 (XBZRLE_PAGE_SIZE ); +uint8_t *buffer512 =g_malloc0 (XBZRLE_PAGE_SIZE ); +uint8_t *compressed512 =g_malloc0 (XBZRLE_PAGE_SIZE ); +int i =0 ; +int dlen =0 , dlen512 =0 ; +int diff_len =g_test_rand_int_range (0 ,XBZRLE_PAGE_SIZE -1006 ); + +for (i = diff_len; i >0 ; i--) { + buffer[1000 + i] = i; + buffer512[1000 + i] = i; + } + + buffer[1000 + diff_len +3 ] =103 ; + buffer[1000 + diff_len +5 ] =105 ; + + buffer512[1000 + diff_len +3 ] =103 ; + buffer512[1000 + diff_len +5 ] =105 ; + +/* encode zero page */ +time_t t_start, t_end, t_start512, t_end512; + t_start =clock (); + dlen =xbzrle_encode_buffer (buffer, buffer,XBZRLE_PAGE_SIZE , compressed, +XBZRLE_PAGE_SIZE ); + t_end =clock (); +float time_val =difftime (t_end, t_start); +g_assert (dlen ==0 ); + + t_start512 =clock (); + dlen512 =xbzrle_encode_buffer_avx512 (buffer512, buffer512,XBZRLE_PAGE_SIZE , + compressed512,XBZRLE_PAGE_SIZE ); + t_end512 =clock (); +float time_val512 =difftime (t_end512, t_start512); +g_assert (dlen512 ==0 ); + +static float result_zero[2 ]; + result_zero[0 ] = time_val; + result_zero[1 ] = time_val512; + +g_free (buffer); +g_free (compressed); +g_free (buffer512); +g_free (compressed512); + +return result_zero; +} + +static void test_encode_decode_zero_avx512 (void ) +{ +int i; +float time_raw =0.0 , time_512 =0.0 ; +float *res; +for (i =0 ; i <10000 ; i++) { + res =encode_decode_zero (); + time_raw += res[0 ]; + time_512 += res[1 ]; + } +printf ("Zero test:\n" ); +printf ("Raw xbzrle_encode time is %f ms\n" , time_raw); +printf ("512 xbzrle_encode time is %f ms\n" , time_512); +} + +static float *encode_decode_unchanged (void ) +{ +uint8_t *compressed =g_malloc0 (XBZRLE_PAGE_SIZE ); +uint8_t *test =g_malloc0 (XBZRLE_PAGE_SIZE ); +uint8_t *compressed512 =g_malloc0 (XBZRLE_PAGE_SIZE ); +uint8_t *test512 =g_malloc0 (XBZRLE_PAGE_SIZE ); +int i =0 ; +int dlen =0 , dlen512 =0 ; +int diff_len =g_test_rand_int_range (0 ,XBZRLE_PAGE_SIZE -1006 ); + +for (i = diff_len; i >0 ; i--) { + test[1000 + i] = i +4 ; + test512[1000 + i] = i +4 ; + } + + test[1000 + diff_len +3 ] =107 ; + test[1000 + diff_len +5 ] =109 ; + + test512[1000 + diff_len +3 ] =107 ; + test512[1000 + diff_len +5 ] =109 ; + +/* test unchanged buffer */ +time_t t_start, t_end, t_start512, t_end512; + t_start =clock (); + dlen =xbzrle_encode_buffer (test, test,XBZRLE_PAGE_SIZE , compressed, +XBZRLE_PAGE_SIZE ); + t_end =clock (); +float time_val =difftime (t_end, t_start); +g_assert (dlen ==0 ); + + t_start512 =clock (); + dlen512 =xbzrle_encode_buffer_avx512 (test512, test512,XBZRLE_PAGE_SIZE , + compressed512,XBZRLE_PAGE_SIZE ); + t_end512 =clock (); +float time_val512 =difftime (t_end512, t_start512); +g_assert (dlen512 ==0 ); + +static float result_unchanged[2 ]; + result_unchanged[0 ] = time_val; + result_unchanged[1 ] = time_val512; + +g_free (test); +g_free (compressed); +g_free (test512); +g_free (compressed512); + +return result_unchanged; +} + +static void test_encode_decode_unchanged_avx512 (void ) +{ +int i; +float time_raw =0.0 , time_512 =0.0 ; +float *res; +for (i =0 ; i <10000 ; i++) { + res =encode_decode_unchanged (); + time_raw += res[0 ]; + time_512 += res[1 ]; + } +printf ("Unchanged test:\n" ); +printf ("Raw xbzrle_encode time is %f ms\n" , time_raw); +printf ("512 xbzrle_encode time is %f ms\n" , time_512); +} + +static float *encode_decode_1_byte (void ) +{ +uint8_t *buffer =g_malloc0 (XBZRLE_PAGE_SIZE ); +uint8_t *test =g_malloc0 (XBZRLE_PAGE_SIZE ); +uint8_t *compressed =g_malloc (XBZRLE_PAGE_SIZE ); +uint8_t *buffer512 =g_malloc0 (XBZRLE_PAGE_SIZE ); +uint8_t *test512 =g_malloc0 (XBZRLE_PAGE_SIZE ); +uint8_t *compressed512 =g_malloc (XBZRLE_PAGE_SIZE ); +int dlen =0 , rc =0 , dlen512 =0 , rc512 =0 ; +uint8_t buf[2 ]; +uint8_t buf512[2 ]; + + test[XBZRLE_PAGE_SIZE -1 ] =1 ; + test512[XBZRLE_PAGE_SIZE -1 ] =1 ; + +time_t t_start, t_end, t_start512, t_end512; + t_start =clock (); + dlen =xbzrle_encode_buffer (buffer, test,XBZRLE_PAGE_SIZE , compressed, +XBZRLE_PAGE_SIZE ); + t_end =clock (); +float time_val =difftime (t_end, t_start); +g_assert (dlen == (uleb128_encode_small (&buf[0 ],4095 ) +2 )); + + rc =xbzrle_decode_buffer (compressed, dlen, buffer,XBZRLE_PAGE_SIZE ); +g_assert (rc ==XBZRLE_PAGE_SIZE ); +g_assert (memcmp (test, buffer,XBZRLE_PAGE_SIZE ) ==0 ); + + t_start512 =clock (); + dlen512 =xbzrle_encode_buffer_avx512 (buffer512, test512,XBZRLE_PAGE_SIZE , + compressed512,XBZRLE_PAGE_SIZE ); + t_end512 =clock (); +float time_val512 =difftime (t_end512, t_start512); +g_assert (dlen512 == (uleb128_encode_small (&buf512[0 ],4095 ) +2 )); + + rc512 =xbzrle_decode_buffer (compressed512, dlen512, buffer512, +XBZRLE_PAGE_SIZE ); +g_assert (rc512 ==XBZRLE_PAGE_SIZE ); +g_assert (memcmp (test512, buffer512,XBZRLE_PAGE_SIZE ) ==0 ); + +static float result_1_byte[2 ]; + result_1_byte[0 ] = time_val; + result_1_byte[1 ] = time_val512; + +g_free (buffer); +g_free (compressed); +g_free (test); +g_free (buffer512); +g_free (compressed512); +g_free (test512); + +return result_1_byte; +} + +static void test_encode_decode_1_byte_avx512 (void ) +{ +int i; +float time_raw =0.0 , time_512 =0.0 ; +float *res; +for (i =0 ; i <10000 ; i++) { + res =encode_decode_1_byte (); + time_raw += res[0 ]; + time_512 += res[1 ]; + } +printf ("1 byte test:\n" ); +printf ("Raw xbzrle_encode time is %f ms\n" , time_raw); +printf ("512 xbzrle_encode time is %f ms\n" , time_512); +} + +static float *encode_decode_overflow (void ) +{ +uint8_t *compressed =g_malloc0 (XBZRLE_PAGE_SIZE ); +uint8_t *test =g_malloc0 (XBZRLE_PAGE_SIZE ); +uint8_t *buffer =g_malloc0 (XBZRLE_PAGE_SIZE ); +uint8_t *compressed512 =g_malloc0 (XBZRLE_PAGE_SIZE ); +uint8_t *test512 =g_malloc0 (XBZRLE_PAGE_SIZE ); +uint8_t *buffer512 =g_malloc0 (XBZRLE_PAGE_SIZE ); +int i =0 , rc =0 , rc512 =0 ; + +for (i =0 ; i <XBZRLE_PAGE_SIZE /2 -1 ; i++) { + test[i *2 ] =1 ; + test512[i *2 ] =1 ; + } + +/* encode overflow */ +time_t t_start, t_end, t_start512, t_end512; + t_start =clock (); + rc =xbzrle_encode_buffer (buffer, test,XBZRLE_PAGE_SIZE , compressed, +XBZRLE_PAGE_SIZE ); + t_end =clock (); +float time_val =difftime (t_end, t_start); +g_assert (rc ==-1 ); + + t_start512 =clock (); + rc512 =xbzrle_encode_buffer_avx512 (buffer512, test512,XBZRLE_PAGE_SIZE , + compressed512,XBZRLE_PAGE_SIZE ); + t_end512 =clock (); +float time_val512 =difftime (t_end512, t_start512); +g_assert (rc512 ==-1 ); + +static float result_overflow[2 ]; + result_overflow[0 ] = time_val; + result_overflow[1 ] = time_val512; + +g_free (buffer); +g_free (compressed); +g_free (test); +g_free (buffer512); +g_free (compressed512); +g_free (test512); + +return result_overflow; +} + +static void test_encode_decode_overflow_avx512 (void ) +{ +int i; +float time_raw =0.0 , time_512 =0.0 ; +float *res; +for (i =0 ; i <10000 ; i++) { + res =encode_decode_overflow (); + time_raw += res[0 ]; + time_512 += res[1 ]; + } +printf ("Overflow test:\n" ); +printf ("Raw xbzrle_encode time is %f ms\n" , time_raw); +printf ("512 xbzrle_encode time is %f ms\n" , time_512); +} + +static float *encode_decode_range_avx512 (void ) +{ +uint8_t *buffer =g_malloc0 (XBZRLE_PAGE_SIZE ); +uint8_t *compressed =g_malloc (XBZRLE_PAGE_SIZE ); +uint8_t *test =g_malloc0 (XBZRLE_PAGE_SIZE ); +uint8_t *buffer512 =g_malloc0 (XBZRLE_PAGE_SIZE ); +uint8_t *compressed512 =g_malloc (XBZRLE_PAGE_SIZE ); +uint8_t *test512 =g_malloc0 (XBZRLE_PAGE_SIZE ); +int i =0 , rc =0 , rc512 =0 ; +int dlen =0 , dlen512 =0 ; + +int diff_len =g_test_rand_int_range (0 ,XBZRLE_PAGE_SIZE -1006 ); + +for (i = diff_len; i >0 ; i--) { + buffer[1000 + i] = i; + test[1000 + i] = i +4 ; + buffer512[1000 + i] = i; + test512[1000 + i] = i +4 ; + } + + buffer[1000 + diff_len +3 ] =103 ; + test[1000 + diff_len +3 ] =107 ; + + buffer[1000 + diff_len +5 ] =105 ; + test[1000 + diff_len +5 ] =109 ; + + buffer512[1000 + diff_len +3 ] =103 ; + test512[1000 + diff_len +3 ] =107 ; + + buffer512[1000 + diff_len +5 ] =105 ; + test512[1000 + diff_len +5 ] =109 ; + +/* test encode/decode */ +time_t t_start, t_end, t_start512, t_end512; + t_start =clock (); + dlen =xbzrle_encode_buffer (test, buffer,XBZRLE_PAGE_SIZE , compressed, +XBZRLE_PAGE_SIZE ); + t_end =clock (); +float time_val =difftime (t_end, t_start); + rc =xbzrle_decode_buffer (compressed, dlen, test,XBZRLE_PAGE_SIZE ); +g_assert (rc <XBZRLE_PAGE_SIZE ); +g_assert (memcmp (test, buffer,XBZRLE_PAGE_SIZE ) ==0 ); + + t_start512 =clock (); + dlen512 =xbzrle_encode_buffer_avx512 (test512, buffer512,XBZRLE_PAGE_SIZE , + compressed512,XBZRLE_PAGE_SIZE ); + t_end512 =clock (); +float time_val512 =difftime (t_end512, t_start512); + rc512 =xbzrle_decode_buffer (compressed512, dlen512, test512,XBZRLE_PAGE_SIZE ); +g_assert (rc512 <XBZRLE_PAGE_SIZE ); +g_assert (memcmp (test512, buffer512,XBZRLE_PAGE_SIZE ) ==0 ); + +static float result_range[2 ]; + result_range[0 ] = time_val; + result_range[1 ] = time_val512; + +g_free (buffer); +g_free (compressed); +g_free (test); +g_free (buffer512); +g_free (compressed512); +g_free (test512); + +return result_range; +} + +static void test_encode_decode_avx512 (void ) +{ +int i; +float time_raw =0.0 , time_512 =0.0 ; +float *res; +for (i =0 ; i <10000 ; i++) { + res =encode_decode_range_avx512 (); + time_raw += res[0 ]; + time_512 += res[1 ]; + } +printf ("Encode decode test:\n" ); +printf ("Raw xbzrle_encode time is %f ms\n" , time_raw); +printf ("512 xbzrle_encode time is %f ms\n" , time_512); +} + +static float *encode_decode_random (void ) +{ +uint8_t *buffer =g_malloc0 (XBZRLE_PAGE_SIZE ); +uint8_t *compressed =g_malloc (XBZRLE_PAGE_SIZE ); +uint8_t *test =g_malloc0 (XBZRLE_PAGE_SIZE ); +uint8_t *buffer512 =g_malloc0 (XBZRLE_PAGE_SIZE ); +uint8_t *compressed512 =g_malloc (XBZRLE_PAGE_SIZE ); +uint8_t *test512 =g_malloc0 (XBZRLE_PAGE_SIZE ); +int i =0 , rc =0 , rc512 =0 ; +int dlen =0 , dlen512 =0 ; + +int diff_len =g_test_rand_int_range (0 ,XBZRLE_PAGE_SIZE -1 ); +/* store the index of diff */ +int dirty_index[diff_len]; +for (int j =0 ; j < diff_len; j++) { + dirty_index[j] =g_test_rand_int_range (0 ,XBZRLE_PAGE_SIZE -1 ); + } +for (i = diff_len -1 ; i >=0 ; i--) { + buffer[dirty_index[i]] = i; + test[dirty_index[i]] = i +4 ; + buffer512[dirty_index[i]] = i; + test512[dirty_index[i]] = i +4 ; + } + +time_t t_start, t_end, t_start512, t_end512; + t_start =clock (); + dlen =xbzrle_encode_buffer (test, buffer,XBZRLE_PAGE_SIZE , compressed, +XBZRLE_PAGE_SIZE ); + t_end =clock (); +float time_val =difftime (t_end, t_start); + rc =xbzrle_decode_buffer (compressed, dlen, test,XBZRLE_PAGE_SIZE ); +g_assert (rc <XBZRLE_PAGE_SIZE ); + + t_start512 =clock (); + dlen512 =xbzrle_encode_buffer_avx512 (test512, buffer512,XBZRLE_PAGE_SIZE , + compressed512,XBZRLE_PAGE_SIZE ); + t_end512 =clock (); +float time_val512 =difftime (t_end512, t_start512); + rc512 =xbzrle_decode_buffer (compressed512, dlen512, test512,XBZRLE_PAGE_SIZE ); +g_assert (rc512 <XBZRLE_PAGE_SIZE ); + +static float result_random[2 ]; + result_random[0 ] = time_val; + result_random[1 ] = time_val512; + +g_free (buffer); +g_free (compressed); +g_free (test); +g_free (buffer512); +g_free (compressed512); +g_free (test512); + +return result_random; +} + +static void test_encode_decode_random_avx512 (void ) +{ +int i; +float time_raw =0.0 , time_512 =0.0 ; +float *res; +for (i =0 ; i <10000 ; i++) { + res =encode_decode_random (); + time_raw += res[0 ]; + time_512 += res[1 ]; + } +printf ("Random test:\n" ); +printf ("Raw xbzrle_encode time is %f ms\n" , time_raw); +printf ("512 xbzrle_encode time is %f ms\n" , time_512); +} +int main (int argc,char **argv) {g_test_init (&argc, &argv,NULL );g_test_rand_int ();g_test_add_func ("/xbzrle/uleb" , test_uleb); +#if defined(CONFIG_AVX512BW_OPT ) +if (likely (is_cpu_support_avx512bw)) { +g_test_add_func ("/xbzrle/encode_decode_zero" , test_encode_decode_zero_avx512); +g_test_add_func ("/xbzrle/encode_decode_unchanged" , + test_encode_decode_unchanged_avx512); +g_test_add_func ("/xbzrle/encode_decode_1_byte" , test_encode_decode_1_byte_avx512); +g_test_add_func ("/xbzrle/encode_decode_overflow" , + test_encode_decode_overflow_avx512); +g_test_add_func ("/xbzrle/encode_decode" , test_encode_decode_avx512); +g_test_add_func ("/xbzrle/encode_decode_random" , test_encode_decode_random_avx512); + }else { +g_test_add_func ("/xbzrle/encode_decode_zero" , test_encode_decode_zero); +g_test_add_func ("/xbzrle/encode_decode_unchanged" , + test_encode_decode_unchanged); +g_test_add_func ("/xbzrle/encode_decode_1_byte" , test_encode_decode_1_byte); +g_test_add_func ("/xbzrle/encode_decode_overflow" , + test_encode_decode_overflow); +g_test_add_func ("/xbzrle/encode_decode" , test_encode_decode); + } +#else g_test_add_func ("/xbzrle/encode_decode_zero" , test_encode_decode_zero);g_test_add_func ("/xbzrle/encode_decode_unchanged" , test_encode_decode_unchanged);
@@ -185,6 +641,6 @@ int main(int argc, char **argv)g_test_add_func ("/xbzrle/encode_decode_overflow" , test_encode_decode_overflow);g_test_add_func ("/xbzrle/encode_decode" , test_encode_decode); - +#endif return g_test_run (); }
Signature · 1 line
2.25.1