iov_iter: streamline iovec/bvec alignment iteration
Subsystems (2)
PATCH 1/1 iov_iter: streamline iovec/bvec alignment iteration
Rewrite iov_iter_aligned_iovec() and iov_iter_aligned_bvec() to be both easier to read, and also significantly more compact in terms of generated code. This saves 178 bytes of text on x86-64 for me (with clang-18) and 136 bytes on arm64 (with gcc-13).
In profiles, also saves a bit of time as well for the same workload:
0.81% -0.18% [kernel.vmlinux] [k] iov_iter_aligned_bvec
0.48% -0.09% [kernel.vmlinux] [k] iov_iter_is_alignedwhich is a nice side benefit as well.
lib/iov_iter.c +15 -14
@@ -714,12 +714,11 @@ EXPORT_SYMBOL(iov_iter_discard);static bool iov_iter_aligned_iovec (const struct iov_iter *i,unsigned addr_mask,unsigned len_mask) { +const struct iovec *iov =iter_iov (i);size_t size = i->count;size_t skip = i->iov_offset; -unsigned k; -for (k =0 ; k < i->nr_segs; k++, skip =0 ) { -const struct iovec *iov =iter_iov (i) + k; + do { size_t len = iov->iov_len - skip; if (len > size)
@@ -729,34 +728,36 @@ static bool iov_iter_aligned_iovec(const struct iov_iter *i, unsigned addr_mask,if ((unsigned long )(iov->iov_base + skip) & addr_mask)return false; + iov++; size -= len; -if (!size) -break ; - } + skip =0 ; + }while (size); +return true; }static bool iov_iter_aligned_bvec (const struct iov_iter *i,unsigned addr_mask,unsigned len_mask) { -size_t size = i->count; +const struct bio_vec *bvec = i->bvec;unsigned skip = i->iov_offset; -unsigned k; +size_t size = i->count; -for (k =0 ; k < i->nr_segs; k++, skip =0 ) { -size_t len = i->bvec[k].bv_len - skip; +do { +size_t len = bvec->bv_len;if (len > size) len = size;if (len & len_mask)return false; -if ((unsigned long )(i->bvec[k].bv_offset + skip) & addr_mask) +if ((unsigned long )(bvec->bv_offset + skip) & addr_mask)return false; + bvec++; size -= len; -if (!size) -break ; - } + skip =0 ; + }while (size); +return true; }
Signature · 1 line
Jens Axboe
From: Jens Axboe
Quoted 7 lines
Sent: 23 January 2024 19:49
Rewrite
iov_iter_aligned_iovec()andiov_iter_aligned_bvec()to be both easier to read, and also significantly more compact in terms of generated code. This saves 178 bytes of text on x86-64 for me (with clang-18) and 136 bytes on arm64 (with gcc-13).
...
Quoted 5 lines
lib/iov_iter.c +0
-0@@ -714,12 +714,11 @@ EXPORT_SYMBOL(iov_iter_discard);
...
lib/iov_iter.c EXPORT_SYMBOL(iov_iter_discard); · line 722
Quoted 4 lines
lib/iov_iter.c +1
-2-for (k =0 ; k < i->nr_segs; k++, skip =0 ) { -const struct iovec *iov =iter_iov (i) + k; + do { size_t len = iov->iov_len - skip;
Is i->nr_segs allowed to be zero?
The old code (seemed to) check for zero.
The new version will go horribly wrong.
David
-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)
Quoted 15 lines · David Laight
From: Jens Axboe
Quoted 7 lines
Sent: 23 January 2024 19:49
Rewrite
iov_iter_aligned_iovec()andiov_iter_aligned_bvec()to be both easier to read, and also significantly more compact in terms of generated code. This saves 178 bytes of text on x86-64 for me (with clang-18) and 136 bytes on arm64 (with gcc-13)....
Quoted 5 lines
lib/iov_iter.c +0
-0@@ -714,12 +714,11 @@ EXPORT_SYMBOL(iov_iter_discard);...
Quoted 6 lines
lib/iov_iter.c
Quoted 4 lines
lib/iov_iter.c +1
-2-for (k =0 ; k < i->nr_segs; k++, skip =0 ) { -const struct iovec *iov =iter_iov (i) + k; + do { size_t len = iov->iov_len - skip;Is
i->nr_segsallowed to be zero?
Not if size is not zero.
Quoted 2 lines
The old code (seemed to) check for zero.
The new version will go horribly wrong.
In other spots we do check both segments and size, but not that nr_segments is non-zero to begin with. So I think this revised version may be better. Saves 176 bytes of text on arm64 for me, and 126 bytes on x86-64. I'll send out a v3.
lib/iov_iter.c +33 -27
@@ -714,12 +714,12 @@ EXPORT_SYMBOL(iov_iter_discard);static bool iov_iter_aligned_iovec (const struct iov_iter *i,unsigned addr_mask,unsigned len_mask) { +const struct iovec *iov =iter_iov (i);size_t size = i->count;size_t skip = i->iov_offset; -unsigned k; +unsigned long nr_segs = i->nr_segs; -for (k =0 ; k < i->nr_segs; k++, skip =0 ) { -const struct iovec *iov =iter_iov (i) + k; + do { size_t len = iov->iov_len - skip; if (len > size)
@@ -729,34 +729,37 @@ static bool iov_iter_aligned_iovec(const struct iov_iter *i, unsigned addr_mask,if ((unsigned long )(iov->iov_base + skip) & addr_mask)return false; + iov++; size -= len; -if (!size) -break ; - } + skip =0 ; + }while (size && --nr_segs); +return true; }static bool iov_iter_aligned_bvec (const struct iov_iter *i,unsigned addr_mask,unsigned len_mask) { -size_t size = i->count; +const struct bio_vec *bvec = i->bvec;unsigned skip = i->iov_offset; -unsigned k; +size_t size = i->count; +unsigned long nr_segs = i->nr_segs; -for (k =0 ; k < i->nr_segs; k++, skip =0 ) { -size_t len = i->bvec[k].bv_len - skip; +do { +size_t len = bvec->bv_len;if (len > size) len = size;if (len & len_mask)return false; -if ((unsigned long )(i->bvec[k].bv_offset + skip) & addr_mask) +if ((unsigned long )(bvec->bv_offset + skip) & addr_mask)return false; + bvec++; size -= len; -if (!size) -break ; - } + skip =0 ; + }while (size && --nr_segs); +return true; }
@@ -800,13 +803,13 @@ EXPORT_SYMBOL_GPL(iov_iter_is_aligned);static unsigned long iov_iter_alignment_iovec (const struct iov_iter *i) { +const struct iovec *iov =iter_iov (i);unsigned long res =0 ;size_t size = i->count;size_t skip = i->iov_offset; -unsigned k; +unsigned long nr_segs = i->nr_segs; -for (k =0 ; k < i->nr_segs; k++, skip =0 ) { -const struct iovec *iov =iter_iov (i) + k; + do { size_t len = iov->iov_len - skip; if (len) { res |= (unsigned long)iov->iov_base + skip;
@@ -814,30 +817,33 @@ static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i) len = size; res |= len; size -= len; -if (!size) -break ; } - } + iov++; + skip =0 ; + }while (size && --nr_segs); +return res; }static unsigned long iov_iter_alignment_bvec (const struct iov_iter *i) { +const struct bio_vec *bvec = i->bvec;unsigned res =0 ;size_t size = i->count;unsigned skip = i->iov_offset; -unsigned k; +unsigned long nr_segs = i->nr_segs; -for (k =0 ; k < i->nr_segs; k++, skip =0 ) { -size_t len = i->bvec[k].bv_len - skip; - res |= (unsigned long )i->bvec[k].bv_offset + skip; +do { +size_t len = bvec->bv_len - skip; + res |= (unsigned long )bvec->bv_offset + skip;if (len > size) len = size; res |= len; + bvec++; size -= len; -if (!size) -break ; - } + skip =0 ; + }while (size && --nr_segs); +return res; }
Signature · 1 line
Jens Axboe