Skip to contentSkip to conversation

iov_iter: streamline iovec/bvec alignment iteration

PATCH Superseded Jens Axboe 2 participants 3 messages Jan 23, 2024 – Jan 24, 2024

Subsystems (2)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Jens Axboe

author
#0 lore raw

PATCH 1/1 iov_iter: streamline iovec/bvec alignment iteration

Rewrite iov_iter_aligned_iovec() and iov_iter_aligned_bvec() to be both easier to read, and also significantly more compact in terms of generated code. This saves 178 bytes of text on x86-64 for me (with clang-18) and 136 bytes on arm64 (with gcc-13).

In profiles, also saves a bit of time as well for the same workload:

     0.81%     -0.18%  [kernel.vmlinux]  [k] iov_iter_aligned_bvec
     0.48%     -0.09%  [kernel.vmlinux]  [k] iov_iter_is_aligned

which is a nice side benefit as well.

lib/iov_iter.c +15 -14
index e0aa6b440ca5..2fcc47d822e3 100644
@@ -714,12 +714,11 @@ EXPORT_SYMBOL(iov_iter_discard);
 static bool iov_iter_aligned_iovec(const struct iov_iter *i, unsigned addr_mask,
 				   unsigned len_mask)
 {
+	const struct iovec *iov = iter_iov(i);
 	size_t size = i->count;
 	size_t skip = i->iov_offset;
-	unsigned k;
 
-	for (k = 0; k < i->nr_segs; k++, skip = 0) {
-		const struct iovec *iov = iter_iov(i) + k;
+	do {
 		size_t len = iov->iov_len - skip;
 
 		if (len > size)
@@ -729,34 +728,36 @@ static bool iov_iter_aligned_iovec(const struct iov_iter *i, unsigned addr_mask,
 		if ((unsigned long)(iov->iov_base + skip) & addr_mask)
 			return false;
 
+		iov++;
 		size -= len;
-		if (!size)
-			break;
-	}
+		skip = 0;
+	} while (size);
+
 	return true;
 }
 
 static bool iov_iter_aligned_bvec(const struct iov_iter *i, unsigned addr_mask,
 				  unsigned len_mask)
 {
-	size_t size = i->count;
+	const struct bio_vec *bvec = i->bvec;
 	unsigned skip = i->iov_offset;
-	unsigned k;
+	size_t size = i->count;
 
-	for (k = 0; k < i->nr_segs; k++, skip = 0) {
-		size_t len = i->bvec[k].bv_len - skip;
+	do {
+		size_t len = bvec->bv_len;
 
 		if (len > size)
 			len = size;
 		if (len & len_mask)
 			return false;
-		if ((unsigned long)(i->bvec[k].bv_offset + skip) & addr_mask)
+		if ((unsigned long)(bvec->bv_offset + skip) & addr_mask)
 			return false;
 
+		bvec++;
 		size -= len;
-		if (!size)
-			break;
-	}
+		skip = 0;
+	} while (size);
+
 	return true;
 }
 
Signature · 1 line
Jens Axboe
#1 lore raw

From: Jens Axboe

Quoted 7 lines

Sent: 23 January 2024 19:49

Rewrite iov_iter_aligned_iovec() and iov_iter_aligned_bvec() to be both easier to read, and also significantly more compact in terms of generated code. This saves 178 bytes of text on x86-64 for me (with clang-18) and 136 bytes on arm64 (with gcc-13).

...

Quoted 5 lines
lib/iov_iter.c +0 -0
index e0aa6b440ca5..2fcc47d822e3 100644
@@ -714,12 +714,11 @@ EXPORT_SYMBOL(iov_iter_discard);

...

lib/iov_iter.c EXPORT_SYMBOL(iov_iter_discard); · line 722

Quoted 4 lines
lib/iov_iter.c +1 -2
-	for (k = 0; k < i->nr_segs; k++, skip = 0) {
-		const struct iovec *iov = iter_iov(i) + k;
+	do {
 		size_t len = iov->iov_len - skip;

Is i->nr_segs allowed to be zero?
The old code (seemed to) check for zero.
The new version will go horribly wrong.

David

-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)

author
#2 lore raw
Quoted 15 lines · David Laight

From: Jens Axboe

Quoted 7 lines

Sent: 23 January 2024 19:49

Rewrite iov_iter_aligned_iovec() and iov_iter_aligned_bvec() to be both easier to read, and also significantly more compact in terms of generated code. This saves 178 bytes of text on x86-64 for me (with clang-18) and 136 bytes on arm64 (with gcc-13).

...

Quoted 5 lines
lib/iov_iter.c +0 -0
index e0aa6b440ca5..2fcc47d822e3 100644
@@ -714,12 +714,11 @@ EXPORT_SYMBOL(iov_iter_discard);

...

Quoted 6 lines

lib/iov_iter.c

Quoted 4 lines
lib/iov_iter.c +1 -2
-	for (k = 0; k < i->nr_segs; k++, skip = 0) {
-		const struct iovec *iov = iter_iov(i) + k;
+	do {
 		size_t len = iov->iov_len - skip;

Is i->nr_segs allowed to be zero?

Not if size is not zero.

Quoted 2 lines

The old code (seemed to) check for zero.
The new version will go horribly wrong.

In other spots we do check both segments and size, but not that nr_segments is non-zero to begin with. So I think this revised version may be better. Saves 176 bytes of text on arm64 for me, and 126 bytes on x86-64. I'll send out a v3.

lib/iov_iter.c +33 -27
index e0aa6b440ca5..5e34639c5d1e 100644
@@ -714,12 +714,12 @@ EXPORT_SYMBOL(iov_iter_discard);
 static bool iov_iter_aligned_iovec(const struct iov_iter *i, unsigned addr_mask,
 				   unsigned len_mask)
 {
+	const struct iovec *iov = iter_iov(i);
 	size_t size = i->count;
 	size_t skip = i->iov_offset;
-	unsigned k;
+	unsigned long nr_segs = i->nr_segs;
 
-	for (k = 0; k < i->nr_segs; k++, skip = 0) {
-		const struct iovec *iov = iter_iov(i) + k;
+	do {
 		size_t len = iov->iov_len - skip;
 
 		if (len > size)
@@ -729,34 +729,37 @@ static bool iov_iter_aligned_iovec(const struct iov_iter *i, unsigned addr_mask,
 		if ((unsigned long)(iov->iov_base + skip) & addr_mask)
 			return false;
 
+		iov++;
 		size -= len;
-		if (!size)
-			break;
-	}
+		skip = 0;
+	} while (size && --nr_segs);
+
 	return true;
 }
 
 static bool iov_iter_aligned_bvec(const struct iov_iter *i, unsigned addr_mask,
 				  unsigned len_mask)
 {
-	size_t size = i->count;
+	const struct bio_vec *bvec = i->bvec;
 	unsigned skip = i->iov_offset;
-	unsigned k;
+	size_t size = i->count;
+	unsigned long nr_segs = i->nr_segs;
 
-	for (k = 0; k < i->nr_segs; k++, skip = 0) {
-		size_t len = i->bvec[k].bv_len - skip;
+	do {
+		size_t len = bvec->bv_len;
 
 		if (len > size)
 			len = size;
 		if (len & len_mask)
 			return false;
-		if ((unsigned long)(i->bvec[k].bv_offset + skip) & addr_mask)
+		if ((unsigned long)(bvec->bv_offset + skip) & addr_mask)
 			return false;
 
+		bvec++;
 		size -= len;
-		if (!size)
-			break;
-	}
+		skip = 0;
+	} while (size && --nr_segs);
+
 	return true;
 }
 
@@ -800,13 +803,13 @@ EXPORT_SYMBOL_GPL(iov_iter_is_aligned);
 
 static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i)
 {
+	const struct iovec *iov = iter_iov(i);
 	unsigned long res = 0;
 	size_t size = i->count;
 	size_t skip = i->iov_offset;
-	unsigned k;
+	unsigned long nr_segs = i->nr_segs;
 
-	for (k = 0; k < i->nr_segs; k++, skip = 0) {
-		const struct iovec *iov = iter_iov(i) + k;
+	do {
 		size_t len = iov->iov_len - skip;
 		if (len) {
 			res |= (unsigned long)iov->iov_base + skip;
@@ -814,30 +817,33 @@ static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i)
 				len = size;
 			res |= len;
 			size -= len;
-			if (!size)
-				break;
 		}
-	}
+		iov++;
+		skip = 0;
+	} while (size && --nr_segs);
+
 	return res;
 }
 
 static unsigned long iov_iter_alignment_bvec(const struct iov_iter *i)
 {
+	const struct bio_vec *bvec = i->bvec;
 	unsigned res = 0;
 	size_t size = i->count;
 	unsigned skip = i->iov_offset;
-	unsigned k;
+	unsigned long nr_segs = i->nr_segs;
 
-	for (k = 0; k < i->nr_segs; k++, skip = 0) {
-		size_t len = i->bvec[k].bv_len - skip;
-		res |= (unsigned long)i->bvec[k].bv_offset + skip;
+	do {
+		size_t len = bvec->bv_len - skip;
+		res |= (unsigned long)bvec->bv_offset + skip;
 		if (len > size)
 			len = size;
 		res |= len;
+		bvec++;
 		size -= len;
-		if (!size)
-			break;
-	}
+		skip = 0;
+	} while (size && --nr_segs);
+
 	return res;
 }
 
Signature · 1 line
Jens Axboe