Skip to contentSkip to conversation

Test the blk_mq_realloc_hw_ctxs() error path

PATCH Applied Bart Van Assche 3 participants 7 messages Mar 28, 2020 – Apr 8, 2020 Block layer

lore
Share & tools
lore
#0 lore raw

v4 cover Test the blk_mq_realloc_hw_ctxs() error path

Hi Omar,

Please consider this patch series for the official blktests repository.

Thanks,

Bart.

Changes compared to v3:
- Addressed Daniel Wagner's review comments.

Changes compared to v2:
- Addressed Omar's review comments.

Changes between v1 and v2:
- Added three patches that refactor null_blk loading, unloading and configuration (Chaitanya).

Bart Van Assche (4):
Make _exit_null_blk remove all null_blk device instances Use _{init,exit}_null_blk instead of open-coding these functions Introduce the function _configure_null_blk()
Add a test that triggers the blk_mq_realloc_hw_ctxs() error path

 common/multipath-over-rdma | 29 ++++++---------------
 common/null_blk            | 21 ++++++++++++++-
 tests/block/022            |  3 ---
 tests/block/029            | 17 ++-----------
 tests/block/030            | 52 ++++++++++++++++++++++++++++++++++++++
 tests/block/030.out        |  1 +
 tests/nvmeof-mp/rc         |  2 +-
 7 files changed, 84 insertions(+), 41 deletions(-)
 create mode 100755 tests/block/030
 create mode 100644 tests/block/030.out
#1 lore raw

PATCH v4 1/4 Make _exit_null_blk remove all null_blk device instances

Instead of making every test remove null_blk device instances before calling _exit_null_blk(), move the null_blk device instance removal code into _exit_null_blk().

---
 common/null_blk | 7 ++++++-
 tests/block/022 | 3 ---
 tests/block/029 | 1 -
 3 files changed, 6 insertions(+), 5 deletions(-)
common/null_blk +6 -1
index 2e300c20bbc7..a4140e365955 100644
@@ -8,11 +8,15 @@ _have_null_blk() {
 	_have_modules null_blk
 }
 
-_init_null_blk() {
+_remove_null_blk_devices() {
 	if [[ -d /sys/kernel/config/nullb ]]; then
 		find /sys/kernel/config/nullb -mindepth 1 -maxdepth 1 \
 		     -type d -delete
 	fi
+}
+
+_init_null_blk() {
+	_remove_null_blk_devices
 
 	local zoned=""
 	if (( RUN_FOR_ZONED )); then zoned="zoned=1"; fi
@@ -26,6 +30,7 @@ _init_null_blk() {
 }
 
 _exit_null_blk() {
+	_remove_null_blk_devices
 	udevadm settle
 	modprobe -r null_blk
 }
tests/block/022 +0 -3
index 1404aacef295..b2c53e266d81 100755
@@ -50,9 +50,6 @@ test() {
 		wait $pid1
 	} 2>/dev/null
 
-	rmdir /sys/kernel/config/nullb/1
-	rmdir /sys/kernel/config/nullb/0
-
 	_exit_null_blk
 	echo "Test complete"
 }
tests/block/029 +0 -1
index d298bac8db5c..0d521edb0cf6 100755
@@ -58,7 +58,6 @@ test() {
 	else
 		echo "Skipping test because $sq cannot be modified" >>"$FULL"
 	fi
-	rmdir /sys/kernel/config/nullb/nullb0
 	_exit_null_blk
 	echo Passed
 }
#2 lore raw

PATCH v4 2/4 Use _{init,exit}_null_blk instead of open-coding these functions

In reply to Bart Van Assche

This patch reduces code duplication.

---
 common/multipath-over-rdma | 12 +++---------
 tests/nvmeof-mp/rc         |  2 +-
 2 files changed, 4 insertions(+), 10 deletions(-)
common/multipath-over-rdma +3 -9
index 40efc4b3aa2e..a56e7a8269db 100644
@@ -5,6 +5,7 @@
 # Functions and global variables used by both the srp and nvmeof-mp tests.
 
 . common/shellcheck
+. common/null_blk
 
 debug=
 filesystem_type=ext4
@@ -634,13 +635,6 @@ configure_null_blk() {
 	ls -l /dev/nullb* &>>"$FULL"
 }
 
-unload_null_blk() {
-	local d
-
-	for d in /sys/kernel/config/nullb/*; do [ -d "$d" ] && rmdir "$d"; done
-	unload_module null_blk
-}
-
 setup_rdma() {
 	start_soft_rdma
 	(
@@ -662,7 +656,7 @@ teardown_uncond() {
 	rm -f /etc/multipath.conf
 	stop_target
 	stop_soft_rdma
-	unload_null_blk
+	_exit_null_blk
 }
 
 teardown() {
@@ -698,7 +692,7 @@ setup_test() {
 		[ -e "/sys/module/$m" ] || modprobe "$m" || return $?
 	done
 
-	modprobe null_blk nr_devices=0 || return $?
+	_init_null_blk nr_devices=0 || return $?
 
 	configure_null_blk || return $?
 
tests/nvmeof-mp/rc +1 -1
index 1fd631445921..136163bc73ad 100755
@@ -278,7 +278,7 @@ stop_nvme_target() {
 	)
 	unload_module nvmet_rdma &&
 		unload_module nvmet &&
-		unload_null_blk
+		_exit_null_blk
 }
 
 start_target() {
#3 lore raw

PATCH v4 3/4 Introduce the function _configure_null_blk()

In reply to Bart Van Assche

Introduce a function for creating a null_blk device instance through configfs.

---
 common/multipath-over-rdma | 17 +++++------------
 common/null_blk            | 14 ++++++++++++++
 tests/block/029            | 16 ++--------------
 3 files changed, 21 insertions(+), 26 deletions(-)
common/multipath-over-rdma +5 -12
index a56e7a8269db..e689a9633d13 100644
@@ -620,18 +620,11 @@ run_fio() {
 configure_null_blk() {
 	local i
 
-	(
-		cd /sys/kernel/config/nullb || return $?
-		for i in nullb0 nullb1; do (
-			{ mkdir -p $i &&
-				  cd $i &&
-				  echo 0 > completion_nsec &&
-				  echo 512 > blocksize &&
-				  echo $((ramdisk_size>>20)) > size &&
-				  echo 1 > memory_backed &&
-				  echo 1 > power; } || exit $?
-		) done
-	)
+	for i in nullb0 nullb1; do
+		_configure_null_blk $i completion_nsec=0 blocksize=512 \
+				    size=$((ramdisk_size>>20)) memory_backed=1 \
+				    power=1 || return $?
+	done
 	ls -l /dev/nullb* &>>"$FULL"
 }
 
common/null_blk +14 -0
index a4140e365955..6611db03a00e 100644
@@ -29,6 +29,20 @@ _init_null_blk() {
 	return 0
 }
 
+# Configure one null_blk instance with name $1 and parameters $2..${$#}.
+_configure_null_blk() {
+	local nullb=/sys/kernel/config/nullb/$1 param val
+
+	shift
+	mkdir "$nullb" || return $?
+	while [[ $# -gt 0 ]]; do
+		param="${1%%=*}"
+		val="${1#*=}"
+		shift
+		echo "$val" > "$nullb/$param" || return $?
+	done
+}
+
 _exit_null_blk() {
 	_remove_null_blk_devices
 	udevadm settle
tests/block/029 +2 -14
index 0d521edb0cf6..dbb582eab473 100755
@@ -14,19 +14,6 @@ requires() {
 	_have_null_blk
 }
 
-# Configure one null_blk instance.
-configure_null_blk() {
-	local nullb0="/sys/kernel/config/nullb/nullb0"
-
-	mkdir "$nullb0" &&
-	echo 0 > "$nullb0/completion_nsec" &&
-	echo 512 > "$nullb0/blocksize" &&
-	echo 16 > "$nullb0/size" &&
-	echo 1 > "$nullb0/memory_backed" &&
-	echo 1 > "$nullb0/power" &&
-	ls -l /dev/nullb* &>>"$FULL"
-}
-
 modify_nr_hw_queues() {
 	local deadline num_cpus
 
@@ -45,7 +32,8 @@ test() {
 
 	: "${TIMEOUT:=30}"
 	_init_null_blk nr_devices=0 queue_mode=2 &&
-	configure_null_blk
+	_configure_null_blk nullb0 completion_nsec=0 blocksize=512 \
+			    size=16 memory_backed=1 power=1 &&
 	if { echo 1 >$sq; } 2>/dev/null; then
 		modify_nr_hw_queues &
 		fio --rw=randwrite --bs=4K --loops=$((10**6)) \
#4 lore raw

PATCH v4 4/4 Add a test that triggers the blk_mq_realloc_hw_ctxs() error path

In reply to Bart Van Assche

Add a test that triggers the code touched by commit d0930bb8f46b ("blk-mq:
Fix a recently introduced regression in blk_mq_realloc_hw_ctxs()"). This test only runs if a recently added fault injection feature is available, namely commit 596444e75705 ("null_blk: Add support for init_hctx() fault injection").

---
 tests/block/030     | 52 +++++++++++++++++++++++++++++++++++++++++++++
 tests/block/030.out |  1 +
 2 files changed, 53 insertions(+)
 create mode 100755 tests/block/030
 create mode 100644 tests/block/030.out
tests/block/030 new file +52 -0
new file mode 100755
index 000000000000..2b159510e586
@@ -0,0 +1,52 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+# Copyright 2020 Google LLC
+#
+# Trigger the blk_mq_realloc_hw_ctxs() error path.
+
+. tests/block/rc
+. common/null_blk
+
+DESCRIPTION="trigger the blk_mq_realloc_hw_ctxs() error path"
+QUICK=1
+
+requires() {
+	_have_null_blk || return $?
+	_have_module_param null_blk init_hctx || return $?
+}
+
+test() {
+	local i sq=/sys/kernel/config/nullb/nullb0/submit_queues
+
+	: "${TIMEOUT:=30}"
+	# Legend: init_hctx=<interval>,<probability>,<space>,<times>
+	# Set <space> to $(nproc) + 1 to make loading of null_blk succeed.
+	if ! _init_null_blk nr_devices=0 queue_mode=2 \
+	     "init_hctx=$(nproc),100,$(($(nproc)+1)),-1"; then
+		echo "Loading null_blk failed"
+		return 1
+	fi
+	if ! _configure_null_blk nullb0 completion_nsec=0 blocksize=512 size=16\
+	     submit_queues="$(nproc)" memory_backed=1 power=1; then
+		echo "Configuring null_blk failed"
+		return 1
+	fi
+	# Since older null_blk versions do not allow "submit_queues" to be
+	# modified, check first whether that configs attribute is writeable.
+	# Each iteration of the loop below triggers $(nproc) + 1
+	# null_init_hctx() calls. Since <interval>=$(nproc), all possible
+	# blk_mq_realloc_hw_ctxs() error paths will be triggered. Whether or
+	# not this test succeeds depends on whether or not _check_dmesg()
+	# detects a kernel warning.
+	if { echo "$(<"$sq")" >$sq; } 2>/dev/null; then
+		for ((i=0;i<100;i++)); do
+			echo 1 >$sq
+			nproc >$sq
+		done
+	else
+		SKIP_REASON="Skipping test because $sq cannot be modified"
+	fi
+	rmdir /sys/kernel/config/nullb/nullb0
+	_exit_null_blk
+	echo Passed
+}
tests/block/030.out new file +1 -0
new file mode 100644
index 000000000000..863339fb8ced
@@ -0,0 +1 @@
+Passed
Reviewed-by on v4 4/4
#5 lore raw

Re: [PATCH blktests v4 4/4] Add a test that triggers the blk_mq_realloc_hw_ctxs() error path

Hi Bart,

Quoted 9 lines · Bart Van Assche

Add a test that triggers the code touched by commit d0930bb8f46b ("blk-mq:
Fix a recently introduced regression in blk_mq_realloc_hw_ctxs()"). This test only runs if a recently added fault injection feature is available, namely commit 596444e75705 ("null_blk: Add support for init_hctx() fault injection").

Thanks,
Daniel

#6 lore raw
In reply to Bart Van Assche
Quoted 24 lines · Bart Van Assche

Hi Omar,

Please consider this patch series for the official blktests repository.

Thanks,

Bart.

Changes compared to v3:
- Addressed Daniel Wagner's review comments.

Changes compared to v2:
- Addressed Omar's review comments.

Changes between v1 and v2:
- Added three patches that refactor null_blk loading, unloading and configuration (Chaitanya).

Bart Van Assche (4):
Make _exit_null_blk remove all null_blk device instances Use _{init,exit}_null_blk instead of open-coding these functions Introduce the function _configure_null_blk()
Add a test that triggers the blk_mq_realloc_hw_ctxs() error path

Quoted 10 lines
 common/multipath-over-rdma | 29 ++++++---------------
 common/null_blk            | 21 ++++++++++++++-
 tests/block/022            |  3 ---
 tests/block/029            | 17 ++-----------
 tests/block/030            | 52 ++++++++++++++++++++++++++++++++++++++
 tests/block/030.out        |  1 +
 tests/nvmeof-mp/rc         |  2 +-
 7 files changed, 84 insertions(+), 41 deletions(-)
 create mode 100755 tests/block/030
 create mode 100644 tests/block/030.out

Merged, thanks, Bart!