Skip to contentSkip to conversation

fs: move the bdex_statx call to vfs_getattr_nosec

PATCH Applied Christoph Hellwig 6 participants 18 messages Apr 17, 2025 – Apr 25, 2025 1 message not in the archive

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 fs: move the bdex_statx call to vfs_getattr_nosec

Currently bdex_statx is only called from the very high-level vfs_statx_path function, and thus bypassing it for in-kernel calls to vfs_getattr or vfs_getattr_nosec.

This breaks querying the block ѕize of the underlying device in the loop driver and also is a pitfall for any other new kernel caller.

Move the call into the lowest level helper to ensure all callers get the right results.

Fixes 2d985f8c6b91 ("vfs: support STATX_DIOALIGN on block devices")
Fixes f4774e92aab8 ("loop: take the file system minimum dio alignment into account")
---
 block/bdev.c           |  3 +--
 fs/stat.c              | 32 ++++++++++++++++++--------------
 include/linux/blkdev.h |  6 +++---
 3 files changed, 22 insertions(+), 19 deletions(-)
block/bdev.c +1 -2
index 4844d1e27b6f..6a34179192c9 100644
@@ -1272,8 +1272,7 @@ void sync_bdevs(bool wait)
 /*
  * Handle STATX_{DIOALIGN, WRITE_ATOMIC} for block devices.
  */
-void bdev_statx(struct path *path, struct kstat *stat,
-		u32 request_mask)
+void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask)
 {
 	struct inode *backing_inode;
 	struct block_device *bdev;
fs/stat.c +18 -14
index f13308bfdc98..3d9222807214 100644
@@ -204,12 +204,25 @@ int vfs_getattr_nosec(const struct path *path, struct kstat *stat,
 				  STATX_ATTR_DAX);
 
 	idmap = mnt_idmap(path->mnt);
-	if (inode->i_op->getattr)
-		return inode->i_op->getattr(idmap, path, stat,
-					    request_mask,
-					    query_flags);
+	if (inode->i_op->getattr) {
+		int ret;
+
+		ret = inode->i_op->getattr(idmap, path, stat, request_mask,
+				query_flags);
+		if (ret)
+			return ret;
+	} else {
+		generic_fillattr(idmap, request_mask, inode, stat);
+	}
+
+	/*
+	 * If this is a block device inode, override the filesystem attributes
+	 * with the block device specific parameters that need to be obtained
+	 * from the bdev backing inode.
+	 */
+	if (S_ISBLK(stat->mode))
+		bdev_statx(path, stat, request_mask);
 
-	generic_fillattr(idmap, request_mask, inode, stat);
 	return 0;
 }
 EXPORT_SYMBOL(vfs_getattr_nosec);
@@ -295,15 +308,6 @@ static int vfs_statx_path(struct path *path, int flags, struct kstat *stat,
 	if (path_mounted(path))
 		stat->attributes |= STATX_ATTR_MOUNT_ROOT;
 	stat->attributes_mask |= STATX_ATTR_MOUNT_ROOT;
-
-	/*
-	 * If this is a block device inode, override the filesystem
-	 * attributes with the block device specific parameters that need to be
-	 * obtained from the bdev backing inode.
-	 */
-	if (S_ISBLK(stat->mode))
-		bdev_statx(path, stat, request_mask);
-
 	return 0;
 }
 
include/linux/blkdev.h +3 -3
index e39c45bc0a97..678dc38442bf 100644
@@ -1685,7 +1685,7 @@ int sync_blockdev(struct block_device *bdev);
 int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend);
 int sync_blockdev_nowait(struct block_device *bdev);
 void sync_bdevs(bool wait);
-void bdev_statx(struct path *, struct kstat *, u32);
+void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask);
 void printk_all_partitions(void);
 int __init early_lookup_bdev(const char *pathname, dev_t *dev);
 #else
@@ -1703,8 +1703,8 @@ static inline int sync_blockdev_nowait(struct block_device *bdev)
 static inline void sync_bdevs(bool wait)
 {
 }
-static inline void bdev_statx(struct path *path, struct kstat *stat,
-				u32 request_mask)
+static inline void bdev_statx(const struct path *path, struct kstat *stat,
+		u32 request_mask)
 {
 }
 static inline void printk_all_partitions(void)
Signature · 1 line
2.47.2
#1 lore raw
Quoted 7 lines · Christoph Hellwig

Currently bdex_statx is only called from the very high-level vfs_statx_path function, and thus bypassing it for in-kernel calls to vfs_getattr or vfs_getattr_nosec.

This breaks querying the block ѕize of the underlying device in the loop driver and also is a pitfall for any other new kernel caller.

Quoted 8 lines

Move the call into the lowest level helper to ensure all callers get the right results.

Fixes 2d985f8c6b91 ("vfs: support STATX_DIOALIGN on block devices")
Fixes f4774e92aab8 ("loop: take the file system minimum dio alignment into account")

---

This looks fine but one thing below.

Quoted 56 lines
 block/bdev.c           |  3 +--
 fs/stat.c              | 32 ++++++++++++++++++--------------
 include/linux/blkdev.h |  6 +++---
 3 files changed, 22 insertions(+), 19 deletions(-)
block/bdev.c +1 -2
index 4844d1e27b6f..6a34179192c9 100644
@@ -1272,8 +1272,7 @@ void sync_bdevs(bool wait)
 /*
  * Handle STATX_{DIOALIGN, WRITE_ATOMIC} for block devices.
  */
-void bdev_statx(struct path *path, struct kstat *stat,
-		u32 request_mask)
+void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask)
 {
 	struct inode *backing_inode;
 	struct block_device *bdev;
fs/stat.c +18 -5
index f13308bfdc98..3d9222807214 100644
@@ -204,12 +204,25 @@ int vfs_getattr_nosec(const struct path *path, struct kstat *stat,
 				  STATX_ATTR_DAX);
 
 	idmap = mnt_idmap(path->mnt);
-	if (inode->i_op->getattr)
-		return inode->i_op->getattr(idmap, path, stat,
-					    request_mask,
-					    query_flags);
+	if (inode->i_op->getattr) {
+		int ret;
+
+		ret = inode->i_op->getattr(idmap, path, stat, request_mask,
+				query_flags);
+		if (ret)
+			return ret;
+	} else {
+		generic_fillattr(idmap, request_mask, inode, stat);
+	}
+
+	/*
+	 * If this is a block device inode, override the filesystem attributes
+	 * with the block device specific parameters that need to be obtained
+	 * from the bdev backing inode.
+	 */
+	if (S_ISBLK(stat->mode))
+		bdev_statx(path, stat, request_mask);
 
-	generic_fillattr(idmap, request_mask, inode, stat);
 	return 0;
 }
 EXPORT_SYMBOL(vfs_getattr_nosec);
@@ -295,15 +308,6 @@ static int vfs_statx_path(struct path *path, int flags, struct kstat *stat,
 	if (path_mounted(path))

fs/stat.c static int vfs_statx_path(struct path *path, int flags, struct kstat *stat, · line 305

Quoted 10 lines
fs/stat.c +0 -8
 		stat->attributes |= STATX_ATTR_MOUNT_ROOT;
 	stat->attributes_mask |= STATX_ATTR_MOUNT_ROOT;
-
-	/*
-	 * If this is a block device inode, override the filesystem
-	 * attributes with the block device specific parameters that need to be
-	 * obtained from the bdev backing inode.
-	 */
-	if (S_ISBLK(stat->mode))
-		bdev_statx(path, stat, request_mask);

bdev_statx()
-> blkdev_get_no_open()

   {
           if (!inode && IS_ENABLED(CONFIG_BLOCK_LEGACY_AUTOLOAD)) {
                blk_request_module(dev);
                inode = ilookup(blockdev_superblock, dev);
                if (inode)
                        pr_warn_ratelimited("block device autoloading is deprecated and will be removed.\n");
        }
   }

So that means any unprivileged Schmock can trigger a module autoload from statx() if that's enabled? It feels like this should really be disabled for statx().

maintainer Applied
#2 lore raw
In reply to Christoph Hellwig
Quoted 8 lines · Christoph Hellwig

Currently bdex_statx is only called from the very high-level vfs_statx_path function, and thus bypassing it for in-kernel calls to vfs_getattr or vfs_getattr_nosec.

This breaks querying the block ѕize of the underlying device in the loop driver and also is a pitfall for any other new kernel caller.

[...]

Applied to the vfs.fixes branch of the vfs/vfs.git tree. Patches in the vfs.fixes branch should appear in linux-next soon.

Please report any outstanding bugs that were missed during review in a new review to the original patch series allowing us to drop it.

It's encouraged to provide Acked-bys and Reviewed-bys even though the patch has now been applied. If possible patch trailers will be updated.

Note that commit hashes shown below are subject to change due to rebase, trailer updates or similar. If in doubt, please check the listed branch.

tree:   https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git
branch: vfs.fixes

[1/1] fs: move the bdex_statx call to vfs_getattr_nosec

      https://git.kernel.org/vfs/vfs/c/777d0961ff95
Tested-by on v1 1/1
#3 lore raw
In reply to Christoph Hellwig
Quoted 7 lines · Christoph Hellwig

Currently bdex_statx is only called from the very high-level vfs_statx_path function, and thus bypassing it for in-kernel calls to vfs_getattr or vfs_getattr_nosec.

This breaks querying the block ѕize of the underlying device in the loop driver and also is a pitfall for any other new kernel caller.

Quoted 7 lines

Move the call into the lowest level helper to ensure all callers get the right results.

Fixes 2d985f8c6b91 ("vfs: support STATX_DIOALIGN on block devices")
Fixes f4774e92aab8 ("loop: take the file system minimum dio alignment into account")

This appears to solve the problem as well.

--D

Quoted 99 lines
---
 block/bdev.c           |  3 +--
 fs/stat.c              | 32 ++++++++++++++++++--------------
 include/linux/blkdev.h |  6 +++---
 3 files changed, 22 insertions(+), 19 deletions(-)
block/bdev.c +1 -2
index 4844d1e27b6f..6a34179192c9 100644
@@ -1272,8 +1272,7 @@ void sync_bdevs(bool wait)
 /*
  * Handle STATX_{DIOALIGN, WRITE_ATOMIC} for block devices.
  */
-void bdev_statx(struct path *path, struct kstat *stat,
-		u32 request_mask)
+void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask)
 {
 	struct inode *backing_inode;
 	struct block_device *bdev;
fs/stat.c +18 -14
index f13308bfdc98..3d9222807214 100644
@@ -204,12 +204,25 @@ int vfs_getattr_nosec(const struct path *path, struct kstat *stat,
 				  STATX_ATTR_DAX);
 
 	idmap = mnt_idmap(path->mnt);
-	if (inode->i_op->getattr)
-		return inode->i_op->getattr(idmap, path, stat,
-					    request_mask,
-					    query_flags);
+	if (inode->i_op->getattr) {
+		int ret;
+
+		ret = inode->i_op->getattr(idmap, path, stat, request_mask,
+				query_flags);
+		if (ret)
+			return ret;
+	} else {
+		generic_fillattr(idmap, request_mask, inode, stat);
+	}
+
+	/*
+	 * If this is a block device inode, override the filesystem attributes
+	 * with the block device specific parameters that need to be obtained
+	 * from the bdev backing inode.
+	 */
+	if (S_ISBLK(stat->mode))
+		bdev_statx(path, stat, request_mask);
 
-	generic_fillattr(idmap, request_mask, inode, stat);
 	return 0;
 }
 EXPORT_SYMBOL(vfs_getattr_nosec);
@@ -295,15 +308,6 @@ static int vfs_statx_path(struct path *path, int flags, struct kstat *stat,
 	if (path_mounted(path))
 		stat->attributes |= STATX_ATTR_MOUNT_ROOT;
 	stat->attributes_mask |= STATX_ATTR_MOUNT_ROOT;
-
-	/*
-	 * If this is a block device inode, override the filesystem
-	 * attributes with the block device specific parameters that need to be
-	 * obtained from the bdev backing inode.
-	 */
-	if (S_ISBLK(stat->mode))
-		bdev_statx(path, stat, request_mask);
-
 	return 0;
 }
 
include/linux/blkdev.h +3 -3
index e39c45bc0a97..678dc38442bf 100644
@@ -1685,7 +1685,7 @@ int sync_blockdev(struct block_device *bdev);
 int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend);
 int sync_blockdev_nowait(struct block_device *bdev);
 void sync_bdevs(bool wait);
-void bdev_statx(struct path *, struct kstat *, u32);
+void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask);
 void printk_all_partitions(void);
 int __init early_lookup_bdev(const char *pathname, dev_t *dev);
 #else
@@ -1703,8 +1703,8 @@ static inline int sync_blockdev_nowait(struct block_device *bdev)
 static inline void sync_bdevs(bool wait)
 {
 }
-static inline void bdev_statx(struct path *path, struct kstat *stat,
-				u32 request_mask)
+static inline void bdev_statx(const struct path *path, struct kstat *stat,
+		u32 request_mask)
 {
 }
 static inline void printk_all_partitions(void)
Signature · 1 line
2.47.2
#4 lore raw
In reply to Christoph Hellwig
Quoted 9 lines · Christoph Hellwig

Currently bdex_statx is only called from the very high-level vfs_statx_path function, and thus bypassing it for in-kernel calls to vfs_getattr or vfs_getattr_nosec.

This breaks querying the block ѕize of the underlying device in the loop driver and also is a pitfall for any other new kernel caller.

Move the call into the lowest level helper to ensure all callers get the right results.

I ran blktests with the kernel v6.15-rc3, and found the test case md/001 hangs. The hang is recreated in stable manner. I bisected and found this patch as the commit 777d0961ff95 is the trigger. When I revert the commit from v6.15-rc3 kernel, the hang disappeared.

Actions for fix will be appreciated.

FYI, the kernel INFO messages recorded functions relevant to the trigger commit, such as bdev_statx or vfs_getattr_nosec [1].

[1]

[ 3341.286639] [ T129061] run blktests md/001 at 2025-04-21 14:23:48
[ 3341.519196] [ T129079] brd: module loaded
[ 3341.629701] [ T129098] nvmet: adding nsid 1 to subsystem blktests-subsystem-1
[ 3341.637856] [ T129099] nvmet_tcp: enabling port 0 (127.0.0.1:4420)
[ 3341.677605] [  T80307] nvmet: Created nvm controller 1 for subsystem blktests-subsystem-1 for NQN nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349.
[ 3341.680557] [ T129106] nvme nvme1: Please enable CONFIG_NVME_MULTIPATH for full support of multi-port devices.
[ 3341.683890] [ T129106] nvme nvme1: creating 4 I/O queues.
[ 3341.685462] [ T129106] nvme nvme1: failed to connect socket: -512
[ 3341.717282] [    T182] nvmet: Created nvm controller 1 for subsystem blktests-subsystem-1 for NQN nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349.
[ 3341.720494] [ T129106] nvme nvme1: Please enable CONFIG_NVME_MULTIPATH for full support of multi-port devices.
[ 3341.722675] [ T129106] nvme nvme1: creating 4 I/O queues.
[ 3341.725909] [ T129106] nvme nvme1: mapped 4/0/0 default/read/poll queues.
[ 3341.728769] [ T129106] nvme nvme1: new ctrl: NQN "blktests-subsystem-1", addr 127.0.0.1:4420, hostnqn: nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349
[ 3347.648972] [ T129119] md/raid1:md127: active with 1 out of 2 mirrors
[ 3347.651175] [ T129119] md127: detected capacity change from 0 to 2093056
[ 3347.700481] [ T129124] md127: detected capacity change from 2093056 to 0
[ 3347.701652] [ T129124] md: md127 stopped.
[ 3347.728822] [ T129126] nvme nvme1: Removing ctrl: NQN "blktests-subsystem-1"
[ 3359.911067] [  T92330] nvmet: ctrl 1 keep-alive timer (5 seconds) expired!
[ 3359.914119] [  T92330] nvmet: ctrl 1 fatal error occurred!
[ 3565.221305] [     T46] INFO: task kdevtmpfs:40 blocked for more than 122 seconds.
[ 3565.222332] [     T46]       Tainted: G    B   W           6.15.0-rc3 #15
[ 3565.223197] [     T46] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3565.224277] [     T46] task:kdevtmpfs       state:D stack:0     pid:40    tgid:40    ppid:2      task_flags:0x208140 flags:0x00004000
[ 3565.225452] [     T46] Call Trace:
[ 3565.226106] [     T46]  <TASK>
[ 3565.226708] [     T46]  __schedule+0x107c/0x5fa0
[ 3565.227416] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.228179] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.228886] [     T46]  ? __pfx___schedule+0x10/0x10
[ 3565.229599] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.230320] [     T46]  ? unwind_next_frame+0x3f6/0x1f10
[ 3565.231090] [     T46]  ? schedule+0x2be/0x390
[ 3565.231759] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.232474] [     T46]  ? unwind_next_frame+0x3f6/0x1f10
[ 3565.233288] [     T46]  ? lock_release+0x217/0x2c0
[ 3565.234013] [     T46]  schedule+0xde/0x390
[ 3565.234672] [     T46]  schedule_timeout+0x176/0x230
[ 3565.235389] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3565.236169] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.236867] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3565.237605] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.238310] [     T46]  __wait_for_common+0x2ea/0x4c0
[ 3565.239040] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3565.239770] [     T46]  ? __pfx___wait_for_common+0x10/0x10
[ 3565.240509] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3565.241240] [     T46]  ? flush_workqueue_prep_pwqs+0x333/0x440
[ 3565.242031] [     T46]  __flush_workqueue+0x373/0x10a0
[ 3565.242715] [     T46]  ? __pfx___might_resched+0x10/0x10
[ 3565.243417] [     T46]  ? kasan_save_track+0x10/0x30
[ 3565.244108] [     T46]  ? kasan_save_free_info+0x37/0x60
[ 3565.244782] [     T46]  ? __mutex_lock+0x538/0x1a40
[ 3565.245430] [     T46]  ? __pfx___flush_workqueue+0x10/0x10
[ 3565.246122] [     T46]  ? find_inode_fast+0x152/0x3f0
[ 3565.246755] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.247391] [     T46]  md_alloc+0x28/0xf10
[ 3565.247987] [     T46]  md_probe+0x25/0x70
[ 3565.248531] [     T46]  blk_probe_dev+0x108/0x180
[ 3565.249134] [     T46]  blk_request_module+0xe/0x60
[ 3565.249716] [     T46]  blkdev_get_no_open+0x5c/0xa0
[ 3565.250311] [     T46]  bdev_statx+0x98/0x550
[ 3565.250859] [     T46]  vfs_getattr_nosec+0x2be/0x3d0
[ 3565.251435] [     T46]  handle_remove+0x115/0x500
[ 3565.252010] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.252558] [     T46]  ? __pfx_handle_remove+0x10/0x10
[ 3565.253144] [     T46]  ? finish_task_switch.isra.0+0x1b9/0x860
[ 3565.253757] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.254312] [     T46]  ? __schedule+0x3099/0x5fa0
[ 3565.254858] [     T46]  ? __pfx_sched_clock_cpu+0x10/0x10
[ 3565.255438] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3565.256011] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3565.256554] [     T46]  ? devtmpfs_work_loop+0xb5/0x5c7
[ 3565.257113] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.257633] [     T46]  ? lock_release+0x217/0x2c0
[ 3565.258150] [     T46]  ? devtmpfs_work_loop+0x55d/0x5c7
[ 3565.258687] [     T46]  devtmpfs_work_loop+0x4e9/0x5c7
[ 3565.259223] [     T46]  ? __pfx_devtmpfs_work_loop+0x10/0x10
[ 3565.259791] [     T46]  ? _raw_spin_unlock_irqrestore+0x35/0x60
[ 3565.260379] [     T46]  ? __pfx_do_raw_spin_lock+0x10/0x10
[ 3565.260959] [     T46]  ? devtmpfsd+0x19/0x30
[ 3565.261442] [     T46]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 3565.262046] [     T46]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 3565.262624] [     T46]  ? __pfx_devtmpfsd+0x10/0x10
[ 3565.263149] [     T46]  devtmpfsd+0x27/0x30
[ 3565.263618] [     T46]  kthread+0x3b0/0x770
[ 3565.264098] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3565.264599] [     T46]  ? ret_from_fork+0x17/0x70
[ 3565.265129] [     T46]  ? ret_from_fork+0x17/0x70
[ 3565.265639] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3565.266197] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3565.266710] [     T46]  ret_from_fork+0x30/0x70
[ 3565.267220] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3565.267732] [     T46]  ret_from_fork_asm+0x1a/0x30
[ 3565.268264] [     T46]  </TASK>
[ 3565.268732] [     T46] INFO: task kworker/1:1:91856 blocked for more than 122 seconds.
[ 3565.269468] [     T46]       Tainted: G    B   W           6.15.0-rc3 #15
[ 3565.270150] [     T46] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3565.270985] [     T46] task:kworker/1:1     state:D stack:0     pid:91856 tgid:91856 ppid:2      task_flags:0x4288060 flags:0x00004000
[ 3565.272043] [     T46] Workqueue: md_misc mddev_delayed_delete
[ 3565.272658] [     T46] Call Trace:
[ 3565.273146] [     T46]  <TASK>
[ 3565.273568] [     T46]  __schedule+0x107c/0x5fa0
[ 3565.274117] [     T46]  ? unwind_next_frame+0x400/0x1f10
[ 3565.274697] [     T46]  ? ret_from_fork_asm+0x1a/0x30
[ 3565.275267] [     T46]  ? __pfx_stack_trace_consume_entry+0x10/0x10
[ 3565.275920] [     T46]  ? __pfx___schedule+0x10/0x10
[ 3565.276489] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3565.277083] [     T46]  ? __pfx_do_raw_spin_lock+0x10/0x10
[ 3565.277674] [     T46]  ? schedule+0x2be/0x390
[ 3565.278208] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.278762] [     T46]  ? lock_release+0x217/0x2c0
[ 3565.279320] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.279902] [     T46]  schedule+0xde/0x390
[ 3565.280418] [     T46]  schedule_timeout+0x176/0x230
[ 3565.281026] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3565.281617] [     T46]  ? llist_add_batch+0xb9/0x130
[ 3565.282190] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.282742] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3565.283330] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.283895] [     T46]  __wait_for_common+0x2ea/0x4c0
[ 3565.284466] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3565.285084] [     T46]  ? __pfx___wait_for_common+0x10/0x10
[ 3565.285682] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3565.286262] [     T46]  ? lock_release+0x217/0x2c0
[ 3565.286825] [     T46]  devtmpfs_delete_node+0x142/0x1b0
[ 3565.287420] [     T46]  ? __pfx_devtmpfs_delete_node+0x10/0x10
[ 3565.288056] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.288593] [     T46]  ? __call_rcu_common.constprop.0+0x45f/0x1090
[ 3565.289260] [     T46]  ? trace_hardirqs_on+0x12/0x120
[ 3565.289854] [     T46]  ? kernfs_put.part.0+0x130/0x470
[ 3565.290441] [     T46]  device_del+0x6e2/0x9b0
[ 3565.290993] [     T46]  ? pm_runtime_set_memalloc_noio+0x8b/0x140
[ 3565.291633] [     T46]  ? __pfx_device_del+0x10/0x10
[ 3565.292206] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.292769] [     T46]  ? trace_hardirqs_on+0x12/0x120
[ 3565.293361] [     T46]  ? _raw_spin_unlock_irq+0x34/0x50
[ 3565.293975] [     T46]  del_gendisk+0x4f5/0xa20
[ 3565.294520] [     T46]  ? __pfx_del_gendisk+0x10/0x10
[ 3565.295198] [     T46]  ? up_write+0x1c6/0x500
[ 3565.295736] [     T46]  md_kobj_release+0x90/0xe0
[ 3565.296304] [     T46]  kobject_put+0x17b/0x4a0
[ 3565.296871] [     T46]  process_one_work+0x84f/0x1460
[ 3565.297466] [     T46]  ? __pfx_process_one_work+0x10/0x10
[ 3565.298100] [     T46]  ? assign_work+0x16c/0x240
[ 3565.298656] [     T46]  worker_thread+0x5ef/0xfd0
[ 3565.299197] [     T46]  ? __kthread_parkme+0xb4/0x200
[ 3565.299761] [     T46]  ? __pfx_worker_thread+0x10/0x10
[ 3565.300322] [     T46]  kthread+0x3b0/0x770
[ 3565.300813] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3565.301339] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.301883] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3565.302441] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.302991] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3565.303505] [     T46]  ret_from_fork+0x30/0x70
[ 3565.304028] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3565.304539] [     T46]  ret_from_fork_asm+0x1a/0x30
[ 3565.305087] [     T46]  </TASK>
[ 3565.305515] [     T46] INFO: task nvme:129126 blocked for more than 122 seconds.
[ 3565.306223] [     T46]       Tainted: G    B   W           6.15.0-rc3 #15
[ 3565.306879] [     T46] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3565.307662] [     T46] task:nvme            state:D stack:0     pid:129126 tgid:129126 ppid:129061 task_flags:0x480100 flags:0x00024002
[ 3565.308665] [     T46] Call Trace:
[ 3565.309140] [     T46]  <TASK>
[ 3565.309574] [     T46]  __schedule+0x107c/0x5fa0
[ 3565.310119] [     T46]  ? lock_release+0x217/0x2c0
[ 3565.310662] [     T46]  ? __pfx_stack_trace_consume_entry+0x10/0x10
[ 3565.311307] [     T46]  ? unwind_get_return_address+0x5a/0xa0
[ 3565.311927] [     T46]  ? __pfx___schedule+0x10/0x10
[ 3565.312490] [     T46]  ? schedule+0x2be/0x390
[ 3565.313037] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.313587] [     T46]  ? stack_trace_save+0x8f/0xc0
[ 3565.314179] [     T46]  ? lock_release+0x217/0x2c0
[ 3565.314706] [     T46]  schedule+0xde/0x390
[ 3565.315208] [     T46]  schedule_timeout+0x176/0x230
[ 3565.315777] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3565.316386] [     T46]  ? llist_add_batch+0xb9/0x130
[ 3565.316981] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.317554] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3565.318155] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.318716] [     T46]  __wait_for_common+0x2ea/0x4c0
[ 3565.319294] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3565.319911] [     T46]  ? __pfx___wait_for_common+0x10/0x10
[ 3565.320520] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3565.321120] [     T46]  ? lock_release+0x217/0x2c0
[ 3565.321673] [     T46]  devtmpfs_delete_node+0x142/0x1b0
[ 3565.322270] [     T46]  ? __pfx_devtmpfs_delete_node+0x10/0x10
[ 3565.322904] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3565.323503] [     T46]  ? klist_release+0x2c9/0x460
[ 3565.324087] [     T46]  ? do_raw_spin_unlock+0x54/0x1e0
[ 3565.324667] [     T46]  ? kobject_put+0x5d/0x4a0
[ 3565.325224] [     T46]  device_del+0x6e2/0x9b0
[ 3565.325755] [     T46]  ? __pfx_device_del+0x10/0x10
[ 3565.326329] [     T46]  ? ktime_get_mono_fast_ns+0x81/0x360
[ 3565.326958] [     T46]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 3565.327586] [     T46]  cdev_device_del+0x19/0xf0
[ 3565.328141] [     T46]  nvme_ns_remove+0x2ab/0x740 [nvme_core]
[ 3565.328762] [     T46]  nvme_remove_namespaces+0x25d/0x3a0 [nvme_core]
[ 3565.329435] [     T46]  ? __pfx_nvme_remove_namespaces+0x10/0x10 [nvme_core]
[ 3565.330162] [     T46]  ? __pfx___might_resched+0x10/0x10
[ 3565.330748] [     T46]  ? __pfx_sysfs_kf_write+0x10/0x10
[ 3565.331337] [     T46]  nvme_do_delete_ctrl+0xf5/0x160 [nvme_core]
[ 3565.332006] [     T46]  nvme_delete_ctrl_sync.cold+0x8/0xd [nvme_core]
[ 3565.332672] [     T46]  nvme_sysfs_delete+0x92/0xb0 [nvme_core]
[ 3565.333317] [     T46]  kernfs_fop_write_iter+0x39f/0x5a0
[ 3565.333926] [     T46]  vfs_write+0x5f7/0xe90
[ 3565.334463] [     T46]  ? __pfx_vfs_write+0x10/0x10
[ 3565.335103] [     T46]  ksys_write+0xf5/0x1c0
[ 3565.335627] [     T46]  ? __pfx_ksys_write+0x10/0x10
[ 3565.336192] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.336736] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3565.337344] [     T46]  ? do_syscall_64+0x54/0x190
[ 3565.337883] [     T46]  do_syscall_64+0x93/0x190
[ 3565.338410] [     T46]  ? kmem_cache_free+0x393/0x580
[ 3565.338977] [     T46]  ? fput_close_sync+0x100/0x170
[ 3565.339519] [     T46]  ? fput_close_sync+0x100/0x170
[ 3565.340084] [     T46]  ? __pfx_fput_close_sync+0x10/0x10
[ 3565.340643] [     T46]  ? lock_release+0x217/0x2c0
[ 3565.341188] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3565.341774] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.342314] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3565.342914] [     T46]  ? trace_hardirqs_on_prepare+0xdb/0x120
[ 3565.343512] [     T46]  ? syscall_exit_to_user_mode+0x93/0x280
[ 3565.344124] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3565.344652] [     T46]  ? do_sys_openat2+0x108/0x180
[ 3565.345201] [     T46]  ? __pfx_do_sys_openat2+0x10/0x10
[ 3565.345757] [     T46]  ? syscall_exit_to_user_mode+0x93/0x280
[ 3565.346360] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3565.346902] [     T46]  ? __x64_sys_openat+0x104/0x1d0
[ 3565.347462] [     T46]  ? __pfx___x64_sys_openat+0x10/0x10
[ 3565.348056] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.348589] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3565.349200] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.349734] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3565.350339] [     T46]  ? trace_hardirqs_on_prepare+0xdb/0x120
[ 3565.350962] [     T46]  ? syscall_exit_to_user_mode+0x93/0x280
[ 3565.351555] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3565.352088] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3565.352635] [     T46]  entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 3565.353249] [     T46] RIP: 0033:0x7fedbaf01504
[ 3565.353760] [     T46] RSP: 002b:00007fff4c0f4858 EFLAGS: 00000202 ORIG_RAX: 0000000000000001
[ 3565.354536] [     T46] RAX: ffffffffffffffda RBX: 0000000000000003 RCX: 00007fedbaf01504
[ 3565.355287] [     T46] RDX: 0000000000000001 RSI: 00007fedbb050746 RDI: 0000000000000003
[ 3565.356037] [     T46] RBP: 00007fedbb050746 R08: 0000000000004000 R09: 00000000ffffffff
[ 3565.356770] [     T46] R10: 0000000000000000 R11: 0000000000000202 R12: 000000000e107e30
[ 3565.357515] [     T46] R13: 00007fff4c0f563b R14: 000000000e107e30 R15: 000000000e108730
[ 3565.358274] [     T46]  </TASK>
[ 3565.358743] [     T46] INFO: lockdep is turned off.
[ 3688.100058] [     T46] INFO: task kdevtmpfs:40 blocked for more than 245 seconds.
[ 3688.101052] [     T46]       Tainted: G    B   W           6.15.0-rc3 #15
[ 3688.101732] [     T46] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3688.102546] [     T46] task:kdevtmpfs       state:D stack:0     pid:40    tgid:40    ppid:2      task_flags:0x208140 flags:0x00004000
[ 3688.103524] [     T46] Call Trace:
[ 3688.104003] [     T46]  <TASK>
[ 3688.104446] [     T46]  __schedule+0x107c/0x5fa0
[ 3688.105102] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.105659] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.106221] [     T46]  ? __pfx___schedule+0x10/0x10
[ 3688.106780] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.107332] [     T46]  ? unwind_next_frame+0x3f6/0x1f10
[ 3688.108088] [     T46]  ? schedule+0x2be/0x390
[ 3688.108611] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.109172] [     T46]  ? unwind_next_frame+0x3f6/0x1f10
[ 3688.109744] [     T46]  ? lock_release+0x217/0x2c0
[ 3688.110299] [     T46]  schedule+0xde/0x390
[ 3688.110828] [     T46]  schedule_timeout+0x176/0x230
[ 3688.111388] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3688.112153] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.112709] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3688.113304] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.113943] [     T46]  __wait_for_common+0x2ea/0x4c0
[ 3688.114515] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3688.115122] [     T46]  ? __pfx___wait_for_common+0x10/0x10
[ 3688.115718] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3688.116313] [     T46]  ? flush_workqueue_prep_pwqs+0x333/0x440
[ 3688.116970] [     T46]  __flush_workqueue+0x373/0x10a0
[ 3688.117543] [     T46]  ? __pfx___might_resched+0x10/0x10
[ 3688.118147] [     T46]  ? kasan_save_track+0x10/0x30
[ 3688.118710] [     T46]  ? kasan_save_free_info+0x37/0x60
[ 3688.119305] [     T46]  ? __mutex_lock+0x538/0x1a40
[ 3688.119941] [     T46]  ? __pfx___flush_workqueue+0x10/0x10
[ 3688.120541] [     T46]  ? find_inode_fast+0x152/0x3f0
[ 3688.121122] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.121682] [     T46]  md_alloc+0x28/0xf10
[ 3688.122208] [     T46]  md_probe+0x25/0x70
[ 3688.122710] [     T46]  blk_probe_dev+0x108/0x180
[ 3688.123266] [     T46]  blk_request_module+0xe/0x60
[ 3688.123988] [     T46]  blkdev_get_no_open+0x5c/0xa0
[ 3688.124550] [     T46]  bdev_statx+0x98/0x550
[ 3688.125083] [     T46]  vfs_getattr_nosec+0x2be/0x3d0
[ 3688.125641] [     T46]  handle_remove+0x115/0x500
[ 3688.126188] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.126728] [     T46]  ? __pfx_handle_remove+0x10/0x10
[ 3688.127296] [     T46]  ? finish_task_switch.isra.0+0x1b9/0x860
[ 3688.128084] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.128623] [     T46]  ? __schedule+0x3099/0x5fa0
[ 3688.129162] [     T46]  ? __pfx_sched_clock_cpu+0x10/0x10
[ 3688.129723] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3688.130282] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3688.130957] [     T46]  ? devtmpfs_work_loop+0xb5/0x5c7
[ 3688.131498] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.132020] [     T46]  ? lock_release+0x217/0x2c0
[ 3688.132514] [     T46]  ? devtmpfs_work_loop+0x55d/0x5c7
[ 3688.133055] [     T46]  devtmpfs_work_loop+0x4e9/0x5c7
[ 3688.133574] [     T46]  ? __pfx_devtmpfs_work_loop+0x10/0x10
[ 3688.134135] [     T46]  ? _raw_spin_unlock_irqrestore+0x35/0x60
[ 3688.134706] [     T46]  ? __pfx_do_raw_spin_lock+0x10/0x10
[ 3688.135261] [     T46]  ? devtmpfsd+0x19/0x30
[ 3688.135811] [     T46]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 3688.136392] [     T46]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 3688.136987] [     T46]  ? __pfx_devtmpfsd+0x10/0x10
[ 3688.137492] [     T46]  devtmpfsd+0x27/0x30
[ 3688.137968] [     T46]  kthread+0x3b0/0x770
[ 3688.138431] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3688.139103] [     T46]  ? ret_from_fork+0x17/0x70
[ 3688.139601] [     T46]  ? ret_from_fork+0x17/0x70
[ 3688.140125] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3688.140653] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3688.141159] [     T46]  ret_from_fork+0x30/0x70
[ 3688.141640] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3688.142144] [     T46]  ret_from_fork_asm+0x1a/0x30
[ 3688.142647] [     T46]  </TASK>
[ 3688.143114] [     T46] INFO: task kworker/1:1:91856 blocked for more than 245 seconds.
[ 3688.143823] [     T46]       Tainted: G    B   W           6.15.0-rc3 #15
[ 3688.144445] [     T46] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3688.145321] [     T46] task:kworker/1:1     state:D stack:0     pid:91856 tgid:91856 ppid:2      task_flags:0x4288060 flags:0x00004000
[ 3688.146310] [     T46] Workqueue: md_misc mddev_delayed_delete
[ 3688.147072] [     T46] Call Trace:
[ 3688.147516] [     T46]  <TASK>
[ 3688.147961] [     T46]  __schedule+0x107c/0x5fa0
[ 3688.148475] [     T46]  ? unwind_next_frame+0x400/0x1f10
[ 3688.149066] [     T46]  ? ret_from_fork_asm+0x1a/0x30
[ 3688.149603] [     T46]  ? __pfx_stack_trace_consume_entry+0x10/0x10
[ 3688.150228] [     T46]  ? __pfx___schedule+0x10/0x10
[ 3688.150802] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3688.151351] [     T46]  ? __pfx_do_raw_spin_lock+0x10/0x10
[ 3688.151946] [     T46]  ? schedule+0x2be/0x390
[ 3688.152449] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.153025] [     T46]  ? lock_release+0x217/0x2c0
[ 3688.153552] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.154097] [     T46]  schedule+0xde/0x390
[ 3688.154584] [     T46]  schedule_timeout+0x176/0x230
[ 3688.155136] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3688.155702] [     T46]  ? llist_add_batch+0xb9/0x130
[ 3688.156252] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.156820] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3688.157378] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.157999] [     T46]  __wait_for_common+0x2ea/0x4c0
[ 3688.158540] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3688.159130] [     T46]  ? __pfx___wait_for_common+0x10/0x10
[ 3688.159702] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3688.160264] [     T46]  ? lock_release+0x217/0x2c0
[ 3688.160844] [     T46]  devtmpfs_delete_node+0x142/0x1b0
[ 3688.161406] [     T46]  ? __pfx_devtmpfs_delete_node+0x10/0x10
[ 3688.162025] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.162558] [     T46]  ? __call_rcu_common.constprop.0+0x45f/0x1090
[ 3688.163195] [     T46]  ? trace_hardirqs_on+0x12/0x120
[ 3688.163810] [     T46]  ? kernfs_put.part.0+0x130/0x470
[ 3688.164370] [     T46]  device_del+0x6e2/0x9b0
[ 3688.164925] [     T46]  ? pm_runtime_set_memalloc_noio+0x8b/0x140
[ 3688.165540] [     T46]  ? __pfx_device_del+0x10/0x10
[ 3688.166096] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.166636] [     T46]  ? trace_hardirqs_on+0x12/0x120
[ 3688.167205] [     T46]  ? _raw_spin_unlock_irq+0x34/0x50
[ 3688.167812] [     T46]  del_gendisk+0x4f5/0xa20
[ 3688.168340] [     T46]  ? __pfx_del_gendisk+0x10/0x10
[ 3688.168939] [     T46]  ? up_write+0x1c6/0x500
[ 3688.169457] [     T46]  md_kobj_release+0x90/0xe0
[ 3688.170025] [     T46]  kobject_put+0x17b/0x4a0
[ 3688.170551] [     T46]  process_one_work+0x84f/0x1460
[ 3688.171184] [     T46]  ? __pfx_process_one_work+0x10/0x10
[ 3688.171795] [     T46]  ? assign_work+0x16c/0x240
[ 3688.172329] [     T46]  worker_thread+0x5ef/0xfd0
[ 3688.172879] [     T46]  ? __kthread_parkme+0xb4/0x200
[ 3688.173424] [     T46]  ? __pfx_worker_thread+0x10/0x10
[ 3688.174011] [     T46]  kthread+0x3b0/0x770
[ 3688.174485] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3688.175008] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.175529] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3688.176083] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.176599] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3688.177119] [     T46]  ret_from_fork+0x30/0x70
[ 3688.177612] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3688.178127] [     T46]  ret_from_fork_asm+0x1a/0x30
[ 3688.178644] [     T46]  </TASK>
[ 3688.179094] [     T46] INFO: task nvme:129126 blocked for more than 245 seconds.
[ 3688.179803] [     T46]       Tainted: G    B   W           6.15.0-rc3 #15
[ 3688.180440] [     T46] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3688.181224] [     T46] task:nvme            state:D stack:0     pid:129126 tgid:129126 ppid:129061 task_flags:0x480100 flags:0x00024002
[ 3688.182234] [     T46] Call Trace:
[ 3688.182686] [     T46]  <TASK>
[ 3688.183128] [     T46]  __schedule+0x107c/0x5fa0
[ 3688.183652] [     T46]  ? lock_release+0x217/0x2c0
[ 3688.184201] [     T46]  ? __pfx_stack_trace_consume_entry+0x10/0x10
[ 3688.184914] [     T46]  ? unwind_get_return_address+0x5a/0xa0
[ 3688.185508] [     T46]  ? __pfx___schedule+0x10/0x10
[ 3688.186068] [     T46]  ? schedule+0x2be/0x390
[ 3688.186583] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.187138] [     T46]  ? stack_trace_save+0x8f/0xc0
[ 3688.187684] [     T46]  ? lock_release+0x217/0x2c0
[ 3688.188234] [     T46]  schedule+0xde/0x390
[ 3688.188735] [     T46]  schedule_timeout+0x176/0x230
[ 3688.189296] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3688.189959] [     T46]  ? llist_add_batch+0xb9/0x130
[ 3688.190503] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.191057] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3688.191610] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.192155] [     T46]  __wait_for_common+0x2ea/0x4c0
[ 3688.192694] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3688.193271] [     T46]  ? __pfx___wait_for_common+0x10/0x10
[ 3688.193878] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3688.194423] [     T46]  ? lock_release+0x217/0x2c0
[ 3688.194976] [     T46]  devtmpfs_delete_node+0x142/0x1b0
[ 3688.195530] [     T46]  ? __pfx_devtmpfs_delete_node+0x10/0x10
[ 3688.196123] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3688.196669] [     T46]  ? klist_release+0x2c9/0x460
[ 3688.197213] [     T46]  ? do_raw_spin_unlock+0x54/0x1e0
[ 3688.197792] [     T46]  ? kobject_put+0x5d/0x4a0
[ 3688.198317] [     T46]  device_del+0x6e2/0x9b0
[ 3688.198863] [     T46]  ? __pfx_device_del+0x10/0x10
[ 3688.199406] [     T46]  ? ktime_get_mono_fast_ns+0x81/0x360
[ 3688.200058] [     T46]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 3688.200658] [     T46]  cdev_device_del+0x19/0xf0
[ 3688.201205] [     T46]  nvme_ns_remove+0x2ab/0x740 [nvme_core]
[ 3688.201859] [     T46]  nvme_remove_namespaces+0x25d/0x3a0 [nvme_core]
[ 3688.202510] [     T46]  ? __pfx_nvme_remove_namespaces+0x10/0x10 [nvme_core]
[ 3688.203270] [     T46]  ? __pfx___might_resched+0x10/0x10
[ 3688.203879] [     T46]  ? __pfx_sysfs_kf_write+0x10/0x10
[ 3688.204451] [     T46]  nvme_do_delete_ctrl+0xf5/0x160 [nvme_core]
[ 3688.205130] [     T46]  nvme_delete_ctrl_sync.cold+0x8/0xd [nvme_core]
[ 3688.205813] [     T46]  nvme_sysfs_delete+0x92/0xb0 [nvme_core]
[ 3688.206438] [     T46]  kernfs_fop_write_iter+0x39f/0x5a0
[ 3688.207046] [     T46]  vfs_write+0x5f7/0xe90
[ 3688.207566] [     T46]  ? __pfx_vfs_write+0x10/0x10
[ 3688.208131] [     T46]  ksys_write+0xf5/0x1c0
[ 3688.208645] [     T46]  ? __pfx_ksys_write+0x10/0x10
[ 3688.209209] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.209784] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3688.210373] [     T46]  ? do_syscall_64+0x54/0x190
[ 3688.210937] [     T46]  do_syscall_64+0x93/0x190
[ 3688.211448] [     T46]  ? kmem_cache_free+0x393/0x580
[ 3688.212063] [     T46]  ? fput_close_sync+0x100/0x170
[ 3688.212625] [     T46]  ? fput_close_sync+0x100/0x170
[ 3688.213203] [     T46]  ? __pfx_fput_close_sync+0x10/0x10
[ 3688.213790] [     T46]  ? lock_release+0x217/0x2c0
[ 3688.214308] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3688.214963] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.215484] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3688.216103] [     T46]  ? trace_hardirqs_on_prepare+0xdb/0x120
[ 3688.216685] [     T46]  ? syscall_exit_to_user_mode+0x93/0x280
[ 3688.217279] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3688.217836] [     T46]  ? do_sys_openat2+0x108/0x180
[ 3688.218369] [     T46]  ? __pfx_do_sys_openat2+0x10/0x10
[ 3688.218965] [     T46]  ? syscall_exit_to_user_mode+0x93/0x280
[ 3688.219546] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3688.220091] [     T46]  ? __x64_sys_openat+0x104/0x1d0
[ 3688.220632] [     T46]  ? __pfx___x64_sys_openat+0x10/0x10
[ 3688.221208] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.221768] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3688.222362] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.222937] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3688.223518] [     T46]  ? trace_hardirqs_on_prepare+0xdb/0x120
[ 3688.224114] [     T46]  ? syscall_exit_to_user_mode+0x93/0x280
[ 3688.224696] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3688.225230] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3688.225787] [     T46]  entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 3688.226384] [     T46] RIP: 0033:0x7fedbaf01504
[ 3688.226943] [     T46] RSP: 002b:00007fff4c0f4858 EFLAGS: 00000202 ORIG_RAX: 0000000000000001
[ 3688.227692] [     T46] RAX: ffffffffffffffda RBX: 0000000000000003 RCX: 00007fedbaf01504
[ 3688.228430] [     T46] RDX: 0000000000000001 RSI: 00007fedbb050746 RDI: 0000000000000003
[ 3688.229195] [     T46] RBP: 00007fedbb050746 R08: 0000000000004000 R09: 00000000ffffffff
[ 3688.229956] [     T46] R10: 0000000000000000 R11: 0000000000000202 R12: 000000000e107e30
[ 3688.230687] [     T46] R13: 00007fff4c0f563b R14: 000000000e107e30 R15: 000000000e108730
[ 3688.231429] [     T46]  </TASK>
[ 3688.232181] [     T46] INFO: lockdep is turned off.
[ 3810.978749] [     T46] INFO: task kdevtmpfs:40 blocked for more than 368 seconds.
[ 3810.981216] [     T46]       Tainted: G    B   W           6.15.0-rc3 #15
[ 3810.983118] [     T46] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3810.985109] [     T46] task:kdevtmpfs       state:D stack:0     pid:40    tgid:40    ppid:2      task_flags:0x208140 flags:0x00004000
[ 3810.987369] [     T46] Call Trace:
[ 3810.988864] [     T46]  <TASK>
[ 3810.989909] [     T46]  __schedule+0x107c/0x5fa0
[ 3810.990439] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3810.990952] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3810.991388] [     T46]  ? __pfx___schedule+0x10/0x10
[ 3810.991907] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3810.992387] [     T46]  ? unwind_next_frame+0x3f6/0x1f10
[ 3810.992931] [     T46]  ? schedule+0x2be/0x390
[ 3810.993384] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3810.993915] [     T46]  ? unwind_next_frame+0x3f6/0x1f10
[ 3810.994372] [     T46]  ? lock_release+0x217/0x2c0
[ 3810.994873] [     T46]  schedule+0xde/0x390
[ 3810.995386] [     T46]  schedule_timeout+0x176/0x230
[ 3810.995941] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3810.996422] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3810.996978] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3810.997449] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3810.997996] [     T46]  __wait_for_common+0x2ea/0x4c0
[ 3810.998463] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3810.998991] [     T46]  ? __pfx___wait_for_common+0x10/0x10
[ 3810.999474] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3811.000007] [     T46]  ? flush_workqueue_prep_pwqs+0x333/0x440
[ 3811.000501] [     T46]  __flush_workqueue+0x373/0x10a0
[ 3811.000990] [     T46]  ? __pfx___might_resched+0x10/0x10
[ 3811.001467] [     T46]  ? kasan_save_track+0x10/0x30
[ 3811.001978] [     T46]  ? kasan_save_free_info+0x37/0x60
[ 3811.002462] [     T46]  ? __mutex_lock+0x538/0x1a40
[ 3811.002967] [     T46]  ? __pfx___flush_workqueue+0x10/0x10
[ 3811.003445] [     T46]  ? find_inode_fast+0x152/0x3f0
[ 3811.003983] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.004451] [     T46]  md_alloc+0x28/0xf10
[ 3811.004940] [     T46]  md_probe+0x25/0x70
[ 3811.005377] [     T46]  blk_probe_dev+0x108/0x180
[ 3811.005886] [     T46]  blk_request_module+0xe/0x60
[ 3811.006363] [     T46]  blkdev_get_no_open+0x5c/0xa0
[ 3811.006872] [     T46]  bdev_statx+0x98/0x550
[ 3811.007391] [     T46]  vfs_getattr_nosec+0x2be/0x3d0
[ 3811.007907] [     T46]  handle_remove+0x115/0x500
[ 3811.008359] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.008866] [     T46]  ? __pfx_handle_remove+0x10/0x10
[ 3811.009420] [     T46]  ? finish_task_switch.isra.0+0x1b9/0x860
[ 3811.009955] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.010412] [     T46]  ? __schedule+0x3099/0x5fa0
[ 3811.010916] [     T46]  ? __pfx_sched_clock_cpu+0x10/0x10
[ 3811.011385] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3811.011911] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3811.012360] [     T46]  ? devtmpfs_work_loop+0xb5/0x5c7
[ 3811.012864] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.013356] [     T46]  ? lock_release+0x217/0x2c0
[ 3811.013832] [     T46]  ? devtmpfs_work_loop+0x55d/0x5c7
[ 3811.014327] [     T46]  devtmpfs_work_loop+0x4e9/0x5c7
[ 3811.014756] [     T46]  ? __pfx_devtmpfs_work_loop+0x10/0x10
[ 3811.015254] [     T46]  ? _raw_spin_unlock_irqrestore+0x35/0x60
[ 3811.015714] [     T46]  ? __pfx_do_raw_spin_lock+0x10/0x10
[ 3811.016206] [     T46]  ? devtmpfsd+0x19/0x30
[ 3811.016610] [     T46]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 3811.017086] [     T46]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 3811.017541] [     T46]  ? __pfx_devtmpfsd+0x10/0x10
[ 3811.017970] [     T46]  devtmpfsd+0x27/0x30
[ 3811.018373] [     T46]  kthread+0x3b0/0x770
[ 3811.018815] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3811.019280] [     T46]  ? ret_from_fork+0x17/0x70
[ 3811.019698] [     T46]  ? ret_from_fork+0x17/0x70
[ 3811.020138] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3811.020563] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3811.020985] [     T46]  ret_from_fork+0x30/0x70
[ 3811.021390] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3811.021845] [     T46]  ret_from_fork_asm+0x1a/0x30
[ 3811.022326] [     T46]  </TASK>
[ 3811.022731] [     T46] INFO: task kworker/1:1:91856 blocked for more than 368 seconds.
[ 3811.023350] [     T46]       Tainted: G    B   W           6.15.0-rc3 #15
[ 3811.023881] [     T46] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3811.024493] [     T46] task:kworker/1:1     state:D stack:0     pid:91856 tgid:91856 ppid:2      task_flags:0x4288060 flags:0x00004000
[ 3811.025268] [     T46] Workqueue: md_misc mddev_delayed_delete
[ 3811.025780] [     T46] Call Trace:
[ 3811.026225] [     T46]  <TASK>
[ 3811.026598] [     T46]  __schedule+0x107c/0x5fa0
[ 3811.027047] [     T46]  ? unwind_next_frame+0x400/0x1f10
[ 3811.027504] [     T46]  ? ret_from_fork_asm+0x1a/0x30
[ 3811.027984] [     T46]  ? __pfx_stack_trace_consume_entry+0x10/0x10
[ 3811.028487] [     T46]  ? __pfx___schedule+0x10/0x10
[ 3811.029016] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3811.029475] [     T46]  ? __pfx_do_raw_spin_lock+0x10/0x10
[ 3811.029997] [     T46]  ? schedule+0x2be/0x390
[ 3811.030435] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.030959] [     T46]  ? lock_release+0x217/0x2c0
[ 3811.031405] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.031906] [     T46]  schedule+0xde/0x390
[ 3811.032397] [     T46]  schedule_timeout+0x176/0x230
[ 3811.032898] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3811.033389] [     T46]  ? llist_add_batch+0xb9/0x130
[ 3811.033890] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.034346] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3811.034868] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.035385] [     T46]  __wait_for_common+0x2ea/0x4c0
[ 3811.035914] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3811.036387] [     T46]  ? __pfx___wait_for_common+0x10/0x10
[ 3811.036916] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3811.037380] [     T46]  ? lock_release+0x217/0x2c0
[ 3811.037907] [     T46]  devtmpfs_delete_node+0x142/0x1b0
[ 3811.038381] [     T46]  ? __pfx_devtmpfs_delete_node+0x10/0x10
[ 3811.038914] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.039374] [     T46]  ? __call_rcu_common.constprop.0+0x45f/0x1090
[ 3811.039940] [     T46]  ? trace_hardirqs_on+0x12/0x120
[ 3811.040409] [     T46]  ? kernfs_put.part.0+0x130/0x470
[ 3811.040970] [     T46]  device_del+0x6e2/0x9b0
[ 3811.041410] [     T46]  ? pm_runtime_set_memalloc_noio+0x8b/0x140
[ 3811.042003] [     T46]  ? __pfx_device_del+0x10/0x10
[ 3811.042469] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.042994] [     T46]  ? trace_hardirqs_on+0x12/0x120
[ 3811.043462] [     T46]  ? _raw_spin_unlock_irq+0x34/0x50
[ 3811.043990] [     T46]  del_gendisk+0x4f5/0xa20
[ 3811.044441] [     T46]  ? __pfx_del_gendisk+0x10/0x10
[ 3811.044958] [     T46]  ? up_write+0x1c6/0x500
[ 3811.045404] [     T46]  md_kobj_release+0x90/0xe0
[ 3811.045908] [     T46]  kobject_put+0x17b/0x4a0
[ 3811.046363] [     T46]  process_one_work+0x84f/0x1460
[ 3811.046881] [     T46]  ? __pfx_process_one_work+0x10/0x10
[ 3811.047364] [     T46]  ? assign_work+0x16c/0x240
[ 3811.047883] [     T46]  worker_thread+0x5ef/0xfd0
[ 3811.048422] [     T46]  ? __kthread_parkme+0xb4/0x200
[ 3811.048920] [     T46]  ? __pfx_worker_thread+0x10/0x10
[ 3811.049383] [     T46]  kthread+0x3b0/0x770
[ 3811.049857] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3811.050357] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.050849] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3811.051369] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.051859] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3811.052367] [     T46]  ret_from_fork+0x30/0x70
[ 3811.052848] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3811.053346] [     T46]  ret_from_fork_asm+0x1a/0x30
[ 3811.053832] [     T46]  </TASK>
[ 3811.054259] [     T46] INFO: task nvme:129126 blocked for more than 368 seconds.
[ 3811.054829] [     T46]       Tainted: G    B   W           6.15.0-rc3 #15
[ 3811.055339] [     T46] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3811.055968] [     T46] task:nvme            state:D stack:0     pid:129126 tgid:129126 ppid:129061 task_flags:0x480100 flags:0x00024002
[ 3811.056668] [     T46] Call Trace:
[ 3811.057088] [     T46]  <TASK>
[ 3811.057478] [     T46]  __schedule+0x107c/0x5fa0
[ 3811.057976] [     T46]  ? lock_release+0x217/0x2c0
[ 3811.058427] [     T46]  ? __pfx_stack_trace_consume_entry+0x10/0x10
[ 3811.058978] [     T46]  ? unwind_get_return_address+0x5a/0xa0
[ 3811.059467] [     T46]  ? __pfx___schedule+0x10/0x10
[ 3811.059996] [     T46]  ? schedule+0x2be/0x390
[ 3811.060432] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.060948] [     T46]  ? stack_trace_save+0x8f/0xc0
[ 3811.061404] [     T46]  ? lock_release+0x217/0x2c0
[ 3811.061938] [     T46]  schedule+0xde/0x390
[ 3811.062379] [     T46]  schedule_timeout+0x176/0x230
[ 3811.062896] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3811.063402] [     T46]  ? llist_add_batch+0xb9/0x130
[ 3811.063923] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.064377] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3811.064894] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.065420] [     T46]  __wait_for_common+0x2ea/0x4c0
[ 3811.065946] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3811.066419] [     T46]  ? __pfx___wait_for_common+0x10/0x10
[ 3811.066939] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3811.067398] [     T46]  ? lock_release+0x217/0x2c0
[ 3811.067891] [     T46]  devtmpfs_delete_node+0x142/0x1b0
[ 3811.068353] [     T46]  ? __pfx_devtmpfs_delete_node+0x10/0x10
[ 3811.068873] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3811.069412] [     T46]  ? klist_release+0x2c9/0x460
[ 3811.069949] [     T46]  ? do_raw_spin_unlock+0x54/0x1e0
[ 3811.070412] [     T46]  ? kobject_put+0x5d/0x4a0
[ 3811.070913] [     T46]  device_del+0x6e2/0x9b0
[ 3811.071344] [     T46]  ? __pfx_device_del+0x10/0x10
[ 3811.071848] [     T46]  ? ktime_get_mono_fast_ns+0x81/0x360
[ 3811.072392] [     T46]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 3811.072935] [     T46]  cdev_device_del+0x19/0xf0
[ 3811.073385] [     T46]  nvme_ns_remove+0x2ab/0x740 [nvme_core]
[ 3811.073938] [     T46]  nvme_remove_namespaces+0x25d/0x3a0 [nvme_core]
[ 3811.074454] [     T46]  ? __pfx_nvme_remove_namespaces+0x10/0x10 [nvme_core]
[ 3811.075060] [     T46]  ? __pfx___might_resched+0x10/0x10
[ 3811.075529] [     T46]  ? __pfx_sysfs_kf_write+0x10/0x10
[ 3811.076003] [     T46]  nvme_do_delete_ctrl+0xf5/0x160 [nvme_core]
[ 3811.076512] [     T46]  nvme_delete_ctrl_sync.cold+0x8/0xd [nvme_core]
[ 3811.077086] [     T46]  nvme_sysfs_delete+0x92/0xb0 [nvme_core]
[ 3811.077575] [     T46]  kernfs_fop_write_iter+0x39f/0x5a0
[ 3811.078049] [     T46]  vfs_write+0x5f7/0xe90
[ 3811.078491] [     T46]  ? __pfx_vfs_write+0x10/0x10
[ 3811.078974] [     T46]  ksys_write+0xf5/0x1c0
[ 3811.079411] [     T46]  ? __pfx_ksys_write+0x10/0x10
[ 3811.079905] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.080358] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3811.080877] [     T46]  ? do_syscall_64+0x54/0x190
[ 3811.081390] [     T46]  do_syscall_64+0x93/0x190
[ 3811.081860] [     T46]  ? kmem_cache_free+0x393/0x580
[ 3811.082373] [     T46]  ? fput_close_sync+0x100/0x170
[ 3811.082880] [     T46]  ? fput_close_sync+0x100/0x170
[ 3811.083334] [     T46]  ? __pfx_fput_close_sync+0x10/0x10
[ 3811.083830] [     T46]  ? lock_release+0x217/0x2c0
[ 3811.084319] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3811.084849] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.085350] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3811.085891] [     T46]  ? trace_hardirqs_on_prepare+0xdb/0x120
[ 3811.086369] [     T46]  ? syscall_exit_to_user_mode+0x93/0x280
[ 3811.086882] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3811.087395] [     T46]  ? do_sys_openat2+0x108/0x180
[ 3811.087880] [     T46]  ? __pfx_do_sys_openat2+0x10/0x10
[ 3811.088341] [     T46]  ? syscall_exit_to_user_mode+0x93/0x280
[ 3811.088875] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3811.089391] [     T46]  ? __x64_sys_openat+0x104/0x1d0
[ 3811.089911] [     T46]  ? __pfx___x64_sys_openat+0x10/0x10
[ 3811.090374] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.090864] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3811.091357] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.091858] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3811.092359] [     T46]  ? trace_hardirqs_on_prepare+0xdb/0x120
[ 3811.092883] [     T46]  ? syscall_exit_to_user_mode+0x93/0x280
[ 3811.093383] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3811.093882] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3811.094404] [     T46]  entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 3811.094973] [     T46] RIP: 0033:0x7fedbaf01504
[ 3811.095420] [     T46] RSP: 002b:00007fff4c0f4858 EFLAGS: 00000202 ORIG_RAX: 0000000000000001
[ 3811.096046] [     T46] RAX: ffffffffffffffda RBX: 0000000000000003 RCX: 00007fedbaf01504
[ 3811.096630] [     T46] RDX: 0000000000000001 RSI: 00007fedbb050746 RDI: 0000000000000003
[ 3811.097321] [     T46] RBP: 00007fedbb050746 R08: 0000000000004000 R09: 00000000ffffffff
[ 3811.098089] [     T46] R10: 0000000000000000 R11: 0000000000000202 R12: 000000000e107e30
[ 3811.099055] [     T46] R13: 00007fff4c0f563b R14: 000000000e107e30 R15: 000000000e108730
[ 3811.099823] [     T46]  </TASK>
[ 3811.100334] [     T46] INFO: lockdep is turned off.
[ 3933.856614] [     T46] INFO: task kdevtmpfs:40 blocked for more than 491 seconds.
[ 3933.857467] [     T46]       Tainted: G    B   W           6.15.0-rc3 #15
[ 3933.858156] [     T46] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3933.859165] [     T46] task:kdevtmpfs       state:D stack:0     pid:40    tgid:40    ppid:2      task_flags:0x208140 flags:0x00004000
[ 3933.860620] [     T46] Call Trace:
[ 3933.861122] [     T46]  <TASK>
[ 3933.861575] [     T46]  __schedule+0x107c/0x5fa0
[ 3933.862257] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3933.862831] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3933.863397] [     T46]  ? __pfx___schedule+0x10/0x10
[ 3933.864129] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3933.864702] [     T46]  ? unwind_next_frame+0x3f6/0x1f10
[ 3933.865291] [     T46]  ? schedule+0x2be/0x390
[ 3933.865822] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3933.866390] [     T46]  ? unwind_next_frame+0x3f6/0x1f10
[ 3933.867232] [     T46]  ? lock_release+0x217/0x2c0
[ 3933.867797] [     T46]  schedule+0xde/0x390
[ 3933.868329] [     T46]  schedule_timeout+0x176/0x230
[ 3933.868912] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3933.869527] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3933.870289] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3933.870930] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3933.871523] [     T46]  __wait_for_common+0x2ea/0x4c0
[ 3933.872251] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3933.872889] [     T46]  ? __pfx___wait_for_common+0x10/0x10
[ 3933.873514] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3933.874186] [     T46]  ? flush_workqueue_prep_pwqs+0x333/0x440
[ 3933.874825] [     T46]  __flush_workqueue+0x373/0x10a0
[ 3933.875432] [     T46]  ? __pfx___might_resched+0x10/0x10
[ 3933.876080] [     T46]  ? kasan_save_track+0x10/0x30
[ 3933.876673] [     T46]  ? kasan_save_free_info+0x37/0x60
[ 3933.877267] [     T46]  ? __mutex_lock+0x538/0x1a40
[ 3933.877843] [     T46]  ? __pfx___flush_workqueue+0x10/0x10
[ 3933.878460] [     T46]  ? find_inode_fast+0x152/0x3f0
[ 3933.879104] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3933.879677] [     T46]  md_alloc+0x28/0xf10
[ 3933.880201] [     T46]  md_probe+0x25/0x70
[ 3933.880718] [     T46]  blk_probe_dev+0x108/0x180
[ 3933.881272] [     T46]  blk_request_module+0xe/0x60
[ 3933.881834] [     T46]  blkdev_get_no_open+0x5c/0xa0
[ 3933.882405] [     T46]  bdev_statx+0x98/0x550
[ 3933.882950] [     T46]  vfs_getattr_nosec+0x2be/0x3d0
[ 3933.883518] [     T46]  handle_remove+0x115/0x500
[ 3933.884067] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3933.884612] [     T46]  ? __pfx_handle_remove+0x10/0x10
[ 3933.885174] [     T46]  ? finish_task_switch.isra.0+0x1b9/0x860
[ 3933.885779] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3933.886325] [     T46]  ? __schedule+0x3099/0x5fa0
[ 3933.886873] [     T46]  ? __pfx_sched_clock_cpu+0x10/0x10
[ 3933.887445] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3933.888004] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3933.888546] [     T46]  ? devtmpfs_work_loop+0xb5/0x5c7
[ 3933.889108] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3933.889628] [     T46]  ? lock_release+0x217/0x2c0
[ 3933.890134] [     T46]  ? devtmpfs_work_loop+0x55d/0x5c7
[ 3933.890676] [     T46]  devtmpfs_work_loop+0x4e9/0x5c7
[ 3933.891207] [     T46]  ? __pfx_devtmpfs_work_loop+0x10/0x10
[ 3933.891768] [     T46]  ? _raw_spin_unlock_irqrestore+0x35/0x60
[ 3933.892356] [     T46]  ? __pfx_do_raw_spin_lock+0x10/0x10
[ 3933.892924] [     T46]  ? devtmpfsd+0x19/0x30
[ 3933.893414] [     T46]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 3933.894004] [     T46]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 3933.894584] [     T46]  ? __pfx_devtmpfsd+0x10/0x10
[ 3933.895099] [     T46]  devtmpfsd+0x27/0x30
[ 3933.895571] [     T46]  kthread+0x3b0/0x770
[ 3933.896086] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3933.896596] [     T46]  ? ret_from_fork+0x17/0x70
[ 3933.897099] [     T46]  ? ret_from_fork+0x17/0x70
[ 3933.897600] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3933.898138] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3933.898642] [     T46]  ret_from_fork+0x30/0x70
[ 3933.899130] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3933.899635] [     T46]  ret_from_fork_asm+0x1a/0x30
[ 3933.900148] [     T46]  </TASK>
[ 3933.900555] [     T46] Future hung task reports are suppressed, see sysctl kernel.hung_task_warnings
[ 3933.901497] [     T46] INFO: lockdep is turned off.
author
#5 lore raw
Quoted 9 lines · Shinichiro Kawasaki

I ran blktests with the kernel v6.15-rc3, and found the test case md/001 hangs. The hang is recreated in stable manner. I bisected and found this patch as the commit 777d0961ff95 is the trigger. When I revert the commit from v6.15-rc3 kernel, the hang disappeared.

Actions for fix will be appreciated.

FYI, the kernel INFO messages recorded functions relevant to the trigger commit, such as bdev_statx or vfs_getattr_nosec [1].

This should fix it:

block/bdev.c +13 -8
index 6a34179192c9..97d4c0ab1670 100644
@@ -1274,18 +1274,23 @@ void sync_bdevs(bool wait)
  */
 void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask)
 {
-	struct inode *backing_inode;
 	struct block_device *bdev;
 
-	backing_inode = d_backing_inode(path->dentry);
-
 	/*
-	 * Note that backing_inode is the inode of a block device node file,
-	 * not the block device's internal inode.  Therefore it is *not* valid
-	 * to use I_BDEV() here; the block device has to be looked up by i_rdev
-	 * instead.
+	 * Note that d_backing_inode() returnsthe inode of a block device node
+	 * file, not the block device's internal inode.
+	 *
+	 * Therefore it is *not* valid to use I_BDEV() here; the block device
+	 * has to be looked up by i_rdev instead.
+	 *
+	 * Only do this lookup if actually needed to avoid the performance
+	 * overhead of the lookup, and to avoid injecting bdev lifetime issues
+	 * into devtmpfs.
 	 */
-	bdev = blkdev_get_no_open(backing_inode->i_rdev);
+	if (!(request_mask & (STATX_DIOALIGN | STATX_WRITE_ATOMIC)))
+		return;
+
+	bdev = blkdev_get_no_open(d_backing_inode(path->dentry)->i_rdev);
 	if (!bdev)
 		return;
 
#6 lore raw
Quoted 6 lines · hch
Quoted 5 lines · Shinichiro Kawasaki

I ran blktests with the kernel v6.15-rc3, and found the test case md/001 hangs. The hang is recreated in stable manner. I bisected and found this patch as the commit 777d0961ff95 is the trigger. When I revert the commit from v6.15-rc3 kernel, the hang disappeared.

Quoted 6 lines
Quoted 4 lines

Actions for fix will be appreciated.

FYI, the kernel INFO messages recorded functions relevant to the trigger commit, such as bdev_statx or vfs_getattr_nosec [1].

This should fix it:

Can you send this as a proper patch so I can pick it up, please? It needs to go upstream.

Quoted 37 lines
block/bdev.c +13 -8
index 6a34179192c9..97d4c0ab1670 100644
@@ -1274,18 +1274,23 @@ void sync_bdevs(bool wait)
  */
 void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask)
 {
-	struct inode *backing_inode;
 	struct block_device *bdev;
 
-	backing_inode = d_backing_inode(path->dentry);
-
 	/*
-	 * Note that backing_inode is the inode of a block device node file,
-	 * not the block device's internal inode.  Therefore it is *not* valid
-	 * to use I_BDEV() here; the block device has to be looked up by i_rdev
-	 * instead.
+	 * Note that d_backing_inode() returnsthe inode of a block device node
+	 * file, not the block device's internal inode.
+	 *
+	 * Therefore it is *not* valid to use I_BDEV() here; the block device
+	 * has to be looked up by i_rdev instead.
+	 *
+	 * Only do this lookup if actually needed to avoid the performance
+	 * overhead of the lookup, and to avoid injecting bdev lifetime issues
+	 * into devtmpfs.
 	 */
-	bdev = blkdev_get_no_open(backing_inode->i_rdev);
+	if (!(request_mask & (STATX_DIOALIGN | STATX_WRITE_ATOMIC)))
+		return;
+
+	bdev = blkdev_get_no_open(d_backing_inode(path->dentry)->i_rdev);
 	if (!bdev)
 		return;
 
author
#7 lore raw
Quoted 2 lines · Christian Brauner

Can you send this as a proper patch so I can pick it up, please? It needs to go upstream.

I'll wait for a tested-by tag from at least on of the reports while I'm running more tests. The previous one was obviously a bit hurried so I'm all for not rushing this round.

#8 lore raw
In reply to hch
Quoted 34 lines · hch
Quoted 9 lines · Shinichiro Kawasaki

I ran blktests with the kernel v6.15-rc3, and found the test case md/001 hangs. The hang is recreated in stable manner. I bisected and found this patch as the commit 777d0961ff95 is the trigger. When I revert the commit from v6.15-rc3 kernel, the hang disappeared.

Actions for fix will be appreciated.

FYI, the kernel INFO messages recorded functions relevant to the trigger commit, such as bdev_statx or vfs_getattr_nosec [1].

This should fix it:

block/bdev.c +3 -7
index 6a34179192c9..97d4c0ab1670 100644
@@ -1274,18 +1274,23 @@ void sync_bdevs(bool wait)
  */
 void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask)
 {
-	struct inode *backing_inode;
 	struct block_device *bdev;
 
-	backing_inode = d_backing_inode(path->dentry);
-
 	/*
-	 * Note that backing_inode is the inode of a block device node file,
-	 * not the block device's internal inode.  Therefore it is *not* valid
-	 * to use I_BDEV() here; the block device has to be looked up by i_rdev
-	 * instead.
+	 * Note that d_backing_inode() returnsthe inode of a block device node
+	 * file, not the block device's internal inode.
+	 *

block/bdev.c void sync_bdevs(bool wait) · line 1291

Quoted 10 lines
block/bdev.c +8 -1
+	 * Therefore it is *not* valid to use I_BDEV() here; the block device
+	 * has to be looked up by i_rdev instead.
+	 *
+	 * Only do this lookup if actually needed to avoid the performance
+	 * overhead of the lookup, and to avoid injecting bdev lifetime issues
+	 * into devtmpfs.
 	 */
-	bdev = blkdev_get_no_open(backing_inode->i_rdev);
+	if (!(request_mask & (STATX_DIOALIGN | STATX_WRITE_ATOMIC)))
+		return;

This leaks the block device reference if blkdev_get_no_open() succeeds.

Quoted 5 lines
block/bdev.c +2 -0
+
+	bdev = blkdev_get_no_open(d_backing_inode(path->dentry)->i_rdev);
 	if (!bdev)
 		return;
 
author
#9 lore raw
Quoted 8 lines · Christian Brauner

block/bdev.c void sync_bdevs(bool wait) · line 1291

Quoted 3 lines
block/bdev.c +2 -1
-	bdev = blkdev_get_no_open(backing_inode->i_rdev);
+	if (!(request_mask & (STATX_DIOALIGN | STATX_WRITE_ATOMIC)))
+		return;

This leaks the block device reference if blkdev_get_no_open() succeeds.

block/bdev.c void sync_bdevs(bool wait) · line 1293

Quoted 2 lines
block/bdev.c +2 -0
+
+	bdev = blkdev_get_no_open(d_backing_inode(path->dentry)->i_rdev);

The one removed by the patch above, or the one added after the check below? :)

#10 lore raw
Quoted 7 lines · hch
Quoted 6 lines · Christian Brauner

block/bdev.c void sync_bdevs(bool wait) · line 1291

Quoted 3 lines
block/bdev.c +2 -1
-	bdev = blkdev_get_no_open(backing_inode->i_rdev);
+	if (!(request_mask & (STATX_DIOALIGN | STATX_WRITE_ATOMIC)))
+		return;

This leaks the block device reference if blkdev_get_no_open() succeeds.

Quoted 5 lines
Quoted 2 lines

block/bdev.c void sync_bdevs(bool wait) · line 1293

Quoted 2 lines
block/bdev.c +2 -0
+
+	bdev = blkdev_get_no_open(d_backing_inode(path->dentry)->i_rdev);

The one removed by the patch above, or the one added after the check below? :)

Ah, I got confused by the inline diff.

author
#11 lore raw
In reply to hch

Turns out this doesn't work. We used to have the request_mask, but it got removed in 25fbcd62d2e1 ("bdev: use bdev_io_min() for statx block size") so that stat can expose the block device min I/O size in st_blkdev, and as the blksize doesn't have it's own request_mask flag is hard to special case.

So maybe the better question is why devtmpfs even calls into vfs_getattr? As far as I can tell handle_remove is only ever called on the actual devtmpfs file system, so we don't need to go through the VFS to query i_mode. i.e. the patch should also fix the issue. The modify_change is probably not needed either, but for now I'm aiming for the minimal fix.

drivers/base/devtmpfs.c +8 -12
index 6dd1a8860f1c..53fb0829eb7b 100644
@@ -296,7 +296,7 @@ static int delete_path(const char *nodepath)
 	return err;
 }
 
-static int dev_mynode(struct device *dev, struct inode *inode, struct kstat *stat)
+static int dev_mynode(struct device *dev, struct inode *inode)
 {
 	/* did we create it */
 	if (inode->i_private != &thread)
@@ -304,13 +304,13 @@ static int dev_mynode(struct device *dev, struct inode *inode, struct kstat *sta
 
 	/* does the dev_t match */
 	if (is_blockdev(dev)) {
-		if (!S_ISBLK(stat->mode))
+		if (!S_ISBLK(inode->i_mode))
 			return 0;
 	} else {
-		if (!S_ISCHR(stat->mode))
+		if (!S_ISCHR(inode->i_mode))
 			return 0;
 	}
-	if (stat->rdev != dev->devt)
+	if (inode->i_rdev != dev->devt)
 		return 0;
 
 	/* ours */
@@ -321,8 +321,7 @@ static int handle_remove(const char *nodename, struct device *dev)
 {
 	struct path parent;
 	struct dentry *dentry;
-	struct kstat stat;
-	struct path p;
+	struct inode *inode;
 	int deleted = 0;
 	int err;
 
@@ -330,11 +329,8 @@ static int handle_remove(const char *nodename, struct device *dev)
 	if (IS_ERR(dentry))
 		return PTR_ERR(dentry);
 
-	p.mnt = parent.mnt;
-	p.dentry = dentry;
-	err = vfs_getattr(&p, &stat, STATX_TYPE | STATX_MODE,
-			  AT_STATX_SYNC_AS_STAT);
-	if (!err && dev_mynode(dev, d_inode(dentry), &stat)) {
+	inode = d_inode(dentry);
+	if (dev_mynode(dev, inode)) {
 		struct iattr newattrs;
 		/*
 		 * before unlinking this node, reset permissions
@@ -342,7 +338,7 @@ static int handle_remove(const char *nodename, struct device *dev)
 		 */
 		newattrs.ia_uid = GLOBAL_ROOT_UID;
 		newattrs.ia_gid = GLOBAL_ROOT_GID;
-		newattrs.ia_mode = stat.mode & ~0777;
+		newattrs.ia_mode = inode->i_mode & ~0777;
 		newattrs.ia_valid =
 			ATTR_UID|ATTR_GID|ATTR_MODE;
 		inode_lock(d_inode(dentry));
#12 lore raw
Quoted 10 lines · hch

Turns out this doesn't work. We used to have the request_mask, but it got removed in 25fbcd62d2e1 ("bdev: use bdev_io_min() for statx block size") so that stat can expose the block device min I/O size in st_blkdev, and as the blksize doesn't have it's own request_mask flag is hard to special case.

So maybe the better question is why devtmpfs even calls into vfs_getattr? As far as I can tell handle_remove is only ever called on the actual devtmpfs file system, so we don't need to go through the VFS to query i_mode. i.e. the patch should also fix the issue. The

Hm, yes. Just looked at the history and it dates back to Kay's original devtmpfs patch.

Quoted 8 lines

modify_change is probably not needed either, but for now I'm aiming for the minimal fix.

drivers/base/devtmpfs.c +0 -0
index 6dd1a8860f1c..53fb0829eb7b 100644

drivers/base/devtmpfs.c static int delete_path(const char *nodepath) · line 302

Quoted 8 lines
drivers/base/devtmpfs.c +1 -1
@@ -296,7 +296,7 @@ static int delete_path(const char *nodepath)
 	return err;
 }
 
-static int dev_mynode(struct device *dev, struct inode *inode, struct kstat *stat)
+static int dev_mynode(struct device *dev, struct inode *inode)
 {
 	/* did we create it */
 	if (inode->i_private != &thread)

It seems off that there are deletion requests coming in for a files that weren't created by devtmpfsd. But maybe that can somehow happen.

Quoted 50 lines
drivers/base/devtmpfs.c +7 -11
@@ -304,13 +304,13 @@ static int dev_mynode(struct device *dev, struct inode *inode, struct kstat *sta
 
 	/* does the dev_t match */
 	if (is_blockdev(dev)) {
-		if (!S_ISBLK(stat->mode))
+		if (!S_ISBLK(inode->i_mode))
 			return 0;
 	} else {
-		if (!S_ISCHR(stat->mode))
+		if (!S_ISCHR(inode->i_mode))
 			return 0;
 	}
-	if (stat->rdev != dev->devt)
+	if (inode->i_rdev != dev->devt)
 		return 0;
 
 	/* ours */
@@ -321,8 +321,7 @@ static int handle_remove(const char *nodename, struct device *dev)
 {
 	struct path parent;
 	struct dentry *dentry;
-	struct kstat stat;
-	struct path p;
+	struct inode *inode;
 	int deleted = 0;
 	int err;
 
@@ -330,11 +329,8 @@ static int handle_remove(const char *nodename, struct device *dev)
 	if (IS_ERR(dentry))
 		return PTR_ERR(dentry);
 
-	p.mnt = parent.mnt;
-	p.dentry = dentry;
-	err = vfs_getattr(&p, &stat, STATX_TYPE | STATX_MODE,
-			  AT_STATX_SYNC_AS_STAT);
-	if (!err && dev_mynode(dev, d_inode(dentry), &stat)) {
+	inode = d_inode(dentry);
+	if (dev_mynode(dev, inode)) {
 		struct iattr newattrs;
 		/*
 		 * before unlinking this node, reset permissions
@@ -342,7 +338,7 @@ static int handle_remove(const char *nodename, struct device *dev)
 		 */
 		newattrs.ia_uid = GLOBAL_ROOT_UID;
 		newattrs.ia_gid = GLOBAL_ROOT_GID;
-		newattrs.ia_mode = stat.mode & ~0777;
+		newattrs.ia_mode = inode->i_mode & ~0777;
 		newattrs.ia_valid =
 			ATTR_UID|ATTR_GID|ATTR_MODE;
 		inode_lock(d_inode(dentry));
#13 lore raw
In reply to hch
Quoted 68 lines · hch

Turns out this doesn't work. We used to have the request_mask, but it got removed in 25fbcd62d2e1 ("bdev: use bdev_io_min() for statx block size") so that stat can expose the block device min I/O size in st_blkdev, and as the blksize doesn't have it's own request_mask flag is hard to special case.

So maybe the better question is why devtmpfs even calls into vfs_getattr? As far as I can tell handle_remove is only ever called on the actual devtmpfs file system, so we don't need to go through the VFS to query i_mode. i.e. the patch should also fix the issue. The modify_change is probably not needed either, but for now I'm aiming for the minimal fix.

drivers/base/devtmpfs.c +7 -11
index 6dd1a8860f1c..53fb0829eb7b 100644
@@ -296,7 +296,7 @@ static int delete_path(const char *nodepath)
 	return err;
 }
 
-static int dev_mynode(struct device *dev, struct inode *inode, struct kstat *stat)
+static int dev_mynode(struct device *dev, struct inode *inode)
 {
 	/* did we create it */
 	if (inode->i_private != &thread)
@@ -304,13 +304,13 @@ static int dev_mynode(struct device *dev, struct inode *inode, struct kstat *sta
 
 	/* does the dev_t match */
 	if (is_blockdev(dev)) {
-		if (!S_ISBLK(stat->mode))
+		if (!S_ISBLK(inode->i_mode))
 			return 0;
 	} else {
-		if (!S_ISCHR(stat->mode))
+		if (!S_ISCHR(inode->i_mode))
 			return 0;
 	}
-	if (stat->rdev != dev->devt)
+	if (inode->i_rdev != dev->devt)
 		return 0;
 
 	/* ours */
@@ -321,8 +321,7 @@ static int handle_remove(const char *nodename, struct device *dev)
 {
 	struct path parent;
 	struct dentry *dentry;
-	struct kstat stat;
-	struct path p;
+	struct inode *inode;
 	int deleted = 0;
 	int err;
 
@@ -330,11 +329,8 @@ static int handle_remove(const char *nodename, struct device *dev)
 	if (IS_ERR(dentry))
 		return PTR_ERR(dentry);
 
-	p.mnt = parent.mnt;
-	p.dentry = dentry;
-	err = vfs_getattr(&p, &stat, STATX_TYPE | STATX_MODE,
-			  AT_STATX_SYNC_AS_STAT);
-	if (!err && dev_mynode(dev, d_inode(dentry), &stat)) {
+	inode = d_inode(dentry);
+	if (dev_mynode(dev, inode)) {
 		struct iattr newattrs;
 		/*
 		 * before unlinking this node, reset permissions

drivers/base/devtmpfs.c static int handle_remove(const char *nodename, struct device *dev) · line 344

Quoted 8 lines
drivers/base/devtmpfs.c +1 -1
@@ -342,7 +338,7 @@ static int handle_remove(const char *nodename, struct device *dev)
 		 */
 		newattrs.ia_uid = GLOBAL_ROOT_UID;
 		newattrs.ia_gid = GLOBAL_ROOT_GID;
-		newattrs.ia_mode = stat.mode & ~0777;
+		newattrs.ia_mode = inode->i_mode & ~0777;
 		newattrs.ia_valid =
 			ATTR_UID|ATTR_GID|ATTR_MODE;
 		inode_lock(d_inode(dentry));

I evaluated this new fix also, and confirmed it avoids the hang. Good. I also ran whole blktests and observed no other regression. Thanks.

Tested-by Ayush Jain
#14 lore raw
In reply to hch
Quoted 68 lines · hch

Turns out this doesn't work. We used to have the request_mask, but it got removed in 25fbcd62d2e1 ("bdev: use bdev_io_min() for statx block size") so that stat can expose the block device min I/O size in st_blkdev, and as the blksize doesn't have it's own request_mask flag is hard to special case.

So maybe the better question is why devtmpfs even calls into vfs_getattr? As far as I can tell handle_remove is only ever called on the actual devtmpfs file system, so we don't need to go through the VFS to query i_mode. i.e. the patch should also fix the issue. The modify_change is probably not needed either, but for now I'm aiming for the minimal fix.

drivers/base/devtmpfs.c +7 -11
index 6dd1a8860f1c..53fb0829eb7b 100644
@@ -296,7 +296,7 @@ static int delete_path(const char *nodepath)
 	return err;
 }
 
-static int dev_mynode(struct device *dev, struct inode *inode, struct kstat *stat)
+static int dev_mynode(struct device *dev, struct inode *inode)
 {
 	/* did we create it */
 	if (inode->i_private != &thread)
@@ -304,13 +304,13 @@ static int dev_mynode(struct device *dev, struct inode *inode, struct kstat *sta
 
 	/* does the dev_t match */
 	if (is_blockdev(dev)) {
-		if (!S_ISBLK(stat->mode))
+		if (!S_ISBLK(inode->i_mode))
 			return 0;
 	} else {
-		if (!S_ISCHR(stat->mode))
+		if (!S_ISCHR(inode->i_mode))
 			return 0;
 	}
-	if (stat->rdev != dev->devt)
+	if (inode->i_rdev != dev->devt)
 		return 0;
 
 	/* ours */
@@ -321,8 +321,7 @@ static int handle_remove(const char *nodename, struct device *dev)
 {
 	struct path parent;
 	struct dentry *dentry;
-	struct kstat stat;
-	struct path p;
+	struct inode *inode;
 	int deleted = 0;
 	int err;
 
@@ -330,11 +329,8 @@ static int handle_remove(const char *nodename, struct device *dev)
 	if (IS_ERR(dentry))
 		return PTR_ERR(dentry);
 
-	p.mnt = parent.mnt;
-	p.dentry = dentry;
-	err = vfs_getattr(&p, &stat, STATX_TYPE | STATX_MODE,
-			  AT_STATX_SYNC_AS_STAT);
-	if (!err && dev_mynode(dev, d_inode(dentry), &stat)) {
+	inode = d_inode(dentry);
+	if (dev_mynode(dev, inode)) {
 		struct iattr newattrs;
 		/*
 		 * before unlinking this node, reset permissions

drivers/base/devtmpfs.c static int handle_remove(const char *nodename, struct device *dev) · line 344

Quoted 8 lines
drivers/base/devtmpfs.c +1 -1
@@ -342,7 +338,7 @@ static int handle_remove(const char *nodename, struct device *dev)
 		 */
 		newattrs.ia_uid = GLOBAL_ROOT_UID;
 		newattrs.ia_gid = GLOBAL_ROOT_GID;
-		newattrs.ia_mode = stat.mode & ~0777;
+		newattrs.ia_mode = inode->i_mode & ~0777;
 		newattrs.ia_valid =
 			ATTR_UID|ATTR_GID|ATTR_MODE;
 		inode_lock(d_inode(dentry));

Thank you for the quick fix, this fixes blk devices hang issues on AMD EPYC systems as well.

Thanks,
Ayush

#15 lore raw
In reply to hch
Quoted 39 lines · hch
Quoted 9 lines · Shinichiro Kawasaki

I ran blktests with the kernel v6.15-rc3, and found the test case md/001 hangs. The hang is recreated in stable manner. I bisected and found this patch as the commit 777d0961ff95 is the trigger. When I revert the commit from v6.15-rc3 kernel, the hang disappeared.

Actions for fix will be appreciated.

FYI, the kernel INFO messages recorded functions relevant to the trigger commit, such as bdev_statx or vfs_getattr_nosec [1].

This should fix it:

block/bdev.c +8 -7
index 6a34179192c9..97d4c0ab1670 100644
@@ -1274,18 +1274,23 @@ void sync_bdevs(bool wait)
  */
 void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask)
 {
-	struct inode *backing_inode;
 	struct block_device *bdev;
 
-	backing_inode = d_backing_inode(path->dentry);
-
 	/*
-	 * Note that backing_inode is the inode of a block device node file,
-	 * not the block device's internal inode.  Therefore it is *not* valid
-	 * to use I_BDEV() here; the block device has to be looked up by i_rdev
-	 * instead.
+	 * Note that d_backing_inode() returnsthe inode of a block device node
+	 * file, not the block device's internal inode.
+	 *
+	 * Therefore it is *not* valid to use I_BDEV() here; the block device
+	 * has to be looked up by i_rdev instead.
+	 *
+	 * Only do this lookup if actually needed to avoid the performance
+	 * overhead of the lookup, and to avoid injecting bdev lifetime issues

block/bdev.c void sync_bdevs(bool wait) · line 1296

Quoted 10 lines
block/bdev.c +5 -1
+	 * into devtmpfs.
 	 */
-	bdev = blkdev_get_no_open(backing_inode->i_rdev);
+	if (!(request_mask & (STATX_DIOALIGN | STATX_WRITE_ATOMIC)))
+		return;
+
+	bdev = blkdev_get_no_open(d_backing_inode(path->dentry)->i_rdev);
 	if (!bdev)
 		return;
 

Thank you for the quick action. I applied this patch on top of the kernel v6.15-rc3 and confirmed the hang disappears. Good. I also ran whole blktests and observed no other regression. When you post the fix as a formal patch, feel free to add,

Tested-by
#16 lore raw
In reply to hch
Quoted 6 lines · hch
Quoted 5 lines · Shinichiro Kawasaki

I ran blktests with the kernel v6.15-rc3, and found the test case md/001 hangs. The hang is recreated in stable manner. I bisected and found this patch as the commit 777d0961ff95 is the trigger. When I revert the commit from v6.15-rc3 kernel, the hang disappeared.

Quoted 6 lines
Quoted 4 lines

Actions for fix will be appreciated.

FYI, the kernel INFO messages recorded functions relevant to the trigger commit, such as bdev_statx or vfs_getattr_nosec [1].

This should fix it:

FWIW, I was also about to report the same problem. Any reboot with dm targets hangs. Your patch fixes it also for me.

A message that is not in the archive (sent off-list or not yet received): [email protected]