nvmet: Kernel oops during nvmetcli 'make test'
Subsystems (2)
Latest nvmetcli fd09200bba73011b4b7086c96e394ce875fdd38f running against 5.5 generic kernel with one commit added.
a099bf88824599a9104c69001e5f3a914fd5c374 nvme-tcp: fix possible crash in write_zeroes processing
BUG: kernel NULL pointer dereference, address: 0000000000000030 #PF: supervisor read access in kernel mode #PF: error_code(0x0000) - not-present page PGD 0 P4D 0 Oops: 0000 [#1] SMP PTI CPU: 0 PID: 20989 Comm: nose2 Not tainted 5.5.0tony+ #59 Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006 RIP: 0010:nvmet_referral_release+0x14/0x30 [nvmet] Code: ff ff e8 df 7d c8 d1 49 c7 c4 f4 ff ff ff e9 4f ff ff ff 0f 1f 00 0f 1f 44 00 00 55 48 8b 47 30 48 8d af f0 fb ff ff 48 89 ee <48> 8b 78 30 48 81 ef 10 04 00 00 e8 7c 39 00 00 48 89 ef 5d e9 b3 RSP: 0018:ffffc0dd002c3e18 EFLAGS: 00010282 RAX: 0000000000000000 RBX: 00000000ffffffff RCX: 0000000000000000 RDX: ffff9ce2c7e140c0 RSI: ffff9ce2c56e3800 RDI: ffff9ce2c56e3c10 RBP: ffff9ce2c56e3800 R08: ffff9ce2c56e0c30 R09: 0000000000000000 R10: 0000000000000200 R11: 0000000000000000 R12: ffffffffc0442620 R13: 0000000000000000 R14: 0000000000000000 R15: ffff9ce2c56e3c10 FS: 00007f1392d0c6c0(0000) GS:ffff9ce31bc00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 0000000000000030 CR3: 00000000c7584002 CR4: 00000000000606f0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: config_item_put+0x85/0xd0 configfs_rmdir+0x1eb/0x2e0 vfs_rmdir+0x7c/0x170 do_rmdir+0x140/0x1a0 do_syscall_64+0x55/0x1a0 entry_SYSCALL_64_after_hwframe+0x44/0xa9 RIP: 0033:0x7f13932c119b Code: 73 01 c3 48 8b 0d ed dc 0c 00 f7 d8 64 89 01 48 83 c8 ff c3 66 2e
0f 1f 84 00 00 00 00 00 90 f3 0f 1e fa b8 54 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d bd dc 0c 00 f7 d8 64 89 01 48
RSP: 002b:00007ffc1401aad8 EFLAGS: 00000246 ORIG_RAX: 0000000000000054
RAX: ffffffffffffffda RBX: 00000000ffffff9c RCX: 00007f13932c119b
RDX: 0000000000000000 RSI: 0000000000000000 RDI: 00007f13919adeb0
RBP: 00007f139302fee0 R08: 0000000000000000 R09: 00007f139318eff0
R10: 00007f1393093753 R11: 0000000000000246 R12: 00005582b43e71d0
R13: 00007f1393165f6c R14: 00007f139299e230 R15: 00007f13919a1890
Modules linked in: nvme_loop nvme_fabrics nvme_core nvmet_tcp nvmet
fuse sunrpc intel_rapl_msr intel_rapl_common iosf_mbi intel_powerclamp crct10dif_pclmul crc32_pclmul dm_thin_pool dm_persistent_data ghash_clmulni_intel dm_bio_prison joydev e1000 intel_rapl_perf pcspkr vboxguest i2c_piix4 ip_tables xfs libcrc32c crc32c_intel serio_raw ata_generic video pata_acpi
CR2: 0000000000000030
---[ end trace a6ec6a3c629ca1cd ]---
(gdb) l *(nvmet_referral_release+0x14)0x32e4 is in nvmet_referral_release (drivers/nvme/target/configfs.c:978).
973 static void nvmet_referral_release(struct config_item *item) 974 {
975 struct nvmet_port *parent =to_nvmet_port(item->ci_parent->ci_parent);
976 struct nvmet_port *port = to_nvmet_port(item);
977
978 nvmet_referral_disable(parent, port);
979 kfree(port);
980 }
Thanks!
-Tony
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Sagi Grimberg
maintainerQuoted 8 lines
(gdb) l *(nvmet_referral_release+0x14)
0x32e4is innvmet_referral_release(drivers/nvme/target/configfs.c:978).973 static void
nvmet_referral_release(struct config_item *item)974 {
975 structnvmet_port*parent =to_nvmet_port(item->ci_parent->ci_parent);
This looks like a typo, I think we need:
--
@@ -1064,7 +1064,7 @@ static ssize_t nvmet_referral_enable_show(struct config_item *item,
static ssize_t nvmet_referral_enable_store(struct config_item *item,
const char *page, size_t count)
{
- struct nvmet_port *parent = to_nvmet_port(item->ci_parent->ci_parent);
+ struct nvmet_port *parent = to_nvmet_port(item->ci_parent);
struct nvmet_port *port = to_nvmet_port(item);
bool enable;--
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Quoted 15 lines
(gdb) l *(nvmet_referral_release+0x14)
0x32e4is innvmet_referral_release(drivers/nvme/target/configfs.c:978).973 static void
nvmet_referral_release(struct config_item *item)974 {
975 structnvmet_port*parent =to_nvmet_port(item->ci_parent->ci_parent);
976 structnvmet_port*port =to_nvmet_port(item);
977
978nvmet_referral_disable(parent, port);
979 kfree(port);
980 }
Quoted 3 lines
Thanks!
-Tony
Seems like referral release has a problem or the testcase related to it which should be test_referral() 353 or 356, do you see any other errors related to the configuration in the dmesg ?
On the other note logging for these testcases could be much better. (totally unrelated to this)
Quoted 1 line
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Quoted 15 lines · Chaitanya Kulkarni
Quoted 15 lines
(gdb) l *(nvmet_referral_release+0x14)
0x32e4is innvmet_referral_release(drivers/nvme/target/configfs.c:978).973 static void
nvmet_referral_release(struct config_item *item)974 {
975 structnvmet_port*parent =to_nvmet_port(item->ci_parent->ci_parent);
976 structnvmet_port*port =to_nvmet_port(item);
977
978nvmet_referral_disable(parent, port);
979 kfree(port);
980 }
Quoted 7 lines
Quoted 3 lines
Thanks!
-TonySeems like referral release has a problem or the testcase related to it which should be
test_referral()353 or 356, do you see any other errors related to the configuration in the dmesg ?
Well we get other messages and some errors, but not sure it's helpful?
nvmet: can't create discovery subsystem through configfs
nvmet: transport type 0 not supported
nvmet: Cannot modify address while enabled
nvmet: Disable the address before modifying
nvmet: Cannot modify address while enabled
nvmet: Disable the address before modifying
nvmet: Cannot modify address while enabled
nvmet: Disable the address before modifying
nvmet: Cannot modify address while enabled
nvmet: Disable the address before modifying
nvmet: Cannot modify address while enabled
nvmet: Disable the address before modifying
nvmet: Cannot modify address while enabled
nvmet: Disable the address before modifying
nvmet: Cannot modify address while enabled
nvmet: Disable the address before modifying
nvmet: Cannot modify address while enabled
nvmet: Disable the address before modifying
nvmet: Cannot modify address while enabled
nvmet: Disable the address before modifying
nvmet: Cannot modify address while enabled
nvmet: Disable the address before modifying
nvmet: adding nsid 3 to subsystem testnqn
nvmet: adding nsid 2 to subsystem testnqn
nvmet: adding nsid 1 to subsystem testnqn
nvmet: adding nsid 4 to subsystem testnqn
nvmet: adding nsid 1 to subsystem testnqn
nvmet: failed to open block device (null): (-22)
nvmet: failed to open block device (null): (-22)
nvmet: failed to open block device /dev/ram0: (-2)
nvmet: failed to open block device /dev/ram0: (-2)
nvmet: adding nsid 1 to subsystem testnqn
nvmet: adding nsid 2 to subsystem testnqn
BUG: kernel NULL pointer dereference, address: 0000000000000030
...
Quoted 2 lines
On the other note logging for these testcases could be much better. (totally unrelated to this)
Or we could break the unit tests into much smaller units so that when you run with '-v' you could identify which specific part of the existing unit test is causing crash instead of just knowing 'test_referral'?
Regardless, I added some debug. We are hitting the issue on line 353.
-Tony
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Quoted 5 lines · Tony Asleson
...
Quoted 2 lines
Quoted 2 lines
On the other note logging for these testcases could be much better. (totally unrelated to this)
Quoted 9 lines
Or we could break the unit tests into much smaller units so that when you run with '-v' you could identify which specific part of the existing unit test is causing crash instead of just knowing '
test_referral'?Regardless, I added some debug. We are hitting the issue on line 353.
-Tony
I don't have your tree, this patch is based on the nvme-5.7 branch. I'm suspecting cleanup issue with some configuration which needs to be sorted out. Can you port this patch on your tree so we will at least know parent pointers and port are valid ? then we can backtrace into testcase.
drivers/nvme/target/configfs.c +0 -0
@@ -1100,9 +1100,27 @@ static struct configfs_attribute
*nvmet_referral_attrs[] = {
static void nvmet_referral_release(struct config_item *item) {
- struct nvmet_port *parent =to_nvmet_port(item->ci_parent->ci_parent);
+ struct nvmet_port *parent;
struct nvmet_port *port = to_nvmet_port(item);+ if (port == NULL) {
+ pr_info("%s %d port = NULL\n", __func__, __LINE__);
+ return;
+ }
+
+ if (item->ci_parent == NULL) {
+ pr_info("%s %d ci_parent = NULL \n", __func__, __LINE__);
+ return;
+ }
+
+ if (item->ci_parent->ci_parent == NULL) {
+ pr_info("%s %d ci_parent->ci_parent = NULL \n",
+ __func__, __LINE__);
+ return;
+ }
+
+ parent = to_nvmet_port(item->ci_parent->ci_parent);
+
nvmet_referral_disable(parent, port);
kfree(port);}
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme