Skip to contentSkip to conversation

fix scsi process problems and clean up the target reap issues

PATCH In review James Bottomley 2 participants 5 messages Feb 23, 2006 – Feb 28, 2006 SCSI

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 fix scsi process problems and clean up the target reap issues

In order to use the new execute_in_process_context() API, you have to provide it with the work storage, which I do in SCSI in scsi_device and scsi_target, but which also means that we can no longer queue up the target reaps, so instead I moved the target to a state model which allows target_alloc to detect if we've received a dying target and wait for it to be gone. Hopefully, this should also solve the target namespace race.

James

---

drivers/scsi/scsi_lib.c +0 -58
Index: BUILD-2.6/drivers/scsi/scsi_lib.c
===================================================================
@@ -2248,61 +2248,3 @@
 		device_for_each_child(dev, NULL, target_unblock);
 }
 EXPORT_SYMBOL_GPL(scsi_target_unblock);
-
-
-struct work_queue_work {
-	struct work_struct	work;
-	void			(*fn)(void *);
-	void			*data;
-};
-
-static void execute_in_process_context_work(void *data)
-{
-	void (*fn)(void *data);
-	struct work_queue_work *wqw = data;
-
-	fn = wqw->fn;
-	data = wqw->data;
-
-	kfree(wqw);
-
-	fn(data);
-}
-
-/**
- * scsi_execute_in_process_context - reliably execute the routine with user context
- * @fn:		the function to execute
- * @data:	data to pass to the function
- *
- * Executes the function immediately if process context is available,
- * otherwise schedules the function for delayed execution.
- *
- * Returns:	0 - function was executed
- *		1 - function was scheduled for execution
- *		<0 - error
- */
-int scsi_execute_in_process_context(void (*fn)(void *data), void *data)
-{
-	struct work_queue_work *wqw;
-
-	if (!in_interrupt()) {
-		fn(data);
-		return 0;
-	}
-
-	wqw = kmalloc(sizeof(struct work_queue_work), GFP_ATOMIC);
-
-	if (unlikely(!wqw)) {
-		printk(KERN_ERR "Failed to allocate memory\n");
-		WARN_ON(1);
-		return -ENOMEM;
-	}
-
-	INIT_WORK(&wqw->work, execute_in_process_context_work, wqw);
-	wqw->fn = fn;
-	wqw->data = data;
-	schedule_work(&wqw->work);
-
-	return 1;
-}
-EXPORT_SYMBOL_GPL(scsi_execute_in_process_context);
drivers/scsi/scsi_scan.c +33 -16

75 lines not on this page · raw message

drivers/scsi/scsi_sysfs.c +3 -1
Index: BUILD-2.6/drivers/scsi/scsi_sysfs.c
===================================================================
@@ -256,7 +256,9 @@
 
 static void scsi_device_dev_release(struct device *dev)
 {
-	scsi_execute_in_process_context(scsi_device_dev_release_usercontext,	dev);
+	struct scsi_device *sdp = to_scsi_device(dev);
+	execute_in_process_context(scsi_device_dev_release_usercontext, dev,
+				   &sdp->ew);
 }
 
 static struct class sdev_class = {
include/scsi/scsi.h +0 -2
Index: BUILD-2.6/include/scsi/scsi.h
===================================================================
@@ -433,6 +433,4 @@
 /* Used to obtain the PCI location of a device */
 #define SCSI_IOCTL_GET_PCI		0x5387
 
-int scsi_execute_in_process_context(void (*fn)(void *data), void *data);
-
 #endif /* _SCSI_SCSI_H */
include/scsi/scsi_device.h +10 -0
Index: BUILD-2.6/include/scsi/scsi_device.h
===================================================================
@@ -4,6 +4,7 @@
 #include <linux/device.h>
 #include <linux/list.h>
 #include <linux/spinlock.h>
+#include <linux/workqueue.h>
 #include <asm/atomic.h>
 
 struct request_queue;
@@ -137,6 +138,8 @@
 	struct device		sdev_gendev;
 	struct class_device	sdev_classdev;
 
+	struct execute_work	ew; /* used to get process context on put */
+
 	enum scsi_device_state sdev_state;
 	unsigned long		sdev_data[0];
 } __attribute__((aligned(sizeof(unsigned long))));
@@ -153,6 +156,11 @@
 #define scmd_printk(prefix, scmd, fmt, a...)	\
 	dev_printk(prefix, &(scmd)->device->sdev_gendev, fmt, ##a)
 
+enum scsi_target_state {
+	STARGET_RUNNING = 1,
+	STARGET_DEL,
+};
+
 /*
  * scsi_target: representation of a scsi target, for now, this is only
  * used for single_lun devices. If no one has active IO to the target,
@@ -169,6 +177,8 @@
 				     * scsi_device.id eventually */
 	unsigned long		create:1; /* signal that it needs to be added */
 	char			scsi_level;
+	struct execute_work	ew;
+	enum scsi_target_state	state;
 	void 			*hostdata; /* available to low-level driver */
 	unsigned long		starget_data[0]; /* for the transport */
 	/* starget_data must be the last element!!!! */
#1 lore raw
Quoted 18 lines · James Bottomley

static void scsi_target_reap_usercontext(void *data)

drivers/scsi/scsi_scan.c +3 -9
@@ -391,21 +400,13 @@
 	struct Scsi_Host *shost = dev_to_shost(starget->dev.parent);
 	unsigned long flags;
 
+	transport_remove_device(&starget->dev);
+	device_del(&starget->dev);
+	transport_destroy_device(&starget->dev);
 	spin_lock_irqsave(shost->host_lock, flags);
-
-	if (--starget->reap_ref == 0 && list_empty(&starget->devices)) {
-		list_del_init(&starget->siblings);
-		spin_unlock_irqrestore(shost->host_lock, flags);
-		transport_remove_device(&starget->dev);
-		device_del(&starget->dev);
-		transport_destroy_device(&starget->dev);
-		put_device(&starget->dev);
-		return;

drivers/scsi/scsi_scan.c line 412

Quoted 10 lines
drivers/scsi/scsi_scan.c +2 -4
-
-	}
+	list_del_init(&starget->siblings);
 	spin_unlock_irqrestore(shost->host_lock, flags);
-
-	return;
+	put_device(&starget->dev);
 }
 
 /**

The patch was tried on the aic7xxx ahc_linux_target_alloc issue I previously mentioned and it did not change the result of a BUG_ON.

Is there some reason we cannot do the list_del_init(&starget->siblings) in scsi_target_dev_release post calling target_destroy? It would appear with the check for STARGET_DEL that being on the list longer should not be a problem.

For the sdev we call the slave_destroy in __scsi_remove_device and then pull it off the list in the release.

-andmike
--
Michael Anderson
[email protected]

#2 lore raw
Quoted 2 lines · Mike Anderson

The patch was tried on the aic7xxx ahc_linux_target_alloc issue I previously mentioned and it did not change the result of a BUG_ON.

Hmm ... could you repost the panic ... I also put in Brian King's check for device_add failure, which should have picked this up ... it sounds like there's something else going on.

Quoted 4 lines

Is there some reason we cannot do the list_del_init(&starget->siblings) in scsi_target_dev_release post calling target_destroy? It would appear with the check for STARGET_DEL that being on the list longer should not be a problem.

No, it shouldn't ... it just potentially delays the allocation to wait for everything to finish using the old target ... we might have to put a reschedule in the retry loop in alloc to avoid a busy wait.

Does that fix your aic problem?

James

#3 lore raw
Quoted 7 lines · James Bottomley
Quoted 2 lines · Mike Anderson

The patch was tried on the aic7xxx ahc_linux_target_alloc issue I previously mentioned and it did not change the result of a BUG_ON.

Hmm ... could you repost the panic ... I also put in Brian King's check for device_add failure, which should have picked this up ... it sounds like there's something else going on.

Posted below. This problem is showing up when a sequence of a delete followed by a scan is executed using the sysfs interface. The device at SCSI Id 1 is a IBM 3580Gen3 LTO Tape device.

Quoted 11 lines
Quoted 5 lines

Is there some reason we cannot do the list_del_init(&starget->siblings) in scsi_target_dev_release post calling target_destroy? It would appear with the check for STARGET_DEL that being on the list longer should not be a problem.

No, it shouldn't ... it just potentially delays the allocation to wait for everything to finish using the old target ... we might have to put a reschedule in the retry loop in alloc to avoid a busy wait.

Does that fix your aic problem?

I believe being on the list longer would solve the aic problem, but I have not tried to a patch to do this. I am trying to get access to the system now.

-andmike
--
Michael Anderson
[email protected]

Feb 27 13:16:22 system1 kernel:   Vendor: IBM       Model: ULTRIUM-TD3     
 Rev: 59D2
Feb 27 13:16:22 system1 kernel:   Type:   Sequential-Access                
 ANSI SCSI revision: 03
Feb 27 13:16:22 system1 kernel:  target0:0:1: Beginning Domain
Validation
Feb 27 13:16:22 system1 kernel:  target0:0:1: wide asynchronous
Feb 27 13:16:23 system1 kernel:  target0:0:1: FAST-20 WIDE SCSI 40.0
MB/s ST
(50 ns, offset 127)
Feb 27 13:16:23 system1 kernel:  target0:0:1: Domain Validation
skipping
write tests
Feb 27 13:16:23 system1 kernel:  target0:0:1: Ending Domain Validation
Feb 27 13:16:23 system1 kernel: st: Version 20050830, fixed bufsize
32768,
s/g segs 256
Feb 27 13:16:23 system1 kernel: st 0:0:1:0: Attached scsi tape
st0<4>st0:
try direct i/o: yes (alignment 512 B)
Feb 27 13:16:23 system1 kernel: st 0:0:1:0: Attached scsi generic sg0
type 1
Feb 27 13:16:33 system1 kernel:   Vendor: IBM       Model: ULTRIUM-TD3     
 Rev: 59D2
Feb 27 13:16:33 system1 kernel:   Type:   Sequential-Access                
 ANSI SCSI revision: 03
Feb 27 13:16:33 system1 kernel:  target0:0:1: Beginning Domain
Validation
Feb 27 13:16:33 system1 kernel:  target0:0:1: wide asynchronous
Feb 27 13:16:33 system1 kernel:  target0:0:1: FAST-20 WIDE SCSI 40.0
MB/s ST
(50 ns, offset 127)
Feb 27 13:16:33 system1 kernel:  target0:0:1: Domain Validation
skipping
write tests
Feb 27 13:16:33 system1 kernel:  target0:0:1: Ending Domain Validation
Feb 27 13:16:33 system1 kernel: st 0:0:1:0: Attached scsi tape
st0<4>st0:
try direct i/o: yes (alignment 512 B)
Feb 27 13:16:33 system1 kernel: st 0:0:1:0: Attached scsi generic sg0
type 1
Feb 27 13:16:48 system1 kernel: ------------[ cut here ]------------
Feb 27 13:16:48 system1 kernel: kernel BUG at
drivers/scsi/aic7xxx/aic7xxx_osm.c:534!
Feb 27 13:16:48 system1 kernel: invalid opcode: 0000 [#1]
Feb 27 13:16:48 system1 kernel: last sysfs file:
/class/scsi_host/host0/scan
Feb 27 13:16:48 system1 kernel: Modules linked in: sg st ip6t_LOG

xt_tcpudp
xt_pkttype ipt_LOG xt_limit snd_pcm_oss snd_mixer_oss snd_seq af_packet edd
button battery ac ip6t_REJECT ipt_REJECT xt_state iptable_mangle iptable_nat
ip_nat iptable_filter ip6table_mangle ip_conntrack nfnetlink ip_tables ip6table_filter ip6_tables x_tables ipv6 loop dm_mod gl620a usbnet generic i2c_viapro i2c_core ns558 ide_cd cdrom parport_pc parport shpchp via_ircc uhci_hcd pci_hotplug irda via_agp usbcore snd_via82xx gameport crc_ccitt agpgart
snd_ac97_codec snd_ac97_bus snd_pcm snd_timer snd_page_alloc snd_mpu401_uart
snd_rawmidi snd_seq_device snd soundcore e100 mii reiserfs fan thermal processor
via82cxxx aic7xxx scsi_transport_spi sd_mod scsi_mod ide_disk ide_core

Feb 27 13:16:48 system1 kernel: CPU:    0
Feb 27 13:16:48 system1 kernel: EIP:    0060:[<d0e81e7d>]
Feb 27 13:16:48 system1 kernel: EFLAGS: 00010086  
(2.6.16-rc3-git3-2-default #1) 
Feb 27 13:16:48 system1 kernel: EIP is at
ahc_linux_target_alloc+0x123/0x244
[aic7xxx]
Feb 27 13:16:48 system1 kernel: eax: 00000282   ebx: 00000001   ecx:
c776d16c   edx: ffffffff
Feb 27 13:16:48 system1 kernel: esi: cf41fa6c   edi: 00000048   ebp:
cf6ab98c   esp: c724dddc
Feb 27 13:16:48 system1 kernel: ds: 007b   es: 007b   ss: 0068
Feb 27 13:16:48 system1 kernel: Process kill_rescan.sh (pid: 5278,
threadinfo=c724c000 task=c5952030)
Feb 27 13:16:48 system1 kernel: Stack: <0>c776d16c c776d338 cf7209d0
00000282 4160f64c 00000007 d082c299 cf60f5f8 
Feb 27 13:16:48 system1 kernel:        c776d170 c020d343 cf60f63c
cf60f64c
00000000 cf60e1bc c776d170 cf60e1c0 
Feb 27 13:16:48 system1 kernel:        c776d16c d0e26a4a 00000001
00000000
cf60e2cc c776d16c c776d25c cf60e1bc 
Feb 27 13:16:48 system1 kernel: Call Trace:
Feb 27 13:16:48 system1 kernel:  [<d082c299>] spi_host_match+0xd/0x54
[scsi_transport_spi]
Feb 27 13:16:48 system1 kernel:  [<c020d343>]
attribute_container_device_trigger+0x3a/0xa1
Feb 27 13:16:48 system1 kernel:  [<d0e26a4a>]
scsi_alloc_target+0x1b4/0x2ab
[scsi_mod]
Feb 27 13:16:48 system1 kernel:  [<d0e26bfa>]
__scsi_scan_target+0x4b/0x59b
[scsi_mod]
Feb 27 13:16:48 system1 kernel:  [<c01656ad>]
notify_change+0x2db/0x2e9
Feb 27 13:16:48 system1 kernel:  [<c01627f6>] __d_path+0x118/0x156
Feb 27 13:16:48 system1 kernel:  [<c01ad446>] vsscanf+0xd4/0x3ef
Feb 27 13:16:48 system1 kernel:  [<d0e2727d>]
scsi_scan_host_selected+0xc5/0xdc [scsi_mod]
Feb 27 13:16:48 system1 kernel:  [<d0e2797b>] store_scan+0x96/0xae
[scsi_mod]
Feb 27 13:16:48 system1 kernel:  [<d0e278e5>] store_scan+0x0/0xae
[scsi_mod]
Feb 27 13:16:48 system1 kernel:  [<c020bd64>]
class_device_attr_store+0x1b/0x1f
Feb 27 13:16:48 system1 kernel:  [<c0182a43>]
sysfs_write_file+0x9b/0xc1
Feb 27 13:16:48 system1 kernel:  [<c01829a8>]
sysfs_write_file+0x0/0xc1
Feb 27 13:16:48 system1 kernel:  [<c014fa1e>] vfs_write+0xa1/0x146
Feb 27 13:16:48 system1 kernel:  [<c014ff34>] sys_write+0x3c/0x63
Feb 27 13:16:48 system1 kernel:  [<c0102ab9>] syscall_call+0x7/0xb
Feb 27 13:16:48 system1 kernel: Code: c0 e9 34 ff ff ff 0f b6 8d 33 01

00 00
83 c3 08 89 4c 24 14 8b 85 a8 00 00 00 83 c0 40 e8 e7 f9 3f ef 89 44 24 0c 83 3e
00 74 08 <0f> 0b 16 02 dc a4 e8 d0 8b 04 24 b9 43 00 00 00 89 06 31 c0 03

Feb 27 13:16:48 system1 kernel:  <3>Debug: sleeping function called
from
invalid context at include/linux/rwsem.h:43
Feb 27 13:16:48 system1 kernel: in_atomic():0, irqs_disabled():1
Feb 27 13:16:48 system1 kernel:  [<c0119174>]
profile_task_exit+0x18/0x3e
Feb 27 13:16:48 system1 kernel:  [<c011a98b>] do_exit+0x1c/0x6b0
Feb 27 13:16:48 system1 kernel:  [<c0118922>] printk+0x14/0x18
Feb 27 13:16:48 system1 kernel:  [<c010442a>] show_stack+0x0/0xa
Feb 27 13:16:48 system1 kernel:  [<c0104979>] do_invalid_op+0x0/0x9d
Feb 27 13:16:48 system1 kernel:  [<c0104a0a>] do_invalid_op+0x91/0x9d
Feb 27 13:16:48 system1 kernel:  [<d0e81e7d>]
ahc_linux_target_alloc+0x123/0x244 [aic7xxx]
Feb 27 13:16:48 system1 kernel:  [<c0114a93>] __wake_up+0x2a/0x3d
Feb 27 13:16:48 system1 kernel:  [<c014c79e>]
cache_alloc_refill+0x1f8/0x527
Feb 27 13:16:48 system1 kernel:  [<c01ab8e7>]
kobject_uevent+0x36b/0x390
Feb 27 13:16:48 system1 kernel:  [<c014bfa9>]
cache_alloc_debugcheck_after+0xb8/0xea
Feb 27 13:16:48 system1 kernel:  [<c0103c3f>] error_code+0x4f/0x60
Feb 27 13:16:48 system1 kernel:  [<d0e81e7d>]
ahc_linux_target_alloc+0x123/0x244 [aic7xxx]
Feb 27 13:16:48 system1 kernel:  [<d082c299>] spi_host_match+0xd/0x54
[scsi_transport_spi]
Feb 27 13:16:48 system1 kernel:  [<c020d343>]
attribute_container_device_trigger+0x3a/0xa1
Feb 27 13:16:48 system1 kernel:  [<d0e26a4a>]
scsi_alloc_target+0x1b4/0x2ab
[scsi_mod]
Feb 27 13:16:48 system1 kernel:  [<d0e26bfa>]
__scsi_scan_target+0x4b/0x59b
[scsi_mod]
Feb 27 13:16:48 system1 kernel:  [<c01656ad>]
notify_change+0x2db/0x2e9
Feb 27 13:16:48 system1 kernel:  [<c01627f6>] __d_path+0x118/0x156
Feb 27 13:16:48 system1 kernel:  [<c01ad446>] vsscanf+0xd4/0x3ef
Feb 27 13:16:48 system1 kernel:  [<d0e2727d>]
scsi_scan_host_selected+0xc5/0xdc [scsi_mod]
Feb 27 13:16:48 system1 kernel:  [<d0e2797b>] store_scan+0x96/0xae
[scsi_mod]
Feb 27 13:16:48 system1 kernel:  [<d0e278e5>] store_scan+0x0/0xae
[scsi_mod]
Feb 27 13:16:48 system1 kernel:  [<c020bd64>]
class_device_attr_store+0x1b/0x1f
Feb 27 13:16:48 system1 kernel:  [<c0182a43>]
sysfs_write_file+0x9b/0xc1
Feb 27 13:16:48 system1 kernel:  [<c01829a8>]
sysfs_write_file+0x0/0xc1
Feb 27 13:16:48 system1 kernel:  [<c014fa1e>] vfs_write+0xa1/0x146
Feb 27 13:16:48 system1 kernel:  [<c014ff34>] sys_write+0x3c/0x63
Feb 27 13:16:48 system1 kernel:  [<c0102ab9>] syscall_call+0x7/0xb
#4 lore raw
Quoted 2 lines · Mike Anderson

I believe being on the list longer would solve the aic problem, but I have not tried to a patch to do this. I am trying to get access to the system now.

Actually, the problem is caused by a misordering of target_alloc, target_destroy.

The aic target_alloc routine has a BUG_ON() to detect that the target entry is null (it's nulled in target_destroy). So, this could be fixed by moving the target_destroy callback, as well. I'll take a look at it and see what springs to mind. I'm leery of putting the list deletion in the release, because that could delay new target creation arbitrarily long ...

James