Skip to contentSkip to conversation

block subsystem related crash with latest -git

bug Ingo Molnar 13 participants 151 messages Oct 17, 2007 – Oct 20, 2007 1 message not in the archive Page 3 of 3

Subsystems (48)
lore
Share & tools
lore
#102 lore raw

Re: [bug] ata subsystem related crash with latest -git

In reply to Jens Axboe
Quoted 16 lines · Jens Axboe

index 4df8311..b858183 100644

drivers/ata/sata_mv.c +1 -1
@@ -1139,6 +1139,7 @@ static void mv_fill_sg(struct ata_queued_cmd *qc)
 	struct mv_port_priv *pp = qc->ap->private_data;
 	struct scatterlist *sg;
 	struct mv_sg *mv_sg;
+	int end_marked = 0;
 
 	mv_sg = pp->sg_tbl;
 	ata_for_each_sg(sg, qc) {
@@ -1159,13 +1160,15 @@ static void mv_fill_sg(struct ata_queued_cmd *qc)
 			sg_len -= len;
 			addr += len;
 
-			if (!sg_len && ata_sg_is_last(sg, qc))

drivers/ata/sata_mv.c static void mv_fill_sg(struct ata_queued_cmd *qc) · line 1171

Quoted 10 lines
drivers/ata/sata_mv.c +4 -1
+			if (!sg_len && ata_sg_is_last(sg, qc)) {
 				mv_sg->flags_size |= cpu_to_le32(EPRD_FLAG_END_OF_TBL);
+				end_marked++;
+			}
 
 			mv_sg++;
 		}
-
 	}
+	BUG_ON(end_marked != 1);

Your BUG_ON() does indeed trip, here.

Its surprising that other folks don't explode, considering that mv_fill_sg() intentionally mirrors the logic in ata_fill_sg().

Jeff

A message that is not in the archive (sent off-list or not yet received): [email protected]

#104 lore raw

Re: [bug] ata subsystem related crash with latest -git

Quoted 26 lines · Benny Halevy

On 10/18/07, *Jeff Garzik* <[email protected] <mailto:[email protected]>> wrote:

    Jens Axboe wrote:
     > index 4df8311..b858183 100644
     > --- a/drivers/ata/sata_mv.c
     > +++ b/drivers/ata/sata_mv.c
     > @@ -1139,6 +1139,7 @@ static void mv_fill_sg(struct
    ata_queued_cmd *qc)
     >       struct mv_port_priv *pp = qc->ap->private_data;
     >       struct scatterlist *sg;
     >       struct mv_sg *mv_sg;
     > +     int end_marked = 0;
     >
     >       mv_sg = pp->sg_tbl;
     >       ata_for_each_sg(sg, qc) {
     > @@ -1159,13 +1160,15 @@ static void mv_fill_sg(struct
    ata_queued_cmd *qc)
     >                       sg_len -= len;
     >                       addr += len;
     >
     > -                     if (!sg_len && ata_sg_is_last(sg, qc))
     > +                     if (!sg_len && ata_sg_is_last(sg, qc)) { 
Quoted 1 line

I'm not sure, but shouldn't that be || rather than &&?

sg_len is zero at the end of each scatterlist entry, so we need to test the additional ata_sg_is_last() condition to determine if we are really at the end of the PRD table.

Jeff

#105 lore raw

Re: [bug] ata subsystem related crash with latest -git

In reply to Jeff Garzik
Quoted 28 lines · Jeff Garzik
Quoted 13 lines · Jens Axboe

index 4df8311..b858183 100644

drivers/ata/sata_mv.c +1 -0
@@ -1139,6 +1139,7 @@ static void mv_fill_sg(struct ata_queued_cmd *qc)
 	struct mv_port_priv *pp = qc->ap->private_data;
 	struct scatterlist *sg;
 	struct mv_sg *mv_sg;
+	int end_marked = 0;
  	mv_sg = pp->sg_tbl;
 	ata_for_each_sg(sg, qc) {
@@ -1159,13 +1160,15 @@ static void mv_fill_sg(struct ata_queued_cmd *qc)
 			sg_len -= len;
 			addr += len;

drivers/ata/sata_mv.c static void mv_fill_sg(struct ata_queued_cmd *qc) · line 1170

Quoted 10 lines
drivers/ata/sata_mv.c +4 -1
 -			if (!sg_len && ata_sg_is_last(sg, qc))
+			if (!sg_len && ata_sg_is_last(sg, qc)) {
 				mv_sg->flags_size |= cpu_to_le32(EPRD_FLAG_END_OF_TBL);
+				end_marked++;
+			}
  			mv_sg++;
 		}
-
 	}
+	BUG_ON(end_marked != 1);

Your BUG_ON() does indeed trip, here.

Quoted 2 lines

Its surprising that other folks don't explode, considering that mv_fill_sg() intentionally mirrors the logic in ata_fill_sg().

The sata_mv construct looks a bit odd. Does this work? That last end_mv_sg test should always be true, just being paranoid...

drivers/ata/sata_mv.c +5 -6
index 4df8311..5397eea 100644
@@ -1138,8 +1138,9 @@ static void mv_fill_sg(struct ata_queued_cmd *qc)
 {
 	struct mv_port_priv *pp = qc->ap->private_data;
 	struct scatterlist *sg;
-	struct mv_sg *mv_sg;
+	struct mv_sg *mv_sg, *end_mv_sg;
 
+	end_mv_sg = NULL;
 	mv_sg = pp->sg_tbl;
 	ata_for_each_sg(sg, qc) {
 		dma_addr_t addr = sg_dma_address(sg);
@@ -1158,14 +1159,12 @@ static void mv_fill_sg(struct ata_queued_cmd *qc)
 
 			sg_len -= len;
 			addr += len;
-
-			if (!sg_len && ata_sg_is_last(sg, qc))
-				mv_sg->flags_size |= cpu_to_le32(EPRD_FLAG_END_OF_TBL);
-
+			end_mv_sg = mv_sg;
 			mv_sg++;
 		}
-
 	}
+	if (end_mv_sg)
+		end_mv_sg->flags_size |= cpu_to_le32(EPRD_FLAG_END_OF_TBL);
 }
 
 static inline void mv_crqb_pack_cmd(__le16 *cmdw, u8 data, u8 addr, unsigned last)
Signature · 1 line
Jens Axboe
#106 lore raw

Re: [bug] ata subsystem related crash with latest -git

Quoted 1 line · Jens Axboe

The sata_mv construct looks a bit odd. Does this work? That last

The sata_mv construct worked just fine before sg chaining :)

Quoted 24 lines

end_mv_sg test should always be true, just being paranoid...

drivers/ata/sata_mv.c +2 -4
index 4df8311..5397eea 100644
@@ -1138,8 +1138,9 @@ static void mv_fill_sg(struct ata_queued_cmd *qc)
 {
 	struct mv_port_priv *pp = qc->ap->private_data;
 	struct scatterlist *sg;
-	struct mv_sg *mv_sg;
+	struct mv_sg *mv_sg, *end_mv_sg;
 
+	end_mv_sg = NULL;
 	mv_sg = pp->sg_tbl;
 	ata_for_each_sg(sg, qc) {
 		dma_addr_t addr = sg_dma_address(sg);
@@ -1158,14 +1159,12 @@ static void mv_fill_sg(struct ata_queued_cmd *qc)
 
 			sg_len -= len;
 			addr += len;
-
-			if (!sg_len && ata_sg_is_last(sg, qc))
-				mv_sg->flags_size |= cpu_to_le32(EPRD_FLAG_END_OF_TBL);

drivers/ata/sata_mv.c static void mv_fill_sg(struct ata_queued_cmd *qc) · line 1169

Quoted 10 lines
drivers/ata/sata_mv.c +3 -2
-
+			end_mv_sg = mv_sg;
 			mv_sg++;
 		}
-
 	}
+	if (end_mv_sg)
+		end_mv_sg->flags_size |= cpu_to_le32(EPRD_FLAG_END_OF_TBL);
 }
 

I'm testing a similar patch based on ata_fill_sg()'s method, which basically does something similar to what you've done here (see attached). I had noticed that ata_fill_sg() did not call ata_sg_is_last().

If this fixes the problem, I think the best solution would be to delete ata_sg_is_last(). In the few users that exist, we should be able to eliminate the test programmatically as you and ata_fill_sg() have done -- thereby eliminating a branch per loop in a hotpath.

Off to test the attached... if that doesn't work I'll try your version, though there shouldn't be much difference.

Jeff

drivers/ata/sata_mv.c +18 -29
index 4df8311..42b5a9e 100644
@@ -421,7 +421,6 @@ static void mv_error_handler(struct ata_port *ap);
 static void mv_post_int_cmd(struct ata_queued_cmd *qc);
 static void mv_eh_freeze(struct ata_port *ap);
 static void mv_eh_thaw(struct ata_port *ap);
-static int mv_slave_config(struct scsi_device *sdev);
 static int mv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
 
 static void mv5_phy_errata(struct mv_host_priv *hpriv, void __iomem *mmio,
@@ -459,7 +458,7 @@ static struct scsi_host_template mv5_sht = {
 	.use_clustering		= 1,
 	.proc_name		= DRV_NAME,
 	.dma_boundary		= MV_DMA_BOUNDARY,
-	.slave_configure	= mv_slave_config,
+	.slave_configure	= ata_scsi_slave_config,
 	.slave_destroy		= ata_scsi_slave_destroy,
 	.bios_param		= ata_std_bios_param,
 };
@@ -477,7 +476,7 @@ static struct scsi_host_template mv6_sht = {
 	.use_clustering		= 1,
 	.proc_name		= DRV_NAME,
 	.dma_boundary		= MV_DMA_BOUNDARY,
-	.slave_configure	= mv_slave_config,
+	.slave_configure	= ata_scsi_slave_config,
 	.slave_destroy		= ata_scsi_slave_destroy,
 	.bios_param		= ata_std_bios_param,
 };
@@ -756,17 +755,6 @@ static void mv_irq_clear(struct ata_port *ap)
 {
 }
 
-static int mv_slave_config(struct scsi_device *sdev)
-{
-	int rc = ata_scsi_slave_config(sdev);
-	if (rc)
-		return rc;
-
-	blk_queue_max_phys_segments(sdev->request_queue, MV_MAX_SG_CT / 2);
-
-	return 0;	/* scsi layer doesn't check return value, sigh */
-}
-
 static void mv_set_edma_ptrs(void __iomem *port_mmio,
 			     struct mv_host_priv *hpriv,
 			     struct mv_port_priv *pp)
@@ -1138,34 +1126,35 @@ static void mv_fill_sg(struct ata_queued_cmd *qc)
 {
 	struct mv_port_priv *pp = qc->ap->private_data;
 	struct scatterlist *sg;
-	struct mv_sg *mv_sg;
+	struct mv_sg *mv_sg = pp->sg_tbl;
+	unsigned int idx = 0;
 
-	mv_sg = pp->sg_tbl;
 	ata_for_each_sg(sg, qc) {
-		dma_addr_t addr = sg_dma_address(sg);
-		u32 sg_len = sg_dma_len(sg);
+		u64 addr;
+		u32 offset, sg_len, len;
+
+		addr = sg_dma_address(sg);
+		sg_len = sg_dma_len(sg);
 
 		while (sg_len) {
-			u32 offset = addr & 0xffff;
-			u32 len = sg_len;
+			offset = addr & 0xffff;
+			len = sg_len;
 
 			if ((offset + sg_len > 0x10000))
 				len = 0x10000 - offset;
 
-			mv_sg->addr = cpu_to_le32(addr & 0xffffffff);
-			mv_sg->addr_hi = cpu_to_le32((addr >> 16) >> 16);
-			mv_sg->flags_size = cpu_to_le32(len & 0xffff);
+			mv_sg[idx].addr = cpu_to_le32(addr & 0xffffffff);
+			mv_sg[idx].addr_hi = cpu_to_le32(addr >> 32);
+			mv_sg[idx].flags_size = cpu_to_le32(len & 0xffff);
 
+			idx++;
 			sg_len -= len;
 			addr += len;
-
-			if (!sg_len && ata_sg_is_last(sg, qc))
-				mv_sg->flags_size |= cpu_to_le32(EPRD_FLAG_END_OF_TBL);
-
-			mv_sg++;
 		}
-
 	}
+
+	if (idx)
+		mv_sg[idx - 1].flags_size |= cpu_to_le32(EPRD_FLAG_END_OF_TBL);
 }
 
 static inline void mv_crqb_pack_cmd(__le16 *cmdw, u8 data, u8 addr, unsigned last)
#107 lore raw

Re: [bug] ata subsystem related crash with latest -git

Quoted 4 lines · Jeff Garzik
Quoted 1 line · Jens Axboe

The sata_mv construct looks a bit odd. Does this work? That last

The sata_mv construct worked just fine before sg chaining :)

Yes I know, but I'm trying to works towards getting rid of sg_last() and ata_sg_is_last() anyway :-)

Quoted 36 lines
Quoted 21 lines

end_mv_sg test should always be true, just being paranoid...

drivers/ata/sata_mv.c +1 -4
index 4df8311..5397eea 100644
@@ -1138,8 +1138,9 @@ static void mv_fill_sg(struct ata_queued_cmd *qc)
 {
 	struct mv_port_priv *pp = qc->ap->private_data;
 	struct scatterlist *sg;
-	struct mv_sg *mv_sg;
+	struct mv_sg *mv_sg, *end_mv_sg;
 +	end_mv_sg = NULL;
 	mv_sg = pp->sg_tbl;
 	ata_for_each_sg(sg, qc) {
 		dma_addr_t addr = sg_dma_address(sg);
@@ -1158,14 +1159,12 @@ static void mv_fill_sg(struct ata_queued_cmd *qc)
  			sg_len -= len;
 			addr += len;
-
-			if (!sg_len && ata_sg_is_last(sg, qc))
-				mv_sg->flags_size |= cpu_to_le32(EPRD_FLAG_END_OF_TBL);

drivers/ata/sata_mv.c static void mv_fill_sg(struct ata_queued_cmd *qc) · line 1168

Quoted 10 lines
drivers/ata/sata_mv.c +3 -2
-
+			end_mv_sg = mv_sg;
 			mv_sg++;
 		}
-
 	}
+	if (end_mv_sg)
+		end_mv_sg->flags_size |= cpu_to_le32(EPRD_FLAG_END_OF_TBL);
 }
 

I'm testing a similar patch based on ata_fill_sg()'s method, which basically does something similar to what you've done here (see attached). I had noticed that ata_fill_sg() did not call ata_sg_is_last().

Quoted 7 lines

If this fixes the problem, I think the best solution would be to delete ata_sg_is_last(). In the few users that exist, we should be able to eliminate the test programmatically as you and ata_fill_sg() have done -- thereby eliminating a branch per loop in a hotpath.

Off to test the attached... if that doesn't work I'll try your version, though there shouldn't be much difference.

That should work as well. WRT ata_sg_is_last(), if we go ahead with my recent sg chaining updates, we can keep the test as it would be a single conditional and not require any looping.

Let me know when you have tested this!

Signature · 1 line
Jens Axboe
#108 lore raw

Re: [bug] ata subsystem related crash with latest -git

Quoted 5 lines · Jens Axboe

That should work as well. WRT ata_sg_is_last(), if we go ahead with my recent sg chaining updates, we can keep the test as it would be a single conditional and not require any looping.

Let me know when you have tested this!

The patch I attached to the last email got both sata_mv test boxes working reliably (so far).

I worked up a patch that kills ata_sg_is_last() (plus the max_phys_segments sata_mv fix), see attached. I'm thinking this is what I like to see in upstream.

Of course, this doesn't explain why ata_sg_is_last() was broken, but since it's working _and_ slightly more efficient, I don't really care :)

Jeff

drivers/ata/pdc_adma.c +6 -3
index 8d1b03d..199f7e1 100644
@@ -318,7 +318,7 @@ static int adma_fill_sg(struct ata_queued_cmd *qc)
 	struct scatterlist *sg;
 	struct ata_port *ap = qc->ap;
 	struct adma_port_priv *pp = ap->private_data;
-	u8  *buf = pp->pkt;
+	u8  *buf = pp->pkt, *last_buf = NULL;
 	int i = (2 + buf[3]) * 8;
 	u8 pFLAGS = pORD | ((qc->tf.flags & ATA_TFLAG_WRITE) ? pDIRO : 0);
 
@@ -334,8 +334,7 @@ static int adma_fill_sg(struct ata_queued_cmd *qc)
 		*(__le32 *)(buf + i) = cpu_to_le32(len);
 		i += 4;
 
-		if (ata_sg_is_last(sg, qc))
-			pFLAGS |= pEND;
+		last_buf = &buf[i];
 		buf[i++] = pFLAGS;
 		buf[i++] = qc->dev->dma_mode & 0xf;
 		buf[i++] = 0;	/* pPKLW */
@@ -348,6 +347,10 @@ static int adma_fill_sg(struct ata_queued_cmd *qc)
 		VPRINTK("PRD[%u] = (0x%lX, 0x%X)\n", i/4,
 					(unsigned long)addr, len);
 	}
+
+	if (likely(last_buf))
+		*last_buf |= pEND;
+
 	return i;
 }
 
drivers/ata/sata_mv.c +7 -19
index 4df8311..7f1b13e 100644
@@ -421,7 +421,6 @@ static void mv_error_handler(struct ata_port *ap);
 static void mv_post_int_cmd(struct ata_queued_cmd *qc);
 static void mv_eh_freeze(struct ata_port *ap);
 static void mv_eh_thaw(struct ata_port *ap);
-static int mv_slave_config(struct scsi_device *sdev);
 static int mv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
 
 static void mv5_phy_errata(struct mv_host_priv *hpriv, void __iomem *mmio,
@@ -459,7 +458,7 @@ static struct scsi_host_template mv5_sht = {
 	.use_clustering		= 1,
 	.proc_name		= DRV_NAME,
 	.dma_boundary		= MV_DMA_BOUNDARY,
-	.slave_configure	= mv_slave_config,
+	.slave_configure	= ata_scsi_slave_config,
 	.slave_destroy		= ata_scsi_slave_destroy,
 	.bios_param		= ata_std_bios_param,
 };
@@ -477,7 +476,7 @@ static struct scsi_host_template mv6_sht = {
 	.use_clustering		= 1,
 	.proc_name		= DRV_NAME,
 	.dma_boundary		= MV_DMA_BOUNDARY,
-	.slave_configure	= mv_slave_config,
+	.slave_configure	= ata_scsi_slave_config,
 	.slave_destroy		= ata_scsi_slave_destroy,
 	.bios_param		= ata_std_bios_param,
 };
@@ -756,17 +755,6 @@ static void mv_irq_clear(struct ata_port *ap)
 {
 }
 
-static int mv_slave_config(struct scsi_device *sdev)
-{
-	int rc = ata_scsi_slave_config(sdev);
-	if (rc)
-		return rc;
-
-	blk_queue_max_phys_segments(sdev->request_queue, MV_MAX_SG_CT / 2);
-
-	return 0;	/* scsi layer doesn't check return value, sigh */
-}
-
 static void mv_set_edma_ptrs(void __iomem *port_mmio,
 			     struct mv_host_priv *hpriv,
 			     struct mv_port_priv *pp)
@@ -1138,7 +1126,7 @@ static void mv_fill_sg(struct ata_queued_cmd *qc)
 {
 	struct mv_port_priv *pp = qc->ap->private_data;
 	struct scatterlist *sg;
-	struct mv_sg *mv_sg;
+	struct mv_sg *mv_sg, *last_sg = NULL;
 
 	mv_sg = pp->sg_tbl;
 	ata_for_each_sg(sg, qc) {
@@ -1159,13 +1147,13 @@ static void mv_fill_sg(struct ata_queued_cmd *qc)
 			sg_len -= len;
 			addr += len;
 
-			if (!sg_len && ata_sg_is_last(sg, qc))
-				mv_sg->flags_size |= cpu_to_le32(EPRD_FLAG_END_OF_TBL);
-
+			last_sg = mv_sg;
 			mv_sg++;
 		}
-
 	}
+
+	if (likely(last_sg))
+		last_sg->flags_size |= cpu_to_le32(EPRD_FLAG_END_OF_TBL);
 }
 
 static inline void mv_crqb_pack_cmd(__le16 *cmdw, u8 data, u8 addr, unsigned last)
drivers/ata/sata_sil24.c +7 -4
index b061927..26ebffc 100644
@@ -796,16 +796,19 @@ static inline void sil24_fill_sg(struct ata_queued_cmd *qc,
 				 struct sil24_sge *sge)
 {
 	struct scatterlist *sg;
+	struct sil24_sge *last_sge = NULL;
 
 	ata_for_each_sg(sg, qc) {
 		sge->addr = cpu_to_le64(sg_dma_address(sg));
 		sge->cnt = cpu_to_le32(sg_dma_len(sg));
-		if (ata_sg_is_last(sg, qc))
-			sge->flags = cpu_to_le32(SGE_TRM);
-		else
-			sge->flags = 0;
+		sge->flags = 0;
+
+		last_sge = sge;
 		sge++;
 	}
+
+	if (likely(last_sge))
+		last_sge->flags = cpu_to_le32(SGE_TRM);
 }
 
 static int sil24_qc_defer(struct ata_queued_cmd *qc)
drivers/scsi/ipr.c +7 -4
index b41dfb5..c316a0b 100644
@@ -5134,6 +5134,7 @@ static void ipr_build_ata_ioadl(struct ipr_cmnd *ipr_cmd,
 	u32 ioadl_flags = 0;
 	struct ipr_ioarcb *ioarcb = &ipr_cmd->ioarcb;
 	struct ipr_ioadl_desc *ioadl = ipr_cmd->ioadl;
+	struct ipr_ioadl_desc *last_ioadl = NULL;
 	int len = qc->nbytes + qc->pad_len;
 	struct scatterlist *sg;
 
@@ -5156,11 +5157,13 @@ static void ipr_build_ata_ioadl(struct ipr_cmnd *ipr_cmd,
 	ata_for_each_sg(sg, qc) {
 		ioadl->flags_and_data_len = cpu_to_be32(ioadl_flags | sg_dma_len(sg));
 		ioadl->address = cpu_to_be32(sg_dma_address(sg));
-		if (ata_sg_is_last(sg, qc))
-			ioadl->flags_and_data_len |= cpu_to_be32(IPR_IOADL_FLAGS_LAST);
-		else
-			ioadl++;
+
+		last_ioadl = ioadl;
+		ioadl++;
 	}
+
+	if (likely(last_ioadl))
+		last_ioadl->flags_and_data_len |= cpu_to_be32(IPR_IOADL_FLAGS_LAST);
 }
 
 /**
include/linux/libata.h +0 -12
index 377e6d4..bc3b6fc 100644
@@ -1037,18 +1037,6 @@ extern void ata_port_pbar_desc(struct ata_port *ap, int bar, ssize_t offset,
 /*
  * qc helpers
  */
-static inline int
-ata_sg_is_last(struct scatterlist *sg, struct ata_queued_cmd *qc)
-{
-	if (sg == &qc->pad_sgent)
-		return 1;
-	if (qc->pad_len)
-		return 0;
-	if (qc->n_iter == qc->n_elem)
-		return 1;
-	return 0;
-}
-
 static inline struct scatterlist *
 ata_qc_first_sg(struct ata_queued_cmd *qc)
 {
#109 lore raw

Re: [bug] ata subsystem related crash with latest -git

Quoted 6 lines · Jeff Garzik
Quoted 4 lines · Jens Axboe

That should work as well. WRT ata_sg_is_last(), if we go ahead with my recent sg chaining updates, we can keep the test as it would be a single conditional and not require any looping.
Let me know when you have tested this!

Quoted 6 lines

The patch I attached to the last email got both sata_mv test boxes working reliably (so far).

I worked up a patch that kills ata_sg_is_last() (plus the max_phys_segments sata_mv fix), see attached. I'm thinking this is what I like to see in upstream.

Great!

Quoted 2 lines

Of course, this doesn't explain why ata_sg_is_last() was broken, but since it's working _and_ slightly more efficient, I don't really care :)

Tomo and I agreed to kill sg_last() a few days ago anyways, so this is perfectly fine with me.

Signature · 1 line
Jens Axboe
#110 lore raw

Re: [bug] ata subsystem related crash with latest -git

Quoted 6 lines · Jens Axboe
Quoted 3 lines

Of course, this doesn't explain why ata_sg_is_last() was broken, but since it's working _and_ slightly more efficient, I don't really care :)

Tomo and I agreed to kill sg_last() a few days ago anyways, so this is perfectly fine with me.

it would still be nice to figure out exactly why it was broken.

Ingo

#111 lore raw

Re: [bug] ata subsystem related crash with latest -git

Quoted 11 lines · Ingo Molnar
Quoted 6 lines · Jens Axboe
Quoted 3 lines

Of course, this doesn't explain why ata_sg_is_last() was broken, but since it's working _and_ slightly more efficient, I don't really care :)

Tomo and I agreed to kill sg_last() a few days ago anyways, so this is perfectly fine with me.

it would still be nice to figure out exactly why it was broken.

It would always be nice. For this case I don't think it's very interesting, if we pursue the improved sg iteration setup.

Signature · 1 line
Jens Axboe
#112 lore raw

Re: [bug] ata subsystem related crash with latest -git

Quoted 8 lines · Jens Axboe
Quoted 7 lines · Ingo Molnar
Quoted 4 lines · Jens Axboe
Quoted 3 lines

Of course, this doesn't explain why ata_sg_is_last() was broken, but since it's working _and_ slightly more efficient, I don't really care :)

Quoted 7 lines
Quoted 4 lines
Quoted 2 lines

Tomo and I agreed to kill sg_last() a few days ago anyways, so this is perfectly fine with me.

it would still be nice to figure out exactly why it was broken.

It would always be nice. For this case I don't think it's very interesting, if we pursue the improved sg iteration setup.

BTW, I think it's pretty clear that ata_sg_is_last() is broken. It's likely a one-off in the n_iter test.

Signature · 1 line
Jens Axboe
#113 lore raw

Re: [bug] ata subsystem related crash with latest -git

Quoted 5 lines · Jens Axboe
Quoted 2 lines

It would always be nice. For this case I don't think it's very interesting, if we pursue the improved sg iteration setup.

BTW, I think it's pretty clear that ata_sg_is_last() is broken. It's likely a one-off in the n_iter test.

fixing that would be a -stable candidate, as a potential data corruptor
- or is it more benign?

Ingo

#114 lore raw

Re: [bug] ata subsystem related crash with latest -git

Quoted 7 lines · Ingo Molnar
Quoted 2 lines · Jens Axboe
Quoted 1 line

It would always be nice. For this case I don't think it's very interesting, if we pursue the improved sg iteration setup.

BTW, I think it's pretty clear that ata_sg_is_last() is broken. It's likely a one-off in the n_iter test.

fixing that would be a -stable candidate, as a potential data corruptor
- or is it more benign?

It is confirmed working prior to the sg-chaining stuff, so 2.6.23.1 is OK...

Jeff

#115 lore raw

Re: [bug] ata subsystem related crash with latest -git

In reply to Ingo Molnar
Quoted 11 lines · Ingo Molnar
Quoted 5 lines · Jens Axboe
Quoted 2 lines

It would always be nice. For this case I don't think it's very interesting, if we pursue the improved sg iteration setup.

BTW, I think it's pretty clear that ata_sg_is_last() is broken. It's likely a one-off in the n_iter test.

fixing that would be a -stable candidate, as a potential data corruptor
- or is it more benign?

I think it's safe to say that it was sg chaining introduced breakage, so it should work fine in 2.6.23.x.

Signature · 1 line
Jens Axboe
#116 lore raw

[PATCH] Re: [bug] ata subsystem related crash with latest -git

In reply to Jens Axboe
Quoted 8 lines · Jens Axboe
Quoted 7 lines · Jeff Garzik
Quoted 4 lines · Jens Axboe

That should work as well. WRT ata_sg_is_last(), if we go ahead with my recent sg chaining updates, we can keep the test as it would be a single conditional and not require any looping.
Let me know when you have tested this!

The patch I attached to the last email got both sata_mv test boxes working reliably (so far).

Quoted 8 lines
Quoted 1 line

I worked up a patch that kills ata_sg_is_last() (plus the max_phys_segments sata_mv fix), see attached. I'm thinking this is what I like to see in upstream.

Great!

Quoted 1 line

Of course, this doesn't explain why ata_sg_is_last() was broken, but since it's working _and_ slightly more efficient, I don't really care :)

Tomo and I agreed to kill sg_last() a few days ago anyways, so this is perfectly fine with me.

Yep, the [attached] patch that kills ata_sg_is_last() is working here on both machines that were previously croaking.

It would be nice to get pdc_adma, sata_sil24 and ipr it-works test done, but IMO the patch is pretty straightforward and should be OK.

Jeff

drivers/ata/pdc_adma.c +6 -3
index 8d1b03d..199f7e1 100644
@@ -318,7 +318,7 @@ static int adma_fill_sg(struct ata_queued_cmd *qc)
 	struct scatterlist *sg;
 	struct ata_port *ap = qc->ap;
 	struct adma_port_priv *pp = ap->private_data;
-	u8  *buf = pp->pkt;
+	u8  *buf = pp->pkt, *last_buf = NULL;
 	int i = (2 + buf[3]) * 8;
 	u8 pFLAGS = pORD | ((qc->tf.flags & ATA_TFLAG_WRITE) ? pDIRO : 0);
 
@@ -334,8 +334,7 @@ static int adma_fill_sg(struct ata_queued_cmd *qc)
 		*(__le32 *)(buf + i) = cpu_to_le32(len);
 		i += 4;
 
-		if (ata_sg_is_last(sg, qc))
-			pFLAGS |= pEND;
+		last_buf = &buf[i];
 		buf[i++] = pFLAGS;
 		buf[i++] = qc->dev->dma_mode & 0xf;
 		buf[i++] = 0;	/* pPKLW */
@@ -348,6 +347,10 @@ static int adma_fill_sg(struct ata_queued_cmd *qc)
 		VPRINTK("PRD[%u] = (0x%lX, 0x%X)\n", i/4,
 					(unsigned long)addr, len);
 	}
+
+	if (likely(last_buf))
+		*last_buf |= pEND;
+
 	return i;
 }
 
drivers/ata/sata_mv.c +7 -19
index 4df8311..7f1b13e 100644
@@ -421,7 +421,6 @@ static void mv_error_handler(struct ata_port *ap);
 static void mv_post_int_cmd(struct ata_queued_cmd *qc);
 static void mv_eh_freeze(struct ata_port *ap);
 static void mv_eh_thaw(struct ata_port *ap);
-static int mv_slave_config(struct scsi_device *sdev);
 static int mv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
 
 static void mv5_phy_errata(struct mv_host_priv *hpriv, void __iomem *mmio,
@@ -459,7 +458,7 @@ static struct scsi_host_template mv5_sht = {
 	.use_clustering		= 1,
 	.proc_name		= DRV_NAME,
 	.dma_boundary		= MV_DMA_BOUNDARY,
-	.slave_configure	= mv_slave_config,
+	.slave_configure	= ata_scsi_slave_config,
 	.slave_destroy		= ata_scsi_slave_destroy,
 	.bios_param		= ata_std_bios_param,
 };
@@ -477,7 +476,7 @@ static struct scsi_host_template mv6_sht = {
 	.use_clustering		= 1,
 	.proc_name		= DRV_NAME,
 	.dma_boundary		= MV_DMA_BOUNDARY,
-	.slave_configure	= mv_slave_config,
+	.slave_configure	= ata_scsi_slave_config,
 	.slave_destroy		= ata_scsi_slave_destroy,
 	.bios_param		= ata_std_bios_param,
 };
@@ -756,17 +755,6 @@ static void mv_irq_clear(struct ata_port *ap)
 {
 }
 
-static int mv_slave_config(struct scsi_device *sdev)
-{
-	int rc = ata_scsi_slave_config(sdev);
-	if (rc)
-		return rc;
-
-	blk_queue_max_phys_segments(sdev->request_queue, MV_MAX_SG_CT / 2);
-
-	return 0;	/* scsi layer doesn't check return value, sigh */
-}
-
 static void mv_set_edma_ptrs(void __iomem *port_mmio,
 			     struct mv_host_priv *hpriv,
 			     struct mv_port_priv *pp)
@@ -1138,7 +1126,7 @@ static void mv_fill_sg(struct ata_queued_cmd *qc)
 {
 	struct mv_port_priv *pp = qc->ap->private_data;
 	struct scatterlist *sg;
-	struct mv_sg *mv_sg;
+	struct mv_sg *mv_sg, *last_sg = NULL;
 
 	mv_sg = pp->sg_tbl;
 	ata_for_each_sg(sg, qc) {
@@ -1159,13 +1147,13 @@ static void mv_fill_sg(struct ata_queued_cmd *qc)
 			sg_len -= len;
 			addr += len;
 
-			if (!sg_len && ata_sg_is_last(sg, qc))
-				mv_sg->flags_size |= cpu_to_le32(EPRD_FLAG_END_OF_TBL);
-
+			last_sg = mv_sg;
 			mv_sg++;
 		}
-
 	}
+
+	if (likely(last_sg))
+		last_sg->flags_size |= cpu_to_le32(EPRD_FLAG_END_OF_TBL);
 }
 
 static inline void mv_crqb_pack_cmd(__le16 *cmdw, u8 data, u8 addr, unsigned last)
drivers/ata/sata_sil24.c +7 -4
index b061927..26ebffc 100644
@@ -796,16 +796,19 @@ static inline void sil24_fill_sg(struct ata_queued_cmd *qc,
 				 struct sil24_sge *sge)
 {
 	struct scatterlist *sg;
+	struct sil24_sge *last_sge = NULL;
 
 	ata_for_each_sg(sg, qc) {
 		sge->addr = cpu_to_le64(sg_dma_address(sg));
 		sge->cnt = cpu_to_le32(sg_dma_len(sg));
-		if (ata_sg_is_last(sg, qc))
-			sge->flags = cpu_to_le32(SGE_TRM);
-		else
-			sge->flags = 0;
+		sge->flags = 0;
+
+		last_sge = sge;
 		sge++;
 	}
+
+	if (likely(last_sge))
+		last_sge->flags = cpu_to_le32(SGE_TRM);
 }
 
 static int sil24_qc_defer(struct ata_queued_cmd *qc)
drivers/scsi/ipr.c +7 -4
index b41dfb5..c316a0b 100644
@@ -5134,6 +5134,7 @@ static void ipr_build_ata_ioadl(struct ipr_cmnd *ipr_cmd,
 	u32 ioadl_flags = 0;
 	struct ipr_ioarcb *ioarcb = &ipr_cmd->ioarcb;
 	struct ipr_ioadl_desc *ioadl = ipr_cmd->ioadl;
+	struct ipr_ioadl_desc *last_ioadl = NULL;
 	int len = qc->nbytes + qc->pad_len;
 	struct scatterlist *sg;
 
@@ -5156,11 +5157,13 @@ static void ipr_build_ata_ioadl(struct ipr_cmnd *ipr_cmd,
 	ata_for_each_sg(sg, qc) {
 		ioadl->flags_and_data_len = cpu_to_be32(ioadl_flags | sg_dma_len(sg));
 		ioadl->address = cpu_to_be32(sg_dma_address(sg));
-		if (ata_sg_is_last(sg, qc))
-			ioadl->flags_and_data_len |= cpu_to_be32(IPR_IOADL_FLAGS_LAST);
-		else
-			ioadl++;
+
+		last_ioadl = ioadl;
+		ioadl++;
 	}
+
+	if (likely(last_ioadl))
+		last_ioadl->flags_and_data_len |= cpu_to_be32(IPR_IOADL_FLAGS_LAST);
 }
 
 /**
include/linux/libata.h +0 -12
index 377e6d4..bc3b6fc 100644
@@ -1037,18 +1037,6 @@ extern void ata_port_pbar_desc(struct ata_port *ap, int bar, ssize_t offset,
 /*
  * qc helpers
  */
-static inline int
-ata_sg_is_last(struct scatterlist *sg, struct ata_queued_cmd *qc)
-{
-	if (sg == &qc->pad_sgent)
-		return 1;
-	if (qc->pad_len)
-		return 0;
-	if (qc->n_iter == qc->n_elem)
-		return 1;
-	return 0;
-}
-
 static inline struct scatterlist *
 ata_qc_first_sg(struct ata_queued_cmd *qc)
 {
#117 lore raw

Re: [PATCH] Re: [bug] ata subsystem related crash with latest -git

Quoted 8 lines · Jeff Garzik
Quoted 2 lines

Tomo and I agreed to kill sg_last() a few days ago anyways, so this is perfectly fine with me.

Yep, the [attached] patch that kills ata_sg_is_last() is working here on both machines that were previously croaking.

It would be nice to get pdc_adma, sata_sil24 and ipr it-works test done, but IMO the patch is pretty straightforward and should be OK.

just a quick question: i have Jens's workarounds applied right now (see patch below). Am i now crash/corruption-safe, or do i need your patch too? And once your patch [and the other sg_*() patches] are upstream i dont need the workaround anymore, correct?

Ingo

---
 block/ll_rw_blk.c         |    2 +-
 drivers/ata/libata-core.c |    2 +-
 drivers/scsi/scsi_lib.c   |    2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)
block/ll_rw_blk.c +1 -1
Index: linux/block/ll_rw_blk.c
===================================================================
@@ -631,7 +631,7 @@ void blk_queue_max_phys_segments(struct 
 		printk("%s: set to minimum %d\n", __FUNCTION__, max_segments);
 	}
 
-	q->max_phys_segments = max_segments;
+	q->max_phys_segments = max_segments - 1;
 }
 
 EXPORT_SYMBOL(blk_queue_max_phys_segments);
drivers/ata/libata-core.c +1 -1
Index: linux/drivers/ata/libata-core.c
===================================================================
@@ -4664,7 +4664,7 @@ static int ata_sg_setup(struct ata_queue
 {
 	struct ata_port *ap = qc->ap;
 	struct scatterlist *sg = qc->__sg;
-	struct scatterlist *lsg = sg_last(qc->__sg, qc->n_elem);
+	struct scatterlist *lsg = &qc->__sg[qc->n_elem - 1];
 	int n_elem, pre_n_elem, dir, trim_sg = 0;
 
 	VPRINTK("ENTER, ata%u\n", ap->print_id);
drivers/scsi/scsi_lib.c +1 -1
Index: linux/drivers/scsi/scsi_lib.c
===================================================================
@@ -39,7 +39,7 @@
  * (unless chaining is used). Should ideally fit inside a single page, to
  * avoid a higher order allocation.
  */
-#define SCSI_MAX_SG_SEGMENTS	128
+#define SCSI_MAX_SG_SEGMENTS	129
 
 struct scsi_host_sg_pool {
 	size_t		size;
#118 lore raw

Re: [PATCH] Re: [bug] ata subsystem related crash with latest -git

Quoted 8 lines · Ingo Molnar
Quoted 4 lines · Jeff Garzik
Quoted 1 line

Tomo and I agreed to kill sg_last() a few days ago anyways, so this is perfectly fine with me.

Yep, the [attached] patch that kills ata_sg_is_last() is working here on both machines that were previously croaking.

It would be nice to get pdc_adma, sata_sil24 and ipr it-works test done, but IMO the patch is pretty straightforward and should be OK.

just a quick question: i have Jens's workarounds applied right now (see patch below). Am i now crash/corruption-safe, or do i need your patch too? And once your patch [and the other sg_*() patches] are upstream i dont need the workaround anymore, correct?

You need my patch if and only if you use one of the drivers touched by the patch. ata_sg_is_last() was a driver helper function, so my fix never really touched core code.

I never had to apply the changes you included, to fix problems here.

And looking at those changes...

Quoted 2 lines
block/ll_rw_blk.c +1 -1
-	q->max_phys_segments = max_segments;
+	q->max_phys_segments = max_segments - 1;

...

drivers/scsi/scsi_lib.c line 42

Quoted 2 lines
drivers/scsi/scsi_lib.c +1 -1
-#define SCSI_MAX_SG_SEGMENTS	128
+#define SCSI_MAX_SG_SEGMENTS	129

I wonder if libata should be doing

	blk_queue_max_phys_segments(q, q->max_phys_segments - 1)

to account for the pad entry that libata owns.

Jeff

#119 lore raw

Re: [PATCH] Re: [bug] ata subsystem related crash with latest -git

Quoted 1 line · Jeff Garzik

I never had to apply the changes you included, to fix problems here.

perhaps because you are not running a CONFIG_DEBUG_PAGEALLOC=y kernel?

I recently fixed DEBUG_PAGEALLOC (it would crash upon bootup on x86 most of the time on any real hardware - so i doubt people were able to use it all that much). As long as you try Linus' latest -git tree (which has the latest arch/x86 merge) and use the 32-bit x86 kernel it should work fine for you too, and you will probably be able to trigger similar crashes too.

Ingo

Tested-by
#120 lore raw

Re: [PATCH] Re: [bug] ata subsystem related crash with latest -git

In reply to Jeff Garzik
Quoted 16 lines · Jeff Garzik
Quoted 15 lines · Jens Axboe
Quoted 11 lines · Jeff Garzik
Quoted 4 lines · Jens Axboe

That should work as well. WRT ata_sg_is_last(), if we go ahead with my recent sg chaining updates, we can keep the test as it would be a single conditional and not require any looping.
Let me know when you have tested this!

The patch I attached to the last email got both sata_mv test boxes working reliably (so far).

I worked up a patch that kills ata_sg_is_last() (plus the max_phys_segments sata_mv fix), see attached. I'm thinking this is what I like to see in upstream.

Great!

Quoted 2 lines

Of course, this doesn't explain why ata_sg_is_last() was broken, but since it's working _and_ slightly more efficient, I don't really care :)

Quoted 8 lines
Quoted 2 lines

Tomo and I agreed to kill sg_last() a few days ago anyways, so this is perfectly fine with me.

Yep, the [attached] patch that kills ata_sg_is_last() is working here on both machines that were previously croaking.

It would be nice to get pdc_adma, sata_sil24 and ipr it-works test done, but IMO the patch is pretty straightforward and should be OK.

Looks ok on SATA_SIL24 and SATA_MV here on PPC (together with Jens' latest patch). Both barfed before.

Thanks!

-Olof

#121 lore raw

Re: [bug] ata subsystem related crash with latest -git

In reply to Jens Axboe

[Just catching with reading lkml to this post]

Quoted 4 lines · Jens Axboe

Theory - ata_sg_is_last() isn't returning true for the last entry. Can you double check that it correcly marks the last entry in mv_fill_sg()? Alternatively, just try this patch.

I "hate" to point this out, but I already reported that sata_sil24 fails on 1. Sep.:
http://lkml.org/lkml/2007/9/1/95

In the thread "sata_sil24 broken since 2.6.23-rc4-mm1" I spent over a week to trace this back to ata_sg_is_last (finally found on 7. Oct):
http://lkml.org/lkml/2007/10/7/43

Thanks for finally patching this now...

Torsten

#122 lore raw

Re: [bug] ata subsystem related crash with latest -git

In reply to Ingo Molnar
Quoted 5 lines · Ingo Molnar

Review

Quoted 2 lines
(quoted diff) +1 -1
-#define SCSI_MAX_SG_SEGMENTS	128
+#define SCSI_MAX_SG_SEGMENTS	129

this one finally made the trick and it's booting fine now, without any crashes!

hm, spoke too soon - i just got the ata_qc_issue() crash below. I've attached further below the current fixes/workarounds that i have applied at the moment. Any ideas?

Ingo

------------->

[  155.259466] kjournald starting.  Commit interval 5 seconds
[  155.265103] EXT3 FS on sda5, internal journal
[  155.269319] EXT3-fs: mounted filesystem with ordered data mode.
[  156.458225] BUG: unable to handle kernel paging request at virtual address 7d5ac000
[  156.465723] printing eip: 784e9300 *pde = 00ddd027 *pte = 055ac000 
[  156.471964] Oops: 0000 [#1] DEBUG_PAGEALLOC
[  156.476123] 
[  156.477597] Pid: 0, comm: swapper Not tainted (2.6.23 #40)
[  156.483055] EIP: 0060:[<784e9300>] EFLAGS: 00010006 CPU: 0
[  156.488520] EIP is at ata_qc_issue+0xd0/0x340
[  156.492848] EAX: 3d328000 EBX: 7d5ac000 ECX: 00000020 EDX: 00000020
[  156.499087] ESI: 7d5ab480 EDI: 7d5abe00 EBP: 7b54007c ESP: 78a13e1c
[  156.505328]  DS: 007b ES: 007b FS: 0000 GS: 0000 SS: 0068
[  156.510700] Process swapper (pid: 0, ti=78a12000 task=789753e0 task.ti=78a12000)
[  156.517893] Stack: 7d5ac000 7b540000 7b540000 00000000 7d5abff0 7b54007c 7d5ab480 7b5417a4 
[  156.526213]        784c2330 784ef49e 784f1ff3 7b52de98 7d5ab480 7b540000 7b5417a4 7d5ab480 
[  156.534531]        7b540000 7b524004 784f20e0 784ef180 784c2330 7d5ab480 00000216 7b524004 
[  156.542851] Call Trace:
[  156.545452]  [<784c2330>] scsi_done+0x0/0x20
[  156.549698]  [<784ef49e>] ata_scsi_translate+0xbe/0x140
[  156.554897]  [<784f1ff3>] ata_scsi_queuecmd+0x33/0x200
[  156.560010]  [<784f20e0>] ata_scsi_queuecmd+0x120/0x200
[  156.565210]  [<784ef180>] ata_scsi_rw_xlat+0x0/0x220
[  156.570150]  [<784c2330>] scsi_done+0x0/0x20
[  156.574395]  [<784c2bb2>] scsi_dispatch_cmd+0x152/0x290
[  156.579596]  [<78135aa7>] trace_hardirqs_on+0x67/0xb0
[  156.584622]  [<784c8abe>] scsi_request_fn+0x1be/0x370
[  156.589649]  [<78407ef6>] blk_run_queue+0x36/0x80
[  156.594328]  [<784c73c0>] scsi_next_command+0x30/0x50
[  156.599354]  [<784c754b>] scsi_end_request+0xab/0xe0
[  156.604294]  [<784c8239>] scsi_io_completion+0xa9/0x3d0
[  156.609493]  [<78135aa7>] trace_hardirqs_on+0x67/0xb0
[  156.614520]  [<78404f85>] blk_done_softirq+0x45/0x80
[  156.619460]  [<78404fb3>] blk_done_softirq+0x73/0x80
[  156.624400]  [<7811d2f3>] __do_softirq+0x53/0xb0
[  156.628992]  [<7811d3b8>] do_softirq+0x68/0x70
[  156.633412]  [<78105351>] do_IRQ+0x51/0x90
[  156.637486]  [<7810290f>] restore_nocheck+0x12/0x15
[  156.642339]  [<7810388e>] common_interrupt+0x2e/0x40
[  156.647277]  [<7810f4c0>] pgd_dtor+0x0/0x50
[  156.651437]  [<7815f1d0>] quicklist_trim+0x0/0x90
[  156.656117]  [<7810f4bb>] check_pgt_cache+0x1b/0x20
[  156.660970]  [<78100c52>] cpu_idle+0x32/0x60
[  156.665217]  [<78a14b35>] start_kernel+0x265/0x300
[  156.669983]  [<78a14380>] unknown_bootoption+0x0/0x1e0
[  156.675096]  =======================
[  156.678649] Code: 84 d9 01 00 00 7e 32 31 d2 89 f6 8b 1c 24 83 c2 01 8b 03 2b 05 18 ed d7 78 c1 f8 05 c1 e0 0c 03 43 04 89 43 08 83 c3 10 89 1c 24 <8b> 03 a8 01 0f 85 58 02 00 00 39 ca 75 d2 f0 83 44 24 00 00 85 
[  156.697455] EIP: [<784e9300>] ata_qc_issue+0xd0/0x340 SS:ESP 0068:78a13e1c
[  156.704822] Kernel panic - not syncing: Fatal exception in interrupt
---
 block/ll_rw_blk.c         |    2 +-
 drivers/ata/libata-core.c |    2 +-
 drivers/scsi/scsi_lib.c   |    2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)
block/ll_rw_blk.c +1 -1
Index: linux/block/ll_rw_blk.c
===================================================================
@@ -631,7 +631,7 @@ void blk_queue_max_phys_segments(struct 
 		printk("%s: set to minimum %d\n", __FUNCTION__, max_segments);
 	}
 
-	q->max_phys_segments = max_segments;
+	q->max_phys_segments = max_segments - 1;
 }
 
 EXPORT_SYMBOL(blk_queue_max_phys_segments);
drivers/ata/libata-core.c +1 -1
Index: linux/drivers/ata/libata-core.c
===================================================================
@@ -4664,7 +4664,7 @@ static int ata_sg_setup(struct ata_queue
 {
 	struct ata_port *ap = qc->ap;
 	struct scatterlist *sg = qc->__sg;
-	struct scatterlist *lsg = sg_last(qc->__sg, qc->n_elem);
+	struct scatterlist *lsg = &qc->__sg[qc->n_elem - 1];
 	int n_elem, pre_n_elem, dir, trim_sg = 0;
 
 	VPRINTK("ENTER, ata%u\n", ap->print_id);
drivers/scsi/scsi_lib.c +1 -1
Index: linux/drivers/scsi/scsi_lib.c
===================================================================
@@ -39,7 +39,7 @@
  * (unless chaining is used). Should ideally fit inside a single page, to
  * avoid a higher order allocation.
  */
-#define SCSI_MAX_SG_SEGMENTS	128
+#define SCSI_MAX_SG_SEGMENTS	129
 
 struct scsi_host_sg_pool {
 	size_t		size;
#123 lore raw

Re: [bug] ata subsystem related crash with latest -git

Quoted 9 lines · Ingo Molnar
Quoted 5 lines · Ingo Molnar

drivers/scsi/scsi_lib.c line 42

Quoted 2 lines
drivers/scsi/scsi_lib.c +1 -1
-#define SCSI_MAX_SG_SEGMENTS	128
+#define SCSI_MAX_SG_SEGMENTS	129

this one finally made the trick and it's booting fine now, without any crashes!

Quoted 3 lines

hm, spoke too soon - i just got the ata_qc_issue() crash below. I've attached further below the current fixes/workarounds that i have applied at the moment. Any ideas?

Hang on Ingo, will post an updated patch soonish!

Signature · 1 line
Jens Axboe
#124 lore raw

Re: [bug] ata subsystem related crash with latest -git

In reply to Linus Torvalds
Quoted 3 lines · Linus Torvalds

I think you'll always hit it if you have a scatter-gather list that is exactly filled up.

In fact, I think you'll hit it even if you use the "for_each_sg()" helper function. Exactly because it does the sg = sg_next(sg) at the end of the for-loop, so it will do it for the last entry too, even though that will access one past the end.

So it really *is* the case that every *single* use of that SG chain needs to be switched over to only do the sg_next() when required, or sg_next() needs to be fixed to look at the current-in-use entry (which we *can* access) when it decides what to do about the next one (which we can *not* access).

Moving the sg_is_chain() bit to the previous entry would work, but it requires that nobody who could have a chained scatterlist must *ever* access "sg->page" directly - you'd always need to use some helper function that masks off the bit, eg

- rename "sg->page" into "sh->page_and_flag", and make it "unsigned long" instead of a pointer.

- change every single "sg->page" access to use "sg_page(sg)" instead:

	#define sg_pointer(sg)	((void *)((sg)->page_and_flag & ~1ul))

	static inline struct page *sg_page(struct scatterist *sg)
	{
		return sg_pointer(sg);
	}

- change "sg_next()" to do

	static inline struct scatterlist *sg_next(struct scatterlist *sg)
	{
		if (sg->page_and_flag & 1)
			sg = sg_pointer(sg+1)-1;
		return ++sg;
	}

where the magic is exactly the fact that now "sg_next()" will *not* derefence the next SG entry unless the current one was marked appropriately.

And then *creating* the chain obviously needs to properly mark the next-to-last entry with the "next entry is a pointer" flag.

Big diff, it sounds like. But I don't see many alternatives. Jens?

Linus

#125 lore raw

Re: [bug] ata subsystem related crash with latest -git

Quoted 12 lines · Linus Torvalds
Quoted 3 lines · Linus Torvalds

I think you'll always hit it if you have a scatter-gather list that is exactly filled up.

In fact, I think you'll hit it even if you use the "for_each_sg()" helper function. Exactly because it does the sg = sg_next(sg) at the end of the for-loop, so it will do it for the last entry too, even though that will access one past the end.

Quoted 5 lines

So it really *is* the case that every *single* use of that SG chain needs to be switched over to only do the sg_next() when required, or sg_next() needs to be fixed to look at the current-in-use entry (which we *can* access) when it decides what to do about the next one (which we can *not* access).

Auch yes, ok nevermind that last email. There really is no way around allocating an extra 'pad' entry right now, so the SCSI_MAX_SG_SEGMENTS at 129 should fix Ingo's oops and the other crap is void.

Quoted 26 lines

Moving the sg_is_chain() bit to the previous entry would work, but it requires that nobody who could have a chained scatterlist must *ever* access "sg->page" directly - you'd always need to use some helper function that masks off the bit, eg

- rename "sg->page" into "sh->page_and_flag", and make it "unsigned long" instead of a pointer.

- change every single "sg->page" access to use "sg_page(sg)" instead:

	#define sg_pointer(sg)	((void *)((sg)->page_and_flag & ~1ul))

	static inline struct page *sg_page(struct scatterist *sg)
	{
		return sg_pointer(sg);
	}

- change "sg_next()" to do

	static inline struct scatterlist *sg_next(struct scatterlist *sg)
	{
		if (sg->page_and_flag & 1)
			sg = sg_pointer(sg+1)-1;
		return ++sg;
	}
Quoted 8 lines

where the magic is exactly the fact that now "sg_next()" will *not* derefence the next SG entry unless the current one was marked appropriately.

And then *creating* the chain obviously needs to properly mark the next-to-last entry with the "next entry is a pointer" flag.

Big diff, it sounds like. But I don't see many alternatives. Jens?

Big diff is not a problem for me, my primary concern would be making sure that the interface is solid. The above also has the nice side effect of making all sg elements usable, so we don't waste any. IIRC this was even debated months ago when I first proposed sg chaining, I shied away from it because I thought it would seem huge and too invasive. Ah well. I'll get digging on this.

Signature · 1 line
Jens Axboe
#126 lore raw

Re: [bug] ata subsystem related crash with latest -git

In reply to Ingo Molnar
Quoted 2 lines · Ingo Molnar

ok, here's a different but similar crash that triggers on the testbox:

Just to confirm - is this with the added memset() in drivers/scsi/scsi_lib.c?

Or did you get this oops before that patch?

Linus

#127 lore raw

Re: [bug] ata subsystem related crash with latest -git

Quoted 8 lines · Linus Torvalds
Quoted 2 lines · Ingo Molnar

ok, here's a different but similar crash that triggers on the testbox:

Just to confirm - is this with the added memset() in drivers/scsi/scsi_lib.c?

yes, Jens's last patch was applied - apparently this ATA crash was masked by the other one. I got one successful bootup after applying Jens's patch and it has crashed on bootup twice since then.

Ingo

#128 lore raw
In reply to Jens Axboe
Quoted 4 lines · Jens Axboe

OK, it is fine, as long as the sglist is cleared initially. And I don't think there's anyway around that, clearly I didn't think long enough before including the memset() removal from Tomo.

Ok, I think that one-liner fixes the real bug.

But I think the rest of your changes are simply bad.

The fix to block/ll_rw_block.c should likely be something like the appended instead:

- remove the "memset()" you had added earlier. It's bogus. It cannot be the right thing. If the sg list wasn't initialized correctly much earlier, trying to initialize it late is pointless - it contains crap.

- the old code was fine, but let's initialize "sg" to NULL to make it clear that the initial value of sg is pointless, and only "next_sg" matters (since sg had better be assigned from that).

Hmm?

Linus

---

block/ll_rw_blk.c +2 -2
index 9e3f3cc..54d974e 100644
@@ -1333,7 +1333,8 @@ int blk_rq_map_sg(struct request_queue *q, struct request *rq,
 	 * for each bio in rq
 	 */
 	bvprv = NULL;
-	sg = next_sg = &sglist[0];
+	sg = NULL;
+	next_sg = &sglist[0];
 	rq_for_each_segment(bvec, rq, iter) {
 		int nbytes = bvec->bv_len;
 
@@ -1352,7 +1353,6 @@ new_segment:
 			sg = next_sg;
 			next_sg = sg_next(sg);
 
-			memset(sg, 0, sizeof(*sg));
 			sg->page = bvec->bv_page;
 			sg->length = nbytes;
 			sg->offset = bvec->bv_offset;
#129 lore raw
Quoted 10 lines · Linus Torvalds
Quoted 4 lines · Jens Axboe

OK, it is fine, as long as the sglist is cleared initially. And I don't think there's anyway around that, clearly I didn't think long enough before including the memset() removal from Tomo.

Ok, I think that one-liner fixes the real bug.

Quoted 8 lines

But I think the rest of your changes are simply bad.

The fix to block/ll_rw_block.c should likely be something like the appended instead:

- remove the "memset()" you had added earlier. It's bogus. It cannot be the right thing. If the sg list wasn't initialized correctly much earlier, trying to initialize it late is pointless - it contains crap.

It's required to clear output members (like dma_len and so on), since some of the IOMU code really wants that initialized.

Quoted 3 lines

- the old code was fine, but let's initialize "sg" to NULL to make it clear that the initial value of sg is pointless, and only "next_sg" matters (since sg had better be assigned from that).

If you prefer the old next_sg approach to my alternative, that is fine with me. But we do need the memset().

Signature · 1 line
Jens Axboe
#130 lore raw
Quoted 7 lines · Jens Axboe
Quoted 4 lines

- remove the "memset()" you had added earlier. It's bogus. It cannot be the right thing. If the sg list wasn't initialized correctly much earlier, trying to initialize it late is pointless - it contains crap.

It's required to clear output members (like dma_len and so on), since some of the IOMU code really wants that initialized.

No it's NOT!

The whole point here is that the sg had *already* better be cleared.

If it wasn't cleared before, that's a bug regardless of anything else. So a memset() is guaranteed to be either a no-op or hiding another bug!

So yes, those members had better be zero. It's just that they had better be zero long before that memset! The memset should have been done when allocating the SG list.

Quoted 2 lines

If you prefer the old next_sg approach to my alternative, that is fine with me. But we do need the memset().

Really? Explain why. Entering that code with a non-initialized SG list is a bug to begin with.

Linus

#131 lore raw
Quoted 13 lines · Linus Torvalds
Quoted 7 lines · Jens Axboe
Quoted 4 lines

- remove the "memset()" you had added earlier. It's bogus. It cannot be the right thing. If the sg list wasn't initialized correctly much earlier, trying to initialize it late is pointless - it contains crap.

It's required to clear output members (like dma_len and so on), since some of the IOMU code really wants that initialized.

No it's NOT!

Quoted 8 lines

The whole point here is that the sg had *already* better be cleared.

If it wasn't cleared before, that's a bug regardless of anything else. So a memset() is guaranteed to be either a no-op or hiding another bug!

So yes, those members had better be zero. It's just that they had better be zero long before that memset! The memset should have been done when allocating the SG list.

For the chain elements - yes, definitely! But we also want to clear dma mapping output values, at least sparc64 wants that. You could argue that the IOMMU code should be fixed up, but I don't think we should mix the two.

So we need the memset() in blk_rq_map_sg() AS WELL AS the initial sg table clear. I'm not arguing about the latter, we clearly do need that.

Quoted 5 lines
Quoted 2 lines

If you prefer the old next_sg approach to my alternative, that is fine with me. But we do need the memset().

Really? Explain why. Entering that code with a non-initialized SG list is a bug to begin with.

Not for the output members, they will be filled AFTER blk_rq_map_sg() returns and someone use iommu_map_sg() to re-iterate the sglist and map the pages appropriately.

Signature · 1 line
Jens Axboe
#132 lore raw
Quoted 8 lines · Jens Axboe

For the chain elements - yes, definitely! But we also want to clear dma mapping output values, at least sparc64 wants that. You could argue that the IOMMU code should be fixed up, but I don't think we should mix the two.

So we need the memset() in blk_rq_map_sg() AS WELL AS the initial sg table clear. I'm not arguing about the latter, we clearly do need that.

I still don't see your argument.

The SG table is *already*zeroed*.

The fact that output values will be changed later is irrelevant. They haven't been changed *before* - or at least you haven't given a reasonable explanation for why they should have.

So explain to me why the memset() in blk_rq_map_sg() helps, considering that the memory must have been zeroed at allocation time anyway? Tell me what it is that fills any of the fields we don't already initialize, and that can happen *before* that memset?

Linus

#133 lore raw
Quoted 16 lines · Linus Torvalds
Quoted 8 lines · Jens Axboe

For the chain elements - yes, definitely! But we also want to clear dma mapping output values, at least sparc64 wants that. You could argue that the IOMMU code should be fixed up, but I don't think we should mix the two.

So we need the memset() in blk_rq_map_sg() AS WELL AS the initial sg table clear. I'm not arguing about the latter, we clearly do need that.

I still don't see your argument.

The SG table is *already*zeroed*.

Quoted 8 lines

The fact that output values will be changed later is irrelevant. They haven't been changed *before* - or at least you haven't given a reasonable explanation for why they should have.

So explain to me why the memset() in blk_rq_map_sg() helps, considering that the memory must have been zeroed at allocation time anyway? Tell me what it is that fills any of the fields we don't already initialize, and that can happen *before* that memset?

Ah ok, I see why you are confused. The SCSI case is one, it allocs and frees the sg table each time. The entries are thus always initialized when they end up in blk_rq_map_sg(). However, other drivers allocate one at driver init time and use that one all the time. It needs to be initially cleared, but it wont be cleared before each request mapping. And that is fine, as long as we do clear entries in blk_rq_map_sg().

Signature · 1 line
Jens Axboe
#134 lore raw
Quoted 7 lines · Jens Axboe

Ah ok, I see why you are confused. The SCSI case is one, it allocs and frees the sg table each time. The entries are thus always initialized when they end up in blk_rq_map_sg(). However, other drivers allocate one at driver init time and use that one all the time. It needs to be initially cleared, but it wont be cleared before each request mapping. And that is fine, as long as we do clear entries in blk_rq_map_sg().

Ahh, ok, thanks for clearing that up.

Linus

#135 lore raw
In reply to Linus Torvalds
Quoted 9 lines · Linus Torvalds

But I think the rest of your changes are simply bad.

The fix to block/ll_rw_block.c should likely be something like the appended instead:

- remove the "memset()" you had added earlier. It's bogus. It cannot be the right thing. If the sg list wasn't initialized correctly much earlier, trying to initialize it late is pointless - it contains crap.

Quoted 5 lines

- the old code was fine, but let's initialize "sg" to NULL to make it clear that the initial value of sg is pointless, and only "next_sg" matters (since sg had better be assigned from that).

Hmm?

built and booted your patch (removed Jens's) but it still crashes in blk_rq_map_sg() - see below. Should i have left something from Jens's patch?

Ingo

------------------->

[   34.698199] VFS: Mounted root (ext3 filesystem) readonly.
[   34.703917] Freeing unused kernel memory: 372k freed
[   34.746609] BUG: unable to handle kernel paging request at virtual address 6e616872
[   34.754106] printing eip: 7840503b *pde = 00000000 
[   34.758960] Oops: 0000 [#1] DEBUG_PAGEALLOC
[   34.763120] 
[   34.764594] Pid: 1, comm: swapper Not tainted (2.6.23 #7)
[   34.769965] EIP: 0060:[<7840503b>] EFLAGS: 00010006 CPU: 0
[   34.775429] EIP is at blk_rq_map_sg+0xbb/0x170
[   34.779845] EAX: 3ffcd000 EBX: 7c88a68c ECX: 3ffce000 EDX: 007ff9a0
[   34.786085] ESI: 6e616862 EDI: 798ea9a0 EBP: 00001000 ESP: 7b421bf4
[   34.792324]  DS: 007b ES: 007b FS: 0000 GS: 0000 SS: 0068
[   34.797697] Process swapper (pid: 1, ti=7b420000 task=7b416000 task.ti=7b420000)
[   34.804890] Stack: 00000046 7b520000 00002000 3ffcf000 7c88ac80 00000001 00000001 00000001 
[   34.813209]        7c88a600 01000000 7c88ab00 7b521d1c 7c88a708 7b520000 784c7ac5 7b520000 
[   34.821529]        784c75ca 7c88a708 7b524ce4 7b521d1c 7c88a708 784e4570 7b416000 7813661b 
[   34.829847] Call Trace:
[   34.832448]  [<784c7ac5>] scsi_init_io+0x55/0xe0
[   34.837042]  [<784c75ca>] scsi_get_cmd_from_req+0x2a/0x40
[   34.842414]  [<784e4570>] sd_prep_fn+0x80/0x940
[   34.846920]  [<7813661b>] __lock_acquire+0x4ab/0xe20
[   34.851860]  [<781450a6>] add_to_page_cache+0x66/0xb0
[   34.856887]  [<78404633>] elv_dispatch_sort+0x23/0xe0
[   34.861912]  [<784041a0>] elv_next_request+0xa0/0x130
[   34.866938]  [<784c8c04>] scsi_request_fn+0x1e4/0x370
[   34.871965]  [<78120f02>] del_timer+0x62/0x70
[   34.876298]  [<78407445>] __generic_unplug_device+0x25/0x30
[   34.881844]  [<78407715>] generic_unplug_device+0x15/0x30
[   34.887217]  [<78404e00>] blk_backing_dev_unplug+0x0/0x10
[   34.892590]  [<78404e0c>] blk_backing_dev_unplug+0xc/0x10
[   34.897963]  [<78180a9d>] block_sync_page+0x2d/0x40
[   34.902817]  [<78144dc9>] sync_page+0x29/0x40
[   34.907150]  [<7876f5bc>] __wait_on_bit_lock+0x3c/0x70
[   34.912263]  [<78144da0>] sync_page+0x0/0x40
[   34.916509]  [<78144d82>] __lock_page+0x52/0x60
[   34.921016]  [<7812adb0>] wake_bit_function+0x0/0x60
[   34.925955]  [<7814554c>] do_generic_mapping_read+0x21c/0x450
[   34.931675]  [<78144b10>] file_read_actor+0x0/0x130
[   34.936528]  [<78147107>] generic_file_aio_read+0x137/0x180
[   34.942074]  [<78144b10>] file_read_actor+0x0/0x130
[   34.946927]  [<78161245>] do_sync_read+0xd5/0x120
[   34.951607]  [<78135646>] mark_lock+0x76/0x550
[   34.956027]  [<7812ad70>] autoremove_wake_function+0x0/0x40
[   34.961573]  [<78155026>] __vma_link+0x36/0x70
[   34.965993]  [<78161170>] do_sync_read+0x0/0x120
[   34.970586]  [<78161ae3>] vfs_read+0xb3/0x110
[   34.974919]  [<781652cd>] kernel_read+0x3d/0x60
[   34.979425]  [<7816539f>] prepare_binprm+0xaf/0xe0
[   34.984192]  [<78166973>] do_execve+0x123/0x1c0
[   34.988698]  [<78100f1f>] sys_execve+0x2f/0x90
[   34.993118]  [<781028a2>] syscall_call+0x7/0xb
[   34.997538]  [<78106f1c>] kernel_execve+0x1c/0x30
[   35.002217]  [<7810016e>] init_post+0x9e/0xe0
[   35.006550]  [<78a147b3>] kernel_init+0x163/0x280
[   35.011230]  [<78a14650>] kernel_init+0x0/0x280
[   35.015736]  [<78103a97>] kernel_thread_helper+0x7/0x10
[   35.020936]  =======================
[   35.024489] Code: 29 c1 c1 f9 05 c1 e1 0c 03 4a 08 8b 52 04 01 ca 89 54 24 0c 8b 3b 89 fa 29 c2 89 d0 c1 f8 05 c1 e0 0c 03 43 08 39 44 24 0c 74 7e <8b> 46 10 8d 56 10 a8 01 75 4c 89 3e 89 6e 0c 8b 43 08 89 46 04 
[   35.043294] EIP: [<7840503b>] blk_rq_map_sg+0xbb/0x170 SS:ESP 0068:7b421bf4
[   35.050228] Kernel panic - not syncing: Attempted to kill init!
#136 lore raw
Quoted 4 lines · Ingo Molnar

built and booted your patch (removed Jens's) but it still crashes in blk_rq_map_sg() - see below. Should i have left something from Jens's patch?

The *real* fix in Jens' patch (and the only thing that really matters) is the one to drivers/scsi/scsi_lib.c.

The rest is all fluff.

Linus

#137 lore raw
In reply to Jens Axboe

Il Wed, Oct 17, 2007 at 07:29:32PM +0200, Jens Axboe ha scritto:

Quoted 53 lines

OK, it is fine, as long as the sglist is cleared initially. And I don't think there's anyway around that, clearly I didn't think long enough before including the memset() removal from Tomo.

Ingo, please try this rolled up version.

Linus, this should work. It would probably be best if you first did a git revert on f5c0dde4c66421a3a2d7d6fa604a712c9b0744e5 and then applied the ll_rw_blk.c bit alone. Do you want me to stuff that (revert + patch) into a branch for you to pull?

block/ll_rw_blk.c +6 -4
index 9e3f3cc..3935469 100644
@@ -1322,8 +1322,8 @@ int blk_rq_map_sg(struct request_queue *q, struct request *rq,
 		  struct scatterlist *sglist)
 {
 	struct bio_vec *bvec, *bvprv;
-	struct scatterlist *next_sg, *sg;
 	struct req_iterator iter;
+	struct scatterlist *sg;
 	int nsegs, cluster;
 
 	nsegs = 0;
@@ -1333,7 +1333,7 @@ int blk_rq_map_sg(struct request_queue *q, struct request *rq,
 	 * for each bio in rq
 	 */
 	bvprv = NULL;
-	sg = next_sg = &sglist[0];
+	sg = NULL;
 	rq_for_each_segment(bvec, rq, iter) {
 		int nbytes = bvec->bv_len;
 
@@ -1349,8 +1349,10 @@ int blk_rq_map_sg(struct request_queue *q, struct request *rq,
 			sg->length += nbytes;
 		} else {
 new_segment:
-			sg = next_sg;
-			next_sg = sg_next(sg);
+			if (!sg)
+				sg = sglist;
+			else
+				sg = sg_next(sg);
 
 			memset(sg, 0, sizeof(*sg));
 			sg->page = bvec->bv_page;
drivers/scsi/scsi_lib.c +0 -0
index 0c86be7..aac8a02 100644

drivers/scsi/scsi_lib.c struct scatterlist *scsi_alloc_sgtable(struct scsi_cmnd *cmd, gfp_t gfp_mask) · line 771

Quoted 8 lines
drivers/scsi/scsi_lib.c +2 -0
@@ -764,6 +764,8 @@ struct scatterlist *scsi_alloc_sgtable(struct scsi_cmnd *cmd, gfp_t gfp_mask)
 		if (unlikely(!sgl))
 			goto enomem;
 
+		memset(sgl, 0, sizeof(*sgl) * sgp->size);
+
 		/*
 		 * first loop through, set initial index and return value
 		 */

Hi Jens,
I just hit what I believe is the same bug, though with a slightly different OOPS (I see a NULL pointer dereference, see below). The patches fixes the bug, and - as Linus suggested - the piece touching scsi_lib.c is enough.

The OOPS:

scsi4 : ata_piix
scsi5 : ata_piix
ata5: PATA max UDMA/100 cmd 0x1f0 ctl 0x3f6 bmdma 0xffa0 irq 14
ata6: PATA max UDMA/100 cmd 0x170 ctl 0x376 bmdma 0xffa8 irq 15
sda1 sda2 sda3 < sda5 sda6 >
sd 0:0:0:0: [sda] Attached SCSI disk
ata5.00: ATAPI: MATSHITADVD-RAM UJ-850S, 1.22, max UDMA/33

Unable to handle kernel NULL pointer dereference at 0000000000000020 RIP: 
 [<ffffffff802d275e>] blk_rq_map_sg+0xf6/0x16a
PGD 4711067 PUD 508c067 PMD 0 
Oops: 0000 [1] PREEMPT SMP 
CPU 0 
Modules linked in: sd_mod ohci1394 ehci_hcd ata_piix ieee1394 ahci uhci_hcd libata scsi_mod usbcore thermal processor fan
Pid: 0, comm: swapper Not tainted 2.6.23-ge6d5a11d #3
RIP: 0010:[<ffffffff802d275e>]  [<ffffffff802d275e>] blk_rq_map_sg+0xf6/0x16a
RSP: 0018:ffffffff804eccd8  EFLAGS: 00010087
RAX: 0000000005331000 RBX: ffff810004740220 RCX: ffff810001000000
RDX: 0000000000000000 RSI: ffff8100052d4f50 RDI: 0000000005142c00
RBP: 0000000000001400 R08: 0000000000000000 R09: ffff8100052a3980
R10: 0000000005143000 R11: 0000000000000400 R12: ffff8100047a4000
R13: 0000000000000000 R14: 0000000000000002 R15: 0000000000000000
FS:  0000000000000000(0000) GS:ffffffff8049a000(0000) knlGS:0000000000000000
CS:  0010 DS: 0018 ES: 0018 CR0: 000000008005003b
CR2: 0000000000000020 CR3: 0000000005118000 CR4: 00000000000006e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Process swapper (pid: 0, threadinfo ffffffff804ac000, task ffffffff8046c340)
Stack:  ffffffff88041834 000000010314ba00 ffff81000477fa80 ffff81000503dd80
 ffff81000503dd80 ffff81000517fc00 000000000000001e 0000000000001d4c
 ffffffff8804193c 0000000015925849 ffff81000473f800 ffff81000503dd80
Call Trace:
 <IRQ>  [<ffffffff88041834>] :scsi_mod:scsi_alloc_sgtable+0xad/0x13c
 [<ffffffff8804193c>] :scsi_mod:scsi_init_io+0x79/0xd9
 [<ffffffff880d1328>] :sd_mod:sd_prep_fn+0x59/0x367
 [<ffffffff802d07ff>] elv_next_request+0xc0/0x14a
 [<ffffffff88042367>] :scsi_mod:scsi_request_fn+0x88/0x356
 [<ffffffff802d46f7>] blk_run_queue+0x41/0x72
 [<ffffffff88041135>] :scsi_mod:scsi_next_command+0x2d/0x39
 [<ffffffff880412ae>] :scsi_mod:scsi_end_request+0xbb/0xc9
 [<ffffffff880415ac>] :scsi_mod:scsi_io_completion+0xf8/0x2d3
 [<ffffffff802d4a83>] blk_done_softirq+0x62/0x6f
 [<ffffffff80238f28>] __do_softirq+0x59/0xc3
 [<ffffffff8020d13c>] call_softirq+0x1c/0x28
 [<ffffffff8020ec26>] do_softirq+0x2c/0x7d
 [<ffffffff80238e30>] irq_exit+0x36/0x81
 [<ffffffff8020edb4>] do_IRQ+0x13d/0x161
 [<ffffffff8020c491>] ret_from_intr+0x0/0xa
 <EOI>  [<ffffffff88005249>] :processor:acpi_processor_idle+0x2cb/0x496
 [<ffffffff88005245>] :processor:acpi_processor_idle+0x2c7/0x496
 [<ffffffff88004f7e>] :processor:acpi_processor_idle+0x0/0x496
 [<ffffffff8020ad00>] default_idle+0x0/0x3d
 [<ffffffff8020adcd>] cpu_idle+0x90/0xcc
 [<ffffffff804b4a60>] start_kernel+0x2ab/0x2b7
 [<ffffffff804b411b>] _sinittext+0x11b/0x122


Code: 49 8b 40 20 4d 8d 50 20 4c 89 c7 fc b9 08 00 00 00 4c 89 c3 
RIP  [<ffffffff802d275e>] blk_rq_map_sg+0xf6/0x16a
 RSP <ffffffff804eccd8>
CR2: 0000000000000020
Kernel panic - not syncing: Aiee, killing interrupt handler!

Luca

Signature · 3 lines
The trouble with computers is that they do what you tell them, 
not what you want.
D. Cohen
#138 lore raw
In reply to Jens Axboe
Quoted 7 lines · Jens Axboe
Quoted 7 lines
block/ll_rw_blk.c +4 -2
-			sg = next_sg;
-			next_sg = sg_next(sg);
+			if (!sg)
+				sg = sglist;
+			else
+				sg = sg_next(sg);
Quoted 5 lines

block/ll_rw_blk.c int blk_rq_map_sg(struct request_queue *q, struct request *rq, · line 1358

Quoted 3 lines
block/ll_rw_blk.c +0 -0
 			memset(sg, 0, sizeof(*sg));
 			sg->page = bvec->bv_page;

Scratch that, it cannot work... I'll think up a different approach.

too late, crashed my box with it already :-)

Ingo

#139 lore raw
Quoted 10 lines · Ingo Molnar
Quoted 7 lines · Jens Axboe
Quoted 7 lines
block/ll_rw_blk.c +4 -2
-			sg = next_sg;
-			next_sg = sg_next(sg);
+			if (!sg)
+				sg = sglist;
+			else
+				sg = sg_next(sg);
Quoted 7 lines
Quoted 5 lines

block/ll_rw_blk.c int blk_rq_map_sg(struct request_queue *q, struct request *rq, · line 1358

Quoted 3 lines
block/ll_rw_blk.c +0 -0
 			memset(sg, 0, sizeof(*sg));
 			sg->page = bvec->bv_page;

Scratch that, it cannot work... I'll think up a different approach.

too late, crashed my box with it already :-)

Sorry about that, please try the next one that includes the scsi_lib.c one liner to clear the sg table on alloc :-)

Signature · 1 line
Jens Axboe
#140 lore raw
In reply to Jens Axboe
Quoted 2 lines · Jens Axboe

[...] It's always safe to zero sg, since it's a valid entry - nothing to save in ->page. Ingo, does this work for you?

with that patch it not crashes on NULL dereference - see crashlog below. Compiler bug perhaps?

Ingo

---------------->

[   34.605614] EXT3-fs: INFO: recovery required on readonly filesystem.
[   34.611842] EXT3-fs: write access will be enabled during recovery.
[   34.635861] BUG: unable to handle kernel NULL pointer dereference at virtual address 00000008
[   34.644227] printing eip: 7840840e *pde = 00000000 
[   34.649081] Oops: 0002 [#1] DEBUG_PAGEALLOC
[   34.653239] 
[   34.654713] Pid: 1, comm: swapper Not tainted (2.6.23 #3)
[   34.660086] EIP: 0060:[<7840840e>] EFLAGS: 00010046 CPU: 0
[   34.665548] EIP is at blk_rq_map_sg+0x8e/0x190
[   34.669965] EAX: 00000000 EBX: 7c885180 ECX: 00000004 EDX: 033b6000
[   34.676205] ESI: 00001000 EDI: 00000008 EBP: 00000000 ESP: 7b4219b8
[   34.682444]  DS: 007b ES: 007b FS: 0000 GS: 0000 SS: 0068
[   34.687818] Process swapper (pid: 1, ti=7b420000 task=7b416000 task.ti=7b420000)
[   34.695010] Stack: 7b521d38 00000008 7c884000 7b520000 00002000 033b6000 7c885080 00000001 
[   34.703329]        00000001 7c885880 01000002 7c886e00 7b521d1c 7c8857c4 7b520000 784c7ae5 
[   34.711649]        7b520000 784c75ea 7c8857c4 7b524ce4 7b521d1c 7c8857c4 784e4590 7b416000 
[   34.719968] Call Trace:
[   34.722568]  [<784c7ae5>] scsi_init_io+0x55/0xe0
[   34.727161]  [<784c75ea>] scsi_get_cmd_from_req+0x2a/0x40
[   34.732534]  [<784e4590>] sd_prep_fn+0x80/0x940
[   34.737041]  [<7813661b>] __lock_acquire+0x4ab/0xe20
[   34.741981]  [<78135c9c>] trace_hardirqs_on+0x9c/0xb0
[   34.747007]  [<78770e30>] _spin_unlock_irq+0x20/0x30
[   34.751946]  [<78404633>] elv_dispatch_sort+0x23/0xe0
[   34.756973]  [<784041a0>] elv_next_request+0xa0/0x130
[   34.761999]  [<784c8c24>] scsi_request_fn+0x1e4/0x370
[   34.767025]  [<78120f02>] del_timer+0x62/0x70
[   34.771358]  [<784072d5>] __generic_unplug_device+0x25/0x30
[   34.776905]  [<784075a5>] generic_unplug_device+0x15/0x30
[   34.782278]  [<78404e0c>] blk_backing_dev_unplug+0xc/0x10
[   34.787650]  [<78181e76>] sync_buffer+0x26/0x40
[   34.792157]  [<7876f6c2>] __wait_on_bit+0x42/0x70
[   34.796836]  [<78181e50>] sync_buffer+0x0/0x40
[   34.801256]  [<78181e50>] sync_buffer+0x0/0x40
[   34.805676]  [<7876f74a>] out_of_line_wait_on_bit+0x5a/0x70
[   34.811223]  [<7812adb0>] wake_bit_function+0x0/0x60
[   34.816163]  [<78181db4>] __wait_on_buffer+0x24/0x30
[   34.821102]  [<78207a27>] jread+0x1b7/0x250
[   34.825262]  [<78207bcd>] do_one_pass+0x10d/0x600
[   34.829942]  [<78135c9c>] trace_hardirqs_on+0x9c/0xb0
[   34.834968]  [<7820823b>] journal_recover+0x9b/0x1a0
[   34.839908]  [<7820b811>] journal_load+0x51/0xf0
[   34.844501]  [<781e6914>] ext3_fill_super+0xdd4/0x1850
[   34.849613]  [<78411cef>] snprintf+0x1f/0x30
[   34.853860]  [<7819ce70>] disk_name+0xb0/0xc0
[   34.858193]  [<78163809>] get_sb_bdev+0x109/0x130
[   34.862873]  [<7815f300>] __kmalloc_node+0x80/0x90
[   34.867639]  [<781e3a80>] ext3_get_sb+0x20/0x30
[   34.872146]  [<781e5b40>] ext3_fill_super+0x0/0x1850
[   34.877085]  [<78163315>] vfs_kern_mount+0xb5/0x130
[   34.881938]  [<781633ed>] do_kern_mount+0x3d/0xe0
[   34.886618]  [<78177c57>] do_mount+0x5e7/0x710
[   34.891038]  [<78770fe5>] _spin_unlock_irqrestore+0x55/0x70
[   34.896584]  [<78135c9c>] trace_hardirqs_on+0x9c/0xb0
[   34.901610]  [<78110935>] change_page_attr+0x3d5/0x400
[   34.906724]  [<7811099a>] kernel_map_pages+0x3a/0x90
[   34.911663]  [<7814972f>] get_page_from_freelist+0x1ff/0x3e0
[   34.917296]  [<7814a2cf>] __alloc_pages+0x5f/0x380
[   34.922063]  [<7814a61e>] __get_free_pages+0x2e/0x50
[   34.927002]  [<78176660>] copy_mount_options+0x40/0x140
[   34.932202]  [<78177df2>] sys_mount+0x72/0xb0
[   34.936535]  [<78a14d39>] mount_block_root+0x89/0x260
[   34.941561]  [<7816b4c7>] sys_mknod+0x27/0x30
[   34.945894]  [<78a14f56>] mount_root+0x46/0x60
[   34.950314]  [<78a1501c>] prepare_namespace+0xac/0x170
[   34.955427]  [<7816000f>] sys_access+0x1f/0x30
[   34.959846]  [<78a147ae>] kernel_init+0x15e/0x280
[   34.964526]  [<78a14650>] kernel_init+0x0/0x280
[   34.969033]  [<78103a97>] kernel_thread_helper+0x7/0x10
[   34.974233]  =======================
[   34.977785] Code: 44 24 20 88 44 24 2b eb 54 8d 74 26 00 83 44 24 04 10 8b 7c 24 04 8b 07 a8 01 0f 85 fb 00 00 00 31 c0 b9 04 00 00 00 8b 7c 24 04 <f3> ab 8b 03 8b 54 24 04 89 02 89 72 0c 8b 43 08 89 42 04 83 44 
[   34.996590] EIP: [<7840840e>] blk_rq_map_sg+0x8e/0x190 SS:ESP 0068:7b4219b8
[   35.003525] Kernel panic - not syncing: Attempted to kill init!
#141 lore raw
In reply to Jens Axboe
Quoted 5 lines · Jens Axboe

OK, the below should actually be safe, I don't know why I talked myself into the next_sg stuff in the beginning. It's always safe to zero sg, since it's a valid entry - nothing to save in ->page. Ingo, does this work for you?

I really don't think this should work.

Doing "sg_next()" on a valid sg is *always* ok. So if the old code didn't work, then "sg" wasn't valid to start with (and the code *after* the sg_next() would have oopsed even if you try to avoid using sg_next.

So avoiding the "sg_next()" on the last entry is pointless.

Also, your patch makes the code almost totally unreadable, with that subtle issue of the "if (bvprv && cluster)" case not triggering on the first case, so the NULL initial sg is "safe".

So at a guess, I think the *real* problem is simply that the passed-in sglist was just too small. What guarantees that the sg list allocation (apparently done by scsi_alloc_sgtable()) is big enough?

If I read things right, scsi_alloc_sgtable() will allocate "cmd->use_sg" SG enties, no? But I also notice that it does not seem to initialize the SG allocation, so those SG entries contain random crap - including, perhaps, a random - and bogus - chain pointer in sg->page..

Yes, we set sh->page *if* we create a chain, but if we don't chain, we leave the old random contents around which in turn may include old and stale chain pointers. Or am I missing something?

So when you added that "memset(sg, 0, sizeof(*sg))" into blk_rq_map_sg(), you did it way too late - it needs to be done when the sg chain is allocated, and for every entry (and then the "link" entry needs to be linked in separately)

I think.

Linus

#142 lore raw
Quoted 9 lines · Linus Torvalds
Quoted 5 lines · Jens Axboe

OK, the below should actually be safe, I don't know why I talked myself into the next_sg stuff in the beginning. It's always safe to zero sg, since it's a valid entry - nothing to save in ->page. Ingo, does this work for you?

Quoted 7 lines

I really don't think this should work.

Doing "sg_next()" on a valid sg is *always* ok. So if the old code didn't work, then "sg" wasn't valid to start with (and the code *after* the sg_next() would have oopsed even if you try to avoid using sg_next.

So avoiding the "sg_next()" on the last entry is pointless.

Yeah, I didn't quite understand why if sg was valid, why dereferencing *(sg + 1)->page would crap out :/

Quoted 3 lines

Also, your patch makes the code almost totally unreadable, with that subtle issue of the "if (bvprv && cluster)" case not triggering on the first case, so the NULL initial sg is "safe".

Hmm I think it's quite readable, but perhaps that's just me :-). The first is much cleaner, and the last part just reads 'if sg is not set yet, set to list. otherwise, goto next entry'.

Quoted 8 lines

So at a guess, I think the *real* problem is simply that the passed-in sglist was just too small. What guarantees that the sg list allocation (apparently done by scsi_alloc_sgtable()) is big enough?

If I read things right, scsi_alloc_sgtable() will allocate "cmd->use_sg" SG enties, no? But I also notice that it does not seem to initialize the SG allocation, so those SG entries contain random crap - including, perhaps, a random - and bogus - chain pointer in sg->page..

Right, we allocate an sgtable that will hold ->use_sg entries, which contains request->nr_phys_segments. And that should definitely fit.

Regarding the init of the sglist, that was the revert I was talking about. We do need that memset() in there, so all those sg entries will be properly zeroed.

Quoted 10 lines

Yes, we set sh->page *if* we create a chain, but if we don't chain, we leave the old random contents around which in turn may include old and stale chain pointers. Or am I missing something?

So when you added that "memset(sg, 0, sizeof(*sg))" into blk_rq_map_sg(), you did it way too late - it needs to be done when the sg chain is allocated, and for every entry (and then the "link" entry needs to be linked in separately)

I think.

Yep, and that is what Ingo did test as well and it worked. For that case, now libata is crapping out elsewhere in sg_next().

Signature · 1 line
Jens Axboe
#143 lore raw
Quoted 5 lines · Jens Axboe
Quoted 2 lines

So avoiding the "sg_next()" on the last entry is pointless.

Yeah, I didn't quite understand why if sg was valid, why dereferencing *(sg + 1)->page would crap out :/

Actually, I take that back. If 'sg' is the last entry in a *non*linked scatter-gather list (ie we don't use the last entry as a link, we actually use it as a real SG entry), then "sg_next(sg)" will indeed access past the end of the whole allocated array, and will access one past the end.

And with page-alloc debugging, that *will* blow up.

So I think your change to use "sg_next()" only when you actually need a next pointer is the correct one after all.

Linus

#144 lore raw
Quoted 14 lines · Linus Torvalds
Quoted 5 lines · Jens Axboe
Quoted 2 lines

So avoiding the "sg_next()" on the last entry is pointless.

Yeah, I didn't quite understand why if sg was valid, why dereferencing *(sg + 1)->page would crap out :/

Actually, I take that back. If 'sg' is the last entry in a *non*linked scatter-gather list (ie we don't use the last entry as a link, we actually use it as a real SG entry), then "sg_next(sg)" will indeed access past the end of the whole allocated array, and will access one past the end.

Quoted 4 lines

And with page-alloc debugging, that *will* blow up.

So I think your change to use "sg_next()" only when you actually need a next pointer is the correct one after all.

Thanks, so I'm not totally crazy :-)

Can you just pull:

git://git.kernel.dk/data/git/linux-2.6-block.git for-linus

then so we get those two pieces correct? Then the remaining issue seems to be a new one that is biting Ingo elsewhere, at least we'll all be on the same page then.

Signature · 1 line
Jens Axboe
#145 lore raw
Quoted 18 lines · Jens Axboe
Quoted 10 lines · Linus Torvalds
Quoted 3 lines · Jens Axboe
Quoted 1 line

So avoiding the "sg_next()" on the last entry is pointless.

Yeah, I didn't quite understand why if sg was valid, why dereferencing *(sg + 1)->page would crap out :/

Actually, I take that back. If 'sg' is the last entry in a *non*linked scatter-gather list (ie we don't use the last entry as a link, we actually use it as a real SG entry), then "sg_next(sg)" will indeed access past the end of the whole allocated array, and will access one past the end.

Quoted 4 lines

And with page-alloc debugging, that *will* blow up.

So I think your change to use "sg_next()" only when you actually need a next pointer is the correct one after all.

Thanks, so I'm not totally crazy :-)

Quoted 8 lines

Can you just pull:

git://git.kernel.dk/data/git/linux-2.6-block.git for-linus

then so we get those two pieces correct? Then the remaining issue seems to be a new one that is biting Ingo elsewhere, at least we'll all be on the same page then.

Jens, for_each_sg still calls sg_next on the last entry which will dereference a possibly bogus sg->page (for the sg_is_chain(sg) condition in sg_next) if the last entry is the last one on the page of unchained entry and sg+1 falls over into an uninitialized page.

How about the following?
(untested yet.
sg.c included here as an example for usage out of scatterlist.h)

include/linux/scatterlist.h +19 -3
index 2dc7464..3a27e03 100644
@@ -30,7 +30,7 @@ static inline void sg_init_one(struct scatterlist *sg, const void *buf,
        ((struct scatterlist *) ((unsigned long) (sg)->page & ~0x01))
 
 /**
- * sg_next - return the next scatterlist entry in a list
+ * sg_next_unsafe - return the next scatterlist entry in a list
  * @sg:                The current sg entry
  *
  * Usually the next entry will be @sg@ + 1, but if this sg element is part
@@ -41,7 +41,7 @@ static inline void sg_init_one(struct scatterlist *sg, const void *buf,
  * the current entry, this function will NOT return NULL for an end-of-list.
  *
  */
-static inline struct scatterlist *sg_next(struct scatterlist *sg)
+static inline struct scatterlist *sg_next_unsafe(struct scatterlist *sg)
 {
        sg++;
 
@@ -51,11 +51,27 @@ static inline struct scatterlist *sg_next(struct scatterlist *sg)
        return sg;
 }
 
+/**
+ * sg_next - return the next scatterlist entry in a list
+ * @sg:                The current sg entry
+ * @next:      Index of next sg entry
+ * @nr:                Number of sg entries in the list
+ *
+ * Note that the caller must ensure that there are further entries after
+ * the current entry, this function will NOT return NULL for an end-of-list.
+ *
+ */
+static inline struct scatterlist *sg_next(struct scatterlist *sg,
+                                          int next, int nr)
+{
+       return next < nr ? sg_next_unsafe(sg) : NULL;
+}
+
 /*
  * Loop over each sg element, following the pointer to a new list if necessary
  */
 #define for_each_sg(sglist, sg, nr, __i)       \
-       for (__i = 0, sg = (sglist); __i < (nr); __i++, sg = sg_next(sg))
+       for (__i = 0, sg = (sglist); sg; sg = sg_next(sg, ++__i, nr))
 
 /**
  * sg_last - return the last scatterlist entry in a list
drivers/scsi/sg.c +9 -9
index 7238b2d..57cc1dd 100644
@@ -1165,7 +1165,7 @@ sg_vma_nopage(struct vm_area_struct *vma, unsigned long addr, int *type)
        sg = rsv_schp->buffer;
        sa = vma->vm_start;
        for (k = 0; (k < rsv_schp->k_use_sg) && (sa < vma->vm_end);
-            ++k, sg = sg_next(sg)) {
+            sg = sg_next(sg, ++k, rsv_schp->k_use_sg)) {
                len = vma->vm_end - sa;
                len = (len < sg->length) ? len : sg->length;
                if (offset < len) {
@@ -1209,7 +1209,7 @@ sg_mmap(struct file *filp, struct vm_area_struct *vma)
        sa = vma->vm_start;
        sg = rsv_schp->buffer;
        for (k = 0; (k < rsv_schp->k_use_sg) && (sa < vma->vm_end);
-            ++k, sg = sg_next(sg)) {
+            sg = sg_next(sg, ++k, rsv_schp->k_use_sg)) {
                len = vma->vm_end - sa;
                len = (len < sg->length) ? len : sg->length;
                sa += len;
@@ -1840,7 +1840,7 @@ sg_build_indirect(Sg_scatter_hold * schp, Sg_fd * sfp, int buff_size)
        }
        for (k = 0, sg = schp->buffer, rem_sz = blk_size;
             (rem_sz > 0) && (k < mx_sc_elems);
-            ++k, rem_sz -= ret_sz, sg = sg_next(sg)) {
+            rem_sz -= ret_sz, sg = sg_next(sg, ++k, mx_sc_elems)) {
 
                num = (rem_sz > scatter_elem_sz_prev) ?
                      scatter_elem_sz_prev : rem_sz;
@@ -1913,7 +1913,7 @@ sg_write_xfer(Sg_request * srp)
                if (res)
                        return res;
 
-               for (; p; sg = sg_next(sg), ksglen = sg->length,
+               for (; p; sg = sg_next_unsafe(sg), ksglen = sg->length,
                     p = page_address(sg->page)) {
                        if (usglen <= 0)
                                break;
@@ -1991,8 +1991,8 @@ sg_remove_scat(Sg_scatter_hold * schp)
                } else {
                        int k;
 
-                       for (k = 0; (k < schp->k_use_sg) && sg->page;
-                            ++k, sg = sg_next(sg)) {
+                       for (k = 0; sg && sg->page;
+                            sg = sg_next(sg, ++k, schp->k_use_sg)) {
                                SCSI_LOG_TIMEOUT(5, printk(
                                    "sg_remove_scat: k=%d, pg=0x%p, len=%d\n",
                                    k, sg->page, sg->length));
@@ -2045,7 +2045,7 @@ sg_read_xfer(Sg_request * srp)
                if (res)
                        return res;
 
-               for (; p; sg = sg_next(sg), ksglen = sg->length,
+               for (; p; sg = sg_next_unsafe(sg), ksglen = sg->length,
                     p = page_address(sg->page)) {
                        if (usglen <= 0)
                                break;
@@ -2092,7 +2092,7 @@ sg_read_oxfer(Sg_request * srp, char __user *outp, int num_read_xfer)
        if ((!outp) || (num_read_xfer <= 0))
                return 0;
 
-       for (k = 0; (k < schp->k_use_sg) && sg->page; ++k, sg = sg_next(sg)) {
+       for (k = 0; sg && sg->page; sg = sg_next(sg, ++k, schp->k_use_sg)) {
                num = sg->length;
                if (num > num_read_xfer) {
                        if (__copy_to_user(outp, page_address(sg->page),
@@ -2142,7 +2142,7 @@ sg_link_reserve(Sg_fd * sfp, Sg_request * srp, int size)
        SCSI_LOG_TIMEOUT(4, printk("sg_link_reserve: size=%d\n", size));
        rem = size;
 
-       for (k = 0; k < rsv_schp->k_use_sg; ++k, sg = sg_next(sg)) {
+       for (k = 0; sg; sg = sg_next(sg, ++k, rsv_schp->k_use_sg)) {
                num = sg->length;
                if (rem <= num) {
                        sfp->save_scat_len = num;
#146 lore raw
Quoted 28 lines · Benny Halevy
Quoted 18 lines · Jens Axboe
Quoted 10 lines · Linus Torvalds
Quoted 3 lines · Jens Axboe
Quoted 1 line

So avoiding the "sg_next()" on the last entry is pointless.

Yeah, I didn't quite understand why if sg was valid, why dereferencing *(sg + 1)->page would crap out :/

Actually, I take that back. If 'sg' is the last entry in a *non*linked scatter-gather list (ie we don't use the last entry as a link, we actually use it as a real SG entry), then "sg_next(sg)" will indeed access past the end of the whole allocated array, and will access one past the end.

Quoted 4 lines

And with page-alloc debugging, that *will* blow up.

So I think your change to use "sg_next()" only when you actually need a next pointer is the correct one after all.

Thanks, so I'm not totally crazy :-)

Quoted 8 lines

Can you just pull:

git://git.kernel.dk/data/git/linux-2.6-block.git for-linus

then so we get those two pieces correct? Then the remaining issue seems to be a new one that is biting Ingo elsewhere, at least we'll all be on the same page then.

Quoted 4 lines

Jens, for_each_sg still calls sg_next on the last entry which will dereference a possibly bogus sg->page (for the sg_is_chain(sg) condition in sg_next) if the last entry is the last one on the page of unchained entry and sg+1 falls over into an uninitialized page.

Things have progressed a lot since, see my recent posting based on Davem's proposal. Will post another patch soonish, that is also tested.

Signature · 1 line
Jens Axboe
maintainer
#147 lore raw

From: Jens Axboe <[email protected]>
Date: Thu, 18 Oct 2007 12:55:17 +0200

Quoted 3 lines

Things have progressed a lot since, see my recent posting based on Davem's proposal. Will post another patch soonish, that is also tested.

One core issue here is that we need to decide whether this thing to be iterated like an array or like a linked list. It's trying to be both.

If we decide upon a looping construct for consumers and stick to it, we'll be in much better shape than we are now and bugs will be eaiser to spot. It would be so much simpler to audit if all we saw in the consumers were things like:

	while (sg) {
		do_stuff(sg);
		sg = sg_next(sg);
	}

I would suggest that we just get it over with and convert the whole tree now rather than trying to do this kind of thing in stages. Because then we can say that ever scatterlist creator has to set the "end" bit and therefore you use well established patterns for scatterlist iteration such as "traverse sg_next() until NULL" as shown above.

I also noticed that there is the issue of on-stack and embedded scatterlist users. We'll need some sort of "DECLARE_SCATTERLIST" and a "scatterlist_init()" thing so that we can keep DEBUG_SG working even in those cases. But for all I know Jens could be working on that already :-)

The only other real option if we don't convert the whole tree now to the "end" marker stuff, is to enforce that every scatterlist iterator only traverse the number of entries there were told are in the one given to them.

#148 lore raw
Quoted 20 lines · David Miller

From: Jens Axboe <[email protected]>
Date: Thu, 18 Oct 2007 12:55:17 +0200

Quoted 3 lines

Things have progressed a lot since, see my recent posting based on Davem's proposal. Will post another patch soonish, that is also tested.

One core issue here is that we need to decide whether this thing to be iterated like an array or like a linked list. It's trying to be both.

If we decide upon a looping construct for consumers and stick to it, we'll be in much better shape than we are now and bugs will be eaiser to spot. It would be so much simpler to audit if all we saw in the consumers were things like:

	while (sg) {
		do_stuff(sg);
		sg = sg_next(sg);
	}
Quoted 6 lines

I would suggest that we just get it over with and convert the whole tree now rather than trying to do this kind of thing in stages. Because then we can say that ever scatterlist creator has to set the "end" bit and therefore you use well established patterns for scatterlist iteration such as "traverse sg_next() until NULL" as shown above.

The above should work now, PROVIDED that the end has been marked properly. I have added some helpers to init an sglist (or sg entry). You can still use

        for_each_sg()

and pass in the number of entries, that'll work even with an end marker. So, it does both.

Quoted 5 lines

I also noticed that there is the issue of on-stack and embedded scatterlist users. We'll need some sort of "DECLARE_SCATTERLIST" and a "scatterlist_init()" thing so that we can keep DEBUG_SG working even in those cases. But for all I know Jens could be working on that already :-)

Heh, got some of it covered! The stack usage is generally just converted to either use sg_init_one() if we can, or sg_init_table(). Adding some soft of DECLARE_SCATTERLIST() sounds like a good idea, though. I'll take patches :-)

Quoted 4 lines

The only other real option if we don't convert the whole tree now to the "end" marker stuff, is to enforce that every scatterlist iterator only traverse the number of entries there were told are in the one given to them.

Driver that do all their own sg management typically still use a manual loop over the number of entries they know are there. I didn't change those, and the original sg chaining didn't convert those either. Some manual labor is required in utilizing chained sg lists, unless you are part of a subsystem (like SCSI) that allocates and inits themf or you. So I still think that baby steps are a good idea - only convert things that matter. Leave the rest to janitors or others willing to dive in.

Signature · 1 line
Jens Axboe
#149 lore raw
In reply to Linus Torvalds
Quoted 3 lines · Linus Torvalds

So I think your change to use "sg_next()" only when you actually need a next pointer is the correct one after all.

That still leaves the initialization issue. The link pointers need to all be initialized at SG allocation time (and not just the last one for the case where it's a linked entry).

And if you initialize them at allocation time, does an allocation ever get re-used without being free'd in between (if it does, you do indeed need to initialize the non-link pointers each time - but I don't see it being the case, so ..)

Linus

#150 lore raw
Quoted 7 lines · Linus Torvalds
Quoted 3 lines · Linus Torvalds

So I think your change to use "sg_next()" only when you actually need a next pointer is the correct one after all.

Quoted 8 lines

That still leaves the initialization issue. The link pointers need to all be initialized at SG allocation time (and not just the last one for the case where it's a linked entry).

And if you initialize them at allocation time, does an allocation ever get re-used without being free'd in between (if it does, you do indeed need to initialize the non-link pointers each time - but I don't see it being the case, so ..)

The links will not change, so reuse should actually be OK. The reuse cannot be larger than the original would hold, but that would be a bug without chaining as well of course.

WRT reuse, if we end up requeuing a request, than we will go through the whole thing again. I don't think Ingo is hitting that either, though.

Signature · 1 line
Jens Axboe
#151 lore raw
In reply to Jens Axboe
Quoted 15 lines · Jens Axboe
Quoted 9 lines

and the oopsing instruction is that load of "sg->page" in the assembly code:

	mov    0x10(%esi),%eax		# %eax = sg->page
	lea    0x10(%esi),%edx		# %edx = sg+1;
	test   $0x1,%al			# if (unlikely(sg_is_chain()))
	jne    +76

Jens?

Yep, that's what I came up with as well - I asked Ingo for a dump in private, but ended up just using ksymoops to decode the line.

The way blk_rq_map_sg() operates is that it ends up doing a

Quoted 7 lines
        next_sg = sg_next(sg);

even though sg may be the last entry. Perhaps this is crapping out, although if sg is a valid address, then sg + 1 should be as well. next_sg may end up being crap, in fact it will, but we'll never use that unless there are more entries to fill. And if there is, then both sg and next_sg were valid.

find below the disassembled code. Here's the faulting source line according to gdb:

(gdb) list *0x78406669
0x78406669 is in blk_rq_map_sg (include/linux/scatterlist.h:48).
43 */
44 static inline struct scatterlist *sg_next(struct scatterlist *sg)
45 {
46 sg++;
47
48 if (unlikely(sg_is_chain(sg)))
49 sg = sg_chain_ptr(sg);
50
51 return sg;
52 }

(gdb) list *0x78406673
0x78406673 is in blk_rq_map_sg (block/ll_rw_blk.c:1355).
1350 } else {
1351 new_segment:
1352 sg = next_sg;
1353 next_sg = sg_next(sg);
1354

1355                            sg->page = bvec->bv_page;
1356                            sg->length = nbytes;
1357                            sg->offset = bvec->bv_offset;
1358                            nsegs++;

1359 }
(gdb)

the compiler is gcc-4.2.2. (vanilla, built from sources)

Ingo

784065b0 <blk_rq_map_sg>:

784065b0:	55                   	push   %ebp
784065b1:	57                   	push   %edi
784065b2:	56                   	push   %esi
784065b3:	53                   	push   %ebx
784065b4:	83 ec 28             	sub    $0x28,%esp
784065b7:	89 44 24 04          	mov    %eax,0x4(%esp)
784065bb:	8b 98 08 01 00 00    	mov    0x108(%eax),%ebx
784065c1:	83 e3 01             	and    $0x1,%ebx
784065c4:	89 5c 24 18          	mov    %ebx,0x18(%esp)
784065c8:	8b 52 3c             	mov    0x3c(%edx),%edx
784065cb:	c7 44 24 14 00 00 00 	movl   $0x0,0x14(%esp)
784065d2:	00 
784065d3:	85 d2                	test   %edx,%edx
784065d5:	89 54 24 20          	mov    %edx,0x20(%esp)
784065d9:	0f 84 fc 00 00 00    	je     784066db <blk_rq_map_sg+0x12b>
784065df:	89 ce                	mov    %ecx,%esi
784065e1:	31 d2                	xor    %edx,%edx
784065e3:	89 4c 24 10          	mov    %ecx,0x10(%esp)
784065e7:	8b 44 24 20          	mov    0x20(%esp),%eax
784065eb:	0f b7 58 1a          	movzwl 0x1a(%eax),%ebx
784065ef:	8b 48 30             	mov    0x30(%eax),%ecx
784065f2:	89 5c 24 1c          	mov    %ebx,0x1c(%esp)
784065f6:	0f b7 40 18          	movzwl 0x18(%eax),%eax
784065fa:	39 d8                	cmp    %ebx,%eax
784065fc:	0f 8e c6 00 00 00    	jle    784066c8 <blk_rq_map_sg+0x118>
78406602:	8d 04 5b             	lea    (%ebx,%ebx,2),%eax
78406605:	8d 1c 81             	lea    (%ecx,%eax,4),%ebx
78406608:	0f b6 44 24 18       	movzbl 0x18(%esp),%eax
7840660d:	88 44 24 27          	mov    %al,0x27(%esp)
78406611:	e9 8b 00 00 00       	jmp    784066a1 <blk_rq_map_sg+0xf1>
78406616:	8b 4c 24 10          	mov    0x10(%esp),%ecx
7840661a:	8b 41 0c             	mov    0xc(%ecx),%eax
7840661d:	8b 4c 24 04          	mov    0x4(%esp),%ecx
78406621:	01 e8                	add    %ebp,%eax
78406623:	89 44 24 08          	mov    %eax,0x8(%esp)
78406627:	3b 81 6c 01 00 00    	cmp    0x16c(%ecx),%eax
7840662d:	0f 87 80 00 00 00    	ja     784066b3 <blk_rq_map_sg+0x103>
78406633:	a1 18 ec d7 78       	mov    0x78d7ec18,%eax
78406638:	8b 0a                	mov    (%edx),%ecx
7840663a:	29 c1                	sub    %eax,%ecx
7840663c:	c1 f9 05             	sar    $0x5,%ecx
7840663f:	c1 e1 0c             	shl    $0xc,%ecx
78406642:	03 4a 08             	add    0x8(%edx),%ecx
78406645:	8b 52 04             	mov    0x4(%edx),%edx
78406648:	01 ca                	add    %ecx,%edx
7840664a:	89 54 24 0c          	mov    %edx,0xc(%esp)
7840664e:	8b 3b                	mov    (%ebx),%edi
78406650:	89 fa                	mov    %edi,%edx
78406652:	29 c2                	sub    %eax,%edx
78406654:	89 d0                	mov    %edx,%eax
78406656:	c1 f8 05             	sar    $0x5,%eax
78406659:	c1 e0 0c             	shl    $0xc,%eax
7840665c:	03 43 08             	add    0x8(%ebx),%eax
7840665f:	39 44 24 0c          	cmp    %eax,0xc(%esp)
78406663:	0f 84 7e 00 00 00    	je     784066e7 <blk_rq_map_sg+0x137>
78406669:	8b 46 10             	mov    0x10(%esi),%eax
7840666c:	8d 56 10             	lea    0x10(%esi),%edx
7840666f:	a8 01                	test   $0x1,%al
78406671:	75 4c                	jne    784066bf <blk_rq_map_sg+0x10f>
78406673:	89 3e                	mov    %edi,(%esi)
78406675:	89 6e 0c             	mov    %ebp,0xc(%esi)
78406678:	8b 43 08             	mov    0x8(%ebx),%eax
7840667b:	89 46 04             	mov    %eax,0x4(%esi)
7840667e:	83 44 24 14 01       	addl   $0x1,0x14(%esp)
78406683:	89 74 24 10          	mov    %esi,0x10(%esp)
78406687:	89 d6                	mov    %edx,%esi
78406689:	8b 54 24 20          	mov    0x20(%esp),%edx
7840668d:	83 44 24 1c 01       	addl   $0x1,0x1c(%esp)
78406692:	0f b7 42 18          	movzwl 0x18(%edx),%eax
78406696:	3b 44 24 1c          	cmp    0x1c(%esp),%eax
7840669a:	7e 2a                	jle    784066c6 <blk_rq_map_sg+0x116>
7840669c:	89 da                	mov    %ebx,%edx
7840669e:	83 c3 0c             	add    $0xc,%ebx
784066a1:	85 d2                	test   %edx,%edx
784066a3:	8b 6b 04             	mov    0x4(%ebx),%ebp
784066a6:	74 0b                	je     784066b3 <blk_rq_map_sg+0x103>
784066a8:	80 7c 24 27 00       	cmpb   $0x0,0x27(%esp)
784066ad:	0f 85 63 ff ff ff    	jne    78406616 <blk_rq_map_sg+0x66>
784066b3:	8b 46 10             	mov    0x10(%esi),%eax
784066b6:	8d 56 10             	lea    0x10(%esi),%edx
784066b9:	8b 3b                	mov    (%ebx),%edi
784066bb:	a8 01                	test   $0x1,%al
784066bd:	74 b4                	je     78406673 <blk_rq_map_sg+0xc3>
784066bf:	89 c2                	mov    %eax,%edx
784066c1:	83 e2 fe             	and    $0xfffffffe,%edx
784066c4:	eb ad                	jmp    78406673 <blk_rq_map_sg+0xc3>
784066c6:	89 da                	mov    %ebx,%edx
784066c8:	8b 4c 24 20          	mov    0x20(%esp),%ecx
784066cc:	8b 49 08             	mov    0x8(%ecx),%ecx
784066cf:	85 c9                	test   %ecx,%ecx
784066d1:	89 4c 24 20          	mov    %ecx,0x20(%esp)
784066d5:	0f 85 0c ff ff ff    	jne    784065e7 <blk_rq_map_sg+0x37>
784066db:	8b 44 24 14          	mov    0x14(%esp),%eax
784066df:	83 c4 28             	add    $0x28,%esp
784066e2:	5b                   	pop    %ebx
784066e3:	5e                   	pop    %esi
784066e4:	5f                   	pop    %edi
784066e5:	5d                   	pop    %ebp
784066e6:	c3                   	ret    
784066e7:	8b 44 24 04          	mov    0x4(%esp),%eax
784066eb:	8b 54 24 0c          	mov    0xc(%esp),%edx
784066ef:	8b 80 70 01 00 00    	mov    0x170(%eax),%eax
784066f5:	89 04 24             	mov    %eax,(%esp)
784066f8:	09 c1                	or     %eax,%ecx
784066fa:	8d 44 2a ff          	lea    0xffffffff(%edx,%ebp,1),%eax
784066fe:	0b 04 24             	or     (%esp),%eax
78406701:	39 c1                	cmp    %eax,%ecx
78406703:	0f 85 60 ff ff ff    	jne    78406669 <blk_rq_map_sg+0xb9>
78406709:	8b 44 24 08          	mov    0x8(%esp),%eax
7840670d:	8b 4c 24 10          	mov    0x10(%esp),%ecx
78406711:	89 41 0c             	mov    %eax,0xc(%ecx)
78406714:	e9 70 ff ff ff       	jmp    78406689 <blk_rq_map_sg+0xd9>

78406719: 8d b4 26 00 00 00 00 lea 0x0(%esi),%esi