Skip to contentSkip to conversation

Address review comments for fnic

PATCH New Abhijeet Joglekar 1 participant 4 messages Mar 25, 2009

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover Address review comments for fnic

The following series addresses Mike C.'s review comments for fnic:
1) Use static workQ item per fnic for link transitions
2) Use static workQ item and skb queue per fnic for FCS frames
3) Use shost_printk to print host_no instead of printing fnic_no

(1) and (2) address the comment that allocating memory in the critical path for link transitions and FCS frames (fabric login etc) is not good since failure in memory allocation can prove fatal. The patches get rid of the event structure and its allocation, and instead use static workQ items.

---

Abhijeet Joglekar (3):
fnic: Use shost_printk and print host no in print messages
fnic: Change FCS frame handling to use skb queues and no events
fnic: Change link transition handling

 drivers/scsi/fnic/fnic.h      |   45 +++-------
 drivers/scsi/fnic/fnic_fcs.c  |  155 +++++++++++++++++++--------------
 drivers/scsi/fnic/fnic_isr.c  |   15 ++-
 drivers/scsi/fnic/fnic_main.c |  186 +++++++++++++++-------------------------
 drivers/scsi/fnic/fnic_res.c  |   80 ++++++++++-------
 drivers/scsi/fnic/fnic_scsi.c |  192 ++++++++++++++++++++---------------------
 6 files changed, 322 insertions(+), 351 deletions(-)
Signature · 1 line
Signature
#1 lore raw

PATCH 1/3 fnic: Change link transition handling

Events were used to wrap a link transition (up or down) and an embedded workQ item would then be posted for fnic workQ thread to pass the event to libFC. This required allocation of event, which if it fails, was not handled.

This fix removes the event wrapper and event allocation. Each fnic structure has an embedded workQ item, and the link ISR simply queues the workQ item. The workQ link handler then decides based on link down count and previous and current link status, how to process the link transition. This does not require the allocation of events and is safer in the face of memory pressure.

---
 drivers/scsi/fnic/fnic.h      |    5 ++
 drivers/scsi/fnic/fnic_fcs.c  |   89 ++++++++++++++++++++++++++++-------------
 drivers/scsi/fnic/fnic_main.c |   29 +------------
 3 files changed, 68 insertions(+), 55 deletions(-)
drivers/scsi/fnic/fnic.h +5 -0
index e76fb35..8a057cd 100644
@@ -188,6 +188,8 @@ struct fnic {
 	u64 fcp_input_bytes;		/* internal statistic */
 	u64 fcp_output_bytes;		/* internal statistic */
 	int event_count;                /* number of events queued in workq */
+	u32 link_down_cnt;
+	int link_status;
 
 	struct list_head list;
 	struct pci_dev *pdev;
@@ -206,6 +208,8 @@ struct fnic {
 	mempool_t *io_sgl_pool[FNIC_SGL_NUM_CACHES];
 	spinlock_t io_req_lock[FNIC_IO_LOCKS];	/* locks for scsi cmnds */
 
+	struct work_struct link_work;
+
 	/* copy work queue cache line section */
 	____cacheline_aligned struct vnic_wq_copy wq_copy[FNIC_WQ_COPY_MAX];
 	/* completion queue cache line section */
@@ -254,6 +258,7 @@ int fnic_request_intr(struct fnic *fnic);
 int fnic_send(struct fc_lport *, struct fc_frame *);
 void fnic_free_wq_buf(struct vnic_wq *wq, struct vnic_wq_buf *buf);
 void fnic_event_work(struct work_struct *work);
+void fnic_handle_link(struct work_struct *work);
 int fnic_rq_cmpl_handler(struct fnic *fnic, int);
 int fnic_alloc_rq_frame(struct vnic_rq *rq);
 void fnic_free_rq_buf(struct vnic_rq *rq, struct vnic_rq_buf *buf);
drivers/scsi/fnic/fnic_fcs.c +60 -29

110 lines not on this page · raw message

drivers/scsi/fnic/fnic_main.c +3 -26
index 97d2f32..11eecac 100644
@@ -237,8 +237,6 @@ void fnic_log_q_error(struct fnic *fnic)
 
 void fnic_handle_link_event(struct fnic *fnic)
 {
-	int link_status = vnic_dev_link_status(fnic->vdev);
-	struct fnic_event *event;
 	unsigned long flags;
 
 	spin_lock_irqsave(&fnic->fnic_lock, flags);
@@ -246,32 +244,9 @@ void fnic_handle_link_event(struct fnic *fnic)
 		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
 		return;
 	}
-
-	FNIC_MAIN_DBG(KERN_DEBUG, DFX "link %s\n", fnic->fnic_no,
-		      (link_status ? "up" : "down"));
-
-	event = kmem_cache_alloc(fnic_ev_cache, GFP_ATOMIC);
-	if (!event) {
-		FNIC_MAIN_DBG(KERN_DEBUG, DFX "Cannot allocate a event, "
-			      "cannot indicate link event to FCS\n",
-			      fnic->fnic_no);
-		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
-		return;
-	}
-
-	/* Queue the link event in fnic workQ */
-	memset(event, 0, sizeof(struct fnic_event));
-	event->fnic = fnic;
-	event->ev_type = EV_TYPE_LINK_UP;
-	if (!link_status) {
-		event->ev_type = EV_TYPE_LINK_DOWN;
-		fnic->lport->host_stats.link_failure_count++;
-	}
-	fnic->event_count++;
 	spin_unlock_irqrestore(&fnic->fnic_lock, flags);
 
-	INIT_WORK(&event->event_work, fnic_event_work);
-	queue_work(fnic_event_queue, &event->event_work);
+	queue_work(fnic_event_queue, &fnic->link_work);
 
 }
 
@@ -745,6 +720,8 @@ static int __devinit fnic_probe(struct pci_dev *pdev,
 	list_add_tail(&fnic->list, &fnic_list);
 	spin_unlock_irqrestore(&fnic_list_lock, flags);
 
+	INIT_WORK(&fnic->link_work, fnic_handle_link);
+
 	/* Enable all queues */
 	for (i = 0; i < fnic->raw_wq_count; i++)
 		vnic_wq_enable(&fnic->wq[i]);
#2 lore raw

PATCH 2/3 fnic: Change FCS frame handling to use skb queues and no events

In reply to Abhijeet Joglekar

Events were used to wrap incoming frames, and an embedded workQ item in the event would then get queued. The workQ handler would then invoke libFC to process the incoming FCS frame. However, this required allocation of event in the critical path, which if failed, could stall fabric login and fail.

This fix uses a per fnic skb queue and per fnic workQ item. Incoming frames are enqueued on the skb queue, and fnic's statically allocated workQ item is queued. The workQ frame handler loops through the skb queue, and forwards the frames to libFC. This avoids memory allocations in critical path.

---
 drivers/scsi/fnic/fnic.h      |   24 ++-------------
 drivers/scsi/fnic/fnic_fcs.c  |   66 ++++++++++++++++-------------------------
 drivers/scsi/fnic/fnic_main.c |   41 +++----------------------
 drivers/scsi/fnic/fnic_scsi.c |   28 ++++-------------
 4 files changed, 39 insertions(+), 120 deletions(-)
drivers/scsi/fnic/fnic.h +3 -21
index 8a057cd..13fa4c5 100644
@@ -187,7 +187,6 @@ struct fnic {
 	u8 data_src_addr[ETH_ALEN];
 	u64 fcp_input_bytes;		/* internal statistic */
 	u64 fcp_output_bytes;		/* internal statistic */
-	int event_count;                /* number of events queued in workq */
 	u32 link_down_cnt;
 	int link_status;
 
@@ -209,6 +208,8 @@ struct fnic {
 	spinlock_t io_req_lock[FNIC_IO_LOCKS];	/* locks for scsi cmnds */
 
 	struct work_struct link_work;
+	struct work_struct frame_work;
+	struct sk_buff_head frame_queue;
 
 	/* copy work queue cache line section */
 	____cacheline_aligned struct vnic_wq_copy wq_copy[FNIC_WQ_COPY_MAX];
@@ -228,26 +229,7 @@ struct fnic {
 	____cacheline_aligned struct vnic_intr intr[FNIC_MSIX_INTR_MAX];
 };
 
-/*
- * This is used to pass incoming frames, link notifications from ISR
- * to fnic workQ which passes it to libFC
- */
-enum fnic_thread_event_type {
-	EV_TYPE_LINK_DOWN = 0,
-	EV_TYPE_LINK_UP,
-	EV_TYPE_FRAME,
-};
-
-struct fnic_event {
-	struct fc_frame *fp;
-	struct fnic *fnic;
-	enum fnic_thread_event_type ev_type;
-	u32 is_flogi_resp_frame:1;
-	struct work_struct event_work;
-};
-
 extern struct workqueue_struct *fnic_event_queue;
-extern struct kmem_cache *fnic_ev_cache;
 extern struct device_attribute *fnic_attrs[];
 
 void fnic_clear_intr_mode(struct fnic *fnic);
@@ -257,7 +239,7 @@ int fnic_request_intr(struct fnic *fnic);
 
 int fnic_send(struct fc_lport *, struct fc_frame *);
 void fnic_free_wq_buf(struct vnic_wq *wq, struct vnic_wq_buf *buf);
-void fnic_event_work(struct work_struct *work);
+void fnic_handle_frame(struct work_struct *work);
 void fnic_handle_link(struct work_struct *work);
 int fnic_rq_cmpl_handler(struct fnic *fnic, int);
 int fnic_alloc_rq_frame(struct vnic_rq *rq);
drivers/scsi/fnic/fnic_fcs.c +25 -41

95 lines not on this page · raw message

drivers/scsi/fnic/fnic_main.c +5 -36

95 lines not on this page · raw message

drivers/scsi/fnic/fnic_scsi.c +6 -22
index 6db6359..0760abe 100644
@@ -543,10 +543,10 @@ static int fnic_fcpio_flogi_reg_cmpl_handler(struct fnic *fnic,
 	u8 type;
 	u8 hdr_status;
 	struct fcpio_tag tag;
-	struct fnic_event *event;
 	int ret = 0;
 	struct fc_frame *flogi_resp = NULL;
 	unsigned long flags;
+	struct sk_buff *skb;
 
 	fcpio_header_dec(&desc->hdr, &type, &hdr_status, &tag);
 
@@ -584,30 +584,14 @@ static int fnic_fcpio_flogi_reg_cmpl_handler(struct fnic *fnic,
 			spin_unlock_irqrestore(&fnic->fnic_lock, flags);
 			goto reg_cmpl_handler_end;
 		}
-
-		event = kmem_cache_alloc(fnic_ev_cache, GFP_ATOMIC);
-
-		if (!event) {
-			spin_unlock_irqrestore(&fnic->fnic_lock, flags);
-			dev_kfree_skb_irq(fp_skb(flogi_resp));
-			ret = -1;
-			goto reg_cmpl_handler_end;
-		}
-
-		memset(event, 0, sizeof(struct fnic_event));
-
-		/* initialize the event structure */
-		event->fp = flogi_resp;
+		skb = (struct sk_buff *)flogi_resp;
+		/* Use fr_flags to indicate whether flogi resp or not */
+		fr_flags(flogi_resp) = 1;
 		fr_dev(flogi_resp) = fnic->lport;
-		event->fnic = fnic;
-		event->ev_type = EV_TYPE_FRAME;
-		event->is_flogi_resp_frame = 1;
-		fnic->event_count++;
 		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
 
-		/* insert it into event queue */
-		INIT_WORK(&event->event_work, fnic_event_work);
-		queue_work(fnic_event_queue, &event->event_work);
+		skb_queue_tail(&fnic->frame_queue, skb);
+		queue_work(fnic_event_queue, &fnic->frame_work);
 
 	} else {
 		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
#3 lore raw

PATCH 3/3 fnic: Use shost_printk and print host no in print messages

In reply to Abhijeet Joglekar
---
 drivers/scsi/fnic/fnic.h      |   16 ++--
 drivers/scsi/fnic/fnic_fcs.c  |   38 +++++-----
 drivers/scsi/fnic/fnic_isr.c  |   15 ++--
 drivers/scsi/fnic/fnic_main.c |  116 ++++++++++++++++-------------
 drivers/scsi/fnic/fnic_res.c  |   80 ++++++++++++--------
 drivers/scsi/fnic/fnic_scsi.c |  164 ++++++++++++++++++++++-------------------
 6 files changed, 234 insertions(+), 195 deletions(-)
drivers/scsi/fnic/fnic.h +8 -8
index 13fa4c5..fc6d720 100644
@@ -94,21 +94,21 @@ do {								\
 		} while (0);					\
 } while (0)
 
-#define FNIC_MAIN_DBG(kern_level, fmt, args...)		\
+#define FNIC_MAIN_DBG(kern_level, host, fmt, args...)		\
 	FNIC_CHECK_LOGGING(FNIC_MAIN_LOGGING,			\
-			 printk(kern_level fmt, ##args);)
+			 shost_printk(kern_level, host, fmt, ##args);)
 
-#define FNIC_FCS_DBG(kern_level, fmt, args...)			\
+#define FNIC_FCS_DBG(kern_level, host, fmt, args...)		\
 	FNIC_CHECK_LOGGING(FNIC_FCS_LOGGING,			\
-			 printk(kern_level fmt, ##args);)
+			 shost_printk(kern_level, host, fmt, ##args);)
 
-#define FNIC_SCSI_DBG(kern_level, fmt, args...)			\
+#define FNIC_SCSI_DBG(kern_level, host, fmt, args...)		\
 	FNIC_CHECK_LOGGING(FNIC_SCSI_LOGGING,			\
-			 printk(kern_level fmt, ##args);)
+			 shost_printk(kern_level, host, fmt, ##args);)
 
-#define FNIC_ISR_DBG(kern_level, fmt, args...)			\
+#define FNIC_ISR_DBG(kern_level, host, fmt, args...)		\
 	FNIC_CHECK_LOGGING(FNIC_ISR_LOGGING,			\
-			 printk(kern_level fmt, ##args);)
+			 shost_printk(kern_level, host, fmt, ##args);)
 
 extern const char *fnic_state_str[];
 
drivers/scsi/fnic/fnic_fcs.c +21 -17
index 9233b8e..9cb15fe 100644
@@ -66,11 +66,11 @@ void fnic_handle_link(struct work_struct *work)
 				/* UP -> DOWN -> UP */
 				fnic->lport->host_stats.link_failure_count++;
 				spin_unlock_irqrestore(&fnic->fnic_lock, flags);
-				FNIC_FCS_DBG(KERN_DEBUG, DFX "link down\n",
-					     fnic->fnic_no);
+				FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host,
+					     "link down\n");
 				fc_linkdown(fnic->lport);
-				FNIC_FCS_DBG(KERN_DEBUG, DFX "link up\n",
-					     fnic->fnic_no);
+				FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host,
+					     "link up\n");
 				fc_linkup(fnic->lport);
 			} else
 				/* UP -> UP */
@@ -79,13 +79,13 @@ void fnic_handle_link(struct work_struct *work)
 	} else if (fnic->link_status) {
 		/* DOWN -> UP */
 		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
-		FNIC_FCS_DBG(KERN_DEBUG, DFX "link up\n", fnic->fnic_no);
+		FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, "link up\n");
 		fc_linkup(fnic->lport);
 	} else {
 		/* UP -> DOWN */
 		fnic->lport->host_stats.link_failure_count++;
 		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
-		FNIC_FCS_DBG(KERN_DEBUG, DFX "link down\n", fnic->fnic_no);
+		FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, "link down\n");
 		fc_linkdown(fnic->lport);
 	}
 
@@ -194,9 +194,10 @@ static inline int fnic_handle_flogi_resp(struct fnic *fnic,
 		 * issued by libFC might have changed the fnic cached oxid
 		 */
 		if (fnic->flogi_oxid != ntohs(fh->fh_ox_id)) {
-			FNIC_FCS_DBG(KERN_DEBUG, DFX "Flogi response oxid not"
+			FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host,
+				     "Flogi response oxid not"
 				     " matching cached oxid, dropping frame"
-				     "\n", fnic->fnic_no);
+				     "\n");
 			ret = -1;
 			spin_unlock_irqrestore(&fnic->fnic_lock, flags);
 			dev_kfree_skb_irq(fp_skb(fp));
@@ -238,7 +239,8 @@ static inline int fnic_handle_flogi_resp(struct fnic *fnic,
 		fnic->state = FNIC_IN_ETH_TRANS_FC_MODE;
 
 	} else {
-		FNIC_FCS_DBG(KERN_DEBUG, PFX "Unexpected fnic state %s while"
+		FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host,
+			     "Unexpected fnic state %s while"
 			     " processing flogi resp\n",
 			     fnic_state_to_str(fnic->state));
 		ret = -1;
@@ -358,16 +360,17 @@ static void fnic_rq_cmpl_frame_recv(struct vnic_rq *rq, struct cq_desc
 
 	} else {
 		/* wrong CQ type*/
-		printk(KERN_ERR DFX "fnic rq_cmpl wrong cq type x%x\n",
-		       fnic->fnic_no, type);
+		shost_printk(KERN_ERR, fnic->lport->host,
+			     "fnic rq_cmpl wrong cq type x%x\n", type);
 		goto drop;
 	}
 
 	if (!fcs_ok || packet_error || !fcoe_fc_crc_ok || fcoe_enc_error) {
-		FNIC_FCS_DBG(KERN_DEBUG, DFX "fnic rq_cmpl fcoe x%x fcsok x%x"
+		FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host,
+			     "fnic rq_cmpl fcoe x%x fcsok x%x"
 			     " pkterr x%x fcoe_fc_crc_ok x%x, fcoe_enc_err"
 			     " x%x\n",
-			     fnic->fnic_no, fcoe, fcs_ok, packet_error,
+			     fcoe, fcs_ok, packet_error,
 			     fcoe_fc_crc_ok, fcoe_enc_error);
 		goto drop;
 	}
@@ -438,8 +441,9 @@ int fnic_rq_cmpl_handler(struct fnic *fnic, int rq_work_to_do)
 		if (cur_work_done) {
 			err = vnic_rq_fill(&fnic->rq[i], fnic_alloc_rq_frame);
 			if (err)
-				printk(KERN_ERR DFX "fnic_alloc_rq_frame"
-				       " cant alloc frame\n", fnic->fnic_no);
+				shost_printk(KERN_ERR, fnic->lport->host,
+					     "fnic_alloc_rq_frame cant alloc"
+					     " frame\n");
 		}
 		tot_rq_work_done += cur_work_done;
 	}
@@ -462,8 +466,8 @@ int fnic_alloc_rq_frame(struct vnic_rq *rq)
 	len = FC_FRAME_HEADROOM + FC_MAX_FRAME + FC_FRAME_TAILROOM;
 	skb = dev_alloc_skb(len);
 	if (!skb) {
-		FNIC_FCS_DBG(KERN_DEBUG, DFX "Unable to allocate RQ sk_buff\n",
-			     fnic->fnic_no);
+		FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host,
+			     "Unable to allocate RQ sk_buff\n");
 		return -ENOMEM;
 	}
 	skb_reset_mac_header(skb);
drivers/scsi/fnic/fnic_isr.c +9 -6
index b4f4771..f5cff08 100644
@@ -202,8 +202,9 @@ int fnic_request_intr(struct fnic *fnic)
 					  fnic->msix[i].devname,
 					  fnic->msix[i].devid);
 			if (err) {
-				printk(KERN_ERR PFX "MSIX: request_irq"
-				       " failed %d\n", err);
+				shost_printk(KERN_ERR, fnic->lport->host,
+					     "MSIX: request_irq"
+					     " failed %d\n", err);
 				fnic_free_intr(fnic);
 				break;
 			}
@@ -253,8 +254,8 @@ int fnic_set_intr_mode(struct fnic *fnic)
 			fnic->intr_count = n + m + o + 1;
 			fnic->err_intr_offset = FNIC_MSIX_ERR_NOTIFY;
 
-			FNIC_ISR_DBG(KERN_DEBUG,
-				     PFX "Using MSI-X Interrupts\n");
+			FNIC_ISR_DBG(KERN_DEBUG, fnic->lport->host,
+				     "Using MSI-X Interrupts\n");
 			vnic_dev_set_intr_mode(fnic->vdev,
 					       VNIC_DEV_INTR_MODE_MSIX);
 			return 0;
@@ -280,7 +281,8 @@ int fnic_set_intr_mode(struct fnic *fnic)
 		fnic->intr_count = 1;
 		fnic->err_intr_offset = 0;
 
-		FNIC_ISR_DBG(KERN_DEBUG, PFX "Using MSI Interrupts\n");
+		FNIC_ISR_DBG(KERN_DEBUG, fnic->lport->host,
+			     "Using MSI Interrupts\n");
 		vnic_dev_set_intr_mode(fnic->vdev, VNIC_DEV_INTR_MODE_MSI);
 
 		return 0;
@@ -305,7 +307,8 @@ int fnic_set_intr_mode(struct fnic *fnic)
 		fnic->cq_count = 3;
 		fnic->intr_count = 3;
 
-		FNIC_ISR_DBG(KERN_DEBUG, PFX "Using Legacy Interrupts\n");
+		FNIC_ISR_DBG(KERN_DEBUG, fnic->lport->host,
+			     "Using Legacy Interrupts\n");
 		vnic_dev_set_intr_mode(fnic->vdev, VNIC_DEV_INTR_MODE_INTX);
 
 		return 0;
drivers/scsi/fnic/fnic_main.c +63 -53

254 lines not on this page · raw message

drivers/scsi/fnic/fnic_res.c +47 -33

111 lines not on this page · raw message

drivers/scsi/fnic/fnic_scsi.c +86 -78

359 lines not on this page · raw message