Skip to contentSkip to conversation

zfcp changes for v6.3

PATCH New Benjamin Block 2 participants 6 messages Feb 21, 2023 – Feb 22, 2023

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

Hello James, Martin,

here is a small set of changes for the zFCP device driver. These are basically follow-up changes I made after the fix I send in <979f6e6019d15f91ba56182f1aaf68d61bf37fc6.1668595505.git.bblock@linux.ibm.com> that refactor some related areas in the driver, and add some additional tracing if we ever should run into a similar situation somehow.

It would be nice, if you could still include them for v6.3. Not sure if I'm too late already.

Benjamin Block (3):
zfcp: make the type for accessing request hashtable buckets size_t
zfcp: change the type of all fsf request id fields and variables to

    u64

zfcp: trace when request remove fails after qdio send fails

 drivers/s390/scsi/zfcp_dbf.c     | 46 ++++++++++++++++++++++++++++++--
 drivers/s390/scsi/zfcp_def.h     |  6 ++---
 drivers/s390/scsi/zfcp_ext.h     |  5 +++-
 drivers/s390/scsi/zfcp_fsf.c     | 22 ++++++++-------
 drivers/s390/scsi/zfcp_qdio.h    |  2 +-
 drivers/s390/scsi/zfcp_reqlist.h | 26 +++++++++---------
 drivers/s390/scsi/zfcp_scsi.c    |  2 +-
 7 files changed, 80 insertions(+), 29 deletions(-)
Signature · 1 line
2.39.2
#1 lore raw

PATCH 1/3 zfcp: make the type for accessing request hashtable buckets size_t

The appropriate type for array indices is size_t and the current implementation in zfcp_reqlist.h mixes int and unsigned int in different places to access the hashtable buckets of our internal request hash table.

To prevent any confusion changed all places to size_t.

---
 drivers/s390/scsi/zfcp_reqlist.h | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)
drivers/s390/scsi/zfcp_reqlist.h +11 -9
index 9b8ff249e31c..f4bac61dfbd0 100644
@@ -5,14 +5,16 @@
  * Data structure and helper functions for tracking pending FSF
  * requests.
  *
- * Copyright IBM Corp. 2009, 2016
+ * Copyright IBM Corp. 2009, 2023
  */
 
 #ifndef ZFCP_REQLIST_H
 #define ZFCP_REQLIST_H
 
+#include <linux/types.h>
+
 /* number of hash buckets */
-#define ZFCP_REQ_LIST_BUCKETS 128
+#define ZFCP_REQ_LIST_BUCKETS 128u
 
 /**
  * struct zfcp_reqlist - Container for request list (reqlist)
@@ -24,7 +26,7 @@ struct zfcp_reqlist {
 	struct list_head buckets[ZFCP_REQ_LIST_BUCKETS];
 };
 
-static inline int zfcp_reqlist_hash(unsigned long req_id)
+static inline size_t zfcp_reqlist_hash(unsigned long req_id)
 {
 	return req_id % ZFCP_REQ_LIST_BUCKETS;
 }
@@ -37,7 +39,7 @@ static inline int zfcp_reqlist_hash(unsigned long req_id)
  */
 static inline struct zfcp_reqlist *zfcp_reqlist_alloc(void)
 {
-	unsigned int i;
+	size_t i;
 	struct zfcp_reqlist *rl;
 
 	rl = kzalloc(sizeof(struct zfcp_reqlist), GFP_KERNEL);
@@ -60,7 +62,7 @@ static inline struct zfcp_reqlist *zfcp_reqlist_alloc(void)
  */
 static inline int zfcp_reqlist_isempty(struct zfcp_reqlist *rl)
 {
-	unsigned int i;
+	size_t i;
 
 	for (i = 0; i < ZFCP_REQ_LIST_BUCKETS; i++)
 		if (!list_empty(&rl->buckets[i]))
@@ -84,7 +86,7 @@ static inline struct zfcp_fsf_req *
 _zfcp_reqlist_find(struct zfcp_reqlist *rl, unsigned long req_id)
 {
 	struct zfcp_fsf_req *req;
-	unsigned int i;
+	size_t i;
 
 	i = zfcp_reqlist_hash(req_id);
 	list_for_each_entry(req, &rl->buckets[i], list)
@@ -154,7 +156,7 @@ zfcp_reqlist_find_rm(struct zfcp_reqlist *rl, unsigned long req_id)
 static inline void zfcp_reqlist_add(struct zfcp_reqlist *rl,
 				    struct zfcp_fsf_req *req)
 {
-	unsigned int i;
+	size_t i;
 	unsigned long flags;
 
 	i = zfcp_reqlist_hash(req->req_id);
@@ -172,7 +174,7 @@ static inline void zfcp_reqlist_add(struct zfcp_reqlist *rl,
 static inline void zfcp_reqlist_move(struct zfcp_reqlist *rl,
 				     struct list_head *list)
 {
-	unsigned int i;
+	size_t i;
 	unsigned long flags;
 
 	spin_lock_irqsave(&rl->lock, flags);
@@ -200,7 +202,7 @@ zfcp_reqlist_apply_for_all(struct zfcp_reqlist *rl,
 {
 	struct zfcp_fsf_req *req;
 	unsigned long flags;
-	unsigned int i;
+	size_t i;
 
 	spin_lock_irqsave(&rl->lock, flags);
 	for (i = 0; i < ZFCP_REQ_LIST_BUCKETS; i++)
Signature · 1 line
2.39.2
#2 lore raw

PATCH 2/3 zfcp: change the type of all fsf request id fields and variables to u64

In reply to Benjamin Block

We use different integer types throughout zfcp to store the FSF request ID and related values; some places use unsigned long and others u64. On s390x these are effectively the same type, but this might cause confusions and is generally inconsistent.

The specification for the used hardware specifies this value as 64 bit number, and ultimately we use this value to communicate with the hardware, so it makes sense to change the type of all these variables to u64 where we can.
The only exception being when we store it in the host_scribble
field of a struct scsi_cmnd; for this case we add a build time check to make sure they are compatible.

---
 drivers/s390/scsi/zfcp_dbf.c     |  2 +-
 drivers/s390/scsi/zfcp_def.h     |  6 +++---
 drivers/s390/scsi/zfcp_fsf.c     | 15 ++++++++-------
 drivers/s390/scsi/zfcp_qdio.h    |  2 +-
 drivers/s390/scsi/zfcp_reqlist.h |  8 ++++----
 drivers/s390/scsi/zfcp_scsi.c    |  2 +-
 6 files changed, 18 insertions(+), 17 deletions(-)
drivers/s390/scsi/zfcp_dbf.c +1 -1
index cbc3b62cd9e5..d28c55ae9f01 100644
@@ -649,7 +649,7 @@ void zfcp_dbf_scsi_common(char *tag, int level, struct scsi_device *sdev,
 		rec->scsi_id = sc->device->id;
 		rec->scsi_lun = (u32)sc->device->lun;
 		rec->scsi_lun_64_hi = (u32)(sc->device->lun >> 32);
-		rec->host_scribble = (unsigned long)sc->host_scribble;
+		rec->host_scribble = (u64)sc->host_scribble;
 
 		memcpy(rec->scsi_opcode, sc->cmnd,
 		       min_t(int, sc->cmd_len, ZFCP_DBF_SCSI_OPCODE));
drivers/s390/scsi/zfcp_def.h +3 -3
index 94de55304a02..6c761299a22f 100644
@@ -129,7 +129,7 @@ struct zfcp_erp_action {
 	struct scsi_device *sdev;
 	u32		status;	      /* recovery status */
 	enum zfcp_erp_steps	step;	/* active step of this erp action */
-	unsigned long		fsf_req_id;
+	u64			fsf_req_id;
 	struct timer_list timer;
 };
 
@@ -163,7 +163,7 @@ struct zfcp_adapter {
 	struct Scsi_Host	*scsi_host;	   /* Pointer to mid-layer */
 	struct list_head	port_list;	   /* remote port list */
 	rwlock_t		port_list_lock;    /* port list lock */
-	unsigned long		req_no;		   /* unique FSF req number */
+	u64			req_no;		   /* unique FSF req number */
 	struct zfcp_reqlist	*req_list;
 	u32			fsf_req_seq_no;	   /* FSF cmnd seq number */
 	rwlock_t		abort_lock;        /* Protects against SCSI
@@ -325,7 +325,7 @@ static inline u64 zfcp_scsi_dev_lun(struct scsi_device *sdev)
  */
 struct zfcp_fsf_req {
 	struct list_head	list;
-	unsigned long		req_id;
+	u64			req_id;
 	struct zfcp_adapter	*adapter;
 	struct zfcp_qdio_req	qdio_req;
 	struct completion	completion;
drivers/s390/scsi/zfcp_fsf.c +8 -7
index ab3ea529cca7..71eabcc26cbb 100644
@@ -884,7 +884,7 @@ static int zfcp_fsf_req_send(struct zfcp_fsf_req *req)
 	const bool is_srb = zfcp_fsf_req_is_status_read_buffer(req);
 	struct zfcp_adapter *adapter = req->adapter;
 	struct zfcp_qdio *qdio = adapter->qdio;
-	unsigned long req_id = req->req_id;
+	u64 req_id = req->req_id;
 
 	zfcp_reqlist_add(adapter->req_list, req);
 
@@ -1042,7 +1042,7 @@ struct zfcp_fsf_req *zfcp_fsf_abort_fcp_cmnd(struct scsi_cmnd *scmnd)
 	struct scsi_device *sdev = scmnd->device;
 	struct zfcp_scsi_dev *zfcp_sdev = sdev_to_zfcp(sdev);
 	struct zfcp_qdio *qdio = zfcp_sdev->port->adapter->qdio;
-	unsigned long old_req_id = (unsigned long) scmnd->host_scribble;
+	u64 old_req_id = (u64) scmnd->host_scribble;
 
 	spin_lock_irq(&qdio->req_q_lock);
 	if (zfcp_qdio_sbal_get(qdio))
@@ -1065,7 +1065,7 @@ struct zfcp_fsf_req *zfcp_fsf_abort_fcp_cmnd(struct scsi_cmnd *scmnd)
 	req->handler = zfcp_fsf_abort_fcp_command_handler;
 	req->qtcb->header.lun_handle = zfcp_sdev->lun_handle;
 	req->qtcb->header.port_handle = zfcp_sdev->port->handle;
-	req->qtcb->bottom.support.req_handle = (u64) old_req_id;
+	req->qtcb->bottom.support.req_handle = old_req_id;
 
 	zfcp_fsf_start_timer(req, ZFCP_FSF_SCSI_ER_TIMEOUT);
 	if (!zfcp_fsf_req_send(req)) {
@@ -1919,7 +1919,7 @@ int zfcp_fsf_open_wka_port(struct zfcp_fc_wka_port *wka_port)
 {
 	struct zfcp_qdio *qdio = wka_port->adapter->qdio;
 	struct zfcp_fsf_req *req;
-	unsigned long req_id = 0;
+	u64 req_id = 0;
 	int retval = -EIO;
 
 	spin_lock_irq(&qdio->req_q_lock);
@@ -1978,7 +1978,7 @@ int zfcp_fsf_close_wka_port(struct zfcp_fc_wka_port *wka_port)
 {
 	struct zfcp_qdio *qdio = wka_port->adapter->qdio;
 	struct zfcp_fsf_req *req;
-	unsigned long req_id = 0;
+	u64 req_id = 0;
 	int retval = -EIO;
 
 	spin_lock_irq(&qdio->req_q_lock);
@@ -2587,6 +2587,7 @@ int zfcp_fsf_fcp_cmnd(struct scsi_cmnd *scsi_cmnd)
 		goto out;
 	}
 
+	BUILD_BUG_ON(sizeof(scsi_cmnd->host_scribble) < sizeof(req->req_id));
 	scsi_cmnd->host_scribble = (unsigned char *) req->req_id;
 
 	io = &req->qtcb->bottom.io;
@@ -2732,7 +2733,7 @@ void zfcp_fsf_reqid_check(struct zfcp_qdio *qdio, int sbal_idx)
 	struct qdio_buffer *sbal = qdio->res_q[sbal_idx];
 	struct qdio_buffer_element *sbale;
 	struct zfcp_fsf_req *fsf_req;
-	unsigned long req_id;
+	u64 req_id;
 	int idx;
 
 	for (idx = 0; idx < QDIO_MAX_ELEMENTS_PER_BUFFER; idx++) {
@@ -2747,7 +2748,7 @@ void zfcp_fsf_reqid_check(struct zfcp_qdio *qdio, int sbal_idx)
 			 * corruption and must stop the machine immediately.
 			 */
 			zfcp_qdio_siosl(adapter);
-			panic("error: unknown req_id (%lx) on adapter %s.\n",
+			panic("error: unknown req_id (%llx) on adapter %s.\n",
 			      req_id, dev_name(&adapter->ccw_device->dev));
 		}
 
drivers/s390/scsi/zfcp_qdio.h +1 -1
index 390706867df3..90134d9b69a7 100644
@@ -115,7 +115,7 @@ zfcp_qdio_sbale_curr(struct zfcp_qdio *qdio, struct zfcp_qdio_req *q_req)
  */
 static inline
 void zfcp_qdio_req_init(struct zfcp_qdio *qdio, struct zfcp_qdio_req *q_req,
-			unsigned long req_id, u8 sbtype, void *data, u32 len)
+			u64 req_id, u8 sbtype, void *data, u32 len)
 {
 	struct qdio_buffer_element *sbale;
 	int count = min(atomic_read(&qdio->req_q_free),
drivers/s390/scsi/zfcp_reqlist.h +4 -4
index f4bac61dfbd0..59fbb1b128cb 100644
@@ -26,7 +26,7 @@ struct zfcp_reqlist {
 	struct list_head buckets[ZFCP_REQ_LIST_BUCKETS];
 };
 
-static inline size_t zfcp_reqlist_hash(unsigned long req_id)
+static inline size_t zfcp_reqlist_hash(u64 req_id)
 {
 	return req_id % ZFCP_REQ_LIST_BUCKETS;
 }
@@ -83,7 +83,7 @@ static inline void zfcp_reqlist_free(struct zfcp_reqlist *rl)
 }
 
 static inline struct zfcp_fsf_req *
-_zfcp_reqlist_find(struct zfcp_reqlist *rl, unsigned long req_id)
+_zfcp_reqlist_find(struct zfcp_reqlist *rl, u64 req_id)
 {
 	struct zfcp_fsf_req *req;
 	size_t i;
@@ -104,7 +104,7 @@ _zfcp_reqlist_find(struct zfcp_reqlist *rl, unsigned long req_id)
  * or NULL if there is no known FSF request with this id.
  */
 static inline struct zfcp_fsf_req *
-zfcp_reqlist_find(struct zfcp_reqlist *rl, unsigned long req_id)
+zfcp_reqlist_find(struct zfcp_reqlist *rl, u64 req_id)
 {
 	unsigned long flags;
 	struct zfcp_fsf_req *req;
@@ -129,7 +129,7 @@ zfcp_reqlist_find(struct zfcp_reqlist *rl, unsigned long req_id)
  * NULL if it has not been found.
  */
 static inline struct zfcp_fsf_req *
-zfcp_reqlist_find_rm(struct zfcp_reqlist *rl, unsigned long req_id)
+zfcp_reqlist_find_rm(struct zfcp_reqlist *rl, u64 req_id)
 {
 	unsigned long flags;
 	struct zfcp_fsf_req *req;
drivers/s390/scsi/zfcp_scsi.c +1 -1
index 526ac240d9fe..3dbf4b21d127 100644
@@ -170,7 +170,7 @@ static int zfcp_scsi_eh_abort_handler(struct scsi_cmnd *scpnt)
 		(struct zfcp_adapter *) scsi_host->hostdata[0];
 	struct zfcp_fsf_req *old_req, *abrt_req;
 	unsigned long flags;
-	unsigned long old_reqid = (unsigned long) scpnt->host_scribble;
+	u64 old_reqid = (u64) scpnt->host_scribble;
 	int retval = SUCCESS, ret;
 	int retry = 3;
 	char *dbf_tag;
Signature · 1 line
2.39.2
#3 lore raw

PATCH 3/3 zfcp: trace when request remove fails after qdio send fails

In reply to Benjamin Block

When we fail to send a FSF request in zfcp_fsf_req_send() when calling zfcp_qdio_send() we try to remove the request object from our internal hash table again to prevent keeping a stale memory reference. This removal might still - very much theoretically - fail.

To store some evidence of when this happens add a new trace record for this case; tag: fsrsrmf.
We reuse the ZFCP_DBF_HBA_RES trace ID for this, but mark all
fields other then the request ID with ~0, to make fairly obvious that these are invalid values. This faking has to be done because we don't have a valid request object at this point, and can not safely access any of the memory of the old object - we just failed to find it in our hash table, so it might be gone already.

Here is an example of a decoded trace record:

    Timestamp      : 2023-02-17-13:09:12:748140
    Area           : HBA
    Subarea        : 1
    Level          : -
    Exception      : 000003ff7ff500c2
    CPU ID         : 0011
    Caller         : 0x0
    Record ID      : 1
    Tag            : fsrsrmf
    Request ID     : 0x0000000080126ab6
    Request status : 0xffffffff
    FSF cmnd       : 0xffffffff
    FSF sequence no: 0xffffffff
    FSF issued     : 2042-09-18-01:53:47:370495
    FSF stat       : 0xffffffff
    FSF stat qual  : ffffffff ffffffff ffffffff ffffffff
    Prot stat      : 0xffffffff
    Prot stat qual : ffffffff ffffffff ffffffff ffffffff
    Port handle    : 0xffffffff
    LUN handle     : 0xffffffff

This provides at least some basic evidence that this event happened, and what object was affected.

---
 drivers/s390/scsi/zfcp_dbf.c | 44 +++++++++++++++++++++++++++++++++++-
 drivers/s390/scsi/zfcp_ext.h |  5 +++-
 drivers/s390/scsi/zfcp_fsf.c |  7 ++++--
 3 files changed, 52 insertions(+), 4 deletions(-)
drivers/s390/scsi/zfcp_dbf.c +43 -1
index d28c55ae9f01..d904625afd40 100644
@@ -4,7 +4,7 @@
  *
  * Debug traces for zfcp.
  *
- * Copyright IBM Corp. 2002, 2020
+ * Copyright IBM Corp. 2002, 2023
  */
 
 #define KMSG_COMPONENT "zfcp"
@@ -145,6 +145,48 @@ void zfcp_dbf_hba_fsf_fces(char *tag, const struct zfcp_fsf_req *req, u64 wwpn,
 	spin_unlock_irqrestore(&dbf->hba_lock, flags);
 }
 
+/**
+ * zfcp_dbf_hba_fsf_reqid - trace only the tag and a request ID
+ * @tag: tag documenting the source
+ * @level: trace level
+ * @adapter: adapter instance the request ID belongs to
+ * @req_id: the request ID to trace
+ */
+void zfcp_dbf_hba_fsf_reqid(const char *const tag, const int level,
+			    struct zfcp_adapter *const adapter,
+			    const u64 req_id)
+{
+	struct zfcp_dbf *const dbf = adapter->dbf;
+	struct zfcp_dbf_hba *const rec = &dbf->hba_buf;
+	struct zfcp_dbf_hba_res *const res = &rec->u.res;
+	unsigned long flags;
+
+	if (unlikely(!debug_level_enabled(dbf->hba, level)))
+		return;
+
+	spin_lock_irqsave(&dbf->hba_lock, flags);
+	memset(rec, 0, sizeof(*rec));
+
+	memcpy(rec->tag, tag, ZFCP_DBF_TAG_LEN);
+
+	rec->id = ZFCP_DBF_HBA_RES;
+	rec->fsf_req_id = req_id;
+	rec->fsf_req_status = ~0u;
+	rec->fsf_cmd = ~0u;
+	rec->fsf_seq_no = ~0u;
+
+	res->req_issued = ~0ull;
+	res->prot_status = ~0u;
+	memset(res->prot_status_qual, 0xff, sizeof(res->prot_status_qual));
+	res->fsf_status = ~0u;
+	memset(res->fsf_status_qual, 0xff, sizeof(res->fsf_status_qual));
+	res->port_handle = ~0u;
+	res->lun_handle = ~0u;
+
+	debug_event(dbf->hba, level, rec, sizeof(*rec));
+	spin_unlock_irqrestore(&dbf->hba_lock, flags);
+}
+
 /**
  * zfcp_dbf_hba_fsf_uss - trace event for an unsolicited status buffer
  * @tag: tag indicating which kind of unsolicited status has been received
drivers/s390/scsi/zfcp_ext.h +4 -1
index c302cbb18a55..9f5152b42b0e 100644
@@ -4,7 +4,7 @@
  *
  * External function declarations.
  *
- * Copyright IBM Corp. 2002, 2020
+ * Copyright IBM Corp. 2002, 2023
  */
 
 #ifndef ZFCP_EXT_H
@@ -46,6 +46,9 @@ extern void zfcp_dbf_hba_fsf_res(char *, int, struct zfcp_fsf_req *);
 extern void zfcp_dbf_hba_fsf_fces(char *tag, const struct zfcp_fsf_req *req,
 				  u64 wwpn, u32 fc_security_old,
 				  u32 fc_security_new);
+extern void zfcp_dbf_hba_fsf_reqid(const char *const tag, const int level,
+				   struct zfcp_adapter *const adapter,
+				   const u64 req_id);
 extern void zfcp_dbf_hba_bit_err(char *, struct zfcp_fsf_req *);
 extern void zfcp_dbf_hba_def_err(struct zfcp_adapter *, u64, u16, void **);
 extern void zfcp_dbf_san_req(char *, struct zfcp_fsf_req *, u32);
drivers/s390/scsi/zfcp_fsf.c +5 -2
index 71eabcc26cbb..ceed1b6f7cb6 100644
@@ -4,7 +4,7 @@
  *
  * Implementation of FSF commands.
  *
- * Copyright IBM Corp. 2002, 2020
+ * Copyright IBM Corp. 2002, 2023
  */
 
 #define KMSG_COMPONENT "zfcp"
@@ -892,8 +892,11 @@ static int zfcp_fsf_req_send(struct zfcp_fsf_req *req)
 	req->issued = get_tod_clock();
 	if (zfcp_qdio_send(qdio, &req->qdio_req)) {
 		del_timer_sync(&req->timer);
+
 		/* lookup request again, list might have changed */
-		zfcp_reqlist_find_rm(adapter->req_list, req_id);
+		if (zfcp_reqlist_find_rm(adapter->req_list, req_id) == NULL)
+			zfcp_dbf_hba_fsf_reqid("fsrsrmf", 1, adapter, req_id);
+
 		zfcp_erp_adapter_reopen(adapter, 0, "fsrs__1");
 		return -EIO;
 	}
Signature · 1 line
2.39.2
#4 lore raw
In reply to Benjamin Block

Benjamin,

Quoted 9 lines

here is a small set of changes for the zFCP device driver. These are basically follow-up changes I made after the fix I send in <979f6e6019d15f91ba56182f1aaf68d61bf37fc6.1668595505.git.bblock@linux.ibm.com> that refactor some related areas in the driver, and add some additional tracing if we ever should run into a similar situation somehow.

It would be nice, if you could still include them for v6.3. Not sure if I'm too late already.

It's a bit late. I merged them to 6.3/scsi-staging for now but may defer to 6.4 depending on how busy the various code checking robots are.

Signature · 1 line
Martin K. Petersen	Oracle Linux Engineering
#5 lore raw
Quoted 7 lines · Martin K. Petersen
Quoted 7 lines

here is a small set of changes for the zFCP device driver. These are basically follow-up changes I made after the fix I send in <979f6e6019d15f91ba56182f1aaf68d61bf37fc6.1668595505.git.bblock@linux.ibm.com> that refactor some related areas in the driver, and add some additional tracing if we ever should run into a similar situation somehow.

Quoted 6 lines
Quoted 2 lines

It would be nice, if you could still include them for v6.3. Not sure if I'm too late already.

It's a bit late. I merged them to 6.3/scsi-staging for now but may defer to 6.4 depending on how busy the various code checking robots are.

Alright, sounds good. Thanks.

Signature · 4 lines
Best Regards, Benjamin Block  / Linux on IBM Z Kernel Development / IBM Systems
IBM Deutschland Research & Development GmbH    /    https://www.ibm.com/privacy
Vorsitz. AufsR.: Gregor Pillen         /         Geschäftsführung: David Faller
Sitz der Gesellschaft: Böblingen / Registergericht: AmtsG Stuttgart, HRB 243294