zfcp changes for v6.3
Subsystems (3)
Hello James, Martin,
here is a small set of changes for the zFCP device driver. These are basically follow-up changes I made after the fix I send in <979f6e6019d15f91ba56182f1aaf68d61bf37fc6.1668595505.git.bblock@linux.ibm.com> that refactor some related areas in the driver, and add some additional tracing if we ever should run into a similar situation somehow.
It would be nice, if you could still include them for v6.3. Not sure if I'm too late already.
Benjamin Block (3):
zfcp: make the type for accessing request hashtable buckets size_t
zfcp: change the type of all fsf request id fields and variables to
u64
zfcp: trace when request remove fails after qdio send fails
drivers/s390/scsi/zfcp_dbf.c | 46 ++++++++++++++++++++++++++++++-- drivers/s390/scsi/zfcp_def.h | 6 ++--- drivers/s390/scsi/zfcp_ext.h | 5 +++- drivers/s390/scsi/zfcp_fsf.c | 22 ++++++++------- drivers/s390/scsi/zfcp_qdio.h | 2 +- drivers/s390/scsi/zfcp_reqlist.h | 26 +++++++++--------- drivers/s390/scsi/zfcp_scsi.c | 2 +- 7 files changed, 80 insertions(+), 29 deletions(-)
Signature · 1 line
2.39.2
PATCH 1/3 zfcp: make the type for accessing request hashtable buckets size_t
The appropriate type for array indices is size_t and the current implementation in zfcp_reqlist.h mixes int and unsigned int in different places to access the hashtable buckets of our internal request hash table.
To prevent any confusion changed all places to size_t.
--- drivers/s390/scsi/zfcp_reqlist.h | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-)
drivers/s390/scsi/zfcp_reqlist.h +11 -9
@@ -5,14 +5,16 @@* Data structure and helper functions for tracking pending FSF * requests. * -* Copyright IBM Corp. 2009, 2016 +* Copyright IBM Corp. 2009, 2023 */ #ifndef ZFCP_REQLIST_H #define ZFCP_REQLIST_H +#include <linux/types.h> +/* number of hash buckets */ -#define ZFCP_REQ_LIST_BUCKETS 128 +#define ZFCP_REQ_LIST_BUCKETS 128u/** * struct zfcp_reqlist - Container for request list (reqlist)
@@ -24,7 +26,7 @@ struct zfcp_reqlist {struct list_head buckets[ZFCP_REQ_LIST_BUCKETS ]; }; -static inline int zfcp_reqlist_hash (unsigned long req_id) +static inline size_t zfcp_reqlist_hash (unsigned long req_id) {return req_id %ZFCP_REQ_LIST_BUCKETS ; }
@@ -37,7 +39,7 @@ static inline int zfcp_reqlist_hash(unsigned long req_id)*/ static inline struct zfcp_reqlist *zfcp_reqlist_alloc (void ) { -unsigned int i; +size_t i;struct zfcp_reqlist *rl; rl =kzalloc (sizeof (struct zfcp_reqlist ),GFP_KERNEL );
@@ -60,7 +62,7 @@ static inline struct zfcp_reqlist *zfcp_reqlist_alloc(void)*/ static inline int zfcp_reqlist_isempty (struct zfcp_reqlist *rl) { -unsigned int i; +size_t i;for (i =0 ; i <ZFCP_REQ_LIST_BUCKETS ; i++)if (!list_empty (&rl->buckets[i]))
@@ -84,7 +86,7 @@ static inline struct zfcp_fsf_req * _zfcp_reqlist_find(struct zfcp_reqlist *rl, unsigned long req_id) {struct zfcp_fsf_req *req; -unsigned int i; +size_t i; i =zfcp_reqlist_hash (req_id);list_for_each_entry (req, &rl->buckets[i], list)
@@ -154,7 +156,7 @@ zfcp_reqlist_find_rm(struct zfcp_reqlist *rl, unsigned long req_id)static inline void zfcp_reqlist_add (struct zfcp_reqlist *rl,struct zfcp_fsf_req *req) { -unsigned int i; +size_t i;unsigned long flags; i =zfcp_reqlist_hash (req->req_id);
@@ -172,7 +174,7 @@ static inline void zfcp_reqlist_add(struct zfcp_reqlist *rl,static inline void zfcp_reqlist_move (struct zfcp_reqlist *rl,struct list_head *list) { -unsigned int i; +size_t i;unsigned long flags;spin_lock_irqsave (&rl->lock, flags);
@@ -200,7 +202,7 @@ zfcp_reqlist_apply_for_all(struct zfcp_reqlist *rl, {struct zfcp_fsf_req *req;unsigned long flags; -unsigned int i; +size_t i;spin_lock_irqsave (&rl->lock, flags);for (i =0 ; i <ZFCP_REQ_LIST_BUCKETS ; i++)
Signature · 1 line
2.39.2
PATCH 2/3 zfcp: change the type of all fsf request id fields and variables to u64
In reply to Benjamin BlockWe use different integer types throughout zfcp to store the FSF request ID and related values; some places use unsigned long and others u64. On s390x these are effectively the same type, but this might cause confusions and is generally inconsistent.
The specification for the used hardware specifies this value as 64 bit number, and ultimately we use this value to communicate with the hardware, so it makes sense to change the type of all these variables to u64 where we can.
The only exception being when we store it in the host_scribble
field of a struct scsi_cmnd; for this case we add a build time check to make sure they are compatible.
--- drivers/s390/scsi/zfcp_dbf.c | 2 +- drivers/s390/scsi/zfcp_def.h | 6 +++--- drivers/s390/scsi/zfcp_fsf.c | 15 ++++++++------- drivers/s390/scsi/zfcp_qdio.h | 2 +- drivers/s390/scsi/zfcp_reqlist.h | 8 ++++---- drivers/s390/scsi/zfcp_scsi.c | 2 +- 6 files changed, 18 insertions(+), 17 deletions(-)
drivers/s390/scsi/zfcp_dbf.c +1 -1
@@ -649,7 +649,7 @@ void zfcp_dbf_scsi_common(char *tag, int level, struct scsi_device *sdev, rec->scsi_id = sc->device->id; rec->scsi_lun = (u32 )sc->device->lun; rec->scsi_lun_64_hi = (u32)(sc->device->lun >>32 ); - rec->host_scribble = (unsigned long )sc->host_scribble; + rec->host_scribble = (u64 )sc->host_scribble;memcpy (rec->scsi_opcode, sc->cmnd,min_t (int, sc->cmd_len,ZFCP_DBF_SCSI_OPCODE ));
drivers/s390/scsi/zfcp_def.h +3 -3
@@ -129,7 +129,7 @@ struct zfcp_erp_action {struct scsi_device *sdev;u32 status;/* recovery status */ enum zfcp_erp_steps step;/* active step of this erp action */ -unsigned long fsf_req_id; +u64 fsf_req_id;struct timer_list timer; };
@@ -163,7 +163,7 @@ struct zfcp_adapter {struct Scsi_Host *scsi_host;/* Pointer to mid-layer */ struct list_head port_list;/* remote port list */ rwlock_t port_list_lock;/* port list lock */ -unsigned long req_no;/* unique FSF req number */ +u64 req_no;/* unique FSF req number */ struct zfcp_reqlist *req_list;u32 fsf_req_seq_no;/* FSF cmnd seq number */ rwlock_t abort_lock;/* Protects against SCSI
@@ -325,7 +325,7 @@ static inline u64 zfcp_scsi_dev_lun(struct scsi_device *sdev)*/ struct zfcp_fsf_req {struct list_head list; -unsigned long req_id; +u64 req_id;struct zfcp_adapter *adapter;struct zfcp_qdio_req qdio_req;struct completion completion;
drivers/s390/scsi/zfcp_fsf.c +8 -7
@@ -884,7 +884,7 @@ static int zfcp_fsf_req_send(struct zfcp_fsf_req *req)const bool is_srb =zfcp_fsf_req_is_status_read_buffer (req);struct zfcp_adapter *adapter = req->adapter;struct zfcp_qdio *qdio = adapter->qdio; -unsigned long req_id = req->req_id; +u64 req_id = req->req_id;zfcp_reqlist_add (adapter->req_list, req);
@@ -1042,7 +1042,7 @@ struct zfcp_fsf_req *zfcp_fsf_abort_fcp_cmnd(struct scsi_cmnd *scmnd)struct scsi_device *sdev = scmnd->device;struct zfcp_scsi_dev *zfcp_sdev =sdev_to_zfcp (sdev);struct zfcp_qdio *qdio = zfcp_sdev->port->adapter->qdio; -unsigned long old_req_id = (unsigned long ) scmnd->host_scribble; +u64 old_req_id = (u64 ) scmnd->host_scribble;spin_lock_irq (&qdio->req_q_lock);if (zfcp_qdio_sbal_get (qdio))
@@ -1065,7 +1065,7 @@ struct zfcp_fsf_req *zfcp_fsf_abort_fcp_cmnd(struct scsi_cmnd *scmnd) req->handler = zfcp_fsf_abort_fcp_command_handler; req->qtcb->header.lun_handle = zfcp_sdev->lun_handle; req->qtcb->header.port_handle = zfcp_sdev->port->handle; - req->qtcb->bottom.support.req_handle = (u64 ) old_req_id; + req->qtcb->bottom.support.req_handle = old_req_id;zfcp_fsf_start_timer (req,ZFCP_FSF_SCSI_ER_TIMEOUT );if (!zfcp_fsf_req_send (req)) {
@@ -1919,7 +1919,7 @@ int zfcp_fsf_open_wka_port(struct zfcp_fc_wka_port *wka_port) {struct zfcp_qdio *qdio = wka_port->adapter->qdio;struct zfcp_fsf_req *req; -unsigned long req_id =0 ; +u64 req_id =0 ;int retval = -EIO ;spin_lock_irq (&qdio->req_q_lock);
@@ -1978,7 +1978,7 @@ int zfcp_fsf_close_wka_port(struct zfcp_fc_wka_port *wka_port) {struct zfcp_qdio *qdio = wka_port->adapter->qdio;struct zfcp_fsf_req *req; -unsigned long req_id =0 ; +u64 req_id =0 ;int retval = -EIO ;spin_lock_irq (&qdio->req_q_lock);
@@ -2587,6 +2587,7 @@ int zfcp_fsf_fcp_cmnd(struct scsi_cmnd *scsi_cmnd) goto out; } +BUILD_BUG_ON (sizeof (scsi_cmnd->host_scribble) <sizeof (req->req_id)); scsi_cmnd->host_scribble = (unsigned char *) req->req_id; io = &req->qtcb->bottom.io;
@@ -2732,7 +2733,7 @@ void zfcp_fsf_reqid_check(struct zfcp_qdio *qdio, int sbal_idx)struct qdio_buffer *sbal = qdio->res_q[sbal_idx];struct qdio_buffer_element *sbale;struct zfcp_fsf_req *fsf_req; -unsigned long req_id; +u64 req_id;int idx;for (idx =0 ; idx <QDIO_MAX_ELEMENTS_PER_BUFFER ; idx++) {
@@ -2747,7 +2748,7 @@ void zfcp_fsf_reqid_check(struct zfcp_qdio *qdio, int sbal_idx)* corruption and must stop the machine immediately. */ zfcp_qdio_siosl (adapter); -panic ("error: unknown req_id (%lx) on adapter %s.\n" , +panic ("error: unknown req_id (%llx) on adapter %s.\n" , req_id,dev_name (&adapter->ccw_device->dev)); }
drivers/s390/scsi/zfcp_qdio.h +1 -1
@@ -115,7 +115,7 @@ zfcp_qdio_sbale_curr(struct zfcp_qdio *qdio, struct zfcp_qdio_req *q_req)*/ static inline void zfcp_qdio_req_init (struct zfcp_qdio *qdio,struct zfcp_qdio_req *q_req, -unsigned long req_id,u8 sbtype,void *data,u32 len) +u64 req_id,u8 sbtype,void *data,u32 len) {struct qdio_buffer_element *sbale; int count = min(atomic_read(&qdio->req_q_free),
drivers/s390/scsi/zfcp_reqlist.h +4 -4
@@ -26,7 +26,7 @@ struct zfcp_reqlist {struct list_head buckets[ZFCP_REQ_LIST_BUCKETS ]; }; -static inline size_t zfcp_reqlist_hash (unsigned long req_id) +static inline size_t zfcp_reqlist_hash (u64 req_id) {return req_id %ZFCP_REQ_LIST_BUCKETS ; }
@@ -83,7 +83,7 @@ static inline void zfcp_reqlist_free(struct zfcp_reqlist *rl) }static inline struct zfcp_fsf_req * -_zfcp_reqlist_find (struct zfcp_reqlist *rl,unsigned long req_id) +_zfcp_reqlist_find (struct zfcp_reqlist *rl,u64 req_id) {struct zfcp_fsf_req *req;size_t i;
@@ -104,7 +104,7 @@ _zfcp_reqlist_find(struct zfcp_reqlist *rl, unsigned long req_id)* or NULL if there is no known FSF request with this id. */ static inline struct zfcp_fsf_req * -zfcp_reqlist_find (struct zfcp_reqlist *rl,unsigned long req_id) +zfcp_reqlist_find (struct zfcp_reqlist *rl,u64 req_id) {unsigned long flags;struct zfcp_fsf_req *req;
@@ -129,7 +129,7 @@ zfcp_reqlist_find(struct zfcp_reqlist *rl, unsigned long req_id)* NULL if it has not been found. */ static inline struct zfcp_fsf_req * -zfcp_reqlist_find_rm (struct zfcp_reqlist *rl,unsigned long req_id) +zfcp_reqlist_find_rm (struct zfcp_reqlist *rl,u64 req_id) {unsigned long flags;struct zfcp_fsf_req *req;
drivers/s390/scsi/zfcp_scsi.c +1 -1
@@ -170,7 +170,7 @@ static int zfcp_scsi_eh_abort_handler(struct scsi_cmnd *scpnt) (struct zfcp_adapter *) scsi_host->hostdata[0 ];struct zfcp_fsf_req *old_req, *abrt_req;unsigned long flags; -unsigned long old_reqid = (unsigned long ) scpnt->host_scribble; +u64 old_reqid = (u64 ) scpnt->host_scribble;int retval =SUCCESS , ret;int retry =3 ;char *dbf_tag;
Signature · 1 line
2.39.2
PATCH 3/3 zfcp: trace when request remove fails after qdio send fails
In reply to Benjamin BlockWhen we fail to send a FSF request in zfcp_fsf_req_send() when calling zfcp_qdio_send() we try to remove the request object from our internal hash table again to prevent keeping a stale memory reference. This removal might still - very much theoretically - fail.
To store some evidence of when this happens add a new trace record for this case; tag: fsrsrmf.
We reuse the ZFCP_DBF_HBA_RES trace ID for this, but mark all
fields other then the request ID with ~0, to make fairly obvious that these are invalid values. This faking has to be done because we don't have a valid request object at this point, and can not safely access any of the memory of the old object - we just failed to find it in our hash table, so it might be gone already.
Here is an example of a decoded trace record:
Timestamp : 2023-02-17-13:09:12:748140
Area : HBA
Subarea : 1
Level : -
Exception : 000003ff7ff500c2
CPU ID : 0011
Caller : 0x0
Record ID : 1
Tag : fsrsrmf
Request ID : 0x0000000080126ab6
Request status : 0xffffffff
FSF cmnd : 0xffffffff
FSF sequence no: 0xffffffff
FSF issued : 2042-09-18-01:53:47:370495
FSF stat : 0xffffffff
FSF stat qual : ffffffff ffffffff ffffffff ffffffff
Prot stat : 0xffffffff
Prot stat qual : ffffffff ffffffff ffffffff ffffffff
Port handle : 0xffffffff
LUN handle : 0xffffffffThis provides at least some basic evidence that this event happened, and what object was affected.
--- drivers/s390/scsi/zfcp_dbf.c | 44 +++++++++++++++++++++++++++++++++++- drivers/s390/scsi/zfcp_ext.h | 5 +++- drivers/s390/scsi/zfcp_fsf.c | 7 ++++-- 3 files changed, 52 insertions(+), 4 deletions(-)
drivers/s390/scsi/zfcp_dbf.c +43 -1
@@ -4,7 +4,7 @@* * Debug traces for zfcp. * -* Copyright IBM Corp. 2002, 2020 +* Copyright IBM Corp. 2002, 2023 */ #define KMSG_COMPONENT "zfcp"
@@ -145,6 +145,48 @@ void zfcp_dbf_hba_fsf_fces(char *tag, const struct zfcp_fsf_req *req, u64 wwpn,spin_unlock_irqrestore (&dbf->hba_lock, flags); } +/** +* zfcp_dbf_hba_fsf_reqid - trace only the tag and a request ID +* @tag: tag documenting the source +* @level: trace level +* @adapter: adapter instance the request ID belongs to +* @req_id: the request ID to trace +*/ +void zfcp_dbf_hba_fsf_reqid (const char *const tag,const int level, +struct zfcp_adapter *const adapter, +const u64 req_id) +{ +struct zfcp_dbf *const dbf = adapter->dbf; +struct zfcp_dbf_hba *const rec = &dbf->hba_buf; +struct zfcp_dbf_hba_res *const res = &rec->u.res; +unsigned long flags; + +if (unlikely (!debug_level_enabled (dbf->hba, level))) +return ; + +spin_lock_irqsave (&dbf->hba_lock, flags); +memset (rec,0 ,sizeof (*rec)); + +memcpy (rec->tag, tag,ZFCP_DBF_TAG_LEN ); + + rec->id =ZFCP_DBF_HBA_RES ; + rec->fsf_req_id = req_id; + rec->fsf_req_status = ~0u ; + rec->fsf_cmd = ~0u ; + rec->fsf_seq_no = ~0u ; + + res->req_issued = ~0ull ; + res->prot_status = ~0u ; +memset (res->prot_status_qual,0xff ,sizeof (res->prot_status_qual)); + res->fsf_status = ~0u ; +memset (res->fsf_status_qual,0xff ,sizeof (res->fsf_status_qual)); + res->port_handle = ~0u ; + res->lun_handle = ~0u ; + +debug_event (dbf->hba, level, rec,sizeof (*rec)); +spin_unlock_irqrestore (&dbf->hba_lock, flags); +} +/** * zfcp_dbf_hba_fsf_uss - trace event for an unsolicited status buffer * @tag: tag indicating which kind of unsolicited status has been received
drivers/s390/scsi/zfcp_ext.h +4 -1
@@ -4,7 +4,7 @@* * External function declarations. * -* Copyright IBM Corp. 2002, 2020 +* Copyright IBM Corp. 2002, 2023 */ #ifndef ZFCP_EXT_H
@@ -46,6 +46,9 @@ extern void zfcp_dbf_hba_fsf_res(char *, int, struct zfcp_fsf_req *);extern void zfcp_dbf_hba_fsf_fces (char *tag,const struct zfcp_fsf_req *req,u64 wwpn,u32 fc_security_old,u32 fc_security_new); +extern void zfcp_dbf_hba_fsf_reqid (const char *const tag,const int level, +struct zfcp_adapter *const adapter, +const u64 req_id);extern void zfcp_dbf_hba_bit_err (char *,struct zfcp_fsf_req *);extern void zfcp_dbf_hba_def_err (struct zfcp_adapter *,u64 ,u16 ,void **);extern void zfcp_dbf_san_req (char *,struct zfcp_fsf_req *,u32 );
drivers/s390/scsi/zfcp_fsf.c +5 -2
@@ -4,7 +4,7 @@* * Implementation of FSF commands. * -* Copyright IBM Corp. 2002, 2020 +* Copyright IBM Corp. 2002, 2023 */ #define KMSG_COMPONENT "zfcp"
@@ -892,8 +892,11 @@ static int zfcp_fsf_req_send(struct zfcp_fsf_req *req) req->issued =get_tod_clock ();if (zfcp_qdio_send (qdio, &req->qdio_req)) {del_timer_sync (&req->timer); +/* lookup request again, list might have changed */ -zfcp_reqlist_find_rm (adapter->req_list, req_id); +if (zfcp_reqlist_find_rm (adapter->req_list, req_id) ==NULL ) +zfcp_dbf_hba_fsf_reqid ("fsrsrmf" ,1 , adapter, req_id); +zfcp_erp_adapter_reopen (adapter,0 ,"fsrs__1" );return -EIO ; }
Signature · 1 line
2.39.2
Benjamin,
Quoted 9 lines
here is a small set of changes for the zFCP device driver. These are basically follow-up changes I made after the fix I send in <979f6e6019d15f91ba56182f1aaf68d61bf37fc6.1668595505.git.bblock@linux.ibm.com> that refactor some related areas in the driver, and add some additional tracing if we ever should run into a similar situation somehow.
It would be nice, if you could still include them for v6.3. Not sure if I'm too late already.
It's a bit late. I merged them to 6.3/scsi-staging for now but may defer to 6.4 depending on how busy the various code checking robots are.
Signature · 1 line
Martin K. Petersen Oracle Linux Engineering
Quoted 7 lines · Martin K. Petersen
Quoted 7 lines
here is a small set of changes for the zFCP device driver. These are basically follow-up changes I made after the fix I send in <979f6e6019d15f91ba56182f1aaf68d61bf37fc6.1668595505.git.bblock@linux.ibm.com> that refactor some related areas in the driver, and add some additional tracing if we ever should run into a similar situation somehow.
Quoted 6 lines
Quoted 2 lines
It would be nice, if you could still include them for v6.3. Not sure if I'm too late already.
It's a bit late. I merged them to 6.3/scsi-staging for now but may defer to 6.4 depending on how busy the various code checking robots are.
Alright, sounds good. Thanks.
Signature · 4 lines
Best Regards, Benjamin Block / Linux on IBM Z Kernel Development / IBM Systems IBM Deutschland Research & Development GmbH / https://www.ibm.com/privacy Vorsitz. AufsR.: Gregor Pillen / Geschäftsführung: David Faller Sitz der Gesellschaft: Böblingen / Registergericht: AmtsG Stuttgart, HRB 243294