enable nvmet-fc for blktests
Subsystems (3)
v4 cover enable nvmet-fc for blktests
The first version which allows to run blktests in loop without crashing or hanging forever somewhere. The missing fix was the argument swap patch for list_add_tail, which ensured no response gets dropped on tbe floor.
Note, this runs with the udev auto connect rule running in the background which injects a lot of additional noise to the test. For example blktests will warn about 'device not removed', which is correct but this device was not created by the test, it was created by the udev rule. nvme/003 fails because it disconnects two devices instead the exptected one.
These are things we have to address in blktests though. So hopefully with these patches we are getting close to run blktests on a regular basis.
changes:
v4:
- dropped patches which got applied
- dropped 'nvmet-fc: free hostport after release reference to tgtport'
- reworked commit message of 'nvmet-fc: untangle cross refcounting objects' and renamed it to 'nvmet-fc: defer cleanup using RCU properly'
- added 'nvmet-fcloop: swap the list_add_tail arguments'
and 'nvmet-fc: use RCU list iterator for assoc_list'
- added RBs
v3:
- collected all patches into one series
- updated ref counting in nvmet-fc
v2:
- added RBs
- added new patches
- https://lore.kernel.org/linux-nvme/[email protected]/
v1:
- https://lore.kernel.org/linux-nvme/[email protected]/
*** BLURB HERE ***
Daniel Wagner (12):
nvme-fc: do not wait in vain when unloading module
nvmet-fcloop: swap the list_add_tail arguments
nvmet-fc: release reference on target port
nvmet-fc: defer cleanup using RCU properly
nvmet-fc: free queue and assoc directly
nvmet-fc: hold reference on hostport match
nvmet-fc: remove null hostport pointer check
nvmet-fc: do not tack refs on tgtports from assoc
nvmet-fc: abort command when there is no binding
nvmet-fc: avoid deadlock on delete association path
nvmet-fc: take ref count on tgtport before delete assoc
nvmet-fc: use RCU list iterator for assoc_list
drivers/nvme/host/fc.c | 47 ++--------- drivers/nvme/target/fc.c | 156 +++++++++++++++++++---------------- drivers/nvme/target/fcloop.c | 6 +- 3 files changed, 93 insertions(+), 116 deletions(-)
Signature · 1 line
2.43.0
PATCH v4 1/12 nvme-fc: do not wait in vain when unloading module
The module exit path has race between deleting all controllers and freeing 'left over IDs'. To prevent double free a synchronization between nvme_delete_ctrl and ida_destroy has been added by the initial commit.
There is some logic around trying to prevent from hanging forever in wait_for_completion, though it does not handling all cases. E.g. blktests is able to reproduce the situation where the module unload hangs forever.
If we completely rely on the cleanup code executed from the nvme_delete_ctrl path, all IDs will be freed eventually. This makes calling ida_destroy unnecessary. We only have to ensure that all nvme_delete_ctrl code has been executed before we leave nvme_fc_exit_module. This is done by flushing the nvme_delete_wq workqueue.
While at it, remove the unused nvme_fc_wq workqueue too.
--- drivers/nvme/host/fc.c | 47 ++++++------------------------------------ 1 file changed, 6 insertions(+), 41 deletions(-)
drivers/nvme/host/fc.c +6 -41
@@ -221,11 +221,6 @@ static LIST_HEAD(nvme_fc_lport_list);static DEFINE_IDA (nvme_fc_local_port_cnt);static DEFINE_IDA (nvme_fc_ctrl_cnt); -static struct workqueue_struct *nvme_fc_wq; - -static bool nvme_fc_waiting_to_unload; -static DECLARE_COMPLETION (nvme_fc_unload_proceed); -/* * These items are short-term. They will eventually be moved into * a generic FC class. See comments in module init.
@@ -255,8 +250,6 @@ nvme_fc_free_lport(struct kref *ref)/* remove from transport list */ spin_lock_irqsave (&nvme_fc_lock, flags);list_del (&lport->port_list); -if (nvme_fc_waiting_to_unload &&list_empty (&nvme_fc_lport_list)) -complete (&nvme_fc_unload_proceed);spin_unlock_irqrestore (&nvme_fc_lock, flags);ida_free (&nvme_fc_local_port_cnt, lport->localport.port_num);
@@ -3894,10 +3887,6 @@ static int __init nvme_fc_init_module(void) {int ret; - nvme_fc_wq =alloc_workqueue ("nvme_fc_wq" ,WQ_MEM_RECLAIM ,0 ); -if (!nvme_fc_wq) -return -ENOMEM ; -/* * NOTE: * It is expected that in the future the kernel will combine
@@ -3915,7 +3904,7 @@ static int __init nvme_fc_init_module(void) ret =class_register (&fc_class);if (ret) {pr_err ("couldn't register class fc\n" ); - goto out_destroy_wq; +return ret; }/*
@@ -3939,8 +3928,6 @@ static int __init nvme_fc_init_module(void)device_destroy (&fc_class,MKDEV (0 ,0 )); out_destroy_class:class_unregister (&fc_class); -out_destroy_wq: -destroy_workqueue (nvme_fc_wq);return ret; }
@@ -3960,45 +3947,23 @@ nvme_fc_delete_controllers(struct nvme_fc_rport *rport)spin_unlock (&rport->lock); } -static void -nvme_fc_cleanup_for_unload (void ) +static void __exit nvme_fc_exit_module(void) {struct nvme_fc_lport *lport;struct nvme_fc_rport *rport; - -list_for_each_entry (lport, &nvme_fc_lport_list, port_list) { -list_for_each_entry (rport, &lport->endp_list, endp_list) { -nvme_fc_delete_controllers (rport); - } - } -} - -static void __exit nvme_fc_exit_module(void) -{unsigned long flags; -bool need_cleanup = false;spin_lock_irqsave (&nvme_fc_lock, flags); - nvme_fc_waiting_to_unload = true; -if (!list_empty (&nvme_fc_lport_list)) { - need_cleanup = true; -nvme_fc_cleanup_for_unload (); - } +list_for_each_entry (lport, &nvme_fc_lport_list, port_list) +list_for_each_entry (rport, &lport->endp_list, endp_list) +nvme_fc_delete_controllers (rport);spin_unlock_irqrestore (&nvme_fc_lock, flags); -if (need_cleanup) { -pr_info ("%s: waiting for ctlr deletes\n" , __func__); -wait_for_completion (&nvme_fc_unload_proceed); -pr_info ("%s: ctrl deletes complete\n" , __func__); - } +flush_workqueue (nvme_delete_wq);nvmf_unregister_transport (&nvme_fc_transport); -ida_destroy (&nvme_fc_local_port_cnt); -ida_destroy (&nvme_fc_ctrl_cnt); -device_destroy (&fc_class,MKDEV (0 ,0 ));class_unregister (&fc_class); -destroy_workqueue (nvme_fc_wq); }module_init (nvme_fc_init_module);
Signature · 1 line
2.43.0
PATCH v4 2/12 nvmet-fcloop: swap the list_add_tail arguments
In reply to Daniel WagnerThe first argument of list_add_tail function is the new element which should be added to the list which is the second argument. Swap the arguments to allow processing more than one element at a time.
--- drivers/nvme/target/fcloop.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
drivers/nvme/target/fcloop.c +3 -3
@@ -358,7 +358,7 @@ fcloop_h2t_ls_req(struct nvme_fc_local_port *localport,if (!rport->targetport) { tls_req->status = -ECONNREFUSED ;spin_lock (&rport->lock); -list_add_tail (&rport->ls_list, &tls_req->ls_list); +list_add_tail (&tls_req->ls_list, &rport->ls_list);spin_unlock (&rport->lock);queue_work (nvmet_wq, &rport->ls_work);return ret;
@@ -391,7 +391,7 @@ fcloop_h2t_xmt_ls_rsp(struct nvmet_fc_target_port *targetport,if (remoteport) { rport = remoteport->private;spin_lock (&rport->lock); -list_add_tail (&rport->ls_list, &tls_req->ls_list); +list_add_tail (&tls_req->ls_list, &rport->ls_list);spin_unlock (&rport->lock);queue_work (nvmet_wq, &rport->ls_work); }
@@ -446,7 +446,7 @@ fcloop_t2h_ls_req(struct nvmet_fc_target_port *targetport, void *hosthandle,if (!tport->remoteport) { tls_req->status = -ECONNREFUSED ;spin_lock (&tport->lock); -list_add_tail (&tport->ls_list, &tls_req->ls_list); +list_add_tail (&tls_req->ls_list, &tport->ls_list);spin_unlock (&tport->lock);queue_work (nvmet_wq, &tport->ls_work);return ret;
Signature · 1 line
2.43.0
Re: [PATCH v4 02/12] nvmet-fcloop: swap the list_add_tail arguments
Quoted 3 lines · Daniel Wagner
The first argument of
list_add_tailfunction is the new element which should be added to the list which is the second argument. Swap the arguments to allow processing more than one element at a time.
Ooops.
Looks good:
PATCH v4 3/12 nvmet-fc: release reference on target port
In reply to Daniel WagnerIn case we return early out of __nvmet_fc_finish_ls_req() we still have to release the reference on the target port.
--- drivers/nvme/target/fc.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
drivers/nvme/target/fc.c +2 -1
@@ -360,7 +360,7 @@ __nvmet_fc_finish_ls_req(struct nvmet_fc_ls_req_op *lsop)if (!lsop->req_queued) {spin_unlock_irqrestore (&tgtport->lock, flags); -return ; + goto out_puttgtport; }list_del (&lsop->lsreq_list);
@@ -373,6 +373,7 @@ __nvmet_fc_finish_ls_req(struct nvmet_fc_ls_req_op *lsop) (lsreq->rqstlen + lsreq->rsplen), DMA_BIDIRECTIONAL); +out_puttgtport:nvmet_fc_tgtport_put (tgtport); }
Signature · 1 line
2.43.0
Re: [PATCH v4 03/12] nvmet-fc: release reference on target port
Looks good:
PATCH v4 4/12 nvmet-fc: defer cleanup using RCU properly
In reply to Daniel WagnerWhen the target executes a disconnect and the host triggers a reconnect immediately, the reconnect command still finds an existing association.
The reconnect crashes later on because nvmet_fc_delete_target_assoc blindly removes resources while the reconnect code wants to use it.
To address this, nvmet_fc_find_target_assoc should not be able to lookup an association which is being removed. The association list is already under RCU lifetime management, so let's properly use it and remove the association from the list and wait for a grace period before cleaning up all. This means we also can drop the RCU management on the queues, because this is now handled via the association itself.
A second step split the execution context so that the initial disconnect command can complete without running the reconnect code in the same context. As usual, this is done by deferring the ->done to a workqueue.
--- drivers/nvme/target/fc.c | 83 ++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 46 deletions(-)
drivers/nvme/target/fc.c +37 -46
@@ -166,7 +166,7 @@ struct nvmet_fc_tgt_assoc {struct nvmet_fc_hostport *hostport;struct nvmet_fc_ls_iod *rcv_disconn;struct list_head a_list; -struct nvmet_fc_tgt_queue __rcu *queues[NVMET_NR_QUEUES +1 ]; +struct nvmet_fc_tgt_queue *queues[NVMET_NR_QUEUES +1 ];struct kref ref;struct work_struct del_work;struct rcu_head rcu;
@@ -803,14 +803,11 @@ nvmet_fc_alloc_target_queue(struct nvmet_fc_tgt_assoc *assoc,if (!queue)return NULL ; -if (!nvmet_fc_tgt_a_get (assoc)) - goto out_free_queue; - queue->work_q =alloc_workqueue ("ntfc%d.%d.%d" ,0 ,0 , assoc->tgtport->fc_target_port.port_num, assoc->a_id, qid);if (!queue->work_q) - goto out_a_put; + goto out_free_queue; queue->qid = qid; queue->sqsize = sqsize;
@@ -832,15 +829,13 @@ nvmet_fc_alloc_target_queue(struct nvmet_fc_tgt_assoc *assoc, goto out_fail_iodlist;WARN_ON (assoc->queues[qid]); -rcu_assign_pointer (assoc->queues[qid], queue); + assoc->queues[qid] = queue;return queue; out_fail_iodlist:nvmet_fc_destroy_fcp_iodlist (assoc->tgtport, queue);destroy_workqueue (queue->work_q); -out_a_put: -nvmet_fc_tgt_a_put (assoc); out_free_queue:kfree (queue);return NULL ;
@@ -853,12 +848,8 @@ nvmet_fc_tgt_queue_free(struct kref *ref)struct nvmet_fc_tgt_queue *queue = container_of(ref, struct nvmet_fc_tgt_queue, ref); -rcu_assign_pointer (queue->assoc->queues[queue->qid],NULL ); -nvmet_fc_destroy_fcp_iodlist (queue->assoc->tgtport, queue); -nvmet_fc_tgt_a_put (queue->assoc); -destroy_workqueue (queue->work_q);kfree_rcu (queue, rcu);
@@ -970,7 +961,7 @@ nvmet_fc_find_target_queue(struct nvmet_fc_tgtport *tgtport,rcu_read_lock ();list_for_each_entry_rcu (assoc, &tgtport->assoc_list, a_list) {if (association_id == assoc->association_id) { - queue =rcu_dereference (assoc->queues[qid]); + queue = assoc->queues[qid];if (queue && (!atomic_read (&queue->connected) || !nvmet_fc_tgt_q_get (queue)))
@@ -1173,13 +1164,18 @@ nvmet_fc_target_assoc_free(struct kref *ref)struct nvmet_fc_tgtport *tgtport = assoc->tgtport;struct nvmet_fc_ls_iod *oldls;unsigned long flags; +int i; + +for (i =NVMET_NR_QUEUES ; i >=0 ; i--) { +if (assoc->queues[i]) +nvmet_fc_delete_target_queue (assoc->queues[i]); + }/* Send Disconnect now that all i/o has completed */ nvmet_fc_xmt_disconnect_assoc (assoc);nvmet_fc_free_hostport (assoc->hostport);spin_lock_irqsave (&tgtport->lock, flags); -list_del_rcu (&assoc->a_list); oldls = assoc->rcv_disconn;spin_unlock_irqrestore (&tgtport->lock, flags);/* if pending Rcv Disconnect Association LS, send rsp now */
@@ -1209,7 +1205,7 @@ static void nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc) {struct nvmet_fc_tgtport *tgtport = assoc->tgtport; -struct nvmet_fc_tgt_queue *queue; +unsigned long flags;int i, terminating; terminating =atomic_xchg (&assoc->terminating,1 );
@@ -1218,29 +1214,21 @@ nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc)if (terminating)return ; +spin_lock_irqsave (&tgtport->lock, flags); +list_del_rcu (&assoc->a_list); +spin_unlock_irqrestore (&tgtport->lock, flags); -for (i =NVMET_NR_QUEUES ; i >=0 ; i--) { -rcu_read_lock (); - queue =rcu_dereference (assoc->queues[i]); -if (!queue) { -rcu_read_unlock (); -continue ; - } +synchronize_rcu (); -if (!nvmet_fc_tgt_q_get (queue)) { -rcu_read_unlock (); -continue ; - } -rcu_read_unlock (); -nvmet_fc_delete_target_queue (queue); -nvmet_fc_tgt_q_put (queue); +/* ensure all in-flight I/Os have been processed */ +for (i =NVMET_NR_QUEUES ; i >=0 ; i--) { +if (assoc->queues[i]) +flush_workqueue (assoc->queues[i]->work_q); }dev_info (tgtport->dev,"{%d:%d} Association deleted\n" , tgtport->fc_target_port.port_num, assoc->a_id); - -nvmet_fc_tgt_a_put (assoc); } static struct nvmet_fc_tgt_assoc *
@@ -1493,9 +1481,8 @@ __nvmet_fc_free_assocs(struct nvmet_fc_tgtport *tgtport)list_for_each_entry_rcu (assoc, &tgtport->assoc_list, a_list) {if (!nvmet_fc_tgt_a_get (assoc))continue ; -if (!queue_work (nvmet_wq, &assoc->del_work)) -/* already deleting - release local reference */ -nvmet_fc_tgt_a_put (assoc); +queue_work (nvmet_wq, &assoc->del_work); +nvmet_fc_tgt_a_put (assoc); }rcu_read_unlock (); }
@@ -1548,9 +1535,8 @@ nvmet_fc_invalidate_host(struct nvmet_fc_target_port *target_port,continue ; assoc->hostport->invalid =1 ; noassoc = false; -if (!queue_work (nvmet_wq, &assoc->del_work)) -/* already deleting - release local reference */ -nvmet_fc_tgt_a_put (assoc); +queue_work (nvmet_wq, &assoc->del_work); +nvmet_fc_tgt_a_put (assoc); }spin_unlock_irqrestore (&tgtport->lock, flags);
@@ -1582,7 +1568,7 @@ nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl)rcu_read_lock ();list_for_each_entry_rcu (assoc, &tgtport->assoc_list, a_list) { - queue =rcu_dereference (assoc->queues[0 ]); + queue = assoc->queues[0 ];if (queue && queue->nvme_sq.ctrl == ctrl) {if (nvmet_fc_tgt_a_get (assoc)) found_ctrl = true;
@@ -1594,9 +1580,8 @@ nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl)nvmet_fc_tgtport_put (tgtport);if (found_ctrl) { -if (!queue_work (nvmet_wq, &assoc->del_work)) -/* already deleting - release local reference */ -nvmet_fc_tgt_a_put (assoc); +queue_work (nvmet_wq, &assoc->del_work); +nvmet_fc_tgt_a_put (assoc);return ; }
@@ -1626,6 +1611,8 @@ nvmet_fc_unregister_targetport(struct nvmet_fc_target_port *target_port)/* terminate any outstanding associations */ __nvmet_fc_free_assocs (tgtport); +flush_workqueue (nvmet_wq); +/* * should terminate LS's as well. However, LS's will be generated * at the tail end of association termination, so they likely don't
@@ -1871,9 +1858,6 @@ nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, sizeof(struct fcnvme_ls_disconnect_assoc_acc)), FCNVME_LS_DISCONNECT_ASSOC); -/* release get taken in nvmet_fc_find_target_assoc */ -nvmet_fc_tgt_a_put (assoc); -/* * The rules for LS response says the response cannot * go back until ABTS's have been sent for all outstanding
@@ -1888,8 +1872,6 @@ nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, assoc->rcv_disconn = iod;spin_unlock_irqrestore (&tgtport->lock, flags); -nvmet_fc_delete_target_assoc (assoc); -if (oldls) {dev_info (tgtport->dev, "{%d:%d} Multiple Disconnect Association LS's "
@@ -1905,6 +1887,9 @@ nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport,nvmet_fc_xmt_ls_rsp (tgtport, oldls); } +queue_work (nvmet_wq, &assoc->del_work); +nvmet_fc_tgt_a_put (assoc); +return false; }
@@ -2903,6 +2888,9 @@ nvmet_fc_remove_port(struct nvmet_port *port)nvmet_fc_portentry_unbind (pe); +/* terminate any outstanding associations */ +__nvmet_fc_free_assocs (pe->tgtport); +kfree (pe); }
@@ -2934,6 +2922,9 @@ static int __init nvmet_fc_init_module(void) static void __exit nvmet_fc_exit_module(void) { +/* ensure any shutdown operation, e.g. delete ctrls have finished */ +flush_workqueue (nvmet_wq); +/* sanity check - all lports should be removed */ if (!list_empty (&nvmet_fc_target_list))pr_warn ("%s: targetport list not empty\n" , __func__);
Signature · 1 line
2.43.0
Re: [PATCH v4 04/12] nvmet-fc: defer cleanup using RCU properly
Looks good:
PATCH v4 5/12 nvmet-fc: free queue and assoc directly
In reply to Daniel WagnerNeither struct nvmet_fc_tgt_queue nor struct nvmet_fc_tgt_assoc are data structure which are used in a RCU context. So there is no reason to delay the free operation.
--- drivers/nvme/target/fc.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-)
drivers/nvme/target/fc.c +2 -4
@@ -145,7 +145,6 @@ struct nvmet_fc_tgt_queue {struct list_head avail_defer_list;struct workqueue_struct *work_q;struct kref ref; -struct rcu_head rcu;/* array of fcp_iods */ struct nvmet_fc_fcp_iod fod[] __counted_by(sqsize); }__aligned(sizeof(unsigned long long)) ;
@@ -169,7 +168,6 @@ struct nvmet_fc_tgt_assoc {struct nvmet_fc_tgt_queue *queues[NVMET_NR_QUEUES +1 ];struct kref ref;struct work_struct del_work; -struct rcu_head rcu; };
@@ -852,7 +850,7 @@ nvmet_fc_tgt_queue_free(struct kref *ref)destroy_workqueue (queue->work_q); -kfree_rcu (queue, rcu); +kfree (queue); } static void
@@ -1185,8 +1183,8 @@ nvmet_fc_target_assoc_free(struct kref *ref)dev_info (tgtport->dev,"{%d:%d} Association freed\n" , tgtport->fc_target_port.port_num, assoc->a_id); -kfree_rcu (assoc, rcu);nvmet_fc_tgtport_put (tgtport); +kfree (assoc); } static void
Signature · 1 line
2.43.0
Re: [PATCH v4 05/12] nvmet-fc: free queue and assoc directly
Looks good:
PATCH v4 6/12 nvmet-fc: hold reference on hostport match
In reply to Daniel WagnerThe hostport data structure is shared between the association, this why we keep track of the users via a refcount. So we should not decrement the refcount on a match and free the hostport several times.
Reported by KASAN.
--- drivers/nvme/target/fc.c | 2 -- 1 file changed, 2 deletions(-)
drivers/nvme/target/fc.c +0 -2
@@ -1068,8 +1068,6 @@ nvmet_fc_alloc_hostport(struct nvmet_fc_tgtport *tgtport, void *hosthandle)/* new allocation not needed */ kfree (newhost); newhost = match; -/* no new allocation - release reference */ -nvmet_fc_tgtport_put (tgtport); }else { newhost->tgtport = tgtport; newhost->hosthandle = hosthandle;
Signature · 1 line
2.43.0
Re: [PATCH v4 06/12] nvmet-fc: hold reference on hostport match
Quoted 8 lines · Daniel Wagner
The hostport data structure is shared between the association, this why we keep track of the users via a refcount. So we should not decrement the refcount on a match and free the hostport several times.
Reported by KASAN.
Reviewed-by Hannes Reinecke <[email protected]>Signed-off-by Daniel Wagner <[email protected]>
Looks good:
PATCH v4 7/12 nvmet-fc: remove null hostport pointer check
In reply to Daniel WagnerAn association has always a valid hostport pointer. Remove useless null pointer check.
--- drivers/nvme/target/fc.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-)
drivers/nvme/target/fc.c +2 -4
@@ -488,8 +488,7 @@ nvmet_fc_xmt_disconnect_assoc(struct nvmet_fc_tgt_assoc *assoc)* message is normal. Otherwise, send unless the hostport has * already been invalidated by the lldd. */ -if (!tgtport->ops->ls_req || !assoc->hostport || - assoc->hostport->invalid) +if (!tgtport->ops->ls_req || assoc->hostport->invalid)return ; lsop = kzalloc((sizeof(*lsop) +
@@ -1524,8 +1523,7 @@ nvmet_fc_invalidate_host(struct nvmet_fc_target_port *target_port,spin_lock_irqsave (&tgtport->lock, flags);list_for_each_entry_safe (assoc, next, &tgtport->assoc_list, a_list) { -if (!assoc->hostport || - assoc->hostport->hosthandle != hosthandle) +if (assoc->hostport->hosthandle != hosthandle)continue ;if (!nvmet_fc_tgt_a_get (assoc))continue ;
Signature · 1 line
2.43.0
Re: [PATCH v4 07/12] nvmet-fc: remove null hostport pointer check
Looks good:
PATCH v4 8/12 nvmet-fc: do not tack refs on tgtports from assoc
In reply to Daniel WagnerThe association life time is tied to the life time of the target port. That means we should not take extra a refcount when creating a association.
--- drivers/nvme/target/fc.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-)
drivers/nvme/target/fc.c +1 -7
@@ -1107,12 +1107,9 @@ nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport, void *hosthandle)if (idx <0 ) goto out_free_assoc; -if (!nvmet_fc_tgtport_get (tgtport)) - goto out_ida; - assoc->hostport =nvmet_fc_alloc_hostport (tgtport, hosthandle);if (IS_ERR (assoc->hostport)) - goto out_put; + goto out_ida; assoc->tgtport = tgtport; assoc->a_id = idx;
@@ -1142,8 +1139,6 @@ nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport, void *hosthandle)return assoc; -out_put: -nvmet_fc_tgtport_put (tgtport); out_ida:ida_free (&tgtport->assoc_cnt, idx); out_free_assoc:
@@ -1180,7 +1175,6 @@ nvmet_fc_target_assoc_free(struct kref *ref)dev_info (tgtport->dev,"{%d:%d} Association freed\n" , tgtport->fc_target_port.port_num, assoc->a_id); -nvmet_fc_tgtport_put (tgtport);kfree (assoc); }
Signature · 1 line
2.43.0
Re: [PATCH v4 08/12] nvmet-fc: do not tack refs on tgtports from assoc
Looks good:
PATCH v4 9/12 nvmet-fc: abort command when there is no binding
In reply to Daniel WagnerWhen the target port has not active port binding, there is no point in trying to process the command as it has to fail anyway. Instead adding checks to all commands abort the command early.
--- drivers/nvme/target/fc.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)
drivers/nvme/target/fc.c +6 -2
@@ -1099,6 +1099,9 @@ nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport, void *hosthandle)int idx;bool needrandom = true; +if (!tgtport->pe) +return NULL ; + assoc =kzalloc (sizeof (*assoc),GFP_KERNEL );if (!assoc)return NULL ;
@@ -2514,8 +2517,9 @@ nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport, fod->req.cmd = &fod->cmdiubuf.sqe; fod->req.cqe = &fod->rspiubuf.cqe; -if (tgtport->pe) - fod->req.port = tgtport->pe->port; +if (!tgtport->pe) + goto transport_error; + fod->req.port = tgtport->pe->port;/* clear any response payload */ memset (&fod->rspiubuf,0 ,sizeof (fod->rspiubuf));
Signature · 1 line
2.43.0
Re: [PATCH v4 09/12] nvmet-fc: abort command when there is no binding
Looks good:
PATCH v4 10/12 nvmet-fc: avoid deadlock on delete association path
In reply to Daniel WagnerWhen deleting an association the shutdown path is deadlocking because we try to flush the nvmet_wq nested. Avoid this by deadlock by deferring the put work into its own work item.
--- drivers/nvme/target/fc.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-)
drivers/nvme/target/fc.c +13 -3
@@ -111,6 +111,8 @@ struct nvmet_fc_tgtport {struct nvmet_fc_port_entry *pe;struct kref ref;u32 max_sg_cnt; + +struct work_struct put_work; };struct nvmet_fc_port_entry {
@@ -247,6 +249,13 @@ static int nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc);static void nvmet_fc_tgt_q_put (struct nvmet_fc_tgt_queue *queue);static int nvmet_fc_tgt_q_get (struct nvmet_fc_tgt_queue *queue);static void nvmet_fc_tgtport_put (struct nvmet_fc_tgtport *tgtport); +static void nvmet_fc_put_tgtport_work (struct work_struct *work) +{ +struct nvmet_fc_tgtport *tgtport = + container_of(work, struct nvmet_fc_tgtport, put_work); + +nvmet_fc_tgtport_put (tgtport); +}static int nvmet_fc_tgtport_get (struct nvmet_fc_tgtport *tgtport);static void nvmet_fc_handle_fcp_rqst (struct nvmet_fc_tgtport *tgtport,struct nvmet_fc_fcp_iod *fod);
@@ -358,7 +367,7 @@ __nvmet_fc_finish_ls_req(struct nvmet_fc_ls_req_op *lsop)if (!lsop->req_queued) {spin_unlock_irqrestore (&tgtport->lock, flags); - goto out_puttgtport; + goto out_putwork; }list_del (&lsop->lsreq_list);
@@ -371,8 +380,8 @@ __nvmet_fc_finish_ls_req(struct nvmet_fc_ls_req_op *lsop) (lsreq->rqstlen + lsreq->rsplen), DMA_BIDIRECTIONAL); -out_puttgtport: -nvmet_fc_tgtport_put (tgtport); +out_putwork: +queue_work (nvmet_wq, &tgtport->put_work); } static int
@@ -1396,6 +1405,7 @@ nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo,kref_init (&newrec->ref);ida_init (&newrec->assoc_cnt); newrec->max_sg_cnt = template->max_sgl_segments; +INIT_WORK (&newrec->put_work, nvmet_fc_put_tgtport_work); ret =nvmet_fc_alloc_ls_iodlist (newrec);if (ret) {
Signature · 1 line
2.43.0
Re: [PATCH v4 10/12] nvmet-fc: avoid deadlock on delete association path
Looks good:
PATCH v4 11/12 nvmet-fc: take ref count on tgtport before delete assoc
In reply to Daniel WagnerWe have to ensure that the tgtport is not going away before be have remove all the associations.
--- drivers/nvme/target/fc.c | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-)
drivers/nvme/target/fc.c +23 -8
@@ -1090,13 +1090,28 @@ nvmet_fc_alloc_hostport(struct nvmet_fc_tgtport *tgtport, void *hosthandle) }static void -nvmet_fc_delete_assoc (struct work_struct *work) +nvmet_fc_delete_assoc (struct nvmet_fc_tgt_assoc *assoc) +{ +nvmet_fc_delete_target_assoc (assoc); +nvmet_fc_tgt_a_put (assoc); +} + +static void +nvmet_fc_delete_assoc_work (struct work_struct *work) {struct nvmet_fc_tgt_assoc *assoc = container_of(work, struct nvmet_fc_tgt_assoc, del_work); +struct nvmet_fc_tgtport *tgtport = assoc->tgtport; -nvmet_fc_delete_target_assoc (assoc); -nvmet_fc_tgt_a_put (assoc); +nvmet_fc_delete_assoc (assoc); +nvmet_fc_tgtport_put (tgtport); +} + +static void +nvmet_fc_schedule_delete_assoc (struct nvmet_fc_tgt_assoc *assoc) +{ +nvmet_fc_tgtport_get (assoc->tgtport); +queue_work (nvmet_wq, &assoc->del_work); } static struct nvmet_fc_tgt_assoc *
@@ -1127,7 +1142,7 @@ nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport, void *hosthandle) assoc->a_id = idx;INIT_LIST_HEAD (&assoc->a_list);kref_init (&assoc->ref); -INIT_WORK (&assoc->del_work, nvmet_fc_delete_assoc); +INIT_WORK (&assoc->del_work, nvmet_fc_delete_assoc_work);atomic_set (&assoc->terminating,0 );while (needrandom) {
@@ -1483,7 +1498,7 @@ __nvmet_fc_free_assocs(struct nvmet_fc_tgtport *tgtport)list_for_each_entry_rcu (assoc, &tgtport->assoc_list, a_list) {if (!nvmet_fc_tgt_a_get (assoc))continue ; -queue_work (nvmet_wq, &assoc->del_work); +nvmet_fc_schedule_delete_assoc (assoc);nvmet_fc_tgt_a_put (assoc); }rcu_read_unlock ();
@@ -1536,7 +1551,7 @@ nvmet_fc_invalidate_host(struct nvmet_fc_target_port *target_port,continue ; assoc->hostport->invalid =1 ; noassoc = false; -queue_work (nvmet_wq, &assoc->del_work); +nvmet_fc_schedule_delete_assoc (assoc);nvmet_fc_tgt_a_put (assoc); }spin_unlock_irqrestore (&tgtport->lock, flags);
@@ -1581,7 +1596,7 @@ nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl)nvmet_fc_tgtport_put (tgtport);if (found_ctrl) { -queue_work (nvmet_wq, &assoc->del_work); +nvmet_fc_schedule_delete_assoc (assoc);nvmet_fc_tgt_a_put (assoc);return ; }
@@ -1888,7 +1903,7 @@ nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport,nvmet_fc_xmt_ls_rsp (tgtport, oldls); } -queue_work (nvmet_wq, &assoc->del_work); +nvmet_fc_schedule_delete_assoc (assoc);nvmet_fc_tgt_a_put (assoc);return false;
Signature · 1 line
2.43.0
Re: [PATCH v4 11/12] nvmet-fc: take ref count on tgtport before delete assoc
Looks good:
PATCH v4 12/12 nvmet-fc: use RCU list iterator for assoc_list
In reply to Daniel WagnerThe assoc_list is a RCU protected list, thus use the RCU flavor of list functions.
--- drivers/nvme/target/fc.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
drivers/nvme/target/fc.c +4 -1
@@ -1151,7 +1151,9 @@ nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport, void *hosthandle)spin_lock_irqsave (&tgtport->lock, flags); needrandom = false; -list_for_each_entry (tmpassoc, &tgtport->assoc_list, a_list) { +rcu_read_lock (); +list_for_each_entry_rcu (tmpassoc, &tgtport->assoc_list, a_list) { +if (ran == tmpassoc->association_id) { needrandom = true;break ;
@@ -1161,6 +1163,7 @@ nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport, void *hosthandle) assoc->association_id = ran;list_add_tail_rcu (&assoc->a_list, &tgtport->assoc_list); } +rcu_read_unlock ();spin_unlock_irqrestore (&tgtport->lock, flags); }
Signature · 1 line
2.43.0
Christoph Hellwig
maintainerRe: [PATCH v4 12/12] nvmet-fc: use RCU list iterator for assoc_list
drivers/nvme/target/fc.c nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport, void *hosthandle) · line 1155
Quoted 1 line
drivers/nvme/target/fc.c +1
-0+list_for_each_entry_rcu (tmpassoc, &tgtport->assoc_list, a_list) {
please avoid the overly long line.
Also in general the (rcu)lock, list walk to find something, return patterns tend to benefit from being factored into a little helper. Not sure if that really fits here, though.