libfc, bnx2fc, fcoe fixes for 3.9-rc
cover libfc, bnx2fc, fcoe fixes for 3.9-rc
This series fixes a few critical issues with FCoE in 3.9.
1) bnx2fc had incorrect scsi host linkage in sysfs which prevents fcoemon (fcoe userspace daemon) from interacting with it correctly.
2) 3 patches fix VN2VN mode configuration with new FCoE control interfaces.
3) Fixed lockdep warning
These patches are needed to allow fcoe to move to the new sysfs interfaces.
I would like to see these pushed upstream for 3.9-rc. A pull-request mail will immediately follow this series.
---
Robert Love (5):
bnx2fc: Make the fcoe_cltr the SCSI host parent
fcoe: Fix deadlock between create and destroy paths
libfc, fcoe, bnx2fc: Always use fcoe_disc_init for discovery layer initialization libfc, fcoe, bnx2fc: Split fc_disc_init into fc_disc_{init, config}
libfcoe: Fix fcoe_sysfs VN2VN mode
drivers/scsi/bnx2fc/bnx2fc_fcoe.c | 6 ++-- drivers/scsi/fcoe/fcoe.c | 15 +++++++-- drivers/scsi/fcoe/fcoe_ctlr.c | 60 ++++++++++++++++++++++++++++--------- drivers/scsi/libfc/fc_disc.c | 26 +++++++++++----- include/scsi/libfc.h | 3 +- 5 files changed, 80 insertions(+), 30 deletions(-)
Signature · 1 line
Thanks, //Rob
PATCH 1/5 bnx2fc: Make the fcoe_cltr the SCSI host parent
The fcoemon userspace daemon is searching for the a hostX under the the /sys/bus/fcoe/devices/ctlrX/ entries. When interfaces created using fcoe_sysfs and fcoe.ko this linkage is setup correctly, but bnx2fc is not doing the same thing and therefore fcoemon does not create the fcoe interface for bnx2fc.
This patch sets up the correct linkage for bnx2fc such that fcoemon will work correctly with fcoe_sysfs and bnx2fc.
--- drivers/scsi/bnx2fc/bnx2fc_fcoe.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)
drivers/scsi/bnx2fc/bnx2fc_fcoe.c +2 -3
@@ -2133,6 +2133,7 @@ static int _bnx2fc_create(struct net_device *netdev, } ctlr =bnx2fc_to_ctlr (interface); + cdev =fcoe_ctlr_to_ctlr_dev (ctlr); interface->vlan_id = vlan_id; interface->timer_work_queue =
@@ -2143,7 +2144,7 @@ static int _bnx2fc_create(struct net_device *netdev, goto ifput_err; } - lport =bnx2fc_if_create (interface, &interface->hba->pcidev->dev,0 ); + lport =bnx2fc_if_create (interface, &cdev->dev,0 );if (!lport) { printk(KERN_ERR PFX "Failed to create interface (%s)\n", netdev->name);
@@ -2159,8 +2160,6 @@ static int _bnx2fc_create(struct net_device *netdev,/* Make this master N_port */ ctlr->lp = lport; - cdev =fcoe_ctlr_to_ctlr_dev (ctlr); -if (link_state ==BNX2FC_CREATE_LINK_UP ) cdev->enabled =FCOE_CTLR_ENABLED ;else
PATCH 2/5 fcoe: Fix deadlock between create and destroy paths
In reply to Robert LoveWe can deadlock (s_active and fcoe_config_mutex) if a port is being destroyed at the same time one is being created.
[ 4200.503113] ======================================================
[ 4200.503114] [ INFO: possible circular locking dependency detected ]
[ 4200.503116] 3.8.0-rc5+ #8 Not tainted
[ 4200.503117] -------------------------------------------------------
[ 4200.503118] kworker/3:2/2492 is trying to acquire lock:
[ 4200.503119] (s_active#292){++++.+}, at: [<ffffffff8122d20b>] sysfs_addrm_finish+0x3b/0x70
[ 4200.503127]
but task is already holding lock:
[ 4200.503128] (fcoe_config_mutex){+.+.+.}, at: [<ffffffffa02f3338>] fcoe_destroy_work+0xe8/0x120 [fcoe]
[ 4200.503133]which lock already depends on the new lock.
[ 4200.503135]
the existing dependency chain (in reverse order) is:
[ 4200.503136]
-> #1 (fcoe_config_mutex){+.+.+.}:
[ 4200.503139] [<ffffffff810c7711>] lock_acquire+0xa1/0x140
[ 4200.503143] [<ffffffff816ca7be>] mutex_lock_nested+0x6e/0x360
[ 4200.503146] [<ffffffffa02f11bd>] fcoe_enable+0x1d/0xb0 [fcoe]
[ 4200.503148] [<ffffffffa02f127d>] fcoe_ctlr_enabled+0x2d/0x50 [fcoe]
[ 4200.503151] [<ffffffffa02ffbe8>] store_ctlr_enabled+0x38/0x90 [libfcoe]
[ 4200.503154] [<ffffffff81424878>] dev_attr_store+0x18/0x30
[ 4200.503157] [<ffffffff8122b750>] sysfs_write_file+0xe0/0x150
[ 4200.503160] [<ffffffff811b334c>] vfs_write+0xac/0x180
[ 4200.503162] [<ffffffff811b3692>] sys_write+0x52/0xa0
[ 4200.503164] [<ffffffff816d7159>] system_call_fastpath+0x16/0x1b
[ 4200.503167]
-> #0 (s_active#292){++++.+}:
[ 4200.503170] [<ffffffff810c680f>] __lock_acquire+0x135f/0x1c90
[ 4200.503172] [<ffffffff810c7711>] lock_acquire+0xa1/0x140
[ 4200.503174] [<ffffffff8122c626>] sysfs_deactivate+0x116/0x160
[ 4200.503176] [<ffffffff8122d20b>] sysfs_addrm_finish+0x3b/0x70
[ 4200.503178] [<ffffffff8122b2eb>] sysfs_hash_and_remove+0x5b/0xb0
[ 4200.503180] [<ffffffff8122f3d1>] sysfs_remove_group+0x61/0x100
[ 4200.503183] [<ffffffff814251eb>] device_remove_groups+0x3b/0x60
[ 4200.503185] [<ffffffff81425534>] device_remove_attrs+0x44/0x80
[ 4200.503187] [<ffffffff81425e97>] device_del+0x127/0x1c0
[ 4200.503189] [<ffffffff81425f52>] device_unregister+0x22/0x60
[ 4200.503191] [<ffffffffa0300970>] fcoe_ctlr_device_delete+0xe0/0xf0 [libfcoe]
[ 4200.503194] [<ffffffffa02f1b5c>] fcoe_interface_cleanup+0x6c/0xa0 [fcoe]
[ 4200.503196] [<ffffffffa02f3355>] fcoe_destroy_work+0x105/0x120 [fcoe]
[ 4200.503198] [<ffffffff8107ee91>] process_one_work+0x1a1/0x580
[ 4200.503203] [<ffffffff81080c6e>] worker_thread+0x15e/0x440
[ 4200.503205] [<ffffffff8108715a>] kthread+0xea/0xf0
[ 4200.503207] [<ffffffff816d70ac>] ret_from_fork+0x7c/0xb0[ 4200.503209]
other info that might help us debug this:
[ 4200.503211] Possible unsafe locking scenario:
[ 4200.503212] CPU0 CPU1 [ 4200.503213] ---- ---- [ 4200.503214] lock(fcoe_config_mutex); [ 4200.503215] lock(s_active#292); [ 4200.503218] lock(fcoe_config_mutex); [ 4200.503219] lock(s_active#292); [ 4200.503221] *** DEADLOCK ***
[ 4200.503223] 3 locks held by kworker/3:2/2492:
[ 4200.503224] #0: (fcoe){.+.+.+}, at: [<ffffffff8107ee2b>] process_one_work+0x13b/0x580
[ 4200.503228] #1: ((&port->destroy_work)){+.+.+.}, at: [<ffffffff8107ee2b>] process_one_work+0x13b/0x580
[ 4200.503232] #2: (fcoe_config_mutex){+.+.+.}, at: [<ffffffffa02f3338>] fcoe_destroy_work+0xe8/0x120 [fcoe]
[ 4200.503236]
stack backtrace:
[ 4200.503238] Pid: 2492, comm: kworker/3:2 Not tainted 3.8.0-rc5+ #8
[ 4200.503240] Call Trace:
[ 4200.503243] [<ffffffff816c2f09>] print_circular_bug+0x1fb/0x20c
[ 4200.503246] [<ffffffff810c680f>] __lock_acquire+0x135f/0x1c90
[ 4200.503248] [<ffffffff810c463a>] ? debug_check_no_locks_freed+0x9a/0x180
[ 4200.503250] [<ffffffff810c7711>] lock_acquire+0xa1/0x140
[ 4200.503253] [<ffffffff8122d20b>] ? sysfs_addrm_finish+0x3b/0x70
[ 4200.503255] [<ffffffff8122c626>] sysfs_deactivate+0x116/0x160
[ 4200.503258] [<ffffffff8122d20b>] ? sysfs_addrm_finish+0x3b/0x70
[ 4200.503260] [<ffffffff8122d20b>] sysfs_addrm_finish+0x3b/0x70
[ 4200.503262] [<ffffffff8122b2eb>] sysfs_hash_and_remove+0x5b/0xb0
[ 4200.503265] [<ffffffff8122f3d1>] sysfs_remove_group+0x61/0x100
[ 4200.503273] [<ffffffff814251eb>] device_remove_groups+0x3b/0x60
[ 4200.503275] [<ffffffff81425534>] device_remove_attrs+0x44/0x80
[ 4200.503277] [<ffffffff81425e97>] device_del+0x127/0x1c0
[ 4200.503279] [<ffffffff81425f52>] device_unregister+0x22/0x60
[ 4200.503282] [<ffffffffa0300970>] fcoe_ctlr_device_delete+0xe0/0xf0 [libfcoe]
[ 4200.503285] [<ffffffffa02f1b5c>] fcoe_interface_cleanup+0x6c/0xa0 [fcoe]
[ 4200.503287] [<ffffffffa02f3355>] fcoe_destroy_work+0x105/0x120 [fcoe]
[ 4200.503290] [<ffffffff8107ee91>] process_one_work+0x1a1/0x580
[ 4200.503292] [<ffffffff8107ee2b>] ? process_one_work+0x13b/0x580
[ 4200.503295] [<ffffffffa02f3250>] ? fcoe_if_destroy+0x230/0x230 [fcoe]
[ 4200.503297] [<ffffffff81080c6e>] worker_thread+0x15e/0x440
[ 4200.503299] [<ffffffff81080b10>] ? busy_worker_rebind_fn+0x100/0x100
[ 4200.503301] [<ffffffff8108715a>] kthread+0xea/0xf0
[ 4200.503304] [<ffffffff81087070>] ? kthread_create_on_node+0x160/0x160
[ 4200.503306] [<ffffffff816d70ac>] ret_from_fork+0x7c/0xb0
[ 4200.503308] [<ffffffff81087070>] ? kthread_create_on_node+0x160/0x160--- drivers/scsi/fcoe/fcoe.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-)
drivers/scsi/fcoe/fcoe.c +11 -4
@@ -490,7 +490,6 @@ static void fcoe_interface_cleanup(struct fcoe_interface *fcoe) {struct net_device *netdev = fcoe->netdev;struct fcoe_ctlr *fip =fcoe_to_ctlr (fcoe); -struct fcoe_ctlr_device *ctlr_dev =fcoe_ctlr_to_ctlr_dev (fip);rtnl_lock ();if (!fcoe->removed)
@@ -501,7 +500,6 @@ static void fcoe_interface_cleanup(struct fcoe_interface *fcoe)/* tear-down the FCoE controller */ fcoe_ctlr_destroy (fip);scsi_host_put (fip->lp->host); -fcoe_ctlr_device_delete (ctlr_dev);dev_put (netdev);module_put (THIS_MODULE ); }
@@ -2194,6 +2192,8 @@ out_nodev:*/ static void fcoe_destroy_work (struct work_struct *work) { +struct fcoe_ctlr_device *cdev; +struct fcoe_ctlr *ctlr;struct fcoe_port *port;struct fcoe_interface *fcoe;struct Scsi_Host *shost;
@@ -2224,10 +2224,15 @@ static void fcoe_destroy_work(struct work_struct *work)mutex_lock (&fcoe_config_mutex); fcoe = port->priv; + ctlr =fcoe_to_ctlr (fcoe); + cdev =fcoe_ctlr_to_ctlr_dev (ctlr); +fcoe_if_destroy (port->lport);fcoe_interface_cleanup (fcoe);mutex_unlock (&fcoe_config_mutex); + +fcoe_ctlr_device_delete (cdev); }/**
@@ -2335,7 +2340,9 @@ static int _fcoe_create(struct net_device *netdev, enum fip_state fip_mode, rc = -EIO ;rtnl_unlock ();fcoe_interface_cleanup (fcoe); - goto out_nortnl; +mutex_unlock (&fcoe_config_mutex); +fcoe_ctlr_device_delete (ctlr_dev); + goto out; }/* Make this the "master" N_Port */
@@ -2375,8 +2382,8 @@ static int _fcoe_create(struct net_device *netdev, enum fip_state fip_mode, out_nodev:rtnl_unlock (); -out_nortnl:mutex_unlock (&fcoe_config_mutex); +out:return rc; }
PATCH 3/5 libfc, fcoe, bnx2fc: Always use fcoe_disc_init for discovery layer initialization
In reply to Robert LoveCurrently libfcoe is doing some libfc discovery layer initialization outside of libfc. This patch moves this code into libfc and sets up a split in discovery (one time) initialization code and (re-configurable) settings that will come in the next patch.
--- drivers/scsi/bnx2fc/bnx2fc_fcoe.c | 2 +- drivers/scsi/fcoe/fcoe_ctlr.c | 11 ++++++----- drivers/scsi/libfc/fc_disc.c | 5 +++-- include/scsi/libfc.h | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-)
drivers/scsi/bnx2fc/bnx2fc_fcoe.c +1 -1
@@ -939,7 +939,7 @@ static int bnx2fc_libfc_config(struct fc_lport *lport)fc_elsct_init (lport);fc_exch_init (lport);fc_rport_init (lport); -fc_disc_init (lport); +fc_disc_init (lport, lport);return 0 ; }
drivers/scsi/fcoe/fcoe_ctlr.c +6 -5
@@ -2826,6 +2826,8 @@ unlock:int fcoe_libfc_config (struct fc_lport *lport,struct fcoe_ctlr *fip,const struct libfc_function_template *tt,int init_fcp) { +void *priv = lport; +/* Set the function pointers set by the LLDD */ memcpy (&lport->tt, tt,sizeof (*tt));if (init_fcp &&fc_fcp_init (lport))
@@ -2842,12 +2844,11 @@ int fcoe_libfc_config(struct fc_lport *lport, struct fcoe_ctlr *fip, lport->tt.disc_start = fcoe_ctlr_disc_start; lport->tt.disc_stop = fcoe_ctlr_disc_stop; lport->tt.disc_stop_final = fcoe_ctlr_disc_stop_final; -mutex_init (&lport->disc.disc_mutex); -INIT_LIST_HEAD (&lport->disc.rports); - lport->disc.priv = fip; - }else { -fc_disc_init (lport); + priv = fip; } + +fc_disc_init (lport, priv); +return 0 ; }EXPORT_SYMBOL_GPL (fcoe_libfc_config);
drivers/scsi/libfc/fc_disc.c +3 -2
@@ -714,8 +714,9 @@ static void fc_disc_stop_final(struct fc_lport *lport)/** * fc_disc_init() - Initialize the discovery layer for a local port * @lport: The local port that needs the discovery layer to be initialized +* @priv: Private data structre for users of the discovery layer */ -int fc_disc_init (struct fc_lport *lport) +int fc_disc_init (struct fc_lport *lport,void *priv) {struct fc_disc *disc;
@@ -736,7 +737,7 @@ int fc_disc_init(struct fc_lport *lport)mutex_init (&disc->disc_mutex);INIT_LIST_HEAD (&disc->rports); - disc->priv = lport; + disc->priv = priv;return 0 ; }
include/scsi/libfc.h +1 -1
@@ -1074,7 +1074,7 @@ void fc_rport_terminate_io(struct fc_rport *);/* * DISCOVERY LAYER *****************************/ -int fc_disc_init (struct fc_lport *); +int fc_disc_init (struct fc_lport *,void *);static inline struct fc_lport *fc_disc_lport (struct fc_disc *disc) {
PATCH 4/5 libfc, fcoe, bnx2fc: Split fc_disc_init into fc_disc_{init, config}
In reply to Robert LoveSplit discovery initialization in code that is setup once (fcoe_disc_init) and code that can be re-configured (fcoe_disc_config).
--- drivers/scsi/bnx2fc/bnx2fc_fcoe.c | 3 ++- drivers/scsi/fcoe/fcoe_ctlr.c | 3 ++- drivers/scsi/libfc/fc_disc.c | 25 +++++++++++++++++-------- include/scsi/libfc.h | 3 ++- 4 files changed, 23 insertions(+), 11 deletions(-)
drivers/scsi/bnx2fc/bnx2fc_fcoe.c +2 -1
@@ -939,7 +939,8 @@ static int bnx2fc_libfc_config(struct fc_lport *lport)fc_elsct_init (lport);fc_exch_init (lport);fc_rport_init (lport); -fc_disc_init (lport, lport); +fc_disc_init (lport); +fc_disc_config (lport, lport);return 0 ; }
drivers/scsi/fcoe/fcoe_ctlr.c +2 -1
@@ -2847,7 +2847,8 @@ int fcoe_libfc_config(struct fc_lport *lport, struct fcoe_ctlr *fip, priv = fip; } -fc_disc_init (lport, priv); +fc_disc_init (lport); +fc_disc_config (lport, priv);return 0 ; }
drivers/scsi/libfc/fc_disc.c +17 -8
@@ -712,13 +712,13 @@ static void fc_disc_stop_final(struct fc_lport *lport) }/** -* fc_disc_init() - Initialize the discovery layer for a local port -* @lport: The local port that needs the discovery layer to be initialized +* fc_disc_config() - Configure the discovery layer for a local port +* @lport: The local port that needs the discovery layer to be configured * @priv: Private data structre for users of the discovery layer */ -int fc_disc_init (struct fc_lport *lport,void *priv) +void fc_disc_config (struct fc_lport *lport,void *priv) { -struct fc_disc *disc; +struct fc_disc *disc = &lport->disc;if (!lport->tt.disc_start) lport->tt.disc_start = fc_disc_start;
@@ -733,12 +733,21 @@ int fc_disc_init(struct fc_lport *lport, void *priv) lport->tt.disc_recv_req = fc_disc_recv_req; disc = &lport->disc; -INIT_DELAYED_WORK (&disc->disc_work, fc_disc_timeout); -mutex_init (&disc->disc_mutex); -INIT_LIST_HEAD (&disc->rports); disc->priv = priv; +} +EXPORT_SYMBOL (fc_disc_config); -return 0 ; +/** +* fc_disc_init() - Initialize the discovery layer for a local port +* @lport: The local port that needs the discovery layer to be initialized +*/ +void fc_disc_init (struct fc_lport *lport) +{ +struct fc_disc *disc = &lport->disc; + +INIT_DELAYED_WORK (&disc->disc_work, fc_disc_timeout); +mutex_init (&disc->disc_mutex); +INIT_LIST_HEAD (&disc->rports); }EXPORT_SYMBOL (fc_disc_init);
include/scsi/libfc.h +2 -1
@@ -1074,7 +1074,8 @@ void fc_rport_terminate_io(struct fc_rport *);/* * DISCOVERY LAYER *****************************/ -int fc_disc_init (struct fc_lport *,void *); +void fc_disc_init (struct fc_lport *); +void fc_disc_config (struct fc_lport *,void *);static inline struct fc_lport *fc_disc_lport (struct fc_disc *disc) {
PATCH 5/5 libfcoe: Fix fcoe_sysfs VN2VN mode
In reply to Robert LoveThe libfc discovery layer is being initialized in the 'create' paths for both legacy libfcoe module parameters and fcoe_sysfs control interfaces. The problem is that for VN2VN mode the discovery layer is initialized as if it were in 'fabric' mode and it is not re-configured when the mode is changed to 'vn2vn'.
This patch splits out code that needs to be initialized once and code that can, and should be, re-configured when the mode changes. Additionally this patch makes that change so that the discovery layer can be reconfigured to the libfcoe implementation when in 'vn2vn' mode.
--- drivers/scsi/fcoe/fcoe_ctlr.c | 60 +++++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 15 deletions(-)
drivers/scsi/fcoe/fcoe_ctlr.c +45 -15
@@ -2815,6 +2815,47 @@ unlock: }/** +* fcoe_ctlr_mode_set() - Set or reset the ctlr's mode +* @lport: The local port to be (re)configured +* @fip: The FCoE controller whose mode is changing +* @fip_mode: The new fip mode +* +* Note that the we shouldn't be changing the libfc discovery settings +* (fc_disc_config) while an lport is going through the libfc state +* machine. The mode can only be changed when a fcoe_ctlr device is +* disabled, so that should ensure that this routine is only called +* when nothing is happening. +*/ +void fcoe_ctlr_mode_set (struct fc_lport *lport,struct fcoe_ctlr *fip, +enum fip_state fip_mode) +{ +void *priv; + +WARN_ON (lport->state !=LPORT_ST_RESET && + lport->state !=LPORT_ST_DISABLED ); + +if (fip_mode ==FIP_MODE_VN2VN ) { + lport->rport_priv_size =sizeof (struct fcoe_rport ); + lport->point_to_multipoint =1 ; + lport->tt.disc_recv_req = fcoe_ctlr_disc_recv; + lport->tt.disc_start = fcoe_ctlr_disc_start; + lport->tt.disc_stop = fcoe_ctlr_disc_stop; + lport->tt.disc_stop_final = fcoe_ctlr_disc_stop_final; + priv = fip; + }else { + lport->rport_priv_size =0 ; + lport->point_to_multipoint =0 ; + lport->tt.disc_recv_req =NULL ; + lport->tt.disc_start =NULL ; + lport->tt.disc_stop =NULL ; + lport->tt.disc_stop_final =NULL ; + priv = lport; + } + +fc_disc_config (lport, priv); +} + +/** * fcoe_libfc_config() - Sets up libfc related properties for local port * @lport: The local port to configure libfc for * @fip: The FCoE controller in use by the local port
@@ -2826,8 +2867,6 @@ unlock:int fcoe_libfc_config (struct fc_lport *lport,struct fcoe_ctlr *fip,const struct libfc_function_template *tt,int init_fcp) { -void *priv = lport; -/* Set the function pointers set by the LLDD */ memcpy (&lport->tt, tt,sizeof (*tt));if (init_fcp &&fc_fcp_init (lport))
@@ -2835,21 +2874,9 @@ int fcoe_libfc_config(struct fc_lport *lport, struct fcoe_ctlr *fip,fc_exch_init (lport);fc_elsct_init (lport);fc_lport_init (lport); -if (fip->mode ==FIP_MODE_VN2VN ) - lport->rport_priv_size =sizeof (struct fcoe_rport );fc_rport_init (lport); -if (fip->mode ==FIP_MODE_VN2VN ) { - lport->point_to_multipoint =1 ; - lport->tt.disc_recv_req = fcoe_ctlr_disc_recv; - lport->tt.disc_start = fcoe_ctlr_disc_start; - lport->tt.disc_stop = fcoe_ctlr_disc_stop; - lport->tt.disc_stop_final = fcoe_ctlr_disc_stop_final; - priv = fip; - } -fc_disc_init (lport); -fc_disc_config (lport, priv); - +fcoe_ctlr_mode_set (lport, fip, fip->mode);return 0 ; }EXPORT_SYMBOL_GPL (fcoe_libfc_config);
@@ -2877,6 +2904,7 @@ EXPORT_SYMBOL(fcoe_fcf_get_selected);void fcoe_ctlr_set_fip_mode (struct fcoe_ctlr_device *ctlr_dev) {struct fcoe_ctlr *ctlr =fcoe_ctlr_device_priv (ctlr_dev); +struct fc_lport *lport = ctlr->lp;mutex_lock (&ctlr->ctlr_mutex);switch (ctlr_dev->mode) {
@@ -2890,5 +2918,7 @@ void fcoe_ctlr_set_fip_mode(struct fcoe_ctlr_device *ctlr_dev) }mutex_unlock (&ctlr->ctlr_mutex); + +fcoe_ctlr_mode_set (lport, ctlr, ctlr->mode); }EXPORT_SYMBOL (fcoe_ctlr_set_fip_mode);