NTB: Bug Fixes
Subsystems (2)
cover NTB: Bug Fixes
This series of patches contains a number of bug fixes for the PCI-E Non-Transparent Bridge (NTB) device driver.
Some of these patches have been posted to LKML before, but I wanted all to be available for review prior to sending a pull request.
Thanks,
Jon
Dan Carpenter (2):
ntb: off by one sanity checks
NTB: fix pointer math issues
Jon Mason (8):
NTB: variable dereferenced before check
NTB: Handle 64bit BAR sizes
NTB: Link toggle memory leak
NTB: reset tx_index on link toggle
NTB: Correctly handle receive buffers of the minimal size
NTB: memcpy lockup workaround
ntb_netdev: remove from list on exit
NTB: Multiple NTB client fix
drivers/net/ntb_netdev.c | 2 + drivers/ntb/ntb_hw.c | 10 +-- drivers/ntb/ntb_transport.c | 175 ++++++++++++++++++++++++++++--------------- 3 files changed, 121 insertions(+), 66 deletions(-)
PATCH 1/10 NTB: variable dereferenced before check
Correct instances of variable dereferencing before checking its value on the functions exported to the client drivers. Also, add sanity checks for all exported functions.
--- drivers/ntb/ntb_transport.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-)
drivers/ntb/ntb_transport.c +14 -2
@@ -1210,12 +1210,14 @@ EXPORT_SYMBOL_GPL(ntb_transport_create_queue);*/ void ntb_transport_free_queue (struct ntb_transport_qp *qp) { -struct pci_dev *pdev =ntb_query_pdev (qp->ndev); +struct pci_dev *pdev;struct ntb_queue_entry *entry;if (!qp)return ; + pdev =ntb_query_pdev (qp->ndev); +cancel_delayed_work_sync (&qp->link_work);ntb_unregister_db_callback (qp->ndev, qp->qp_num);
@@ -1371,12 +1373,13 @@ EXPORT_SYMBOL_GPL(ntb_transport_link_up);*/ void ntb_transport_link_down (struct ntb_transport_qp *qp) { -struct pci_dev *pdev =ntb_query_pdev (qp->ndev); +struct pci_dev *pdev;int rc, val;if (!qp)return ; + pdev =ntb_query_pdev (qp->ndev); qp->client_ready =NTB_LINK_DOWN ; rc =ntb_read_local_spad (qp->ndev,QP_LINKS , &val);
@@ -1408,6 +1411,9 @@ EXPORT_SYMBOL_GPL(ntb_transport_link_down);*/ bool ntb_transport_link_query (struct ntb_transport_qp *qp) { +if (!qp) +return false; +return qp->qp_link ==NTB_LINK_UP ; }EXPORT_SYMBOL_GPL (ntb_transport_link_query);
@@ -1422,6 +1428,9 @@ EXPORT_SYMBOL_GPL(ntb_transport_link_query);*/ unsigned char ntb_transport_qp_num (struct ntb_transport_qp *qp) { +if (!qp) +return 0 ; +return qp->qp_num; }EXPORT_SYMBOL_GPL (ntb_transport_qp_num);
@@ -1436,6 +1445,9 @@ EXPORT_SYMBOL_GPL(ntb_transport_qp_num);*/ unsigned int ntb_transport_max_size (struct ntb_transport_qp *qp) { +if (!qp) +return 0 ; +return qp->tx_max_frame -sizeof (struct ntb_payload_header ); }EXPORT_SYMBOL_GPL (ntb_transport_max_size);
Signature · 1 line
1.7.9.5
PATCH 2/10 ntb: off by one sanity checks
In reply to Jon MasonFrom: Dan Carpenter <[email protected]>
These tests are off by one. If "mw" is equal to NTB_NUM_MW then we would go beyond the end of the ndev->mw[] array.
--- drivers/ntb/ntb_hw.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
drivers/ntb/ntb_hw.c +3 -3
@@ -345,7 +345,7 @@ int ntb_read_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 *val)*/ void __iomem *ntb_get_mw_vbase (struct ntb_device *ndev,unsigned int mw) { -if (mw >NTB_NUM_MW ) +if (mw >=NTB_NUM_MW )return NULL ;return ndev->mw[mw].vbase;
@@ -362,7 +362,7 @@ void __iomem *ntb_get_mw_vbase(struct ntb_device *ndev, unsigned int mw)*/ resource_size_t ntb_get_mw_size (struct ntb_device *ndev,unsigned int mw) { -if (mw >NTB_NUM_MW ) +if (mw >=NTB_NUM_MW )return 0 ;return ndev->mw[mw].bar_sz;
@@ -380,7 +380,7 @@ resource_size_t ntb_get_mw_size(struct ntb_device *ndev, unsigned int mw)*/ void ntb_set_mw_addr (struct ntb_device *ndev,unsigned int mw,u64 addr) { -if (mw >NTB_NUM_MW ) +if (mw >=NTB_NUM_MW )return ; dev_dbg(&ndev->pdev->dev, "Writing addr %Lx to BAR %d\n", addr,
Signature · 1 line
1.7.9.5
PATCH 3/10 NTB: fix pointer math issues
In reply to Jon MasonFrom: Dan Carpenter <[email protected]>
->remote_rx_info and ->rx_info are struct ntb_rx_info pointers. If we add sizeof(struct ntb_rx_info) then it goes too far.
--- drivers/ntb/ntb_transport.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
drivers/ntb/ntb_transport.c +2 -2
@@ -486,7 +486,7 @@ static void ntb_transport_setup_qp_mw(struct ntb_transport *nt, (qp_num /NTB_NUM_MW * rx_size); rx_size -=sizeof (struct ntb_rx_info ); - qp->rx_buff = qp->remote_rx_info +sizeof (struct ntb_rx_info ); + qp->rx_buff = qp->remote_rx_info +1 ; qp->rx_max_frame =min (transport_mtu, rx_size); qp->rx_max_entry = rx_size / qp->rx_max_frame; qp->rx_index =0 ;
@@ -780,7 +780,7 @@ static void ntb_transport_init_queue(struct ntb_transport *nt, (qp_num /NTB_NUM_MW * tx_size); tx_size -=sizeof (struct ntb_rx_info ); - qp->tx_mw = qp->rx_info +sizeof (struct ntb_rx_info ); + qp->tx_mw = qp->rx_info +1 ; qp->tx_max_frame =min (transport_mtu, tx_size); qp->tx_max_entry = tx_size / qp->tx_max_frame; qp->tx_index =0 ;
Signature · 1 line
1.7.9.5
PATCH 4/10 NTB: Handle 64bit BAR sizes
In reply to Jon Mason64bit BAR sizes are permissible with an NTB device. To support them various modifications and clean-ups were required, most significantly using 2 32bit scratch pad registers for each BAR.
Also, modify the driver to allow more than 2 Memory Windows.
--- drivers/ntb/ntb_hw.c | 4 +- drivers/ntb/ntb_transport.c | 121 ++++++++++++++++++++++++++----------------- 2 files changed, 75 insertions(+), 50 deletions(-)
drivers/ntb/ntb_hw.c +2 -2
@@ -1027,8 +1027,8 @@ static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) ndev->mw[i].vbase =ioremap_wc (pci_resource_start (pdev,MW_TO_BAR (i)), ndev->mw[i].bar_sz); -dev_info (&pdev->dev,"MW %d size %d\n" , i, - (u32 )pci_resource_len (pdev,MW_TO_BAR (i))); +dev_info (&pdev->dev,"MW %d size %llu\n" , i, +pci_resource_len (pdev,MW_TO_BAR (i)));if (!ndev->mw[i].vbase) {dev_warn (&pdev->dev,"Cannot remap BAR %d\n" ,MW_TO_BAR (i));
drivers/ntb/ntb_transport.c +73 -48
186 lines not on this page · raw message
Signature · 1 line
1.7.9.5
PATCH 5/10 NTB: Link toggle memory leak
In reply to Jon MasonEach link-up will allocate a new NTB receive buffer when the NTB properties are negotiated with the remote system. These allocations did not check for existing buffers and thus did not free them. Now, the driver will check for an existing buffer and free it if not of the correct size, before trying to alloc a new one.
--- drivers/ntb/ntb_transport.c | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-)
drivers/ntb/ntb_transport.c +20 -12
@@ -507,17 +507,37 @@ static void ntb_transport_setup_qp_mw(struct ntb_transport *nt, qp->tx_pkts =0 ; } +static void ntb_free_mw (struct ntb_transport *nt,int num_mw) +{ +struct ntb_transport_mw *mw = &nt->mw[num_mw]; +struct pci_dev *pdev =ntb_query_pdev (nt->ndev); + +if (!mw->virt_addr) +return ; + +dma_free_coherent (&pdev->dev, mw->size, mw->virt_addr, mw->dma_addr); + mw->virt_addr =NULL ; +} +static int ntb_set_mw (struct ntb_transport *nt,int num_mw,unsigned int size) {struct ntb_transport_mw *mw = &nt->mw[num_mw];struct pci_dev *pdev =ntb_query_pdev (nt->ndev); +/* No need to re-setup */ +if (mw->size ==ALIGN (size,4096 )) +return 0 ; + +if (mw->size !=0 ) +ntb_free_mw (nt, num_mw); +/* Alloc memory for receiving data. Must be 4k aligned */ mw->size =ALIGN (size,4096 ); mw->virt_addr =dma_alloc_coherent (&pdev->dev, mw->size, &mw->dma_addr,GFP_KERNEL );if (!mw->virt_addr) { + mw->size =0 ;dev_err (&pdev->dev,"Unable to allocate MW buffer of size %d\n" , (int ) mw->size);return -ENOMEM ;
@@ -529,18 +549,6 @@ static int ntb_set_mw(struct ntb_transport *nt, int num_mw, unsigned int size)return 0 ; } -static void ntb_free_mw (struct ntb_transport *nt,int num_mw) -{ -struct ntb_transport_mw *mw = &nt->mw[num_mw]; -struct pci_dev *pdev =ntb_query_pdev (nt->ndev); - -if (!mw->virt_addr) -return ; - -dma_free_coherent (&pdev->dev, mw->size, mw->virt_addr, mw->dma_addr); - mw->virt_addr =NULL ; -} -static void ntb_qp_link_cleanup (struct work_struct *work) { struct ntb_transport_qp *qp = container_of(work,
Signature · 1 line
1.7.9.5
PATCH 6/10 NTB: reset tx_index on link toggle
In reply to Jon MasonIf the NTB link toggles, the driver could stop receiving due to the tx_index not being set to 0 on the transmitting size on a link-up event. This is due to the driver expecting the incoming data to start at the beginning of the receive buffer and not at a random place.
--- drivers/ntb/ntb_transport.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/ntb/ntb_transport.c +1 -1
@@ -505,6 +505,7 @@ static void ntb_transport_setup_qp_mw(struct ntb_transport *nt, qp->rx_pkts =0 ; qp->tx_pkts =0 ; + qp->tx_index =0 ; }static void ntb_free_mw (struct ntb_transport *nt,int num_mw)
@@ -819,7 +820,6 @@ static void ntb_transport_init_queue(struct ntb_transport *nt, qp->tx_mw = qp->rx_info +1 ; qp->tx_max_frame =min (transport_mtu, tx_size); qp->tx_max_entry = tx_size / qp->tx_max_frame; - qp->tx_index =0 ;if (nt->debugfs_dir) {char debugfs_name[4 ];
Signature · 1 line
1.7.9.5
PATCH 7/10 NTB: Correctly handle receive buffers of the minimal size
In reply to Jon MasonThe ring logic of the NTB receive buffer/transmit memory window requires there to be at least 2 payload sized allotments. For the minimal size case, split the buffer into two and set the transport_mtu to the appropriate size.
--- drivers/ntb/ntb_transport.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-)
drivers/ntb/ntb_transport.c +5 -3
@@ -490,11 +490,12 @@ static void ntb_transport_setup_qp_mw(struct ntb_transport *nt, rx_size -=sizeof (struct ntb_rx_info ); qp->rx_buff = qp->remote_rx_info +1 ; - qp->rx_max_frame =min (transport_mtu, rx_size); +/* Due to housekeeping, there must be atleast 2 buffs */ + qp->rx_max_frame =min (transport_mtu, rx_size /2 ); qp->rx_max_entry = rx_size / qp->rx_max_frame; qp->rx_index =0 ; - qp->remote_rx_info->entry = qp->rx_max_entry; + qp->remote_rx_info->entry = qp->rx_max_entry -1 ;/* setup the hdr offsets with 0's */ for (i =0 ; i < qp->rx_max_entry; i++) {
@@ -818,7 +819,8 @@ static void ntb_transport_init_queue(struct ntb_transport *nt, tx_size -=sizeof (struct ntb_rx_info ); qp->tx_mw = qp->rx_info +1 ; - qp->tx_max_frame =min (transport_mtu, tx_size); +/* Due to housekeeping, there must be atleast 2 buffs */ + qp->tx_max_frame =min (transport_mtu, tx_size /2 ); qp->tx_max_entry = tx_size / qp->tx_max_frame;if (nt->debugfs_dir) {
Signature · 1 line
1.7.9.5
PATCH 8/10 NTB: memcpy lockup workaround
In reply to Jon MasonThe system will appear to lockup for long periods of time due to the NTB driver spending too much time in memcpy. Avoid this by reducing the number of packets that can be serviced on a given interrupt.
--- drivers/ntb/ntb_transport.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-)
drivers/ntb/ntb_transport.c +8 -3
@@ -1034,11 +1034,16 @@ out:static void ntb_transport_rx (unsigned long data) {struct ntb_transport_qp *qp = (struct ntb_transport_qp *)data; -int rc; +int rc, i; -do { +/* Limit the number of packets processed in a single interrupt to +* provide fairness to others +*/ +for (i =0 ; i < qp->rx_max_entry; i++) { rc =ntb_process_rxc (qp); - }while (!rc); +if (rc) +break ; + } }static void ntb_transport_rxc_db (void *data,int db_num)
Signature · 1 line
1.7.9.5
PATCH 9/10 ntb_netdev: remove from list on exit
In reply to Jon MasonThe ntb_netdev device is not removed from the global list of devices upon device removal. If the device is re-added, the removal code would find the first instance and try to remove an already removed device.
--- drivers/net/ntb_netdev.c | 2 ++ 1 file changed, 2 insertions(+)
drivers/net/ntb_netdev.c +2 -0
@@ -375,6 +375,8 @@ static void ntb_netdev_remove(struct pci_dev *pdev)if (dev ==NULL )return ; +list_del (&dev->list); + ndev = dev->ndev;unregister_netdev (ndev);
Signature · 1 line
1.7.9.5
PATCH 10/10 NTB: Multiple NTB client fix
In reply to Jon MasonFix issue with adding multiple ntb client devices to the ntb virtual bus. Previously, multiple devices would be added with the same name, resulting in crashes. To get around this issue, add a unique number to the device when it is added.
--- drivers/ntb/ntb_transport.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
drivers/ntb/ntb_transport.c +3 -2
@@ -300,7 +300,7 @@ int ntb_register_client_dev(char *device_name) {struct ntb_transport_client_dev *client_dev;struct ntb_transport *nt; -int rc; +int rc, i =0 ;if (list_empty (&ntb_transport_list))return -ENODEV ;
@@ -318,7 +318,7 @@ int ntb_register_client_dev(char *device_name) dev = &client_dev->dev;/* setup and register client devices */ -dev_set_name (dev,"%s" , device_name); +dev_set_name (dev,"%s%d" , device_name, i); dev->bus = &ntb_bus_type; dev->release = ntb_client_release; dev->parent = &ntb_query_pdev (nt->ndev)->dev;
@@ -330,6 +330,7 @@ int ntb_register_client_dev(char *device_name) }list_add_tail (&client_dev->entry, &nt->client_devs); + i++; }return 0 ;
Signature · 1 line
1.7.9.5