I/OAT: Add support for DCA - Direct Cache Access
Subsystems (3)
cover I/OAT: Add support for DCA - Direct Cache Access
The following series implements support for providers and clients of Direct Cache Access (DCA), a method for warming the cache in the correct CPU before needing data.
This series applies on GIT commit 5bae7ac9feba925fd0099057f6b23d7be80b7b41
ioat-new-device-ids.patch
- add devices id's for newer Intel chipsets which support DMA and DCA
ioat-rename-source-file.patch
- prepare for adding new functionality
ioat-dma-cleanups.patch
- cleanup some code ugliness
ioat-split-startup-code.patch
- split the DMA support code from the PCI startup
ioat-add-msi-msix-support.patch
- add support for various interrupt handling schemes
ioat-add-dca-support.patch
- add the dca driver
ioat-add-ioat-dca.patch
- add DCA services to the ioatdma driver
Please pull from my git tree at
git://lost.foo-projects.org/~sln/linux-2.6 dca-upstream
Thanks to Dan Williams, Auke Kok, PJ Waskiewicz, and Chris Leech for their help.
sln
--
======================================================================
Mr. Shannon Nelson LAN Access Division, Intel Corp.
[email protected] I don't speak for Intel
(503) 712-7659 Parents can't afford to be squeamish.
PATCH 1/7 I/OAT: New device ids
Add device ids for new revs of the Intel I/OAT DMA engine
---
drivers/dma/ioatdma.c | 5 +++-- include/linux/pci_ids.h | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-)
drivers/dma/ioatdma.c +3 -2
@@ -517,8 +517,9 @@ static enum dma_status ioat_dma_is_complete(struct dma_chan *chan,static struct pci_device_id ioat_pci_tbl[] = { {PCI_DEVICE (PCI_VENDOR_ID_INTEL ,PCI_DEVICE_ID_INTEL_IOAT ) }, - {PCI_DEVICE (PCI_VENDOR_ID_UNISYS , -PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR ) }, + {PCI_DEVICE (PCI_VENDOR_ID_INTEL ,PCI_DEVICE_ID_INTEL_IOAT_CNB ) }, + {PCI_DEVICE (PCI_VENDOR_ID_INTEL ,PCI_DEVICE_ID_INTEL_IOAT_SCNB ) }, + {PCI_DEVICE (PCI_VENDOR_ID_UNISYS ,PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR ) }, {0 , } };
include/linux/pci_ids.h +2 -0
@@ -2258,6 +2258,8 @@#define PCI_DEVICE_ID_INTEL_MCH_PC 0x3599#define PCI_DEVICE_ID_INTEL_MCH_PC1 0x359a#define PCI_DEVICE_ID_INTEL_E7525_MCH 0x359e +#define PCI_DEVICE_ID_INTEL_IOAT_CNB 0x360b +#define PCI_DEVICE_ID_INTEL_IOAT_SCNB 0x65ff#define PCI_DEVICE_ID_INTEL_82371SB_0 0x7000#define PCI_DEVICE_ID_INTEL_82371SB_1 0x7010#define PCI_DEVICE_ID_INTEL_82371SB_2 0x7020
Re: [PATCH 1/7] I/OAT: New device ids
From: Shannon Nelson <[email protected]>
Date: Thu, 19 Jul 2007 17:44:52 -0700
Quoted 3 lines
Add device ids for new revs of the Intel I/OAT DMA engine
Signed-off-by Shannon Nelson <[email protected]>
PATCH 2/7 I/OAT: Rename the source file
In reply to Shannon NelsonRename the ioatdma.c file in preparation for splitting into multiple files, which will allow for easier adding new functionality.
---
drivers/dma/Makefile | 1 drivers/dma/ioat_dma.c | 829 ++++++++++++++++++++++++++++++++++++++++++++++++ drivers/dma/ioatdma.c | 829 ------------------------------------------------ 3 files changed, 830 insertions(+), 829 deletions(-)
drivers/dma/Makefile +1 -0
@@ -1,4 +1,5 @@ obj-$(CONFIG_DMA_ENGINE) += dmaengine.o obj-$(CONFIG_NET_DMA) += iovlock.o obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o +ioatdma-objs := ioat_dma.o obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o
drivers/dma/ioat_dma.c new file +829 -0
829 lines not on this page · raw message
drivers/dma/ioatdma.c deleted +0 -829
829 lines not on this page · raw message
Re: [PATCH 2/7] I/OAT: Rename the source file
From: Shannon Nelson <[email protected]>
Date: Thu, 19 Jul 2007 17:44:57 -0700
Quoted 4 lines
Rename the
ioatdma.cfile in preparation for splitting into multiple files, which will allow for easier adding new functionality.Signed-off-by Shannon Nelson <[email protected]>
PATCH 3/7 I/OAT: code cleanup from checkpatch output
In reply to Shannon NelsonTake care of a bunch of little code nits in ioatdma files
---
drivers/dma/ioat_dma.c | 200 +++++++++++++++++++++++++++--------------------- 1 files changed, 111 insertions(+), 89 deletions(-)
drivers/dma/ioat_dma.c +111 -89
@@ -1,10 +1,10 @@/* -* Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. +* Intel I/OAT DMA Linux driver +* Copyright(c) 2004 - 2007 Intel Corporation. * * This program is free software; you can redistribute it and/or modify it -* under the terms of the GNU General Public License as published by the Free -* Software Foundation; either version 2 of the License, or (at your option) -* any later version. +* under the terms and conditions of the GNU General Public License, +* version 2, as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
@@ -12,11 +12,12 @@* more details. * * You should have received a copy of the GNU General Public License along with -* this program; if not, write to the Free Software Foundation, Inc., 59 -* Temple Place - Suite 330, Boston, MA 02111-1307, USA. +* this program; if not, write to the Free Software Foundation, Inc., +* 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. +* +* The full GNU General Public License is included in this distribution in +* the file called "COPYING". * -* The full GNU General Public License is included in this distribution in the -* file called COPYING. */ /*
@@ -35,17 +36,22 @@#include "ioatdma_registers.h" #include "ioatdma_hw.h" +#define INITIAL_IOAT_DESC_COUNT 128 +#define to_ioat_chan (chan) container_of(chan, struct ioat_dma_chan, common)#define to_ioat_device (dev) container_of(dev, struct ioat_device, common)#define to_ioat_desc (lh) container_of(lh, struct ioat_desc_sw, node)#define tx_to_ioat_desc (tx) container_of(tx, struct ioat_desc_sw, async_tx)/* internal functions */ -static int __devinit ioat_probe (struct pci_dev *pdev,const struct pci_device_id *ent); +static void ioat_dma_start_null_desc (struct ioat_dma_chan *ioat_chan); +static void ioat_dma_memcpy_cleanup (struct ioat_dma_chan *ioat_chan); +static int __devinit ioat_probe (struct pci_dev *pdev, +const struct pci_device_id *ent);static void ioat_shutdown (struct pci_dev *pdev); static void __devexit ioat_remove(struct pci_dev *pdev); -static int enumerate_dma_channels (struct ioat_device *device) +static int ioat_dma_enumerate_channels (struct ioat_device *device) {u8 xfercap_scale;u32 xfercap;
@@ -73,13 +79,14 @@ static int enumerate_dma_channels(struct ioat_device *device)/* This should be made common somewhere in dmaengine.c */ ioat_chan->common.device = &device->common;list_add_tail (&ioat_chan->common.device_node, - &device->common.channels); + &device->common.channels); }return device->common.chancnt; } -static void -ioat_set_src (dma_addr_t addr,struct dma_async_tx_descriptor *tx,int index) +static void ioat_set_src (dma_addr_t addr, +struct dma_async_tx_descriptor *tx, +int index) {struct ioat_desc_sw *iter, *desc =tx_to_ioat_desc (tx);struct ioat_dma_chan *ioat_chan =to_ioat_chan (tx->chan);
@@ -93,8 +100,9 @@ ioat_set_src(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index) } -static void -ioat_set_dest (dma_addr_t addr,struct dma_async_tx_descriptor *tx,int index) +static void ioat_set_dest (dma_addr_t addr, +struct dma_async_tx_descriptor *tx, +int index) {struct ioat_desc_sw *iter, *desc =tx_to_ioat_desc (tx);struct ioat_dma_chan *ioat_chan =to_ioat_chan (tx->chan);
@@ -107,8 +115,7 @@ ioat_set_dest(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index) } } -static dma_cookie_t -ioat_tx_submit (struct dma_async_tx_descriptor *tx) +static dma_cookie_t ioat_tx_submit (struct dma_async_tx_descriptor *tx) {struct ioat_dma_chan *ioat_chan =to_ioat_chan (tx->chan);struct ioat_desc_sw *desc =tx_to_ioat_desc (tx);
@@ -141,13 +148,13 @@ ioat_tx_submit(struct dma_async_tx_descriptor *tx)if (append)writeb (IOAT_CHANCMD_APPEND , ioat_chan->reg_base +IOAT_CHANCMD_OFFSET ); - +return cookie; }static struct ioat_desc_sw *ioat_dma_alloc_descriptor ( -struct ioat_dma_chan *ioat_chan, -gfp_t flags) +struct ioat_dma_chan *ioat_chan, +gfp_t flags) {struct ioat_dma_descriptor *desc;struct ioat_desc_sw *desc_sw;
@@ -177,10 +184,6 @@ static struct ioat_desc_sw *ioat_dma_alloc_descriptor(return desc_sw; } -#define INITIAL_IOAT_DESC_COUNT 128 - -static void ioat_start_null_desc (struct ioat_dma_chan *ioat_chan); -/* returns the actual number of allocated descriptors */ static int ioat_dma_alloc_chan_resources (struct dma_chan *chan) {
@@ -199,16 +202,17 @@ static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)if (chanctrl &IOAT_CHANCTRL_CHANNEL_IN_USE )return -EBUSY ; -/* Setup register to interrupt and write completion status on error */ +/* Setup register to interrupt and write completion status on error */ chanctrl =IOAT_CHANCTRL_CHANNEL_IN_USE |IOAT_CHANCTRL_ERR_INT_EN |IOAT_CHANCTRL_ANY_ERR_ABORT_EN |IOAT_CHANCTRL_ERR_COMPLETION_EN ; -writew (chanctrl, ioat_chan->reg_base +IOAT_CHANCTRL_OFFSET ); +writew (chanctrl, ioat_chan->reg_base +IOAT_CHANCTRL_OFFSET ); chanerr =readl (ioat_chan->reg_base +IOAT_CHANERR_OFFSET );if (chanerr) { -printk ("IOAT: CHANERR = %x, clearing\n" , chanerr); +dev_err (&ioat_chan->device->pdev->dev, +"ioatdma: CHANERR = %x, clearing\n" , chanerr);writel (chanerr, ioat_chan->reg_base +IOAT_CHANERR_OFFSET ); }
@@ -216,7 +220,8 @@ static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)for (i =0 ; i <INITIAL_IOAT_DESC_COUNT ; i++) { desc =ioat_dma_alloc_descriptor (ioat_chan,GFP_KERNEL );if (!desc) { -printk (KERN_ERR "IOAT: Only %d initial descriptors\n" , i); +dev_err (&ioat_chan->device->pdev->dev, +"ioatdma: Only %d initial descriptors\n" , i);break ; }list_add_tail (&desc->node, &tmp_list);
@@ -229,8 +234,8 @@ static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)/* doing 2 32bit writes to mmio since 1 64b write doesn't work */ ioat_chan->completion_virt =pci_pool_alloc (ioat_chan->device->completion_pool, -GFP_KERNEL , - &ioat_chan->completion_addr); +GFP_KERNEL , + &ioat_chan->completion_addr);memset (ioat_chan->completion_virt,0 ,sizeof (*ioat_chan->completion_virt)); writel(((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF,
@@ -238,12 +243,10 @@ static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)writel (((u64 ) ioat_chan->completion_addr) >>32 , ioat_chan->reg_base +IOAT_CHANCMP_OFFSET_HIGH ); -ioat_start_null_desc (ioat_chan); +ioat_dma_start_null_desc (ioat_chan);return i; } -static void ioat_dma_memcpy_cleanup (struct ioat_dma_chan *ioat_chan); -static void ioat_dma_free_chan_resources (struct dma_chan *chan) {struct ioat_dma_chan *ioat_chan =to_ioat_chan (chan);
@@ -273,12 +276,13 @@ static void ioat_dma_free_chan_resources(struct dma_chan *chan)spin_unlock_bh (&ioat_chan->desc_lock);pci_pool_free (ioat_device->completion_pool, - ioat_chan->completion_virt, - ioat_chan->completion_addr); + ioat_chan->completion_virt, + ioat_chan->completion_addr);/* one is ok since we left it on there on purpose */ if (in_use_descs >1 ) -printk (KERN_ERR "IOAT: Freeing %d in use descriptors!\n" , +dev_err (&ioat_chan->device->pdev->dev, +"ioatdma: Freeing %d in use descriptors!\n" , in_use_descs -1 ); ioat_chan->last_completion = ioat_chan->completion_addr =0 ;
@@ -289,8 +293,10 @@ static void ioat_dma_free_chan_resources(struct dma_chan *chan)writew (chanctrl, ioat_chan->reg_base +IOAT_CHANCTRL_OFFSET ); } -static struct dma_async_tx_descriptor * -ioat_dma_prep_memcpy (struct dma_chan *chan,size_t len,int int_en) +static struct dma_async_tx_descriptor *ioat_dma_prep_memcpy ( +struct dma_chan *chan, +size_t len, +int int_en) {struct ioat_dma_chan *ioat_chan =to_ioat_chan (chan);struct ioat_desc_sw *first, *prev, *new;
@@ -354,12 +360,11 @@ ioat_dma_prep_memcpy(struct dma_chan *chan, size_t len, int int_en)return new ? &new->async_tx :NULL ; } -/** -* ioat_dma_memcpy_issue_pending - push potentially unrecognized appended descriptors to hw +* ioat_dma_memcpy_issue_pending - push potentially unrecognized appended +* descriptors to hw * @chan: DMA channel handle */ -static void ioat_dma_memcpy_issue_pending (struct dma_chan *chan) {struct ioat_dma_chan *ioat_chan =to_ioat_chan (chan);
@@ -371,15 +376,15 @@ static void ioat_dma_memcpy_issue_pending(struct dma_chan *chan) } } -static void ioat_dma_memcpy_cleanup (struct ioat_dma_chan *chan) +static void ioat_dma_memcpy_cleanup (struct ioat_dma_chan *ioat_chan) {unsigned long phys_complete;struct ioat_desc_sw *desc, *_desc;dma_cookie_t cookie =0 ; -prefetch (chan->completion_virt); +prefetch (ioat_chan->completion_virt); -if (!spin_trylock (&chan->cleanup_lock)) +if (!spin_trylock (&ioat_chan->cleanup_lock))return ;/* The completion writeback can happen at any time,
@@ -389,26 +394,27 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan)#if (BITS_PER_LONG ==64 ) phys_complete = - chan->completion_virt->full &IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR ; + ioat_chan->completion_virt->full &IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR ;#else - phys_complete = chan->completion_virt->low &IOAT_LOW_COMPLETION_MASK ; + phys_complete = ioat_chan->completion_virt->low &IOAT_LOW_COMPLETION_MASK ;#endif -if ((chan->completion_virt->full &IOAT_CHANSTS_DMA_TRANSFER_STATUS ) == -IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED ) { -printk ("IOAT: Channel halted, chanerr = %x\n" , -readl (chan->reg_base +IOAT_CHANERR_OFFSET )); +if ((ioat_chan->completion_virt->full &IOAT_CHANSTS_DMA_TRANSFER_STATUS ) == +IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED ) { +dev_err (&ioat_chan->device->pdev->dev, +"ioatdma: Channel halted, chanerr = %x\n" , +readl (ioat_chan->reg_base +IOAT_CHANERR_OFFSET ));/* TODO do something to salvage the situation */ } -if (phys_complete == chan->last_completion) { -spin_unlock (&chan->cleanup_lock); +if (phys_complete == ioat_chan->last_completion) { +spin_unlock (&ioat_chan->cleanup_lock);return ; } -spin_lock_bh (&chan->desc_lock); -list_for_each_entry_safe (desc, _desc, &chan->used_desc, node) { +spin_lock_bh (&ioat_chan->desc_lock); +list_for_each_entry_safe (desc, _desc, &ioat_chan->used_desc, node) {/* * Incoming DMA requests may use multiple descriptors, due to
@@ -418,31 +424,36 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan)if (desc->async_tx.cookie) { cookie = desc->async_tx.cookie; -/* yes we are unmapping both _page and _single alloc'd -regions with unmap_page. Is this *really* that bad? -*/ -pci_unmap_page (chan->device->pdev, +/* +* yes we are unmapping both _page and _single alloc'd +* regions with unmap_page. Is this *really* that bad? +*/ +pci_unmap_page (ioat_chan->device->pdev,pci_unmap_addr (desc, dst),pci_unmap_len (desc, dst_len),PCI_DMA_FROMDEVICE ); -pci_unmap_page (chan->device->pdev, +pci_unmap_page (ioat_chan->device->pdev,pci_unmap_addr (desc, src),pci_unmap_len (desc, src_len),PCI_DMA_TODEVICE ); }if (desc->async_tx.phys != phys_complete) { -/* a completed entry, but not the last, so cleanup +/* +* a completed entry, but not the last, so cleanup * if the client is done with the descriptor */ if (desc->async_tx.ack) {list_del (&desc->node); -list_add_tail (&desc->node, &chan->free_desc); +list_add_tail (&desc->node, + &ioat_chan->free_desc); }else desc->async_tx.cookie =0 ; }else { -/* last used desc. Do not remove, so we can append from -it, but don't look at it next time, either */ +/* +* last used desc. Do not remove, so we can append from +* it, but don't look at it next time, either +*/ desc->async_tx.cookie =0 ;/* TODO check status bits? */
@@ -450,13 +461,13 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan) } } -spin_unlock_bh (&chan->desc_lock); +spin_unlock_bh (&ioat_chan->desc_lock); - chan->last_completion = phys_complete; + ioat_chan->last_completion = phys_complete;if (cookie !=0 ) - chan->completed_cookie = cookie; + ioat_chan->completed_cookie = cookie; -spin_unlock (&chan->cleanup_lock); +spin_unlock (&ioat_chan->cleanup_lock); }static void ioat_dma_dependency_added (struct dma_chan *chan)
@@ -477,11 +488,10 @@ static void ioat_dma_dependency_added(struct dma_chan *chan)* @done: if not %NULL, updated with last completed transaction * @used: if not %NULL, updated with last used transaction */ -static enum dma_status ioat_dma_is_complete (struct dma_chan *chan, -dma_cookie_t cookie, -dma_cookie_t *done, -dma_cookie_t *used) +dma_cookie_t cookie, +dma_cookie_t *done, +dma_cookie_t *used) {struct ioat_dma_chan *ioat_chan =to_ioat_chan (chan);dma_cookie_t last_used;
@@ -492,7 +502,7 @@ static enum dma_status ioat_dma_is_complete(struct dma_chan *chan, last_complete = ioat_chan->completed_cookie;if (done) - *done= last_complete; + *done = last_complete;if (used) *used = last_used;
@@ -506,7 +516,7 @@ static enum dma_status ioat_dma_is_complete(struct dma_chan *chan, last_complete = ioat_chan->completed_cookie;if (done) - *done= last_complete; + *done = last_complete;if (used) *used = last_used;
@@ -549,13 +559,13 @@ static irqreturn_t ioat_do_interrupt(int irq, void *data) attnstatus =readl (instance->reg_base +IOAT_ATTNSTATUS_OFFSET ); -printk (KERN_ERR "ioatdma error: interrupt! status %lx\n" , attnstatus); +printk (KERN_ERR "ioatdma: interrupt! status %lx\n" , attnstatus);writeb (intrctrl, instance->reg_base +IOAT_INTRCTRL_OFFSET );return IRQ_HANDLED ; } -static void ioat_start_null_desc (struct ioat_dma_chan *ioat_chan) +static void ioat_dma_start_null_desc (struct ioat_dma_chan *ioat_chan) {struct ioat_desc_sw *desc;
@@ -619,9 +629,11 @@ static int ioat_self_test(struct ioat_device *device)/* Start copy, using first DMA channel */ dma_chan =container_of (device->common.channels.next, - struct dma_chan, - device_node); + struct dma_chan, + device_node);if (ioat_dma_alloc_chan_resources (dma_chan) <1 ) { +dev_err (&device->pdev->dev, +"selftest cannot allocate chan resource\n" ); err = -ENODEV ; goto out; }
@@ -639,12 +651,14 @@ static int ioat_self_test(struct ioat_device *device)msleep (1 );if (ioat_dma_is_complete (dma_chan, cookie,NULL ,NULL ) !=DMA_SUCCESS ) { -printk (KERN_ERR "ioatdma: Self-test copy timed out, disabling\n" ); +dev_err (&device->pdev->dev, +"ioatdma: Self-test copy timed out, disabling\n" ); err = -ENODEV ; goto free_resources; }if (memcmp (src, dest,IOAT_TEST_SIZE )) { -printk (KERN_ERR "ioatdma: Self-test copy failed compare, disabling\n" ); +dev_err (&device->pdev->dev, +"ioatdma: Self-test copy failed compare, disabling\n" ); err = -ENODEV ; goto free_resources; }
@@ -658,7 +672,7 @@ out: }static int __devinit ioat_probe (struct pci_dev *pdev, -const struct pci_device_id *ent) +const struct pci_device_id *ent) {int err;unsigned long mmio_start, mmio_len;
@@ -702,7 +716,9 @@ static int __devinit ioat_probe(struct pci_dev *pdev, goto err_dma_pool; } - device->completion_pool =pci_pool_create ("completion_pool" , pdev,sizeof (u64),SMP_CACHE_BYTES ,SMP_CACHE_BYTES ); + device->completion_pool =pci_pool_create ("completion_pool" , pdev, +sizeof (u64),SMP_CACHE_BYTES , +SMP_CACHE_BYTES );if (!device->completion_pool) { err = -ENOMEM ; goto err_completion_pool;
@@ -724,22 +740,26 @@ static int __devinit ioat_probe(struct pci_dev *pdev, device->reg_base = reg_base; -writeb (IOAT_INTRCTRL_MASTER_INT_EN , device->reg_base +IOAT_INTRCTRL_OFFSET ); +writeb (IOAT_INTRCTRL_MASTER_INT_EN , + device->reg_base +IOAT_INTRCTRL_OFFSET );pci_set_master (pdev);INIT_LIST_HEAD (&device->common.channels); -enumerate_dma_channels (device); +ioat_dma_enumerate_channels (device);dma_cap_set (DMA_MEMCPY , device->common.cap_mask); - device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources; - device->common.device_free_chan_resources = ioat_dma_free_chan_resources; + device->common.device_alloc_chan_resources = + ioat_dma_alloc_chan_resources; + device->common.device_free_chan_resources = + ioat_dma_free_chan_resources; device->common.device_prep_dma_memcpy = ioat_dma_prep_memcpy; device->common.device_is_tx_complete = ioat_dma_is_complete; device->common.device_issue_pending = ioat_dma_memcpy_issue_pending; device->common.device_dependency_added = ioat_dma_dependency_added; device->common.dev = &pdev->dev; -printk (KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n" , - device->common.chancnt); +printk (KERN_INFO " " +"ioatdma: Intel(R) I/OAT DMA Engine found, %d channels\n" , + device->common.chancnt); err =ioat_self_test (device);if (err)
@@ -765,7 +785,8 @@ err_set_dma_mask:pci_disable_device (pdev); err_enable_device: -printk (KERN_ERR "Intel(R) I/OAT DMA Engine initialization failed\n" ); +printk (KERN_INFO " " +"ioatdma: Intel(R) I/OAT DMA Engine initialization failed\n" );return err; }
@@ -797,7 +818,8 @@ static void __devexit ioat_remove(struct pci_dev *pdev)iounmap (device->reg_base);pci_release_regions (pdev);pci_disable_device (pdev); -list_for_each_entry_safe (chan, _chan, &device->common.channels, device_node) { +list_for_each_entry_safe (chan, _chan, + &device->common.channels, device_node) { ioat_chan =to_ioat_chan (chan);list_del (&chan->device_node);kfree (ioat_chan);
Re: [PATCH 3/7] I/OAT: code cleanup from checkpatch output
From: Shannon Nelson <[email protected]>
Date: Thu, 19 Jul 2007 17:45:02 -0700
Quoted 3 lines
Take care of a bunch of little code nits in ioatdma files
Signed-off-by Shannon Nelson <[email protected]>
PATCH 4/7 I/OAT: Split PCI startup from DMA handling code
In reply to Shannon NelsonSplit the general PCI startup from the DMA handling code in order to prepare for adding support for DCA services and future versions of the ioatdma device.
---
drivers/dma/Makefile | 2 drivers/dma/ioat.c | 186 ++++++++++++++++++++++++++++++++++++++++++++ drivers/dma/ioat_dma.c | 196 +++++++++++----------------------------------- drivers/dma/ioatdma.h | 16 +++- drivers/dma/ioatdma_hw.h | 2 5 files changed, 245 insertions(+), 157 deletions(-)
drivers/dma/Makefile +1 -1
@@ -1,5 +1,5 @@ obj-$(CONFIG_DMA_ENGINE) += dmaengine.o obj-$(CONFIG_NET_DMA) += iovlock.o obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o -ioatdma-objs := ioat_dma.o +ioatdma-objs := ioat.o ioat_dma.o obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o
drivers/dma/ioat.c new file +186 -0
186 lines not on this page · raw message
drivers/dma/ioat_dma.c +45 -151
331 lines not on this page · raw message
drivers/dma/ioatdma.h +12 -4
@@ -31,7 +31,7 @@#define IOAT_LOW_COMPLETION_MASK 0xffffffc0/** -* struct ioat_device - internal representation of a IOAT device +* struct ioatdma_device - internal representation of a IOAT device * @pdev: PCI-Express device * @reg_base: MMIO register space base address * @dma_pool: for allocating DMA descriptors
@@ -39,14 +39,14 @@* @msi: Message Signaled Interrupt number */ -struct ioat_device { +struct ioatdma_device {struct pci_dev *pdev;void __iomem *reg_base;struct pci_pool *dma_pool;struct pci_pool *completion_pool;struct dma_device common; -u8 msi; +u8 version; };/**
@@ -84,7 +84,7 @@ struct ioat_dma_chan {int pending; -struct ioat_device *device; +struct ioatdma_device *device;struct dma_chan common;dma_addr_t completion_addr;
@@ -118,4 +118,12 @@ struct ioat_desc_sw { struct dma_async_tx_descriptor async_tx; }; +#if defined(CONFIG_INTEL_IOATDMA ) || defined(CONFIG_INTEL_IOATDMA_MODULE ) +struct ioatdma_device *ioat_dma_probe (struct pci_dev *,void __iomem *); +void ioat_dma_remove (struct ioatdma_device *device); +#else +#define ioat_dma_probe (pdev, io) NULL +#define ioat_dma_remove (dev) do { } while (0) +#endif +#endif /* IOATDMA_H */
drivers/dma/ioatdma_hw.h +1 -1
@@ -27,7 +27,7 @@#define IOAT_PCI_RID 0x00#define IOAT_PCI_SVID 0x8086#define IOAT_PCI_SID 0x8086 -#define IOAT_VER 0x12/* Version 1.2 */ +#define IOAT_VER_1_2 0x12/* Version 1.2 */ struct ioat_dma_descriptor {uint32_t size;
Re: [PATCH 4/7] I/OAT: Split PCI startup from DMA handling code
From: Shannon Nelson <[email protected]>
Date: Thu, 19 Jul 2007 17:45:07 -0700
Quoted 5 lines
Split the general PCI startup from the DMA handling code in order to prepare for adding support for DCA services and future versions of the ioatdma device.
Signed-off-by Shannon Nelson <[email protected]>
Re: [PATCH 4/7] I/OAT: Split PCI startup from DMA handling code
In reply to Shannon NelsonQuoted 67 lines · Shannon Nelson
Split the general PCI startup from the DMA handling code in order to prepare for adding support for DCA services and future versions of the ioatdma device.
Signed-off-by Shannon Nelson <[email protected]>---
drivers/dma/Makefile | 2 drivers/dma/ioat.c | 186 ++++++++++++++++++++++++++++++++++++++++++++ drivers/dma/ioat_dma.c | 196 +++++++++++----------------------------------- drivers/dma/ioatdma.h | 16 +++- drivers/dma/ioatdma_hw.h | 2 5 files changed, 245 insertions(+), 157 deletions(-)drivers/dma/Makefile +1
-1@@ -1,5 +1,5 @@ obj-$(CONFIG_DMA_ENGINE) += dmaengine.o obj-$(CONFIG_NET_DMA) += iovlock.o obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o -ioatdma-objs := ioat_dma.o +ioatdma-objs := ioat.o ioat_dma.o obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.odrivers/dma/ioat.c new file +36
-0@@ -0,0 +1,186 @@ +/* +* Intel I/OAT DMA Linux driver +* Copyright(c) 2004 - 2007 Intel Corporation. +* +* This program is free software; you can redistribute it and/or modify it +* under the terms and conditions of the GNU General Public License, +* version 2, as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, but WITHOUT +* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +* more details. +* +* You should have received a copy of the GNU General Public License along with +* this program; if not, write to the Free Software Foundation, Inc., +* 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. +* +* The full GNU General Public License is included in this distribution in +* the file called "COPYING". +* +*/ + +/* +* This driver supports an Intel I/OAT DMA engine, which does asynchronous +* copy operations. +*/ + +#include <linux/init.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/interrupt.h> +#include "ioatdma.h" +#include "ioatdma_registers.h" +#include "ioatdma_hw.h" + +MODULE_VERSION ("1.24" );
drivers/dma/ioat.c line 46
Quoted 10 lines
drivers/dma/ioat.c new file +10
-0+MODULE_LICENSE ("GPL" ); +MODULE_AUTHOR ("Intel Corporation" ); + +static struct pci_device_id ioat_pci_tbl[] = { + {PCI_DEVICE (PCI_VENDOR_ID_INTEL ,PCI_DEVICE_ID_INTEL_IOAT ) }, + {PCI_DEVICE (PCI_VENDOR_ID_INTEL ,PCI_DEVICE_ID_INTEL_IOAT_CNB ) }, + {PCI_DEVICE (PCI_VENDOR_ID_INTEL ,PCI_DEVICE_ID_INTEL_IOAT_SCNB ) }, + {PCI_DEVICE (PCI_VENDOR_ID_UNISYS ,PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR ) }, + {0 , } +};
Why this driver lacks MODULE_DEVICE_TABLE() ? Is it intentionaly omitted ?
Signature · 2 lines
Andrey Panin | Linux and UNIX system administrator [email protected] | PGP key: wwwkeys.pgp.net
RE: [PATCH 4/7] I/OAT: Split PCI startup from DMA handling code
Andrey Panin [mailto:[email protected]]
Quoted 3 lines
Why this driver lacks
MODULE_DEVICE_TABLE()? Is it intentionaly omitted ?
Hmmm - good catch. It has been missing for a long time in the code. I'll double check with the original developer to see if it was left out on purpose and probably add it in a later patch.
Thanks,
sln
--
======================================================================
Mr. Shannon Nelson LAN Access Division, Intel Corp.
[email protected] I don't speak for Intel
(503) 712-7659 Parents can't afford to be squeamish.
PATCH 5/7 I/OAT: Add support for MSI and MSI-X
In reply to Shannon NelsonAdd support for MSI and MSI-X interrupt handling, including the ability to choose the desired interrupt method.
---
drivers/dma/ioat_dma.c | 353 ++++++++++++++++++++++++++++++++------- drivers/dma/ioatdma.h | 12 + drivers/dma/ioatdma_registers.h | 6 + 3 files changed, 305 insertions(+), 66 deletions(-)
drivers/dma/ioat_dma.c +288 -65
@@ -47,6 +47,71 @@static void ioat_dma_start_null_desc (struct ioat_dma_chan *ioat_chan);static void ioat_dma_memcpy_cleanup (struct ioat_dma_chan *ioat_chan); +#define for_each_bit (bit, addr, size) \ + for ((bit) = find_first_bit((addr), (size)); \ + (bit) < (size); \ + (bit) = find_next_bit((addr), (size), (bit) + 1)) + + +struct ioat_dma_chan *ioat_lookup_chan_by_index (struct ioatdma_device *device, +int index) +{ +return device->idx[index]; +} + +/** +* ioat_dma_do_interrupt - handler used for single vector interrupt mode +* @irq: interrupt id +* @data: interrupt data +*/ +static irqreturn_t ioat_dma_do_interrupt (int irq,void *data) +{ +struct ioatdma_device *instance = data; +struct ioat_dma_chan *ioat_chan; +unsigned long attnstatus; +int bit; +u8 intrctrl; + + intrctrl =readb (instance->reg_base +IOAT_INTRCTRL_OFFSET ); + +if (!(intrctrl &IOAT_INTRCTRL_MASTER_INT_EN )) +return IRQ_NONE ; + +if (!(intrctrl &IOAT_INTRCTRL_INT_STATUS )) { +writeb (intrctrl, instance->reg_base +IOAT_INTRCTRL_OFFSET ); +return IRQ_NONE ; + } + + attnstatus =readl (instance->reg_base +IOAT_ATTNSTATUS_OFFSET ); + for_each_bit (bit, &attnstatus, BITS_PER_LONG) { + ioat_chan =ioat_lookup_chan_by_index (instance, bit); +tasklet_schedule (&ioat_chan->cleanup_task); + } + +writeb (intrctrl, instance->reg_base +IOAT_INTRCTRL_OFFSET ); +return IRQ_HANDLED ; +} + +/** +* ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode +* @irq: interrupt id +* @data: interrupt data +*/ +static irqreturn_t ioat_dma_do_interrupt_msix (int irq,void *data) +{ +struct ioat_dma_chan *ioat_chan = data; + +tasklet_schedule (&ioat_chan->cleanup_task); + +return IRQ_HANDLED ; +} + +static void ioat_dma_cleanup_tasklet (unsigned long data); + +/** +* ioat_dma_enumerate_channels - find and initialize the device's channels +* @device: the device to be enumerated +*/ static int ioat_dma_enumerate_channels (struct ioatdma_device *device) {u8 xfercap_scale;
@@ -76,6 +141,11 @@ static int ioat_dma_enumerate_channels(struct ioatdma_device *device) ioat_chan->common.device = &device->common;list_add_tail (&ioat_chan->common.device_node, &device->common.channels); + device->idx[i] = ioat_chan; +tasklet_init (&ioat_chan->cleanup_task, + ioat_dma_cleanup_tasklet, + (unsigned long ) ioat_chan); +tasklet_disable (&ioat_chan->cleanup_task); }return device->common.chancnt; }
@@ -234,6 +304,7 @@ static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)writel (((u64 ) ioat_chan->completion_addr) >>32 , ioat_chan->reg_base +IOAT_CHANCMP_OFFSET_HIGH ); +tasklet_enable (&ioat_chan->cleanup_task);ioat_dma_start_null_desc (ioat_chan);return i; }
@@ -245,9 +316,14 @@ static void ioat_dma_free_chan_resources(struct dma_chan *chan)struct ioat_desc_sw *desc, *_desc;int in_use_descs =0 ; +tasklet_disable (&ioat_chan->cleanup_task);ioat_dma_memcpy_cleanup (ioat_chan); +/* Delay 100ms after reset to allow internal DMA logic to quiesce +* before removing DMA descriptor resources. +*/ writeb (IOAT_CHANCMD_RESET , ioat_chan->reg_base +IOAT_CHANCMD_OFFSET ); +mdelay (100 );spin_lock_bh (&ioat_chan->desc_lock);list_for_each_entry_safe (desc, _desc, &ioat_chan->used_desc, node) {
@@ -276,6 +352,34 @@ static void ioat_dma_free_chan_resources(struct dma_chan *chan) in_use_descs - 1); ioat_chan->last_completion = ioat_chan->completion_addr =0 ; + ioat_chan->pending =0 ; +} +/** +* ioat_dma_get_next_descriptor - return the next available descriptor from +* the chain +* @ioat_chan: IOAT DMA channel handle +* Gets the next descriptor from the chain, and must be called with the +* channel's desc_lock held. Allocates more descriptors if the channel +* has run out. +*/ +static struct ioat_desc_sw *ioat_dma_get_next_descriptor ( +struct ioat_dma_chan *ioat_chan) +{ +struct ioat_desc_sw *new =NULL ; + +if (!list_empty (&ioat_chan->free_desc)) { + new =to_ioat_desc (ioat_chan->free_desc.next); +list_del (&new->node); + }else { +/* try to get another desc */ + new =ioat_dma_alloc_descriptor (ioat_chan,GFP_ATOMIC ); +/* will this ever happen? */ +/* TODO add upper limit on these */ +BUG_ON (!new); + } + +prefetch (new->hw); +return new; } static struct dma_async_tx_descriptor *ioat_dma_prep_memcpy(
@@ -300,17 +404,7 @@ static struct dma_async_tx_descriptor *ioat_dma_prep_memcpy(spin_lock_bh (&ioat_chan->desc_lock);while (len) { -if (!list_empty (&ioat_chan->free_desc)) { - new =to_ioat_desc (ioat_chan->free_desc.next); -list_del (&new->node); - }else { -/* try to get another desc */ - new =ioat_dma_alloc_descriptor (ioat_chan,GFP_ATOMIC ); -/* will this ever happen? */ -/* TODO add upper limit on these */ -BUG_ON (!new); - } - + new =ioat_dma_get_next_descriptor (ioat_chan); copy =min ((u32 ) len, ioat_chan->xfercap); new->hw->size = copy;
@@ -361,6 +455,14 @@ static void ioat_dma_memcpy_issue_pending(struct dma_chan *chan) } } +static void ioat_dma_cleanup_tasklet (unsigned long data) +{ +struct ioat_dma_chan *chan = (void *)data; +ioat_dma_memcpy_cleanup (chan); +writew (IOAT_CHANCTRL_INT_DISABLE , + chan->reg_base +IOAT_CHANCTRL_OFFSET ); +} +static void ioat_dma_memcpy_cleanup (struct ioat_dma_chan *ioat_chan) {unsigned long phys_complete;
@@ -398,6 +500,7 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan)return ; } + cookie =0 ;spin_lock_bh (&ioat_chan->desc_lock);list_for_each_entry_safe (desc, _desc, &ioat_chan->used_desc, node) {
@@ -510,48 +613,13 @@ static enum dma_status ioat_dma_is_complete(struct dma_chan *chan,/* PCI API */ -static irqreturn_t ioat_do_interrupt (int irq,void *data) -{ -struct ioatdma_device *instance = data; -unsigned long attnstatus; -u8 intrctrl; - - intrctrl =readb (instance->reg_base +IOAT_INTRCTRL_OFFSET ); - -if (!(intrctrl &IOAT_INTRCTRL_MASTER_INT_EN )) -return IRQ_NONE ; - -if (!(intrctrl &IOAT_INTRCTRL_INT_STATUS )) { -writeb (intrctrl, instance->reg_base +IOAT_INTRCTRL_OFFSET ); -return IRQ_NONE ; - } - - attnstatus =readl (instance->reg_base +IOAT_ATTNSTATUS_OFFSET ); - -printk (KERN_ERR "ioatdma: interrupt! status %lx\n" , attnstatus); - -writeb (intrctrl, instance->reg_base +IOAT_INTRCTRL_OFFSET ); -return IRQ_HANDLED ; -} -static void ioat_dma_start_null_desc (struct ioat_dma_chan *ioat_chan) {struct ioat_desc_sw *desc;spin_lock_bh (&ioat_chan->desc_lock); -if (!list_empty (&ioat_chan->free_desc)) { - desc =to_ioat_desc (ioat_chan->free_desc.next); -list_del (&desc->node); - }else { -/* try to get another desc */ -spin_unlock_bh (&ioat_chan->desc_lock); - desc =ioat_dma_alloc_descriptor (ioat_chan,GFP_KERNEL ); -spin_lock_bh (&ioat_chan->desc_lock); -/* will this ever happen? */ -BUG_ON (!desc); - } - + desc =ioat_dma_get_next_descriptor (ioat_chan); desc->hw->ctl =IOAT_DMA_DESCRIPTOR_NUL ; desc->hw->next =0 ; desc->async_tx.ack =1 ;
@@ -572,7 +640,11 @@ static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat_chan)*/ #define IOAT_TEST_SIZE 2000 -static int ioat_self_test (struct ioatdma_device *device) +/** +* ioat_dma_self_test - Perform a IOAT transaction to verify the HW works. +* @device: device to be tested +*/ +static int ioat_dma_self_test (struct ioatdma_device *device) {int i;u8 *src;
@@ -640,6 +712,162 @@ out:return err; } +static char ioat_interrupt_style[32 ]; +module_param_string (ioat_interrupt_style, ioat_interrupt_style, +sizeof (ioat_interrupt_style),0644 ); +MODULE_PARM_DESC (ioat_interrupt_style, +"set ioat interrupt style: msix (default), " +"msix-single-vector, msi, intx)" ); + +/** +* ioat_dma_setup_interrupts - setup interrupt handler, choosing btwn msix, +* msi, and legacy +* @device: ioat device +*/ +int ioat_dma_setup_interrupts (struct ioatdma_device *device) +{ +struct ioat_dma_chan *ioat_chan; +int err, i, j, msixcnt; +u8 intrctrl =0 ; + +if (!strcmp (ioat_interrupt_style,"msix" )) + goto msix; +else if (!strcmp (ioat_interrupt_style,"msix-single-vector" )) + goto msix_single_vector; +else if (!strcmp (ioat_interrupt_style,"msi" )) + goto msi; +else if (!strcmp (ioat_interrupt_style,"intx" )) + goto intx; + +msix: +/* The number of MSI-X vectors should equal the number of channels */ + msixcnt = device->common.chancnt; +for (i =0 ; i < msixcnt; i++) + device->msix_entries[i].entry = i; + + err =pci_enable_msix (device->pdev, device->msix_entries, msixcnt); +if (err <0 ) + goto msi; +if (err >0 ) + goto msix_single_vector; + +for (i =0 ; i < msixcnt; i++) { + ioat_chan =ioat_lookup_chan_by_index (device, i); + err =request_irq (device->msix_entries[i].vector, + ioat_dma_do_interrupt_msix, +0 ,"ioat-msix" , ioat_chan); +if (err) { +for (j =0 ; j < i; j++) { + ioat_chan = +ioat_lookup_chan_by_index (device, j); +free_irq (device->msix_entries[j].vector, + ioat_chan); + } + goto msix_single_vector; + } + } + intrctrl |=IOAT_INTRCTRL_MSIX_VECTOR_CONTROL ; + device->irq_mode = msix_multi_vector; + goto done; + +msix_single_vector: + device->msix_entries[0 ].entry =0 ; + err =pci_enable_msix (device->pdev, device->msix_entries,1 ); +if (err) + goto msi; + + err =request_irq (device->msix_entries[0 ].vector, ioat_dma_do_interrupt, +0 ,"ioat-msix" , device); +if (err) { +pci_disable_msix (device->pdev); + goto msi; + } + device->irq_mode = msix_single_vector; + goto done; + +msi: + err =pci_enable_msi (device->pdev); +if (err) + goto intx; + + err =request_irq (device->pdev->irq, ioat_dma_do_interrupt, +0 ,"ioat-msi" , device); +if (err) { +pci_disable_msi (device->pdev); + goto intx; + } +/* +* CB 1.2 devices need a bit set in configuration space to enable MSI +*/ +if (device->version ==IOAT_VER_1_2 ) { +u32 dmactrl; +pci_read_config_dword (device->pdev, +IOAT_PCI_DMACTRL_OFFSET , &dmactrl); + dmactrl |=IOAT_PCI_DMACTRL_MSI_EN ; +pci_write_config_dword (device->pdev, +IOAT_PCI_DMACTRL_OFFSET , dmactrl); + } + device->irq_mode = msi; + goto done; + +intx: + err =request_irq (device->pdev->irq, ioat_dma_do_interrupt, +IRQF_SHARED ,"ioat-intx" , device); +if (err) + goto err_no_irq; + device->irq_mode = intx; + +done: + intrctrl |=IOAT_INTRCTRL_MASTER_INT_EN ; +writeb (intrctrl, device->reg_base +IOAT_INTRCTRL_OFFSET ); +return 0 ; + +err_no_irq: +/* Disable all interrupt generation */ +writeb (0 , device->reg_base +IOAT_INTRCTRL_OFFSET ); +dev_err (&device->pdev->dev,"no usable interrupts\n" ); + device->irq_mode = none; +return -1 ; +} + +/** +* ioat_dma_remove_interrupts - remove whatever interrupts were set +* @device: ioat device +*/ +void ioat_dma_remove_interrupts (struct ioatdma_device *device) +{ +struct ioat_dma_chan *ioat_chan; +int i; + +/* Disable all interrupt generation */ +writeb (0 , device->reg_base +IOAT_INTRCTRL_OFFSET ); + +switch (device->irq_mode) { +case msix_multi_vector: +for (i =0 ; i < device->common.chancnt; i++) { + ioat_chan =ioat_lookup_chan_by_index (device, i); +free_irq (device->msix_entries[i].vector, ioat_chan); + } +pci_disable_msix (device->pdev); +break ; +case msix_single_vector: +free_irq (device->msix_entries[0 ].vector, device); +pci_disable_msix (device->pdev); +break ; +case msi: +free_irq (device->pdev->irq, device); +pci_disable_msi (device->pdev); +break ; +case intx: +free_irq (device->pdev->irq, device); +break ; +case none: +dev_warn (&device->pdev->dev, +"call to %s without interrupts setup\n" , __func__); + } + device->irq_mode = none; +} +struct ioatdma_device *ioat_dma_probe (struct pci_dev *pdev,void __iomem *iobase) {
@@ -685,21 +913,16 @@ struct ioatdma_device *ioat_dma_probe(struct pci_dev *pdev, device->common.device_issue_pending = ioat_dma_memcpy_issue_pending; device->common.device_dependency_added = ioat_dma_dependency_added; device->common.dev = &pdev->dev; -printk (KERN_INFO "ioatdma: Intel(R) I/OAT DMA Engine found," -" %d channels, device version 0x%02x\n" , - device->common.chancnt, device->version); +dev_err (&device->pdev->dev, +"ioatdma: Intel(R) I/OAT DMA Engine found," +" %d channels, device version 0x%02x\n" , + device->common.chancnt, device->version); -pci_set_drvdata (pdev, device); - err =request_irq (pdev->irq, &ioat_do_interrupt,IRQF_SHARED ,"ioat" , - device); + err =ioat_dma_setup_interrupts (device);if (err) - goto err_irq; - -writeb (IOAT_INTRCTRL_MASTER_INT_EN , - device->reg_base +IOAT_INTRCTRL_OFFSET ); -pci_set_master (pdev); + goto err_setup_interrupts; - err =ioat_self_test (device); + err =ioat_dma_self_test (device);if (err) goto err_self_test;
@@ -708,8 +931,8 @@ struct ioatdma_device *ioat_dma_probe(struct pci_dev *pdev,return device; err_self_test: -free_irq (device->pdev->irq, device); -err_irq: +ioat_dma_remove_interrupts (device); +err_setup_interrupts:pci_pool_destroy (device->completion_pool); err_completion_pool:pci_pool_destroy (device->dma_pool);
@@ -717,8 +940,8 @@ err_dma_pool:kfree (device); err_kzalloc:iounmap (iobase); -printk (KERN_ERR " " -"ioatdma: Intel(R) I/OAT DMA Engine initialization failed\n" ); +dev_err (&device->pdev->dev, +"ioatdma: Intel(R) I/OAT DMA Engine initialization failed\n" );return NULL ; }
@@ -729,7 +952,7 @@ void ioat_dma_remove(struct ioatdma_device *device)dma_async_device_unregister (&device->common); -free_irq (device->pdev->irq, device); +ioat_dma_remove_interrupts (device);pci_pool_destroy (device->dma_pool);pci_pool_destroy (device->completion_pool);
drivers/dma/ioatdma.h +12 -0
@@ -28,6 +28,14 @@#include <linux/cache.h> #include <linux/pci_ids.h> +enum ioat_interrupt { + none =0 , + msix_multi_vector =1 , + msix_single_vector =2 , + msi =3 , + intx =4 , +}; +#define IOAT_LOW_COMPLETION_MASK 0xffffffc0/**
@@ -47,6 +55,9 @@ struct ioatdma_device {struct dma_device common;u8 version; +enum ioat_interrupt irq_mode; +struct msix_entry msix_entries[4 ]; +struct ioat_dma_chan *idx[4 ]; };/**
@@ -95,6 +106,7 @@ struct ioat_dma_chan {u32 high; }; } *completion_virt; +struct tasklet_struct cleanup_task; };/* wrapper around hardware descriptor format + additional software fields */
drivers/dma/ioatdma_registers.h +5 -1
@@ -1,5 +1,5 @@/* -* Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. +* Copyright(c) 2004 - 2007 Intel Corporation. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free
@@ -21,6 +21,9 @@#ifndef _IOAT_REGISTERS_H_#define _IOAT_REGISTERS_H_ +#define IOAT_PCI_DMACTRL_OFFSET 0x48 +#define IOAT_PCI_DMACTRL_DMA_EN 0x00000001 +#define IOAT_PCI_DMACTRL_MSI_EN 0x00000002/* MMIO Device Registers */ #define IOAT_CHANCNT_OFFSET 0x00/* 8-bit */
@@ -39,6 +42,7 @@#define IOAT_INTRCTRL_MASTER_INT_EN 0x01/* Master Interrupt Enable */ #define IOAT_INTRCTRL_INT_STATUS 0x02/* ATTNSTATUS -or- Channel Int */ #define IOAT_INTRCTRL_INT 0x04/* INT_STATUS -and- MASTER_INT_EN */ +#define IOAT_INTRCTRL_MSIX_VECTOR_CONTROL 0x08/* Enable all MSI-X vectors */ #define IOAT_ATTNSTATUS_OFFSET 0x04/* Each bit is a channel */
Re: [PATCH 5/7] I/OAT: Add support for MSI and MSI-X
From: Shannon Nelson <[email protected]>
Date: Thu, 19 Jul 2007 17:45:12 -0700
Quoted 4 lines
Add support for MSI and MSI-X interrupt handling, including the ability to choose the desired interrupt method.
Signed-off-by Shannon Nelson <[email protected]>
But:
drivers/dma/ioat_dma.c line 53
Quoted 4 lines
drivers/dma/ioat_dma.c +4
-0+#define for_each_bit (bit, addr, size) \ + for ((bit) = find_first_bit((addr), (size)); \ + (bit) < (size); \ + (bit) = find_next_bit((addr), (size), (bit) + 1))
This or something like it is codified in a few spots now, namely now here and cpumask.h, it would be nice to have this in some standard place.
Re: [PATCH 5/7] I/OAT: Add support for MSI and MSI-X
In reply to Shannon Nelsonare there any devices that support MSI but not MSI-X? If not, is there any point in having code to support both?
- R.
RE: [PATCH 5/7] I/OAT: Add support for MSI and MSI-X
Quoted 9 lines
-----Original Message-----
From: Roland Dreier [mailto:[email protected]]
Sent: Friday, July 20, 2007 10:43 AM
To: Nelson, Shannon
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Williams, Dan J; Leech, Christopher; Waskiewicz Jr, Peter P
Subject: Re: [PATCH 5/7] I/OAT: Add support for MSI and MSI-X
Quoted 5 lines
are there any devices that support MSI but not MSI-X? If not, is there any point in having code to support both?
- R.
Both igb (recently posted) and ixgbe (also recently posted) support both MSI and MSI-X. Right now when we try to request MSI-X vectors, if we fail to acquire what we've asked for, we fall back to MSI support. If MSI fails to initialize, we fall back to legacy interrupts. So it needs to be there in case MSI-X allocation fails for the NIC driver.
Thanks,
-PJ Waskiewicz
Re: [PATCH 5/7] I/OAT: Add support for MSI and MSI-X
Quoted 5 lines
Both igb (recently posted) and ixgbe (also recently posted) support both MSI and MSI-X. Right now when we try to request MSI-X vectors, if we fail to acquire what we've asked for, we fall back to MSI support. If MSI fails to initialize, we fall back to legacy interrupts. So it needs to be there in case MSI-X allocation fails for the NIC driver.
Hmm, I see I don't understand what this driver is doing. What is a "struct ioatdma_device"? Is this driver requesting interrupts that come from the NIC or the IOAT DMA engine?
Anyway, if the NICs support MSI-X, is there any chance of failing to get one MSI-X vectors but then succeeding in getting MSI enabled? How could that happen? I don't see what falling back to MSI buys you beyond more code.
- R.
RE: [PATCH 5/7] I/OAT: Add support for MSI and MSI-X
Quoted 3 lines
Hmm, I see I don't understand what this driver is doing. What is a "struct
ioatdma_device"? Is this driver requesting interrupts that come from the NIC or the IOAT DMA engine?
I might have caused some confusion. You had asked if any drivers support MSI but not MSI-X, so I threw 2 drivers out there that currently support both, and why we support MSI for compatibility.
Quoted 5 lines
Anyway, if the NICs support MSI-X, is there any chance of failing to get one MSI-X vectors but then succeeding in getting MSI enabled?
How could that happen? I don't see what falling back to MSI buys you beyond more code.
MSI-X doesn't make much sense if you have 1 Rx queue on your NIC, since 1 vector essentially acts like MSI. In the case of why MSI-X could fail, I have had it fail when I misconfigured my driver and didn't ask for enough vectors for what I was assigning, so the driver disabled the multiple Rx queues, and fell back to MSI.
I hope this helps.
-PJ
Re: [PATCH 5/7] I/OAT: Add support for MSI and MSI-X
Quoted 14 lines
Quoted 3 lines
Hmm, I see I don't understand what this driver is doing. What is a "struct
ioatdma_device"? Is this driver requesting interrupts that come from the NIC or the IOAT DMA engine?I might have caused some confusion. You had asked if any drivers support MSI but not MSI-X, so I threw 2 drivers out there that currently support both, and why we support MSI for compatibility.
Quoted 5 lines
Anyway, if the NICs support MSI-X, is there any chance of failing to get one MSI-X vectors but then succeeding in getting MSI enabled?
How could that happen? I don't see what falling back to MSI buys you beyond more code.
Quoted 5 lines
MSI-X doesn't make much sense if you have 1 Rx queue on your NIC, since 1 vector essentially acts like MSI. In the case of why MSI-X could fail, I have had it fail when I misconfigured my driver and didn't ask for enough vectors for what I was assigning, so the driver disabled the multiple Rx queues, and fell back to MSI.
OK, let's try to avoid going off into the weeds here. In the context of the specific patch that this thread started with, is there any point in having both "msix-single-vector" and "msi" interrupt support?
- R.
RE: [PATCH 5/7] I/OAT: Add support for MSI and MSI-X
Quoted 3 lines · Roland Dreier
OK, let's try to avoid going off into the weeds here. In the context of the specific patch that this thread started with, is there any point in having both "msix-single-vector" and "msi" interrupt support?
This driver supports some chipsets that do MSI, and some that do MSI-X, but none that can do both.
- Chris
Re: [PATCH 5/7] I/OAT: Add support for MSI and MSI-X
Quoted 2 lines
This driver supports some chipsets that do MSI, and some that do MSI-X, but none that can do both.
Thanks, that's the simple answer I was hoping for. Obviously if some chipsets only do MSI then you need the MSI code in addition to the MSI-X code.
Re: [PATCH 5/7] I/OAT: Add support for MSI and MSI-X
Quoted 6 lines · Roland Dreier
Quoted 2 lines
This driver supports some chipsets that do MSI, and some that do MSI-X, but none that can do both.
Thanks, that's the simple answer I was hoping for. Obviously if some chipsets only do MSI then you need the MSI code in addition to the MSI-X code.
In a case where you have a device that which supports MSI-X (multiple interrupts) but the device in default is in MSI mode, ie some configuration change is needed on the device. In such a case, how would one handle between MSI-X and MSI ?
ie the device initially doesn't support MSI-X
Re: [PATCH 5/7] I/OAT: Add support for MSI and MSI-X
Quoted 6 lines
In a case where you have a device that which supports MSI-X (multiple interrupts) but the device in default is in MSI mode, ie some configuration change is needed on the device. In such a case, how would one handle between MSI-X and MSI ?
ie the device initially doesn't support MSI-X
I don't understand the question really. Does the PCI spec even allow a device to be in MSI mode by default? Surely the OS must initialize the address/message before the device can generate an MSI?
What device do you have in mind? I guess the interesting case is a PCIe device that supports MSI and MSI-X but not legacy interrupts. However I would assume such a device would come up with both MSI and MSI-X disabled.
- R.
Re: [PATCH 5/7] I/OAT: Add support for MSI and MSI-X
Quoted 11 lines · Roland Dreier
Quoted 6 lines
In a case where you have a device that which supports MSI-X (multiple interrupts) but the device in default is in MSI mode, ie some configuration change is needed on the device. In such a case, how would one handle between MSI-X and MSI ?
ie the device initially doesn't support MSI-X
I don't understand the question really. Does the PCI spec even allow a device to be in MSI mode by default? Surely the OS must initialize the address/message before the device can generate an MSI?
Sorry for being not clear. What i was asking is thus:
A device that has legacy interrupts and MSI-X. I was thinking that if MSI-X failed one should fall back to MSI mode (single message), that's what i was assuming.
In such a case, i do enable MSI-X mode on the device, when the request for 2 ^ n number of messages (where messages can be a max of 32), If the request fails one falls into a single message mode, ie MSI ?
Quoted 1 line
What device do you have in mind?
The device that i have in mind is a SAA7160.
Quoted 7 lines
I guess the interesting case is a
PCIe device that supports MSI and MSI-X but not legacy interrupts. However I would assume such a device would come up with both MSI and MSI-X disabled.- R.
RE: [PATCH 5/7] I/OAT: Add support for MSI and MSI-X
Manu Abraham [mailto:[email protected]]
Quoted 11 lines
Sorry for being not clear. What i was asking is thus:
A device that has legacy interrupts and MSI-X. I was thinking that if MSI-X failed one should fall back to MSI mode (single message), that's what i was assuming.
In such a case, i do enable MSI-X mode on the device, when the request for 2 ^ n number of messages (where messages can be a max of 32), If the request fails one falls into a single message mode, ie MSI ?
Quoted 4 lines
Quoted 1 line
What device do you have in mind?
The device that i have in mind is a SAA7160.
Notice our code looks at the return from pci_enable_msix() - it will give you a hint whether MSI-X is not supported (returns < 0) or you simply asked for too many (returns > 0). If the former, then fallback to legacy; if the latter, try MSI-X with only one interrupt, which essentially emulates MSI mode.
sln
--
======================================================================
Mr. Shannon Nelson LAN Access Division, Intel Corp.
[email protected] I don't speak for Intel
(503) 712-7659 Parents can't afford to be squeamish.
Re: [PATCH 5/7] I/OAT: Add support for MSI and MSI-X
Quoted 15 lines · Shannon
Manu Abraham [mailto:[email protected]]
Quoted 14 lines
Sorry for being not clear. What i was asking is thus:
A device that has legacy interrupts and MSI-X. I was thinking that if MSI-X failed one should fall back to MSI mode (single message), that's what i was assuming.
In such a case, i do enable MSI-X mode on the device, when the request for 2 ^ n number of messages (where messages can be a max of 32), If the request fails one falls into a single message mode, ie MSI ?
Quoted 1 line
What device do you have in mind?
Quoted 7 lines
Quoted 1 line
The device that i have in mind is a SAA7160.
Notice our code looks at the return from
pci_enable_msix()- it will give you a hint whether MSI-X is not supported (returns < 0) or you simply asked for too many (returns > 0). If the former, then fallback to legacy; if the latter, try MSI-X with only one interrupt, which essentially emulates MSI mode.
Ok. Thanks for clearing it up. So the idea would be that if pci_enable_msix() for "n" number of messages failed, then settle down for 1 message, which is equivalent to MSI mode
RE: [PATCH 5/7] I/OAT: Add support for MSI and MSI-X
Manu Abraham [mailto:[email protected]]
Quoted 5 lines
Ok. Thanks for clearing it up. So the idea would be that if
pci_enable_msix()for "n" number of messages failed, then settle down for 1 message, which is equivalent to MSI mode
yep
RE: [PATCH 5/7] I/OAT: Add support for MSI and MSI-X
In reply to Manu AbrahamQuoted 4 lines
Ok. Thanks for clearing it up. So the idea would be that if
pci_enable_msix()for "n" number of messages failed, then settle down for 1 message, which is equivalent to MSI mode
And if that fails to acquire the one vector for whatever reason (MSI isn't enabled, or something is buggy in your chipset), fail back to legacy interrupts.
Cheers,
-PJ Waskiewicz
RE: [PATCH 5/7] I/OAT: Add support for MSI and MSI-X
In reply to Roland DreierRoland Dreier [mailto:[email protected]]
Quoted 4 lines
OK, let's try to avoid going off into the weeds here. In the context of the specific patch that this thread started with, is there any point in having both "msix-single-vector" and "msi" interrupt support?
Some versions of this hardware support MSI-X and not MSI. If it can't get all the MSI-X interrupts, it will try a single interrupt before fallign all the way down to Legacy interrupts. Basically, if we can only get one interrupt, we'd still like to use the MSI-X mechanics if possible. If totally out of the question, then we'll use legacy.
sln
--
======================================================================
Mr. Shannon Nelson LAN Access Division, Intel Corp.
[email protected] I don't speak for Intel
(503) 712-7659 Parents can't afford to be squeamish.
PATCH 6/7 DCA: Add Direct Cache Access driver
In reply to Shannon NelsonDirect Cache Access (DCA) is a method for warming the CPU cache before data is used, with the intent of lessening the impact of cache misses. This patch adds a manager and interface for matching up client requests for DCA services with devices that offer DCA services.
In order to use DCA, a module must do bus writes with the appropriate tag bits set to trigger a cache read for a specific CPU. However, different CPUs and chipsets can require different sets of tag bits, and the methods for determining the correct bits may be simple hardcoding or may be a hardware specific magic incantation. This interface is a way for DCA clients to find the correct tag bits for the targeted CPU without needing to know the specifics.
---
drivers/Kconfig | 2 + drivers/Makefile | 1 drivers/dca/Kconfig | 11 +++ drivers/dca/Makefile | 2 + drivers/dca/dca-core.c | 170 +++++++++++++++++++++++++++++++++++++++++++++++ drivers/dca/dca-sysfs.c | 88 ++++++++++++++++++++++++ include/linux/dca.h | 47 +++++++++++++ 7 files changed, 321 insertions(+), 0 deletions(-)
drivers/Kconfig +2 -0
@@ -80,6 +80,8 @@ source "drivers/rtc/Kconfig" source "drivers/dma/Kconfig" +source "drivers/dca/Kconfig" + source "drivers/auxdisplay/Kconfig" source "drivers/kvm/Kconfig"
drivers/Makefile +1 -0
@@ -82,5 +82,6 @@ obj-$(CONFIG_CRYPTO) += crypto/ obj-$(CONFIG_SUPERH) += sh/ obj-$(CONFIG_GENERIC_TIME) += clocksource/ obj-$(CONFIG_DMA_ENGINE) += dma/ +obj-$(CONFIG_DCA) += dca/ obj-$(CONFIG_HID) += hid/ obj-$(CONFIG_PPC_PS3) += ps3/
drivers/dca/Kconfig new file +11 -0
@@ -0,0 +1,11 @@ +# +# DCA server configuration +# + +config DCA + tristate "DCA support for clients and providers" + ---help--- + This is a server to help modules that want to use Direct Cache + Access to find DCA providers that will supply correct CPU tags. + default m +
drivers/dca/Makefile new file +2 -0
@@ -0,0 +1,2 @@ +obj-$(CONFIG_DCA) += dca.o +dca-objs := dca-core.o dca-sysfs.o
drivers/dca/dca-core.c new file +170 -0
170 lines not on this page · raw message
drivers/dca/dca-sysfs.c new file +88 -0
88 lines not on this page · raw message
include/linux/dca.h new file +47 -0
47 lines not on this page · raw message
Re: [PATCH 6/7] DCA: Add Direct Cache Access driver
From: Shannon Nelson <[email protected]>
Date: Thu, 19 Jul 2007 17:45:17 -0700
Quoted 1 line
+static
spinlock_tdca_lock;
...
drivers/dca/dca-core.c line 155
Quoted 1 line
drivers/dca/dca-core.c new file +1
-0+spin_lock_init (&dca_lock);
It's easier to use DEFINE_SPINLOCK().
RE: [PATCH 6/7] DCA: Add Direct Cache Access driver
David Miller [mailto:[email protected]]
Quoted 7 lines
Quoted 2 lines
+static
spinlock_tdca_lock;...
drivers/dca/dca-core.c line 155
Quoted 1 line
drivers/dca/dca-core.c new file +1
-0+spin_lock_init (&dca_lock);It's easier to use
DEFINE_SPINLOCK().
Thanks - I'll adjust that in a future patch.
sln
--
======================================================================
Mr. Shannon Nelson LAN Access Division, Intel Corp.
[email protected] I don't speak for Intel
(503) 712-7659 Parents can't afford to be squeamish.
PATCH 7/7 I/OAT: Add DCA services
In reply to Shannon NelsonAdd code to connect to the DCA driver and provide cpu tags for use by drivers that would like to use Direct Cache Access hints.
---
drivers/dma/Makefile | 2 drivers/dma/ioat.c | 12 ++ drivers/dma/ioat_dca.c | 259 ++++++++++++++++++++++++++++++++++++++++++++++++ drivers/dma/ioatdma.h | 2 4 files changed, 273 insertions(+), 2 deletions(-)
drivers/dma/Makefile +1 -1
@@ -1,5 +1,5 @@ obj-$(CONFIG_DMA_ENGINE) += dmaengine.o obj-$(CONFIG_NET_DMA) += iovlock.o obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o -ioatdma-objs := ioat.o ioat_dma.o +ioatdma-objs := ioat.o ioat_dma.o ioat_dca.o obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o
drivers/dma/ioat.c +11 -1
@@ -1,6 +1,6 @@/* * Intel I/OAT DMA Linux driver -* Copyright(c) 2004 - 2007 Intel Corporation. +* Copyright(c) 2007 Intel Corporation. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License,
@@ -29,6 +29,7 @@#include <linux/module.h> #include <linux/pci.h> #include <linux/interrupt.h> +#include <linux/dca.h> #include "ioatdma.h" #include "ioatdma_registers.h" #include "ioatdma_hw.h"
@@ -49,6 +50,7 @@ struct ioat_device {struct pci_dev *pdev;void __iomem *iobase;struct ioatdma_device *dma; +struct dca_provider *dca; }; static int __devinit ioat_probe(struct pci_dev *pdev,
@@ -65,6 +67,7 @@ static int ioat_setup_functionality(struct pci_dev *pdev, void __iomem *iobase)switch (version) {case IOAT_VER_1_2 : device->dma =ioat_dma_probe (pdev, iobase); + device->dca =ioat_dca_init (pdev, iobase);break ;default : err = -ENODEV ;
@@ -81,6 +84,13 @@ static void ioat_shutdown_functionality(struct pci_dev *pdev)ioat_dma_remove (device->dma); device->dma =NULL ; } + +if (device->dca) { +unregister_dca_provider (device->dca); +free_dca_provider (device->dca); + device->dca =NULL ; + } + }static struct pci_driver ioat_pci_drv = {
drivers/dma/ioat_dca.c new file +259 -0
259 lines not on this page · raw message
drivers/dma/ioatdma.h +2 -0
@@ -133,9 +133,11 @@ struct ioat_desc_sw {#if defined(CONFIG_INTEL_IOATDMA ) || defined(CONFIG_INTEL_IOATDMA_MODULE )struct ioatdma_device *ioat_dma_probe (struct pci_dev *,void __iomem *);void ioat_dma_remove (struct ioatdma_device *device); +struct dca_provider *ioat_dca_init (struct pci_dev *,void __iomem *);#else #define ioat_dma_probe (pdev, io) NULL#define ioat_dma_remove (dev) do { } while (0) +#define ioat_dca_init (pdev, io) NULL#endif #endif /* IOATDMA_H */
Re: [PATCH 7/7] I/OAT: Add DCA services
From: Shannon Nelson <[email protected]>
Date: Thu, 19 Jul 2007 17:45:22 -0700
Quoted 4 lines
Add code to connect to the DCA driver and provide cpu tags for use by drivers that would like to use Direct Cache Access hints.
Signed-off-by Shannon Nelson <[email protected]>