ARM: add PrimeCell generic DMA to PL011
PATCH 1/1 ARM: add PrimeCell generic DMA to PL011
This extends the PL011 UART driver with generic DMA engine support using the PrimeCell DMA engine interface.
---
Changes from previous version that was in the patch set for PrimeCell DMA (I've lost count):
This adds support for the ST-Ericsson specific DMA watermarking via a vendor data-specified setup function, a design pattern likely to be useful for other users as well.
--- drivers/serial/amba-pl011.c | 871 ++++++++++++++++++++++++++++++++++++++++++- include/linux/amba/serial.h | 21 + 2 files changed, 883 insertions(+), 9 deletions(-)
drivers/serial/amba-pl011.c +862 -9
@@ -7,6 +7,7 @@* * Copyright 1999 ARM Limited * Copyright (C) 2000 Deep Blue Solutions Ltd. +* Copyright (C) 2010 ST-Ericsson SA * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by
@@ -48,6 +49,11 @@#include <linux/amba/serial.h> #include <linux/clk.h> #include <linux/slab.h> +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> +#include <linux/scatterlist.h> +#include <linux/completion.h> +#include <linux/delay.h> #include <asm/io.h> #include <asm/sizes.h>
@@ -63,6 +69,27 @@#define UART_DR_ERROR (UART011_DR_OE|UART011_DR_BE|UART011_DR_PE|UART011_DR_FE)#define UART_DUMMY_DR_RX (1 << 16) +struct uart_amba_port ; +typedef void (*dma_init_fn )(struct uart_amba_port *uap); + +/* Deals with DMA transactions */ +struct pl011_dma_rx_transaction { +struct completion complete; +bool use_buffer_b; +struct scatterlist scatter_a; +struct scatterlist scatter_b; +char *rx_dma_buf_a; +char *rx_dma_buf_b; +dma_cookie_t cookie; +}; + +struct pl011_dma_tx_transaction { +struct completion complete; +struct scatterlist scatter; +char *tx_dma_buf; +dma_cookie_t cookie; +}; +/* * We wrap our port structure around the generic uart_port. */
@@ -75,7 +102,18 @@ struct uart_amba_port {unsigned int lcrh_tx;/* vendor-specific */ unsigned int lcrh_rx;/* vendor-specific */ bool oversampling;/* vendor-specific */ +dma_init_fn dma_init;/* vendor-specific */ bool autorts; +unsigned int fifosize; +/* DMA stuff */ +bool enable_dma; +bool rx_dma_running; +#ifdef CONFIG_DMA_ENGINE +struct dma_chan *dma_rx_channel; +struct dma_chan *dma_tx_channel; +struct pl011_dma_rx_transaction dmarx; +struct pl011_dma_tx_transaction dmatx; +#endif };/* There is by now@least one vendor with differing details, so handle it */
@@ -85,6 +123,7 @@ struct vendor_data {unsigned int lcrh_tx;unsigned int lcrh_rx;bool oversampling; +dma_init_fn dma_init; };static struct vendor_data vendor_arm = {
@@ -95,14 +134,747 @@ static struct vendor_data vendor_arm = { .oversampling = false, }; +static void pl011_st_dma_startup (struct uart_amba_port *uap); +static struct vendor_data vendor_st = { .ifls =UART011_IFLS_RX_HALF |UART011_IFLS_TX_HALF , .fifosize =64 , .lcrh_tx =ST_UART011_LCRH_TX , .lcrh_rx =ST_UART011_LCRH_RX , .oversampling = true, + .dma_init = pl011_st_dma_startup, }; +/* +* All the DMA operation mode stuff goes inside this ifdef. +* This assumes that you have a generic DMA device interface, +* no custom DMA interfaces are supported. +* +* If we had discardable probe() functions akin to +* platform_device_probe() in the PrimeCell/AMBA bus, we could +* discard most of this code after use, but since we haven't, +* we have to keep it all around. +*/ +#ifdef CONFIG_DMA_ENGINE + +#define PL011_DMA_BUFFER_SIZE PAGE_SIZE + +static void pl011_dma_probe_initcall (struct uart_amba_port *uap) +{ +/* DMA is the sole user of the platform data right now */ +struct amba_pl011_data *plat = uap->port.dev->platform_data; +struct pl011_dma_rx_transaction *dmarx = &uap->dmarx; +struct pl011_dma_tx_transaction *dmatx = &uap->dmatx; +struct dma_slave_config rx_conf = { + .src_addr = uap->port.mapbase + UART01x_DR, + .src_addr_width =DMA_SLAVE_BUSWIDTH_1_BYTE , + .direction =DMA_FROM_DEVICE , + .src_maxburst = uap->fifosize >>1 , + }; +struct dma_slave_config tx_conf = { + .dst_addr = uap->port.mapbase + UART01x_DR, + .dst_addr_width =DMA_SLAVE_BUSWIDTH_1_BYTE , + .direction =DMA_TO_DEVICE , + .dst_maxburst = uap->fifosize >>1 , + }; +dma_cap_mask_t mask; +int sglen; + +/* We need platform data */ +if (!plat) { +dev_err (uap->port.dev,"no DMA platform data!\n" ); +return ; + } + +/* Try to acquire a generic DMA engine slave channel */ +dma_cap_zero (mask); +dma_cap_set (DMA_SLAVE , mask); + +/* +* We need both RX and TX channels to do DMA, else do none +* of them. +*/ + uap->dma_rx_channel =dma_request_channel (mask, + plat->dma_filter, + plat->dma_rx_param); +if (!uap->dma_rx_channel) { +dev_err (uap->port.dev,"no RX DMA channel!\n" ); +return ; + } + uap->dma_rx_channel->device->device_control (uap->dma_rx_channel, +DMA_SLAVE_CONFIG , + (unsigned long ) &rx_conf); + + uap->dma_tx_channel =dma_request_channel (mask, + plat->dma_filter, + plat->dma_tx_param); +if (!uap->dma_tx_channel) { +dev_err (uap->port.dev,"no TX DMA channel!\n" ); + goto err_no_txchan; + } + uap->dma_tx_channel->device->device_control (uap->dma_tx_channel, +DMA_SLAVE_CONFIG , + (unsigned long ) &tx_conf); + +/* Allocate DMA RX and TX buffers */ + dmarx->rx_dma_buf_a =kmalloc (PL011_DMA_BUFFER_SIZE ,GFP_KERNEL ); +if (!dmarx->rx_dma_buf_a) { +dev_err (uap->port.dev,"failed to allocate DMA RX buffer A\n" ); + goto err_no_rxbuf_a; + } + + dmarx->rx_dma_buf_b =kmalloc (PL011_DMA_BUFFER_SIZE ,GFP_KERNEL ); +if (!dmarx->rx_dma_buf_b) { +dev_err (uap->port.dev,"failed to allocate DMA RX buffer B\n" ); + goto err_no_rxbuf_b; + } + + dmatx->tx_dma_buf =kmalloc (PL011_DMA_BUFFER_SIZE ,GFP_KERNEL ); +if (!dmatx->tx_dma_buf) { +dev_err (uap->port.dev,"failed to allocate DMA TX buffer\n" ); + goto err_no_txbuf; + } + +/* Provide single SG list with one item to the buffers */ +sg_init_one (&dmarx->scatter_a, dmarx->rx_dma_buf_a, +PL011_DMA_BUFFER_SIZE ); +sg_init_one (&dmarx->scatter_b, dmarx->rx_dma_buf_b, +PL011_DMA_BUFFER_SIZE ); +sg_init_one (&dmatx->scatter, dmatx->tx_dma_buf,PL011_DMA_BUFFER_SIZE ); + +/* Map DMA buffers */ + sglen =dma_map_sg (uap->port.dev, &dmarx->scatter_a, +1 ,DMA_FROM_DEVICE ); +if (sglen !=1 ) + goto err_rx_sgmap_a; + + sglen =dma_map_sg (uap->port.dev, &dmarx->scatter_b, +1 ,DMA_FROM_DEVICE ); +if (sglen !=1 ) + goto err_rx_sgmap_b; + + sglen =dma_map_sg (uap->port.dev, &dmatx->scatter, +1 ,DMA_TO_DEVICE ); +if (sglen !=1 ) + goto err_tx_sgmap; + +/* Initially we say the transfers are incomplete */ +init_completion (&uap->dmatx.complete); +complete (&uap->dmatx.complete); + +/* The DMA buffer is now the FIFO the TTY subsystem can use */ + uap->port.fifosize =PL011_DMA_BUFFER_SIZE ; + + uap->enable_dma = true; +dev_info (uap->port.dev,"setup for DMA on RX %s, TX %s\n" , +dma_chan_name (uap->dma_rx_channel), +dma_chan_name (uap->dma_tx_channel)); +return ; + +err_tx_sgmap: +dma_unmap_sg (uap->port.dev, &dmarx->scatter_b, +1 ,DMA_FROM_DEVICE ); +err_rx_sgmap_b: +dma_unmap_sg (uap->port.dev, &dmarx->scatter_a, +1 ,DMA_FROM_DEVICE ); +err_rx_sgmap_a: +kfree (dmatx->tx_dma_buf); +err_no_txbuf: +kfree (dmarx->rx_dma_buf_b); +err_no_rxbuf_b: +kfree (dmarx->rx_dma_buf_a); +err_no_rxbuf_a: +dma_release_channel (uap->dma_tx_channel); + uap->dma_tx_channel =NULL ; +err_no_txchan: +dma_release_channel (uap->dma_rx_channel); + uap->dma_rx_channel =NULL ; +return ; +} + +/* +* Stack up the UARTs and let the above initcall be done at +* device initcall time, because the serial driver is called as +* an arch initcall, and at this time the DMA subsystem is not yet +* registered. At this point the driver will switch over to using +* DMA where desired. +*/ + +struct dma_uap { +struct list_head node; +struct uart_amba_port *uap; +}; + +struct list_head pl011_dma_uarts =LIST_HEAD_INIT (pl011_dma_uarts); + +static int pl011_dma_initcall (void ) +{ +struct list_head *node, *tmp; + +list_for_each_safe (node, tmp, &pl011_dma_uarts) { + struct dma_uap *dmau = list_entry(node, struct dma_uap, node); +pl011_dma_probe_initcall (dmau->uap); +list_del (node); +kfree (dmau); + } +return 0 ; +} + +device_initcall (pl011_dma_initcall); + +static void pl011_dma_probe (struct uart_amba_port *uap) +{ +struct dma_uap *dmau =kzalloc (sizeof (struct dma_uap ),GFP_KERNEL ); + +if (dmau ==NULL ) +return ; + dmau->uap = uap; +list_add_tail (&dmau->node, &pl011_dma_uarts); +} + +static void pl011_dma_remove (struct uart_amba_port *uap) +{ +struct pl011_dma_rx_transaction *dmarx = &uap->dmarx; +struct pl011_dma_tx_transaction *dmatx = &uap->dmatx; + +/* TODO: remove the initcall if it has not yet executed */ +/* Unmap and free DMA buffers */ +if (uap->dma_rx_channel) +dma_release_channel (uap->dma_rx_channel); +if (uap->dma_tx_channel) +dma_release_channel (uap->dma_tx_channel); +if (dmatx->tx_dma_buf) { +dma_unmap_sg (uap->port.dev, &dmatx->scatter, +1 ,DMA_TO_DEVICE ); +kfree (dmatx->tx_dma_buf); + } +if (dmarx->rx_dma_buf_b) { +dma_unmap_sg (uap->port.dev, &dmarx->scatter_b, +1 ,DMA_FROM_DEVICE ); +kfree (dmarx->rx_dma_buf_b); + } +if (dmarx->rx_dma_buf_a) { +dma_unmap_sg (uap->port.dev, &dmarx->scatter_a, +1 ,DMA_FROM_DEVICE ); +kfree (dmarx->rx_dma_buf_a); + } +} + +/* Forward declare this for the refill routine */ +static int pl011_dma_tx_refill (struct uart_amba_port *uap); + +/* +* Move the tail when this IRQ occurs, if not empty refill and +* fire another transaction +*/ +static void pl011_dma_tx_callback (void *data) +{ +struct uart_amba_port *uap = data; +struct pl011_dma_tx_transaction *dmatx = &uap->dmatx; +struct circ_buf *xmit = &uap->port.state->xmit; +u16 val; +int ret; + +/* Temporarily disable TX DMA */ + val =readw (uap->port.membase +UART011_DMACR ); + val &= ~(UART011_TXDMAE ); +writew (val, uap->port.membase +UART011_DMACR ); + +/* Refill the TX if the buffer is not empty */ +if (!uart_circ_empty (xmit)) { + ret =pl011_dma_tx_refill (uap); +if (ret == -EBUSY ) +/* +* If DMA cannot be used right now, we complete this +* transaction and let the TTY layer retry. If the +* firs following xfer fails to set up for DMA, it +* will fall through to interrupt mode. +*/ +dev_dbg (uap->port.dev,"DMA busy\n" ); + }else { +complete (&dmatx->complete); + } +} + +static int pl011_dma_tx_refill (struct uart_amba_port *uap) +{ +struct pl011_dma_tx_transaction *dmatx = &uap->dmatx; +struct dma_chan *chan = uap->dma_tx_channel; +struct dma_async_tx_descriptor *desc; +struct circ_buf *xmit = &uap->port.state->xmit; +unsigned int count; +unsigned long flags; +u16 val; + +/* Don't bother about using DMA on XON/XOFF */ +if (uap->port.x_char) { +/* If we can't get it into the FIFO, retry later */ +if (readw (uap->port.membase + UART01x_FR) & + UART01x_FR_TXFF) { +complete (&dmatx->complete); +return 0 ; + } +writew (uap->port.x_char, uap->port.membase + UART01x_DR); + uap->port.icount.tx++; + uap->port.x_char =0 ; +complete (&dmatx->complete); +return 0 ; + } + +/* +* Try to avoid the overhead involved in using DMA if the +* transaction fits in the first half of the FIFO and it's not +* full. Unfortunately there is only one single bit in the +* hardware to tell whether the FIFO is full or not, so +* we don't know exactly how many chars we can fit in. +*/ +if (uart_circ_chars_pending (xmit) < (uap->fifosize >>1 )) { +while (uart_circ_chars_pending (xmit)) { +if (readw (uap->port.membase + UART01x_FR) & + UART01x_FR_TXFF) { +/* +* Ooops TX FIFO is full, we'd better stop +* this. Let's enable TX interrupt here to get +* informed when there is again some space in +* the TX FIFO so we can continue the transfer. +* This interrupt will be cleared just before +* setting up DMA, as it could interfere with +* TX interrupt handling routine. +*/ + uap->im |=UART011_TXIM ; +writew (uap->im, + uap->port.membase +UART011_IMSC ); +break ; + } +writew (xmit->buf[xmit->tail], + uap->port.membase + UART01x_DR); + uap->port.icount.tx++; + xmit->tail = (xmit->tail +1 ) & (UART_XMIT_SIZE -1 ); + } +complete (&dmatx->complete); +return 0 ; + } + +/* +* Clear TX interrupt to be sure that DMA will not interfere with +* TX ISR +*/ +local_irq_save (flags); + uap->im &= ~UART011_TXIM ; +writew (uap->im, uap->port.membase +UART011_IMSC ); +local_irq_restore (flags); + +/* Sync the buffer for the CPU so we can write into it */ +dma_sync_sg_for_cpu (uap->port.dev, + &dmatx->scatter, +1 , +DMA_TO_DEVICE ); + +/* Else proceed to copy the TX chars to the DMA buffer and fire DMA */ + count =uart_circ_chars_pending (xmit); +if (count >PL011_DMA_BUFFER_SIZE ) + count =PL011_DMA_BUFFER_SIZE ; + +if (xmit->tail < xmit->head) +memcpy (&dmatx->tx_dma_buf[0 ], &xmit->buf[xmit->tail], count); +else { +size_t first =UART_XMIT_SIZE - xmit->tail; +size_t second = xmit->head; + +memcpy (&dmatx->tx_dma_buf[0 ], &xmit->buf[xmit->tail], first); +memcpy (&dmatx->tx_dma_buf[first], &xmit->buf[0 ], second); + } + + dmatx->scatter.length = count; + +/* Synchronize the scatterlist, invalidate buffers, caches etc */ +dma_sync_sg_for_device (uap->port.dev, + &dmatx->scatter, +1 , +DMA_TO_DEVICE ); + +/* Prepare the scatterlist */ + desc = chan->device->device_prep_slave_sg (chan, + &dmatx->scatter, +1 , +DMA_TO_DEVICE , +DMA_PREP_INTERRUPT |DMA_CTRL_ACK ); +if (!desc) { +/* "Complete" DMA (errorpath) */ +complete (&dmatx->complete); + chan->device->device_control (chan,DMA_TERMINATE_ALL ,0 ); +return -EBUSY ; + } + +/* Some data to go along to the callback */ + desc->callback = pl011_dma_tx_callback; + desc->callback_param = uap; + +/* Here is where overloaded DMA controllers can fail */ + dmatx->cookie = desc->tx_submit (desc); +if (dma_submit_error (dmatx->cookie)) { +/* "Complete" DMA (errorpath) */ +complete (&dmatx->complete); + chan->device->device_control (chan,DMA_TERMINATE_ALL ,0 ); +return dmatx->cookie; + } + +/* +* Now we know that DMA will fire, so advance the ring buffer +* with the stuff we just dispatched +*/ + xmit->tail = (xmit->tail + count) & (UART_XMIT_SIZE -1 ); + uap->port.icount.tx += count; +if (uart_circ_chars_pending (xmit) <WAKEUP_CHARS ) +uart_write_wakeup (&uap->port); + +/* Fire the DMA transaction */ + chan->device->device_issue_pending (chan); + + val =readw (uap->port.membase +UART011_DMACR ); + val |=UART011_TXDMAE ; +writew (val, uap->port.membase +UART011_DMACR ); +return 0 ; +} + +static void pl011_dma_rx_callback (void *data); + +static int pl011_dma_rx_trigger_dma (struct uart_amba_port *uap) +{ +struct dma_chan *rxchan = uap->dma_rx_channel; +struct pl011_dma_rx_transaction *dmarx = &uap->dmarx; +struct dma_async_tx_descriptor *desc; +struct scatterlist *scatter = dmarx->use_buffer_b ? + &dmarx->scatter_b : &dmarx->scatter_a; +u16 val; + +/* Start the RX DMA job */ + desc = rxchan->device->device_prep_slave_sg (rxchan, + scatter, +1 , +DMA_FROM_DEVICE , +DMA_PREP_INTERRUPT |DMA_CTRL_ACK ); +/* +* If the DMA engine is busy and cannot prepare a +* channel, no big deal, the driver will fall back +* to interrupt mode as a result of this error code. +*/ +if (!desc) { + uap->rx_dma_running = false; + rxchan->device->device_control (rxchan,DMA_TERMINATE_ALL ,0 ); +return -EBUSY ; + } + +/* Some data to go along to the callback */ + desc->callback = pl011_dma_rx_callback; + desc->callback_param = uap; +/* This is another point where an overloaded engine can fail */ + dmarx->cookie = desc->tx_submit (desc); +if (dma_submit_error (dmarx->cookie)) { + uap->rx_dma_running = false; + rxchan->device->device_control (rxchan,DMA_TERMINATE_ALL ,0 ); +return -EBUSY ; + } + + rxchan->device->device_issue_pending (rxchan); + + val =readw (uap->port.membase +UART011_DMACR ); + val |=UART011_RXDMAE ; +writew (val, uap->port.membase +UART011_DMACR ); + uap->rx_dma_running = true; + +return 0 ; +} + +/* +* This is called when either the DMA job is complete, or +* the FIFO timeout interrupt occurred. This must be called +* with the port spinlock uap->port.lock held. +*/ +static void pl011_dma_rx_chars (struct uart_amba_port *uap, +u32 pending,bool use_buffer_b, +bool readfifo) +{ +struct pl011_dma_rx_transaction *dmarx = &uap->dmarx; +struct tty_struct *tty = uap->port.state->port.tty; +char *buf = use_buffer_b ? dmarx->rx_dma_buf_b : dmarx->rx_dma_buf_a; +struct scatterlist *scatter = use_buffer_b ? + &dmarx->scatter_b : &dmarx->scatter_a; +unsigned int status, ch, flag; +u32 count = pending; +u32 bufp =0 ; +u32 fifotaken =0 ;/* only used for vdbg() */ + +/* Sync in buffer */ +dma_sync_sg_for_cpu (uap->port.dev, + scatter, +1 , +DMA_FROM_DEVICE ); + + status =readw (uap->port.membase + UART01x_FR); + +/* +* First take all chars in the DMA pipe, then look +* in the FIFO. So loop while we have chars in the +* DMA buffer or the FIFO. If we came here from a +* DMA buffer full interrupt, there is already another +* DMA job triggered to read the FIFO, so don't look +* at it. +*/ +while (count || + (readfifo && (status & UART01x_FR_RXFE) ==0 )) { + + flag =TTY_NORMAL ; + uap->port.icount.rx++; + +if (count) { +/* Take chars from the DMA buffer */ +int inserted =tty_insert_flip_string ( + uap->port.state->port.tty, buf, count); + +/* +* Check if insertion is successful to avoid +* infinite loop. This can happen when TTY is full. +*/ +if (unlikely (inserted ==0 )) + count =0 ; +else { + count -= inserted; + bufp += inserted; + } +continue ; + }else { +/* Take chars from the FIFO and update status */ + ch =readw (uap->port.membase + UART01x_DR); + status =readw (uap->port.membase + UART01x_FR); + fifotaken++; + +/* +* Error conditions will only occur in the FIFO, +* these will trigger an immediate interrupt and +* stop the DMA job, so we will always find the +* error in the FIFO, never in the DMA buffer. +*/ +if (unlikely (ch &UART_DR_ERROR )) { +if (ch &UART011_DR_BE ) { + ch &= ~(UART011_DR_FE |UART011_DR_PE ); + uap->port.icount.brk++; +if (uart_handle_break (&uap->port)) +continue ; + }else if (ch &UART011_DR_PE ) + uap->port.icount.parity++; +else if (ch &UART011_DR_FE ) + uap->port.icount.frame++; +if (ch &UART011_DR_OE ) + uap->port.icount.overrun++; + + ch &= uap->port.read_status_mask; + +if (ch &UART011_DR_BE ) + flag =TTY_BREAK ; +else if (ch &UART011_DR_PE ) + flag =TTY_PARITY ; +else if (ch &UART011_DR_FE ) + flag =TTY_FRAME ; + } + } + +if (uart_handle_sysrq_char (&uap->port, ch &255 )) +continue ; + +uart_insert_char (&uap->port, ch,UART011_DR_OE , ch, flag); + } + +spin_unlock (&uap->port.lock); +dev_vdbg (uap->port.dev, +"Took %d chars from DMA buffer and %d chars from the FIFO\n" , + bufp, fifotaken); +tty_flip_buffer_push (tty); +spin_lock (&uap->port.lock); +} + +static void pl011_dma_rx_irq (struct uart_amba_port *uap) +{ +struct dma_chan *rxchan = uap->dma_rx_channel; +struct pl011_dma_rx_transaction *dmarx = &uap->dmarx; +struct scatterlist *scatter = dmarx->use_buffer_b ? + &dmarx->scatter_b : &dmarx->scatter_a; +u32 pending; +int ret; +struct dma_tx_state state; +enum dma_status dmastat; +u16 val; + +/* Use PrimeCell DMA extensions to stop the transfer */ + ret = rxchan->device->device_control (rxchan,DMA_PAUSE ,0 ); +if (ret) +dev_err (uap->port.dev,"unable to pause DMA transfer\n" ); + dmastat = rxchan->device->device_tx_status (rxchan, + dmarx->cookie, &state); + +/* Disable RX DMA temporarily */ + val =readw (uap->port.membase +UART011_DMACR ); + val &= ~(UART011_RXDMAE ); +writew (val, uap->port.membase +UART011_DMACR ); + uap->rx_dma_running = false; + +if (dmastat !=DMA_PAUSED ) +dev_err (uap->port.dev,"unable to pause DMA transfer\n" ); + pending = scatter->length - state.residue; + +BUG_ON (pending >PL011_DMA_BUFFER_SIZE ); + + ret = rxchan->device->device_control (rxchan,DMA_TERMINATE_ALL ,0 ); +if (ret) +dev_err (uap->port.dev,"unable to terminate DMA transfer\n" ); + +/* +* This will take the chars we have so far and insert +* into the framework. +*/ +pl011_dma_rx_chars (uap, pending, dmarx->use_buffer_b, true); + +/* Switch buffer & re-trigger DMA job */ + dmarx->use_buffer_b = !dmarx->use_buffer_b; + ret =pl011_dma_rx_trigger_dma (uap); +if (ret) { +dev_dbg (uap->port.dev,"could not retrigger RX DMA job " +"fall back to interrupt mode\n" ); + uap->im |=UART011_RXIM ; +writew (uap->im, uap->port.membase +UART011_IMSC ); + } +} + +static void pl011_dma_rx_callback (void *data) +{ +struct uart_amba_port *uap = data; +struct pl011_dma_rx_transaction *dmarx = &uap->dmarx; +bool lastbuf = dmarx->use_buffer_b; +int ret; + +/* +* This completion interrupt occurs typically when the +* RX buffer is totally stuffed but no timeout has yet +* occurred. When that happens, we just want the RX +* routine to flush out the secondary DMA buffer while +* we immediately trigger the next DMA job. +*/ + uap->rx_dma_running = false; + dmarx->use_buffer_b = !lastbuf; + ret =pl011_dma_rx_trigger_dma (uap); + +spin_lock_irq (&uap->port.lock); +pl011_dma_rx_chars (uap,PL011_DMA_BUFFER_SIZE , lastbuf, false); +spin_unlock_irq (&uap->port.lock); +/* +* Do this check after we picked the DMA chars so we don't +* get some IRQ immediately from RX. +*/ +if (ret) { +dev_dbg (uap->port.dev,"could not retrigger RX DMA job " +"fall back to interrupt mode\n" ); + uap->im |=UART011_RXIM ; +writew (uap->im, uap->port.membase +UART011_IMSC ); + } +} + +static void pl011_st_dma_startup (struct uart_amba_port *uap) +{ +/* Set DMABREQ threshold */ +writew (ST_UART011_DMAWM_RX_16 |ST_UART011_DMAWM_TX_16 , + uap->port.membase +ST_UART011_DMAWM ); +} + +static void pl011_dma_startup (struct uart_amba_port *uap) +{ +u16 val; +int ret =0 ; + +if (!uap->enable_dma) +return ; + +/* Turn on DMA error (RX/TX will be enabled on demand) */ + val =readw (uap->port.membase +UART011_DMACR ); + val |=UART011_DMAONERR ; +writew (val, uap->port.membase +UART011_DMACR ); + +/* call vendor specific dma init */ +if (uap->dma_init) + uap->dma_init (uap); + + ret =pl011_dma_rx_trigger_dma (uap); +if (ret) +dev_dbg (uap->port.dev,"could not trigger initial " +"RX DMA job, fall back to interrupt mode\n" ); +} + +static void pl011_dma_shutdown (struct uart_amba_port *uap) +{ +struct dma_chan *rxchan = uap->dma_rx_channel; +struct dma_chan *txchan = uap->dma_tx_channel; +u16 val; + +if (!uap->enable_dma) +return ; + +/* Disable RX and TX DMA */ +while (readw (uap->port.membase + UART01x_FR) & UART01x_FR_BUSY) +barrier (); + val =readw (uap->port.membase +UART011_DMACR ); + val &= ~(UART011_DMAONERR |UART011_RXDMAE |UART011_TXDMAE ); +writew (val, uap->port.membase +UART011_DMACR ); +/* Terminate any RX and TX DMA jobs */ + rxchan->device->device_control (rxchan,DMA_TERMINATE_ALL ,0 ); + txchan->device->device_control (txchan,DMA_TERMINATE_ALL ,0 ); +} + +static int pl011_dma_tx_chars (struct uart_amba_port *uap) +{ +struct pl011_dma_tx_transaction *dmatx = &uap->dmatx; + +/* Try to wait for completion, return if something is in progress */ +if (!try_wait_for_completion (&dmatx->complete)) +return -EINPROGRESS ; + +/* Set up and fire the DMA job */ +init_completion (&dmatx->complete); +return pl011_dma_tx_refill (uap); +} + +#else +/* Blank functions if the DMA engine is not available */ +static inline void pl011_dma_probe (struct uart_amba_port *uap) +{ +} + +static inline void pl011_dma_remove (struct uart_amba_port *uap) +{ +} + +static inline void pl011_dma_rx_irq (struct uart_amba_port *uap) +{ +} + +static inline int pl011_dma_rx_trigger_dma (struct uart_amba_port *uap) +{ +return -EIO ; +} + +static inline void pl011_dma_startup (struct uart_amba_port *uap) +{ +} + +static inline void pl011_dma_shutdown (struct uart_amba_port *uap) +{ +} + +static inline int pl011_dma_tx_chars (struct uart_amba_port *uap) +{ +return -EIO ; +} +#endif + +static void pl011_stop_tx (struct uart_port *port) {struct uart_amba_port *uap = (struct uart_amba_port *)port;
@@ -111,10 +883,18 @@ static void pl011_stop_tx(struct uart_port *port)writew (uap->im, uap->port.membase +UART011_IMSC ); } +static void pl011_tx_chars (struct uart_amba_port *uap); +static void pl011_start_tx (struct uart_port *port) {struct uart_amba_port *uap = (struct uart_amba_port *)port; +if (uap->enable_dma) { +/* Immediately push out chars in DMA mode */ +pl011_tx_chars (uap); +return ; + } +/* In interrupt mode, let the interrupt pull chars */ uap->im |=UART011_TXIM ;writew (uap->im, uap->port.membase +UART011_IMSC ); }
@@ -140,6 +920,7 @@ static void pl011_rx_chars(struct uart_amba_port *uap) {struct tty_struct *tty = uap->port.state->port.tty;unsigned int status, ch, flag, max_count =256 ; +int ret; status =readw (uap->port.membase + UART01x_FR);while ((status & UART01x_FR_RXFE) ==0 && max_count--) {
@@ -184,6 +965,21 @@ static void pl011_rx_chars(struct uart_amba_port *uap) }spin_unlock (&uap->port.lock);tty_flip_buffer_push (tty); +/* +* If we were temporarily out of DMA mode for a while, +* attempt to switch back to DMA mode again. +*/ +if (uap->enable_dma) { + uap->im &= ~UART011_RXIM ; +writew (uap->im, uap->port.membase +UART011_IMSC ); + ret =pl011_dma_rx_trigger_dma (uap); +if (ret) { +dev_dbg (uap->port.dev,"could not trigger RX DMA job " +"fall back to interrupt mode again\n" ); + uap->im |=UART011_RXIM ; +writew (uap->im, uap->port.membase +UART011_IMSC ); + } + }spin_lock (&uap->port.lock); }
@@ -192,6 +988,25 @@ static void pl011_tx_chars(struct uart_amba_port *uap)struct circ_buf *xmit = &uap->port.state->xmit;int count; +if (uap->enable_dma) { +int ret; + + ret =pl011_dma_tx_chars (uap); +if (!ret) +return ; +if (ret == -EINPROGRESS ) +return ; + +/* +* On any other error (including -EBUSY which is emitted +* in case the DMA engine is out of physical channels +* for example) we fall through to interrupt mode +*/ +dev_dbg (uap->port.dev,"DMA unavailable for TX\n" ); + uap->im |=UART011_TXIM ; +writew (uap->im, uap->port.membase +UART011_IMSC ); + } +if (uap->port.x_char) {writew (uap->port.x_char, uap->port.membase + UART01x_DR); uap->port.icount.tx++;
@@ -203,8 +1018,10 @@ static void pl011_tx_chars(struct uart_amba_port *uap)return ; } - count = uap->port.fifosize >>1 ; + count = uap->fifosize >>1 ; do { + if (readw(uap->port.membase + UART01x_FR) & UART01x_FR_TXFF) + break; writew(xmit->buf[xmit->tail], uap->port.membase + UART01x_DR); xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1); uap->port.icount.tx++;
@@ -249,7 +1066,7 @@ static irqreturn_t pl011_int(int irq, void *dev_id)unsigned int status, pass_counter =AMBA_ISR_PASS_LIMIT ;int handled =0 ; -spin_lock (&uap->port.lock); +spin_lock_irq (&uap->port.lock); status =readw (uap->port.membase +UART011_MIS );if (status) {
@@ -258,13 +1075,30 @@ static irqreturn_t pl011_int(int irq, void *dev_id) UART011_RXIS), uap->port.membase + UART011_ICR); -if (status & (UART011_RTIS |UART011_RXIS )) -pl011_rx_chars (uap); +if (status & (UART011_RTIS |UART011_RXIS )) { +if (uap->enable_dma && uap->rx_dma_running) +pl011_dma_rx_irq (uap); +else +pl011_rx_chars (uap); + }if (status & (UART011_DSRMIS |UART011_DCDMIS |UART011_CTSMIS |UART011_RIMIS ))pl011_modem_status (uap); -if (status &UART011_TXIS ) +if (status &UART011_TXIS ) { +/* +* When DMA is enabled we still use TX +* interrupt to send small amounts of data, +* and as a fallback when the DMA channel is +* not available. This interrupt is cleared +* here and will be enabled when it's needed. +*/ +if (uap->enable_dma) { + uap->im &= ~UART011_TXIM ; +writew (uap->im, + uap->port.membase +UART011_IMSC ); + }pl011_tx_chars (uap); + }if (pass_counter-- ==0 )break ;
@@ -274,7 +1108,7 @@ static irqreturn_t pl011_int(int irq, void *dev_id) handled =1 ; } -spin_unlock (&uap->port.lock); +spin_unlock_irq (&uap->port.lock);return IRQ_RETVAL (handled); }
@@ -423,16 +1257,28 @@ static int pl011_startup(struct uart_port *port) cr = UART01x_CR_UARTEN |UART011_CR_RXE |UART011_CR_TXE ;writew (cr, uap->port.membase +UART011_CR ); +/* Clear pending error interrupts*/ +writew (0xFFFF & ~(UART011_TXIS |UART011_RTIS |UART011_RXIS ), + uap->port.membase +UART011_ICR ); +/* * initialise the old status of the modem signals */ uap->old_status =readw (uap->port.membase + UART01x_FR) & UART01x_FR_MODEM_ANY; +/* Startup DMA */ +pl011_dma_startup (uap); +/* -* Finally, enable interrupts +* Finally, enable interrupts, only timeouts when using DMA +* if initial RX DMA job failed, start in interrupt mode +* as well. */ spin_lock_irq (&uap->port.lock); - uap->im =UART011_RXIM |UART011_RTIM ; +if (uap->enable_dma && uap->rx_dma_running) + uap->im =UART011_RTIM ; +else + uap->im =UART011_RXIM |UART011_RTIM ;writew (uap->im, uap->port.membase +UART011_IMSC );spin_unlock_irq (&uap->port.lock);
@@ -467,6 +1313,8 @@ static void pl011_shutdown(struct uart_port *port)writew (0xffff , uap->port.membase +UART011_ICR );spin_unlock_irq (&uap->port.lock); +pl011_dma_shutdown (uap); +/* * Free the interrupt */
@@ -532,7 +1380,7 @@ pl011_set_termios(struct uart_port *port, struct ktermios *termios,if (!(termios->c_cflag &PARODD )) lcr_h |= UART01x_LCRH_EPS; } -if (port->fifosize >1 ) +if (uap->fifosize >1 ) lcr_h |= UART01x_LCRH_FEN;spin_lock_irqsave (&port->lock, flags);
@@ -862,6 +1710,7 @@ static int pl011_probe(struct amba_device *dev, struct amba_id *id) uap->lcrh_rx = vendor->lcrh_rx; uap->lcrh_tx = vendor->lcrh_tx; uap->oversampling = vendor->oversampling; + uap->dma_init = vendor->dma_init; uap->port.dev = &dev->dev; uap->port.mapbase = dev->res.start; uap->port.membase = base;
@@ -871,6 +1720,8 @@ static int pl011_probe(struct amba_device *dev, struct amba_id *id) uap->port.ops = &amba_pl011_pops; uap->port.flags =UPF_BOOT_AUTOCONF ; uap->port.line = i; + uap->fifosize = vendor->fifosize; +pl011_dma_probe (uap); amba_ports[i] = uap;
@@ -879,6 +1730,7 @@ static int pl011_probe(struct amba_device *dev, struct amba_id *id)if (ret) {amba_set_drvdata (dev,NULL ); amba_ports[i] =NULL ; +pl011_dma_remove (uap);clk_put (uap->clk); unmap:iounmap (base);
@@ -902,6 +1754,7 @@ static int pl011_remove(struct amba_device *dev)if (amba_ports[i] == uap) amba_ports[i] =NULL ; +pl011_dma_remove (uap);iounmap (uap->port.membase);clk_put (uap->clk);kfree (uap);
include/linux/amba/serial.h +21 -0
@@ -113,6 +113,21 @@#define UART01x_LCRH_PEN 0x02#define UART01x_LCRH_BRK 0x01 +#define ST_UART011_DMAWM_RX_1 (0 << 3) +#define ST_UART011_DMAWM_RX_2 (1 << 3) +#define ST_UART011_DMAWM_RX_4 (2 << 3) +#define ST_UART011_DMAWM_RX_8 (3 << 3) +#define ST_UART011_DMAWM_RX_16 (4 << 3) +#define ST_UART011_DMAWM_RX_32 (5 << 3) +#define ST_UART011_DMAWM_RX_48 (6 << 3) +#define ST_UART011_DMAWM_TX_1 0 +#define ST_UART011_DMAWM_TX_2 1 +#define ST_UART011_DMAWM_TX_4 2 +#define ST_UART011_DMAWM_TX_8 3 +#define ST_UART011_DMAWM_TX_16 4 +#define ST_UART011_DMAWM_TX_32 5 +#define ST_UART011_DMAWM_TX_48 6 +#define UART010_IIR_RTIS 0x08#define UART010_IIR_TIS 0x04#define UART010_IIR_RIS 0x02
@@ -180,6 +195,12 @@ struct amba_device; /* in uncompress this is included but amba/bus.h is not */struct amba_pl010_data {void (*set_mctrl)(struct amba_device *dev,void __iomem *base,unsigned int mctrl); }; +struct dma_chan ; +struct amba_pl011_data { +bool (*dma_filter)(struct dma_chan *chan,void *filter_param); +void *dma_rx_param; +void *dma_tx_param; +};#endif #endif
Signature · 1 line
1.6.3.3
PATCH 1/1 ARM: add PrimeCell generic DMA to PL011
Linus,
As these patches depend on several other trees, it's difficult for me to check them on my platforms, and I don't want my tree to break on Realview/Versatile platforms without these dependents - it's one of the platforms I'm using to check out SMP/highmem/memblock stuff.
Unless you can assure me that there will be no breakage through these patches without having their dependencies merged first, I think these will have to wait another cycle...
Quoted 1,114 lines · Linus Walleij
This extends the PL011 UART driver with generic DMA engine support using the PrimeCell DMA engine interface.
Tested-by Jerzy Kasenberg <[email protected]>Tested-by Grzegorz Sygieda <[email protected]>Tested-by Marcin Mielczarczyk <[email protected]>Signed-off-by Linus Walleij <[email protected]>---
Changes from previous version that was in the patch set for PrimeCell DMA (I've lost count):This adds support for the ST-Ericsson specific DMA watermarking via a vendor data-specified setup function, a design pattern likely to be useful for other users as well.
--- drivers/serial/amba-pl011.c | 871 ++++++++++++++++++++++++++++++++++++++++++- include/linux/amba/serial.h | 21 + 2 files changed, 883 insertions(+), 9 deletions(-)drivers/serial/amba-pl011.c +862
-9@@ -7,6 +7,7 @@* * Copyright 1999 ARM Limited * Copyright (C) 2000 Deep Blue Solutions Ltd. +* Copyright (C) 2010 ST-Ericsson SA * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -48,6 +49,11 @@#include <linux/amba/serial.h> #include <linux/clk.h> #include <linux/slab.h> +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> +#include <linux/scatterlist.h> +#include <linux/completion.h> +#include <linux/delay.h> #include <asm/io.h> #include <asm/sizes.h> @@ -63,6 +69,27 @@#define UART_DR_ERROR (UART011_DR_OE|UART011_DR_BE|UART011_DR_PE|UART011_DR_FE)#define UART_DUMMY_DR_RX (1 << 16) +struct uart_amba_port ; +typedef void (*dma_init_fn )(struct uart_amba_port *uap); + +/* Deals with DMA transactions */ +struct pl011_dma_rx_transaction { +struct completion complete; +bool use_buffer_b; +struct scatterlist scatter_a; +struct scatterlist scatter_b; +char *rx_dma_buf_a; +char *rx_dma_buf_b; +dma_cookie_t cookie; +}; + +struct pl011_dma_tx_transaction { +struct completion complete; +struct scatterlist scatter; +char *tx_dma_buf; +dma_cookie_t cookie; +}; +/* * We wrap our port structure around the generic uart_port. */ @@ -75,7 +102,18 @@ struct uart_amba_port {unsigned int lcrh_tx;/* vendor-specific */ unsigned int lcrh_rx;/* vendor-specific */ bool oversampling;/* vendor-specific */ +dma_init_fn dma_init;/* vendor-specific */ bool autorts; +unsigned int fifosize; +/* DMA stuff */ +bool enable_dma; +bool rx_dma_running; +#ifdef CONFIG_DMA_ENGINE +struct dma_chan *dma_rx_channel; +struct dma_chan *dma_tx_channel; +struct pl011_dma_rx_transaction dmarx; +struct pl011_dma_tx_transaction dmatx; +#endif };/* There is by now at least one vendor with differing details, so handle it */ @@ -85,6 +123,7 @@ struct vendor_data {unsigned int lcrh_tx;unsigned int lcrh_rx;bool oversampling; +dma_init_fn dma_init; };static struct vendor_data vendor_arm = {@@ -95,14 +134,747 @@ static struct vendor_data vendor_arm = { .oversampling = false, }; +static void pl011_st_dma_startup (struct uart_amba_port *uap); +static struct vendor_data vendor_st = { .ifls =UART011_IFLS_RX_HALF |UART011_IFLS_TX_HALF , .fifosize =64 , .lcrh_tx =ST_UART011_LCRH_TX , .lcrh_rx =ST_UART011_LCRH_RX , .oversampling = true, + .dma_init = pl011_st_dma_startup, }; +/* +* All the DMA operation mode stuff goes inside this ifdef. +* This assumes that you have a generic DMA device interface, +* no custom DMA interfaces are supported. +* +* If we had discardable probe() functions akin to +* platform_device_probe() in the PrimeCell/AMBA bus, we could +* discard most of this code after use, but since we haven't, +* we have to keep it all around. +*/ +#ifdef CONFIG_DMA_ENGINE + +#define PL011_DMA_BUFFER_SIZE PAGE_SIZE + +static void pl011_dma_probe_initcall (struct uart_amba_port *uap) +{ +/* DMA is the sole user of the platform data right now */ +struct amba_pl011_data *plat = uap->port.dev->platform_data; +struct pl011_dma_rx_transaction *dmarx = &uap->dmarx; +struct pl011_dma_tx_transaction *dmatx = &uap->dmatx; +struct dma_slave_config rx_conf = { + .src_addr = uap->port.mapbase + UART01x_DR, + .src_addr_width =DMA_SLAVE_BUSWIDTH_1_BYTE , + .direction =DMA_FROM_DEVICE , + .src_maxburst = uap->fifosize >>1 , + }; +struct dma_slave_config tx_conf = { + .dst_addr = uap->port.mapbase + UART01x_DR, + .dst_addr_width =DMA_SLAVE_BUSWIDTH_1_BYTE , + .direction =DMA_TO_DEVICE , + .dst_maxburst = uap->fifosize >>1 , + }; +dma_cap_mask_t mask; +int sglen; + +/* We need platform data */ +if (!plat) { +dev_err (uap->port.dev,"no DMA platform data!\n" ); +return ; + } + +/* Try to acquire a generic DMA engine slave channel */ +dma_cap_zero (mask); +dma_cap_set (DMA_SLAVE , mask); + +/* +* We need both RX and TX channels to do DMA, else do none +* of them. +*/ + uap->dma_rx_channel =dma_request_channel (mask, + plat->dma_filter, + plat->dma_rx_param); +if (!uap->dma_rx_channel) { +dev_err (uap->port.dev,"no RX DMA channel!\n" ); +return ; + } + uap->dma_rx_channel->device->device_control (uap->dma_rx_channel, +DMA_SLAVE_CONFIG , + (unsigned long ) &rx_conf); + + uap->dma_tx_channel =dma_request_channel (mask, + plat->dma_filter, + plat->dma_tx_param); +if (!uap->dma_tx_channel) { +dev_err (uap->port.dev,"no TX DMA channel!\n" ); + goto err_no_txchan; + } + uap->dma_tx_channel->device->device_control (uap->dma_tx_channel, +DMA_SLAVE_CONFIG , + (unsigned long ) &tx_conf); + +/* Allocate DMA RX and TX buffers */ + dmarx->rx_dma_buf_a =kmalloc (PL011_DMA_BUFFER_SIZE ,GFP_KERNEL ); +if (!dmarx->rx_dma_buf_a) { +dev_err (uap->port.dev,"failed to allocate DMA RX buffer A\n" ); + goto err_no_rxbuf_a; + } + + dmarx->rx_dma_buf_b =kmalloc (PL011_DMA_BUFFER_SIZE ,GFP_KERNEL ); +if (!dmarx->rx_dma_buf_b) { +dev_err (uap->port.dev,"failed to allocate DMA RX buffer B\n" ); + goto err_no_rxbuf_b; + } + + dmatx->tx_dma_buf =kmalloc (PL011_DMA_BUFFER_SIZE ,GFP_KERNEL ); +if (!dmatx->tx_dma_buf) { +dev_err (uap->port.dev,"failed to allocate DMA TX buffer\n" ); + goto err_no_txbuf; + } + +/* Provide single SG list with one item to the buffers */ +sg_init_one (&dmarx->scatter_a, dmarx->rx_dma_buf_a, +PL011_DMA_BUFFER_SIZE ); +sg_init_one (&dmarx->scatter_b, dmarx->rx_dma_buf_b, +PL011_DMA_BUFFER_SIZE ); +sg_init_one (&dmatx->scatter, dmatx->tx_dma_buf,PL011_DMA_BUFFER_SIZE ); + +/* Map DMA buffers */ + sglen =dma_map_sg (uap->port.dev, &dmarx->scatter_a, +1 ,DMA_FROM_DEVICE ); +if (sglen !=1 ) + goto err_rx_sgmap_a; + + sglen =dma_map_sg (uap->port.dev, &dmarx->scatter_b, +1 ,DMA_FROM_DEVICE ); +if (sglen !=1 ) + goto err_rx_sgmap_b; + + sglen =dma_map_sg (uap->port.dev, &dmatx->scatter, +1 ,DMA_TO_DEVICE ); +if (sglen !=1 ) + goto err_tx_sgmap; + +/* Initially we say the transfers are incomplete */ +init_completion (&uap->dmatx.complete); +complete (&uap->dmatx.complete); + +/* The DMA buffer is now the FIFO the TTY subsystem can use */ + uap->port.fifosize =PL011_DMA_BUFFER_SIZE ; + + uap->enable_dma = true; +dev_info (uap->port.dev,"setup for DMA on RX %s, TX %s\n" , +dma_chan_name (uap->dma_rx_channel), +dma_chan_name (uap->dma_tx_channel)); +return ; + +err_tx_sgmap: +dma_unmap_sg (uap->port.dev, &dmarx->scatter_b, +1 ,DMA_FROM_DEVICE ); +err_rx_sgmap_b: +dma_unmap_sg (uap->port.dev, &dmarx->scatter_a, +1 ,DMA_FROM_DEVICE ); +err_rx_sgmap_a: +kfree (dmatx->tx_dma_buf); +err_no_txbuf: +kfree (dmarx->rx_dma_buf_b); +err_no_rxbuf_b: +kfree (dmarx->rx_dma_buf_a); +err_no_rxbuf_a: +dma_release_channel (uap->dma_tx_channel); + uap->dma_tx_channel =NULL ; +err_no_txchan: +dma_release_channel (uap->dma_rx_channel); + uap->dma_rx_channel =NULL ; +return ; +} + +/* +* Stack up the UARTs and let the above initcall be done at +* device initcall time, because the serial driver is called as +* an arch initcall, and at this time the DMA subsystem is not yet +* registered. At this point the driver will switch over to using +* DMA where desired. +*/ + +struct dma_uap { +struct list_head node; +struct uart_amba_port *uap; +}; + +struct list_head pl011_dma_uarts =LIST_HEAD_INIT (pl011_dma_uarts); + +static int pl011_dma_initcall (void ) +{ +struct list_head *node, *tmp; + +list_for_each_safe (node, tmp, &pl011_dma_uarts) { + struct dma_uap *dmau = list_entry(node, struct dma_uap, node); +pl011_dma_probe_initcall (dmau->uap); +list_del (node); +kfree (dmau); + } +return 0 ; +} + +device_initcall (pl011_dma_initcall); + +static void pl011_dma_probe (struct uart_amba_port *uap) +{ +struct dma_uap *dmau =kzalloc (sizeof (struct dma_uap ),GFP_KERNEL ); + +if (dmau ==NULL ) +return ; + dmau->uap = uap; +list_add_tail (&dmau->node, &pl011_dma_uarts); +} + +static void pl011_dma_remove (struct uart_amba_port *uap) +{ +struct pl011_dma_rx_transaction *dmarx = &uap->dmarx; +struct pl011_dma_tx_transaction *dmatx = &uap->dmatx; + +/* TODO: remove the initcall if it has not yet executed */ +/* Unmap and free DMA buffers */ +if (uap->dma_rx_channel) +dma_release_channel (uap->dma_rx_channel); +if (uap->dma_tx_channel) +dma_release_channel (uap->dma_tx_channel); +if (dmatx->tx_dma_buf) { +dma_unmap_sg (uap->port.dev, &dmatx->scatter, +1 ,DMA_TO_DEVICE ); +kfree (dmatx->tx_dma_buf); + } +if (dmarx->rx_dma_buf_b) { +dma_unmap_sg (uap->port.dev, &dmarx->scatter_b, +1 ,DMA_FROM_DEVICE ); +kfree (dmarx->rx_dma_buf_b); + } +if (dmarx->rx_dma_buf_a) { +dma_unmap_sg (uap->port.dev, &dmarx->scatter_a, +1 ,DMA_FROM_DEVICE ); +kfree (dmarx->rx_dma_buf_a); + } +} + +/* Forward declare this for the refill routine */ +static int pl011_dma_tx_refill (struct uart_amba_port *uap); + +/* +* Move the tail when this IRQ occurs, if not empty refill and +* fire another transaction +*/ +static void pl011_dma_tx_callback (void *data) +{ +struct uart_amba_port *uap = data; +struct pl011_dma_tx_transaction *dmatx = &uap->dmatx; +struct circ_buf *xmit = &uap->port.state->xmit; +u16 val; +int ret; + +/* Temporarily disable TX DMA */ + val =readw (uap->port.membase +UART011_DMACR ); + val &= ~(UART011_TXDMAE ); +writew (val, uap->port.membase +UART011_DMACR ); + +/* Refill the TX if the buffer is not empty */ +if (!uart_circ_empty (xmit)) { + ret =pl011_dma_tx_refill (uap); +if (ret == -EBUSY ) +/* +* If DMA cannot be used right now, we complete this +* transaction and let the TTY layer retry. If the +* firs following xfer fails to set up for DMA, it +* will fall through to interrupt mode. +*/ +dev_dbg (uap->port.dev,"DMA busy\n" ); + }else { +complete (&dmatx->complete); + } +} + +static int pl011_dma_tx_refill (struct uart_amba_port *uap) +{ +struct pl011_dma_tx_transaction *dmatx = &uap->dmatx; +struct dma_chan *chan = uap->dma_tx_channel; +struct dma_async_tx_descriptor *desc; +struct circ_buf *xmit = &uap->port.state->xmit; +unsigned int count; +unsigned long flags; +u16 val; + +/* Don't bother about using DMA on XON/XOFF */ +if (uap->port.x_char) { +/* If we can't get it into the FIFO, retry later */ +if (readw (uap->port.membase + UART01x_FR) & + UART01x_FR_TXFF) { +complete (&dmatx->complete); +return 0 ; + } +writew (uap->port.x_char, uap->port.membase + UART01x_DR); + uap->port.icount.tx++; + uap->port.x_char =0 ; +complete (&dmatx->complete); +return 0 ; + } + +/* +* Try to avoid the overhead involved in using DMA if the +* transaction fits in the first half of the FIFO and it's not +* full. Unfortunately there is only one single bit in the +* hardware to tell whether the FIFO is full or not, so +* we don't know exactly how many chars we can fit in. +*/ +if (uart_circ_chars_pending (xmit) < (uap->fifosize >>1 )) { +while (uart_circ_chars_pending (xmit)) { +if (readw (uap->port.membase + UART01x_FR) & + UART01x_FR_TXFF) { +/* +* Ooops TX FIFO is full, we'd better stop +* this. Let's enable TX interrupt here to get +* informed when there is again some space in +* the TX FIFO so we can continue the transfer. +* This interrupt will be cleared just before +* setting up DMA, as it could interfere with +* TX interrupt handling routine. +*/ + uap->im |=UART011_TXIM ; +writew (uap->im, + uap->port.membase +UART011_IMSC ); +break ; + } +writew (xmit->buf[xmit->tail], + uap->port.membase + UART01x_DR); + uap->port.icount.tx++; + xmit->tail = (xmit->tail +1 ) & (UART_XMIT_SIZE -1 ); + } +complete (&dmatx->complete); +return 0 ; + } + +/* +* Clear TX interrupt to be sure that DMA will not interfere with +* TX ISR +*/ +local_irq_save (flags); + uap->im &= ~UART011_TXIM ; +writew (uap->im, uap->port.membase +UART011_IMSC ); +local_irq_restore (flags); + +/* Sync the buffer for the CPU so we can write into it */ +dma_sync_sg_for_cpu (uap->port.dev, + &dmatx->scatter, +1 , +DMA_TO_DEVICE ); + +/* Else proceed to copy the TX chars to the DMA buffer and fire DMA */ + count =uart_circ_chars_pending (xmit); +if (count >PL011_DMA_BUFFER_SIZE ) + count =PL011_DMA_BUFFER_SIZE ; + +if (xmit->tail < xmit->head) +memcpy (&dmatx->tx_dma_buf[0 ], &xmit->buf[xmit->tail], count); +else { +size_t first =UART_XMIT_SIZE - xmit->tail; +size_t second = xmit->head; + +memcpy (&dmatx->tx_dma_buf[0 ], &xmit->buf[xmit->tail], first); +memcpy (&dmatx->tx_dma_buf[first], &xmit->buf[0 ], second); + } + + dmatx->scatter.length = count; + +/* Synchronize the scatterlist, invalidate buffers, caches etc */ +dma_sync_sg_for_device (uap->port.dev, + &dmatx->scatter, +1 , +DMA_TO_DEVICE ); + +/* Prepare the scatterlist */ + desc = chan->device->device_prep_slave_sg (chan, + &dmatx->scatter, +1 , +DMA_TO_DEVICE , +DMA_PREP_INTERRUPT |DMA_CTRL_ACK ); +if (!desc) { +/* "Complete" DMA (errorpath) */ +complete (&dmatx->complete); + chan->device->device_control (chan,DMA_TERMINATE_ALL ,0 ); +return -EBUSY ; + } + +/* Some data to go along to the callback */ + desc->callback = pl011_dma_tx_callback; + desc->callback_param = uap; + +/* Here is where overloaded DMA controllers can fail */ + dmatx->cookie = desc->tx_submit (desc); +if (dma_submit_error (dmatx->cookie)) { +/* "Complete" DMA (errorpath) */ +complete (&dmatx->complete); + chan->device->device_control (chan,DMA_TERMINATE_ALL ,0 ); +return dmatx->cookie; + } + +/* +* Now we know that DMA will fire, so advance the ring buffer +* with the stuff we just dispatched +*/ + xmit->tail = (xmit->tail + count) & (UART_XMIT_SIZE -1 ); + uap->port.icount.tx += count; +if (uart_circ_chars_pending (xmit) <WAKEUP_CHARS ) +uart_write_wakeup (&uap->port); + +/* Fire the DMA transaction */ + chan->device->device_issue_pending (chan); + + val =readw (uap->port.membase +UART011_DMACR ); + val |=UART011_TXDMAE ; +writew (val, uap->port.membase +UART011_DMACR ); +return 0 ; +} + +static void pl011_dma_rx_callback (void *data); + +static int pl011_dma_rx_trigger_dma (struct uart_amba_port *uap) +{ +struct dma_chan *rxchan = uap->dma_rx_channel; +struct pl011_dma_rx_transaction *dmarx = &uap->dmarx; +struct dma_async_tx_descriptor *desc; +struct scatterlist *scatter = dmarx->use_buffer_b ? + &dmarx->scatter_b : &dmarx->scatter_a; +u16 val; + +/* Start the RX DMA job */ + desc = rxchan->device->device_prep_slave_sg (rxchan, + scatter, +1 , +DMA_FROM_DEVICE , +DMA_PREP_INTERRUPT |DMA_CTRL_ACK ); +/* +* If the DMA engine is busy and cannot prepare a +* channel, no big deal, the driver will fall back +* to interrupt mode as a result of this error code. +*/ +if (!desc) { + uap->rx_dma_running = false; + rxchan->device->device_control (rxchan,DMA_TERMINATE_ALL ,0 ); +return -EBUSY ; + } + +/* Some data to go along to the callback */ + desc->callback = pl011_dma_rx_callback; + desc->callback_param = uap; +/* This is another point where an overloaded engine can fail */ + dmarx->cookie = desc->tx_submit (desc); +if (dma_submit_error (dmarx->cookie)) { + uap->rx_dma_running = false; + rxchan->device->device_control (rxchan,DMA_TERMINATE_ALL ,0 ); +return -EBUSY ; + } + + rxchan->device->device_issue_pending (rxchan); + + val =readw (uap->port.membase +UART011_DMACR ); + val |=UART011_RXDMAE ; +writew (val, uap->port.membase +UART011_DMACR ); + uap->rx_dma_running = true; + +return 0 ; +} + +/* +* This is called when either the DMA job is complete, or +* the FIFO timeout interrupt occurred. This must be called +* with the port spinlock uap->port.lock held. +*/ +static void pl011_dma_rx_chars (struct uart_amba_port *uap, +u32 pending,bool use_buffer_b, +bool readfifo) +{ +struct pl011_dma_rx_transaction *dmarx = &uap->dmarx; +struct tty_struct *tty = uap->port.state->port.tty; +char *buf = use_buffer_b ? dmarx->rx_dma_buf_b : dmarx->rx_dma_buf_a; +struct scatterlist *scatter = use_buffer_b ? + &dmarx->scatter_b : &dmarx->scatter_a; +unsigned int status, ch, flag; +u32 count = pending; +u32 bufp =0 ; +u32 fifotaken =0 ;/* only used for vdbg() */ + +/* Sync in buffer */ +dma_sync_sg_for_cpu (uap->port.dev, + scatter, +1 , +DMA_FROM_DEVICE ); + + status =readw (uap->port.membase + UART01x_FR); + +/* +* First take all chars in the DMA pipe, then look +* in the FIFO. So loop while we have chars in the +* DMA buffer or the FIFO. If we came here from a +* DMA buffer full interrupt, there is already another +* DMA job triggered to read the FIFO, so don't look +* at it. +*/ +while (count || + (readfifo && (status & UART01x_FR_RXFE) ==0 )) { + + flag =TTY_NORMAL ; + uap->port.icount.rx++; + +if (count) { +/* Take chars from the DMA buffer */ +int inserted =tty_insert_flip_string ( + uap->port.state->port.tty, buf, count); + +/* +* Check if insertion is successful to avoid +* infinite loop. This can happen when TTY is full. +*/ +if (unlikely (inserted ==0 )) + count =0 ; +else { + count -= inserted; + bufp += inserted; + } +continue ; + }else { +/* Take chars from the FIFO and update status */ + ch =readw (uap->port.membase + UART01x_DR); + status =readw (uap->port.membase + UART01x_FR); + fifotaken++; + +/* +* Error conditions will only occur in the FIFO, +* these will trigger an immediate interrupt and +* stop the DMA job, so we will always find the +* error in the FIFO, never in the DMA buffer. +*/ +if (unlikely (ch &UART_DR_ERROR )) { +if (ch &UART011_DR_BE ) { + ch &= ~(UART011_DR_FE |UART011_DR_PE ); + uap->port.icount.brk++; +if (uart_handle_break (&uap->port)) +continue ; + }else if (ch &UART011_DR_PE ) + uap->port.icount.parity++; +else if (ch &UART011_DR_FE ) + uap->port.icount.frame++; +if (ch &UART011_DR_OE ) + uap->port.icount.overrun++; + + ch &= uap->port.read_status_mask; + +if (ch &UART011_DR_BE ) + flag =TTY_BREAK ; +else if (ch &UART011_DR_PE ) + flag =TTY_PARITY ; +else if (ch &UART011_DR_FE ) + flag =TTY_FRAME ; + } + } + +if (uart_handle_sysrq_char (&uap->port, ch &255 )) +continue ; + +uart_insert_char (&uap->port, ch,UART011_DR_OE , ch, flag); + } + +spin_unlock (&uap->port.lock); +dev_vdbg (uap->port.dev, +"Took %d chars from DMA buffer and %d chars from the FIFO\n" , + bufp, fifotaken); +tty_flip_buffer_push (tty); +spin_lock (&uap->port.lock); +} + +static void pl011_dma_rx_irq (struct uart_amba_port *uap) +{ +struct dma_chan *rxchan = uap->dma_rx_channel; +struct pl011_dma_rx_transaction *dmarx = &uap->dmarx; +struct scatterlist *scatter = dmarx->use_buffer_b ? + &dmarx->scatter_b : &dmarx->scatter_a; +u32 pending; +int ret; +struct dma_tx_state state; +enum dma_status dmastat; +u16 val; + +/* Use PrimeCell DMA extensions to stop the transfer */ + ret = rxchan->device->device_control (rxchan,DMA_PAUSE ,0 ); +if (ret) +dev_err (uap->port.dev,"unable to pause DMA transfer\n" ); + dmastat = rxchan->device->device_tx_status (rxchan, + dmarx->cookie, &state); + +/* Disable RX DMA temporarily */ + val =readw (uap->port.membase +UART011_DMACR ); + val &= ~(UART011_RXDMAE ); +writew (val, uap->port.membase +UART011_DMACR ); + uap->rx_dma_running = false; + +if (dmastat !=DMA_PAUSED ) +dev_err (uap->port.dev,"unable to pause DMA transfer\n" ); + pending = scatter->length - state.residue; + +BUG_ON (pending >PL011_DMA_BUFFER_SIZE ); + + ret = rxchan->device->device_control (rxchan,DMA_TERMINATE_ALL ,0 ); +if (ret) +dev_err (uap->port.dev,"unable to terminate DMA transfer\n" ); + +/* +* This will take the chars we have so far and insert +* into the framework. +*/ +pl011_dma_rx_chars (uap, pending, dmarx->use_buffer_b, true); + +/* Switch buffer & re-trigger DMA job */ + dmarx->use_buffer_b = !dmarx->use_buffer_b; + ret =pl011_dma_rx_trigger_dma (uap); +if (ret) { +dev_dbg (uap->port.dev,"could not retrigger RX DMA job " +"fall back to interrupt mode\n" ); + uap->im |=UART011_RXIM ; +writew (uap->im, uap->port.membase +UART011_IMSC ); + } +} + +static void pl011_dma_rx_callback (void *data) +{ +struct uart_amba_port *uap = data; +struct pl011_dma_rx_transaction *dmarx = &uap->dmarx; +bool lastbuf = dmarx->use_buffer_b; +int ret; + +/* +* This completion interrupt occurs typically when the +* RX buffer is totally stuffed but no timeout has yet +* occurred. When that happens, we just want the RX +* routine to flush out the secondary DMA buffer while +* we immediately trigger the next DMA job. +*/ + uap->rx_dma_running = false; + dmarx->use_buffer_b = !lastbuf; + ret =pl011_dma_rx_trigger_dma (uap); + +spin_lock_irq (&uap->port.lock); +pl011_dma_rx_chars (uap,PL011_DMA_BUFFER_SIZE , lastbuf, false); +spin_unlock_irq (&uap->port.lock); +/* +* Do this check after we picked the DMA chars so we don't +* get some IRQ immediately from RX. +*/ +if (ret) { +dev_dbg (uap->port.dev,"could not retrigger RX DMA job " +"fall back to interrupt mode\n" ); + uap->im |=UART011_RXIM ; +writew (uap->im, uap->port.membase +UART011_IMSC ); + } +} + +static void pl011_st_dma_startup (struct uart_amba_port *uap) +{ +/* Set DMABREQ threshold */ +writew (ST_UART011_DMAWM_RX_16 |ST_UART011_DMAWM_TX_16 , + uap->port.membase +ST_UART011_DMAWM ); +} + +static void pl011_dma_startup (struct uart_amba_port *uap) +{ +u16 val; +int ret =0 ; + +if (!uap->enable_dma) +return ; + +/* Turn on DMA error (RX/TX will be enabled on demand) */ + val =readw (uap->port.membase +UART011_DMACR ); + val |=UART011_DMAONERR ; +writew (val, uap->port.membase +UART011_DMACR ); + +/* call vendor specific dma init */ +if (uap->dma_init) + uap->dma_init (uap); + + ret =pl011_dma_rx_trigger_dma (uap); +if (ret) +dev_dbg (uap->port.dev,"could not trigger initial " +"RX DMA job, fall back to interrupt mode\n" ); +} + +static void pl011_dma_shutdown (struct uart_amba_port *uap) +{ +struct dma_chan *rxchan = uap->dma_rx_channel; +struct dma_chan *txchan = uap->dma_tx_channel; +u16 val; + +if (!uap->enable_dma) +return ; + +/* Disable RX and TX DMA */ +while (readw (uap->port.membase + UART01x_FR) & UART01x_FR_BUSY) +barrier (); + val =readw (uap->port.membase +UART011_DMACR ); + val &= ~(UART011_DMAONERR |UART011_RXDMAE |UART011_TXDMAE ); +writew (val, uap->port.membase +UART011_DMACR ); +/* Terminate any RX and TX DMA jobs */ + rxchan->device->device_control (rxchan,DMA_TERMINATE_ALL ,0 ); + txchan->device->device_control (txchan,DMA_TERMINATE_ALL ,0 ); +} + +static int pl011_dma_tx_chars (struct uart_amba_port *uap) +{ +struct pl011_dma_tx_transaction *dmatx = &uap->dmatx; + +/* Try to wait for completion, return if something is in progress */ +if (!try_wait_for_completion (&dmatx->complete)) +return -EINPROGRESS ; + +/* Set up and fire the DMA job */ +init_completion (&dmatx->complete); +return pl011_dma_tx_refill (uap); +} + +#else +/* Blank functions if the DMA engine is not available */ +static inline void pl011_dma_probe (struct uart_amba_port *uap) +{ +} + +static inline void pl011_dma_remove (struct uart_amba_port *uap) +{ +} + +static inline void pl011_dma_rx_irq (struct uart_amba_port *uap) +{ +} + +static inline int pl011_dma_rx_trigger_dma (struct uart_amba_port *uap) +{ +return -EIO ; +} + +static inline void pl011_dma_startup (struct uart_amba_port *uap) +{ +} + +static inline void pl011_dma_shutdown (struct uart_amba_port *uap) +{ +} + +static inline int pl011_dma_tx_chars (struct uart_amba_port *uap) +{ +return -EIO ; +} +#endif + +static void pl011_stop_tx (struct uart_port *port) {struct uart_amba_port *uap = (struct uart_amba_port *)port;@@ -111,10 +883,18 @@ static void pl011_stop_tx(struct uart_port *port)writew (uap->im, uap->port.membase +UART011_IMSC ); } +static void pl011_tx_chars (struct uart_amba_port *uap); +static void pl011_start_tx (struct uart_port *port) {struct uart_amba_port *uap = (struct uart_amba_port *)port; +if (uap->enable_dma) { +/* Immediately push out chars in DMA mode */ +pl011_tx_chars (uap); +return ; + } +/* In interrupt mode, let the interrupt pull chars */ uap->im |=UART011_TXIM ;writew (uap->im, uap->port.membase +UART011_IMSC ); }@@ -140,6 +920,7 @@ static void pl011_rx_chars(struct uart_amba_port *uap) {struct tty_struct *tty = uap->port.state->port.tty;unsigned int status, ch, flag, max_count =256 ; +int ret; status =readw (uap->port.membase + UART01x_FR);while ((status & UART01x_FR_RXFE) ==0 && max_count--) {@@ -184,6 +965,21 @@ static void pl011_rx_chars(struct uart_amba_port *uap) }spin_unlock (&uap->port.lock);tty_flip_buffer_push (tty); +/* +* If we were temporarily out of DMA mode for a while, +* attempt to switch back to DMA mode again. +*/ +if (uap->enable_dma) { + uap->im &= ~UART011_RXIM ; +writew (uap->im, uap->port.membase +UART011_IMSC ); + ret =pl011_dma_rx_trigger_dma (uap); +if (ret) { +dev_dbg (uap->port.dev,"could not trigger RX DMA job " +"fall back to interrupt mode again\n" ); + uap->im |=UART011_RXIM ; +writew (uap->im, uap->port.membase +UART011_IMSC ); + } + }spin_lock (&uap->port.lock); }@@ -192,6 +988,25 @@ static void pl011_tx_chars(struct uart_amba_port *uap)struct circ_buf *xmit = &uap->port.state->xmit;int count; +if (uap->enable_dma) { +int ret; + + ret =pl011_dma_tx_chars (uap); +if (!ret) +return ; +if (ret == -EINPROGRESS ) +return ; + +/* +* On any other error (including -EBUSY which is emitted +* in case the DMA engine is out of physical channels +* for example) we fall through to interrupt mode +*/ +dev_dbg (uap->port.dev,"DMA unavailable for TX\n" ); + uap->im |=UART011_TXIM ; +writew (uap->im, uap->port.membase +UART011_IMSC ); + } +if (uap->port.x_char) {writew (uap->port.x_char, uap->port.membase + UART01x_DR); uap->port.icount.tx++;@@ -203,8 +1018,10 @@ static void pl011_tx_chars(struct uart_amba_port *uap)return ; } - count = uap->port.fifosize >>1 ; + count = uap->fifosize >>1 ; do { + if (readw(uap->port.membase + UART01x_FR) & UART01x_FR_TXFF) + break; writew(xmit->buf[xmit->tail], uap->port.membase + UART01x_DR); xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1); uap->port.icount.tx++;@@ -249,7 +1066,7 @@ static irqreturn_t pl011_int(int irq, void *dev_id)unsigned int status, pass_counter =AMBA_ISR_PASS_LIMIT ;int handled =0 ; -spin_lock (&uap->port.lock); +spin_lock_irq (&uap->port.lock); status =readw (uap->port.membase +UART011_MIS );if (status) {@@ -258,13 +1075,30 @@ static irqreturn_t pl011_int(int irq, void *dev_id) UART011_RXIS), uap->port.membase + UART011_ICR); -if (status & (UART011_RTIS |UART011_RXIS )) -pl011_rx_chars (uap); +if (status & (UART011_RTIS |UART011_RXIS )) { +if (uap->enable_dma && uap->rx_dma_running) +pl011_dma_rx_irq (uap); +else +pl011_rx_chars (uap); + }if (status & (UART011_DSRMIS |UART011_DCDMIS |UART011_CTSMIS |UART011_RIMIS ))pl011_modem_status (uap); -if (status &UART011_TXIS ) +if (status &UART011_TXIS ) { +/* +* When DMA is enabled we still use TX +* interrupt to send small amounts of data, +* and as a fallback when the DMA channel is +* not available. This interrupt is cleared +* here and will be enabled when it's needed. +*/ +if (uap->enable_dma) { + uap->im &= ~UART011_TXIM ; +writew (uap->im, + uap->port.membase +UART011_IMSC ); + }pl011_tx_chars (uap); + }if (pass_counter-- ==0 )break ;@@ -274,7 +1108,7 @@ static irqreturn_t pl011_int(int irq, void *dev_id) handled =1 ; } -spin_unlock (&uap->port.lock); +spin_unlock_irq (&uap->port.lock);return IRQ_RETVAL (handled); }@@ -423,16 +1257,28 @@ static int pl011_startup(struct uart_port *port) cr = UART01x_CR_UARTEN |UART011_CR_RXE |UART011_CR_TXE ;writew (cr, uap->port.membase +UART011_CR ); +/* Clear pending error interrupts*/ +writew (0xFFFF & ~(UART011_TXIS |UART011_RTIS |UART011_RXIS ), + uap->port.membase +UART011_ICR ); +/* * initialise the old status of the modem signals */ uap->old_status =readw (uap->port.membase + UART01x_FR) & UART01x_FR_MODEM_ANY; +/* Startup DMA */ +pl011_dma_startup (uap); +/* -* Finally, enable interrupts +* Finally, enable interrupts, only timeouts when using DMA +* if initial RX DMA job failed, start in interrupt mode +* as well. */ spin_lock_irq (&uap->port.lock); - uap->im =UART011_RXIM |UART011_RTIM ; +if (uap->enable_dma && uap->rx_dma_running) + uap->im =UART011_RTIM ; +else + uap->im =UART011_RXIM |UART011_RTIM ;writew (uap->im, uap->port.membase +UART011_IMSC );spin_unlock_irq (&uap->port.lock);@@ -467,6 +1313,8 @@ static void pl011_shutdown(struct uart_port *port)writew (0xffff , uap->port.membase +UART011_ICR );spin_unlock_irq (&uap->port.lock); +pl011_dma_shutdown (uap); +/* * Free the interrupt */ @@ -532,7 +1380,7 @@ pl011_set_termios(struct uart_port *port, struct ktermios *termios,if (!(termios->c_cflag &PARODD )) lcr_h |= UART01x_LCRH_EPS; } -if (port->fifosize >1 ) +if (uap->fifosize >1 ) lcr_h |= UART01x_LCRH_FEN;spin_lock_irqsave (&port->lock, flags);@@ -862,6 +1710,7 @@ static int pl011_probe(struct amba_device *dev, struct amba_id *id) uap->lcrh_rx = vendor->lcrh_rx; uap->lcrh_tx = vendor->lcrh_tx; uap->oversampling = vendor->oversampling; + uap->dma_init = vendor->dma_init; uap->port.dev = &dev->dev; uap->port.mapbase = dev->res.start; uap->port.membase = base;@@ -871,6 +1720,8 @@ static int pl011_probe(struct amba_device *dev, struct amba_id *id) uap->port.ops = &amba_pl011_pops; uap->port.flags =UPF_BOOT_AUTOCONF ; uap->port.line = i; + uap->fifosize = vendor->fifosize; +pl011_dma_probe (uap); amba_ports[i] = uap;@@ -879,6 +1730,7 @@ static int pl011_probe(struct amba_device *dev, struct amba_id *id)if (ret) {amba_set_drvdata (dev,NULL ); amba_ports[i] =NULL ; +pl011_dma_remove (uap);clk_put (uap->clk); unmap:iounmap (base);@@ -902,6 +1754,7 @@ static int pl011_remove(struct amba_device *dev)if (amba_ports[i] == uap) amba_ports[i] =NULL ; +pl011_dma_remove (uap);iounmap (uap->port.membase);clk_put (uap->clk);kfree (uap);include/linux/amba/serial.h +21
-0@@ -113,6 +113,21 @@#define UART01x_LCRH_PEN 0x02#define UART01x_LCRH_BRK 0x01 +#define ST_UART011_DMAWM_RX_1 (0 << 3) +#define ST_UART011_DMAWM_RX_2 (1 << 3) +#define ST_UART011_DMAWM_RX_4 (2 << 3) +#define ST_UART011_DMAWM_RX_8 (3 << 3) +#define ST_UART011_DMAWM_RX_16 (4 << 3) +#define ST_UART011_DMAWM_RX_32 (5 << 3) +#define ST_UART011_DMAWM_RX_48 (6 << 3) +#define ST_UART011_DMAWM_TX_1 0 +#define ST_UART011_DMAWM_TX_2 1 +#define ST_UART011_DMAWM_TX_4 2 +#define ST_UART011_DMAWM_TX_8 3 +#define ST_UART011_DMAWM_TX_16 4 +#define ST_UART011_DMAWM_TX_32 5 +#define ST_UART011_DMAWM_TX_48 6 +#define UART010_IIR_RTIS 0x08#define UART010_IIR_TIS 0x04#define UART010_IIR_RIS 0x02@@ -180,6 +195,12 @@ struct amba_device; /* in uncompress this is included but amba/bus.h is not */struct amba_pl010_data {void (*set_mctrl)(struct amba_device *dev,void __iomem *base,unsigned int mctrl); }; +struct dma_chan ; +struct amba_pl011_data { +bool (*dma_filter)(struct dma_chan *chan,void *filter_param); +void *dma_rx_param; +void *dma_tx_param; +};#endif #endif Signature · 7 lines
1.6.3.3 _______________________________________________ linux-arm-kernel mailing list linux-arm-kernel at lists.infradead.org http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
PATCH 1/1 ARM: add PrimeCell generic DMA to PL011
2010/10/12 Russell King - ARM Linux <[email protected]>:
Quoted 3 lines
Unless you can assure me that there will be no breakage through these patches without having their dependencies merged first, I think these will have to wait another cycle...
It's OK Russell I have all patience in the world :-)
We ended up in a bit of cross-subsystem hell and such things happen.
We'll have the PL08x driver and other necessary bits in by 2.6.37 and then we can perhaps look at these just after that merge window. Then the PL022 that is not in as wide use as the others and which is still marked experimental is merged, so we'll use that thing as a guninea pig, if it doesn't blow up it's probably safe to merge the others as well.
If we then merge them and they seem to work my plan is to move the DMA channel assignments for the drivers into the struct amba_device as the final step.
Yours,
Linus Walleij
PATCH 1/1 ARM: add PrimeCell generic DMA to PL011
In reply to Linus Walleijdrivers/serial/amba-pl011.c static int pl011_startup(struct uart_port *port) · line 1282
Quoted 13 lines · Linus Walleij
drivers/serial/amba-pl011.c +7
-2/* -* Finally, enable interrupts +* Finally, enable interrupts, only timeouts when using DMA +* if initial RX DMA job failed, start in interrupt mode +* as well. */ spin_lock_irq (&uap->port.lock); - uap->im =UART011_RXIM |UART011_RTIM ; +if (uap->enable_dma && uap->rx_dma_running) + uap->im =UART011_RTIM ; +else + uap->im =UART011_RXIM |UART011_RTIM ;writew (uap->im, uap->port.membase +UART011_IMSC );
There's a big hole here, using DMA on the receive path.
| 3.4.4 UARTRTINTR | The receive timeout interrupt is asserted when the receive FIFO is not | empty, and no further data is received over a 32-bit period. The receive | timeout interrupt is cleared either when the FIFO becomes empty through | reading all the data (or by reading the holding register), or when a 1 | is written to the corresponding bit of the UARTICR register.
There must be unread data in the FIFO for us to get a RT interrupt. The problem is that if we have DMA enabled, then the received characters are read from the FIFO as soon as they appear, and so the FIFO becomes empty, which negates the conditions required for the RT interrupt to be generated.
Let's say I have a login prompt on the serial port. I type 'root'. The DMA reads the contents of the FIFO into memory, and causing it to be emptied. As there's no characters in the FIFO, we don't receive a RT interrupt, so we don't transfer these characters into the TTY subsystem. The only time that the input is received is after a further ~4092 newline characters are submitted, at which point the DMA engine says the buffer is complete, and calls the RX callback. This sucks for interactive use of the port, and probably prevents it being used for any kind of sane protocol too.
I don't see any way to bring back interactivity while keeping RX DMA support, so I don't think we can use DMA for the RX channel with these UARTs.
PATCH 1/1 ARM: add PrimeCell generic DMA to PL011
2010/12/22 Russell King - ARM Linux <[email protected]>:
Quoted 24 lines · Linus Walleij
Quoted 5 lines
? ? ? /*
- ? ? ?* Finally, enable interrupts
+ ? ? ?* Finally, enable interrupts, only timeouts when using DMA
+ ? ? ?* if initial RX DMA job failed, start in interrupt mode
+ ? ? ?* as well.Quoted 8 lines
? ? ? ?*/
? ? ?spin_lock_irq(&uap->port.lock);
- ? ?uap->im=UART011_RXIM|UART011_RTIM;
+ ? ? if (uap->enable_dma&&uap->rx_dma_running)
+ ? ? ? ? ? ?uap->im=UART011_RTIM;
+ ? ? else
+ ? ? ? ? ? ?uap->im=UART011_RXIM|UART011_RTIM; ? ? ?writew(uap->im, uap->port.membase + UART011_IMSC);There's a big hole here, using DMA on the receive path.
| 3.4.4 UARTRTINTR | The receive timeout interrupt is asserted when the receive FIFO is not | empty, and no further data is received over a 32-bit period. The receive | timeout interrupt is cleared either when the FIFO becomes empty through | reading all the data (or by reading the holding register), or when a 1 | is written to the corresponding bit of the UARTICR register.
Quoted 5 lines
There must be unread data in the FIFO for us to get a RT interrupt. ?The problem is that if we have DMA enabled, then the received characters are read from the FIFO as soon as they appear, and so the FIFO becomes empty, which negates the conditions required for the RT interrupt to be generated.
The DMA on the RX channel will not read single bytes at all, instead it waits until it can retrieve a burst, which is set to half a FIFO: src_maxburst = uap->fifosize >> 1,
(OK this is max burst, but of course the DMA engine will try to configure as large transfers as possible, so in practice this will be 8 bytes for an unmodified PL011.)
In the interactive case the RT interrupt is the most expected execution path. If you type in a few characters pl011_dma_rx_chars() will not recieve anything using DMA really, it will get all characters from the FIFO after a RT interrupt.
If you feed the DMA real quick (e.g. try pasting a large buffer into a terminal) it will run some DMA rounds of pages, then it will stop with some characters in the FIFO, and then it will time out, and the remaining characters are read out from the FIFO. So the characters get in for all cases I can think of.
If it ain't working there is something more fishy to it...
Quoted 10 lines
Let's say I have a login prompt on the serial port. ?I type 'root'. The DMA reads the contents of the FIFO into memory, and causing it to be emptied. ?As there's no characters in the FIFO, we don't receive a RT interrupt, so we don't transfer these characters into the TTY subsystem. The only time that the input is received is after a further ~4092 newline characters are submitted, at which point the DMA engine says the buffer is complete, and calls the RX callback. ?This sucks for interactive use of the port, and probably prevents it being used for any kind of sane protocol too.
Quoted 3 lines
I don't see any way to bring back interactivity while keeping RX DMA support, so I don't think we can use DMA for the RX channel with these UARTs.
But I'm using this interactively on ux500, u300 (pure PL011) and RealView PB11MPCore..?
There *may* be a corner case when you're transferring an even multiple of the burst size though, then you can *maybe* get into the situation you're describing. Such as paste a buffer with n MOD 8 bytes, but I haven't been able to provoke it :-/ (and no hardware here sadly).
In that case however, what we would could do is to have some last fallback timer that will do what the RT interrupt does if it doesn't eventually happen.
(Will comment more on the thread(s) soonish, just need to put the kids to sleep.)
Yours,
Linus Walleij
PATCH 1/1 ARM: add PrimeCell generic DMA to PL011
Quoted 31 lines · Linus Walleij
2010/12/22 Russell King - ARM Linux <[email protected]>:
Quoted 24 lines · Linus Walleij
Quoted 5 lines
? ? ? /*
- ? ? ?* Finally, enable interrupts
+ ? ? ?* Finally, enable interrupts, only timeouts when using DMA
+ ? ? ?* if initial RX DMA job failed, start in interrupt mode
+ ? ? ?* as well.Quoted 8 lines
? ? ? ?*/
? ? ?spin_lock_irq(&uap->port.lock);
- ? ?uap->im=UART011_RXIM|UART011_RTIM;
+ ? ? if (uap->enable_dma&&uap->rx_dma_running)
+ ? ? ? ? ? ?uap->im=UART011_RTIM;
+ ? ? else
+ ? ? ? ? ? ?uap->im=UART011_RXIM|UART011_RTIM; ? ? ?writew(uap->im, uap->port.membase + UART011_IMSC);There's a big hole here, using DMA on the receive path.
| 3.4.4 UARTRTINTR | The receive timeout interrupt is asserted when the receive FIFO is not | empty, and no further data is received over a 32-bit period. The receive | timeout interrupt is cleared either when the FIFO becomes empty through | reading all the data (or by reading the holding register), or when a 1 | is written to the corresponding bit of the UARTICR register.Quoted 5 lines
There must be unread data in the FIFO for us to get a RT interrupt. ?The problem is that if we have DMA enabled, then the received characters are read from the FIFO as soon as they appear, and so the FIFO becomes empty, which negates the conditions required for the RT interrupt to be generated.
Quoted 3 lines
The DMA on the RX channel will not read single bytes at all, instead it waits until it can retrieve a burst, which is set to half a FIFO:
src_maxburst=uap->fifosize>> 1,
I refer you to section 2.7, which describes the DMA interface.
UARTRXDMASREQ
Single character DMA transfer request, asserted by the UART. For receive, one character consists of up to 12 bits. This signal is asserted when the receive FIFO contains at least one character.
UARTRXDMABREQ
Burst DMA transfer request, asserted by the UART. This signal is asserted when the receive FIFO contains more characters than the programmed watermark level. You can program the watermark level for each FIFO through the UARTIFLS register.
If your DMA controller is using UARTRXDMASREQ, then you'll end up DMAing single bytes as long as the FIFO contains any bytes - thereby causing it to be emptied completely. As 3.4.4 says, if the FIFO is empty, you don't get the timeout.
The PB926 doesn't specify whether it connects the burst or single- character DMA request signal to the DMA controller - from the behaviour I observe, I suspect it's the single character request which is.
PATCH 1/1 ARM: add PrimeCell generic DMA to PL011
Quoted 49 lines · Russell King - ARM Linux
Quoted 31 lines · Linus Walleij
2010/12/22 Russell King - ARM Linux <[email protected]>:
Quoted 24 lines · Linus Walleij
Quoted 5 lines
? ? ? /*
- ? ? ?* Finally, enable interrupts
+ ? ? ?* Finally, enable interrupts, only timeouts when using DMA
+ ? ? ?* if initial RX DMA job failed, start in interrupt mode
+ ? ? ?* as well.Quoted 8 lines
? ? ? ?*/
? ? ?spin_lock_irq(&uap->port.lock);
- ? ?uap->im=UART011_RXIM|UART011_RTIM;
+ ? ? if (uap->enable_dma&&uap->rx_dma_running)
+ ? ? ? ? ? ?uap->im=UART011_RTIM;
+ ? ? else
+ ? ? ? ? ? ?uap->im=UART011_RXIM|UART011_RTIM; ? ? ?writew(uap->im, uap->port.membase + UART011_IMSC);There's a big hole here, using DMA on the receive path.
| 3.4.4 UARTRTINTR | The receive timeout interrupt is asserted when the receive FIFO is not | empty, and no further data is received over a 32-bit period. The receive | timeout interrupt is cleared either when the FIFO becomes empty through | reading all the data (or by reading the holding register), or when a 1 | is written to the corresponding bit of the UARTICR register.Quoted 5 lines
There must be unread data in the FIFO for us to get a RT interrupt. ?The problem is that if we have DMA enabled, then the received characters are read from the FIFO as soon as they appear, and so the FIFO becomes empty, which negates the conditions required for the RT interrupt to be generated.
Quoted 3 lines
The DMA on the RX channel will not read single bytes at all, instead it waits until it can retrieve a burst, which is set to half a FIFO:
src_maxburst=uap->fifosize>> 1,I refer you to section 2.7, which describes the DMA interface.
UARTRXDMASREQ
Single character DMA transfer request, asserted by the UART. For receive, one character consists of up to 12 bits. This signal is asserted when the receive FIFO contains at least one character.UARTRXDMABREQ
Burst DMA transfer request, asserted by the UART. This signal is asserted when the receive FIFO contains more characters than the programmed watermark level. You can program the watermark level for each FIFO through the UARTIFLS register.
Quoted 8 lines
If your DMA controller is using UARTRXDMASREQ, then you'll end up DMAing single bytes as long as the FIFO contains any bytes - thereby causing it to be emptied completely. As 3.4.4 says, if the FIFO is empty, you don't get the timeout.
The PB926 doesn't specify whether it connects the burst or single- character DMA request signal to the DMA controller - from the behaviour I observe, I suspect it's the single character request which is.
And here's what /proc/interrupts reports:
12: 38 VIC uart-pl011
Now, I send exactly one 'o' to the port:
12: 38 VIC uart-pl011
no interrupt received. If I extend PL08x's debugfs output to include a register dump (src, dest, lli, ctrl, conf):
0 uart0rx
[101f1000 019c4007 0111c011 1a0123f9 0000d01d]
now if I send three characters:
0 uart0rx
[101f1000 019c400a 0111c011 1a0123f6 0000d01d]
Note that the address has increased by three, and the count has decremented by three. This confirms that the DMA is reading the data as the UART FIFO fills - rather than DMA waiting for the FIFO to partially fill before starting a request.
This emptying of the RX UART FIFO of all received data prevents the RX timeout condition being satisfied, which in turn prevents any reasonable use of the UART port.
I think the answer here is to allow the platform to supply just the TX DMA channel, so when a UART is wired up to use the single-character request, RX DMA support can be omitted.
PATCH 1/1 ARM: add PrimeCell generic DMA to PL011
Quoted 83 lines · Russell King - ARM Linux
Quoted 49 lines · Russell King - ARM Linux
Quoted 31 lines · Linus Walleij
2010/12/22 Russell King - ARM Linux <[email protected]>:
Quoted 24 lines · Linus Walleij
Quoted 5 lines
? ? ? /*
- ? ? ?* Finally, enable interrupts
+ ? ? ?* Finally, enable interrupts, only timeouts when using DMA
+ ? ? ?* if initial RX DMA job failed, start in interrupt mode
+ ? ? ?* as well.Quoted 8 lines
? ? ? ?*/
? ? ?spin_lock_irq(&uap->port.lock);
- ? ?uap->im=UART011_RXIM|UART011_RTIM;
+ ? ? if (uap->enable_dma&&uap->rx_dma_running)
+ ? ? ? ? ? ?uap->im=UART011_RTIM;
+ ? ? else
+ ? ? ? ? ? ?uap->im=UART011_RXIM|UART011_RTIM; ? ? ?writew(uap->im, uap->port.membase + UART011_IMSC);There's a big hole here, using DMA on the receive path.
| 3.4.4 UARTRTINTR | The receive timeout interrupt is asserted when the receive FIFO is not | empty, and no further data is received over a 32-bit period. The receive | timeout interrupt is cleared either when the FIFO becomes empty through | reading all the data (or by reading the holding register), or when a 1 | is written to the corresponding bit of the UARTICR register.Quoted 5 lines
There must be unread data in the FIFO for us to get a RT interrupt. ?The problem is that if we have DMA enabled, then the received characters are read from the FIFO as soon as they appear, and so the FIFO becomes empty, which negates the conditions required for the RT interrupt to be generated.
Quoted 3 lines
The DMA on the RX channel will not read single bytes at all, instead it waits until it can retrieve a burst, which is set to half a FIFO:
src_maxburst=uap->fifosize>> 1,I refer you to section 2.7, which describes the DMA interface.
UARTRXDMASREQ
Single character DMA transfer request, asserted by the UART. For receive, one character consists of up to 12 bits. This signal is asserted when the receive FIFO contains at least one character.UARTRXDMABREQ
Burst DMA transfer request, asserted by the UART. This signal is asserted when the receive FIFO contains more characters than the programmed watermark level. You can program the watermark level for each FIFO through the UARTIFLS register.Quoted 8 lines
If your DMA controller is using UARTRXDMASREQ, then you'll end up DMAing single bytes as long as the FIFO contains any bytes - thereby causing it to be emptied completely. As 3.4.4 says, if the FIFO is empty, you don't get the timeout.
The PB926 doesn't specify whether it connects the burst or single- character DMA request signal to the DMA controller - from the behaviour I observe, I suspect it's the single character request which is.
And here's what
/proc/interruptsreports:12: 38 VIC uart-pl011
Now, I send exactly one 'o' to the port:
12: 38 VIC uart-pl011
no interrupt received. If I extend PL08x's debugfs output to include a register dump (src, dest, lli, ctrl, conf):
0 uart0rx
[101f1000 019c4007 0111c011 1a0123f9 0000d01d]now if I send three characters:
0 uart0rx
[101f1000 019c400a 0111c011 1a0123f6 0000d01d]Note that the address has increased by three, and the count has decremented by three. This confirms that the DMA is reading the data as the UART FIFO fills - rather than DMA waiting for the FIFO to partially fill before starting a request.
Quoted 7 lines
This emptying of the RX UART FIFO of all received data prevents the RX timeout condition being satisfied, which in turn prevents any reasonable use of the UART port.
I think the answer here is to allow the platform to supply just the TX DMA channel, so when a UART is wired up to use the single-character request, RX DMA support can be omitted.
One final point on this: it _will_ work if you send a corrupted character or break first - that sets an error interrupt flag, which remains latched because the ICR is never written to clear them. Could that also have something to do with why your setup works?
I'll continue hacking the RX DMA support around, but for the time being, I've committed my revised TX DMA support in several patches to my tree. That won't work on its own because the mainline amba-pl08x.c is rather functional on PB/926 - and of course it has the deadlock issue.
PATCH 1/1 ARM: add PrimeCell generic DMA to PL011
In reply to Russell King - ARM Linux2010/12/22 Russell King - ARM Linux <[email protected]>:
Quoted 7 lines
This emptying of the RX UART FIFO of all received data prevents the RX timeout condition being satisfied, which in turn prevents any reasonable use of the UART port.
I think the answer here is to allow the platform to supply just the TX DMA channel, so when a UART is wired up to use the single-character request, RX DMA support can be omitted.
Yes atleast that is fully OK until someone comes up with some clever workaround. I don't know if a software fallback timer flipping the buffers would work.
There's another reason for not doing UART RX at all: for example it will occupy one of the channels (PB11MPCores PL081 has only two!) for good, and then MMCI and AACI cannot use it at the same time.
Yours,
Linus Walleij
PATCH 1/1 ARM: add PrimeCell generic DMA to PL011
In reply to Russell King - ARM Linux2010/12/22 Russell King - ARM Linux <[email protected]>:
Quoted 3 lines
The PB926 doesn't specify whether it connects the burst or single- character DMA request signal to the DMA controller - from the behaviour I observe, I suspect it's the single character request which is.
Aha I suspect this is it... :-(
FYI I have repeatedly requested access to the PB926 board from ARM but these can't even be bought any more, even if I had the money. Thanks for testing at it!
Yours,
Linus Walleij
PATCH 1/1 ARM: add PrimeCell generic DMA to PL011
In reply to Linus WalleijQuoted 5 lines · Linus Walleij
There *may* be a corner case when you're transferring an even multiple of the burst size though, then you can *maybe* get into the situation you're describing. Such as paste a buffer with n MOD 8 bytes, but I haven't been able to provoke it :-/ (and no hardware here sadly).
This will only happen if you send all 8 bytes without triggering a RTIM interrupt, and your DMA controller reads all 8 bytes upon receiving the burst request.
If you send 4 bytes, pause, 4 bytes, and your DMA is only transferring when 8 bytes have been received then you won't see the problem (as after the first four bytes are received, you'll get a RTIM interrupt, and another RTIM after the second group of four bytes.
I'm not sure that I'd trust pasting characters into a terminal to guarantee that we receive all 8 bytes as a single back-to-back burst.
echo -n 01234567 >/dev/ttySn
where /dev/ttySn is the uart on your host PC to which your terminal program is attached. If you can't provoke it with fast baud rates, try slowing down the baud rate so the 32-bit period takes longer to expire - your DMA controller may not be reading all data from the FIFO before this period expires, thereby allowing the RTIM interrupt to occur.
Note: I've no idea how the ST Micro parts operate - you're setting the DMA threshold to 16 bytes, yet the FIFO is 64 bytes, but you're asking the DMA controller to do bursts of 32 bytes. What's the effect of IFLS on the DMA thesholds too? Does that result in the DMA BREQ signal only being activated after 32 bytes and the ST_UART011_DMAWM register being ignored? If so that could mean you need to send 32 bytes instead of 8. Or maybe it's 16 bytes?
Anyway, this is easy to work around. Let's assume a 16-byte FIFO for everything that follows.
Set a threshold of 8 characters for the DMA burst request to be triggered in the PL011 and set the DMA to transfer a burst of 4 bytes. This will guarantee that bytes are left in the PL011 FIFO, which means we have the conditions for the PL011 to generate a RTIM interrupt when there's a pause of 32 bit times in the transmission no matter what.
In your RTIM interrupt handler, you disable and read data from the DMA buffer as before. However, you also check for characters in the FIFO and empty those into the TTY buffers before re-enabling DMA.
What this means is that the DMA engine controller _absolutely_ _must_ respect the requested burst size and never transfer anything but the requested burst size per burst request. If the DMA engine controller it is unable to make this guarantee, we need some way of ensuring that the PL011 driver will never use RX DMA.
In other words:
- if you make use of the PL011's SREQ signal for the RX path, you lose.
- if you DMA 8 or more characters per BREQ for the RX path, you lose.
Lastly, I've confirmed what's going on on the PB926 - the PL080 DMA controller is definitely violating the statement in B.4.1 below figure B-3. However, it's rather iffy that this is in an appendix rather than the main text describing the behaviour of the DMA controller - given the observed behaviour I believe this paragraph has been accidentally left in the documentation. This at least explains why it doesn't work
- and can _never_ work on this platform.