Merge "spi: spi_qsd: Improve latencies for small transfers"

This commit is contained in:
Linux Build Service Account 2016-12-20 23:44:47 -08:00 committed by Gerrit - the friendly Code Review server
commit 4802720f7b
2 changed files with 133 additions and 76 deletions

View file

@ -1,4 +1,4 @@
/* Copyright (c) 2008-2015, The Linux Foundation. All rights reserved.
/* Copyright (c) 2008-2016, The Linux Foundation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 and
@ -45,6 +45,8 @@
#include <linux/msm-bus-board.h>
#include "spi_qsd.h"
#define SPI_MAX_BYTES_PER_WORD (4)
static int msm_spi_pm_resume_runtime(struct device *device);
static int msm_spi_pm_suspend_runtime(struct device *device);
static inline void msm_spi_dma_unmap_buffers(struct msm_spi *dd);
@ -438,10 +440,12 @@ static void msm_spi_read_word_from_fifo(struct msm_spi *dd)
u32 data_in;
int i;
int shift;
int read_bytes = (dd->pack_words ?
SPI_MAX_BYTES_PER_WORD : dd->bytes_per_word);
data_in = readl_relaxed(dd->base + SPI_INPUT_FIFO);
if (dd->read_buf) {
for (i = 0; (i < dd->bytes_per_word) &&
for (i = 0; (i < read_bytes) &&
dd->rx_bytes_remaining; i++) {
/* The data format depends on bytes_per_word:
4 bytes: 0x12345678
@ -454,8 +458,8 @@ static void msm_spi_read_word_from_fifo(struct msm_spi *dd)
dd->rx_bytes_remaining--;
}
} else {
if (dd->rx_bytes_remaining >= dd->bytes_per_word)
dd->rx_bytes_remaining -= dd->bytes_per_word;
if (dd->rx_bytes_remaining >= read_bytes)
dd->rx_bytes_remaining -= read_bytes;
else
dd->rx_bytes_remaining = 0;
}
@ -552,7 +556,7 @@ msm_spi_set_bpw_and_no_io_flags(struct msm_spi *dd, u32 *config, int n)
if (n != (*config & SPI_CFG_N))
*config = (*config & ~SPI_CFG_N) | n;
if (dd->mode == SPI_BAM_MODE) {
if (dd->tx_mode == SPI_BAM_MODE) {
if (dd->read_buf == NULL)
*config |= SPI_NO_INPUT;
if (dd->write_buf == NULL)
@ -617,25 +621,34 @@ static void msm_spi_set_spi_config(struct msm_spi *dd, int bpw)
static void msm_spi_set_mx_counts(struct msm_spi *dd, u32 n_words)
{
/*
* n_words cannot exceed fifo_size, and only one READ COUNT
* interrupt is generated per transaction, so for transactions
* larger than fifo size READ COUNT must be disabled.
* For those transactions we usually move to Data Mover mode.
* For FIFO mode:
* - Set the MX_OUTPUT_COUNT/MX_INPUT_COUNT registers to 0
* - Set the READ/WRITE_COUNT registers to 0 (infinite mode)
* or num bytes (finite mode) if less than fifo worth of data.
* For Block mode:
* - Set the MX_OUTPUT/MX_INPUT_COUNT registers to num xfer bytes.
* - Set the READ/WRITE_COUNT registers to 0.
*/
if (dd->mode == SPI_FIFO_MODE) {
if (n_words <= dd->input_fifo_size) {
writel_relaxed(n_words,
dd->base + SPI_MX_READ_COUNT);
msm_spi_set_write_count(dd, n_words);
} else {
writel_relaxed(0, dd->base + SPI_MX_READ_COUNT);
msm_spi_set_write_count(dd, 0);
}
if (dd->qup_ver == SPI_QUP_VERSION_BFAM) {
/* must be zero for FIFO */
writel_relaxed(0, dd->base + SPI_MX_INPUT_COUNT);
if (dd->tx_mode != SPI_BAM_MODE) {
if (dd->tx_mode == SPI_FIFO_MODE) {
if (n_words <= dd->input_fifo_size)
msm_spi_set_write_count(dd, n_words);
else
msm_spi_set_write_count(dd, 0);
writel_relaxed(0, dd->base + SPI_MX_OUTPUT_COUNT);
}
} else
writel_relaxed(n_words, dd->base + SPI_MX_OUTPUT_COUNT);
if (dd->rx_mode == SPI_FIFO_MODE) {
if (n_words <= dd->input_fifo_size)
writel_relaxed(n_words,
dd->base + SPI_MX_READ_COUNT);
else
writel_relaxed(0,
dd->base + SPI_MX_READ_COUNT);
writel_relaxed(0, dd->base + SPI_MX_INPUT_COUNT);
} else
writel_relaxed(n_words, dd->base + SPI_MX_INPUT_COUNT);
} else {
/* must be zero for BAM and DMOV */
writel_relaxed(0, dd->base + SPI_MX_READ_COUNT);
@ -882,7 +895,7 @@ xfr_err:
static int
msm_spi_bam_next_transfer(struct msm_spi *dd)
{
if (dd->mode != SPI_BAM_MODE)
if (dd->tx_mode != SPI_BAM_MODE)
return 0;
if (dd->tx_bytes_remaining > 0) {
@ -901,7 +914,7 @@ msm_spi_bam_next_transfer(struct msm_spi *dd)
static int msm_spi_dma_send_next(struct msm_spi *dd)
{
int ret = 0;
if (dd->mode == SPI_BAM_MODE)
if (dd->tx_mode == SPI_BAM_MODE)
ret = msm_spi_bam_next_transfer(dd);
return ret;
}
@ -932,32 +945,38 @@ static inline irqreturn_t msm_spi_qup_irq(int irq, void *dev_id)
}
op = readl_relaxed(dd->base + SPI_OPERATIONAL);
writel_relaxed(op, dd->base + SPI_OPERATIONAL);
/*
* Ensure service flag was cleared before further
* processing of interrupt.
*/
mb();
if (op & SPI_OP_INPUT_SERVICE_FLAG) {
writel_relaxed(SPI_OP_INPUT_SERVICE_FLAG,
dd->base + SPI_OPERATIONAL);
/*
* Ensure service flag was cleared before further
* processing of interrupt.
*/
mb();
ret |= msm_spi_input_irq(irq, dev_id);
}
if (op & SPI_OP_OUTPUT_SERVICE_FLAG) {
writel_relaxed(SPI_OP_OUTPUT_SERVICE_FLAG,
dd->base + SPI_OPERATIONAL);
/*
* Ensure service flag was cleared before further
* processing of interrupt.
*/
mb();
ret |= msm_spi_output_irq(irq, dev_id);
}
if (dd->done) {
if (dd->tx_mode != SPI_BAM_MODE) {
if (!dd->rx_done) {
if (dd->rx_bytes_remaining == 0)
dd->rx_done = true;
}
if (!dd->tx_done) {
if (!dd->tx_bytes_remaining &&
(op & SPI_OP_IP_FIFO_NOT_EMPTY)) {
dd->tx_done = true;
}
}
}
if (dd->tx_done && dd->rx_done) {
msm_spi_set_state(dd, SPI_OP_STATE_RESET);
dd->tx_done = false;
dd->rx_done = false;
complete(&dd->rx_transfer_complete);
complete(&dd->tx_transfer_complete);
dd->done = 0;
}
return ret;
}
@ -968,17 +987,23 @@ static irqreturn_t msm_spi_input_irq(int irq, void *dev_id)
dd->stat_rx++;
if (dd->mode == SPI_MODE_NONE)
if (dd->rx_mode == SPI_MODE_NONE)
return IRQ_HANDLED;
if (dd->mode == SPI_FIFO_MODE) {
if (dd->rx_mode == SPI_FIFO_MODE) {
while ((readl_relaxed(dd->base + SPI_OPERATIONAL) &
SPI_OP_IP_FIFO_NOT_EMPTY) &&
(dd->rx_bytes_remaining > 0)) {
msm_spi_read_word_from_fifo(dd);
}
if (dd->rx_bytes_remaining == 0)
msm_spi_complete(dd);
} else if (dd->rx_mode == SPI_BLOCK_MODE) {
int count = 0;
while (dd->rx_bytes_remaining &&
(count < dd->input_block_size)) {
msm_spi_read_word_from_fifo(dd);
count += SPI_MAX_BYTES_PER_WORD;
}
}
return IRQ_HANDLED;
@ -989,18 +1014,20 @@ static void msm_spi_write_word_to_fifo(struct msm_spi *dd)
u32 word;
u8 byte;
int i;
int write_bytes =
(dd->pack_words ? SPI_MAX_BYTES_PER_WORD : dd->bytes_per_word);
word = 0;
if (dd->write_buf) {
for (i = 0; (i < dd->bytes_per_word) &&
for (i = 0; (i < write_bytes) &&
dd->tx_bytes_remaining; i++) {
dd->tx_bytes_remaining--;
byte = *dd->write_buf++;
word |= (byte << (BITS_PER_BYTE * i));
}
} else
if (dd->tx_bytes_remaining > dd->bytes_per_word)
dd->tx_bytes_remaining -= dd->bytes_per_word;
if (dd->tx_bytes_remaining > write_bytes)
dd->tx_bytes_remaining -= write_bytes;
else
dd->tx_bytes_remaining = 0;
dd->write_xfr_cnt++;
@ -1012,11 +1039,22 @@ static inline void msm_spi_write_rmn_to_fifo(struct msm_spi *dd)
{
int count = 0;
while ((dd->tx_bytes_remaining > 0) && (count < dd->input_fifo_size) &&
!(readl_relaxed(dd->base + SPI_OPERATIONAL) &
SPI_OP_OUTPUT_FIFO_FULL)) {
msm_spi_write_word_to_fifo(dd);
count++;
if (dd->tx_mode == SPI_FIFO_MODE) {
while ((dd->tx_bytes_remaining > 0) &&
(count < dd->input_fifo_size) &&
!(readl_relaxed(dd->base + SPI_OPERATIONAL)
& SPI_OP_OUTPUT_FIFO_FULL)) {
msm_spi_write_word_to_fifo(dd);
count++;
}
}
if (dd->tx_mode == SPI_BLOCK_MODE) {
while (dd->tx_bytes_remaining &&
(count < dd->output_block_size)) {
msm_spi_write_word_to_fifo(dd);
count += SPI_MAX_BYTES_PER_WORD;
}
}
}
@ -1026,11 +1064,11 @@ static irqreturn_t msm_spi_output_irq(int irq, void *dev_id)
dd->stat_tx++;
if (dd->mode == SPI_MODE_NONE)
if (dd->tx_mode == SPI_MODE_NONE)
return IRQ_HANDLED;
/* Output FIFO is empty. Transmit any outstanding write data. */
if (dd->mode == SPI_FIFO_MODE)
if ((dd->tx_mode == SPI_FIFO_MODE) || (dd->tx_mode == SPI_BLOCK_MODE))
msm_spi_write_rmn_to_fifo(dd);
return IRQ_HANDLED;
@ -1106,7 +1144,7 @@ error:
static int msm_spi_dma_map_buffers(struct msm_spi *dd)
{
int ret = 0;
if (dd->mode == SPI_BAM_MODE)
if (dd->tx_mode == SPI_BAM_MODE)
ret = msm_spi_bam_map_buffers(dd);
return ret;
}
@ -1135,7 +1173,7 @@ static void msm_spi_bam_unmap_buffers(struct msm_spi *dd)
static inline void msm_spi_dma_unmap_buffers(struct msm_spi *dd)
{
if (dd->mode == SPI_BAM_MODE)
if (dd->tx_mode == SPI_BAM_MODE)
msm_spi_bam_unmap_buffers(dd);
}
@ -1197,9 +1235,11 @@ static void
msm_spi_set_transfer_mode(struct msm_spi *dd, u8 bpw, u32 read_count)
{
if (msm_spi_use_dma(dd, dd->cur_transfer, bpw)) {
dd->mode = SPI_BAM_MODE;
dd->tx_mode = SPI_BAM_MODE;
dd->rx_mode = SPI_BAM_MODE;
} else {
dd->mode = SPI_FIFO_MODE;
dd->rx_mode = SPI_FIFO_MODE;
dd->tx_mode = SPI_FIFO_MODE;
dd->read_len = dd->cur_transfer->len;
dd->write_len = dd->cur_transfer->len;
}
@ -1215,14 +1255,19 @@ static void msm_spi_set_qup_io_modes(struct msm_spi *dd)
spi_iom = readl_relaxed(dd->base + SPI_IO_MODES);
/* Set input and output transfer mode: FIFO, DMOV, or BAM */
spi_iom &= ~(SPI_IO_M_INPUT_MODE | SPI_IO_M_OUTPUT_MODE);
spi_iom = (spi_iom | (dd->mode << OUTPUT_MODE_SHIFT));
spi_iom = (spi_iom | (dd->mode << INPUT_MODE_SHIFT));
/* Turn on packing for data mover */
if (dd->mode == SPI_BAM_MODE)
spi_iom = (spi_iom | (dd->tx_mode << OUTPUT_MODE_SHIFT));
spi_iom = (spi_iom | (dd->rx_mode << INPUT_MODE_SHIFT));
/* Always enable packing for all % 8 bits_per_word */
if (dd->cur_transfer->bits_per_word &&
((dd->cur_transfer->bits_per_word == 8) ||
(dd->cur_transfer->bits_per_word == 16) ||
(dd->cur_transfer->bits_per_word == 32))) {
spi_iom |= SPI_IO_M_PACK_EN | SPI_IO_M_UNPACK_EN;
else {
dd->pack_words = true;
} else {
spi_iom &= ~(SPI_IO_M_PACK_EN | SPI_IO_M_UNPACK_EN);
spi_iom |= SPI_IO_M_OUTPUT_BIT_SHIFT_EN;
dd->pack_words = false;
}
/*if (dd->mode == SPI_BAM_MODE) {
@ -1280,7 +1325,7 @@ static void msm_spi_set_qup_op_mask(struct msm_spi *dd)
{
/* mask INPUT and OUTPUT service flags in to prevent IRQs on FIFO status
* change in BAM mode */
u32 mask = (dd->mode == SPI_BAM_MODE) ?
u32 mask = (dd->tx_mode == SPI_BAM_MODE) ?
QUP_OP_MASK_OUTPUT_SERVICE_FLAG | QUP_OP_MASK_INPUT_SERVICE_FLAG
: 0;
writel_relaxed(mask, dd->base + QUP_OPERATIONAL_MASK);
@ -1321,6 +1366,8 @@ static int msm_spi_process_transfer(struct msm_spi *dd)
dd->rx_bytes_remaining = dd->cur_msg_len;
dd->read_buf = dd->cur_transfer->rx_buf;
dd->write_buf = dd->cur_transfer->tx_buf;
dd->tx_done = false;
dd->rx_done = false;
init_completion(&dd->tx_transfer_complete);
init_completion(&dd->rx_transfer_complete);
if (dd->cur_transfer->bits_per_word)
@ -1351,10 +1398,12 @@ static int msm_spi_process_transfer(struct msm_spi *dd)
msm_spi_set_transfer_mode(dd, bpw, read_count);
msm_spi_set_mx_counts(dd, read_count);
if (dd->mode == SPI_BAM_MODE) {
if (dd->tx_mode == SPI_BAM_MODE) {
ret = msm_spi_dma_map_buffers(dd);
if (ret < 0) {
pr_err("Mapping DMA buffers\n");
dd->tx_mode = SPI_MODE_NONE;
dd->rx_mode = SPI_MODE_NONE;
return ret;
}
}
@ -1368,11 +1417,11 @@ static int msm_spi_process_transfer(struct msm_spi *dd)
the first. Restricting this to one write avoids contention
issues and race conditions between this thread and the int handler
*/
if (dd->mode == SPI_FIFO_MODE) {
if (dd->tx_mode != SPI_BAM_MODE) {
if (msm_spi_prepare_for_write(dd))
goto transfer_end;
msm_spi_start_write(dd, read_count);
} else if (dd->mode == SPI_BAM_MODE) {
} else {
if ((msm_spi_bam_begin_transfer(dd)) < 0) {
dev_err(dd->dev, "%s: BAM transfer setup failed\n",
__func__);
@ -1388,11 +1437,11 @@ static int msm_spi_process_transfer(struct msm_spi *dd)
* might fire before the first word is written resulting in a
* possible race condition.
*/
if (dd->mode != SPI_BAM_MODE)
if (dd->tx_mode != SPI_BAM_MODE)
if (msm_spi_set_state(dd, SPI_OP_STATE_RUN)) {
dev_warn(dd->dev,
"%s: Failed to set QUP to run-state. Mode:%d",
__func__, dd->mode);
__func__, dd->tx_mode);
goto transfer_end;
}
@ -1422,10 +1471,11 @@ static int msm_spi_process_transfer(struct msm_spi *dd)
msm_spi_udelay(dd->xfrs_delay_usec);
transfer_end:
if ((dd->mode == SPI_BAM_MODE) && status)
if ((dd->tx_mode == SPI_BAM_MODE) && status)
msm_spi_bam_flush(dd);
msm_spi_dma_unmap_buffers(dd);
dd->mode = SPI_MODE_NONE;
dd->tx_mode = SPI_MODE_NONE;
dd->rx_mode = SPI_MODE_NONE;
msm_spi_set_state(dd, SPI_OP_STATE_RESET);
if (!dd->cur_transfer->cs_change)
@ -2350,7 +2400,8 @@ static int init_resources(struct platform_device *pdev)
pclk_enabled = 0;
dd->transfer_pending = 0;
dd->mode = SPI_MODE_NONE;
dd->tx_mode = SPI_MODE_NONE;
dd->rx_mode = SPI_MODE_NONE;
rc = msm_spi_request_irq(dd, pdev, master);
if (rc)

View file

@ -1,4 +1,4 @@
/* Copyright (c) 2014-2015, The Linux Foundation. All rights reserved.
/* Copyright (c) 2014-2016, The Linux Foundation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 and
@ -113,6 +113,8 @@
#define INPUT_MODE_SHIFT QSD_REG(10) QUP_REG(12)
/* SPI_OPERATIONAL fields */
#define SPI_OP_IN_BLK_RD_REQ_FLAG 0x00002000
#define SPI_OP_OUT_BLK_WR_REQ_FLAG 0x00001000
#define SPI_OP_MAX_INPUT_DONE_FLAG 0x00000800
#define SPI_OP_MAX_OUTPUT_DONE_FLAG 0x00000400
#define SPI_OP_INPUT_SERVICE_FLAG 0x00000200
@ -318,7 +320,8 @@ struct msm_spi {
bool transfer_pending;
wait_queue_head_t continue_suspend;
/* DMA data */
enum msm_spi_mode mode;
enum msm_spi_mode tx_mode;
enum msm_spi_mode rx_mode;
bool use_dma;
int tx_dma_chan;
int tx_dma_crci;
@ -349,7 +352,8 @@ struct msm_spi {
#endif
struct msm_spi_platform_data *pdata; /* Platform data */
/* When set indicates multiple transfers in a single message */
bool done;
bool rx_done;
bool tx_done;
u32 cur_msg_len;
/* Used in FIFO mode to keep track of the transfer being processed */
struct spi_transfer *cur_tx_transfer;
@ -367,6 +371,7 @@ struct msm_spi {
struct pinctrl_state *pins_active;
struct pinctrl_state *pins_sleep;
bool is_init_complete;
bool pack_words;
};
/* Forward declaration */
@ -523,7 +528,8 @@ static inline void msm_spi_set_write_count(struct msm_spi *dd, int val)
static inline void msm_spi_complete(struct msm_spi *dd)
{
dd->done = 1;
dd->tx_done = true;
dd->rx_done = true;
}
static inline void msm_spi_enable_error_flags(struct msm_spi *dd)