target/linux/ipq806x/patches/0179-spi-qup-Add-DMA-capabilities.patch

   1 From 8322bafdcee1d7eaf15540ff013415bff1eacb28 Mon Sep 17 00:00:00 2001
   2 From: Andy Gross <agross@codeaurora.org>
   3 Date: Thu, 26 Jun 2014 10:50:24 -0500
   4 Subject: [PATCH 179/182] spi: qup: Add DMA capabilities
   5
   6 This patch adds DMA capabilities to the spi-qup driver.  If DMA channels are
   7 present, the QUP will use DMA instead of block mode for transfers to/from SPI
   8 peripherals for transactions larger greater than the length of a block.
   9
  10 Signed-off-by: Andy Gross <agross@codeaurora.org>
  11 ---
  12  drivers/spi/spi-qup.c |  361 ++++++++++++++++++++++++++++++++++++++++++++++---
  13  1 file changed, 340 insertions(+), 21 deletions(-)
  14
  15 --- a/drivers/spi/spi-qup.c
  16 +++ b/drivers/spi/spi-qup.c
  17 @@ -22,6 +22,8 @@
  18  #include <linux/platform_device.h>
  19  #include <linux/pm_runtime.h>
  20  #include <linux/spi/spi.h>
  21 +#include <linux/dmaengine.h>
  22 +#include <linux/dma-mapping.h>
  23
  24  #define QUP_CONFIG                     0x0000
  25  #define QUP_STATE                      0x0004
  26 @@ -116,6 +118,8 @@
  27
  28  #define SPI_NUM_CHIPSELECTS            4
  29
  30 +#define SPI_MAX_XFER                   (SZ_64K - 64)
  31 +
  32  /* high speed mode is when bus rate is greater then 26MHz */
  33  #define SPI_HS_MIN_RATE                        26000000
  34  #define SPI_MAX_RATE                   50000000
  35 @@ -143,6 +147,14 @@ struct spi_qup {
  36         int                     tx_bytes;
  37         int                     rx_bytes;
  38         int                     qup_v1;
  39 +       int                     use_dma;
  40 +
  41 +       struct dma_chan         *rx_chan;
  42 +       struct dma_slave_config rx_conf;
  43 +       struct dma_chan         *tx_chan;
  44 +       struct dma_slave_config tx_conf;
  45 +       void *dummy;
  46 +       atomic_t                dma_outstanding;
  47  };
  48
  49
  50 @@ -266,6 +278,221 @@ static void spi_qup_fifo_write(struct sp
  51         }
  52  }
  53
  54 +static void qup_dma_callback(void *data)
  55 +{
  56 +       struct spi_qup *controller = data;
  57 +
  58 +       if (atomic_dec_and_test(&controller->dma_outstanding))
  59 +               complete(&controller->done);
  60 +}
  61 +
  62 +static int spi_qup_do_dma(struct spi_qup *controller, struct spi_transfer *xfer)
  63 +{
  64 +       struct dma_async_tx_descriptor *rxd, *txd;
  65 +       dma_cookie_t rx_cookie, tx_cookie;
  66 +       u32 xfer_len, rx_align = 0, tx_align = 0, n_words;
  67 +       struct scatterlist tx_sg[2], rx_sg[2];
  68 +       int ret = 0;
  69 +       u32 bytes_to_xfer = xfer->len;
  70 +       u32 offset = 0;
  71 +       u32 rx_nents = 0, tx_nents = 0;
  72 +       dma_addr_t rx_dma = 0, tx_dma = 0, rx_dummy_dma = 0, tx_dummy_dma = 0;
  73 +
  74 +
  75 +       if (xfer->rx_buf) {
  76 +               rx_dma = dma_map_single(controller->dev, xfer->rx_buf,
  77 +                       xfer->len, DMA_FROM_DEVICE);
  78 +
  79 +               if (dma_mapping_error(controller->dev, rx_dma)) {
  80 +                       ret = -ENOMEM;
  81 +                       return ret;
  82 +               }
  83 +
  84 +               /* check to see if we need dummy buffer for leftover bytes */
  85 +               rx_align = xfer->len % controller->in_blk_sz;
  86 +               if (rx_align) {
  87 +                       rx_dummy_dma = dma_map_single(controller->dev,
  88 +                               controller->dummy, controller->in_fifo_sz,
  89 +                               DMA_FROM_DEVICE);
  90 +
  91 +                       if (dma_mapping_error(controller->dev, rx_dummy_dma)) {
  92 +                               ret = -ENOMEM;
  93 +                               goto err_map_rx_dummy;
  94 +                       }
  95 +               }
  96 +       }
  97 +
  98 +       if (xfer->tx_buf) {
  99 +               tx_dma = dma_map_single(controller->dev,
 100 +                       (void *)xfer->tx_buf, xfer->len, DMA_TO_DEVICE);
 101 +
 102 +               if (dma_mapping_error(controller->dev, tx_dma)) {
 103 +                       ret = -ENOMEM;
 104 +                       goto err_map_tx;
 105 +               }
 106 +
 107 +               /* check to see if we need dummy buffer for leftover bytes */
 108 +               tx_align = xfer->len % controller->out_blk_sz;
 109 +               if (tx_align) {
 110 +                       memcpy(controller->dummy + SZ_1K,
 111 +                               xfer->tx_buf + xfer->len - tx_align,
 112 +                               tx_align);
 113 +                       memset(controller->dummy + SZ_1K + tx_align, 0,
 114 +                               controller->out_blk_sz - tx_align);
 115 +
 116 +                       tx_dummy_dma = dma_map_single(controller->dev,
 117 +                               controller->dummy + SZ_1K,
 118 +                               controller->out_blk_sz, DMA_TO_DEVICE);
 119 +
 120 +                       if (dma_mapping_error(controller->dev, tx_dummy_dma)) {
 121 +                               ret = -ENOMEM;
 122 +                               goto err_map_tx_dummy;
 123 +                       }
 124 +               }
 125 +       }
 126 +
 127 +       atomic_set(&controller->dma_outstanding, 0);
 128 +
 129 +       while (bytes_to_xfer > 0) {
 130 +               xfer_len = min_t(u32, bytes_to_xfer, SPI_MAX_XFER);
 131 +               n_words = DIV_ROUND_UP(xfer_len, controller->w_size);
 132 +
 133 +               /* write out current word count to controller */
 134 +               writel_relaxed(n_words, controller->base + QUP_MX_INPUT_CNT);
 135 +               writel_relaxed(n_words, controller->base + QUP_MX_OUTPUT_CNT);
 136 +
 137 +               reinit_completion(&controller->done);
 138 +
 139 +               if (xfer->tx_buf) {
 140 +                       /* recalc align for each transaction */
 141 +                       tx_align = xfer_len % controller->out_blk_sz;
 142 +
 143 +                       if (tx_align)
 144 +                               tx_nents = 2;
 145 +                       else
 146 +                               tx_nents = 1;
 147 +
 148 +                       /* initialize scatterlists */
 149 +                       sg_init_table(tx_sg, tx_nents);
 150 +                       sg_dma_len(&tx_sg[0]) = xfer_len - tx_align;
 151 +                       sg_dma_address(&tx_sg[0]) = tx_dma + offset;
 152 +
 153 +                       /* account for non block size transfer */
 154 +                       if (tx_align) {
 155 +                               sg_dma_len(&tx_sg[1]) = controller->out_blk_sz;
 156 +                               sg_dma_address(&tx_sg[1]) = tx_dummy_dma;
 157 +                       }
 158 +
 159 +                       txd = dmaengine_prep_slave_sg(controller->tx_chan,
 160 +                                       tx_sg, tx_nents, DMA_MEM_TO_DEV, 0);
 161 +                       if (!txd) {
 162 +                               ret = -ENOMEM;
 163 +                               goto err_unmap;
 164 +                       }
 165 +
 166 +                       atomic_inc(&controller->dma_outstanding);
 167 +
 168 +                       txd->callback = qup_dma_callback;
 169 +                       txd->callback_param = controller;
 170 +
 171 +                       tx_cookie = dmaengine_submit(txd);
 172 +
 173 +                       dma_async_issue_pending(controller->tx_chan);
 174 +               }
 175 +
 176 +               if (xfer->rx_buf) {
 177 +                       /* recalc align for each transaction */
 178 +                       rx_align = xfer_len % controller->in_blk_sz;
 179 +
 180 +                       if (rx_align)
 181 +                               rx_nents = 2;
 182 +                       else
 183 +                               rx_nents = 1;
 184 +
 185 +                       /* initialize scatterlists */
 186 +                       sg_init_table(rx_sg, rx_nents);
 187 +                       sg_dma_address(&rx_sg[0]) = rx_dma + offset;
 188 +                       sg_dma_len(&rx_sg[0]) = xfer_len - rx_align;
 189 +
 190 +                       /* account for non block size transfer */
 191 +                       if (rx_align) {
 192 +                               sg_dma_len(&rx_sg[1]) = controller->in_blk_sz;
 193 +                               sg_dma_address(&rx_sg[1]) = rx_dummy_dma;
 194 +                       }
 195 +
 196 +                       rxd = dmaengine_prep_slave_sg(controller->rx_chan,
 197 +                                       rx_sg, rx_nents, DMA_DEV_TO_MEM, 0);
 198 +                       if (!rxd) {
 199 +                               ret = -ENOMEM;
 200 +                               goto err_unmap;
 201 +                       }
 202 +
 203 +                       atomic_inc(&controller->dma_outstanding);
 204 +
 205 +                       rxd->callback = qup_dma_callback;
 206 +                       rxd->callback_param = controller;
 207 +
 208 +                       rx_cookie = dmaengine_submit(rxd);
 209 +
 210 +                       dma_async_issue_pending(controller->rx_chan);
 211 +               }
 212 +
 213 +               if (spi_qup_set_state(controller, QUP_STATE_RUN)) {
 214 +                       dev_warn(controller->dev, "cannot set EXECUTE state\n");
 215 +                       goto err_unmap;
 216 +               }
 217 +
 218 +               if (!wait_for_completion_timeout(&controller->done,
 219 +                       msecs_to_jiffies(1000))) {
 220 +                       ret = -ETIMEDOUT;
 221 +
 222 +                       /* clear out all the DMA transactions */
 223 +                       if (xfer->tx_buf)
 224 +                               dmaengine_terminate_all(controller->tx_chan);
 225 +                       if (xfer->rx_buf)
 226 +                               dmaengine_terminate_all(controller->rx_chan);
 227 +
 228 +                       goto err_unmap;
 229 +               }
 230 +
 231 +               if (rx_align)
 232 +                       memcpy(xfer->rx_buf + offset + xfer->len - rx_align,
 233 +                               controller->dummy, rx_align);
 234 +
 235 +               /* adjust remaining bytes to transfer */
 236 +               bytes_to_xfer -= xfer_len;
 237 +               offset += xfer_len;
 238 +
 239 +
 240 +               /* reset mini-core state so we can program next transaction */
 241 +               if (spi_qup_set_state(controller, QUP_STATE_RESET)) {
 242 +                       dev_err(controller->dev, "cannot set RESET state\n");
 243 +                       goto err_unmap;
 244 +               }
 245 +       }
 246 +
 247 +       ret = 0;
 248 +
 249 +err_unmap:
 250 +       if (tx_align)
 251 +               dma_unmap_single(controller->dev, tx_dummy_dma,
 252 +                       controller->out_fifo_sz, DMA_TO_DEVICE);
 253 +err_map_tx_dummy:
 254 +       if (xfer->tx_buf)
 255 +               dma_unmap_single(controller->dev, tx_dma, xfer->len,
 256 +                       DMA_TO_DEVICE);
 257 +err_map_tx:
 258 +       if (rx_align)
 259 +               dma_unmap_single(controller->dev, rx_dummy_dma,
 260 +                       controller->in_fifo_sz, DMA_FROM_DEVICE);
 261 +err_map_rx_dummy:
 262 +       if (xfer->rx_buf)
 263 +               dma_unmap_single(controller->dev, rx_dma, xfer->len,
 264 +                       DMA_FROM_DEVICE);
 265 +
 266 +       return ret;
 267 +}
 268 +
 269  static irqreturn_t spi_qup_qup_irq(int irq, void *dev_id)
 270  {
 271         struct spi_qup *controller = dev_id;
 272 @@ -315,11 +542,13 @@ static irqreturn_t spi_qup_qup_irq(int i
 273                 error = -EIO;
 274         }
 275
 276 -       if (opflags & QUP_OP_IN_SERVICE_FLAG)
 277 -               spi_qup_fifo_read(controller, xfer);
 278 +       if (!controller->use_dma) {
 279 +               if (opflags & QUP_OP_IN_SERVICE_FLAG)
 280 +                       spi_qup_fifo_read(controller, xfer);
 281
 282 -       if (opflags & QUP_OP_OUT_SERVICE_FLAG)
 283 -               spi_qup_fifo_write(controller, xfer);
 284 +               if (opflags & QUP_OP_OUT_SERVICE_FLAG)
 285 +                       spi_qup_fifo_write(controller, xfer);
 286 +       }
 287
 288         spin_lock_irqsave(&controller->lock, flags);
 289         controller->error = error;
 290 @@ -339,6 +568,8 @@ static int spi_qup_io_config(struct spi_
 291         struct spi_qup *controller = spi_master_get_devdata(spi->master);
 292         u32 config, iomode, mode;
 293         int ret, n_words, w_size;
 294 +       size_t dma_align = dma_get_cache_alignment();
 295 +       u32 dma_available = 0;
 296
 297         if (spi->mode & SPI_LOOP && xfer->len > controller->in_fifo_sz) {
 298                 dev_err(controller->dev, "too big size for loopback %d > %d\n",
 299 @@ -367,6 +598,13 @@ static int spi_qup_io_config(struct spi_
 300         n_words = xfer->len / w_size;
 301         controller->w_size = w_size;
 302
 303 +       if (controller->rx_chan &&
 304 +               IS_ALIGNED((size_t)xfer->tx_buf, dma_align) &&
 305 +               IS_ALIGNED((size_t)xfer->rx_buf, dma_align) &&
 306 +               !is_vmalloc_addr(xfer->tx_buf) &&
 307 +               !is_vmalloc_addr(xfer->rx_buf))
 308 +               dma_available = 1;
 309 +
 310         if (n_words <= (controller->in_fifo_sz / sizeof(u32))) {
 311                 mode = QUP_IO_M_MODE_FIFO;
 312                 writel_relaxed(n_words, controller->base + QUP_MX_READ_CNT);
 313 @@ -374,19 +612,30 @@ static int spi_qup_io_config(struct spi_
 314                 /* must be zero for FIFO */
 315                 writel_relaxed(0, controller->base + QUP_MX_INPUT_CNT);
 316                 writel_relaxed(0, controller->base + QUP_MX_OUTPUT_CNT);
 317 -       } else {
 318 +               controller->use_dma = 0;
 319 +       } else if (!dma_available) {
 320                 mode = QUP_IO_M_MODE_BLOCK;
 321                 writel_relaxed(n_words, controller->base + QUP_MX_INPUT_CNT);
 322                 writel_relaxed(n_words, controller->base + QUP_MX_OUTPUT_CNT);
 323                 /* must be zero for BLOCK and BAM */
 324                 writel_relaxed(0, controller->base + QUP_MX_READ_CNT);
 325                 writel_relaxed(0, controller->base + QUP_MX_WRITE_CNT);
 326 +               controller->use_dma = 0;
 327 +       } else {
 328 +               mode = QUP_IO_M_MODE_DMOV;
 329 +               writel_relaxed(0, controller->base + QUP_MX_READ_CNT);
 330 +               writel_relaxed(0, controller->base + QUP_MX_WRITE_CNT);
 331 +               controller->use_dma = 1;
 332         }
 333
 334         iomode = readl_relaxed(controller->base + QUP_IO_M_MODES);
 335         /* Set input and output transfer mode */
 336         iomode &= ~(QUP_IO_M_INPUT_MODE_MASK | QUP_IO_M_OUTPUT_MODE_MASK);
 337 -       iomode &= ~(QUP_IO_M_PACK_EN | QUP_IO_M_UNPACK_EN);
 338 +       if (!controller->use_dma)
 339 +               iomode &= ~(QUP_IO_M_PACK_EN | QUP_IO_M_UNPACK_EN);
 340 +       else
 341 +               iomode |= QUP_IO_M_PACK_EN | QUP_IO_M_UNPACK_EN;
 342 +
 343         iomode |= (mode << QUP_IO_M_OUTPUT_MODE_MASK_SHIFT);
 344         iomode |= (mode << QUP_IO_M_INPUT_MODE_MASK_SHIFT);
 345
 346 @@ -419,11 +668,20 @@ static int spi_qup_io_config(struct spi_
 347         config &= ~(QUP_CONFIG_NO_INPUT | QUP_CONFIG_NO_OUTPUT | QUP_CONFIG_N);
 348         config |= xfer->bits_per_word - 1;
 349         config |= QUP_CONFIG_SPI_MODE;
 350 +
 351 +       if (controller->use_dma) {
 352 +               if (!xfer->tx_buf)
 353 +                       config |= QUP_CONFIG_NO_OUTPUT;
 354 +               if (!xfer->rx_buf)
 355 +                       config |= QUP_CONFIG_NO_INPUT;
 356 +       }
 357 +
 358         writel_relaxed(config, controller->base + QUP_CONFIG);
 359
 360         /* only write to OPERATIONAL_MASK when register is present */
 361         if (!controller->qup_v1)
 362                 writel_relaxed(0, controller->base + QUP_OPERATIONAL_MASK);
 363 +
 364         return 0;
 365  }
 366
 367 @@ -452,26 +710,32 @@ static int spi_qup_transfer_one(struct s
 368         controller->tx_bytes = 0;
 369         spin_unlock_irqrestore(&controller->lock, flags);
 370
 371 -       if (spi_qup_set_state(controller, QUP_STATE_RUN)) {
 372 -               dev_warn(controller->dev, "cannot set RUN state\n");
 373 -               goto exit;
 374 -       }
 375 +       if (controller->use_dma) {
 376 +               ret = spi_qup_do_dma(controller, xfer);
 377 +       } else {
 378 +               if (spi_qup_set_state(controller, QUP_STATE_RUN)) {
 379 +                       dev_warn(controller->dev, "cannot set RUN state\n");
 380 +                       goto exit;
 381 +               }
 382
 383 -       if (spi_qup_set_state(controller, QUP_STATE_PAUSE)) {
 384 -               dev_warn(controller->dev, "cannot set PAUSE state\n");
 385 -               goto exit;
 386 -       }
 387 +               if (spi_qup_set_state(controller, QUP_STATE_PAUSE)) {
 388 +                       dev_warn(controller->dev, "cannot set PAUSE state\n");
 389 +                       goto exit;
 390 +               }
 391
 392 -       spi_qup_fifo_write(controller, xfer);
 393 +               spi_qup_fifo_write(controller, xfer);
 394
 395 -       if (spi_qup_set_state(controller, QUP_STATE_RUN)) {
 396 -               dev_warn(controller->dev, "cannot set EXECUTE state\n");
 397 -               goto exit;
 398 -       }
 399 +               if (spi_qup_set_state(controller, QUP_STATE_RUN)) {
 400 +                       dev_warn(controller->dev, "cannot set EXECUTE state\n");
 401 +                       goto exit;
 402 +               }
 403
 404 -       if (!wait_for_completion_timeout(&controller->done, timeout))
 405 -               ret = -ETIMEDOUT;
 406 +               if (!ret && !wait_for_completion_timeout(&controller->done,
 407 +                               timeout))
 408 +                       ret = -ETIMEDOUT;
 409 +       }
 410  exit:
 411 +
 412         spi_qup_set_state(controller, QUP_STATE_RESET);
 413         spin_lock_irqsave(&controller->lock, flags);
 414         controller->xfer = NULL;
 415 @@ -553,6 +817,7 @@ static int spi_qup_probe(struct platform
 416         master->transfer_one = spi_qup_transfer_one;
 417         master->dev.of_node = pdev->dev.of_node;
 418         master->auto_runtime_pm = true;
 419 +       master->dma_alignment = dma_get_cache_alignment();
 420
 421         platform_set_drvdata(pdev, master);
 422
 423 @@ -612,6 +877,55 @@ static int spi_qup_probe(struct platform
 424         writel_relaxed(SPI_ERROR_CLK_UNDER_RUN | SPI_ERROR_CLK_OVER_RUN,
 425                        base + SPI_ERROR_FLAGS_EN);
 426
 427 +       /* allocate dma resources, if available */
 428 +       controller->rx_chan = dma_request_slave_channel(&pdev->dev, "rx");
 429 +       if (controller->rx_chan) {
 430 +               controller->tx_chan =
 431 +                       dma_request_slave_channel(&pdev->dev, "tx");
 432 +
 433 +               if (!controller->tx_chan) {
 434 +                       dev_err(&pdev->dev, "Failed to allocate dma tx chan");
 435 +                       dma_release_channel(controller->rx_chan);
 436 +               }
 437 +
 438 +               /* set DMA parameters */
 439 +               controller->rx_conf.device_fc = 1;
 440 +               controller->rx_conf.src_addr = res->start + QUP_INPUT_FIFO;
 441 +               controller->rx_conf.src_maxburst = controller->in_blk_sz;
 442 +
 443 +               controller->tx_conf.device_fc = 1;
 444 +               controller->tx_conf.dst_addr = res->start + QUP_OUTPUT_FIFO;
 445 +               controller->tx_conf.dst_maxburst = controller->out_blk_sz;
 446 +
 447 +               if (dmaengine_slave_config(controller->rx_chan,
 448 +                               &controller->rx_conf)) {
 449 +                       dev_err(&pdev->dev, "failed to configure RX channel\n");
 450 +
 451 +                       dma_release_channel(controller->rx_chan);
 452 +                       dma_release_channel(controller->tx_chan);
 453 +                       controller->tx_chan = NULL;
 454 +                       controller->rx_chan = NULL;
 455 +               } else if (dmaengine_slave_config(controller->tx_chan,
 456 +                               &controller->tx_conf)) {
 457 +                       dev_err(&pdev->dev, "failed to configure TX channel\n");
 458 +
 459 +                       dma_release_channel(controller->rx_chan);
 460 +                       dma_release_channel(controller->tx_chan);
 461 +                       controller->tx_chan = NULL;
 462 +                       controller->rx_chan = NULL;
 463 +               }
 464 +
 465 +               controller->dummy = devm_kmalloc(controller->dev, PAGE_SIZE,
 466 +                       GFP_KERNEL);
 467 +
 468 +               if (!controller->dummy) {
 469 +                       dma_release_channel(controller->rx_chan);
 470 +                       dma_release_channel(controller->tx_chan);
 471 +                       controller->tx_chan = NULL;
 472 +                       controller->rx_chan = NULL;
 473 +               }
 474 +       }
 475 +
 476         /* if earlier version of the QUP, disable INPUT_OVERRUN */
 477         if (controller->qup_v1)
 478                 writel_relaxed(QUP_ERROR_OUTPUT_OVER_RUN |
 479 @@ -730,6 +1044,11 @@ static int spi_qup_remove(struct platfor
 480         if (ret)
 481                 return ret;
 482
 483 +       if (controller->rx_chan)
 484 +               dma_release_channel(controller->rx_chan);
 485 +       if (controller->tx_chan)
 486 +               dma_release_channel(controller->tx_chan);
 487 +
 488         clk_disable_unprepare(controller->cclk);
 489         clk_disable_unprepare(controller->iclk);
 490