target/linux/brcm2708/patches-4.4/0149-bcm2835-sdhost-Major-revision.patch

   1 From 5a117bd925d13a305d94eeb28919dedeaa9be17d Mon Sep 17 00:00:00 2001
   2 From: Phil Elwell <phil@raspberrypi.org>
   3 Date: Thu, 11 Feb 2016 16:51:01 +0000
   4 Subject: [PATCH 149/170] bcm2835-sdhost: Major revision
   5
   6 This is a significant revision of the bcm2835-sdhost driver. It
   7 improves on the original in a number of ways:
   8
   9 1) Through the use of CMD23 for reads it appears to avoid problems
  10    reading some sectors on certain high speed cards.
  11 2) Better atomicity to prevent crashes.
  12 3) Higher performance.
  13 4) Activity logging included, for easier diagnosis in the event
  14    of a problem.
  15
  16 Signed-off-by: Phil Elwell <phil@raspberrypi.org>
  17 ---
  18  drivers/mmc/host/bcm2835-sdhost.c | 1284 ++++++++++++++++++++-----------------
  19  1 file changed, 686 insertions(+), 598 deletions(-)
  20
  21 --- a/drivers/mmc/host/bcm2835-sdhost.c
  22 +++ b/drivers/mmc/host/bcm2835-sdhost.c
  23 @@ -2,7 +2,7 @@
  24   * BCM2835 SD host driver.
  25   *
  26   * Author:      Phil Elwell <phil@raspberrypi.org>
  27 - *              Copyright 2015
  28 + *              Copyright (C) 2015-2016 Raspberry Pi (Trading) Ltd.
  29   *
  30   * Based on
  31   *  mmc-bcm2835.c by Gellert Weisz
  32 @@ -24,12 +24,13 @@
  33   * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  34   */
  35
  36 -#define SAFE_READ_THRESHOLD     4
  37 -#define SAFE_WRITE_THRESHOLD    4
  38 -#define ALLOW_DMA               1
  39 -#define ALLOW_CMD23             0
  40 -#define ALLOW_FAST              1
  41 -#define USE_BLOCK_IRQ           1
  42 +#define FIFO_READ_THRESHOLD     4
  43 +#define FIFO_WRITE_THRESHOLD    4
  44 +#define ALLOW_CMD23_READ        1
  45 +#define ALLOW_CMD23_WRITE       0
  46 +#define ENABLE_LOG              1
  47 +#define SDDATA_FIFO_PIO_BURST   8
  48 +#define CMD_DALLY_US            1
  49
  50  #include <linux/delay.h>
  51  #include <linux/module.h>
  52 @@ -48,6 +49,7 @@
  53  #include <linux/dma-mapping.h>
  54  #include <linux/of_dma.h>
  55  #include <linux/time.h>
  56 +#include <linux/workqueue.h>
  57
  58  #define DRIVER_NAME "sdhost-bcm2835"
  59
  60 @@ -110,6 +112,28 @@
  61  #define SDEDM_READ_THRESHOLD_SHIFT 14
  62  #define SDEDM_THRESHOLD_MASK     0x1f
  63
  64 +#define SDEDM_FSM_MASK           0xf
  65 +#define SDEDM_FSM_IDENTMODE      0x0
  66 +#define SDEDM_FSM_DATAMODE       0x1
  67 +#define SDEDM_FSM_READDATA       0x2
  68 +#define SDEDM_FSM_WRITEDATA      0x3
  69 +#define SDEDM_FSM_READWAIT       0x4
  70 +#define SDEDM_FSM_READCRC        0x5
  71 +#define SDEDM_FSM_WRITECRC       0x6
  72 +#define SDEDM_FSM_WRITEWAIT1     0x7
  73 +#define SDEDM_FSM_POWERDOWN      0x8
  74 +#define SDEDM_FSM_POWERUP        0x9
  75 +#define SDEDM_FSM_WRITESTART1    0xa
  76 +#define SDEDM_FSM_WRITESTART2    0xb
  77 +#define SDEDM_FSM_GENPULSES      0xc
  78 +#define SDEDM_FSM_WRITEWAIT2     0xd
  79 +#define SDEDM_FSM_STARTPOWDOWN   0xf
  80 +
  81 +#define SDDATA_FIFO_WORDS        16
  82 +
  83 +#define USE_CMD23_FLAGS          ((ALLOW_CMD23_READ * MMC_DATA_READ) | \
  84 +                                 (ALLOW_CMD23_WRITE * MMC_DATA_WRITE))
  85 +
  86  #define MHZ 1000000
  87
  88
  89 @@ -131,15 +155,17 @@ struct bcm2835_host {
  90
  91         struct tasklet_struct   finish_tasklet; /* Tasklet structures */
  92
  93 -       struct timer_list       timer;          /* Timer for timeouts */
  94 +       struct work_struct      cmd_wait_wq;    /* Workqueue function */
  95
  96 -       struct timer_list       pio_timer;      /* PIO error detection timer */
  97 +       struct timer_list       timer;          /* Timer for timeouts */
  98
  99         struct sg_mapping_iter  sg_miter;       /* SG state for PIO */
 100         unsigned int            blocks;         /* remaining PIO blocks */
 101
 102         int                     irq;            /* Device IRQ */
 103
 104 +       u32                     cmd_quick_poll_retries;
 105 +       u32                     ns_per_fifo_word;
 106
 107         /* cached registers */
 108         u32                     hcfg;
 109 @@ -154,16 +180,21 @@ struct bcm2835_host {
 110
 111         unsigned int                    use_busy:1;             /* Wait for busy interrupt */
 112
 113 -       unsigned int                    debug:1;                /* Enable debug output */
 114 +       unsigned int                    use_sbc:1;              /* Send CMD23 */
 115
 116 -       u32                             thread_isr;
 117 +       unsigned int                    debug:1;                /* Enable debug output */
 118
 119         /*DMA part*/
 120         struct dma_chan                 *dma_chan_rx;           /* DMA channel for reads */
 121         struct dma_chan                 *dma_chan_tx;           /* DMA channel for writes */
 122 +       struct dma_chan                 *dma_chan;              /* Channel in used */
 123 +       struct dma_async_tx_descriptor  *dma_desc;
 124 +       u32                             dma_dir;
 125 +       u32                             drain_words;
 126 +       struct page                     *drain_page;
 127 +       u32                             drain_offset;
 128
 129         bool                            allow_dma;
 130 -       bool                            have_dma;
 131         bool                            use_dma;
 132         /*end of DMA part*/
 133
 134 @@ -173,13 +204,98 @@ struct bcm2835_host {
 135         u32                             overclock_50;   /* frequency to use when 50MHz is requested (in MHz) */
 136         u32                             overclock;      /* Current frequency if overclocked, else zero */
 137         u32                             pio_limit;      /* Maximum block count for PIO (0 = always DMA) */
 138 +};
 139
 140 -       u32                             debug_flags;
 141 +#if ENABLE_LOG
 142
 143 -       u32                             sectors;        /* Cached card size in sectors */
 144 -       u32                             single_read_sectors[8];
 145 +struct log_entry_struct {
 146 +       char event[4];
 147 +       u32 timestamp;
 148 +       u32 param1;
 149 +       u32 param2;
 150  };
 151
 152 +typedef struct log_entry_struct LOG_ENTRY_T;
 153 +
 154 +LOG_ENTRY_T *sdhost_log_buf;
 155 +dma_addr_t sdhost_log_addr;
 156 +static u32 sdhost_log_idx;
 157 +static spinlock_t log_lock;
 158 +static void __iomem *timer_base;
 159 +
 160 +#define LOG_ENTRIES (256*1)
 161 +#define LOG_SIZE (sizeof(LOG_ENTRY_T)*LOG_ENTRIES)
 162 +
 163 +static void log_init(u32 bus_to_phys)
 164 +{
 165 +       spin_lock_init(&log_lock);
 166 +       sdhost_log_buf = dma_zalloc_coherent(NULL, LOG_SIZE, &sdhost_log_addr,
 167 +                                            GFP_KERNEL);
 168 +       if (sdhost_log_buf) {
 169 +               pr_err("sdhost: log_buf @ %p (%x)\n",
 170 +                      sdhost_log_buf, sdhost_log_addr);
 171 +               timer_base = ioremap_nocache(bus_to_phys + 0x7e003000, SZ_4K);
 172 +               if (!timer_base)
 173 +                       pr_err("sdhost: failed to remap timer\n");
 174 +       }
 175 +       else
 176 +               pr_err("sdhost: failed to allocate log buf\n");
 177 +}
 178 +
 179 +static void log_event_impl(const char *event, u32 param1, u32 param2)
 180 +{
 181 +       if (sdhost_log_buf) {
 182 +               LOG_ENTRY_T *entry;
 183 +               unsigned long flags;
 184 +
 185 +               spin_lock_irqsave(&log_lock, flags);
 186 +
 187 +               entry = sdhost_log_buf + sdhost_log_idx;
 188 +               memcpy(entry->event, event, 4);
 189 +               entry->timestamp = (readl(timer_base + 4) & 0x3fffffff) +
 190 +                       (smp_processor_id()<<30);
 191 +               entry->param1 = param1;
 192 +               entry->param2 = param2;
 193 +               sdhost_log_idx = (sdhost_log_idx + 1) % LOG_ENTRIES;
 194 +
 195 +               spin_unlock_irqrestore(&log_lock, flags);
 196 +       }
 197 +}
 198 +
 199 +static void log_dump(void)
 200 +{
 201 +       if (sdhost_log_buf) {
 202 +               LOG_ENTRY_T *entry;
 203 +               unsigned long flags;
 204 +               int idx;
 205 +
 206 +               spin_lock_irqsave(&log_lock, flags);
 207 +
 208 +               idx = sdhost_log_idx;
 209 +               do {
 210 +                       entry = sdhost_log_buf + idx;
 211 +                       if (entry->event[0] != '\0')
 212 +                               pr_err("[%08x] %.4s %x %x\n",
 213 +                                      entry->timestamp,
 214 +                                      entry->event,
 215 +                                      entry->param1,
 216 +                                      entry->param2);
 217 +                       idx = (idx + 1) % LOG_ENTRIES;
 218 +               } while (idx != sdhost_log_idx);
 219 +
 220 +               spin_unlock_irqrestore(&log_lock, flags);
 221 +       }
 222 +}
 223 +
 224 +#define log_event(event, param1, param2) log_event_impl(event, param1, param2)
 225 +
 226 +#else
 227 +
 228 +#define log_init(x) (void)0
 229 +#define log_event(event, param1, param2) (void)0
 230 +#define log_dump() (void)0
 231 +
 232 +#endif
 233
 234  static inline void bcm2835_sdhost_write(struct bcm2835_host *host, u32 val, int reg)
 235  {
 236 @@ -201,7 +317,7 @@ static void bcm2835_sdhost_dumpcmd(struc
 237                                    const char *label)
 238  {
 239         if (cmd)
 240 -               pr_info("%s:%c%s op %d arg 0x%x flags 0x%x - resp %08x %08x %08x %08x, err %d\n",
 241 +               pr_err("%s:%c%s op %d arg 0x%x flags 0x%x - resp %08x %08x %08x %08x, err %d\n",
 242                         mmc_hostname(host->mmc),
 243                         (cmd == host->cmd) ? '>' : ' ',
 244                         label, cmd->opcode, cmd->arg, cmd->flags,
 245 @@ -211,73 +327,74 @@ static void bcm2835_sdhost_dumpcmd(struc
 246
 247  static void bcm2835_sdhost_dumpregs(struct bcm2835_host *host)
 248  {
 249 -       bcm2835_sdhost_dumpcmd(host, host->mrq->sbc, "sbc");
 250 -       bcm2835_sdhost_dumpcmd(host, host->mrq->cmd, "cmd");
 251 -       if (host->mrq->data)
 252 -               pr_err("%s: data blocks %x blksz %x - err %d\n",
 253 -                      mmc_hostname(host->mmc),
 254 -                      host->mrq->data->blocks,
 255 -                      host->mrq->data->blksz,
 256 -                      host->mrq->data->error);
 257 -       bcm2835_sdhost_dumpcmd(host, host->mrq->stop, "stop");
 258 +       if (host->mrq)
 259 +       {
 260 +               bcm2835_sdhost_dumpcmd(host, host->mrq->sbc, "sbc");
 261 +               bcm2835_sdhost_dumpcmd(host, host->mrq->cmd, "cmd");
 262 +               if (host->mrq->data)
 263 +                       pr_err("%s: data blocks %x blksz %x - err %d\n",
 264 +                              mmc_hostname(host->mmc),
 265 +                              host->mrq->data->blocks,
 266 +                              host->mrq->data->blksz,
 267 +                              host->mrq->data->error);
 268 +               bcm2835_sdhost_dumpcmd(host, host->mrq->stop, "stop");
 269 +       }
 270
 271 -       pr_info("%s: =========== REGISTER DUMP ===========\n",
 272 +       pr_err("%s: =========== REGISTER DUMP ===========\n",
 273                 mmc_hostname(host->mmc));
 274
 275 -       pr_info("%s: SDCMD  0x%08x\n",
 276 +       pr_err("%s: SDCMD  0x%08x\n",
 277                 mmc_hostname(host->mmc),
 278                 bcm2835_sdhost_read(host, SDCMD));
 279 -       pr_info("%s: SDARG  0x%08x\n",
 280 +       pr_err("%s: SDARG  0x%08x\n",
 281                 mmc_hostname(host->mmc),
 282                 bcm2835_sdhost_read(host, SDARG));
 283 -       pr_info("%s: SDTOUT 0x%08x\n",
 284 +       pr_err("%s: SDTOUT 0x%08x\n",
 285                 mmc_hostname(host->mmc),
 286                 bcm2835_sdhost_read(host, SDTOUT));
 287 -       pr_info("%s: SDCDIV 0x%08x\n",
 288 +       pr_err("%s: SDCDIV 0x%08x\n",
 289                 mmc_hostname(host->mmc),
 290                 bcm2835_sdhost_read(host, SDCDIV));
 291 -       pr_info("%s: SDRSP0 0x%08x\n",
 292 +       pr_err("%s: SDRSP0 0x%08x\n",
 293                 mmc_hostname(host->mmc),
 294                 bcm2835_sdhost_read(host, SDRSP0));
 295 -       pr_info("%s: SDRSP1 0x%08x\n",
 296 +       pr_err("%s: SDRSP1 0x%08x\n",
 297                 mmc_hostname(host->mmc),
 298                 bcm2835_sdhost_read(host, SDRSP1));
 299 -       pr_info("%s: SDRSP2 0x%08x\n",
 300 +       pr_err("%s: SDRSP2 0x%08x\n",
 301                 mmc_hostname(host->mmc),
 302                 bcm2835_sdhost_read(host, SDRSP2));
 303 -       pr_info("%s: SDRSP3 0x%08x\n",
 304 +       pr_err("%s: SDRSP3 0x%08x\n",
 305                 mmc_hostname(host->mmc),
 306                 bcm2835_sdhost_read(host, SDRSP3));
 307 -       pr_info("%s: SDHSTS 0x%08x\n",
 308 +       pr_err("%s: SDHSTS 0x%08x\n",
 309                 mmc_hostname(host->mmc),
 310                 bcm2835_sdhost_read(host, SDHSTS));
 311 -       pr_info("%s: SDVDD  0x%08x\n",
 312 +       pr_err("%s: SDVDD  0x%08x\n",
 313                 mmc_hostname(host->mmc),
 314                 bcm2835_sdhost_read(host, SDVDD));
 315 -       pr_info("%s: SDEDM  0x%08x\n",
 316 +       pr_err("%s: SDEDM  0x%08x\n",
 317                 mmc_hostname(host->mmc),
 318                 bcm2835_sdhost_read(host, SDEDM));
 319 -       pr_info("%s: SDHCFG 0x%08x\n",
 320 +       pr_err("%s: SDHCFG 0x%08x\n",
 321                 mmc_hostname(host->mmc),
 322                 bcm2835_sdhost_read(host, SDHCFG));
 323 -       pr_info("%s: SDHBCT 0x%08x\n",
 324 +       pr_err("%s: SDHBCT 0x%08x\n",
 325                 mmc_hostname(host->mmc),
 326                 bcm2835_sdhost_read(host, SDHBCT));
 327 -       pr_info("%s: SDHBLC 0x%08x\n",
 328 +       pr_err("%s: SDHBLC 0x%08x\n",
 329                 mmc_hostname(host->mmc),
 330                 bcm2835_sdhost_read(host, SDHBLC));
 331
 332 -       pr_info("%s: ===========================================\n",
 333 +       pr_err("%s: ===========================================\n",
 334                 mmc_hostname(host->mmc));
 335  }
 336
 337 -
 338  static void bcm2835_sdhost_set_power(struct bcm2835_host *host, bool on)
 339  {
 340         bcm2835_sdhost_write(host, on ? 1 : 0, SDVDD);
 341  }
 342
 343 -
 344  static void bcm2835_sdhost_reset_internal(struct bcm2835_host *host)
 345  {
 346         u32 temp;
 347 @@ -300,26 +417,24 @@ static void bcm2835_sdhost_reset_interna
 348         temp = bcm2835_sdhost_read(host, SDEDM);
 349         temp &= ~((SDEDM_THRESHOLD_MASK<<SDEDM_READ_THRESHOLD_SHIFT) |
 350                   (SDEDM_THRESHOLD_MASK<<SDEDM_WRITE_THRESHOLD_SHIFT));
 351 -       temp |= (SAFE_READ_THRESHOLD << SDEDM_READ_THRESHOLD_SHIFT) |
 352 -               (SAFE_WRITE_THRESHOLD << SDEDM_WRITE_THRESHOLD_SHIFT);
 353 +       temp |= (FIFO_READ_THRESHOLD << SDEDM_READ_THRESHOLD_SHIFT) |
 354 +               (FIFO_WRITE_THRESHOLD << SDEDM_WRITE_THRESHOLD_SHIFT);
 355         bcm2835_sdhost_write(host, temp, SDEDM);
 356         mdelay(10);
 357         bcm2835_sdhost_set_power(host, true);
 358         mdelay(10);
 359         host->clock = 0;
 360 -       host->sectors = 0;
 361 -       host->single_read_sectors[0] = ~0;
 362         bcm2835_sdhost_write(host, host->hcfg, SDHCFG);
 363         bcm2835_sdhost_write(host, host->cdiv, SDCDIV);
 364         mmiowb();
 365  }
 366
 367 -
 368  static void bcm2835_sdhost_reset(struct mmc_host *mmc)
 369  {
 370         struct bcm2835_host *host = mmc_priv(mmc);
 371         unsigned long flags;
 372         spin_lock_irqsave(&host->lock, flags);
 373 +       log_event("RST<", 0, 0);
 374
 375         bcm2835_sdhost_reset_internal(host);
 376
 377 @@ -344,82 +459,48 @@ static void bcm2835_sdhost_init(struct b
 378         }
 379  }
 380
 381 -static bool bcm2835_sdhost_is_write_complete(struct bcm2835_host *host)
 382 +static void bcm2835_sdhost_wait_transfer_complete(struct bcm2835_host *host)
 383  {
 384 -       bool write_complete = ((bcm2835_sdhost_read(host, SDEDM) & 0xf) == 1);
 385 +       int timediff;
 386 +       u32 alternate_idle;
 387 +       u32 edm;
 388
 389 -       if (!write_complete) {
 390 -               /* Request an IRQ for the last block */
 391 -               host->hcfg |= SDHCFG_BLOCK_IRPT_EN;
 392 -               bcm2835_sdhost_write(host, host->hcfg, SDHCFG);
 393 -               if ((bcm2835_sdhost_read(host, SDEDM) & 0xf) == 1) {
 394 -                       /* The write has now completed. Disable the interrupt
 395 -                          and clear the status flag */
 396 -                       host->hcfg &= ~SDHCFG_BLOCK_IRPT_EN;
 397 -                       bcm2835_sdhost_write(host, host->hcfg, SDHCFG);
 398 -                       bcm2835_sdhost_write(host, SDHSTS_BLOCK_IRPT, SDHSTS);
 399 -                       write_complete = true;
 400 -               }
 401 -       }
 402 +       alternate_idle = (host->mrq->data->flags & MMC_DATA_READ) ?
 403 +               SDEDM_FSM_READWAIT : SDEDM_FSM_WRITESTART1;
 404
 405 -       return write_complete;
 406 -}
 407 +       edm = bcm2835_sdhost_read(host, SDEDM);
 408
 409 -static void bcm2835_sdhost_wait_write_complete(struct bcm2835_host *host)
 410 -{
 411 -       int timediff;
 412 -#ifdef DEBUG
 413 -       static struct timeval start_time;
 414 -       static int max_stall_time = 0;
 415 -       static int total_stall_time = 0;
 416 -       struct timeval before, after;
 417 -
 418 -       do_gettimeofday(&before);
 419 -       if (max_stall_time == 0)
 420 -               start_time = before;
 421 -#endif
 422 +       log_event("WTC<", edm, 0);
 423
 424         timediff = 0;
 425
 426         while (1) {
 427 -               u32 edm = bcm2835_sdhost_read(host, SDEDM);
 428 -               if ((edm & 0xf) == 1)
 429 +               u32 fsm = edm & SDEDM_FSM_MASK;
 430 +               if ((fsm == SDEDM_FSM_IDENTMODE) ||
 431 +                   (fsm == SDEDM_FSM_DATAMODE))
 432                         break;
 433 -               timediff++;
 434 -               if (timediff > 5000000) {
 435 -#ifdef DEBUG
 436 -                       do_gettimeofday(&after);
 437 -                       timediff = (after.tv_sec - before.tv_sec)*1000000 +
 438 -                               (after.tv_usec - before.tv_usec);
 439 +               if (fsm == alternate_idle) {
 440 +                       bcm2835_sdhost_write(host,
 441 +                                            edm | SDEDM_FORCE_DATA_MODE,
 442 +                                            SDEDM);
 443 +                       break;
 444 +               }
 445
 446 -                       pr_err(" wait_write_complete - still waiting after %dus\n",
 447 -                              timediff);
 448 -#else
 449 -                       pr_err(" wait_write_complete - still waiting after %d retries\n",
 450 +               timediff++;
 451 +               if (timediff == 100000) {
 452 +                       pr_err("%s: wait_transfer_complete - still waiting after %d retries\n",
 453 +                              mmc_hostname(host->mmc),
 454                                timediff);
 455 -#endif
 456 +                       log_dump();
 457                         bcm2835_sdhost_dumpregs(host);
 458 -                       host->data->error = -ETIMEDOUT;
 459 +                       host->mrq->data->error = -ETIMEDOUT;
 460 +                       log_event("WTC!", edm, 0);
 461                         return;
 462                 }
 463 +               cpu_relax();
 464 +               edm = bcm2835_sdhost_read(host, SDEDM);
 465         }
 466 -
 467 -#ifdef DEBUG
 468 -       do_gettimeofday(&after);
 469 -       timediff = (after.tv_sec - before.tv_sec)*1000000 + (after.tv_usec - before.tv_usec);
 470 -
 471 -       total_stall_time += timediff;
 472 -       if (timediff > max_stall_time)
 473 -               max_stall_time = timediff;
 474 -
 475 -       if ((after.tv_sec - start_time.tv_sec) > 10) {
 476 -               pr_debug(" wait_write_complete - max wait %dus, total %dus\n",
 477 -                        max_stall_time, total_stall_time);
 478 -               start_time = after;
 479 -               max_stall_time = 0;
 480 -               total_stall_time = 0;
 481 -       }
 482 -#endif
 483 +       log_event("WTC>", edm, 0);
 484  }
 485
 486  static void bcm2835_sdhost_finish_data(struct bcm2835_host *host);
 487 @@ -427,65 +508,44 @@ static void bcm2835_sdhost_finish_data(s
 488  static void bcm2835_sdhost_dma_complete(void *param)
 489  {
 490         struct bcm2835_host *host = param;
 491 -       struct dma_chan *dma_chan;
 492 +       struct mmc_data *data = host->data;
 493         unsigned long flags;
 494 -       u32 dir_data;
 495
 496         spin_lock_irqsave(&host->lock, flags);
 497 +       log_event("DMA<", (u32)host->data, bcm2835_sdhost_read(host, SDHSTS));
 498 +       log_event("DMA ", bcm2835_sdhost_read(host, SDCMD),
 499 +                 bcm2835_sdhost_read(host, SDEDM));
 500
 501 -       if (host->data) {
 502 -               bool write_complete;
 503 -               if (USE_BLOCK_IRQ)
 504 -                       write_complete = bcm2835_sdhost_is_write_complete(host);
 505 -               else {
 506 -                       bcm2835_sdhost_wait_write_complete(host);
 507 -                       write_complete = true;
 508 -               }
 509 -               pr_debug("dma_complete() - write_complete=%d\n",
 510 -                        write_complete);
 511 -
 512 -               if (write_complete || (host->data->flags & MMC_DATA_READ))
 513 -               {
 514 -                       if (write_complete) {
 515 -                               dma_chan = host->dma_chan_tx;
 516 -                               dir_data = DMA_TO_DEVICE;
 517 -                       } else {
 518 -                               dma_chan = host->dma_chan_rx;
 519 -                               dir_data = DMA_FROM_DEVICE;
 520 -                       }
 521 -
 522 -                       dma_unmap_sg(dma_chan->device->dev,
 523 -                                    host->data->sg, host->data->sg_len,
 524 -                                    dir_data);
 525 +       if (host->dma_chan) {
 526 +               dma_unmap_sg(host->dma_chan->device->dev,
 527 +                            data->sg, data->sg_len,
 528 +                            host->dma_dir);
 529
 530 -                       bcm2835_sdhost_finish_data(host);
 531 -               }
 532 +               host->dma_chan = NULL;
 533         }
 534
 535 -       spin_unlock_irqrestore(&host->lock, flags);
 536 -}
 537 +       if (host->drain_words) {
 538 +               void *page;
 539 +               u32 *buf;
 540
 541 -static bool data_transfer_wait(struct bcm2835_host *host)
 542 -{
 543 -       unsigned long timeout = 1000000;
 544 -       while (timeout)
 545 -       {
 546 -               u32 sdhsts = bcm2835_sdhost_read(host, SDHSTS);
 547 -               if (sdhsts & SDHSTS_DATA_FLAG) {
 548 -                       bcm2835_sdhost_write(host, SDHSTS_DATA_FLAG, SDHSTS);
 549 -                       break;
 550 +               page = kmap_atomic(host->drain_page);
 551 +               buf = page + host->drain_offset;
 552 +
 553 +               while (host->drain_words) {
 554 +                       u32 edm = bcm2835_sdhost_read(host, SDEDM);
 555 +                       if ((edm >> 4) & 0x1f)
 556 +                               *(buf++) = bcm2835_sdhost_read(host,
 557 +                                                              SDDATA);
 558 +                       host->drain_words--;
 559                 }
 560 -               timeout--;
 561 -       }
 562 -       if (timeout == 0) {
 563 -           pr_err("%s: Data %s timeout\n",
 564 -                  mmc_hostname(host->mmc),
 565 -                  (host->data->flags & MMC_DATA_READ) ? "read" : "write");
 566 -           bcm2835_sdhost_dumpregs(host);
 567 -           host->data->error = -ETIMEDOUT;
 568 -           return false;
 569 +
 570 +               kunmap_atomic(page);
 571         }
 572 -       return true;
 573 +
 574 +       bcm2835_sdhost_finish_data(host);
 575 +
 576 +       log_event("DMA>", (u32)host->data, 0);
 577 +       spin_unlock_irqrestore(&host->lock, flags);
 578  }
 579
 580  static void bcm2835_sdhost_read_block_pio(struct bcm2835_host *host)
 581 @@ -493,32 +553,83 @@ static void bcm2835_sdhost_read_block_pi
 582         unsigned long flags;
 583         size_t blksize, len;
 584         u32 *buf;
 585 +       unsigned long wait_max;
 586
 587         blksize = host->data->blksz;
 588
 589 +       wait_max = jiffies + msecs_to_jiffies(host->pio_timeout);
 590 +
 591         local_irq_save(flags);
 592
 593         while (blksize) {
 594 -               if (!sg_miter_next(&host->sg_miter))
 595 -                       BUG();
 596 +               int copy_words;
 597 +               u32 hsts = 0;
 598 +
 599 +               if (!sg_miter_next(&host->sg_miter)) {
 600 +                       host->data->error = -EINVAL;
 601 +                       break;
 602 +               }
 603
 604                 len = min(host->sg_miter.length, blksize);
 605 -               BUG_ON(len % 4);
 606 +               if (len % 4) {
 607 +                       host->data->error = -EINVAL;
 608 +                       break;
 609 +               }
 610
 611                 blksize -= len;
 612                 host->sg_miter.consumed = len;
 613
 614                 buf = (u32 *)host->sg_miter.addr;
 615
 616 -               while (len) {
 617 -                       if (!data_transfer_wait(host))
 618 -                               break;
 619 +               copy_words = len/4;
 620 +
 621 +               while (copy_words) {
 622 +                       int burst_words, words;
 623 +                       u32 edm;
 624 +
 625 +                       burst_words = SDDATA_FIFO_PIO_BURST;
 626 +                       if (burst_words > copy_words)
 627 +                               burst_words = copy_words;
 628 +                       edm = bcm2835_sdhost_read(host, SDEDM);
 629 +                       words = ((edm >> 4) & 0x1f);
 630 +
 631 +                       if (words < burst_words) {
 632 +                               int fsm_state = (edm & SDEDM_FSM_MASK);
 633 +                               if ((fsm_state != SDEDM_FSM_READDATA) &&
 634 +                                   (fsm_state != SDEDM_FSM_READWAIT) &&
 635 +                                   (fsm_state != SDEDM_FSM_READCRC)) {
 636 +                                       hsts = bcm2835_sdhost_read(host,
 637 +                                                                  SDHSTS);
 638 +                                       pr_err("%s: fsm %x, hsts %x\n",
 639 +                                              mmc_hostname(host->mmc),
 640 +                                              fsm_state, hsts);
 641 +                                       if (hsts & SDHSTS_ERROR_MASK)
 642 +                                               break;
 643 +                               }
 644 +
 645 +                               if (time_after(jiffies, wait_max)) {
 646 +                                       pr_err("%s: PIO read timeout - EDM %x\n",
 647 +                                              mmc_hostname(host->mmc),
 648 +                                              edm);
 649 +                                       hsts = SDHSTS_REW_TIME_OUT;
 650 +                                       break;
 651 +                               }
 652 +                               ndelay((burst_words - words) *
 653 +                                      host->ns_per_fifo_word);
 654 +                               continue;
 655 +                       } else if (words > copy_words) {
 656 +                               words = copy_words;
 657 +                       }
 658 +
 659 +                       copy_words -= words;
 660
 661 -                       *(buf++) = bcm2835_sdhost_read(host, SDDATA);
 662 -                       len -= 4;
 663 +                       while (words) {
 664 +                               *(buf++) = bcm2835_sdhost_read(host, SDDATA);
 665 +                               words--;
 666 +                       }
 667                 }
 668
 669 -               if (host->data->error)
 670 +               if (hsts & SDHSTS_ERROR_MASK)
 671                         break;
 672         }
 673
 674 @@ -532,32 +643,83 @@ static void bcm2835_sdhost_write_block_p
 675         unsigned long flags;
 676         size_t blksize, len;
 677         u32 *buf;
 678 +       unsigned long wait_max;
 679
 680         blksize = host->data->blksz;
 681
 682 +       wait_max = jiffies + msecs_to_jiffies(host->pio_timeout);
 683 +
 684         local_irq_save(flags);
 685
 686         while (blksize) {
 687 -               if (!sg_miter_next(&host->sg_miter))
 688 -                       BUG();
 689 +               int copy_words;
 690 +               u32 hsts = 0;
 691 +
 692 +               if (!sg_miter_next(&host->sg_miter)) {
 693 +                       host->data->error = -EINVAL;
 694 +                       break;
 695 +               }
 696
 697                 len = min(host->sg_miter.length, blksize);
 698 -               BUG_ON(len % 4);
 699 +               if (len % 4) {
 700 +                       host->data->error = -EINVAL;
 701 +                       break;
 702 +               }
 703
 704                 blksize -= len;
 705                 host->sg_miter.consumed = len;
 706
 707 -               buf = host->sg_miter.addr;
 708 +               buf = (u32 *)host->sg_miter.addr;
 709
 710 -               while (len) {
 711 -                       if (!data_transfer_wait(host))
 712 -                               break;
 713 +               copy_words = len/4;
 714 +
 715 +               while (copy_words) {
 716 +                       int burst_words, words;
 717 +                       u32 edm;
 718 +
 719 +                       burst_words = SDDATA_FIFO_PIO_BURST;
 720 +                       if (burst_words > copy_words)
 721 +                               burst_words = copy_words;
 722 +                       edm = bcm2835_sdhost_read(host, SDEDM);
 723 +                       words = SDDATA_FIFO_WORDS - ((edm >> 4) & 0x1f);
 724 +
 725 +                       if (words < burst_words) {
 726 +                               int fsm_state = (edm & SDEDM_FSM_MASK);
 727 +                               if ((fsm_state != SDEDM_FSM_WRITEDATA) &&
 728 +                                   (fsm_state != SDEDM_FSM_WRITESTART1) &&
 729 +                                   (fsm_state != SDEDM_FSM_WRITESTART2)) {
 730 +                                       hsts = bcm2835_sdhost_read(host,
 731 +                                                                  SDHSTS);
 732 +                                       pr_err("%s: fsm %x, hsts %x\n",
 733 +                                              mmc_hostname(host->mmc),
 734 +                                              fsm_state, hsts);
 735 +                                       if (hsts & SDHSTS_ERROR_MASK)
 736 +                                               break;
 737 +                               }
 738
 739 -                       bcm2835_sdhost_write(host, *(buf++), SDDATA);
 740 -                       len -= 4;
 741 +                               if (time_after(jiffies, wait_max)) {
 742 +                                       pr_err("%s: PIO write timeout - EDM %x\n",
 743 +                                              mmc_hostname(host->mmc),
 744 +                                              edm);
 745 +                                       hsts = SDHSTS_REW_TIME_OUT;
 746 +                                       break;
 747 +                               }
 748 +                               ndelay((burst_words - words) *
 749 +                                      host->ns_per_fifo_word);
 750 +                               continue;
 751 +                       } else if (words > copy_words) {
 752 +                               words = copy_words;
 753 +                       }
 754 +
 755 +                       copy_words -= words;
 756 +
 757 +                       while (words) {
 758 +                               bcm2835_sdhost_write(host, *(buf++), SDDATA);
 759 +                               words--;
 760 +                       }
 761                 }
 762
 763 -               if (host->data->error)
 764 +               if (hsts & SDHSTS_ERROR_MASK)
 765                         break;
 766         }
 767
 768 @@ -566,12 +728,12 @@ static void bcm2835_sdhost_write_block_p
 769         local_irq_restore(flags);
 770  }
 771
 772 -
 773  static void bcm2835_sdhost_transfer_pio(struct bcm2835_host *host)
 774  {
 775         u32 sdhsts;
 776         bool is_read;
 777         BUG_ON(!host->data);
 778 +       log_event("XFP<", (u32)host->data, host->blocks);
 779
 780         is_read = (host->data->flags & MMC_DATA_READ) != 0;
 781         if (is_read)
 782 @@ -595,28 +757,21 @@ static void bcm2835_sdhost_transfer_pio(
 783                        is_read ? "read" : "write",
 784                        sdhsts);
 785                 host->data->error = -ETIMEDOUT;
 786 -       } else if (!is_read && !host->data->error) {
 787 -               /* Start a timer in case a transfer error occurs because
 788 -                  there is no error interrupt */
 789 -               mod_timer(&host->pio_timer, jiffies + host->pio_timeout);
 790         }
 791 +       log_event("XFP>", (u32)host->data, host->blocks);
 792  }
 793
 794 -
 795 -static void bcm2835_sdhost_transfer_dma(struct bcm2835_host *host)
 796 +static void bcm2835_sdhost_prepare_dma(struct bcm2835_host *host,
 797 +       struct mmc_data *data)
 798  {
 799 -       u32 len, dir_data, dir_slave;
 800 +       int len, dir_data, dir_slave;
 801         struct dma_async_tx_descriptor *desc = NULL;
 802         struct dma_chan *dma_chan;
 803
 804 -       pr_debug("bcm2835_sdhost_transfer_dma()\n");
 805 -
 806 -       WARN_ON(!host->data);
 807 +       log_event("PRD<", (u32)data, 0);
 808 +       pr_debug("bcm2835_sdhost_prepare_dma()\n");
 809
 810 -       if (!host->data)
 811 -               return;
 812 -
 813 -       if (host->data->flags & MMC_DATA_READ) {
 814 +       if (data->flags & MMC_DATA_READ) {
 815                 dma_chan = host->dma_chan_rx;
 816                 dir_data = DMA_FROM_DEVICE;
 817                 dir_slave = DMA_DEV_TO_MEM;
 818 @@ -625,35 +780,71 @@ static void bcm2835_sdhost_transfer_dma(
 819                 dir_data = DMA_TO_DEVICE;
 820                 dir_slave = DMA_MEM_TO_DEV;
 821         }
 822 +       log_event("PRD1", (u32)dma_chan, 0);
 823
 824         BUG_ON(!dma_chan->device);
 825         BUG_ON(!dma_chan->device->dev);
 826 -       BUG_ON(!host->data->sg);
 827 +       BUG_ON(!data->sg);
 828
 829 -       len = dma_map_sg(dma_chan->device->dev, host->data->sg,
 830 -                        host->data->sg_len, dir_data);
 831 -       if (len > 0) {
 832 -               desc = dmaengine_prep_slave_sg(dma_chan, host->data->sg,
 833 +       /* The block doesn't manage the FIFO DREQs properly for multi-block
 834 +          transfers, so don't attempt to DMA the final few words.
 835 +          Unfortunately this requires the final sg entry to be trimmed.
 836 +          N.B. This code demands that the overspill is contained in
 837 +          a single sg entry.
 838 +       */
 839 +
 840 +       host->drain_words = 0;
 841 +       if ((data->blocks > 1) && (dir_data == DMA_FROM_DEVICE)) {
 842 +               struct scatterlist *sg;
 843 +               u32 len;
 844 +               int i;
 845 +
 846 +               len = min((u32)(FIFO_READ_THRESHOLD - 1) * 4,
 847 +                         (u32)data->blocks * data->blksz);
 848 +
 849 +               for_each_sg(data->sg, sg, data->sg_len, i) {
 850 +                       if (sg_is_last(sg)) {
 851 +                               BUG_ON(sg->length < len);
 852 +                               sg->length -= len;
 853 +                               host->drain_page = (struct page *)sg->page_link;
 854 +                               host->drain_offset = sg->offset + sg->length;
 855 +                       }
 856 +               }
 857 +               host->drain_words = len/4;
 858 +       }
 859 +
 860 +       len = dma_map_sg(dma_chan->device->dev, data->sg, data->sg_len,
 861 +                        dir_data);
 862 +
 863 +       log_event("PRD2", len, 0);
 864 +       if (len > 0)
 865 +               desc = dmaengine_prep_slave_sg(dma_chan, data->sg,
 866                                                len, dir_slave,
 867                                                DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 868 -       } else {
 869 -               dev_err(mmc_dev(host->mmc), "dma_map_sg returned zero length\n");
 870 -       }
 871 +       log_event("PRD3", (u32)desc, 0);
 872 +
 873         if (desc) {
 874                 desc->callback = bcm2835_sdhost_dma_complete;
 875                 desc->callback_param = host;
 876 -               dmaengine_submit(desc);
 877 -               dma_async_issue_pending(dma_chan);
 878 +               host->dma_desc = desc;
 879 +               host->dma_chan = dma_chan;
 880 +               host->dma_dir = dir_data;
 881         }
 882 -
 883 +       log_event("PDM>", (u32)data, 0);
 884  }
 885
 886 +static void bcm2835_sdhost_start_dma(struct bcm2835_host *host)
 887 +{
 888 +       log_event("SDMA", (u32)host->data, (u32)host->dma_chan);
 889 +       dmaengine_submit(host->dma_desc);
 890 +       dma_async_issue_pending(host->dma_chan);
 891 +}
 892
 893  static void bcm2835_sdhost_set_transfer_irqs(struct bcm2835_host *host)
 894  {
 895         u32 all_irqs = SDHCFG_DATA_IRPT_EN | SDHCFG_BLOCK_IRPT_EN |
 896                 SDHCFG_BUSY_IRPT_EN;
 897 -       if (host->use_dma)
 898 +       if (host->dma_desc)
 899                 host->hcfg = (host->hcfg & ~all_irqs) |
 900                         SDHCFG_BUSY_IRPT_EN;
 901         else
 902 @@ -664,13 +855,13 @@ static void bcm2835_sdhost_set_transfer_
 903         bcm2835_sdhost_write(host, host->hcfg, SDHCFG);
 904  }
 905
 906 -
 907  static void bcm2835_sdhost_prepare_data(struct bcm2835_host *host, struct mmc_command *cmd)
 908  {
 909         struct mmc_data *data = cmd->data;
 910
 911         WARN_ON(host->data);
 912
 913 +       host->data = data;
 914         if (!data)
 915                 return;
 916
 917 @@ -679,46 +870,19 @@ static void bcm2835_sdhost_prepare_data(
 918         BUG_ON(data->blksz > host->mmc->max_blk_size);
 919         BUG_ON(data->blocks > 65535);
 920
 921 -       host->data = data;
 922         host->data_complete = 0;
 923         host->flush_fifo = 0;
 924         host->data->bytes_xfered = 0;
 925
 926 -       if (!host->sectors && host->mmc->card && !(host->debug_flags & 1))
 927 -       {
 928 -               struct mmc_card *card = host->mmc->card;
 929 -               if (!mmc_card_sd(card) && mmc_card_blockaddr(card)) {
 930 -                       /*
 931 -                        * The EXT_CSD sector count is in number of 512 byte
 932 -                        * sectors.
 933 -                        */
 934 -                       host->sectors = card->ext_csd.sectors;
 935 -                       pr_err("%s: using ext_csd!\n", mmc_hostname(host->mmc));
 936 -               } else {
 937 -                       /*
 938 -                        * The CSD capacity field is in units of read_blkbits.
 939 -                        * set_capacity takes units of 512 bytes.
 940 -                        */
 941 -                       host->sectors = card->csd.capacity <<
 942 -                               (card->csd.read_blkbits - 9);
 943 -               }
 944 -               host->single_read_sectors[0] = host->sectors - 65;
 945 -               host->single_read_sectors[1] = host->sectors - 64;
 946 -               host->single_read_sectors[2] = host->sectors - 33;
 947 -               host->single_read_sectors[3] = host->sectors - 32;
 948 -               host->single_read_sectors[4] = host->sectors - 1;
 949 -               host->single_read_sectors[5] = ~0; /* Safety net */
 950 -       }
 951
 952 -       host->use_dma = host->have_dma && (data->blocks > host->pio_limit);
 953 -       if (!host->use_dma) {
 954 +       if (!host->dma_desc) {
 955 +               /* Use PIO */
 956                 int flags;
 957
 958 -               flags = SG_MITER_ATOMIC;
 959                 if (data->flags & MMC_DATA_READ)
 960 -                       flags |= SG_MITER_TO_SG;
 961 +                       flags = SG_MITER_TO_SG;
 962                 else
 963 -                       flags |= SG_MITER_FROM_SG;
 964 +                       flags = SG_MITER_FROM_SG;
 965                 sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags);
 966                 host->blocks = data->blocks;
 967         }
 968 @@ -726,19 +890,20 @@ static void bcm2835_sdhost_prepare_data(
 969         bcm2835_sdhost_set_transfer_irqs(host);
 970
 971         bcm2835_sdhost_write(host, data->blksz, SDHBCT);
 972 -       bcm2835_sdhost_write(host, host->use_dma ? data->blocks : 0, SDHBLC);
 973 +       bcm2835_sdhost_write(host, data->blocks, SDHBLC);
 974
 975         BUG_ON(!host->data);
 976  }
 977
 978 -
 979 -void bcm2835_sdhost_send_command(struct bcm2835_host *host, struct mmc_command *cmd)
 980 +bool bcm2835_sdhost_send_command(struct bcm2835_host *host,
 981 +                                struct mmc_command *cmd)
 982  {
 983         u32 sdcmd, sdhsts;
 984         unsigned long timeout;
 985         int delay;
 986
 987         WARN_ON(host->cmd);
 988 +       log_event("CMD<", cmd->opcode, cmd->arg);
 989
 990         if (cmd->data)
 991                 pr_debug("%s: send_command %d 0x%x "
 992 @@ -761,9 +926,9 @@ void bcm2835_sdhost_send_command(struct
 993                         pr_err("%s: previous command never completed.\n",
 994                                 mmc_hostname(host->mmc));
 995                         bcm2835_sdhost_dumpregs(host);
 996 -                       cmd->error = -EIO;
 997 +                       cmd->error = -EILSEQ;
 998                         tasklet_schedule(&host->finish_tasklet);
 999 -                       return;
1000 +                       return false;
1001                 }
1002                 timeout--;
1003                 udelay(10);
1004 @@ -791,23 +956,24 @@ void bcm2835_sdhost_send_command(struct
1005         if (sdhsts & SDHSTS_ERROR_MASK)
1006                 bcm2835_sdhost_write(host, sdhsts, SDHSTS);
1007
1008 -       bcm2835_sdhost_prepare_data(host, cmd);
1009 -
1010 -       bcm2835_sdhost_write(host, cmd->arg, SDARG);
1011 -
1012         if ((cmd->flags & MMC_RSP_136) && (cmd->flags & MMC_RSP_BUSY)) {
1013                 pr_err("%s: unsupported response type!\n",
1014                         mmc_hostname(host->mmc));
1015                 cmd->error = -EINVAL;
1016                 tasklet_schedule(&host->finish_tasklet);
1017 -               return;
1018 +               return false;
1019         }
1020
1021 +       bcm2835_sdhost_prepare_data(host, cmd);
1022 +
1023 +       bcm2835_sdhost_write(host, cmd->arg, SDARG);
1024 +
1025         sdcmd = cmd->opcode & SDCMD_CMD_MASK;
1026
1027 -       if (!(cmd->flags & MMC_RSP_PRESENT))
1028 +       host->use_busy = 0;
1029 +       if (!(cmd->flags & MMC_RSP_PRESENT)) {
1030                 sdcmd |= SDCMD_NO_RESPONSE;
1031 -       else {
1032 +       } else {
1033                 if (cmd->flags & MMC_RSP_136)
1034                         sdcmd |= SDCMD_LONG_RESPONSE;
1035                 if (cmd->flags & MMC_RSP_BUSY) {
1036 @@ -817,6 +983,7 @@ void bcm2835_sdhost_send_command(struct
1037         }
1038
1039         if (cmd->data) {
1040 +               log_event("CMDD", cmd->data->blocks, cmd->data->blksz);
1041                 if (host->delay_after_stop) {
1042                         struct timeval now;
1043                         int time_since_stop;
1044 @@ -839,10 +1006,12 @@ void bcm2835_sdhost_send_command(struct
1045         }
1046
1047         bcm2835_sdhost_write(host, sdcmd | SDCMD_NEW_FLAG, SDCMD);
1048 -}
1049
1050 +       return true;
1051 +}
1052
1053 -static void bcm2835_sdhost_finish_command(struct bcm2835_host *host);
1054 +static void bcm2835_sdhost_finish_command(struct bcm2835_host *host,
1055 +                                         unsigned long *irq_flags);
1056  static void bcm2835_sdhost_transfer_complete(struct bcm2835_host *host);
1057
1058  static void bcm2835_sdhost_finish_data(struct bcm2835_host *host)
1059 @@ -852,6 +1021,7 @@ static void bcm2835_sdhost_finish_data(s
1060         data = host->data;
1061         BUG_ON(!data);
1062
1063 +       log_event("FDA<", (u32)host->mrq, (u32)host->cmd);
1064         pr_debug("finish_data(error %d, stop %d, sbc %d)\n",
1065                data->error, data->stop ? 1 : 0,
1066                host->mrq->sbc ? 1 : 0);
1067 @@ -859,10 +1029,7 @@ static void bcm2835_sdhost_finish_data(s
1068         host->hcfg &= ~(SDHCFG_DATA_IRPT_EN | SDHCFG_BLOCK_IRPT_EN);
1069         bcm2835_sdhost_write(host, host->hcfg, SDHCFG);
1070
1071 -       if (data->error) {
1072 -               data->bytes_xfered = 0;
1073 -       } else
1074 -               data->bytes_xfered = data->blksz * data->blocks;
1075 +       data->bytes_xfered = data->error ? 0 : (data->blksz * data->blocks);
1076
1077         host->data_complete = 1;
1078
1079 @@ -877,9 +1044,9 @@ static void bcm2835_sdhost_finish_data(s
1080         }
1081         else
1082                 bcm2835_sdhost_transfer_complete(host);
1083 +       log_event("FDA>", (u32)host->mrq, (u32)host->cmd);
1084  }
1085
1086 -
1087  static void bcm2835_sdhost_transfer_complete(struct bcm2835_host *host)
1088  {
1089         struct mmc_data *data;
1090 @@ -891,6 +1058,7 @@ static void bcm2835_sdhost_transfer_comp
1091         data = host->data;
1092         host->data = NULL;
1093
1094 +       log_event("TCM<", (u32)data, data->error);
1095         pr_debug("transfer_complete(error %d, stop %d)\n",
1096                data->error, data->stop ? 1 : 0);
1097
1098 @@ -899,88 +1067,114 @@ static void bcm2835_sdhost_transfer_comp
1099          * a) open-ended multiblock transfer (no CMD23)
1100          * b) error in multiblock transfer
1101          */
1102 -       if (data->stop &&
1103 -           (data->error ||
1104 -            !host->mrq->sbc)) {
1105 -               host->flush_fifo = 1;
1106 -               bcm2835_sdhost_send_command(host, data->stop);
1107 -               if (host->delay_after_stop)
1108 -                       do_gettimeofday(&host->stop_time);
1109 -               if (!host->use_busy)
1110 -                       bcm2835_sdhost_finish_command(host);
1111 +       if (host->mrq->stop && (data->error || !host->use_sbc)) {
1112 +               if (bcm2835_sdhost_send_command(host, host->mrq->stop)) {
1113 +                       /* No busy, so poll for completion */
1114 +                       if (!host->use_busy)
1115 +                               bcm2835_sdhost_finish_command(host, NULL);
1116 +
1117 +                       if (host->delay_after_stop)
1118 +                               do_gettimeofday(&host->stop_time);
1119 +               }
1120         } else {
1121 +               bcm2835_sdhost_wait_transfer_complete(host);
1122                 tasklet_schedule(&host->finish_tasklet);
1123         }
1124 +       log_event("TCM>", (u32)data, 0);
1125  }
1126
1127 -static void bcm2835_sdhost_finish_command(struct bcm2835_host *host)
1128 +/* If irq_flags is valid, the caller is in a thread context and is allowed
1129 +   to sleep */
1130 +static void bcm2835_sdhost_finish_command(struct bcm2835_host *host,
1131 +                                         unsigned long *irq_flags)
1132  {
1133         u32 sdcmd;
1134 -       unsigned long timeout;
1135 +       u32 retries;
1136  #ifdef DEBUG
1137         struct timeval before, after;
1138         int timediff = 0;
1139  #endif
1140
1141 +       log_event("FCM<", (u32)host->mrq, (u32)host->cmd);
1142         pr_debug("finish_command(%x)\n", bcm2835_sdhost_read(host, SDCMD));
1143
1144         BUG_ON(!host->cmd || !host->mrq);
1145
1146 -#ifdef DEBUG
1147 -       do_gettimeofday(&before);
1148 -#endif
1149 -       /* Wait max 100 ms */
1150 -       timeout = 10000;
1151 +       /* Poll quickly at first */
1152 +
1153 +       retries = host->cmd_quick_poll_retries;
1154 +       if (!retries) {
1155 +               /* Work out how many polls take 1us by timing 10us */
1156 +               struct timeval start, now;
1157 +               int us_diff;
1158 +
1159 +               retries = 1;
1160 +               do {
1161 +                       int i;
1162 +
1163 +                       retries *= 2;
1164 +
1165 +                       do_gettimeofday(&start);
1166 +
1167 +                       for (i = 0; i < retries; i++) {
1168 +                               cpu_relax();
1169 +                               sdcmd = bcm2835_sdhost_read(host, SDCMD);
1170 +                       }
1171 +
1172 +                       do_gettimeofday(&now);
1173 +                       us_diff = (now.tv_sec - start.tv_sec) * 1000000 +
1174 +                               (now.tv_usec - start.tv_usec);
1175 +               } while (us_diff < 10);
1176 +
1177 +               host->cmd_quick_poll_retries = ((retries * us_diff + 9)*CMD_DALLY_US)/10 + 1;
1178 +               retries = 1; // We've already waited long enough this time
1179 +       }
1180 +
1181 +       retries = host->cmd_quick_poll_retries;
1182         for (sdcmd = bcm2835_sdhost_read(host, SDCMD);
1183 -            (sdcmd & SDCMD_NEW_FLAG) && timeout;
1184 -            timeout--) {
1185 -               if (host->flush_fifo) {
1186 -                       while (bcm2835_sdhost_read(host, SDHSTS) &
1187 -                              SDHSTS_DATA_FLAG)
1188 -                               (void)bcm2835_sdhost_read(host, SDDATA);
1189 -               }
1190 -               udelay(10);
1191 +            (sdcmd & SDCMD_NEW_FLAG) && !(sdcmd & SDCMD_FAIL_FLAG) && retries;
1192 +            retries--) {
1193 +               cpu_relax();
1194                 sdcmd = bcm2835_sdhost_read(host, SDCMD);
1195         }
1196 -#ifdef DEBUG
1197 -       do_gettimeofday(&after);
1198 -       timediff = (after.tv_sec - before.tv_sec)*1000000 +
1199 -               (after.tv_usec - before.tv_usec);
1200
1201 -       pr_debug(" finish_command - waited %dus\n", timediff);
1202 -#endif
1203 +       if (!retries) {
1204 +               unsigned long wait_max;
1205 +
1206 +               if (!irq_flags) {
1207 +                       /* Schedule the work */
1208 +                       log_event("CWWQ", 0, 0);
1209 +                       schedule_work(&host->cmd_wait_wq);
1210 +                       return;
1211 +               }
1212 +
1213 +               /* Wait max 100 ms */
1214 +               wait_max = jiffies + msecs_to_jiffies(100);
1215 +               while (time_before(jiffies, wait_max)) {
1216 +                       spin_unlock_irqrestore(&host->lock, *irq_flags);
1217 +                       usleep_range(1, 10);
1218 +                       spin_lock_irqsave(&host->lock, *irq_flags);
1219 +                       sdcmd = bcm2835_sdhost_read(host, SDCMD);
1220 +                       if (!(sdcmd & SDCMD_NEW_FLAG) ||
1221 +                           (sdcmd & SDCMD_FAIL_FLAG))
1222 +                               break;
1223 +               }
1224 +       }
1225
1226 -       if (timeout == 0) {
1227 +       /* Check for errors */
1228 +       if (sdcmd & SDCMD_NEW_FLAG) {
1229                 pr_err("%s: command never completed.\n",
1230                        mmc_hostname(host->mmc));
1231                 bcm2835_sdhost_dumpregs(host);
1232                 host->cmd->error = -EIO;
1233                 tasklet_schedule(&host->finish_tasklet);
1234                 return;
1235 -       }
1236 -
1237 -       if (host->flush_fifo) {
1238 -               for (timeout = 100;
1239 -                    (bcm2835_sdhost_read(host, SDHSTS) & SDHSTS_DATA_FLAG) && timeout;
1240 -                    timeout--) {
1241 -                       (void)bcm2835_sdhost_read(host, SDDATA);
1242 -               }
1243 -               host->flush_fifo = 0;
1244 -               if (timeout == 0) {
1245 -                       pr_err("%s: FIFO never drained.\n",
1246 -                              mmc_hostname(host->mmc));
1247 -                       bcm2835_sdhost_dumpregs(host);
1248 -                       host->cmd->error = -EIO;
1249 -                       tasklet_schedule(&host->finish_tasklet);
1250 -                       return;
1251 -               }
1252 -       }
1253 -
1254 -       /* Check for errors */
1255 -       if (sdcmd & SDCMD_FAIL_FLAG)
1256 -       {
1257 +       } else if (sdcmd & SDCMD_FAIL_FLAG) {
1258                 u32 sdhsts = bcm2835_sdhost_read(host, SDHSTS);
1259
1260 +               /* Clear the errors */
1261 +               bcm2835_sdhost_write(host, SDHSTS_ERROR_MASK, SDHSTS);
1262 +
1263                 if (host->debug)
1264                         pr_info("%s: error detected - CMD %x, HSTS %03x, EDM %x\n",
1265                                 mmc_hostname(host->mmc), sdcmd, sdhsts,
1266 @@ -1003,7 +1197,7 @@ static void bcm2835_sdhost_finish_comman
1267                                        mmc_hostname(host->mmc),
1268                                        host->cmd->opcode);
1269                                 bcm2835_sdhost_dumpregs(host);
1270 -                               host->cmd->error = -EIO;
1271 +                               host->cmd->error = -EILSEQ;
1272                         }
1273                         tasklet_schedule(&host->finish_tasklet);
1274                         return;
1275 @@ -1018,31 +1212,31 @@ static void bcm2835_sdhost_finish_comman
1276                         pr_debug("%s: finish_command %08x %08x %08x %08x\n",
1277                                  mmc_hostname(host->mmc),
1278                                  host->cmd->resp[0], host->cmd->resp[1], host->cmd->resp[2], host->cmd->resp[3]);
1279 +                       log_event("RSP ", host->cmd->resp[0], host->cmd->resp[1]);
1280                 } else {
1281                         host->cmd->resp[0] = bcm2835_sdhost_read(host, SDRSP0);
1282                         pr_debug("%s: finish_command %08x\n",
1283                                  mmc_hostname(host->mmc),
1284                                  host->cmd->resp[0]);
1285 +                       log_event("RSP ", host->cmd->resp[0], 0);
1286                 }
1287         }
1288
1289 -       host->cmd->error = 0;
1290 -
1291         if (host->cmd == host->mrq->sbc) {
1292                 /* Finished CMD23, now send actual command. */
1293                 host->cmd = NULL;
1294 -               bcm2835_sdhost_send_command(host, host->mrq->cmd);
1295 +               if (bcm2835_sdhost_send_command(host, host->mrq->cmd)) {
1296 +                       if (host->data && host->dma_desc)
1297 +                               /* DMA transfer starts now, PIO starts after irq */
1298 +                               bcm2835_sdhost_start_dma(host);
1299
1300 -               if (host->cmd->data && host->use_dma)
1301 -                       /* DMA transfer starts now, PIO starts after irq */
1302 -                       bcm2835_sdhost_transfer_dma(host);
1303 -
1304 -               if (!host->use_busy)
1305 -                       bcm2835_sdhost_finish_command(host);
1306 -       } else if (host->cmd == host->mrq->stop)
1307 +                       if (!host->use_busy)
1308 +                               bcm2835_sdhost_finish_command(host, NULL);
1309 +               }
1310 +       } else if (host->cmd == host->mrq->stop) {
1311                 /* Finished CMD12 */
1312                 tasklet_schedule(&host->finish_tasklet);
1313 -       else {
1314 +       } else {
1315                 /* Processed actual command. */
1316                 host->cmd = NULL;
1317                 if (!host->data)
1318 @@ -1050,6 +1244,7 @@ static void bcm2835_sdhost_finish_comman
1319                 else if (host->data_complete)
1320                         bcm2835_sdhost_transfer_complete(host);
1321         }
1322 +       log_event("FCM>", (u32)host->mrq, (u32)host->cmd);
1323  }
1324
1325  static void bcm2835_sdhost_timeout(unsigned long data)
1326 @@ -1060,10 +1255,12 @@ static void bcm2835_sdhost_timeout(unsig
1327         host = (struct bcm2835_host *)data;
1328
1329         spin_lock_irqsave(&host->lock, flags);
1330 +       log_event("TIM<", 0, 0);
1331
1332         if (host->mrq) {
1333                 pr_err("%s: timeout waiting for hardware interrupt.\n",
1334                         mmc_hostname(host->mmc));
1335 +               log_dump();
1336                 bcm2835_sdhost_dumpregs(host);
1337
1338                 if (host->data) {
1339 @@ -1084,74 +1281,15 @@ static void bcm2835_sdhost_timeout(unsig
1340         spin_unlock_irqrestore(&host->lock, flags);
1341  }
1342
1343 -static void bcm2835_sdhost_pio_timeout(unsigned long data)
1344 -{
1345 -       struct bcm2835_host *host;
1346 -       unsigned long flags;
1347 -
1348 -       host = (struct bcm2835_host *)data;
1349 -
1350 -       spin_lock_irqsave(&host->lock, flags);
1351 -
1352 -       if (host->data) {
1353 -               u32 sdhsts = bcm2835_sdhost_read(host, SDHSTS);
1354 -
1355 -               if (sdhsts & SDHSTS_REW_TIME_OUT) {
1356 -                       pr_err("%s: transfer timeout\n",
1357 -                              mmc_hostname(host->mmc));
1358 -                       if (host->debug)
1359 -                               bcm2835_sdhost_dumpregs(host);
1360 -               } else {
1361 -                       pr_err("%s: unexpected transfer timeout\n",
1362 -                              mmc_hostname(host->mmc));
1363 -                       bcm2835_sdhost_dumpregs(host);
1364 -               }
1365 -
1366 -               bcm2835_sdhost_write(host, SDHSTS_TRANSFER_ERROR_MASK,
1367 -                                    SDHSTS);
1368 -
1369 -               host->data->error = -ETIMEDOUT;
1370 -
1371 -               bcm2835_sdhost_finish_data(host);
1372 -       }
1373 -
1374 -       mmiowb();
1375 -       spin_unlock_irqrestore(&host->lock, flags);
1376 -}
1377 -
1378 -static void bcm2835_sdhost_enable_sdio_irq_nolock(struct bcm2835_host *host, int enable)
1379 -{
1380 -       if (enable)
1381 -               host->hcfg |= SDHCFG_SDIO_IRPT_EN;
1382 -       else
1383 -               host->hcfg &= ~SDHCFG_SDIO_IRPT_EN;
1384 -       bcm2835_sdhost_write(host, host->hcfg, SDHCFG);
1385 -       mmiowb();
1386 -}
1387 -
1388 -static void bcm2835_sdhost_enable_sdio_irq(struct mmc_host *mmc, int enable)
1389 -{
1390 -       struct bcm2835_host *host = mmc_priv(mmc);
1391 -       unsigned long flags;
1392 -
1393 -       pr_debug("%s: enable_sdio_irq(%d)\n", mmc_hostname(mmc), enable);
1394 -       spin_lock_irqsave(&host->lock, flags);
1395 -       bcm2835_sdhost_enable_sdio_irq_nolock(host, enable);
1396 -       spin_unlock_irqrestore(&host->lock, flags);
1397 -}
1398 -
1399 -static u32 bcm2835_sdhost_busy_irq(struct bcm2835_host *host, u32 intmask)
1400 +static void bcm2835_sdhost_busy_irq(struct bcm2835_host *host, u32 intmask)
1401  {
1402 -       const u32 handled = (SDHSTS_REW_TIME_OUT | SDHSTS_CMD_TIME_OUT |
1403 -                            SDHSTS_CRC16_ERROR | SDHSTS_CRC7_ERROR |
1404 -                            SDHSTS_FIFO_ERROR);
1405 -
1406 +       log_event("IRQB", (u32)host->cmd, intmask);
1407         if (!host->cmd) {
1408                 pr_err("%s: got command busy interrupt 0x%08x even "
1409                         "though no command operation was in progress.\n",
1410                         mmc_hostname(host->mmc), (unsigned)intmask);
1411                 bcm2835_sdhost_dumpregs(host);
1412 -               return 0;
1413 +               return;
1414         }
1415
1416         if (!host->use_busy) {
1417 @@ -1159,7 +1297,7 @@ static u32 bcm2835_sdhost_busy_irq(struc
1418                         "though not expecting one.\n",
1419                         mmc_hostname(host->mmc), (unsigned)intmask);
1420                 bcm2835_sdhost_dumpregs(host);
1421 -               return 0;
1422 +               return;
1423         }
1424         host->use_busy = 0;
1425
1426 @@ -1182,28 +1320,23 @@ static u32 bcm2835_sdhost_busy_irq(struc
1427                 } else if (intmask & SDHSTS_CMD_TIME_OUT)
1428                         host->cmd->error = -ETIMEDOUT;
1429
1430 +               log_dump();
1431                 bcm2835_sdhost_dumpregs(host);
1432 -               tasklet_schedule(&host->finish_tasklet);
1433         }
1434         else
1435 -               bcm2835_sdhost_finish_command(host);
1436 -
1437 -       return handled;
1438 +               bcm2835_sdhost_finish_command(host, NULL);
1439  }
1440
1441 -static u32 bcm2835_sdhost_data_irq(struct bcm2835_host *host, u32 intmask)
1442 +static void bcm2835_sdhost_data_irq(struct bcm2835_host *host, u32 intmask)
1443  {
1444 -       const u32 handled = (SDHSTS_REW_TIME_OUT |
1445 -                            SDHSTS_CRC16_ERROR |
1446 -                            SDHSTS_FIFO_ERROR);
1447 -
1448         /* There are no dedicated data/space available interrupt
1449            status bits, so it is necessary to use the single shared
1450            data/space available FIFO status bits. It is therefore not
1451            an error to get here when there is no data transfer in
1452            progress. */
1453 +       log_event("IRQD", (u32)host->data, intmask);
1454         if (!host->data)
1455 -               return 0;
1456 +               return;
1457
1458         if (intmask & (SDHSTS_CRC16_ERROR |
1459                        SDHSTS_FIFO_ERROR |
1460 @@ -1214,46 +1347,37 @@ static u32 bcm2835_sdhost_data_irq(struc
1461                 else
1462                         host->data->error = -ETIMEDOUT;
1463
1464 -               bcm2835_sdhost_dumpregs(host);
1465 -               tasklet_schedule(&host->finish_tasklet);
1466 -               return handled;
1467 +               if (host->debug) {
1468 +                       log_dump();
1469 +                       bcm2835_sdhost_dumpregs(host);
1470 +               }
1471         }
1472
1473 -       /* Use the block interrupt for writes after the first block */
1474 -       if (host->data->flags & MMC_DATA_WRITE) {
1475 +       if (host->data->error) {
1476 +               bcm2835_sdhost_finish_data(host);
1477 +       } else if (host->data->flags & MMC_DATA_WRITE) {
1478 +               /* Use the block interrupt for writes after the first block */
1479                 host->hcfg &= ~(SDHCFG_DATA_IRPT_EN);
1480                 host->hcfg |= SDHCFG_BLOCK_IRPT_EN;
1481                 bcm2835_sdhost_write(host, host->hcfg, SDHCFG);
1482 -               if (host->data->error)
1483 -                       bcm2835_sdhost_finish_data(host);
1484 -               else
1485 -                       bcm2835_sdhost_transfer_pio(host);
1486 +               bcm2835_sdhost_transfer_pio(host);
1487         } else {
1488 -               if (!host->data->error) {
1489 -                       bcm2835_sdhost_transfer_pio(host);
1490 -                       host->blocks--;
1491 -               }
1492 +               bcm2835_sdhost_transfer_pio(host);
1493 +               host->blocks--;
1494                 if ((host->blocks == 0) || host->data->error)
1495                         bcm2835_sdhost_finish_data(host);
1496         }
1497 -
1498 -       return handled;
1499  }
1500
1501 -static u32 bcm2835_sdhost_block_irq(struct bcm2835_host *host, u32 intmask)
1502 +static void bcm2835_sdhost_block_irq(struct bcm2835_host *host, u32 intmask)
1503  {
1504 -       struct dma_chan *dma_chan;
1505 -       u32 dir_data;
1506 -       const u32 handled = (SDHSTS_REW_TIME_OUT |
1507 -                            SDHSTS_CRC16_ERROR |
1508 -                            SDHSTS_FIFO_ERROR);
1509 -
1510 +       log_event("IRQK", (u32)host->data, intmask);
1511         if (!host->data) {
1512                 pr_err("%s: got block interrupt 0x%08x even "
1513                         "though no data operation was in progress.\n",
1514                         mmc_hostname(host->mmc), (unsigned)intmask);
1515                 bcm2835_sdhost_dumpregs(host);
1516 -               return handled;
1517 +               return;
1518         }
1519
1520         if (intmask & (SDHSTS_CRC16_ERROR |
1521 @@ -1265,149 +1389,69 @@ static u32 bcm2835_sdhost_block_irq(stru
1522                 else
1523                         host->data->error = -ETIMEDOUT;
1524
1525 -               if (host->debug)
1526 +               if (host->debug) {
1527 +                       log_dump();
1528                         bcm2835_sdhost_dumpregs(host);
1529 -               tasklet_schedule(&host->finish_tasklet);
1530 -               return handled;
1531 +               }
1532         }
1533
1534 -       if (!host->use_dma) {
1535 +       if (!host->dma_desc) {
1536                 BUG_ON(!host->blocks);
1537 -               host->blocks--;
1538 -               if ((host->blocks == 0) || host->data->error) {
1539 -                       /* Cancel the timer */
1540 -                       del_timer(&host->pio_timer);
1541 -
1542 +               if (host->data->error || (--host->blocks == 0)) {
1543                         bcm2835_sdhost_finish_data(host);
1544                 } else {
1545 -                       /* Reset the timer */
1546 -                       mod_timer(&host->pio_timer,
1547 -                                 jiffies + host->pio_timeout);
1548 -
1549                         bcm2835_sdhost_transfer_pio(host);
1550 -
1551 -                       /* Reset the timer */
1552 -                       mod_timer(&host->pio_timer,
1553 -                                 jiffies + host->pio_timeout);
1554                 }
1555         } else if (host->data->flags & MMC_DATA_WRITE) {
1556 -               dma_chan = host->dma_chan_tx;
1557 -               dir_data = DMA_TO_DEVICE;
1558 -               dma_unmap_sg(dma_chan->device->dev,
1559 -                            host->data->sg, host->data->sg_len,
1560 -                            dir_data);
1561 -
1562                 bcm2835_sdhost_finish_data(host);
1563         }
1564 -
1565 -       return handled;
1566  }
1567
1568 -
1569  static irqreturn_t bcm2835_sdhost_irq(int irq, void *dev_id)
1570  {
1571         irqreturn_t result = IRQ_NONE;
1572         struct bcm2835_host *host = dev_id;
1573 -       u32 unexpected = 0, early = 0;
1574 -       int loops = 0;
1575 +       u32 intmask;
1576
1577         spin_lock(&host->lock);
1578
1579 -       for (loops = 0; loops < 1; loops++) {
1580 -               u32 intmask, handled;
1581 -
1582 -               intmask = bcm2835_sdhost_read(host, SDHSTS);
1583 -               handled = intmask & (SDHSTS_BUSY_IRPT |
1584 -                                    SDHSTS_BLOCK_IRPT |
1585 -                                    SDHSTS_SDIO_IRPT |
1586 -                                    SDHSTS_DATA_FLAG);
1587 -               if ((handled == SDHSTS_DATA_FLAG) &&
1588 -                   (loops == 0) && !host->data) {
1589 -                       pr_err("%s: sdhost_irq data interrupt 0x%08x even "
1590 -                              "though no data operation was in progress.\n",
1591 -                              mmc_hostname(host->mmc),
1592 -                              (unsigned)intmask);
1593 -
1594 -                       bcm2835_sdhost_dumpregs(host);
1595 -               }
1596 -
1597 -               if (!handled)
1598 -                       break;
1599 +       intmask = bcm2835_sdhost_read(host, SDHSTS);
1600 +       log_event("IRQ<", intmask, 0);
1601
1602 -               if (loops)
1603 -                       early |= handled;
1604 +       bcm2835_sdhost_write(host,
1605 +                            SDHSTS_BUSY_IRPT |
1606 +                            SDHSTS_BLOCK_IRPT |
1607 +                            SDHSTS_SDIO_IRPT |
1608 +                            SDHSTS_DATA_FLAG,
1609 +                            SDHSTS);
1610
1611 +       if (intmask & SDHSTS_BLOCK_IRPT) {
1612 +               bcm2835_sdhost_block_irq(host, intmask);
1613                 result = IRQ_HANDLED;
1614 +       }
1615
1616 -               /* Clear all interrupts and notifications */
1617 -               bcm2835_sdhost_write(host, intmask, SDHSTS);
1618 -
1619 -               if (intmask & SDHSTS_BUSY_IRPT)
1620 -                       handled |= bcm2835_sdhost_busy_irq(host, intmask);
1621 -
1622 -               /* There is no true data interrupt status bit, so it is
1623 -                  necessary to qualify the data flag with the interrupt
1624 -                  enable bit */
1625 -               if ((intmask & SDHSTS_DATA_FLAG) &&
1626 -                   (host->hcfg & SDHCFG_DATA_IRPT_EN))
1627 -                       handled |= bcm2835_sdhost_data_irq(host, intmask);
1628 -
1629 -               if (intmask & SDHSTS_BLOCK_IRPT)
1630 -                       handled |= bcm2835_sdhost_block_irq(host, intmask);
1631 -
1632 -               if (intmask & SDHSTS_SDIO_IRPT) {
1633 -                       bcm2835_sdhost_enable_sdio_irq_nolock(host, false);
1634 -                       host->thread_isr |= SDHSTS_SDIO_IRPT;
1635 -                       result = IRQ_WAKE_THREAD;
1636 -               }
1637 +       if (intmask & SDHSTS_BUSY_IRPT) {
1638 +               bcm2835_sdhost_busy_irq(host, intmask);
1639 +               result = IRQ_HANDLED;
1640 +       }
1641
1642 -               unexpected |= (intmask & ~handled);
1643 +       /* There is no true data interrupt status bit, so it is
1644 +          necessary to qualify the data flag with the interrupt
1645 +          enable bit */
1646 +       if ((intmask & SDHSTS_DATA_FLAG) &&
1647 +           (host->hcfg & SDHCFG_DATA_IRPT_EN)) {
1648 +               bcm2835_sdhost_data_irq(host, intmask);
1649 +               result = IRQ_HANDLED;
1650         }
1651
1652         mmiowb();
1653
1654 +       log_event("IRQ>", bcm2835_sdhost_read(host, SDHSTS), 0);
1655         spin_unlock(&host->lock);
1656
1657 -       if (early)
1658 -               pr_debug("%s: early %x (loops %d)\n",
1659 -                        mmc_hostname(host->mmc), early, loops);
1660 -
1661 -       if (unexpected) {
1662 -               pr_err("%s: unexpected interrupt 0x%08x.\n",
1663 -                          mmc_hostname(host->mmc), unexpected);
1664 -               bcm2835_sdhost_dumpregs(host);
1665 -       }
1666 -
1667         return result;
1668  }
1669
1670 -static irqreturn_t bcm2835_sdhost_thread_irq(int irq, void *dev_id)
1671 -{
1672 -       struct bcm2835_host *host = dev_id;
1673 -       unsigned long flags;
1674 -       u32 isr;
1675 -
1676 -       spin_lock_irqsave(&host->lock, flags);
1677 -       isr = host->thread_isr;
1678 -       host->thread_isr = 0;
1679 -       spin_unlock_irqrestore(&host->lock, flags);
1680 -
1681 -       if (isr & SDHSTS_SDIO_IRPT) {
1682 -               sdio_run_irqs(host->mmc);
1683 -
1684 -/* Is this necessary? Why re-enable an interrupt which is enabled?
1685 -               spin_lock_irqsave(&host->lock, flags);
1686 -               if (host->flags & SDHSTS_SDIO_IRPT_ENABLED)
1687 -                       bcm2835_sdhost_enable_sdio_irq_nolock(host, true);
1688 -               spin_unlock_irqrestore(&host->lock, flags);
1689 -*/
1690 -       }
1691 -
1692 -       return isr ? IRQ_HANDLED : IRQ_NONE;
1693 -}
1694 -
1695 -
1696 -
1697  void bcm2835_sdhost_set_clock(struct bcm2835_host *host, unsigned int clock)
1698  {
1699         int div = 0; /* Initialized for compiler warning */
1700 @@ -1417,9 +1461,8 @@ void bcm2835_sdhost_set_clock(struct bcm
1701                 pr_info("%s: set_clock(%d)\n", mmc_hostname(host->mmc), clock);
1702
1703         if ((host->overclock_50 > 50) &&
1704 -           (clock == 50*MHZ)) {
1705 +           (clock == 50*MHZ))
1706                 clock = host->overclock_50 * MHZ + (MHZ - 1);
1707 -       }
1708
1709         /* The SDCDIV register has 11 bits, and holds (div - 2).
1710            But in data mode the max is 50MHz wihout a minimum, and only the
1711 @@ -1466,6 +1509,11 @@ void bcm2835_sdhost_set_clock(struct bcm
1712         clock = host->max_clk / (div + 2);
1713         host->mmc->actual_clock = clock;
1714
1715 +       /* Calibrate some delays */
1716 +
1717 +       host->ns_per_fifo_word = (1000000000/clock) *
1718 +               ((host->mmc->caps & MMC_CAP_4_BIT_DATA) ? 8 : 32);
1719 +
1720         if (clock > input_clock) {
1721                 /* Save the closest value, to make it easier
1722                    to reduce in the event of error */
1723 @@ -1501,6 +1549,7 @@ static void bcm2835_sdhost_request(struc
1724  {
1725         struct bcm2835_host *host;
1726         unsigned long flags;
1727 +       u32 edm, fsm;
1728
1729         host = mmc_priv(mmc);
1730
1731 @@ -1521,6 +1570,8 @@ static void bcm2835_sdhost_request(struc
1732         }
1733
1734         /* Reset the error statuses in case this is a retry */
1735 +       if (mrq->sbc)
1736 +               mrq->sbc->error = 0;
1737         if (mrq->cmd)
1738                 mrq->cmd->error = 0;
1739         if (mrq->data)
1740 @@ -1536,28 +1587,58 @@ static void bcm2835_sdhost_request(struc
1741                 return;
1742         }
1743
1744 +       if (host->use_dma && mrq->data &&
1745 +           (mrq->data->blocks > host->pio_limit))
1746 +               bcm2835_sdhost_prepare_dma(host, mrq->data);
1747 +
1748         spin_lock_irqsave(&host->lock, flags);
1749
1750         WARN_ON(host->mrq != NULL);
1751 -
1752         host->mrq = mrq;
1753
1754 -       if (mrq->sbc)
1755 -               bcm2835_sdhost_send_command(host, mrq->sbc);
1756 -       else
1757 -               bcm2835_sdhost_send_command(host, mrq->cmd);
1758 +       edm = bcm2835_sdhost_read(host, SDEDM);
1759 +       fsm = edm & SDEDM_FSM_MASK;
1760
1761 -       mmiowb();
1762 -       spin_unlock_irqrestore(&host->lock, flags);
1763 +       log_event("REQ<", (u32)mrq, edm);
1764 +       if ((fsm != SDEDM_FSM_IDENTMODE) &&
1765 +           (fsm != SDEDM_FSM_DATAMODE)) {
1766 +               pr_err("%s: previous command (%d) not complete (EDM %x)\n",
1767 +                      mmc_hostname(host->mmc),
1768 +                      bcm2835_sdhost_read(host, SDCMD) & SDCMD_CMD_MASK,
1769 +                      edm);
1770 +               log_event("REQ!", (u32)mrq, edm);
1771 +               log_dump();
1772 +               bcm2835_sdhost_dumpregs(host);
1773 +               mrq->cmd->error = -EILSEQ;
1774 +               tasklet_schedule(&host->finish_tasklet);
1775 +               mmiowb();
1776 +               spin_unlock_irqrestore(&host->lock, flags);
1777 +               return;
1778 +       }
1779 +
1780 +       host->use_sbc = !!mrq->sbc &&
1781 +               (host->mrq->data->flags & USE_CMD23_FLAGS);
1782 +       if (host->use_sbc) {
1783 +               if (bcm2835_sdhost_send_command(host, mrq->sbc)) {
1784 +                       if (!host->use_busy)
1785 +                               bcm2835_sdhost_finish_command(host, &flags);
1786 +               }
1787 +       } else if (bcm2835_sdhost_send_command(host, mrq->cmd)) {
1788 +               if (host->data && host->dma_desc)
1789 +                       /* DMA transfer starts now, PIO starts after irq */
1790 +                       bcm2835_sdhost_start_dma(host);
1791
1792 -       if (!mrq->sbc && mrq->cmd->data && host->use_dma)
1793 -               /* DMA transfer starts now, PIO starts after irq */
1794 -               bcm2835_sdhost_transfer_dma(host);
1795 +               if (!host->use_busy)
1796 +                       bcm2835_sdhost_finish_command(host, &flags);
1797 +       }
1798
1799 -       if (!host->use_busy)
1800 -               bcm2835_sdhost_finish_command(host);
1801 -}
1802 +       log_event("CMD ", (u32)mrq->cmd->opcode,
1803 +                  mrq->data ? (u32)mrq->data->blksz : 0);
1804 +       mmiowb();
1805
1806 +       log_event("REQ>", (u32)mrq, 0);
1807 +       spin_unlock_irqrestore(&host->lock, flags);
1808 +}
1809
1810  static void bcm2835_sdhost_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
1811  {
1812 @@ -1574,6 +1655,8 @@ static void bcm2835_sdhost_set_ios(struc
1813
1814         spin_lock_irqsave(&host->lock, flags);
1815
1816 +       log_event("IOS<", ios->clock, 0);
1817 +
1818         if (!ios->clock || ios->clock != host->clock) {
1819                 bcm2835_sdhost_set_clock(host, ios->clock);
1820                 host->clock = ios->clock;
1821 @@ -1596,59 +1679,53 @@ static void bcm2835_sdhost_set_ios(struc
1822         spin_unlock_irqrestore(&host->lock, flags);
1823  }
1824
1825 -static int bcm2835_sdhost_multi_io_quirk(struct mmc_card *card,
1826 -                                        unsigned int direction,
1827 -                                        u32 blk_pos, int blk_size)
1828 -{
1829 -       /* There is a bug in the host controller hardware that makes
1830 -          reading the final sector of the card as part of a multiple read
1831 -          problematic. Detect that case and shorten the read accordingly.
1832 -       */
1833 +static struct mmc_host_ops bcm2835_sdhost_ops = {
1834 +       .request = bcm2835_sdhost_request,
1835 +       .set_ios = bcm2835_sdhost_set_ios,
1836 +       .hw_reset = bcm2835_sdhost_reset,
1837 +};
1838 +
1839 +static void bcm2835_sdhost_cmd_wait_work(struct work_struct *work)
1840 +{
1841         struct bcm2835_host *host;
1842 +       unsigned long flags;
1843
1844 -       host = mmc_priv(card->host);
1845 +       host = container_of(work, struct bcm2835_host, cmd_wait_wq);
1846
1847 -       if (!host->sectors) {
1848 -               /* csd.capacity is in weird units - convert to sectors */
1849 -               u32 card_sectors = (card->csd.capacity << (card->csd.read_blkbits - 9));
1850 -               if ((direction == MMC_DATA_READ) &&
1851 -                   ((blk_pos + blk_size) == card_sectors))
1852 -                       blk_size--;
1853 -               return blk_size;
1854 -       }
1855 +       spin_lock_irqsave(&host->lock, flags);
1856
1857 -       if (direction == MMC_DATA_READ) {
1858 -               int i;
1859 -               int sector;
1860 -               for (i = 0; blk_pos > (sector = host->single_read_sectors[i]); i++)
1861 -                       continue;
1862 +       log_event("CWK<", (u32)host->cmd, (u32)host->mrq);
1863
1864 -               if ((blk_pos + blk_size) > sector)
1865 -                       blk_size = (blk_pos == sector) ? 1 : (sector - blk_pos);
1866 +       /*
1867 +        * If this tasklet gets rescheduled while running, it will
1868 +        * be run again afterwards but without any active request.
1869 +        */
1870 +       if (!host->mrq) {
1871 +               spin_unlock_irqrestore(&host->lock, flags);
1872 +               return;
1873         }
1874 -       return blk_size;
1875 -}
1876
1877 +       bcm2835_sdhost_finish_command(host, &flags);
1878
1879 -static struct mmc_host_ops bcm2835_sdhost_ops = {
1880 -       .request = bcm2835_sdhost_request,
1881 -       .set_ios = bcm2835_sdhost_set_ios,
1882 -       .enable_sdio_irq = bcm2835_sdhost_enable_sdio_irq,
1883 -       .hw_reset = bcm2835_sdhost_reset,
1884 -       .multi_io_quirk = bcm2835_sdhost_multi_io_quirk,
1885 -};
1886 +       mmiowb();
1887 +
1888 +       log_event("CWK>", (u32)host->cmd, 0);
1889
1890 +       spin_unlock_irqrestore(&host->lock, flags);
1891 +}
1892
1893  static void bcm2835_sdhost_tasklet_finish(unsigned long param)
1894  {
1895         struct bcm2835_host *host;
1896         unsigned long flags;
1897         struct mmc_request *mrq;
1898 +       struct dma_chan *terminate_chan = NULL;
1899
1900         host = (struct bcm2835_host *)param;
1901
1902         spin_lock_irqsave(&host->lock, flags);
1903
1904 +       log_event("TSK<", (u32)host->mrq, 0);
1905         /*
1906          * If this tasklet gets rescheduled while running, it will
1907          * be run again afterwards but without any active request.
1908 @@ -1683,11 +1760,23 @@ static void bcm2835_sdhost_tasklet_finis
1909
1910         mmiowb();
1911
1912 +       host->dma_desc = NULL;
1913 +       terminate_chan = host->dma_chan;
1914 +       host->dma_chan = NULL;
1915 +
1916         spin_unlock_irqrestore(&host->lock, flags);
1917 -       mmc_request_done(host->mmc, mrq);
1918 -}
1919
1920 +       if (terminate_chan)
1921 +       {
1922 +               int err = dmaengine_terminate_all(terminate_chan);
1923 +               if (err)
1924 +                       pr_err("%s: failed to terminate DMA (%d)\n",
1925 +                              mmc_hostname(host->mmc), err);
1926 +       }
1927
1928 +       mmc_request_done(host->mmc, mrq);
1929 +       log_event("TSK>", (u32)mrq, 0);
1930 +}
1931
1932  int bcm2835_sdhost_add_host(struct bcm2835_host *host)
1933  {
1934 @@ -1709,10 +1798,10 @@ int bcm2835_sdhost_add_host(struct bcm28
1935                  mmc->f_max, mmc->f_min, mmc->max_busy_timeout);
1936
1937         /* host controller capabilities */
1938 -       mmc->caps |= /* MMC_CAP_SDIO_IRQ |*/ MMC_CAP_4_BIT_DATA |
1939 +       mmc->caps |=
1940                 MMC_CAP_SD_HIGHSPEED | MMC_CAP_MMC_HIGHSPEED |
1941                 MMC_CAP_NEEDS_POLL | MMC_CAP_HW_RESET | MMC_CAP_ERASE |
1942 -               (ALLOW_CMD23 * MMC_CAP_CMD23);
1943 +               ((ALLOW_CMD23_READ|ALLOW_CMD23_WRITE) * MMC_CAP_CMD23);
1944
1945         spin_lock_init(&host->lock);
1946
1947 @@ -1722,9 +1811,9 @@ int bcm2835_sdhost_add_host(struct bcm28
1948                         pr_err("%s: unable to initialise DMA channels. "
1949                                "Falling back to PIO\n",
1950                                mmc_hostname(mmc));
1951 -                       host->have_dma = false;
1952 +                       host->use_dma = false;
1953                 } else {
1954 -                       host->have_dma = true;
1955 +                       host->use_dma = true;
1956
1957                         cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
1958                         cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
1959 @@ -1741,7 +1830,7 @@ int bcm2835_sdhost_add_host(struct bcm28
1960                         ret = dmaengine_slave_config(host->dma_chan_rx, &cfg);
1961                 }
1962         } else {
1963 -               host->have_dma = false;
1964 +               host->use_dma = false;
1965         }
1966
1967         mmc->max_segs = 128;
1968 @@ -1756,16 +1845,15 @@ int bcm2835_sdhost_add_host(struct bcm28
1969         tasklet_init(&host->finish_tasklet,
1970                 bcm2835_sdhost_tasklet_finish, (unsigned long)host);
1971
1972 -       setup_timer(&host->timer, bcm2835_sdhost_timeout,
1973 -                   (unsigned long)host);
1974 +       INIT_WORK(&host->cmd_wait_wq, bcm2835_sdhost_cmd_wait_work);
1975
1976 -       setup_timer(&host->pio_timer, bcm2835_sdhost_pio_timeout,
1977 +       setup_timer(&host->timer, bcm2835_sdhost_timeout,
1978                     (unsigned long)host);
1979
1980         bcm2835_sdhost_init(host, 0);
1981 -       ret = request_threaded_irq(host->irq, bcm2835_sdhost_irq,
1982 -                                  bcm2835_sdhost_thread_irq,
1983 -                                  IRQF_SHARED, mmc_hostname(mmc), host);
1984 +
1985 +       ret = request_irq(host->irq, bcm2835_sdhost_irq, 0 /*IRQF_SHARED*/,
1986 +                                 mmc_hostname(mmc), host);
1987         if (ret) {
1988                 pr_err("%s: failed to request IRQ %d: %d\n",
1989                        mmc_hostname(mmc), host->irq, ret);
1990 @@ -1776,11 +1864,11 @@ int bcm2835_sdhost_add_host(struct bcm28
1991         mmc_add_host(mmc);
1992
1993         pio_limit_string[0] = '\0';
1994 -       if (host->have_dma && (host->pio_limit > 0))
1995 +       if (host->use_dma && (host->pio_limit > 0))
1996                 sprintf(pio_limit_string, " (>%d)", host->pio_limit);
1997         pr_info("%s: %s loaded - DMA %s%s\n",
1998                 mmc_hostname(mmc), DRIVER_NAME,
1999 -               host->have_dma ? "enabled" : "disabled",
2000 +               host->use_dma ? "enabled" : "disabled",
2001                 pio_limit_string);
2002
2003         return 0;
2004 @@ -1810,8 +1898,11 @@ static int bcm2835_sdhost_probe(struct p
2005         mmc->ops = &bcm2835_sdhost_ops;
2006         host = mmc_priv(mmc);
2007         host->mmc = mmc;
2008 +       host->cmd_quick_poll_retries = 0;
2009         host->pio_timeout = msecs_to_jiffies(500);
2010 +       host->pio_limit = 1;
2011         host->max_delay = 1; /* Warn if over 1ms */
2012 +       host->allow_dma = 1;
2013         spin_lock_init(&host->lock);
2014
2015         iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2016 @@ -1827,13 +1918,12 @@ static int bcm2835_sdhost_probe(struct p
2017                 return -ENODEV;
2018         }
2019         host->bus_addr = be32_to_cpup(addr);
2020 +       log_init(iomem->start - host->bus_addr);
2021         pr_debug(" - ioaddr %lx, iomem->start %lx, bus_addr %lx\n",
2022                  (unsigned long)host->ioaddr,
2023                  (unsigned long)iomem->start,
2024                  (unsigned long)host->bus_addr);
2025
2026 -       host->allow_dma = ALLOW_DMA;
2027 -
2028         if (node) {
2029                 /* Read any custom properties */
2030                 of_property_read_u32(node,
2031 @@ -1845,16 +1935,17 @@ static int bcm2835_sdhost_probe(struct p
2032                 of_property_read_u32(node,
2033                                      "brcm,pio-limit",
2034                                      &host->pio_limit);
2035 -               host->allow_dma = ALLOW_DMA &&
2036 +               host->allow_dma =
2037                         !of_property_read_bool(node, "brcm,force-pio");
2038                 host->debug = of_property_read_bool(node, "brcm,debug");
2039 -               of_property_read_u32(node,
2040 -                                    "brcm,debug-flags",
2041 -                                    &host->debug_flags);
2042         }
2043
2044 -       if (host->debug_flags)
2045 -               dev_err(dev, "debug_flags=%x\n", host->debug_flags);
2046 +       host->dma_chan = NULL;
2047 +       host->dma_desc = NULL;
2048 +
2049 +       /* Formally recognise the other way of disabling DMA */
2050 +       if (host->pio_limit == 0x7fffffff)
2051 +               host->allow_dma = false;
2052
2053         if (host->allow_dma) {
2054                 if (node) {
2055 @@ -1940,15 +2031,12 @@ static int bcm2835_sdhost_remove(struct
2056         return 0;
2057  }
2058
2059 -
2060  static const struct of_device_id bcm2835_sdhost_match[] = {
2061         { .compatible = "brcm,bcm2835-sdhost" },
2062         { }
2063  };
2064  MODULE_DEVICE_TABLE(of, bcm2835_sdhost_match);
2065
2066 -
2067 -
2068  static struct platform_driver bcm2835_sdhost_driver = {
2069         .probe      = bcm2835_sdhost_probe,
2070         .remove     = bcm2835_sdhost_remove,