target/linux/xburst/files/drivers/usb/gadget/jz4740_udc.c

   1 /*
   2  * linux/drivers/usb/gadget/jz4740_udc.c
   3  *
   4  * Ingenic JZ4740 on-chip high speed USB device controller
   5  *
   6  * Copyright (C) 2006 - 2008 Ingenic Semiconductor Inc.
   7  * Author: <jlwei@ingenic.cn>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  */
  14
  15 /*
  16  * This device has ep0, two bulk-in/interrupt-in endpoints, and one bulk-out endpoint.
  17  *
  18  *  - Endpoint numbering is fixed: ep0, ep1in-int, ep2in-bulk, ep1out-bulk.
  19  *  - DMA works with bulk-in (channel 1) and bulk-out (channel 2) endpoints.
  20  */
  21
  22 #include <linux/kernel.h>
  23 #include <linux/module.h>
  24 #include <linux/platform_device.h>
  25 #include <linux/delay.h>
  26 #include <linux/ioport.h>
  27 #include <linux/slab.h>
  28 #include <linux/errno.h>
  29 #include <linux/init.h>
  30 #include <linux/list.h>
  31 #include <linux/interrupt.h>
  32 #include <linux/proc_fs.h>
  33 #include <linux/usb.h>
  34 #include <linux/usb/gadget.h>
  35 #include <linux/clk.h>
  36
  37 #include <asm/byteorder.h>
  38 #include <asm/io.h>
  39 #include <asm/irq.h>
  40 #include <asm/system.h>
  41 #include <asm/mach-jz4740/jz4740.h>
  42 #include <asm/mach-jz4740/clock.h>
  43
  44 #include "jz4740_udc.h"
  45
  46 #define JZ_REG_UDC_FADDR        0x00 /* Function Address 8-bit */
  47 #define JZ_REG_UDC_POWER        0x01 /* Power Managemetn 8-bit */
  48 #define JZ_REG_UDC_INTRIN       0x02 /* Interrupt IN 16-bit */
  49 #define JZ_REG_UDC_INTROUT      0x04 /* Interrupt OUT 16-bit */
  50 #define JZ_REG_UDC_INTRINE      0x06 /* Intr IN enable 16-bit */
  51 #define JZ_REG_UDC_INTROUTE     0x08 /* Intr OUT enable 16-bit */
  52 #define JZ_REG_UDC_INTRUSB      0x0a /* Interrupt USB 8-bit */
  53         #define JZ_REG_UDC_INTRUSBE     0x0b /* Interrupt USB Enable 8-bit */
  54 #define JZ_REG_UDC_FRAME        0x0c /* Frame number 16-bit */
  55 #define JZ_REG_UDC_INDEX        0x0e /* Index register 8-bit */
  56 #define JZ_REG_UDC_TESTMODE     0x0f /* USB test mode 8-bit */
  57
  58 #define JZ_REG_UDC_CSR0         0x12 /* EP0 CSR 8-bit */
  59 #define JZ_REG_UDC_INMAXP       0x10 /* EP1-2 IN Max Pkt Size 16-bit */
  60 #define JZ_REG_UDC_INCSR        0x12 /* EP1-2 IN CSR LSB 8/16bit */
  61 #define JZ_REG_UDC_INCSRH       0x13 /* EP1-2 IN CSR MSB 8-bit */
  62 #define JZ_REG_UDC_OUTMAXP      0x14 /* EP1 OUT Max Pkt Size 16-bit */
  63 #define JZ_REG_UDC_OUTCSR       0x16 /* EP1 OUT CSR LSB 8/16bit */
  64 #define JZ_REG_UDC_OUTCSRH      0x17 /* EP1 OUT CSR MSB 8-bit */
  65 #define JZ_REG_UDC_OUTCOUNT     0x18 /* bytes in EP0/1 OUT FIFO 16-bit */
  66
  67 #define JZ_REG_UDC_EP_FIFO(x)   (4 * (x) + 0x20)
  68
  69 #define JZ_REG_UDC_EPINFO       0x78 /* Endpoint information */
  70 #define JZ_REG_UDC_RAMINFO      0x79 /* RAM information */
  71
  72 #define JZ_REG_UDC_INTR         0x200 /* DMA pending interrupts */
  73 #define JZ_REG_UDC_CNTL1        0x204 /* DMA channel 1 control */
  74 #define JZ_REG_UDC_ADDR1        0x208 /* DMA channel 1 AHB memory addr */
  75 #define JZ_REG_UDC_COUNT1       0x20c /* DMA channel 1 byte count */
  76 #define JZ_REG_UDC_CNTL2        0x214 /* DMA channel 2 control */
  77 #define JZ_REG_UDC_ADDR2        0x218 /* DMA channel 2 AHB memory addr */
  78 #define JZ_REG_UDC_COUNT2       0x21c /* DMA channel 2 byte count */
  79
  80 /* Power register bit masks */
  81 #define USB_POWER_SUSPENDM      0x01
  82 #define USB_POWER_RESUME        0x04
  83 #define USB_POWER_HSMODE        0x10
  84 #define USB_POWER_HSENAB        0x20
  85 #define USB_POWER_SOFTCONN      0x40
  86
  87 /* Interrupt register bit masks */
  88 #define USB_INTR_SUSPEND        0x01
  89 #define USB_INTR_RESUME         0x02
  90 #define USB_INTR_RESET          0x04
  91
  92 #define USB_INTR_EP0            0x0001
  93 #define USB_INTR_INEP1          0x0002
  94 #define USB_INTR_INEP2          0x0004
  95 #define USB_INTR_OUTEP1         0x0002
  96
  97 /* CSR0 bit masks */
  98 #define USB_CSR0_OUTPKTRDY      0x01
  99 #define USB_CSR0_INPKTRDY       0x02
 100 #define USB_CSR0_SENTSTALL      0x04
 101 #define USB_CSR0_DATAEND        0x08
 102 #define USB_CSR0_SETUPEND       0x10
 103 #define USB_CSR0_SENDSTALL      0x20
 104 #define USB_CSR0_SVDOUTPKTRDY   0x40
 105 #define USB_CSR0_SVDSETUPEND    0x80
 106
 107 /* Endpoint CSR register bits */
 108 #define USB_INCSRH_AUTOSET      0x80
 109 #define USB_INCSRH_ISO          0x40
 110 #define USB_INCSRH_MODE         0x20
 111 #define USB_INCSRH_DMAREQENAB   0x10
 112 #define USB_INCSRH_DMAREQMODE   0x04
 113 #define USB_INCSR_CDT           0x40
 114 #define USB_INCSR_SENTSTALL     0x20
 115 #define USB_INCSR_SENDSTALL     0x10
 116 #define USB_INCSR_FF            0x08
 117 #define USB_INCSR_UNDERRUN      0x04
 118 #define USB_INCSR_FFNOTEMPT     0x02
 119 #define USB_INCSR_INPKTRDY      0x01
 120 #define USB_OUTCSRH_AUTOCLR     0x80
 121 #define USB_OUTCSRH_ISO         0x40
 122 #define USB_OUTCSRH_DMAREQENAB  0x20
 123 #define USB_OUTCSRH_DNYT        0x10
 124 #define USB_OUTCSRH_DMAREQMODE  0x08
 125 #define USB_OUTCSR_CDT          0x80
 126 #define USB_OUTCSR_SENTSTALL    0x40
 127 #define USB_OUTCSR_SENDSTALL    0x20
 128 #define USB_OUTCSR_FF           0x10
 129 #define USB_OUTCSR_DATAERR      0x08
 130 #define USB_OUTCSR_OVERRUN      0x04
 131 #define USB_OUTCSR_FFFULL       0x02
 132 #define USB_OUTCSR_OUTPKTRDY    0x01
 133
 134 /* Testmode register bits */
 135 #define USB_TEST_SE0NAK         0x01
 136 #define USB_TEST_J              0x02
 137 #define USB_TEST_K              0x04
 138 #define USB_TEST_PACKET         0x08
 139
 140 /* DMA control bits */
 141 #define USB_CNTL_ENA            0x01
 142 #define USB_CNTL_DIR_IN         0x02
 143 #define USB_CNTL_MODE_1         0x04
 144 #define USB_CNTL_INTR_EN        0x08
 145 #define USB_CNTL_EP(n)          ((n) << 4)
 146 #define USB_CNTL_BURST_0        (0 << 9)
 147 #define USB_CNTL_BURST_4        (1 << 9)
 148 #define USB_CNTL_BURST_8        (2 << 9)
 149 #define USB_CNTL_BURST_16       (3 << 9)
 150
 151
 152 #ifndef DEBUG
 153 # define DEBUG(fmt,args...) do {} while(0)
 154 #endif
 155 #ifndef DEBUG_EP0
 156 # define NO_STATES
 157 # define DEBUG_EP0(fmt,args...) do {} while(0)
 158 #endif
 159 #ifndef DEBUG_SETUP
 160 # define DEBUG_SETUP(fmt,args...) do {} while(0)
 161 #endif
 162
 163 static unsigned int use_dma = 0;   /* 1: use DMA, 0: use PIO */
 164
 165 module_param(use_dma, int, 0);
 166 MODULE_PARM_DESC(use_dma, "DMA mode enable flag");
 167
 168 struct jz4740_udc *the_controller;
 169
 170 /*
 171  * Local declarations.
 172  */
 173 static void jz4740_ep0_kick(struct jz4740_udc *dev, struct jz4740_ep *ep);
 174 static void jz4740_handle_ep0(struct jz4740_udc *dev, uint32_t intr);
 175
 176 static void done(struct jz4740_ep *ep, struct jz4740_request *req,
 177                  int status);
 178 static void pio_irq_enable(struct jz4740_ep *ep);
 179 static void pio_irq_disable(struct jz4740_ep *ep);
 180 static void stop_activity(struct jz4740_udc *dev,
 181                           struct usb_gadget_driver *driver);
 182 static void nuke(struct jz4740_ep *ep, int status);
 183 static void flush(struct jz4740_ep *ep);
 184 static void udc_set_address(struct jz4740_udc *dev, unsigned char address);
 185
 186 /*-------------------------------------------------------------------------*/
 187
 188 /* inline functions of register read/write/set/clear  */
 189
 190 static inline uint8_t usb_readb(struct jz4740_udc *udc, size_t reg)
 191 {
 192         return readb(udc->base + reg);
 193 }
 194
 195 static inline uint16_t usb_readw(struct jz4740_udc *udc, size_t reg)
 196 {
 197         return readw(udc->base + reg);
 198 }
 199
 200 static inline uint32_t usb_readl(struct jz4740_udc *udc, size_t reg)
 201 {
 202         return readl(udc->base + reg);
 203 }
 204
 205 static inline void usb_writeb(struct jz4740_udc *udc, size_t reg, uint8_t val)
 206 {
 207         writeb(val, udc->base + reg);
 208 }
 209
 210 static inline void usb_writew(struct jz4740_udc *udc, size_t reg, uint16_t val)
 211 {
 212         writew(val, udc->base + reg);
 213 }
 214
 215 static inline void usb_writel(struct jz4740_udc *udc, size_t reg, uint32_t val)
 216 {
 217         writel(val, udc->base + reg);
 218 }
 219
 220 static inline void usb_setb(struct jz4740_udc *udc, size_t reg, uint8_t mask)
 221 {
 222         usb_writeb(udc, reg, usb_readb(udc, reg) | mask);
 223 }
 224
 225 static inline void usb_setw(struct jz4740_udc *udc, size_t reg, uint8_t mask)
 226 {
 227         usb_writew(udc, reg, usb_readw(udc, reg) | mask);
 228 }
 229
 230 static inline void usb_setl(struct jz4740_udc *udc, size_t reg, uint32_t mask)
 231 {
 232         usb_writel(udc, reg, usb_readl(udc, reg) | mask);
 233 }
 234
 235 static inline void usb_clearb(struct jz4740_udc *udc, size_t reg, uint8_t mask)
 236 {
 237         usb_writeb(udc, reg, usb_readb(udc, reg) & ~mask);
 238 }
 239
 240 static inline void usb_clearw(struct jz4740_udc *udc, size_t reg, uint16_t mask)
 241 {
 242         usb_writew(udc, reg, usb_readw(udc, reg) & ~mask);
 243 }
 244
 245 static inline void usb_clearl(struct jz4740_udc *udc, size_t reg, uint32_t mask)
 246 {
 247         usb_writel(udc, reg, usb_readl(udc, reg) & ~mask);
 248 }
 249
 250 /*-------------------------------------------------------------------------*/
 251
 252 static inline void jz_udc_set_index(struct jz4740_udc *udc, uint8_t index)
 253 {
 254         usb_writeb(udc, JZ_REG_UDC_INDEX, index);
 255 }
 256
 257 static inline void jz_udc_select_ep(struct jz4740_ep *ep)
 258 {
 259         jz_udc_set_index(ep->dev, ep_index(ep));
 260 }
 261
 262 static inline int write_packet(struct jz4740_ep *ep,
 263                                    struct jz4740_request *req, int max)
 264 {
 265         uint8_t *buf;
 266         int length, nlong, nbyte;
 267         DEBUG("%s:%s[%d]\n", __FILE__, __func__, __LINE__);
 268
 269         buf = req->req.buf + req->req.actual;
 270         prefetch(buf);
 271
 272         length = req->req.length - req->req.actual;
 273         length = min(length, max);
 274         req->req.actual += length;
 275
 276         DEBUG("Write %d (max %d), fifo %x\n", length, max, ep->fifo);
 277
 278         nlong = length >> 2;
 279         nbyte = length & 0x3;
 280         while (nlong--) {
 281                 usb_writel(ep->dev, ep->fifo, *((uint32_t *)buf));
 282                 buf += 4;
 283         }
 284         while (nbyte--) {
 285                 usb_writeb(ep->dev, ep->fifo, *buf++);
 286         }
 287
 288         return length;
 289 }
 290
 291 static inline int read_packet(struct jz4740_ep *ep,
 292                                   struct jz4740_request *req, int count)
 293 {
 294         uint8_t *buf;
 295         int length, nlong, nbyte;
 296         DEBUG("%s:%s[%d]\n", __FILE__, __func__, __LINE__);
 297
 298         buf = req->req.buf + req->req.actual;
 299         prefetchw(buf);
 300
 301         length = req->req.length - req->req.actual;
 302         length = min(length, count);
 303         req->req.actual += length;
 304
 305         DEBUG("Read %d, fifo %x\n", length, ep->fifo);
 306
 307         nlong = length >> 2;
 308         nbyte = length & 0x3;
 309         while (nlong--) {
 310                 *((uint32_t *)buf) = usb_readl(ep->dev, ep->fifo);
 311                 buf += 4;
 312         }
 313         while (nbyte--) {
 314                 *buf++ = usb_readb(ep->dev, ep->fifo);
 315         }
 316
 317         return length;
 318 }
 319
 320 /*-------------------------------------------------------------------------*/
 321
 322 /*
 323  *      udc_disable - disable USB device controller
 324  */
 325 static void udc_disable(struct jz4740_udc *dev)
 326 {
 327         DEBUG("%s:%s[%d]\n", __FILE__, __func__, __LINE__);
 328
 329         udc_set_address(dev, 0);
 330
 331         /* Disable interrupts */
 332         usb_writew(dev, JZ_REG_UDC_INTRINE, 0);
 333         usb_writew(dev, JZ_REG_UDC_INTROUTE, 0);
 334         usb_writeb(dev, JZ_REG_UDC_INTRUSBE, 0);
 335
 336         /* Disable DMA */
 337         usb_writel(dev, JZ_REG_UDC_CNTL1, 0);
 338         usb_writel(dev, JZ_REG_UDC_CNTL2, 0);
 339
 340         /* Disconnect from usb */
 341         usb_clearb(dev, JZ_REG_UDC_POWER, USB_POWER_SOFTCONN);
 342
 343         /* Disable the USB PHY */
 344         clk_disable(dev->clk);
 345
 346         dev->ep0state = WAIT_FOR_SETUP;
 347         dev->gadget.speed = USB_SPEED_UNKNOWN;
 348
 349         return;
 350 }
 351
 352 /*
 353  *      udc_reinit - initialize software state
 354  */
 355 static void udc_reinit(struct jz4740_udc *dev)
 356 {
 357         int i;
 358         DEBUG("%s:%s[%d]\n", __FILE__, __func__, __LINE__);
 359
 360         /* device/ep0 records init */
 361         INIT_LIST_HEAD(&dev->gadget.ep_list);
 362         INIT_LIST_HEAD(&dev->gadget.ep0->ep_list);
 363         dev->ep0state = WAIT_FOR_SETUP;
 364
 365         for (i = 0; i < UDC_MAX_ENDPOINTS; i++) {
 366                 struct jz4740_ep *ep = &dev->ep[i];
 367
 368                 if (i != 0)
 369                         list_add_tail(&ep->ep.ep_list, &dev->gadget.ep_list);
 370
 371                 INIT_LIST_HEAD(&ep->queue);
 372                 ep->desc = 0;
 373                 ep->stopped = 0;
 374                 ep->pio_irqs = 0;
 375         }
 376 }
 377
 378 /* until it's enabled, this UDC should be completely invisible
 379  * to any USB host.
 380  */
 381 static void udc_enable(struct jz4740_udc *dev)
 382 {
 383         int i;
 384         DEBUG("%s:%s[%d]\n", __FILE__, __func__, __LINE__);
 385
 386         /* UDC state is incorrect - Added by River */
 387         if (dev->state != UDC_STATE_ENABLE) {
 388                 return;
 389         }
 390
 391         dev->gadget.speed = USB_SPEED_UNKNOWN;
 392
 393         /* Flush FIFO for each */
 394         for (i = 0; i < UDC_MAX_ENDPOINTS; i++) {
 395                 struct jz4740_ep *ep = &dev->ep[i];
 396
 397                 jz_udc_set_index(dev, ep_index(ep));
 398                 flush(ep);
 399         }
 400
 401         /* Set this bit to allow the UDC entering low-power mode when
 402          * there are no actions on the USB bus.
 403          * UDC still works during this bit was set.
 404          */
 405         jz4740_clock_udc_enable_auto_suspend();
 406
 407         /* Enable the USB PHY */
 408         clk_enable(dev->clk);
 409
 410         /* Disable interrupts */
 411 /*      usb_writew(dev, JZ_REG_UDC_INTRINE, 0);
 412         usb_writew(dev, JZ_REG_UDC_INTROUTE, 0);
 413         usb_writeb(dev, JZ_REG_UDC_INTRUSBE, 0);*/
 414
 415         /* Enable interrupts */
 416         usb_setw(dev, JZ_REG_UDC_INTRINE, USB_INTR_EP0);
 417         usb_setb(dev, JZ_REG_UDC_INTRUSBE, USB_INTR_RESET);
 418         /* Don't enable rest of the interrupts */
 419         /* usb_setw(dev, JZ_REG_UDC_INTRINE, USB_INTR_INEP1 | USB_INTR_INEP2);
 420            usb_setw(dev, JZ_REG_UDC_INTROUTE, USB_INTR_OUTEP1); */
 421
 422         /* Enable SUSPEND */
 423         /* usb_setb(dev, JZ_REG_UDC_POWER, USB_POWER_SUSPENDM); */
 424
 425         /* Enable HS Mode */
 426         usb_setb(dev, JZ_REG_UDC_POWER, USB_POWER_HSENAB);
 427
 428         /* Let host detect UDC:
 429          * Software must write a 1 to the PMR:USB_POWER_SOFTCONN bit to turn this
 430          * transistor on and pull the USBDP pin HIGH.
 431          */
 432         usb_setb(dev, JZ_REG_UDC_POWER, USB_POWER_SOFTCONN);
 433
 434         return;
 435 }
 436
 437 /*-------------------------------------------------------------------------*/
 438
 439 /* keeping it simple:
 440  * - one bus driver, initted first;
 441  * - one function driver, initted second
 442  */
 443
 444 /*
 445  * Register entry point for the peripheral controller driver.
 446  */
 447
 448 int usb_gadget_register_driver(struct usb_gadget_driver *driver)
 449 {
 450         struct jz4740_udc *dev = the_controller;
 451         int retval;
 452
 453         if (!driver || !driver->bind) {
 454                 return -EINVAL;
 455         }
 456
 457         if (!dev) {
 458                 return -ENODEV;
 459         }
 460
 461         if (dev->driver) {
 462                 return -EBUSY;
 463         }
 464
 465         /* hook up the driver */
 466         dev->driver = driver;
 467         dev->gadget.dev.driver = &driver->driver;
 468
 469         retval = driver->bind(&dev->gadget);
 470         if (retval) {
 471                 DEBUG("%s: bind to driver %s --> error %d\n", dev->gadget.name,
 472                             driver->driver.name, retval);
 473                 dev->driver = 0;
 474                 return retval;
 475         }
 476
 477         /* then enable host detection and ep0; and we're ready
 478          * for set_configuration as well as eventual disconnect.
 479          */
 480         udc_enable(dev);
 481
 482         DEBUG("%s: registered gadget driver '%s'\n", dev->gadget.name,
 483               driver->driver.name);
 484
 485         return 0;
 486 }
 487
 488 EXPORT_SYMBOL(usb_gadget_register_driver);
 489
 490 static void stop_activity(struct jz4740_udc *dev,
 491                           struct usb_gadget_driver *driver)
 492 {
 493         int i;
 494
 495         DEBUG("%s:%s[%d]\n", __FILE__, __func__, __LINE__);
 496
 497         /* don't disconnect drivers more than once */
 498         if (dev->gadget.speed == USB_SPEED_UNKNOWN)
 499                 driver = 0;
 500         dev->gadget.speed = USB_SPEED_UNKNOWN;
 501
 502         /* prevent new request submissions, kill any outstanding requests  */
 503         for (i = 0; i < UDC_MAX_ENDPOINTS; i++) {
 504                 struct jz4740_ep *ep = &dev->ep[i];
 505
 506                 ep->stopped = 1;
 507
 508                 jz_udc_set_index(dev, ep_index(ep));
 509                 nuke(ep, -ESHUTDOWN);
 510         }
 511
 512         /* report disconnect; the driver is already quiesced */
 513         if (driver) {
 514                 spin_unlock(&dev->lock);
 515                 driver->disconnect(&dev->gadget);
 516                 spin_lock(&dev->lock);
 517         }
 518
 519         /* re-init driver-visible data structures */
 520         udc_reinit(dev);
 521 }
 522
 523
 524 /*
 525  * Unregister entry point for the peripheral controller driver.
 526  */
 527 int usb_gadget_unregister_driver(struct usb_gadget_driver *driver)
 528 {
 529         struct jz4740_udc *dev = the_controller;
 530         unsigned long flags;
 531         DEBUG("%s:%s[%d]\n", __FILE__, __func__, __LINE__);
 532
 533         if (!dev)
 534                 return -ENODEV;
 535         if (!driver || driver != dev->driver)
 536                 return -EINVAL;
 537         if (!driver->unbind)
 538                 return -EBUSY;
 539
 540         spin_lock_irqsave(&dev->lock, flags);
 541         dev->driver = 0;
 542         stop_activity(dev, driver);
 543         spin_unlock_irqrestore(&dev->lock, flags);
 544
 545         driver->unbind(&dev->gadget);
 546
 547         udc_disable(dev);
 548
 549         DEBUG("unregistered driver '%s'\n", driver->driver.name);
 550
 551         return 0;
 552 }
 553
 554 EXPORT_SYMBOL(usb_gadget_unregister_driver);
 555
 556 /*-------------------------------------------------------------------------*/
 557
 558 /*
 559  * Starting DMA using mode 1
 560  */
 561 static void kick_dma(struct jz4740_ep *ep, struct jz4740_request *req)
 562 {
 563         struct jz4740_udc *dev = ep->dev;
 564         uint32_t count = req->req.length;
 565         uint32_t physaddr = virt_to_phys((void *)req->req.buf);
 566
 567         DEBUG("%s:%s[%d]\n", __FILE__, __func__, __LINE__);
 568
 569         jz_udc_select_ep(ep);
 570
 571         if (ep_is_in(ep)) { /* Bulk-IN transfer using DMA channel 1 */
 572                 ep->reg_addr = JZ_REG_UDC_ADDR1;
 573
 574                 dma_cache_wback_inv((unsigned long)req->req.buf, count);
 575
 576                 pio_irq_enable(ep);
 577
 578                 usb_writeb(dev, JZ_REG_UDC_INCSRH,
 579                            USB_INCSRH_DMAREQENAB | USB_INCSRH_AUTOSET | USB_INCSRH_DMAREQMODE);
 580
 581                 usb_writel(dev, JZ_REG_UDC_ADDR1, physaddr);
 582                 usb_writel(dev, JZ_REG_UDC_COUNT1, count);
 583                 usb_writel(dev, JZ_REG_UDC_CNTL1, USB_CNTL_ENA | USB_CNTL_DIR_IN | USB_CNTL_MODE_1 |
 584                            USB_CNTL_INTR_EN | USB_CNTL_BURST_16 | USB_CNTL_EP(ep_index(ep)));
 585         }
 586         else { /* Bulk-OUT transfer using DMA channel 2 */
 587                 ep->reg_addr = JZ_REG_UDC_ADDR2;
 588
 589                 dma_cache_wback_inv((unsigned long)req->req.buf, count);
 590
 591                 pio_irq_enable(ep);
 592
 593                 usb_setb(dev, JZ_REG_UDC_OUTCSRH,
 594                          USB_OUTCSRH_DMAREQENAB | USB_OUTCSRH_AUTOCLR | USB_OUTCSRH_DMAREQMODE);
 595
 596                 usb_writel(dev, JZ_REG_UDC_ADDR2, physaddr);
 597                 usb_writel(dev, JZ_REG_UDC_COUNT2, count);
 598                 usb_writel(dev, JZ_REG_UDC_CNTL2, USB_CNTL_ENA | USB_CNTL_MODE_1 |
 599                            USB_CNTL_INTR_EN | USB_CNTL_BURST_16 | USB_CNTL_EP(ep_index(ep)));
 600         }
 601 }
 602
 603 /*-------------------------------------------------------------------------*/
 604
 605 /** Write request to FIFO (max write == maxp size)
 606  *  Return:  0 = still running, 1 = completed, negative = errno
 607  *  NOTE: INDEX register must be set for EP
 608  */
 609 static int write_fifo(struct jz4740_ep *ep, struct jz4740_request *req)
 610 {
 611         struct jz4740_udc *dev = ep->dev;
 612         uint32_t max, csr;
 613         uint32_t physaddr = virt_to_phys((void *)req->req.buf);
 614
 615         DEBUG("%s:%s[%d]\n", __FILE__, __func__, __LINE__);
 616         max = le16_to_cpu(ep->desc->wMaxPacketSize);
 617
 618         if (use_dma) {
 619                 uint32_t dma_count;
 620
 621                 /* DMA interrupt generated due to the last packet loaded into the FIFO */
 622
 623                 dma_count = usb_readl(dev, ep->reg_addr) - physaddr;
 624                 req->req.actual += dma_count;
 625
 626                 if (dma_count % max) {
 627                         /* If the last packet is less than MAXP, set INPKTRDY manually */
 628                         usb_setb(dev, ep->csr, USB_INCSR_INPKTRDY);
 629                 }
 630
 631                 done(ep, req, 0);
 632                 if (list_empty(&ep->queue)) {
 633                         pio_irq_disable(ep);
 634                         return 1;
 635                 }
 636                 else {
 637                         /* advance the request queue */
 638                         req = list_entry(ep->queue.next, struct jz4740_request, queue);
 639                         kick_dma(ep, req);
 640                         return 0;
 641                 }
 642         }
 643
 644         /*
 645          * PIO mode handling starts here ...
 646          */
 647
 648         csr = usb_readb(dev, ep->csr);
 649
 650         if (!(csr & USB_INCSR_FFNOTEMPT)) {
 651                 unsigned count;
 652                 int is_last, is_short;
 653
 654                 count = write_packet(ep, req, max);
 655                 usb_setb(dev, ep->csr, USB_INCSR_INPKTRDY);
 656
 657                 /* last packet is usually short (or a zlp) */
 658                 if (unlikely(count != max))
 659                         is_last = is_short = 1;
 660                 else {
 661                         if (likely(req->req.length != req->req.actual)
 662                             || req->req.zero)
 663                                 is_last = 0;
 664                         else
 665                                 is_last = 1;
 666                         /* interrupt/iso maxpacket may not fill the fifo */
 667                         is_short = unlikely(max < ep_maxpacket(ep));
 668                 }
 669
 670                 DEBUG("%s: wrote %s %d bytes%s%s %d left %p\n", __FUNCTION__,
 671                       ep->ep.name, count,
 672                       is_last ? "/L" : "", is_short ? "/S" : "",
 673                       req->req.length - req->req.actual, req);
 674
 675                 /* requests complete when all IN data is in the FIFO */
 676                 if (is_last) {
 677                         done(ep, req, 0);
 678                         if (list_empty(&ep->queue)) {
 679                                 pio_irq_disable(ep);
 680                         }
 681                         return 1;
 682                 }
 683         } else {
 684                 DEBUG("Hmm.. %d ep FIFO is not empty!\n", ep_index(ep));
 685         }
 686
 687         return 0;
 688 }
 689
 690 /** Read to request from FIFO (max read == bytes in fifo)
 691  *  Return:  0 = still running, 1 = completed, negative = errno
 692  *  NOTE: INDEX register must be set for EP
 693  */
 694 static int read_fifo(struct jz4740_ep *ep, struct jz4740_request *req)
 695 {
 696         struct jz4740_udc *dev = ep->dev;
 697         uint32_t csr;
 698         unsigned count, is_short;
 699         uint32_t physaddr = virt_to_phys((void *)req->req.buf);
 700
 701         if (use_dma) {
 702                 uint32_t dma_count;
 703
 704                 /* DMA interrupt generated due to a packet less than MAXP loaded into the FIFO */
 705
 706                 dma_count = usb_readl(dev, ep->reg_addr) - physaddr;
 707                 req->req.actual += dma_count;
 708
 709                 /* Disable interrupt and DMA */
 710                 pio_irq_disable(ep);
 711                 usb_writel(dev, JZ_REG_UDC_CNTL2, 0);
 712
 713                 /* Read all bytes from this packet */
 714                 count = usb_readw(dev, JZ_REG_UDC_OUTCOUNT);
 715                 count = read_packet(ep, req, count);
 716
 717                 if (count) {
 718                         /* If the last packet is greater than zero, clear OUTPKTRDY manually */
 719                         usb_clearb(dev, ep->csr, USB_OUTCSR_OUTPKTRDY);
 720                 }
 721                 done(ep, req, 0);
 722
 723                 if (!list_empty(&ep->queue)) {
 724                         /* advance the request queue */
 725                         req = list_entry(ep->queue.next, struct jz4740_request, queue);
 726                         kick_dma(ep, req);
 727                 }
 728
 729                 return 1;
 730         }
 731
 732         /*
 733          * PIO mode handling starts here ...
 734          */
 735
 736         /* make sure there's a packet in the FIFO. */
 737         csr = usb_readb(dev, ep->csr);
 738         if (!(csr & USB_OUTCSR_OUTPKTRDY)) {
 739                 DEBUG("%s: Packet NOT ready!\n", __FUNCTION__);
 740                 return -EINVAL;
 741         }
 742
 743         /* read all bytes from this packet */
 744         count = usb_readw(dev, JZ_REG_UDC_OUTCOUNT);
 745
 746         is_short = (count < ep->ep.maxpacket);
 747
 748         count = read_packet(ep, req, count);
 749
 750         DEBUG("read %s %02x, %d bytes%s req %p %d/%d\n",
 751               ep->ep.name, csr, count,
 752               is_short ? "/S" : "", req, req->req.actual, req->req.length);
 753
 754         /* Clear OutPktRdy */
 755         usb_clearb(dev, ep->csr, USB_OUTCSR_OUTPKTRDY);
 756
 757         /* completion */
 758         if (is_short || req->req.actual == req->req.length) {
 759                 done(ep, req, 0);
 760
 761                 if (list_empty(&ep->queue))
 762                         pio_irq_disable(ep);
 763                 return 1;
 764         }
 765
 766         /* finished that packet.  the next one may be waiting... */
 767         return 0;
 768 }
 769
 770 /*
 771  *      done - retire a request; caller blocked irqs
 772  *  INDEX register is preserved to keep same
 773  */
 774 static void done(struct jz4740_ep *ep, struct jz4740_request *req, int status)
 775 {
 776         unsigned int stopped = ep->stopped;
 777         unsigned long flags;
 778         uint32_t index;
 779
 780         DEBUG("%s, %p\n", __FUNCTION__, ep);
 781         list_del_init(&req->queue);
 782
 783         if (likely(req->req.status == -EINPROGRESS))
 784                 req->req.status = status;
 785         else
 786                 status = req->req.status;
 787
 788         if (status && status != -ESHUTDOWN)
 789                 DEBUG("complete %s req %p stat %d len %u/%u\n",
 790                       ep->ep.name, &req->req, status,
 791                       req->req.actual, req->req.length);
 792
 793         /* don't modify queue heads during completion callback */
 794         ep->stopped = 1;
 795         /* Read current index (completion may modify it) */
 796         spin_lock_irqsave(&ep->dev->lock, flags);
 797         index = usb_readb(ep->dev, JZ_REG_UDC_INDEX);
 798
 799         req->req.complete(&ep->ep, &req->req);
 800
 801         /* Restore index */
 802         jz_udc_set_index(ep->dev, index);
 803         spin_unlock_irqrestore(&ep->dev->lock, flags);
 804         ep->stopped = stopped;
 805 }
 806
 807 /** Enable EP interrupt */
 808 static void pio_irq_enable(struct jz4740_ep *ep)
 809 {
 810         uint8_t index = ep_index(ep);
 811         struct jz4740_udc *dev = ep->dev;
 812         DEBUG("%s: EP%d %s\n", __FUNCTION__, ep_index(ep), ep_is_in(ep) ? "IN": "OUT");
 813
 814         if (ep_is_in(ep)) {
 815                 switch (index) {
 816                 case 1:
 817                 case 2:
 818                         usb_setw(dev, JZ_REG_UDC_INTRINE, BIT(index));
 819                         dev->in_mask |= BIT(index);
 820                         break;
 821                 default:
 822                         DEBUG("Unknown endpoint: %d\n", index);
 823                         break;
 824                 }
 825         }
 826         else {
 827                 switch (index) {
 828                 case 1:
 829                         usb_setw(dev, JZ_REG_UDC_INTROUTE, BIT(index));
 830                         dev->out_mask |= BIT(index);
 831                         break;
 832                 default:
 833                         DEBUG("Unknown endpoint: %d\n", index);
 834                         break;
 835                 }
 836         }
 837 }
 838
 839 /** Disable EP interrupt */
 840 static void pio_irq_disable(struct jz4740_ep *ep)
 841 {
 842         uint8_t index = ep_index(ep);
 843         struct jz4740_udc *dev = ep->dev;
 844
 845         DEBUG("%s: EP%d %s\n", __FUNCTION__, ep_index(ep), ep_is_in(ep) ? "IN": "OUT");
 846
 847         if (ep_is_in(ep)) {
 848                 switch (ep_index(ep)) {
 849                 case 1:
 850                 case 2:
 851                         usb_clearw(ep->dev, JZ_REG_UDC_INTRINE, BIT(index));
 852                         dev->in_mask &= ~BIT(index);
 853                         break;
 854                 default:
 855                         DEBUG("Unknown endpoint: %d\n", index);
 856                         break;
 857                 }
 858         }
 859         else {
 860                 switch (ep_index(ep)) {
 861                 case 1:
 862                         usb_clearw(ep->dev, JZ_REG_UDC_INTROUTE, BIT(index));
 863                         dev->out_mask &= ~BIT(index);
 864                         break;
 865                 default:
 866                         DEBUG("Unknown endpoint: %d\n", index);
 867                         break;
 868             }
 869         }
 870 }
 871
 872 /*
 873  *      nuke - dequeue ALL requests
 874  */
 875 static void nuke(struct jz4740_ep *ep, int status)
 876 {
 877         struct jz4740_request *req;
 878
 879         DEBUG("%s, %p\n", __FUNCTION__, ep);
 880
 881         /* Flush FIFO */
 882         flush(ep);
 883
 884         /* called with irqs blocked */
 885         while (!list_empty(&ep->queue)) {
 886                 req = list_entry(ep->queue.next, struct jz4740_request, queue);
 887                 done(ep, req, status);
 888         }
 889
 890         /* Disable IRQ if EP is enabled (has descriptor) */
 891         if (ep->desc)
 892                 pio_irq_disable(ep);
 893 }
 894
 895 /** Flush EP FIFO
 896  * NOTE: INDEX register must be set before this call
 897  */
 898 static void flush(struct jz4740_ep *ep)
 899 {
 900         DEBUG("%s: %s\n", __FUNCTION__, ep->ep.name);
 901
 902         switch (ep->type) {
 903         case ep_bulk_in:
 904         case ep_interrupt:
 905                 usb_setb(ep->dev, ep->csr, USB_INCSR_FF);
 906                 break;
 907         case ep_bulk_out:
 908                 usb_setb(ep->dev, ep->csr, USB_OUTCSR_FF);
 909                 break;
 910         case ep_control:
 911                 break;
 912         }
 913 }
 914
 915 /**
 916  * jz4740_in_epn - handle IN interrupt
 917  */
 918 static void jz4740_in_epn(struct jz4740_udc *dev, uint32_t ep_idx, uint32_t intr)
 919 {
 920         uint32_t csr;
 921         struct jz4740_ep *ep = &dev->ep[ep_idx + 1];
 922         struct jz4740_request *req;
 923         DEBUG("%s:%s[%d]\n", __FILE__, __func__, __LINE__);
 924
 925         jz_udc_set_index(dev, ep_index(ep));
 926
 927         csr = usb_readb(dev, ep->csr);
 928         DEBUG("%s: %d, csr %x\n", __FUNCTION__, ep_idx, csr);
 929
 930         if (csr & USB_INCSR_SENTSTALL) {
 931                 DEBUG("USB_INCSR_SENTSTALL\n");
 932                 usb_clearb(dev, ep->csr, USB_INCSR_SENTSTALL);
 933                 return;
 934         }
 935
 936         if (!ep->desc) {
 937                 DEBUG("%s: NO EP DESC\n", __FUNCTION__);
 938                 return;
 939         }
 940
 941         if (list_empty(&ep->queue))
 942                 req = 0;
 943         else
 944                 req = list_entry(ep->queue.next, struct jz4740_request, queue);
 945
 946         DEBUG("req: %p\n", req);
 947
 948         if (!req)
 949                 return;
 950
 951         write_fifo(ep, req);
 952 }
 953
 954 /*
 955  * Bulk OUT (recv)
 956  */
 957 static void jz4740_out_epn(struct jz4740_udc *dev, uint32_t ep_idx, uint32_t intr)
 958 {
 959         struct jz4740_ep *ep = &dev->ep[ep_idx];
 960         struct jz4740_request *req;
 961
 962         DEBUG("%s: %d\n", __FUNCTION__, ep_idx);
 963
 964         jz_udc_set_index(dev, ep_index(ep));
 965         if (ep->desc) {
 966                 uint32_t csr;
 967
 968                 if (use_dma) {
 969                         /* DMA starts here ... */
 970                         if (list_empty(&ep->queue))
 971                                 req = 0;
 972                         else
 973                                 req = list_entry(ep->queue.next, struct jz4740_request, queue);
 974
 975                         if (req)
 976                                 read_fifo(ep, req);
 977                         return;
 978                 }
 979
 980                 /*
 981                  * PIO mode starts here ...
 982                  */
 983
 984                 while ((csr = usb_readb(dev, ep->csr)) &
 985                        (USB_OUTCSR_OUTPKTRDY | USB_OUTCSR_SENTSTALL)) {
 986                         DEBUG("%s: %x\n", __FUNCTION__, csr);
 987
 988                         if (csr & USB_OUTCSR_SENTSTALL) {
 989                                 DEBUG("%s: stall sent, flush fifo\n",
 990                                       __FUNCTION__);
 991                                 /* usb_set(USB_OUT_CSR1_FIFO_FLUSH, ep->csr1); */
 992                                 flush(ep);
 993                         } else if (csr & USB_OUTCSR_OUTPKTRDY) {
 994                                 if (list_empty(&ep->queue))
 995                                         req = 0;
 996                                 else
 997                                         req =
 998                                                 list_entry(ep->queue.next,
 999                                                            struct jz4740_request,
1000                                                            queue);
1001
1002                                 if (!req) {
1003                                         DEBUG("%s: NULL REQ %d\n",
1004                                               __FUNCTION__, ep_idx);
1005                                         break;
1006                                 } else {
1007                                         read_fifo(ep, req);
1008                                 }
1009                         }
1010                 }
1011         } else {
1012                 /* Throw packet away.. */
1013                 DEBUG("%s: ep %p ep_indx %d No descriptor?!?\n", __FUNCTION__, ep, ep_idx);
1014                 flush(ep);
1015         }
1016 }
1017
1018 /** Halt specific EP
1019  *  Return 0 if success
1020  *  NOTE: Sets INDEX register to EP !
1021  */
1022 static int jz4740_set_halt(struct usb_ep *_ep, int value)
1023 {
1024         struct jz4740_udc *dev;
1025         struct jz4740_ep *ep;
1026         unsigned long flags;
1027
1028         DEBUG("%s:%s[%d]\n", __FILE__, __func__, __LINE__);
1029
1030         ep = container_of(_ep, struct jz4740_ep, ep);
1031         if (unlikely(!_ep || (!ep->desc && ep->type != ep_control))) {
1032                 DEBUG("%s, bad ep\n", __FUNCTION__);
1033                 return -EINVAL;
1034         }
1035
1036         dev = ep->dev;
1037
1038         spin_lock_irqsave(&dev->lock, flags);
1039
1040         jz_udc_select_ep(ep);
1041
1042         DEBUG("%s, ep %d, val %d\n", __FUNCTION__, ep_index(ep), value);
1043
1044         if (ep_index(ep) == 0) {
1045                 /* EP0 */
1046                 usb_setb(dev, JZ_REG_UDC_CSR0, USB_CSR0_SENDSTALL);
1047         } else if (ep_is_in(ep)) {
1048                 uint32_t csr = usb_readb(dev, ep->csr);
1049                 if (value && ((csr & USB_INCSR_FFNOTEMPT)
1050                               || !list_empty(&ep->queue))) {
1051                         /*
1052                          * Attempts to halt IN endpoints will fail (returning -EAGAIN)
1053                          * if any transfer requests are still queued, or if the controller
1054                          * FIFO still holds bytes that the host hasn\92t collected.
1055                          */
1056                         spin_unlock_irqrestore(&dev->lock, flags);
1057                         DEBUG
1058                             ("Attempt to halt IN endpoint failed (returning -EAGAIN) %d %d\n",
1059                              (csr & USB_INCSR_FFNOTEMPT),
1060                              !list_empty(&ep->queue));
1061                         return -EAGAIN;
1062                 }
1063                 flush(ep);
1064                 if (value) {
1065                         usb_setb(dev, ep->csr, USB_INCSR_SENDSTALL);
1066                 }
1067                 else {
1068                         usb_clearb(dev, ep->csr, USB_INCSR_SENDSTALL);
1069                         usb_setb(dev, ep->csr, USB_INCSR_CDT);
1070                 }
1071         } else {
1072
1073                 flush(ep);
1074                 if (value) {
1075                         usb_setb(dev, ep->csr, USB_OUTCSR_SENDSTALL);
1076                 }
1077                 else {
1078                         usb_clearb(dev, ep->csr, USB_OUTCSR_SENDSTALL);
1079                         usb_setb(dev, ep->csr, USB_OUTCSR_CDT);
1080                 }
1081         }
1082
1083         if (value) {
1084                 ep->stopped = 1;
1085         } else {
1086                 ep->stopped = 0;
1087         }
1088
1089         spin_unlock_irqrestore(&dev->lock, flags);
1090
1091         DEBUG("%s %s halted\n", _ep->name, value == 0 ? "NOT" : "IS");
1092
1093         return 0;
1094 }
1095
1096
1097 static int jz4740_ep_enable(struct usb_ep *_ep,
1098                             const struct usb_endpoint_descriptor *desc)
1099 {
1100         struct jz4740_ep *ep;
1101         struct jz4740_udc *dev;
1102         unsigned long flags;
1103         uint32_t max, csrh = 0;
1104
1105         DEBUG("%s: trying to enable %s\n", __FUNCTION__, _ep->name);
1106
1107         if (!_ep || !desc)
1108                 return -EINVAL;
1109
1110         ep = container_of(_ep, struct jz4740_ep, ep);
1111         if (ep->desc || ep->type == ep_control
1112             || desc->bDescriptorType != USB_DT_ENDPOINT
1113             || ep->bEndpointAddress != desc->bEndpointAddress) {
1114                 DEBUG("%s, bad ep or descriptor\n", __FUNCTION__);
1115                 return -EINVAL;
1116         }
1117
1118         /* xfer types must match, except that interrupt ~= bulk */
1119         if (ep->bmAttributes != desc->bmAttributes
1120             && ep->bmAttributes != USB_ENDPOINT_XFER_BULK
1121             && desc->bmAttributes != USB_ENDPOINT_XFER_INT) {
1122                 DEBUG("%s, %s type mismatch\n", __FUNCTION__, _ep->name);
1123                 return -EINVAL;
1124         }
1125
1126         dev = ep->dev;
1127         if (!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN) {
1128                 DEBUG("%s, bogus device state\n", __FUNCTION__);
1129                 return -ESHUTDOWN;
1130         }
1131
1132         max = le16_to_cpu(desc->wMaxPacketSize);
1133
1134         spin_lock_irqsave(&ep->dev->lock, flags);
1135
1136         /* Configure the endpoint */
1137         jz_udc_set_index(dev, desc->bEndpointAddress & 0x0F);
1138         if (ep_is_in(ep)) {
1139                 usb_writew(dev, JZ_REG_UDC_INMAXP, max);
1140                 switch (desc->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) {
1141                 case USB_ENDPOINT_XFER_BULK:
1142                 case USB_ENDPOINT_XFER_INT:
1143                         csrh &= ~USB_INCSRH_ISO;
1144                         break;
1145                 case USB_ENDPOINT_XFER_ISOC:
1146                         csrh |= USB_INCSRH_ISO;
1147                         break;
1148                 }
1149                 usb_writeb(dev, JZ_REG_UDC_INCSRH, csrh);
1150         }
1151         else {
1152                 usb_writew(dev, JZ_REG_UDC_OUTMAXP, max);
1153                 switch (desc->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) {
1154                 case USB_ENDPOINT_XFER_BULK:
1155                          csrh &= ~USB_OUTCSRH_ISO;
1156                         break;
1157                 case USB_ENDPOINT_XFER_INT:
1158                         csrh &= ~USB_OUTCSRH_ISO;
1159                         csrh |= USB_OUTCSRH_DNYT;
1160                         break;
1161                 case USB_ENDPOINT_XFER_ISOC:
1162                         csrh |= USB_OUTCSRH_ISO;
1163                         break;
1164                 }
1165                 usb_writeb(dev, JZ_REG_UDC_OUTCSRH, csrh);
1166         }
1167
1168
1169         ep->stopped = 0;
1170         ep->desc = desc;
1171         ep->pio_irqs = 0;
1172         ep->ep.maxpacket = max;
1173
1174         spin_unlock_irqrestore(&ep->dev->lock, flags);
1175
1176         /* Reset halt state (does flush) */
1177         jz4740_set_halt(_ep, 0);
1178
1179         DEBUG("%s: enabled %s\n", __FUNCTION__, _ep->name);
1180
1181         return 0;
1182 }
1183
1184 /** Disable EP
1185  *  NOTE: Sets INDEX register
1186  */
1187 static int jz4740_ep_disable(struct usb_ep *_ep)
1188 {
1189         struct jz4740_ep *ep;
1190         unsigned long flags;
1191
1192         DEBUG("%s, %p\n", __FUNCTION__, _ep);
1193
1194         ep = container_of(_ep, struct jz4740_ep, ep);
1195         if (!_ep || !ep->desc) {
1196                 DEBUG("%s, %s not enabled\n", __FUNCTION__,
1197                       _ep ? ep->ep.name : NULL);
1198                 return -EINVAL;
1199         }
1200
1201         spin_lock_irqsave(&ep->dev->lock, flags);
1202
1203         jz_udc_select_ep(ep);
1204
1205         /* Nuke all pending requests (does flush) */
1206         nuke(ep, -ESHUTDOWN);
1207
1208         /* Disable ep IRQ */
1209         pio_irq_disable(ep);
1210
1211         ep->desc = 0;
1212         ep->stopped = 1;
1213
1214         spin_unlock_irqrestore(&ep->dev->lock, flags);
1215
1216         DEBUG("%s: disabled %s\n", __FUNCTION__, _ep->name);
1217         return 0;
1218 }
1219
1220 static struct usb_request *jz4740_alloc_request(struct usb_ep *ep, gfp_t gfp_flags)
1221 {
1222         struct jz4740_request *req;
1223
1224         DEBUG("%s, %p\n", __FUNCTION__, ep);
1225
1226         req = kzalloc(sizeof(*req), gfp_flags);
1227         if (!req)
1228                 return 0;
1229
1230         INIT_LIST_HEAD(&req->queue);
1231
1232         return &req->req;
1233 }
1234
1235 static void jz4740_free_request(struct usb_ep *ep, struct usb_request *_req)
1236 {
1237         struct jz4740_request *req;
1238
1239         DEBUG("%s, %p\n", __FUNCTION__, ep);
1240
1241         req = container_of(_req, struct jz4740_request, req);
1242         WARN_ON(!list_empty(&req->queue));
1243         kfree(req);
1244 }
1245
1246 /*--------------------------------------------------------------------*/
1247
1248 /** Queue one request
1249  *  Kickstart transfer if needed
1250  *  NOTE: Sets INDEX register
1251  */
1252 static int jz4740_queue(struct usb_ep *_ep, struct usb_request *_req,
1253                         gfp_t gfp_flags)
1254 {
1255         struct jz4740_request *req;
1256         struct jz4740_ep *ep;
1257         struct jz4740_udc *dev;
1258         unsigned long flags;
1259
1260         DEBUG("%s, %p\n", __FUNCTION__, _ep);
1261
1262         req = container_of(_req, struct jz4740_request, req);
1263         if (unlikely
1264             (!_req || !_req->complete || !_req->buf
1265              || !list_empty(&req->queue))) {
1266                 DEBUG("%s, bad params\n", __FUNCTION__);
1267                 return -EINVAL;
1268         }
1269
1270         ep = container_of(_ep, struct jz4740_ep, ep);
1271         if (unlikely(!_ep || (!ep->desc && ep->type != ep_control))) {
1272                 DEBUG("%s, bad ep\n", __FUNCTION__);
1273                 return -EINVAL;
1274         }
1275
1276         dev = ep->dev;
1277         if (unlikely(!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN)) {
1278                 DEBUG("%s, bogus device state %p\n", __FUNCTION__, dev->driver);
1279                 return -ESHUTDOWN;
1280         }
1281
1282         DEBUG("%s queue req %p, len %d buf %p\n", _ep->name, _req, _req->length,
1283               _req->buf);
1284
1285         spin_lock_irqsave(&dev->lock, flags);
1286
1287         _req->status = -EINPROGRESS;
1288         _req->actual = 0;
1289
1290         /* kickstart this i/o queue? */
1291         DEBUG("Add to %d Q %d %d\n", ep_index(ep), list_empty(&ep->queue),
1292               ep->stopped);
1293         if (list_empty(&ep->queue) && likely(!ep->stopped)) {
1294                 uint32_t csr;
1295
1296                 if (unlikely(ep_index(ep) == 0)) {
1297                         /* EP0 */
1298                         list_add_tail(&req->queue, &ep->queue);
1299                         jz4740_ep0_kick(dev, ep);
1300                         req = 0;
1301                 } else if (use_dma) {
1302                         /* DMA */
1303                         kick_dma(ep, req);
1304                 }
1305                 /* PIO */
1306                 else if (ep_is_in(ep)) {
1307                         /* EP1 & EP2 */
1308                         jz_udc_set_index(dev, ep_index(ep));
1309                         csr = usb_readb(dev, ep->csr);
1310                         pio_irq_enable(ep);
1311                         if (!(csr & USB_INCSR_FFNOTEMPT)) {
1312                                 if (write_fifo(ep, req) == 1)
1313                                         req = 0;
1314                         }
1315                 } else {
1316                         /* EP1 */
1317                         jz_udc_set_index(dev, ep_index(ep));
1318                         csr = usb_readb(dev, ep->csr);
1319                         pio_irq_enable(ep);
1320                         if (csr & USB_OUTCSR_OUTPKTRDY) {
1321                                 if (read_fifo(ep, req) == 1)
1322                                         req = 0;
1323                         }
1324                 }
1325         }
1326
1327         /* pio or dma irq handler advances the queue. */
1328         if (likely(req != 0))
1329                 list_add_tail(&req->queue, &ep->queue);
1330
1331         spin_unlock_irqrestore(&dev->lock, flags);
1332
1333         return 0;
1334 }
1335
1336 /* dequeue JUST ONE request */
1337 static int jz4740_dequeue(struct usb_ep *_ep, struct usb_request *_req)
1338 {
1339         struct jz4740_ep *ep;
1340         struct jz4740_request *req;
1341         unsigned long flags;
1342
1343         DEBUG("%s, %p\n", __FUNCTION__, _ep);
1344
1345         ep = container_of(_ep, struct jz4740_ep, ep);
1346         if (!_ep || ep->type == ep_control)
1347                 return -EINVAL;
1348
1349         spin_lock_irqsave(&ep->dev->lock, flags);
1350
1351         /* make sure it's actually queued on this endpoint */
1352         list_for_each_entry(req, &ep->queue, queue) {
1353                 if (&req->req == _req)
1354                         break;
1355         }
1356         if (&req->req != _req) {
1357                 spin_unlock_irqrestore(&ep->dev->lock, flags);
1358                 return -EINVAL;
1359         }
1360         done(ep, req, -ECONNRESET);
1361
1362         spin_unlock_irqrestore(&ep->dev->lock, flags);
1363         return 0;
1364 }
1365
1366 /** Return bytes in EP FIFO
1367  *  NOTE: Sets INDEX register to EP
1368  */
1369 static int jz4740_fifo_status(struct usb_ep *_ep)
1370 {
1371         uint32_t csr;
1372         int count = 0;
1373         struct jz4740_ep *ep;
1374         unsigned long flags;
1375
1376         ep = container_of(_ep, struct jz4740_ep, ep);
1377         if (!_ep) {
1378                 DEBUG("%s, bad ep\n", __FUNCTION__);
1379                 return -ENODEV;
1380         }
1381
1382         DEBUG("%s, %d\n", __FUNCTION__, ep_index(ep));
1383
1384         /* LPD can't report unclaimed bytes from IN fifos */
1385         if (ep_is_in(ep))
1386                 return -EOPNOTSUPP;
1387
1388         spin_lock_irqsave(&ep->dev->lock, flags);
1389         jz_udc_set_index(ep->dev, ep_index(ep));
1390
1391         csr = usb_readb(ep->dev, ep->csr);
1392         if (ep->dev->gadget.speed != USB_SPEED_UNKNOWN ||
1393             csr & 0x1) {
1394                 count = usb_readw(ep->dev, JZ_REG_UDC_OUTCOUNT);
1395         }
1396
1397         spin_unlock_irqrestore(&ep->dev->lock, flags);
1398
1399         return count;
1400 }
1401
1402 /** Flush EP FIFO
1403  *  NOTE: Sets INDEX register to EP
1404  */
1405 static void jz4740_fifo_flush(struct usb_ep *_ep)
1406 {
1407         struct jz4740_ep *ep;
1408         unsigned long flags;
1409
1410         DEBUG("%s:%s[%d]\n", __FILE__, __func__, __LINE__);
1411
1412         ep = container_of(_ep, struct jz4740_ep, ep);
1413         if (unlikely(!_ep || (!ep->desc && ep->type == ep_control))) {
1414                 DEBUG("%s, bad ep\n", __FUNCTION__);
1415                 return;
1416         }
1417
1418         spin_lock_irqsave(&ep->dev->lock, flags);
1419
1420         jz_udc_set_index(ep->dev, ep_index(ep));
1421         flush(ep);
1422
1423         spin_unlock_irqrestore(&ep->dev->lock, flags);
1424 }
1425
1426 /****************************************************************/
1427 /* End Point 0 related functions                                */
1428 /****************************************************************/
1429
1430 /* return:  0 = still running, 1 = completed, negative = errno */
1431 static int write_fifo_ep0(struct jz4740_ep *ep, struct jz4740_request *req)
1432 {
1433         uint32_t max;
1434         unsigned count;
1435         int is_last;
1436
1437     DEBUG("%s:%s[%d]\n", __FILE__, __func__, __LINE__);
1438         max = ep_maxpacket(ep);
1439
1440         count = write_packet(ep, req, max);
1441
1442         /* last packet is usually short (or a zlp) */
1443         if (unlikely(count != max))
1444                 is_last = 1;
1445         else {
1446                 if (likely(req->req.length != req->req.actual) || req->req.zero)
1447                         is_last = 0;
1448                 else
1449                         is_last = 1;
1450         }
1451
1452         DEBUG_EP0("%s: wrote %s %d bytes%s %d left %p\n", __FUNCTION__,
1453                   ep->ep.name, count,
1454                   is_last ? "/L" : "", req->req.length - req->req.actual, req);
1455
1456         /* requests complete when all IN data is in the FIFO */
1457         if (is_last) {
1458                 done(ep, req, 0);
1459                 return 1;
1460         }
1461
1462         return 0;
1463 }
1464
1465 static inline int jz4740_fifo_read(struct jz4740_ep *ep,
1466                                        unsigned char *cp, int max)
1467 {
1468         int bytes;
1469         int count = usb_readw(ep->dev, JZ_REG_UDC_OUTCOUNT);
1470
1471         if (count > max)
1472                 count = max;
1473         bytes = count;
1474         while (count--)
1475                 *cp++ = usb_readb(ep->dev, ep->fifo);
1476
1477         return bytes;
1478 }
1479
1480 static inline void jz4740_fifo_write(struct jz4740_ep *ep,
1481                                          unsigned char *cp, int count)
1482 {
1483         DEBUG("fifo_write: %d %d\n", ep_index(ep), count);
1484         while (count--)
1485                 usb_writeb(ep->dev, ep->fifo, *cp++);
1486 }
1487
1488 static int read_fifo_ep0(struct jz4740_ep *ep, struct jz4740_request *req)
1489 {
1490         struct jz4740_udc *dev = ep->dev;
1491         uint32_t csr;
1492         uint8_t *buf;
1493         unsigned bufferspace, count, is_short;
1494
1495         DEBUG_EP0("%s\n", __FUNCTION__);
1496
1497         csr = usb_readb(dev, JZ_REG_UDC_CSR0);
1498         if (!(csr & USB_CSR0_OUTPKTRDY))
1499                 return 0;
1500
1501         buf = req->req.buf + req->req.actual;
1502         prefetchw(buf);
1503         bufferspace = req->req.length - req->req.actual;
1504
1505         /* read all bytes from this packet */
1506         if (likely(csr & USB_CSR0_OUTPKTRDY)) {
1507                 count = usb_readw(dev, JZ_REG_UDC_OUTCOUNT);
1508                 req->req.actual += min(count, bufferspace);
1509         } else                  /* zlp */
1510                 count = 0;
1511
1512         is_short = (count < ep->ep.maxpacket);
1513         DEBUG_EP0("read %s %02x, %d bytes%s req %p %d/%d\n",
1514                   ep->ep.name, csr, count,
1515                   is_short ? "/S" : "", req, req->req.actual, req->req.length);
1516
1517         while (likely(count-- != 0)) {
1518                 uint8_t byte = (uint8_t)usb_readl(dev, ep->fifo);
1519
1520                 if (unlikely(bufferspace == 0)) {
1521                         /* this happens when the driver's buffer
1522                          * is smaller than what the host sent.
1523                          * discard the extra data.
1524                          */
1525                         if (req->req.status != -EOVERFLOW)
1526                                 DEBUG_EP0("%s overflow %d\n", ep->ep.name,
1527                                           count);
1528                         req->req.status = -EOVERFLOW;
1529                 } else {
1530                         *buf++ = byte;
1531                         bufferspace--;
1532                 }
1533         }
1534
1535         /* completion */
1536         if (is_short || req->req.actual == req->req.length) {
1537                 done(ep, req, 0);
1538                 return 1;
1539         }
1540
1541         /* finished that packet.  the next one may be waiting... */
1542         return 0;
1543 }
1544
1545 /**
1546  * udc_set_address - set the USB address for this device
1547  * @address:
1548  *
1549  * Called from control endpoint function after it decodes a set address setup packet.
1550  */
1551 static void udc_set_address(struct jz4740_udc *dev, unsigned char address)
1552 {
1553         DEBUG_EP0("%s: %d\n", __FUNCTION__, address);
1554
1555         dev->usb_address = address;
1556         usb_writeb(dev, JZ_REG_UDC_FADDR, address);
1557 }
1558
1559 /*
1560  * DATA_STATE_RECV (USB_CSR0_OUTPKTRDY)
1561  *      - if error
1562  *              set USB_CSR0_SVDOUTPKTRDY | USB_CSR0_DATAEND | USB_CSR0_SENDSTALL bits
1563  *      - else
1564  *              set USB_CSR0_SVDOUTPKTRDY bit
1565                                 if last set USB_CSR0_DATAEND bit
1566  */
1567 static void jz4740_ep0_out(struct jz4740_udc *dev, uint32_t csr, int kickstart)
1568 {
1569         struct jz4740_request *req;
1570         struct jz4740_ep *ep = &dev->ep[0];
1571         int ret;
1572
1573         DEBUG_EP0("%s: %x\n", __FUNCTION__, csr);
1574
1575         if (list_empty(&ep->queue))
1576                 req = 0;
1577         else
1578                 req = list_entry(ep->queue.next, struct jz4740_request, queue);
1579
1580         if (req) {
1581                 if (req->req.length == 0) {
1582                         DEBUG_EP0("ZERO LENGTH OUT!\n");
1583                         usb_setb(dev, JZ_REG_UDC_CSR0, (USB_CSR0_SVDOUTPKTRDY | USB_CSR0_DATAEND));
1584                         dev->ep0state = WAIT_FOR_SETUP;
1585                         return;
1586                 } else if (kickstart) {
1587                         usb_setb(dev, JZ_REG_UDC_CSR0, (USB_CSR0_SVDOUTPKTRDY));
1588                         return;
1589                 }
1590                 ret = read_fifo_ep0(ep, req);
1591                 if (ret) {
1592                         /* Done! */
1593                         DEBUG_EP0("%s: finished, waiting for status\n",
1594                                   __FUNCTION__);
1595                         usb_setb(dev, JZ_REG_UDC_CSR0, (USB_CSR0_SVDOUTPKTRDY | USB_CSR0_DATAEND));
1596                         dev->ep0state = WAIT_FOR_SETUP;
1597                 } else {
1598                         /* Not done yet.. */
1599                         DEBUG_EP0("%s: not finished\n", __FUNCTION__);
1600                         usb_setb(dev, JZ_REG_UDC_CSR0, USB_CSR0_SVDOUTPKTRDY);
1601                 }
1602         } else {
1603                 DEBUG_EP0("NO REQ??!\n");
1604         }
1605 }
1606
1607 /*
1608  * DATA_STATE_XMIT
1609  */
1610 static int jz4740_ep0_in(struct jz4740_udc *dev, uint32_t csr)
1611 {
1612         struct jz4740_request *req;
1613         struct jz4740_ep *ep = &dev->ep[0];
1614         int ret, need_zlp = 0;
1615
1616         DEBUG_EP0("%s: %x\n", __FUNCTION__, csr);
1617
1618         if (list_empty(&ep->queue))
1619                 req = 0;
1620         else
1621                 req = list_entry(ep->queue.next, struct jz4740_request, queue);
1622
1623         if (!req) {
1624                 DEBUG_EP0("%s: NULL REQ\n", __FUNCTION__);
1625                 return 0;
1626         }
1627
1628         if (req->req.length == 0) {
1629                 usb_setb(dev, JZ_REG_UDC_CSR0, (USB_CSR0_INPKTRDY | USB_CSR0_DATAEND));
1630                 dev->ep0state = WAIT_FOR_SETUP;
1631                 return 1;
1632         }
1633
1634         if (req->req.length - req->req.actual == EP0_MAXPACKETSIZE) {
1635                 /* Next write will end with the packet size, */
1636                 /* so we need zero-length-packet */
1637                 need_zlp = 1;
1638         }
1639
1640         ret = write_fifo_ep0(ep, req);
1641
1642         if (ret == 1 && !need_zlp) {
1643                 /* Last packet */
1644                 DEBUG_EP0("%s: finished, waiting for status\n", __FUNCTION__);
1645
1646                 usb_setb(dev, JZ_REG_UDC_CSR0, (USB_CSR0_INPKTRDY | USB_CSR0_DATAEND));
1647                 dev->ep0state = WAIT_FOR_SETUP;
1648         } else {
1649                 DEBUG_EP0("%s: not finished\n", __FUNCTION__);
1650                 usb_setb(dev, JZ_REG_UDC_CSR0, USB_CSR0_INPKTRDY);
1651         }
1652
1653         if (need_zlp) {
1654                 DEBUG_EP0("%s: Need ZLP!\n", __FUNCTION__);
1655                 usb_setb(dev, JZ_REG_UDC_CSR0, USB_CSR0_INPKTRDY);
1656                 dev->ep0state = DATA_STATE_NEED_ZLP;
1657         }
1658
1659         return 1;
1660 }
1661
1662 static int jz4740_handle_get_status(struct jz4740_udc *dev,
1663                                     struct usb_ctrlrequest *ctrl)
1664 {
1665         struct jz4740_ep *ep0 = &dev->ep[0];
1666         struct jz4740_ep *qep;
1667         int reqtype = (ctrl->bRequestType & USB_RECIP_MASK);
1668         uint16_t val = 0;
1669
1670     DEBUG("%s:%s[%d]\n", __FILE__, __func__, __LINE__);
1671
1672         if (reqtype == USB_RECIP_INTERFACE) {
1673                 /* This is not supported.
1674                  * And according to the USB spec, this one does nothing..
1675                  * Just return 0
1676                  */
1677                 DEBUG_SETUP("GET_STATUS: USB_RECIP_INTERFACE\n");
1678         } else if (reqtype == USB_RECIP_DEVICE) {
1679                 DEBUG_SETUP("GET_STATUS: USB_RECIP_DEVICE\n");
1680                 val |= (1 << 0);        /* Self powered */
1681                 /*val |= (1<<1); *//* Remote wakeup */
1682         } else if (reqtype == USB_RECIP_ENDPOINT) {
1683                 int ep_num = (ctrl->wIndex & ~USB_DIR_IN);
1684
1685                 DEBUG_SETUP
1686                         ("GET_STATUS: USB_RECIP_ENDPOINT (%d), ctrl->wLength = %d\n",
1687                          ep_num, ctrl->wLength);
1688
1689                 if (ctrl->wLength > 2 || ep_num > 3)
1690                         return -EOPNOTSUPP;
1691
1692                 qep = &dev->ep[ep_num];
1693                 if (ep_is_in(qep) != ((ctrl->wIndex & USB_DIR_IN) ? 1 : 0)
1694                     && ep_index(qep) != 0) {
1695                         return -EOPNOTSUPP;
1696                 }
1697
1698                 jz_udc_set_index(dev, ep_index(qep));
1699
1700                 /* Return status on next IN token */
1701                 switch (qep->type) {
1702                 case ep_control:
1703                         val =
1704                             (usb_readb(dev, qep->csr) & USB_CSR0_SENDSTALL) ==
1705                             USB_CSR0_SENDSTALL;
1706                         break;
1707                 case ep_bulk_in:
1708                 case ep_interrupt:
1709                         val =
1710                             (usb_readb(dev, qep->csr) & USB_INCSR_SENDSTALL) ==
1711                             USB_INCSR_SENDSTALL;
1712                         break;
1713                 case ep_bulk_out:
1714                         val =
1715                             (usb_readb(dev, qep->csr) & USB_OUTCSR_SENDSTALL) ==
1716                             USB_OUTCSR_SENDSTALL;
1717                         break;
1718                 }
1719
1720                 /* Back to EP0 index */
1721                 jz_udc_set_index(dev, 0);
1722
1723                 DEBUG_SETUP("GET_STATUS, ep: %d (%x), val = %d\n", ep_num,
1724                             ctrl->wIndex, val);
1725         } else {
1726                 DEBUG_SETUP("Unknown REQ TYPE: %d\n", reqtype);
1727                 return -EOPNOTSUPP;
1728         }
1729
1730         /* Clear "out packet ready" */
1731         usb_setb(dev, JZ_REG_UDC_CSR0, USB_CSR0_SVDOUTPKTRDY);
1732         /* Put status to FIFO */
1733         jz4740_fifo_write(ep0, (uint8_t *)&val, sizeof(val));
1734         /* Issue "In packet ready" */
1735         usb_setb(dev, JZ_REG_UDC_CSR0, (USB_CSR0_INPKTRDY | USB_CSR0_DATAEND));
1736
1737         return 0;
1738 }
1739
1740 /*
1741  * WAIT_FOR_SETUP (OUTPKTRDY)
1742  *      - read data packet from EP0 FIFO
1743  *      - decode command
1744  *      - if error
1745  *              set USB_CSR0_SVDOUTPKTRDY | USB_CSR0_DATAEND | USB_CSR0_SENDSTALL bits
1746  *      - else
1747  *              set USB_CSR0_SVDOUTPKTRDY | USB_CSR0_DATAEND bits
1748  */
1749 static void jz4740_ep0_setup(struct jz4740_udc *dev, uint32_t csr)
1750 {
1751         struct jz4740_ep *ep = &dev->ep[0];
1752         struct usb_ctrlrequest ctrl;
1753         int i;
1754
1755         DEBUG_SETUP("%s: %x\n", __FUNCTION__, csr);
1756
1757         /* Nuke all previous transfers */
1758         nuke(ep, -EPROTO);
1759
1760         /* read control req from fifo (8 bytes) */
1761         jz4740_fifo_read(ep, (unsigned char *)&ctrl, 8);
1762
1763         DEBUG_SETUP("SETUP %02x.%02x v%04x i%04x l%04x\n",
1764                     ctrl.bRequestType, ctrl.bRequest,
1765                     ctrl.wValue, ctrl.wIndex, ctrl.wLength);
1766
1767         /* Set direction of EP0 */
1768         if (likely(ctrl.bRequestType & USB_DIR_IN)) {
1769                 ep->bEndpointAddress |= USB_DIR_IN;
1770         } else {
1771                 ep->bEndpointAddress &= ~USB_DIR_IN;
1772         }
1773
1774         /* Handle some SETUP packets ourselves */
1775         switch (ctrl.bRequest) {
1776         case USB_REQ_SET_ADDRESS:
1777                 if (ctrl.bRequestType != (USB_TYPE_STANDARD | USB_RECIP_DEVICE))
1778                         break;
1779
1780                 DEBUG_SETUP("USB_REQ_SET_ADDRESS (%d)\n", ctrl.wValue);
1781                 udc_set_address(dev, ctrl.wValue);
1782                 usb_setb(dev, JZ_REG_UDC_CSR0, (USB_CSR0_SVDOUTPKTRDY | USB_CSR0_DATAEND));
1783                 return;
1784
1785         case USB_REQ_SET_CONFIGURATION:
1786                 if (ctrl.bRequestType != (USB_TYPE_STANDARD | USB_RECIP_DEVICE))
1787                         break;
1788
1789                 DEBUG_SETUP("USB_REQ_SET_CONFIGURATION (%d)\n", ctrl.wValue);
1790 /*              usb_setb(JZ_REG_UDC_CSR0, (USB_CSR0_SVDOUTPKTRDY | USB_CSR0_DATAEND));*/
1791
1792                 /* Enable RESUME and SUSPEND interrupts */
1793                 usb_setb(dev, JZ_REG_UDC_INTRUSBE, (USB_INTR_RESUME | USB_INTR_SUSPEND));
1794                 break;
1795
1796         case USB_REQ_SET_INTERFACE:
1797                 if (ctrl.bRequestType != (USB_TYPE_STANDARD | USB_RECIP_DEVICE))
1798                         break;
1799
1800                 DEBUG_SETUP("USB_REQ_SET_INTERFACE (%d)\n", ctrl.wValue);
1801 /*              usb_setb(JZ_REG_UDC_CSR0, (USB_CSR0_SVDOUTPKTRDY | USB_CSR0_DATAEND));*/
1802                 break;
1803
1804         case USB_REQ_GET_STATUS:
1805                 if (jz4740_handle_get_status(dev, &ctrl) == 0)
1806                         return;
1807
1808         case USB_REQ_CLEAR_FEATURE:
1809         case USB_REQ_SET_FEATURE:
1810                 if (ctrl.bRequestType == USB_RECIP_ENDPOINT) {
1811                         struct jz4740_ep *qep;
1812                         int ep_num = (ctrl.wIndex & 0x0f);
1813
1814                         /* Support only HALT feature */
1815                         if (ctrl.wValue != 0 || ctrl.wLength != 0
1816                             || ep_num > 3 || ep_num < 1)
1817                                 break;
1818
1819                         qep = &dev->ep[ep_num];
1820                         spin_unlock(&dev->lock);
1821                         if (ctrl.bRequest == USB_REQ_SET_FEATURE) {
1822                                 DEBUG_SETUP("SET_FEATURE (%d)\n",
1823                                             ep_num);
1824                                 jz4740_set_halt(&qep->ep, 1);
1825                         } else {
1826                                 DEBUG_SETUP("CLR_FEATURE (%d)\n",
1827                                             ep_num);
1828                                 jz4740_set_halt(&qep->ep, 0);
1829                         }
1830                         spin_lock(&dev->lock);
1831
1832                         jz_udc_set_index(dev, 0);
1833
1834                         /* Reply with a ZLP on next IN token */
1835                         usb_setb(dev, JZ_REG_UDC_CSR0,
1836                                  (USB_CSR0_SVDOUTPKTRDY | USB_CSR0_DATAEND));
1837                         return;
1838                 }
1839                 break;
1840
1841         default:
1842                 break;
1843         }
1844
1845         /* gadget drivers see class/vendor specific requests,
1846          * {SET,GET}_{INTERFACE,DESCRIPTOR,CONFIGURATION},
1847          * and more.
1848          */
1849         if (dev->driver) {
1850                 /* device-2-host (IN) or no data setup command, process immediately */
1851                 spin_unlock(&dev->lock);
1852
1853                 i = dev->driver->setup(&dev->gadget, &ctrl);
1854                 spin_lock(&dev->lock);
1855
1856                 if (unlikely(i < 0)) {
1857                         /* setup processing failed, force stall */
1858                         DEBUG_SETUP
1859                             ("  --> ERROR: gadget setup FAILED (stalling), setup returned %d\n",
1860                              i);
1861                         jz_udc_set_index(dev, 0);
1862                         usb_setb(dev, JZ_REG_UDC_CSR0, (USB_CSR0_SVDOUTPKTRDY | USB_CSR0_DATAEND | USB_CSR0_SENDSTALL));
1863
1864                         /* ep->stopped = 1; */
1865                         dev->ep0state = WAIT_FOR_SETUP;
1866                 }
1867                 else {
1868                         DEBUG_SETUP("gadget driver setup ok (%d)\n", ctrl.wLength);
1869 /*                      if (!ctrl.wLength) {
1870                                 usb_setb(JZ_REG_UDC_CSR0, USB_CSR0_SVDOUTPKTRDY);
1871                         }*/
1872                 }
1873         }
1874 }
1875
1876 /*
1877  * DATA_STATE_NEED_ZLP
1878  */
1879 static void jz4740_ep0_in_zlp(struct jz4740_udc *dev, uint32_t csr)
1880 {
1881         DEBUG_EP0("%s: %x\n", __FUNCTION__, csr);
1882
1883         usb_setb(dev, JZ_REG_UDC_CSR0, (USB_CSR0_INPKTRDY | USB_CSR0_DATAEND));
1884         dev->ep0state = WAIT_FOR_SETUP;
1885 }
1886
1887 /*
1888  * handle ep0 interrupt
1889  */
1890 static void jz4740_handle_ep0(struct jz4740_udc *dev, uint32_t intr)
1891 {
1892         struct jz4740_ep *ep = &dev->ep[0];
1893         uint32_t csr;
1894
1895     DEBUG("%s:%s[%d]\n", __FILE__, __func__, __LINE__);
1896         /* Set index 0 */
1897         jz_udc_set_index(dev, 0);
1898         csr = usb_readb(dev, JZ_REG_UDC_CSR0);
1899
1900         DEBUG_EP0("%s: csr = %x  state = \n", __FUNCTION__, csr);//, state_names[dev->ep0state]);
1901
1902         /*
1903          * if SENT_STALL is set
1904          *      - clear the SENT_STALL bit
1905          */
1906         if (csr & USB_CSR0_SENTSTALL) {
1907                 DEBUG_EP0("%s: USB_CSR0_SENTSTALL is set: %x\n", __FUNCTION__, csr);
1908                 usb_clearb(dev, JZ_REG_UDC_CSR0, USB_CSR0_SENDSTALL | USB_CSR0_SENTSTALL);
1909                 nuke(ep, -ECONNABORTED);
1910                 dev->ep0state = WAIT_FOR_SETUP;
1911                 return;
1912         }
1913
1914         /*
1915          * if a transfer is in progress && INPKTRDY and OUTPKTRDY are clear
1916          *      - fill EP0 FIFO
1917          *      - if last packet
1918          *      -       set IN_PKT_RDY | DATA_END
1919          *      - else
1920          *              set IN_PKT_RDY
1921          */
1922         if (!(csr & (USB_CSR0_INPKTRDY | USB_CSR0_OUTPKTRDY))) {
1923                 DEBUG_EP0("%s: INPKTRDY and OUTPKTRDY are clear\n",
1924                           __FUNCTION__);
1925
1926                 switch (dev->ep0state) {
1927                 case DATA_STATE_XMIT:
1928                         DEBUG_EP0("continue with DATA_STATE_XMIT\n");
1929                         jz4740_ep0_in(dev, csr);
1930                         return;
1931                 case DATA_STATE_NEED_ZLP:
1932                         DEBUG_EP0("continue with DATA_STATE_NEED_ZLP\n");
1933                         jz4740_ep0_in_zlp(dev, csr);
1934                         return;
1935                 default:
1936                         /* Stall? */
1937 //                      DEBUG_EP0("Odd state!! state = %s\n",
1938 //                                state_names[dev->ep0state]);
1939                         dev->ep0state = WAIT_FOR_SETUP;
1940                         /* nuke(ep, 0); */
1941                         /* usb_setb(ep->csr, USB_CSR0_SENDSTALL); */
1942 //                      break;
1943                         return;
1944                 }
1945         }
1946
1947         /*
1948          * if SETUPEND is set
1949          *      - abort the last transfer
1950          *      - set SERVICED_SETUP_END_BIT
1951          */
1952         if (csr & USB_CSR0_SETUPEND) {
1953                 DEBUG_EP0("%s: USB_CSR0_SETUPEND is set: %x\n", __FUNCTION__, csr);
1954
1955                 usb_setb(dev, JZ_REG_UDC_CSR0, USB_CSR0_SVDSETUPEND);
1956                 nuke(ep, 0);
1957                 dev->ep0state = WAIT_FOR_SETUP;
1958         }
1959
1960         /*
1961          * if USB_CSR0_OUTPKTRDY is set
1962          *      - read data packet from EP0 FIFO
1963          *      - decode command
1964          *      - if error
1965          *              set SVDOUTPKTRDY | DATAEND | SENDSTALL bits
1966          *      - else
1967          *              set SVDOUTPKTRDY | DATAEND bits
1968          */
1969         if (csr & USB_CSR0_OUTPKTRDY) {
1970
1971                 DEBUG_EP0("%s: EP0_OUT_PKT_RDY is set: %x\n", __FUNCTION__,
1972                           csr);
1973
1974                 switch (dev->ep0state) {
1975                 case WAIT_FOR_SETUP:
1976                         DEBUG_EP0("WAIT_FOR_SETUP\n");
1977                         jz4740_ep0_setup(dev, csr);
1978                         break;
1979
1980                 case DATA_STATE_RECV:
1981                         DEBUG_EP0("DATA_STATE_RECV\n");
1982                         jz4740_ep0_out(dev, csr, 0);
1983                         break;
1984
1985                 default:
1986                         /* send stall? */
1987                         DEBUG_EP0("strange state!! 2. send stall? state = %d\n",
1988                                   dev->ep0state);
1989                         break;
1990                 }
1991         }
1992 }
1993
1994 static void jz4740_ep0_kick(struct jz4740_udc *dev, struct jz4740_ep *ep)
1995 {
1996         uint32_t csr;
1997
1998         jz_udc_set_index(dev, 0);
1999
2000         DEBUG_EP0("%s: %x\n", __FUNCTION__, csr);
2001
2002         /* Clear "out packet ready" */
2003
2004         if (ep_is_in(ep)) {
2005                 usb_setb(dev, JZ_REG_UDC_CSR0, USB_CSR0_SVDOUTPKTRDY);
2006                 csr = usb_readb(dev, JZ_REG_UDC_CSR0);
2007                 dev->ep0state = DATA_STATE_XMIT;
2008                 jz4740_ep0_in(dev, csr);
2009         } else {
2010                 csr = usb_readb(dev, JZ_REG_UDC_CSR0);
2011                 dev->ep0state = DATA_STATE_RECV;
2012                 jz4740_ep0_out(dev, csr, 1);
2013         }
2014 }
2015
2016 /** Handle USB RESET interrupt
2017  */
2018 static void jz4740_reset_irq(struct jz4740_udc *dev)
2019 {
2020         dev->gadget.speed = (usb_readb(dev, JZ_REG_UDC_POWER) & USB_POWER_HSMODE) ?
2021                 USB_SPEED_HIGH : USB_SPEED_FULL;
2022
2023         DEBUG_SETUP("%s: address = %d, speed = %s\n", __FUNCTION__, dev->usb_address,
2024                     (dev->gadget.speed == USB_SPEED_HIGH) ? "HIGH":"FULL" );
2025 }
2026
2027 /*
2028  *      jz4740 usb device interrupt handler.
2029  */
2030 static irqreturn_t jz4740_udc_irq(int irq, void *_dev)
2031 {
2032         struct jz4740_udc *dev = _dev;
2033         uint8_t index;
2034
2035         uint32_t intr_usb = usb_readb(dev, JZ_REG_UDC_INTRUSB) & 0x7; /* mask SOF */
2036         uint32_t intr_in  = usb_readw(dev, JZ_REG_UDC_INTRIN);
2037         uint32_t intr_out = usb_readw(dev, JZ_REG_UDC_INTROUT);
2038         uint32_t intr_dma = usb_readb(dev, JZ_REG_UDC_INTR);
2039
2040         if (!intr_usb && !intr_in && !intr_out && !intr_dma)
2041                 return IRQ_HANDLED;
2042
2043
2044         DEBUG("intr_out=%x intr_in=%x intr_usb=%x\n",
2045               intr_out, intr_in, intr_usb);
2046
2047         spin_lock(&dev->lock);
2048         index = usb_readb(dev, JZ_REG_UDC_INDEX);
2049
2050         /* Check for resume from suspend mode */
2051         if ((intr_usb & USB_INTR_RESUME) &&
2052             (usb_readb(dev, JZ_REG_UDC_INTRUSBE) & USB_INTR_RESUME)) {
2053                 DEBUG("USB resume\n");
2054                 dev->driver->resume(&dev->gadget); /* We have suspend(), so we must have resume() too. */
2055         }
2056
2057         /* Check for system interrupts */
2058         if (intr_usb & USB_INTR_RESET) {
2059                 DEBUG("USB reset\n");
2060                 jz4740_reset_irq(dev);
2061         }
2062
2063         /* Check for endpoint 0 interrupt */
2064         if (intr_in & USB_INTR_EP0) {
2065                 DEBUG("USB_INTR_EP0 (control)\n");
2066                 jz4740_handle_ep0(dev, intr_in);
2067         }
2068
2069         /* Check for Bulk-IN DMA interrupt */
2070         if (intr_dma & 0x1) {
2071                 int ep_num;
2072                 struct jz4740_ep *ep;
2073                 ep_num = (usb_readl(dev, JZ_REG_UDC_CNTL1) >> 4) & 0xf;
2074                 ep = &dev->ep[ep_num + 1];
2075                 jz_udc_set_index(dev, ep_num);
2076                 usb_setb(dev, ep->csr, USB_INCSR_INPKTRDY);
2077 /*              jz4740_in_epn(dev, ep_num, intr_in);*/
2078         }
2079
2080         /* Check for Bulk-OUT DMA interrupt */
2081         if (intr_dma & 0x2) {
2082                 int ep_num;
2083                 ep_num = (usb_readl(dev, JZ_REG_UDC_CNTL2) >> 4) & 0xf;
2084                 jz4740_out_epn(dev, ep_num, intr_out);
2085         }
2086
2087         /* Check for each configured endpoint interrupt */
2088         if (intr_in & USB_INTR_INEP1) {
2089                 DEBUG("USB_INTR_INEP1\n");
2090                 jz4740_in_epn(dev, 1, intr_in);
2091         }
2092
2093         if (intr_in & USB_INTR_INEP2) {
2094                 DEBUG("USB_INTR_INEP2\n");
2095                 jz4740_in_epn(dev, 2, intr_in);
2096         }
2097
2098         if (intr_out & USB_INTR_OUTEP1) {
2099                 DEBUG("USB_INTR_OUTEP1\n");
2100                 jz4740_out_epn(dev, 1, intr_out);
2101         }
2102
2103         /* Check for suspend mode */
2104         if ((intr_usb & USB_INTR_SUSPEND) &&
2105             (usb_readb(dev, JZ_REG_UDC_INTRUSBE) & USB_INTR_SUSPEND)) {
2106                 DEBUG("USB suspend\n");
2107                 dev->driver->suspend(&dev->gadget);
2108                 /* Host unloaded from us, can do something, such as flushing
2109                  the NAND block cache etc. */
2110         }
2111
2112     jz_udc_set_index(dev, index);
2113
2114         spin_unlock(&dev->lock);
2115
2116         return IRQ_HANDLED;
2117 }
2118
2119
2120
2121 /*-------------------------------------------------------------------------*/
2122
2123 /* Common functions - Added by River */
2124 static struct jz4740_udc udc_dev;
2125
2126 static inline struct jz4740_udc *gadget_to_udc(struct usb_gadget *gadget)
2127 {
2128         return container_of(gadget, struct jz4740_udc, gadget);
2129 }
2130 /* End added */
2131
2132 static int jz4740_udc_get_frame(struct usb_gadget *_gadget)
2133 {
2134         DEBUG("%s, %p\n", __FUNCTION__, _gadget);
2135         return usb_readw(gadget_to_udc(_gadget), JZ_REG_UDC_FRAME);
2136 }
2137
2138 static int jz4740_udc_wakeup(struct usb_gadget *_gadget)
2139 {
2140         /* host may not have enabled remote wakeup */
2141         /*if ((UDCCS0 & UDCCS0_DRWF) == 0)
2142            return -EHOSTUNREACH;
2143            udc_set_mask_UDCCR(UDCCR_RSM); */
2144         return -ENOTSUPP;
2145 }
2146
2147 static int jz4740_udc_pullup(struct usb_gadget *_gadget, int on)
2148 {
2149         struct jz4740_udc *udc = gadget_to_udc(_gadget);
2150         unsigned long flags;
2151
2152         local_irq_save(flags);
2153
2154         if (on) {
2155                 udc->state = UDC_STATE_ENABLE;
2156                 udc_enable(udc);
2157         } else {
2158                 udc->state = UDC_STATE_DISABLE;
2159                 udc_disable(udc);
2160         }
2161
2162         local_irq_restore(flags);
2163
2164         return 0;
2165 }
2166
2167
2168 static const struct usb_gadget_ops jz4740_udc_ops = {
2169         .get_frame = jz4740_udc_get_frame,
2170         .wakeup = jz4740_udc_wakeup,
2171         .pullup = jz4740_udc_pullup,
2172         /* current versions must always be self-powered */
2173 };
2174
2175 static struct usb_ep_ops jz4740_ep_ops = {
2176         .enable         = jz4740_ep_enable,
2177         .disable        = jz4740_ep_disable,
2178
2179         .alloc_request  = jz4740_alloc_request,
2180         .free_request   = jz4740_free_request,
2181
2182         .queue          = jz4740_queue,
2183         .dequeue        = jz4740_dequeue,
2184
2185         .set_halt       = jz4740_set_halt,
2186         .fifo_status    = jz4740_fifo_status,
2187         .fifo_flush     = jz4740_fifo_flush,
2188 };
2189
2190
2191 /*-------------------------------------------------------------------------*/
2192
2193 static struct jz4740_udc udc_dev = {
2194         .usb_address = 0,
2195         .gadget = {
2196                 .ops = &jz4740_udc_ops,
2197                 .ep0 = &udc_dev.ep[0].ep,
2198                 .name = "jz-udc",
2199                 .dev = {
2200                         .init_name = "gadget",
2201                 },
2202         },
2203
2204         /* control endpoint */
2205         .ep[0] = {
2206                 .ep = {
2207                         .name = "ep0",
2208                         .ops = &jz4740_ep_ops,
2209                         .maxpacket = EP0_MAXPACKETSIZE,
2210                 },
2211                 .dev = &udc_dev,
2212
2213                 .bEndpointAddress = 0,
2214                 .bmAttributes = 0,
2215
2216                 .type = ep_control,
2217                 .fifo = JZ_REG_UDC_EP_FIFO(0),
2218                 .csr = JZ_REG_UDC_CSR0,
2219         },
2220
2221         /* bulk out endpoint */
2222         .ep[1] = {
2223                 .ep = {
2224                         .name = "ep1out-bulk",
2225                         .ops = &jz4740_ep_ops,
2226                         .maxpacket = EPBULK_MAXPACKETSIZE,
2227                 },
2228                 .dev = &udc_dev,
2229
2230                 .bEndpointAddress = 1,
2231                 .bmAttributes = USB_ENDPOINT_XFER_BULK,
2232
2233                 .type = ep_bulk_out,
2234                 .fifo = JZ_REG_UDC_EP_FIFO(1),
2235                 .csr = JZ_REG_UDC_OUTCSR,
2236         },
2237
2238         /* bulk in endpoint */
2239         .ep[2] = {
2240                 .ep = {
2241                         .name = "ep1in-bulk",
2242                         .ops = &jz4740_ep_ops,
2243                         .maxpacket = EPBULK_MAXPACKETSIZE,
2244                 },
2245                 .dev = &udc_dev,
2246
2247                 .bEndpointAddress = 1 | USB_DIR_IN,
2248                 .bmAttributes = USB_ENDPOINT_XFER_BULK,
2249
2250                 .type = ep_bulk_in,
2251                 .fifo = JZ_REG_UDC_EP_FIFO(1),
2252                 .csr = JZ_REG_UDC_INCSR,
2253         },
2254
2255         /* interrupt in endpoint */
2256         .ep[3] = {
2257                 .ep = {
2258                         .name = "ep2in-int",
2259                         .ops = &jz4740_ep_ops,
2260                         .maxpacket = EPINTR_MAXPACKETSIZE,
2261                 },
2262                 .dev = &udc_dev,
2263
2264                 .bEndpointAddress = 2 | USB_DIR_IN,
2265                 .bmAttributes = USB_ENDPOINT_XFER_INT,
2266
2267                 .type = ep_interrupt,
2268                 .fifo = JZ_REG_UDC_EP_FIFO(2),
2269                 .csr = JZ_REG_UDC_INCSR,
2270         },
2271 };
2272
2273 static void gadget_release(struct device *_dev)
2274 {
2275 }
2276
2277
2278 static int jz4740_udc_probe(struct platform_device *pdev)
2279 {
2280         struct jz4740_udc *dev = &udc_dev;
2281         int ret;
2282
2283         spin_lock_init(&dev->lock);
2284         the_controller = dev;
2285
2286         dev->dev = &pdev->dev;
2287         dev_set_name(&dev->gadget.dev, "gadget");
2288         dev->gadget.dev.parent = &pdev->dev;
2289         dev->gadget.dev.dma_mask = pdev->dev.dma_mask;
2290         dev->gadget.dev.release = gadget_release;
2291
2292         ret = device_register(&dev->gadget.dev);
2293         if (ret)
2294                 return ret;
2295
2296         dev->clk = clk_get(&pdev->dev, "udc");
2297         if (IS_ERR(dev->clk)) {
2298                 ret = PTR_ERR(dev->clk);
2299                 dev_err(&pdev->dev, "Failed to get udc clock: %d\n", ret);
2300                 goto err_device_unregister;
2301         }
2302
2303         platform_set_drvdata(pdev, dev);
2304
2305         dev->mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2306
2307         if (!dev->mem) {
2308                 ret = -ENOENT;
2309                 dev_err(&pdev->dev, "Failed to get mmio memory resource\n");
2310                 goto err_clk_put;
2311         }
2312
2313         dev->mem = request_mem_region(dev->mem->start, resource_size(dev->mem), pdev->name);
2314
2315         if (!dev->mem) {
2316                 ret = -EBUSY;
2317                 dev_err(&pdev->dev, "Failed to request mmio memory region\n");
2318                 goto err_device_unregister;
2319         }
2320
2321         dev->base = ioremap(dev->mem->start, resource_size(dev->mem));
2322
2323         if (!dev->base) {
2324                 ret = -EBUSY;
2325                 dev_err(&pdev->dev, "Failed to ioremap mmio memory\n");
2326                 goto err_release_mem_region;
2327         }
2328
2329         dev->irq = platform_get_irq(pdev, 0);
2330
2331         ret = request_irq(dev->irq, jz4740_udc_irq, IRQF_DISABLED,
2332                             pdev->name, dev);
2333         if (ret) {
2334                 dev_err(&pdev->dev, "Failed to request irq: %d\n", ret);
2335                 goto err_iounmap;
2336         }
2337
2338         udc_disable(dev);
2339         udc_reinit(dev);
2340
2341         return 0;
2342
2343 err_iounmap:
2344         iounmap(dev->base);
2345 err_release_mem_region:
2346         release_mem_region(dev->mem->start, resource_size(dev->mem));
2347 err_clk_put:
2348         clk_put(dev->clk);
2349 err_device_unregister:
2350         device_unregister(&dev->gadget.dev);
2351         platform_set_drvdata(pdev, NULL);
2352
2353         the_controller = 0;
2354
2355         return ret;
2356 }
2357
2358 static int jz4740_udc_remove(struct platform_device *pdev)
2359 {
2360         struct jz4740_udc *dev = platform_get_drvdata(pdev);
2361
2362         if (dev->driver)
2363                 return -EBUSY;
2364
2365         udc_disable(dev);
2366
2367         free_irq(dev->irq, dev);
2368         iounmap(dev->base);
2369         release_mem_region(dev->mem->start, resource_size(dev->mem));
2370         clk_put(dev->clk);
2371
2372         platform_set_drvdata(pdev, NULL);
2373         device_unregister(&dev->gadget.dev);
2374         the_controller = NULL;
2375
2376         return 0;
2377 }
2378
2379 #ifdef CONFIG_PM
2380
2381 static int jz4740_udc_suspend(struct device *dev)
2382 {
2383         struct jz4740_udc *udc = dev_get_drvdata(dev);
2384
2385         if (udc->state == UDC_STATE_ENABLE)
2386                 udc_disable(udc);
2387
2388         return 0;
2389 }
2390
2391 static int jz4740_udc_resume(struct device *dev)
2392 {
2393         struct jz4740_udc *udc = dev_get_drvdata(dev);
2394
2395         if (udc->state == UDC_STATE_ENABLE)
2396                 udc_enable(udc);
2397
2398         return 0;
2399 }
2400
2401 static struct dev_pm_ops jz4740_udc_pm_ops = {
2402         .suspend = jz4740_udc_suspend,
2403         .resume = jz4740_udc_resume,
2404 };
2405
2406 #define JZ4740_UDC_PM_OPS (&jz4740_udc_pm_ops)
2407
2408 #else
2409
2410 #define JZ4740_UDC_PM_OPS NULL
2411
2412 #endif
2413
2414 static struct platform_driver udc_driver = {
2415         .probe          = jz4740_udc_probe,
2416         .remove         = jz4740_udc_remove,
2417         .driver         = {
2418                 .name   = "jz-udc",
2419                 .owner  = THIS_MODULE,
2420                 .pm             = JZ4740_UDC_PM_OPS,
2421         },
2422 };
2423
2424 /*-------------------------------------------------------------------------*/
2425
2426 static int __init udc_init (void)
2427 {
2428         return platform_driver_register(&udc_driver);
2429 }
2430 module_init(udc_init);
2431
2432 static void __exit udc_exit (void)
2433 {
2434         platform_driver_unregister(&udc_driver);
2435 }
2436 module_exit(udc_exit);
2437
2438 MODULE_DESCRIPTION("JZ4740 USB Device Controller");
2439 MODULE_AUTHOR("Wei Jianli <jlwei@ingenic.cn>");
2440 MODULE_LICENSE("GPL");