Pileus Git - ~andy/linux/blob - drivers/usb/gadget/amd5536udc.c

   1 /*
   2  * amd5536.c -- AMD 5536 UDC high/full speed USB device controller
   3  *
   4  * Copyright (C) 2005-2007 AMD (http://www.amd.com)
   5  * Author: Thomas Dahlmann
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  */
  12
  13 /*
  14  * The AMD5536 UDC is part of the x86 southbridge AMD Geode CS5536.
  15  * It is a USB Highspeed DMA capable USB device controller. Beside ep0 it
  16  * provides 4 IN and 4 OUT endpoints (bulk or interrupt type).
  17  *
  18  * Make sure that UDC is assigned to port 4 by BIOS settings (port can also
  19  * be used as host port) and UOC bits PAD_EN and APU are set (should be done
  20  * by BIOS init).
  21  *
  22  * UDC DMA requires 32-bit aligned buffers so DMA with gadget ether does not
  23  * work without updating NET_IP_ALIGN. Or PIO mode (module param "use_dma=0")
  24  * can be used with gadget ether.
  25  */
  26
  27 /* debug control */
  28 /* #define UDC_VERBOSE */
  29
  30 /* Driver strings */
  31 #define UDC_MOD_DESCRIPTION             "AMD 5536 UDC - USB Device Controller"
  32 #define UDC_DRIVER_VERSION_STRING       "01.00.0206"
  33
  34 /* system */
  35 #include <linux/module.h>
  36 #include <linux/pci.h>
  37 #include <linux/kernel.h>
  38 #include <linux/delay.h>
  39 #include <linux/ioport.h>
  40 #include <linux/sched.h>
  41 #include <linux/slab.h>
  42 #include <linux/errno.h>
  43 #include <linux/init.h>
  44 #include <linux/timer.h>
  45 #include <linux/list.h>
  46 #include <linux/interrupt.h>
  47 #include <linux/ioctl.h>
  48 #include <linux/fs.h>
  49 #include <linux/dmapool.h>
  50 #include <linux/moduleparam.h>
  51 #include <linux/device.h>
  52 #include <linux/io.h>
  53 #include <linux/irq.h>
  54 #include <linux/prefetch.h>
  55
  56 #include <asm/byteorder.h>
  57 #include <asm/system.h>
  58 #include <asm/unaligned.h>
  59
  60 /* gadget stack */
  61 #include <linux/usb/ch9.h>
  62 #include <linux/usb/gadget.h>
  63
  64 /* udc specific */
  65 #include "amd5536udc.h"
  66
  67
  68 static void udc_tasklet_disconnect(unsigned long);
  69 static void empty_req_queue(struct udc_ep *);
  70 static int udc_probe(struct udc *dev);
  71 static void udc_basic_init(struct udc *dev);
  72 static void udc_setup_endpoints(struct udc *dev);
  73 static void udc_soft_reset(struct udc *dev);
  74 static struct udc_request *udc_alloc_bna_dummy(struct udc_ep *ep);
  75 static void udc_free_request(struct usb_ep *usbep, struct usb_request *usbreq);
  76 static int udc_free_dma_chain(struct udc *dev, struct udc_request *req);
  77 static int udc_create_dma_chain(struct udc_ep *ep, struct udc_request *req,
  78                                 unsigned long buf_len, gfp_t gfp_flags);
  79 static int udc_remote_wakeup(struct udc *dev);
  80 static int udc_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id);
  81 static void udc_pci_remove(struct pci_dev *pdev);
  82
  83 /* description */
  84 static const char mod_desc[] = UDC_MOD_DESCRIPTION;
  85 static const char name[] = "amd5536udc";
  86
  87 /* structure to hold endpoint function pointers */
  88 static const struct usb_ep_ops udc_ep_ops;
  89
  90 /* received setup data */
  91 static union udc_setup_data setup_data;
  92
  93 /* pointer to device object */
  94 static struct udc *udc;
  95
  96 /* irq spin lock for soft reset */
  97 static DEFINE_SPINLOCK(udc_irq_spinlock);
  98 /* stall spin lock */
  99 static DEFINE_SPINLOCK(udc_stall_spinlock);
 100
 101 /*
 102 * slave mode: pending bytes in rx fifo after nyet,
 103 * used if EPIN irq came but no req was available
 104 */
 105 static unsigned int udc_rxfifo_pending;
 106
 107 /* count soft resets after suspend to avoid loop */
 108 static int soft_reset_occured;
 109 static int soft_reset_after_usbreset_occured;
 110
 111 /* timer */
 112 static struct timer_list udc_timer;
 113 static int stop_timer;
 114
 115 /* set_rde -- Is used to control enabling of RX DMA. Problem is
 116  * that UDC has only one bit (RDE) to enable/disable RX DMA for
 117  * all OUT endpoints. So we have to handle race conditions like
 118  * when OUT data reaches the fifo but no request was queued yet.
 119  * This cannot be solved by letting the RX DMA disabled until a
 120  * request gets queued because there may be other OUT packets
 121  * in the FIFO (important for not blocking control traffic).
 122  * The value of set_rde controls the correspondig timer.
 123  *
 124  * set_rde -1 == not used, means it is alloed to be set to 0 or 1
 125  * set_rde  0 == do not touch RDE, do no start the RDE timer
 126  * set_rde  1 == timer function will look whether FIFO has data
 127  * set_rde  2 == set by timer function to enable RX DMA on next call
 128  */
 129 static int set_rde = -1;
 130
 131 static DECLARE_COMPLETION(on_exit);
 132 static struct timer_list udc_pollstall_timer;
 133 static int stop_pollstall_timer;
 134 static DECLARE_COMPLETION(on_pollstall_exit);
 135
 136 /* tasklet for usb disconnect */
 137 static DECLARE_TASKLET(disconnect_tasklet, udc_tasklet_disconnect,
 138                 (unsigned long) &udc);
 139
 140
 141 /* endpoint names used for print */
 142 static const char ep0_string[] = "ep0in";
 143 static const char *const ep_string[] = {
 144         ep0_string,
 145         "ep1in-int", "ep2in-bulk", "ep3in-bulk", "ep4in-bulk", "ep5in-bulk",
 146         "ep6in-bulk", "ep7in-bulk", "ep8in-bulk", "ep9in-bulk", "ep10in-bulk",
 147         "ep11in-bulk", "ep12in-bulk", "ep13in-bulk", "ep14in-bulk",
 148         "ep15in-bulk", "ep0out", "ep1out-bulk", "ep2out-bulk", "ep3out-bulk",
 149         "ep4out-bulk", "ep5out-bulk", "ep6out-bulk", "ep7out-bulk",
 150         "ep8out-bulk", "ep9out-bulk", "ep10out-bulk", "ep11out-bulk",
 151         "ep12out-bulk", "ep13out-bulk", "ep14out-bulk", "ep15out-bulk"
 152 };
 153
 154 /* DMA usage flag */
 155 static bool use_dma = 1;
 156 /* packet per buffer dma */
 157 static bool use_dma_ppb = 1;
 158 /* with per descr. update */
 159 static bool use_dma_ppb_du;
 160 /* buffer fill mode */
 161 static int use_dma_bufferfill_mode;
 162 /* full speed only mode */
 163 static bool use_fullspeed;
 164 /* tx buffer size for high speed */
 165 static unsigned long hs_tx_buf = UDC_EPIN_BUFF_SIZE;
 166
 167 /* module parameters */
 168 module_param(use_dma, bool, S_IRUGO);
 169 MODULE_PARM_DESC(use_dma, "true for DMA");
 170 module_param(use_dma_ppb, bool, S_IRUGO);
 171 MODULE_PARM_DESC(use_dma_ppb, "true for DMA in packet per buffer mode");
 172 module_param(use_dma_ppb_du, bool, S_IRUGO);
 173 MODULE_PARM_DESC(use_dma_ppb_du,
 174         "true for DMA in packet per buffer mode with descriptor update");
 175 module_param(use_fullspeed, bool, S_IRUGO);
 176 MODULE_PARM_DESC(use_fullspeed, "true for fullspeed only");
 177
 178 /*---------------------------------------------------------------------------*/
 179 /* Prints UDC device registers and endpoint irq registers */
 180 static void print_regs(struct udc *dev)
 181 {
 182         DBG(dev, "------- Device registers -------\n");
 183         DBG(dev, "dev config     = %08x\n", readl(&dev->regs->cfg));
 184         DBG(dev, "dev control    = %08x\n", readl(&dev->regs->ctl));
 185         DBG(dev, "dev status     = %08x\n", readl(&dev->regs->sts));
 186         DBG(dev, "\n");
 187         DBG(dev, "dev int's      = %08x\n", readl(&dev->regs->irqsts));
 188         DBG(dev, "dev intmask    = %08x\n", readl(&dev->regs->irqmsk));
 189         DBG(dev, "\n");
 190         DBG(dev, "dev ep int's   = %08x\n", readl(&dev->regs->ep_irqsts));
 191         DBG(dev, "dev ep intmask = %08x\n", readl(&dev->regs->ep_irqmsk));
 192         DBG(dev, "\n");
 193         DBG(dev, "USE DMA        = %d\n", use_dma);
 194         if (use_dma && use_dma_ppb && !use_dma_ppb_du) {
 195                 DBG(dev, "DMA mode       = PPBNDU (packet per buffer "
 196                         "WITHOUT desc. update)\n");
 197                 dev_info(&dev->pdev->dev, "DMA mode (%s)\n", "PPBNDU");
 198         } else if (use_dma && use_dma_ppb && use_dma_ppb_du) {
 199                 DBG(dev, "DMA mode       = PPBDU (packet per buffer "
 200                         "WITH desc. update)\n");
 201                 dev_info(&dev->pdev->dev, "DMA mode (%s)\n", "PPBDU");
 202         }
 203         if (use_dma && use_dma_bufferfill_mode) {
 204                 DBG(dev, "DMA mode       = BF (buffer fill mode)\n");
 205                 dev_info(&dev->pdev->dev, "DMA mode (%s)\n", "BF");
 206         }
 207         if (!use_dma)
 208                 dev_info(&dev->pdev->dev, "FIFO mode\n");
 209         DBG(dev, "-------------------------------------------------------\n");
 210 }
 211
 212 /* Masks unused interrupts */
 213 static int udc_mask_unused_interrupts(struct udc *dev)
 214 {
 215         u32 tmp;
 216
 217         /* mask all dev interrupts */
 218         tmp =   AMD_BIT(UDC_DEVINT_SVC) |
 219                 AMD_BIT(UDC_DEVINT_ENUM) |
 220                 AMD_BIT(UDC_DEVINT_US) |
 221                 AMD_BIT(UDC_DEVINT_UR) |
 222                 AMD_BIT(UDC_DEVINT_ES) |
 223                 AMD_BIT(UDC_DEVINT_SI) |
 224                 AMD_BIT(UDC_DEVINT_SOF)|
 225                 AMD_BIT(UDC_DEVINT_SC);
 226         writel(tmp, &dev->regs->irqmsk);
 227
 228         /* mask all ep interrupts */
 229         writel(UDC_EPINT_MSK_DISABLE_ALL, &dev->regs->ep_irqmsk);
 230
 231         return 0;
 232 }
 233
 234 /* Enables endpoint 0 interrupts */
 235 static int udc_enable_ep0_interrupts(struct udc *dev)
 236 {
 237         u32 tmp;
 238
 239         DBG(dev, "udc_enable_ep0_interrupts()\n");
 240
 241         /* read irq mask */
 242         tmp = readl(&dev->regs->ep_irqmsk);
 243         /* enable ep0 irq's */
 244         tmp &= AMD_UNMASK_BIT(UDC_EPINT_IN_EP0)
 245                 & AMD_UNMASK_BIT(UDC_EPINT_OUT_EP0);
 246         writel(tmp, &dev->regs->ep_irqmsk);
 247
 248         return 0;
 249 }
 250
 251 /* Enables device interrupts for SET_INTF and SET_CONFIG */
 252 static int udc_enable_dev_setup_interrupts(struct udc *dev)
 253 {
 254         u32 tmp;
 255
 256         DBG(dev, "enable device interrupts for setup data\n");
 257
 258         /* read irq mask */
 259         tmp = readl(&dev->regs->irqmsk);
 260
 261         /* enable SET_INTERFACE, SET_CONFIG and other needed irq's */
 262         tmp &= AMD_UNMASK_BIT(UDC_DEVINT_SI)
 263                 & AMD_UNMASK_BIT(UDC_DEVINT_SC)
 264                 & AMD_UNMASK_BIT(UDC_DEVINT_UR)
 265                 & AMD_UNMASK_BIT(UDC_DEVINT_SVC)
 266                 & AMD_UNMASK_BIT(UDC_DEVINT_ENUM);
 267         writel(tmp, &dev->regs->irqmsk);
 268
 269         return 0;
 270 }
 271
 272 /* Calculates fifo start of endpoint based on preceding endpoints */
 273 static int udc_set_txfifo_addr(struct udc_ep *ep)
 274 {
 275         struct udc      *dev;
 276         u32 tmp;
 277         int i;
 278
 279         if (!ep || !(ep->in))
 280                 return -EINVAL;
 281
 282         dev = ep->dev;
 283         ep->txfifo = dev->txfifo;
 284
 285         /* traverse ep's */
 286         for (i = 0; i < ep->num; i++) {
 287                 if (dev->ep[i].regs) {
 288                         /* read fifo size */
 289                         tmp = readl(&dev->ep[i].regs->bufin_framenum);
 290                         tmp = AMD_GETBITS(tmp, UDC_EPIN_BUFF_SIZE);
 291                         ep->txfifo += tmp;
 292                 }
 293         }
 294         return 0;
 295 }
 296
 297 /* CNAK pending field: bit0 = ep0in, bit16 = ep0out */
 298 static u32 cnak_pending;
 299
 300 static void UDC_QUEUE_CNAK(struct udc_ep *ep, unsigned num)
 301 {
 302         if (readl(&ep->regs->ctl) & AMD_BIT(UDC_EPCTL_NAK)) {
 303                 DBG(ep->dev, "NAK could not be cleared for ep%d\n", num);
 304                 cnak_pending |= 1 << (num);
 305                 ep->naking = 1;
 306         } else
 307                 cnak_pending = cnak_pending & (~(1 << (num)));
 308 }
 309
 310
 311 /* Enables endpoint, is called by gadget driver */
 312 static int
 313 udc_ep_enable(struct usb_ep *usbep, const struct usb_endpoint_descriptor *desc)
 314 {
 315         struct udc_ep           *ep;
 316         struct udc              *dev;
 317         u32                     tmp;
 318         unsigned long           iflags;
 319         u8 udc_csr_epix;
 320         unsigned                maxpacket;
 321
 322         if (!usbep
 323                         || usbep->name == ep0_string
 324                         || !desc
 325                         || desc->bDescriptorType != USB_DT_ENDPOINT)
 326                 return -EINVAL;
 327
 328         ep = container_of(usbep, struct udc_ep, ep);
 329         dev = ep->dev;
 330
 331         DBG(dev, "udc_ep_enable() ep %d\n", ep->num);
 332
 333         if (!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN)
 334                 return -ESHUTDOWN;
 335
 336         spin_lock_irqsave(&dev->lock, iflags);
 337         ep->desc = desc;
 338
 339         ep->halted = 0;
 340
 341         /* set traffic type */
 342         tmp = readl(&dev->ep[ep->num].regs->ctl);
 343         tmp = AMD_ADDBITS(tmp, desc->bmAttributes, UDC_EPCTL_ET);
 344         writel(tmp, &dev->ep[ep->num].regs->ctl);
 345
 346         /* set max packet size */
 347         maxpacket = usb_endpoint_maxp(desc);
 348         tmp = readl(&dev->ep[ep->num].regs->bufout_maxpkt);
 349         tmp = AMD_ADDBITS(tmp, maxpacket, UDC_EP_MAX_PKT_SIZE);
 350         ep->ep.maxpacket = maxpacket;
 351         writel(tmp, &dev->ep[ep->num].regs->bufout_maxpkt);
 352
 353         /* IN ep */
 354         if (ep->in) {
 355
 356                 /* ep ix in UDC CSR register space */
 357                 udc_csr_epix = ep->num;
 358
 359                 /* set buffer size (tx fifo entries) */
 360                 tmp = readl(&dev->ep[ep->num].regs->bufin_framenum);
 361                 /* double buffering: fifo size = 2 x max packet size */
 362                 tmp = AMD_ADDBITS(
 363                                 tmp,
 364                                 maxpacket * UDC_EPIN_BUFF_SIZE_MULT
 365                                           / UDC_DWORD_BYTES,
 366                                 UDC_EPIN_BUFF_SIZE);
 367                 writel(tmp, &dev->ep[ep->num].regs->bufin_framenum);
 368
 369                 /* calc. tx fifo base addr */
 370                 udc_set_txfifo_addr(ep);
 371
 372                 /* flush fifo */
 373                 tmp = readl(&ep->regs->ctl);
 374                 tmp |= AMD_BIT(UDC_EPCTL_F);
 375                 writel(tmp, &ep->regs->ctl);
 376
 377         /* OUT ep */
 378         } else {
 379                 /* ep ix in UDC CSR register space */
 380                 udc_csr_epix = ep->num - UDC_CSR_EP_OUT_IX_OFS;
 381
 382                 /* set max packet size UDC CSR  */
 383                 tmp = readl(&dev->csr->ne[ep->num - UDC_CSR_EP_OUT_IX_OFS]);
 384                 tmp = AMD_ADDBITS(tmp, maxpacket,
 385                                         UDC_CSR_NE_MAX_PKT);
 386                 writel(tmp, &dev->csr->ne[ep->num - UDC_CSR_EP_OUT_IX_OFS]);
 387
 388                 if (use_dma && !ep->in) {
 389                         /* alloc and init BNA dummy request */
 390                         ep->bna_dummy_req = udc_alloc_bna_dummy(ep);
 391                         ep->bna_occurred = 0;
 392                 }
 393
 394                 if (ep->num != UDC_EP0OUT_IX)
 395                         dev->data_ep_enabled = 1;
 396         }
 397
 398         /* set ep values */
 399         tmp = readl(&dev->csr->ne[udc_csr_epix]);
 400         /* max packet */
 401         tmp = AMD_ADDBITS(tmp, maxpacket, UDC_CSR_NE_MAX_PKT);
 402         /* ep number */
 403         tmp = AMD_ADDBITS(tmp, desc->bEndpointAddress, UDC_CSR_NE_NUM);
 404         /* ep direction */
 405         tmp = AMD_ADDBITS(tmp, ep->in, UDC_CSR_NE_DIR);
 406         /* ep type */
 407         tmp = AMD_ADDBITS(tmp, desc->bmAttributes, UDC_CSR_NE_TYPE);
 408         /* ep config */
 409         tmp = AMD_ADDBITS(tmp, ep->dev->cur_config, UDC_CSR_NE_CFG);
 410         /* ep interface */
 411         tmp = AMD_ADDBITS(tmp, ep->dev->cur_intf, UDC_CSR_NE_INTF);
 412         /* ep alt */
 413         tmp = AMD_ADDBITS(tmp, ep->dev->cur_alt, UDC_CSR_NE_ALT);
 414         /* write reg */
 415         writel(tmp, &dev->csr->ne[udc_csr_epix]);
 416
 417         /* enable ep irq */
 418         tmp = readl(&dev->regs->ep_irqmsk);
 419         tmp &= AMD_UNMASK_BIT(ep->num);
 420         writel(tmp, &dev->regs->ep_irqmsk);
 421
 422         /*
 423          * clear NAK by writing CNAK
 424          * avoid BNA for OUT DMA, don't clear NAK until DMA desc. written
 425          */
 426         if (!use_dma || ep->in) {
 427                 tmp = readl(&ep->regs->ctl);
 428                 tmp |= AMD_BIT(UDC_EPCTL_CNAK);
 429                 writel(tmp, &ep->regs->ctl);
 430                 ep->naking = 0;
 431                 UDC_QUEUE_CNAK(ep, ep->num);
 432         }
 433         tmp = desc->bEndpointAddress;
 434         DBG(dev, "%s enabled\n", usbep->name);
 435
 436         spin_unlock_irqrestore(&dev->lock, iflags);
 437         return 0;
 438 }
 439
 440 /* Resets endpoint */
 441 static void ep_init(struct udc_regs __iomem *regs, struct udc_ep *ep)
 442 {
 443         u32             tmp;
 444
 445         VDBG(ep->dev, "ep-%d reset\n", ep->num);
 446         ep->desc = NULL;
 447         ep->ep.desc = NULL;
 448         ep->ep.ops = &udc_ep_ops;
 449         INIT_LIST_HEAD(&ep->queue);
 450
 451         ep->ep.maxpacket = (u16) ~0;
 452         /* set NAK */
 453         tmp = readl(&ep->regs->ctl);
 454         tmp |= AMD_BIT(UDC_EPCTL_SNAK);
 455         writel(tmp, &ep->regs->ctl);
 456         ep->naking = 1;
 457
 458         /* disable interrupt */
 459         tmp = readl(&regs->ep_irqmsk);
 460         tmp |= AMD_BIT(ep->num);
 461         writel(tmp, &regs->ep_irqmsk);
 462
 463         if (ep->in) {
 464                 /* unset P and IN bit of potential former DMA */
 465                 tmp = readl(&ep->regs->ctl);
 466                 tmp &= AMD_UNMASK_BIT(UDC_EPCTL_P);
 467                 writel(tmp, &ep->regs->ctl);
 468
 469                 tmp = readl(&ep->regs->sts);
 470                 tmp |= AMD_BIT(UDC_EPSTS_IN);
 471                 writel(tmp, &ep->regs->sts);
 472
 473                 /* flush the fifo */
 474                 tmp = readl(&ep->regs->ctl);
 475                 tmp |= AMD_BIT(UDC_EPCTL_F);
 476                 writel(tmp, &ep->regs->ctl);
 477
 478         }
 479         /* reset desc pointer */
 480         writel(0, &ep->regs->desptr);
 481 }
 482
 483 /* Disables endpoint, is called by gadget driver */
 484 static int udc_ep_disable(struct usb_ep *usbep)
 485 {
 486         struct udc_ep   *ep = NULL;
 487         unsigned long   iflags;
 488
 489         if (!usbep)
 490                 return -EINVAL;
 491
 492         ep = container_of(usbep, struct udc_ep, ep);
 493         if (usbep->name == ep0_string || !ep->desc)
 494                 return -EINVAL;
 495
 496         DBG(ep->dev, "Disable ep-%d\n", ep->num);
 497
 498         spin_lock_irqsave(&ep->dev->lock, iflags);
 499         udc_free_request(&ep->ep, &ep->bna_dummy_req->req);
 500         empty_req_queue(ep);
 501         ep_init(ep->dev->regs, ep);
 502         spin_unlock_irqrestore(&ep->dev->lock, iflags);
 503
 504         return 0;
 505 }
 506
 507 /* Allocates request packet, called by gadget driver */
 508 static struct usb_request *
 509 udc_alloc_request(struct usb_ep *usbep, gfp_t gfp)
 510 {
 511         struct udc_request      *req;
 512         struct udc_data_dma     *dma_desc;
 513         struct udc_ep   *ep;
 514
 515         if (!usbep)
 516                 return NULL;
 517
 518         ep = container_of(usbep, struct udc_ep, ep);
 519
 520         VDBG(ep->dev, "udc_alloc_req(): ep%d\n", ep->num);
 521         req = kzalloc(sizeof(struct udc_request), gfp);
 522         if (!req)
 523                 return NULL;
 524
 525         req->req.dma = DMA_DONT_USE;
 526         INIT_LIST_HEAD(&req->queue);
 527
 528         if (ep->dma) {
 529                 /* ep0 in requests are allocated from data pool here */
 530                 dma_desc = pci_pool_alloc(ep->dev->data_requests, gfp,
 531                                                 &req->td_phys);
 532                 if (!dma_desc) {
 533                         kfree(req);
 534                         return NULL;
 535                 }
 536
 537                 VDBG(ep->dev, "udc_alloc_req: req = %p dma_desc = %p, "
 538                                 "td_phys = %lx\n",
 539                                 req, dma_desc,
 540                                 (unsigned long)req->td_phys);
 541                 /* prevent from using desc. - set HOST BUSY */
 542                 dma_desc->status = AMD_ADDBITS(dma_desc->status,
 543                                                 UDC_DMA_STP_STS_BS_HOST_BUSY,
 544                                                 UDC_DMA_STP_STS_BS);
 545                 dma_desc->bufptr = cpu_to_le32(DMA_DONT_USE);
 546                 req->td_data = dma_desc;
 547                 req->td_data_last = NULL;
 548                 req->chain_len = 1;
 549         }
 550
 551         return &req->req;
 552 }
 553
 554 /* Frees request packet, called by gadget driver */
 555 static void
 556 udc_free_request(struct usb_ep *usbep, struct usb_request *usbreq)
 557 {
 558         struct udc_ep   *ep;
 559         struct udc_request      *req;
 560
 561         if (!usbep || !usbreq)
 562                 return;
 563
 564         ep = container_of(usbep, struct udc_ep, ep);
 565         req = container_of(usbreq, struct udc_request, req);
 566         VDBG(ep->dev, "free_req req=%p\n", req);
 567         BUG_ON(!list_empty(&req->queue));
 568         if (req->td_data) {
 569                 VDBG(ep->dev, "req->td_data=%p\n", req->td_data);
 570
 571                 /* free dma chain if created */
 572                 if (req->chain_len > 1)
 573                         udc_free_dma_chain(ep->dev, req);
 574
 575                 pci_pool_free(ep->dev->data_requests, req->td_data,
 576                                                         req->td_phys);
 577         }
 578         kfree(req);
 579 }
 580
 581 /* Init BNA dummy descriptor for HOST BUSY and pointing to itself */
 582 static void udc_init_bna_dummy(struct udc_request *req)
 583 {
 584         if (req) {
 585                 /* set last bit */
 586                 req->td_data->status |= AMD_BIT(UDC_DMA_IN_STS_L);
 587                 /* set next pointer to itself */
 588                 req->td_data->next = req->td_phys;
 589                 /* set HOST BUSY */
 590                 req->td_data->status
 591                         = AMD_ADDBITS(req->td_data->status,
 592                                         UDC_DMA_STP_STS_BS_DMA_DONE,
 593                                         UDC_DMA_STP_STS_BS);
 594 #ifdef UDC_VERBOSE
 595                 pr_debug("bna desc = %p, sts = %08x\n",
 596                         req->td_data, req->td_data->status);
 597 #endif
 598         }
 599 }
 600
 601 /* Allocate BNA dummy descriptor */
 602 static struct udc_request *udc_alloc_bna_dummy(struct udc_ep *ep)
 603 {
 604         struct udc_request *req = NULL;
 605         struct usb_request *_req = NULL;
 606
 607         /* alloc the dummy request */
 608         _req = udc_alloc_request(&ep->ep, GFP_ATOMIC);
 609         if (_req) {
 610                 req = container_of(_req, struct udc_request, req);
 611                 ep->bna_dummy_req = req;
 612                 udc_init_bna_dummy(req);
 613         }
 614         return req;
 615 }
 616
 617 /* Write data to TX fifo for IN packets */
 618 static void
 619 udc_txfifo_write(struct udc_ep *ep, struct usb_request *req)
 620 {
 621         u8                      *req_buf;
 622         u32                     *buf;
 623         int                     i, j;
 624         unsigned                bytes = 0;
 625         unsigned                remaining = 0;
 626
 627         if (!req || !ep)
 628                 return;
 629
 630         req_buf = req->buf + req->actual;
 631         prefetch(req_buf);
 632         remaining = req->length - req->actual;
 633
 634         buf = (u32 *) req_buf;
 635
 636         bytes = ep->ep.maxpacket;
 637         if (bytes > remaining)
 638                 bytes = remaining;
 639
 640         /* dwords first */
 641         for (i = 0; i < bytes / UDC_DWORD_BYTES; i++)
 642                 writel(*(buf + i), ep->txfifo);
 643
 644         /* remaining bytes must be written by byte access */
 645         for (j = 0; j < bytes % UDC_DWORD_BYTES; j++) {
 646                 writeb((u8)(*(buf + i) >> (j << UDC_BITS_PER_BYTE_SHIFT)),
 647                                                         ep->txfifo);
 648         }
 649
 650         /* dummy write confirm */
 651         writel(0, &ep->regs->confirm);
 652 }
 653
 654 /* Read dwords from RX fifo for OUT transfers */
 655 static int udc_rxfifo_read_dwords(struct udc *dev, u32 *buf, int dwords)
 656 {
 657         int i;
 658
 659         VDBG(dev, "udc_read_dwords(): %d dwords\n", dwords);
 660
 661         for (i = 0; i < dwords; i++)
 662                 *(buf + i) = readl(dev->rxfifo);
 663         return 0;
 664 }
 665
 666 /* Read bytes from RX fifo for OUT transfers */
 667 static int udc_rxfifo_read_bytes(struct udc *dev, u8 *buf, int bytes)
 668 {
 669         int i, j;
 670         u32 tmp;
 671
 672         VDBG(dev, "udc_read_bytes(): %d bytes\n", bytes);
 673
 674         /* dwords first */
 675         for (i = 0; i < bytes / UDC_DWORD_BYTES; i++)
 676                 *((u32 *)(buf + (i<<2))) = readl(dev->rxfifo);
 677
 678         /* remaining bytes must be read by byte access */
 679         if (bytes % UDC_DWORD_BYTES) {
 680                 tmp = readl(dev->rxfifo);
 681                 for (j = 0; j < bytes % UDC_DWORD_BYTES; j++) {
 682                         *(buf + (i<<2) + j) = (u8)(tmp & UDC_BYTE_MASK);
 683                         tmp = tmp >> UDC_BITS_PER_BYTE;
 684                 }
 685         }
 686
 687         return 0;
 688 }
 689
 690 /* Read data from RX fifo for OUT transfers */
 691 static int
 692 udc_rxfifo_read(struct udc_ep *ep, struct udc_request *req)
 693 {
 694         u8 *buf;
 695         unsigned buf_space;
 696         unsigned bytes = 0;
 697         unsigned finished = 0;
 698
 699         /* received number bytes */
 700         bytes = readl(&ep->regs->sts);
 701         bytes = AMD_GETBITS(bytes, UDC_EPSTS_RX_PKT_SIZE);
 702
 703         buf_space = req->req.length - req->req.actual;
 704         buf = req->req.buf + req->req.actual;
 705         if (bytes > buf_space) {
 706                 if ((buf_space % ep->ep.maxpacket) != 0) {
 707                         DBG(ep->dev,
 708                                 "%s: rx %d bytes, rx-buf space = %d bytesn\n",
 709                                 ep->ep.name, bytes, buf_space);
 710                         req->req.status = -EOVERFLOW;
 711                 }
 712                 bytes = buf_space;
 713         }
 714         req->req.actual += bytes;
 715
 716         /* last packet ? */
 717         if (((bytes % ep->ep.maxpacket) != 0) || (!bytes)
 718                 || ((req->req.actual == req->req.length) && !req->req.zero))
 719                 finished = 1;
 720
 721         /* read rx fifo bytes */
 722         VDBG(ep->dev, "ep %s: rxfifo read %d bytes\n", ep->ep.name, bytes);
 723         udc_rxfifo_read_bytes(ep->dev, buf, bytes);
 724
 725         return finished;
 726 }
 727
 728 /* create/re-init a DMA descriptor or a DMA descriptor chain */
 729 static int prep_dma(struct udc_ep *ep, struct udc_request *req, gfp_t gfp)
 730 {
 731         int     retval = 0;
 732         u32     tmp;
 733
 734         VDBG(ep->dev, "prep_dma\n");
 735         VDBG(ep->dev, "prep_dma ep%d req->td_data=%p\n",
 736                         ep->num, req->td_data);
 737
 738         /* set buffer pointer */
 739         req->td_data->bufptr = req->req.dma;
 740
 741         /* set last bit */
 742         req->td_data->status |= AMD_BIT(UDC_DMA_IN_STS_L);
 743
 744         /* build/re-init dma chain if maxpkt scatter mode, not for EP0 */
 745         if (use_dma_ppb) {
 746
 747                 retval = udc_create_dma_chain(ep, req, ep->ep.maxpacket, gfp);
 748                 if (retval != 0) {
 749                         if (retval == -ENOMEM)
 750                                 DBG(ep->dev, "Out of DMA memory\n");
 751                         return retval;
 752                 }
 753                 if (ep->in) {
 754                         if (req->req.length == ep->ep.maxpacket) {
 755                                 /* write tx bytes */
 756                                 req->td_data->status =
 757                                         AMD_ADDBITS(req->td_data->status,
 758                                                 ep->ep.maxpacket,
 759                                                 UDC_DMA_IN_STS_TXBYTES);
 760
 761                         }
 762                 }
 763
 764         }
 765
 766         if (ep->in) {
 767                 VDBG(ep->dev, "IN: use_dma_ppb=%d req->req.len=%d "
 768                                 "maxpacket=%d ep%d\n",
 769                                 use_dma_ppb, req->req.length,
 770                                 ep->ep.maxpacket, ep->num);
 771                 /*
 772                  * if bytes < max packet then tx bytes must
 773                  * be written in packet per buffer mode
 774                  */
 775                 if (!use_dma_ppb || req->req.length < ep->ep.maxpacket
 776                                 || ep->num == UDC_EP0OUT_IX
 777                                 || ep->num == UDC_EP0IN_IX) {
 778                         /* write tx bytes */
 779                         req->td_data->status =
 780                                 AMD_ADDBITS(req->td_data->status,
 781                                                 req->req.length,
 782                                                 UDC_DMA_IN_STS_TXBYTES);
 783                         /* reset frame num */
 784                         req->td_data->status =
 785                                 AMD_ADDBITS(req->td_data->status,
 786                                                 0,
 787                                                 UDC_DMA_IN_STS_FRAMENUM);
 788                 }
 789                 /* set HOST BUSY */
 790                 req->td_data->status =
 791                         AMD_ADDBITS(req->td_data->status,
 792                                 UDC_DMA_STP_STS_BS_HOST_BUSY,
 793                                 UDC_DMA_STP_STS_BS);
 794         } else {
 795                 VDBG(ep->dev, "OUT set host ready\n");
 796                 /* set HOST READY */
 797                 req->td_data->status =
 798                         AMD_ADDBITS(req->td_data->status,
 799                                 UDC_DMA_STP_STS_BS_HOST_READY,
 800                                 UDC_DMA_STP_STS_BS);
 801
 802
 803                         /* clear NAK by writing CNAK */
 804                         if (ep->naking) {
 805                                 tmp = readl(&ep->regs->ctl);
 806                                 tmp |= AMD_BIT(UDC_EPCTL_CNAK);
 807                                 writel(tmp, &ep->regs->ctl);
 808                                 ep->naking = 0;
 809                                 UDC_QUEUE_CNAK(ep, ep->num);
 810                         }
 811
 812         }
 813
 814         return retval;
 815 }
 816
 817 /* Completes request packet ... caller MUST hold lock */
 818 static void
 819 complete_req(struct udc_ep *ep, struct udc_request *req, int sts)
 820 __releases(ep->dev->lock)
 821 __acquires(ep->dev->lock)
 822 {
 823         struct udc              *dev;
 824         unsigned                halted;
 825
 826         VDBG(ep->dev, "complete_req(): ep%d\n", ep->num);
 827
 828         dev = ep->dev;
 829         /* unmap DMA */
 830         if (ep->dma)
 831                 usb_gadget_unmap_request(&dev->gadget, &req->req, ep->in);
 832
 833         halted = ep->halted;
 834         ep->halted = 1;
 835
 836         /* set new status if pending */
 837         if (req->req.status == -EINPROGRESS)
 838                 req->req.status = sts;
 839
 840         /* remove from ep queue */
 841         list_del_init(&req->queue);
 842
 843         VDBG(ep->dev, "req %p => complete %d bytes at %s with sts %d\n",
 844                 &req->req, req->req.length, ep->ep.name, sts);
 845
 846         spin_unlock(&dev->lock);
 847         req->req.complete(&ep->ep, &req->req);
 848         spin_lock(&dev->lock);
 849         ep->halted = halted;
 850 }
 851
 852 /* frees pci pool descriptors of a DMA chain */
 853 static int udc_free_dma_chain(struct udc *dev, struct udc_request *req)
 854 {
 855
 856         int ret_val = 0;
 857         struct udc_data_dma     *td;
 858         struct udc_data_dma     *td_last = NULL;
 859         unsigned int i;
 860
 861         DBG(dev, "free chain req = %p\n", req);
 862
 863         /* do not free first desc., will be done by free for request */
 864         td_last = req->td_data;
 865         td = phys_to_virt(td_last->next);
 866
 867         for (i = 1; i < req->chain_len; i++) {
 868
 869                 pci_pool_free(dev->data_requests, td,
 870                                 (dma_addr_t) td_last->next);
 871                 td_last = td;
 872                 td = phys_to_virt(td_last->next);
 873         }
 874
 875         return ret_val;
 876 }
 877
 878 /* Iterates to the end of a DMA chain and returns last descriptor */
 879 static struct udc_data_dma *udc_get_last_dma_desc(struct udc_request *req)
 880 {
 881         struct udc_data_dma     *td;
 882
 883         td = req->td_data;
 884         while (td && !(td->status & AMD_BIT(UDC_DMA_IN_STS_L)))
 885                 td = phys_to_virt(td->next);
 886
 887         return td;
 888
 889 }
 890
 891 /* Iterates to the end of a DMA chain and counts bytes received */
 892 static u32 udc_get_ppbdu_rxbytes(struct udc_request *req)
 893 {
 894         struct udc_data_dma     *td;
 895         u32 count;
 896
 897         td = req->td_data;
 898         /* received number bytes */
 899         count = AMD_GETBITS(td->status, UDC_DMA_OUT_STS_RXBYTES);
 900
 901         while (td && !(td->status & AMD_BIT(UDC_DMA_IN_STS_L))) {
 902                 td = phys_to_virt(td->next);
 903                 /* received number bytes */
 904                 if (td) {
 905                         count += AMD_GETBITS(td->status,
 906                                 UDC_DMA_OUT_STS_RXBYTES);
 907                 }
 908         }
 909
 910         return count;
 911
 912 }
 913
 914 /* Creates or re-inits a DMA chain */
 915 static int udc_create_dma_chain(
 916         struct udc_ep *ep,
 917         struct udc_request *req,
 918         unsigned long buf_len, gfp_t gfp_flags
 919 )
 920 {
 921         unsigned long bytes = req->req.length;
 922         unsigned int i;
 923         dma_addr_t dma_addr;
 924         struct udc_data_dma     *td = NULL;
 925         struct udc_data_dma     *last = NULL;
 926         unsigned long txbytes;
 927         unsigned create_new_chain = 0;
 928         unsigned len;
 929
 930         VDBG(ep->dev, "udc_create_dma_chain: bytes=%ld buf_len=%ld\n",
 931                         bytes, buf_len);
 932         dma_addr = DMA_DONT_USE;
 933
 934         /* unset L bit in first desc for OUT */
 935         if (!ep->in)
 936                 req->td_data->status &= AMD_CLEAR_BIT(UDC_DMA_IN_STS_L);
 937
 938         /* alloc only new desc's if not already available */
 939         len = req->req.length / ep->ep.maxpacket;
 940         if (req->req.length % ep->ep.maxpacket)
 941                 len++;
 942
 943         if (len > req->chain_len) {
 944                 /* shorter chain already allocated before */
 945                 if (req->chain_len > 1)
 946                         udc_free_dma_chain(ep->dev, req);
 947                 req->chain_len = len;
 948                 create_new_chain = 1;
 949         }
 950
 951         td = req->td_data;
 952         /* gen. required number of descriptors and buffers */
 953         for (i = buf_len; i < bytes; i += buf_len) {
 954                 /* create or determine next desc. */
 955                 if (create_new_chain) {
 956
 957                         td = pci_pool_alloc(ep->dev->data_requests,
 958                                         gfp_flags, &dma_addr);
 959                         if (!td)
 960                                 return -ENOMEM;
 961
 962                         td->status = 0;
 963                 } else if (i == buf_len) {
 964                         /* first td */
 965                         td = (struct udc_data_dma *) phys_to_virt(
 966                                                 req->td_data->next);
 967                         td->status = 0;
 968                 } else {
 969                         td = (struct udc_data_dma *) phys_to_virt(last->next);
 970                         td->status = 0;
 971                 }
 972
 973
 974                 if (td)
 975                         td->bufptr = req->req.dma + i; /* assign buffer */
 976                 else
 977                         break;
 978
 979                 /* short packet ? */
 980                 if ((bytes - i) >= buf_len) {
 981                         txbytes = buf_len;
 982                 } else {
 983                         /* short packet */
 984                         txbytes = bytes - i;
 985                 }
 986
 987                 /* link td and assign tx bytes */
 988                 if (i == buf_len) {
 989                         if (create_new_chain)
 990                                 req->td_data->next = dma_addr;
 991                         /*
 992                         else
 993                                 req->td_data->next = virt_to_phys(td);
 994                         */
 995                         /* write tx bytes */
 996                         if (ep->in) {
 997                                 /* first desc */
 998                                 req->td_data->status =
 999                                         AMD_ADDBITS(req->td_data->status,
1000                                                         ep->ep.maxpacket,
1001                                                         UDC_DMA_IN_STS_TXBYTES);
1002                                 /* second desc */
1003                                 td->status = AMD_ADDBITS(td->status,
1004                                                         txbytes,
1005                                                         UDC_DMA_IN_STS_TXBYTES);
1006                         }
1007                 } else {
1008                         if (create_new_chain)
1009                                 last->next = dma_addr;
1010                         /*
1011                         else
1012                                 last->next = virt_to_phys(td);
1013                         */
1014                         if (ep->in) {
1015                                 /* write tx bytes */
1016                                 td->status = AMD_ADDBITS(td->status,
1017                                                         txbytes,
1018                                                         UDC_DMA_IN_STS_TXBYTES);
1019                         }
1020                 }
1021                 last = td;
1022         }
1023         /* set last bit */
1024         if (td) {
1025                 td->status |= AMD_BIT(UDC_DMA_IN_STS_L);
1026                 /* last desc. points to itself */
1027                 req->td_data_last = td;
1028         }
1029
1030         return 0;
1031 }
1032
1033 /* Enabling RX DMA */
1034 static void udc_set_rde(struct udc *dev)
1035 {
1036         u32 tmp;
1037
1038         VDBG(dev, "udc_set_rde()\n");
1039         /* stop RDE timer */
1040         if (timer_pending(&udc_timer)) {
1041                 set_rde = 0;
1042                 mod_timer(&udc_timer, jiffies - 1);
1043         }
1044         /* set RDE */
1045         tmp = readl(&dev->regs->ctl);
1046         tmp |= AMD_BIT(UDC_DEVCTL_RDE);
1047         writel(tmp, &dev->regs->ctl);
1048 }
1049
1050 /* Queues a request packet, called by gadget driver */
1051 static int
1052 udc_queue(struct usb_ep *usbep, struct usb_request *usbreq, gfp_t gfp)
1053 {
1054         int                     retval = 0;
1055         u8                      open_rxfifo = 0;
1056         unsigned long           iflags;
1057         struct udc_ep           *ep;
1058         struct udc_request      *req;
1059         struct udc              *dev;
1060         u32                     tmp;
1061
1062         /* check the inputs */
1063         req = container_of(usbreq, struct udc_request, req);
1064
1065         if (!usbep || !usbreq || !usbreq->complete || !usbreq->buf
1066                         || !list_empty(&req->queue))
1067                 return -EINVAL;
1068
1069         ep = container_of(usbep, struct udc_ep, ep);
1070         if (!ep->desc && (ep->num != 0 && ep->num != UDC_EP0OUT_IX))
1071                 return -EINVAL;
1072
1073         VDBG(ep->dev, "udc_queue(): ep%d-in=%d\n", ep->num, ep->in);
1074         dev = ep->dev;
1075
1076         if (!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN)
1077                 return -ESHUTDOWN;
1078
1079         /* map dma (usually done before) */
1080         if (ep->dma) {
1081                 VDBG(dev, "DMA map req %p\n", req);
1082                 retval = usb_gadget_map_request(&udc->gadget, usbreq, ep->in);
1083                 if (retval)
1084                         return retval;
1085         }
1086
1087         VDBG(dev, "%s queue req %p, len %d req->td_data=%p buf %p\n",
1088                         usbep->name, usbreq, usbreq->length,
1089                         req->td_data, usbreq->buf);
1090
1091         spin_lock_irqsave(&dev->lock, iflags);
1092         usbreq->actual = 0;
1093         usbreq->status = -EINPROGRESS;
1094         req->dma_done = 0;
1095
1096         /* on empty queue just do first transfer */
1097         if (list_empty(&ep->queue)) {
1098                 /* zlp */
1099                 if (usbreq->length == 0) {
1100                         /* IN zlp's are handled by hardware */
1101                         complete_req(ep, req, 0);
1102                         VDBG(dev, "%s: zlp\n", ep->ep.name);
1103                         /*
1104                          * if set_config or set_intf is waiting for ack by zlp
1105                          * then set CSR_DONE
1106                          */
1107                         if (dev->set_cfg_not_acked) {
1108                                 tmp = readl(&dev->regs->ctl);
1109                                 tmp |= AMD_BIT(UDC_DEVCTL_CSR_DONE);
1110                                 writel(tmp, &dev->regs->ctl);
1111                                 dev->set_cfg_not_acked = 0;
1112                         }
1113                         /* setup command is ACK'ed now by zlp */
1114                         if (dev->waiting_zlp_ack_ep0in) {
1115                                 /* clear NAK by writing CNAK in EP0_IN */
1116                                 tmp = readl(&dev->ep[UDC_EP0IN_IX].regs->ctl);
1117                                 tmp |= AMD_BIT(UDC_EPCTL_CNAK);
1118                                 writel(tmp, &dev->ep[UDC_EP0IN_IX].regs->ctl);
1119                                 dev->ep[UDC_EP0IN_IX].naking = 0;
1120                                 UDC_QUEUE_CNAK(&dev->ep[UDC_EP0IN_IX],
1121                                                         UDC_EP0IN_IX);
1122                                 dev->waiting_zlp_ack_ep0in = 0;
1123                         }
1124                         goto finished;
1125                 }
1126                 if (ep->dma) {
1127                         retval = prep_dma(ep, req, gfp);
1128                         if (retval != 0)
1129                                 goto finished;
1130                         /* write desc pointer to enable DMA */
1131                         if (ep->in) {
1132                                 /* set HOST READY */
1133                                 req->td_data->status =
1134                                         AMD_ADDBITS(req->td_data->status,
1135                                                 UDC_DMA_IN_STS_BS_HOST_READY,
1136                                                 UDC_DMA_IN_STS_BS);
1137                         }
1138
1139                         /* disabled rx dma while descriptor update */
1140                         if (!ep->in) {
1141                                 /* stop RDE timer */
1142                                 if (timer_pending(&udc_timer)) {
1143                                         set_rde = 0;
1144                                         mod_timer(&udc_timer, jiffies - 1);
1145                                 }
1146                                 /* clear RDE */
1147                                 tmp = readl(&dev->regs->ctl);
1148                                 tmp &= AMD_UNMASK_BIT(UDC_DEVCTL_RDE);
1149                                 writel(tmp, &dev->regs->ctl);
1150                                 open_rxfifo = 1;
1151
1152                                 /*
1153                                  * if BNA occurred then let BNA dummy desc.
1154                                  * point to current desc.
1155                                  */
1156                                 if (ep->bna_occurred) {
1157                                         VDBG(dev, "copy to BNA dummy desc.\n");
1158                                         memcpy(ep->bna_dummy_req->td_data,
1159                                                 req->td_data,
1160                                                 sizeof(struct udc_data_dma));
1161                                 }
1162                         }
1163                         /* write desc pointer */
1164                         writel(req->td_phys, &ep->regs->desptr);
1165
1166                         /* clear NAK by writing CNAK */
1167                         if (ep->naking) {
1168                                 tmp = readl(&ep->regs->ctl);
1169                                 tmp |= AMD_BIT(UDC_EPCTL_CNAK);
1170                                 writel(tmp, &ep->regs->ctl);
1171                                 ep->naking = 0;
1172                                 UDC_QUEUE_CNAK(ep, ep->num);
1173                         }
1174
1175                         if (ep->in) {
1176                                 /* enable ep irq */
1177                                 tmp = readl(&dev->regs->ep_irqmsk);
1178                                 tmp &= AMD_UNMASK_BIT(ep->num);
1179                                 writel(tmp, &dev->regs->ep_irqmsk);
1180                         }
1181                 } else if (ep->in) {
1182                                 /* enable ep irq */
1183                                 tmp = readl(&dev->regs->ep_irqmsk);
1184                                 tmp &= AMD_UNMASK_BIT(ep->num);
1185                                 writel(tmp, &dev->regs->ep_irqmsk);
1186                         }
1187
1188         } else if (ep->dma) {
1189
1190                 /*
1191                  * prep_dma not used for OUT ep's, this is not possible
1192                  * for PPB modes, because of chain creation reasons
1193                  */
1194                 if (ep->in) {
1195                         retval = prep_dma(ep, req, gfp);
1196                         if (retval != 0)
1197                                 goto finished;
1198                 }
1199         }
1200         VDBG(dev, "list_add\n");
1201         /* add request to ep queue */
1202         if (req) {
1203
1204                 list_add_tail(&req->queue, &ep->queue);
1205
1206                 /* open rxfifo if out data queued */
1207                 if (open_rxfifo) {
1208                         /* enable DMA */
1209                         req->dma_going = 1;
1210                         udc_set_rde(dev);
1211                         if (ep->num != UDC_EP0OUT_IX)
1212                                 dev->data_ep_queued = 1;
1213                 }
1214                 /* stop OUT naking */
1215                 if (!ep->in) {
1216                         if (!use_dma && udc_rxfifo_pending) {
1217                                 DBG(dev, "udc_queue(): pending bytes in "
1218                                         "rxfifo after nyet\n");
1219                                 /*
1220                                  * read pending bytes afer nyet:
1221                                  * referring to isr
1222                                  */
1223                                 if (udc_rxfifo_read(ep, req)) {
1224                                         /* finish */
1225                                         complete_req(ep, req, 0);
1226                                 }
1227                                 udc_rxfifo_pending = 0;
1228
1229                         }
1230                 }
1231         }
1232
1233 finished:
1234         spin_unlock_irqrestore(&dev->lock, iflags);
1235         return retval;
1236 }
1237
1238 /* Empty request queue of an endpoint; caller holds spinlock */
1239 static void empty_req_queue(struct udc_ep *ep)
1240 {
1241         struct udc_request      *req;
1242
1243         ep->halted = 1;
1244         while (!list_empty(&ep->queue)) {
1245                 req = list_entry(ep->queue.next,
1246                         struct udc_request,
1247                         queue);
1248                 complete_req(ep, req, -ESHUTDOWN);
1249         }
1250 }
1251
1252 /* Dequeues a request packet, called by gadget driver */
1253 static int udc_dequeue(struct usb_ep *usbep, struct usb_request *usbreq)
1254 {
1255         struct udc_ep           *ep;
1256         struct udc_request      *req;
1257         unsigned                halted;
1258         unsigned long           iflags;
1259
1260         ep = container_of(usbep, struct udc_ep, ep);
1261         if (!usbep || !usbreq || (!ep->desc && (ep->num != 0
1262                                 && ep->num != UDC_EP0OUT_IX)))
1263                 return -EINVAL;
1264
1265         req = container_of(usbreq, struct udc_request, req);
1266
1267         spin_lock_irqsave(&ep->dev->lock, iflags);
1268         halted = ep->halted;
1269         ep->halted = 1;
1270         /* request in processing or next one */
1271         if (ep->queue.next == &req->queue) {
1272                 if (ep->dma && req->dma_going) {
1273                         if (ep->in)
1274                                 ep->cancel_transfer = 1;
1275                         else {
1276                                 u32 tmp;
1277                                 u32 dma_sts;
1278                                 /* stop potential receive DMA */
1279                                 tmp = readl(&udc->regs->ctl);
1280                                 writel(tmp & AMD_UNMASK_BIT(UDC_DEVCTL_RDE),
1281                                                         &udc->regs->ctl);
1282                                 /*
1283                                  * Cancel transfer later in ISR
1284                                  * if descriptor was touched.
1285                                  */
1286                                 dma_sts = AMD_GETBITS(req->td_data->status,
1287                                                         UDC_DMA_OUT_STS_BS);
1288                                 if (dma_sts != UDC_DMA_OUT_STS_BS_HOST_READY)
1289                                         ep->cancel_transfer = 1;
1290                                 else {
1291                                         udc_init_bna_dummy(ep->req);
1292                                         writel(ep->bna_dummy_req->td_phys,
1293                                                 &ep->regs->desptr);
1294                                 }
1295                                 writel(tmp, &udc->regs->ctl);
1296                         }
1297                 }
1298         }
1299         complete_req(ep, req, -ECONNRESET);
1300         ep->halted = halted;
1301
1302         spin_unlock_irqrestore(&ep->dev->lock, iflags);
1303         return 0;
1304 }
1305
1306 /* Halt or clear halt of endpoint */
1307 static int
1308 udc_set_halt(struct usb_ep *usbep, int halt)
1309 {
1310         struct udc_ep   *ep;
1311         u32 tmp;
1312         unsigned long iflags;
1313         int retval = 0;
1314
1315         if (!usbep)
1316                 return -EINVAL;
1317
1318         pr_debug("set_halt %s: halt=%d\n", usbep->name, halt);
1319
1320         ep = container_of(usbep, struct udc_ep, ep);
1321         if (!ep->desc && (ep->num != 0 && ep->num != UDC_EP0OUT_IX))
1322                 return -EINVAL;
1323         if (!ep->dev->driver || ep->dev->gadget.speed == USB_SPEED_UNKNOWN)
1324                 return -ESHUTDOWN;
1325
1326         spin_lock_irqsave(&udc_stall_spinlock, iflags);
1327         /* halt or clear halt */
1328         if (halt) {
1329                 if (ep->num == 0)
1330                         ep->dev->stall_ep0in = 1;
1331                 else {
1332                         /*
1333                          * set STALL
1334                          * rxfifo empty not taken into acount
1335                          */
1336                         tmp = readl(&ep->regs->ctl);
1337                         tmp |= AMD_BIT(UDC_EPCTL_S);
1338                         writel(tmp, &ep->regs->ctl);
1339                         ep->halted = 1;
1340
1341                         /* setup poll timer */
1342                         if (!timer_pending(&udc_pollstall_timer)) {
1343                                 udc_pollstall_timer.expires = jiffies +
1344                                         HZ * UDC_POLLSTALL_TIMER_USECONDS
1345                                         / (1000 * 1000);
1346                                 if (!stop_pollstall_timer) {
1347                                         DBG(ep->dev, "start polltimer\n");
1348                                         add_timer(&udc_pollstall_timer);
1349                                 }
1350                         }
1351                 }
1352         } else {
1353                 /* ep is halted by set_halt() before */
1354                 if (ep->halted) {
1355                         tmp = readl(&ep->regs->ctl);
1356                         /* clear stall bit */
1357                         tmp = tmp & AMD_CLEAR_BIT(UDC_EPCTL_S);
1358                         /* clear NAK by writing CNAK */
1359                         tmp |= AMD_BIT(UDC_EPCTL_CNAK);
1360                         writel(tmp, &ep->regs->ctl);
1361                         ep->halted = 0;
1362                         UDC_QUEUE_CNAK(ep, ep->num);
1363                 }
1364         }
1365         spin_unlock_irqrestore(&udc_stall_spinlock, iflags);
1366         return retval;
1367 }
1368
1369 /* gadget interface */
1370 static const struct usb_ep_ops udc_ep_ops = {
1371         .enable         = udc_ep_enable,
1372         .disable        = udc_ep_disable,
1373
1374         .alloc_request  = udc_alloc_request,
1375         .free_request   = udc_free_request,
1376
1377         .queue          = udc_queue,
1378         .dequeue        = udc_dequeue,
1379
1380         .set_halt       = udc_set_halt,
1381         /* fifo ops not implemented */
1382 };
1383
1384 /*-------------------------------------------------------------------------*/
1385
1386 /* Get frame counter (not implemented) */
1387 static int udc_get_frame(struct usb_gadget *gadget)
1388 {
1389         return -EOPNOTSUPP;
1390 }
1391
1392 /* Remote wakeup gadget interface */
1393 static int udc_wakeup(struct usb_gadget *gadget)
1394 {
1395         struct udc              *dev;
1396
1397         if (!gadget)
1398                 return -EINVAL;
1399         dev = container_of(gadget, struct udc, gadget);
1400         udc_remote_wakeup(dev);
1401
1402         return 0;
1403 }
1404
1405 static int amd5536_start(struct usb_gadget_driver *driver,
1406                 int (*bind)(struct usb_gadget *));
1407 static int amd5536_stop(struct usb_gadget_driver *driver);
1408 /* gadget operations */
1409 static const struct usb_gadget_ops udc_ops = {
1410         .wakeup         = udc_wakeup,
1411         .get_frame      = udc_get_frame,
1412         .start          = amd5536_start,
1413         .stop           = amd5536_stop,
1414 };
1415
1416 /* Setups endpoint parameters, adds endpoints to linked list */
1417 static void make_ep_lists(struct udc *dev)
1418 {
1419         /* make gadget ep lists */
1420         INIT_LIST_HEAD(&dev->gadget.ep_list);
1421         list_add_tail(&dev->ep[UDC_EPIN_STATUS_IX].ep.ep_list,
1422                                                 &dev->gadget.ep_list);
1423         list_add_tail(&dev->ep[UDC_EPIN_IX].ep.ep_list,
1424                                                 &dev->gadget.ep_list);
1425         list_add_tail(&dev->ep[UDC_EPOUT_IX].ep.ep_list,
1426                                                 &dev->gadget.ep_list);
1427
1428         /* fifo config */
1429         dev->ep[UDC_EPIN_STATUS_IX].fifo_depth = UDC_EPIN_SMALLINT_BUFF_SIZE;
1430         if (dev->gadget.speed == USB_SPEED_FULL)
1431                 dev->ep[UDC_EPIN_IX].fifo_depth = UDC_FS_EPIN_BUFF_SIZE;
1432         else if (dev->gadget.speed == USB_SPEED_HIGH)
1433                 dev->ep[UDC_EPIN_IX].fifo_depth = hs_tx_buf;
1434         dev->ep[UDC_EPOUT_IX].fifo_depth = UDC_RXFIFO_SIZE;
1435 }
1436
1437 /* init registers at driver load time */
1438 static int startup_registers(struct udc *dev)
1439 {
1440         u32 tmp;
1441
1442         /* init controller by soft reset */
1443         udc_soft_reset(dev);
1444
1445         /* mask not needed interrupts */
1446         udc_mask_unused_interrupts(dev);
1447
1448         /* put into initial config */
1449         udc_basic_init(dev);
1450         /* link up all endpoints */
1451         udc_setup_endpoints(dev);
1452
1453         /* program speed */
1454         tmp = readl(&dev->regs->cfg);
1455         if (use_fullspeed)
1456                 tmp = AMD_ADDBITS(tmp, UDC_DEVCFG_SPD_FS, UDC_DEVCFG_SPD);
1457         else
1458                 tmp = AMD_ADDBITS(tmp, UDC_DEVCFG_SPD_HS, UDC_DEVCFG_SPD);
1459         writel(tmp, &dev->regs->cfg);
1460
1461         return 0;
1462 }
1463
1464 /* Inits UDC context */
1465 static void udc_basic_init(struct udc *dev)
1466 {
1467         u32     tmp;
1468
1469         DBG(dev, "udc_basic_init()\n");
1470
1471         dev->gadget.speed = USB_SPEED_UNKNOWN;
1472
1473         /* stop RDE timer */
1474         if (timer_pending(&udc_timer)) {
1475                 set_rde = 0;
1476                 mod_timer(&udc_timer, jiffies - 1);
1477         }
1478         /* stop poll stall timer */
1479         if (timer_pending(&udc_pollstall_timer))
1480                 mod_timer(&udc_pollstall_timer, jiffies - 1);
1481         /* disable DMA */
1482         tmp = readl(&dev->regs->ctl);
1483         tmp &= AMD_UNMASK_BIT(UDC_DEVCTL_RDE);
1484         tmp &= AMD_UNMASK_BIT(UDC_DEVCTL_TDE);
1485         writel(tmp, &dev->regs->ctl);
1486
1487         /* enable dynamic CSR programming */
1488         tmp = readl(&dev->regs->cfg);
1489         tmp |= AMD_BIT(UDC_DEVCFG_CSR_PRG);
1490         /* set self powered */
1491         tmp |= AMD_BIT(UDC_DEVCFG_SP);
1492         /* set remote wakeupable */
1493         tmp |= AMD_BIT(UDC_DEVCFG_RWKP);
1494         writel(tmp, &dev->regs->cfg);
1495
1496         make_ep_lists(dev);
1497
1498         dev->data_ep_enabled = 0;
1499         dev->data_ep_queued = 0;
1500 }
1501
1502 /* Sets initial endpoint parameters */
1503 static void udc_setup_endpoints(struct udc *dev)
1504 {
1505         struct udc_ep   *ep;
1506         u32     tmp;
1507         u32     reg;
1508
1509         DBG(dev, "udc_setup_endpoints()\n");
1510
1511         /* read enum speed */
1512         tmp = readl(&dev->regs->sts);
1513         tmp = AMD_GETBITS(tmp, UDC_DEVSTS_ENUM_SPEED);
1514         if (tmp == UDC_DEVSTS_ENUM_SPEED_HIGH)
1515                 dev->gadget.speed = USB_SPEED_HIGH;
1516         else if (tmp == UDC_DEVSTS_ENUM_SPEED_FULL)
1517                 dev->gadget.speed = USB_SPEED_FULL;
1518
1519         /* set basic ep parameters */
1520         for (tmp = 0; tmp < UDC_EP_NUM; tmp++) {
1521                 ep = &dev->ep[tmp];
1522                 ep->dev = dev;
1523                 ep->ep.name = ep_string[tmp];
1524                 ep->num = tmp;
1525                 /* txfifo size is calculated at enable time */
1526                 ep->txfifo = dev->txfifo;
1527
1528                 /* fifo size */
1529                 if (tmp < UDC_EPIN_NUM) {
1530                         ep->fifo_depth = UDC_TXFIFO_SIZE;
1531                         ep->in = 1;
1532                 } else {
1533                         ep->fifo_depth = UDC_RXFIFO_SIZE;
1534                         ep->in = 0;
1535
1536                 }
1537                 ep->regs = &dev->ep_regs[tmp];
1538                 /*
1539                  * ep will be reset only if ep was not enabled before to avoid
1540                  * disabling ep interrupts when ENUM interrupt occurs but ep is
1541                  * not enabled by gadget driver
1542                  */
1543                 if (!ep->desc)
1544                         ep_init(dev->regs, ep);
1545
1546                 if (use_dma) {
1547                         /*
1548                          * ep->dma is not really used, just to indicate that
1549                          * DMA is active: remove this
1550                          * dma regs = dev control regs
1551                          */
1552                         ep->dma = &dev->regs->ctl;
1553
1554                         /* nak OUT endpoints until enable - not for ep0 */
1555                         if (tmp != UDC_EP0IN_IX && tmp != UDC_EP0OUT_IX
1556                                                 && tmp > UDC_EPIN_NUM) {
1557                                 /* set NAK */
1558                                 reg = readl(&dev->ep[tmp].regs->ctl);
1559                                 reg |= AMD_BIT(UDC_EPCTL_SNAK);
1560                                 writel(reg, &dev->ep[tmp].regs->ctl);
1561                                 dev->ep[tmp].naking = 1;
1562
1563                         }
1564                 }
1565         }
1566         /* EP0 max packet */
1567         if (dev->gadget.speed == USB_SPEED_FULL) {
1568                 dev->ep[UDC_EP0IN_IX].ep.maxpacket = UDC_FS_EP0IN_MAX_PKT_SIZE;
1569                 dev->ep[UDC_EP0OUT_IX].ep.maxpacket =
1570                                                 UDC_FS_EP0OUT_MAX_PKT_SIZE;
1571         } else if (dev->gadget.speed == USB_SPEED_HIGH) {
1572                 dev->ep[UDC_EP0IN_IX].ep.maxpacket = UDC_EP0IN_MAX_PKT_SIZE;
1573                 dev->ep[UDC_EP0OUT_IX].ep.maxpacket = UDC_EP0OUT_MAX_PKT_SIZE;
1574         }
1575
1576         /*
1577          * with suspend bug workaround, ep0 params for gadget driver
1578          * are set at gadget driver bind() call
1579          */
1580         dev->gadget.ep0 = &dev->ep[UDC_EP0IN_IX].ep;
1581         dev->ep[UDC_EP0IN_IX].halted = 0;
1582         INIT_LIST_HEAD(&dev->gadget.ep0->ep_list);
1583
1584         /* init cfg/alt/int */
1585         dev->cur_config = 0;
1586         dev->cur_intf = 0;
1587         dev->cur_alt = 0;
1588 }
1589
1590 /* Bringup after Connect event, initial bringup to be ready for ep0 events */
1591 static void usb_connect(struct udc *dev)
1592 {
1593
1594         dev_info(&dev->pdev->dev, "USB Connect\n");
1595
1596         dev->connected = 1;
1597
1598         /* put into initial config */
1599         udc_basic_init(dev);
1600
1601         /* enable device setup interrupts */
1602         udc_enable_dev_setup_interrupts(dev);
1603 }
1604
1605 /*
1606  * Calls gadget with disconnect event and resets the UDC and makes
1607  * initial bringup to be ready for ep0 events
1608  */
1609 static void usb_disconnect(struct udc *dev)
1610 {
1611
1612         dev_info(&dev->pdev->dev, "USB Disconnect\n");
1613
1614         dev->connected = 0;
1615
1616         /* mask interrupts */
1617         udc_mask_unused_interrupts(dev);
1618
1619         /* REVISIT there doesn't seem to be a point to having this
1620          * talk to a tasklet ... do it directly, we already hold
1621          * the spinlock needed to process the disconnect.
1622          */
1623
1624         tasklet_schedule(&disconnect_tasklet);
1625 }
1626
1627 /* Tasklet for disconnect to be outside of interrupt context */
1628 static void udc_tasklet_disconnect(unsigned long par)
1629 {
1630         struct udc *dev = (struct udc *)(*((struct udc **) par));
1631         u32 tmp;
1632
1633         DBG(dev, "Tasklet disconnect\n");
1634         spin_lock_irq(&dev->lock);
1635
1636         if (dev->driver) {
1637                 spin_unlock(&dev->lock);
1638                 dev->driver->disconnect(&dev->gadget);
1639                 spin_lock(&dev->lock);
1640
1641                 /* empty queues */
1642                 for (tmp = 0; tmp < UDC_EP_NUM; tmp++)
1643                         empty_req_queue(&dev->ep[tmp]);
1644
1645         }
1646
1647         /* disable ep0 */
1648         ep_init(dev->regs,
1649                         &dev->ep[UDC_EP0IN_IX]);
1650
1651
1652         if (!soft_reset_occured) {
1653                 /* init controller by soft reset */
1654                 udc_soft_reset(dev);
1655                 soft_reset_occured++;
1656         }
1657
1658         /* re-enable dev interrupts */
1659         udc_enable_dev_setup_interrupts(dev);
1660         /* back to full speed ? */
1661         if (use_fullspeed) {
1662                 tmp = readl(&dev->regs->cfg);
1663                 tmp = AMD_ADDBITS(tmp, UDC_DEVCFG_SPD_FS, UDC_DEVCFG_SPD);
1664                 writel(tmp, &dev->regs->cfg);
1665         }
1666
1667         spin_unlock_irq(&dev->lock);
1668 }
1669
1670 /* Reset the UDC core */
1671 static void udc_soft_reset(struct udc *dev)
1672 {
1673         unsigned long   flags;
1674
1675         DBG(dev, "Soft reset\n");
1676         /*
1677          * reset possible waiting interrupts, because int.
1678          * status is lost after soft reset,
1679          * ep int. status reset
1680          */
1681         writel(UDC_EPINT_MSK_DISABLE_ALL, &dev->regs->ep_irqsts);
1682         /* device int. status reset */
1683         writel(UDC_DEV_MSK_DISABLE, &dev->regs->irqsts);
1684
1685         spin_lock_irqsave(&udc_irq_spinlock, flags);
1686         writel(AMD_BIT(UDC_DEVCFG_SOFTRESET), &dev->regs->cfg);
1687         readl(&dev->regs->cfg);
1688         spin_unlock_irqrestore(&udc_irq_spinlock, flags);
1689
1690 }
1691
1692 /* RDE timer callback to set RDE bit */
1693 static void udc_timer_function(unsigned long v)
1694 {
1695         u32 tmp;
1696
1697         spin_lock_irq(&udc_irq_spinlock);
1698
1699         if (set_rde > 0) {
1700                 /*
1701                  * open the fifo if fifo was filled on last timer call
1702                  * conditionally
1703                  */
1704                 if (set_rde > 1) {
1705                         /* set RDE to receive setup data */
1706                         tmp = readl(&udc->regs->ctl);
1707                         tmp |= AMD_BIT(UDC_DEVCTL_RDE);
1708                         writel(tmp, &udc->regs->ctl);
1709                         set_rde = -1;
1710                 } else if (readl(&udc->regs->sts)
1711                                 & AMD_BIT(UDC_DEVSTS_RXFIFO_EMPTY)) {
1712                         /*
1713                          * if fifo empty setup polling, do not just
1714                          * open the fifo
1715                          */
1716                         udc_timer.expires = jiffies + HZ/UDC_RDE_TIMER_DIV;
1717                         if (!stop_timer)
1718                                 add_timer(&udc_timer);
1719                 } else {
1720                         /*
1721                          * fifo contains data now, setup timer for opening
1722                          * the fifo when timer expires to be able to receive
1723                          * setup packets, when data packets gets queued by
1724                          * gadget layer then timer will forced to expire with
1725                          * set_rde=0 (RDE is set in udc_queue())
1726                          */
1727                         set_rde++;
1728                         /* debug: lhadmot_timer_start = 221070 */
1729                         udc_timer.expires = jiffies + HZ*UDC_RDE_TIMER_SECONDS;
1730                         if (!stop_timer)
1731                                 add_timer(&udc_timer);
1732                 }
1733
1734         } else
1735                 set_rde = -1; /* RDE was set by udc_queue() */
1736         spin_unlock_irq(&udc_irq_spinlock);
1737         if (stop_timer)
1738                 complete(&on_exit);
1739
1740 }
1741
1742 /* Handle halt state, used in stall poll timer */
1743 static void udc_handle_halt_state(struct udc_ep *ep)
1744 {
1745         u32 tmp;
1746         /* set stall as long not halted */
1747         if (ep->halted == 1) {
1748                 tmp = readl(&ep->regs->ctl);
1749                 /* STALL cleared ? */
1750                 if (!(tmp & AMD_BIT(UDC_EPCTL_S))) {
1751                         /*
1752                          * FIXME: MSC spec requires that stall remains
1753                          * even on receivng of CLEAR_FEATURE HALT. So
1754                          * we would set STALL again here to be compliant.
1755                          * But with current mass storage drivers this does
1756                          * not work (would produce endless host retries).
1757                          * So we clear halt on CLEAR_FEATURE.
1758                          *
1759                         DBG(ep->dev, "ep %d: set STALL again\n", ep->num);
1760                         tmp |= AMD_BIT(UDC_EPCTL_S);
1761                         writel(tmp, &ep->regs->ctl);*/
1762
1763                         /* clear NAK by writing CNAK */
1764                         tmp |= AMD_BIT(UDC_EPCTL_CNAK);
1765                         writel(tmp, &ep->regs->ctl);
1766                         ep->halted = 0;
1767                         UDC_QUEUE_CNAK(ep, ep->num);
1768                 }
1769         }
1770 }
1771
1772 /* Stall timer callback to poll S bit and set it again after */
1773 static void udc_pollstall_timer_function(unsigned long v)
1774 {
1775         struct udc_ep *ep;
1776         int halted = 0;
1777
1778         spin_lock_irq(&udc_stall_spinlock);
1779         /*
1780          * only one IN and OUT endpoints are handled
1781          * IN poll stall
1782          */
1783         ep = &udc->ep[UDC_EPIN_IX];
1784         udc_handle_halt_state(ep);
1785         if (ep->halted)
1786                 halted = 1;
1787         /* OUT poll stall */
1788         ep = &udc->ep[UDC_EPOUT_IX];
1789         udc_handle_halt_state(ep);
1790         if (ep->halted)
1791                 halted = 1;
1792
1793         /* setup timer again when still halted */
1794         if (!stop_pollstall_timer && halted) {
1795                 udc_pollstall_timer.expires = jiffies +
1796                                         HZ * UDC_POLLSTALL_TIMER_USECONDS
1797                                         / (1000 * 1000);
1798                 add_timer(&udc_pollstall_timer);
1799         }
1800         spin_unlock_irq(&udc_stall_spinlock);
1801
1802         if (stop_pollstall_timer)
1803                 complete(&on_pollstall_exit);
1804 }
1805
1806 /* Inits endpoint 0 so that SETUP packets are processed */
1807 static void activate_control_endpoints(struct udc *dev)
1808 {
1809         u32 tmp;
1810
1811         DBG(dev, "activate_control_endpoints\n");
1812
1813         /* flush fifo */
1814         tmp = readl(&dev->ep[UDC_EP0IN_IX].regs->ctl);
1815         tmp |= AMD_BIT(UDC_EPCTL_F);
1816         writel(tmp, &dev->ep[UDC_EP0IN_IX].regs->ctl);
1817
1818         /* set ep0 directions */
1819         dev->ep[UDC_EP0IN_IX].in = 1;
1820         dev->ep[UDC_EP0OUT_IX].in = 0;
1821
1822         /* set buffer size (tx fifo entries) of EP0_IN */
1823         tmp = readl(&dev->ep[UDC_EP0IN_IX].regs->bufin_framenum);
1824         if (dev->gadget.speed == USB_SPEED_FULL)
1825                 tmp = AMD_ADDBITS(tmp, UDC_FS_EPIN0_BUFF_SIZE,
1826                                         UDC_EPIN_BUFF_SIZE);
1827         else if (dev->gadget.speed == USB_SPEED_HIGH)
1828                 tmp = AMD_ADDBITS(tmp, UDC_EPIN0_BUFF_SIZE,
1829                                         UDC_EPIN_BUFF_SIZE);
1830         writel(tmp, &dev->ep[UDC_EP0IN_IX].regs->bufin_framenum);
1831
1832         /* set max packet size of EP0_IN */
1833         tmp = readl(&dev->ep[UDC_EP0IN_IX].regs->bufout_maxpkt);
1834         if (dev->gadget.speed == USB_SPEED_FULL)
1835                 tmp = AMD_ADDBITS(tmp, UDC_FS_EP0IN_MAX_PKT_SIZE,
1836                                         UDC_EP_MAX_PKT_SIZE);
1837         else if (dev->gadget.speed == USB_SPEED_HIGH)
1838                 tmp = AMD_ADDBITS(tmp, UDC_EP0IN_MAX_PKT_SIZE,
1839                                 UDC_EP_MAX_PKT_SIZE);
1840         writel(tmp, &dev->ep[UDC_EP0IN_IX].regs->bufout_maxpkt);
1841
1842         /* set max packet size of EP0_OUT */
1843         tmp = readl(&dev->ep[UDC_EP0OUT_IX].regs->bufout_maxpkt);
1844         if (dev->gadget.speed == USB_SPEED_FULL)
1845                 tmp = AMD_ADDBITS(tmp, UDC_FS_EP0OUT_MAX_PKT_SIZE,
1846                                         UDC_EP_MAX_PKT_SIZE);
1847         else if (dev->gadget.speed == USB_SPEED_HIGH)
1848                 tmp = AMD_ADDBITS(tmp, UDC_EP0OUT_MAX_PKT_SIZE,
1849                                         UDC_EP_MAX_PKT_SIZE);
1850         writel(tmp, &dev->ep[UDC_EP0OUT_IX].regs->bufout_maxpkt);
1851
1852         /* set max packet size of EP0 in UDC CSR */
1853         tmp = readl(&dev->csr->ne[0]);
1854         if (dev->gadget.speed == USB_SPEED_FULL)
1855                 tmp = AMD_ADDBITS(tmp, UDC_FS_EP0OUT_MAX_PKT_SIZE,
1856                                         UDC_CSR_NE_MAX_PKT);
1857         else if (dev->gadget.speed == USB_SPEED_HIGH)
1858                 tmp = AMD_ADDBITS(tmp, UDC_EP0OUT_MAX_PKT_SIZE,
1859                                         UDC_CSR_NE_MAX_PKT);
1860         writel(tmp, &dev->csr->ne[0]);
1861
1862         if (use_dma) {
1863                 dev->ep[UDC_EP0OUT_IX].td->status |=
1864                         AMD_BIT(UDC_DMA_OUT_STS_L);
1865                 /* write dma desc address */
1866                 writel(dev->ep[UDC_EP0OUT_IX].td_stp_dma,
1867                         &dev->ep[UDC_EP0OUT_IX].regs->subptr);
1868                 writel(dev->ep[UDC_EP0OUT_IX].td_phys,
1869                         &dev->ep[UDC_EP0OUT_IX].regs->desptr);
1870                 /* stop RDE timer */
1871                 if (timer_pending(&udc_timer)) {
1872                         set_rde = 0;
1873                         mod_timer(&udc_timer, jiffies - 1);
1874                 }
1875                 /* stop pollstall timer */
1876                 if (timer_pending(&udc_pollstall_timer))
1877                         mod_timer(&udc_pollstall_timer, jiffies - 1);
1878                 /* enable DMA */
1879                 tmp = readl(&dev->regs->ctl);
1880                 tmp |= AMD_BIT(UDC_DEVCTL_MODE)
1881                                 | AMD_BIT(UDC_DEVCTL_RDE)
1882                                 | AMD_BIT(UDC_DEVCTL_TDE);
1883                 if (use_dma_bufferfill_mode)
1884                         tmp |= AMD_BIT(UDC_DEVCTL_BF);
1885                 else if (use_dma_ppb_du)
1886                         tmp |= AMD_BIT(UDC_DEVCTL_DU);
1887                 writel(tmp, &dev->regs->ctl);
1888         }
1889
1890         /* clear NAK by writing CNAK for EP0IN */
1891         tmp = readl(&dev->ep[UDC_EP0IN_IX].regs->ctl);
1892         tmp |= AMD_BIT(UDC_EPCTL_CNAK);
1893         writel(tmp, &dev->ep[UDC_EP0IN_IX].regs->ctl);
1894         dev->ep[UDC_EP0IN_IX].naking = 0;
1895         UDC_QUEUE_CNAK(&dev->ep[UDC_EP0IN_IX], UDC_EP0IN_IX);
1896
1897         /* clear NAK by writing CNAK for EP0OUT */
1898         tmp = readl(&dev->ep[UDC_EP0OUT_IX].regs->ctl);
1899         tmp |= AMD_BIT(UDC_EPCTL_CNAK);
1900         writel(tmp, &dev->ep[UDC_EP0OUT_IX].regs->ctl);
1901         dev->ep[UDC_EP0OUT_IX].naking = 0;
1902         UDC_QUEUE_CNAK(&dev->ep[UDC_EP0OUT_IX], UDC_EP0OUT_IX);
1903 }
1904
1905 /* Make endpoint 0 ready for control traffic */
1906 static int setup_ep0(struct udc *dev)
1907 {
1908         activate_control_endpoints(dev);
1909         /* enable ep0 interrupts */
1910         udc_enable_ep0_interrupts(dev);
1911         /* enable device setup interrupts */
1912         udc_enable_dev_setup_interrupts(dev);
1913
1914         return 0;
1915 }
1916
1917 /* Called by gadget driver to register itself */
1918 static int amd5536_start(struct usb_gadget_driver *driver,
1919                 int (*bind)(struct usb_gadget *))
1920 {
1921         struct udc              *dev = udc;
1922         int                     retval;
1923         u32 tmp;
1924
1925         if (!driver || !bind || !driver->setup
1926                         || driver->max_speed < USB_SPEED_HIGH)
1927                 return -EINVAL;
1928         if (!dev)
1929                 return -ENODEV;
1930         if (dev->driver)
1931                 return -EBUSY;
1932
1933         driver->driver.bus = NULL;
1934         dev->driver = driver;
1935         dev->gadget.dev.driver = &driver->driver;
1936
1937         retval = bind(&dev->gadget);
1938
1939         /* Some gadget drivers use both ep0 directions.
1940          * NOTE: to gadget driver, ep0 is just one endpoint...
1941          */
1942         dev->ep[UDC_EP0OUT_IX].ep.driver_data =
1943                 dev->ep[UDC_EP0IN_IX].ep.driver_data;
1944
1945         if (retval) {
1946                 DBG(dev, "binding to %s returning %d\n",
1947                                 driver->driver.name, retval);
1948                 dev->driver = NULL;
1949                 dev->gadget.dev.driver = NULL;
1950                 return retval;
1951         }
1952
1953         /* get ready for ep0 traffic */
1954         setup_ep0(dev);
1955
1956         /* clear SD */
1957         tmp = readl(&dev->regs->ctl);
1958         tmp = tmp & AMD_CLEAR_BIT(UDC_DEVCTL_SD);
1959         writel(tmp, &dev->regs->ctl);
1960
1961         usb_connect(dev);
1962
1963         return 0;
1964 }
1965
1966 /* shutdown requests and disconnect from gadget */
1967 static void
1968 shutdown(struct udc *dev, struct usb_gadget_driver *driver)
1969 __releases(dev->lock)
1970 __acquires(dev->lock)
1971 {
1972         int tmp;
1973
1974         if (dev->gadget.speed != USB_SPEED_UNKNOWN) {
1975                 spin_unlock(&dev->lock);
1976                 driver->disconnect(&dev->gadget);
1977                 spin_lock(&dev->lock);
1978         }
1979
1980         /* empty queues and init hardware */
1981         udc_basic_init(dev);
1982         for (tmp = 0; tmp < UDC_EP_NUM; tmp++)
1983                 empty_req_queue(&dev->ep[tmp]);
1984
1985         udc_setup_endpoints(dev);
1986 }
1987
1988 /* Called by gadget driver to unregister itself */
1989 static int amd5536_stop(struct usb_gadget_driver *driver)
1990 {
1991         struct udc      *dev = udc;
1992         unsigned long   flags;
1993         u32 tmp;
1994
1995         if (!dev)
1996                 return -ENODEV;
1997         if (!driver || driver != dev->driver || !driver->unbind)
1998                 return -EINVAL;
1999
2000         spin_lock_irqsave(&dev->lock, flags);
2001         udc_mask_unused_interrupts(dev);
2002         shutdown(dev, driver);
2003         spin_unlock_irqrestore(&dev->lock, flags);
2004
2005         driver->unbind(&dev->gadget);
2006         dev->gadget.dev.driver = NULL;
2007         dev->driver = NULL;
2008
2009         /* set SD */
2010         tmp = readl(&dev->regs->ctl);
2011         tmp |= AMD_BIT(UDC_DEVCTL_SD);
2012         writel(tmp, &dev->regs->ctl);
2013
2014
2015         DBG(dev, "%s: unregistered\n", driver->driver.name);
2016
2017         return 0;
2018 }
2019
2020 /* Clear pending NAK bits */
2021 static void udc_process_cnak_queue(struct udc *dev)
2022 {
2023         u32 tmp;
2024         u32 reg;
2025
2026         /* check epin's */
2027         DBG(dev, "CNAK pending queue processing\n");
2028         for (tmp = 0; tmp < UDC_EPIN_NUM_USED; tmp++) {
2029                 if (cnak_pending & (1 << tmp)) {
2030                         DBG(dev, "CNAK pending for ep%d\n", tmp);
2031                         /* clear NAK by writing CNAK */
2032                         reg = readl(&dev->ep[tmp].regs->ctl);
2033                         reg |= AMD_BIT(UDC_EPCTL_CNAK);
2034                         writel(reg, &dev->ep[tmp].regs->ctl);
2035                         dev->ep[tmp].naking = 0;
2036                         UDC_QUEUE_CNAK(&dev->ep[tmp], dev->ep[tmp].num);
2037                 }
2038         }
2039         /* ...  and ep0out */
2040         if (cnak_pending & (1 << UDC_EP0OUT_IX)) {
2041                 DBG(dev, "CNAK pending for ep%d\n", UDC_EP0OUT_IX);
2042                 /* clear NAK by writing CNAK */
2043                 reg = readl(&dev->ep[UDC_EP0OUT_IX].regs->ctl);
2044                 reg |= AMD_BIT(UDC_EPCTL_CNAK);
2045                 writel(reg, &dev->ep[UDC_EP0OUT_IX].regs->ctl);
2046                 dev->ep[UDC_EP0OUT_IX].naking = 0;
2047                 UDC_QUEUE_CNAK(&dev->ep[UDC_EP0OUT_IX],
2048                                 dev->ep[UDC_EP0OUT_IX].num);
2049         }
2050 }
2051
2052 /* Enabling RX DMA after setup packet */
2053 static void udc_ep0_set_rde(struct udc *dev)
2054 {
2055         if (use_dma) {
2056                 /*
2057                  * only enable RXDMA when no data endpoint enabled
2058                  * or data is queued
2059                  */
2060                 if (!dev->data_ep_enabled || dev->data_ep_queued) {
2061                         udc_set_rde(dev);
2062                 } else {
2063                         /*
2064                          * setup timer for enabling RDE (to not enable
2065                          * RXFIFO DMA for data endpoints to early)
2066                          */
2067                         if (set_rde != 0 && !timer_pending(&udc_timer)) {
2068                                 udc_timer.expires =
2069                                         jiffies + HZ/UDC_RDE_TIMER_DIV;
2070                                 set_rde = 1;
2071                                 if (!stop_timer)
2072                                         add_timer(&udc_timer);
2073                         }
2074                 }
2075         }
2076 }
2077
2078
2079 /* Interrupt handler for data OUT traffic */
2080 static irqreturn_t udc_data_out_isr(struct udc *dev, int ep_ix)
2081 {
2082         irqreturn_t             ret_val = IRQ_NONE;
2083         u32                     tmp;
2084         struct udc_ep           *ep;
2085         struct udc_request      *req;
2086         unsigned int            count;
2087         struct udc_data_dma     *td = NULL;
2088         unsigned                dma_done;
2089
2090         VDBG(dev, "ep%d irq\n", ep_ix);
2091         ep = &dev->ep[ep_ix];
2092
2093         tmp = readl(&ep->regs->sts);
2094         if (use_dma) {
2095                 /* BNA event ? */
2096                 if (tmp & AMD_BIT(UDC_EPSTS_BNA)) {
2097                         DBG(dev, "BNA ep%dout occurred - DESPTR = %x\n",
2098                                         ep->num, readl(&ep->regs->desptr));
2099                         /* clear BNA */
2100                         writel(tmp | AMD_BIT(UDC_EPSTS_BNA), &ep->regs->sts);
2101                         if (!ep->cancel_transfer)
2102                                 ep->bna_occurred = 1;
2103                         else
2104                                 ep->cancel_transfer = 0;
2105                         ret_val = IRQ_HANDLED;
2106                         goto finished;
2107                 }
2108         }
2109         /* HE event ? */
2110         if (tmp & AMD_BIT(UDC_EPSTS_HE)) {
2111                 dev_err(&dev->pdev->dev, "HE ep%dout occurred\n", ep->num);
2112
2113                 /* clear HE */
2114                 writel(tmp | AMD_BIT(UDC_EPSTS_HE), &ep->regs->sts);
2115                 ret_val = IRQ_HANDLED;
2116                 goto finished;
2117         }
2118
2119         if (!list_empty(&ep->queue)) {
2120
2121                 /* next request */
2122                 req = list_entry(ep->queue.next,
2123                         struct udc_request, queue);
2124         } else {
2125                 req = NULL;
2126                 udc_rxfifo_pending = 1;
2127         }
2128         VDBG(dev, "req = %p\n", req);
2129         /* fifo mode */
2130         if (!use_dma) {
2131
2132                 /* read fifo */
2133                 if (req && udc_rxfifo_read(ep, req)) {
2134                         ret_val = IRQ_HANDLED;
2135
2136                         /* finish */
2137                         complete_req(ep, req, 0);
2138                         /* next request */
2139                         if (!list_empty(&ep->queue) && !ep->halted) {
2140                                 req = list_entry(ep->queue.next,
2141                                         struct udc_request, queue);
2142                         } else
2143                                 req = NULL;
2144                 }
2145
2146         /* DMA */
2147         } else if (!ep->cancel_transfer && req != NULL) {
2148                 ret_val = IRQ_HANDLED;
2149
2150                 /* check for DMA done */
2151                 if (!use_dma_ppb) {
2152                         dma_done = AMD_GETBITS(req->td_data->status,
2153                                                 UDC_DMA_OUT_STS_BS);
2154                 /* packet per buffer mode - rx bytes */
2155                 } else {
2156                         /*
2157                          * if BNA occurred then recover desc. from
2158                          * BNA dummy desc.
2159                          */
2160                         if (ep->bna_occurred) {
2161                                 VDBG(dev, "Recover desc. from BNA dummy\n");
2162                                 memcpy(req->td_data, ep->bna_dummy_req->td_data,
2163                                                 sizeof(struct udc_data_dma));
2164                                 ep->bna_occurred = 0;
2165                                 udc_init_bna_dummy(ep->req);
2166                         }
2167                         td = udc_get_last_dma_desc(req);
2168                         dma_done = AMD_GETBITS(td->status, UDC_DMA_OUT_STS_BS);
2169                 }
2170                 if (dma_done == UDC_DMA_OUT_STS_BS_DMA_DONE) {
2171                         /* buffer fill mode - rx bytes */
2172                         if (!use_dma_ppb) {
2173                                 /* received number bytes */
2174                                 count = AMD_GETBITS(req->td_data->status,
2175                                                 UDC_DMA_OUT_STS_RXBYTES);
2176                                 VDBG(dev, "rx bytes=%u\n", count);
2177                         /* packet per buffer mode - rx bytes */
2178                         } else {
2179                                 VDBG(dev, "req->td_data=%p\n", req->td_data);
2180                                 VDBG(dev, "last desc = %p\n", td);
2181                                 /* received number bytes */
2182                                 if (use_dma_ppb_du) {
2183                                         /* every desc. counts bytes */
2184                                         count = udc_get_ppbdu_rxbytes(req);
2185                                 } else {
2186                                         /* last desc. counts bytes */
2187                                         count = AMD_GETBITS(td->status,
2188                                                 UDC_DMA_OUT_STS_RXBYTES);
2189                                         if (!count && req->req.length
2190                                                 == UDC_DMA_MAXPACKET) {
2191                                                 /*
2192                                                  * on 64k packets the RXBYTES
2193                                                  * field is zero
2194                                                  */
2195                                                 count = UDC_DMA_MAXPACKET;
2196                                         }
2197                                 }
2198                                 VDBG(dev, "last desc rx bytes=%u\n", count);
2199                         }
2200
2201                         tmp = req->req.length - req->req.actual;
2202                         if (count > tmp) {
2203                                 if ((tmp % ep->ep.maxpacket) != 0) {
2204                                         DBG(dev, "%s: rx %db, space=%db\n",
2205                                                 ep->ep.name, count, tmp);
2206                                         req->req.status = -EOVERFLOW;
2207                                 }
2208                                 count = tmp;
2209                         }
2210                         req->req.actual += count;
2211                         req->dma_going = 0;
2212                         /* complete request */
2213                         complete_req(ep, req, 0);
2214
2215                         /* next request */
2216                         if (!list_empty(&ep->queue) && !ep->halted) {
2217                                 req = list_entry(ep->queue.next,
2218                                         struct udc_request,
2219                                         queue);
2220                                 /*
2221                                  * DMA may be already started by udc_queue()
2222                                  * called by gadget drivers completion
2223                                  * routine. This happens when queue
2224                                  * holds one request only.
2225                                  */
2226                                 if (req->dma_going == 0) {
2227                                         /* next dma */
2228                                         if (prep_dma(ep, req, GFP_ATOMIC) != 0)
2229                                                 goto finished;
2230                                         /* write desc pointer */
2231                                         writel(req->td_phys,
2232                                                 &ep->regs->desptr);
2233                                         req->dma_going = 1;
2234                                         /* enable DMA */
2235                                         udc_set_rde(dev);
2236                                 }
2237                         } else {
2238                                 /*
2239                                  * implant BNA dummy descriptor to allow
2240                                  * RXFIFO opening by RDE
2241                                  */
2242                                 if (ep->bna_dummy_req) {
2243                                         /* write desc pointer */
2244                                         writel(ep->bna_dummy_req->td_phys,
2245                                                 &ep->regs->desptr);
2246                                         ep->bna_occurred = 0;
2247                                 }
2248
2249                                 /*
2250                                  * schedule timer for setting RDE if queue
2251                                  * remains empty to allow ep0 packets pass
2252                                  * through
2253                                  */
2254                                 if (set_rde != 0
2255                                                 && !timer_pending(&udc_timer)) {
2256                                         udc_timer.expires =
2257                                                 jiffies
2258                                                 + HZ*UDC_RDE_TIMER_SECONDS;
2259                                         set_rde = 1;
2260                                         if (!stop_timer)
2261                                                 add_timer(&udc_timer);
2262                                 }
2263                                 if (ep->num != UDC_EP0OUT_IX)
2264                                         dev->data_ep_queued = 0;
2265                         }
2266
2267                 } else {
2268                         /*
2269                         * RX DMA must be reenabled for each desc in PPBDU mode
2270                         * and must be enabled for PPBNDU mode in case of BNA
2271                         */
2272                         udc_set_rde(dev);
2273                 }
2274
2275         } else if (ep->cancel_transfer) {
2276                 ret_val = IRQ_HANDLED;
2277                 ep->cancel_transfer = 0;
2278         }
2279
2280         /* check pending CNAKS */
2281         if (cnak_pending) {
2282                 /* CNAk processing when rxfifo empty only */
2283                 if (readl(&dev->regs->sts) & AMD_BIT(UDC_DEVSTS_RXFIFO_EMPTY))
2284                         udc_process_cnak_queue(dev);
2285         }
2286
2287         /* clear OUT bits in ep status */
2288         writel(UDC_EPSTS_OUT_CLEAR, &ep->regs->sts);
2289 finished:
2290         return ret_val;
2291 }
2292
2293 /* Interrupt handler for data IN traffic */
2294 static irqreturn_t udc_data_in_isr(struct udc *dev, int ep_ix)
2295 {
2296         irqreturn_t ret_val = IRQ_NONE;
2297         u32 tmp;
2298         u32 epsts;
2299         struct udc_ep *ep;
2300         struct udc_request *req;
2301         struct udc_data_dma *td;
2302         unsigned dma_done;
2303         unsigned len;
2304
2305         ep = &dev->ep[ep_ix];
2306
2307         epsts = readl(&ep->regs->sts);
2308         if (use_dma) {
2309                 /* BNA ? */
2310                 if (epsts & AMD_BIT(UDC_EPSTS_BNA)) {
2311                         dev_err(&dev->pdev->dev,
2312                                 "BNA ep%din occurred - DESPTR = %08lx\n",
2313                                 ep->num,
2314                                 (unsigned long) readl(&ep->regs->desptr));
2315
2316                         /* clear BNA */
2317                         writel(epsts, &ep->regs->sts);
2318                         ret_val = IRQ_HANDLED;
2319                         goto finished;
2320                 }
2321         }
2322         /* HE event ? */
2323         if (epsts & AMD_BIT(UDC_EPSTS_HE)) {
2324                 dev_err(&dev->pdev->dev,
2325                         "HE ep%dn occurred - DESPTR = %08lx\n",
2326                         ep->num, (unsigned long) readl(&ep->regs->desptr));
2327
2328                 /* clear HE */
2329                 writel(epsts | AMD_BIT(UDC_EPSTS_HE), &ep->regs->sts);
2330                 ret_val = IRQ_HANDLED;
2331                 goto finished;
2332         }
2333
2334         /* DMA completion */
2335         if (epsts & AMD_BIT(UDC_EPSTS_TDC)) {
2336                 VDBG(dev, "TDC set- completion\n");
2337                 ret_val = IRQ_HANDLED;
2338                 if (!ep->cancel_transfer && !list_empty(&ep->queue)) {
2339                         req = list_entry(ep->queue.next,
2340                                         struct udc_request, queue);
2341                         /*
2342                          * length bytes transferred
2343                          * check dma done of last desc. in PPBDU mode
2344                          */
2345                         if (use_dma_ppb_du) {
2346                                 td = udc_get_last_dma_desc(req);
2347                                 if (td) {
2348                                         dma_done =
2349                                                 AMD_GETBITS(td->status,
2350                                                 UDC_DMA_IN_STS_BS);
2351                                         /* don't care DMA done */
2352                                         req->req.actual = req->req.length;
2353                                 }
2354                         } else {
2355                                 /* assume all bytes transferred */
2356                                 req->req.actual = req->req.length;
2357                         }
2358
2359                         if (req->req.actual == req->req.length) {
2360                                 /* complete req */
2361                                 complete_req(ep, req, 0);
2362                                 req->dma_going = 0;
2363                                 /* further request available ? */
2364                                 if (list_empty(&ep->queue)) {
2365                                         /* disable interrupt */
2366                                         tmp = readl(&dev->regs->ep_irqmsk);
2367                                         tmp |= AMD_BIT(ep->num);
2368                                         writel(tmp, &dev->regs->ep_irqmsk);
2369                                 }
2370                         }
2371                 }
2372                 ep->cancel_transfer = 0;
2373
2374         }
2375         /*
2376          * status reg has IN bit set and TDC not set (if TDC was handled,
2377          * IN must not be handled (UDC defect) ?
2378          */
2379         if ((epsts & AMD_BIT(UDC_EPSTS_IN))
2380                         && !(epsts & AMD_BIT(UDC_EPSTS_TDC))) {
2381                 ret_val = IRQ_HANDLED;
2382                 if (!list_empty(&ep->queue)) {
2383                         /* next request */
2384                         req = list_entry(ep->queue.next,
2385                                         struct udc_request, queue);
2386                         /* FIFO mode */
2387                         if (!use_dma) {
2388                                 /* write fifo */
2389                                 udc_txfifo_write(ep, &req->req);
2390                                 len = req->req.length - req->req.actual;
2391                                 if (len > ep->ep.maxpacket)
2392                                         len = ep->ep.maxpacket;
2393                                 req->req.actual += len;
2394                                 if (req->req.actual == req->req.length
2395                                         || (len != ep->ep.maxpacket)) {
2396                                         /* complete req */
2397                                         complete_req(ep, req, 0);
2398                                 }
2399                         /* DMA */
2400                         } else if (req && !req->dma_going) {
2401                                 VDBG(dev, "IN DMA : req=%p req->td_data=%p\n",
2402                                         req, req->td_data);
2403                                 if (req->td_data) {
2404
2405                                         req->dma_going = 1;
2406
2407                                         /*
2408                                          * unset L bit of first desc.
2409                                          * for chain
2410                                          */
2411                                         if (use_dma_ppb && req->req.length >
2412                                                         ep->ep.maxpacket) {
2413                                                 req->td_data->status &=
2414                                                         AMD_CLEAR_BIT(
2415                                                         UDC_DMA_IN_STS_L);
2416                                         }
2417
2418                                         /* write desc pointer */
2419                                         writel(req->td_phys, &ep->regs->desptr);
2420
2421                                         /* set HOST READY */
2422                                         req->td_data->status =
2423                                                 AMD_ADDBITS(
2424                                                 req->td_data->status,
2425                                                 UDC_DMA_IN_STS_BS_HOST_READY,
2426                                                 UDC_DMA_IN_STS_BS);
2427
2428                                         /* set poll demand bit */
2429                                         tmp = readl(&ep->regs->ctl);
2430                                         tmp |= AMD_BIT(UDC_EPCTL_P);
2431                                         writel(tmp, &ep->regs->ctl);
2432                                 }
2433                         }
2434
2435                 } else if (!use_dma && ep->in) {
2436                         /* disable interrupt */
2437                         tmp = readl(
2438                                 &dev->regs->ep_irqmsk);
2439                         tmp |= AMD_BIT(ep->num);
2440                         writel(tmp,
2441                                 &dev->regs->ep_irqmsk);
2442                 }
2443         }
2444         /* clear status bits */
2445         writel(epsts, &ep->regs->sts);
2446
2447 finished:
2448         return ret_val;
2449
2450 }
2451
2452 /* Interrupt handler for Control OUT traffic */
2453 static irqreturn_t udc_control_out_isr(struct udc *dev)
2454 __releases(dev->lock)
2455 __acquires(dev->lock)
2456 {
2457         irqreturn_t ret_val = IRQ_NONE;
2458         u32 tmp;
2459         int setup_supported;
2460         u32 count;
2461         int set = 0;
2462         struct udc_ep   *ep;
2463         struct udc_ep   *ep_tmp;
2464
2465         ep = &dev->ep[UDC_EP0OUT_IX];
2466
2467         /* clear irq */
2468         writel(AMD_BIT(UDC_EPINT_OUT_EP0), &dev->regs->ep_irqsts);
2469
2470         tmp = readl(&dev->ep[UDC_EP0OUT_IX].regs->sts);
2471         /* check BNA and clear if set */
2472         if (tmp & AMD_BIT(UDC_EPSTS_BNA)) {
2473                 VDBG(dev, "ep0: BNA set\n");
2474                 writel(AMD_BIT(UDC_EPSTS_BNA),
2475                         &dev->ep[UDC_EP0OUT_IX].regs->sts);
2476                 ep->bna_occurred = 1;
2477                 ret_val = IRQ_HANDLED;
2478                 goto finished;
2479         }
2480
2481         /* type of data: SETUP or DATA 0 bytes */
2482         tmp = AMD_GETBITS(tmp, UDC_EPSTS_OUT);
2483         VDBG(dev, "data_typ = %x\n", tmp);
2484
2485         /* setup data */
2486         if (tmp == UDC_EPSTS_OUT_SETUP) {
2487                 ret_val = IRQ_HANDLED;
2488
2489                 ep->dev->stall_ep0in = 0;
2490                 dev->waiting_zlp_ack_ep0in = 0;
2491
2492                 /* set NAK for EP0_IN */
2493                 tmp = readl(&dev->ep[UDC_EP0IN_IX].regs->ctl);
2494                 tmp |= AMD_BIT(UDC_EPCTL_SNAK);
2495                 writel(tmp, &dev->ep[UDC_EP0IN_IX].regs->ctl);
2496                 dev->ep[UDC_EP0IN_IX].naking = 1;
2497                 /* get setup data */
2498                 if (use_dma) {
2499
2500                         /* clear OUT bits in ep status */
2501                         writel(UDC_EPSTS_OUT_CLEAR,
2502                                 &dev->ep[UDC_EP0OUT_IX].regs->sts);
2503
2504                         setup_data.data[0] =
2505                                 dev->ep[UDC_EP0OUT_IX].td_stp->data12;
2506                         setup_data.data[1] =
2507                                 dev->ep[UDC_EP0OUT_IX].td_stp->data34;
2508                         /* set HOST READY */
2509                         dev->ep[UDC_EP0OUT_IX].td_stp->status =
2510                                         UDC_DMA_STP_STS_BS_HOST_READY;
2511                 } else {
2512                         /* read fifo */
2513                         udc_rxfifo_read_dwords(dev, setup_data.data, 2);
2514                 }
2515
2516                 /* determine direction of control data */
2517                 if ((setup_data.request.bRequestType & USB_DIR_IN) != 0) {
2518                         dev->gadget.ep0 = &dev->ep[UDC_EP0IN_IX].ep;
2519                         /* enable RDE */
2520                         udc_ep0_set_rde(dev);
2521                         set = 0;
2522                 } else {
2523                         dev->gadget.ep0 = &dev->ep[UDC_EP0OUT_IX].ep;
2524                         /*
2525                          * implant BNA dummy descriptor to allow RXFIFO opening
2526                          * by RDE
2527                          */
2528                         if (ep->bna_dummy_req) {
2529                                 /* write desc pointer */
2530                                 writel(ep->bna_dummy_req->td_phys,
2531                                         &dev->ep[UDC_EP0OUT_IX].regs->desptr);
2532                                 ep->bna_occurred = 0;
2533                         }
2534
2535                         set = 1;
2536                         dev->ep[UDC_EP0OUT_IX].naking = 1;
2537                         /*
2538                          * setup timer for enabling RDE (to not enable
2539                          * RXFIFO DMA for data to early)
2540                          */
2541                         set_rde = 1;
2542                         if (!timer_pending(&udc_timer)) {
2543                                 udc_timer.expires = jiffies +
2544                                                         HZ/UDC_RDE_TIMER_DIV;
2545                                 if (!stop_timer)
2546                                         add_timer(&udc_timer);
2547                         }
2548                 }
2549
2550                 /*
2551                  * mass storage reset must be processed here because
2552                  * next packet may be a CLEAR_FEATURE HALT which would not
2553                  * clear the stall bit when no STALL handshake was received
2554                  * before (autostall can cause this)
2555                  */
2556                 if (setup_data.data[0] == UDC_MSCRES_DWORD0
2557                                 && setup_data.data[1] == UDC_MSCRES_DWORD1) {
2558                         DBG(dev, "MSC Reset\n");
2559                         /*
2560                          * clear stall bits
2561                          * only one IN and OUT endpoints are handled
2562                          */
2563                         ep_tmp = &udc->ep[UDC_EPIN_IX];
2564                         udc_set_halt(&ep_tmp->ep, 0);
2565                         ep_tmp = &udc->ep[UDC_EPOUT_IX];
2566                         udc_set_halt(&ep_tmp->ep, 0);
2567                 }
2568
2569                 /* call gadget with setup data received */
2570                 spin_unlock(&dev->lock);
2571                 setup_supported = dev->driver->setup(&dev->gadget,
2572                                                 &setup_data.request);
2573                 spin_lock(&dev->lock);
2574
2575                 tmp = readl(&dev->ep[UDC_EP0IN_IX].regs->ctl);
2576                 /* ep0 in returns data (not zlp) on IN phase */
2577                 if (setup_supported >= 0 && setup_supported <
2578                                 UDC_EP0IN_MAXPACKET) {
2579                         /* clear NAK by writing CNAK in EP0_IN */
2580                         tmp |= AMD_BIT(UDC_EPCTL_CNAK);
2581                         writel(tmp, &dev->ep[UDC_EP0IN_IX].regs->ctl);
2582                         dev->ep[UDC_EP0IN_IX].naking = 0;
2583                         UDC_QUEUE_CNAK(&dev->ep[UDC_EP0IN_IX], UDC_EP0IN_IX);
2584
2585                 /* if unsupported request then stall */
2586                 } else if (setup_supported < 0) {
2587                         tmp |= AMD_BIT(UDC_EPCTL_S);
2588                         writel(tmp, &dev->ep[UDC_EP0IN_IX].regs->ctl);
2589                 } else
2590                         dev->waiting_zlp_ack_ep0in = 1;
2591
2592
2593                 /* clear NAK by writing CNAK in EP0_OUT */
2594                 if (!set) {
2595                         tmp = readl(&dev->ep[UDC_EP0OUT_IX].regs->ctl);
2596                         tmp |= AMD_BIT(UDC_EPCTL_CNAK);
2597                         writel(tmp, &dev->ep[UDC_EP0OUT_IX].regs->ctl);
2598                         dev->ep[UDC_EP0OUT_IX].naking = 0;
2599                         UDC_QUEUE_CNAK(&dev->ep[UDC_EP0OUT_IX], UDC_EP0OUT_IX);
2600                 }
2601
2602                 if (!use_dma) {
2603                         /* clear OUT bits in ep status */
2604                         writel(UDC_EPSTS_OUT_CLEAR,
2605                                 &dev->ep[UDC_EP0OUT_IX].regs->sts);
2606                 }
2607
2608         /* data packet 0 bytes */
2609         } else if (tmp == UDC_EPSTS_OUT_DATA) {
2610                 /* clear OUT bits in ep status */
2611                 writel(UDC_EPSTS_OUT_CLEAR, &dev->ep[UDC_EP0OUT_IX].regs->sts);
2612
2613                 /* get setup data: only 0 packet */
2614                 if (use_dma) {
2615                         /* no req if 0 packet, just reactivate */
2616                         if (list_empty(&dev->ep[UDC_EP0OUT_IX].queue)) {
2617                                 VDBG(dev, "ZLP\n");
2618
2619                                 /* set HOST READY */
2620                                 dev->ep[UDC_EP0OUT_IX].td->status =
2621                                         AMD_ADDBITS(
2622                                         dev->ep[UDC_EP0OUT_IX].td->status,
2623                                         UDC_DMA_OUT_STS_BS_HOST_READY,
2624                                         UDC_DMA_OUT_STS_BS);
2625                                 /* enable RDE */
2626                                 udc_ep0_set_rde(dev);
2627                                 ret_val = IRQ_HANDLED;
2628
2629                         } else {
2630                                 /* control write */
2631                                 ret_val |= udc_data_out_isr(dev, UDC_EP0OUT_IX);
2632                                 /* re-program desc. pointer for possible ZLPs */
2633                                 writel(dev->ep[UDC_EP0OUT_IX].td_phys,
2634                                         &dev->ep[UDC_EP0OUT_IX].regs->desptr);
2635                                 /* enable RDE */
2636                                 udc_ep0_set_rde(dev);
2637                         }
2638                 } else {
2639
2640                         /* received number bytes */
2641                         count = readl(&dev->ep[UDC_EP0OUT_IX].regs->sts);
2642                         count = AMD_GETBITS(count, UDC_EPSTS_RX_PKT_SIZE);
2643                         /* out data for fifo mode not working */
2644                         count = 0;
2645
2646                         /* 0 packet or real data ? */
2647                         if (count != 0) {
2648                                 ret_val |= udc_data_out_isr(dev, UDC_EP0OUT_IX);
2649                         } else {
2650                                 /* dummy read confirm */
2651                                 readl(&dev->ep[UDC_EP0OUT_IX].regs->confirm);
2652                                 ret_val = IRQ_HANDLED;
2653                         }
2654                 }
2655         }
2656
2657         /* check pending CNAKS */
2658         if (cnak_pending) {
2659                 /* CNAk processing when rxfifo empty only */
2660                 if (readl(&dev->regs->sts) & AMD_BIT(UDC_DEVSTS_RXFIFO_EMPTY))
2661                         udc_process_cnak_queue(dev);
2662         }
2663
2664 finished:
2665         return ret_val;
2666 }
2667
2668 /* Interrupt handler for Control IN traffic */
2669 static irqreturn_t udc_control_in_isr(struct udc *dev)
2670 {
2671         irqreturn_t ret_val = IRQ_NONE;
2672         u32 tmp;
2673         struct udc_ep *ep;
2674         struct udc_request *req;
2675         unsigned len;
2676
2677         ep = &dev->ep[UDC_EP0IN_IX];
2678
2679         /* clear irq */
2680         writel(AMD_BIT(UDC_EPINT_IN_EP0), &dev->regs->ep_irqsts);
2681
2682         tmp = readl(&dev->ep[UDC_EP0IN_IX].regs->sts);
2683         /* DMA completion */
2684         if (tmp & AMD_BIT(UDC_EPSTS_TDC)) {
2685                 VDBG(dev, "isr: TDC clear\n");
2686                 ret_val = IRQ_HANDLED;
2687
2688                 /* clear TDC bit */
2689                 writel(AMD_BIT(UDC_EPSTS_TDC),
2690                                 &dev->ep[UDC_EP0IN_IX].regs->sts);
2691
2692         /* status reg has IN bit set ? */
2693         } else if (tmp & AMD_BIT(UDC_EPSTS_IN)) {
2694                 ret_val = IRQ_HANDLED;
2695
2696                 if (ep->dma) {
2697                         /* clear IN bit */
2698                         writel(AMD_BIT(UDC_EPSTS_IN),
2699                                 &dev->ep[UDC_EP0IN_IX].regs->sts);
2700                 }
2701                 if (dev->stall_ep0in) {
2702                         DBG(dev, "stall ep0in\n");
2703                         /* halt ep0in */
2704                         tmp = readl(&ep->regs->ctl);
2705                         tmp |= AMD_BIT(UDC_EPCTL_S);
2706                         writel(tmp, &ep->regs->ctl);
2707                 } else {
2708                         if (!list_empty(&ep->queue)) {
2709                                 /* next request */
2710                                 req = list_entry(ep->queue.next,
2711                                                 struct udc_request, queue);
2712
2713                                 if (ep->dma) {
2714                                         /* write desc pointer */
2715                                         writel(req->td_phys, &ep->regs->desptr);
2716                                         /* set HOST READY */
2717                                         req->td_data->status =
2718                                                 AMD_ADDBITS(
2719                                                 req->td_data->status,
2720                                                 UDC_DMA_STP_STS_BS_HOST_READY,
2721                                                 UDC_DMA_STP_STS_BS);
2722
2723                                         /* set poll demand bit */
2724                                         tmp =
2725                                         readl(&dev->ep[UDC_EP0IN_IX].regs->ctl);
2726                                         tmp |= AMD_BIT(UDC_EPCTL_P);
2727                                         writel(tmp,
2728                                         &dev->ep[UDC_EP0IN_IX].regs->ctl);
2729
2730                                         /* all bytes will be transferred */
2731                                         req->req.actual = req->req.length;
2732
2733                                         /* complete req */
2734                                         complete_req(ep, req, 0);
2735
2736                                 } else {
2737                                         /* write fifo */
2738                                         udc_txfifo_write(ep, &req->req);
2739
2740                                         /* lengh bytes transferred */
2741                                         len = req->req.length - req->req.actual;
2742                                         if (len > ep->ep.maxpacket)
2743                                                 len = ep->ep.maxpacket;
2744
2745                                         req->req.actual += len;
2746                                         if (req->req.actual == req->req.length
2747                                                 || (len != ep->ep.maxpacket)) {
2748                                                 /* complete req */
2749                                                 complete_req(ep, req, 0);
2750                                         }
2751                                 }
2752
2753                         }
2754                 }
2755                 ep->halted = 0;
2756                 dev->stall_ep0in = 0;
2757                 if (!ep->dma) {
2758                         /* clear IN bit */
2759                         writel(AMD_BIT(UDC_EPSTS_IN),
2760                                 &dev->ep[UDC_EP0IN_IX].regs->sts);
2761                 }
2762         }
2763
2764         return ret_val;
2765 }
2766
2767
2768 /* Interrupt handler for global device events */
2769 static irqreturn_t udc_dev_isr(struct udc *dev, u32 dev_irq)
2770 __releases(dev->lock)
2771 __acquires(dev->lock)
2772 {
2773         irqreturn_t ret_val = IRQ_NONE;
2774         u32 tmp;
2775         u32 cfg;
2776         struct udc_ep *ep;
2777         u16 i;
2778         u8 udc_csr_epix;
2779
2780         /* SET_CONFIG irq ? */
2781         if (dev_irq & AMD_BIT(UDC_DEVINT_SC)) {
2782                 ret_val = IRQ_HANDLED;
2783
2784                 /* read config value */
2785                 tmp = readl(&dev->regs->sts);
2786                 cfg = AMD_GETBITS(tmp, UDC_DEVSTS_CFG);
2787                 DBG(dev, "SET_CONFIG interrupt: config=%d\n", cfg);
2788                 dev->cur_config = cfg;
2789                 dev->set_cfg_not_acked = 1;
2790
2791                 /* make usb request for gadget driver */
2792                 memset(&setup_data, 0 , sizeof(union udc_setup_data));
2793                 setup_data.request.bRequest = USB_REQ_SET_CONFIGURATION;
2794                 setup_data.request.wValue = cpu_to_le16(dev->cur_config);
2795
2796                 /* programm the NE registers */
2797                 for (i = 0; i < UDC_EP_NUM; i++) {
2798                         ep = &dev->ep[i];
2799                         if (ep->in) {
2800
2801                                 /* ep ix in UDC CSR register space */
2802                                 udc_csr_epix = ep->num;
2803
2804
2805                         /* OUT ep */
2806                         } else {
2807                                 /* ep ix in UDC CSR register space */
2808                                 udc_csr_epix = ep->num - UDC_CSR_EP_OUT_IX_OFS;
2809                         }
2810
2811                         tmp = readl(&dev->csr->ne[udc_csr_epix]);
2812                         /* ep cfg */
2813                         tmp = AMD_ADDBITS(tmp, ep->dev->cur_config,
2814                                                 UDC_CSR_NE_CFG);
2815                         /* write reg */
2816                         writel(tmp, &dev->csr->ne[udc_csr_epix]);
2817
2818                         /* clear stall bits */
2819                         ep->halted = 0;
2820                         tmp = readl(&ep->regs->ctl);
2821                         tmp = tmp & AMD_CLEAR_BIT(UDC_EPCTL_S);
2822                         writel(tmp, &ep->regs->ctl);
2823                 }
2824                 /* call gadget zero with setup data received */
2825                 spin_unlock(&dev->lock);
2826                 tmp = dev->driver->setup(&dev->gadget, &setup_data.request);
2827                 spin_lock(&dev->lock);
2828
2829         } /* SET_INTERFACE ? */
2830         if (dev_irq & AMD_BIT(UDC_DEVINT_SI)) {
2831                 ret_val = IRQ_HANDLED;
2832
2833                 dev->set_cfg_not_acked = 1;
2834                 /* read interface and alt setting values */
2835                 tmp = readl(&dev->regs->sts);
2836                 dev->cur_alt = AMD_GETBITS(tmp, UDC_DEVSTS_ALT);
2837                 dev->cur_intf = AMD_GETBITS(tmp, UDC_DEVSTS_INTF);
2838
2839                 /* make usb request for gadget driver */
2840                 memset(&setup_data, 0 , sizeof(union udc_setup_data));
2841                 setup_data.request.bRequest = USB_REQ_SET_INTERFACE;
2842                 setup_data.request.bRequestType = USB_RECIP_INTERFACE;
2843                 setup_data.request.wValue = cpu_to_le16(dev->cur_alt);
2844                 setup_data.request.wIndex = cpu_to_le16(dev->cur_intf);
2845
2846                 DBG(dev, "SET_INTERFACE interrupt: alt=%d intf=%d\n",
2847                                 dev->cur_alt, dev->cur_intf);
2848
2849                 /* programm the NE registers */
2850                 for (i = 0; i < UDC_EP_NUM; i++) {
2851                         ep = &dev->ep[i];
2852                         if (ep->in) {
2853
2854                                 /* ep ix in UDC CSR register space */
2855                                 udc_csr_epix = ep->num;
2856
2857
2858                         /* OUT ep */
2859                         } else {
2860                                 /* ep ix in UDC CSR register space */
2861                                 udc_csr_epix = ep->num - UDC_CSR_EP_OUT_IX_OFS;
2862                         }
2863
2864                         /* UDC CSR reg */
2865                         /* set ep values */
2866                         tmp = readl(&dev->csr->ne[udc_csr_epix]);
2867                         /* ep interface */
2868                         tmp = AMD_ADDBITS(tmp, ep->dev->cur_intf,
2869                                                 UDC_CSR_NE_INTF);
2870                         /* tmp = AMD_ADDBITS(tmp, 2, UDC_CSR_NE_INTF); */
2871                         /* ep alt */
2872                         tmp = AMD_ADDBITS(tmp, ep->dev->cur_alt,
2873                                                 UDC_CSR_NE_ALT);
2874                         /* write reg */
2875                         writel(tmp, &dev->csr->ne[udc_csr_epix]);
2876
2877                         /* clear stall bits */
2878                         ep->halted = 0;
2879                         tmp = readl(&ep->regs->ctl);
2880                         tmp = tmp & AMD_CLEAR_BIT(UDC_EPCTL_S);
2881                         writel(tmp, &ep->regs->ctl);
2882                 }
2883
2884                 /* call gadget zero with setup data received */
2885                 spin_unlock(&dev->lock);
2886                 tmp = dev->driver->setup(&dev->gadget, &setup_data.request);
2887                 spin_lock(&dev->lock);
2888
2889         } /* USB reset */
2890         if (dev_irq & AMD_BIT(UDC_DEVINT_UR)) {
2891                 DBG(dev, "USB Reset interrupt\n");
2892                 ret_val = IRQ_HANDLED;
2893
2894                 /* allow soft reset when suspend occurs */
2895                 soft_reset_occured = 0;
2896
2897                 dev->waiting_zlp_ack_ep0in = 0;
2898                 dev->set_cfg_not_acked = 0;
2899
2900                 /* mask not needed interrupts */
2901                 udc_mask_unused_interrupts(dev);
2902
2903                 /* call gadget to resume and reset configs etc. */
2904                 spin_unlock(&dev->lock);
2905                 if (dev->sys_suspended && dev->driver->resume) {
2906                         dev->driver->resume(&dev->gadget);
2907                         dev->sys_suspended = 0;
2908                 }
2909                 dev->driver->disconnect(&dev->gadget);
2910                 spin_lock(&dev->lock);
2911
2912                 /* disable ep0 to empty req queue */
2913                 empty_req_queue(&dev->ep[UDC_EP0IN_IX]);
2914                 ep_init(dev->regs, &dev->ep[UDC_EP0IN_IX]);
2915
2916                 /* soft reset when rxfifo not empty */
2917                 tmp = readl(&dev->regs->sts);
2918                 if (!(tmp & AMD_BIT(UDC_DEVSTS_RXFIFO_EMPTY))
2919                                 && !soft_reset_after_usbreset_occured) {
2920                         udc_soft_reset(dev);
2921                         soft_reset_after_usbreset_occured++;
2922                 }
2923
2924                 /*
2925                  * DMA reset to kill potential old DMA hw hang,
2926                  * POLL bit is already reset by ep_init() through
2927                  * disconnect()
2928                  */
2929                 DBG(dev, "DMA machine reset\n");
2930                 tmp = readl(&dev->regs->cfg);
2931                 writel(tmp | AMD_BIT(UDC_DEVCFG_DMARST), &dev->regs->cfg);
2932                 writel(tmp, &dev->regs->cfg);
2933
2934                 /* put into initial config */
2935                 udc_basic_init(dev);
2936
2937                 /* enable device setup interrupts */
2938                 udc_enable_dev_setup_interrupts(dev);
2939
2940                 /* enable suspend interrupt */
2941                 tmp = readl(&dev->regs->irqmsk);
2942                 tmp &= AMD_UNMASK_BIT(UDC_DEVINT_US);
2943                 writel(tmp, &dev->regs->irqmsk);
2944
2945         } /* USB suspend */
2946         if (dev_irq & AMD_BIT(UDC_DEVINT_US)) {
2947                 DBG(dev, "USB Suspend interrupt\n");
2948                 ret_val = IRQ_HANDLED;
2949                 if (dev->driver->suspend) {
2950                         spin_unlock(&dev->lock);
2951                         dev->sys_suspended = 1;
2952                         dev->driver->suspend(&dev->gadget);
2953                         spin_lock(&dev->lock);
2954                 }
2955         } /* new speed ? */
2956         if (dev_irq & AMD_BIT(UDC_DEVINT_ENUM)) {
2957                 DBG(dev, "ENUM interrupt\n");
2958                 ret_val = IRQ_HANDLED;
2959                 soft_reset_after_usbreset_occured = 0;
2960
2961                 /* disable ep0 to empty req queue */
2962                 empty_req_queue(&dev->ep[UDC_EP0IN_IX]);
2963                 ep_init(dev->regs, &dev->ep[UDC_EP0IN_IX]);
2964
2965                 /* link up all endpoints */
2966                 udc_setup_endpoints(dev);
2967                 dev_info(&dev->pdev->dev, "Connect: %s\n",
2968                          usb_speed_string(dev->gadget.speed));
2969
2970                 /* init ep 0 */
2971                 activate_control_endpoints(dev);
2972
2973                 /* enable ep0 interrupts */
2974                 udc_enable_ep0_interrupts(dev);
2975         }
2976         /* session valid change interrupt */
2977         if (dev_irq & AMD_BIT(UDC_DEVINT_SVC)) {
2978                 DBG(dev, "USB SVC interrupt\n");
2979                 ret_val = IRQ_HANDLED;
2980
2981                 /* check that session is not valid to detect disconnect */
2982                 tmp = readl(&dev->regs->sts);
2983                 if (!(tmp & AMD_BIT(UDC_DEVSTS_SESSVLD))) {
2984                         /* disable suspend interrupt */
2985                         tmp = readl(&dev->regs->irqmsk);
2986                         tmp |= AMD_BIT(UDC_DEVINT_US);
2987                         writel(tmp, &dev->regs->irqmsk);
2988                         DBG(dev, "USB Disconnect (session valid low)\n");
2989                         /* cleanup on disconnect */
2990                         usb_disconnect(udc);
2991                 }
2992
2993         }
2994
2995         return ret_val;
2996 }
2997
2998 /* Interrupt Service Routine, see Linux Kernel Doc for parameters */
2999 static irqreturn_t udc_irq(int irq, void *pdev)
3000 {
3001         struct udc *dev = pdev;
3002         u32 reg;
3003         u16 i;
3004         u32 ep_irq;
3005         irqreturn_t ret_val = IRQ_NONE;
3006
3007         spin_lock(&dev->lock);
3008
3009         /* check for ep irq */
3010         reg = readl(&dev->regs->ep_irqsts);
3011         if (reg) {
3012                 if (reg & AMD_BIT(UDC_EPINT_OUT_EP0))
3013                         ret_val |= udc_control_out_isr(dev);
3014                 if (reg & AMD_BIT(UDC_EPINT_IN_EP0))
3015                         ret_val |= udc_control_in_isr(dev);
3016
3017                 /*
3018                  * data endpoint
3019                  * iterate ep's
3020                  */
3021                 for (i = 1; i < UDC_EP_NUM; i++) {
3022                         ep_irq = 1 << i;
3023                         if (!(reg & ep_irq) || i == UDC_EPINT_OUT_EP0)
3024                                 continue;
3025
3026                         /* clear irq status */
3027                         writel(ep_irq, &dev->regs->ep_irqsts);
3028
3029                         /* irq for out ep ? */
3030                         if (i > UDC_EPIN_NUM)
3031                                 ret_val |= udc_data_out_isr(dev, i);
3032                         else
3033                                 ret_val |= udc_data_in_isr(dev, i);
3034                 }
3035
3036         }
3037
3038
3039         /* check for dev irq */
3040         reg = readl(&dev->regs->irqsts);
3041         if (reg) {
3042                 /* clear irq */
3043                 writel(reg, &dev->regs->irqsts);
3044                 ret_val |= udc_dev_isr(dev, reg);
3045         }
3046
3047
3048         spin_unlock(&dev->lock);
3049         return ret_val;
3050 }
3051
3052 /* Tears down device */
3053 static void gadget_release(struct device *pdev)
3054 {
3055         struct amd5536udc *dev = dev_get_drvdata(pdev);
3056         kfree(dev);
3057 }
3058
3059 /* Cleanup on device remove */
3060 static void udc_remove(struct udc *dev)
3061 {
3062         /* remove timer */
3063         stop_timer++;
3064         if (timer_pending(&udc_timer))
3065                 wait_for_completion(&on_exit);
3066         if (udc_timer.data)
3067                 del_timer_sync(&udc_timer);
3068         /* remove pollstall timer */
3069         stop_pollstall_timer++;
3070         if (timer_pending(&udc_pollstall_timer))
3071                 wait_for_completion(&on_pollstall_exit);
3072         if (udc_pollstall_timer.data)
3073                 del_timer_sync(&udc_pollstall_timer);
3074         udc = NULL;
3075 }
3076
3077 /* Reset all pci context */
3078 static void udc_pci_remove(struct pci_dev *pdev)
3079 {
3080         struct udc              *dev;
3081
3082         dev = pci_get_drvdata(pdev);
3083
3084         usb_del_gadget_udc(&udc->gadget);
3085         /* gadget driver must not be registered */
3086         BUG_ON(dev->driver != NULL);
3087
3088         /* dma pool cleanup */
3089         if (dev->data_requests)
3090                 pci_pool_destroy(dev->data_requests);
3091
3092         if (dev->stp_requests) {
3093                 /* cleanup DMA desc's for ep0in */
3094                 pci_pool_free(dev->stp_requests,
3095                         dev->ep[UDC_EP0OUT_IX].td_stp,
3096                         dev->ep[UDC_EP0OUT_IX].td_stp_dma);
3097                 pci_pool_free(dev->stp_requests,
3098                         dev->ep[UDC_EP0OUT_IX].td,
3099                         dev->ep[UDC_EP0OUT_IX].td_phys);
3100
3101                 pci_pool_destroy(dev->stp_requests);
3102         }
3103
3104         /* reset controller */
3105         writel(AMD_BIT(UDC_DEVCFG_SOFTRESET), &dev->regs->cfg);
3106         if (dev->irq_registered)
3107                 free_irq(pdev->irq, dev);
3108         if (dev->regs)
3109                 iounmap(dev->regs);
3110         if (dev->mem_region)
3111                 release_mem_region(pci_resource_start(pdev, 0),
3112                                 pci_resource_len(pdev, 0));
3113         if (dev->active)
3114                 pci_disable_device(pdev);
3115
3116         device_unregister(&dev->gadget.dev);
3117         pci_set_drvdata(pdev, NULL);
3118
3119         udc_remove(dev);
3120 }
3121
3122 /* create dma pools on init */
3123 static int init_dma_pools(struct udc *dev)
3124 {
3125         struct udc_stp_dma      *td_stp;
3126         struct udc_data_dma     *td_data;
3127         int retval;
3128
3129         /* consistent DMA mode setting ? */
3130         if (use_dma_ppb) {
3131                 use_dma_bufferfill_mode = 0;
3132         } else {
3133                 use_dma_ppb_du = 0;
3134                 use_dma_bufferfill_mode = 1;
3135         }
3136
3137         /* DMA setup */
3138         dev->data_requests = dma_pool_create("data_requests", NULL,
3139                 sizeof(struct udc_data_dma), 0, 0);
3140         if (!dev->data_requests) {
3141                 DBG(dev, "can't get request data pool\n");
3142                 retval = -ENOMEM;
3143                 goto finished;
3144         }
3145
3146         /* EP0 in dma regs = dev control regs */
3147         dev->ep[UDC_EP0IN_IX].dma = &dev->regs->ctl;
3148
3149         /* dma desc for setup data */
3150         dev->stp_requests = dma_pool_create("setup requests", NULL,
3151                 sizeof(struct udc_stp_dma), 0, 0);
3152         if (!dev->stp_requests) {
3153                 DBG(dev, "can't get stp request pool\n");
3154                 retval = -ENOMEM;
3155                 goto finished;
3156         }
3157         /* setup */
3158         td_stp = dma_pool_alloc(dev->stp_requests, GFP_KERNEL,
3159                                 &dev->ep[UDC_EP0OUT_IX].td_stp_dma);
3160         if (td_stp == NULL) {
3161                 retval = -ENOMEM;
3162                 goto finished;
3163         }
3164         dev->ep[UDC_EP0OUT_IX].td_stp = td_stp;
3165
3166         /* data: 0 packets !? */
3167         td_data = dma_pool_alloc(dev->stp_requests, GFP_KERNEL,
3168                                 &dev->ep[UDC_EP0OUT_IX].td_phys);
3169         if (td_data == NULL) {
3170                 retval = -ENOMEM;
3171                 goto finished;
3172         }
3173         dev->ep[UDC_EP0OUT_IX].td = td_data;
3174         return 0;
3175
3176 finished:
3177         return retval;
3178 }
3179
3180 /* Called by pci bus driver to init pci context */
3181 static int udc_pci_probe(
3182         struct pci_dev *pdev,
3183         const struct pci_device_id *id
3184 )
3185 {
3186         struct udc              *dev;
3187         unsigned long           resource;
3188         unsigned long           len;
3189         int                     retval = 0;
3190
3191         /* one udc only */
3192         if (udc) {
3193                 dev_dbg(&pdev->dev, "already probed\n");
3194                 return -EBUSY;
3195         }
3196
3197         /* init */
3198         dev = kzalloc(sizeof(struct udc), GFP_KERNEL);
3199         if (!dev) {
3200                 retval = -ENOMEM;
3201                 goto finished;
3202         }
3203
3204         /* pci setup */
3205         if (pci_enable_device(pdev) < 0) {
3206                 kfree(dev);
3207                 dev = NULL;
3208                 retval = -ENODEV;
3209                 goto finished;
3210         }
3211         dev->active = 1;
3212
3213         /* PCI resource allocation */
3214         resource = pci_resource_start(pdev, 0);
3215         len = pci_resource_len(pdev, 0);
3216
3217         if (!request_mem_region(resource, len, name)) {
3218                 dev_dbg(&pdev->dev, "pci device used already\n");
3219                 kfree(dev);
3220                 dev = NULL;
3221                 retval = -EBUSY;
3222                 goto finished;
3223         }
3224         dev->mem_region = 1;
3225
3226         dev->virt_addr = ioremap_nocache(resource, len);
3227         if (dev->virt_addr == NULL) {
3228                 dev_dbg(&pdev->dev, "start address cannot be mapped\n");
3229                 kfree(dev);
3230                 dev = NULL;
3231                 retval = -EFAULT;
3232                 goto finished;
3233         }
3234
3235         if (!pdev->irq) {
3236                 dev_err(&dev->pdev->dev, "irq not set\n");
3237                 kfree(dev);
3238                 dev = NULL;
3239                 retval = -ENODEV;
3240                 goto finished;
3241         }
3242
3243         spin_lock_init(&dev->lock);
3244         /* udc csr registers base */
3245         dev->csr = dev->virt_addr + UDC_CSR_ADDR;
3246         /* dev registers base */
3247         dev->regs = dev->virt_addr + UDC_DEVCFG_ADDR;
3248         /* ep registers base */
3249         dev->ep_regs = dev->virt_addr + UDC_EPREGS_ADDR;
3250         /* fifo's base */
3251         dev->rxfifo = (u32 __iomem *)(dev->virt_addr + UDC_RXFIFO_ADDR);
3252         dev->txfifo = (u32 __iomem *)(dev->virt_addr + UDC_TXFIFO_ADDR);
3253
3254         if (request_irq(pdev->irq, udc_irq, IRQF_SHARED, name, dev) != 0) {
3255                 dev_dbg(&dev->pdev->dev, "request_irq(%d) fail\n", pdev->irq);
3256                 kfree(dev);
3257                 dev = NULL;
3258                 retval = -EBUSY;
3259                 goto finished;
3260         }
3261         dev->irq_registered = 1;
3262
3263         pci_set_drvdata(pdev, dev);
3264
3265         /* chip revision for Hs AMD5536 */
3266         dev->chiprev = pdev->revision;
3267
3268         pci_set_master(pdev);
3269         pci_try_set_mwi(pdev);
3270
3271         /* init dma pools */
3272         if (use_dma) {
3273                 retval = init_dma_pools(dev);
3274                 if (retval != 0)
3275                         goto finished;
3276         }
3277
3278         dev->phys_addr = resource;
3279         dev->irq = pdev->irq;
3280         dev->pdev = pdev;
3281         dev->gadget.dev.parent = &pdev->dev;
3282         dev->gadget.dev.dma_mask = pdev->dev.dma_mask;
3283
3284         /* general probing */
3285         if (udc_probe(dev) == 0)
3286                 return 0;
3287
3288 finished:
3289         if (dev)
3290                 udc_pci_remove(pdev);
3291         return retval;
3292 }
3293
3294 /* general probe */
3295 static int udc_probe(struct udc *dev)
3296 {
3297         char            tmp[128];
3298         u32             reg;
3299         int             retval;
3300
3301         /* mark timer as not initialized */
3302         udc_timer.data = 0;
3303         udc_pollstall_timer.data = 0;
3304
3305         /* device struct setup */
3306         dev->gadget.ops = &udc_ops;
3307
3308         dev_set_name(&dev->gadget.dev, "gadget");
3309         dev->gadget.dev.release = gadget_release;
3310         dev->gadget.name = name;
3311         dev->gadget.max_speed = USB_SPEED_HIGH;
3312
3313         /* init registers, interrupts, ... */
3314         startup_registers(dev);
3315
3316         dev_info(&dev->pdev->dev, "%s\n", mod_desc);
3317
3318         snprintf(tmp, sizeof tmp, "%d", dev->irq);
3319         dev_info(&dev->pdev->dev,
3320                 "irq %s, pci mem %08lx, chip rev %02x(Geode5536 %s)\n",
3321                 tmp, dev->phys_addr, dev->chiprev,
3322                 (dev->chiprev == UDC_HSA0_REV) ? "A0" : "B1");
3323         strcpy(tmp, UDC_DRIVER_VERSION_STRING);
3324         if (dev->chiprev == UDC_HSA0_REV) {
3325                 dev_err(&dev->pdev->dev, "chip revision is A0; too old\n");
3326                 retval = -ENODEV;
3327                 goto finished;
3328         }
3329         dev_info(&dev->pdev->dev,
3330                 "driver version: %s(for Geode5536 B1)\n", tmp);
3331         udc = dev;
3332
3333         retval = usb_add_gadget_udc(&udc->pdev->dev, &dev->gadget);
3334         if (retval)
3335                 goto finished;
3336
3337         retval = device_register(&dev->gadget.dev);
3338         if (retval) {
3339                 usb_del_gadget_udc(&dev->gadget);
3340                 put_device(&dev->gadget.dev);
3341                 goto finished;
3342         }
3343
3344         /* timer init */
3345         init_timer(&udc_timer);
3346         udc_timer.function = udc_timer_function;
3347         udc_timer.data = 1;
3348         /* timer pollstall init */
3349         init_timer(&udc_pollstall_timer);
3350         udc_pollstall_timer.function = udc_pollstall_timer_function;
3351         udc_pollstall_timer.data = 1;
3352
3353         /* set SD */
3354         reg = readl(&dev->regs->ctl);
3355         reg |= AMD_BIT(UDC_DEVCTL_SD);
3356         writel(reg, &dev->regs->ctl);
3357
3358         /* print dev register info */
3359         print_regs(dev);
3360
3361         return 0;
3362
3363 finished:
3364         return retval;
3365 }
3366
3367 /* Initiates a remote wakeup */
3368 static int udc_remote_wakeup(struct udc *dev)
3369 {
3370         unsigned long flags;
3371         u32 tmp;
3372
3373         DBG(dev, "UDC initiates remote wakeup\n");
3374
3375         spin_lock_irqsave(&dev->lock, flags);
3376
3377         tmp = readl(&dev->regs->ctl);
3378         tmp |= AMD_BIT(UDC_DEVCTL_RES);
3379         writel(tmp, &dev->regs->ctl);
3380         tmp &= AMD_CLEAR_BIT(UDC_DEVCTL_RES);
3381         writel(tmp, &dev->regs->ctl);
3382
3383         spin_unlock_irqrestore(&dev->lock, flags);
3384         return 0;
3385 }
3386
3387 /* PCI device parameters */
3388 static DEFINE_PCI_DEVICE_TABLE(pci_id) = {
3389         {
3390                 PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x2096),
3391                 .class =        (PCI_CLASS_SERIAL_USB << 8) | 0xfe,
3392                 .class_mask =   0xffffffff,
3393         },
3394         {},
3395 };
3396 MODULE_DEVICE_TABLE(pci, pci_id);
3397
3398 /* PCI functions */
3399 static struct pci_driver udc_pci_driver = {
3400         .name =         (char *) name,
3401         .id_table =     pci_id,
3402         .probe =        udc_pci_probe,
3403         .remove =       udc_pci_remove,
3404 };
3405
3406 /* Inits driver */
3407 static int __init init(void)
3408 {
3409         return pci_register_driver(&udc_pci_driver);
3410 }
3411 module_init(init);
3412
3413 /* Cleans driver */
3414 static void __exit cleanup(void)
3415 {
3416         pci_unregister_driver(&udc_pci_driver);
3417 }
3418 module_exit(cleanup);
3419
3420 MODULE_DESCRIPTION(UDC_MOD_DESCRIPTION);
3421 MODULE_AUTHOR("Thomas Dahlmann");
3422 MODULE_LICENSE("GPL");
3423