Pileus Git - ~andy/linux/blob - drivers/staging/octeon-usb/octeon-hcd.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 2008 Cavium Networks
   7  *
   8  * Some parts of the code were originally released under BSD license:
   9  *
  10  * Copyright (c) 2003-2010 Cavium Networks (support@cavium.com). All rights
  11  * reserved.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions are
  15  * met:
  16  *
  17  *   * Redistributions of source code must retain the above copyright
  18  *     notice, this list of conditions and the following disclaimer.
  19  *
  20  *   * Redistributions in binary form must reproduce the above
  21  *     copyright notice, this list of conditions and the following
  22  *     disclaimer in the documentation and/or other materials provided
  23  *     with the distribution.
  24  *
  25  *   * Neither the name of Cavium Networks nor the names of
  26  *     its contributors may be used to endorse or promote products
  27  *     derived from this software without specific prior written
  28  *     permission.
  29  *
  30  * This Software, including technical data, may be subject to U.S. export
  31  * control laws, including the U.S. Export Administration Act and its associated
  32  * regulations, and may be subject to export or import regulations in other
  33  * countries.
  34  *
  35  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
  36  * AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR
  37  * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
  38  * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION
  39  * OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
  40  * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
  41  * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
  42  * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
  43  * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
  44  * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
  45  */
  46 #include <linux/kernel.h>
  47 #include <linux/module.h>
  48 #include <linux/init.h>
  49 #include <linux/pci.h>
  50 #include <linux/interrupt.h>
  51 #include <linux/platform_device.h>
  52 #include <linux/usb.h>
  53
  54 #include <linux/time.h>
  55 #include <linux/delay.h>
  56
  57 #include <asm/octeon/cvmx.h>
  58 #include <asm/octeon/cvmx-iob-defs.h>
  59
  60 #include <linux/usb/hcd.h>
  61
  62 #include <linux/err.h>
  63
  64 #include <asm/octeon/octeon.h>
  65 #include <asm/octeon/cvmx-helper.h>
  66 #include <asm/octeon/cvmx-sysinfo.h>
  67 #include <asm/octeon/cvmx-helper-board.h>
  68
  69 #include "octeon-hcd.h"
  70
  71 /**
  72  * enum cvmx_usb_speed - the possible USB device speeds
  73  *
  74  * @CVMX_USB_SPEED_HIGH: Device is operation at 480Mbps
  75  * @CVMX_USB_SPEED_FULL: Device is operation at 12Mbps
  76  * @CVMX_USB_SPEED_LOW:  Device is operation at 1.5Mbps
  77  */
  78 enum cvmx_usb_speed {
  79         CVMX_USB_SPEED_HIGH = 0,
  80         CVMX_USB_SPEED_FULL = 1,
  81         CVMX_USB_SPEED_LOW = 2,
  82 };
  83
  84 /**
  85  * enum cvmx_usb_transfer - the possible USB transfer types
  86  *
  87  * @CVMX_USB_TRANSFER_CONTROL:     USB transfer type control for hub and status
  88  *                                 transfers
  89  * @CVMX_USB_TRANSFER_ISOCHRONOUS: USB transfer type isochronous for low
  90  *                                 priority periodic transfers
  91  * @CVMX_USB_TRANSFER_BULK:        USB transfer type bulk for large low priority
  92  *                                 transfers
  93  * @CVMX_USB_TRANSFER_INTERRUPT:   USB transfer type interrupt for high priority
  94  *                                 periodic transfers
  95  */
  96 enum cvmx_usb_transfer {
  97         CVMX_USB_TRANSFER_CONTROL = 0,
  98         CVMX_USB_TRANSFER_ISOCHRONOUS = 1,
  99         CVMX_USB_TRANSFER_BULK = 2,
 100         CVMX_USB_TRANSFER_INTERRUPT = 3,
 101 };
 102
 103 /**
 104  * enum cvmx_usb_direction - the transfer directions
 105  *
 106  * @CVMX_USB_DIRECTION_OUT: Data is transferring from Octeon to the device/host
 107  * @CVMX_USB_DIRECTION_IN:  Data is transferring from the device/host to Octeon
 108  */
 109 enum cvmx_usb_direction {
 110         CVMX_USB_DIRECTION_OUT,
 111         CVMX_USB_DIRECTION_IN,
 112 };
 113
 114 /**
 115  * enum cvmx_usb_complete - possible callback function status codes
 116  *
 117  * @CVMX_USB_COMPLETE_SUCCESS:    The transaction / operation finished without
 118  *                                any errors
 119  * @CVMX_USB_COMPLETE_SHORT:      FIXME: This is currently not implemented
 120  * @CVMX_USB_COMPLETE_CANCEL:     The transaction was canceled while in flight
 121  *                                by a user call to cvmx_usb_cancel
 122  * @CVMX_USB_COMPLETE_ERROR:      The transaction aborted with an unexpected
 123  *                                error status
 124  * @CVMX_USB_COMPLETE_STALL:      The transaction received a USB STALL response
 125  *                                from the device
 126  * @CVMX_USB_COMPLETE_XACTERR:    The transaction failed with an error from the
 127  *                                device even after a number of retries
 128  * @CVMX_USB_COMPLETE_DATATGLERR: The transaction failed with a data toggle
 129  *                                error even after a number of retries
 130  * @CVMX_USB_COMPLETE_BABBLEERR:  The transaction failed with a babble error
 131  * @CVMX_USB_COMPLETE_FRAMEERR:   The transaction failed with a frame error
 132  *                                even after a number of retries
 133  */
 134 enum cvmx_usb_complete {
 135         CVMX_USB_COMPLETE_SUCCESS,
 136         CVMX_USB_COMPLETE_SHORT,
 137         CVMX_USB_COMPLETE_CANCEL,
 138         CVMX_USB_COMPLETE_ERROR,
 139         CVMX_USB_COMPLETE_STALL,
 140         CVMX_USB_COMPLETE_XACTERR,
 141         CVMX_USB_COMPLETE_DATATGLERR,
 142         CVMX_USB_COMPLETE_BABBLEERR,
 143         CVMX_USB_COMPLETE_FRAMEERR,
 144 };
 145
 146 /**
 147  * struct cvmx_usb_port_status - the USB port status information
 148  *
 149  * @port_enabled:       1 = Usb port is enabled, 0 = disabled
 150  * @port_over_current:  1 = Over current detected, 0 = Over current not
 151  *                      detected. Octeon doesn't support over current detection.
 152  * @port_powered:       1 = Port power is being supplied to the device, 0 =
 153  *                      power is off. Octeon doesn't support turning port power
 154  *                      off.
 155  * @port_speed:         Current port speed.
 156  * @connected:          1 = A device is connected to the port, 0 = No device is
 157  *                      connected.
 158  * @connect_change:     1 = Device connected state changed since the last set
 159  *                      status call.
 160  */
 161 struct cvmx_usb_port_status {
 162         uint32_t reserved               : 25;
 163         uint32_t port_enabled           : 1;
 164         uint32_t port_over_current      : 1;
 165         uint32_t port_powered           : 1;
 166         enum cvmx_usb_speed port_speed  : 2;
 167         uint32_t connected              : 1;
 168         uint32_t connect_change         : 1;
 169 };
 170
 171 /**
 172  * union cvmx_usb_control_header - the structure of a Control packet header
 173  *
 174  * @s.request_type:     Bit 7 tells the direction: 1=IN, 0=OUT
 175  * @s.request           The standard usb request to make
 176  * @s.value             Value parameter for the request in little endian format
 177  * @s.index             Index for the request in little endian format
 178  * @s.length            Length of the data associated with this request in
 179  *                      little endian format
 180  */
 181 union cvmx_usb_control_header {
 182         uint64_t u64;
 183         struct {
 184                 uint64_t request_type   : 8;
 185                 uint64_t request        : 8;
 186                 uint64_t value          : 16;
 187                 uint64_t index          : 16;
 188                 uint64_t length         : 16;
 189         } s;
 190 };
 191
 192 /**
 193  * struct cvmx_usb_iso_packet - descriptor for Isochronous packets
 194  *
 195  * @offset:     This is the offset in bytes into the main buffer where this data
 196  *              is stored.
 197  * @length:     This is the length in bytes of the data.
 198  * @status:     This is the status of this individual packet transfer.
 199  */
 200 struct cvmx_usb_iso_packet {
 201         int offset;
 202         int length;
 203         enum cvmx_usb_complete status;
 204 };
 205
 206 /**
 207  * enum cvmx_usb_initialize_flags - flags used by the initialization function
 208  *
 209  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI:    The USB port uses a 12MHz crystal
 210  *                                            as clock source at USB_XO and
 211  *                                            USB_XI.
 212  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND:   The USB port uses 12/24/48MHz 2.5V
 213  *                                            board clock source at USB_XO.
 214  *                                            USB_XI should be tied to GND.
 215  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK: Mask for clock speed field
 216  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:    Speed of reference clock or
 217  *                                            crystal
 218  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:    Speed of reference clock
 219  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:    Speed of reference clock
 220  * @CVMX_USB_INITIALIZE_FLAGS_NO_DMA:         Disable DMA and used polled IO for
 221  *                                            data transfer use for the USB
 222  */
 223 enum cvmx_usb_initialize_flags {
 224         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI           = 1 << 0,
 225         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND          = 1 << 1,
 226         CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK        = 3 << 3,
 227         CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ           = 1 << 3,
 228         CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ           = 2 << 3,
 229         CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ           = 3 << 3,
 230         /* Bits 3-4 used to encode the clock frequency */
 231         CVMX_USB_INITIALIZE_FLAGS_NO_DMA                = 1 << 5,
 232 };
 233
 234 /**
 235  * enum cvmx_usb_pipe_flags - internal flags for a pipe.
 236  *
 237  * @__CVMX_USB_PIPE_FLAGS_SCHEDULED: Used internally to determine if a pipe is
 238  *                                   actively using hardware. Do not use.
 239  * @__CVMX_USB_PIPE_FLAGS_NEED_PING: Used internally to determine if a high
 240  *                                   speed pipe is in the ping state. Do not
 241  *                                   use.
 242  */
 243 enum cvmx_usb_pipe_flags {
 244         __CVMX_USB_PIPE_FLAGS_SCHEDULED = 1 << 17,
 245         __CVMX_USB_PIPE_FLAGS_NEED_PING = 1 << 18,
 246 };
 247
 248 /* Normal prefetch that use the pref instruction. */
 249 #define CVMX_PREFETCH(address, offset) asm volatile ("pref %[type], %[off](%[rbase])" : : [rbase] "d" (address), [off] "I" (offset), [type] "n" (0))
 250
 251 /* Maximum number of times to retry failed transactions */
 252 #define MAX_RETRIES             3
 253
 254 /* Maximum number of hardware channels supported by the USB block */
 255 #define MAX_CHANNELS            8
 256
 257 /* The highest valid USB device address */
 258 #define MAX_USB_ADDRESS         127
 259
 260 /* The highest valid USB endpoint number */
 261 #define MAX_USB_ENDPOINT        15
 262
 263 /* The highest valid port number on a hub */
 264 #define MAX_USB_HUB_PORT        15
 265
 266 /*
 267  * The low level hardware can transfer a maximum of this number of bytes in each
 268  * transfer. The field is 19 bits wide
 269  */
 270 #define MAX_TRANSFER_BYTES      ((1<<19)-1)
 271
 272 /*
 273  * The low level hardware can transfer a maximum of this number of packets in
 274  * each transfer. The field is 10 bits wide
 275  */
 276 #define MAX_TRANSFER_PACKETS    ((1<<10)-1)
 277
 278 enum {
 279         USB_CLOCK_TYPE_REF_12,
 280         USB_CLOCK_TYPE_REF_24,
 281         USB_CLOCK_TYPE_REF_48,
 282         USB_CLOCK_TYPE_CRYSTAL_12,
 283 };
 284
 285 /**
 286  * Logical transactions may take numerous low level
 287  * transactions, especially when splits are concerned. This
 288  * enum represents all of the possible stages a transaction can
 289  * be in. Note that split completes are always even. This is so
 290  * the NAK handler can backup to the previous low level
 291  * transaction with a simple clearing of bit 0.
 292  */
 293 enum cvmx_usb_stage {
 294         CVMX_USB_STAGE_NON_CONTROL,
 295         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE,
 296         CVMX_USB_STAGE_SETUP,
 297         CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE,
 298         CVMX_USB_STAGE_DATA,
 299         CVMX_USB_STAGE_DATA_SPLIT_COMPLETE,
 300         CVMX_USB_STAGE_STATUS,
 301         CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE,
 302 };
 303
 304 /**
 305  * struct cvmx_usb_transaction - describes each pending USB transaction
 306  *                               regardless of type. These are linked together
 307  *                               to form a list of pending requests for a pipe.
 308  *
 309  * @prev:               Transaction before this one in the pipe.
 310  * @next:               Transaction after this one in the pipe.
 311  * @type:               Type of transaction, duplicated of the pipe.
 312  * @flags:              State flags for this transaction.
 313  * @buffer:             User's physical buffer address to read/write.
 314  * @buffer_length:      Size of the user's buffer in bytes.
 315  * @control_header:     For control transactions, physical address of the 8
 316  *                      byte standard header.
 317  * @iso_start_frame:    For ISO transactions, the starting frame number.
 318  * @iso_number_packets: For ISO transactions, the number of packets in the
 319  *                      request.
 320  * @iso_packets:        For ISO transactions, the sub packets in the request.
 321  * @actual_bytes:       Actual bytes transfer for this transaction.
 322  * @stage:              For control transactions, the current stage.
 323  * @urb:                URB.
 324  */
 325 struct cvmx_usb_transaction {
 326         struct cvmx_usb_transaction *prev;
 327         struct cvmx_usb_transaction *next;
 328         enum cvmx_usb_transfer type;
 329         uint64_t buffer;
 330         int buffer_length;
 331         uint64_t control_header;
 332         int iso_start_frame;
 333         int iso_number_packets;
 334         struct cvmx_usb_iso_packet *iso_packets;
 335         int xfersize;
 336         int pktcnt;
 337         int retries;
 338         int actual_bytes;
 339         enum cvmx_usb_stage stage;
 340         struct urb *urb;
 341 };
 342
 343 /**
 344  * struct cvmx_usb_pipe - a pipe represents a virtual connection between Octeon
 345  *                        and some USB device. It contains a list of pending
 346  *                        request to the device.
 347  *
 348  * @node:               List node for pipe list
 349  * @next:               Pipe after this one in the list
 350  * @head:               The first pending transaction
 351  * @tail:               The last pending transaction
 352  * @interval:           For periodic pipes, the interval between packets in
 353  *                      frames
 354  * @next_tx_frame:      The next frame this pipe is allowed to transmit on
 355  * @flags:              State flags for this pipe
 356  * @device_speed:       Speed of device connected to this pipe
 357  * @transfer_type:      Type of transaction supported by this pipe
 358  * @transfer_dir:       IN or OUT. Ignored for Control
 359  * @multi_count:        Max packet in a row for the device
 360  * @max_packet:         The device's maximum packet size in bytes
 361  * @device_addr:        USB device address at other end of pipe
 362  * @endpoint_num:       USB endpoint number at other end of pipe
 363  * @hub_device_addr:    Hub address this device is connected to
 364  * @hub_port:           Hub port this device is connected to
 365  * @pid_toggle:         This toggles between 0/1 on every packet send to track
 366  *                      the data pid needed
 367  * @channel:            Hardware DMA channel for this pipe
 368  * @split_sc_frame:     The low order bits of the frame number the split
 369  *                      complete should be sent on
 370  */
 371 struct cvmx_usb_pipe {
 372         struct list_head node;
 373         struct cvmx_usb_transaction *head;
 374         struct cvmx_usb_transaction *tail;
 375         uint64_t interval;
 376         uint64_t next_tx_frame;
 377         enum cvmx_usb_pipe_flags flags;
 378         enum cvmx_usb_speed device_speed;
 379         enum cvmx_usb_transfer transfer_type;
 380         enum cvmx_usb_direction transfer_dir;
 381         int multi_count;
 382         uint16_t max_packet;
 383         uint8_t device_addr;
 384         uint8_t endpoint_num;
 385         uint8_t hub_device_addr;
 386         uint8_t hub_port;
 387         uint8_t pid_toggle;
 388         uint8_t channel;
 389         int8_t split_sc_frame;
 390 };
 391
 392 struct cvmx_usb_tx_fifo {
 393         struct {
 394                 int channel;
 395                 int size;
 396                 uint64_t address;
 397         } entry[MAX_CHANNELS+1];
 398         int head;
 399         int tail;
 400 };
 401
 402 /**
 403  * struct cvmx_usb_state - the state of the USB block
 404  *
 405  * init_flags:             Flags passed to initialize.
 406  * index:                  Which USB block this is for.
 407  * idle_hardware_channels: Bit set for every idle hardware channel.
 408  * usbcx_hprt:             Stored port status so we don't need to read a CSR to
 409  *                         determine splits.
 410  * pipe_for_channel:       Map channels to pipes.
 411  * pipe:                   Storage for pipes.
 412  * indent:                 Used by debug output to indent functions.
 413  * port_status:            Last port status used for change notification.
 414  * idle_pipes:             List of open pipes that have no transactions.
 415  * active_pipes:           Active pipes indexed by transfer type.
 416  * frame_number:           Increments every SOF interrupt for time keeping.
 417  * active_split:           Points to the current active split, or NULL.
 418  */
 419 struct cvmx_usb_state {
 420         int init_flags;
 421         int index;
 422         int idle_hardware_channels;
 423         union cvmx_usbcx_hprt usbcx_hprt;
 424         struct cvmx_usb_pipe *pipe_for_channel[MAX_CHANNELS];
 425         int indent;
 426         struct cvmx_usb_port_status port_status;
 427         struct list_head idle_pipes;
 428         struct list_head active_pipes[4];
 429         uint64_t frame_number;
 430         struct cvmx_usb_transaction *active_split;
 431         struct cvmx_usb_tx_fifo periodic;
 432         struct cvmx_usb_tx_fifo nonperiodic;
 433 };
 434
 435 struct octeon_hcd {
 436         spinlock_t lock;
 437         struct cvmx_usb_state usb;
 438         struct tasklet_struct dequeue_tasklet;
 439         struct list_head dequeue_list;
 440 };
 441
 442 /* This macro spins on a field waiting for it to reach a value */
 443 #define CVMX_WAIT_FOR_FIELD32(address, type, field, op, value, timeout_usec)\
 444         ({int result;                                                       \
 445         do {                                                                \
 446                 uint64_t done = cvmx_get_cycle() + (uint64_t)timeout_usec * \
 447                         octeon_get_clock_rate() / 1000000;                  \
 448                 type c;                                                     \
 449                 while (1) {                                                 \
 450                         c.u32 = __cvmx_usb_read_csr32(usb, address);        \
 451                         if (c.s.field op (value)) {                         \
 452                                 result = 0;                                 \
 453                                 break;                                      \
 454                         } else if (cvmx_get_cycle() > done) {               \
 455                                 result = -1;                                \
 456                                 break;                                      \
 457                         } else                                              \
 458                                 cvmx_wait(100);                             \
 459                 }                                                           \
 460         } while (0);                                                        \
 461         result; })
 462
 463 /*
 464  * This macro logically sets a single field in a CSR. It does the sequence
 465  * read, modify, and write
 466  */
 467 #define USB_SET_FIELD32(address, type, field, value)            \
 468         do {                                                    \
 469                 type c;                                         \
 470                 c.u32 = __cvmx_usb_read_csr32(usb, address);    \
 471                 c.s.field = value;                              \
 472                 __cvmx_usb_write_csr32(usb, address, c.u32);    \
 473         } while (0)
 474
 475 /* Returns the IO address to push/pop stuff data from the FIFOs */
 476 #define USB_FIFO_ADDRESS(channel, usb_index) (CVMX_USBCX_GOTGCTL(usb_index) + ((channel)+1)*0x1000)
 477
 478 static int octeon_usb_get_clock_type(void)
 479 {
 480         switch (cvmx_sysinfo_get()->board_type) {
 481         case CVMX_BOARD_TYPE_BBGW_REF:
 482         case CVMX_BOARD_TYPE_LANAI2_A:
 483         case CVMX_BOARD_TYPE_LANAI2_U:
 484         case CVMX_BOARD_TYPE_LANAI2_G:
 485         case CVMX_BOARD_TYPE_UBNT_E100:
 486                 return USB_CLOCK_TYPE_CRYSTAL_12;
 487         }
 488         return USB_CLOCK_TYPE_REF_48;
 489 }
 490
 491 /**
 492  * Read a USB 32bit CSR. It performs the necessary address swizzle
 493  * for 32bit CSRs and logs the value in a readable format if
 494  * debugging is on.
 495  *
 496  * @usb:     USB block this access is for
 497  * @address: 64bit address to read
 498  *
 499  * Returns: Result of the read
 500  */
 501 static inline uint32_t __cvmx_usb_read_csr32(struct cvmx_usb_state *usb,
 502                                              uint64_t address)
 503 {
 504         uint32_t result = cvmx_read64_uint32(address ^ 4);
 505         return result;
 506 }
 507
 508
 509 /**
 510  * Write a USB 32bit CSR. It performs the necessary address
 511  * swizzle for 32bit CSRs and logs the value in a readable format
 512  * if debugging is on.
 513  *
 514  * @usb:     USB block this access is for
 515  * @address: 64bit address to write
 516  * @value:   Value to write
 517  */
 518 static inline void __cvmx_usb_write_csr32(struct cvmx_usb_state *usb,
 519                                           uint64_t address, uint32_t value)
 520 {
 521         cvmx_write64_uint32(address ^ 4, value);
 522         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
 523 }
 524
 525
 526 /**
 527  * Read a USB 64bit CSR. It logs the value in a readable format if
 528  * debugging is on.
 529  *
 530  * @usb:     USB block this access is for
 531  * @address: 64bit address to read
 532  *
 533  * Returns: Result of the read
 534  */
 535 static inline uint64_t __cvmx_usb_read_csr64(struct cvmx_usb_state *usb,
 536                                              uint64_t address)
 537 {
 538         uint64_t result = cvmx_read64_uint64(address);
 539         return result;
 540 }
 541
 542
 543 /**
 544  * Write a USB 64bit CSR. It logs the value in a readable format
 545  * if debugging is on.
 546  *
 547  * @usb:     USB block this access is for
 548  * @address: 64bit address to write
 549  * @value:   Value to write
 550  */
 551 static inline void __cvmx_usb_write_csr64(struct cvmx_usb_state *usb,
 552                                           uint64_t address, uint64_t value)
 553 {
 554         cvmx_write64_uint64(address, value);
 555 }
 556
 557 /**
 558  * Return non zero if this pipe connects to a non HIGH speed
 559  * device through a high speed hub.
 560  *
 561  * @usb:    USB block this access is for
 562  * @pipe:   Pipe to check
 563  *
 564  * Returns: Non zero if we need to do split transactions
 565  */
 566 static inline int __cvmx_usb_pipe_needs_split(struct cvmx_usb_state *usb,
 567                                               struct cvmx_usb_pipe *pipe)
 568 {
 569         return pipe->device_speed != CVMX_USB_SPEED_HIGH &&
 570                usb->usbcx_hprt.s.prtspd == CVMX_USB_SPEED_HIGH;
 571 }
 572
 573
 574 /**
 575  * Trivial utility function to return the correct PID for a pipe
 576  *
 577  * @pipe:   pipe to check
 578  *
 579  * Returns: PID for pipe
 580  */
 581 static inline int __cvmx_usb_get_data_pid(struct cvmx_usb_pipe *pipe)
 582 {
 583         if (pipe->pid_toggle)
 584                 return 2; /* Data1 */
 585         else
 586                 return 0; /* Data0 */
 587 }
 588
 589
 590 /**
 591  * Return the number of USB ports supported by this Octeon
 592  * chip. If the chip doesn't support USB, or is not supported
 593  * by this API, a zero will be returned. Most Octeon chips
 594  * support one usb port, but some support two ports.
 595  * cvmx_usb_initialize() must be called on independent
 596  * struct cvmx_usb_state.
 597  *
 598  * Returns: Number of port, zero if usb isn't supported
 599  */
 600 static int cvmx_usb_get_num_ports(void)
 601 {
 602         int arch_ports = 0;
 603
 604         if (OCTEON_IS_MODEL(OCTEON_CN56XX))
 605                 arch_ports = 1;
 606         else if (OCTEON_IS_MODEL(OCTEON_CN52XX))
 607                 arch_ports = 2;
 608         else if (OCTEON_IS_MODEL(OCTEON_CN50XX))
 609                 arch_ports = 1;
 610         else if (OCTEON_IS_MODEL(OCTEON_CN31XX))
 611                 arch_ports = 1;
 612         else if (OCTEON_IS_MODEL(OCTEON_CN30XX))
 613                 arch_ports = 1;
 614         else
 615                 arch_ports = 0;
 616
 617         return arch_ports;
 618 }
 619
 620 /**
 621  * Initialize a USB port for use. This must be called before any
 622  * other access to the Octeon USB port is made. The port starts
 623  * off in the disabled state.
 624  *
 625  * @usb:         Pointer to an empty struct cvmx_usb_state
 626  *               that will be populated by the initialize call.
 627  *               This structure is then passed to all other USB
 628  *               functions.
 629  * @usb_port_number:
 630  *               Which Octeon USB port to initialize.
 631  *
 632  * Returns: 0 or a negative error code.
 633  */
 634 static int cvmx_usb_initialize(struct cvmx_usb_state *usb,
 635                                int usb_port_number)
 636 {
 637         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 638         union cvmx_usbnx_usbp_ctl_status usbn_usbp_ctl_status;
 639         enum cvmx_usb_initialize_flags flags = 0;
 640         int i;
 641
 642         /* At first allow 0-1 for the usb port number */
 643         if ((usb_port_number < 0) || (usb_port_number > 1))
 644                 return -EINVAL;
 645         /* For all chips except 52XX there is only one port */
 646         if (!OCTEON_IS_MODEL(OCTEON_CN52XX) && (usb_port_number > 0))
 647                 return -EINVAL;
 648         /* Try to determine clock type automatically */
 649         if (octeon_usb_get_clock_type() == USB_CLOCK_TYPE_CRYSTAL_12) {
 650                 /* Only 12 MHZ crystals are supported */
 651                 flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI;
 652         } else {
 653                 flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND;
 654
 655                 switch (octeon_usb_get_clock_type()) {
 656                 case USB_CLOCK_TYPE_REF_12:
 657                         flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ;
 658                         break;
 659                 case USB_CLOCK_TYPE_REF_24:
 660                         flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ;
 661                         break;
 662                 case USB_CLOCK_TYPE_REF_48:
 663                         flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ;
 664                         break;
 665                 default:
 666                         return -EINVAL;
 667                         break;
 668                 }
 669         }
 670
 671         memset(usb, 0, sizeof(*usb));
 672         usb->init_flags = flags;
 673
 674         /* Initialize the USB state structure */
 675         usb->index = usb_port_number;
 676         INIT_LIST_HEAD(&usb->idle_pipes);
 677         for (i = 0; i < ARRAY_SIZE(usb->active_pipes); i++)
 678                 INIT_LIST_HEAD(&usb->active_pipes[i]);
 679
 680         /*
 681          * Power On Reset and PHY Initialization
 682          *
 683          * 1. Wait for DCOK to assert (nothing to do)
 684          *
 685          * 2a. Write USBN0/1_CLK_CTL[POR] = 1 and
 686          *     USBN0/1_CLK_CTL[HRST,PRST,HCLK_RST] = 0
 687          */
 688         usbn_clk_ctl.u64 = __cvmx_usb_read_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index));
 689         usbn_clk_ctl.s.por = 1;
 690         usbn_clk_ctl.s.hrst = 0;
 691         usbn_clk_ctl.s.prst = 0;
 692         usbn_clk_ctl.s.hclk_rst = 0;
 693         usbn_clk_ctl.s.enable = 0;
 694         /*
 695          * 2b. Select the USB reference clock/crystal parameters by writing
 696          *     appropriate values to USBN0/1_CLK_CTL[P_C_SEL, P_RTYPE, P_COM_ON]
 697          */
 698         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND) {
 699                 /*
 700                  * The USB port uses 12/24/48MHz 2.5V board clock
 701                  * source at USB_XO. USB_XI should be tied to GND.
 702                  * Most Octeon evaluation boards require this setting
 703                  */
 704                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX) ||
 705                     OCTEON_IS_MODEL(OCTEON_CN56XX) ||
 706                     OCTEON_IS_MODEL(OCTEON_CN50XX))
 707                         /* From CN56XX,CN50XX,CN31XX,CN30XX manuals */
 708                         usbn_clk_ctl.s.p_rtype = 2; /* p_rclk=1 & p_xenbn=0 */
 709                 else
 710                         /* From CN52XX manual */
 711                         usbn_clk_ctl.s.p_rtype = 1;
 712
 713                 switch (flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK) {
 714                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:
 715                         usbn_clk_ctl.s.p_c_sel = 0;
 716                         break;
 717                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:
 718                         usbn_clk_ctl.s.p_c_sel = 1;
 719                         break;
 720                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:
 721                         usbn_clk_ctl.s.p_c_sel = 2;
 722                         break;
 723                 }
 724         } else {
 725                 /*
 726                  * The USB port uses a 12MHz crystal as clock source
 727                  * at USB_XO and USB_XI
 728                  */
 729                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
 730                         /* From CN31XX,CN30XX manual */
 731                         usbn_clk_ctl.s.p_rtype = 3; /* p_rclk=1 & p_xenbn=1 */
 732                 else
 733                         /* From CN56XX,CN52XX,CN50XX manuals. */
 734                         usbn_clk_ctl.s.p_rtype = 0;
 735
 736                 usbn_clk_ctl.s.p_c_sel = 0;
 737         }
 738         /*
 739          * 2c. Select the HCLK via writing USBN0/1_CLK_CTL[DIVIDE, DIVIDE2] and
 740          *     setting USBN0/1_CLK_CTL[ENABLE] = 1. Divide the core clock down
 741          *     such that USB is as close as possible to 125Mhz
 742          */
 743         {
 744                 int divisor = (octeon_get_clock_rate()+125000000-1)/125000000;
 745                 /* Lower than 4 doesn't seem to work properly */
 746                 if (divisor < 4)
 747                         divisor = 4;
 748                 usbn_clk_ctl.s.divide = divisor;
 749                 usbn_clk_ctl.s.divide2 = 0;
 750         }
 751         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 752                                usbn_clk_ctl.u64);
 753         /* 2d. Write USBN0/1_CLK_CTL[HCLK_RST] = 1 */
 754         usbn_clk_ctl.s.hclk_rst = 1;
 755         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 756                                usbn_clk_ctl.u64);
 757         /* 2e.  Wait 64 core-clock cycles for HCLK to stabilize */
 758         cvmx_wait(64);
 759         /*
 760          * 3. Program the power-on reset field in the USBN clock-control
 761          *    register:
 762          *    USBN_CLK_CTL[POR] = 0
 763          */
 764         usbn_clk_ctl.s.por = 0;
 765         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 766                                usbn_clk_ctl.u64);
 767         /* 4. Wait 1 ms for PHY clock to start */
 768         mdelay(1);
 769         /*
 770          * 5. Program the Reset input from automatic test equipment field in the
 771          *    USBP control and status register:
 772          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 1
 773          */
 774         usbn_usbp_ctl_status.u64 = __cvmx_usb_read_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index));
 775         usbn_usbp_ctl_status.s.ate_reset = 1;
 776         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 777                                usbn_usbp_ctl_status.u64);
 778         /* 6. Wait 10 cycles */
 779         cvmx_wait(10);
 780         /*
 781          * 7. Clear ATE_RESET field in the USBN clock-control register:
 782          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 0
 783          */
 784         usbn_usbp_ctl_status.s.ate_reset = 0;
 785         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 786                                usbn_usbp_ctl_status.u64);
 787         /*
 788          * 8. Program the PHY reset field in the USBN clock-control register:
 789          *    USBN_CLK_CTL[PRST] = 1
 790          */
 791         usbn_clk_ctl.s.prst = 1;
 792         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 793                                usbn_clk_ctl.u64);
 794         /*
 795          * 9. Program the USBP control and status register to select host or
 796          *    device mode. USBN_USBP_CTL_STATUS[HST_MODE] = 0 for host, = 1 for
 797          *    device
 798          */
 799         usbn_usbp_ctl_status.s.hst_mode = 0;
 800         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 801                                usbn_usbp_ctl_status.u64);
 802         /* 10. Wait 1 us */
 803         udelay(1);
 804         /*
 805          * 11. Program the hreset_n field in the USBN clock-control register:
 806          *     USBN_CLK_CTL[HRST] = 1
 807          */
 808         usbn_clk_ctl.s.hrst = 1;
 809         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 810                                usbn_clk_ctl.u64);
 811         /* 12. Proceed to USB core initialization */
 812         usbn_clk_ctl.s.enable = 1;
 813         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 814                                usbn_clk_ctl.u64);
 815         udelay(1);
 816
 817         /*
 818          * USB Core Initialization
 819          *
 820          * 1. Read USBC_GHWCFG1, USBC_GHWCFG2, USBC_GHWCFG3, USBC_GHWCFG4 to
 821          *    determine USB core configuration parameters.
 822          *
 823          *    Nothing needed
 824          *
 825          * 2. Program the following fields in the global AHB configuration
 826          *    register (USBC_GAHBCFG)
 827          *    DMA mode, USBC_GAHBCFG[DMAEn]: 1 = DMA mode, 0 = slave mode
 828          *    Burst length, USBC_GAHBCFG[HBSTLEN] = 0
 829          *    Nonperiodic TxFIFO empty level (slave mode only),
 830          *    USBC_GAHBCFG[NPTXFEMPLVL]
 831          *    Periodic TxFIFO empty level (slave mode only),
 832          *    USBC_GAHBCFG[PTXFEMPLVL]
 833          *    Global interrupt mask, USBC_GAHBCFG[GLBLINTRMSK] = 1
 834          */
 835         {
 836                 union cvmx_usbcx_gahbcfg usbcx_gahbcfg;
 837                 /* Due to an errata, CN31XX doesn't support DMA */
 838                 if (OCTEON_IS_MODEL(OCTEON_CN31XX))
 839                         usb->init_flags |= CVMX_USB_INITIALIZE_FLAGS_NO_DMA;
 840                 usbcx_gahbcfg.u32 = 0;
 841                 usbcx_gahbcfg.s.dmaen = !(usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA);
 842                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 843                         /* Only use one channel with non DMA */
 844                         usb->idle_hardware_channels = 0x1;
 845                 else if (OCTEON_IS_MODEL(OCTEON_CN5XXX))
 846                         /* CN5XXX have an errata with channel 3 */
 847                         usb->idle_hardware_channels = 0xf7;
 848                 else
 849                         usb->idle_hardware_channels = 0xff;
 850                 usbcx_gahbcfg.s.hbstlen = 0;
 851                 usbcx_gahbcfg.s.nptxfemplvl = 1;
 852                 usbcx_gahbcfg.s.ptxfemplvl = 1;
 853                 usbcx_gahbcfg.s.glblintrmsk = 1;
 854                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GAHBCFG(usb->index),
 855                                        usbcx_gahbcfg.u32);
 856         }
 857         /*
 858          * 3. Program the following fields in USBC_GUSBCFG register.
 859          *    HS/FS timeout calibration, USBC_GUSBCFG[TOUTCAL] = 0
 860          *    ULPI DDR select, USBC_GUSBCFG[DDRSEL] = 0
 861          *    USB turnaround time, USBC_GUSBCFG[USBTRDTIM] = 0x5
 862          *    PHY low-power clock select, USBC_GUSBCFG[PHYLPWRCLKSEL] = 0
 863          */
 864         {
 865                 union cvmx_usbcx_gusbcfg usbcx_gusbcfg;
 866                 usbcx_gusbcfg.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index));
 867                 usbcx_gusbcfg.s.toutcal = 0;
 868                 usbcx_gusbcfg.s.ddrsel = 0;
 869                 usbcx_gusbcfg.s.usbtrdtim = 0x5;
 870                 usbcx_gusbcfg.s.phylpwrclksel = 0;
 871                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index),
 872                                        usbcx_gusbcfg.u32);
 873         }
 874         /*
 875          * 4. The software must unmask the following bits in the USBC_GINTMSK
 876          *    register.
 877          *    OTG interrupt mask, USBC_GINTMSK[OTGINTMSK] = 1
 878          *    Mode mismatch interrupt mask, USBC_GINTMSK[MODEMISMSK] = 1
 879          */
 880         {
 881                 union cvmx_usbcx_gintmsk usbcx_gintmsk;
 882                 int channel;
 883
 884                 usbcx_gintmsk.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GINTMSK(usb->index));
 885                 usbcx_gintmsk.s.otgintmsk = 1;
 886                 usbcx_gintmsk.s.modemismsk = 1;
 887                 usbcx_gintmsk.s.hchintmsk = 1;
 888                 usbcx_gintmsk.s.sofmsk = 0;
 889                 /* We need RX FIFO interrupts if we don't have DMA */
 890                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 891                         usbcx_gintmsk.s.rxflvlmsk = 1;
 892                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTMSK(usb->index),
 893                                        usbcx_gintmsk.u32);
 894
 895                 /*
 896                  * Disable all channel interrupts. We'll enable them per channel
 897                  * later.
 898                  */
 899                 for (channel = 0; channel < 8; channel++)
 900                         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
 901         }
 902
 903         {
 904                 /*
 905                  * Host Port Initialization
 906                  *
 907                  * 1. Program the host-port interrupt-mask field to unmask,
 908                  *    USBC_GINTMSK[PRTINT] = 1
 909                  */
 910                 USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk,
 911                                 prtintmsk, 1);
 912                 USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk,
 913                                 disconnintmsk, 1);
 914                 /*
 915                  * 2. Program the USBC_HCFG register to select full-speed host
 916                  *    or high-speed host.
 917                  */
 918                 {
 919                         union cvmx_usbcx_hcfg usbcx_hcfg;
 920                         usbcx_hcfg.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCFG(usb->index));
 921                         usbcx_hcfg.s.fslssupp = 0;
 922                         usbcx_hcfg.s.fslspclksel = 0;
 923                         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCFG(usb->index), usbcx_hcfg.u32);
 924                 }
 925                 /*
 926                  * 3. Program the port power bit to drive VBUS on the USB,
 927                  *    USBC_HPRT[PRTPWR] = 1
 928                  */
 929                 USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt, prtpwr, 1);
 930
 931                 /*
 932                  * Steps 4-15 from the manual are done later in the port enable
 933                  */
 934         }
 935
 936         return 0;
 937 }
 938
 939
 940 /**
 941  * Shutdown a USB port after a call to cvmx_usb_initialize().
 942  * The port should be disabled with all pipes closed when this
 943  * function is called.
 944  *
 945  * @usb: USB device state populated by cvmx_usb_initialize().
 946  *
 947  * Returns: 0 or a negative error code.
 948  */
 949 static int cvmx_usb_shutdown(struct cvmx_usb_state *usb)
 950 {
 951         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 952
 953         /* Make sure all pipes are closed */
 954         if (!list_empty(&usb->idle_pipes) ||
 955             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_ISOCHRONOUS]) ||
 956             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_INTERRUPT]) ||
 957             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_CONTROL]) ||
 958             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_BULK]))
 959                 return -EBUSY;
 960
 961         /* Disable the clocks and put them in power on reset */
 962         usbn_clk_ctl.u64 = __cvmx_usb_read_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index));
 963         usbn_clk_ctl.s.enable = 1;
 964         usbn_clk_ctl.s.por = 1;
 965         usbn_clk_ctl.s.hclk_rst = 1;
 966         usbn_clk_ctl.s.prst = 0;
 967         usbn_clk_ctl.s.hrst = 0;
 968         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 969                                usbn_clk_ctl.u64);
 970         return 0;
 971 }
 972
 973
 974 /**
 975  * Enable a USB port. After this call succeeds, the USB port is
 976  * online and servicing requests.
 977  *
 978  * @usb: USB device state populated by cvmx_usb_initialize().
 979  *
 980  * Returns: 0 or a negative error code.
 981  */
 982 static int cvmx_usb_enable(struct cvmx_usb_state *usb)
 983 {
 984         union cvmx_usbcx_ghwcfg3 usbcx_ghwcfg3;
 985
 986         usb->usbcx_hprt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
 987
 988         /*
 989          * If the port is already enabled the just return. We don't need to do
 990          * anything
 991          */
 992         if (usb->usbcx_hprt.s.prtena)
 993                 return 0;
 994
 995         /* If there is nothing plugged into the port then fail immediately */
 996         if (!usb->usbcx_hprt.s.prtconnsts) {
 997                 return -ETIMEDOUT;
 998         }
 999
1000         /* Program the port reset bit to start the reset process */
1001         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt, prtrst, 1);
1002
1003         /*
1004          * Wait at least 50ms (high speed), or 10ms (full speed) for the reset
1005          * process to complete.
1006          */
1007         mdelay(50);
1008
1009         /* Program the port reset bit to 0, USBC_HPRT[PRTRST] = 0 */
1010         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt, prtrst, 0);
1011
1012         /* Wait for the USBC_HPRT[PRTENA]. */
1013         if (CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt,
1014                                   prtena, ==, 1, 100000))
1015                 return -ETIMEDOUT;
1016
1017         /*
1018          * Read the port speed field to get the enumerated speed,
1019          * USBC_HPRT[PRTSPD].
1020          */
1021         usb->usbcx_hprt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
1022         usbcx_ghwcfg3.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GHWCFG3(usb->index));
1023
1024         /*
1025          * 13. Program the USBC_GRXFSIZ register to select the size of the
1026          *     receive FIFO (25%).
1027          */
1028         USB_SET_FIELD32(CVMX_USBCX_GRXFSIZ(usb->index), union cvmx_usbcx_grxfsiz,
1029                         rxfdep, usbcx_ghwcfg3.s.dfifodepth / 4);
1030         /*
1031          * 14. Program the USBC_GNPTXFSIZ register to select the size and the
1032          *     start address of the non- periodic transmit FIFO for nonperiodic
1033          *     transactions (50%).
1034          */
1035         {
1036                 union cvmx_usbcx_gnptxfsiz siz;
1037                 siz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index));
1038                 siz.s.nptxfdep = usbcx_ghwcfg3.s.dfifodepth / 2;
1039                 siz.s.nptxfstaddr = usbcx_ghwcfg3.s.dfifodepth / 4;
1040                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index), siz.u32);
1041         }
1042         /*
1043          * 15. Program the USBC_HPTXFSIZ register to select the size and start
1044          *     address of the periodic transmit FIFO for periodic transactions
1045          *     (25%).
1046          */
1047         {
1048                 union cvmx_usbcx_hptxfsiz siz;
1049                 siz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index));
1050                 siz.s.ptxfsize = usbcx_ghwcfg3.s.dfifodepth / 4;
1051                 siz.s.ptxfstaddr = 3 * usbcx_ghwcfg3.s.dfifodepth / 4;
1052                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index), siz.u32);
1053         }
1054         /* Flush all FIFOs */
1055         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), union cvmx_usbcx_grstctl, txfnum, 0x10);
1056         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), union cvmx_usbcx_grstctl, txfflsh, 1);
1057         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), union cvmx_usbcx_grstctl,
1058                               txfflsh, ==, 0, 100);
1059         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), union cvmx_usbcx_grstctl, rxfflsh, 1);
1060         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), union cvmx_usbcx_grstctl,
1061                               rxfflsh, ==, 0, 100);
1062
1063         return 0;
1064 }
1065
1066
1067 /**
1068  * Disable a USB port. After this call the USB port will not
1069  * generate data transfers and will not generate events.
1070  * Transactions in process will fail and call their
1071  * associated callbacks.
1072  *
1073  * @usb: USB device state populated by cvmx_usb_initialize().
1074  *
1075  * Returns: 0 or a negative error code.
1076  */
1077 static int cvmx_usb_disable(struct cvmx_usb_state *usb)
1078 {
1079         /* Disable the port */
1080         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt, prtena, 1);
1081         return 0;
1082 }
1083
1084
1085 /**
1086  * Get the current state of the USB port. Use this call to
1087  * determine if the usb port has anything connected, is enabled,
1088  * or has some sort of error condition. The return value of this
1089  * call has "changed" bits to signal of the value of some fields
1090  * have changed between calls.
1091  *
1092  * @usb: USB device state populated by cvmx_usb_initialize().
1093  *
1094  * Returns: Port status information
1095  */
1096 static struct cvmx_usb_port_status cvmx_usb_get_status(struct cvmx_usb_state *usb)
1097 {
1098         union cvmx_usbcx_hprt usbc_hprt;
1099         struct cvmx_usb_port_status result;
1100
1101         memset(&result, 0, sizeof(result));
1102
1103         usbc_hprt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
1104         result.port_enabled = usbc_hprt.s.prtena;
1105         result.port_over_current = usbc_hprt.s.prtovrcurract;
1106         result.port_powered = usbc_hprt.s.prtpwr;
1107         result.port_speed = usbc_hprt.s.prtspd;
1108         result.connected = usbc_hprt.s.prtconnsts;
1109         result.connect_change = (result.connected != usb->port_status.connected);
1110
1111         return result;
1112 }
1113
1114 /**
1115  * Open a virtual pipe between the host and a USB device. A pipe
1116  * must be opened before data can be transferred between a device
1117  * and Octeon.
1118  *
1119  * @usb:             USB device state populated by cvmx_usb_initialize().
1120  * @device_addr:
1121  *                   USB device address to open the pipe to
1122  *                   (0-127).
1123  * @endpoint_num:
1124  *                   USB endpoint number to open the pipe to
1125  *                   (0-15).
1126  * @device_speed:
1127  *                   The speed of the device the pipe is going
1128  *                   to. This must match the device's speed,
1129  *                   which may be different than the port speed.
1130  * @max_packet:      The maximum packet length the device can
1131  *                   transmit/receive (low speed=0-8, full
1132  *                   speed=0-1023, high speed=0-1024). This value
1133  *                   comes from the standard endpoint descriptor
1134  *                   field wMaxPacketSize bits <10:0>.
1135  * @transfer_type:
1136  *                   The type of transfer this pipe is for.
1137  * @transfer_dir:
1138  *                   The direction the pipe is in. This is not
1139  *                   used for control pipes.
1140  * @interval:        For ISOCHRONOUS and INTERRUPT transfers,
1141  *                   this is how often the transfer is scheduled
1142  *                   for. All other transfers should specify
1143  *                   zero. The units are in frames (8000/sec at
1144  *                   high speed, 1000/sec for full speed).
1145  * @multi_count:
1146  *                   For high speed devices, this is the maximum
1147  *                   allowed number of packet per microframe.
1148  *                   Specify zero for non high speed devices. This
1149  *                   value comes from the standard endpoint descriptor
1150  *                   field wMaxPacketSize bits <12:11>.
1151  * @hub_device_addr:
1152  *                   Hub device address this device is connected
1153  *                   to. Devices connected directly to Octeon
1154  *                   use zero. This is only used when the device
1155  *                   is full/low speed behind a high speed hub.
1156  *                   The address will be of the high speed hub,
1157  *                   not and full speed hubs after it.
1158  * @hub_port:        Which port on the hub the device is
1159  *                   connected. Use zero for devices connected
1160  *                   directly to Octeon. Like hub_device_addr,
1161  *                   this is only used for full/low speed
1162  *                   devices behind a high speed hub.
1163  *
1164  * Returns: A non-NULL value is a pipe. NULL means an error.
1165  */
1166 static struct cvmx_usb_pipe *cvmx_usb_open_pipe(struct cvmx_usb_state *usb,
1167                                                 int device_addr, int
1168                                                 endpoint_num,
1169                                                 enum cvmx_usb_speed
1170                                                         device_speed,
1171                                                 int max_packet,
1172                                                 enum cvmx_usb_transfer
1173                                                         transfer_type,
1174                                                 enum cvmx_usb_direction
1175                                                         transfer_dir,
1176                                                 int interval, int multi_count,
1177                                                 int hub_device_addr,
1178                                                 int hub_port)
1179 {
1180         struct cvmx_usb_pipe *pipe;
1181
1182         if (unlikely((device_addr < 0) || (device_addr > MAX_USB_ADDRESS)))
1183                 return NULL;
1184         if (unlikely((endpoint_num < 0) || (endpoint_num > MAX_USB_ENDPOINT)))
1185                 return NULL;
1186         if (unlikely(device_speed > CVMX_USB_SPEED_LOW))
1187                 return NULL;
1188         if (unlikely((max_packet <= 0) || (max_packet > 1024)))
1189                 return NULL;
1190         if (unlikely(transfer_type > CVMX_USB_TRANSFER_INTERRUPT))
1191                 return NULL;
1192         if (unlikely((transfer_dir != CVMX_USB_DIRECTION_OUT) &&
1193                 (transfer_dir != CVMX_USB_DIRECTION_IN)))
1194                 return NULL;
1195         if (unlikely(interval < 0))
1196                 return NULL;
1197         if (unlikely((transfer_type == CVMX_USB_TRANSFER_CONTROL) && interval))
1198                 return NULL;
1199         if (unlikely(multi_count < 0))
1200                 return NULL;
1201         if (unlikely((device_speed != CVMX_USB_SPEED_HIGH) &&
1202                 (multi_count != 0)))
1203                 return NULL;
1204         if (unlikely((hub_device_addr < 0) || (hub_device_addr > MAX_USB_ADDRESS)))
1205                 return NULL;
1206         if (unlikely((hub_port < 0) || (hub_port > MAX_USB_HUB_PORT)))
1207                 return NULL;
1208
1209         pipe = kzalloc(sizeof(*pipe), GFP_ATOMIC);
1210         if (!pipe)
1211                 return NULL;
1212         if ((device_speed == CVMX_USB_SPEED_HIGH) &&
1213                 (transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1214                 (transfer_type == CVMX_USB_TRANSFER_BULK))
1215                 pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
1216         pipe->device_addr = device_addr;
1217         pipe->endpoint_num = endpoint_num;
1218         pipe->device_speed = device_speed;
1219         pipe->max_packet = max_packet;
1220         pipe->transfer_type = transfer_type;
1221         pipe->transfer_dir = transfer_dir;
1222         /*
1223          * All pipes use interval to rate limit NAK processing. Force an
1224          * interval if one wasn't supplied
1225          */
1226         if (!interval)
1227                 interval = 1;
1228         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1229                 pipe->interval = interval*8;
1230                 /* Force start splits to be schedule on uFrame 0 */
1231                 pipe->next_tx_frame = ((usb->frame_number+7)&~7) + pipe->interval;
1232         } else {
1233                 pipe->interval = interval;
1234                 pipe->next_tx_frame = usb->frame_number + pipe->interval;
1235         }
1236         pipe->multi_count = multi_count;
1237         pipe->hub_device_addr = hub_device_addr;
1238         pipe->hub_port = hub_port;
1239         pipe->pid_toggle = 0;
1240         pipe->split_sc_frame = -1;
1241         list_add_tail(&pipe->node, &usb->idle_pipes);
1242
1243         /*
1244          * We don't need to tell the hardware about this pipe yet since
1245          * it doesn't have any submitted requests
1246          */
1247
1248         return pipe;
1249 }
1250
1251
1252 /**
1253  * Poll the RX FIFOs and remove data as needed. This function is only used
1254  * in non DMA mode. It is very important that this function be called quickly
1255  * enough to prevent FIFO overflow.
1256  *
1257  * @usb:        USB device state populated by cvmx_usb_initialize().
1258  */
1259 static void __cvmx_usb_poll_rx_fifo(struct cvmx_usb_state *usb)
1260 {
1261         union cvmx_usbcx_grxstsph rx_status;
1262         int channel;
1263         int bytes;
1264         uint64_t address;
1265         uint32_t *ptr;
1266
1267         rx_status.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GRXSTSPH(usb->index));
1268         /* Only read data if IN data is there */
1269         if (rx_status.s.pktsts != 2)
1270                 return;
1271         /* Check if no data is available */
1272         if (!rx_status.s.bcnt)
1273                 return;
1274
1275         channel = rx_status.s.chnum;
1276         bytes = rx_status.s.bcnt;
1277         if (!bytes)
1278                 return;
1279
1280         /* Get where the DMA engine would have written this data */
1281         address = __cvmx_usb_read_csr64(usb, CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8);
1282         ptr = cvmx_phys_to_ptr(address);
1283         __cvmx_usb_write_csr64(usb, CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8, address + bytes);
1284
1285         /* Loop writing the FIFO data for this packet into memory */
1286         while (bytes > 0) {
1287                 *ptr++ = __cvmx_usb_read_csr32(usb, USB_FIFO_ADDRESS(channel, usb->index));
1288                 bytes -= 4;
1289         }
1290         CVMX_SYNCW;
1291
1292         return;
1293 }
1294
1295
1296 /**
1297  * Fill the TX hardware fifo with data out of the software
1298  * fifos
1299  *
1300  * @usb:            USB device state populated by cvmx_usb_initialize().
1301  * @fifo:           Software fifo to use
1302  * @available:      Amount of space in the hardware fifo
1303  *
1304  * Returns: Non zero if the hardware fifo was too small and needs
1305  *          to be serviced again.
1306  */
1307 static int __cvmx_usb_fill_tx_hw(struct cvmx_usb_state *usb,
1308                                  struct cvmx_usb_tx_fifo *fifo, int available)
1309 {
1310         /*
1311          * We're done either when there isn't anymore space or the software FIFO
1312          * is empty
1313          */
1314         while (available && (fifo->head != fifo->tail)) {
1315                 int i = fifo->tail;
1316                 const uint32_t *ptr = cvmx_phys_to_ptr(fifo->entry[i].address);
1317                 uint64_t csr_address = USB_FIFO_ADDRESS(fifo->entry[i].channel, usb->index) ^ 4;
1318                 int words = available;
1319
1320                 /* Limit the amount of data to waht the SW fifo has */
1321                 if (fifo->entry[i].size <= available) {
1322                         words = fifo->entry[i].size;
1323                         fifo->tail++;
1324                         if (fifo->tail > MAX_CHANNELS)
1325                                 fifo->tail = 0;
1326                 }
1327
1328                 /* Update the next locations and counts */
1329                 available -= words;
1330                 fifo->entry[i].address += words * 4;
1331                 fifo->entry[i].size -= words;
1332
1333                 /*
1334                  * Write the HW fifo data. The read every three writes is due
1335                  * to an errata on CN3XXX chips
1336                  */
1337                 while (words > 3) {
1338                         cvmx_write64_uint32(csr_address, *ptr++);
1339                         cvmx_write64_uint32(csr_address, *ptr++);
1340                         cvmx_write64_uint32(csr_address, *ptr++);
1341                         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1342                         words -= 3;
1343                 }
1344                 cvmx_write64_uint32(csr_address, *ptr++);
1345                 if (--words) {
1346                         cvmx_write64_uint32(csr_address, *ptr++);
1347                         if (--words)
1348                                 cvmx_write64_uint32(csr_address, *ptr++);
1349                 }
1350                 cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1351         }
1352         return fifo->head != fifo->tail;
1353 }
1354
1355
1356 /**
1357  * Check the hardware FIFOs and fill them as needed
1358  *
1359  * @usb:        USB device state populated by cvmx_usb_initialize().
1360  */
1361 static void __cvmx_usb_poll_tx_fifo(struct cvmx_usb_state *usb)
1362 {
1363         if (usb->periodic.head != usb->periodic.tail) {
1364                 union cvmx_usbcx_hptxsts tx_status;
1365                 tx_status.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPTXSTS(usb->index));
1366                 if (__cvmx_usb_fill_tx_hw(usb, &usb->periodic, tx_status.s.ptxfspcavail))
1367                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk, ptxfempmsk, 1);
1368                 else
1369                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk, ptxfempmsk, 0);
1370         }
1371
1372         if (usb->nonperiodic.head != usb->nonperiodic.tail) {
1373                 union cvmx_usbcx_gnptxsts tx_status;
1374                 tx_status.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GNPTXSTS(usb->index));
1375                 if (__cvmx_usb_fill_tx_hw(usb, &usb->nonperiodic, tx_status.s.nptxfspcavail))
1376                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk, nptxfempmsk, 1);
1377                 else
1378                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk, nptxfempmsk, 0);
1379         }
1380
1381         return;
1382 }
1383
1384
1385 /**
1386  * Fill the TX FIFO with an outgoing packet
1387  *
1388  * @usb:          USB device state populated by cvmx_usb_initialize().
1389  * @channel:      Channel number to get packet from
1390  */
1391 static void __cvmx_usb_fill_tx_fifo(struct cvmx_usb_state *usb, int channel)
1392 {
1393         union cvmx_usbcx_hccharx hcchar;
1394         union cvmx_usbcx_hcspltx usbc_hcsplt;
1395         union cvmx_usbcx_hctsizx usbc_hctsiz;
1396         struct cvmx_usb_tx_fifo *fifo;
1397
1398         /* We only need to fill data on outbound channels */
1399         hcchar.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index));
1400         if (hcchar.s.epdir != CVMX_USB_DIRECTION_OUT)
1401                 return;
1402
1403         /* OUT Splits only have data on the start and not the complete */
1404         usbc_hcsplt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCSPLTX(channel, usb->index));
1405         if (usbc_hcsplt.s.spltena && usbc_hcsplt.s.compsplt)
1406                 return;
1407
1408         /*
1409          * Find out how many bytes we need to fill and convert it into 32bit
1410          * words.
1411          */
1412         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index));
1413         if (!usbc_hctsiz.s.xfersize)
1414                 return;
1415
1416         if ((hcchar.s.eptype == CVMX_USB_TRANSFER_INTERRUPT) ||
1417                 (hcchar.s.eptype == CVMX_USB_TRANSFER_ISOCHRONOUS))
1418                 fifo = &usb->periodic;
1419         else
1420                 fifo = &usb->nonperiodic;
1421
1422         fifo->entry[fifo->head].channel = channel;
1423         fifo->entry[fifo->head].address = __cvmx_usb_read_csr64(usb, CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8);
1424         fifo->entry[fifo->head].size = (usbc_hctsiz.s.xfersize+3)>>2;
1425         fifo->head++;
1426         if (fifo->head > MAX_CHANNELS)
1427                 fifo->head = 0;
1428
1429         __cvmx_usb_poll_tx_fifo(usb);
1430
1431         return;
1432 }
1433
1434 /**
1435  * Perform channel specific setup for Control transactions. All
1436  * the generic stuff will already have been done in
1437  * __cvmx_usb_start_channel()
1438  *
1439  * @usb:          USB device state populated by cvmx_usb_initialize().
1440  * @channel:      Channel to setup
1441  * @pipe:         Pipe for control transaction
1442  */
1443 static void __cvmx_usb_start_channel_control(struct cvmx_usb_state *usb,
1444                                              int channel,
1445                                              struct cvmx_usb_pipe *pipe)
1446 {
1447         struct cvmx_usb_transaction *transaction = pipe->head;
1448         union cvmx_usb_control_header *header =
1449                 cvmx_phys_to_ptr(transaction->control_header);
1450         int bytes_to_transfer = transaction->buffer_length - transaction->actual_bytes;
1451         int packets_to_transfer;
1452         union cvmx_usbcx_hctsizx usbc_hctsiz;
1453
1454         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index));
1455
1456         switch (transaction->stage) {
1457         case CVMX_USB_STAGE_NON_CONTROL:
1458         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
1459                 cvmx_dprintf("%s: ERROR - Non control stage\n", __FUNCTION__);
1460                 break;
1461         case CVMX_USB_STAGE_SETUP:
1462                 usbc_hctsiz.s.pid = 3; /* Setup */
1463                 bytes_to_transfer = sizeof(*header);
1464                 /* All Control operations start with a setup going OUT */
1465                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), union cvmx_usbcx_hccharx, epdir, CVMX_USB_DIRECTION_OUT);
1466                 /*
1467                  * Setup send the control header instead of the buffer data. The
1468                  * buffer data will be used in the next stage
1469                  */
1470                 __cvmx_usb_write_csr64(usb, CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8, transaction->control_header);
1471                 break;
1472         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
1473                 usbc_hctsiz.s.pid = 3; /* Setup */
1474                 bytes_to_transfer = 0;
1475                 /* All Control operations start with a setup going OUT */
1476                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), union cvmx_usbcx_hccharx, epdir, CVMX_USB_DIRECTION_OUT);
1477                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index), union cvmx_usbcx_hcspltx, compsplt, 1);
1478                 break;
1479         case CVMX_USB_STAGE_DATA:
1480                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1481                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1482                         if (header->s.request_type & 0x80)
1483                                 bytes_to_transfer = 0;
1484                         else if (bytes_to_transfer > pipe->max_packet)
1485                                 bytes_to_transfer = pipe->max_packet;
1486                 }
1487                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1488                                 union cvmx_usbcx_hccharx, epdir,
1489                                 ((header->s.request_type & 0x80) ?
1490                                         CVMX_USB_DIRECTION_IN :
1491                                         CVMX_USB_DIRECTION_OUT));
1492                 break;
1493         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
1494                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1495                 if (!(header->s.request_type & 0x80))
1496                         bytes_to_transfer = 0;
1497                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1498                                 union cvmx_usbcx_hccharx, epdir,
1499                                 ((header->s.request_type & 0x80) ?
1500                                         CVMX_USB_DIRECTION_IN :
1501                                         CVMX_USB_DIRECTION_OUT));
1502                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index), union cvmx_usbcx_hcspltx, compsplt, 1);
1503                 break;
1504         case CVMX_USB_STAGE_STATUS:
1505                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1506                 bytes_to_transfer = 0;
1507                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), union cvmx_usbcx_hccharx, epdir,
1508                                 ((header->s.request_type & 0x80) ?
1509                                         CVMX_USB_DIRECTION_OUT :
1510                                         CVMX_USB_DIRECTION_IN));
1511                 break;
1512         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
1513                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1514                 bytes_to_transfer = 0;
1515                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), union cvmx_usbcx_hccharx, epdir,
1516                                 ((header->s.request_type & 0x80) ?
1517                                         CVMX_USB_DIRECTION_OUT :
1518                                         CVMX_USB_DIRECTION_IN));
1519                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index), union cvmx_usbcx_hcspltx, compsplt, 1);
1520                 break;
1521         }
1522
1523         /*
1524          * Make sure the transfer never exceeds the byte limit of the hardware.
1525          * Further bytes will be sent as continued transactions
1526          */
1527         if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1528                 /* Round MAX_TRANSFER_BYTES to a multiple of out packet size */
1529                 bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
1530                 bytes_to_transfer *= pipe->max_packet;
1531         }
1532
1533         /*
1534          * Calculate the number of packets to transfer. If the length is zero
1535          * we still need to transfer one packet
1536          */
1537         packets_to_transfer = (bytes_to_transfer + pipe->max_packet - 1) / pipe->max_packet;
1538         if (packets_to_transfer == 0)
1539                 packets_to_transfer = 1;
1540         else if ((packets_to_transfer > 1) && (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1541                 /*
1542                  * Limit to one packet when not using DMA. Channels must be
1543                  * restarted between every packet for IN transactions, so there
1544                  * is no reason to do multiple packets in a row
1545                  */
1546                 packets_to_transfer = 1;
1547                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1548         } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1549                 /*
1550                  * Limit the number of packet and data transferred to what the
1551                  * hardware can handle
1552                  */
1553                 packets_to_transfer = MAX_TRANSFER_PACKETS;
1554                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1555         }
1556
1557         usbc_hctsiz.s.xfersize = bytes_to_transfer;
1558         usbc_hctsiz.s.pktcnt = packets_to_transfer;
1559
1560         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index), usbc_hctsiz.u32);
1561         return;
1562 }
1563
1564
1565 /**
1566  * Start a channel to perform the pipe's head transaction
1567  *
1568  * @usb:          USB device state populated by cvmx_usb_initialize().
1569  * @channel:      Channel to setup
1570  * @pipe:         Pipe to start
1571  */
1572 static void __cvmx_usb_start_channel(struct cvmx_usb_state *usb,
1573                                      int channel,
1574                                      struct cvmx_usb_pipe *pipe)
1575 {
1576         struct cvmx_usb_transaction *transaction = pipe->head;
1577
1578         /* Make sure all writes to the DMA region get flushed */
1579         CVMX_SYNCW;
1580
1581         /* Attach the channel to the pipe */
1582         usb->pipe_for_channel[channel] = pipe;
1583         pipe->channel = channel;
1584         pipe->flags |= __CVMX_USB_PIPE_FLAGS_SCHEDULED;
1585
1586         /* Mark this channel as in use */
1587         usb->idle_hardware_channels &= ~(1<<channel);
1588
1589         /* Enable the channel interrupt bits */
1590         {
1591                 union cvmx_usbcx_hcintx usbc_hcint;
1592                 union cvmx_usbcx_hcintmskx usbc_hcintmsk;
1593                 union cvmx_usbcx_haintmsk usbc_haintmsk;
1594
1595                 /* Clear all channel status bits */
1596                 usbc_hcint.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCINTX(channel, usb->index));
1597                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTX(channel, usb->index), usbc_hcint.u32);
1598
1599                 usbc_hcintmsk.u32 = 0;
1600                 usbc_hcintmsk.s.chhltdmsk = 1;
1601                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1602                         /*
1603                          * Channels need these extra interrupts when we aren't
1604                          * in DMA mode.
1605                          */
1606                         usbc_hcintmsk.s.datatglerrmsk = 1;
1607                         usbc_hcintmsk.s.frmovrunmsk = 1;
1608                         usbc_hcintmsk.s.bblerrmsk = 1;
1609                         usbc_hcintmsk.s.xacterrmsk = 1;
1610                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1611                                 /*
1612                                  * Splits don't generate xfercompl, so we need
1613                                  * ACK and NYET.
1614                                  */
1615                                 usbc_hcintmsk.s.nyetmsk = 1;
1616                                 usbc_hcintmsk.s.ackmsk = 1;
1617                         }
1618                         usbc_hcintmsk.s.nakmsk = 1;
1619                         usbc_hcintmsk.s.stallmsk = 1;
1620                         usbc_hcintmsk.s.xfercomplmsk = 1;
1621                 }
1622                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), usbc_hcintmsk.u32);
1623
1624                 /* Enable the channel interrupt to propagate */
1625                 usbc_haintmsk.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HAINTMSK(usb->index));
1626                 usbc_haintmsk.s.haintmsk |= 1<<channel;
1627                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HAINTMSK(usb->index), usbc_haintmsk.u32);
1628         }
1629
1630         /* Setup the locations the DMA engines use  */
1631         {
1632                 uint64_t dma_address = transaction->buffer + transaction->actual_bytes;
1633                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1634                         dma_address = transaction->buffer + transaction->iso_packets[0].offset + transaction->actual_bytes;
1635                 __cvmx_usb_write_csr64(usb, CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8, dma_address);
1636                 __cvmx_usb_write_csr64(usb, CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8, dma_address);
1637         }
1638
1639         /* Setup both the size of the transfer and the SPLIT characteristics */
1640         {
1641                 union cvmx_usbcx_hcspltx usbc_hcsplt = {.u32 = 0};
1642                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 = 0};
1643                 int packets_to_transfer;
1644                 int bytes_to_transfer = transaction->buffer_length - transaction->actual_bytes;
1645
1646                 /*
1647                  * ISOCHRONOUS transactions store each individual transfer size
1648                  * in the packet structure, not the global buffer_length
1649                  */
1650                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1651                         bytes_to_transfer = transaction->iso_packets[0].length - transaction->actual_bytes;
1652
1653                 /*
1654                  * We need to do split transactions when we are talking to non
1655                  * high speed devices that are behind a high speed hub
1656                  */
1657                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1658                         /*
1659                          * On the start split phase (stage is even) record the
1660                          * frame number we will need to send the split complete.
1661                          * We only store the lower two bits since the time ahead
1662                          * can only be two frames
1663                          */
1664                         if ((transaction->stage&1) == 0) {
1665                                 if (transaction->type == CVMX_USB_TRANSFER_BULK)
1666                                         pipe->split_sc_frame = (usb->frame_number + 1) & 0x7f;
1667                                 else
1668                                         pipe->split_sc_frame = (usb->frame_number + 2) & 0x7f;
1669                         } else
1670                                 pipe->split_sc_frame = -1;
1671
1672                         usbc_hcsplt.s.spltena = 1;
1673                         usbc_hcsplt.s.hubaddr = pipe->hub_device_addr;
1674                         usbc_hcsplt.s.prtaddr = pipe->hub_port;
1675                         usbc_hcsplt.s.compsplt = (transaction->stage == CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE);
1676
1677                         /*
1678                          * SPLIT transactions can only ever transmit one data
1679                          * packet so limit the transfer size to the max packet
1680                          * size
1681                          */
1682                         if (bytes_to_transfer > pipe->max_packet)
1683                                 bytes_to_transfer = pipe->max_packet;
1684
1685                         /*
1686                          * ISOCHRONOUS OUT splits are unique in that they limit
1687                          * data transfers to 188 byte chunks representing the
1688                          * begin/middle/end of the data or all
1689                          */
1690                         if (!usbc_hcsplt.s.compsplt &&
1691                                 (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1692                                 (pipe->transfer_type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
1693                                 /*
1694                                  * Clear the split complete frame number as
1695                                  * there isn't going to be a split complete
1696                                  */
1697                                 pipe->split_sc_frame = -1;
1698                                 /*
1699                                  * See if we've started this transfer and sent
1700                                  * data
1701                                  */
1702                                 if (transaction->actual_bytes == 0) {
1703                                         /*
1704                                          * Nothing sent yet, this is either a
1705                                          * begin or the entire payload
1706                                          */
1707                                         if (bytes_to_transfer <= 188)
1708                                                 /* Entire payload in one go */
1709                                                 usbc_hcsplt.s.xactpos = 3;
1710                                         else
1711                                                 /* First part of payload */
1712                                                 usbc_hcsplt.s.xactpos = 2;
1713                                 } else {
1714                                         /*
1715                                          * Continuing the previous data, we must
1716                                          * either be in the middle or at the end
1717                                          */
1718                                         if (bytes_to_transfer <= 188)
1719                                                 /* End of payload */
1720                                                 usbc_hcsplt.s.xactpos = 1;
1721                                         else
1722                                                 /* Middle of payload */
1723                                                 usbc_hcsplt.s.xactpos = 0;
1724                                 }
1725                                 /*
1726                                  * Again, the transfer size is limited to 188
1727                                  * bytes
1728                                  */
1729                                 if (bytes_to_transfer > 188)
1730                                         bytes_to_transfer = 188;
1731                         }
1732                 }
1733
1734                 /*
1735                  * Make sure the transfer never exceeds the byte limit of the
1736                  * hardware. Further bytes will be sent as continued
1737                  * transactions
1738                  */
1739                 if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1740                         /*
1741                          * Round MAX_TRANSFER_BYTES to a multiple of out packet
1742                          * size
1743                          */
1744                         bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
1745                         bytes_to_transfer *= pipe->max_packet;
1746                 }
1747
1748                 /*
1749                  * Calculate the number of packets to transfer. If the length is
1750                  * zero we still need to transfer one packet
1751                  */
1752                 packets_to_transfer = (bytes_to_transfer + pipe->max_packet - 1) / pipe->max_packet;
1753                 if (packets_to_transfer == 0)
1754                         packets_to_transfer = 1;
1755                 else if ((packets_to_transfer > 1) && (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1756                         /*
1757                          * Limit to one packet when not using DMA. Channels must
1758                          * be restarted between every packet for IN
1759                          * transactions, so there is no reason to do multiple
1760                          * packets in a row
1761                          */
1762                         packets_to_transfer = 1;
1763                         bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1764                 } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1765                         /*
1766                          * Limit the number of packet and data transferred to
1767                          * what the hardware can handle
1768                          */
1769                         packets_to_transfer = MAX_TRANSFER_PACKETS;
1770                         bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1771                 }
1772
1773                 usbc_hctsiz.s.xfersize = bytes_to_transfer;
1774                 usbc_hctsiz.s.pktcnt = packets_to_transfer;
1775
1776                 /* Update the DATA0/DATA1 toggle */
1777                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1778                 /*
1779                  * High speed pipes may need a hardware ping before they start
1780                  */
1781                 if (pipe->flags & __CVMX_USB_PIPE_FLAGS_NEED_PING)
1782                         usbc_hctsiz.s.dopng = 1;
1783
1784                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCSPLTX(channel, usb->index), usbc_hcsplt.u32);
1785                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index), usbc_hctsiz.u32);
1786         }
1787
1788         /* Setup the Host Channel Characteristics Register */
1789         {
1790                 union cvmx_usbcx_hccharx usbc_hcchar = {.u32 = 0};
1791
1792                 /*
1793                  * Set the startframe odd/even properly. This is only used for
1794                  * periodic
1795                  */
1796                 usbc_hcchar.s.oddfrm = usb->frame_number&1;
1797
1798                 /*
1799                  * Set the number of back to back packets allowed by this
1800                  * endpoint. Split transactions interpret "ec" as the number of
1801                  * immediate retries of failure. These retries happen too
1802                  * quickly, so we disable these entirely for splits
1803                  */
1804                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
1805                         usbc_hcchar.s.ec = 1;
1806                 else if (pipe->multi_count < 1)
1807                         usbc_hcchar.s.ec = 1;
1808                 else if (pipe->multi_count > 3)
1809                         usbc_hcchar.s.ec = 3;
1810                 else
1811                         usbc_hcchar.s.ec = pipe->multi_count;
1812
1813                 /* Set the rest of the endpoint specific settings */
1814                 usbc_hcchar.s.devaddr = pipe->device_addr;
1815                 usbc_hcchar.s.eptype = transaction->type;
1816                 usbc_hcchar.s.lspddev = (pipe->device_speed == CVMX_USB_SPEED_LOW);
1817                 usbc_hcchar.s.epdir = pipe->transfer_dir;
1818                 usbc_hcchar.s.epnum = pipe->endpoint_num;
1819                 usbc_hcchar.s.mps = pipe->max_packet;
1820                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index), usbc_hcchar.u32);
1821         }
1822
1823         /* Do transaction type specific fixups as needed */
1824         switch (transaction->type) {
1825         case CVMX_USB_TRANSFER_CONTROL:
1826                 __cvmx_usb_start_channel_control(usb, channel, pipe);
1827                 break;
1828         case CVMX_USB_TRANSFER_BULK:
1829         case CVMX_USB_TRANSFER_INTERRUPT:
1830                 break;
1831         case CVMX_USB_TRANSFER_ISOCHRONOUS:
1832                 if (!__cvmx_usb_pipe_needs_split(usb, pipe)) {
1833                         /*
1834                          * ISO transactions require different PIDs depending on
1835                          * direction and how many packets are needed
1836                          */
1837                         if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
1838                                 if (pipe->multi_count < 2) /* Need DATA0 */
1839                                         USB_SET_FIELD32(CVMX_USBCX_HCTSIZX(channel, usb->index), union cvmx_usbcx_hctsizx, pid, 0);
1840                                 else /* Need MDATA */
1841                                         USB_SET_FIELD32(CVMX_USBCX_HCTSIZX(channel, usb->index), union cvmx_usbcx_hctsizx, pid, 3);
1842                         }
1843                 }
1844                 break;
1845         }
1846         {
1847                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index))};
1848                 transaction->xfersize = usbc_hctsiz.s.xfersize;
1849                 transaction->pktcnt = usbc_hctsiz.s.pktcnt;
1850         }
1851         /* Remeber when we start a split transaction */
1852         if (__cvmx_usb_pipe_needs_split(usb, pipe))
1853                 usb->active_split = transaction;
1854         USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), union cvmx_usbcx_hccharx, chena, 1);
1855         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
1856                 __cvmx_usb_fill_tx_fifo(usb, channel);
1857         return;
1858 }
1859
1860
1861 /**
1862  * Find a pipe that is ready to be scheduled to hardware.
1863  * @usb:         USB device state populated by cvmx_usb_initialize().
1864  * @list:        Pipe list to search
1865  * @current_frame:
1866  *               Frame counter to use as a time reference.
1867  *
1868  * Returns: Pipe or NULL if none are ready
1869  */
1870 static struct cvmx_usb_pipe *__cvmx_usb_find_ready_pipe(struct cvmx_usb_state *usb, struct list_head *list, uint64_t current_frame)
1871 {
1872         struct cvmx_usb_pipe *pipe;
1873
1874         list_for_each_entry(pipe, list, node) {
1875                 if (!(pipe->flags & __CVMX_USB_PIPE_FLAGS_SCHEDULED) && pipe->head &&
1876                         (pipe->next_tx_frame <= current_frame) &&
1877                         ((pipe->split_sc_frame == -1) || ((((int)current_frame - (int)pipe->split_sc_frame) & 0x7f) < 0x40)) &&
1878                         (!usb->active_split || (usb->active_split == pipe->head))) {
1879                         CVMX_PREFETCH(pipe, 128);
1880                         CVMX_PREFETCH(pipe->head, 0);
1881                         return pipe;
1882                 }
1883         }
1884         return NULL;
1885 }
1886
1887
1888 /**
1889  * Called whenever a pipe might need to be scheduled to the
1890  * hardware.
1891  *
1892  * @usb:         USB device state populated by cvmx_usb_initialize().
1893  * @is_sof:      True if this schedule was called on a SOF interrupt.
1894  */
1895 static void __cvmx_usb_schedule(struct cvmx_usb_state *usb, int is_sof)
1896 {
1897         int channel;
1898         struct cvmx_usb_pipe *pipe;
1899         int need_sof;
1900         enum cvmx_usb_transfer ttype;
1901
1902         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1903                 /*
1904                  * Without DMA we need to be careful to not schedule something
1905                  * at the end of a frame and cause an overrun.
1906                  */
1907                 union cvmx_usbcx_hfnum hfnum = {.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index))};
1908                 union cvmx_usbcx_hfir hfir = {.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HFIR(usb->index))};
1909                 if (hfnum.s.frrem < hfir.s.frint/4)
1910                         goto done;
1911         }
1912
1913         while (usb->idle_hardware_channels) {
1914                 /* Find an idle channel */
1915                 channel = __fls(usb->idle_hardware_channels);
1916                 if (unlikely(channel > 7))
1917                         break;
1918
1919                 /* Find a pipe needing service */
1920                 pipe = NULL;
1921                 if (is_sof) {
1922                         /*
1923                          * Only process periodic pipes on SOF interrupts. This
1924                          * way we are sure that the periodic data is sent in the
1925                          * beginning of the frame
1926                          */
1927                         pipe = __cvmx_usb_find_ready_pipe(usb, usb->active_pipes + CVMX_USB_TRANSFER_ISOCHRONOUS, usb->frame_number);
1928                         if (likely(!pipe))
1929                                 pipe = __cvmx_usb_find_ready_pipe(usb, usb->active_pipes + CVMX_USB_TRANSFER_INTERRUPT, usb->frame_number);
1930                 }
1931                 if (likely(!pipe)) {
1932                         pipe = __cvmx_usb_find_ready_pipe(usb, usb->active_pipes + CVMX_USB_TRANSFER_CONTROL, usb->frame_number);
1933                         if (likely(!pipe))
1934                                 pipe = __cvmx_usb_find_ready_pipe(usb, usb->active_pipes + CVMX_USB_TRANSFER_BULK, usb->frame_number);
1935                 }
1936                 if (!pipe)
1937                         break;
1938
1939                 __cvmx_usb_start_channel(usb, channel, pipe);
1940         }
1941
1942 done:
1943         /*
1944          * Only enable SOF interrupts when we have transactions pending in the
1945          * future that might need to be scheduled
1946          */
1947         need_sof = 0;
1948         for (ttype = CVMX_USB_TRANSFER_CONTROL; ttype <= CVMX_USB_TRANSFER_INTERRUPT; ttype++) {
1949                 list_for_each_entry(pipe, &usb->active_pipes[ttype], node) {
1950                         if (pipe->next_tx_frame > usb->frame_number) {
1951                                 need_sof = 1;
1952                                 break;
1953                         }
1954                 }
1955         }
1956         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk, sofmsk, need_sof);
1957         return;
1958 }
1959
1960 static inline struct octeon_hcd *cvmx_usb_to_octeon(struct cvmx_usb_state *p)
1961 {
1962         return container_of(p, struct octeon_hcd, usb);
1963 }
1964
1965 static inline struct usb_hcd *octeon_to_hcd(struct octeon_hcd *p)
1966 {
1967         return container_of((void *)p, struct usb_hcd, hcd_priv);
1968 }
1969
1970 static void octeon_usb_urb_complete_callback(struct cvmx_usb_state *usb,
1971                                              enum cvmx_usb_complete status,
1972                                              struct cvmx_usb_pipe *pipe,
1973                                              struct cvmx_usb_transaction
1974                                                 *transaction,
1975                                              int bytes_transferred,
1976                                              struct urb *urb)
1977 {
1978         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
1979         struct usb_hcd *hcd = octeon_to_hcd(priv);
1980         struct device *dev = hcd->self.controller;
1981
1982         urb->actual_length = bytes_transferred;
1983         urb->hcpriv = NULL;
1984
1985         if (!list_empty(&urb->urb_list))
1986                 /*
1987                  * It is on the dequeue_list, but we are going to call
1988                  * usb_hcd_giveback_urb(), so we must clear it from
1989                  * the list.  We got to it before the
1990                  * octeon_usb_urb_dequeue_work() tasklet did.
1991                  */
1992                 list_del_init(&urb->urb_list);
1993
1994         /* For Isochronous transactions we need to update the URB packet status
1995            list from data in our private copy */
1996         if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
1997                 int i;
1998                 /*
1999                  * The pointer to the private list is stored in the setup_packet
2000                  * field.
2001                  */
2002                 struct cvmx_usb_iso_packet *iso_packet =
2003                         (struct cvmx_usb_iso_packet *) urb->setup_packet;
2004                 /* Recalculate the transfer size by adding up each packet */
2005                 urb->actual_length = 0;
2006                 for (i = 0; i < urb->number_of_packets; i++) {
2007                         if (iso_packet[i].status == CVMX_USB_COMPLETE_SUCCESS) {
2008                                 urb->iso_frame_desc[i].status = 0;
2009                                 urb->iso_frame_desc[i].actual_length = iso_packet[i].length;
2010                                 urb->actual_length += urb->iso_frame_desc[i].actual_length;
2011                         } else {
2012                                 dev_dbg(dev, "ISOCHRONOUS packet=%d of %d status=%d pipe=%p transaction=%p size=%d\n",
2013                                         i, urb->number_of_packets,
2014                                         iso_packet[i].status, pipe,
2015                                         transaction, iso_packet[i].length);
2016                                 urb->iso_frame_desc[i].status = -EREMOTEIO;
2017                         }
2018                 }
2019                 /* Free the private list now that we don't need it anymore */
2020                 kfree(iso_packet);
2021                 urb->setup_packet = NULL;
2022         }
2023
2024         switch (status) {
2025         case CVMX_USB_COMPLETE_SUCCESS:
2026                 urb->status = 0;
2027                 break;
2028         case CVMX_USB_COMPLETE_CANCEL:
2029                 if (urb->status == 0)
2030                         urb->status = -ENOENT;
2031                 break;
2032         case CVMX_USB_COMPLETE_STALL:
2033                 dev_dbg(dev, "status=stall pipe=%p transaction=%p size=%d\n",
2034                         pipe, transaction, bytes_transferred);
2035                 urb->status = -EPIPE;
2036                 break;
2037         case CVMX_USB_COMPLETE_BABBLEERR:
2038                 dev_dbg(dev, "status=babble pipe=%p transaction=%p size=%d\n",
2039                         pipe, transaction, bytes_transferred);
2040                 urb->status = -EPIPE;
2041                 break;
2042         case CVMX_USB_COMPLETE_SHORT:
2043                 dev_dbg(dev, "status=short pipe=%p transaction=%p size=%d\n",
2044                         pipe, transaction, bytes_transferred);
2045                 urb->status = -EREMOTEIO;
2046                 break;
2047         case CVMX_USB_COMPLETE_ERROR:
2048         case CVMX_USB_COMPLETE_XACTERR:
2049         case CVMX_USB_COMPLETE_DATATGLERR:
2050         case CVMX_USB_COMPLETE_FRAMEERR:
2051                 dev_dbg(dev, "status=%d pipe=%p transaction=%p size=%d\n",
2052                         status, pipe, transaction, bytes_transferred);
2053                 urb->status = -EPROTO;
2054                 break;
2055         }
2056         spin_unlock(&priv->lock);
2057         usb_hcd_giveback_urb(octeon_to_hcd(priv), urb, urb->status);
2058         spin_lock(&priv->lock);
2059 }
2060
2061 /**
2062  * Signal the completion of a transaction and free it. The
2063  * transaction will be removed from the pipe transaction list.
2064  *
2065  * @usb:         USB device state populated by cvmx_usb_initialize().
2066  * @pipe:        Pipe the transaction is on
2067  * @transaction:
2068  *               Transaction that completed
2069  * @complete_code:
2070  *               Completion code
2071  */
2072 static void __cvmx_usb_perform_complete(struct cvmx_usb_state *usb,
2073                                         struct cvmx_usb_pipe *pipe,
2074                                         struct cvmx_usb_transaction *transaction,
2075                                         enum cvmx_usb_complete complete_code)
2076 {
2077         /* If this was a split then clear our split in progress marker */
2078         if (usb->active_split == transaction)
2079                 usb->active_split = NULL;
2080
2081         /*
2082          * Isochronous transactions need extra processing as they might not be
2083          * done after a single data transfer
2084          */
2085         if (unlikely(transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
2086                 /* Update the number of bytes transferred in this ISO packet */
2087                 transaction->iso_packets[0].length = transaction->actual_bytes;
2088                 transaction->iso_packets[0].status = complete_code;
2089
2090                 /*
2091                  * If there are more ISOs pending and we succeeded, schedule the
2092                  * next one
2093                  */
2094                 if ((transaction->iso_number_packets > 1) && (complete_code == CVMX_USB_COMPLETE_SUCCESS)) {
2095                         /* No bytes transferred for this packet as of yet */
2096                         transaction->actual_bytes = 0;
2097                         /* One less ISO waiting to transfer */
2098                         transaction->iso_number_packets--;
2099                         /* Increment to the next location in our packet array */
2100                         transaction->iso_packets++;
2101                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2102                         goto done;
2103                 }
2104         }
2105
2106         /* Remove the transaction from the pipe list */
2107         if (transaction->next)
2108                 transaction->next->prev = transaction->prev;
2109         else
2110                 pipe->tail = transaction->prev;
2111         if (transaction->prev)
2112                 transaction->prev->next = transaction->next;
2113         else
2114                 pipe->head = transaction->next;
2115         if (!pipe->head)
2116                 list_move_tail(&pipe->node, &usb->idle_pipes);
2117         octeon_usb_urb_complete_callback(usb, complete_code, pipe,
2118                                          transaction,
2119                                          transaction->actual_bytes,
2120                                          transaction->urb);
2121         kfree(transaction);
2122 done:
2123         return;
2124 }
2125
2126
2127 /**
2128  * Submit a usb transaction to a pipe. Called for all types
2129  * of transactions.
2130  *
2131  * @usb:
2132  * @pipe:           Which pipe to submit to.
2133  * @type:           Transaction type
2134  * @buffer:         User buffer for the transaction
2135  * @buffer_length:
2136  *                  User buffer's length in bytes
2137  * @control_header:
2138  *                  For control transactions, the 8 byte standard header
2139  * @iso_start_frame:
2140  *                  For ISO transactions, the start frame
2141  * @iso_number_packets:
2142  *                  For ISO, the number of packet in the transaction.
2143  * @iso_packets:
2144  *                  A description of each ISO packet
2145  * @urb:            URB for the callback
2146  *
2147  * Returns: Transaction or NULL on failure.
2148  */
2149 static struct cvmx_usb_transaction *__cvmx_usb_submit_transaction(struct cvmx_usb_state *usb,
2150                                                                   struct cvmx_usb_pipe *pipe,
2151                                                                   enum cvmx_usb_transfer type,
2152                                                                   uint64_t buffer,
2153                                                                   int buffer_length,
2154                                                                   uint64_t control_header,
2155                                                                   int iso_start_frame,
2156                                                                   int iso_number_packets,
2157                                                                   struct cvmx_usb_iso_packet *iso_packets,
2158                                                                   struct urb *urb)
2159 {
2160         struct cvmx_usb_transaction *transaction;
2161
2162         if (unlikely(pipe->transfer_type != type))
2163                 return NULL;
2164
2165         transaction = kzalloc(sizeof(*transaction), GFP_ATOMIC);
2166         if (unlikely(!transaction))
2167                 return NULL;
2168
2169         transaction->type = type;
2170         transaction->buffer = buffer;
2171         transaction->buffer_length = buffer_length;
2172         transaction->control_header = control_header;
2173         /* FIXME: This is not used, implement it. */
2174         transaction->iso_start_frame = iso_start_frame;
2175         transaction->iso_number_packets = iso_number_packets;
2176         transaction->iso_packets = iso_packets;
2177         transaction->urb = urb;
2178         if (transaction->type == CVMX_USB_TRANSFER_CONTROL)
2179                 transaction->stage = CVMX_USB_STAGE_SETUP;
2180         else
2181                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2182
2183         transaction->next = NULL;
2184         if (pipe->tail) {
2185                 transaction->prev = pipe->tail;
2186                 transaction->prev->next = transaction;
2187         } else {
2188                 if (pipe->next_tx_frame < usb->frame_number)
2189                         pipe->next_tx_frame = usb->frame_number + pipe->interval -
2190                                 (usb->frame_number - pipe->next_tx_frame) % pipe->interval;
2191                 transaction->prev = NULL;
2192                 pipe->head = transaction;
2193                 list_move_tail(&pipe->node,
2194                                &usb->active_pipes[pipe->transfer_type]);
2195         }
2196         pipe->tail = transaction;
2197
2198         /* We may need to schedule the pipe if this was the head of the pipe */
2199         if (!transaction->prev)
2200                 __cvmx_usb_schedule(usb, 0);
2201
2202         return transaction;
2203 }
2204
2205
2206 /**
2207  * Call to submit a USB Bulk transfer to a pipe.
2208  *
2209  * @usb:            USB device state populated by cvmx_usb_initialize().
2210  * @pipe:           Handle to the pipe for the transfer.
2211  * @urb:            URB.
2212  *
2213  * Returns: A submitted transaction or NULL on failure.
2214  */
2215 static struct cvmx_usb_transaction *cvmx_usb_submit_bulk(struct cvmx_usb_state *usb,
2216                                                          struct cvmx_usb_pipe *pipe,
2217                                                          struct urb *urb)
2218 {
2219         return __cvmx_usb_submit_transaction(usb, pipe, CVMX_USB_TRANSFER_BULK,
2220                                              urb->transfer_dma,
2221                                              urb->transfer_buffer_length,
2222                                              0, /* control_header */
2223                                              0, /* iso_start_frame */
2224                                              0, /* iso_number_packets */
2225                                              NULL, /* iso_packets */
2226                                              urb);
2227 }
2228
2229
2230 /**
2231  * Call to submit a USB Interrupt transfer to a pipe.
2232  *
2233  * @usb:            USB device state populated by cvmx_usb_initialize().
2234  * @pipe:           Handle to the pipe for the transfer.
2235  * @urb:            URB returned when the callback is called.
2236  *
2237  * Returns: A submitted transaction or NULL on failure.
2238  */
2239 static struct cvmx_usb_transaction *cvmx_usb_submit_interrupt(struct cvmx_usb_state *usb,
2240                                                               struct cvmx_usb_pipe *pipe,
2241                                                               struct urb *urb)
2242 {
2243         return __cvmx_usb_submit_transaction(usb, pipe,
2244                                              CVMX_USB_TRANSFER_INTERRUPT,
2245                                              urb->transfer_dma,
2246                                              urb->transfer_buffer_length,
2247                                              0, /* control_header */
2248                                              0, /* iso_start_frame */
2249                                              0, /* iso_number_packets */
2250                                              NULL, /* iso_packets */
2251                                              urb);
2252 }
2253
2254
2255 /**
2256  * Call to submit a USB Control transfer to a pipe.
2257  *
2258  * @usb:            USB device state populated by cvmx_usb_initialize().
2259  * @pipe:           Handle to the pipe for the transfer.
2260  * @urb:            URB.
2261  *
2262  * Returns: A submitted transaction or NULL on failure.
2263  */
2264 static struct cvmx_usb_transaction *cvmx_usb_submit_control(struct cvmx_usb_state *usb,
2265                                                             struct cvmx_usb_pipe *pipe,
2266                                                             struct urb *urb)
2267 {
2268         int buffer_length = urb->transfer_buffer_length;
2269         uint64_t control_header = urb->setup_dma;
2270         union cvmx_usb_control_header *header =
2271                 cvmx_phys_to_ptr(control_header);
2272
2273         if ((header->s.request_type & 0x80) == 0)
2274                 buffer_length = le16_to_cpu(header->s.length);
2275
2276         return __cvmx_usb_submit_transaction(usb, pipe,
2277                                              CVMX_USB_TRANSFER_CONTROL,
2278                                              urb->transfer_dma, buffer_length,
2279                                              control_header,
2280                                              0, /* iso_start_frame */
2281                                              0, /* iso_number_packets */
2282                                              NULL, /* iso_packets */
2283                                              urb);
2284 }
2285
2286
2287 /**
2288  * Call to submit a USB Isochronous transfer to a pipe.
2289  *
2290  * @usb:            USB device state populated by cvmx_usb_initialize().
2291  * @pipe:           Handle to the pipe for the transfer.
2292  * @urb:            URB returned when the callback is called.
2293  *
2294  * Returns: A submitted transaction or NULL on failure.
2295  */
2296 static struct cvmx_usb_transaction *cvmx_usb_submit_isochronous(struct cvmx_usb_state *usb,
2297                                                                 struct cvmx_usb_pipe *pipe,
2298                                                                 struct urb *urb)
2299 {
2300         struct cvmx_usb_iso_packet *packets;
2301
2302         packets = (struct cvmx_usb_iso_packet *) urb->setup_packet;
2303         return __cvmx_usb_submit_transaction(usb, pipe,
2304                                              CVMX_USB_TRANSFER_ISOCHRONOUS,
2305                                              urb->transfer_dma,
2306                                              urb->transfer_buffer_length,
2307                                              0, /* control_header */
2308                                              urb->start_frame,
2309                                              urb->number_of_packets,
2310                                              packets, urb);
2311 }
2312
2313
2314 /**
2315  * Cancel one outstanding request in a pipe. Canceling a request
2316  * can fail if the transaction has already completed before cancel
2317  * is called. Even after a successful cancel call, it may take
2318  * a frame or two for the cvmx_usb_poll() function to call the
2319  * associated callback.
2320  *
2321  * @usb:         USB device state populated by cvmx_usb_initialize().
2322  * @pipe:        Pipe to cancel requests in.
2323  * @transaction: Transaction to cancel, returned by the submit function.
2324  *
2325  * Returns: 0 or a negative error code.
2326  */
2327 static int cvmx_usb_cancel(struct cvmx_usb_state *usb,
2328                            struct cvmx_usb_pipe *pipe,
2329                            struct cvmx_usb_transaction *transaction)
2330 {
2331         /*
2332          * If the transaction is the HEAD of the queue and scheduled. We need to
2333          * treat it special
2334          */
2335         if ((pipe->head == transaction) &&
2336                 (pipe->flags & __CVMX_USB_PIPE_FLAGS_SCHEDULED)) {
2337                 union cvmx_usbcx_hccharx usbc_hcchar;
2338
2339                 usb->pipe_for_channel[pipe->channel] = NULL;
2340                 pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_SCHEDULED;
2341
2342                 CVMX_SYNCW;
2343
2344                 usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCCHARX(pipe->channel, usb->index));
2345                 /*
2346                  * If the channel isn't enabled then the transaction already
2347                  * completed.
2348                  */
2349                 if (usbc_hcchar.s.chena) {
2350                         usbc_hcchar.s.chdis = 1;
2351                         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCCHARX(pipe->channel, usb->index), usbc_hcchar.u32);
2352                 }
2353         }
2354         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_CANCEL);
2355         return 0;
2356 }
2357
2358
2359 /**
2360  * Cancel all outstanding requests in a pipe. Logically all this
2361  * does is call cvmx_usb_cancel() in a loop.
2362  *
2363  * @usb:         USB device state populated by cvmx_usb_initialize().
2364  * @pipe:        Pipe to cancel requests in.
2365  *
2366  * Returns: 0 or a negative error code.
2367  */
2368 static int cvmx_usb_cancel_all(struct cvmx_usb_state *usb,
2369                                struct cvmx_usb_pipe *pipe)
2370 {
2371         /* Simply loop through and attempt to cancel each transaction */
2372         while (pipe->head) {
2373                 int result = cvmx_usb_cancel(usb, pipe, pipe->head);
2374                 if (unlikely(result != 0))
2375                         return result;
2376         }
2377         return 0;
2378 }
2379
2380
2381 /**
2382  * Close a pipe created with cvmx_usb_open_pipe().
2383  *
2384  * @usb:         USB device state populated by cvmx_usb_initialize().
2385  * @pipe:        Pipe to close.
2386  *
2387  * Returns: 0 or a negative error code. EBUSY is returned if the pipe has
2388  *          outstanding transfers.
2389  */
2390 static int cvmx_usb_close_pipe(struct cvmx_usb_state *usb,
2391                                struct cvmx_usb_pipe *pipe)
2392 {
2393         /* Fail if the pipe has pending transactions */
2394         if (unlikely(pipe->head))
2395                 return -EBUSY;
2396
2397         list_del(&pipe->node);
2398         kfree(pipe);
2399
2400         return 0;
2401 }
2402
2403 /**
2404  * Get the current USB protocol level frame number. The frame
2405  * number is always in the range of 0-0x7ff.
2406  *
2407  * @usb: USB device state populated by cvmx_usb_initialize().
2408  *
2409  * Returns: USB frame number
2410  */
2411 static int cvmx_usb_get_frame_number(struct cvmx_usb_state *usb)
2412 {
2413         int frame_number;
2414         union cvmx_usbcx_hfnum usbc_hfnum;
2415
2416         usbc_hfnum.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
2417         frame_number = usbc_hfnum.s.frnum;
2418
2419         return frame_number;
2420 }
2421
2422
2423 /**
2424  * Poll a channel for status
2425  *
2426  * @usb:     USB device
2427  * @channel: Channel to poll
2428  *
2429  * Returns: Zero on success
2430  */
2431 static int __cvmx_usb_poll_channel(struct cvmx_usb_state *usb, int channel)
2432 {
2433         union cvmx_usbcx_hcintx usbc_hcint;
2434         union cvmx_usbcx_hctsizx usbc_hctsiz;
2435         union cvmx_usbcx_hccharx usbc_hcchar;
2436         struct cvmx_usb_pipe *pipe;
2437         struct cvmx_usb_transaction *transaction;
2438         int bytes_this_transfer;
2439         int bytes_in_last_packet;
2440         int packets_processed;
2441         int buffer_space_left;
2442
2443         /* Read the interrupt status bits for the channel */
2444         usbc_hcint.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCINTX(channel, usb->index));
2445
2446         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
2447                 usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index));
2448
2449                 if (usbc_hcchar.s.chena && usbc_hcchar.s.chdis) {
2450                         /*
2451                          * There seems to be a bug in CN31XX which can cause
2452                          * interrupt IN transfers to get stuck until we do a
2453                          * write of HCCHARX without changing things
2454                          */
2455                         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index), usbc_hcchar.u32);
2456                         return 0;
2457                 }
2458
2459                 /*
2460                  * In non DMA mode the channels don't halt themselves. We need
2461                  * to manually disable channels that are left running
2462                  */
2463                 if (!usbc_hcint.s.chhltd) {
2464                         if (usbc_hcchar.s.chena) {
2465                                 union cvmx_usbcx_hcintmskx hcintmsk;
2466                                 /* Disable all interrupts except CHHLTD */
2467                                 hcintmsk.u32 = 0;
2468                                 hcintmsk.s.chhltdmsk = 1;
2469                                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), hcintmsk.u32);
2470                                 usbc_hcchar.s.chdis = 1;
2471                                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index), usbc_hcchar.u32);
2472                                 return 0;
2473                         } else if (usbc_hcint.s.xfercompl) {
2474                                 /*
2475                                  * Successful IN/OUT with transfer complete.
2476                                  * Channel halt isn't needed.
2477                                  */
2478                         } else {
2479                                 cvmx_dprintf("USB%d: Channel %d interrupt without halt\n", usb->index, channel);
2480                                 return 0;
2481                         }
2482                 }
2483         } else {
2484                 /*
2485                  * There is are no interrupts that we need to process when the
2486                  * channel is still running
2487                  */
2488                 if (!usbc_hcint.s.chhltd)
2489                         return 0;
2490         }
2491
2492         /* Disable the channel interrupts now that it is done */
2493         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
2494         usb->idle_hardware_channels |= (1<<channel);
2495
2496         /* Make sure this channel is tied to a valid pipe */
2497         pipe = usb->pipe_for_channel[channel];
2498         CVMX_PREFETCH(pipe, 0);
2499         CVMX_PREFETCH(pipe, 128);
2500         if (!pipe)
2501                 return 0;
2502         transaction = pipe->head;
2503         CVMX_PREFETCH(transaction, 0);
2504
2505         /*
2506          * Disconnect this pipe from the HW channel. Later the schedule
2507          * function will figure out which pipe needs to go
2508          */
2509         usb->pipe_for_channel[channel] = NULL;
2510         pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_SCHEDULED;
2511
2512         /*
2513          * Read the channel config info so we can figure out how much data
2514          * transfered
2515          */
2516         usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index));
2517         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index));
2518
2519         /*
2520          * Calculating the number of bytes successfully transferred is dependent
2521          * on the transfer direction
2522          */
2523         packets_processed = transaction->pktcnt - usbc_hctsiz.s.pktcnt;
2524         if (usbc_hcchar.s.epdir) {
2525                 /*
2526                  * IN transactions are easy. For every byte received the
2527                  * hardware decrements xfersize. All we need to do is subtract
2528                  * the current value of xfersize from its starting value and we
2529                  * know how many bytes were written to the buffer
2530                  */
2531                 bytes_this_transfer = transaction->xfersize - usbc_hctsiz.s.xfersize;
2532         } else {
2533                 /*
2534                  * OUT transaction don't decrement xfersize. Instead pktcnt is
2535                  * decremented on every successful packet send. The hardware
2536                  * does this when it receives an ACK, or NYET. If it doesn't
2537                  * receive one of these responses pktcnt doesn't change
2538                  */
2539                 bytes_this_transfer = packets_processed * usbc_hcchar.s.mps;
2540                 /*
2541                  * The last packet may not be a full transfer if we didn't have
2542                  * enough data
2543                  */
2544                 if (bytes_this_transfer > transaction->xfersize)
2545                         bytes_this_transfer = transaction->xfersize;
2546         }
2547         /* Figure out how many bytes were in the last packet of the transfer */
2548         if (packets_processed)
2549                 bytes_in_last_packet = bytes_this_transfer - (packets_processed-1) * usbc_hcchar.s.mps;
2550         else
2551                 bytes_in_last_packet = bytes_this_transfer;
2552
2553         /*
2554          * As a special case, setup transactions output the setup header, not
2555          * the user's data. For this reason we don't count setup data as bytes
2556          * transferred
2557          */
2558         if ((transaction->stage == CVMX_USB_STAGE_SETUP) ||
2559                 (transaction->stage == CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE))
2560                 bytes_this_transfer = 0;
2561
2562         /*
2563          * Add the bytes transferred to the running total. It is important that
2564          * bytes_this_transfer doesn't count any data that needs to be
2565          * retransmitted
2566          */
2567         transaction->actual_bytes += bytes_this_transfer;
2568         if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
2569                 buffer_space_left = transaction->iso_packets[0].length - transaction->actual_bytes;
2570         else
2571                 buffer_space_left = transaction->buffer_length - transaction->actual_bytes;
2572
2573         /*
2574          * We need to remember the PID toggle state for the next transaction.
2575          * The hardware already updated it for the next transaction
2576          */
2577         pipe->pid_toggle = !(usbc_hctsiz.s.pid == 0);
2578
2579         /*
2580          * For high speed bulk out, assume the next transaction will need to do
2581          * a ping before proceeding. If this isn't true the ACK processing below
2582          * will clear this flag
2583          */
2584         if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2585                 (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
2586                 (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT))
2587                 pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
2588
2589         if (usbc_hcint.s.stall) {
2590                 /*
2591                  * STALL as a response means this transaction cannot be
2592                  * completed because the device can't process transactions. Tell
2593                  * the user. Any data that was transferred will be counted on
2594                  * the actual bytes transferred
2595                  */
2596                 pipe->pid_toggle = 0;
2597                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_STALL);
2598         } else if (usbc_hcint.s.xacterr) {
2599                 /*
2600                  * We know at least one packet worked if we get a ACK or NAK.
2601                  * Reset the retry counter
2602                  */
2603                 if (usbc_hcint.s.nak || usbc_hcint.s.ack)
2604                         transaction->retries = 0;
2605                 transaction->retries++;
2606                 if (transaction->retries > MAX_RETRIES) {
2607                         /*
2608                          * XactErr as a response means the device signaled
2609                          * something wrong with the transfer. For example, PID
2610                          * toggle errors cause these
2611                          */
2612                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_XACTERR);
2613                 } else {
2614                         /*
2615                          * If this was a split then clear our split in progress
2616                          * marker
2617                          */
2618                         if (usb->active_split == transaction)
2619                                 usb->active_split = NULL;
2620                         /*
2621                          * Rewind to the beginning of the transaction by anding
2622                          * off the split complete bit
2623                          */
2624                         transaction->stage &= ~1;
2625                         pipe->split_sc_frame = -1;
2626                         pipe->next_tx_frame += pipe->interval;
2627                         if (pipe->next_tx_frame < usb->frame_number)
2628                                 pipe->next_tx_frame = usb->frame_number + pipe->interval -
2629                                                       (usb->frame_number - pipe->next_tx_frame) % pipe->interval;
2630                 }
2631         } else if (usbc_hcint.s.bblerr) {
2632                 /* Babble Error (BblErr) */
2633                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_BABBLEERR);
2634         } else if (usbc_hcint.s.datatglerr) {
2635                 /* We'll retry the exact same transaction again */
2636                 transaction->retries++;
2637         } else if (usbc_hcint.s.nyet) {
2638                 /*
2639                  * NYET as a response is only allowed in three cases: as a
2640                  * response to a ping, as a response to a split transaction, and
2641                  * as a response to a bulk out. The ping case is handled by
2642                  * hardware, so we only have splits and bulk out
2643                  */
2644                 if (!__cvmx_usb_pipe_needs_split(usb, pipe)) {
2645                         transaction->retries = 0;
2646                         /*
2647                          * If there is more data to go then we need to try
2648                          * again. Otherwise this transaction is complete
2649                          */
2650                         if ((buffer_space_left == 0) || (bytes_in_last_packet < pipe->max_packet))
2651                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2652                 } else {
2653                         /*
2654                          * Split transactions retry the split complete 4 times
2655                          * then rewind to the start split and do the entire
2656                          * transactions again
2657                          */
2658                         transaction->retries++;
2659                         if ((transaction->retries & 0x3) == 0) {
2660                                 /*
2661                                  * Rewind to the beginning of the transaction by
2662                                  * anding off the split complete bit
2663                                  */
2664                                 transaction->stage &= ~1;
2665                                 pipe->split_sc_frame = -1;
2666                         }
2667                 }
2668         } else if (usbc_hcint.s.ack) {
2669                 transaction->retries = 0;
2670                 /*
2671                  * The ACK bit can only be checked after the other error bits.
2672                  * This is because a multi packet transfer may succeed in a
2673                  * number of packets and then get a different response on the
2674                  * last packet. In this case both ACK and the last response bit
2675                  * will be set. If none of the other response bits is set, then
2676                  * the last packet must have been an ACK
2677                  *
2678                  * Since we got an ACK, we know we don't need to do a ping on
2679                  * this pipe
2680                  */
2681                 pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_NEED_PING;
2682
2683                 switch (transaction->type) {
2684                 case CVMX_USB_TRANSFER_CONTROL:
2685                         switch (transaction->stage) {
2686                         case CVMX_USB_STAGE_NON_CONTROL:
2687                         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
2688                                 /* This should be impossible */
2689                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_ERROR);
2690                                 break;
2691                         case CVMX_USB_STAGE_SETUP:
2692                                 pipe->pid_toggle = 1;
2693                                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
2694                                         transaction->stage = CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE;
2695                                 else {
2696                                         union cvmx_usb_control_header *header =
2697                                                 cvmx_phys_to_ptr(transaction->control_header);
2698                                         if (header->s.length)
2699                                                 transaction->stage = CVMX_USB_STAGE_DATA;
2700                                         else
2701                                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2702                                 }
2703                                 break;
2704                         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
2705                                 {
2706                                         union cvmx_usb_control_header *header =
2707                                                 cvmx_phys_to_ptr(transaction->control_header);
2708                                         if (header->s.length)
2709                                                 transaction->stage = CVMX_USB_STAGE_DATA;
2710                                         else
2711                                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2712                                 }
2713                                 break;
2714                         case CVMX_USB_STAGE_DATA:
2715                                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2716                                         transaction->stage = CVMX_USB_STAGE_DATA_SPLIT_COMPLETE;
2717                                         /*
2718                                          * For setup OUT data that are splits,
2719                                          * the hardware doesn't appear to count
2720                                          * transferred data. Here we manually
2721                                          * update the data transferred
2722                                          */
2723                                         if (!usbc_hcchar.s.epdir) {
2724                                                 if (buffer_space_left < pipe->max_packet)
2725                                                         transaction->actual_bytes += buffer_space_left;
2726                                                 else
2727                                                         transaction->actual_bytes += pipe->max_packet;
2728                                         }
2729                                 } else if ((buffer_space_left == 0) || (bytes_in_last_packet < pipe->max_packet)) {
2730                                         pipe->pid_toggle = 1;
2731                                         transaction->stage = CVMX_USB_STAGE_STATUS;
2732                                 }
2733                                 break;
2734                         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
2735                                 if ((buffer_space_left == 0) || (bytes_in_last_packet < pipe->max_packet)) {
2736                                         pipe->pid_toggle = 1;
2737                                         transaction->stage = CVMX_USB_STAGE_STATUS;
2738                                 } else {
2739                                         transaction->stage = CVMX_USB_STAGE_DATA;
2740                                 }
2741                                 break;
2742                         case CVMX_USB_STAGE_STATUS:
2743                                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
2744                                         transaction->stage = CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE;
2745                                 else
2746                                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2747                                 break;
2748                         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
2749                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2750                                 break;
2751                         }
2752                         break;
2753                 case CVMX_USB_TRANSFER_BULK:
2754                 case CVMX_USB_TRANSFER_INTERRUPT:
2755                         /*
2756                          * The only time a bulk transfer isn't complete when it
2757                          * finishes with an ACK is during a split transaction.
2758                          * For splits we need to continue the transfer if more
2759                          * data is needed
2760                          */
2761                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2762                                 if (transaction->stage == CVMX_USB_STAGE_NON_CONTROL)
2763                                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2764                                 else {
2765                                         if (buffer_space_left && (bytes_in_last_packet == pipe->max_packet))
2766                                                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2767                                         else {
2768                                                 if (transaction->type == CVMX_USB_TRANSFER_INTERRUPT)
2769                                                         pipe->next_tx_frame += pipe->interval;
2770                                                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2771                                         }
2772                                 }
2773                         } else {
2774                                 if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2775                                     (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
2776                                     (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) &&
2777                                     (usbc_hcint.s.nak))
2778                                         pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
2779                                 if (!buffer_space_left || (bytes_in_last_packet < pipe->max_packet)) {
2780                                         if (transaction->type == CVMX_USB_TRANSFER_INTERRUPT)
2781                                                 pipe->next_tx_frame += pipe->interval;
2782                                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2783                                 }
2784                         }
2785                         break;
2786                 case CVMX_USB_TRANSFER_ISOCHRONOUS:
2787                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2788                                 /*
2789                                  * ISOCHRONOUS OUT splits don't require a
2790                                  * complete split stage. Instead they use a
2791                                  * sequence of begin OUT splits to transfer the
2792                                  * data 188 bytes at a time. Once the transfer
2793                                  * is complete, the pipe sleeps until the next
2794                                  * schedule interval
2795                                  */
2796                                 if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
2797                                         /*
2798                                          * If no space left or this wasn't a max
2799                                          * size packet then this transfer is
2800                                          * complete. Otherwise start it again to
2801                                          * send the next 188 bytes
2802                                          */
2803                                         if (!buffer_space_left || (bytes_this_transfer < 188)) {
2804                                                 pipe->next_tx_frame += pipe->interval;
2805                                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2806                                         }
2807                                 } else {
2808                                         if (transaction->stage == CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE) {
2809                                                 /*
2810                                                  * We are in the incoming data
2811                                                  * phase. Keep getting data
2812                                                  * until we run out of space or
2813                                                  * get a small packet
2814                                                  */
2815                                                 if ((buffer_space_left == 0) || (bytes_in_last_packet < pipe->max_packet)) {
2816                                                         pipe->next_tx_frame += pipe->interval;
2817                                                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2818                                                 }
2819                                         } else
2820                                                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2821                                 }
2822                         } else {
2823                                 pipe->next_tx_frame += pipe->interval;
2824                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2825                         }
2826                         break;
2827                 }
2828         } else if (usbc_hcint.s.nak) {
2829                 /*
2830                  * If this was a split then clear our split in progress marker.
2831                  */
2832                 if (usb->active_split == transaction)
2833                         usb->active_split = NULL;
2834                 /*
2835                  * NAK as a response means the device couldn't accept the
2836                  * transaction, but it should be retried in the future. Rewind
2837                  * to the beginning of the transaction by anding off the split
2838                  * complete bit. Retry in the next interval
2839                  */
2840                 transaction->retries = 0;
2841                 transaction->stage &= ~1;
2842                 pipe->next_tx_frame += pipe->interval;
2843                 if (pipe->next_tx_frame < usb->frame_number)
2844                         pipe->next_tx_frame = usb->frame_number + pipe->interval -
2845                                 (usb->frame_number - pipe->next_tx_frame) % pipe->interval;
2846         } else {
2847                 struct cvmx_usb_port_status port;
2848                 port = cvmx_usb_get_status(usb);
2849                 if (port.port_enabled) {
2850                         /* We'll retry the exact same transaction again */
2851                         transaction->retries++;
2852                 } else {
2853                         /*
2854                          * We get channel halted interrupts with no result bits
2855                          * sets when the cable is unplugged
2856                          */
2857                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_ERROR);
2858                 }
2859         }
2860         return 0;
2861 }
2862
2863 static void octeon_usb_port_callback(struct cvmx_usb_state *usb)
2864 {
2865         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
2866
2867         spin_unlock(&priv->lock);
2868         usb_hcd_poll_rh_status(octeon_to_hcd(priv));
2869         spin_lock(&priv->lock);
2870 }
2871
2872 /**
2873  * Poll the USB block for status and call all needed callback
2874  * handlers. This function is meant to be called in the interrupt
2875  * handler for the USB controller. It can also be called
2876  * periodically in a loop for non-interrupt based operation.
2877  *
2878  * @usb: USB device state populated by cvmx_usb_initialize().
2879  *
2880  * Returns: 0 or a negative error code.
2881  */
2882 static int cvmx_usb_poll(struct cvmx_usb_state *usb)
2883 {
2884         union cvmx_usbcx_hfnum usbc_hfnum;
2885         union cvmx_usbcx_gintsts usbc_gintsts;
2886
2887         CVMX_PREFETCH(usb, 0);
2888         CVMX_PREFETCH(usb, 1*128);
2889         CVMX_PREFETCH(usb, 2*128);
2890         CVMX_PREFETCH(usb, 3*128);
2891         CVMX_PREFETCH(usb, 4*128);
2892
2893         /* Update the frame counter */
2894         usbc_hfnum.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
2895         if ((usb->frame_number&0x3fff) > usbc_hfnum.s.frnum)
2896                 usb->frame_number += 0x4000;
2897         usb->frame_number &= ~0x3fffull;
2898         usb->frame_number |= usbc_hfnum.s.frnum;
2899
2900         /* Read the pending interrupts */
2901         usbc_gintsts.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GINTSTS(usb->index));
2902
2903         /* Clear the interrupts now that we know about them */
2904         __cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTSTS(usb->index), usbc_gintsts.u32);
2905
2906         if (usbc_gintsts.s.rxflvl) {
2907                 /*
2908                  * RxFIFO Non-Empty (RxFLvl)
2909                  * Indicates that there is at least one packet pending to be
2910                  * read from the RxFIFO.
2911                  *
2912                  * In DMA mode this is handled by hardware
2913                  */
2914                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
2915                         __cvmx_usb_poll_rx_fifo(usb);
2916         }
2917         if (usbc_gintsts.s.ptxfemp || usbc_gintsts.s.nptxfemp) {
2918                 /* Fill the Tx FIFOs when not in DMA mode */
2919                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
2920                         __cvmx_usb_poll_tx_fifo(usb);
2921         }
2922         if (usbc_gintsts.s.disconnint || usbc_gintsts.s.prtint) {
2923                 union cvmx_usbcx_hprt usbc_hprt;
2924                 /*
2925                  * Disconnect Detected Interrupt (DisconnInt)
2926                  * Asserted when a device disconnect is detected.
2927                  *
2928                  * Host Port Interrupt (PrtInt)
2929                  * The core sets this bit to indicate a change in port status of
2930                  * one of the O2P USB core ports in Host mode. The application
2931                  * must read the Host Port Control and Status (HPRT) register to
2932                  * determine the exact event that caused this interrupt. The
2933                  * application must clear the appropriate status bit in the Host
2934                  * Port Control and Status register to clear this bit.
2935                  *
2936                  * Call the user's port callback
2937                  */
2938                 octeon_usb_port_callback(usb);
2939                 /* Clear the port change bits */
2940                 usbc_hprt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
2941                 usbc_hprt.s.prtena = 0;
2942                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HPRT(usb->index), usbc_hprt.u32);
2943         }
2944         if (usbc_gintsts.s.hchint) {
2945                 /*
2946                  * Host Channels Interrupt (HChInt)
2947                  * The core sets this bit to indicate that an interrupt is
2948                  * pending on one of the channels of the core (in Host mode).
2949                  * The application must read the Host All Channels Interrupt
2950                  * (HAINT) register to determine the exact number of the channel
2951                  * on which the interrupt occurred, and then read the
2952                  * corresponding Host Channel-n Interrupt (HCINTn) register to
2953                  * determine the exact cause of the interrupt. The application
2954                  * must clear the appropriate status bit in the HCINTn register
2955                  * to clear this bit.
2956                  */
2957                 union cvmx_usbcx_haint usbc_haint;
2958                 usbc_haint.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HAINT(usb->index));
2959                 while (usbc_haint.u32) {
2960                         int channel;
2961
2962                         channel = __fls(usbc_haint.u32);
2963                         __cvmx_usb_poll_channel(usb, channel);
2964                         usbc_haint.u32 ^= 1<<channel;
2965                 }
2966         }
2967
2968         __cvmx_usb_schedule(usb, usbc_gintsts.s.sof);
2969
2970         return 0;
2971 }
2972
2973 /* convert between an HCD pointer and the corresponding struct octeon_hcd */
2974 static inline struct octeon_hcd *hcd_to_octeon(struct usb_hcd *hcd)
2975 {
2976         return (struct octeon_hcd *)(hcd->hcd_priv);
2977 }
2978
2979 static irqreturn_t octeon_usb_irq(struct usb_hcd *hcd)
2980 {
2981         struct octeon_hcd *priv = hcd_to_octeon(hcd);
2982         unsigned long flags;
2983
2984         spin_lock_irqsave(&priv->lock, flags);
2985         cvmx_usb_poll(&priv->usb);
2986         spin_unlock_irqrestore(&priv->lock, flags);
2987         return IRQ_HANDLED;
2988 }
2989
2990 static int octeon_usb_start(struct usb_hcd *hcd)
2991 {
2992         hcd->state = HC_STATE_RUNNING;
2993         return 0;
2994 }
2995
2996 static void octeon_usb_stop(struct usb_hcd *hcd)
2997 {
2998         hcd->state = HC_STATE_HALT;
2999 }
3000
3001 static int octeon_usb_get_frame_number(struct usb_hcd *hcd)
3002 {
3003         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3004
3005         return cvmx_usb_get_frame_number(&priv->usb);
3006 }
3007
3008 static int octeon_usb_urb_enqueue(struct usb_hcd *hcd,
3009                                   struct urb *urb,
3010                                   gfp_t mem_flags)
3011 {
3012         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3013         struct device *dev = hcd->self.controller;
3014         struct cvmx_usb_transaction *transaction = NULL;
3015         struct cvmx_usb_pipe *pipe;
3016         unsigned long flags;
3017         struct cvmx_usb_iso_packet *iso_packet;
3018         struct usb_host_endpoint *ep = urb->ep;
3019
3020         urb->status = 0;
3021         INIT_LIST_HEAD(&urb->urb_list); /* not enqueued on dequeue_list */
3022         spin_lock_irqsave(&priv->lock, flags);
3023
3024         if (!ep->hcpriv) {
3025                 enum cvmx_usb_transfer transfer_type;
3026                 enum cvmx_usb_speed speed;
3027                 int split_device = 0;
3028                 int split_port = 0;
3029                 switch (usb_pipetype(urb->pipe)) {
3030                 case PIPE_ISOCHRONOUS:
3031                         transfer_type = CVMX_USB_TRANSFER_ISOCHRONOUS;
3032                         break;
3033                 case PIPE_INTERRUPT:
3034                         transfer_type = CVMX_USB_TRANSFER_INTERRUPT;
3035                         break;
3036                 case PIPE_CONTROL:
3037                         transfer_type = CVMX_USB_TRANSFER_CONTROL;
3038                         break;
3039                 default:
3040                         transfer_type = CVMX_USB_TRANSFER_BULK;
3041                         break;
3042                 }
3043                 switch (urb->dev->speed) {
3044                 case USB_SPEED_LOW:
3045                         speed = CVMX_USB_SPEED_LOW;
3046                         break;
3047                 case USB_SPEED_FULL:
3048                         speed = CVMX_USB_SPEED_FULL;
3049                         break;
3050                 default:
3051                         speed = CVMX_USB_SPEED_HIGH;
3052                         break;
3053                 }
3054                 /*
3055                  * For slow devices on high speed ports we need to find the hub
3056                  * that does the speed translation so we know where to send the
3057                  * split transactions.
3058                  */
3059                 if (speed != CVMX_USB_SPEED_HIGH) {
3060                         /*
3061                          * Start at this device and work our way up the usb
3062                          * tree.
3063                          */
3064                         struct usb_device *dev = urb->dev;
3065                         while (dev->parent) {
3066                                 /*
3067                                  * If our parent is high speed then he'll
3068                                  * receive the splits.
3069                                  */
3070                                 if (dev->parent->speed == USB_SPEED_HIGH) {
3071                                         split_device = dev->parent->devnum;
3072                                         split_port = dev->portnum;
3073                                         break;
3074                                 }
3075                                 /*
3076                                  * Move up the tree one level. If we make it all
3077                                  * the way up the tree, then the port must not
3078                                  * be in high speed mode and we don't need a
3079                                  * split.
3080                                  */
3081                                 dev = dev->parent;
3082                         }
3083                 }
3084                 pipe = cvmx_usb_open_pipe(&priv->usb, usb_pipedevice(urb->pipe),
3085                                           usb_pipeendpoint(urb->pipe), speed,
3086                                           le16_to_cpu(ep->desc.wMaxPacketSize) & 0x7ff,
3087                                           transfer_type,
3088                                           usb_pipein(urb->pipe) ?
3089                                                 CVMX_USB_DIRECTION_IN :
3090                                                 CVMX_USB_DIRECTION_OUT,
3091                                           urb->interval,
3092                                           (le16_to_cpu(ep->desc.wMaxPacketSize) >> 11) & 0x3,
3093                                           split_device, split_port);
3094                 if (!pipe) {
3095                         spin_unlock_irqrestore(&priv->lock, flags);
3096                         dev_dbg(dev, "Failed to create pipe\n");
3097                         return -ENOMEM;
3098                 }
3099                 ep->hcpriv = pipe;
3100         } else {
3101                 pipe = ep->hcpriv;
3102         }
3103
3104         switch (usb_pipetype(urb->pipe)) {
3105         case PIPE_ISOCHRONOUS:
3106                 dev_dbg(dev, "Submit isochronous to %d.%d\n",
3107                         usb_pipedevice(urb->pipe), usb_pipeendpoint(urb->pipe));
3108                 /*
3109                  * Allocate a structure to use for our private list of
3110                  * isochronous packets.
3111                  */
3112                 iso_packet = kmalloc(urb->number_of_packets *
3113                                      sizeof(struct cvmx_usb_iso_packet),
3114                                      GFP_ATOMIC);
3115                 if (iso_packet) {
3116                         int i;
3117                         /* Fill the list with the data from the URB */
3118                         for (i = 0; i < urb->number_of_packets; i++) {
3119                                 iso_packet[i].offset = urb->iso_frame_desc[i].offset;
3120                                 iso_packet[i].length = urb->iso_frame_desc[i].length;
3121                                 iso_packet[i].status = CVMX_USB_COMPLETE_ERROR;
3122                         }
3123                         /*
3124                          * Store a pointer to the list in the URB setup_packet
3125                          * field. We know this currently isn't being used and
3126                          * this saves us a bunch of logic.
3127                          */
3128                         urb->setup_packet = (char *)iso_packet;
3129                         transaction = cvmx_usb_submit_isochronous(&priv->usb,
3130                                                                   pipe, urb);
3131                         /*
3132                          * If submit failed we need to free our private packet
3133                          * list.
3134                          */
3135                         if (!transaction) {
3136                                 urb->setup_packet = NULL;
3137                                 kfree(iso_packet);
3138                         }
3139                 }
3140                 break;
3141         case PIPE_INTERRUPT:
3142                 dev_dbg(dev, "Submit interrupt to %d.%d\n",
3143                         usb_pipedevice(urb->pipe), usb_pipeendpoint(urb->pipe));
3144                 transaction = cvmx_usb_submit_interrupt(&priv->usb, pipe, urb);
3145                 break;
3146         case PIPE_CONTROL:
3147                 dev_dbg(dev, "Submit control to %d.%d\n",
3148                         usb_pipedevice(urb->pipe), usb_pipeendpoint(urb->pipe));
3149                 transaction = cvmx_usb_submit_control(&priv->usb, pipe, urb);
3150                 break;
3151         case PIPE_BULK:
3152                 dev_dbg(dev, "Submit bulk to %d.%d\n",
3153                         usb_pipedevice(urb->pipe), usb_pipeendpoint(urb->pipe));
3154                 transaction = cvmx_usb_submit_bulk(&priv->usb, pipe, urb);
3155                 break;
3156         }
3157         if (!transaction) {
3158                 spin_unlock_irqrestore(&priv->lock, flags);
3159                 dev_dbg(dev, "Failed to submit\n");
3160                 return -ENOMEM;
3161         }
3162         urb->hcpriv = transaction;
3163         spin_unlock_irqrestore(&priv->lock, flags);
3164         return 0;
3165 }
3166
3167 static void octeon_usb_urb_dequeue_work(unsigned long arg)
3168 {
3169         struct urb *urb;
3170         struct urb *next;
3171         unsigned long flags;
3172         struct octeon_hcd *priv = (struct octeon_hcd *)arg;
3173
3174         spin_lock_irqsave(&priv->lock, flags);
3175
3176         list_for_each_entry_safe(urb, next, &priv->dequeue_list, urb_list) {
3177                 list_del_init(&urb->urb_list);
3178                 cvmx_usb_cancel(&priv->usb, urb->ep->hcpriv, urb->hcpriv);
3179         }
3180
3181         spin_unlock_irqrestore(&priv->lock, flags);
3182 }
3183
3184 static int octeon_usb_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
3185 {
3186         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3187         unsigned long flags;
3188
3189         if (!urb->dev)
3190                 return -EINVAL;
3191
3192         spin_lock_irqsave(&priv->lock, flags);
3193
3194         urb->status = status;
3195         list_add_tail(&urb->urb_list, &priv->dequeue_list);
3196
3197         spin_unlock_irqrestore(&priv->lock, flags);
3198
3199         tasklet_schedule(&priv->dequeue_tasklet);
3200
3201         return 0;
3202 }
3203
3204 static void octeon_usb_endpoint_disable(struct usb_hcd *hcd, struct usb_host_endpoint *ep)
3205 {
3206         struct device *dev = hcd->self.controller;
3207
3208         if (ep->hcpriv) {
3209                 struct octeon_hcd *priv = hcd_to_octeon(hcd);
3210                 struct cvmx_usb_pipe *pipe = ep->hcpriv;
3211                 unsigned long flags;
3212                 spin_lock_irqsave(&priv->lock, flags);
3213                 cvmx_usb_cancel_all(&priv->usb, pipe);
3214                 if (cvmx_usb_close_pipe(&priv->usb, pipe))
3215                         dev_dbg(dev, "Closing pipe %p failed\n", pipe);
3216                 spin_unlock_irqrestore(&priv->lock, flags);
3217                 ep->hcpriv = NULL;
3218         }
3219 }
3220
3221 static int octeon_usb_hub_status_data(struct usb_hcd *hcd, char *buf)
3222 {
3223         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3224         struct cvmx_usb_port_status port_status;
3225         unsigned long flags;
3226
3227         spin_lock_irqsave(&priv->lock, flags);
3228         port_status = cvmx_usb_get_status(&priv->usb);
3229         spin_unlock_irqrestore(&priv->lock, flags);
3230         buf[0] = 0;
3231         buf[0] = port_status.connect_change << 1;
3232
3233         return (buf[0] != 0);
3234 }
3235
3236 static int octeon_usb_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, u16 wIndex, char *buf, u16 wLength)
3237 {
3238         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3239         struct device *dev = hcd->self.controller;
3240         struct cvmx_usb_port_status usb_port_status;
3241         int port_status;
3242         struct usb_hub_descriptor *desc;
3243         unsigned long flags;
3244
3245         switch (typeReq) {
3246         case ClearHubFeature:
3247                 dev_dbg(dev, "ClearHubFeature\n");
3248                 switch (wValue) {
3249                 case C_HUB_LOCAL_POWER:
3250                 case C_HUB_OVER_CURRENT:
3251                         /* Nothing required here */
3252                         break;
3253                 default:
3254                         return -EINVAL;
3255                 }
3256                 break;
3257         case ClearPortFeature:
3258                 dev_dbg(dev, "ClearPortFeature\n");
3259                 if (wIndex != 1) {
3260                         dev_dbg(dev, " INVALID\n");
3261                         return -EINVAL;
3262                 }
3263
3264                 switch (wValue) {
3265                 case USB_PORT_FEAT_ENABLE:
3266                         dev_dbg(dev, " ENABLE\n");
3267                         spin_lock_irqsave(&priv->lock, flags);
3268                         cvmx_usb_disable(&priv->usb);
3269                         spin_unlock_irqrestore(&priv->lock, flags);
3270                         break;
3271                 case USB_PORT_FEAT_SUSPEND:
3272                         dev_dbg(dev, " SUSPEND\n");
3273                         /* Not supported on Octeon */
3274                         break;
3275                 case USB_PORT_FEAT_POWER:
3276                         dev_dbg(dev, " POWER\n");
3277                         /* Not supported on Octeon */
3278                         break;
3279                 case USB_PORT_FEAT_INDICATOR:
3280                         dev_dbg(dev, " INDICATOR\n");
3281                         /* Port inidicator not supported */
3282                         break;
3283                 case USB_PORT_FEAT_C_CONNECTION:
3284                         dev_dbg(dev, " C_CONNECTION\n");
3285                         /* Clears drivers internal connect status change flag */
3286                         spin_lock_irqsave(&priv->lock, flags);
3287                         priv->usb.port_status = cvmx_usb_get_status(&priv->usb);
3288                         spin_unlock_irqrestore(&priv->lock, flags);
3289                         break;
3290                 case USB_PORT_FEAT_C_RESET:
3291                         dev_dbg(dev, " C_RESET\n");
3292                         /*
3293                          * Clears the driver's internal Port Reset Change flag.
3294                          */
3295                         spin_lock_irqsave(&priv->lock, flags);
3296                         priv->usb.port_status = cvmx_usb_get_status(&priv->usb);
3297                         spin_unlock_irqrestore(&priv->lock, flags);
3298                         break;
3299                 case USB_PORT_FEAT_C_ENABLE:
3300                         dev_dbg(dev, " C_ENABLE\n");
3301                         /*
3302                          * Clears the driver's internal Port Enable/Disable
3303                          * Change flag.
3304                          */
3305                         spin_lock_irqsave(&priv->lock, flags);
3306                         priv->usb.port_status = cvmx_usb_get_status(&priv->usb);
3307                         spin_unlock_irqrestore(&priv->lock, flags);
3308                         break;
3309                 case USB_PORT_FEAT_C_SUSPEND:
3310                         dev_dbg(dev, " C_SUSPEND\n");
3311                         /*
3312                          * Clears the driver's internal Port Suspend Change
3313                          * flag, which is set when resume signaling on the host
3314                          * port is complete.
3315                          */
3316                         break;
3317                 case USB_PORT_FEAT_C_OVER_CURRENT:
3318                         dev_dbg(dev, " C_OVER_CURRENT\n");
3319                         /* Clears the driver's overcurrent Change flag */
3320                         spin_lock_irqsave(&priv->lock, flags);
3321                         priv->usb.port_status = cvmx_usb_get_status(&priv->usb);
3322                         spin_unlock_irqrestore(&priv->lock, flags);
3323                         break;
3324                 default:
3325                         dev_dbg(dev, " UNKNOWN\n");
3326                         return -EINVAL;
3327                 }
3328                 break;
3329         case GetHubDescriptor:
3330                 dev_dbg(dev, "GetHubDescriptor\n");
3331                 desc = (struct usb_hub_descriptor *)buf;
3332                 desc->bDescLength = 9;
3333                 desc->bDescriptorType = 0x29;
3334                 desc->bNbrPorts = 1;
3335                 desc->wHubCharacteristics = 0x08;
3336                 desc->bPwrOn2PwrGood = 1;
3337                 desc->bHubContrCurrent = 0;
3338                 desc->u.hs.DeviceRemovable[0] = 0;
3339                 desc->u.hs.DeviceRemovable[1] = 0xff;
3340                 break;
3341         case GetHubStatus:
3342                 dev_dbg(dev, "GetHubStatus\n");
3343                 *(__le32 *) buf = 0;
3344                 break;
3345         case GetPortStatus:
3346                 dev_dbg(dev, "GetPortStatus\n");
3347                 if (wIndex != 1) {
3348                         dev_dbg(dev, " INVALID\n");
3349                         return -EINVAL;
3350                 }
3351
3352                 spin_lock_irqsave(&priv->lock, flags);
3353                 usb_port_status = cvmx_usb_get_status(&priv->usb);
3354                 spin_unlock_irqrestore(&priv->lock, flags);
3355                 port_status = 0;
3356
3357                 if (usb_port_status.connect_change) {
3358                         port_status |= (1 << USB_PORT_FEAT_C_CONNECTION);
3359                         dev_dbg(dev, " C_CONNECTION\n");
3360                 }
3361
3362                 if (usb_port_status.port_enabled) {
3363                         port_status |= (1 << USB_PORT_FEAT_C_ENABLE);
3364                         dev_dbg(dev, " C_ENABLE\n");
3365                 }
3366
3367                 if (usb_port_status.connected) {
3368                         port_status |= (1 << USB_PORT_FEAT_CONNECTION);
3369                         dev_dbg(dev, " CONNECTION\n");
3370                 }
3371
3372                 if (usb_port_status.port_enabled) {
3373                         port_status |= (1 << USB_PORT_FEAT_ENABLE);
3374                         dev_dbg(dev, " ENABLE\n");
3375                 }
3376
3377                 if (usb_port_status.port_over_current) {
3378                         port_status |= (1 << USB_PORT_FEAT_OVER_CURRENT);
3379                         dev_dbg(dev, " OVER_CURRENT\n");
3380                 }
3381
3382                 if (usb_port_status.port_powered) {
3383                         port_status |= (1 << USB_PORT_FEAT_POWER);
3384                         dev_dbg(dev, " POWER\n");
3385                 }
3386
3387                 if (usb_port_status.port_speed == CVMX_USB_SPEED_HIGH) {
3388                         port_status |= USB_PORT_STAT_HIGH_SPEED;
3389                         dev_dbg(dev, " HIGHSPEED\n");
3390                 } else if (usb_port_status.port_speed == CVMX_USB_SPEED_LOW) {
3391                         port_status |= (1 << USB_PORT_FEAT_LOWSPEED);
3392                         dev_dbg(dev, " LOWSPEED\n");
3393                 }
3394
3395                 *((__le32 *) buf) = cpu_to_le32(port_status);
3396                 break;
3397         case SetHubFeature:
3398                 dev_dbg(dev, "SetHubFeature\n");
3399                 /* No HUB features supported */
3400                 break;
3401         case SetPortFeature:
3402                 dev_dbg(dev, "SetPortFeature\n");
3403                 if (wIndex != 1) {
3404                         dev_dbg(dev, " INVALID\n");
3405                         return -EINVAL;
3406                 }
3407
3408                 switch (wValue) {
3409                 case USB_PORT_FEAT_SUSPEND:
3410                         dev_dbg(dev, " SUSPEND\n");
3411                         return -EINVAL;
3412                 case USB_PORT_FEAT_POWER:
3413                         dev_dbg(dev, " POWER\n");
3414                         return -EINVAL;
3415                 case USB_PORT_FEAT_RESET:
3416                         dev_dbg(dev, " RESET\n");
3417                         spin_lock_irqsave(&priv->lock, flags);
3418                         cvmx_usb_disable(&priv->usb);
3419                         if (cvmx_usb_enable(&priv->usb))
3420                                 dev_dbg(dev, "Failed to enable the port\n");
3421                         spin_unlock_irqrestore(&priv->lock, flags);
3422                         return 0;
3423                 case USB_PORT_FEAT_INDICATOR:
3424                         dev_dbg(dev, " INDICATOR\n");
3425                         /* Not supported */
3426                         break;
3427                 default:
3428                         dev_dbg(dev, " UNKNOWN\n");
3429                         return -EINVAL;
3430                 }
3431                 break;
3432         default:
3433                 dev_dbg(dev, "Unknown root hub request\n");
3434                 return -EINVAL;
3435         }
3436         return 0;
3437 }
3438
3439
3440 static const struct hc_driver octeon_hc_driver = {
3441         .description            = "Octeon USB",
3442         .product_desc           = "Octeon Host Controller",
3443         .hcd_priv_size          = sizeof(struct octeon_hcd),
3444         .irq                    = octeon_usb_irq,
3445         .flags                  = HCD_MEMORY | HCD_USB2,
3446         .start                  = octeon_usb_start,
3447         .stop                   = octeon_usb_stop,
3448         .urb_enqueue            = octeon_usb_urb_enqueue,
3449         .urb_dequeue            = octeon_usb_urb_dequeue,
3450         .endpoint_disable       = octeon_usb_endpoint_disable,
3451         .get_frame_number       = octeon_usb_get_frame_number,
3452         .hub_status_data        = octeon_usb_hub_status_data,
3453         .hub_control            = octeon_usb_hub_control,
3454 };
3455
3456
3457 static int octeon_usb_driver_probe(struct device *dev)
3458 {
3459         int status;
3460         int usb_num = to_platform_device(dev)->id;
3461         int irq = platform_get_irq(to_platform_device(dev), 0);
3462         struct octeon_hcd *priv;
3463         struct usb_hcd *hcd;
3464         unsigned long flags;
3465
3466         /*
3467          * Set the DMA mask to 64bits so we get buffers already translated for
3468          * DMA.
3469          */
3470         dev->coherent_dma_mask = ~0;
3471         dev->dma_mask = &dev->coherent_dma_mask;
3472
3473         hcd = usb_create_hcd(&octeon_hc_driver, dev, dev_name(dev));
3474         if (!hcd) {
3475                 dev_dbg(dev, "Failed to allocate memory for HCD\n");
3476                 return -1;
3477         }
3478         hcd->uses_new_polling = 1;
3479         priv = (struct octeon_hcd *)hcd->hcd_priv;
3480
3481         spin_lock_init(&priv->lock);
3482
3483         tasklet_init(&priv->dequeue_tasklet, octeon_usb_urb_dequeue_work, (unsigned long)priv);
3484         INIT_LIST_HEAD(&priv->dequeue_list);
3485
3486         status = cvmx_usb_initialize(&priv->usb, usb_num);
3487         if (status) {
3488                 dev_dbg(dev, "USB initialization failed with %d\n", status);
3489                 kfree(hcd);
3490                 return -1;
3491         }
3492
3493         /* This delay is needed for CN3010, but I don't know why... */
3494         mdelay(10);
3495
3496         spin_lock_irqsave(&priv->lock, flags);
3497         cvmx_usb_poll(&priv->usb);
3498         spin_unlock_irqrestore(&priv->lock, flags);
3499
3500         status = usb_add_hcd(hcd, irq, IRQF_SHARED);
3501         if (status) {
3502                 dev_dbg(dev, "USB add HCD failed with %d\n", status);
3503                 kfree(hcd);
3504                 return -1;
3505         }
3506
3507         dev_dbg(dev, "Registered HCD for port %d on irq %d\n", usb_num, irq);
3508
3509         return 0;
3510 }
3511
3512 static int octeon_usb_driver_remove(struct device *dev)
3513 {
3514         int status;
3515         struct usb_hcd *hcd = dev_get_drvdata(dev);
3516         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3517         unsigned long flags;
3518
3519         usb_remove_hcd(hcd);
3520         tasklet_kill(&priv->dequeue_tasklet);
3521         spin_lock_irqsave(&priv->lock, flags);
3522         status = cvmx_usb_shutdown(&priv->usb);
3523         spin_unlock_irqrestore(&priv->lock, flags);
3524         if (status)
3525                 dev_dbg(dev, "USB shutdown failed with %d\n", status);
3526
3527         kfree(hcd);
3528
3529         return 0;
3530 }
3531
3532 static struct device_driver octeon_usb_driver = {
3533         .name   = "OcteonUSB",
3534         .bus    = &platform_bus_type,
3535         .probe  = octeon_usb_driver_probe,
3536         .remove = octeon_usb_driver_remove,
3537 };
3538
3539
3540 #define MAX_USB_PORTS   10
3541 static struct platform_device *pdev_glob[MAX_USB_PORTS];
3542 static int octeon_usb_registered;
3543 static int __init octeon_usb_module_init(void)
3544 {
3545         int num_devices = cvmx_usb_get_num_ports();
3546         int device;
3547
3548         if (usb_disabled() || num_devices == 0)
3549                 return -ENODEV;
3550
3551         if (driver_register(&octeon_usb_driver))
3552                 return -ENOMEM;
3553
3554         octeon_usb_registered = 1;
3555
3556         /*
3557          * Only cn52XX and cn56XX have DWC_OTG USB hardware and the
3558          * IOB priority registers.  Under heavy network load USB
3559          * hardware can be starved by the IOB causing a crash.  Give
3560          * it a priority boost if it has been waiting more than 400
3561          * cycles to avoid this situation.
3562          *
3563          * Testing indicates that a cnt_val of 8192 is not sufficient,
3564          * but no failures are seen with 4096.  We choose a value of
3565          * 400 to give a safety factor of 10.
3566          */
3567         if (OCTEON_IS_MODEL(OCTEON_CN52XX) || OCTEON_IS_MODEL(OCTEON_CN56XX)) {
3568                 union cvmx_iob_n2c_l2c_pri_cnt pri_cnt;
3569
3570                 pri_cnt.u64 = 0;
3571                 pri_cnt.s.cnt_enb = 1;
3572                 pri_cnt.s.cnt_val = 400;
3573                 cvmx_write_csr(CVMX_IOB_N2C_L2C_PRI_CNT, pri_cnt.u64);
3574         }
3575
3576         for (device = 0; device < num_devices; device++) {
3577                 struct resource irq_resource;
3578                 struct platform_device *pdev;
3579                 memset(&irq_resource, 0, sizeof(irq_resource));
3580                 irq_resource.start = (device == 0) ? OCTEON_IRQ_USB0 : OCTEON_IRQ_USB1;
3581                 irq_resource.end = irq_resource.start;
3582                 irq_resource.flags = IORESOURCE_IRQ;
3583                 pdev = platform_device_register_simple((char *)octeon_usb_driver.  name, device, &irq_resource, 1);
3584                 if (IS_ERR(pdev)) {
3585                         driver_unregister(&octeon_usb_driver);
3586                         octeon_usb_registered = 0;
3587                         return PTR_ERR(pdev);
3588                 }
3589                 if (device < MAX_USB_PORTS)
3590                         pdev_glob[device] = pdev;
3591
3592         }
3593         return 0;
3594 }
3595
3596 static void __exit octeon_usb_module_cleanup(void)
3597 {
3598         int i;
3599
3600         for (i = 0; i < MAX_USB_PORTS; i++)
3601                 if (pdev_glob[i]) {
3602                         platform_device_unregister(pdev_glob[i]);
3603                         pdev_glob[i] = NULL;
3604                 }
3605         if (octeon_usb_registered)
3606                 driver_unregister(&octeon_usb_driver);
3607 }
3608
3609 MODULE_LICENSE("GPL");
3610 MODULE_AUTHOR("Cavium Networks <support@caviumnetworks.com>");
3611 MODULE_DESCRIPTION("Cavium Networks Octeon USB Host driver.");
3612 module_init(octeon_usb_module_init);
3613 module_exit(octeon_usb_module_cleanup);