Pileus Git - ~andy/linux/blob - drivers/staging/octeon-usb/cvmx-usb.c

   1 /***********************license start***************
   2  * Copyright (c) 2003-2010  Cavium Networks (support@cavium.com). All rights
   3  * reserved.
   4  *
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions are
   8  * met:
   9  *
  10  *   * Redistributions of source code must retain the above copyright
  11  *     notice, this list of conditions and the following disclaimer.
  12  *
  13  *   * Redistributions in binary form must reproduce the above
  14  *     copyright notice, this list of conditions and the following
  15  *     disclaimer in the documentation and/or other materials provided
  16  *     with the distribution.
  17
  18  *   * Neither the name of Cavium Networks nor the names of
  19  *     its contributors may be used to endorse or promote products
  20  *     derived from this software without specific prior written
  21  *     permission.
  22
  23  * This Software, including technical data, may be subject to U.S. export  control
  24  * laws, including the U.S. Export Administration Act and its  associated
  25  * regulations, and may be subject to export or import  regulations in other
  26  * countries.
  27
  28  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
  29  * AND WITH ALL FAULTS AND CAVIUM  NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR
  30  * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
  31  * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR
  32  * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
  33  * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
  34  * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
  35  * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
  36  * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE  RISK ARISING OUT OF USE OR
  37  * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
  38  ***********************license end**************************************/
  39
  40
  41 /**
  42  * @file
  43  *
  44  * "cvmx-usb.c" defines a set of low level USB functions to help
  45  * developers create Octeon USB drivers for various operating
  46  * systems. These functions provide a generic API to the Octeon
  47  * USB blocks, hiding the internal hardware specific
  48  * operations.
  49  */
  50 #include <linux/delay.h>
  51 #include <asm/octeon/cvmx.h>
  52 #include <asm/octeon/octeon.h>
  53 #include <asm/octeon/cvmx-sysinfo.h>
  54 #include "cvmx-usbnx-defs.h"
  55 #include "cvmx-usbcx-defs.h"
  56 #include "cvmx-usb.h"
  57 #include <asm/octeon/cvmx-helper.h>
  58 #include <asm/octeon/cvmx-helper-board.h>
  59
  60 #define CVMX_PREFETCH0(address) CVMX_PREFETCH(address, 0)
  61 #define CVMX_PREFETCH128(address) CVMX_PREFETCH(address, 128)
  62 // a normal prefetch
  63 #define CVMX_PREFETCH(address, offset) CVMX_PREFETCH_PREF0(address, offset)
  64 // normal prefetches that use the pref instruction
  65 #define CVMX_PREFETCH_PREFX(X, address, offset) asm volatile ("pref %[type], %[off](%[rbase])" : : [rbase] "d" (address), [off] "I" (offset), [type] "n" (X))
  66 #define CVMX_PREFETCH_PREF0(address, offset) CVMX_PREFETCH_PREFX(0, address, offset)
  67 #define CVMX_CLZ(result, input) asm ("clz %[rd],%[rs]" : [rd] "=d" (result) : [rs] "d" (input))
  68
  69 #define MAX_RETRIES             3               /* Maximum number of times to retry failed transactions */
  70 #define MAX_PIPES               32              /* Maximum number of pipes that can be open at once */
  71 #define MAX_TRANSACTIONS        256             /* Maximum number of outstanding transactions across all pipes */
  72 #define MAX_CHANNELS            8               /* Maximum number of hardware channels supported by the USB block */
  73 #define MAX_USB_ADDRESS         127             /* The highest valid USB device address */
  74 #define MAX_USB_ENDPOINT        15              /* The highest valid USB endpoint number */
  75 #define MAX_USB_HUB_PORT        15              /* The highest valid port number on a hub */
  76 #define MAX_TRANSFER_BYTES      ((1<<19)-1)     /* The low level hardware can transfer a maximum of this number of bytes in each transfer. The field is 19 bits wide */
  77 #define MAX_TRANSFER_PACKETS    ((1<<10)-1)     /* The low level hardware can transfer a maximum of this number of packets in each transfer. The field is 10 bits wide */
  78
  79 /*
  80  * These defines disable the normal read and write csr. This is so I can add
  81  * extra debug stuff to the usb specific version and I won't use the normal
  82  * version by mistake
  83  */
  84 #define cvmx_read_csr use_cvmx_usb_read_csr64_instead_of_cvmx_read_csr
  85 #define cvmx_write_csr use_cvmx_usb_write_csr64_instead_of_cvmx_write_csr
  86
  87 enum cvmx_usb_transaction_flags {
  88         __CVMX_USB_TRANSACTION_FLAGS_IN_USE = 1<<16,
  89 };
  90
  91 enum {
  92         USB_CLOCK_TYPE_REF_12,
  93         USB_CLOCK_TYPE_REF_24,
  94         USB_CLOCK_TYPE_REF_48,
  95         USB_CLOCK_TYPE_CRYSTAL_12,
  96 };
  97
  98 /**
  99  * Logical transactions may take numerous low level
 100  * transactions, especially when splits are concerned. This
 101  * enum represents all of the possible stages a transaction can
 102  * be in. Note that split completes are always even. This is so
 103  * the NAK handler can backup to the previous low level
 104  * transaction with a simple clearing of bit 0.
 105  */
 106 enum cvmx_usb_stage {
 107         CVMX_USB_STAGE_NON_CONTROL,
 108         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE,
 109         CVMX_USB_STAGE_SETUP,
 110         CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE,
 111         CVMX_USB_STAGE_DATA,
 112         CVMX_USB_STAGE_DATA_SPLIT_COMPLETE,
 113         CVMX_USB_STAGE_STATUS,
 114         CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE,
 115 };
 116
 117 /**
 118  * struct cvmx_usb_transaction - describes each pending USB transaction
 119  *                               regardless of type. These are linked together
 120  *                               to form a list of pending requests for a pipe.
 121  *
 122  * @prev:               Transaction before this one in the pipe.
 123  * @next:               Transaction after this one in the pipe.
 124  * @type:               Type of transaction, duplicated of the pipe.
 125  * @flags:              State flags for this transaction.
 126  * @buffer:             User's physical buffer address to read/write.
 127  * @buffer_length:      Size of the user's buffer in bytes.
 128  * @control_header:     For control transactions, physical address of the 8
 129  *                      byte standard header.
 130  * @iso_start_frame:    For ISO transactions, the starting frame number.
 131  * @iso_number_packets: For ISO transactions, the number of packets in the
 132  *                      request.
 133  * @iso_packets:        For ISO transactions, the sub packets in the request.
 134  * @actual_bytes:       Actual bytes transfer for this transaction.
 135  * @stage:              For control transactions, the current stage.
 136  * @callback:           User's callback function when complete.
 137  * @callback_data:      User's data.
 138  */
 139 struct cvmx_usb_transaction {
 140         struct cvmx_usb_transaction *prev;
 141         struct cvmx_usb_transaction *next;
 142         enum cvmx_usb_transfer type;
 143         enum cvmx_usb_transaction_flags flags;
 144         uint64_t buffer;
 145         int buffer_length;
 146         uint64_t control_header;
 147         int iso_start_frame;
 148         int iso_number_packets;
 149         struct cvmx_usb_iso_packet *iso_packets;
 150         int xfersize;
 151         int pktcnt;
 152         int retries;
 153         int actual_bytes;
 154         enum cvmx_usb_stage stage;
 155         cvmx_usb_callback_func_t callback;
 156         void *callback_data;
 157 };
 158
 159 /**
 160  * struct cvmx_usb_pipe - a pipe represents a virtual connection between Octeon
 161  *                        and some USB device. It contains a list of pending
 162  *                        request to the device.
 163  *
 164  * @prev:               Pipe before this one in the list
 165  * @next:               Pipe after this one in the list
 166  * @head:               The first pending transaction
 167  * @tail:               The last pending transaction
 168  * @interval:           For periodic pipes, the interval between packets in
 169  *                      frames
 170  * @next_tx_frame:      The next frame this pipe is allowed to transmit on
 171  * @flags:              State flags for this pipe
 172  * @device_speed:       Speed of device connected to this pipe
 173  * @transfer_type:      Type of transaction supported by this pipe
 174  * @transfer_dir:       IN or OUT. Ignored for Control
 175  * @multi_count:        Max packet in a row for the device
 176  * @max_packet:         The device's maximum packet size in bytes
 177  * @device_addr:        USB device address at other end of pipe
 178  * @endpoint_num:       USB endpoint number at other end of pipe
 179  * @hub_device_addr:    Hub address this device is connected to
 180  * @hub_port:           Hub port this device is connected to
 181  * @pid_toggle:         This toggles between 0/1 on every packet send to track
 182  *                      the data pid needed
 183  * @channel:            Hardware DMA channel for this pipe
 184  * @split_sc_frame:     The low order bits of the frame number the split
 185  *                      complete should be sent on
 186  */
 187 struct cvmx_usb_pipe {
 188         struct cvmx_usb_pipe *prev;
 189         struct cvmx_usb_pipe *next;
 190         struct cvmx_usb_transaction *head;
 191         struct cvmx_usb_transaction *tail;
 192         uint64_t interval;
 193         uint64_t next_tx_frame;
 194         enum cvmx_usb_pipe_flags flags;
 195         enum cvmx_usb_speed device_speed;
 196         enum cvmx_usb_transfer transfer_type;
 197         enum cvmx_usb_direction transfer_dir;
 198         int multi_count;
 199         uint16_t max_packet;
 200         uint8_t device_addr;
 201         uint8_t endpoint_num;
 202         uint8_t hub_device_addr;
 203         uint8_t hub_port;
 204         uint8_t pid_toggle;
 205         uint8_t channel;
 206         int8_t split_sc_frame;
 207 };
 208
 209 /**
 210  * struct cvmx_usb_pipe_list
 211  *
 212  * @head: Head of the list, or NULL if empty.
 213  * @tail: Tail if the list, or NULL if empty.
 214  */
 215 struct cvmx_usb_pipe_list {
 216         struct cvmx_usb_pipe *head;
 217         struct cvmx_usb_pipe *tail;
 218 };
 219
 220 struct cvmx_usb_tx_fifo {
 221         struct {
 222                 int channel;
 223                 int size;
 224                 uint64_t address;
 225         } entry[MAX_CHANNELS+1];
 226         int head;
 227         int tail;
 228 };
 229
 230 /**
 231  * struct cvmx_usb_internal_state - the state of the USB block
 232  *
 233  * init_flags:             Flags passed to initialize.
 234  * index:                  Which USB block this is for.
 235  * idle_hardware_channels: Bit set for every idle hardware channel.
 236  * usbcx_hprt:             Stored port status so we don't need to read a CSR to
 237  *                         determine splits.
 238  * pipe_for_channel:       Map channels to pipes.
 239  * free_transaction_head:  List of free transactions head.
 240  * free_transaction_tail:  List of free transactions tail.
 241  * pipe:                   Storage for pipes.
 242  * transaction:            Storage for transactions.
 243  * callback:               User global callbacks.
 244  * callback_data:          User data for each callback.
 245  * indent:                 Used by debug output to indent functions.
 246  * port_status:            Last port status used for change notification.
 247  * free_pipes:             List of all pipes that are currently closed.
 248  * idle_pipes:             List of open pipes that have no transactions.
 249  * active_pipes:           Active pipes indexed by transfer type.
 250  * frame_number:           Increments every SOF interrupt for time keeping.
 251  * active_split:           Points to the current active split, or NULL.
 252  */
 253 struct cvmx_usb_internal_state {
 254         int init_flags;
 255         int index;
 256         int idle_hardware_channels;
 257         union cvmx_usbcx_hprt usbcx_hprt;
 258         struct cvmx_usb_pipe *pipe_for_channel[MAX_CHANNELS];
 259         struct cvmx_usb_transaction *free_transaction_head;
 260         struct cvmx_usb_transaction *free_transaction_tail;
 261         struct cvmx_usb_pipe pipe[MAX_PIPES];
 262         struct cvmx_usb_transaction transaction[MAX_TRANSACTIONS];
 263         cvmx_usb_callback_func_t callback[__CVMX_USB_CALLBACK_END];
 264         void *callback_data[__CVMX_USB_CALLBACK_END];
 265         int indent;
 266         struct cvmx_usb_port_status port_status;
 267         struct cvmx_usb_pipe_list free_pipes;
 268         struct cvmx_usb_pipe_list idle_pipes;
 269         struct cvmx_usb_pipe_list active_pipes[4];
 270         uint64_t frame_number;
 271         struct cvmx_usb_transaction *active_split;
 272         struct cvmx_usb_tx_fifo periodic;
 273         struct cvmx_usb_tx_fifo nonperiodic;
 274 };
 275
 276 /* This macro spins on a field waiting for it to reach a value */
 277 #define CVMX_WAIT_FOR_FIELD32(address, type, field, op, value, timeout_usec)\
 278         ({int result;                                                       \
 279         do {                                                                \
 280                 uint64_t done = cvmx_get_cycle() + (uint64_t)timeout_usec * \
 281                         octeon_get_clock_rate() / 1000000;                  \
 282                 type c;                                                     \
 283                 while (1) {                                                 \
 284                         c.u32 = __cvmx_usb_read_csr32(usb, address);        \
 285                         if (c.s.field op (value)) {                         \
 286                                 result = 0;                                 \
 287                                 break;                                      \
 288                         } else if (cvmx_get_cycle() > done) {               \
 289                                 result = -1;                                \
 290                                 break;                                      \
 291                         } else                                              \
 292                                 cvmx_wait(100);                             \
 293                 }                                                           \
 294         } while (0);                                                        \
 295         result; })
 296
 297 /*
 298  * This macro logically sets a single field in a CSR. It does the sequence
 299  * read, modify, and write
 300  */
 301 #define USB_SET_FIELD32(address, type, field, value)            \
 302         do {                                                    \
 303                 type c;                                         \
 304                 c.u32 = __cvmx_usb_read_csr32(usb, address);    \
 305                 c.s.field = value;                              \
 306                 __cvmx_usb_write_csr32(usb, address, c.u32);    \
 307         } while (0)
 308
 309 /* Returns the IO address to push/pop stuff data from the FIFOs */
 310 #define USB_FIFO_ADDRESS(channel, usb_index) (CVMX_USBCX_GOTGCTL(usb_index) + ((channel)+1)*0x1000)
 311
 312 static int octeon_usb_get_clock_type(void)
 313 {
 314         switch (cvmx_sysinfo_get()->board_type) {
 315         case CVMX_BOARD_TYPE_BBGW_REF:
 316         case CVMX_BOARD_TYPE_LANAI2_A:
 317         case CVMX_BOARD_TYPE_LANAI2_U:
 318         case CVMX_BOARD_TYPE_LANAI2_G:
 319         case CVMX_BOARD_TYPE_UBNT_E100:
 320                 return USB_CLOCK_TYPE_CRYSTAL_12;
 321         }
 322         return USB_CLOCK_TYPE_REF_48;
 323 }
 324
 325 /**
 326  * Read a USB 32bit CSR. It performs the necessary address swizzle
 327  * for 32bit CSRs and logs the value in a readable format if
 328  * debugging is on.
 329  *
 330  * @usb:     USB block this access is for
 331  * @address: 64bit address to read
 332  *
 333  * Returns: Result of the read
 334  */
 335 static inline uint32_t __cvmx_usb_read_csr32(struct cvmx_usb_internal_state *usb,
 336                                              uint64_t address)
 337 {
 338         uint32_t result = cvmx_read64_uint32(address ^ 4);
 339         return result;
 340 }
 341
 342
 343 /**
 344  * Write a USB 32bit CSR. It performs the necessary address
 345  * swizzle for 32bit CSRs and logs the value in a readable format
 346  * if debugging is on.
 347  *
 348  * @usb:     USB block this access is for
 349  * @address: 64bit address to write
 350  * @value:   Value to write
 351  */
 352 static inline void __cvmx_usb_write_csr32(struct cvmx_usb_internal_state *usb,
 353                                           uint64_t address, uint32_t value)
 354 {
 355         cvmx_write64_uint32(address ^ 4, value);
 356         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
 357 }
 358
 359
 360 /**
 361  * Read a USB 64bit CSR. It logs the value in a readable format if
 362  * debugging is on.
 363  *
 364  * @usb:     USB block this access is for
 365  * @address: 64bit address to read
 366  *
 367  * Returns: Result of the read
 368  */
 369 static inline uint64_t __cvmx_usb_read_csr64(struct cvmx_usb_internal_state *usb,
 370                                              uint64_t address)
 371 {
 372         uint64_t result = cvmx_read64_uint64(address);
 373         return result;
 374 }
 375
 376
 377 /**
 378  * Write a USB 64bit CSR. It logs the value in a readable format
 379  * if debugging is on.
 380  *
 381  * @usb:     USB block this access is for
 382  * @address: 64bit address to write
 383  * @value:   Value to write
 384  */
 385 static inline void __cvmx_usb_write_csr64(struct cvmx_usb_internal_state *usb,
 386                                           uint64_t address, uint64_t value)
 387 {
 388         cvmx_write64_uint64(address, value);
 389 }
 390
 391 /**
 392  * Return non zero if this pipe connects to a non HIGH speed
 393  * device through a high speed hub.
 394  *
 395  * @usb:    USB block this access is for
 396  * @pipe:   Pipe to check
 397  *
 398  * Returns: Non zero if we need to do split transactions
 399  */
 400 static inline int __cvmx_usb_pipe_needs_split(struct cvmx_usb_internal_state *usb, struct cvmx_usb_pipe *pipe)
 401 {
 402         return ((pipe->device_speed != CVMX_USB_SPEED_HIGH) && (usb->usbcx_hprt.s.prtspd == CVMX_USB_SPEED_HIGH));
 403 }
 404
 405
 406 /**
 407  * Trivial utility function to return the correct PID for a pipe
 408  *
 409  * @pipe:   pipe to check
 410  *
 411  * Returns: PID for pipe
 412  */
 413 static inline int __cvmx_usb_get_data_pid(struct cvmx_usb_pipe *pipe)
 414 {
 415         if (pipe->pid_toggle)
 416                 return 2; /* Data1 */
 417         else
 418                 return 0; /* Data0 */
 419 }
 420
 421
 422 /**
 423  * Return the number of USB ports supported by this Octeon
 424  * chip. If the chip doesn't support USB, or is not supported
 425  * by this API, a zero will be returned. Most Octeon chips
 426  * support one usb port, but some support two ports.
 427  * cvmx_usb_initialize() must be called on independent
 428  * struct cvmx_usb_state.
 429  *
 430  * Returns: Number of port, zero if usb isn't supported
 431  */
 432 int cvmx_usb_get_num_ports(void)
 433 {
 434         int arch_ports = 0;
 435
 436         if (OCTEON_IS_MODEL(OCTEON_CN56XX))
 437                 arch_ports = 1;
 438         else if (OCTEON_IS_MODEL(OCTEON_CN52XX))
 439                 arch_ports = 2;
 440         else if (OCTEON_IS_MODEL(OCTEON_CN50XX))
 441                 arch_ports = 1;
 442         else if (OCTEON_IS_MODEL(OCTEON_CN31XX))
 443                 arch_ports = 1;
 444         else if (OCTEON_IS_MODEL(OCTEON_CN30XX))
 445                 arch_ports = 1;
 446         else
 447                 arch_ports = 0;
 448
 449         return arch_ports;
 450 }
 451
 452
 453 /**
 454  * Allocate a usb transaction for use
 455  *
 456  * @usb:         USB device state populated by
 457  *               cvmx_usb_initialize().
 458  *
 459  * Returns: Transaction or NULL
 460  */
 461 static inline struct cvmx_usb_transaction *__cvmx_usb_alloc_transaction(struct cvmx_usb_internal_state *usb)
 462 {
 463         struct cvmx_usb_transaction *t;
 464         t = usb->free_transaction_head;
 465         if (t) {
 466                 usb->free_transaction_head = t->next;
 467                 if (!usb->free_transaction_head)
 468                         usb->free_transaction_tail = NULL;
 469         }
 470         if (t) {
 471                 memset(t, 0, sizeof(*t));
 472                 t->flags = __CVMX_USB_TRANSACTION_FLAGS_IN_USE;
 473         }
 474         return t;
 475 }
 476
 477
 478 /**
 479  * Free a usb transaction
 480  *
 481  * @usb:         USB device state populated by
 482  *               cvmx_usb_initialize().
 483  * @transaction:
 484  *               Transaction to free
 485  */
 486 static inline void __cvmx_usb_free_transaction(struct cvmx_usb_internal_state *usb,
 487                                                struct cvmx_usb_transaction *transaction)
 488 {
 489         transaction->flags = 0;
 490         transaction->prev = NULL;
 491         transaction->next = NULL;
 492         if (usb->free_transaction_tail)
 493                 usb->free_transaction_tail->next = transaction;
 494         else
 495                 usb->free_transaction_head = transaction;
 496         usb->free_transaction_tail = transaction;
 497 }
 498
 499
 500 /**
 501  * Add a pipe to the tail of a list
 502  * @list:   List to add pipe to
 503  * @pipe:   Pipe to add
 504  */
 505 static inline void __cvmx_usb_append_pipe(struct cvmx_usb_pipe_list *list, struct cvmx_usb_pipe *pipe)
 506 {
 507         pipe->next = NULL;
 508         pipe->prev = list->tail;
 509         if (list->tail)
 510                 list->tail->next = pipe;
 511         else
 512                 list->head = pipe;
 513         list->tail = pipe;
 514 }
 515
 516
 517 /**
 518  * Remove a pipe from a list
 519  * @list:   List to remove pipe from
 520  * @pipe:   Pipe to remove
 521  */
 522 static inline void __cvmx_usb_remove_pipe(struct cvmx_usb_pipe_list *list, struct cvmx_usb_pipe *pipe)
 523 {
 524         if (list->head == pipe) {
 525                 list->head = pipe->next;
 526                 pipe->next = NULL;
 527                 if (list->head)
 528                         list->head->prev = NULL;
 529                 else
 530                         list->tail = NULL;
 531         } else if (list->tail == pipe) {
 532                 list->tail = pipe->prev;
 533                 list->tail->next = NULL;
 534                 pipe->prev = NULL;
 535         } else {
 536                 pipe->prev->next = pipe->next;
 537                 pipe->next->prev = pipe->prev;
 538                 pipe->prev = NULL;
 539                 pipe->next = NULL;
 540         }
 541 }
 542
 543
 544 /**
 545  * Initialize a USB port for use. This must be called before any
 546  * other access to the Octeon USB port is made. The port starts
 547  * off in the disabled state.
 548  *
 549  * @state:       Pointer to an empty struct cvmx_usb_state
 550  *               that will be populated by the initialize call.
 551  *               This structure is then passed to all other USB
 552  *               functions.
 553  * @usb_port_number:
 554  *               Which Octeon USB port to initialize.
 555  *
 556  * Returns: 0 or a negative error code.
 557  */
 558 int cvmx_usb_initialize(struct cvmx_usb_state *state, int usb_port_number)
 559 {
 560         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 561         union cvmx_usbnx_usbp_ctl_status usbn_usbp_ctl_status;
 562         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
 563         enum cvmx_usb_initialize_flags flags = 0;
 564
 565         /* Make sure that state is large enough to store the internal state */
 566         if (sizeof(*state) < sizeof(*usb))
 567                 return -EINVAL;
 568         /* At first allow 0-1 for the usb port number */
 569         if ((usb_port_number < 0) || (usb_port_number > 1))
 570                 return -EINVAL;
 571         /* For all chips except 52XX there is only one port */
 572         if (!OCTEON_IS_MODEL(OCTEON_CN52XX) && (usb_port_number > 0))
 573                 return -EINVAL;
 574         /* Try to determine clock type automatically */
 575         if (octeon_usb_get_clock_type() == USB_CLOCK_TYPE_CRYSTAL_12) {
 576                 /* Only 12 MHZ crystals are supported */
 577                 flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI;
 578         } else {
 579                 flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND;
 580
 581                 switch (octeon_usb_get_clock_type()) {
 582                 case USB_CLOCK_TYPE_REF_12:
 583                         flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ;
 584                         break;
 585                 case USB_CLOCK_TYPE_REF_24:
 586                         flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ;
 587                         break;
 588                 case USB_CLOCK_TYPE_REF_48:
 589                         flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ;
 590                         break;
 591                 default:
 592                         return -EINVAL;
 593                         break;
 594                 }
 595         }
 596
 597         memset(usb, 0, sizeof(*usb));
 598         usb->init_flags = flags;
 599
 600         /* Initialize the USB state structure */
 601         {
 602                 int i;
 603                 usb->index = usb_port_number;
 604
 605                 /* Initialize the transaction double linked list */
 606                 usb->free_transaction_head = NULL;
 607                 usb->free_transaction_tail = NULL;
 608                 for (i = 0; i < MAX_TRANSACTIONS; i++)
 609                         __cvmx_usb_free_transaction(usb, usb->transaction + i);
 610                 for (i = 0; i < MAX_PIPES; i++)
 611                         __cvmx_usb_append_pipe(&usb->free_pipes, usb->pipe + i);
 612         }
 613
 614         /*
 615          * Power On Reset and PHY Initialization
 616          *
 617          * 1. Wait for DCOK to assert (nothing to do)
 618          *
 619          * 2a. Write USBN0/1_CLK_CTL[POR] = 1 and
 620          *     USBN0/1_CLK_CTL[HRST,PRST,HCLK_RST] = 0
 621          */
 622         usbn_clk_ctl.u64 = __cvmx_usb_read_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index));
 623         usbn_clk_ctl.s.por = 1;
 624         usbn_clk_ctl.s.hrst = 0;
 625         usbn_clk_ctl.s.prst = 0;
 626         usbn_clk_ctl.s.hclk_rst = 0;
 627         usbn_clk_ctl.s.enable = 0;
 628         /*
 629          * 2b. Select the USB reference clock/crystal parameters by writing
 630          *     appropriate values to USBN0/1_CLK_CTL[P_C_SEL, P_RTYPE, P_COM_ON]
 631          */
 632         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND) {
 633                 /*
 634                  * The USB port uses 12/24/48MHz 2.5V board clock
 635                  * source at USB_XO. USB_XI should be tied to GND.
 636                  * Most Octeon evaluation boards require this setting
 637                  */
 638                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX)) {
 639                         usbn_clk_ctl.cn31xx.p_rclk  = 1; /* From CN31XX,CN30XX manual */
 640                         usbn_clk_ctl.cn31xx.p_xenbn = 0;
 641                 } else if (OCTEON_IS_MODEL(OCTEON_CN56XX) || OCTEON_IS_MODEL(OCTEON_CN50XX))
 642                         usbn_clk_ctl.cn56xx.p_rtype = 2; /* From CN56XX,CN50XX manual */
 643                 else
 644                         usbn_clk_ctl.cn52xx.p_rtype = 1; /* From CN52XX manual */
 645
 646                 switch (flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK) {
 647                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:
 648                         usbn_clk_ctl.s.p_c_sel = 0;
 649                         break;
 650                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:
 651                         usbn_clk_ctl.s.p_c_sel = 1;
 652                         break;
 653                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:
 654                         usbn_clk_ctl.s.p_c_sel = 2;
 655                         break;
 656                 }
 657         } else {
 658                 /*
 659                  * The USB port uses a 12MHz crystal as clock source
 660                  * at USB_XO and USB_XI
 661                  */
 662                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX)) {
 663                         usbn_clk_ctl.cn31xx.p_rclk  = 1; /* From CN31XX,CN30XX manual */
 664                         usbn_clk_ctl.cn31xx.p_xenbn = 1;
 665                 } else if (OCTEON_IS_MODEL(OCTEON_CN56XX) || OCTEON_IS_MODEL(OCTEON_CN50XX))
 666                         usbn_clk_ctl.cn56xx.p_rtype = 0; /* From CN56XX,CN50XX manual */
 667                 else
 668                         usbn_clk_ctl.cn52xx.p_rtype = 0; /* From CN52XX manual */
 669
 670                 usbn_clk_ctl.s.p_c_sel = 0;
 671         }
 672         /*
 673          * 2c. Select the HCLK via writing USBN0/1_CLK_CTL[DIVIDE, DIVIDE2] and
 674          *     setting USBN0/1_CLK_CTL[ENABLE] = 1. Divide the core clock down
 675          *     such that USB is as close as possible to 125Mhz
 676          */
 677         {
 678                 int divisor = (octeon_get_clock_rate()+125000000-1)/125000000;
 679                 if (divisor < 4)  /* Lower than 4 doesn't seem to work properly */
 680                         divisor = 4;
 681                 usbn_clk_ctl.s.divide = divisor;
 682                 usbn_clk_ctl.s.divide2 = 0;
 683         }
 684         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 685                                usbn_clk_ctl.u64);
 686         /* 2d. Write USBN0/1_CLK_CTL[HCLK_RST] = 1 */
 687         usbn_clk_ctl.s.hclk_rst = 1;
 688         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 689                                usbn_clk_ctl.u64);
 690         /* 2e.  Wait 64 core-clock cycles for HCLK to stabilize */
 691         cvmx_wait(64);
 692         /*
 693          * 3. Program the power-on reset field in the USBN clock-control
 694          *    register:
 695          *    USBN_CLK_CTL[POR] = 0
 696          */
 697         usbn_clk_ctl.s.por = 0;
 698         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 699                                usbn_clk_ctl.u64);
 700         /* 4. Wait 1 ms for PHY clock to start */
 701         mdelay(1);
 702         /*
 703          * 5. Program the Reset input from automatic test equipment field in the
 704          *    USBP control and status register:
 705          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 1
 706          */
 707         usbn_usbp_ctl_status.u64 = __cvmx_usb_read_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index));
 708         usbn_usbp_ctl_status.s.ate_reset = 1;
 709         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 710                                usbn_usbp_ctl_status.u64);
 711         /* 6. Wait 10 cycles */
 712         cvmx_wait(10);
 713         /*
 714          * 7. Clear ATE_RESET field in the USBN clock-control register:
 715          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 0
 716          */
 717         usbn_usbp_ctl_status.s.ate_reset = 0;
 718         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 719                                usbn_usbp_ctl_status.u64);
 720         /*
 721          * 8. Program the PHY reset field in the USBN clock-control register:
 722          *    USBN_CLK_CTL[PRST] = 1
 723          */
 724         usbn_clk_ctl.s.prst = 1;
 725         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 726                                usbn_clk_ctl.u64);
 727         /*
 728          * 9. Program the USBP control and status register to select host or
 729          *    device mode. USBN_USBP_CTL_STATUS[HST_MODE] = 0 for host, = 1 for
 730          *    device
 731          */
 732         usbn_usbp_ctl_status.s.hst_mode = 0;
 733         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 734                                usbn_usbp_ctl_status.u64);
 735         /* 10. Wait 1 us */
 736         udelay(1);
 737         /*
 738          * 11. Program the hreset_n field in the USBN clock-control register:
 739          *     USBN_CLK_CTL[HRST] = 1
 740          */
 741         usbn_clk_ctl.s.hrst = 1;
 742         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 743                                usbn_clk_ctl.u64);
 744         /* 12. Proceed to USB core initialization */
 745         usbn_clk_ctl.s.enable = 1;
 746         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 747                                usbn_clk_ctl.u64);
 748         udelay(1);
 749
 750         /*
 751          * USB Core Initialization
 752          *
 753          * 1. Read USBC_GHWCFG1, USBC_GHWCFG2, USBC_GHWCFG3, USBC_GHWCFG4 to
 754          *    determine USB core configuration parameters.
 755          *
 756          *    Nothing needed
 757          *
 758          * 2. Program the following fields in the global AHB configuration
 759          *    register (USBC_GAHBCFG)
 760          *    DMA mode, USBC_GAHBCFG[DMAEn]: 1 = DMA mode, 0 = slave mode
 761          *    Burst length, USBC_GAHBCFG[HBSTLEN] = 0
 762          *    Nonperiodic TxFIFO empty level (slave mode only),
 763          *    USBC_GAHBCFG[NPTXFEMPLVL]
 764          *    Periodic TxFIFO empty level (slave mode only),
 765          *    USBC_GAHBCFG[PTXFEMPLVL]
 766          *    Global interrupt mask, USBC_GAHBCFG[GLBLINTRMSK] = 1
 767          */
 768         {
 769                 union cvmx_usbcx_gahbcfg usbcx_gahbcfg;
 770                 /* Due to an errata, CN31XX doesn't support DMA */
 771                 if (OCTEON_IS_MODEL(OCTEON_CN31XX))
 772                         usb->init_flags |= CVMX_USB_INITIALIZE_FLAGS_NO_DMA;
 773                 usbcx_gahbcfg.u32 = 0;
 774                 usbcx_gahbcfg.s.dmaen = !(usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA);
 775                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 776                         usb->idle_hardware_channels = 0x1;  /* Only use one channel with non DMA */
 777                 else if (OCTEON_IS_MODEL(OCTEON_CN5XXX))
 778                         usb->idle_hardware_channels = 0xf7; /* CN5XXX have an errata with channel 3 */
 779                 else
 780                         usb->idle_hardware_channels = 0xff;
 781                 usbcx_gahbcfg.s.hbstlen = 0;
 782                 usbcx_gahbcfg.s.nptxfemplvl = 1;
 783                 usbcx_gahbcfg.s.ptxfemplvl = 1;
 784                 usbcx_gahbcfg.s.glblintrmsk = 1;
 785                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GAHBCFG(usb->index),
 786                                        usbcx_gahbcfg.u32);
 787         }
 788         /*
 789          * 3. Program the following fields in USBC_GUSBCFG register.
 790          *    HS/FS timeout calibration, USBC_GUSBCFG[TOUTCAL] = 0
 791          *    ULPI DDR select, USBC_GUSBCFG[DDRSEL] = 0
 792          *    USB turnaround time, USBC_GUSBCFG[USBTRDTIM] = 0x5
 793          *    PHY low-power clock select, USBC_GUSBCFG[PHYLPWRCLKSEL] = 0
 794          */
 795         {
 796                 union cvmx_usbcx_gusbcfg usbcx_gusbcfg;
 797                 usbcx_gusbcfg.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index));
 798                 usbcx_gusbcfg.s.toutcal = 0;
 799                 usbcx_gusbcfg.s.ddrsel = 0;
 800                 usbcx_gusbcfg.s.usbtrdtim = 0x5;
 801                 usbcx_gusbcfg.s.phylpwrclksel = 0;
 802                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index),
 803                                        usbcx_gusbcfg.u32);
 804         }
 805         /*
 806          * 4. The software must unmask the following bits in the USBC_GINTMSK
 807          *    register.
 808          *    OTG interrupt mask, USBC_GINTMSK[OTGINTMSK] = 1
 809          *    Mode mismatch interrupt mask, USBC_GINTMSK[MODEMISMSK] = 1
 810          */
 811         {
 812                 union cvmx_usbcx_gintmsk usbcx_gintmsk;
 813                 int channel;
 814
 815                 usbcx_gintmsk.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GINTMSK(usb->index));
 816                 usbcx_gintmsk.s.otgintmsk = 1;
 817                 usbcx_gintmsk.s.modemismsk = 1;
 818                 usbcx_gintmsk.s.hchintmsk = 1;
 819                 usbcx_gintmsk.s.sofmsk = 0;
 820                 /* We need RX FIFO interrupts if we don't have DMA */
 821                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 822                         usbcx_gintmsk.s.rxflvlmsk = 1;
 823                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTMSK(usb->index),
 824                                        usbcx_gintmsk.u32);
 825
 826                 /* Disable all channel interrupts. We'll enable them per channel later */
 827                 for (channel = 0; channel < 8; channel++)
 828                         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
 829         }
 830
 831         {
 832                 /*
 833                  * Host Port Initialization
 834                  *
 835                  * 1. Program the host-port interrupt-mask field to unmask,
 836                  *    USBC_GINTMSK[PRTINT] = 1
 837                  */
 838                 USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk,
 839                                 prtintmsk, 1);
 840                 USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk,
 841                                 disconnintmsk, 1);
 842                 /*
 843                  * 2. Program the USBC_HCFG register to select full-speed host
 844                  *    or high-speed host.
 845                  */
 846                 {
 847                         union cvmx_usbcx_hcfg usbcx_hcfg;
 848                         usbcx_hcfg.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCFG(usb->index));
 849                         usbcx_hcfg.s.fslssupp = 0;
 850                         usbcx_hcfg.s.fslspclksel = 0;
 851                         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCFG(usb->index), usbcx_hcfg.u32);
 852                 }
 853                 /*
 854                  * 3. Program the port power bit to drive VBUS on the USB,
 855                  *    USBC_HPRT[PRTPWR] = 1
 856                  */
 857                 USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt, prtpwr, 1);
 858
 859                 /*
 860                  * Steps 4-15 from the manual are done later in the port enable
 861                  */
 862         }
 863
 864         return 0;
 865 }
 866
 867
 868 /**
 869  * Shutdown a USB port after a call to cvmx_usb_initialize().
 870  * The port should be disabled with all pipes closed when this
 871  * function is called.
 872  *
 873  * @state: USB device state populated by
 874  *         cvmx_usb_initialize().
 875  *
 876  * Returns: 0 or a negative error code.
 877  */
 878 int cvmx_usb_shutdown(struct cvmx_usb_state *state)
 879 {
 880         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 881         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
 882
 883         /* Make sure all pipes are closed */
 884         if (usb->idle_pipes.head ||
 885                 usb->active_pipes[CVMX_USB_TRANSFER_ISOCHRONOUS].head ||
 886                 usb->active_pipes[CVMX_USB_TRANSFER_INTERRUPT].head ||
 887                 usb->active_pipes[CVMX_USB_TRANSFER_CONTROL].head ||
 888                 usb->active_pipes[CVMX_USB_TRANSFER_BULK].head)
 889                 return -EBUSY;
 890
 891         /* Disable the clocks and put them in power on reset */
 892         usbn_clk_ctl.u64 = __cvmx_usb_read_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index));
 893         usbn_clk_ctl.s.enable = 1;
 894         usbn_clk_ctl.s.por = 1;
 895         usbn_clk_ctl.s.hclk_rst = 1;
 896         usbn_clk_ctl.s.prst = 0;
 897         usbn_clk_ctl.s.hrst = 0;
 898         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 899                                usbn_clk_ctl.u64);
 900         return 0;
 901 }
 902
 903
 904 /**
 905  * Enable a USB port. After this call succeeds, the USB port is
 906  * online and servicing requests.
 907  *
 908  * @state: USB device state populated by
 909  *         cvmx_usb_initialize().
 910  *
 911  * Returns: 0 or a negative error code.
 912  */
 913 int cvmx_usb_enable(struct cvmx_usb_state *state)
 914 {
 915         union cvmx_usbcx_ghwcfg3 usbcx_ghwcfg3;
 916         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
 917
 918         usb->usbcx_hprt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
 919
 920         /*
 921          * If the port is already enabled the just return. We don't need to do
 922          * anything
 923          */
 924         if (usb->usbcx_hprt.s.prtena)
 925                 return 0;
 926
 927         /* If there is nothing plugged into the port then fail immediately */
 928         if (!usb->usbcx_hprt.s.prtconnsts) {
 929                 return -ETIMEDOUT;
 930         }
 931
 932         /* Program the port reset bit to start the reset process */
 933         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt, prtrst, 1);
 934
 935         /*
 936          * Wait at least 50ms (high speed), or 10ms (full speed) for the reset
 937          * process to complete.
 938          */
 939         mdelay(50);
 940
 941         /* Program the port reset bit to 0, USBC_HPRT[PRTRST] = 0 */
 942         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt, prtrst, 0);
 943
 944         /* Wait for the USBC_HPRT[PRTENA]. */
 945         if (CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt,
 946                                   prtena, ==, 1, 100000))
 947                 return -ETIMEDOUT;
 948
 949         /* Read the port speed field to get the enumerated speed, USBC_HPRT[PRTSPD]. */
 950         usb->usbcx_hprt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
 951         usbcx_ghwcfg3.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GHWCFG3(usb->index));
 952
 953         /*
 954          * 13. Program the USBC_GRXFSIZ register to select the size of the
 955          *     receive FIFO (25%).
 956          */
 957         USB_SET_FIELD32(CVMX_USBCX_GRXFSIZ(usb->index), union cvmx_usbcx_grxfsiz,
 958                         rxfdep, usbcx_ghwcfg3.s.dfifodepth / 4);
 959         /*
 960          * 14. Program the USBC_GNPTXFSIZ register to select the size and the
 961          *     start address of the non- periodic transmit FIFO for nonperiodic
 962          *     transactions (50%).
 963          */
 964         {
 965                 union cvmx_usbcx_gnptxfsiz siz;
 966                 siz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index));
 967                 siz.s.nptxfdep = usbcx_ghwcfg3.s.dfifodepth / 2;
 968                 siz.s.nptxfstaddr = usbcx_ghwcfg3.s.dfifodepth / 4;
 969                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index), siz.u32);
 970         }
 971         /*
 972          * 15. Program the USBC_HPTXFSIZ register to select the size and start
 973          *     address of the periodic transmit FIFO for periodic transactions
 974          *     (25%).
 975          */
 976         {
 977                 union cvmx_usbcx_hptxfsiz siz;
 978                 siz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index));
 979                 siz.s.ptxfsize = usbcx_ghwcfg3.s.dfifodepth / 4;
 980                 siz.s.ptxfstaddr = 3 * usbcx_ghwcfg3.s.dfifodepth / 4;
 981                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index), siz.u32);
 982         }
 983         /* Flush all FIFOs */
 984         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), union cvmx_usbcx_grstctl, txfnum, 0x10);
 985         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), union cvmx_usbcx_grstctl, txfflsh, 1);
 986         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), union cvmx_usbcx_grstctl,
 987                               txfflsh, ==, 0, 100);
 988         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), union cvmx_usbcx_grstctl, rxfflsh, 1);
 989         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), union cvmx_usbcx_grstctl,
 990                               rxfflsh, ==, 0, 100);
 991
 992         return 0;
 993 }
 994
 995
 996 /**
 997  * Disable a USB port. After this call the USB port will not
 998  * generate data transfers and will not generate events.
 999  * Transactions in process will fail and call their
1000  * associated callbacks.
1001  *
1002  * @state: USB device state populated by
1003  *         cvmx_usb_initialize().
1004  *
1005  * Returns: 0 or a negative error code.
1006  */
1007 int cvmx_usb_disable(struct cvmx_usb_state *state)
1008 {
1009         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
1010
1011         /* Disable the port */
1012         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt, prtena, 1);
1013         return 0;
1014 }
1015
1016
1017 /**
1018  * Get the current state of the USB port. Use this call to
1019  * determine if the usb port has anything connected, is enabled,
1020  * or has some sort of error condition. The return value of this
1021  * call has "changed" bits to signal of the value of some fields
1022  * have changed between calls. These "changed" fields are based
1023  * on the last call to cvmx_usb_set_status(). In order to clear
1024  * them, you must update the status through cvmx_usb_set_status().
1025  *
1026  * @state: USB device state populated by
1027  *         cvmx_usb_initialize().
1028  *
1029  * Returns: Port status information
1030  */
1031 struct cvmx_usb_port_status cvmx_usb_get_status(struct cvmx_usb_state *state)
1032 {
1033         union cvmx_usbcx_hprt usbc_hprt;
1034         struct cvmx_usb_port_status result;
1035         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
1036
1037         memset(&result, 0, sizeof(result));
1038
1039         usbc_hprt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
1040         result.port_enabled = usbc_hprt.s.prtena;
1041         result.port_over_current = usbc_hprt.s.prtovrcurract;
1042         result.port_powered = usbc_hprt.s.prtpwr;
1043         result.port_speed = usbc_hprt.s.prtspd;
1044         result.connected = usbc_hprt.s.prtconnsts;
1045         result.connect_change = (result.connected != usb->port_status.connected);
1046
1047         return result;
1048 }
1049
1050
1051 /**
1052  * Set the current state of the USB port. The status is used as
1053  * a reference for the "changed" bits returned by
1054  * cvmx_usb_get_status(). Other than serving as a reference, the
1055  * status passed to this function is not used. No fields can be
1056  * changed through this call.
1057  *
1058  * @state:       USB device state populated by
1059  *               cvmx_usb_initialize().
1060  * @port_status:
1061  *               Port status to set, most like returned by cvmx_usb_get_status()
1062  */
1063 void cvmx_usb_set_status(struct cvmx_usb_state *state, struct cvmx_usb_port_status port_status)
1064 {
1065         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
1066         usb->port_status = port_status;
1067         return;
1068 }
1069
1070
1071 /**
1072  * Convert a USB transaction into a handle
1073  *
1074  * @usb:         USB device state populated by
1075  *               cvmx_usb_initialize().
1076  * @transaction:
1077  *               Transaction to get handle for
1078  *
1079  * Returns: Handle
1080  */
1081 static inline int __cvmx_usb_get_submit_handle(struct cvmx_usb_internal_state *usb,
1082                                                struct cvmx_usb_transaction *transaction)
1083 {
1084         return ((unsigned long)transaction - (unsigned long)usb->transaction) /
1085                         sizeof(*transaction);
1086 }
1087
1088
1089 /**
1090  * Convert a USB pipe into a handle
1091  *
1092  * @usb:         USB device state populated by
1093  *               cvmx_usb_initialize().
1094  * @pipe:        Pipe to get handle for
1095  *
1096  * Returns: Handle
1097  */
1098 static inline int __cvmx_usb_get_pipe_handle(struct cvmx_usb_internal_state *usb,
1099                                              struct cvmx_usb_pipe *pipe)
1100 {
1101         return ((unsigned long)pipe - (unsigned long)usb->pipe) / sizeof(*pipe);
1102 }
1103
1104
1105 /**
1106  * Open a virtual pipe between the host and a USB device. A pipe
1107  * must be opened before data can be transferred between a device
1108  * and Octeon.
1109  *
1110  * @state:           USB device state populated by
1111  *                   cvmx_usb_initialize().
1112  * @flags:           Optional pipe flags defined in
1113  *                   enum cvmx_usb_pipe_flags.
1114  * @device_addr:
1115  *                   USB device address to open the pipe to
1116  *                   (0-127).
1117  * @endpoint_num:
1118  *                   USB endpoint number to open the pipe to
1119  *                   (0-15).
1120  * @device_speed:
1121  *                   The speed of the device the pipe is going
1122  *                   to. This must match the device's speed,
1123  *                   which may be different than the port speed.
1124  * @max_packet:      The maximum packet length the device can
1125  *                   transmit/receive (low speed=0-8, full
1126  *                   speed=0-1023, high speed=0-1024). This value
1127  *                   comes from the standard endpoint descriptor
1128  *                   field wMaxPacketSize bits <10:0>.
1129  * @transfer_type:
1130  *                   The type of transfer this pipe is for.
1131  * @transfer_dir:
1132  *                   The direction the pipe is in. This is not
1133  *                   used for control pipes.
1134  * @interval:        For ISOCHRONOUS and INTERRUPT transfers,
1135  *                   this is how often the transfer is scheduled
1136  *                   for. All other transfers should specify
1137  *                   zero. The units are in frames (8000/sec at
1138  *                   high speed, 1000/sec for full speed).
1139  * @multi_count:
1140  *                   For high speed devices, this is the maximum
1141  *                   allowed number of packet per microframe.
1142  *                   Specify zero for non high speed devices. This
1143  *                   value comes from the standard endpoint descriptor
1144  *                   field wMaxPacketSize bits <12:11>.
1145  * @hub_device_addr:
1146  *                   Hub device address this device is connected
1147  *                   to. Devices connected directly to Octeon
1148  *                   use zero. This is only used when the device
1149  *                   is full/low speed behind a high speed hub.
1150  *                   The address will be of the high speed hub,
1151  *                   not and full speed hubs after it.
1152  * @hub_port:        Which port on the hub the device is
1153  *                   connected. Use zero for devices connected
1154  *                   directly to Octeon. Like hub_device_addr,
1155  *                   this is only used for full/low speed
1156  *                   devices behind a high speed hub.
1157  *
1158  * Returns: A non negative value is a pipe handle. Negative
1159  *          values are error codes.
1160  */
1161 int cvmx_usb_open_pipe(struct cvmx_usb_state *state, enum cvmx_usb_pipe_flags flags,
1162                        int device_addr, int endpoint_num,
1163                        enum cvmx_usb_speed device_speed, int max_packet,
1164                        enum cvmx_usb_transfer transfer_type,
1165                        enum cvmx_usb_direction transfer_dir, int interval,
1166                        int multi_count, int hub_device_addr, int hub_port)
1167 {
1168         struct cvmx_usb_pipe *pipe;
1169         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
1170
1171         if (unlikely((device_addr < 0) || (device_addr > MAX_USB_ADDRESS)))
1172                 return -EINVAL;
1173         if (unlikely((endpoint_num < 0) || (endpoint_num > MAX_USB_ENDPOINT)))
1174                 return -EINVAL;
1175         if (unlikely(device_speed > CVMX_USB_SPEED_LOW))
1176                 return -EINVAL;
1177         if (unlikely((max_packet <= 0) || (max_packet > 1024)))
1178                 return -EINVAL;
1179         if (unlikely(transfer_type > CVMX_USB_TRANSFER_INTERRUPT))
1180                 return -EINVAL;
1181         if (unlikely((transfer_dir != CVMX_USB_DIRECTION_OUT) &&
1182                 (transfer_dir != CVMX_USB_DIRECTION_IN)))
1183                 return -EINVAL;
1184         if (unlikely(interval < 0))
1185                 return -EINVAL;
1186         if (unlikely((transfer_type == CVMX_USB_TRANSFER_CONTROL) && interval))
1187                 return -EINVAL;
1188         if (unlikely(multi_count < 0))
1189                 return -EINVAL;
1190         if (unlikely((device_speed != CVMX_USB_SPEED_HIGH) &&
1191                 (multi_count != 0)))
1192                 return -EINVAL;
1193         if (unlikely((hub_device_addr < 0) || (hub_device_addr > MAX_USB_ADDRESS)))
1194                 return -EINVAL;
1195         if (unlikely((hub_port < 0) || (hub_port > MAX_USB_HUB_PORT)))
1196                 return -EINVAL;
1197
1198         /* Find a free pipe */
1199         pipe = usb->free_pipes.head;
1200         if (!pipe)
1201                 return -ENOMEM;
1202         __cvmx_usb_remove_pipe(&usb->free_pipes, pipe);
1203         pipe->flags = flags | __CVMX_USB_PIPE_FLAGS_OPEN;
1204         if ((device_speed == CVMX_USB_SPEED_HIGH) &&
1205                 (transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1206                 (transfer_type == CVMX_USB_TRANSFER_BULK))
1207                 pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
1208         pipe->device_addr = device_addr;
1209         pipe->endpoint_num = endpoint_num;
1210         pipe->device_speed = device_speed;
1211         pipe->max_packet = max_packet;
1212         pipe->transfer_type = transfer_type;
1213         pipe->transfer_dir = transfer_dir;
1214         /*
1215          * All pipes use interval to rate limit NAK processing. Force an
1216          * interval if one wasn't supplied
1217          */
1218         if (!interval)
1219                 interval = 1;
1220         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1221                 pipe->interval = interval*8;
1222                 /* Force start splits to be schedule on uFrame 0 */
1223                 pipe->next_tx_frame = ((usb->frame_number+7)&~7) + pipe->interval;
1224         } else {
1225                 pipe->interval = interval;
1226                 pipe->next_tx_frame = usb->frame_number + pipe->interval;
1227         }
1228         pipe->multi_count = multi_count;
1229         pipe->hub_device_addr = hub_device_addr;
1230         pipe->hub_port = hub_port;
1231         pipe->pid_toggle = 0;
1232         pipe->split_sc_frame = -1;
1233         __cvmx_usb_append_pipe(&usb->idle_pipes, pipe);
1234
1235         /*
1236          * We don't need to tell the hardware about this pipe yet since
1237          * it doesn't have any submitted requests
1238          */
1239
1240         return __cvmx_usb_get_pipe_handle(usb, pipe);
1241 }
1242
1243
1244 /**
1245  * Poll the RX FIFOs and remove data as needed. This function is only used
1246  * in non DMA mode. It is very important that this function be called quickly
1247  * enough to prevent FIFO overflow.
1248  *
1249  * @usb:        USB device state populated by
1250  *              cvmx_usb_initialize().
1251  */
1252 static void __cvmx_usb_poll_rx_fifo(struct cvmx_usb_internal_state *usb)
1253 {
1254         union cvmx_usbcx_grxstsph rx_status;
1255         int channel;
1256         int bytes;
1257         uint64_t address;
1258         uint32_t *ptr;
1259
1260         rx_status.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GRXSTSPH(usb->index));
1261         /* Only read data if IN data is there */
1262         if (rx_status.s.pktsts != 2)
1263                 return;
1264         /* Check if no data is available */
1265         if (!rx_status.s.bcnt)
1266                 return;
1267
1268         channel = rx_status.s.chnum;
1269         bytes = rx_status.s.bcnt;
1270         if (!bytes)
1271                 return;
1272
1273         /* Get where the DMA engine would have written this data */
1274         address = __cvmx_usb_read_csr64(usb, CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8);
1275         ptr = cvmx_phys_to_ptr(address);
1276         __cvmx_usb_write_csr64(usb, CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8, address + bytes);
1277
1278         /* Loop writing the FIFO data for this packet into memory */
1279         while (bytes > 0) {
1280                 *ptr++ = __cvmx_usb_read_csr32(usb, USB_FIFO_ADDRESS(channel, usb->index));
1281                 bytes -= 4;
1282         }
1283         CVMX_SYNCW;
1284
1285         return;
1286 }
1287
1288
1289 /**
1290  * Fill the TX hardware fifo with data out of the software
1291  * fifos
1292  *
1293  * @usb:            USB device state populated by
1294  *                  cvmx_usb_initialize().
1295  * @fifo:           Software fifo to use
1296  * @available:      Amount of space in the hardware fifo
1297  *
1298  * Returns: Non zero if the hardware fifo was too small and needs
1299  *          to be serviced again.
1300  */
1301 static int __cvmx_usb_fill_tx_hw(struct cvmx_usb_internal_state *usb, struct cvmx_usb_tx_fifo *fifo, int available)
1302 {
1303         /*
1304          * We're done either when there isn't anymore space or the software FIFO
1305          * is empty
1306          */
1307         while (available && (fifo->head != fifo->tail)) {
1308                 int i = fifo->tail;
1309                 const uint32_t *ptr = cvmx_phys_to_ptr(fifo->entry[i].address);
1310                 uint64_t csr_address = USB_FIFO_ADDRESS(fifo->entry[i].channel, usb->index) ^ 4;
1311                 int words = available;
1312
1313                 /* Limit the amount of data to waht the SW fifo has */
1314                 if (fifo->entry[i].size <= available) {
1315                         words = fifo->entry[i].size;
1316                         fifo->tail++;
1317                         if (fifo->tail > MAX_CHANNELS)
1318                                 fifo->tail = 0;
1319                 }
1320
1321                 /* Update the next locations and counts */
1322                 available -= words;
1323                 fifo->entry[i].address += words * 4;
1324                 fifo->entry[i].size -= words;
1325
1326                 /*
1327                  * Write the HW fifo data. The read every three writes is due
1328                  * to an errata on CN3XXX chips
1329                  */
1330                 while (words > 3) {
1331                         cvmx_write64_uint32(csr_address, *ptr++);
1332                         cvmx_write64_uint32(csr_address, *ptr++);
1333                         cvmx_write64_uint32(csr_address, *ptr++);
1334                         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1335                         words -= 3;
1336                 }
1337                 cvmx_write64_uint32(csr_address, *ptr++);
1338                 if (--words) {
1339                         cvmx_write64_uint32(csr_address, *ptr++);
1340                         if (--words)
1341                                 cvmx_write64_uint32(csr_address, *ptr++);
1342                 }
1343                 cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1344         }
1345         return fifo->head != fifo->tail;
1346 }
1347
1348
1349 /**
1350  * Check the hardware FIFOs and fill them as needed
1351  *
1352  * @usb:        USB device state populated by
1353  *              cvmx_usb_initialize().
1354  */
1355 static void __cvmx_usb_poll_tx_fifo(struct cvmx_usb_internal_state *usb)
1356 {
1357         if (usb->periodic.head != usb->periodic.tail) {
1358                 union cvmx_usbcx_hptxsts tx_status;
1359                 tx_status.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPTXSTS(usb->index));
1360                 if (__cvmx_usb_fill_tx_hw(usb, &usb->periodic, tx_status.s.ptxfspcavail))
1361                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk, ptxfempmsk, 1);
1362                 else
1363                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk, ptxfempmsk, 0);
1364         }
1365
1366         if (usb->nonperiodic.head != usb->nonperiodic.tail) {
1367                 union cvmx_usbcx_gnptxsts tx_status;
1368                 tx_status.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GNPTXSTS(usb->index));
1369                 if (__cvmx_usb_fill_tx_hw(usb, &usb->nonperiodic, tx_status.s.nptxfspcavail))
1370                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk, nptxfempmsk, 1);
1371                 else
1372                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk, nptxfempmsk, 0);
1373         }
1374
1375         return;
1376 }
1377
1378
1379 /**
1380  * Fill the TX FIFO with an outgoing packet
1381  *
1382  * @usb:          USB device state populated by
1383  *                cvmx_usb_initialize().
1384  * @channel:      Channel number to get packet from
1385  */
1386 static void __cvmx_usb_fill_tx_fifo(struct cvmx_usb_internal_state *usb, int channel)
1387 {
1388         union cvmx_usbcx_hccharx hcchar;
1389         union cvmx_usbcx_hcspltx usbc_hcsplt;
1390         union cvmx_usbcx_hctsizx usbc_hctsiz;
1391         struct cvmx_usb_tx_fifo *fifo;
1392
1393         /* We only need to fill data on outbound channels */
1394         hcchar.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index));
1395         if (hcchar.s.epdir != CVMX_USB_DIRECTION_OUT)
1396                 return;
1397
1398         /* OUT Splits only have data on the start and not the complete */
1399         usbc_hcsplt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCSPLTX(channel, usb->index));
1400         if (usbc_hcsplt.s.spltena && usbc_hcsplt.s.compsplt)
1401                 return;
1402
1403         /* Find out how many bytes we need to fill and convert it into 32bit words */
1404         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index));
1405         if (!usbc_hctsiz.s.xfersize)
1406                 return;
1407
1408         if ((hcchar.s.eptype == CVMX_USB_TRANSFER_INTERRUPT) ||
1409                 (hcchar.s.eptype == CVMX_USB_TRANSFER_ISOCHRONOUS))
1410                 fifo = &usb->periodic;
1411         else
1412                 fifo = &usb->nonperiodic;
1413
1414         fifo->entry[fifo->head].channel = channel;
1415         fifo->entry[fifo->head].address = __cvmx_usb_read_csr64(usb, CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8);
1416         fifo->entry[fifo->head].size = (usbc_hctsiz.s.xfersize+3)>>2;
1417         fifo->head++;
1418         if (fifo->head > MAX_CHANNELS)
1419                 fifo->head = 0;
1420
1421         __cvmx_usb_poll_tx_fifo(usb);
1422
1423         return;
1424 }
1425
1426 /**
1427  * Perform channel specific setup for Control transactions. All
1428  * the generic stuff will already have been done in
1429  * __cvmx_usb_start_channel()
1430  *
1431  * @usb:          USB device state populated by
1432  *                cvmx_usb_initialize().
1433  * @channel:      Channel to setup
1434  * @pipe:         Pipe for control transaction
1435  */
1436 static void __cvmx_usb_start_channel_control(struct cvmx_usb_internal_state *usb,
1437                                              int channel,
1438                                              struct cvmx_usb_pipe *pipe)
1439 {
1440         struct cvmx_usb_transaction *transaction = pipe->head;
1441         union cvmx_usb_control_header *header =
1442                 cvmx_phys_to_ptr(transaction->control_header);
1443         int bytes_to_transfer = transaction->buffer_length - transaction->actual_bytes;
1444         int packets_to_transfer;
1445         union cvmx_usbcx_hctsizx usbc_hctsiz;
1446
1447         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index));
1448
1449         switch (transaction->stage) {
1450         case CVMX_USB_STAGE_NON_CONTROL:
1451         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
1452                 cvmx_dprintf("%s: ERROR - Non control stage\n", __FUNCTION__);
1453                 break;
1454         case CVMX_USB_STAGE_SETUP:
1455                 usbc_hctsiz.s.pid = 3; /* Setup */
1456                 bytes_to_transfer = sizeof(*header);
1457                 /* All Control operations start with a setup going OUT */
1458                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), union cvmx_usbcx_hccharx, epdir, CVMX_USB_DIRECTION_OUT);
1459                 /*
1460                  * Setup send the control header instead of the buffer data. The
1461                  * buffer data will be used in the next stage
1462                  */
1463                 __cvmx_usb_write_csr64(usb, CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8, transaction->control_header);
1464                 break;
1465         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
1466                 usbc_hctsiz.s.pid = 3; /* Setup */
1467                 bytes_to_transfer = 0;
1468                 /* All Control operations start with a setup going OUT */
1469                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), union cvmx_usbcx_hccharx, epdir, CVMX_USB_DIRECTION_OUT);
1470                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index), union cvmx_usbcx_hcspltx, compsplt, 1);
1471                 break;
1472         case CVMX_USB_STAGE_DATA:
1473                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1474                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1475                         if (header->s.request_type & 0x80)
1476                                 bytes_to_transfer = 0;
1477                         else if (bytes_to_transfer > pipe->max_packet)
1478                                 bytes_to_transfer = pipe->max_packet;
1479                 }
1480                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1481                                 union cvmx_usbcx_hccharx, epdir,
1482                                 ((header->s.request_type & 0x80) ?
1483                                         CVMX_USB_DIRECTION_IN :
1484                                         CVMX_USB_DIRECTION_OUT));
1485                 break;
1486         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
1487                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1488                 if (!(header->s.request_type & 0x80))
1489                         bytes_to_transfer = 0;
1490                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1491                                 union cvmx_usbcx_hccharx, epdir,
1492                                 ((header->s.request_type & 0x80) ?
1493                                         CVMX_USB_DIRECTION_IN :
1494                                         CVMX_USB_DIRECTION_OUT));
1495                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index), union cvmx_usbcx_hcspltx, compsplt, 1);
1496                 break;
1497         case CVMX_USB_STAGE_STATUS:
1498                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1499                 bytes_to_transfer = 0;
1500                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), union cvmx_usbcx_hccharx, epdir,
1501                                 ((header->s.request_type & 0x80) ?
1502                                         CVMX_USB_DIRECTION_OUT :
1503                                         CVMX_USB_DIRECTION_IN));
1504                 break;
1505         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
1506                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1507                 bytes_to_transfer = 0;
1508                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), union cvmx_usbcx_hccharx, epdir,
1509                                 ((header->s.request_type & 0x80) ?
1510                                         CVMX_USB_DIRECTION_OUT :
1511                                         CVMX_USB_DIRECTION_IN));
1512                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index), union cvmx_usbcx_hcspltx, compsplt, 1);
1513                 break;
1514         }
1515
1516         /*
1517          * Make sure the transfer never exceeds the byte limit of the hardware.
1518          * Further bytes will be sent as continued transactions
1519          */
1520         if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1521                 /* Round MAX_TRANSFER_BYTES to a multiple of out packet size */
1522                 bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
1523                 bytes_to_transfer *= pipe->max_packet;
1524         }
1525
1526         /*
1527          * Calculate the number of packets to transfer. If the length is zero
1528          * we still need to transfer one packet
1529          */
1530         packets_to_transfer = (bytes_to_transfer + pipe->max_packet - 1) / pipe->max_packet;
1531         if (packets_to_transfer == 0)
1532                 packets_to_transfer = 1;
1533         else if ((packets_to_transfer > 1) && (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1534                 /*
1535                  * Limit to one packet when not using DMA. Channels must be
1536                  * restarted between every packet for IN transactions, so there
1537                  * is no reason to do multiple packets in a row
1538                  */
1539                 packets_to_transfer = 1;
1540                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1541         } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1542                 /*
1543                  * Limit the number of packet and data transferred to what the
1544                  * hardware can handle
1545                  */
1546                 packets_to_transfer = MAX_TRANSFER_PACKETS;
1547                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1548         }
1549
1550         usbc_hctsiz.s.xfersize = bytes_to_transfer;
1551         usbc_hctsiz.s.pktcnt = packets_to_transfer;
1552
1553         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index), usbc_hctsiz.u32);
1554         return;
1555 }
1556
1557
1558 /**
1559  * Start a channel to perform the pipe's head transaction
1560  *
1561  * @usb:          USB device state populated by
1562  *                cvmx_usb_initialize().
1563  * @channel:      Channel to setup
1564  * @pipe:         Pipe to start
1565  */
1566 static void __cvmx_usb_start_channel(struct cvmx_usb_internal_state *usb,
1567                                      int channel,
1568                                      struct cvmx_usb_pipe *pipe)
1569 {
1570         struct cvmx_usb_transaction *transaction = pipe->head;
1571
1572         /* Make sure all writes to the DMA region get flushed */
1573         CVMX_SYNCW;
1574
1575         /* Attach the channel to the pipe */
1576         usb->pipe_for_channel[channel] = pipe;
1577         pipe->channel = channel;
1578         pipe->flags |= __CVMX_USB_PIPE_FLAGS_SCHEDULED;
1579
1580         /* Mark this channel as in use */
1581         usb->idle_hardware_channels &= ~(1<<channel);
1582
1583         /* Enable the channel interrupt bits */
1584         {
1585                 union cvmx_usbcx_hcintx usbc_hcint;
1586                 union cvmx_usbcx_hcintmskx usbc_hcintmsk;
1587                 union cvmx_usbcx_haintmsk usbc_haintmsk;
1588
1589                 /* Clear all channel status bits */
1590                 usbc_hcint.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCINTX(channel, usb->index));
1591                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTX(channel, usb->index), usbc_hcint.u32);
1592
1593                 usbc_hcintmsk.u32 = 0;
1594                 usbc_hcintmsk.s.chhltdmsk = 1;
1595                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1596                         /* Channels need these extra interrupts when we aren't in DMA mode */
1597                         usbc_hcintmsk.s.datatglerrmsk = 1;
1598                         usbc_hcintmsk.s.frmovrunmsk = 1;
1599                         usbc_hcintmsk.s.bblerrmsk = 1;
1600                         usbc_hcintmsk.s.xacterrmsk = 1;
1601                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1602                                 /* Splits don't generate xfercompl, so we need ACK and NYET */
1603                                 usbc_hcintmsk.s.nyetmsk = 1;
1604                                 usbc_hcintmsk.s.ackmsk = 1;
1605                         }
1606                         usbc_hcintmsk.s.nakmsk = 1;
1607                         usbc_hcintmsk.s.stallmsk = 1;
1608                         usbc_hcintmsk.s.xfercomplmsk = 1;
1609                 }
1610                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), usbc_hcintmsk.u32);
1611
1612                 /* Enable the channel interrupt to propagate */
1613                 usbc_haintmsk.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HAINTMSK(usb->index));
1614                 usbc_haintmsk.s.haintmsk |= 1<<channel;
1615                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HAINTMSK(usb->index), usbc_haintmsk.u32);
1616         }
1617
1618         /* Setup the locations the DMA engines use  */
1619         {
1620                 uint64_t dma_address = transaction->buffer + transaction->actual_bytes;
1621                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1622                         dma_address = transaction->buffer + transaction->iso_packets[0].offset + transaction->actual_bytes;
1623                 __cvmx_usb_write_csr64(usb, CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8, dma_address);
1624                 __cvmx_usb_write_csr64(usb, CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8, dma_address);
1625         }
1626
1627         /* Setup both the size of the transfer and the SPLIT characteristics */
1628         {
1629                 union cvmx_usbcx_hcspltx usbc_hcsplt = {.u32 = 0};
1630                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 = 0};
1631                 int packets_to_transfer;
1632                 int bytes_to_transfer = transaction->buffer_length - transaction->actual_bytes;
1633
1634                 /*
1635                  * ISOCHRONOUS transactions store each individual transfer size
1636                  * in the packet structure, not the global buffer_length
1637                  */
1638                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1639                         bytes_to_transfer = transaction->iso_packets[0].length - transaction->actual_bytes;
1640
1641                 /*
1642                  * We need to do split transactions when we are talking to non
1643                  * high speed devices that are behind a high speed hub
1644                  */
1645                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1646                         /*
1647                          * On the start split phase (stage is even) record the
1648                          * frame number we will need to send the split complete.
1649                          * We only store the lower two bits since the time ahead
1650                          * can only be two frames
1651                          */
1652                         if ((transaction->stage&1) == 0) {
1653                                 if (transaction->type == CVMX_USB_TRANSFER_BULK)
1654                                         pipe->split_sc_frame = (usb->frame_number + 1) & 0x7f;
1655                                 else
1656                                         pipe->split_sc_frame = (usb->frame_number + 2) & 0x7f;
1657                         } else
1658                                 pipe->split_sc_frame = -1;
1659
1660                         usbc_hcsplt.s.spltena = 1;
1661                         usbc_hcsplt.s.hubaddr = pipe->hub_device_addr;
1662                         usbc_hcsplt.s.prtaddr = pipe->hub_port;
1663                         usbc_hcsplt.s.compsplt = (transaction->stage == CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE);
1664
1665                         /*
1666                          * SPLIT transactions can only ever transmit one data
1667                          * packet so limit the transfer size to the max packet
1668                          * size
1669                          */
1670                         if (bytes_to_transfer > pipe->max_packet)
1671                                 bytes_to_transfer = pipe->max_packet;
1672
1673                         /*
1674                          * ISOCHRONOUS OUT splits are unique in that they limit
1675                          * data transfers to 188 byte chunks representing the
1676                          * begin/middle/end of the data or all
1677                          */
1678                         if (!usbc_hcsplt.s.compsplt &&
1679                                 (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1680                                 (pipe->transfer_type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
1681                                 /*
1682                                  * Clear the split complete frame number as
1683                                  * there isn't going to be a split complete
1684                                  */
1685                                 pipe->split_sc_frame = -1;
1686                                 /*
1687                                  * See if we've started this transfer and sent
1688                                  * data
1689                                  */
1690                                 if (transaction->actual_bytes == 0) {
1691                                         /*
1692                                          * Nothing sent yet, this is either a
1693                                          * begin or the entire payload
1694                                          */
1695                                         if (bytes_to_transfer <= 188)
1696                                                 usbc_hcsplt.s.xactpos = 3; /* Entire payload in one go */
1697                                         else
1698                                                 usbc_hcsplt.s.xactpos = 2; /* First part of payload */
1699                                 } else {
1700                                         /*
1701                                          * Continuing the previous data, we must
1702                                          * either be in the middle or at the end
1703                                          */
1704                                         if (bytes_to_transfer <= 188)
1705                                                 usbc_hcsplt.s.xactpos = 1; /* End of payload */
1706                                         else
1707                                                 usbc_hcsplt.s.xactpos = 0; /* Middle of payload */
1708                                 }
1709                                 /*
1710                                  * Again, the transfer size is limited to 188
1711                                  * bytes
1712                                  */
1713                                 if (bytes_to_transfer > 188)
1714                                         bytes_to_transfer = 188;
1715                         }
1716                 }
1717
1718                 /*
1719                  * Make sure the transfer never exceeds the byte limit of the
1720                  * hardware. Further bytes will be sent as continued
1721                  * transactions
1722                  */
1723                 if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1724                         /*
1725                          * Round MAX_TRANSFER_BYTES to a multiple of out packet
1726                          * size
1727                          */
1728                         bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
1729                         bytes_to_transfer *= pipe->max_packet;
1730                 }
1731
1732                 /*
1733                  * Calculate the number of packets to transfer. If the length is
1734                  * zero we still need to transfer one packet
1735                  */
1736                 packets_to_transfer = (bytes_to_transfer + pipe->max_packet - 1) / pipe->max_packet;
1737                 if (packets_to_transfer == 0)
1738                         packets_to_transfer = 1;
1739                 else if ((packets_to_transfer > 1) && (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1740                         /*
1741                          * Limit to one packet when not using DMA. Channels must
1742                          * be restarted between every packet for IN
1743                          * transactions, so there is no reason to do multiple
1744                          * packets in a row
1745                          */
1746                         packets_to_transfer = 1;
1747                         bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1748                 } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1749                         /*
1750                          * Limit the number of packet and data transferred to
1751                          * what the hardware can handle
1752                          */
1753                         packets_to_transfer = MAX_TRANSFER_PACKETS;
1754                         bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1755                 }
1756
1757                 usbc_hctsiz.s.xfersize = bytes_to_transfer;
1758                 usbc_hctsiz.s.pktcnt = packets_to_transfer;
1759
1760                 /* Update the DATA0/DATA1 toggle */
1761                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1762                 /*
1763                  * High speed pipes may need a hardware ping before they start
1764                  */
1765                 if (pipe->flags & __CVMX_USB_PIPE_FLAGS_NEED_PING)
1766                         usbc_hctsiz.s.dopng = 1;
1767
1768                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCSPLTX(channel, usb->index), usbc_hcsplt.u32);
1769                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index), usbc_hctsiz.u32);
1770         }
1771
1772         /* Setup the Host Channel Characteristics Register */
1773         {
1774                 union cvmx_usbcx_hccharx usbc_hcchar = {.u32 = 0};
1775
1776                 /*
1777                  * Set the startframe odd/even properly. This is only used for
1778                  * periodic
1779                  */
1780                 usbc_hcchar.s.oddfrm = usb->frame_number&1;
1781
1782                 /*
1783                  * Set the number of back to back packets allowed by this
1784                  * endpoint. Split transactions interpret "ec" as the number of
1785                  * immediate retries of failure. These retries happen too
1786                  * quickly, so we disable these entirely for splits
1787                  */
1788                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
1789                         usbc_hcchar.s.ec = 1;
1790                 else if (pipe->multi_count < 1)
1791                         usbc_hcchar.s.ec = 1;
1792                 else if (pipe->multi_count > 3)
1793                         usbc_hcchar.s.ec = 3;
1794                 else
1795                         usbc_hcchar.s.ec = pipe->multi_count;
1796
1797                 /* Set the rest of the endpoint specific settings */
1798                 usbc_hcchar.s.devaddr = pipe->device_addr;
1799                 usbc_hcchar.s.eptype = transaction->type;
1800                 usbc_hcchar.s.lspddev = (pipe->device_speed == CVMX_USB_SPEED_LOW);
1801                 usbc_hcchar.s.epdir = pipe->transfer_dir;
1802                 usbc_hcchar.s.epnum = pipe->endpoint_num;
1803                 usbc_hcchar.s.mps = pipe->max_packet;
1804                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index), usbc_hcchar.u32);
1805         }
1806
1807         /* Do transaction type specific fixups as needed */
1808         switch (transaction->type) {
1809         case CVMX_USB_TRANSFER_CONTROL:
1810                 __cvmx_usb_start_channel_control(usb, channel, pipe);
1811                 break;
1812         case CVMX_USB_TRANSFER_BULK:
1813         case CVMX_USB_TRANSFER_INTERRUPT:
1814                 break;
1815         case CVMX_USB_TRANSFER_ISOCHRONOUS:
1816                 if (!__cvmx_usb_pipe_needs_split(usb, pipe)) {
1817                         /*
1818                          * ISO transactions require different PIDs depending on
1819                          * direction and how many packets are needed
1820                          */
1821                         if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
1822                                 if (pipe->multi_count < 2) /* Need DATA0 */
1823                                         USB_SET_FIELD32(CVMX_USBCX_HCTSIZX(channel, usb->index), union cvmx_usbcx_hctsizx, pid, 0);
1824                                 else /* Need MDATA */
1825                                         USB_SET_FIELD32(CVMX_USBCX_HCTSIZX(channel, usb->index), union cvmx_usbcx_hctsizx, pid, 3);
1826                         }
1827                 }
1828                 break;
1829         }
1830         {
1831                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index))};
1832                 transaction->xfersize = usbc_hctsiz.s.xfersize;
1833                 transaction->pktcnt = usbc_hctsiz.s.pktcnt;
1834         }
1835         /* Remeber when we start a split transaction */
1836         if (__cvmx_usb_pipe_needs_split(usb, pipe))
1837                 usb->active_split = transaction;
1838         USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), union cvmx_usbcx_hccharx, chena, 1);
1839         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
1840                 __cvmx_usb_fill_tx_fifo(usb, channel);
1841         return;
1842 }
1843
1844
1845 /**
1846  * Find a pipe that is ready to be scheduled to hardware.
1847  * @usb:         USB device state populated by
1848  *               cvmx_usb_initialize().
1849  * @list:        Pipe list to search
1850  * @current_frame:
1851  *               Frame counter to use as a time reference.
1852  *
1853  * Returns: Pipe or NULL if none are ready
1854  */
1855 static struct cvmx_usb_pipe *__cvmx_usb_find_ready_pipe(struct cvmx_usb_internal_state *usb, struct cvmx_usb_pipe_list *list, uint64_t current_frame)
1856 {
1857         struct cvmx_usb_pipe *pipe = list->head;
1858         while (pipe) {
1859                 if (!(pipe->flags & __CVMX_USB_PIPE_FLAGS_SCHEDULED) && pipe->head &&
1860                         (pipe->next_tx_frame <= current_frame) &&
1861                         ((pipe->split_sc_frame == -1) || ((((int)current_frame - (int)pipe->split_sc_frame) & 0x7f) < 0x40)) &&
1862                         (!usb->active_split || (usb->active_split == pipe->head))) {
1863                         CVMX_PREFETCH(pipe, 128);
1864                         CVMX_PREFETCH(pipe->head, 0);
1865                         return pipe;
1866                 }
1867                 pipe = pipe->next;
1868         }
1869         return NULL;
1870 }
1871
1872
1873 /**
1874  * Called whenever a pipe might need to be scheduled to the
1875  * hardware.
1876  *
1877  * @usb:         USB device state populated by
1878  *               cvmx_usb_initialize().
1879  * @is_sof:      True if this schedule was called on a SOF interrupt.
1880  */
1881 static void __cvmx_usb_schedule(struct cvmx_usb_internal_state *usb, int is_sof)
1882 {
1883         int channel;
1884         struct cvmx_usb_pipe *pipe;
1885         int need_sof;
1886         enum cvmx_usb_transfer ttype;
1887
1888         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1889                 /* Without DMA we need to be careful to not schedule something at the end of a frame and cause an overrun */
1890                 union cvmx_usbcx_hfnum hfnum = {.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index))};
1891                 union cvmx_usbcx_hfir hfir = {.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HFIR(usb->index))};
1892                 if (hfnum.s.frrem < hfir.s.frint/4)
1893                         goto done;
1894         }
1895
1896         while (usb->idle_hardware_channels) {
1897                 /* Find an idle channel */
1898                 CVMX_CLZ(channel, usb->idle_hardware_channels);
1899                 channel = 31 - channel;
1900                 if (unlikely(channel > 7))
1901                         break;
1902
1903                 /* Find a pipe needing service */
1904                 pipe = NULL;
1905                 if (is_sof) {
1906                         /*
1907                          * Only process periodic pipes on SOF interrupts. This
1908                          * way we are sure that the periodic data is sent in the
1909                          * beginning of the frame
1910                          */
1911                         pipe = __cvmx_usb_find_ready_pipe(usb, usb->active_pipes + CVMX_USB_TRANSFER_ISOCHRONOUS, usb->frame_number);
1912                         if (likely(!pipe))
1913                                 pipe = __cvmx_usb_find_ready_pipe(usb, usb->active_pipes + CVMX_USB_TRANSFER_INTERRUPT, usb->frame_number);
1914                 }
1915                 if (likely(!pipe)) {
1916                         pipe = __cvmx_usb_find_ready_pipe(usb, usb->active_pipes + CVMX_USB_TRANSFER_CONTROL, usb->frame_number);
1917                         if (likely(!pipe))
1918                                 pipe = __cvmx_usb_find_ready_pipe(usb, usb->active_pipes + CVMX_USB_TRANSFER_BULK, usb->frame_number);
1919                 }
1920                 if (!pipe)
1921                         break;
1922
1923                 __cvmx_usb_start_channel(usb, channel, pipe);
1924         }
1925
1926 done:
1927         /*
1928          * Only enable SOF interrupts when we have transactions pending in the
1929          * future that might need to be scheduled
1930          */
1931         need_sof = 0;
1932         for (ttype = CVMX_USB_TRANSFER_CONTROL; ttype <= CVMX_USB_TRANSFER_INTERRUPT; ttype++) {
1933                 pipe = usb->active_pipes[ttype].head;
1934                 while (pipe) {
1935                         if (pipe->next_tx_frame > usb->frame_number) {
1936                                 need_sof = 1;
1937                                 break;
1938                         }
1939                         pipe = pipe->next;
1940                 }
1941         }
1942         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk, sofmsk, need_sof);
1943         return;
1944 }
1945
1946
1947 /**
1948  * Call a user's callback for a specific reason.
1949  *
1950  * @usb:         USB device state populated by
1951  *               cvmx_usb_initialize().
1952  * @pipe:        Pipe the callback is for or NULL
1953  * @transaction:
1954  *               Transaction the callback is for or NULL
1955  * @reason:      Reason this callback is being called
1956  * @complete_code:
1957  *               Completion code for the transaction, if any
1958  */
1959 static void __cvmx_usb_perform_callback(struct cvmx_usb_internal_state *usb,
1960                                         struct cvmx_usb_pipe *pipe,
1961                                         struct cvmx_usb_transaction *transaction,
1962                                         enum cvmx_usb_callback reason,
1963                                         enum cvmx_usb_complete complete_code)
1964 {
1965         cvmx_usb_callback_func_t callback = usb->callback[reason];
1966         void *user_data = usb->callback_data[reason];
1967         int submit_handle = -1;
1968         int pipe_handle = -1;
1969         int bytes_transferred = 0;
1970
1971         if (pipe)
1972                 pipe_handle = __cvmx_usb_get_pipe_handle(usb, pipe);
1973
1974         if (transaction) {
1975                 submit_handle = __cvmx_usb_get_submit_handle(usb, transaction);
1976                 bytes_transferred = transaction->actual_bytes;
1977                 /* Transactions are allowed to override the default callback */
1978                 if ((reason == CVMX_USB_CALLBACK_TRANSFER_COMPLETE) && transaction->callback) {
1979                         callback = transaction->callback;
1980                         user_data = transaction->callback_data;
1981                 }
1982         }
1983
1984         if (!callback)
1985                 return;
1986
1987         callback((struct cvmx_usb_state *)usb, reason, complete_code, pipe_handle, submit_handle,
1988                  bytes_transferred, user_data);
1989 }
1990
1991
1992 /**
1993  * Signal the completion of a transaction and free it. The
1994  * transaction will be removed from the pipe transaction list.
1995  *
1996  * @usb:         USB device state populated by
1997  *               cvmx_usb_initialize().
1998  * @pipe:        Pipe the transaction is on
1999  * @transaction:
2000  *               Transaction that completed
2001  * @complete_code:
2002  *               Completion code
2003  */
2004 static void __cvmx_usb_perform_complete(struct cvmx_usb_internal_state *usb,
2005                                         struct cvmx_usb_pipe *pipe,
2006                                         struct cvmx_usb_transaction *transaction,
2007                                         enum cvmx_usb_complete complete_code)
2008 {
2009         /* If this was a split then clear our split in progress marker */
2010         if (usb->active_split == transaction)
2011                 usb->active_split = NULL;
2012
2013         /*
2014          * Isochronous transactions need extra processing as they might not be
2015          * done after a single data transfer
2016          */
2017         if (unlikely(transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
2018                 /* Update the number of bytes transferred in this ISO packet */
2019                 transaction->iso_packets[0].length = transaction->actual_bytes;
2020                 transaction->iso_packets[0].status = complete_code;
2021
2022                 /*
2023                  * If there are more ISOs pending and we succeeded, schedule the
2024                  * next one
2025                  */
2026                 if ((transaction->iso_number_packets > 1) && (complete_code == CVMX_USB_COMPLETE_SUCCESS)) {
2027                         transaction->actual_bytes = 0;     /* No bytes transferred for this packet as of yet */
2028                         transaction->iso_number_packets--; /* One less ISO waiting to transfer */
2029                         transaction->iso_packets++;        /* Increment to the next location in our packet array */
2030                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2031                         goto done;
2032                 }
2033         }
2034
2035         /* Remove the transaction from the pipe list */
2036         if (transaction->next)
2037                 transaction->next->prev = transaction->prev;
2038         else
2039                 pipe->tail = transaction->prev;
2040         if (transaction->prev)
2041                 transaction->prev->next = transaction->next;
2042         else
2043                 pipe->head = transaction->next;
2044         if (!pipe->head) {
2045                 __cvmx_usb_remove_pipe(usb->active_pipes + pipe->transfer_type, pipe);
2046                 __cvmx_usb_append_pipe(&usb->idle_pipes, pipe);
2047
2048         }
2049         __cvmx_usb_perform_callback(usb, pipe, transaction,
2050                                     CVMX_USB_CALLBACK_TRANSFER_COMPLETE,
2051                                     complete_code);
2052         __cvmx_usb_free_transaction(usb, transaction);
2053 done:
2054         return;
2055 }
2056
2057
2058 /**
2059  * Submit a usb transaction to a pipe. Called for all types
2060  * of transactions.
2061  *
2062  * @usb:
2063  * @pipe_handle:
2064  *                  Which pipe to submit to. Will be validated in this function.
2065  * @type:           Transaction type
2066  * @flags:          Flags for the transaction
2067  * @buffer:         User buffer for the transaction
2068  * @buffer_length:
2069  *                  User buffer's length in bytes
2070  * @control_header:
2071  *                  For control transactions, the 8 byte standard header
2072  * @iso_start_frame:
2073  *                  For ISO transactions, the start frame
2074  * @iso_number_packets:
2075  *                  For ISO, the number of packet in the transaction.
2076  * @iso_packets:
2077  *                  A description of each ISO packet
2078  * @callback:       User callback to call when the transaction completes
2079  * @user_data:      User's data for the callback
2080  *
2081  * Returns: Submit handle or negative on failure. Matches the result
2082  *          in the external API.
2083  */
2084 static int __cvmx_usb_submit_transaction(struct cvmx_usb_internal_state *usb,
2085                                          int pipe_handle,
2086                                          enum cvmx_usb_transfer type,
2087                                          int flags,
2088                                          uint64_t buffer,
2089                                          int buffer_length,
2090                                          uint64_t control_header,
2091                                          int iso_start_frame,
2092                                          int iso_number_packets,
2093                                          struct cvmx_usb_iso_packet *iso_packets,
2094                                          cvmx_usb_callback_func_t callback,
2095                                          void *user_data)
2096 {
2097         int submit_handle;
2098         struct cvmx_usb_transaction *transaction;
2099         struct cvmx_usb_pipe *pipe = usb->pipe + pipe_handle;
2100
2101         if (unlikely((pipe_handle < 0) || (pipe_handle >= MAX_PIPES)))
2102                 return -EINVAL;
2103         /* Fail if the pipe isn't open */
2104         if (unlikely((pipe->flags & __CVMX_USB_PIPE_FLAGS_OPEN) == 0))
2105                 return -EINVAL;
2106         if (unlikely(pipe->transfer_type != type))
2107                 return -EINVAL;
2108
2109         transaction = __cvmx_usb_alloc_transaction(usb);
2110         if (unlikely(!transaction))
2111                 return -ENOMEM;
2112
2113         transaction->type = type;
2114         transaction->flags |= flags;
2115         transaction->buffer = buffer;
2116         transaction->buffer_length = buffer_length;
2117         transaction->control_header = control_header;
2118         transaction->iso_start_frame = iso_start_frame; // FIXME: This is not used, implement it
2119         transaction->iso_number_packets = iso_number_packets;
2120         transaction->iso_packets = iso_packets;
2121         transaction->callback = callback;
2122         transaction->callback_data = user_data;
2123         if (transaction->type == CVMX_USB_TRANSFER_CONTROL)
2124                 transaction->stage = CVMX_USB_STAGE_SETUP;
2125         else
2126                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2127
2128         transaction->next = NULL;
2129         if (pipe->tail) {
2130                 transaction->prev = pipe->tail;
2131                 transaction->prev->next = transaction;
2132         } else {
2133                 if (pipe->next_tx_frame < usb->frame_number)
2134                         pipe->next_tx_frame = usb->frame_number + pipe->interval -
2135                                 (usb->frame_number - pipe->next_tx_frame) % pipe->interval;
2136                 transaction->prev = NULL;
2137                 pipe->head = transaction;
2138                 __cvmx_usb_remove_pipe(&usb->idle_pipes, pipe);
2139                 __cvmx_usb_append_pipe(usb->active_pipes + pipe->transfer_type, pipe);
2140         }
2141         pipe->tail = transaction;
2142
2143         submit_handle = __cvmx_usb_get_submit_handle(usb, transaction);
2144
2145         /* We may need to schedule the pipe if this was the head of the pipe */
2146         if (!transaction->prev)
2147                 __cvmx_usb_schedule(usb, 0);
2148
2149         return submit_handle;
2150 }
2151
2152
2153 /**
2154  * Call to submit a USB Bulk transfer to a pipe.
2155  *
2156  * @state:          USB device state populated by
2157  *                  cvmx_usb_initialize().
2158  * @pipe_handle:
2159  *                  Handle to the pipe for the transfer.
2160  * @buffer:         Physical address of the data buffer in
2161  *                  memory. Note that this is NOT A POINTER, but
2162  *                  the full 64bit physical address of the
2163  *                  buffer. This may be zero if buffer_length is
2164  *                  zero.
2165  * @buffer_length:
2166  *                  Length of buffer in bytes.
2167  * @callback:       Function to call when this transaction
2168  *                  completes. If the return value of this
2169  *                  function isn't an error, then this function
2170  *                  is guaranteed to be called when the
2171  *                  transaction completes. If this parameter is
2172  *                  NULL, then the generic callback registered
2173  *                  through cvmx_usb_register_callback is
2174  *                  called. If both are NULL, then there is no
2175  *                  way to know when a transaction completes.
2176  * @user_data:      User supplied data returned when the
2177  *                  callback is called. This is only used if
2178  *                  callback in not NULL.
2179  *
2180  * Returns: A submitted transaction handle or negative on
2181  *          failure. Negative values are error codes.
2182  */
2183 int cvmx_usb_submit_bulk(struct cvmx_usb_state *state, int pipe_handle,
2184                          uint64_t buffer, int buffer_length,
2185                          cvmx_usb_callback_func_t callback,
2186                          void *user_data)
2187 {
2188         int submit_handle;
2189         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
2190
2191         /* Pipe handle checking is done later in a common place */
2192         if (unlikely(!buffer))
2193                 return -EINVAL;
2194         if (unlikely(buffer_length < 0))
2195                 return -EINVAL;
2196
2197         submit_handle = __cvmx_usb_submit_transaction(usb, pipe_handle,
2198                                                       CVMX_USB_TRANSFER_BULK,
2199                                                       0, /* flags */
2200                                                       buffer,
2201                                                       buffer_length,
2202                                                       0, /* control_header */
2203                                                       0, /* iso_start_frame */
2204                                                       0, /* iso_number_packets */
2205                                                       NULL, /* iso_packets */
2206                                                       callback,
2207                                                       user_data);
2208         return submit_handle;
2209 }
2210
2211
2212 /**
2213  * Call to submit a USB Interrupt transfer to a pipe.
2214  *
2215  * @state:          USB device state populated by
2216  *                  cvmx_usb_initialize().
2217  * @pipe_handle:
2218  *                  Handle to the pipe for the transfer.
2219  * @buffer:         Physical address of the data buffer in
2220  *                  memory. Note that this is NOT A POINTER, but
2221  *                  the full 64bit physical address of the
2222  *                  buffer. This may be zero if buffer_length is
2223  *                  zero.
2224  * @buffer_length:
2225  *                  Length of buffer in bytes.
2226  * @callback:       Function to call when this transaction
2227  *                  completes. If the return value of this
2228  *                  function isn't an error, then this function
2229  *                  is guaranteed to be called when the
2230  *                  transaction completes. If this parameter is
2231  *                  NULL, then the generic callback registered
2232  *                  through cvmx_usb_register_callback is
2233  *                  called. If both are NULL, then there is no
2234  *                  way to know when a transaction completes.
2235  * @user_data:      User supplied data returned when the
2236  *                  callback is called. This is only used if
2237  *                  callback in not NULL.
2238  *
2239  * Returns: A submitted transaction handle or negative on
2240  *          failure. Negative values are error codes.
2241  */
2242 int cvmx_usb_submit_interrupt(struct cvmx_usb_state *state, int pipe_handle,
2243                               uint64_t buffer, int buffer_length,
2244                               cvmx_usb_callback_func_t callback,
2245                               void *user_data)
2246 {
2247         int submit_handle;
2248         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
2249
2250         /* Pipe handle checking is done later in a common place */
2251         if (unlikely(!buffer))
2252                 return -EINVAL;
2253         if (unlikely(buffer_length < 0))
2254                 return -EINVAL;
2255
2256         submit_handle = __cvmx_usb_submit_transaction(usb, pipe_handle,
2257                                                       CVMX_USB_TRANSFER_INTERRUPT,
2258                                                       0, /* flags */
2259                                                       buffer,
2260                                                       buffer_length,
2261                                                       0, /* control_header */
2262                                                       0, /* iso_start_frame */
2263                                                       0, /* iso_number_packets */
2264                                                       NULL, /* iso_packets */
2265                                                       callback,
2266                                                       user_data);
2267         return submit_handle;
2268 }
2269
2270
2271 /**
2272  * Call to submit a USB Control transfer to a pipe.
2273  *
2274  * @state:          USB device state populated by
2275  *                  cvmx_usb_initialize().
2276  * @pipe_handle:
2277  *                  Handle to the pipe for the transfer.
2278  * @control_header:
2279  *                  USB 8 byte control header physical address.
2280  *                  Note that this is NOT A POINTER, but the
2281  *                  full 64bit physical address of the buffer.
2282  * @buffer:         Physical address of the data buffer in
2283  *                  memory. Note that this is NOT A POINTER, but
2284  *                  the full 64bit physical address of the
2285  *                  buffer. This may be zero if buffer_length is
2286  *                  zero.
2287  * @buffer_length:
2288  *                  Length of buffer in bytes.
2289  * @callback:       Function to call when this transaction
2290  *                  completes. If the return value of this
2291  *                  function isn't an error, then this function
2292  *                  is guaranteed to be called when the
2293  *                  transaction completes. If this parameter is
2294  *                  NULL, then the generic callback registered
2295  *                  through cvmx_usb_register_callback is
2296  *                  called. If both are NULL, then there is no
2297  *                  way to know when a transaction completes.
2298  * @user_data:      User supplied data returned when the
2299  *                  callback is called. This is only used if
2300  *                  callback in not NULL.
2301  *
2302  * Returns: A submitted transaction handle or negative on
2303  *          failure. Negative values are error codes.
2304  */
2305 int cvmx_usb_submit_control(struct cvmx_usb_state *state, int pipe_handle,
2306                             uint64_t control_header,
2307                             uint64_t buffer, int buffer_length,
2308                             cvmx_usb_callback_func_t callback,
2309                             void *user_data)
2310 {
2311         int submit_handle;
2312         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
2313         union cvmx_usb_control_header *header =
2314                 cvmx_phys_to_ptr(control_header);
2315
2316         /* Pipe handle checking is done later in a common place */
2317         if (unlikely(!control_header))
2318                 return -EINVAL;
2319         /* Some drivers send a buffer with a zero length. God only knows why */
2320         if (unlikely(buffer && (buffer_length < 0)))
2321                 return -EINVAL;
2322         if (unlikely(!buffer && (buffer_length != 0)))
2323                 return -EINVAL;
2324         if ((header->s.request_type & 0x80) == 0)
2325                 buffer_length = le16_to_cpu(header->s.length);
2326
2327         submit_handle = __cvmx_usb_submit_transaction(usb, pipe_handle,
2328                                                       CVMX_USB_TRANSFER_CONTROL,
2329                                                       0, /* flags */
2330                                                       buffer,
2331                                                       buffer_length,
2332                                                       control_header,
2333                                                       0, /* iso_start_frame */
2334                                                       0, /* iso_number_packets */
2335                                                       NULL, /* iso_packets */
2336                                                       callback,
2337                                                       user_data);
2338         return submit_handle;
2339 }
2340
2341
2342 /**
2343  * Call to submit a USB Isochronous transfer to a pipe.
2344  *
2345  * @state:          USB device state populated by
2346  *                  cvmx_usb_initialize().
2347  * @pipe_handle:
2348  *                  Handle to the pipe for the transfer.
2349  * @start_frame:
2350  *                  Number of frames into the future to schedule
2351  *                  this transaction.
2352  * @flags:          Flags to control the transfer. See
2353  *                  enum cvmx_usb_isochronous_flags for the flag
2354  *                  definitions.
2355  * @number_packets:
2356  *                  Number of sequential packets to transfer.
2357  *                  "packets" is a pointer to an array of this
2358  *                  many packet structures.
2359  * @packets:        Description of each transfer packet as
2360  *                  defined by struct cvmx_usb_iso_packet. The array
2361  *                  pointed to here must stay valid until the
2362  *                  complete callback is called.
2363  * @buffer:         Physical address of the data buffer in
2364  *                  memory. Note that this is NOT A POINTER, but
2365  *                  the full 64bit physical address of the
2366  *                  buffer. This may be zero if buffer_length is
2367  *                  zero.
2368  * @buffer_length:
2369  *                  Length of buffer in bytes.
2370  * @callback:       Function to call when this transaction
2371  *                  completes. If the return value of this
2372  *                  function isn't an error, then this function
2373  *                  is guaranteed to be called when the
2374  *                  transaction completes. If this parameter is
2375  *                  NULL, then the generic callback registered
2376  *                  through cvmx_usb_register_callback is
2377  *                  called. If both are NULL, then there is no
2378  *                  way to know when a transaction completes.
2379  * @user_data:      User supplied data returned when the
2380  *                  callback is called. This is only used if
2381  *                  callback in not NULL.
2382  *
2383  * Returns: A submitted transaction handle or negative on
2384  *          failure. Negative values are error codes.
2385  */
2386 int cvmx_usb_submit_isochronous(struct cvmx_usb_state *state, int pipe_handle,
2387                                 int start_frame, int flags,
2388                                 int number_packets,
2389                                 struct cvmx_usb_iso_packet packets[],
2390                                 uint64_t buffer, int buffer_length,
2391                                 cvmx_usb_callback_func_t callback,
2392                                 void *user_data)
2393 {
2394         int submit_handle;
2395         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
2396
2397         /* Pipe handle checking is done later in a common place */
2398         if (unlikely(start_frame < 0))
2399                 return -EINVAL;
2400         if (unlikely(flags & ~(CVMX_USB_ISOCHRONOUS_FLAGS_ALLOW_SHORT | CVMX_USB_ISOCHRONOUS_FLAGS_ASAP)))
2401                 return -EINVAL;
2402         if (unlikely(number_packets < 1))
2403                 return -EINVAL;
2404         if (unlikely(!packets))
2405                 return -EINVAL;
2406         if (unlikely(!buffer))
2407                 return -EINVAL;
2408         if (unlikely(buffer_length < 0))
2409                 return -EINVAL;
2410
2411         submit_handle = __cvmx_usb_submit_transaction(usb, pipe_handle,
2412                                                       CVMX_USB_TRANSFER_ISOCHRONOUS,
2413                                                       flags,
2414                                                       buffer,
2415                                                       buffer_length,
2416                                                       0, /* control_header */
2417                                                       start_frame,
2418                                                       number_packets,
2419                                                       packets,
2420                                                       callback,
2421                                                       user_data);
2422         return submit_handle;
2423 }
2424
2425
2426 /**
2427  * Cancel one outstanding request in a pipe. Canceling a request
2428  * can fail if the transaction has already completed before cancel
2429  * is called. Even after a successful cancel call, it may take
2430  * a frame or two for the cvmx_usb_poll() function to call the
2431  * associated callback.
2432  *
2433  * @state:       USB device state populated by
2434  *               cvmx_usb_initialize().
2435  * @pipe_handle:
2436  *               Pipe handle to cancel requests in.
2437  * @submit_handle:
2438  *               Handle to transaction to cancel, returned by the submit function.
2439  *
2440  * Returns: 0 or a negative error code.
2441  */
2442 int cvmx_usb_cancel(struct cvmx_usb_state *state, int pipe_handle, int submit_handle)
2443 {
2444         struct cvmx_usb_transaction *transaction;
2445         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
2446         struct cvmx_usb_pipe *pipe = usb->pipe + pipe_handle;
2447
2448         if (unlikely((pipe_handle < 0) || (pipe_handle >= MAX_PIPES)))
2449                 return -EINVAL;
2450         if (unlikely((submit_handle < 0) || (submit_handle >= MAX_TRANSACTIONS)))
2451                 return -EINVAL;
2452
2453         /* Fail if the pipe isn't open */
2454         if (unlikely((pipe->flags & __CVMX_USB_PIPE_FLAGS_OPEN) == 0))
2455                 return -EINVAL;
2456
2457         transaction = usb->transaction + submit_handle;
2458
2459         /* Fail if this transaction already completed */
2460         if (unlikely((transaction->flags & __CVMX_USB_TRANSACTION_FLAGS_IN_USE) == 0))
2461                 return -EINVAL;
2462
2463         /*
2464          * If the transaction is the HEAD of the queue and scheduled. We need to
2465          * treat it special
2466          */
2467         if ((pipe->head == transaction) &&
2468                 (pipe->flags & __CVMX_USB_PIPE_FLAGS_SCHEDULED)) {
2469                 union cvmx_usbcx_hccharx usbc_hcchar;
2470
2471                 usb->pipe_for_channel[pipe->channel] = NULL;
2472                 pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_SCHEDULED;
2473
2474                 CVMX_SYNCW;
2475
2476                 usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCCHARX(pipe->channel, usb->index));
2477                 /* If the channel isn't enabled then the transaction already completed */
2478                 if (usbc_hcchar.s.chena) {
2479                         usbc_hcchar.s.chdis = 1;
2480                         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCCHARX(pipe->channel, usb->index), usbc_hcchar.u32);
2481                 }
2482         }
2483         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_CANCEL);
2484         return 0;
2485 }
2486
2487
2488 /**
2489  * Cancel all outstanding requests in a pipe. Logically all this
2490  * does is call cvmx_usb_cancel() in a loop.
2491  *
2492  * @state:       USB device state populated by
2493  *               cvmx_usb_initialize().
2494  * @pipe_handle:
2495  *               Pipe handle to cancel requests in.
2496  *
2497  * Returns: 0 or a negative error code.
2498  */
2499 int cvmx_usb_cancel_all(struct cvmx_usb_state *state, int pipe_handle)
2500 {
2501         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
2502         struct cvmx_usb_pipe *pipe = usb->pipe + pipe_handle;
2503
2504         if (unlikely((pipe_handle < 0) || (pipe_handle >= MAX_PIPES)))
2505                 return -EINVAL;
2506
2507         /* Fail if the pipe isn't open */
2508         if (unlikely((pipe->flags & __CVMX_USB_PIPE_FLAGS_OPEN) == 0))
2509                 return -EINVAL;
2510
2511         /* Simply loop through and attempt to cancel each transaction */
2512         while (pipe->head) {
2513                 int result = cvmx_usb_cancel(state, pipe_handle,
2514                         __cvmx_usb_get_submit_handle(usb, pipe->head));
2515                 if (unlikely(result != 0))
2516                         return result;
2517         }
2518         return 0;
2519 }
2520
2521
2522 /**
2523  * Close a pipe created with cvmx_usb_open_pipe().
2524  *
2525  * @state:       USB device state populated by
2526  *               cvmx_usb_initialize().
2527  * @pipe_handle:
2528  *               Pipe handle to close.
2529  *
2530  * Returns: 0 or a negative error code. EBUSY is returned if the pipe has
2531  *          outstanding transfers.
2532  */
2533 int cvmx_usb_close_pipe(struct cvmx_usb_state *state, int pipe_handle)
2534 {
2535         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
2536         struct cvmx_usb_pipe *pipe = usb->pipe + pipe_handle;
2537
2538         if (unlikely((pipe_handle < 0) || (pipe_handle >= MAX_PIPES)))
2539                 return -EINVAL;
2540
2541         /* Fail if the pipe isn't open */
2542         if (unlikely((pipe->flags & __CVMX_USB_PIPE_FLAGS_OPEN) == 0))
2543                 return -EINVAL;
2544
2545         /* Fail if the pipe has pending transactions */
2546         if (unlikely(pipe->head))
2547                 return -EBUSY;
2548
2549         pipe->flags = 0;
2550         __cvmx_usb_remove_pipe(&usb->idle_pipes, pipe);
2551         __cvmx_usb_append_pipe(&usb->free_pipes, pipe);
2552
2553         return 0;
2554 }
2555
2556
2557 /**
2558  * Register a function to be called when various USB events occur.
2559  *
2560  * @state:     USB device state populated by
2561  *             cvmx_usb_initialize().
2562  * @reason:    Which event to register for.
2563  * @callback:  Function to call when the event occurs.
2564  * @user_data: User data parameter to the function.
2565  *
2566  * Returns: 0 or a negative error code.
2567  */
2568 int cvmx_usb_register_callback(struct cvmx_usb_state *state,
2569                                enum cvmx_usb_callback reason,
2570                                cvmx_usb_callback_func_t callback,
2571                                void *user_data)
2572 {
2573         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
2574
2575         if (unlikely(reason >= __CVMX_USB_CALLBACK_END))
2576                 return -EINVAL;
2577         if (unlikely(!callback))
2578                 return -EINVAL;
2579
2580         usb->callback[reason] = callback;
2581         usb->callback_data[reason] = user_data;
2582
2583         return 0;
2584 }
2585
2586
2587 /**
2588  * Get the current USB protocol level frame number. The frame
2589  * number is always in the range of 0-0x7ff.
2590  *
2591  * @state: USB device state populated by
2592  *         cvmx_usb_initialize().
2593  *
2594  * Returns: USB frame number
2595  */
2596 int cvmx_usb_get_frame_number(struct cvmx_usb_state *state)
2597 {
2598         int frame_number;
2599         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
2600         union cvmx_usbcx_hfnum usbc_hfnum;
2601
2602         usbc_hfnum.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
2603         frame_number = usbc_hfnum.s.frnum;
2604
2605         return frame_number;
2606 }
2607
2608
2609 /**
2610  * Poll a channel for status
2611  *
2612  * @usb:     USB device
2613  * @channel: Channel to poll
2614  *
2615  * Returns: Zero on success
2616  */
2617 static int __cvmx_usb_poll_channel(struct cvmx_usb_internal_state *usb, int channel)
2618 {
2619         union cvmx_usbcx_hcintx usbc_hcint;
2620         union cvmx_usbcx_hctsizx usbc_hctsiz;
2621         union cvmx_usbcx_hccharx usbc_hcchar;
2622         struct cvmx_usb_pipe *pipe;
2623         struct cvmx_usb_transaction *transaction;
2624         int bytes_this_transfer;
2625         int bytes_in_last_packet;
2626         int packets_processed;
2627         int buffer_space_left;
2628
2629         /* Read the interrupt status bits for the channel */
2630         usbc_hcint.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCINTX(channel, usb->index));
2631
2632         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
2633                 usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index));
2634
2635                 if (usbc_hcchar.s.chena && usbc_hcchar.s.chdis) {
2636                         /*
2637                          * There seems to be a bug in CN31XX which can cause
2638                          * interrupt IN transfers to get stuck until we do a
2639                          * write of HCCHARX without changing things
2640                          */
2641                         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index), usbc_hcchar.u32);
2642                         return 0;
2643                 }
2644
2645                 /*
2646                  * In non DMA mode the channels don't halt themselves. We need
2647                  * to manually disable channels that are left running
2648                  */
2649                 if (!usbc_hcint.s.chhltd) {
2650                         if (usbc_hcchar.s.chena) {
2651                                 union cvmx_usbcx_hcintmskx hcintmsk;
2652                                 /* Disable all interrupts except CHHLTD */
2653                                 hcintmsk.u32 = 0;
2654                                 hcintmsk.s.chhltdmsk = 1;
2655                                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), hcintmsk.u32);
2656                                 usbc_hcchar.s.chdis = 1;
2657                                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index), usbc_hcchar.u32);
2658                                 return 0;
2659                         } else if (usbc_hcint.s.xfercompl) {
2660                                 /* Successful IN/OUT with transfer complete. Channel halt isn't needed */
2661                         } else {
2662                                 cvmx_dprintf("USB%d: Channel %d interrupt without halt\n", usb->index, channel);
2663                                 return 0;
2664                         }
2665                 }
2666         } else {
2667                 /*
2668                  * There is are no interrupts that we need to process when the
2669                  * channel is still running
2670                  */
2671                 if (!usbc_hcint.s.chhltd)
2672                         return 0;
2673         }
2674
2675         /* Disable the channel interrupts now that it is done */
2676         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
2677         usb->idle_hardware_channels |= (1<<channel);
2678
2679         /* Make sure this channel is tied to a valid pipe */
2680         pipe = usb->pipe_for_channel[channel];
2681         CVMX_PREFETCH(pipe, 0);
2682         CVMX_PREFETCH(pipe, 128);
2683         if (!pipe)
2684                 return 0;
2685         transaction = pipe->head;
2686         CVMX_PREFETCH0(transaction);
2687
2688         /*
2689          * Disconnect this pipe from the HW channel. Later the schedule
2690          * function will figure out which pipe needs to go
2691          */
2692         usb->pipe_for_channel[channel] = NULL;
2693         pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_SCHEDULED;
2694
2695         /*
2696          * Read the channel config info so we can figure out how much data
2697          * transfered
2698          */
2699         usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index));
2700         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index));
2701
2702         /*
2703          * Calculating the number of bytes successfully transferred is dependent
2704          * on the transfer direction
2705          */
2706         packets_processed = transaction->pktcnt - usbc_hctsiz.s.pktcnt;
2707         if (usbc_hcchar.s.epdir) {
2708                 /*
2709                  * IN transactions are easy. For every byte received the
2710                  * hardware decrements xfersize. All we need to do is subtract
2711                  * the current value of xfersize from its starting value and we
2712                  * know how many bytes were written to the buffer
2713                  */
2714                 bytes_this_transfer = transaction->xfersize - usbc_hctsiz.s.xfersize;
2715         } else {
2716                 /*
2717                  * OUT transaction don't decrement xfersize. Instead pktcnt is
2718                  * decremented on every successful packet send. The hardware
2719                  * does this when it receives an ACK, or NYET. If it doesn't
2720                  * receive one of these responses pktcnt doesn't change
2721                  */
2722                 bytes_this_transfer = packets_processed * usbc_hcchar.s.mps;
2723                 /*
2724                  * The last packet may not be a full transfer if we didn't have
2725                  * enough data
2726                  */
2727                 if (bytes_this_transfer > transaction->xfersize)
2728                         bytes_this_transfer = transaction->xfersize;
2729         }
2730         /* Figure out how many bytes were in the last packet of the transfer */
2731         if (packets_processed)
2732                 bytes_in_last_packet = bytes_this_transfer - (packets_processed-1) * usbc_hcchar.s.mps;
2733         else
2734                 bytes_in_last_packet = bytes_this_transfer;
2735
2736         /*
2737          * As a special case, setup transactions output the setup header, not
2738          * the user's data. For this reason we don't count setup data as bytes
2739          * transferred
2740          */
2741         if ((transaction->stage == CVMX_USB_STAGE_SETUP) ||
2742                 (transaction->stage == CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE))
2743                 bytes_this_transfer = 0;
2744
2745         /*
2746          * Add the bytes transferred to the running total. It is important that
2747          * bytes_this_transfer doesn't count any data that needs to be
2748          * retransmitted
2749          */
2750         transaction->actual_bytes += bytes_this_transfer;
2751         if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
2752                 buffer_space_left = transaction->iso_packets[0].length - transaction->actual_bytes;
2753         else
2754                 buffer_space_left = transaction->buffer_length - transaction->actual_bytes;
2755
2756         /*
2757          * We need to remember the PID toggle state for the next transaction.
2758          * The hardware already updated it for the next transaction
2759          */
2760         pipe->pid_toggle = !(usbc_hctsiz.s.pid == 0);
2761
2762         /*
2763          * For high speed bulk out, assume the next transaction will need to do
2764          * a ping before proceeding. If this isn't true the ACK processing below
2765          * will clear this flag
2766          */
2767         if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2768                 (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
2769                 (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT))
2770                 pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
2771
2772         if (usbc_hcint.s.stall) {
2773                 /*
2774                  * STALL as a response means this transaction cannot be
2775                  * completed because the device can't process transactions. Tell
2776                  * the user. Any data that was transferred will be counted on
2777                  * the actual bytes transferred
2778                  */
2779                 pipe->pid_toggle = 0;
2780                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_STALL);
2781         } else if (usbc_hcint.s.xacterr) {
2782                 /*
2783                  * We know at least one packet worked if we get a ACK or NAK.
2784                  * Reset the retry counter
2785                  */
2786                 if (usbc_hcint.s.nak || usbc_hcint.s.ack)
2787                         transaction->retries = 0;
2788                 transaction->retries++;
2789                 if (transaction->retries > MAX_RETRIES) {
2790                         /*
2791                          * XactErr as a response means the device signaled
2792                          * something wrong with the transfer. For example, PID
2793                          * toggle errors cause these
2794                          */
2795                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_XACTERR);
2796                 } else {
2797                         /*
2798                          * If this was a split then clear our split in progress
2799                          * marker
2800                          */
2801                         if (usb->active_split == transaction)
2802                                 usb->active_split = NULL;
2803                         /*
2804                          * Rewind to the beginning of the transaction by anding
2805                          * off the split complete bit
2806                          */
2807                         transaction->stage &= ~1;
2808                         pipe->split_sc_frame = -1;
2809                         pipe->next_tx_frame += pipe->interval;
2810                         if (pipe->next_tx_frame < usb->frame_number)
2811                                 pipe->next_tx_frame = usb->frame_number + pipe->interval -
2812                                                       (usb->frame_number - pipe->next_tx_frame) % pipe->interval;
2813                 }
2814         } else if (usbc_hcint.s.bblerr) {
2815                 /* Babble Error (BblErr) */
2816                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_BABBLEERR);
2817         } else if (usbc_hcint.s.datatglerr) {
2818                 /* We'll retry the exact same transaction again */
2819                 transaction->retries++;
2820         } else if (usbc_hcint.s.nyet) {
2821                 /*
2822                  * NYET as a response is only allowed in three cases: as a
2823                  * response to a ping, as a response to a split transaction, and
2824                  * as a response to a bulk out. The ping case is handled by
2825                  * hardware, so we only have splits and bulk out
2826                  */
2827                 if (!__cvmx_usb_pipe_needs_split(usb, pipe)) {
2828                         transaction->retries = 0;
2829                         /*
2830                          * If there is more data to go then we need to try
2831                          * again. Otherwise this transaction is complete
2832                          */
2833                         if ((buffer_space_left == 0) || (bytes_in_last_packet < pipe->max_packet))
2834                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2835                 } else {
2836                         /*
2837                          * Split transactions retry the split complete 4 times
2838                          * then rewind to the start split and do the entire
2839                          * transactions again
2840                          */
2841                         transaction->retries++;
2842                         if ((transaction->retries & 0x3) == 0) {
2843                                 /*
2844                                  * Rewind to the beginning of the transaction by
2845                                  * anding off the split complete bit
2846                                  */
2847                                 transaction->stage &= ~1;
2848                                 pipe->split_sc_frame = -1;
2849                         }
2850                 }
2851         } else if (usbc_hcint.s.ack) {
2852                 transaction->retries = 0;
2853                 /*
2854                  * The ACK bit can only be checked after the other error bits.
2855                  * This is because a multi packet transfer may succeed in a
2856                  * number of packets and then get a different response on the
2857                  * last packet. In this case both ACK and the last response bit
2858                  * will be set. If none of the other response bits is set, then
2859                  * the last packet must have been an ACK
2860                  *
2861                  * Since we got an ACK, we know we don't need to do a ping on
2862                  * this pipe
2863                  */
2864                 pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_NEED_PING;
2865
2866                 switch (transaction->type) {
2867                 case CVMX_USB_TRANSFER_CONTROL:
2868                         switch (transaction->stage) {
2869                         case CVMX_USB_STAGE_NON_CONTROL:
2870                         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
2871                                 /* This should be impossible */
2872                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_ERROR);
2873                                 break;
2874                         case CVMX_USB_STAGE_SETUP:
2875                                 pipe->pid_toggle = 1;
2876                                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
2877                                         transaction->stage = CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE;
2878                                 else {
2879                                         union cvmx_usb_control_header *header =
2880                                                 cvmx_phys_to_ptr(transaction->control_header);
2881                                         if (header->s.length)
2882                                                 transaction->stage = CVMX_USB_STAGE_DATA;
2883                                         else
2884                                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2885                                 }
2886                                 break;
2887                         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
2888                                 {
2889                                         union cvmx_usb_control_header *header =
2890                                                 cvmx_phys_to_ptr(transaction->control_header);
2891                                         if (header->s.length)
2892                                                 transaction->stage = CVMX_USB_STAGE_DATA;
2893                                         else
2894                                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2895                                 }
2896                                 break;
2897                         case CVMX_USB_STAGE_DATA:
2898                                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2899                                         transaction->stage = CVMX_USB_STAGE_DATA_SPLIT_COMPLETE;
2900                                         /*
2901                                          * For setup OUT data that are splits,
2902                                          * the hardware doesn't appear to count
2903                                          * transferred data. Here we manually
2904                                          * update the data transferred
2905                                          */
2906                                         if (!usbc_hcchar.s.epdir) {
2907                                                 if (buffer_space_left < pipe->max_packet)
2908                                                         transaction->actual_bytes += buffer_space_left;
2909                                                 else
2910                                                         transaction->actual_bytes += pipe->max_packet;
2911                                         }
2912                                 } else if ((buffer_space_left == 0) || (bytes_in_last_packet < pipe->max_packet)) {
2913                                         pipe->pid_toggle = 1;
2914                                         transaction->stage = CVMX_USB_STAGE_STATUS;
2915                                 }
2916                                 break;
2917                         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
2918                                 if ((buffer_space_left == 0) || (bytes_in_last_packet < pipe->max_packet)) {
2919                                         pipe->pid_toggle = 1;
2920                                         transaction->stage = CVMX_USB_STAGE_STATUS;
2921                                 } else {
2922                                         transaction->stage = CVMX_USB_STAGE_DATA;
2923                                 }
2924                                 break;
2925                         case CVMX_USB_STAGE_STATUS:
2926                                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
2927                                         transaction->stage = CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE;
2928                                 else
2929                                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2930                                 break;
2931                         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
2932                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2933                                 break;
2934                         }
2935                         break;
2936                 case CVMX_USB_TRANSFER_BULK:
2937                 case CVMX_USB_TRANSFER_INTERRUPT:
2938                         /*
2939                          * The only time a bulk transfer isn't complete when it
2940                          * finishes with an ACK is during a split transaction.
2941                          * For splits we need to continue the transfer if more
2942                          * data is needed
2943                          */
2944                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2945                                 if (transaction->stage == CVMX_USB_STAGE_NON_CONTROL)
2946                                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2947                                 else {
2948                                         if (buffer_space_left && (bytes_in_last_packet == pipe->max_packet))
2949                                                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2950                                         else {
2951                                                 if (transaction->type == CVMX_USB_TRANSFER_INTERRUPT)
2952                                                         pipe->next_tx_frame += pipe->interval;
2953                                                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2954                                         }
2955                                 }
2956                         } else {
2957                                 if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2958                                     (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
2959                                     (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) &&
2960                                     (usbc_hcint.s.nak))
2961                                         pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
2962                                 if (!buffer_space_left || (bytes_in_last_packet < pipe->max_packet)) {
2963                                         if (transaction->type == CVMX_USB_TRANSFER_INTERRUPT)
2964                                                 pipe->next_tx_frame += pipe->interval;
2965                                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2966                                 }
2967                         }
2968                         break;
2969                 case CVMX_USB_TRANSFER_ISOCHRONOUS:
2970                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2971                                 /*
2972                                  * ISOCHRONOUS OUT splits don't require a
2973                                  * complete split stage. Instead they use a
2974                                  * sequence of begin OUT splits to transfer the
2975                                  * data 188 bytes at a time. Once the transfer
2976                                  * is complete, the pipe sleeps until the next
2977                                  * schedule interval
2978                                  */
2979                                 if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
2980                                         /*
2981                                          * If no space left or this wasn't a max
2982                                          * size packet then this transfer is
2983                                          * complete. Otherwise start it again to
2984                                          * send the next 188 bytes
2985                                          */
2986                                         if (!buffer_space_left || (bytes_this_transfer < 188)) {
2987                                                 pipe->next_tx_frame += pipe->interval;
2988                                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2989                                         }
2990                                 } else {
2991                                         if (transaction->stage == CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE) {
2992                                                 /*
2993                                                  * We are in the incoming data
2994                                                  * phase. Keep getting data
2995                                                  * until we run out of space or
2996                                                  * get a small packet
2997                                                  */
2998                                                 if ((buffer_space_left == 0) || (bytes_in_last_packet < pipe->max_packet)) {
2999                                                         pipe->next_tx_frame += pipe->interval;
3000                                                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
3001                                                 }
3002                                         } else
3003                                                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
3004                                 }
3005                         } else {
3006                                 pipe->next_tx_frame += pipe->interval;
3007                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
3008                         }
3009                         break;
3010                 }
3011         } else if (usbc_hcint.s.nak) {
3012                 /* If this was a split then clear our split in progress marker */
3013                 if (usb->active_split == transaction)
3014                         usb->active_split = NULL;
3015                 /*
3016                  * NAK as a response means the device couldn't accept the
3017                  * transaction, but it should be retried in the future. Rewind
3018                  * to the beginning of the transaction by anding off the split
3019                  * complete bit. Retry in the next interval
3020                  */
3021                 transaction->retries = 0;
3022                 transaction->stage &= ~1;
3023                 pipe->next_tx_frame += pipe->interval;
3024                 if (pipe->next_tx_frame < usb->frame_number)
3025                         pipe->next_tx_frame = usb->frame_number + pipe->interval -
3026                                 (usb->frame_number - pipe->next_tx_frame) % pipe->interval;
3027         } else {
3028                 struct cvmx_usb_port_status port;
3029                 port = cvmx_usb_get_status((struct cvmx_usb_state *)usb);
3030                 if (port.port_enabled) {
3031                         /* We'll retry the exact same transaction again */
3032                         transaction->retries++;
3033                 } else {
3034                         /*
3035                          * We get channel halted interrupts with no result bits
3036                          * sets when the cable is unplugged
3037                          */
3038                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_ERROR);
3039                 }
3040         }
3041         return 0;
3042 }
3043
3044
3045 /**
3046  * Poll the USB block for status and call all needed callback
3047  * handlers. This function is meant to be called in the interrupt
3048  * handler for the USB controller. It can also be called
3049  * periodically in a loop for non-interrupt based operation.
3050  *
3051  * @state:      USB device state populated by
3052  *              cvmx_usb_initialize().
3053  *
3054  * Returns: 0 or a negative error code.
3055  */
3056 int cvmx_usb_poll(struct cvmx_usb_state *state)
3057 {
3058         union cvmx_usbcx_hfnum usbc_hfnum;
3059         union cvmx_usbcx_gintsts usbc_gintsts;
3060         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
3061
3062         CVMX_PREFETCH(usb, 0);
3063         CVMX_PREFETCH(usb, 1*128);
3064         CVMX_PREFETCH(usb, 2*128);
3065         CVMX_PREFETCH(usb, 3*128);
3066         CVMX_PREFETCH(usb, 4*128);
3067
3068         /* Update the frame counter */
3069         usbc_hfnum.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
3070         if ((usb->frame_number&0x3fff) > usbc_hfnum.s.frnum)
3071                 usb->frame_number += 0x4000;
3072         usb->frame_number &= ~0x3fffull;
3073         usb->frame_number |= usbc_hfnum.s.frnum;
3074
3075         /* Read the pending interrupts */
3076         usbc_gintsts.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GINTSTS(usb->index));
3077
3078         /* Clear the interrupts now that we know about them */
3079         __cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTSTS(usb->index), usbc_gintsts.u32);
3080
3081         if (usbc_gintsts.s.rxflvl) {
3082                 /*
3083                  * RxFIFO Non-Empty (RxFLvl)
3084                  * Indicates that there is at least one packet pending to be
3085                  * read from the RxFIFO.
3086                  *
3087                  * In DMA mode this is handled by hardware
3088                  */
3089                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
3090                         __cvmx_usb_poll_rx_fifo(usb);
3091         }
3092         if (usbc_gintsts.s.ptxfemp || usbc_gintsts.s.nptxfemp) {
3093                 /* Fill the Tx FIFOs when not in DMA mode */
3094                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
3095                         __cvmx_usb_poll_tx_fifo(usb);
3096         }
3097         if (usbc_gintsts.s.disconnint || usbc_gintsts.s.prtint) {
3098                 union cvmx_usbcx_hprt usbc_hprt;
3099                 /*
3100                  * Disconnect Detected Interrupt (DisconnInt)
3101                  * Asserted when a device disconnect is detected.
3102                  *
3103                  * Host Port Interrupt (PrtInt)
3104                  * The core sets this bit to indicate a change in port status of
3105                  * one of the O2P USB core ports in Host mode. The application
3106                  * must read the Host Port Control and Status (HPRT) register to
3107                  * determine the exact event that caused this interrupt. The
3108                  * application must clear the appropriate status bit in the Host
3109                  * Port Control and Status register to clear this bit.
3110                  *
3111                  * Call the user's port callback
3112                  */
3113                 __cvmx_usb_perform_callback(usb, NULL, NULL,
3114                                             CVMX_USB_CALLBACK_PORT_CHANGED,
3115                                             CVMX_USB_COMPLETE_SUCCESS);
3116                 /* Clear the port change bits */
3117                 usbc_hprt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
3118                 usbc_hprt.s.prtena = 0;
3119                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HPRT(usb->index), usbc_hprt.u32);
3120         }
3121         if (usbc_gintsts.s.hchint) {
3122                 /*
3123                  * Host Channels Interrupt (HChInt)
3124                  * The core sets this bit to indicate that an interrupt is
3125                  * pending on one of the channels of the core (in Host mode).
3126                  * The application must read the Host All Channels Interrupt
3127                  * (HAINT) register to determine the exact number of the channel
3128                  * on which the interrupt occurred, and then read the
3129                  * corresponding Host Channel-n Interrupt (HCINTn) register to
3130                  * determine the exact cause of the interrupt. The application
3131                  * must clear the appropriate status bit in the HCINTn register
3132                  * to clear this bit.
3133                  */
3134                 union cvmx_usbcx_haint usbc_haint;
3135                 usbc_haint.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HAINT(usb->index));
3136                 while (usbc_haint.u32) {
3137                         int channel;
3138                         CVMX_CLZ(channel, usbc_haint.u32);
3139                         channel = 31 - channel;
3140                         __cvmx_usb_poll_channel(usb, channel);
3141                         usbc_haint.u32 ^= 1<<channel;
3142                 }
3143         }
3144
3145         __cvmx_usb_schedule(usb, usbc_gintsts.s.sof);
3146
3147         return 0;
3148 }