Pileus Git - ~andy/linux/blob - drivers/staging/octeon-usb/cvmx-usb.c

   1 /***********************license start***************
   2  * Copyright (c) 2003-2010  Cavium Networks (support@cavium.com). All rights
   3  * reserved.
   4  *
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions are
   8  * met:
   9  *
  10  *   * Redistributions of source code must retain the above copyright
  11  *     notice, this list of conditions and the following disclaimer.
  12  *
  13  *   * Redistributions in binary form must reproduce the above
  14  *     copyright notice, this list of conditions and the following
  15  *     disclaimer in the documentation and/or other materials provided
  16  *     with the distribution.
  17
  18  *   * Neither the name of Cavium Networks nor the names of
  19  *     its contributors may be used to endorse or promote products
  20  *     derived from this software without specific prior written
  21  *     permission.
  22
  23  * This Software, including technical data, may be subject to U.S. export  control
  24  * laws, including the U.S. Export Administration Act and its  associated
  25  * regulations, and may be subject to export or import  regulations in other
  26  * countries.
  27
  28  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
  29  * AND WITH ALL FAULTS AND CAVIUM  NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR
  30  * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
  31  * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR
  32  * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
  33  * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
  34  * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
  35  * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
  36  * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE  RISK ARISING OUT OF USE OR
  37  * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
  38  ***********************license end**************************************/
  39
  40
  41 /**
  42  * @file
  43  *
  44  * "cvmx-usb.c" defines a set of low level USB functions to help
  45  * developers create Octeon USB drivers for various operating
  46  * systems. These functions provide a generic API to the Octeon
  47  * USB blocks, hiding the internal hardware specific
  48  * operations.
  49  */
  50 #include <linux/delay.h>
  51 #include <asm/octeon/cvmx.h>
  52 #include <asm/octeon/octeon.h>
  53 #include <asm/octeon/cvmx-sysinfo.h>
  54 #include "cvmx-usbnx-defs.h"
  55 #include "cvmx-usbcx-defs.h"
  56 #include "cvmx-usb.h"
  57 #include <asm/octeon/cvmx-helper.h>
  58 #include <asm/octeon/cvmx-helper-board.h>
  59
  60 #define CVMX_PREFETCH128(address) CVMX_PREFETCH(address, 128)
  61 // a normal prefetch
  62 #define CVMX_PREFETCH(address, offset) CVMX_PREFETCH_PREF0(address, offset)
  63 // normal prefetches that use the pref instruction
  64 #define CVMX_PREFETCH_PREFX(X, address, offset) asm volatile ("pref %[type], %[off](%[rbase])" : : [rbase] "d" (address), [off] "I" (offset), [type] "n" (X))
  65 #define CVMX_PREFETCH_PREF0(address, offset) CVMX_PREFETCH_PREFX(0, address, offset)
  66
  67 #define MAX_RETRIES             3               /* Maximum number of times to retry failed transactions */
  68 #define MAX_PIPES               32              /* Maximum number of pipes that can be open at once */
  69 #define MAX_TRANSACTIONS        256             /* Maximum number of outstanding transactions across all pipes */
  70 #define MAX_CHANNELS            8               /* Maximum number of hardware channels supported by the USB block */
  71 #define MAX_USB_ADDRESS         127             /* The highest valid USB device address */
  72 #define MAX_USB_ENDPOINT        15              /* The highest valid USB endpoint number */
  73 #define MAX_USB_HUB_PORT        15              /* The highest valid port number on a hub */
  74 #define MAX_TRANSFER_BYTES      ((1<<19)-1)     /* The low level hardware can transfer a maximum of this number of bytes in each transfer. The field is 19 bits wide */
  75 #define MAX_TRANSFER_PACKETS    ((1<<10)-1)     /* The low level hardware can transfer a maximum of this number of packets in each transfer. The field is 10 bits wide */
  76
  77 /*
  78  * These defines disable the normal read and write csr. This is so I can add
  79  * extra debug stuff to the usb specific version and I won't use the normal
  80  * version by mistake
  81  */
  82 #define cvmx_read_csr use_cvmx_usb_read_csr64_instead_of_cvmx_read_csr
  83 #define cvmx_write_csr use_cvmx_usb_write_csr64_instead_of_cvmx_write_csr
  84
  85 enum cvmx_usb_transaction_flags {
  86         __CVMX_USB_TRANSACTION_FLAGS_IN_USE = 1<<16,
  87 };
  88
  89 enum {
  90         USB_CLOCK_TYPE_REF_12,
  91         USB_CLOCK_TYPE_REF_24,
  92         USB_CLOCK_TYPE_REF_48,
  93         USB_CLOCK_TYPE_CRYSTAL_12,
  94 };
  95
  96 /**
  97  * Logical transactions may take numerous low level
  98  * transactions, especially when splits are concerned. This
  99  * enum represents all of the possible stages a transaction can
 100  * be in. Note that split completes are always even. This is so
 101  * the NAK handler can backup to the previous low level
 102  * transaction with a simple clearing of bit 0.
 103  */
 104 enum cvmx_usb_stage {
 105         CVMX_USB_STAGE_NON_CONTROL,
 106         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE,
 107         CVMX_USB_STAGE_SETUP,
 108         CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE,
 109         CVMX_USB_STAGE_DATA,
 110         CVMX_USB_STAGE_DATA_SPLIT_COMPLETE,
 111         CVMX_USB_STAGE_STATUS,
 112         CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE,
 113 };
 114
 115 /**
 116  * struct cvmx_usb_transaction - describes each pending USB transaction
 117  *                               regardless of type. These are linked together
 118  *                               to form a list of pending requests for a pipe.
 119  *
 120  * @prev:               Transaction before this one in the pipe.
 121  * @next:               Transaction after this one in the pipe.
 122  * @type:               Type of transaction, duplicated of the pipe.
 123  * @flags:              State flags for this transaction.
 124  * @buffer:             User's physical buffer address to read/write.
 125  * @buffer_length:      Size of the user's buffer in bytes.
 126  * @control_header:     For control transactions, physical address of the 8
 127  *                      byte standard header.
 128  * @iso_start_frame:    For ISO transactions, the starting frame number.
 129  * @iso_number_packets: For ISO transactions, the number of packets in the
 130  *                      request.
 131  * @iso_packets:        For ISO transactions, the sub packets in the request.
 132  * @actual_bytes:       Actual bytes transfer for this transaction.
 133  * @stage:              For control transactions, the current stage.
 134  * @callback:           User's callback function when complete.
 135  * @callback_data:      User's data.
 136  */
 137 struct cvmx_usb_transaction {
 138         struct cvmx_usb_transaction *prev;
 139         struct cvmx_usb_transaction *next;
 140         enum cvmx_usb_transfer type;
 141         enum cvmx_usb_transaction_flags flags;
 142         uint64_t buffer;
 143         int buffer_length;
 144         uint64_t control_header;
 145         int iso_start_frame;
 146         int iso_number_packets;
 147         struct cvmx_usb_iso_packet *iso_packets;
 148         int xfersize;
 149         int pktcnt;
 150         int retries;
 151         int actual_bytes;
 152         enum cvmx_usb_stage stage;
 153         cvmx_usb_callback_func_t callback;
 154         void *callback_data;
 155 };
 156
 157 /**
 158  * struct cvmx_usb_pipe - a pipe represents a virtual connection between Octeon
 159  *                        and some USB device. It contains a list of pending
 160  *                        request to the device.
 161  *
 162  * @prev:               Pipe before this one in the list
 163  * @next:               Pipe after this one in the list
 164  * @head:               The first pending transaction
 165  * @tail:               The last pending transaction
 166  * @interval:           For periodic pipes, the interval between packets in
 167  *                      frames
 168  * @next_tx_frame:      The next frame this pipe is allowed to transmit on
 169  * @flags:              State flags for this pipe
 170  * @device_speed:       Speed of device connected to this pipe
 171  * @transfer_type:      Type of transaction supported by this pipe
 172  * @transfer_dir:       IN or OUT. Ignored for Control
 173  * @multi_count:        Max packet in a row for the device
 174  * @max_packet:         The device's maximum packet size in bytes
 175  * @device_addr:        USB device address at other end of pipe
 176  * @endpoint_num:       USB endpoint number at other end of pipe
 177  * @hub_device_addr:    Hub address this device is connected to
 178  * @hub_port:           Hub port this device is connected to
 179  * @pid_toggle:         This toggles between 0/1 on every packet send to track
 180  *                      the data pid needed
 181  * @channel:            Hardware DMA channel for this pipe
 182  * @split_sc_frame:     The low order bits of the frame number the split
 183  *                      complete should be sent on
 184  */
 185 struct cvmx_usb_pipe {
 186         struct cvmx_usb_pipe *prev;
 187         struct cvmx_usb_pipe *next;
 188         struct cvmx_usb_transaction *head;
 189         struct cvmx_usb_transaction *tail;
 190         uint64_t interval;
 191         uint64_t next_tx_frame;
 192         enum cvmx_usb_pipe_flags flags;
 193         enum cvmx_usb_speed device_speed;
 194         enum cvmx_usb_transfer transfer_type;
 195         enum cvmx_usb_direction transfer_dir;
 196         int multi_count;
 197         uint16_t max_packet;
 198         uint8_t device_addr;
 199         uint8_t endpoint_num;
 200         uint8_t hub_device_addr;
 201         uint8_t hub_port;
 202         uint8_t pid_toggle;
 203         uint8_t channel;
 204         int8_t split_sc_frame;
 205 };
 206
 207 /**
 208  * struct cvmx_usb_pipe_list
 209  *
 210  * @head: Head of the list, or NULL if empty.
 211  * @tail: Tail if the list, or NULL if empty.
 212  */
 213 struct cvmx_usb_pipe_list {
 214         struct cvmx_usb_pipe *head;
 215         struct cvmx_usb_pipe *tail;
 216 };
 217
 218 struct cvmx_usb_tx_fifo {
 219         struct {
 220                 int channel;
 221                 int size;
 222                 uint64_t address;
 223         } entry[MAX_CHANNELS+1];
 224         int head;
 225         int tail;
 226 };
 227
 228 /**
 229  * struct cvmx_usb_internal_state - the state of the USB block
 230  *
 231  * init_flags:             Flags passed to initialize.
 232  * index:                  Which USB block this is for.
 233  * idle_hardware_channels: Bit set for every idle hardware channel.
 234  * usbcx_hprt:             Stored port status so we don't need to read a CSR to
 235  *                         determine splits.
 236  * pipe_for_channel:       Map channels to pipes.
 237  * free_transaction_head:  List of free transactions head.
 238  * free_transaction_tail:  List of free transactions tail.
 239  * pipe:                   Storage for pipes.
 240  * transaction:            Storage for transactions.
 241  * callback:               User global callbacks.
 242  * callback_data:          User data for each callback.
 243  * indent:                 Used by debug output to indent functions.
 244  * port_status:            Last port status used for change notification.
 245  * free_pipes:             List of all pipes that are currently closed.
 246  * idle_pipes:             List of open pipes that have no transactions.
 247  * active_pipes:           Active pipes indexed by transfer type.
 248  * frame_number:           Increments every SOF interrupt for time keeping.
 249  * active_split:           Points to the current active split, or NULL.
 250  */
 251 struct cvmx_usb_internal_state {
 252         int init_flags;
 253         int index;
 254         int idle_hardware_channels;
 255         union cvmx_usbcx_hprt usbcx_hprt;
 256         struct cvmx_usb_pipe *pipe_for_channel[MAX_CHANNELS];
 257         struct cvmx_usb_transaction *free_transaction_head;
 258         struct cvmx_usb_transaction *free_transaction_tail;
 259         struct cvmx_usb_pipe pipe[MAX_PIPES];
 260         struct cvmx_usb_transaction transaction[MAX_TRANSACTIONS];
 261         cvmx_usb_callback_func_t callback[__CVMX_USB_CALLBACK_END];
 262         void *callback_data[__CVMX_USB_CALLBACK_END];
 263         int indent;
 264         struct cvmx_usb_port_status port_status;
 265         struct cvmx_usb_pipe_list free_pipes;
 266         struct cvmx_usb_pipe_list idle_pipes;
 267         struct cvmx_usb_pipe_list active_pipes[4];
 268         uint64_t frame_number;
 269         struct cvmx_usb_transaction *active_split;
 270         struct cvmx_usb_tx_fifo periodic;
 271         struct cvmx_usb_tx_fifo nonperiodic;
 272 };
 273
 274 /* This macro spins on a field waiting for it to reach a value */
 275 #define CVMX_WAIT_FOR_FIELD32(address, type, field, op, value, timeout_usec)\
 276         ({int result;                                                       \
 277         do {                                                                \
 278                 uint64_t done = cvmx_get_cycle() + (uint64_t)timeout_usec * \
 279                         octeon_get_clock_rate() / 1000000;                  \
 280                 type c;                                                     \
 281                 while (1) {                                                 \
 282                         c.u32 = __cvmx_usb_read_csr32(usb, address);        \
 283                         if (c.s.field op (value)) {                         \
 284                                 result = 0;                                 \
 285                                 break;                                      \
 286                         } else if (cvmx_get_cycle() > done) {               \
 287                                 result = -1;                                \
 288                                 break;                                      \
 289                         } else                                              \
 290                                 cvmx_wait(100);                             \
 291                 }                                                           \
 292         } while (0);                                                        \
 293         result; })
 294
 295 /*
 296  * This macro logically sets a single field in a CSR. It does the sequence
 297  * read, modify, and write
 298  */
 299 #define USB_SET_FIELD32(address, type, field, value)            \
 300         do {                                                    \
 301                 type c;                                         \
 302                 c.u32 = __cvmx_usb_read_csr32(usb, address);    \
 303                 c.s.field = value;                              \
 304                 __cvmx_usb_write_csr32(usb, address, c.u32);    \
 305         } while (0)
 306
 307 /* Returns the IO address to push/pop stuff data from the FIFOs */
 308 #define USB_FIFO_ADDRESS(channel, usb_index) (CVMX_USBCX_GOTGCTL(usb_index) + ((channel)+1)*0x1000)
 309
 310 static int octeon_usb_get_clock_type(void)
 311 {
 312         switch (cvmx_sysinfo_get()->board_type) {
 313         case CVMX_BOARD_TYPE_BBGW_REF:
 314         case CVMX_BOARD_TYPE_LANAI2_A:
 315         case CVMX_BOARD_TYPE_LANAI2_U:
 316         case CVMX_BOARD_TYPE_LANAI2_G:
 317         case CVMX_BOARD_TYPE_UBNT_E100:
 318                 return USB_CLOCK_TYPE_CRYSTAL_12;
 319         }
 320         return USB_CLOCK_TYPE_REF_48;
 321 }
 322
 323 /**
 324  * Read a USB 32bit CSR. It performs the necessary address swizzle
 325  * for 32bit CSRs and logs the value in a readable format if
 326  * debugging is on.
 327  *
 328  * @usb:     USB block this access is for
 329  * @address: 64bit address to read
 330  *
 331  * Returns: Result of the read
 332  */
 333 static inline uint32_t __cvmx_usb_read_csr32(struct cvmx_usb_internal_state *usb,
 334                                              uint64_t address)
 335 {
 336         uint32_t result = cvmx_read64_uint32(address ^ 4);
 337         return result;
 338 }
 339
 340
 341 /**
 342  * Write a USB 32bit CSR. It performs the necessary address
 343  * swizzle for 32bit CSRs and logs the value in a readable format
 344  * if debugging is on.
 345  *
 346  * @usb:     USB block this access is for
 347  * @address: 64bit address to write
 348  * @value:   Value to write
 349  */
 350 static inline void __cvmx_usb_write_csr32(struct cvmx_usb_internal_state *usb,
 351                                           uint64_t address, uint32_t value)
 352 {
 353         cvmx_write64_uint32(address ^ 4, value);
 354         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
 355 }
 356
 357
 358 /**
 359  * Read a USB 64bit CSR. It logs the value in a readable format if
 360  * debugging is on.
 361  *
 362  * @usb:     USB block this access is for
 363  * @address: 64bit address to read
 364  *
 365  * Returns: Result of the read
 366  */
 367 static inline uint64_t __cvmx_usb_read_csr64(struct cvmx_usb_internal_state *usb,
 368                                              uint64_t address)
 369 {
 370         uint64_t result = cvmx_read64_uint64(address);
 371         return result;
 372 }
 373
 374
 375 /**
 376  * Write a USB 64bit CSR. It logs the value in a readable format
 377  * if debugging is on.
 378  *
 379  * @usb:     USB block this access is for
 380  * @address: 64bit address to write
 381  * @value:   Value to write
 382  */
 383 static inline void __cvmx_usb_write_csr64(struct cvmx_usb_internal_state *usb,
 384                                           uint64_t address, uint64_t value)
 385 {
 386         cvmx_write64_uint64(address, value);
 387 }
 388
 389 /**
 390  * Return non zero if this pipe connects to a non HIGH speed
 391  * device through a high speed hub.
 392  *
 393  * @usb:    USB block this access is for
 394  * @pipe:   Pipe to check
 395  *
 396  * Returns: Non zero if we need to do split transactions
 397  */
 398 static inline int __cvmx_usb_pipe_needs_split(struct cvmx_usb_internal_state *usb, struct cvmx_usb_pipe *pipe)
 399 {
 400         return ((pipe->device_speed != CVMX_USB_SPEED_HIGH) && (usb->usbcx_hprt.s.prtspd == CVMX_USB_SPEED_HIGH));
 401 }
 402
 403
 404 /**
 405  * Trivial utility function to return the correct PID for a pipe
 406  *
 407  * @pipe:   pipe to check
 408  *
 409  * Returns: PID for pipe
 410  */
 411 static inline int __cvmx_usb_get_data_pid(struct cvmx_usb_pipe *pipe)
 412 {
 413         if (pipe->pid_toggle)
 414                 return 2; /* Data1 */
 415         else
 416                 return 0; /* Data0 */
 417 }
 418
 419
 420 /**
 421  * Return the number of USB ports supported by this Octeon
 422  * chip. If the chip doesn't support USB, or is not supported
 423  * by this API, a zero will be returned. Most Octeon chips
 424  * support one usb port, but some support two ports.
 425  * cvmx_usb_initialize() must be called on independent
 426  * struct cvmx_usb_state.
 427  *
 428  * Returns: Number of port, zero if usb isn't supported
 429  */
 430 int cvmx_usb_get_num_ports(void)
 431 {
 432         int arch_ports = 0;
 433
 434         if (OCTEON_IS_MODEL(OCTEON_CN56XX))
 435                 arch_ports = 1;
 436         else if (OCTEON_IS_MODEL(OCTEON_CN52XX))
 437                 arch_ports = 2;
 438         else if (OCTEON_IS_MODEL(OCTEON_CN50XX))
 439                 arch_ports = 1;
 440         else if (OCTEON_IS_MODEL(OCTEON_CN31XX))
 441                 arch_ports = 1;
 442         else if (OCTEON_IS_MODEL(OCTEON_CN30XX))
 443                 arch_ports = 1;
 444         else
 445                 arch_ports = 0;
 446
 447         return arch_ports;
 448 }
 449
 450
 451 /**
 452  * Allocate a usb transaction for use
 453  *
 454  * @usb:         USB device state populated by
 455  *               cvmx_usb_initialize().
 456  *
 457  * Returns: Transaction or NULL
 458  */
 459 static inline struct cvmx_usb_transaction *__cvmx_usb_alloc_transaction(struct cvmx_usb_internal_state *usb)
 460 {
 461         struct cvmx_usb_transaction *t;
 462         t = usb->free_transaction_head;
 463         if (t) {
 464                 usb->free_transaction_head = t->next;
 465                 if (!usb->free_transaction_head)
 466                         usb->free_transaction_tail = NULL;
 467         }
 468         if (t) {
 469                 memset(t, 0, sizeof(*t));
 470                 t->flags = __CVMX_USB_TRANSACTION_FLAGS_IN_USE;
 471         }
 472         return t;
 473 }
 474
 475
 476 /**
 477  * Free a usb transaction
 478  *
 479  * @usb:         USB device state populated by
 480  *               cvmx_usb_initialize().
 481  * @transaction:
 482  *               Transaction to free
 483  */
 484 static inline void __cvmx_usb_free_transaction(struct cvmx_usb_internal_state *usb,
 485                                                struct cvmx_usb_transaction *transaction)
 486 {
 487         transaction->flags = 0;
 488         transaction->prev = NULL;
 489         transaction->next = NULL;
 490         if (usb->free_transaction_tail)
 491                 usb->free_transaction_tail->next = transaction;
 492         else
 493                 usb->free_transaction_head = transaction;
 494         usb->free_transaction_tail = transaction;
 495 }
 496
 497
 498 /**
 499  * Add a pipe to the tail of a list
 500  * @list:   List to add pipe to
 501  * @pipe:   Pipe to add
 502  */
 503 static inline void __cvmx_usb_append_pipe(struct cvmx_usb_pipe_list *list, struct cvmx_usb_pipe *pipe)
 504 {
 505         pipe->next = NULL;
 506         pipe->prev = list->tail;
 507         if (list->tail)
 508                 list->tail->next = pipe;
 509         else
 510                 list->head = pipe;
 511         list->tail = pipe;
 512 }
 513
 514
 515 /**
 516  * Remove a pipe from a list
 517  * @list:   List to remove pipe from
 518  * @pipe:   Pipe to remove
 519  */
 520 static inline void __cvmx_usb_remove_pipe(struct cvmx_usb_pipe_list *list, struct cvmx_usb_pipe *pipe)
 521 {
 522         if (list->head == pipe) {
 523                 list->head = pipe->next;
 524                 pipe->next = NULL;
 525                 if (list->head)
 526                         list->head->prev = NULL;
 527                 else
 528                         list->tail = NULL;
 529         } else if (list->tail == pipe) {
 530                 list->tail = pipe->prev;
 531                 list->tail->next = NULL;
 532                 pipe->prev = NULL;
 533         } else {
 534                 pipe->prev->next = pipe->next;
 535                 pipe->next->prev = pipe->prev;
 536                 pipe->prev = NULL;
 537                 pipe->next = NULL;
 538         }
 539 }
 540
 541
 542 /**
 543  * Initialize a USB port for use. This must be called before any
 544  * other access to the Octeon USB port is made. The port starts
 545  * off in the disabled state.
 546  *
 547  * @state:       Pointer to an empty struct cvmx_usb_state
 548  *               that will be populated by the initialize call.
 549  *               This structure is then passed to all other USB
 550  *               functions.
 551  * @usb_port_number:
 552  *               Which Octeon USB port to initialize.
 553  *
 554  * Returns: 0 or a negative error code.
 555  */
 556 int cvmx_usb_initialize(struct cvmx_usb_state *state, int usb_port_number)
 557 {
 558         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 559         union cvmx_usbnx_usbp_ctl_status usbn_usbp_ctl_status;
 560         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
 561         enum cvmx_usb_initialize_flags flags = 0;
 562
 563         /* Make sure that state is large enough to store the internal state */
 564         if (sizeof(*state) < sizeof(*usb))
 565                 return -EINVAL;
 566         /* At first allow 0-1 for the usb port number */
 567         if ((usb_port_number < 0) || (usb_port_number > 1))
 568                 return -EINVAL;
 569         /* For all chips except 52XX there is only one port */
 570         if (!OCTEON_IS_MODEL(OCTEON_CN52XX) && (usb_port_number > 0))
 571                 return -EINVAL;
 572         /* Try to determine clock type automatically */
 573         if (octeon_usb_get_clock_type() == USB_CLOCK_TYPE_CRYSTAL_12) {
 574                 /* Only 12 MHZ crystals are supported */
 575                 flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI;
 576         } else {
 577                 flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND;
 578
 579                 switch (octeon_usb_get_clock_type()) {
 580                 case USB_CLOCK_TYPE_REF_12:
 581                         flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ;
 582                         break;
 583                 case USB_CLOCK_TYPE_REF_24:
 584                         flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ;
 585                         break;
 586                 case USB_CLOCK_TYPE_REF_48:
 587                         flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ;
 588                         break;
 589                 default:
 590                         return -EINVAL;
 591                         break;
 592                 }
 593         }
 594
 595         memset(usb, 0, sizeof(*usb));
 596         usb->init_flags = flags;
 597
 598         /* Initialize the USB state structure */
 599         {
 600                 int i;
 601                 usb->index = usb_port_number;
 602
 603                 /* Initialize the transaction double linked list */
 604                 usb->free_transaction_head = NULL;
 605                 usb->free_transaction_tail = NULL;
 606                 for (i = 0; i < MAX_TRANSACTIONS; i++)
 607                         __cvmx_usb_free_transaction(usb, usb->transaction + i);
 608                 for (i = 0; i < MAX_PIPES; i++)
 609                         __cvmx_usb_append_pipe(&usb->free_pipes, usb->pipe + i);
 610         }
 611
 612         /*
 613          * Power On Reset and PHY Initialization
 614          *
 615          * 1. Wait for DCOK to assert (nothing to do)
 616          *
 617          * 2a. Write USBN0/1_CLK_CTL[POR] = 1 and
 618          *     USBN0/1_CLK_CTL[HRST,PRST,HCLK_RST] = 0
 619          */
 620         usbn_clk_ctl.u64 = __cvmx_usb_read_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index));
 621         usbn_clk_ctl.s.por = 1;
 622         usbn_clk_ctl.s.hrst = 0;
 623         usbn_clk_ctl.s.prst = 0;
 624         usbn_clk_ctl.s.hclk_rst = 0;
 625         usbn_clk_ctl.s.enable = 0;
 626         /*
 627          * 2b. Select the USB reference clock/crystal parameters by writing
 628          *     appropriate values to USBN0/1_CLK_CTL[P_C_SEL, P_RTYPE, P_COM_ON]
 629          */
 630         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND) {
 631                 /*
 632                  * The USB port uses 12/24/48MHz 2.5V board clock
 633                  * source at USB_XO. USB_XI should be tied to GND.
 634                  * Most Octeon evaluation boards require this setting
 635                  */
 636                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX)) {
 637                         usbn_clk_ctl.cn31xx.p_rclk  = 1; /* From CN31XX,CN30XX manual */
 638                         usbn_clk_ctl.cn31xx.p_xenbn = 0;
 639                 } else if (OCTEON_IS_MODEL(OCTEON_CN56XX) || OCTEON_IS_MODEL(OCTEON_CN50XX))
 640                         usbn_clk_ctl.cn56xx.p_rtype = 2; /* From CN56XX,CN50XX manual */
 641                 else
 642                         usbn_clk_ctl.cn52xx.p_rtype = 1; /* From CN52XX manual */
 643
 644                 switch (flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK) {
 645                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:
 646                         usbn_clk_ctl.s.p_c_sel = 0;
 647                         break;
 648                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:
 649                         usbn_clk_ctl.s.p_c_sel = 1;
 650                         break;
 651                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:
 652                         usbn_clk_ctl.s.p_c_sel = 2;
 653                         break;
 654                 }
 655         } else {
 656                 /*
 657                  * The USB port uses a 12MHz crystal as clock source
 658                  * at USB_XO and USB_XI
 659                  */
 660                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX)) {
 661                         usbn_clk_ctl.cn31xx.p_rclk  = 1; /* From CN31XX,CN30XX manual */
 662                         usbn_clk_ctl.cn31xx.p_xenbn = 1;
 663                 } else if (OCTEON_IS_MODEL(OCTEON_CN56XX) || OCTEON_IS_MODEL(OCTEON_CN50XX))
 664                         usbn_clk_ctl.cn56xx.p_rtype = 0; /* From CN56XX,CN50XX manual */
 665                 else
 666                         usbn_clk_ctl.cn52xx.p_rtype = 0; /* From CN52XX manual */
 667
 668                 usbn_clk_ctl.s.p_c_sel = 0;
 669         }
 670         /*
 671          * 2c. Select the HCLK via writing USBN0/1_CLK_CTL[DIVIDE, DIVIDE2] and
 672          *     setting USBN0/1_CLK_CTL[ENABLE] = 1. Divide the core clock down
 673          *     such that USB is as close as possible to 125Mhz
 674          */
 675         {
 676                 int divisor = (octeon_get_clock_rate()+125000000-1)/125000000;
 677                 if (divisor < 4)  /* Lower than 4 doesn't seem to work properly */
 678                         divisor = 4;
 679                 usbn_clk_ctl.s.divide = divisor;
 680                 usbn_clk_ctl.s.divide2 = 0;
 681         }
 682         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 683                                usbn_clk_ctl.u64);
 684         /* 2d. Write USBN0/1_CLK_CTL[HCLK_RST] = 1 */
 685         usbn_clk_ctl.s.hclk_rst = 1;
 686         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 687                                usbn_clk_ctl.u64);
 688         /* 2e.  Wait 64 core-clock cycles for HCLK to stabilize */
 689         cvmx_wait(64);
 690         /*
 691          * 3. Program the power-on reset field in the USBN clock-control
 692          *    register:
 693          *    USBN_CLK_CTL[POR] = 0
 694          */
 695         usbn_clk_ctl.s.por = 0;
 696         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 697                                usbn_clk_ctl.u64);
 698         /* 4. Wait 1 ms for PHY clock to start */
 699         mdelay(1);
 700         /*
 701          * 5. Program the Reset input from automatic test equipment field in the
 702          *    USBP control and status register:
 703          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 1
 704          */
 705         usbn_usbp_ctl_status.u64 = __cvmx_usb_read_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index));
 706         usbn_usbp_ctl_status.s.ate_reset = 1;
 707         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 708                                usbn_usbp_ctl_status.u64);
 709         /* 6. Wait 10 cycles */
 710         cvmx_wait(10);
 711         /*
 712          * 7. Clear ATE_RESET field in the USBN clock-control register:
 713          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 0
 714          */
 715         usbn_usbp_ctl_status.s.ate_reset = 0;
 716         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 717                                usbn_usbp_ctl_status.u64);
 718         /*
 719          * 8. Program the PHY reset field in the USBN clock-control register:
 720          *    USBN_CLK_CTL[PRST] = 1
 721          */
 722         usbn_clk_ctl.s.prst = 1;
 723         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 724                                usbn_clk_ctl.u64);
 725         /*
 726          * 9. Program the USBP control and status register to select host or
 727          *    device mode. USBN_USBP_CTL_STATUS[HST_MODE] = 0 for host, = 1 for
 728          *    device
 729          */
 730         usbn_usbp_ctl_status.s.hst_mode = 0;
 731         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 732                                usbn_usbp_ctl_status.u64);
 733         /* 10. Wait 1 us */
 734         udelay(1);
 735         /*
 736          * 11. Program the hreset_n field in the USBN clock-control register:
 737          *     USBN_CLK_CTL[HRST] = 1
 738          */
 739         usbn_clk_ctl.s.hrst = 1;
 740         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 741                                usbn_clk_ctl.u64);
 742         /* 12. Proceed to USB core initialization */
 743         usbn_clk_ctl.s.enable = 1;
 744         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 745                                usbn_clk_ctl.u64);
 746         udelay(1);
 747
 748         /*
 749          * USB Core Initialization
 750          *
 751          * 1. Read USBC_GHWCFG1, USBC_GHWCFG2, USBC_GHWCFG3, USBC_GHWCFG4 to
 752          *    determine USB core configuration parameters.
 753          *
 754          *    Nothing needed
 755          *
 756          * 2. Program the following fields in the global AHB configuration
 757          *    register (USBC_GAHBCFG)
 758          *    DMA mode, USBC_GAHBCFG[DMAEn]: 1 = DMA mode, 0 = slave mode
 759          *    Burst length, USBC_GAHBCFG[HBSTLEN] = 0
 760          *    Nonperiodic TxFIFO empty level (slave mode only),
 761          *    USBC_GAHBCFG[NPTXFEMPLVL]
 762          *    Periodic TxFIFO empty level (slave mode only),
 763          *    USBC_GAHBCFG[PTXFEMPLVL]
 764          *    Global interrupt mask, USBC_GAHBCFG[GLBLINTRMSK] = 1
 765          */
 766         {
 767                 union cvmx_usbcx_gahbcfg usbcx_gahbcfg;
 768                 /* Due to an errata, CN31XX doesn't support DMA */
 769                 if (OCTEON_IS_MODEL(OCTEON_CN31XX))
 770                         usb->init_flags |= CVMX_USB_INITIALIZE_FLAGS_NO_DMA;
 771                 usbcx_gahbcfg.u32 = 0;
 772                 usbcx_gahbcfg.s.dmaen = !(usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA);
 773                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 774                         usb->idle_hardware_channels = 0x1;  /* Only use one channel with non DMA */
 775                 else if (OCTEON_IS_MODEL(OCTEON_CN5XXX))
 776                         usb->idle_hardware_channels = 0xf7; /* CN5XXX have an errata with channel 3 */
 777                 else
 778                         usb->idle_hardware_channels = 0xff;
 779                 usbcx_gahbcfg.s.hbstlen = 0;
 780                 usbcx_gahbcfg.s.nptxfemplvl = 1;
 781                 usbcx_gahbcfg.s.ptxfemplvl = 1;
 782                 usbcx_gahbcfg.s.glblintrmsk = 1;
 783                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GAHBCFG(usb->index),
 784                                        usbcx_gahbcfg.u32);
 785         }
 786         /*
 787          * 3. Program the following fields in USBC_GUSBCFG register.
 788          *    HS/FS timeout calibration, USBC_GUSBCFG[TOUTCAL] = 0
 789          *    ULPI DDR select, USBC_GUSBCFG[DDRSEL] = 0
 790          *    USB turnaround time, USBC_GUSBCFG[USBTRDTIM] = 0x5
 791          *    PHY low-power clock select, USBC_GUSBCFG[PHYLPWRCLKSEL] = 0
 792          */
 793         {
 794                 union cvmx_usbcx_gusbcfg usbcx_gusbcfg;
 795                 usbcx_gusbcfg.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index));
 796                 usbcx_gusbcfg.s.toutcal = 0;
 797                 usbcx_gusbcfg.s.ddrsel = 0;
 798                 usbcx_gusbcfg.s.usbtrdtim = 0x5;
 799                 usbcx_gusbcfg.s.phylpwrclksel = 0;
 800                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index),
 801                                        usbcx_gusbcfg.u32);
 802         }
 803         /*
 804          * 4. The software must unmask the following bits in the USBC_GINTMSK
 805          *    register.
 806          *    OTG interrupt mask, USBC_GINTMSK[OTGINTMSK] = 1
 807          *    Mode mismatch interrupt mask, USBC_GINTMSK[MODEMISMSK] = 1
 808          */
 809         {
 810                 union cvmx_usbcx_gintmsk usbcx_gintmsk;
 811                 int channel;
 812
 813                 usbcx_gintmsk.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GINTMSK(usb->index));
 814                 usbcx_gintmsk.s.otgintmsk = 1;
 815                 usbcx_gintmsk.s.modemismsk = 1;
 816                 usbcx_gintmsk.s.hchintmsk = 1;
 817                 usbcx_gintmsk.s.sofmsk = 0;
 818                 /* We need RX FIFO interrupts if we don't have DMA */
 819                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 820                         usbcx_gintmsk.s.rxflvlmsk = 1;
 821                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTMSK(usb->index),
 822                                        usbcx_gintmsk.u32);
 823
 824                 /* Disable all channel interrupts. We'll enable them per channel later */
 825                 for (channel = 0; channel < 8; channel++)
 826                         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
 827         }
 828
 829         {
 830                 /*
 831                  * Host Port Initialization
 832                  *
 833                  * 1. Program the host-port interrupt-mask field to unmask,
 834                  *    USBC_GINTMSK[PRTINT] = 1
 835                  */
 836                 USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk,
 837                                 prtintmsk, 1);
 838                 USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk,
 839                                 disconnintmsk, 1);
 840                 /*
 841                  * 2. Program the USBC_HCFG register to select full-speed host
 842                  *    or high-speed host.
 843                  */
 844                 {
 845                         union cvmx_usbcx_hcfg usbcx_hcfg;
 846                         usbcx_hcfg.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCFG(usb->index));
 847                         usbcx_hcfg.s.fslssupp = 0;
 848                         usbcx_hcfg.s.fslspclksel = 0;
 849                         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCFG(usb->index), usbcx_hcfg.u32);
 850                 }
 851                 /*
 852                  * 3. Program the port power bit to drive VBUS on the USB,
 853                  *    USBC_HPRT[PRTPWR] = 1
 854                  */
 855                 USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt, prtpwr, 1);
 856
 857                 /*
 858                  * Steps 4-15 from the manual are done later in the port enable
 859                  */
 860         }
 861
 862         return 0;
 863 }
 864
 865
 866 /**
 867  * Shutdown a USB port after a call to cvmx_usb_initialize().
 868  * The port should be disabled with all pipes closed when this
 869  * function is called.
 870  *
 871  * @state: USB device state populated by
 872  *         cvmx_usb_initialize().
 873  *
 874  * Returns: 0 or a negative error code.
 875  */
 876 int cvmx_usb_shutdown(struct cvmx_usb_state *state)
 877 {
 878         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 879         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
 880
 881         /* Make sure all pipes are closed */
 882         if (usb->idle_pipes.head ||
 883                 usb->active_pipes[CVMX_USB_TRANSFER_ISOCHRONOUS].head ||
 884                 usb->active_pipes[CVMX_USB_TRANSFER_INTERRUPT].head ||
 885                 usb->active_pipes[CVMX_USB_TRANSFER_CONTROL].head ||
 886                 usb->active_pipes[CVMX_USB_TRANSFER_BULK].head)
 887                 return -EBUSY;
 888
 889         /* Disable the clocks and put them in power on reset */
 890         usbn_clk_ctl.u64 = __cvmx_usb_read_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index));
 891         usbn_clk_ctl.s.enable = 1;
 892         usbn_clk_ctl.s.por = 1;
 893         usbn_clk_ctl.s.hclk_rst = 1;
 894         usbn_clk_ctl.s.prst = 0;
 895         usbn_clk_ctl.s.hrst = 0;
 896         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 897                                usbn_clk_ctl.u64);
 898         return 0;
 899 }
 900
 901
 902 /**
 903  * Enable a USB port. After this call succeeds, the USB port is
 904  * online and servicing requests.
 905  *
 906  * @state: USB device state populated by
 907  *         cvmx_usb_initialize().
 908  *
 909  * Returns: 0 or a negative error code.
 910  */
 911 int cvmx_usb_enable(struct cvmx_usb_state *state)
 912 {
 913         union cvmx_usbcx_ghwcfg3 usbcx_ghwcfg3;
 914         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
 915
 916         usb->usbcx_hprt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
 917
 918         /*
 919          * If the port is already enabled the just return. We don't need to do
 920          * anything
 921          */
 922         if (usb->usbcx_hprt.s.prtena)
 923                 return 0;
 924
 925         /* If there is nothing plugged into the port then fail immediately */
 926         if (!usb->usbcx_hprt.s.prtconnsts) {
 927                 return -ETIMEDOUT;
 928         }
 929
 930         /* Program the port reset bit to start the reset process */
 931         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt, prtrst, 1);
 932
 933         /*
 934          * Wait at least 50ms (high speed), or 10ms (full speed) for the reset
 935          * process to complete.
 936          */
 937         mdelay(50);
 938
 939         /* Program the port reset bit to 0, USBC_HPRT[PRTRST] = 0 */
 940         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt, prtrst, 0);
 941
 942         /* Wait for the USBC_HPRT[PRTENA]. */
 943         if (CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt,
 944                                   prtena, ==, 1, 100000))
 945                 return -ETIMEDOUT;
 946
 947         /* Read the port speed field to get the enumerated speed, USBC_HPRT[PRTSPD]. */
 948         usb->usbcx_hprt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
 949         usbcx_ghwcfg3.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GHWCFG3(usb->index));
 950
 951         /*
 952          * 13. Program the USBC_GRXFSIZ register to select the size of the
 953          *     receive FIFO (25%).
 954          */
 955         USB_SET_FIELD32(CVMX_USBCX_GRXFSIZ(usb->index), union cvmx_usbcx_grxfsiz,
 956                         rxfdep, usbcx_ghwcfg3.s.dfifodepth / 4);
 957         /*
 958          * 14. Program the USBC_GNPTXFSIZ register to select the size and the
 959          *     start address of the non- periodic transmit FIFO for nonperiodic
 960          *     transactions (50%).
 961          */
 962         {
 963                 union cvmx_usbcx_gnptxfsiz siz;
 964                 siz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index));
 965                 siz.s.nptxfdep = usbcx_ghwcfg3.s.dfifodepth / 2;
 966                 siz.s.nptxfstaddr = usbcx_ghwcfg3.s.dfifodepth / 4;
 967                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index), siz.u32);
 968         }
 969         /*
 970          * 15. Program the USBC_HPTXFSIZ register to select the size and start
 971          *     address of the periodic transmit FIFO for periodic transactions
 972          *     (25%).
 973          */
 974         {
 975                 union cvmx_usbcx_hptxfsiz siz;
 976                 siz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index));
 977                 siz.s.ptxfsize = usbcx_ghwcfg3.s.dfifodepth / 4;
 978                 siz.s.ptxfstaddr = 3 * usbcx_ghwcfg3.s.dfifodepth / 4;
 979                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index), siz.u32);
 980         }
 981         /* Flush all FIFOs */
 982         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), union cvmx_usbcx_grstctl, txfnum, 0x10);
 983         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), union cvmx_usbcx_grstctl, txfflsh, 1);
 984         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), union cvmx_usbcx_grstctl,
 985                               txfflsh, ==, 0, 100);
 986         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), union cvmx_usbcx_grstctl, rxfflsh, 1);
 987         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), union cvmx_usbcx_grstctl,
 988                               rxfflsh, ==, 0, 100);
 989
 990         return 0;
 991 }
 992
 993
 994 /**
 995  * Disable a USB port. After this call the USB port will not
 996  * generate data transfers and will not generate events.
 997  * Transactions in process will fail and call their
 998  * associated callbacks.
 999  *
1000  * @state: USB device state populated by
1001  *         cvmx_usb_initialize().
1002  *
1003  * Returns: 0 or a negative error code.
1004  */
1005 int cvmx_usb_disable(struct cvmx_usb_state *state)
1006 {
1007         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
1008
1009         /* Disable the port */
1010         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt, prtena, 1);
1011         return 0;
1012 }
1013
1014
1015 /**
1016  * Get the current state of the USB port. Use this call to
1017  * determine if the usb port has anything connected, is enabled,
1018  * or has some sort of error condition. The return value of this
1019  * call has "changed" bits to signal of the value of some fields
1020  * have changed between calls. These "changed" fields are based
1021  * on the last call to cvmx_usb_set_status(). In order to clear
1022  * them, you must update the status through cvmx_usb_set_status().
1023  *
1024  * @state: USB device state populated by
1025  *         cvmx_usb_initialize().
1026  *
1027  * Returns: Port status information
1028  */
1029 struct cvmx_usb_port_status cvmx_usb_get_status(struct cvmx_usb_state *state)
1030 {
1031         union cvmx_usbcx_hprt usbc_hprt;
1032         struct cvmx_usb_port_status result;
1033         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
1034
1035         memset(&result, 0, sizeof(result));
1036
1037         usbc_hprt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
1038         result.port_enabled = usbc_hprt.s.prtena;
1039         result.port_over_current = usbc_hprt.s.prtovrcurract;
1040         result.port_powered = usbc_hprt.s.prtpwr;
1041         result.port_speed = usbc_hprt.s.prtspd;
1042         result.connected = usbc_hprt.s.prtconnsts;
1043         result.connect_change = (result.connected != usb->port_status.connected);
1044
1045         return result;
1046 }
1047
1048
1049 /**
1050  * Set the current state of the USB port. The status is used as
1051  * a reference for the "changed" bits returned by
1052  * cvmx_usb_get_status(). Other than serving as a reference, the
1053  * status passed to this function is not used. No fields can be
1054  * changed through this call.
1055  *
1056  * @state:       USB device state populated by
1057  *               cvmx_usb_initialize().
1058  * @port_status:
1059  *               Port status to set, most like returned by cvmx_usb_get_status()
1060  */
1061 void cvmx_usb_set_status(struct cvmx_usb_state *state, struct cvmx_usb_port_status port_status)
1062 {
1063         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
1064         usb->port_status = port_status;
1065         return;
1066 }
1067
1068
1069 /**
1070  * Convert a USB transaction into a handle
1071  *
1072  * @usb:         USB device state populated by
1073  *               cvmx_usb_initialize().
1074  * @transaction:
1075  *               Transaction to get handle for
1076  *
1077  * Returns: Handle
1078  */
1079 static inline int __cvmx_usb_get_submit_handle(struct cvmx_usb_internal_state *usb,
1080                                                struct cvmx_usb_transaction *transaction)
1081 {
1082         return ((unsigned long)transaction - (unsigned long)usb->transaction) /
1083                         sizeof(*transaction);
1084 }
1085
1086
1087 /**
1088  * Convert a USB pipe into a handle
1089  *
1090  * @usb:         USB device state populated by
1091  *               cvmx_usb_initialize().
1092  * @pipe:        Pipe to get handle for
1093  *
1094  * Returns: Handle
1095  */
1096 static inline int __cvmx_usb_get_pipe_handle(struct cvmx_usb_internal_state *usb,
1097                                              struct cvmx_usb_pipe *pipe)
1098 {
1099         return ((unsigned long)pipe - (unsigned long)usb->pipe) / sizeof(*pipe);
1100 }
1101
1102
1103 /**
1104  * Open a virtual pipe between the host and a USB device. A pipe
1105  * must be opened before data can be transferred between a device
1106  * and Octeon.
1107  *
1108  * @state:           USB device state populated by
1109  *                   cvmx_usb_initialize().
1110  * @device_addr:
1111  *                   USB device address to open the pipe to
1112  *                   (0-127).
1113  * @endpoint_num:
1114  *                   USB endpoint number to open the pipe to
1115  *                   (0-15).
1116  * @device_speed:
1117  *                   The speed of the device the pipe is going
1118  *                   to. This must match the device's speed,
1119  *                   which may be different than the port speed.
1120  * @max_packet:      The maximum packet length the device can
1121  *                   transmit/receive (low speed=0-8, full
1122  *                   speed=0-1023, high speed=0-1024). This value
1123  *                   comes from the standard endpoint descriptor
1124  *                   field wMaxPacketSize bits <10:0>.
1125  * @transfer_type:
1126  *                   The type of transfer this pipe is for.
1127  * @transfer_dir:
1128  *                   The direction the pipe is in. This is not
1129  *                   used for control pipes.
1130  * @interval:        For ISOCHRONOUS and INTERRUPT transfers,
1131  *                   this is how often the transfer is scheduled
1132  *                   for. All other transfers should specify
1133  *                   zero. The units are in frames (8000/sec at
1134  *                   high speed, 1000/sec for full speed).
1135  * @multi_count:
1136  *                   For high speed devices, this is the maximum
1137  *                   allowed number of packet per microframe.
1138  *                   Specify zero for non high speed devices. This
1139  *                   value comes from the standard endpoint descriptor
1140  *                   field wMaxPacketSize bits <12:11>.
1141  * @hub_device_addr:
1142  *                   Hub device address this device is connected
1143  *                   to. Devices connected directly to Octeon
1144  *                   use zero. This is only used when the device
1145  *                   is full/low speed behind a high speed hub.
1146  *                   The address will be of the high speed hub,
1147  *                   not and full speed hubs after it.
1148  * @hub_port:        Which port on the hub the device is
1149  *                   connected. Use zero for devices connected
1150  *                   directly to Octeon. Like hub_device_addr,
1151  *                   this is only used for full/low speed
1152  *                   devices behind a high speed hub.
1153  *
1154  * Returns: A non negative value is a pipe handle. Negative
1155  *          values are error codes.
1156  */
1157 int cvmx_usb_open_pipe(struct cvmx_usb_state *state,
1158                        int device_addr, int endpoint_num,
1159                        enum cvmx_usb_speed device_speed, int max_packet,
1160                        enum cvmx_usb_transfer transfer_type,
1161                        enum cvmx_usb_direction transfer_dir, int interval,
1162                        int multi_count, int hub_device_addr, int hub_port)
1163 {
1164         struct cvmx_usb_pipe *pipe;
1165         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
1166
1167         if (unlikely((device_addr < 0) || (device_addr > MAX_USB_ADDRESS)))
1168                 return -EINVAL;
1169         if (unlikely((endpoint_num < 0) || (endpoint_num > MAX_USB_ENDPOINT)))
1170                 return -EINVAL;
1171         if (unlikely(device_speed > CVMX_USB_SPEED_LOW))
1172                 return -EINVAL;
1173         if (unlikely((max_packet <= 0) || (max_packet > 1024)))
1174                 return -EINVAL;
1175         if (unlikely(transfer_type > CVMX_USB_TRANSFER_INTERRUPT))
1176                 return -EINVAL;
1177         if (unlikely((transfer_dir != CVMX_USB_DIRECTION_OUT) &&
1178                 (transfer_dir != CVMX_USB_DIRECTION_IN)))
1179                 return -EINVAL;
1180         if (unlikely(interval < 0))
1181                 return -EINVAL;
1182         if (unlikely((transfer_type == CVMX_USB_TRANSFER_CONTROL) && interval))
1183                 return -EINVAL;
1184         if (unlikely(multi_count < 0))
1185                 return -EINVAL;
1186         if (unlikely((device_speed != CVMX_USB_SPEED_HIGH) &&
1187                 (multi_count != 0)))
1188                 return -EINVAL;
1189         if (unlikely((hub_device_addr < 0) || (hub_device_addr > MAX_USB_ADDRESS)))
1190                 return -EINVAL;
1191         if (unlikely((hub_port < 0) || (hub_port > MAX_USB_HUB_PORT)))
1192                 return -EINVAL;
1193
1194         /* Find a free pipe */
1195         pipe = usb->free_pipes.head;
1196         if (!pipe)
1197                 return -ENOMEM;
1198         __cvmx_usb_remove_pipe(&usb->free_pipes, pipe);
1199         pipe->flags = __CVMX_USB_PIPE_FLAGS_OPEN;
1200         if ((device_speed == CVMX_USB_SPEED_HIGH) &&
1201                 (transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1202                 (transfer_type == CVMX_USB_TRANSFER_BULK))
1203                 pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
1204         pipe->device_addr = device_addr;
1205         pipe->endpoint_num = endpoint_num;
1206         pipe->device_speed = device_speed;
1207         pipe->max_packet = max_packet;
1208         pipe->transfer_type = transfer_type;
1209         pipe->transfer_dir = transfer_dir;
1210         /*
1211          * All pipes use interval to rate limit NAK processing. Force an
1212          * interval if one wasn't supplied
1213          */
1214         if (!interval)
1215                 interval = 1;
1216         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1217                 pipe->interval = interval*8;
1218                 /* Force start splits to be schedule on uFrame 0 */
1219                 pipe->next_tx_frame = ((usb->frame_number+7)&~7) + pipe->interval;
1220         } else {
1221                 pipe->interval = interval;
1222                 pipe->next_tx_frame = usb->frame_number + pipe->interval;
1223         }
1224         pipe->multi_count = multi_count;
1225         pipe->hub_device_addr = hub_device_addr;
1226         pipe->hub_port = hub_port;
1227         pipe->pid_toggle = 0;
1228         pipe->split_sc_frame = -1;
1229         __cvmx_usb_append_pipe(&usb->idle_pipes, pipe);
1230
1231         /*
1232          * We don't need to tell the hardware about this pipe yet since
1233          * it doesn't have any submitted requests
1234          */
1235
1236         return __cvmx_usb_get_pipe_handle(usb, pipe);
1237 }
1238
1239
1240 /**
1241  * Poll the RX FIFOs and remove data as needed. This function is only used
1242  * in non DMA mode. It is very important that this function be called quickly
1243  * enough to prevent FIFO overflow.
1244  *
1245  * @usb:        USB device state populated by
1246  *              cvmx_usb_initialize().
1247  */
1248 static void __cvmx_usb_poll_rx_fifo(struct cvmx_usb_internal_state *usb)
1249 {
1250         union cvmx_usbcx_grxstsph rx_status;
1251         int channel;
1252         int bytes;
1253         uint64_t address;
1254         uint32_t *ptr;
1255
1256         rx_status.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GRXSTSPH(usb->index));
1257         /* Only read data if IN data is there */
1258         if (rx_status.s.pktsts != 2)
1259                 return;
1260         /* Check if no data is available */
1261         if (!rx_status.s.bcnt)
1262                 return;
1263
1264         channel = rx_status.s.chnum;
1265         bytes = rx_status.s.bcnt;
1266         if (!bytes)
1267                 return;
1268
1269         /* Get where the DMA engine would have written this data */
1270         address = __cvmx_usb_read_csr64(usb, CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8);
1271         ptr = cvmx_phys_to_ptr(address);
1272         __cvmx_usb_write_csr64(usb, CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8, address + bytes);
1273
1274         /* Loop writing the FIFO data for this packet into memory */
1275         while (bytes > 0) {
1276                 *ptr++ = __cvmx_usb_read_csr32(usb, USB_FIFO_ADDRESS(channel, usb->index));
1277                 bytes -= 4;
1278         }
1279         CVMX_SYNCW;
1280
1281         return;
1282 }
1283
1284
1285 /**
1286  * Fill the TX hardware fifo with data out of the software
1287  * fifos
1288  *
1289  * @usb:            USB device state populated by
1290  *                  cvmx_usb_initialize().
1291  * @fifo:           Software fifo to use
1292  * @available:      Amount of space in the hardware fifo
1293  *
1294  * Returns: Non zero if the hardware fifo was too small and needs
1295  *          to be serviced again.
1296  */
1297 static int __cvmx_usb_fill_tx_hw(struct cvmx_usb_internal_state *usb, struct cvmx_usb_tx_fifo *fifo, int available)
1298 {
1299         /*
1300          * We're done either when there isn't anymore space or the software FIFO
1301          * is empty
1302          */
1303         while (available && (fifo->head != fifo->tail)) {
1304                 int i = fifo->tail;
1305                 const uint32_t *ptr = cvmx_phys_to_ptr(fifo->entry[i].address);
1306                 uint64_t csr_address = USB_FIFO_ADDRESS(fifo->entry[i].channel, usb->index) ^ 4;
1307                 int words = available;
1308
1309                 /* Limit the amount of data to waht the SW fifo has */
1310                 if (fifo->entry[i].size <= available) {
1311                         words = fifo->entry[i].size;
1312                         fifo->tail++;
1313                         if (fifo->tail > MAX_CHANNELS)
1314                                 fifo->tail = 0;
1315                 }
1316
1317                 /* Update the next locations and counts */
1318                 available -= words;
1319                 fifo->entry[i].address += words * 4;
1320                 fifo->entry[i].size -= words;
1321
1322                 /*
1323                  * Write the HW fifo data. The read every three writes is due
1324                  * to an errata on CN3XXX chips
1325                  */
1326                 while (words > 3) {
1327                         cvmx_write64_uint32(csr_address, *ptr++);
1328                         cvmx_write64_uint32(csr_address, *ptr++);
1329                         cvmx_write64_uint32(csr_address, *ptr++);
1330                         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1331                         words -= 3;
1332                 }
1333                 cvmx_write64_uint32(csr_address, *ptr++);
1334                 if (--words) {
1335                         cvmx_write64_uint32(csr_address, *ptr++);
1336                         if (--words)
1337                                 cvmx_write64_uint32(csr_address, *ptr++);
1338                 }
1339                 cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1340         }
1341         return fifo->head != fifo->tail;
1342 }
1343
1344
1345 /**
1346  * Check the hardware FIFOs and fill them as needed
1347  *
1348  * @usb:        USB device state populated by
1349  *              cvmx_usb_initialize().
1350  */
1351 static void __cvmx_usb_poll_tx_fifo(struct cvmx_usb_internal_state *usb)
1352 {
1353         if (usb->periodic.head != usb->periodic.tail) {
1354                 union cvmx_usbcx_hptxsts tx_status;
1355                 tx_status.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPTXSTS(usb->index));
1356                 if (__cvmx_usb_fill_tx_hw(usb, &usb->periodic, tx_status.s.ptxfspcavail))
1357                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk, ptxfempmsk, 1);
1358                 else
1359                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk, ptxfempmsk, 0);
1360         }
1361
1362         if (usb->nonperiodic.head != usb->nonperiodic.tail) {
1363                 union cvmx_usbcx_gnptxsts tx_status;
1364                 tx_status.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GNPTXSTS(usb->index));
1365                 if (__cvmx_usb_fill_tx_hw(usb, &usb->nonperiodic, tx_status.s.nptxfspcavail))
1366                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk, nptxfempmsk, 1);
1367                 else
1368                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk, nptxfempmsk, 0);
1369         }
1370
1371         return;
1372 }
1373
1374
1375 /**
1376  * Fill the TX FIFO with an outgoing packet
1377  *
1378  * @usb:          USB device state populated by
1379  *                cvmx_usb_initialize().
1380  * @channel:      Channel number to get packet from
1381  */
1382 static void __cvmx_usb_fill_tx_fifo(struct cvmx_usb_internal_state *usb, int channel)
1383 {
1384         union cvmx_usbcx_hccharx hcchar;
1385         union cvmx_usbcx_hcspltx usbc_hcsplt;
1386         union cvmx_usbcx_hctsizx usbc_hctsiz;
1387         struct cvmx_usb_tx_fifo *fifo;
1388
1389         /* We only need to fill data on outbound channels */
1390         hcchar.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index));
1391         if (hcchar.s.epdir != CVMX_USB_DIRECTION_OUT)
1392                 return;
1393
1394         /* OUT Splits only have data on the start and not the complete */
1395         usbc_hcsplt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCSPLTX(channel, usb->index));
1396         if (usbc_hcsplt.s.spltena && usbc_hcsplt.s.compsplt)
1397                 return;
1398
1399         /* Find out how many bytes we need to fill and convert it into 32bit words */
1400         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index));
1401         if (!usbc_hctsiz.s.xfersize)
1402                 return;
1403
1404         if ((hcchar.s.eptype == CVMX_USB_TRANSFER_INTERRUPT) ||
1405                 (hcchar.s.eptype == CVMX_USB_TRANSFER_ISOCHRONOUS))
1406                 fifo = &usb->periodic;
1407         else
1408                 fifo = &usb->nonperiodic;
1409
1410         fifo->entry[fifo->head].channel = channel;
1411         fifo->entry[fifo->head].address = __cvmx_usb_read_csr64(usb, CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8);
1412         fifo->entry[fifo->head].size = (usbc_hctsiz.s.xfersize+3)>>2;
1413         fifo->head++;
1414         if (fifo->head > MAX_CHANNELS)
1415                 fifo->head = 0;
1416
1417         __cvmx_usb_poll_tx_fifo(usb);
1418
1419         return;
1420 }
1421
1422 /**
1423  * Perform channel specific setup for Control transactions. All
1424  * the generic stuff will already have been done in
1425  * __cvmx_usb_start_channel()
1426  *
1427  * @usb:          USB device state populated by
1428  *                cvmx_usb_initialize().
1429  * @channel:      Channel to setup
1430  * @pipe:         Pipe for control transaction
1431  */
1432 static void __cvmx_usb_start_channel_control(struct cvmx_usb_internal_state *usb,
1433                                              int channel,
1434                                              struct cvmx_usb_pipe *pipe)
1435 {
1436         struct cvmx_usb_transaction *transaction = pipe->head;
1437         union cvmx_usb_control_header *header =
1438                 cvmx_phys_to_ptr(transaction->control_header);
1439         int bytes_to_transfer = transaction->buffer_length - transaction->actual_bytes;
1440         int packets_to_transfer;
1441         union cvmx_usbcx_hctsizx usbc_hctsiz;
1442
1443         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index));
1444
1445         switch (transaction->stage) {
1446         case CVMX_USB_STAGE_NON_CONTROL:
1447         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
1448                 cvmx_dprintf("%s: ERROR - Non control stage\n", __FUNCTION__);
1449                 break;
1450         case CVMX_USB_STAGE_SETUP:
1451                 usbc_hctsiz.s.pid = 3; /* Setup */
1452                 bytes_to_transfer = sizeof(*header);
1453                 /* All Control operations start with a setup going OUT */
1454                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), union cvmx_usbcx_hccharx, epdir, CVMX_USB_DIRECTION_OUT);
1455                 /*
1456                  * Setup send the control header instead of the buffer data. The
1457                  * buffer data will be used in the next stage
1458                  */
1459                 __cvmx_usb_write_csr64(usb, CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8, transaction->control_header);
1460                 break;
1461         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
1462                 usbc_hctsiz.s.pid = 3; /* Setup */
1463                 bytes_to_transfer = 0;
1464                 /* All Control operations start with a setup going OUT */
1465                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), union cvmx_usbcx_hccharx, epdir, CVMX_USB_DIRECTION_OUT);
1466                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index), union cvmx_usbcx_hcspltx, compsplt, 1);
1467                 break;
1468         case CVMX_USB_STAGE_DATA:
1469                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1470                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1471                         if (header->s.request_type & 0x80)
1472                                 bytes_to_transfer = 0;
1473                         else if (bytes_to_transfer > pipe->max_packet)
1474                                 bytes_to_transfer = pipe->max_packet;
1475                 }
1476                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1477                                 union cvmx_usbcx_hccharx, epdir,
1478                                 ((header->s.request_type & 0x80) ?
1479                                         CVMX_USB_DIRECTION_IN :
1480                                         CVMX_USB_DIRECTION_OUT));
1481                 break;
1482         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
1483                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1484                 if (!(header->s.request_type & 0x80))
1485                         bytes_to_transfer = 0;
1486                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1487                                 union cvmx_usbcx_hccharx, epdir,
1488                                 ((header->s.request_type & 0x80) ?
1489                                         CVMX_USB_DIRECTION_IN :
1490                                         CVMX_USB_DIRECTION_OUT));
1491                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index), union cvmx_usbcx_hcspltx, compsplt, 1);
1492                 break;
1493         case CVMX_USB_STAGE_STATUS:
1494                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1495                 bytes_to_transfer = 0;
1496                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), union cvmx_usbcx_hccharx, epdir,
1497                                 ((header->s.request_type & 0x80) ?
1498                                         CVMX_USB_DIRECTION_OUT :
1499                                         CVMX_USB_DIRECTION_IN));
1500                 break;
1501         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
1502                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1503                 bytes_to_transfer = 0;
1504                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), union cvmx_usbcx_hccharx, epdir,
1505                                 ((header->s.request_type & 0x80) ?
1506                                         CVMX_USB_DIRECTION_OUT :
1507                                         CVMX_USB_DIRECTION_IN));
1508                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index), union cvmx_usbcx_hcspltx, compsplt, 1);
1509                 break;
1510         }
1511
1512         /*
1513          * Make sure the transfer never exceeds the byte limit of the hardware.
1514          * Further bytes will be sent as continued transactions
1515          */
1516         if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1517                 /* Round MAX_TRANSFER_BYTES to a multiple of out packet size */
1518                 bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
1519                 bytes_to_transfer *= pipe->max_packet;
1520         }
1521
1522         /*
1523          * Calculate the number of packets to transfer. If the length is zero
1524          * we still need to transfer one packet
1525          */
1526         packets_to_transfer = (bytes_to_transfer + pipe->max_packet - 1) / pipe->max_packet;
1527         if (packets_to_transfer == 0)
1528                 packets_to_transfer = 1;
1529         else if ((packets_to_transfer > 1) && (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1530                 /*
1531                  * Limit to one packet when not using DMA. Channels must be
1532                  * restarted between every packet for IN transactions, so there
1533                  * is no reason to do multiple packets in a row
1534                  */
1535                 packets_to_transfer = 1;
1536                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1537         } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1538                 /*
1539                  * Limit the number of packet and data transferred to what the
1540                  * hardware can handle
1541                  */
1542                 packets_to_transfer = MAX_TRANSFER_PACKETS;
1543                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1544         }
1545
1546         usbc_hctsiz.s.xfersize = bytes_to_transfer;
1547         usbc_hctsiz.s.pktcnt = packets_to_transfer;
1548
1549         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index), usbc_hctsiz.u32);
1550         return;
1551 }
1552
1553
1554 /**
1555  * Start a channel to perform the pipe's head transaction
1556  *
1557  * @usb:          USB device state populated by
1558  *                cvmx_usb_initialize().
1559  * @channel:      Channel to setup
1560  * @pipe:         Pipe to start
1561  */
1562 static void __cvmx_usb_start_channel(struct cvmx_usb_internal_state *usb,
1563                                      int channel,
1564                                      struct cvmx_usb_pipe *pipe)
1565 {
1566         struct cvmx_usb_transaction *transaction = pipe->head;
1567
1568         /* Make sure all writes to the DMA region get flushed */
1569         CVMX_SYNCW;
1570
1571         /* Attach the channel to the pipe */
1572         usb->pipe_for_channel[channel] = pipe;
1573         pipe->channel = channel;
1574         pipe->flags |= __CVMX_USB_PIPE_FLAGS_SCHEDULED;
1575
1576         /* Mark this channel as in use */
1577         usb->idle_hardware_channels &= ~(1<<channel);
1578
1579         /* Enable the channel interrupt bits */
1580         {
1581                 union cvmx_usbcx_hcintx usbc_hcint;
1582                 union cvmx_usbcx_hcintmskx usbc_hcintmsk;
1583                 union cvmx_usbcx_haintmsk usbc_haintmsk;
1584
1585                 /* Clear all channel status bits */
1586                 usbc_hcint.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCINTX(channel, usb->index));
1587                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTX(channel, usb->index), usbc_hcint.u32);
1588
1589                 usbc_hcintmsk.u32 = 0;
1590                 usbc_hcintmsk.s.chhltdmsk = 1;
1591                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1592                         /* Channels need these extra interrupts when we aren't in DMA mode */
1593                         usbc_hcintmsk.s.datatglerrmsk = 1;
1594                         usbc_hcintmsk.s.frmovrunmsk = 1;
1595                         usbc_hcintmsk.s.bblerrmsk = 1;
1596                         usbc_hcintmsk.s.xacterrmsk = 1;
1597                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1598                                 /* Splits don't generate xfercompl, so we need ACK and NYET */
1599                                 usbc_hcintmsk.s.nyetmsk = 1;
1600                                 usbc_hcintmsk.s.ackmsk = 1;
1601                         }
1602                         usbc_hcintmsk.s.nakmsk = 1;
1603                         usbc_hcintmsk.s.stallmsk = 1;
1604                         usbc_hcintmsk.s.xfercomplmsk = 1;
1605                 }
1606                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), usbc_hcintmsk.u32);
1607
1608                 /* Enable the channel interrupt to propagate */
1609                 usbc_haintmsk.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HAINTMSK(usb->index));
1610                 usbc_haintmsk.s.haintmsk |= 1<<channel;
1611                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HAINTMSK(usb->index), usbc_haintmsk.u32);
1612         }
1613
1614         /* Setup the locations the DMA engines use  */
1615         {
1616                 uint64_t dma_address = transaction->buffer + transaction->actual_bytes;
1617                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1618                         dma_address = transaction->buffer + transaction->iso_packets[0].offset + transaction->actual_bytes;
1619                 __cvmx_usb_write_csr64(usb, CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8, dma_address);
1620                 __cvmx_usb_write_csr64(usb, CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8, dma_address);
1621         }
1622
1623         /* Setup both the size of the transfer and the SPLIT characteristics */
1624         {
1625                 union cvmx_usbcx_hcspltx usbc_hcsplt = {.u32 = 0};
1626                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 = 0};
1627                 int packets_to_transfer;
1628                 int bytes_to_transfer = transaction->buffer_length - transaction->actual_bytes;
1629
1630                 /*
1631                  * ISOCHRONOUS transactions store each individual transfer size
1632                  * in the packet structure, not the global buffer_length
1633                  */
1634                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1635                         bytes_to_transfer = transaction->iso_packets[0].length - transaction->actual_bytes;
1636
1637                 /*
1638                  * We need to do split transactions when we are talking to non
1639                  * high speed devices that are behind a high speed hub
1640                  */
1641                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1642                         /*
1643                          * On the start split phase (stage is even) record the
1644                          * frame number we will need to send the split complete.
1645                          * We only store the lower two bits since the time ahead
1646                          * can only be two frames
1647                          */
1648                         if ((transaction->stage&1) == 0) {
1649                                 if (transaction->type == CVMX_USB_TRANSFER_BULK)
1650                                         pipe->split_sc_frame = (usb->frame_number + 1) & 0x7f;
1651                                 else
1652                                         pipe->split_sc_frame = (usb->frame_number + 2) & 0x7f;
1653                         } else
1654                                 pipe->split_sc_frame = -1;
1655
1656                         usbc_hcsplt.s.spltena = 1;
1657                         usbc_hcsplt.s.hubaddr = pipe->hub_device_addr;
1658                         usbc_hcsplt.s.prtaddr = pipe->hub_port;
1659                         usbc_hcsplt.s.compsplt = (transaction->stage == CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE);
1660
1661                         /*
1662                          * SPLIT transactions can only ever transmit one data
1663                          * packet so limit the transfer size to the max packet
1664                          * size
1665                          */
1666                         if (bytes_to_transfer > pipe->max_packet)
1667                                 bytes_to_transfer = pipe->max_packet;
1668
1669                         /*
1670                          * ISOCHRONOUS OUT splits are unique in that they limit
1671                          * data transfers to 188 byte chunks representing the
1672                          * begin/middle/end of the data or all
1673                          */
1674                         if (!usbc_hcsplt.s.compsplt &&
1675                                 (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1676                                 (pipe->transfer_type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
1677                                 /*
1678                                  * Clear the split complete frame number as
1679                                  * there isn't going to be a split complete
1680                                  */
1681                                 pipe->split_sc_frame = -1;
1682                                 /*
1683                                  * See if we've started this transfer and sent
1684                                  * data
1685                                  */
1686                                 if (transaction->actual_bytes == 0) {
1687                                         /*
1688                                          * Nothing sent yet, this is either a
1689                                          * begin or the entire payload
1690                                          */
1691                                         if (bytes_to_transfer <= 188)
1692                                                 usbc_hcsplt.s.xactpos = 3; /* Entire payload in one go */
1693                                         else
1694                                                 usbc_hcsplt.s.xactpos = 2; /* First part of payload */
1695                                 } else {
1696                                         /*
1697                                          * Continuing the previous data, we must
1698                                          * either be in the middle or at the end
1699                                          */
1700                                         if (bytes_to_transfer <= 188)
1701                                                 usbc_hcsplt.s.xactpos = 1; /* End of payload */
1702                                         else
1703                                                 usbc_hcsplt.s.xactpos = 0; /* Middle of payload */
1704                                 }
1705                                 /*
1706                                  * Again, the transfer size is limited to 188
1707                                  * bytes
1708                                  */
1709                                 if (bytes_to_transfer > 188)
1710                                         bytes_to_transfer = 188;
1711                         }
1712                 }
1713
1714                 /*
1715                  * Make sure the transfer never exceeds the byte limit of the
1716                  * hardware. Further bytes will be sent as continued
1717                  * transactions
1718                  */
1719                 if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1720                         /*
1721                          * Round MAX_TRANSFER_BYTES to a multiple of out packet
1722                          * size
1723                          */
1724                         bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
1725                         bytes_to_transfer *= pipe->max_packet;
1726                 }
1727
1728                 /*
1729                  * Calculate the number of packets to transfer. If the length is
1730                  * zero we still need to transfer one packet
1731                  */
1732                 packets_to_transfer = (bytes_to_transfer + pipe->max_packet - 1) / pipe->max_packet;
1733                 if (packets_to_transfer == 0)
1734                         packets_to_transfer = 1;
1735                 else if ((packets_to_transfer > 1) && (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1736                         /*
1737                          * Limit to one packet when not using DMA. Channels must
1738                          * be restarted between every packet for IN
1739                          * transactions, so there is no reason to do multiple
1740                          * packets in a row
1741                          */
1742                         packets_to_transfer = 1;
1743                         bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1744                 } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1745                         /*
1746                          * Limit the number of packet and data transferred to
1747                          * what the hardware can handle
1748                          */
1749                         packets_to_transfer = MAX_TRANSFER_PACKETS;
1750                         bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1751                 }
1752
1753                 usbc_hctsiz.s.xfersize = bytes_to_transfer;
1754                 usbc_hctsiz.s.pktcnt = packets_to_transfer;
1755
1756                 /* Update the DATA0/DATA1 toggle */
1757                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1758                 /*
1759                  * High speed pipes may need a hardware ping before they start
1760                  */
1761                 if (pipe->flags & __CVMX_USB_PIPE_FLAGS_NEED_PING)
1762                         usbc_hctsiz.s.dopng = 1;
1763
1764                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCSPLTX(channel, usb->index), usbc_hcsplt.u32);
1765                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index), usbc_hctsiz.u32);
1766         }
1767
1768         /* Setup the Host Channel Characteristics Register */
1769         {
1770                 union cvmx_usbcx_hccharx usbc_hcchar = {.u32 = 0};
1771
1772                 /*
1773                  * Set the startframe odd/even properly. This is only used for
1774                  * periodic
1775                  */
1776                 usbc_hcchar.s.oddfrm = usb->frame_number&1;
1777
1778                 /*
1779                  * Set the number of back to back packets allowed by this
1780                  * endpoint. Split transactions interpret "ec" as the number of
1781                  * immediate retries of failure. These retries happen too
1782                  * quickly, so we disable these entirely for splits
1783                  */
1784                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
1785                         usbc_hcchar.s.ec = 1;
1786                 else if (pipe->multi_count < 1)
1787                         usbc_hcchar.s.ec = 1;
1788                 else if (pipe->multi_count > 3)
1789                         usbc_hcchar.s.ec = 3;
1790                 else
1791                         usbc_hcchar.s.ec = pipe->multi_count;
1792
1793                 /* Set the rest of the endpoint specific settings */
1794                 usbc_hcchar.s.devaddr = pipe->device_addr;
1795                 usbc_hcchar.s.eptype = transaction->type;
1796                 usbc_hcchar.s.lspddev = (pipe->device_speed == CVMX_USB_SPEED_LOW);
1797                 usbc_hcchar.s.epdir = pipe->transfer_dir;
1798                 usbc_hcchar.s.epnum = pipe->endpoint_num;
1799                 usbc_hcchar.s.mps = pipe->max_packet;
1800                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index), usbc_hcchar.u32);
1801         }
1802
1803         /* Do transaction type specific fixups as needed */
1804         switch (transaction->type) {
1805         case CVMX_USB_TRANSFER_CONTROL:
1806                 __cvmx_usb_start_channel_control(usb, channel, pipe);
1807                 break;
1808         case CVMX_USB_TRANSFER_BULK:
1809         case CVMX_USB_TRANSFER_INTERRUPT:
1810                 break;
1811         case CVMX_USB_TRANSFER_ISOCHRONOUS:
1812                 if (!__cvmx_usb_pipe_needs_split(usb, pipe)) {
1813                         /*
1814                          * ISO transactions require different PIDs depending on
1815                          * direction and how many packets are needed
1816                          */
1817                         if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
1818                                 if (pipe->multi_count < 2) /* Need DATA0 */
1819                                         USB_SET_FIELD32(CVMX_USBCX_HCTSIZX(channel, usb->index), union cvmx_usbcx_hctsizx, pid, 0);
1820                                 else /* Need MDATA */
1821                                         USB_SET_FIELD32(CVMX_USBCX_HCTSIZX(channel, usb->index), union cvmx_usbcx_hctsizx, pid, 3);
1822                         }
1823                 }
1824                 break;
1825         }
1826         {
1827                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index))};
1828                 transaction->xfersize = usbc_hctsiz.s.xfersize;
1829                 transaction->pktcnt = usbc_hctsiz.s.pktcnt;
1830         }
1831         /* Remeber when we start a split transaction */
1832         if (__cvmx_usb_pipe_needs_split(usb, pipe))
1833                 usb->active_split = transaction;
1834         USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), union cvmx_usbcx_hccharx, chena, 1);
1835         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
1836                 __cvmx_usb_fill_tx_fifo(usb, channel);
1837         return;
1838 }
1839
1840
1841 /**
1842  * Find a pipe that is ready to be scheduled to hardware.
1843  * @usb:         USB device state populated by
1844  *               cvmx_usb_initialize().
1845  * @list:        Pipe list to search
1846  * @current_frame:
1847  *               Frame counter to use as a time reference.
1848  *
1849  * Returns: Pipe or NULL if none are ready
1850  */
1851 static struct cvmx_usb_pipe *__cvmx_usb_find_ready_pipe(struct cvmx_usb_internal_state *usb, struct cvmx_usb_pipe_list *list, uint64_t current_frame)
1852 {
1853         struct cvmx_usb_pipe *pipe = list->head;
1854         while (pipe) {
1855                 if (!(pipe->flags & __CVMX_USB_PIPE_FLAGS_SCHEDULED) && pipe->head &&
1856                         (pipe->next_tx_frame <= current_frame) &&
1857                         ((pipe->split_sc_frame == -1) || ((((int)current_frame - (int)pipe->split_sc_frame) & 0x7f) < 0x40)) &&
1858                         (!usb->active_split || (usb->active_split == pipe->head))) {
1859                         CVMX_PREFETCH(pipe, 128);
1860                         CVMX_PREFETCH(pipe->head, 0);
1861                         return pipe;
1862                 }
1863                 pipe = pipe->next;
1864         }
1865         return NULL;
1866 }
1867
1868
1869 /**
1870  * Called whenever a pipe might need to be scheduled to the
1871  * hardware.
1872  *
1873  * @usb:         USB device state populated by
1874  *               cvmx_usb_initialize().
1875  * @is_sof:      True if this schedule was called on a SOF interrupt.
1876  */
1877 static void __cvmx_usb_schedule(struct cvmx_usb_internal_state *usb, int is_sof)
1878 {
1879         int channel;
1880         struct cvmx_usb_pipe *pipe;
1881         int need_sof;
1882         enum cvmx_usb_transfer ttype;
1883
1884         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1885                 /* Without DMA we need to be careful to not schedule something at the end of a frame and cause an overrun */
1886                 union cvmx_usbcx_hfnum hfnum = {.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index))};
1887                 union cvmx_usbcx_hfir hfir = {.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HFIR(usb->index))};
1888                 if (hfnum.s.frrem < hfir.s.frint/4)
1889                         goto done;
1890         }
1891
1892         while (usb->idle_hardware_channels) {
1893                 /* Find an idle channel */
1894                 channel = __fls(usb->idle_hardware_channels);
1895                 if (unlikely(channel > 7))
1896                         break;
1897
1898                 /* Find a pipe needing service */
1899                 pipe = NULL;
1900                 if (is_sof) {
1901                         /*
1902                          * Only process periodic pipes on SOF interrupts. This
1903                          * way we are sure that the periodic data is sent in the
1904                          * beginning of the frame
1905                          */
1906                         pipe = __cvmx_usb_find_ready_pipe(usb, usb->active_pipes + CVMX_USB_TRANSFER_ISOCHRONOUS, usb->frame_number);
1907                         if (likely(!pipe))
1908                                 pipe = __cvmx_usb_find_ready_pipe(usb, usb->active_pipes + CVMX_USB_TRANSFER_INTERRUPT, usb->frame_number);
1909                 }
1910                 if (likely(!pipe)) {
1911                         pipe = __cvmx_usb_find_ready_pipe(usb, usb->active_pipes + CVMX_USB_TRANSFER_CONTROL, usb->frame_number);
1912                         if (likely(!pipe))
1913                                 pipe = __cvmx_usb_find_ready_pipe(usb, usb->active_pipes + CVMX_USB_TRANSFER_BULK, usb->frame_number);
1914                 }
1915                 if (!pipe)
1916                         break;
1917
1918                 __cvmx_usb_start_channel(usb, channel, pipe);
1919         }
1920
1921 done:
1922         /*
1923          * Only enable SOF interrupts when we have transactions pending in the
1924          * future that might need to be scheduled
1925          */
1926         need_sof = 0;
1927         for (ttype = CVMX_USB_TRANSFER_CONTROL; ttype <= CVMX_USB_TRANSFER_INTERRUPT; ttype++) {
1928                 pipe = usb->active_pipes[ttype].head;
1929                 while (pipe) {
1930                         if (pipe->next_tx_frame > usb->frame_number) {
1931                                 need_sof = 1;
1932                                 break;
1933                         }
1934                         pipe = pipe->next;
1935                 }
1936         }
1937         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), union cvmx_usbcx_gintmsk, sofmsk, need_sof);
1938         return;
1939 }
1940
1941
1942 /**
1943  * Call a user's callback for a specific reason.
1944  *
1945  * @usb:         USB device state populated by
1946  *               cvmx_usb_initialize().
1947  * @pipe:        Pipe the callback is for or NULL
1948  * @transaction:
1949  *               Transaction the callback is for or NULL
1950  * @reason:      Reason this callback is being called
1951  * @complete_code:
1952  *               Completion code for the transaction, if any
1953  */
1954 static void __cvmx_usb_perform_callback(struct cvmx_usb_internal_state *usb,
1955                                         struct cvmx_usb_pipe *pipe,
1956                                         struct cvmx_usb_transaction *transaction,
1957                                         enum cvmx_usb_callback reason,
1958                                         enum cvmx_usb_complete complete_code)
1959 {
1960         cvmx_usb_callback_func_t callback = usb->callback[reason];
1961         void *user_data = usb->callback_data[reason];
1962         int submit_handle = -1;
1963         int pipe_handle = -1;
1964         int bytes_transferred = 0;
1965
1966         if (pipe)
1967                 pipe_handle = __cvmx_usb_get_pipe_handle(usb, pipe);
1968
1969         if (transaction) {
1970                 submit_handle = __cvmx_usb_get_submit_handle(usb, transaction);
1971                 bytes_transferred = transaction->actual_bytes;
1972                 /* Transactions are allowed to override the default callback */
1973                 if ((reason == CVMX_USB_CALLBACK_TRANSFER_COMPLETE) && transaction->callback) {
1974                         callback = transaction->callback;
1975                         user_data = transaction->callback_data;
1976                 }
1977         }
1978
1979         if (!callback)
1980                 return;
1981
1982         callback((struct cvmx_usb_state *)usb, reason, complete_code, pipe_handle, submit_handle,
1983                  bytes_transferred, user_data);
1984 }
1985
1986
1987 /**
1988  * Signal the completion of a transaction and free it. The
1989  * transaction will be removed from the pipe transaction list.
1990  *
1991  * @usb:         USB device state populated by
1992  *               cvmx_usb_initialize().
1993  * @pipe:        Pipe the transaction is on
1994  * @transaction:
1995  *               Transaction that completed
1996  * @complete_code:
1997  *               Completion code
1998  */
1999 static void __cvmx_usb_perform_complete(struct cvmx_usb_internal_state *usb,
2000                                         struct cvmx_usb_pipe *pipe,
2001                                         struct cvmx_usb_transaction *transaction,
2002                                         enum cvmx_usb_complete complete_code)
2003 {
2004         /* If this was a split then clear our split in progress marker */
2005         if (usb->active_split == transaction)
2006                 usb->active_split = NULL;
2007
2008         /*
2009          * Isochronous transactions need extra processing as they might not be
2010          * done after a single data transfer
2011          */
2012         if (unlikely(transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
2013                 /* Update the number of bytes transferred in this ISO packet */
2014                 transaction->iso_packets[0].length = transaction->actual_bytes;
2015                 transaction->iso_packets[0].status = complete_code;
2016
2017                 /*
2018                  * If there are more ISOs pending and we succeeded, schedule the
2019                  * next one
2020                  */
2021                 if ((transaction->iso_number_packets > 1) && (complete_code == CVMX_USB_COMPLETE_SUCCESS)) {
2022                         transaction->actual_bytes = 0;     /* No bytes transferred for this packet as of yet */
2023                         transaction->iso_number_packets--; /* One less ISO waiting to transfer */
2024                         transaction->iso_packets++;        /* Increment to the next location in our packet array */
2025                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2026                         goto done;
2027                 }
2028         }
2029
2030         /* Remove the transaction from the pipe list */
2031         if (transaction->next)
2032                 transaction->next->prev = transaction->prev;
2033         else
2034                 pipe->tail = transaction->prev;
2035         if (transaction->prev)
2036                 transaction->prev->next = transaction->next;
2037         else
2038                 pipe->head = transaction->next;
2039         if (!pipe->head) {
2040                 __cvmx_usb_remove_pipe(usb->active_pipes + pipe->transfer_type, pipe);
2041                 __cvmx_usb_append_pipe(&usb->idle_pipes, pipe);
2042
2043         }
2044         __cvmx_usb_perform_callback(usb, pipe, transaction,
2045                                     CVMX_USB_CALLBACK_TRANSFER_COMPLETE,
2046                                     complete_code);
2047         __cvmx_usb_free_transaction(usb, transaction);
2048 done:
2049         return;
2050 }
2051
2052
2053 /**
2054  * Submit a usb transaction to a pipe. Called for all types
2055  * of transactions.
2056  *
2057  * @usb:
2058  * @pipe_handle:
2059  *                  Which pipe to submit to. Will be validated in this function.
2060  * @type:           Transaction type
2061  * @buffer:         User buffer for the transaction
2062  * @buffer_length:
2063  *                  User buffer's length in bytes
2064  * @control_header:
2065  *                  For control transactions, the 8 byte standard header
2066  * @iso_start_frame:
2067  *                  For ISO transactions, the start frame
2068  * @iso_number_packets:
2069  *                  For ISO, the number of packet in the transaction.
2070  * @iso_packets:
2071  *                  A description of each ISO packet
2072  * @callback:       User callback to call when the transaction completes
2073  * @user_data:      User's data for the callback
2074  *
2075  * Returns: Submit handle or negative on failure. Matches the result
2076  *          in the external API.
2077  */
2078 static int __cvmx_usb_submit_transaction(struct cvmx_usb_internal_state *usb,
2079                                          int pipe_handle,
2080                                          enum cvmx_usb_transfer type,
2081                                          uint64_t buffer,
2082                                          int buffer_length,
2083                                          uint64_t control_header,
2084                                          int iso_start_frame,
2085                                          int iso_number_packets,
2086                                          struct cvmx_usb_iso_packet *iso_packets,
2087                                          cvmx_usb_callback_func_t callback,
2088                                          void *user_data)
2089 {
2090         int submit_handle;
2091         struct cvmx_usb_transaction *transaction;
2092         struct cvmx_usb_pipe *pipe = usb->pipe + pipe_handle;
2093
2094         if (unlikely((pipe_handle < 0) || (pipe_handle >= MAX_PIPES)))
2095                 return -EINVAL;
2096         /* Fail if the pipe isn't open */
2097         if (unlikely((pipe->flags & __CVMX_USB_PIPE_FLAGS_OPEN) == 0))
2098                 return -EINVAL;
2099         if (unlikely(pipe->transfer_type != type))
2100                 return -EINVAL;
2101
2102         transaction = __cvmx_usb_alloc_transaction(usb);
2103         if (unlikely(!transaction))
2104                 return -ENOMEM;
2105
2106         transaction->type = type;
2107         transaction->buffer = buffer;
2108         transaction->buffer_length = buffer_length;
2109         transaction->control_header = control_header;
2110         transaction->iso_start_frame = iso_start_frame; // FIXME: This is not used, implement it
2111         transaction->iso_number_packets = iso_number_packets;
2112         transaction->iso_packets = iso_packets;
2113         transaction->callback = callback;
2114         transaction->callback_data = user_data;
2115         if (transaction->type == CVMX_USB_TRANSFER_CONTROL)
2116                 transaction->stage = CVMX_USB_STAGE_SETUP;
2117         else
2118                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2119
2120         transaction->next = NULL;
2121         if (pipe->tail) {
2122                 transaction->prev = pipe->tail;
2123                 transaction->prev->next = transaction;
2124         } else {
2125                 if (pipe->next_tx_frame < usb->frame_number)
2126                         pipe->next_tx_frame = usb->frame_number + pipe->interval -
2127                                 (usb->frame_number - pipe->next_tx_frame) % pipe->interval;
2128                 transaction->prev = NULL;
2129                 pipe->head = transaction;
2130                 __cvmx_usb_remove_pipe(&usb->idle_pipes, pipe);
2131                 __cvmx_usb_append_pipe(usb->active_pipes + pipe->transfer_type, pipe);
2132         }
2133         pipe->tail = transaction;
2134
2135         submit_handle = __cvmx_usb_get_submit_handle(usb, transaction);
2136
2137         /* We may need to schedule the pipe if this was the head of the pipe */
2138         if (!transaction->prev)
2139                 __cvmx_usb_schedule(usb, 0);
2140
2141         return submit_handle;
2142 }
2143
2144
2145 /**
2146  * Call to submit a USB Bulk transfer to a pipe.
2147  *
2148  * @state:          USB device state populated by
2149  *                  cvmx_usb_initialize().
2150  * @pipe_handle:
2151  *                  Handle to the pipe for the transfer.
2152  * @buffer:         Physical address of the data buffer in
2153  *                  memory. Note that this is NOT A POINTER, but
2154  *                  the full 64bit physical address of the
2155  *                  buffer. This may be zero if buffer_length is
2156  *                  zero.
2157  * @buffer_length:
2158  *                  Length of buffer in bytes.
2159  * @callback:       Function to call when this transaction
2160  *                  completes. If the return value of this
2161  *                  function isn't an error, then this function
2162  *                  is guaranteed to be called when the
2163  *                  transaction completes. If this parameter is
2164  *                  NULL, then the generic callback registered
2165  *                  through cvmx_usb_register_callback is
2166  *                  called. If both are NULL, then there is no
2167  *                  way to know when a transaction completes.
2168  * @user_data:      User supplied data returned when the
2169  *                  callback is called. This is only used if
2170  *                  callback in not NULL.
2171  *
2172  * Returns: A submitted transaction handle or negative on
2173  *          failure. Negative values are error codes.
2174  */
2175 int cvmx_usb_submit_bulk(struct cvmx_usb_state *state, int pipe_handle,
2176                          uint64_t buffer, int buffer_length,
2177                          cvmx_usb_callback_func_t callback,
2178                          void *user_data)
2179 {
2180         int submit_handle;
2181         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
2182
2183         /* Pipe handle checking is done later in a common place */
2184         if (unlikely(!buffer))
2185                 return -EINVAL;
2186         if (unlikely(buffer_length < 0))
2187                 return -EINVAL;
2188
2189         submit_handle = __cvmx_usb_submit_transaction(usb, pipe_handle,
2190                                                       CVMX_USB_TRANSFER_BULK,
2191                                                       buffer,
2192                                                       buffer_length,
2193                                                       0, /* control_header */
2194                                                       0, /* iso_start_frame */
2195                                                       0, /* iso_number_packets */
2196                                                       NULL, /* iso_packets */
2197                                                       callback,
2198                                                       user_data);
2199         return submit_handle;
2200 }
2201
2202
2203 /**
2204  * Call to submit a USB Interrupt transfer to a pipe.
2205  *
2206  * @state:          USB device state populated by
2207  *                  cvmx_usb_initialize().
2208  * @pipe_handle:
2209  *                  Handle to the pipe for the transfer.
2210  * @buffer:         Physical address of the data buffer in
2211  *                  memory. Note that this is NOT A POINTER, but
2212  *                  the full 64bit physical address of the
2213  *                  buffer. This may be zero if buffer_length is
2214  *                  zero.
2215  * @buffer_length:
2216  *                  Length of buffer in bytes.
2217  * @callback:       Function to call when this transaction
2218  *                  completes. If the return value of this
2219  *                  function isn't an error, then this function
2220  *                  is guaranteed to be called when the
2221  *                  transaction completes. If this parameter is
2222  *                  NULL, then the generic callback registered
2223  *                  through cvmx_usb_register_callback is
2224  *                  called. If both are NULL, then there is no
2225  *                  way to know when a transaction completes.
2226  * @user_data:      User supplied data returned when the
2227  *                  callback is called. This is only used if
2228  *                  callback in not NULL.
2229  *
2230  * Returns: A submitted transaction handle or negative on
2231  *          failure. Negative values are error codes.
2232  */
2233 int cvmx_usb_submit_interrupt(struct cvmx_usb_state *state, int pipe_handle,
2234                               uint64_t buffer, int buffer_length,
2235                               cvmx_usb_callback_func_t callback,
2236                               void *user_data)
2237 {
2238         int submit_handle;
2239         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
2240
2241         /* Pipe handle checking is done later in a common place */
2242         if (unlikely(!buffer))
2243                 return -EINVAL;
2244         if (unlikely(buffer_length < 0))
2245                 return -EINVAL;
2246
2247         submit_handle = __cvmx_usb_submit_transaction(usb, pipe_handle,
2248                                                       CVMX_USB_TRANSFER_INTERRUPT,
2249                                                       buffer,
2250                                                       buffer_length,
2251                                                       0, /* control_header */
2252                                                       0, /* iso_start_frame */
2253                                                       0, /* iso_number_packets */
2254                                                       NULL, /* iso_packets */
2255                                                       callback,
2256                                                       user_data);
2257         return submit_handle;
2258 }
2259
2260
2261 /**
2262  * Call to submit a USB Control transfer to a pipe.
2263  *
2264  * @state:          USB device state populated by
2265  *                  cvmx_usb_initialize().
2266  * @pipe_handle:
2267  *                  Handle to the pipe for the transfer.
2268  * @control_header:
2269  *                  USB 8 byte control header physical address.
2270  *                  Note that this is NOT A POINTER, but the
2271  *                  full 64bit physical address of the buffer.
2272  * @buffer:         Physical address of the data buffer in
2273  *                  memory. Note that this is NOT A POINTER, but
2274  *                  the full 64bit physical address of the
2275  *                  buffer. This may be zero if buffer_length is
2276  *                  zero.
2277  * @buffer_length:
2278  *                  Length of buffer in bytes.
2279  * @callback:       Function to call when this transaction
2280  *                  completes. If the return value of this
2281  *                  function isn't an error, then this function
2282  *                  is guaranteed to be called when the
2283  *                  transaction completes. If this parameter is
2284  *                  NULL, then the generic callback registered
2285  *                  through cvmx_usb_register_callback is
2286  *                  called. If both are NULL, then there is no
2287  *                  way to know when a transaction completes.
2288  * @user_data:      User supplied data returned when the
2289  *                  callback is called. This is only used if
2290  *                  callback in not NULL.
2291  *
2292  * Returns: A submitted transaction handle or negative on
2293  *          failure. Negative values are error codes.
2294  */
2295 int cvmx_usb_submit_control(struct cvmx_usb_state *state, int pipe_handle,
2296                             uint64_t control_header,
2297                             uint64_t buffer, int buffer_length,
2298                             cvmx_usb_callback_func_t callback,
2299                             void *user_data)
2300 {
2301         int submit_handle;
2302         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
2303         union cvmx_usb_control_header *header =
2304                 cvmx_phys_to_ptr(control_header);
2305
2306         /* Pipe handle checking is done later in a common place */
2307         if (unlikely(!control_header))
2308                 return -EINVAL;
2309         /* Some drivers send a buffer with a zero length. God only knows why */
2310         if (unlikely(buffer && (buffer_length < 0)))
2311                 return -EINVAL;
2312         if (unlikely(!buffer && (buffer_length != 0)))
2313                 return -EINVAL;
2314         if ((header->s.request_type & 0x80) == 0)
2315                 buffer_length = le16_to_cpu(header->s.length);
2316
2317         submit_handle = __cvmx_usb_submit_transaction(usb, pipe_handle,
2318                                                       CVMX_USB_TRANSFER_CONTROL,
2319                                                       buffer,
2320                                                       buffer_length,
2321                                                       control_header,
2322                                                       0, /* iso_start_frame */
2323                                                       0, /* iso_number_packets */
2324                                                       NULL, /* iso_packets */
2325                                                       callback,
2326                                                       user_data);
2327         return submit_handle;
2328 }
2329
2330
2331 /**
2332  * Call to submit a USB Isochronous transfer to a pipe.
2333  *
2334  * @state:          USB device state populated by
2335  *                  cvmx_usb_initialize().
2336  * @pipe_handle:
2337  *                  Handle to the pipe for the transfer.
2338  * @start_frame:
2339  *                  Number of frames into the future to schedule
2340  *                  this transaction.
2341  * @number_packets:
2342  *                  Number of sequential packets to transfer.
2343  *                  "packets" is a pointer to an array of this
2344  *                  many packet structures.
2345  * @packets:        Description of each transfer packet as
2346  *                  defined by struct cvmx_usb_iso_packet. The array
2347  *                  pointed to here must stay valid until the
2348  *                  complete callback is called.
2349  * @buffer:         Physical address of the data buffer in
2350  *                  memory. Note that this is NOT A POINTER, but
2351  *                  the full 64bit physical address of the
2352  *                  buffer. This may be zero if buffer_length is
2353  *                  zero.
2354  * @buffer_length:
2355  *                  Length of buffer in bytes.
2356  * @callback:       Function to call when this transaction
2357  *                  completes. If the return value of this
2358  *                  function isn't an error, then this function
2359  *                  is guaranteed to be called when the
2360  *                  transaction completes. If this parameter is
2361  *                  NULL, then the generic callback registered
2362  *                  through cvmx_usb_register_callback is
2363  *                  called. If both are NULL, then there is no
2364  *                  way to know when a transaction completes.
2365  * @user_data:      User supplied data returned when the
2366  *                  callback is called. This is only used if
2367  *                  callback in not NULL.
2368  *
2369  * Returns: A submitted transaction handle or negative on
2370  *          failure. Negative values are error codes.
2371  */
2372 int cvmx_usb_submit_isochronous(struct cvmx_usb_state *state, int pipe_handle,
2373                                 int start_frame,
2374                                 int number_packets,
2375                                 struct cvmx_usb_iso_packet packets[],
2376                                 uint64_t buffer, int buffer_length,
2377                                 cvmx_usb_callback_func_t callback,
2378                                 void *user_data)
2379 {
2380         int submit_handle;
2381         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
2382
2383         /* Pipe handle checking is done later in a common place */
2384         if (unlikely(start_frame < 0))
2385                 return -EINVAL;
2386         if (unlikely(number_packets < 1))
2387                 return -EINVAL;
2388         if (unlikely(!packets))
2389                 return -EINVAL;
2390         if (unlikely(!buffer))
2391                 return -EINVAL;
2392         if (unlikely(buffer_length < 0))
2393                 return -EINVAL;
2394
2395         submit_handle = __cvmx_usb_submit_transaction(usb, pipe_handle,
2396                                                       CVMX_USB_TRANSFER_ISOCHRONOUS,
2397                                                       buffer,
2398                                                       buffer_length,
2399                                                       0, /* control_header */
2400                                                       start_frame,
2401                                                       number_packets,
2402                                                       packets,
2403                                                       callback,
2404                                                       user_data);
2405         return submit_handle;
2406 }
2407
2408
2409 /**
2410  * Cancel one outstanding request in a pipe. Canceling a request
2411  * can fail if the transaction has already completed before cancel
2412  * is called. Even after a successful cancel call, it may take
2413  * a frame or two for the cvmx_usb_poll() function to call the
2414  * associated callback.
2415  *
2416  * @state:       USB device state populated by
2417  *               cvmx_usb_initialize().
2418  * @pipe_handle:
2419  *               Pipe handle to cancel requests in.
2420  * @submit_handle:
2421  *               Handle to transaction to cancel, returned by the submit function.
2422  *
2423  * Returns: 0 or a negative error code.
2424  */
2425 int cvmx_usb_cancel(struct cvmx_usb_state *state, int pipe_handle, int submit_handle)
2426 {
2427         struct cvmx_usb_transaction *transaction;
2428         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
2429         struct cvmx_usb_pipe *pipe = usb->pipe + pipe_handle;
2430
2431         if (unlikely((pipe_handle < 0) || (pipe_handle >= MAX_PIPES)))
2432                 return -EINVAL;
2433         if (unlikely((submit_handle < 0) || (submit_handle >= MAX_TRANSACTIONS)))
2434                 return -EINVAL;
2435
2436         /* Fail if the pipe isn't open */
2437         if (unlikely((pipe->flags & __CVMX_USB_PIPE_FLAGS_OPEN) == 0))
2438                 return -EINVAL;
2439
2440         transaction = usb->transaction + submit_handle;
2441
2442         /* Fail if this transaction already completed */
2443         if (unlikely((transaction->flags & __CVMX_USB_TRANSACTION_FLAGS_IN_USE) == 0))
2444                 return -EINVAL;
2445
2446         /*
2447          * If the transaction is the HEAD of the queue and scheduled. We need to
2448          * treat it special
2449          */
2450         if ((pipe->head == transaction) &&
2451                 (pipe->flags & __CVMX_USB_PIPE_FLAGS_SCHEDULED)) {
2452                 union cvmx_usbcx_hccharx usbc_hcchar;
2453
2454                 usb->pipe_for_channel[pipe->channel] = NULL;
2455                 pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_SCHEDULED;
2456
2457                 CVMX_SYNCW;
2458
2459                 usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCCHARX(pipe->channel, usb->index));
2460                 /* If the channel isn't enabled then the transaction already completed */
2461                 if (usbc_hcchar.s.chena) {
2462                         usbc_hcchar.s.chdis = 1;
2463                         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCCHARX(pipe->channel, usb->index), usbc_hcchar.u32);
2464                 }
2465         }
2466         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_CANCEL);
2467         return 0;
2468 }
2469
2470
2471 /**
2472  * Cancel all outstanding requests in a pipe. Logically all this
2473  * does is call cvmx_usb_cancel() in a loop.
2474  *
2475  * @state:       USB device state populated by
2476  *               cvmx_usb_initialize().
2477  * @pipe_handle:
2478  *               Pipe handle to cancel requests in.
2479  *
2480  * Returns: 0 or a negative error code.
2481  */
2482 int cvmx_usb_cancel_all(struct cvmx_usb_state *state, int pipe_handle)
2483 {
2484         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
2485         struct cvmx_usb_pipe *pipe = usb->pipe + pipe_handle;
2486
2487         if (unlikely((pipe_handle < 0) || (pipe_handle >= MAX_PIPES)))
2488                 return -EINVAL;
2489
2490         /* Fail if the pipe isn't open */
2491         if (unlikely((pipe->flags & __CVMX_USB_PIPE_FLAGS_OPEN) == 0))
2492                 return -EINVAL;
2493
2494         /* Simply loop through and attempt to cancel each transaction */
2495         while (pipe->head) {
2496                 int result = cvmx_usb_cancel(state, pipe_handle,
2497                         __cvmx_usb_get_submit_handle(usb, pipe->head));
2498                 if (unlikely(result != 0))
2499                         return result;
2500         }
2501         return 0;
2502 }
2503
2504
2505 /**
2506  * Close a pipe created with cvmx_usb_open_pipe().
2507  *
2508  * @state:       USB device state populated by
2509  *               cvmx_usb_initialize().
2510  * @pipe_handle:
2511  *               Pipe handle to close.
2512  *
2513  * Returns: 0 or a negative error code. EBUSY is returned if the pipe has
2514  *          outstanding transfers.
2515  */
2516 int cvmx_usb_close_pipe(struct cvmx_usb_state *state, int pipe_handle)
2517 {
2518         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
2519         struct cvmx_usb_pipe *pipe = usb->pipe + pipe_handle;
2520
2521         if (unlikely((pipe_handle < 0) || (pipe_handle >= MAX_PIPES)))
2522                 return -EINVAL;
2523
2524         /* Fail if the pipe isn't open */
2525         if (unlikely((pipe->flags & __CVMX_USB_PIPE_FLAGS_OPEN) == 0))
2526                 return -EINVAL;
2527
2528         /* Fail if the pipe has pending transactions */
2529         if (unlikely(pipe->head))
2530                 return -EBUSY;
2531
2532         pipe->flags = 0;
2533         __cvmx_usb_remove_pipe(&usb->idle_pipes, pipe);
2534         __cvmx_usb_append_pipe(&usb->free_pipes, pipe);
2535
2536         return 0;
2537 }
2538
2539
2540 /**
2541  * Register a function to be called when various USB events occur.
2542  *
2543  * @state:     USB device state populated by
2544  *             cvmx_usb_initialize().
2545  * @reason:    Which event to register for.
2546  * @callback:  Function to call when the event occurs.
2547  * @user_data: User data parameter to the function.
2548  *
2549  * Returns: 0 or a negative error code.
2550  */
2551 int cvmx_usb_register_callback(struct cvmx_usb_state *state,
2552                                enum cvmx_usb_callback reason,
2553                                cvmx_usb_callback_func_t callback,
2554                                void *user_data)
2555 {
2556         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
2557
2558         if (unlikely(reason >= __CVMX_USB_CALLBACK_END))
2559                 return -EINVAL;
2560         if (unlikely(!callback))
2561                 return -EINVAL;
2562
2563         usb->callback[reason] = callback;
2564         usb->callback_data[reason] = user_data;
2565
2566         return 0;
2567 }
2568
2569
2570 /**
2571  * Get the current USB protocol level frame number. The frame
2572  * number is always in the range of 0-0x7ff.
2573  *
2574  * @state: USB device state populated by
2575  *         cvmx_usb_initialize().
2576  *
2577  * Returns: USB frame number
2578  */
2579 int cvmx_usb_get_frame_number(struct cvmx_usb_state *state)
2580 {
2581         int frame_number;
2582         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
2583         union cvmx_usbcx_hfnum usbc_hfnum;
2584
2585         usbc_hfnum.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
2586         frame_number = usbc_hfnum.s.frnum;
2587
2588         return frame_number;
2589 }
2590
2591
2592 /**
2593  * Poll a channel for status
2594  *
2595  * @usb:     USB device
2596  * @channel: Channel to poll
2597  *
2598  * Returns: Zero on success
2599  */
2600 static int __cvmx_usb_poll_channel(struct cvmx_usb_internal_state *usb, int channel)
2601 {
2602         union cvmx_usbcx_hcintx usbc_hcint;
2603         union cvmx_usbcx_hctsizx usbc_hctsiz;
2604         union cvmx_usbcx_hccharx usbc_hcchar;
2605         struct cvmx_usb_pipe *pipe;
2606         struct cvmx_usb_transaction *transaction;
2607         int bytes_this_transfer;
2608         int bytes_in_last_packet;
2609         int packets_processed;
2610         int buffer_space_left;
2611
2612         /* Read the interrupt status bits for the channel */
2613         usbc_hcint.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCINTX(channel, usb->index));
2614
2615         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
2616                 usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index));
2617
2618                 if (usbc_hcchar.s.chena && usbc_hcchar.s.chdis) {
2619                         /*
2620                          * There seems to be a bug in CN31XX which can cause
2621                          * interrupt IN transfers to get stuck until we do a
2622                          * write of HCCHARX without changing things
2623                          */
2624                         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index), usbc_hcchar.u32);
2625                         return 0;
2626                 }
2627
2628                 /*
2629                  * In non DMA mode the channels don't halt themselves. We need
2630                  * to manually disable channels that are left running
2631                  */
2632                 if (!usbc_hcint.s.chhltd) {
2633                         if (usbc_hcchar.s.chena) {
2634                                 union cvmx_usbcx_hcintmskx hcintmsk;
2635                                 /* Disable all interrupts except CHHLTD */
2636                                 hcintmsk.u32 = 0;
2637                                 hcintmsk.s.chhltdmsk = 1;
2638                                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), hcintmsk.u32);
2639                                 usbc_hcchar.s.chdis = 1;
2640                                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index), usbc_hcchar.u32);
2641                                 return 0;
2642                         } else if (usbc_hcint.s.xfercompl) {
2643                                 /* Successful IN/OUT with transfer complete. Channel halt isn't needed */
2644                         } else {
2645                                 cvmx_dprintf("USB%d: Channel %d interrupt without halt\n", usb->index, channel);
2646                                 return 0;
2647                         }
2648                 }
2649         } else {
2650                 /*
2651                  * There is are no interrupts that we need to process when the
2652                  * channel is still running
2653                  */
2654                 if (!usbc_hcint.s.chhltd)
2655                         return 0;
2656         }
2657
2658         /* Disable the channel interrupts now that it is done */
2659         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
2660         usb->idle_hardware_channels |= (1<<channel);
2661
2662         /* Make sure this channel is tied to a valid pipe */
2663         pipe = usb->pipe_for_channel[channel];
2664         CVMX_PREFETCH(pipe, 0);
2665         CVMX_PREFETCH(pipe, 128);
2666         if (!pipe)
2667                 return 0;
2668         transaction = pipe->head;
2669         CVMX_PREFETCH(transaction, 0);
2670
2671         /*
2672          * Disconnect this pipe from the HW channel. Later the schedule
2673          * function will figure out which pipe needs to go
2674          */
2675         usb->pipe_for_channel[channel] = NULL;
2676         pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_SCHEDULED;
2677
2678         /*
2679          * Read the channel config info so we can figure out how much data
2680          * transfered
2681          */
2682         usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index));
2683         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index));
2684
2685         /*
2686          * Calculating the number of bytes successfully transferred is dependent
2687          * on the transfer direction
2688          */
2689         packets_processed = transaction->pktcnt - usbc_hctsiz.s.pktcnt;
2690         if (usbc_hcchar.s.epdir) {
2691                 /*
2692                  * IN transactions are easy. For every byte received the
2693                  * hardware decrements xfersize. All we need to do is subtract
2694                  * the current value of xfersize from its starting value and we
2695                  * know how many bytes were written to the buffer
2696                  */
2697                 bytes_this_transfer = transaction->xfersize - usbc_hctsiz.s.xfersize;
2698         } else {
2699                 /*
2700                  * OUT transaction don't decrement xfersize. Instead pktcnt is
2701                  * decremented on every successful packet send. The hardware
2702                  * does this when it receives an ACK, or NYET. If it doesn't
2703                  * receive one of these responses pktcnt doesn't change
2704                  */
2705                 bytes_this_transfer = packets_processed * usbc_hcchar.s.mps;
2706                 /*
2707                  * The last packet may not be a full transfer if we didn't have
2708                  * enough data
2709                  */
2710                 if (bytes_this_transfer > transaction->xfersize)
2711                         bytes_this_transfer = transaction->xfersize;
2712         }
2713         /* Figure out how many bytes were in the last packet of the transfer */
2714         if (packets_processed)
2715                 bytes_in_last_packet = bytes_this_transfer - (packets_processed-1) * usbc_hcchar.s.mps;
2716         else
2717                 bytes_in_last_packet = bytes_this_transfer;
2718
2719         /*
2720          * As a special case, setup transactions output the setup header, not
2721          * the user's data. For this reason we don't count setup data as bytes
2722          * transferred
2723          */
2724         if ((transaction->stage == CVMX_USB_STAGE_SETUP) ||
2725                 (transaction->stage == CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE))
2726                 bytes_this_transfer = 0;
2727
2728         /*
2729          * Add the bytes transferred to the running total. It is important that
2730          * bytes_this_transfer doesn't count any data that needs to be
2731          * retransmitted
2732          */
2733         transaction->actual_bytes += bytes_this_transfer;
2734         if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
2735                 buffer_space_left = transaction->iso_packets[0].length - transaction->actual_bytes;
2736         else
2737                 buffer_space_left = transaction->buffer_length - transaction->actual_bytes;
2738
2739         /*
2740          * We need to remember the PID toggle state for the next transaction.
2741          * The hardware already updated it for the next transaction
2742          */
2743         pipe->pid_toggle = !(usbc_hctsiz.s.pid == 0);
2744
2745         /*
2746          * For high speed bulk out, assume the next transaction will need to do
2747          * a ping before proceeding. If this isn't true the ACK processing below
2748          * will clear this flag
2749          */
2750         if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2751                 (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
2752                 (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT))
2753                 pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
2754
2755         if (usbc_hcint.s.stall) {
2756                 /*
2757                  * STALL as a response means this transaction cannot be
2758                  * completed because the device can't process transactions. Tell
2759                  * the user. Any data that was transferred will be counted on
2760                  * the actual bytes transferred
2761                  */
2762                 pipe->pid_toggle = 0;
2763                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_STALL);
2764         } else if (usbc_hcint.s.xacterr) {
2765                 /*
2766                  * We know at least one packet worked if we get a ACK or NAK.
2767                  * Reset the retry counter
2768                  */
2769                 if (usbc_hcint.s.nak || usbc_hcint.s.ack)
2770                         transaction->retries = 0;
2771                 transaction->retries++;
2772                 if (transaction->retries > MAX_RETRIES) {
2773                         /*
2774                          * XactErr as a response means the device signaled
2775                          * something wrong with the transfer. For example, PID
2776                          * toggle errors cause these
2777                          */
2778                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_XACTERR);
2779                 } else {
2780                         /*
2781                          * If this was a split then clear our split in progress
2782                          * marker
2783                          */
2784                         if (usb->active_split == transaction)
2785                                 usb->active_split = NULL;
2786                         /*
2787                          * Rewind to the beginning of the transaction by anding
2788                          * off the split complete bit
2789                          */
2790                         transaction->stage &= ~1;
2791                         pipe->split_sc_frame = -1;
2792                         pipe->next_tx_frame += pipe->interval;
2793                         if (pipe->next_tx_frame < usb->frame_number)
2794                                 pipe->next_tx_frame = usb->frame_number + pipe->interval -
2795                                                       (usb->frame_number - pipe->next_tx_frame) % pipe->interval;
2796                 }
2797         } else if (usbc_hcint.s.bblerr) {
2798                 /* Babble Error (BblErr) */
2799                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_BABBLEERR);
2800         } else if (usbc_hcint.s.datatglerr) {
2801                 /* We'll retry the exact same transaction again */
2802                 transaction->retries++;
2803         } else if (usbc_hcint.s.nyet) {
2804                 /*
2805                  * NYET as a response is only allowed in three cases: as a
2806                  * response to a ping, as a response to a split transaction, and
2807                  * as a response to a bulk out. The ping case is handled by
2808                  * hardware, so we only have splits and bulk out
2809                  */
2810                 if (!__cvmx_usb_pipe_needs_split(usb, pipe)) {
2811                         transaction->retries = 0;
2812                         /*
2813                          * If there is more data to go then we need to try
2814                          * again. Otherwise this transaction is complete
2815                          */
2816                         if ((buffer_space_left == 0) || (bytes_in_last_packet < pipe->max_packet))
2817                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2818                 } else {
2819                         /*
2820                          * Split transactions retry the split complete 4 times
2821                          * then rewind to the start split and do the entire
2822                          * transactions again
2823                          */
2824                         transaction->retries++;
2825                         if ((transaction->retries & 0x3) == 0) {
2826                                 /*
2827                                  * Rewind to the beginning of the transaction by
2828                                  * anding off the split complete bit
2829                                  */
2830                                 transaction->stage &= ~1;
2831                                 pipe->split_sc_frame = -1;
2832                         }
2833                 }
2834         } else if (usbc_hcint.s.ack) {
2835                 transaction->retries = 0;
2836                 /*
2837                  * The ACK bit can only be checked after the other error bits.
2838                  * This is because a multi packet transfer may succeed in a
2839                  * number of packets and then get a different response on the
2840                  * last packet. In this case both ACK and the last response bit
2841                  * will be set. If none of the other response bits is set, then
2842                  * the last packet must have been an ACK
2843                  *
2844                  * Since we got an ACK, we know we don't need to do a ping on
2845                  * this pipe
2846                  */
2847                 pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_NEED_PING;
2848
2849                 switch (transaction->type) {
2850                 case CVMX_USB_TRANSFER_CONTROL:
2851                         switch (transaction->stage) {
2852                         case CVMX_USB_STAGE_NON_CONTROL:
2853                         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
2854                                 /* This should be impossible */
2855                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_ERROR);
2856                                 break;
2857                         case CVMX_USB_STAGE_SETUP:
2858                                 pipe->pid_toggle = 1;
2859                                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
2860                                         transaction->stage = CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE;
2861                                 else {
2862                                         union cvmx_usb_control_header *header =
2863                                                 cvmx_phys_to_ptr(transaction->control_header);
2864                                         if (header->s.length)
2865                                                 transaction->stage = CVMX_USB_STAGE_DATA;
2866                                         else
2867                                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2868                                 }
2869                                 break;
2870                         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
2871                                 {
2872                                         union cvmx_usb_control_header *header =
2873                                                 cvmx_phys_to_ptr(transaction->control_header);
2874                                         if (header->s.length)
2875                                                 transaction->stage = CVMX_USB_STAGE_DATA;
2876                                         else
2877                                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2878                                 }
2879                                 break;
2880                         case CVMX_USB_STAGE_DATA:
2881                                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2882                                         transaction->stage = CVMX_USB_STAGE_DATA_SPLIT_COMPLETE;
2883                                         /*
2884                                          * For setup OUT data that are splits,
2885                                          * the hardware doesn't appear to count
2886                                          * transferred data. Here we manually
2887                                          * update the data transferred
2888                                          */
2889                                         if (!usbc_hcchar.s.epdir) {
2890                                                 if (buffer_space_left < pipe->max_packet)
2891                                                         transaction->actual_bytes += buffer_space_left;
2892                                                 else
2893                                                         transaction->actual_bytes += pipe->max_packet;
2894                                         }
2895                                 } else if ((buffer_space_left == 0) || (bytes_in_last_packet < pipe->max_packet)) {
2896                                         pipe->pid_toggle = 1;
2897                                         transaction->stage = CVMX_USB_STAGE_STATUS;
2898                                 }
2899                                 break;
2900                         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
2901                                 if ((buffer_space_left == 0) || (bytes_in_last_packet < pipe->max_packet)) {
2902                                         pipe->pid_toggle = 1;
2903                                         transaction->stage = CVMX_USB_STAGE_STATUS;
2904                                 } else {
2905                                         transaction->stage = CVMX_USB_STAGE_DATA;
2906                                 }
2907                                 break;
2908                         case CVMX_USB_STAGE_STATUS:
2909                                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
2910                                         transaction->stage = CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE;
2911                                 else
2912                                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2913                                 break;
2914                         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
2915                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2916                                 break;
2917                         }
2918                         break;
2919                 case CVMX_USB_TRANSFER_BULK:
2920                 case CVMX_USB_TRANSFER_INTERRUPT:
2921                         /*
2922                          * The only time a bulk transfer isn't complete when it
2923                          * finishes with an ACK is during a split transaction.
2924                          * For splits we need to continue the transfer if more
2925                          * data is needed
2926                          */
2927                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2928                                 if (transaction->stage == CVMX_USB_STAGE_NON_CONTROL)
2929                                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2930                                 else {
2931                                         if (buffer_space_left && (bytes_in_last_packet == pipe->max_packet))
2932                                                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2933                                         else {
2934                                                 if (transaction->type == CVMX_USB_TRANSFER_INTERRUPT)
2935                                                         pipe->next_tx_frame += pipe->interval;
2936                                                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2937                                         }
2938                                 }
2939                         } else {
2940                                 if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2941                                     (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
2942                                     (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) &&
2943                                     (usbc_hcint.s.nak))
2944                                         pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
2945                                 if (!buffer_space_left || (bytes_in_last_packet < pipe->max_packet)) {
2946                                         if (transaction->type == CVMX_USB_TRANSFER_INTERRUPT)
2947                                                 pipe->next_tx_frame += pipe->interval;
2948                                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2949                                 }
2950                         }
2951                         break;
2952                 case CVMX_USB_TRANSFER_ISOCHRONOUS:
2953                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2954                                 /*
2955                                  * ISOCHRONOUS OUT splits don't require a
2956                                  * complete split stage. Instead they use a
2957                                  * sequence of begin OUT splits to transfer the
2958                                  * data 188 bytes at a time. Once the transfer
2959                                  * is complete, the pipe sleeps until the next
2960                                  * schedule interval
2961                                  */
2962                                 if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
2963                                         /*
2964                                          * If no space left or this wasn't a max
2965                                          * size packet then this transfer is
2966                                          * complete. Otherwise start it again to
2967                                          * send the next 188 bytes
2968                                          */
2969                                         if (!buffer_space_left || (bytes_this_transfer < 188)) {
2970                                                 pipe->next_tx_frame += pipe->interval;
2971                                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2972                                         }
2973                                 } else {
2974                                         if (transaction->stage == CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE) {
2975                                                 /*
2976                                                  * We are in the incoming data
2977                                                  * phase. Keep getting data
2978                                                  * until we run out of space or
2979                                                  * get a small packet
2980                                                  */
2981                                                 if ((buffer_space_left == 0) || (bytes_in_last_packet < pipe->max_packet)) {
2982                                                         pipe->next_tx_frame += pipe->interval;
2983                                                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2984                                                 }
2985                                         } else
2986                                                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2987                                 }
2988                         } else {
2989                                 pipe->next_tx_frame += pipe->interval;
2990                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2991                         }
2992                         break;
2993                 }
2994         } else if (usbc_hcint.s.nak) {
2995                 /* If this was a split then clear our split in progress marker */
2996                 if (usb->active_split == transaction)
2997                         usb->active_split = NULL;
2998                 /*
2999                  * NAK as a response means the device couldn't accept the
3000                  * transaction, but it should be retried in the future. Rewind
3001                  * to the beginning of the transaction by anding off the split
3002                  * complete bit. Retry in the next interval
3003                  */
3004                 transaction->retries = 0;
3005                 transaction->stage &= ~1;
3006                 pipe->next_tx_frame += pipe->interval;
3007                 if (pipe->next_tx_frame < usb->frame_number)
3008                         pipe->next_tx_frame = usb->frame_number + pipe->interval -
3009                                 (usb->frame_number - pipe->next_tx_frame) % pipe->interval;
3010         } else {
3011                 struct cvmx_usb_port_status port;
3012                 port = cvmx_usb_get_status((struct cvmx_usb_state *)usb);
3013                 if (port.port_enabled) {
3014                         /* We'll retry the exact same transaction again */
3015                         transaction->retries++;
3016                 } else {
3017                         /*
3018                          * We get channel halted interrupts with no result bits
3019                          * sets when the cable is unplugged
3020                          */
3021                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_ERROR);
3022                 }
3023         }
3024         return 0;
3025 }
3026
3027
3028 /**
3029  * Poll the USB block for status and call all needed callback
3030  * handlers. This function is meant to be called in the interrupt
3031  * handler for the USB controller. It can also be called
3032  * periodically in a loop for non-interrupt based operation.
3033  *
3034  * @state:      USB device state populated by
3035  *              cvmx_usb_initialize().
3036  *
3037  * Returns: 0 or a negative error code.
3038  */
3039 int cvmx_usb_poll(struct cvmx_usb_state *state)
3040 {
3041         union cvmx_usbcx_hfnum usbc_hfnum;
3042         union cvmx_usbcx_gintsts usbc_gintsts;
3043         struct cvmx_usb_internal_state *usb = (struct cvmx_usb_internal_state *)state;
3044
3045         CVMX_PREFETCH(usb, 0);
3046         CVMX_PREFETCH(usb, 1*128);
3047         CVMX_PREFETCH(usb, 2*128);
3048         CVMX_PREFETCH(usb, 3*128);
3049         CVMX_PREFETCH(usb, 4*128);
3050
3051         /* Update the frame counter */
3052         usbc_hfnum.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
3053         if ((usb->frame_number&0x3fff) > usbc_hfnum.s.frnum)
3054                 usb->frame_number += 0x4000;
3055         usb->frame_number &= ~0x3fffull;
3056         usb->frame_number |= usbc_hfnum.s.frnum;
3057
3058         /* Read the pending interrupts */
3059         usbc_gintsts.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GINTSTS(usb->index));
3060
3061         /* Clear the interrupts now that we know about them */
3062         __cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTSTS(usb->index), usbc_gintsts.u32);
3063
3064         if (usbc_gintsts.s.rxflvl) {
3065                 /*
3066                  * RxFIFO Non-Empty (RxFLvl)
3067                  * Indicates that there is at least one packet pending to be
3068                  * read from the RxFIFO.
3069                  *
3070                  * In DMA mode this is handled by hardware
3071                  */
3072                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
3073                         __cvmx_usb_poll_rx_fifo(usb);
3074         }
3075         if (usbc_gintsts.s.ptxfemp || usbc_gintsts.s.nptxfemp) {
3076                 /* Fill the Tx FIFOs when not in DMA mode */
3077                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
3078                         __cvmx_usb_poll_tx_fifo(usb);
3079         }
3080         if (usbc_gintsts.s.disconnint || usbc_gintsts.s.prtint) {
3081                 union cvmx_usbcx_hprt usbc_hprt;
3082                 /*
3083                  * Disconnect Detected Interrupt (DisconnInt)
3084                  * Asserted when a device disconnect is detected.
3085                  *
3086                  * Host Port Interrupt (PrtInt)
3087                  * The core sets this bit to indicate a change in port status of
3088                  * one of the O2P USB core ports in Host mode. The application
3089                  * must read the Host Port Control and Status (HPRT) register to
3090                  * determine the exact event that caused this interrupt. The
3091                  * application must clear the appropriate status bit in the Host
3092                  * Port Control and Status register to clear this bit.
3093                  *
3094                  * Call the user's port callback
3095                  */
3096                 __cvmx_usb_perform_callback(usb, NULL, NULL,
3097                                             CVMX_USB_CALLBACK_PORT_CHANGED,
3098                                             CVMX_USB_COMPLETE_SUCCESS);
3099                 /* Clear the port change bits */
3100                 usbc_hprt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
3101                 usbc_hprt.s.prtena = 0;
3102                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HPRT(usb->index), usbc_hprt.u32);
3103         }
3104         if (usbc_gintsts.s.hchint) {
3105                 /*
3106                  * Host Channels Interrupt (HChInt)
3107                  * The core sets this bit to indicate that an interrupt is
3108                  * pending on one of the channels of the core (in Host mode).
3109                  * The application must read the Host All Channels Interrupt
3110                  * (HAINT) register to determine the exact number of the channel
3111                  * on which the interrupt occurred, and then read the
3112                  * corresponding Host Channel-n Interrupt (HCINTn) register to
3113                  * determine the exact cause of the interrupt. The application
3114                  * must clear the appropriate status bit in the HCINTn register
3115                  * to clear this bit.
3116                  */
3117                 union cvmx_usbcx_haint usbc_haint;
3118                 usbc_haint.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HAINT(usb->index));
3119                 while (usbc_haint.u32) {
3120                         int channel;
3121
3122                         channel = __fls(usbc_haint.u32);
3123                         __cvmx_usb_poll_channel(usb, channel);
3124                         usbc_haint.u32 ^= 1<<channel;
3125                 }
3126         }
3127
3128         __cvmx_usb_schedule(usb, usbc_gintsts.s.sof);
3129
3130         return 0;
3131 }