drivers/staging/octeon-usb/cvmx-usb.c

   1 /***********************license start***************
   2  * Copyright (c) 2003-2010  Cavium Networks (support@cavium.com). All rights
   3  * reserved.
   4  *
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions are
   8  * met:
   9  *
  10  *   * Redistributions of source code must retain the above copyright
  11  *     notice, this list of conditions and the following disclaimer.
  12  *
  13  *   * Redistributions in binary form must reproduce the above
  14  *     copyright notice, this list of conditions and the following
  15  *     disclaimer in the documentation and/or other materials provided
  16  *     with the distribution.
  17
  18  *   * Neither the name of Cavium Networks nor the names of
  19  *     its contributors may be used to endorse or promote products
  20  *     derived from this software without specific prior written
  21  *     permission.
  22
  23  * This Software, including technical data, may be subject to U.S. export  control
  24  * laws, including the U.S. Export Administration Act and its  associated
  25  * regulations, and may be subject to export or import  regulations in other
  26  * countries.
  27
  28  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
  29  * AND WITH ALL FAULTS AND CAVIUM  NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR
  30  * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
  31  * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR
  32  * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
  33  * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
  34  * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
  35  * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
  36  * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE  RISK ARISING OUT OF USE OR
  37  * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
  38  ***********************license end**************************************/
  39
  40
  41 /**
  42  * @file
  43  *
  44  * "cvmx-usb.c" defines a set of low level USB functions to help
  45  * developers create Octeon USB drivers for various operating
  46  * systems. These functions provide a generic API to the Octeon
  47  * USB blocks, hiding the internal hardware specific
  48  * operations.
  49  *
  50  * <hr>$Revision: 32636 $<hr>
  51  */
  52 #include <linux/delay.h>
  53 #include <asm/octeon/cvmx.h>
  54 #include <asm/octeon/octeon.h>
  55 #include <asm/octeon/cvmx-sysinfo.h>
  56 #include "cvmx-usbnx-defs.h"
  57 #include "cvmx-usbcx-defs.h"
  58 #include "cvmx-usb.h"
  59 #include <asm/octeon/cvmx-helper.h>
  60 #include <asm/octeon/cvmx-helper-board.h>
  61
  62 #define CVMX_PREFETCH0(address) CVMX_PREFETCH(address, 0)
  63 #define CVMX_PREFETCH128(address) CVMX_PREFETCH(address, 128)
  64 // a normal prefetch
  65 #define CVMX_PREFETCH(address, offset) CVMX_PREFETCH_PREF0(address, offset)
  66 // normal prefetches that use the pref instruction
  67 #define CVMX_PREFETCH_PREFX(X, address, offset) asm volatile ("pref %[type], %[off](%[rbase])" : : [rbase] "d" (address), [off] "I" (offset), [type] "n" (X))
  68 #define CVMX_PREFETCH_PREF0(address, offset) CVMX_PREFETCH_PREFX(0, address, offset)
  69 #define CVMX_CLZ(result, input) asm ("clz %[rd],%[rs]" : [rd] "=d" (result) : [rs] "d" (input))
  70
  71 #define MAX_RETRIES             3               /* Maximum number of times to retry failed transactions */
  72 #define MAX_PIPES               32              /* Maximum number of pipes that can be open at once */
  73 #define MAX_TRANSACTIONS        256             /* Maximum number of outstanding transactions across all pipes */
  74 #define MAX_CHANNELS            8               /* Maximum number of hardware channels supported by the USB block */
  75 #define MAX_USB_ADDRESS         127             /* The highest valid USB device address */
  76 #define MAX_USB_ENDPOINT        15              /* The highest valid USB endpoint number */
  77 #define MAX_USB_HUB_PORT        15              /* The highest valid port number on a hub */
  78 #define MAX_TRANSFER_BYTES      ((1<<19)-1)     /* The low level hardware can transfer a maximum of this number of bytes in each transfer. The field is 19 bits wide */
  79 #define MAX_TRANSFER_PACKETS    ((1<<10)-1)     /* The low level hardware can transfer a maximum of this number of packets in each transfer. The field is 10 bits wide */
  80
  81 /*
  82  * These defines disable the normal read and write csr. This is so I can add
  83  * extra debug stuff to the usb specific version and I won't use the normal
  84  * version by mistake
  85  */
  86 #define cvmx_read_csr use_cvmx_usb_read_csr64_instead_of_cvmx_read_csr
  87 #define cvmx_write_csr use_cvmx_usb_write_csr64_instead_of_cvmx_write_csr
  88
  89 typedef enum {
  90         __CVMX_USB_TRANSACTION_FLAGS_IN_USE = 1<<16,
  91 } cvmx_usb_transaction_flags_t;
  92
  93 enum {
  94         USB_CLOCK_TYPE_REF_12,
  95         USB_CLOCK_TYPE_REF_24,
  96         USB_CLOCK_TYPE_REF_48,
  97         USB_CLOCK_TYPE_CRYSTAL_12,
  98 };
  99
 100 /**
 101  * Logical transactions may take numerous low level
 102  * transactions, especially when splits are concerned. This
 103  * enum represents all of the possible stages a transaction can
 104  * be in. Note that split completes are always even. This is so
 105  * the NAK handler can backup to the previous low level
 106  * transaction with a simple clearing of bit 0.
 107  */
 108 typedef enum {
 109         CVMX_USB_STAGE_NON_CONTROL,
 110         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE,
 111         CVMX_USB_STAGE_SETUP,
 112         CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE,
 113         CVMX_USB_STAGE_DATA,
 114         CVMX_USB_STAGE_DATA_SPLIT_COMPLETE,
 115         CVMX_USB_STAGE_STATUS,
 116         CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE,
 117 } cvmx_usb_stage_t;
 118
 119 /**
 120  * This structure describes each pending USB transaction
 121  * regardless of type. These are linked together to form a list
 122  * of pending requests for a pipe.
 123  */
 124 typedef struct cvmx_usb_transaction {
 125         struct cvmx_usb_transaction *prev;      /**< Transaction before this one in the pipe */
 126         struct cvmx_usb_transaction *next;      /**< Transaction after this one in the pipe */
 127         cvmx_usb_transfer_t type;               /**< Type of transaction, duplicated of the pipe */
 128         cvmx_usb_transaction_flags_t flags;     /**< State flags for this transaction */
 129         uint64_t buffer;                        /**< User's physical buffer address to read/write */
 130         int buffer_length;                      /**< Size of the user's buffer in bytes */
 131         uint64_t control_header;                /**< For control transactions, physical address of the 8 byte standard header */
 132         int iso_start_frame;                    /**< For ISO transactions, the starting frame number */
 133         int iso_number_packets;                 /**< For ISO transactions, the number of packets in the request */
 134         cvmx_usb_iso_packet_t *iso_packets;     /**< For ISO transactions, the sub packets in the request */
 135         int xfersize;
 136         int pktcnt;
 137         int retries;
 138         int actual_bytes;                       /**< Actual bytes transfer for this transaction */
 139         cvmx_usb_stage_t stage;                 /**< For control transactions, the current stage */
 140         cvmx_usb_callback_func_t callback;      /**< User's callback function when complete */
 141         void *callback_data;                    /**< User's data */
 142 } cvmx_usb_transaction_t;
 143
 144 /**
 145  * A pipe represents a virtual connection between Octeon and some
 146  * USB device. It contains a list of pending request to the device.
 147  */
 148 typedef struct cvmx_usb_pipe {
 149         struct cvmx_usb_pipe *prev;             /**< Pipe before this one in the list */
 150         struct cvmx_usb_pipe *next;             /**< Pipe after this one in the list */
 151         cvmx_usb_transaction_t *head;           /**< The first pending transaction */
 152         cvmx_usb_transaction_t *tail;           /**< The last pending transaction */
 153         uint64_t interval;                      /**< For periodic pipes, the interval between packets in frames */
 154         uint64_t next_tx_frame;                 /**< The next frame this pipe is allowed to transmit on */
 155         cvmx_usb_pipe_flags_t flags;            /**< State flags for this pipe */
 156         cvmx_usb_speed_t device_speed;          /**< Speed of device connected to this pipe */
 157         cvmx_usb_transfer_t transfer_type;      /**< Type of transaction supported by this pipe */
 158         cvmx_usb_direction_t transfer_dir;      /**< IN or OUT. Ignored for Control */
 159         int multi_count;                        /**< Max packet in a row for the device */
 160         uint16_t max_packet;                    /**< The device's maximum packet size in bytes */
 161         uint8_t device_addr;                    /**< USB device address at other end of pipe */
 162         uint8_t endpoint_num;                   /**< USB endpoint number at other end of pipe */
 163         uint8_t hub_device_addr;                /**< Hub address this device is connected to */
 164         uint8_t hub_port;                       /**< Hub port this device is connected to */
 165         uint8_t pid_toggle;                     /**< This toggles between 0/1 on every packet send to track the data pid needed */
 166         uint8_t channel;                        /**< Hardware DMA channel for this pipe */
 167         int8_t split_sc_frame;                  /**< The low order bits of the frame number the split complete should be sent on */
 168 } cvmx_usb_pipe_t;
 169
 170 typedef struct {
 171         cvmx_usb_pipe_t *head;  /**< Head of the list, or NULL if empty */
 172         cvmx_usb_pipe_t *tail;  /**< Tail if the list, or NULL if empty */
 173 } cvmx_usb_pipe_list_t;
 174
 175 typedef struct {
 176         struct {
 177                 int channel;
 178                 int size;
 179                 uint64_t address;
 180         } entry[MAX_CHANNELS+1];
 181         int head;
 182         int tail;
 183 } cvmx_usb_tx_fifo_t;
 184
 185 /**
 186  * The state of the USB block is stored in this structure
 187  */
 188 typedef struct {
 189         int init_flags;                                                 /**< Flags passed to initialize */
 190         int index;                                                      /**< Which USB block this is for */
 191         int idle_hardware_channels;                                     /**< Bit set for every idle hardware channel */
 192         cvmx_usbcx_hprt_t usbcx_hprt;                                   /**< Stored port status so we don't need to read a CSR to determine splits */
 193         cvmx_usb_pipe_t *pipe_for_channel[MAX_CHANNELS];                /**< Map channels to pipes */
 194         cvmx_usb_transaction_t *free_transaction_head;                  /**< List of free transactions head */
 195         cvmx_usb_transaction_t *free_transaction_tail;                  /**< List of free transactions tail */
 196         cvmx_usb_pipe_t pipe[MAX_PIPES];                                /**< Storage for pipes */
 197         cvmx_usb_transaction_t transaction[MAX_TRANSACTIONS];           /**< Storage for transactions */
 198         cvmx_usb_callback_func_t callback[__CVMX_USB_CALLBACK_END];     /**< User global callbacks */
 199         void *callback_data[__CVMX_USB_CALLBACK_END];                   /**< User data for each callback */
 200         int indent;                                                     /**< Used by debug output to indent functions */
 201         cvmx_usb_port_status_t port_status;                             /**< Last port status used for change notification */
 202         cvmx_usb_pipe_list_t free_pipes;                                /**< List of all pipes that are currently closed */
 203         cvmx_usb_pipe_list_t idle_pipes;                                /**< List of open pipes that have no transactions */
 204         cvmx_usb_pipe_list_t active_pipes[4];                           /**< Active pipes indexed by transfer type */
 205         uint64_t frame_number;                                          /**< Increments every SOF interrupt for time keeping */
 206         cvmx_usb_transaction_t *active_split;                           /**< Points to the current active split, or NULL */
 207         cvmx_usb_tx_fifo_t periodic;
 208         cvmx_usb_tx_fifo_t nonperiodic;
 209 } cvmx_usb_internal_state_t;
 210
 211 /* This macro spins on a field waiting for it to reach a value */
 212 #define CVMX_WAIT_FOR_FIELD32(address, type, field, op, value, timeout_usec)\
 213         ({int result;                                                       \
 214         do {                                                                \
 215                 uint64_t done = cvmx_get_cycle() + (uint64_t)timeout_usec * \
 216                         octeon_get_clock_rate() / 1000000;                  \
 217                 type c;                                                     \
 218                 while (1) {                                                 \
 219                         c.u32 = __cvmx_usb_read_csr32(usb, address);        \
 220                         if (c.s.field op (value)) {                         \
 221                                 result = 0;                                 \
 222                                 break;                                      \
 223                         } else if (cvmx_get_cycle() > done) {               \
 224                                 result = -1;                                \
 225                                 break;                                      \
 226                         } else                                              \
 227                                 cvmx_wait(100);                             \
 228                 }                                                           \
 229         } while (0);                                                        \
 230         result; })
 231
 232 /*
 233  * This macro logically sets a single field in a CSR. It does the sequence
 234  * read, modify, and write
 235  */
 236 #define USB_SET_FIELD32(address, type, field, value)            \
 237         do {                                                    \
 238                 type c;                                         \
 239                 c.u32 = __cvmx_usb_read_csr32(usb, address);    \
 240                 c.s.field = value;                              \
 241                 __cvmx_usb_write_csr32(usb, address, c.u32);    \
 242         } while (0)
 243
 244 /* Returns the IO address to push/pop stuff data from the FIFOs */
 245 #define USB_FIFO_ADDRESS(channel, usb_index) (CVMX_USBCX_GOTGCTL(usb_index) + ((channel)+1)*0x1000)
 246
 247 static int octeon_usb_get_clock_type(void)
 248 {
 249         switch (cvmx_sysinfo_get()->board_type) {
 250         case CVMX_BOARD_TYPE_BBGW_REF:
 251         case CVMX_BOARD_TYPE_LANAI2_A:
 252         case CVMX_BOARD_TYPE_LANAI2_U:
 253         case CVMX_BOARD_TYPE_LANAI2_G:
 254         case CVMX_BOARD_TYPE_UBNT_E100:
 255                 return USB_CLOCK_TYPE_CRYSTAL_12;
 256         }
 257         return USB_CLOCK_TYPE_REF_48;
 258 }
 259
 260 /**
 261  * @INTERNAL
 262  * Read a USB 32bit CSR. It performs the necessary address swizzle
 263  * for 32bit CSRs and logs the value in a readable format if
 264  * debugging is on.
 265  *
 266  * @param usb     USB block this access is for
 267  * @param address 64bit address to read
 268  *
 269  * @return Result of the read
 270  */
 271 static inline uint32_t __cvmx_usb_read_csr32(cvmx_usb_internal_state_t *usb,
 272                                              uint64_t address)
 273 {
 274         uint32_t result = cvmx_read64_uint32(address ^ 4);
 275         return result;
 276 }
 277
 278
 279 /**
 280  * @INTERNAL
 281  * Write a USB 32bit CSR. It performs the necessary address
 282  * swizzle for 32bit CSRs and logs the value in a readable format
 283  * if debugging is on.
 284  *
 285  * @param usb     USB block this access is for
 286  * @param address 64bit address to write
 287  * @param value   Value to write
 288  */
 289 static inline void __cvmx_usb_write_csr32(cvmx_usb_internal_state_t *usb,
 290                                           uint64_t address, uint32_t value)
 291 {
 292         cvmx_write64_uint32(address ^ 4, value);
 293         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
 294 }
 295
 296
 297 /**
 298  * @INTERNAL
 299  * Read a USB 64bit CSR. It logs the value in a readable format if
 300  * debugging is on.
 301  *
 302  * @param usb     USB block this access is for
 303  * @param address 64bit address to read
 304  *
 305  * @return Result of the read
 306  */
 307 static inline uint64_t __cvmx_usb_read_csr64(cvmx_usb_internal_state_t *usb,
 308                                              uint64_t address)
 309 {
 310         uint64_t result = cvmx_read64_uint64(address);
 311         return result;
 312 }
 313
 314
 315 /**
 316  * @INTERNAL
 317  * Write a USB 64bit CSR. It logs the value in a readable format
 318  * if debugging is on.
 319  *
 320  * @param usb     USB block this access is for
 321  * @param address 64bit address to write
 322  * @param value   Value to write
 323  */
 324 static inline void __cvmx_usb_write_csr64(cvmx_usb_internal_state_t *usb,
 325                                           uint64_t address, uint64_t value)
 326 {
 327         cvmx_write64_uint64(address, value);
 328 }
 329
 330 /**
 331  * @INTERNAL
 332  * Return non zero if this pipe connects to a non HIGH speed
 333  * device through a high speed hub.
 334  *
 335  * @param usb    USB block this access is for
 336  * @param pipe   Pipe to check
 337  *
 338  * @return Non zero if we need to do split transactions
 339  */
 340 static inline int __cvmx_usb_pipe_needs_split(cvmx_usb_internal_state_t *usb, cvmx_usb_pipe_t *pipe)
 341 {
 342         return ((pipe->device_speed != CVMX_USB_SPEED_HIGH) && (usb->usbcx_hprt.s.prtspd == CVMX_USB_SPEED_HIGH));
 343 }
 344
 345
 346 /**
 347  * @INTERNAL
 348  * Trivial utility function to return the correct PID for a pipe
 349  *
 350  * @param pipe   pipe to check
 351  *
 352  * @return PID for pipe
 353  */
 354 static inline int __cvmx_usb_get_data_pid(cvmx_usb_pipe_t *pipe)
 355 {
 356         if (pipe->pid_toggle)
 357                 return 2; /* Data1 */
 358         else
 359                 return 0; /* Data0 */
 360 }
 361
 362
 363 /**
 364  * Return the number of USB ports supported by this Octeon
 365  * chip. If the chip doesn't support USB, or is not supported
 366  * by this API, a zero will be returned. Most Octeon chips
 367  * support one usb port, but some support two ports.
 368  * cvmx_usb_initialize() must be called on independent
 369  * cvmx_usb_state_t structures.
 370  *
 371  * @return Number of port, zero if usb isn't supported
 372  */
 373 int cvmx_usb_get_num_ports(void)
 374 {
 375         int arch_ports = 0;
 376
 377         if (OCTEON_IS_MODEL(OCTEON_CN56XX))
 378                 arch_ports = 1;
 379         else if (OCTEON_IS_MODEL(OCTEON_CN52XX))
 380                 arch_ports = 2;
 381         else if (OCTEON_IS_MODEL(OCTEON_CN50XX))
 382                 arch_ports = 1;
 383         else if (OCTEON_IS_MODEL(OCTEON_CN31XX))
 384                 arch_ports = 1;
 385         else if (OCTEON_IS_MODEL(OCTEON_CN30XX))
 386                 arch_ports = 1;
 387         else
 388                 arch_ports = 0;
 389
 390         return arch_ports;
 391 }
 392
 393
 394 /**
 395  * @INTERNAL
 396  * Allocate a usb transaction for use
 397  *
 398  * @param usb    USB device state populated by
 399  *               cvmx_usb_initialize().
 400  *
 401  * @return Transaction or NULL
 402  */
 403 static inline cvmx_usb_transaction_t *__cvmx_usb_alloc_transaction(cvmx_usb_internal_state_t *usb)
 404 {
 405         cvmx_usb_transaction_t *t;
 406         t = usb->free_transaction_head;
 407         if (t) {
 408                 usb->free_transaction_head = t->next;
 409                 if (!usb->free_transaction_head)
 410                         usb->free_transaction_tail = NULL;
 411         }
 412         if (t) {
 413                 memset(t, 0, sizeof(*t));
 414                 t->flags = __CVMX_USB_TRANSACTION_FLAGS_IN_USE;
 415         }
 416         return t;
 417 }
 418
 419
 420 /**
 421  * @INTERNAL
 422  * Free a usb transaction
 423  *
 424  * @param usb    USB device state populated by
 425  *               cvmx_usb_initialize().
 426  * @param transaction
 427  *               Transaction to free
 428  */
 429 static inline void __cvmx_usb_free_transaction(cvmx_usb_internal_state_t *usb,
 430                                                cvmx_usb_transaction_t *transaction)
 431 {
 432         transaction->flags = 0;
 433         transaction->prev = NULL;
 434         transaction->next = NULL;
 435         if (usb->free_transaction_tail)
 436                 usb->free_transaction_tail->next = transaction;
 437         else
 438                 usb->free_transaction_head = transaction;
 439         usb->free_transaction_tail = transaction;
 440 }
 441
 442
 443 /**
 444  * @INTERNAL
 445  * Add a pipe to the tail of a list
 446  * @param list   List to add pipe to
 447  * @param pipe   Pipe to add
 448  */
 449 static inline void __cvmx_usb_append_pipe(cvmx_usb_pipe_list_t *list, cvmx_usb_pipe_t *pipe)
 450 {
 451         pipe->next = NULL;
 452         pipe->prev = list->tail;
 453         if (list->tail)
 454                 list->tail->next = pipe;
 455         else
 456                 list->head = pipe;
 457         list->tail = pipe;
 458 }
 459
 460
 461 /**
 462  * @INTERNAL
 463  * Remove a pipe from a list
 464  * @param list   List to remove pipe from
 465  * @param pipe   Pipe to remove
 466  */
 467 static inline void __cvmx_usb_remove_pipe(cvmx_usb_pipe_list_t *list, cvmx_usb_pipe_t *pipe)
 468 {
 469         if (list->head == pipe) {
 470                 list->head = pipe->next;
 471                 pipe->next = NULL;
 472                 if (list->head)
 473                         list->head->prev = NULL;
 474                 else
 475                         list->tail = NULL;
 476         } else if (list->tail == pipe) {
 477                 list->tail = pipe->prev;
 478                 list->tail->next = NULL;
 479                 pipe->prev = NULL;
 480         } else {
 481                 pipe->prev->next = pipe->next;
 482                 pipe->next->prev = pipe->prev;
 483                 pipe->prev = NULL;
 484                 pipe->next = NULL;
 485         }
 486 }
 487
 488
 489 /**
 490  * Initialize a USB port for use. This must be called before any
 491  * other access to the Octeon USB port is made. The port starts
 492  * off in the disabled state.
 493  *
 494  * @param state  Pointer to an empty cvmx_usb_state_t structure
 495  *               that will be populated by the initialize call.
 496  *               This structure is then passed to all other USB
 497  *               functions.
 498  * @param usb_port_number
 499  *               Which Octeon USB port to initialize.
 500  * @param flags  Flags to control hardware initialization. See
 501  *               cvmx_usb_initialize_flags_t for the flag
 502  *               definitions. Some flags are mandatory.
 503  *
 504  * @return 0 or a negative error code.
 505  */
 506 int cvmx_usb_initialize(cvmx_usb_state_t *state, int usb_port_number,
 507                         cvmx_usb_initialize_flags_t flags)
 508 {
 509         cvmx_usbnx_clk_ctl_t usbn_clk_ctl;
 510         cvmx_usbnx_usbp_ctl_status_t usbn_usbp_ctl_status;
 511         cvmx_usb_internal_state_t *usb = (cvmx_usb_internal_state_t *)state;
 512
 513         usb->init_flags = flags;
 514
 515         /* Make sure that state is large enough to store the internal state */
 516         if (sizeof(*state) < sizeof(*usb))
 517                 return -EINVAL;
 518         /* At first allow 0-1 for the usb port number */
 519         if ((usb_port_number < 0) || (usb_port_number > 1))
 520                 return -EINVAL;
 521         /* For all chips except 52XX there is only one port */
 522         if (!OCTEON_IS_MODEL(OCTEON_CN52XX) && (usb_port_number > 0))
 523                 return -EINVAL;
 524         /* Try to determine clock type automatically */
 525         if ((flags & (CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI |
 526                       CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND)) == 0) {
 527                 if (octeon_usb_get_clock_type() == USB_CLOCK_TYPE_CRYSTAL_12)
 528                         flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI;  /* Only 12 MHZ crystals are supported */
 529                 else
 530                         flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND;
 531         }
 532
 533         if (flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND) {
 534                 /* Check for auto ref clock frequency */
 535                 if (!(flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK))
 536                         switch (octeon_usb_get_clock_type()) {
 537                         case USB_CLOCK_TYPE_REF_12:
 538                                 flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ;
 539                                 break;
 540                         case USB_CLOCK_TYPE_REF_24:
 541                                 flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ;
 542                                 break;
 543                         case USB_CLOCK_TYPE_REF_48:
 544                                 flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ;
 545                                 break;
 546                         default:
 547                                 return -EINVAL;
 548                                 break;
 549                         }
 550         }
 551
 552         memset(usb, 0, sizeof(usb));
 553         usb->init_flags = flags;
 554
 555         /* Initialize the USB state structure */
 556         {
 557                 int i;
 558                 usb->index = usb_port_number;
 559
 560                 /* Initialize the transaction double linked list */
 561                 usb->free_transaction_head = NULL;
 562                 usb->free_transaction_tail = NULL;
 563                 for (i = 0; i < MAX_TRANSACTIONS; i++)
 564                         __cvmx_usb_free_transaction(usb, usb->transaction + i);
 565                 for (i = 0; i < MAX_PIPES; i++)
 566                         __cvmx_usb_append_pipe(&usb->free_pipes, usb->pipe + i);
 567         }
 568
 569         /*
 570          * Power On Reset and PHY Initialization
 571          *
 572          * 1. Wait for DCOK to assert (nothing to do)
 573          *
 574          * 2a. Write USBN0/1_CLK_CTL[POR] = 1 and
 575          *     USBN0/1_CLK_CTL[HRST,PRST,HCLK_RST] = 0
 576          */
 577         usbn_clk_ctl.u64 = __cvmx_usb_read_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index));
 578         usbn_clk_ctl.s.por = 1;
 579         usbn_clk_ctl.s.hrst = 0;
 580         usbn_clk_ctl.s.prst = 0;
 581         usbn_clk_ctl.s.hclk_rst = 0;
 582         usbn_clk_ctl.s.enable = 0;
 583         /*
 584          * 2b. Select the USB reference clock/crystal parameters by writing
 585          *     appropriate values to USBN0/1_CLK_CTL[P_C_SEL, P_RTYPE, P_COM_ON]
 586          */
 587         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND) {
 588                 /*
 589                  * The USB port uses 12/24/48MHz 2.5V board clock
 590                  * source at USB_XO. USB_XI should be tied to GND.
 591                  * Most Octeon evaluation boards require this setting
 592                  */
 593                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX)) {
 594                         usbn_clk_ctl.cn31xx.p_rclk  = 1; /* From CN31XX,CN30XX manual */
 595                         usbn_clk_ctl.cn31xx.p_xenbn = 0;
 596                 } else if (OCTEON_IS_MODEL(OCTEON_CN56XX) || OCTEON_IS_MODEL(OCTEON_CN50XX))
 597                         usbn_clk_ctl.cn56xx.p_rtype = 2; /* From CN56XX,CN50XX manual */
 598                 else
 599                         usbn_clk_ctl.cn52xx.p_rtype = 1; /* From CN52XX manual */
 600
 601                 switch (flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK) {
 602                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:
 603                         usbn_clk_ctl.s.p_c_sel = 0;
 604                         break;
 605                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:
 606                         usbn_clk_ctl.s.p_c_sel = 1;
 607                         break;
 608                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:
 609                         usbn_clk_ctl.s.p_c_sel = 2;
 610                         break;
 611                 }
 612         } else {
 613                 /*
 614                  * The USB port uses a 12MHz crystal as clock source
 615                  * at USB_XO and USB_XI
 616                  */
 617                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX)) {
 618                         usbn_clk_ctl.cn31xx.p_rclk  = 1; /* From CN31XX,CN30XX manual */
 619                         usbn_clk_ctl.cn31xx.p_xenbn = 1;
 620                 } else if (OCTEON_IS_MODEL(OCTEON_CN56XX) || OCTEON_IS_MODEL(OCTEON_CN50XX))
 621                         usbn_clk_ctl.cn56xx.p_rtype = 0; /* From CN56XX,CN50XX manual */
 622                 else
 623                         usbn_clk_ctl.cn52xx.p_rtype = 0; /* From CN52XX manual */
 624
 625                 usbn_clk_ctl.s.p_c_sel = 0;
 626         }
 627         /*
 628          * 2c. Select the HCLK via writing USBN0/1_CLK_CTL[DIVIDE, DIVIDE2] and
 629          *     setting USBN0/1_CLK_CTL[ENABLE] = 1. Divide the core clock down
 630          *     such that USB is as close as possible to 125Mhz
 631          */
 632         {
 633                 int divisor = (octeon_get_clock_rate()+125000000-1)/125000000;
 634                 if (divisor < 4)  /* Lower than 4 doesn't seem to work properly */
 635                         divisor = 4;
 636                 usbn_clk_ctl.s.divide = divisor;
 637                 usbn_clk_ctl.s.divide2 = 0;
 638         }
 639         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 640                                usbn_clk_ctl.u64);
 641         /* 2d. Write USBN0/1_CLK_CTL[HCLK_RST] = 1 */
 642         usbn_clk_ctl.s.hclk_rst = 1;
 643         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 644                                usbn_clk_ctl.u64);
 645         /* 2e.  Wait 64 core-clock cycles for HCLK to stabilize */
 646         cvmx_wait(64);
 647         /*
 648          * 3. Program the power-on reset field in the USBN clock-control
 649          *    register:
 650          *    USBN_CLK_CTL[POR] = 0
 651          */
 652         usbn_clk_ctl.s.por = 0;
 653         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 654                                usbn_clk_ctl.u64);
 655         /* 4. Wait 1 ms for PHY clock to start */
 656         mdelay(1);
 657         /*
 658          * 5. Program the Reset input from automatic test equipment field in the
 659          *    USBP control and status register:
 660          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 1
 661          */
 662         usbn_usbp_ctl_status.u64 = __cvmx_usb_read_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index));
 663         usbn_usbp_ctl_status.s.ate_reset = 1;
 664         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 665                                usbn_usbp_ctl_status.u64);
 666         /* 6. Wait 10 cycles */
 667         cvmx_wait(10);
 668         /*
 669          * 7. Clear ATE_RESET field in the USBN clock-control register:
 670          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 0
 671          */
 672         usbn_usbp_ctl_status.s.ate_reset = 0;
 673         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 674                                usbn_usbp_ctl_status.u64);
 675         /*
 676          * 8. Program the PHY reset field in the USBN clock-control register:
 677          *    USBN_CLK_CTL[PRST] = 1
 678          */
 679         usbn_clk_ctl.s.prst = 1;
 680         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 681                                usbn_clk_ctl.u64);
 682         /*
 683          * 9. Program the USBP control and status register to select host or
 684          *    device mode. USBN_USBP_CTL_STATUS[HST_MODE] = 0 for host, = 1 for
 685          *    device
 686          */
 687         usbn_usbp_ctl_status.s.hst_mode = 0;
 688         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 689                                usbn_usbp_ctl_status.u64);
 690         /* 10. Wait 1 us */
 691         udelay(1);
 692         /*
 693          * 11. Program the hreset_n field in the USBN clock-control register:
 694          *     USBN_CLK_CTL[HRST] = 1
 695          */
 696         usbn_clk_ctl.s.hrst = 1;
 697         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 698                                usbn_clk_ctl.u64);
 699         /* 12. Proceed to USB core initialization */
 700         usbn_clk_ctl.s.enable = 1;
 701         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 702                                usbn_clk_ctl.u64);
 703         udelay(1);
 704
 705         /*
 706          * USB Core Initialization
 707          *
 708          * 1. Read USBC_GHWCFG1, USBC_GHWCFG2, USBC_GHWCFG3, USBC_GHWCFG4 to
 709          *    determine USB core configuration parameters.
 710          *
 711          *    Nothing needed
 712          *
 713          * 2. Program the following fields in the global AHB configuration
 714          *    register (USBC_GAHBCFG)
 715          *    DMA mode, USBC_GAHBCFG[DMAEn]: 1 = DMA mode, 0 = slave mode
 716          *    Burst length, USBC_GAHBCFG[HBSTLEN] = 0
 717          *    Nonperiodic TxFIFO empty level (slave mode only),
 718          *    USBC_GAHBCFG[NPTXFEMPLVL]
 719          *    Periodic TxFIFO empty level (slave mode only),
 720          *    USBC_GAHBCFG[PTXFEMPLVL]
 721          *    Global interrupt mask, USBC_GAHBCFG[GLBLINTRMSK] = 1
 722          */
 723         {
 724                 cvmx_usbcx_gahbcfg_t usbcx_gahbcfg;
 725                 /* Due to an errata, CN31XX doesn't support DMA */
 726                 if (OCTEON_IS_MODEL(OCTEON_CN31XX))
 727                         usb->init_flags |= CVMX_USB_INITIALIZE_FLAGS_NO_DMA;
 728                 usbcx_gahbcfg.u32 = 0;
 729                 usbcx_gahbcfg.s.dmaen = !(usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA);
 730                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 731                         usb->idle_hardware_channels = 0x1;  /* Only use one channel with non DMA */
 732                 else if (OCTEON_IS_MODEL(OCTEON_CN5XXX))
 733                         usb->idle_hardware_channels = 0xf7; /* CN5XXX have an errata with channel 3 */
 734                 else
 735                         usb->idle_hardware_channels = 0xff;
 736                 usbcx_gahbcfg.s.hbstlen = 0;
 737                 usbcx_gahbcfg.s.nptxfemplvl = 1;
 738                 usbcx_gahbcfg.s.ptxfemplvl = 1;
 739                 usbcx_gahbcfg.s.glblintrmsk = 1;
 740                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GAHBCFG(usb->index),
 741                                        usbcx_gahbcfg.u32);
 742         }
 743         /*
 744          * 3. Program the following fields in USBC_GUSBCFG register.
 745          *    HS/FS timeout calibration, USBC_GUSBCFG[TOUTCAL] = 0
 746          *    ULPI DDR select, USBC_GUSBCFG[DDRSEL] = 0
 747          *    USB turnaround time, USBC_GUSBCFG[USBTRDTIM] = 0x5
 748          *    PHY low-power clock select, USBC_GUSBCFG[PHYLPWRCLKSEL] = 0
 749          */
 750         {
 751                 cvmx_usbcx_gusbcfg_t usbcx_gusbcfg;
 752                 usbcx_gusbcfg.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index));
 753                 usbcx_gusbcfg.s.toutcal = 0;
 754                 usbcx_gusbcfg.s.ddrsel = 0;
 755                 usbcx_gusbcfg.s.usbtrdtim = 0x5;
 756                 usbcx_gusbcfg.s.phylpwrclksel = 0;
 757                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index),
 758                                        usbcx_gusbcfg.u32);
 759         }
 760         /*
 761          * 4. The software must unmask the following bits in the USBC_GINTMSK
 762          *    register.
 763          *    OTG interrupt mask, USBC_GINTMSK[OTGINTMSK] = 1
 764          *    Mode mismatch interrupt mask, USBC_GINTMSK[MODEMISMSK] = 1
 765          */
 766         {
 767                 cvmx_usbcx_gintmsk_t usbcx_gintmsk;
 768                 int channel;
 769
 770                 usbcx_gintmsk.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GINTMSK(usb->index));
 771                 usbcx_gintmsk.s.otgintmsk = 1;
 772                 usbcx_gintmsk.s.modemismsk = 1;
 773                 usbcx_gintmsk.s.hchintmsk = 1;
 774                 usbcx_gintmsk.s.sofmsk = 0;
 775                 /* We need RX FIFO interrupts if we don't have DMA */
 776                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 777                         usbcx_gintmsk.s.rxflvlmsk = 1;
 778                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTMSK(usb->index),
 779                                        usbcx_gintmsk.u32);
 780
 781                 /* Disable all channel interrupts. We'll enable them per channel later */
 782                 for (channel = 0; channel < 8; channel++)
 783                         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
 784         }
 785
 786         {
 787                 /*
 788                  * Host Port Initialization
 789                  *
 790                  * 1. Program the host-port interrupt-mask field to unmask,
 791                  *    USBC_GINTMSK[PRTINT] = 1
 792                  */
 793                 USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), cvmx_usbcx_gintmsk_t,
 794                                 prtintmsk, 1);
 795                 USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), cvmx_usbcx_gintmsk_t,
 796                                 disconnintmsk, 1);
 797                 /*
 798                  * 2. Program the USBC_HCFG register to select full-speed host
 799                  *    or high-speed host.
 800                  */
 801                 {
 802                         cvmx_usbcx_hcfg_t usbcx_hcfg;
 803                         usbcx_hcfg.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCFG(usb->index));
 804                         usbcx_hcfg.s.fslssupp = 0;
 805                         usbcx_hcfg.s.fslspclksel = 0;
 806                         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCFG(usb->index), usbcx_hcfg.u32);
 807                 }
 808                 /*
 809                  * 3. Program the port power bit to drive VBUS on the USB,
 810                  *    USBC_HPRT[PRTPWR] = 1
 811                  */
 812                 USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt_t, prtpwr, 1);
 813
 814                 /*
 815                  * Steps 4-15 from the manual are done later in the port enable
 816                  */
 817         }
 818
 819         return 0;
 820 }
 821
 822
 823 /**
 824  * Shutdown a USB port after a call to cvmx_usb_initialize().
 825  * The port should be disabled with all pipes closed when this
 826  * function is called.
 827  *
 828  * @param state  USB device state populated by
 829  *               cvmx_usb_initialize().
 830  *
 831  * @return 0 or a negative error code.
 832  */
 833 int cvmx_usb_shutdown(cvmx_usb_state_t *state)
 834 {
 835         cvmx_usbnx_clk_ctl_t usbn_clk_ctl;
 836         cvmx_usb_internal_state_t *usb = (cvmx_usb_internal_state_t *)state;
 837
 838         /* Make sure all pipes are closed */
 839         if (usb->idle_pipes.head ||
 840                 usb->active_pipes[CVMX_USB_TRANSFER_ISOCHRONOUS].head ||
 841                 usb->active_pipes[CVMX_USB_TRANSFER_INTERRUPT].head ||
 842                 usb->active_pipes[CVMX_USB_TRANSFER_CONTROL].head ||
 843                 usb->active_pipes[CVMX_USB_TRANSFER_BULK].head)
 844                 return -EBUSY;
 845
 846         /* Disable the clocks and put them in power on reset */
 847         usbn_clk_ctl.u64 = __cvmx_usb_read_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index));
 848         usbn_clk_ctl.s.enable = 1;
 849         usbn_clk_ctl.s.por = 1;
 850         usbn_clk_ctl.s.hclk_rst = 1;
 851         usbn_clk_ctl.s.prst = 0;
 852         usbn_clk_ctl.s.hrst = 0;
 853         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 854                                usbn_clk_ctl.u64);
 855         return 0;
 856 }
 857
 858
 859 /**
 860  * Enable a USB port. After this call succeeds, the USB port is
 861  * online and servicing requests.
 862  *
 863  * @param state  USB device state populated by
 864  *               cvmx_usb_initialize().
 865  *
 866  * @return 0 or a negative error code.
 867  */
 868 int cvmx_usb_enable(cvmx_usb_state_t *state)
 869 {
 870         cvmx_usbcx_ghwcfg3_t usbcx_ghwcfg3;
 871         cvmx_usb_internal_state_t *usb = (cvmx_usb_internal_state_t *)state;
 872
 873         usb->usbcx_hprt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
 874
 875         /*
 876          * If the port is already enabled the just return. We don't need to do
 877          * anything
 878          */
 879         if (usb->usbcx_hprt.s.prtena)
 880                 return 0;
 881
 882         /* If there is nothing plugged into the port then fail immediately */
 883         if (!usb->usbcx_hprt.s.prtconnsts) {
 884                 return -ETIMEDOUT;
 885         }
 886
 887         /* Program the port reset bit to start the reset process */
 888         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt_t, prtrst, 1);
 889
 890         /*
 891          * Wait at least 50ms (high speed), or 10ms (full speed) for the reset
 892          * process to complete.
 893          */
 894         mdelay(50);
 895
 896         /* Program the port reset bit to 0, USBC_HPRT[PRTRST] = 0 */
 897         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt_t, prtrst, 0);
 898
 899         /* Wait for the USBC_HPRT[PRTENA]. */
 900         if (CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt_t,
 901                                   prtena, ==, 1, 100000))
 902                 return -ETIMEDOUT;
 903
 904         /* Read the port speed field to get the enumerated speed, USBC_HPRT[PRTSPD]. */
 905         usb->usbcx_hprt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
 906         usbcx_ghwcfg3.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GHWCFG3(usb->index));
 907
 908         /*
 909          * 13. Program the USBC_GRXFSIZ register to select the size of the
 910          *     receive FIFO (25%).
 911          */
 912         USB_SET_FIELD32(CVMX_USBCX_GRXFSIZ(usb->index), cvmx_usbcx_grxfsiz_t,
 913                         rxfdep, usbcx_ghwcfg3.s.dfifodepth / 4);
 914         /*
 915          * 14. Program the USBC_GNPTXFSIZ register to select the size and the
 916          *     start address of the non- periodic transmit FIFO for nonperiodic
 917          *     transactions (50%).
 918          */
 919         {
 920                 cvmx_usbcx_gnptxfsiz_t siz;
 921                 siz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index));
 922                 siz.s.nptxfdep = usbcx_ghwcfg3.s.dfifodepth / 2;
 923                 siz.s.nptxfstaddr = usbcx_ghwcfg3.s.dfifodepth / 4;
 924                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index), siz.u32);
 925         }
 926         /*
 927          * 15. Program the USBC_HPTXFSIZ register to select the size and start
 928          *     address of the periodic transmit FIFO for periodic transactions
 929          *     (25%).
 930          */
 931         {
 932                 cvmx_usbcx_hptxfsiz_t siz;
 933                 siz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index));
 934                 siz.s.ptxfsize = usbcx_ghwcfg3.s.dfifodepth / 4;
 935                 siz.s.ptxfstaddr = 3 * usbcx_ghwcfg3.s.dfifodepth / 4;
 936                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index), siz.u32);
 937         }
 938         /* Flush all FIFOs */
 939         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), cvmx_usbcx_grstctl_t, txfnum, 0x10);
 940         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), cvmx_usbcx_grstctl_t, txfflsh, 1);
 941         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), cvmx_usbcx_grstctl_t,
 942                               txfflsh, ==, 0, 100);
 943         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), cvmx_usbcx_grstctl_t, rxfflsh, 1);
 944         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index), cvmx_usbcx_grstctl_t,
 945                               rxfflsh, ==, 0, 100);
 946
 947         return 0;
 948 }
 949
 950
 951 /**
 952  * Disable a USB port. After this call the USB port will not
 953  * generate data transfers and will not generate events.
 954  * Transactions in process will fail and call their
 955  * associated callbacks.
 956  *
 957  * @param state  USB device state populated by
 958  *               cvmx_usb_initialize().
 959  *
 960  * @return 0 or a negative error code.
 961  */
 962 int cvmx_usb_disable(cvmx_usb_state_t *state)
 963 {
 964         cvmx_usb_internal_state_t *usb = (cvmx_usb_internal_state_t *)state;
 965
 966         /* Disable the port */
 967         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt_t, prtena, 1);
 968         return 0;
 969 }
 970
 971
 972 /**
 973  * Get the current state of the USB port. Use this call to
 974  * determine if the usb port has anything connected, is enabled,
 975  * or has some sort of error condition. The return value of this
 976  * call has "changed" bits to signal of the value of some fields
 977  * have changed between calls. These "changed" fields are based
 978  * on the last call to cvmx_usb_set_status(). In order to clear
 979  * them, you must update the status through cvmx_usb_set_status().
 980  *
 981  * @param state  USB device state populated by
 982  *               cvmx_usb_initialize().
 983  *
 984  * @return Port status information
 985  */
 986 cvmx_usb_port_status_t cvmx_usb_get_status(cvmx_usb_state_t *state)
 987 {
 988         cvmx_usbcx_hprt_t usbc_hprt;
 989         cvmx_usb_port_status_t result;
 990         cvmx_usb_internal_state_t *usb = (cvmx_usb_internal_state_t *)state;
 991
 992         memset(&result, 0, sizeof(result));
 993
 994         usbc_hprt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
 995         result.port_enabled = usbc_hprt.s.prtena;
 996         result.port_over_current = usbc_hprt.s.prtovrcurract;
 997         result.port_powered = usbc_hprt.s.prtpwr;
 998         result.port_speed = usbc_hprt.s.prtspd;
 999         result.connected = usbc_hprt.s.prtconnsts;
1000         result.connect_change = (result.connected != usb->port_status.connected);
1001
1002         return result;
1003 }
1004
1005
1006 /**
1007  * Set the current state of the USB port. The status is used as
1008  * a reference for the "changed" bits returned by
1009  * cvmx_usb_get_status(). Other than serving as a reference, the
1010  * status passed to this function is not used. No fields can be
1011  * changed through this call.
1012  *
1013  * @param state  USB device state populated by
1014  *               cvmx_usb_initialize().
1015  * @param port_status
1016  *               Port status to set, most like returned by cvmx_usb_get_status()
1017  */
1018 void cvmx_usb_set_status(cvmx_usb_state_t *state, cvmx_usb_port_status_t port_status)
1019 {
1020         cvmx_usb_internal_state_t *usb = (cvmx_usb_internal_state_t *)state;
1021         usb->port_status = port_status;
1022         return;
1023 }
1024
1025
1026 /**
1027  * @INTERNAL
1028  * Convert a USB transaction into a handle
1029  *
1030  * @param usb    USB device state populated by
1031  *               cvmx_usb_initialize().
1032  * @param transaction
1033  *               Transaction to get handle for
1034  *
1035  * @return Handle
1036  */
1037 static inline int __cvmx_usb_get_submit_handle(cvmx_usb_internal_state_t *usb,
1038                                                cvmx_usb_transaction_t *transaction)
1039 {
1040         return ((unsigned long)transaction - (unsigned long)usb->transaction) /
1041                         sizeof(*transaction);
1042 }
1043
1044
1045 /**
1046  * @INTERNAL
1047  * Convert a USB pipe into a handle
1048  *
1049  * @param usb    USB device state populated by
1050  *               cvmx_usb_initialize().
1051  * @param pipe   Pipe to get handle for
1052  *
1053  * @return Handle
1054  */
1055 static inline int __cvmx_usb_get_pipe_handle(cvmx_usb_internal_state_t *usb,
1056                                              cvmx_usb_pipe_t *pipe)
1057 {
1058         return ((unsigned long)pipe - (unsigned long)usb->pipe) / sizeof(*pipe);
1059 }
1060
1061
1062 /**
1063  * Open a virtual pipe between the host and a USB device. A pipe
1064  * must be opened before data can be transferred between a device
1065  * and Octeon.
1066  *
1067  * @param state      USB device state populated by
1068  *                   cvmx_usb_initialize().
1069  * @param flags      Optional pipe flags defined in
1070  *                   cvmx_usb_pipe_flags_t.
1071  * @param device_addr
1072  *                   USB device address to open the pipe to
1073  *                   (0-127).
1074  * @param endpoint_num
1075  *                   USB endpoint number to open the pipe to
1076  *                   (0-15).
1077  * @param device_speed
1078  *                   The speed of the device the pipe is going
1079  *                   to. This must match the device's speed,
1080  *                   which may be different than the port speed.
1081  * @param max_packet The maximum packet length the device can
1082  *                   transmit/receive (low speed=0-8, full
1083  *                   speed=0-1023, high speed=0-1024). This value
1084  *                   comes from the standard endpoint descriptor
1085  *                   field wMaxPacketSize bits <10:0>.
1086  * @param transfer_type
1087  *                   The type of transfer this pipe is for.
1088  * @param transfer_dir
1089  *                   The direction the pipe is in. This is not
1090  *                   used for control pipes.
1091  * @param interval   For ISOCHRONOUS and INTERRUPT transfers,
1092  *                   this is how often the transfer is scheduled
1093  *                   for. All other transfers should specify
1094  *                   zero. The units are in frames (8000/sec at
1095  *                   high speed, 1000/sec for full speed).
1096  * @param multi_count
1097  *                   For high speed devices, this is the maximum
1098  *                   allowed number of packet per microframe.
1099  *                   Specify zero for non high speed devices. This
1100  *                   value comes from the standard endpoint descriptor
1101  *                   field wMaxPacketSize bits <12:11>.
1102  * @param hub_device_addr
1103  *                   Hub device address this device is connected
1104  *                   to. Devices connected directly to Octeon
1105  *                   use zero. This is only used when the device
1106  *                   is full/low speed behind a high speed hub.
1107  *                   The address will be of the high speed hub,
1108  *                   not and full speed hubs after it.
1109  * @param hub_port   Which port on the hub the device is
1110  *                   connected. Use zero for devices connected
1111  *                   directly to Octeon. Like hub_device_addr,
1112  *                   this is only used for full/low speed
1113  *                   devices behind a high speed hub.
1114  *
1115  * @return A non negative value is a pipe handle. Negative
1116  *         values are error codes.
1117  */
1118 int cvmx_usb_open_pipe(cvmx_usb_state_t *state, cvmx_usb_pipe_flags_t flags,
1119                        int device_addr, int endpoint_num,
1120                        cvmx_usb_speed_t device_speed, int max_packet,
1121                        cvmx_usb_transfer_t transfer_type,
1122                        cvmx_usb_direction_t transfer_dir, int interval,
1123                        int multi_count, int hub_device_addr, int hub_port)
1124 {
1125         cvmx_usb_pipe_t *pipe;
1126         cvmx_usb_internal_state_t *usb = (cvmx_usb_internal_state_t *)state;
1127
1128         if (unlikely((device_addr < 0) || (device_addr > MAX_USB_ADDRESS)))
1129                 return -EINVAL;
1130         if (unlikely((endpoint_num < 0) || (endpoint_num > MAX_USB_ENDPOINT)))
1131                 return -EINVAL;
1132         if (unlikely(device_speed > CVMX_USB_SPEED_LOW))
1133                 return -EINVAL;
1134         if (unlikely((max_packet <= 0) || (max_packet > 1024)))
1135                 return -EINVAL;
1136         if (unlikely(transfer_type > CVMX_USB_TRANSFER_INTERRUPT))
1137                 return -EINVAL;
1138         if (unlikely((transfer_dir != CVMX_USB_DIRECTION_OUT) &&
1139                 (transfer_dir != CVMX_USB_DIRECTION_IN)))
1140                 return -EINVAL;
1141         if (unlikely(interval < 0))
1142                 return -EINVAL;
1143         if (unlikely((transfer_type == CVMX_USB_TRANSFER_CONTROL) && interval))
1144                 return -EINVAL;
1145         if (unlikely(multi_count < 0))
1146                 return -EINVAL;
1147         if (unlikely((device_speed != CVMX_USB_SPEED_HIGH) &&
1148                 (multi_count != 0)))
1149                 return -EINVAL;
1150         if (unlikely((hub_device_addr < 0) || (hub_device_addr > MAX_USB_ADDRESS)))
1151                 return -EINVAL;
1152         if (unlikely((hub_port < 0) || (hub_port > MAX_USB_HUB_PORT)))
1153                 return -EINVAL;
1154
1155         /* Find a free pipe */
1156         pipe = usb->free_pipes.head;
1157         if (!pipe)
1158                 return -ENOMEM;
1159         __cvmx_usb_remove_pipe(&usb->free_pipes, pipe);
1160         pipe->flags = flags | __CVMX_USB_PIPE_FLAGS_OPEN;
1161         if ((device_speed == CVMX_USB_SPEED_HIGH) &&
1162                 (transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1163                 (transfer_type == CVMX_USB_TRANSFER_BULK))
1164                 pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
1165         pipe->device_addr = device_addr;
1166         pipe->endpoint_num = endpoint_num;
1167         pipe->device_speed = device_speed;
1168         pipe->max_packet = max_packet;
1169         pipe->transfer_type = transfer_type;
1170         pipe->transfer_dir = transfer_dir;
1171         /*
1172          * All pipes use interval to rate limit NAK processing. Force an
1173          * interval if one wasn't supplied
1174          */
1175         if (!interval)
1176                 interval = 1;
1177         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1178                 pipe->interval = interval*8;
1179                 /* Force start splits to be schedule on uFrame 0 */
1180                 pipe->next_tx_frame = ((usb->frame_number+7)&~7) + pipe->interval;
1181         } else {
1182                 pipe->interval = interval;
1183                 pipe->next_tx_frame = usb->frame_number + pipe->interval;
1184         }
1185         pipe->multi_count = multi_count;
1186         pipe->hub_device_addr = hub_device_addr;
1187         pipe->hub_port = hub_port;
1188         pipe->pid_toggle = 0;
1189         pipe->split_sc_frame = -1;
1190         __cvmx_usb_append_pipe(&usb->idle_pipes, pipe);
1191
1192         /*
1193          * We don't need to tell the hardware about this pipe yet since
1194          * it doesn't have any submitted requests
1195          */
1196
1197         return __cvmx_usb_get_pipe_handle(usb, pipe);
1198 }
1199
1200
1201 /**
1202  * @INTERNAL
1203  * Poll the RX FIFOs and remove data as needed. This function is only used
1204  * in non DMA mode. It is very important that this function be called quickly
1205  * enough to prevent FIFO overflow.
1206  *
1207  * @param usb   USB device state populated by
1208  *              cvmx_usb_initialize().
1209  */
1210 static void __cvmx_usb_poll_rx_fifo(cvmx_usb_internal_state_t *usb)
1211 {
1212         cvmx_usbcx_grxstsph_t rx_status;
1213         int channel;
1214         int bytes;
1215         uint64_t address;
1216         uint32_t *ptr;
1217
1218         rx_status.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GRXSTSPH(usb->index));
1219         /* Only read data if IN data is there */
1220         if (rx_status.s.pktsts != 2)
1221                 return;
1222         /* Check if no data is available */
1223         if (!rx_status.s.bcnt)
1224                 return;
1225
1226         channel = rx_status.s.chnum;
1227         bytes = rx_status.s.bcnt;
1228         if (!bytes)
1229                 return;
1230
1231         /* Get where the DMA engine would have written this data */
1232         address = __cvmx_usb_read_csr64(usb, CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8);
1233         ptr = cvmx_phys_to_ptr(address);
1234         __cvmx_usb_write_csr64(usb, CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8, address + bytes);
1235
1236         /* Loop writing the FIFO data for this packet into memory */
1237         while (bytes > 0) {
1238                 *ptr++ = __cvmx_usb_read_csr32(usb, USB_FIFO_ADDRESS(channel, usb->index));
1239                 bytes -= 4;
1240         }
1241         CVMX_SYNCW;
1242
1243         return;
1244 }
1245
1246
1247 /**
1248  * Fill the TX hardware fifo with data out of the software
1249  * fifos
1250  *
1251  * @param usb       USB device state populated by
1252  *                  cvmx_usb_initialize().
1253  * @param fifo      Software fifo to use
1254  * @param available Amount of space in the hardware fifo
1255  *
1256  * @return Non zero if the hardware fifo was too small and needs
1257  *         to be serviced again.
1258  */
1259 static int __cvmx_usb_fill_tx_hw(cvmx_usb_internal_state_t *usb, cvmx_usb_tx_fifo_t *fifo, int available)
1260 {
1261         /*
1262          * We're done either when there isn't anymore space or the software FIFO
1263          * is empty
1264          */
1265         while (available && (fifo->head != fifo->tail)) {
1266                 int i = fifo->tail;
1267                 const uint32_t *ptr = cvmx_phys_to_ptr(fifo->entry[i].address);
1268                 uint64_t csr_address = USB_FIFO_ADDRESS(fifo->entry[i].channel, usb->index) ^ 4;
1269                 int words = available;
1270
1271                 /* Limit the amount of data to waht the SW fifo has */
1272                 if (fifo->entry[i].size <= available) {
1273                         words = fifo->entry[i].size;
1274                         fifo->tail++;
1275                         if (fifo->tail > MAX_CHANNELS)
1276                                 fifo->tail = 0;
1277                 }
1278
1279                 /* Update the next locations and counts */
1280                 available -= words;
1281                 fifo->entry[i].address += words * 4;
1282                 fifo->entry[i].size -= words;
1283
1284                 /*
1285                  * Write the HW fifo data. The read every three writes is due
1286                  * to an errata on CN3XXX chips
1287                  */
1288                 while (words > 3) {
1289                         cvmx_write64_uint32(csr_address, *ptr++);
1290                         cvmx_write64_uint32(csr_address, *ptr++);
1291                         cvmx_write64_uint32(csr_address, *ptr++);
1292                         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1293                         words -= 3;
1294                 }
1295                 cvmx_write64_uint32(csr_address, *ptr++);
1296                 if (--words) {
1297                         cvmx_write64_uint32(csr_address, *ptr++);
1298                         if (--words)
1299                                 cvmx_write64_uint32(csr_address, *ptr++);
1300                 }
1301                 cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1302         }
1303         return fifo->head != fifo->tail;
1304 }
1305
1306
1307 /**
1308  * Check the hardware FIFOs and fill them as needed
1309  *
1310  * @param usb   USB device state populated by
1311  *              cvmx_usb_initialize().
1312  */
1313 static void __cvmx_usb_poll_tx_fifo(cvmx_usb_internal_state_t *usb)
1314 {
1315         if (usb->periodic.head != usb->periodic.tail) {
1316                 cvmx_usbcx_hptxsts_t tx_status;
1317                 tx_status.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPTXSTS(usb->index));
1318                 if (__cvmx_usb_fill_tx_hw(usb, &usb->periodic, tx_status.s.ptxfspcavail))
1319                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), cvmx_usbcx_gintmsk_t, ptxfempmsk, 1);
1320                 else
1321                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), cvmx_usbcx_gintmsk_t, ptxfempmsk, 0);
1322         }
1323
1324         if (usb->nonperiodic.head != usb->nonperiodic.tail) {
1325                 cvmx_usbcx_gnptxsts_t tx_status;
1326                 tx_status.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GNPTXSTS(usb->index));
1327                 if (__cvmx_usb_fill_tx_hw(usb, &usb->nonperiodic, tx_status.s.nptxfspcavail))
1328                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), cvmx_usbcx_gintmsk_t, nptxfempmsk, 1);
1329                 else
1330                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), cvmx_usbcx_gintmsk_t, nptxfempmsk, 0);
1331         }
1332
1333         return;
1334 }
1335
1336
1337 /**
1338  * @INTERNAL
1339  * Fill the TX FIFO with an outgoing packet
1340  *
1341  * @param usb     USB device state populated by
1342  *                cvmx_usb_initialize().
1343  * @param channel Channel number to get packet from
1344  */
1345 static void __cvmx_usb_fill_tx_fifo(cvmx_usb_internal_state_t *usb, int channel)
1346 {
1347         cvmx_usbcx_hccharx_t hcchar;
1348         cvmx_usbcx_hcspltx_t usbc_hcsplt;
1349         cvmx_usbcx_hctsizx_t usbc_hctsiz;
1350         cvmx_usb_tx_fifo_t *fifo;
1351
1352         /* We only need to fill data on outbound channels */
1353         hcchar.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index));
1354         if (hcchar.s.epdir != CVMX_USB_DIRECTION_OUT)
1355                 return;
1356
1357         /* OUT Splits only have data on the start and not the complete */
1358         usbc_hcsplt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCSPLTX(channel, usb->index));
1359         if (usbc_hcsplt.s.spltena && usbc_hcsplt.s.compsplt)
1360                 return;
1361
1362         /* Find out how many bytes we need to fill and convert it into 32bit words */
1363         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index));
1364         if (!usbc_hctsiz.s.xfersize)
1365                 return;
1366
1367         if ((hcchar.s.eptype == CVMX_USB_TRANSFER_INTERRUPT) ||
1368                 (hcchar.s.eptype == CVMX_USB_TRANSFER_ISOCHRONOUS))
1369                 fifo = &usb->periodic;
1370         else
1371                 fifo = &usb->nonperiodic;
1372
1373         fifo->entry[fifo->head].channel = channel;
1374         fifo->entry[fifo->head].address = __cvmx_usb_read_csr64(usb, CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8);
1375         fifo->entry[fifo->head].size = (usbc_hctsiz.s.xfersize+3)>>2;
1376         fifo->head++;
1377         if (fifo->head > MAX_CHANNELS)
1378                 fifo->head = 0;
1379
1380         __cvmx_usb_poll_tx_fifo(usb);
1381
1382         return;
1383 }
1384
1385 /**
1386  * @INTERNAL
1387  * Perform channel specific setup for Control transactions. All
1388  * the generic stuff will already have been done in
1389  * __cvmx_usb_start_channel()
1390  *
1391  * @param usb     USB device state populated by
1392  *                cvmx_usb_initialize().
1393  * @param channel Channel to setup
1394  * @param pipe    Pipe for control transaction
1395  */
1396 static void __cvmx_usb_start_channel_control(cvmx_usb_internal_state_t *usb,
1397                                              int channel,
1398                                              cvmx_usb_pipe_t *pipe)
1399 {
1400         cvmx_usb_transaction_t *transaction = pipe->head;
1401         cvmx_usb_control_header_t *header = cvmx_phys_to_ptr(transaction->control_header);
1402         int bytes_to_transfer = transaction->buffer_length - transaction->actual_bytes;
1403         int packets_to_transfer;
1404         cvmx_usbcx_hctsizx_t usbc_hctsiz;
1405
1406         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index));
1407
1408         switch (transaction->stage) {
1409         case CVMX_USB_STAGE_NON_CONTROL:
1410         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
1411                 cvmx_dprintf("%s: ERROR - Non control stage\n", __FUNCTION__);
1412                 break;
1413         case CVMX_USB_STAGE_SETUP:
1414                 usbc_hctsiz.s.pid = 3; /* Setup */
1415                 bytes_to_transfer = sizeof(*header);
1416                 /* All Control operations start with a setup going OUT */
1417                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), cvmx_usbcx_hccharx_t, epdir, CVMX_USB_DIRECTION_OUT);
1418                 /*
1419                  * Setup send the control header instead of the buffer data. The
1420                  * buffer data will be used in the next stage
1421                  */
1422                 __cvmx_usb_write_csr64(usb, CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8, transaction->control_header);
1423                 break;
1424         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
1425                 usbc_hctsiz.s.pid = 3; /* Setup */
1426                 bytes_to_transfer = 0;
1427                 /* All Control operations start with a setup going OUT */
1428                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), cvmx_usbcx_hccharx_t, epdir, CVMX_USB_DIRECTION_OUT);
1429                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index), cvmx_usbcx_hcspltx_t, compsplt, 1);
1430                 break;
1431         case CVMX_USB_STAGE_DATA:
1432                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1433                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1434                         if (header->s.request_type & 0x80)
1435                                 bytes_to_transfer = 0;
1436                         else if (bytes_to_transfer > pipe->max_packet)
1437                                 bytes_to_transfer = pipe->max_packet;
1438                 }
1439                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1440                                 cvmx_usbcx_hccharx_t, epdir,
1441                                 ((header->s.request_type & 0x80) ?
1442                                         CVMX_USB_DIRECTION_IN :
1443                                         CVMX_USB_DIRECTION_OUT));
1444                 break;
1445         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
1446                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1447                 if (!(header->s.request_type & 0x80))
1448                         bytes_to_transfer = 0;
1449                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1450                                 cvmx_usbcx_hccharx_t, epdir,
1451                                 ((header->s.request_type & 0x80) ?
1452                                         CVMX_USB_DIRECTION_IN :
1453                                         CVMX_USB_DIRECTION_OUT));
1454                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index), cvmx_usbcx_hcspltx_t, compsplt, 1);
1455                 break;
1456         case CVMX_USB_STAGE_STATUS:
1457                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1458                 bytes_to_transfer = 0;
1459                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), cvmx_usbcx_hccharx_t, epdir,
1460                                 ((header->s.request_type & 0x80) ?
1461                                         CVMX_USB_DIRECTION_OUT :
1462                                         CVMX_USB_DIRECTION_IN));
1463                 break;
1464         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
1465                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1466                 bytes_to_transfer = 0;
1467                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), cvmx_usbcx_hccharx_t, epdir,
1468                                 ((header->s.request_type & 0x80) ?
1469                                         CVMX_USB_DIRECTION_OUT :
1470                                         CVMX_USB_DIRECTION_IN));
1471                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index), cvmx_usbcx_hcspltx_t, compsplt, 1);
1472                 break;
1473         }
1474
1475         /*
1476          * Make sure the transfer never exceeds the byte limit of the hardware.
1477          * Further bytes will be sent as continued transactions
1478          */
1479         if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1480                 /* Round MAX_TRANSFER_BYTES to a multiple of out packet size */
1481                 bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
1482                 bytes_to_transfer *= pipe->max_packet;
1483         }
1484
1485         /*
1486          * Calculate the number of packets to transfer. If the length is zero
1487          * we still need to transfer one packet
1488          */
1489         packets_to_transfer = (bytes_to_transfer + pipe->max_packet - 1) / pipe->max_packet;
1490         if (packets_to_transfer == 0)
1491                 packets_to_transfer = 1;
1492         else if ((packets_to_transfer > 1) && (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1493                 /*
1494                  * Limit to one packet when not using DMA. Channels must be
1495                  * restarted between every packet for IN transactions, so there
1496                  * is no reason to do multiple packets in a row
1497                  */
1498                 packets_to_transfer = 1;
1499                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1500         } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1501                 /*
1502                  * Limit the number of packet and data transferred to what the
1503                  * hardware can handle
1504                  */
1505                 packets_to_transfer = MAX_TRANSFER_PACKETS;
1506                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1507         }
1508
1509         usbc_hctsiz.s.xfersize = bytes_to_transfer;
1510         usbc_hctsiz.s.pktcnt = packets_to_transfer;
1511
1512         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index), usbc_hctsiz.u32);
1513         return;
1514 }
1515
1516
1517 /**
1518  * @INTERNAL
1519  * Start a channel to perform the pipe's head transaction
1520  *
1521  * @param usb     USB device state populated by
1522  *                cvmx_usb_initialize().
1523  * @param channel Channel to setup
1524  * @param pipe    Pipe to start
1525  */
1526 static void __cvmx_usb_start_channel(cvmx_usb_internal_state_t *usb,
1527                                      int channel,
1528                                      cvmx_usb_pipe_t *pipe)
1529 {
1530         cvmx_usb_transaction_t *transaction = pipe->head;
1531
1532         /* Make sure all writes to the DMA region get flushed */
1533         CVMX_SYNCW;
1534
1535         /* Attach the channel to the pipe */
1536         usb->pipe_for_channel[channel] = pipe;
1537         pipe->channel = channel;
1538         pipe->flags |= __CVMX_USB_PIPE_FLAGS_SCHEDULED;
1539
1540         /* Mark this channel as in use */
1541         usb->idle_hardware_channels &= ~(1<<channel);
1542
1543         /* Enable the channel interrupt bits */
1544         {
1545                 cvmx_usbcx_hcintx_t usbc_hcint;
1546                 cvmx_usbcx_hcintmskx_t usbc_hcintmsk;
1547                 cvmx_usbcx_haintmsk_t usbc_haintmsk;
1548
1549                 /* Clear all channel status bits */
1550                 usbc_hcint.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCINTX(channel, usb->index));
1551                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTX(channel, usb->index), usbc_hcint.u32);
1552
1553                 usbc_hcintmsk.u32 = 0;
1554                 usbc_hcintmsk.s.chhltdmsk = 1;
1555                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1556                         /* Channels need these extra interrupts when we aren't in DMA mode */
1557                         usbc_hcintmsk.s.datatglerrmsk = 1;
1558                         usbc_hcintmsk.s.frmovrunmsk = 1;
1559                         usbc_hcintmsk.s.bblerrmsk = 1;
1560                         usbc_hcintmsk.s.xacterrmsk = 1;
1561                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1562                                 /* Splits don't generate xfercompl, so we need ACK and NYET */
1563                                 usbc_hcintmsk.s.nyetmsk = 1;
1564                                 usbc_hcintmsk.s.ackmsk = 1;
1565                         }
1566                         usbc_hcintmsk.s.nakmsk = 1;
1567                         usbc_hcintmsk.s.stallmsk = 1;
1568                         usbc_hcintmsk.s.xfercomplmsk = 1;
1569                 }
1570                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), usbc_hcintmsk.u32);
1571
1572                 /* Enable the channel interrupt to propagate */
1573                 usbc_haintmsk.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HAINTMSK(usb->index));
1574                 usbc_haintmsk.s.haintmsk |= 1<<channel;
1575                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HAINTMSK(usb->index), usbc_haintmsk.u32);
1576         }
1577
1578         /* Setup the locations the DMA engines use  */
1579         {
1580                 uint64_t dma_address = transaction->buffer + transaction->actual_bytes;
1581                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1582                         dma_address = transaction->buffer + transaction->iso_packets[0].offset + transaction->actual_bytes;
1583                 __cvmx_usb_write_csr64(usb, CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8, dma_address);
1584                 __cvmx_usb_write_csr64(usb, CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8, dma_address);
1585         }
1586
1587         /* Setup both the size of the transfer and the SPLIT characteristics */
1588         {
1589                 cvmx_usbcx_hcspltx_t usbc_hcsplt = {.u32 = 0};
1590                 cvmx_usbcx_hctsizx_t usbc_hctsiz = {.u32 = 0};
1591                 int packets_to_transfer;
1592                 int bytes_to_transfer = transaction->buffer_length - transaction->actual_bytes;
1593
1594                 /*
1595                  * ISOCHRONOUS transactions store each individual transfer size
1596                  * in the packet structure, not the global buffer_length
1597                  */
1598                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1599                         bytes_to_transfer = transaction->iso_packets[0].length - transaction->actual_bytes;
1600
1601                 /*
1602                  * We need to do split transactions when we are talking to non
1603                  * high speed devices that are behind a high speed hub
1604                  */
1605                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1606                         /*
1607                          * On the start split phase (stage is even) record the
1608                          * frame number we will need to send the split complete.
1609                          * We only store the lower two bits since the time ahead
1610                          * can only be two frames
1611                          */
1612                         if ((transaction->stage&1) == 0) {
1613                                 if (transaction->type == CVMX_USB_TRANSFER_BULK)
1614                                         pipe->split_sc_frame = (usb->frame_number + 1) & 0x7f;
1615                                 else
1616                                         pipe->split_sc_frame = (usb->frame_number + 2) & 0x7f;
1617                         } else
1618                                 pipe->split_sc_frame = -1;
1619
1620                         usbc_hcsplt.s.spltena = 1;
1621                         usbc_hcsplt.s.hubaddr = pipe->hub_device_addr;
1622                         usbc_hcsplt.s.prtaddr = pipe->hub_port;
1623                         usbc_hcsplt.s.compsplt = (transaction->stage == CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE);
1624
1625                         /*
1626                          * SPLIT transactions can only ever transmit one data
1627                          * packet so limit the transfer size to the max packet
1628                          * size
1629                          */
1630                         if (bytes_to_transfer > pipe->max_packet)
1631                                 bytes_to_transfer = pipe->max_packet;
1632
1633                         /*
1634                          * ISOCHRONOUS OUT splits are unique in that they limit
1635                          * data transfers to 188 byte chunks representing the
1636                          * begin/middle/end of the data or all
1637                          */
1638                         if (!usbc_hcsplt.s.compsplt &&
1639                                 (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1640                                 (pipe->transfer_type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
1641                                 /*
1642                                  * Clear the split complete frame number as
1643                                  * there isn't going to be a split complete
1644                                  */
1645                                 pipe->split_sc_frame = -1;
1646                                 /*
1647                                  * See if we've started this transfer and sent
1648                                  * data
1649                                  */
1650                                 if (transaction->actual_bytes == 0) {
1651                                         /*
1652                                          * Nothing sent yet, this is either a
1653                                          * begin or the entire payload
1654                                          */
1655                                         if (bytes_to_transfer <= 188)
1656                                                 usbc_hcsplt.s.xactpos = 3; /* Entire payload in one go */
1657                                         else
1658                                                 usbc_hcsplt.s.xactpos = 2; /* First part of payload */
1659                                 } else {
1660                                         /*
1661                                          * Continuing the previous data, we must
1662                                          * either be in the middle or at the end
1663                                          */
1664                                         if (bytes_to_transfer <= 188)
1665                                                 usbc_hcsplt.s.xactpos = 1; /* End of payload */
1666                                         else
1667                                                 usbc_hcsplt.s.xactpos = 0; /* Middle of payload */
1668                                 }
1669                                 /*
1670                                  * Again, the transfer size is limited to 188
1671                                  * bytes
1672                                  */
1673                                 if (bytes_to_transfer > 188)
1674                                         bytes_to_transfer = 188;
1675                         }
1676                 }
1677
1678                 /*
1679                  * Make sure the transfer never exceeds the byte limit of the
1680                  * hardware. Further bytes will be sent as continued
1681                  * transactions
1682                  */
1683                 if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1684                         /*
1685                          * Round MAX_TRANSFER_BYTES to a multiple of out packet
1686                          * size
1687                          */
1688                         bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
1689                         bytes_to_transfer *= pipe->max_packet;
1690                 }
1691
1692                 /*
1693                  * Calculate the number of packets to transfer. If the length is
1694                  * zero we still need to transfer one packet
1695                  */
1696                 packets_to_transfer = (bytes_to_transfer + pipe->max_packet - 1) / pipe->max_packet;
1697                 if (packets_to_transfer == 0)
1698                         packets_to_transfer = 1;
1699                 else if ((packets_to_transfer > 1) && (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1700                         /*
1701                          * Limit to one packet when not using DMA. Channels must
1702                          * be restarted between every packet for IN
1703                          * transactions, so there is no reason to do multiple
1704                          * packets in a row
1705                          */
1706                         packets_to_transfer = 1;
1707                         bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1708                 } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1709                         /*
1710                          * Limit the number of packet and data transferred to
1711                          * what the hardware can handle
1712                          */
1713                         packets_to_transfer = MAX_TRANSFER_PACKETS;
1714                         bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1715                 }
1716
1717                 usbc_hctsiz.s.xfersize = bytes_to_transfer;
1718                 usbc_hctsiz.s.pktcnt = packets_to_transfer;
1719
1720                 /* Update the DATA0/DATA1 toggle */
1721                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1722                 /*
1723                  * High speed pipes may need a hardware ping before they start
1724                  */
1725                 if (pipe->flags & __CVMX_USB_PIPE_FLAGS_NEED_PING)
1726                         usbc_hctsiz.s.dopng = 1;
1727
1728                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCSPLTX(channel, usb->index), usbc_hcsplt.u32);
1729                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index), usbc_hctsiz.u32);
1730         }
1731
1732         /* Setup the Host Channel Characteristics Register */
1733         {
1734                 cvmx_usbcx_hccharx_t usbc_hcchar = {.u32 = 0};
1735
1736                 /*
1737                  * Set the startframe odd/even properly. This is only used for
1738                  * periodic
1739                  */
1740                 usbc_hcchar.s.oddfrm = usb->frame_number&1;
1741
1742                 /*
1743                  * Set the number of back to back packets allowed by this
1744                  * endpoint. Split transactions interpret "ec" as the number of
1745                  * immediate retries of failure. These retries happen too
1746                  * quickly, so we disable these entirely for splits
1747                  */
1748                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
1749                         usbc_hcchar.s.ec = 1;
1750                 else if (pipe->multi_count < 1)
1751                         usbc_hcchar.s.ec = 1;
1752                 else if (pipe->multi_count > 3)
1753                         usbc_hcchar.s.ec = 3;
1754                 else
1755                         usbc_hcchar.s.ec = pipe->multi_count;
1756
1757                 /* Set the rest of the endpoint specific settings */
1758                 usbc_hcchar.s.devaddr = pipe->device_addr;
1759                 usbc_hcchar.s.eptype = transaction->type;
1760                 usbc_hcchar.s.lspddev = (pipe->device_speed == CVMX_USB_SPEED_LOW);
1761                 usbc_hcchar.s.epdir = pipe->transfer_dir;
1762                 usbc_hcchar.s.epnum = pipe->endpoint_num;
1763                 usbc_hcchar.s.mps = pipe->max_packet;
1764                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index), usbc_hcchar.u32);
1765         }
1766
1767         /* Do transaction type specific fixups as needed */
1768         switch (transaction->type) {
1769         case CVMX_USB_TRANSFER_CONTROL:
1770                 __cvmx_usb_start_channel_control(usb, channel, pipe);
1771                 break;
1772         case CVMX_USB_TRANSFER_BULK:
1773         case CVMX_USB_TRANSFER_INTERRUPT:
1774                 break;
1775         case CVMX_USB_TRANSFER_ISOCHRONOUS:
1776                 if (!__cvmx_usb_pipe_needs_split(usb, pipe)) {
1777                         /*
1778                          * ISO transactions require different PIDs depending on
1779                          * direction and how many packets are needed
1780                          */
1781                         if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
1782                                 if (pipe->multi_count < 2) /* Need DATA0 */
1783                                         USB_SET_FIELD32(CVMX_USBCX_HCTSIZX(channel, usb->index), cvmx_usbcx_hctsizx_t, pid, 0);
1784                                 else /* Need MDATA */
1785                                         USB_SET_FIELD32(CVMX_USBCX_HCTSIZX(channel, usb->index), cvmx_usbcx_hctsizx_t, pid, 3);
1786                         }
1787                 }
1788                 break;
1789         }
1790         {
1791                 cvmx_usbcx_hctsizx_t usbc_hctsiz = {.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index))};
1792                 transaction->xfersize = usbc_hctsiz.s.xfersize;
1793                 transaction->pktcnt = usbc_hctsiz.s.pktcnt;
1794         }
1795         /* Remeber when we start a split transaction */
1796         if (__cvmx_usb_pipe_needs_split(usb, pipe))
1797                 usb->active_split = transaction;
1798         USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index), cvmx_usbcx_hccharx_t, chena, 1);
1799         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
1800                 __cvmx_usb_fill_tx_fifo(usb, channel);
1801         return;
1802 }
1803
1804
1805 /**
1806  * @INTERNAL
1807  * Find a pipe that is ready to be scheduled to hardware.
1808  * @param usb    USB device state populated by
1809  *               cvmx_usb_initialize().
1810  * @param list   Pipe list to search
1811  * @param current_frame
1812  *               Frame counter to use as a time reference.
1813  *
1814  * @return Pipe or NULL if none are ready
1815  */
1816 static cvmx_usb_pipe_t *__cvmx_usb_find_ready_pipe(cvmx_usb_internal_state_t *usb, cvmx_usb_pipe_list_t *list, uint64_t current_frame)
1817 {
1818         cvmx_usb_pipe_t *pipe = list->head;
1819         while (pipe) {
1820                 if (!(pipe->flags & __CVMX_USB_PIPE_FLAGS_SCHEDULED) && pipe->head &&
1821                         (pipe->next_tx_frame <= current_frame) &&
1822                         ((pipe->split_sc_frame == -1) || ((((int)current_frame - (int)pipe->split_sc_frame) & 0x7f) < 0x40)) &&
1823                         (!usb->active_split || (usb->active_split == pipe->head))) {
1824                         CVMX_PREFETCH(pipe, 128);
1825                         CVMX_PREFETCH(pipe->head, 0);
1826                         return pipe;
1827                 }
1828                 pipe = pipe->next;
1829         }
1830         return NULL;
1831 }
1832
1833
1834 /**
1835  * @INTERNAL
1836  * Called whenever a pipe might need to be scheduled to the
1837  * hardware.
1838  *
1839  * @param usb    USB device state populated by
1840  *               cvmx_usb_initialize().
1841  * @param is_sof True if this schedule was called on a SOF interrupt.
1842  */
1843 static void __cvmx_usb_schedule(cvmx_usb_internal_state_t *usb, int is_sof)
1844 {
1845         int channel;
1846         cvmx_usb_pipe_t *pipe;
1847         int need_sof;
1848         cvmx_usb_transfer_t ttype;
1849
1850         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1851                 /* Without DMA we need to be careful to not schedule something at the end of a frame and cause an overrun */
1852                 cvmx_usbcx_hfnum_t hfnum = {.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index))};
1853                 cvmx_usbcx_hfir_t hfir = {.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HFIR(usb->index))};
1854                 if (hfnum.s.frrem < hfir.s.frint/4)
1855                         goto done;
1856         }
1857
1858         while (usb->idle_hardware_channels) {
1859                 /* Find an idle channel */
1860                 CVMX_CLZ(channel, usb->idle_hardware_channels);
1861                 channel = 31 - channel;
1862                 if (unlikely(channel > 7))
1863                         break;
1864
1865                 /* Find a pipe needing service */
1866                 pipe = NULL;
1867                 if (is_sof) {
1868                         /*
1869                          * Only process periodic pipes on SOF interrupts. This
1870                          * way we are sure that the periodic data is sent in the
1871                          * beginning of the frame
1872                          */
1873                         pipe = __cvmx_usb_find_ready_pipe(usb, usb->active_pipes + CVMX_USB_TRANSFER_ISOCHRONOUS, usb->frame_number);
1874                         if (likely(!pipe))
1875                                 pipe = __cvmx_usb_find_ready_pipe(usb, usb->active_pipes + CVMX_USB_TRANSFER_INTERRUPT, usb->frame_number);
1876                 }
1877                 if (likely(!pipe)) {
1878                         pipe = __cvmx_usb_find_ready_pipe(usb, usb->active_pipes + CVMX_USB_TRANSFER_CONTROL, usb->frame_number);
1879                         if (likely(!pipe))
1880                                 pipe = __cvmx_usb_find_ready_pipe(usb, usb->active_pipes + CVMX_USB_TRANSFER_BULK, usb->frame_number);
1881                 }
1882                 if (!pipe)
1883                         break;
1884
1885                 __cvmx_usb_start_channel(usb, channel, pipe);
1886         }
1887
1888 done:
1889         /*
1890          * Only enable SOF interrupts when we have transactions pending in the
1891          * future that might need to be scheduled
1892          */
1893         need_sof = 0;
1894         for (ttype = CVMX_USB_TRANSFER_CONTROL; ttype <= CVMX_USB_TRANSFER_INTERRUPT; ttype++) {
1895                 pipe = usb->active_pipes[ttype].head;
1896                 while (pipe) {
1897                         if (pipe->next_tx_frame > usb->frame_number) {
1898                                 need_sof = 1;
1899                                 break;
1900                         }
1901                         pipe = pipe->next;
1902                 }
1903         }
1904         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), cvmx_usbcx_gintmsk_t, sofmsk, need_sof);
1905         return;
1906 }
1907
1908
1909 /**
1910  * @INTERNAL
1911  * Call a user's callback for a specific reason.
1912  *
1913  * @param usb    USB device state populated by
1914  *               cvmx_usb_initialize().
1915  * @param pipe   Pipe the callback is for or NULL
1916  * @param transaction
1917  *               Transaction the callback is for or NULL
1918  * @param reason Reason this callback is being called
1919  * @param complete_code
1920  *               Completion code for the transaction, if any
1921  */
1922 static void __cvmx_usb_perform_callback(cvmx_usb_internal_state_t *usb,
1923                                         cvmx_usb_pipe_t *pipe,
1924                                         cvmx_usb_transaction_t *transaction,
1925                                         cvmx_usb_callback_t reason,
1926                                         cvmx_usb_complete_t complete_code)
1927 {
1928         cvmx_usb_callback_func_t callback = usb->callback[reason];
1929         void *user_data = usb->callback_data[reason];
1930         int submit_handle = -1;
1931         int pipe_handle = -1;
1932         int bytes_transferred = 0;
1933
1934         if (pipe)
1935                 pipe_handle = __cvmx_usb_get_pipe_handle(usb, pipe);
1936
1937         if (transaction) {
1938                 submit_handle = __cvmx_usb_get_submit_handle(usb, transaction);
1939                 bytes_transferred = transaction->actual_bytes;
1940                 /* Transactions are allowed to override the default callback */
1941                 if ((reason == CVMX_USB_CALLBACK_TRANSFER_COMPLETE) && transaction->callback) {
1942                         callback = transaction->callback;
1943                         user_data = transaction->callback_data;
1944                 }
1945         }
1946
1947         if (!callback)
1948                 return;
1949
1950         callback((cvmx_usb_state_t *)usb, reason, complete_code, pipe_handle, submit_handle,
1951                  bytes_transferred, user_data);
1952 }
1953
1954
1955 /**
1956  * @INTERNAL
1957  * Signal the completion of a transaction and free it. The
1958  * transaction will be removed from the pipe transaction list.
1959  *
1960  * @param usb    USB device state populated by
1961  *               cvmx_usb_initialize().
1962  * @param pipe   Pipe the transaction is on
1963  * @param transaction
1964  *               Transaction that completed
1965  * @param complete_code
1966  *               Completion code
1967  */
1968 static void __cvmx_usb_perform_complete(cvmx_usb_internal_state_t *usb,
1969                                         cvmx_usb_pipe_t *pipe,
1970                                         cvmx_usb_transaction_t *transaction,
1971                                         cvmx_usb_complete_t complete_code)
1972 {
1973         /* If this was a split then clear our split in progress marker */
1974         if (usb->active_split == transaction)
1975                 usb->active_split = NULL;
1976
1977         /*
1978          * Isochronous transactions need extra processing as they might not be
1979          * done after a single data transfer
1980          */
1981         if (unlikely(transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
1982                 /* Update the number of bytes transferred in this ISO packet */
1983                 transaction->iso_packets[0].length = transaction->actual_bytes;
1984                 transaction->iso_packets[0].status = complete_code;
1985
1986                 /*
1987                  * If there are more ISOs pending and we succeeded, schedule the
1988                  * next one
1989                  */
1990                 if ((transaction->iso_number_packets > 1) && (complete_code == CVMX_USB_COMPLETE_SUCCESS)) {
1991                         transaction->actual_bytes = 0;     /* No bytes transferred for this packet as of yet */
1992                         transaction->iso_number_packets--; /* One less ISO waiting to transfer */
1993                         transaction->iso_packets++;        /* Increment to the next location in our packet array */
1994                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
1995                         goto done;
1996                 }
1997         }
1998
1999         /* Remove the transaction from the pipe list */
2000         if (transaction->next)
2001                 transaction->next->prev = transaction->prev;
2002         else
2003                 pipe->tail = transaction->prev;
2004         if (transaction->prev)
2005                 transaction->prev->next = transaction->next;
2006         else
2007                 pipe->head = transaction->next;
2008         if (!pipe->head) {
2009                 __cvmx_usb_remove_pipe(usb->active_pipes + pipe->transfer_type, pipe);
2010                 __cvmx_usb_append_pipe(&usb->idle_pipes, pipe);
2011
2012         }
2013         __cvmx_usb_perform_callback(usb, pipe, transaction,
2014                                     CVMX_USB_CALLBACK_TRANSFER_COMPLETE,
2015                                     complete_code);
2016         __cvmx_usb_free_transaction(usb, transaction);
2017 done:
2018         return;
2019 }
2020
2021
2022 /**
2023  * @INTERNAL
2024  * Submit a usb transaction to a pipe. Called for all types
2025  * of transactions.
2026  *
2027  * @param usb
2028  * @param pipe_handle
2029  *                  Which pipe to submit to. Will be validated in this function.
2030  * @param type      Transaction type
2031  * @param flags     Flags for the transaction
2032  * @param buffer    User buffer for the transaction
2033  * @param buffer_length
2034  *                  User buffer's length in bytes
2035  * @param control_header
2036  *                  For control transactions, the 8 byte standard header
2037  * @param iso_start_frame
2038  *                  For ISO transactions, the start frame
2039  * @param iso_number_packets
2040  *                  For ISO, the number of packet in the transaction.
2041  * @param iso_packets
2042  *                  A description of each ISO packet
2043  * @param callback  User callback to call when the transaction completes
2044  * @param user_data User's data for the callback
2045  *
2046  * @return Submit handle or negative on failure. Matches the result
2047  *         in the external API.
2048  */
2049 static int __cvmx_usb_submit_transaction(cvmx_usb_internal_state_t *usb,
2050                                          int pipe_handle,
2051                                          cvmx_usb_transfer_t type,
2052                                          int flags,
2053                                          uint64_t buffer,
2054                                          int buffer_length,
2055                                          uint64_t control_header,
2056                                          int iso_start_frame,
2057                                          int iso_number_packets,
2058                                          cvmx_usb_iso_packet_t *iso_packets,
2059                                          cvmx_usb_callback_func_t callback,
2060                                          void *user_data)
2061 {
2062         int submit_handle;
2063         cvmx_usb_transaction_t *transaction;
2064         cvmx_usb_pipe_t *pipe = usb->pipe + pipe_handle;
2065
2066         if (unlikely((pipe_handle < 0) || (pipe_handle >= MAX_PIPES)))
2067                 return -EINVAL;
2068         /* Fail if the pipe isn't open */
2069         if (unlikely((pipe->flags & __CVMX_USB_PIPE_FLAGS_OPEN) == 0))
2070                 return -EINVAL;
2071         if (unlikely(pipe->transfer_type != type))
2072                 return -EINVAL;
2073
2074         transaction = __cvmx_usb_alloc_transaction(usb);
2075         if (unlikely(!transaction))
2076                 return -ENOMEM;
2077
2078         transaction->type = type;
2079         transaction->flags |= flags;
2080         transaction->buffer = buffer;
2081         transaction->buffer_length = buffer_length;
2082         transaction->control_header = control_header;
2083         transaction->iso_start_frame = iso_start_frame; // FIXME: This is not used, implement it
2084         transaction->iso_number_packets = iso_number_packets;
2085         transaction->iso_packets = iso_packets;
2086         transaction->callback = callback;
2087         transaction->callback_data = user_data;
2088         if (transaction->type == CVMX_USB_TRANSFER_CONTROL)
2089                 transaction->stage = CVMX_USB_STAGE_SETUP;
2090         else
2091                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2092
2093         transaction->next = NULL;
2094         if (pipe->tail) {
2095                 transaction->prev = pipe->tail;
2096                 transaction->prev->next = transaction;
2097         } else {
2098                 if (pipe->next_tx_frame < usb->frame_number)
2099                         pipe->next_tx_frame = usb->frame_number + pipe->interval -
2100                                 (usb->frame_number - pipe->next_tx_frame) % pipe->interval;
2101                 transaction->prev = NULL;
2102                 pipe->head = transaction;
2103                 __cvmx_usb_remove_pipe(&usb->idle_pipes, pipe);
2104                 __cvmx_usb_append_pipe(usb->active_pipes + pipe->transfer_type, pipe);
2105         }
2106         pipe->tail = transaction;
2107
2108         submit_handle = __cvmx_usb_get_submit_handle(usb, transaction);
2109
2110         /* We may need to schedule the pipe if this was the head of the pipe */
2111         if (!transaction->prev)
2112                 __cvmx_usb_schedule(usb, 0);
2113
2114         return submit_handle;
2115 }
2116
2117
2118 /**
2119  * Call to submit a USB Bulk transfer to a pipe.
2120  *
2121  * @param state     USB device state populated by
2122  *                  cvmx_usb_initialize().
2123  * @param pipe_handle
2124  *                  Handle to the pipe for the transfer.
2125  * @param buffer    Physical address of the data buffer in
2126  *                  memory. Note that this is NOT A POINTER, but
2127  *                  the full 64bit physical address of the
2128  *                  buffer. This may be zero if buffer_length is
2129  *                  zero.
2130  * @param buffer_length
2131  *                  Length of buffer in bytes.
2132  * @param callback  Function to call when this transaction
2133  *                  completes. If the return value of this
2134  *                  function isn't an error, then this function
2135  *                  is guaranteed to be called when the
2136  *                  transaction completes. If this parameter is
2137  *                  NULL, then the generic callback registered
2138  *                  through cvmx_usb_register_callback is
2139  *                  called. If both are NULL, then there is no
2140  *                  way to know when a transaction completes.
2141  * @param user_data User supplied data returned when the
2142  *                  callback is called. This is only used if
2143  *                  callback in not NULL.
2144  *
2145  * @return A submitted transaction handle or negative on
2146  *         failure. Negative values are error codes.
2147  */
2148 int cvmx_usb_submit_bulk(cvmx_usb_state_t *state, int pipe_handle,
2149                          uint64_t buffer, int buffer_length,
2150                          cvmx_usb_callback_func_t callback,
2151                          void *user_data)
2152 {
2153         int submit_handle;
2154         cvmx_usb_internal_state_t *usb = (cvmx_usb_internal_state_t *)state;
2155
2156         /* Pipe handle checking is done later in a common place */
2157         if (unlikely(!buffer))
2158                 return -EINVAL;
2159         if (unlikely(buffer_length < 0))
2160                 return -EINVAL;
2161
2162         submit_handle = __cvmx_usb_submit_transaction(usb, pipe_handle,
2163                                                       CVMX_USB_TRANSFER_BULK,
2164                                                       0, /* flags */
2165                                                       buffer,
2166                                                       buffer_length,
2167                                                       0, /* control_header */
2168                                                       0, /* iso_start_frame */
2169                                                       0, /* iso_number_packets */
2170                                                       NULL, /* iso_packets */
2171                                                       callback,
2172                                                       user_data);
2173         return submit_handle;
2174 }
2175
2176
2177 /**
2178  * Call to submit a USB Interrupt transfer to a pipe.
2179  *
2180  * @param state     USB device state populated by
2181  *                  cvmx_usb_initialize().
2182  * @param pipe_handle
2183  *                  Handle to the pipe for the transfer.
2184  * @param buffer    Physical address of the data buffer in
2185  *                  memory. Note that this is NOT A POINTER, but
2186  *                  the full 64bit physical address of the
2187  *                  buffer. This may be zero if buffer_length is
2188  *                  zero.
2189  * @param buffer_length
2190  *                  Length of buffer in bytes.
2191  * @param callback  Function to call when this transaction
2192  *                  completes. If the return value of this
2193  *                  function isn't an error, then this function
2194  *                  is guaranteed to be called when the
2195  *                  transaction completes. If this parameter is
2196  *                  NULL, then the generic callback registered
2197  *                  through cvmx_usb_register_callback is
2198  *                  called. If both are NULL, then there is no
2199  *                  way to know when a transaction completes.
2200  * @param user_data User supplied data returned when the
2201  *                  callback is called. This is only used if
2202  *                  callback in not NULL.
2203  *
2204  * @return A submitted transaction handle or negative on
2205  *         failure. Negative values are error codes.
2206  */
2207 int cvmx_usb_submit_interrupt(cvmx_usb_state_t *state, int pipe_handle,
2208                               uint64_t buffer, int buffer_length,
2209                               cvmx_usb_callback_func_t callback,
2210                               void *user_data)
2211 {
2212         int submit_handle;
2213         cvmx_usb_internal_state_t *usb = (cvmx_usb_internal_state_t *)state;
2214
2215         /* Pipe handle checking is done later in a common place */
2216         if (unlikely(!buffer))
2217                 return -EINVAL;
2218         if (unlikely(buffer_length < 0))
2219                 return -EINVAL;
2220
2221         submit_handle = __cvmx_usb_submit_transaction(usb, pipe_handle,
2222                                                       CVMX_USB_TRANSFER_INTERRUPT,
2223                                                       0, /* flags */
2224                                                       buffer,
2225                                                       buffer_length,
2226                                                       0, /* control_header */
2227                                                       0, /* iso_start_frame */
2228                                                       0, /* iso_number_packets */
2229                                                       NULL, /* iso_packets */
2230                                                       callback,
2231                                                       user_data);
2232         return submit_handle;
2233 }
2234
2235
2236 /**
2237  * Call to submit a USB Control transfer to a pipe.
2238  *
2239  * @param state     USB device state populated by
2240  *                  cvmx_usb_initialize().
2241  * @param pipe_handle
2242  *                  Handle to the pipe for the transfer.
2243  * @param control_header
2244  *                  USB 8 byte control header physical address.
2245  *                  Note that this is NOT A POINTER, but the
2246  *                  full 64bit physical address of the buffer.
2247  * @param buffer    Physical address of the data buffer in
2248  *                  memory. Note that this is NOT A POINTER, but
2249  *                  the full 64bit physical address of the
2250  *                  buffer. This may be zero if buffer_length is
2251  *                  zero.
2252  * @param buffer_length
2253  *                  Length of buffer in bytes.
2254  * @param callback  Function to call when this transaction
2255  *                  completes. If the return value of this
2256  *                  function isn't an error, then this function
2257  *                  is guaranteed to be called when the
2258  *                  transaction completes. If this parameter is
2259  *                  NULL, then the generic callback registered
2260  *                  through cvmx_usb_register_callback is
2261  *                  called. If both are NULL, then there is no
2262  *                  way to know when a transaction completes.
2263  * @param user_data User supplied data returned when the
2264  *                  callback is called. This is only used if
2265  *                  callback in not NULL.
2266  *
2267  * @return A submitted transaction handle or negative on
2268  *         failure. Negative values are error codes.
2269  */
2270 int cvmx_usb_submit_control(cvmx_usb_state_t *state, int pipe_handle,
2271                             uint64_t control_header,
2272                             uint64_t buffer, int buffer_length,
2273                             cvmx_usb_callback_func_t callback,
2274                             void *user_data)
2275 {
2276         int submit_handle;
2277         cvmx_usb_internal_state_t *usb = (cvmx_usb_internal_state_t *)state;
2278         cvmx_usb_control_header_t *header = cvmx_phys_to_ptr(control_header);
2279
2280         /* Pipe handle checking is done later in a common place */
2281         if (unlikely(!control_header))
2282                 return -EINVAL;
2283         /* Some drivers send a buffer with a zero length. God only knows why */
2284         if (unlikely(buffer && (buffer_length < 0)))
2285                 return -EINVAL;
2286         if (unlikely(!buffer && (buffer_length != 0)))
2287                 return -EINVAL;
2288         if ((header->s.request_type & 0x80) == 0)
2289                 buffer_length = le16_to_cpu(header->s.length);
2290
2291         submit_handle = __cvmx_usb_submit_transaction(usb, pipe_handle,
2292                                                       CVMX_USB_TRANSFER_CONTROL,
2293                                                       0, /* flags */
2294                                                       buffer,
2295                                                       buffer_length,
2296                                                       control_header,
2297                                                       0, /* iso_start_frame */
2298                                                       0, /* iso_number_packets */
2299                                                       NULL, /* iso_packets */
2300                                                       callback,
2301                                                       user_data);
2302         return submit_handle;
2303 }
2304
2305
2306 /**
2307  * Call to submit a USB Isochronous transfer to a pipe.
2308  *
2309  * @param state     USB device state populated by
2310  *                  cvmx_usb_initialize().
2311  * @param pipe_handle
2312  *                  Handle to the pipe for the transfer.
2313  * @param start_frame
2314  *                  Number of frames into the future to schedule
2315  *                  this transaction.
2316  * @param flags     Flags to control the transfer. See
2317  *                  cvmx_usb_isochronous_flags_t for the flag
2318  *                  definitions.
2319  * @param number_packets
2320  *                  Number of sequential packets to transfer.
2321  *                  "packets" is a pointer to an array of this
2322  *                  many packet structures.
2323  * @param packets   Description of each transfer packet as
2324  *                  defined by cvmx_usb_iso_packet_t. The array
2325  *                  pointed to here must stay valid until the
2326  *                  complete callback is called.
2327  * @param buffer    Physical address of the data buffer in
2328  *                  memory. Note that this is NOT A POINTER, but
2329  *                  the full 64bit physical address of the
2330  *                  buffer. This may be zero if buffer_length is
2331  *                  zero.
2332  * @param buffer_length
2333  *                  Length of buffer in bytes.
2334  * @param callback  Function to call when this transaction
2335  *                  completes. If the return value of this
2336  *                  function isn't an error, then this function
2337  *                  is guaranteed to be called when the
2338  *                  transaction completes. If this parameter is
2339  *                  NULL, then the generic callback registered
2340  *                  through cvmx_usb_register_callback is
2341  *                  called. If both are NULL, then there is no
2342  *                  way to know when a transaction completes.
2343  * @param user_data User supplied data returned when the
2344  *                  callback is called. This is only used if
2345  *                  callback in not NULL.
2346  *
2347  * @return A submitted transaction handle or negative on
2348  *         failure. Negative values are error codes.
2349  */
2350 int cvmx_usb_submit_isochronous(cvmx_usb_state_t *state, int pipe_handle,
2351                                 int start_frame, int flags,
2352                                 int number_packets,
2353                                 cvmx_usb_iso_packet_t packets[],
2354                                 uint64_t buffer, int buffer_length,
2355                                 cvmx_usb_callback_func_t callback,
2356                                 void *user_data)
2357 {
2358         int submit_handle;
2359         cvmx_usb_internal_state_t *usb = (cvmx_usb_internal_state_t *)state;
2360
2361         /* Pipe handle checking is done later in a common place */
2362         if (unlikely(start_frame < 0))
2363                 return -EINVAL;
2364         if (unlikely(flags & ~(CVMX_USB_ISOCHRONOUS_FLAGS_ALLOW_SHORT | CVMX_USB_ISOCHRONOUS_FLAGS_ASAP)))
2365                 return -EINVAL;
2366         if (unlikely(number_packets < 1))
2367                 return -EINVAL;
2368         if (unlikely(!packets))
2369                 return -EINVAL;
2370         if (unlikely(!buffer))
2371                 return -EINVAL;
2372         if (unlikely(buffer_length < 0))
2373                 return -EINVAL;
2374
2375         submit_handle = __cvmx_usb_submit_transaction(usb, pipe_handle,
2376                                                       CVMX_USB_TRANSFER_ISOCHRONOUS,
2377                                                       flags,
2378                                                       buffer,
2379                                                       buffer_length,
2380                                                       0, /* control_header */
2381                                                       start_frame,
2382                                                       number_packets,
2383                                                       packets,
2384                                                       callback,
2385                                                       user_data);
2386         return submit_handle;
2387 }
2388
2389
2390 /**
2391  * Cancel one outstanding request in a pipe. Canceling a request
2392  * can fail if the transaction has already completed before cancel
2393  * is called. Even after a successful cancel call, it may take
2394  * a frame or two for the cvmx_usb_poll() function to call the
2395  * associated callback.
2396  *
2397  * @param state  USB device state populated by
2398  *               cvmx_usb_initialize().
2399  * @param pipe_handle
2400  *               Pipe handle to cancel requests in.
2401  * @param submit_handle
2402  *               Handle to transaction to cancel, returned by the submit function.
2403  *
2404  * @return 0 or a negative error code.
2405  */
2406 int cvmx_usb_cancel(cvmx_usb_state_t *state, int pipe_handle, int submit_handle)
2407 {
2408         cvmx_usb_transaction_t *transaction;
2409         cvmx_usb_internal_state_t *usb = (cvmx_usb_internal_state_t *)state;
2410         cvmx_usb_pipe_t *pipe = usb->pipe + pipe_handle;
2411
2412         if (unlikely((pipe_handle < 0) || (pipe_handle >= MAX_PIPES)))
2413                 return -EINVAL;
2414         if (unlikely((submit_handle < 0) || (submit_handle >= MAX_TRANSACTIONS)))
2415                 return -EINVAL;
2416
2417         /* Fail if the pipe isn't open */
2418         if (unlikely((pipe->flags & __CVMX_USB_PIPE_FLAGS_OPEN) == 0))
2419                 return -EINVAL;
2420
2421         transaction = usb->transaction + submit_handle;
2422
2423         /* Fail if this transaction already completed */
2424         if (unlikely((transaction->flags & __CVMX_USB_TRANSACTION_FLAGS_IN_USE) == 0))
2425                 return -EINVAL;
2426
2427         /*
2428          * If the transaction is the HEAD of the queue and scheduled. We need to
2429          * treat it special
2430          */
2431         if ((pipe->head == transaction) &&
2432                 (pipe->flags & __CVMX_USB_PIPE_FLAGS_SCHEDULED)) {
2433                 cvmx_usbcx_hccharx_t usbc_hcchar;
2434
2435                 usb->pipe_for_channel[pipe->channel] = NULL;
2436                 pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_SCHEDULED;
2437
2438                 CVMX_SYNCW;
2439
2440                 usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCCHARX(pipe->channel, usb->index));
2441                 /* If the channel isn't enabled then the transaction already completed */
2442                 if (usbc_hcchar.s.chena) {
2443                         usbc_hcchar.s.chdis = 1;
2444                         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCCHARX(pipe->channel, usb->index), usbc_hcchar.u32);
2445                 }
2446         }
2447         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_CANCEL);
2448         return 0;
2449 }
2450
2451
2452 /**
2453  * Cancel all outstanding requests in a pipe. Logically all this
2454  * does is call cvmx_usb_cancel() in a loop.
2455  *
2456  * @param state  USB device state populated by
2457  *               cvmx_usb_initialize().
2458  * @param pipe_handle
2459  *               Pipe handle to cancel requests in.
2460  *
2461  * @return 0 or a negative error code.
2462  */
2463 int cvmx_usb_cancel_all(cvmx_usb_state_t *state, int pipe_handle)
2464 {
2465         cvmx_usb_internal_state_t *usb = (cvmx_usb_internal_state_t *)state;
2466         cvmx_usb_pipe_t *pipe = usb->pipe + pipe_handle;
2467
2468         if (unlikely((pipe_handle < 0) || (pipe_handle >= MAX_PIPES)))
2469                 return -EINVAL;
2470
2471         /* Fail if the pipe isn't open */
2472         if (unlikely((pipe->flags & __CVMX_USB_PIPE_FLAGS_OPEN) == 0))
2473                 return -EINVAL;
2474
2475         /* Simply loop through and attempt to cancel each transaction */
2476         while (pipe->head) {
2477                 int result = cvmx_usb_cancel(state, pipe_handle,
2478                         __cvmx_usb_get_submit_handle(usb, pipe->head));
2479                 if (unlikely(result != 0))
2480                         return result;
2481         }
2482         return 0;
2483 }
2484
2485
2486 /**
2487  * Close a pipe created with cvmx_usb_open_pipe().
2488  *
2489  * @param state  USB device state populated by
2490  *               cvmx_usb_initialize().
2491  * @param pipe_handle
2492  *               Pipe handle to close.
2493  *
2494  * @return 0 or a negative error code. EBUSY is returned if the pipe has
2495  *         outstanding transfers.
2496  */
2497 int cvmx_usb_close_pipe(cvmx_usb_state_t *state, int pipe_handle)
2498 {
2499         cvmx_usb_internal_state_t *usb = (cvmx_usb_internal_state_t *)state;
2500         cvmx_usb_pipe_t *pipe = usb->pipe + pipe_handle;
2501
2502         if (unlikely((pipe_handle < 0) || (pipe_handle >= MAX_PIPES)))
2503                 return -EINVAL;
2504
2505         /* Fail if the pipe isn't open */
2506         if (unlikely((pipe->flags & __CVMX_USB_PIPE_FLAGS_OPEN) == 0))
2507                 return -EINVAL;
2508
2509         /* Fail if the pipe has pending transactions */
2510         if (unlikely(pipe->head))
2511                 return -EBUSY;
2512
2513         pipe->flags = 0;
2514         __cvmx_usb_remove_pipe(&usb->idle_pipes, pipe);
2515         __cvmx_usb_append_pipe(&usb->free_pipes, pipe);
2516
2517         return 0;
2518 }
2519
2520
2521 /**
2522  * Register a function to be called when various USB events occur.
2523  *
2524  * @param state     USB device state populated by
2525  *                  cvmx_usb_initialize().
2526  * @param reason    Which event to register for.
2527  * @param callback  Function to call when the event occurs.
2528  * @param user_data User data parameter to the function.
2529  *
2530  * @return 0 or a negative error code.
2531  */
2532 int cvmx_usb_register_callback(cvmx_usb_state_t *state,
2533                                cvmx_usb_callback_t reason,
2534                                cvmx_usb_callback_func_t callback,
2535                                void *user_data)
2536 {
2537         cvmx_usb_internal_state_t *usb = (cvmx_usb_internal_state_t *)state;
2538
2539         if (unlikely(reason >= __CVMX_USB_CALLBACK_END))
2540                 return -EINVAL;
2541         if (unlikely(!callback))
2542                 return -EINVAL;
2543
2544         usb->callback[reason] = callback;
2545         usb->callback_data[reason] = user_data;
2546
2547         return 0;
2548 }
2549
2550
2551 /**
2552  * Get the current USB protocol level frame number. The frame
2553  * number is always in the range of 0-0x7ff.
2554  *
2555  * @param state  USB device state populated by
2556  *               cvmx_usb_initialize().
2557  *
2558  * @return USB frame number
2559  */
2560 int cvmx_usb_get_frame_number(cvmx_usb_state_t *state)
2561 {
2562         int frame_number;
2563         cvmx_usb_internal_state_t *usb = (cvmx_usb_internal_state_t *)state;
2564         cvmx_usbcx_hfnum_t usbc_hfnum;
2565
2566         usbc_hfnum.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
2567         frame_number = usbc_hfnum.s.frnum;
2568
2569         return frame_number;
2570 }
2571
2572
2573 /**
2574  * @INTERNAL
2575  * Poll a channel for status
2576  *
2577  * @param usb     USB device
2578  * @param channel Channel to poll
2579  *
2580  * @return Zero on success
2581  */
2582 static int __cvmx_usb_poll_channel(cvmx_usb_internal_state_t *usb, int channel)
2583 {
2584         cvmx_usbcx_hcintx_t usbc_hcint;
2585         cvmx_usbcx_hctsizx_t usbc_hctsiz;
2586         cvmx_usbcx_hccharx_t usbc_hcchar;
2587         cvmx_usb_pipe_t *pipe;
2588         cvmx_usb_transaction_t *transaction;
2589         int bytes_this_transfer;
2590         int bytes_in_last_packet;
2591         int packets_processed;
2592         int buffer_space_left;
2593
2594         /* Read the interrupt status bits for the channel */
2595         usbc_hcint.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCINTX(channel, usb->index));
2596
2597         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
2598                 usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index));
2599
2600                 if (usbc_hcchar.s.chena && usbc_hcchar.s.chdis) {
2601                         /*
2602                          * There seems to be a bug in CN31XX which can cause
2603                          * interrupt IN transfers to get stuck until we do a
2604                          * write of HCCHARX without changing things
2605                          */
2606                         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index), usbc_hcchar.u32);
2607                         return 0;
2608                 }
2609
2610                 /*
2611                  * In non DMA mode the channels don't halt themselves. We need
2612                  * to manually disable channels that are left running
2613                  */
2614                 if (!usbc_hcint.s.chhltd) {
2615                         if (usbc_hcchar.s.chena) {
2616                                 cvmx_usbcx_hcintmskx_t hcintmsk;
2617                                 /* Disable all interrupts except CHHLTD */
2618                                 hcintmsk.u32 = 0;
2619                                 hcintmsk.s.chhltdmsk = 1;
2620                                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), hcintmsk.u32);
2621                                 usbc_hcchar.s.chdis = 1;
2622                                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index), usbc_hcchar.u32);
2623                                 return 0;
2624                         } else if (usbc_hcint.s.xfercompl) {
2625                                 /* Successful IN/OUT with transfer complete. Channel halt isn't needed */
2626                         } else {
2627                                 cvmx_dprintf("USB%d: Channel %d interrupt without halt\n", usb->index, channel);
2628                                 return 0;
2629                         }
2630                 }
2631         } else {
2632                 /*
2633                  * There is are no interrupts that we need to process when the
2634                  * channel is still running
2635                  */
2636                 if (!usbc_hcint.s.chhltd)
2637                         return 0;
2638         }
2639
2640         /* Disable the channel interrupts now that it is done */
2641         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
2642         usb->idle_hardware_channels |= (1<<channel);
2643
2644         /* Make sure this channel is tied to a valid pipe */
2645         pipe = usb->pipe_for_channel[channel];
2646         CVMX_PREFETCH(pipe, 0);
2647         CVMX_PREFETCH(pipe, 128);
2648         if (!pipe)
2649                 return 0;
2650         transaction = pipe->head;
2651         CVMX_PREFETCH0(transaction);
2652
2653         /*
2654          * Disconnect this pipe from the HW channel. Later the schedule
2655          * function will figure out which pipe needs to go
2656          */
2657         usb->pipe_for_channel[channel] = NULL;
2658         pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_SCHEDULED;
2659
2660         /*
2661          * Read the channel config info so we can figure out how much data
2662          * transfered
2663          */
2664         usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCCHARX(channel, usb->index));
2665         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index));
2666
2667         /*
2668          * Calculating the number of bytes successfully transferred is dependent
2669          * on the transfer direction
2670          */
2671         packets_processed = transaction->pktcnt - usbc_hctsiz.s.pktcnt;
2672         if (usbc_hcchar.s.epdir) {
2673                 /*
2674                  * IN transactions are easy. For every byte received the
2675                  * hardware decrements xfersize. All we need to do is subtract
2676                  * the current value of xfersize from its starting value and we
2677                  * know how many bytes were written to the buffer
2678                  */
2679                 bytes_this_transfer = transaction->xfersize - usbc_hctsiz.s.xfersize;
2680         } else {
2681                 /*
2682                  * OUT transaction don't decrement xfersize. Instead pktcnt is
2683                  * decremented on every successful packet send. The hardware
2684                  * does this when it receives an ACK, or NYET. If it doesn't
2685                  * receive one of these responses pktcnt doesn't change
2686                  */
2687                 bytes_this_transfer = packets_processed * usbc_hcchar.s.mps;
2688                 /*
2689                  * The last packet may not be a full transfer if we didn't have
2690                  * enough data
2691                  */
2692                 if (bytes_this_transfer > transaction->xfersize)
2693                         bytes_this_transfer = transaction->xfersize;
2694         }
2695         /* Figure out how many bytes were in the last packet of the transfer */
2696         if (packets_processed)
2697                 bytes_in_last_packet = bytes_this_transfer - (packets_processed-1) * usbc_hcchar.s.mps;
2698         else
2699                 bytes_in_last_packet = bytes_this_transfer;
2700
2701         /*
2702          * As a special case, setup transactions output the setup header, not
2703          * the user's data. For this reason we don't count setup data as bytes
2704          * transferred
2705          */
2706         if ((transaction->stage == CVMX_USB_STAGE_SETUP) ||
2707                 (transaction->stage == CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE))
2708                 bytes_this_transfer = 0;
2709
2710         /*
2711          * Add the bytes transferred to the running total. It is important that
2712          * bytes_this_transfer doesn't count any data that needs to be
2713          * retransmitted
2714          */
2715         transaction->actual_bytes += bytes_this_transfer;
2716         if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
2717                 buffer_space_left = transaction->iso_packets[0].length - transaction->actual_bytes;
2718         else
2719                 buffer_space_left = transaction->buffer_length - transaction->actual_bytes;
2720
2721         /*
2722          * We need to remember the PID toggle state for the next transaction.
2723          * The hardware already updated it for the next transaction
2724          */
2725         pipe->pid_toggle = !(usbc_hctsiz.s.pid == 0);
2726
2727         /*
2728          * For high speed bulk out, assume the next transaction will need to do
2729          * a ping before proceeding. If this isn't true the ACK processing below
2730          * will clear this flag
2731          */
2732         if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2733                 (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
2734                 (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT))
2735                 pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
2736
2737         if (usbc_hcint.s.stall) {
2738                 /*
2739                  * STALL as a response means this transaction cannot be
2740                  * completed because the device can't process transactions. Tell
2741                  * the user. Any data that was transferred will be counted on
2742                  * the actual bytes transferred
2743                  */
2744                 pipe->pid_toggle = 0;
2745                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_STALL);
2746         } else if (usbc_hcint.s.xacterr) {
2747                 /*
2748                  * We know at least one packet worked if we get a ACK or NAK.
2749                  * Reset the retry counter
2750                  */
2751                 if (usbc_hcint.s.nak || usbc_hcint.s.ack)
2752                         transaction->retries = 0;
2753                 transaction->retries++;
2754                 if (transaction->retries > MAX_RETRIES) {
2755                         /*
2756                          * XactErr as a response means the device signaled
2757                          * something wrong with the transfer. For example, PID
2758                          * toggle errors cause these
2759                          */
2760                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_XACTERR);
2761                 } else {
2762                         /*
2763                          * If this was a split then clear our split in progress
2764                          * marker
2765                          */
2766                         if (usb->active_split == transaction)
2767                                 usb->active_split = NULL;
2768                         /*
2769                          * Rewind to the beginning of the transaction by anding
2770                          * off the split complete bit
2771                          */
2772                         transaction->stage &= ~1;
2773                         pipe->split_sc_frame = -1;
2774                         pipe->next_tx_frame += pipe->interval;
2775                         if (pipe->next_tx_frame < usb->frame_number)
2776                                 pipe->next_tx_frame = usb->frame_number + pipe->interval -
2777                                                       (usb->frame_number - pipe->next_tx_frame) % pipe->interval;
2778                 }
2779         } else if (usbc_hcint.s.bblerr) {
2780                 /* Babble Error (BblErr) */
2781                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_BABBLEERR);
2782         } else if (usbc_hcint.s.datatglerr) {
2783                 /* We'll retry the exact same transaction again */
2784                 transaction->retries++;
2785         } else if (usbc_hcint.s.nyet) {
2786                 /*
2787                  * NYET as a response is only allowed in three cases: as a
2788                  * response to a ping, as a response to a split transaction, and
2789                  * as a response to a bulk out. The ping case is handled by
2790                  * hardware, so we only have splits and bulk out
2791                  */
2792                 if (!__cvmx_usb_pipe_needs_split(usb, pipe)) {
2793                         transaction->retries = 0;
2794                         /*
2795                          * If there is more data to go then we need to try
2796                          * again. Otherwise this transaction is complete
2797                          */
2798                         if ((buffer_space_left == 0) || (bytes_in_last_packet < pipe->max_packet))
2799                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2800                 } else {
2801                         /*
2802                          * Split transactions retry the split complete 4 times
2803                          * then rewind to the start split and do the entire
2804                          * transactions again
2805                          */
2806                         transaction->retries++;
2807                         if ((transaction->retries & 0x3) == 0) {
2808                                 /*
2809                                  * Rewind to the beginning of the transaction by
2810                                  * anding off the split complete bit
2811                                  */
2812                                 transaction->stage &= ~1;
2813                                 pipe->split_sc_frame = -1;
2814                         }
2815                 }
2816         } else if (usbc_hcint.s.ack) {
2817                 transaction->retries = 0;
2818                 /*
2819                  * The ACK bit can only be checked after the other error bits.
2820                  * This is because a multi packet transfer may succeed in a
2821                  * number of packets and then get a different response on the
2822                  * last packet. In this case both ACK and the last response bit
2823                  * will be set. If none of the other response bits is set, then
2824                  * the last packet must have been an ACK
2825                  *
2826                  * Since we got an ACK, we know we don't need to do a ping on
2827                  * this pipe
2828                  */
2829                 pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_NEED_PING;
2830
2831                 switch (transaction->type) {
2832                 case CVMX_USB_TRANSFER_CONTROL:
2833                         switch (transaction->stage) {
2834                         case CVMX_USB_STAGE_NON_CONTROL:
2835                         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
2836                                 /* This should be impossible */
2837                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_ERROR);
2838                                 break;
2839                         case CVMX_USB_STAGE_SETUP:
2840                                 pipe->pid_toggle = 1;
2841                                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
2842                                         transaction->stage = CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE;
2843                                 else {
2844                                         cvmx_usb_control_header_t *header = cvmx_phys_to_ptr(transaction->control_header);
2845                                         if (header->s.length)
2846                                                 transaction->stage = CVMX_USB_STAGE_DATA;
2847                                         else
2848                                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2849                                 }
2850                                 break;
2851                         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
2852                                 {
2853                                         cvmx_usb_control_header_t *header = cvmx_phys_to_ptr(transaction->control_header);
2854                                         if (header->s.length)
2855                                                 transaction->stage = CVMX_USB_STAGE_DATA;
2856                                         else
2857                                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2858                                 }
2859                                 break;
2860                         case CVMX_USB_STAGE_DATA:
2861                                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2862                                         transaction->stage = CVMX_USB_STAGE_DATA_SPLIT_COMPLETE;
2863                                         /*
2864                                          * For setup OUT data that are splits,
2865                                          * the hardware doesn't appear to count
2866                                          * transferred data. Here we manually
2867                                          * update the data transferred
2868                                          */
2869                                         if (!usbc_hcchar.s.epdir) {
2870                                                 if (buffer_space_left < pipe->max_packet)
2871                                                         transaction->actual_bytes += buffer_space_left;
2872                                                 else
2873                                                         transaction->actual_bytes += pipe->max_packet;
2874                                         }
2875                                 } else if ((buffer_space_left == 0) || (bytes_in_last_packet < pipe->max_packet)) {
2876                                         pipe->pid_toggle = 1;
2877                                         transaction->stage = CVMX_USB_STAGE_STATUS;
2878                                 }
2879                                 break;
2880                         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
2881                                 if ((buffer_space_left == 0) || (bytes_in_last_packet < pipe->max_packet)) {
2882                                         pipe->pid_toggle = 1;
2883                                         transaction->stage = CVMX_USB_STAGE_STATUS;
2884                                 } else {
2885                                         transaction->stage = CVMX_USB_STAGE_DATA;
2886                                 }
2887                                 break;
2888                         case CVMX_USB_STAGE_STATUS:
2889                                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
2890                                         transaction->stage = CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE;
2891                                 else
2892                                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2893                                 break;
2894                         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
2895                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2896                                 break;
2897                         }
2898                         break;
2899                 case CVMX_USB_TRANSFER_BULK:
2900                 case CVMX_USB_TRANSFER_INTERRUPT:
2901                         /*
2902                          * The only time a bulk transfer isn't complete when it
2903                          * finishes with an ACK is during a split transaction.
2904                          * For splits we need to continue the transfer if more
2905                          * data is needed
2906                          */
2907                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2908                                 if (transaction->stage == CVMX_USB_STAGE_NON_CONTROL)
2909                                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2910                                 else {
2911                                         if (buffer_space_left && (bytes_in_last_packet == pipe->max_packet))
2912                                                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2913                                         else {
2914                                                 if (transaction->type == CVMX_USB_TRANSFER_INTERRUPT)
2915                                                         pipe->next_tx_frame += pipe->interval;
2916                                                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2917                                         }
2918                                 }
2919                         } else {
2920                                 if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2921                                     (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
2922                                     (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) &&
2923                                     (usbc_hcint.s.nak))
2924                                         pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
2925                                 if (!buffer_space_left || (bytes_in_last_packet < pipe->max_packet)) {
2926                                         if (transaction->type == CVMX_USB_TRANSFER_INTERRUPT)
2927                                                 pipe->next_tx_frame += pipe->interval;
2928                                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2929                                 }
2930                         }
2931                         break;
2932                 case CVMX_USB_TRANSFER_ISOCHRONOUS:
2933                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2934                                 /*
2935                                  * ISOCHRONOUS OUT splits don't require a
2936                                  * complete split stage. Instead they use a
2937                                  * sequence of begin OUT splits to transfer the
2938                                  * data 188 bytes at a time. Once the transfer
2939                                  * is complete, the pipe sleeps until the next
2940                                  * schedule interval
2941                                  */
2942                                 if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
2943                                         /*
2944                                          * If no space left or this wasn't a max
2945                                          * size packet then this transfer is
2946                                          * complete. Otherwise start it again to
2947                                          * send the next 188 bytes
2948                                          */
2949                                         if (!buffer_space_left || (bytes_this_transfer < 188)) {
2950                                                 pipe->next_tx_frame += pipe->interval;
2951                                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2952                                         }
2953                                 } else {
2954                                         if (transaction->stage == CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE) {
2955                                                 /*
2956                                                  * We are in the incoming data
2957                                                  * phase. Keep getting data
2958                                                  * until we run out of space or
2959                                                  * get a small packet
2960                                                  */
2961                                                 if ((buffer_space_left == 0) || (bytes_in_last_packet < pipe->max_packet)) {
2962                                                         pipe->next_tx_frame += pipe->interval;
2963                                                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2964                                                 }
2965                                         } else
2966                                                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2967                                 }
2968                         } else {
2969                                 pipe->next_tx_frame += pipe->interval;
2970                                 __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_SUCCESS);
2971                         }
2972                         break;
2973                 }
2974         } else if (usbc_hcint.s.nak) {
2975                 /* If this was a split then clear our split in progress marker */
2976                 if (usb->active_split == transaction)
2977                         usb->active_split = NULL;
2978                 /*
2979                  * NAK as a response means the device couldn't accept the
2980                  * transaction, but it should be retried in the future. Rewind
2981                  * to the beginning of the transaction by anding off the split
2982                  * complete bit. Retry in the next interval
2983                  */
2984                 transaction->retries = 0;
2985                 transaction->stage &= ~1;
2986                 pipe->next_tx_frame += pipe->interval;
2987                 if (pipe->next_tx_frame < usb->frame_number)
2988                         pipe->next_tx_frame = usb->frame_number + pipe->interval -
2989                                 (usb->frame_number - pipe->next_tx_frame) % pipe->interval;
2990         } else {
2991                 cvmx_usb_port_status_t port;
2992                 port = cvmx_usb_get_status((cvmx_usb_state_t *)usb);
2993                 if (port.port_enabled) {
2994                         /* We'll retry the exact same transaction again */
2995                         transaction->retries++;
2996                 } else {
2997                         /*
2998                          * We get channel halted interrupts with no result bits
2999                          * sets when the cable is unplugged
3000                          */
3001                         __cvmx_usb_perform_complete(usb, pipe, transaction, CVMX_USB_COMPLETE_ERROR);
3002                 }
3003         }
3004         return 0;
3005 }
3006
3007
3008 /**
3009  * Poll the USB block for status and call all needed callback
3010  * handlers. This function is meant to be called in the interrupt
3011  * handler for the USB controller. It can also be called
3012  * periodically in a loop for non-interrupt based operation.
3013  *
3014  * @param state USB device state populated by
3015  *              cvmx_usb_initialize().
3016  *
3017  * @return 0 or a negative error code.
3018  */
3019 int cvmx_usb_poll(cvmx_usb_state_t *state)
3020 {
3021         cvmx_usbcx_hfnum_t usbc_hfnum;
3022         cvmx_usbcx_gintsts_t usbc_gintsts;
3023         cvmx_usb_internal_state_t *usb = (cvmx_usb_internal_state_t *)state;
3024
3025         CVMX_PREFETCH(usb, 0);
3026         CVMX_PREFETCH(usb, 1*128);
3027         CVMX_PREFETCH(usb, 2*128);
3028         CVMX_PREFETCH(usb, 3*128);
3029         CVMX_PREFETCH(usb, 4*128);
3030
3031         /* Update the frame counter */
3032         usbc_hfnum.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
3033         if ((usb->frame_number&0x3fff) > usbc_hfnum.s.frnum)
3034                 usb->frame_number += 0x4000;
3035         usb->frame_number &= ~0x3fffull;
3036         usb->frame_number |= usbc_hfnum.s.frnum;
3037
3038         /* Read the pending interrupts */
3039         usbc_gintsts.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_GINTSTS(usb->index));
3040
3041         /* Clear the interrupts now that we know about them */
3042         __cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTSTS(usb->index), usbc_gintsts.u32);
3043
3044         if (usbc_gintsts.s.rxflvl) {
3045                 /*
3046                  * RxFIFO Non-Empty (RxFLvl)
3047                  * Indicates that there is at least one packet pending to be
3048                  * read from the RxFIFO.
3049                  *
3050                  * In DMA mode this is handled by hardware
3051                  */
3052                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
3053                         __cvmx_usb_poll_rx_fifo(usb);
3054         }
3055         if (usbc_gintsts.s.ptxfemp || usbc_gintsts.s.nptxfemp) {
3056                 /* Fill the Tx FIFOs when not in DMA mode */
3057                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
3058                         __cvmx_usb_poll_tx_fifo(usb);
3059         }
3060         if (usbc_gintsts.s.disconnint || usbc_gintsts.s.prtint) {
3061                 cvmx_usbcx_hprt_t usbc_hprt;
3062                 /*
3063                  * Disconnect Detected Interrupt (DisconnInt)
3064                  * Asserted when a device disconnect is detected.
3065                  *
3066                  * Host Port Interrupt (PrtInt)
3067                  * The core sets this bit to indicate a change in port status of
3068                  * one of the O2P USB core ports in Host mode. The application
3069                  * must read the Host Port Control and Status (HPRT) register to
3070                  * determine the exact event that caused this interrupt. The
3071                  * application must clear the appropriate status bit in the Host
3072                  * Port Control and Status register to clear this bit.
3073                  *
3074                  * Call the user's port callback
3075                  */
3076                 __cvmx_usb_perform_callback(usb, NULL, NULL,
3077                                             CVMX_USB_CALLBACK_PORT_CHANGED,
3078                                             CVMX_USB_COMPLETE_SUCCESS);
3079                 /* Clear the port change bits */
3080                 usbc_hprt.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
3081                 usbc_hprt.s.prtena = 0;
3082                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HPRT(usb->index), usbc_hprt.u32);
3083         }
3084         if (usbc_gintsts.s.hchint) {
3085                 /*
3086                  * Host Channels Interrupt (HChInt)
3087                  * The core sets this bit to indicate that an interrupt is
3088                  * pending on one of the channels of the core (in Host mode).
3089                  * The application must read the Host All Channels Interrupt
3090                  * (HAINT) register to determine the exact number of the channel
3091                  * on which the interrupt occurred, and then read the
3092                  * corresponding Host Channel-n Interrupt (HCINTn) register to
3093                  * determine the exact cause of the interrupt. The application
3094                  * must clear the appropriate status bit in the HCINTn register
3095                  * to clear this bit.
3096                  */
3097                 cvmx_usbcx_haint_t usbc_haint;
3098                 usbc_haint.u32 = __cvmx_usb_read_csr32(usb, CVMX_USBCX_HAINT(usb->index));
3099                 while (usbc_haint.u32) {
3100                         int channel;
3101                         CVMX_CLZ(channel, usbc_haint.u32);
3102                         channel = 31 - channel;
3103                         __cvmx_usb_poll_channel(usb, channel);
3104                         usbc_haint.u32 ^= 1<<channel;
3105                 }
3106         }
3107
3108         __cvmx_usb_schedule(usb, usbc_gintsts.s.sof);
3109
3110         return 0;
3111 }