drivers/staging/octeon-usb/octeon-hcd.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 2008 Cavium Networks
   7  *
   8  * Some parts of the code were originally released under BSD license:
   9  *
  10  * Copyright (c) 2003-2010 Cavium Networks (support@cavium.com). All rights
  11  * reserved.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions are
  15  * met:
  16  *
  17  *   * Redistributions of source code must retain the above copyright
  18  *     notice, this list of conditions and the following disclaimer.
  19  *
  20  *   * Redistributions in binary form must reproduce the above
  21  *     copyright notice, this list of conditions and the following
  22  *     disclaimer in the documentation and/or other materials provided
  23  *     with the distribution.
  24  *
  25  *   * Neither the name of Cavium Networks nor the names of
  26  *     its contributors may be used to endorse or promote products
  27  *     derived from this software without specific prior written
  28  *     permission.
  29  *
  30  * This Software, including technical data, may be subject to U.S. export
  31  * control laws, including the U.S. Export Administration Act and its associated
  32  * regulations, and may be subject to export or import regulations in other
  33  * countries.
  34  *
  35  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
  36  * AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR
  37  * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
  38  * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION
  39  * OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
  40  * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
  41  * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
  42  * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
  43  * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
  44  * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
  45  */
  46 #include <linux/kernel.h>
  47 #include <linux/module.h>
  48 #include <linux/init.h>
  49 #include <linux/pci.h>
  50 #include <linux/prefetch.h>
  51 #include <linux/interrupt.h>
  52 #include <linux/platform_device.h>
  53 #include <linux/usb.h>
  54
  55 #include <linux/time.h>
  56 #include <linux/delay.h>
  57
  58 #include <asm/octeon/cvmx.h>
  59 #include <asm/octeon/cvmx-iob-defs.h>
  60
  61 #include <linux/usb/hcd.h>
  62
  63 #include <linux/err.h>
  64
  65 #include <asm/octeon/octeon.h>
  66 #include <asm/octeon/cvmx-helper.h>
  67 #include <asm/octeon/cvmx-sysinfo.h>
  68 #include <asm/octeon/cvmx-helper-board.h>
  69
  70 #include "octeon-hcd.h"
  71
  72 /**
  73  * enum cvmx_usb_speed - the possible USB device speeds
  74  *
  75  * @CVMX_USB_SPEED_HIGH: Device is operation at 480Mbps
  76  * @CVMX_USB_SPEED_FULL: Device is operation at 12Mbps
  77  * @CVMX_USB_SPEED_LOW:  Device is operation at 1.5Mbps
  78  */
  79 enum cvmx_usb_speed {
  80         CVMX_USB_SPEED_HIGH = 0,
  81         CVMX_USB_SPEED_FULL = 1,
  82         CVMX_USB_SPEED_LOW = 2,
  83 };
  84
  85 /**
  86  * enum cvmx_usb_transfer - the possible USB transfer types
  87  *
  88  * @CVMX_USB_TRANSFER_CONTROL:     USB transfer type control for hub and status
  89  *                                 transfers
  90  * @CVMX_USB_TRANSFER_ISOCHRONOUS: USB transfer type isochronous for low
  91  *                                 priority periodic transfers
  92  * @CVMX_USB_TRANSFER_BULK:        USB transfer type bulk for large low priority
  93  *                                 transfers
  94  * @CVMX_USB_TRANSFER_INTERRUPT:   USB transfer type interrupt for high priority
  95  *                                 periodic transfers
  96  */
  97 enum cvmx_usb_transfer {
  98         CVMX_USB_TRANSFER_CONTROL = 0,
  99         CVMX_USB_TRANSFER_ISOCHRONOUS = 1,
 100         CVMX_USB_TRANSFER_BULK = 2,
 101         CVMX_USB_TRANSFER_INTERRUPT = 3,
 102 };
 103
 104 /**
 105  * enum cvmx_usb_direction - the transfer directions
 106  *
 107  * @CVMX_USB_DIRECTION_OUT: Data is transferring from Octeon to the device/host
 108  * @CVMX_USB_DIRECTION_IN:  Data is transferring from the device/host to Octeon
 109  */
 110 enum cvmx_usb_direction {
 111         CVMX_USB_DIRECTION_OUT,
 112         CVMX_USB_DIRECTION_IN,
 113 };
 114
 115 /**
 116  * enum cvmx_usb_complete - possible callback function status codes
 117  *
 118  * @CVMX_USB_COMPLETE_SUCCESS:    The transaction / operation finished without
 119  *                                any errors
 120  * @CVMX_USB_COMPLETE_SHORT:      FIXME: This is currently not implemented
 121  * @CVMX_USB_COMPLETE_CANCEL:     The transaction was canceled while in flight
 122  *                                by a user call to cvmx_usb_cancel
 123  * @CVMX_USB_COMPLETE_ERROR:      The transaction aborted with an unexpected
 124  *                                error status
 125  * @CVMX_USB_COMPLETE_STALL:      The transaction received a USB STALL response
 126  *                                from the device
 127  * @CVMX_USB_COMPLETE_XACTERR:    The transaction failed with an error from the
 128  *                                device even after a number of retries
 129  * @CVMX_USB_COMPLETE_DATATGLERR: The transaction failed with a data toggle
 130  *                                error even after a number of retries
 131  * @CVMX_USB_COMPLETE_BABBLEERR:  The transaction failed with a babble error
 132  * @CVMX_USB_COMPLETE_FRAMEERR:   The transaction failed with a frame error
 133  *                                even after a number of retries
 134  */
 135 enum cvmx_usb_complete {
 136         CVMX_USB_COMPLETE_SUCCESS,
 137         CVMX_USB_COMPLETE_SHORT,
 138         CVMX_USB_COMPLETE_CANCEL,
 139         CVMX_USB_COMPLETE_ERROR,
 140         CVMX_USB_COMPLETE_STALL,
 141         CVMX_USB_COMPLETE_XACTERR,
 142         CVMX_USB_COMPLETE_DATATGLERR,
 143         CVMX_USB_COMPLETE_BABBLEERR,
 144         CVMX_USB_COMPLETE_FRAMEERR,
 145 };
 146
 147 /**
 148  * struct cvmx_usb_port_status - the USB port status information
 149  *
 150  * @port_enabled:       1 = Usb port is enabled, 0 = disabled
 151  * @port_over_current:  1 = Over current detected, 0 = Over current not
 152  *                      detected. Octeon doesn't support over current detection.
 153  * @port_powered:       1 = Port power is being supplied to the device, 0 =
 154  *                      power is off. Octeon doesn't support turning port power
 155  *                      off.
 156  * @port_speed:         Current port speed.
 157  * @connected:          1 = A device is connected to the port, 0 = No device is
 158  *                      connected.
 159  * @connect_change:     1 = Device connected state changed since the last set
 160  *                      status call.
 161  */
 162 struct cvmx_usb_port_status {
 163         uint32_t reserved               : 25;
 164         uint32_t port_enabled           : 1;
 165         uint32_t port_over_current      : 1;
 166         uint32_t port_powered           : 1;
 167         enum cvmx_usb_speed port_speed  : 2;
 168         uint32_t connected              : 1;
 169         uint32_t connect_change         : 1;
 170 };
 171
 172 /**
 173  * union cvmx_usb_control_header - the structure of a Control packet header
 174  *
 175  * @s.request_type:     Bit 7 tells the direction: 1=IN, 0=OUT
 176  * @s.request           The standard usb request to make
 177  * @s.value             Value parameter for the request in little endian format
 178  * @s.index             Index for the request in little endian format
 179  * @s.length            Length of the data associated with this request in
 180  *                      little endian format
 181  */
 182 union cvmx_usb_control_header {
 183         uint64_t u64;
 184         struct {
 185                 uint64_t request_type   : 8;
 186                 uint64_t request        : 8;
 187                 uint64_t value          : 16;
 188                 uint64_t index          : 16;
 189                 uint64_t length         : 16;
 190         } s;
 191 };
 192
 193 /**
 194  * struct cvmx_usb_iso_packet - descriptor for Isochronous packets
 195  *
 196  * @offset:     This is the offset in bytes into the main buffer where this data
 197  *              is stored.
 198  * @length:     This is the length in bytes of the data.
 199  * @status:     This is the status of this individual packet transfer.
 200  */
 201 struct cvmx_usb_iso_packet {
 202         int offset;
 203         int length;
 204         enum cvmx_usb_complete status;
 205 };
 206
 207 /**
 208  * enum cvmx_usb_initialize_flags - flags used by the initialization function
 209  *
 210  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI:    The USB port uses a 12MHz crystal
 211  *                                            as clock source at USB_XO and
 212  *                                            USB_XI.
 213  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND:   The USB port uses 12/24/48MHz 2.5V
 214  *                                            board clock source at USB_XO.
 215  *                                            USB_XI should be tied to GND.
 216  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK: Mask for clock speed field
 217  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:    Speed of reference clock or
 218  *                                            crystal
 219  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:    Speed of reference clock
 220  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:    Speed of reference clock
 221  * @CVMX_USB_INITIALIZE_FLAGS_NO_DMA:         Disable DMA and used polled IO for
 222  *                                            data transfer use for the USB
 223  */
 224 enum cvmx_usb_initialize_flags {
 225         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI           = 1 << 0,
 226         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND          = 1 << 1,
 227         CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK        = 3 << 3,
 228         CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ           = 1 << 3,
 229         CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ           = 2 << 3,
 230         CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ           = 3 << 3,
 231         /* Bits 3-4 used to encode the clock frequency */
 232         CVMX_USB_INITIALIZE_FLAGS_NO_DMA                = 1 << 5,
 233 };
 234
 235 /**
 236  * enum cvmx_usb_pipe_flags - internal flags for a pipe.
 237  *
 238  * @__CVMX_USB_PIPE_FLAGS_SCHEDULED: Used internally to determine if a pipe is
 239  *                                   actively using hardware. Do not use.
 240  * @__CVMX_USB_PIPE_FLAGS_NEED_PING: Used internally to determine if a high
 241  *                                   speed pipe is in the ping state. Do not
 242  *                                   use.
 243  */
 244 enum cvmx_usb_pipe_flags {
 245         __CVMX_USB_PIPE_FLAGS_SCHEDULED = 1 << 17,
 246         __CVMX_USB_PIPE_FLAGS_NEED_PING = 1 << 18,
 247 };
 248
 249 /* Maximum number of times to retry failed transactions */
 250 #define MAX_RETRIES             3
 251
 252 /* Maximum number of hardware channels supported by the USB block */
 253 #define MAX_CHANNELS            8
 254
 255 /* The highest valid USB device address */
 256 #define MAX_USB_ADDRESS         127
 257
 258 /* The highest valid USB endpoint number */
 259 #define MAX_USB_ENDPOINT        15
 260
 261 /* The highest valid port number on a hub */
 262 #define MAX_USB_HUB_PORT        15
 263
 264 /*
 265  * The low level hardware can transfer a maximum of this number of bytes in each
 266  * transfer. The field is 19 bits wide
 267  */
 268 #define MAX_TRANSFER_BYTES      ((1<<19)-1)
 269
 270 /*
 271  * The low level hardware can transfer a maximum of this number of packets in
 272  * each transfer. The field is 10 bits wide
 273  */
 274 #define MAX_TRANSFER_PACKETS    ((1<<10)-1)
 275
 276 /**
 277  * Logical transactions may take numerous low level
 278  * transactions, especially when splits are concerned. This
 279  * enum represents all of the possible stages a transaction can
 280  * be in. Note that split completes are always even. This is so
 281  * the NAK handler can backup to the previous low level
 282  * transaction with a simple clearing of bit 0.
 283  */
 284 enum cvmx_usb_stage {
 285         CVMX_USB_STAGE_NON_CONTROL,
 286         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE,
 287         CVMX_USB_STAGE_SETUP,
 288         CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE,
 289         CVMX_USB_STAGE_DATA,
 290         CVMX_USB_STAGE_DATA_SPLIT_COMPLETE,
 291         CVMX_USB_STAGE_STATUS,
 292         CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE,
 293 };
 294
 295 /**
 296  * struct cvmx_usb_transaction - describes each pending USB transaction
 297  *                               regardless of type. These are linked together
 298  *                               to form a list of pending requests for a pipe.
 299  *
 300  * @node:               List node for transactions in the pipe.
 301  * @type:               Type of transaction, duplicated of the pipe.
 302  * @flags:              State flags for this transaction.
 303  * @buffer:             User's physical buffer address to read/write.
 304  * @buffer_length:      Size of the user's buffer in bytes.
 305  * @control_header:     For control transactions, physical address of the 8
 306  *                      byte standard header.
 307  * @iso_start_frame:    For ISO transactions, the starting frame number.
 308  * @iso_number_packets: For ISO transactions, the number of packets in the
 309  *                      request.
 310  * @iso_packets:        For ISO transactions, the sub packets in the request.
 311  * @actual_bytes:       Actual bytes transfer for this transaction.
 312  * @stage:              For control transactions, the current stage.
 313  * @urb:                URB.
 314  */
 315 struct cvmx_usb_transaction {
 316         struct list_head node;
 317         enum cvmx_usb_transfer type;
 318         uint64_t buffer;
 319         int buffer_length;
 320         uint64_t control_header;
 321         int iso_start_frame;
 322         int iso_number_packets;
 323         struct cvmx_usb_iso_packet *iso_packets;
 324         int xfersize;
 325         int pktcnt;
 326         int retries;
 327         int actual_bytes;
 328         enum cvmx_usb_stage stage;
 329         struct urb *urb;
 330 };
 331
 332 /**
 333  * struct cvmx_usb_pipe - a pipe represents a virtual connection between Octeon
 334  *                        and some USB device. It contains a list of pending
 335  *                        request to the device.
 336  *
 337  * @node:               List node for pipe list
 338  * @next:               Pipe after this one in the list
 339  * @transactions:       List of pending transactions
 340  * @interval:           For periodic pipes, the interval between packets in
 341  *                      frames
 342  * @next_tx_frame:      The next frame this pipe is allowed to transmit on
 343  * @flags:              State flags for this pipe
 344  * @device_speed:       Speed of device connected to this pipe
 345  * @transfer_type:      Type of transaction supported by this pipe
 346  * @transfer_dir:       IN or OUT. Ignored for Control
 347  * @multi_count:        Max packet in a row for the device
 348  * @max_packet:         The device's maximum packet size in bytes
 349  * @device_addr:        USB device address at other end of pipe
 350  * @endpoint_num:       USB endpoint number at other end of pipe
 351  * @hub_device_addr:    Hub address this device is connected to
 352  * @hub_port:           Hub port this device is connected to
 353  * @pid_toggle:         This toggles between 0/1 on every packet send to track
 354  *                      the data pid needed
 355  * @channel:            Hardware DMA channel for this pipe
 356  * @split_sc_frame:     The low order bits of the frame number the split
 357  *                      complete should be sent on
 358  */
 359 struct cvmx_usb_pipe {
 360         struct list_head node;
 361         struct list_head transactions;
 362         uint64_t interval;
 363         uint64_t next_tx_frame;
 364         enum cvmx_usb_pipe_flags flags;
 365         enum cvmx_usb_speed device_speed;
 366         enum cvmx_usb_transfer transfer_type;
 367         enum cvmx_usb_direction transfer_dir;
 368         int multi_count;
 369         uint16_t max_packet;
 370         uint8_t device_addr;
 371         uint8_t endpoint_num;
 372         uint8_t hub_device_addr;
 373         uint8_t hub_port;
 374         uint8_t pid_toggle;
 375         uint8_t channel;
 376         int8_t split_sc_frame;
 377 };
 378
 379 struct cvmx_usb_tx_fifo {
 380         struct {
 381                 int channel;
 382                 int size;
 383                 uint64_t address;
 384         } entry[MAX_CHANNELS+1];
 385         int head;
 386         int tail;
 387 };
 388
 389 /**
 390  * struct cvmx_usb_state - the state of the USB block
 391  *
 392  * init_flags:             Flags passed to initialize.
 393  * index:                  Which USB block this is for.
 394  * idle_hardware_channels: Bit set for every idle hardware channel.
 395  * usbcx_hprt:             Stored port status so we don't need to read a CSR to
 396  *                         determine splits.
 397  * pipe_for_channel:       Map channels to pipes.
 398  * pipe:                   Storage for pipes.
 399  * indent:                 Used by debug output to indent functions.
 400  * port_status:            Last port status used for change notification.
 401  * idle_pipes:             List of open pipes that have no transactions.
 402  * active_pipes:           Active pipes indexed by transfer type.
 403  * frame_number:           Increments every SOF interrupt for time keeping.
 404  * active_split:           Points to the current active split, or NULL.
 405  */
 406 struct cvmx_usb_state {
 407         int init_flags;
 408         int index;
 409         int idle_hardware_channels;
 410         union cvmx_usbcx_hprt usbcx_hprt;
 411         struct cvmx_usb_pipe *pipe_for_channel[MAX_CHANNELS];
 412         int indent;
 413         struct cvmx_usb_port_status port_status;
 414         struct list_head idle_pipes;
 415         struct list_head active_pipes[4];
 416         uint64_t frame_number;
 417         struct cvmx_usb_transaction *active_split;
 418         struct cvmx_usb_tx_fifo periodic;
 419         struct cvmx_usb_tx_fifo nonperiodic;
 420 };
 421
 422 struct octeon_hcd {
 423         spinlock_t lock;
 424         struct cvmx_usb_state usb;
 425         struct tasklet_struct dequeue_tasklet;
 426         struct list_head dequeue_list;
 427 };
 428
 429 /* This macro spins on a field waiting for it to reach a value */
 430 #define CVMX_WAIT_FOR_FIELD32(address, type, field, op, value, timeout_usec)\
 431         ({int result;                                                       \
 432         do {                                                                \
 433                 uint64_t done = cvmx_get_cycle() + (uint64_t)timeout_usec * \
 434                         octeon_get_clock_rate() / 1000000;                  \
 435                 type c;                                                     \
 436                 while (1) {                                                 \
 437                         c.u32 = __cvmx_usb_read_csr32(usb, address);        \
 438                         if (c.s.field op (value)) {                         \
 439                                 result = 0;                                 \
 440                                 break;                                      \
 441                         } else if (cvmx_get_cycle() > done) {               \
 442                                 result = -1;                                \
 443                                 break;                                      \
 444                         } else                                              \
 445                                 cvmx_wait(100);                             \
 446                 }                                                           \
 447         } while (0);                                                        \
 448         result; })
 449
 450 /*
 451  * This macro logically sets a single field in a CSR. It does the sequence
 452  * read, modify, and write
 453  */
 454 #define USB_SET_FIELD32(address, type, field, value)            \
 455         do {                                                    \
 456                 type c;                                         \
 457                 c.u32 = __cvmx_usb_read_csr32(usb, address);    \
 458                 c.s.field = value;                              \
 459                 __cvmx_usb_write_csr32(usb, address, c.u32);    \
 460         } while (0)
 461
 462 /* Returns the IO address to push/pop stuff data from the FIFOs */
 463 #define USB_FIFO_ADDRESS(channel, usb_index) \
 464         (CVMX_USBCX_GOTGCTL(usb_index) + ((channel)+1)*0x1000)
 465
 466 /**
 467  * struct octeon_temp_buffer - a bounce buffer for USB transfers
 468  * @temp_buffer: the newly allocated temporary buffer (including meta-data)
 469  * @orig_buffer: the original buffer passed by the USB stack
 470  * @data:        the newly allocated temporary buffer (excluding meta-data)
 471  *
 472  * Both the DMA engine and FIFO mode will always transfer full 32-bit words. If
 473  * the buffer is too short, we need to allocate a temporary one, and this struct
 474  * represents it.
 475  */
 476 struct octeon_temp_buffer {
 477         void *temp_buffer;
 478         void *orig_buffer;
 479         u8 data[0];
 480 };
 481
 482 /**
 483  * octeon_alloc_temp_buffer - allocate a temporary buffer for USB transfer
 484  *                            (if needed)
 485  * @urb:        URB.
 486  * @mem_flags:  Memory allocation flags.
 487  *
 488  * This function allocates a temporary bounce buffer whenever it's needed
 489  * due to HW limitations.
 490  */
 491 static int octeon_alloc_temp_buffer(struct urb *urb, gfp_t mem_flags)
 492 {
 493         struct octeon_temp_buffer *temp;
 494
 495         if (urb->num_sgs || urb->sg ||
 496             (urb->transfer_flags & URB_NO_TRANSFER_DMA_MAP) ||
 497             !(urb->transfer_buffer_length % sizeof(u32)))
 498                 return 0;
 499
 500         temp = kmalloc(ALIGN(urb->transfer_buffer_length, sizeof(u32)) +
 501                        sizeof(*temp), mem_flags);
 502         if (!temp)
 503                 return -ENOMEM;
 504
 505         temp->temp_buffer = temp;
 506         temp->orig_buffer = urb->transfer_buffer;
 507         if (usb_urb_dir_out(urb))
 508                 memcpy(temp->data, urb->transfer_buffer,
 509                        urb->transfer_buffer_length);
 510         urb->transfer_buffer = temp->data;
 511         urb->transfer_flags |= URB_ALIGNED_TEMP_BUFFER;
 512
 513         return 0;
 514 }
 515
 516 /**
 517  * octeon_free_temp_buffer - free a temporary buffer used by USB transfers.
 518  * @urb: URB.
 519  *
 520  * Frees a buffer allocated by octeon_alloc_temp_buffer().
 521  */
 522 static void octeon_free_temp_buffer(struct urb *urb)
 523 {
 524         struct octeon_temp_buffer *temp;
 525
 526         if (!(urb->transfer_flags & URB_ALIGNED_TEMP_BUFFER))
 527                 return;
 528
 529         temp = container_of(urb->transfer_buffer, struct octeon_temp_buffer,
 530                             data);
 531         if (usb_urb_dir_in(urb))
 532                 memcpy(temp->orig_buffer, urb->transfer_buffer,
 533                        urb->actual_length);
 534         urb->transfer_buffer = temp->orig_buffer;
 535         urb->transfer_flags &= ~URB_ALIGNED_TEMP_BUFFER;
 536         kfree(temp->temp_buffer);
 537 }
 538
 539 /**
 540  * octeon_map_urb_for_dma - Octeon-specific map_urb_for_dma().
 541  * @hcd:        USB HCD structure.
 542  * @urb:        URB.
 543  * @mem_flags:  Memory allocation flags.
 544  */
 545 static int octeon_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
 546                                   gfp_t mem_flags)
 547 {
 548         int ret;
 549
 550         ret = octeon_alloc_temp_buffer(urb, mem_flags);
 551         if (ret)
 552                 return ret;
 553
 554         ret = usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);
 555         if (ret)
 556                 octeon_free_temp_buffer(urb);
 557
 558         return ret;
 559 }
 560
 561 /**
 562  * octeon_unmap_urb_for_dma - Octeon-specific unmap_urb_for_dma()
 563  * @hcd:        USB HCD structure.
 564  * @urb:        URB.
 565  */
 566 static void octeon_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)
 567 {
 568         usb_hcd_unmap_urb_for_dma(hcd, urb);
 569         octeon_free_temp_buffer(urb);
 570 }
 571
 572 /**
 573  * Read a USB 32bit CSR. It performs the necessary address swizzle
 574  * for 32bit CSRs and logs the value in a readable format if
 575  * debugging is on.
 576  *
 577  * @usb:     USB block this access is for
 578  * @address: 64bit address to read
 579  *
 580  * Returns: Result of the read
 581  */
 582 static inline uint32_t __cvmx_usb_read_csr32(struct cvmx_usb_state *usb,
 583                                              uint64_t address)
 584 {
 585         uint32_t result = cvmx_read64_uint32(address ^ 4);
 586         return result;
 587 }
 588
 589
 590 /**
 591  * Write a USB 32bit CSR. It performs the necessary address
 592  * swizzle for 32bit CSRs and logs the value in a readable format
 593  * if debugging is on.
 594  *
 595  * @usb:     USB block this access is for
 596  * @address: 64bit address to write
 597  * @value:   Value to write
 598  */
 599 static inline void __cvmx_usb_write_csr32(struct cvmx_usb_state *usb,
 600                                           uint64_t address, uint32_t value)
 601 {
 602         cvmx_write64_uint32(address ^ 4, value);
 603         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
 604 }
 605
 606
 607 /**
 608  * Read a USB 64bit CSR. It logs the value in a readable format if
 609  * debugging is on.
 610  *
 611  * @usb:     USB block this access is for
 612  * @address: 64bit address to read
 613  *
 614  * Returns: Result of the read
 615  */
 616 static inline uint64_t __cvmx_usb_read_csr64(struct cvmx_usb_state *usb,
 617                                              uint64_t address)
 618 {
 619         uint64_t result = cvmx_read64_uint64(address);
 620         return result;
 621 }
 622
 623
 624 /**
 625  * Write a USB 64bit CSR. It logs the value in a readable format
 626  * if debugging is on.
 627  *
 628  * @usb:     USB block this access is for
 629  * @address: 64bit address to write
 630  * @value:   Value to write
 631  */
 632 static inline void __cvmx_usb_write_csr64(struct cvmx_usb_state *usb,
 633                                           uint64_t address, uint64_t value)
 634 {
 635         cvmx_write64_uint64(address, value);
 636 }
 637
 638 /**
 639  * Return non zero if this pipe connects to a non HIGH speed
 640  * device through a high speed hub.
 641  *
 642  * @usb:    USB block this access is for
 643  * @pipe:   Pipe to check
 644  *
 645  * Returns: Non zero if we need to do split transactions
 646  */
 647 static inline int __cvmx_usb_pipe_needs_split(struct cvmx_usb_state *usb,
 648                                               struct cvmx_usb_pipe *pipe)
 649 {
 650         return pipe->device_speed != CVMX_USB_SPEED_HIGH &&
 651                usb->usbcx_hprt.s.prtspd == CVMX_USB_SPEED_HIGH;
 652 }
 653
 654
 655 /**
 656  * Trivial utility function to return the correct PID for a pipe
 657  *
 658  * @pipe:   pipe to check
 659  *
 660  * Returns: PID for pipe
 661  */
 662 static inline int __cvmx_usb_get_data_pid(struct cvmx_usb_pipe *pipe)
 663 {
 664         if (pipe->pid_toggle)
 665                 return 2; /* Data1 */
 666         else
 667                 return 0; /* Data0 */
 668 }
 669
 670 /**
 671  * Initialize a USB port for use. This must be called before any
 672  * other access to the Octeon USB port is made. The port starts
 673  * off in the disabled state.
 674  *
 675  * @usb:         Pointer to an empty struct cvmx_usb_state
 676  *               that will be populated by the initialize call.
 677  *               This structure is then passed to all other USB
 678  *               functions.
 679  * @usb_port_number:
 680  *               Which Octeon USB port to initialize.
 681  *
 682  * Returns: 0 or a negative error code.
 683  */
 684 static int cvmx_usb_initialize(struct cvmx_usb_state *usb,
 685                                int usb_port_number,
 686                                enum cvmx_usb_initialize_flags flags)
 687 {
 688         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 689         union cvmx_usbnx_usbp_ctl_status usbn_usbp_ctl_status;
 690         int i;
 691
 692         /* At first allow 0-1 for the usb port number */
 693         if ((usb_port_number < 0) || (usb_port_number > 1))
 694                 return -EINVAL;
 695
 696         memset(usb, 0, sizeof(*usb));
 697         usb->init_flags = flags;
 698
 699         /* Initialize the USB state structure */
 700         usb->index = usb_port_number;
 701         INIT_LIST_HEAD(&usb->idle_pipes);
 702         for (i = 0; i < ARRAY_SIZE(usb->active_pipes); i++)
 703                 INIT_LIST_HEAD(&usb->active_pipes[i]);
 704
 705         /*
 706          * Power On Reset and PHY Initialization
 707          *
 708          * 1. Wait for DCOK to assert (nothing to do)
 709          *
 710          * 2a. Write USBN0/1_CLK_CTL[POR] = 1 and
 711          *     USBN0/1_CLK_CTL[HRST,PRST,HCLK_RST] = 0
 712          */
 713         usbn_clk_ctl.u64 =
 714                 __cvmx_usb_read_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index));
 715         usbn_clk_ctl.s.por = 1;
 716         usbn_clk_ctl.s.hrst = 0;
 717         usbn_clk_ctl.s.prst = 0;
 718         usbn_clk_ctl.s.hclk_rst = 0;
 719         usbn_clk_ctl.s.enable = 0;
 720         /*
 721          * 2b. Select the USB reference clock/crystal parameters by writing
 722          *     appropriate values to USBN0/1_CLK_CTL[P_C_SEL, P_RTYPE, P_COM_ON]
 723          */
 724         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND) {
 725                 /*
 726                  * The USB port uses 12/24/48MHz 2.5V board clock
 727                  * source at USB_XO. USB_XI should be tied to GND.
 728                  * Most Octeon evaluation boards require this setting
 729                  */
 730                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX) ||
 731                     OCTEON_IS_MODEL(OCTEON_CN56XX) ||
 732                     OCTEON_IS_MODEL(OCTEON_CN50XX))
 733                         /* From CN56XX,CN50XX,CN31XX,CN30XX manuals */
 734                         usbn_clk_ctl.s.p_rtype = 2; /* p_rclk=1 & p_xenbn=0 */
 735                 else
 736                         /* From CN52XX manual */
 737                         usbn_clk_ctl.s.p_rtype = 1;
 738
 739                 switch (flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK) {
 740                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:
 741                         usbn_clk_ctl.s.p_c_sel = 0;
 742                         break;
 743                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:
 744                         usbn_clk_ctl.s.p_c_sel = 1;
 745                         break;
 746                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:
 747                         usbn_clk_ctl.s.p_c_sel = 2;
 748                         break;
 749                 }
 750         } else {
 751                 /*
 752                  * The USB port uses a 12MHz crystal as clock source
 753                  * at USB_XO and USB_XI
 754                  */
 755                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
 756                         /* From CN31XX,CN30XX manual */
 757                         usbn_clk_ctl.s.p_rtype = 3; /* p_rclk=1 & p_xenbn=1 */
 758                 else
 759                         /* From CN56XX,CN52XX,CN50XX manuals. */
 760                         usbn_clk_ctl.s.p_rtype = 0;
 761
 762                 usbn_clk_ctl.s.p_c_sel = 0;
 763         }
 764         /*
 765          * 2c. Select the HCLK via writing USBN0/1_CLK_CTL[DIVIDE, DIVIDE2] and
 766          *     setting USBN0/1_CLK_CTL[ENABLE] = 1. Divide the core clock down
 767          *     such that USB is as close as possible to 125Mhz
 768          */
 769         {
 770                 int divisor = (octeon_get_clock_rate()+125000000-1)/125000000;
 771                 /* Lower than 4 doesn't seem to work properly */
 772                 if (divisor < 4)
 773                         divisor = 4;
 774                 usbn_clk_ctl.s.divide = divisor;
 775                 usbn_clk_ctl.s.divide2 = 0;
 776         }
 777         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 778                                usbn_clk_ctl.u64);
 779         /* 2d. Write USBN0/1_CLK_CTL[HCLK_RST] = 1 */
 780         usbn_clk_ctl.s.hclk_rst = 1;
 781         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 782                                usbn_clk_ctl.u64);
 783         /* 2e.  Wait 64 core-clock cycles for HCLK to stabilize */
 784         cvmx_wait(64);
 785         /*
 786          * 3. Program the power-on reset field in the USBN clock-control
 787          *    register:
 788          *    USBN_CLK_CTL[POR] = 0
 789          */
 790         usbn_clk_ctl.s.por = 0;
 791         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 792                                usbn_clk_ctl.u64);
 793         /* 4. Wait 1 ms for PHY clock to start */
 794         mdelay(1);
 795         /*
 796          * 5. Program the Reset input from automatic test equipment field in the
 797          *    USBP control and status register:
 798          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 1
 799          */
 800         usbn_usbp_ctl_status.u64 = __cvmx_usb_read_csr64(usb,
 801                         CVMX_USBNX_USBP_CTL_STATUS(usb->index));
 802         usbn_usbp_ctl_status.s.ate_reset = 1;
 803         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 804                                usbn_usbp_ctl_status.u64);
 805         /* 6. Wait 10 cycles */
 806         cvmx_wait(10);
 807         /*
 808          * 7. Clear ATE_RESET field in the USBN clock-control register:
 809          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 0
 810          */
 811         usbn_usbp_ctl_status.s.ate_reset = 0;
 812         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 813                                usbn_usbp_ctl_status.u64);
 814         /*
 815          * 8. Program the PHY reset field in the USBN clock-control register:
 816          *    USBN_CLK_CTL[PRST] = 1
 817          */
 818         usbn_clk_ctl.s.prst = 1;
 819         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 820                                usbn_clk_ctl.u64);
 821         /*
 822          * 9. Program the USBP control and status register to select host or
 823          *    device mode. USBN_USBP_CTL_STATUS[HST_MODE] = 0 for host, = 1 for
 824          *    device
 825          */
 826         usbn_usbp_ctl_status.s.hst_mode = 0;
 827         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 828                                usbn_usbp_ctl_status.u64);
 829         /* 10. Wait 1 us */
 830         udelay(1);
 831         /*
 832          * 11. Program the hreset_n field in the USBN clock-control register:
 833          *     USBN_CLK_CTL[HRST] = 1
 834          */
 835         usbn_clk_ctl.s.hrst = 1;
 836         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 837                                usbn_clk_ctl.u64);
 838         /* 12. Proceed to USB core initialization */
 839         usbn_clk_ctl.s.enable = 1;
 840         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 841                                usbn_clk_ctl.u64);
 842         udelay(1);
 843
 844         /*
 845          * USB Core Initialization
 846          *
 847          * 1. Read USBC_GHWCFG1, USBC_GHWCFG2, USBC_GHWCFG3, USBC_GHWCFG4 to
 848          *    determine USB core configuration parameters.
 849          *
 850          *    Nothing needed
 851          *
 852          * 2. Program the following fields in the global AHB configuration
 853          *    register (USBC_GAHBCFG)
 854          *    DMA mode, USBC_GAHBCFG[DMAEn]: 1 = DMA mode, 0 = slave mode
 855          *    Burst length, USBC_GAHBCFG[HBSTLEN] = 0
 856          *    Nonperiodic TxFIFO empty level (slave mode only),
 857          *    USBC_GAHBCFG[NPTXFEMPLVL]
 858          *    Periodic TxFIFO empty level (slave mode only),
 859          *    USBC_GAHBCFG[PTXFEMPLVL]
 860          *    Global interrupt mask, USBC_GAHBCFG[GLBLINTRMSK] = 1
 861          */
 862         {
 863                 union cvmx_usbcx_gahbcfg usbcx_gahbcfg;
 864                 /* Due to an errata, CN31XX doesn't support DMA */
 865                 if (OCTEON_IS_MODEL(OCTEON_CN31XX))
 866                         usb->init_flags |= CVMX_USB_INITIALIZE_FLAGS_NO_DMA;
 867                 usbcx_gahbcfg.u32 = 0;
 868                 usbcx_gahbcfg.s.dmaen = !(usb->init_flags &
 869                                           CVMX_USB_INITIALIZE_FLAGS_NO_DMA);
 870                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 871                         /* Only use one channel with non DMA */
 872                         usb->idle_hardware_channels = 0x1;
 873                 else if (OCTEON_IS_MODEL(OCTEON_CN5XXX))
 874                         /* CN5XXX have an errata with channel 3 */
 875                         usb->idle_hardware_channels = 0xf7;
 876                 else
 877                         usb->idle_hardware_channels = 0xff;
 878                 usbcx_gahbcfg.s.hbstlen = 0;
 879                 usbcx_gahbcfg.s.nptxfemplvl = 1;
 880                 usbcx_gahbcfg.s.ptxfemplvl = 1;
 881                 usbcx_gahbcfg.s.glblintrmsk = 1;
 882                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GAHBCFG(usb->index),
 883                                        usbcx_gahbcfg.u32);
 884         }
 885         /*
 886          * 3. Program the following fields in USBC_GUSBCFG register.
 887          *    HS/FS timeout calibration, USBC_GUSBCFG[TOUTCAL] = 0
 888          *    ULPI DDR select, USBC_GUSBCFG[DDRSEL] = 0
 889          *    USB turnaround time, USBC_GUSBCFG[USBTRDTIM] = 0x5
 890          *    PHY low-power clock select, USBC_GUSBCFG[PHYLPWRCLKSEL] = 0
 891          */
 892         {
 893                 union cvmx_usbcx_gusbcfg usbcx_gusbcfg;
 894
 895                 usbcx_gusbcfg.u32 = __cvmx_usb_read_csr32(usb,
 896                                 CVMX_USBCX_GUSBCFG(usb->index));
 897                 usbcx_gusbcfg.s.toutcal = 0;
 898                 usbcx_gusbcfg.s.ddrsel = 0;
 899                 usbcx_gusbcfg.s.usbtrdtim = 0x5;
 900                 usbcx_gusbcfg.s.phylpwrclksel = 0;
 901                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index),
 902                                        usbcx_gusbcfg.u32);
 903         }
 904         /*
 905          * 4. The software must unmask the following bits in the USBC_GINTMSK
 906          *    register.
 907          *    OTG interrupt mask, USBC_GINTMSK[OTGINTMSK] = 1
 908          *    Mode mismatch interrupt mask, USBC_GINTMSK[MODEMISMSK] = 1
 909          */
 910         {
 911                 union cvmx_usbcx_gintmsk usbcx_gintmsk;
 912                 int channel;
 913
 914                 usbcx_gintmsk.u32 = __cvmx_usb_read_csr32(usb,
 915                                 CVMX_USBCX_GINTMSK(usb->index));
 916                 usbcx_gintmsk.s.otgintmsk = 1;
 917                 usbcx_gintmsk.s.modemismsk = 1;
 918                 usbcx_gintmsk.s.hchintmsk = 1;
 919                 usbcx_gintmsk.s.sofmsk = 0;
 920                 /* We need RX FIFO interrupts if we don't have DMA */
 921                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 922                         usbcx_gintmsk.s.rxflvlmsk = 1;
 923                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTMSK(usb->index),
 924                                        usbcx_gintmsk.u32);
 925
 926                 /*
 927                  * Disable all channel interrupts. We'll enable them per channel
 928                  * later.
 929                  */
 930                 for (channel = 0; channel < 8; channel++)
 931                         __cvmx_usb_write_csr32(usb,
 932                                 CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
 933         }
 934
 935         {
 936                 /*
 937                  * Host Port Initialization
 938                  *
 939                  * 1. Program the host-port interrupt-mask field to unmask,
 940                  *    USBC_GINTMSK[PRTINT] = 1
 941                  */
 942                 USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
 943                                 union cvmx_usbcx_gintmsk, prtintmsk, 1);
 944                 USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
 945                                 union cvmx_usbcx_gintmsk, disconnintmsk, 1);
 946                 /*
 947                  * 2. Program the USBC_HCFG register to select full-speed host
 948                  *    or high-speed host.
 949                  */
 950                 {
 951                         union cvmx_usbcx_hcfg usbcx_hcfg;
 952
 953                         usbcx_hcfg.u32 = __cvmx_usb_read_csr32(usb,
 954                                         CVMX_USBCX_HCFG(usb->index));
 955                         usbcx_hcfg.s.fslssupp = 0;
 956                         usbcx_hcfg.s.fslspclksel = 0;
 957                         __cvmx_usb_write_csr32(usb,
 958                                         CVMX_USBCX_HCFG(usb->index),
 959                                         usbcx_hcfg.u32);
 960                 }
 961                 /*
 962                  * 3. Program the port power bit to drive VBUS on the USB,
 963                  *    USBC_HPRT[PRTPWR] = 1
 964                  */
 965                 USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index),
 966                                 union cvmx_usbcx_hprt, prtpwr, 1);
 967
 968                 /*
 969                  * Steps 4-15 from the manual are done later in the port enable
 970                  */
 971         }
 972
 973         return 0;
 974 }
 975
 976
 977 /**
 978  * Shutdown a USB port after a call to cvmx_usb_initialize().
 979  * The port should be disabled with all pipes closed when this
 980  * function is called.
 981  *
 982  * @usb: USB device state populated by cvmx_usb_initialize().
 983  *
 984  * Returns: 0 or a negative error code.
 985  */
 986 static int cvmx_usb_shutdown(struct cvmx_usb_state *usb)
 987 {
 988         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 989
 990         /* Make sure all pipes are closed */
 991         if (!list_empty(&usb->idle_pipes) ||
 992             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_ISOCHRONOUS]) ||
 993             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_INTERRUPT]) ||
 994             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_CONTROL]) ||
 995             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_BULK]))
 996                 return -EBUSY;
 997
 998         /* Disable the clocks and put them in power on reset */
 999         usbn_clk_ctl.u64 = __cvmx_usb_read_csr64(usb,
1000                         CVMX_USBNX_CLK_CTL(usb->index));
1001         usbn_clk_ctl.s.enable = 1;
1002         usbn_clk_ctl.s.por = 1;
1003         usbn_clk_ctl.s.hclk_rst = 1;
1004         usbn_clk_ctl.s.prst = 0;
1005         usbn_clk_ctl.s.hrst = 0;
1006         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
1007                                usbn_clk_ctl.u64);
1008         return 0;
1009 }
1010
1011
1012 /**
1013  * Enable a USB port. After this call succeeds, the USB port is
1014  * online and servicing requests.
1015  *
1016  * @usb: USB device state populated by cvmx_usb_initialize().
1017  *
1018  * Returns: 0 or a negative error code.
1019  */
1020 static int cvmx_usb_enable(struct cvmx_usb_state *usb)
1021 {
1022         union cvmx_usbcx_ghwcfg3 usbcx_ghwcfg3;
1023
1024         usb->usbcx_hprt.u32 = __cvmx_usb_read_csr32(usb,
1025                         CVMX_USBCX_HPRT(usb->index));
1026
1027         /*
1028          * If the port is already enabled the just return. We don't need to do
1029          * anything
1030          */
1031         if (usb->usbcx_hprt.s.prtena)
1032                 return 0;
1033
1034         /* If there is nothing plugged into the port then fail immediately */
1035         if (!usb->usbcx_hprt.s.prtconnsts)
1036                 return -ETIMEDOUT;
1037
1038         /* Program the port reset bit to start the reset process */
1039         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt,
1040                         prtrst, 1);
1041
1042         /*
1043          * Wait at least 50ms (high speed), or 10ms (full speed) for the reset
1044          * process to complete.
1045          */
1046         mdelay(50);
1047
1048         /* Program the port reset bit to 0, USBC_HPRT[PRTRST] = 0 */
1049         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt,
1050                         prtrst, 0);
1051
1052         /* Wait for the USBC_HPRT[PRTENA]. */
1053         if (CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_HPRT(usb->index),
1054                                 union cvmx_usbcx_hprt, prtena, ==, 1, 100000))
1055                 return -ETIMEDOUT;
1056
1057         /*
1058          * Read the port speed field to get the enumerated speed,
1059          * USBC_HPRT[PRTSPD].
1060          */
1061         usb->usbcx_hprt.u32 = __cvmx_usb_read_csr32(usb,
1062                         CVMX_USBCX_HPRT(usb->index));
1063         usbcx_ghwcfg3.u32 = __cvmx_usb_read_csr32(usb,
1064                         CVMX_USBCX_GHWCFG3(usb->index));
1065
1066         /*
1067          * 13. Program the USBC_GRXFSIZ register to select the size of the
1068          *     receive FIFO (25%).
1069          */
1070         USB_SET_FIELD32(CVMX_USBCX_GRXFSIZ(usb->index),
1071                         union cvmx_usbcx_grxfsiz, rxfdep,
1072                         usbcx_ghwcfg3.s.dfifodepth / 4);
1073         /*
1074          * 14. Program the USBC_GNPTXFSIZ register to select the size and the
1075          *     start address of the non- periodic transmit FIFO for nonperiodic
1076          *     transactions (50%).
1077          */
1078         {
1079                 union cvmx_usbcx_gnptxfsiz siz;
1080
1081                 siz.u32 = __cvmx_usb_read_csr32(usb,
1082                                 CVMX_USBCX_GNPTXFSIZ(usb->index));
1083                 siz.s.nptxfdep = usbcx_ghwcfg3.s.dfifodepth / 2;
1084                 siz.s.nptxfstaddr = usbcx_ghwcfg3.s.dfifodepth / 4;
1085                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index),
1086                                        siz.u32);
1087         }
1088         /*
1089          * 15. Program the USBC_HPTXFSIZ register to select the size and start
1090          *     address of the periodic transmit FIFO for periodic transactions
1091          *     (25%).
1092          */
1093         {
1094                 union cvmx_usbcx_hptxfsiz siz;
1095
1096                 siz.u32 = __cvmx_usb_read_csr32(usb,
1097                                 CVMX_USBCX_HPTXFSIZ(usb->index));
1098                 siz.s.ptxfsize = usbcx_ghwcfg3.s.dfifodepth / 4;
1099                 siz.s.ptxfstaddr = 3 * usbcx_ghwcfg3.s.dfifodepth / 4;
1100                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index),
1101                                        siz.u32);
1102         }
1103         /* Flush all FIFOs */
1104         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1105                         union cvmx_usbcx_grstctl, txfnum, 0x10);
1106         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1107                         union cvmx_usbcx_grstctl, txfflsh, 1);
1108         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1109                               union cvmx_usbcx_grstctl,
1110                               txfflsh, ==, 0, 100);
1111         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1112                         union cvmx_usbcx_grstctl, rxfflsh, 1);
1113         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1114                               union cvmx_usbcx_grstctl,
1115                               rxfflsh, ==, 0, 100);
1116
1117         return 0;
1118 }
1119
1120
1121 /**
1122  * Disable a USB port. After this call the USB port will not
1123  * generate data transfers and will not generate events.
1124  * Transactions in process will fail and call their
1125  * associated callbacks.
1126  *
1127  * @usb: USB device state populated by cvmx_usb_initialize().
1128  *
1129  * Returns: 0 or a negative error code.
1130  */
1131 static int cvmx_usb_disable(struct cvmx_usb_state *usb)
1132 {
1133         /* Disable the port */
1134         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt,
1135                         prtena, 1);
1136         return 0;
1137 }
1138
1139
1140 /**
1141  * Get the current state of the USB port. Use this call to
1142  * determine if the usb port has anything connected, is enabled,
1143  * or has some sort of error condition. The return value of this
1144  * call has "changed" bits to signal of the value of some fields
1145  * have changed between calls.
1146  *
1147  * @usb: USB device state populated by cvmx_usb_initialize().
1148  *
1149  * Returns: Port status information
1150  */
1151 static struct cvmx_usb_port_status cvmx_usb_get_status(
1152                 struct cvmx_usb_state *usb)
1153 {
1154         union cvmx_usbcx_hprt usbc_hprt;
1155         struct cvmx_usb_port_status result;
1156
1157         memset(&result, 0, sizeof(result));
1158
1159         usbc_hprt.u32 = __cvmx_usb_read_csr32(usb,
1160                         CVMX_USBCX_HPRT(usb->index));
1161         result.port_enabled = usbc_hprt.s.prtena;
1162         result.port_over_current = usbc_hprt.s.prtovrcurract;
1163         result.port_powered = usbc_hprt.s.prtpwr;
1164         result.port_speed = usbc_hprt.s.prtspd;
1165         result.connected = usbc_hprt.s.prtconnsts;
1166         result.connect_change =
1167                 (result.connected != usb->port_status.connected);
1168
1169         return result;
1170 }
1171
1172 /**
1173  * Open a virtual pipe between the host and a USB device. A pipe
1174  * must be opened before data can be transferred between a device
1175  * and Octeon.
1176  *
1177  * @usb:             USB device state populated by cvmx_usb_initialize().
1178  * @device_addr:
1179  *                   USB device address to open the pipe to
1180  *                   (0-127).
1181  * @endpoint_num:
1182  *                   USB endpoint number to open the pipe to
1183  *                   (0-15).
1184  * @device_speed:
1185  *                   The speed of the device the pipe is going
1186  *                   to. This must match the device's speed,
1187  *                   which may be different than the port speed.
1188  * @max_packet:      The maximum packet length the device can
1189  *                   transmit/receive (low speed=0-8, full
1190  *                   speed=0-1023, high speed=0-1024). This value
1191  *                   comes from the standard endpoint descriptor
1192  *                   field wMaxPacketSize bits <10:0>.
1193  * @transfer_type:
1194  *                   The type of transfer this pipe is for.
1195  * @transfer_dir:
1196  *                   The direction the pipe is in. This is not
1197  *                   used for control pipes.
1198  * @interval:        For ISOCHRONOUS and INTERRUPT transfers,
1199  *                   this is how often the transfer is scheduled
1200  *                   for. All other transfers should specify
1201  *                   zero. The units are in frames (8000/sec at
1202  *                   high speed, 1000/sec for full speed).
1203  * @multi_count:
1204  *                   For high speed devices, this is the maximum
1205  *                   allowed number of packet per microframe.
1206  *                   Specify zero for non high speed devices. This
1207  *                   value comes from the standard endpoint descriptor
1208  *                   field wMaxPacketSize bits <12:11>.
1209  * @hub_device_addr:
1210  *                   Hub device address this device is connected
1211  *                   to. Devices connected directly to Octeon
1212  *                   use zero. This is only used when the device
1213  *                   is full/low speed behind a high speed hub.
1214  *                   The address will be of the high speed hub,
1215  *                   not and full speed hubs after it.
1216  * @hub_port:        Which port on the hub the device is
1217  *                   connected. Use zero for devices connected
1218  *                   directly to Octeon. Like hub_device_addr,
1219  *                   this is only used for full/low speed
1220  *                   devices behind a high speed hub.
1221  *
1222  * Returns: A non-NULL value is a pipe. NULL means an error.
1223  */
1224 static struct cvmx_usb_pipe *cvmx_usb_open_pipe(struct cvmx_usb_state *usb,
1225                                                 int device_addr,
1226                                                 int endpoint_num,
1227                                                 enum cvmx_usb_speed
1228                                                         device_speed,
1229                                                 int max_packet,
1230                                                 enum cvmx_usb_transfer
1231                                                         transfer_type,
1232                                                 enum cvmx_usb_direction
1233                                                         transfer_dir,
1234                                                 int interval, int multi_count,
1235                                                 int hub_device_addr,
1236                                                 int hub_port)
1237 {
1238         struct cvmx_usb_pipe *pipe;
1239
1240         if (unlikely((device_addr < 0) || (device_addr > MAX_USB_ADDRESS)))
1241                 return NULL;
1242         if (unlikely((endpoint_num < 0) || (endpoint_num > MAX_USB_ENDPOINT)))
1243                 return NULL;
1244         if (unlikely(device_speed > CVMX_USB_SPEED_LOW))
1245                 return NULL;
1246         if (unlikely((max_packet <= 0) || (max_packet > 1024)))
1247                 return NULL;
1248         if (unlikely(transfer_type > CVMX_USB_TRANSFER_INTERRUPT))
1249                 return NULL;
1250         if (unlikely((transfer_dir != CVMX_USB_DIRECTION_OUT) &&
1251                 (transfer_dir != CVMX_USB_DIRECTION_IN)))
1252                 return NULL;
1253         if (unlikely(interval < 0))
1254                 return NULL;
1255         if (unlikely((transfer_type == CVMX_USB_TRANSFER_CONTROL) && interval))
1256                 return NULL;
1257         if (unlikely(multi_count < 0))
1258                 return NULL;
1259         if (unlikely((device_speed != CVMX_USB_SPEED_HIGH) &&
1260                 (multi_count != 0)))
1261                 return NULL;
1262         if (unlikely((hub_device_addr < 0) ||
1263                 (hub_device_addr > MAX_USB_ADDRESS)))
1264                 return NULL;
1265         if (unlikely((hub_port < 0) || (hub_port > MAX_USB_HUB_PORT)))
1266                 return NULL;
1267
1268         pipe = kzalloc(sizeof(*pipe), GFP_ATOMIC);
1269         if (!pipe)
1270                 return NULL;
1271         if ((device_speed == CVMX_USB_SPEED_HIGH) &&
1272                 (transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1273                 (transfer_type == CVMX_USB_TRANSFER_BULK))
1274                 pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
1275         pipe->device_addr = device_addr;
1276         pipe->endpoint_num = endpoint_num;
1277         pipe->device_speed = device_speed;
1278         pipe->max_packet = max_packet;
1279         pipe->transfer_type = transfer_type;
1280         pipe->transfer_dir = transfer_dir;
1281         INIT_LIST_HEAD(&pipe->transactions);
1282
1283         /*
1284          * All pipes use interval to rate limit NAK processing. Force an
1285          * interval if one wasn't supplied
1286          */
1287         if (!interval)
1288                 interval = 1;
1289         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1290                 pipe->interval = interval*8;
1291                 /* Force start splits to be schedule on uFrame 0 */
1292                 pipe->next_tx_frame = ((usb->frame_number+7)&~7) +
1293                                         pipe->interval;
1294         } else {
1295                 pipe->interval = interval;
1296                 pipe->next_tx_frame = usb->frame_number + pipe->interval;
1297         }
1298         pipe->multi_count = multi_count;
1299         pipe->hub_device_addr = hub_device_addr;
1300         pipe->hub_port = hub_port;
1301         pipe->pid_toggle = 0;
1302         pipe->split_sc_frame = -1;
1303         list_add_tail(&pipe->node, &usb->idle_pipes);
1304
1305         /*
1306          * We don't need to tell the hardware about this pipe yet since
1307          * it doesn't have any submitted requests
1308          */
1309
1310         return pipe;
1311 }
1312
1313
1314 /**
1315  * Poll the RX FIFOs and remove data as needed. This function is only used
1316  * in non DMA mode. It is very important that this function be called quickly
1317  * enough to prevent FIFO overflow.
1318  *
1319  * @usb:        USB device state populated by cvmx_usb_initialize().
1320  */
1321 static void __cvmx_usb_poll_rx_fifo(struct cvmx_usb_state *usb)
1322 {
1323         union cvmx_usbcx_grxstsph rx_status;
1324         int channel;
1325         int bytes;
1326         uint64_t address;
1327         uint32_t *ptr;
1328
1329         rx_status.u32 = __cvmx_usb_read_csr32(usb,
1330                         CVMX_USBCX_GRXSTSPH(usb->index));
1331         /* Only read data if IN data is there */
1332         if (rx_status.s.pktsts != 2)
1333                 return;
1334         /* Check if no data is available */
1335         if (!rx_status.s.bcnt)
1336                 return;
1337
1338         channel = rx_status.s.chnum;
1339         bytes = rx_status.s.bcnt;
1340         if (!bytes)
1341                 return;
1342
1343         /* Get where the DMA engine would have written this data */
1344         address = __cvmx_usb_read_csr64(usb,
1345                         CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8);
1346
1347         ptr = cvmx_phys_to_ptr(address);
1348         __cvmx_usb_write_csr64(usb,
1349                                CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8,
1350                                address + bytes);
1351
1352         /* Loop writing the FIFO data for this packet into memory */
1353         while (bytes > 0) {
1354                 *ptr++ = __cvmx_usb_read_csr32(usb, USB_FIFO_ADDRESS(channel, usb->index));
1355                 bytes -= 4;
1356         }
1357         CVMX_SYNCW;
1358
1359         return;
1360 }
1361
1362
1363 /**
1364  * Fill the TX hardware fifo with data out of the software
1365  * fifos
1366  *
1367  * @usb:            USB device state populated by cvmx_usb_initialize().
1368  * @fifo:           Software fifo to use
1369  * @available:      Amount of space in the hardware fifo
1370  *
1371  * Returns: Non zero if the hardware fifo was too small and needs
1372  *          to be serviced again.
1373  */
1374 static int __cvmx_usb_fill_tx_hw(struct cvmx_usb_state *usb,
1375                                  struct cvmx_usb_tx_fifo *fifo, int available)
1376 {
1377         /*
1378          * We're done either when there isn't anymore space or the software FIFO
1379          * is empty
1380          */
1381         while (available && (fifo->head != fifo->tail)) {
1382                 int i = fifo->tail;
1383                 const uint32_t *ptr = cvmx_phys_to_ptr(fifo->entry[i].address);
1384                 uint64_t csr_address = USB_FIFO_ADDRESS(fifo->entry[i].channel,
1385                                                         usb->index) ^ 4;
1386                 int words = available;
1387
1388                 /* Limit the amount of data to waht the SW fifo has */
1389                 if (fifo->entry[i].size <= available) {
1390                         words = fifo->entry[i].size;
1391                         fifo->tail++;
1392                         if (fifo->tail > MAX_CHANNELS)
1393                                 fifo->tail = 0;
1394                 }
1395
1396                 /* Update the next locations and counts */
1397                 available -= words;
1398                 fifo->entry[i].address += words * 4;
1399                 fifo->entry[i].size -= words;
1400
1401                 /*
1402                  * Write the HW fifo data. The read every three writes is due
1403                  * to an errata on CN3XXX chips
1404                  */
1405                 while (words > 3) {
1406                         cvmx_write64_uint32(csr_address, *ptr++);
1407                         cvmx_write64_uint32(csr_address, *ptr++);
1408                         cvmx_write64_uint32(csr_address, *ptr++);
1409                         cvmx_read64_uint64(
1410                                         CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1411                         words -= 3;
1412                 }
1413                 cvmx_write64_uint32(csr_address, *ptr++);
1414                 if (--words) {
1415                         cvmx_write64_uint32(csr_address, *ptr++);
1416                         if (--words)
1417                                 cvmx_write64_uint32(csr_address, *ptr++);
1418                 }
1419                 cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1420         }
1421         return fifo->head != fifo->tail;
1422 }
1423
1424
1425 /**
1426  * Check the hardware FIFOs and fill them as needed
1427  *
1428  * @usb:        USB device state populated by cvmx_usb_initialize().
1429  */
1430 static void __cvmx_usb_poll_tx_fifo(struct cvmx_usb_state *usb)
1431 {
1432         if (usb->periodic.head != usb->periodic.tail) {
1433                 union cvmx_usbcx_hptxsts tx_status;
1434
1435                 tx_status.u32 = __cvmx_usb_read_csr32(usb,
1436                                 CVMX_USBCX_HPTXSTS(usb->index));
1437                 if (__cvmx_usb_fill_tx_hw(usb, &usb->periodic,
1438                                           tx_status.s.ptxfspcavail))
1439                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1440                                         union cvmx_usbcx_gintmsk,
1441                                         ptxfempmsk, 1);
1442                 else
1443                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1444                                         union cvmx_usbcx_gintmsk,
1445                                         ptxfempmsk, 0);
1446         }
1447
1448         if (usb->nonperiodic.head != usb->nonperiodic.tail) {
1449                 union cvmx_usbcx_gnptxsts tx_status;
1450
1451                 tx_status.u32 = __cvmx_usb_read_csr32(usb,
1452                                 CVMX_USBCX_GNPTXSTS(usb->index));
1453                 if (__cvmx_usb_fill_tx_hw(usb, &usb->nonperiodic,
1454                                           tx_status.s.nptxfspcavail))
1455                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1456                                         union cvmx_usbcx_gintmsk,
1457                                         nptxfempmsk, 1);
1458                 else
1459                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1460                                         union cvmx_usbcx_gintmsk,
1461                                         nptxfempmsk, 0);
1462         }
1463
1464         return;
1465 }
1466
1467
1468 /**
1469  * Fill the TX FIFO with an outgoing packet
1470  *
1471  * @usb:          USB device state populated by cvmx_usb_initialize().
1472  * @channel:      Channel number to get packet from
1473  */
1474 static void __cvmx_usb_fill_tx_fifo(struct cvmx_usb_state *usb, int channel)
1475 {
1476         union cvmx_usbcx_hccharx hcchar;
1477         union cvmx_usbcx_hcspltx usbc_hcsplt;
1478         union cvmx_usbcx_hctsizx usbc_hctsiz;
1479         struct cvmx_usb_tx_fifo *fifo;
1480
1481         /* We only need to fill data on outbound channels */
1482         hcchar.u32 = __cvmx_usb_read_csr32(usb,
1483                         CVMX_USBCX_HCCHARX(channel, usb->index));
1484         if (hcchar.s.epdir != CVMX_USB_DIRECTION_OUT)
1485                 return;
1486
1487         /* OUT Splits only have data on the start and not the complete */
1488         usbc_hcsplt.u32 = __cvmx_usb_read_csr32(usb,
1489                         CVMX_USBCX_HCSPLTX(channel, usb->index));
1490         if (usbc_hcsplt.s.spltena && usbc_hcsplt.s.compsplt)
1491                 return;
1492
1493         /*
1494          * Find out how many bytes we need to fill and convert it into 32bit
1495          * words.
1496          */
1497         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb,
1498                         CVMX_USBCX_HCTSIZX(channel, usb->index));
1499         if (!usbc_hctsiz.s.xfersize)
1500                 return;
1501
1502         if ((hcchar.s.eptype == CVMX_USB_TRANSFER_INTERRUPT) ||
1503                 (hcchar.s.eptype == CVMX_USB_TRANSFER_ISOCHRONOUS))
1504                 fifo = &usb->periodic;
1505         else
1506                 fifo = &usb->nonperiodic;
1507
1508         fifo->entry[fifo->head].channel = channel;
1509         fifo->entry[fifo->head].address = __cvmx_usb_read_csr64(usb, CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8);
1510         fifo->entry[fifo->head].size = (usbc_hctsiz.s.xfersize+3)>>2;
1511         fifo->head++;
1512         if (fifo->head > MAX_CHANNELS)
1513                 fifo->head = 0;
1514
1515         __cvmx_usb_poll_tx_fifo(usb);
1516
1517         return;
1518 }
1519
1520 /**
1521  * Perform channel specific setup for Control transactions. All
1522  * the generic stuff will already have been done in
1523  * __cvmx_usb_start_channel()
1524  *
1525  * @usb:          USB device state populated by cvmx_usb_initialize().
1526  * @channel:      Channel to setup
1527  * @pipe:         Pipe for control transaction
1528  */
1529 static void __cvmx_usb_start_channel_control(struct cvmx_usb_state *usb,
1530                                              int channel,
1531                                              struct cvmx_usb_pipe *pipe)
1532 {
1533         struct cvmx_usb_transaction *transaction =
1534                 list_first_entry(&pipe->transactions, typeof(*transaction),
1535                                  node);
1536         union cvmx_usb_control_header *header =
1537                 cvmx_phys_to_ptr(transaction->control_header);
1538         int bytes_to_transfer = transaction->buffer_length -
1539                 transaction->actual_bytes;
1540         int packets_to_transfer;
1541         union cvmx_usbcx_hctsizx usbc_hctsiz;
1542
1543         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb,
1544                         CVMX_USBCX_HCTSIZX(channel, usb->index));
1545
1546         switch (transaction->stage) {
1547         case CVMX_USB_STAGE_NON_CONTROL:
1548         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
1549                 cvmx_dprintf("%s: ERROR - Non control stage\n", __func__);
1550                 break;
1551         case CVMX_USB_STAGE_SETUP:
1552                 usbc_hctsiz.s.pid = 3; /* Setup */
1553                 bytes_to_transfer = sizeof(*header);
1554                 /* All Control operations start with a setup going OUT */
1555                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1556                                 union cvmx_usbcx_hccharx, epdir,
1557                                 CVMX_USB_DIRECTION_OUT);
1558                 /*
1559                  * Setup send the control header instead of the buffer data. The
1560                  * buffer data will be used in the next stage
1561                  */
1562                 __cvmx_usb_write_csr64(usb,
1563                         CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8,
1564                         transaction->control_header);
1565                 break;
1566         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
1567                 usbc_hctsiz.s.pid = 3; /* Setup */
1568                 bytes_to_transfer = 0;
1569                 /* All Control operations start with a setup going OUT */
1570                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1571                                 union cvmx_usbcx_hccharx, epdir,
1572                                 CVMX_USB_DIRECTION_OUT);
1573
1574                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1575                                 union cvmx_usbcx_hcspltx, compsplt, 1);
1576                 break;
1577         case CVMX_USB_STAGE_DATA:
1578                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1579                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1580                         if (header->s.request_type & 0x80)
1581                                 bytes_to_transfer = 0;
1582                         else if (bytes_to_transfer > pipe->max_packet)
1583                                 bytes_to_transfer = pipe->max_packet;
1584                 }
1585                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1586                                 union cvmx_usbcx_hccharx, epdir,
1587                                 ((header->s.request_type & 0x80) ?
1588                                         CVMX_USB_DIRECTION_IN :
1589                                         CVMX_USB_DIRECTION_OUT));
1590                 break;
1591         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
1592                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1593                 if (!(header->s.request_type & 0x80))
1594                         bytes_to_transfer = 0;
1595                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1596                                 union cvmx_usbcx_hccharx, epdir,
1597                                 ((header->s.request_type & 0x80) ?
1598                                         CVMX_USB_DIRECTION_IN :
1599                                         CVMX_USB_DIRECTION_OUT));
1600                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1601                                 union cvmx_usbcx_hcspltx, compsplt, 1);
1602                 break;
1603         case CVMX_USB_STAGE_STATUS:
1604                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1605                 bytes_to_transfer = 0;
1606                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1607                                 union cvmx_usbcx_hccharx, epdir,
1608                                 ((header->s.request_type & 0x80) ?
1609                                         CVMX_USB_DIRECTION_OUT :
1610                                         CVMX_USB_DIRECTION_IN));
1611                 break;
1612         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
1613                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1614                 bytes_to_transfer = 0;
1615                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1616                                 union cvmx_usbcx_hccharx, epdir,
1617                                 ((header->s.request_type & 0x80) ?
1618                                         CVMX_USB_DIRECTION_OUT :
1619                                         CVMX_USB_DIRECTION_IN));
1620                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1621                                 union cvmx_usbcx_hcspltx, compsplt, 1);
1622                 break;
1623         }
1624
1625         /*
1626          * Make sure the transfer never exceeds the byte limit of the hardware.
1627          * Further bytes will be sent as continued transactions
1628          */
1629         if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1630                 /* Round MAX_TRANSFER_BYTES to a multiple of out packet size */
1631                 bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
1632                 bytes_to_transfer *= pipe->max_packet;
1633         }
1634
1635         /*
1636          * Calculate the number of packets to transfer. If the length is zero
1637          * we still need to transfer one packet
1638          */
1639         packets_to_transfer = (bytes_to_transfer + pipe->max_packet - 1) /
1640                 pipe->max_packet;
1641         if (packets_to_transfer == 0)
1642                 packets_to_transfer = 1;
1643         else if ((packets_to_transfer > 1) &&
1644                         (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1645                 /*
1646                  * Limit to one packet when not using DMA. Channels must be
1647                  * restarted between every packet for IN transactions, so there
1648                  * is no reason to do multiple packets in a row
1649                  */
1650                 packets_to_transfer = 1;
1651                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1652         } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1653                 /*
1654                  * Limit the number of packet and data transferred to what the
1655                  * hardware can handle
1656                  */
1657                 packets_to_transfer = MAX_TRANSFER_PACKETS;
1658                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1659         }
1660
1661         usbc_hctsiz.s.xfersize = bytes_to_transfer;
1662         usbc_hctsiz.s.pktcnt = packets_to_transfer;
1663
1664         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index),
1665                                usbc_hctsiz.u32);
1666         return;
1667 }
1668
1669
1670 /**
1671  * Start a channel to perform the pipe's head transaction
1672  *
1673  * @usb:          USB device state populated by cvmx_usb_initialize().
1674  * @channel:      Channel to setup
1675  * @pipe:         Pipe to start
1676  */
1677 static void __cvmx_usb_start_channel(struct cvmx_usb_state *usb,
1678                                      int channel,
1679                                      struct cvmx_usb_pipe *pipe)
1680 {
1681         struct cvmx_usb_transaction *transaction =
1682                 list_first_entry(&pipe->transactions, typeof(*transaction),
1683                                  node);
1684
1685         /* Make sure all writes to the DMA region get flushed */
1686         CVMX_SYNCW;
1687
1688         /* Attach the channel to the pipe */
1689         usb->pipe_for_channel[channel] = pipe;
1690         pipe->channel = channel;
1691         pipe->flags |= __CVMX_USB_PIPE_FLAGS_SCHEDULED;
1692
1693         /* Mark this channel as in use */
1694         usb->idle_hardware_channels &= ~(1<<channel);
1695
1696         /* Enable the channel interrupt bits */
1697         {
1698                 union cvmx_usbcx_hcintx usbc_hcint;
1699                 union cvmx_usbcx_hcintmskx usbc_hcintmsk;
1700                 union cvmx_usbcx_haintmsk usbc_haintmsk;
1701
1702                 /* Clear all channel status bits */
1703                 usbc_hcint.u32 = __cvmx_usb_read_csr32(usb,
1704                                 CVMX_USBCX_HCINTX(channel, usb->index));
1705
1706                 __cvmx_usb_write_csr32(usb,
1707                                        CVMX_USBCX_HCINTX(channel, usb->index),
1708                                        usbc_hcint.u32);
1709
1710                 usbc_hcintmsk.u32 = 0;
1711                 usbc_hcintmsk.s.chhltdmsk = 1;
1712                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1713                         /*
1714                          * Channels need these extra interrupts when we aren't
1715                          * in DMA mode.
1716                          */
1717                         usbc_hcintmsk.s.datatglerrmsk = 1;
1718                         usbc_hcintmsk.s.frmovrunmsk = 1;
1719                         usbc_hcintmsk.s.bblerrmsk = 1;
1720                         usbc_hcintmsk.s.xacterrmsk = 1;
1721                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1722                                 /*
1723                                  * Splits don't generate xfercompl, so we need
1724                                  * ACK and NYET.
1725                                  */
1726                                 usbc_hcintmsk.s.nyetmsk = 1;
1727                                 usbc_hcintmsk.s.ackmsk = 1;
1728                         }
1729                         usbc_hcintmsk.s.nakmsk = 1;
1730                         usbc_hcintmsk.s.stallmsk = 1;
1731                         usbc_hcintmsk.s.xfercomplmsk = 1;
1732                 }
1733                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), usbc_hcintmsk.u32);
1734
1735                 /* Enable the channel interrupt to propagate */
1736                 usbc_haintmsk.u32 = __cvmx_usb_read_csr32(usb,
1737                                         CVMX_USBCX_HAINTMSK(usb->index));
1738                 usbc_haintmsk.s.haintmsk |= 1<<channel;
1739                 __cvmx_usb_write_csr32(usb,
1740                                         CVMX_USBCX_HAINTMSK(usb->index),
1741                                         usbc_haintmsk.u32);
1742         }
1743
1744         /* Setup the locations the DMA engines use  */
1745         {
1746                 uint64_t dma_address = transaction->buffer +
1747                                         transaction->actual_bytes;
1748
1749                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1750                         dma_address = transaction->buffer +
1751                                         transaction->iso_packets[0].offset +
1752                                         transaction->actual_bytes;
1753
1754                 __cvmx_usb_write_csr64(usb,
1755                         CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8,
1756                         dma_address);
1757
1758                 __cvmx_usb_write_csr64(usb,
1759                         CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8,
1760                         dma_address);
1761         }
1762
1763         /* Setup both the size of the transfer and the SPLIT characteristics */
1764         {
1765                 union cvmx_usbcx_hcspltx usbc_hcsplt = {.u32 = 0};
1766                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 = 0};
1767                 int packets_to_transfer;
1768                 int bytes_to_transfer = transaction->buffer_length -
1769                         transaction->actual_bytes;
1770
1771                 /*
1772                  * ISOCHRONOUS transactions store each individual transfer size
1773                  * in the packet structure, not the global buffer_length
1774                  */
1775                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1776                         bytes_to_transfer =
1777                                 transaction->iso_packets[0].length -
1778                                 transaction->actual_bytes;
1779
1780                 /*
1781                  * We need to do split transactions when we are talking to non
1782                  * high speed devices that are behind a high speed hub
1783                  */
1784                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1785                         /*
1786                          * On the start split phase (stage is even) record the
1787                          * frame number we will need to send the split complete.
1788                          * We only store the lower two bits since the time ahead
1789                          * can only be two frames
1790                          */
1791                         if ((transaction->stage&1) == 0) {
1792                                 if (transaction->type == CVMX_USB_TRANSFER_BULK)
1793                                         pipe->split_sc_frame =
1794                                                 (usb->frame_number + 1) & 0x7f;
1795                                 else
1796                                         pipe->split_sc_frame =
1797                                                 (usb->frame_number + 2) & 0x7f;
1798                         } else
1799                                 pipe->split_sc_frame = -1;
1800
1801                         usbc_hcsplt.s.spltena = 1;
1802                         usbc_hcsplt.s.hubaddr = pipe->hub_device_addr;
1803                         usbc_hcsplt.s.prtaddr = pipe->hub_port;
1804                         usbc_hcsplt.s.compsplt = (transaction->stage ==
1805                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE);
1806
1807                         /*
1808                          * SPLIT transactions can only ever transmit one data
1809                          * packet so limit the transfer size to the max packet
1810                          * size
1811                          */
1812                         if (bytes_to_transfer > pipe->max_packet)
1813                                 bytes_to_transfer = pipe->max_packet;
1814
1815                         /*
1816                          * ISOCHRONOUS OUT splits are unique in that they limit
1817                          * data transfers to 188 byte chunks representing the
1818                          * begin/middle/end of the data or all
1819                          */
1820                         if (!usbc_hcsplt.s.compsplt &&
1821                                 (pipe->transfer_dir ==
1822                                  CVMX_USB_DIRECTION_OUT) &&
1823                                 (pipe->transfer_type ==
1824                                  CVMX_USB_TRANSFER_ISOCHRONOUS)) {
1825                                 /*
1826                                  * Clear the split complete frame number as
1827                                  * there isn't going to be a split complete
1828                                  */
1829                                 pipe->split_sc_frame = -1;
1830                                 /*
1831                                  * See if we've started this transfer and sent
1832                                  * data
1833                                  */
1834                                 if (transaction->actual_bytes == 0) {
1835                                         /*
1836                                          * Nothing sent yet, this is either a
1837                                          * begin or the entire payload
1838                                          */
1839                                         if (bytes_to_transfer <= 188)
1840                                                 /* Entire payload in one go */
1841                                                 usbc_hcsplt.s.xactpos = 3;
1842                                         else
1843                                                 /* First part of payload */
1844                                                 usbc_hcsplt.s.xactpos = 2;
1845                                 } else {
1846                                         /*
1847                                          * Continuing the previous data, we must
1848                                          * either be in the middle or at the end
1849                                          */
1850                                         if (bytes_to_transfer <= 188)
1851                                                 /* End of payload */
1852                                                 usbc_hcsplt.s.xactpos = 1;
1853                                         else
1854                                                 /* Middle of payload */
1855                                                 usbc_hcsplt.s.xactpos = 0;
1856                                 }
1857                                 /*
1858                                  * Again, the transfer size is limited to 188
1859                                  * bytes
1860                                  */
1861                                 if (bytes_to_transfer > 188)
1862                                         bytes_to_transfer = 188;
1863                         }
1864                 }
1865
1866                 /*
1867                  * Make sure the transfer never exceeds the byte limit of the
1868                  * hardware. Further bytes will be sent as continued
1869                  * transactions
1870                  */
1871                 if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1872                         /*
1873                          * Round MAX_TRANSFER_BYTES to a multiple of out packet
1874                          * size
1875                          */
1876                         bytes_to_transfer = MAX_TRANSFER_BYTES /
1877                                 pipe->max_packet;
1878                         bytes_to_transfer *= pipe->max_packet;
1879                 }
1880
1881                 /*
1882                  * Calculate the number of packets to transfer. If the length is
1883                  * zero we still need to transfer one packet
1884                  */
1885                 packets_to_transfer =
1886                         (bytes_to_transfer + pipe->max_packet - 1) /
1887                         pipe->max_packet;
1888                 if (packets_to_transfer == 0)
1889                         packets_to_transfer = 1;
1890                 else if ((packets_to_transfer > 1) &&
1891                                 (usb->init_flags &
1892                                  CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1893                         /*
1894                          * Limit to one packet when not using DMA. Channels must
1895                          * be restarted between every packet for IN
1896                          * transactions, so there is no reason to do multiple
1897                          * packets in a row
1898                          */
1899                         packets_to_transfer = 1;
1900                         bytes_to_transfer = packets_to_transfer *
1901                                 pipe->max_packet;
1902                 } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1903                         /*
1904                          * Limit the number of packet and data transferred to
1905                          * what the hardware can handle
1906                          */
1907                         packets_to_transfer = MAX_TRANSFER_PACKETS;
1908                         bytes_to_transfer = packets_to_transfer *
1909                                 pipe->max_packet;
1910                 }
1911
1912                 usbc_hctsiz.s.xfersize = bytes_to_transfer;
1913                 usbc_hctsiz.s.pktcnt = packets_to_transfer;
1914
1915                 /* Update the DATA0/DATA1 toggle */
1916                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1917                 /*
1918                  * High speed pipes may need a hardware ping before they start
1919                  */
1920                 if (pipe->flags & __CVMX_USB_PIPE_FLAGS_NEED_PING)
1921                         usbc_hctsiz.s.dopng = 1;
1922
1923                 __cvmx_usb_write_csr32(usb,
1924                                        CVMX_USBCX_HCSPLTX(channel, usb->index),
1925                                        usbc_hcsplt.u32);
1926                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel,
1927                                         usb->index), usbc_hctsiz.u32);
1928         }
1929
1930         /* Setup the Host Channel Characteristics Register */
1931         {
1932                 union cvmx_usbcx_hccharx usbc_hcchar = {.u32 = 0};
1933
1934                 /*
1935                  * Set the startframe odd/even properly. This is only used for
1936                  * periodic
1937                  */
1938                 usbc_hcchar.s.oddfrm = usb->frame_number&1;
1939
1940                 /*
1941                  * Set the number of back to back packets allowed by this
1942                  * endpoint. Split transactions interpret "ec" as the number of
1943                  * immediate retries of failure. These retries happen too
1944                  * quickly, so we disable these entirely for splits
1945                  */
1946                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
1947                         usbc_hcchar.s.ec = 1;
1948                 else if (pipe->multi_count < 1)
1949                         usbc_hcchar.s.ec = 1;
1950                 else if (pipe->multi_count > 3)
1951                         usbc_hcchar.s.ec = 3;
1952                 else
1953                         usbc_hcchar.s.ec = pipe->multi_count;
1954
1955                 /* Set the rest of the endpoint specific settings */
1956                 usbc_hcchar.s.devaddr = pipe->device_addr;
1957                 usbc_hcchar.s.eptype = transaction->type;
1958                 usbc_hcchar.s.lspddev =
1959                         (pipe->device_speed == CVMX_USB_SPEED_LOW);
1960                 usbc_hcchar.s.epdir = pipe->transfer_dir;
1961                 usbc_hcchar.s.epnum = pipe->endpoint_num;
1962                 usbc_hcchar.s.mps = pipe->max_packet;
1963                 __cvmx_usb_write_csr32(usb,
1964                                        CVMX_USBCX_HCCHARX(channel, usb->index),
1965                                        usbc_hcchar.u32);
1966         }
1967
1968         /* Do transaction type specific fixups as needed */
1969         switch (transaction->type) {
1970         case CVMX_USB_TRANSFER_CONTROL:
1971                 __cvmx_usb_start_channel_control(usb, channel, pipe);
1972                 break;
1973         case CVMX_USB_TRANSFER_BULK:
1974         case CVMX_USB_TRANSFER_INTERRUPT:
1975                 break;
1976         case CVMX_USB_TRANSFER_ISOCHRONOUS:
1977                 if (!__cvmx_usb_pipe_needs_split(usb, pipe)) {
1978                         /*
1979                          * ISO transactions require different PIDs depending on
1980                          * direction and how many packets are needed
1981                          */
1982                         if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
1983                                 if (pipe->multi_count < 2) /* Need DATA0 */
1984                                         USB_SET_FIELD32(
1985                                                 CVMX_USBCX_HCTSIZX(channel,
1986                                                                    usb->index),
1987                                                 union cvmx_usbcx_hctsizx,
1988                                                 pid, 0);
1989                                 else /* Need MDATA */
1990                                         USB_SET_FIELD32(
1991                                                 CVMX_USBCX_HCTSIZX(channel,
1992                                                                    usb->index),
1993                                                 union cvmx_usbcx_hctsizx,
1994                                                 pid, 3);
1995                         }
1996                 }
1997                 break;
1998         }
1999         {
2000                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 =
2001                         __cvmx_usb_read_csr32(usb,
2002                                 CVMX_USBCX_HCTSIZX(channel, usb->index))};
2003                 transaction->xfersize = usbc_hctsiz.s.xfersize;
2004                 transaction->pktcnt = usbc_hctsiz.s.pktcnt;
2005         }
2006         /* Remeber when we start a split transaction */
2007         if (__cvmx_usb_pipe_needs_split(usb, pipe))
2008                 usb->active_split = transaction;
2009         USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
2010                         union cvmx_usbcx_hccharx, chena, 1);
2011         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
2012                 __cvmx_usb_fill_tx_fifo(usb, channel);
2013         return;
2014 }
2015
2016
2017 /**
2018  * Find a pipe that is ready to be scheduled to hardware.
2019  * @usb:         USB device state populated by cvmx_usb_initialize().
2020  * @list:        Pipe list to search
2021  * @current_frame:
2022  *               Frame counter to use as a time reference.
2023  *
2024  * Returns: Pipe or NULL if none are ready
2025  */
2026 static struct cvmx_usb_pipe *__cvmx_usb_find_ready_pipe(
2027                 struct cvmx_usb_state *usb,
2028                 struct list_head *list,
2029                 uint64_t current_frame)
2030 {
2031         struct cvmx_usb_pipe *pipe;
2032
2033         list_for_each_entry(pipe, list, node) {
2034                 struct cvmx_usb_transaction *t =
2035                         list_first_entry(&pipe->transactions, typeof(*t),
2036                                          node);
2037                 if (!(pipe->flags & __CVMX_USB_PIPE_FLAGS_SCHEDULED) && t &&
2038                         (pipe->next_tx_frame <= current_frame) &&
2039                         ((pipe->split_sc_frame == -1) ||
2040                          ((((int)current_frame - (int)pipe->split_sc_frame)
2041                            & 0x7f) < 0x40)) &&
2042                         (!usb->active_split || (usb->active_split == t))) {
2043                         prefetch(t);
2044                         return pipe;
2045                 }
2046         }
2047         return NULL;
2048 }
2049
2050
2051 /**
2052  * Called whenever a pipe might need to be scheduled to the
2053  * hardware.
2054  *
2055  * @usb:         USB device state populated by cvmx_usb_initialize().
2056  * @is_sof:      True if this schedule was called on a SOF interrupt.
2057  */
2058 static void __cvmx_usb_schedule(struct cvmx_usb_state *usb, int is_sof)
2059 {
2060         int channel;
2061         struct cvmx_usb_pipe *pipe;
2062         int need_sof;
2063         enum cvmx_usb_transfer ttype;
2064
2065         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
2066                 /*
2067                  * Without DMA we need to be careful to not schedule something
2068                  * at the end of a frame and cause an overrun.
2069                  */
2070                 union cvmx_usbcx_hfnum hfnum = {
2071                         .u32 = __cvmx_usb_read_csr32(usb,
2072                                                 CVMX_USBCX_HFNUM(usb->index))
2073                 };
2074
2075                 union cvmx_usbcx_hfir hfir = {
2076                         .u32 = __cvmx_usb_read_csr32(usb,
2077                                                 CVMX_USBCX_HFIR(usb->index))
2078                 };
2079
2080                 if (hfnum.s.frrem < hfir.s.frint/4)
2081                         goto done;
2082         }
2083
2084         while (usb->idle_hardware_channels) {
2085                 /* Find an idle channel */
2086                 channel = __fls(usb->idle_hardware_channels);
2087                 if (unlikely(channel > 7))
2088                         break;
2089
2090                 /* Find a pipe needing service */
2091                 pipe = NULL;
2092                 if (is_sof) {
2093                         /*
2094                          * Only process periodic pipes on SOF interrupts. This
2095                          * way we are sure that the periodic data is sent in the
2096                          * beginning of the frame
2097                          */
2098                         pipe = __cvmx_usb_find_ready_pipe(usb,
2099                                         usb->active_pipes +
2100                                         CVMX_USB_TRANSFER_ISOCHRONOUS,
2101                                         usb->frame_number);
2102                         if (likely(!pipe))
2103                                 pipe = __cvmx_usb_find_ready_pipe(usb,
2104                                                 usb->active_pipes +
2105                                                 CVMX_USB_TRANSFER_INTERRUPT,
2106                                                 usb->frame_number);
2107                 }
2108                 if (likely(!pipe)) {
2109                         pipe = __cvmx_usb_find_ready_pipe(usb,
2110                                         usb->active_pipes +
2111                                         CVMX_USB_TRANSFER_CONTROL,
2112                                         usb->frame_number);
2113                         if (likely(!pipe))
2114                                 pipe = __cvmx_usb_find_ready_pipe(usb,
2115                                                 usb->active_pipes +
2116                                                 CVMX_USB_TRANSFER_BULK,
2117                                                 usb->frame_number);
2118                 }
2119                 if (!pipe)
2120                         break;
2121
2122                 __cvmx_usb_start_channel(usb, channel, pipe);
2123         }
2124
2125 done:
2126         /*
2127          * Only enable SOF interrupts when we have transactions pending in the
2128          * future that might need to be scheduled
2129          */
2130         need_sof = 0;
2131         for (ttype = CVMX_USB_TRANSFER_CONTROL;
2132                         ttype <= CVMX_USB_TRANSFER_INTERRUPT; ttype++) {
2133                 list_for_each_entry(pipe, &usb->active_pipes[ttype], node) {
2134                         if (pipe->next_tx_frame > usb->frame_number) {
2135                                 need_sof = 1;
2136                                 break;
2137                         }
2138                 }
2139         }
2140         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
2141                         union cvmx_usbcx_gintmsk, sofmsk, need_sof);
2142         return;
2143 }
2144
2145 static inline struct octeon_hcd *cvmx_usb_to_octeon(struct cvmx_usb_state *p)
2146 {
2147         return container_of(p, struct octeon_hcd, usb);
2148 }
2149
2150 static inline struct usb_hcd *octeon_to_hcd(struct octeon_hcd *p)
2151 {
2152         return container_of((void *)p, struct usb_hcd, hcd_priv);
2153 }
2154
2155 static void octeon_usb_urb_complete_callback(struct cvmx_usb_state *usb,
2156                                              enum cvmx_usb_complete status,
2157                                              struct cvmx_usb_pipe *pipe,
2158                                              struct cvmx_usb_transaction
2159                                                 *transaction,
2160                                              int bytes_transferred,
2161                                              struct urb *urb)
2162 {
2163         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
2164         struct usb_hcd *hcd = octeon_to_hcd(priv);
2165         struct device *dev = hcd->self.controller;
2166
2167         urb->actual_length = bytes_transferred;
2168         urb->hcpriv = NULL;
2169
2170         if (!list_empty(&urb->urb_list))
2171                 /*
2172                  * It is on the dequeue_list, but we are going to call
2173                  * usb_hcd_giveback_urb(), so we must clear it from
2174                  * the list.  We got to it before the
2175                  * octeon_usb_urb_dequeue_work() tasklet did.
2176                  */
2177                 list_del_init(&urb->urb_list);
2178
2179         /* For Isochronous transactions we need to update the URB packet status
2180            list from data in our private copy */
2181         if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
2182                 int i;
2183                 /*
2184                  * The pointer to the private list is stored in the setup_packet
2185                  * field.
2186                  */
2187                 struct cvmx_usb_iso_packet *iso_packet =
2188                         (struct cvmx_usb_iso_packet *) urb->setup_packet;
2189                 /* Recalculate the transfer size by adding up each packet */
2190                 urb->actual_length = 0;
2191                 for (i = 0; i < urb->number_of_packets; i++) {
2192                         if (iso_packet[i].status ==
2193                                         CVMX_USB_COMPLETE_SUCCESS) {
2194                                 urb->iso_frame_desc[i].status = 0;
2195                                 urb->iso_frame_desc[i].actual_length =
2196                                         iso_packet[i].length;
2197                                 urb->actual_length +=
2198                                         urb->iso_frame_desc[i].actual_length;
2199                         } else {
2200                                 dev_dbg(dev, "ISOCHRONOUS packet=%d of %d status=%d pipe=%p transaction=%p size=%d\n",
2201                                         i, urb->number_of_packets,
2202                                         iso_packet[i].status, pipe,
2203                                         transaction, iso_packet[i].length);
2204                                 urb->iso_frame_desc[i].status = -EREMOTEIO;
2205                         }
2206                 }
2207                 /* Free the private list now that we don't need it anymore */
2208                 kfree(iso_packet);
2209                 urb->setup_packet = NULL;
2210         }
2211
2212         switch (status) {
2213         case CVMX_USB_COMPLETE_SUCCESS:
2214                 urb->status = 0;
2215                 break;
2216         case CVMX_USB_COMPLETE_CANCEL:
2217                 if (urb->status == 0)
2218                         urb->status = -ENOENT;
2219                 break;
2220         case CVMX_USB_COMPLETE_STALL:
2221                 dev_dbg(dev, "status=stall pipe=%p transaction=%p size=%d\n",
2222                         pipe, transaction, bytes_transferred);
2223                 urb->status = -EPIPE;
2224                 break;
2225         case CVMX_USB_COMPLETE_BABBLEERR:
2226                 dev_dbg(dev, "status=babble pipe=%p transaction=%p size=%d\n",
2227                         pipe, transaction, bytes_transferred);
2228                 urb->status = -EPIPE;
2229                 break;
2230         case CVMX_USB_COMPLETE_SHORT:
2231                 dev_dbg(dev, "status=short pipe=%p transaction=%p size=%d\n",
2232                         pipe, transaction, bytes_transferred);
2233                 urb->status = -EREMOTEIO;
2234                 break;
2235         case CVMX_USB_COMPLETE_ERROR:
2236         case CVMX_USB_COMPLETE_XACTERR:
2237         case CVMX_USB_COMPLETE_DATATGLERR:
2238         case CVMX_USB_COMPLETE_FRAMEERR:
2239                 dev_dbg(dev, "status=%d pipe=%p transaction=%p size=%d\n",
2240                         status, pipe, transaction, bytes_transferred);
2241                 urb->status = -EPROTO;
2242                 break;
2243         }
2244         spin_unlock(&priv->lock);
2245         usb_hcd_giveback_urb(octeon_to_hcd(priv), urb, urb->status);
2246         spin_lock(&priv->lock);
2247 }
2248
2249 /**
2250  * Signal the completion of a transaction and free it. The
2251  * transaction will be removed from the pipe transaction list.
2252  *
2253  * @usb:         USB device state populated by cvmx_usb_initialize().
2254  * @pipe:        Pipe the transaction is on
2255  * @transaction:
2256  *               Transaction that completed
2257  * @complete_code:
2258  *               Completion code
2259  */
2260 static void __cvmx_usb_perform_complete(
2261                                 struct cvmx_usb_state *usb,
2262                                 struct cvmx_usb_pipe *pipe,
2263                                 struct cvmx_usb_transaction *transaction,
2264                                 enum cvmx_usb_complete complete_code)
2265 {
2266         /* If this was a split then clear our split in progress marker */
2267         if (usb->active_split == transaction)
2268                 usb->active_split = NULL;
2269
2270         /*
2271          * Isochronous transactions need extra processing as they might not be
2272          * done after a single data transfer
2273          */
2274         if (unlikely(transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
2275                 /* Update the number of bytes transferred in this ISO packet */
2276                 transaction->iso_packets[0].length = transaction->actual_bytes;
2277                 transaction->iso_packets[0].status = complete_code;
2278
2279                 /*
2280                  * If there are more ISOs pending and we succeeded, schedule the
2281                  * next one
2282                  */
2283                 if ((transaction->iso_number_packets > 1) &&
2284                         (complete_code == CVMX_USB_COMPLETE_SUCCESS)) {
2285                         /* No bytes transferred for this packet as of yet */
2286                         transaction->actual_bytes = 0;
2287                         /* One less ISO waiting to transfer */
2288                         transaction->iso_number_packets--;
2289                         /* Increment to the next location in our packet array */
2290                         transaction->iso_packets++;
2291                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2292                         goto done;
2293                 }
2294         }
2295
2296         /* Remove the transaction from the pipe list */
2297         list_del(&transaction->node);
2298         if (list_empty(&pipe->transactions))
2299                 list_move_tail(&pipe->node, &usb->idle_pipes);
2300         octeon_usb_urb_complete_callback(usb, complete_code, pipe,
2301                                          transaction,
2302                                          transaction->actual_bytes,
2303                                          transaction->urb);
2304         kfree(transaction);
2305 done:
2306         return;
2307 }
2308
2309
2310 /**
2311  * Submit a usb transaction to a pipe. Called for all types
2312  * of transactions.
2313  *
2314  * @usb:
2315  * @pipe:           Which pipe to submit to.
2316  * @type:           Transaction type
2317  * @buffer:         User buffer for the transaction
2318  * @buffer_length:
2319  *                  User buffer's length in bytes
2320  * @control_header:
2321  *                  For control transactions, the 8 byte standard header
2322  * @iso_start_frame:
2323  *                  For ISO transactions, the start frame
2324  * @iso_number_packets:
2325  *                  For ISO, the number of packet in the transaction.
2326  * @iso_packets:
2327  *                  A description of each ISO packet
2328  * @urb:            URB for the callback
2329  *
2330  * Returns: Transaction or NULL on failure.
2331  */
2332 static struct cvmx_usb_transaction *__cvmx_usb_submit_transaction(
2333                                 struct cvmx_usb_state *usb,
2334                                 struct cvmx_usb_pipe *pipe,
2335                                 enum cvmx_usb_transfer type,
2336                                 uint64_t buffer,
2337                                 int buffer_length,
2338                                 uint64_t control_header,
2339                                 int iso_start_frame,
2340                                 int iso_number_packets,
2341                                 struct cvmx_usb_iso_packet *iso_packets,
2342                                 struct urb *urb)
2343 {
2344         struct cvmx_usb_transaction *transaction;
2345
2346         if (unlikely(pipe->transfer_type != type))
2347                 return NULL;
2348
2349         transaction = kzalloc(sizeof(*transaction), GFP_ATOMIC);
2350         if (unlikely(!transaction))
2351                 return NULL;
2352
2353         transaction->type = type;
2354         transaction->buffer = buffer;
2355         transaction->buffer_length = buffer_length;
2356         transaction->control_header = control_header;
2357         /* FIXME: This is not used, implement it. */
2358         transaction->iso_start_frame = iso_start_frame;
2359         transaction->iso_number_packets = iso_number_packets;
2360         transaction->iso_packets = iso_packets;
2361         transaction->urb = urb;
2362         if (transaction->type == CVMX_USB_TRANSFER_CONTROL)
2363                 transaction->stage = CVMX_USB_STAGE_SETUP;
2364         else
2365                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2366
2367         if (!list_empty(&pipe->transactions)) {
2368                 list_add_tail(&transaction->node, &pipe->transactions);
2369         } else {
2370                 list_add_tail(&transaction->node, &pipe->transactions);
2371                 list_move_tail(&pipe->node,
2372                                &usb->active_pipes[pipe->transfer_type]);
2373
2374                 /*
2375                  * We may need to schedule the pipe if this was the head of the
2376                  * pipe.
2377                  */
2378                 __cvmx_usb_schedule(usb, 0);
2379         }
2380
2381         return transaction;
2382 }
2383
2384
2385 /**
2386  * Call to submit a USB Bulk transfer to a pipe.
2387  *
2388  * @usb:            USB device state populated by cvmx_usb_initialize().
2389  * @pipe:           Handle to the pipe for the transfer.
2390  * @urb:            URB.
2391  *
2392  * Returns: A submitted transaction or NULL on failure.
2393  */
2394 static struct cvmx_usb_transaction *cvmx_usb_submit_bulk(
2395                                                 struct cvmx_usb_state *usb,
2396                                                 struct cvmx_usb_pipe *pipe,
2397                                                 struct urb *urb)
2398 {
2399         return __cvmx_usb_submit_transaction(usb, pipe, CVMX_USB_TRANSFER_BULK,
2400                                              urb->transfer_dma,
2401                                              urb->transfer_buffer_length,
2402                                              0, /* control_header */
2403                                              0, /* iso_start_frame */
2404                                              0, /* iso_number_packets */
2405                                              NULL, /* iso_packets */
2406                                              urb);
2407 }
2408
2409
2410 /**
2411  * Call to submit a USB Interrupt transfer to a pipe.
2412  *
2413  * @usb:            USB device state populated by cvmx_usb_initialize().
2414  * @pipe:           Handle to the pipe for the transfer.
2415  * @urb:            URB returned when the callback is called.
2416  *
2417  * Returns: A submitted transaction or NULL on failure.
2418  */
2419 static struct cvmx_usb_transaction *cvmx_usb_submit_interrupt(
2420                                                 struct cvmx_usb_state *usb,
2421                                                 struct cvmx_usb_pipe *pipe,
2422                                                 struct urb *urb)
2423 {
2424         return __cvmx_usb_submit_transaction(usb, pipe,
2425                                              CVMX_USB_TRANSFER_INTERRUPT,
2426                                              urb->transfer_dma,
2427                                              urb->transfer_buffer_length,
2428                                              0, /* control_header */
2429                                              0, /* iso_start_frame */
2430                                              0, /* iso_number_packets */
2431                                              NULL, /* iso_packets */
2432                                              urb);
2433 }
2434
2435
2436 /**
2437  * Call to submit a USB Control transfer to a pipe.
2438  *
2439  * @usb:            USB device state populated by cvmx_usb_initialize().
2440  * @pipe:           Handle to the pipe for the transfer.
2441  * @urb:            URB.
2442  *
2443  * Returns: A submitted transaction or NULL on failure.
2444  */
2445 static struct cvmx_usb_transaction *cvmx_usb_submit_control(
2446                                                 struct cvmx_usb_state *usb,
2447                                                 struct cvmx_usb_pipe *pipe,
2448                                                 struct urb *urb)
2449 {
2450         int buffer_length = urb->transfer_buffer_length;
2451         uint64_t control_header = urb->setup_dma;
2452         union cvmx_usb_control_header *header =
2453                 cvmx_phys_to_ptr(control_header);
2454
2455         if ((header->s.request_type & 0x80) == 0)
2456                 buffer_length = le16_to_cpu(header->s.length);
2457
2458         return __cvmx_usb_submit_transaction(usb, pipe,
2459                                              CVMX_USB_TRANSFER_CONTROL,
2460                                              urb->transfer_dma, buffer_length,
2461                                              control_header,
2462                                              0, /* iso_start_frame */
2463                                              0, /* iso_number_packets */
2464                                              NULL, /* iso_packets */
2465                                              urb);
2466 }
2467
2468
2469 /**
2470  * Call to submit a USB Isochronous transfer to a pipe.
2471  *
2472  * @usb:            USB device state populated by cvmx_usb_initialize().
2473  * @pipe:           Handle to the pipe for the transfer.
2474  * @urb:            URB returned when the callback is called.
2475  *
2476  * Returns: A submitted transaction or NULL on failure.
2477  */
2478 static struct cvmx_usb_transaction *cvmx_usb_submit_isochronous(
2479                                                 struct cvmx_usb_state *usb,
2480                                                 struct cvmx_usb_pipe *pipe,
2481                                                 struct urb *urb)
2482 {
2483         struct cvmx_usb_iso_packet *packets;
2484
2485         packets = (struct cvmx_usb_iso_packet *) urb->setup_packet;
2486         return __cvmx_usb_submit_transaction(usb, pipe,
2487                                              CVMX_USB_TRANSFER_ISOCHRONOUS,
2488                                              urb->transfer_dma,
2489                                              urb->transfer_buffer_length,
2490                                              0, /* control_header */
2491                                              urb->start_frame,
2492                                              urb->number_of_packets,
2493                                              packets, urb);
2494 }
2495
2496
2497 /**
2498  * Cancel one outstanding request in a pipe. Canceling a request
2499  * can fail if the transaction has already completed before cancel
2500  * is called. Even after a successful cancel call, it may take
2501  * a frame or two for the cvmx_usb_poll() function to call the
2502  * associated callback.
2503  *
2504  * @usb:         USB device state populated by cvmx_usb_initialize().
2505  * @pipe:        Pipe to cancel requests in.
2506  * @transaction: Transaction to cancel, returned by the submit function.
2507  *
2508  * Returns: 0 or a negative error code.
2509  */
2510 static int cvmx_usb_cancel(struct cvmx_usb_state *usb,
2511                            struct cvmx_usb_pipe *pipe,
2512                            struct cvmx_usb_transaction *transaction)
2513 {
2514         /*
2515          * If the transaction is the HEAD of the queue and scheduled. We need to
2516          * treat it special
2517          */
2518         if (list_first_entry(&pipe->transactions, typeof(*transaction), node) ==
2519             transaction && (pipe->flags & __CVMX_USB_PIPE_FLAGS_SCHEDULED)) {
2520                 union cvmx_usbcx_hccharx usbc_hcchar;
2521
2522                 usb->pipe_for_channel[pipe->channel] = NULL;
2523                 pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_SCHEDULED;
2524
2525                 CVMX_SYNCW;
2526
2527                 usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb,
2528                                 CVMX_USBCX_HCCHARX(pipe->channel, usb->index));
2529                 /*
2530                  * If the channel isn't enabled then the transaction already
2531                  * completed.
2532                  */
2533                 if (usbc_hcchar.s.chena) {
2534                         usbc_hcchar.s.chdis = 1;
2535                         __cvmx_usb_write_csr32(usb,
2536                                         CVMX_USBCX_HCCHARX(pipe->channel,
2537                                                 usb->index),
2538                                         usbc_hcchar.u32);
2539                 }
2540         }
2541         __cvmx_usb_perform_complete(usb, pipe, transaction,
2542                                     CVMX_USB_COMPLETE_CANCEL);
2543         return 0;
2544 }
2545
2546
2547 /**
2548  * Cancel all outstanding requests in a pipe. Logically all this
2549  * does is call cvmx_usb_cancel() in a loop.
2550  *
2551  * @usb:         USB device state populated by cvmx_usb_initialize().
2552  * @pipe:        Pipe to cancel requests in.
2553  *
2554  * Returns: 0 or a negative error code.
2555  */
2556 static int cvmx_usb_cancel_all(struct cvmx_usb_state *usb,
2557                                struct cvmx_usb_pipe *pipe)
2558 {
2559         struct cvmx_usb_transaction *transaction, *next;
2560
2561         /* Simply loop through and attempt to cancel each transaction */
2562         list_for_each_entry_safe(transaction, next, &pipe->transactions, node) {
2563                 int result = cvmx_usb_cancel(usb, pipe, transaction);
2564
2565                 if (unlikely(result != 0))
2566                         return result;
2567         }
2568         return 0;
2569 }
2570
2571
2572 /**
2573  * Close a pipe created with cvmx_usb_open_pipe().
2574  *
2575  * @usb:         USB device state populated by cvmx_usb_initialize().
2576  * @pipe:        Pipe to close.
2577  *
2578  * Returns: 0 or a negative error code. EBUSY is returned if the pipe has
2579  *          outstanding transfers.
2580  */
2581 static int cvmx_usb_close_pipe(struct cvmx_usb_state *usb,
2582                                struct cvmx_usb_pipe *pipe)
2583 {
2584         /* Fail if the pipe has pending transactions */
2585         if (!list_empty(&pipe->transactions))
2586                 return -EBUSY;
2587
2588         list_del(&pipe->node);
2589         kfree(pipe);
2590
2591         return 0;
2592 }
2593
2594 /**
2595  * Get the current USB protocol level frame number. The frame
2596  * number is always in the range of 0-0x7ff.
2597  *
2598  * @usb: USB device state populated by cvmx_usb_initialize().
2599  *
2600  * Returns: USB frame number
2601  */
2602 static int cvmx_usb_get_frame_number(struct cvmx_usb_state *usb)
2603 {
2604         int frame_number;
2605         union cvmx_usbcx_hfnum usbc_hfnum;
2606
2607         usbc_hfnum.u32 = __cvmx_usb_read_csr32(usb,
2608                         CVMX_USBCX_HFNUM(usb->index));
2609         frame_number = usbc_hfnum.s.frnum;
2610
2611         return frame_number;
2612 }
2613
2614
2615 /**
2616  * Poll a channel for status
2617  *
2618  * @usb:     USB device
2619  * @channel: Channel to poll
2620  *
2621  * Returns: Zero on success
2622  */
2623 static int __cvmx_usb_poll_channel(struct cvmx_usb_state *usb, int channel)
2624 {
2625         union cvmx_usbcx_hcintx usbc_hcint;
2626         union cvmx_usbcx_hctsizx usbc_hctsiz;
2627         union cvmx_usbcx_hccharx usbc_hcchar;
2628         struct cvmx_usb_pipe *pipe;
2629         struct cvmx_usb_transaction *transaction;
2630         int bytes_this_transfer;
2631         int bytes_in_last_packet;
2632         int packets_processed;
2633         int buffer_space_left;
2634
2635         /* Read the interrupt status bits for the channel */
2636         usbc_hcint.u32 = __cvmx_usb_read_csr32(usb,
2637                         CVMX_USBCX_HCINTX(channel, usb->index));
2638
2639         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
2640                 usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb,
2641                                 CVMX_USBCX_HCCHARX(channel, usb->index));
2642
2643                 if (usbc_hcchar.s.chena && usbc_hcchar.s.chdis) {
2644                         /*
2645                          * There seems to be a bug in CN31XX which can cause
2646                          * interrupt IN transfers to get stuck until we do a
2647                          * write of HCCHARX without changing things
2648                          */
2649                         __cvmx_usb_write_csr32(usb,
2650                                         CVMX_USBCX_HCCHARX(channel,
2651                                                            usb->index),
2652                                         usbc_hcchar.u32);
2653                         return 0;
2654                 }
2655
2656                 /*
2657                  * In non DMA mode the channels don't halt themselves. We need
2658                  * to manually disable channels that are left running
2659                  */
2660                 if (!usbc_hcint.s.chhltd) {
2661                         if (usbc_hcchar.s.chena) {
2662                                 union cvmx_usbcx_hcintmskx hcintmsk;
2663                                 /* Disable all interrupts except CHHLTD */
2664                                 hcintmsk.u32 = 0;
2665                                 hcintmsk.s.chhltdmsk = 1;
2666                                 __cvmx_usb_write_csr32(usb,
2667                                                 CVMX_USBCX_HCINTMSKX(channel,
2668                                                         usb->index),
2669                                                 hcintmsk.u32);
2670                                 usbc_hcchar.s.chdis = 1;
2671                                 __cvmx_usb_write_csr32(usb,
2672                                                 CVMX_USBCX_HCCHARX(channel,
2673                                                         usb->index),
2674                                                 usbc_hcchar.u32);
2675                                 return 0;
2676                         } else if (usbc_hcint.s.xfercompl) {
2677                                 /*
2678                                  * Successful IN/OUT with transfer complete.
2679                                  * Channel halt isn't needed.
2680                                  */
2681                         } else {
2682                                 cvmx_dprintf("USB%d: Channel %d interrupt without halt\n",
2683                                                 usb->index, channel);
2684                                 return 0;
2685                         }
2686                 }
2687         } else {
2688                 /*
2689                  * There is are no interrupts that we need to process when the
2690                  * channel is still running
2691                  */
2692                 if (!usbc_hcint.s.chhltd)
2693                         return 0;
2694         }
2695
2696         /* Disable the channel interrupts now that it is done */
2697         __cvmx_usb_write_csr32(usb,
2698                                 CVMX_USBCX_HCINTMSKX(channel, usb->index),
2699                                 0);
2700         usb->idle_hardware_channels |= (1<<channel);
2701
2702         /* Make sure this channel is tied to a valid pipe */
2703         pipe = usb->pipe_for_channel[channel];
2704         prefetch(pipe);
2705         if (!pipe)
2706                 return 0;
2707         transaction = list_first_entry(&pipe->transactions,
2708                                        typeof(*transaction),
2709                                        node);
2710         prefetch(transaction);
2711
2712         /*
2713          * Disconnect this pipe from the HW channel. Later the schedule
2714          * function will figure out which pipe needs to go
2715          */
2716         usb->pipe_for_channel[channel] = NULL;
2717         pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_SCHEDULED;
2718
2719         /*
2720          * Read the channel config info so we can figure out how much data
2721          * transfered
2722          */
2723         usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb,
2724                         CVMX_USBCX_HCCHARX(channel, usb->index));
2725         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb,
2726                         CVMX_USBCX_HCTSIZX(channel, usb->index));
2727
2728         /*
2729          * Calculating the number of bytes successfully transferred is dependent
2730          * on the transfer direction
2731          */
2732         packets_processed = transaction->pktcnt - usbc_hctsiz.s.pktcnt;
2733         if (usbc_hcchar.s.epdir) {
2734                 /*
2735                  * IN transactions are easy. For every byte received the
2736                  * hardware decrements xfersize. All we need to do is subtract
2737                  * the current value of xfersize from its starting value and we
2738                  * know how many bytes were written to the buffer
2739                  */
2740                 bytes_this_transfer = transaction->xfersize -
2741                         usbc_hctsiz.s.xfersize;
2742         } else {
2743                 /*
2744                  * OUT transaction don't decrement xfersize. Instead pktcnt is
2745                  * decremented on every successful packet send. The hardware
2746                  * does this when it receives an ACK, or NYET. If it doesn't
2747                  * receive one of these responses pktcnt doesn't change
2748                  */
2749                 bytes_this_transfer = packets_processed * usbc_hcchar.s.mps;
2750                 /*
2751                  * The last packet may not be a full transfer if we didn't have
2752                  * enough data
2753                  */
2754                 if (bytes_this_transfer > transaction->xfersize)
2755                         bytes_this_transfer = transaction->xfersize;
2756         }
2757         /* Figure out how many bytes were in the last packet of the transfer */
2758         if (packets_processed)
2759                 bytes_in_last_packet = bytes_this_transfer -
2760                         (packets_processed - 1) * usbc_hcchar.s.mps;
2761         else
2762                 bytes_in_last_packet = bytes_this_transfer;
2763
2764         /*
2765          * As a special case, setup transactions output the setup header, not
2766          * the user's data. For this reason we don't count setup data as bytes
2767          * transferred
2768          */
2769         if ((transaction->stage == CVMX_USB_STAGE_SETUP) ||
2770                 (transaction->stage == CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE))
2771                 bytes_this_transfer = 0;
2772
2773         /*
2774          * Add the bytes transferred to the running total. It is important that
2775          * bytes_this_transfer doesn't count any data that needs to be
2776          * retransmitted
2777          */
2778         transaction->actual_bytes += bytes_this_transfer;
2779         if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
2780                 buffer_space_left = transaction->iso_packets[0].length -
2781                         transaction->actual_bytes;
2782         else
2783                 buffer_space_left = transaction->buffer_length -
2784                         transaction->actual_bytes;
2785
2786         /*
2787          * We need to remember the PID toggle state for the next transaction.
2788          * The hardware already updated it for the next transaction
2789          */
2790         pipe->pid_toggle = !(usbc_hctsiz.s.pid == 0);
2791
2792         /*
2793          * For high speed bulk out, assume the next transaction will need to do
2794          * a ping before proceeding. If this isn't true the ACK processing below
2795          * will clear this flag
2796          */
2797         if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2798                 (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
2799                 (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT))
2800                 pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
2801
2802         if (usbc_hcint.s.stall) {
2803                 /*
2804                  * STALL as a response means this transaction cannot be
2805                  * completed because the device can't process transactions. Tell
2806                  * the user. Any data that was transferred will be counted on
2807                  * the actual bytes transferred
2808                  */
2809                 pipe->pid_toggle = 0;
2810                 __cvmx_usb_perform_complete(usb, pipe, transaction,
2811                                             CVMX_USB_COMPLETE_STALL);
2812         } else if (usbc_hcint.s.xacterr) {
2813                 /*
2814                  * We know at least one packet worked if we get a ACK or NAK.
2815                  * Reset the retry counter
2816                  */
2817                 if (usbc_hcint.s.nak || usbc_hcint.s.ack)
2818                         transaction->retries = 0;
2819                 transaction->retries++;
2820                 if (transaction->retries > MAX_RETRIES) {
2821                         /*
2822                          * XactErr as a response means the device signaled
2823                          * something wrong with the transfer. For example, PID
2824                          * toggle errors cause these
2825                          */
2826                         __cvmx_usb_perform_complete(usb, pipe, transaction,
2827                                                     CVMX_USB_COMPLETE_XACTERR);
2828                 } else {
2829                         /*
2830                          * If this was a split then clear our split in progress
2831                          * marker
2832                          */
2833                         if (usb->active_split == transaction)
2834                                 usb->active_split = NULL;
2835                         /*
2836                          * Rewind to the beginning of the transaction by anding
2837                          * off the split complete bit
2838                          */
2839                         transaction->stage &= ~1;
2840                         pipe->split_sc_frame = -1;
2841                         pipe->next_tx_frame += pipe->interval;
2842                         if (pipe->next_tx_frame < usb->frame_number)
2843                                 pipe->next_tx_frame =
2844                                         usb->frame_number + pipe->interval -
2845                                         (usb->frame_number -
2846                                          pipe->next_tx_frame) % pipe->interval;
2847                 }
2848         } else if (usbc_hcint.s.bblerr) {
2849                 /* Babble Error (BblErr) */
2850                 __cvmx_usb_perform_complete(usb, pipe, transaction,
2851                                             CVMX_USB_COMPLETE_BABBLEERR);
2852         } else if (usbc_hcint.s.datatglerr) {
2853                 /* We'll retry the exact same transaction again */
2854                 transaction->retries++;
2855         } else if (usbc_hcint.s.nyet) {
2856                 /*
2857                  * NYET as a response is only allowed in three cases: as a
2858                  * response to a ping, as a response to a split transaction, and
2859                  * as a response to a bulk out. The ping case is handled by
2860                  * hardware, so we only have splits and bulk out
2861                  */
2862                 if (!__cvmx_usb_pipe_needs_split(usb, pipe)) {
2863                         transaction->retries = 0;
2864                         /*
2865                          * If there is more data to go then we need to try
2866                          * again. Otherwise this transaction is complete
2867                          */
2868                         if ((buffer_space_left == 0) ||
2869                                 (bytes_in_last_packet < pipe->max_packet))
2870                                 __cvmx_usb_perform_complete(usb, pipe,
2871                                                 transaction,
2872                                                 CVMX_USB_COMPLETE_SUCCESS);
2873                 } else {
2874                         /*
2875                          * Split transactions retry the split complete 4 times
2876                          * then rewind to the start split and do the entire
2877                          * transactions again
2878                          */
2879                         transaction->retries++;
2880                         if ((transaction->retries & 0x3) == 0) {
2881                                 /*
2882                                  * Rewind to the beginning of the transaction by
2883                                  * anding off the split complete bit
2884                                  */
2885                                 transaction->stage &= ~1;
2886                                 pipe->split_sc_frame = -1;
2887                         }
2888                 }
2889         } else if (usbc_hcint.s.ack) {
2890                 transaction->retries = 0;
2891                 /*
2892                  * The ACK bit can only be checked after the other error bits.
2893                  * This is because a multi packet transfer may succeed in a
2894                  * number of packets and then get a different response on the
2895                  * last packet. In this case both ACK and the last response bit
2896                  * will be set. If none of the other response bits is set, then
2897                  * the last packet must have been an ACK
2898                  *
2899                  * Since we got an ACK, we know we don't need to do a ping on
2900                  * this pipe
2901                  */
2902                 pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_NEED_PING;
2903
2904                 switch (transaction->type) {
2905                 case CVMX_USB_TRANSFER_CONTROL:
2906                         switch (transaction->stage) {
2907                         case CVMX_USB_STAGE_NON_CONTROL:
2908                         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
2909                                 /* This should be impossible */
2910                                 __cvmx_usb_perform_complete(usb, pipe,
2911                                         transaction, CVMX_USB_COMPLETE_ERROR);
2912                                 break;
2913                         case CVMX_USB_STAGE_SETUP:
2914                                 pipe->pid_toggle = 1;
2915                                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
2916                                         transaction->stage =
2917                                                 CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE;
2918                                 else {
2919                                         union cvmx_usb_control_header *header =
2920                                                 cvmx_phys_to_ptr(transaction->control_header);
2921                                         if (header->s.length)
2922                                                 transaction->stage = CVMX_USB_STAGE_DATA;
2923                                         else
2924                                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2925                                 }
2926                                 break;
2927                         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
2928                                 {
2929                                         union cvmx_usb_control_header *header =
2930                                                 cvmx_phys_to_ptr(transaction->control_header);
2931                                         if (header->s.length)
2932                                                 transaction->stage = CVMX_USB_STAGE_DATA;
2933                                         else
2934                                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2935                                 }
2936                                 break;
2937                         case CVMX_USB_STAGE_DATA:
2938                                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2939                                         transaction->stage =
2940                                                 CVMX_USB_STAGE_DATA_SPLIT_COMPLETE;
2941                                         /*
2942                                          * For setup OUT data that are splits,
2943                                          * the hardware doesn't appear to count
2944                                          * transferred data. Here we manually
2945                                          * update the data transferred
2946                                          */
2947                                         if (!usbc_hcchar.s.epdir) {
2948                                                 if (buffer_space_left < pipe->max_packet)
2949                                                         transaction->actual_bytes +=
2950                                                                 buffer_space_left;
2951                                                 else
2952                                                         transaction->actual_bytes +=
2953                                                                 pipe->max_packet;
2954                                         }
2955                                 } else if ((buffer_space_left == 0) ||
2956                                                 (bytes_in_last_packet <
2957                                                  pipe->max_packet)) {
2958                                         pipe->pid_toggle = 1;
2959                                         transaction->stage =
2960                                                 CVMX_USB_STAGE_STATUS;
2961                                 }
2962                                 break;
2963                         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
2964                                 if ((buffer_space_left == 0) ||
2965                                                 (bytes_in_last_packet <
2966                                                  pipe->max_packet)) {
2967                                         pipe->pid_toggle = 1;
2968                                         transaction->stage =
2969                                                 CVMX_USB_STAGE_STATUS;
2970                                 } else {
2971                                         transaction->stage =
2972                                                 CVMX_USB_STAGE_DATA;
2973                                 }
2974                                 break;
2975                         case CVMX_USB_STAGE_STATUS:
2976                                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
2977                                         transaction->stage =
2978                                                 CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE;
2979                                 else
2980                                         __cvmx_usb_perform_complete(usb, pipe,
2981                                                 transaction,
2982                                                 CVMX_USB_COMPLETE_SUCCESS);
2983                                 break;
2984                         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
2985                                 __cvmx_usb_perform_complete(usb, pipe,
2986                                                 transaction,
2987                                                 CVMX_USB_COMPLETE_SUCCESS);
2988                                 break;
2989                         }
2990                         break;
2991                 case CVMX_USB_TRANSFER_BULK:
2992                 case CVMX_USB_TRANSFER_INTERRUPT:
2993                         /*
2994                          * The only time a bulk transfer isn't complete when it
2995                          * finishes with an ACK is during a split transaction.
2996                          * For splits we need to continue the transfer if more
2997                          * data is needed
2998                          */
2999                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
3000                                 if (transaction->stage ==
3001                                                 CVMX_USB_STAGE_NON_CONTROL)
3002                                         transaction->stage =
3003                                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
3004                                 else {
3005                                         if (buffer_space_left &&
3006                                                 (bytes_in_last_packet ==
3007                                                  pipe->max_packet))
3008                                                 transaction->stage =
3009                                                         CVMX_USB_STAGE_NON_CONTROL;
3010                                         else {
3011                                                 if (transaction->type ==
3012                                                         CVMX_USB_TRANSFER_INTERRUPT)
3013                                                         pipe->next_tx_frame +=
3014                                                                 pipe->interval;
3015                                                         __cvmx_usb_perform_complete(
3016                                                                 usb,
3017                                                                 pipe,
3018                                                                 transaction,
3019                                                                 CVMX_USB_COMPLETE_SUCCESS);
3020                                         }
3021                                 }
3022                         } else {
3023                                 if ((pipe->device_speed ==
3024                                         CVMX_USB_SPEED_HIGH) &&
3025                                     (pipe->transfer_type ==
3026                                      CVMX_USB_TRANSFER_BULK) &&
3027                                     (pipe->transfer_dir ==
3028                                      CVMX_USB_DIRECTION_OUT) &&
3029                                     (usbc_hcint.s.nak))
3030                                         pipe->flags |=
3031                                                 __CVMX_USB_PIPE_FLAGS_NEED_PING;
3032                                 if (!buffer_space_left ||
3033                                         (bytes_in_last_packet <
3034                                          pipe->max_packet)) {
3035                                         if (transaction->type ==
3036                                                 CVMX_USB_TRANSFER_INTERRUPT)
3037                                                 pipe->next_tx_frame +=
3038                                                         pipe->interval;
3039                                         __cvmx_usb_perform_complete(usb,
3040                                                 pipe,
3041                                                 transaction,
3042                                                 CVMX_USB_COMPLETE_SUCCESS);
3043                                 }
3044                         }
3045                         break;
3046                 case CVMX_USB_TRANSFER_ISOCHRONOUS:
3047                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
3048                                 /*
3049                                  * ISOCHRONOUS OUT splits don't require a
3050                                  * complete split stage. Instead they use a
3051                                  * sequence of begin OUT splits to transfer the
3052                                  * data 188 bytes at a time. Once the transfer
3053                                  * is complete, the pipe sleeps until the next
3054                                  * schedule interval
3055                                  */
3056                                 if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
3057                                         /*
3058                                          * If no space left or this wasn't a max
3059                                          * size packet then this transfer is
3060                                          * complete. Otherwise start it again to
3061                                          * send the next 188 bytes
3062                                          */
3063                                         if (!buffer_space_left ||
3064                                                 (bytes_this_transfer < 188)) {
3065                                                 pipe->next_tx_frame +=
3066                                                         pipe->interval;
3067                                                 __cvmx_usb_perform_complete(
3068                                                         usb,
3069                                                         pipe,
3070                                                         transaction,
3071                                                         CVMX_USB_COMPLETE_SUCCESS);
3072                                         }
3073                                 } else {
3074                                         if (transaction->stage ==
3075                                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE) {
3076                                                 /*
3077                                                  * We are in the incoming data
3078                                                  * phase. Keep getting data
3079                                                  * until we run out of space or
3080                                                  * get a small packet
3081                                                  */
3082                                                 if ((buffer_space_left == 0) ||
3083                                                         (bytes_in_last_packet <
3084                                                          pipe->max_packet)) {
3085                                                         pipe->next_tx_frame +=
3086                                                                 pipe->interval;
3087                                                         __cvmx_usb_perform_complete(
3088                                                                 usb,
3089                                                                 pipe,
3090                                                                 transaction,
3091                                                                 CVMX_USB_COMPLETE_SUCCESS);
3092                                                 }
3093                                         } else
3094                                                 transaction->stage =
3095                                                         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
3096                                 }
3097                         } else {
3098                                 pipe->next_tx_frame += pipe->interval;
3099                                 __cvmx_usb_perform_complete(usb,
3100                                                 pipe,
3101                                                 transaction,
3102                                                 CVMX_USB_COMPLETE_SUCCESS);
3103                         }
3104                         break;
3105                 }
3106         } else if (usbc_hcint.s.nak) {
3107                 /*
3108                  * If this was a split then clear our split in progress marker.
3109                  */
3110                 if (usb->active_split == transaction)
3111                         usb->active_split = NULL;
3112                 /*
3113                  * NAK as a response means the device couldn't accept the
3114                  * transaction, but it should be retried in the future. Rewind
3115                  * to the beginning of the transaction by anding off the split
3116                  * complete bit. Retry in the next interval
3117                  */
3118                 transaction->retries = 0;
3119                 transaction->stage &= ~1;
3120                 pipe->next_tx_frame += pipe->interval;
3121                 if (pipe->next_tx_frame < usb->frame_number)
3122                         pipe->next_tx_frame = usb->frame_number +
3123                                 pipe->interval -
3124                                 (usb->frame_number - pipe->next_tx_frame) %
3125                                 pipe->interval;
3126         } else {
3127                 struct cvmx_usb_port_status port;
3128
3129                 port = cvmx_usb_get_status(usb);
3130                 if (port.port_enabled) {
3131                         /* We'll retry the exact same transaction again */
3132                         transaction->retries++;
3133                 } else {
3134                         /*
3135                          * We get channel halted interrupts with no result bits
3136                          * sets when the cable is unplugged
3137                          */
3138                         __cvmx_usb_perform_complete(usb, pipe, transaction,
3139                                         CVMX_USB_COMPLETE_ERROR);
3140                 }
3141         }
3142         return 0;
3143 }
3144
3145 static void octeon_usb_port_callback(struct cvmx_usb_state *usb)
3146 {
3147         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
3148
3149         spin_unlock(&priv->lock);
3150         usb_hcd_poll_rh_status(octeon_to_hcd(priv));
3151         spin_lock(&priv->lock);
3152 }
3153
3154 /**
3155  * Poll the USB block for status and call all needed callback
3156  * handlers. This function is meant to be called in the interrupt
3157  * handler for the USB controller. It can also be called
3158  * periodically in a loop for non-interrupt based operation.
3159  *
3160  * @usb: USB device state populated by cvmx_usb_initialize().
3161  *
3162  * Returns: 0 or a negative error code.
3163  */
3164 static int cvmx_usb_poll(struct cvmx_usb_state *usb)
3165 {
3166         union cvmx_usbcx_hfnum usbc_hfnum;
3167         union cvmx_usbcx_gintsts usbc_gintsts;
3168
3169         prefetch_range(usb, sizeof(*usb));
3170
3171         /* Update the frame counter */
3172         usbc_hfnum.u32 = __cvmx_usb_read_csr32(usb,
3173                                                 CVMX_USBCX_HFNUM(usb->index));
3174         if ((usb->frame_number&0x3fff) > usbc_hfnum.s.frnum)
3175                 usb->frame_number += 0x4000;
3176         usb->frame_number &= ~0x3fffull;
3177         usb->frame_number |= usbc_hfnum.s.frnum;
3178
3179         /* Read the pending interrupts */
3180         usbc_gintsts.u32 = __cvmx_usb_read_csr32(usb,
3181                                                 CVMX_USBCX_GINTSTS(usb->index));
3182
3183         /* Clear the interrupts now that we know about them */
3184         __cvmx_usb_write_csr32(usb,
3185                                 CVMX_USBCX_GINTSTS(usb->index),
3186                                 usbc_gintsts.u32);
3187
3188         if (usbc_gintsts.s.rxflvl) {
3189                 /*
3190                  * RxFIFO Non-Empty (RxFLvl)
3191                  * Indicates that there is at least one packet pending to be
3192                  * read from the RxFIFO.
3193                  *
3194                  * In DMA mode this is handled by hardware
3195                  */
3196                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
3197                         __cvmx_usb_poll_rx_fifo(usb);
3198         }
3199         if (usbc_gintsts.s.ptxfemp || usbc_gintsts.s.nptxfemp) {
3200                 /* Fill the Tx FIFOs when not in DMA mode */
3201                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
3202                         __cvmx_usb_poll_tx_fifo(usb);
3203         }
3204         if (usbc_gintsts.s.disconnint || usbc_gintsts.s.prtint) {
3205                 union cvmx_usbcx_hprt usbc_hprt;
3206                 /*
3207                  * Disconnect Detected Interrupt (DisconnInt)
3208                  * Asserted when a device disconnect is detected.
3209                  *
3210                  * Host Port Interrupt (PrtInt)
3211                  * The core sets this bit to indicate a change in port status of
3212                  * one of the O2P USB core ports in Host mode. The application
3213                  * must read the Host Port Control and Status (HPRT) register to
3214                  * determine the exact event that caused this interrupt. The
3215                  * application must clear the appropriate status bit in the Host
3216                  * Port Control and Status register to clear this bit.
3217                  *
3218                  * Call the user's port callback
3219                  */
3220                 octeon_usb_port_callback(usb);
3221                 /* Clear the port change bits */
3222                 usbc_hprt.u32 = __cvmx_usb_read_csr32(usb,
3223                                 CVMX_USBCX_HPRT(usb->index));
3224                 usbc_hprt.s.prtena = 0;
3225                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HPRT(usb->index),
3226                                        usbc_hprt.u32);
3227         }
3228         if (usbc_gintsts.s.hchint) {
3229                 /*
3230                  * Host Channels Interrupt (HChInt)
3231                  * The core sets this bit to indicate that an interrupt is
3232                  * pending on one of the channels of the core (in Host mode).
3233                  * The application must read the Host All Channels Interrupt
3234                  * (HAINT) register to determine the exact number of the channel
3235                  * on which the interrupt occurred, and then read the
3236                  * corresponding Host Channel-n Interrupt (HCINTn) register to
3237                  * determine the exact cause of the interrupt. The application
3238                  * must clear the appropriate status bit in the HCINTn register
3239                  * to clear this bit.
3240                  */
3241                 union cvmx_usbcx_haint usbc_haint;
3242
3243                 usbc_haint.u32 = __cvmx_usb_read_csr32(usb,
3244                                         CVMX_USBCX_HAINT(usb->index));
3245                 while (usbc_haint.u32) {
3246                         int channel;
3247
3248                         channel = __fls(usbc_haint.u32);
3249                         __cvmx_usb_poll_channel(usb, channel);
3250                         usbc_haint.u32 ^= 1<<channel;
3251                 }
3252         }
3253
3254         __cvmx_usb_schedule(usb, usbc_gintsts.s.sof);
3255
3256         return 0;
3257 }
3258
3259 /* convert between an HCD pointer and the corresponding struct octeon_hcd */
3260 static inline struct octeon_hcd *hcd_to_octeon(struct usb_hcd *hcd)
3261 {
3262         return (struct octeon_hcd *)(hcd->hcd_priv);
3263 }
3264
3265 static irqreturn_t octeon_usb_irq(struct usb_hcd *hcd)
3266 {
3267         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3268         unsigned long flags;
3269
3270         spin_lock_irqsave(&priv->lock, flags);
3271         cvmx_usb_poll(&priv->usb);
3272         spin_unlock_irqrestore(&priv->lock, flags);
3273         return IRQ_HANDLED;
3274 }
3275
3276 static int octeon_usb_start(struct usb_hcd *hcd)
3277 {
3278         hcd->state = HC_STATE_RUNNING;
3279         return 0;
3280 }
3281
3282 static void octeon_usb_stop(struct usb_hcd *hcd)
3283 {
3284         hcd->state = HC_STATE_HALT;
3285 }
3286
3287 static int octeon_usb_get_frame_number(struct usb_hcd *hcd)
3288 {
3289         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3290
3291         return cvmx_usb_get_frame_number(&priv->usb);
3292 }
3293
3294 static int octeon_usb_urb_enqueue(struct usb_hcd *hcd,
3295                                   struct urb *urb,
3296                                   gfp_t mem_flags)
3297 {
3298         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3299         struct device *dev = hcd->self.controller;
3300         struct cvmx_usb_transaction *transaction = NULL;
3301         struct cvmx_usb_pipe *pipe;
3302         unsigned long flags;
3303         struct cvmx_usb_iso_packet *iso_packet;
3304         struct usb_host_endpoint *ep = urb->ep;
3305
3306         urb->status = 0;
3307         INIT_LIST_HEAD(&urb->urb_list); /* not enqueued on dequeue_list */
3308         spin_lock_irqsave(&priv->lock, flags);
3309
3310         if (!ep->hcpriv) {
3311                 enum cvmx_usb_transfer transfer_type;
3312                 enum cvmx_usb_speed speed;
3313                 int split_device = 0;
3314                 int split_port = 0;
3315
3316                 switch (usb_pipetype(urb->pipe)) {
3317                 case PIPE_ISOCHRONOUS:
3318                         transfer_type = CVMX_USB_TRANSFER_ISOCHRONOUS;
3319                         break;
3320                 case PIPE_INTERRUPT:
3321                         transfer_type = CVMX_USB_TRANSFER_INTERRUPT;
3322                         break;
3323                 case PIPE_CONTROL:
3324                         transfer_type = CVMX_USB_TRANSFER_CONTROL;
3325                         break;
3326                 default:
3327                         transfer_type = CVMX_USB_TRANSFER_BULK;
3328                         break;
3329                 }
3330                 switch (urb->dev->speed) {
3331                 case USB_SPEED_LOW:
3332                         speed = CVMX_USB_SPEED_LOW;
3333                         break;
3334                 case USB_SPEED_FULL:
3335                         speed = CVMX_USB_SPEED_FULL;
3336                         break;
3337                 default:
3338                         speed = CVMX_USB_SPEED_HIGH;
3339                         break;
3340                 }
3341                 /*
3342                  * For slow devices on high speed ports we need to find the hub
3343                  * that does the speed translation so we know where to send the
3344                  * split transactions.
3345                  */
3346                 if (speed != CVMX_USB_SPEED_HIGH) {
3347                         /*
3348                          * Start at this device and work our way up the usb
3349                          * tree.
3350                          */
3351                         struct usb_device *dev = urb->dev;
3352
3353                         while (dev->parent) {
3354                                 /*
3355                                  * If our parent is high speed then he'll
3356                                  * receive the splits.
3357                                  */
3358                                 if (dev->parent->speed == USB_SPEED_HIGH) {
3359                                         split_device = dev->parent->devnum;
3360                                         split_port = dev->portnum;
3361                                         break;
3362                                 }
3363                                 /*
3364                                  * Move up the tree one level. If we make it all
3365                                  * the way up the tree, then the port must not
3366                                  * be in high speed mode and we don't need a
3367                                  * split.
3368                                  */
3369                                 dev = dev->parent;
3370                         }
3371                 }
3372                 pipe = cvmx_usb_open_pipe(&priv->usb, usb_pipedevice(urb->pipe),
3373                                           usb_pipeendpoint(urb->pipe), speed,
3374                                           le16_to_cpu(ep->desc.wMaxPacketSize)
3375                                           & 0x7ff,
3376                                           transfer_type,
3377                                           usb_pipein(urb->pipe) ?
3378                                                 CVMX_USB_DIRECTION_IN :
3379                                                 CVMX_USB_DIRECTION_OUT,
3380                                           urb->interval,
3381                                           (le16_to_cpu(ep->desc.wMaxPacketSize)
3382                                            >> 11) & 0x3,
3383                                           split_device, split_port);
3384                 if (!pipe) {
3385                         spin_unlock_irqrestore(&priv->lock, flags);
3386                         dev_dbg(dev, "Failed to create pipe\n");
3387                         return -ENOMEM;
3388                 }
3389                 ep->hcpriv = pipe;
3390         } else {
3391                 pipe = ep->hcpriv;
3392         }
3393
3394         switch (usb_pipetype(urb->pipe)) {
3395         case PIPE_ISOCHRONOUS:
3396                 dev_dbg(dev, "Submit isochronous to %d.%d\n",
3397                         usb_pipedevice(urb->pipe),
3398                         usb_pipeendpoint(urb->pipe));
3399                 /*
3400                  * Allocate a structure to use for our private list of
3401                  * isochronous packets.
3402                  */
3403                 iso_packet = kmalloc(urb->number_of_packets *
3404                                      sizeof(struct cvmx_usb_iso_packet),
3405                                      GFP_ATOMIC);
3406                 if (iso_packet) {
3407                         int i;
3408                         /* Fill the list with the data from the URB */
3409                         for (i = 0; i < urb->number_of_packets; i++) {
3410                                 iso_packet[i].offset =
3411                                         urb->iso_frame_desc[i].offset;
3412                                 iso_packet[i].length =
3413                                         urb->iso_frame_desc[i].length;
3414                                 iso_packet[i].status =
3415                                         CVMX_USB_COMPLETE_ERROR;
3416                         }
3417                         /*
3418                          * Store a pointer to the list in the URB setup_packet
3419                          * field. We know this currently isn't being used and
3420                          * this saves us a bunch of logic.
3421                          */
3422                         urb->setup_packet = (char *)iso_packet;
3423                         transaction = cvmx_usb_submit_isochronous(&priv->usb,
3424                                                                   pipe, urb);
3425                         /*
3426                          * If submit failed we need to free our private packet
3427                          * list.
3428                          */
3429                         if (!transaction) {
3430                                 urb->setup_packet = NULL;
3431                                 kfree(iso_packet);
3432                         }
3433                 }
3434                 break;
3435         case PIPE_INTERRUPT:
3436                 dev_dbg(dev, "Submit interrupt to %d.%d\n",
3437                         usb_pipedevice(urb->pipe),
3438                         usb_pipeendpoint(urb->pipe));
3439                 transaction = cvmx_usb_submit_interrupt(&priv->usb, pipe, urb);
3440                 break;
3441         case PIPE_CONTROL:
3442                 dev_dbg(dev, "Submit control to %d.%d\n",
3443                         usb_pipedevice(urb->pipe),
3444                         usb_pipeendpoint(urb->pipe));
3445                 transaction = cvmx_usb_submit_control(&priv->usb, pipe, urb);
3446                 break;
3447         case PIPE_BULK:
3448                 dev_dbg(dev, "Submit bulk to %d.%d\n",
3449                         usb_pipedevice(urb->pipe),
3450                         usb_pipeendpoint(urb->pipe));
3451                 transaction = cvmx_usb_submit_bulk(&priv->usb, pipe, urb);
3452                 break;
3453         }
3454         if (!transaction) {
3455                 spin_unlock_irqrestore(&priv->lock, flags);
3456                 dev_dbg(dev, "Failed to submit\n");
3457                 return -ENOMEM;
3458         }
3459         urb->hcpriv = transaction;
3460         spin_unlock_irqrestore(&priv->lock, flags);
3461         return 0;
3462 }
3463
3464 static void octeon_usb_urb_dequeue_work(unsigned long arg)
3465 {
3466         struct urb *urb;
3467         struct urb *next;
3468         unsigned long flags;
3469         struct octeon_hcd *priv = (struct octeon_hcd *)arg;
3470
3471         spin_lock_irqsave(&priv->lock, flags);
3472
3473         list_for_each_entry_safe(urb, next, &priv->dequeue_list, urb_list) {
3474                 list_del_init(&urb->urb_list);
3475                 cvmx_usb_cancel(&priv->usb, urb->ep->hcpriv, urb->hcpriv);
3476         }
3477
3478         spin_unlock_irqrestore(&priv->lock, flags);
3479 }
3480
3481 static int octeon_usb_urb_dequeue(struct usb_hcd *hcd,
3482                                   struct urb *urb,
3483                                   int status)
3484 {
3485         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3486         unsigned long flags;
3487
3488         if (!urb->dev)
3489                 return -EINVAL;
3490
3491         spin_lock_irqsave(&priv->lock, flags);
3492
3493         urb->status = status;
3494         list_add_tail(&urb->urb_list, &priv->dequeue_list);
3495
3496         spin_unlock_irqrestore(&priv->lock, flags);
3497
3498         tasklet_schedule(&priv->dequeue_tasklet);
3499
3500         return 0;
3501 }
3502
3503 static void octeon_usb_endpoint_disable(struct usb_hcd *hcd,
3504                                         struct usb_host_endpoint *ep)
3505 {
3506         struct device *dev = hcd->self.controller;
3507
3508         if (ep->hcpriv) {
3509                 struct octeon_hcd *priv = hcd_to_octeon(hcd);
3510                 struct cvmx_usb_pipe *pipe = ep->hcpriv;
3511                 unsigned long flags;
3512
3513                 spin_lock_irqsave(&priv->lock, flags);
3514                 cvmx_usb_cancel_all(&priv->usb, pipe);
3515                 if (cvmx_usb_close_pipe(&priv->usb, pipe))
3516                         dev_dbg(dev, "Closing pipe %p failed\n", pipe);
3517                 spin_unlock_irqrestore(&priv->lock, flags);
3518                 ep->hcpriv = NULL;
3519         }
3520 }
3521
3522 static int octeon_usb_hub_status_data(struct usb_hcd *hcd, char *buf)
3523 {
3524         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3525         struct cvmx_usb_port_status port_status;
3526         unsigned long flags;
3527
3528         spin_lock_irqsave(&priv->lock, flags);
3529         port_status = cvmx_usb_get_status(&priv->usb);
3530         spin_unlock_irqrestore(&priv->lock, flags);
3531         buf[0] = 0;
3532         buf[0] = port_status.connect_change << 1;
3533
3534         return buf[0] != 0;
3535 }
3536
3537 static int octeon_usb_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
3538                                 u16 wIndex, char *buf, u16 wLength)
3539 {
3540         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3541         struct device *dev = hcd->self.controller;
3542         struct cvmx_usb_port_status usb_port_status;
3543         int port_status;
3544         struct usb_hub_descriptor *desc;
3545         unsigned long flags;
3546
3547         switch (typeReq) {
3548         case ClearHubFeature:
3549                 dev_dbg(dev, "ClearHubFeature\n");
3550                 switch (wValue) {
3551                 case C_HUB_LOCAL_POWER:
3552                 case C_HUB_OVER_CURRENT:
3553                         /* Nothing required here */
3554                         break;
3555                 default:
3556                         return -EINVAL;
3557                 }
3558                 break;
3559         case ClearPortFeature:
3560                 dev_dbg(dev, "ClearPortFeature\n");
3561                 if (wIndex != 1) {
3562                         dev_dbg(dev, " INVALID\n");
3563                         return -EINVAL;
3564                 }
3565
3566                 switch (wValue) {
3567                 case USB_PORT_FEAT_ENABLE:
3568                         dev_dbg(dev, " ENABLE\n");
3569                         spin_lock_irqsave(&priv->lock, flags);
3570                         cvmx_usb_disable(&priv->usb);
3571                         spin_unlock_irqrestore(&priv->lock, flags);
3572                         break;
3573                 case USB_PORT_FEAT_SUSPEND:
3574                         dev_dbg(dev, " SUSPEND\n");
3575                         /* Not supported on Octeon */
3576                         break;
3577                 case USB_PORT_FEAT_POWER:
3578                         dev_dbg(dev, " POWER\n");
3579                         /* Not supported on Octeon */
3580                         break;
3581                 case USB_PORT_FEAT_INDICATOR:
3582                         dev_dbg(dev, " INDICATOR\n");
3583                         /* Port inidicator not supported */
3584                         break;
3585                 case USB_PORT_FEAT_C_CONNECTION:
3586                         dev_dbg(dev, " C_CONNECTION\n");
3587                         /* Clears drivers internal connect status change flag */
3588                         spin_lock_irqsave(&priv->lock, flags);
3589                         priv->usb.port_status =
3590                                 cvmx_usb_get_status(&priv->usb);
3591                         spin_unlock_irqrestore(&priv->lock, flags);
3592                         break;
3593                 case USB_PORT_FEAT_C_RESET:
3594                         dev_dbg(dev, " C_RESET\n");
3595                         /*
3596                          * Clears the driver's internal Port Reset Change flag.
3597                          */
3598                         spin_lock_irqsave(&priv->lock, flags);
3599                         priv->usb.port_status =
3600                                 cvmx_usb_get_status(&priv->usb);
3601                         spin_unlock_irqrestore(&priv->lock, flags);
3602                         break;
3603                 case USB_PORT_FEAT_C_ENABLE:
3604                         dev_dbg(dev, " C_ENABLE\n");
3605                         /*
3606                          * Clears the driver's internal Port Enable/Disable
3607                          * Change flag.
3608                          */
3609                         spin_lock_irqsave(&priv->lock, flags);
3610                         priv->usb.port_status =
3611                                 cvmx_usb_get_status(&priv->usb);
3612                         spin_unlock_irqrestore(&priv->lock, flags);
3613                         break;
3614                 case USB_PORT_FEAT_C_SUSPEND:
3615                         dev_dbg(dev, " C_SUSPEND\n");
3616                         /*
3617                          * Clears the driver's internal Port Suspend Change
3618                          * flag, which is set when resume signaling on the host
3619                          * port is complete.
3620                          */
3621                         break;
3622                 case USB_PORT_FEAT_C_OVER_CURRENT:
3623                         dev_dbg(dev, " C_OVER_CURRENT\n");
3624                         /* Clears the driver's overcurrent Change flag */
3625                         spin_lock_irqsave(&priv->lock, flags);
3626                         priv->usb.port_status =
3627                                 cvmx_usb_get_status(&priv->usb);
3628                         spin_unlock_irqrestore(&priv->lock, flags);
3629                         break;
3630                 default:
3631                         dev_dbg(dev, " UNKNOWN\n");
3632                         return -EINVAL;
3633                 }
3634                 break;
3635         case GetHubDescriptor:
3636                 dev_dbg(dev, "GetHubDescriptor\n");
3637                 desc = (struct usb_hub_descriptor *)buf;
3638                 desc->bDescLength = 9;
3639                 desc->bDescriptorType = 0x29;
3640                 desc->bNbrPorts = 1;
3641                 desc->wHubCharacteristics = 0x08;
3642                 desc->bPwrOn2PwrGood = 1;
3643                 desc->bHubContrCurrent = 0;
3644                 desc->u.hs.DeviceRemovable[0] = 0;
3645                 desc->u.hs.DeviceRemovable[1] = 0xff;
3646                 break;
3647         case GetHubStatus:
3648                 dev_dbg(dev, "GetHubStatus\n");
3649                 *(__le32 *) buf = 0;
3650                 break;
3651         case GetPortStatus:
3652                 dev_dbg(dev, "GetPortStatus\n");
3653                 if (wIndex != 1) {
3654                         dev_dbg(dev, " INVALID\n");
3655                         return -EINVAL;
3656                 }
3657
3658                 spin_lock_irqsave(&priv->lock, flags);
3659                 usb_port_status = cvmx_usb_get_status(&priv->usb);
3660                 spin_unlock_irqrestore(&priv->lock, flags);
3661                 port_status = 0;
3662
3663                 if (usb_port_status.connect_change) {
3664                         port_status |= (1 << USB_PORT_FEAT_C_CONNECTION);
3665                         dev_dbg(dev, " C_CONNECTION\n");
3666                 }
3667
3668                 if (usb_port_status.port_enabled) {
3669                         port_status |= (1 << USB_PORT_FEAT_C_ENABLE);
3670                         dev_dbg(dev, " C_ENABLE\n");
3671                 }
3672
3673                 if (usb_port_status.connected) {
3674                         port_status |= (1 << USB_PORT_FEAT_CONNECTION);
3675                         dev_dbg(dev, " CONNECTION\n");
3676                 }
3677
3678                 if (usb_port_status.port_enabled) {
3679                         port_status |= (1 << USB_PORT_FEAT_ENABLE);
3680                         dev_dbg(dev, " ENABLE\n");
3681                 }
3682
3683                 if (usb_port_status.port_over_current) {
3684                         port_status |= (1 << USB_PORT_FEAT_OVER_CURRENT);
3685                         dev_dbg(dev, " OVER_CURRENT\n");
3686                 }
3687
3688                 if (usb_port_status.port_powered) {
3689                         port_status |= (1 << USB_PORT_FEAT_POWER);
3690                         dev_dbg(dev, " POWER\n");
3691                 }
3692
3693                 if (usb_port_status.port_speed == CVMX_USB_SPEED_HIGH) {
3694                         port_status |= USB_PORT_STAT_HIGH_SPEED;
3695                         dev_dbg(dev, " HIGHSPEED\n");
3696                 } else if (usb_port_status.port_speed == CVMX_USB_SPEED_LOW) {
3697                         port_status |= (1 << USB_PORT_FEAT_LOWSPEED);
3698                         dev_dbg(dev, " LOWSPEED\n");
3699                 }
3700
3701                 *((__le32 *) buf) = cpu_to_le32(port_status);
3702                 break;
3703         case SetHubFeature:
3704                 dev_dbg(dev, "SetHubFeature\n");
3705                 /* No HUB features supported */
3706                 break;
3707         case SetPortFeature:
3708                 dev_dbg(dev, "SetPortFeature\n");
3709                 if (wIndex != 1) {
3710                         dev_dbg(dev, " INVALID\n");
3711                         return -EINVAL;
3712                 }
3713
3714                 switch (wValue) {
3715                 case USB_PORT_FEAT_SUSPEND:
3716                         dev_dbg(dev, " SUSPEND\n");
3717                         return -EINVAL;
3718                 case USB_PORT_FEAT_POWER:
3719                         dev_dbg(dev, " POWER\n");
3720                         return -EINVAL;
3721                 case USB_PORT_FEAT_RESET:
3722                         dev_dbg(dev, " RESET\n");
3723                         spin_lock_irqsave(&priv->lock, flags);
3724                         cvmx_usb_disable(&priv->usb);
3725                         if (cvmx_usb_enable(&priv->usb))
3726                                 dev_dbg(dev, "Failed to enable the port\n");
3727                         spin_unlock_irqrestore(&priv->lock, flags);
3728                         return 0;
3729                 case USB_PORT_FEAT_INDICATOR:
3730                         dev_dbg(dev, " INDICATOR\n");
3731                         /* Not supported */
3732                         break;
3733                 default:
3734                         dev_dbg(dev, " UNKNOWN\n");
3735                         return -EINVAL;
3736                 }
3737                 break;
3738         default:
3739                 dev_dbg(dev, "Unknown root hub request\n");
3740                 return -EINVAL;
3741         }
3742         return 0;
3743 }
3744
3745 static const struct hc_driver octeon_hc_driver = {
3746         .description            = "Octeon USB",
3747         .product_desc           = "Octeon Host Controller",
3748         .hcd_priv_size          = sizeof(struct octeon_hcd),
3749         .irq                    = octeon_usb_irq,
3750         .flags                  = HCD_MEMORY | HCD_USB2,
3751         .start                  = octeon_usb_start,
3752         .stop                   = octeon_usb_stop,
3753         .urb_enqueue            = octeon_usb_urb_enqueue,
3754         .urb_dequeue            = octeon_usb_urb_dequeue,
3755         .endpoint_disable       = octeon_usb_endpoint_disable,
3756         .get_frame_number       = octeon_usb_get_frame_number,
3757         .hub_status_data        = octeon_usb_hub_status_data,
3758         .hub_control            = octeon_usb_hub_control,
3759         .map_urb_for_dma        = octeon_map_urb_for_dma,
3760         .unmap_urb_for_dma      = octeon_unmap_urb_for_dma,
3761 };
3762
3763 static int octeon_usb_probe(struct platform_device *pdev)
3764 {
3765         int status;
3766         int initialize_flags;
3767         int usb_num;
3768         struct resource *res_mem;
3769         struct device_node *usbn_node;
3770         int irq = platform_get_irq(pdev, 0);
3771         struct device *dev = &pdev->dev;
3772         struct octeon_hcd *priv;
3773         struct usb_hcd *hcd;
3774         unsigned long flags;
3775         u32 clock_rate = 48000000;
3776         bool is_crystal_clock = false;
3777         const char *clock_type;
3778         int i;
3779
3780         if (dev->of_node == NULL) {
3781                 dev_err(dev, "Error: empty of_node\n");
3782                 return -ENXIO;
3783         }
3784         usbn_node = dev->of_node->parent;
3785
3786         i = of_property_read_u32(usbn_node,
3787                                  "refclk-frequency", &clock_rate);
3788         if (i) {
3789                 dev_err(dev, "No USBN \"refclk-frequency\"\n");
3790                 return -ENXIO;
3791         }
3792         switch (clock_rate) {
3793         case 12000000:
3794                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ;
3795                 break;
3796         case 24000000:
3797                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ;
3798                 break;
3799         case 48000000:
3800                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ;
3801                 break;
3802         default:
3803                 dev_err(dev, "Illebal USBN \"refclk-frequency\" %u\n",
3804                                 clock_rate);
3805                 return -ENXIO;
3806
3807         }
3808
3809         i = of_property_read_string(usbn_node,
3810                                     "refclk-type", &clock_type);
3811
3812         if (!i && strcmp("crystal", clock_type) == 0)
3813                 is_crystal_clock = true;
3814
3815         if (is_crystal_clock)
3816                 initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI;
3817         else
3818                 initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND;
3819
3820         res_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3821         if (res_mem == NULL) {
3822                 dev_err(dev, "found no memory resource\n");
3823                 return -ENXIO;
3824         }
3825         usb_num = (res_mem->start >> 44) & 1;
3826
3827         if (irq < 0) {
3828                 /* Defective device tree, but we know how to fix it. */
3829                 irq_hw_number_t hwirq = usb_num ? (1 << 6) + 17 : 56;
3830
3831                 irq = irq_create_mapping(NULL, hwirq);
3832         }
3833
3834         /*
3835          * Set the DMA mask to 64bits so we get buffers already translated for
3836          * DMA.
3837          */
3838         dev->coherent_dma_mask = ~0;
3839         dev->dma_mask = &dev->coherent_dma_mask;
3840
3841         /*
3842          * Only cn52XX and cn56XX have DWC_OTG USB hardware and the
3843          * IOB priority registers.  Under heavy network load USB
3844          * hardware can be starved by the IOB causing a crash.  Give
3845          * it a priority boost if it has been waiting more than 400
3846          * cycles to avoid this situation.
3847          *
3848          * Testing indicates that a cnt_val of 8192 is not sufficient,
3849          * but no failures are seen with 4096.  We choose a value of
3850          * 400 to give a safety factor of 10.
3851          */
3852         if (OCTEON_IS_MODEL(OCTEON_CN52XX) || OCTEON_IS_MODEL(OCTEON_CN56XX)) {
3853                 union cvmx_iob_n2c_l2c_pri_cnt pri_cnt;
3854
3855                 pri_cnt.u64 = 0;
3856                 pri_cnt.s.cnt_enb = 1;
3857                 pri_cnt.s.cnt_val = 400;
3858                 cvmx_write_csr(CVMX_IOB_N2C_L2C_PRI_CNT, pri_cnt.u64);
3859         }
3860
3861         hcd = usb_create_hcd(&octeon_hc_driver, dev, dev_name(dev));
3862         if (!hcd) {
3863                 dev_dbg(dev, "Failed to allocate memory for HCD\n");
3864                 return -1;
3865         }
3866         hcd->uses_new_polling = 1;
3867         priv = (struct octeon_hcd *)hcd->hcd_priv;
3868
3869         spin_lock_init(&priv->lock);
3870
3871         tasklet_init(&priv->dequeue_tasklet, octeon_usb_urb_dequeue_work,
3872                      (unsigned long)priv);
3873         INIT_LIST_HEAD(&priv->dequeue_list);
3874
3875         status = cvmx_usb_initialize(&priv->usb, usb_num, initialize_flags);
3876         if (status) {
3877                 dev_dbg(dev, "USB initialization failed with %d\n", status);
3878                 kfree(hcd);
3879                 return -1;
3880         }
3881
3882         /* This delay is needed for CN3010, but I don't know why... */
3883         mdelay(10);
3884
3885         spin_lock_irqsave(&priv->lock, flags);
3886         cvmx_usb_poll(&priv->usb);
3887         spin_unlock_irqrestore(&priv->lock, flags);
3888
3889         status = usb_add_hcd(hcd, irq, 0);
3890         if (status) {
3891                 dev_dbg(dev, "USB add HCD failed with %d\n", status);
3892                 kfree(hcd);
3893                 return -1;
3894         }
3895         device_wakeup_enable(hcd->self.controller);
3896
3897         dev_info(dev, "Registered HCD for port %d on irq %d\n", usb_num, irq);
3898
3899         return 0;
3900 }
3901
3902 static int octeon_usb_remove(struct platform_device *pdev)
3903 {
3904         int status;
3905         struct device *dev = &pdev->dev;
3906         struct usb_hcd *hcd = dev_get_drvdata(dev);
3907         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3908         unsigned long flags;
3909
3910         usb_remove_hcd(hcd);
3911         tasklet_kill(&priv->dequeue_tasklet);
3912         spin_lock_irqsave(&priv->lock, flags);
3913         status = cvmx_usb_shutdown(&priv->usb);
3914         spin_unlock_irqrestore(&priv->lock, flags);
3915         if (status)
3916                 dev_dbg(dev, "USB shutdown failed with %d\n", status);
3917
3918         kfree(hcd);
3919
3920         return 0;
3921 }
3922
3923 static struct of_device_id octeon_usb_match[] = {
3924         {
3925                 .compatible = "cavium,octeon-5750-usbc",
3926         },
3927         {},
3928 };
3929
3930 static struct platform_driver octeon_usb_driver = {
3931         .driver = {
3932                 .name       = "OcteonUSB",
3933                 .owner          = THIS_MODULE,
3934                 .of_match_table = octeon_usb_match,
3935         },
3936         .probe      = octeon_usb_probe,
3937         .remove     = octeon_usb_remove,
3938 };
3939
3940 static int __init octeon_usb_driver_init(void)
3941 {
3942         if (usb_disabled())
3943                 return 0;
3944
3945         return platform_driver_register(&octeon_usb_driver);
3946 }
3947 module_init(octeon_usb_driver_init);
3948
3949 static void __exit octeon_usb_driver_exit(void)
3950 {
3951         if (usb_disabled())
3952                 return;
3953
3954         platform_driver_unregister(&octeon_usb_driver);
3955 }
3956 module_exit(octeon_usb_driver_exit);
3957
3958 MODULE_LICENSE("GPL");
3959 MODULE_AUTHOR("Cavium, Inc. <support@cavium.com>");
3960 MODULE_DESCRIPTION("Cavium Inc. OCTEON USB Host driver.");