drivers/staging/octeon-usb/octeon-hcd.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 2008 Cavium Networks
   7  *
   8  * Some parts of the code were originally released under BSD license:
   9  *
  10  * Copyright (c) 2003-2010 Cavium Networks (support@cavium.com). All rights
  11  * reserved.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions are
  15  * met:
  16  *
  17  *   * Redistributions of source code must retain the above copyright
  18  *     notice, this list of conditions and the following disclaimer.
  19  *
  20  *   * Redistributions in binary form must reproduce the above
  21  *     copyright notice, this list of conditions and the following
  22  *     disclaimer in the documentation and/or other materials provided
  23  *     with the distribution.
  24  *
  25  *   * Neither the name of Cavium Networks nor the names of
  26  *     its contributors may be used to endorse or promote products
  27  *     derived from this software without specific prior written
  28  *     permission.
  29  *
  30  * This Software, including technical data, may be subject to U.S. export
  31  * control laws, including the U.S. Export Administration Act and its associated
  32  * regulations, and may be subject to export or import regulations in other
  33  * countries.
  34  *
  35  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
  36  * AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR
  37  * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
  38  * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION
  39  * OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
  40  * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
  41  * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
  42  * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
  43  * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
  44  * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
  45  */
  46 #include <linux/kernel.h>
  47 #include <linux/module.h>
  48 #include <linux/init.h>
  49 #include <linux/pci.h>
  50 #include <linux/prefetch.h>
  51 #include <linux/interrupt.h>
  52 #include <linux/platform_device.h>
  53 #include <linux/usb.h>
  54
  55 #include <linux/time.h>
  56 #include <linux/delay.h>
  57
  58 #include <asm/octeon/cvmx.h>
  59 #include <asm/octeon/cvmx-iob-defs.h>
  60
  61 #include <linux/usb/hcd.h>
  62
  63 #include <linux/err.h>
  64
  65 #include <asm/octeon/octeon.h>
  66 #include <asm/octeon/cvmx-helper.h>
  67 #include <asm/octeon/cvmx-sysinfo.h>
  68 #include <asm/octeon/cvmx-helper-board.h>
  69
  70 #include "octeon-hcd.h"
  71
  72 /**
  73  * enum cvmx_usb_speed - the possible USB device speeds
  74  *
  75  * @CVMX_USB_SPEED_HIGH: Device is operation at 480Mbps
  76  * @CVMX_USB_SPEED_FULL: Device is operation at 12Mbps
  77  * @CVMX_USB_SPEED_LOW:  Device is operation at 1.5Mbps
  78  */
  79 enum cvmx_usb_speed {
  80         CVMX_USB_SPEED_HIGH = 0,
  81         CVMX_USB_SPEED_FULL = 1,
  82         CVMX_USB_SPEED_LOW = 2,
  83 };
  84
  85 /**
  86  * enum cvmx_usb_transfer - the possible USB transfer types
  87  *
  88  * @CVMX_USB_TRANSFER_CONTROL:     USB transfer type control for hub and status
  89  *                                 transfers
  90  * @CVMX_USB_TRANSFER_ISOCHRONOUS: USB transfer type isochronous for low
  91  *                                 priority periodic transfers
  92  * @CVMX_USB_TRANSFER_BULK:        USB transfer type bulk for large low priority
  93  *                                 transfers
  94  * @CVMX_USB_TRANSFER_INTERRUPT:   USB transfer type interrupt for high priority
  95  *                                 periodic transfers
  96  */
  97 enum cvmx_usb_transfer {
  98         CVMX_USB_TRANSFER_CONTROL = 0,
  99         CVMX_USB_TRANSFER_ISOCHRONOUS = 1,
 100         CVMX_USB_TRANSFER_BULK = 2,
 101         CVMX_USB_TRANSFER_INTERRUPT = 3,
 102 };
 103
 104 /**
 105  * enum cvmx_usb_direction - the transfer directions
 106  *
 107  * @CVMX_USB_DIRECTION_OUT: Data is transferring from Octeon to the device/host
 108  * @CVMX_USB_DIRECTION_IN:  Data is transferring from the device/host to Octeon
 109  */
 110 enum cvmx_usb_direction {
 111         CVMX_USB_DIRECTION_OUT,
 112         CVMX_USB_DIRECTION_IN,
 113 };
 114
 115 /**
 116  * enum cvmx_usb_complete - possible callback function status codes
 117  *
 118  * @CVMX_USB_COMPLETE_SUCCESS:    The transaction / operation finished without
 119  *                                any errors
 120  * @CVMX_USB_COMPLETE_SHORT:      FIXME: This is currently not implemented
 121  * @CVMX_USB_COMPLETE_CANCEL:     The transaction was canceled while in flight
 122  *                                by a user call to cvmx_usb_cancel
 123  * @CVMX_USB_COMPLETE_ERROR:      The transaction aborted with an unexpected
 124  *                                error status
 125  * @CVMX_USB_COMPLETE_STALL:      The transaction received a USB STALL response
 126  *                                from the device
 127  * @CVMX_USB_COMPLETE_XACTERR:    The transaction failed with an error from the
 128  *                                device even after a number of retries
 129  * @CVMX_USB_COMPLETE_DATATGLERR: The transaction failed with a data toggle
 130  *                                error even after a number of retries
 131  * @CVMX_USB_COMPLETE_BABBLEERR:  The transaction failed with a babble error
 132  * @CVMX_USB_COMPLETE_FRAMEERR:   The transaction failed with a frame error
 133  *                                even after a number of retries
 134  */
 135 enum cvmx_usb_complete {
 136         CVMX_USB_COMPLETE_SUCCESS,
 137         CVMX_USB_COMPLETE_SHORT,
 138         CVMX_USB_COMPLETE_CANCEL,
 139         CVMX_USB_COMPLETE_ERROR,
 140         CVMX_USB_COMPLETE_STALL,
 141         CVMX_USB_COMPLETE_XACTERR,
 142         CVMX_USB_COMPLETE_DATATGLERR,
 143         CVMX_USB_COMPLETE_BABBLEERR,
 144         CVMX_USB_COMPLETE_FRAMEERR,
 145 };
 146
 147 /**
 148  * struct cvmx_usb_port_status - the USB port status information
 149  *
 150  * @port_enabled:       1 = Usb port is enabled, 0 = disabled
 151  * @port_over_current:  1 = Over current detected, 0 = Over current not
 152  *                      detected. Octeon doesn't support over current detection.
 153  * @port_powered:       1 = Port power is being supplied to the device, 0 =
 154  *                      power is off. Octeon doesn't support turning port power
 155  *                      off.
 156  * @port_speed:         Current port speed.
 157  * @connected:          1 = A device is connected to the port, 0 = No device is
 158  *                      connected.
 159  * @connect_change:     1 = Device connected state changed since the last set
 160  *                      status call.
 161  */
 162 struct cvmx_usb_port_status {
 163         uint32_t reserved               : 25;
 164         uint32_t port_enabled           : 1;
 165         uint32_t port_over_current      : 1;
 166         uint32_t port_powered           : 1;
 167         enum cvmx_usb_speed port_speed  : 2;
 168         uint32_t connected              : 1;
 169         uint32_t connect_change         : 1;
 170 };
 171
 172 /**
 173  * struct cvmx_usb_iso_packet - descriptor for Isochronous packets
 174  *
 175  * @offset:     This is the offset in bytes into the main buffer where this data
 176  *              is stored.
 177  * @length:     This is the length in bytes of the data.
 178  * @status:     This is the status of this individual packet transfer.
 179  */
 180 struct cvmx_usb_iso_packet {
 181         int offset;
 182         int length;
 183         enum cvmx_usb_complete status;
 184 };
 185
 186 /**
 187  * enum cvmx_usb_initialize_flags - flags used by the initialization function
 188  *
 189  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI:    The USB port uses a 12MHz crystal
 190  *                                            as clock source at USB_XO and
 191  *                                            USB_XI.
 192  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND:   The USB port uses 12/24/48MHz 2.5V
 193  *                                            board clock source at USB_XO.
 194  *                                            USB_XI should be tied to GND.
 195  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK: Mask for clock speed field
 196  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:    Speed of reference clock or
 197  *                                            crystal
 198  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:    Speed of reference clock
 199  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:    Speed of reference clock
 200  * @CVMX_USB_INITIALIZE_FLAGS_NO_DMA:         Disable DMA and used polled IO for
 201  *                                            data transfer use for the USB
 202  */
 203 enum cvmx_usb_initialize_flags {
 204         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI           = 1 << 0,
 205         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND          = 1 << 1,
 206         CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK        = 3 << 3,
 207         CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ           = 1 << 3,
 208         CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ           = 2 << 3,
 209         CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ           = 3 << 3,
 210         /* Bits 3-4 used to encode the clock frequency */
 211         CVMX_USB_INITIALIZE_FLAGS_NO_DMA                = 1 << 5,
 212 };
 213
 214 /**
 215  * enum cvmx_usb_pipe_flags - internal flags for a pipe.
 216  *
 217  * @__CVMX_USB_PIPE_FLAGS_SCHEDULED: Used internally to determine if a pipe is
 218  *                                   actively using hardware. Do not use.
 219  * @__CVMX_USB_PIPE_FLAGS_NEED_PING: Used internally to determine if a high
 220  *                                   speed pipe is in the ping state. Do not
 221  *                                   use.
 222  */
 223 enum cvmx_usb_pipe_flags {
 224         __CVMX_USB_PIPE_FLAGS_SCHEDULED = 1 << 17,
 225         __CVMX_USB_PIPE_FLAGS_NEED_PING = 1 << 18,
 226 };
 227
 228 /* Maximum number of times to retry failed transactions */
 229 #define MAX_RETRIES             3
 230
 231 /* Maximum number of hardware channels supported by the USB block */
 232 #define MAX_CHANNELS            8
 233
 234 /* The highest valid USB device address */
 235 #define MAX_USB_ADDRESS         127
 236
 237 /* The highest valid USB endpoint number */
 238 #define MAX_USB_ENDPOINT        15
 239
 240 /* The highest valid port number on a hub */
 241 #define MAX_USB_HUB_PORT        15
 242
 243 /*
 244  * The low level hardware can transfer a maximum of this number of bytes in each
 245  * transfer. The field is 19 bits wide
 246  */
 247 #define MAX_TRANSFER_BYTES      ((1<<19)-1)
 248
 249 /*
 250  * The low level hardware can transfer a maximum of this number of packets in
 251  * each transfer. The field is 10 bits wide
 252  */
 253 #define MAX_TRANSFER_PACKETS    ((1<<10)-1)
 254
 255 /**
 256  * Logical transactions may take numerous low level
 257  * transactions, especially when splits are concerned. This
 258  * enum represents all of the possible stages a transaction can
 259  * be in. Note that split completes are always even. This is so
 260  * the NAK handler can backup to the previous low level
 261  * transaction with a simple clearing of bit 0.
 262  */
 263 enum cvmx_usb_stage {
 264         CVMX_USB_STAGE_NON_CONTROL,
 265         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE,
 266         CVMX_USB_STAGE_SETUP,
 267         CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE,
 268         CVMX_USB_STAGE_DATA,
 269         CVMX_USB_STAGE_DATA_SPLIT_COMPLETE,
 270         CVMX_USB_STAGE_STATUS,
 271         CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE,
 272 };
 273
 274 /**
 275  * struct cvmx_usb_transaction - describes each pending USB transaction
 276  *                               regardless of type. These are linked together
 277  *                               to form a list of pending requests for a pipe.
 278  *
 279  * @node:               List node for transactions in the pipe.
 280  * @type:               Type of transaction, duplicated of the pipe.
 281  * @flags:              State flags for this transaction.
 282  * @buffer:             User's physical buffer address to read/write.
 283  * @buffer_length:      Size of the user's buffer in bytes.
 284  * @control_header:     For control transactions, physical address of the 8
 285  *                      byte standard header.
 286  * @iso_start_frame:    For ISO transactions, the starting frame number.
 287  * @iso_number_packets: For ISO transactions, the number of packets in the
 288  *                      request.
 289  * @iso_packets:        For ISO transactions, the sub packets in the request.
 290  * @actual_bytes:       Actual bytes transfer for this transaction.
 291  * @stage:              For control transactions, the current stage.
 292  * @urb:                URB.
 293  */
 294 struct cvmx_usb_transaction {
 295         struct list_head node;
 296         enum cvmx_usb_transfer type;
 297         uint64_t buffer;
 298         int buffer_length;
 299         uint64_t control_header;
 300         int iso_start_frame;
 301         int iso_number_packets;
 302         struct cvmx_usb_iso_packet *iso_packets;
 303         int xfersize;
 304         int pktcnt;
 305         int retries;
 306         int actual_bytes;
 307         enum cvmx_usb_stage stage;
 308         struct urb *urb;
 309 };
 310
 311 /**
 312  * struct cvmx_usb_pipe - a pipe represents a virtual connection between Octeon
 313  *                        and some USB device. It contains a list of pending
 314  *                        request to the device.
 315  *
 316  * @node:               List node for pipe list
 317  * @next:               Pipe after this one in the list
 318  * @transactions:       List of pending transactions
 319  * @interval:           For periodic pipes, the interval between packets in
 320  *                      frames
 321  * @next_tx_frame:      The next frame this pipe is allowed to transmit on
 322  * @flags:              State flags for this pipe
 323  * @device_speed:       Speed of device connected to this pipe
 324  * @transfer_type:      Type of transaction supported by this pipe
 325  * @transfer_dir:       IN or OUT. Ignored for Control
 326  * @multi_count:        Max packet in a row for the device
 327  * @max_packet:         The device's maximum packet size in bytes
 328  * @device_addr:        USB device address at other end of pipe
 329  * @endpoint_num:       USB endpoint number at other end of pipe
 330  * @hub_device_addr:    Hub address this device is connected to
 331  * @hub_port:           Hub port this device is connected to
 332  * @pid_toggle:         This toggles between 0/1 on every packet send to track
 333  *                      the data pid needed
 334  * @channel:            Hardware DMA channel for this pipe
 335  * @split_sc_frame:     The low order bits of the frame number the split
 336  *                      complete should be sent on
 337  */
 338 struct cvmx_usb_pipe {
 339         struct list_head node;
 340         struct list_head transactions;
 341         uint64_t interval;
 342         uint64_t next_tx_frame;
 343         enum cvmx_usb_pipe_flags flags;
 344         enum cvmx_usb_speed device_speed;
 345         enum cvmx_usb_transfer transfer_type;
 346         enum cvmx_usb_direction transfer_dir;
 347         int multi_count;
 348         uint16_t max_packet;
 349         uint8_t device_addr;
 350         uint8_t endpoint_num;
 351         uint8_t hub_device_addr;
 352         uint8_t hub_port;
 353         uint8_t pid_toggle;
 354         uint8_t channel;
 355         int8_t split_sc_frame;
 356 };
 357
 358 struct cvmx_usb_tx_fifo {
 359         struct {
 360                 int channel;
 361                 int size;
 362                 uint64_t address;
 363         } entry[MAX_CHANNELS+1];
 364         int head;
 365         int tail;
 366 };
 367
 368 /**
 369  * struct cvmx_usb_state - the state of the USB block
 370  *
 371  * init_flags:             Flags passed to initialize.
 372  * index:                  Which USB block this is for.
 373  * idle_hardware_channels: Bit set for every idle hardware channel.
 374  * usbcx_hprt:             Stored port status so we don't need to read a CSR to
 375  *                         determine splits.
 376  * pipe_for_channel:       Map channels to pipes.
 377  * pipe:                   Storage for pipes.
 378  * indent:                 Used by debug output to indent functions.
 379  * port_status:            Last port status used for change notification.
 380  * idle_pipes:             List of open pipes that have no transactions.
 381  * active_pipes:           Active pipes indexed by transfer type.
 382  * frame_number:           Increments every SOF interrupt for time keeping.
 383  * active_split:           Points to the current active split, or NULL.
 384  */
 385 struct cvmx_usb_state {
 386         int init_flags;
 387         int index;
 388         int idle_hardware_channels;
 389         union cvmx_usbcx_hprt usbcx_hprt;
 390         struct cvmx_usb_pipe *pipe_for_channel[MAX_CHANNELS];
 391         int indent;
 392         struct cvmx_usb_port_status port_status;
 393         struct list_head idle_pipes;
 394         struct list_head active_pipes[4];
 395         uint64_t frame_number;
 396         struct cvmx_usb_transaction *active_split;
 397         struct cvmx_usb_tx_fifo periodic;
 398         struct cvmx_usb_tx_fifo nonperiodic;
 399 };
 400
 401 struct octeon_hcd {
 402         spinlock_t lock;
 403         struct cvmx_usb_state usb;
 404 };
 405
 406 /* This macro spins on a field waiting for it to reach a value */
 407 #define CVMX_WAIT_FOR_FIELD32(address, type, field, op, value, timeout_usec)\
 408         ({int result;                                                       \
 409         do {                                                                \
 410                 uint64_t done = cvmx_get_cycle() + (uint64_t)timeout_usec * \
 411                         octeon_get_clock_rate() / 1000000;                  \
 412                 type c;                                                     \
 413                 while (1) {                                                 \
 414                         c.u32 = __cvmx_usb_read_csr32(usb, address);        \
 415                         if (c.s.field op (value)) {                         \
 416                                 result = 0;                                 \
 417                                 break;                                      \
 418                         } else if (cvmx_get_cycle() > done) {               \
 419                                 result = -1;                                \
 420                                 break;                                      \
 421                         } else                                              \
 422                                 cvmx_wait(100);                             \
 423                 }                                                           \
 424         } while (0);                                                        \
 425         result; })
 426
 427 /*
 428  * This macro logically sets a single field in a CSR. It does the sequence
 429  * read, modify, and write
 430  */
 431 #define USB_SET_FIELD32(address, type, field, value)            \
 432         do {                                                    \
 433                 type c;                                         \
 434                 c.u32 = __cvmx_usb_read_csr32(usb, address);    \
 435                 c.s.field = value;                              \
 436                 __cvmx_usb_write_csr32(usb, address, c.u32);    \
 437         } while (0)
 438
 439 /* Returns the IO address to push/pop stuff data from the FIFOs */
 440 #define USB_FIFO_ADDRESS(channel, usb_index) \
 441         (CVMX_USBCX_GOTGCTL(usb_index) + ((channel)+1)*0x1000)
 442
 443 /**
 444  * struct octeon_temp_buffer - a bounce buffer for USB transfers
 445  * @temp_buffer: the newly allocated temporary buffer (including meta-data)
 446  * @orig_buffer: the original buffer passed by the USB stack
 447  * @data:        the newly allocated temporary buffer (excluding meta-data)
 448  *
 449  * Both the DMA engine and FIFO mode will always transfer full 32-bit words. If
 450  * the buffer is too short, we need to allocate a temporary one, and this struct
 451  * represents it.
 452  */
 453 struct octeon_temp_buffer {
 454         void *temp_buffer;
 455         void *orig_buffer;
 456         u8 data[0];
 457 };
 458
 459 static inline struct octeon_hcd *cvmx_usb_to_octeon(struct cvmx_usb_state *p)
 460 {
 461         return container_of(p, struct octeon_hcd, usb);
 462 }
 463
 464 static inline struct usb_hcd *octeon_to_hcd(struct octeon_hcd *p)
 465 {
 466         return container_of((void *)p, struct usb_hcd, hcd_priv);
 467 }
 468
 469 /**
 470  * octeon_alloc_temp_buffer - allocate a temporary buffer for USB transfer
 471  *                            (if needed)
 472  * @urb:        URB.
 473  * @mem_flags:  Memory allocation flags.
 474  *
 475  * This function allocates a temporary bounce buffer whenever it's needed
 476  * due to HW limitations.
 477  */
 478 static int octeon_alloc_temp_buffer(struct urb *urb, gfp_t mem_flags)
 479 {
 480         struct octeon_temp_buffer *temp;
 481
 482         if (urb->num_sgs || urb->sg ||
 483             (urb->transfer_flags & URB_NO_TRANSFER_DMA_MAP) ||
 484             !(urb->transfer_buffer_length % sizeof(u32)))
 485                 return 0;
 486
 487         temp = kmalloc(ALIGN(urb->transfer_buffer_length, sizeof(u32)) +
 488                        sizeof(*temp), mem_flags);
 489         if (!temp)
 490                 return -ENOMEM;
 491
 492         temp->temp_buffer = temp;
 493         temp->orig_buffer = urb->transfer_buffer;
 494         if (usb_urb_dir_out(urb))
 495                 memcpy(temp->data, urb->transfer_buffer,
 496                        urb->transfer_buffer_length);
 497         urb->transfer_buffer = temp->data;
 498         urb->transfer_flags |= URB_ALIGNED_TEMP_BUFFER;
 499
 500         return 0;
 501 }
 502
 503 /**
 504  * octeon_free_temp_buffer - free a temporary buffer used by USB transfers.
 505  * @urb: URB.
 506  *
 507  * Frees a buffer allocated by octeon_alloc_temp_buffer().
 508  */
 509 static void octeon_free_temp_buffer(struct urb *urb)
 510 {
 511         struct octeon_temp_buffer *temp;
 512
 513         if (!(urb->transfer_flags & URB_ALIGNED_TEMP_BUFFER))
 514                 return;
 515
 516         temp = container_of(urb->transfer_buffer, struct octeon_temp_buffer,
 517                             data);
 518         if (usb_urb_dir_in(urb))
 519                 memcpy(temp->orig_buffer, urb->transfer_buffer,
 520                        urb->actual_length);
 521         urb->transfer_buffer = temp->orig_buffer;
 522         urb->transfer_flags &= ~URB_ALIGNED_TEMP_BUFFER;
 523         kfree(temp->temp_buffer);
 524 }
 525
 526 /**
 527  * octeon_map_urb_for_dma - Octeon-specific map_urb_for_dma().
 528  * @hcd:        USB HCD structure.
 529  * @urb:        URB.
 530  * @mem_flags:  Memory allocation flags.
 531  */
 532 static int octeon_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
 533                                   gfp_t mem_flags)
 534 {
 535         int ret;
 536
 537         ret = octeon_alloc_temp_buffer(urb, mem_flags);
 538         if (ret)
 539                 return ret;
 540
 541         ret = usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);
 542         if (ret)
 543                 octeon_free_temp_buffer(urb);
 544
 545         return ret;
 546 }
 547
 548 /**
 549  * octeon_unmap_urb_for_dma - Octeon-specific unmap_urb_for_dma()
 550  * @hcd:        USB HCD structure.
 551  * @urb:        URB.
 552  */
 553 static void octeon_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)
 554 {
 555         usb_hcd_unmap_urb_for_dma(hcd, urb);
 556         octeon_free_temp_buffer(urb);
 557 }
 558
 559 /**
 560  * Read a USB 32bit CSR. It performs the necessary address swizzle
 561  * for 32bit CSRs and logs the value in a readable format if
 562  * debugging is on.
 563  *
 564  * @usb:     USB block this access is for
 565  * @address: 64bit address to read
 566  *
 567  * Returns: Result of the read
 568  */
 569 static inline uint32_t __cvmx_usb_read_csr32(struct cvmx_usb_state *usb,
 570                                              uint64_t address)
 571 {
 572         uint32_t result = cvmx_read64_uint32(address ^ 4);
 573         return result;
 574 }
 575
 576
 577 /**
 578  * Write a USB 32bit CSR. It performs the necessary address
 579  * swizzle for 32bit CSRs and logs the value in a readable format
 580  * if debugging is on.
 581  *
 582  * @usb:     USB block this access is for
 583  * @address: 64bit address to write
 584  * @value:   Value to write
 585  */
 586 static inline void __cvmx_usb_write_csr32(struct cvmx_usb_state *usb,
 587                                           uint64_t address, uint32_t value)
 588 {
 589         cvmx_write64_uint32(address ^ 4, value);
 590         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
 591 }
 592
 593
 594 /**
 595  * Read a USB 64bit CSR. It logs the value in a readable format if
 596  * debugging is on.
 597  *
 598  * @usb:     USB block this access is for
 599  * @address: 64bit address to read
 600  *
 601  * Returns: Result of the read
 602  */
 603 static inline uint64_t __cvmx_usb_read_csr64(struct cvmx_usb_state *usb,
 604                                              uint64_t address)
 605 {
 606         uint64_t result = cvmx_read64_uint64(address);
 607         return result;
 608 }
 609
 610
 611 /**
 612  * Write a USB 64bit CSR. It logs the value in a readable format
 613  * if debugging is on.
 614  *
 615  * @usb:     USB block this access is for
 616  * @address: 64bit address to write
 617  * @value:   Value to write
 618  */
 619 static inline void __cvmx_usb_write_csr64(struct cvmx_usb_state *usb,
 620                                           uint64_t address, uint64_t value)
 621 {
 622         cvmx_write64_uint64(address, value);
 623 }
 624
 625 /**
 626  * Return non zero if this pipe connects to a non HIGH speed
 627  * device through a high speed hub.
 628  *
 629  * @usb:    USB block this access is for
 630  * @pipe:   Pipe to check
 631  *
 632  * Returns: Non zero if we need to do split transactions
 633  */
 634 static inline int __cvmx_usb_pipe_needs_split(struct cvmx_usb_state *usb,
 635                                               struct cvmx_usb_pipe *pipe)
 636 {
 637         return pipe->device_speed != CVMX_USB_SPEED_HIGH &&
 638                usb->usbcx_hprt.s.prtspd == CVMX_USB_SPEED_HIGH;
 639 }
 640
 641
 642 /**
 643  * Trivial utility function to return the correct PID for a pipe
 644  *
 645  * @pipe:   pipe to check
 646  *
 647  * Returns: PID for pipe
 648  */
 649 static inline int __cvmx_usb_get_data_pid(struct cvmx_usb_pipe *pipe)
 650 {
 651         if (pipe->pid_toggle)
 652                 return 2; /* Data1 */
 653         return 0; /* Data0 */
 654 }
 655
 656 /**
 657  * Initialize a USB port for use. This must be called before any
 658  * other access to the Octeon USB port is made. The port starts
 659  * off in the disabled state.
 660  *
 661  * @usb:         Pointer to an empty struct cvmx_usb_state
 662  *               that will be populated by the initialize call.
 663  *               This structure is then passed to all other USB
 664  *               functions.
 665  * @usb_port_number:
 666  *               Which Octeon USB port to initialize.
 667  *
 668  * Returns: 0 or a negative error code.
 669  */
 670 static int cvmx_usb_initialize(struct cvmx_usb_state *usb,
 671                                int usb_port_number,
 672                                enum cvmx_usb_initialize_flags flags)
 673 {
 674         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 675         union cvmx_usbnx_usbp_ctl_status usbn_usbp_ctl_status;
 676         int i;
 677
 678         /* At first allow 0-1 for the usb port number */
 679         if ((usb_port_number < 0) || (usb_port_number > 1))
 680                 return -EINVAL;
 681
 682         memset(usb, 0, sizeof(*usb));
 683         usb->init_flags = flags;
 684
 685         /* Initialize the USB state structure */
 686         usb->index = usb_port_number;
 687         INIT_LIST_HEAD(&usb->idle_pipes);
 688         for (i = 0; i < ARRAY_SIZE(usb->active_pipes); i++)
 689                 INIT_LIST_HEAD(&usb->active_pipes[i]);
 690
 691         /*
 692          * Power On Reset and PHY Initialization
 693          *
 694          * 1. Wait for DCOK to assert (nothing to do)
 695          *
 696          * 2a. Write USBN0/1_CLK_CTL[POR] = 1 and
 697          *     USBN0/1_CLK_CTL[HRST,PRST,HCLK_RST] = 0
 698          */
 699         usbn_clk_ctl.u64 =
 700                 __cvmx_usb_read_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index));
 701         usbn_clk_ctl.s.por = 1;
 702         usbn_clk_ctl.s.hrst = 0;
 703         usbn_clk_ctl.s.prst = 0;
 704         usbn_clk_ctl.s.hclk_rst = 0;
 705         usbn_clk_ctl.s.enable = 0;
 706         /*
 707          * 2b. Select the USB reference clock/crystal parameters by writing
 708          *     appropriate values to USBN0/1_CLK_CTL[P_C_SEL, P_RTYPE, P_COM_ON]
 709          */
 710         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND) {
 711                 /*
 712                  * The USB port uses 12/24/48MHz 2.5V board clock
 713                  * source at USB_XO. USB_XI should be tied to GND.
 714                  * Most Octeon evaluation boards require this setting
 715                  */
 716                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX) ||
 717                     OCTEON_IS_MODEL(OCTEON_CN56XX) ||
 718                     OCTEON_IS_MODEL(OCTEON_CN50XX))
 719                         /* From CN56XX,CN50XX,CN31XX,CN30XX manuals */
 720                         usbn_clk_ctl.s.p_rtype = 2; /* p_rclk=1 & p_xenbn=0 */
 721                 else
 722                         /* From CN52XX manual */
 723                         usbn_clk_ctl.s.p_rtype = 1;
 724
 725                 switch (flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK) {
 726                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:
 727                         usbn_clk_ctl.s.p_c_sel = 0;
 728                         break;
 729                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:
 730                         usbn_clk_ctl.s.p_c_sel = 1;
 731                         break;
 732                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:
 733                         usbn_clk_ctl.s.p_c_sel = 2;
 734                         break;
 735                 }
 736         } else {
 737                 /*
 738                  * The USB port uses a 12MHz crystal as clock source
 739                  * at USB_XO and USB_XI
 740                  */
 741                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
 742                         /* From CN31XX,CN30XX manual */
 743                         usbn_clk_ctl.s.p_rtype = 3; /* p_rclk=1 & p_xenbn=1 */
 744                 else
 745                         /* From CN56XX,CN52XX,CN50XX manuals. */
 746                         usbn_clk_ctl.s.p_rtype = 0;
 747
 748                 usbn_clk_ctl.s.p_c_sel = 0;
 749         }
 750         /*
 751          * 2c. Select the HCLK via writing USBN0/1_CLK_CTL[DIVIDE, DIVIDE2] and
 752          *     setting USBN0/1_CLK_CTL[ENABLE] = 1. Divide the core clock down
 753          *     such that USB is as close as possible to 125Mhz
 754          */
 755         {
 756                 int divisor = DIV_ROUND_UP(octeon_get_clock_rate(), 125000000);
 757                 /* Lower than 4 doesn't seem to work properly */
 758                 if (divisor < 4)
 759                         divisor = 4;
 760                 usbn_clk_ctl.s.divide = divisor;
 761                 usbn_clk_ctl.s.divide2 = 0;
 762         }
 763         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 764                                usbn_clk_ctl.u64);
 765         /* 2d. Write USBN0/1_CLK_CTL[HCLK_RST] = 1 */
 766         usbn_clk_ctl.s.hclk_rst = 1;
 767         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 768                                usbn_clk_ctl.u64);
 769         /* 2e.  Wait 64 core-clock cycles for HCLK to stabilize */
 770         cvmx_wait(64);
 771         /*
 772          * 3. Program the power-on reset field in the USBN clock-control
 773          *    register:
 774          *    USBN_CLK_CTL[POR] = 0
 775          */
 776         usbn_clk_ctl.s.por = 0;
 777         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 778                                usbn_clk_ctl.u64);
 779         /* 4. Wait 1 ms for PHY clock to start */
 780         mdelay(1);
 781         /*
 782          * 5. Program the Reset input from automatic test equipment field in the
 783          *    USBP control and status register:
 784          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 1
 785          */
 786         usbn_usbp_ctl_status.u64 = __cvmx_usb_read_csr64(usb,
 787                         CVMX_USBNX_USBP_CTL_STATUS(usb->index));
 788         usbn_usbp_ctl_status.s.ate_reset = 1;
 789         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 790                                usbn_usbp_ctl_status.u64);
 791         /* 6. Wait 10 cycles */
 792         cvmx_wait(10);
 793         /*
 794          * 7. Clear ATE_RESET field in the USBN clock-control register:
 795          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 0
 796          */
 797         usbn_usbp_ctl_status.s.ate_reset = 0;
 798         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 799                                usbn_usbp_ctl_status.u64);
 800         /*
 801          * 8. Program the PHY reset field in the USBN clock-control register:
 802          *    USBN_CLK_CTL[PRST] = 1
 803          */
 804         usbn_clk_ctl.s.prst = 1;
 805         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 806                                usbn_clk_ctl.u64);
 807         /*
 808          * 9. Program the USBP control and status register to select host or
 809          *    device mode. USBN_USBP_CTL_STATUS[HST_MODE] = 0 for host, = 1 for
 810          *    device
 811          */
 812         usbn_usbp_ctl_status.s.hst_mode = 0;
 813         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 814                                usbn_usbp_ctl_status.u64);
 815         /* 10. Wait 1 us */
 816         udelay(1);
 817         /*
 818          * 11. Program the hreset_n field in the USBN clock-control register:
 819          *     USBN_CLK_CTL[HRST] = 1
 820          */
 821         usbn_clk_ctl.s.hrst = 1;
 822         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 823                                usbn_clk_ctl.u64);
 824         /* 12. Proceed to USB core initialization */
 825         usbn_clk_ctl.s.enable = 1;
 826         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 827                                usbn_clk_ctl.u64);
 828         udelay(1);
 829
 830         /*
 831          * USB Core Initialization
 832          *
 833          * 1. Read USBC_GHWCFG1, USBC_GHWCFG2, USBC_GHWCFG3, USBC_GHWCFG4 to
 834          *    determine USB core configuration parameters.
 835          *
 836          *    Nothing needed
 837          *
 838          * 2. Program the following fields in the global AHB configuration
 839          *    register (USBC_GAHBCFG)
 840          *    DMA mode, USBC_GAHBCFG[DMAEn]: 1 = DMA mode, 0 = slave mode
 841          *    Burst length, USBC_GAHBCFG[HBSTLEN] = 0
 842          *    Nonperiodic TxFIFO empty level (slave mode only),
 843          *    USBC_GAHBCFG[NPTXFEMPLVL]
 844          *    Periodic TxFIFO empty level (slave mode only),
 845          *    USBC_GAHBCFG[PTXFEMPLVL]
 846          *    Global interrupt mask, USBC_GAHBCFG[GLBLINTRMSK] = 1
 847          */
 848         {
 849                 union cvmx_usbcx_gahbcfg usbcx_gahbcfg;
 850                 /* Due to an errata, CN31XX doesn't support DMA */
 851                 if (OCTEON_IS_MODEL(OCTEON_CN31XX))
 852                         usb->init_flags |= CVMX_USB_INITIALIZE_FLAGS_NO_DMA;
 853                 usbcx_gahbcfg.u32 = 0;
 854                 usbcx_gahbcfg.s.dmaen = !(usb->init_flags &
 855                                           CVMX_USB_INITIALIZE_FLAGS_NO_DMA);
 856                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 857                         /* Only use one channel with non DMA */
 858                         usb->idle_hardware_channels = 0x1;
 859                 else if (OCTEON_IS_MODEL(OCTEON_CN5XXX))
 860                         /* CN5XXX have an errata with channel 3 */
 861                         usb->idle_hardware_channels = 0xf7;
 862                 else
 863                         usb->idle_hardware_channels = 0xff;
 864                 usbcx_gahbcfg.s.hbstlen = 0;
 865                 usbcx_gahbcfg.s.nptxfemplvl = 1;
 866                 usbcx_gahbcfg.s.ptxfemplvl = 1;
 867                 usbcx_gahbcfg.s.glblintrmsk = 1;
 868                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GAHBCFG(usb->index),
 869                                        usbcx_gahbcfg.u32);
 870         }
 871         /*
 872          * 3. Program the following fields in USBC_GUSBCFG register.
 873          *    HS/FS timeout calibration, USBC_GUSBCFG[TOUTCAL] = 0
 874          *    ULPI DDR select, USBC_GUSBCFG[DDRSEL] = 0
 875          *    USB turnaround time, USBC_GUSBCFG[USBTRDTIM] = 0x5
 876          *    PHY low-power clock select, USBC_GUSBCFG[PHYLPWRCLKSEL] = 0
 877          */
 878         {
 879                 union cvmx_usbcx_gusbcfg usbcx_gusbcfg;
 880
 881                 usbcx_gusbcfg.u32 = __cvmx_usb_read_csr32(usb,
 882                                 CVMX_USBCX_GUSBCFG(usb->index));
 883                 usbcx_gusbcfg.s.toutcal = 0;
 884                 usbcx_gusbcfg.s.ddrsel = 0;
 885                 usbcx_gusbcfg.s.usbtrdtim = 0x5;
 886                 usbcx_gusbcfg.s.phylpwrclksel = 0;
 887                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index),
 888                                        usbcx_gusbcfg.u32);
 889         }
 890         /*
 891          * 4. The software must unmask the following bits in the USBC_GINTMSK
 892          *    register.
 893          *    OTG interrupt mask, USBC_GINTMSK[OTGINTMSK] = 1
 894          *    Mode mismatch interrupt mask, USBC_GINTMSK[MODEMISMSK] = 1
 895          */
 896         {
 897                 union cvmx_usbcx_gintmsk usbcx_gintmsk;
 898                 int channel;
 899
 900                 usbcx_gintmsk.u32 = __cvmx_usb_read_csr32(usb,
 901                                 CVMX_USBCX_GINTMSK(usb->index));
 902                 usbcx_gintmsk.s.otgintmsk = 1;
 903                 usbcx_gintmsk.s.modemismsk = 1;
 904                 usbcx_gintmsk.s.hchintmsk = 1;
 905                 usbcx_gintmsk.s.sofmsk = 0;
 906                 /* We need RX FIFO interrupts if we don't have DMA */
 907                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 908                         usbcx_gintmsk.s.rxflvlmsk = 1;
 909                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTMSK(usb->index),
 910                                        usbcx_gintmsk.u32);
 911
 912                 /*
 913                  * Disable all channel interrupts. We'll enable them per channel
 914                  * later.
 915                  */
 916                 for (channel = 0; channel < 8; channel++)
 917                         __cvmx_usb_write_csr32(usb,
 918                                 CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
 919         }
 920
 921         {
 922                 /*
 923                  * Host Port Initialization
 924                  *
 925                  * 1. Program the host-port interrupt-mask field to unmask,
 926                  *    USBC_GINTMSK[PRTINT] = 1
 927                  */
 928                 USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
 929                                 union cvmx_usbcx_gintmsk, prtintmsk, 1);
 930                 USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
 931                                 union cvmx_usbcx_gintmsk, disconnintmsk, 1);
 932                 /*
 933                  * 2. Program the USBC_HCFG register to select full-speed host
 934                  *    or high-speed host.
 935                  */
 936                 {
 937                         union cvmx_usbcx_hcfg usbcx_hcfg;
 938
 939                         usbcx_hcfg.u32 = __cvmx_usb_read_csr32(usb,
 940                                         CVMX_USBCX_HCFG(usb->index));
 941                         usbcx_hcfg.s.fslssupp = 0;
 942                         usbcx_hcfg.s.fslspclksel = 0;
 943                         __cvmx_usb_write_csr32(usb,
 944                                         CVMX_USBCX_HCFG(usb->index),
 945                                         usbcx_hcfg.u32);
 946                 }
 947                 /*
 948                  * 3. Program the port power bit to drive VBUS on the USB,
 949                  *    USBC_HPRT[PRTPWR] = 1
 950                  */
 951                 USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index),
 952                                 union cvmx_usbcx_hprt, prtpwr, 1);
 953
 954                 /*
 955                  * Steps 4-15 from the manual are done later in the port enable
 956                  */
 957         }
 958
 959         return 0;
 960 }
 961
 962
 963 /**
 964  * Shutdown a USB port after a call to cvmx_usb_initialize().
 965  * The port should be disabled with all pipes closed when this
 966  * function is called.
 967  *
 968  * @usb: USB device state populated by cvmx_usb_initialize().
 969  *
 970  * Returns: 0 or a negative error code.
 971  */
 972 static int cvmx_usb_shutdown(struct cvmx_usb_state *usb)
 973 {
 974         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 975
 976         /* Make sure all pipes are closed */
 977         if (!list_empty(&usb->idle_pipes) ||
 978             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_ISOCHRONOUS]) ||
 979             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_INTERRUPT]) ||
 980             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_CONTROL]) ||
 981             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_BULK]))
 982                 return -EBUSY;
 983
 984         /* Disable the clocks and put them in power on reset */
 985         usbn_clk_ctl.u64 = __cvmx_usb_read_csr64(usb,
 986                         CVMX_USBNX_CLK_CTL(usb->index));
 987         usbn_clk_ctl.s.enable = 1;
 988         usbn_clk_ctl.s.por = 1;
 989         usbn_clk_ctl.s.hclk_rst = 1;
 990         usbn_clk_ctl.s.prst = 0;
 991         usbn_clk_ctl.s.hrst = 0;
 992         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 993                                usbn_clk_ctl.u64);
 994         return 0;
 995 }
 996
 997
 998 /**
 999  * Enable a USB port. After this call succeeds, the USB port is
1000  * online and servicing requests.
1001  *
1002  * @usb: USB device state populated by cvmx_usb_initialize().
1003  *
1004  * Returns: 0 or a negative error code.
1005  */
1006 static int cvmx_usb_enable(struct cvmx_usb_state *usb)
1007 {
1008         union cvmx_usbcx_ghwcfg3 usbcx_ghwcfg3;
1009
1010         usb->usbcx_hprt.u32 = __cvmx_usb_read_csr32(usb,
1011                         CVMX_USBCX_HPRT(usb->index));
1012
1013         /*
1014          * If the port is already enabled the just return. We don't need to do
1015          * anything
1016          */
1017         if (usb->usbcx_hprt.s.prtena)
1018                 return 0;
1019
1020         /* If there is nothing plugged into the port then fail immediately */
1021         if (!usb->usbcx_hprt.s.prtconnsts)
1022                 return -ETIMEDOUT;
1023
1024         /* Program the port reset bit to start the reset process */
1025         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt,
1026                         prtrst, 1);
1027
1028         /*
1029          * Wait at least 50ms (high speed), or 10ms (full speed) for the reset
1030          * process to complete.
1031          */
1032         mdelay(50);
1033
1034         /* Program the port reset bit to 0, USBC_HPRT[PRTRST] = 0 */
1035         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt,
1036                         prtrst, 0);
1037
1038         /* Wait for the USBC_HPRT[PRTENA]. */
1039         if (CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_HPRT(usb->index),
1040                                 union cvmx_usbcx_hprt, prtena, ==, 1, 100000))
1041                 return -ETIMEDOUT;
1042
1043         /*
1044          * Read the port speed field to get the enumerated speed,
1045          * USBC_HPRT[PRTSPD].
1046          */
1047         usb->usbcx_hprt.u32 = __cvmx_usb_read_csr32(usb,
1048                         CVMX_USBCX_HPRT(usb->index));
1049         usbcx_ghwcfg3.u32 = __cvmx_usb_read_csr32(usb,
1050                         CVMX_USBCX_GHWCFG3(usb->index));
1051
1052         /*
1053          * 13. Program the USBC_GRXFSIZ register to select the size of the
1054          *     receive FIFO (25%).
1055          */
1056         USB_SET_FIELD32(CVMX_USBCX_GRXFSIZ(usb->index),
1057                         union cvmx_usbcx_grxfsiz, rxfdep,
1058                         usbcx_ghwcfg3.s.dfifodepth / 4);
1059         /*
1060          * 14. Program the USBC_GNPTXFSIZ register to select the size and the
1061          *     start address of the non- periodic transmit FIFO for nonperiodic
1062          *     transactions (50%).
1063          */
1064         {
1065                 union cvmx_usbcx_gnptxfsiz siz;
1066
1067                 siz.u32 = __cvmx_usb_read_csr32(usb,
1068                                 CVMX_USBCX_GNPTXFSIZ(usb->index));
1069                 siz.s.nptxfdep = usbcx_ghwcfg3.s.dfifodepth / 2;
1070                 siz.s.nptxfstaddr = usbcx_ghwcfg3.s.dfifodepth / 4;
1071                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index),
1072                                        siz.u32);
1073         }
1074         /*
1075          * 15. Program the USBC_HPTXFSIZ register to select the size and start
1076          *     address of the periodic transmit FIFO for periodic transactions
1077          *     (25%).
1078          */
1079         {
1080                 union cvmx_usbcx_hptxfsiz siz;
1081
1082                 siz.u32 = __cvmx_usb_read_csr32(usb,
1083                                 CVMX_USBCX_HPTXFSIZ(usb->index));
1084                 siz.s.ptxfsize = usbcx_ghwcfg3.s.dfifodepth / 4;
1085                 siz.s.ptxfstaddr = 3 * usbcx_ghwcfg3.s.dfifodepth / 4;
1086                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index),
1087                                        siz.u32);
1088         }
1089         /* Flush all FIFOs */
1090         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1091                         union cvmx_usbcx_grstctl, txfnum, 0x10);
1092         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1093                         union cvmx_usbcx_grstctl, txfflsh, 1);
1094         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1095                               union cvmx_usbcx_grstctl,
1096                               txfflsh, ==, 0, 100);
1097         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1098                         union cvmx_usbcx_grstctl, rxfflsh, 1);
1099         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1100                               union cvmx_usbcx_grstctl,
1101                               rxfflsh, ==, 0, 100);
1102
1103         return 0;
1104 }
1105
1106
1107 /**
1108  * Disable a USB port. After this call the USB port will not
1109  * generate data transfers and will not generate events.
1110  * Transactions in process will fail and call their
1111  * associated callbacks.
1112  *
1113  * @usb: USB device state populated by cvmx_usb_initialize().
1114  *
1115  * Returns: 0 or a negative error code.
1116  */
1117 static int cvmx_usb_disable(struct cvmx_usb_state *usb)
1118 {
1119         /* Disable the port */
1120         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt,
1121                         prtena, 1);
1122         return 0;
1123 }
1124
1125
1126 /**
1127  * Get the current state of the USB port. Use this call to
1128  * determine if the usb port has anything connected, is enabled,
1129  * or has some sort of error condition. The return value of this
1130  * call has "changed" bits to signal of the value of some fields
1131  * have changed between calls.
1132  *
1133  * @usb: USB device state populated by cvmx_usb_initialize().
1134  *
1135  * Returns: Port status information
1136  */
1137 static struct cvmx_usb_port_status cvmx_usb_get_status(
1138                 struct cvmx_usb_state *usb)
1139 {
1140         union cvmx_usbcx_hprt usbc_hprt;
1141         struct cvmx_usb_port_status result;
1142
1143         memset(&result, 0, sizeof(result));
1144
1145         usbc_hprt.u32 = __cvmx_usb_read_csr32(usb,
1146                         CVMX_USBCX_HPRT(usb->index));
1147         result.port_enabled = usbc_hprt.s.prtena;
1148         result.port_over_current = usbc_hprt.s.prtovrcurract;
1149         result.port_powered = usbc_hprt.s.prtpwr;
1150         result.port_speed = usbc_hprt.s.prtspd;
1151         result.connected = usbc_hprt.s.prtconnsts;
1152         result.connect_change =
1153                 (result.connected != usb->port_status.connected);
1154
1155         return result;
1156 }
1157
1158 /**
1159  * Open a virtual pipe between the host and a USB device. A pipe
1160  * must be opened before data can be transferred between a device
1161  * and Octeon.
1162  *
1163  * @usb:             USB device state populated by cvmx_usb_initialize().
1164  * @device_addr:
1165  *                   USB device address to open the pipe to
1166  *                   (0-127).
1167  * @endpoint_num:
1168  *                   USB endpoint number to open the pipe to
1169  *                   (0-15).
1170  * @device_speed:
1171  *                   The speed of the device the pipe is going
1172  *                   to. This must match the device's speed,
1173  *                   which may be different than the port speed.
1174  * @max_packet:      The maximum packet length the device can
1175  *                   transmit/receive (low speed=0-8, full
1176  *                   speed=0-1023, high speed=0-1024). This value
1177  *                   comes from the standard endpoint descriptor
1178  *                   field wMaxPacketSize bits <10:0>.
1179  * @transfer_type:
1180  *                   The type of transfer this pipe is for.
1181  * @transfer_dir:
1182  *                   The direction the pipe is in. This is not
1183  *                   used for control pipes.
1184  * @interval:        For ISOCHRONOUS and INTERRUPT transfers,
1185  *                   this is how often the transfer is scheduled
1186  *                   for. All other transfers should specify
1187  *                   zero. The units are in frames (8000/sec at
1188  *                   high speed, 1000/sec for full speed).
1189  * @multi_count:
1190  *                   For high speed devices, this is the maximum
1191  *                   allowed number of packet per microframe.
1192  *                   Specify zero for non high speed devices. This
1193  *                   value comes from the standard endpoint descriptor
1194  *                   field wMaxPacketSize bits <12:11>.
1195  * @hub_device_addr:
1196  *                   Hub device address this device is connected
1197  *                   to. Devices connected directly to Octeon
1198  *                   use zero. This is only used when the device
1199  *                   is full/low speed behind a high speed hub.
1200  *                   The address will be of the high speed hub,
1201  *                   not and full speed hubs after it.
1202  * @hub_port:        Which port on the hub the device is
1203  *                   connected. Use zero for devices connected
1204  *                   directly to Octeon. Like hub_device_addr,
1205  *                   this is only used for full/low speed
1206  *                   devices behind a high speed hub.
1207  *
1208  * Returns: A non-NULL value is a pipe. NULL means an error.
1209  */
1210 static struct cvmx_usb_pipe *cvmx_usb_open_pipe(struct cvmx_usb_state *usb,
1211                                                 int device_addr,
1212                                                 int endpoint_num,
1213                                                 enum cvmx_usb_speed
1214                                                         device_speed,
1215                                                 int max_packet,
1216                                                 enum cvmx_usb_transfer
1217                                                         transfer_type,
1218                                                 enum cvmx_usb_direction
1219                                                         transfer_dir,
1220                                                 int interval, int multi_count,
1221                                                 int hub_device_addr,
1222                                                 int hub_port)
1223 {
1224         struct cvmx_usb_pipe *pipe;
1225
1226         if (unlikely((device_addr < 0) || (device_addr > MAX_USB_ADDRESS)))
1227                 return NULL;
1228         if (unlikely((endpoint_num < 0) || (endpoint_num > MAX_USB_ENDPOINT)))
1229                 return NULL;
1230         if (unlikely(device_speed > CVMX_USB_SPEED_LOW))
1231                 return NULL;
1232         if (unlikely((max_packet <= 0) || (max_packet > 1024)))
1233                 return NULL;
1234         if (unlikely(transfer_type > CVMX_USB_TRANSFER_INTERRUPT))
1235                 return NULL;
1236         if (unlikely((transfer_dir != CVMX_USB_DIRECTION_OUT) &&
1237                 (transfer_dir != CVMX_USB_DIRECTION_IN)))
1238                 return NULL;
1239         if (unlikely(interval < 0))
1240                 return NULL;
1241         if (unlikely((transfer_type == CVMX_USB_TRANSFER_CONTROL) && interval))
1242                 return NULL;
1243         if (unlikely(multi_count < 0))
1244                 return NULL;
1245         if (unlikely((device_speed != CVMX_USB_SPEED_HIGH) &&
1246                 (multi_count != 0)))
1247                 return NULL;
1248         if (unlikely((hub_device_addr < 0) ||
1249                 (hub_device_addr > MAX_USB_ADDRESS)))
1250                 return NULL;
1251         if (unlikely((hub_port < 0) || (hub_port > MAX_USB_HUB_PORT)))
1252                 return NULL;
1253
1254         pipe = kzalloc(sizeof(*pipe), GFP_ATOMIC);
1255         if (!pipe)
1256                 return NULL;
1257         if ((device_speed == CVMX_USB_SPEED_HIGH) &&
1258                 (transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1259                 (transfer_type == CVMX_USB_TRANSFER_BULK))
1260                 pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
1261         pipe->device_addr = device_addr;
1262         pipe->endpoint_num = endpoint_num;
1263         pipe->device_speed = device_speed;
1264         pipe->max_packet = max_packet;
1265         pipe->transfer_type = transfer_type;
1266         pipe->transfer_dir = transfer_dir;
1267         INIT_LIST_HEAD(&pipe->transactions);
1268
1269         /*
1270          * All pipes use interval to rate limit NAK processing. Force an
1271          * interval if one wasn't supplied
1272          */
1273         if (!interval)
1274                 interval = 1;
1275         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1276                 pipe->interval = interval*8;
1277                 /* Force start splits to be schedule on uFrame 0 */
1278                 pipe->next_tx_frame = ((usb->frame_number+7)&~7) +
1279                                         pipe->interval;
1280         } else {
1281                 pipe->interval = interval;
1282                 pipe->next_tx_frame = usb->frame_number + pipe->interval;
1283         }
1284         pipe->multi_count = multi_count;
1285         pipe->hub_device_addr = hub_device_addr;
1286         pipe->hub_port = hub_port;
1287         pipe->pid_toggle = 0;
1288         pipe->split_sc_frame = -1;
1289         list_add_tail(&pipe->node, &usb->idle_pipes);
1290
1291         /*
1292          * We don't need to tell the hardware about this pipe yet since
1293          * it doesn't have any submitted requests
1294          */
1295
1296         return pipe;
1297 }
1298
1299
1300 /**
1301  * Poll the RX FIFOs and remove data as needed. This function is only used
1302  * in non DMA mode. It is very important that this function be called quickly
1303  * enough to prevent FIFO overflow.
1304  *
1305  * @usb:        USB device state populated by cvmx_usb_initialize().
1306  */
1307 static void __cvmx_usb_poll_rx_fifo(struct cvmx_usb_state *usb)
1308 {
1309         union cvmx_usbcx_grxstsph rx_status;
1310         int channel;
1311         int bytes;
1312         uint64_t address;
1313         uint32_t *ptr;
1314
1315         rx_status.u32 = __cvmx_usb_read_csr32(usb,
1316                         CVMX_USBCX_GRXSTSPH(usb->index));
1317         /* Only read data if IN data is there */
1318         if (rx_status.s.pktsts != 2)
1319                 return;
1320         /* Check if no data is available */
1321         if (!rx_status.s.bcnt)
1322                 return;
1323
1324         channel = rx_status.s.chnum;
1325         bytes = rx_status.s.bcnt;
1326         if (!bytes)
1327                 return;
1328
1329         /* Get where the DMA engine would have written this data */
1330         address = __cvmx_usb_read_csr64(usb,
1331                         CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8);
1332
1333         ptr = cvmx_phys_to_ptr(address);
1334         __cvmx_usb_write_csr64(usb,
1335                                CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8,
1336                                address + bytes);
1337
1338         /* Loop writing the FIFO data for this packet into memory */
1339         while (bytes > 0) {
1340                 *ptr++ = __cvmx_usb_read_csr32(usb,
1341                                 USB_FIFO_ADDRESS(channel, usb->index));
1342                 bytes -= 4;
1343         }
1344         CVMX_SYNCW;
1345 }
1346
1347
1348 /**
1349  * Fill the TX hardware fifo with data out of the software
1350  * fifos
1351  *
1352  * @usb:            USB device state populated by cvmx_usb_initialize().
1353  * @fifo:           Software fifo to use
1354  * @available:      Amount of space in the hardware fifo
1355  *
1356  * Returns: Non zero if the hardware fifo was too small and needs
1357  *          to be serviced again.
1358  */
1359 static int __cvmx_usb_fill_tx_hw(struct cvmx_usb_state *usb,
1360                                  struct cvmx_usb_tx_fifo *fifo, int available)
1361 {
1362         /*
1363          * We're done either when there isn't anymore space or the software FIFO
1364          * is empty
1365          */
1366         while (available && (fifo->head != fifo->tail)) {
1367                 int i = fifo->tail;
1368                 const uint32_t *ptr = cvmx_phys_to_ptr(fifo->entry[i].address);
1369                 uint64_t csr_address = USB_FIFO_ADDRESS(fifo->entry[i].channel,
1370                                                         usb->index) ^ 4;
1371                 int words = available;
1372
1373                 /* Limit the amount of data to waht the SW fifo has */
1374                 if (fifo->entry[i].size <= available) {
1375                         words = fifo->entry[i].size;
1376                         fifo->tail++;
1377                         if (fifo->tail > MAX_CHANNELS)
1378                                 fifo->tail = 0;
1379                 }
1380
1381                 /* Update the next locations and counts */
1382                 available -= words;
1383                 fifo->entry[i].address += words * 4;
1384                 fifo->entry[i].size -= words;
1385
1386                 /*
1387                  * Write the HW fifo data. The read every three writes is due
1388                  * to an errata on CN3XXX chips
1389                  */
1390                 while (words > 3) {
1391                         cvmx_write64_uint32(csr_address, *ptr++);
1392                         cvmx_write64_uint32(csr_address, *ptr++);
1393                         cvmx_write64_uint32(csr_address, *ptr++);
1394                         cvmx_read64_uint64(
1395                                         CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1396                         words -= 3;
1397                 }
1398                 cvmx_write64_uint32(csr_address, *ptr++);
1399                 if (--words) {
1400                         cvmx_write64_uint32(csr_address, *ptr++);
1401                         if (--words)
1402                                 cvmx_write64_uint32(csr_address, *ptr++);
1403                 }
1404                 cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1405         }
1406         return fifo->head != fifo->tail;
1407 }
1408
1409
1410 /**
1411  * Check the hardware FIFOs and fill them as needed
1412  *
1413  * @usb:        USB device state populated by cvmx_usb_initialize().
1414  */
1415 static void __cvmx_usb_poll_tx_fifo(struct cvmx_usb_state *usb)
1416 {
1417         if (usb->periodic.head != usb->periodic.tail) {
1418                 union cvmx_usbcx_hptxsts tx_status;
1419
1420                 tx_status.u32 = __cvmx_usb_read_csr32(usb,
1421                                 CVMX_USBCX_HPTXSTS(usb->index));
1422                 if (__cvmx_usb_fill_tx_hw(usb, &usb->periodic,
1423                                           tx_status.s.ptxfspcavail))
1424                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1425                                         union cvmx_usbcx_gintmsk,
1426                                         ptxfempmsk, 1);
1427                 else
1428                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1429                                         union cvmx_usbcx_gintmsk,
1430                                         ptxfempmsk, 0);
1431         }
1432
1433         if (usb->nonperiodic.head != usb->nonperiodic.tail) {
1434                 union cvmx_usbcx_gnptxsts tx_status;
1435
1436                 tx_status.u32 = __cvmx_usb_read_csr32(usb,
1437                                 CVMX_USBCX_GNPTXSTS(usb->index));
1438                 if (__cvmx_usb_fill_tx_hw(usb, &usb->nonperiodic,
1439                                           tx_status.s.nptxfspcavail))
1440                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1441                                         union cvmx_usbcx_gintmsk,
1442                                         nptxfempmsk, 1);
1443                 else
1444                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1445                                         union cvmx_usbcx_gintmsk,
1446                                         nptxfempmsk, 0);
1447         }
1448 }
1449
1450
1451 /**
1452  * Fill the TX FIFO with an outgoing packet
1453  *
1454  * @usb:          USB device state populated by cvmx_usb_initialize().
1455  * @channel:      Channel number to get packet from
1456  */
1457 static void __cvmx_usb_fill_tx_fifo(struct cvmx_usb_state *usb, int channel)
1458 {
1459         union cvmx_usbcx_hccharx hcchar;
1460         union cvmx_usbcx_hcspltx usbc_hcsplt;
1461         union cvmx_usbcx_hctsizx usbc_hctsiz;
1462         struct cvmx_usb_tx_fifo *fifo;
1463
1464         /* We only need to fill data on outbound channels */
1465         hcchar.u32 = __cvmx_usb_read_csr32(usb,
1466                         CVMX_USBCX_HCCHARX(channel, usb->index));
1467         if (hcchar.s.epdir != CVMX_USB_DIRECTION_OUT)
1468                 return;
1469
1470         /* OUT Splits only have data on the start and not the complete */
1471         usbc_hcsplt.u32 = __cvmx_usb_read_csr32(usb,
1472                         CVMX_USBCX_HCSPLTX(channel, usb->index));
1473         if (usbc_hcsplt.s.spltena && usbc_hcsplt.s.compsplt)
1474                 return;
1475
1476         /*
1477          * Find out how many bytes we need to fill and convert it into 32bit
1478          * words.
1479          */
1480         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb,
1481                         CVMX_USBCX_HCTSIZX(channel, usb->index));
1482         if (!usbc_hctsiz.s.xfersize)
1483                 return;
1484
1485         if ((hcchar.s.eptype == CVMX_USB_TRANSFER_INTERRUPT) ||
1486                 (hcchar.s.eptype == CVMX_USB_TRANSFER_ISOCHRONOUS))
1487                 fifo = &usb->periodic;
1488         else
1489                 fifo = &usb->nonperiodic;
1490
1491         fifo->entry[fifo->head].channel = channel;
1492         fifo->entry[fifo->head].address = __cvmx_usb_read_csr64(usb,
1493                         CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8);
1494         fifo->entry[fifo->head].size = (usbc_hctsiz.s.xfersize+3)>>2;
1495         fifo->head++;
1496         if (fifo->head > MAX_CHANNELS)
1497                 fifo->head = 0;
1498
1499         __cvmx_usb_poll_tx_fifo(usb);
1500 }
1501
1502 /**
1503  * Perform channel specific setup for Control transactions. All
1504  * the generic stuff will already have been done in
1505  * __cvmx_usb_start_channel()
1506  *
1507  * @usb:          USB device state populated by cvmx_usb_initialize().
1508  * @channel:      Channel to setup
1509  * @pipe:         Pipe for control transaction
1510  */
1511 static void __cvmx_usb_start_channel_control(struct cvmx_usb_state *usb,
1512                                              int channel,
1513                                              struct cvmx_usb_pipe *pipe)
1514 {
1515         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
1516         struct usb_hcd *hcd = octeon_to_hcd(priv);
1517         struct device *dev = hcd->self.controller;
1518         struct cvmx_usb_transaction *transaction =
1519                 list_first_entry(&pipe->transactions, typeof(*transaction),
1520                                  node);
1521         struct usb_ctrlrequest *header =
1522                 cvmx_phys_to_ptr(transaction->control_header);
1523         int bytes_to_transfer = transaction->buffer_length -
1524                 transaction->actual_bytes;
1525         int packets_to_transfer;
1526         union cvmx_usbcx_hctsizx usbc_hctsiz;
1527
1528         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb,
1529                         CVMX_USBCX_HCTSIZX(channel, usb->index));
1530
1531         switch (transaction->stage) {
1532         case CVMX_USB_STAGE_NON_CONTROL:
1533         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
1534                 dev_err(dev, "%s: ERROR - Non control stage\n", __func__);
1535                 break;
1536         case CVMX_USB_STAGE_SETUP:
1537                 usbc_hctsiz.s.pid = 3; /* Setup */
1538                 bytes_to_transfer = sizeof(*header);
1539                 /* All Control operations start with a setup going OUT */
1540                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1541                                 union cvmx_usbcx_hccharx, epdir,
1542                                 CVMX_USB_DIRECTION_OUT);
1543                 /*
1544                  * Setup send the control header instead of the buffer data. The
1545                  * buffer data will be used in the next stage
1546                  */
1547                 __cvmx_usb_write_csr64(usb,
1548                         CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8,
1549                         transaction->control_header);
1550                 break;
1551         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
1552                 usbc_hctsiz.s.pid = 3; /* Setup */
1553                 bytes_to_transfer = 0;
1554                 /* All Control operations start with a setup going OUT */
1555                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1556                                 union cvmx_usbcx_hccharx, epdir,
1557                                 CVMX_USB_DIRECTION_OUT);
1558
1559                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1560                                 union cvmx_usbcx_hcspltx, compsplt, 1);
1561                 break;
1562         case CVMX_USB_STAGE_DATA:
1563                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1564                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1565                         if (header->bRequestType & USB_DIR_IN)
1566                                 bytes_to_transfer = 0;
1567                         else if (bytes_to_transfer > pipe->max_packet)
1568                                 bytes_to_transfer = pipe->max_packet;
1569                 }
1570                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1571                                 union cvmx_usbcx_hccharx, epdir,
1572                                 ((header->bRequestType & USB_DIR_IN) ?
1573                                         CVMX_USB_DIRECTION_IN :
1574                                         CVMX_USB_DIRECTION_OUT));
1575                 break;
1576         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
1577                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1578                 if (!(header->bRequestType & USB_DIR_IN))
1579                         bytes_to_transfer = 0;
1580                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1581                                 union cvmx_usbcx_hccharx, epdir,
1582                                 ((header->bRequestType & USB_DIR_IN) ?
1583                                         CVMX_USB_DIRECTION_IN :
1584                                         CVMX_USB_DIRECTION_OUT));
1585                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1586                                 union cvmx_usbcx_hcspltx, compsplt, 1);
1587                 break;
1588         case CVMX_USB_STAGE_STATUS:
1589                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1590                 bytes_to_transfer = 0;
1591                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1592                                 union cvmx_usbcx_hccharx, epdir,
1593                                 ((header->bRequestType & USB_DIR_IN) ?
1594                                         CVMX_USB_DIRECTION_OUT :
1595                                         CVMX_USB_DIRECTION_IN));
1596                 break;
1597         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
1598                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1599                 bytes_to_transfer = 0;
1600                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1601                                 union cvmx_usbcx_hccharx, epdir,
1602                                 ((header->bRequestType & USB_DIR_IN) ?
1603                                         CVMX_USB_DIRECTION_OUT :
1604                                         CVMX_USB_DIRECTION_IN));
1605                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1606                                 union cvmx_usbcx_hcspltx, compsplt, 1);
1607                 break;
1608         }
1609
1610         /*
1611          * Make sure the transfer never exceeds the byte limit of the hardware.
1612          * Further bytes will be sent as continued transactions
1613          */
1614         if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1615                 /* Round MAX_TRANSFER_BYTES to a multiple of out packet size */
1616                 bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
1617                 bytes_to_transfer *= pipe->max_packet;
1618         }
1619
1620         /*
1621          * Calculate the number of packets to transfer. If the length is zero
1622          * we still need to transfer one packet
1623          */
1624         packets_to_transfer = DIV_ROUND_UP(bytes_to_transfer,
1625                                            pipe->max_packet);
1626         if (packets_to_transfer == 0)
1627                 packets_to_transfer = 1;
1628         else if ((packets_to_transfer > 1) &&
1629                         (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1630                 /*
1631                  * Limit to one packet when not using DMA. Channels must be
1632                  * restarted between every packet for IN transactions, so there
1633                  * is no reason to do multiple packets in a row
1634                  */
1635                 packets_to_transfer = 1;
1636                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1637         } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1638                 /*
1639                  * Limit the number of packet and data transferred to what the
1640                  * hardware can handle
1641                  */
1642                 packets_to_transfer = MAX_TRANSFER_PACKETS;
1643                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1644         }
1645
1646         usbc_hctsiz.s.xfersize = bytes_to_transfer;
1647         usbc_hctsiz.s.pktcnt = packets_to_transfer;
1648
1649         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index),
1650                                usbc_hctsiz.u32);
1651 }
1652
1653
1654 /**
1655  * Start a channel to perform the pipe's head transaction
1656  *
1657  * @usb:          USB device state populated by cvmx_usb_initialize().
1658  * @channel:      Channel to setup
1659  * @pipe:         Pipe to start
1660  */
1661 static void __cvmx_usb_start_channel(struct cvmx_usb_state *usb,
1662                                      int channel,
1663                                      struct cvmx_usb_pipe *pipe)
1664 {
1665         struct cvmx_usb_transaction *transaction =
1666                 list_first_entry(&pipe->transactions, typeof(*transaction),
1667                                  node);
1668
1669         /* Make sure all writes to the DMA region get flushed */
1670         CVMX_SYNCW;
1671
1672         /* Attach the channel to the pipe */
1673         usb->pipe_for_channel[channel] = pipe;
1674         pipe->channel = channel;
1675         pipe->flags |= __CVMX_USB_PIPE_FLAGS_SCHEDULED;
1676
1677         /* Mark this channel as in use */
1678         usb->idle_hardware_channels &= ~(1<<channel);
1679
1680         /* Enable the channel interrupt bits */
1681         {
1682                 union cvmx_usbcx_hcintx usbc_hcint;
1683                 union cvmx_usbcx_hcintmskx usbc_hcintmsk;
1684                 union cvmx_usbcx_haintmsk usbc_haintmsk;
1685
1686                 /* Clear all channel status bits */
1687                 usbc_hcint.u32 = __cvmx_usb_read_csr32(usb,
1688                                 CVMX_USBCX_HCINTX(channel, usb->index));
1689
1690                 __cvmx_usb_write_csr32(usb,
1691                                        CVMX_USBCX_HCINTX(channel, usb->index),
1692                                        usbc_hcint.u32);
1693
1694                 usbc_hcintmsk.u32 = 0;
1695                 usbc_hcintmsk.s.chhltdmsk = 1;
1696                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1697                         /*
1698                          * Channels need these extra interrupts when we aren't
1699                          * in DMA mode.
1700                          */
1701                         usbc_hcintmsk.s.datatglerrmsk = 1;
1702                         usbc_hcintmsk.s.frmovrunmsk = 1;
1703                         usbc_hcintmsk.s.bblerrmsk = 1;
1704                         usbc_hcintmsk.s.xacterrmsk = 1;
1705                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1706                                 /*
1707                                  * Splits don't generate xfercompl, so we need
1708                                  * ACK and NYET.
1709                                  */
1710                                 usbc_hcintmsk.s.nyetmsk = 1;
1711                                 usbc_hcintmsk.s.ackmsk = 1;
1712                         }
1713                         usbc_hcintmsk.s.nakmsk = 1;
1714                         usbc_hcintmsk.s.stallmsk = 1;
1715                         usbc_hcintmsk.s.xfercomplmsk = 1;
1716                 }
1717                 __cvmx_usb_write_csr32(usb,
1718                                 CVMX_USBCX_HCINTMSKX(channel, usb->index),
1719                                 usbc_hcintmsk.u32);
1720
1721                 /* Enable the channel interrupt to propagate */
1722                 usbc_haintmsk.u32 = __cvmx_usb_read_csr32(usb,
1723                                         CVMX_USBCX_HAINTMSK(usb->index));
1724                 usbc_haintmsk.s.haintmsk |= 1<<channel;
1725                 __cvmx_usb_write_csr32(usb,
1726                                         CVMX_USBCX_HAINTMSK(usb->index),
1727                                         usbc_haintmsk.u32);
1728         }
1729
1730         /* Setup the locations the DMA engines use  */
1731         {
1732                 uint64_t dma_address = transaction->buffer +
1733                                         transaction->actual_bytes;
1734
1735                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1736                         dma_address = transaction->buffer +
1737                                         transaction->iso_packets[0].offset +
1738                                         transaction->actual_bytes;
1739
1740                 __cvmx_usb_write_csr64(usb,
1741                         CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8,
1742                         dma_address);
1743
1744                 __cvmx_usb_write_csr64(usb,
1745                         CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8,
1746                         dma_address);
1747         }
1748
1749         /* Setup both the size of the transfer and the SPLIT characteristics */
1750         {
1751                 union cvmx_usbcx_hcspltx usbc_hcsplt = {.u32 = 0};
1752                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 = 0};
1753                 int packets_to_transfer;
1754                 int bytes_to_transfer = transaction->buffer_length -
1755                         transaction->actual_bytes;
1756
1757                 /*
1758                  * ISOCHRONOUS transactions store each individual transfer size
1759                  * in the packet structure, not the global buffer_length
1760                  */
1761                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1762                         bytes_to_transfer =
1763                                 transaction->iso_packets[0].length -
1764                                 transaction->actual_bytes;
1765
1766                 /*
1767                  * We need to do split transactions when we are talking to non
1768                  * high speed devices that are behind a high speed hub
1769                  */
1770                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1771                         /*
1772                          * On the start split phase (stage is even) record the
1773                          * frame number we will need to send the split complete.
1774                          * We only store the lower two bits since the time ahead
1775                          * can only be two frames
1776                          */
1777                         if ((transaction->stage&1) == 0) {
1778                                 if (transaction->type == CVMX_USB_TRANSFER_BULK)
1779                                         pipe->split_sc_frame =
1780                                                 (usb->frame_number + 1) & 0x7f;
1781                                 else
1782                                         pipe->split_sc_frame =
1783                                                 (usb->frame_number + 2) & 0x7f;
1784                         } else
1785                                 pipe->split_sc_frame = -1;
1786
1787                         usbc_hcsplt.s.spltena = 1;
1788                         usbc_hcsplt.s.hubaddr = pipe->hub_device_addr;
1789                         usbc_hcsplt.s.prtaddr = pipe->hub_port;
1790                         usbc_hcsplt.s.compsplt = (transaction->stage ==
1791                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE);
1792
1793                         /*
1794                          * SPLIT transactions can only ever transmit one data
1795                          * packet so limit the transfer size to the max packet
1796                          * size
1797                          */
1798                         if (bytes_to_transfer > pipe->max_packet)
1799                                 bytes_to_transfer = pipe->max_packet;
1800
1801                         /*
1802                          * ISOCHRONOUS OUT splits are unique in that they limit
1803                          * data transfers to 188 byte chunks representing the
1804                          * begin/middle/end of the data or all
1805                          */
1806                         if (!usbc_hcsplt.s.compsplt &&
1807                                 (pipe->transfer_dir ==
1808                                  CVMX_USB_DIRECTION_OUT) &&
1809                                 (pipe->transfer_type ==
1810                                  CVMX_USB_TRANSFER_ISOCHRONOUS)) {
1811                                 /*
1812                                  * Clear the split complete frame number as
1813                                  * there isn't going to be a split complete
1814                                  */
1815                                 pipe->split_sc_frame = -1;
1816                                 /*
1817                                  * See if we've started this transfer and sent
1818                                  * data
1819                                  */
1820                                 if (transaction->actual_bytes == 0) {
1821                                         /*
1822                                          * Nothing sent yet, this is either a
1823                                          * begin or the entire payload
1824                                          */
1825                                         if (bytes_to_transfer <= 188)
1826                                                 /* Entire payload in one go */
1827                                                 usbc_hcsplt.s.xactpos = 3;
1828                                         else
1829                                                 /* First part of payload */
1830                                                 usbc_hcsplt.s.xactpos = 2;
1831                                 } else {
1832                                         /*
1833                                          * Continuing the previous data, we must
1834                                          * either be in the middle or at the end
1835                                          */
1836                                         if (bytes_to_transfer <= 188)
1837                                                 /* End of payload */
1838                                                 usbc_hcsplt.s.xactpos = 1;
1839                                         else
1840                                                 /* Middle of payload */
1841                                                 usbc_hcsplt.s.xactpos = 0;
1842                                 }
1843                                 /*
1844                                  * Again, the transfer size is limited to 188
1845                                  * bytes
1846                                  */
1847                                 if (bytes_to_transfer > 188)
1848                                         bytes_to_transfer = 188;
1849                         }
1850                 }
1851
1852                 /*
1853                  * Make sure the transfer never exceeds the byte limit of the
1854                  * hardware. Further bytes will be sent as continued
1855                  * transactions
1856                  */
1857                 if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1858                         /*
1859                          * Round MAX_TRANSFER_BYTES to a multiple of out packet
1860                          * size
1861                          */
1862                         bytes_to_transfer = MAX_TRANSFER_BYTES /
1863                                 pipe->max_packet;
1864                         bytes_to_transfer *= pipe->max_packet;
1865                 }
1866
1867                 /*
1868                  * Calculate the number of packets to transfer. If the length is
1869                  * zero we still need to transfer one packet
1870                  */
1871                 packets_to_transfer =
1872                         DIV_ROUND_UP(bytes_to_transfer, pipe->max_packet);
1873                 if (packets_to_transfer == 0)
1874                         packets_to_transfer = 1;
1875                 else if ((packets_to_transfer > 1) &&
1876                                 (usb->init_flags &
1877                                  CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1878                         /*
1879                          * Limit to one packet when not using DMA. Channels must
1880                          * be restarted between every packet for IN
1881                          * transactions, so there is no reason to do multiple
1882                          * packets in a row
1883                          */
1884                         packets_to_transfer = 1;
1885                         bytes_to_transfer = packets_to_transfer *
1886                                 pipe->max_packet;
1887                 } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1888                         /*
1889                          * Limit the number of packet and data transferred to
1890                          * what the hardware can handle
1891                          */
1892                         packets_to_transfer = MAX_TRANSFER_PACKETS;
1893                         bytes_to_transfer = packets_to_transfer *
1894                                 pipe->max_packet;
1895                 }
1896
1897                 usbc_hctsiz.s.xfersize = bytes_to_transfer;
1898                 usbc_hctsiz.s.pktcnt = packets_to_transfer;
1899
1900                 /* Update the DATA0/DATA1 toggle */
1901                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1902                 /*
1903                  * High speed pipes may need a hardware ping before they start
1904                  */
1905                 if (pipe->flags & __CVMX_USB_PIPE_FLAGS_NEED_PING)
1906                         usbc_hctsiz.s.dopng = 1;
1907
1908                 __cvmx_usb_write_csr32(usb,
1909                                        CVMX_USBCX_HCSPLTX(channel, usb->index),
1910                                        usbc_hcsplt.u32);
1911                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel,
1912                                         usb->index), usbc_hctsiz.u32);
1913         }
1914
1915         /* Setup the Host Channel Characteristics Register */
1916         {
1917                 union cvmx_usbcx_hccharx usbc_hcchar = {.u32 = 0};
1918
1919                 /*
1920                  * Set the startframe odd/even properly. This is only used for
1921                  * periodic
1922                  */
1923                 usbc_hcchar.s.oddfrm = usb->frame_number&1;
1924
1925                 /*
1926                  * Set the number of back to back packets allowed by this
1927                  * endpoint. Split transactions interpret "ec" as the number of
1928                  * immediate retries of failure. These retries happen too
1929                  * quickly, so we disable these entirely for splits
1930                  */
1931                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
1932                         usbc_hcchar.s.ec = 1;
1933                 else if (pipe->multi_count < 1)
1934                         usbc_hcchar.s.ec = 1;
1935                 else if (pipe->multi_count > 3)
1936                         usbc_hcchar.s.ec = 3;
1937                 else
1938                         usbc_hcchar.s.ec = pipe->multi_count;
1939
1940                 /* Set the rest of the endpoint specific settings */
1941                 usbc_hcchar.s.devaddr = pipe->device_addr;
1942                 usbc_hcchar.s.eptype = transaction->type;
1943                 usbc_hcchar.s.lspddev =
1944                         (pipe->device_speed == CVMX_USB_SPEED_LOW);
1945                 usbc_hcchar.s.epdir = pipe->transfer_dir;
1946                 usbc_hcchar.s.epnum = pipe->endpoint_num;
1947                 usbc_hcchar.s.mps = pipe->max_packet;
1948                 __cvmx_usb_write_csr32(usb,
1949                                        CVMX_USBCX_HCCHARX(channel, usb->index),
1950                                        usbc_hcchar.u32);
1951         }
1952
1953         /* Do transaction type specific fixups as needed */
1954         switch (transaction->type) {
1955         case CVMX_USB_TRANSFER_CONTROL:
1956                 __cvmx_usb_start_channel_control(usb, channel, pipe);
1957                 break;
1958         case CVMX_USB_TRANSFER_BULK:
1959         case CVMX_USB_TRANSFER_INTERRUPT:
1960                 break;
1961         case CVMX_USB_TRANSFER_ISOCHRONOUS:
1962                 if (!__cvmx_usb_pipe_needs_split(usb, pipe)) {
1963                         /*
1964                          * ISO transactions require different PIDs depending on
1965                          * direction and how many packets are needed
1966                          */
1967                         if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
1968                                 if (pipe->multi_count < 2) /* Need DATA0 */
1969                                         USB_SET_FIELD32(
1970                                                 CVMX_USBCX_HCTSIZX(channel,
1971                                                                    usb->index),
1972                                                 union cvmx_usbcx_hctsizx,
1973                                                 pid, 0);
1974                                 else /* Need MDATA */
1975                                         USB_SET_FIELD32(
1976                                                 CVMX_USBCX_HCTSIZX(channel,
1977                                                                    usb->index),
1978                                                 union cvmx_usbcx_hctsizx,
1979                                                 pid, 3);
1980                         }
1981                 }
1982                 break;
1983         }
1984         {
1985                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 =
1986                         __cvmx_usb_read_csr32(usb,
1987                                 CVMX_USBCX_HCTSIZX(channel, usb->index))};
1988                 transaction->xfersize = usbc_hctsiz.s.xfersize;
1989                 transaction->pktcnt = usbc_hctsiz.s.pktcnt;
1990         }
1991         /* Remeber when we start a split transaction */
1992         if (__cvmx_usb_pipe_needs_split(usb, pipe))
1993                 usb->active_split = transaction;
1994         USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1995                         union cvmx_usbcx_hccharx, chena, 1);
1996         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
1997                 __cvmx_usb_fill_tx_fifo(usb, channel);
1998 }
1999
2000
2001 /**
2002  * Find a pipe that is ready to be scheduled to hardware.
2003  * @usb:         USB device state populated by cvmx_usb_initialize().
2004  * @list:        Pipe list to search
2005  * @current_frame:
2006  *               Frame counter to use as a time reference.
2007  *
2008  * Returns: Pipe or NULL if none are ready
2009  */
2010 static struct cvmx_usb_pipe *__cvmx_usb_find_ready_pipe(
2011                 struct cvmx_usb_state *usb,
2012                 struct list_head *list,
2013                 uint64_t current_frame)
2014 {
2015         struct cvmx_usb_pipe *pipe;
2016
2017         list_for_each_entry(pipe, list, node) {
2018                 struct cvmx_usb_transaction *t =
2019                         list_first_entry(&pipe->transactions, typeof(*t),
2020                                          node);
2021                 if (!(pipe->flags & __CVMX_USB_PIPE_FLAGS_SCHEDULED) && t &&
2022                         (pipe->next_tx_frame <= current_frame) &&
2023                         ((pipe->split_sc_frame == -1) ||
2024                          ((((int)current_frame - (int)pipe->split_sc_frame)
2025                            & 0x7f) < 0x40)) &&
2026                         (!usb->active_split || (usb->active_split == t))) {
2027                         prefetch(t);
2028                         return pipe;
2029                 }
2030         }
2031         return NULL;
2032 }
2033
2034
2035 /**
2036  * Called whenever a pipe might need to be scheduled to the
2037  * hardware.
2038  *
2039  * @usb:         USB device state populated by cvmx_usb_initialize().
2040  * @is_sof:      True if this schedule was called on a SOF interrupt.
2041  */
2042 static void __cvmx_usb_schedule(struct cvmx_usb_state *usb, int is_sof)
2043 {
2044         int channel;
2045         struct cvmx_usb_pipe *pipe;
2046         int need_sof;
2047         enum cvmx_usb_transfer ttype;
2048
2049         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
2050                 /*
2051                  * Without DMA we need to be careful to not schedule something
2052                  * at the end of a frame and cause an overrun.
2053                  */
2054                 union cvmx_usbcx_hfnum hfnum = {
2055                         .u32 = __cvmx_usb_read_csr32(usb,
2056                                                 CVMX_USBCX_HFNUM(usb->index))
2057                 };
2058
2059                 union cvmx_usbcx_hfir hfir = {
2060                         .u32 = __cvmx_usb_read_csr32(usb,
2061                                                 CVMX_USBCX_HFIR(usb->index))
2062                 };
2063
2064                 if (hfnum.s.frrem < hfir.s.frint/4)
2065                         goto done;
2066         }
2067
2068         while (usb->idle_hardware_channels) {
2069                 /* Find an idle channel */
2070                 channel = __fls(usb->idle_hardware_channels);
2071                 if (unlikely(channel > 7))
2072                         break;
2073
2074                 /* Find a pipe needing service */
2075                 pipe = NULL;
2076                 if (is_sof) {
2077                         /*
2078                          * Only process periodic pipes on SOF interrupts. This
2079                          * way we are sure that the periodic data is sent in the
2080                          * beginning of the frame
2081                          */
2082                         pipe = __cvmx_usb_find_ready_pipe(usb,
2083                                         usb->active_pipes +
2084                                         CVMX_USB_TRANSFER_ISOCHRONOUS,
2085                                         usb->frame_number);
2086                         if (likely(!pipe))
2087                                 pipe = __cvmx_usb_find_ready_pipe(usb,
2088                                                 usb->active_pipes +
2089                                                 CVMX_USB_TRANSFER_INTERRUPT,
2090                                                 usb->frame_number);
2091                 }
2092                 if (likely(!pipe)) {
2093                         pipe = __cvmx_usb_find_ready_pipe(usb,
2094                                         usb->active_pipes +
2095                                         CVMX_USB_TRANSFER_CONTROL,
2096                                         usb->frame_number);
2097                         if (likely(!pipe))
2098                                 pipe = __cvmx_usb_find_ready_pipe(usb,
2099                                                 usb->active_pipes +
2100                                                 CVMX_USB_TRANSFER_BULK,
2101                                                 usb->frame_number);
2102                 }
2103                 if (!pipe)
2104                         break;
2105
2106                 __cvmx_usb_start_channel(usb, channel, pipe);
2107         }
2108
2109 done:
2110         /*
2111          * Only enable SOF interrupts when we have transactions pending in the
2112          * future that might need to be scheduled
2113          */
2114         need_sof = 0;
2115         for (ttype = CVMX_USB_TRANSFER_CONTROL;
2116                         ttype <= CVMX_USB_TRANSFER_INTERRUPT; ttype++) {
2117                 list_for_each_entry(pipe, &usb->active_pipes[ttype], node) {
2118                         if (pipe->next_tx_frame > usb->frame_number) {
2119                                 need_sof = 1;
2120                                 break;
2121                         }
2122                 }
2123         }
2124         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
2125                         union cvmx_usbcx_gintmsk, sofmsk, need_sof);
2126 }
2127
2128 static void octeon_usb_urb_complete_callback(struct cvmx_usb_state *usb,
2129                                              enum cvmx_usb_complete status,
2130                                              struct cvmx_usb_pipe *pipe,
2131                                              struct cvmx_usb_transaction
2132                                                 *transaction,
2133                                              int bytes_transferred,
2134                                              struct urb *urb)
2135 {
2136         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
2137         struct usb_hcd *hcd = octeon_to_hcd(priv);
2138         struct device *dev = hcd->self.controller;
2139
2140         if (likely(status == CVMX_USB_COMPLETE_SUCCESS))
2141                 urb->actual_length = bytes_transferred;
2142         else
2143                 urb->actual_length = 0;
2144
2145         urb->hcpriv = NULL;
2146
2147         /* For Isochronous transactions we need to update the URB packet status
2148            list from data in our private copy */
2149         if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
2150                 int i;
2151                 /*
2152                  * The pointer to the private list is stored in the setup_packet
2153                  * field.
2154                  */
2155                 struct cvmx_usb_iso_packet *iso_packet =
2156                         (struct cvmx_usb_iso_packet *) urb->setup_packet;
2157                 /* Recalculate the transfer size by adding up each packet */
2158                 urb->actual_length = 0;
2159                 for (i = 0; i < urb->number_of_packets; i++) {
2160                         if (iso_packet[i].status ==
2161                                         CVMX_USB_COMPLETE_SUCCESS) {
2162                                 urb->iso_frame_desc[i].status = 0;
2163                                 urb->iso_frame_desc[i].actual_length =
2164                                         iso_packet[i].length;
2165                                 urb->actual_length +=
2166                                         urb->iso_frame_desc[i].actual_length;
2167                         } else {
2168                                 dev_dbg(dev, "ISOCHRONOUS packet=%d of %d status=%d pipe=%p transaction=%p size=%d\n",
2169                                         i, urb->number_of_packets,
2170                                         iso_packet[i].status, pipe,
2171                                         transaction, iso_packet[i].length);
2172                                 urb->iso_frame_desc[i].status = -EREMOTEIO;
2173                         }
2174                 }
2175                 /* Free the private list now that we don't need it anymore */
2176                 kfree(iso_packet);
2177                 urb->setup_packet = NULL;
2178         }
2179
2180         switch (status) {
2181         case CVMX_USB_COMPLETE_SUCCESS:
2182                 urb->status = 0;
2183                 break;
2184         case CVMX_USB_COMPLETE_CANCEL:
2185                 if (urb->status == 0)
2186                         urb->status = -ENOENT;
2187                 break;
2188         case CVMX_USB_COMPLETE_STALL:
2189                 dev_dbg(dev, "status=stall pipe=%p transaction=%p size=%d\n",
2190                         pipe, transaction, bytes_transferred);
2191                 urb->status = -EPIPE;
2192                 break;
2193         case CVMX_USB_COMPLETE_BABBLEERR:
2194                 dev_dbg(dev, "status=babble pipe=%p transaction=%p size=%d\n",
2195                         pipe, transaction, bytes_transferred);
2196                 urb->status = -EPIPE;
2197                 break;
2198         case CVMX_USB_COMPLETE_SHORT:
2199                 dev_dbg(dev, "status=short pipe=%p transaction=%p size=%d\n",
2200                         pipe, transaction, bytes_transferred);
2201                 urb->status = -EREMOTEIO;
2202                 break;
2203         case CVMX_USB_COMPLETE_ERROR:
2204         case CVMX_USB_COMPLETE_XACTERR:
2205         case CVMX_USB_COMPLETE_DATATGLERR:
2206         case CVMX_USB_COMPLETE_FRAMEERR:
2207                 dev_dbg(dev, "status=%d pipe=%p transaction=%p size=%d\n",
2208                         status, pipe, transaction, bytes_transferred);
2209                 urb->status = -EPROTO;
2210                 break;
2211         }
2212         usb_hcd_unlink_urb_from_ep(octeon_to_hcd(priv), urb);
2213         spin_unlock(&priv->lock);
2214         usb_hcd_giveback_urb(octeon_to_hcd(priv), urb, urb->status);
2215         spin_lock(&priv->lock);
2216 }
2217
2218 /**
2219  * Signal the completion of a transaction and free it. The
2220  * transaction will be removed from the pipe transaction list.
2221  *
2222  * @usb:         USB device state populated by cvmx_usb_initialize().
2223  * @pipe:        Pipe the transaction is on
2224  * @transaction:
2225  *               Transaction that completed
2226  * @complete_code:
2227  *               Completion code
2228  */
2229 static void __cvmx_usb_perform_complete(
2230                                 struct cvmx_usb_state *usb,
2231                                 struct cvmx_usb_pipe *pipe,
2232                                 struct cvmx_usb_transaction *transaction,
2233                                 enum cvmx_usb_complete complete_code)
2234 {
2235         /* If this was a split then clear our split in progress marker */
2236         if (usb->active_split == transaction)
2237                 usb->active_split = NULL;
2238
2239         /*
2240          * Isochronous transactions need extra processing as they might not be
2241          * done after a single data transfer
2242          */
2243         if (unlikely(transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
2244                 /* Update the number of bytes transferred in this ISO packet */
2245                 transaction->iso_packets[0].length = transaction->actual_bytes;
2246                 transaction->iso_packets[0].status = complete_code;
2247
2248                 /*
2249                  * If there are more ISOs pending and we succeeded, schedule the
2250                  * next one
2251                  */
2252                 if ((transaction->iso_number_packets > 1) &&
2253                         (complete_code == CVMX_USB_COMPLETE_SUCCESS)) {
2254                         /* No bytes transferred for this packet as of yet */
2255                         transaction->actual_bytes = 0;
2256                         /* One less ISO waiting to transfer */
2257                         transaction->iso_number_packets--;
2258                         /* Increment to the next location in our packet array */
2259                         transaction->iso_packets++;
2260                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2261                         return;
2262                 }
2263         }
2264
2265         /* Remove the transaction from the pipe list */
2266         list_del(&transaction->node);
2267         if (list_empty(&pipe->transactions))
2268                 list_move_tail(&pipe->node, &usb->idle_pipes);
2269         octeon_usb_urb_complete_callback(usb, complete_code, pipe,
2270                                          transaction,
2271                                          transaction->actual_bytes,
2272                                          transaction->urb);
2273         kfree(transaction);
2274 }
2275
2276
2277 /**
2278  * Submit a usb transaction to a pipe. Called for all types
2279  * of transactions.
2280  *
2281  * @usb:
2282  * @pipe:           Which pipe to submit to.
2283  * @type:           Transaction type
2284  * @buffer:         User buffer for the transaction
2285  * @buffer_length:
2286  *                  User buffer's length in bytes
2287  * @control_header:
2288  *                  For control transactions, the 8 byte standard header
2289  * @iso_start_frame:
2290  *                  For ISO transactions, the start frame
2291  * @iso_number_packets:
2292  *                  For ISO, the number of packet in the transaction.
2293  * @iso_packets:
2294  *                  A description of each ISO packet
2295  * @urb:            URB for the callback
2296  *
2297  * Returns: Transaction or NULL on failure.
2298  */
2299 static struct cvmx_usb_transaction *__cvmx_usb_submit_transaction(
2300                                 struct cvmx_usb_state *usb,
2301                                 struct cvmx_usb_pipe *pipe,
2302                                 enum cvmx_usb_transfer type,
2303                                 uint64_t buffer,
2304                                 int buffer_length,
2305                                 uint64_t control_header,
2306                                 int iso_start_frame,
2307                                 int iso_number_packets,
2308                                 struct cvmx_usb_iso_packet *iso_packets,
2309                                 struct urb *urb)
2310 {
2311         struct cvmx_usb_transaction *transaction;
2312
2313         if (unlikely(pipe->transfer_type != type))
2314                 return NULL;
2315
2316         transaction = kzalloc(sizeof(*transaction), GFP_ATOMIC);
2317         if (unlikely(!transaction))
2318                 return NULL;
2319
2320         transaction->type = type;
2321         transaction->buffer = buffer;
2322         transaction->buffer_length = buffer_length;
2323         transaction->control_header = control_header;
2324         /* FIXME: This is not used, implement it. */
2325         transaction->iso_start_frame = iso_start_frame;
2326         transaction->iso_number_packets = iso_number_packets;
2327         transaction->iso_packets = iso_packets;
2328         transaction->urb = urb;
2329         if (transaction->type == CVMX_USB_TRANSFER_CONTROL)
2330                 transaction->stage = CVMX_USB_STAGE_SETUP;
2331         else
2332                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2333
2334         if (!list_empty(&pipe->transactions)) {
2335                 list_add_tail(&transaction->node, &pipe->transactions);
2336         } else {
2337                 list_add_tail(&transaction->node, &pipe->transactions);
2338                 list_move_tail(&pipe->node,
2339                                &usb->active_pipes[pipe->transfer_type]);
2340
2341                 /*
2342                  * We may need to schedule the pipe if this was the head of the
2343                  * pipe.
2344                  */
2345                 __cvmx_usb_schedule(usb, 0);
2346         }
2347
2348         return transaction;
2349 }
2350
2351
2352 /**
2353  * Call to submit a USB Bulk transfer to a pipe.
2354  *
2355  * @usb:            USB device state populated by cvmx_usb_initialize().
2356  * @pipe:           Handle to the pipe for the transfer.
2357  * @urb:            URB.
2358  *
2359  * Returns: A submitted transaction or NULL on failure.
2360  */
2361 static struct cvmx_usb_transaction *cvmx_usb_submit_bulk(
2362                                                 struct cvmx_usb_state *usb,
2363                                                 struct cvmx_usb_pipe *pipe,
2364                                                 struct urb *urb)
2365 {
2366         return __cvmx_usb_submit_transaction(usb, pipe, CVMX_USB_TRANSFER_BULK,
2367                                              urb->transfer_dma,
2368                                              urb->transfer_buffer_length,
2369                                              0, /* control_header */
2370                                              0, /* iso_start_frame */
2371                                              0, /* iso_number_packets */
2372                                              NULL, /* iso_packets */
2373                                              urb);
2374 }
2375
2376
2377 /**
2378  * Call to submit a USB Interrupt transfer to a pipe.
2379  *
2380  * @usb:            USB device state populated by cvmx_usb_initialize().
2381  * @pipe:           Handle to the pipe for the transfer.
2382  * @urb:            URB returned when the callback is called.
2383  *
2384  * Returns: A submitted transaction or NULL on failure.
2385  */
2386 static struct cvmx_usb_transaction *cvmx_usb_submit_interrupt(
2387                                                 struct cvmx_usb_state *usb,
2388                                                 struct cvmx_usb_pipe *pipe,
2389                                                 struct urb *urb)
2390 {
2391         return __cvmx_usb_submit_transaction(usb, pipe,
2392                                              CVMX_USB_TRANSFER_INTERRUPT,
2393                                              urb->transfer_dma,
2394                                              urb->transfer_buffer_length,
2395                                              0, /* control_header */
2396                                              0, /* iso_start_frame */
2397                                              0, /* iso_number_packets */
2398                                              NULL, /* iso_packets */
2399                                              urb);
2400 }
2401
2402
2403 /**
2404  * Call to submit a USB Control transfer to a pipe.
2405  *
2406  * @usb:            USB device state populated by cvmx_usb_initialize().
2407  * @pipe:           Handle to the pipe for the transfer.
2408  * @urb:            URB.
2409  *
2410  * Returns: A submitted transaction or NULL on failure.
2411  */
2412 static struct cvmx_usb_transaction *cvmx_usb_submit_control(
2413                                                 struct cvmx_usb_state *usb,
2414                                                 struct cvmx_usb_pipe *pipe,
2415                                                 struct urb *urb)
2416 {
2417         int buffer_length = urb->transfer_buffer_length;
2418         uint64_t control_header = urb->setup_dma;
2419         struct usb_ctrlrequest *header = cvmx_phys_to_ptr(control_header);
2420
2421         if ((header->bRequestType & USB_DIR_IN) == 0)
2422                 buffer_length = le16_to_cpu(header->wLength);
2423
2424         return __cvmx_usb_submit_transaction(usb, pipe,
2425                                              CVMX_USB_TRANSFER_CONTROL,
2426                                              urb->transfer_dma, buffer_length,
2427                                              control_header,
2428                                              0, /* iso_start_frame */
2429                                              0, /* iso_number_packets */
2430                                              NULL, /* iso_packets */
2431                                              urb);
2432 }
2433
2434
2435 /**
2436  * Call to submit a USB Isochronous transfer to a pipe.
2437  *
2438  * @usb:            USB device state populated by cvmx_usb_initialize().
2439  * @pipe:           Handle to the pipe for the transfer.
2440  * @urb:            URB returned when the callback is called.
2441  *
2442  * Returns: A submitted transaction or NULL on failure.
2443  */
2444 static struct cvmx_usb_transaction *cvmx_usb_submit_isochronous(
2445                                                 struct cvmx_usb_state *usb,
2446                                                 struct cvmx_usb_pipe *pipe,
2447                                                 struct urb *urb)
2448 {
2449         struct cvmx_usb_iso_packet *packets;
2450
2451         packets = (struct cvmx_usb_iso_packet *) urb->setup_packet;
2452         return __cvmx_usb_submit_transaction(usb, pipe,
2453                                              CVMX_USB_TRANSFER_ISOCHRONOUS,
2454                                              urb->transfer_dma,
2455                                              urb->transfer_buffer_length,
2456                                              0, /* control_header */
2457                                              urb->start_frame,
2458                                              urb->number_of_packets,
2459                                              packets, urb);
2460 }
2461
2462
2463 /**
2464  * Cancel one outstanding request in a pipe. Canceling a request
2465  * can fail if the transaction has already completed before cancel
2466  * is called. Even after a successful cancel call, it may take
2467  * a frame or two for the cvmx_usb_poll() function to call the
2468  * associated callback.
2469  *
2470  * @usb:         USB device state populated by cvmx_usb_initialize().
2471  * @pipe:        Pipe to cancel requests in.
2472  * @transaction: Transaction to cancel, returned by the submit function.
2473  *
2474  * Returns: 0 or a negative error code.
2475  */
2476 static int cvmx_usb_cancel(struct cvmx_usb_state *usb,
2477                            struct cvmx_usb_pipe *pipe,
2478                            struct cvmx_usb_transaction *transaction)
2479 {
2480         /*
2481          * If the transaction is the HEAD of the queue and scheduled. We need to
2482          * treat it special
2483          */
2484         if (list_first_entry(&pipe->transactions, typeof(*transaction), node) ==
2485             transaction && (pipe->flags & __CVMX_USB_PIPE_FLAGS_SCHEDULED)) {
2486                 union cvmx_usbcx_hccharx usbc_hcchar;
2487
2488                 usb->pipe_for_channel[pipe->channel] = NULL;
2489                 pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_SCHEDULED;
2490
2491                 CVMX_SYNCW;
2492
2493                 usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb,
2494                                 CVMX_USBCX_HCCHARX(pipe->channel, usb->index));
2495                 /*
2496                  * If the channel isn't enabled then the transaction already
2497                  * completed.
2498                  */
2499                 if (usbc_hcchar.s.chena) {
2500                         usbc_hcchar.s.chdis = 1;
2501                         __cvmx_usb_write_csr32(usb,
2502                                         CVMX_USBCX_HCCHARX(pipe->channel,
2503                                                 usb->index),
2504                                         usbc_hcchar.u32);
2505                 }
2506         }
2507         __cvmx_usb_perform_complete(usb, pipe, transaction,
2508                                     CVMX_USB_COMPLETE_CANCEL);
2509         return 0;
2510 }
2511
2512
2513 /**
2514  * Cancel all outstanding requests in a pipe. Logically all this
2515  * does is call cvmx_usb_cancel() in a loop.
2516  *
2517  * @usb:         USB device state populated by cvmx_usb_initialize().
2518  * @pipe:        Pipe to cancel requests in.
2519  *
2520  * Returns: 0 or a negative error code.
2521  */
2522 static int cvmx_usb_cancel_all(struct cvmx_usb_state *usb,
2523                                struct cvmx_usb_pipe *pipe)
2524 {
2525         struct cvmx_usb_transaction *transaction, *next;
2526
2527         /* Simply loop through and attempt to cancel each transaction */
2528         list_for_each_entry_safe(transaction, next, &pipe->transactions, node) {
2529                 int result = cvmx_usb_cancel(usb, pipe, transaction);
2530
2531                 if (unlikely(result != 0))
2532                         return result;
2533         }
2534         return 0;
2535 }
2536
2537
2538 /**
2539  * Close a pipe created with cvmx_usb_open_pipe().
2540  *
2541  * @usb:         USB device state populated by cvmx_usb_initialize().
2542  * @pipe:        Pipe to close.
2543  *
2544  * Returns: 0 or a negative error code. EBUSY is returned if the pipe has
2545  *          outstanding transfers.
2546  */
2547 static int cvmx_usb_close_pipe(struct cvmx_usb_state *usb,
2548                                struct cvmx_usb_pipe *pipe)
2549 {
2550         /* Fail if the pipe has pending transactions */
2551         if (!list_empty(&pipe->transactions))
2552                 return -EBUSY;
2553
2554         list_del(&pipe->node);
2555         kfree(pipe);
2556
2557         return 0;
2558 }
2559
2560 /**
2561  * Get the current USB protocol level frame number. The frame
2562  * number is always in the range of 0-0x7ff.
2563  *
2564  * @usb: USB device state populated by cvmx_usb_initialize().
2565  *
2566  * Returns: USB frame number
2567  */
2568 static int cvmx_usb_get_frame_number(struct cvmx_usb_state *usb)
2569 {
2570         int frame_number;
2571         union cvmx_usbcx_hfnum usbc_hfnum;
2572
2573         usbc_hfnum.u32 = __cvmx_usb_read_csr32(usb,
2574                         CVMX_USBCX_HFNUM(usb->index));
2575         frame_number = usbc_hfnum.s.frnum;
2576
2577         return frame_number;
2578 }
2579
2580
2581 /**
2582  * Poll a channel for status
2583  *
2584  * @usb:     USB device
2585  * @channel: Channel to poll
2586  *
2587  * Returns: Zero on success
2588  */
2589 static int __cvmx_usb_poll_channel(struct cvmx_usb_state *usb, int channel)
2590 {
2591         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
2592         struct usb_hcd *hcd = octeon_to_hcd(priv);
2593         struct device *dev = hcd->self.controller;
2594         union cvmx_usbcx_hcintx usbc_hcint;
2595         union cvmx_usbcx_hctsizx usbc_hctsiz;
2596         union cvmx_usbcx_hccharx usbc_hcchar;
2597         struct cvmx_usb_pipe *pipe;
2598         struct cvmx_usb_transaction *transaction;
2599         int bytes_this_transfer;
2600         int bytes_in_last_packet;
2601         int packets_processed;
2602         int buffer_space_left;
2603
2604         /* Read the interrupt status bits for the channel */
2605         usbc_hcint.u32 = __cvmx_usb_read_csr32(usb,
2606                         CVMX_USBCX_HCINTX(channel, usb->index));
2607
2608         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
2609                 usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb,
2610                                 CVMX_USBCX_HCCHARX(channel, usb->index));
2611
2612                 if (usbc_hcchar.s.chena && usbc_hcchar.s.chdis) {
2613                         /*
2614                          * There seems to be a bug in CN31XX which can cause
2615                          * interrupt IN transfers to get stuck until we do a
2616                          * write of HCCHARX without changing things
2617                          */
2618                         __cvmx_usb_write_csr32(usb,
2619                                         CVMX_USBCX_HCCHARX(channel,
2620                                                            usb->index),
2621                                         usbc_hcchar.u32);
2622                         return 0;
2623                 }
2624
2625                 /*
2626                  * In non DMA mode the channels don't halt themselves. We need
2627                  * to manually disable channels that are left running
2628                  */
2629                 if (!usbc_hcint.s.chhltd) {
2630                         if (usbc_hcchar.s.chena) {
2631                                 union cvmx_usbcx_hcintmskx hcintmsk;
2632                                 /* Disable all interrupts except CHHLTD */
2633                                 hcintmsk.u32 = 0;
2634                                 hcintmsk.s.chhltdmsk = 1;
2635                                 __cvmx_usb_write_csr32(usb,
2636                                                 CVMX_USBCX_HCINTMSKX(channel,
2637                                                         usb->index),
2638                                                 hcintmsk.u32);
2639                                 usbc_hcchar.s.chdis = 1;
2640                                 __cvmx_usb_write_csr32(usb,
2641                                                 CVMX_USBCX_HCCHARX(channel,
2642                                                         usb->index),
2643                                                 usbc_hcchar.u32);
2644                                 return 0;
2645                         } else if (usbc_hcint.s.xfercompl) {
2646                                 /*
2647                                  * Successful IN/OUT with transfer complete.
2648                                  * Channel halt isn't needed.
2649                                  */
2650                         } else {
2651                                 dev_err(dev, "USB%d: Channel %d interrupt without halt\n",
2652                                         usb->index, channel);
2653                                 return 0;
2654                         }
2655                 }
2656         } else {
2657                 /*
2658                  * There is are no interrupts that we need to process when the
2659                  * channel is still running
2660                  */
2661                 if (!usbc_hcint.s.chhltd)
2662                         return 0;
2663         }
2664
2665         /* Disable the channel interrupts now that it is done */
2666         __cvmx_usb_write_csr32(usb,
2667                                 CVMX_USBCX_HCINTMSKX(channel, usb->index),
2668                                 0);
2669         usb->idle_hardware_channels |= (1<<channel);
2670
2671         /* Make sure this channel is tied to a valid pipe */
2672         pipe = usb->pipe_for_channel[channel];
2673         prefetch(pipe);
2674         if (!pipe)
2675                 return 0;
2676         transaction = list_first_entry(&pipe->transactions,
2677                                        typeof(*transaction),
2678                                        node);
2679         prefetch(transaction);
2680
2681         /*
2682          * Disconnect this pipe from the HW channel. Later the schedule
2683          * function will figure out which pipe needs to go
2684          */
2685         usb->pipe_for_channel[channel] = NULL;
2686         pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_SCHEDULED;
2687
2688         /*
2689          * Read the channel config info so we can figure out how much data
2690          * transferred
2691          */
2692         usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb,
2693                         CVMX_USBCX_HCCHARX(channel, usb->index));
2694         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb,
2695                         CVMX_USBCX_HCTSIZX(channel, usb->index));
2696
2697         /*
2698          * Calculating the number of bytes successfully transferred is dependent
2699          * on the transfer direction
2700          */
2701         packets_processed = transaction->pktcnt - usbc_hctsiz.s.pktcnt;
2702         if (usbc_hcchar.s.epdir) {
2703                 /*
2704                  * IN transactions are easy. For every byte received the
2705                  * hardware decrements xfersize. All we need to do is subtract
2706                  * the current value of xfersize from its starting value and we
2707                  * know how many bytes were written to the buffer
2708                  */
2709                 bytes_this_transfer = transaction->xfersize -
2710                         usbc_hctsiz.s.xfersize;
2711         } else {
2712                 /*
2713                  * OUT transaction don't decrement xfersize. Instead pktcnt is
2714                  * decremented on every successful packet send. The hardware
2715                  * does this when it receives an ACK, or NYET. If it doesn't
2716                  * receive one of these responses pktcnt doesn't change
2717                  */
2718                 bytes_this_transfer = packets_processed * usbc_hcchar.s.mps;
2719                 /*
2720                  * The last packet may not be a full transfer if we didn't have
2721                  * enough data
2722                  */
2723                 if (bytes_this_transfer > transaction->xfersize)
2724                         bytes_this_transfer = transaction->xfersize;
2725         }
2726         /* Figure out how many bytes were in the last packet of the transfer */
2727         if (packets_processed)
2728                 bytes_in_last_packet = bytes_this_transfer -
2729                         (packets_processed - 1) * usbc_hcchar.s.mps;
2730         else
2731                 bytes_in_last_packet = bytes_this_transfer;
2732
2733         /*
2734          * As a special case, setup transactions output the setup header, not
2735          * the user's data. For this reason we don't count setup data as bytes
2736          * transferred
2737          */
2738         if ((transaction->stage == CVMX_USB_STAGE_SETUP) ||
2739                 (transaction->stage == CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE))
2740                 bytes_this_transfer = 0;
2741
2742         /*
2743          * Add the bytes transferred to the running total. It is important that
2744          * bytes_this_transfer doesn't count any data that needs to be
2745          * retransmitted
2746          */
2747         transaction->actual_bytes += bytes_this_transfer;
2748         if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
2749                 buffer_space_left = transaction->iso_packets[0].length -
2750                         transaction->actual_bytes;
2751         else
2752                 buffer_space_left = transaction->buffer_length -
2753                         transaction->actual_bytes;
2754
2755         /*
2756          * We need to remember the PID toggle state for the next transaction.
2757          * The hardware already updated it for the next transaction
2758          */
2759         pipe->pid_toggle = !(usbc_hctsiz.s.pid == 0);
2760
2761         /*
2762          * For high speed bulk out, assume the next transaction will need to do
2763          * a ping before proceeding. If this isn't true the ACK processing below
2764          * will clear this flag
2765          */
2766         if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2767                 (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
2768                 (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT))
2769                 pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
2770
2771         if (usbc_hcint.s.stall) {
2772                 /*
2773                  * STALL as a response means this transaction cannot be
2774                  * completed because the device can't process transactions. Tell
2775                  * the user. Any data that was transferred will be counted on
2776                  * the actual bytes transferred
2777                  */
2778                 pipe->pid_toggle = 0;
2779                 __cvmx_usb_perform_complete(usb, pipe, transaction,
2780                                             CVMX_USB_COMPLETE_STALL);
2781         } else if (usbc_hcint.s.xacterr) {
2782                 /*
2783                  * We know at least one packet worked if we get a ACK or NAK.
2784                  * Reset the retry counter
2785                  */
2786                 if (usbc_hcint.s.nak || usbc_hcint.s.ack)
2787                         transaction->retries = 0;
2788                 transaction->retries++;
2789                 if (transaction->retries > MAX_RETRIES) {
2790                         /*
2791                          * XactErr as a response means the device signaled
2792                          * something wrong with the transfer. For example, PID
2793                          * toggle errors cause these
2794                          */
2795                         __cvmx_usb_perform_complete(usb, pipe, transaction,
2796                                                     CVMX_USB_COMPLETE_XACTERR);
2797                 } else {
2798                         /*
2799                          * If this was a split then clear our split in progress
2800                          * marker
2801                          */
2802                         if (usb->active_split == transaction)
2803                                 usb->active_split = NULL;
2804                         /*
2805                          * Rewind to the beginning of the transaction by anding
2806                          * off the split complete bit
2807                          */
2808                         transaction->stage &= ~1;
2809                         pipe->split_sc_frame = -1;
2810                         pipe->next_tx_frame += pipe->interval;
2811                         if (pipe->next_tx_frame < usb->frame_number)
2812                                 pipe->next_tx_frame =
2813                                         usb->frame_number + pipe->interval -
2814                                         (usb->frame_number -
2815                                          pipe->next_tx_frame) % pipe->interval;
2816                 }
2817         } else if (usbc_hcint.s.bblerr) {
2818                 /* Babble Error (BblErr) */
2819                 __cvmx_usb_perform_complete(usb, pipe, transaction,
2820                                             CVMX_USB_COMPLETE_BABBLEERR);
2821         } else if (usbc_hcint.s.datatglerr) {
2822                 /* Data toggle error */
2823                 __cvmx_usb_perform_complete(usb, pipe, transaction,
2824                                             CVMX_USB_COMPLETE_DATATGLERR);
2825         } else if (usbc_hcint.s.nyet) {
2826                 /*
2827                  * NYET as a response is only allowed in three cases: as a
2828                  * response to a ping, as a response to a split transaction, and
2829                  * as a response to a bulk out. The ping case is handled by
2830                  * hardware, so we only have splits and bulk out
2831                  */
2832                 if (!__cvmx_usb_pipe_needs_split(usb, pipe)) {
2833                         transaction->retries = 0;
2834                         /*
2835                          * If there is more data to go then we need to try
2836                          * again. Otherwise this transaction is complete
2837                          */
2838                         if ((buffer_space_left == 0) ||
2839                                 (bytes_in_last_packet < pipe->max_packet))
2840                                 __cvmx_usb_perform_complete(usb, pipe,
2841                                                 transaction,
2842                                                 CVMX_USB_COMPLETE_SUCCESS);
2843                 } else {
2844                         /*
2845                          * Split transactions retry the split complete 4 times
2846                          * then rewind to the start split and do the entire
2847                          * transactions again
2848                          */
2849                         transaction->retries++;
2850                         if ((transaction->retries & 0x3) == 0) {
2851                                 /*
2852                                  * Rewind to the beginning of the transaction by
2853                                  * anding off the split complete bit
2854                                  */
2855                                 transaction->stage &= ~1;
2856                                 pipe->split_sc_frame = -1;
2857                         }
2858                 }
2859         } else if (usbc_hcint.s.ack) {
2860                 transaction->retries = 0;
2861                 /*
2862                  * The ACK bit can only be checked after the other error bits.
2863                  * This is because a multi packet transfer may succeed in a
2864                  * number of packets and then get a different response on the
2865                  * last packet. In this case both ACK and the last response bit
2866                  * will be set. If none of the other response bits is set, then
2867                  * the last packet must have been an ACK
2868                  *
2869                  * Since we got an ACK, we know we don't need to do a ping on
2870                  * this pipe
2871                  */
2872                 pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_NEED_PING;
2873
2874                 switch (transaction->type) {
2875                 case CVMX_USB_TRANSFER_CONTROL:
2876                         switch (transaction->stage) {
2877                         case CVMX_USB_STAGE_NON_CONTROL:
2878                         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
2879                                 /* This should be impossible */
2880                                 __cvmx_usb_perform_complete(usb, pipe,
2881                                         transaction, CVMX_USB_COMPLETE_ERROR);
2882                                 break;
2883                         case CVMX_USB_STAGE_SETUP:
2884                                 pipe->pid_toggle = 1;
2885                                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
2886                                         transaction->stage =
2887                                                 CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE;
2888                                 else {
2889                                         struct usb_ctrlrequest *header =
2890                                                 cvmx_phys_to_ptr(transaction->control_header);
2891                                         if (header->wLength)
2892                                                 transaction->stage =
2893                                                         CVMX_USB_STAGE_DATA;
2894                                         else
2895                                                 transaction->stage =
2896                                                         CVMX_USB_STAGE_STATUS;
2897                                 }
2898                                 break;
2899                         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
2900                                 {
2901                                         struct usb_ctrlrequest *header =
2902                                                 cvmx_phys_to_ptr(transaction->control_header);
2903                                         if (header->wLength)
2904                                                 transaction->stage =
2905                                                         CVMX_USB_STAGE_DATA;
2906                                         else
2907                                                 transaction->stage =
2908                                                         CVMX_USB_STAGE_STATUS;
2909                                 }
2910                                 break;
2911                         case CVMX_USB_STAGE_DATA:
2912                                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2913                                         transaction->stage =
2914                                                 CVMX_USB_STAGE_DATA_SPLIT_COMPLETE;
2915                                         /*
2916                                          * For setup OUT data that are splits,
2917                                          * the hardware doesn't appear to count
2918                                          * transferred data. Here we manually
2919                                          * update the data transferred
2920                                          */
2921                                         if (!usbc_hcchar.s.epdir) {
2922                                                 if (buffer_space_left < pipe->max_packet)
2923                                                         transaction->actual_bytes +=
2924                                                                 buffer_space_left;
2925                                                 else
2926                                                         transaction->actual_bytes +=
2927                                                                 pipe->max_packet;
2928                                         }
2929                                 } else if ((buffer_space_left == 0) ||
2930                                                 (bytes_in_last_packet <
2931                                                  pipe->max_packet)) {
2932                                         pipe->pid_toggle = 1;
2933                                         transaction->stage =
2934                                                 CVMX_USB_STAGE_STATUS;
2935                                 }
2936                                 break;
2937                         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
2938                                 if ((buffer_space_left == 0) ||
2939                                                 (bytes_in_last_packet <
2940                                                  pipe->max_packet)) {
2941                                         pipe->pid_toggle = 1;
2942                                         transaction->stage =
2943                                                 CVMX_USB_STAGE_STATUS;
2944                                 } else {
2945                                         transaction->stage =
2946                                                 CVMX_USB_STAGE_DATA;
2947                                 }
2948                                 break;
2949                         case CVMX_USB_STAGE_STATUS:
2950                                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
2951                                         transaction->stage =
2952                                                 CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE;
2953                                 else
2954                                         __cvmx_usb_perform_complete(usb, pipe,
2955                                                 transaction,
2956                                                 CVMX_USB_COMPLETE_SUCCESS);
2957                                 break;
2958                         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
2959                                 __cvmx_usb_perform_complete(usb, pipe,
2960                                                 transaction,
2961                                                 CVMX_USB_COMPLETE_SUCCESS);
2962                                 break;
2963                         }
2964                         break;
2965                 case CVMX_USB_TRANSFER_BULK:
2966                 case CVMX_USB_TRANSFER_INTERRUPT:
2967                         /*
2968                          * The only time a bulk transfer isn't complete when it
2969                          * finishes with an ACK is during a split transaction.
2970                          * For splits we need to continue the transfer if more
2971                          * data is needed
2972                          */
2973                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2974                                 if (transaction->stage ==
2975                                                 CVMX_USB_STAGE_NON_CONTROL)
2976                                         transaction->stage =
2977                                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2978                                 else {
2979                                         if (buffer_space_left &&
2980                                                 (bytes_in_last_packet ==
2981                                                  pipe->max_packet))
2982                                                 transaction->stage =
2983                                                         CVMX_USB_STAGE_NON_CONTROL;
2984                                         else {
2985                                                 if (transaction->type ==
2986                                                         CVMX_USB_TRANSFER_INTERRUPT)
2987                                                         pipe->next_tx_frame +=
2988                                                                 pipe->interval;
2989                                                         __cvmx_usb_perform_complete(
2990                                                                 usb,
2991                                                                 pipe,
2992                                                                 transaction,
2993                                                                 CVMX_USB_COMPLETE_SUCCESS);
2994                                         }
2995                                 }
2996                         } else {
2997                                 if ((pipe->device_speed ==
2998                                         CVMX_USB_SPEED_HIGH) &&
2999                                     (pipe->transfer_type ==
3000                                      CVMX_USB_TRANSFER_BULK) &&
3001                                     (pipe->transfer_dir ==
3002                                      CVMX_USB_DIRECTION_OUT) &&
3003                                     (usbc_hcint.s.nak))
3004                                         pipe->flags |=
3005                                                 __CVMX_USB_PIPE_FLAGS_NEED_PING;
3006                                 if (!buffer_space_left ||
3007                                         (bytes_in_last_packet <
3008                                          pipe->max_packet)) {
3009                                         if (transaction->type ==
3010                                                 CVMX_USB_TRANSFER_INTERRUPT)
3011                                                 pipe->next_tx_frame +=
3012                                                         pipe->interval;
3013                                         __cvmx_usb_perform_complete(usb,
3014                                                 pipe,
3015                                                 transaction,
3016                                                 CVMX_USB_COMPLETE_SUCCESS);
3017                                 }
3018                         }
3019                         break;
3020                 case CVMX_USB_TRANSFER_ISOCHRONOUS:
3021                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
3022                                 /*
3023                                  * ISOCHRONOUS OUT splits don't require a
3024                                  * complete split stage. Instead they use a
3025                                  * sequence of begin OUT splits to transfer the
3026                                  * data 188 bytes at a time. Once the transfer
3027                                  * is complete, the pipe sleeps until the next
3028                                  * schedule interval
3029                                  */
3030                                 if (pipe->transfer_dir ==
3031                                         CVMX_USB_DIRECTION_OUT) {
3032                                         /*
3033                                          * If no space left or this wasn't a max
3034                                          * size packet then this transfer is
3035                                          * complete. Otherwise start it again to
3036                                          * send the next 188 bytes
3037                                          */
3038                                         if (!buffer_space_left ||
3039                                                 (bytes_this_transfer < 188)) {
3040                                                 pipe->next_tx_frame +=
3041                                                         pipe->interval;
3042                                                 __cvmx_usb_perform_complete(
3043                                                         usb,
3044                                                         pipe,
3045                                                         transaction,
3046                                                         CVMX_USB_COMPLETE_SUCCESS);
3047                                         }
3048                                 } else {
3049                                         if (transaction->stage ==
3050                                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE) {
3051                                                 /*
3052                                                  * We are in the incoming data
3053                                                  * phase. Keep getting data
3054                                                  * until we run out of space or
3055                                                  * get a small packet
3056                                                  */
3057                                                 if ((buffer_space_left == 0) ||
3058                                                         (bytes_in_last_packet <
3059                                                          pipe->max_packet)) {
3060                                                         pipe->next_tx_frame +=
3061                                                                 pipe->interval;
3062                                                         __cvmx_usb_perform_complete(
3063                                                                 usb,
3064                                                                 pipe,
3065                                                                 transaction,
3066                                                                 CVMX_USB_COMPLETE_SUCCESS);
3067                                                 }
3068                                         } else
3069                                                 transaction->stage =
3070                                                         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
3071                                 }
3072                         } else {
3073                                 pipe->next_tx_frame += pipe->interval;
3074                                 __cvmx_usb_perform_complete(usb,
3075                                                 pipe,
3076                                                 transaction,
3077                                                 CVMX_USB_COMPLETE_SUCCESS);
3078                         }
3079                         break;
3080                 }
3081         } else if (usbc_hcint.s.nak) {
3082                 /*
3083                  * If this was a split then clear our split in progress marker.
3084                  */
3085                 if (usb->active_split == transaction)
3086                         usb->active_split = NULL;
3087                 /*
3088                  * NAK as a response means the device couldn't accept the
3089                  * transaction, but it should be retried in the future. Rewind
3090                  * to the beginning of the transaction by anding off the split
3091                  * complete bit. Retry in the next interval
3092                  */
3093                 transaction->retries = 0;
3094                 transaction->stage &= ~1;
3095                 pipe->next_tx_frame += pipe->interval;
3096                 if (pipe->next_tx_frame < usb->frame_number)
3097                         pipe->next_tx_frame = usb->frame_number +
3098                                 pipe->interval -
3099                                 (usb->frame_number - pipe->next_tx_frame) %
3100                                 pipe->interval;
3101         } else {
3102                 struct cvmx_usb_port_status port;
3103
3104                 port = cvmx_usb_get_status(usb);
3105                 if (port.port_enabled) {
3106                         /* We'll retry the exact same transaction again */
3107                         transaction->retries++;
3108                 } else {
3109                         /*
3110                          * We get channel halted interrupts with no result bits
3111                          * sets when the cable is unplugged
3112                          */
3113                         __cvmx_usb_perform_complete(usb, pipe, transaction,
3114                                         CVMX_USB_COMPLETE_ERROR);
3115                 }
3116         }
3117         return 0;
3118 }
3119
3120 static void octeon_usb_port_callback(struct cvmx_usb_state *usb)
3121 {
3122         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
3123
3124         spin_unlock(&priv->lock);
3125         usb_hcd_poll_rh_status(octeon_to_hcd(priv));
3126         spin_lock(&priv->lock);
3127 }
3128
3129 /**
3130  * Poll the USB block for status and call all needed callback
3131  * handlers. This function is meant to be called in the interrupt
3132  * handler for the USB controller. It can also be called
3133  * periodically in a loop for non-interrupt based operation.
3134  *
3135  * @usb: USB device state populated by cvmx_usb_initialize().
3136  *
3137  * Returns: 0 or a negative error code.
3138  */
3139 static int cvmx_usb_poll(struct cvmx_usb_state *usb)
3140 {
3141         union cvmx_usbcx_hfnum usbc_hfnum;
3142         union cvmx_usbcx_gintsts usbc_gintsts;
3143
3144         prefetch_range(usb, sizeof(*usb));
3145
3146         /* Update the frame counter */
3147         usbc_hfnum.u32 = __cvmx_usb_read_csr32(usb,
3148                                                 CVMX_USBCX_HFNUM(usb->index));
3149         if ((usb->frame_number&0x3fff) > usbc_hfnum.s.frnum)
3150                 usb->frame_number += 0x4000;
3151         usb->frame_number &= ~0x3fffull;
3152         usb->frame_number |= usbc_hfnum.s.frnum;
3153
3154         /* Read the pending interrupts */
3155         usbc_gintsts.u32 = __cvmx_usb_read_csr32(usb,
3156                                                 CVMX_USBCX_GINTSTS(usb->index));
3157
3158         /* Clear the interrupts now that we know about them */
3159         __cvmx_usb_write_csr32(usb,
3160                                 CVMX_USBCX_GINTSTS(usb->index),
3161                                 usbc_gintsts.u32);
3162
3163         if (usbc_gintsts.s.rxflvl) {
3164                 /*
3165                  * RxFIFO Non-Empty (RxFLvl)
3166                  * Indicates that there is at least one packet pending to be
3167                  * read from the RxFIFO.
3168                  *
3169                  * In DMA mode this is handled by hardware
3170                  */
3171                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
3172                         __cvmx_usb_poll_rx_fifo(usb);
3173         }
3174         if (usbc_gintsts.s.ptxfemp || usbc_gintsts.s.nptxfemp) {
3175                 /* Fill the Tx FIFOs when not in DMA mode */
3176                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
3177                         __cvmx_usb_poll_tx_fifo(usb);
3178         }
3179         if (usbc_gintsts.s.disconnint || usbc_gintsts.s.prtint) {
3180                 union cvmx_usbcx_hprt usbc_hprt;
3181                 /*
3182                  * Disconnect Detected Interrupt (DisconnInt)
3183                  * Asserted when a device disconnect is detected.
3184                  *
3185                  * Host Port Interrupt (PrtInt)
3186                  * The core sets this bit to indicate a change in port status of
3187                  * one of the O2P USB core ports in Host mode. The application
3188                  * must read the Host Port Control and Status (HPRT) register to
3189                  * determine the exact event that caused this interrupt. The
3190                  * application must clear the appropriate status bit in the Host
3191                  * Port Control and Status register to clear this bit.
3192                  *
3193                  * Call the user's port callback
3194                  */
3195                 octeon_usb_port_callback(usb);
3196                 /* Clear the port change bits */
3197                 usbc_hprt.u32 = __cvmx_usb_read_csr32(usb,
3198                                 CVMX_USBCX_HPRT(usb->index));
3199                 usbc_hprt.s.prtena = 0;
3200                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HPRT(usb->index),
3201                                        usbc_hprt.u32);
3202         }
3203         if (usbc_gintsts.s.hchint) {
3204                 /*
3205                  * Host Channels Interrupt (HChInt)
3206                  * The core sets this bit to indicate that an interrupt is
3207                  * pending on one of the channels of the core (in Host mode).
3208                  * The application must read the Host All Channels Interrupt
3209                  * (HAINT) register to determine the exact number of the channel
3210                  * on which the interrupt occurred, and then read the
3211                  * corresponding Host Channel-n Interrupt (HCINTn) register to
3212                  * determine the exact cause of the interrupt. The application
3213                  * must clear the appropriate status bit in the HCINTn register
3214                  * to clear this bit.
3215                  */
3216                 union cvmx_usbcx_haint usbc_haint;
3217
3218                 usbc_haint.u32 = __cvmx_usb_read_csr32(usb,
3219                                         CVMX_USBCX_HAINT(usb->index));
3220                 while (usbc_haint.u32) {
3221                         int channel;
3222
3223                         channel = __fls(usbc_haint.u32);
3224                         __cvmx_usb_poll_channel(usb, channel);
3225                         usbc_haint.u32 ^= 1<<channel;
3226                 }
3227         }
3228
3229         __cvmx_usb_schedule(usb, usbc_gintsts.s.sof);
3230
3231         return 0;
3232 }
3233
3234 /* convert between an HCD pointer and the corresponding struct octeon_hcd */
3235 static inline struct octeon_hcd *hcd_to_octeon(struct usb_hcd *hcd)
3236 {
3237         return (struct octeon_hcd *)(hcd->hcd_priv);
3238 }
3239
3240 static irqreturn_t octeon_usb_irq(struct usb_hcd *hcd)
3241 {
3242         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3243         unsigned long flags;
3244
3245         spin_lock_irqsave(&priv->lock, flags);
3246         cvmx_usb_poll(&priv->usb);
3247         spin_unlock_irqrestore(&priv->lock, flags);
3248         return IRQ_HANDLED;
3249 }
3250
3251 static int octeon_usb_start(struct usb_hcd *hcd)
3252 {
3253         hcd->state = HC_STATE_RUNNING;
3254         return 0;
3255 }
3256
3257 static void octeon_usb_stop(struct usb_hcd *hcd)
3258 {
3259         hcd->state = HC_STATE_HALT;
3260 }
3261
3262 static int octeon_usb_get_frame_number(struct usb_hcd *hcd)
3263 {
3264         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3265
3266         return cvmx_usb_get_frame_number(&priv->usb);
3267 }
3268
3269 static int octeon_usb_urb_enqueue(struct usb_hcd *hcd,
3270                                   struct urb *urb,
3271                                   gfp_t mem_flags)
3272 {
3273         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3274         struct device *dev = hcd->self.controller;
3275         struct cvmx_usb_transaction *transaction = NULL;
3276         struct cvmx_usb_pipe *pipe;
3277         unsigned long flags;
3278         struct cvmx_usb_iso_packet *iso_packet;
3279         struct usb_host_endpoint *ep = urb->ep;
3280         int rc;
3281
3282         urb->status = 0;
3283         spin_lock_irqsave(&priv->lock, flags);
3284
3285         rc = usb_hcd_link_urb_to_ep(hcd, urb);
3286         if (rc) {
3287                 spin_unlock_irqrestore(&priv->lock, flags);
3288                 return rc;
3289         }
3290
3291         if (!ep->hcpriv) {
3292                 enum cvmx_usb_transfer transfer_type;
3293                 enum cvmx_usb_speed speed;
3294                 int split_device = 0;
3295                 int split_port = 0;
3296
3297                 switch (usb_pipetype(urb->pipe)) {
3298                 case PIPE_ISOCHRONOUS:
3299                         transfer_type = CVMX_USB_TRANSFER_ISOCHRONOUS;
3300                         break;
3301                 case PIPE_INTERRUPT:
3302                         transfer_type = CVMX_USB_TRANSFER_INTERRUPT;
3303                         break;
3304                 case PIPE_CONTROL:
3305                         transfer_type = CVMX_USB_TRANSFER_CONTROL;
3306                         break;
3307                 default:
3308                         transfer_type = CVMX_USB_TRANSFER_BULK;
3309                         break;
3310                 }
3311                 switch (urb->dev->speed) {
3312                 case USB_SPEED_LOW:
3313                         speed = CVMX_USB_SPEED_LOW;
3314                         break;
3315                 case USB_SPEED_FULL:
3316                         speed = CVMX_USB_SPEED_FULL;
3317                         break;
3318                 default:
3319                         speed = CVMX_USB_SPEED_HIGH;
3320                         break;
3321                 }
3322                 /*
3323                  * For slow devices on high speed ports we need to find the hub
3324                  * that does the speed translation so we know where to send the
3325                  * split transactions.
3326                  */
3327                 if (speed != CVMX_USB_SPEED_HIGH) {
3328                         /*
3329                          * Start at this device and work our way up the usb
3330                          * tree.
3331                          */
3332                         struct usb_device *dev = urb->dev;
3333
3334                         while (dev->parent) {
3335                                 /*
3336                                  * If our parent is high speed then he'll
3337                                  * receive the splits.
3338                                  */
3339                                 if (dev->parent->speed == USB_SPEED_HIGH) {
3340                                         split_device = dev->parent->devnum;
3341                                         split_port = dev->portnum;
3342                                         break;
3343                                 }
3344                                 /*
3345                                  * Move up the tree one level. If we make it all
3346                                  * the way up the tree, then the port must not
3347                                  * be in high speed mode and we don't need a
3348                                  * split.
3349                                  */
3350                                 dev = dev->parent;
3351                         }
3352                 }
3353                 pipe = cvmx_usb_open_pipe(&priv->usb, usb_pipedevice(urb->pipe),
3354                                           usb_pipeendpoint(urb->pipe), speed,
3355                                           le16_to_cpu(ep->desc.wMaxPacketSize)
3356                                           & 0x7ff,
3357                                           transfer_type,
3358                                           usb_pipein(urb->pipe) ?
3359                                                 CVMX_USB_DIRECTION_IN :
3360                                                 CVMX_USB_DIRECTION_OUT,
3361                                           urb->interval,
3362                                           (le16_to_cpu(ep->desc.wMaxPacketSize)
3363                                            >> 11) & 0x3,
3364                                           split_device, split_port);
3365                 if (!pipe) {
3366                         usb_hcd_unlink_urb_from_ep(hcd, urb);
3367                         spin_unlock_irqrestore(&priv->lock, flags);
3368                         dev_dbg(dev, "Failed to create pipe\n");
3369                         return -ENOMEM;
3370                 }
3371                 ep->hcpriv = pipe;
3372         } else {
3373                 pipe = ep->hcpriv;
3374         }
3375
3376         switch (usb_pipetype(urb->pipe)) {
3377         case PIPE_ISOCHRONOUS:
3378                 dev_dbg(dev, "Submit isochronous to %d.%d\n",
3379                         usb_pipedevice(urb->pipe),
3380                         usb_pipeendpoint(urb->pipe));
3381                 /*
3382                  * Allocate a structure to use for our private list of
3383                  * isochronous packets.
3384                  */
3385                 iso_packet = kmalloc(urb->number_of_packets *
3386                                      sizeof(struct cvmx_usb_iso_packet),
3387                                      GFP_ATOMIC);
3388                 if (iso_packet) {
3389                         int i;
3390                         /* Fill the list with the data from the URB */
3391                         for (i = 0; i < urb->number_of_packets; i++) {
3392                                 iso_packet[i].offset =
3393                                         urb->iso_frame_desc[i].offset;
3394                                 iso_packet[i].length =
3395                                         urb->iso_frame_desc[i].length;
3396                                 iso_packet[i].status =
3397                                         CVMX_USB_COMPLETE_ERROR;
3398                         }
3399                         /*
3400                          * Store a pointer to the list in the URB setup_packet
3401                          * field. We know this currently isn't being used and
3402                          * this saves us a bunch of logic.
3403                          */
3404                         urb->setup_packet = (char *)iso_packet;
3405                         transaction = cvmx_usb_submit_isochronous(&priv->usb,
3406                                                                   pipe, urb);
3407                         /*
3408                          * If submit failed we need to free our private packet
3409                          * list.
3410                          */
3411                         if (!transaction) {
3412                                 urb->setup_packet = NULL;
3413                                 kfree(iso_packet);
3414                         }
3415                 }
3416                 break;
3417         case PIPE_INTERRUPT:
3418                 dev_dbg(dev, "Submit interrupt to %d.%d\n",
3419                         usb_pipedevice(urb->pipe),
3420                         usb_pipeendpoint(urb->pipe));
3421                 transaction = cvmx_usb_submit_interrupt(&priv->usb, pipe, urb);
3422                 break;
3423         case PIPE_CONTROL:
3424                 dev_dbg(dev, "Submit control to %d.%d\n",
3425                         usb_pipedevice(urb->pipe),
3426                         usb_pipeendpoint(urb->pipe));
3427                 transaction = cvmx_usb_submit_control(&priv->usb, pipe, urb);
3428                 break;
3429         case PIPE_BULK:
3430                 dev_dbg(dev, "Submit bulk to %d.%d\n",
3431                         usb_pipedevice(urb->pipe),
3432                         usb_pipeendpoint(urb->pipe));
3433                 transaction = cvmx_usb_submit_bulk(&priv->usb, pipe, urb);
3434                 break;
3435         }
3436         if (!transaction) {
3437                 usb_hcd_unlink_urb_from_ep(hcd, urb);
3438                 spin_unlock_irqrestore(&priv->lock, flags);
3439                 dev_dbg(dev, "Failed to submit\n");
3440                 return -ENOMEM;
3441         }
3442         urb->hcpriv = transaction;
3443         spin_unlock_irqrestore(&priv->lock, flags);
3444         return 0;
3445 }
3446
3447 static int octeon_usb_urb_dequeue(struct usb_hcd *hcd,
3448                                   struct urb *urb,
3449                                   int status)
3450 {
3451         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3452         unsigned long flags;
3453         int rc;
3454
3455         if (!urb->dev)
3456                 return -EINVAL;
3457
3458         spin_lock_irqsave(&priv->lock, flags);
3459
3460         rc = usb_hcd_check_unlink_urb(hcd, urb, status);
3461         if (rc)
3462                 goto out;
3463
3464         urb->status = status;
3465         cvmx_usb_cancel(&priv->usb, urb->ep->hcpriv, urb->hcpriv);
3466
3467 out:
3468         spin_unlock_irqrestore(&priv->lock, flags);
3469
3470         return rc;
3471 }
3472
3473 static void octeon_usb_endpoint_disable(struct usb_hcd *hcd,
3474                                         struct usb_host_endpoint *ep)
3475 {
3476         struct device *dev = hcd->self.controller;
3477
3478         if (ep->hcpriv) {
3479                 struct octeon_hcd *priv = hcd_to_octeon(hcd);
3480                 struct cvmx_usb_pipe *pipe = ep->hcpriv;
3481                 unsigned long flags;
3482
3483                 spin_lock_irqsave(&priv->lock, flags);
3484                 cvmx_usb_cancel_all(&priv->usb, pipe);
3485                 if (cvmx_usb_close_pipe(&priv->usb, pipe))
3486                         dev_dbg(dev, "Closing pipe %p failed\n", pipe);
3487                 spin_unlock_irqrestore(&priv->lock, flags);
3488                 ep->hcpriv = NULL;
3489         }
3490 }
3491
3492 static int octeon_usb_hub_status_data(struct usb_hcd *hcd, char *buf)
3493 {
3494         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3495         struct cvmx_usb_port_status port_status;
3496         unsigned long flags;
3497
3498         spin_lock_irqsave(&priv->lock, flags);
3499         port_status = cvmx_usb_get_status(&priv->usb);
3500         spin_unlock_irqrestore(&priv->lock, flags);
3501         buf[0] = 0;
3502         buf[0] = port_status.connect_change << 1;
3503
3504         return buf[0] != 0;
3505 }
3506
3507 static int octeon_usb_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
3508                                 u16 wIndex, char *buf, u16 wLength)
3509 {
3510         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3511         struct device *dev = hcd->self.controller;
3512         struct cvmx_usb_port_status usb_port_status;
3513         int port_status;
3514         struct usb_hub_descriptor *desc;
3515         unsigned long flags;
3516
3517         switch (typeReq) {
3518         case ClearHubFeature:
3519                 dev_dbg(dev, "ClearHubFeature\n");
3520                 switch (wValue) {
3521                 case C_HUB_LOCAL_POWER:
3522                 case C_HUB_OVER_CURRENT:
3523                         /* Nothing required here */
3524                         break;
3525                 default:
3526                         return -EINVAL;
3527                 }
3528                 break;
3529         case ClearPortFeature:
3530                 dev_dbg(dev, "ClearPortFeature\n");
3531                 if (wIndex != 1) {
3532                         dev_dbg(dev, " INVALID\n");
3533                         return -EINVAL;
3534                 }
3535
3536                 switch (wValue) {
3537                 case USB_PORT_FEAT_ENABLE:
3538                         dev_dbg(dev, " ENABLE\n");
3539                         spin_lock_irqsave(&priv->lock, flags);
3540                         cvmx_usb_disable(&priv->usb);
3541                         spin_unlock_irqrestore(&priv->lock, flags);
3542                         break;
3543                 case USB_PORT_FEAT_SUSPEND:
3544                         dev_dbg(dev, " SUSPEND\n");
3545                         /* Not supported on Octeon */
3546                         break;
3547                 case USB_PORT_FEAT_POWER:
3548                         dev_dbg(dev, " POWER\n");
3549                         /* Not supported on Octeon */
3550                         break;
3551                 case USB_PORT_FEAT_INDICATOR:
3552                         dev_dbg(dev, " INDICATOR\n");
3553                         /* Port inidicator not supported */
3554                         break;
3555                 case USB_PORT_FEAT_C_CONNECTION:
3556                         dev_dbg(dev, " C_CONNECTION\n");
3557                         /* Clears drivers internal connect status change flag */
3558                         spin_lock_irqsave(&priv->lock, flags);
3559                         priv->usb.port_status =
3560                                 cvmx_usb_get_status(&priv->usb);
3561                         spin_unlock_irqrestore(&priv->lock, flags);
3562                         break;
3563                 case USB_PORT_FEAT_C_RESET:
3564                         dev_dbg(dev, " C_RESET\n");
3565                         /*
3566                          * Clears the driver's internal Port Reset Change flag.
3567                          */
3568                         spin_lock_irqsave(&priv->lock, flags);
3569                         priv->usb.port_status =
3570                                 cvmx_usb_get_status(&priv->usb);
3571                         spin_unlock_irqrestore(&priv->lock, flags);
3572                         break;
3573                 case USB_PORT_FEAT_C_ENABLE:
3574                         dev_dbg(dev, " C_ENABLE\n");
3575                         /*
3576                          * Clears the driver's internal Port Enable/Disable
3577                          * Change flag.
3578                          */
3579                         spin_lock_irqsave(&priv->lock, flags);
3580                         priv->usb.port_status =
3581                                 cvmx_usb_get_status(&priv->usb);
3582                         spin_unlock_irqrestore(&priv->lock, flags);
3583                         break;
3584                 case USB_PORT_FEAT_C_SUSPEND:
3585                         dev_dbg(dev, " C_SUSPEND\n");
3586                         /*
3587                          * Clears the driver's internal Port Suspend Change
3588                          * flag, which is set when resume signaling on the host
3589                          * port is complete.
3590                          */
3591                         break;
3592                 case USB_PORT_FEAT_C_OVER_CURRENT:
3593                         dev_dbg(dev, " C_OVER_CURRENT\n");
3594                         /* Clears the driver's overcurrent Change flag */
3595                         spin_lock_irqsave(&priv->lock, flags);
3596                         priv->usb.port_status =
3597                                 cvmx_usb_get_status(&priv->usb);
3598                         spin_unlock_irqrestore(&priv->lock, flags);
3599                         break;
3600                 default:
3601                         dev_dbg(dev, " UNKNOWN\n");
3602                         return -EINVAL;
3603                 }
3604                 break;
3605         case GetHubDescriptor:
3606                 dev_dbg(dev, "GetHubDescriptor\n");
3607                 desc = (struct usb_hub_descriptor *)buf;
3608                 desc->bDescLength = 9;
3609                 desc->bDescriptorType = 0x29;
3610                 desc->bNbrPorts = 1;
3611                 desc->wHubCharacteristics = cpu_to_le16(0x08);
3612                 desc->bPwrOn2PwrGood = 1;
3613                 desc->bHubContrCurrent = 0;
3614                 desc->u.hs.DeviceRemovable[0] = 0;
3615                 desc->u.hs.DeviceRemovable[1] = 0xff;
3616                 break;
3617         case GetHubStatus:
3618                 dev_dbg(dev, "GetHubStatus\n");
3619                 *(__le32 *) buf = 0;
3620                 break;
3621         case GetPortStatus:
3622                 dev_dbg(dev, "GetPortStatus\n");
3623                 if (wIndex != 1) {
3624                         dev_dbg(dev, " INVALID\n");
3625                         return -EINVAL;
3626                 }
3627
3628                 spin_lock_irqsave(&priv->lock, flags);
3629                 usb_port_status = cvmx_usb_get_status(&priv->usb);
3630                 spin_unlock_irqrestore(&priv->lock, flags);
3631                 port_status = 0;
3632
3633                 if (usb_port_status.connect_change) {
3634                         port_status |= (1 << USB_PORT_FEAT_C_CONNECTION);
3635                         dev_dbg(dev, " C_CONNECTION\n");
3636                 }
3637
3638                 if (usb_port_status.port_enabled) {
3639                         port_status |= (1 << USB_PORT_FEAT_C_ENABLE);
3640                         dev_dbg(dev, " C_ENABLE\n");
3641                 }
3642
3643                 if (usb_port_status.connected) {
3644                         port_status |= (1 << USB_PORT_FEAT_CONNECTION);
3645                         dev_dbg(dev, " CONNECTION\n");
3646                 }
3647
3648                 if (usb_port_status.port_enabled) {
3649                         port_status |= (1 << USB_PORT_FEAT_ENABLE);
3650                         dev_dbg(dev, " ENABLE\n");
3651                 }
3652
3653                 if (usb_port_status.port_over_current) {
3654                         port_status |= (1 << USB_PORT_FEAT_OVER_CURRENT);
3655                         dev_dbg(dev, " OVER_CURRENT\n");
3656                 }
3657
3658                 if (usb_port_status.port_powered) {
3659                         port_status |= (1 << USB_PORT_FEAT_POWER);
3660                         dev_dbg(dev, " POWER\n");
3661                 }
3662
3663                 if (usb_port_status.port_speed == CVMX_USB_SPEED_HIGH) {
3664                         port_status |= USB_PORT_STAT_HIGH_SPEED;
3665                         dev_dbg(dev, " HIGHSPEED\n");
3666                 } else if (usb_port_status.port_speed == CVMX_USB_SPEED_LOW) {
3667                         port_status |= (1 << USB_PORT_FEAT_LOWSPEED);
3668                         dev_dbg(dev, " LOWSPEED\n");
3669                 }
3670
3671                 *((__le32 *) buf) = cpu_to_le32(port_status);
3672                 break;
3673         case SetHubFeature:
3674                 dev_dbg(dev, "SetHubFeature\n");
3675                 /* No HUB features supported */
3676                 break;
3677         case SetPortFeature:
3678                 dev_dbg(dev, "SetPortFeature\n");
3679                 if (wIndex != 1) {
3680                         dev_dbg(dev, " INVALID\n");
3681                         return -EINVAL;
3682                 }
3683
3684                 switch (wValue) {
3685                 case USB_PORT_FEAT_SUSPEND:
3686                         dev_dbg(dev, " SUSPEND\n");
3687                         return -EINVAL;
3688                 case USB_PORT_FEAT_POWER:
3689                         dev_dbg(dev, " POWER\n");
3690                         return -EINVAL;
3691                 case USB_PORT_FEAT_RESET:
3692                         dev_dbg(dev, " RESET\n");
3693                         spin_lock_irqsave(&priv->lock, flags);
3694                         cvmx_usb_disable(&priv->usb);
3695                         if (cvmx_usb_enable(&priv->usb))
3696                                 dev_dbg(dev, "Failed to enable the port\n");
3697                         spin_unlock_irqrestore(&priv->lock, flags);
3698                         return 0;
3699                 case USB_PORT_FEAT_INDICATOR:
3700                         dev_dbg(dev, " INDICATOR\n");
3701                         /* Not supported */
3702                         break;
3703                 default:
3704                         dev_dbg(dev, " UNKNOWN\n");
3705                         return -EINVAL;
3706                 }
3707                 break;
3708         default:
3709                 dev_dbg(dev, "Unknown root hub request\n");
3710                 return -EINVAL;
3711         }
3712         return 0;
3713 }
3714
3715 static const struct hc_driver octeon_hc_driver = {
3716         .description            = "Octeon USB",
3717         .product_desc           = "Octeon Host Controller",
3718         .hcd_priv_size          = sizeof(struct octeon_hcd),
3719         .irq                    = octeon_usb_irq,
3720         .flags                  = HCD_MEMORY | HCD_USB2,
3721         .start                  = octeon_usb_start,
3722         .stop                   = octeon_usb_stop,
3723         .urb_enqueue            = octeon_usb_urb_enqueue,
3724         .urb_dequeue            = octeon_usb_urb_dequeue,
3725         .endpoint_disable       = octeon_usb_endpoint_disable,
3726         .get_frame_number       = octeon_usb_get_frame_number,
3727         .hub_status_data        = octeon_usb_hub_status_data,
3728         .hub_control            = octeon_usb_hub_control,
3729         .map_urb_for_dma        = octeon_map_urb_for_dma,
3730         .unmap_urb_for_dma      = octeon_unmap_urb_for_dma,
3731 };
3732
3733 static int octeon_usb_probe(struct platform_device *pdev)
3734 {
3735         int status;
3736         int initialize_flags;
3737         int usb_num;
3738         struct resource *res_mem;
3739         struct device_node *usbn_node;
3740         int irq = platform_get_irq(pdev, 0);
3741         struct device *dev = &pdev->dev;
3742         struct octeon_hcd *priv;
3743         struct usb_hcd *hcd;
3744         unsigned long flags;
3745         u32 clock_rate = 48000000;
3746         bool is_crystal_clock = false;
3747         const char *clock_type;
3748         int i;
3749
3750         if (dev->of_node == NULL) {
3751                 dev_err(dev, "Error: empty of_node\n");
3752                 return -ENXIO;
3753         }
3754         usbn_node = dev->of_node->parent;
3755
3756         i = of_property_read_u32(usbn_node,
3757                                  "refclk-frequency", &clock_rate);
3758         if (i) {
3759                 dev_err(dev, "No USBN \"refclk-frequency\"\n");
3760                 return -ENXIO;
3761         }
3762         switch (clock_rate) {
3763         case 12000000:
3764                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ;
3765                 break;
3766         case 24000000:
3767                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ;
3768                 break;
3769         case 48000000:
3770                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ;
3771                 break;
3772         default:
3773                 dev_err(dev, "Illebal USBN \"refclk-frequency\" %u\n",
3774                                 clock_rate);
3775                 return -ENXIO;
3776
3777         }
3778
3779         i = of_property_read_string(usbn_node,
3780                                     "refclk-type", &clock_type);
3781
3782         if (!i && strcmp("crystal", clock_type) == 0)
3783                 is_crystal_clock = true;
3784
3785         if (is_crystal_clock)
3786                 initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI;
3787         else
3788                 initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND;
3789
3790         res_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3791         if (res_mem == NULL) {
3792                 dev_err(dev, "found no memory resource\n");
3793                 return -ENXIO;
3794         }
3795         usb_num = (res_mem->start >> 44) & 1;
3796
3797         if (irq < 0) {
3798                 /* Defective device tree, but we know how to fix it. */
3799                 irq_hw_number_t hwirq = usb_num ? (1 << 6) + 17 : 56;
3800
3801                 irq = irq_create_mapping(NULL, hwirq);
3802         }
3803
3804         /*
3805          * Set the DMA mask to 64bits so we get buffers already translated for
3806          * DMA.
3807          */
3808         dev->coherent_dma_mask = ~0;
3809         dev->dma_mask = &dev->coherent_dma_mask;
3810
3811         /*
3812          * Only cn52XX and cn56XX have DWC_OTG USB hardware and the
3813          * IOB priority registers.  Under heavy network load USB
3814          * hardware can be starved by the IOB causing a crash.  Give
3815          * it a priority boost if it has been waiting more than 400
3816          * cycles to avoid this situation.
3817          *
3818          * Testing indicates that a cnt_val of 8192 is not sufficient,
3819          * but no failures are seen with 4096.  We choose a value of
3820          * 400 to give a safety factor of 10.
3821          */
3822         if (OCTEON_IS_MODEL(OCTEON_CN52XX) || OCTEON_IS_MODEL(OCTEON_CN56XX)) {
3823                 union cvmx_iob_n2c_l2c_pri_cnt pri_cnt;
3824
3825                 pri_cnt.u64 = 0;
3826                 pri_cnt.s.cnt_enb = 1;
3827                 pri_cnt.s.cnt_val = 400;
3828                 cvmx_write_csr(CVMX_IOB_N2C_L2C_PRI_CNT, pri_cnt.u64);
3829         }
3830
3831         hcd = usb_create_hcd(&octeon_hc_driver, dev, dev_name(dev));
3832         if (!hcd) {
3833                 dev_dbg(dev, "Failed to allocate memory for HCD\n");
3834                 return -1;
3835         }
3836         hcd->uses_new_polling = 1;
3837         priv = (struct octeon_hcd *)hcd->hcd_priv;
3838
3839         spin_lock_init(&priv->lock);
3840
3841         status = cvmx_usb_initialize(&priv->usb, usb_num, initialize_flags);
3842         if (status) {
3843                 dev_dbg(dev, "USB initialization failed with %d\n", status);
3844                 kfree(hcd);
3845                 return -1;
3846         }
3847
3848         /* This delay is needed for CN3010, but I don't know why... */
3849         mdelay(10);
3850
3851         spin_lock_irqsave(&priv->lock, flags);
3852         cvmx_usb_poll(&priv->usb);
3853         spin_unlock_irqrestore(&priv->lock, flags);
3854
3855         status = usb_add_hcd(hcd, irq, 0);
3856         if (status) {
3857                 dev_dbg(dev, "USB add HCD failed with %d\n", status);
3858                 kfree(hcd);
3859                 return -1;
3860         }
3861         device_wakeup_enable(hcd->self.controller);
3862
3863         dev_info(dev, "Registered HCD for port %d on irq %d\n", usb_num, irq);
3864
3865         return 0;
3866 }
3867
3868 static int octeon_usb_remove(struct platform_device *pdev)
3869 {
3870         int status;
3871         struct device *dev = &pdev->dev;
3872         struct usb_hcd *hcd = dev_get_drvdata(dev);
3873         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3874         unsigned long flags;
3875
3876         usb_remove_hcd(hcd);
3877         spin_lock_irqsave(&priv->lock, flags);
3878         status = cvmx_usb_shutdown(&priv->usb);
3879         spin_unlock_irqrestore(&priv->lock, flags);
3880         if (status)
3881                 dev_dbg(dev, "USB shutdown failed with %d\n", status);
3882
3883         kfree(hcd);
3884
3885         return 0;
3886 }
3887
3888 static struct of_device_id octeon_usb_match[] = {
3889         {
3890                 .compatible = "cavium,octeon-5750-usbc",
3891         },
3892         {},
3893 };
3894
3895 static struct platform_driver octeon_usb_driver = {
3896         .driver = {
3897                 .name       = "OcteonUSB",
3898                 .of_match_table = octeon_usb_match,
3899         },
3900         .probe      = octeon_usb_probe,
3901         .remove     = octeon_usb_remove,
3902 };
3903
3904 static int __init octeon_usb_driver_init(void)
3905 {
3906         if (usb_disabled())
3907                 return 0;
3908
3909         return platform_driver_register(&octeon_usb_driver);
3910 }
3911 module_init(octeon_usb_driver_init);
3912
3913 static void __exit octeon_usb_driver_exit(void)
3914 {
3915         if (usb_disabled())
3916                 return;
3917
3918         platform_driver_unregister(&octeon_usb_driver);
3919 }
3920 module_exit(octeon_usb_driver_exit);
3921
3922 MODULE_LICENSE("GPL");
3923 MODULE_AUTHOR("Cavium, Inc. <support@cavium.com>");
3924 MODULE_DESCRIPTION("Cavium Inc. OCTEON USB Host driver.");