drivers/staging/rdma/hfi1/diag.c

   1 /*
   2  *
   3  * This file is provided under a dual BSD/GPLv2 license.  When using or
   4  * redistributing this file, you may do so under either license.
   5  *
   6  * GPL LICENSE SUMMARY
   7  *
   8  * Copyright(c) 2015, 2016 Intel Corporation.
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of version 2 of the GNU General Public License as
  12  * published by the Free Software Foundation.
  13  *
  14  * This program is distributed in the hope that it will be useful, but
  15  * WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * General Public License for more details.
  18  *
  19  * BSD LICENSE
  20  *
  21  * Copyright(c) 2015, 2016 Intel Corporation.
  22  *
  23  * Redistribution and use in source and binary forms, with or without
  24  * modification, are permitted provided that the following conditions
  25  * are met:
  26  *
  27  *  - Redistributions of source code must retain the above copyright
  28  *    notice, this list of conditions and the following disclaimer.
  29  *  - Redistributions in binary form must reproduce the above copyright
  30  *    notice, this list of conditions and the following disclaimer in
  31  *    the documentation and/or other materials provided with the
  32  *    distribution.
  33  *  - Neither the name of Intel Corporation nor the names of its
  34  *    contributors may be used to endorse or promote products derived
  35  *    from this software without specific prior written permission.
  36  *
  37  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  38  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  39  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  40  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  41  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  42  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  43  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  44  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  45  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  46  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  47  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  48  *
  49  */
  50
  51 /*
  52  * This file contains support for diagnostic functions.  It is accessed by
  53  * opening the hfi1_diag device, normally minor number 129.  Diagnostic use
  54  * of the chip may render the chip or board unusable until the driver
  55  * is unloaded, or in some cases, until the system is rebooted.
  56  *
  57  * Accesses to the chip through this interface are not similar to going
  58  * through the /sys/bus/pci resource mmap interface.
  59  */
  60
  61 #include <linux/io.h>
  62 #include <linux/pci.h>
  63 #include <linux/poll.h>
  64 #include <linux/vmalloc.h>
  65 #include <linux/export.h>
  66 #include <linux/fs.h>
  67 #include <linux/uaccess.h>
  68 #include <linux/module.h>
  69 #include <rdma/ib_smi.h>
  70 #include "hfi.h"
  71 #include "device.h"
  72 #include "common.h"
  73 #include "verbs_txreq.h"
  74 #include "trace.h"
  75
  76 #undef pr_fmt
  77 #define pr_fmt(fmt) DRIVER_NAME ": " fmt
  78 #define snoop_dbg(fmt, ...) \
  79         hfi1_cdbg(SNOOP, fmt, ##__VA_ARGS__)
  80
  81 /* Snoop option mask */
  82 #define SNOOP_DROP_SEND         BIT(0)
  83 #define SNOOP_USE_METADATA      BIT(1)
  84 #define SNOOP_SET_VL0TOVL15     BIT(2)
  85
  86 static u8 snoop_flags;
  87
  88 /*
  89  * Extract packet length from LRH header.
  90  * This is in Dwords so multiply by 4 to get size in bytes
  91  */
  92 #define HFI1_GET_PKT_LEN(x)      (((be16_to_cpu((x)->lrh[2]) & 0xFFF)) << 2)
  93
  94 enum hfi1_filter_status {
  95         HFI1_FILTER_HIT,
  96         HFI1_FILTER_ERR,
  97         HFI1_FILTER_MISS
  98 };
  99
 100 /* snoop processing functions */
 101 rhf_rcv_function_ptr snoop_rhf_rcv_functions[8] = {
 102         [RHF_RCV_TYPE_EXPECTED] = snoop_recv_handler,
 103         [RHF_RCV_TYPE_EAGER]    = snoop_recv_handler,
 104         [RHF_RCV_TYPE_IB]       = snoop_recv_handler,
 105         [RHF_RCV_TYPE_ERROR]    = snoop_recv_handler,
 106         [RHF_RCV_TYPE_BYPASS]   = snoop_recv_handler,
 107         [RHF_RCV_TYPE_INVALID5] = process_receive_invalid,
 108         [RHF_RCV_TYPE_INVALID6] = process_receive_invalid,
 109         [RHF_RCV_TYPE_INVALID7] = process_receive_invalid
 110 };
 111
 112 /* Snoop packet structure */
 113 struct snoop_packet {
 114         struct list_head list;
 115         u32 total_len;
 116         u8 data[];
 117 };
 118
 119 /* Do not make these an enum or it will blow up the capture_md */
 120 #define PKT_DIR_EGRESS 0x0
 121 #define PKT_DIR_INGRESS 0x1
 122
 123 /* Packet capture metadata returned to the user with the packet. */
 124 struct capture_md {
 125         u8 port;
 126         u8 dir;
 127         u8 reserved[6];
 128         union {
 129                 u64 pbc;
 130                 u64 rhf;
 131         } u;
 132 };
 133
 134 static atomic_t diagpkt_count = ATOMIC_INIT(0);
 135 static struct cdev diagpkt_cdev;
 136 static struct device *diagpkt_device;
 137
 138 static ssize_t diagpkt_write(struct file *fp, const char __user *data,
 139                              size_t count, loff_t *off);
 140
 141 static const struct file_operations diagpkt_file_ops = {
 142         .owner = THIS_MODULE,
 143         .write = diagpkt_write,
 144         .llseek = noop_llseek,
 145 };
 146
 147 /*
 148  * This is used for communication with user space for snoop extended IOCTLs
 149  */
 150 struct hfi1_link_info {
 151         __be64 node_guid;
 152         u8 port_mode;
 153         u8 port_state;
 154         u16 link_speed_active;
 155         u16 link_width_active;
 156         u16 vl15_init;
 157         u8 port_number;
 158         /*
 159          * Add padding to make this a full IB SMP payload. Note: changing the
 160          * size of this structure will make the IOCTLs created with _IOWR
 161          * change.
 162          * Be sure to run tests on all IOCTLs when making changes to this
 163          * structure.
 164          */
 165         u8 res[47];
 166 };
 167
 168 /*
 169  * This starts our ioctl sequence numbers *way* off from the ones
 170  * defined in ib_core.
 171  */
 172 #define SNOOP_CAPTURE_VERSION 0x1
 173
 174 #define IB_IOCTL_MAGIC          0x1b /* See Documentation/ioctl-number.txt */
 175 #define HFI1_SNOOP_IOC_MAGIC IB_IOCTL_MAGIC
 176 #define HFI1_SNOOP_IOC_BASE_SEQ 0x80
 177
 178 #define HFI1_SNOOP_IOCGETLINKSTATE \
 179         _IO(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ)
 180 #define HFI1_SNOOP_IOCSETLINKSTATE \
 181         _IO(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ + 1)
 182 #define HFI1_SNOOP_IOCCLEARQUEUE \
 183         _IO(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ + 2)
 184 #define HFI1_SNOOP_IOCCLEARFILTER \
 185         _IO(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ + 3)
 186 #define HFI1_SNOOP_IOCSETFILTER \
 187         _IO(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ + 4)
 188 #define HFI1_SNOOP_IOCGETVERSION \
 189         _IO(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ + 5)
 190 #define HFI1_SNOOP_IOCSET_OPTS \
 191         _IO(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ + 6)
 192
 193 /*
 194  * These offsets +6/+7 could change, but these are already known and used
 195  * IOCTL numbers so don't change them without a good reason.
 196  */
 197 #define HFI1_SNOOP_IOCGETLINKSTATE_EXTRA \
 198         _IOWR(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ + 6, \
 199                 struct hfi1_link_info)
 200 #define HFI1_SNOOP_IOCSETLINKSTATE_EXTRA \
 201         _IOWR(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ + 7, \
 202                 struct hfi1_link_info)
 203
 204 static int hfi1_snoop_open(struct inode *in, struct file *fp);
 205 static ssize_t hfi1_snoop_read(struct file *fp, char __user *data,
 206                                size_t pkt_len, loff_t *off);
 207 static ssize_t hfi1_snoop_write(struct file *fp, const char __user *data,
 208                                 size_t count, loff_t *off);
 209 static long hfi1_ioctl(struct file *fp, unsigned int cmd, unsigned long arg);
 210 static unsigned int hfi1_snoop_poll(struct file *fp,
 211                                     struct poll_table_struct *wait);
 212 static int hfi1_snoop_release(struct inode *in, struct file *fp);
 213
 214 struct hfi1_packet_filter_command {
 215         int opcode;
 216         int length;
 217         void *value_ptr;
 218 };
 219
 220 /* Can't re-use PKT_DIR_*GRESS here because 0 means no packets for this */
 221 #define HFI1_SNOOP_INGRESS 0x1
 222 #define HFI1_SNOOP_EGRESS  0x2
 223
 224 enum hfi1_packet_filter_opcodes {
 225         FILTER_BY_LID,
 226         FILTER_BY_DLID,
 227         FILTER_BY_MAD_MGMT_CLASS,
 228         FILTER_BY_QP_NUMBER,
 229         FILTER_BY_PKT_TYPE,
 230         FILTER_BY_SERVICE_LEVEL,
 231         FILTER_BY_PKEY,
 232         FILTER_BY_DIRECTION,
 233 };
 234
 235 static const struct file_operations snoop_file_ops = {
 236         .owner = THIS_MODULE,
 237         .open = hfi1_snoop_open,
 238         .read = hfi1_snoop_read,
 239         .unlocked_ioctl = hfi1_ioctl,
 240         .poll = hfi1_snoop_poll,
 241         .write = hfi1_snoop_write,
 242         .release = hfi1_snoop_release
 243 };
 244
 245 struct hfi1_filter_array {
 246         int (*filter)(void *, void *, void *);
 247 };
 248
 249 static int hfi1_filter_lid(void *ibhdr, void *packet_data, void *value);
 250 static int hfi1_filter_dlid(void *ibhdr, void *packet_data, void *value);
 251 static int hfi1_filter_mad_mgmt_class(void *ibhdr, void *packet_data,
 252                                       void *value);
 253 static int hfi1_filter_qp_number(void *ibhdr, void *packet_data, void *value);
 254 static int hfi1_filter_ibpacket_type(void *ibhdr, void *packet_data,
 255                                      void *value);
 256 static int hfi1_filter_ib_service_level(void *ibhdr, void *packet_data,
 257                                         void *value);
 258 static int hfi1_filter_ib_pkey(void *ibhdr, void *packet_data, void *value);
 259 static int hfi1_filter_direction(void *ibhdr, void *packet_data, void *value);
 260
 261 static struct hfi1_filter_array hfi1_filters[] = {
 262         { hfi1_filter_lid },
 263         { hfi1_filter_dlid },
 264         { hfi1_filter_mad_mgmt_class },
 265         { hfi1_filter_qp_number },
 266         { hfi1_filter_ibpacket_type },
 267         { hfi1_filter_ib_service_level },
 268         { hfi1_filter_ib_pkey },
 269         { hfi1_filter_direction },
 270 };
 271
 272 #define HFI1_MAX_FILTERS        ARRAY_SIZE(hfi1_filters)
 273 #define HFI1_DIAG_MINOR_BASE    129
 274
 275 static int hfi1_snoop_add(struct hfi1_devdata *dd, const char *name);
 276
 277 int hfi1_diag_add(struct hfi1_devdata *dd)
 278 {
 279         char name[16];
 280         int ret = 0;
 281
 282         snprintf(name, sizeof(name), "%s_diagpkt%d", class_name(),
 283                  dd->unit);
 284         /*
 285          * Do this for each device as opposed to the normal diagpkt
 286          * interface which is one per host
 287          */
 288         ret = hfi1_snoop_add(dd, name);
 289         if (ret)
 290                 dd_dev_err(dd, "Unable to init snoop/capture device");
 291
 292         snprintf(name, sizeof(name), "%s_diagpkt", class_name());
 293         if (atomic_inc_return(&diagpkt_count) == 1) {
 294                 ret = hfi1_cdev_init(HFI1_DIAGPKT_MINOR, name,
 295                                      &diagpkt_file_ops, &diagpkt_cdev,
 296                                      &diagpkt_device, false);
 297         }
 298
 299         return ret;
 300 }
 301
 302 /* this must be called w/ dd->snoop_in_lock held */
 303 static void drain_snoop_list(struct list_head *queue)
 304 {
 305         struct list_head *pos, *q;
 306         struct snoop_packet *packet;
 307
 308         list_for_each_safe(pos, q, queue) {
 309                 packet = list_entry(pos, struct snoop_packet, list);
 310                 list_del(pos);
 311                 kfree(packet);
 312         }
 313 }
 314
 315 static void hfi1_snoop_remove(struct hfi1_devdata *dd)
 316 {
 317         unsigned long flags = 0;
 318
 319         spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
 320         drain_snoop_list(&dd->hfi1_snoop.queue);
 321         hfi1_cdev_cleanup(&dd->hfi1_snoop.cdev, &dd->hfi1_snoop.class_dev);
 322         spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
 323 }
 324
 325 void hfi1_diag_remove(struct hfi1_devdata *dd)
 326 {
 327         hfi1_snoop_remove(dd);
 328         if (atomic_dec_and_test(&diagpkt_count))
 329                 hfi1_cdev_cleanup(&diagpkt_cdev, &diagpkt_device);
 330         hfi1_cdev_cleanup(&dd->diag_cdev, &dd->diag_device);
 331 }
 332
 333 /*
 334  * Allocated structure shared between the credit return mechanism and
 335  * diagpkt_send().
 336  */
 337 struct diagpkt_wait {
 338         struct completion credits_returned;
 339         int code;
 340         atomic_t count;
 341 };
 342
 343 /*
 344  * When each side is finished with the structure, they call this.
 345  * The last user frees the structure.
 346  */
 347 static void put_diagpkt_wait(struct diagpkt_wait *wait)
 348 {
 349         if (atomic_dec_and_test(&wait->count))
 350                 kfree(wait);
 351 }
 352
 353 /*
 354  * Callback from the credit return code.  Set the complete, which
 355  * will let diapkt_send() continue.
 356  */
 357 static void diagpkt_complete(void *arg, int code)
 358 {
 359         struct diagpkt_wait *wait = (struct diagpkt_wait *)arg;
 360
 361         wait->code = code;
 362         complete(&wait->credits_returned);
 363         put_diagpkt_wait(wait); /* finished with the structure */
 364 }
 365
 366 /**
 367  * diagpkt_send - send a packet
 368  * @dp: diag packet descriptor
 369  */
 370 static ssize_t diagpkt_send(struct diag_pkt *dp)
 371 {
 372         struct hfi1_devdata *dd;
 373         struct send_context *sc;
 374         struct pio_buf *pbuf;
 375         u32 *tmpbuf = NULL;
 376         ssize_t ret = 0;
 377         u32 pkt_len, total_len;
 378         pio_release_cb credit_cb = NULL;
 379         void *credit_arg = NULL;
 380         struct diagpkt_wait *wait = NULL;
 381         int trycount = 0;
 382
 383         dd = hfi1_lookup(dp->unit);
 384         if (!dd || !(dd->flags & HFI1_PRESENT) || !dd->kregbase) {
 385                 ret = -ENODEV;
 386                 goto bail;
 387         }
 388         if (!(dd->flags & HFI1_INITTED)) {
 389                 /* no hardware, freeze, etc. */
 390                 ret = -ENODEV;
 391                 goto bail;
 392         }
 393
 394         if (dp->version != _DIAG_PKT_VERS) {
 395                 dd_dev_err(dd, "Invalid version %u for diagpkt_write\n",
 396                            dp->version);
 397                 ret = -EINVAL;
 398                 goto bail;
 399         }
 400
 401         /* send count must be an exact number of dwords */
 402         if (dp->len & 3) {
 403                 ret = -EINVAL;
 404                 goto bail;
 405         }
 406
 407         /* there is only port 1 */
 408         if (dp->port != 1) {
 409                 ret = -EINVAL;
 410                 goto bail;
 411         }
 412
 413         /* need a valid context */
 414         if (dp->sw_index >= dd->num_send_contexts) {
 415                 ret = -EINVAL;
 416                 goto bail;
 417         }
 418         /* can only use kernel contexts */
 419         if (dd->send_contexts[dp->sw_index].type != SC_KERNEL) {
 420                 ret = -EINVAL;
 421                 goto bail;
 422         }
 423         /* must be allocated */
 424         sc = dd->send_contexts[dp->sw_index].sc;
 425         if (!sc) {
 426                 ret = -EINVAL;
 427                 goto bail;
 428         }
 429         /* must be enabled */
 430         if (!(sc->flags & SCF_ENABLED)) {
 431                 ret = -EINVAL;
 432                 goto bail;
 433         }
 434
 435         /* allocate a buffer and copy the data in */
 436         tmpbuf = vmalloc(dp->len);
 437         if (!tmpbuf) {
 438                 ret = -ENOMEM;
 439                 goto bail;
 440         }
 441
 442         if (copy_from_user(tmpbuf,
 443                            (const void __user *)(unsigned long)dp->data,
 444                            dp->len)) {
 445                 ret = -EFAULT;
 446                 goto bail;
 447         }
 448
 449         /*
 450          * pkt_len is how much data we have to write, includes header and data.
 451          * total_len is length of the packet in Dwords plus the PBC should not
 452          * include the CRC.
 453          */
 454         pkt_len = dp->len >> 2;
 455         total_len = pkt_len + 2; /* PBC + packet */
 456
 457         /* if 0, fill in a default */
 458         if (dp->pbc == 0) {
 459                 struct hfi1_pportdata *ppd = dd->pport;
 460
 461                 hfi1_cdbg(PKT, "Generating PBC");
 462                 dp->pbc = create_pbc(ppd, 0, 0, 0, total_len);
 463         } else {
 464                 hfi1_cdbg(PKT, "Using passed in PBC");
 465         }
 466
 467         hfi1_cdbg(PKT, "Egress PBC content is 0x%llx", dp->pbc);
 468
 469         /*
 470          * The caller wants to wait until the packet is sent and to
 471          * check for errors.  The best we can do is wait until
 472          * the buffer credits are returned and check if any packet
 473          * error has occurred.  If there are any late errors, this
 474          * could miss it.  If there are other senders who generate
 475          * an error, this may find it.  However, in general, it
 476          * should catch most.
 477          */
 478         if (dp->flags & F_DIAGPKT_WAIT) {
 479                 /* always force a credit return */
 480                 dp->pbc |= PBC_CREDIT_RETURN;
 481                 /* turn on credit return interrupts */
 482                 sc_add_credit_return_intr(sc);
 483                 wait = kmalloc(sizeof(*wait), GFP_KERNEL);
 484                 if (!wait) {
 485                         ret = -ENOMEM;
 486                         goto bail;
 487                 }
 488                 init_completion(&wait->credits_returned);
 489                 atomic_set(&wait->count, 2);
 490                 wait->code = PRC_OK;
 491
 492                 credit_cb = diagpkt_complete;
 493                 credit_arg = wait;
 494         }
 495
 496 retry:
 497         pbuf = sc_buffer_alloc(sc, total_len, credit_cb, credit_arg);
 498         if (!pbuf) {
 499                 if (trycount == 0) {
 500                         /* force a credit return and try again */
 501                         sc_return_credits(sc);
 502                         trycount = 1;
 503                         goto retry;
 504                 }
 505                 /*
 506                  * No send buffer means no credit callback.  Undo
 507                  * the wait set-up that was done above.  We free wait
 508                  * because the callback will never be called.
 509                  */
 510                 if (dp->flags & F_DIAGPKT_WAIT) {
 511                         sc_del_credit_return_intr(sc);
 512                         kfree(wait);
 513                         wait = NULL;
 514                 }
 515                 ret = -ENOSPC;
 516                 goto bail;
 517         }
 518
 519         pio_copy(dd, pbuf, dp->pbc, tmpbuf, pkt_len);
 520         /* no flush needed as the HW knows the packet size */
 521
 522         ret = sizeof(*dp);
 523
 524         if (dp->flags & F_DIAGPKT_WAIT) {
 525                 /* wait for credit return */
 526                 ret = wait_for_completion_interruptible(
 527                                                 &wait->credits_returned);
 528                 /*
 529                  * If the wait returns an error, the wait was interrupted,
 530                  * e.g. with a ^C in the user program.  The callback is
 531                  * still pending.  This is OK as the wait structure is
 532                  * kmalloc'ed and the structure will free itself when
 533                  * all users are done with it.
 534                  *
 535                  * A context disable occurs on a send context restart, so
 536                  * include that in the list of errors below to check for.
 537                  * NOTE: PRC_FILL_ERR is at best informational and cannot
 538                  * be depended on.
 539                  */
 540                 if (!ret && (((wait->code & PRC_STATUS_ERR) ||
 541                               (wait->code & PRC_FILL_ERR) ||
 542                               (wait->code & PRC_SC_DISABLE))))
 543                         ret = -EIO;
 544
 545                 put_diagpkt_wait(wait); /* finished with the structure */
 546                 sc_del_credit_return_intr(sc);
 547         }
 548
 549 bail:
 550         vfree(tmpbuf);
 551         return ret;
 552 }
 553
 554 static ssize_t diagpkt_write(struct file *fp, const char __user *data,
 555                              size_t count, loff_t *off)
 556 {
 557         struct hfi1_devdata *dd;
 558         struct send_context *sc;
 559         u8 vl;
 560
 561         struct diag_pkt dp;
 562
 563         if (count != sizeof(dp))
 564                 return -EINVAL;
 565
 566         if (copy_from_user(&dp, data, sizeof(dp)))
 567                 return -EFAULT;
 568
 569         /*
 570         * The Send Context is derived from the PbcVL value
 571         * if PBC is populated
 572         */
 573         if (dp.pbc) {
 574                 dd = hfi1_lookup(dp.unit);
 575                 if (!dd)
 576                         return -ENODEV;
 577                 vl = (dp.pbc >> PBC_VL_SHIFT) & PBC_VL_MASK;
 578                 sc = dd->vld[vl].sc;
 579                 if (sc) {
 580                         dp.sw_index = sc->sw_index;
 581                         hfi1_cdbg(
 582                                PKT,
 583                                "Packet sent over VL %d via Send Context %u(%u)",
 584                                vl, sc->sw_index, sc->hw_context);
 585                 }
 586         }
 587
 588         return diagpkt_send(&dp);
 589 }
 590
 591 static int hfi1_snoop_add(struct hfi1_devdata *dd, const char *name)
 592 {
 593         int ret = 0;
 594
 595         dd->hfi1_snoop.mode_flag = 0;
 596         spin_lock_init(&dd->hfi1_snoop.snoop_lock);
 597         INIT_LIST_HEAD(&dd->hfi1_snoop.queue);
 598         init_waitqueue_head(&dd->hfi1_snoop.waitq);
 599
 600         ret = hfi1_cdev_init(HFI1_SNOOP_CAPTURE_BASE + dd->unit, name,
 601                              &snoop_file_ops,
 602                              &dd->hfi1_snoop.cdev, &dd->hfi1_snoop.class_dev,
 603                              false);
 604
 605         if (ret) {
 606                 dd_dev_err(dd, "Couldn't create %s device: %d", name, ret);
 607                 hfi1_cdev_cleanup(&dd->hfi1_snoop.cdev,
 608                                   &dd->hfi1_snoop.class_dev);
 609         }
 610
 611         return ret;
 612 }
 613
 614 static struct hfi1_devdata *hfi1_dd_from_sc_inode(struct inode *in)
 615 {
 616         int unit = iminor(in) - HFI1_SNOOP_CAPTURE_BASE;
 617         struct hfi1_devdata *dd;
 618
 619         dd = hfi1_lookup(unit);
 620         return dd;
 621 }
 622
 623 /* clear or restore send context integrity checks */
 624 static void adjust_integrity_checks(struct hfi1_devdata *dd)
 625 {
 626         struct send_context *sc;
 627         unsigned long sc_flags;
 628         int i;
 629
 630         spin_lock_irqsave(&dd->sc_lock, sc_flags);
 631         for (i = 0; i < dd->num_send_contexts; i++) {
 632                 int enable;
 633
 634                 sc = dd->send_contexts[i].sc;
 635
 636                 if (!sc)
 637                         continue;       /* not allocated */
 638
 639                 enable = likely(!HFI1_CAP_IS_KSET(NO_INTEGRITY)) &&
 640                          dd->hfi1_snoop.mode_flag != HFI1_PORT_SNOOP_MODE;
 641
 642                 set_pio_integrity(sc);
 643
 644                 if (enable) /* take HFI_CAP_* flags into account */
 645                         hfi1_init_ctxt(sc);
 646         }
 647         spin_unlock_irqrestore(&dd->sc_lock, sc_flags);
 648 }
 649
 650 static int hfi1_snoop_open(struct inode *in, struct file *fp)
 651 {
 652         int ret;
 653         int mode_flag = 0;
 654         unsigned long flags = 0;
 655         struct hfi1_devdata *dd;
 656         struct list_head *queue;
 657
 658         mutex_lock(&hfi1_mutex);
 659
 660         dd = hfi1_dd_from_sc_inode(in);
 661         if (!dd) {
 662                 ret = -ENODEV;
 663                 goto bail;
 664         }
 665
 666         /*
 667          * File mode determines snoop or capture. Some existing user
 668          * applications expect the capture device to be able to be opened RDWR
 669          * because they expect a dedicated capture device. For this reason we
 670          * support a module param to force capture mode even if the file open
 671          * mode matches snoop.
 672          */
 673         if ((fp->f_flags & O_ACCMODE) == O_RDONLY) {
 674                 snoop_dbg("Capture Enabled");
 675                 mode_flag = HFI1_PORT_CAPTURE_MODE;
 676         } else if ((fp->f_flags & O_ACCMODE) == O_RDWR) {
 677                 snoop_dbg("Snoop Enabled");
 678                 mode_flag = HFI1_PORT_SNOOP_MODE;
 679         } else {
 680                 snoop_dbg("Invalid");
 681                 ret =  -EINVAL;
 682                 goto bail;
 683         }
 684         queue = &dd->hfi1_snoop.queue;
 685
 686         /*
 687          * We are not supporting snoop and capture at the same time.
 688          */
 689         spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
 690         if (dd->hfi1_snoop.mode_flag) {
 691                 ret = -EBUSY;
 692                 spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
 693                 goto bail;
 694         }
 695
 696         dd->hfi1_snoop.mode_flag = mode_flag;
 697         drain_snoop_list(queue);
 698
 699         dd->hfi1_snoop.filter_callback = NULL;
 700         dd->hfi1_snoop.filter_value = NULL;
 701
 702         /*
 703          * Send side packet integrity checks are not helpful when snooping so
 704          * disable and re-enable when we stop snooping.
 705          */
 706         if (mode_flag == HFI1_PORT_SNOOP_MODE) {
 707                 /* clear after snoop mode is on */
 708                 adjust_integrity_checks(dd); /* clear */
 709
 710                 /*
 711                  * We also do not want to be doing the DLID LMC check for
 712                  * ingressed packets.
 713                  */
 714                 dd->hfi1_snoop.dcc_cfg = read_csr(dd, DCC_CFG_PORT_CONFIG1);
 715                 write_csr(dd, DCC_CFG_PORT_CONFIG1,
 716                           (dd->hfi1_snoop.dcc_cfg >> 32) << 32);
 717         }
 718
 719         /*
 720          * As soon as we set these function pointers the recv and send handlers
 721          * are active. This is a race condition so we must make sure to drain
 722          * the queue and init filter values above. Technically we should add
 723          * locking here but all that will happen is on recv a packet will get
 724          * allocated and get stuck on the snoop_lock before getting added to the
 725          * queue. Same goes for send.
 726          */
 727         dd->rhf_rcv_function_map = snoop_rhf_rcv_functions;
 728         dd->process_pio_send = snoop_send_pio_handler;
 729         dd->process_dma_send = snoop_send_pio_handler;
 730         dd->pio_inline_send = snoop_inline_pio_send;
 731
 732         spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
 733         ret = 0;
 734
 735 bail:
 736         mutex_unlock(&hfi1_mutex);
 737
 738         return ret;
 739 }
 740
 741 static int hfi1_snoop_release(struct inode *in, struct file *fp)
 742 {
 743         unsigned long flags = 0;
 744         struct hfi1_devdata *dd;
 745         int mode_flag;
 746
 747         dd = hfi1_dd_from_sc_inode(in);
 748         if (!dd)
 749                 return -ENODEV;
 750
 751         spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
 752
 753         /* clear the snoop mode before re-adjusting send context CSRs */
 754         mode_flag = dd->hfi1_snoop.mode_flag;
 755         dd->hfi1_snoop.mode_flag = 0;
 756
 757         /*
 758          * Drain the queue and clear the filters we are done with it. Don't
 759          * forget to restore the packet integrity checks
 760          */
 761         drain_snoop_list(&dd->hfi1_snoop.queue);
 762         if (mode_flag == HFI1_PORT_SNOOP_MODE) {
 763                 /* restore after snoop mode is clear */
 764                 adjust_integrity_checks(dd); /* restore */
 765
 766                 /*
 767                  * Also should probably reset the DCC_CONFIG1 register for DLID
 768                  * checking on incoming packets again. Use the value saved when
 769                  * opening the snoop device.
 770                  */
 771                 write_csr(dd, DCC_CFG_PORT_CONFIG1, dd->hfi1_snoop.dcc_cfg);
 772         }
 773
 774         dd->hfi1_snoop.filter_callback = NULL;
 775         kfree(dd->hfi1_snoop.filter_value);
 776         dd->hfi1_snoop.filter_value = NULL;
 777
 778         /*
 779          * User is done snooping and capturing, return control to the normal
 780          * handler. Re-enable SDMA handling.
 781          */
 782         dd->rhf_rcv_function_map = dd->normal_rhf_rcv_functions;
 783         dd->process_pio_send = hfi1_verbs_send_pio;
 784         dd->process_dma_send = hfi1_verbs_send_dma;
 785         dd->pio_inline_send = pio_copy;
 786
 787         spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
 788
 789         snoop_dbg("snoop/capture device released");
 790
 791         return 0;
 792 }
 793
 794 static unsigned int hfi1_snoop_poll(struct file *fp,
 795                                     struct poll_table_struct *wait)
 796 {
 797         int ret = 0;
 798         unsigned long flags = 0;
 799
 800         struct hfi1_devdata *dd;
 801
 802         dd = hfi1_dd_from_sc_inode(fp->f_inode);
 803         if (!dd)
 804                 return -ENODEV;
 805
 806         spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
 807
 808         poll_wait(fp, &dd->hfi1_snoop.waitq, wait);
 809         if (!list_empty(&dd->hfi1_snoop.queue))
 810                 ret |= POLLIN | POLLRDNORM;
 811
 812         spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
 813         return ret;
 814 }
 815
 816 static ssize_t hfi1_snoop_write(struct file *fp, const char __user *data,
 817                                 size_t count, loff_t *off)
 818 {
 819         struct diag_pkt dpkt;
 820         struct hfi1_devdata *dd;
 821         size_t ret;
 822         u8 byte_two, sl, sc5, sc4, vl, byte_one;
 823         struct send_context *sc;
 824         u32 len;
 825         u64 pbc;
 826         struct hfi1_ibport *ibp;
 827         struct hfi1_pportdata *ppd;
 828
 829         dd = hfi1_dd_from_sc_inode(fp->f_inode);
 830         if (!dd)
 831                 return -ENODEV;
 832
 833         ppd = dd->pport;
 834         snoop_dbg("received %lu bytes from user", count);
 835
 836         memset(&dpkt, 0, sizeof(struct diag_pkt));
 837         dpkt.version = _DIAG_PKT_VERS;
 838         dpkt.unit = dd->unit;
 839         dpkt.port = 1;
 840
 841         if (likely(!(snoop_flags & SNOOP_USE_METADATA))) {
 842                 /*
 843                 * We need to generate the PBC and not let diagpkt_send do it,
 844                 * to do this we need the VL and the length in dwords.
 845                 * The VL can be determined by using the SL and looking up the
 846                 * SC. Then the SC can be converted into VL. The exception to
 847                 * this is those packets which are from an SMI queue pair.
 848                 * Since we can't detect anything about the QP here we have to
 849                 * rely on the SC. If its 0xF then we assume its SMI and
 850                 * do not look at the SL.
 851                 */
 852                 if (copy_from_user(&byte_one, data, 1))
 853                         return -EINVAL;
 854
 855                 if (copy_from_user(&byte_two, data + 1, 1))
 856                         return -EINVAL;
 857
 858                 sc4 = (byte_one >> 4) & 0xf;
 859                 if (sc4 == 0xF) {
 860                         snoop_dbg("Detected VL15 packet ignoring SL in packet");
 861                         vl = sc4;
 862                 } else {
 863                         sl = (byte_two >> 4) & 0xf;
 864                         ibp = to_iport(&dd->verbs_dev.rdi.ibdev, 1);
 865                         sc5 = ibp->sl_to_sc[sl];
 866                         vl = sc_to_vlt(dd, sc5);
 867                         if (vl != sc4) {
 868                                 snoop_dbg("VL %d does not match SC %d of packet",
 869                                           vl, sc4);
 870                                 return -EINVAL;
 871                         }
 872                 }
 873
 874                 sc = dd->vld[vl].sc; /* Look up the context based on VL */
 875                 if (sc) {
 876                         dpkt.sw_index = sc->sw_index;
 877                         snoop_dbg("Sending on context %u(%u)", sc->sw_index,
 878                                   sc->hw_context);
 879                 } else {
 880                         snoop_dbg("Could not find context for vl %d", vl);
 881                         return -EINVAL;
 882                 }
 883
 884                 len = (count >> 2) + 2; /* Add in PBC */
 885                 pbc = create_pbc(ppd, 0, 0, vl, len);
 886         } else {
 887                 if (copy_from_user(&pbc, data, sizeof(pbc)))
 888                         return -EINVAL;
 889                 vl = (pbc >> PBC_VL_SHIFT) & PBC_VL_MASK;
 890                 sc = dd->vld[vl].sc; /* Look up the context based on VL */
 891                 if (sc) {
 892                         dpkt.sw_index = sc->sw_index;
 893                 } else {
 894                         snoop_dbg("Could not find context for vl %d", vl);
 895                         return -EINVAL;
 896                 }
 897                 data += sizeof(pbc);
 898                 count -= sizeof(pbc);
 899         }
 900         dpkt.len = count;
 901         dpkt.data = (unsigned long)data;
 902
 903         snoop_dbg("PBC: vl=0x%llx Length=0x%llx",
 904                   (pbc >> 12) & 0xf,
 905                   (pbc & 0xfff));
 906
 907         dpkt.pbc = pbc;
 908         ret = diagpkt_send(&dpkt);
 909         /*
 910          * diagpkt_send only returns number of bytes in the diagpkt so patch
 911          * that up here before returning.
 912          */
 913         if (ret == sizeof(dpkt))
 914                 return count;
 915
 916         return ret;
 917 }
 918
 919 static ssize_t hfi1_snoop_read(struct file *fp, char __user *data,
 920                                size_t pkt_len, loff_t *off)
 921 {
 922         ssize_t ret = 0;
 923         unsigned long flags = 0;
 924         struct snoop_packet *packet = NULL;
 925         struct hfi1_devdata *dd;
 926
 927         dd = hfi1_dd_from_sc_inode(fp->f_inode);
 928         if (!dd)
 929                 return -ENODEV;
 930
 931         spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
 932
 933         while (list_empty(&dd->hfi1_snoop.queue)) {
 934                 spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
 935
 936                 if (fp->f_flags & O_NONBLOCK)
 937                         return -EAGAIN;
 938
 939                 if (wait_event_interruptible(
 940                                 dd->hfi1_snoop.waitq,
 941                                 !list_empty(&dd->hfi1_snoop.queue)))
 942                         return -EINTR;
 943
 944                 spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
 945         }
 946
 947         if (!list_empty(&dd->hfi1_snoop.queue)) {
 948                 packet = list_entry(dd->hfi1_snoop.queue.next,
 949                                     struct snoop_packet, list);
 950                 list_del(&packet->list);
 951                 spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
 952                 if (pkt_len >= packet->total_len) {
 953                         if (copy_to_user(data, packet->data,
 954                                          packet->total_len))
 955                                 ret = -EFAULT;
 956                         else
 957                                 ret = packet->total_len;
 958                 } else {
 959                         ret = -EINVAL;
 960                 }
 961
 962                 kfree(packet);
 963         } else {
 964                 spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
 965         }
 966
 967         return ret;
 968 }
 969
 970 /**
 971  * hfi1_assign_snoop_link_credits -- Set up credits for VL15 and others
 972  * @ppd : ptr to hfi1 port data
 973  * @value : options from user space
 974  *
 975  * Assumes the rest of the CM credit registers are zero from a
 976  * previous global or credit reset.
 977  * Leave shared count at zero for both global and all vls.
 978  * In snoop mode ideally we don't use shared credits
 979  * Reserve 8.5k for VL15
 980  * If total credits less than 8.5kbytes return error.
 981  * Divide the rest of the credits across VL0 to VL7 and if
 982  * each of these levels has less than 34 credits (at least 2048 + 128 bytes)
 983  * return with an error.
 984  * The credit registers will be reset to zero on link negotiation or link up
 985  * so this function should be activated from user space only if the port has
 986  * gone past link negotiation and link up.
 987  *
 988  * Return -- 0 if successful else error condition
 989  *
 990  */
 991 static long hfi1_assign_snoop_link_credits(struct hfi1_pportdata *ppd,
 992                                            int value)
 993 {
 994 #define  OPA_MIN_PER_VL_CREDITS  34  /* 2048 + 128 bytes */
 995         struct buffer_control t;
 996         int i;
 997         struct hfi1_devdata *dd = ppd->dd;
 998         u16  total_credits = (value >> 16) & 0xffff;
 999         u16  vl15_credits = dd->vl15_init / 2;
1000         u16  per_vl_credits;
1001         __be16 be_per_vl_credits;
1002
1003         if (!(ppd->host_link_state & HLS_UP))
1004                 goto err_exit;
1005         if (total_credits  <  vl15_credits)
1006                 goto err_exit;
1007
1008         per_vl_credits = (total_credits - vl15_credits) / TXE_NUM_DATA_VL;
1009
1010         if (per_vl_credits < OPA_MIN_PER_VL_CREDITS)
1011                 goto err_exit;
1012
1013         memset(&t, 0, sizeof(t));
1014         be_per_vl_credits = cpu_to_be16(per_vl_credits);
1015
1016         for (i = 0; i < TXE_NUM_DATA_VL; i++)
1017                 t.vl[i].dedicated = be_per_vl_credits;
1018
1019         t.vl[15].dedicated  = cpu_to_be16(vl15_credits);
1020         return set_buffer_control(ppd, &t);
1021
1022 err_exit:
1023         snoop_dbg("port_state = 0x%x, total_credits = %d, vl15_credits = %d",
1024                   ppd->host_link_state, total_credits, vl15_credits);
1025
1026         return -EINVAL;
1027 }
1028
1029 static long hfi1_ioctl(struct file *fp, unsigned int cmd, unsigned long arg)
1030 {
1031         struct hfi1_devdata *dd;
1032         void *filter_value = NULL;
1033         long ret = 0;
1034         int value = 0;
1035         u8 phys_state = 0;
1036         u8 link_state = 0;
1037         u16 dev_state = 0;
1038         unsigned long flags = 0;
1039         unsigned long *argp = NULL;
1040         struct hfi1_packet_filter_command filter_cmd = {0};
1041         int mode_flag = 0;
1042         struct hfi1_pportdata *ppd = NULL;
1043         unsigned int index;
1044         struct hfi1_link_info link_info;
1045         int read_cmd, write_cmd, read_ok, write_ok;
1046
1047         dd = hfi1_dd_from_sc_inode(fp->f_inode);
1048         if (!dd)
1049                 return -ENODEV;
1050
1051         mode_flag = dd->hfi1_snoop.mode_flag;
1052         read_cmd = _IOC_DIR(cmd) & _IOC_READ;
1053         write_cmd = _IOC_DIR(cmd) & _IOC_WRITE;
1054         write_ok = access_ok(VERIFY_WRITE, (void __user *)arg, _IOC_SIZE(cmd));
1055         read_ok = access_ok(VERIFY_READ, (void __user *)arg, _IOC_SIZE(cmd));
1056
1057         if ((read_cmd && !write_ok) || (write_cmd && !read_ok))
1058                 return -EFAULT;
1059
1060         if (!capable(CAP_SYS_ADMIN))
1061                 return -EPERM;
1062
1063         if ((mode_flag & HFI1_PORT_CAPTURE_MODE) &&
1064             (cmd != HFI1_SNOOP_IOCCLEARQUEUE) &&
1065             (cmd != HFI1_SNOOP_IOCCLEARFILTER) &&
1066             (cmd != HFI1_SNOOP_IOCSETFILTER))
1067                 /* Capture devices are allowed only 3 operations
1068                  * 1.Clear capture queue
1069                  * 2.Clear capture filter
1070                  * 3.Set capture filter
1071                  * Other are invalid.
1072                  */
1073                 return -EINVAL;
1074
1075         switch (cmd) {
1076         case HFI1_SNOOP_IOCSETLINKSTATE_EXTRA:
1077                 memset(&link_info, 0, sizeof(link_info));
1078
1079                 if (copy_from_user(&link_info,
1080                                    (struct hfi1_link_info __user *)arg,
1081                                    sizeof(link_info)))
1082                         return -EFAULT;
1083
1084                 value = link_info.port_state;
1085                 index = link_info.port_number;
1086                 if (index > dd->num_pports - 1)
1087                         return -EINVAL;
1088
1089                 ppd = &dd->pport[index];
1090                 if (!ppd)
1091                         return -EINVAL;
1092
1093                 /* What we want to transition to */
1094                 phys_state = (value >> 4) & 0xF;
1095                 link_state = value & 0xF;
1096                 snoop_dbg("Setting link state 0x%x", value);
1097
1098                 switch (link_state) {
1099                 case IB_PORT_NOP:
1100                         if (phys_state == 0)
1101                                 break;
1102                                 /* fall through */
1103                 case IB_PORT_DOWN:
1104                         switch (phys_state) {
1105                         case 0:
1106                                 dev_state = HLS_DN_DOWNDEF;
1107                                 break;
1108                         case 2:
1109                                 dev_state = HLS_DN_POLL;
1110                                 break;
1111                         case 3:
1112                                 dev_state = HLS_DN_DISABLE;
1113                                 break;
1114                         default:
1115                                 return -EINVAL;
1116                         }
1117                         ret = set_link_state(ppd, dev_state);
1118                         break;
1119                 case IB_PORT_ARMED:
1120                         ret = set_link_state(ppd, HLS_UP_ARMED);
1121                         if (!ret)
1122                                 send_idle_sma(dd, SMA_IDLE_ARM);
1123                         break;
1124                 case IB_PORT_ACTIVE:
1125                         ret = set_link_state(ppd, HLS_UP_ACTIVE);
1126                         if (!ret)
1127                                 send_idle_sma(dd, SMA_IDLE_ACTIVE);
1128                         break;
1129                 default:
1130                         return -EINVAL;
1131                 }
1132
1133                 if (ret)
1134                         break;
1135                 /* fall through */
1136         case HFI1_SNOOP_IOCGETLINKSTATE:
1137         case HFI1_SNOOP_IOCGETLINKSTATE_EXTRA:
1138                 if (cmd == HFI1_SNOOP_IOCGETLINKSTATE_EXTRA) {
1139                         memset(&link_info, 0, sizeof(link_info));
1140                         if (copy_from_user(&link_info,
1141                                            (struct hfi1_link_info __user *)arg,
1142                                            sizeof(link_info)))
1143                                 return -EFAULT;
1144                         index = link_info.port_number;
1145                 } else {
1146                         ret = __get_user(index, (int __user *)arg);
1147                         if (ret !=  0)
1148                                 break;
1149                 }
1150
1151                 if (index > dd->num_pports - 1)
1152                         return -EINVAL;
1153
1154                 ppd = &dd->pport[index];
1155                 if (!ppd)
1156                         return -EINVAL;
1157
1158                 value = hfi1_ibphys_portstate(ppd);
1159                 value <<= 4;
1160                 value |= driver_lstate(ppd);
1161
1162                 snoop_dbg("Link port | Link State: %d", value);
1163
1164                 if ((cmd == HFI1_SNOOP_IOCGETLINKSTATE_EXTRA) ||
1165                     (cmd == HFI1_SNOOP_IOCSETLINKSTATE_EXTRA)) {
1166                         link_info.port_state = value;
1167                         link_info.node_guid = cpu_to_be64(ppd->guid);
1168                         link_info.link_speed_active =
1169                                                 ppd->link_speed_active;
1170                         link_info.link_width_active =
1171                                                 ppd->link_width_active;
1172                         if (copy_to_user((struct hfi1_link_info __user *)arg,
1173                                          &link_info, sizeof(link_info)))
1174                                 return -EFAULT;
1175                 } else {
1176                         ret = __put_user(value, (int __user *)arg);
1177                 }
1178                 break;
1179
1180         case HFI1_SNOOP_IOCCLEARQUEUE:
1181                 snoop_dbg("Clearing snoop queue");
1182                 spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
1183                 drain_snoop_list(&dd->hfi1_snoop.queue);
1184                 spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
1185                 break;
1186
1187         case HFI1_SNOOP_IOCCLEARFILTER:
1188                 snoop_dbg("Clearing filter");
1189                 spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
1190                 if (dd->hfi1_snoop.filter_callback) {
1191                         /* Drain packets first */
1192                         drain_snoop_list(&dd->hfi1_snoop.queue);
1193                         dd->hfi1_snoop.filter_callback = NULL;
1194                 }
1195                 kfree(dd->hfi1_snoop.filter_value);
1196                 dd->hfi1_snoop.filter_value = NULL;
1197                 spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
1198                 break;
1199
1200         case HFI1_SNOOP_IOCSETFILTER:
1201                 snoop_dbg("Setting filter");
1202                 /* just copy command structure */
1203                 argp = (unsigned long *)arg;
1204                 if (copy_from_user(&filter_cmd, (void __user *)argp,
1205                                    sizeof(filter_cmd)))
1206                         return -EFAULT;
1207
1208                 if (filter_cmd.opcode >= HFI1_MAX_FILTERS) {
1209                         pr_alert("Invalid opcode in request\n");
1210                         return -EINVAL;
1211                 }
1212
1213                 snoop_dbg("Opcode %d Len %d Ptr %p",
1214                           filter_cmd.opcode, filter_cmd.length,
1215                           filter_cmd.value_ptr);
1216
1217                 filter_value = kcalloc(filter_cmd.length, sizeof(u8),
1218                                        GFP_KERNEL);
1219                 if (!filter_value)
1220                         return -ENOMEM;
1221
1222                 /* copy remaining data from userspace */
1223                 if (copy_from_user((u8 *)filter_value,
1224                                    (void __user *)filter_cmd.value_ptr,
1225                                    filter_cmd.length)) {
1226                         kfree(filter_value);
1227                         return -EFAULT;
1228                 }
1229                 /* Drain packets first */
1230                 spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
1231                 drain_snoop_list(&dd->hfi1_snoop.queue);
1232                 dd->hfi1_snoop.filter_callback =
1233                         hfi1_filters[filter_cmd.opcode].filter;
1234                 /* just in case we see back to back sets */
1235                 kfree(dd->hfi1_snoop.filter_value);
1236                 dd->hfi1_snoop.filter_value = filter_value;
1237                 spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
1238                 break;
1239         case HFI1_SNOOP_IOCGETVERSION:
1240                 value = SNOOP_CAPTURE_VERSION;
1241                 snoop_dbg("Getting version: %d", value);
1242                 ret = __put_user(value, (int __user *)arg);
1243                 break;
1244         case HFI1_SNOOP_IOCSET_OPTS:
1245                 snoop_flags = 0;
1246                 ret = __get_user(value, (int __user *)arg);
1247                 if (ret != 0)
1248                         break;
1249
1250                 snoop_dbg("Setting snoop option %d", value);
1251                 if (value & SNOOP_DROP_SEND)
1252                         snoop_flags |= SNOOP_DROP_SEND;
1253                 if (value & SNOOP_USE_METADATA)
1254                         snoop_flags |= SNOOP_USE_METADATA;
1255                 if (value & (SNOOP_SET_VL0TOVL15)) {
1256                         ppd = &dd->pport[0];  /* first port will do */
1257                         ret = hfi1_assign_snoop_link_credits(ppd, value);
1258                 }
1259                 break;
1260         default:
1261                 return -ENOTTY;
1262         }
1263
1264         return ret;
1265 }
1266
1267 static void snoop_list_add_tail(struct snoop_packet *packet,
1268                                 struct hfi1_devdata *dd)
1269 {
1270         unsigned long flags = 0;
1271
1272         spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
1273         if (likely((dd->hfi1_snoop.mode_flag & HFI1_PORT_SNOOP_MODE) ||
1274                    (dd->hfi1_snoop.mode_flag & HFI1_PORT_CAPTURE_MODE))) {
1275                 list_add_tail(&packet->list, &dd->hfi1_snoop.queue);
1276                 snoop_dbg("Added packet to list");
1277         }
1278
1279         /*
1280          * Technically we can could have closed the snoop device while waiting
1281          * on the above lock and it is gone now. The snoop mode_flag will
1282          * prevent us from adding the packet to the queue though.
1283          */
1284
1285         spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
1286         wake_up_interruptible(&dd->hfi1_snoop.waitq);
1287 }
1288
1289 static inline int hfi1_filter_check(void *val, const char *msg)
1290 {
1291         if (!val) {
1292                 snoop_dbg("Error invalid %s value for filter", msg);
1293                 return HFI1_FILTER_ERR;
1294         }
1295         return 0;
1296 }
1297
1298 static int hfi1_filter_lid(void *ibhdr, void *packet_data, void *value)
1299 {
1300         struct hfi1_ib_header *hdr;
1301         int ret;
1302
1303         ret = hfi1_filter_check(ibhdr, "header");
1304         if (ret)
1305                 return ret;
1306         ret = hfi1_filter_check(value, "user");
1307         if (ret)
1308                 return ret;
1309         hdr = (struct hfi1_ib_header *)ibhdr;
1310
1311         if (*((u16 *)value) == be16_to_cpu(hdr->lrh[3])) /* matches slid */
1312                 return HFI1_FILTER_HIT; /* matched */
1313
1314         return HFI1_FILTER_MISS; /* Not matched */
1315 }
1316
1317 static int hfi1_filter_dlid(void *ibhdr, void *packet_data, void *value)
1318 {
1319         struct hfi1_ib_header *hdr;
1320         int ret;
1321
1322         ret = hfi1_filter_check(ibhdr, "header");
1323         if (ret)
1324                 return ret;
1325         ret = hfi1_filter_check(value, "user");
1326         if (ret)
1327                 return ret;
1328
1329         hdr = (struct hfi1_ib_header *)ibhdr;
1330
1331         if (*((u16 *)value) == be16_to_cpu(hdr->lrh[1]))
1332                 return HFI1_FILTER_HIT;
1333
1334         return HFI1_FILTER_MISS;
1335 }
1336
1337 /* Not valid for outgoing packets, send handler passes null for data*/
1338 static int hfi1_filter_mad_mgmt_class(void *ibhdr, void *packet_data,
1339                                       void *value)
1340 {
1341         struct hfi1_ib_header *hdr;
1342         struct hfi1_other_headers *ohdr = NULL;
1343         struct ib_smp *smp = NULL;
1344         u32 qpn = 0;
1345         int ret;
1346
1347         ret = hfi1_filter_check(ibhdr, "header");
1348         if (ret)
1349                 return ret;
1350         ret = hfi1_filter_check(packet_data, "packet_data");
1351         if (ret)
1352                 return ret;
1353         ret = hfi1_filter_check(value, "user");
1354         if (ret)
1355                 return ret;
1356
1357         hdr = (struct hfi1_ib_header *)ibhdr;
1358
1359         /* Check for GRH */
1360         if ((be16_to_cpu(hdr->lrh[0]) & 3) == HFI1_LRH_BTH)
1361                 ohdr = &hdr->u.oth; /* LRH + BTH + DETH */
1362         else
1363                 ohdr = &hdr->u.l.oth; /* LRH + GRH + BTH + DETH */
1364
1365         qpn = be32_to_cpu(ohdr->bth[1]) & 0x00FFFFFF;
1366         if (qpn <= 1) {
1367                 smp = (struct ib_smp *)packet_data;
1368                 if (*((u8 *)value) == smp->mgmt_class)
1369                         return HFI1_FILTER_HIT;
1370                 else
1371                         return HFI1_FILTER_MISS;
1372         }
1373         return HFI1_FILTER_ERR;
1374 }
1375
1376 static int hfi1_filter_qp_number(void *ibhdr, void *packet_data, void *value)
1377 {
1378         struct hfi1_ib_header *hdr;
1379         struct hfi1_other_headers *ohdr = NULL;
1380         int ret;
1381
1382         ret = hfi1_filter_check(ibhdr, "header");
1383         if (ret)
1384                 return ret;
1385         ret = hfi1_filter_check(value, "user");
1386         if (ret)
1387                 return ret;
1388
1389         hdr = (struct hfi1_ib_header *)ibhdr;
1390
1391         /* Check for GRH */
1392         if ((be16_to_cpu(hdr->lrh[0]) & 3) == HFI1_LRH_BTH)
1393                 ohdr = &hdr->u.oth; /* LRH + BTH + DETH */
1394         else
1395                 ohdr = &hdr->u.l.oth; /* LRH + GRH + BTH + DETH */
1396         if (*((u32 *)value) == (be32_to_cpu(ohdr->bth[1]) & 0x00FFFFFF))
1397                 return HFI1_FILTER_HIT;
1398
1399         return HFI1_FILTER_MISS;
1400 }
1401
1402 static int hfi1_filter_ibpacket_type(void *ibhdr, void *packet_data,
1403                                      void *value)
1404 {
1405         u32 lnh = 0;
1406         u8 opcode = 0;
1407         struct hfi1_ib_header *hdr;
1408         struct hfi1_other_headers *ohdr = NULL;
1409         int ret;
1410
1411         ret = hfi1_filter_check(ibhdr, "header");
1412         if (ret)
1413                 return ret;
1414         ret = hfi1_filter_check(value, "user");
1415         if (ret)
1416                 return ret;
1417
1418         hdr = (struct hfi1_ib_header *)ibhdr;
1419
1420         lnh = (be16_to_cpu(hdr->lrh[0]) & 3);
1421
1422         if (lnh == HFI1_LRH_BTH)
1423                 ohdr = &hdr->u.oth;
1424         else if (lnh == HFI1_LRH_GRH)
1425                 ohdr = &hdr->u.l.oth;
1426         else
1427                 return HFI1_FILTER_ERR;
1428
1429         opcode = be32_to_cpu(ohdr->bth[0]) >> 24;
1430
1431         if (*((u8 *)value) == ((opcode >> 5) & 0x7))
1432                 return HFI1_FILTER_HIT;
1433
1434         return HFI1_FILTER_MISS;
1435 }
1436
1437 static int hfi1_filter_ib_service_level(void *ibhdr, void *packet_data,
1438                                         void *value)
1439 {
1440         struct hfi1_ib_header *hdr;
1441         int ret;
1442
1443         ret = hfi1_filter_check(ibhdr, "header");
1444         if (ret)
1445                 return ret;
1446         ret = hfi1_filter_check(value, "user");
1447         if (ret)
1448                 return ret;
1449
1450         hdr = (struct hfi1_ib_header *)ibhdr;
1451
1452         if ((*((u8 *)value)) == ((be16_to_cpu(hdr->lrh[0]) >> 4) & 0xF))
1453                 return HFI1_FILTER_HIT;
1454
1455         return HFI1_FILTER_MISS;
1456 }
1457
1458 static int hfi1_filter_ib_pkey(void *ibhdr, void *packet_data, void *value)
1459 {
1460         u32 lnh = 0;
1461         struct hfi1_ib_header *hdr;
1462         struct hfi1_other_headers *ohdr = NULL;
1463         int ret;
1464
1465         ret = hfi1_filter_check(ibhdr, "header");
1466         if (ret)
1467                 return ret;
1468         ret = hfi1_filter_check(value, "user");
1469         if (ret)
1470                 return ret;
1471
1472         hdr = (struct hfi1_ib_header *)ibhdr;
1473
1474         lnh = (be16_to_cpu(hdr->lrh[0]) & 3);
1475         if (lnh == HFI1_LRH_BTH)
1476                 ohdr = &hdr->u.oth;
1477         else if (lnh == HFI1_LRH_GRH)
1478                 ohdr = &hdr->u.l.oth;
1479         else
1480                 return HFI1_FILTER_ERR;
1481
1482         /* P_key is 16-bit entity, however top most bit indicates
1483          * type of membership. 0 for limited and 1 for Full.
1484          * Limited members cannot accept information from other
1485          * Limited members, but communication is allowed between
1486          * every other combination of membership.
1487          * Hence we'll omit comparing top-most bit while filtering
1488          */
1489
1490         if ((*(u16 *)value & 0x7FFF) ==
1491                 ((be32_to_cpu(ohdr->bth[0])) & 0x7FFF))
1492                 return HFI1_FILTER_HIT;
1493
1494         return HFI1_FILTER_MISS;
1495 }
1496
1497 /*
1498  * If packet_data is NULL then this is coming from one of the send functions.
1499  * Thus we know if its an ingressed or egressed packet.
1500  */
1501 static int hfi1_filter_direction(void *ibhdr, void *packet_data, void *value)
1502 {
1503         u8 user_dir = *(u8 *)value;
1504         int ret;
1505
1506         ret = hfi1_filter_check(value, "user");
1507         if (ret)
1508                 return ret;
1509
1510         if (packet_data) {
1511                 /* Incoming packet */
1512                 if (user_dir & HFI1_SNOOP_INGRESS)
1513                         return HFI1_FILTER_HIT;
1514         } else {
1515                 /* Outgoing packet */
1516                 if (user_dir & HFI1_SNOOP_EGRESS)
1517                         return HFI1_FILTER_HIT;
1518         }
1519
1520         return HFI1_FILTER_MISS;
1521 }
1522
1523 /*
1524  * Allocate a snoop packet. The structure that is stored in the ring buffer, not
1525  * to be confused with an hfi packet type.
1526  */
1527 static struct snoop_packet *allocate_snoop_packet(u32 hdr_len,
1528                                                   u32 data_len,
1529                                                   u32 md_len)
1530 {
1531         struct snoop_packet *packet;
1532
1533         packet = kzalloc(sizeof(*packet) + hdr_len + data_len
1534                          + md_len,
1535                          GFP_ATOMIC | __GFP_NOWARN);
1536         if (likely(packet))
1537                 INIT_LIST_HEAD(&packet->list);
1538
1539         return packet;
1540 }
1541
1542 /*
1543  * Instead of having snoop and capture code intermixed with the recv functions,
1544  * both the interrupt handler and hfi1_ib_rcv() we are going to hijack the call
1545  * and land in here for snoop/capture but if not enabled the call will go
1546  * through as before. This gives us a single point to constrain all of the snoop
1547  * snoop recv logic. There is nothing special that needs to happen for bypass
1548  * packets. This routine should not try to look into the packet. It just copied
1549  * it. There is no guarantee for filters when it comes to bypass packets as
1550  * there is no specific support. Bottom line is this routine does now even know
1551  * what a bypass packet is.
1552  */
1553 int snoop_recv_handler(struct hfi1_packet *packet)
1554 {
1555         struct hfi1_pportdata *ppd = packet->rcd->ppd;
1556         struct hfi1_ib_header *hdr = packet->hdr;
1557         int header_size = packet->hlen;
1558         void *data = packet->ebuf;
1559         u32 tlen = packet->tlen;
1560         struct snoop_packet *s_packet = NULL;
1561         int ret;
1562         int snoop_mode = 0;
1563         u32 md_len = 0;
1564         struct capture_md md;
1565
1566         snoop_dbg("PACKET IN: hdr size %d tlen %d data %p", header_size, tlen,
1567                   data);
1568
1569         trace_snoop_capture(ppd->dd, header_size, hdr, tlen - header_size,
1570                             data);
1571
1572         if (!ppd->dd->hfi1_snoop.filter_callback) {
1573                 snoop_dbg("filter not set");
1574                 ret = HFI1_FILTER_HIT;
1575         } else {
1576                 ret = ppd->dd->hfi1_snoop.filter_callback(hdr, data,
1577                                         ppd->dd->hfi1_snoop.filter_value);
1578         }
1579
1580         switch (ret) {
1581         case HFI1_FILTER_ERR:
1582                 snoop_dbg("Error in filter call");
1583                 break;
1584         case HFI1_FILTER_MISS:
1585                 snoop_dbg("Filter Miss");
1586                 break;
1587         case HFI1_FILTER_HIT:
1588
1589                 if (ppd->dd->hfi1_snoop.mode_flag & HFI1_PORT_SNOOP_MODE)
1590                         snoop_mode = 1;
1591                 if ((snoop_mode == 0) ||
1592                     unlikely(snoop_flags & SNOOP_USE_METADATA))
1593                         md_len = sizeof(struct capture_md);
1594
1595                 s_packet = allocate_snoop_packet(header_size,
1596                                                  tlen - header_size,
1597                                                  md_len);
1598
1599                 if (unlikely(!s_packet)) {
1600                         dd_dev_warn_ratelimited(ppd->dd, "Unable to allocate snoop/capture packet\n");
1601                         break;
1602                 }
1603
1604                 if (md_len > 0) {
1605                         memset(&md, 0, sizeof(struct capture_md));
1606                         md.port = 1;
1607                         md.dir = PKT_DIR_INGRESS;
1608                         md.u.rhf = packet->rhf;
1609                         memcpy(s_packet->data, &md, md_len);
1610                 }
1611
1612                 /* We should always have a header */
1613                 if (hdr) {
1614                         memcpy(s_packet->data + md_len, hdr, header_size);
1615                 } else {
1616                         dd_dev_err(ppd->dd, "Unable to copy header to snoop/capture packet\n");
1617                         kfree(s_packet);
1618                         break;
1619                 }
1620
1621                 /*
1622                  * Packets with no data are possible. If there is no data needed
1623                  * to take care of the last 4 bytes which are normally included
1624                  * with data buffers and are included in tlen.  Since we kzalloc
1625                  * the buffer we do not need to set any values but if we decide
1626                  * not to use kzalloc we should zero them.
1627                  */
1628                 if (data)
1629                         memcpy(s_packet->data + header_size + md_len, data,
1630                                tlen - header_size);
1631
1632                 s_packet->total_len = tlen + md_len;
1633                 snoop_list_add_tail(s_packet, ppd->dd);
1634
1635                 /*
1636                  * If we are snooping the packet not capturing then throw away
1637                  * after adding to the list.
1638                  */
1639                 snoop_dbg("Capturing packet");
1640                 if (ppd->dd->hfi1_snoop.mode_flag & HFI1_PORT_SNOOP_MODE) {
1641                         snoop_dbg("Throwing packet away");
1642                         /*
1643                          * If we are dropping the packet we still may need to
1644                          * handle the case where error flags are set, this is
1645                          * normally done by the type specific handler but that
1646                          * won't be called in this case.
1647                          */
1648                         if (unlikely(rhf_err_flags(packet->rhf)))
1649                                 handle_eflags(packet);
1650
1651                         /* throw the packet on the floor */
1652                         return RHF_RCV_CONTINUE;
1653                 }
1654                 break;
1655         default:
1656                 break;
1657         }
1658
1659         /*
1660          * We do not care what type of packet came in here - just pass it off
1661          * to the normal handler.
1662          */
1663         return ppd->dd->normal_rhf_rcv_functions[rhf_rcv_type(packet->rhf)]
1664                         (packet);
1665 }
1666
1667 /*
1668  * Handle snooping and capturing packets when sdma is being used.
1669  */
1670 int snoop_send_dma_handler(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
1671                            u64 pbc)
1672 {
1673         pr_alert("Snooping/Capture of Send DMA Packets Is Not Supported!\n");
1674         snoop_dbg("Unsupported Operation");
1675         return hfi1_verbs_send_dma(qp, ps, 0);
1676 }
1677
1678 /*
1679  * Handle snooping and capturing packets when pio is being used. Does not handle
1680  * bypass packets. The only way to send a bypass packet currently is to use the
1681  * diagpkt interface. When that interface is enable snoop/capture is not.
1682  */
1683 int snoop_send_pio_handler(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
1684                            u64 pbc)
1685 {
1686         u32 hdrwords = qp->s_hdrwords;
1687         struct rvt_sge_state *ss = qp->s_cur_sge;
1688         u32 len = qp->s_cur_size;
1689         u32 dwords = (len + 3) >> 2;
1690         u32 plen = hdrwords + dwords + 2; /* includes pbc */
1691         struct hfi1_pportdata *ppd = ps->ppd;
1692         struct snoop_packet *s_packet = NULL;
1693         u32 *hdr = (u32 *)&ps->s_txreq->phdr.hdr;
1694         u32 length = 0;
1695         struct rvt_sge_state temp_ss;
1696         void *data = NULL;
1697         void *data_start = NULL;
1698         int ret;
1699         int snoop_mode = 0;
1700         int md_len = 0;
1701         struct capture_md md;
1702         u32 vl;
1703         u32 hdr_len = hdrwords << 2;
1704         u32 tlen = HFI1_GET_PKT_LEN(&ps->s_txreq->phdr.hdr);
1705
1706         md.u.pbc = 0;
1707
1708         snoop_dbg("PACKET OUT: hdrword %u len %u plen %u dwords %u tlen %u",
1709                   hdrwords, len, plen, dwords, tlen);
1710         if (ppd->dd->hfi1_snoop.mode_flag & HFI1_PORT_SNOOP_MODE)
1711                 snoop_mode = 1;
1712         if ((snoop_mode == 0) ||
1713             unlikely(snoop_flags & SNOOP_USE_METADATA))
1714                 md_len = sizeof(struct capture_md);
1715
1716         /* not using ss->total_len as arg 2 b/c that does not count CRC */
1717         s_packet = allocate_snoop_packet(hdr_len, tlen - hdr_len, md_len);
1718
1719         if (unlikely(!s_packet)) {
1720                 dd_dev_warn_ratelimited(ppd->dd, "Unable to allocate snoop/capture packet\n");
1721                 goto out;
1722         }
1723
1724         s_packet->total_len = tlen + md_len;
1725
1726         if (md_len > 0) {
1727                 memset(&md, 0, sizeof(struct capture_md));
1728                 md.port = 1;
1729                 md.dir = PKT_DIR_EGRESS;
1730                 if (likely(pbc == 0)) {
1731                         vl = be16_to_cpu(ps->s_txreq->phdr.hdr.lrh[0]) >> 12;
1732                         md.u.pbc = create_pbc(ppd, 0, qp->s_srate, vl, plen);
1733                 } else {
1734                         md.u.pbc = 0;
1735                 }
1736                 memcpy(s_packet->data, &md, md_len);
1737         } else {
1738                 md.u.pbc = pbc;
1739         }
1740
1741         /* Copy header */
1742         if (likely(hdr)) {
1743                 memcpy(s_packet->data + md_len, hdr, hdr_len);
1744         } else {
1745                 dd_dev_err(ppd->dd,
1746                            "Unable to copy header to snoop/capture packet\n");
1747                 kfree(s_packet);
1748                 goto out;
1749         }
1750
1751         if (ss) {
1752                 data = s_packet->data + hdr_len + md_len;
1753                 data_start = data;
1754
1755                 /*
1756                  * Copy SGE State
1757                  * The update_sge() function below will not modify the
1758                  * individual SGEs in the array. It will make a copy each time
1759                  * and operate on that. So we only need to copy this instance
1760                  * and it won't impact PIO.
1761                  */
1762                 temp_ss = *ss;
1763                 length = len;
1764
1765                 snoop_dbg("Need to copy %d bytes", length);
1766                 while (length) {
1767                         void *addr = temp_ss.sge.vaddr;
1768                         u32 slen = temp_ss.sge.length;
1769
1770                         if (slen > length) {
1771                                 slen = length;
1772                                 snoop_dbg("slen %d > len %d", slen, length);
1773                         }
1774                         snoop_dbg("copy %d to %p", slen, addr);
1775                         memcpy(data, addr, slen);
1776                         update_sge(&temp_ss, slen);
1777                         length -= slen;
1778                         data += slen;
1779                         snoop_dbg("data is now %p bytes left %d", data, length);
1780                 }
1781                 snoop_dbg("Completed SGE copy");
1782         }
1783
1784         /*
1785          * Why do the filter check down here? Because the event tracing has its
1786          * own filtering and we need to have the walked the SGE list.
1787          */
1788         if (!ppd->dd->hfi1_snoop.filter_callback) {
1789                 snoop_dbg("filter not set\n");
1790                 ret = HFI1_FILTER_HIT;
1791         } else {
1792                 ret = ppd->dd->hfi1_snoop.filter_callback(
1793                                         &ps->s_txreq->phdr.hdr,
1794                                         NULL,
1795                                         ppd->dd->hfi1_snoop.filter_value);
1796         }
1797
1798         switch (ret) {
1799         case HFI1_FILTER_ERR:
1800                 snoop_dbg("Error in filter call");
1801                 /* fall through */
1802         case HFI1_FILTER_MISS:
1803                 snoop_dbg("Filter Miss");
1804                 kfree(s_packet);
1805                 break;
1806         case HFI1_FILTER_HIT:
1807                 snoop_dbg("Capturing packet");
1808                 snoop_list_add_tail(s_packet, ppd->dd);
1809
1810                 if (unlikely((snoop_flags & SNOOP_DROP_SEND) &&
1811                              (ppd->dd->hfi1_snoop.mode_flag &
1812                               HFI1_PORT_SNOOP_MODE))) {
1813                         unsigned long flags;
1814
1815                         snoop_dbg("Dropping packet");
1816                         if (qp->s_wqe) {
1817                                 spin_lock_irqsave(&qp->s_lock, flags);
1818                                 hfi1_send_complete(
1819                                         qp,
1820                                         qp->s_wqe,
1821                                         IB_WC_SUCCESS);
1822                                 spin_unlock_irqrestore(&qp->s_lock, flags);
1823                         } else if (qp->ibqp.qp_type == IB_QPT_RC) {
1824                                 spin_lock_irqsave(&qp->s_lock, flags);
1825                                 hfi1_rc_send_complete(qp,
1826                                                       &ps->s_txreq->phdr.hdr);
1827                                 spin_unlock_irqrestore(&qp->s_lock, flags);
1828                         }
1829
1830                         /*
1831                          * If snoop is dropping the packet we need to put the
1832                          * txreq back because no one else will.
1833                          */
1834                         hfi1_put_txreq(ps->s_txreq);
1835                         return 0;
1836                 }
1837                 break;
1838         default:
1839                 kfree(s_packet);
1840                 break;
1841         }
1842 out:
1843         return hfi1_verbs_send_pio(qp, ps, md.u.pbc);
1844 }
1845
1846 /*
1847  * Callers of this must pass a hfi1_ib_header type for the from ptr. Currently
1848  * this can be used anywhere, but the intention is for inline ACKs for RC and
1849  * CCA packets. We don't restrict this usage though.
1850  */
1851 void snoop_inline_pio_send(struct hfi1_devdata *dd, struct pio_buf *pbuf,
1852                            u64 pbc, const void *from, size_t count)
1853 {
1854         int snoop_mode = 0;
1855         int md_len = 0;
1856         struct capture_md md;
1857         struct snoop_packet *s_packet = NULL;
1858
1859         /*
1860          * count is in dwords so we need to convert to bytes.
1861          * We also need to account for CRC which would be tacked on by hardware.
1862          */
1863         int packet_len = (count << 2) + 4;
1864         int ret;
1865
1866         snoop_dbg("ACK OUT: len %d", packet_len);
1867
1868         if (!dd->hfi1_snoop.filter_callback) {
1869                 snoop_dbg("filter not set");
1870                 ret = HFI1_FILTER_HIT;
1871         } else {
1872                 ret = dd->hfi1_snoop.filter_callback(
1873                                 (struct hfi1_ib_header *)from,
1874                                 NULL,
1875                                 dd->hfi1_snoop.filter_value);
1876         }
1877
1878         switch (ret) {
1879         case HFI1_FILTER_ERR:
1880                 snoop_dbg("Error in filter call");
1881                 /* fall through */
1882         case HFI1_FILTER_MISS:
1883                 snoop_dbg("Filter Miss");
1884                 break;
1885         case HFI1_FILTER_HIT:
1886                 snoop_dbg("Capturing packet");
1887                 if (dd->hfi1_snoop.mode_flag & HFI1_PORT_SNOOP_MODE)
1888                         snoop_mode = 1;
1889                 if ((snoop_mode == 0) ||
1890                     unlikely(snoop_flags & SNOOP_USE_METADATA))
1891                         md_len = sizeof(struct capture_md);
1892
1893                 s_packet = allocate_snoop_packet(packet_len, 0, md_len);
1894
1895                 if (unlikely(!s_packet)) {
1896                         dd_dev_warn_ratelimited(dd, "Unable to allocate snoop/capture packet\n");
1897                         goto inline_pio_out;
1898                 }
1899
1900                 s_packet->total_len = packet_len + md_len;
1901
1902                 /* Fill in the metadata for the packet */
1903                 if (md_len > 0) {
1904                         memset(&md, 0, sizeof(struct capture_md));
1905                         md.port = 1;
1906                         md.dir = PKT_DIR_EGRESS;
1907                         md.u.pbc = pbc;
1908                         memcpy(s_packet->data, &md, md_len);
1909                 }
1910
1911                 /* Add the packet data which is a single buffer */
1912                 memcpy(s_packet->data + md_len, from, packet_len);
1913
1914                 snoop_list_add_tail(s_packet, dd);
1915
1916                 if (unlikely((snoop_flags & SNOOP_DROP_SEND) && snoop_mode)) {
1917                         snoop_dbg("Dropping packet");
1918                         return;
1919                 }
1920                 break;
1921         default:
1922                 break;
1923         }
1924
1925 inline_pio_out:
1926         pio_copy(dd, pbuf, pbc, from, count);
1927 }