3 * This file is provided under a dual BSD/GPLv2 license. When using or
4 * redistributing this file, you may do so under either license.
8 * Copyright(c) 2015 Intel Corporation.
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of version 2 of the GNU General Public License as
12 * published by the Free Software Foundation.
14 * This program is distributed in the hope that it will be useful, but
15 * WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * General Public License for more details.
21 * Copyright(c) 2015 Intel Corporation.
23 * Redistribution and use in source and binary forms, with or without
24 * modification, are permitted provided that the following conditions
27 * - Redistributions of source code must retain the above copyright
28 * notice, this list of conditions and the following disclaimer.
29 * - Redistributions in binary form must reproduce the above copyright
30 * notice, this list of conditions and the following disclaimer in
31 * the documentation and/or other materials provided with the
33 * - Neither the name of Intel Corporation nor the names of its
34 * contributors may be used to endorse or promote products derived
35 * from this software without specific prior written permission.
37 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
38 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
39 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
40 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
41 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
42 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
43 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
44 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
45 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
46 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
47 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
52 * This file contains support for diagnostic functions. It is accessed by
53 * opening the hfi1_diag device, normally minor number 129. Diagnostic use
54 * of the chip may render the chip or board unusable until the driver
55 * is unloaded, or in some cases, until the system is rebooted.
57 * Accesses to the chip through this interface are not similar to going
58 * through the /sys/bus/pci resource mmap interface.
62 #include <linux/pci.h>
63 #include <linux/poll.h>
64 #include <linux/vmalloc.h>
65 #include <linux/export.h>
67 #include <linux/uaccess.h>
68 #include <linux/module.h>
69 #include <rdma/ib_smi.h>
76 #define pr_fmt(fmt) DRIVER_NAME ": " fmt
77 #define snoop_dbg(fmt, ...) \
78 hfi1_cdbg(SNOOP, fmt, ##__VA_ARGS__)
80 /* Snoop option mask */
81 #define SNOOP_DROP_SEND BIT(0)
82 #define SNOOP_USE_METADATA BIT(1)
84 static u8 snoop_flags;
87 * Extract packet length from LRH header.
88 * Why & 0x7FF? Because len is only 11 bits in case it wasn't 0'd we throw the
89 * bogus bits away. This is in Dwords so multiply by 4 to get size in bytes
91 #define HFI1_GET_PKT_LEN(x) (((be16_to_cpu((x)->lrh[2]) & 0x7FF)) << 2)
93 enum hfi1_filter_status {
99 /* snoop processing functions */
100 rhf_rcv_function_ptr snoop_rhf_rcv_functions[8] = {
101 [RHF_RCV_TYPE_EXPECTED] = snoop_recv_handler,
102 [RHF_RCV_TYPE_EAGER] = snoop_recv_handler,
103 [RHF_RCV_TYPE_IB] = snoop_recv_handler,
104 [RHF_RCV_TYPE_ERROR] = snoop_recv_handler,
105 [RHF_RCV_TYPE_BYPASS] = snoop_recv_handler,
106 [RHF_RCV_TYPE_INVALID5] = process_receive_invalid,
107 [RHF_RCV_TYPE_INVALID6] = process_receive_invalid,
108 [RHF_RCV_TYPE_INVALID7] = process_receive_invalid
111 /* Snoop packet structure */
112 struct snoop_packet {
113 struct list_head list;
118 /* Do not make these an enum or it will blow up the capture_md */
119 #define PKT_DIR_EGRESS 0x0
120 #define PKT_DIR_INGRESS 0x1
122 /* Packet capture metadata returned to the user with the packet. */
133 static atomic_t diagpkt_count = ATOMIC_INIT(0);
134 static struct cdev diagpkt_cdev;
135 static struct device *diagpkt_device;
137 static ssize_t diagpkt_write(struct file *fp, const char __user *data,
138 size_t count, loff_t *off);
140 static const struct file_operations diagpkt_file_ops = {
141 .owner = THIS_MODULE,
142 .write = diagpkt_write,
143 .llseek = noop_llseek,
147 * This is used for communication with user space for snoop extended IOCTLs
149 struct hfi1_link_info {
153 u16 link_speed_active;
154 u16 link_width_active;
158 * Add padding to make this a full IB SMP payload. Note: changing the
159 * size of this structure will make the IOCTLs created with _IOWR
161 * Be sure to run tests on all IOCTLs when making changes to this
168 * This starts our ioctl sequence numbers *way* off from the ones
169 * defined in ib_core.
171 #define SNOOP_CAPTURE_VERSION 0x1
173 #define IB_IOCTL_MAGIC 0x1b /* See Documentation/ioctl-number.txt */
174 #define HFI1_SNOOP_IOC_MAGIC IB_IOCTL_MAGIC
175 #define HFI1_SNOOP_IOC_BASE_SEQ 0x80
177 #define HFI1_SNOOP_IOCGETLINKSTATE \
178 _IO(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ)
179 #define HFI1_SNOOP_IOCSETLINKSTATE \
180 _IO(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ + 1)
181 #define HFI1_SNOOP_IOCCLEARQUEUE \
182 _IO(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ + 2)
183 #define HFI1_SNOOP_IOCCLEARFILTER \
184 _IO(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ + 3)
185 #define HFI1_SNOOP_IOCSETFILTER \
186 _IO(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ + 4)
187 #define HFI1_SNOOP_IOCGETVERSION \
188 _IO(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ + 5)
189 #define HFI1_SNOOP_IOCSET_OPTS \
190 _IO(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ + 6)
193 * These offsets +6/+7 could change, but these are already known and used
194 * IOCTL numbers so don't change them without a good reason.
196 #define HFI1_SNOOP_IOCGETLINKSTATE_EXTRA \
197 _IOWR(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ + 6, \
198 struct hfi1_link_info)
199 #define HFI1_SNOOP_IOCSETLINKSTATE_EXTRA \
200 _IOWR(HFI1_SNOOP_IOC_MAGIC, HFI1_SNOOP_IOC_BASE_SEQ + 7, \
201 struct hfi1_link_info)
203 static int hfi1_snoop_open(struct inode *in, struct file *fp);
204 static ssize_t hfi1_snoop_read(struct file *fp, char __user *data,
205 size_t pkt_len, loff_t *off);
206 static ssize_t hfi1_snoop_write(struct file *fp, const char __user *data,
207 size_t count, loff_t *off);
208 static long hfi1_ioctl(struct file *fp, unsigned int cmd, unsigned long arg);
209 static unsigned int hfi1_snoop_poll(struct file *fp,
210 struct poll_table_struct *wait);
211 static int hfi1_snoop_release(struct inode *in, struct file *fp);
213 struct hfi1_packet_filter_command {
219 /* Can't re-use PKT_DIR_*GRESS here because 0 means no packets for this */
220 #define HFI1_SNOOP_INGRESS 0x1
221 #define HFI1_SNOOP_EGRESS 0x2
223 enum hfi1_packet_filter_opcodes {
226 FILTER_BY_MAD_MGMT_CLASS,
229 FILTER_BY_SERVICE_LEVEL,
234 static const struct file_operations snoop_file_ops = {
235 .owner = THIS_MODULE,
236 .open = hfi1_snoop_open,
237 .read = hfi1_snoop_read,
238 .unlocked_ioctl = hfi1_ioctl,
239 .poll = hfi1_snoop_poll,
240 .write = hfi1_snoop_write,
241 .release = hfi1_snoop_release
244 struct hfi1_filter_array {
245 int (*filter)(void *, void *, void *);
248 static int hfi1_filter_lid(void *ibhdr, void *packet_data, void *value);
249 static int hfi1_filter_dlid(void *ibhdr, void *packet_data, void *value);
250 static int hfi1_filter_mad_mgmt_class(void *ibhdr, void *packet_data,
252 static int hfi1_filter_qp_number(void *ibhdr, void *packet_data, void *value);
253 static int hfi1_filter_ibpacket_type(void *ibhdr, void *packet_data,
255 static int hfi1_filter_ib_service_level(void *ibhdr, void *packet_data,
257 static int hfi1_filter_ib_pkey(void *ibhdr, void *packet_data, void *value);
258 static int hfi1_filter_direction(void *ibhdr, void *packet_data, void *value);
260 static struct hfi1_filter_array hfi1_filters[] = {
262 { hfi1_filter_dlid },
263 { hfi1_filter_mad_mgmt_class },
264 { hfi1_filter_qp_number },
265 { hfi1_filter_ibpacket_type },
266 { hfi1_filter_ib_service_level },
267 { hfi1_filter_ib_pkey },
268 { hfi1_filter_direction },
271 #define HFI1_MAX_FILTERS ARRAY_SIZE(hfi1_filters)
272 #define HFI1_DIAG_MINOR_BASE 129
274 static int hfi1_snoop_add(struct hfi1_devdata *dd, const char *name);
276 int hfi1_diag_add(struct hfi1_devdata *dd)
281 snprintf(name, sizeof(name), "%s_diagpkt%d", class_name(),
284 * Do this for each device as opposed to the normal diagpkt
285 * interface which is one per host
287 ret = hfi1_snoop_add(dd, name);
289 dd_dev_err(dd, "Unable to init snoop/capture device");
291 snprintf(name, sizeof(name), "%s_diagpkt", class_name());
292 if (atomic_inc_return(&diagpkt_count) == 1) {
293 ret = hfi1_cdev_init(HFI1_DIAGPKT_MINOR, name,
294 &diagpkt_file_ops, &diagpkt_cdev,
295 &diagpkt_device, false);
301 /* this must be called w/ dd->snoop_in_lock held */
302 static void drain_snoop_list(struct list_head *queue)
304 struct list_head *pos, *q;
305 struct snoop_packet *packet;
307 list_for_each_safe(pos, q, queue) {
308 packet = list_entry(pos, struct snoop_packet, list);
314 static void hfi1_snoop_remove(struct hfi1_devdata *dd)
316 unsigned long flags = 0;
318 spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
319 drain_snoop_list(&dd->hfi1_snoop.queue);
320 hfi1_cdev_cleanup(&dd->hfi1_snoop.cdev, &dd->hfi1_snoop.class_dev);
321 spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
324 void hfi1_diag_remove(struct hfi1_devdata *dd)
326 hfi1_snoop_remove(dd);
327 if (atomic_dec_and_test(&diagpkt_count))
328 hfi1_cdev_cleanup(&diagpkt_cdev, &diagpkt_device);
329 hfi1_cdev_cleanup(&dd->diag_cdev, &dd->diag_device);
333 * Allocated structure shared between the credit return mechanism and
336 struct diagpkt_wait {
337 struct completion credits_returned;
343 * When each side is finished with the structure, they call this.
344 * The last user frees the structure.
346 static void put_diagpkt_wait(struct diagpkt_wait *wait)
348 if (atomic_dec_and_test(&wait->count))
353 * Callback from the credit return code. Set the complete, which
354 * will let diapkt_send() continue.
356 static void diagpkt_complete(void *arg, int code)
358 struct diagpkt_wait *wait = (struct diagpkt_wait *)arg;
361 complete(&wait->credits_returned);
362 put_diagpkt_wait(wait); /* finished with the structure */
366 * diagpkt_send - send a packet
367 * @dp: diag packet descriptor
369 static ssize_t diagpkt_send(struct diag_pkt *dp)
371 struct hfi1_devdata *dd;
372 struct send_context *sc;
373 struct pio_buf *pbuf;
376 u32 pkt_len, total_len;
377 pio_release_cb credit_cb = NULL;
378 void *credit_arg = NULL;
379 struct diagpkt_wait *wait = NULL;
382 dd = hfi1_lookup(dp->unit);
383 if (!dd || !(dd->flags & HFI1_PRESENT) || !dd->kregbase) {
387 if (!(dd->flags & HFI1_INITTED)) {
388 /* no hardware, freeze, etc. */
393 if (dp->version != _DIAG_PKT_VERS) {
394 dd_dev_err(dd, "Invalid version %u for diagpkt_write\n",
400 /* send count must be an exact number of dwords */
406 /* there is only port 1 */
412 /* need a valid context */
413 if (dp->sw_index >= dd->num_send_contexts) {
417 /* can only use kernel contexts */
418 if (dd->send_contexts[dp->sw_index].type != SC_KERNEL) {
422 /* must be allocated */
423 sc = dd->send_contexts[dp->sw_index].sc;
428 /* must be enabled */
429 if (!(sc->flags & SCF_ENABLED)) {
434 /* allocate a buffer and copy the data in */
435 tmpbuf = vmalloc(dp->len);
441 if (copy_from_user(tmpbuf,
442 (const void __user *)(unsigned long)dp->data,
449 * pkt_len is how much data we have to write, includes header and data.
450 * total_len is length of the packet in Dwords plus the PBC should not
453 pkt_len = dp->len >> 2;
454 total_len = pkt_len + 2; /* PBC + packet */
456 /* if 0, fill in a default */
458 struct hfi1_pportdata *ppd = dd->pport;
460 hfi1_cdbg(PKT, "Generating PBC");
461 dp->pbc = create_pbc(ppd, 0, 0, 0, total_len);
463 hfi1_cdbg(PKT, "Using passed in PBC");
466 hfi1_cdbg(PKT, "Egress PBC content is 0x%llx", dp->pbc);
469 * The caller wants to wait until the packet is sent and to
470 * check for errors. The best we can do is wait until
471 * the buffer credits are returned and check if any packet
472 * error has occurred. If there are any late errors, this
473 * could miss it. If there are other senders who generate
474 * an error, this may find it. However, in general, it
477 if (dp->flags & F_DIAGPKT_WAIT) {
478 /* always force a credit return */
479 dp->pbc |= PBC_CREDIT_RETURN;
480 /* turn on credit return interrupts */
481 sc_add_credit_return_intr(sc);
482 wait = kmalloc(sizeof(*wait), GFP_KERNEL);
487 init_completion(&wait->credits_returned);
488 atomic_set(&wait->count, 2);
491 credit_cb = diagpkt_complete;
496 pbuf = sc_buffer_alloc(sc, total_len, credit_cb, credit_arg);
499 /* force a credit return and try again */
500 sc_return_credits(sc);
505 * No send buffer means no credit callback. Undo
506 * the wait set-up that was done above. We free wait
507 * because the callback will never be called.
509 if (dp->flags & F_DIAGPKT_WAIT) {
510 sc_del_credit_return_intr(sc);
518 pio_copy(dd, pbuf, dp->pbc, tmpbuf, pkt_len);
519 /* no flush needed as the HW knows the packet size */
523 if (dp->flags & F_DIAGPKT_WAIT) {
524 /* wait for credit return */
525 ret = wait_for_completion_interruptible(
526 &wait->credits_returned);
528 * If the wait returns an error, the wait was interrupted,
529 * e.g. with a ^C in the user program. The callback is
530 * still pending. This is OK as the wait structure is
531 * kmalloc'ed and the structure will free itself when
532 * all users are done with it.
534 * A context disable occurs on a send context restart, so
535 * include that in the list of errors below to check for.
536 * NOTE: PRC_FILL_ERR is at best informational and cannot
539 if (!ret && (((wait->code & PRC_STATUS_ERR) ||
540 (wait->code & PRC_FILL_ERR) ||
541 (wait->code & PRC_SC_DISABLE))))
544 put_diagpkt_wait(wait); /* finished with the structure */
545 sc_del_credit_return_intr(sc);
553 static ssize_t diagpkt_write(struct file *fp, const char __user *data,
554 size_t count, loff_t *off)
556 struct hfi1_devdata *dd;
557 struct send_context *sc;
562 if (count != sizeof(dp))
565 if (copy_from_user(&dp, data, sizeof(dp)))
569 * The Send Context is derived from the PbcVL value
570 * if PBC is populated
573 dd = hfi1_lookup(dp.unit);
576 vl = (dp.pbc >> PBC_VL_SHIFT) & PBC_VL_MASK;
579 dp.sw_index = sc->sw_index;
582 "Packet sent over VL %d via Send Context %u(%u)",
583 vl, sc->sw_index, sc->hw_context);
587 return diagpkt_send(&dp);
590 static int hfi1_snoop_add(struct hfi1_devdata *dd, const char *name)
594 dd->hfi1_snoop.mode_flag = 0;
595 spin_lock_init(&dd->hfi1_snoop.snoop_lock);
596 INIT_LIST_HEAD(&dd->hfi1_snoop.queue);
597 init_waitqueue_head(&dd->hfi1_snoop.waitq);
599 ret = hfi1_cdev_init(HFI1_SNOOP_CAPTURE_BASE + dd->unit, name,
601 &dd->hfi1_snoop.cdev, &dd->hfi1_snoop.class_dev,
605 dd_dev_err(dd, "Couldn't create %s device: %d", name, ret);
606 hfi1_cdev_cleanup(&dd->hfi1_snoop.cdev,
607 &dd->hfi1_snoop.class_dev);
613 static struct hfi1_devdata *hfi1_dd_from_sc_inode(struct inode *in)
615 int unit = iminor(in) - HFI1_SNOOP_CAPTURE_BASE;
616 struct hfi1_devdata *dd;
618 dd = hfi1_lookup(unit);
622 /* clear or restore send context integrity checks */
623 static void adjust_integrity_checks(struct hfi1_devdata *dd)
625 struct send_context *sc;
626 unsigned long sc_flags;
629 spin_lock_irqsave(&dd->sc_lock, sc_flags);
630 for (i = 0; i < dd->num_send_contexts; i++) {
633 sc = dd->send_contexts[i].sc;
636 continue; /* not allocated */
638 enable = likely(!HFI1_CAP_IS_KSET(NO_INTEGRITY)) &&
639 dd->hfi1_snoop.mode_flag != HFI1_PORT_SNOOP_MODE;
641 set_pio_integrity(sc);
643 if (enable) /* take HFI_CAP_* flags into account */
646 spin_unlock_irqrestore(&dd->sc_lock, sc_flags);
649 static int hfi1_snoop_open(struct inode *in, struct file *fp)
653 unsigned long flags = 0;
654 struct hfi1_devdata *dd;
655 struct list_head *queue;
657 mutex_lock(&hfi1_mutex);
659 dd = hfi1_dd_from_sc_inode(in);
666 * File mode determines snoop or capture. Some existing user
667 * applications expect the capture device to be able to be opened RDWR
668 * because they expect a dedicated capture device. For this reason we
669 * support a module param to force capture mode even if the file open
670 * mode matches snoop.
672 if ((fp->f_flags & O_ACCMODE) == O_RDONLY) {
673 snoop_dbg("Capture Enabled");
674 mode_flag = HFI1_PORT_CAPTURE_MODE;
675 } else if ((fp->f_flags & O_ACCMODE) == O_RDWR) {
676 snoop_dbg("Snoop Enabled");
677 mode_flag = HFI1_PORT_SNOOP_MODE;
679 snoop_dbg("Invalid");
683 queue = &dd->hfi1_snoop.queue;
686 * We are not supporting snoop and capture at the same time.
688 spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
689 if (dd->hfi1_snoop.mode_flag) {
691 spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
695 dd->hfi1_snoop.mode_flag = mode_flag;
696 drain_snoop_list(queue);
698 dd->hfi1_snoop.filter_callback = NULL;
699 dd->hfi1_snoop.filter_value = NULL;
702 * Send side packet integrity checks are not helpful when snooping so
703 * disable and re-enable when we stop snooping.
705 if (mode_flag == HFI1_PORT_SNOOP_MODE) {
706 /* clear after snoop mode is on */
707 adjust_integrity_checks(dd); /* clear */
710 * We also do not want to be doing the DLID LMC check for
713 dd->hfi1_snoop.dcc_cfg = read_csr(dd, DCC_CFG_PORT_CONFIG1);
714 write_csr(dd, DCC_CFG_PORT_CONFIG1,
715 (dd->hfi1_snoop.dcc_cfg >> 32) << 32);
719 * As soon as we set these function pointers the recv and send handlers
720 * are active. This is a race condition so we must make sure to drain
721 * the queue and init filter values above. Technically we should add
722 * locking here but all that will happen is on recv a packet will get
723 * allocated and get stuck on the snoop_lock before getting added to the
724 * queue. Same goes for send.
726 dd->rhf_rcv_function_map = snoop_rhf_rcv_functions;
727 dd->process_pio_send = snoop_send_pio_handler;
728 dd->process_dma_send = snoop_send_pio_handler;
729 dd->pio_inline_send = snoop_inline_pio_send;
731 spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
735 mutex_unlock(&hfi1_mutex);
740 static int hfi1_snoop_release(struct inode *in, struct file *fp)
742 unsigned long flags = 0;
743 struct hfi1_devdata *dd;
746 dd = hfi1_dd_from_sc_inode(in);
750 spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
752 /* clear the snoop mode before re-adjusting send context CSRs */
753 mode_flag = dd->hfi1_snoop.mode_flag;
754 dd->hfi1_snoop.mode_flag = 0;
757 * Drain the queue and clear the filters we are done with it. Don't
758 * forget to restore the packet integrity checks
760 drain_snoop_list(&dd->hfi1_snoop.queue);
761 if (mode_flag == HFI1_PORT_SNOOP_MODE) {
762 /* restore after snoop mode is clear */
763 adjust_integrity_checks(dd); /* restore */
766 * Also should probably reset the DCC_CONFIG1 register for DLID
767 * checking on incoming packets again. Use the value saved when
768 * opening the snoop device.
770 write_csr(dd, DCC_CFG_PORT_CONFIG1, dd->hfi1_snoop.dcc_cfg);
773 dd->hfi1_snoop.filter_callback = NULL;
774 kfree(dd->hfi1_snoop.filter_value);
775 dd->hfi1_snoop.filter_value = NULL;
778 * User is done snooping and capturing, return control to the normal
779 * handler. Re-enable SDMA handling.
781 dd->rhf_rcv_function_map = dd->normal_rhf_rcv_functions;
782 dd->process_pio_send = hfi1_verbs_send_pio;
783 dd->process_dma_send = hfi1_verbs_send_dma;
784 dd->pio_inline_send = pio_copy;
786 spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
788 snoop_dbg("snoop/capture device released");
793 static unsigned int hfi1_snoop_poll(struct file *fp,
794 struct poll_table_struct *wait)
797 unsigned long flags = 0;
799 struct hfi1_devdata *dd;
801 dd = hfi1_dd_from_sc_inode(fp->f_inode);
805 spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
807 poll_wait(fp, &dd->hfi1_snoop.waitq, wait);
808 if (!list_empty(&dd->hfi1_snoop.queue))
809 ret |= POLLIN | POLLRDNORM;
811 spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
815 static ssize_t hfi1_snoop_write(struct file *fp, const char __user *data,
816 size_t count, loff_t *off)
818 struct diag_pkt dpkt;
819 struct hfi1_devdata *dd;
821 u8 byte_two, sl, sc5, sc4, vl, byte_one;
822 struct send_context *sc;
825 struct hfi1_ibport *ibp;
826 struct hfi1_pportdata *ppd;
828 dd = hfi1_dd_from_sc_inode(fp->f_inode);
833 snoop_dbg("received %lu bytes from user", count);
835 memset(&dpkt, 0, sizeof(struct diag_pkt));
836 dpkt.version = _DIAG_PKT_VERS;
837 dpkt.unit = dd->unit;
840 if (likely(!(snoop_flags & SNOOP_USE_METADATA))) {
842 * We need to generate the PBC and not let diagpkt_send do it,
843 * to do this we need the VL and the length in dwords.
844 * The VL can be determined by using the SL and looking up the
845 * SC. Then the SC can be converted into VL. The exception to
846 * this is those packets which are from an SMI queue pair.
847 * Since we can't detect anything about the QP here we have to
848 * rely on the SC. If its 0xF then we assume its SMI and
849 * do not look at the SL.
851 if (copy_from_user(&byte_one, data, 1))
854 if (copy_from_user(&byte_two, data + 1, 1))
857 sc4 = (byte_one >> 4) & 0xf;
859 snoop_dbg("Detected VL15 packet ignoring SL in packet");
862 sl = (byte_two >> 4) & 0xf;
863 ibp = to_iport(&dd->verbs_dev.ibdev, 1);
864 sc5 = ibp->sl_to_sc[sl];
865 vl = sc_to_vlt(dd, sc5);
867 snoop_dbg("VL %d does not match SC %d of packet",
873 sc = dd->vld[vl].sc; /* Look up the context based on VL */
875 dpkt.sw_index = sc->sw_index;
876 snoop_dbg("Sending on context %u(%u)", sc->sw_index,
879 snoop_dbg("Could not find context for vl %d", vl);
883 len = (count >> 2) + 2; /* Add in PBC */
884 pbc = create_pbc(ppd, 0, 0, vl, len);
886 if (copy_from_user(&pbc, data, sizeof(pbc)))
888 vl = (pbc >> PBC_VL_SHIFT) & PBC_VL_MASK;
889 sc = dd->vld[vl].sc; /* Look up the context based on VL */
891 dpkt.sw_index = sc->sw_index;
893 snoop_dbg("Could not find context for vl %d", vl);
897 count -= sizeof(pbc);
900 dpkt.data = (unsigned long)data;
902 snoop_dbg("PBC: vl=0x%llx Length=0x%llx",
907 ret = diagpkt_send(&dpkt);
909 * diagpkt_send only returns number of bytes in the diagpkt so patch
910 * that up here before returning.
912 if (ret == sizeof(dpkt))
918 static ssize_t hfi1_snoop_read(struct file *fp, char __user *data,
919 size_t pkt_len, loff_t *off)
922 unsigned long flags = 0;
923 struct snoop_packet *packet = NULL;
924 struct hfi1_devdata *dd;
926 dd = hfi1_dd_from_sc_inode(fp->f_inode);
930 spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
932 while (list_empty(&dd->hfi1_snoop.queue)) {
933 spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
935 if (fp->f_flags & O_NONBLOCK)
938 if (wait_event_interruptible(
939 dd->hfi1_snoop.waitq,
940 !list_empty(&dd->hfi1_snoop.queue)))
943 spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
946 if (!list_empty(&dd->hfi1_snoop.queue)) {
947 packet = list_entry(dd->hfi1_snoop.queue.next,
948 struct snoop_packet, list);
949 list_del(&packet->list);
950 spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
951 if (pkt_len >= packet->total_len) {
952 if (copy_to_user(data, packet->data,
956 ret = packet->total_len;
963 spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
969 static long hfi1_ioctl(struct file *fp, unsigned int cmd, unsigned long arg)
971 struct hfi1_devdata *dd;
972 void *filter_value = NULL;
978 unsigned long flags = 0;
979 unsigned long *argp = NULL;
980 struct hfi1_packet_filter_command filter_cmd = {0};
982 struct hfi1_pportdata *ppd = NULL;
984 struct hfi1_link_info link_info;
986 dd = hfi1_dd_from_sc_inode(fp->f_inode);
990 mode_flag = dd->hfi1_snoop.mode_flag;
992 if (((_IOC_DIR(cmd) & _IOC_READ)
993 && !access_ok(VERIFY_WRITE, (void __user *)arg, _IOC_SIZE(cmd)))
994 || ((_IOC_DIR(cmd) & _IOC_WRITE)
995 && !access_ok(VERIFY_READ, (void __user *)arg, _IOC_SIZE(cmd)))) {
997 } else if (!capable(CAP_SYS_ADMIN)) {
999 } else if ((mode_flag & HFI1_PORT_CAPTURE_MODE) &&
1000 (cmd != HFI1_SNOOP_IOCCLEARQUEUE) &&
1001 (cmd != HFI1_SNOOP_IOCCLEARFILTER) &&
1002 (cmd != HFI1_SNOOP_IOCSETFILTER)) {
1003 /* Capture devices are allowed only 3 operations
1004 * 1.Clear capture queue
1005 * 2.Clear capture filter
1006 * 3.Set capture filter
1007 * Other are invalid.
1012 spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
1015 case HFI1_SNOOP_IOCSETLINKSTATE:
1016 snoop_dbg("HFI1_SNOOP_IOCSETLINKSTATE is not valid");
1020 case HFI1_SNOOP_IOCSETLINKSTATE_EXTRA:
1021 memset(&link_info, 0, sizeof(link_info));
1023 if (copy_from_user(&link_info,
1024 (struct hfi1_link_info __user *)arg,
1028 value = link_info.port_state;
1029 index = link_info.port_number;
1030 if (index > dd->num_pports - 1) {
1035 ppd = &dd->pport[index];
1041 /* What we want to transition to */
1042 phys_state = (value >> 4) & 0xF;
1043 link_state = value & 0xF;
1044 snoop_dbg("Setting link state 0x%x", value);
1046 switch (link_state) {
1048 if (phys_state == 0)
1052 switch (phys_state) {
1054 dev_state = HLS_DN_DOWNDEF;
1057 dev_state = HLS_DN_POLL;
1060 dev_state = HLS_DN_DISABLE;
1066 ret = set_link_state(ppd, dev_state);
1069 ret = set_link_state(ppd, HLS_UP_ARMED);
1071 send_idle_sma(dd, SMA_IDLE_ARM);
1073 case IB_PORT_ACTIVE:
1074 ret = set_link_state(ppd, HLS_UP_ACTIVE);
1076 send_idle_sma(dd, SMA_IDLE_ACTIVE);
1086 case HFI1_SNOOP_IOCGETLINKSTATE:
1087 case HFI1_SNOOP_IOCGETLINKSTATE_EXTRA:
1088 if (cmd == HFI1_SNOOP_IOCGETLINKSTATE_EXTRA) {
1089 memset(&link_info, 0, sizeof(link_info));
1090 if (copy_from_user(&link_info,
1091 (struct hfi1_link_info __user *)arg,
1094 index = link_info.port_number;
1096 ret = __get_user(index, (int __user *)arg);
1101 if (index > dd->num_pports - 1) {
1106 ppd = &dd->pport[index];
1111 value = hfi1_ibphys_portstate(ppd);
1113 value |= driver_lstate(ppd);
1115 snoop_dbg("Link port | Link State: %d", value);
1117 if ((cmd == HFI1_SNOOP_IOCGETLINKSTATE_EXTRA) ||
1118 (cmd == HFI1_SNOOP_IOCSETLINKSTATE_EXTRA)) {
1119 link_info.port_state = value;
1120 link_info.node_guid = cpu_to_be64(ppd->guid);
1121 link_info.link_speed_active =
1122 ppd->link_speed_active;
1123 link_info.link_width_active =
1124 ppd->link_width_active;
1125 if (copy_to_user((struct hfi1_link_info __user *)arg,
1126 &link_info, sizeof(link_info)))
1129 ret = __put_user(value, (int __user *)arg);
1133 case HFI1_SNOOP_IOCCLEARQUEUE:
1134 snoop_dbg("Clearing snoop queue");
1135 drain_snoop_list(&dd->hfi1_snoop.queue);
1138 case HFI1_SNOOP_IOCCLEARFILTER:
1139 snoop_dbg("Clearing filter");
1140 if (dd->hfi1_snoop.filter_callback) {
1141 /* Drain packets first */
1142 drain_snoop_list(&dd->hfi1_snoop.queue);
1143 dd->hfi1_snoop.filter_callback = NULL;
1145 kfree(dd->hfi1_snoop.filter_value);
1146 dd->hfi1_snoop.filter_value = NULL;
1149 case HFI1_SNOOP_IOCSETFILTER:
1150 snoop_dbg("Setting filter");
1151 /* just copy command structure */
1152 argp = (unsigned long *)arg;
1153 if (copy_from_user(&filter_cmd, (void __user *)argp,
1154 sizeof(filter_cmd))) {
1158 if (filter_cmd.opcode >= HFI1_MAX_FILTERS) {
1159 pr_alert("Invalid opcode in request\n");
1164 snoop_dbg("Opcode %d Len %d Ptr %p",
1165 filter_cmd.opcode, filter_cmd.length,
1166 filter_cmd.value_ptr);
1168 filter_value = kcalloc(filter_cmd.length, sizeof(u8),
1170 if (!filter_value) {
1174 /* copy remaining data from userspace */
1175 if (copy_from_user((u8 *)filter_value,
1176 (void __user *)filter_cmd.value_ptr,
1177 filter_cmd.length)) {
1178 kfree(filter_value);
1182 /* Drain packets first */
1183 drain_snoop_list(&dd->hfi1_snoop.queue);
1184 dd->hfi1_snoop.filter_callback =
1185 hfi1_filters[filter_cmd.opcode].filter;
1186 /* just in case we see back to back sets */
1187 kfree(dd->hfi1_snoop.filter_value);
1188 dd->hfi1_snoop.filter_value = filter_value;
1191 case HFI1_SNOOP_IOCGETVERSION:
1192 value = SNOOP_CAPTURE_VERSION;
1193 snoop_dbg("Getting version: %d", value);
1194 ret = __put_user(value, (int __user *)arg);
1196 case HFI1_SNOOP_IOCSET_OPTS:
1198 ret = __get_user(value, (int __user *)arg);
1202 snoop_dbg("Setting snoop option %d", value);
1203 if (value & SNOOP_DROP_SEND)
1204 snoop_flags |= SNOOP_DROP_SEND;
1205 if (value & SNOOP_USE_METADATA)
1206 snoop_flags |= SNOOP_USE_METADATA;
1213 spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
1217 static void snoop_list_add_tail(struct snoop_packet *packet,
1218 struct hfi1_devdata *dd)
1220 unsigned long flags = 0;
1222 spin_lock_irqsave(&dd->hfi1_snoop.snoop_lock, flags);
1223 if (likely((dd->hfi1_snoop.mode_flag & HFI1_PORT_SNOOP_MODE) ||
1224 (dd->hfi1_snoop.mode_flag & HFI1_PORT_CAPTURE_MODE))) {
1225 list_add_tail(&packet->list, &dd->hfi1_snoop.queue);
1226 snoop_dbg("Added packet to list");
1230 * Technically we can could have closed the snoop device while waiting
1231 * on the above lock and it is gone now. The snoop mode_flag will
1232 * prevent us from adding the packet to the queue though.
1235 spin_unlock_irqrestore(&dd->hfi1_snoop.snoop_lock, flags);
1236 wake_up_interruptible(&dd->hfi1_snoop.waitq);
1239 static inline int hfi1_filter_check(void *val, const char *msg)
1242 snoop_dbg("Error invalid %s value for filter", msg);
1243 return HFI1_FILTER_ERR;
1248 static int hfi1_filter_lid(void *ibhdr, void *packet_data, void *value)
1250 struct hfi1_ib_header *hdr;
1253 ret = hfi1_filter_check(ibhdr, "header");
1256 ret = hfi1_filter_check(value, "user");
1259 hdr = (struct hfi1_ib_header *)ibhdr;
1261 if (*((u16 *)value) == be16_to_cpu(hdr->lrh[3])) /* matches slid */
1262 return HFI1_FILTER_HIT; /* matched */
1264 return HFI1_FILTER_MISS; /* Not matched */
1267 static int hfi1_filter_dlid(void *ibhdr, void *packet_data, void *value)
1269 struct hfi1_ib_header *hdr;
1272 ret = hfi1_filter_check(ibhdr, "header");
1275 ret = hfi1_filter_check(value, "user");
1279 hdr = (struct hfi1_ib_header *)ibhdr;
1281 if (*((u16 *)value) == be16_to_cpu(hdr->lrh[1]))
1282 return HFI1_FILTER_HIT;
1284 return HFI1_FILTER_MISS;
1287 /* Not valid for outgoing packets, send handler passes null for data*/
1288 static int hfi1_filter_mad_mgmt_class(void *ibhdr, void *packet_data,
1291 struct hfi1_ib_header *hdr;
1292 struct hfi1_other_headers *ohdr = NULL;
1293 struct ib_smp *smp = NULL;
1297 ret = hfi1_filter_check(ibhdr, "header");
1300 ret = hfi1_filter_check(packet_data, "packet_data");
1303 ret = hfi1_filter_check(value, "user");
1307 hdr = (struct hfi1_ib_header *)ibhdr;
1310 if ((be16_to_cpu(hdr->lrh[0]) & 3) == HFI1_LRH_BTH)
1311 ohdr = &hdr->u.oth; /* LRH + BTH + DETH */
1313 ohdr = &hdr->u.l.oth; /* LRH + GRH + BTH + DETH */
1315 qpn = be32_to_cpu(ohdr->bth[1]) & 0x00FFFFFF;
1317 smp = (struct ib_smp *)packet_data;
1318 if (*((u8 *)value) == smp->mgmt_class)
1319 return HFI1_FILTER_HIT;
1321 return HFI1_FILTER_MISS;
1323 return HFI1_FILTER_ERR;
1326 static int hfi1_filter_qp_number(void *ibhdr, void *packet_data, void *value)
1328 struct hfi1_ib_header *hdr;
1329 struct hfi1_other_headers *ohdr = NULL;
1332 ret = hfi1_filter_check(ibhdr, "header");
1335 ret = hfi1_filter_check(value, "user");
1339 hdr = (struct hfi1_ib_header *)ibhdr;
1342 if ((be16_to_cpu(hdr->lrh[0]) & 3) == HFI1_LRH_BTH)
1343 ohdr = &hdr->u.oth; /* LRH + BTH + DETH */
1345 ohdr = &hdr->u.l.oth; /* LRH + GRH + BTH + DETH */
1346 if (*((u32 *)value) == (be32_to_cpu(ohdr->bth[1]) & 0x00FFFFFF))
1347 return HFI1_FILTER_HIT;
1349 return HFI1_FILTER_MISS;
1352 static int hfi1_filter_ibpacket_type(void *ibhdr, void *packet_data,
1357 struct hfi1_ib_header *hdr;
1358 struct hfi1_other_headers *ohdr = NULL;
1361 ret = hfi1_filter_check(ibhdr, "header");
1364 ret = hfi1_filter_check(value, "user");
1368 hdr = (struct hfi1_ib_header *)ibhdr;
1370 lnh = (be16_to_cpu(hdr->lrh[0]) & 3);
1372 if (lnh == HFI1_LRH_BTH)
1374 else if (lnh == HFI1_LRH_GRH)
1375 ohdr = &hdr->u.l.oth;
1377 return HFI1_FILTER_ERR;
1379 opcode = be32_to_cpu(ohdr->bth[0]) >> 24;
1381 if (*((u8 *)value) == ((opcode >> 5) & 0x7))
1382 return HFI1_FILTER_HIT;
1384 return HFI1_FILTER_MISS;
1387 static int hfi1_filter_ib_service_level(void *ibhdr, void *packet_data,
1390 struct hfi1_ib_header *hdr;
1393 ret = hfi1_filter_check(ibhdr, "header");
1396 ret = hfi1_filter_check(value, "user");
1400 hdr = (struct hfi1_ib_header *)ibhdr;
1402 if ((*((u8 *)value)) == ((be16_to_cpu(hdr->lrh[0]) >> 4) & 0xF))
1403 return HFI1_FILTER_HIT;
1405 return HFI1_FILTER_MISS;
1408 static int hfi1_filter_ib_pkey(void *ibhdr, void *packet_data, void *value)
1411 struct hfi1_ib_header *hdr;
1412 struct hfi1_other_headers *ohdr = NULL;
1415 ret = hfi1_filter_check(ibhdr, "header");
1418 ret = hfi1_filter_check(value, "user");
1422 hdr = (struct hfi1_ib_header *)ibhdr;
1424 lnh = (be16_to_cpu(hdr->lrh[0]) & 3);
1425 if (lnh == HFI1_LRH_BTH)
1427 else if (lnh == HFI1_LRH_GRH)
1428 ohdr = &hdr->u.l.oth;
1430 return HFI1_FILTER_ERR;
1432 /* P_key is 16-bit entity, however top most bit indicates
1433 * type of membership. 0 for limited and 1 for Full.
1434 * Limited members cannot accept information from other
1435 * Limited members, but communication is allowed between
1436 * every other combination of membership.
1437 * Hence we'll omit comparing top-most bit while filtering
1440 if ((*(u16 *)value & 0x7FFF) ==
1441 ((be32_to_cpu(ohdr->bth[0])) & 0x7FFF))
1442 return HFI1_FILTER_HIT;
1444 return HFI1_FILTER_MISS;
1448 * If packet_data is NULL then this is coming from one of the send functions.
1449 * Thus we know if its an ingressed or egressed packet.
1451 static int hfi1_filter_direction(void *ibhdr, void *packet_data, void *value)
1453 u8 user_dir = *(u8 *)value;
1456 ret = hfi1_filter_check(value, "user");
1461 /* Incoming packet */
1462 if (user_dir & HFI1_SNOOP_INGRESS)
1463 return HFI1_FILTER_HIT;
1465 /* Outgoing packet */
1466 if (user_dir & HFI1_SNOOP_EGRESS)
1467 return HFI1_FILTER_HIT;
1470 return HFI1_FILTER_MISS;
1474 * Allocate a snoop packet. The structure that is stored in the ring buffer, not
1475 * to be confused with an hfi packet type.
1477 static struct snoop_packet *allocate_snoop_packet(u32 hdr_len,
1481 struct snoop_packet *packet;
1483 packet = kzalloc(sizeof(*packet) + hdr_len + data_len
1485 GFP_ATOMIC | __GFP_NOWARN);
1487 INIT_LIST_HEAD(&packet->list);
1493 * Instead of having snoop and capture code intermixed with the recv functions,
1494 * both the interrupt handler and hfi1_ib_rcv() we are going to hijack the call
1495 * and land in here for snoop/capture but if not enabled the call will go
1496 * through as before. This gives us a single point to constrain all of the snoop
1497 * snoop recv logic. There is nothing special that needs to happen for bypass
1498 * packets. This routine should not try to look into the packet. It just copied
1499 * it. There is no guarantee for filters when it comes to bypass packets as
1500 * there is no specific support. Bottom line is this routine does now even know
1501 * what a bypass packet is.
1503 int snoop_recv_handler(struct hfi1_packet *packet)
1505 struct hfi1_pportdata *ppd = packet->rcd->ppd;
1506 struct hfi1_ib_header *hdr = packet->hdr;
1507 int header_size = packet->hlen;
1508 void *data = packet->ebuf;
1509 u32 tlen = packet->tlen;
1510 struct snoop_packet *s_packet = NULL;
1514 struct capture_md md;
1516 snoop_dbg("PACKET IN: hdr size %d tlen %d data %p", header_size, tlen,
1519 trace_snoop_capture(ppd->dd, header_size, hdr, tlen - header_size,
1522 if (!ppd->dd->hfi1_snoop.filter_callback) {
1523 snoop_dbg("filter not set");
1524 ret = HFI1_FILTER_HIT;
1526 ret = ppd->dd->hfi1_snoop.filter_callback(hdr, data,
1527 ppd->dd->hfi1_snoop.filter_value);
1531 case HFI1_FILTER_ERR:
1532 snoop_dbg("Error in filter call");
1534 case HFI1_FILTER_MISS:
1535 snoop_dbg("Filter Miss");
1537 case HFI1_FILTER_HIT:
1539 if (ppd->dd->hfi1_snoop.mode_flag & HFI1_PORT_SNOOP_MODE)
1541 if ((snoop_mode == 0) ||
1542 unlikely(snoop_flags & SNOOP_USE_METADATA))
1543 md_len = sizeof(struct capture_md);
1545 s_packet = allocate_snoop_packet(header_size,
1549 if (unlikely(!s_packet)) {
1550 dd_dev_warn_ratelimited(ppd->dd, "Unable to allocate snoop/capture packet\n");
1555 memset(&md, 0, sizeof(struct capture_md));
1557 md.dir = PKT_DIR_INGRESS;
1558 md.u.rhf = packet->rhf;
1559 memcpy(s_packet->data, &md, md_len);
1562 /* We should always have a header */
1564 memcpy(s_packet->data + md_len, hdr, header_size);
1566 dd_dev_err(ppd->dd, "Unable to copy header to snoop/capture packet\n");
1572 * Packets with no data are possible. If there is no data needed
1573 * to take care of the last 4 bytes which are normally included
1574 * with data buffers and are included in tlen. Since we kzalloc
1575 * the buffer we do not need to set any values but if we decide
1576 * not to use kzalloc we should zero them.
1579 memcpy(s_packet->data + header_size + md_len, data,
1580 tlen - header_size);
1582 s_packet->total_len = tlen + md_len;
1583 snoop_list_add_tail(s_packet, ppd->dd);
1586 * If we are snooping the packet not capturing then throw away
1587 * after adding to the list.
1589 snoop_dbg("Capturing packet");
1590 if (ppd->dd->hfi1_snoop.mode_flag & HFI1_PORT_SNOOP_MODE) {
1591 snoop_dbg("Throwing packet away");
1593 * If we are dropping the packet we still may need to
1594 * handle the case where error flags are set, this is
1595 * normally done by the type specific handler but that
1596 * won't be called in this case.
1598 if (unlikely(rhf_err_flags(packet->rhf)))
1599 handle_eflags(packet);
1601 /* throw the packet on the floor */
1602 return RHF_RCV_CONTINUE;
1610 * We do not care what type of packet came in here - just pass it off
1611 * to the normal handler.
1613 return ppd->dd->normal_rhf_rcv_functions[rhf_rcv_type(packet->rhf)]
1618 * Handle snooping and capturing packets when sdma is being used.
1620 int snoop_send_dma_handler(struct hfi1_qp *qp, struct hfi1_pkt_state *ps,
1623 pr_alert("Snooping/Capture of Send DMA Packets Is Not Supported!\n");
1624 snoop_dbg("Unsupported Operation");
1625 return hfi1_verbs_send_dma(qp, ps, 0);
1629 * Handle snooping and capturing packets when pio is being used. Does not handle
1630 * bypass packets. The only way to send a bypass packet currently is to use the
1631 * diagpkt interface. When that interface is enable snoop/capture is not.
1633 int snoop_send_pio_handler(struct hfi1_qp *qp, struct hfi1_pkt_state *ps,
1636 struct ahg_ib_header *ahdr = qp->s_hdr;
1637 u32 hdrwords = qp->s_hdrwords;
1638 struct hfi1_sge_state *ss = qp->s_cur_sge;
1639 u32 len = qp->s_cur_size;
1640 u32 dwords = (len + 3) >> 2;
1641 u32 plen = hdrwords + dwords + 2; /* includes pbc */
1642 struct hfi1_pportdata *ppd = ps->ppd;
1643 struct snoop_packet *s_packet = NULL;
1644 u32 *hdr = (u32 *)&ahdr->ibh;
1646 struct hfi1_sge_state temp_ss;
1648 void *data_start = NULL;
1652 struct capture_md md;
1654 u32 hdr_len = hdrwords << 2;
1655 u32 tlen = HFI1_GET_PKT_LEN(&ahdr->ibh);
1659 snoop_dbg("PACKET OUT: hdrword %u len %u plen %u dwords %u tlen %u",
1660 hdrwords, len, plen, dwords, tlen);
1661 if (ppd->dd->hfi1_snoop.mode_flag & HFI1_PORT_SNOOP_MODE)
1663 if ((snoop_mode == 0) ||
1664 unlikely(snoop_flags & SNOOP_USE_METADATA))
1665 md_len = sizeof(struct capture_md);
1667 /* not using ss->total_len as arg 2 b/c that does not count CRC */
1668 s_packet = allocate_snoop_packet(hdr_len, tlen - hdr_len, md_len);
1670 if (unlikely(!s_packet)) {
1671 dd_dev_warn_ratelimited(ppd->dd, "Unable to allocate snoop/capture packet\n");
1675 s_packet->total_len = tlen + md_len;
1678 memset(&md, 0, sizeof(struct capture_md));
1680 md.dir = PKT_DIR_EGRESS;
1681 if (likely(pbc == 0)) {
1682 vl = be16_to_cpu(ahdr->ibh.lrh[0]) >> 12;
1683 md.u.pbc = create_pbc(ppd, 0, qp->s_srate, vl, plen);
1687 memcpy(s_packet->data, &md, md_len);
1694 memcpy(s_packet->data + md_len, hdr, hdr_len);
1697 "Unable to copy header to snoop/capture packet\n");
1703 data = s_packet->data + hdr_len + md_len;
1708 * The update_sge() function below will not modify the
1709 * individual SGEs in the array. It will make a copy each time
1710 * and operate on that. So we only need to copy this instance
1711 * and it won't impact PIO.
1716 snoop_dbg("Need to copy %d bytes", length);
1718 void *addr = temp_ss.sge.vaddr;
1719 u32 slen = temp_ss.sge.length;
1721 if (slen > length) {
1723 snoop_dbg("slen %d > len %d", slen, length);
1725 snoop_dbg("copy %d to %p", slen, addr);
1726 memcpy(data, addr, slen);
1727 update_sge(&temp_ss, slen);
1730 snoop_dbg("data is now %p bytes left %d", data, length);
1732 snoop_dbg("Completed SGE copy");
1736 * Why do the filter check down here? Because the event tracing has its
1737 * own filtering and we need to have the walked the SGE list.
1739 if (!ppd->dd->hfi1_snoop.filter_callback) {
1740 snoop_dbg("filter not set\n");
1741 ret = HFI1_FILTER_HIT;
1743 ret = ppd->dd->hfi1_snoop.filter_callback(
1746 ppd->dd->hfi1_snoop.filter_value);
1750 case HFI1_FILTER_ERR:
1751 snoop_dbg("Error in filter call");
1753 case HFI1_FILTER_MISS:
1754 snoop_dbg("Filter Miss");
1757 case HFI1_FILTER_HIT:
1758 snoop_dbg("Capturing packet");
1759 snoop_list_add_tail(s_packet, ppd->dd);
1761 if (unlikely((snoop_flags & SNOOP_DROP_SEND) &&
1762 (ppd->dd->hfi1_snoop.mode_flag &
1763 HFI1_PORT_SNOOP_MODE))) {
1764 unsigned long flags;
1766 snoop_dbg("Dropping packet");
1768 spin_lock_irqsave(&qp->s_lock, flags);
1773 spin_unlock_irqrestore(&qp->s_lock, flags);
1774 } else if (qp->ibqp.qp_type == IB_QPT_RC) {
1775 spin_lock_irqsave(&qp->s_lock, flags);
1776 hfi1_rc_send_complete(qp, &ahdr->ibh);
1777 spin_unlock_irqrestore(&qp->s_lock, flags);
1787 return hfi1_verbs_send_pio(qp, ps, md.u.pbc);
1791 * Callers of this must pass a hfi1_ib_header type for the from ptr. Currently
1792 * this can be used anywhere, but the intention is for inline ACKs for RC and
1793 * CCA packets. We don't restrict this usage though.
1795 void snoop_inline_pio_send(struct hfi1_devdata *dd, struct pio_buf *pbuf,
1796 u64 pbc, const void *from, size_t count)
1800 struct capture_md md;
1801 struct snoop_packet *s_packet = NULL;
1804 * count is in dwords so we need to convert to bytes.
1805 * We also need to account for CRC which would be tacked on by hardware.
1807 int packet_len = (count << 2) + 4;
1810 snoop_dbg("ACK OUT: len %d", packet_len);
1812 if (!dd->hfi1_snoop.filter_callback) {
1813 snoop_dbg("filter not set");
1814 ret = HFI1_FILTER_HIT;
1816 ret = dd->hfi1_snoop.filter_callback(
1817 (struct hfi1_ib_header *)from,
1819 dd->hfi1_snoop.filter_value);
1823 case HFI1_FILTER_ERR:
1824 snoop_dbg("Error in filter call");
1826 case HFI1_FILTER_MISS:
1827 snoop_dbg("Filter Miss");
1829 case HFI1_FILTER_HIT:
1830 snoop_dbg("Capturing packet");
1831 if (dd->hfi1_snoop.mode_flag & HFI1_PORT_SNOOP_MODE)
1833 if ((snoop_mode == 0) ||
1834 unlikely(snoop_flags & SNOOP_USE_METADATA))
1835 md_len = sizeof(struct capture_md);
1837 s_packet = allocate_snoop_packet(packet_len, 0, md_len);
1839 if (unlikely(!s_packet)) {
1840 dd_dev_warn_ratelimited(dd, "Unable to allocate snoop/capture packet\n");
1841 goto inline_pio_out;
1844 s_packet->total_len = packet_len + md_len;
1846 /* Fill in the metadata for the packet */
1848 memset(&md, 0, sizeof(struct capture_md));
1850 md.dir = PKT_DIR_EGRESS;
1852 memcpy(s_packet->data, &md, md_len);
1855 /* Add the packet data which is a single buffer */
1856 memcpy(s_packet->data + md_len, from, packet_len);
1858 snoop_list_add_tail(s_packet, dd);
1860 if (unlikely((snoop_flags & SNOOP_DROP_SEND) && snoop_mode)) {
1861 snoop_dbg("Dropping packet");
1870 pio_copy(dd, pbuf, pbc, from, count);