2 * Copyright (c) 2006, 2007, 2008, 2009 QLogic Corporation. All rights reserved.
3 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
38 /* cut down ridiculously long IB macro names */
39 #define OP(x) IB_OPCODE_RC_##x
42 static u32 restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe,
47 len = ((psn - wqe->psn) & QIB_PSN_MASK) * pmtu;
48 ss->sge = wqe->sg_list[0];
49 ss->sg_list = wqe->sg_list + 1;
50 ss->num_sge = wqe->wr.num_sge;
51 ss->total_len = wqe->length;
52 rvt_skip_sge(ss, len, false);
53 return wqe->length - len;
57 * qib_make_rc_ack - construct a response packet (ACK, NAK, or RDMA read)
58 * @dev: the device for this QP
59 * @qp: a pointer to the QP
60 * @ohdr: a pointer to the IB header being constructed
63 * Return 1 if constructed; otherwise, return 0.
64 * Note that we are in the responder's side of the QP context.
65 * Note the QP s_lock must be held.
67 static int qib_make_rc_ack(struct qib_ibdev *dev, struct rvt_qp *qp,
68 struct ib_other_headers *ohdr, u32 pmtu)
70 struct rvt_ack_entry *e;
76 /* Don't send an ACK if we aren't supposed to. */
77 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
80 /* header size in 32-bit words LRH+BTH = (8+12)/4. */
83 switch (qp->s_ack_state) {
84 case OP(RDMA_READ_RESPONSE_LAST):
85 case OP(RDMA_READ_RESPONSE_ONLY):
86 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
88 rvt_put_mr(e->rdma_sge.mr);
89 e->rdma_sge.mr = NULL;
92 case OP(ATOMIC_ACKNOWLEDGE):
94 * We can increment the tail pointer now that the last
95 * response has been sent instead of only being
98 if (++qp->s_tail_ack_queue > QIB_MAX_RDMA_ATOMIC)
99 qp->s_tail_ack_queue = 0;
102 case OP(ACKNOWLEDGE):
103 /* Check for no next entry in the queue. */
104 if (qp->r_head_ack_queue == qp->s_tail_ack_queue) {
105 if (qp->s_flags & RVT_S_ACK_PENDING)
110 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
111 if (e->opcode == OP(RDMA_READ_REQUEST)) {
113 * If a RDMA read response is being resent and
114 * we haven't seen the duplicate request yet,
115 * then stop sending the remaining responses the
116 * responder has seen until the requester resends it.
118 len = e->rdma_sge.sge_length;
119 if (len && !e->rdma_sge.mr) {
120 qp->s_tail_ack_queue = qp->r_head_ack_queue;
123 /* Copy SGE state in case we need to resend */
124 qp->s_rdma_mr = e->rdma_sge.mr;
126 rvt_get_mr(qp->s_rdma_mr);
127 qp->s_ack_rdma_sge.sge = e->rdma_sge;
128 qp->s_ack_rdma_sge.num_sge = 1;
129 qp->s_cur_sge = &qp->s_ack_rdma_sge;
132 qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST);
134 qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY);
137 ohdr->u.aeth = rvt_compute_aeth(qp);
139 qp->s_ack_rdma_psn = e->psn;
140 bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK;
142 /* COMPARE_SWAP or FETCH_ADD */
143 qp->s_cur_sge = NULL;
145 qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE);
146 ohdr->u.at.aeth = rvt_compute_aeth(qp);
147 ib_u64_put(e->atomic_data, &ohdr->u.at.atomic_ack_eth);
148 hwords += sizeof(ohdr->u.at) / sizeof(u32);
149 bth2 = e->psn & QIB_PSN_MASK;
152 bth0 = qp->s_ack_state << 24;
155 case OP(RDMA_READ_RESPONSE_FIRST):
156 qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE);
158 case OP(RDMA_READ_RESPONSE_MIDDLE):
159 qp->s_cur_sge = &qp->s_ack_rdma_sge;
160 qp->s_rdma_mr = qp->s_ack_rdma_sge.sge.mr;
162 rvt_get_mr(qp->s_rdma_mr);
163 len = qp->s_ack_rdma_sge.sge.sge_length;
167 ohdr->u.aeth = rvt_compute_aeth(qp);
169 qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST);
170 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
173 bth0 = qp->s_ack_state << 24;
174 bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK;
180 * Send a regular ACK.
181 * Set the s_ack_state so we wait until after sending
182 * the ACK before setting s_ack_state to ACKNOWLEDGE
185 qp->s_ack_state = OP(SEND_ONLY);
186 qp->s_flags &= ~RVT_S_ACK_PENDING;
187 qp->s_cur_sge = NULL;
190 cpu_to_be32((qp->r_msn & IB_MSN_MASK) |
192 IB_AETH_CREDIT_SHIFT));
194 ohdr->u.aeth = rvt_compute_aeth(qp);
197 bth0 = OP(ACKNOWLEDGE) << 24;
198 bth2 = qp->s_ack_psn & QIB_PSN_MASK;
200 qp->s_rdma_ack_cnt++;
201 qp->s_hdrwords = hwords;
202 qp->s_cur_size = len;
203 qib_make_ruc_header(qp, ohdr, bth0, bth2);
207 qp->s_ack_state = OP(ACKNOWLEDGE);
208 qp->s_flags &= ~(RVT_S_RESP_PENDING | RVT_S_ACK_PENDING);
213 * qib_make_rc_req - construct a request packet (SEND, RDMA r/w, ATOMIC)
214 * @qp: a pointer to the QP
216 * Assumes the s_lock is held.
218 * Return 1 if constructed; otherwise, return 0.
220 int qib_make_rc_req(struct rvt_qp *qp, unsigned long *flags)
222 struct qib_qp_priv *priv = qp->priv;
223 struct qib_ibdev *dev = to_idev(qp->ibqp.device);
224 struct ib_other_headers *ohdr;
225 struct rvt_sge_state *ss;
226 struct rvt_swqe *wqe;
236 ohdr = &priv->s_hdr->u.oth;
237 if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)
238 ohdr = &priv->s_hdr->u.l.oth;
240 /* Sending responses has higher priority over sending requests. */
241 if ((qp->s_flags & RVT_S_RESP_PENDING) &&
242 qib_make_rc_ack(dev, qp, ohdr, pmtu))
245 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) {
246 if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND))
248 /* We are in the error state, flush the work request. */
249 smp_read_barrier_depends(); /* see post_one_send() */
250 if (qp->s_last == READ_ONCE(qp->s_head))
252 /* If DMAs are in progress, we can't flush immediately. */
253 if (atomic_read(&priv->s_dma_busy)) {
254 qp->s_flags |= RVT_S_WAIT_DMA;
257 wqe = rvt_get_swqe_ptr(qp, qp->s_last);
258 qib_send_complete(qp, wqe, qp->s_last != qp->s_acked ?
259 IB_WC_SUCCESS : IB_WC_WR_FLUSH_ERR);
260 /* will get called again */
264 if (qp->s_flags & (RVT_S_WAIT_RNR | RVT_S_WAIT_ACK))
267 if (qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) {
268 if (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) {
269 qp->s_flags |= RVT_S_WAIT_PSN;
272 qp->s_sending_psn = qp->s_psn;
273 qp->s_sending_hpsn = qp->s_psn - 1;
276 /* header size in 32-bit words LRH+BTH = (8+12)/4. */
280 /* Send a request. */
281 wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
282 switch (qp->s_state) {
284 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_NEXT_SEND_OK))
287 * Resend an old request or start a new one.
289 * We keep track of the current SWQE so that
290 * we don't reset the "furthest progress" state
291 * if we need to back up.
294 if (qp->s_cur == qp->s_tail) {
295 /* Check if send work queue is empty. */
296 smp_read_barrier_depends(); /* see post_one_send() */
297 if (qp->s_tail == READ_ONCE(qp->s_head))
300 * If a fence is requested, wait for previous
301 * RDMA read and atomic operations to finish.
303 if ((wqe->wr.send_flags & IB_SEND_FENCE) &&
304 qp->s_num_rd_atomic) {
305 qp->s_flags |= RVT_S_WAIT_FENCE;
309 qp->s_psn = wqe->psn;
312 * Note that we have to be careful not to modify the
313 * original work request since we may need to resend
318 bth2 = qp->s_psn & QIB_PSN_MASK;
319 switch (wqe->wr.opcode) {
321 case IB_WR_SEND_WITH_IMM:
322 /* If no credit, return. */
323 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT) &&
324 rvt_cmp_msn(wqe->ssn, qp->s_lsn + 1) > 0) {
325 qp->s_flags |= RVT_S_WAIT_SSN_CREDIT;
329 qp->s_state = OP(SEND_FIRST);
333 if (wqe->wr.opcode == IB_WR_SEND)
334 qp->s_state = OP(SEND_ONLY);
336 qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE);
337 /* Immediate data comes after the BTH */
338 ohdr->u.imm_data = wqe->wr.ex.imm_data;
341 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
342 bth0 |= IB_BTH_SOLICITED;
343 bth2 |= IB_BTH_REQ_ACK;
344 if (++qp->s_cur == qp->s_size)
348 case IB_WR_RDMA_WRITE:
349 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
352 case IB_WR_RDMA_WRITE_WITH_IMM:
353 /* If no credit, return. */
354 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT) &&
355 rvt_cmp_msn(wqe->ssn, qp->s_lsn + 1) > 0) {
356 qp->s_flags |= RVT_S_WAIT_SSN_CREDIT;
360 ohdr->u.rc.reth.vaddr =
361 cpu_to_be64(wqe->rdma_wr.remote_addr);
362 ohdr->u.rc.reth.rkey =
363 cpu_to_be32(wqe->rdma_wr.rkey);
364 ohdr->u.rc.reth.length = cpu_to_be32(len);
365 hwords += sizeof(struct ib_reth) / sizeof(u32);
367 qp->s_state = OP(RDMA_WRITE_FIRST);
371 if (wqe->rdma_wr.wr.opcode == IB_WR_RDMA_WRITE)
372 qp->s_state = OP(RDMA_WRITE_ONLY);
374 qp->s_state = OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE);
375 /* Immediate data comes after RETH */
376 ohdr->u.rc.imm_data =
377 wqe->rdma_wr.wr.ex.imm_data;
379 if (wqe->rdma_wr.wr.send_flags & IB_SEND_SOLICITED)
380 bth0 |= IB_BTH_SOLICITED;
382 bth2 |= IB_BTH_REQ_ACK;
383 if (++qp->s_cur == qp->s_size)
387 case IB_WR_RDMA_READ:
389 * Don't allow more operations to be started
390 * than the QP limits allow.
393 if (qp->s_num_rd_atomic >=
394 qp->s_max_rd_atomic) {
395 qp->s_flags |= RVT_S_WAIT_RDMAR;
398 qp->s_num_rd_atomic++;
399 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
403 ohdr->u.rc.reth.vaddr =
404 cpu_to_be64(wqe->rdma_wr.remote_addr);
405 ohdr->u.rc.reth.rkey =
406 cpu_to_be32(wqe->rdma_wr.rkey);
407 ohdr->u.rc.reth.length = cpu_to_be32(len);
408 qp->s_state = OP(RDMA_READ_REQUEST);
409 hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32);
412 bth2 |= IB_BTH_REQ_ACK;
413 if (++qp->s_cur == qp->s_size)
417 case IB_WR_ATOMIC_CMP_AND_SWP:
418 case IB_WR_ATOMIC_FETCH_AND_ADD:
420 * Don't allow more operations to be started
421 * than the QP limits allow.
424 if (qp->s_num_rd_atomic >=
425 qp->s_max_rd_atomic) {
426 qp->s_flags |= RVT_S_WAIT_RDMAR;
429 qp->s_num_rd_atomic++;
430 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
433 if (wqe->atomic_wr.wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
434 qp->s_state = OP(COMPARE_SWAP);
435 put_ib_ateth_swap(wqe->atomic_wr.swap,
436 &ohdr->u.atomic_eth);
437 put_ib_ateth_swap(wqe->atomic_wr.compare_add,
438 &ohdr->u.atomic_eth);
440 qp->s_state = OP(FETCH_ADD);
441 put_ib_ateth_swap(wqe->atomic_wr.compare_add,
442 &ohdr->u.atomic_eth);
443 put_ib_ateth_swap(0, &ohdr->u.atomic_eth);
445 put_ib_ateth_vaddr(wqe->atomic_wr.remote_addr,
446 &ohdr->u.atomic_eth);
447 ohdr->u.atomic_eth.rkey = cpu_to_be32(
448 wqe->atomic_wr.rkey);
449 hwords += sizeof(struct ib_atomic_eth) / sizeof(u32);
452 bth2 |= IB_BTH_REQ_ACK;
453 if (++qp->s_cur == qp->s_size)
460 qp->s_sge.sge = wqe->sg_list[0];
461 qp->s_sge.sg_list = wqe->sg_list + 1;
462 qp->s_sge.num_sge = wqe->wr.num_sge;
463 qp->s_sge.total_len = wqe->length;
464 qp->s_len = wqe->length;
467 if (qp->s_tail >= qp->s_size)
470 if (wqe->wr.opcode == IB_WR_RDMA_READ)
471 qp->s_psn = wqe->lpsn + 1;
476 case OP(RDMA_READ_RESPONSE_FIRST):
478 * qp->s_state is normally set to the opcode of the
479 * last packet constructed for new requests and therefore
480 * is never set to RDMA read response.
481 * RDMA_READ_RESPONSE_FIRST is used by the ACK processing
482 * thread to indicate a SEND needs to be restarted from an
483 * earlier PSN without interferring with the sending thread.
484 * See qib_restart_rc().
486 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
489 qp->s_state = OP(SEND_MIDDLE);
491 case OP(SEND_MIDDLE):
492 bth2 = qp->s_psn++ & QIB_PSN_MASK;
499 if (wqe->wr.opcode == IB_WR_SEND)
500 qp->s_state = OP(SEND_LAST);
502 qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE);
503 /* Immediate data comes after the BTH */
504 ohdr->u.imm_data = wqe->wr.ex.imm_data;
507 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
508 bth0 |= IB_BTH_SOLICITED;
509 bth2 |= IB_BTH_REQ_ACK;
511 if (qp->s_cur >= qp->s_size)
515 case OP(RDMA_READ_RESPONSE_LAST):
517 * qp->s_state is normally set to the opcode of the
518 * last packet constructed for new requests and therefore
519 * is never set to RDMA read response.
520 * RDMA_READ_RESPONSE_LAST is used by the ACK processing
521 * thread to indicate a RDMA write needs to be restarted from
522 * an earlier PSN without interferring with the sending thread.
523 * See qib_restart_rc().
525 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
527 case OP(RDMA_WRITE_FIRST):
528 qp->s_state = OP(RDMA_WRITE_MIDDLE);
530 case OP(RDMA_WRITE_MIDDLE):
531 bth2 = qp->s_psn++ & QIB_PSN_MASK;
538 if (wqe->wr.opcode == IB_WR_RDMA_WRITE)
539 qp->s_state = OP(RDMA_WRITE_LAST);
541 qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE);
542 /* Immediate data comes after the BTH */
543 ohdr->u.imm_data = wqe->wr.ex.imm_data;
545 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
546 bth0 |= IB_BTH_SOLICITED;
548 bth2 |= IB_BTH_REQ_ACK;
550 if (qp->s_cur >= qp->s_size)
554 case OP(RDMA_READ_RESPONSE_MIDDLE):
556 * qp->s_state is normally set to the opcode of the
557 * last packet constructed for new requests and therefore
558 * is never set to RDMA read response.
559 * RDMA_READ_RESPONSE_MIDDLE is used by the ACK processing
560 * thread to indicate a RDMA read needs to be restarted from
561 * an earlier PSN without interferring with the sending thread.
562 * See qib_restart_rc().
564 len = ((qp->s_psn - wqe->psn) & QIB_PSN_MASK) * pmtu;
565 ohdr->u.rc.reth.vaddr =
566 cpu_to_be64(wqe->rdma_wr.remote_addr + len);
567 ohdr->u.rc.reth.rkey =
568 cpu_to_be32(wqe->rdma_wr.rkey);
569 ohdr->u.rc.reth.length = cpu_to_be32(wqe->length - len);
570 qp->s_state = OP(RDMA_READ_REQUEST);
571 hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32);
572 bth2 = (qp->s_psn & QIB_PSN_MASK) | IB_BTH_REQ_ACK;
573 qp->s_psn = wqe->lpsn + 1;
577 if (qp->s_cur == qp->s_size)
581 qp->s_sending_hpsn = bth2;
582 delta = (((int) bth2 - (int) wqe->psn) << 8) >> 8;
583 if (delta && delta % QIB_PSN_CREDIT == 0)
584 bth2 |= IB_BTH_REQ_ACK;
585 if (qp->s_flags & RVT_S_SEND_ONE) {
586 qp->s_flags &= ~RVT_S_SEND_ONE;
587 qp->s_flags |= RVT_S_WAIT_ACK;
588 bth2 |= IB_BTH_REQ_ACK;
591 qp->s_hdrwords = hwords;
593 qp->s_cur_size = len;
594 qib_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24), bth2);
598 qp->s_flags &= ~RVT_S_BUSY;
603 * qib_send_rc_ack - Construct an ACK packet and send it
604 * @qp: a pointer to the QP
606 * This is called from qib_rc_rcv() and qib_kreceive().
607 * Note that RDMA reads and atomics are handled in the
608 * send side QP state and tasklet.
610 void qib_send_rc_ack(struct rvt_qp *qp)
612 struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device);
613 struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
614 struct qib_pportdata *ppd = ppd_from_ibp(ibp);
621 struct ib_header hdr;
622 struct ib_other_headers *ohdr;
626 spin_lock_irqsave(&qp->s_lock, flags);
628 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
631 /* Don't send ACK or NAK if a RDMA read or atomic is pending. */
632 if ((qp->s_flags & RVT_S_RESP_PENDING) || qp->s_rdma_ack_cnt)
635 /* Construct the header with s_lock held so APM doesn't change it. */
638 /* header size in 32-bit words LRH+BTH+AETH = (8+12+4)/4. */
640 if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) &
642 hwords += qib_make_grh(ibp, &hdr.u.l.grh,
643 rdma_ah_read_grh(&qp->remote_ah_attr),
648 /* read pkey_index w/o lock (its atomic) */
649 bth0 = qib_get_pkey(ibp, qp->s_pkey_index) | (OP(ACKNOWLEDGE) << 24);
650 if (qp->s_mig_state == IB_MIG_MIGRATED)
651 bth0 |= IB_BTH_MIG_REQ;
653 ohdr->u.aeth = cpu_to_be32((qp->r_msn & IB_MSN_MASK) |
655 IB_AETH_CREDIT_SHIFT));
657 ohdr->u.aeth = rvt_compute_aeth(qp);
658 lrh0 |= ibp->sl_to_vl[rdma_ah_get_sl(&qp->remote_ah_attr)] << 12 |
659 rdma_ah_get_sl(&qp->remote_ah_attr) << 4;
660 hdr.lrh[0] = cpu_to_be16(lrh0);
661 hdr.lrh[1] = cpu_to_be16(rdma_ah_get_dlid(&qp->remote_ah_attr));
662 hdr.lrh[2] = cpu_to_be16(hwords + SIZE_OF_CRC);
663 hdr.lrh[3] = cpu_to_be16(ppd->lid |
664 rdma_ah_get_path_bits(&qp->remote_ah_attr));
665 ohdr->bth[0] = cpu_to_be32(bth0);
666 ohdr->bth[1] = cpu_to_be32(qp->remote_qpn);
667 ohdr->bth[2] = cpu_to_be32(qp->r_ack_psn & QIB_PSN_MASK);
669 spin_unlock_irqrestore(&qp->s_lock, flags);
671 /* Don't try to send ACKs if the link isn't ACTIVE */
672 if (!(ppd->lflags & QIBL_LINKACTIVE))
675 control = dd->f_setpbc_control(ppd, hwords + SIZE_OF_CRC,
676 qp->s_srate, lrh0 >> 12);
677 /* length is + 1 for the control dword */
678 pbc = ((u64) control << 32) | (hwords + 1);
680 piobuf = dd->f_getsendbuf(ppd, pbc, &pbufn);
683 * We are out of PIO buffers at the moment.
684 * Pass responsibility for sending the ACK to the
685 * send tasklet so that when a PIO buffer becomes
686 * available, the ACK is sent ahead of other outgoing
689 spin_lock_irqsave(&qp->s_lock, flags);
695 * We have to flush after the PBC for correctness
696 * on some cpus or WC buffer can be written out of order.
700 if (dd->flags & QIB_PIO_FLUSH_WC) {
701 u32 *hdrp = (u32 *) &hdr;
704 qib_pio_copy(piobuf + 2, hdrp, hwords - 1);
706 __raw_writel(hdrp[hwords - 1], piobuf + hwords + 1);
708 qib_pio_copy(piobuf + 2, (u32 *) &hdr, hwords);
710 if (dd->flags & QIB_USE_SPCL_TRIG) {
711 u32 spcl_off = (pbufn >= dd->piobcnt2k) ? 2047 : 1023;
714 __raw_writel(0xaebecede, piobuf + spcl_off);
718 qib_sendbuf_done(dd, pbufn);
720 this_cpu_inc(ibp->pmastats->n_unicast_xmit);
724 if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) {
725 this_cpu_inc(*ibp->rvp.rc_qacks);
726 qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING;
727 qp->s_nak_state = qp->r_nak_state;
728 qp->s_ack_psn = qp->r_ack_psn;
730 /* Schedule the send tasklet. */
731 qib_schedule_send(qp);
734 spin_unlock_irqrestore(&qp->s_lock, flags);
740 * reset_psn - reset the QP state to send starting from PSN
742 * @psn: the packet sequence number to restart at
744 * This is called from qib_rc_rcv() to process an incoming RC ACK
746 * Called at interrupt level with the QP s_lock held.
748 static void reset_psn(struct rvt_qp *qp, u32 psn)
751 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n);
757 * If we are starting the request from the beginning,
758 * let the normal send code handle initialization.
760 if (qib_cmp24(psn, wqe->psn) <= 0) {
761 qp->s_state = OP(SEND_LAST);
765 /* Find the work request opcode corresponding to the given PSN. */
766 opcode = wqe->wr.opcode;
770 if (++n == qp->s_size)
774 wqe = rvt_get_swqe_ptr(qp, n);
775 diff = qib_cmp24(psn, wqe->psn);
780 * If we are starting the request from the beginning,
781 * let the normal send code handle initialization.
784 qp->s_state = OP(SEND_LAST);
787 opcode = wqe->wr.opcode;
791 * Set the state to restart in the middle of a request.
792 * Don't change the s_sge, s_cur_sge, or s_cur_size.
793 * See qib_make_rc_req().
797 case IB_WR_SEND_WITH_IMM:
798 qp->s_state = OP(RDMA_READ_RESPONSE_FIRST);
801 case IB_WR_RDMA_WRITE:
802 case IB_WR_RDMA_WRITE_WITH_IMM:
803 qp->s_state = OP(RDMA_READ_RESPONSE_LAST);
806 case IB_WR_RDMA_READ:
807 qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE);
812 * This case shouldn't happen since its only
815 qp->s_state = OP(SEND_LAST);
820 * Set RVT_S_WAIT_PSN as qib_rc_complete() may start the timer
821 * asynchronously before the send tasklet can get scheduled.
822 * Doing it in qib_make_rc_req() is too late.
824 if ((qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) &&
825 (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0))
826 qp->s_flags |= RVT_S_WAIT_PSN;
830 * Back up requester to resend the last un-ACKed request.
831 * The QP r_lock and s_lock should be held and interrupts disabled.
833 void qib_restart_rc(struct rvt_qp *qp, u32 psn, int wait)
835 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
836 struct qib_ibport *ibp;
838 if (qp->s_retry == 0) {
839 if (qp->s_mig_state == IB_MIG_ARMED) {
841 qp->s_retry = qp->s_retry_cnt;
842 } else if (qp->s_last == qp->s_acked) {
843 qib_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR);
844 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
846 } else /* XXX need to handle delayed completion */
851 ibp = to_iport(qp->ibqp.device, qp->port_num);
852 if (wqe->wr.opcode == IB_WR_RDMA_READ)
853 ibp->rvp.n_rc_resends++;
855 ibp->rvp.n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK;
857 qp->s_flags &= ~(RVT_S_WAIT_FENCE | RVT_S_WAIT_RDMAR |
858 RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_PSN |
861 qp->s_flags |= RVT_S_SEND_ONE;
866 * Set qp->s_sending_psn to the next PSN after the given one.
867 * This would be psn+1 except when RDMA reads are present.
869 static void reset_sending_psn(struct rvt_qp *qp, u32 psn)
871 struct rvt_swqe *wqe;
874 /* Find the work request corresponding to the given PSN. */
876 wqe = rvt_get_swqe_ptr(qp, n);
877 if (qib_cmp24(psn, wqe->lpsn) <= 0) {
878 if (wqe->wr.opcode == IB_WR_RDMA_READ)
879 qp->s_sending_psn = wqe->lpsn + 1;
881 qp->s_sending_psn = psn + 1;
884 if (++n == qp->s_size)
892 * This should be called with the QP s_lock held and interrupts disabled.
894 void qib_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr)
896 struct ib_other_headers *ohdr;
897 struct rvt_swqe *wqe;
901 if (!(ib_rvt_state_ops[qp->state] & RVT_SEND_OR_FLUSH_OR_RECV_OK))
904 /* Find out where the BTH is */
905 if ((be16_to_cpu(hdr->lrh[0]) & 3) == QIB_LRH_BTH)
908 ohdr = &hdr->u.l.oth;
910 opcode = be32_to_cpu(ohdr->bth[0]) >> 24;
911 if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) &&
912 opcode <= OP(ATOMIC_ACKNOWLEDGE)) {
913 WARN_ON(!qp->s_rdma_ack_cnt);
914 qp->s_rdma_ack_cnt--;
918 psn = be32_to_cpu(ohdr->bth[2]);
919 reset_sending_psn(qp, psn);
922 * Start timer after a packet requesting an ACK has been sent and
923 * there are still requests that haven't been acked.
925 if ((psn & IB_BTH_REQ_ACK) && qp->s_acked != qp->s_tail &&
926 !(qp->s_flags & (RVT_S_TIMER | RVT_S_WAIT_RNR | RVT_S_WAIT_PSN)) &&
927 (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
928 rvt_add_retry_timer(qp);
930 while (qp->s_last != qp->s_acked) {
933 wqe = rvt_get_swqe_ptr(qp, qp->s_last);
934 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 &&
935 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)
938 if (++s_last >= qp->s_size)
941 /* see post_send() */
944 rvt_qp_swqe_complete(qp,
946 ib_qib_wc_opcode[wqe->wr.opcode],
950 * If we were waiting for sends to complete before resending,
951 * and they are now complete, restart sending.
953 if (qp->s_flags & RVT_S_WAIT_PSN &&
954 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) {
955 qp->s_flags &= ~RVT_S_WAIT_PSN;
956 qp->s_sending_psn = qp->s_psn;
957 qp->s_sending_hpsn = qp->s_psn - 1;
958 qib_schedule_send(qp);
962 static inline void update_last_psn(struct rvt_qp *qp, u32 psn)
964 qp->s_last_psn = psn;
968 * Generate a SWQE completion.
969 * This is similar to qib_send_complete but has to check to be sure
970 * that the SGEs are not being referenced if the SWQE is being resent.
972 static struct rvt_swqe *do_rc_completion(struct rvt_qp *qp,
973 struct rvt_swqe *wqe,
974 struct qib_ibport *ibp)
977 * Don't decrement refcount and don't generate a
978 * completion if the SWQE is being resent until the send
981 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 ||
982 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) {
987 if (++s_last >= qp->s_size)
990 /* see post_send() */
992 rvt_qp_swqe_complete(qp,
994 ib_qib_wc_opcode[wqe->wr.opcode],
997 this_cpu_inc(*ibp->rvp.rc_delayed_comp);
999 qp->s_retry = qp->s_retry_cnt;
1000 update_last_psn(qp, wqe->lpsn);
1003 * If we are completing a request which is in the process of
1004 * being resent, we can stop resending it since we know the
1005 * responder has already seen it.
1007 if (qp->s_acked == qp->s_cur) {
1008 if (++qp->s_cur >= qp->s_size)
1010 qp->s_acked = qp->s_cur;
1011 wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
1012 if (qp->s_acked != qp->s_tail) {
1013 qp->s_state = OP(SEND_LAST);
1014 qp->s_psn = wqe->psn;
1017 if (++qp->s_acked >= qp->s_size)
1019 if (qp->state == IB_QPS_SQD && qp->s_acked == qp->s_cur)
1021 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1027 * do_rc_ack - process an incoming RC ACK
1028 * @qp: the QP the ACK came in on
1029 * @psn: the packet sequence number of the ACK
1030 * @opcode: the opcode of the request that resulted in the ACK
1032 * This is called from qib_rc_rcv_resp() to process an incoming RC ACK
1034 * Called at interrupt level with the QP s_lock held.
1035 * Returns 1 if OK, 0 if current operation should be aborted (NAK).
1037 static int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode,
1038 u64 val, struct qib_ctxtdata *rcd)
1040 struct qib_ibport *ibp;
1041 enum ib_wc_status status;
1042 struct rvt_swqe *wqe;
1048 * Note that NAKs implicitly ACK outstanding SEND and RDMA write
1049 * requests and implicitly NAK RDMA read and atomic requests issued
1050 * before the NAK'ed request. The MSN won't include the NAK'ed
1051 * request but will include an ACK'ed request(s).
1054 if (aeth >> IB_AETH_NAK_SHIFT)
1056 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1057 ibp = to_iport(qp->ibqp.device, qp->port_num);
1060 * The MSN might be for a later WQE than the PSN indicates so
1061 * only complete WQEs that the PSN finishes.
1063 while ((diff = qib_cmp24(ack_psn, wqe->lpsn)) >= 0) {
1065 * RDMA_READ_RESPONSE_ONLY is a special case since
1066 * we want to generate completion events for everything
1067 * before the RDMA read, copy the data, then generate
1068 * the completion for the read.
1070 if (wqe->wr.opcode == IB_WR_RDMA_READ &&
1071 opcode == OP(RDMA_READ_RESPONSE_ONLY) &&
1077 * If this request is a RDMA read or atomic, and the ACK is
1078 * for a later operation, this ACK NAKs the RDMA read or
1079 * atomic. In other words, only a RDMA_READ_LAST or ONLY
1080 * can ACK a RDMA read and likewise for atomic ops. Note
1081 * that the NAK case can only happen if relaxed ordering is
1082 * used and requests are sent after an RDMA read or atomic
1083 * is sent but before the response is received.
1085 if ((wqe->wr.opcode == IB_WR_RDMA_READ &&
1086 (opcode != OP(RDMA_READ_RESPONSE_LAST) || diff != 0)) ||
1087 ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
1088 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) &&
1089 (opcode != OP(ATOMIC_ACKNOWLEDGE) || diff != 0))) {
1090 /* Retry this request. */
1091 if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) {
1092 qp->r_flags |= RVT_R_RDMAR_SEQ;
1093 qib_restart_rc(qp, qp->s_last_psn + 1, 0);
1094 if (list_empty(&qp->rspwait)) {
1095 qp->r_flags |= RVT_R_RSP_SEND;
1097 list_add_tail(&qp->rspwait,
1098 &rcd->qp_wait_list);
1102 * No need to process the ACK/NAK since we are
1103 * restarting an earlier request.
1107 if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
1108 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
1109 u64 *vaddr = wqe->sg_list[0].vaddr;
1112 if (qp->s_num_rd_atomic &&
1113 (wqe->wr.opcode == IB_WR_RDMA_READ ||
1114 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
1115 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) {
1116 qp->s_num_rd_atomic--;
1117 /* Restart sending task if fence is complete */
1118 if ((qp->s_flags & RVT_S_WAIT_FENCE) &&
1119 !qp->s_num_rd_atomic) {
1120 qp->s_flags &= ~(RVT_S_WAIT_FENCE |
1122 qib_schedule_send(qp);
1123 } else if (qp->s_flags & RVT_S_WAIT_RDMAR) {
1124 qp->s_flags &= ~(RVT_S_WAIT_RDMAR |
1126 qib_schedule_send(qp);
1129 wqe = do_rc_completion(qp, wqe, ibp);
1130 if (qp->s_acked == qp->s_tail)
1134 switch (aeth >> IB_AETH_NAK_SHIFT) {
1136 this_cpu_inc(*ibp->rvp.rc_acks);
1137 if (qp->s_acked != qp->s_tail) {
1139 * We are expecting more ACKs so
1140 * reset the retransmit timer.
1142 rvt_mod_retry_timer(qp);
1144 * We can stop resending the earlier packets and
1145 * continue with the next packet the receiver wants.
1147 if (qib_cmp24(qp->s_psn, psn) <= 0)
1148 reset_psn(qp, psn + 1);
1150 /* No more acks - kill all timers */
1151 rvt_stop_rc_timers(qp);
1152 if (qib_cmp24(qp->s_psn, psn) <= 0) {
1153 qp->s_state = OP(SEND_LAST);
1154 qp->s_psn = psn + 1;
1157 if (qp->s_flags & RVT_S_WAIT_ACK) {
1158 qp->s_flags &= ~RVT_S_WAIT_ACK;
1159 qib_schedule_send(qp);
1161 rvt_get_credit(qp, aeth);
1162 qp->s_rnr_retry = qp->s_rnr_retry_cnt;
1163 qp->s_retry = qp->s_retry_cnt;
1164 update_last_psn(qp, psn);
1167 case 1: /* RNR NAK */
1168 ibp->rvp.n_rnr_naks++;
1169 if (qp->s_acked == qp->s_tail)
1171 if (qp->s_flags & RVT_S_WAIT_RNR)
1173 if (qp->s_rnr_retry == 0) {
1174 status = IB_WC_RNR_RETRY_EXC_ERR;
1177 if (qp->s_rnr_retry_cnt < 7)
1180 /* The last valid PSN is the previous PSN. */
1181 update_last_psn(qp, psn - 1);
1183 ibp->rvp.n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK;
1187 qp->s_flags &= ~(RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_ACK);
1188 rvt_stop_rc_timers(qp);
1189 rvt_add_rnr_timer(qp, aeth);
1193 if (qp->s_acked == qp->s_tail)
1195 /* The last valid PSN is the previous PSN. */
1196 update_last_psn(qp, psn - 1);
1197 switch ((aeth >> IB_AETH_CREDIT_SHIFT) &
1198 IB_AETH_CREDIT_MASK) {
1199 case 0: /* PSN sequence error */
1200 ibp->rvp.n_seq_naks++;
1202 * Back up to the responder's expected PSN.
1203 * Note that we might get a NAK in the middle of an
1204 * RDMA READ response which terminates the RDMA
1207 qib_restart_rc(qp, psn, 0);
1208 qib_schedule_send(qp);
1211 case 1: /* Invalid Request */
1212 status = IB_WC_REM_INV_REQ_ERR;
1213 ibp->rvp.n_other_naks++;
1216 case 2: /* Remote Access Error */
1217 status = IB_WC_REM_ACCESS_ERR;
1218 ibp->rvp.n_other_naks++;
1221 case 3: /* Remote Operation Error */
1222 status = IB_WC_REM_OP_ERR;
1223 ibp->rvp.n_other_naks++;
1225 if (qp->s_last == qp->s_acked) {
1226 qib_send_complete(qp, wqe, status);
1227 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
1232 /* Ignore other reserved NAK error codes */
1235 qp->s_retry = qp->s_retry_cnt;
1236 qp->s_rnr_retry = qp->s_rnr_retry_cnt;
1239 default: /* 2: reserved */
1241 /* Ignore reserved NAK codes. */
1246 rvt_stop_rc_timers(qp);
1251 * We have seen an out of sequence RDMA read middle or last packet.
1252 * This ACKs SENDs and RDMA writes up to the first RDMA read or atomic SWQE.
1254 static void rdma_seq_err(struct rvt_qp *qp, struct qib_ibport *ibp, u32 psn,
1255 struct qib_ctxtdata *rcd)
1257 struct rvt_swqe *wqe;
1259 /* Remove QP from retry timer */
1260 rvt_stop_rc_timers(qp);
1262 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1264 while (qib_cmp24(psn, wqe->lpsn) > 0) {
1265 if (wqe->wr.opcode == IB_WR_RDMA_READ ||
1266 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
1267 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)
1269 wqe = do_rc_completion(qp, wqe, ibp);
1272 ibp->rvp.n_rdma_seq++;
1273 qp->r_flags |= RVT_R_RDMAR_SEQ;
1274 qib_restart_rc(qp, qp->s_last_psn + 1, 0);
1275 if (list_empty(&qp->rspwait)) {
1276 qp->r_flags |= RVT_R_RSP_SEND;
1278 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
1283 * qib_rc_rcv_resp - process an incoming RC response packet
1284 * @ibp: the port this packet came in on
1285 * @ohdr: the other headers for this packet
1286 * @data: the packet data
1287 * @tlen: the packet length
1288 * @qp: the QP for this packet
1289 * @opcode: the opcode for this packet
1290 * @psn: the packet sequence number for this packet
1291 * @hdrsize: the header length
1292 * @pmtu: the path MTU
1294 * This is called from qib_rc_rcv() to process an incoming RC response
1295 * packet for the given QP.
1296 * Called at interrupt level.
1298 static void qib_rc_rcv_resp(struct qib_ibport *ibp,
1299 struct ib_other_headers *ohdr,
1300 void *data, u32 tlen,
1303 u32 psn, u32 hdrsize, u32 pmtu,
1304 struct qib_ctxtdata *rcd)
1306 struct rvt_swqe *wqe;
1307 struct qib_pportdata *ppd = ppd_from_ibp(ibp);
1308 enum ib_wc_status status;
1309 unsigned long flags;
1315 if (opcode != OP(RDMA_READ_RESPONSE_MIDDLE)) {
1317 * If ACK'd PSN on SDMA busy list try to make progress to
1318 * reclaim SDMA credits.
1320 if ((qib_cmp24(psn, qp->s_sending_psn) >= 0) &&
1321 (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) {
1324 * If send tasklet not running attempt to progress
1327 if (!(qp->s_flags & RVT_S_BUSY)) {
1328 /* Acquire SDMA Lock */
1329 spin_lock_irqsave(&ppd->sdma_lock, flags);
1330 /* Invoke sdma make progress */
1331 qib_sdma_make_progress(ppd);
1332 /* Release SDMA Lock */
1333 spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1338 spin_lock_irqsave(&qp->s_lock, flags);
1339 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
1342 /* Ignore invalid responses. */
1343 smp_read_barrier_depends(); /* see post_one_send */
1344 if (qib_cmp24(psn, READ_ONCE(qp->s_next_psn)) >= 0)
1347 /* Ignore duplicate responses. */
1348 diff = qib_cmp24(psn, qp->s_last_psn);
1349 if (unlikely(diff <= 0)) {
1350 /* Update credits for "ghost" ACKs */
1351 if (diff == 0 && opcode == OP(ACKNOWLEDGE)) {
1352 aeth = be32_to_cpu(ohdr->u.aeth);
1353 if ((aeth >> IB_AETH_NAK_SHIFT) == 0)
1354 rvt_get_credit(qp, aeth);
1360 * Skip everything other than the PSN we expect, if we are waiting
1361 * for a reply to a restarted RDMA read or atomic op.
1363 if (qp->r_flags & RVT_R_RDMAR_SEQ) {
1364 if (qib_cmp24(psn, qp->s_last_psn + 1) != 0)
1366 qp->r_flags &= ~RVT_R_RDMAR_SEQ;
1369 if (unlikely(qp->s_acked == qp->s_tail))
1371 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1372 status = IB_WC_SUCCESS;
1375 case OP(ACKNOWLEDGE):
1376 case OP(ATOMIC_ACKNOWLEDGE):
1377 case OP(RDMA_READ_RESPONSE_FIRST):
1378 aeth = be32_to_cpu(ohdr->u.aeth);
1379 if (opcode == OP(ATOMIC_ACKNOWLEDGE))
1380 val = ib_u64_get(&ohdr->u.at.atomic_ack_eth);
1383 if (!do_rc_ack(qp, aeth, psn, opcode, val, rcd) ||
1384 opcode != OP(RDMA_READ_RESPONSE_FIRST))
1387 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1388 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
1391 * If this is a response to a resent RDMA read, we
1392 * have to be careful to copy the data to the right
1395 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge,
1399 case OP(RDMA_READ_RESPONSE_MIDDLE):
1400 /* no AETH, no ACK */
1401 if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1)))
1403 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
1406 if (unlikely(tlen != (hdrsize + pmtu + 4)))
1408 if (unlikely(pmtu >= qp->s_rdma_read_len))
1412 * We got a response so update the timeout.
1413 * 4.096 usec. * (1 << qp->timeout)
1415 rvt_mod_retry_timer(qp);
1416 if (qp->s_flags & RVT_S_WAIT_ACK) {
1417 qp->s_flags &= ~RVT_S_WAIT_ACK;
1418 qib_schedule_send(qp);
1421 if (opcode == OP(RDMA_READ_RESPONSE_MIDDLE))
1422 qp->s_retry = qp->s_retry_cnt;
1425 * Update the RDMA receive state but do the copy w/o
1426 * holding the locks and blocking interrupts.
1428 qp->s_rdma_read_len -= pmtu;
1429 update_last_psn(qp, psn);
1430 spin_unlock_irqrestore(&qp->s_lock, flags);
1431 qib_copy_sge(&qp->s_rdma_read_sge, data, pmtu, 0);
1434 case OP(RDMA_READ_RESPONSE_ONLY):
1435 aeth = be32_to_cpu(ohdr->u.aeth);
1436 if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd))
1438 /* Get the number of bytes the message was padded by. */
1439 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
1441 * Check that the data size is >= 0 && <= pmtu.
1442 * Remember to account for the AETH header (4) and
1445 if (unlikely(tlen < (hdrsize + pad + 8)))
1448 * If this is a response to a resent RDMA read, we
1449 * have to be careful to copy the data to the right
1452 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1453 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge,
1457 case OP(RDMA_READ_RESPONSE_LAST):
1458 /* ACKs READ req. */
1459 if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1)))
1461 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
1463 /* Get the number of bytes the message was padded by. */
1464 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
1466 * Check that the data size is >= 1 && <= pmtu.
1467 * Remember to account for the AETH header (4) and
1470 if (unlikely(tlen <= (hdrsize + pad + 8)))
1473 tlen -= hdrsize + pad + 8;
1474 if (unlikely(tlen != qp->s_rdma_read_len))
1476 aeth = be32_to_cpu(ohdr->u.aeth);
1477 qib_copy_sge(&qp->s_rdma_read_sge, data, tlen, 0);
1478 WARN_ON(qp->s_rdma_read_sge.num_sge);
1479 (void) do_rc_ack(qp, aeth, psn,
1480 OP(RDMA_READ_RESPONSE_LAST), 0, rcd);
1485 status = IB_WC_LOC_QP_OP_ERR;
1489 rdma_seq_err(qp, ibp, psn, rcd);
1493 status = IB_WC_LOC_LEN_ERR;
1495 if (qp->s_last == qp->s_acked) {
1496 qib_send_complete(qp, wqe, status);
1497 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
1500 spin_unlock_irqrestore(&qp->s_lock, flags);
1506 * qib_rc_rcv_error - process an incoming duplicate or error RC packet
1507 * @ohdr: the other headers for this packet
1508 * @data: the packet data
1509 * @qp: the QP for this packet
1510 * @opcode: the opcode for this packet
1511 * @psn: the packet sequence number for this packet
1512 * @diff: the difference between the PSN and the expected PSN
1514 * This is called from qib_rc_rcv() to process an unexpected
1515 * incoming RC packet for the given QP.
1516 * Called at interrupt level.
1517 * Return 1 if no more processing is needed; otherwise return 0 to
1518 * schedule a response to be sent.
1520 static int qib_rc_rcv_error(struct ib_other_headers *ohdr,
1526 struct qib_ctxtdata *rcd)
1528 struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
1529 struct rvt_ack_entry *e;
1530 unsigned long flags;
1536 * Packet sequence error.
1537 * A NAK will ACK earlier sends and RDMA writes.
1538 * Don't queue the NAK if we already sent one.
1540 if (!qp->r_nak_state) {
1541 ibp->rvp.n_rc_seqnak++;
1542 qp->r_nak_state = IB_NAK_PSN_ERROR;
1543 /* Use the expected PSN. */
1544 qp->r_ack_psn = qp->r_psn;
1546 * Wait to send the sequence NAK until all packets
1547 * in the receive queue have been processed.
1548 * Otherwise, we end up propagating congestion.
1550 if (list_empty(&qp->rspwait)) {
1551 qp->r_flags |= RVT_R_RSP_NAK;
1553 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
1560 * Handle a duplicate request. Don't re-execute SEND, RDMA
1561 * write or atomic op. Don't NAK errors, just silently drop
1562 * the duplicate request. Note that r_sge, r_len, and
1563 * r_rcv_len may be in use so don't modify them.
1565 * We are supposed to ACK the earliest duplicate PSN but we
1566 * can coalesce an outstanding duplicate ACK. We have to
1567 * send the earliest so that RDMA reads can be restarted at
1568 * the requester's expected PSN.
1570 * First, find where this duplicate PSN falls within the
1571 * ACKs previously sent.
1572 * old_req is true if there is an older response that is scheduled
1573 * to be sent before sending this one.
1577 ibp->rvp.n_rc_dupreq++;
1579 spin_lock_irqsave(&qp->s_lock, flags);
1581 for (i = qp->r_head_ack_queue; ; i = prev) {
1582 if (i == qp->s_tail_ack_queue)
1587 prev = QIB_MAX_RDMA_ATOMIC;
1588 if (prev == qp->r_head_ack_queue) {
1592 e = &qp->s_ack_queue[prev];
1597 if (qib_cmp24(psn, e->psn) >= 0) {
1598 if (prev == qp->s_tail_ack_queue &&
1599 qib_cmp24(psn, e->lpsn) <= 0)
1605 case OP(RDMA_READ_REQUEST): {
1606 struct ib_reth *reth;
1611 * If we didn't find the RDMA read request in the ack queue,
1612 * we can ignore this request.
1614 if (!e || e->opcode != OP(RDMA_READ_REQUEST))
1616 /* RETH comes after BTH */
1617 reth = &ohdr->u.rc.reth;
1619 * Address range must be a subset of the original
1620 * request and start on pmtu boundaries.
1621 * We reuse the old ack_queue slot since the requester
1622 * should not back up and request an earlier PSN for the
1625 offset = ((psn - e->psn) & QIB_PSN_MASK) *
1627 len = be32_to_cpu(reth->length);
1628 if (unlikely(offset + len != e->rdma_sge.sge_length))
1630 if (e->rdma_sge.mr) {
1631 rvt_put_mr(e->rdma_sge.mr);
1632 e->rdma_sge.mr = NULL;
1635 u32 rkey = be32_to_cpu(reth->rkey);
1636 u64 vaddr = be64_to_cpu(reth->vaddr);
1639 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey,
1640 IB_ACCESS_REMOTE_READ);
1644 e->rdma_sge.vaddr = NULL;
1645 e->rdma_sge.length = 0;
1646 e->rdma_sge.sge_length = 0;
1651 qp->s_tail_ack_queue = prev;
1655 case OP(COMPARE_SWAP):
1656 case OP(FETCH_ADD): {
1658 * If we didn't find the atomic request in the ack queue
1659 * or the send tasklet is already backed up to send an
1660 * earlier entry, we can ignore this request.
1662 if (!e || e->opcode != (u8) opcode || old_req)
1664 qp->s_tail_ack_queue = prev;
1670 * Ignore this operation if it doesn't request an ACK
1671 * or an earlier RDMA read or atomic is going to be resent.
1673 if (!(psn & IB_BTH_REQ_ACK) || old_req)
1676 * Resend the most recent ACK if this request is
1677 * after all the previous RDMA reads and atomics.
1679 if (i == qp->r_head_ack_queue) {
1680 spin_unlock_irqrestore(&qp->s_lock, flags);
1681 qp->r_nak_state = 0;
1682 qp->r_ack_psn = qp->r_psn - 1;
1686 * Try to send a simple ACK to work around a Mellanox bug
1687 * which doesn't accept a RDMA read response or atomic
1688 * response as an ACK for earlier SENDs or RDMA writes.
1690 if (!(qp->s_flags & RVT_S_RESP_PENDING)) {
1691 spin_unlock_irqrestore(&qp->s_lock, flags);
1692 qp->r_nak_state = 0;
1693 qp->r_ack_psn = qp->s_ack_queue[i].psn - 1;
1697 * Resend the RDMA read or atomic op which
1698 * ACKs this duplicate request.
1700 qp->s_tail_ack_queue = i;
1703 qp->s_ack_state = OP(ACKNOWLEDGE);
1704 qp->s_flags |= RVT_S_RESP_PENDING;
1705 qp->r_nak_state = 0;
1706 qib_schedule_send(qp);
1709 spin_unlock_irqrestore(&qp->s_lock, flags);
1717 static inline void qib_update_ack_queue(struct rvt_qp *qp, unsigned n)
1722 if (next > QIB_MAX_RDMA_ATOMIC)
1724 qp->s_tail_ack_queue = next;
1725 qp->s_ack_state = OP(ACKNOWLEDGE);
1729 * qib_rc_rcv - process an incoming RC packet
1730 * @rcd: the context pointer
1731 * @hdr: the header of this packet
1732 * @has_grh: true if the header has a GRH
1733 * @data: the packet data
1734 * @tlen: the packet length
1735 * @qp: the QP for this packet
1737 * This is called from qib_qp_rcv() to process an incoming RC packet
1739 * Called at interrupt level.
1741 void qib_rc_rcv(struct qib_ctxtdata *rcd, struct ib_header *hdr,
1742 int has_grh, void *data, u32 tlen, struct rvt_qp *qp)
1744 struct qib_ibport *ibp = &rcd->ppd->ibport_data;
1745 struct ib_other_headers *ohdr;
1751 u32 pmtu = qp->pmtu;
1753 struct ib_reth *reth;
1754 unsigned long flags;
1760 hdrsize = 8 + 12; /* LRH + BTH */
1762 ohdr = &hdr->u.l.oth;
1763 hdrsize = 8 + 40 + 12; /* LRH + GRH + BTH */
1766 opcode = be32_to_cpu(ohdr->bth[0]);
1767 if (qib_ruc_check_hdr(ibp, hdr, has_grh, qp, opcode))
1770 psn = be32_to_cpu(ohdr->bth[2]);
1774 * Process responses (ACKs) before anything else. Note that the
1775 * packet sequence number will be for something in the send work
1776 * queue rather than the expected receive packet sequence number.
1777 * In other words, this QP is the requester.
1779 if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) &&
1780 opcode <= OP(ATOMIC_ACKNOWLEDGE)) {
1781 qib_rc_rcv_resp(ibp, ohdr, data, tlen, qp, opcode, psn,
1782 hdrsize, pmtu, rcd);
1786 /* Compute 24 bits worth of difference. */
1787 diff = qib_cmp24(psn, qp->r_psn);
1788 if (unlikely(diff)) {
1789 if (qib_rc_rcv_error(ohdr, data, qp, opcode, psn, diff, rcd))
1794 /* Check for opcode sequence errors. */
1795 switch (qp->r_state) {
1796 case OP(SEND_FIRST):
1797 case OP(SEND_MIDDLE):
1798 if (opcode == OP(SEND_MIDDLE) ||
1799 opcode == OP(SEND_LAST) ||
1800 opcode == OP(SEND_LAST_WITH_IMMEDIATE))
1804 case OP(RDMA_WRITE_FIRST):
1805 case OP(RDMA_WRITE_MIDDLE):
1806 if (opcode == OP(RDMA_WRITE_MIDDLE) ||
1807 opcode == OP(RDMA_WRITE_LAST) ||
1808 opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE))
1813 if (opcode == OP(SEND_MIDDLE) ||
1814 opcode == OP(SEND_LAST) ||
1815 opcode == OP(SEND_LAST_WITH_IMMEDIATE) ||
1816 opcode == OP(RDMA_WRITE_MIDDLE) ||
1817 opcode == OP(RDMA_WRITE_LAST) ||
1818 opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE))
1821 * Note that it is up to the requester to not send a new
1822 * RDMA read or atomic operation before receiving an ACK
1823 * for the previous operation.
1828 if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST))
1831 /* OK, process the packet. */
1833 case OP(SEND_FIRST):
1834 ret = qib_get_rwqe(qp, 0);
1841 case OP(SEND_MIDDLE):
1842 case OP(RDMA_WRITE_MIDDLE):
1844 /* Check for invalid length PMTU or posted rwqe len. */
1845 if (unlikely(tlen != (hdrsize + pmtu + 4)))
1847 qp->r_rcv_len += pmtu;
1848 if (unlikely(qp->r_rcv_len > qp->r_len))
1850 qib_copy_sge(&qp->r_sge, data, pmtu, 1);
1853 case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE):
1855 ret = qib_get_rwqe(qp, 1);
1863 case OP(SEND_ONLY_WITH_IMMEDIATE):
1864 ret = qib_get_rwqe(qp, 0);
1870 if (opcode == OP(SEND_ONLY))
1871 goto no_immediate_data;
1872 /* FALLTHROUGH for SEND_ONLY_WITH_IMMEDIATE */
1873 case OP(SEND_LAST_WITH_IMMEDIATE):
1875 wc.ex.imm_data = ohdr->u.imm_data;
1877 wc.wc_flags = IB_WC_WITH_IMM;
1880 case OP(RDMA_WRITE_LAST):
1885 /* Get the number of bytes the message was padded by. */
1886 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
1887 /* Check for invalid length. */
1888 /* XXX LAST len should be >= 1 */
1889 if (unlikely(tlen < (hdrsize + pad + 4)))
1891 /* Don't count the CRC. */
1892 tlen -= (hdrsize + pad + 4);
1893 wc.byte_len = tlen + qp->r_rcv_len;
1894 if (unlikely(wc.byte_len > qp->r_len))
1896 qib_copy_sge(&qp->r_sge, data, tlen, 1);
1897 rvt_put_ss(&qp->r_sge);
1899 if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags))
1901 wc.wr_id = qp->r_wr_id;
1902 wc.status = IB_WC_SUCCESS;
1903 if (opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE) ||
1904 opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE))
1905 wc.opcode = IB_WC_RECV_RDMA_WITH_IMM;
1907 wc.opcode = IB_WC_RECV;
1909 wc.src_qp = qp->remote_qpn;
1910 wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr);
1911 wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr);
1912 /* zero fields that are N/A */
1915 wc.dlid_path_bits = 0;
1917 /* Signal completion event if the solicited bit is set. */
1918 rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc,
1920 cpu_to_be32(IB_BTH_SOLICITED)) != 0);
1923 case OP(RDMA_WRITE_FIRST):
1924 case OP(RDMA_WRITE_ONLY):
1925 case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE):
1926 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))
1929 reth = &ohdr->u.rc.reth;
1930 hdrsize += sizeof(*reth);
1931 qp->r_len = be32_to_cpu(reth->length);
1933 qp->r_sge.sg_list = NULL;
1934 if (qp->r_len != 0) {
1935 u32 rkey = be32_to_cpu(reth->rkey);
1936 u64 vaddr = be64_to_cpu(reth->vaddr);
1939 /* Check rkey & NAK */
1940 ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, vaddr,
1941 rkey, IB_ACCESS_REMOTE_WRITE);
1944 qp->r_sge.num_sge = 1;
1946 qp->r_sge.num_sge = 0;
1947 qp->r_sge.sge.mr = NULL;
1948 qp->r_sge.sge.vaddr = NULL;
1949 qp->r_sge.sge.length = 0;
1950 qp->r_sge.sge.sge_length = 0;
1952 if (opcode == OP(RDMA_WRITE_FIRST))
1954 else if (opcode == OP(RDMA_WRITE_ONLY))
1955 goto no_immediate_data;
1956 ret = qib_get_rwqe(qp, 1);
1961 wc.ex.imm_data = ohdr->u.rc.imm_data;
1963 wc.wc_flags = IB_WC_WITH_IMM;
1966 case OP(RDMA_READ_REQUEST): {
1967 struct rvt_ack_entry *e;
1971 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ)))
1973 next = qp->r_head_ack_queue + 1;
1974 /* s_ack_queue is size QIB_MAX_RDMA_ATOMIC+1 so use > not >= */
1975 if (next > QIB_MAX_RDMA_ATOMIC)
1977 spin_lock_irqsave(&qp->s_lock, flags);
1978 if (unlikely(next == qp->s_tail_ack_queue)) {
1979 if (!qp->s_ack_queue[next].sent)
1980 goto nack_inv_unlck;
1981 qib_update_ack_queue(qp, next);
1983 e = &qp->s_ack_queue[qp->r_head_ack_queue];
1984 if (e->opcode == OP(RDMA_READ_REQUEST) && e->rdma_sge.mr) {
1985 rvt_put_mr(e->rdma_sge.mr);
1986 e->rdma_sge.mr = NULL;
1988 reth = &ohdr->u.rc.reth;
1989 len = be32_to_cpu(reth->length);
1991 u32 rkey = be32_to_cpu(reth->rkey);
1992 u64 vaddr = be64_to_cpu(reth->vaddr);
1995 /* Check rkey & NAK */
1996 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr,
1997 rkey, IB_ACCESS_REMOTE_READ);
1999 goto nack_acc_unlck;
2001 * Update the next expected PSN. We add 1 later
2002 * below, so only add the remainder here.
2004 qp->r_psn += rvt_div_mtu(qp, len - 1);
2006 e->rdma_sge.mr = NULL;
2007 e->rdma_sge.vaddr = NULL;
2008 e->rdma_sge.length = 0;
2009 e->rdma_sge.sge_length = 0;
2014 e->lpsn = qp->r_psn;
2016 * We need to increment the MSN here instead of when we
2017 * finish sending the result since a duplicate request would
2018 * increment it more than once.
2022 qp->r_state = opcode;
2023 qp->r_nak_state = 0;
2024 qp->r_head_ack_queue = next;
2026 /* Schedule the send tasklet. */
2027 qp->s_flags |= RVT_S_RESP_PENDING;
2028 qib_schedule_send(qp);
2033 case OP(COMPARE_SWAP):
2034 case OP(FETCH_ADD): {
2035 struct ib_atomic_eth *ateth;
2036 struct rvt_ack_entry *e;
2043 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)))
2045 next = qp->r_head_ack_queue + 1;
2046 if (next > QIB_MAX_RDMA_ATOMIC)
2048 spin_lock_irqsave(&qp->s_lock, flags);
2049 if (unlikely(next == qp->s_tail_ack_queue)) {
2050 if (!qp->s_ack_queue[next].sent)
2051 goto nack_inv_unlck;
2052 qib_update_ack_queue(qp, next);
2054 e = &qp->s_ack_queue[qp->r_head_ack_queue];
2055 if (e->opcode == OP(RDMA_READ_REQUEST) && e->rdma_sge.mr) {
2056 rvt_put_mr(e->rdma_sge.mr);
2057 e->rdma_sge.mr = NULL;
2059 ateth = &ohdr->u.atomic_eth;
2060 vaddr = get_ib_ateth_vaddr(ateth);
2061 if (unlikely(vaddr & (sizeof(u64) - 1)))
2062 goto nack_inv_unlck;
2063 rkey = be32_to_cpu(ateth->rkey);
2064 /* Check rkey & NAK */
2065 if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64),
2067 IB_ACCESS_REMOTE_ATOMIC)))
2068 goto nack_acc_unlck;
2069 /* Perform atomic OP and save result. */
2070 maddr = (atomic64_t *) qp->r_sge.sge.vaddr;
2071 sdata = get_ib_ateth_swap(ateth);
2072 e->atomic_data = (opcode == OP(FETCH_ADD)) ?
2073 (u64) atomic64_add_return(sdata, maddr) - sdata :
2074 (u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr,
2075 get_ib_ateth_compare(ateth),
2077 rvt_put_mr(qp->r_sge.sge.mr);
2078 qp->r_sge.num_sge = 0;
2085 qp->r_state = opcode;
2086 qp->r_nak_state = 0;
2087 qp->r_head_ack_queue = next;
2089 /* Schedule the send tasklet. */
2090 qp->s_flags |= RVT_S_RESP_PENDING;
2091 qib_schedule_send(qp);
2097 /* NAK unknown opcodes. */
2101 qp->r_state = opcode;
2102 qp->r_ack_psn = psn;
2103 qp->r_nak_state = 0;
2104 /* Send an ACK if requested or required. */
2105 if (psn & (1 << 31))
2110 qp->r_nak_state = IB_RNR_NAK | qp->r_min_rnr_timer;
2111 qp->r_ack_psn = qp->r_psn;
2112 /* Queue RNR NAK for later */
2113 if (list_empty(&qp->rspwait)) {
2114 qp->r_flags |= RVT_R_RSP_NAK;
2116 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
2121 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
2122 qp->r_nak_state = IB_NAK_REMOTE_OPERATIONAL_ERROR;
2123 qp->r_ack_psn = qp->r_psn;
2124 /* Queue NAK for later */
2125 if (list_empty(&qp->rspwait)) {
2126 qp->r_flags |= RVT_R_RSP_NAK;
2128 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
2133 spin_unlock_irqrestore(&qp->s_lock, flags);
2135 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
2136 qp->r_nak_state = IB_NAK_INVALID_REQUEST;
2137 qp->r_ack_psn = qp->r_psn;
2138 /* Queue NAK for later */
2139 if (list_empty(&qp->rspwait)) {
2140 qp->r_flags |= RVT_R_RSP_NAK;
2142 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
2147 spin_unlock_irqrestore(&qp->s_lock, flags);
2149 rvt_rc_error(qp, IB_WC_LOC_PROT_ERR);
2150 qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR;
2151 qp->r_ack_psn = qp->r_psn;
2153 qib_send_rc_ack(qp);
2157 spin_unlock_irqrestore(&qp->s_lock, flags);