drivers/infiniband/core/cm.c

   1 /*
   2  * Copyright (c) 2004-2007 Intel Corporation.  All rights reserved.
   3  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
   4  * Copyright (c) 2004, 2005 Voltaire Corporation.  All rights reserved.
   5  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * OpenIB.org BSD license below:
  12  *
  13  *     Redistribution and use in source and binary forms, with or
  14  *     without modification, are permitted provided that the following
  15  *     conditions are met:
  16  *
  17  *      - Redistributions of source code must retain the above
  18  *        copyright notice, this list of conditions and the following
  19  *        disclaimer.
  20  *
  21  *      - Redistributions in binary form must reproduce the above
  22  *        copyright notice, this list of conditions and the following
  23  *        disclaimer in the documentation and/or other materials
  24  *        provided with the distribution.
  25  *
  26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  33  * SOFTWARE.
  34  */
  35
  36 #include <linux/completion.h>
  37 #include <linux/dma-mapping.h>
  38 #include <linux/device.h>
  39 #include <linux/module.h>
  40 #include <linux/err.h>
  41 #include <linux/idr.h>
  42 #include <linux/interrupt.h>
  43 #include <linux/random.h>
  44 #include <linux/rbtree.h>
  45 #include <linux/spinlock.h>
  46 #include <linux/slab.h>
  47 #include <linux/sysfs.h>
  48 #include <linux/workqueue.h>
  49 #include <linux/kdev_t.h>
  50 #include <linux/etherdevice.h>
  51
  52 #include <rdma/ib_cache.h>
  53 #include <rdma/ib_cm.h>
  54 #include "cm_msgs.h"
  55
  56 MODULE_AUTHOR("Sean Hefty");
  57 MODULE_DESCRIPTION("InfiniBand CM");
  58 MODULE_LICENSE("Dual BSD/GPL");
  59
  60 static const char * const ibcm_rej_reason_strs[] = {
  61         [IB_CM_REJ_NO_QP]                       = "no QP",
  62         [IB_CM_REJ_NO_EEC]                      = "no EEC",
  63         [IB_CM_REJ_NO_RESOURCES]                = "no resources",
  64         [IB_CM_REJ_TIMEOUT]                     = "timeout",
  65         [IB_CM_REJ_UNSUPPORTED]                 = "unsupported",
  66         [IB_CM_REJ_INVALID_COMM_ID]             = "invalid comm ID",
  67         [IB_CM_REJ_INVALID_COMM_INSTANCE]       = "invalid comm instance",
  68         [IB_CM_REJ_INVALID_SERVICE_ID]          = "invalid service ID",
  69         [IB_CM_REJ_INVALID_TRANSPORT_TYPE]      = "invalid transport type",
  70         [IB_CM_REJ_STALE_CONN]                  = "stale conn",
  71         [IB_CM_REJ_RDC_NOT_EXIST]               = "RDC not exist",
  72         [IB_CM_REJ_INVALID_GID]                 = "invalid GID",
  73         [IB_CM_REJ_INVALID_LID]                 = "invalid LID",
  74         [IB_CM_REJ_INVALID_SL]                  = "invalid SL",
  75         [IB_CM_REJ_INVALID_TRAFFIC_CLASS]       = "invalid traffic class",
  76         [IB_CM_REJ_INVALID_HOP_LIMIT]           = "invalid hop limit",
  77         [IB_CM_REJ_INVALID_PACKET_RATE]         = "invalid packet rate",
  78         [IB_CM_REJ_INVALID_ALT_GID]             = "invalid alt GID",
  79         [IB_CM_REJ_INVALID_ALT_LID]             = "invalid alt LID",
  80         [IB_CM_REJ_INVALID_ALT_SL]              = "invalid alt SL",
  81         [IB_CM_REJ_INVALID_ALT_TRAFFIC_CLASS]   = "invalid alt traffic class",
  82         [IB_CM_REJ_INVALID_ALT_HOP_LIMIT]       = "invalid alt hop limit",
  83         [IB_CM_REJ_INVALID_ALT_PACKET_RATE]     = "invalid alt packet rate",
  84         [IB_CM_REJ_PORT_CM_REDIRECT]            = "port CM redirect",
  85         [IB_CM_REJ_PORT_REDIRECT]               = "port redirect",
  86         [IB_CM_REJ_INVALID_MTU]                 = "invalid MTU",
  87         [IB_CM_REJ_INSUFFICIENT_RESP_RESOURCES] = "insufficient resp resources",
  88         [IB_CM_REJ_CONSUMER_DEFINED]            = "consumer defined",
  89         [IB_CM_REJ_INVALID_RNR_RETRY]           = "invalid RNR retry",
  90         [IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID]     = "duplicate local comm ID",
  91         [IB_CM_REJ_INVALID_CLASS_VERSION]       = "invalid class version",
  92         [IB_CM_REJ_INVALID_FLOW_LABEL]          = "invalid flow label",
  93         [IB_CM_REJ_INVALID_ALT_FLOW_LABEL]      = "invalid alt flow label",
  94 };
  95
  96 const char *__attribute_const__ ibcm_reject_msg(int reason)
  97 {
  98         size_t index = reason;
  99
 100         if (index < ARRAY_SIZE(ibcm_rej_reason_strs) &&
 101             ibcm_rej_reason_strs[index])
 102                 return ibcm_rej_reason_strs[index];
 103         else
 104                 return "unrecognized reason";
 105 }
 106 EXPORT_SYMBOL(ibcm_reject_msg);
 107
 108 static void cm_add_one(struct ib_device *device);
 109 static void cm_remove_one(struct ib_device *device, void *client_data);
 110
 111 static struct ib_client cm_client = {
 112         .name   = "cm",
 113         .add    = cm_add_one,
 114         .remove = cm_remove_one
 115 };
 116
 117 static struct ib_cm {
 118         spinlock_t lock;
 119         struct list_head device_list;
 120         rwlock_t device_lock;
 121         struct rb_root listen_service_table;
 122         u64 listen_service_id;
 123         /* struct rb_root peer_service_table; todo: fix peer to peer */
 124         struct rb_root remote_qp_table;
 125         struct rb_root remote_id_table;
 126         struct rb_root remote_sidr_table;
 127         struct idr local_id_table;
 128         __be32 random_id_operand;
 129         struct list_head timewait_list;
 130         struct workqueue_struct *wq;
 131         /* Sync on cm change port state */
 132         spinlock_t state_lock;
 133 } cm;
 134
 135 /* Counter indexes ordered by attribute ID */
 136 enum {
 137         CM_REQ_COUNTER,
 138         CM_MRA_COUNTER,
 139         CM_REJ_COUNTER,
 140         CM_REP_COUNTER,
 141         CM_RTU_COUNTER,
 142         CM_DREQ_COUNTER,
 143         CM_DREP_COUNTER,
 144         CM_SIDR_REQ_COUNTER,
 145         CM_SIDR_REP_COUNTER,
 146         CM_LAP_COUNTER,
 147         CM_APR_COUNTER,
 148         CM_ATTR_COUNT,
 149         CM_ATTR_ID_OFFSET = 0x0010,
 150 };
 151
 152 enum {
 153         CM_XMIT,
 154         CM_XMIT_RETRIES,
 155         CM_RECV,
 156         CM_RECV_DUPLICATES,
 157         CM_COUNTER_GROUPS
 158 };
 159
 160 static char const counter_group_names[CM_COUNTER_GROUPS]
 161                                      [sizeof("cm_rx_duplicates")] = {
 162         "cm_tx_msgs", "cm_tx_retries",
 163         "cm_rx_msgs", "cm_rx_duplicates"
 164 };
 165
 166 struct cm_counter_group {
 167         struct kobject obj;
 168         atomic_long_t counter[CM_ATTR_COUNT];
 169 };
 170
 171 struct cm_counter_attribute {
 172         struct attribute attr;
 173         int index;
 174 };
 175
 176 #define CM_COUNTER_ATTR(_name, _index) \
 177 struct cm_counter_attribute cm_##_name##_counter_attr = { \
 178         .attr = { .name = __stringify(_name), .mode = 0444 }, \
 179         .index = _index \
 180 }
 181
 182 static CM_COUNTER_ATTR(req, CM_REQ_COUNTER);
 183 static CM_COUNTER_ATTR(mra, CM_MRA_COUNTER);
 184 static CM_COUNTER_ATTR(rej, CM_REJ_COUNTER);
 185 static CM_COUNTER_ATTR(rep, CM_REP_COUNTER);
 186 static CM_COUNTER_ATTR(rtu, CM_RTU_COUNTER);
 187 static CM_COUNTER_ATTR(dreq, CM_DREQ_COUNTER);
 188 static CM_COUNTER_ATTR(drep, CM_DREP_COUNTER);
 189 static CM_COUNTER_ATTR(sidr_req, CM_SIDR_REQ_COUNTER);
 190 static CM_COUNTER_ATTR(sidr_rep, CM_SIDR_REP_COUNTER);
 191 static CM_COUNTER_ATTR(lap, CM_LAP_COUNTER);
 192 static CM_COUNTER_ATTR(apr, CM_APR_COUNTER);
 193
 194 static struct attribute *cm_counter_default_attrs[] = {
 195         &cm_req_counter_attr.attr,
 196         &cm_mra_counter_attr.attr,
 197         &cm_rej_counter_attr.attr,
 198         &cm_rep_counter_attr.attr,
 199         &cm_rtu_counter_attr.attr,
 200         &cm_dreq_counter_attr.attr,
 201         &cm_drep_counter_attr.attr,
 202         &cm_sidr_req_counter_attr.attr,
 203         &cm_sidr_rep_counter_attr.attr,
 204         &cm_lap_counter_attr.attr,
 205         &cm_apr_counter_attr.attr,
 206         NULL
 207 };
 208
 209 struct cm_port {
 210         struct cm_device *cm_dev;
 211         struct ib_mad_agent *mad_agent;
 212         struct kobject port_obj;
 213         u8 port_num;
 214         struct list_head cm_priv_prim_list;
 215         struct list_head cm_priv_altr_list;
 216         struct cm_counter_group counter_group[CM_COUNTER_GROUPS];
 217 };
 218
 219 struct cm_device {
 220         struct list_head list;
 221         struct ib_device *ib_device;
 222         struct device *device;
 223         u8 ack_delay;
 224         int going_down;
 225         struct cm_port *port[0];
 226 };
 227
 228 struct cm_av {
 229         struct cm_port *port;
 230         union ib_gid dgid;
 231         struct rdma_ah_attr ah_attr;
 232         u16 pkey_index;
 233         u8 timeout;
 234 };
 235
 236 struct cm_work {
 237         struct delayed_work work;
 238         struct list_head list;
 239         struct cm_port *port;
 240         struct ib_mad_recv_wc *mad_recv_wc;     /* Received MADs */
 241         __be32 local_id;                        /* Established / timewait */
 242         __be32 remote_id;
 243         struct ib_cm_event cm_event;
 244         struct sa_path_rec path[0];
 245 };
 246
 247 struct cm_timewait_info {
 248         struct cm_work work;                    /* Must be first. */
 249         struct list_head list;
 250         struct rb_node remote_qp_node;
 251         struct rb_node remote_id_node;
 252         __be64 remote_ca_guid;
 253         __be32 remote_qpn;
 254         u8 inserted_remote_qp;
 255         u8 inserted_remote_id;
 256 };
 257
 258 struct cm_id_private {
 259         struct ib_cm_id id;
 260
 261         struct rb_node service_node;
 262         struct rb_node sidr_id_node;
 263         spinlock_t lock;        /* Do not acquire inside cm.lock */
 264         struct completion comp;
 265         atomic_t refcount;
 266         /* Number of clients sharing this ib_cm_id. Only valid for listeners.
 267          * Protected by the cm.lock spinlock. */
 268         int listen_sharecount;
 269
 270         struct ib_mad_send_buf *msg;
 271         struct cm_timewait_info *timewait_info;
 272         /* todo: use alternate port on send failure */
 273         struct cm_av av;
 274         struct cm_av alt_av;
 275
 276         void *private_data;
 277         __be64 tid;
 278         __be32 local_qpn;
 279         __be32 remote_qpn;
 280         enum ib_qp_type qp_type;
 281         __be32 sq_psn;
 282         __be32 rq_psn;
 283         int timeout_ms;
 284         enum ib_mtu path_mtu;
 285         __be16 pkey;
 286         u8 private_data_len;
 287         u8 max_cm_retries;
 288         u8 peer_to_peer;
 289         u8 responder_resources;
 290         u8 initiator_depth;
 291         u8 retry_count;
 292         u8 rnr_retry_count;
 293         u8 service_timeout;
 294         u8 target_ack_delay;
 295
 296         struct list_head prim_list;
 297         struct list_head altr_list;
 298         /* Indicates that the send port mad is registered and av is set */
 299         int prim_send_port_not_ready;
 300         int altr_send_port_not_ready;
 301
 302         struct list_head work_list;
 303         atomic_t work_count;
 304 };
 305
 306 static void cm_work_handler(struct work_struct *work);
 307
 308 static inline void cm_deref_id(struct cm_id_private *cm_id_priv)
 309 {
 310         if (atomic_dec_and_test(&cm_id_priv->refcount))
 311                 complete(&cm_id_priv->comp);
 312 }
 313
 314 static int cm_alloc_msg(struct cm_id_private *cm_id_priv,
 315                         struct ib_mad_send_buf **msg)
 316 {
 317         struct ib_mad_agent *mad_agent;
 318         struct ib_mad_send_buf *m;
 319         struct ib_ah *ah;
 320         struct cm_av *av;
 321         unsigned long flags, flags2;
 322         int ret = 0;
 323
 324         /* don't let the port to be released till the agent is down */
 325         spin_lock_irqsave(&cm.state_lock, flags2);
 326         spin_lock_irqsave(&cm.lock, flags);
 327         if (!cm_id_priv->prim_send_port_not_ready)
 328                 av = &cm_id_priv->av;
 329         else if (!cm_id_priv->altr_send_port_not_ready &&
 330                  (cm_id_priv->alt_av.port))
 331                 av = &cm_id_priv->alt_av;
 332         else {
 333                 pr_info("%s: not valid CM id\n", __func__);
 334                 ret = -ENODEV;
 335                 spin_unlock_irqrestore(&cm.lock, flags);
 336                 goto out;
 337         }
 338         spin_unlock_irqrestore(&cm.lock, flags);
 339         /* Make sure the port haven't released the mad yet */
 340         mad_agent = cm_id_priv->av.port->mad_agent;
 341         if (!mad_agent) {
 342                 pr_info("%s: not a valid MAD agent\n", __func__);
 343                 ret = -ENODEV;
 344                 goto out;
 345         }
 346         ah = rdma_create_ah(mad_agent->qp->pd, &av->ah_attr);
 347         if (IS_ERR(ah)) {
 348                 ret = PTR_ERR(ah);
 349                 goto out;
 350         }
 351
 352         m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn,
 353                                av->pkey_index,
 354                                0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
 355                                GFP_ATOMIC,
 356                                IB_MGMT_BASE_VERSION);
 357         if (IS_ERR(m)) {
 358                 rdma_destroy_ah(ah);
 359                 ret = PTR_ERR(m);
 360                 goto out;
 361         }
 362
 363         /* Timeout set by caller if response is expected. */
 364         m->ah = ah;
 365         m->retries = cm_id_priv->max_cm_retries;
 366
 367         atomic_inc(&cm_id_priv->refcount);
 368         m->context[0] = cm_id_priv;
 369         *msg = m;
 370
 371 out:
 372         spin_unlock_irqrestore(&cm.state_lock, flags2);
 373         return ret;
 374 }
 375
 376 static int cm_alloc_response_msg(struct cm_port *port,
 377                                  struct ib_mad_recv_wc *mad_recv_wc,
 378                                  struct ib_mad_send_buf **msg)
 379 {
 380         struct ib_mad_send_buf *m;
 381         struct ib_ah *ah;
 382
 383         ah = ib_create_ah_from_wc(port->mad_agent->qp->pd, mad_recv_wc->wc,
 384                                   mad_recv_wc->recv_buf.grh, port->port_num);
 385         if (IS_ERR(ah))
 386                 return PTR_ERR(ah);
 387
 388         m = ib_create_send_mad(port->mad_agent, 1, mad_recv_wc->wc->pkey_index,
 389                                0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
 390                                GFP_ATOMIC,
 391                                IB_MGMT_BASE_VERSION);
 392         if (IS_ERR(m)) {
 393                 rdma_destroy_ah(ah);
 394                 return PTR_ERR(m);
 395         }
 396         m->ah = ah;
 397         *msg = m;
 398         return 0;
 399 }
 400
 401 static void cm_free_msg(struct ib_mad_send_buf *msg)
 402 {
 403         rdma_destroy_ah(msg->ah);
 404         if (msg->context[0])
 405                 cm_deref_id(msg->context[0]);
 406         ib_free_send_mad(msg);
 407 }
 408
 409 static void * cm_copy_private_data(const void *private_data,
 410                                    u8 private_data_len)
 411 {
 412         void *data;
 413
 414         if (!private_data || !private_data_len)
 415                 return NULL;
 416
 417         data = kmemdup(private_data, private_data_len, GFP_KERNEL);
 418         if (!data)
 419                 return ERR_PTR(-ENOMEM);
 420
 421         return data;
 422 }
 423
 424 static void cm_set_private_data(struct cm_id_private *cm_id_priv,
 425                                  void *private_data, u8 private_data_len)
 426 {
 427         if (cm_id_priv->private_data && cm_id_priv->private_data_len)
 428                 kfree(cm_id_priv->private_data);
 429
 430         cm_id_priv->private_data = private_data;
 431         cm_id_priv->private_data_len = private_data_len;
 432 }
 433
 434 static void cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,
 435                                     struct ib_grh *grh, struct cm_av *av)
 436 {
 437         av->port = port;
 438         av->pkey_index = wc->pkey_index;
 439         ib_init_ah_from_wc(port->cm_dev->ib_device, port->port_num, wc,
 440                            grh, &av->ah_attr);
 441 }
 442
 443 static int cm_init_av_by_path(struct sa_path_rec *path, struct cm_av *av,
 444                               struct cm_id_private *cm_id_priv)
 445 {
 446         struct cm_device *cm_dev;
 447         struct cm_port *port = NULL;
 448         unsigned long flags;
 449         int ret;
 450         u8 p;
 451         struct net_device *ndev = ib_get_ndev_from_path(path);
 452
 453         read_lock_irqsave(&cm.device_lock, flags);
 454         list_for_each_entry(cm_dev, &cm.device_list, list) {
 455                 if (!ib_find_cached_gid(cm_dev->ib_device, &path->sgid,
 456                                         sa_conv_pathrec_to_gid_type(path),
 457                                         ndev, &p, NULL)) {
 458                         port = cm_dev->port[p-1];
 459                         break;
 460                 }
 461         }
 462         read_unlock_irqrestore(&cm.device_lock, flags);
 463
 464         if (ndev)
 465                 dev_put(ndev);
 466
 467         if (!port)
 468                 return -EINVAL;
 469
 470         ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num,
 471                                   be16_to_cpu(path->pkey), &av->pkey_index);
 472         if (ret)
 473                 return ret;
 474
 475         av->port = port;
 476         ib_init_ah_from_path(cm_dev->ib_device, port->port_num, path,
 477                              &av->ah_attr);
 478         av->timeout = path->packet_life_time + 1;
 479
 480         spin_lock_irqsave(&cm.lock, flags);
 481         if (&cm_id_priv->av == av)
 482                 list_add_tail(&cm_id_priv->prim_list, &port->cm_priv_prim_list);
 483         else if (&cm_id_priv->alt_av == av)
 484                 list_add_tail(&cm_id_priv->altr_list, &port->cm_priv_altr_list);
 485         else
 486                 ret = -EINVAL;
 487
 488         spin_unlock_irqrestore(&cm.lock, flags);
 489
 490         return ret;
 491 }
 492
 493 static int cm_alloc_id(struct cm_id_private *cm_id_priv)
 494 {
 495         unsigned long flags;
 496         int id;
 497
 498         idr_preload(GFP_KERNEL);
 499         spin_lock_irqsave(&cm.lock, flags);
 500
 501         id = idr_alloc_cyclic(&cm.local_id_table, cm_id_priv, 0, 0, GFP_NOWAIT);
 502
 503         spin_unlock_irqrestore(&cm.lock, flags);
 504         idr_preload_end();
 505
 506         cm_id_priv->id.local_id = (__force __be32)id ^ cm.random_id_operand;
 507         return id < 0 ? id : 0;
 508 }
 509
 510 static void cm_free_id(__be32 local_id)
 511 {
 512         spin_lock_irq(&cm.lock);
 513         idr_remove(&cm.local_id_table,
 514                    (__force int) (local_id ^ cm.random_id_operand));
 515         spin_unlock_irq(&cm.lock);
 516 }
 517
 518 static struct cm_id_private * cm_get_id(__be32 local_id, __be32 remote_id)
 519 {
 520         struct cm_id_private *cm_id_priv;
 521
 522         cm_id_priv = idr_find(&cm.local_id_table,
 523                               (__force int) (local_id ^ cm.random_id_operand));
 524         if (cm_id_priv) {
 525                 if (cm_id_priv->id.remote_id == remote_id)
 526                         atomic_inc(&cm_id_priv->refcount);
 527                 else
 528                         cm_id_priv = NULL;
 529         }
 530
 531         return cm_id_priv;
 532 }
 533
 534 static struct cm_id_private * cm_acquire_id(__be32 local_id, __be32 remote_id)
 535 {
 536         struct cm_id_private *cm_id_priv;
 537
 538         spin_lock_irq(&cm.lock);
 539         cm_id_priv = cm_get_id(local_id, remote_id);
 540         spin_unlock_irq(&cm.lock);
 541
 542         return cm_id_priv;
 543 }
 544
 545 /*
 546  * Trivial helpers to strip endian annotation and compare; the
 547  * endianness doesn't actually matter since we just need a stable
 548  * order for the RB tree.
 549  */
 550 static int be32_lt(__be32 a, __be32 b)
 551 {
 552         return (__force u32) a < (__force u32) b;
 553 }
 554
 555 static int be32_gt(__be32 a, __be32 b)
 556 {
 557         return (__force u32) a > (__force u32) b;
 558 }
 559
 560 static int be64_lt(__be64 a, __be64 b)
 561 {
 562         return (__force u64) a < (__force u64) b;
 563 }
 564
 565 static int be64_gt(__be64 a, __be64 b)
 566 {
 567         return (__force u64) a > (__force u64) b;
 568 }
 569
 570 static struct cm_id_private * cm_insert_listen(struct cm_id_private *cm_id_priv)
 571 {
 572         struct rb_node **link = &cm.listen_service_table.rb_node;
 573         struct rb_node *parent = NULL;
 574         struct cm_id_private *cur_cm_id_priv;
 575         __be64 service_id = cm_id_priv->id.service_id;
 576         __be64 service_mask = cm_id_priv->id.service_mask;
 577
 578         while (*link) {
 579                 parent = *link;
 580                 cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
 581                                           service_node);
 582                 if ((cur_cm_id_priv->id.service_mask & service_id) ==
 583                     (service_mask & cur_cm_id_priv->id.service_id) &&
 584                     (cm_id_priv->id.device == cur_cm_id_priv->id.device))
 585                         return cur_cm_id_priv;
 586
 587                 if (cm_id_priv->id.device < cur_cm_id_priv->id.device)
 588                         link = &(*link)->rb_left;
 589                 else if (cm_id_priv->id.device > cur_cm_id_priv->id.device)
 590                         link = &(*link)->rb_right;
 591                 else if (be64_lt(service_id, cur_cm_id_priv->id.service_id))
 592                         link = &(*link)->rb_left;
 593                 else if (be64_gt(service_id, cur_cm_id_priv->id.service_id))
 594                         link = &(*link)->rb_right;
 595                 else
 596                         link = &(*link)->rb_right;
 597         }
 598         rb_link_node(&cm_id_priv->service_node, parent, link);
 599         rb_insert_color(&cm_id_priv->service_node, &cm.listen_service_table);
 600         return NULL;
 601 }
 602
 603 static struct cm_id_private * cm_find_listen(struct ib_device *device,
 604                                              __be64 service_id)
 605 {
 606         struct rb_node *node = cm.listen_service_table.rb_node;
 607         struct cm_id_private *cm_id_priv;
 608
 609         while (node) {
 610                 cm_id_priv = rb_entry(node, struct cm_id_private, service_node);
 611                 if ((cm_id_priv->id.service_mask & service_id) ==
 612                      cm_id_priv->id.service_id &&
 613                     (cm_id_priv->id.device == device))
 614                         return cm_id_priv;
 615
 616                 if (device < cm_id_priv->id.device)
 617                         node = node->rb_left;
 618                 else if (device > cm_id_priv->id.device)
 619                         node = node->rb_right;
 620                 else if (be64_lt(service_id, cm_id_priv->id.service_id))
 621                         node = node->rb_left;
 622                 else if (be64_gt(service_id, cm_id_priv->id.service_id))
 623                         node = node->rb_right;
 624                 else
 625                         node = node->rb_right;
 626         }
 627         return NULL;
 628 }
 629
 630 static struct cm_timewait_info * cm_insert_remote_id(struct cm_timewait_info
 631                                                      *timewait_info)
 632 {
 633         struct rb_node **link = &cm.remote_id_table.rb_node;
 634         struct rb_node *parent = NULL;
 635         struct cm_timewait_info *cur_timewait_info;
 636         __be64 remote_ca_guid = timewait_info->remote_ca_guid;
 637         __be32 remote_id = timewait_info->work.remote_id;
 638
 639         while (*link) {
 640                 parent = *link;
 641                 cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
 642                                              remote_id_node);
 643                 if (be32_lt(remote_id, cur_timewait_info->work.remote_id))
 644                         link = &(*link)->rb_left;
 645                 else if (be32_gt(remote_id, cur_timewait_info->work.remote_id))
 646                         link = &(*link)->rb_right;
 647                 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 648                         link = &(*link)->rb_left;
 649                 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 650                         link = &(*link)->rb_right;
 651                 else
 652                         return cur_timewait_info;
 653         }
 654         timewait_info->inserted_remote_id = 1;
 655         rb_link_node(&timewait_info->remote_id_node, parent, link);
 656         rb_insert_color(&timewait_info->remote_id_node, &cm.remote_id_table);
 657         return NULL;
 658 }
 659
 660 static struct cm_timewait_info * cm_find_remote_id(__be64 remote_ca_guid,
 661                                                    __be32 remote_id)
 662 {
 663         struct rb_node *node = cm.remote_id_table.rb_node;
 664         struct cm_timewait_info *timewait_info;
 665
 666         while (node) {
 667                 timewait_info = rb_entry(node, struct cm_timewait_info,
 668                                          remote_id_node);
 669                 if (be32_lt(remote_id, timewait_info->work.remote_id))
 670                         node = node->rb_left;
 671                 else if (be32_gt(remote_id, timewait_info->work.remote_id))
 672                         node = node->rb_right;
 673                 else if (be64_lt(remote_ca_guid, timewait_info->remote_ca_guid))
 674                         node = node->rb_left;
 675                 else if (be64_gt(remote_ca_guid, timewait_info->remote_ca_guid))
 676                         node = node->rb_right;
 677                 else
 678                         return timewait_info;
 679         }
 680         return NULL;
 681 }
 682
 683 static struct cm_timewait_info * cm_insert_remote_qpn(struct cm_timewait_info
 684                                                       *timewait_info)
 685 {
 686         struct rb_node **link = &cm.remote_qp_table.rb_node;
 687         struct rb_node *parent = NULL;
 688         struct cm_timewait_info *cur_timewait_info;
 689         __be64 remote_ca_guid = timewait_info->remote_ca_guid;
 690         __be32 remote_qpn = timewait_info->remote_qpn;
 691
 692         while (*link) {
 693                 parent = *link;
 694                 cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
 695                                              remote_qp_node);
 696                 if (be32_lt(remote_qpn, cur_timewait_info->remote_qpn))
 697                         link = &(*link)->rb_left;
 698                 else if (be32_gt(remote_qpn, cur_timewait_info->remote_qpn))
 699                         link = &(*link)->rb_right;
 700                 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 701                         link = &(*link)->rb_left;
 702                 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 703                         link = &(*link)->rb_right;
 704                 else
 705                         return cur_timewait_info;
 706         }
 707         timewait_info->inserted_remote_qp = 1;
 708         rb_link_node(&timewait_info->remote_qp_node, parent, link);
 709         rb_insert_color(&timewait_info->remote_qp_node, &cm.remote_qp_table);
 710         return NULL;
 711 }
 712
 713 static struct cm_id_private * cm_insert_remote_sidr(struct cm_id_private
 714                                                     *cm_id_priv)
 715 {
 716         struct rb_node **link = &cm.remote_sidr_table.rb_node;
 717         struct rb_node *parent = NULL;
 718         struct cm_id_private *cur_cm_id_priv;
 719         union ib_gid *port_gid = &cm_id_priv->av.dgid;
 720         __be32 remote_id = cm_id_priv->id.remote_id;
 721
 722         while (*link) {
 723                 parent = *link;
 724                 cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
 725                                           sidr_id_node);
 726                 if (be32_lt(remote_id, cur_cm_id_priv->id.remote_id))
 727                         link = &(*link)->rb_left;
 728                 else if (be32_gt(remote_id, cur_cm_id_priv->id.remote_id))
 729                         link = &(*link)->rb_right;
 730                 else {
 731                         int cmp;
 732                         cmp = memcmp(port_gid, &cur_cm_id_priv->av.dgid,
 733                                      sizeof *port_gid);
 734                         if (cmp < 0)
 735                                 link = &(*link)->rb_left;
 736                         else if (cmp > 0)
 737                                 link = &(*link)->rb_right;
 738                         else
 739                                 return cur_cm_id_priv;
 740                 }
 741         }
 742         rb_link_node(&cm_id_priv->sidr_id_node, parent, link);
 743         rb_insert_color(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
 744         return NULL;
 745 }
 746
 747 static void cm_reject_sidr_req(struct cm_id_private *cm_id_priv,
 748                                enum ib_cm_sidr_status status)
 749 {
 750         struct ib_cm_sidr_rep_param param;
 751
 752         memset(&param, 0, sizeof param);
 753         param.status = status;
 754         ib_send_cm_sidr_rep(&cm_id_priv->id, &param);
 755 }
 756
 757 struct ib_cm_id *ib_create_cm_id(struct ib_device *device,
 758                                  ib_cm_handler cm_handler,
 759                                  void *context)
 760 {
 761         struct cm_id_private *cm_id_priv;
 762         int ret;
 763
 764         cm_id_priv = kzalloc(sizeof *cm_id_priv, GFP_KERNEL);
 765         if (!cm_id_priv)
 766                 return ERR_PTR(-ENOMEM);
 767
 768         cm_id_priv->id.state = IB_CM_IDLE;
 769         cm_id_priv->id.device = device;
 770         cm_id_priv->id.cm_handler = cm_handler;
 771         cm_id_priv->id.context = context;
 772         cm_id_priv->id.remote_cm_qpn = 1;
 773         ret = cm_alloc_id(cm_id_priv);
 774         if (ret)
 775                 goto error;
 776
 777         spin_lock_init(&cm_id_priv->lock);
 778         init_completion(&cm_id_priv->comp);
 779         INIT_LIST_HEAD(&cm_id_priv->work_list);
 780         INIT_LIST_HEAD(&cm_id_priv->prim_list);
 781         INIT_LIST_HEAD(&cm_id_priv->altr_list);
 782         atomic_set(&cm_id_priv->work_count, -1);
 783         atomic_set(&cm_id_priv->refcount, 1);
 784         return &cm_id_priv->id;
 785
 786 error:
 787         kfree(cm_id_priv);
 788         return ERR_PTR(-ENOMEM);
 789 }
 790 EXPORT_SYMBOL(ib_create_cm_id);
 791
 792 static struct cm_work * cm_dequeue_work(struct cm_id_private *cm_id_priv)
 793 {
 794         struct cm_work *work;
 795
 796         if (list_empty(&cm_id_priv->work_list))
 797                 return NULL;
 798
 799         work = list_entry(cm_id_priv->work_list.next, struct cm_work, list);
 800         list_del(&work->list);
 801         return work;
 802 }
 803
 804 static void cm_free_work(struct cm_work *work)
 805 {
 806         if (work->mad_recv_wc)
 807                 ib_free_recv_mad(work->mad_recv_wc);
 808         kfree(work);
 809 }
 810
 811 static inline int cm_convert_to_ms(int iba_time)
 812 {
 813         /* approximate conversion to ms from 4.096us x 2^iba_time */
 814         return 1 << max(iba_time - 8, 0);
 815 }
 816
 817 /*
 818  * calculate: 4.096x2^ack_timeout = 4.096x2^ack_delay + 2x4.096x2^life_time
 819  * Because of how ack_timeout is stored, adding one doubles the timeout.
 820  * To avoid large timeouts, select the max(ack_delay, life_time + 1), and
 821  * increment it (round up) only if the other is within 50%.
 822  */
 823 static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time)
 824 {
 825         int ack_timeout = packet_life_time + 1;
 826
 827         if (ack_timeout >= ca_ack_delay)
 828                 ack_timeout += (ca_ack_delay >= (ack_timeout - 1));
 829         else
 830                 ack_timeout = ca_ack_delay +
 831                               (ack_timeout >= (ca_ack_delay - 1));
 832
 833         return min(31, ack_timeout);
 834 }
 835
 836 static void cm_cleanup_timewait(struct cm_timewait_info *timewait_info)
 837 {
 838         if (timewait_info->inserted_remote_id) {
 839                 rb_erase(&timewait_info->remote_id_node, &cm.remote_id_table);
 840                 timewait_info->inserted_remote_id = 0;
 841         }
 842
 843         if (timewait_info->inserted_remote_qp) {
 844                 rb_erase(&timewait_info->remote_qp_node, &cm.remote_qp_table);
 845                 timewait_info->inserted_remote_qp = 0;
 846         }
 847 }
 848
 849 static struct cm_timewait_info * cm_create_timewait_info(__be32 local_id)
 850 {
 851         struct cm_timewait_info *timewait_info;
 852
 853         timewait_info = kzalloc(sizeof *timewait_info, GFP_KERNEL);
 854         if (!timewait_info)
 855                 return ERR_PTR(-ENOMEM);
 856
 857         timewait_info->work.local_id = local_id;
 858         INIT_DELAYED_WORK(&timewait_info->work.work, cm_work_handler);
 859         timewait_info->work.cm_event.event = IB_CM_TIMEWAIT_EXIT;
 860         return timewait_info;
 861 }
 862
 863 static void cm_enter_timewait(struct cm_id_private *cm_id_priv)
 864 {
 865         int wait_time;
 866         unsigned long flags;
 867         struct cm_device *cm_dev;
 868
 869         cm_dev = ib_get_client_data(cm_id_priv->id.device, &cm_client);
 870         if (!cm_dev)
 871                 return;
 872
 873         spin_lock_irqsave(&cm.lock, flags);
 874         cm_cleanup_timewait(cm_id_priv->timewait_info);
 875         list_add_tail(&cm_id_priv->timewait_info->list, &cm.timewait_list);
 876         spin_unlock_irqrestore(&cm.lock, flags);
 877
 878         /*
 879          * The cm_id could be destroyed by the user before we exit timewait.
 880          * To protect against this, we search for the cm_id after exiting
 881          * timewait before notifying the user that we've exited timewait.
 882          */
 883         cm_id_priv->id.state = IB_CM_TIMEWAIT;
 884         wait_time = cm_convert_to_ms(cm_id_priv->av.timeout);
 885
 886         /* Check if the device started its remove_one */
 887         spin_lock_irqsave(&cm.lock, flags);
 888         if (!cm_dev->going_down)
 889                 queue_delayed_work(cm.wq, &cm_id_priv->timewait_info->work.work,
 890                                    msecs_to_jiffies(wait_time));
 891         spin_unlock_irqrestore(&cm.lock, flags);
 892
 893         cm_id_priv->timewait_info = NULL;
 894 }
 895
 896 static void cm_reset_to_idle(struct cm_id_private *cm_id_priv)
 897 {
 898         unsigned long flags;
 899
 900         cm_id_priv->id.state = IB_CM_IDLE;
 901         if (cm_id_priv->timewait_info) {
 902                 spin_lock_irqsave(&cm.lock, flags);
 903                 cm_cleanup_timewait(cm_id_priv->timewait_info);
 904                 spin_unlock_irqrestore(&cm.lock, flags);
 905                 kfree(cm_id_priv->timewait_info);
 906                 cm_id_priv->timewait_info = NULL;
 907         }
 908 }
 909
 910 static void cm_destroy_id(struct ib_cm_id *cm_id, int err)
 911 {
 912         struct cm_id_private *cm_id_priv;
 913         struct cm_work *work;
 914
 915         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
 916 retest:
 917         spin_lock_irq(&cm_id_priv->lock);
 918         switch (cm_id->state) {
 919         case IB_CM_LISTEN:
 920                 spin_unlock_irq(&cm_id_priv->lock);
 921
 922                 spin_lock_irq(&cm.lock);
 923                 if (--cm_id_priv->listen_sharecount > 0) {
 924                         /* The id is still shared. */
 925                         cm_deref_id(cm_id_priv);
 926                         spin_unlock_irq(&cm.lock);
 927                         return;
 928                 }
 929                 rb_erase(&cm_id_priv->service_node, &cm.listen_service_table);
 930                 spin_unlock_irq(&cm.lock);
 931                 break;
 932         case IB_CM_SIDR_REQ_SENT:
 933                 cm_id->state = IB_CM_IDLE;
 934                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
 935                 spin_unlock_irq(&cm_id_priv->lock);
 936                 break;
 937         case IB_CM_SIDR_REQ_RCVD:
 938                 spin_unlock_irq(&cm_id_priv->lock);
 939                 cm_reject_sidr_req(cm_id_priv, IB_SIDR_REJECT);
 940                 spin_lock_irq(&cm.lock);
 941                 if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node))
 942                         rb_erase(&cm_id_priv->sidr_id_node,
 943                                  &cm.remote_sidr_table);
 944                 spin_unlock_irq(&cm.lock);
 945                 break;
 946         case IB_CM_REQ_SENT:
 947         case IB_CM_MRA_REQ_RCVD:
 948                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
 949                 spin_unlock_irq(&cm_id_priv->lock);
 950                 ib_send_cm_rej(cm_id, IB_CM_REJ_TIMEOUT,
 951                                &cm_id_priv->id.device->node_guid,
 952                                sizeof cm_id_priv->id.device->node_guid,
 953                                NULL, 0);
 954                 break;
 955         case IB_CM_REQ_RCVD:
 956                 if (err == -ENOMEM) {
 957                         /* Do not reject to allow future retries. */
 958                         cm_reset_to_idle(cm_id_priv);
 959                         spin_unlock_irq(&cm_id_priv->lock);
 960                 } else {
 961                         spin_unlock_irq(&cm_id_priv->lock);
 962                         ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED,
 963                                        NULL, 0, NULL, 0);
 964                 }
 965                 break;
 966         case IB_CM_REP_SENT:
 967         case IB_CM_MRA_REP_RCVD:
 968                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
 969                 /* Fall through */
 970         case IB_CM_MRA_REQ_SENT:
 971         case IB_CM_REP_RCVD:
 972         case IB_CM_MRA_REP_SENT:
 973                 spin_unlock_irq(&cm_id_priv->lock);
 974                 ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED,
 975                                NULL, 0, NULL, 0);
 976                 break;
 977         case IB_CM_ESTABLISHED:
 978                 spin_unlock_irq(&cm_id_priv->lock);
 979                 if (cm_id_priv->qp_type == IB_QPT_XRC_TGT)
 980                         break;
 981                 ib_send_cm_dreq(cm_id, NULL, 0);
 982                 goto retest;
 983         case IB_CM_DREQ_SENT:
 984                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
 985                 cm_enter_timewait(cm_id_priv);
 986                 spin_unlock_irq(&cm_id_priv->lock);
 987                 break;
 988         case IB_CM_DREQ_RCVD:
 989                 spin_unlock_irq(&cm_id_priv->lock);
 990                 ib_send_cm_drep(cm_id, NULL, 0);
 991                 break;
 992         default:
 993                 spin_unlock_irq(&cm_id_priv->lock);
 994                 break;
 995         }
 996
 997         spin_lock_irq(&cm.lock);
 998         if (!list_empty(&cm_id_priv->altr_list) &&
 999             (!cm_id_priv->altr_send_port_not_ready))
1000                 list_del(&cm_id_priv->altr_list);
1001         if (!list_empty(&cm_id_priv->prim_list) &&
1002             (!cm_id_priv->prim_send_port_not_ready))
1003                 list_del(&cm_id_priv->prim_list);
1004         spin_unlock_irq(&cm.lock);
1005
1006         cm_free_id(cm_id->local_id);
1007         cm_deref_id(cm_id_priv);
1008         wait_for_completion(&cm_id_priv->comp);
1009         while ((work = cm_dequeue_work(cm_id_priv)) != NULL)
1010                 cm_free_work(work);
1011         kfree(cm_id_priv->private_data);
1012         kfree(cm_id_priv);
1013 }
1014
1015 void ib_destroy_cm_id(struct ib_cm_id *cm_id)
1016 {
1017         cm_destroy_id(cm_id, 0);
1018 }
1019 EXPORT_SYMBOL(ib_destroy_cm_id);
1020
1021 /**
1022  * __ib_cm_listen - Initiates listening on the specified service ID for
1023  *   connection and service ID resolution requests.
1024  * @cm_id: Connection identifier associated with the listen request.
1025  * @service_id: Service identifier matched against incoming connection
1026  *   and service ID resolution requests.  The service ID should be specified
1027  *   network-byte order.  If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1028  *   assign a service ID to the caller.
1029  * @service_mask: Mask applied to service ID used to listen across a
1030  *   range of service IDs.  If set to 0, the service ID is matched
1031  *   exactly.  This parameter is ignored if %service_id is set to
1032  *   IB_CM_ASSIGN_SERVICE_ID.
1033  */
1034 static int __ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id,
1035                           __be64 service_mask)
1036 {
1037         struct cm_id_private *cm_id_priv, *cur_cm_id_priv;
1038         int ret = 0;
1039
1040         service_mask = service_mask ? service_mask : ~cpu_to_be64(0);
1041         service_id &= service_mask;
1042         if ((service_id & IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID &&
1043             (service_id != IB_CM_ASSIGN_SERVICE_ID))
1044                 return -EINVAL;
1045
1046         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1047         if (cm_id->state != IB_CM_IDLE)
1048                 return -EINVAL;
1049
1050         cm_id->state = IB_CM_LISTEN;
1051         ++cm_id_priv->listen_sharecount;
1052
1053         if (service_id == IB_CM_ASSIGN_SERVICE_ID) {
1054                 cm_id->service_id = cpu_to_be64(cm.listen_service_id++);
1055                 cm_id->service_mask = ~cpu_to_be64(0);
1056         } else {
1057                 cm_id->service_id = service_id;
1058                 cm_id->service_mask = service_mask;
1059         }
1060         cur_cm_id_priv = cm_insert_listen(cm_id_priv);
1061
1062         if (cur_cm_id_priv) {
1063                 cm_id->state = IB_CM_IDLE;
1064                 --cm_id_priv->listen_sharecount;
1065                 ret = -EBUSY;
1066         }
1067         return ret;
1068 }
1069
1070 int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id, __be64 service_mask)
1071 {
1072         unsigned long flags;
1073         int ret;
1074
1075         spin_lock_irqsave(&cm.lock, flags);
1076         ret = __ib_cm_listen(cm_id, service_id, service_mask);
1077         spin_unlock_irqrestore(&cm.lock, flags);
1078
1079         return ret;
1080 }
1081 EXPORT_SYMBOL(ib_cm_listen);
1082
1083 /**
1084  * Create a new listening ib_cm_id and listen on the given service ID.
1085  *
1086  * If there's an existing ID listening on that same device and service ID,
1087  * return it.
1088  *
1089  * @device: Device associated with the cm_id.  All related communication will
1090  * be associated with the specified device.
1091  * @cm_handler: Callback invoked to notify the user of CM events.
1092  * @service_id: Service identifier matched against incoming connection
1093  *   and service ID resolution requests.  The service ID should be specified
1094  *   network-byte order.  If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1095  *   assign a service ID to the caller.
1096  *
1097  * Callers should call ib_destroy_cm_id when done with the listener ID.
1098  */
1099 struct ib_cm_id *ib_cm_insert_listen(struct ib_device *device,
1100                                      ib_cm_handler cm_handler,
1101                                      __be64 service_id)
1102 {
1103         struct cm_id_private *cm_id_priv;
1104         struct ib_cm_id *cm_id;
1105         unsigned long flags;
1106         int err = 0;
1107
1108         /* Create an ID in advance, since the creation may sleep */
1109         cm_id = ib_create_cm_id(device, cm_handler, NULL);
1110         if (IS_ERR(cm_id))
1111                 return cm_id;
1112
1113         spin_lock_irqsave(&cm.lock, flags);
1114
1115         if (service_id == IB_CM_ASSIGN_SERVICE_ID)
1116                 goto new_id;
1117
1118         /* Find an existing ID */
1119         cm_id_priv = cm_find_listen(device, service_id);
1120         if (cm_id_priv) {
1121                 if (cm_id->cm_handler != cm_handler || cm_id->context) {
1122                         /* Sharing an ib_cm_id with different handlers is not
1123                          * supported */
1124                         spin_unlock_irqrestore(&cm.lock, flags);
1125                         return ERR_PTR(-EINVAL);
1126                 }
1127                 atomic_inc(&cm_id_priv->refcount);
1128                 ++cm_id_priv->listen_sharecount;
1129                 spin_unlock_irqrestore(&cm.lock, flags);
1130
1131                 ib_destroy_cm_id(cm_id);
1132                 cm_id = &cm_id_priv->id;
1133                 return cm_id;
1134         }
1135
1136 new_id:
1137         /* Use newly created ID */
1138         err = __ib_cm_listen(cm_id, service_id, 0);
1139
1140         spin_unlock_irqrestore(&cm.lock, flags);
1141
1142         if (err) {
1143                 ib_destroy_cm_id(cm_id);
1144                 return ERR_PTR(err);
1145         }
1146         return cm_id;
1147 }
1148 EXPORT_SYMBOL(ib_cm_insert_listen);
1149
1150 static __be64 cm_form_tid(struct cm_id_private *cm_id_priv,
1151                           enum cm_msg_sequence msg_seq)
1152 {
1153         u64 hi_tid, low_tid;
1154
1155         hi_tid   = ((u64) cm_id_priv->av.port->mad_agent->hi_tid) << 32;
1156         low_tid  = (u64) ((__force u32)cm_id_priv->id.local_id |
1157                           (msg_seq << 30));
1158         return cpu_to_be64(hi_tid | low_tid);
1159 }
1160
1161 static void cm_format_mad_hdr(struct ib_mad_hdr *hdr,
1162                               __be16 attr_id, __be64 tid)
1163 {
1164         hdr->base_version  = IB_MGMT_BASE_VERSION;
1165         hdr->mgmt_class    = IB_MGMT_CLASS_CM;
1166         hdr->class_version = IB_CM_CLASS_VERSION;
1167         hdr->method        = IB_MGMT_METHOD_SEND;
1168         hdr->attr_id       = attr_id;
1169         hdr->tid           = tid;
1170 }
1171
1172 static void cm_format_req(struct cm_req_msg *req_msg,
1173                           struct cm_id_private *cm_id_priv,
1174                           struct ib_cm_req_param *param)
1175 {
1176         struct sa_path_rec *pri_path = param->primary_path;
1177         struct sa_path_rec *alt_path = param->alternate_path;
1178
1179         cm_format_mad_hdr(&req_msg->hdr, CM_REQ_ATTR_ID,
1180                           cm_form_tid(cm_id_priv, CM_MSG_SEQUENCE_REQ));
1181
1182         req_msg->local_comm_id = cm_id_priv->id.local_id;
1183         req_msg->service_id = param->service_id;
1184         req_msg->local_ca_guid = cm_id_priv->id.device->node_guid;
1185         cm_req_set_local_qpn(req_msg, cpu_to_be32(param->qp_num));
1186         cm_req_set_init_depth(req_msg, param->initiator_depth);
1187         cm_req_set_remote_resp_timeout(req_msg,
1188                                        param->remote_cm_response_timeout);
1189         cm_req_set_qp_type(req_msg, param->qp_type);
1190         cm_req_set_flow_ctrl(req_msg, param->flow_control);
1191         cm_req_set_starting_psn(req_msg, cpu_to_be32(param->starting_psn));
1192         cm_req_set_local_resp_timeout(req_msg,
1193                                       param->local_cm_response_timeout);
1194         req_msg->pkey = param->primary_path->pkey;
1195         cm_req_set_path_mtu(req_msg, param->primary_path->mtu);
1196         cm_req_set_max_cm_retries(req_msg, param->max_cm_retries);
1197
1198         if (param->qp_type != IB_QPT_XRC_INI) {
1199                 cm_req_set_resp_res(req_msg, param->responder_resources);
1200                 cm_req_set_retry_count(req_msg, param->retry_count);
1201                 cm_req_set_rnr_retry_count(req_msg, param->rnr_retry_count);
1202                 cm_req_set_srq(req_msg, param->srq);
1203         }
1204
1205         if (pri_path->hop_limit <= 1) {
1206                 req_msg->primary_local_lid = sa_path_get_slid(pri_path);
1207                 req_msg->primary_remote_lid = sa_path_get_dlid(pri_path);
1208         } else {
1209                 /* Work-around until there's a way to obtain remote LID info */
1210                 req_msg->primary_local_lid = IB_LID_PERMISSIVE;
1211                 req_msg->primary_remote_lid = IB_LID_PERMISSIVE;
1212         }
1213         req_msg->primary_local_gid = pri_path->sgid;
1214         req_msg->primary_remote_gid = pri_path->dgid;
1215         cm_req_set_primary_flow_label(req_msg, pri_path->flow_label);
1216         cm_req_set_primary_packet_rate(req_msg, pri_path->rate);
1217         req_msg->primary_traffic_class = pri_path->traffic_class;
1218         req_msg->primary_hop_limit = pri_path->hop_limit;
1219         cm_req_set_primary_sl(req_msg, pri_path->sl);
1220         cm_req_set_primary_subnet_local(req_msg, (pri_path->hop_limit <= 1));
1221         cm_req_set_primary_local_ack_timeout(req_msg,
1222                 cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1223                                pri_path->packet_life_time));
1224
1225         if (alt_path) {
1226                 if (alt_path->hop_limit <= 1) {
1227                         req_msg->alt_local_lid = sa_path_get_slid(alt_path);
1228                         req_msg->alt_remote_lid = sa_path_get_dlid(alt_path);
1229                 } else {
1230                         req_msg->alt_local_lid = IB_LID_PERMISSIVE;
1231                         req_msg->alt_remote_lid = IB_LID_PERMISSIVE;
1232                 }
1233                 req_msg->alt_local_gid = alt_path->sgid;
1234                 req_msg->alt_remote_gid = alt_path->dgid;
1235                 cm_req_set_alt_flow_label(req_msg,
1236                                           alt_path->flow_label);
1237                 cm_req_set_alt_packet_rate(req_msg, alt_path->rate);
1238                 req_msg->alt_traffic_class = alt_path->traffic_class;
1239                 req_msg->alt_hop_limit = alt_path->hop_limit;
1240                 cm_req_set_alt_sl(req_msg, alt_path->sl);
1241                 cm_req_set_alt_subnet_local(req_msg, (alt_path->hop_limit <= 1));
1242                 cm_req_set_alt_local_ack_timeout(req_msg,
1243                         cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1244                                        alt_path->packet_life_time));
1245         }
1246
1247         if (param->private_data && param->private_data_len)
1248                 memcpy(req_msg->private_data, param->private_data,
1249                        param->private_data_len);
1250 }
1251
1252 static int cm_validate_req_param(struct ib_cm_req_param *param)
1253 {
1254         /* peer-to-peer not supported */
1255         if (param->peer_to_peer)
1256                 return -EINVAL;
1257
1258         if (!param->primary_path)
1259                 return -EINVAL;
1260
1261         if (param->qp_type != IB_QPT_RC && param->qp_type != IB_QPT_UC &&
1262             param->qp_type != IB_QPT_XRC_INI)
1263                 return -EINVAL;
1264
1265         if (param->private_data &&
1266             param->private_data_len > IB_CM_REQ_PRIVATE_DATA_SIZE)
1267                 return -EINVAL;
1268
1269         if (param->alternate_path &&
1270             (param->alternate_path->pkey != param->primary_path->pkey ||
1271              param->alternate_path->mtu != param->primary_path->mtu))
1272                 return -EINVAL;
1273
1274         return 0;
1275 }
1276
1277 int ib_send_cm_req(struct ib_cm_id *cm_id,
1278                    struct ib_cm_req_param *param)
1279 {
1280         struct cm_id_private *cm_id_priv;
1281         struct cm_req_msg *req_msg;
1282         unsigned long flags;
1283         int ret;
1284
1285         ret = cm_validate_req_param(param);
1286         if (ret)
1287                 return ret;
1288
1289         /* Verify that we're not in timewait. */
1290         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1291         spin_lock_irqsave(&cm_id_priv->lock, flags);
1292         if (cm_id->state != IB_CM_IDLE) {
1293                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1294                 ret = -EINVAL;
1295                 goto out;
1296         }
1297         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1298
1299         cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
1300                                                             id.local_id);
1301         if (IS_ERR(cm_id_priv->timewait_info)) {
1302                 ret = PTR_ERR(cm_id_priv->timewait_info);
1303                 goto out;
1304         }
1305
1306         ret = cm_init_av_by_path(param->primary_path, &cm_id_priv->av,
1307                                  cm_id_priv);
1308         if (ret)
1309                 goto error1;
1310         if (param->alternate_path) {
1311                 ret = cm_init_av_by_path(param->alternate_path,
1312                                          &cm_id_priv->alt_av, cm_id_priv);
1313                 if (ret)
1314                         goto error1;
1315         }
1316         cm_id->service_id = param->service_id;
1317         cm_id->service_mask = ~cpu_to_be64(0);
1318         cm_id_priv->timeout_ms = cm_convert_to_ms(
1319                                     param->primary_path->packet_life_time) * 2 +
1320                                  cm_convert_to_ms(
1321                                     param->remote_cm_response_timeout);
1322         cm_id_priv->max_cm_retries = param->max_cm_retries;
1323         cm_id_priv->initiator_depth = param->initiator_depth;
1324         cm_id_priv->responder_resources = param->responder_resources;
1325         cm_id_priv->retry_count = param->retry_count;
1326         cm_id_priv->path_mtu = param->primary_path->mtu;
1327         cm_id_priv->pkey = param->primary_path->pkey;
1328         cm_id_priv->qp_type = param->qp_type;
1329
1330         ret = cm_alloc_msg(cm_id_priv, &cm_id_priv->msg);
1331         if (ret)
1332                 goto error1;
1333
1334         req_msg = (struct cm_req_msg *) cm_id_priv->msg->mad;
1335         cm_format_req(req_msg, cm_id_priv, param);
1336         cm_id_priv->tid = req_msg->hdr.tid;
1337         cm_id_priv->msg->timeout_ms = cm_id_priv->timeout_ms;
1338         cm_id_priv->msg->context[1] = (void *) (unsigned long) IB_CM_REQ_SENT;
1339
1340         cm_id_priv->local_qpn = cm_req_get_local_qpn(req_msg);
1341         cm_id_priv->rq_psn = cm_req_get_starting_psn(req_msg);
1342
1343         spin_lock_irqsave(&cm_id_priv->lock, flags);
1344         ret = ib_post_send_mad(cm_id_priv->msg, NULL);
1345         if (ret) {
1346                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1347                 goto error2;
1348         }
1349         BUG_ON(cm_id->state != IB_CM_IDLE);
1350         cm_id->state = IB_CM_REQ_SENT;
1351         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1352         return 0;
1353
1354 error2: cm_free_msg(cm_id_priv->msg);
1355 error1: kfree(cm_id_priv->timewait_info);
1356 out:    return ret;
1357 }
1358 EXPORT_SYMBOL(ib_send_cm_req);
1359
1360 static int cm_issue_rej(struct cm_port *port,
1361                         struct ib_mad_recv_wc *mad_recv_wc,
1362                         enum ib_cm_rej_reason reason,
1363                         enum cm_msg_response msg_rejected,
1364                         void *ari, u8 ari_length)
1365 {
1366         struct ib_mad_send_buf *msg = NULL;
1367         struct cm_rej_msg *rej_msg, *rcv_msg;
1368         int ret;
1369
1370         ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
1371         if (ret)
1372                 return ret;
1373
1374         /* We just need common CM header information.  Cast to any message. */
1375         rcv_msg = (struct cm_rej_msg *) mad_recv_wc->recv_buf.mad;
1376         rej_msg = (struct cm_rej_msg *) msg->mad;
1377
1378         cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, rcv_msg->hdr.tid);
1379         rej_msg->remote_comm_id = rcv_msg->local_comm_id;
1380         rej_msg->local_comm_id = rcv_msg->remote_comm_id;
1381         cm_rej_set_msg_rejected(rej_msg, msg_rejected);
1382         rej_msg->reason = cpu_to_be16(reason);
1383
1384         if (ari && ari_length) {
1385                 cm_rej_set_reject_info_len(rej_msg, ari_length);
1386                 memcpy(rej_msg->ari, ari, ari_length);
1387         }
1388
1389         ret = ib_post_send_mad(msg, NULL);
1390         if (ret)
1391                 cm_free_msg(msg);
1392
1393         return ret;
1394 }
1395
1396 static inline int cm_is_active_peer(__be64 local_ca_guid, __be64 remote_ca_guid,
1397                                     __be32 local_qpn, __be32 remote_qpn)
1398 {
1399         return (be64_to_cpu(local_ca_guid) > be64_to_cpu(remote_ca_guid) ||
1400                 ((local_ca_guid == remote_ca_guid) &&
1401                  (be32_to_cpu(local_qpn) > be32_to_cpu(remote_qpn))));
1402 }
1403
1404 static void cm_format_paths_from_req(struct cm_req_msg *req_msg,
1405                                      struct sa_path_rec *primary_path,
1406                                      struct sa_path_rec *alt_path)
1407 {
1408         primary_path->dgid = req_msg->primary_local_gid;
1409         primary_path->sgid = req_msg->primary_remote_gid;
1410         sa_path_set_dlid(primary_path, req_msg->primary_local_lid);
1411         sa_path_set_slid(primary_path, req_msg->primary_remote_lid);
1412         primary_path->flow_label = cm_req_get_primary_flow_label(req_msg);
1413         primary_path->hop_limit = req_msg->primary_hop_limit;
1414         primary_path->traffic_class = req_msg->primary_traffic_class;
1415         primary_path->reversible = 1;
1416         primary_path->pkey = req_msg->pkey;
1417         primary_path->sl = cm_req_get_primary_sl(req_msg);
1418         primary_path->mtu_selector = IB_SA_EQ;
1419         primary_path->mtu = cm_req_get_path_mtu(req_msg);
1420         primary_path->rate_selector = IB_SA_EQ;
1421         primary_path->rate = cm_req_get_primary_packet_rate(req_msg);
1422         primary_path->packet_life_time_selector = IB_SA_EQ;
1423         primary_path->packet_life_time =
1424                 cm_req_get_primary_local_ack_timeout(req_msg);
1425         primary_path->packet_life_time -= (primary_path->packet_life_time > 0);
1426         sa_path_set_service_id(primary_path, req_msg->service_id);
1427
1428         if (req_msg->alt_local_lid) {
1429                 alt_path->dgid = req_msg->alt_local_gid;
1430                 alt_path->sgid = req_msg->alt_remote_gid;
1431                 sa_path_set_dlid(alt_path, req_msg->alt_local_lid);
1432                 sa_path_set_slid(alt_path, req_msg->alt_remote_lid);
1433                 alt_path->flow_label = cm_req_get_alt_flow_label(req_msg);
1434                 alt_path->hop_limit = req_msg->alt_hop_limit;
1435                 alt_path->traffic_class = req_msg->alt_traffic_class;
1436                 alt_path->reversible = 1;
1437                 alt_path->pkey = req_msg->pkey;
1438                 alt_path->sl = cm_req_get_alt_sl(req_msg);
1439                 alt_path->mtu_selector = IB_SA_EQ;
1440                 alt_path->mtu = cm_req_get_path_mtu(req_msg);
1441                 alt_path->rate_selector = IB_SA_EQ;
1442                 alt_path->rate = cm_req_get_alt_packet_rate(req_msg);
1443                 alt_path->packet_life_time_selector = IB_SA_EQ;
1444                 alt_path->packet_life_time =
1445                         cm_req_get_alt_local_ack_timeout(req_msg);
1446                 alt_path->packet_life_time -= (alt_path->packet_life_time > 0);
1447                 sa_path_set_service_id(alt_path, req_msg->service_id);
1448         }
1449 }
1450
1451 static u16 cm_get_bth_pkey(struct cm_work *work)
1452 {
1453         struct ib_device *ib_dev = work->port->cm_dev->ib_device;
1454         u8 port_num = work->port->port_num;
1455         u16 pkey_index = work->mad_recv_wc->wc->pkey_index;
1456         u16 pkey;
1457         int ret;
1458
1459         ret = ib_get_cached_pkey(ib_dev, port_num, pkey_index, &pkey);
1460         if (ret) {
1461                 dev_warn_ratelimited(&ib_dev->dev, "ib_cm: Couldn't retrieve pkey for incoming request (port %d, pkey index %d). %d\n",
1462                                      port_num, pkey_index, ret);
1463                 return 0;
1464         }
1465
1466         return pkey;
1467 }
1468
1469 static void cm_format_req_event(struct cm_work *work,
1470                                 struct cm_id_private *cm_id_priv,
1471                                 struct ib_cm_id *listen_id)
1472 {
1473         struct cm_req_msg *req_msg;
1474         struct ib_cm_req_event_param *param;
1475
1476         req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1477         param = &work->cm_event.param.req_rcvd;
1478         param->listen_id = listen_id;
1479         param->bth_pkey = cm_get_bth_pkey(work);
1480         param->port = cm_id_priv->av.port->port_num;
1481         param->primary_path = &work->path[0];
1482         if (req_msg->alt_local_lid)
1483                 param->alternate_path = &work->path[1];
1484         else
1485                 param->alternate_path = NULL;
1486         param->remote_ca_guid = req_msg->local_ca_guid;
1487         param->remote_qkey = be32_to_cpu(req_msg->local_qkey);
1488         param->remote_qpn = be32_to_cpu(cm_req_get_local_qpn(req_msg));
1489         param->qp_type = cm_req_get_qp_type(req_msg);
1490         param->starting_psn = be32_to_cpu(cm_req_get_starting_psn(req_msg));
1491         param->responder_resources = cm_req_get_init_depth(req_msg);
1492         param->initiator_depth = cm_req_get_resp_res(req_msg);
1493         param->local_cm_response_timeout =
1494                                         cm_req_get_remote_resp_timeout(req_msg);
1495         param->flow_control = cm_req_get_flow_ctrl(req_msg);
1496         param->remote_cm_response_timeout =
1497                                         cm_req_get_local_resp_timeout(req_msg);
1498         param->retry_count = cm_req_get_retry_count(req_msg);
1499         param->rnr_retry_count = cm_req_get_rnr_retry_count(req_msg);
1500         param->srq = cm_req_get_srq(req_msg);
1501         work->cm_event.private_data = &req_msg->private_data;
1502 }
1503
1504 static void cm_process_work(struct cm_id_private *cm_id_priv,
1505                             struct cm_work *work)
1506 {
1507         int ret;
1508
1509         /* We will typically only have the current event to report. */
1510         ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
1511         cm_free_work(work);
1512
1513         while (!ret && !atomic_add_negative(-1, &cm_id_priv->work_count)) {
1514                 spin_lock_irq(&cm_id_priv->lock);
1515                 work = cm_dequeue_work(cm_id_priv);
1516                 spin_unlock_irq(&cm_id_priv->lock);
1517                 BUG_ON(!work);
1518                 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id,
1519                                                 &work->cm_event);
1520                 cm_free_work(work);
1521         }
1522         cm_deref_id(cm_id_priv);
1523         if (ret)
1524                 cm_destroy_id(&cm_id_priv->id, ret);
1525 }
1526
1527 static void cm_format_mra(struct cm_mra_msg *mra_msg,
1528                           struct cm_id_private *cm_id_priv,
1529                           enum cm_msg_response msg_mraed, u8 service_timeout,
1530                           const void *private_data, u8 private_data_len)
1531 {
1532         cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid);
1533         cm_mra_set_msg_mraed(mra_msg, msg_mraed);
1534         mra_msg->local_comm_id = cm_id_priv->id.local_id;
1535         mra_msg->remote_comm_id = cm_id_priv->id.remote_id;
1536         cm_mra_set_service_timeout(mra_msg, service_timeout);
1537
1538         if (private_data && private_data_len)
1539                 memcpy(mra_msg->private_data, private_data, private_data_len);
1540 }
1541
1542 static void cm_format_rej(struct cm_rej_msg *rej_msg,
1543                           struct cm_id_private *cm_id_priv,
1544                           enum ib_cm_rej_reason reason,
1545                           void *ari,
1546                           u8 ari_length,
1547                           const void *private_data,
1548                           u8 private_data_len)
1549 {
1550         cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, cm_id_priv->tid);
1551         rej_msg->remote_comm_id = cm_id_priv->id.remote_id;
1552
1553         switch(cm_id_priv->id.state) {
1554         case IB_CM_REQ_RCVD:
1555                 rej_msg->local_comm_id = 0;
1556                 cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REQ);
1557                 break;
1558         case IB_CM_MRA_REQ_SENT:
1559                 rej_msg->local_comm_id = cm_id_priv->id.local_id;
1560                 cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REQ);
1561                 break;
1562         case IB_CM_REP_RCVD:
1563         case IB_CM_MRA_REP_SENT:
1564                 rej_msg->local_comm_id = cm_id_priv->id.local_id;
1565                 cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REP);
1566                 break;
1567         default:
1568                 rej_msg->local_comm_id = cm_id_priv->id.local_id;
1569                 cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_OTHER);
1570                 break;
1571         }
1572
1573         rej_msg->reason = cpu_to_be16(reason);
1574         if (ari && ari_length) {
1575                 cm_rej_set_reject_info_len(rej_msg, ari_length);
1576                 memcpy(rej_msg->ari, ari, ari_length);
1577         }
1578
1579         if (private_data && private_data_len)
1580                 memcpy(rej_msg->private_data, private_data, private_data_len);
1581 }
1582
1583 static void cm_dup_req_handler(struct cm_work *work,
1584                                struct cm_id_private *cm_id_priv)
1585 {
1586         struct ib_mad_send_buf *msg = NULL;
1587         int ret;
1588
1589         atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
1590                         counter[CM_REQ_COUNTER]);
1591
1592         /* Quick state check to discard duplicate REQs. */
1593         if (cm_id_priv->id.state == IB_CM_REQ_RCVD)
1594                 return;
1595
1596         ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
1597         if (ret)
1598                 return;
1599
1600         spin_lock_irq(&cm_id_priv->lock);
1601         switch (cm_id_priv->id.state) {
1602         case IB_CM_MRA_REQ_SENT:
1603                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
1604                               CM_MSG_RESPONSE_REQ, cm_id_priv->service_timeout,
1605                               cm_id_priv->private_data,
1606                               cm_id_priv->private_data_len);
1607                 break;
1608         case IB_CM_TIMEWAIT:
1609                 cm_format_rej((struct cm_rej_msg *) msg->mad, cm_id_priv,
1610                               IB_CM_REJ_STALE_CONN, NULL, 0, NULL, 0);
1611                 break;
1612         default:
1613                 goto unlock;
1614         }
1615         spin_unlock_irq(&cm_id_priv->lock);
1616
1617         ret = ib_post_send_mad(msg, NULL);
1618         if (ret)
1619                 goto free;
1620         return;
1621
1622 unlock: spin_unlock_irq(&cm_id_priv->lock);
1623 free:   cm_free_msg(msg);
1624 }
1625
1626 static struct cm_id_private * cm_match_req(struct cm_work *work,
1627                                            struct cm_id_private *cm_id_priv)
1628 {
1629         struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv;
1630         struct cm_timewait_info *timewait_info;
1631         struct cm_req_msg *req_msg;
1632         struct ib_cm_id *cm_id;
1633
1634         req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1635
1636         /* Check for possible duplicate REQ. */
1637         spin_lock_irq(&cm.lock);
1638         timewait_info = cm_insert_remote_id(cm_id_priv->timewait_info);
1639         if (timewait_info) {
1640                 cur_cm_id_priv = cm_get_id(timewait_info->work.local_id,
1641                                            timewait_info->work.remote_id);
1642                 spin_unlock_irq(&cm.lock);
1643                 if (cur_cm_id_priv) {
1644                         cm_dup_req_handler(work, cur_cm_id_priv);
1645                         cm_deref_id(cur_cm_id_priv);
1646                 }
1647                 return NULL;
1648         }
1649
1650         /* Check for stale connections. */
1651         timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
1652         if (timewait_info) {
1653                 cm_cleanup_timewait(cm_id_priv->timewait_info);
1654                 cur_cm_id_priv = cm_get_id(timewait_info->work.local_id,
1655                                            timewait_info->work.remote_id);
1656
1657                 spin_unlock_irq(&cm.lock);
1658                 cm_issue_rej(work->port, work->mad_recv_wc,
1659                              IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REQ,
1660                              NULL, 0);
1661                 if (cur_cm_id_priv) {
1662                         cm_id = &cur_cm_id_priv->id;
1663                         ib_send_cm_dreq(cm_id, NULL, 0);
1664                         cm_deref_id(cur_cm_id_priv);
1665                 }
1666                 return NULL;
1667         }
1668
1669         /* Find matching listen request. */
1670         listen_cm_id_priv = cm_find_listen(cm_id_priv->id.device,
1671                                            req_msg->service_id);
1672         if (!listen_cm_id_priv) {
1673                 cm_cleanup_timewait(cm_id_priv->timewait_info);
1674                 spin_unlock_irq(&cm.lock);
1675                 cm_issue_rej(work->port, work->mad_recv_wc,
1676                              IB_CM_REJ_INVALID_SERVICE_ID, CM_MSG_RESPONSE_REQ,
1677                              NULL, 0);
1678                 goto out;
1679         }
1680         atomic_inc(&listen_cm_id_priv->refcount);
1681         atomic_inc(&cm_id_priv->refcount);
1682         cm_id_priv->id.state = IB_CM_REQ_RCVD;
1683         atomic_inc(&cm_id_priv->work_count);
1684         spin_unlock_irq(&cm.lock);
1685 out:
1686         return listen_cm_id_priv;
1687 }
1688
1689 /*
1690  * Work-around for inter-subnet connections.  If the LIDs are permissive,
1691  * we need to override the LID/SL data in the REQ with the LID information
1692  * in the work completion.
1693  */
1694 static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc)
1695 {
1696         if (!cm_req_get_primary_subnet_local(req_msg)) {
1697                 if (req_msg->primary_local_lid == IB_LID_PERMISSIVE) {
1698                         req_msg->primary_local_lid = cpu_to_be16(wc->slid);
1699                         cm_req_set_primary_sl(req_msg, wc->sl);
1700                 }
1701
1702                 if (req_msg->primary_remote_lid == IB_LID_PERMISSIVE)
1703                         req_msg->primary_remote_lid = cpu_to_be16(wc->dlid_path_bits);
1704         }
1705
1706         if (!cm_req_get_alt_subnet_local(req_msg)) {
1707                 if (req_msg->alt_local_lid == IB_LID_PERMISSIVE) {
1708                         req_msg->alt_local_lid = cpu_to_be16(wc->slid);
1709                         cm_req_set_alt_sl(req_msg, wc->sl);
1710                 }
1711
1712                 if (req_msg->alt_remote_lid == IB_LID_PERMISSIVE)
1713                         req_msg->alt_remote_lid = cpu_to_be16(wc->dlid_path_bits);
1714         }
1715 }
1716
1717 static int cm_req_handler(struct cm_work *work)
1718 {
1719         struct ib_cm_id *cm_id;
1720         struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
1721         struct cm_req_msg *req_msg;
1722         union ib_gid gid;
1723         struct ib_gid_attr gid_attr;
1724         const struct ib_global_route *grh;
1725         int ret;
1726
1727         req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1728
1729         cm_id = ib_create_cm_id(work->port->cm_dev->ib_device, NULL, NULL);
1730         if (IS_ERR(cm_id))
1731                 return PTR_ERR(cm_id);
1732
1733         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1734         cm_id_priv->id.remote_id = req_msg->local_comm_id;
1735         cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
1736                                 work->mad_recv_wc->recv_buf.grh,
1737                                 &cm_id_priv->av);
1738         cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
1739                                                             id.local_id);
1740         if (IS_ERR(cm_id_priv->timewait_info)) {
1741                 ret = PTR_ERR(cm_id_priv->timewait_info);
1742                 goto destroy;
1743         }
1744         cm_id_priv->timewait_info->work.remote_id = req_msg->local_comm_id;
1745         cm_id_priv->timewait_info->remote_ca_guid = req_msg->local_ca_guid;
1746         cm_id_priv->timewait_info->remote_qpn = cm_req_get_local_qpn(req_msg);
1747
1748         listen_cm_id_priv = cm_match_req(work, cm_id_priv);
1749         if (!listen_cm_id_priv) {
1750                 ret = -EINVAL;
1751                 kfree(cm_id_priv->timewait_info);
1752                 goto destroy;
1753         }
1754
1755         cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
1756         cm_id_priv->id.context = listen_cm_id_priv->id.context;
1757         cm_id_priv->id.service_id = req_msg->service_id;
1758         cm_id_priv->id.service_mask = ~cpu_to_be64(0);
1759
1760         cm_process_routed_req(req_msg, work->mad_recv_wc->wc);
1761
1762         memset(&work->path[0], 0, sizeof(work->path[0]));
1763         memset(&work->path[1], 0, sizeof(work->path[1]));
1764         grh = rdma_ah_read_grh(&cm_id_priv->av.ah_attr);
1765         ret = ib_get_cached_gid(work->port->cm_dev->ib_device,
1766                                 work->port->port_num,
1767                                 grh->sgid_index,
1768                                 &gid, &gid_attr);
1769         if (!ret) {
1770                 if (gid_attr.ndev) {
1771                         work->path[0].rec_type =
1772                                 sa_conv_gid_to_pathrec_type(gid_attr.gid_type);
1773                         sa_path_set_ifindex(&work->path[0],
1774                                             gid_attr.ndev->ifindex);
1775                         sa_path_set_ndev(&work->path[0],
1776                                          dev_net(gid_attr.ndev));
1777                         dev_put(gid_attr.ndev);
1778                 } else {
1779                         work->path[0].rec_type = SA_PATH_REC_TYPE_IB;
1780                 }
1781                 if (req_msg->alt_local_lid)
1782                         work->path[1].rec_type = work->path[0].rec_type;
1783                 cm_format_paths_from_req(req_msg, &work->path[0],
1784                                          &work->path[1]);
1785                 if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)
1786                         sa_path_set_dmac(&work->path[0],
1787                                          cm_id_priv->av.ah_attr.roce.dmac);
1788                 work->path[0].hop_limit = grh->hop_limit;
1789                 ret = cm_init_av_by_path(&work->path[0], &cm_id_priv->av,
1790                                          cm_id_priv);
1791         }
1792         if (ret) {
1793                 int err = ib_get_cached_gid(work->port->cm_dev->ib_device,
1794                                             work->port->port_num, 0,
1795                                             &work->path[0].sgid,
1796                                             &gid_attr);
1797                 if (!err && gid_attr.ndev) {
1798                         work->path[0].rec_type =
1799                                 sa_conv_gid_to_pathrec_type(gid_attr.gid_type);
1800                         sa_path_set_ifindex(&work->path[0],
1801                                             gid_attr.ndev->ifindex);
1802                         sa_path_set_ndev(&work->path[0],
1803                                          dev_net(gid_attr.ndev));
1804                         dev_put(gid_attr.ndev);
1805                 } else {
1806                         work->path[0].rec_type = SA_PATH_REC_TYPE_IB;
1807                 }
1808                 if (req_msg->alt_local_lid)
1809                         work->path[1].rec_type = work->path[0].rec_type;
1810                 ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID,
1811                                &work->path[0].sgid, sizeof work->path[0].sgid,
1812                                NULL, 0);
1813                 goto rejected;
1814         }
1815         if (req_msg->alt_local_lid) {
1816                 ret = cm_init_av_by_path(&work->path[1], &cm_id_priv->alt_av,
1817                                          cm_id_priv);
1818                 if (ret) {
1819                         ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_ALT_GID,
1820                                        &work->path[0].sgid,
1821                                        sizeof work->path[0].sgid, NULL, 0);
1822                         goto rejected;
1823                 }
1824         }
1825         cm_id_priv->tid = req_msg->hdr.tid;
1826         cm_id_priv->timeout_ms = cm_convert_to_ms(
1827                                         cm_req_get_local_resp_timeout(req_msg));
1828         cm_id_priv->max_cm_retries = cm_req_get_max_cm_retries(req_msg);
1829         cm_id_priv->remote_qpn = cm_req_get_local_qpn(req_msg);
1830         cm_id_priv->initiator_depth = cm_req_get_resp_res(req_msg);
1831         cm_id_priv->responder_resources = cm_req_get_init_depth(req_msg);
1832         cm_id_priv->path_mtu = cm_req_get_path_mtu(req_msg);
1833         cm_id_priv->pkey = req_msg->pkey;
1834         cm_id_priv->sq_psn = cm_req_get_starting_psn(req_msg);
1835         cm_id_priv->retry_count = cm_req_get_retry_count(req_msg);
1836         cm_id_priv->rnr_retry_count = cm_req_get_rnr_retry_count(req_msg);
1837         cm_id_priv->qp_type = cm_req_get_qp_type(req_msg);
1838
1839         cm_format_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
1840         cm_process_work(cm_id_priv, work);
1841         cm_deref_id(listen_cm_id_priv);
1842         return 0;
1843
1844 rejected:
1845         atomic_dec(&cm_id_priv->refcount);
1846         cm_deref_id(listen_cm_id_priv);
1847 destroy:
1848         ib_destroy_cm_id(cm_id);
1849         return ret;
1850 }
1851
1852 static void cm_format_rep(struct cm_rep_msg *rep_msg,
1853                           struct cm_id_private *cm_id_priv,
1854                           struct ib_cm_rep_param *param)
1855 {
1856         cm_format_mad_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid);
1857         rep_msg->local_comm_id = cm_id_priv->id.local_id;
1858         rep_msg->remote_comm_id = cm_id_priv->id.remote_id;
1859         cm_rep_set_starting_psn(rep_msg, cpu_to_be32(param->starting_psn));
1860         rep_msg->resp_resources = param->responder_resources;
1861         cm_rep_set_target_ack_delay(rep_msg,
1862                                     cm_id_priv->av.port->cm_dev->ack_delay);
1863         cm_rep_set_failover(rep_msg, param->failover_accepted);
1864         cm_rep_set_rnr_retry_count(rep_msg, param->rnr_retry_count);
1865         rep_msg->local_ca_guid = cm_id_priv->id.device->node_guid;
1866
1867         if (cm_id_priv->qp_type != IB_QPT_XRC_TGT) {
1868                 rep_msg->initiator_depth = param->initiator_depth;
1869                 cm_rep_set_flow_ctrl(rep_msg, param->flow_control);
1870                 cm_rep_set_srq(rep_msg, param->srq);
1871                 cm_rep_set_local_qpn(rep_msg, cpu_to_be32(param->qp_num));
1872         } else {
1873                 cm_rep_set_srq(rep_msg, 1);
1874                 cm_rep_set_local_eecn(rep_msg, cpu_to_be32(param->qp_num));
1875         }
1876
1877         if (param->private_data && param->private_data_len)
1878                 memcpy(rep_msg->private_data, param->private_data,
1879                        param->private_data_len);
1880 }
1881
1882 int ib_send_cm_rep(struct ib_cm_id *cm_id,
1883                    struct ib_cm_rep_param *param)
1884 {
1885         struct cm_id_private *cm_id_priv;
1886         struct ib_mad_send_buf *msg;
1887         struct cm_rep_msg *rep_msg;
1888         unsigned long flags;
1889         int ret;
1890
1891         if (param->private_data &&
1892             param->private_data_len > IB_CM_REP_PRIVATE_DATA_SIZE)
1893                 return -EINVAL;
1894
1895         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1896         spin_lock_irqsave(&cm_id_priv->lock, flags);
1897         if (cm_id->state != IB_CM_REQ_RCVD &&
1898             cm_id->state != IB_CM_MRA_REQ_SENT) {
1899                 ret = -EINVAL;
1900                 goto out;
1901         }
1902
1903         ret = cm_alloc_msg(cm_id_priv, &msg);
1904         if (ret)
1905                 goto out;
1906
1907         rep_msg = (struct cm_rep_msg *) msg->mad;
1908         cm_format_rep(rep_msg, cm_id_priv, param);
1909         msg->timeout_ms = cm_id_priv->timeout_ms;
1910         msg->context[1] = (void *) (unsigned long) IB_CM_REP_SENT;
1911
1912         ret = ib_post_send_mad(msg, NULL);
1913         if (ret) {
1914                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1915                 cm_free_msg(msg);
1916                 return ret;
1917         }
1918
1919         cm_id->state = IB_CM_REP_SENT;
1920         cm_id_priv->msg = msg;
1921         cm_id_priv->initiator_depth = param->initiator_depth;
1922         cm_id_priv->responder_resources = param->responder_resources;
1923         cm_id_priv->rq_psn = cm_rep_get_starting_psn(rep_msg);
1924         cm_id_priv->local_qpn = cpu_to_be32(param->qp_num & 0xFFFFFF);
1925
1926 out:    spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1927         return ret;
1928 }
1929 EXPORT_SYMBOL(ib_send_cm_rep);
1930
1931 static void cm_format_rtu(struct cm_rtu_msg *rtu_msg,
1932                           struct cm_id_private *cm_id_priv,
1933                           const void *private_data,
1934                           u8 private_data_len)
1935 {
1936         cm_format_mad_hdr(&rtu_msg->hdr, CM_RTU_ATTR_ID, cm_id_priv->tid);
1937         rtu_msg->local_comm_id = cm_id_priv->id.local_id;
1938         rtu_msg->remote_comm_id = cm_id_priv->id.remote_id;
1939
1940         if (private_data && private_data_len)
1941                 memcpy(rtu_msg->private_data, private_data, private_data_len);
1942 }
1943
1944 int ib_send_cm_rtu(struct ib_cm_id *cm_id,
1945                    const void *private_data,
1946                    u8 private_data_len)
1947 {
1948         struct cm_id_private *cm_id_priv;
1949         struct ib_mad_send_buf *msg;
1950         unsigned long flags;
1951         void *data;
1952         int ret;
1953
1954         if (private_data && private_data_len > IB_CM_RTU_PRIVATE_DATA_SIZE)
1955                 return -EINVAL;
1956
1957         data = cm_copy_private_data(private_data, private_data_len);
1958         if (IS_ERR(data))
1959                 return PTR_ERR(data);
1960
1961         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1962         spin_lock_irqsave(&cm_id_priv->lock, flags);
1963         if (cm_id->state != IB_CM_REP_RCVD &&
1964             cm_id->state != IB_CM_MRA_REP_SENT) {
1965                 ret = -EINVAL;
1966                 goto error;
1967         }
1968
1969         ret = cm_alloc_msg(cm_id_priv, &msg);
1970         if (ret)
1971                 goto error;
1972
1973         cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
1974                       private_data, private_data_len);
1975
1976         ret = ib_post_send_mad(msg, NULL);
1977         if (ret) {
1978                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1979                 cm_free_msg(msg);
1980                 kfree(data);
1981                 return ret;
1982         }
1983
1984         cm_id->state = IB_CM_ESTABLISHED;
1985         cm_set_private_data(cm_id_priv, data, private_data_len);
1986         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1987         return 0;
1988
1989 error:  spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1990         kfree(data);
1991         return ret;
1992 }
1993 EXPORT_SYMBOL(ib_send_cm_rtu);
1994
1995 static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type)
1996 {
1997         struct cm_rep_msg *rep_msg;
1998         struct ib_cm_rep_event_param *param;
1999
2000         rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2001         param = &work->cm_event.param.rep_rcvd;
2002         param->remote_ca_guid = rep_msg->local_ca_guid;
2003         param->remote_qkey = be32_to_cpu(rep_msg->local_qkey);
2004         param->remote_qpn = be32_to_cpu(cm_rep_get_qpn(rep_msg, qp_type));
2005         param->starting_psn = be32_to_cpu(cm_rep_get_starting_psn(rep_msg));
2006         param->responder_resources = rep_msg->initiator_depth;
2007         param->initiator_depth = rep_msg->resp_resources;
2008         param->target_ack_delay = cm_rep_get_target_ack_delay(rep_msg);
2009         param->failover_accepted = cm_rep_get_failover(rep_msg);
2010         param->flow_control = cm_rep_get_flow_ctrl(rep_msg);
2011         param->rnr_retry_count = cm_rep_get_rnr_retry_count(rep_msg);
2012         param->srq = cm_rep_get_srq(rep_msg);
2013         work->cm_event.private_data = &rep_msg->private_data;
2014 }
2015
2016 static void cm_dup_rep_handler(struct cm_work *work)
2017 {
2018         struct cm_id_private *cm_id_priv;
2019         struct cm_rep_msg *rep_msg;
2020         struct ib_mad_send_buf *msg = NULL;
2021         int ret;
2022
2023         rep_msg = (struct cm_rep_msg *) work->mad_recv_wc->recv_buf.mad;
2024         cm_id_priv = cm_acquire_id(rep_msg->remote_comm_id,
2025                                    rep_msg->local_comm_id);
2026         if (!cm_id_priv)
2027                 return;
2028
2029         atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2030                         counter[CM_REP_COUNTER]);
2031         ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
2032         if (ret)
2033                 goto deref;
2034
2035         spin_lock_irq(&cm_id_priv->lock);
2036         if (cm_id_priv->id.state == IB_CM_ESTABLISHED)
2037                 cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2038                               cm_id_priv->private_data,
2039                               cm_id_priv->private_data_len);
2040         else if (cm_id_priv->id.state == IB_CM_MRA_REP_SENT)
2041                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
2042                               CM_MSG_RESPONSE_REP, cm_id_priv->service_timeout,
2043                               cm_id_priv->private_data,
2044                               cm_id_priv->private_data_len);
2045         else
2046                 goto unlock;
2047         spin_unlock_irq(&cm_id_priv->lock);
2048
2049         ret = ib_post_send_mad(msg, NULL);
2050         if (ret)
2051                 goto free;
2052         goto deref;
2053
2054 unlock: spin_unlock_irq(&cm_id_priv->lock);
2055 free:   cm_free_msg(msg);
2056 deref:  cm_deref_id(cm_id_priv);
2057 }
2058
2059 static int cm_rep_handler(struct cm_work *work)
2060 {
2061         struct cm_id_private *cm_id_priv;
2062         struct cm_rep_msg *rep_msg;
2063         int ret;
2064         struct cm_id_private *cur_cm_id_priv;
2065         struct ib_cm_id *cm_id;
2066         struct cm_timewait_info *timewait_info;
2067
2068         rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2069         cm_id_priv = cm_acquire_id(rep_msg->remote_comm_id, 0);
2070         if (!cm_id_priv) {
2071                 cm_dup_rep_handler(work);
2072                 return -EINVAL;
2073         }
2074
2075         cm_format_rep_event(work, cm_id_priv->qp_type);
2076
2077         spin_lock_irq(&cm_id_priv->lock);
2078         switch (cm_id_priv->id.state) {
2079         case IB_CM_REQ_SENT:
2080         case IB_CM_MRA_REQ_RCVD:
2081                 break;
2082         default:
2083                 spin_unlock_irq(&cm_id_priv->lock);
2084                 ret = -EINVAL;
2085                 goto error;
2086         }
2087
2088         cm_id_priv->timewait_info->work.remote_id = rep_msg->local_comm_id;
2089         cm_id_priv->timewait_info->remote_ca_guid = rep_msg->local_ca_guid;
2090         cm_id_priv->timewait_info->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2091
2092         spin_lock(&cm.lock);
2093         /* Check for duplicate REP. */
2094         if (cm_insert_remote_id(cm_id_priv->timewait_info)) {
2095                 spin_unlock(&cm.lock);
2096                 spin_unlock_irq(&cm_id_priv->lock);
2097                 ret = -EINVAL;
2098                 goto error;
2099         }
2100         /* Check for a stale connection. */
2101         timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
2102         if (timewait_info) {
2103                 rb_erase(&cm_id_priv->timewait_info->remote_id_node,
2104                          &cm.remote_id_table);
2105                 cm_id_priv->timewait_info->inserted_remote_id = 0;
2106                 cur_cm_id_priv = cm_get_id(timewait_info->work.local_id,
2107                                            timewait_info->work.remote_id);
2108
2109                 spin_unlock(&cm.lock);
2110                 spin_unlock_irq(&cm_id_priv->lock);
2111                 cm_issue_rej(work->port, work->mad_recv_wc,
2112                              IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP,
2113                              NULL, 0);
2114                 ret = -EINVAL;
2115                 if (cur_cm_id_priv) {
2116                         cm_id = &cur_cm_id_priv->id;
2117                         ib_send_cm_dreq(cm_id, NULL, 0);
2118                         cm_deref_id(cur_cm_id_priv);
2119                 }
2120
2121                 goto error;
2122         }
2123         spin_unlock(&cm.lock);
2124
2125         cm_id_priv->id.state = IB_CM_REP_RCVD;
2126         cm_id_priv->id.remote_id = rep_msg->local_comm_id;
2127         cm_id_priv->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2128         cm_id_priv->initiator_depth = rep_msg->resp_resources;
2129         cm_id_priv->responder_resources = rep_msg->initiator_depth;
2130         cm_id_priv->sq_psn = cm_rep_get_starting_psn(rep_msg);
2131         cm_id_priv->rnr_retry_count = cm_rep_get_rnr_retry_count(rep_msg);
2132         cm_id_priv->target_ack_delay = cm_rep_get_target_ack_delay(rep_msg);
2133         cm_id_priv->av.timeout =
2134                         cm_ack_timeout(cm_id_priv->target_ack_delay,
2135                                        cm_id_priv->av.timeout - 1);
2136         cm_id_priv->alt_av.timeout =
2137                         cm_ack_timeout(cm_id_priv->target_ack_delay,
2138                                        cm_id_priv->alt_av.timeout - 1);
2139
2140         /* todo: handle peer_to_peer */
2141
2142         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2143         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2144         if (!ret)
2145                 list_add_tail(&work->list, &cm_id_priv->work_list);
2146         spin_unlock_irq(&cm_id_priv->lock);
2147
2148         if (ret)
2149                 cm_process_work(cm_id_priv, work);
2150         else
2151                 cm_deref_id(cm_id_priv);
2152         return 0;
2153
2154 error:
2155         cm_deref_id(cm_id_priv);
2156         return ret;
2157 }
2158
2159 static int cm_establish_handler(struct cm_work *work)
2160 {
2161         struct cm_id_private *cm_id_priv;
2162         int ret;
2163
2164         /* See comment in cm_establish about lookup. */
2165         cm_id_priv = cm_acquire_id(work->local_id, work->remote_id);
2166         if (!cm_id_priv)
2167                 return -EINVAL;
2168
2169         spin_lock_irq(&cm_id_priv->lock);
2170         if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2171                 spin_unlock_irq(&cm_id_priv->lock);
2172                 goto out;
2173         }
2174
2175         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2176         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2177         if (!ret)
2178                 list_add_tail(&work->list, &cm_id_priv->work_list);
2179         spin_unlock_irq(&cm_id_priv->lock);
2180
2181         if (ret)
2182                 cm_process_work(cm_id_priv, work);
2183         else
2184                 cm_deref_id(cm_id_priv);
2185         return 0;
2186 out:
2187         cm_deref_id(cm_id_priv);
2188         return -EINVAL;
2189 }
2190
2191 static int cm_rtu_handler(struct cm_work *work)
2192 {
2193         struct cm_id_private *cm_id_priv;
2194         struct cm_rtu_msg *rtu_msg;
2195         int ret;
2196
2197         rtu_msg = (struct cm_rtu_msg *)work->mad_recv_wc->recv_buf.mad;
2198         cm_id_priv = cm_acquire_id(rtu_msg->remote_comm_id,
2199                                    rtu_msg->local_comm_id);
2200         if (!cm_id_priv)
2201                 return -EINVAL;
2202
2203         work->cm_event.private_data = &rtu_msg->private_data;
2204
2205         spin_lock_irq(&cm_id_priv->lock);
2206         if (cm_id_priv->id.state != IB_CM_REP_SENT &&
2207             cm_id_priv->id.state != IB_CM_MRA_REP_RCVD) {
2208                 spin_unlock_irq(&cm_id_priv->lock);
2209                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2210                                 counter[CM_RTU_COUNTER]);
2211                 goto out;
2212         }
2213         cm_id_priv->id.state = IB_CM_ESTABLISHED;
2214
2215         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2216         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2217         if (!ret)
2218                 list_add_tail(&work->list, &cm_id_priv->work_list);
2219         spin_unlock_irq(&cm_id_priv->lock);
2220
2221         if (ret)
2222                 cm_process_work(cm_id_priv, work);
2223         else
2224                 cm_deref_id(cm_id_priv);
2225         return 0;
2226 out:
2227         cm_deref_id(cm_id_priv);
2228         return -EINVAL;
2229 }
2230
2231 static void cm_format_dreq(struct cm_dreq_msg *dreq_msg,
2232                           struct cm_id_private *cm_id_priv,
2233                           const void *private_data,
2234                           u8 private_data_len)
2235 {
2236         cm_format_mad_hdr(&dreq_msg->hdr, CM_DREQ_ATTR_ID,
2237                           cm_form_tid(cm_id_priv, CM_MSG_SEQUENCE_DREQ));
2238         dreq_msg->local_comm_id = cm_id_priv->id.local_id;
2239         dreq_msg->remote_comm_id = cm_id_priv->id.remote_id;
2240         cm_dreq_set_remote_qpn(dreq_msg, cm_id_priv->remote_qpn);
2241
2242         if (private_data && private_data_len)
2243                 memcpy(dreq_msg->private_data, private_data, private_data_len);
2244 }
2245
2246 int ib_send_cm_dreq(struct ib_cm_id *cm_id,
2247                     const void *private_data,
2248                     u8 private_data_len)
2249 {
2250         struct cm_id_private *cm_id_priv;
2251         struct ib_mad_send_buf *msg;
2252         unsigned long flags;
2253         int ret;
2254
2255         if (private_data && private_data_len > IB_CM_DREQ_PRIVATE_DATA_SIZE)
2256                 return -EINVAL;
2257
2258         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2259         spin_lock_irqsave(&cm_id_priv->lock, flags);
2260         if (cm_id->state != IB_CM_ESTABLISHED) {
2261                 ret = -EINVAL;
2262                 goto out;
2263         }
2264
2265         if (cm_id->lap_state == IB_CM_LAP_SENT ||
2266             cm_id->lap_state == IB_CM_MRA_LAP_RCVD)
2267                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2268
2269         ret = cm_alloc_msg(cm_id_priv, &msg);
2270         if (ret) {
2271                 cm_enter_timewait(cm_id_priv);
2272                 goto out;
2273         }
2274
2275         cm_format_dreq((struct cm_dreq_msg *) msg->mad, cm_id_priv,
2276                        private_data, private_data_len);
2277         msg->timeout_ms = cm_id_priv->timeout_ms;
2278         msg->context[1] = (void *) (unsigned long) IB_CM_DREQ_SENT;
2279
2280         ret = ib_post_send_mad(msg, NULL);
2281         if (ret) {
2282                 cm_enter_timewait(cm_id_priv);
2283                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2284                 cm_free_msg(msg);
2285                 return ret;
2286         }
2287
2288         cm_id->state = IB_CM_DREQ_SENT;
2289         cm_id_priv->msg = msg;
2290 out:    spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2291         return ret;
2292 }
2293 EXPORT_SYMBOL(ib_send_cm_dreq);
2294
2295 static void cm_format_drep(struct cm_drep_msg *drep_msg,
2296                           struct cm_id_private *cm_id_priv,
2297                           const void *private_data,
2298                           u8 private_data_len)
2299 {
2300         cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, cm_id_priv->tid);
2301         drep_msg->local_comm_id = cm_id_priv->id.local_id;
2302         drep_msg->remote_comm_id = cm_id_priv->id.remote_id;
2303
2304         if (private_data && private_data_len)
2305                 memcpy(drep_msg->private_data, private_data, private_data_len);
2306 }
2307
2308 int ib_send_cm_drep(struct ib_cm_id *cm_id,
2309                     const void *private_data,
2310                     u8 private_data_len)
2311 {
2312         struct cm_id_private *cm_id_priv;
2313         struct ib_mad_send_buf *msg;
2314         unsigned long flags;
2315         void *data;
2316         int ret;
2317
2318         if (private_data && private_data_len > IB_CM_DREP_PRIVATE_DATA_SIZE)
2319                 return -EINVAL;
2320
2321         data = cm_copy_private_data(private_data, private_data_len);
2322         if (IS_ERR(data))
2323                 return PTR_ERR(data);
2324
2325         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2326         spin_lock_irqsave(&cm_id_priv->lock, flags);
2327         if (cm_id->state != IB_CM_DREQ_RCVD) {
2328                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2329                 kfree(data);
2330                 return -EINVAL;
2331         }
2332
2333         cm_set_private_data(cm_id_priv, data, private_data_len);
2334         cm_enter_timewait(cm_id_priv);
2335
2336         ret = cm_alloc_msg(cm_id_priv, &msg);
2337         if (ret)
2338                 goto out;
2339
2340         cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2341                        private_data, private_data_len);
2342
2343         ret = ib_post_send_mad(msg, NULL);
2344         if (ret) {
2345                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2346                 cm_free_msg(msg);
2347                 return ret;
2348         }
2349
2350 out:    spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2351         return ret;
2352 }
2353 EXPORT_SYMBOL(ib_send_cm_drep);
2354
2355 static int cm_issue_drep(struct cm_port *port,
2356                          struct ib_mad_recv_wc *mad_recv_wc)
2357 {
2358         struct ib_mad_send_buf *msg = NULL;
2359         struct cm_dreq_msg *dreq_msg;
2360         struct cm_drep_msg *drep_msg;
2361         int ret;
2362
2363         ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
2364         if (ret)
2365                 return ret;
2366
2367         dreq_msg = (struct cm_dreq_msg *) mad_recv_wc->recv_buf.mad;
2368         drep_msg = (struct cm_drep_msg *) msg->mad;
2369
2370         cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, dreq_msg->hdr.tid);
2371         drep_msg->remote_comm_id = dreq_msg->local_comm_id;
2372         drep_msg->local_comm_id = dreq_msg->remote_comm_id;
2373
2374         ret = ib_post_send_mad(msg, NULL);
2375         if (ret)
2376                 cm_free_msg(msg);
2377
2378         return ret;
2379 }
2380
2381 static int cm_dreq_handler(struct cm_work *work)
2382 {
2383         struct cm_id_private *cm_id_priv;
2384         struct cm_dreq_msg *dreq_msg;
2385         struct ib_mad_send_buf *msg = NULL;
2386         int ret;
2387
2388         dreq_msg = (struct cm_dreq_msg *)work->mad_recv_wc->recv_buf.mad;
2389         cm_id_priv = cm_acquire_id(dreq_msg->remote_comm_id,
2390                                    dreq_msg->local_comm_id);
2391         if (!cm_id_priv) {
2392                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2393                                 counter[CM_DREQ_COUNTER]);
2394                 cm_issue_drep(work->port, work->mad_recv_wc);
2395                 return -EINVAL;
2396         }
2397
2398         work->cm_event.private_data = &dreq_msg->private_data;
2399
2400         spin_lock_irq(&cm_id_priv->lock);
2401         if (cm_id_priv->local_qpn != cm_dreq_get_remote_qpn(dreq_msg))
2402                 goto unlock;
2403
2404         switch (cm_id_priv->id.state) {
2405         case IB_CM_REP_SENT:
2406         case IB_CM_DREQ_SENT:
2407                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2408                 break;
2409         case IB_CM_ESTABLISHED:
2410                 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2411                     cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2412                         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2413                 break;
2414         case IB_CM_MRA_REP_RCVD:
2415                 break;
2416         case IB_CM_TIMEWAIT:
2417                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2418                                 counter[CM_DREQ_COUNTER]);
2419                 if (cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg))
2420                         goto unlock;
2421
2422                 cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2423                                cm_id_priv->private_data,
2424                                cm_id_priv->private_data_len);
2425                 spin_unlock_irq(&cm_id_priv->lock);
2426
2427                 if (ib_post_send_mad(msg, NULL))
2428                         cm_free_msg(msg);
2429                 goto deref;
2430         case IB_CM_DREQ_RCVD:
2431                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2432                                 counter[CM_DREQ_COUNTER]);
2433                 goto unlock;
2434         default:
2435                 goto unlock;
2436         }
2437         cm_id_priv->id.state = IB_CM_DREQ_RCVD;
2438         cm_id_priv->tid = dreq_msg->hdr.tid;
2439         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2440         if (!ret)
2441                 list_add_tail(&work->list, &cm_id_priv->work_list);
2442         spin_unlock_irq(&cm_id_priv->lock);
2443
2444         if (ret)
2445                 cm_process_work(cm_id_priv, work);
2446         else
2447                 cm_deref_id(cm_id_priv);
2448         return 0;
2449
2450 unlock: spin_unlock_irq(&cm_id_priv->lock);
2451 deref:  cm_deref_id(cm_id_priv);
2452         return -EINVAL;
2453 }
2454
2455 static int cm_drep_handler(struct cm_work *work)
2456 {
2457         struct cm_id_private *cm_id_priv;
2458         struct cm_drep_msg *drep_msg;
2459         int ret;
2460
2461         drep_msg = (struct cm_drep_msg *)work->mad_recv_wc->recv_buf.mad;
2462         cm_id_priv = cm_acquire_id(drep_msg->remote_comm_id,
2463                                    drep_msg->local_comm_id);
2464         if (!cm_id_priv)
2465                 return -EINVAL;
2466
2467         work->cm_event.private_data = &drep_msg->private_data;
2468
2469         spin_lock_irq(&cm_id_priv->lock);
2470         if (cm_id_priv->id.state != IB_CM_DREQ_SENT &&
2471             cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2472                 spin_unlock_irq(&cm_id_priv->lock);
2473                 goto out;
2474         }
2475         cm_enter_timewait(cm_id_priv);
2476
2477         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2478         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2479         if (!ret)
2480                 list_add_tail(&work->list, &cm_id_priv->work_list);
2481         spin_unlock_irq(&cm_id_priv->lock);
2482
2483         if (ret)
2484                 cm_process_work(cm_id_priv, work);
2485         else
2486                 cm_deref_id(cm_id_priv);
2487         return 0;
2488 out:
2489         cm_deref_id(cm_id_priv);
2490         return -EINVAL;
2491 }
2492
2493 int ib_send_cm_rej(struct ib_cm_id *cm_id,
2494                    enum ib_cm_rej_reason reason,
2495                    void *ari,
2496                    u8 ari_length,
2497                    const void *private_data,
2498                    u8 private_data_len)
2499 {
2500         struct cm_id_private *cm_id_priv;
2501         struct ib_mad_send_buf *msg;
2502         unsigned long flags;
2503         int ret;
2504
2505         if ((private_data && private_data_len > IB_CM_REJ_PRIVATE_DATA_SIZE) ||
2506             (ari && ari_length > IB_CM_REJ_ARI_LENGTH))
2507                 return -EINVAL;
2508
2509         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2510
2511         spin_lock_irqsave(&cm_id_priv->lock, flags);
2512         switch (cm_id->state) {
2513         case IB_CM_REQ_SENT:
2514         case IB_CM_MRA_REQ_RCVD:
2515         case IB_CM_REQ_RCVD:
2516         case IB_CM_MRA_REQ_SENT:
2517         case IB_CM_REP_RCVD:
2518         case IB_CM_MRA_REP_SENT:
2519                 ret = cm_alloc_msg(cm_id_priv, &msg);
2520                 if (!ret)
2521                         cm_format_rej((struct cm_rej_msg *) msg->mad,
2522                                       cm_id_priv, reason, ari, ari_length,
2523                                       private_data, private_data_len);
2524
2525                 cm_reset_to_idle(cm_id_priv);
2526                 break;
2527         case IB_CM_REP_SENT:
2528         case IB_CM_MRA_REP_RCVD:
2529                 ret = cm_alloc_msg(cm_id_priv, &msg);
2530                 if (!ret)
2531                         cm_format_rej((struct cm_rej_msg *) msg->mad,
2532                                       cm_id_priv, reason, ari, ari_length,
2533                                       private_data, private_data_len);
2534
2535                 cm_enter_timewait(cm_id_priv);
2536                 break;
2537         default:
2538                 ret = -EINVAL;
2539                 goto out;
2540         }
2541
2542         if (ret)
2543                 goto out;
2544
2545         ret = ib_post_send_mad(msg, NULL);
2546         if (ret)
2547                 cm_free_msg(msg);
2548
2549 out:    spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2550         return ret;
2551 }
2552 EXPORT_SYMBOL(ib_send_cm_rej);
2553
2554 static void cm_format_rej_event(struct cm_work *work)
2555 {
2556         struct cm_rej_msg *rej_msg;
2557         struct ib_cm_rej_event_param *param;
2558
2559         rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
2560         param = &work->cm_event.param.rej_rcvd;
2561         param->ari = rej_msg->ari;
2562         param->ari_length = cm_rej_get_reject_info_len(rej_msg);
2563         param->reason = __be16_to_cpu(rej_msg->reason);
2564         work->cm_event.private_data = &rej_msg->private_data;
2565 }
2566
2567 static struct cm_id_private * cm_acquire_rejected_id(struct cm_rej_msg *rej_msg)
2568 {
2569         struct cm_timewait_info *timewait_info;
2570         struct cm_id_private *cm_id_priv;
2571         __be32 remote_id;
2572
2573         remote_id = rej_msg->local_comm_id;
2574
2575         if (__be16_to_cpu(rej_msg->reason) == IB_CM_REJ_TIMEOUT) {
2576                 spin_lock_irq(&cm.lock);
2577                 timewait_info = cm_find_remote_id( *((__be64 *) rej_msg->ari),
2578                                                   remote_id);
2579                 if (!timewait_info) {
2580                         spin_unlock_irq(&cm.lock);
2581                         return NULL;
2582                 }
2583                 cm_id_priv = idr_find(&cm.local_id_table, (__force int)
2584                                       (timewait_info->work.local_id ^
2585                                        cm.random_id_operand));
2586                 if (cm_id_priv) {
2587                         if (cm_id_priv->id.remote_id == remote_id)
2588                                 atomic_inc(&cm_id_priv->refcount);
2589                         else
2590                                 cm_id_priv = NULL;
2591                 }
2592                 spin_unlock_irq(&cm.lock);
2593         } else if (cm_rej_get_msg_rejected(rej_msg) == CM_MSG_RESPONSE_REQ)
2594                 cm_id_priv = cm_acquire_id(rej_msg->remote_comm_id, 0);
2595         else
2596                 cm_id_priv = cm_acquire_id(rej_msg->remote_comm_id, remote_id);
2597
2598         return cm_id_priv;
2599 }
2600
2601 static int cm_rej_handler(struct cm_work *work)
2602 {
2603         struct cm_id_private *cm_id_priv;
2604         struct cm_rej_msg *rej_msg;
2605         int ret;
2606
2607         rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
2608         cm_id_priv = cm_acquire_rejected_id(rej_msg);
2609         if (!cm_id_priv)
2610                 return -EINVAL;
2611
2612         cm_format_rej_event(work);
2613
2614         spin_lock_irq(&cm_id_priv->lock);
2615         switch (cm_id_priv->id.state) {
2616         case IB_CM_REQ_SENT:
2617         case IB_CM_MRA_REQ_RCVD:
2618         case IB_CM_REP_SENT:
2619         case IB_CM_MRA_REP_RCVD:
2620                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2621                 /* fall through */
2622         case IB_CM_REQ_RCVD:
2623         case IB_CM_MRA_REQ_SENT:
2624                 if (__be16_to_cpu(rej_msg->reason) == IB_CM_REJ_STALE_CONN)
2625                         cm_enter_timewait(cm_id_priv);
2626                 else
2627                         cm_reset_to_idle(cm_id_priv);
2628                 break;
2629         case IB_CM_DREQ_SENT:
2630                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2631                 /* fall through */
2632         case IB_CM_REP_RCVD:
2633         case IB_CM_MRA_REP_SENT:
2634                 cm_enter_timewait(cm_id_priv);
2635                 break;
2636         case IB_CM_ESTABLISHED:
2637                 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT ||
2638                     cm_id_priv->id.lap_state == IB_CM_LAP_SENT) {
2639                         if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT)
2640                                 ib_cancel_mad(cm_id_priv->av.port->mad_agent,
2641                                               cm_id_priv->msg);
2642                         cm_enter_timewait(cm_id_priv);
2643                         break;
2644                 }
2645                 /* fall through */
2646         default:
2647                 spin_unlock_irq(&cm_id_priv->lock);
2648                 ret = -EINVAL;
2649                 goto out;
2650         }
2651
2652         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2653         if (!ret)
2654                 list_add_tail(&work->list, &cm_id_priv->work_list);
2655         spin_unlock_irq(&cm_id_priv->lock);
2656
2657         if (ret)
2658                 cm_process_work(cm_id_priv, work);
2659         else
2660                 cm_deref_id(cm_id_priv);
2661         return 0;
2662 out:
2663         cm_deref_id(cm_id_priv);
2664         return -EINVAL;
2665 }
2666
2667 int ib_send_cm_mra(struct ib_cm_id *cm_id,
2668                    u8 service_timeout,
2669                    const void *private_data,
2670                    u8 private_data_len)
2671 {
2672         struct cm_id_private *cm_id_priv;
2673         struct ib_mad_send_buf *msg;
2674         enum ib_cm_state cm_state;
2675         enum ib_cm_lap_state lap_state;
2676         enum cm_msg_response msg_response;
2677         void *data;
2678         unsigned long flags;
2679         int ret;
2680
2681         if (private_data && private_data_len > IB_CM_MRA_PRIVATE_DATA_SIZE)
2682                 return -EINVAL;
2683
2684         data = cm_copy_private_data(private_data, private_data_len);
2685         if (IS_ERR(data))
2686                 return PTR_ERR(data);
2687
2688         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2689
2690         spin_lock_irqsave(&cm_id_priv->lock, flags);
2691         switch(cm_id_priv->id.state) {
2692         case IB_CM_REQ_RCVD:
2693                 cm_state = IB_CM_MRA_REQ_SENT;
2694                 lap_state = cm_id->lap_state;
2695                 msg_response = CM_MSG_RESPONSE_REQ;
2696                 break;
2697         case IB_CM_REP_RCVD:
2698                 cm_state = IB_CM_MRA_REP_SENT;
2699                 lap_state = cm_id->lap_state;
2700                 msg_response = CM_MSG_RESPONSE_REP;
2701                 break;
2702         case IB_CM_ESTABLISHED:
2703                 if (cm_id->lap_state == IB_CM_LAP_RCVD) {
2704                         cm_state = cm_id->state;
2705                         lap_state = IB_CM_MRA_LAP_SENT;
2706                         msg_response = CM_MSG_RESPONSE_OTHER;
2707                         break;
2708                 }
2709         default:
2710                 ret = -EINVAL;
2711                 goto error1;
2712         }
2713
2714         if (!(service_timeout & IB_CM_MRA_FLAG_DELAY)) {
2715                 ret = cm_alloc_msg(cm_id_priv, &msg);
2716                 if (ret)
2717                         goto error1;
2718
2719                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
2720                               msg_response, service_timeout,
2721                               private_data, private_data_len);
2722                 ret = ib_post_send_mad(msg, NULL);
2723                 if (ret)
2724                         goto error2;
2725         }
2726
2727         cm_id->state = cm_state;
2728         cm_id->lap_state = lap_state;
2729         cm_id_priv->service_timeout = service_timeout;
2730         cm_set_private_data(cm_id_priv, data, private_data_len);
2731         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2732         return 0;
2733
2734 error1: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2735         kfree(data);
2736         return ret;
2737
2738 error2: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2739         kfree(data);
2740         cm_free_msg(msg);
2741         return ret;
2742 }
2743 EXPORT_SYMBOL(ib_send_cm_mra);
2744
2745 static struct cm_id_private * cm_acquire_mraed_id(struct cm_mra_msg *mra_msg)
2746 {
2747         switch (cm_mra_get_msg_mraed(mra_msg)) {
2748         case CM_MSG_RESPONSE_REQ:
2749                 return cm_acquire_id(mra_msg->remote_comm_id, 0);
2750         case CM_MSG_RESPONSE_REP:
2751         case CM_MSG_RESPONSE_OTHER:
2752                 return cm_acquire_id(mra_msg->remote_comm_id,
2753                                      mra_msg->local_comm_id);
2754         default:
2755                 return NULL;
2756         }
2757 }
2758
2759 static int cm_mra_handler(struct cm_work *work)
2760 {
2761         struct cm_id_private *cm_id_priv;
2762         struct cm_mra_msg *mra_msg;
2763         int timeout, ret;
2764
2765         mra_msg = (struct cm_mra_msg *)work->mad_recv_wc->recv_buf.mad;
2766         cm_id_priv = cm_acquire_mraed_id(mra_msg);
2767         if (!cm_id_priv)
2768                 return -EINVAL;
2769
2770         work->cm_event.private_data = &mra_msg->private_data;
2771         work->cm_event.param.mra_rcvd.service_timeout =
2772                                         cm_mra_get_service_timeout(mra_msg);
2773         timeout = cm_convert_to_ms(cm_mra_get_service_timeout(mra_msg)) +
2774                   cm_convert_to_ms(cm_id_priv->av.timeout);
2775
2776         spin_lock_irq(&cm_id_priv->lock);
2777         switch (cm_id_priv->id.state) {
2778         case IB_CM_REQ_SENT:
2779                 if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_REQ ||
2780                     ib_modify_mad(cm_id_priv->av.port->mad_agent,
2781                                   cm_id_priv->msg, timeout))
2782                         goto out;
2783                 cm_id_priv->id.state = IB_CM_MRA_REQ_RCVD;
2784                 break;
2785         case IB_CM_REP_SENT:
2786                 if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_REP ||
2787                     ib_modify_mad(cm_id_priv->av.port->mad_agent,
2788                                   cm_id_priv->msg, timeout))
2789                         goto out;
2790                 cm_id_priv->id.state = IB_CM_MRA_REP_RCVD;
2791                 break;
2792         case IB_CM_ESTABLISHED:
2793                 if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_OTHER ||
2794                     cm_id_priv->id.lap_state != IB_CM_LAP_SENT ||
2795                     ib_modify_mad(cm_id_priv->av.port->mad_agent,
2796                                   cm_id_priv->msg, timeout)) {
2797                         if (cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2798                                 atomic_long_inc(&work->port->
2799                                                 counter_group[CM_RECV_DUPLICATES].
2800                                                 counter[CM_MRA_COUNTER]);
2801                         goto out;
2802                 }
2803                 cm_id_priv->id.lap_state = IB_CM_MRA_LAP_RCVD;
2804                 break;
2805         case IB_CM_MRA_REQ_RCVD:
2806         case IB_CM_MRA_REP_RCVD:
2807                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2808                                 counter[CM_MRA_COUNTER]);
2809                 /* fall through */
2810         default:
2811                 goto out;
2812         }
2813
2814         cm_id_priv->msg->context[1] = (void *) (unsigned long)
2815                                       cm_id_priv->id.state;
2816         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2817         if (!ret)
2818                 list_add_tail(&work->list, &cm_id_priv->work_list);
2819         spin_unlock_irq(&cm_id_priv->lock);
2820
2821         if (ret)
2822                 cm_process_work(cm_id_priv, work);
2823         else
2824                 cm_deref_id(cm_id_priv);
2825         return 0;
2826 out:
2827         spin_unlock_irq(&cm_id_priv->lock);
2828         cm_deref_id(cm_id_priv);
2829         return -EINVAL;
2830 }
2831
2832 static void cm_format_lap(struct cm_lap_msg *lap_msg,
2833                           struct cm_id_private *cm_id_priv,
2834                           struct sa_path_rec *alternate_path,
2835                           const void *private_data,
2836                           u8 private_data_len)
2837 {
2838         cm_format_mad_hdr(&lap_msg->hdr, CM_LAP_ATTR_ID,
2839                           cm_form_tid(cm_id_priv, CM_MSG_SEQUENCE_LAP));
2840         lap_msg->local_comm_id = cm_id_priv->id.local_id;
2841         lap_msg->remote_comm_id = cm_id_priv->id.remote_id;
2842         cm_lap_set_remote_qpn(lap_msg, cm_id_priv->remote_qpn);
2843         /* todo: need remote CM response timeout */
2844         cm_lap_set_remote_resp_timeout(lap_msg, 0x1F);
2845         lap_msg->alt_local_lid = sa_path_get_slid(alternate_path);
2846         lap_msg->alt_remote_lid = sa_path_get_dlid(alternate_path);
2847         lap_msg->alt_local_gid = alternate_path->sgid;
2848         lap_msg->alt_remote_gid = alternate_path->dgid;
2849         cm_lap_set_flow_label(lap_msg, alternate_path->flow_label);
2850         cm_lap_set_traffic_class(lap_msg, alternate_path->traffic_class);
2851         lap_msg->alt_hop_limit = alternate_path->hop_limit;
2852         cm_lap_set_packet_rate(lap_msg, alternate_path->rate);
2853         cm_lap_set_sl(lap_msg, alternate_path->sl);
2854         cm_lap_set_subnet_local(lap_msg, 1); /* local only... */
2855         cm_lap_set_local_ack_timeout(lap_msg,
2856                 cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
2857                                alternate_path->packet_life_time));
2858
2859         if (private_data && private_data_len)
2860                 memcpy(lap_msg->private_data, private_data, private_data_len);
2861 }
2862
2863 int ib_send_cm_lap(struct ib_cm_id *cm_id,
2864                    struct sa_path_rec *alternate_path,
2865                    const void *private_data,
2866                    u8 private_data_len)
2867 {
2868         struct cm_id_private *cm_id_priv;
2869         struct ib_mad_send_buf *msg;
2870         unsigned long flags;
2871         int ret;
2872
2873         if (private_data && private_data_len > IB_CM_LAP_PRIVATE_DATA_SIZE)
2874                 return -EINVAL;
2875
2876         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2877         spin_lock_irqsave(&cm_id_priv->lock, flags);
2878         if (cm_id->state != IB_CM_ESTABLISHED ||
2879             (cm_id->lap_state != IB_CM_LAP_UNINIT &&
2880              cm_id->lap_state != IB_CM_LAP_IDLE)) {
2881                 ret = -EINVAL;
2882                 goto out;
2883         }
2884
2885         ret = cm_init_av_by_path(alternate_path, &cm_id_priv->alt_av,
2886                                  cm_id_priv);
2887         if (ret)
2888                 goto out;
2889         cm_id_priv->alt_av.timeout =
2890                         cm_ack_timeout(cm_id_priv->target_ack_delay,
2891                                        cm_id_priv->alt_av.timeout - 1);
2892
2893         ret = cm_alloc_msg(cm_id_priv, &msg);
2894         if (ret)
2895                 goto out;
2896
2897         cm_format_lap((struct cm_lap_msg *) msg->mad, cm_id_priv,
2898                       alternate_path, private_data, private_data_len);
2899         msg->timeout_ms = cm_id_priv->timeout_ms;
2900         msg->context[1] = (void *) (unsigned long) IB_CM_ESTABLISHED;
2901
2902         ret = ib_post_send_mad(msg, NULL);
2903         if (ret) {
2904                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2905                 cm_free_msg(msg);
2906                 return ret;
2907         }
2908
2909         cm_id->lap_state = IB_CM_LAP_SENT;
2910         cm_id_priv->msg = msg;
2911
2912 out:    spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2913         return ret;
2914 }
2915 EXPORT_SYMBOL(ib_send_cm_lap);
2916
2917 static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv,
2918                                     struct sa_path_rec *path,
2919                                     struct cm_lap_msg *lap_msg)
2920 {
2921         memset(path, 0, sizeof *path);
2922         path->rec_type = SA_PATH_REC_TYPE_IB;
2923         path->dgid = lap_msg->alt_local_gid;
2924         path->sgid = lap_msg->alt_remote_gid;
2925         sa_path_set_dlid(path, lap_msg->alt_local_lid);
2926         sa_path_set_slid(path, lap_msg->alt_remote_lid);
2927         path->flow_label = cm_lap_get_flow_label(lap_msg);
2928         path->hop_limit = lap_msg->alt_hop_limit;
2929         path->traffic_class = cm_lap_get_traffic_class(lap_msg);
2930         path->reversible = 1;
2931         path->pkey = cm_id_priv->pkey;
2932         path->sl = cm_lap_get_sl(lap_msg);
2933         path->mtu_selector = IB_SA_EQ;
2934         path->mtu = cm_id_priv->path_mtu;
2935         path->rate_selector = IB_SA_EQ;
2936         path->rate = cm_lap_get_packet_rate(lap_msg);
2937         path->packet_life_time_selector = IB_SA_EQ;
2938         path->packet_life_time = cm_lap_get_local_ack_timeout(lap_msg);
2939         path->packet_life_time -= (path->packet_life_time > 0);
2940 }
2941
2942 static int cm_lap_handler(struct cm_work *work)
2943 {
2944         struct cm_id_private *cm_id_priv;
2945         struct cm_lap_msg *lap_msg;
2946         struct ib_cm_lap_event_param *param;
2947         struct ib_mad_send_buf *msg = NULL;
2948         int ret;
2949
2950         /* todo: verify LAP request and send reject APR if invalid. */
2951         lap_msg = (struct cm_lap_msg *)work->mad_recv_wc->recv_buf.mad;
2952         cm_id_priv = cm_acquire_id(lap_msg->remote_comm_id,
2953                                    lap_msg->local_comm_id);
2954         if (!cm_id_priv)
2955                 return -EINVAL;
2956
2957         param = &work->cm_event.param.lap_rcvd;
2958         param->alternate_path = &work->path[0];
2959         cm_format_path_from_lap(cm_id_priv, param->alternate_path, lap_msg);
2960         work->cm_event.private_data = &lap_msg->private_data;
2961
2962         spin_lock_irq(&cm_id_priv->lock);
2963         if (cm_id_priv->id.state != IB_CM_ESTABLISHED)
2964                 goto unlock;
2965
2966         switch (cm_id_priv->id.lap_state) {
2967         case IB_CM_LAP_UNINIT:
2968         case IB_CM_LAP_IDLE:
2969                 break;
2970         case IB_CM_MRA_LAP_SENT:
2971                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2972                                 counter[CM_LAP_COUNTER]);
2973                 if (cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg))
2974                         goto unlock;
2975
2976                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
2977                               CM_MSG_RESPONSE_OTHER,
2978                               cm_id_priv->service_timeout,
2979                               cm_id_priv->private_data,
2980                               cm_id_priv->private_data_len);
2981                 spin_unlock_irq(&cm_id_priv->lock);
2982
2983                 if (ib_post_send_mad(msg, NULL))
2984                         cm_free_msg(msg);
2985                 goto deref;
2986         case IB_CM_LAP_RCVD:
2987                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2988                                 counter[CM_LAP_COUNTER]);
2989                 goto unlock;
2990         default:
2991                 goto unlock;
2992         }
2993
2994         cm_id_priv->id.lap_state = IB_CM_LAP_RCVD;
2995         cm_id_priv->tid = lap_msg->hdr.tid;
2996         cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
2997                                 work->mad_recv_wc->recv_buf.grh,
2998                                 &cm_id_priv->av);
2999         cm_init_av_by_path(param->alternate_path, &cm_id_priv->alt_av,
3000                            cm_id_priv);
3001         ret = atomic_inc_and_test(&cm_id_priv->work_count);
3002         if (!ret)
3003                 list_add_tail(&work->list, &cm_id_priv->work_list);
3004         spin_unlock_irq(&cm_id_priv->lock);
3005
3006         if (ret)
3007                 cm_process_work(cm_id_priv, work);
3008         else
3009                 cm_deref_id(cm_id_priv);
3010         return 0;
3011
3012 unlock: spin_unlock_irq(&cm_id_priv->lock);
3013 deref:  cm_deref_id(cm_id_priv);
3014         return -EINVAL;
3015 }
3016
3017 static void cm_format_apr(struct cm_apr_msg *apr_msg,
3018                           struct cm_id_private *cm_id_priv,
3019                           enum ib_cm_apr_status status,
3020                           void *info,
3021                           u8 info_length,
3022                           const void *private_data,
3023                           u8 private_data_len)
3024 {
3025         cm_format_mad_hdr(&apr_msg->hdr, CM_APR_ATTR_ID, cm_id_priv->tid);
3026         apr_msg->local_comm_id = cm_id_priv->id.local_id;
3027         apr_msg->remote_comm_id = cm_id_priv->id.remote_id;
3028         apr_msg->ap_status = (u8) status;
3029
3030         if (info && info_length) {
3031                 apr_msg->info_length = info_length;
3032                 memcpy(apr_msg->info, info, info_length);
3033         }
3034
3035         if (private_data && private_data_len)
3036                 memcpy(apr_msg->private_data, private_data, private_data_len);
3037 }
3038
3039 int ib_send_cm_apr(struct ib_cm_id *cm_id,
3040                    enum ib_cm_apr_status status,
3041                    void *info,
3042                    u8 info_length,
3043                    const void *private_data,
3044                    u8 private_data_len)
3045 {
3046         struct cm_id_private *cm_id_priv;
3047         struct ib_mad_send_buf *msg;
3048         unsigned long flags;
3049         int ret;
3050
3051         if ((private_data && private_data_len > IB_CM_APR_PRIVATE_DATA_SIZE) ||
3052             (info && info_length > IB_CM_APR_INFO_LENGTH))
3053                 return -EINVAL;
3054
3055         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3056         spin_lock_irqsave(&cm_id_priv->lock, flags);
3057         if (cm_id->state != IB_CM_ESTABLISHED ||
3058             (cm_id->lap_state != IB_CM_LAP_RCVD &&
3059              cm_id->lap_state != IB_CM_MRA_LAP_SENT)) {
3060                 ret = -EINVAL;
3061                 goto out;
3062         }
3063
3064         ret = cm_alloc_msg(cm_id_priv, &msg);
3065         if (ret)
3066                 goto out;
3067
3068         cm_format_apr((struct cm_apr_msg *) msg->mad, cm_id_priv, status,
3069                       info, info_length, private_data, private_data_len);
3070         ret = ib_post_send_mad(msg, NULL);
3071         if (ret) {
3072                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3073                 cm_free_msg(msg);
3074                 return ret;
3075         }
3076
3077         cm_id->lap_state = IB_CM_LAP_IDLE;
3078 out:    spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3079         return ret;
3080 }
3081 EXPORT_SYMBOL(ib_send_cm_apr);
3082
3083 static int cm_apr_handler(struct cm_work *work)
3084 {
3085         struct cm_id_private *cm_id_priv;
3086         struct cm_apr_msg *apr_msg;
3087         int ret;
3088
3089         apr_msg = (struct cm_apr_msg *)work->mad_recv_wc->recv_buf.mad;
3090         cm_id_priv = cm_acquire_id(apr_msg->remote_comm_id,
3091                                    apr_msg->local_comm_id);
3092         if (!cm_id_priv)
3093                 return -EINVAL; /* Unmatched reply. */
3094
3095         work->cm_event.param.apr_rcvd.ap_status = apr_msg->ap_status;
3096         work->cm_event.param.apr_rcvd.apr_info = &apr_msg->info;
3097         work->cm_event.param.apr_rcvd.info_len = apr_msg->info_length;
3098         work->cm_event.private_data = &apr_msg->private_data;
3099
3100         spin_lock_irq(&cm_id_priv->lock);
3101         if (cm_id_priv->id.state != IB_CM_ESTABLISHED ||
3102             (cm_id_priv->id.lap_state != IB_CM_LAP_SENT &&
3103              cm_id_priv->id.lap_state != IB_CM_MRA_LAP_RCVD)) {
3104                 spin_unlock_irq(&cm_id_priv->lock);
3105                 goto out;
3106         }
3107         cm_id_priv->id.lap_state = IB_CM_LAP_IDLE;
3108         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
3109         cm_id_priv->msg = NULL;
3110
3111         ret = atomic_inc_and_test(&cm_id_priv->work_count);
3112         if (!ret)
3113                 list_add_tail(&work->list, &cm_id_priv->work_list);
3114         spin_unlock_irq(&cm_id_priv->lock);
3115
3116         if (ret)
3117                 cm_process_work(cm_id_priv, work);
3118         else
3119                 cm_deref_id(cm_id_priv);
3120         return 0;
3121 out:
3122         cm_deref_id(cm_id_priv);
3123         return -EINVAL;
3124 }
3125
3126 static int cm_timewait_handler(struct cm_work *work)
3127 {
3128         struct cm_timewait_info *timewait_info;
3129         struct cm_id_private *cm_id_priv;
3130         int ret;
3131
3132         timewait_info = (struct cm_timewait_info *)work;
3133         spin_lock_irq(&cm.lock);
3134         list_del(&timewait_info->list);
3135         spin_unlock_irq(&cm.lock);
3136
3137         cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
3138                                    timewait_info->work.remote_id);
3139         if (!cm_id_priv)
3140                 return -EINVAL;
3141
3142         spin_lock_irq(&cm_id_priv->lock);
3143         if (cm_id_priv->id.state != IB_CM_TIMEWAIT ||
3144             cm_id_priv->remote_qpn != timewait_info->remote_qpn) {
3145                 spin_unlock_irq(&cm_id_priv->lock);
3146                 goto out;
3147         }
3148         cm_id_priv->id.state = IB_CM_IDLE;
3149         ret = atomic_inc_and_test(&cm_id_priv->work_count);
3150         if (!ret)
3151                 list_add_tail(&work->list, &cm_id_priv->work_list);
3152         spin_unlock_irq(&cm_id_priv->lock);
3153
3154         if (ret)
3155                 cm_process_work(cm_id_priv, work);
3156         else
3157                 cm_deref_id(cm_id_priv);
3158         return 0;
3159 out:
3160         cm_deref_id(cm_id_priv);
3161         return -EINVAL;
3162 }
3163
3164 static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg,
3165                                struct cm_id_private *cm_id_priv,
3166                                struct ib_cm_sidr_req_param *param)
3167 {
3168         cm_format_mad_hdr(&sidr_req_msg->hdr, CM_SIDR_REQ_ATTR_ID,
3169                           cm_form_tid(cm_id_priv, CM_MSG_SEQUENCE_SIDR));
3170         sidr_req_msg->request_id = cm_id_priv->id.local_id;
3171         sidr_req_msg->pkey = param->path->pkey;
3172         sidr_req_msg->service_id = param->service_id;
3173
3174         if (param->private_data && param->private_data_len)
3175                 memcpy(sidr_req_msg->private_data, param->private_data,
3176                        param->private_data_len);
3177 }
3178
3179 int ib_send_cm_sidr_req(struct ib_cm_id *cm_id,
3180                         struct ib_cm_sidr_req_param *param)
3181 {
3182         struct cm_id_private *cm_id_priv;
3183         struct ib_mad_send_buf *msg;
3184         unsigned long flags;
3185         int ret;
3186
3187         if (!param->path || (param->private_data &&
3188              param->private_data_len > IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE))
3189                 return -EINVAL;
3190
3191         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3192         ret = cm_init_av_by_path(param->path, &cm_id_priv->av, cm_id_priv);
3193         if (ret)
3194                 goto out;
3195
3196         cm_id->service_id = param->service_id;
3197         cm_id->service_mask = ~cpu_to_be64(0);
3198         cm_id_priv->timeout_ms = param->timeout_ms;
3199         cm_id_priv->max_cm_retries = param->max_cm_retries;
3200         ret = cm_alloc_msg(cm_id_priv, &msg);
3201         if (ret)
3202                 goto out;
3203
3204         cm_format_sidr_req((struct cm_sidr_req_msg *) msg->mad, cm_id_priv,
3205                            param);
3206         msg->timeout_ms = cm_id_priv->timeout_ms;
3207         msg->context[1] = (void *) (unsigned long) IB_CM_SIDR_REQ_SENT;
3208
3209         spin_lock_irqsave(&cm_id_priv->lock, flags);
3210         if (cm_id->state == IB_CM_IDLE)
3211                 ret = ib_post_send_mad(msg, NULL);
3212         else
3213                 ret = -EINVAL;
3214
3215         if (ret) {
3216                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3217                 cm_free_msg(msg);
3218                 goto out;
3219         }
3220         cm_id->state = IB_CM_SIDR_REQ_SENT;
3221         cm_id_priv->msg = msg;
3222         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3223 out:
3224         return ret;
3225 }
3226 EXPORT_SYMBOL(ib_send_cm_sidr_req);
3227
3228 static void cm_format_sidr_req_event(struct cm_work *work,
3229                                      struct ib_cm_id *listen_id)
3230 {
3231         struct cm_sidr_req_msg *sidr_req_msg;
3232         struct ib_cm_sidr_req_event_param *param;
3233
3234         sidr_req_msg = (struct cm_sidr_req_msg *)
3235                                 work->mad_recv_wc->recv_buf.mad;
3236         param = &work->cm_event.param.sidr_req_rcvd;
3237         param->pkey = __be16_to_cpu(sidr_req_msg->pkey);
3238         param->listen_id = listen_id;
3239         param->service_id = sidr_req_msg->service_id;
3240         param->bth_pkey = cm_get_bth_pkey(work);
3241         param->port = work->port->port_num;
3242         work->cm_event.private_data = &sidr_req_msg->private_data;
3243 }
3244
3245 static int cm_sidr_req_handler(struct cm_work *work)
3246 {
3247         struct ib_cm_id *cm_id;
3248         struct cm_id_private *cm_id_priv, *cur_cm_id_priv;
3249         struct cm_sidr_req_msg *sidr_req_msg;
3250         struct ib_wc *wc;
3251
3252         cm_id = ib_create_cm_id(work->port->cm_dev->ib_device, NULL, NULL);
3253         if (IS_ERR(cm_id))
3254                 return PTR_ERR(cm_id);
3255         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3256
3257         /* Record SGID/SLID and request ID for lookup. */
3258         sidr_req_msg = (struct cm_sidr_req_msg *)
3259                                 work->mad_recv_wc->recv_buf.mad;
3260         wc = work->mad_recv_wc->wc;
3261         cm_id_priv->av.dgid.global.subnet_prefix = cpu_to_be64(wc->slid);
3262         cm_id_priv->av.dgid.global.interface_id = 0;
3263         cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
3264                                 work->mad_recv_wc->recv_buf.grh,
3265                                 &cm_id_priv->av);
3266         cm_id_priv->id.remote_id = sidr_req_msg->request_id;
3267         cm_id_priv->tid = sidr_req_msg->hdr.tid;
3268         atomic_inc(&cm_id_priv->work_count);
3269
3270         spin_lock_irq(&cm.lock);
3271         cur_cm_id_priv = cm_insert_remote_sidr(cm_id_priv);
3272         if (cur_cm_id_priv) {
3273                 spin_unlock_irq(&cm.lock);
3274                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
3275                                 counter[CM_SIDR_REQ_COUNTER]);
3276                 goto out; /* Duplicate message. */
3277         }
3278         cm_id_priv->id.state = IB_CM_SIDR_REQ_RCVD;
3279         cur_cm_id_priv = cm_find_listen(cm_id->device,
3280                                         sidr_req_msg->service_id);
3281         if (!cur_cm_id_priv) {
3282                 spin_unlock_irq(&cm.lock);
3283                 cm_reject_sidr_req(cm_id_priv, IB_SIDR_UNSUPPORTED);
3284                 goto out; /* No match. */
3285         }
3286         atomic_inc(&cur_cm_id_priv->refcount);
3287         atomic_inc(&cm_id_priv->refcount);
3288         spin_unlock_irq(&cm.lock);
3289
3290         cm_id_priv->id.cm_handler = cur_cm_id_priv->id.cm_handler;
3291         cm_id_priv->id.context = cur_cm_id_priv->id.context;
3292         cm_id_priv->id.service_id = sidr_req_msg->service_id;
3293         cm_id_priv->id.service_mask = ~cpu_to_be64(0);
3294
3295         cm_format_sidr_req_event(work, &cur_cm_id_priv->id);
3296         cm_process_work(cm_id_priv, work);
3297         cm_deref_id(cur_cm_id_priv);
3298         return 0;
3299 out:
3300         ib_destroy_cm_id(&cm_id_priv->id);
3301         return -EINVAL;
3302 }
3303
3304 static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg,
3305                                struct cm_id_private *cm_id_priv,
3306                                struct ib_cm_sidr_rep_param *param)
3307 {
3308         cm_format_mad_hdr(&sidr_rep_msg->hdr, CM_SIDR_REP_ATTR_ID,
3309                           cm_id_priv->tid);
3310         sidr_rep_msg->request_id = cm_id_priv->id.remote_id;
3311         sidr_rep_msg->status = param->status;
3312         cm_sidr_rep_set_qpn(sidr_rep_msg, cpu_to_be32(param->qp_num));
3313         sidr_rep_msg->service_id = cm_id_priv->id.service_id;
3314         sidr_rep_msg->qkey = cpu_to_be32(param->qkey);
3315
3316         if (param->info && param->info_length)
3317                 memcpy(sidr_rep_msg->info, param->info, param->info_length);
3318
3319         if (param->private_data && param->private_data_len)
3320                 memcpy(sidr_rep_msg->private_data, param->private_data,
3321                        param->private_data_len);
3322 }
3323
3324 int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id,
3325                         struct ib_cm_sidr_rep_param *param)
3326 {
3327         struct cm_id_private *cm_id_priv;
3328         struct ib_mad_send_buf *msg;
3329         unsigned long flags;
3330         int ret;
3331
3332         if ((param->info && param->info_length > IB_CM_SIDR_REP_INFO_LENGTH) ||
3333             (param->private_data &&
3334              param->private_data_len > IB_CM_SIDR_REP_PRIVATE_DATA_SIZE))
3335                 return -EINVAL;
3336
3337         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3338         spin_lock_irqsave(&cm_id_priv->lock, flags);
3339         if (cm_id->state != IB_CM_SIDR_REQ_RCVD) {
3340                 ret = -EINVAL;
3341                 goto error;
3342         }
3343
3344         ret = cm_alloc_msg(cm_id_priv, &msg);
3345         if (ret)
3346                 goto error;
3347
3348         cm_format_sidr_rep((struct cm_sidr_rep_msg *) msg->mad, cm_id_priv,
3349                            param);
3350         ret = ib_post_send_mad(msg, NULL);
3351         if (ret) {
3352                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3353                 cm_free_msg(msg);
3354                 return ret;
3355         }
3356         cm_id->state = IB_CM_IDLE;
3357         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3358
3359         spin_lock_irqsave(&cm.lock, flags);
3360         if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node)) {
3361                 rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
3362                 RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
3363         }
3364         spin_unlock_irqrestore(&cm.lock, flags);
3365         return 0;
3366
3367 error:  spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3368         return ret;
3369 }
3370 EXPORT_SYMBOL(ib_send_cm_sidr_rep);
3371
3372 static void cm_format_sidr_rep_event(struct cm_work *work)
3373 {
3374         struct cm_sidr_rep_msg *sidr_rep_msg;
3375         struct ib_cm_sidr_rep_event_param *param;
3376
3377         sidr_rep_msg = (struct cm_sidr_rep_msg *)
3378                                 work->mad_recv_wc->recv_buf.mad;
3379         param = &work->cm_event.param.sidr_rep_rcvd;
3380         param->status = sidr_rep_msg->status;
3381         param->qkey = be32_to_cpu(sidr_rep_msg->qkey);
3382         param->qpn = be32_to_cpu(cm_sidr_rep_get_qpn(sidr_rep_msg));
3383         param->info = &sidr_rep_msg->info;
3384         param->info_len = sidr_rep_msg->info_length;
3385         work->cm_event.private_data = &sidr_rep_msg->private_data;
3386 }
3387
3388 static int cm_sidr_rep_handler(struct cm_work *work)
3389 {
3390         struct cm_sidr_rep_msg *sidr_rep_msg;
3391         struct cm_id_private *cm_id_priv;
3392
3393         sidr_rep_msg = (struct cm_sidr_rep_msg *)
3394                                 work->mad_recv_wc->recv_buf.mad;
3395         cm_id_priv = cm_acquire_id(sidr_rep_msg->request_id, 0);
3396         if (!cm_id_priv)
3397                 return -EINVAL; /* Unmatched reply. */
3398
3399         spin_lock_irq(&cm_id_priv->lock);
3400         if (cm_id_priv->id.state != IB_CM_SIDR_REQ_SENT) {
3401                 spin_unlock_irq(&cm_id_priv->lock);
3402                 goto out;
3403         }
3404         cm_id_priv->id.state = IB_CM_IDLE;
3405         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
3406         spin_unlock_irq(&cm_id_priv->lock);
3407
3408         cm_format_sidr_rep_event(work);
3409         cm_process_work(cm_id_priv, work);
3410         return 0;
3411 out:
3412         cm_deref_id(cm_id_priv);
3413         return -EINVAL;
3414 }
3415
3416 static void cm_process_send_error(struct ib_mad_send_buf *msg,
3417                                   enum ib_wc_status wc_status)
3418 {
3419         struct cm_id_private *cm_id_priv;
3420         struct ib_cm_event cm_event;
3421         enum ib_cm_state state;
3422         int ret;
3423
3424         memset(&cm_event, 0, sizeof cm_event);
3425         cm_id_priv = msg->context[0];
3426
3427         /* Discard old sends or ones without a response. */
3428         spin_lock_irq(&cm_id_priv->lock);
3429         state = (enum ib_cm_state) (unsigned long) msg->context[1];
3430         if (msg != cm_id_priv->msg || state != cm_id_priv->id.state)
3431                 goto discard;
3432
3433         pr_debug_ratelimited("CM: failed sending MAD in state %d. (%s)\n",
3434                              state, ib_wc_status_msg(wc_status));
3435         switch (state) {
3436         case IB_CM_REQ_SENT:
3437         case IB_CM_MRA_REQ_RCVD:
3438                 cm_reset_to_idle(cm_id_priv);
3439                 cm_event.event = IB_CM_REQ_ERROR;
3440                 break;
3441         case IB_CM_REP_SENT:
3442         case IB_CM_MRA_REP_RCVD:
3443                 cm_reset_to_idle(cm_id_priv);
3444                 cm_event.event = IB_CM_REP_ERROR;
3445                 break;
3446         case IB_CM_DREQ_SENT:
3447                 cm_enter_timewait(cm_id_priv);
3448                 cm_event.event = IB_CM_DREQ_ERROR;
3449                 break;
3450         case IB_CM_SIDR_REQ_SENT:
3451                 cm_id_priv->id.state = IB_CM_IDLE;
3452                 cm_event.event = IB_CM_SIDR_REQ_ERROR;
3453                 break;
3454         default:
3455                 goto discard;
3456         }
3457         spin_unlock_irq(&cm_id_priv->lock);
3458         cm_event.param.send_status = wc_status;
3459
3460         /* No other events can occur on the cm_id at this point. */
3461         ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &cm_event);
3462         cm_free_msg(msg);
3463         if (ret)
3464                 ib_destroy_cm_id(&cm_id_priv->id);
3465         return;
3466 discard:
3467         spin_unlock_irq(&cm_id_priv->lock);
3468         cm_free_msg(msg);
3469 }
3470
3471 static void cm_send_handler(struct ib_mad_agent *mad_agent,
3472                             struct ib_mad_send_wc *mad_send_wc)
3473 {
3474         struct ib_mad_send_buf *msg = mad_send_wc->send_buf;
3475         struct cm_port *port;
3476         u16 attr_index;
3477
3478         port = mad_agent->context;
3479         attr_index = be16_to_cpu(((struct ib_mad_hdr *)
3480                                   msg->mad)->attr_id) - CM_ATTR_ID_OFFSET;
3481
3482         /*
3483          * If the send was in response to a received message (context[0] is not
3484          * set to a cm_id), and is not a REJ, then it is a send that was
3485          * manually retried.
3486          */
3487         if (!msg->context[0] && (attr_index != CM_REJ_COUNTER))
3488                 msg->retries = 1;
3489
3490         atomic_long_add(1 + msg->retries,
3491                         &port->counter_group[CM_XMIT].counter[attr_index]);
3492         if (msg->retries)
3493                 atomic_long_add(msg->retries,
3494                                 &port->counter_group[CM_XMIT_RETRIES].
3495                                 counter[attr_index]);
3496
3497         switch (mad_send_wc->status) {
3498         case IB_WC_SUCCESS:
3499         case IB_WC_WR_FLUSH_ERR:
3500                 cm_free_msg(msg);
3501                 break;
3502         default:
3503                 if (msg->context[0] && msg->context[1])
3504                         cm_process_send_error(msg, mad_send_wc->status);
3505                 else
3506                         cm_free_msg(msg);
3507                 break;
3508         }
3509 }
3510
3511 static void cm_work_handler(struct work_struct *_work)
3512 {
3513         struct cm_work *work = container_of(_work, struct cm_work, work.work);
3514         int ret;
3515
3516         switch (work->cm_event.event) {
3517         case IB_CM_REQ_RECEIVED:
3518                 ret = cm_req_handler(work);
3519                 break;
3520         case IB_CM_MRA_RECEIVED:
3521                 ret = cm_mra_handler(work);
3522                 break;
3523         case IB_CM_REJ_RECEIVED:
3524                 ret = cm_rej_handler(work);
3525                 break;
3526         case IB_CM_REP_RECEIVED:
3527                 ret = cm_rep_handler(work);
3528                 break;
3529         case IB_CM_RTU_RECEIVED:
3530                 ret = cm_rtu_handler(work);
3531                 break;
3532         case IB_CM_USER_ESTABLISHED:
3533                 ret = cm_establish_handler(work);
3534                 break;
3535         case IB_CM_DREQ_RECEIVED:
3536                 ret = cm_dreq_handler(work);
3537                 break;
3538         case IB_CM_DREP_RECEIVED:
3539                 ret = cm_drep_handler(work);
3540                 break;
3541         case IB_CM_SIDR_REQ_RECEIVED:
3542                 ret = cm_sidr_req_handler(work);
3543                 break;
3544         case IB_CM_SIDR_REP_RECEIVED:
3545                 ret = cm_sidr_rep_handler(work);
3546                 break;
3547         case IB_CM_LAP_RECEIVED:
3548                 ret = cm_lap_handler(work);
3549                 break;
3550         case IB_CM_APR_RECEIVED:
3551                 ret = cm_apr_handler(work);
3552                 break;
3553         case IB_CM_TIMEWAIT_EXIT:
3554                 ret = cm_timewait_handler(work);
3555                 break;
3556         default:
3557                 ret = -EINVAL;
3558                 break;
3559         }
3560         if (ret)
3561                 cm_free_work(work);
3562 }
3563
3564 static int cm_establish(struct ib_cm_id *cm_id)
3565 {
3566         struct cm_id_private *cm_id_priv;
3567         struct cm_work *work;
3568         unsigned long flags;
3569         int ret = 0;
3570         struct cm_device *cm_dev;
3571
3572         cm_dev = ib_get_client_data(cm_id->device, &cm_client);
3573         if (!cm_dev)
3574                 return -ENODEV;
3575
3576         work = kmalloc(sizeof *work, GFP_ATOMIC);
3577         if (!work)
3578                 return -ENOMEM;
3579
3580         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3581         spin_lock_irqsave(&cm_id_priv->lock, flags);
3582         switch (cm_id->state)
3583         {
3584         case IB_CM_REP_SENT:
3585         case IB_CM_MRA_REP_RCVD:
3586                 cm_id->state = IB_CM_ESTABLISHED;
3587                 break;
3588         case IB_CM_ESTABLISHED:
3589                 ret = -EISCONN;
3590                 break;
3591         default:
3592                 ret = -EINVAL;
3593                 break;
3594         }
3595         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3596
3597         if (ret) {
3598                 kfree(work);
3599                 goto out;
3600         }
3601
3602         /*
3603          * The CM worker thread may try to destroy the cm_id before it
3604          * can execute this work item.  To prevent potential deadlock,
3605          * we need to find the cm_id once we're in the context of the
3606          * worker thread, rather than holding a reference on it.
3607          */
3608         INIT_DELAYED_WORK(&work->work, cm_work_handler);
3609         work->local_id = cm_id->local_id;
3610         work->remote_id = cm_id->remote_id;
3611         work->mad_recv_wc = NULL;
3612         work->cm_event.event = IB_CM_USER_ESTABLISHED;
3613
3614         /* Check if the device started its remove_one */
3615         spin_lock_irqsave(&cm.lock, flags);
3616         if (!cm_dev->going_down) {
3617                 queue_delayed_work(cm.wq, &work->work, 0);
3618         } else {
3619                 kfree(work);
3620                 ret = -ENODEV;
3621         }
3622         spin_unlock_irqrestore(&cm.lock, flags);
3623
3624 out:
3625         return ret;
3626 }
3627
3628 static int cm_migrate(struct ib_cm_id *cm_id)
3629 {
3630         struct cm_id_private *cm_id_priv;
3631         struct cm_av tmp_av;
3632         unsigned long flags;
3633         int tmp_send_port_not_ready;
3634         int ret = 0;
3635
3636         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3637         spin_lock_irqsave(&cm_id_priv->lock, flags);
3638         if (cm_id->state == IB_CM_ESTABLISHED &&
3639             (cm_id->lap_state == IB_CM_LAP_UNINIT ||
3640              cm_id->lap_state == IB_CM_LAP_IDLE)) {
3641                 cm_id->lap_state = IB_CM_LAP_IDLE;
3642                 /* Swap address vector */
3643                 tmp_av = cm_id_priv->av;
3644                 cm_id_priv->av = cm_id_priv->alt_av;
3645                 cm_id_priv->alt_av = tmp_av;
3646                 /* Swap port send ready state */
3647                 tmp_send_port_not_ready = cm_id_priv->prim_send_port_not_ready;
3648                 cm_id_priv->prim_send_port_not_ready = cm_id_priv->altr_send_port_not_ready;
3649                 cm_id_priv->altr_send_port_not_ready = tmp_send_port_not_ready;
3650         } else
3651                 ret = -EINVAL;
3652         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3653
3654         return ret;
3655 }
3656
3657 int ib_cm_notify(struct ib_cm_id *cm_id, enum ib_event_type event)
3658 {
3659         int ret;
3660
3661         switch (event) {
3662         case IB_EVENT_COMM_EST:
3663                 ret = cm_establish(cm_id);
3664                 break;
3665         case IB_EVENT_PATH_MIG:
3666                 ret = cm_migrate(cm_id);
3667                 break;
3668         default:
3669                 ret = -EINVAL;
3670         }
3671         return ret;
3672 }
3673 EXPORT_SYMBOL(ib_cm_notify);
3674
3675 static void cm_recv_handler(struct ib_mad_agent *mad_agent,
3676                             struct ib_mad_send_buf *send_buf,
3677                             struct ib_mad_recv_wc *mad_recv_wc)
3678 {
3679         struct cm_port *port = mad_agent->context;
3680         struct cm_work *work;
3681         enum ib_cm_event_type event;
3682         u16 attr_id;
3683         int paths = 0;
3684         int going_down = 0;
3685
3686         switch (mad_recv_wc->recv_buf.mad->mad_hdr.attr_id) {
3687         case CM_REQ_ATTR_ID:
3688                 paths = 1 + (((struct cm_req_msg *) mad_recv_wc->recv_buf.mad)->
3689                                                     alt_local_lid != 0);
3690                 event = IB_CM_REQ_RECEIVED;
3691                 break;
3692         case CM_MRA_ATTR_ID:
3693                 event = IB_CM_MRA_RECEIVED;
3694                 break;
3695         case CM_REJ_ATTR_ID:
3696                 event = IB_CM_REJ_RECEIVED;
3697                 break;
3698         case CM_REP_ATTR_ID:
3699                 event = IB_CM_REP_RECEIVED;
3700                 break;
3701         case CM_RTU_ATTR_ID:
3702                 event = IB_CM_RTU_RECEIVED;
3703                 break;
3704         case CM_DREQ_ATTR_ID:
3705                 event = IB_CM_DREQ_RECEIVED;
3706                 break;
3707         case CM_DREP_ATTR_ID:
3708                 event = IB_CM_DREP_RECEIVED;
3709                 break;
3710         case CM_SIDR_REQ_ATTR_ID:
3711                 event = IB_CM_SIDR_REQ_RECEIVED;
3712                 break;
3713         case CM_SIDR_REP_ATTR_ID:
3714                 event = IB_CM_SIDR_REP_RECEIVED;
3715                 break;
3716         case CM_LAP_ATTR_ID:
3717                 paths = 1;
3718                 event = IB_CM_LAP_RECEIVED;
3719                 break;
3720         case CM_APR_ATTR_ID:
3721                 event = IB_CM_APR_RECEIVED;
3722                 break;
3723         default:
3724                 ib_free_recv_mad(mad_recv_wc);
3725                 return;
3726         }
3727
3728         attr_id = be16_to_cpu(mad_recv_wc->recv_buf.mad->mad_hdr.attr_id);
3729         atomic_long_inc(&port->counter_group[CM_RECV].
3730                         counter[attr_id - CM_ATTR_ID_OFFSET]);
3731
3732         work = kmalloc(sizeof(*work) + sizeof(struct sa_path_rec) * paths,
3733                        GFP_KERNEL);
3734         if (!work) {
3735                 ib_free_recv_mad(mad_recv_wc);
3736                 return;
3737         }
3738
3739         INIT_DELAYED_WORK(&work->work, cm_work_handler);
3740         work->cm_event.event = event;
3741         work->mad_recv_wc = mad_recv_wc;
3742         work->port = port;
3743
3744         /* Check if the device started its remove_one */
3745         spin_lock_irq(&cm.lock);
3746         if (!port->cm_dev->going_down)
3747                 queue_delayed_work(cm.wq, &work->work, 0);
3748         else
3749                 going_down = 1;
3750         spin_unlock_irq(&cm.lock);
3751
3752         if (going_down) {
3753                 kfree(work);
3754                 ib_free_recv_mad(mad_recv_wc);
3755         }
3756 }
3757
3758 static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv,
3759                                 struct ib_qp_attr *qp_attr,
3760                                 int *qp_attr_mask)
3761 {
3762         unsigned long flags;
3763         int ret;
3764
3765         spin_lock_irqsave(&cm_id_priv->lock, flags);
3766         switch (cm_id_priv->id.state) {
3767         case IB_CM_REQ_SENT:
3768         case IB_CM_MRA_REQ_RCVD:
3769         case IB_CM_REQ_RCVD:
3770         case IB_CM_MRA_REQ_SENT:
3771         case IB_CM_REP_RCVD:
3772         case IB_CM_MRA_REP_SENT:
3773         case IB_CM_REP_SENT:
3774         case IB_CM_MRA_REP_RCVD:
3775         case IB_CM_ESTABLISHED:
3776                 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS |
3777                                 IB_QP_PKEY_INDEX | IB_QP_PORT;
3778                 qp_attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
3779                 if (cm_id_priv->responder_resources)
3780                         qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ |
3781                                                     IB_ACCESS_REMOTE_ATOMIC;
3782                 qp_attr->pkey_index = cm_id_priv->av.pkey_index;
3783                 qp_attr->port_num = cm_id_priv->av.port->port_num;
3784                 ret = 0;
3785                 break;
3786         default:
3787                 ret = -EINVAL;
3788                 break;
3789         }
3790         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3791         return ret;
3792 }
3793
3794 static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv,
3795                                struct ib_qp_attr *qp_attr,
3796                                int *qp_attr_mask)
3797 {
3798         unsigned long flags;
3799         int ret;
3800
3801         spin_lock_irqsave(&cm_id_priv->lock, flags);
3802         switch (cm_id_priv->id.state) {
3803         case IB_CM_REQ_RCVD:
3804         case IB_CM_MRA_REQ_SENT:
3805         case IB_CM_REP_RCVD:
3806         case IB_CM_MRA_REP_SENT:
3807         case IB_CM_REP_SENT:
3808         case IB_CM_MRA_REP_RCVD:
3809         case IB_CM_ESTABLISHED:
3810                 *qp_attr_mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU |
3811                                 IB_QP_DEST_QPN | IB_QP_RQ_PSN;
3812                 qp_attr->ah_attr = cm_id_priv->av.ah_attr;
3813                 qp_attr->path_mtu = cm_id_priv->path_mtu;
3814                 qp_attr->dest_qp_num = be32_to_cpu(cm_id_priv->remote_qpn);
3815                 qp_attr->rq_psn = be32_to_cpu(cm_id_priv->rq_psn);
3816                 if (cm_id_priv->qp_type == IB_QPT_RC ||
3817                     cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
3818                         *qp_attr_mask |= IB_QP_MAX_DEST_RD_ATOMIC |
3819                                          IB_QP_MIN_RNR_TIMER;
3820                         qp_attr->max_dest_rd_atomic =
3821                                         cm_id_priv->responder_resources;
3822                         qp_attr->min_rnr_timer = 0;
3823                 }
3824                 if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) {
3825                         *qp_attr_mask |= IB_QP_ALT_PATH;
3826                         qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num;
3827                         qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
3828                         qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
3829                         qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
3830                 }
3831                 ret = 0;
3832                 break;
3833         default:
3834                 ret = -EINVAL;
3835                 break;
3836         }
3837         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3838         return ret;
3839 }
3840
3841 static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv,
3842                                struct ib_qp_attr *qp_attr,
3843                                int *qp_attr_mask)
3844 {
3845         unsigned long flags;
3846         int ret;
3847
3848         spin_lock_irqsave(&cm_id_priv->lock, flags);
3849         switch (cm_id_priv->id.state) {
3850         /* Allow transition to RTS before sending REP */
3851         case IB_CM_REQ_RCVD:
3852         case IB_CM_MRA_REQ_SENT:
3853
3854         case IB_CM_REP_RCVD:
3855         case IB_CM_MRA_REP_SENT:
3856         case IB_CM_REP_SENT:
3857         case IB_CM_MRA_REP_RCVD:
3858         case IB_CM_ESTABLISHED:
3859                 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT) {
3860                         *qp_attr_mask = IB_QP_STATE | IB_QP_SQ_PSN;
3861                         qp_attr->sq_psn = be32_to_cpu(cm_id_priv->sq_psn);
3862                         switch (cm_id_priv->qp_type) {
3863                         case IB_QPT_RC:
3864                         case IB_QPT_XRC_INI:
3865                                 *qp_attr_mask |= IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
3866                                                  IB_QP_MAX_QP_RD_ATOMIC;
3867                                 qp_attr->retry_cnt = cm_id_priv->retry_count;
3868                                 qp_attr->rnr_retry = cm_id_priv->rnr_retry_count;
3869                                 qp_attr->max_rd_atomic = cm_id_priv->initiator_depth;
3870                                 /* fall through */
3871                         case IB_QPT_XRC_TGT:
3872                                 *qp_attr_mask |= IB_QP_TIMEOUT;
3873                                 qp_attr->timeout = cm_id_priv->av.timeout;
3874                                 break;
3875                         default:
3876                                 break;
3877                         }
3878                         if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) {
3879                                 *qp_attr_mask |= IB_QP_PATH_MIG_STATE;
3880                                 qp_attr->path_mig_state = IB_MIG_REARM;
3881                         }
3882                 } else {
3883                         *qp_attr_mask = IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE;
3884                         qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num;
3885                         qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
3886                         qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
3887                         qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
3888                         qp_attr->path_mig_state = IB_MIG_REARM;
3889                 }
3890                 ret = 0;
3891                 break;
3892         default:
3893                 ret = -EINVAL;
3894                 break;
3895         }
3896         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3897         return ret;
3898 }
3899
3900 int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
3901                        struct ib_qp_attr *qp_attr,
3902                        int *qp_attr_mask)
3903 {
3904         struct cm_id_private *cm_id_priv;
3905         int ret;
3906
3907         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3908         switch (qp_attr->qp_state) {
3909         case IB_QPS_INIT:
3910                 ret = cm_init_qp_init_attr(cm_id_priv, qp_attr, qp_attr_mask);
3911                 break;
3912         case IB_QPS_RTR:
3913                 ret = cm_init_qp_rtr_attr(cm_id_priv, qp_attr, qp_attr_mask);
3914                 break;
3915         case IB_QPS_RTS:
3916                 ret = cm_init_qp_rts_attr(cm_id_priv, qp_attr, qp_attr_mask);
3917                 break;
3918         default:
3919                 ret = -EINVAL;
3920                 break;
3921         }
3922         return ret;
3923 }
3924 EXPORT_SYMBOL(ib_cm_init_qp_attr);
3925
3926 static ssize_t cm_show_counter(struct kobject *obj, struct attribute *attr,
3927                                char *buf)
3928 {
3929         struct cm_counter_group *group;
3930         struct cm_counter_attribute *cm_attr;
3931
3932         group = container_of(obj, struct cm_counter_group, obj);
3933         cm_attr = container_of(attr, struct cm_counter_attribute, attr);
3934
3935         return sprintf(buf, "%ld\n",
3936                        atomic_long_read(&group->counter[cm_attr->index]));
3937 }
3938
3939 static const struct sysfs_ops cm_counter_ops = {
3940         .show = cm_show_counter
3941 };
3942
3943 static struct kobj_type cm_counter_obj_type = {
3944         .sysfs_ops = &cm_counter_ops,
3945         .default_attrs = cm_counter_default_attrs
3946 };
3947
3948 static void cm_release_port_obj(struct kobject *obj)
3949 {
3950         struct cm_port *cm_port;
3951
3952         cm_port = container_of(obj, struct cm_port, port_obj);
3953         kfree(cm_port);
3954 }
3955
3956 static struct kobj_type cm_port_obj_type = {
3957         .release = cm_release_port_obj
3958 };
3959
3960 static char *cm_devnode(struct device *dev, umode_t *mode)
3961 {
3962         if (mode)
3963                 *mode = 0666;
3964         return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev));
3965 }
3966
3967 struct class cm_class = {
3968         .owner   = THIS_MODULE,
3969         .name    = "infiniband_cm",
3970         .devnode = cm_devnode,
3971 };
3972 EXPORT_SYMBOL(cm_class);
3973
3974 static int cm_create_port_fs(struct cm_port *port)
3975 {
3976         int i, ret;
3977
3978         ret = kobject_init_and_add(&port->port_obj, &cm_port_obj_type,
3979                                    &port->cm_dev->device->kobj,
3980                                    "%d", port->port_num);
3981         if (ret) {
3982                 kfree(port);
3983                 return ret;
3984         }
3985
3986         for (i = 0; i < CM_COUNTER_GROUPS; i++) {
3987                 ret = kobject_init_and_add(&port->counter_group[i].obj,
3988                                            &cm_counter_obj_type,
3989                                            &port->port_obj,
3990                                            "%s", counter_group_names[i]);
3991                 if (ret)
3992                         goto error;
3993         }
3994
3995         return 0;
3996
3997 error:
3998         while (i--)
3999                 kobject_put(&port->counter_group[i].obj);
4000         kobject_put(&port->port_obj);
4001         return ret;
4002
4003 }
4004
4005 static void cm_remove_port_fs(struct cm_port *port)
4006 {
4007         int i;
4008
4009         for (i = 0; i < CM_COUNTER_GROUPS; i++)
4010                 kobject_put(&port->counter_group[i].obj);
4011
4012         kobject_put(&port->port_obj);
4013 }
4014
4015 static void cm_add_one(struct ib_device *ib_device)
4016 {
4017         struct cm_device *cm_dev;
4018         struct cm_port *port;
4019         struct ib_mad_reg_req reg_req = {
4020                 .mgmt_class = IB_MGMT_CLASS_CM,
4021                 .mgmt_class_version = IB_CM_CLASS_VERSION,
4022         };
4023         struct ib_port_modify port_modify = {
4024                 .set_port_cap_mask = IB_PORT_CM_SUP
4025         };
4026         unsigned long flags;
4027         int ret;
4028         int count = 0;
4029         u8 i;
4030
4031         cm_dev = kzalloc(sizeof(*cm_dev) + sizeof(*port) *
4032                          ib_device->phys_port_cnt, GFP_KERNEL);
4033         if (!cm_dev)
4034                 return;
4035
4036         cm_dev->ib_device = ib_device;
4037         cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay;
4038         cm_dev->going_down = 0;
4039         cm_dev->device = device_create(&cm_class, &ib_device->dev,
4040                                        MKDEV(0, 0), NULL,
4041                                        "%s", ib_device->name);
4042         if (IS_ERR(cm_dev->device)) {
4043                 kfree(cm_dev);
4044                 return;
4045         }
4046
4047         set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask);
4048         for (i = 1; i <= ib_device->phys_port_cnt; i++) {
4049                 if (!rdma_cap_ib_cm(ib_device, i))
4050                         continue;
4051
4052                 port = kzalloc(sizeof *port, GFP_KERNEL);
4053                 if (!port)
4054                         goto error1;
4055
4056                 cm_dev->port[i-1] = port;
4057                 port->cm_dev = cm_dev;
4058                 port->port_num = i;
4059
4060                 INIT_LIST_HEAD(&port->cm_priv_prim_list);
4061                 INIT_LIST_HEAD(&port->cm_priv_altr_list);
4062
4063                 ret = cm_create_port_fs(port);
4064                 if (ret)
4065                         goto error1;
4066
4067                 port->mad_agent = ib_register_mad_agent(ib_device, i,
4068                                                         IB_QPT_GSI,
4069                                                         &reg_req,
4070                                                         0,
4071                                                         cm_send_handler,
4072                                                         cm_recv_handler,
4073                                                         port,
4074                                                         0);
4075                 if (IS_ERR(port->mad_agent))
4076                         goto error2;
4077
4078                 ret = ib_modify_port(ib_device, i, 0, &port_modify);
4079                 if (ret)
4080                         goto error3;
4081
4082                 count++;
4083         }
4084
4085         if (!count)
4086                 goto free;
4087
4088         ib_set_client_data(ib_device, &cm_client, cm_dev);
4089
4090         write_lock_irqsave(&cm.device_lock, flags);
4091         list_add_tail(&cm_dev->list, &cm.device_list);
4092         write_unlock_irqrestore(&cm.device_lock, flags);
4093         return;
4094
4095 error3:
4096         ib_unregister_mad_agent(port->mad_agent);
4097 error2:
4098         cm_remove_port_fs(port);
4099 error1:
4100         port_modify.set_port_cap_mask = 0;
4101         port_modify.clr_port_cap_mask = IB_PORT_CM_SUP;
4102         while (--i) {
4103                 if (!rdma_cap_ib_cm(ib_device, i))
4104                         continue;
4105
4106                 port = cm_dev->port[i-1];
4107                 ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4108                 ib_unregister_mad_agent(port->mad_agent);
4109                 cm_remove_port_fs(port);
4110         }
4111 free:
4112         device_unregister(cm_dev->device);
4113         kfree(cm_dev);
4114 }
4115
4116 static void cm_remove_one(struct ib_device *ib_device, void *client_data)
4117 {
4118         struct cm_device *cm_dev = client_data;
4119         struct cm_port *port;
4120         struct cm_id_private *cm_id_priv;
4121         struct ib_mad_agent *cur_mad_agent;
4122         struct ib_port_modify port_modify = {
4123                 .clr_port_cap_mask = IB_PORT_CM_SUP
4124         };
4125         unsigned long flags;
4126         int i;
4127
4128         if (!cm_dev)
4129                 return;
4130
4131         write_lock_irqsave(&cm.device_lock, flags);
4132         list_del(&cm_dev->list);
4133         write_unlock_irqrestore(&cm.device_lock, flags);
4134
4135         spin_lock_irq(&cm.lock);
4136         cm_dev->going_down = 1;
4137         spin_unlock_irq(&cm.lock);
4138
4139         for (i = 1; i <= ib_device->phys_port_cnt; i++) {
4140                 if (!rdma_cap_ib_cm(ib_device, i))
4141                         continue;
4142
4143                 port = cm_dev->port[i-1];
4144                 ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4145                 /* Mark all the cm_id's as not valid */
4146                 spin_lock_irq(&cm.lock);
4147                 list_for_each_entry(cm_id_priv, &port->cm_priv_altr_list, altr_list)
4148                         cm_id_priv->altr_send_port_not_ready = 1;
4149                 list_for_each_entry(cm_id_priv, &port->cm_priv_prim_list, prim_list)
4150                         cm_id_priv->prim_send_port_not_ready = 1;
4151                 spin_unlock_irq(&cm.lock);
4152                 /*
4153                  * We flush the queue here after the going_down set, this
4154                  * verify that no new works will be queued in the recv handler,
4155                  * after that we can call the unregister_mad_agent
4156                  */
4157                 flush_workqueue(cm.wq);
4158                 spin_lock_irq(&cm.state_lock);
4159                 cur_mad_agent = port->mad_agent;
4160                 port->mad_agent = NULL;
4161                 spin_unlock_irq(&cm.state_lock);
4162                 ib_unregister_mad_agent(cur_mad_agent);
4163                 cm_remove_port_fs(port);
4164         }
4165
4166         device_unregister(cm_dev->device);
4167         kfree(cm_dev);
4168 }
4169
4170 static int __init ib_cm_init(void)
4171 {
4172         int ret;
4173
4174         memset(&cm, 0, sizeof cm);
4175         INIT_LIST_HEAD(&cm.device_list);
4176         rwlock_init(&cm.device_lock);
4177         spin_lock_init(&cm.lock);
4178         spin_lock_init(&cm.state_lock);
4179         cm.listen_service_table = RB_ROOT;
4180         cm.listen_service_id = be64_to_cpu(IB_CM_ASSIGN_SERVICE_ID);
4181         cm.remote_id_table = RB_ROOT;
4182         cm.remote_qp_table = RB_ROOT;
4183         cm.remote_sidr_table = RB_ROOT;
4184         idr_init(&cm.local_id_table);
4185         get_random_bytes(&cm.random_id_operand, sizeof cm.random_id_operand);
4186         INIT_LIST_HEAD(&cm.timewait_list);
4187
4188         ret = class_register(&cm_class);
4189         if (ret) {
4190                 ret = -ENOMEM;
4191                 goto error1;
4192         }
4193
4194         cm.wq = create_workqueue("ib_cm");
4195         if (!cm.wq) {
4196                 ret = -ENOMEM;
4197                 goto error2;
4198         }
4199
4200         ret = ib_register_client(&cm_client);
4201         if (ret)
4202                 goto error3;
4203
4204         return 0;
4205 error3:
4206         destroy_workqueue(cm.wq);
4207 error2:
4208         class_unregister(&cm_class);
4209 error1:
4210         idr_destroy(&cm.local_id_table);
4211         return ret;
4212 }
4213
4214 static void __exit ib_cm_cleanup(void)
4215 {
4216         struct cm_timewait_info *timewait_info, *tmp;
4217
4218         spin_lock_irq(&cm.lock);
4219         list_for_each_entry(timewait_info, &cm.timewait_list, list)
4220                 cancel_delayed_work(&timewait_info->work.work);
4221         spin_unlock_irq(&cm.lock);
4222
4223         ib_unregister_client(&cm_client);
4224         destroy_workqueue(cm.wq);
4225
4226         list_for_each_entry_safe(timewait_info, tmp, &cm.timewait_list, list) {
4227                 list_del(&timewait_info->list);
4228                 kfree(timewait_info);
4229         }
4230
4231         class_unregister(&cm_class);
4232         idr_destroy(&cm.local_id_table);
4233 }
4234
4235 module_init(ib_cm_init);
4236 module_exit(ib_cm_cleanup);
4237