drivers/staging/lustre/lnet/lnet/api-ni.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  *
  30  * Copyright (c) 2011, 2015, Intel Corporation.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  */
  36
  37 #define DEBUG_SUBSYSTEM S_LNET
  38 #include <linux/log2.h>
  39 #include <linux/ktime.h>
  40
  41 #include "../../include/linux/lnet/lib-lnet.h"
  42 #include "../../include/linux/lnet/lib-dlc.h"
  43
  44 #define D_LNI D_CONSOLE
  45
  46 lnet_t the_lnet;                           /* THE state of the network */
  47 EXPORT_SYMBOL(the_lnet);
  48
  49 static char *ip2nets = "";
  50 module_param(ip2nets, charp, 0444);
  51 MODULE_PARM_DESC(ip2nets, "LNET network <- IP table");
  52
  53 static char *networks = "";
  54 module_param(networks, charp, 0444);
  55 MODULE_PARM_DESC(networks, "local networks");
  56
  57 static char *routes = "";
  58 module_param(routes, charp, 0444);
  59 MODULE_PARM_DESC(routes, "routes to non-local networks");
  60
  61 static int rnet_htable_size = LNET_REMOTE_NETS_HASH_DEFAULT;
  62 module_param(rnet_htable_size, int, 0444);
  63 MODULE_PARM_DESC(rnet_htable_size, "size of remote network hash table");
  64
  65 static int lnet_ping(lnet_process_id_t id, int timeout_ms,
  66                      lnet_process_id_t __user *ids, int n_ids);
  67
  68 static char *
  69 lnet_get_routes(void)
  70 {
  71         return routes;
  72 }
  73
  74 static char *
  75 lnet_get_networks(void)
  76 {
  77         char *nets;
  78         int rc;
  79
  80         if (*networks && *ip2nets) {
  81                 LCONSOLE_ERROR_MSG(0x101, "Please specify EITHER 'networks' or 'ip2nets' but not both at once\n");
  82                 return NULL;
  83         }
  84
  85         if (*ip2nets) {
  86                 rc = lnet_parse_ip2nets(&nets, ip2nets);
  87                 return !rc ? nets : NULL;
  88         }
  89
  90         if (*networks)
  91                 return networks;
  92
  93         return "tcp";
  94 }
  95
  96 static void
  97 lnet_init_locks(void)
  98 {
  99         spin_lock_init(&the_lnet.ln_eq_wait_lock);
 100         init_waitqueue_head(&the_lnet.ln_eq_waitq);
 101         mutex_init(&the_lnet.ln_lnd_mutex);
 102         mutex_init(&the_lnet.ln_api_mutex);
 103 }
 104
 105 static int
 106 lnet_create_remote_nets_table(void)
 107 {
 108         int i;
 109         struct list_head *hash;
 110
 111         LASSERT(!the_lnet.ln_remote_nets_hash);
 112         LASSERT(the_lnet.ln_remote_nets_hbits > 0);
 113         LIBCFS_ALLOC(hash, LNET_REMOTE_NETS_HASH_SIZE * sizeof(*hash));
 114         if (!hash) {
 115                 CERROR("Failed to create remote nets hash table\n");
 116                 return -ENOMEM;
 117         }
 118
 119         for (i = 0; i < LNET_REMOTE_NETS_HASH_SIZE; i++)
 120                 INIT_LIST_HEAD(&hash[i]);
 121         the_lnet.ln_remote_nets_hash = hash;
 122         return 0;
 123 }
 124
 125 static void
 126 lnet_destroy_remote_nets_table(void)
 127 {
 128         int i;
 129
 130         if (!the_lnet.ln_remote_nets_hash)
 131                 return;
 132
 133         for (i = 0; i < LNET_REMOTE_NETS_HASH_SIZE; i++)
 134                 LASSERT(list_empty(&the_lnet.ln_remote_nets_hash[i]));
 135
 136         LIBCFS_FREE(the_lnet.ln_remote_nets_hash,
 137                     LNET_REMOTE_NETS_HASH_SIZE *
 138                     sizeof(the_lnet.ln_remote_nets_hash[0]));
 139         the_lnet.ln_remote_nets_hash = NULL;
 140 }
 141
 142 static void
 143 lnet_destroy_locks(void)
 144 {
 145         if (the_lnet.ln_res_lock) {
 146                 cfs_percpt_lock_free(the_lnet.ln_res_lock);
 147                 the_lnet.ln_res_lock = NULL;
 148         }
 149
 150         if (the_lnet.ln_net_lock) {
 151                 cfs_percpt_lock_free(the_lnet.ln_net_lock);
 152                 the_lnet.ln_net_lock = NULL;
 153         }
 154 }
 155
 156 static int
 157 lnet_create_locks(void)
 158 {
 159         lnet_init_locks();
 160
 161         the_lnet.ln_res_lock = cfs_percpt_lock_alloc(lnet_cpt_table());
 162         if (!the_lnet.ln_res_lock)
 163                 goto failed;
 164
 165         the_lnet.ln_net_lock = cfs_percpt_lock_alloc(lnet_cpt_table());
 166         if (!the_lnet.ln_net_lock)
 167                 goto failed;
 168
 169         return 0;
 170
 171  failed:
 172         lnet_destroy_locks();
 173         return -ENOMEM;
 174 }
 175
 176 static void lnet_assert_wire_constants(void)
 177 {
 178         /*
 179          * Wire protocol assertions generated by 'wirecheck'
 180          * running on Linux robert.bartonsoftware.com 2.6.8-1.521
 181          * #1 Mon Aug 16 09:01:18 EDT 2004 i686 athlon i386 GNU/Linux
 182          * with gcc version 3.3.3 20040412 (Red Hat Linux 3.3.3-7)
 183          */
 184
 185         /* Constants... */
 186         CLASSERT(LNET_PROTO_TCP_MAGIC == 0xeebc0ded);
 187         CLASSERT(LNET_PROTO_TCP_VERSION_MAJOR == 1);
 188         CLASSERT(LNET_PROTO_TCP_VERSION_MINOR == 0);
 189         CLASSERT(LNET_MSG_ACK == 0);
 190         CLASSERT(LNET_MSG_PUT == 1);
 191         CLASSERT(LNET_MSG_GET == 2);
 192         CLASSERT(LNET_MSG_REPLY == 3);
 193         CLASSERT(LNET_MSG_HELLO == 4);
 194
 195         /* Checks for struct ptl_handle_wire_t */
 196         CLASSERT((int)sizeof(lnet_handle_wire_t) == 16);
 197         CLASSERT((int)offsetof(lnet_handle_wire_t, wh_interface_cookie) == 0);
 198         CLASSERT((int)sizeof(((lnet_handle_wire_t *)0)->wh_interface_cookie) == 8);
 199         CLASSERT((int)offsetof(lnet_handle_wire_t, wh_object_cookie) == 8);
 200         CLASSERT((int)sizeof(((lnet_handle_wire_t *)0)->wh_object_cookie) == 8);
 201
 202         /* Checks for struct lnet_magicversion_t */
 203         CLASSERT((int)sizeof(lnet_magicversion_t) == 8);
 204         CLASSERT((int)offsetof(lnet_magicversion_t, magic) == 0);
 205         CLASSERT((int)sizeof(((lnet_magicversion_t *)0)->magic) == 4);
 206         CLASSERT((int)offsetof(lnet_magicversion_t, version_major) == 4);
 207         CLASSERT((int)sizeof(((lnet_magicversion_t *)0)->version_major) == 2);
 208         CLASSERT((int)offsetof(lnet_magicversion_t, version_minor) == 6);
 209         CLASSERT((int)sizeof(((lnet_magicversion_t *)0)->version_minor) == 2);
 210
 211         /* Checks for struct lnet_hdr_t */
 212         CLASSERT((int)sizeof(lnet_hdr_t) == 72);
 213         CLASSERT((int)offsetof(lnet_hdr_t, dest_nid) == 0);
 214         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->dest_nid) == 8);
 215         CLASSERT((int)offsetof(lnet_hdr_t, src_nid) == 8);
 216         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->src_nid) == 8);
 217         CLASSERT((int)offsetof(lnet_hdr_t, dest_pid) == 16);
 218         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->dest_pid) == 4);
 219         CLASSERT((int)offsetof(lnet_hdr_t, src_pid) == 20);
 220         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->src_pid) == 4);
 221         CLASSERT((int)offsetof(lnet_hdr_t, type) == 24);
 222         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->type) == 4);
 223         CLASSERT((int)offsetof(lnet_hdr_t, payload_length) == 28);
 224         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->payload_length) == 4);
 225         CLASSERT((int)offsetof(lnet_hdr_t, msg) == 32);
 226         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->msg) == 40);
 227
 228         /* Ack */
 229         CLASSERT((int)offsetof(lnet_hdr_t, msg.ack.dst_wmd) == 32);
 230         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->msg.ack.dst_wmd) == 16);
 231         CLASSERT((int)offsetof(lnet_hdr_t, msg.ack.match_bits) == 48);
 232         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->msg.ack.match_bits) == 8);
 233         CLASSERT((int)offsetof(lnet_hdr_t, msg.ack.mlength) == 56);
 234         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->msg.ack.mlength) == 4);
 235
 236         /* Put */
 237         CLASSERT((int)offsetof(lnet_hdr_t, msg.put.ack_wmd) == 32);
 238         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->msg.put.ack_wmd) == 16);
 239         CLASSERT((int)offsetof(lnet_hdr_t, msg.put.match_bits) == 48);
 240         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->msg.put.match_bits) == 8);
 241         CLASSERT((int)offsetof(lnet_hdr_t, msg.put.hdr_data) == 56);
 242         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->msg.put.hdr_data) == 8);
 243         CLASSERT((int)offsetof(lnet_hdr_t, msg.put.ptl_index) == 64);
 244         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->msg.put.ptl_index) == 4);
 245         CLASSERT((int)offsetof(lnet_hdr_t, msg.put.offset) == 68);
 246         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->msg.put.offset) == 4);
 247
 248         /* Get */
 249         CLASSERT((int)offsetof(lnet_hdr_t, msg.get.return_wmd) == 32);
 250         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->msg.get.return_wmd) == 16);
 251         CLASSERT((int)offsetof(lnet_hdr_t, msg.get.match_bits) == 48);
 252         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->msg.get.match_bits) == 8);
 253         CLASSERT((int)offsetof(lnet_hdr_t, msg.get.ptl_index) == 56);
 254         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->msg.get.ptl_index) == 4);
 255         CLASSERT((int)offsetof(lnet_hdr_t, msg.get.src_offset) == 60);
 256         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->msg.get.src_offset) == 4);
 257         CLASSERT((int)offsetof(lnet_hdr_t, msg.get.sink_length) == 64);
 258         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->msg.get.sink_length) == 4);
 259
 260         /* Reply */
 261         CLASSERT((int)offsetof(lnet_hdr_t, msg.reply.dst_wmd) == 32);
 262         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->msg.reply.dst_wmd) == 16);
 263
 264         /* Hello */
 265         CLASSERT((int)offsetof(lnet_hdr_t, msg.hello.incarnation) == 32);
 266         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->msg.hello.incarnation) == 8);
 267         CLASSERT((int)offsetof(lnet_hdr_t, msg.hello.type) == 40);
 268         CLASSERT((int)sizeof(((lnet_hdr_t *)0)->msg.hello.type) == 4);
 269 }
 270
 271 static lnd_t *
 272 lnet_find_lnd_by_type(__u32 type)
 273 {
 274         lnd_t *lnd;
 275         struct list_head *tmp;
 276
 277         /* holding lnd mutex */
 278         list_for_each(tmp, &the_lnet.ln_lnds) {
 279                 lnd = list_entry(tmp, lnd_t, lnd_list);
 280
 281                 if (lnd->lnd_type == type)
 282                         return lnd;
 283         }
 284
 285         return NULL;
 286 }
 287
 288 void
 289 lnet_register_lnd(lnd_t *lnd)
 290 {
 291         mutex_lock(&the_lnet.ln_lnd_mutex);
 292
 293         LASSERT(the_lnet.ln_init);
 294         LASSERT(libcfs_isknown_lnd(lnd->lnd_type));
 295         LASSERT(!lnet_find_lnd_by_type(lnd->lnd_type));
 296
 297         list_add_tail(&lnd->lnd_list, &the_lnet.ln_lnds);
 298         lnd->lnd_refcount = 0;
 299
 300         CDEBUG(D_NET, "%s LND registered\n", libcfs_lnd2str(lnd->lnd_type));
 301
 302         mutex_unlock(&the_lnet.ln_lnd_mutex);
 303 }
 304 EXPORT_SYMBOL(lnet_register_lnd);
 305
 306 void
 307 lnet_unregister_lnd(lnd_t *lnd)
 308 {
 309         mutex_lock(&the_lnet.ln_lnd_mutex);
 310
 311         LASSERT(the_lnet.ln_init);
 312         LASSERT(lnet_find_lnd_by_type(lnd->lnd_type) == lnd);
 313         LASSERT(!lnd->lnd_refcount);
 314
 315         list_del(&lnd->lnd_list);
 316         CDEBUG(D_NET, "%s LND unregistered\n", libcfs_lnd2str(lnd->lnd_type));
 317
 318         mutex_unlock(&the_lnet.ln_lnd_mutex);
 319 }
 320 EXPORT_SYMBOL(lnet_unregister_lnd);
 321
 322 void
 323 lnet_counters_get(lnet_counters_t *counters)
 324 {
 325         lnet_counters_t *ctr;
 326         int i;
 327
 328         memset(counters, 0, sizeof(*counters));
 329
 330         lnet_net_lock(LNET_LOCK_EX);
 331
 332         cfs_percpt_for_each(ctr, i, the_lnet.ln_counters) {
 333                 counters->msgs_max     += ctr->msgs_max;
 334                 counters->msgs_alloc   += ctr->msgs_alloc;
 335                 counters->errors       += ctr->errors;
 336                 counters->send_count   += ctr->send_count;
 337                 counters->recv_count   += ctr->recv_count;
 338                 counters->route_count  += ctr->route_count;
 339                 counters->drop_count   += ctr->drop_count;
 340                 counters->send_length  += ctr->send_length;
 341                 counters->recv_length  += ctr->recv_length;
 342                 counters->route_length += ctr->route_length;
 343                 counters->drop_length  += ctr->drop_length;
 344         }
 345         lnet_net_unlock(LNET_LOCK_EX);
 346 }
 347 EXPORT_SYMBOL(lnet_counters_get);
 348
 349 void
 350 lnet_counters_reset(void)
 351 {
 352         lnet_counters_t *counters;
 353         int i;
 354
 355         lnet_net_lock(LNET_LOCK_EX);
 356
 357         cfs_percpt_for_each(counters, i, the_lnet.ln_counters)
 358                 memset(counters, 0, sizeof(lnet_counters_t));
 359
 360         lnet_net_unlock(LNET_LOCK_EX);
 361 }
 362
 363 static char *
 364 lnet_res_type2str(int type)
 365 {
 366         switch (type) {
 367         default:
 368                 LBUG();
 369         case LNET_COOKIE_TYPE_MD:
 370                 return "MD";
 371         case LNET_COOKIE_TYPE_ME:
 372                 return "ME";
 373         case LNET_COOKIE_TYPE_EQ:
 374                 return "EQ";
 375         }
 376 }
 377
 378 static void
 379 lnet_res_container_cleanup(struct lnet_res_container *rec)
 380 {
 381         int count = 0;
 382
 383         if (!rec->rec_type) /* not set yet, it's uninitialized */
 384                 return;
 385
 386         while (!list_empty(&rec->rec_active)) {
 387                 struct list_head *e = rec->rec_active.next;
 388
 389                 list_del_init(e);
 390                 if (rec->rec_type == LNET_COOKIE_TYPE_EQ) {
 391                         lnet_eq_free(list_entry(e, lnet_eq_t, eq_list));
 392
 393                 } else if (rec->rec_type == LNET_COOKIE_TYPE_MD) {
 394                         lnet_md_free(list_entry(e, lnet_libmd_t, md_list));
 395
 396                 } else { /* NB: Active MEs should be attached on portals */
 397                         LBUG();
 398                 }
 399                 count++;
 400         }
 401
 402         if (count > 0) {
 403                 /*
 404                  * Found alive MD/ME/EQ, user really should unlink/free
 405                  * all of them before finalize LNet, but if someone didn't,
 406                  * we have to recycle garbage for him
 407                  */
 408                 CERROR("%d active elements on exit of %s container\n",
 409                        count, lnet_res_type2str(rec->rec_type));
 410         }
 411
 412         if (rec->rec_lh_hash) {
 413                 LIBCFS_FREE(rec->rec_lh_hash,
 414                             LNET_LH_HASH_SIZE * sizeof(rec->rec_lh_hash[0]));
 415                 rec->rec_lh_hash = NULL;
 416         }
 417
 418         rec->rec_type = 0; /* mark it as finalized */
 419 }
 420
 421 static int
 422 lnet_res_container_setup(struct lnet_res_container *rec, int cpt, int type)
 423 {
 424         int rc = 0;
 425         int i;
 426
 427         LASSERT(!rec->rec_type);
 428
 429         rec->rec_type = type;
 430         INIT_LIST_HEAD(&rec->rec_active);
 431         rec->rec_lh_cookie = (cpt << LNET_COOKIE_TYPE_BITS) | type;
 432
 433         /* Arbitrary choice of hash table size */
 434         LIBCFS_CPT_ALLOC(rec->rec_lh_hash, lnet_cpt_table(), cpt,
 435                          LNET_LH_HASH_SIZE * sizeof(rec->rec_lh_hash[0]));
 436         if (!rec->rec_lh_hash) {
 437                 rc = -ENOMEM;
 438                 goto out;
 439         }
 440
 441         for (i = 0; i < LNET_LH_HASH_SIZE; i++)
 442                 INIT_LIST_HEAD(&rec->rec_lh_hash[i]);
 443
 444         return 0;
 445
 446 out:
 447         CERROR("Failed to setup %s resource container\n",
 448                lnet_res_type2str(type));
 449         lnet_res_container_cleanup(rec);
 450         return rc;
 451 }
 452
 453 static void
 454 lnet_res_containers_destroy(struct lnet_res_container **recs)
 455 {
 456         struct lnet_res_container *rec;
 457         int i;
 458
 459         cfs_percpt_for_each(rec, i, recs)
 460                 lnet_res_container_cleanup(rec);
 461
 462         cfs_percpt_free(recs);
 463 }
 464
 465 static struct lnet_res_container **
 466 lnet_res_containers_create(int type)
 467 {
 468         struct lnet_res_container **recs;
 469         struct lnet_res_container *rec;
 470         int rc;
 471         int i;
 472
 473         recs = cfs_percpt_alloc(lnet_cpt_table(), sizeof(*rec));
 474         if (!recs) {
 475                 CERROR("Failed to allocate %s resource containers\n",
 476                        lnet_res_type2str(type));
 477                 return NULL;
 478         }
 479
 480         cfs_percpt_for_each(rec, i, recs) {
 481                 rc = lnet_res_container_setup(rec, i, type);
 482                 if (rc) {
 483                         lnet_res_containers_destroy(recs);
 484                         return NULL;
 485                 }
 486         }
 487
 488         return recs;
 489 }
 490
 491 lnet_libhandle_t *
 492 lnet_res_lh_lookup(struct lnet_res_container *rec, __u64 cookie)
 493 {
 494         /* ALWAYS called with lnet_res_lock held */
 495         struct list_head *head;
 496         lnet_libhandle_t *lh;
 497         unsigned int hash;
 498
 499         if ((cookie & LNET_COOKIE_MASK) != rec->rec_type)
 500                 return NULL;
 501
 502         hash = cookie >> (LNET_COOKIE_TYPE_BITS + LNET_CPT_BITS);
 503         head = &rec->rec_lh_hash[hash & LNET_LH_HASH_MASK];
 504
 505         list_for_each_entry(lh, head, lh_hash_chain) {
 506                 if (lh->lh_cookie == cookie)
 507                         return lh;
 508         }
 509
 510         return NULL;
 511 }
 512
 513 void
 514 lnet_res_lh_initialize(struct lnet_res_container *rec, lnet_libhandle_t *lh)
 515 {
 516         /* ALWAYS called with lnet_res_lock held */
 517         unsigned int ibits = LNET_COOKIE_TYPE_BITS + LNET_CPT_BITS;
 518         unsigned int hash;
 519
 520         lh->lh_cookie = rec->rec_lh_cookie;
 521         rec->rec_lh_cookie += 1 << ibits;
 522
 523         hash = (lh->lh_cookie >> ibits) & LNET_LH_HASH_MASK;
 524
 525         list_add(&lh->lh_hash_chain, &rec->rec_lh_hash[hash]);
 526 }
 527
 528 static int lnet_unprepare(void);
 529
 530 static int
 531 lnet_prepare(lnet_pid_t requested_pid)
 532 {
 533         /* Prepare to bring up the network */
 534         struct lnet_res_container **recs;
 535         int rc = 0;
 536
 537         if (requested_pid == LNET_PID_ANY) {
 538                 /* Don't instantiate LNET just for me */
 539                 return -ENETDOWN;
 540         }
 541
 542         LASSERT(!the_lnet.ln_refcount);
 543
 544         the_lnet.ln_routing = 0;
 545
 546         LASSERT(!(requested_pid & LNET_PID_USERFLAG));
 547         the_lnet.ln_pid = requested_pid;
 548
 549         INIT_LIST_HEAD(&the_lnet.ln_test_peers);
 550         INIT_LIST_HEAD(&the_lnet.ln_nis);
 551         INIT_LIST_HEAD(&the_lnet.ln_nis_cpt);
 552         INIT_LIST_HEAD(&the_lnet.ln_nis_zombie);
 553         INIT_LIST_HEAD(&the_lnet.ln_routers);
 554
 555         rc = lnet_create_remote_nets_table();
 556         if (rc)
 557                 goto failed;
 558         /*
 559          * NB the interface cookie in wire handles guards against delayed
 560          * replies and ACKs appearing valid after reboot.
 561          */
 562         the_lnet.ln_interface_cookie = ktime_get_ns();
 563
 564         the_lnet.ln_counters = cfs_percpt_alloc(lnet_cpt_table(),
 565                                                 sizeof(lnet_counters_t));
 566         if (!the_lnet.ln_counters) {
 567                 CERROR("Failed to allocate counters for LNet\n");
 568                 rc = -ENOMEM;
 569                 goto failed;
 570         }
 571
 572         rc = lnet_peer_tables_create();
 573         if (rc)
 574                 goto failed;
 575
 576         rc = lnet_msg_containers_create();
 577         if (rc)
 578                 goto failed;
 579
 580         rc = lnet_res_container_setup(&the_lnet.ln_eq_container, 0,
 581                                       LNET_COOKIE_TYPE_EQ);
 582         if (rc)
 583                 goto failed;
 584
 585         recs = lnet_res_containers_create(LNET_COOKIE_TYPE_ME);
 586         if (!recs) {
 587                 rc = -ENOMEM;
 588                 goto failed;
 589         }
 590
 591         the_lnet.ln_me_containers = recs;
 592
 593         recs = lnet_res_containers_create(LNET_COOKIE_TYPE_MD);
 594         if (!recs) {
 595                 rc = -ENOMEM;
 596                 goto failed;
 597         }
 598
 599         the_lnet.ln_md_containers = recs;
 600
 601         rc = lnet_portals_create();
 602         if (rc) {
 603                 CERROR("Failed to create portals for LNet: %d\n", rc);
 604                 goto failed;
 605         }
 606
 607         return 0;
 608
 609  failed:
 610         lnet_unprepare();
 611         return rc;
 612 }
 613
 614 static int
 615 lnet_unprepare(void)
 616 {
 617         /*
 618          * NB no LNET_LOCK since this is the last reference.  All LND instances
 619          * have shut down already, so it is safe to unlink and free all
 620          * descriptors, even those that appear committed to a network op (eg MD
 621          * with non-zero pending count)
 622          */
 623         lnet_fail_nid(LNET_NID_ANY, 0);
 624
 625         LASSERT(!the_lnet.ln_refcount);
 626         LASSERT(list_empty(&the_lnet.ln_test_peers));
 627         LASSERT(list_empty(&the_lnet.ln_nis));
 628         LASSERT(list_empty(&the_lnet.ln_nis_cpt));
 629         LASSERT(list_empty(&the_lnet.ln_nis_zombie));
 630
 631         lnet_portals_destroy();
 632
 633         if (the_lnet.ln_md_containers) {
 634                 lnet_res_containers_destroy(the_lnet.ln_md_containers);
 635                 the_lnet.ln_md_containers = NULL;
 636         }
 637
 638         if (the_lnet.ln_me_containers) {
 639                 lnet_res_containers_destroy(the_lnet.ln_me_containers);
 640                 the_lnet.ln_me_containers = NULL;
 641         }
 642
 643         lnet_res_container_cleanup(&the_lnet.ln_eq_container);
 644
 645         lnet_msg_containers_destroy();
 646         lnet_peer_tables_destroy();
 647         lnet_rtrpools_free(0);
 648
 649         if (the_lnet.ln_counters) {
 650                 cfs_percpt_free(the_lnet.ln_counters);
 651                 the_lnet.ln_counters = NULL;
 652         }
 653         lnet_destroy_remote_nets_table();
 654
 655         return 0;
 656 }
 657
 658 lnet_ni_t  *
 659 lnet_net2ni_locked(__u32 net, int cpt)
 660 {
 661         struct list_head *tmp;
 662         lnet_ni_t *ni;
 663
 664         LASSERT(cpt != LNET_LOCK_EX);
 665
 666         list_for_each(tmp, &the_lnet.ln_nis) {
 667                 ni = list_entry(tmp, lnet_ni_t, ni_list);
 668
 669                 if (LNET_NIDNET(ni->ni_nid) == net) {
 670                         lnet_ni_addref_locked(ni, cpt);
 671                         return ni;
 672                 }
 673         }
 674
 675         return NULL;
 676 }
 677
 678 lnet_ni_t *
 679 lnet_net2ni(__u32 net)
 680 {
 681         lnet_ni_t *ni;
 682
 683         lnet_net_lock(0);
 684         ni = lnet_net2ni_locked(net, 0);
 685         lnet_net_unlock(0);
 686
 687         return ni;
 688 }
 689 EXPORT_SYMBOL(lnet_net2ni);
 690
 691 static unsigned int
 692 lnet_nid_cpt_hash(lnet_nid_t nid, unsigned int number)
 693 {
 694         __u64 key = nid;
 695         unsigned int val;
 696
 697         LASSERT(number >= 1 && number <= LNET_CPT_NUMBER);
 698
 699         if (number == 1)
 700                 return 0;
 701
 702         val = hash_long(key, LNET_CPT_BITS);
 703         /* NB: LNET_CP_NUMBER doesn't have to be PO2 */
 704         if (val < number)
 705                 return val;
 706
 707         return (unsigned int)(key + val + (val >> 1)) % number;
 708 }
 709
 710 int
 711 lnet_cpt_of_nid_locked(lnet_nid_t nid)
 712 {
 713         struct lnet_ni *ni;
 714
 715         /* must called with hold of lnet_net_lock */
 716         if (LNET_CPT_NUMBER == 1)
 717                 return 0; /* the only one */
 718
 719         /* take lnet_net_lock(any) would be OK */
 720         if (!list_empty(&the_lnet.ln_nis_cpt)) {
 721                 list_for_each_entry(ni, &the_lnet.ln_nis_cpt, ni_cptlist) {
 722                         if (LNET_NIDNET(ni->ni_nid) != LNET_NIDNET(nid))
 723                                 continue;
 724
 725                         LASSERT(ni->ni_cpts);
 726                         return ni->ni_cpts[lnet_nid_cpt_hash
 727                                            (nid, ni->ni_ncpts)];
 728                 }
 729         }
 730
 731         return lnet_nid_cpt_hash(nid, LNET_CPT_NUMBER);
 732 }
 733
 734 int
 735 lnet_cpt_of_nid(lnet_nid_t nid)
 736 {
 737         int cpt;
 738         int cpt2;
 739
 740         if (LNET_CPT_NUMBER == 1)
 741                 return 0; /* the only one */
 742
 743         if (list_empty(&the_lnet.ln_nis_cpt))
 744                 return lnet_nid_cpt_hash(nid, LNET_CPT_NUMBER);
 745
 746         cpt = lnet_net_lock_current();
 747         cpt2 = lnet_cpt_of_nid_locked(nid);
 748         lnet_net_unlock(cpt);
 749
 750         return cpt2;
 751 }
 752 EXPORT_SYMBOL(lnet_cpt_of_nid);
 753
 754 int
 755 lnet_islocalnet(__u32 net)
 756 {
 757         struct lnet_ni *ni;
 758         int cpt;
 759
 760         cpt = lnet_net_lock_current();
 761
 762         ni = lnet_net2ni_locked(net, cpt);
 763         if (ni)
 764                 lnet_ni_decref_locked(ni, cpt);
 765
 766         lnet_net_unlock(cpt);
 767
 768         return !!ni;
 769 }
 770
 771 lnet_ni_t  *
 772 lnet_nid2ni_locked(lnet_nid_t nid, int cpt)
 773 {
 774         struct lnet_ni *ni;
 775         struct list_head *tmp;
 776
 777         LASSERT(cpt != LNET_LOCK_EX);
 778
 779         list_for_each(tmp, &the_lnet.ln_nis) {
 780                 ni = list_entry(tmp, lnet_ni_t, ni_list);
 781
 782                 if (ni->ni_nid == nid) {
 783                         lnet_ni_addref_locked(ni, cpt);
 784                         return ni;
 785                 }
 786         }
 787
 788         return NULL;
 789 }
 790
 791 int
 792 lnet_islocalnid(lnet_nid_t nid)
 793 {
 794         struct lnet_ni *ni;
 795         int cpt;
 796
 797         cpt = lnet_net_lock_current();
 798         ni = lnet_nid2ni_locked(nid, cpt);
 799         if (ni)
 800                 lnet_ni_decref_locked(ni, cpt);
 801         lnet_net_unlock(cpt);
 802
 803         return !!ni;
 804 }
 805
 806 int
 807 lnet_count_acceptor_nis(void)
 808 {
 809         /* Return the # of NIs that need the acceptor. */
 810         int count = 0;
 811         struct list_head *tmp;
 812         struct lnet_ni *ni;
 813         int cpt;
 814
 815         cpt = lnet_net_lock_current();
 816         list_for_each(tmp, &the_lnet.ln_nis) {
 817                 ni = list_entry(tmp, lnet_ni_t, ni_list);
 818
 819                 if (ni->ni_lnd->lnd_accept)
 820                         count++;
 821         }
 822
 823         lnet_net_unlock(cpt);
 824
 825         return count;
 826 }
 827
 828 static lnet_ping_info_t *
 829 lnet_ping_info_create(int num_ni)
 830 {
 831         lnet_ping_info_t *ping_info;
 832         unsigned int infosz;
 833
 834         infosz = offsetof(lnet_ping_info_t, pi_ni[num_ni]);
 835         LIBCFS_ALLOC(ping_info, infosz);
 836         if (!ping_info) {
 837                 CERROR("Can't allocate ping info[%d]\n", num_ni);
 838                 return NULL;
 839         }
 840
 841         ping_info->pi_nnis = num_ni;
 842         ping_info->pi_pid = the_lnet.ln_pid;
 843         ping_info->pi_magic = LNET_PROTO_PING_MAGIC;
 844         ping_info->pi_features = LNET_PING_FEAT_NI_STATUS;
 845
 846         return ping_info;
 847 }
 848
 849 static inline int
 850 lnet_get_ni_count(void)
 851 {
 852         struct lnet_ni *ni;
 853         int count = 0;
 854
 855         lnet_net_lock(0);
 856
 857         list_for_each_entry(ni, &the_lnet.ln_nis, ni_list)
 858                 count++;
 859
 860         lnet_net_unlock(0);
 861
 862         return count;
 863 }
 864
 865 static inline void
 866 lnet_ping_info_free(lnet_ping_info_t *pinfo)
 867 {
 868         LIBCFS_FREE(pinfo,
 869                     offsetof(lnet_ping_info_t,
 870                              pi_ni[pinfo->pi_nnis]));
 871 }
 872
 873 static void
 874 lnet_ping_info_destroy(void)
 875 {
 876         struct lnet_ni *ni;
 877
 878         lnet_net_lock(LNET_LOCK_EX);
 879
 880         list_for_each_entry(ni, &the_lnet.ln_nis, ni_list) {
 881                 lnet_ni_lock(ni);
 882                 ni->ni_status = NULL;
 883                 lnet_ni_unlock(ni);
 884         }
 885
 886         lnet_ping_info_free(the_lnet.ln_ping_info);
 887         the_lnet.ln_ping_info = NULL;
 888
 889         lnet_net_unlock(LNET_LOCK_EX);
 890 }
 891
 892 static void
 893 lnet_ping_event_handler(lnet_event_t *event)
 894 {
 895         lnet_ping_info_t *pinfo = event->md.user_ptr;
 896
 897         if (event->unlinked)
 898                 pinfo->pi_features = LNET_PING_FEAT_INVAL;
 899 }
 900
 901 static int
 902 lnet_ping_info_setup(lnet_ping_info_t **ppinfo, lnet_handle_md_t *md_handle,
 903                      int ni_count, bool set_eq)
 904 {
 905         lnet_process_id_t id = {LNET_NID_ANY, LNET_PID_ANY};
 906         lnet_handle_me_t me_handle;
 907         lnet_md_t md = { NULL };
 908         int rc, rc2;
 909
 910         if (set_eq) {
 911                 rc = LNetEQAlloc(0, lnet_ping_event_handler,
 912                                  &the_lnet.ln_ping_target_eq);
 913                 if (rc) {
 914                         CERROR("Can't allocate ping EQ: %d\n", rc);
 915                         return rc;
 916                 }
 917         }
 918
 919         *ppinfo = lnet_ping_info_create(ni_count);
 920         if (!*ppinfo) {
 921                 rc = -ENOMEM;
 922                 goto failed_0;
 923         }
 924
 925         rc = LNetMEAttach(LNET_RESERVED_PORTAL, id,
 926                           LNET_PROTO_PING_MATCHBITS, 0,
 927                           LNET_UNLINK, LNET_INS_AFTER,
 928                           &me_handle);
 929         if (rc) {
 930                 CERROR("Can't create ping ME: %d\n", rc);
 931                 goto failed_1;
 932         }
 933
 934         /* initialize md content */
 935         md.start = *ppinfo;
 936         md.length = offsetof(lnet_ping_info_t,
 937                              pi_ni[(*ppinfo)->pi_nnis]);
 938         md.threshold = LNET_MD_THRESH_INF;
 939         md.max_size = 0;
 940         md.options = LNET_MD_OP_GET | LNET_MD_TRUNCATE |
 941                      LNET_MD_MANAGE_REMOTE;
 942         md.user_ptr  = NULL;
 943         md.eq_handle = the_lnet.ln_ping_target_eq;
 944         md.user_ptr = *ppinfo;
 945
 946         rc = LNetMDAttach(me_handle, md, LNET_RETAIN, md_handle);
 947         if (rc) {
 948                 CERROR("Can't attach ping MD: %d\n", rc);
 949                 goto failed_2;
 950         }
 951
 952         return 0;
 953
 954 failed_2:
 955         rc2 = LNetMEUnlink(me_handle);
 956         LASSERT(!rc2);
 957 failed_1:
 958         lnet_ping_info_free(*ppinfo);
 959         *ppinfo = NULL;
 960 failed_0:
 961         if (set_eq)
 962                 LNetEQFree(the_lnet.ln_ping_target_eq);
 963         return rc;
 964 }
 965
 966 static void
 967 lnet_ping_md_unlink(lnet_ping_info_t *pinfo, lnet_handle_md_t *md_handle)
 968 {
 969         sigset_t blocked = cfs_block_allsigs();
 970
 971         LNetMDUnlink(*md_handle);
 972         LNetInvalidateHandle(md_handle);
 973
 974         /* NB md could be busy; this just starts the unlink */
 975         while (pinfo->pi_features != LNET_PING_FEAT_INVAL) {
 976                 CDEBUG(D_NET, "Still waiting for ping MD to unlink\n");
 977                 set_current_state(TASK_UNINTERRUPTIBLE);
 978                 schedule_timeout(cfs_time_seconds(1));
 979         }
 980
 981         cfs_restore_sigs(blocked);
 982 }
 983
 984 static void
 985 lnet_ping_info_install_locked(lnet_ping_info_t *ping_info)
 986 {
 987         lnet_ni_status_t *ns;
 988         lnet_ni_t *ni;
 989         int i = 0;
 990
 991         list_for_each_entry(ni, &the_lnet.ln_nis, ni_list) {
 992                 LASSERT(i < ping_info->pi_nnis);
 993
 994                 ns = &ping_info->pi_ni[i];
 995
 996                 ns->ns_nid = ni->ni_nid;
 997
 998                 lnet_ni_lock(ni);
 999                 ns->ns_status = (ni->ni_status) ?
1000                                  ni->ni_status->ns_status : LNET_NI_STATUS_UP;
1001                 ni->ni_status = ns;
1002                 lnet_ni_unlock(ni);
1003
1004                 i++;
1005         }
1006 }
1007
1008 static void
1009 lnet_ping_target_update(lnet_ping_info_t *pinfo, lnet_handle_md_t md_handle)
1010 {
1011         lnet_ping_info_t *old_pinfo = NULL;
1012         lnet_handle_md_t old_md;
1013
1014         /* switch the NIs to point to the new ping info created */
1015         lnet_net_lock(LNET_LOCK_EX);
1016
1017         if (!the_lnet.ln_routing)
1018                 pinfo->pi_features |= LNET_PING_FEAT_RTE_DISABLED;
1019         lnet_ping_info_install_locked(pinfo);
1020
1021         if (the_lnet.ln_ping_info) {
1022                 old_pinfo = the_lnet.ln_ping_info;
1023                 old_md = the_lnet.ln_ping_target_md;
1024         }
1025         the_lnet.ln_ping_target_md = md_handle;
1026         the_lnet.ln_ping_info = pinfo;
1027
1028         lnet_net_unlock(LNET_LOCK_EX);
1029
1030         if (old_pinfo) {
1031                 /* unlink the old ping info */
1032                 lnet_ping_md_unlink(old_pinfo, &old_md);
1033                 lnet_ping_info_free(old_pinfo);
1034         }
1035 }
1036
1037 static void
1038 lnet_ping_target_fini(void)
1039 {
1040         int rc;
1041
1042         lnet_ping_md_unlink(the_lnet.ln_ping_info,
1043                             &the_lnet.ln_ping_target_md);
1044
1045         rc = LNetEQFree(the_lnet.ln_ping_target_eq);
1046         LASSERT(!rc);
1047
1048         lnet_ping_info_destroy();
1049 }
1050
1051 static int
1052 lnet_ni_tq_credits(lnet_ni_t *ni)
1053 {
1054         int credits;
1055
1056         LASSERT(ni->ni_ncpts >= 1);
1057
1058         if (ni->ni_ncpts == 1)
1059                 return ni->ni_maxtxcredits;
1060
1061         credits = ni->ni_maxtxcredits / ni->ni_ncpts;
1062         credits = max(credits, 8 * ni->ni_peertxcredits);
1063         credits = min(credits, ni->ni_maxtxcredits);
1064
1065         return credits;
1066 }
1067
1068 static void
1069 lnet_ni_unlink_locked(lnet_ni_t *ni)
1070 {
1071         if (!list_empty(&ni->ni_cptlist)) {
1072                 list_del_init(&ni->ni_cptlist);
1073                 lnet_ni_decref_locked(ni, 0);
1074         }
1075
1076         /* move it to zombie list and nobody can find it anymore */
1077         LASSERT(!list_empty(&ni->ni_list));
1078         list_move(&ni->ni_list, &the_lnet.ln_nis_zombie);
1079         lnet_ni_decref_locked(ni, 0);   /* drop ln_nis' ref */
1080 }
1081
1082 static void
1083 lnet_clear_zombies_nis_locked(void)
1084 {
1085         int i;
1086         int islo;
1087         lnet_ni_t *ni;
1088
1089         /*
1090          * Now wait for the NI's I just nuked to show up on ln_zombie_nis
1091          * and shut them down in guaranteed thread context
1092          */
1093         i = 2;
1094         while (!list_empty(&the_lnet.ln_nis_zombie)) {
1095                 int *ref;
1096                 int j;
1097
1098                 ni = list_entry(the_lnet.ln_nis_zombie.next,
1099                                 lnet_ni_t, ni_list);
1100                 list_del_init(&ni->ni_list);
1101                 cfs_percpt_for_each(ref, j, ni->ni_refs) {
1102                         if (!*ref)
1103                                 continue;
1104                         /* still busy, add it back to zombie list */
1105                         list_add(&ni->ni_list, &the_lnet.ln_nis_zombie);
1106                         break;
1107                 }
1108
1109                 if (!list_empty(&ni->ni_list)) {
1110                         lnet_net_unlock(LNET_LOCK_EX);
1111                         ++i;
1112                         if ((i & (-i)) == i) {
1113                                 CDEBUG(D_WARNING, "Waiting for zombie LNI %s\n",
1114                                        libcfs_nid2str(ni->ni_nid));
1115                         }
1116                         set_current_state(TASK_UNINTERRUPTIBLE);
1117                         schedule_timeout(cfs_time_seconds(1));
1118                         lnet_net_lock(LNET_LOCK_EX);
1119                         continue;
1120                 }
1121
1122                 ni->ni_lnd->lnd_refcount--;
1123                 lnet_net_unlock(LNET_LOCK_EX);
1124
1125                 islo = ni->ni_lnd->lnd_type == LOLND;
1126
1127                 LASSERT(!in_interrupt());
1128                 ni->ni_lnd->lnd_shutdown(ni);
1129
1130                 /*
1131                  * can't deref lnd anymore now; it might have unregistered
1132                  * itself...
1133                  */
1134                 if (!islo)
1135                         CDEBUG(D_LNI, "Removed LNI %s\n",
1136                                libcfs_nid2str(ni->ni_nid));
1137
1138                 lnet_ni_free(ni);
1139                 i = 2;
1140
1141                 lnet_net_lock(LNET_LOCK_EX);
1142         }
1143 }
1144
1145 static void
1146 lnet_shutdown_lndnis(void)
1147 {
1148         lnet_ni_t *ni;
1149         int i;
1150
1151         /* NB called holding the global mutex */
1152
1153         /* All quiet on the API front */
1154         LASSERT(!the_lnet.ln_shutdown);
1155         LASSERT(!the_lnet.ln_refcount);
1156         LASSERT(list_empty(&the_lnet.ln_nis_zombie));
1157
1158         lnet_net_lock(LNET_LOCK_EX);
1159         the_lnet.ln_shutdown = 1;       /* flag shutdown */
1160
1161         /* Unlink NIs from the global table */
1162         while (!list_empty(&the_lnet.ln_nis)) {
1163                 ni = list_entry(the_lnet.ln_nis.next,
1164                                 lnet_ni_t, ni_list);
1165                 lnet_ni_unlink_locked(ni);
1166         }
1167
1168         /* Drop the cached eqwait NI. */
1169         if (the_lnet.ln_eq_waitni) {
1170                 lnet_ni_decref_locked(the_lnet.ln_eq_waitni, 0);
1171                 the_lnet.ln_eq_waitni = NULL;
1172         }
1173
1174         /* Drop the cached loopback NI. */
1175         if (the_lnet.ln_loni) {
1176                 lnet_ni_decref_locked(the_lnet.ln_loni, 0);
1177                 the_lnet.ln_loni = NULL;
1178         }
1179
1180         lnet_net_unlock(LNET_LOCK_EX);
1181
1182         /*
1183          * Clear lazy portals and drop delayed messages which hold refs
1184          * on their lnet_msg_t::msg_rxpeer
1185          */
1186         for (i = 0; i < the_lnet.ln_nportals; i++)
1187                 LNetClearLazyPortal(i);
1188
1189         /*
1190          * Clear the peer table and wait for all peers to go (they hold refs on
1191          * their NIs)
1192          */
1193         lnet_peer_tables_cleanup(NULL);
1194
1195         lnet_net_lock(LNET_LOCK_EX);
1196
1197         lnet_clear_zombies_nis_locked();
1198         the_lnet.ln_shutdown = 0;
1199         lnet_net_unlock(LNET_LOCK_EX);
1200 }
1201
1202 /* shutdown down the NI and release refcount */
1203 static void
1204 lnet_shutdown_lndni(struct lnet_ni *ni)
1205 {
1206         lnet_net_lock(LNET_LOCK_EX);
1207         lnet_ni_unlink_locked(ni);
1208         lnet_net_unlock(LNET_LOCK_EX);
1209
1210         /* Do peer table cleanup for this ni */
1211         lnet_peer_tables_cleanup(ni);
1212
1213         lnet_net_lock(LNET_LOCK_EX);
1214         lnet_clear_zombies_nis_locked();
1215         lnet_net_unlock(LNET_LOCK_EX);
1216 }
1217
1218 static int
1219 lnet_startup_lndni(struct lnet_ni *ni, __s32 peer_timeout,
1220                    __s32 peer_cr, __s32 peer_buf_cr, __s32 credits)
1221 {
1222         int rc = 0;
1223         int lnd_type;
1224         lnd_t *lnd;
1225         struct lnet_tx_queue *tq;
1226         int i;
1227
1228         lnd_type = LNET_NETTYP(LNET_NIDNET(ni->ni_nid));
1229
1230         LASSERT(libcfs_isknown_lnd(lnd_type));
1231
1232         if (lnd_type == CIBLND || lnd_type == OPENIBLND ||
1233             lnd_type == IIBLND || lnd_type == VIBLND) {
1234                 CERROR("LND %s obsoleted\n", libcfs_lnd2str(lnd_type));
1235                 goto failed0;
1236         }
1237
1238         /* Make sure this new NI is unique. */
1239         lnet_net_lock(LNET_LOCK_EX);
1240         if (!lnet_net_unique(LNET_NIDNET(ni->ni_nid), &the_lnet.ln_nis)) {
1241                 if (lnd_type == LOLND) {
1242                         lnet_net_unlock(LNET_LOCK_EX);
1243                         lnet_ni_free(ni);
1244                         return 0;
1245                 }
1246                 lnet_net_unlock(LNET_LOCK_EX);
1247
1248                 CERROR("Net %s is not unique\n",
1249                        libcfs_net2str(LNET_NIDNET(ni->ni_nid)));
1250                 goto failed0;
1251         }
1252         lnet_net_unlock(LNET_LOCK_EX);
1253
1254         mutex_lock(&the_lnet.ln_lnd_mutex);
1255         lnd = lnet_find_lnd_by_type(lnd_type);
1256
1257         if (!lnd) {
1258                 mutex_unlock(&the_lnet.ln_lnd_mutex);
1259                 rc = request_module("%s", libcfs_lnd2modname(lnd_type));
1260                 mutex_lock(&the_lnet.ln_lnd_mutex);
1261
1262                 lnd = lnet_find_lnd_by_type(lnd_type);
1263                 if (!lnd) {
1264                         mutex_unlock(&the_lnet.ln_lnd_mutex);
1265                         CERROR("Can't load LND %s, module %s, rc=%d\n",
1266                                libcfs_lnd2str(lnd_type),
1267                                libcfs_lnd2modname(lnd_type), rc);
1268                         goto failed0;
1269                 }
1270         }
1271
1272         lnet_net_lock(LNET_LOCK_EX);
1273         lnd->lnd_refcount++;
1274         lnet_net_unlock(LNET_LOCK_EX);
1275
1276         ni->ni_lnd = lnd;
1277
1278         rc = lnd->lnd_startup(ni);
1279
1280         mutex_unlock(&the_lnet.ln_lnd_mutex);
1281
1282         if (rc) {
1283                 LCONSOLE_ERROR_MSG(0x105, "Error %d starting up LNI %s\n",
1284                                    rc, libcfs_lnd2str(lnd->lnd_type));
1285                 lnet_net_lock(LNET_LOCK_EX);
1286                 lnd->lnd_refcount--;
1287                 lnet_net_unlock(LNET_LOCK_EX);
1288                 goto failed0;
1289         }
1290
1291         /*
1292          * If given some LND tunable parameters, parse those now to
1293          * override the values in the NI structure.
1294          */
1295         if (peer_buf_cr >= 0)
1296                 ni->ni_peerrtrcredits = peer_buf_cr;
1297         if (peer_timeout >= 0)
1298                 ni->ni_peertimeout = peer_timeout;
1299         /*
1300          * TODO
1301          * Note: For now, don't allow the user to change
1302          * peertxcredits as this number is used in the
1303          * IB LND to control queue depth.
1304          * if (peer_cr != -1)
1305          *      ni->ni_peertxcredits = peer_cr;
1306          */
1307         if (credits >= 0)
1308                 ni->ni_maxtxcredits = credits;
1309
1310         LASSERT(ni->ni_peertimeout <= 0 || lnd->lnd_query);
1311
1312         lnet_net_lock(LNET_LOCK_EX);
1313         /* refcount for ln_nis */
1314         lnet_ni_addref_locked(ni, 0);
1315         list_add_tail(&ni->ni_list, &the_lnet.ln_nis);
1316         if (ni->ni_cpts) {
1317                 lnet_ni_addref_locked(ni, 0);
1318                 list_add_tail(&ni->ni_cptlist, &the_lnet.ln_nis_cpt);
1319         }
1320
1321         lnet_net_unlock(LNET_LOCK_EX);
1322
1323         if (lnd->lnd_type == LOLND) {
1324                 lnet_ni_addref(ni);
1325                 LASSERT(!the_lnet.ln_loni);
1326                 the_lnet.ln_loni = ni;
1327                 return 0;
1328         }
1329
1330         if (!ni->ni_peertxcredits || !ni->ni_maxtxcredits) {
1331                 LCONSOLE_ERROR_MSG(0x107, "LNI %s has no %scredits\n",
1332                                    libcfs_lnd2str(lnd->lnd_type),
1333                                    !ni->ni_peertxcredits ?
1334                                    "" : "per-peer ");
1335                 /*
1336                  * shutdown the NI since if we get here then it must've already
1337                  * been started
1338                  */
1339                 lnet_shutdown_lndni(ni);
1340                 return -EINVAL;
1341         }
1342
1343         cfs_percpt_for_each(tq, i, ni->ni_tx_queues) {
1344                 tq->tq_credits_min =
1345                 tq->tq_credits_max =
1346                 tq->tq_credits = lnet_ni_tq_credits(ni);
1347         }
1348
1349         CDEBUG(D_LNI, "Added LNI %s [%d/%d/%d/%d]\n",
1350                libcfs_nid2str(ni->ni_nid), ni->ni_peertxcredits,
1351                lnet_ni_tq_credits(ni) * LNET_CPT_NUMBER,
1352                ni->ni_peerrtrcredits, ni->ni_peertimeout);
1353
1354         return 0;
1355 failed0:
1356         lnet_ni_free(ni);
1357         return -EINVAL;
1358 }
1359
1360 static int
1361 lnet_startup_lndnis(struct list_head *nilist)
1362 {
1363         struct lnet_ni *ni;
1364         int rc;
1365         int lnd_type;
1366         int ni_count = 0;
1367
1368         while (!list_empty(nilist)) {
1369                 ni = list_entry(nilist->next, lnet_ni_t, ni_list);
1370                 list_del(&ni->ni_list);
1371                 rc = lnet_startup_lndni(ni, -1, -1, -1, -1);
1372
1373                 if (rc < 0)
1374                         goto failed;
1375
1376                 ni_count++;
1377         }
1378
1379         if (the_lnet.ln_eq_waitni && ni_count > 1) {
1380                 lnd_type = the_lnet.ln_eq_waitni->ni_lnd->lnd_type;
1381                 LCONSOLE_ERROR_MSG(0x109, "LND %s can only run single-network\n",
1382                                    libcfs_lnd2str(lnd_type));
1383                 rc = -EINVAL;
1384                 goto failed;
1385         }
1386
1387         return ni_count;
1388 failed:
1389         lnet_shutdown_lndnis();
1390
1391         return rc;
1392 }
1393
1394 /**
1395  * Initialize LNet library.
1396  *
1397  * Only userspace program needs to call this function - it's automatically
1398  * called in the kernel at module loading time. Caller has to call lnet_fini()
1399  * after a call to lnet_init(), if and only if the latter returned 0. It must
1400  * be called exactly once.
1401  *
1402  * \return 0 on success, and -ve on failures.
1403  */
1404 int
1405 lnet_init(void)
1406 {
1407         int rc;
1408
1409         lnet_assert_wire_constants();
1410         LASSERT(!the_lnet.ln_init);
1411
1412         memset(&the_lnet, 0, sizeof(the_lnet));
1413
1414         /* refer to global cfs_cpt_table for now */
1415         the_lnet.ln_cpt_table   = cfs_cpt_table;
1416         the_lnet.ln_cpt_number  = cfs_cpt_number(cfs_cpt_table);
1417
1418         LASSERT(the_lnet.ln_cpt_number > 0);
1419         if (the_lnet.ln_cpt_number > LNET_CPT_MAX) {
1420                 /* we are under risk of consuming all lh_cookie */
1421                 CERROR("Can't have %d CPTs for LNet (max allowed is %d), please change setting of CPT-table and retry\n",
1422                        the_lnet.ln_cpt_number, LNET_CPT_MAX);
1423                 return -1;
1424         }
1425
1426         while ((1 << the_lnet.ln_cpt_bits) < the_lnet.ln_cpt_number)
1427                 the_lnet.ln_cpt_bits++;
1428
1429         rc = lnet_create_locks();
1430         if (rc) {
1431                 CERROR("Can't create LNet global locks: %d\n", rc);
1432                 return -1;
1433         }
1434
1435         the_lnet.ln_refcount = 0;
1436         the_lnet.ln_init = 1;
1437         LNetInvalidateHandle(&the_lnet.ln_rc_eqh);
1438         INIT_LIST_HEAD(&the_lnet.ln_lnds);
1439         INIT_LIST_HEAD(&the_lnet.ln_rcd_zombie);
1440         INIT_LIST_HEAD(&the_lnet.ln_rcd_deathrow);
1441
1442         /*
1443          * The hash table size is the number of bits it takes to express the set
1444          * ln_num_routes, minus 1 (better to under estimate than over so we
1445          * don't waste memory).
1446          */
1447         if (rnet_htable_size <= 0)
1448                 rnet_htable_size = LNET_REMOTE_NETS_HASH_DEFAULT;
1449         else if (rnet_htable_size > LNET_REMOTE_NETS_HASH_MAX)
1450                 rnet_htable_size = LNET_REMOTE_NETS_HASH_MAX;
1451         the_lnet.ln_remote_nets_hbits = max_t(int, 1,
1452                                            order_base_2(rnet_htable_size) - 1);
1453
1454         /*
1455          * All LNDs apart from the LOLND are in separate modules.  They
1456          * register themselves when their module loads, and unregister
1457          * themselves when their module is unloaded.
1458          */
1459         lnet_register_lnd(&the_lolnd);
1460         return 0;
1461 }
1462
1463 /**
1464  * Finalize LNet library.
1465  *
1466  * Only userspace program needs to call this function. It can be called
1467  * at most once.
1468  *
1469  * \pre lnet_init() called with success.
1470  * \pre All LNet users called LNetNIFini() for matching LNetNIInit() calls.
1471  */
1472 void
1473 lnet_fini(void)
1474 {
1475         LASSERT(the_lnet.ln_init);
1476         LASSERT(!the_lnet.ln_refcount);
1477
1478         while (!list_empty(&the_lnet.ln_lnds))
1479                 lnet_unregister_lnd(list_entry(the_lnet.ln_lnds.next,
1480                                                lnd_t, lnd_list));
1481         lnet_destroy_locks();
1482
1483         the_lnet.ln_init = 0;
1484 }
1485
1486 /**
1487  * Set LNet PID and start LNet interfaces, routing, and forwarding.
1488  *
1489  * Userspace program should call this after a successful call to lnet_init().
1490  * Users must call this function at least once before any other functions.
1491  * For each successful call there must be a corresponding call to
1492  * LNetNIFini(). For subsequent calls to LNetNIInit(), \a requested_pid is
1493  * ignored.
1494  *
1495  * The PID used by LNet may be different from the one requested.
1496  * See LNetGetId().
1497  *
1498  * \param requested_pid PID requested by the caller.
1499  *
1500  * \return >= 0 on success, and < 0 error code on failures.
1501  */
1502 int
1503 LNetNIInit(lnet_pid_t requested_pid)
1504 {
1505         int im_a_router = 0;
1506         int rc, rc2;
1507         int ni_count;
1508         lnet_ping_info_t *pinfo;
1509         lnet_handle_md_t md_handle;
1510         struct list_head net_head;
1511
1512         INIT_LIST_HEAD(&net_head);
1513
1514         mutex_lock(&the_lnet.ln_api_mutex);
1515
1516         LASSERT(the_lnet.ln_init);
1517         CDEBUG(D_OTHER, "refs %d\n", the_lnet.ln_refcount);
1518
1519         if (the_lnet.ln_refcount > 0) {
1520                 rc = the_lnet.ln_refcount++;
1521                 mutex_unlock(&the_lnet.ln_api_mutex);
1522                 return rc;
1523         }
1524
1525         rc = lnet_prepare(requested_pid);
1526         if (rc) {
1527                 mutex_unlock(&the_lnet.ln_api_mutex);
1528                 return rc;
1529         }
1530
1531         /* Add in the loopback network */
1532         if (!lnet_ni_alloc(LNET_MKNET(LOLND, 0), NULL, &net_head)) {
1533                 rc = -ENOMEM;
1534                 goto err_empty_list;
1535         }
1536
1537         /*
1538          * If LNet is being initialized via DLC it is possible
1539          * that the user requests not to load module parameters (ones which
1540          * are supported by DLC) on initialization.  Therefore, make sure not
1541          * to load networks, routes and forwarding from module parameters
1542          * in this case. On cleanup in case of failure only clean up
1543          * routes if it has been loaded
1544          */
1545         if (!the_lnet.ln_nis_from_mod_params) {
1546                 rc = lnet_parse_networks(&net_head, lnet_get_networks());
1547                 if (rc < 0)
1548                         goto err_empty_list;
1549         }
1550
1551         ni_count = lnet_startup_lndnis(&net_head);
1552         if (ni_count < 0) {
1553                 rc = ni_count;
1554                 goto err_empty_list;
1555         }
1556
1557         if (!the_lnet.ln_nis_from_mod_params) {
1558                 rc = lnet_parse_routes(lnet_get_routes(), &im_a_router);
1559                 if (rc)
1560                         goto err_shutdown_lndnis;
1561
1562                 rc = lnet_check_routes();
1563                 if (rc)
1564                         goto err_destory_routes;
1565
1566                 rc = lnet_rtrpools_alloc(im_a_router);
1567                 if (rc)
1568                         goto err_destory_routes;
1569         }
1570
1571         rc = lnet_acceptor_start();
1572         if (rc)
1573                 goto err_destory_routes;
1574
1575         the_lnet.ln_refcount = 1;
1576         /* Now I may use my own API functions... */
1577
1578         rc = lnet_ping_info_setup(&pinfo, &md_handle, ni_count, true);
1579         if (rc)
1580                 goto err_acceptor_stop;
1581
1582         lnet_ping_target_update(pinfo, md_handle);
1583
1584         rc = lnet_router_checker_start();
1585         if (rc)
1586                 goto err_stop_ping;
1587
1588         lnet_router_debugfs_init();
1589
1590         mutex_unlock(&the_lnet.ln_api_mutex);
1591
1592         return 0;
1593
1594 err_stop_ping:
1595         lnet_ping_md_unlink(pinfo, &md_handle);
1596         lnet_ping_info_free(pinfo);
1597         rc2 = LNetEQFree(the_lnet.ln_ping_target_eq);
1598         LASSERT(!rc2);
1599 err_acceptor_stop:
1600         the_lnet.ln_refcount = 0;
1601         lnet_acceptor_stop();
1602 err_destory_routes:
1603         if (!the_lnet.ln_nis_from_mod_params)
1604                 lnet_destroy_routes();
1605 err_shutdown_lndnis:
1606         lnet_shutdown_lndnis();
1607 err_empty_list:
1608         lnet_unprepare();
1609         LASSERT(rc < 0);
1610         mutex_unlock(&the_lnet.ln_api_mutex);
1611         while (!list_empty(&net_head)) {
1612                 struct lnet_ni *ni;
1613
1614                 ni = list_entry(net_head.next, struct lnet_ni, ni_list);
1615                 list_del_init(&ni->ni_list);
1616                 lnet_ni_free(ni);
1617         }
1618         return rc;
1619 }
1620 EXPORT_SYMBOL(LNetNIInit);
1621
1622 /**
1623  * Stop LNet interfaces, routing, and forwarding.
1624  *
1625  * Users must call this function once for each successful call to LNetNIInit().
1626  * Once the LNetNIFini() operation has been started, the results of pending
1627  * API operations are undefined.
1628  *
1629  * \return always 0 for current implementation.
1630  */
1631 int
1632 LNetNIFini(void)
1633 {
1634         mutex_lock(&the_lnet.ln_api_mutex);
1635
1636         LASSERT(the_lnet.ln_init);
1637         LASSERT(the_lnet.ln_refcount > 0);
1638
1639         if (the_lnet.ln_refcount != 1) {
1640                 the_lnet.ln_refcount--;
1641         } else {
1642                 LASSERT(!the_lnet.ln_niinit_self);
1643
1644                 lnet_router_debugfs_fini();
1645                 lnet_router_checker_stop();
1646                 lnet_ping_target_fini();
1647
1648                 /* Teardown fns that use my own API functions BEFORE here */
1649                 the_lnet.ln_refcount = 0;
1650
1651                 lnet_acceptor_stop();
1652                 lnet_destroy_routes();
1653                 lnet_shutdown_lndnis();
1654                 lnet_unprepare();
1655         }
1656
1657         mutex_unlock(&the_lnet.ln_api_mutex);
1658         return 0;
1659 }
1660 EXPORT_SYMBOL(LNetNIFini);
1661
1662 /**
1663  * Grabs the ni data from the ni structure and fills the out
1664  * parameters
1665  *
1666  * \param[in] ni network       interface structure
1667  * \param[out] cpt_count       the number of cpts the ni is on
1668  * \param[out] nid             Network Interface ID
1669  * \param[out] peer_timeout    NI peer timeout
1670  * \param[out] peer_tx_crdits  NI peer transmit credits
1671  * \param[out] peer_rtr_credits NI peer router credits
1672  * \param[out] max_tx_credits  NI max transmit credit
1673  * \param[out] net_config      Network configuration
1674  */
1675 static void
1676 lnet_fill_ni_info(struct lnet_ni *ni, __u32 *cpt_count, __u64 *nid,
1677                   int *peer_timeout, int *peer_tx_credits,
1678                   int *peer_rtr_credits, int *max_tx_credits,
1679                   struct lnet_ioctl_net_config *net_config)
1680 {
1681         int i;
1682
1683         if (!ni)
1684                 return;
1685
1686         if (!net_config)
1687                 return;
1688
1689         BUILD_BUG_ON(ARRAY_SIZE(ni->ni_interfaces) !=
1690                      ARRAY_SIZE(net_config->ni_interfaces));
1691
1692         for (i = 0; i < ARRAY_SIZE(ni->ni_interfaces); i++) {
1693                 if (!ni->ni_interfaces[i])
1694                         break;
1695
1696                 strncpy(net_config->ni_interfaces[i],
1697                         ni->ni_interfaces[i],
1698                         sizeof(net_config->ni_interfaces[i]));
1699         }
1700
1701         *nid = ni->ni_nid;
1702         *peer_timeout = ni->ni_peertimeout;
1703         *peer_tx_credits = ni->ni_peertxcredits;
1704         *peer_rtr_credits = ni->ni_peerrtrcredits;
1705         *max_tx_credits = ni->ni_maxtxcredits;
1706
1707         net_config->ni_status = ni->ni_status->ns_status;
1708
1709         if (ni->ni_cpts) {
1710                 int num_cpts = min(ni->ni_ncpts, LNET_MAX_SHOW_NUM_CPT);
1711
1712                 for (i = 0; i < num_cpts; i++)
1713                         net_config->ni_cpts[i] = ni->ni_cpts[i];
1714
1715                 *cpt_count = num_cpts;
1716         }
1717 }
1718
1719 int
1720 lnet_get_net_config(int idx, __u32 *cpt_count, __u64 *nid, int *peer_timeout,
1721                     int *peer_tx_credits, int *peer_rtr_credits,
1722                     int *max_tx_credits,
1723                     struct lnet_ioctl_net_config *net_config)
1724 {
1725         struct lnet_ni *ni;
1726         struct list_head *tmp;
1727         int cpt, i = 0;
1728         int rc = -ENOENT;
1729
1730         cpt = lnet_net_lock_current();
1731
1732         list_for_each(tmp, &the_lnet.ln_nis) {
1733                 if (i++ != idx)
1734                         continue;
1735
1736                 ni = list_entry(tmp, lnet_ni_t, ni_list);
1737                 lnet_ni_lock(ni);
1738                 lnet_fill_ni_info(ni, cpt_count, nid, peer_timeout,
1739                                   peer_tx_credits, peer_rtr_credits,
1740                                   max_tx_credits, net_config);
1741                 lnet_ni_unlock(ni);
1742                 rc = 0;
1743                 break;
1744         }
1745
1746         lnet_net_unlock(cpt);
1747         return rc;
1748 }
1749
1750 int
1751 lnet_dyn_add_ni(lnet_pid_t requested_pid, char *nets,
1752                 __s32 peer_timeout, __s32 peer_cr, __s32 peer_buf_cr,
1753                 __s32 credits)
1754 {
1755         lnet_ping_info_t *pinfo;
1756         lnet_handle_md_t md_handle;
1757         struct lnet_ni *ni;
1758         struct list_head net_head;
1759         lnet_remotenet_t *rnet;
1760         int rc;
1761
1762         INIT_LIST_HEAD(&net_head);
1763
1764         /* Create a ni structure for the network string */
1765         rc = lnet_parse_networks(&net_head, nets);
1766         if (rc <= 0)
1767                 return !rc ? -EINVAL : rc;
1768
1769         mutex_lock(&the_lnet.ln_api_mutex);
1770
1771         if (rc > 1) {
1772                 rc = -EINVAL; /* only add one interface per call */
1773                 goto failed0;
1774         }
1775
1776         ni = list_entry(net_head.next, struct lnet_ni, ni_list);
1777
1778         lnet_net_lock(LNET_LOCK_EX);
1779         rnet = lnet_find_net_locked(LNET_NIDNET(ni->ni_nid));
1780         lnet_net_unlock(LNET_LOCK_EX);
1781         /*
1782          * make sure that the net added doesn't invalidate the current
1783          * configuration LNet is keeping
1784          */
1785         if (rnet) {
1786                 CERROR("Adding net %s will invalidate routing configuration\n",
1787                        nets);
1788                 rc = -EUSERS;
1789                 goto failed0;
1790         }
1791
1792         rc = lnet_ping_info_setup(&pinfo, &md_handle, 1 + lnet_get_ni_count(),
1793                                   false);
1794         if (rc)
1795                 goto failed0;
1796
1797         list_del_init(&ni->ni_list);
1798
1799         rc = lnet_startup_lndni(ni, peer_timeout, peer_cr,
1800                                 peer_buf_cr, credits);
1801         if (rc)
1802                 goto failed1;
1803
1804         if (ni->ni_lnd->lnd_accept) {
1805                 rc = lnet_acceptor_start();
1806                 if (rc < 0) {
1807                         /* shutdown the ni that we just started */
1808                         CERROR("Failed to start up acceptor thread\n");
1809                         lnet_shutdown_lndni(ni);
1810                         goto failed1;
1811                 }
1812         }
1813
1814         lnet_ping_target_update(pinfo, md_handle);
1815         mutex_unlock(&the_lnet.ln_api_mutex);
1816
1817         return 0;
1818
1819 failed1:
1820         lnet_ping_md_unlink(pinfo, &md_handle);
1821         lnet_ping_info_free(pinfo);
1822 failed0:
1823         mutex_unlock(&the_lnet.ln_api_mutex);
1824         while (!list_empty(&net_head)) {
1825                 ni = list_entry(net_head.next, struct lnet_ni, ni_list);
1826                 list_del_init(&ni->ni_list);
1827                 lnet_ni_free(ni);
1828         }
1829         return rc;
1830 }
1831
1832 int
1833 lnet_dyn_del_ni(__u32 net)
1834 {
1835         lnet_ni_t *ni;
1836         lnet_ping_info_t *pinfo;
1837         lnet_handle_md_t md_handle;
1838         int rc;
1839
1840         /* don't allow userspace to shutdown the LOLND */
1841         if (LNET_NETTYP(net) == LOLND)
1842                 return -EINVAL;
1843
1844         mutex_lock(&the_lnet.ln_api_mutex);
1845         /* create and link a new ping info, before removing the old one */
1846         rc = lnet_ping_info_setup(&pinfo, &md_handle,
1847                                   lnet_get_ni_count() - 1, false);
1848         if (rc)
1849                 goto out;
1850
1851         ni = lnet_net2ni(net);
1852         if (!ni) {
1853                 rc = -EINVAL;
1854                 goto failed;
1855         }
1856
1857         /* decrement the reference counter taken by lnet_net2ni() */
1858         lnet_ni_decref_locked(ni, 0);
1859
1860         lnet_shutdown_lndni(ni);
1861
1862         if (!lnet_count_acceptor_nis())
1863                 lnet_acceptor_stop();
1864
1865         lnet_ping_target_update(pinfo, md_handle);
1866         goto out;
1867 failed:
1868         lnet_ping_md_unlink(pinfo, &md_handle);
1869         lnet_ping_info_free(pinfo);
1870 out:
1871         mutex_unlock(&the_lnet.ln_api_mutex);
1872
1873         return rc;
1874 }
1875
1876 /**
1877  * LNet ioctl handler.
1878  *
1879  */
1880 int
1881 LNetCtl(unsigned int cmd, void *arg)
1882 {
1883         struct libcfs_ioctl_data *data = arg;
1884         struct lnet_ioctl_config_data *config;
1885         lnet_process_id_t id = {0};
1886         lnet_ni_t *ni;
1887         int rc;
1888         unsigned long secs_passed;
1889
1890         LASSERT(the_lnet.ln_init);
1891
1892         switch (cmd) {
1893         case IOC_LIBCFS_GET_NI:
1894                 rc = LNetGetId(data->ioc_count, &id);
1895                 data->ioc_nid = id.nid;
1896                 return rc;
1897
1898         case IOC_LIBCFS_FAIL_NID:
1899                 return lnet_fail_nid(data->ioc_nid, data->ioc_count);
1900
1901         case IOC_LIBCFS_ADD_ROUTE:
1902                 config = arg;
1903
1904                 if (config->cfg_hdr.ioc_len < sizeof(*config))
1905                         return -EINVAL;
1906
1907                 mutex_lock(&the_lnet.ln_api_mutex);
1908                 rc = lnet_add_route(config->cfg_net,
1909                                     config->cfg_config_u.cfg_route.rtr_hop,
1910                                     config->cfg_nid,
1911                                     config->cfg_config_u.cfg_route.rtr_priority);
1912                 mutex_unlock(&the_lnet.ln_api_mutex);
1913                 return rc ? rc : lnet_check_routes();
1914
1915         case IOC_LIBCFS_DEL_ROUTE:
1916                 config = arg;
1917
1918                 if (config->cfg_hdr.ioc_len < sizeof(*config))
1919                         return -EINVAL;
1920
1921                 mutex_lock(&the_lnet.ln_api_mutex);
1922                 rc = lnet_del_route(config->cfg_net, config->cfg_nid);
1923                 mutex_unlock(&the_lnet.ln_api_mutex);
1924                 return rc;
1925
1926         case IOC_LIBCFS_GET_ROUTE:
1927                 config = arg;
1928
1929                 if (config->cfg_hdr.ioc_len < sizeof(*config))
1930                         return -EINVAL;
1931
1932                 return lnet_get_route(config->cfg_count,
1933                                       &config->cfg_net,
1934                                       &config->cfg_config_u.cfg_route.rtr_hop,
1935                                       &config->cfg_nid,
1936                                       &config->cfg_config_u.cfg_route.rtr_flags,
1937                                       &config->cfg_config_u.cfg_route.rtr_priority);
1938
1939         case IOC_LIBCFS_GET_NET: {
1940                 struct lnet_ioctl_net_config *net_config;
1941                 size_t total = sizeof(*config) + sizeof(*net_config);
1942
1943                 config = arg;
1944
1945                 if (config->cfg_hdr.ioc_len < total)
1946                         return -EINVAL;
1947
1948                 net_config = (struct lnet_ioctl_net_config *)
1949                                 config->cfg_bulk;
1950                 if (!net_config)
1951                         return -EINVAL;
1952
1953                 return lnet_get_net_config(config->cfg_count,
1954                                            &config->cfg_ncpts,
1955                                            &config->cfg_nid,
1956                                            &config->cfg_config_u.cfg_net.net_peer_timeout,
1957                                            &config->cfg_config_u.cfg_net.net_peer_tx_credits,
1958                                            &config->cfg_config_u.cfg_net.net_peer_rtr_credits,
1959                                            &config->cfg_config_u.cfg_net.net_max_tx_credits,
1960                                            net_config);
1961         }
1962
1963         case IOC_LIBCFS_GET_LNET_STATS: {
1964                 struct lnet_ioctl_lnet_stats *lnet_stats = arg;
1965
1966                 if (lnet_stats->st_hdr.ioc_len < sizeof(*lnet_stats))
1967                         return -EINVAL;
1968
1969                 lnet_counters_get(&lnet_stats->st_cntrs);
1970                 return 0;
1971         }
1972
1973         case IOC_LIBCFS_CONFIG_RTR:
1974                 config = arg;
1975
1976                 if (config->cfg_hdr.ioc_len < sizeof(*config))
1977                         return -EINVAL;
1978
1979                 mutex_lock(&the_lnet.ln_api_mutex);
1980                 if (config->cfg_config_u.cfg_buffers.buf_enable) {
1981                         rc = lnet_rtrpools_enable();
1982                         mutex_unlock(&the_lnet.ln_api_mutex);
1983                         return rc;
1984                 }
1985                 lnet_rtrpools_disable();
1986                 mutex_unlock(&the_lnet.ln_api_mutex);
1987                 return 0;
1988
1989         case IOC_LIBCFS_ADD_BUF:
1990                 config = arg;
1991
1992                 if (config->cfg_hdr.ioc_len < sizeof(*config))
1993                         return -EINVAL;
1994
1995                 mutex_lock(&the_lnet.ln_api_mutex);
1996                 rc = lnet_rtrpools_adjust(config->cfg_config_u.cfg_buffers.buf_tiny,
1997                                           config->cfg_config_u.cfg_buffers.buf_small,
1998                                           config->cfg_config_u.cfg_buffers.buf_large);
1999                 mutex_unlock(&the_lnet.ln_api_mutex);
2000                 return rc;
2001
2002         case IOC_LIBCFS_GET_BUF: {
2003                 struct lnet_ioctl_pool_cfg *pool_cfg;
2004                 size_t total = sizeof(*config) + sizeof(*pool_cfg);
2005
2006                 config = arg;
2007
2008                 if (config->cfg_hdr.ioc_len < total)
2009                         return -EINVAL;
2010
2011                 pool_cfg = (struct lnet_ioctl_pool_cfg *)config->cfg_bulk;
2012                 return lnet_get_rtr_pool_cfg(config->cfg_count, pool_cfg);
2013         }
2014
2015         case IOC_LIBCFS_GET_PEER_INFO: {
2016                 struct lnet_ioctl_peer *peer_info = arg;
2017
2018                 if (peer_info->pr_hdr.ioc_len < sizeof(*peer_info))
2019                         return -EINVAL;
2020
2021                 return lnet_get_peer_info(peer_info->pr_count,
2022                         &peer_info->pr_nid,
2023                         peer_info->pr_lnd_u.pr_peer_credits.cr_aliveness,
2024                         &peer_info->pr_lnd_u.pr_peer_credits.cr_ncpt,
2025                         &peer_info->pr_lnd_u.pr_peer_credits.cr_refcount,
2026                         &peer_info->pr_lnd_u.pr_peer_credits.cr_ni_peer_tx_credits,
2027                         &peer_info->pr_lnd_u.pr_peer_credits.cr_peer_tx_credits,
2028                         &peer_info->pr_lnd_u.pr_peer_credits.cr_peer_rtr_credits,
2029                         &peer_info->pr_lnd_u.pr_peer_credits.cr_peer_min_rtr_credits,
2030                         &peer_info->pr_lnd_u.pr_peer_credits.cr_peer_tx_qnob);
2031         }
2032
2033         case IOC_LIBCFS_NOTIFY_ROUTER:
2034                 secs_passed = (ktime_get_real_seconds() - data->ioc_u64[0]);
2035                 return lnet_notify(NULL, data->ioc_nid, data->ioc_flags,
2036                                    jiffies - secs_passed * HZ);
2037
2038         case IOC_LIBCFS_LNET_DIST:
2039                 rc = LNetDist(data->ioc_nid, &data->ioc_nid, &data->ioc_u32[1]);
2040                 if (rc < 0 && rc != -EHOSTUNREACH)
2041                         return rc;
2042
2043                 data->ioc_u32[0] = rc;
2044                 return 0;
2045
2046         case IOC_LIBCFS_TESTPROTOCOMPAT:
2047                 lnet_net_lock(LNET_LOCK_EX);
2048                 the_lnet.ln_testprotocompat = data->ioc_flags;
2049                 lnet_net_unlock(LNET_LOCK_EX);
2050                 return 0;
2051
2052         case IOC_LIBCFS_PING:
2053                 id.nid = data->ioc_nid;
2054                 id.pid = data->ioc_u32[0];
2055                 rc = lnet_ping(id, data->ioc_u32[1], /* timeout */
2056                                data->ioc_pbuf1,
2057                                data->ioc_plen1 / sizeof(lnet_process_id_t));
2058                 if (rc < 0)
2059                         return rc;
2060                 data->ioc_count = rc;
2061                 return 0;
2062
2063         default:
2064                 ni = lnet_net2ni(data->ioc_net);
2065                 if (!ni)
2066                         return -EINVAL;
2067
2068                 if (!ni->ni_lnd->lnd_ctl)
2069                         rc = -EINVAL;
2070                 else
2071                         rc = ni->ni_lnd->lnd_ctl(ni, cmd, arg);
2072
2073                 lnet_ni_decref(ni);
2074                 return rc;
2075         }
2076         /* not reached */
2077 }
2078 EXPORT_SYMBOL(LNetCtl);
2079
2080 void LNetDebugPeer(lnet_process_id_t id)
2081 {
2082         lnet_debug_peer(id.nid);
2083 }
2084 EXPORT_SYMBOL(LNetDebugPeer);
2085
2086 /**
2087  * Retrieve the lnet_process_id_t ID of LNet interface at \a index. Note that
2088  * all interfaces share a same PID, as requested by LNetNIInit().
2089  *
2090  * \param index Index of the interface to look up.
2091  * \param id On successful return, this location will hold the
2092  * lnet_process_id_t ID of the interface.
2093  *
2094  * \retval 0 If an interface exists at \a index.
2095  * \retval -ENOENT If no interface has been found.
2096  */
2097 int
2098 LNetGetId(unsigned int index, lnet_process_id_t *id)
2099 {
2100         struct lnet_ni *ni;
2101         struct list_head *tmp;
2102         int cpt;
2103         int rc = -ENOENT;
2104
2105         LASSERT(the_lnet.ln_init);
2106
2107         /* LNetNI initilization failed? */
2108         if (!the_lnet.ln_refcount)
2109                 return rc;
2110
2111         cpt = lnet_net_lock_current();
2112
2113         list_for_each(tmp, &the_lnet.ln_nis) {
2114                 if (index--)
2115                         continue;
2116
2117                 ni = list_entry(tmp, lnet_ni_t, ni_list);
2118
2119                 id->nid = ni->ni_nid;
2120                 id->pid = the_lnet.ln_pid;
2121                 rc = 0;
2122                 break;
2123         }
2124
2125         lnet_net_unlock(cpt);
2126         return rc;
2127 }
2128 EXPORT_SYMBOL(LNetGetId);
2129
2130 /**
2131  * Print a string representation of handle \a h into buffer \a str of
2132  * \a len bytes.
2133  */
2134 void
2135 LNetSnprintHandle(char *str, int len, lnet_handle_any_t h)
2136 {
2137         snprintf(str, len, "%#llx", h.cookie);
2138 }
2139 EXPORT_SYMBOL(LNetSnprintHandle);
2140
2141 static int lnet_ping(lnet_process_id_t id, int timeout_ms,
2142                      lnet_process_id_t __user *ids, int n_ids)
2143 {
2144         lnet_handle_eq_t eqh;
2145         lnet_handle_md_t mdh;
2146         lnet_event_t event;
2147         lnet_md_t md = { NULL };
2148         int which;
2149         int unlinked = 0;
2150         int replied = 0;
2151         const int a_long_time = 60000; /* mS */
2152         int infosz;
2153         lnet_ping_info_t *info;
2154         lnet_process_id_t tmpid;
2155         int i;
2156         int nob;
2157         int rc;
2158         int rc2;
2159         sigset_t blocked;
2160
2161         infosz = offsetof(lnet_ping_info_t, pi_ni[n_ids]);
2162
2163         if (n_ids <= 0 ||
2164             id.nid == LNET_NID_ANY ||
2165             timeout_ms > 500000 ||            /* arbitrary limit! */
2166             n_ids > 20)                  /* arbitrary limit! */
2167                 return -EINVAL;
2168
2169         if (id.pid == LNET_PID_ANY)
2170                 id.pid = LNET_PID_LUSTRE;
2171
2172         LIBCFS_ALLOC(info, infosz);
2173         if (!info)
2174                 return -ENOMEM;
2175
2176         /* NB 2 events max (including any unlink event) */
2177         rc = LNetEQAlloc(2, LNET_EQ_HANDLER_NONE, &eqh);
2178         if (rc) {
2179                 CERROR("Can't allocate EQ: %d\n", rc);
2180                 goto out_0;
2181         }
2182
2183         /* initialize md content */
2184         md.start     = info;
2185         md.length    = infosz;
2186         md.threshold = 2; /*GET/REPLY*/
2187         md.max_size  = 0;
2188         md.options   = LNET_MD_TRUNCATE;
2189         md.user_ptr  = NULL;
2190         md.eq_handle = eqh;
2191
2192         rc = LNetMDBind(md, LNET_UNLINK, &mdh);
2193         if (rc) {
2194                 CERROR("Can't bind MD: %d\n", rc);
2195                 goto out_1;
2196         }
2197
2198         rc = LNetGet(LNET_NID_ANY, mdh, id,
2199                      LNET_RESERVED_PORTAL,
2200                      LNET_PROTO_PING_MATCHBITS, 0);
2201
2202         if (rc) {
2203                 /* Don't CERROR; this could be deliberate! */
2204
2205                 rc2 = LNetMDUnlink(mdh);
2206                 LASSERT(!rc2);
2207
2208                 /* NB must wait for the UNLINK event below... */
2209                 unlinked = 1;
2210                 timeout_ms = a_long_time;
2211         }
2212
2213         do {
2214                 /* MUST block for unlink to complete */
2215                 if (unlinked)
2216                         blocked = cfs_block_allsigs();
2217
2218                 rc2 = LNetEQPoll(&eqh, 1, timeout_ms, &event, &which);
2219
2220                 if (unlinked)
2221                         cfs_restore_sigs(blocked);
2222
2223                 CDEBUG(D_NET, "poll %d(%d %d)%s\n", rc2,
2224                        (rc2 <= 0) ? -1 : event.type,
2225                        (rc2 <= 0) ? -1 : event.status,
2226                        (rc2 > 0 && event.unlinked) ? " unlinked" : "");
2227
2228                 LASSERT(rc2 != -EOVERFLOW);     /* can't miss anything */
2229
2230                 if (rc2 <= 0 || event.status) {
2231                         /* timeout or error */
2232                         if (!replied && !rc)
2233                                 rc = (rc2 < 0) ? rc2 :
2234                                      !rc2 ? -ETIMEDOUT :
2235                                      event.status;
2236
2237                         if (!unlinked) {
2238                                 /* Ensure completion in finite time... */
2239                                 LNetMDUnlink(mdh);
2240                                 /* No assertion (racing with network) */
2241                                 unlinked = 1;
2242                                 timeout_ms = a_long_time;
2243                         } else if (!rc2) {
2244                                 /* timed out waiting for unlink */
2245                                 CWARN("ping %s: late network completion\n",
2246                                       libcfs_id2str(id));
2247                         }
2248                 } else if (event.type == LNET_EVENT_REPLY) {
2249                         replied = 1;
2250                         rc = event.mlength;
2251                 }
2252
2253         } while (rc2 <= 0 || !event.unlinked);
2254
2255         if (!replied) {
2256                 if (rc >= 0)
2257                         CWARN("%s: Unexpected rc >= 0 but no reply!\n",
2258                               libcfs_id2str(id));
2259                 rc = -EIO;
2260                 goto out_1;
2261         }
2262
2263         nob = rc;
2264         LASSERT(nob >= 0 && nob <= infosz);
2265
2266         rc = -EPROTO;                      /* if I can't parse... */
2267
2268         if (nob < 8) {
2269                 /* can't check magic/version */
2270                 CERROR("%s: ping info too short %d\n",
2271                        libcfs_id2str(id), nob);
2272                 goto out_1;
2273         }
2274
2275         if (info->pi_magic == __swab32(LNET_PROTO_PING_MAGIC)) {
2276                 lnet_swap_pinginfo(info);
2277         } else if (info->pi_magic != LNET_PROTO_PING_MAGIC) {
2278                 CERROR("%s: Unexpected magic %08x\n",
2279                        libcfs_id2str(id), info->pi_magic);
2280                 goto out_1;
2281         }
2282
2283         if (!(info->pi_features & LNET_PING_FEAT_NI_STATUS)) {
2284                 CERROR("%s: ping w/o NI status: 0x%x\n",
2285                        libcfs_id2str(id), info->pi_features);
2286                 goto out_1;
2287         }
2288
2289         if (nob < offsetof(lnet_ping_info_t, pi_ni[0])) {
2290                 CERROR("%s: Short reply %d(%d min)\n", libcfs_id2str(id),
2291                        nob, (int)offsetof(lnet_ping_info_t, pi_ni[0]));
2292                 goto out_1;
2293         }
2294
2295         if (info->pi_nnis < n_ids)
2296                 n_ids = info->pi_nnis;
2297
2298         if (nob < offsetof(lnet_ping_info_t, pi_ni[n_ids])) {
2299                 CERROR("%s: Short reply %d(%d expected)\n", libcfs_id2str(id),
2300                        nob, (int)offsetof(lnet_ping_info_t, pi_ni[n_ids]));
2301                 goto out_1;
2302         }
2303
2304         rc = -EFAULT;                      /* If I SEGV... */
2305
2306         memset(&tmpid, 0, sizeof(tmpid));
2307         for (i = 0; i < n_ids; i++) {
2308                 tmpid.pid = info->pi_pid;
2309                 tmpid.nid = info->pi_ni[i].ns_nid;
2310                 if (copy_to_user(&ids[i], &tmpid, sizeof(tmpid)))
2311                         goto out_1;
2312         }
2313         rc = info->pi_nnis;
2314
2315  out_1:
2316         rc2 = LNetEQFree(eqh);
2317         if (rc2)
2318                 CERROR("rc2 %d\n", rc2);
2319         LASSERT(!rc2);
2320
2321  out_0:
2322         LIBCFS_FREE(info, infosz);
2323         return rc;
2324 }