drivers/staging/lustre/lnet/selftest/console.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  *
  30  * Copyright (c) 2012, Intel Corporation.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * lnet/selftest/conctl.c
  37  *
  38  * Infrastructure of LST console
  39  *
  40  * Author: Liang Zhen <liangzhen@clusterfs.com>
  41  */
  42
  43 #include "../../include/linux/libcfs/libcfs.h"
  44 #include "../../include/linux/lnet/lib-lnet.h"
  45 #include "console.h"
  46 #include "conrpc.h"
  47
  48 #define LST_NODE_STATE_COUNTER(nd, p)                   \
  49 do {                                                    \
  50         if ((nd)->nd_state == LST_NODE_ACTIVE)          \
  51                 (p)->nle_nactive++;                     \
  52         else if ((nd)->nd_state == LST_NODE_BUSY)       \
  53                 (p)->nle_nbusy++;                       \
  54         else if ((nd)->nd_state == LST_NODE_DOWN)       \
  55                 (p)->nle_ndown++;                       \
  56         else                                            \
  57                 (p)->nle_nunknown++;                    \
  58         (p)->nle_nnode++;                               \
  59 } while (0)
  60
  61 lstcon_session_t console_session;
  62
  63 static void
  64 lstcon_node_get(lstcon_node_t *nd)
  65 {
  66         LASSERT(nd->nd_ref >= 1);
  67
  68         nd->nd_ref++;
  69 }
  70
  71 static int
  72 lstcon_node_find(lnet_process_id_t id, lstcon_node_t **ndpp, int create)
  73 {
  74         lstcon_ndlink_t *ndl;
  75         unsigned int idx = LNET_NIDADDR(id.nid) % LST_GLOBAL_HASHSIZE;
  76
  77         LASSERT(id.nid != LNET_NID_ANY);
  78
  79         list_for_each_entry(ndl, &console_session.ses_ndl_hash[idx], ndl_hlink) {
  80                 if (ndl->ndl_node->nd_id.nid != id.nid ||
  81                     ndl->ndl_node->nd_id.pid != id.pid)
  82                         continue;
  83
  84                 lstcon_node_get(ndl->ndl_node);
  85                 *ndpp = ndl->ndl_node;
  86                 return 0;
  87         }
  88
  89         if (!create)
  90                 return -ENOENT;
  91
  92         LIBCFS_ALLOC(*ndpp, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
  93         if (*ndpp == NULL)
  94                 return -ENOMEM;
  95
  96         ndl = (lstcon_ndlink_t *)(*ndpp + 1);
  97
  98         ndl->ndl_node = *ndpp;
  99
 100         ndl->ndl_node->nd_ref   = 1;
 101         ndl->ndl_node->nd_id    = id;
 102         ndl->ndl_node->nd_stamp = cfs_time_current();
 103         ndl->ndl_node->nd_state = LST_NODE_UNKNOWN;
 104         ndl->ndl_node->nd_timeout = 0;
 105         memset(&ndl->ndl_node->nd_ping, 0, sizeof(lstcon_rpc_t));
 106
 107         /* queued in global hash & list, no refcount is taken by
 108          * global hash & list, if caller release his refcount,
 109          * node will be released */
 110         list_add_tail(&ndl->ndl_hlink, &console_session.ses_ndl_hash[idx]);
 111         list_add_tail(&ndl->ndl_link, &console_session.ses_ndl_list);
 112
 113         return 0;
 114 }
 115
 116 static void
 117 lstcon_node_put(lstcon_node_t *nd)
 118 {
 119         lstcon_ndlink_t *ndl;
 120
 121         LASSERT(nd->nd_ref > 0);
 122
 123         if (--nd->nd_ref > 0)
 124                 return;
 125
 126         ndl = (lstcon_ndlink_t *)(nd + 1);
 127
 128         LASSERT(!list_empty(&ndl->ndl_link));
 129         LASSERT(!list_empty(&ndl->ndl_hlink));
 130
 131         /* remove from session */
 132         list_del(&ndl->ndl_link);
 133         list_del(&ndl->ndl_hlink);
 134
 135         LIBCFS_FREE(nd, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
 136 }
 137
 138 static int
 139 lstcon_ndlink_find(struct list_head *hash,
 140                    lnet_process_id_t id, lstcon_ndlink_t **ndlpp, int create)
 141 {
 142         unsigned int idx = LNET_NIDADDR(id.nid) % LST_NODE_HASHSIZE;
 143         lstcon_ndlink_t *ndl;
 144         lstcon_node_t *nd;
 145         int rc;
 146
 147         if (id.nid == LNET_NID_ANY)
 148                 return -EINVAL;
 149
 150         /* search in hash */
 151         list_for_each_entry(ndl, &hash[idx], ndl_hlink) {
 152                 if (ndl->ndl_node->nd_id.nid != id.nid ||
 153                     ndl->ndl_node->nd_id.pid != id.pid)
 154                         continue;
 155
 156                 *ndlpp = ndl;
 157                 return 0;
 158         }
 159
 160         if (create == 0)
 161                 return -ENOENT;
 162
 163         /* find or create in session hash */
 164         rc = lstcon_node_find(id, &nd, (create == 1) ? 1 : 0);
 165         if (rc != 0)
 166                 return rc;
 167
 168         LIBCFS_ALLOC(ndl, sizeof(lstcon_ndlink_t));
 169         if (ndl == NULL) {
 170                 lstcon_node_put(nd);
 171                 return -ENOMEM;
 172         }
 173
 174         *ndlpp = ndl;
 175
 176         ndl->ndl_node = nd;
 177         INIT_LIST_HEAD(&ndl->ndl_link);
 178         list_add_tail(&ndl->ndl_hlink, &hash[idx]);
 179
 180         return  0;
 181 }
 182
 183 static void
 184 lstcon_ndlink_release(lstcon_ndlink_t *ndl)
 185 {
 186         LASSERT(list_empty(&ndl->ndl_link));
 187         LASSERT(!list_empty(&ndl->ndl_hlink));
 188
 189         list_del(&ndl->ndl_hlink); /* delete from hash */
 190         lstcon_node_put(ndl->ndl_node);
 191
 192         LIBCFS_FREE(ndl, sizeof(*ndl));
 193 }
 194
 195 static int
 196 lstcon_group_alloc(char *name, lstcon_group_t **grpp)
 197 {
 198         lstcon_group_t *grp;
 199         int i;
 200
 201         LIBCFS_ALLOC(grp, offsetof(lstcon_group_t,
 202                                    grp_ndl_hash[LST_NODE_HASHSIZE]));
 203         if (grp == NULL)
 204                 return -ENOMEM;
 205
 206         grp->grp_ref = 1;
 207         if (name != NULL)
 208                 strcpy(grp->grp_name, name);
 209
 210         INIT_LIST_HEAD(&grp->grp_link);
 211         INIT_LIST_HEAD(&grp->grp_ndl_list);
 212         INIT_LIST_HEAD(&grp->grp_trans_list);
 213
 214         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 215                 INIT_LIST_HEAD(&grp->grp_ndl_hash[i]);
 216
 217         *grpp = grp;
 218
 219         return 0;
 220 }
 221
 222 static void
 223 lstcon_group_addref(lstcon_group_t *grp)
 224 {
 225         grp->grp_ref++;
 226 }
 227
 228 static void lstcon_group_ndlink_release(lstcon_group_t *, lstcon_ndlink_t *);
 229
 230 static void
 231 lstcon_group_drain(lstcon_group_t *grp, int keep)
 232 {
 233         lstcon_ndlink_t *ndl;
 234         lstcon_ndlink_t *tmp;
 235
 236         list_for_each_entry_safe(ndl, tmp, &grp->grp_ndl_list, ndl_link) {
 237                 if ((ndl->ndl_node->nd_state & keep) == 0)
 238                         lstcon_group_ndlink_release(grp, ndl);
 239         }
 240 }
 241
 242 static void
 243 lstcon_group_decref(lstcon_group_t *grp)
 244 {
 245         int i;
 246
 247         if (--grp->grp_ref > 0)
 248                 return;
 249
 250         if (!list_empty(&grp->grp_link))
 251                 list_del(&grp->grp_link);
 252
 253         lstcon_group_drain(grp, 0);
 254
 255         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 256                 LASSERT(list_empty(&grp->grp_ndl_hash[i]));
 257         }
 258
 259         LIBCFS_FREE(grp, offsetof(lstcon_group_t,
 260                                   grp_ndl_hash[LST_NODE_HASHSIZE]));
 261 }
 262
 263 static int
 264 lstcon_group_find(const char *name, lstcon_group_t **grpp)
 265 {
 266         lstcon_group_t *grp;
 267
 268         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 269                 if (strncmp(grp->grp_name, name, LST_NAME_SIZE) != 0)
 270                         continue;
 271
 272                 lstcon_group_addref(grp);  /* +1 ref for caller */
 273                 *grpp = grp;
 274                 return 0;
 275         }
 276
 277         return -ENOENT;
 278 }
 279
 280 static int
 281 lstcon_group_ndlink_find(lstcon_group_t *grp, lnet_process_id_t id,
 282                          lstcon_ndlink_t **ndlpp, int create)
 283 {
 284         int rc;
 285
 286         rc = lstcon_ndlink_find(&grp->grp_ndl_hash[0], id, ndlpp, create);
 287         if (rc != 0)
 288                 return rc;
 289
 290         if (!list_empty(&(*ndlpp)->ndl_link))
 291                 return 0;
 292
 293         list_add_tail(&(*ndlpp)->ndl_link, &grp->grp_ndl_list);
 294         grp->grp_nnode++;
 295
 296         return 0;
 297 }
 298
 299 static void
 300 lstcon_group_ndlink_release(lstcon_group_t *grp, lstcon_ndlink_t *ndl)
 301 {
 302         list_del_init(&ndl->ndl_link);
 303         lstcon_ndlink_release(ndl);
 304         grp->grp_nnode--;
 305 }
 306
 307 static void
 308 lstcon_group_ndlink_move(lstcon_group_t *old,
 309                          lstcon_group_t *new, lstcon_ndlink_t *ndl)
 310 {
 311         unsigned int idx = LNET_NIDADDR(ndl->ndl_node->nd_id.nid) %
 312                            LST_NODE_HASHSIZE;
 313
 314         list_del(&ndl->ndl_hlink);
 315         list_del(&ndl->ndl_link);
 316         old->grp_nnode--;
 317
 318         list_add_tail(&ndl->ndl_hlink, &new->grp_ndl_hash[idx]);
 319         list_add_tail(&ndl->ndl_link, &new->grp_ndl_list);
 320         new->grp_nnode++;
 321 }
 322
 323 static void
 324 lstcon_group_move(lstcon_group_t *old, lstcon_group_t *new)
 325 {
 326         lstcon_ndlink_t *ndl;
 327
 328         while (!list_empty(&old->grp_ndl_list)) {
 329                 ndl = list_entry(old->grp_ndl_list.next,
 330                                      lstcon_ndlink_t, ndl_link);
 331                 lstcon_group_ndlink_move(old, new, ndl);
 332         }
 333 }
 334
 335 static int
 336 lstcon_sesrpc_condition(int transop, lstcon_node_t *nd, void *arg)
 337 {
 338         lstcon_group_t *grp = (lstcon_group_t *)arg;
 339
 340         switch (transop) {
 341         case LST_TRANS_SESNEW:
 342                 if (nd->nd_state == LST_NODE_ACTIVE)
 343                         return 0;
 344                 break;
 345
 346         case LST_TRANS_SESEND:
 347                 if (nd->nd_state != LST_NODE_ACTIVE)
 348                         return 0;
 349
 350                 if (grp != NULL && nd->nd_ref > 1)
 351                         return 0;
 352                 break;
 353
 354         case LST_TRANS_SESQRY:
 355                 break;
 356
 357         default:
 358                 LBUG();
 359         }
 360
 361         return 1;
 362 }
 363
 364 static int
 365 lstcon_sesrpc_readent(int transop, srpc_msg_t *msg,
 366                       lstcon_rpc_ent_t *ent_up)
 367 {
 368         srpc_debug_reply_t *rep;
 369
 370         switch (transop) {
 371         case LST_TRANS_SESNEW:
 372         case LST_TRANS_SESEND:
 373                 return 0;
 374
 375         case LST_TRANS_SESQRY:
 376                 rep = &msg->msg_body.dbg_reply;
 377
 378                 if (copy_to_user(&ent_up->rpe_priv[0],
 379                                      &rep->dbg_timeout, sizeof(int)) ||
 380                     copy_to_user(&ent_up->rpe_payload[0],
 381                                      &rep->dbg_name, LST_NAME_SIZE))
 382                         return -EFAULT;
 383
 384                 return 0;
 385
 386         default:
 387                 LBUG();
 388         }
 389
 390         return 0;
 391 }
 392
 393 static int
 394 lstcon_group_nodes_add(lstcon_group_t *grp,
 395                        int count, lnet_process_id_t *ids_up,
 396                        unsigned *featp, struct list_head *result_up)
 397 {
 398         lstcon_rpc_trans_t *trans;
 399         lstcon_ndlink_t *ndl;
 400         lstcon_group_t *tmp;
 401         lnet_process_id_t id;
 402         int i;
 403         int rc;
 404
 405         rc = lstcon_group_alloc(NULL, &tmp);
 406         if (rc != 0) {
 407                 CERROR("Out of memory\n");
 408                 return -ENOMEM;
 409         }
 410
 411         for (i = 0 ; i < count; i++) {
 412                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 413                         rc = -EFAULT;
 414                         break;
 415                 }
 416
 417                 /* skip if it's in this group already */
 418                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 0);
 419                 if (rc == 0)
 420                         continue;
 421
 422                 /* add to tmp group */
 423                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 1);
 424                 if (rc != 0) {
 425                         CERROR("Can't create ndlink, out of memory\n");
 426                         break;
 427                 }
 428         }
 429
 430         if (rc != 0) {
 431                 lstcon_group_decref(tmp);
 432                 return rc;
 433         }
 434
 435         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 436                                      &tmp->grp_trans_list, LST_TRANS_SESNEW,
 437                                      tmp, lstcon_sesrpc_condition, &trans);
 438         if (rc != 0) {
 439                 CERROR("Can't create transaction: %d\n", rc);
 440                 lstcon_group_decref(tmp);
 441                 return rc;
 442         }
 443
 444         /* post all RPCs */
 445         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 446
 447         rc = lstcon_rpc_trans_interpreter(trans, result_up,
 448                                           lstcon_sesrpc_readent);
 449         *featp = trans->tas_features;
 450
 451         /* destroy all RPGs */
 452         lstcon_rpc_trans_destroy(trans);
 453
 454         lstcon_group_move(tmp, grp);
 455         lstcon_group_decref(tmp);
 456
 457         return rc;
 458 }
 459
 460 static int
 461 lstcon_group_nodes_remove(lstcon_group_t *grp,
 462                           int count, lnet_process_id_t *ids_up,
 463                           struct list_head *result_up)
 464 {
 465         lstcon_rpc_trans_t *trans;
 466         lstcon_ndlink_t *ndl;
 467         lstcon_group_t *tmp;
 468         lnet_process_id_t id;
 469         int rc;
 470         int i;
 471
 472         /* End session and remove node from the group */
 473
 474         rc = lstcon_group_alloc(NULL, &tmp);
 475         if (rc != 0) {
 476                 CERROR("Out of memory\n");
 477                 return -ENOMEM;
 478         }
 479
 480         for (i = 0; i < count; i++) {
 481                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 482                         rc = -EFAULT;
 483                         goto error;
 484                 }
 485
 486                 /* move node to tmp group */
 487                 if (lstcon_group_ndlink_find(grp, id, &ndl, 0) == 0)
 488                         lstcon_group_ndlink_move(grp, tmp, ndl);
 489         }
 490
 491         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 492                                      &tmp->grp_trans_list, LST_TRANS_SESEND,
 493                                      tmp, lstcon_sesrpc_condition, &trans);
 494         if (rc != 0) {
 495                 CERROR("Can't create transaction: %d\n", rc);
 496                 goto error;
 497         }
 498
 499         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 500
 501         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 502
 503         lstcon_rpc_trans_destroy(trans);
 504         /* release nodes anyway, because we can't rollback status */
 505         lstcon_group_decref(tmp);
 506
 507         return rc;
 508 error:
 509         lstcon_group_move(tmp, grp);
 510         lstcon_group_decref(tmp);
 511
 512         return rc;
 513 }
 514
 515 int
 516 lstcon_group_add(char *name)
 517 {
 518         lstcon_group_t *grp;
 519         int rc;
 520
 521         rc = (lstcon_group_find(name, &grp) == 0) ? -EEXIST : 0;
 522         if (rc != 0) {
 523                 /* find a group with same name */
 524                 lstcon_group_decref(grp);
 525                 return rc;
 526         }
 527
 528         rc = lstcon_group_alloc(name, &grp);
 529         if (rc != 0) {
 530                 CERROR("Can't allocate descriptor for group %s\n", name);
 531                 return -ENOMEM;
 532         }
 533
 534         list_add_tail(&grp->grp_link, &console_session.ses_grp_list);
 535
 536         return rc;
 537 }
 538
 539 int
 540 lstcon_nodes_add(char *name, int count, lnet_process_id_t *ids_up,
 541                  unsigned *featp, struct list_head *result_up)
 542 {
 543         lstcon_group_t *grp;
 544         int rc;
 545
 546         LASSERT(count > 0);
 547         LASSERT(ids_up != NULL);
 548
 549         rc = lstcon_group_find(name, &grp);
 550         if (rc != 0) {
 551                 CDEBUG(D_NET, "Can't find group %s\n", name);
 552                 return rc;
 553         }
 554
 555         if (grp->grp_ref > 2) {
 556                 /* referred by other threads or test */
 557                 CDEBUG(D_NET, "Group %s is busy\n", name);
 558                 lstcon_group_decref(grp);
 559
 560                 return -EBUSY;
 561         }
 562
 563         rc = lstcon_group_nodes_add(grp, count, ids_up, featp, result_up);
 564
 565         lstcon_group_decref(grp);
 566
 567         return rc;
 568 }
 569
 570 int
 571 lstcon_group_del(char *name)
 572 {
 573         lstcon_rpc_trans_t *trans;
 574         lstcon_group_t *grp;
 575         int rc;
 576
 577         rc = lstcon_group_find(name, &grp);
 578         if (rc != 0) {
 579                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 580                 return rc;
 581         }
 582
 583         if (grp->grp_ref > 2) {
 584                 /* referred by others threads or test */
 585                 CDEBUG(D_NET, "Group %s is busy\n", name);
 586                 lstcon_group_decref(grp);
 587                 return -EBUSY;
 588         }
 589
 590         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 591                                      &grp->grp_trans_list, LST_TRANS_SESEND,
 592                                      grp, lstcon_sesrpc_condition, &trans);
 593         if (rc != 0) {
 594                 CERROR("Can't create transaction: %d\n", rc);
 595                 lstcon_group_decref(grp);
 596                 return rc;
 597         }
 598
 599         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 600
 601         lstcon_rpc_trans_destroy(trans);
 602
 603         lstcon_group_decref(grp);
 604         /* -ref for session, it's destroyed,
 605          * status can't be rolled back, destroy group anyway */
 606         lstcon_group_decref(grp);
 607
 608         return rc;
 609 }
 610
 611 int
 612 lstcon_group_clean(char *name, int args)
 613 {
 614         lstcon_group_t *grp = NULL;
 615         int rc;
 616
 617         rc = lstcon_group_find(name, &grp);
 618         if (rc != 0) {
 619                 CDEBUG(D_NET, "Can't find group %s\n", name);
 620                 return rc;
 621         }
 622
 623         if (grp->grp_ref > 2) {
 624                 /* referred by test */
 625                 CDEBUG(D_NET, "Group %s is busy\n", name);
 626                 lstcon_group_decref(grp);
 627                 return -EBUSY;
 628         }
 629
 630         args = (LST_NODE_ACTIVE | LST_NODE_BUSY |
 631                 LST_NODE_DOWN | LST_NODE_UNKNOWN) & ~args;
 632
 633         lstcon_group_drain(grp, args);
 634
 635         lstcon_group_decref(grp);
 636         /* release empty group */
 637         if (list_empty(&grp->grp_ndl_list))
 638                 lstcon_group_decref(grp);
 639
 640         return 0;
 641 }
 642
 643 int
 644 lstcon_nodes_remove(char *name, int count,
 645                     lnet_process_id_t *ids_up, struct list_head *result_up)
 646 {
 647         lstcon_group_t *grp = NULL;
 648         int rc;
 649
 650         rc = lstcon_group_find(name, &grp);
 651         if (rc != 0) {
 652                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 653                 return rc;
 654         }
 655
 656         if (grp->grp_ref > 2) {
 657                 /* referred by test */
 658                 CDEBUG(D_NET, "Group %s is busy\n", name);
 659                 lstcon_group_decref(grp);
 660                 return -EBUSY;
 661         }
 662
 663         rc = lstcon_group_nodes_remove(grp, count, ids_up, result_up);
 664
 665         lstcon_group_decref(grp);
 666         /* release empty group */
 667         if (list_empty(&grp->grp_ndl_list))
 668                 lstcon_group_decref(grp);
 669
 670         return rc;
 671 }
 672
 673 int
 674 lstcon_group_refresh(char *name, struct list_head *result_up)
 675 {
 676         lstcon_rpc_trans_t *trans;
 677         lstcon_group_t *grp;
 678         int rc;
 679
 680         rc = lstcon_group_find(name, &grp);
 681         if (rc != 0) {
 682                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 683                 return rc;
 684         }
 685
 686         if (grp->grp_ref > 2) {
 687                 /* referred by test */
 688                 CDEBUG(D_NET, "Group %s is busy\n", name);
 689                 lstcon_group_decref(grp);
 690                 return -EBUSY;
 691         }
 692
 693         /* re-invite all inactive nodes int the group */
 694         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 695                                      &grp->grp_trans_list, LST_TRANS_SESNEW,
 696                                      grp, lstcon_sesrpc_condition, &trans);
 697         if (rc != 0) {
 698                 /* local error, return */
 699                 CDEBUG(D_NET, "Can't create transaction: %d\n", rc);
 700                 lstcon_group_decref(grp);
 701                 return rc;
 702         }
 703
 704         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 705
 706         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 707
 708         lstcon_rpc_trans_destroy(trans);
 709         /* -ref for me */
 710         lstcon_group_decref(grp);
 711
 712         return rc;
 713 }
 714
 715 int
 716 lstcon_group_list(int index, int len, char *name_up)
 717 {
 718         lstcon_group_t *grp;
 719
 720         LASSERT(index >= 0);
 721         LASSERT(name_up != NULL);
 722
 723         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 724                 if (index-- == 0) {
 725                         return copy_to_user(name_up, grp->grp_name, len) ?
 726                                -EFAULT : 0;
 727                 }
 728         }
 729
 730         return -ENOENT;
 731 }
 732
 733 static int
 734 lstcon_nodes_getent(struct list_head *head, int *index_p,
 735                     int *count_p, lstcon_node_ent_t *dents_up)
 736 {
 737         lstcon_ndlink_t *ndl;
 738         lstcon_node_t *nd;
 739         int count = 0;
 740         int index = 0;
 741
 742         LASSERT(index_p != NULL && count_p != NULL);
 743         LASSERT(dents_up != NULL);
 744         LASSERT(*index_p >= 0);
 745         LASSERT(*count_p > 0);
 746
 747         list_for_each_entry(ndl, head, ndl_link) {
 748                 if (index++ < *index_p)
 749                         continue;
 750
 751                 if (count >= *count_p)
 752                         break;
 753
 754                 nd = ndl->ndl_node;
 755                 if (copy_to_user(&dents_up[count].nde_id,
 756                                      &nd->nd_id, sizeof(nd->nd_id)) ||
 757                     copy_to_user(&dents_up[count].nde_state,
 758                                      &nd->nd_state, sizeof(nd->nd_state)))
 759                         return -EFAULT;
 760
 761                 count++;
 762         }
 763
 764         if (index <= *index_p)
 765                 return -ENOENT;
 766
 767         *count_p = count;
 768         *index_p = index;
 769
 770         return 0;
 771 }
 772
 773 int
 774 lstcon_group_info(char *name, lstcon_ndlist_ent_t *gents_p,
 775                   int *index_p, int *count_p, lstcon_node_ent_t *dents_up)
 776 {
 777         lstcon_ndlist_ent_t *gentp;
 778         lstcon_group_t *grp;
 779         lstcon_ndlink_t *ndl;
 780         int rc;
 781
 782         rc = lstcon_group_find(name, &grp);
 783         if (rc != 0) {
 784                 CDEBUG(D_NET, "Can't find group %s\n", name);
 785                 return rc;
 786         }
 787
 788         if (dents_up) {
 789                 /* verbose query */
 790                 rc = lstcon_nodes_getent(&grp->grp_ndl_list,
 791                                          index_p, count_p, dents_up);
 792                 lstcon_group_decref(grp);
 793
 794                 return rc;
 795         }
 796
 797         /* non-verbose query */
 798         LIBCFS_ALLOC(gentp, sizeof(lstcon_ndlist_ent_t));
 799         if (gentp == NULL) {
 800                 CERROR("Can't allocate ndlist_ent\n");
 801                 lstcon_group_decref(grp);
 802
 803                 return -ENOMEM;
 804         }
 805
 806         list_for_each_entry(ndl, &grp->grp_ndl_list, ndl_link)
 807                 LST_NODE_STATE_COUNTER(ndl->ndl_node, gentp);
 808
 809         rc = copy_to_user(gents_p, gentp,
 810                               sizeof(lstcon_ndlist_ent_t)) ? -EFAULT : 0;
 811
 812         LIBCFS_FREE(gentp, sizeof(lstcon_ndlist_ent_t));
 813
 814         lstcon_group_decref(grp);
 815
 816         return 0;
 817 }
 818
 819 static int
 820 lstcon_batch_find(const char *name, lstcon_batch_t **batpp)
 821 {
 822         lstcon_batch_t *bat;
 823
 824         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 825                 if (strncmp(bat->bat_name, name, LST_NAME_SIZE) == 0) {
 826                         *batpp = bat;
 827                         return 0;
 828                 }
 829         }
 830
 831         return -ENOENT;
 832 }
 833
 834 int
 835 lstcon_batch_add(char *name)
 836 {
 837         lstcon_batch_t *bat;
 838         int i;
 839         int rc;
 840
 841         rc = (lstcon_batch_find(name, &bat) == 0) ? -EEXIST : 0;
 842         if (rc != 0) {
 843                 CDEBUG(D_NET, "Batch %s already exists\n", name);
 844                 return rc;
 845         }
 846
 847         LIBCFS_ALLOC(bat, sizeof(lstcon_batch_t));
 848         if (bat == NULL) {
 849                 CERROR("Can't allocate descriptor for batch %s\n", name);
 850                 return -ENOMEM;
 851         }
 852
 853         LIBCFS_ALLOC(bat->bat_cli_hash,
 854                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 855         if (bat->bat_cli_hash == NULL) {
 856                 CERROR("Can't allocate hash for batch %s\n", name);
 857                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
 858
 859                 return -ENOMEM;
 860         }
 861
 862         LIBCFS_ALLOC(bat->bat_srv_hash,
 863                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 864         if (bat->bat_srv_hash == NULL) {
 865                 CERROR("Can't allocate hash for batch %s\n", name);
 866                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 867                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
 868
 869                 return -ENOMEM;
 870         }
 871
 872         strcpy(bat->bat_name, name);
 873         bat->bat_hdr.tsb_index = 0;
 874         bat->bat_hdr.tsb_id.bat_id = ++console_session.ses_id_cookie;
 875
 876         bat->bat_ntest = 0;
 877         bat->bat_state = LST_BATCH_IDLE;
 878
 879         INIT_LIST_HEAD(&bat->bat_cli_list);
 880         INIT_LIST_HEAD(&bat->bat_srv_list);
 881         INIT_LIST_HEAD(&bat->bat_test_list);
 882         INIT_LIST_HEAD(&bat->bat_trans_list);
 883
 884         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 885                 INIT_LIST_HEAD(&bat->bat_cli_hash[i]);
 886                 INIT_LIST_HEAD(&bat->bat_srv_hash[i]);
 887         }
 888
 889         list_add_tail(&bat->bat_link, &console_session.ses_bat_list);
 890
 891         return rc;
 892 }
 893
 894 int
 895 lstcon_batch_list(int index, int len, char *name_up)
 896 {
 897         lstcon_batch_t *bat;
 898
 899         LASSERT(name_up != NULL);
 900         LASSERT(index >= 0);
 901
 902         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 903                 if (index-- == 0) {
 904                         return copy_to_user(name_up, bat->bat_name, len) ?
 905                                -EFAULT : 0;
 906                 }
 907         }
 908
 909         return -ENOENT;
 910 }
 911
 912 int
 913 lstcon_batch_info(char *name, lstcon_test_batch_ent_t *ent_up, int server,
 914                   int testidx, int *index_p, int *ndent_p,
 915                   lstcon_node_ent_t *dents_up)
 916 {
 917         lstcon_test_batch_ent_t *entp;
 918         struct list_head *clilst;
 919         struct list_head *srvlst;
 920         lstcon_test_t *test = NULL;
 921         lstcon_batch_t *bat;
 922         lstcon_ndlink_t *ndl;
 923         int rc;
 924
 925         rc = lstcon_batch_find(name, &bat);
 926         if (rc != 0) {
 927                 CDEBUG(D_NET, "Can't find batch %s\n", name);
 928                 return -ENOENT;
 929         }
 930
 931         if (testidx > 0) {
 932                 /* query test, test index start from 1 */
 933                 list_for_each_entry(test, &bat->bat_test_list, tes_link) {
 934                         if (testidx-- == 1)
 935                                 break;
 936                 }
 937
 938                 if (testidx > 0) {
 939                         CDEBUG(D_NET, "Can't find specified test in batch\n");
 940                         return -ENOENT;
 941                 }
 942         }
 943
 944         clilst = (test == NULL) ? &bat->bat_cli_list :
 945                                   &test->tes_src_grp->grp_ndl_list;
 946         srvlst = (test == NULL) ? &bat->bat_srv_list :
 947                                   &test->tes_dst_grp->grp_ndl_list;
 948
 949         if (dents_up != NULL) {
 950                 rc = lstcon_nodes_getent((server ? srvlst : clilst),
 951                                          index_p, ndent_p, dents_up);
 952                 return rc;
 953         }
 954
 955         /* non-verbose query */
 956         LIBCFS_ALLOC(entp, sizeof(lstcon_test_batch_ent_t));
 957         if (entp == NULL)
 958                 return -ENOMEM;
 959
 960         if (test == NULL) {
 961                 entp->u.tbe_batch.bae_ntest = bat->bat_ntest;
 962                 entp->u.tbe_batch.bae_state = bat->bat_state;
 963
 964         } else {
 965
 966                 entp->u.tbe_test.tse_type   = test->tes_type;
 967                 entp->u.tbe_test.tse_loop   = test->tes_loop;
 968                 entp->u.tbe_test.tse_concur = test->tes_concur;
 969         }
 970
 971         list_for_each_entry(ndl, clilst, ndl_link)
 972                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_cli_nle);
 973
 974         list_for_each_entry(ndl, srvlst, ndl_link)
 975                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_srv_nle);
 976
 977         rc = copy_to_user(ent_up, entp,
 978                               sizeof(lstcon_test_batch_ent_t)) ? -EFAULT : 0;
 979
 980         LIBCFS_FREE(entp, sizeof(lstcon_test_batch_ent_t));
 981
 982         return rc;
 983 }
 984
 985 static int
 986 lstcon_batrpc_condition(int transop, lstcon_node_t *nd, void *arg)
 987 {
 988         switch (transop) {
 989         case LST_TRANS_TSBRUN:
 990                 if (nd->nd_state != LST_NODE_ACTIVE)
 991                         return -ENETDOWN;
 992                 break;
 993
 994         case LST_TRANS_TSBSTOP:
 995                 if (nd->nd_state != LST_NODE_ACTIVE)
 996                         return 0;
 997                 break;
 998
 999         case LST_TRANS_TSBCLIQRY:
1000         case LST_TRANS_TSBSRVQRY:
1001                 break;
1002         }
1003
1004         return 1;
1005 }
1006
1007 static int
1008 lstcon_batch_op(lstcon_batch_t *bat, int transop,
1009                 struct list_head *result_up)
1010 {
1011         lstcon_rpc_trans_t *trans;
1012         int rc;
1013
1014         rc = lstcon_rpc_trans_ndlist(&bat->bat_cli_list,
1015                                      &bat->bat_trans_list, transop,
1016                                      bat, lstcon_batrpc_condition, &trans);
1017         if (rc != 0) {
1018                 CERROR("Can't create transaction: %d\n", rc);
1019                 return rc;
1020         }
1021
1022         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1023
1024         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1025
1026         lstcon_rpc_trans_destroy(trans);
1027
1028         return rc;
1029 }
1030
1031 int
1032 lstcon_batch_run(char *name, int timeout, struct list_head *result_up)
1033 {
1034         lstcon_batch_t *bat;
1035         int rc;
1036
1037         if (lstcon_batch_find(name, &bat) != 0) {
1038                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1039                 return -ENOENT;
1040         }
1041
1042         bat->bat_arg = timeout;
1043
1044         rc = lstcon_batch_op(bat, LST_TRANS_TSBRUN, result_up);
1045
1046         /* mark batch as running if it's started in any node */
1047         if (lstcon_tsbop_stat_success(lstcon_trans_stat(), 0) != 0)
1048                 bat->bat_state = LST_BATCH_RUNNING;
1049
1050         return rc;
1051 }
1052
1053 int
1054 lstcon_batch_stop(char *name, int force, struct list_head *result_up)
1055 {
1056         lstcon_batch_t *bat;
1057         int rc;
1058
1059         if (lstcon_batch_find(name, &bat) != 0) {
1060                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1061                 return -ENOENT;
1062         }
1063
1064         bat->bat_arg = force;
1065
1066         rc = lstcon_batch_op(bat, LST_TRANS_TSBSTOP, result_up);
1067
1068         /* mark batch as stopped if all RPCs finished */
1069         if (lstcon_tsbop_stat_failure(lstcon_trans_stat(), 0) == 0)
1070                 bat->bat_state = LST_BATCH_IDLE;
1071
1072         return rc;
1073 }
1074
1075 static void
1076 lstcon_batch_destroy(lstcon_batch_t *bat)
1077 {
1078         lstcon_ndlink_t *ndl;
1079         lstcon_test_t *test;
1080         int i;
1081
1082         list_del(&bat->bat_link);
1083
1084         while (!list_empty(&bat->bat_test_list)) {
1085                 test = list_entry(bat->bat_test_list.next,
1086                                       lstcon_test_t, tes_link);
1087                 LASSERT(list_empty(&test->tes_trans_list));
1088
1089                 list_del(&test->tes_link);
1090
1091                 lstcon_group_decref(test->tes_src_grp);
1092                 lstcon_group_decref(test->tes_dst_grp);
1093
1094                 LIBCFS_FREE(test, offsetof(lstcon_test_t,
1095                                            tes_param[test->tes_paramlen]));
1096         }
1097
1098         LASSERT(list_empty(&bat->bat_trans_list));
1099
1100         while (!list_empty(&bat->bat_cli_list)) {
1101                 ndl = list_entry(bat->bat_cli_list.next,
1102                                      lstcon_ndlink_t, ndl_link);
1103                 list_del_init(&ndl->ndl_link);
1104
1105                 lstcon_ndlink_release(ndl);
1106         }
1107
1108         while (!list_empty(&bat->bat_srv_list)) {
1109                 ndl = list_entry(bat->bat_srv_list.next,
1110                                      lstcon_ndlink_t, ndl_link);
1111                 list_del_init(&ndl->ndl_link);
1112
1113                 lstcon_ndlink_release(ndl);
1114         }
1115
1116         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
1117                 LASSERT(list_empty(&bat->bat_cli_hash[i]));
1118                 LASSERT(list_empty(&bat->bat_srv_hash[i]));
1119         }
1120
1121         LIBCFS_FREE(bat->bat_cli_hash,
1122                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1123         LIBCFS_FREE(bat->bat_srv_hash,
1124                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1125         LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
1126 }
1127
1128 static int
1129 lstcon_testrpc_condition(int transop, lstcon_node_t *nd, void *arg)
1130 {
1131         lstcon_test_t *test;
1132         lstcon_batch_t *batch;
1133         lstcon_ndlink_t *ndl;
1134         struct list_head *hash;
1135         struct list_head *head;
1136
1137         test = (lstcon_test_t *)arg;
1138         LASSERT(test != NULL);
1139
1140         batch = test->tes_batch;
1141         LASSERT(batch != NULL);
1142
1143         if (test->tes_oneside &&
1144             transop == LST_TRANS_TSBSRVADD)
1145                 return 0;
1146
1147         if (nd->nd_state != LST_NODE_ACTIVE)
1148                 return -ENETDOWN;
1149
1150         if (transop == LST_TRANS_TSBCLIADD) {
1151                 hash = batch->bat_cli_hash;
1152                 head = &batch->bat_cli_list;
1153
1154         } else {
1155                 LASSERT(transop == LST_TRANS_TSBSRVADD);
1156
1157                 hash = batch->bat_srv_hash;
1158                 head = &batch->bat_srv_list;
1159         }
1160
1161         LASSERT(nd->nd_id.nid != LNET_NID_ANY);
1162
1163         if (lstcon_ndlink_find(hash, nd->nd_id, &ndl, 1) != 0)
1164                 return -ENOMEM;
1165
1166         if (list_empty(&ndl->ndl_link))
1167                 list_add_tail(&ndl->ndl_link, head);
1168
1169         return 1;
1170 }
1171
1172 static int
1173 lstcon_test_nodes_add(lstcon_test_t *test, struct list_head *result_up)
1174 {
1175         lstcon_rpc_trans_t *trans;
1176         lstcon_group_t *grp;
1177         int transop;
1178         int rc;
1179
1180         LASSERT(test->tes_src_grp != NULL);
1181         LASSERT(test->tes_dst_grp != NULL);
1182
1183         transop = LST_TRANS_TSBSRVADD;
1184         grp  = test->tes_dst_grp;
1185 again:
1186         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
1187                                      &test->tes_trans_list, transop,
1188                                      test, lstcon_testrpc_condition, &trans);
1189         if (rc != 0) {
1190                 CERROR("Can't create transaction: %d\n", rc);
1191                 return rc;
1192         }
1193
1194         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1195
1196         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1197             lstcon_trans_stat()->trs_fwk_errno != 0) {
1198                 lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1199
1200                 lstcon_rpc_trans_destroy(trans);
1201                 /* return if any error */
1202                 CDEBUG(D_NET, "Failed to add test %s, RPC error %d, framework error %d\n",
1203                        transop == LST_TRANS_TSBCLIADD ? "client" : "server",
1204                        lstcon_trans_stat()->trs_rpc_errno,
1205                        lstcon_trans_stat()->trs_fwk_errno);
1206
1207                 return rc;
1208         }
1209
1210         lstcon_rpc_trans_destroy(trans);
1211
1212         if (transop == LST_TRANS_TSBCLIADD)
1213                 return rc;
1214
1215         transop = LST_TRANS_TSBCLIADD;
1216         grp = test->tes_src_grp;
1217         test->tes_cliidx = 0;
1218
1219         /* requests to test clients */
1220         goto again;
1221 }
1222
1223 static int
1224 lstcon_verify_batch(const char *name, lstcon_batch_t **batch)
1225 {
1226         int rc;
1227
1228         rc = lstcon_batch_find(name, batch);
1229         if (rc != 0) {
1230                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1231                 return rc;
1232         }
1233
1234         if ((*batch)->bat_state != LST_BATCH_IDLE) {
1235                 CDEBUG(D_NET, "Can't change running batch %s\n", name);
1236                 return -EINVAL;
1237         }
1238
1239         return 0;
1240 }
1241
1242 static int
1243 lstcon_verify_group(const char *name, lstcon_group_t **grp)
1244 {
1245         int rc;
1246         lstcon_ndlink_t *ndl;
1247
1248         rc = lstcon_group_find(name, grp);
1249         if (rc != 0) {
1250                 CDEBUG(D_NET, "can't find group %s\n", name);
1251                 return rc;
1252         }
1253
1254         list_for_each_entry(ndl, &(*grp)->grp_ndl_list, ndl_link) {
1255                 if (ndl->ndl_node->nd_state == LST_NODE_ACTIVE)
1256                         return 0;
1257         }
1258
1259         CDEBUG(D_NET, "Group %s has no ACTIVE nodes\n", name);
1260
1261         return -EINVAL;
1262 }
1263
1264 int
1265 lstcon_test_add(char *batch_name, int type, int loop,
1266                 int concur, int dist, int span,
1267                 char *src_name, char *dst_name,
1268                 void *param, int paramlen, int *retp,
1269                 struct list_head *result_up)
1270 {
1271         lstcon_test_t    *test   = NULL;
1272         int              rc;
1273         lstcon_group_t   *src_grp = NULL;
1274         lstcon_group_t   *dst_grp = NULL;
1275         lstcon_batch_t   *batch = NULL;
1276
1277         /*
1278          * verify that a batch of the given name exists, and the groups
1279          * that will be part of the batch exist and have at least one
1280          * active node
1281          */
1282         rc = lstcon_verify_batch(batch_name, &batch);
1283         if (rc != 0)
1284                 goto out;
1285
1286         rc = lstcon_verify_group(src_name, &src_grp);
1287         if (rc != 0)
1288                 goto out;
1289
1290         rc = lstcon_verify_group(dst_name, &dst_grp);
1291         if (rc != 0)
1292                 goto out;
1293
1294         if (dst_grp->grp_userland)
1295                 *retp = 1;
1296
1297         LIBCFS_ALLOC(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1298         if (!test) {
1299                 CERROR("Can't allocate test descriptor\n");
1300                 rc = -ENOMEM;
1301
1302                 goto out;
1303         }
1304
1305         test->tes_hdr.tsb_id    = batch->bat_hdr.tsb_id;
1306         test->tes_batch         = batch;
1307         test->tes_type          = type;
1308         test->tes_oneside       = 0; /* TODO */
1309         test->tes_loop          = loop;
1310         test->tes_concur        = concur;
1311         test->tes_stop_onerr    = 1; /* TODO */
1312         test->tes_span          = span;
1313         test->tes_dist          = dist;
1314         test->tes_cliidx        = 0; /* just used for creating RPC */
1315         test->tes_src_grp       = src_grp;
1316         test->tes_dst_grp       = dst_grp;
1317         INIT_LIST_HEAD(&test->tes_trans_list);
1318
1319         if (param != NULL) {
1320                 test->tes_paramlen = paramlen;
1321                 memcpy(&test->tes_param[0], param, paramlen);
1322         }
1323
1324         rc = lstcon_test_nodes_add(test, result_up);
1325
1326         if (rc != 0)
1327                 goto out;
1328
1329         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1330             lstcon_trans_stat()->trs_fwk_errno != 0)
1331                 CDEBUG(D_NET, "Failed to add test %d to batch %s\n", type,
1332                        batch_name);
1333
1334         /* add to test list anyway, so user can check what's going on */
1335         list_add_tail(&test->tes_link, &batch->bat_test_list);
1336
1337         batch->bat_ntest++;
1338         test->tes_hdr.tsb_index = batch->bat_ntest;
1339
1340         /*  hold groups so nobody can change them */
1341         return rc;
1342 out:
1343         if (test != NULL)
1344                 LIBCFS_FREE(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1345
1346         if (dst_grp != NULL)
1347                 lstcon_group_decref(dst_grp);
1348
1349         if (src_grp != NULL)
1350                 lstcon_group_decref(src_grp);
1351
1352         return rc;
1353 }
1354
1355 static int
1356 lstcon_test_find(lstcon_batch_t *batch, int idx, lstcon_test_t **testpp)
1357 {
1358         lstcon_test_t *test;
1359
1360         list_for_each_entry(test, &batch->bat_test_list, tes_link) {
1361                 if (idx == test->tes_hdr.tsb_index) {
1362                         *testpp = test;
1363                         return 0;
1364                 }
1365         }
1366
1367         return -ENOENT;
1368 }
1369
1370 static int
1371 lstcon_tsbrpc_readent(int transop, srpc_msg_t *msg,
1372                       lstcon_rpc_ent_t *ent_up)
1373 {
1374         srpc_batch_reply_t *rep = &msg->msg_body.bat_reply;
1375
1376         LASSERT(transop == LST_TRANS_TSBCLIQRY ||
1377                  transop == LST_TRANS_TSBSRVQRY);
1378
1379         /* positive errno, framework error code */
1380         if (copy_to_user(&ent_up->rpe_priv[0],
1381                              &rep->bar_active, sizeof(rep->bar_active)))
1382                 return -EFAULT;
1383
1384         return 0;
1385 }
1386
1387 int
1388 lstcon_test_batch_query(char *name, int testidx, int client,
1389                         int timeout, struct list_head *result_up)
1390 {
1391         lstcon_rpc_trans_t *trans;
1392         struct list_head *translist;
1393         struct list_head *ndlist;
1394         lstcon_tsb_hdr_t *hdr;
1395         lstcon_batch_t *batch;
1396         lstcon_test_t *test = NULL;
1397         int transop;
1398         int rc;
1399
1400         rc = lstcon_batch_find(name, &batch);
1401         if (rc != 0) {
1402                 CDEBUG(D_NET, "Can't find batch: %s\n", name);
1403                 return rc;
1404         }
1405
1406         if (testidx == 0) {
1407                 translist = &batch->bat_trans_list;
1408                 ndlist    = &batch->bat_cli_list;
1409                 hdr       = &batch->bat_hdr;
1410
1411         } else {
1412                 /* query specified test only */
1413                 rc = lstcon_test_find(batch, testidx, &test);
1414                 if (rc != 0) {
1415                         CDEBUG(D_NET, "Can't find test: %d\n", testidx);
1416                         return rc;
1417                 }
1418
1419                 translist = &test->tes_trans_list;
1420                 ndlist    = &test->tes_src_grp->grp_ndl_list;
1421                 hdr       = &test->tes_hdr;
1422         }
1423
1424         transop = client ? LST_TRANS_TSBCLIQRY : LST_TRANS_TSBSRVQRY;
1425
1426         rc = lstcon_rpc_trans_ndlist(ndlist, translist, transop, hdr,
1427                                      lstcon_batrpc_condition, &trans);
1428         if (rc != 0) {
1429                 CERROR("Can't create transaction: %d\n", rc);
1430                 return rc;
1431         }
1432
1433         lstcon_rpc_trans_postwait(trans, timeout);
1434
1435         if (testidx == 0 && /* query a batch, not a test */
1436             lstcon_rpc_stat_failure(lstcon_trans_stat(), 0) == 0 &&
1437             lstcon_tsbqry_stat_run(lstcon_trans_stat(), 0) == 0) {
1438                 /* all RPCs finished, and no active test */
1439                 batch->bat_state = LST_BATCH_IDLE;
1440         }
1441
1442         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1443                                           lstcon_tsbrpc_readent);
1444         lstcon_rpc_trans_destroy(trans);
1445
1446         return rc;
1447 }
1448
1449 static int
1450 lstcon_statrpc_readent(int transop, srpc_msg_t *msg,
1451                        lstcon_rpc_ent_t *ent_up)
1452 {
1453         srpc_stat_reply_t *rep = &msg->msg_body.stat_reply;
1454         sfw_counters_t *sfwk_stat;
1455         srpc_counters_t *srpc_stat;
1456         lnet_counters_t *lnet_stat;
1457
1458         if (rep->str_status != 0)
1459                 return 0;
1460
1461         sfwk_stat = (sfw_counters_t *)&ent_up->rpe_payload[0];
1462         srpc_stat = (srpc_counters_t *)((char *)sfwk_stat + sizeof(*sfwk_stat));
1463         lnet_stat = (lnet_counters_t *)((char *)srpc_stat + sizeof(*srpc_stat));
1464
1465         if (copy_to_user(sfwk_stat, &rep->str_fw, sizeof(*sfwk_stat)) ||
1466             copy_to_user(srpc_stat, &rep->str_rpc, sizeof(*srpc_stat)) ||
1467             copy_to_user(lnet_stat, &rep->str_lnet, sizeof(*lnet_stat)))
1468                 return -EFAULT;
1469
1470         return 0;
1471 }
1472
1473 static int
1474 lstcon_ndlist_stat(struct list_head *ndlist,
1475                    int timeout, struct list_head *result_up)
1476 {
1477         struct list_head head;
1478         lstcon_rpc_trans_t *trans;
1479         int rc;
1480
1481         INIT_LIST_HEAD(&head);
1482
1483         rc = lstcon_rpc_trans_ndlist(ndlist, &head,
1484                                      LST_TRANS_STATQRY, NULL, NULL, &trans);
1485         if (rc != 0) {
1486                 CERROR("Can't create transaction: %d\n", rc);
1487                 return rc;
1488         }
1489
1490         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1491
1492         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1493                                           lstcon_statrpc_readent);
1494         lstcon_rpc_trans_destroy(trans);
1495
1496         return rc;
1497 }
1498
1499 int
1500 lstcon_group_stat(char *grp_name, int timeout, struct list_head *result_up)
1501 {
1502         lstcon_group_t *grp;
1503         int rc;
1504
1505         rc = lstcon_group_find(grp_name, &grp);
1506         if (rc != 0) {
1507                 CDEBUG(D_NET, "Can't find group %s\n", grp_name);
1508                 return rc;
1509         }
1510
1511         rc = lstcon_ndlist_stat(&grp->grp_ndl_list, timeout, result_up);
1512
1513         lstcon_group_decref(grp);
1514
1515         return rc;
1516 }
1517
1518 int
1519 lstcon_nodes_stat(int count, lnet_process_id_t *ids_up,
1520                   int timeout, struct list_head *result_up)
1521 {
1522         lstcon_ndlink_t *ndl;
1523         lstcon_group_t *tmp;
1524         lnet_process_id_t id;
1525         int i;
1526         int rc;
1527
1528         rc = lstcon_group_alloc(NULL, &tmp);
1529         if (rc != 0) {
1530                 CERROR("Out of memory\n");
1531                 return -ENOMEM;
1532         }
1533
1534         for (i = 0 ; i < count; i++) {
1535                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1536                         rc = -EFAULT;
1537                         break;
1538                 }
1539
1540                 /* add to tmp group */
1541                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 2);
1542                 if (rc != 0) {
1543                         CDEBUG((rc == -ENOMEM) ? D_ERROR : D_NET,
1544                                "Failed to find or create %s: %d\n",
1545                                libcfs_id2str(id), rc);
1546                         break;
1547                 }
1548         }
1549
1550         if (rc != 0) {
1551                 lstcon_group_decref(tmp);
1552                 return rc;
1553         }
1554
1555         rc = lstcon_ndlist_stat(&tmp->grp_ndl_list, timeout, result_up);
1556
1557         lstcon_group_decref(tmp);
1558
1559         return rc;
1560 }
1561
1562 static int
1563 lstcon_debug_ndlist(struct list_head *ndlist,
1564                     struct list_head *translist,
1565                     int timeout, struct list_head *result_up)
1566 {
1567         lstcon_rpc_trans_t *trans;
1568         int              rc;
1569
1570         rc = lstcon_rpc_trans_ndlist(ndlist, translist, LST_TRANS_SESQRY,
1571                                      NULL, lstcon_sesrpc_condition, &trans);
1572         if (rc != 0) {
1573                 CERROR("Can't create transaction: %d\n", rc);
1574                 return rc;
1575         }
1576
1577         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1578
1579         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1580                                           lstcon_sesrpc_readent);
1581         lstcon_rpc_trans_destroy(trans);
1582
1583         return rc;
1584 }
1585
1586 int
1587 lstcon_session_debug(int timeout, struct list_head *result_up)
1588 {
1589         return lstcon_debug_ndlist(&console_session.ses_ndl_list,
1590                                    NULL, timeout, result_up);
1591 }
1592
1593 int
1594 lstcon_batch_debug(int timeout, char *name,
1595                    int client, struct list_head *result_up)
1596 {
1597         lstcon_batch_t *bat;
1598         int rc;
1599
1600         rc = lstcon_batch_find(name, &bat);
1601         if (rc != 0)
1602                 return -ENOENT;
1603
1604         rc = lstcon_debug_ndlist(client ? &bat->bat_cli_list :
1605                                           &bat->bat_srv_list,
1606                                  NULL, timeout, result_up);
1607
1608         return rc;
1609 }
1610
1611 int
1612 lstcon_group_debug(int timeout, char *name,
1613                    struct list_head *result_up)
1614 {
1615         lstcon_group_t *grp;
1616         int rc;
1617
1618         rc = lstcon_group_find(name, &grp);
1619         if (rc != 0)
1620                 return -ENOENT;
1621
1622         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1623                                  timeout, result_up);
1624         lstcon_group_decref(grp);
1625
1626         return rc;
1627 }
1628
1629 int
1630 lstcon_nodes_debug(int timeout,
1631                    int count, lnet_process_id_t *ids_up,
1632                    struct list_head *result_up)
1633 {
1634         lnet_process_id_t id;
1635         lstcon_ndlink_t *ndl;
1636         lstcon_group_t *grp;
1637         int i;
1638         int rc;
1639
1640         rc = lstcon_group_alloc(NULL, &grp);
1641         if (rc != 0) {
1642                 CDEBUG(D_NET, "Out of memory\n");
1643                 return rc;
1644         }
1645
1646         for (i = 0; i < count; i++) {
1647                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1648                         rc = -EFAULT;
1649                         break;
1650                 }
1651
1652                 /* node is added to tmp group */
1653                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 1);
1654                 if (rc != 0) {
1655                         CERROR("Can't create node link\n");
1656                         break;
1657                 }
1658         }
1659
1660         if (rc != 0) {
1661                 lstcon_group_decref(grp);
1662                 return rc;
1663         }
1664
1665         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1666                                  timeout, result_up);
1667
1668         lstcon_group_decref(grp);
1669
1670         return rc;
1671 }
1672
1673 int
1674 lstcon_session_match(lst_sid_t sid)
1675 {
1676         return (console_session.ses_id.ses_nid   == sid.ses_nid &&
1677                 console_session.ses_id.ses_stamp == sid.ses_stamp) ?  1 : 0;
1678 }
1679
1680 static void
1681 lstcon_new_session_id(lst_sid_t *sid)
1682 {
1683         lnet_process_id_t id;
1684
1685         LASSERT(console_session.ses_state == LST_SESSION_NONE);
1686
1687         LNetGetId(1, &id);
1688         sid->ses_nid   = id.nid;
1689         sid->ses_stamp = cfs_time_current();
1690 }
1691
1692 extern srpc_service_t lstcon_acceptor_service;
1693
1694 int
1695 lstcon_session_new(char *name, int key, unsigned feats,
1696                    int timeout, int force, lst_sid_t *sid_up)
1697 {
1698         int rc = 0;
1699         int i;
1700
1701         if (console_session.ses_state != LST_SESSION_NONE) {
1702                 /* session exists */
1703                 if (!force) {
1704                         CNETERR("Session %s already exists\n",
1705                                 console_session.ses_name);
1706                         return -EEXIST;
1707                 }
1708
1709                 rc = lstcon_session_end();
1710
1711                 /* lstcon_session_end() only return local error */
1712                 if  (rc != 0)
1713                         return rc;
1714         }
1715
1716         if ((feats & ~LST_FEATS_MASK) != 0) {
1717                 CNETERR("Unknown session features %x\n",
1718                         (feats & ~LST_FEATS_MASK));
1719                 return -EINVAL;
1720         }
1721
1722         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
1723                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
1724
1725         lstcon_new_session_id(&console_session.ses_id);
1726
1727         console_session.ses_key = key;
1728         console_session.ses_state = LST_SESSION_ACTIVE;
1729         console_session.ses_force = !!force;
1730         console_session.ses_features = feats;
1731         console_session.ses_feats_updated = 0;
1732         console_session.ses_timeout = (timeout <= 0) ?
1733                                       LST_CONSOLE_TIMEOUT : timeout;
1734         strlcpy(console_session.ses_name, name,
1735                 sizeof(console_session.ses_name));
1736
1737         rc = lstcon_batch_add(LST_DEFAULT_BATCH);
1738         if (rc != 0)
1739                 return rc;
1740
1741         rc = lstcon_rpc_pinger_start();
1742         if (rc != 0) {
1743                 lstcon_batch_t *bat = NULL;
1744
1745                 lstcon_batch_find(LST_DEFAULT_BATCH, &bat);
1746                 lstcon_batch_destroy(bat);
1747
1748                 return rc;
1749         }
1750
1751         if (copy_to_user(sid_up, &console_session.ses_id,
1752                              sizeof(lst_sid_t)) == 0)
1753                 return rc;
1754
1755         lstcon_session_end();
1756
1757         return -EFAULT;
1758 }
1759
1760 int
1761 lstcon_session_info(lst_sid_t *sid_up, int *key_up, unsigned *featp,
1762                     lstcon_ndlist_ent_t *ndinfo_up, char *name_up, int len)
1763 {
1764         lstcon_ndlist_ent_t *entp;
1765         lstcon_ndlink_t *ndl;
1766         int rc = 0;
1767
1768         if (console_session.ses_state != LST_SESSION_ACTIVE)
1769                 return -ESRCH;
1770
1771         LIBCFS_ALLOC(entp, sizeof(*entp));
1772         if (entp == NULL)
1773                 return -ENOMEM;
1774
1775         list_for_each_entry(ndl, &console_session.ses_ndl_list, ndl_link)
1776                 LST_NODE_STATE_COUNTER(ndl->ndl_node, entp);
1777
1778         if (copy_to_user(sid_up, &console_session.ses_id,
1779                              sizeof(lst_sid_t)) ||
1780             copy_to_user(key_up, &console_session.ses_key,
1781                              sizeof(*key_up)) ||
1782             copy_to_user(featp, &console_session.ses_features,
1783                              sizeof(*featp)) ||
1784             copy_to_user(ndinfo_up, entp, sizeof(*entp)) ||
1785             copy_to_user(name_up, console_session.ses_name, len))
1786                 rc = -EFAULT;
1787
1788         LIBCFS_FREE(entp, sizeof(*entp));
1789
1790         return rc;
1791 }
1792
1793 int
1794 lstcon_session_end(void)
1795 {
1796         lstcon_rpc_trans_t *trans;
1797         lstcon_group_t *grp;
1798         lstcon_batch_t *bat;
1799         int rc = 0;
1800
1801         LASSERT(console_session.ses_state == LST_SESSION_ACTIVE);
1802
1803         rc = lstcon_rpc_trans_ndlist(&console_session.ses_ndl_list,
1804                                      NULL, LST_TRANS_SESEND, NULL,
1805                                      lstcon_sesrpc_condition, &trans);
1806         if (rc != 0) {
1807                 CERROR("Can't create transaction: %d\n", rc);
1808                 return rc;
1809         }
1810
1811         console_session.ses_shutdown = 1;
1812
1813         lstcon_rpc_pinger_stop();
1814
1815         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1816
1817         lstcon_rpc_trans_destroy(trans);
1818         /* User can do nothing even rpc failed, so go on */
1819
1820         /* waiting for orphan rpcs to die */
1821         lstcon_rpc_cleanup_wait();
1822
1823         console_session.ses_id    = LST_INVALID_SID;
1824         console_session.ses_state = LST_SESSION_NONE;
1825         console_session.ses_key   = 0;
1826         console_session.ses_force = 0;
1827         console_session.ses_feats_updated = 0;
1828
1829         /* destroy all batches */
1830         while (!list_empty(&console_session.ses_bat_list)) {
1831                 bat = list_entry(console_session.ses_bat_list.next,
1832                                      lstcon_batch_t, bat_link);
1833
1834                 lstcon_batch_destroy(bat);
1835         }
1836
1837         /* destroy all groups */
1838         while (!list_empty(&console_session.ses_grp_list)) {
1839                 grp = list_entry(console_session.ses_grp_list.next,
1840                                      lstcon_group_t, grp_link);
1841                 LASSERT(grp->grp_ref == 1);
1842
1843                 lstcon_group_decref(grp);
1844         }
1845
1846         /* all nodes should be released */
1847         LASSERT(list_empty(&console_session.ses_ndl_list));
1848
1849         console_session.ses_shutdown = 0;
1850         console_session.ses_expired  = 0;
1851
1852         return rc;
1853 }
1854
1855 int
1856 lstcon_session_feats_check(unsigned feats)
1857 {
1858         int rc = 0;
1859
1860         if ((feats & ~LST_FEATS_MASK) != 0) {
1861                 CERROR("Can't support these features: %x\n",
1862                        (feats & ~LST_FEATS_MASK));
1863                 return -EPROTO;
1864         }
1865
1866         spin_lock(&console_session.ses_rpc_lock);
1867
1868         if (!console_session.ses_feats_updated) {
1869                 console_session.ses_feats_updated = 1;
1870                 console_session.ses_features = feats;
1871         }
1872
1873         if (console_session.ses_features != feats)
1874                 rc = -EPROTO;
1875
1876         spin_unlock(&console_session.ses_rpc_lock);
1877
1878         if (rc != 0) {
1879                 CERROR("remote features %x do not match with session features %x of console\n",
1880                        feats, console_session.ses_features);
1881         }
1882
1883         return rc;
1884 }
1885
1886 static int
1887 lstcon_acceptor_handle(struct srpc_server_rpc *rpc)
1888 {
1889         srpc_msg_t *rep  = &rpc->srpc_replymsg;
1890         srpc_msg_t *req  = &rpc->srpc_reqstbuf->buf_msg;
1891         srpc_join_reqst_t *jreq = &req->msg_body.join_reqst;
1892         srpc_join_reply_t *jrep = &rep->msg_body.join_reply;
1893         lstcon_group_t *grp  = NULL;
1894         lstcon_ndlink_t *ndl;
1895         int rc   = 0;
1896
1897         sfw_unpack_message(req);
1898
1899         mutex_lock(&console_session.ses_mutex);
1900
1901         jrep->join_sid = console_session.ses_id;
1902
1903         if (console_session.ses_id.ses_nid == LNET_NID_ANY) {
1904                 jrep->join_status = ESRCH;
1905                 goto out;
1906         }
1907
1908         if (lstcon_session_feats_check(req->msg_ses_feats) != 0) {
1909                 jrep->join_status = EPROTO;
1910                 goto out;
1911         }
1912
1913         if (jreq->join_sid.ses_nid != LNET_NID_ANY &&
1914              !lstcon_session_match(jreq->join_sid)) {
1915                 jrep->join_status = EBUSY;
1916                 goto out;
1917         }
1918
1919         if (lstcon_group_find(jreq->join_group, &grp) != 0) {
1920                 rc = lstcon_group_alloc(jreq->join_group, &grp);
1921                 if (rc != 0) {
1922                         CERROR("Out of memory\n");
1923                         goto out;
1924                 }
1925
1926                 list_add_tail(&grp->grp_link,
1927                                   &console_session.ses_grp_list);
1928                 lstcon_group_addref(grp);
1929         }
1930
1931         if (grp->grp_ref > 2) {
1932                 /* Group in using */
1933                 jrep->join_status = EBUSY;
1934                 goto out;
1935         }
1936
1937         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 0);
1938         if (rc == 0) {
1939                 jrep->join_status = EEXIST;
1940                 goto out;
1941         }
1942
1943         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 1);
1944         if (rc != 0) {
1945                 CERROR("Out of memory\n");
1946                 goto out;
1947         }
1948
1949         ndl->ndl_node->nd_state   = LST_NODE_ACTIVE;
1950         ndl->ndl_node->nd_timeout = console_session.ses_timeout;
1951
1952         if (grp->grp_userland == 0)
1953                 grp->grp_userland = 1;
1954
1955         strlcpy(jrep->join_session, console_session.ses_name,
1956                 sizeof(jrep->join_session));
1957         jrep->join_timeout = console_session.ses_timeout;
1958         jrep->join_status  = 0;
1959
1960 out:
1961         rep->msg_ses_feats = console_session.ses_features;
1962         if (grp != NULL)
1963                 lstcon_group_decref(grp);
1964
1965         mutex_unlock(&console_session.ses_mutex);
1966
1967         return rc;
1968 }
1969
1970 srpc_service_t lstcon_acceptor_service;
1971 static void lstcon_init_acceptor_service(void)
1972 {
1973         /* initialize selftest console acceptor service table */
1974         lstcon_acceptor_service.sv_name     = "join session";
1975         lstcon_acceptor_service.sv_handler  = lstcon_acceptor_handle;
1976         lstcon_acceptor_service.sv_id       = SRPC_SERVICE_JOIN;
1977         lstcon_acceptor_service.sv_wi_total = SFW_FRWK_WI_MAX;
1978 }
1979
1980 extern int lstcon_ioctl_entry(unsigned int cmd, struct libcfs_ioctl_data *data);
1981
1982 static DECLARE_IOCTL_HANDLER(lstcon_ioctl_handler, lstcon_ioctl_entry);
1983
1984 /* initialize console */
1985 int
1986 lstcon_console_init(void)
1987 {
1988         int i;
1989         int rc;
1990
1991         memset(&console_session, 0, sizeof(lstcon_session_t));
1992
1993         console_session.ses_id            = LST_INVALID_SID;
1994         console_session.ses_state         = LST_SESSION_NONE;
1995         console_session.ses_timeout       = 0;
1996         console_session.ses_force         = 0;
1997         console_session.ses_expired       = 0;
1998         console_session.ses_feats_updated = 0;
1999         console_session.ses_features      = LST_FEATS_MASK;
2000         console_session.ses_laststamp     = ktime_get_real_seconds();
2001
2002         mutex_init(&console_session.ses_mutex);
2003
2004         INIT_LIST_HEAD(&console_session.ses_ndl_list);
2005         INIT_LIST_HEAD(&console_session.ses_grp_list);
2006         INIT_LIST_HEAD(&console_session.ses_bat_list);
2007         INIT_LIST_HEAD(&console_session.ses_trans_list);
2008
2009         LIBCFS_ALLOC(console_session.ses_ndl_hash,
2010                      sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2011         if (console_session.ses_ndl_hash == NULL)
2012                 return -ENOMEM;
2013
2014         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
2015                 INIT_LIST_HEAD(&console_session.ses_ndl_hash[i]);
2016
2017         /* initialize acceptor service table */
2018         lstcon_init_acceptor_service();
2019
2020         rc = srpc_add_service(&lstcon_acceptor_service);
2021         LASSERT(rc != -EBUSY);
2022         if (rc != 0) {
2023                 LIBCFS_FREE(console_session.ses_ndl_hash,
2024                             sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2025                 return rc;
2026         }
2027
2028         rc = srpc_service_add_buffers(&lstcon_acceptor_service,
2029                                       lstcon_acceptor_service.sv_wi_total);
2030         if (rc != 0) {
2031                 rc = -ENOMEM;
2032                 goto out;
2033         }
2034
2035         rc = libcfs_register_ioctl(&lstcon_ioctl_handler);
2036
2037         if (rc == 0) {
2038                 lstcon_rpc_module_init();
2039                 return 0;
2040         }
2041
2042 out:
2043         srpc_shutdown_service(&lstcon_acceptor_service);
2044         srpc_remove_service(&lstcon_acceptor_service);
2045
2046         LIBCFS_FREE(console_session.ses_ndl_hash,
2047                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2048
2049         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2050
2051         return rc;
2052 }
2053
2054 int
2055 lstcon_console_fini(void)
2056 {
2057         int i;
2058
2059         libcfs_deregister_ioctl(&lstcon_ioctl_handler);
2060
2061         mutex_lock(&console_session.ses_mutex);
2062
2063         srpc_shutdown_service(&lstcon_acceptor_service);
2064         srpc_remove_service(&lstcon_acceptor_service);
2065
2066         if (console_session.ses_state != LST_SESSION_NONE)
2067                 lstcon_session_end();
2068
2069         lstcon_rpc_module_fini();
2070
2071         mutex_unlock(&console_session.ses_mutex);
2072
2073         LASSERT(list_empty(&console_session.ses_ndl_list));
2074         LASSERT(list_empty(&console_session.ses_grp_list));
2075         LASSERT(list_empty(&console_session.ses_bat_list));
2076         LASSERT(list_empty(&console_session.ses_trans_list));
2077
2078         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
2079                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
2080         }
2081
2082         LIBCFS_FREE(console_session.ses_ndl_hash,
2083                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2084
2085         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2086
2087         return 0;
2088 }