drivers/staging/lustre/lnet/selftest/console.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  *
  30  * Copyright (c) 2012, Intel Corporation.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * lnet/selftest/conctl.c
  37  *
  38  * Infrastructure of LST console
  39  *
  40  * Author: Liang Zhen <liangzhen@clusterfs.com>
  41  */
  42
  43 #include "../../include/linux/libcfs/libcfs.h"
  44 #include "../../include/linux/lnet/lib-lnet.h"
  45 #include "console.h"
  46 #include "conrpc.h"
  47
  48 #define LST_NODE_STATE_COUNTER(nd, p)                   \
  49 do {                                                    \
  50         if ((nd)->nd_state == LST_NODE_ACTIVE)          \
  51                 (p)->nle_nactive++;                     \
  52         else if ((nd)->nd_state == LST_NODE_BUSY)       \
  53                 (p)->nle_nbusy++;                       \
  54         else if ((nd)->nd_state == LST_NODE_DOWN)       \
  55                 (p)->nle_ndown++;                       \
  56         else                                            \
  57                 (p)->nle_nunknown++;                    \
  58         (p)->nle_nnode++;                               \
  59 } while (0)
  60
  61 struct lstcon_session console_session;
  62
  63 static void
  64 lstcon_node_get(lstcon_node_t *nd)
  65 {
  66         LASSERT(nd->nd_ref >= 1);
  67
  68         nd->nd_ref++;
  69 }
  70
  71 static int
  72 lstcon_node_find(lnet_process_id_t id, lstcon_node_t **ndpp, int create)
  73 {
  74         lstcon_ndlink_t *ndl;
  75         unsigned int idx = LNET_NIDADDR(id.nid) % LST_GLOBAL_HASHSIZE;
  76
  77         LASSERT(id.nid != LNET_NID_ANY);
  78
  79         list_for_each_entry(ndl, &console_session.ses_ndl_hash[idx], ndl_hlink) {
  80                 if (ndl->ndl_node->nd_id.nid != id.nid ||
  81                     ndl->ndl_node->nd_id.pid != id.pid)
  82                         continue;
  83
  84                 lstcon_node_get(ndl->ndl_node);
  85                 *ndpp = ndl->ndl_node;
  86                 return 0;
  87         }
  88
  89         if (!create)
  90                 return -ENOENT;
  91
  92         LIBCFS_ALLOC(*ndpp, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
  93         if (!*ndpp)
  94                 return -ENOMEM;
  95
  96         ndl = (lstcon_ndlink_t *)(*ndpp + 1);
  97
  98         ndl->ndl_node = *ndpp;
  99
 100         ndl->ndl_node->nd_ref   = 1;
 101         ndl->ndl_node->nd_id    = id;
 102         ndl->ndl_node->nd_stamp = cfs_time_current();
 103         ndl->ndl_node->nd_state = LST_NODE_UNKNOWN;
 104         ndl->ndl_node->nd_timeout = 0;
 105         memset(&ndl->ndl_node->nd_ping, 0, sizeof(lstcon_rpc_t));
 106
 107         /*
 108          * queued in global hash & list, no refcount is taken by
 109          * global hash & list, if caller release his refcount,
 110          * node will be released
 111          */
 112         list_add_tail(&ndl->ndl_hlink, &console_session.ses_ndl_hash[idx]);
 113         list_add_tail(&ndl->ndl_link, &console_session.ses_ndl_list);
 114
 115         return 0;
 116 }
 117
 118 static void
 119 lstcon_node_put(lstcon_node_t *nd)
 120 {
 121         lstcon_ndlink_t *ndl;
 122
 123         LASSERT(nd->nd_ref > 0);
 124
 125         if (--nd->nd_ref > 0)
 126                 return;
 127
 128         ndl = (lstcon_ndlink_t *)(nd + 1);
 129
 130         LASSERT(!list_empty(&ndl->ndl_link));
 131         LASSERT(!list_empty(&ndl->ndl_hlink));
 132
 133         /* remove from session */
 134         list_del(&ndl->ndl_link);
 135         list_del(&ndl->ndl_hlink);
 136
 137         LIBCFS_FREE(nd, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
 138 }
 139
 140 static int
 141 lstcon_ndlink_find(struct list_head *hash,
 142                    lnet_process_id_t id, lstcon_ndlink_t **ndlpp, int create)
 143 {
 144         unsigned int idx = LNET_NIDADDR(id.nid) % LST_NODE_HASHSIZE;
 145         lstcon_ndlink_t *ndl;
 146         lstcon_node_t *nd;
 147         int rc;
 148
 149         if (id.nid == LNET_NID_ANY)
 150                 return -EINVAL;
 151
 152         /* search in hash */
 153         list_for_each_entry(ndl, &hash[idx], ndl_hlink) {
 154                 if (ndl->ndl_node->nd_id.nid != id.nid ||
 155                     ndl->ndl_node->nd_id.pid != id.pid)
 156                         continue;
 157
 158                 *ndlpp = ndl;
 159                 return 0;
 160         }
 161
 162         if (!create)
 163                 return -ENOENT;
 164
 165         /* find or create in session hash */
 166         rc = lstcon_node_find(id, &nd, (create == 1) ? 1 : 0);
 167         if (rc)
 168                 return rc;
 169
 170         LIBCFS_ALLOC(ndl, sizeof(lstcon_ndlink_t));
 171         if (!ndl) {
 172                 lstcon_node_put(nd);
 173                 return -ENOMEM;
 174         }
 175
 176         *ndlpp = ndl;
 177
 178         ndl->ndl_node = nd;
 179         INIT_LIST_HEAD(&ndl->ndl_link);
 180         list_add_tail(&ndl->ndl_hlink, &hash[idx]);
 181
 182         return  0;
 183 }
 184
 185 static void
 186 lstcon_ndlink_release(lstcon_ndlink_t *ndl)
 187 {
 188         LASSERT(list_empty(&ndl->ndl_link));
 189         LASSERT(!list_empty(&ndl->ndl_hlink));
 190
 191         list_del(&ndl->ndl_hlink); /* delete from hash */
 192         lstcon_node_put(ndl->ndl_node);
 193
 194         LIBCFS_FREE(ndl, sizeof(*ndl));
 195 }
 196
 197 static int
 198 lstcon_group_alloc(char *name, lstcon_group_t **grpp)
 199 {
 200         lstcon_group_t *grp;
 201         int i;
 202
 203         LIBCFS_ALLOC(grp, offsetof(lstcon_group_t,
 204                                    grp_ndl_hash[LST_NODE_HASHSIZE]));
 205         if (!grp)
 206                 return -ENOMEM;
 207
 208         grp->grp_ref = 1;
 209         if (name)
 210                 strcpy(grp->grp_name, name);
 211
 212         INIT_LIST_HEAD(&grp->grp_link);
 213         INIT_LIST_HEAD(&grp->grp_ndl_list);
 214         INIT_LIST_HEAD(&grp->grp_trans_list);
 215
 216         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 217                 INIT_LIST_HEAD(&grp->grp_ndl_hash[i]);
 218
 219         *grpp = grp;
 220
 221         return 0;
 222 }
 223
 224 static void
 225 lstcon_group_addref(lstcon_group_t *grp)
 226 {
 227         grp->grp_ref++;
 228 }
 229
 230 static void lstcon_group_ndlink_release(lstcon_group_t *, lstcon_ndlink_t *);
 231
 232 static void
 233 lstcon_group_drain(lstcon_group_t *grp, int keep)
 234 {
 235         lstcon_ndlink_t *ndl;
 236         lstcon_ndlink_t *tmp;
 237
 238         list_for_each_entry_safe(ndl, tmp, &grp->grp_ndl_list, ndl_link) {
 239                 if (!(ndl->ndl_node->nd_state & keep))
 240                         lstcon_group_ndlink_release(grp, ndl);
 241         }
 242 }
 243
 244 static void
 245 lstcon_group_decref(lstcon_group_t *grp)
 246 {
 247         int i;
 248
 249         if (--grp->grp_ref > 0)
 250                 return;
 251
 252         if (!list_empty(&grp->grp_link))
 253                 list_del(&grp->grp_link);
 254
 255         lstcon_group_drain(grp, 0);
 256
 257         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 258                 LASSERT(list_empty(&grp->grp_ndl_hash[i]));
 259
 260         LIBCFS_FREE(grp, offsetof(lstcon_group_t,
 261                                   grp_ndl_hash[LST_NODE_HASHSIZE]));
 262 }
 263
 264 static int
 265 lstcon_group_find(const char *name, lstcon_group_t **grpp)
 266 {
 267         lstcon_group_t *grp;
 268
 269         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 270                 if (strncmp(grp->grp_name, name, LST_NAME_SIZE))
 271                         continue;
 272
 273                 lstcon_group_addref(grp);  /* +1 ref for caller */
 274                 *grpp = grp;
 275                 return 0;
 276         }
 277
 278         return -ENOENT;
 279 }
 280
 281 static int
 282 lstcon_group_ndlink_find(lstcon_group_t *grp, lnet_process_id_t id,
 283                          lstcon_ndlink_t **ndlpp, int create)
 284 {
 285         int rc;
 286
 287         rc = lstcon_ndlink_find(&grp->grp_ndl_hash[0], id, ndlpp, create);
 288         if (rc)
 289                 return rc;
 290
 291         if (!list_empty(&(*ndlpp)->ndl_link))
 292                 return 0;
 293
 294         list_add_tail(&(*ndlpp)->ndl_link, &grp->grp_ndl_list);
 295         grp->grp_nnode++;
 296
 297         return 0;
 298 }
 299
 300 static void
 301 lstcon_group_ndlink_release(lstcon_group_t *grp, lstcon_ndlink_t *ndl)
 302 {
 303         list_del_init(&ndl->ndl_link);
 304         lstcon_ndlink_release(ndl);
 305         grp->grp_nnode--;
 306 }
 307
 308 static void
 309 lstcon_group_ndlink_move(lstcon_group_t *old,
 310                          lstcon_group_t *new, lstcon_ndlink_t *ndl)
 311 {
 312         unsigned int idx = LNET_NIDADDR(ndl->ndl_node->nd_id.nid) %
 313                            LST_NODE_HASHSIZE;
 314
 315         list_del(&ndl->ndl_hlink);
 316         list_del(&ndl->ndl_link);
 317         old->grp_nnode--;
 318
 319         list_add_tail(&ndl->ndl_hlink, &new->grp_ndl_hash[idx]);
 320         list_add_tail(&ndl->ndl_link, &new->grp_ndl_list);
 321         new->grp_nnode++;
 322 }
 323
 324 static void
 325 lstcon_group_move(lstcon_group_t *old, lstcon_group_t *new)
 326 {
 327         lstcon_ndlink_t *ndl;
 328
 329         while (!list_empty(&old->grp_ndl_list)) {
 330                 ndl = list_entry(old->grp_ndl_list.next,
 331                                  lstcon_ndlink_t, ndl_link);
 332                 lstcon_group_ndlink_move(old, new, ndl);
 333         }
 334 }
 335
 336 static int
 337 lstcon_sesrpc_condition(int transop, lstcon_node_t *nd, void *arg)
 338 {
 339         lstcon_group_t *grp = (lstcon_group_t *)arg;
 340
 341         switch (transop) {
 342         case LST_TRANS_SESNEW:
 343                 if (nd->nd_state == LST_NODE_ACTIVE)
 344                         return 0;
 345                 break;
 346
 347         case LST_TRANS_SESEND:
 348                 if (nd->nd_state != LST_NODE_ACTIVE)
 349                         return 0;
 350
 351                 if (grp && nd->nd_ref > 1)
 352                         return 0;
 353                 break;
 354
 355         case LST_TRANS_SESQRY:
 356                 break;
 357
 358         default:
 359                 LBUG();
 360         }
 361
 362         return 1;
 363 }
 364
 365 static int
 366 lstcon_sesrpc_readent(int transop, srpc_msg_t *msg,
 367                       lstcon_rpc_ent_t __user *ent_up)
 368 {
 369         srpc_debug_reply_t *rep;
 370
 371         switch (transop) {
 372         case LST_TRANS_SESNEW:
 373         case LST_TRANS_SESEND:
 374                 return 0;
 375
 376         case LST_TRANS_SESQRY:
 377                 rep = &msg->msg_body.dbg_reply;
 378
 379                 if (copy_to_user(&ent_up->rpe_priv[0],
 380                                  &rep->dbg_timeout, sizeof(int)) ||
 381                     copy_to_user(&ent_up->rpe_payload[0],
 382                                  &rep->dbg_name, LST_NAME_SIZE))
 383                         return -EFAULT;
 384
 385                 return 0;
 386
 387         default:
 388                 LBUG();
 389         }
 390
 391         return 0;
 392 }
 393
 394 static int
 395 lstcon_group_nodes_add(lstcon_group_t *grp,
 396                        int count, lnet_process_id_t __user *ids_up,
 397                        unsigned *featp, struct list_head __user *result_up)
 398 {
 399         lstcon_rpc_trans_t *trans;
 400         lstcon_ndlink_t *ndl;
 401         lstcon_group_t *tmp;
 402         lnet_process_id_t id;
 403         int i;
 404         int rc;
 405
 406         rc = lstcon_group_alloc(NULL, &tmp);
 407         if (rc) {
 408                 CERROR("Out of memory\n");
 409                 return -ENOMEM;
 410         }
 411
 412         for (i = 0 ; i < count; i++) {
 413                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 414                         rc = -EFAULT;
 415                         break;
 416                 }
 417
 418                 /* skip if it's in this group already */
 419                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 0);
 420                 if (!rc)
 421                         continue;
 422
 423                 /* add to tmp group */
 424                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 1);
 425                 if (rc) {
 426                         CERROR("Can't create ndlink, out of memory\n");
 427                         break;
 428                 }
 429         }
 430
 431         if (rc) {
 432                 lstcon_group_decref(tmp);
 433                 return rc;
 434         }
 435
 436         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 437                                      &tmp->grp_trans_list, LST_TRANS_SESNEW,
 438                                      tmp, lstcon_sesrpc_condition, &trans);
 439         if (rc) {
 440                 CERROR("Can't create transaction: %d\n", rc);
 441                 lstcon_group_decref(tmp);
 442                 return rc;
 443         }
 444
 445         /* post all RPCs */
 446         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 447
 448         rc = lstcon_rpc_trans_interpreter(trans, result_up,
 449                                           lstcon_sesrpc_readent);
 450         *featp = trans->tas_features;
 451
 452         /* destroy all RPGs */
 453         lstcon_rpc_trans_destroy(trans);
 454
 455         lstcon_group_move(tmp, grp);
 456         lstcon_group_decref(tmp);
 457
 458         return rc;
 459 }
 460
 461 static int
 462 lstcon_group_nodes_remove(lstcon_group_t *grp,
 463                           int count, lnet_process_id_t __user *ids_up,
 464                           struct list_head __user *result_up)
 465 {
 466         lstcon_rpc_trans_t *trans;
 467         lstcon_ndlink_t *ndl;
 468         lstcon_group_t *tmp;
 469         lnet_process_id_t id;
 470         int rc;
 471         int i;
 472
 473         /* End session and remove node from the group */
 474
 475         rc = lstcon_group_alloc(NULL, &tmp);
 476         if (rc) {
 477                 CERROR("Out of memory\n");
 478                 return -ENOMEM;
 479         }
 480
 481         for (i = 0; i < count; i++) {
 482                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 483                         rc = -EFAULT;
 484                         goto error;
 485                 }
 486
 487                 /* move node to tmp group */
 488                 if (!lstcon_group_ndlink_find(grp, id, &ndl, 0))
 489                         lstcon_group_ndlink_move(grp, tmp, ndl);
 490         }
 491
 492         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 493                                      &tmp->grp_trans_list, LST_TRANS_SESEND,
 494                                      tmp, lstcon_sesrpc_condition, &trans);
 495         if (rc) {
 496                 CERROR("Can't create transaction: %d\n", rc);
 497                 goto error;
 498         }
 499
 500         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 501
 502         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 503
 504         lstcon_rpc_trans_destroy(trans);
 505         /* release nodes anyway, because we can't rollback status */
 506         lstcon_group_decref(tmp);
 507
 508         return rc;
 509 error:
 510         lstcon_group_move(tmp, grp);
 511         lstcon_group_decref(tmp);
 512
 513         return rc;
 514 }
 515
 516 int
 517 lstcon_group_add(char *name)
 518 {
 519         lstcon_group_t *grp;
 520         int rc;
 521
 522         rc = lstcon_group_find(name, &grp) ? 0: -EEXIST;
 523         if (rc) {
 524                 /* find a group with same name */
 525                 lstcon_group_decref(grp);
 526                 return rc;
 527         }
 528
 529         rc = lstcon_group_alloc(name, &grp);
 530         if (rc) {
 531                 CERROR("Can't allocate descriptor for group %s\n", name);
 532                 return -ENOMEM;
 533         }
 534
 535         list_add_tail(&grp->grp_link, &console_session.ses_grp_list);
 536
 537         return rc;
 538 }
 539
 540 int
 541 lstcon_nodes_add(char *name, int count, lnet_process_id_t __user *ids_up,
 542                  unsigned *featp, struct list_head __user *result_up)
 543 {
 544         lstcon_group_t *grp;
 545         int rc;
 546
 547         LASSERT(count > 0);
 548         LASSERT(ids_up);
 549
 550         rc = lstcon_group_find(name, &grp);
 551         if (rc) {
 552                 CDEBUG(D_NET, "Can't find group %s\n", name);
 553                 return rc;
 554         }
 555
 556         if (grp->grp_ref > 2) {
 557                 /* referred by other threads or test */
 558                 CDEBUG(D_NET, "Group %s is busy\n", name);
 559                 lstcon_group_decref(grp);
 560
 561                 return -EBUSY;
 562         }
 563
 564         rc = lstcon_group_nodes_add(grp, count, ids_up, featp, result_up);
 565
 566         lstcon_group_decref(grp);
 567
 568         return rc;
 569 }
 570
 571 int
 572 lstcon_group_del(char *name)
 573 {
 574         lstcon_rpc_trans_t *trans;
 575         lstcon_group_t *grp;
 576         int rc;
 577
 578         rc = lstcon_group_find(name, &grp);
 579         if (rc) {
 580                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 581                 return rc;
 582         }
 583
 584         if (grp->grp_ref > 2) {
 585                 /* referred by others threads or test */
 586                 CDEBUG(D_NET, "Group %s is busy\n", name);
 587                 lstcon_group_decref(grp);
 588                 return -EBUSY;
 589         }
 590
 591         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 592                                      &grp->grp_trans_list, LST_TRANS_SESEND,
 593                                      grp, lstcon_sesrpc_condition, &trans);
 594         if (rc) {
 595                 CERROR("Can't create transaction: %d\n", rc);
 596                 lstcon_group_decref(grp);
 597                 return rc;
 598         }
 599
 600         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 601
 602         lstcon_rpc_trans_destroy(trans);
 603
 604         lstcon_group_decref(grp);
 605         /*
 606          * -ref for session, it's destroyed,
 607          * status can't be rolled back, destroy group anyway
 608          */
 609         lstcon_group_decref(grp);
 610
 611         return rc;
 612 }
 613
 614 int
 615 lstcon_group_clean(char *name, int args)
 616 {
 617         lstcon_group_t *grp = NULL;
 618         int rc;
 619
 620         rc = lstcon_group_find(name, &grp);
 621         if (rc) {
 622                 CDEBUG(D_NET, "Can't find group %s\n", name);
 623                 return rc;
 624         }
 625
 626         if (grp->grp_ref > 2) {
 627                 /* referred by test */
 628                 CDEBUG(D_NET, "Group %s is busy\n", name);
 629                 lstcon_group_decref(grp);
 630                 return -EBUSY;
 631         }
 632
 633         args = (LST_NODE_ACTIVE | LST_NODE_BUSY |
 634                 LST_NODE_DOWN | LST_NODE_UNKNOWN) & ~args;
 635
 636         lstcon_group_drain(grp, args);
 637
 638         lstcon_group_decref(grp);
 639         /* release empty group */
 640         if (list_empty(&grp->grp_ndl_list))
 641                 lstcon_group_decref(grp);
 642
 643         return 0;
 644 }
 645
 646 int
 647 lstcon_nodes_remove(char *name, int count, lnet_process_id_t __user *ids_up,
 648                     struct list_head __user *result_up)
 649 {
 650         lstcon_group_t *grp = NULL;
 651         int rc;
 652
 653         rc = lstcon_group_find(name, &grp);
 654         if (rc) {
 655                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 656                 return rc;
 657         }
 658
 659         if (grp->grp_ref > 2) {
 660                 /* referred by test */
 661                 CDEBUG(D_NET, "Group %s is busy\n", name);
 662                 lstcon_group_decref(grp);
 663                 return -EBUSY;
 664         }
 665
 666         rc = lstcon_group_nodes_remove(grp, count, ids_up, result_up);
 667
 668         lstcon_group_decref(grp);
 669         /* release empty group */
 670         if (list_empty(&grp->grp_ndl_list))
 671                 lstcon_group_decref(grp);
 672
 673         return rc;
 674 }
 675
 676 int
 677 lstcon_group_refresh(char *name, struct list_head __user *result_up)
 678 {
 679         lstcon_rpc_trans_t *trans;
 680         lstcon_group_t *grp;
 681         int rc;
 682
 683         rc = lstcon_group_find(name, &grp);
 684         if (rc) {
 685                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 686                 return rc;
 687         }
 688
 689         if (grp->grp_ref > 2) {
 690                 /* referred by test */
 691                 CDEBUG(D_NET, "Group %s is busy\n", name);
 692                 lstcon_group_decref(grp);
 693                 return -EBUSY;
 694         }
 695
 696         /* re-invite all inactive nodes int the group */
 697         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 698                                      &grp->grp_trans_list, LST_TRANS_SESNEW,
 699                                      grp, lstcon_sesrpc_condition, &trans);
 700         if (rc) {
 701                 /* local error, return */
 702                 CDEBUG(D_NET, "Can't create transaction: %d\n", rc);
 703                 lstcon_group_decref(grp);
 704                 return rc;
 705         }
 706
 707         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 708
 709         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 710
 711         lstcon_rpc_trans_destroy(trans);
 712         /* -ref for me */
 713         lstcon_group_decref(grp);
 714
 715         return rc;
 716 }
 717
 718 int
 719 lstcon_group_list(int index, int len, char __user *name_up)
 720 {
 721         lstcon_group_t *grp;
 722
 723         LASSERT(index >= 0);
 724         LASSERT(name_up);
 725
 726         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 727                 if (!index--) {
 728                         return copy_to_user(name_up, grp->grp_name, len) ?
 729                                -EFAULT : 0;
 730                 }
 731         }
 732
 733         return -ENOENT;
 734 }
 735
 736 static int
 737 lstcon_nodes_getent(struct list_head *head, int *index_p,
 738                     int *count_p, lstcon_node_ent_t __user *dents_up)
 739 {
 740         lstcon_ndlink_t *ndl;
 741         lstcon_node_t *nd;
 742         int count = 0;
 743         int index = 0;
 744
 745         LASSERT(index_p && count_p);
 746         LASSERT(dents_up);
 747         LASSERT(*index_p >= 0);
 748         LASSERT(*count_p > 0);
 749
 750         list_for_each_entry(ndl, head, ndl_link) {
 751                 if (index++ < *index_p)
 752                         continue;
 753
 754                 if (count >= *count_p)
 755                         break;
 756
 757                 nd = ndl->ndl_node;
 758                 if (copy_to_user(&dents_up[count].nde_id,
 759                                  &nd->nd_id, sizeof(nd->nd_id)) ||
 760                     copy_to_user(&dents_up[count].nde_state,
 761                                  &nd->nd_state, sizeof(nd->nd_state)))
 762                         return -EFAULT;
 763
 764                 count++;
 765         }
 766
 767         if (index <= *index_p)
 768                 return -ENOENT;
 769
 770         *count_p = count;
 771         *index_p = index;
 772
 773         return 0;
 774 }
 775
 776 int
 777 lstcon_group_info(char *name, lstcon_ndlist_ent_t __user *gents_p,
 778                   int *index_p, int *count_p,
 779                   lstcon_node_ent_t __user *dents_up)
 780 {
 781         lstcon_ndlist_ent_t *gentp;
 782         lstcon_group_t *grp;
 783         lstcon_ndlink_t *ndl;
 784         int rc;
 785
 786         rc = lstcon_group_find(name, &grp);
 787         if (rc) {
 788                 CDEBUG(D_NET, "Can't find group %s\n", name);
 789                 return rc;
 790         }
 791
 792         if (dents_up) {
 793                 /* verbose query */
 794                 rc = lstcon_nodes_getent(&grp->grp_ndl_list,
 795                                          index_p, count_p, dents_up);
 796                 lstcon_group_decref(grp);
 797
 798                 return rc;
 799         }
 800
 801         /* non-verbose query */
 802         LIBCFS_ALLOC(gentp, sizeof(lstcon_ndlist_ent_t));
 803         if (!gentp) {
 804                 CERROR("Can't allocate ndlist_ent\n");
 805                 lstcon_group_decref(grp);
 806
 807                 return -ENOMEM;
 808         }
 809
 810         list_for_each_entry(ndl, &grp->grp_ndl_list, ndl_link)
 811                 LST_NODE_STATE_COUNTER(ndl->ndl_node, gentp);
 812
 813         rc = copy_to_user(gents_p, gentp,
 814                           sizeof(lstcon_ndlist_ent_t)) ? -EFAULT : 0;
 815
 816         LIBCFS_FREE(gentp, sizeof(lstcon_ndlist_ent_t));
 817
 818         lstcon_group_decref(grp);
 819
 820         return 0;
 821 }
 822
 823 static int
 824 lstcon_batch_find(const char *name, lstcon_batch_t **batpp)
 825 {
 826         lstcon_batch_t *bat;
 827
 828         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 829                 if (!strncmp(bat->bat_name, name, LST_NAME_SIZE)) {
 830                         *batpp = bat;
 831                         return 0;
 832                 }
 833         }
 834
 835         return -ENOENT;
 836 }
 837
 838 int
 839 lstcon_batch_add(char *name)
 840 {
 841         lstcon_batch_t *bat;
 842         int i;
 843         int rc;
 844
 845         rc = !lstcon_batch_find(name, &bat) ? -EEXIST : 0;
 846         if (rc) {
 847                 CDEBUG(D_NET, "Batch %s already exists\n", name);
 848                 return rc;
 849         }
 850
 851         LIBCFS_ALLOC(bat, sizeof(lstcon_batch_t));
 852         if (!bat) {
 853                 CERROR("Can't allocate descriptor for batch %s\n", name);
 854                 return -ENOMEM;
 855         }
 856
 857         LIBCFS_ALLOC(bat->bat_cli_hash,
 858                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 859         if (!bat->bat_cli_hash) {
 860                 CERROR("Can't allocate hash for batch %s\n", name);
 861                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
 862
 863                 return -ENOMEM;
 864         }
 865
 866         LIBCFS_ALLOC(bat->bat_srv_hash,
 867                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 868         if (!bat->bat_srv_hash) {
 869                 CERROR("Can't allocate hash for batch %s\n", name);
 870                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 871                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
 872
 873                 return -ENOMEM;
 874         }
 875
 876         strcpy(bat->bat_name, name);
 877         bat->bat_hdr.tsb_index = 0;
 878         bat->bat_hdr.tsb_id.bat_id = ++console_session.ses_id_cookie;
 879
 880         bat->bat_ntest = 0;
 881         bat->bat_state = LST_BATCH_IDLE;
 882
 883         INIT_LIST_HEAD(&bat->bat_cli_list);
 884         INIT_LIST_HEAD(&bat->bat_srv_list);
 885         INIT_LIST_HEAD(&bat->bat_test_list);
 886         INIT_LIST_HEAD(&bat->bat_trans_list);
 887
 888         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 889                 INIT_LIST_HEAD(&bat->bat_cli_hash[i]);
 890                 INIT_LIST_HEAD(&bat->bat_srv_hash[i]);
 891         }
 892
 893         list_add_tail(&bat->bat_link, &console_session.ses_bat_list);
 894
 895         return rc;
 896 }
 897
 898 int
 899 lstcon_batch_list(int index, int len, char __user *name_up)
 900 {
 901         lstcon_batch_t *bat;
 902
 903         LASSERT(name_up);
 904         LASSERT(index >= 0);
 905
 906         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 907                 if (!index--) {
 908                         return copy_to_user(name_up, bat->bat_name, len) ?
 909                                -EFAULT : 0;
 910                 }
 911         }
 912
 913         return -ENOENT;
 914 }
 915
 916 int
 917 lstcon_batch_info(char *name, lstcon_test_batch_ent_t __user *ent_up,
 918                   int server, int testidx, int *index_p, int *ndent_p,
 919                   lstcon_node_ent_t __user *dents_up)
 920 {
 921         lstcon_test_batch_ent_t *entp;
 922         struct list_head *clilst;
 923         struct list_head *srvlst;
 924         lstcon_test_t *test = NULL;
 925         lstcon_batch_t *bat;
 926         lstcon_ndlink_t *ndl;
 927         int rc;
 928
 929         rc = lstcon_batch_find(name, &bat);
 930         if (rc) {
 931                 CDEBUG(D_NET, "Can't find batch %s\n", name);
 932                 return -ENOENT;
 933         }
 934
 935         if (testidx > 0) {
 936                 /* query test, test index start from 1 */
 937                 list_for_each_entry(test, &bat->bat_test_list, tes_link) {
 938                         if (testidx-- == 1)
 939                                 break;
 940                 }
 941
 942                 if (testidx > 0) {
 943                         CDEBUG(D_NET, "Can't find specified test in batch\n");
 944                         return -ENOENT;
 945                 }
 946         }
 947
 948         clilst = !test ? &bat->bat_cli_list :
 949                          &test->tes_src_grp->grp_ndl_list;
 950         srvlst = !test ? &bat->bat_srv_list :
 951                          &test->tes_dst_grp->grp_ndl_list;
 952
 953         if (dents_up) {
 954                 rc = lstcon_nodes_getent((server ? srvlst : clilst),
 955                                          index_p, ndent_p, dents_up);
 956                 return rc;
 957         }
 958
 959         /* non-verbose query */
 960         LIBCFS_ALLOC(entp, sizeof(lstcon_test_batch_ent_t));
 961         if (!entp)
 962                 return -ENOMEM;
 963
 964         if (!test) {
 965                 entp->u.tbe_batch.bae_ntest = bat->bat_ntest;
 966                 entp->u.tbe_batch.bae_state = bat->bat_state;
 967
 968         } else {
 969                 entp->u.tbe_test.tse_type   = test->tes_type;
 970                 entp->u.tbe_test.tse_loop   = test->tes_loop;
 971                 entp->u.tbe_test.tse_concur = test->tes_concur;
 972         }
 973
 974         list_for_each_entry(ndl, clilst, ndl_link)
 975                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_cli_nle);
 976
 977         list_for_each_entry(ndl, srvlst, ndl_link)
 978                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_srv_nle);
 979
 980         rc = copy_to_user(ent_up, entp,
 981                           sizeof(lstcon_test_batch_ent_t)) ? -EFAULT : 0;
 982
 983         LIBCFS_FREE(entp, sizeof(lstcon_test_batch_ent_t));
 984
 985         return rc;
 986 }
 987
 988 static int
 989 lstcon_batrpc_condition(int transop, lstcon_node_t *nd, void *arg)
 990 {
 991         switch (transop) {
 992         case LST_TRANS_TSBRUN:
 993                 if (nd->nd_state != LST_NODE_ACTIVE)
 994                         return -ENETDOWN;
 995                 break;
 996
 997         case LST_TRANS_TSBSTOP:
 998                 if (nd->nd_state != LST_NODE_ACTIVE)
 999                         return 0;
1000                 break;
1001
1002         case LST_TRANS_TSBCLIQRY:
1003         case LST_TRANS_TSBSRVQRY:
1004                 break;
1005         }
1006
1007         return 1;
1008 }
1009
1010 static int
1011 lstcon_batch_op(lstcon_batch_t *bat, int transop,
1012                 struct list_head __user *result_up)
1013 {
1014         lstcon_rpc_trans_t *trans;
1015         int rc;
1016
1017         rc = lstcon_rpc_trans_ndlist(&bat->bat_cli_list,
1018                                      &bat->bat_trans_list, transop,
1019                                      bat, lstcon_batrpc_condition, &trans);
1020         if (rc) {
1021                 CERROR("Can't create transaction: %d\n", rc);
1022                 return rc;
1023         }
1024
1025         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1026
1027         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1028
1029         lstcon_rpc_trans_destroy(trans);
1030
1031         return rc;
1032 }
1033
1034 int
1035 lstcon_batch_run(char *name, int timeout, struct list_head __user *result_up)
1036 {
1037         lstcon_batch_t *bat;
1038         int rc;
1039
1040         if (lstcon_batch_find(name, &bat)) {
1041                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1042                 return -ENOENT;
1043         }
1044
1045         bat->bat_arg = timeout;
1046
1047         rc = lstcon_batch_op(bat, LST_TRANS_TSBRUN, result_up);
1048
1049         /* mark batch as running if it's started in any node */
1050         if (lstcon_tsbop_stat_success(lstcon_trans_stat(), 0))
1051                 bat->bat_state = LST_BATCH_RUNNING;
1052
1053         return rc;
1054 }
1055
1056 int
1057 lstcon_batch_stop(char *name, int force, struct list_head __user *result_up)
1058 {
1059         lstcon_batch_t *bat;
1060         int rc;
1061
1062         if (lstcon_batch_find(name, &bat)) {
1063                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1064                 return -ENOENT;
1065         }
1066
1067         bat->bat_arg = force;
1068
1069         rc = lstcon_batch_op(bat, LST_TRANS_TSBSTOP, result_up);
1070
1071         /* mark batch as stopped if all RPCs finished */
1072         if (!lstcon_tsbop_stat_failure(lstcon_trans_stat(), 0))
1073                 bat->bat_state = LST_BATCH_IDLE;
1074
1075         return rc;
1076 }
1077
1078 static void
1079 lstcon_batch_destroy(lstcon_batch_t *bat)
1080 {
1081         lstcon_ndlink_t *ndl;
1082         lstcon_test_t *test;
1083         int i;
1084
1085         list_del(&bat->bat_link);
1086
1087         while (!list_empty(&bat->bat_test_list)) {
1088                 test = list_entry(bat->bat_test_list.next,
1089                                   lstcon_test_t, tes_link);
1090                 LASSERT(list_empty(&test->tes_trans_list));
1091
1092                 list_del(&test->tes_link);
1093
1094                 lstcon_group_decref(test->tes_src_grp);
1095                 lstcon_group_decref(test->tes_dst_grp);
1096
1097                 LIBCFS_FREE(test, offsetof(lstcon_test_t,
1098                                            tes_param[test->tes_paramlen]));
1099         }
1100
1101         LASSERT(list_empty(&bat->bat_trans_list));
1102
1103         while (!list_empty(&bat->bat_cli_list)) {
1104                 ndl = list_entry(bat->bat_cli_list.next,
1105                                  lstcon_ndlink_t, ndl_link);
1106                 list_del_init(&ndl->ndl_link);
1107
1108                 lstcon_ndlink_release(ndl);
1109         }
1110
1111         while (!list_empty(&bat->bat_srv_list)) {
1112                 ndl = list_entry(bat->bat_srv_list.next,
1113                                  lstcon_ndlink_t, ndl_link);
1114                 list_del_init(&ndl->ndl_link);
1115
1116                 lstcon_ndlink_release(ndl);
1117         }
1118
1119         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
1120                 LASSERT(list_empty(&bat->bat_cli_hash[i]));
1121                 LASSERT(list_empty(&bat->bat_srv_hash[i]));
1122         }
1123
1124         LIBCFS_FREE(bat->bat_cli_hash,
1125                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1126         LIBCFS_FREE(bat->bat_srv_hash,
1127                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1128         LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
1129 }
1130
1131 static int
1132 lstcon_testrpc_condition(int transop, lstcon_node_t *nd, void *arg)
1133 {
1134         lstcon_test_t *test;
1135         lstcon_batch_t *batch;
1136         lstcon_ndlink_t *ndl;
1137         struct list_head *hash;
1138         struct list_head *head;
1139
1140         test = (lstcon_test_t *)arg;
1141         LASSERT(test);
1142
1143         batch = test->tes_batch;
1144         LASSERT(batch);
1145
1146         if (test->tes_oneside &&
1147             transop == LST_TRANS_TSBSRVADD)
1148                 return 0;
1149
1150         if (nd->nd_state != LST_NODE_ACTIVE)
1151                 return -ENETDOWN;
1152
1153         if (transop == LST_TRANS_TSBCLIADD) {
1154                 hash = batch->bat_cli_hash;
1155                 head = &batch->bat_cli_list;
1156
1157         } else {
1158                 LASSERT(transop == LST_TRANS_TSBSRVADD);
1159
1160                 hash = batch->bat_srv_hash;
1161                 head = &batch->bat_srv_list;
1162         }
1163
1164         LASSERT(nd->nd_id.nid != LNET_NID_ANY);
1165
1166         if (lstcon_ndlink_find(hash, nd->nd_id, &ndl, 1))
1167                 return -ENOMEM;
1168
1169         if (list_empty(&ndl->ndl_link))
1170                 list_add_tail(&ndl->ndl_link, head);
1171
1172         return 1;
1173 }
1174
1175 static int
1176 lstcon_test_nodes_add(lstcon_test_t *test, struct list_head __user *result_up)
1177 {
1178         lstcon_rpc_trans_t *trans;
1179         lstcon_group_t *grp;
1180         int transop;
1181         int rc;
1182
1183         LASSERT(test->tes_src_grp);
1184         LASSERT(test->tes_dst_grp);
1185
1186         transop = LST_TRANS_TSBSRVADD;
1187         grp  = test->tes_dst_grp;
1188 again:
1189         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
1190                                      &test->tes_trans_list, transop,
1191                                      test, lstcon_testrpc_condition, &trans);
1192         if (rc) {
1193                 CERROR("Can't create transaction: %d\n", rc);
1194                 return rc;
1195         }
1196
1197         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1198
1199         if (lstcon_trans_stat()->trs_rpc_errno ||
1200             lstcon_trans_stat()->trs_fwk_errno) {
1201                 lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1202
1203                 lstcon_rpc_trans_destroy(trans);
1204                 /* return if any error */
1205                 CDEBUG(D_NET, "Failed to add test %s, RPC error %d, framework error %d\n",
1206                        transop == LST_TRANS_TSBCLIADD ? "client" : "server",
1207                        lstcon_trans_stat()->trs_rpc_errno,
1208                        lstcon_trans_stat()->trs_fwk_errno);
1209
1210                 return rc;
1211         }
1212
1213         lstcon_rpc_trans_destroy(trans);
1214
1215         if (transop == LST_TRANS_TSBCLIADD)
1216                 return rc;
1217
1218         transop = LST_TRANS_TSBCLIADD;
1219         grp = test->tes_src_grp;
1220         test->tes_cliidx = 0;
1221
1222         /* requests to test clients */
1223         goto again;
1224 }
1225
1226 static int
1227 lstcon_verify_batch(const char *name, lstcon_batch_t **batch)
1228 {
1229         int rc;
1230
1231         rc = lstcon_batch_find(name, batch);
1232         if (rc) {
1233                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1234                 return rc;
1235         }
1236
1237         if ((*batch)->bat_state != LST_BATCH_IDLE) {
1238                 CDEBUG(D_NET, "Can't change running batch %s\n", name);
1239                 return -EINVAL;
1240         }
1241
1242         return 0;
1243 }
1244
1245 static int
1246 lstcon_verify_group(const char *name, lstcon_group_t **grp)
1247 {
1248         int rc;
1249         lstcon_ndlink_t *ndl;
1250
1251         rc = lstcon_group_find(name, grp);
1252         if (rc) {
1253                 CDEBUG(D_NET, "can't find group %s\n", name);
1254                 return rc;
1255         }
1256
1257         list_for_each_entry(ndl, &(*grp)->grp_ndl_list, ndl_link) {
1258                 if (ndl->ndl_node->nd_state == LST_NODE_ACTIVE)
1259                         return 0;
1260         }
1261
1262         CDEBUG(D_NET, "Group %s has no ACTIVE nodes\n", name);
1263
1264         return -EINVAL;
1265 }
1266
1267 int
1268 lstcon_test_add(char *batch_name, int type, int loop,
1269                 int concur, int dist, int span,
1270                 char *src_name, char *dst_name,
1271                 void *param, int paramlen, int *retp,
1272                 struct list_head __user *result_up)
1273 {
1274         lstcon_test_t    *test   = NULL;
1275         int              rc;
1276         lstcon_group_t   *src_grp = NULL;
1277         lstcon_group_t   *dst_grp = NULL;
1278         lstcon_batch_t   *batch = NULL;
1279
1280         /*
1281          * verify that a batch of the given name exists, and the groups
1282          * that will be part of the batch exist and have at least one
1283          * active node
1284          */
1285         rc = lstcon_verify_batch(batch_name, &batch);
1286         if (rc)
1287                 goto out;
1288
1289         rc = lstcon_verify_group(src_name, &src_grp);
1290         if (rc)
1291                 goto out;
1292
1293         rc = lstcon_verify_group(dst_name, &dst_grp);
1294         if (rc)
1295                 goto out;
1296
1297         if (dst_grp->grp_userland)
1298                 *retp = 1;
1299
1300         LIBCFS_ALLOC(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1301         if (!test) {
1302                 CERROR("Can't allocate test descriptor\n");
1303                 rc = -ENOMEM;
1304
1305                 goto out;
1306         }
1307
1308         test->tes_hdr.tsb_id    = batch->bat_hdr.tsb_id;
1309         test->tes_batch         = batch;
1310         test->tes_type          = type;
1311         test->tes_oneside       = 0; /* TODO */
1312         test->tes_loop          = loop;
1313         test->tes_concur        = concur;
1314         test->tes_stop_onerr    = 1; /* TODO */
1315         test->tes_span          = span;
1316         test->tes_dist          = dist;
1317         test->tes_cliidx        = 0; /* just used for creating RPC */
1318         test->tes_src_grp       = src_grp;
1319         test->tes_dst_grp       = dst_grp;
1320         INIT_LIST_HEAD(&test->tes_trans_list);
1321
1322         if (param) {
1323                 test->tes_paramlen = paramlen;
1324                 memcpy(&test->tes_param[0], param, paramlen);
1325         }
1326
1327         rc = lstcon_test_nodes_add(test, result_up);
1328
1329         if (rc)
1330                 goto out;
1331
1332         if (lstcon_trans_stat()->trs_rpc_errno ||
1333             lstcon_trans_stat()->trs_fwk_errno)
1334                 CDEBUG(D_NET, "Failed to add test %d to batch %s\n", type,
1335                        batch_name);
1336
1337         /* add to test list anyway, so user can check what's going on */
1338         list_add_tail(&test->tes_link, &batch->bat_test_list);
1339
1340         batch->bat_ntest++;
1341         test->tes_hdr.tsb_index = batch->bat_ntest;
1342
1343         /*  hold groups so nobody can change them */
1344         return rc;
1345 out:
1346         if (test)
1347                 LIBCFS_FREE(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1348
1349         if (dst_grp)
1350                 lstcon_group_decref(dst_grp);
1351
1352         if (src_grp)
1353                 lstcon_group_decref(src_grp);
1354
1355         return rc;
1356 }
1357
1358 static int
1359 lstcon_test_find(lstcon_batch_t *batch, int idx, lstcon_test_t **testpp)
1360 {
1361         lstcon_test_t *test;
1362
1363         list_for_each_entry(test, &batch->bat_test_list, tes_link) {
1364                 if (idx == test->tes_hdr.tsb_index) {
1365                         *testpp = test;
1366                         return 0;
1367                 }
1368         }
1369
1370         return -ENOENT;
1371 }
1372
1373 static int
1374 lstcon_tsbrpc_readent(int transop, srpc_msg_t *msg,
1375                       lstcon_rpc_ent_t __user *ent_up)
1376 {
1377         srpc_batch_reply_t *rep = &msg->msg_body.bat_reply;
1378
1379         LASSERT(transop == LST_TRANS_TSBCLIQRY ||
1380                 transop == LST_TRANS_TSBSRVQRY);
1381
1382         /* positive errno, framework error code */
1383         if (copy_to_user(&ent_up->rpe_priv[0], &rep->bar_active,
1384                          sizeof(rep->bar_active)))
1385                 return -EFAULT;
1386
1387         return 0;
1388 }
1389
1390 int
1391 lstcon_test_batch_query(char *name, int testidx, int client,
1392                         int timeout, struct list_head __user *result_up)
1393 {
1394         lstcon_rpc_trans_t *trans;
1395         struct list_head *translist;
1396         struct list_head *ndlist;
1397         lstcon_tsb_hdr_t *hdr;
1398         lstcon_batch_t *batch;
1399         lstcon_test_t *test = NULL;
1400         int transop;
1401         int rc;
1402
1403         rc = lstcon_batch_find(name, &batch);
1404         if (rc) {
1405                 CDEBUG(D_NET, "Can't find batch: %s\n", name);
1406                 return rc;
1407         }
1408
1409         if (!testidx) {
1410                 translist = &batch->bat_trans_list;
1411                 ndlist    = &batch->bat_cli_list;
1412                 hdr       = &batch->bat_hdr;
1413
1414         } else {
1415                 /* query specified test only */
1416                 rc = lstcon_test_find(batch, testidx, &test);
1417                 if (rc) {
1418                         CDEBUG(D_NET, "Can't find test: %d\n", testidx);
1419                         return rc;
1420                 }
1421
1422                 translist = &test->tes_trans_list;
1423                 ndlist    = &test->tes_src_grp->grp_ndl_list;
1424                 hdr       = &test->tes_hdr;
1425         }
1426
1427         transop = client ? LST_TRANS_TSBCLIQRY : LST_TRANS_TSBSRVQRY;
1428
1429         rc = lstcon_rpc_trans_ndlist(ndlist, translist, transop, hdr,
1430                                      lstcon_batrpc_condition, &trans);
1431         if (rc) {
1432                 CERROR("Can't create transaction: %d\n", rc);
1433                 return rc;
1434         }
1435
1436         lstcon_rpc_trans_postwait(trans, timeout);
1437
1438         if (!testidx && /* query a batch, not a test */
1439             !lstcon_rpc_stat_failure(lstcon_trans_stat(), 0) &&
1440             !lstcon_tsbqry_stat_run(lstcon_trans_stat(), 0)) {
1441                 /* all RPCs finished, and no active test */
1442                 batch->bat_state = LST_BATCH_IDLE;
1443         }
1444
1445         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1446                                           lstcon_tsbrpc_readent);
1447         lstcon_rpc_trans_destroy(trans);
1448
1449         return rc;
1450 }
1451
1452 static int
1453 lstcon_statrpc_readent(int transop, srpc_msg_t *msg,
1454                        lstcon_rpc_ent_t __user *ent_up)
1455 {
1456         srpc_stat_reply_t *rep = &msg->msg_body.stat_reply;
1457         sfw_counters_t __user *sfwk_stat;
1458         srpc_counters_t __user *srpc_stat;
1459         lnet_counters_t __user *lnet_stat;
1460
1461         if (rep->str_status)
1462                 return 0;
1463
1464         sfwk_stat = (sfw_counters_t __user *)&ent_up->rpe_payload[0];
1465         srpc_stat = (srpc_counters_t __user *)(sfwk_stat + 1);
1466         lnet_stat = (lnet_counters_t __user *)(srpc_stat + 1);
1467
1468         if (copy_to_user(sfwk_stat, &rep->str_fw, sizeof(*sfwk_stat)) ||
1469             copy_to_user(srpc_stat, &rep->str_rpc, sizeof(*srpc_stat)) ||
1470             copy_to_user(lnet_stat, &rep->str_lnet, sizeof(*lnet_stat)))
1471                 return -EFAULT;
1472
1473         return 0;
1474 }
1475
1476 static int
1477 lstcon_ndlist_stat(struct list_head *ndlist,
1478                    int timeout, struct list_head __user *result_up)
1479 {
1480         struct list_head head;
1481         lstcon_rpc_trans_t *trans;
1482         int rc;
1483
1484         INIT_LIST_HEAD(&head);
1485
1486         rc = lstcon_rpc_trans_ndlist(ndlist, &head,
1487                                      LST_TRANS_STATQRY, NULL, NULL, &trans);
1488         if (rc) {
1489                 CERROR("Can't create transaction: %d\n", rc);
1490                 return rc;
1491         }
1492
1493         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1494
1495         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1496                                           lstcon_statrpc_readent);
1497         lstcon_rpc_trans_destroy(trans);
1498
1499         return rc;
1500 }
1501
1502 int
1503 lstcon_group_stat(char *grp_name, int timeout,
1504                   struct list_head __user *result_up)
1505 {
1506         lstcon_group_t *grp;
1507         int rc;
1508
1509         rc = lstcon_group_find(grp_name, &grp);
1510         if (rc) {
1511                 CDEBUG(D_NET, "Can't find group %s\n", grp_name);
1512                 return rc;
1513         }
1514
1515         rc = lstcon_ndlist_stat(&grp->grp_ndl_list, timeout, result_up);
1516
1517         lstcon_group_decref(grp);
1518
1519         return rc;
1520 }
1521
1522 int
1523 lstcon_nodes_stat(int count, lnet_process_id_t __user *ids_up,
1524                   int timeout, struct list_head __user *result_up)
1525 {
1526         lstcon_ndlink_t *ndl;
1527         lstcon_group_t *tmp;
1528         lnet_process_id_t id;
1529         int i;
1530         int rc;
1531
1532         rc = lstcon_group_alloc(NULL, &tmp);
1533         if (rc) {
1534                 CERROR("Out of memory\n");
1535                 return -ENOMEM;
1536         }
1537
1538         for (i = 0 ; i < count; i++) {
1539                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1540                         rc = -EFAULT;
1541                         break;
1542                 }
1543
1544                 /* add to tmp group */
1545                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 2);
1546                 if (rc) {
1547                         CDEBUG((rc == -ENOMEM) ? D_ERROR : D_NET,
1548                                "Failed to find or create %s: %d\n",
1549                                libcfs_id2str(id), rc);
1550                         break;
1551                 }
1552         }
1553
1554         if (rc) {
1555                 lstcon_group_decref(tmp);
1556                 return rc;
1557         }
1558
1559         rc = lstcon_ndlist_stat(&tmp->grp_ndl_list, timeout, result_up);
1560
1561         lstcon_group_decref(tmp);
1562
1563         return rc;
1564 }
1565
1566 static int
1567 lstcon_debug_ndlist(struct list_head *ndlist,
1568                     struct list_head *translist,
1569                     int timeout, struct list_head __user *result_up)
1570 {
1571         lstcon_rpc_trans_t *trans;
1572         int              rc;
1573
1574         rc = lstcon_rpc_trans_ndlist(ndlist, translist, LST_TRANS_SESQRY,
1575                                      NULL, lstcon_sesrpc_condition, &trans);
1576         if (rc) {
1577                 CERROR("Can't create transaction: %d\n", rc);
1578                 return rc;
1579         }
1580
1581         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1582
1583         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1584                                           lstcon_sesrpc_readent);
1585         lstcon_rpc_trans_destroy(trans);
1586
1587         return rc;
1588 }
1589
1590 int
1591 lstcon_session_debug(int timeout, struct list_head __user *result_up)
1592 {
1593         return lstcon_debug_ndlist(&console_session.ses_ndl_list,
1594                                    NULL, timeout, result_up);
1595 }
1596
1597 int
1598 lstcon_batch_debug(int timeout, char *name,
1599                    int client, struct list_head __user *result_up)
1600 {
1601         lstcon_batch_t *bat;
1602         int rc;
1603
1604         rc = lstcon_batch_find(name, &bat);
1605         if (rc)
1606                 return -ENOENT;
1607
1608         rc = lstcon_debug_ndlist(client ? &bat->bat_cli_list :
1609                                           &bat->bat_srv_list,
1610                                  NULL, timeout, result_up);
1611
1612         return rc;
1613 }
1614
1615 int
1616 lstcon_group_debug(int timeout, char *name,
1617                    struct list_head __user *result_up)
1618 {
1619         lstcon_group_t *grp;
1620         int rc;
1621
1622         rc = lstcon_group_find(name, &grp);
1623         if (rc)
1624                 return -ENOENT;
1625
1626         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1627                                  timeout, result_up);
1628         lstcon_group_decref(grp);
1629
1630         return rc;
1631 }
1632
1633 int
1634 lstcon_nodes_debug(int timeout,
1635                    int count, lnet_process_id_t __user *ids_up,
1636                    struct list_head __user *result_up)
1637 {
1638         lnet_process_id_t id;
1639         lstcon_ndlink_t *ndl;
1640         lstcon_group_t *grp;
1641         int i;
1642         int rc;
1643
1644         rc = lstcon_group_alloc(NULL, &grp);
1645         if (rc) {
1646                 CDEBUG(D_NET, "Out of memory\n");
1647                 return rc;
1648         }
1649
1650         for (i = 0; i < count; i++) {
1651                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1652                         rc = -EFAULT;
1653                         break;
1654                 }
1655
1656                 /* node is added to tmp group */
1657                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 1);
1658                 if (rc) {
1659                         CERROR("Can't create node link\n");
1660                         break;
1661                 }
1662         }
1663
1664         if (rc) {
1665                 lstcon_group_decref(grp);
1666                 return rc;
1667         }
1668
1669         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1670                                  timeout, result_up);
1671
1672         lstcon_group_decref(grp);
1673
1674         return rc;
1675 }
1676
1677 int
1678 lstcon_session_match(lst_sid_t sid)
1679 {
1680         return (console_session.ses_id.ses_nid   == sid.ses_nid &&
1681                 console_session.ses_id.ses_stamp == sid.ses_stamp) ?  1 : 0;
1682 }
1683
1684 static void
1685 lstcon_new_session_id(lst_sid_t *sid)
1686 {
1687         lnet_process_id_t id;
1688
1689         LASSERT(console_session.ses_state == LST_SESSION_NONE);
1690
1691         LNetGetId(1, &id);
1692         sid->ses_nid   = id.nid;
1693         sid->ses_stamp = cfs_time_current();
1694 }
1695
1696 extern srpc_service_t lstcon_acceptor_service;
1697
1698 int
1699 lstcon_session_new(char *name, int key, unsigned feats,
1700                    int timeout, int force, lst_sid_t __user *sid_up)
1701 {
1702         int rc = 0;
1703         int i;
1704
1705         if (console_session.ses_state != LST_SESSION_NONE) {
1706                 /* session exists */
1707                 if (!force) {
1708                         CNETERR("Session %s already exists\n",
1709                                 console_session.ses_name);
1710                         return -EEXIST;
1711                 }
1712
1713                 rc = lstcon_session_end();
1714
1715                 /* lstcon_session_end() only return local error */
1716                 if  (rc)
1717                         return rc;
1718         }
1719
1720         if (feats & ~LST_FEATS_MASK) {
1721                 CNETERR("Unknown session features %x\n",
1722                         (feats & ~LST_FEATS_MASK));
1723                 return -EINVAL;
1724         }
1725
1726         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
1727                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
1728
1729         lstcon_new_session_id(&console_session.ses_id);
1730
1731         console_session.ses_key = key;
1732         console_session.ses_state = LST_SESSION_ACTIVE;
1733         console_session.ses_force = !!force;
1734         console_session.ses_features = feats;
1735         console_session.ses_feats_updated = 0;
1736         console_session.ses_timeout = (timeout <= 0) ?
1737                                       LST_CONSOLE_TIMEOUT : timeout;
1738         strlcpy(console_session.ses_name, name,
1739                 sizeof(console_session.ses_name));
1740
1741         rc = lstcon_batch_add(LST_DEFAULT_BATCH);
1742         if (rc)
1743                 return rc;
1744
1745         rc = lstcon_rpc_pinger_start();
1746         if (rc) {
1747                 lstcon_batch_t *bat = NULL;
1748
1749                 lstcon_batch_find(LST_DEFAULT_BATCH, &bat);
1750                 lstcon_batch_destroy(bat);
1751
1752                 return rc;
1753         }
1754
1755         if (!copy_to_user(sid_up, &console_session.ses_id,
1756                           sizeof(lst_sid_t)))
1757                 return rc;
1758
1759         lstcon_session_end();
1760
1761         return -EFAULT;
1762 }
1763
1764 int
1765 lstcon_session_info(lst_sid_t __user *sid_up, int __user *key_up,
1766                     unsigned __user *featp,
1767                     lstcon_ndlist_ent_t __user *ndinfo_up,
1768                     char __user *name_up, int len)
1769 {
1770         lstcon_ndlist_ent_t *entp;
1771         lstcon_ndlink_t *ndl;
1772         int rc = 0;
1773
1774         if (console_session.ses_state != LST_SESSION_ACTIVE)
1775                 return -ESRCH;
1776
1777         LIBCFS_ALLOC(entp, sizeof(*entp));
1778         if (!entp)
1779                 return -ENOMEM;
1780
1781         list_for_each_entry(ndl, &console_session.ses_ndl_list, ndl_link)
1782                 LST_NODE_STATE_COUNTER(ndl->ndl_node, entp);
1783
1784         if (copy_to_user(sid_up, &console_session.ses_id,
1785                          sizeof(lst_sid_t)) ||
1786             copy_to_user(key_up, &console_session.ses_key,
1787                          sizeof(*key_up)) ||
1788             copy_to_user(featp, &console_session.ses_features,
1789                          sizeof(*featp)) ||
1790             copy_to_user(ndinfo_up, entp, sizeof(*entp)) ||
1791             copy_to_user(name_up, console_session.ses_name, len))
1792                 rc = -EFAULT;
1793
1794         LIBCFS_FREE(entp, sizeof(*entp));
1795
1796         return rc;
1797 }
1798
1799 int
1800 lstcon_session_end(void)
1801 {
1802         lstcon_rpc_trans_t *trans;
1803         lstcon_group_t *grp;
1804         lstcon_batch_t *bat;
1805         int rc = 0;
1806
1807         LASSERT(console_session.ses_state == LST_SESSION_ACTIVE);
1808
1809         rc = lstcon_rpc_trans_ndlist(&console_session.ses_ndl_list,
1810                                      NULL, LST_TRANS_SESEND, NULL,
1811                                      lstcon_sesrpc_condition, &trans);
1812         if (rc) {
1813                 CERROR("Can't create transaction: %d\n", rc);
1814                 return rc;
1815         }
1816
1817         console_session.ses_shutdown = 1;
1818
1819         lstcon_rpc_pinger_stop();
1820
1821         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1822
1823         lstcon_rpc_trans_destroy(trans);
1824         /* User can do nothing even rpc failed, so go on */
1825
1826         /* waiting for orphan rpcs to die */
1827         lstcon_rpc_cleanup_wait();
1828
1829         console_session.ses_id    = LST_INVALID_SID;
1830         console_session.ses_state = LST_SESSION_NONE;
1831         console_session.ses_key   = 0;
1832         console_session.ses_force = 0;
1833         console_session.ses_feats_updated = 0;
1834
1835         /* destroy all batches */
1836         while (!list_empty(&console_session.ses_bat_list)) {
1837                 bat = list_entry(console_session.ses_bat_list.next,
1838                                  lstcon_batch_t, bat_link);
1839
1840                 lstcon_batch_destroy(bat);
1841         }
1842
1843         /* destroy all groups */
1844         while (!list_empty(&console_session.ses_grp_list)) {
1845                 grp = list_entry(console_session.ses_grp_list.next,
1846                                  lstcon_group_t, grp_link);
1847                 LASSERT(grp->grp_ref == 1);
1848
1849                 lstcon_group_decref(grp);
1850         }
1851
1852         /* all nodes should be released */
1853         LASSERT(list_empty(&console_session.ses_ndl_list));
1854
1855         console_session.ses_shutdown = 0;
1856         console_session.ses_expired  = 0;
1857
1858         return rc;
1859 }
1860
1861 int
1862 lstcon_session_feats_check(unsigned feats)
1863 {
1864         int rc = 0;
1865
1866         if (feats & ~LST_FEATS_MASK) {
1867                 CERROR("Can't support these features: %x\n",
1868                        (feats & ~LST_FEATS_MASK));
1869                 return -EPROTO;
1870         }
1871
1872         spin_lock(&console_session.ses_rpc_lock);
1873
1874         if (!console_session.ses_feats_updated) {
1875                 console_session.ses_feats_updated = 1;
1876                 console_session.ses_features = feats;
1877         }
1878
1879         if (console_session.ses_features != feats)
1880                 rc = -EPROTO;
1881
1882         spin_unlock(&console_session.ses_rpc_lock);
1883
1884         if (rc) {
1885                 CERROR("remote features %x do not match with session features %x of console\n",
1886                        feats, console_session.ses_features);
1887         }
1888
1889         return rc;
1890 }
1891
1892 static int
1893 lstcon_acceptor_handle(struct srpc_server_rpc *rpc)
1894 {
1895         srpc_msg_t *rep  = &rpc->srpc_replymsg;
1896         srpc_msg_t *req  = &rpc->srpc_reqstbuf->buf_msg;
1897         srpc_join_reqst_t *jreq = &req->msg_body.join_reqst;
1898         srpc_join_reply_t *jrep = &rep->msg_body.join_reply;
1899         lstcon_group_t *grp  = NULL;
1900         lstcon_ndlink_t *ndl;
1901         int rc   = 0;
1902
1903         sfw_unpack_message(req);
1904
1905         mutex_lock(&console_session.ses_mutex);
1906
1907         jrep->join_sid = console_session.ses_id;
1908
1909         if (console_session.ses_id.ses_nid == LNET_NID_ANY) {
1910                 jrep->join_status = ESRCH;
1911                 goto out;
1912         }
1913
1914         if (lstcon_session_feats_check(req->msg_ses_feats)) {
1915                 jrep->join_status = EPROTO;
1916                 goto out;
1917         }
1918
1919         if (jreq->join_sid.ses_nid != LNET_NID_ANY &&
1920             !lstcon_session_match(jreq->join_sid)) {
1921                 jrep->join_status = EBUSY;
1922                 goto out;
1923         }
1924
1925         if (lstcon_group_find(jreq->join_group, &grp)) {
1926                 rc = lstcon_group_alloc(jreq->join_group, &grp);
1927                 if (rc) {
1928                         CERROR("Out of memory\n");
1929                         goto out;
1930                 }
1931
1932                 list_add_tail(&grp->grp_link,
1933                               &console_session.ses_grp_list);
1934                 lstcon_group_addref(grp);
1935         }
1936
1937         if (grp->grp_ref > 2) {
1938                 /* Group in using */
1939                 jrep->join_status = EBUSY;
1940                 goto out;
1941         }
1942
1943         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 0);
1944         if (!rc) {
1945                 jrep->join_status = EEXIST;
1946                 goto out;
1947         }
1948
1949         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 1);
1950         if (rc) {
1951                 CERROR("Out of memory\n");
1952                 goto out;
1953         }
1954
1955         ndl->ndl_node->nd_state   = LST_NODE_ACTIVE;
1956         ndl->ndl_node->nd_timeout = console_session.ses_timeout;
1957
1958         if (!grp->grp_userland)
1959                 grp->grp_userland = 1;
1960
1961         strlcpy(jrep->join_session, console_session.ses_name,
1962                 sizeof(jrep->join_session));
1963         jrep->join_timeout = console_session.ses_timeout;
1964         jrep->join_status  = 0;
1965
1966 out:
1967         rep->msg_ses_feats = console_session.ses_features;
1968         if (grp)
1969                 lstcon_group_decref(grp);
1970
1971         mutex_unlock(&console_session.ses_mutex);
1972
1973         return rc;
1974 }
1975
1976 srpc_service_t lstcon_acceptor_service;
1977 static void lstcon_init_acceptor_service(void)
1978 {
1979         /* initialize selftest console acceptor service table */
1980         lstcon_acceptor_service.sv_name     = "join session";
1981         lstcon_acceptor_service.sv_handler  = lstcon_acceptor_handle;
1982         lstcon_acceptor_service.sv_id       = SRPC_SERVICE_JOIN;
1983         lstcon_acceptor_service.sv_wi_total = SFW_FRWK_WI_MAX;
1984 }
1985
1986 extern int lstcon_ioctl_entry(unsigned int cmd, struct libcfs_ioctl_hdr *hdr);
1987
1988 static DECLARE_IOCTL_HANDLER(lstcon_ioctl_handler, lstcon_ioctl_entry);
1989
1990 /* initialize console */
1991 int
1992 lstcon_console_init(void)
1993 {
1994         int i;
1995         int rc;
1996
1997         memset(&console_session, 0, sizeof(struct lstcon_session));
1998
1999         console_session.ses_id            = LST_INVALID_SID;
2000         console_session.ses_state         = LST_SESSION_NONE;
2001         console_session.ses_timeout       = 0;
2002         console_session.ses_force         = 0;
2003         console_session.ses_expired       = 0;
2004         console_session.ses_feats_updated = 0;
2005         console_session.ses_features      = LST_FEATS_MASK;
2006         console_session.ses_laststamp     = ktime_get_real_seconds();
2007
2008         mutex_init(&console_session.ses_mutex);
2009
2010         INIT_LIST_HEAD(&console_session.ses_ndl_list);
2011         INIT_LIST_HEAD(&console_session.ses_grp_list);
2012         INIT_LIST_HEAD(&console_session.ses_bat_list);
2013         INIT_LIST_HEAD(&console_session.ses_trans_list);
2014
2015         LIBCFS_ALLOC(console_session.ses_ndl_hash,
2016                      sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2017         if (!console_session.ses_ndl_hash)
2018                 return -ENOMEM;
2019
2020         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
2021                 INIT_LIST_HEAD(&console_session.ses_ndl_hash[i]);
2022
2023         /* initialize acceptor service table */
2024         lstcon_init_acceptor_service();
2025
2026         rc = srpc_add_service(&lstcon_acceptor_service);
2027         LASSERT(rc != -EBUSY);
2028         if (rc) {
2029                 LIBCFS_FREE(console_session.ses_ndl_hash,
2030                             sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2031                 return rc;
2032         }
2033
2034         rc = srpc_service_add_buffers(&lstcon_acceptor_service,
2035                                       lstcon_acceptor_service.sv_wi_total);
2036         if (rc) {
2037                 rc = -ENOMEM;
2038                 goto out;
2039         }
2040
2041         rc = libcfs_register_ioctl(&lstcon_ioctl_handler);
2042
2043         if (!rc) {
2044                 lstcon_rpc_module_init();
2045                 return 0;
2046         }
2047
2048 out:
2049         srpc_shutdown_service(&lstcon_acceptor_service);
2050         srpc_remove_service(&lstcon_acceptor_service);
2051
2052         LIBCFS_FREE(console_session.ses_ndl_hash,
2053                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2054
2055         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2056
2057         return rc;
2058 }
2059
2060 int
2061 lstcon_console_fini(void)
2062 {
2063         int i;
2064
2065         libcfs_deregister_ioctl(&lstcon_ioctl_handler);
2066
2067         mutex_lock(&console_session.ses_mutex);
2068
2069         srpc_shutdown_service(&lstcon_acceptor_service);
2070         srpc_remove_service(&lstcon_acceptor_service);
2071
2072         if (console_session.ses_state != LST_SESSION_NONE)
2073                 lstcon_session_end();
2074
2075         lstcon_rpc_module_fini();
2076
2077         mutex_unlock(&console_session.ses_mutex);
2078
2079         LASSERT(list_empty(&console_session.ses_ndl_list));
2080         LASSERT(list_empty(&console_session.ses_grp_list));
2081         LASSERT(list_empty(&console_session.ses_bat_list));
2082         LASSERT(list_empty(&console_session.ses_trans_list));
2083
2084         for (i = 0; i < LST_NODE_HASHSIZE; i++)
2085                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
2086
2087         LIBCFS_FREE(console_session.ses_ndl_hash,
2088                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2089
2090         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2091
2092         return 0;
2093 }