drivers/s390/cio/qdio_main.c

   1 /*
   2  * linux/drivers/s390/cio/qdio_main.c
   3  *
   4  * Linux for s390 qdio support, buffer handling, qdio API and module support.
   5  *
   6  * Copyright 2000,2008 IBM Corp.
   7  * Author(s): Utz Bacher <utz.bacher@de.ibm.com>
   8  *            Jan Glauber <jang@linux.vnet.ibm.com>
   9  * 2.6 cio integration by Cornelia Huck <cornelia.huck@de.ibm.com>
  10  */
  11 #include <linux/module.h>
  12 #include <linux/init.h>
  13 #include <linux/kernel.h>
  14 #include <linux/timer.h>
  15 #include <linux/delay.h>
  16 #include <linux/gfp.h>
  17 #include <linux/kernel_stat.h>
  18 #include <asm/atomic.h>
  19 #include <asm/debug.h>
  20 #include <asm/qdio.h>
  21
  22 #include "cio.h"
  23 #include "css.h"
  24 #include "device.h"
  25 #include "qdio.h"
  26 #include "qdio_debug.h"
  27
  28 MODULE_AUTHOR("Utz Bacher <utz.bacher@de.ibm.com>,"\
  29         "Jan Glauber <jang@linux.vnet.ibm.com>");
  30 MODULE_DESCRIPTION("QDIO base support");
  31 MODULE_LICENSE("GPL");
  32
  33 static inline int do_siga_sync(unsigned long schid,
  34                                unsigned int out_mask, unsigned int in_mask,
  35                                unsigned int fc)
  36 {
  37         register unsigned long __fc asm ("0") = fc;
  38         register unsigned long __schid asm ("1") = schid;
  39         register unsigned long out asm ("2") = out_mask;
  40         register unsigned long in asm ("3") = in_mask;
  41         int cc;
  42
  43         asm volatile(
  44                 "       siga    0\n"
  45                 "       ipm     %0\n"
  46                 "       srl     %0,28\n"
  47                 : "=d" (cc)
  48                 : "d" (__fc), "d" (__schid), "d" (out), "d" (in) : "cc");
  49         return cc;
  50 }
  51
  52 static inline int do_siga_input(unsigned long schid, unsigned int mask,
  53                                 unsigned int fc)
  54 {
  55         register unsigned long __fc asm ("0") = fc;
  56         register unsigned long __schid asm ("1") = schid;
  57         register unsigned long __mask asm ("2") = mask;
  58         int cc;
  59
  60         asm volatile(
  61                 "       siga    0\n"
  62                 "       ipm     %0\n"
  63                 "       srl     %0,28\n"
  64                 : "=d" (cc)
  65                 : "d" (__fc), "d" (__schid), "d" (__mask) : "cc", "memory");
  66         return cc;
  67 }
  68
  69 /**
  70  * do_siga_output - perform SIGA-w/wt function
  71  * @schid: subchannel id or in case of QEBSM the subchannel token
  72  * @mask: which output queues to process
  73  * @bb: busy bit indicator, set only if SIGA-w/wt could not access a buffer
  74  * @fc: function code to perform
  75  *
  76  * Returns cc or QDIO_ERROR_SIGA_ACCESS_EXCEPTION.
  77  * Note: For IQDC unicast queues only the highest priority queue is processed.
  78  */
  79 static inline int do_siga_output(unsigned long schid, unsigned long mask,
  80                                  unsigned int *bb, unsigned int fc)
  81 {
  82         register unsigned long __fc asm("0") = fc;
  83         register unsigned long __schid asm("1") = schid;
  84         register unsigned long __mask asm("2") = mask;
  85         int cc = QDIO_ERROR_SIGA_ACCESS_EXCEPTION;
  86
  87         asm volatile(
  88                 "       siga    0\n"
  89                 "0:     ipm     %0\n"
  90                 "       srl     %0,28\n"
  91                 "1:\n"
  92                 EX_TABLE(0b, 1b)
  93                 : "+d" (cc), "+d" (__fc), "+d" (__schid), "+d" (__mask)
  94                 : : "cc", "memory");
  95         *bb = ((unsigned int) __fc) >> 31;
  96         return cc;
  97 }
  98
  99 static inline int qdio_check_ccq(struct qdio_q *q, unsigned int ccq)
 100 {
 101         /* all done or next buffer state different */
 102         if (ccq == 0 || ccq == 32)
 103                 return 0;
 104         /* not all buffers processed */
 105         if (ccq == 96 || ccq == 97)
 106                 return 1;
 107         /* notify devices immediately */
 108         DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq);
 109         return -EIO;
 110 }
 111
 112 /**
 113  * qdio_do_eqbs - extract buffer states for QEBSM
 114  * @q: queue to manipulate
 115  * @state: state of the extracted buffers
 116  * @start: buffer number to start at
 117  * @count: count of buffers to examine
 118  * @auto_ack: automatically acknowledge buffers
 119  *
 120  * Returns the number of successfully extracted equal buffer states.
 121  * Stops processing if a state is different from the last buffers state.
 122  */
 123 static int qdio_do_eqbs(struct qdio_q *q, unsigned char *state,
 124                         int start, int count, int auto_ack)
 125 {
 126         unsigned int ccq = 0;
 127         int tmp_count = count, tmp_start = start;
 128         int nr = q->nr;
 129         int rc;
 130
 131         BUG_ON(!q->irq_ptr->sch_token);
 132         qperf_inc(q, eqbs);
 133
 134         if (!q->is_input_q)
 135                 nr += q->irq_ptr->nr_input_qs;
 136 again:
 137         ccq = do_eqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count,
 138                       auto_ack);
 139         rc = qdio_check_ccq(q, ccq);
 140
 141         /* At least one buffer was processed, return and extract the remaining
 142          * buffers later.
 143          */
 144         if ((ccq == 96) && (count != tmp_count)) {
 145                 qperf_inc(q, eqbs_partial);
 146                 return (count - tmp_count);
 147         }
 148
 149         if (rc == 1) {
 150                 DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "EQBS again:%2d", ccq);
 151                 goto again;
 152         }
 153
 154         if (rc < 0) {
 155                 DBF_ERROR("%4x EQBS ERROR", SCH_NO(q));
 156                 DBF_ERROR("%3d%3d%2d", count, tmp_count, nr);
 157                 q->handler(q->irq_ptr->cdev,
 158                            QDIO_ERROR_ACTIVATE_CHECK_CONDITION,
 159                            0, -1, -1, q->irq_ptr->int_parm);
 160                 return 0;
 161         }
 162         return count - tmp_count;
 163 }
 164
 165 /**
 166  * qdio_do_sqbs - set buffer states for QEBSM
 167  * @q: queue to manipulate
 168  * @state: new state of the buffers
 169  * @start: first buffer number to change
 170  * @count: how many buffers to change
 171  *
 172  * Returns the number of successfully changed buffers.
 173  * Does retrying until the specified count of buffer states is set or an
 174  * error occurs.
 175  */
 176 static int qdio_do_sqbs(struct qdio_q *q, unsigned char state, int start,
 177                         int count)
 178 {
 179         unsigned int ccq = 0;
 180         int tmp_count = count, tmp_start = start;
 181         int nr = q->nr;
 182         int rc;
 183
 184         if (!count)
 185                 return 0;
 186
 187         BUG_ON(!q->irq_ptr->sch_token);
 188         qperf_inc(q, sqbs);
 189
 190         if (!q->is_input_q)
 191                 nr += q->irq_ptr->nr_input_qs;
 192 again:
 193         ccq = do_sqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count);
 194         rc = qdio_check_ccq(q, ccq);
 195         if (rc == 1) {
 196                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "SQBS again:%2d", ccq);
 197                 qperf_inc(q, sqbs_partial);
 198                 goto again;
 199         }
 200         if (rc < 0) {
 201                 DBF_ERROR("%4x SQBS ERROR", SCH_NO(q));
 202                 DBF_ERROR("%3d%3d%2d", count, tmp_count, nr);
 203                 q->handler(q->irq_ptr->cdev,
 204                            QDIO_ERROR_ACTIVATE_CHECK_CONDITION,
 205                            0, -1, -1, q->irq_ptr->int_parm);
 206                 return 0;
 207         }
 208         WARN_ON(tmp_count);
 209         return count - tmp_count;
 210 }
 211
 212 /* returns number of examined buffers and their common state in *state */
 213 static inline int get_buf_states(struct qdio_q *q, unsigned int bufnr,
 214                                  unsigned char *state, unsigned int count,
 215                                  int auto_ack)
 216 {
 217         unsigned char __state = 0;
 218         int i;
 219
 220         BUG_ON(bufnr > QDIO_MAX_BUFFERS_MASK);
 221         BUG_ON(count > QDIO_MAX_BUFFERS_PER_Q);
 222
 223         if (is_qebsm(q))
 224                 return qdio_do_eqbs(q, state, bufnr, count, auto_ack);
 225
 226         for (i = 0; i < count; i++) {
 227                 if (!__state)
 228                         __state = q->slsb.val[bufnr];
 229                 else if (q->slsb.val[bufnr] != __state)
 230                         break;
 231                 bufnr = next_buf(bufnr);
 232         }
 233         *state = __state;
 234         return i;
 235 }
 236
 237 static inline int get_buf_state(struct qdio_q *q, unsigned int bufnr,
 238                                 unsigned char *state, int auto_ack)
 239 {
 240         return get_buf_states(q, bufnr, state, 1, auto_ack);
 241 }
 242
 243 /* wrap-around safe setting of slsb states, returns number of changed buffers */
 244 static inline int set_buf_states(struct qdio_q *q, int bufnr,
 245                                  unsigned char state, int count)
 246 {
 247         int i;
 248
 249         BUG_ON(bufnr > QDIO_MAX_BUFFERS_MASK);
 250         BUG_ON(count > QDIO_MAX_BUFFERS_PER_Q);
 251
 252         if (is_qebsm(q))
 253                 return qdio_do_sqbs(q, state, bufnr, count);
 254
 255         for (i = 0; i < count; i++) {
 256                 xchg(&q->slsb.val[bufnr], state);
 257                 bufnr = next_buf(bufnr);
 258         }
 259         return count;
 260 }
 261
 262 static inline int set_buf_state(struct qdio_q *q, int bufnr,
 263                                 unsigned char state)
 264 {
 265         return set_buf_states(q, bufnr, state, 1);
 266 }
 267
 268 /* set slsb states to initial state */
 269 void qdio_init_buf_states(struct qdio_irq *irq_ptr)
 270 {
 271         struct qdio_q *q;
 272         int i;
 273
 274         for_each_input_queue(irq_ptr, q, i)
 275                 set_buf_states(q, 0, SLSB_P_INPUT_NOT_INIT,
 276                                QDIO_MAX_BUFFERS_PER_Q);
 277         for_each_output_queue(irq_ptr, q, i)
 278                 set_buf_states(q, 0, SLSB_P_OUTPUT_NOT_INIT,
 279                                QDIO_MAX_BUFFERS_PER_Q);
 280 }
 281
 282 static inline int qdio_siga_sync(struct qdio_q *q, unsigned int output,
 283                           unsigned int input)
 284 {
 285         unsigned long schid = *((u32 *) &q->irq_ptr->schid);
 286         unsigned int fc = QDIO_SIGA_SYNC;
 287         int cc;
 288
 289         if (!need_siga_sync(q))
 290                 return 0;
 291
 292         DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-s:%1d", q->nr);
 293         qperf_inc(q, siga_sync);
 294
 295         if (is_qebsm(q)) {
 296                 schid = q->irq_ptr->sch_token;
 297                 fc |= QDIO_SIGA_QEBSM_FLAG;
 298         }
 299
 300         cc = do_siga_sync(schid, output, input, fc);
 301         if (cc)
 302                 DBF_ERROR("%4x SIGA-S:%2d", SCH_NO(q), cc);
 303         return cc;
 304 }
 305
 306 static inline int qdio_siga_sync_q(struct qdio_q *q)
 307 {
 308         if (q->is_input_q)
 309                 return qdio_siga_sync(q, 0, q->mask);
 310         else
 311                 return qdio_siga_sync(q, q->mask, 0);
 312 }
 313
 314 static inline int qdio_siga_sync_out(struct qdio_q *q)
 315 {
 316         return qdio_siga_sync(q, ~0U, 0);
 317 }
 318
 319 static inline int qdio_siga_sync_all(struct qdio_q *q)
 320 {
 321         return qdio_siga_sync(q, ~0U, ~0U);
 322 }
 323
 324 static int qdio_siga_output(struct qdio_q *q, unsigned int *busy_bit)
 325 {
 326         unsigned long schid = *((u32 *) &q->irq_ptr->schid);
 327         unsigned int fc = QDIO_SIGA_WRITE;
 328         u64 start_time = 0;
 329         int cc;
 330
 331         if (q->u.out.use_enh_siga)
 332                 fc = 3;
 333
 334         if (is_qebsm(q)) {
 335                 schid = q->irq_ptr->sch_token;
 336                 fc |= QDIO_SIGA_QEBSM_FLAG;
 337         }
 338 again:
 339         cc = do_siga_output(schid, q->mask, busy_bit, fc);
 340
 341         /* hipersocket busy condition */
 342         if (*busy_bit) {
 343                 WARN_ON(queue_type(q) != QDIO_IQDIO_QFMT || cc != 2);
 344
 345                 if (!start_time) {
 346                         start_time = get_clock();
 347                         goto again;
 348                 }
 349                 if ((get_clock() - start_time) < QDIO_BUSY_BIT_PATIENCE)
 350                         goto again;
 351         }
 352         return cc;
 353 }
 354
 355 static inline int qdio_siga_input(struct qdio_q *q)
 356 {
 357         unsigned long schid = *((u32 *) &q->irq_ptr->schid);
 358         unsigned int fc = QDIO_SIGA_READ;
 359         int cc;
 360
 361         DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-r:%1d", q->nr);
 362         qperf_inc(q, siga_read);
 363
 364         if (is_qebsm(q)) {
 365                 schid = q->irq_ptr->sch_token;
 366                 fc |= QDIO_SIGA_QEBSM_FLAG;
 367         }
 368
 369         cc = do_siga_input(schid, q->mask, fc);
 370         if (cc)
 371                 DBF_ERROR("%4x SIGA-R:%2d", SCH_NO(q), cc);
 372         return cc;
 373 }
 374
 375 static inline void qdio_sync_after_thinint(struct qdio_q *q)
 376 {
 377         if (pci_out_supported(q)) {
 378                 if (need_siga_sync_thinint(q))
 379                         qdio_siga_sync_all(q);
 380                 else if (need_siga_sync_out_thinint(q))
 381                         qdio_siga_sync_out(q);
 382         } else
 383                 qdio_siga_sync_q(q);
 384 }
 385
 386 int debug_get_buf_state(struct qdio_q *q, unsigned int bufnr,
 387                         unsigned char *state)
 388 {
 389         qdio_siga_sync_q(q);
 390         return get_buf_states(q, bufnr, state, 1, 0);
 391 }
 392
 393 static inline void qdio_stop_polling(struct qdio_q *q)
 394 {
 395         if (!q->u.in.polling)
 396                 return;
 397
 398         q->u.in.polling = 0;
 399         qperf_inc(q, stop_polling);
 400
 401         /* show the card that we are not polling anymore */
 402         if (is_qebsm(q)) {
 403                 set_buf_states(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT,
 404                                q->u.in.ack_count);
 405                 q->u.in.ack_count = 0;
 406         } else
 407                 set_buf_state(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT);
 408 }
 409
 410 static inline void account_sbals(struct qdio_q *q, int count)
 411 {
 412         int pos = 0;
 413
 414         q->q_stats.nr_sbal_total += count;
 415         if (count == QDIO_MAX_BUFFERS_MASK) {
 416                 q->q_stats.nr_sbals[7]++;
 417                 return;
 418         }
 419         while (count >>= 1)
 420                 pos++;
 421         q->q_stats.nr_sbals[pos]++;
 422 }
 423
 424 static void announce_buffer_error(struct qdio_q *q, int count)
 425 {
 426         q->qdio_error |= QDIO_ERROR_SLSB_STATE;
 427
 428         /* special handling for no target buffer empty */
 429         if ((!q->is_input_q &&
 430             (q->sbal[q->first_to_check]->element[15].flags & 0xff) == 0x10)) {
 431                 qperf_inc(q, target_full);
 432                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x",
 433                               q->first_to_check);
 434                 return;
 435         }
 436
 437         DBF_ERROR("%4x BUF ERROR", SCH_NO(q));
 438         DBF_ERROR((q->is_input_q) ? "IN:%2d" : "OUT:%2d", q->nr);
 439         DBF_ERROR("FTC:%3d C:%3d", q->first_to_check, count);
 440         DBF_ERROR("F14:%2x F15:%2x",
 441                   q->sbal[q->first_to_check]->element[14].flags & 0xff,
 442                   q->sbal[q->first_to_check]->element[15].flags & 0xff);
 443 }
 444
 445 static inline void inbound_primed(struct qdio_q *q, int count)
 446 {
 447         int new;
 448
 449         DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim: %02x", count);
 450
 451         /* for QEBSM the ACK was already set by EQBS */
 452         if (is_qebsm(q)) {
 453                 if (!q->u.in.polling) {
 454                         q->u.in.polling = 1;
 455                         q->u.in.ack_count = count;
 456                         q->u.in.ack_start = q->first_to_check;
 457                         return;
 458                 }
 459
 460                 /* delete the previous ACK's */
 461                 set_buf_states(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT,
 462                                q->u.in.ack_count);
 463                 q->u.in.ack_count = count;
 464                 q->u.in.ack_start = q->first_to_check;
 465                 return;
 466         }
 467
 468         /*
 469          * ACK the newest buffer. The ACK will be removed in qdio_stop_polling
 470          * or by the next inbound run.
 471          */
 472         new = add_buf(q->first_to_check, count - 1);
 473         if (q->u.in.polling) {
 474                 /* reset the previous ACK but first set the new one */
 475                 set_buf_state(q, new, SLSB_P_INPUT_ACK);
 476                 set_buf_state(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT);
 477         } else {
 478                 q->u.in.polling = 1;
 479                 set_buf_state(q, new, SLSB_P_INPUT_ACK);
 480         }
 481
 482         q->u.in.ack_start = new;
 483         count--;
 484         if (!count)
 485                 return;
 486         /* need to change ALL buffers to get more interrupts */
 487         set_buf_states(q, q->first_to_check, SLSB_P_INPUT_NOT_INIT, count);
 488 }
 489
 490 static int get_inbound_buffer_frontier(struct qdio_q *q)
 491 {
 492         int count, stop;
 493         unsigned char state;
 494
 495         /*
 496          * Don't check 128 buffers, as otherwise qdio_inbound_q_moved
 497          * would return 0.
 498          */
 499         count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK);
 500         stop = add_buf(q->first_to_check, count);
 501
 502         if (q->first_to_check == stop)
 503                 goto out;
 504
 505         /*
 506          * No siga sync here, as a PCI or we after a thin interrupt
 507          * already sync'ed the queues.
 508          */
 509         count = get_buf_states(q, q->first_to_check, &state, count, 1);
 510         if (!count)
 511                 goto out;
 512
 513         switch (state) {
 514         case SLSB_P_INPUT_PRIMED:
 515                 inbound_primed(q, count);
 516                 q->first_to_check = add_buf(q->first_to_check, count);
 517                 if (atomic_sub(count, &q->nr_buf_used) == 0)
 518                         qperf_inc(q, inbound_queue_full);
 519                 if (q->irq_ptr->perf_stat_enabled)
 520                         account_sbals(q, count);
 521                 break;
 522         case SLSB_P_INPUT_ERROR:
 523                 announce_buffer_error(q, count);
 524                 /* process the buffer, the upper layer will take care of it */
 525                 q->first_to_check = add_buf(q->first_to_check, count);
 526                 atomic_sub(count, &q->nr_buf_used);
 527                 if (q->irq_ptr->perf_stat_enabled)
 528                         account_sbals_error(q, count);
 529                 break;
 530         case SLSB_CU_INPUT_EMPTY:
 531         case SLSB_P_INPUT_NOT_INIT:
 532         case SLSB_P_INPUT_ACK:
 533                 if (q->irq_ptr->perf_stat_enabled)
 534                         q->q_stats.nr_sbal_nop++;
 535                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in nop");
 536                 break;
 537         default:
 538                 BUG();
 539         }
 540 out:
 541         return q->first_to_check;
 542 }
 543
 544 static int qdio_inbound_q_moved(struct qdio_q *q)
 545 {
 546         int bufnr;
 547
 548         bufnr = get_inbound_buffer_frontier(q);
 549
 550         if ((bufnr != q->last_move) || q->qdio_error) {
 551                 q->last_move = bufnr;
 552                 if (!is_thinint_irq(q->irq_ptr) && MACHINE_IS_LPAR)
 553                         q->u.in.timestamp = get_clock();
 554                 return 1;
 555         } else
 556                 return 0;
 557 }
 558
 559 static inline int qdio_inbound_q_done(struct qdio_q *q)
 560 {
 561         unsigned char state = 0;
 562
 563         if (!atomic_read(&q->nr_buf_used))
 564                 return 1;
 565
 566         qdio_siga_sync_q(q);
 567         get_buf_state(q, q->first_to_check, &state, 0);
 568
 569         if (state == SLSB_P_INPUT_PRIMED || state == SLSB_P_INPUT_ERROR)
 570                 /* more work coming */
 571                 return 0;
 572
 573         if (is_thinint_irq(q->irq_ptr))
 574                 return 1;
 575
 576         /* don't poll under z/VM */
 577         if (MACHINE_IS_VM)
 578                 return 1;
 579
 580         /*
 581          * At this point we know, that inbound first_to_check
 582          * has (probably) not moved (see qdio_inbound_processing).
 583          */
 584         if (get_clock() > q->u.in.timestamp + QDIO_INPUT_THRESHOLD) {
 585                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in done:%02x",
 586                               q->first_to_check);
 587                 return 1;
 588         } else
 589                 return 0;
 590 }
 591
 592 static void qdio_kick_handler(struct qdio_q *q)
 593 {
 594         int start = q->first_to_kick;
 595         int end = q->first_to_check;
 596         int count;
 597
 598         if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))
 599                 return;
 600
 601         count = sub_buf(end, start);
 602
 603         if (q->is_input_q) {
 604                 qperf_inc(q, inbound_handler);
 605                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "kih s:%02x c:%02x", start, count);
 606         } else {
 607                 qperf_inc(q, outbound_handler);
 608                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "koh: s:%02x c:%02x",
 609                               start, count);
 610         }
 611
 612         q->handler(q->irq_ptr->cdev, q->qdio_error, q->nr, start, count,
 613                    q->irq_ptr->int_parm);
 614
 615         /* for the next time */
 616         q->first_to_kick = end;
 617         q->qdio_error = 0;
 618 }
 619
 620 static void __qdio_inbound_processing(struct qdio_q *q)
 621 {
 622         qperf_inc(q, tasklet_inbound);
 623
 624         if (!qdio_inbound_q_moved(q))
 625                 return;
 626
 627         qdio_kick_handler(q);
 628
 629         if (!qdio_inbound_q_done(q)) {
 630                 /* means poll time is not yet over */
 631                 qperf_inc(q, tasklet_inbound_resched);
 632                 if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED)) {
 633                         tasklet_schedule(&q->tasklet);
 634                         return;
 635                 }
 636         }
 637
 638         qdio_stop_polling(q);
 639         /*
 640          * We need to check again to not lose initiative after
 641          * resetting the ACK state.
 642          */
 643         if (!qdio_inbound_q_done(q)) {
 644                 qperf_inc(q, tasklet_inbound_resched2);
 645                 if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED))
 646                         tasklet_schedule(&q->tasklet);
 647         }
 648 }
 649
 650 void qdio_inbound_processing(unsigned long data)
 651 {
 652         struct qdio_q *q = (struct qdio_q *)data;
 653         __qdio_inbound_processing(q);
 654 }
 655
 656 static int get_outbound_buffer_frontier(struct qdio_q *q)
 657 {
 658         int count, stop;
 659         unsigned char state;
 660
 661         if (((queue_type(q) != QDIO_IQDIO_QFMT) && !pci_out_supported(q)) ||
 662             (queue_type(q) == QDIO_IQDIO_QFMT && multicast_outbound(q)))
 663                 qdio_siga_sync_q(q);
 664
 665         /*
 666          * Don't check 128 buffers, as otherwise qdio_inbound_q_moved
 667          * would return 0.
 668          */
 669         count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK);
 670         stop = add_buf(q->first_to_check, count);
 671
 672         if (q->first_to_check == stop)
 673                 return q->first_to_check;
 674
 675         count = get_buf_states(q, q->first_to_check, &state, count, 0);
 676         if (!count)
 677                 return q->first_to_check;
 678
 679         switch (state) {
 680         case SLSB_P_OUTPUT_EMPTY:
 681                 /* the adapter got it */
 682                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out empty:%1d %02x", q->nr, count);
 683
 684                 atomic_sub(count, &q->nr_buf_used);
 685                 q->first_to_check = add_buf(q->first_to_check, count);
 686                 if (q->irq_ptr->perf_stat_enabled)
 687                         account_sbals(q, count);
 688                 break;
 689         case SLSB_P_OUTPUT_ERROR:
 690                 announce_buffer_error(q, count);
 691                 /* process the buffer, the upper layer will take care of it */
 692                 q->first_to_check = add_buf(q->first_to_check, count);
 693                 atomic_sub(count, &q->nr_buf_used);
 694                 if (q->irq_ptr->perf_stat_enabled)
 695                         account_sbals_error(q, count);
 696                 break;
 697         case SLSB_CU_OUTPUT_PRIMED:
 698                 /* the adapter has not fetched the output yet */
 699                 if (q->irq_ptr->perf_stat_enabled)
 700                         q->q_stats.nr_sbal_nop++;
 701                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out primed:%1d", q->nr);
 702                 break;
 703         case SLSB_P_OUTPUT_NOT_INIT:
 704         case SLSB_P_OUTPUT_HALTED:
 705                 break;
 706         default:
 707                 BUG();
 708         }
 709         return q->first_to_check;
 710 }
 711
 712 /* all buffers processed? */
 713 static inline int qdio_outbound_q_done(struct qdio_q *q)
 714 {
 715         return atomic_read(&q->nr_buf_used) == 0;
 716 }
 717
 718 static inline int qdio_outbound_q_moved(struct qdio_q *q)
 719 {
 720         int bufnr;
 721
 722         bufnr = get_outbound_buffer_frontier(q);
 723
 724         if ((bufnr != q->last_move) || q->qdio_error) {
 725                 q->last_move = bufnr;
 726                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out moved:%1d", q->nr);
 727                 return 1;
 728         } else
 729                 return 0;
 730 }
 731
 732 static int qdio_kick_outbound_q(struct qdio_q *q)
 733 {
 734         unsigned int busy_bit;
 735         int cc;
 736
 737         if (!need_siga_out(q))
 738                 return 0;
 739
 740         DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w:%1d", q->nr);
 741         qperf_inc(q, siga_write);
 742
 743         cc = qdio_siga_output(q, &busy_bit);
 744         switch (cc) {
 745         case 0:
 746                 break;
 747         case 2:
 748                 if (busy_bit) {
 749                         DBF_ERROR("%4x cc2 REP:%1d", SCH_NO(q), q->nr);
 750                         cc |= QDIO_ERROR_SIGA_BUSY;
 751                 } else
 752                         DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w cc2:%1d", q->nr);
 753                 break;
 754         case 1:
 755         case 3:
 756                 DBF_ERROR("%4x SIGA-W:%1d", SCH_NO(q), cc);
 757                 break;
 758         }
 759         return cc;
 760 }
 761
 762 static void __qdio_outbound_processing(struct qdio_q *q)
 763 {
 764         qperf_inc(q, tasklet_outbound);
 765         BUG_ON(atomic_read(&q->nr_buf_used) < 0);
 766
 767         if (qdio_outbound_q_moved(q))
 768                 qdio_kick_handler(q);
 769
 770         if (queue_type(q) == QDIO_ZFCP_QFMT)
 771                 if (!pci_out_supported(q) && !qdio_outbound_q_done(q))
 772                         goto sched;
 773
 774         /* bail out for HiperSockets unicast queues */
 775         if (queue_type(q) == QDIO_IQDIO_QFMT && !multicast_outbound(q))
 776                 return;
 777
 778         if ((queue_type(q) == QDIO_IQDIO_QFMT) &&
 779             (atomic_read(&q->nr_buf_used)) > QDIO_IQDIO_POLL_LVL)
 780                 goto sched;
 781
 782         if (q->u.out.pci_out_enabled)
 783                 return;
 784
 785         /*
 786          * Now we know that queue type is either qeth without pci enabled
 787          * or HiperSockets multicast. Make sure buffer switch from PRIMED to
 788          * EMPTY is noticed and outbound_handler is called after some time.
 789          */
 790         if (qdio_outbound_q_done(q))
 791                 del_timer(&q->u.out.timer);
 792         else
 793                 if (!timer_pending(&q->u.out.timer))
 794                         mod_timer(&q->u.out.timer, jiffies + 10 * HZ);
 795         return;
 796
 797 sched:
 798         if (unlikely(q->irq_ptr->state == QDIO_IRQ_STATE_STOPPED))
 799                 return;
 800         tasklet_schedule(&q->tasklet);
 801 }
 802
 803 /* outbound tasklet */
 804 void qdio_outbound_processing(unsigned long data)
 805 {
 806         struct qdio_q *q = (struct qdio_q *)data;
 807         __qdio_outbound_processing(q);
 808 }
 809
 810 void qdio_outbound_timer(unsigned long data)
 811 {
 812         struct qdio_q *q = (struct qdio_q *)data;
 813
 814         if (unlikely(q->irq_ptr->state == QDIO_IRQ_STATE_STOPPED))
 815                 return;
 816         tasklet_schedule(&q->tasklet);
 817 }
 818
 819 static inline void qdio_check_outbound_after_thinint(struct qdio_q *q)
 820 {
 821         struct qdio_q *out;
 822         int i;
 823
 824         if (!pci_out_supported(q))
 825                 return;
 826
 827         for_each_output_queue(q->irq_ptr, out, i)
 828                 if (!qdio_outbound_q_done(out))
 829                         tasklet_schedule(&out->tasklet);
 830 }
 831
 832 static void __tiqdio_inbound_processing(struct qdio_q *q)
 833 {
 834         qperf_inc(q, tasklet_inbound);
 835         qdio_sync_after_thinint(q);
 836
 837         /*
 838          * The interrupt could be caused by a PCI request. Check the
 839          * PCI capable outbound queues.
 840          */
 841         qdio_check_outbound_after_thinint(q);
 842
 843         if (!qdio_inbound_q_moved(q))
 844                 return;
 845
 846         qdio_kick_handler(q);
 847
 848         if (!qdio_inbound_q_done(q)) {
 849                 qperf_inc(q, tasklet_inbound_resched);
 850                 if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED)) {
 851                         tasklet_schedule(&q->tasklet);
 852                         return;
 853                 }
 854         }
 855
 856         qdio_stop_polling(q);
 857         /*
 858          * We need to check again to not lose initiative after
 859          * resetting the ACK state.
 860          */
 861         if (!qdio_inbound_q_done(q)) {
 862                 qperf_inc(q, tasklet_inbound_resched2);
 863                 if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED))
 864                         tasklet_schedule(&q->tasklet);
 865         }
 866 }
 867
 868 void tiqdio_inbound_processing(unsigned long data)
 869 {
 870         struct qdio_q *q = (struct qdio_q *)data;
 871         __tiqdio_inbound_processing(q);
 872 }
 873
 874 static inline void qdio_set_state(struct qdio_irq *irq_ptr,
 875                                   enum qdio_irq_states state)
 876 {
 877         DBF_DEV_EVENT(DBF_INFO, irq_ptr, "newstate: %1d", state);
 878
 879         irq_ptr->state = state;
 880         mb();
 881 }
 882
 883 static void qdio_irq_check_sense(struct qdio_irq *irq_ptr, struct irb *irb)
 884 {
 885         if (irb->esw.esw0.erw.cons) {
 886                 DBF_ERROR("%4x sense:", irq_ptr->schid.sch_no);
 887                 DBF_ERROR_HEX(irb, 64);
 888                 DBF_ERROR_HEX(irb->ecw, 64);
 889         }
 890 }
 891
 892 /* PCI interrupt handler */
 893 static void qdio_int_handler_pci(struct qdio_irq *irq_ptr)
 894 {
 895         int i;
 896         struct qdio_q *q;
 897
 898         if (unlikely(irq_ptr->state == QDIO_IRQ_STATE_STOPPED))
 899                 return;
 900
 901         for_each_input_queue(irq_ptr, q, i) {
 902                 if (q->u.in.queue_start_poll) {
 903                         /* skip if polling is enabled or already in work */
 904                         if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED,
 905                                      &q->u.in.queue_irq_state)) {
 906                                 qperf_inc(q, int_discarded);
 907                                 continue;
 908                         }
 909                         q->u.in.queue_start_poll(q->irq_ptr->cdev, q->nr,
 910                                                  q->irq_ptr->int_parm);
 911                 } else
 912                         tasklet_schedule(&q->tasklet);
 913         }
 914
 915         if (!(irq_ptr->qib.ac & QIB_AC_OUTBOUND_PCI_SUPPORTED))
 916                 return;
 917
 918         for_each_output_queue(irq_ptr, q, i) {
 919                 if (qdio_outbound_q_done(q))
 920                         continue;
 921
 922                 if (!siga_syncs_out_pci(q))
 923                         qdio_siga_sync_q(q);
 924
 925                 tasklet_schedule(&q->tasklet);
 926         }
 927 }
 928
 929 static void qdio_handle_activate_check(struct ccw_device *cdev,
 930                                 unsigned long intparm, int cstat, int dstat)
 931 {
 932         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
 933         struct qdio_q *q;
 934
 935         DBF_ERROR("%4x ACT CHECK", irq_ptr->schid.sch_no);
 936         DBF_ERROR("intp :%lx", intparm);
 937         DBF_ERROR("ds: %2x cs:%2x", dstat, cstat);
 938
 939         if (irq_ptr->nr_input_qs) {
 940                 q = irq_ptr->input_qs[0];
 941         } else if (irq_ptr->nr_output_qs) {
 942                 q = irq_ptr->output_qs[0];
 943         } else {
 944                 dump_stack();
 945                 goto no_handler;
 946         }
 947         q->handler(q->irq_ptr->cdev, QDIO_ERROR_ACTIVATE_CHECK_CONDITION,
 948                    0, -1, -1, irq_ptr->int_parm);
 949 no_handler:
 950         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);
 951 }
 952
 953 static void qdio_establish_handle_irq(struct ccw_device *cdev, int cstat,
 954                                       int dstat)
 955 {
 956         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
 957
 958         DBF_DEV_EVENT(DBF_INFO, irq_ptr, "qest irq");
 959
 960         if (cstat)
 961                 goto error;
 962         if (dstat & ~(DEV_STAT_DEV_END | DEV_STAT_CHN_END))
 963                 goto error;
 964         if (!(dstat & DEV_STAT_DEV_END))
 965                 goto error;
 966         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ESTABLISHED);
 967         return;
 968
 969 error:
 970         DBF_ERROR("%4x EQ:error", irq_ptr->schid.sch_no);
 971         DBF_ERROR("ds: %2x cs:%2x", dstat, cstat);
 972         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);
 973 }
 974
 975 /* qdio interrupt handler */
 976 void qdio_int_handler(struct ccw_device *cdev, unsigned long intparm,
 977                       struct irb *irb)
 978 {
 979         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
 980         int cstat, dstat;
 981
 982         if (!intparm || !irq_ptr) {
 983                 DBF_ERROR("qint:%4x", cdev->private->schid.sch_no);
 984                 return;
 985         }
 986
 987         kstat_cpu(smp_processor_id()).irqs[IOINT_QDI]++;
 988         if (irq_ptr->perf_stat_enabled)
 989                 irq_ptr->perf_stat.qdio_int++;
 990
 991         if (IS_ERR(irb)) {
 992                 switch (PTR_ERR(irb)) {
 993                 case -EIO:
 994                         DBF_ERROR("%4x IO error", irq_ptr->schid.sch_no);
 995                         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);
 996                         wake_up(&cdev->private->wait_q);
 997                         return;
 998                 default:
 999                         WARN_ON(1);
1000                         return;
1001                 }
1002         }
1003         qdio_irq_check_sense(irq_ptr, irb);
1004         cstat = irb->scsw.cmd.cstat;
1005         dstat = irb->scsw.cmd.dstat;
1006
1007         switch (irq_ptr->state) {
1008         case QDIO_IRQ_STATE_INACTIVE:
1009                 qdio_establish_handle_irq(cdev, cstat, dstat);
1010                 break;
1011         case QDIO_IRQ_STATE_CLEANUP:
1012                 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
1013                 break;
1014         case QDIO_IRQ_STATE_ESTABLISHED:
1015         case QDIO_IRQ_STATE_ACTIVE:
1016                 if (cstat & SCHN_STAT_PCI) {
1017                         qdio_int_handler_pci(irq_ptr);
1018                         return;
1019                 }
1020                 if (cstat || dstat)
1021                         qdio_handle_activate_check(cdev, intparm, cstat,
1022                                                    dstat);
1023                 break;
1024         case QDIO_IRQ_STATE_STOPPED:
1025                 break;
1026         default:
1027                 WARN_ON(1);
1028         }
1029         wake_up(&cdev->private->wait_q);
1030 }
1031
1032 /**
1033  * qdio_get_ssqd_desc - get qdio subchannel description
1034  * @cdev: ccw device to get description for
1035  * @data: where to store the ssqd
1036  *
1037  * Returns 0 or an error code. The results of the chsc are stored in the
1038  * specified structure.
1039  */
1040 int qdio_get_ssqd_desc(struct ccw_device *cdev,
1041                        struct qdio_ssqd_desc *data)
1042 {
1043
1044         if (!cdev || !cdev->private)
1045                 return -EINVAL;
1046
1047         DBF_EVENT("get ssqd:%4x", cdev->private->schid.sch_no);
1048         return qdio_setup_get_ssqd(NULL, &cdev->private->schid, data);
1049 }
1050 EXPORT_SYMBOL_GPL(qdio_get_ssqd_desc);
1051
1052 static void qdio_shutdown_queues(struct ccw_device *cdev)
1053 {
1054         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1055         struct qdio_q *q;
1056         int i;
1057
1058         for_each_input_queue(irq_ptr, q, i)
1059                 tasklet_kill(&q->tasklet);
1060
1061         for_each_output_queue(irq_ptr, q, i) {
1062                 del_timer(&q->u.out.timer);
1063                 tasklet_kill(&q->tasklet);
1064         }
1065 }
1066
1067 /**
1068  * qdio_shutdown - shut down a qdio subchannel
1069  * @cdev: associated ccw device
1070  * @how: use halt or clear to shutdown
1071  */
1072 int qdio_shutdown(struct ccw_device *cdev, int how)
1073 {
1074         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1075         int rc;
1076         unsigned long flags;
1077
1078         if (!irq_ptr)
1079                 return -ENODEV;
1080
1081         BUG_ON(irqs_disabled());
1082         DBF_EVENT("qshutdown:%4x", cdev->private->schid.sch_no);
1083
1084         mutex_lock(&irq_ptr->setup_mutex);
1085         /*
1086          * Subchannel was already shot down. We cannot prevent being called
1087          * twice since cio may trigger a shutdown asynchronously.
1088          */
1089         if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) {
1090                 mutex_unlock(&irq_ptr->setup_mutex);
1091                 return 0;
1092         }
1093
1094         /*
1095          * Indicate that the device is going down. Scheduling the queue
1096          * tasklets is forbidden from here on.
1097          */
1098         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);
1099
1100         tiqdio_remove_input_queues(irq_ptr);
1101         qdio_shutdown_queues(cdev);
1102         qdio_shutdown_debug_entries(irq_ptr, cdev);
1103
1104         /* cleanup subchannel */
1105         spin_lock_irqsave(get_ccwdev_lock(cdev), flags);
1106
1107         if (how & QDIO_FLAG_CLEANUP_USING_CLEAR)
1108                 rc = ccw_device_clear(cdev, QDIO_DOING_CLEANUP);
1109         else
1110                 /* default behaviour is halt */
1111                 rc = ccw_device_halt(cdev, QDIO_DOING_CLEANUP);
1112         if (rc) {
1113                 DBF_ERROR("%4x SHUTD ERR", irq_ptr->schid.sch_no);
1114                 DBF_ERROR("rc:%4d", rc);
1115                 goto no_cleanup;
1116         }
1117
1118         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_CLEANUP);
1119         spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
1120         wait_event_interruptible_timeout(cdev->private->wait_q,
1121                 irq_ptr->state == QDIO_IRQ_STATE_INACTIVE ||
1122                 irq_ptr->state == QDIO_IRQ_STATE_ERR,
1123                 10 * HZ);
1124         spin_lock_irqsave(get_ccwdev_lock(cdev), flags);
1125
1126 no_cleanup:
1127         qdio_shutdown_thinint(irq_ptr);
1128
1129         /* restore interrupt handler */
1130         if ((void *)cdev->handler == (void *)qdio_int_handler)
1131                 cdev->handler = irq_ptr->orig_handler;
1132         spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
1133
1134         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
1135         mutex_unlock(&irq_ptr->setup_mutex);
1136         if (rc)
1137                 return rc;
1138         return 0;
1139 }
1140 EXPORT_SYMBOL_GPL(qdio_shutdown);
1141
1142 /**
1143  * qdio_free - free data structures for a qdio subchannel
1144  * @cdev: associated ccw device
1145  */
1146 int qdio_free(struct ccw_device *cdev)
1147 {
1148         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1149
1150         if (!irq_ptr)
1151                 return -ENODEV;
1152
1153         DBF_EVENT("qfree:%4x", cdev->private->schid.sch_no);
1154         mutex_lock(&irq_ptr->setup_mutex);
1155
1156         if (irq_ptr->debug_area != NULL) {
1157                 debug_unregister(irq_ptr->debug_area);
1158                 irq_ptr->debug_area = NULL;
1159         }
1160         cdev->private->qdio_data = NULL;
1161         mutex_unlock(&irq_ptr->setup_mutex);
1162
1163         qdio_release_memory(irq_ptr);
1164         return 0;
1165 }
1166 EXPORT_SYMBOL_GPL(qdio_free);
1167
1168 /**
1169  * qdio_allocate - allocate qdio queues and associated data
1170  * @init_data: initialization data
1171  */
1172 int qdio_allocate(struct qdio_initialize *init_data)
1173 {
1174         struct qdio_irq *irq_ptr;
1175
1176         DBF_EVENT("qallocate:%4x", init_data->cdev->private->schid.sch_no);
1177
1178         if ((init_data->no_input_qs && !init_data->input_handler) ||
1179             (init_data->no_output_qs && !init_data->output_handler))
1180                 return -EINVAL;
1181
1182         if ((init_data->no_input_qs > QDIO_MAX_QUEUES_PER_IRQ) ||
1183             (init_data->no_output_qs > QDIO_MAX_QUEUES_PER_IRQ))
1184                 return -EINVAL;
1185
1186         if ((!init_data->input_sbal_addr_array) ||
1187             (!init_data->output_sbal_addr_array))
1188                 return -EINVAL;
1189
1190         /* irq_ptr must be in GFP_DMA since it contains ccw1.cda */
1191         irq_ptr = (void *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1192         if (!irq_ptr)
1193                 goto out_err;
1194
1195         mutex_init(&irq_ptr->setup_mutex);
1196         qdio_allocate_dbf(init_data, irq_ptr);
1197
1198         /*
1199          * Allocate a page for the chsc calls in qdio_establish.
1200          * Must be pre-allocated since a zfcp recovery will call
1201          * qdio_establish. In case of low memory and swap on a zfcp disk
1202          * we may not be able to allocate memory otherwise.
1203          */
1204         irq_ptr->chsc_page = get_zeroed_page(GFP_KERNEL);
1205         if (!irq_ptr->chsc_page)
1206                 goto out_rel;
1207
1208         /* qdr is used in ccw1.cda which is u32 */
1209         irq_ptr->qdr = (struct qdr *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1210         if (!irq_ptr->qdr)
1211                 goto out_rel;
1212         WARN_ON((unsigned long)irq_ptr->qdr & 0xfff);
1213
1214         if (qdio_allocate_qs(irq_ptr, init_data->no_input_qs,
1215                              init_data->no_output_qs))
1216                 goto out_rel;
1217
1218         init_data->cdev->private->qdio_data = irq_ptr;
1219         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
1220         return 0;
1221 out_rel:
1222         qdio_release_memory(irq_ptr);
1223 out_err:
1224         return -ENOMEM;
1225 }
1226 EXPORT_SYMBOL_GPL(qdio_allocate);
1227
1228 /**
1229  * qdio_establish - establish queues on a qdio subchannel
1230  * @init_data: initialization data
1231  */
1232 int qdio_establish(struct qdio_initialize *init_data)
1233 {
1234         struct qdio_irq *irq_ptr;
1235         struct ccw_device *cdev = init_data->cdev;
1236         unsigned long saveflags;
1237         int rc;
1238
1239         DBF_EVENT("qestablish:%4x", cdev->private->schid.sch_no);
1240
1241         irq_ptr = cdev->private->qdio_data;
1242         if (!irq_ptr)
1243                 return -ENODEV;
1244
1245         if (cdev->private->state != DEV_STATE_ONLINE)
1246                 return -EINVAL;
1247
1248         mutex_lock(&irq_ptr->setup_mutex);
1249         qdio_setup_irq(init_data);
1250
1251         rc = qdio_establish_thinint(irq_ptr);
1252         if (rc) {
1253                 mutex_unlock(&irq_ptr->setup_mutex);
1254                 qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
1255                 return rc;
1256         }
1257
1258         /* establish q */
1259         irq_ptr->ccw.cmd_code = irq_ptr->equeue.cmd;
1260         irq_ptr->ccw.flags = CCW_FLAG_SLI;
1261         irq_ptr->ccw.count = irq_ptr->equeue.count;
1262         irq_ptr->ccw.cda = (u32)((addr_t)irq_ptr->qdr);
1263
1264         spin_lock_irqsave(get_ccwdev_lock(cdev), saveflags);
1265         ccw_device_set_options_mask(cdev, 0);
1266
1267         rc = ccw_device_start(cdev, &irq_ptr->ccw, QDIO_DOING_ESTABLISH, 0, 0);
1268         if (rc) {
1269                 DBF_ERROR("%4x est IO ERR", irq_ptr->schid.sch_no);
1270                 DBF_ERROR("rc:%4x", rc);
1271         }
1272         spin_unlock_irqrestore(get_ccwdev_lock(cdev), saveflags);
1273
1274         if (rc) {
1275                 mutex_unlock(&irq_ptr->setup_mutex);
1276                 qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
1277                 return rc;
1278         }
1279
1280         wait_event_interruptible_timeout(cdev->private->wait_q,
1281                 irq_ptr->state == QDIO_IRQ_STATE_ESTABLISHED ||
1282                 irq_ptr->state == QDIO_IRQ_STATE_ERR, HZ);
1283
1284         if (irq_ptr->state != QDIO_IRQ_STATE_ESTABLISHED) {
1285                 mutex_unlock(&irq_ptr->setup_mutex);
1286                 qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
1287                 return -EIO;
1288         }
1289
1290         qdio_setup_ssqd_info(irq_ptr);
1291         DBF_EVENT("qDmmwc:%2x", irq_ptr->ssqd_desc.mmwc);
1292         DBF_EVENT("qib ac:%4x", irq_ptr->qib.ac);
1293
1294         /* qebsm is now setup if available, initialize buffer states */
1295         qdio_init_buf_states(irq_ptr);
1296
1297         mutex_unlock(&irq_ptr->setup_mutex);
1298         qdio_print_subchannel_info(irq_ptr, cdev);
1299         qdio_setup_debug_entries(irq_ptr, cdev);
1300         return 0;
1301 }
1302 EXPORT_SYMBOL_GPL(qdio_establish);
1303
1304 /**
1305  * qdio_activate - activate queues on a qdio subchannel
1306  * @cdev: associated cdev
1307  */
1308 int qdio_activate(struct ccw_device *cdev)
1309 {
1310         struct qdio_irq *irq_ptr;
1311         int rc;
1312         unsigned long saveflags;
1313
1314         DBF_EVENT("qactivate:%4x", cdev->private->schid.sch_no);
1315
1316         irq_ptr = cdev->private->qdio_data;
1317         if (!irq_ptr)
1318                 return -ENODEV;
1319
1320         if (cdev->private->state != DEV_STATE_ONLINE)
1321                 return -EINVAL;
1322
1323         mutex_lock(&irq_ptr->setup_mutex);
1324         if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) {
1325                 rc = -EBUSY;
1326                 goto out;
1327         }
1328
1329         irq_ptr->ccw.cmd_code = irq_ptr->aqueue.cmd;
1330         irq_ptr->ccw.flags = CCW_FLAG_SLI;
1331         irq_ptr->ccw.count = irq_ptr->aqueue.count;
1332         irq_ptr->ccw.cda = 0;
1333
1334         spin_lock_irqsave(get_ccwdev_lock(cdev), saveflags);
1335         ccw_device_set_options(cdev, CCWDEV_REPORT_ALL);
1336
1337         rc = ccw_device_start(cdev, &irq_ptr->ccw, QDIO_DOING_ACTIVATE,
1338                               0, DOIO_DENY_PREFETCH);
1339         if (rc) {
1340                 DBF_ERROR("%4x act IO ERR", irq_ptr->schid.sch_no);
1341                 DBF_ERROR("rc:%4x", rc);
1342         }
1343         spin_unlock_irqrestore(get_ccwdev_lock(cdev), saveflags);
1344
1345         if (rc)
1346                 goto out;
1347
1348         if (is_thinint_irq(irq_ptr))
1349                 tiqdio_add_input_queues(irq_ptr);
1350
1351         /* wait for subchannel to become active */
1352         msleep(5);
1353
1354         switch (irq_ptr->state) {
1355         case QDIO_IRQ_STATE_STOPPED:
1356         case QDIO_IRQ_STATE_ERR:
1357                 rc = -EIO;
1358                 break;
1359         default:
1360                 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ACTIVE);
1361                 rc = 0;
1362         }
1363 out:
1364         mutex_unlock(&irq_ptr->setup_mutex);
1365         return rc;
1366 }
1367 EXPORT_SYMBOL_GPL(qdio_activate);
1368
1369 static inline int buf_in_between(int bufnr, int start, int count)
1370 {
1371         int end = add_buf(start, count);
1372
1373         if (end > start) {
1374                 if (bufnr >= start && bufnr < end)
1375                         return 1;
1376                 else
1377                         return 0;
1378         }
1379
1380         /* wrap-around case */
1381         if ((bufnr >= start && bufnr <= QDIO_MAX_BUFFERS_PER_Q) ||
1382             (bufnr < end))
1383                 return 1;
1384         else
1385                 return 0;
1386 }
1387
1388 /**
1389  * handle_inbound - reset processed input buffers
1390  * @q: queue containing the buffers
1391  * @callflags: flags
1392  * @bufnr: first buffer to process
1393  * @count: how many buffers are emptied
1394  */
1395 static int handle_inbound(struct qdio_q *q, unsigned int callflags,
1396                           int bufnr, int count)
1397 {
1398         int used, diff;
1399
1400         qperf_inc(q, inbound_call);
1401
1402         if (!q->u.in.polling)
1403                 goto set;
1404
1405         /* protect against stop polling setting an ACK for an emptied slsb */
1406         if (count == QDIO_MAX_BUFFERS_PER_Q) {
1407                 /* overwriting everything, just delete polling status */
1408                 q->u.in.polling = 0;
1409                 q->u.in.ack_count = 0;
1410                 goto set;
1411         } else if (buf_in_between(q->u.in.ack_start, bufnr, count)) {
1412                 if (is_qebsm(q)) {
1413                         /* partial overwrite, just update ack_start */
1414                         diff = add_buf(bufnr, count);
1415                         diff = sub_buf(diff, q->u.in.ack_start);
1416                         q->u.in.ack_count -= diff;
1417                         if (q->u.in.ack_count <= 0) {
1418                                 q->u.in.polling = 0;
1419                                 q->u.in.ack_count = 0;
1420                                 goto set;
1421                         }
1422                         q->u.in.ack_start = add_buf(q->u.in.ack_start, diff);
1423                 }
1424                 else
1425                         /* the only ACK will be deleted, so stop polling */
1426                         q->u.in.polling = 0;
1427         }
1428
1429 set:
1430         count = set_buf_states(q, bufnr, SLSB_CU_INPUT_EMPTY, count);
1431
1432         used = atomic_add_return(count, &q->nr_buf_used) - count;
1433         BUG_ON(used + count > QDIO_MAX_BUFFERS_PER_Q);
1434
1435         /* no need to signal as long as the adapter had free buffers */
1436         if (used)
1437                 return 0;
1438
1439         if (need_siga_in(q))
1440                 return qdio_siga_input(q);
1441         return 0;
1442 }
1443
1444 /**
1445  * handle_outbound - process filled outbound buffers
1446  * @q: queue containing the buffers
1447  * @callflags: flags
1448  * @bufnr: first buffer to process
1449  * @count: how many buffers are filled
1450  */
1451 static int handle_outbound(struct qdio_q *q, unsigned int callflags,
1452                            int bufnr, int count)
1453 {
1454         unsigned char state;
1455         int used, rc = 0;
1456
1457         qperf_inc(q, outbound_call);
1458
1459         count = set_buf_states(q, bufnr, SLSB_CU_OUTPUT_PRIMED, count);
1460         used = atomic_add_return(count, &q->nr_buf_used);
1461         BUG_ON(used > QDIO_MAX_BUFFERS_PER_Q);
1462
1463         if (used == QDIO_MAX_BUFFERS_PER_Q)
1464                 qperf_inc(q, outbound_queue_full);
1465
1466         if (callflags & QDIO_FLAG_PCI_OUT) {
1467                 q->u.out.pci_out_enabled = 1;
1468                 qperf_inc(q, pci_request_int);
1469         }
1470         else
1471                 q->u.out.pci_out_enabled = 0;
1472
1473         if (queue_type(q) == QDIO_IQDIO_QFMT) {
1474                 if (multicast_outbound(q))
1475                         rc = qdio_kick_outbound_q(q);
1476                 else
1477                         if ((q->irq_ptr->ssqd_desc.mmwc > 1) &&
1478                             (count > 1) &&
1479                             (count <= q->irq_ptr->ssqd_desc.mmwc)) {
1480                                 /* exploit enhanced SIGA */
1481                                 q->u.out.use_enh_siga = 1;
1482                                 rc = qdio_kick_outbound_q(q);
1483                         } else {
1484                                 /*
1485                                 * One siga-w per buffer required for unicast
1486                                 * HiperSockets.
1487                                 */
1488                                 q->u.out.use_enh_siga = 0;
1489                                 while (count--) {
1490                                         rc = qdio_kick_outbound_q(q);
1491                                         if (rc)
1492                                                 goto out;
1493                                 }
1494                         }
1495                 goto out;
1496         }
1497
1498         if (need_siga_sync(q)) {
1499                 qdio_siga_sync_q(q);
1500                 goto out;
1501         }
1502
1503         /* try to fast requeue buffers */
1504         get_buf_state(q, prev_buf(bufnr), &state, 0);
1505         if (state != SLSB_CU_OUTPUT_PRIMED)
1506                 rc = qdio_kick_outbound_q(q);
1507         else
1508                 qperf_inc(q, fast_requeue);
1509
1510 out:
1511         /* in case of SIGA errors we must process the error immediately */
1512         if (used >= q->u.out.scan_threshold || rc)
1513                 tasklet_schedule(&q->tasklet);
1514         else
1515                 /* free the SBALs in case of no further traffic */
1516                 if (!timer_pending(&q->u.out.timer))
1517                         mod_timer(&q->u.out.timer, jiffies + HZ);
1518         return rc;
1519 }
1520
1521 /**
1522  * do_QDIO - process input or output buffers
1523  * @cdev: associated ccw_device for the qdio subchannel
1524  * @callflags: input or output and special flags from the program
1525  * @q_nr: queue number
1526  * @bufnr: buffer number
1527  * @count: how many buffers to process
1528  */
1529 int do_QDIO(struct ccw_device *cdev, unsigned int callflags,
1530             int q_nr, unsigned int bufnr, unsigned int count)
1531 {
1532         struct qdio_irq *irq_ptr;
1533
1534         if (bufnr >= QDIO_MAX_BUFFERS_PER_Q || count > QDIO_MAX_BUFFERS_PER_Q)
1535                 return -EINVAL;
1536
1537         irq_ptr = cdev->private->qdio_data;
1538         if (!irq_ptr)
1539                 return -ENODEV;
1540
1541         DBF_DEV_EVENT(DBF_INFO, irq_ptr,
1542                       "do%02x b:%02x c:%02x", callflags, bufnr, count);
1543
1544         if (irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)
1545                 return -EBUSY;
1546
1547         if (callflags & QDIO_FLAG_SYNC_INPUT)
1548                 return handle_inbound(irq_ptr->input_qs[q_nr],
1549                                       callflags, bufnr, count);
1550         else if (callflags & QDIO_FLAG_SYNC_OUTPUT)
1551                 return handle_outbound(irq_ptr->output_qs[q_nr],
1552                                        callflags, bufnr, count);
1553         return -EINVAL;
1554 }
1555 EXPORT_SYMBOL_GPL(do_QDIO);
1556
1557 /**
1558  * qdio_start_irq - process input buffers
1559  * @cdev: associated ccw_device for the qdio subchannel
1560  * @nr: input queue number
1561  *
1562  * Return codes
1563  *   0 - success
1564  *   1 - irqs not started since new data is available
1565  */
1566 int qdio_start_irq(struct ccw_device *cdev, int nr)
1567 {
1568         struct qdio_q *q;
1569         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1570
1571         if (!irq_ptr)
1572                 return -ENODEV;
1573         q = irq_ptr->input_qs[nr];
1574
1575         WARN_ON(queue_irqs_enabled(q));
1576
1577         if (!shared_ind(q->irq_ptr->dsci))
1578                 xchg(q->irq_ptr->dsci, 0);
1579
1580         qdio_stop_polling(q);
1581         clear_bit(QDIO_QUEUE_IRQS_DISABLED, &q->u.in.queue_irq_state);
1582
1583         /*
1584          * We need to check again to not lose initiative after
1585          * resetting the ACK state.
1586          */
1587         if (!shared_ind(q->irq_ptr->dsci) && *q->irq_ptr->dsci)
1588                 goto rescan;
1589         if (!qdio_inbound_q_done(q))
1590                 goto rescan;
1591         return 0;
1592
1593 rescan:
1594         if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED,
1595                              &q->u.in.queue_irq_state))
1596                 return 0;
1597         else
1598                 return 1;
1599
1600 }
1601 EXPORT_SYMBOL(qdio_start_irq);
1602
1603 /**
1604  * qdio_get_next_buffers - process input buffers
1605  * @cdev: associated ccw_device for the qdio subchannel
1606  * @nr: input queue number
1607  * @bufnr: first filled buffer number
1608  * @error: buffers are in error state
1609  *
1610  * Return codes
1611  *   < 0 - error
1612  *   = 0 - no new buffers found
1613  *   > 0 - number of processed buffers
1614  */
1615 int qdio_get_next_buffers(struct ccw_device *cdev, int nr, int *bufnr,
1616                           int *error)
1617 {
1618         struct qdio_q *q;
1619         int start, end;
1620         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1621
1622         if (!irq_ptr)
1623                 return -ENODEV;
1624         q = irq_ptr->input_qs[nr];
1625         WARN_ON(queue_irqs_enabled(q));
1626
1627         qdio_sync_after_thinint(q);
1628
1629         /*
1630          * The interrupt could be caused by a PCI request. Check the
1631          * PCI capable outbound queues.
1632          */
1633         qdio_check_outbound_after_thinint(q);
1634
1635         if (!qdio_inbound_q_moved(q))
1636                 return 0;
1637
1638         /* Note: upper-layer MUST stop processing immediately here ... */
1639         if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))
1640                 return -EIO;
1641
1642         start = q->first_to_kick;
1643         end = q->first_to_check;
1644         *bufnr = start;
1645         *error = q->qdio_error;
1646
1647         /* for the next time */
1648         q->first_to_kick = end;
1649         q->qdio_error = 0;
1650         return sub_buf(end, start);
1651 }
1652 EXPORT_SYMBOL(qdio_get_next_buffers);
1653
1654 /**
1655  * qdio_stop_irq - disable interrupt processing for the device
1656  * @cdev: associated ccw_device for the qdio subchannel
1657  * @nr: input queue number
1658  *
1659  * Return codes
1660  *   0 - interrupts were already disabled
1661  *   1 - interrupts successfully disabled
1662  */
1663 int qdio_stop_irq(struct ccw_device *cdev, int nr)
1664 {
1665         struct qdio_q *q;
1666         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1667
1668         if (!irq_ptr)
1669                 return -ENODEV;
1670         q = irq_ptr->input_qs[nr];
1671
1672         if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED,
1673                              &q->u.in.queue_irq_state))
1674                 return 0;
1675         else
1676                 return 1;
1677 }
1678 EXPORT_SYMBOL(qdio_stop_irq);
1679
1680 static int __init init_QDIO(void)
1681 {
1682         int rc;
1683
1684         rc = qdio_setup_init();
1685         if (rc)
1686                 return rc;
1687         rc = tiqdio_allocate_memory();
1688         if (rc)
1689                 goto out_cache;
1690         rc = qdio_debug_init();
1691         if (rc)
1692                 goto out_ti;
1693         rc = tiqdio_register_thinints();
1694         if (rc)
1695                 goto out_debug;
1696         return 0;
1697
1698 out_debug:
1699         qdio_debug_exit();
1700 out_ti:
1701         tiqdio_free_memory();
1702 out_cache:
1703         qdio_setup_exit();
1704         return rc;
1705 }
1706
1707 static void __exit exit_QDIO(void)
1708 {
1709         tiqdio_unregister_thinints();
1710         tiqdio_free_memory();
1711         qdio_debug_exit();
1712         qdio_setup_exit();
1713 }
1714
1715 module_init(init_QDIO);
1716 module_exit(exit_QDIO);