drivers/target/iscsi/iscsi_target_erl0.c

   1 /******************************************************************************
   2  * This file contains error recovery level zero functions used by
   3  * the iSCSI Target driver.
   4  *
   5  * (c) Copyright 2007-2013 Datera, Inc.
   6  *
   7  * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  ******************************************************************************/
  19
  20 #include <scsi/iscsi_proto.h>
  21 #include <target/target_core_base.h>
  22 #include <target/target_core_fabric.h>
  23
  24 #include <target/iscsi/iscsi_target_core.h>
  25 #include <target/iscsi/iscsi_transport.h>
  26 #include "iscsi_target_seq_pdu_list.h"
  27 #include "iscsi_target_tq.h"
  28 #include "iscsi_target_erl0.h"
  29 #include "iscsi_target_erl1.h"
  30 #include "iscsi_target_erl2.h"
  31 #include "iscsi_target_util.h"
  32 #include "iscsi_target.h"
  33
  34 /*
  35  *      Used to set values in struct iscsi_cmd that iscsit_dataout_check_sequence()
  36  *      checks against to determine a PDU's Offset+Length is within the current
  37  *      DataOUT Sequence.  Used for DataSequenceInOrder=Yes only.
  38  */
  39 void iscsit_set_dataout_sequence_values(
  40         struct iscsi_cmd *cmd)
  41 {
  42         struct iscsi_conn *conn = cmd->conn;
  43         /*
  44          * Still set seq_start_offset and seq_end_offset for Unsolicited
  45          * DataOUT, even if DataSequenceInOrder=No.
  46          */
  47         if (cmd->unsolicited_data) {
  48                 cmd->seq_start_offset = cmd->write_data_done;
  49                 cmd->seq_end_offset = (cmd->write_data_done +
  50                         ((cmd->se_cmd.data_length >
  51                           conn->sess->sess_ops->FirstBurstLength) ?
  52                          conn->sess->sess_ops->FirstBurstLength : cmd->se_cmd.data_length));
  53                 return;
  54         }
  55
  56         if (!conn->sess->sess_ops->DataSequenceInOrder)
  57                 return;
  58
  59         if (!cmd->seq_start_offset && !cmd->seq_end_offset) {
  60                 cmd->seq_start_offset = cmd->write_data_done;
  61                 cmd->seq_end_offset = (cmd->se_cmd.data_length >
  62                         conn->sess->sess_ops->MaxBurstLength) ?
  63                         (cmd->write_data_done +
  64                         conn->sess->sess_ops->MaxBurstLength) : cmd->se_cmd.data_length;
  65         } else {
  66                 cmd->seq_start_offset = cmd->seq_end_offset;
  67                 cmd->seq_end_offset = ((cmd->seq_end_offset +
  68                         conn->sess->sess_ops->MaxBurstLength) >=
  69                         cmd->se_cmd.data_length) ? cmd->se_cmd.data_length :
  70                         (cmd->seq_end_offset +
  71                          conn->sess->sess_ops->MaxBurstLength);
  72         }
  73 }
  74
  75 static int iscsit_dataout_within_command_recovery_check(
  76         struct iscsi_cmd *cmd,
  77         unsigned char *buf)
  78 {
  79         struct iscsi_conn *conn = cmd->conn;
  80         struct iscsi_data *hdr = (struct iscsi_data *) buf;
  81         u32 payload_length = ntoh24(hdr->dlength);
  82
  83         /*
  84          * We do the within-command recovery checks here as it is
  85          * the first function called in iscsi_check_pre_dataout().
  86          * Basically, if we are in within-command recovery and
  87          * the PDU does not contain the offset the sequence needs,
  88          * dump the payload.
  89          *
  90          * This only applies to DataPDUInOrder=Yes, for
  91          * DataPDUInOrder=No we only re-request the failed PDU
  92          * and check that all PDUs in a sequence are received
  93          * upon end of sequence.
  94          */
  95         if (conn->sess->sess_ops->DataSequenceInOrder) {
  96                 if ((cmd->cmd_flags & ICF_WITHIN_COMMAND_RECOVERY) &&
  97                     cmd->write_data_done != be32_to_cpu(hdr->offset))
  98                         goto dump;
  99
 100                 cmd->cmd_flags &= ~ICF_WITHIN_COMMAND_RECOVERY;
 101         } else {
 102                 struct iscsi_seq *seq;
 103
 104                 seq = iscsit_get_seq_holder(cmd, be32_to_cpu(hdr->offset),
 105                                             payload_length);
 106                 if (!seq)
 107                         return DATAOUT_CANNOT_RECOVER;
 108                 /*
 109                  * Set the struct iscsi_seq pointer to reuse later.
 110                  */
 111                 cmd->seq_ptr = seq;
 112
 113                 if (conn->sess->sess_ops->DataPDUInOrder) {
 114                         if (seq->status ==
 115                             DATAOUT_SEQUENCE_WITHIN_COMMAND_RECOVERY &&
 116                            (seq->offset != be32_to_cpu(hdr->offset) ||
 117                             seq->data_sn != be32_to_cpu(hdr->datasn)))
 118                                 goto dump;
 119                 } else {
 120                         if (seq->status ==
 121                              DATAOUT_SEQUENCE_WITHIN_COMMAND_RECOVERY &&
 122                             seq->data_sn != be32_to_cpu(hdr->datasn))
 123                                 goto dump;
 124                 }
 125
 126                 if (seq->status == DATAOUT_SEQUENCE_COMPLETE)
 127                         goto dump;
 128
 129                 if (seq->status != DATAOUT_SEQUENCE_COMPLETE)
 130                         seq->status = 0;
 131         }
 132
 133         return DATAOUT_NORMAL;
 134
 135 dump:
 136         pr_err("Dumping DataOUT PDU Offset: %u Length: %d DataSN:"
 137                 " 0x%08x\n", hdr->offset, payload_length, hdr->datasn);
 138         return iscsit_dump_data_payload(conn, payload_length, 1);
 139 }
 140
 141 static int iscsit_dataout_check_unsolicited_sequence(
 142         struct iscsi_cmd *cmd,
 143         unsigned char *buf)
 144 {
 145         u32 first_burst_len;
 146         struct iscsi_conn *conn = cmd->conn;
 147         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 148         u32 payload_length = ntoh24(hdr->dlength);
 149
 150
 151         if ((be32_to_cpu(hdr->offset) < cmd->seq_start_offset) ||
 152            ((be32_to_cpu(hdr->offset) + payload_length) > cmd->seq_end_offset)) {
 153                 pr_err("Command ITT: 0x%08x with Offset: %u,"
 154                 " Length: %u outside of Unsolicited Sequence %u:%u while"
 155                 " DataSequenceInOrder=Yes.\n", cmd->init_task_tag,
 156                 be32_to_cpu(hdr->offset), payload_length, cmd->seq_start_offset,
 157                         cmd->seq_end_offset);
 158                 return DATAOUT_CANNOT_RECOVER;
 159         }
 160
 161         first_burst_len = (cmd->first_burst_len + payload_length);
 162
 163         if (first_burst_len > conn->sess->sess_ops->FirstBurstLength) {
 164                 pr_err("Total %u bytes exceeds FirstBurstLength: %u"
 165                         " for this Unsolicited DataOut Burst.\n",
 166                         first_burst_len, conn->sess->sess_ops->FirstBurstLength);
 167                 transport_send_check_condition_and_sense(&cmd->se_cmd,
 168                                 TCM_INCORRECT_AMOUNT_OF_DATA, 0);
 169                 return DATAOUT_CANNOT_RECOVER;
 170         }
 171
 172         /*
 173          * Perform various MaxBurstLength and ISCSI_FLAG_CMD_FINAL sanity
 174          * checks for the current Unsolicited DataOUT Sequence.
 175          */
 176         if (hdr->flags & ISCSI_FLAG_CMD_FINAL) {
 177                 /*
 178                  * Ignore ISCSI_FLAG_CMD_FINAL checks while DataPDUInOrder=No, end of
 179                  * sequence checks are handled in
 180                  * iscsit_dataout_datapduinorder_no_fbit().
 181                  */
 182                 if (!conn->sess->sess_ops->DataPDUInOrder)
 183                         goto out;
 184
 185                 if ((first_burst_len != cmd->se_cmd.data_length) &&
 186                     (first_burst_len != conn->sess->sess_ops->FirstBurstLength)) {
 187                         pr_err("Unsolicited non-immediate data"
 188                         " received %u does not equal FirstBurstLength: %u, and"
 189                         " does not equal ExpXferLen %u.\n", first_burst_len,
 190                                 conn->sess->sess_ops->FirstBurstLength,
 191                                 cmd->se_cmd.data_length);
 192                         transport_send_check_condition_and_sense(&cmd->se_cmd,
 193                                         TCM_INCORRECT_AMOUNT_OF_DATA, 0);
 194                         return DATAOUT_CANNOT_RECOVER;
 195                 }
 196         } else {
 197                 if (first_burst_len == conn->sess->sess_ops->FirstBurstLength) {
 198                         pr_err("Command ITT: 0x%08x reached"
 199                         " FirstBurstLength: %u, but ISCSI_FLAG_CMD_FINAL is not set. protocol"
 200                                 " error.\n", cmd->init_task_tag,
 201                                 conn->sess->sess_ops->FirstBurstLength);
 202                         return DATAOUT_CANNOT_RECOVER;
 203                 }
 204                 if (first_burst_len == cmd->se_cmd.data_length) {
 205                         pr_err("Command ITT: 0x%08x reached"
 206                         " ExpXferLen: %u, but ISCSI_FLAG_CMD_FINAL is not set. protocol"
 207                         " error.\n", cmd->init_task_tag, cmd->se_cmd.data_length);
 208                         return DATAOUT_CANNOT_RECOVER;
 209                 }
 210         }
 211
 212 out:
 213         return DATAOUT_NORMAL;
 214 }
 215
 216 static int iscsit_dataout_check_sequence(
 217         struct iscsi_cmd *cmd,
 218         unsigned char *buf)
 219 {
 220         u32 next_burst_len;
 221         struct iscsi_conn *conn = cmd->conn;
 222         struct iscsi_seq *seq = NULL;
 223         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 224         u32 payload_length = ntoh24(hdr->dlength);
 225
 226         /*
 227          * For DataSequenceInOrder=Yes: Check that the offset and offset+length
 228          * is within range as defined by iscsi_set_dataout_sequence_values().
 229          *
 230          * For DataSequenceInOrder=No: Check that an struct iscsi_seq exists for
 231          * offset+length tuple.
 232          */
 233         if (conn->sess->sess_ops->DataSequenceInOrder) {
 234                 /*
 235                  * Due to possibility of recovery DataOUT sent by the initiator
 236                  * fullfilling an Recovery R2T, it's best to just dump the
 237                  * payload here, instead of erroring out.
 238                  */
 239                 if ((be32_to_cpu(hdr->offset) < cmd->seq_start_offset) ||
 240                    ((be32_to_cpu(hdr->offset) + payload_length) > cmd->seq_end_offset)) {
 241                         pr_err("Command ITT: 0x%08x with Offset: %u,"
 242                         " Length: %u outside of Sequence %u:%u while"
 243                         " DataSequenceInOrder=Yes.\n", cmd->init_task_tag,
 244                         be32_to_cpu(hdr->offset), payload_length, cmd->seq_start_offset,
 245                                 cmd->seq_end_offset);
 246
 247                         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 248                                 return DATAOUT_CANNOT_RECOVER;
 249                         return DATAOUT_WITHIN_COMMAND_RECOVERY;
 250                 }
 251
 252                 next_burst_len = (cmd->next_burst_len + payload_length);
 253         } else {
 254                 seq = iscsit_get_seq_holder(cmd, be32_to_cpu(hdr->offset),
 255                                             payload_length);
 256                 if (!seq)
 257                         return DATAOUT_CANNOT_RECOVER;
 258                 /*
 259                  * Set the struct iscsi_seq pointer to reuse later.
 260                  */
 261                 cmd->seq_ptr = seq;
 262
 263                 if (seq->status == DATAOUT_SEQUENCE_COMPLETE) {
 264                         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 265                                 return DATAOUT_CANNOT_RECOVER;
 266                         return DATAOUT_WITHIN_COMMAND_RECOVERY;
 267                 }
 268
 269                 next_burst_len = (seq->next_burst_len + payload_length);
 270         }
 271
 272         if (next_burst_len > conn->sess->sess_ops->MaxBurstLength) {
 273                 pr_err("Command ITT: 0x%08x, NextBurstLength: %u and"
 274                         " Length: %u exceeds MaxBurstLength: %u. protocol"
 275                         " error.\n", cmd->init_task_tag,
 276                         (next_burst_len - payload_length),
 277                         payload_length, conn->sess->sess_ops->MaxBurstLength);
 278                 return DATAOUT_CANNOT_RECOVER;
 279         }
 280
 281         /*
 282          * Perform various MaxBurstLength and ISCSI_FLAG_CMD_FINAL sanity
 283          * checks for the current DataOUT Sequence.
 284          */
 285         if (hdr->flags & ISCSI_FLAG_CMD_FINAL) {
 286                 /*
 287                  * Ignore ISCSI_FLAG_CMD_FINAL checks while DataPDUInOrder=No, end of
 288                  * sequence checks are handled in
 289                  * iscsit_dataout_datapduinorder_no_fbit().
 290                  */
 291                 if (!conn->sess->sess_ops->DataPDUInOrder)
 292                         goto out;
 293
 294                 if (conn->sess->sess_ops->DataSequenceInOrder) {
 295                         if ((next_burst_len <
 296                              conn->sess->sess_ops->MaxBurstLength) &&
 297                            ((cmd->write_data_done + payload_length) <
 298                              cmd->se_cmd.data_length)) {
 299                                 pr_err("Command ITT: 0x%08x set ISCSI_FLAG_CMD_FINAL"
 300                                 " before end of DataOUT sequence, protocol"
 301                                 " error.\n", cmd->init_task_tag);
 302                                 return DATAOUT_CANNOT_RECOVER;
 303                         }
 304                 } else {
 305                         if (next_burst_len < seq->xfer_len) {
 306                                 pr_err("Command ITT: 0x%08x set ISCSI_FLAG_CMD_FINAL"
 307                                 " before end of DataOUT sequence, protocol"
 308                                 " error.\n", cmd->init_task_tag);
 309                                 return DATAOUT_CANNOT_RECOVER;
 310                         }
 311                 }
 312         } else {
 313                 if (conn->sess->sess_ops->DataSequenceInOrder) {
 314                         if (next_burst_len ==
 315                                         conn->sess->sess_ops->MaxBurstLength) {
 316                                 pr_err("Command ITT: 0x%08x reached"
 317                                 " MaxBurstLength: %u, but ISCSI_FLAG_CMD_FINAL is"
 318                                 " not set, protocol error.", cmd->init_task_tag,
 319                                         conn->sess->sess_ops->MaxBurstLength);
 320                                 return DATAOUT_CANNOT_RECOVER;
 321                         }
 322                         if ((cmd->write_data_done + payload_length) ==
 323                                         cmd->se_cmd.data_length) {
 324                                 pr_err("Command ITT: 0x%08x reached"
 325                                 " last DataOUT PDU in sequence but ISCSI_FLAG_"
 326                                 "CMD_FINAL is not set, protocol error.\n",
 327                                         cmd->init_task_tag);
 328                                 return DATAOUT_CANNOT_RECOVER;
 329                         }
 330                 } else {
 331                         if (next_burst_len == seq->xfer_len) {
 332                                 pr_err("Command ITT: 0x%08x reached"
 333                                 " last DataOUT PDU in sequence but ISCSI_FLAG_"
 334                                 "CMD_FINAL is not set, protocol error.\n",
 335                                         cmd->init_task_tag);
 336                                 return DATAOUT_CANNOT_RECOVER;
 337                         }
 338                 }
 339         }
 340
 341 out:
 342         return DATAOUT_NORMAL;
 343 }
 344
 345 static int iscsit_dataout_check_datasn(
 346         struct iscsi_cmd *cmd,
 347         unsigned char *buf)
 348 {
 349         u32 data_sn = 0;
 350         struct iscsi_conn *conn = cmd->conn;
 351         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 352         u32 payload_length = ntoh24(hdr->dlength);
 353
 354         /*
 355          * Considering the target has no method of re-requesting DataOUT
 356          * by DataSN, if we receieve a greater DataSN than expected we
 357          * assume the functions for DataPDUInOrder=[Yes,No] below will
 358          * handle it.
 359          *
 360          * If the DataSN is less than expected, dump the payload.
 361          */
 362         if (conn->sess->sess_ops->DataSequenceInOrder)
 363                 data_sn = cmd->data_sn;
 364         else {
 365                 struct iscsi_seq *seq = cmd->seq_ptr;
 366                 data_sn = seq->data_sn;
 367         }
 368
 369         if (be32_to_cpu(hdr->datasn) > data_sn) {
 370                 pr_err("Command ITT: 0x%08x, received DataSN: 0x%08x"
 371                         " higher than expected 0x%08x.\n", cmd->init_task_tag,
 372                                 be32_to_cpu(hdr->datasn), data_sn);
 373                 goto recover;
 374         } else if (be32_to_cpu(hdr->datasn) < data_sn) {
 375                 pr_err("Command ITT: 0x%08x, received DataSN: 0x%08x"
 376                         " lower than expected 0x%08x, discarding payload.\n",
 377                         cmd->init_task_tag, be32_to_cpu(hdr->datasn), data_sn);
 378                 goto dump;
 379         }
 380
 381         return DATAOUT_NORMAL;
 382
 383 recover:
 384         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 385                 pr_err("Unable to perform within-command recovery"
 386                                 " while ERL=0.\n");
 387                 return DATAOUT_CANNOT_RECOVER;
 388         }
 389 dump:
 390         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 391                 return DATAOUT_CANNOT_RECOVER;
 392
 393         return DATAOUT_WITHIN_COMMAND_RECOVERY;
 394 }
 395
 396 static int iscsit_dataout_pre_datapduinorder_yes(
 397         struct iscsi_cmd *cmd,
 398         unsigned char *buf)
 399 {
 400         int dump = 0, recovery = 0;
 401         struct iscsi_conn *conn = cmd->conn;
 402         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 403         u32 payload_length = ntoh24(hdr->dlength);
 404
 405         /*
 406          * For DataSequenceInOrder=Yes: If the offset is greater than the global
 407          * DataPDUInOrder=Yes offset counter in struct iscsi_cmd a protcol error has
 408          * occurred and fail the connection.
 409          *
 410          * For DataSequenceInOrder=No: If the offset is greater than the per
 411          * sequence DataPDUInOrder=Yes offset counter in struct iscsi_seq a protocol
 412          * error has occurred and fail the connection.
 413          */
 414         if (conn->sess->sess_ops->DataSequenceInOrder) {
 415                 if (be32_to_cpu(hdr->offset) != cmd->write_data_done) {
 416                         pr_err("Command ITT: 0x%08x, received offset"
 417                         " %u different than expected %u.\n", cmd->init_task_tag,
 418                                 be32_to_cpu(hdr->offset), cmd->write_data_done);
 419                         recovery = 1;
 420                         goto recover;
 421                 }
 422         } else {
 423                 struct iscsi_seq *seq = cmd->seq_ptr;
 424
 425                 if (be32_to_cpu(hdr->offset) > seq->offset) {
 426                         pr_err("Command ITT: 0x%08x, received offset"
 427                         " %u greater than expected %u.\n", cmd->init_task_tag,
 428                                 be32_to_cpu(hdr->offset), seq->offset);
 429                         recovery = 1;
 430                         goto recover;
 431                 } else if (be32_to_cpu(hdr->offset) < seq->offset) {
 432                         pr_err("Command ITT: 0x%08x, received offset"
 433                         " %u less than expected %u, discarding payload.\n",
 434                                 cmd->init_task_tag, be32_to_cpu(hdr->offset),
 435                                 seq->offset);
 436                         dump = 1;
 437                         goto dump;
 438                 }
 439         }
 440
 441         return DATAOUT_NORMAL;
 442
 443 recover:
 444         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 445                 pr_err("Unable to perform within-command recovery"
 446                                 " while ERL=0.\n");
 447                 return DATAOUT_CANNOT_RECOVER;
 448         }
 449 dump:
 450         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 451                 return DATAOUT_CANNOT_RECOVER;
 452
 453         return (recovery) ? iscsit_recover_dataout_sequence(cmd,
 454                 be32_to_cpu(hdr->offset), payload_length) :
 455                (dump) ? DATAOUT_WITHIN_COMMAND_RECOVERY : DATAOUT_NORMAL;
 456 }
 457
 458 static int iscsit_dataout_pre_datapduinorder_no(
 459         struct iscsi_cmd *cmd,
 460         unsigned char *buf)
 461 {
 462         struct iscsi_pdu *pdu;
 463         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 464         u32 payload_length = ntoh24(hdr->dlength);
 465
 466         pdu = iscsit_get_pdu_holder(cmd, be32_to_cpu(hdr->offset),
 467                                     payload_length);
 468         if (!pdu)
 469                 return DATAOUT_CANNOT_RECOVER;
 470
 471         cmd->pdu_ptr = pdu;
 472
 473         switch (pdu->status) {
 474         case ISCSI_PDU_NOT_RECEIVED:
 475         case ISCSI_PDU_CRC_FAILED:
 476         case ISCSI_PDU_TIMED_OUT:
 477                 break;
 478         case ISCSI_PDU_RECEIVED_OK:
 479                 pr_err("Command ITT: 0x%08x received already gotten"
 480                         " Offset: %u, Length: %u\n", cmd->init_task_tag,
 481                                 be32_to_cpu(hdr->offset), payload_length);
 482                 return iscsit_dump_data_payload(cmd->conn, payload_length, 1);
 483         default:
 484                 return DATAOUT_CANNOT_RECOVER;
 485         }
 486
 487         return DATAOUT_NORMAL;
 488 }
 489
 490 static int iscsit_dataout_update_r2t(struct iscsi_cmd *cmd, u32 offset, u32 length)
 491 {
 492         struct iscsi_r2t *r2t;
 493
 494         if (cmd->unsolicited_data)
 495                 return 0;
 496
 497         r2t = iscsit_get_r2t_for_eos(cmd, offset, length);
 498         if (!r2t)
 499                 return -1;
 500
 501         spin_lock_bh(&cmd->r2t_lock);
 502         r2t->seq_complete = 1;
 503         cmd->outstanding_r2ts--;
 504         spin_unlock_bh(&cmd->r2t_lock);
 505
 506         return 0;
 507 }
 508
 509 static int iscsit_dataout_update_datapduinorder_no(
 510         struct iscsi_cmd *cmd,
 511         u32 data_sn,
 512         int f_bit)
 513 {
 514         int ret = 0;
 515         struct iscsi_pdu *pdu = cmd->pdu_ptr;
 516
 517         pdu->data_sn = data_sn;
 518
 519         switch (pdu->status) {
 520         case ISCSI_PDU_NOT_RECEIVED:
 521                 pdu->status = ISCSI_PDU_RECEIVED_OK;
 522                 break;
 523         case ISCSI_PDU_CRC_FAILED:
 524                 pdu->status = ISCSI_PDU_RECEIVED_OK;
 525                 break;
 526         case ISCSI_PDU_TIMED_OUT:
 527                 pdu->status = ISCSI_PDU_RECEIVED_OK;
 528                 break;
 529         default:
 530                 return DATAOUT_CANNOT_RECOVER;
 531         }
 532
 533         if (f_bit) {
 534                 ret = iscsit_dataout_datapduinorder_no_fbit(cmd, pdu);
 535                 if (ret == DATAOUT_CANNOT_RECOVER)
 536                         return ret;
 537         }
 538
 539         return DATAOUT_NORMAL;
 540 }
 541
 542 static int iscsit_dataout_post_crc_passed(
 543         struct iscsi_cmd *cmd,
 544         unsigned char *buf)
 545 {
 546         int ret, send_r2t = 0;
 547         struct iscsi_conn *conn = cmd->conn;
 548         struct iscsi_seq *seq = NULL;
 549         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 550         u32 payload_length = ntoh24(hdr->dlength);
 551
 552         if (cmd->unsolicited_data) {
 553                 if ((cmd->first_burst_len + payload_length) ==
 554                      conn->sess->sess_ops->FirstBurstLength) {
 555                         if (iscsit_dataout_update_r2t(cmd, be32_to_cpu(hdr->offset),
 556                                         payload_length) < 0)
 557                                 return DATAOUT_CANNOT_RECOVER;
 558                         send_r2t = 1;
 559                 }
 560
 561                 if (!conn->sess->sess_ops->DataPDUInOrder) {
 562                         ret = iscsit_dataout_update_datapduinorder_no(cmd,
 563                                 be32_to_cpu(hdr->datasn),
 564                                 (hdr->flags & ISCSI_FLAG_CMD_FINAL));
 565                         if (ret == DATAOUT_CANNOT_RECOVER)
 566                                 return ret;
 567                 }
 568
 569                 cmd->first_burst_len += payload_length;
 570
 571                 if (conn->sess->sess_ops->DataSequenceInOrder)
 572                         cmd->data_sn++;
 573                 else {
 574                         seq = cmd->seq_ptr;
 575                         seq->data_sn++;
 576                         seq->offset += payload_length;
 577                 }
 578
 579                 if (send_r2t) {
 580                         if (seq)
 581                                 seq->status = DATAOUT_SEQUENCE_COMPLETE;
 582                         cmd->first_burst_len = 0;
 583                         cmd->unsolicited_data = 0;
 584                 }
 585         } else {
 586                 if (conn->sess->sess_ops->DataSequenceInOrder) {
 587                         if ((cmd->next_burst_len + payload_length) ==
 588                              conn->sess->sess_ops->MaxBurstLength) {
 589                                 if (iscsit_dataout_update_r2t(cmd,
 590                                                 be32_to_cpu(hdr->offset),
 591                                                 payload_length) < 0)
 592                                         return DATAOUT_CANNOT_RECOVER;
 593                                 send_r2t = 1;
 594                         }
 595
 596                         if (!conn->sess->sess_ops->DataPDUInOrder) {
 597                                 ret = iscsit_dataout_update_datapduinorder_no(
 598                                                 cmd, be32_to_cpu(hdr->datasn),
 599                                                 (hdr->flags & ISCSI_FLAG_CMD_FINAL));
 600                                 if (ret == DATAOUT_CANNOT_RECOVER)
 601                                         return ret;
 602                         }
 603
 604                         cmd->next_burst_len += payload_length;
 605                         cmd->data_sn++;
 606
 607                         if (send_r2t)
 608                                 cmd->next_burst_len = 0;
 609                 } else {
 610                         seq = cmd->seq_ptr;
 611
 612                         if ((seq->next_burst_len + payload_length) ==
 613                              seq->xfer_len) {
 614                                 if (iscsit_dataout_update_r2t(cmd,
 615                                                 be32_to_cpu(hdr->offset),
 616                                                 payload_length) < 0)
 617                                         return DATAOUT_CANNOT_RECOVER;
 618                                 send_r2t = 1;
 619                         }
 620
 621                         if (!conn->sess->sess_ops->DataPDUInOrder) {
 622                                 ret = iscsit_dataout_update_datapduinorder_no(
 623                                                 cmd, be32_to_cpu(hdr->datasn),
 624                                                 (hdr->flags & ISCSI_FLAG_CMD_FINAL));
 625                                 if (ret == DATAOUT_CANNOT_RECOVER)
 626                                         return ret;
 627                         }
 628
 629                         seq->data_sn++;
 630                         seq->offset += payload_length;
 631                         seq->next_burst_len += payload_length;
 632
 633                         if (send_r2t) {
 634                                 seq->next_burst_len = 0;
 635                                 seq->status = DATAOUT_SEQUENCE_COMPLETE;
 636                         }
 637                 }
 638         }
 639
 640         if (send_r2t && conn->sess->sess_ops->DataSequenceInOrder)
 641                 cmd->data_sn = 0;
 642
 643         cmd->write_data_done += payload_length;
 644
 645         if (cmd->write_data_done == cmd->se_cmd.data_length)
 646                 return DATAOUT_SEND_TO_TRANSPORT;
 647         else if (send_r2t)
 648                 return DATAOUT_SEND_R2T;
 649         else
 650                 return DATAOUT_NORMAL;
 651 }
 652
 653 static int iscsit_dataout_post_crc_failed(
 654         struct iscsi_cmd *cmd,
 655         unsigned char *buf)
 656 {
 657         struct iscsi_conn *conn = cmd->conn;
 658         struct iscsi_pdu *pdu;
 659         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 660         u32 payload_length = ntoh24(hdr->dlength);
 661
 662         if (conn->sess->sess_ops->DataPDUInOrder)
 663                 goto recover;
 664         /*
 665          * The rest of this function is only called when DataPDUInOrder=No.
 666          */
 667         pdu = cmd->pdu_ptr;
 668
 669         switch (pdu->status) {
 670         case ISCSI_PDU_NOT_RECEIVED:
 671                 pdu->status = ISCSI_PDU_CRC_FAILED;
 672                 break;
 673         case ISCSI_PDU_CRC_FAILED:
 674                 break;
 675         case ISCSI_PDU_TIMED_OUT:
 676                 pdu->status = ISCSI_PDU_CRC_FAILED;
 677                 break;
 678         default:
 679                 return DATAOUT_CANNOT_RECOVER;
 680         }
 681
 682 recover:
 683         return iscsit_recover_dataout_sequence(cmd, be32_to_cpu(hdr->offset),
 684                                                 payload_length);
 685 }
 686
 687 /*
 688  *      Called from iscsit_handle_data_out() before DataOUT Payload is received
 689  *      and CRC computed.
 690  */
 691 int iscsit_check_pre_dataout(
 692         struct iscsi_cmd *cmd,
 693         unsigned char *buf)
 694 {
 695         int ret;
 696         struct iscsi_conn *conn = cmd->conn;
 697
 698         ret = iscsit_dataout_within_command_recovery_check(cmd, buf);
 699         if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 700             (ret == DATAOUT_CANNOT_RECOVER))
 701                 return ret;
 702
 703         ret = iscsit_dataout_check_datasn(cmd, buf);
 704         if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 705             (ret == DATAOUT_CANNOT_RECOVER))
 706                 return ret;
 707
 708         if (cmd->unsolicited_data) {
 709                 ret = iscsit_dataout_check_unsolicited_sequence(cmd, buf);
 710                 if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 711                     (ret == DATAOUT_CANNOT_RECOVER))
 712                         return ret;
 713         } else {
 714                 ret = iscsit_dataout_check_sequence(cmd, buf);
 715                 if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 716                     (ret == DATAOUT_CANNOT_RECOVER))
 717                         return ret;
 718         }
 719
 720         return (conn->sess->sess_ops->DataPDUInOrder) ?
 721                 iscsit_dataout_pre_datapduinorder_yes(cmd, buf) :
 722                 iscsit_dataout_pre_datapduinorder_no(cmd, buf);
 723 }
 724
 725 /*
 726  *      Called from iscsit_handle_data_out() after DataOUT Payload is received
 727  *      and CRC computed.
 728  */
 729 int iscsit_check_post_dataout(
 730         struct iscsi_cmd *cmd,
 731         unsigned char *buf,
 732         u8 data_crc_failed)
 733 {
 734         struct iscsi_conn *conn = cmd->conn;
 735
 736         cmd->dataout_timeout_retries = 0;
 737
 738         if (!data_crc_failed)
 739                 return iscsit_dataout_post_crc_passed(cmd, buf);
 740         else {
 741                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 742                         pr_err("Unable to recover from DataOUT CRC"
 743                                 " failure while ERL=0, closing session.\n");
 744                         iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
 745                                           buf);
 746                         return DATAOUT_CANNOT_RECOVER;
 747                 }
 748
 749                 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, buf);
 750                 return iscsit_dataout_post_crc_failed(cmd, buf);
 751         }
 752 }
 753
 754 static void iscsit_handle_time2retain_timeout(unsigned long data)
 755 {
 756         struct iscsi_session *sess = (struct iscsi_session *) data;
 757         struct iscsi_portal_group *tpg = sess->tpg;
 758         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
 759
 760         spin_lock_bh(&se_tpg->session_lock);
 761         if (sess->time2retain_timer_flags & ISCSI_TF_STOP) {
 762                 spin_unlock_bh(&se_tpg->session_lock);
 763                 return;
 764         }
 765         if (atomic_read(&sess->session_reinstatement)) {
 766                 pr_err("Exiting Time2Retain handler because"
 767                                 " session_reinstatement=1\n");
 768                 spin_unlock_bh(&se_tpg->session_lock);
 769                 return;
 770         }
 771         sess->time2retain_timer_flags |= ISCSI_TF_EXPIRED;
 772
 773         pr_err("Time2Retain timer expired for SID: %u, cleaning up"
 774                         " iSCSI session.\n", sess->sid);
 775         {
 776         struct iscsi_tiqn *tiqn = tpg->tpg_tiqn;
 777
 778         if (tiqn) {
 779                 spin_lock(&tiqn->sess_err_stats.lock);
 780                 strcpy(tiqn->sess_err_stats.last_sess_fail_rem_name,
 781                         (void *)sess->sess_ops->InitiatorName);
 782                 tiqn->sess_err_stats.last_sess_failure_type =
 783                                 ISCSI_SESS_ERR_CXN_TIMEOUT;
 784                 tiqn->sess_err_stats.cxn_timeout_errors++;
 785                 atomic_long_inc(&sess->conn_timeout_errors);
 786                 spin_unlock(&tiqn->sess_err_stats.lock);
 787         }
 788         }
 789
 790         spin_unlock_bh(&se_tpg->session_lock);
 791         target_put_session(sess->se_sess);
 792 }
 793
 794 void iscsit_start_time2retain_handler(struct iscsi_session *sess)
 795 {
 796         int tpg_active;
 797         /*
 798          * Only start Time2Retain timer when the associated TPG is still in
 799          * an ACTIVE (eg: not disabled or shutdown) state.
 800          */
 801         spin_lock(&sess->tpg->tpg_state_lock);
 802         tpg_active = (sess->tpg->tpg_state == TPG_STATE_ACTIVE);
 803         spin_unlock(&sess->tpg->tpg_state_lock);
 804
 805         if (!tpg_active)
 806                 return;
 807
 808         if (sess->time2retain_timer_flags & ISCSI_TF_RUNNING)
 809                 return;
 810
 811         pr_debug("Starting Time2Retain timer for %u seconds on"
 812                 " SID: %u\n", sess->sess_ops->DefaultTime2Retain, sess->sid);
 813
 814         init_timer(&sess->time2retain_timer);
 815         sess->time2retain_timer.expires =
 816                 (get_jiffies_64() + sess->sess_ops->DefaultTime2Retain * HZ);
 817         sess->time2retain_timer.data = (unsigned long)sess;
 818         sess->time2retain_timer.function = iscsit_handle_time2retain_timeout;
 819         sess->time2retain_timer_flags &= ~ISCSI_TF_STOP;
 820         sess->time2retain_timer_flags |= ISCSI_TF_RUNNING;
 821         add_timer(&sess->time2retain_timer);
 822 }
 823
 824 /*
 825  *      Called with spin_lock_bh(&struct se_portal_group->session_lock) held
 826  */
 827 int iscsit_stop_time2retain_timer(struct iscsi_session *sess)
 828 {
 829         struct iscsi_portal_group *tpg = sess->tpg;
 830         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
 831
 832         if (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)
 833                 return -1;
 834
 835         if (!(sess->time2retain_timer_flags & ISCSI_TF_RUNNING))
 836                 return 0;
 837
 838         sess->time2retain_timer_flags |= ISCSI_TF_STOP;
 839         spin_unlock(&se_tpg->session_lock);
 840
 841         del_timer_sync(&sess->time2retain_timer);
 842
 843         spin_lock(&se_tpg->session_lock);
 844         sess->time2retain_timer_flags &= ~ISCSI_TF_RUNNING;
 845         pr_debug("Stopped Time2Retain Timer for SID: %u\n",
 846                         sess->sid);
 847         return 0;
 848 }
 849
 850 void iscsit_connection_reinstatement_rcfr(struct iscsi_conn *conn)
 851 {
 852         spin_lock_bh(&conn->state_lock);
 853         if (atomic_read(&conn->connection_exit)) {
 854                 spin_unlock_bh(&conn->state_lock);
 855                 goto sleep;
 856         }
 857
 858         if (atomic_read(&conn->transport_failed)) {
 859                 spin_unlock_bh(&conn->state_lock);
 860                 goto sleep;
 861         }
 862         spin_unlock_bh(&conn->state_lock);
 863
 864         iscsi_thread_set_force_reinstatement(conn);
 865
 866 sleep:
 867         wait_for_completion(&conn->conn_wait_rcfr_comp);
 868         complete(&conn->conn_post_wait_comp);
 869 }
 870
 871 void iscsit_cause_connection_reinstatement(struct iscsi_conn *conn, int sleep)
 872 {
 873         spin_lock_bh(&conn->state_lock);
 874         if (atomic_read(&conn->connection_exit)) {
 875                 spin_unlock_bh(&conn->state_lock);
 876                 return;
 877         }
 878
 879         if (atomic_read(&conn->transport_failed)) {
 880                 spin_unlock_bh(&conn->state_lock);
 881                 return;
 882         }
 883
 884         if (atomic_read(&conn->connection_reinstatement)) {
 885                 spin_unlock_bh(&conn->state_lock);
 886                 return;
 887         }
 888
 889         if (iscsi_thread_set_force_reinstatement(conn) < 0) {
 890                 spin_unlock_bh(&conn->state_lock);
 891                 return;
 892         }
 893
 894         atomic_set(&conn->connection_reinstatement, 1);
 895         if (!sleep) {
 896                 spin_unlock_bh(&conn->state_lock);
 897                 return;
 898         }
 899
 900         atomic_set(&conn->sleep_on_conn_wait_comp, 1);
 901         spin_unlock_bh(&conn->state_lock);
 902
 903         wait_for_completion(&conn->conn_wait_comp);
 904         complete(&conn->conn_post_wait_comp);
 905 }
 906 EXPORT_SYMBOL(iscsit_cause_connection_reinstatement);
 907
 908 void iscsit_fall_back_to_erl0(struct iscsi_session *sess)
 909 {
 910         pr_debug("Falling back to ErrorRecoveryLevel=0 for SID:"
 911                         " %u\n", sess->sid);
 912
 913         atomic_set(&sess->session_fall_back_to_erl0, 1);
 914 }
 915
 916 static void iscsit_handle_connection_cleanup(struct iscsi_conn *conn)
 917 {
 918         struct iscsi_session *sess = conn->sess;
 919
 920         if ((sess->sess_ops->ErrorRecoveryLevel == 2) &&
 921             !atomic_read(&sess->session_reinstatement) &&
 922             !atomic_read(&sess->session_fall_back_to_erl0))
 923                 iscsit_connection_recovery_transport_reset(conn);
 924         else {
 925                 pr_debug("Performing cleanup for failed iSCSI"
 926                         " Connection ID: %hu from %s\n", conn->cid,
 927                         sess->sess_ops->InitiatorName);
 928                 iscsit_close_connection(conn);
 929         }
 930 }
 931
 932 void iscsit_take_action_for_connection_exit(struct iscsi_conn *conn)
 933 {
 934         spin_lock_bh(&conn->state_lock);
 935         if (atomic_read(&conn->connection_exit)) {
 936                 spin_unlock_bh(&conn->state_lock);
 937                 return;
 938         }
 939         atomic_set(&conn->connection_exit, 1);
 940
 941         if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) {
 942                 spin_unlock_bh(&conn->state_lock);
 943                 if (conn->conn_transport->transport_type == ISCSI_TCP)
 944                         iscsit_close_connection(conn);
 945                 return;
 946         }
 947
 948         if (conn->conn_state == TARG_CONN_STATE_CLEANUP_WAIT) {
 949                 spin_unlock_bh(&conn->state_lock);
 950                 return;
 951         }
 952
 953         pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n");
 954         conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT;
 955         spin_unlock_bh(&conn->state_lock);
 956
 957         iscsit_handle_connection_cleanup(conn);
 958 }
 959
 960 /*
 961  *      This is the simple function that makes the magic of
 962  *      sync and steering happen in the follow paradoxical order:
 963  *
 964  *      0) Receive conn->of_marker (bytes left until next OFMarker)
 965  *         bytes into an offload buffer.  When we pass the exact number
 966  *         of bytes in conn->of_marker, iscsit_dump_data_payload() and hence
 967  *         rx_data() will automatically receive the identical u32 marker
 968  *         values and store it in conn->of_marker_offset;
 969  *      1) Now conn->of_marker_offset will contain the offset to the start
 970  *         of the next iSCSI PDU.  Dump these remaining bytes into another
 971  *         offload buffer.
 972  *      2) We are done!
 973  *         Next byte in the TCP stream will contain the next iSCSI PDU!
 974  *         Cool Huh?!
 975  */
 976 int iscsit_recover_from_unknown_opcode(struct iscsi_conn *conn)
 977 {
 978         /*
 979          * Make sure the remaining bytes to next maker is a sane value.
 980          */
 981         if (conn->of_marker > (conn->conn_ops->OFMarkInt * 4)) {
 982                 pr_err("Remaining bytes to OFMarker: %u exceeds"
 983                         " OFMarkInt bytes: %u.\n", conn->of_marker,
 984                                 conn->conn_ops->OFMarkInt * 4);
 985                 return -1;
 986         }
 987
 988         pr_debug("Advancing %u bytes in TCP stream to get to the"
 989                         " next OFMarker.\n", conn->of_marker);
 990
 991         if (iscsit_dump_data_payload(conn, conn->of_marker, 0) < 0)
 992                 return -1;
 993
 994         /*
 995          * Make sure the offset marker we retrived is a valid value.
 996          */
 997         if (conn->of_marker_offset > (ISCSI_HDR_LEN + (ISCSI_CRC_LEN * 2) +
 998             conn->conn_ops->MaxRecvDataSegmentLength)) {
 999                 pr_err("OfMarker offset value: %u exceeds limit.\n",
1000                         conn->of_marker_offset);
1001                 return -1;
1002         }
1003
1004         pr_debug("Discarding %u bytes of TCP stream to get to the"
1005                         " next iSCSI Opcode.\n", conn->of_marker_offset);
1006
1007         if (iscsit_dump_data_payload(conn, conn->of_marker_offset, 0) < 0)
1008                 return -1;
1009
1010         return 0;
1011 }