drivers/scsi/ufs/ufshcd.c

   1 /*
   2  * Universal Flash Storage Host controller driver Core
   3  *
   4  * This code is based on drivers/scsi/ufs/ufshcd.c
   5  * Copyright (C) 2011-2013 Samsung India Software Operations
   6  *
   7  * Authors:
   8  *      Santosh Yaraganavi <santosh.sy@samsung.com>
   9  *      Vinayak Holikatti <h.vinayak@samsung.com>
  10  *
  11  * This program is free software; you can redistribute it and/or
  12  * modify it under the terms of the GNU General Public License
  13  * as published by the Free Software Foundation; either version 2
  14  * of the License, or (at your option) any later version.
  15  * See the COPYING file in the top-level directory or visit
  16  * <http://www.gnu.org/licenses/gpl-2.0.html>
  17  *
  18  * This program is distributed in the hope that it will be useful,
  19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21  * GNU General Public License for more details.
  22  *
  23  * This program is provided "AS IS" and "WITH ALL FAULTS" and
  24  * without warranty of any kind. You are solely responsible for
  25  * determining the appropriateness of using and distributing
  26  * the program and assume all risks associated with your exercise
  27  * of rights with respect to the program, including but not limited
  28  * to infringement of third party rights, the risks and costs of
  29  * program errors, damage to or loss of data, programs or equipment,
  30  * and unavailability or interruption of operations. Under no
  31  * circumstances will the contributor of this Program be liable for
  32  * any damages of any kind arising from your use or distribution of
  33  * this program.
  34  */
  35
  36 #include <linux/async.h>
  37
  38 #include "ufshcd.h"
  39
  40 #define UFSHCD_ENABLE_INTRS     (UTP_TRANSFER_REQ_COMPL |\
  41                                  UTP_TASK_REQ_COMPL |\
  42                                  UFSHCD_ERROR_MASK)
  43 /* UIC command timeout, unit: ms */
  44 #define UIC_CMD_TIMEOUT 500
  45
  46 /* NOP OUT retries waiting for NOP IN response */
  47 #define NOP_OUT_RETRIES    10
  48 /* Timeout after 30 msecs if NOP OUT hangs without response */
  49 #define NOP_OUT_TIMEOUT    30 /* msecs */
  50
  51 /* Query request retries */
  52 #define QUERY_REQ_RETRIES 10
  53 /* Query request timeout */
  54 #define QUERY_REQ_TIMEOUT 30 /* msec */
  55
  56 /* Expose the flag value from utp_upiu_query.value */
  57 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
  58
  59 enum {
  60         UFSHCD_MAX_CHANNEL      = 0,
  61         UFSHCD_MAX_ID           = 1,
  62         UFSHCD_MAX_LUNS         = 8,
  63         UFSHCD_CMD_PER_LUN      = 32,
  64         UFSHCD_CAN_QUEUE        = 32,
  65 };
  66
  67 /* UFSHCD states */
  68 enum {
  69         UFSHCD_STATE_OPERATIONAL,
  70         UFSHCD_STATE_RESET,
  71         UFSHCD_STATE_ERROR,
  72 };
  73
  74 /* Interrupt configuration options */
  75 enum {
  76         UFSHCD_INT_DISABLE,
  77         UFSHCD_INT_ENABLE,
  78         UFSHCD_INT_CLEAR,
  79 };
  80
  81 /* Interrupt aggregation options */
  82 enum {
  83         INT_AGGR_RESET,
  84         INT_AGGR_CONFIG,
  85 };
  86
  87 /*
  88  * ufshcd_wait_for_register - wait for register value to change
  89  * @hba - per-adapter interface
  90  * @reg - mmio register offset
  91  * @mask - mask to apply to read register value
  92  * @val - wait condition
  93  * @interval_us - polling interval in microsecs
  94  * @timeout_ms - timeout in millisecs
  95  *
  96  * Returns -ETIMEDOUT on error, zero on success
  97  */
  98 static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
  99                 u32 val, unsigned long interval_us, unsigned long timeout_ms)
 100 {
 101         int err = 0;
 102         unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
 103
 104         /* ignore bits that we don't intend to wait on */
 105         val = val & mask;
 106
 107         while ((ufshcd_readl(hba, reg) & mask) != val) {
 108                 /* wakeup within 50us of expiry */
 109                 usleep_range(interval_us, interval_us + 50);
 110
 111                 if (time_after(jiffies, timeout)) {
 112                         if ((ufshcd_readl(hba, reg) & mask) != val)
 113                                 err = -ETIMEDOUT;
 114                         break;
 115                 }
 116         }
 117
 118         return err;
 119 }
 120
 121 /**
 122  * ufshcd_get_intr_mask - Get the interrupt bit mask
 123  * @hba - Pointer to adapter instance
 124  *
 125  * Returns interrupt bit mask per version
 126  */
 127 static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
 128 {
 129         if (hba->ufs_version == UFSHCI_VERSION_10)
 130                 return INTERRUPT_MASK_ALL_VER_10;
 131         else
 132                 return INTERRUPT_MASK_ALL_VER_11;
 133 }
 134
 135 /**
 136  * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
 137  * @hba - Pointer to adapter instance
 138  *
 139  * Returns UFSHCI version supported by the controller
 140  */
 141 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
 142 {
 143         return ufshcd_readl(hba, REG_UFS_VERSION);
 144 }
 145
 146 /**
 147  * ufshcd_is_device_present - Check if any device connected to
 148  *                            the host controller
 149  * @reg_hcs - host controller status register value
 150  *
 151  * Returns 1 if device present, 0 if no device detected
 152  */
 153 static inline int ufshcd_is_device_present(u32 reg_hcs)
 154 {
 155         return (DEVICE_PRESENT & reg_hcs) ? 1 : 0;
 156 }
 157
 158 /**
 159  * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
 160  * @lrb: pointer to local command reference block
 161  *
 162  * This function is used to get the OCS field from UTRD
 163  * Returns the OCS field in the UTRD
 164  */
 165 static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
 166 {
 167         return lrbp->utr_descriptor_ptr->header.dword_2 & MASK_OCS;
 168 }
 169
 170 /**
 171  * ufshcd_get_tmr_ocs - Get the UTMRD Overall Command Status
 172  * @task_req_descp: pointer to utp_task_req_desc structure
 173  *
 174  * This function is used to get the OCS field from UTMRD
 175  * Returns the OCS field in the UTMRD
 176  */
 177 static inline int
 178 ufshcd_get_tmr_ocs(struct utp_task_req_desc *task_req_descp)
 179 {
 180         return task_req_descp->header.dword_2 & MASK_OCS;
 181 }
 182
 183 /**
 184  * ufshcd_get_tm_free_slot - get a free slot for task management request
 185  * @hba: per adapter instance
 186  *
 187  * Returns maximum number of task management request slots in case of
 188  * task management queue full or returns the free slot number
 189  */
 190 static inline int ufshcd_get_tm_free_slot(struct ufs_hba *hba)
 191 {
 192         return find_first_zero_bit(&hba->outstanding_tasks, hba->nutmrs);
 193 }
 194
 195 /**
 196  * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
 197  * @hba: per adapter instance
 198  * @pos: position of the bit to be cleared
 199  */
 200 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
 201 {
 202         ufshcd_writel(hba, ~(1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
 203 }
 204
 205 /**
 206  * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
 207  * @reg: Register value of host controller status
 208  *
 209  * Returns integer, 0 on Success and positive value if failed
 210  */
 211 static inline int ufshcd_get_lists_status(u32 reg)
 212 {
 213         /*
 214          * The mask 0xFF is for the following HCS register bits
 215          * Bit          Description
 216          *  0           Device Present
 217          *  1           UTRLRDY
 218          *  2           UTMRLRDY
 219          *  3           UCRDY
 220          *  4           HEI
 221          *  5           DEI
 222          * 6-7          reserved
 223          */
 224         return (((reg) & (0xFF)) >> 1) ^ (0x07);
 225 }
 226
 227 /**
 228  * ufshcd_get_uic_cmd_result - Get the UIC command result
 229  * @hba: Pointer to adapter instance
 230  *
 231  * This function gets the result of UIC command completion
 232  * Returns 0 on success, non zero value on error
 233  */
 234 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
 235 {
 236         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
 237                MASK_UIC_COMMAND_RESULT;
 238 }
 239
 240 /**
 241  * ufshcd_get_req_rsp - returns the TR response transaction type
 242  * @ucd_rsp_ptr: pointer to response UPIU
 243  */
 244 static inline int
 245 ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
 246 {
 247         return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
 248 }
 249
 250 /**
 251  * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
 252  * @ucd_rsp_ptr: pointer to response UPIU
 253  *
 254  * This function gets the response status and scsi_status from response UPIU
 255  * Returns the response result code.
 256  */
 257 static inline int
 258 ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
 259 {
 260         return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
 261 }
 262
 263 /**
 264  * ufshcd_is_exception_event - Check if the device raised an exception event
 265  * @ucd_rsp_ptr: pointer to response UPIU
 266  *
 267  * The function checks if the device raised an exception event indicated in
 268  * the Device Information field of response UPIU.
 269  *
 270  * Returns true if exception is raised, false otherwise.
 271  */
 272 static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
 273 {
 274         return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
 275                         MASK_RSP_EXCEPTION_EVENT ? true : false;
 276 }
 277
 278 /**
 279  * ufshcd_config_int_aggr - Configure interrupt aggregation values.
 280  *              Currently there is no use case where we want to configure
 281  *              interrupt aggregation dynamically. So to configure interrupt
 282  *              aggregation, #define INT_AGGR_COUNTER_THRESHOLD_VALUE and
 283  *              INT_AGGR_TIMEOUT_VALUE are used.
 284  * @hba: per adapter instance
 285  * @option: Interrupt aggregation option
 286  */
 287 static inline void
 288 ufshcd_config_int_aggr(struct ufs_hba *hba, int option)
 289 {
 290         switch (option) {
 291         case INT_AGGR_RESET:
 292                 ufshcd_writel(hba, INT_AGGR_ENABLE |
 293                               INT_AGGR_COUNTER_AND_TIMER_RESET,
 294                               REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 295                 break;
 296         case INT_AGGR_CONFIG:
 297                 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
 298                               INT_AGGR_COUNTER_THRESHOLD_VALUE |
 299                               INT_AGGR_TIMEOUT_VALUE,
 300                               REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 301                 break;
 302         }
 303 }
 304
 305 /**
 306  * ufshcd_enable_run_stop_reg - Enable run-stop registers,
 307  *                      When run-stop registers are set to 1, it indicates the
 308  *                      host controller that it can process the requests
 309  * @hba: per adapter instance
 310  */
 311 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
 312 {
 313         ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
 314                       REG_UTP_TASK_REQ_LIST_RUN_STOP);
 315         ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
 316                       REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
 317 }
 318
 319 /**
 320  * ufshcd_hba_start - Start controller initialization sequence
 321  * @hba: per adapter instance
 322  */
 323 static inline void ufshcd_hba_start(struct ufs_hba *hba)
 324 {
 325         ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE);
 326 }
 327
 328 /**
 329  * ufshcd_is_hba_active - Get controller state
 330  * @hba: per adapter instance
 331  *
 332  * Returns zero if controller is active, 1 otherwise
 333  */
 334 static inline int ufshcd_is_hba_active(struct ufs_hba *hba)
 335 {
 336         return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & 0x1) ? 0 : 1;
 337 }
 338
 339 /**
 340  * ufshcd_send_command - Send SCSI or device management commands
 341  * @hba: per adapter instance
 342  * @task_tag: Task tag of the command
 343  */
 344 static inline
 345 void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
 346 {
 347         __set_bit(task_tag, &hba->outstanding_reqs);
 348         ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
 349 }
 350
 351 /**
 352  * ufshcd_copy_sense_data - Copy sense data in case of check condition
 353  * @lrb - pointer to local reference block
 354  */
 355 static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
 356 {
 357         int len;
 358         if (lrbp->sense_buffer) {
 359                 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
 360                 memcpy(lrbp->sense_buffer,
 361                         lrbp->ucd_rsp_ptr->sr.sense_data,
 362                         min_t(int, len, SCSI_SENSE_BUFFERSIZE));
 363         }
 364 }
 365
 366 /**
 367  * ufshcd_query_to_cpu() - formats the buffer to native cpu endian
 368  * @response: upiu query response to convert
 369  */
 370 static inline void ufshcd_query_to_cpu(struct utp_upiu_query *response)
 371 {
 372         response->length = be16_to_cpu(response->length);
 373         response->value = be32_to_cpu(response->value);
 374 }
 375
 376 /**
 377  * ufshcd_query_to_be() - formats the buffer to big endian
 378  * @request: upiu query request to convert
 379  */
 380 static inline void ufshcd_query_to_be(struct utp_upiu_query *request)
 381 {
 382         request->length = cpu_to_be16(request->length);
 383         request->value = cpu_to_be32(request->value);
 384 }
 385
 386 /**
 387  * ufshcd_copy_query_response() - Copy the Query Response and the data
 388  * descriptor
 389  * @hba: per adapter instance
 390  * @lrb - pointer to local reference block
 391  */
 392 static
 393 void ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
 394 {
 395         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
 396
 397         /* Get the UPIU response */
 398         query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
 399                         UPIU_RSP_CODE_OFFSET;
 400
 401         memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
 402         ufshcd_query_to_cpu(&query_res->upiu_res);
 403
 404
 405         /* Get the descriptor */
 406         if (lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
 407                 u8 *descp = (u8 *)&lrbp->ucd_rsp_ptr +
 408                                 GENERAL_UPIU_REQUEST_SIZE;
 409                 u16 len;
 410
 411                 /* data segment length */
 412                 len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
 413                                                 MASK_QUERY_DATA_SEG_LEN;
 414
 415                 memcpy(hba->dev_cmd.query.descriptor, descp,
 416                                 min_t(u16, len, QUERY_DESC_MAX_SIZE));
 417         }
 418 }
 419
 420 /**
 421  * ufshcd_hba_capabilities - Read controller capabilities
 422  * @hba: per adapter instance
 423  */
 424 static inline void ufshcd_hba_capabilities(struct ufs_hba *hba)
 425 {
 426         hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
 427
 428         /* nutrs and nutmrs are 0 based values */
 429         hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
 430         hba->nutmrs =
 431         ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
 432 }
 433
 434 /**
 435  * ufshcd_ready_for_uic_cmd - Check if controller is ready
 436  *                            to accept UIC commands
 437  * @hba: per adapter instance
 438  * Return true on success, else false
 439  */
 440 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
 441 {
 442         if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
 443                 return true;
 444         else
 445                 return false;
 446 }
 447
 448 /**
 449  * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
 450  * @hba: per adapter instance
 451  * @uic_cmd: UIC command
 452  *
 453  * Mutex must be held.
 454  */
 455 static inline void
 456 ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
 457 {
 458         WARN_ON(hba->active_uic_cmd);
 459
 460         hba->active_uic_cmd = uic_cmd;
 461
 462         /* Write Args */
 463         ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
 464         ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
 465         ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
 466
 467         /* Write UIC Cmd */
 468         ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
 469                       REG_UIC_COMMAND);
 470 }
 471
 472 /**
 473  * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
 474  * @hba: per adapter instance
 475  * @uic_command: UIC command
 476  *
 477  * Must be called with mutex held.
 478  * Returns 0 only if success.
 479  */
 480 static int
 481 ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
 482 {
 483         int ret;
 484         unsigned long flags;
 485
 486         if (wait_for_completion_timeout(&uic_cmd->done,
 487                                         msecs_to_jiffies(UIC_CMD_TIMEOUT)))
 488                 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
 489         else
 490                 ret = -ETIMEDOUT;
 491
 492         spin_lock_irqsave(hba->host->host_lock, flags);
 493         hba->active_uic_cmd = NULL;
 494         spin_unlock_irqrestore(hba->host->host_lock, flags);
 495
 496         return ret;
 497 }
 498
 499 /**
 500  * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
 501  * @hba: per adapter instance
 502  * @uic_cmd: UIC command
 503  *
 504  * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
 505  * with mutex held.
 506  * Returns 0 only if success.
 507  */
 508 static int
 509 __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
 510 {
 511         int ret;
 512         unsigned long flags;
 513
 514         if (!ufshcd_ready_for_uic_cmd(hba)) {
 515                 dev_err(hba->dev,
 516                         "Controller not ready to accept UIC commands\n");
 517                 return -EIO;
 518         }
 519
 520         init_completion(&uic_cmd->done);
 521
 522         spin_lock_irqsave(hba->host->host_lock, flags);
 523         ufshcd_dispatch_uic_cmd(hba, uic_cmd);
 524         spin_unlock_irqrestore(hba->host->host_lock, flags);
 525
 526         ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
 527
 528         return ret;
 529 }
 530
 531 /**
 532  * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
 533  * @hba: per adapter instance
 534  * @uic_cmd: UIC command
 535  *
 536  * Returns 0 only if success.
 537  */
 538 static int
 539 ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
 540 {
 541         int ret;
 542
 543         mutex_lock(&hba->uic_cmd_mutex);
 544         ret = __ufshcd_send_uic_cmd(hba, uic_cmd);
 545         mutex_unlock(&hba->uic_cmd_mutex);
 546
 547         return ret;
 548 }
 549
 550 /**
 551  * ufshcd_map_sg - Map scatter-gather list to prdt
 552  * @lrbp - pointer to local reference block
 553  *
 554  * Returns 0 in case of success, non-zero value in case of failure
 555  */
 556 static int ufshcd_map_sg(struct ufshcd_lrb *lrbp)
 557 {
 558         struct ufshcd_sg_entry *prd_table;
 559         struct scatterlist *sg;
 560         struct scsi_cmnd *cmd;
 561         int sg_segments;
 562         int i;
 563
 564         cmd = lrbp->cmd;
 565         sg_segments = scsi_dma_map(cmd);
 566         if (sg_segments < 0)
 567                 return sg_segments;
 568
 569         if (sg_segments) {
 570                 lrbp->utr_descriptor_ptr->prd_table_length =
 571                                         cpu_to_le16((u16) (sg_segments));
 572
 573                 prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
 574
 575                 scsi_for_each_sg(cmd, sg, sg_segments, i) {
 576                         prd_table[i].size  =
 577                                 cpu_to_le32(((u32) sg_dma_len(sg))-1);
 578                         prd_table[i].base_addr =
 579                                 cpu_to_le32(lower_32_bits(sg->dma_address));
 580                         prd_table[i].upper_addr =
 581                                 cpu_to_le32(upper_32_bits(sg->dma_address));
 582                 }
 583         } else {
 584                 lrbp->utr_descriptor_ptr->prd_table_length = 0;
 585         }
 586
 587         return 0;
 588 }
 589
 590 /**
 591  * ufshcd_enable_intr - enable interrupts
 592  * @hba: per adapter instance
 593  * @intrs: interrupt bits
 594  */
 595 static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
 596 {
 597         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
 598
 599         if (hba->ufs_version == UFSHCI_VERSION_10) {
 600                 u32 rw;
 601                 rw = set & INTERRUPT_MASK_RW_VER_10;
 602                 set = rw | ((set ^ intrs) & intrs);
 603         } else {
 604                 set |= intrs;
 605         }
 606
 607         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
 608 }
 609
 610 /**
 611  * ufshcd_disable_intr - disable interrupts
 612  * @hba: per adapter instance
 613  * @intrs: interrupt bits
 614  */
 615 static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
 616 {
 617         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
 618
 619         if (hba->ufs_version == UFSHCI_VERSION_10) {
 620                 u32 rw;
 621                 rw = (set & INTERRUPT_MASK_RW_VER_10) &
 622                         ~(intrs & INTERRUPT_MASK_RW_VER_10);
 623                 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
 624
 625         } else {
 626                 set &= ~intrs;
 627         }
 628
 629         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
 630 }
 631
 632 /**
 633  * ufshcd_prepare_req_desc_hdr() - Fills the requests header
 634  * descriptor according to request
 635  * @lrbp: pointer to local reference block
 636  * @upiu_flags: flags required in the header
 637  * @cmd_dir: requests data direction
 638  */
 639 static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
 640                 u32 *upiu_flags, enum dma_data_direction cmd_dir)
 641 {
 642         struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
 643         u32 data_direction;
 644         u32 dword_0;
 645
 646         if (cmd_dir == DMA_FROM_DEVICE) {
 647                 data_direction = UTP_DEVICE_TO_HOST;
 648                 *upiu_flags = UPIU_CMD_FLAGS_READ;
 649         } else if (cmd_dir == DMA_TO_DEVICE) {
 650                 data_direction = UTP_HOST_TO_DEVICE;
 651                 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
 652         } else {
 653                 data_direction = UTP_NO_DATA_TRANSFER;
 654                 *upiu_flags = UPIU_CMD_FLAGS_NONE;
 655         }
 656
 657         dword_0 = data_direction | (lrbp->command_type
 658                                 << UPIU_COMMAND_TYPE_OFFSET);
 659         if (lrbp->intr_cmd)
 660                 dword_0 |= UTP_REQ_DESC_INT_CMD;
 661
 662         /* Transfer request descriptor header fields */
 663         req_desc->header.dword_0 = cpu_to_le32(dword_0);
 664
 665         /*
 666          * assigning invalid value for command status. Controller
 667          * updates OCS on command completion, with the command
 668          * status
 669          */
 670         req_desc->header.dword_2 =
 671                 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
 672 }
 673
 674 /**
 675  * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
 676  * for scsi commands
 677  * @lrbp - local reference block pointer
 678  * @upiu_flags - flags
 679  */
 680 static
 681 void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u32 upiu_flags)
 682 {
 683         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
 684
 685         /* command descriptor fields */
 686         ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
 687                                 UPIU_TRANSACTION_COMMAND, upiu_flags,
 688                                 lrbp->lun, lrbp->task_tag);
 689         ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
 690                                 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
 691
 692         /* Total EHS length and Data segment length will be zero */
 693         ucd_req_ptr->header.dword_2 = 0;
 694
 695         ucd_req_ptr->sc.exp_data_transfer_len =
 696                 cpu_to_be32(lrbp->cmd->sdb.length);
 697
 698         memcpy(ucd_req_ptr->sc.cdb, lrbp->cmd->cmnd,
 699                 (min_t(unsigned short, lrbp->cmd->cmd_len, MAX_CDB_SIZE)));
 700 }
 701
 702 /**
 703  * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
 704  * for query requsts
 705  * @hba: UFS hba
 706  * @lrbp: local reference block pointer
 707  * @upiu_flags: flags
 708  */
 709 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
 710                                 struct ufshcd_lrb *lrbp, u32 upiu_flags)
 711 {
 712         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
 713         struct ufs_query *query = &hba->dev_cmd.query;
 714         u16 len = query->request.upiu_req.length;
 715         u8 *descp = (u8 *)lrbp->ucd_req_ptr + GENERAL_UPIU_REQUEST_SIZE;
 716
 717         /* Query request header */
 718         ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
 719                         UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
 720                         lrbp->lun, lrbp->task_tag);
 721         ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
 722                         0, query->request.query_func, 0, 0);
 723
 724         /* Data segment length */
 725         ucd_req_ptr->header.dword_2 = UPIU_HEADER_DWORD(
 726                         0, 0, len >> 8, (u8)len);
 727
 728         /* Copy the Query Request buffer as is */
 729         memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
 730                         QUERY_OSF_SIZE);
 731         ufshcd_query_to_be(&ucd_req_ptr->qr);
 732
 733         /* Copy the Descriptor */
 734         if ((len > 0) && (query->request.upiu_req.opcode ==
 735                                         UPIU_QUERY_OPCODE_WRITE_DESC)) {
 736                 memcpy(descp, query->descriptor,
 737                                 min_t(u16, len, QUERY_DESC_MAX_SIZE));
 738         }
 739 }
 740
 741 static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
 742 {
 743         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
 744
 745         memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
 746
 747         /* command descriptor fields */
 748         ucd_req_ptr->header.dword_0 =
 749                 UPIU_HEADER_DWORD(
 750                         UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
 751 }
 752
 753 /**
 754  * ufshcd_compose_upiu - form UFS Protocol Information Unit(UPIU)
 755  * @hba - per adapter instance
 756  * @lrb - pointer to local reference block
 757  */
 758 static int ufshcd_compose_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
 759 {
 760         u32 upiu_flags;
 761         int ret = 0;
 762
 763         switch (lrbp->command_type) {
 764         case UTP_CMD_TYPE_SCSI:
 765                 if (likely(lrbp->cmd)) {
 766                         ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
 767                                         lrbp->cmd->sc_data_direction);
 768                         ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
 769                 } else {
 770                         ret = -EINVAL;
 771                 }
 772                 break;
 773         case UTP_CMD_TYPE_DEV_MANAGE:
 774                 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
 775                 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
 776                         ufshcd_prepare_utp_query_req_upiu(
 777                                         hba, lrbp, upiu_flags);
 778                 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
 779                         ufshcd_prepare_utp_nop_upiu(lrbp);
 780                 else
 781                         ret = -EINVAL;
 782                 break;
 783         case UTP_CMD_TYPE_UFS:
 784                 /* For UFS native command implementation */
 785                 ret = -ENOTSUPP;
 786                 dev_err(hba->dev, "%s: UFS native command are not supported\n",
 787                         __func__);
 788                 break;
 789         default:
 790                 ret = -ENOTSUPP;
 791                 dev_err(hba->dev, "%s: unknown command type: 0x%x\n",
 792                                 __func__, lrbp->command_type);
 793                 break;
 794         } /* end of switch */
 795
 796         return ret;
 797 }
 798
 799 /**
 800  * ufshcd_queuecommand - main entry point for SCSI requests
 801  * @cmd: command from SCSI Midlayer
 802  * @done: call back function
 803  *
 804  * Returns 0 for success, non-zero in case of failure
 805  */
 806 static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
 807 {
 808         struct ufshcd_lrb *lrbp;
 809         struct ufs_hba *hba;
 810         unsigned long flags;
 811         int tag;
 812         int err = 0;
 813
 814         hba = shost_priv(host);
 815
 816         tag = cmd->request->tag;
 817
 818         if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
 819                 err = SCSI_MLQUEUE_HOST_BUSY;
 820                 goto out;
 821         }
 822
 823         /* acquire the tag to make sure device cmds don't use it */
 824         if (test_and_set_bit_lock(tag, &hba->lrb_in_use)) {
 825                 /*
 826                  * Dev manage command in progress, requeue the command.
 827                  * Requeuing the command helps in cases where the request *may*
 828                  * find different tag instead of waiting for dev manage command
 829                  * completion.
 830                  */
 831                 err = SCSI_MLQUEUE_HOST_BUSY;
 832                 goto out;
 833         }
 834
 835         lrbp = &hba->lrb[tag];
 836
 837         WARN_ON(lrbp->cmd);
 838         lrbp->cmd = cmd;
 839         lrbp->sense_bufflen = SCSI_SENSE_BUFFERSIZE;
 840         lrbp->sense_buffer = cmd->sense_buffer;
 841         lrbp->task_tag = tag;
 842         lrbp->lun = cmd->device->lun;
 843         lrbp->intr_cmd = false;
 844         lrbp->command_type = UTP_CMD_TYPE_SCSI;
 845
 846         /* form UPIU before issuing the command */
 847         ufshcd_compose_upiu(hba, lrbp);
 848         err = ufshcd_map_sg(lrbp);
 849         if (err) {
 850                 lrbp->cmd = NULL;
 851                 clear_bit_unlock(tag, &hba->lrb_in_use);
 852                 goto out;
 853         }
 854
 855         /* issue command to the controller */
 856         spin_lock_irqsave(hba->host->host_lock, flags);
 857         ufshcd_send_command(hba, tag);
 858         spin_unlock_irqrestore(hba->host->host_lock, flags);
 859 out:
 860         return err;
 861 }
 862
 863 static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
 864                 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
 865 {
 866         lrbp->cmd = NULL;
 867         lrbp->sense_bufflen = 0;
 868         lrbp->sense_buffer = NULL;
 869         lrbp->task_tag = tag;
 870         lrbp->lun = 0; /* device management cmd is not specific to any LUN */
 871         lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
 872         lrbp->intr_cmd = true; /* No interrupt aggregation */
 873         hba->dev_cmd.type = cmd_type;
 874
 875         return ufshcd_compose_upiu(hba, lrbp);
 876 }
 877
 878 static int
 879 ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
 880 {
 881         int err = 0;
 882         unsigned long flags;
 883         u32 mask = 1 << tag;
 884
 885         /* clear outstanding transaction before retry */
 886         spin_lock_irqsave(hba->host->host_lock, flags);
 887         ufshcd_utrl_clear(hba, tag);
 888         spin_unlock_irqrestore(hba->host->host_lock, flags);
 889
 890         /*
 891          * wait for for h/w to clear corresponding bit in door-bell.
 892          * max. wait is 1 sec.
 893          */
 894         err = ufshcd_wait_for_register(hba,
 895                         REG_UTP_TRANSFER_REQ_DOOR_BELL,
 896                         mask, ~mask, 1000, 1000);
 897
 898         return err;
 899 }
 900
 901 /**
 902  * ufshcd_dev_cmd_completion() - handles device management command responses
 903  * @hba: per adapter instance
 904  * @lrbp: pointer to local reference block
 905  */
 906 static int
 907 ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
 908 {
 909         int resp;
 910         int err = 0;
 911
 912         resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
 913
 914         switch (resp) {
 915         case UPIU_TRANSACTION_NOP_IN:
 916                 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
 917                         err = -EINVAL;
 918                         dev_err(hba->dev, "%s: unexpected response %x\n",
 919                                         __func__, resp);
 920                 }
 921                 break;
 922         case UPIU_TRANSACTION_QUERY_RSP:
 923                 ufshcd_copy_query_response(hba, lrbp);
 924                 break;
 925         case UPIU_TRANSACTION_REJECT_UPIU:
 926                 /* TODO: handle Reject UPIU Response */
 927                 err = -EPERM;
 928                 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
 929                                 __func__);
 930                 break;
 931         default:
 932                 err = -EINVAL;
 933                 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
 934                                 __func__, resp);
 935                 break;
 936         }
 937
 938         return err;
 939 }
 940
 941 static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
 942                 struct ufshcd_lrb *lrbp, int max_timeout)
 943 {
 944         int err = 0;
 945         unsigned long time_left;
 946         unsigned long flags;
 947
 948         time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
 949                         msecs_to_jiffies(max_timeout));
 950
 951         spin_lock_irqsave(hba->host->host_lock, flags);
 952         hba->dev_cmd.complete = NULL;
 953         if (likely(time_left)) {
 954                 err = ufshcd_get_tr_ocs(lrbp);
 955                 if (!err)
 956                         err = ufshcd_dev_cmd_completion(hba, lrbp);
 957         }
 958         spin_unlock_irqrestore(hba->host->host_lock, flags);
 959
 960         if (!time_left) {
 961                 err = -ETIMEDOUT;
 962                 if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
 963                         /* sucessfully cleared the command, retry if needed */
 964                         err = -EAGAIN;
 965         }
 966
 967         return err;
 968 }
 969
 970 /**
 971  * ufshcd_get_dev_cmd_tag - Get device management command tag
 972  * @hba: per-adapter instance
 973  * @tag: pointer to variable with available slot value
 974  *
 975  * Get a free slot and lock it until device management command
 976  * completes.
 977  *
 978  * Returns false if free slot is unavailable for locking, else
 979  * return true with tag value in @tag.
 980  */
 981 static bool ufshcd_get_dev_cmd_tag(struct ufs_hba *hba, int *tag_out)
 982 {
 983         int tag;
 984         bool ret = false;
 985         unsigned long tmp;
 986
 987         if (!tag_out)
 988                 goto out;
 989
 990         do {
 991                 tmp = ~hba->lrb_in_use;
 992                 tag = find_last_bit(&tmp, hba->nutrs);
 993                 if (tag >= hba->nutrs)
 994                         goto out;
 995         } while (test_and_set_bit_lock(tag, &hba->lrb_in_use));
 996
 997         *tag_out = tag;
 998         ret = true;
 999 out:
1000         return ret;
1001 }
1002
1003 static inline void ufshcd_put_dev_cmd_tag(struct ufs_hba *hba, int tag)
1004 {
1005         clear_bit_unlock(tag, &hba->lrb_in_use);
1006 }
1007
1008 /**
1009  * ufshcd_exec_dev_cmd - API for sending device management requests
1010  * @hba - UFS hba
1011  * @cmd_type - specifies the type (NOP, Query...)
1012  * @timeout - time in seconds
1013  *
1014  * NOTE: Since there is only one available tag for device management commands,
1015  * it is expected you hold the hba->dev_cmd.lock mutex.
1016  */
1017 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
1018                 enum dev_cmd_type cmd_type, int timeout)
1019 {
1020         struct ufshcd_lrb *lrbp;
1021         int err;
1022         int tag;
1023         struct completion wait;
1024         unsigned long flags;
1025
1026         /*
1027          * Get free slot, sleep if slots are unavailable.
1028          * Even though we use wait_event() which sleeps indefinitely,
1029          * the maximum wait time is bounded by SCSI request timeout.
1030          */
1031         wait_event(hba->dev_cmd.tag_wq, ufshcd_get_dev_cmd_tag(hba, &tag));
1032
1033         init_completion(&wait);
1034         lrbp = &hba->lrb[tag];
1035         WARN_ON(lrbp->cmd);
1036         err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
1037         if (unlikely(err))
1038                 goto out_put_tag;
1039
1040         hba->dev_cmd.complete = &wait;
1041
1042         spin_lock_irqsave(hba->host->host_lock, flags);
1043         ufshcd_send_command(hba, tag);
1044         spin_unlock_irqrestore(hba->host->host_lock, flags);
1045
1046         err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
1047
1048 out_put_tag:
1049         ufshcd_put_dev_cmd_tag(hba, tag);
1050         wake_up(&hba->dev_cmd.tag_wq);
1051         return err;
1052 }
1053
1054 /**
1055  * ufshcd_query_flag() - API function for sending flag query requests
1056  * hba: per-adapter instance
1057  * query_opcode: flag query to perform
1058  * idn: flag idn to access
1059  * flag_res: the flag value after the query request completes
1060  *
1061  * Returns 0 for success, non-zero in case of failure
1062  */
1063 static int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
1064                         enum flag_idn idn, bool *flag_res)
1065 {
1066         struct ufs_query_req *request;
1067         struct ufs_query_res *response;
1068         int err;
1069
1070         BUG_ON(!hba);
1071
1072         mutex_lock(&hba->dev_cmd.lock);
1073         request = &hba->dev_cmd.query.request;
1074         response = &hba->dev_cmd.query.response;
1075         memset(request, 0, sizeof(struct ufs_query_req));
1076         memset(response, 0, sizeof(struct ufs_query_res));
1077
1078         switch (opcode) {
1079         case UPIU_QUERY_OPCODE_SET_FLAG:
1080         case UPIU_QUERY_OPCODE_CLEAR_FLAG:
1081         case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
1082                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
1083                 break;
1084         case UPIU_QUERY_OPCODE_READ_FLAG:
1085                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
1086                 if (!flag_res) {
1087                         /* No dummy reads */
1088                         dev_err(hba->dev, "%s: Invalid argument for read request\n",
1089                                         __func__);
1090                         err = -EINVAL;
1091                         goto out_unlock;
1092                 }
1093                 break;
1094         default:
1095                 dev_err(hba->dev,
1096                         "%s: Expected query flag opcode but got = %d\n",
1097                         __func__, opcode);
1098                 err = -EINVAL;
1099                 goto out_unlock;
1100         }
1101         request->upiu_req.opcode = opcode;
1102         request->upiu_req.idn = idn;
1103
1104         /* Send query request */
1105         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY,
1106                         QUERY_REQ_TIMEOUT);
1107
1108         if (err) {
1109                 dev_err(hba->dev,
1110                         "%s: Sending flag query for idn %d failed, err = %d\n",
1111                         __func__, idn, err);
1112                 goto out_unlock;
1113         }
1114
1115         if (flag_res)
1116                 *flag_res = (response->upiu_res.value &
1117                                 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
1118
1119 out_unlock:
1120         mutex_unlock(&hba->dev_cmd.lock);
1121         return err;
1122 }
1123
1124 /**
1125  * ufshcd_query_attr - API function for sending attribute requests
1126  * hba: per-adapter instance
1127  * opcode: attribute opcode
1128  * idn: attribute idn to access
1129  * index: index field
1130  * selector: selector field
1131  * attr_val: the attribute value after the query request completes
1132  *
1133  * Returns 0 for success, non-zero in case of failure
1134 */
1135 int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
1136                         enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
1137 {
1138         struct ufs_query_req *request;
1139         struct ufs_query_res *response;
1140         int err;
1141
1142         BUG_ON(!hba);
1143
1144         if (!attr_val) {
1145                 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
1146                                 __func__, opcode);
1147                 err = -EINVAL;
1148                 goto out;
1149         }
1150
1151         mutex_lock(&hba->dev_cmd.lock);
1152         request = &hba->dev_cmd.query.request;
1153         response = &hba->dev_cmd.query.response;
1154         memset(request, 0, sizeof(struct ufs_query_req));
1155         memset(response, 0, sizeof(struct ufs_query_res));
1156
1157         switch (opcode) {
1158         case UPIU_QUERY_OPCODE_WRITE_ATTR:
1159                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
1160                 request->upiu_req.value = *attr_val;
1161                 break;
1162         case UPIU_QUERY_OPCODE_READ_ATTR:
1163                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
1164                 break;
1165         default:
1166                 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
1167                                 __func__, opcode);
1168                 err = -EINVAL;
1169                 goto out_unlock;
1170         }
1171
1172         request->upiu_req.opcode = opcode;
1173         request->upiu_req.idn = idn;
1174         request->upiu_req.index = index;
1175         request->upiu_req.selector = selector;
1176
1177         /* Send query request */
1178         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY,
1179                                                 QUERY_REQ_TIMEOUT);
1180
1181         if (err) {
1182                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, err = %d\n",
1183                                 __func__, opcode, idn, err);
1184                 goto out_unlock;
1185         }
1186
1187         *attr_val = response->upiu_res.value;
1188
1189 out_unlock:
1190         mutex_unlock(&hba->dev_cmd.lock);
1191 out:
1192         return err;
1193 }
1194
1195 /**
1196  * ufshcd_memory_alloc - allocate memory for host memory space data structures
1197  * @hba: per adapter instance
1198  *
1199  * 1. Allocate DMA memory for Command Descriptor array
1200  *      Each command descriptor consist of Command UPIU, Response UPIU and PRDT
1201  * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
1202  * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
1203  *      (UTMRDL)
1204  * 4. Allocate memory for local reference block(lrb).
1205  *
1206  * Returns 0 for success, non-zero in case of failure
1207  */
1208 static int ufshcd_memory_alloc(struct ufs_hba *hba)
1209 {
1210         size_t utmrdl_size, utrdl_size, ucdl_size;
1211
1212         /* Allocate memory for UTP command descriptors */
1213         ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
1214         hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
1215                                                   ucdl_size,
1216                                                   &hba->ucdl_dma_addr,
1217                                                   GFP_KERNEL);
1218
1219         /*
1220          * UFSHCI requires UTP command descriptor to be 128 byte aligned.
1221          * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
1222          * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
1223          * be aligned to 128 bytes as well
1224          */
1225         if (!hba->ucdl_base_addr ||
1226             WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
1227                 dev_err(hba->dev,
1228                         "Command Descriptor Memory allocation failed\n");
1229                 goto out;
1230         }
1231
1232         /*
1233          * Allocate memory for UTP Transfer descriptors
1234          * UFSHCI requires 1024 byte alignment of UTRD
1235          */
1236         utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
1237         hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
1238                                                    utrdl_size,
1239                                                    &hba->utrdl_dma_addr,
1240                                                    GFP_KERNEL);
1241         if (!hba->utrdl_base_addr ||
1242             WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
1243                 dev_err(hba->dev,
1244                         "Transfer Descriptor Memory allocation failed\n");
1245                 goto out;
1246         }
1247
1248         /*
1249          * Allocate memory for UTP Task Management descriptors
1250          * UFSHCI requires 1024 byte alignment of UTMRD
1251          */
1252         utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
1253         hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
1254                                                     utmrdl_size,
1255                                                     &hba->utmrdl_dma_addr,
1256                                                     GFP_KERNEL);
1257         if (!hba->utmrdl_base_addr ||
1258             WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
1259                 dev_err(hba->dev,
1260                 "Task Management Descriptor Memory allocation failed\n");
1261                 goto out;
1262         }
1263
1264         /* Allocate memory for local reference block */
1265         hba->lrb = devm_kzalloc(hba->dev,
1266                                 hba->nutrs * sizeof(struct ufshcd_lrb),
1267                                 GFP_KERNEL);
1268         if (!hba->lrb) {
1269                 dev_err(hba->dev, "LRB Memory allocation failed\n");
1270                 goto out;
1271         }
1272         return 0;
1273 out:
1274         return -ENOMEM;
1275 }
1276
1277 /**
1278  * ufshcd_host_memory_configure - configure local reference block with
1279  *                              memory offsets
1280  * @hba: per adapter instance
1281  *
1282  * Configure Host memory space
1283  * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
1284  * address.
1285  * 2. Update each UTRD with Response UPIU offset, Response UPIU length
1286  * and PRDT offset.
1287  * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
1288  * into local reference block.
1289  */
1290 static void ufshcd_host_memory_configure(struct ufs_hba *hba)
1291 {
1292         struct utp_transfer_cmd_desc *cmd_descp;
1293         struct utp_transfer_req_desc *utrdlp;
1294         dma_addr_t cmd_desc_dma_addr;
1295         dma_addr_t cmd_desc_element_addr;
1296         u16 response_offset;
1297         u16 prdt_offset;
1298         int cmd_desc_size;
1299         int i;
1300
1301         utrdlp = hba->utrdl_base_addr;
1302         cmd_descp = hba->ucdl_base_addr;
1303
1304         response_offset =
1305                 offsetof(struct utp_transfer_cmd_desc, response_upiu);
1306         prdt_offset =
1307                 offsetof(struct utp_transfer_cmd_desc, prd_table);
1308
1309         cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
1310         cmd_desc_dma_addr = hba->ucdl_dma_addr;
1311
1312         for (i = 0; i < hba->nutrs; i++) {
1313                 /* Configure UTRD with command descriptor base address */
1314                 cmd_desc_element_addr =
1315                                 (cmd_desc_dma_addr + (cmd_desc_size * i));
1316                 utrdlp[i].command_desc_base_addr_lo =
1317                                 cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
1318                 utrdlp[i].command_desc_base_addr_hi =
1319                                 cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
1320
1321                 /* Response upiu and prdt offset should be in double words */
1322                 utrdlp[i].response_upiu_offset =
1323                                 cpu_to_le16((response_offset >> 2));
1324                 utrdlp[i].prd_table_offset =
1325                                 cpu_to_le16((prdt_offset >> 2));
1326                 utrdlp[i].response_upiu_length =
1327                                 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
1328
1329                 hba->lrb[i].utr_descriptor_ptr = (utrdlp + i);
1330                 hba->lrb[i].ucd_req_ptr =
1331                         (struct utp_upiu_req *)(cmd_descp + i);
1332                 hba->lrb[i].ucd_rsp_ptr =
1333                         (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
1334                 hba->lrb[i].ucd_prdt_ptr =
1335                         (struct ufshcd_sg_entry *)cmd_descp[i].prd_table;
1336         }
1337 }
1338
1339 /**
1340  * ufshcd_dme_link_startup - Notify Unipro to perform link startup
1341  * @hba: per adapter instance
1342  *
1343  * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
1344  * in order to initialize the Unipro link startup procedure.
1345  * Once the Unipro links are up, the device connected to the controller
1346  * is detected.
1347  *
1348  * Returns 0 on success, non-zero value on failure
1349  */
1350 static int ufshcd_dme_link_startup(struct ufs_hba *hba)
1351 {
1352         struct uic_command uic_cmd = {0};
1353         int ret;
1354
1355         uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
1356
1357         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
1358         if (ret)
1359                 dev_err(hba->dev,
1360                         "dme-link-startup: error code %d\n", ret);
1361         return ret;
1362 }
1363
1364 /**
1365  * ufshcd_complete_dev_init() - checks device readiness
1366  * hba: per-adapter instance
1367  *
1368  * Set fDeviceInit flag and poll until device toggles it.
1369  */
1370 static int ufshcd_complete_dev_init(struct ufs_hba *hba)
1371 {
1372         int i, retries, err = 0;
1373         bool flag_res = 1;
1374
1375         for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
1376                 /* Set the fDeviceInit flag */
1377                 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_SET_FLAG,
1378                                         QUERY_FLAG_IDN_FDEVICEINIT, NULL);
1379                 if (!err || err == -ETIMEDOUT)
1380                         break;
1381                 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
1382         }
1383         if (err) {
1384                 dev_err(hba->dev,
1385                         "%s setting fDeviceInit flag failed with error %d\n",
1386                         __func__, err);
1387                 goto out;
1388         }
1389
1390         /* poll for max. 100 iterations for fDeviceInit flag to clear */
1391         for (i = 0; i < 100 && !err && flag_res; i++) {
1392                 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
1393                         err = ufshcd_query_flag(hba,
1394                                         UPIU_QUERY_OPCODE_READ_FLAG,
1395                                         QUERY_FLAG_IDN_FDEVICEINIT, &flag_res);
1396                         if (!err || err == -ETIMEDOUT)
1397                                 break;
1398                         dev_dbg(hba->dev, "%s: error %d retrying\n", __func__,
1399                                         err);
1400                 }
1401         }
1402         if (err)
1403                 dev_err(hba->dev,
1404                         "%s reading fDeviceInit flag failed with error %d\n",
1405                         __func__, err);
1406         else if (flag_res)
1407                 dev_err(hba->dev,
1408                         "%s fDeviceInit was not cleared by the device\n",
1409                         __func__);
1410
1411 out:
1412         return err;
1413 }
1414
1415 /**
1416  * ufshcd_make_hba_operational - Make UFS controller operational
1417  * @hba: per adapter instance
1418  *
1419  * To bring UFS host controller to operational state,
1420  * 1. Check if device is present
1421  * 2. Enable required interrupts
1422  * 3. Configure interrupt aggregation
1423  * 4. Program UTRL and UTMRL base addres
1424  * 5. Configure run-stop-registers
1425  *
1426  * Returns 0 on success, non-zero value on failure
1427  */
1428 static int ufshcd_make_hba_operational(struct ufs_hba *hba)
1429 {
1430         int err = 0;
1431         u32 reg;
1432
1433         /* check if device present */
1434         reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
1435         if (!ufshcd_is_device_present(reg)) {
1436                 dev_err(hba->dev, "cc: Device not present\n");
1437                 err = -ENXIO;
1438                 goto out;
1439         }
1440
1441         /* Enable required interrupts */
1442         ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
1443
1444         /* Configure interrupt aggregation */
1445         ufshcd_config_int_aggr(hba, INT_AGGR_CONFIG);
1446
1447         /* Configure UTRL and UTMRL base address registers */
1448         ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
1449                         REG_UTP_TRANSFER_REQ_LIST_BASE_L);
1450         ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
1451                         REG_UTP_TRANSFER_REQ_LIST_BASE_H);
1452         ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
1453                         REG_UTP_TASK_REQ_LIST_BASE_L);
1454         ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
1455                         REG_UTP_TASK_REQ_LIST_BASE_H);
1456
1457         /*
1458          * UCRDY, UTMRLDY and UTRLRDY bits must be 1
1459          * DEI, HEI bits must be 0
1460          */
1461         if (!(ufshcd_get_lists_status(reg))) {
1462                 ufshcd_enable_run_stop_reg(hba);
1463         } else {
1464                 dev_err(hba->dev,
1465                         "Host controller not ready to process requests");
1466                 err = -EIO;
1467                 goto out;
1468         }
1469
1470         if (hba->ufshcd_state == UFSHCD_STATE_RESET)
1471                 scsi_unblock_requests(hba->host);
1472
1473         hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
1474
1475 out:
1476         return err;
1477 }
1478
1479 /**
1480  * ufshcd_hba_enable - initialize the controller
1481  * @hba: per adapter instance
1482  *
1483  * The controller resets itself and controller firmware initialization
1484  * sequence kicks off. When controller is ready it will set
1485  * the Host Controller Enable bit to 1.
1486  *
1487  * Returns 0 on success, non-zero value on failure
1488  */
1489 static int ufshcd_hba_enable(struct ufs_hba *hba)
1490 {
1491         int retry;
1492
1493         /*
1494          * msleep of 1 and 5 used in this function might result in msleep(20),
1495          * but it was necessary to send the UFS FPGA to reset mode during
1496          * development and testing of this driver. msleep can be changed to
1497          * mdelay and retry count can be reduced based on the controller.
1498          */
1499         if (!ufshcd_is_hba_active(hba)) {
1500
1501                 /* change controller state to "reset state" */
1502                 ufshcd_hba_stop(hba);
1503
1504                 /*
1505                  * This delay is based on the testing done with UFS host
1506                  * controller FPGA. The delay can be changed based on the
1507                  * host controller used.
1508                  */
1509                 msleep(5);
1510         }
1511
1512         /* start controller initialization sequence */
1513         ufshcd_hba_start(hba);
1514
1515         /*
1516          * To initialize a UFS host controller HCE bit must be set to 1.
1517          * During initialization the HCE bit value changes from 1->0->1.
1518          * When the host controller completes initialization sequence
1519          * it sets the value of HCE bit to 1. The same HCE bit is read back
1520          * to check if the controller has completed initialization sequence.
1521          * So without this delay the value HCE = 1, set in the previous
1522          * instruction might be read back.
1523          * This delay can be changed based on the controller.
1524          */
1525         msleep(1);
1526
1527         /* wait for the host controller to complete initialization */
1528         retry = 10;
1529         while (ufshcd_is_hba_active(hba)) {
1530                 if (retry) {
1531                         retry--;
1532                 } else {
1533                         dev_err(hba->dev,
1534                                 "Controller enable failed\n");
1535                         return -EIO;
1536                 }
1537                 msleep(5);
1538         }
1539         return 0;
1540 }
1541
1542 /**
1543  * ufshcd_link_startup - Initialize unipro link startup
1544  * @hba: per adapter instance
1545  *
1546  * Returns 0 for success, non-zero in case of failure
1547  */
1548 static int ufshcd_link_startup(struct ufs_hba *hba)
1549 {
1550         int ret;
1551
1552         /* enable UIC related interrupts */
1553         ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
1554
1555         ret = ufshcd_dme_link_startup(hba);
1556         if (ret)
1557                 goto out;
1558
1559         ret = ufshcd_make_hba_operational(hba);
1560
1561 out:
1562         if (ret)
1563                 dev_err(hba->dev, "link startup failed %d\n", ret);
1564         return ret;
1565 }
1566
1567 /**
1568  * ufshcd_verify_dev_init() - Verify device initialization
1569  * @hba: per-adapter instance
1570  *
1571  * Send NOP OUT UPIU and wait for NOP IN response to check whether the
1572  * device Transport Protocol (UTP) layer is ready after a reset.
1573  * If the UTP layer at the device side is not initialized, it may
1574  * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
1575  * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
1576  */
1577 static int ufshcd_verify_dev_init(struct ufs_hba *hba)
1578 {
1579         int err = 0;
1580         int retries;
1581
1582         mutex_lock(&hba->dev_cmd.lock);
1583         for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
1584                 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
1585                                                NOP_OUT_TIMEOUT);
1586
1587                 if (!err || err == -ETIMEDOUT)
1588                         break;
1589
1590                 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
1591         }
1592         mutex_unlock(&hba->dev_cmd.lock);
1593
1594         if (err)
1595                 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
1596         return err;
1597 }
1598
1599 /**
1600  * ufshcd_do_reset - reset the host controller
1601  * @hba: per adapter instance
1602  *
1603  * Returns SUCCESS/FAILED
1604  */
1605 static int ufshcd_do_reset(struct ufs_hba *hba)
1606 {
1607         struct ufshcd_lrb *lrbp;
1608         unsigned long flags;
1609         int tag;
1610
1611         /* block commands from midlayer */
1612         scsi_block_requests(hba->host);
1613
1614         spin_lock_irqsave(hba->host->host_lock, flags);
1615         hba->ufshcd_state = UFSHCD_STATE_RESET;
1616
1617         /* send controller to reset state */
1618         ufshcd_hba_stop(hba);
1619         spin_unlock_irqrestore(hba->host->host_lock, flags);
1620
1621         /* abort outstanding commands */
1622         for (tag = 0; tag < hba->nutrs; tag++) {
1623                 if (test_bit(tag, &hba->outstanding_reqs)) {
1624                         lrbp = &hba->lrb[tag];
1625                         if (lrbp->cmd) {
1626                                 scsi_dma_unmap(lrbp->cmd);
1627                                 lrbp->cmd->result = DID_RESET << 16;
1628                                 lrbp->cmd->scsi_done(lrbp->cmd);
1629                                 lrbp->cmd = NULL;
1630                                 clear_bit_unlock(tag, &hba->lrb_in_use);
1631                         }
1632                 }
1633         }
1634
1635         /* complete device management command */
1636         if (hba->dev_cmd.complete)
1637                 complete(hba->dev_cmd.complete);
1638
1639         /* clear outstanding request/task bit maps */
1640         hba->outstanding_reqs = 0;
1641         hba->outstanding_tasks = 0;
1642
1643         /* Host controller enable */
1644         if (ufshcd_hba_enable(hba)) {
1645                 dev_err(hba->dev,
1646                         "Reset: Controller initialization failed\n");
1647                 return FAILED;
1648         }
1649
1650         if (ufshcd_link_startup(hba)) {
1651                 dev_err(hba->dev,
1652                         "Reset: Link start-up failed\n");
1653                 return FAILED;
1654         }
1655
1656         return SUCCESS;
1657 }
1658
1659 /**
1660  * ufshcd_slave_alloc - handle initial SCSI device configurations
1661  * @sdev: pointer to SCSI device
1662  *
1663  * Returns success
1664  */
1665 static int ufshcd_slave_alloc(struct scsi_device *sdev)
1666 {
1667         struct ufs_hba *hba;
1668
1669         hba = shost_priv(sdev->host);
1670         sdev->tagged_supported = 1;
1671
1672         /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
1673         sdev->use_10_for_ms = 1;
1674         scsi_set_tag_type(sdev, MSG_SIMPLE_TAG);
1675
1676         /*
1677          * Inform SCSI Midlayer that the LUN queue depth is same as the
1678          * controller queue depth. If a LUN queue depth is less than the
1679          * controller queue depth and if the LUN reports
1680          * SAM_STAT_TASK_SET_FULL, the LUN queue depth will be adjusted
1681          * with scsi_adjust_queue_depth.
1682          */
1683         scsi_activate_tcq(sdev, hba->nutrs);
1684         return 0;
1685 }
1686
1687 /**
1688  * ufshcd_slave_destroy - remove SCSI device configurations
1689  * @sdev: pointer to SCSI device
1690  */
1691 static void ufshcd_slave_destroy(struct scsi_device *sdev)
1692 {
1693         struct ufs_hba *hba;
1694
1695         hba = shost_priv(sdev->host);
1696         scsi_deactivate_tcq(sdev, hba->nutrs);
1697 }
1698
1699 /**
1700  * ufshcd_task_req_compl - handle task management request completion
1701  * @hba: per adapter instance
1702  * @index: index of the completed request
1703  *
1704  * Returns SUCCESS/FAILED
1705  */
1706 static int ufshcd_task_req_compl(struct ufs_hba *hba, u32 index)
1707 {
1708         struct utp_task_req_desc *task_req_descp;
1709         struct utp_upiu_task_rsp *task_rsp_upiup;
1710         unsigned long flags;
1711         int ocs_value;
1712         int task_result;
1713
1714         spin_lock_irqsave(hba->host->host_lock, flags);
1715
1716         /* Clear completed tasks from outstanding_tasks */
1717         __clear_bit(index, &hba->outstanding_tasks);
1718
1719         task_req_descp = hba->utmrdl_base_addr;
1720         ocs_value = ufshcd_get_tmr_ocs(&task_req_descp[index]);
1721
1722         if (ocs_value == OCS_SUCCESS) {
1723                 task_rsp_upiup = (struct utp_upiu_task_rsp *)
1724                                 task_req_descp[index].task_rsp_upiu;
1725                 task_result = be32_to_cpu(task_rsp_upiup->header.dword_1);
1726                 task_result = ((task_result & MASK_TASK_RESPONSE) >> 8);
1727
1728                 if (task_result != UPIU_TASK_MANAGEMENT_FUNC_COMPL &&
1729                     task_result != UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED)
1730                         task_result = FAILED;
1731                 else
1732                         task_result = SUCCESS;
1733         } else {
1734                 task_result = FAILED;
1735                 dev_err(hba->dev,
1736                         "trc: Invalid ocs = %x\n", ocs_value);
1737         }
1738         spin_unlock_irqrestore(hba->host->host_lock, flags);
1739         return task_result;
1740 }
1741
1742 /**
1743  * ufshcd_adjust_lun_qdepth - Update LUN queue depth if device responds with
1744  *                            SAM_STAT_TASK_SET_FULL SCSI command status.
1745  * @cmd: pointer to SCSI command
1746  */
1747 static void ufshcd_adjust_lun_qdepth(struct scsi_cmnd *cmd)
1748 {
1749         struct ufs_hba *hba;
1750         int i;
1751         int lun_qdepth = 0;
1752
1753         hba = shost_priv(cmd->device->host);
1754
1755         /*
1756          * LUN queue depth can be obtained by counting outstanding commands
1757          * on the LUN.
1758          */
1759         for (i = 0; i < hba->nutrs; i++) {
1760                 if (test_bit(i, &hba->outstanding_reqs)) {
1761
1762                         /*
1763                          * Check if the outstanding command belongs
1764                          * to the LUN which reported SAM_STAT_TASK_SET_FULL.
1765                          */
1766                         if (cmd->device->lun == hba->lrb[i].lun)
1767                                 lun_qdepth++;
1768                 }
1769         }
1770
1771         /*
1772          * LUN queue depth will be total outstanding commands, except the
1773          * command for which the LUN reported SAM_STAT_TASK_SET_FULL.
1774          */
1775         scsi_adjust_queue_depth(cmd->device, MSG_SIMPLE_TAG, lun_qdepth - 1);
1776 }
1777
1778 /**
1779  * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
1780  * @lrb: pointer to local reference block of completed command
1781  * @scsi_status: SCSI command status
1782  *
1783  * Returns value base on SCSI command status
1784  */
1785 static inline int
1786 ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
1787 {
1788         int result = 0;
1789
1790         switch (scsi_status) {
1791         case SAM_STAT_GOOD:
1792                 result |= DID_OK << 16 |
1793                           COMMAND_COMPLETE << 8 |
1794                           SAM_STAT_GOOD;
1795                 break;
1796         case SAM_STAT_CHECK_CONDITION:
1797                 result |= DID_OK << 16 |
1798                           COMMAND_COMPLETE << 8 |
1799                           SAM_STAT_CHECK_CONDITION;
1800                 ufshcd_copy_sense_data(lrbp);
1801                 break;
1802         case SAM_STAT_BUSY:
1803                 result |= SAM_STAT_BUSY;
1804                 break;
1805         case SAM_STAT_TASK_SET_FULL:
1806
1807                 /*
1808                  * If a LUN reports SAM_STAT_TASK_SET_FULL, then the LUN queue
1809                  * depth needs to be adjusted to the exact number of
1810                  * outstanding commands the LUN can handle at any given time.
1811                  */
1812                 ufshcd_adjust_lun_qdepth(lrbp->cmd);
1813                 result |= SAM_STAT_TASK_SET_FULL;
1814                 break;
1815         case SAM_STAT_TASK_ABORTED:
1816                 result |= SAM_STAT_TASK_ABORTED;
1817                 break;
1818         default:
1819                 result |= DID_ERROR << 16;
1820                 break;
1821         } /* end of switch */
1822
1823         return result;
1824 }
1825
1826 /**
1827  * ufshcd_transfer_rsp_status - Get overall status of the response
1828  * @hba: per adapter instance
1829  * @lrb: pointer to local reference block of completed command
1830  *
1831  * Returns result of the command to notify SCSI midlayer
1832  */
1833 static inline int
1834 ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
1835 {
1836         int result = 0;
1837         int scsi_status;
1838         int ocs;
1839
1840         /* overall command status of utrd */
1841         ocs = ufshcd_get_tr_ocs(lrbp);
1842
1843         switch (ocs) {
1844         case OCS_SUCCESS:
1845                 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
1846
1847                 switch (result) {
1848                 case UPIU_TRANSACTION_RESPONSE:
1849                         /*
1850                          * get the response UPIU result to extract
1851                          * the SCSI command status
1852                          */
1853                         result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
1854
1855                         /*
1856                          * get the result based on SCSI status response
1857                          * to notify the SCSI midlayer of the command status
1858                          */
1859                         scsi_status = result & MASK_SCSI_STATUS;
1860                         result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
1861
1862                         if (ufshcd_is_exception_event(lrbp->ucd_rsp_ptr))
1863                                 schedule_work(&hba->eeh_work);
1864                         break;
1865                 case UPIU_TRANSACTION_REJECT_UPIU:
1866                         /* TODO: handle Reject UPIU Response */
1867                         result = DID_ERROR << 16;
1868                         dev_err(hba->dev,
1869                                 "Reject UPIU not fully implemented\n");
1870                         break;
1871                 default:
1872                         result = DID_ERROR << 16;
1873                         dev_err(hba->dev,
1874                                 "Unexpected request response code = %x\n",
1875                                 result);
1876                         break;
1877                 }
1878                 break;
1879         case OCS_ABORTED:
1880                 result |= DID_ABORT << 16;
1881                 break;
1882         case OCS_INVALID_CMD_TABLE_ATTR:
1883         case OCS_INVALID_PRDT_ATTR:
1884         case OCS_MISMATCH_DATA_BUF_SIZE:
1885         case OCS_MISMATCH_RESP_UPIU_SIZE:
1886         case OCS_PEER_COMM_FAILURE:
1887         case OCS_FATAL_ERROR:
1888         default:
1889                 result |= DID_ERROR << 16;
1890                 dev_err(hba->dev,
1891                 "OCS error from controller = %x\n", ocs);
1892                 break;
1893         } /* end of switch */
1894
1895         return result;
1896 }
1897
1898 /**
1899  * ufshcd_uic_cmd_compl - handle completion of uic command
1900  * @hba: per adapter instance
1901  */
1902 static void ufshcd_uic_cmd_compl(struct ufs_hba *hba)
1903 {
1904         if (hba->active_uic_cmd) {
1905                 hba->active_uic_cmd->argument2 |=
1906                         ufshcd_get_uic_cmd_result(hba);
1907                 complete(&hba->active_uic_cmd->done);
1908         }
1909 }
1910
1911 /**
1912  * ufshcd_transfer_req_compl - handle SCSI and query command completion
1913  * @hba: per adapter instance
1914  */
1915 static void ufshcd_transfer_req_compl(struct ufs_hba *hba)
1916 {
1917         struct ufshcd_lrb *lrbp;
1918         struct scsi_cmnd *cmd;
1919         unsigned long completed_reqs;
1920         u32 tr_doorbell;
1921         int result;
1922         int index;
1923         bool int_aggr_reset = false;
1924
1925         tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
1926         completed_reqs = tr_doorbell ^ hba->outstanding_reqs;
1927
1928         for (index = 0; index < hba->nutrs; index++) {
1929                 if (test_bit(index, &completed_reqs)) {
1930                         lrbp = &hba->lrb[index];
1931                         cmd = lrbp->cmd;
1932                         /*
1933                          * Don't skip resetting interrupt aggregation counters
1934                          * if a regular command is present.
1935                          */
1936                         int_aggr_reset |= !lrbp->intr_cmd;
1937
1938                         if (cmd) {
1939                                 result = ufshcd_transfer_rsp_status(hba, lrbp);
1940                                 scsi_dma_unmap(cmd);
1941                                 cmd->result = result;
1942                                 /* Mark completed command as NULL in LRB */
1943                                 lrbp->cmd = NULL;
1944                                 clear_bit_unlock(index, &hba->lrb_in_use);
1945                                 /* Do not touch lrbp after scsi done */
1946                                 cmd->scsi_done(cmd);
1947                         } else if (lrbp->command_type ==
1948                                         UTP_CMD_TYPE_DEV_MANAGE) {
1949                                 if (hba->dev_cmd.complete)
1950                                         complete(hba->dev_cmd.complete);
1951                         }
1952                 } /* end of if */
1953         } /* end of for */
1954
1955         /* clear corresponding bits of completed commands */
1956         hba->outstanding_reqs ^= completed_reqs;
1957
1958         /* we might have free'd some tags above */
1959         wake_up(&hba->dev_cmd.tag_wq);
1960
1961         /* Reset interrupt aggregation counters */
1962         if (int_aggr_reset)
1963                 ufshcd_config_int_aggr(hba, INT_AGGR_RESET);
1964 }
1965
1966 /**
1967  * ufshcd_disable_ee - disable exception event
1968  * @hba: per-adapter instance
1969  * @mask: exception event to disable
1970  *
1971  * Disables exception event in the device so that the EVENT_ALERT
1972  * bit is not set.
1973  *
1974  * Returns zero on success, non-zero error value on failure.
1975  */
1976 static int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
1977 {
1978         int err = 0;
1979         u32 val;
1980
1981         if (!(hba->ee_ctrl_mask & mask))
1982                 goto out;
1983
1984         val = hba->ee_ctrl_mask & ~mask;
1985         val &= 0xFFFF; /* 2 bytes */
1986         err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
1987                         QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
1988         if (!err)
1989                 hba->ee_ctrl_mask &= ~mask;
1990 out:
1991         return err;
1992 }
1993
1994 /**
1995  * ufshcd_enable_ee - enable exception event
1996  * @hba: per-adapter instance
1997  * @mask: exception event to enable
1998  *
1999  * Enable corresponding exception event in the device to allow
2000  * device to alert host in critical scenarios.
2001  *
2002  * Returns zero on success, non-zero error value on failure.
2003  */
2004 static int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
2005 {
2006         int err = 0;
2007         u32 val;
2008
2009         if (hba->ee_ctrl_mask & mask)
2010                 goto out;
2011
2012         val = hba->ee_ctrl_mask | mask;
2013         val &= 0xFFFF; /* 2 bytes */
2014         err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
2015                         QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
2016         if (!err)
2017                 hba->ee_ctrl_mask |= mask;
2018 out:
2019         return err;
2020 }
2021
2022 /**
2023  * ufshcd_enable_auto_bkops - Allow device managed BKOPS
2024  * @hba: per-adapter instance
2025  *
2026  * Allow device to manage background operations on its own. Enabling
2027  * this might lead to inconsistent latencies during normal data transfers
2028  * as the device is allowed to manage its own way of handling background
2029  * operations.
2030  *
2031  * Returns zero on success, non-zero on failure.
2032  */
2033 static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
2034 {
2035         int err = 0;
2036
2037         if (hba->auto_bkops_enabled)
2038                 goto out;
2039
2040         err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_SET_FLAG,
2041                         QUERY_FLAG_IDN_BKOPS_EN, NULL);
2042         if (err) {
2043                 dev_err(hba->dev, "%s: failed to enable bkops %d\n",
2044                                 __func__, err);
2045                 goto out;
2046         }
2047
2048         hba->auto_bkops_enabled = true;
2049
2050         /* No need of URGENT_BKOPS exception from the device */
2051         err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
2052         if (err)
2053                 dev_err(hba->dev, "%s: failed to disable exception event %d\n",
2054                                 __func__, err);
2055 out:
2056         return err;
2057 }
2058
2059 /**
2060  * ufshcd_disable_auto_bkops - block device in doing background operations
2061  * @hba: per-adapter instance
2062  *
2063  * Disabling background operations improves command response latency but
2064  * has drawback of device moving into critical state where the device is
2065  * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
2066  * host is idle so that BKOPS are managed effectively without any negative
2067  * impacts.
2068  *
2069  * Returns zero on success, non-zero on failure.
2070  */
2071 static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
2072 {
2073         int err = 0;
2074
2075         if (!hba->auto_bkops_enabled)
2076                 goto out;
2077
2078         /*
2079          * If host assisted BKOPs is to be enabled, make sure
2080          * urgent bkops exception is allowed.
2081          */
2082         err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
2083         if (err) {
2084                 dev_err(hba->dev, "%s: failed to enable exception event %d\n",
2085                                 __func__, err);
2086                 goto out;
2087         }
2088
2089         err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
2090                         QUERY_FLAG_IDN_BKOPS_EN, NULL);
2091         if (err) {
2092                 dev_err(hba->dev, "%s: failed to disable bkops %d\n",
2093                                 __func__, err);
2094                 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
2095                 goto out;
2096         }
2097
2098         hba->auto_bkops_enabled = false;
2099 out:
2100         return err;
2101 }
2102
2103 /**
2104  * ufshcd_force_reset_auto_bkops - force enable of auto bkops
2105  * @hba: per adapter instance
2106  *
2107  * After a device reset the device may toggle the BKOPS_EN flag
2108  * to default value. The s/w tracking variables should be updated
2109  * as well. Do this by forcing enable of auto bkops.
2110  */
2111 static void  ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
2112 {
2113         hba->auto_bkops_enabled = false;
2114         hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
2115         ufshcd_enable_auto_bkops(hba);
2116 }
2117
2118 static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
2119 {
2120         return ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_READ_ATTR,
2121                         QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
2122 }
2123
2124 /**
2125  * ufshcd_urgent_bkops - handle urgent bkops exception event
2126  * @hba: per-adapter instance
2127  *
2128  * Enable fBackgroundOpsEn flag in the device to permit background
2129  * operations.
2130  */
2131 static int ufshcd_urgent_bkops(struct ufs_hba *hba)
2132 {
2133         int err;
2134         u32 status = 0;
2135
2136         err = ufshcd_get_bkops_status(hba, &status);
2137         if (err) {
2138                 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
2139                                 __func__, err);
2140                 goto out;
2141         }
2142
2143         status = status & 0xF;
2144
2145         /* handle only if status indicates performance impact or critical */
2146         if (status >= BKOPS_STATUS_PERF_IMPACT)
2147                 err = ufshcd_enable_auto_bkops(hba);
2148 out:
2149         return err;
2150 }
2151
2152 static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
2153 {
2154         return ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_READ_ATTR,
2155                         QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
2156 }
2157
2158 /**
2159  * ufshcd_exception_event_handler - handle exceptions raised by device
2160  * @work: pointer to work data
2161  *
2162  * Read bExceptionEventStatus attribute from the device and handle the
2163  * exception event accordingly.
2164  */
2165 static void ufshcd_exception_event_handler(struct work_struct *work)
2166 {
2167         struct ufs_hba *hba;
2168         int err;
2169         u32 status = 0;
2170         hba = container_of(work, struct ufs_hba, eeh_work);
2171
2172         pm_runtime_get_sync(hba->dev);
2173         err = ufshcd_get_ee_status(hba, &status);
2174         if (err) {
2175                 dev_err(hba->dev, "%s: failed to get exception status %d\n",
2176                                 __func__, err);
2177                 goto out;
2178         }
2179
2180         status &= hba->ee_ctrl_mask;
2181         if (status & MASK_EE_URGENT_BKOPS) {
2182                 err = ufshcd_urgent_bkops(hba);
2183                 if (err)
2184                         dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
2185                                         __func__, err);
2186         }
2187 out:
2188         pm_runtime_put_sync(hba->dev);
2189         return;
2190 }
2191
2192 /**
2193  * ufshcd_fatal_err_handler - handle fatal errors
2194  * @hba: per adapter instance
2195  */
2196 static void ufshcd_fatal_err_handler(struct work_struct *work)
2197 {
2198         struct ufs_hba *hba;
2199         hba = container_of(work, struct ufs_hba, feh_workq);
2200
2201         pm_runtime_get_sync(hba->dev);
2202         /* check if reset is already in progress */
2203         if (hba->ufshcd_state != UFSHCD_STATE_RESET)
2204                 ufshcd_do_reset(hba);
2205         pm_runtime_put_sync(hba->dev);
2206 }
2207
2208 /**
2209  * ufshcd_err_handler - Check for fatal errors
2210  * @work: pointer to a work queue structure
2211  */
2212 static void ufshcd_err_handler(struct ufs_hba *hba)
2213 {
2214         u32 reg;
2215
2216         if (hba->errors & INT_FATAL_ERRORS)
2217                 goto fatal_eh;
2218
2219         if (hba->errors & UIC_ERROR) {
2220                 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
2221                 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
2222                         goto fatal_eh;
2223         }
2224         return;
2225 fatal_eh:
2226         hba->ufshcd_state = UFSHCD_STATE_ERROR;
2227         schedule_work(&hba->feh_workq);
2228 }
2229
2230 /**
2231  * ufshcd_tmc_handler - handle task management function completion
2232  * @hba: per adapter instance
2233  */
2234 static void ufshcd_tmc_handler(struct ufs_hba *hba)
2235 {
2236         u32 tm_doorbell;
2237
2238         tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
2239         hba->tm_condition = tm_doorbell ^ hba->outstanding_tasks;
2240         wake_up_interruptible(&hba->ufshcd_tm_wait_queue);
2241 }
2242
2243 /**
2244  * ufshcd_sl_intr - Interrupt service routine
2245  * @hba: per adapter instance
2246  * @intr_status: contains interrupts generated by the controller
2247  */
2248 static void ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
2249 {
2250         hba->errors = UFSHCD_ERROR_MASK & intr_status;
2251         if (hba->errors)
2252                 ufshcd_err_handler(hba);
2253
2254         if (intr_status & UIC_COMMAND_COMPL)
2255                 ufshcd_uic_cmd_compl(hba);
2256
2257         if (intr_status & UTP_TASK_REQ_COMPL)
2258                 ufshcd_tmc_handler(hba);
2259
2260         if (intr_status & UTP_TRANSFER_REQ_COMPL)
2261                 ufshcd_transfer_req_compl(hba);
2262 }
2263
2264 /**
2265  * ufshcd_intr - Main interrupt service routine
2266  * @irq: irq number
2267  * @__hba: pointer to adapter instance
2268  *
2269  * Returns IRQ_HANDLED - If interrupt is valid
2270  *              IRQ_NONE - If invalid interrupt
2271  */
2272 static irqreturn_t ufshcd_intr(int irq, void *__hba)
2273 {
2274         u32 intr_status;
2275         irqreturn_t retval = IRQ_NONE;
2276         struct ufs_hba *hba = __hba;
2277
2278         spin_lock(hba->host->host_lock);
2279         intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
2280
2281         if (intr_status) {
2282                 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
2283                 ufshcd_sl_intr(hba, intr_status);
2284                 retval = IRQ_HANDLED;
2285         }
2286         spin_unlock(hba->host->host_lock);
2287         return retval;
2288 }
2289
2290 /**
2291  * ufshcd_issue_tm_cmd - issues task management commands to controller
2292  * @hba: per adapter instance
2293  * @lrbp: pointer to local reference block
2294  *
2295  * Returns SUCCESS/FAILED
2296  */
2297 static int
2298 ufshcd_issue_tm_cmd(struct ufs_hba *hba,
2299                     struct ufshcd_lrb *lrbp,
2300                     u8 tm_function)
2301 {
2302         struct utp_task_req_desc *task_req_descp;
2303         struct utp_upiu_task_req *task_req_upiup;
2304         struct Scsi_Host *host;
2305         unsigned long flags;
2306         int free_slot = 0;
2307         int err;
2308
2309         host = hba->host;
2310
2311         spin_lock_irqsave(host->host_lock, flags);
2312
2313         /* If task management queue is full */
2314         free_slot = ufshcd_get_tm_free_slot(hba);
2315         if (free_slot >= hba->nutmrs) {
2316                 spin_unlock_irqrestore(host->host_lock, flags);
2317                 dev_err(hba->dev, "Task management queue full\n");
2318                 err = FAILED;
2319                 goto out;
2320         }
2321
2322         task_req_descp = hba->utmrdl_base_addr;
2323         task_req_descp += free_slot;
2324
2325         /* Configure task request descriptor */
2326         task_req_descp->header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
2327         task_req_descp->header.dword_2 =
2328                         cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
2329
2330         /* Configure task request UPIU */
2331         task_req_upiup =
2332                 (struct utp_upiu_task_req *) task_req_descp->task_req_upiu;
2333         task_req_upiup->header.dword_0 =
2334                 UPIU_HEADER_DWORD(UPIU_TRANSACTION_TASK_REQ, 0,
2335                                               lrbp->lun, lrbp->task_tag);
2336         task_req_upiup->header.dword_1 =
2337                 UPIU_HEADER_DWORD(0, tm_function, 0, 0);
2338
2339         task_req_upiup->input_param1 = lrbp->lun;
2340         task_req_upiup->input_param1 =
2341                 cpu_to_be32(task_req_upiup->input_param1);
2342         task_req_upiup->input_param2 = lrbp->task_tag;
2343         task_req_upiup->input_param2 =
2344                 cpu_to_be32(task_req_upiup->input_param2);
2345
2346         /* send command to the controller */
2347         __set_bit(free_slot, &hba->outstanding_tasks);
2348         ufshcd_writel(hba, 1 << free_slot, REG_UTP_TASK_REQ_DOOR_BELL);
2349
2350         spin_unlock_irqrestore(host->host_lock, flags);
2351
2352         /* wait until the task management command is completed */
2353         err =
2354         wait_event_interruptible_timeout(hba->ufshcd_tm_wait_queue,
2355                                          (test_bit(free_slot,
2356                                          &hba->tm_condition) != 0),
2357                                          60 * HZ);
2358         if (!err) {
2359                 dev_err(hba->dev,
2360                         "Task management command timed-out\n");
2361                 err = FAILED;
2362                 goto out;
2363         }
2364         clear_bit(free_slot, &hba->tm_condition);
2365         err = ufshcd_task_req_compl(hba, free_slot);
2366 out:
2367         return err;
2368 }
2369
2370 /**
2371  * ufshcd_device_reset - reset device and abort all the pending commands
2372  * @cmd: SCSI command pointer
2373  *
2374  * Returns SUCCESS/FAILED
2375  */
2376 static int ufshcd_device_reset(struct scsi_cmnd *cmd)
2377 {
2378         struct Scsi_Host *host;
2379         struct ufs_hba *hba;
2380         unsigned int tag;
2381         u32 pos;
2382         int err;
2383
2384         host = cmd->device->host;
2385         hba = shost_priv(host);
2386         tag = cmd->request->tag;
2387
2388         err = ufshcd_issue_tm_cmd(hba, &hba->lrb[tag], UFS_LOGICAL_RESET);
2389         if (err == FAILED)
2390                 goto out;
2391
2392         for (pos = 0; pos < hba->nutrs; pos++) {
2393                 if (test_bit(pos, &hba->outstanding_reqs) &&
2394                     (hba->lrb[tag].lun == hba->lrb[pos].lun)) {
2395
2396                         /* clear the respective UTRLCLR register bit */
2397                         ufshcd_utrl_clear(hba, pos);
2398
2399                         clear_bit(pos, &hba->outstanding_reqs);
2400
2401                         if (hba->lrb[pos].cmd) {
2402                                 scsi_dma_unmap(hba->lrb[pos].cmd);
2403                                 hba->lrb[pos].cmd->result =
2404                                         DID_ABORT << 16;
2405                                 hba->lrb[pos].cmd->scsi_done(cmd);
2406                                 hba->lrb[pos].cmd = NULL;
2407                                 clear_bit_unlock(pos, &hba->lrb_in_use);
2408                                 wake_up(&hba->dev_cmd.tag_wq);
2409                         }
2410                 }
2411         } /* end of for */
2412 out:
2413         return err;
2414 }
2415
2416 /**
2417  * ufshcd_host_reset - Main reset function registered with scsi layer
2418  * @cmd: SCSI command pointer
2419  *
2420  * Returns SUCCESS/FAILED
2421  */
2422 static int ufshcd_host_reset(struct scsi_cmnd *cmd)
2423 {
2424         struct ufs_hba *hba;
2425
2426         hba = shost_priv(cmd->device->host);
2427
2428         if (hba->ufshcd_state == UFSHCD_STATE_RESET)
2429                 return SUCCESS;
2430
2431         return ufshcd_do_reset(hba);
2432 }
2433
2434 /**
2435  * ufshcd_abort - abort a specific command
2436  * @cmd: SCSI command pointer
2437  *
2438  * Returns SUCCESS/FAILED
2439  */
2440 static int ufshcd_abort(struct scsi_cmnd *cmd)
2441 {
2442         struct Scsi_Host *host;
2443         struct ufs_hba *hba;
2444         unsigned long flags;
2445         unsigned int tag;
2446         int err;
2447
2448         host = cmd->device->host;
2449         hba = shost_priv(host);
2450         tag = cmd->request->tag;
2451
2452         spin_lock_irqsave(host->host_lock, flags);
2453
2454         /* check if command is still pending */
2455         if (!(test_bit(tag, &hba->outstanding_reqs))) {
2456                 err = FAILED;
2457                 spin_unlock_irqrestore(host->host_lock, flags);
2458                 goto out;
2459         }
2460         spin_unlock_irqrestore(host->host_lock, flags);
2461
2462         err = ufshcd_issue_tm_cmd(hba, &hba->lrb[tag], UFS_ABORT_TASK);
2463         if (err == FAILED)
2464                 goto out;
2465
2466         scsi_dma_unmap(cmd);
2467
2468         spin_lock_irqsave(host->host_lock, flags);
2469
2470         /* clear the respective UTRLCLR register bit */
2471         ufshcd_utrl_clear(hba, tag);
2472
2473         __clear_bit(tag, &hba->outstanding_reqs);
2474         hba->lrb[tag].cmd = NULL;
2475         spin_unlock_irqrestore(host->host_lock, flags);
2476
2477         clear_bit_unlock(tag, &hba->lrb_in_use);
2478         wake_up(&hba->dev_cmd.tag_wq);
2479 out:
2480         return err;
2481 }
2482
2483 /**
2484  * ufshcd_async_scan - asynchronous execution for link startup
2485  * @data: data pointer to pass to this function
2486  * @cookie: cookie data
2487  */
2488 static void ufshcd_async_scan(void *data, async_cookie_t cookie)
2489 {
2490         struct ufs_hba *hba = (struct ufs_hba *)data;
2491         int ret;
2492
2493         ret = ufshcd_link_startup(hba);
2494         if (ret)
2495                 goto out;
2496
2497         ret = ufshcd_verify_dev_init(hba);
2498         if (ret)
2499                 goto out;
2500
2501         ret = ufshcd_complete_dev_init(hba);
2502         if (ret)
2503                 goto out;
2504
2505         ufshcd_force_reset_auto_bkops(hba);
2506         scsi_scan_host(hba->host);
2507         pm_runtime_put_sync(hba->dev);
2508 out:
2509         return;
2510 }
2511
2512 static struct scsi_host_template ufshcd_driver_template = {
2513         .module                 = THIS_MODULE,
2514         .name                   = UFSHCD,
2515         .proc_name              = UFSHCD,
2516         .queuecommand           = ufshcd_queuecommand,
2517         .slave_alloc            = ufshcd_slave_alloc,
2518         .slave_destroy          = ufshcd_slave_destroy,
2519         .eh_abort_handler       = ufshcd_abort,
2520         .eh_device_reset_handler = ufshcd_device_reset,
2521         .eh_host_reset_handler  = ufshcd_host_reset,
2522         .this_id                = -1,
2523         .sg_tablesize           = SG_ALL,
2524         .cmd_per_lun            = UFSHCD_CMD_PER_LUN,
2525         .can_queue              = UFSHCD_CAN_QUEUE,
2526 };
2527
2528 /**
2529  * ufshcd_suspend - suspend power management function
2530  * @hba: per adapter instance
2531  * @state: power state
2532  *
2533  * Returns -ENOSYS
2534  */
2535 int ufshcd_suspend(struct ufs_hba *hba, pm_message_t state)
2536 {
2537         /*
2538          * TODO:
2539          * 1. Block SCSI requests from SCSI midlayer
2540          * 2. Change the internal driver state to non operational
2541          * 3. Set UTRLRSR and UTMRLRSR bits to zero
2542          * 4. Wait until outstanding commands are completed
2543          * 5. Set HCE to zero to send the UFS host controller to reset state
2544          */
2545
2546         return -ENOSYS;
2547 }
2548 EXPORT_SYMBOL_GPL(ufshcd_suspend);
2549
2550 /**
2551  * ufshcd_resume - resume power management function
2552  * @hba: per adapter instance
2553  *
2554  * Returns -ENOSYS
2555  */
2556 int ufshcd_resume(struct ufs_hba *hba)
2557 {
2558         /*
2559          * TODO:
2560          * 1. Set HCE to 1, to start the UFS host controller
2561          * initialization process
2562          * 2. Set UTRLRSR and UTMRLRSR bits to 1
2563          * 3. Change the internal driver state to operational
2564          * 4. Unblock SCSI requests from SCSI midlayer
2565          */
2566
2567         return -ENOSYS;
2568 }
2569 EXPORT_SYMBOL_GPL(ufshcd_resume);
2570
2571 int ufshcd_runtime_suspend(struct ufs_hba *hba)
2572 {
2573         if (!hba)
2574                 return 0;
2575
2576         /*
2577          * The device is idle with no requests in the queue,
2578          * allow background operations.
2579          */
2580         return ufshcd_enable_auto_bkops(hba);
2581 }
2582 EXPORT_SYMBOL(ufshcd_runtime_suspend);
2583
2584 int ufshcd_runtime_resume(struct ufs_hba *hba)
2585 {
2586         if (!hba)
2587                 return 0;
2588
2589         return ufshcd_disable_auto_bkops(hba);
2590 }
2591 EXPORT_SYMBOL(ufshcd_runtime_resume);
2592
2593 int ufshcd_runtime_idle(struct ufs_hba *hba)
2594 {
2595         return 0;
2596 }
2597 EXPORT_SYMBOL(ufshcd_runtime_idle);
2598
2599 /**
2600  * ufshcd_remove - de-allocate SCSI host and host memory space
2601  *              data structure memory
2602  * @hba - per adapter instance
2603  */
2604 void ufshcd_remove(struct ufs_hba *hba)
2605 {
2606         scsi_remove_host(hba->host);
2607         /* disable interrupts */
2608         ufshcd_disable_intr(hba, hba->intr_mask);
2609         ufshcd_hba_stop(hba);
2610
2611         scsi_host_put(hba->host);
2612 }
2613 EXPORT_SYMBOL_GPL(ufshcd_remove);
2614
2615 /**
2616  * ufshcd_init - Driver initialization routine
2617  * @dev: pointer to device handle
2618  * @hba_handle: driver private handle
2619  * @mmio_base: base register address
2620  * @irq: Interrupt line of device
2621  * Returns 0 on success, non-zero value on failure
2622  */
2623 int ufshcd_init(struct device *dev, struct ufs_hba **hba_handle,
2624                  void __iomem *mmio_base, unsigned int irq)
2625 {
2626         struct Scsi_Host *host;
2627         struct ufs_hba *hba;
2628         int err;
2629
2630         if (!dev) {
2631                 dev_err(dev,
2632                 "Invalid memory reference for dev is NULL\n");
2633                 err = -ENODEV;
2634                 goto out_error;
2635         }
2636
2637         if (!mmio_base) {
2638                 dev_err(dev,
2639                 "Invalid memory reference for mmio_base is NULL\n");
2640                 err = -ENODEV;
2641                 goto out_error;
2642         }
2643
2644         host = scsi_host_alloc(&ufshcd_driver_template,
2645                                 sizeof(struct ufs_hba));
2646         if (!host) {
2647                 dev_err(dev, "scsi_host_alloc failed\n");
2648                 err = -ENOMEM;
2649                 goto out_error;
2650         }
2651         hba = shost_priv(host);
2652         hba->host = host;
2653         hba->dev = dev;
2654         hba->mmio_base = mmio_base;
2655         hba->irq = irq;
2656
2657         /* Read capabilities registers */
2658         ufshcd_hba_capabilities(hba);
2659
2660         /* Get UFS version supported by the controller */
2661         hba->ufs_version = ufshcd_get_ufs_version(hba);
2662
2663         /* Get Interrupt bit mask per version */
2664         hba->intr_mask = ufshcd_get_intr_mask(hba);
2665
2666         /* Allocate memory for host memory space */
2667         err = ufshcd_memory_alloc(hba);
2668         if (err) {
2669                 dev_err(hba->dev, "Memory allocation failed\n");
2670                 goto out_disable;
2671         }
2672
2673         /* Configure LRB */
2674         ufshcd_host_memory_configure(hba);
2675
2676         host->can_queue = hba->nutrs;
2677         host->cmd_per_lun = hba->nutrs;
2678         host->max_id = UFSHCD_MAX_ID;
2679         host->max_lun = UFSHCD_MAX_LUNS;
2680         host->max_channel = UFSHCD_MAX_CHANNEL;
2681         host->unique_id = host->host_no;
2682         host->max_cmd_len = MAX_CDB_SIZE;
2683
2684         /* Initailize wait queue for task management */
2685         init_waitqueue_head(&hba->ufshcd_tm_wait_queue);
2686
2687         /* Initialize work queues */
2688         INIT_WORK(&hba->feh_workq, ufshcd_fatal_err_handler);
2689         INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
2690
2691         /* Initialize UIC command mutex */
2692         mutex_init(&hba->uic_cmd_mutex);
2693
2694         /* Initialize mutex for device management commands */
2695         mutex_init(&hba->dev_cmd.lock);
2696
2697         /* Initialize device management tag acquire wait queue */
2698         init_waitqueue_head(&hba->dev_cmd.tag_wq);
2699
2700         /* IRQ registration */
2701         err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
2702         if (err) {
2703                 dev_err(hba->dev, "request irq failed\n");
2704                 goto out_disable;
2705         }
2706
2707         /* Enable SCSI tag mapping */
2708         err = scsi_init_shared_tag_map(host, host->can_queue);
2709         if (err) {
2710                 dev_err(hba->dev, "init shared queue failed\n");
2711                 goto out_disable;
2712         }
2713
2714         err = scsi_add_host(host, hba->dev);
2715         if (err) {
2716                 dev_err(hba->dev, "scsi_add_host failed\n");
2717                 goto out_disable;
2718         }
2719
2720         /* Host controller enable */
2721         err = ufshcd_hba_enable(hba);
2722         if (err) {
2723                 dev_err(hba->dev, "Host controller enable failed\n");
2724                 goto out_remove_scsi_host;
2725         }
2726
2727         *hba_handle = hba;
2728
2729         /* Hold auto suspend until async scan completes */
2730         pm_runtime_get_sync(dev);
2731
2732         async_schedule(ufshcd_async_scan, hba);
2733
2734         return 0;
2735
2736 out_remove_scsi_host:
2737         scsi_remove_host(hba->host);
2738 out_disable:
2739         scsi_host_put(host);
2740 out_error:
2741         return err;
2742 }
2743 EXPORT_SYMBOL_GPL(ufshcd_init);
2744
2745 MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
2746 MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
2747 MODULE_DESCRIPTION("Generic UFS host controller driver Core");
2748 MODULE_LICENSE("GPL");
2749 MODULE_VERSION(UFSHCD_DRIVER_VERSION);