2 * Universal Flash Storage Host controller driver Core
4 * This code is based on drivers/scsi/ufs/ufshcd.c
5 * Copyright (C) 2011-2013 Samsung India Software Operations
8 * Santosh Yaraganavi <santosh.sy@samsung.com>
9 * Vinayak Holikatti <h.vinayak@samsung.com>
11 * This program is free software; you can redistribute it and/or
12 * modify it under the terms of the GNU General Public License
13 * as published by the Free Software Foundation; either version 2
14 * of the License, or (at your option) any later version.
15 * See the COPYING file in the top-level directory or visit
16 * <http://www.gnu.org/licenses/gpl-2.0.html>
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
23 * This program is provided "AS IS" and "WITH ALL FAULTS" and
24 * without warranty of any kind. You are solely responsible for
25 * determining the appropriateness of using and distributing
26 * the program and assume all risks associated with your exercise
27 * of rights with respect to the program, including but not limited
28 * to infringement of third party rights, the risks and costs of
29 * program errors, damage to or loss of data, programs or equipment,
30 * and unavailability or interruption of operations. Under no
31 * circumstances will the contributor of this Program be liable for
32 * any damages of any kind arising from your use or distribution of
36 #include <linux/async.h>
40 #define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
43 /* UIC command timeout, unit: ms */
44 #define UIC_CMD_TIMEOUT 500
46 /* NOP OUT retries waiting for NOP IN response */
47 #define NOP_OUT_RETRIES 10
48 /* Timeout after 30 msecs if NOP OUT hangs without response */
49 #define NOP_OUT_TIMEOUT 30 /* msecs */
52 UFSHCD_MAX_CHANNEL = 0,
55 UFSHCD_CMD_PER_LUN = 32,
56 UFSHCD_CAN_QUEUE = 32,
61 UFSHCD_STATE_OPERATIONAL,
66 /* Interrupt configuration options */
73 /* Interrupt aggregation options */
80 * ufshcd_wait_for_register - wait for register value to change
81 * @hba - per-adapter interface
82 * @reg - mmio register offset
83 * @mask - mask to apply to read register value
84 * @val - wait condition
85 * @interval_us - polling interval in microsecs
86 * @timeout_ms - timeout in millisecs
88 * Returns -ETIMEDOUT on error, zero on success
90 static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
91 u32 val, unsigned long interval_us, unsigned long timeout_ms)
94 unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
96 /* ignore bits that we don't intend to wait on */
99 while ((ufshcd_readl(hba, reg) & mask) != val) {
100 /* wakeup within 50us of expiry */
101 usleep_range(interval_us, interval_us + 50);
103 if (time_after(jiffies, timeout)) {
104 if ((ufshcd_readl(hba, reg) & mask) != val)
114 * ufshcd_get_intr_mask - Get the interrupt bit mask
115 * @hba - Pointer to adapter instance
117 * Returns interrupt bit mask per version
119 static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
121 if (hba->ufs_version == UFSHCI_VERSION_10)
122 return INTERRUPT_MASK_ALL_VER_10;
124 return INTERRUPT_MASK_ALL_VER_11;
128 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
129 * @hba - Pointer to adapter instance
131 * Returns UFSHCI version supported by the controller
133 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
135 return ufshcd_readl(hba, REG_UFS_VERSION);
139 * ufshcd_is_device_present - Check if any device connected to
140 * the host controller
141 * @reg_hcs - host controller status register value
143 * Returns 1 if device present, 0 if no device detected
145 static inline int ufshcd_is_device_present(u32 reg_hcs)
147 return (DEVICE_PRESENT & reg_hcs) ? 1 : 0;
151 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
152 * @lrb: pointer to local command reference block
154 * This function is used to get the OCS field from UTRD
155 * Returns the OCS field in the UTRD
157 static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
159 return lrbp->utr_descriptor_ptr->header.dword_2 & MASK_OCS;
163 * ufshcd_get_tmr_ocs - Get the UTMRD Overall Command Status
164 * @task_req_descp: pointer to utp_task_req_desc structure
166 * This function is used to get the OCS field from UTMRD
167 * Returns the OCS field in the UTMRD
170 ufshcd_get_tmr_ocs(struct utp_task_req_desc *task_req_descp)
172 return task_req_descp->header.dword_2 & MASK_OCS;
176 * ufshcd_get_tm_free_slot - get a free slot for task management request
177 * @hba: per adapter instance
179 * Returns maximum number of task management request slots in case of
180 * task management queue full or returns the free slot number
182 static inline int ufshcd_get_tm_free_slot(struct ufs_hba *hba)
184 return find_first_zero_bit(&hba->outstanding_tasks, hba->nutmrs);
188 * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
189 * @hba: per adapter instance
190 * @pos: position of the bit to be cleared
192 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
194 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
198 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
199 * @reg: Register value of host controller status
201 * Returns integer, 0 on Success and positive value if failed
203 static inline int ufshcd_get_lists_status(u32 reg)
206 * The mask 0xFF is for the following HCS register bits
216 return (((reg) & (0xFF)) >> 1) ^ (0x07);
220 * ufshcd_get_uic_cmd_result - Get the UIC command result
221 * @hba: Pointer to adapter instance
223 * This function gets the result of UIC command completion
224 * Returns 0 on success, non zero value on error
226 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
228 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
229 MASK_UIC_COMMAND_RESULT;
233 * ufshcd_get_req_rsp - returns the TR response transaction type
234 * @ucd_rsp_ptr: pointer to response UPIU
237 ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
239 return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
243 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
244 * @ucd_rsp_ptr: pointer to response UPIU
246 * This function gets the response status and scsi_status from response UPIU
247 * Returns the response result code.
250 ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
252 return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
256 * ufshcd_config_int_aggr - Configure interrupt aggregation values.
257 * Currently there is no use case where we want to configure
258 * interrupt aggregation dynamically. So to configure interrupt
259 * aggregation, #define INT_AGGR_COUNTER_THRESHOLD_VALUE and
260 * INT_AGGR_TIMEOUT_VALUE are used.
261 * @hba: per adapter instance
262 * @option: Interrupt aggregation option
265 ufshcd_config_int_aggr(struct ufs_hba *hba, int option)
269 ufshcd_writel(hba, INT_AGGR_ENABLE |
270 INT_AGGR_COUNTER_AND_TIMER_RESET,
271 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
273 case INT_AGGR_CONFIG:
274 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
275 INT_AGGR_COUNTER_THRESHOLD_VALUE |
276 INT_AGGR_TIMEOUT_VALUE,
277 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
283 * ufshcd_enable_run_stop_reg - Enable run-stop registers,
284 * When run-stop registers are set to 1, it indicates the
285 * host controller that it can process the requests
286 * @hba: per adapter instance
288 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
290 ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
291 REG_UTP_TASK_REQ_LIST_RUN_STOP);
292 ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
293 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
297 * ufshcd_hba_start - Start controller initialization sequence
298 * @hba: per adapter instance
300 static inline void ufshcd_hba_start(struct ufs_hba *hba)
302 ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE);
306 * ufshcd_is_hba_active - Get controller state
307 * @hba: per adapter instance
309 * Returns zero if controller is active, 1 otherwise
311 static inline int ufshcd_is_hba_active(struct ufs_hba *hba)
313 return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & 0x1) ? 0 : 1;
317 * ufshcd_send_command - Send SCSI or device management commands
318 * @hba: per adapter instance
319 * @task_tag: Task tag of the command
322 void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
324 __set_bit(task_tag, &hba->outstanding_reqs);
325 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
329 * ufshcd_copy_sense_data - Copy sense data in case of check condition
330 * @lrb - pointer to local reference block
332 static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
335 if (lrbp->sense_buffer) {
336 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
337 memcpy(lrbp->sense_buffer,
338 lrbp->ucd_rsp_ptr->sr.sense_data,
339 min_t(int, len, SCSI_SENSE_BUFFERSIZE));
344 * ufshcd_hba_capabilities - Read controller capabilities
345 * @hba: per adapter instance
347 static inline void ufshcd_hba_capabilities(struct ufs_hba *hba)
349 hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
351 /* nutrs and nutmrs are 0 based values */
352 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
354 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
358 * ufshcd_ready_for_uic_cmd - Check if controller is ready
359 * to accept UIC commands
360 * @hba: per adapter instance
361 * Return true on success, else false
363 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
365 if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
372 * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
373 * @hba: per adapter instance
374 * @uic_cmd: UIC command
376 * Mutex must be held.
379 ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
381 WARN_ON(hba->active_uic_cmd);
383 hba->active_uic_cmd = uic_cmd;
386 ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
387 ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
388 ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
391 ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
396 * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
397 * @hba: per adapter instance
398 * @uic_command: UIC command
400 * Must be called with mutex held.
401 * Returns 0 only if success.
404 ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
409 if (wait_for_completion_timeout(&uic_cmd->done,
410 msecs_to_jiffies(UIC_CMD_TIMEOUT)))
411 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
415 spin_lock_irqsave(hba->host->host_lock, flags);
416 hba->active_uic_cmd = NULL;
417 spin_unlock_irqrestore(hba->host->host_lock, flags);
423 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
424 * @hba: per adapter instance
425 * @uic_cmd: UIC command
427 * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
429 * Returns 0 only if success.
432 __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
437 if (!ufshcd_ready_for_uic_cmd(hba)) {
439 "Controller not ready to accept UIC commands\n");
443 init_completion(&uic_cmd->done);
445 spin_lock_irqsave(hba->host->host_lock, flags);
446 ufshcd_dispatch_uic_cmd(hba, uic_cmd);
447 spin_unlock_irqrestore(hba->host->host_lock, flags);
449 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
455 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
456 * @hba: per adapter instance
457 * @uic_cmd: UIC command
459 * Returns 0 only if success.
462 ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
466 mutex_lock(&hba->uic_cmd_mutex);
467 ret = __ufshcd_send_uic_cmd(hba, uic_cmd);
468 mutex_unlock(&hba->uic_cmd_mutex);
474 * ufshcd_map_sg - Map scatter-gather list to prdt
475 * @lrbp - pointer to local reference block
477 * Returns 0 in case of success, non-zero value in case of failure
479 static int ufshcd_map_sg(struct ufshcd_lrb *lrbp)
481 struct ufshcd_sg_entry *prd_table;
482 struct scatterlist *sg;
483 struct scsi_cmnd *cmd;
488 sg_segments = scsi_dma_map(cmd);
493 lrbp->utr_descriptor_ptr->prd_table_length =
494 cpu_to_le16((u16) (sg_segments));
496 prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
498 scsi_for_each_sg(cmd, sg, sg_segments, i) {
500 cpu_to_le32(((u32) sg_dma_len(sg))-1);
501 prd_table[i].base_addr =
502 cpu_to_le32(lower_32_bits(sg->dma_address));
503 prd_table[i].upper_addr =
504 cpu_to_le32(upper_32_bits(sg->dma_address));
507 lrbp->utr_descriptor_ptr->prd_table_length = 0;
514 * ufshcd_enable_intr - enable interrupts
515 * @hba: per adapter instance
516 * @intrs: interrupt bits
518 static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
520 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
522 if (hba->ufs_version == UFSHCI_VERSION_10) {
524 rw = set & INTERRUPT_MASK_RW_VER_10;
525 set = rw | ((set ^ intrs) & intrs);
530 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
534 * ufshcd_disable_intr - disable interrupts
535 * @hba: per adapter instance
536 * @intrs: interrupt bits
538 static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
540 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
542 if (hba->ufs_version == UFSHCI_VERSION_10) {
544 rw = (set & INTERRUPT_MASK_RW_VER_10) &
545 ~(intrs & INTERRUPT_MASK_RW_VER_10);
546 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
552 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
556 * ufshcd_prepare_req_desc_hdr() - Fills the requests header
557 * descriptor according to request
558 * @lrbp: pointer to local reference block
559 * @upiu_flags: flags required in the header
560 * @cmd_dir: requests data direction
562 static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
563 u32 *upiu_flags, enum dma_data_direction cmd_dir)
565 struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
569 if (cmd_dir == DMA_FROM_DEVICE) {
570 data_direction = UTP_DEVICE_TO_HOST;
571 *upiu_flags = UPIU_CMD_FLAGS_READ;
572 } else if (cmd_dir == DMA_TO_DEVICE) {
573 data_direction = UTP_HOST_TO_DEVICE;
574 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
576 data_direction = UTP_NO_DATA_TRANSFER;
577 *upiu_flags = UPIU_CMD_FLAGS_NONE;
580 dword_0 = data_direction | (lrbp->command_type
581 << UPIU_COMMAND_TYPE_OFFSET);
583 dword_0 |= UTP_REQ_DESC_INT_CMD;
585 /* Transfer request descriptor header fields */
586 req_desc->header.dword_0 = cpu_to_le32(dword_0);
589 * assigning invalid value for command status. Controller
590 * updates OCS on command completion, with the command
593 req_desc->header.dword_2 =
594 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
598 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
600 * @lrbp - local reference block pointer
601 * @upiu_flags - flags
604 void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u32 upiu_flags)
606 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
608 /* command descriptor fields */
609 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
610 UPIU_TRANSACTION_COMMAND, upiu_flags,
611 lrbp->lun, lrbp->task_tag);
612 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
613 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
615 /* Total EHS length and Data segment length will be zero */
616 ucd_req_ptr->header.dword_2 = 0;
618 ucd_req_ptr->sc.exp_data_transfer_len =
619 cpu_to_be32(lrbp->cmd->sdb.length);
621 memcpy(ucd_req_ptr->sc.cdb, lrbp->cmd->cmnd,
622 (min_t(unsigned short, lrbp->cmd->cmd_len, MAX_CDB_SIZE)));
625 static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
627 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
629 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
631 /* command descriptor fields */
632 ucd_req_ptr->header.dword_0 =
634 UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
638 * ufshcd_compose_upiu - form UFS Protocol Information Unit(UPIU)
639 * @hba - per adapter instance
640 * @lrb - pointer to local reference block
642 static int ufshcd_compose_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
647 switch (lrbp->command_type) {
648 case UTP_CMD_TYPE_SCSI:
649 if (likely(lrbp->cmd)) {
650 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
651 lrbp->cmd->sc_data_direction);
652 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
657 case UTP_CMD_TYPE_DEV_MANAGE:
658 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
659 if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
660 ufshcd_prepare_utp_nop_upiu(lrbp);
664 case UTP_CMD_TYPE_UFS:
665 /* For UFS native command implementation */
667 dev_err(hba->dev, "%s: UFS native command are not supported\n",
672 dev_err(hba->dev, "%s: unknown command type: 0x%x\n",
673 __func__, lrbp->command_type);
675 } /* end of switch */
681 * ufshcd_queuecommand - main entry point for SCSI requests
682 * @cmd: command from SCSI Midlayer
683 * @done: call back function
685 * Returns 0 for success, non-zero in case of failure
687 static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
689 struct ufshcd_lrb *lrbp;
695 hba = shost_priv(host);
697 tag = cmd->request->tag;
699 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
700 err = SCSI_MLQUEUE_HOST_BUSY;
704 /* acquire the tag to make sure device cmds don't use it */
705 if (test_and_set_bit_lock(tag, &hba->lrb_in_use)) {
707 * Dev manage command in progress, requeue the command.
708 * Requeuing the command helps in cases where the request *may*
709 * find different tag instead of waiting for dev manage command
712 err = SCSI_MLQUEUE_HOST_BUSY;
716 lrbp = &hba->lrb[tag];
720 lrbp->sense_bufflen = SCSI_SENSE_BUFFERSIZE;
721 lrbp->sense_buffer = cmd->sense_buffer;
722 lrbp->task_tag = tag;
723 lrbp->lun = cmd->device->lun;
724 lrbp->intr_cmd = false;
725 lrbp->command_type = UTP_CMD_TYPE_SCSI;
727 /* form UPIU before issuing the command */
728 ufshcd_compose_upiu(hba, lrbp);
729 err = ufshcd_map_sg(lrbp);
732 clear_bit_unlock(tag, &hba->lrb_in_use);
736 /* issue command to the controller */
737 spin_lock_irqsave(hba->host->host_lock, flags);
738 ufshcd_send_command(hba, tag);
739 spin_unlock_irqrestore(hba->host->host_lock, flags);
744 static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
745 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
748 lrbp->sense_bufflen = 0;
749 lrbp->sense_buffer = NULL;
750 lrbp->task_tag = tag;
751 lrbp->lun = 0; /* device management cmd is not specific to any LUN */
752 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
753 lrbp->intr_cmd = true; /* No interrupt aggregation */
754 hba->dev_cmd.type = cmd_type;
756 return ufshcd_compose_upiu(hba, lrbp);
760 ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
766 /* clear outstanding transaction before retry */
767 spin_lock_irqsave(hba->host->host_lock, flags);
768 ufshcd_utrl_clear(hba, tag);
769 spin_unlock_irqrestore(hba->host->host_lock, flags);
772 * wait for for h/w to clear corresponding bit in door-bell.
773 * max. wait is 1 sec.
775 err = ufshcd_wait_for_register(hba,
776 REG_UTP_TRANSFER_REQ_DOOR_BELL,
777 mask, ~mask, 1000, 1000);
783 * ufshcd_dev_cmd_completion() - handles device management command responses
784 * @hba: per adapter instance
785 * @lrbp: pointer to local reference block
788 ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
793 resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
796 case UPIU_TRANSACTION_NOP_IN:
797 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
799 dev_err(hba->dev, "%s: unexpected response %x\n",
803 case UPIU_TRANSACTION_REJECT_UPIU:
804 /* TODO: handle Reject UPIU Response */
806 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
811 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
819 static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
820 struct ufshcd_lrb *lrbp, int max_timeout)
823 unsigned long time_left;
826 time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
827 msecs_to_jiffies(max_timeout));
829 spin_lock_irqsave(hba->host->host_lock, flags);
830 hba->dev_cmd.complete = NULL;
831 if (likely(time_left)) {
832 err = ufshcd_get_tr_ocs(lrbp);
834 err = ufshcd_dev_cmd_completion(hba, lrbp);
836 spin_unlock_irqrestore(hba->host->host_lock, flags);
840 if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
841 /* sucessfully cleared the command, retry if needed */
849 * ufshcd_get_dev_cmd_tag - Get device management command tag
850 * @hba: per-adapter instance
851 * @tag: pointer to variable with available slot value
853 * Get a free slot and lock it until device management command
856 * Returns false if free slot is unavailable for locking, else
857 * return true with tag value in @tag.
859 static bool ufshcd_get_dev_cmd_tag(struct ufs_hba *hba, int *tag_out)
869 tmp = ~hba->lrb_in_use;
870 tag = find_last_bit(&tmp, hba->nutrs);
871 if (tag >= hba->nutrs)
873 } while (test_and_set_bit_lock(tag, &hba->lrb_in_use));
881 static inline void ufshcd_put_dev_cmd_tag(struct ufs_hba *hba, int tag)
883 clear_bit_unlock(tag, &hba->lrb_in_use);
887 * ufshcd_exec_dev_cmd - API for sending device management requests
889 * @cmd_type - specifies the type (NOP, Query...)
890 * @timeout - time in seconds
892 * NOTE: There is only one available tag for device management commands. Thus
893 * synchronisation is the responsibilty of the user.
895 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
896 enum dev_cmd_type cmd_type, int timeout)
898 struct ufshcd_lrb *lrbp;
901 struct completion wait;
905 * Get free slot, sleep if slots are unavailable.
906 * Even though we use wait_event() which sleeps indefinitely,
907 * the maximum wait time is bounded by SCSI request timeout.
909 wait_event(hba->dev_cmd.tag_wq, ufshcd_get_dev_cmd_tag(hba, &tag));
911 init_completion(&wait);
912 lrbp = &hba->lrb[tag];
914 err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
918 hba->dev_cmd.complete = &wait;
920 spin_lock_irqsave(hba->host->host_lock, flags);
921 ufshcd_send_command(hba, tag);
922 spin_unlock_irqrestore(hba->host->host_lock, flags);
924 err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
927 ufshcd_put_dev_cmd_tag(hba, tag);
928 wake_up(&hba->dev_cmd.tag_wq);
933 * ufshcd_memory_alloc - allocate memory for host memory space data structures
934 * @hba: per adapter instance
936 * 1. Allocate DMA memory for Command Descriptor array
937 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT
938 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
939 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
941 * 4. Allocate memory for local reference block(lrb).
943 * Returns 0 for success, non-zero in case of failure
945 static int ufshcd_memory_alloc(struct ufs_hba *hba)
947 size_t utmrdl_size, utrdl_size, ucdl_size;
949 /* Allocate memory for UTP command descriptors */
950 ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
951 hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
957 * UFSHCI requires UTP command descriptor to be 128 byte aligned.
958 * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
959 * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
960 * be aligned to 128 bytes as well
962 if (!hba->ucdl_base_addr ||
963 WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
965 "Command Descriptor Memory allocation failed\n");
970 * Allocate memory for UTP Transfer descriptors
971 * UFSHCI requires 1024 byte alignment of UTRD
973 utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
974 hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
976 &hba->utrdl_dma_addr,
978 if (!hba->utrdl_base_addr ||
979 WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
981 "Transfer Descriptor Memory allocation failed\n");
986 * Allocate memory for UTP Task Management descriptors
987 * UFSHCI requires 1024 byte alignment of UTMRD
989 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
990 hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
992 &hba->utmrdl_dma_addr,
994 if (!hba->utmrdl_base_addr ||
995 WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
997 "Task Management Descriptor Memory allocation failed\n");
1001 /* Allocate memory for local reference block */
1002 hba->lrb = devm_kzalloc(hba->dev,
1003 hba->nutrs * sizeof(struct ufshcd_lrb),
1006 dev_err(hba->dev, "LRB Memory allocation failed\n");
1015 * ufshcd_host_memory_configure - configure local reference block with
1017 * @hba: per adapter instance
1019 * Configure Host memory space
1020 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
1022 * 2. Update each UTRD with Response UPIU offset, Response UPIU length
1024 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
1025 * into local reference block.
1027 static void ufshcd_host_memory_configure(struct ufs_hba *hba)
1029 struct utp_transfer_cmd_desc *cmd_descp;
1030 struct utp_transfer_req_desc *utrdlp;
1031 dma_addr_t cmd_desc_dma_addr;
1032 dma_addr_t cmd_desc_element_addr;
1033 u16 response_offset;
1038 utrdlp = hba->utrdl_base_addr;
1039 cmd_descp = hba->ucdl_base_addr;
1042 offsetof(struct utp_transfer_cmd_desc, response_upiu);
1044 offsetof(struct utp_transfer_cmd_desc, prd_table);
1046 cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
1047 cmd_desc_dma_addr = hba->ucdl_dma_addr;
1049 for (i = 0; i < hba->nutrs; i++) {
1050 /* Configure UTRD with command descriptor base address */
1051 cmd_desc_element_addr =
1052 (cmd_desc_dma_addr + (cmd_desc_size * i));
1053 utrdlp[i].command_desc_base_addr_lo =
1054 cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
1055 utrdlp[i].command_desc_base_addr_hi =
1056 cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
1058 /* Response upiu and prdt offset should be in double words */
1059 utrdlp[i].response_upiu_offset =
1060 cpu_to_le16((response_offset >> 2));
1061 utrdlp[i].prd_table_offset =
1062 cpu_to_le16((prdt_offset >> 2));
1063 utrdlp[i].response_upiu_length =
1064 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
1066 hba->lrb[i].utr_descriptor_ptr = (utrdlp + i);
1067 hba->lrb[i].ucd_req_ptr =
1068 (struct utp_upiu_req *)(cmd_descp + i);
1069 hba->lrb[i].ucd_rsp_ptr =
1070 (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
1071 hba->lrb[i].ucd_prdt_ptr =
1072 (struct ufshcd_sg_entry *)cmd_descp[i].prd_table;
1077 * ufshcd_dme_link_startup - Notify Unipro to perform link startup
1078 * @hba: per adapter instance
1080 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
1081 * in order to initialize the Unipro link startup procedure.
1082 * Once the Unipro links are up, the device connected to the controller
1085 * Returns 0 on success, non-zero value on failure
1087 static int ufshcd_dme_link_startup(struct ufs_hba *hba)
1089 struct uic_command uic_cmd = {0};
1092 uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
1094 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
1097 "dme-link-startup: error code %d\n", ret);
1102 * ufshcd_make_hba_operational - Make UFS controller operational
1103 * @hba: per adapter instance
1105 * To bring UFS host controller to operational state,
1106 * 1. Check if device is present
1107 * 2. Enable required interrupts
1108 * 3. Configure interrupt aggregation
1109 * 4. Program UTRL and UTMRL base addres
1110 * 5. Configure run-stop-registers
1112 * Returns 0 on success, non-zero value on failure
1114 static int ufshcd_make_hba_operational(struct ufs_hba *hba)
1119 /* check if device present */
1120 reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
1121 if (!ufshcd_is_device_present(reg)) {
1122 dev_err(hba->dev, "cc: Device not present\n");
1127 /* Enable required interrupts */
1128 ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
1130 /* Configure interrupt aggregation */
1131 ufshcd_config_int_aggr(hba, INT_AGGR_CONFIG);
1133 /* Configure UTRL and UTMRL base address registers */
1134 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
1135 REG_UTP_TRANSFER_REQ_LIST_BASE_L);
1136 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
1137 REG_UTP_TRANSFER_REQ_LIST_BASE_H);
1138 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
1139 REG_UTP_TASK_REQ_LIST_BASE_L);
1140 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
1141 REG_UTP_TASK_REQ_LIST_BASE_H);
1144 * UCRDY, UTMRLDY and UTRLRDY bits must be 1
1145 * DEI, HEI bits must be 0
1147 if (!(ufshcd_get_lists_status(reg))) {
1148 ufshcd_enable_run_stop_reg(hba);
1151 "Host controller not ready to process requests");
1156 if (hba->ufshcd_state == UFSHCD_STATE_RESET)
1157 scsi_unblock_requests(hba->host);
1159 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
1166 * ufshcd_hba_enable - initialize the controller
1167 * @hba: per adapter instance
1169 * The controller resets itself and controller firmware initialization
1170 * sequence kicks off. When controller is ready it will set
1171 * the Host Controller Enable bit to 1.
1173 * Returns 0 on success, non-zero value on failure
1175 static int ufshcd_hba_enable(struct ufs_hba *hba)
1180 * msleep of 1 and 5 used in this function might result in msleep(20),
1181 * but it was necessary to send the UFS FPGA to reset mode during
1182 * development and testing of this driver. msleep can be changed to
1183 * mdelay and retry count can be reduced based on the controller.
1185 if (!ufshcd_is_hba_active(hba)) {
1187 /* change controller state to "reset state" */
1188 ufshcd_hba_stop(hba);
1191 * This delay is based on the testing done with UFS host
1192 * controller FPGA. The delay can be changed based on the
1193 * host controller used.
1198 /* start controller initialization sequence */
1199 ufshcd_hba_start(hba);
1202 * To initialize a UFS host controller HCE bit must be set to 1.
1203 * During initialization the HCE bit value changes from 1->0->1.
1204 * When the host controller completes initialization sequence
1205 * it sets the value of HCE bit to 1. The same HCE bit is read back
1206 * to check if the controller has completed initialization sequence.
1207 * So without this delay the value HCE = 1, set in the previous
1208 * instruction might be read back.
1209 * This delay can be changed based on the controller.
1213 /* wait for the host controller to complete initialization */
1215 while (ufshcd_is_hba_active(hba)) {
1220 "Controller enable failed\n");
1229 * ufshcd_link_startup - Initialize unipro link startup
1230 * @hba: per adapter instance
1232 * Returns 0 for success, non-zero in case of failure
1234 static int ufshcd_link_startup(struct ufs_hba *hba)
1238 /* enable UIC related interrupts */
1239 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
1241 ret = ufshcd_dme_link_startup(hba);
1245 ret = ufshcd_make_hba_operational(hba);
1249 dev_err(hba->dev, "link startup failed %d\n", ret);
1254 * ufshcd_verify_dev_init() - Verify device initialization
1255 * @hba: per-adapter instance
1257 * Send NOP OUT UPIU and wait for NOP IN response to check whether the
1258 * device Transport Protocol (UTP) layer is ready after a reset.
1259 * If the UTP layer at the device side is not initialized, it may
1260 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
1261 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
1263 static int ufshcd_verify_dev_init(struct ufs_hba *hba)
1268 mutex_lock(&hba->dev_cmd.lock);
1269 for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
1270 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
1273 if (!err || err == -ETIMEDOUT)
1276 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
1278 mutex_unlock(&hba->dev_cmd.lock);
1281 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
1286 * ufshcd_do_reset - reset the host controller
1287 * @hba: per adapter instance
1289 * Returns SUCCESS/FAILED
1291 static int ufshcd_do_reset(struct ufs_hba *hba)
1293 struct ufshcd_lrb *lrbp;
1294 unsigned long flags;
1297 /* block commands from midlayer */
1298 scsi_block_requests(hba->host);
1300 spin_lock_irqsave(hba->host->host_lock, flags);
1301 hba->ufshcd_state = UFSHCD_STATE_RESET;
1303 /* send controller to reset state */
1304 ufshcd_hba_stop(hba);
1305 spin_unlock_irqrestore(hba->host->host_lock, flags);
1307 /* abort outstanding commands */
1308 for (tag = 0; tag < hba->nutrs; tag++) {
1309 if (test_bit(tag, &hba->outstanding_reqs)) {
1310 lrbp = &hba->lrb[tag];
1312 scsi_dma_unmap(lrbp->cmd);
1313 lrbp->cmd->result = DID_RESET << 16;
1314 lrbp->cmd->scsi_done(lrbp->cmd);
1316 clear_bit_unlock(tag, &hba->lrb_in_use);
1321 /* complete device management command */
1322 if (hba->dev_cmd.complete)
1323 complete(hba->dev_cmd.complete);
1325 /* clear outstanding request/task bit maps */
1326 hba->outstanding_reqs = 0;
1327 hba->outstanding_tasks = 0;
1329 /* Host controller enable */
1330 if (ufshcd_hba_enable(hba)) {
1332 "Reset: Controller initialization failed\n");
1336 if (ufshcd_link_startup(hba)) {
1338 "Reset: Link start-up failed\n");
1346 * ufshcd_slave_alloc - handle initial SCSI device configurations
1347 * @sdev: pointer to SCSI device
1351 static int ufshcd_slave_alloc(struct scsi_device *sdev)
1353 struct ufs_hba *hba;
1355 hba = shost_priv(sdev->host);
1356 sdev->tagged_supported = 1;
1358 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
1359 sdev->use_10_for_ms = 1;
1360 scsi_set_tag_type(sdev, MSG_SIMPLE_TAG);
1363 * Inform SCSI Midlayer that the LUN queue depth is same as the
1364 * controller queue depth. If a LUN queue depth is less than the
1365 * controller queue depth and if the LUN reports
1366 * SAM_STAT_TASK_SET_FULL, the LUN queue depth will be adjusted
1367 * with scsi_adjust_queue_depth.
1369 scsi_activate_tcq(sdev, hba->nutrs);
1374 * ufshcd_slave_destroy - remove SCSI device configurations
1375 * @sdev: pointer to SCSI device
1377 static void ufshcd_slave_destroy(struct scsi_device *sdev)
1379 struct ufs_hba *hba;
1381 hba = shost_priv(sdev->host);
1382 scsi_deactivate_tcq(sdev, hba->nutrs);
1386 * ufshcd_task_req_compl - handle task management request completion
1387 * @hba: per adapter instance
1388 * @index: index of the completed request
1390 * Returns SUCCESS/FAILED
1392 static int ufshcd_task_req_compl(struct ufs_hba *hba, u32 index)
1394 struct utp_task_req_desc *task_req_descp;
1395 struct utp_upiu_task_rsp *task_rsp_upiup;
1396 unsigned long flags;
1400 spin_lock_irqsave(hba->host->host_lock, flags);
1402 /* Clear completed tasks from outstanding_tasks */
1403 __clear_bit(index, &hba->outstanding_tasks);
1405 task_req_descp = hba->utmrdl_base_addr;
1406 ocs_value = ufshcd_get_tmr_ocs(&task_req_descp[index]);
1408 if (ocs_value == OCS_SUCCESS) {
1409 task_rsp_upiup = (struct utp_upiu_task_rsp *)
1410 task_req_descp[index].task_rsp_upiu;
1411 task_result = be32_to_cpu(task_rsp_upiup->header.dword_1);
1412 task_result = ((task_result & MASK_TASK_RESPONSE) >> 8);
1414 if (task_result != UPIU_TASK_MANAGEMENT_FUNC_COMPL &&
1415 task_result != UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED)
1416 task_result = FAILED;
1418 task_result = SUCCESS;
1420 task_result = FAILED;
1422 "trc: Invalid ocs = %x\n", ocs_value);
1424 spin_unlock_irqrestore(hba->host->host_lock, flags);
1429 * ufshcd_adjust_lun_qdepth - Update LUN queue depth if device responds with
1430 * SAM_STAT_TASK_SET_FULL SCSI command status.
1431 * @cmd: pointer to SCSI command
1433 static void ufshcd_adjust_lun_qdepth(struct scsi_cmnd *cmd)
1435 struct ufs_hba *hba;
1439 hba = shost_priv(cmd->device->host);
1442 * LUN queue depth can be obtained by counting outstanding commands
1445 for (i = 0; i < hba->nutrs; i++) {
1446 if (test_bit(i, &hba->outstanding_reqs)) {
1449 * Check if the outstanding command belongs
1450 * to the LUN which reported SAM_STAT_TASK_SET_FULL.
1452 if (cmd->device->lun == hba->lrb[i].lun)
1458 * LUN queue depth will be total outstanding commands, except the
1459 * command for which the LUN reported SAM_STAT_TASK_SET_FULL.
1461 scsi_adjust_queue_depth(cmd->device, MSG_SIMPLE_TAG, lun_qdepth - 1);
1465 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
1466 * @lrb: pointer to local reference block of completed command
1467 * @scsi_status: SCSI command status
1469 * Returns value base on SCSI command status
1472 ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
1476 switch (scsi_status) {
1478 result |= DID_OK << 16 |
1479 COMMAND_COMPLETE << 8 |
1482 case SAM_STAT_CHECK_CONDITION:
1483 result |= DID_OK << 16 |
1484 COMMAND_COMPLETE << 8 |
1485 SAM_STAT_CHECK_CONDITION;
1486 ufshcd_copy_sense_data(lrbp);
1489 result |= SAM_STAT_BUSY;
1491 case SAM_STAT_TASK_SET_FULL:
1494 * If a LUN reports SAM_STAT_TASK_SET_FULL, then the LUN queue
1495 * depth needs to be adjusted to the exact number of
1496 * outstanding commands the LUN can handle at any given time.
1498 ufshcd_adjust_lun_qdepth(lrbp->cmd);
1499 result |= SAM_STAT_TASK_SET_FULL;
1501 case SAM_STAT_TASK_ABORTED:
1502 result |= SAM_STAT_TASK_ABORTED;
1505 result |= DID_ERROR << 16;
1507 } /* end of switch */
1513 * ufshcd_transfer_rsp_status - Get overall status of the response
1514 * @hba: per adapter instance
1515 * @lrb: pointer to local reference block of completed command
1517 * Returns result of the command to notify SCSI midlayer
1520 ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
1526 /* overall command status of utrd */
1527 ocs = ufshcd_get_tr_ocs(lrbp);
1531 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
1534 case UPIU_TRANSACTION_RESPONSE:
1536 * get the response UPIU result to extract
1537 * the SCSI command status
1539 result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
1542 * get the result based on SCSI status response
1543 * to notify the SCSI midlayer of the command status
1545 scsi_status = result & MASK_SCSI_STATUS;
1546 result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
1548 case UPIU_TRANSACTION_REJECT_UPIU:
1549 /* TODO: handle Reject UPIU Response */
1550 result = DID_ERROR << 16;
1552 "Reject UPIU not fully implemented\n");
1555 result = DID_ERROR << 16;
1557 "Unexpected request response code = %x\n",
1563 result |= DID_ABORT << 16;
1565 case OCS_INVALID_CMD_TABLE_ATTR:
1566 case OCS_INVALID_PRDT_ATTR:
1567 case OCS_MISMATCH_DATA_BUF_SIZE:
1568 case OCS_MISMATCH_RESP_UPIU_SIZE:
1569 case OCS_PEER_COMM_FAILURE:
1570 case OCS_FATAL_ERROR:
1572 result |= DID_ERROR << 16;
1574 "OCS error from controller = %x\n", ocs);
1576 } /* end of switch */
1582 * ufshcd_uic_cmd_compl - handle completion of uic command
1583 * @hba: per adapter instance
1585 static void ufshcd_uic_cmd_compl(struct ufs_hba *hba)
1587 if (hba->active_uic_cmd) {
1588 hba->active_uic_cmd->argument2 |=
1589 ufshcd_get_uic_cmd_result(hba);
1590 complete(&hba->active_uic_cmd->done);
1595 * ufshcd_transfer_req_compl - handle SCSI and query command completion
1596 * @hba: per adapter instance
1598 static void ufshcd_transfer_req_compl(struct ufs_hba *hba)
1600 struct ufshcd_lrb *lrbp;
1601 struct scsi_cmnd *cmd;
1602 unsigned long completed_reqs;
1606 bool int_aggr_reset = false;
1608 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
1609 completed_reqs = tr_doorbell ^ hba->outstanding_reqs;
1611 for (index = 0; index < hba->nutrs; index++) {
1612 if (test_bit(index, &completed_reqs)) {
1613 lrbp = &hba->lrb[index];
1616 * Don't skip resetting interrupt aggregation counters
1617 * if a regular command is present.
1619 int_aggr_reset |= !lrbp->intr_cmd;
1622 result = ufshcd_transfer_rsp_status(hba, lrbp);
1623 scsi_dma_unmap(cmd);
1624 cmd->result = result;
1625 /* Mark completed command as NULL in LRB */
1627 clear_bit_unlock(index, &hba->lrb_in_use);
1628 /* Do not touch lrbp after scsi done */
1629 cmd->scsi_done(cmd);
1630 } else if (lrbp->command_type ==
1631 UTP_CMD_TYPE_DEV_MANAGE) {
1632 if (hba->dev_cmd.complete)
1633 complete(hba->dev_cmd.complete);
1638 /* clear corresponding bits of completed commands */
1639 hba->outstanding_reqs ^= completed_reqs;
1641 /* we might have free'd some tags above */
1642 wake_up(&hba->dev_cmd.tag_wq);
1644 /* Reset interrupt aggregation counters */
1646 ufshcd_config_int_aggr(hba, INT_AGGR_RESET);
1650 * ufshcd_fatal_err_handler - handle fatal errors
1651 * @hba: per adapter instance
1653 static void ufshcd_fatal_err_handler(struct work_struct *work)
1655 struct ufs_hba *hba;
1656 hba = container_of(work, struct ufs_hba, feh_workq);
1658 /* check if reset is already in progress */
1659 if (hba->ufshcd_state != UFSHCD_STATE_RESET)
1660 ufshcd_do_reset(hba);
1664 * ufshcd_err_handler - Check for fatal errors
1665 * @work: pointer to a work queue structure
1667 static void ufshcd_err_handler(struct ufs_hba *hba)
1671 if (hba->errors & INT_FATAL_ERRORS)
1674 if (hba->errors & UIC_ERROR) {
1675 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
1676 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
1681 hba->ufshcd_state = UFSHCD_STATE_ERROR;
1682 schedule_work(&hba->feh_workq);
1686 * ufshcd_tmc_handler - handle task management function completion
1687 * @hba: per adapter instance
1689 static void ufshcd_tmc_handler(struct ufs_hba *hba)
1693 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
1694 hba->tm_condition = tm_doorbell ^ hba->outstanding_tasks;
1695 wake_up_interruptible(&hba->ufshcd_tm_wait_queue);
1699 * ufshcd_sl_intr - Interrupt service routine
1700 * @hba: per adapter instance
1701 * @intr_status: contains interrupts generated by the controller
1703 static void ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
1705 hba->errors = UFSHCD_ERROR_MASK & intr_status;
1707 ufshcd_err_handler(hba);
1709 if (intr_status & UIC_COMMAND_COMPL)
1710 ufshcd_uic_cmd_compl(hba);
1712 if (intr_status & UTP_TASK_REQ_COMPL)
1713 ufshcd_tmc_handler(hba);
1715 if (intr_status & UTP_TRANSFER_REQ_COMPL)
1716 ufshcd_transfer_req_compl(hba);
1720 * ufshcd_intr - Main interrupt service routine
1722 * @__hba: pointer to adapter instance
1724 * Returns IRQ_HANDLED - If interrupt is valid
1725 * IRQ_NONE - If invalid interrupt
1727 static irqreturn_t ufshcd_intr(int irq, void *__hba)
1730 irqreturn_t retval = IRQ_NONE;
1731 struct ufs_hba *hba = __hba;
1733 spin_lock(hba->host->host_lock);
1734 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
1737 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
1738 ufshcd_sl_intr(hba, intr_status);
1739 retval = IRQ_HANDLED;
1741 spin_unlock(hba->host->host_lock);
1746 * ufshcd_issue_tm_cmd - issues task management commands to controller
1747 * @hba: per adapter instance
1748 * @lrbp: pointer to local reference block
1750 * Returns SUCCESS/FAILED
1753 ufshcd_issue_tm_cmd(struct ufs_hba *hba,
1754 struct ufshcd_lrb *lrbp,
1757 struct utp_task_req_desc *task_req_descp;
1758 struct utp_upiu_task_req *task_req_upiup;
1759 struct Scsi_Host *host;
1760 unsigned long flags;
1766 spin_lock_irqsave(host->host_lock, flags);
1768 /* If task management queue is full */
1769 free_slot = ufshcd_get_tm_free_slot(hba);
1770 if (free_slot >= hba->nutmrs) {
1771 spin_unlock_irqrestore(host->host_lock, flags);
1772 dev_err(hba->dev, "Task management queue full\n");
1777 task_req_descp = hba->utmrdl_base_addr;
1778 task_req_descp += free_slot;
1780 /* Configure task request descriptor */
1781 task_req_descp->header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
1782 task_req_descp->header.dword_2 =
1783 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
1785 /* Configure task request UPIU */
1787 (struct utp_upiu_task_req *) task_req_descp->task_req_upiu;
1788 task_req_upiup->header.dword_0 =
1789 UPIU_HEADER_DWORD(UPIU_TRANSACTION_TASK_REQ, 0,
1790 lrbp->lun, lrbp->task_tag);
1791 task_req_upiup->header.dword_1 =
1792 UPIU_HEADER_DWORD(0, tm_function, 0, 0);
1794 task_req_upiup->input_param1 = lrbp->lun;
1795 task_req_upiup->input_param1 =
1796 cpu_to_be32(task_req_upiup->input_param1);
1797 task_req_upiup->input_param2 = lrbp->task_tag;
1798 task_req_upiup->input_param2 =
1799 cpu_to_be32(task_req_upiup->input_param2);
1801 /* send command to the controller */
1802 __set_bit(free_slot, &hba->outstanding_tasks);
1803 ufshcd_writel(hba, 1 << free_slot, REG_UTP_TASK_REQ_DOOR_BELL);
1805 spin_unlock_irqrestore(host->host_lock, flags);
1807 /* wait until the task management command is completed */
1809 wait_event_interruptible_timeout(hba->ufshcd_tm_wait_queue,
1810 (test_bit(free_slot,
1811 &hba->tm_condition) != 0),
1815 "Task management command timed-out\n");
1819 clear_bit(free_slot, &hba->tm_condition);
1820 err = ufshcd_task_req_compl(hba, free_slot);
1826 * ufshcd_device_reset - reset device and abort all the pending commands
1827 * @cmd: SCSI command pointer
1829 * Returns SUCCESS/FAILED
1831 static int ufshcd_device_reset(struct scsi_cmnd *cmd)
1833 struct Scsi_Host *host;
1834 struct ufs_hba *hba;
1839 host = cmd->device->host;
1840 hba = shost_priv(host);
1841 tag = cmd->request->tag;
1843 err = ufshcd_issue_tm_cmd(hba, &hba->lrb[tag], UFS_LOGICAL_RESET);
1847 for (pos = 0; pos < hba->nutrs; pos++) {
1848 if (test_bit(pos, &hba->outstanding_reqs) &&
1849 (hba->lrb[tag].lun == hba->lrb[pos].lun)) {
1851 /* clear the respective UTRLCLR register bit */
1852 ufshcd_utrl_clear(hba, pos);
1854 clear_bit(pos, &hba->outstanding_reqs);
1856 if (hba->lrb[pos].cmd) {
1857 scsi_dma_unmap(hba->lrb[pos].cmd);
1858 hba->lrb[pos].cmd->result =
1860 hba->lrb[pos].cmd->scsi_done(cmd);
1861 hba->lrb[pos].cmd = NULL;
1862 clear_bit_unlock(pos, &hba->lrb_in_use);
1863 wake_up(&hba->dev_cmd.tag_wq);
1872 * ufshcd_host_reset - Main reset function registered with scsi layer
1873 * @cmd: SCSI command pointer
1875 * Returns SUCCESS/FAILED
1877 static int ufshcd_host_reset(struct scsi_cmnd *cmd)
1879 struct ufs_hba *hba;
1881 hba = shost_priv(cmd->device->host);
1883 if (hba->ufshcd_state == UFSHCD_STATE_RESET)
1886 return ufshcd_do_reset(hba);
1890 * ufshcd_abort - abort a specific command
1891 * @cmd: SCSI command pointer
1893 * Returns SUCCESS/FAILED
1895 static int ufshcd_abort(struct scsi_cmnd *cmd)
1897 struct Scsi_Host *host;
1898 struct ufs_hba *hba;
1899 unsigned long flags;
1903 host = cmd->device->host;
1904 hba = shost_priv(host);
1905 tag = cmd->request->tag;
1907 spin_lock_irqsave(host->host_lock, flags);
1909 /* check if command is still pending */
1910 if (!(test_bit(tag, &hba->outstanding_reqs))) {
1912 spin_unlock_irqrestore(host->host_lock, flags);
1915 spin_unlock_irqrestore(host->host_lock, flags);
1917 err = ufshcd_issue_tm_cmd(hba, &hba->lrb[tag], UFS_ABORT_TASK);
1921 scsi_dma_unmap(cmd);
1923 spin_lock_irqsave(host->host_lock, flags);
1925 /* clear the respective UTRLCLR register bit */
1926 ufshcd_utrl_clear(hba, tag);
1928 __clear_bit(tag, &hba->outstanding_reqs);
1929 hba->lrb[tag].cmd = NULL;
1930 spin_unlock_irqrestore(host->host_lock, flags);
1932 clear_bit_unlock(tag, &hba->lrb_in_use);
1933 wake_up(&hba->dev_cmd.tag_wq);
1939 * ufshcd_async_scan - asynchronous execution for link startup
1940 * @data: data pointer to pass to this function
1941 * @cookie: cookie data
1943 static void ufshcd_async_scan(void *data, async_cookie_t cookie)
1945 struct ufs_hba *hba = (struct ufs_hba *)data;
1948 ret = ufshcd_link_startup(hba);
1952 ret = ufshcd_verify_dev_init(hba);
1956 scsi_scan_host(hba->host);
1961 static struct scsi_host_template ufshcd_driver_template = {
1962 .module = THIS_MODULE,
1964 .proc_name = UFSHCD,
1965 .queuecommand = ufshcd_queuecommand,
1966 .slave_alloc = ufshcd_slave_alloc,
1967 .slave_destroy = ufshcd_slave_destroy,
1968 .eh_abort_handler = ufshcd_abort,
1969 .eh_device_reset_handler = ufshcd_device_reset,
1970 .eh_host_reset_handler = ufshcd_host_reset,
1972 .sg_tablesize = SG_ALL,
1973 .cmd_per_lun = UFSHCD_CMD_PER_LUN,
1974 .can_queue = UFSHCD_CAN_QUEUE,
1978 * ufshcd_suspend - suspend power management function
1979 * @hba: per adapter instance
1980 * @state: power state
1984 int ufshcd_suspend(struct ufs_hba *hba, pm_message_t state)
1988 * 1. Block SCSI requests from SCSI midlayer
1989 * 2. Change the internal driver state to non operational
1990 * 3. Set UTRLRSR and UTMRLRSR bits to zero
1991 * 4. Wait until outstanding commands are completed
1992 * 5. Set HCE to zero to send the UFS host controller to reset state
1997 EXPORT_SYMBOL_GPL(ufshcd_suspend);
2000 * ufshcd_resume - resume power management function
2001 * @hba: per adapter instance
2005 int ufshcd_resume(struct ufs_hba *hba)
2009 * 1. Set HCE to 1, to start the UFS host controller
2010 * initialization process
2011 * 2. Set UTRLRSR and UTMRLRSR bits to 1
2012 * 3. Change the internal driver state to operational
2013 * 4. Unblock SCSI requests from SCSI midlayer
2018 EXPORT_SYMBOL_GPL(ufshcd_resume);
2021 * ufshcd_remove - de-allocate SCSI host and host memory space
2022 * data structure memory
2023 * @hba - per adapter instance
2025 void ufshcd_remove(struct ufs_hba *hba)
2027 /* disable interrupts */
2028 ufshcd_disable_intr(hba, hba->intr_mask);
2029 ufshcd_hba_stop(hba);
2031 scsi_remove_host(hba->host);
2032 scsi_host_put(hba->host);
2034 EXPORT_SYMBOL_GPL(ufshcd_remove);
2037 * ufshcd_init - Driver initialization routine
2038 * @dev: pointer to device handle
2039 * @hba_handle: driver private handle
2040 * @mmio_base: base register address
2041 * @irq: Interrupt line of device
2042 * Returns 0 on success, non-zero value on failure
2044 int ufshcd_init(struct device *dev, struct ufs_hba **hba_handle,
2045 void __iomem *mmio_base, unsigned int irq)
2047 struct Scsi_Host *host;
2048 struct ufs_hba *hba;
2053 "Invalid memory reference for dev is NULL\n");
2060 "Invalid memory reference for mmio_base is NULL\n");
2065 host = scsi_host_alloc(&ufshcd_driver_template,
2066 sizeof(struct ufs_hba));
2068 dev_err(dev, "scsi_host_alloc failed\n");
2072 hba = shost_priv(host);
2075 hba->mmio_base = mmio_base;
2078 /* Read capabilities registers */
2079 ufshcd_hba_capabilities(hba);
2081 /* Get UFS version supported by the controller */
2082 hba->ufs_version = ufshcd_get_ufs_version(hba);
2084 /* Get Interrupt bit mask per version */
2085 hba->intr_mask = ufshcd_get_intr_mask(hba);
2087 /* Allocate memory for host memory space */
2088 err = ufshcd_memory_alloc(hba);
2090 dev_err(hba->dev, "Memory allocation failed\n");
2095 ufshcd_host_memory_configure(hba);
2097 host->can_queue = hba->nutrs;
2098 host->cmd_per_lun = hba->nutrs;
2099 host->max_id = UFSHCD_MAX_ID;
2100 host->max_lun = UFSHCD_MAX_LUNS;
2101 host->max_channel = UFSHCD_MAX_CHANNEL;
2102 host->unique_id = host->host_no;
2103 host->max_cmd_len = MAX_CDB_SIZE;
2105 /* Initailize wait queue for task management */
2106 init_waitqueue_head(&hba->ufshcd_tm_wait_queue);
2108 /* Initialize work queues */
2109 INIT_WORK(&hba->feh_workq, ufshcd_fatal_err_handler);
2111 /* Initialize UIC command mutex */
2112 mutex_init(&hba->uic_cmd_mutex);
2114 /* Initialize mutex for device management commands */
2115 mutex_init(&hba->dev_cmd.lock);
2117 /* Initialize device management tag acquire wait queue */
2118 init_waitqueue_head(&hba->dev_cmd.tag_wq);
2120 /* IRQ registration */
2121 err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
2123 dev_err(hba->dev, "request irq failed\n");
2127 /* Enable SCSI tag mapping */
2128 err = scsi_init_shared_tag_map(host, host->can_queue);
2130 dev_err(hba->dev, "init shared queue failed\n");
2134 err = scsi_add_host(host, hba->dev);
2136 dev_err(hba->dev, "scsi_add_host failed\n");
2140 /* Host controller enable */
2141 err = ufshcd_hba_enable(hba);
2143 dev_err(hba->dev, "Host controller enable failed\n");
2144 goto out_remove_scsi_host;
2149 async_schedule(ufshcd_async_scan, hba);
2153 out_remove_scsi_host:
2154 scsi_remove_host(hba->host);
2156 scsi_host_put(host);
2160 EXPORT_SYMBOL_GPL(ufshcd_init);
2162 MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
2163 MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
2164 MODULE_DESCRIPTION("Generic UFS host controller driver Core");
2165 MODULE_LICENSE("GPL");
2166 MODULE_VERSION(UFSHCD_DRIVER_VERSION);