/* * Copyright (C) 2018 MediaTek Inc. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. * See http://www.gnu.org/licenses/gpl-2.0.html for more details. */ #include "ufs.h" #include #include #include #include #include #include #include #include #include #include #include "ufshcd.h" #include "ufshcd-pltfrm.h" #include "unipro.h" #include "ufs_quirks.h" #include "ufs-mtk.h" #include "ufs-mtk-block.h" #include "ufs-mtk-platform.h" #include "ufs-mtk-dbg.h" #ifdef CONFIG_MTK_AEE_FEATURE #include #endif #include #include #include #include #include #ifdef CONFIG_MTK_UFS_LBA_CRC16_CHECK #include #endif #if defined(CONFIG_UFSHPB) #include "ufshpb.h" #endif #include "mtk_spm_resource_req.h" #define CREATE_TRACE_POINTS #include "ufs-mtk-trace.h" /* Query request retries */ #define QUERY_REQ_RETRIES 10 #define MAX_WRITE_BUFFER_SIZE (512 * 1024) /* refer to ufs_mtk_init() for default value of these globals */ int ufs_mtk_rpm_autosuspend_delay; /* runtime PM: auto suspend delay */ bool ufs_mtk_rpm_enabled; /* runtime PM: on/off */ bool ufs_mtk_auto_hibern8_enabled; bool ufs_mtk_host_deep_stall_enable; bool ufs_mtk_host_scramble_enable; int ufs_mtk_hs_gear; struct ufs_hba *ufs_mtk_hba; static bool ufs_mtk_is_data_write_cmd(struct scsi_cmnd *cmd, bool isolation); static bool ufs_mtk_is_data_cmd(struct scsi_cmnd *cmd, bool isolation); static bool ufs_mtk_is_unmap_cmd(struct scsi_cmnd *cmd); #if defined(PMIC_RG_LDO_VUFS_LP_ADDR) && defined(pmic_config_interface) #define ufs_mtk_vufs_lpm(on) \ pmic_config_interface(PMIC_RG_LDO_VUFS_LP_ADDR, \ (on), \ PMIC_RG_LDO_VUFS_LP_MASK, \ PMIC_RG_LDO_VUFS_LP_SHIFT) #else #define ufs_mtk_vufs_lpm(on) #endif #ifdef CONFIG_MTK_UFS_LBA_CRC16_CHECK /* * Sometimes a lba may use encrypted r/w sometime * and use raw r/w at other times. * Use two CRC16 Arrays to record Encrypted and NoEncrypted data */ /* Self init Encryption and No Encryption array */ static u8 di_init; /* Logical block count */ static u64 di_blkcnt; /* CRC value of each 4KB block */ static u16 *di_crc; /* private data */ static u8 *di_priv; /* only do crc to first 32 byte of a 4KB block to reduce cpu overhead */ #define DI_CRC_DATA_SIZE (32) /* Init Encryption and No Encryption array and others */ void ufs_mtk_di_init(struct ufs_hba *hba) { u8 ud_buf[UNIT_DESC_PARAM_ERASE_BLK_SIZE] = { 0 }; int len = UNIT_DESC_PARAM_ERASE_BLK_SIZE; /* Already init */ if (di_init != 0) return; /* Read LU2 unit descriptor */ ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC, QUERY_DESC_IDN_UNIT, 0x2, 0, ud_buf, &len); di_blkcnt = ( ((u64)ud_buf[UNIT_DESC_PARAM_LOGICAL_BLK_COUNT] << 56) | ((u64)ud_buf[UNIT_DESC_PARAM_LOGICAL_BLK_COUNT+1] << 48) | ((u64)ud_buf[UNIT_DESC_PARAM_LOGICAL_BLK_COUNT+2] << 40) | ((u64)ud_buf[UNIT_DESC_PARAM_LOGICAL_BLK_COUNT+3] << 32) | ((u64)ud_buf[UNIT_DESC_PARAM_LOGICAL_BLK_COUNT+4] << 24) | ((u64)ud_buf[UNIT_DESC_PARAM_LOGICAL_BLK_COUNT+5] << 16) | ((u64)ud_buf[UNIT_DESC_PARAM_LOGICAL_BLK_COUNT+6] << 8) | ((u64)ud_buf[UNIT_DESC_PARAM_LOGICAL_BLK_COUNT+7])); di_crc = vzalloc(di_blkcnt * sizeof(u16)); if (!di_crc) return; di_priv = vzalloc(di_blkcnt * sizeof(u8)); if (!di_priv) { vfree(di_crc); di_crc = NULL; return; } dev_info(hba->dev, "%s: need %llu MB memory for total lba %llu(0x%llx)\n", __func__, di_blkcnt * (sizeof(u16) + sizeof(u8)) / 1024 / 1024 , di_blkcnt, di_blkcnt); di_init = 1; } static void ufs_mtk_di_reset(struct ufs_hba *hba) { if (di_crc) memset(di_crc, 0, di_blkcnt * sizeof(u16)); if (di_priv) memset(di_priv, 0, di_blkcnt * sizeof(u8)); dev_info(hba->dev, "%s: Reset di %llu MB memory done!", __func__, di_blkcnt * (sizeof(u16) + sizeof(u8)) / 1024 / 1024); } /* Clear record for UNMAP command */ static int ufs_mtk_di_clr(struct scsi_cmnd *cmd) { u32 lba, lba_cnt, i; /* * Discard request has only one continuous sector range, * so we can assume UNMAP command has only one UNMAP block * descriptor. */ /* __sector: 512-byte based */ lba = cmd->request->__sector >> 3; /* __data_len: bytes */ lba_cnt = cmd->request->__data_len >> 12; /* clear both encrypted and non-encrypted records */ for (i = 0; i < lba_cnt; i++) { di_crc[lba + i] = di_priv[lba + i] = 0; } return 0; } static int ufs_mtk_di_cmp_read(struct ufs_hba *hba, u32 lba, int mode, u16 crc, u8 priv) { /* * For read, update CRC16 value * if corresponding CRC16[lba] is 0 (not set before), * Otherwise, compare current calculated crc16 * value to CRC16[lba]. */ if (mode == UFS_CRYPTO_HW_FDE) { if (di_crc[lba] == 0 || di_priv[lba] == 0) { di_crc[lba] = crc; di_priv[lba] = priv; } else { if (di_crc[lba] != crc) { dev_info(hba->dev, "%s: crc err! existed: 0x%x, read: 0x%x, lba: %u\n", __func__, di_crc[lba], crc, lba); WARN_ON(1); return -EIO; } } } else if (mode == UFS_CRYPTO_HW_FBE) { if (di_crc[lba] == 0 || di_priv[lba] == 0) { di_crc[lba] = crc; di_priv[lba] = priv; } else { if (priv != di_priv[lba]) { dev_info(hba->dev, "%s: key err (fde)! existed: 0x%x, read: 0x%x, lba: %u\n", __func__, di_priv[lba], priv, lba); return 0; } else if (crc != di_crc[lba]) { dev_info(hba->dev, "%s: crc err (fbe)! existed: 0x%x, read: 0x%x, lba: %u\n", __func__, di_crc[lba], crc, lba); return 0; } } } else { if (di_crc[lba] == 0 || di_priv[lba] != 0) { di_crc[lba] = crc; di_priv[lba] = priv; } else { if (di_crc[lba] != crc) { dev_info(hba->dev, "%s: crc err (ne)! existed: 0x%x, read: 0x%x, lba: %u\n", __func__, di_crc[lba], crc, lba); WARN_ON(1); return -EIO; } } } return 0; } static int ufs_mtk_di_cmp(struct ufs_hba *hba, struct scsi_cmnd *cmd) { char *buffer; int i, len, err = 0; struct scatterlist *sg; u32 lba, blk_cnt, end_lba; u16 crc = 0; int mode = 0, tag; u8 priv = 0; sg = scsi_sglist(cmd); tag = cmd->request->tag; lba = cmd->cmnd[5] | (cmd->cmnd[4] << 8) | (cmd->cmnd[3] << 16) | (cmd->cmnd[2] << 24); if ((u64)lba >= di_blkcnt) { dev_info(hba->dev, "%s: lba err! expected: di_blkcnt: 0x%llx, LBA: 0x%x\n", __func__, di_blkcnt, lba); WARN_ON(1); return -EIO; } /* HPB use READ_16, Transfer_len in cmd[15]*/ if (cmd->cmnd[0] == READ_16) { if ((hba->card->wmanufacturerid == UFS_VENDOR_SAMSUNG) || (hba->card->wmanufacturerid == UFS_VENDOR_MICRON)) blk_cnt = cmd->cmnd[15]; else blk_cnt = cmd->cmnd[14]; /* JEDEC version */ #if defined(CONFIG_SCSI_SKHPB) } else if (cmd->cmnd[0] == SKHPB_READ) { /* JEDEC version */ blk_cnt = cmd->cmnd[14]; #endif } else { blk_cnt = cmd->cmnd[8] | (cmd->cmnd[7] << 8); } end_lba = lba + blk_cnt; /* Only data commands in LU2 need to check crc */ if (ufshcd_scsi_to_upiu_lun(cmd->device->lun) != 0x2) return 0; if (!scsi_sg_count(cmd)) return 0; if (hba->lrb[tag].crypto_enable) { mode = UFS_CRYPTO_HW_FBE; /* * ufshcd_crypto_enable() will re-program all of the keys with * the same index in keyslot_manager_reprogram_all_keys() * so use key slot as key index for checking different keys */ priv = hba->lrb[tag].crypto_key_slot; priv++; } for (i = 0; i < scsi_sg_count(cmd); i++) { buffer = (char *)sg_virt(sg); for (len = 0; len < sg->length; len = len + 0x1000, lba++) { /* * Use value 0 as empty slot in crc array * * if calculated crc value is 0, use crc + 1 * instead to avoid conflict */ crc = crc16(0x0, &buffer[len], DI_CRC_DATA_SIZE); if (crc == 0) crc++; if (ufs_mtk_is_data_write_cmd(cmd, false)) { /* For write, update crc value */ di_crc[lba] = crc; di_priv[lba] = priv; } else { err = ufs_mtk_di_cmp_read(hba, lba, mode, crc, priv); if (err) return err; } } sg = sg_next(sg); } /* * Check lba # traverse from * scatter is the same as end_lba */ if (end_lba != lba) { dev_info(hba->dev, "expect end_lba is 0x%x, but 0x%x, cmd=0x%x, blk_cnt=0x%x\n", end_lba, lba, cmd->cmnd[0], blk_cnt); } return 0; } int ufs_mtk_di_inspect(struct ufs_hba *hba, struct scsi_cmnd *cmd) { if (!di_crc) return 0; /* do inspection in LU2 (user LU) only */ if (ufshcd_scsi_to_upiu_lun(cmd->device->lun) != 0x2) return -ENODEV; if (ufs_mtk_is_data_cmd(cmd, false)) return ufs_mtk_di_cmp(hba, cmd); if (ufs_mtk_is_unmap_cmd(cmd)) return ufs_mtk_di_clr(cmd); return -ENODEV; } #endif static int ufs_mtk_query_desc(struct ufs_hba *hba, enum query_opcode opcode, enum desc_idn idn, u8 index, void *desc, int len) { return ufshcd_query_descriptor_retry(hba, opcode, idn, index, 0, desc, &len); } static int ufs_mtk_send_uic_command(struct ufs_hba *hba, u32 cmd, u32 arg1, u32 arg2, u32 *arg3, u8 *err_code) { int result; struct uic_command uic_cmd = { .command = cmd, .argument1 = arg1, .argument2 = arg2, .argument3 = *arg3, }; result = ufshcd_send_uic_cmd(hba, &uic_cmd); if (err_code) *err_code = uic_cmd.argument2 & MASK_UIC_COMMAND_RESULT; if (result) { dev_err(hba->dev, "UIC command error: %#x\n", result); return -EIO; } if (cmd == UIC_CMD_DME_GET || cmd == UIC_CMD_DME_PEER_GET) *arg3 = uic_cmd.argument3; return 0; } int ufs_mtk_run_batch_uic_cmd(struct ufs_hba *hba, struct uic_command *cmds, int ncmds) { int i; int err = 0; for (i = 0; i < ncmds; i++) { err = ufshcd_send_uic_cmd(hba, &cmds[i]); if (err) { dev_err(hba->dev, "%s fail, cmd: %x, arg1: %x\n", __func__, cmds->command, cmds->argument1); /* return err; */ } } return err; } int ufs_mtk_cfg_unipro_cg(struct ufs_hba *hba, bool enable) { u32 tmp = 0; if (enable) { ufshcd_dme_get(hba, UIC_ARG_MIB(VENDOR_SAVEPOWERCONTROL), &tmp); tmp = tmp | (1 << RX_SYMBOL_CLK_GATE_EN) | (1 << SYS_CLK_GATE_EN) | (1 << TX_CLK_GATE_EN); ufshcd_dme_set(hba, UIC_ARG_MIB(VENDOR_SAVEPOWERCONTROL), tmp); ufshcd_dme_get(hba, UIC_ARG_MIB(VENDOR_DEBUGCLOCKENABLE), &tmp); tmp = tmp & ~(1 << TX_SYMBOL_CLK_REQ_FORCE); ufshcd_dme_set(hba, UIC_ARG_MIB(VENDOR_DEBUGCLOCKENABLE), tmp); } else { ufshcd_dme_get(hba, UIC_ARG_MIB(VENDOR_SAVEPOWERCONTROL), &tmp); tmp = tmp & ~((1 << RX_SYMBOL_CLK_GATE_EN) | (1 << SYS_CLK_GATE_EN) | (1 << TX_CLK_GATE_EN)); ufshcd_dme_set(hba, UIC_ARG_MIB(VENDOR_SAVEPOWERCONTROL), tmp); ufshcd_dme_get(hba, UIC_ARG_MIB(VENDOR_DEBUGCLOCKENABLE), &tmp); tmp = tmp | (1 << TX_SYMBOL_CLK_REQ_FORCE); ufshcd_dme_set(hba, UIC_ARG_MIB(VENDOR_DEBUGCLOCKENABLE), tmp); } return 0; } /** * ufs_mtk_advertise_quirks - advertise the known mtk UFS controller quirks * @hba: host controller instance * * mtk UFS host controller might have some non standard behaviours (quirks) * than what is specified by UFSHCI specification. Advertise all such * quirks to standard UFS host controller driver so standard takes them into * account. */ static void ufs_mtk_advertise_hci_quirks(struct ufs_hba *hba) { #if defined(UFS_MTK_PLATFORM_UFS_HCI_PERF_HEURISTIC) hba->quirks |= UFSHCD_QUIRK_UFS_HCI_PERF_HEURISTIC; #endif #if defined(UFS_MTK_PLATFORM_UFS_HCI_RST_DEV_FOR_LINKUP_FAIL) hba->quirks |= UFSHCD_QUIRK_UFS_HCI_DEV_RST_FOR_LINKUP_FAIL; #endif #if defined(UFS_MTK_PLATFORM_UFS_HCI_VENDOR_HOST_RST) hba->quirks |= UFSHCD_QUIRK_UFS_HCI_VENDOR_HOST_RST; #endif #if defined(UFS_MTK_PLATFORM_VCC_ALWAYS_ON) hba->quirks |= UFSHCD_QUIRK_UFS_VCC_ALWAYS_ON; #endif /* Always enable "Disable AH8 before RDB" */ hba->quirks |= UFSHCD_QUIRK_UFS_HCI_DISABLE_AH8_BEFORE_RDB; dev_info(hba->dev, "hci quirks: %#x\n", hba->quirks); } #ifdef CONFIG_MTK_HW_FDE /** * ufs_mtk_hwfde_cfg_cmd - configure command for hw fde * @hba: host controller instance * @cmd: scsi command instance * * HW FDE key may be changed by updating master key by end-user's behavior. * Update new key to crypto IP if necessary. * * Host lock must be held during key update in atf. */ void ufs_mtk_hwfde_cfg_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd) { u64 lba; u32 dunl, dunu; unsigned long flags; int hwfde_key_idx_old; struct ufshcd_lrb *lrbp; lrbp = &hba->lrb[cmd->request->tag]; /* for r/w request only */ if (cmd->request->bio && cmd->request->bio->bi_hw_fde) { /* in case hw fde is enabled and key index is updated by * dm-crypt */ if (cmd->request->bio->bi_key_idx != hba->crypto_hwfde_key_idx) { /* acquire host lock to ensure atomicity during key * change in atf */ spin_lock_irqsave(hba->host->host_lock, flags); /* do key change */ mt_secure_call(MTK_SIP_KERNEL_HW_FDE_UFS_CTL, (1 << 3), 0, 0, 0); hwfde_key_idx_old = hba->crypto_hwfde_key_idx; hba->crypto_hwfde_key_idx = cmd->request->bio->bi_key_idx; spin_unlock_irqrestore(hba->host->host_lock, flags); dev_info(hba->dev, "update hw-fde key, ver %d->%d\n", hwfde_key_idx_old, hba->crypto_hwfde_key_idx); } lba = blk_rq_pos(cmd->request) >> 3; ufs_mtk_crypto_cal_dun(UFS_CRYPTO_ALGO_ESSIV_AES_CBC, lba, &dunl, &dunu); lrbp->crypto_key_slot = 0; lrbp->crypto_enable = true; lrbp->data_unit_num = ((u64)dunu << 32 || dunl); /* mark data has ever gone through encryption/decryption path */ hba->crypto_feature |= UFS_CRYPTO_HW_FDE_ENCRYPTED; } } #else void ufs_mtk_hwfde_cfg_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd) { }; #endif static enum ufs_pm_level ufs_mtk_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state, enum uic_link_state link_state) { enum ufs_pm_level lvl; for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) { if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) && (ufs_pm_lvl_states[lvl].link_state == link_state)) return lvl; } /* if no match found, return the level 0 */ return UFS_PM_LVL_0; } static bool ufs_mtk_is_valid_pm_lvl(int lvl) { if (lvl >= 0 && lvl < ufs_pm_lvl_states_size) return true; else return false; } static int ufs_mtk_host_clk_get(struct device *dev, const char *name, struct clk **clk_out) { struct clk *clk; int err = 0; clk = devm_clk_get(dev, name); if (IS_ERR(clk)) err = PTR_ERR(clk); else *clk_out = clk; return err; } bool ufs_mtk_perf_is_supported(struct ufs_mtk_host *host) { if (!host->crypto_clk_mux || !host->crypto_parent_clk_normal || !host->crypto_parent_clk_perf || !host->req_vcore || host->crypto_vcore_opp < 0) return false; else return true; } int ufs_mtk_perf_setup_req(struct ufs_mtk_host *host, bool perf) { int err = 0; err = clk_prepare_enable(host->crypto_clk_mux); if (err) { dev_info(host->hba->dev, "%s: clk_prepare_enable(): %d\n", __func__, err); goto out; } if (perf) { pm_qos_update_request(host->req_vcore, host->crypto_vcore_opp); err = clk_set_parent(host->crypto_clk_mux, host->crypto_parent_clk_perf); } else { err = clk_set_parent(host->crypto_clk_mux, host->crypto_parent_clk_normal); pm_qos_update_request(host->req_vcore, PM_QOS_VCORE_OPP_DEFAULT_VALUE); } if (err) dev_info(host->hba->dev, "%s: clk_set_parent(): %d\n", __func__, err); clk_disable_unprepare(host->crypto_clk_mux); out: ufs_mtk_dbg_add_trace(host->hba, UFS_TRACE_PERF_MODE, perf, 0, (u32)err, 0, 0, 0, 0, 0, 0); return err; } int ufs_mtk_perf_setup_crypto_clk(struct ufs_mtk_host *host, bool perf) { int err = 0; bool rpm_resumed = false; bool clk_prepared = false; if (!ufs_mtk_perf_is_supported(host)) { dev_info(host->hba->dev, "%s: perf mode is unsupported\n", __func__); err = -ENOTSUPP; goto out; } /* runtime resume shall be prior to blocking requests */ pm_runtime_get_sync(host->hba->dev); rpm_resumed = true; /* * reuse clk scaling preparation function to wait until all * on-going commands are done, and then block future commands */ err = ufshcd_clock_scaling_prepare(host->hba); if (err) { dev_info(host->hba->dev, "%s: ufshcd_clock_scaling_prepare(): %d\n", __func__, err); goto out; } clk_prepared = true; err = ufs_mtk_perf_setup_req(host, perf); out: /* * add event before any possible incoming commands * by unblocking requests in ufshcd_clock_scaling_unprepare() */ dev_info(host->hba->dev, "perf mode: request %s %s\n", perf ? "on" : "off", err ? "failed" : "ok"); if (clk_prepared) ufshcd_clock_scaling_unprepare(host->hba); if (rpm_resumed) pm_runtime_put_sync(host->hba->dev); if (!err) host->perf_en = perf; return err; } int ufs_mtk_perf_setup(struct ufs_mtk_host *host, bool perf) { int err = 0; if (!ufs_mtk_perf_is_supported(host) || (host->perf_mode != PERF_AUTO)) { /* return without error */ return 0; } #ifdef CONFIG_UFSTW /* Turbo write may disable or not support */ if (!host->hba->ufsf.tw_lup[2] || !host->hba->ufsf.tw_lup[2]->tw_enable) return 0; #endif err = ufs_mtk_perf_setup_req(host, perf); if (!err) host->perf_en = perf; else dev_info(host->hba->dev, "%s: %s fail %d\n", __func__, perf ? "en":"dis", err); return err; } static int ufs_mtk_perf_init_crypto(struct ufs_hba *hba) { int err = 0; struct ufs_mtk_host *host = ufshcd_get_variant(hba); struct device_node *np = hba->dev->of_node; err = ufs_mtk_host_clk_get(hba->dev, "ufs-vendor-crypto-clk-mux", &host->crypto_clk_mux); if (err) { dev_info(hba->dev, "%s: failed to get ufs-vendor-crypto-clk-mux, err: %d", __func__, err); goto out; } err = ufs_mtk_host_clk_get(hba->dev, "ufs-vendor-crypto-normal-parent-clk", &host->crypto_parent_clk_normal); if (err) { dev_info(hba->dev, "%s: failed to get ufs-vendor-crypto-normal-parent-clk, err: %d", __func__, err); goto out; } err = ufs_mtk_host_clk_get(hba->dev, "ufs-vendor-crypto-perf-parent-clk", &host->crypto_parent_clk_perf); if (err) { dev_info(hba->dev, "%s: failed to get ufs-vendor-crypto-perf-parent-clk, err: %d", __func__, err); goto out; } err = of_property_read_s32(np, "mediatek,perf-crypto-vcore", &host->crypto_vcore_opp); if (err) { dev_info(hba->dev, "%s: failed to get mediatek,perf-crypto-vcore", __func__); host->crypto_vcore_opp = -1; goto out; } /* init VCORE QOS */ host->req_vcore = devm_kzalloc(hba->dev, sizeof(*host->req_vcore), GFP_KERNEL); if (!host->req_vcore) { err = -ENOMEM; goto out; } pm_qos_add_request(host->req_vcore, PM_QOS_VCORE_OPP, PM_QOS_VCORE_OPP_DEFAULT_VALUE); out: #ifdef CONFIG_UFSTW if (!err) host->perf_mode = PERF_AUTO; else #endif host->perf_mode = PERF_FORCE_DISABLE; return err; } static int ufs_mtk_setup_clocks(struct ufs_hba *hba, bool on, enum ufs_notify_change_status stage) { int ret = 0; struct ufs_mtk_host *host = ufshcd_get_variant(hba); switch (stage) { case PRE_CHANGE: if (!on) { if (host && host->pm_qos_init) { pm_qos_update_request( &host->req_cpu_dma_latency, PM_QOS_DEFAULT_VALUE); pm_qos_update_request( &host->req_mm_bandwidth, 0); ret = ufs_mtk_perf_setup(host, false); if (ret) goto out; } ret = ufs_mtk_pltfrm_ref_clk_ctrl(hba, false); if (ret) goto out; if (host && host->qos_enabled) ufs_mtk_biolog_clk_gating(on); } break; case POST_CHANGE: if (on) { ret = ufs_mtk_pltfrm_ref_clk_ctrl(hba, true); if (ret) goto out; if (host && host->pm_qos_init) { if (hba->quirks & UFSHCD_QUIRK_UFS_HCI_PERF_HEURISTIC) { pm_qos_update_request( &host->req_mm_bandwidth, 5554); } pm_qos_update_request( &host->req_cpu_dma_latency, 0); ret = ufs_mtk_perf_setup(host, true); if (ret) goto out; } if (host && host->qos_enabled) ufs_mtk_biolog_clk_gating(on); } break; default: break; } out: return ret; } static void ufs_mtk_set_caps(struct ufs_hba *hba) { hba->caps |= UFSHCD_CAP_AUTO_BKOPS_SUSPEND; } #if !defined(DISABLE_LOW_BATTERY_PROTECT) && defined(LOW_BATTERY_PT_SETTING_V2) static void ufs_mtk_low_batt_callback(LOW_BATTERY_LEVEL level) { struct ufs_hba *hba = ufs_mtk_hba; bool scale = false; bool up; if (!hba) return; if (level > LOW_BATTERY_LEVEL_2) { if (hba->pwr_info.gear_rx > 1 || hba->pwr_info.gear_tx > 1) { scale = true; up = false; } } else { if (hba->pwr_info.gear_rx <= 1 || hba->pwr_info.gear_tx <= 1) { scale = true; up = true; } } if (scale) { pm_runtime_get_sync(hba->dev); ufshcd_devfreq_scale(hba, up); pm_runtime_put_sync(hba->dev); } else { dev_info(hba->dev, "%s: skip scaling gear\n", __func__); } } #endif /** * ufs_mtk_init - find other essential mmio bases * @hba: host controller instance */ static int ufs_mtk_init(struct ufs_hba *hba) { struct ufs_mtk_host *host; int err = 0; struct platform_device *pdev; host = devm_kzalloc(hba->dev, sizeof(*host), GFP_KERNEL); if (!host) { err = -ENOMEM; dev_info(hba->dev, "%s: no memory for mtk ufs host\n", __func__); goto out; } host->hba = hba; ufshcd_set_variant(hba, host); /* initialize globals */ ufs_mtk_rpm_autosuspend_delay = -1; ufs_mtk_rpm_enabled = false; ufs_mtk_auto_hibern8_enabled = false; ufs_mtk_host_deep_stall_enable = 0; ufs_mtk_host_scramble_enable = 0; ufs_mtk_hba = hba; hba->crypto_hwfde_key_idx = -1; /* * Rename device to unify device path for booting storage device. * * Device rename shall be prior to any pinctrl operation to avoid * known kernel panic issue which can be triggered by dumping pin * information, for example, * * "cat /sys/kernel/debug/pinctrl/10005000.pinctrl/pinmux-pins". * * The panic is because create_pinctrl() will keep the original * device name string instance in kobject. However, old name string * instance will be freed during device_rename() but NOT awared by * pinctrl. * * Please also remove default pin state in device tree and related * code because create_pinctrl() will be activated before device * probing if default pin state is declared. */ device_rename(hba->dev, "bootdevice"); /* * fix uaf(use afer free) issue: modify pdev->name, * device_rename will free pdev->name */ pdev = to_platform_device(hba->dev); pdev->name = pdev->dev.kobj.name; ufs_mtk_pltfrm_init(); ufs_mtk_pltfrm_parse_dt(hba); ufs_mtk_set_caps(hba); ufs_mtk_advertise_hci_quirks(hba); ufs_mtk_parse_dt(hba); /* * If rpm_lvl and and spm_lvl are not already set to valid levels, * set the default power management level for UFS runtime and system * suspend. Default power saving mode selected is keeping UFS link in * Hibern8 state and UFS device in sleep. */ if (!ufs_mtk_is_valid_pm_lvl(hba->rpm_lvl)) hba->rpm_lvl = ufs_mtk_get_desired_pm_lvl_for_dev_link_state( UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE); if (!ufs_mtk_is_valid_pm_lvl(hba->spm_lvl)) hba->spm_lvl = ufs_mtk_get_desired_pm_lvl_for_dev_link_state( UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE); /* Get auto-hibern8 timeout from device tree */ ufs_mtk_parse_auto_hibern8_timer(hba); ufs_mtk_perf_init_crypto(hba); pm_qos_add_request(&host->req_cpu_dma_latency, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE); pm_qos_add_request(&host->req_mm_bandwidth, PM_QOS_MM_MEMORY_BANDWIDTH, 0); host->pm_qos_init = true; ufs_mtk_biolog_init(host->qos_allowed); #if !defined(DISABLE_LOW_BATTERY_PROTECT) && defined(LOW_BATTERY_PT_SETTING_V2) register_low_battery_notify(&ufs_mtk_low_batt_callback, LOW_BATTERY_PRIO_UFS); #endif out: return err; } /** * ufs_mtk_exit - release resource * @hba: host controller instance */ void ufs_mtk_exit(struct ufs_hba *hba) { struct ufs_mtk_host *host; host = ufshcd_get_variant(hba); if (host->pm_qos_init) { /* remove pm_qos when exit */ pm_qos_remove_request(host->req_vcore); pm_qos_remove_request(&host->req_cpu_dma_latency); host->pm_qos_init = false; } /* prevent pointer is used after hba is freed */ ufs_mtk_hba = NULL; } static int ufs_mtk_pre_pwr_change(struct ufs_hba *hba, struct ufs_pa_layer_attr *desired, struct ufs_pa_layer_attr *final) { int err = 0; struct ufs_descriptor desc; /* get manu ID for vendor specific configuration in the future */ if (hba->manu_id == 0) { /* read device descriptor */ desc.descriptor_idn = 0; desc.index = 0; err = ufs_mtk_query_desc(hba, UPIU_QUERY_FUNC_STANDARD_READ_REQUEST, desc.descriptor_idn, desc.index, desc.descriptor, sizeof(desc.descriptor)); if (err) return err; /* get wManufacturerID */ hba->manu_id = desc.descriptor[0x18] << 8 | desc.descriptor[0x19]; dev_dbg(hba->dev, "wManufacturerID: 0x%x\n", hba->manu_id); } /* HSG3B as default power mode, only use HSG1B at FPGA */ #ifndef CONFIG_FPGA_EARLY_PORTING final->gear_rx = desired->gear_rx; final->gear_tx = desired->gear_tx; final->gear_rx = min_t(u32, final->gear_rx, final->gear_tx); final->gear_tx = final->gear_rx; if (ufs_mtk_hs_gear == UFS_HS_G4) { if ((desired->gear_rx == UFS_HS_G4) && (desired->gear_tx == UFS_HS_G4)) { /* INITIAL ADAPT */ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXHSADAPTTYPE), PA_INITIAL_ADAPT); } else { /* NO ADAPT */ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXHSADAPTTYPE), PA_NO_ADAPT); } } #else final->gear_rx = UFS_HS_G1; final->gear_tx = UFS_HS_G1; #endif /* Change by dts setting */ if (hba->lanes_per_direction == 2) { final->lane_rx = 2; final->lane_tx = 2; } else { final->lane_rx = 1; final->lane_tx = 1; } final->hs_rate = desired->hs_rate; final->pwr_rx = desired->pwr_rx; final->pwr_tx = desired->pwr_tx; /* Set PAPowerModeUserData[0~5] = 0xffff, default is 0 */ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0), 0x1fff); ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1), 0xffff); ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2), 0x7fff); ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3), 0x1fff); ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4), 0xffff); ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5), 0x7fff); return err; } static int ufs_mtk_pwr_change_notify(struct ufs_hba *hba, enum ufs_notify_change_status stage, struct ufs_pa_layer_attr *desired, struct ufs_pa_layer_attr *final) { int ret = 0; switch (stage) { case PRE_CHANGE: ret = ufs_mtk_pre_pwr_change(hba, desired, final); break; case POST_CHANGE: break; default: break; } return ret; } static int ufs_mtk_init_mphy(struct ufs_hba *hba) { return 0; } static int ufs_mtk_enable_crypto(struct ufs_hba *hba) { /* restore vendor crypto setting by re-using resume operation */ mt_secure_call(MTK_SIP_KERNEL_HW_FDE_UFS_CTL, (1 << 2), 0, 0, 0); return 0; } static int ufs_mtk_reset_device(struct ufs_hba *hba) { dev_info(hba->dev, "reset device\n"); /* do device hw reset */ mt_secure_call(MTK_SIP_KERNEL_HW_FDE_UFS_CTL, (1 << 5), 0, 0, 0); return 0; } int ufs_mtk_linkup_fail_handler(struct ufs_hba *hba, int left_retry) { if (!(hba->quirks & UFSHCD_QUIRK_UFS_HCI_DEV_RST_FOR_LINKUP_FAIL)) return 0; if (left_retry <= 1) ufs_mtk_reset_device(hba); return 0; } int ufs_mtk_check_powerctl(struct ufs_hba *hba) { struct ufs_mtk_host *host = ufshcd_get_variant(hba); int err = 0; u32 val = 0; /* check if host in power saving */ err = ufshcd_dme_get(hba, UIC_ARG_MIB(VENDOR_UNIPROPOWERDOWNCONTROL), &val); if (!err && val == 0x1) { err = ufshcd_dme_set(hba, UIC_ARG_MIB(VENDOR_UNIPROPOWERDOWNCONTROL), 0); dev_info(hba->dev, "get dme 0x%x = %d, set 0 (%d)\n", VENDOR_UNIPROPOWERDOWNCONTROL, val, err); } /* * Set unipro as non-lpm mode anyway for initialization and error * recovery */ host->unipro_lpm = false; return err; } static int ufs_mtk_hce_enable_notify(struct ufs_hba *hba, enum ufs_notify_change_status stage) { struct ufs_mtk_host *host = ufshcd_get_variant(hba); int ret = 0; switch (stage) { case PRE_CHANGE: if (host->unipro_lpm) hba->hba_enable_delay_us = 0; else { hba->hba_enable_delay_us = 600; ufs_mtk_pltfrm_host_sw_rst(hba, SW_RST_TARGET_UFSHCI); } break; case POST_CHANGE: ret = ufs_mtk_enable_crypto(hba); /* * After HCE enable, need disable xoufs_req_s in ufshci * when xoufs hw solution is not ready. */ #ifndef UFS_REF_CLK_CTRL_BY_UFSHCI /* * Old chip not use this, disable it always after HCE enable * to deactivate UFS_SRCCLKENA/UFS_INFRA_REQ/UFS_VRF18_REQ * after ufs_mtk_pltfrm_resume. */ ufshcd_writel(hba, 0, REG_UFS_ADDR_XOUFS_ST); #endif if (hba->quirks & UFSHCD_QUIRK_UFS_HCI_PERF_HEURISTIC) { /* [31:16] PRE_ULTRA, [15:0] ULTRA */ ufshcd_writel(hba, 0x00400080, REG_UFS_MTK_AXI_W_ULTRA_THR); } break; default: break; } return ret; } static int ufs_mtk_pre_link(struct ufs_hba *hba) { int ret = 0; u32 tmp; ufs_mtk_pltfrm_bootrom_deputy(hba); ufs_mtk_init_mphy(hba); /* ensure auto-hibern8 is disabled during hba probing */ ufshcd_vops_auto_hibern8(hba, false); /* powerup unipro if unipro powerdown */ ret = ufs_mtk_check_powerctl(hba); if (ret) return ret; /* configure deep stall */ ret = ufshcd_dme_get(hba, UIC_ARG_MIB(VENDOR_SAVEPOWERCONTROL), &tmp); if (ret) return ret; if (ufs_mtk_host_deep_stall_enable) /* enable deep stall */ tmp |= (1 << 6); else tmp &= ~(1 << 6); /* disable deep stall */ ret = ufshcd_dme_set(hba, UIC_ARG_MIB(VENDOR_SAVEPOWERCONTROL), tmp); if (ret) return ret; /* configure scrambling */ if (ufs_mtk_host_scramble_enable) ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_SCRAMBLING), 1); else ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_SCRAMBLING), 0); return ret; } static int ufs_mtk_post_link(struct ufs_hba *hba) { int ret = 0; u32 arg = 0; u32 ah_ms; /* disable device LCC */ ret = ufs_mtk_send_uic_command(hba, UIC_CMD_DME_SET, UIC_ARG_MIB(PA_LOCALTXLCCENABLE), 0, &arg, NULL); if (ret) { dev_err(hba->dev, "dme_setting_after_link fail\n"); ret = 0; /* skip error */ } /* enable unipro clock gating feature */ ufs_mtk_cfg_unipro_cg(hba, true); /* configure clk gating delay */ if (ufshcd_is_clkgating_allowed(hba)) { if (ufshcd_is_auto_hibern8_supported(hba) && hba->ahit) ah_ms = FIELD_GET(UFSHCI_AHIBERN8_TIMER_MASK, hba->ahit); else ah_ms = 10; hba->clk_gating.delay_ms = ah_ms + 5; } else hba->clk_gating.delay_ms = 0; return ret; } static void ufs_mtk_vreg_lpm(struct ufs_hba *hba, bool lpm) { struct ufs_mtk_host *host = ufshcd_get_variant(hba); if (!host->vreg_lpm_supported || !hba->vreg_info.vcc) return; if (lpm & !hba->vreg_info.vcc->enabled) ufs_mtk_vufs_lpm(1); else if (!lpm) ufs_mtk_vufs_lpm(0); } static int ufs_mtk_link_startup_notify(struct ufs_hba *hba, enum ufs_notify_change_status stage) { int ret = 0; switch (stage) { case PRE_CHANGE: ret = ufs_mtk_pre_link(hba); break; case POST_CHANGE: ret = ufs_mtk_post_link(hba); break; default: break; } return ret; } static int ufs_mtk_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) { struct ufs_mtk_host *host = ufshcd_get_variant(hba); int ret = 0; if (ufshcd_is_link_hibern8(hba)) { /* * HCI power-down flow with link in hibern8 * Enter vendor-specific power down mode to keep UniPro state */ ret = ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VENDOR_UNIPROPOWERDOWNCONTROL, 0), 1); if (ret) { /* dump ufs debug Info like XO_UFS/VEMC/VUFS18 */ ufs_mtk_pltfrm_gpio_trigger_and_debugInfo_dump(hba); /* * Power down fail leave vendor-specific power down mode * to resume UniPro state */ (void)ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VENDOR_UNIPROPOWERDOWNCONTROL, 0), 0); ret = -EAGAIN; goto out; } if (ufshcd_is_link_off(hba)) mt_secure_call(MTK_SIP_KERNEL_UFS_CTL, 2, 0, 0, 0); host->unipro_lpm = true; ufs_mtk_pltfrm_suspend(hba); /* vendor-specific crypto suspend */ mt_secure_call(MTK_SIP_KERNEL_HW_FDE_UFS_CTL, (1 << 1), 0, 0, 0); /* * Make sure no error will be returned by suspend callback * before making regulators enter low-power mode because any * error will lead to re-enable regulators by error handling * in ufshcd_suspend(). */ ufs_mtk_vreg_lpm(hba, true); } out: if (ret) { ufs_mtk_pltfrm_gpio_trigger_and_debugInfo_dump(hba); ufshcd_print_host_state(hba, 0, NULL, NULL, NULL); ufs_mtk_dbg_dump_trace(NULL, NULL, 50, NULL); } return ret; } static void ufs_mtk_dbg_register_dump(struct ufs_hba *hba) { u32 val; /* read debugging register REG_UFS_MTK_PROBE */ /* * configure REG_UFS_MTK_DEBUG_SEL to direct debugging information * to REG_UFS_MTK_PROBE. * * REG_UFS_MTK_DEBUG_SEL is not required to set back * as 0x0 (default value) after dump. */ ufshcd_writel(hba, 0x20, REG_UFS_MTK_DEBUG_SEL); val = ufshcd_readl(hba, REG_UFS_MTK_PROBE); dev_info(hba->dev, "REG_UFS_MTK_PROBE: 0x%x\n", val); dev_info(hba->dev, "outstanding_reqs: 0x%x, tasks: 0x%x", hba->outstanding_reqs, hba->outstanding_tasks); dev_info(hba->dev, "r_cmd_cnt: %d, w_cmd_cnt: %d\n", hba->ufs_mtk_qcmd_r_cmd_cnt, hba->ufs_mtk_qcmd_w_cmd_cnt); } static int ufs_mtk_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) { struct ufs_mtk_host *host = ufshcd_get_variant(hba); int ret = 0; if (ufshcd_is_link_hibern8(hba)) { ufs_mtk_vreg_lpm(hba, false); ufs_mtk_pltfrm_resume(hba); /* * HCI power-on flow with link in hibern8 * * Enable UFSHCI * NOTE: interrupt mask UFSHCD_UIC_MASK * (UIC_COMMAND_COMPL | UFSHCD_UIC_PWR_MASK) * will be enabled inside. */ ret = ufshcd_hba_enable(hba); if (ret) { dev_err(hba->dev, "%s: hba_enable failed. ret = %d\n", __func__, ret); goto out; } /* Leave vendor-specific power down mode to resume * UniPro state */ ret = ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VENDOR_UNIPROPOWERDOWNCONTROL, 0), 0); if (ret) { dev_err(hba->dev, "%s: UniProPowerDownControl failed. ret = %d\n", __func__, ret); goto out; } host->unipro_lpm = false; /* * Leave hibern8 state * NOTE: ufshcd_make_hba_operational will be ok only if link * is in active state. * * The reason is: ufshcd_make_hba_operational * will check if REG_CONTROLLER_STATUS * is OK. In REG_CONTROLLER_STATUS, UTMRLRDY * and UTRLRDY will be ready only if * DP is set. DP will be set only if link * is in active state in MTK design. */ ret = ufshcd_uic_hibern8_exit(hba); if (!ret) ufshcd_set_link_active(hba); /* Re-start hba */ ret = ufshcd_make_hba_operational(hba); if (ret) { dev_err(hba->dev, "%s: make_hba_operational failed. ret = %d\n", __func__, ret); goto out; } /* vendor-specific crypto resume */ mt_secure_call(MTK_SIP_KERNEL_HW_FDE_UFS_CTL, (1 << 2), 0, 0, 0); } out: if (ret) { ufs_mtk_pltfrm_gpio_trigger_and_debugInfo_dump(hba); ufshcd_print_host_state(hba, 0, NULL, NULL, NULL); ufs_mtk_dbg_dump_trace(NULL, NULL, 50, NULL); } return ret; } void ufs_mtk_parse_dt(struct ufs_hba *hba) { struct device *dev = hba->dev; struct device_node *np = dev->of_node; struct ufs_mtk_host *host = ufshcd_get_variant(hba); u32 val; if (np) { /* system PM */ if (of_property_read_u32(np, "mediatek,spm-level", &hba->spm_lvl)) hba->spm_lvl = -1; /* runtime PM */ if (of_property_read_u32(np, "mediatek,rpm-level", &hba->rpm_lvl)) hba->rpm_lvl = -1; if (!of_property_read_u32(np, "mediatek,rpm-enable", &val)) { if (val) ufs_mtk_rpm_enabled = true; } if (of_property_read_u32(np, "mediatek,rpm-autosuspend-delay", &ufs_mtk_rpm_autosuspend_delay)) ufs_mtk_rpm_autosuspend_delay = -1; if (of_property_read_bool(np, "mediatek,spm_sw_mode")) host->spm_sw_mode = true; else host->spm_sw_mode = false; if (of_property_read_u32(np, "highspeed-gear", &ufs_mtk_hs_gear)) ufs_mtk_hs_gear = UFS_HS_G3; } #ifdef UFS_MTK_PLATFORM_EARA_IO host->qos_allowed = true; host->qos_enabled = true; #endif } void ufs_mtk_parse_auto_hibern8_timer(struct ufs_hba *hba) { struct device *dev = hba->dev; struct device_node *np = dev->of_node; unsigned long flags; u32 ahit; u32 ah_ms = 0; if (np) { if (of_property_read_u32(np, "mediatek,auto-hibern8-timer", &ah_ms)) ah_ms = 0; } dev_info(hba->dev, "auto-hibern8 timer %d ms\n", ah_ms); if (!ah_ms) return; /* update auto-hibern8 timer */ ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, ah_ms) | FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3); spin_lock_irqsave(hba->host->host_lock, flags); if (hba->ahit == ahit) goto out_unlock; hba->ahit = ahit; out_unlock: spin_unlock_irqrestore(hba->host->host_lock, flags); } /** * ufs_mtk_ffu_send_cmd - sends WRITE BUFFER command to do FFU * @hba: per adapter instance * @idata: ioctl data for ffu * * Returns 0 if ffu operation is sccessfull * Returns non-zero if failed to do ffu */ static int ufs_mtk_ffu_send_cmd(struct scsi_device *dev, struct ufs_ioctl_ffu_data *idata) { struct ufs_hba *hba; unsigned char cmd[10]; struct scsi_sense_hdr sshdr; unsigned long flags; int ret; int size_to_write, written; if (dev) hba = shost_priv(dev->host); else return -ENODEV; spin_lock_irqsave(hba->host->host_lock, flags); ret = scsi_device_get(dev); if (!ret && !scsi_device_online(dev)) { ret = -ENODEV; scsi_device_put(dev); } spin_unlock_irqrestore(hba->host->host_lock, flags); if (ret) return ret; for (written = 0; written < idata->buf_byte; written += size_to_write) { if ((idata->buf_byte - written) > MAX_WRITE_BUFFER_SIZE) size_to_write = MAX_WRITE_BUFFER_SIZE; else size_to_write = (idata->buf_byte - written); /* * If scsi commands fail, the scsi mid-layer schedules scsi * error-handling, which would wait for host to be resumed. * Since we know we are functional while we are here, skip * host resume in error handling context. */ hba->host->eh_noresume = 1; cmd[0] = WRITE_BUFFER; /* Opcode */ /* 0xE: Download firmware */ cmd[1] = 0xE; cmd[2] = 0; /* Buffer ID = 0 */ /* Buffer Offset[23:16] = 0 */ cmd[3] = (unsigned char)((written >> 16) & 0xff); /* Buffer Offset[15:08] = 0 */ cmd[4] = (unsigned char)((written >> 8) & 0xff); /* Buffer Offset[07:00] = 0 */ cmd[5] = (unsigned char)(written & 0xff); cmd[6] = (size_to_write >> 16) & 0xff; /* Length[23:16] */ cmd[7] = (size_to_write >> 8) & 0xff; /* Length[15:08] */ cmd[8] = (size_to_write) & 0xff; /* Length[07:00] */ cmd[9] = 0x0; /* Control = 0 */ /* * Current function would be generally called from the power * management callbacks hence set the RQF_PM flag so that it * doesn't resume the already suspended children. */ ret = scsi_execute(dev, cmd, DMA_TO_DEVICE, idata->buf_ptr + written, size_to_write, NULL, &sshdr, msecs_to_jiffies(1000), 0, 0, RQF_PM, NULL); if (ret) { sdev_printk(KERN_ERR, dev, "WRITE BUFFER failed for firmware upgrade\n"); } } scsi_device_put(dev); hba->host->eh_noresume = 0; return ret; } /** * ufs_mtk_ioctl_get_fw_ver - perform user request: query fw ver * @hba: per-adapter instance * @buffer: user space buffer for ffu ioctl data * @return: 0 for success negative error code otherwise * * Expected/Submitted buffer structure is struct ufs_ioctl_ffu_data. * It will read the buffer information of new firmware. */ int ufs_mtk_ioctl_get_fw_ver(struct scsi_device *dev, void __user *buf_user) { struct ufs_hba *hba; struct ufs_ioctl_query_fw_ver_data *idata = NULL; int err; if (dev) hba = shost_priv(dev->host); else return -ENODEV; /* check scsi device instance */ if (!dev->rev) { dev_err(hba->dev, "%s: scsi_device or rev is NULL\n", __func__); err = -ENOENT; goto out; } idata = kzalloc(sizeof(struct ufs_ioctl_query_fw_ver_data), GFP_KERNEL); if (!idata) { err = -ENOMEM; goto out; } /* extract params from user buffer */ err = copy_from_user(idata, buf_user, sizeof(struct ufs_ioctl_query_fw_ver_data)); if (err) { dev_err(hba->dev, "%s: failed copying buffer from user, err %d\n", __func__, err); goto out_release_mem; } idata->buf_byte = min_t(int, UFS_IOCTL_FFU_MAX_FW_VER_BYTES, idata->buf_byte); /* * Copy firmware version string to user buffer * * We get firmware version from scsi_device->rev, * which is ready in scsi_add_lun() * during SCSI device probe process. * * If probe failed, rev will be NULL. * We checked it in the beginning of this function. */ err = copy_to_user(idata->buf_ptr, dev->rev, idata->buf_byte); if (err) { dev_err(hba->dev, "%s: err %d copying back to user.\n", __func__, err); goto out_release_mem; } out_release_mem: kfree(idata); out: return 0; } void ufs_mtk_device_quiesce(struct ufs_hba *hba) { struct scsi_device *scsi_d; int i; /* * Wait all cmds done & block user issue cmds to * general LUs, wlun device, wlun rpmb and wlun boot. * To avoid new cmds coming after device has been * stopped by SSU cmd in ufshcd_suspend(). */ for (i = 0; i < UFS_UPIU_MAX_GENERAL_LUN; i++) { scsi_d = scsi_device_lookup(hba->host, 0, 0, i); if (scsi_d) scsi_device_quiesce(scsi_d); } scsi_d = scsi_device_lookup(hba->host, 0, 0, ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN)); if (scsi_d) scsi_device_quiesce(scsi_d); if (hba->sdev_ufs_device) scsi_device_quiesce(hba->sdev_ufs_device); if (hba->sdev_ufs_rpmb) scsi_device_quiesce(hba->sdev_ufs_rpmb); } void ufs_mtk_device_resume(struct ufs_hba *hba) { struct scsi_device *scsi_d; int i; for (i = 0; i < UFS_UPIU_MAX_GENERAL_LUN; i++) { scsi_d = scsi_device_lookup(hba->host, 0, 0, i); if (scsi_d) scsi_device_resume(scsi_d); } scsi_d = scsi_device_lookup(hba->host, 0, 0, ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN)); if (scsi_d) scsi_device_resume(scsi_d); if (hba->sdev_ufs_device) scsi_device_resume(hba->sdev_ufs_device); if (hba->sdev_ufs_rpmb) scsi_device_resume(hba->sdev_ufs_rpmb); } /** * ufs_mtk_ioctl_ffu - perform user ffu * @hba: per-adapter instance * @buffer: user space buffer for ffu ioctl data * @return: 0 for success negative error code otherwise * * Expected/Submitted buffer structure is struct ufs_ioctl_ffu_data. * It will read the buffer information of new firmware. */ int ufs_mtk_ioctl_ffu(struct scsi_device *dev, const void __user *buf_user) { struct ufs_hba *hba = shost_priv(dev->host); struct ufs_ioctl_ffu_data *idata = NULL; struct ufs_ioctl_ffu_data *idata_user = NULL; int err = 0; u32 attr = 0; idata = kzalloc(sizeof(struct ufs_ioctl_ffu_data), GFP_KERNEL); if (!idata) { err = -ENOMEM; goto out; } idata_user = kzalloc(sizeof(struct ufs_ioctl_ffu_data), GFP_KERNEL); if (!idata_user) { kfree(idata); err = -ENOMEM; goto out; } /* extract struct idata from user buffer */ err = copy_from_user(idata_user, buf_user, sizeof(struct ufs_ioctl_ffu_data)); if (err) { dev_err(hba->dev, "%s: failed copying buffer from user, err %d\n", __func__, err); goto out_release_mem; } memcpy(idata, idata_user, sizeof(struct ufs_ioctl_ffu_data)); /* extract firmware from user buffer */ if (idata->buf_byte > (u32)UFS_IOCTL_FFU_MAX_FW_SIZE_BYTES) { dev_err(hba->dev, "%s: idata->buf_byte:0x%x > max 0x%x bytes\n", __func__, idata->buf_byte, (u32)UFS_IOCTL_FFU_MAX_FW_SIZE_BYTES); err = -ENOMEM; goto out_release_mem; } idata->buf_ptr = kzalloc(idata->buf_byte, GFP_KERNEL); if (!idata->buf_ptr) { err = -ENOMEM; goto out_release_mem; } if (copy_from_user(idata->buf_ptr, (void __user *)idata_user->buf_ptr, idata->buf_byte)) { err = -EFAULT; goto out_release_idata_buf_ptr; } ufs_mtk_device_quiesce(hba); /* do FFU */ err = ufs_mtk_ffu_send_cmd(dev, idata); if (err) { dev_err(hba->dev, "%s: ffu failed, err %d\n", __func__, err); ufs_mtk_device_resume(hba); } else { dev_info(hba->dev, "%s: ffu ok\n", __func__); } /* * Check bDeviceFFUStatus attribute * * For reference only since UFS spec. said the status is valid after * device power cycle. */ err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_READ_ATTR, QUERY_ATTR_IDN_DEVICE_FFU_STATUS, 0, 0, &attr); if (err) { dev_err(hba->dev, "%s: query bDeviceFFUStatus failed, err %d\n", __func__, err); goto out_release_idata_buf_ptr; } if (attr > UFS_FFU_STATUS_OK) dev_err(hba->dev, "%s: bDeviceFFUStatus shows fail %d (ref only)\n", __func__, attr); out_release_idata_buf_ptr: kfree(idata->buf_ptr); out_release_mem: kfree(idata); kfree(idata_user); out: /* * UFS might not be used normally after FFU. * Just reboot system (including device) to avoid following * false alarm. For example, I/O errors. */ emergency_restart(); return err; } /** * ufs_mtk_ioctl_query - perform user read queries * @hba: per-adapter instance * @lun: used for lun specific queries * @buffer: user space buffer for reading and submitting query data and params * @return: 0 for success negative error code otherwise * * Expected/Submitted buffer structure is struct ufs_ioctl_query_data. * It will read the opcode, idn and buf_length parameters, and, put the * response in the buffer field while updating the used size in buf_length. */ #if defined(CONFIG_UFSFEATURE) int ufsf_query_ioctl(struct ufsf_feature *ufsf, unsigned int lun, void __user *buffer, struct ufs_ioctl_query_data_hpb *ioctl_data, u8 selector); #endif int ufs_mtk_ioctl_query(struct ufs_hba *hba, u8 lun, void __user *buf_user) { struct ufs_ioctl_query_data *idata; void __user *user_buf_ptr; int err = 0; int length = 0; void *data_ptr; bool flag; u32 att = 0; u8 *desc = NULL; u8 read_desc, read_attr, write_attr, read_flag; #if defined(CONFIG_UFSFEATURE) u8 selector; #endif idata = kzalloc(sizeof(struct ufs_ioctl_query_data), GFP_KERNEL); if (!idata) { err = -ENOMEM; goto out; } /* extract params from user buffer */ err = copy_from_user(idata, buf_user, sizeof(struct ufs_ioctl_query_data)); if (err) { dev_err(hba->dev, "%s: failed copying buffer from user, err %d\n", __func__, err); goto out_release_mem; } #if defined(CONFIG_UFSFEATURE) if (hba->card->wmanufacturerid == UFS_VENDOR_SAMSUNG || hba->card->wmanufacturerid == UFS_VENDOR_MICRON) selector = UFSFEATURE_SELECTOR; else selector = 0; if (ufsf_check_query(idata->opcode)) { err = ufsf_query_ioctl(&hba->ufsf, lun, buf_user, (struct ufs_ioctl_query_data_hpb *)idata, selector); goto out_release_mem; } #endif user_buf_ptr = idata->buf_ptr; /* * save idata->idn to specific local variable to avoid * confusion by comapring idata->idn with different enums below. */ switch (idata->opcode) { case UPIU_QUERY_OPCODE_READ_DESC: read_desc = idata->idn; break; case UPIU_QUERY_OPCODE_READ_ATTR: read_attr = idata->idn; break; case UPIU_QUERY_OPCODE_WRITE_ATTR: write_attr = idata->idn; break; case UPIU_QUERY_OPCODE_READ_FLAG: read_flag = idata->idn; break; default: goto out_einval; } /* verify legal parameters & send query */ switch (idata->opcode) { case UPIU_QUERY_OPCODE_READ_DESC: switch (read_desc) { case QUERY_DESC_IDN_DEVICE: case QUERY_DESC_IDN_STRING: break; default: goto out_einval; } length = min_t(int, QUERY_DESC_MAX_SIZE, idata->buf_byte); desc = kzalloc(length, GFP_KERNEL); if (!desc) { err = -ENOMEM; goto out_release_mem; } err = ufshcd_query_descriptor_retry(hba, idata->opcode, read_desc, idata->idx, 0, desc, &length); break; case UPIU_QUERY_OPCODE_READ_ATTR: switch (read_attr) { case QUERY_ATTR_IDN_BOOT_LUN_EN: break; case QUERY_ATTR_IDN_DEVICE_FFU_STATUS: break; default: goto out_einval; } err = ufshcd_query_attr(hba, idata->opcode, read_attr, idata->idx, 0, &att); break; case UPIU_QUERY_OPCODE_WRITE_ATTR: switch (write_attr) { case QUERY_ATTR_IDN_BOOT_LUN_EN: break; default: goto out_einval; } length = min_t(int, sizeof(int), idata->buf_byte); if (copy_from_user(&att, (void __user *)(unsigned long) idata->buf_ptr, length)) { err = -EFAULT; goto out_release_mem; } err = ufshcd_query_attr(hba, idata->opcode, write_attr, idata->idx, 0, &att); break; case UPIU_QUERY_OPCODE_READ_FLAG: switch (read_flag) { case QUERY_FLAG_IDN_PERMANENTLY_DISABLE_FW_UPDATE: break; default: goto out_einval; } err = ufshcd_query_flag(hba, idata->opcode, read_flag, &flag); break; default: goto out_einval; } if (err) { dev_err(hba->dev, "%s: query for idn %d failed\n", __func__, idata->idn); goto out_release_mem; } /* * copy response data * As we might end up reading less data then what is specified in * "ioct_data->buf_byte". So we are updating "ioct_data-> * buf_byte" to what exactly we have read. */ switch (idata->opcode) { case UPIU_QUERY_OPCODE_READ_DESC: idata->buf_byte = min_t(int, idata->buf_byte, length); data_ptr = desc; break; case UPIU_QUERY_OPCODE_READ_ATTR: idata->buf_byte = sizeof(att); data_ptr = &att; break; case UPIU_QUERY_OPCODE_READ_FLAG: idata->buf_byte = 1; data_ptr = &flag; break; case UPIU_QUERY_OPCODE_WRITE_ATTR: /* write attribute does not require coping response data */ goto out_release_mem; default: goto out_einval; } /* copy to user */ err = copy_to_user(buf_user, idata, sizeof(struct ufs_ioctl_query_data)); if (err) dev_err(hba->dev, "%s: failed copying back to user.\n", __func__); err = copy_to_user(user_buf_ptr, data_ptr, idata->buf_byte); if (err) dev_err(hba->dev, "%s: err %d copying back to user.\n", __func__, err); goto out_release_mem; out_einval: dev_err(hba->dev, "%s: illegal ufs query ioctl data, opcode 0x%x, idn 0x%x\n", __func__, idata->opcode, (unsigned int)idata->idn); err = -EINVAL; out_release_mem: kfree(idata); kfree(desc); out: return err; } /** * ufs_mtk_ioctl_rpmb - perform user rpmb read/write request * @hba: per-adapter instance * @buf_user: user space buffer for ioctl rpmb_cmd data * @return: 0 for success negative error code otherwise * * Expected/Submitted buffer structure is struct rpmb_cmd. * It will read/write data to rpmb */ int ufs_mtk_ioctl_rpmb(struct ufs_hba *hba, const void __user *buf_user) { struct rpmb_cmd cmd[3]; struct rpmb_frame *frame_buf = NULL; struct rpmb_frame *frames = NULL; int size = 0; int nframes = 0; unsigned long flags; struct scsi_device *sdev; int ret; int i; /* Get scsi device */ spin_lock_irqsave(hba->host->host_lock, flags); sdev = hba->sdev_ufs_rpmb; if (sdev) { ret = scsi_device_get(sdev); if (!ret && !scsi_device_online(sdev)) { ret = -ENODEV; scsi_device_put(sdev); } } else { ret = -ENODEV; } spin_unlock_irqrestore(hba->host->host_lock, flags); if (ret) { dev_info(hba->dev, "%s: failed get rpmb device, ret %d\n", __func__, ret); goto out; } /* Get cmd params from user buffer */ ret = copy_from_user((void *) cmd, buf_user, sizeof(struct rpmb_cmd) * 3); if (ret) { dev_info(hba->dev, "%s: failed copying cmd buffer from user, ret %d\n", __func__, ret); goto out_put; } /* Check number of rpmb frames */ for (i = 0; i < 3; i++) { ret = (int)rpmb_get_rw_size(ufs_mtk_rpmb_get_raw_dev()); if (cmd[i].nframes > ret) { dev_info(hba->dev, "%s: number of rpmb frames %u exceeds limit %d\n", __func__, cmd[i].nframes, ret); ret = -EINVAL; goto out_put; } } /* Prepaer frame buffer */ for (i = 0; i < 3; i++) nframes += cmd[i].nframes; frame_buf = kcalloc(nframes, sizeof(struct rpmb_frame), GFP_KERNEL); if (!frame_buf) { ret = -ENOMEM; goto out_put; } frames = frame_buf; /* * Send all command one by one. * Use rpmb lock to prevent other rpmb read/write threads cut in line. * Use mutex not spin lock because in/out function might sleep. */ mutex_lock(&hba->rpmb_lock); for (i = 0; i < 3; i++) { if (cmd[i].nframes == 0) break; /* Get frames from user buffer */ size = sizeof(struct rpmb_frame) * cmd[i].nframes; ret = copy_from_user((void *) frames, cmd[i].frames, size); if (ret) { dev_err(hba->dev, "%s: failed from user, ret %d\n", __func__, ret); break; } /* Do rpmb in out */ if (cmd[i].flags & RPMB_F_WRITE) { ret = ufshcd_rpmb_security_out(sdev, frames, cmd[i].nframes); if (ret) { dev_err(hba->dev, "%s: failed rpmb out, err %d\n", __func__, ret); break; } } else { ret = ufshcd_rpmb_security_in(sdev, frames, cmd[i].nframes); if (ret) { dev_err(hba->dev, "%s: failed rpmb in, err %d\n", __func__, ret); break; } /* Copy frames to user buffer */ ret = copy_to_user((void *) cmd[i].frames, frames, size); if (ret) { dev_err(hba->dev, "%s: failed to user, err %d\n", __func__, ret); break; } } frames += cmd[i].nframes; } mutex_unlock(&hba->rpmb_lock); kfree(frame_buf); out_put: scsi_device_put(sdev); out: return ret; } static int ufs_mtk_scsi_dev_cfg(struct scsi_device *sdev, enum ufs_scsi_dev_cfg op) { if (op == UFS_SCSI_DEV_SLAVE_CONFIGURE) { if (ufs_mtk_rpm_enabled) { sdev->use_rpm_auto = 1; sdev->autosuspend_delay = ufs_mtk_rpm_autosuspend_delay; } else { sdev->use_rpm_auto = 0; sdev->autosuspend_delay = -1; } } return 0; } void ufs_mtk_runtime_pm_init(struct scsi_device *sdev) { /* * If runtime PM is enabled for UFS device, use auto-suspend mechanism * if assigned. * * This is only for those SCSI devices which runtime PM is not managed * by block layer. Thus autosuspend of this device is not configured by * sd_probe_async. */ if (ufs_mtk_rpm_enabled && sdev->autosuspend_delay >= 0) { pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev, sdev->autosuspend_delay); pm_runtime_use_autosuspend(&sdev->sdev_gendev); } } static void ufs_mtk_device_reset(struct ufs_hba *hba) { /* disable hba before device reset */ ufshcd_hba_stop(hba, true); (void)ufs_mtk_pltfrm_ufs_device_reset(hba); #ifdef CONFIG_MTK_UFS_LBA_CRC16_CHECK /* Clear di memory to avoid false alarm */ ufs_mtk_di_reset(hba); #endif } static void ufs_mtk_auto_hibern8(struct ufs_hba *hba, bool enable) { /* if auto-hibern8 is not enabled by device tree, return */ if (!hba->ahit) return; dev_dbg(hba->dev, "ah8: %d, ahit: 0x%x\n", enable, hba->ahit); /* if already enabled or disabled, return */ if (!(ufs_mtk_auto_hibern8_enabled ^ enable)) return; if (enable) { ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER); ufs_mtk_auto_hibern8_enabled = true; } else { /* disable auto-hibern8 */ ufshcd_writel(hba, 0, REG_AUTO_HIBERNATE_IDLE_TIMER); ufs_mtk_auto_hibern8_enabled = false; /* wait host return to idle state when ah8 off */ ufs_mtk_wait_idle_state(hba, 5); } } int ufs_mtk_auto_hiber8_quirk_handler(struct ufs_hba *hba, bool enable) { /* * do not toggle ah8 during suspend/resume callback * since ah8 policy is always fixed * as below, * * 1. suspend: ah8 is disabled before suspend flow starts. * 2. resume: ah8 is enabled after resume flow is finished. */ if (hba->quirks & UFSHCD_QUIRK_UFS_HCI_DISABLE_AH8_BEFORE_RDB && !hba->outstanding_reqs && !hba->outstanding_tasks && !hba->pm_op_in_progress && (hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL)) { ufshcd_vops_auto_hibern8(hba, enable); } return 0; } void ufs_mtk_wait_idle_state(struct ufs_hba *hba, unsigned long retry_ms) { u64 timeout, time_checked; u32 val, sm; bool wait_idle; timeout = sched_clock() + retry_ms * 1000000UL; /* wait a specific time after check base */ udelay(10); wait_idle = false; do { time_checked = sched_clock(); ufshcd_writel(hba, 0x20, REG_UFS_MTK_DEBUG_SEL); val = ufshcd_readl(hba, REG_UFS_MTK_PROBE); sm = val & 0x1f; /* * if state is in H8 enter and H8 enter confirm * wait until return to idle state. */ if ((sm >= 0x8) && (sm <= 0xd)) { wait_idle = true; udelay(50); continue; } else if (!wait_idle) break; if (wait_idle && (sm == 0x1)) break; } while (time_checked < timeout); if (wait_idle && sm != 1) dev_info(hba->dev, "wait idle tmo: 0x%x\n", val); } int ufs_mtk_wait_link_state(struct ufs_hba *hba, u32 *state, unsigned long retry_ms) { u64 timeout, time_checked; u32 val; timeout = sched_clock() + retry_ms * 1000000UL; do { time_checked = sched_clock(); ufshcd_writel(hba, 0x20, REG_UFS_MTK_DEBUG_SEL); val = ufshcd_readl(hba, REG_UFS_MTK_PROBE); val = val >> 28; if (val == *state) break; /* sleep for max. 200us */ usleep_range(100, 200); } while (time_checked < timeout); if (val == *state) return 0; *state = val; return -ETIMEDOUT; } int ufs_mtk_generic_read_dme_no_check(u32 uic_cmd, u16 mib_attribute, u16 gen_select_index, u32 *value, unsigned long retry_ms) { struct ufs_hba *hba = ufs_mtk_hba; struct uic_command ucmd = {0}; u32 val; int ret = 0; unsigned long elapsed_us = 0; bool reenable_intr = false; val = ufshcd_readl(hba, REG_CONTROLLER_STATUS); if (!(val & UIC_COMMAND_READY) || hba->active_uic_cmd) { dev_info(hba->dev, "uic not rdy, host sta(0x%x), uic cmd(%d)\n", val, hba->active_uic_cmd ? hba->active_uic_cmd->command : -1); return -EIO; } if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) { ufshcd_disable_intr(hba, UIC_COMMAND_COMPL); /* * Make sure UIC command completion interrupt is disabled before * issuing UIC command. */ wmb(); reenable_intr = true; } ucmd.command = uic_cmd; ucmd.argument1 = UIC_ARG_MIB_SEL( (u32)mib_attribute, (u32)gen_select_index); ucmd.argument2 = 0; ucmd.argument3 = 0; ufshcd_writel(hba, ucmd.argument1, REG_UIC_COMMAND_ARG_1); ufshcd_writel(hba, ucmd.argument2, REG_UIC_COMMAND_ARG_2); ufshcd_writel(hba, ucmd.argument3, REG_UIC_COMMAND_ARG_3); ufs_mtk_dme_cmd_log(hba, &ucmd, UFS_TRACE_UIC_SEND); ufshcd_writel(hba, ucmd.command & COMMAND_OPCODE_MASK, REG_UIC_COMMAND); while ((ufshcd_readl(hba, REG_INTERRUPT_STATUS) & UIC_COMMAND_COMPL) != UIC_COMMAND_COMPL) { /* busy waiting 1us */ udelay(1); elapsed_us += 1; if (elapsed_us > (retry_ms * 1000)) { ret = -ETIMEDOUT; goto out; } } ufshcd_writel(hba, UIC_COMMAND_COMPL, REG_INTERRUPT_STATUS); ufs_mtk_dme_cmd_log(hba, &ucmd, UFS_TRACE_UIC_CMPL_GENERAL); val = ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2); if (val & MASK_UIC_COMMAND_RESULT) { ret = val; goto out; } *value = ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3); out: if (reenable_intr) ufshcd_enable_intr(hba, UIC_COMMAND_COMPL); return ret; } /* Notice: this function must be called in automic context */ /* Because it is not protected by ufs spin_lock or mutex */ /* it access ufs host directly. */ int ufs_mtk_generic_read_dme(u32 uic_cmd, u16 mib_attribute, u16 gen_select_index, u32 *value, unsigned long retry_ms) { if (ufs_mtk_hba->outstanding_reqs || ufs_mtk_hba->outstanding_tasks || ufs_mtk_hba->active_uic_cmd || ufs_mtk_hba->pm_op_in_progress) { dev_dbg(ufs_mtk_hba->dev, "req: %lx, task %lx, pm %x\n", ufs_mtk_hba->outstanding_reqs , ufs_mtk_hba->outstanding_tasks, ufs_mtk_hba->pm_op_in_progress); if (ufs_mtk_hba->active_uic_cmd != NULL) dev_dbg(ufs_mtk_hba->dev, "uic not null\n"); return -1; } return ufs_mtk_generic_read_dme_no_check(uic_cmd, mib_attribute, gen_select_index, value, retry_ms); } /** * ufs_mtk_deepidle_hibern8_check - callback function for Deepidle & SODI. * Release all resources: DRAM/26M clk/Main PLL and dsiable 26M ref clk if in H8 * * @return: 0 for success, negative/postive error code otherwise */ int ufs_mtk_deepidle_hibern8_check(void) { return ufs_mtk_pltfrm_deepidle_check_h8(); } /** * ufs_mtk_deepidle_leave - callback function for leaving Deepidle & SODI. */ void ufs_mtk_deepidle_leave(void) { ufs_mtk_pltfrm_deepidle_leave(); } struct rpmb_dev *ufs_mtk_rpmb_get_raw_dev(void) { return ufs_mtk_hba->rawdev_ufs_rpmb; } #ifdef CONFIG_MTK_UFS_DEBUG void ufs_mtk_dump_asc_ascq(struct ufs_hba *hba, u8 asc, u8 ascq) { dev_info(hba->dev, "Sense Data: ASC=%#04x, ASCQ=%#04x\n", asc, ascq); if (asc == 0x25) { if (ascq == 0x00) dev_err(hba->dev, "Logical unit not supported!\n"); } else if (asc == 0x29) { if (ascq == 0x00) dev_err(hba->dev, "Power on, reset, or bus device reset occupied\n"); } } #endif void ufs_mtk_crypto_cal_dun(u32 alg_id, u64 iv, u32 *dunl, u32 *dunu) { /* bitlocker dun use byte address */ if (alg_id == UFS_CRYPTO_ALGO_BITLOCKER_AES_CBC) iv = iv << 12; *dunl = iv & 0xffffffff; *dunu = (iv >> 32) & 0xffffffff; } bool ufs_mtk_is_data_write_cmd(struct scsi_cmnd *cmd, bool isolation) { char cmd_op = cmd->cmnd[0]; if (cmd_op == WRITE_10 || cmd_op == WRITE_16 || cmd_op == WRITE_6) return true; if (isolation) { if (cmd->sc_data_direction == DMA_TO_DEVICE) return true; } return false; } static inline bool ufs_mtk_is_unmap_cmd(struct scsi_cmnd *cmd) { char cmd_op = cmd->cmnd[0]; if (cmd_op == UNMAP) return true; return false; } static bool ufs_mtk_is_data_cmd(struct scsi_cmnd *cmd, bool isolation) { char cmd_op = cmd->cmnd[0]; if (cmd_op == WRITE_10 || cmd_op == READ_10 || cmd_op == WRITE_16 || cmd_op == READ_16 || cmd_op == WRITE_6 || cmd_op == READ_6) return true; if (isolation) { if ((cmd->sc_data_direction == DMA_FROM_DEVICE) || (cmd->sc_data_direction == DMA_TO_DEVICE)) return true; } return false; } static struct ufs_cmd_str_struct ufs_cmd_str_tbl[] = { {"TEST_UNIT_READY", 0x00}, {"REQUEST_SENSE", 0x03}, {"FORMAT_UNIT", 0x04}, {"READ_BLOCK_LIMITS", 0x05}, {"INQUIRY", 0x12}, {"RECOVER_BUFFERED_DATA", 0x14}, {"MODE_SENSE", 0x1a}, {"START_STOP", 0x1b}, {"SEND_DIAGNOSTIC", 0x1d}, {"READ_FORMAT_CAPACITIES", 0x23}, {"READ_CAPACITY", 0x25}, {"READ_10", 0x28}, {"WRITE_10", 0x2a}, {"PRE_FETCH", 0x34}, {"SYNCHRONIZE_CACHE", 0x35}, {"WRITE_BUFFER", 0x3b}, {"READ_BUFFER", 0x3c}, {"UNMAP", 0x42}, {"MODE_SELECT_10", 0x55}, {"MODE_SENSE_10", 0x5a}, {"REPORT_LUNS", 0xa0}, {"READ_CAPACITY_16", 0x9e}, {"SECURITY_PROTOCOL_IN", 0xa2}, {"MAINTENANCE_IN", 0xa3}, {"MAINTENANCE_OUT", 0xa4}, {"SECURITY_PROTOCOL_OUT", 0xb5}, {"UNKNOWN", 0xFF} }; static unsigned int ufs_mtk_get_cmd_str_idx(char cmd) { unsigned int i; for (i = 0; ufs_cmd_str_tbl[i].cmd != 0xFF; i++) { if (ufs_cmd_str_tbl[i].cmd == cmd) return i; } return i; } void ufs_mtk_dbg_dump_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, u32 flag) { #ifdef CONFIG_MTK_UFS_DEBUG u32 lba = 0; u32 blk_cnt; u32 fua, flush; char str[32]; strncpy(str, ufs_cmd_str_tbl[ufs_mtk_get_cmd_str_idx(cmd->cmnd[0])].str, 32 - 1); if (ufs_mtk_is_data_cmd(cmd, false)) { lba = cmd->cmnd[5] | (cmd->cmnd[4] << 8) | (cmd->cmnd[3] << 16) | (cmd->cmnd[2] << 24); blk_cnt = cmd->cmnd[8] | (cmd->cmnd[7] << 8); fua = (cmd->cmnd[1] & 0x8) ? 1 : 0; flush = (cmd->request->cmd_flags & REQ_FUA) ? 1 : 0; #ifdef CONFIG_MTK_HW_FDE if (cmd->request->bio && cmd->request->bio->bi_hw_fde) { dev_dbg(hba->dev, "QCMD(C),L:%x,T:%d,0x%x,%s,LBA:%d,BCNT:%d,FUA:%d,FLUSH:%d\n", ufshcd_scsi_to_upiu_lun(cmd->device->lun), cmd->request->tag, cmd->cmnd[0], str, lba, blk_cnt, fua, flush); } else #endif { dev_dbg(hba->dev, "QCMD,L:%x,T:%d,0x%x,%s,LBA:%d,BCNT:%d,FUA:%d,FLUSH:%d\n", ufshcd_scsi_to_upiu_lun(cmd->device->lun), cmd->request->tag, cmd->cmnd[0], str, lba, blk_cnt, fua, flush); } } else { dev_dbg(hba->dev, "QCMD,L:%x,T:%d,0x%x,%s\n", ufshcd_scsi_to_upiu_lun(cmd->device->lun), cmd->request->tag, cmd->cmnd[0], str); } #endif } #ifdef SPM_READY void ufs_mtk_res_ctrl(struct ufs_hba *hba, unsigned int op) { int res_type = 0; struct ufs_mtk_host *host = ufshcd_get_variant(hba); if (!host->spm_sw_mode) return; if (!hba->outstanding_tasks && !hba->outstanding_reqs) { if (op == UFS_RESCTL_CMD_SEND) { if (hba->active_uic_cmd) res_type = 1; else res_type = 2; } else if (op == UFS_RESCTL_CMD_COMP) res_type = 1; } if (res_type == 1) spm_resource_req(SPM_RESOURCE_USER_UFS, SPM_RESOURCE_MAINPLL | SPM_RESOURCE_CK_26M); else if (res_type == 2) spm_resource_req(SPM_RESOURCE_USER_UFS, SPM_RESOURCE_ALL); } #else #define ufs_mtk_res_ctrl NULL #endif static void ufs_mtk_abort_handler(struct ufs_hba *hba, int tag, char *file, int line) { #ifdef CONFIG_MTK_AEE_FEATURE u8 cmd = 0; dev_info(hba->dev, "%s: tag: %d\n", __func__, tag); if (tag == -1) { aee_kernel_warning_api(file, line, DB_OPT_FS_IO_LOG, "[UFS] Invalid Resp or OCS", "Invalid Resp or OCS, %s:%d", file, line); } else { if (hba->lrb[tag].cmd) cmd = hba->lrb[tag].cmd->cmnd[0]; aee_kernel_warning_api(file, line, DB_OPT_FS_IO_LOG, "[UFS] Command Timeout", "Command 0x%x timeout, %s:%d", cmd, file, line); } #endif } int ufs_mtk_perf_heurisic_if_allow_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd) { if (!(hba->quirks & UFSHCD_QUIRK_UFS_HCI_PERF_HEURISTIC)) return 0; /* Check rw commands only and allow all other commands. */ if (ufs_mtk_is_data_cmd(cmd, true)) { if (!hba->ufs_mtk_qcmd_r_cmd_cnt && !hba->ufs_mtk_qcmd_w_cmd_cnt) { /* Case: no on-going r or w commands. */ if (ufs_mtk_is_data_write_cmd(cmd, true)) hba->ufs_mtk_qcmd_w_cmd_cnt++; else hba->ufs_mtk_qcmd_r_cmd_cnt++; } else { if (ufs_mtk_is_data_write_cmd(cmd, true)) { if (hba->ufs_mtk_qcmd_r_cmd_cnt) return 1; hba->ufs_mtk_qcmd_w_cmd_cnt++; } else { if (hba->ufs_mtk_qcmd_w_cmd_cnt) return 1; hba->ufs_mtk_qcmd_r_cmd_cnt++; } } } return 0; } void ufs_mtk_perf_heurisic_req_done(struct ufs_hba *hba, struct scsi_cmnd *cmd) { if (!(hba->quirks & UFSHCD_QUIRK_UFS_HCI_PERF_HEURISTIC)) return; if (ufs_mtk_is_data_cmd(cmd, true)) { if (ufs_mtk_is_data_write_cmd(cmd, true)) hba->ufs_mtk_qcmd_w_cmd_cnt--; else hba->ufs_mtk_qcmd_r_cmd_cnt--; } } static void ufs_mtk_event_notify(struct ufs_hba *hba, enum ufs_event_type evt, void *data) { static bool skip_first_dev_reset = true; unsigned int val = *(u32 *)data; /* Ignore the first device reset during initialization */ if ((hba->lanes_per_direction == 2) && (evt == UFS_EVT_DEV_RESET) && skip_first_dev_reset) { skip_first_dev_reset = false; return; } if ((evt == UFS_EVT_SUSPEND_ERR && val == -EAGAIN) || (evt == UFS_EVT_PERF_WARN)) return; trace_ufs_mtk_event(evt, val); } static void ufs_mtk_setup_xfer_req(struct ufs_hba *hba, int tag, bool is_scsi) { struct ufshcd_lrb *lrbp; struct scsi_cmnd *cmd; if (is_scsi) { lrbp = &hba->lrb[tag]; cmd = lrbp->cmd; if (!ufs_mtk_is_data_cmd(cmd, false)) return; ufs_mtk_biolog_send_command(tag, cmd); ufs_mtk_biolog_check(hba->outstanding_reqs | (1 << tag)); } } static void ufs_mtk_compl_xfer_req(struct ufs_hba *hba, int tag, bool is_scsi) { struct ufshcd_lrb *lrbp; struct scsi_cmnd *cmd; unsigned long req_mask; if (is_scsi) { lrbp = &hba->lrb[tag]; cmd = lrbp->cmd; if (!ufs_mtk_is_data_cmd(cmd, false)) return; req_mask = hba->outstanding_reqs & ~(1 << tag); ufs_mtk_biolog_transfer_req_compl(tag, req_mask); ufs_mtk_biolog_check(req_mask); } } /** * struct ufs_hba_mtk_vops - UFS MTK specific variant operations * * The variant operations configure the necessary controller and PHY * handshake during initialization. */ static struct ufs_hba_variant_ops ufs_hba_mtk_vops = { "mediatek.ufshci", /* name */ ufs_mtk_init, /* init */ ufs_mtk_exit, /* exit */ NULL, /* get_ufs_hci_version */ NULL, /* clk_scale_notify */ ufs_mtk_setup_clocks, /* setup_clocks */ NULL, /* setup_regulators */ ufs_mtk_hce_enable_notify, /* hce_enable_notify */ ufs_mtk_link_startup_notify, /* link_startup_notify */ ufs_mtk_pwr_change_notify, /* pwr_change_notify */ ufs_mtk_setup_xfer_req, /* setup_xfer_req */ ufs_mtk_compl_xfer_req, /* compl_xfer_req */ NULL, /* setup_task_mgmt */ NULL, /* hibern8_notify */ NULL, /* apply_dev_quirks */ ufs_mtk_suspend, /* suspend */ ufs_mtk_resume, /* resume */ ufs_mtk_dbg_register_dump, /* dbg_register_dump */ NULL, /* phy_initialization */ ufs_mtk_device_reset, /* device_reset */ ufs_mtk_auto_hibern8, /* auto_hibern8 */ ufs_mtk_res_ctrl, /* res_ctrl */ ufs_mtk_pltfrm_deepidle_lock, /* deepidle_lock */ ufs_mtk_scsi_dev_cfg, /* scsi_dev_cfg */ NULL, /* program_key */ ufs_mtk_abort_handler, /* abort_handler */ ufs_mtk_event_notify /* event_notify */ }; /** * ufs_mtk_probe - probe routine of the driver * @pdev: pointer to Platform device handle * * Return zero for success and non-zero for failure */ static int ufs_mtk_probe(struct platform_device *pdev) { int err; struct device *dev = &pdev->dev; int boot_type; void __iomem *ufs_base; /* * Disable xoufs_req_s in ufshci to deactivate * 1. UFS_SRCCLKENA * 2. UFS_INFRA_REQ * 3. UFS_VRF18_REQ * Which block 26M off */ ufs_base = of_iomap(pdev->dev.of_node, 0); if (!ufs_base) { dev_err(dev, "ufs iomap failed\n"); return -ENODEV; }; #ifndef UFS_REF_CLK_CTRL_BY_UFSHCI /* Old chip not use this, disable it always */ writel(0, ufs_base + REG_UFS_ADDR_XOUFS_ST); #endif /* Add get_boot_type check and return ENODEV if not ufs boot */ boot_type = get_boot_type(); if (boot_type != BOOTDEV_UFS) { #ifdef UFS_REF_CLK_CTRL_BY_UFSHCI /* New chip disable it when eMMC boot */ writel(0, ufs_base + REG_UFS_ADDR_XOUFS_ST); #endif return -ENODEV; } /* perform generic probe */ err = ufshcd_pltfrm_init(pdev, &ufs_hba_mtk_vops); if (err) { dev_err(dev, "ufshcd_pltfrm_init() failed %d\n", err); goto out; } out: return err; } /** * ufs_mtk_remove - set driver_data of the device to NULL * @pdev: pointer to platform device handle * * Always return 0 */ static int ufs_mtk_remove(struct platform_device *pdev) { struct ufs_hba *hba = platform_get_drvdata(pdev); pm_runtime_get_sync(&(pdev)->dev); ufshcd_remove(hba); ufs_mtk_biolog_exit(); return 0; } const struct of_device_id ufs_mtk_of_match[] = { { .compatible = "mediatek,ufshci"}, {}, }; static const struct dev_pm_ops ufs_mtk_pm_ops = { .suspend = ufshcd_pltfrm_suspend, .resume = ufshcd_pltfrm_resume, .runtime_suspend = ufshcd_pltfrm_runtime_suspend, .runtime_resume = ufshcd_pltfrm_runtime_resume, .runtime_idle = ufshcd_pltfrm_runtime_idle, }; static struct platform_driver ufs_mtk_pltform = { .probe = ufs_mtk_probe, .remove = ufs_mtk_remove, .shutdown = ufshcd_pltfrm_shutdown, .driver = { .name = "ufshcd", .owner = THIS_MODULE, .pm = &ufs_mtk_pm_ops, .of_match_table = ufs_mtk_of_match, }, }; module_platform_driver(ufs_mtk_pltform);