diff --git a/drivers/mmc/card/block.c b/drivers/mmc/card/block.c index 4a0dba010cfd..cbf798babe5c 100644 --- a/drivers/mmc/card/block.c +++ b/drivers/mmc/card/block.c @@ -88,6 +88,8 @@ MODULE_ALIAS("mmc:block"); #define PCKD_TRGR_LOWER_BOUND 5 #define PCKD_TRGR_PRECISION_MULTIPLIER 100 +static struct mmc_cmdq_req *mmc_cmdq_prep_dcmd( + struct mmc_queue_req *mqrq, struct mmc_queue *mq); static DEFINE_MUTEX(block_mutex); /* @@ -1508,6 +1510,87 @@ int mmc_access_rpmb(struct mmc_queue *mq) return false; } +static struct mmc_cmdq_req *mmc_blk_cmdq_prep_discard_req(struct mmc_queue *mq, + struct request *req) +{ + struct mmc_blk_data *md = mq->data; + struct mmc_card *card = md->queue.card; + struct mmc_host *host = card->host; + struct mmc_cmdq_context_info *ctx_info = &host->cmdq_ctx; + struct mmc_cmdq_req *cmdq_req; + struct mmc_queue_req *active_mqrq; + + BUG_ON(req->tag > card->ext_csd.cmdq_depth); + BUG_ON(test_and_set_bit(req->tag, &host->cmdq_ctx.active_reqs)); + + set_bit(CMDQ_STATE_DCMD_ACTIVE, &ctx_info->curr_state); + + active_mqrq = &mq->mqrq_cmdq[req->tag]; + active_mqrq->req = req; + + cmdq_req = mmc_cmdq_prep_dcmd(active_mqrq, mq); + cmdq_req->cmdq_req_flags |= QBR; + cmdq_req->mrq.cmd = &cmdq_req->cmd; + cmdq_req->tag = req->tag; + return cmdq_req; +} + +static int mmc_blk_cmdq_issue_discard_rq(struct mmc_queue *mq, + struct request *req) +{ + struct mmc_blk_data *md = mq->data; + struct mmc_card *card = md->queue.card; + struct mmc_cmdq_req *cmdq_req = NULL; + struct mmc_host *host = card->host; + struct mmc_cmdq_context_info *ctx_info = &host->cmdq_ctx; + unsigned int from, nr, arg; + int err = 0; + + if (!mmc_can_erase(card)) { + err = -EOPNOTSUPP; + goto out; + } + + from = blk_rq_pos(req); + nr = blk_rq_sectors(req); + + if (mmc_can_discard(card)) + arg = MMC_DISCARD_ARG; + else if (mmc_can_trim(card)) + arg = MMC_TRIM_ARG; + else + arg = MMC_ERASE_ARG; + + cmdq_req = mmc_blk_cmdq_prep_discard_req(mq, req); + if (card->quirks & MMC_QUIRK_INAND_CMD38) { + __mmc_switch_cmdq_mode(cmdq_req->mrq.cmd, + EXT_CSD_CMD_SET_NORMAL, + INAND_CMD38_ARG_EXT_CSD, + arg == MMC_TRIM_ARG ? + INAND_CMD38_ARG_TRIM : + INAND_CMD38_ARG_ERASE, + 0, true, false); + err = mmc_cmdq_wait_for_dcmd(card->host, cmdq_req); + if (err) + goto clear_dcmd; + } + err = mmc_cmdq_erase(cmdq_req, card, from, nr, arg); +clear_dcmd: + /* clear pending request */ + if (cmdq_req) { + BUG_ON(!test_and_clear_bit(cmdq_req->tag, + &ctx_info->active_reqs)); + clear_bit(CMDQ_STATE_DCMD_ACTIVE, &ctx_info->curr_state); + } +out: + blk_end_request(req, err, blk_rq_bytes(req)); + + if (test_and_clear_bit(0, &ctx_info->req_starved)) + blk_run_queue(mq->queue); + mmc_release_host(host); + return err ? 1 : 0; +} + static int mmc_blk_issue_discard_rq(struct mmc_queue *mq, struct request *req) { struct mmc_blk_data *md = mq->data; @@ -1551,6 +1634,79 @@ out: return err ? 0 : 1; } +static int mmc_blk_cmdq_issue_secdiscard_rq(struct mmc_queue *mq, + struct request *req) +{ + struct mmc_blk_data *md = mq->data; + struct mmc_card *card = md->queue.card; + struct mmc_cmdq_req *cmdq_req = NULL; + unsigned int from, nr, arg; + struct mmc_host *host = card->host; + struct mmc_cmdq_context_info *ctx_info = &host->cmdq_ctx; + int err = 0; + + if (!(mmc_can_secure_erase_trim(card))) { + err = -EOPNOTSUPP; + goto out; + } + + from = blk_rq_pos(req); + nr = blk_rq_sectors(req); + + if (mmc_can_trim(card) && !mmc_erase_group_aligned(card, from, nr)) + arg = MMC_SECURE_TRIM1_ARG; + else + arg = MMC_SECURE_ERASE_ARG; + + cmdq_req = mmc_blk_cmdq_prep_discard_req(mq, req); + if (card->quirks & MMC_QUIRK_INAND_CMD38) { + __mmc_switch_cmdq_mode(cmdq_req->mrq.cmd, + EXT_CSD_CMD_SET_NORMAL, + INAND_CMD38_ARG_EXT_CSD, + arg == MMC_SECURE_TRIM1_ARG ? + INAND_CMD38_ARG_SECTRIM1 : + INAND_CMD38_ARG_SECERASE, + 0, true, false); + err = mmc_cmdq_wait_for_dcmd(card->host, cmdq_req); + if (err) + goto clear_dcmd; + } + + err = mmc_cmdq_erase(cmdq_req, card, from, nr, arg); + if (err) + goto clear_dcmd; + + if (arg == MMC_SECURE_TRIM1_ARG) { + if (card->quirks & MMC_QUIRK_INAND_CMD38) { + __mmc_switch_cmdq_mode(cmdq_req->mrq.cmd, + EXT_CSD_CMD_SET_NORMAL, + INAND_CMD38_ARG_EXT_CSD, + INAND_CMD38_ARG_SECTRIM2, + 0, true, false); + err = mmc_cmdq_wait_for_dcmd(card->host, cmdq_req); + if (err) + goto clear_dcmd; + } + + err = mmc_cmdq_erase(cmdq_req, card, from, nr, + MMC_SECURE_TRIM2_ARG); + } +clear_dcmd: + /* clear pending request */ + if (cmdq_req) { + BUG_ON(!test_and_clear_bit(cmdq_req->tag, + &ctx_info->active_reqs)); + clear_bit(CMDQ_STATE_DCMD_ACTIVE, &ctx_info->curr_state); + } +out: + blk_end_request(req, err, blk_rq_bytes(req)); + + if (test_and_clear_bit(0, &ctx_info->req_starved)) + blk_run_queue(mq->queue); + mmc_release_host(host); + return err ? 1 : 0; +} + static int mmc_blk_issue_secdiscard_rq(struct mmc_queue *mq, struct request *req) { @@ -3180,10 +3336,17 @@ static int mmc_blk_cmdq_issue_rq(struct mmc_queue *mq, struct request *req) } if (req) { - if (cmd_flags & REQ_FLUSH) + if (cmd_flags & REQ_DISCARD) { + if (cmd_flags & REQ_SECURE && + !(card->quirks & MMC_QUIRK_SEC_ERASE_TRIM_BROKEN)) + ret = mmc_blk_cmdq_issue_secdiscard_rq(mq, req); + else + ret = mmc_blk_cmdq_issue_discard_rq(mq, req); + } else if (cmd_flags & REQ_FLUSH) { ret = mmc_blk_cmdq_issue_flush_rq(mq, req); - else + } else { ret = mmc_blk_cmdq_issue_rw_rq(mq, req); + } } switch_failure: diff --git a/drivers/mmc/core/core.c b/drivers/mmc/core/core.c index 21d3968626b4..fd9f1878c742 100644 --- a/drivers/mmc/core/core.c +++ b/drivers/mmc/core/core.c @@ -1267,6 +1267,35 @@ int mmc_cmdq_start_req(struct mmc_host *host, struct mmc_cmdq_req *cmdq_req) } EXPORT_SYMBOL(mmc_cmdq_start_req); +static void mmc_cmdq_dcmd_req_done(struct mmc_request *mrq) +{ + complete(&mrq->completion); +} + +int mmc_cmdq_wait_for_dcmd(struct mmc_host *host, + struct mmc_cmdq_req *cmdq_req) +{ + struct mmc_request *mrq = &cmdq_req->mrq; + struct mmc_command *cmd = mrq->cmd; + int err = 0; + + init_completion(&mrq->completion); + mrq->done = mmc_cmdq_dcmd_req_done; + err = mmc_cmdq_start_req(host, cmdq_req); + if (err) + return err; + + wait_for_completion_io(&mrq->completion); + if (cmd->error) { + pr_err("%s: DCMD %d failed with err %d\n", + mmc_hostname(host), cmd->opcode, + cmd->error); + err = cmd->error; + } + return err; +} +EXPORT_SYMBOL(mmc_cmdq_wait_for_dcmd); + int mmc_cmdq_prepare_flush(struct mmc_command *cmd) { return __mmc_switch_cmdq_mode(cmd, EXT_CSD_CMD_SET_NORMAL, @@ -2911,20 +2940,9 @@ static unsigned int mmc_erase_timeout(struct mmc_card *card, return mmc_mmc_erase_timeout(card, arg, qty); } -static int mmc_do_erase(struct mmc_card *card, unsigned int from, - unsigned int to, unsigned int arg) +static u32 mmc_get_erase_qty(struct mmc_card *card, u32 from, u32 to) { - struct mmc_command cmd = {0}; - unsigned int qty = 0; - unsigned long timeout; - unsigned int fr, nr; - int err; - - fr = from; - nr = to - from + 1; - trace_mmc_blk_erase_start(arg, fr, nr); - - mmc_retune_hold(card->host); + u32 qty = 0; /* * qty is used to calculate the erase timeout which depends on how many @@ -2950,12 +2968,122 @@ static int mmc_do_erase(struct mmc_card *card, unsigned int from, else qty += ((to / card->erase_size) - (from / card->erase_size)) + 1; + return qty; +} + +static int mmc_cmdq_send_erase_cmd(struct mmc_cmdq_req *cmdq_req, + struct mmc_card *card, u32 opcode, u32 arg, u32 qty) +{ + struct mmc_command *cmd = cmdq_req->mrq.cmd; + int err; + + memset(cmd, 0, sizeof(struct mmc_command)); + + cmd->opcode = opcode; + cmd->arg = arg; + if (cmd->opcode == MMC_ERASE) { + cmd->flags = MMC_RSP_SPI_R1B | MMC_RSP_R1B | MMC_CMD_AC; + cmd->busy_timeout = mmc_erase_timeout(card, arg, qty); + } else { + cmd->flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_AC; + } + + err = mmc_cmdq_wait_for_dcmd(card->host, cmdq_req); + if (err) { + pr_err("mmc_erase: group start error %d, status %#x\n", + err, cmd->resp[0]); + return -EIO; + } + return 0; +} + +static int mmc_cmdq_do_erase(struct mmc_cmdq_req *cmdq_req, + struct mmc_card *card, unsigned int from, + unsigned int to, unsigned int arg) +{ + struct mmc_command *cmd = cmdq_req->mrq.cmd; + unsigned int qty = 0; + unsigned long timeout; + unsigned int fr, nr; + int err; + + fr = from; + nr = to - from + 1; + trace_mmc_blk_erase_start(arg, fr, nr); + + qty = mmc_get_erase_qty(card, from, to); if (!mmc_card_blockaddr(card)) { from <<= 9; to <<= 9; } + err = mmc_cmdq_send_erase_cmd(cmdq_req, card, MMC_ERASE_GROUP_START, + from, qty); + if (err) + goto out; + + err = mmc_cmdq_send_erase_cmd(cmdq_req, card, MMC_ERASE_GROUP_END, + to, qty); + if (err) + goto out; + + err = mmc_cmdq_send_erase_cmd(cmdq_req, card, MMC_ERASE, + arg, qty); + if (err) + goto out; + + timeout = jiffies + msecs_to_jiffies(MMC_CORE_TIMEOUT_MS); + do { + memset(cmd, 0, sizeof(struct mmc_command)); + cmd->opcode = MMC_SEND_STATUS; + cmd->arg = card->rca << 16; + cmd->flags = MMC_RSP_R1 | MMC_CMD_AC; + /* Do not retry else we can't see errors */ + err = mmc_cmdq_wait_for_dcmd(card->host, cmdq_req); + if (err || (cmd->resp[0] & 0xFDF92000)) { + pr_err("error %d requesting status %#x\n", + err, cmd->resp[0]); + err = -EIO; + goto out; + } + /* Timeout if the device never becomes ready for data and + * never leaves the program state. + */ + if (time_after(jiffies, timeout)) { + pr_err("%s: Card stuck in programming state! %s\n", + mmc_hostname(card->host), __func__); + err = -EIO; + goto out; + } + } while (!(cmd->resp[0] & R1_READY_FOR_DATA) || + (R1_CURRENT_STATE(cmd->resp[0]) == R1_STATE_PRG)); +out: + trace_mmc_blk_erase_end(arg, fr, nr); + return err; +} + +static int mmc_do_erase(struct mmc_card *card, unsigned int from, + unsigned int to, unsigned int arg) +{ + struct mmc_command cmd = {0}; + unsigned int qty = 0; + unsigned long timeout; + unsigned int fr, nr; + int err; + + fr = from; + nr = to - from + 1; + trace_mmc_blk_erase_start(arg, fr, nr); + + qty = mmc_get_erase_qty(card, from, to); + + if (!mmc_card_blockaddr(card)) { + from <<= 9; + to <<= 9; + } + + mmc_retune_hold(card->host); if (mmc_card_sd(card)) cmd.opcode = SD_ERASE_WR_BLK_START; else @@ -3034,21 +3162,9 @@ out: return err; } -/** - * mmc_erase - erase sectors. - * @card: card to erase - * @from: first sector to erase - * @nr: number of sectors to erase - * @arg: erase command argument (SD supports only %MMC_ERASE_ARG) - * - * Caller must claim host before calling this function. - */ -int mmc_erase(struct mmc_card *card, unsigned int from, unsigned int nr, - unsigned int arg) +int mmc_erase_sanity_check(struct mmc_card *card, unsigned int from, + unsigned int nr, unsigned int arg) { - unsigned int rem, to = from + nr; - int err; - if (!(card->host->caps & MMC_CAP_ERASE) || !(card->csd.cmdclass & CCC_ERASE)) return -EOPNOTSUPP; @@ -3071,6 +3187,68 @@ int mmc_erase(struct mmc_card *card, unsigned int from, unsigned int nr, if (from % card->erase_size || nr % card->erase_size) return -EINVAL; } + return 0; +} + +int mmc_cmdq_erase(struct mmc_cmdq_req *cmdq_req, + struct mmc_card *card, unsigned int from, unsigned int nr, + unsigned int arg) +{ + unsigned int rem, to = from + nr; + int ret; + + ret = mmc_erase_sanity_check(card, from, nr, arg); + if (ret) + return ret; + + if (arg == MMC_ERASE_ARG) { + rem = from % card->erase_size; + if (rem) { + rem = card->erase_size - rem; + from += rem; + if (nr > rem) + nr -= rem; + else + return 0; + } + rem = nr % card->erase_size; + if (rem) + nr -= rem; + } + + if (nr == 0) + return 0; + + to = from + nr; + + if (to <= from) + return -EINVAL; + + /* 'from' and 'to' are inclusive */ + to -= 1; + + return mmc_cmdq_do_erase(cmdq_req, card, from, to, arg); +} +EXPORT_SYMBOL(mmc_cmdq_erase); + +/** + * mmc_erase - erase sectors. + * @card: card to erase + * @from: first sector to erase + * @nr: number of sectors to erase + * @arg: erase command argument (SD supports only %MMC_ERASE_ARG) + * + * Caller must claim host before calling this function. + */ +int mmc_erase(struct mmc_card *card, unsigned int from, unsigned int nr, + unsigned int arg) +{ + unsigned int rem, to = from + nr; + int ret; + + ret = mmc_erase_sanity_check(card, from, nr, arg); + if (ret) + return ret; if (arg == MMC_ERASE_ARG) { rem = from % card->erase_size; @@ -3108,10 +3286,10 @@ int mmc_erase(struct mmc_card *card, unsigned int from, unsigned int nr, */ rem = card->erase_size - (from % card->erase_size); if ((arg & MMC_TRIM_ARGS) && (card->eg_boundary) && (nr > rem)) { - err = mmc_do_erase(card, from, from + rem - 1, arg); + ret = mmc_do_erase(card, from, from + rem - 1, arg); from += rem; - if ((err) || (to <= from)) - return err; + if ((ret) || (to <= from)) + return ret; } return mmc_do_erase(card, from, to, arg); diff --git a/include/linux/mmc/core.h b/include/linux/mmc/core.h index 59d9196250c1..fcad0aa31b2e 100644 --- a/include/linux/mmc/core.h +++ b/include/linux/mmc/core.h @@ -123,6 +123,11 @@ extern void mmc_cmdq_post_req(struct mmc_host *host, struct mmc_request *mrq, extern int mmc_cmdq_start_req(struct mmc_host *host, struct mmc_cmdq_req *cmdq_req); extern int mmc_cmdq_prepare_flush(struct mmc_command *cmd); +extern int mmc_cmdq_wait_for_dcmd(struct mmc_host *host, + struct mmc_cmdq_req *cmdq_req); +extern int mmc_cmdq_erase(struct mmc_cmdq_req *cmdq_req, + struct mmc_card *card, unsigned int from, unsigned int nr, + unsigned int arg); extern int mmc_stop_bkops(struct mmc_card *); extern int mmc_read_bkops_status(struct mmc_card *);