// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Copyright (C) 2017 Free Electrons
* Copyright (C) 2017 NextThing Co
*
* Author: Boris Brezillon <boris.brezillon@free-electrons.com>
*/
#include <linux/slab.h>
#include "internals.h"
/*
* Special Micron status bit 3 indicates that the block has been
* corrected by on-die ECC and should be rewritten.
*/
#define NAND_ECC_STATUS_WRITE_RECOMMENDED BIT(3)
/*
* On chips with 8-bit ECC and additional bit can be used to distinguish
* cases where a errors were corrected without needing a rewrite
*
* Bit 4 Bit 3 Bit 0 Description
* ----- ----- ----- -----------
* 0 0 0 No Errors
* 0 0 1 Multiple uncorrected errors
* 0 1 0 4 - 6 errors corrected, recommend rewrite
* 0 1 1 Reserved
* 1 0 0 1 - 3 errors corrected
* 1 0 1 Reserved
* 1 1 0 7 - 8 errors corrected, recommend rewrite
*/
#define NAND_ECC_STATUS_MASK (BIT(4) | BIT(3) | BIT(0))
#define NAND_ECC_STATUS_UNCORRECTABLE BIT(0)
#define NAND_ECC_STATUS_4_6_CORRECTED BIT(3)
#define NAND_ECC_STATUS_1_3_CORRECTED BIT(4)
#define NAND_ECC_STATUS_7_8_CORRECTED (BIT(4) | BIT(3))
struct nand_onfi_vendor_micron {
u8 two_plane_read;
u8 read_cache;
u8 read_unique_id;
u8 dq_imped;
u8 dq_imped_num_settings;
u8 dq_imped_feat_addr;
u8 rb_pulldown_strength;
u8 rb_pulldown_strength_feat_addr;
u8 rb_pulldown_strength_num_settings;
u8 otp_mode;
u8 otp_page_start;
u8 otp_data_prot_addr;
u8 otp_num_pages;
u8 otp_feat_addr;
u8 read_retry_options;
u8 reserved[72];
u8 param_revision;
} __packed;
struct micron_on_die_ecc {
bool forced;
bool enabled;
void *rawbuf;
};
struct micron_nand {
struct micron_on_die_ecc ecc;
};
static int micron_nand_setup_read_retry(struct nand_chip *chip, int retry_mode)
{
u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = {retry_mode};
return nand_set_features(chip, ONFI_FEATURE_ADDR_READ_RETRY, feature);
}
/*
* Configure chip properties from Micron vendor-specific ONFI table
*/
static int micron_nand_onfi_init(struct nand_chip *chip)
{
struct nand_parameters *p = &chip->parameters;
if (p->onfi) {
struct nand_onfi_vendor_micron *micron = (void *)p->onfi->vendor;
chip->read_retries = micron->read_retry_options;
chip->setup_read_retry = micron_nand_setup_read_retry;
}
if (p->supports_set_get_features) {
set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->set_feature_list);
set_bit(ONFI_FEATURE_ON_DIE_ECC, p->set_feature_list);
set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->get_feature_list);
set_bit(ONFI_FEATURE_ON_DIE_ECC, p->get_feature_list);
}
return 0;
}
static int micron_nand_on_die_4_ooblayout_ecc(struct mtd_info *mtd,
int section,
struct mtd_oob_region *oobregion)
{
if (section >= 4)
return -ERANGE;
oobregion->offset = (section * 16) + 8;
oobregion->length = 8;
return 0;
}
static int micron_nand_on_die_4_ooblayout_free(struct mtd_info *mtd,
int section,
struct mtd_oob_region *oobregion)
{
if (section >= 4)
return -ERANGE;
oobregion->offset = (section * 16) + 2;
oobregion->length = 6;
return 0;
}
static const struct mtd_ooblayout_ops micron_nand_on_die_4_ooblayout_ops = {
.ecc = micron_nand_on_die_4_ooblayout_ecc,
.free = micron_nand_on_die_4_ooblayout_free,
};
static int micron_nand_on_die_8_ooblayout_ecc(struct mtd_info *mtd,
int section,
struct mtd_oob_region *oobregion)
{
struct nand_chip *chip = mtd_to_nand(mtd);
if (section)
return -ERANGE;
oobregion->offset = mtd->oobsize - chip->ecc.total;
oobregion->length = chip->ecc.total;
return 0;
}
static int micron_nand_on_die_8_ooblayout_free(struct mtd_info *mtd,
int section,
struct mtd_oob_region *oobregion)
{
struct nand_chip *chip = mtd_to_nand(mtd);
if (section)
return -ERANGE;
oobregion->offset = 2;
oobregion->length = mtd->oobsize - chip->ecc.total - 2;
return 0;
}
static const struct mtd_ooblayout_ops micron_nand_on_die_8_ooblayout_ops = {
.ecc = micron_nand_on_die_8_ooblayout_ecc,
.free = micron_nand_on_die_8_ooblayout_free,
};
static int micron_nand_on_die_ecc_setup(struct nand_chip *chip, bool enable)
{
struct micron_nand *micron = nand_get_manufacturer_data(chip);
u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = { 0, };
int ret;
if (micron->ecc.forced)
return 0;
if (micron->ecc.enabled == enable)
return 0;
if (enable)
feature[0] |= ONFI_FEATURE_ON_DIE_ECC_EN;
ret = nand_set_features(chip, ONFI_FEATURE_ON_DIE_ECC, feature);
if (!ret)
micron->ecc.enabled = enable;
return ret;
}
static int micron_nand_on_die_ecc_status_4(struct nand_chip *chip, u8 status,
void *buf, int page,
int oob_required)
{
struct micron_nand *micron = nand_get_manufacturer_data(chip);
struct mtd_info *mtd = nand_to_mtd(chip);
unsigned int step, max_bitflips = 0;
int ret;
if (!(status & NAND_ECC_STATUS_WRITE_RECOMMENDED)) {
if (status & NAND_STATUS_FAIL)
mtd->ecc_stats.failed++;
return 0;
}
/*
* The internal ECC doesn't tell us the number of bitflips that have
* been corrected, but tells us if it recommends to rewrite the block.
* If it's the case, we need to read the page in raw mode and compare
* its content to the corrected version to extract the actual number of
* bitflips.
* But before we do that, we must make sure we have all OOB bytes read
* in non-raw mode, even if the user did not request those bytes.
*/
if (!oob_required) {
ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize,
false);
if (ret)
return ret;
}
micron_nand_on_die_ecc_setup(chip, false);
ret = nand_read_page_op(chip, page, 0, micron->ecc.rawbuf,
mtd->writesize + mtd->oobsize);
if (ret)
return ret;
for (step = 0; step < chip->ecc.steps; step++) {
unsigned int offs, i, nbitflips = 0;
u8 *rawbuf, *corrbuf;
offs = step * chip->ecc.size;
rawbuf = micron->ecc.rawbuf + offs;
corrbuf = buf + offs;
for (i = 0; i < chip->ecc.size; i++)
nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]);
offs = (step * 16) + 4;
rawbuf = micron->ecc.rawbuf + mtd->writesize + offs;
corrbuf = chip->oob_poi + offs;
for (i = 0; i < chip->ecc.bytes + 4; i++)
nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]);
if (WARN_ON(nbitflips > chip->ecc.strength))
return -EINVAL;
max_bitflips = max(nbitflips, max_bitflips);
mtd->ecc_stats.corrected += nbitflips;
}
return max_bitflips;
}
static int micron_nand_on_die_ecc_status_8(struct nand_chip *chip, u8 status)
{
struct mtd_info *mtd = nand_to_mtd(chip);
/*
* With 8/512 we have more information but still don't know precisely
* how many bit-flips were seen.
*/
switch (status & NAND_ECC_STATUS_MASK) {
case NAND_ECC_STATUS_UNCORRECTABLE:
mtd->ecc_stats.failed++;
return 0;
case NAND_ECC_STATUS_1_3_CORRECTED:
mtd->ecc_stats.corrected += 3;
return 3;
case NAND_ECC_STATUS_4_6_CORRECTED:
mtd->ecc_stats.corrected += 6;
/* rewrite recommended */
return 6;
case NAND_ECC_STATUS_7_8_CORRECTED:
mtd->ecc_stats.corrected += 8;
/* rewrite recommended */
return 8;
default:
return 0;
}
}
static int
micron_nand_read_page_on_die_ecc(struct nand_chip *chip, uint8_t *buf,
int oob_required, int page)
{
struct mtd_info *mtd = nand_to_mtd(chip);
u8 status;
int ret, max_bitflips = 0;
ret = micron_nand_on_die_ecc_setup(chip, true);
if (ret)
return ret;
ret = nand_read_page_op(chip, page, 0, NULL, 0);
if (ret)
goto out;
ret = nand_status_op(chip, &status);
if (ret)
goto out;
ret = nand_exit_status_op(chip);
if (ret)
goto out;
ret = nand_read_data_op(chip, buf, mtd->writesize, false);
if (!ret && oob_required)
ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize,
false);
if (chip->ecc.strength == 4)
max_bitflips = micron_nand_on_die_ecc_status_4(chip, status,
buf, page,
oob_required);
else
max_bitflips = micron_nand_on_die_ecc_status_8(chip, status);
out:
micron_nand_on_die_ecc_setup(chip, false);
return ret ? ret : max_bitflips;
}
static int
micron_nand_write_page_on_die_ecc(struct nand_chip *chip, const uint8_t *buf,
int oob_required, int page)
{
int ret;
ret = micron_nand_on_die_ecc_setup(chip, true);
if (ret)
return ret;
ret = nand_write_page_raw(chip, buf, oob_required, page);
micron_nand_on_die_ecc_setup(chip, false);
return ret;
}
enum {
/* The NAND flash doesn't support on-die ECC */
MICRON_ON_DIE_UNSUPPORTED,
/*
* The NAND flash supports on-die ECC and it can be
* enabled/disabled by a set features command.
*/
MICRON_ON_DIE_SUPPORTED,
/*
* The NAND flash supports on-die ECC, and it cannot be
* disabled.
*/
MICRON_ON_DIE_MANDATORY,
};
#define MICRON_ID_INTERNAL_ECC_MASK GENMASK(1, 0)
#define MICRON_ID_ECC_ENABLED BIT(7)
/*
* Try to detect if the NAND support on-die ECC. To do this, we enable
* the feature, and read back if it has been enabled as expected. We
* also check if it can be disabled, because some Micron NANDs do not
* allow disabling the on-die ECC and we don't support such NANDs for
* now.
*
* This function also has the side effect of disabling on-die ECC if
* it had been left enabled by the firmware/bootloader.
*/
static int micron_supports_on_die_ecc(struct nand_chip *chip)
{
u8 id[5];
int ret;
if (!chip->parameters.onfi)
return MICRON_ON_DIE_UNSUPPORTED;
if (nanddev_bits_per_cell(&chip->base) != 1)
return MICRON_ON_DIE_UNSUPPORTED;
/*
* We only support on-die ECC of 4/512 or 8/512
*/
if (chip->base.eccreq.strength != 4 && chip->base.eccreq.strength != 8)
return MICRON_ON_DIE_UNSUPPORTED;
/* 0x2 means on-die ECC is available. */
if (chip->id.len != 5 ||
(chip->id.data[4] & MICRON_ID_INTERNAL_ECC_MASK) != 0x2)
return MICRON_ON_DIE_UNSUPPORTED;
/*
* It seems that there are devices which do not support ECC officially.
* At least the MT29F2G08ABAGA / MT29F2G08ABBGA devices supports
* enabling the ECC feature but don't reflect that to the READ_ID table.
* So we have to guarantee that we disable the ECC feature directly
* after we did the READ_ID table command. Later we can evaluate the
* ECC_ENABLE support.
*/
ret = micron_nand_on_die_ecc_setup(chip, true);
if (ret)
return MICRON_ON_DIE_UNSUPPORTED;
ret = nand_readid_op(chip, 0, id, sizeof(id));
if (ret)
return MICRON_ON_DIE_UNSUPPORTED;
ret = micron_nand_on_die_ecc_setup(chip, false);
if (ret)
return MICRON_ON_DIE_UNSUPPORTED;
if (!(id[4] & MICRON_ID_ECC_ENABLED))
return MICRON_ON_DIE_UNSUPPORTED;
ret = nand_readid_op(chip, 0, id, sizeof(id));
if (ret)
return MICRON_ON_DIE_UNSUPPORTED;
if (id[4] & MICRON_ID_ECC_ENABLED)
return MICRON_ON_DIE_MANDATORY;
/*
* We only support on-die ECC of 4/512 or 8/512
*/
if (chip->base.eccreq.strength != 4 && chip->base.eccreq.strength != 8)
return MICRON_ON_DIE_UNSUPPORTED;
return MICRON_ON_DIE_SUPPORTED;
}
static int micron_nand_init(struct nand_chip *chip)
{
struct mtd_info *mtd = nand_to_mtd(chip);
struct micron_nand *micron;
int ondie;
int ret;
micron = kzalloc(sizeof(*micron), GFP_KERNEL);
if (!micron)
return -ENOMEM;
nand_set_manufacturer_data(chip, micron);
ret = micron_nand_onfi_init(chip);
if (ret)
goto err_free_manuf_data;
if (mtd->writesize == 2048)
chip->options |= NAND_BBM_FIRSTPAGE | NAND_BBM_SECONDPAGE;
ondie = micron_supports_on_die_ecc(chip);
if (ondie == MICRON_ON_DIE_MANDATORY &&
chip->ecc.mode != NAND_ECC_ON_DIE) {
pr_err("On-die ECC forcefully enabled, not supported\n");
ret = -EINVAL;
goto err_free_manuf_data;
}
if (chip->ecc.mode == NAND_ECC_ON_DIE) {
if (ondie == MICRON_ON_DIE_UNSUPPORTED) {
pr_err("On-die ECC selected but not supported\n");
ret = -EINVAL;
goto err_free_manuf_data;
}
if (ondie == MICRON_ON_DIE_MANDATORY) {
micron->ecc.forced = true;
micron->ecc.enabled = true;
}
/*
* In case of 4bit on-die ECC, we need a buffer to store a
* page dumped in raw mode so that we can compare its content
* to the same page after ECC correction happened and extract
* the real number of bitflips from this comparison.
* That's not needed for 8-bit ECC, because the status expose
* a better approximation of the number of bitflips in a page.
*/
if (chip->base.eccreq.strength == 4) {
micron->ecc.rawbuf = kmalloc(mtd->writesize +
mtd->oobsize,
GFP_KERNEL);
if (!micron->ecc.rawbuf) {
ret = -ENOMEM;
goto err_free_manuf_data;
}
}
if (chip->base.eccreq.strength == 4)
mtd_set_ooblayout(mtd,
µn_nand_on_die_4_ooblayout_ops);
else
mtd_set_ooblayout(mtd,
µn_nand_on_die_8_ooblayout_ops);
chip->ecc.bytes = chip->base.eccreq.strength * 2;
chip->ecc.size = 512;
chip->ecc.strength = chip->base.eccreq.strength;
chip->ecc.algo = NAND_ECC_BCH;
chip->ecc.read_page = micron_nand_read_page_on_die_ecc;
chip->ecc.write_page = micron_nand_write_page_on_die_ecc;
if (ondie == MICRON_ON_DIE_MANDATORY) {
chip->ecc.read_page_raw = nand_read_page_raw_notsupp;
chip->ecc.write_page_raw = nand_write_page_raw_notsupp;
} else {
chip->ecc.read_page_raw = nand_read_page_raw;
chip->ecc.write_page_raw = nand_write_page_raw;
}
}
return 0;
err_free_manuf_data:
kfree(micron->ecc.rawbuf);
kfree(micron);
return ret;
}
static void micron_nand_cleanup(struct nand_chip *chip)
{
struct micron_nand *micron = nand_get_manufacturer_data(chip);
kfree(micron->ecc.rawbuf);
kfree(micron);
}
static void micron_fixup_onfi_param_page(struct nand_chip *chip,
struct nand_onfi_params *p)
{
/*
* MT29F1G08ABAFAWP-ITE:F and possibly others report 00 00 for the
* revision number field of the ONFI parameter page. Assume ONFI
* version 1.0 if the revision number is 00 00.
*/
if (le16_to_cpu(p->revision) == 0)
p->revision = cpu_to_le16(ONFI_VERSION_1_0);
}
const struct nand_manufacturer_ops micron_nand_manuf_ops = {
.init = micron_nand_init,
.cleanup = micron_nand_cleanup,
.fixup_onfi_param_page = micron_fixup_onfi_param_page,
};