mtd: nand: omap: optimized chip->ecc.correct() for H/W ECC schemes
Subsystems (2)
v5 cover mtd: nand: omap: optimized chip->ecc.correct() for H/W ECC schemes
This patch series is split version of earlier patch:
http://lists.infradead.org/pipermail/linux-mtd/2013-November/050241.html
chip->ecc.correct() is used for detecting and correcting bit-flips during read operations. In OMAP NAND driver different ecc-schemes have different callbacks:
- omap_correct_data() for HAM1_HW ecc-schemes (Untouched)
- nand_bch_correct_data() for BCHx_HW_DETECTION_SW ecc-schemes (Untouched)
- omap_elm_correct_data() for BCHx_HW ecc-schemes
This patch-series fixes following issues in omap_elm_correct_data():
(1) Dependency on a specific reserved byte-position in OOB area to differentiates between erased-pages v/s programmed-pages.
Problem: reserved byte-position cannot be accomodated in all ecc-schemes
Problem: reserved byte-position can itself be subjected upto 8 bit-flips
causing the 0xff to become 0x00, causing page to be mis-recognized as erased-page.
(2) Bit-flips in erased-pages are detected by comparing each byte of Data & OOB with 0xff in check_erased_page().
Problem: This is causes performance penalty when erased-pages are checked.
(3) Current code is not scalable for future ECC schemes due to presence of tweaks for BCH4_ECC and BCH8_ECC at multiple places.
(4) Currently, bit-flips are evaluated and fixed even when ELM reports them as un-correctable bit-flips, this should not happen as 'number-of-error' field in ELM_LOCATION_STATUS becomes invalid when un-correctable flag is set.
(5) Driver should return with error-code = '-EBADMSG' when uncorrectable bit-flip is detected
bit-flip outside valid Data and OOB region is detected
Pekon Gupta (5):
mtd: nand: omap: add field to indicate current ecc-scheme in 'struct
omap_nand_info'
mtd: nand: omap: ecc.correct: omap_elm_correct_data: rename eccsize
and ecc_vector_size to avoid confusion
mtd: nand: omap: ecc.correct: omap_elm_correct_data: fix erased-pages
and bit-flips detection for H/W ECC schemes
mtd: nand: omap: ecc.correct: omap_elm_correct_data: cleanup to make
it scalable for future ecc-schemes
mtd: nand: omap: ecc.correct: omap_elm_correct_data: fix bit-flip
correction logic
drivers/mtd/nand/omap2.c | 248 +++++++++++++++++++---------------------------- 1 file changed, 100 insertions(+), 148 deletions(-)
Signature · 1 line
1.8.1
PATCH v5 1/5 mtd: nand: omap: add field to indicate current ecc-scheme in 'struct omap_nand_info'
Information of currently selected ECC scheme 'enum omap_ecc ecc_opt' should available outside platform-data, so that single nand_chip->ecc callback can support multiple ecc-scheme configurations.
--- drivers/mtd/nand/omap2.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
drivers/mtd/nand/omap2.c +3 -1
@@ -160,6 +160,7 @@ struct omap_nand_info {int gpmc_cs;unsigned long phys_base;unsigned long mem_size; +enum omap_ecc ecc_opt;struct completion comp;struct dma_chan *dma;int gpmc_irq_fifo;
@@ -1657,6 +1658,7 @@ static int omap_nand_probe(struct platform_device *pdev) info->gpmc_cs = pdata->cs; info->reg = pdata->reg; info->of_node = pdata->of_node; + info->ecc_opt = pdata->ecc_opt; mtd = &info->mtd; mtd->priv = &info->nand; mtd->name =dev_name (&pdev->dev);
@@ -1818,7 +1820,7 @@ static int omap_nand_probe(struct platform_device *pdev)/* populate MTD interface based on ECC scheme */ nand_chip->ecc.layout = &omap_oobinfo; ecclayout = &omap_oobinfo; -switch (pdata->ecc_opt) { +switch (info->ecc_opt) {case OMAP_ECC_HAM1_CODE_HW :pr_info ("nand: using OMAP_ECC_HAM1_CODE_HW\n" ); nand_chip->ecc.mode =NAND_ECC_HW ;
Signature · 1 line
1.8.1
PATCH v5 2/5 mtd: nand: omap: ecc.correct: omap_elm_correct_data: rename eccsize and ecc_vector_size to avoid confusion
In reply to Pekon Guptarenaming following variables as they cause confusion due to resemblence to another similar field in 'struct nand_ecc_ctrl' (nand_chip->ecc.size).
ecc_vector_size = info->nand.ecc.bytes ---> eccbytes = info->nand.ecc.bytes;
eccsize = ecc_vector_size - 1; ---> eccbytes - 1;
--- drivers/mtd/nand/omap2.c | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-)
drivers/mtd/nand/omap2.c +11 -16
@@ -1359,9 +1359,10 @@ static int omap_elm_correct_data(struct mtd_info *mtd, u_char *data, { struct omap_nand_info *info = container_of(mtd, struct omap_nand_info, mtd); +int eccbytes = info->nand.ecc.bytes;int eccsteps = info->nand.ecc.steps;int i , j, stat =0 ; -int eccsize, eccflag, ecc_vector_size; +int eccflag;struct elm_errorvec err_vec[ERROR_VECTOR_MAX ];u_char *ecc_vec = calc_ecc;u_char *spare_ecc = read_ecc;
@@ -1380,23 +1381,16 @@ static int omap_elm_correct_data(struct mtd_info *mtd, u_char *data, erased_ecc_vec = bch4_vector; } - ecc_vector_size = info->nand.ecc.bytes; - -/* -* Remove extra byte padding for BCH8 RBL -* compatibility and erased page handling -*/ - eccsize = ecc_vector_size -1 ; -for (i =0 ; i < eccsteps ; i++) { eccflag =0 ;/* initialize eccflag */ /* * Check any error reported, * In case of error, non zero ecc reported. +* - BCH4_HW: omit 7th ECC byte reserved as 0x0 +* - BCH8_HW: omit 14th ECC byte reserved as 0x0 */ - -for (j =0 ; (j < eccsize); j++) { +for (j =0 ; j < (eccbytes -1 ); j++) {if (calc_ecc[j] !=0 ) { eccflag =1 ;/* non zero ecc, error present */ break ;
@@ -1421,7 +1415,7 @@ static int omap_elm_correct_data(struct mtd_info *mtd, u_char *data,* zeros are more than threshold erased page, either * case page reported as uncorrectable. */ -if (hweight8 (~read_ecc[eccsize]) >= threshold) { +if (hweight8 (~read_ecc[eccbytes -1 ]) >= threshold) {/* * Update elm error vector as * data area is programmed
@@ -1433,7 +1427,8 @@ static int omap_elm_correct_data(struct mtd_info *mtd, u_char *data,int bitflip_count;u_char *buf = &data[info->nand.ecc.size * i]; -if (memcmp (calc_ecc, erased_ecc_vec, eccsize)) { +if (memcmp (calc_ecc, erased_ecc_vec, + (eccbytes -1 ))) { bitflip_count =erased_sector_bitflips ( buf, read_ecc, info);
@@ -1446,8 +1441,8 @@ static int omap_elm_correct_data(struct mtd_info *mtd, u_char *data, }/* Update the ecc vector */ - calc_ecc += ecc_vector_size; - read_ecc += ecc_vector_size; + calc_ecc += eccbytes; + read_ecc += eccbytes; }/* Check if any error reported */
@@ -1496,7 +1491,7 @@ static int omap_elm_correct_data(struct mtd_info *mtd, u_char *data,/* Update page data with sector size */ data += info->nand.ecc.size; - spare_ecc += ecc_vector_size; + spare_ecc += eccbytes; }for (i =0 ; i < eccsteps; i++)
Signature · 1 line
1.8.1
PATCH v5 3/5 mtd: nand: omap: ecc.correct: omap_elm_correct_data: fix erased-pages and bit-flips detection for H/W ECC schemes
In reply to Pekon Guptachip->ecc.correct() is used for detecting and correcting bit-flips during read operations. In OMAP NAND driver different ecc-schemes have different callbacks:
- omap_correct_data() for HAM1_HW ecc-schemes (Untouched)
- nand_bch_correct_data() for BCHx_HW_DETECTION_SW ecc-schemes (Untouched)
- omap_elm_correct_data() for BCHx_HW ecc-schemes
Current implementation of omap_elm_correct_data() has some issues:
a) It depends on a specific byte-position in OOB area (reserved as 0x00) to differentiate between programmed-pages v/s erased-pages.
Problem-1: reserved byte-position cannot be accomodated in all ecc-schemes
Problem-2: reserved byte-position can itself be subjected upto 8 bit-flips
causing the 0xff to become 0x00, causing page to be mis-recognized as erased-page.
b) Bit-flips in erased-pages are detected by comparing each byte of Data & OOB with 0xff in check_erased_page().
Problem-3: This is causes performance penalty when erased-pages are checked.
This patch optimizes omap_elm_correct_data() in following ways:
- Removes dependency on specific reserved-byte (0x00) in OOB area instead identifies erased-pages based on (1).
- Removes check_erased_page() instead bit-flips in erased-page are detected based on (2)
- If bit-flips are detected in erased-page then corrective action is taken based on (3), (4) and (5).
(1) A page can be assumed 'erased' only if its OOB == all(0xff).
- However an abrupt termination of page programming (e.g. power failure) may result in partial write, corrupting only main area. But such data should not be trusted so NAND driver should indicate upper-layers to re-erase such page.
(2) Instead of comparing each byte of main-area with 0xff, just compare
calc_ecc[] with known ECC-syndromp-of-all(0xff). This confirms that
both main-area + OOB are all(0xff). This will save CPU cycles.(3) Bit-flips in erased-page can't be corrected because there is no ECC present in OOB
(4) Irrespective of number of bit-flips an erased-page will only contain all(0xff),So its safe to return error-code -EUCLEAN with data_buf=0xff, instead of -EBADMSG
(5) And incrementing ecc_stat->corrected makes chip->read_page() return -EUCLEAN
--- drivers/mtd/nand/omap2.c | 159 ++++++++++++++++++----------------------------- 1 file changed, 59 insertions(+), 100 deletions(-)
drivers/mtd/nand/omap2.c +59 -100
@@ -1292,81 +1292,31 @@ static int omap3_calculate_ecc_bch(struct mtd_info *mtd, const u_char *dat, }/** -* erased_sector_bitflips - count bit flips -* @data: data sector buffer -* @oob: oob buffer -* @info: omap_nand_info -* -* Check the bit flips in erased page falls below correctable level. -* If falls below, report the page as erased with correctable bit -* flip, else report as uncorrectable page. -*/ -static int erased_sector_bitflips (u_char *data,u_char *oob, -struct omap_nand_info *info) -{ -int flip_bits =0 , i; - -for (i =0 ; i < info->nand.ecc.size; i++) { - flip_bits +=hweight8 (~data[i]); -if (flip_bits > info->nand.ecc.strength) -return 0 ; - } - -for (i =0 ; i < info->nand.ecc.bytes -1 ; i++) { - flip_bits +=hweight8 (~oob[i]); -if (flip_bits > info->nand.ecc.strength) -return 0 ; - } - -/* -* Bit flips falls in correctable level. -* Fill data area with 0xFF -*/ -if (flip_bits) { -memset (data,0xFF , info->nand.ecc.size); -memset (oob,0xFF , info->nand.ecc.bytes); - } - -return flip_bits; -} - -/** * omap_elm_correct_data - corrects page data area in case error reported * @mtd: MTD device structure * @data: page data * @read_ecc: ecc read from nand flash -* @calc_ecc: ecc read from HW ECC registers -* -* Calculated ecc vector reported as zero in case of non-error pages. -* In case of error/erased pages non-zero error vector is reported. -* In case of non-zero ecc vector, check read_ecc at fixed offset -* (x = 13/7 in case of BCH8/4 == 0) to find page programmed or not. -* To handle bit flips in this data, count the number of 0's in -* read_ecc[x] and check if it greater than 4. If it is less, it is -* programmed page, else erased page. -* -* 1. If page is erased, check with standard ecc vector (ecc vector -* for erased page to find any bit flip). If check fails, bit flip -* is present in erased page. Count the bit flips in erased page and -* if it falls under correctable level, report page with 0xFF and -* update the correctable bit information. -* 2. If error is reported on programmed page, update elm error -* vector and correct the page with ELM error correction routine. -* +* @calc_ecc: ecc calculated after reading Data and OOB regions from flash +* calc_ecc would be non-zero only in following cases: +* - bit-flips in data or oob region +* - erased page, where no ECC is written in OOB area */ static int omap_elm_correct_data (struct mtd_info *mtd,u_char *data,u_char *read_ecc,u_char *calc_ecc) { struct omap_nand_info *info = container_of(mtd, struct omap_nand_info, mtd); +enum omap_ecc ecc_opt = info->ecc_opt; +int eccsize = info->nand.ecc.size;int eccbytes = info->nand.ecc.bytes; -int eccsteps = info->nand.ecc.steps; +int eccstrength = info->nand.ecc.strength; +int eccsteps = info->nand.ecc.steps; +bool page_is_erased;int i , j, stat =0 ;int eccflag;struct elm_errorvec err_vec[ERROR_VECTOR_MAX ];u_char *ecc_vec = calc_ecc;u_char *spare_ecc = read_ecc; -u_char *erased_ecc_vec;enum bch_ecc type;bool is_error_reported = false;
@@ -1375,10 +1325,8 @@ static int omap_elm_correct_data(struct mtd_info *mtd, u_char *data,if (info->nand.ecc.strength ==BCH8_MAX_ERROR ) { type =BCH8_ECC ; - erased_ecc_vec = bch8_vector; }else { type =BCH4_ECC ; - erased_ecc_vec = bch4_vector; }for (i =0 ; i < eccsteps ; i++) {
@@ -1397,46 +1345,57 @@ static int omap_elm_correct_data(struct mtd_info *mtd, u_char *data, } } -if (eccflag ==1 ) { -/* -* Set threshold to minimum of 4, half of ecc.strength/2 -* to allow max bit flip in byte to 4 -*/ - unsigned int threshold = min_t(unsigned int, 4, - info->nand.ecc.strength /2 ); - -/* -* Check data area is programmed by counting -* number of 0's at fixed offset in spare area. -* Checking count of 0's against threshold. -* In case programmed page expects at least threshold -* zeros in byte. -* If zeros are less than threshold for programmed page/ -* zeros are more than threshold erased page, either -* case page reported as uncorrectable. -*/ -if (hweight8 (~read_ecc[eccbytes -1 ]) >= threshold) { -/* -* Update elm error vector as -* data area is programmed -*/ - err_vec[i].error_reported = true; - is_error_reported = true; - }else { -/* Error reported in erased page */ -int bitflip_count; -u_char *buf = &data[info->nand.ecc.size * i]; - -if (memcmp (calc_ecc, erased_ecc_vec, - (eccbytes -1 ))) { - bitflip_count =erased_sector_bitflips ( - buf, read_ecc, info); - -if (bitflip_count) - stat += bitflip_count; -else -return -EINVAL ; +/* A page can be assumed 'erased' only if read_ecc == all(0xff). +* However an abrupt termination of page programming +* (e.g. power failure) may result in partial write, corrupting +* only main area. But such data should not be trusted so NAND +* driver should indicate upper-layers to re-erase the page */ +if (eccflag) { +/* check if OOB is all(0xff) */ + page_is_erased = true; +for (j =0 ; j < (eccbytes -1 ) && page_is_erased; j++) { +if (read_ecc[j] !=0xff ) + page_is_erased = false; + } +if (page_is_erased) { +/* check bit-flips in erased-pages */ +/* - Instead of comparing each byte of main-area +* with 0xff, comparing just calc_ecc[] with +* known ECC-syndrome-of-all(0xff). This +* confirms that main-area + OOB are all(0xff) +* This will save CPU cycles. +* - Bit-flips in erased-page can't be corrected +* because there is no ECC present in OOB +* - Irrespective of number of bit-flips an +* erased-page will only contain all(0xff), +* So its safe to return error-code -EUCLEAN +* with data_buf=0xff, instead of -EBADMSG +* - And incrementing ecc_stat->corrected makes +* chip->read_page() return -EUCLEAN */ +switch (ecc_opt) { +case OMAP_ECC_BCH4_CODE_HW : +/* omit 7th ECC byte reserved as 0x0 */ +if (memcmp (calc_ecc, bch4_vector, + eccbytes -1 )) + stat += eccstrength; +break ; +case OMAP_ECC_BCH8_CODE_HW : +/* omit 14th ECC byte reserved as 0x0 */ +if (memcmp (calc_ecc, bch8_vector, + eccbytes -1 )) + stat += eccstrength; +break ; +default : +return -EINVAL ; } +memset (&data[i * eccsize],0xff , eccsize); + }else { +/* OOB != 0xff means either +* - page is programmed OR +* - OOB has bit-flips +* in either cases go fix the bit-flips */ + is_error_reported = true; + err_vec[i].error_reported = true; } }
Signature · 1 line
1.8.1
PATCH v5 4/5 mtd: nand: omap: ecc.correct: omap_elm_correct_data: cleanup to make it scalable for future ecc-schemes
In reply to Pekon GuptaCurrent omap_elm_correct_data() code is not scalable for future ecc-schemes due to presence of tweaks and hard-coded macros for BCH4_ECC and BCH8_ECC ecc-schemes at multiple places.
This patch:
- replaces 'ecc_opt' with '(info->nand.ecc.strength == BCH8_MAX_ERROR) used to differentiate between BCH8_HW and BCH4_SW
- replaces macros (defining magic number for specific ecc-scheme) with generic variables
- removes dependency on macros defined in elm.h (like BCHx_ECC_OOB_BYTES)
--- drivers/mtd/nand/omap2.c | 43 ++++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 25 deletions(-)
drivers/mtd/nand/omap2.c +18 -25
@@ -118,14 +118,9 @@#define OMAP24XX_DMA_GPMC 4 -#define BCH8_MAX_ERROR 8/* upto 8 bit correctable */ -#define BCH4_MAX_ERROR 4/* upto 4 bit correctable */ -#define SECTOR_BYTES 512/* 4 bit padding to make byte aligned, 56 = 52 + 4 */ #define BCH4_BIT_PAD 4 -#define BCH8_ECC_MAX ((SECTOR_BYTES + BCH8_ECC_OOB_BYTES) * 8) -#define BCH4_ECC_MAX ((SECTOR_BYTES + BCH4_ECC_OOB_BYTES) * 8)/* GPMC ecc engine settings for read */ #define BCH_WRAPMODE_1 1/* BCH wrap mode 1 */
@@ -1317,18 +1312,12 @@ static int omap_elm_correct_data(struct mtd_info *mtd, u_char *data,struct elm_errorvec err_vec[ERROR_VECTOR_MAX ];u_char *ecc_vec = calc_ecc;u_char *spare_ecc = read_ecc; -enum bch_ecc type;bool is_error_reported = false; +u32 bit_pos, byte_pos, error_max, pos;/* Initialize elm error vector to zero */ memset (err_vec,0 ,sizeof (err_vec)); -if (info->nand.ecc.strength ==BCH8_MAX_ERROR ) { - type =BCH8_ECC ; - }else { - type =BCH4_ECC ; - } -for (i =0 ; i < eccsteps ; i++) { eccflag =0 ;/* initialize eccflag */
@@ -1414,20 +1403,24 @@ static int omap_elm_correct_data(struct mtd_info *mtd, u_char *data,for (i =0 ; i < eccsteps; i++) {if (err_vec[i].error_reported) {for (j =0 ; j < err_vec[i].error_count; j++) { -u32 bit_pos, byte_pos, error_max, pos; - -if (type ==BCH8_ECC ) - error_max =BCH8_ECC_MAX ; -else - error_max =BCH4_ECC_MAX ; - -if (info->nand.ecc.strength ==BCH8_MAX_ERROR ) - pos = err_vec[i].error_loc[j]; -else -/* Add 4 to take care 4 bit padding */ +switch (ecc_opt) { +case OMAP_ECC_BCH4_CODE_HW : +/* omit 7th ECC byte reserved as 0x0 */ + error_max = + (eccsize + (eccbytes -1 )) *8 ; +/* Add 4 bits to take care of padding */ pos = err_vec[i].error_loc[j] +BCH4_BIT_PAD ; - +break ; +case OMAP_ECC_BCH8_CODE_HW : +/* omit 14th ECC byte reserved as 0x0 */ + error_max = + (eccsize + (eccbytes -1 )) *8 ; + pos = err_vec[i].error_loc[j]; +break ; +default : +return -EINVAL ; + }/* Calculate bit position of error */ bit_pos = pos %8 ;
@@ -1449,7 +1442,7 @@ static int omap_elm_correct_data(struct mtd_info *mtd, u_char *data, stat += err_vec[i].error_count;/* Update page data with sector size */ - data += info->nand.ecc.size; + data += eccsize; spare_ecc += eccbytes; }
Signature · 1 line
1.8.1
PATCH v5 5/5 mtd: nand: omap: ecc.correct: omap_elm_correct_data: fix bit-flip correction logic
In reply to Pekon GuptaThis patch updates following checks when bit-flips are detected by ELM:
- Do not evaluate bit-flips when un-correctable bit-flips is reported by ELM, because as per [1] when ELM reports an un-correctable bit-flips, 'number of error' field in its ELM_LOCATION_STATUS register is also invalid.
- Return with error-code '-EBADMSG' on detection of un-correctable bit-flip.
- Return with error-code '-EBADMSG' when bit-flips position is outside current Sector and OOB area.
[1] ELM IP spec Table-25 ELM_LOCATION_STATUS Register. ELM_LOCATION_STATUS[8] = ECC_CORRECTABLE: Error location process exit status 0x0: ECC error location process failed.
Number of errors and error locations are invalid.
0x1: all errors were successfully located.
Number of errors and error locations are valid.
--- drivers/mtd/nand/omap2.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-)
drivers/mtd/nand/omap2.c +12 -9
@@ -1307,7 +1307,7 @@ static int omap_elm_correct_data(struct mtd_info *mtd, u_char *data,int eccstrength = info->nand.ecc.strength;int eccsteps = info->nand.ecc.steps;bool page_is_erased; -int i , j, stat =0 ; +int i , j, stat =0 , err =0 ;int eccflag;struct elm_errorvec err_vec[ERROR_VECTOR_MAX ];u_char *ecc_vec = calc_ecc;
@@ -1401,7 +1401,10 @@ static int omap_elm_correct_data(struct mtd_info *mtd, u_char *data,elm_decode_bch_error_page (info->elm_dev, ecc_vec, err_vec);for (i =0 ; i < eccsteps; i++) { -if (err_vec[i].error_reported) { +if (err_vec[i].error_uncorrectable) { +pr_err ("nand: uncorrectable bit-flips found\n" ); + err = -EBADMSG ; + }else if (err_vec[i].error_reported) {for (j =0 ; j < err_vec[i].error_count; j++) {switch (ecc_opt) {case OMAP_ECC_BCH4_CODE_HW :
@@ -1428,13 +1431,18 @@ static int omap_elm_correct_data(struct mtd_info *mtd, u_char *data, byte_pos = (error_max - pos -1 ) /8 ;if (pos < error_max) { +pr_debug ("fixed bit-flip @ [%d][%d]\n" , + byte_pos, bit_pos);if (byte_pos <512 ) data[byte_pos] ^=1 << bit_pos;else spare_ecc[byte_pos -512 ] ^=1 << bit_pos; + }else { +pr_err ("invalid bit-flip @ [%d][%d]\n" , + byte_pos, bit_pos); + err = -EBADMSG ; } -/* else, not interested to correct ecc */ } }
@@ -1446,12 +1454,7 @@ static int omap_elm_correct_data(struct mtd_info *mtd, u_char *data, spare_ecc += eccbytes; } -for (i =0 ; i < eccsteps; i++) -/* Return error if uncorrectable error present */ -if (err_vec[i].error_uncorrectable) -return -EINVAL ; - -return stat; +return (err) ? err : stat; }/**
Signature · 1 line
1.8.1