DAX poison recovery
Subsystems (9)
v3 cover DAX poison recovery
In v3, dax recovery code path is independent of that of normal write. Competing dax recovery threads are serialized, racing read threads are guaranteed not overlapping with the recovery process.
In this phase, the recovery granularity is page, future patch will explore recovery in finer granularity.
Please refer to below discussions for more information:
v2:
https://lore.kernel.org/all/[email protected]/
Disussions about marking poisoned page as 'np':
https://lore.kernel.org/all/CAPcyv4hrXPb1tASBZUg-GgdVs0OOFKXMXLiHmktg_kFi7YBMyQ@mail.gmail.com/
Jane Chu (7):
mce: fix set_mce_nospec to always unmap the whole page
dax: introduce dax device flag DAXDEV_RECOVERY
dm: make dm aware of target's DAXDEV_RECOVERY capability
dax: add dax_recovery_write to dax_op and dm target type
pmem: add pmem_recovery_write() dax op
dax: add recovery_write to dax_iomap_iter in failure path
pmem: fix pmem_do_write() avoid writing to 'np' page
arch/x86/include/asm/set_memory.h | 17 ++--- arch/x86/kernel/cpu/mce/core.c | 6 +- arch/x86/mm/pat/set_memory.c | 8 ++- drivers/dax/super.c | 53 ++++++++++++++++ drivers/md/dm-linear.c | 13 ++++ drivers/md/dm-log-writes.c | 14 +++++ drivers/md/dm-stripe.c | 13 ++++ drivers/md/dm-table.c | 33 ++++++++++ drivers/md/dm.c | 27 ++++++++ drivers/nvdimm/pmem.c | 101 +++++++++++++++++++++++++++--- drivers/nvdimm/pmem.h | 1 + fs/dax.c | 25 +++++++- include/linux/dax.h | 9 +++ include/linux/device-mapper.h | 3 + include/linux/set_memory.h | 2 +- 15 files changed, 298 insertions(+), 27 deletions(-)
Signature · 1 line
2.18.4
PATCH v3 1/7 mce: fix set_mce_nospec to always unmap the whole page
Mark poisoned page as not present, and to reverse the 'np' effect, restate the _PAGE_PRESENT bit. Please refer to discussions here for reason behind the decision.
https://lore.kernel.org/all/CAPcyv4hrXPb1tASBZUg-GgdVs0OOFKXMXLiHmktg_kFi7YBMyQ@mail.gmail.com/
--- arch/x86/include/asm/set_memory.h | 17 +++++------------ arch/x86/kernel/cpu/mce/core.c | 6 +++--- arch/x86/mm/pat/set_memory.c | 8 +++++++- include/linux/set_memory.h | 2 +- 4 files changed, 16 insertions(+), 17 deletions(-)
arch/x86/include/asm/set_memory.h +5 -12
@@ -49,6 +49,7 @@ int set_memory_decrypted(unsigned long addr, int numpages);int set_memory_np_noalias (unsigned long addr,int numpages);int set_memory_nonglobal (unsigned long addr,int numpages);int set_memory_global (unsigned long addr,int numpages); +int _set_memory_present (unsigned long addr,int numpages);int set_pages_array_uc (struct page **pages,int addrinarray);int set_pages_array_wc (struct page **pages,int addrinarray);
@@ -88,13 +89,8 @@ void notify_range_enc_status_changed(unsigned long vaddr, int npages, bool enc);extern int kernel_set_to_readonly;#ifdef CONFIG_X86_64 -/* -* Prevent speculative access to the page by either unmapping -* it (if we do not require access to any part of the page) or -* marking it uncacheable (if we want to try to retrieve data -* from non-poisoned lines in the page). -*/ -static inline int set_mce_nospec (unsigned long pfn,bool unmap) +/* Prevent speculative access to a page by marking it not-present */ +static inline int set_mce_nospec (unsigned long pfn) {unsigned long decoy_addr;int rc;
@@ -113,10 +109,7 @@ static inline int set_mce_nospec(unsigned long pfn, bool unmap)*/ decoy_addr = (pfn <<PAGE_SHIFT ) + (PAGE_OFFSET ^BIT (63 )); -if (unmap) - rc =set_memory_np (decoy_addr,1 ); -else - rc =set_memory_uc (decoy_addr,1 ); + rc =set_memory_np (decoy_addr,1 );if (rc)pr_warn ("Could not invalidate pfn=0x%lx from 1:1 map\n" , pfn);return rc;
@@ -126,7 +119,7 @@ static inline int set_mce_nospec(unsigned long pfn, bool unmap)/* Restore full speculative operation to the pfn. */ static inline int clear_mce_nospec (unsigned long pfn) { -return set_memory_wb ((unsigned long )pfn_to_kaddr (pfn),1 ); +return _set_memory_present ((unsigned long )pfn_to_kaddr (pfn),1 ); }#define clear_mce_nospec clear_mce_nospec#else
arch/x86/kernel/cpu/mce/core.c +3 -3
@@ -601,7 +601,7 @@ static int uc_decode_notifier(struct notifier_block *nb, unsigned long val, pfn = mce->addr >>PAGE_SHIFT ;if (!memory_failure (pfn,0 )) { -set_mce_nospec (pfn,whole_page (mce)); +set_mce_nospec (pfn); mce->kflags |=MCE_HANDLED_UC ; }
@@ -1255,7 +1255,7 @@ static void kill_me_maybe(struct callback_head *cb) ret =memory_failure (p->mce_addr >>PAGE_SHIFT , flags);if (!ret) { -set_mce_nospec (p->mce_addr >>PAGE_SHIFT , p->mce_whole_page); +set_mce_nospec (p->mce_addr >>PAGE_SHIFT );sync_core ();return ; }
@@ -1279,7 +1279,7 @@ static void kill_me_never(struct callback_head *cb) p->mce_count =0 ;pr_err ("Kernel accessed poison in user space at %llx\n" , p->mce_addr);if (!memory_failure (p->mce_addr >>PAGE_SHIFT ,0 )) -set_mce_nospec (p->mce_addr >>PAGE_SHIFT , p->mce_whole_page); +set_mce_nospec (p->mce_addr >>PAGE_SHIFT ); }static void queue_task_work (struct mce *m,char *msg,void (*func)(struct callback_head *))
arch/x86/mm/pat/set_memory.c +7 -1
@@ -1816,7 +1816,7 @@ static inline int cpa_clear_pages_array(struct page **pages, int numpages, }/* -* _set_memory_prot is an internal helper for callers that have been passed +* __set_memory_prot is an internal helper for callers that have been passed * a pgprot_t value from upper layers and a reservation has already been taken. * If you want to set the pgprot to a specific page protocol, use the * set_memory_xx() functions.
@@ -1983,6 +1983,12 @@ int set_memory_global(unsigned long addr, int numpages) __pgprot(_PAGE_GLOBAL), 0); } +int _set_memory_present (unsigned long addr,int numpages) +{ +return change_page_attr_set (&addr, numpages,__pgprot (_PAGE_PRESENT),0 ); +} +EXPORT_SYMBOL_GPL (_set_memory_present); +/* * __set_memory_enc_pgtable() is used for the hypervisors that get * informed about "encryption" status via page tables.
include/linux/set_memory.h +1 -1
@@ -43,7 +43,7 @@ static inline bool can_set_direct_map(void)#endif /* CONFIG_ARCH_HAS_SET_DIRECT_MAP */ #ifndef set_mce_nospec -static inline int set_mce_nospec (unsigned long pfn,bool unmap) +static inline int set_mce_nospec (unsigned long pfn) {return 0 ; }
Signature · 1 line
2.18.4
PATCH v3 2/7 dax: introduce dax device flag DAXDEV_RECOVERY
In reply to Jane ChuIntroduce dax device flag DAXDEV_RECOVERY to indicate a device that is capable of recoverying from media poison. For MD raid DAX devices, the capability is allowed for partial device as oppose to the entire device. And the final poison detection and repair rely on the provisioning base drivers.
--- drivers/dax/super.c | 34 ++++++++++++++++++++++++++++++++++ drivers/nvdimm/pmem.c | 1 + include/linux/dax.h | 3 +++ 3 files changed, 38 insertions(+)
drivers/dax/super.c +34 -0
@@ -105,6 +105,8 @@ enum dax_device_flags {DAXDEV_WRITE_CACHE ,/* flag to check if device supports synchronous flush */ DAXDEV_SYNC , +/* flag to indicate device capable of poison recovery */ + DAXDEV_RECOVERY, };/**
@@ -227,6 +229,38 @@ bool dax_alive(struct dax_device *dax_dev) }EXPORT_SYMBOL_GPL (dax_alive); +/* set by driver */ +/* for md: set if any participant target has the capability */ +void set_dax_recovery (struct dax_device *dax_dev) +{ +set_bit (DAXDEV_RECOVERY , &dax_dev->flags); +} +EXPORT_SYMBOL_GPL (set_dax_recovery); + +/* query by dax, and md drivers, and .... */ +/* can be true for md raid, but untrue for a target within */ +bool dax_recovery_capable (struct dax_device *dax_dev) +{ +return test_bit (DAXDEV_RECOVERY , &dax_dev->flags); +} +EXPORT_SYMBOL_GPL (dax_recovery_capable); + +/* requested by fs */ +/* can be true for md raid, and ought to be true for the target +* because kaddr ought to fall into the target which complained +* about BB and only pmem do that. however in theory, this could +* be true for md but untrue for a target strip +*/ +int dax_prep_recovery (struct dax_device *dax_dev,void **kaddr) +{ +if (dax_recovery_capable (dax_dev)) { +set_bit (DAXDEV_RECOVERY , (unsigned long *)kaddr); +return 0 ; + } +return -EINVAL ; +} +EXPORT_SYMBOL_GPL (dax_prep_recovery); +/* * Note, rcu is not protecting the liveness of dax_dev, rcu is ensuring * that any fault handlers or operations that might have seen
drivers/nvdimm/pmem.c +1 -0
@@ -506,6 +506,7 @@ static int pmem_attach_disk(struct device *dev,if (rc) goto out_cleanup_dax;dax_write_cache (dax_dev,nvdimm_has_cache (nd_region)); +set_dax_recovery (dax_dev); pmem->dax_dev = dax_dev; rc =device_add_disk (dev, disk, pmem_attribute_groups);
include/linux/dax.h +3 -0
@@ -49,6 +49,9 @@ void kill_dax(struct dax_device *dax_dev);void dax_write_cache (struct dax_device *dax_dev,bool wc);bool dax_write_cache_enabled (struct dax_device *dax_dev);bool __dax_synchronous (struct dax_device *dax_dev); +void set_dax_recovery (struct dax_device *dax_dev); +bool dax_recovery_capable (struct dax_device *dax_dev); +int dax_prep_recovery (struct dax_device *dax_dev,void **kaddr);static inline bool dax_synchronous (struct dax_device *dax_dev) {return __dax_synchronous (dax_dev);
Signature · 1 line
2.18.4
PATCH v3 3/7 dm: make dm aware of target's DAXDEV_RECOVERY capability
In reply to Jane ChuIf one of the MD raid participating target dax device supports DAXDEV_RECOVERY, then it'll be declared on the whole that the MD device is capable of DAXDEV_RECOVERY.
And only when the recovery process reaches to the target driver, it becomes deterministic whether a certain dax address range maybe recovered, or not.
--- drivers/md/dm-table.c | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+)
drivers/md/dm-table.c +33 -0
@@ -844,6 +844,36 @@ static bool dm_table_supports_dax(struct dm_table *t,return true; } +/* Check whether device is capable of dax poison recovery */ +static int device_poison_recovery_capable (struct dm_target *ti, +struct dm_dev *dev,sector_t start,sector_t len,void *data) +{ +if (!dev->dax_dev) +return false; +return dax_recovery_capable (dev->dax_dev); +} + +static bool dm_table_supports_poison_recovery (struct dm_table *t, +iterate_devices_callout_fn func) +{ +struct dm_target *ti; +unsigned int i; + +/* Check if any DAX target supports poison recovery */ +for (i =0 ; i <dm_table_get_num_targets (t); i++) { + ti =dm_table_get_target (t, i); + +if (!ti->type->direct_access) +return false; + +if (ti->type->iterate_devices && + ti->type->iterate_devices (ti, func,NULL )) +return true; + } + +return false; +} +static int device_is_rq_stackable (struct dm_target *ti,struct dm_dev *dev,sector_t start,sector_t len,void *data) {
@@ -2014,6 +2044,9 @@ int dm_table_set_restrictions(struct dm_table *t, struct request_queue *q,blk_queue_flag_set (QUEUE_FLAG_DAX , q);if (dm_table_supports_dax (t, device_not_dax_synchronous_capable))set_dax_synchronous (t->md->dax_dev); +if (dm_table_supports_poison_recovery (t, + device_poison_recovery_capable)) +set_dax_recovery (t->md->dax_dev); }else blk_queue_flag_clear (QUEUE_FLAG_DAX , q);
Signature · 1 line
2.18.4
PATCH v3 4/7 dax: add dax_recovery_write to dax_op and dm target type
In reply to Jane Chudax_recovery_write() dax op is only required for DAX device that export DAXDEV_RECOVERY indicating its capability to recover from poisons.
DM may be nested, if part of the base dax devices forming a DM device support dax recovery, the DM device is marked with such capability.
--- drivers/dax/super.c | 19 +++++++++++++++++++ drivers/md/dm-linear.c | 13 +++++++++++++ drivers/md/dm-log-writes.c | 14 ++++++++++++++ drivers/md/dm-stripe.c | 13 +++++++++++++ drivers/md/dm.c | 27 +++++++++++++++++++++++++++ drivers/nvdimm/pmem.c | 7 +++++++ include/linux/dax.h | 6 ++++++ include/linux/device-mapper.h | 3 +++ 8 files changed, 102 insertions(+)
drivers/dax/super.c +19 -0
@@ -162,6 +162,16 @@ size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, }EXPORT_SYMBOL_GPL (dax_copy_to_iter); +size_t dax_recovery_write (struct dax_device *dax_dev,pgoff_t pgoff,void *addr, +size_t bytes,struct iov_iter *i) +{ +if (!dax_recovery_capable (dax_dev) || !dax_dev->ops->recovery_write) +return (size_t )-EOPNOTSUPP ; + +return dax_dev->ops->recovery_write (dax_dev, pgoff, addr, bytes, i); +} +EXPORT_SYMBOL_GPL (dax_recovery_write); +int dax_zero_page_range (struct dax_device *dax_dev,pgoff_t pgoff,size_t nr_pages) {
@@ -261,6 +271,15 @@ int dax_prep_recovery(struct dax_device *dax_dev, void **kaddr) }EXPORT_SYMBOL_GPL (dax_prep_recovery); +bool dax_recovery_started (struct dax_device *dax_dev,void **kaddr) +{ +if (!kaddr || !dax_recovery_capable (dax_dev)) +return false; + +return test_bit (DAXDEV_RECOVERY , (unsigned long *)kaddr); +} +EXPORT_SYMBOL_GPL (dax_recovery_started); +/* * Note, rcu is not protecting the liveness of dax_dev, rcu is ensuring * that any fault handlers or operations that might have seen
drivers/md/dm-linear.c +13 -0
@@ -204,11 +204,23 @@ static int linear_dax_zero_page_range(struct dm_target *ti, pgoff_t pgoff,return dax_zero_page_range (dax_dev, pgoff, nr_pages); } +static size_t linear_dax_recovery_write (struct dm_target *ti,pgoff_t pgoff, +void *addr,size_t bytes,struct iov_iter *i) +{ +struct dax_device *dax_dev =linear_dax_pgoff (ti, &pgoff); + +if (!dax_recovery_capable (dax_dev)) +return (size_t ) -EOPNOTSUPP ; + +return dax_recovery_write (dax_dev, pgoff, addr, bytes, i); +} +#else #define linear_dax_direct_access NULL#define linear_dax_copy_from_iter NULL#define linear_dax_copy_to_iter NULL#define linear_dax_zero_page_range NULL +#define linear_dax_recovery_write NULL#endif static struct target_type linear_target = {
@@ -228,6 +240,7 @@ static struct target_type linear_target = { .dax_copy_from_iter = linear_dax_copy_from_iter, .dax_copy_to_iter = linear_dax_copy_to_iter, .dax_zero_page_range = linear_dax_zero_page_range, + .dax_recovery_write = linear_dax_recovery_write, };int __init dm_linear_init (void )
drivers/md/dm-log-writes.c +14 -0
@@ -1003,11 +1003,24 @@ static int log_writes_dax_zero_page_range(struct dm_target *ti, pgoff_t pgoff,return dax_zero_page_range (dax_dev, pgoff, nr_pages <<PAGE_SHIFT ); } +static size_t log_writes_dax_recovery_write (struct dm_target *ti, +pgoff_t pgoff,void *addr,size_t bytes, +struct iov_iter *i) +{ +struct dax_device *dax_dev =log_writes_dax_pgoff (ti, &pgoff); + +if (!dax_recovery_capable (dax_dev)) +return (size_t ) -EOPNOTSUPP ; + +return dax_recovery_write (dax_dev, pgoff, addr, bytes, i); +} +#else #define log_writes_dax_direct_access NULL#define log_writes_dax_copy_from_iter NULL#define log_writes_dax_copy_to_iter NULL#define log_writes_dax_zero_page_range NULL +#define log_writes_dax_recovery_write NULL#endif static struct target_type log_writes_target = {
@@ -1027,6 +1040,7 @@ static struct target_type log_writes_target = { .dax_copy_from_iter = log_writes_dax_copy_from_iter, .dax_copy_to_iter = log_writes_dax_copy_to_iter, .dax_zero_page_range = log_writes_dax_zero_page_range, + .dax_recovery_write = log_writes_dax_recovery_write, };static int __init dm_log_writes_init (void )
drivers/md/dm-stripe.c +13 -0
@@ -348,11 +348,23 @@ static int stripe_dax_zero_page_range(struct dm_target *ti, pgoff_t pgoff, return dax_zero_page_range(dax_dev, pgoff, nr_pages); } +static size_t stripe_dax_recovery_write (struct dm_target *ti,pgoff_t pgoff, +void *addr,size_t bytes,struct iov_iter *i) +{ +struct dax_device *dax_dev =stripe_dax_pgoff (ti, &pgoff); + +if (!dax_recovery_capable (dax_dev)) +return (size_t ) -EOPNOTSUPP ; + +return dax_recovery_write (dax_dev, pgoff, addr, bytes, i); +} +#else #define stripe_dax_direct_access NULL#define stripe_dax_copy_from_iter NULL#define stripe_dax_copy_to_iter NULL#define stripe_dax_zero_page_range NULL +#define stripe_dax_recovery_write NULL#endif /*
@@ -491,6 +503,7 @@ static struct target_type stripe_target = { .dax_copy_from_iter = stripe_dax_copy_from_iter, .dax_copy_to_iter = stripe_dax_copy_to_iter, .dax_zero_page_range = stripe_dax_zero_page_range, + .dax_recovery_write = stripe_dax_recovery_write, };int __init dm_stripe_init (void )
drivers/md/dm.c +27 -0
@@ -1102,6 +1102,32 @@ static int dm_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,return ret; } +static size_t dm_dax_recovery_write (struct dax_device *dax_dev,pgoff_t pgoff, +void *addr,size_t bytes,struct iov_iter *i) +{ +struct mapped_device *md =dax_get_private (dax_dev); +sector_t sector = pgoff *PAGE_SECTORS ; +struct dm_target *ti; +long ret =0 ; +int srcu_idx; + + ti =dm_dax_get_live_target (md, sector, &srcu_idx); + +if (!ti) + goto out; + +if (!ti->type->dax_recovery_write) { + ret = (size_t )-EOPNOTSUPP ; + goto out; + } + + ret = ti->type->dax_recovery_write (ti, pgoff, addr, bytes, i); + out: +dm_put_live_table (md, srcu_idx); + +return ret; +} +/* * A target may call dm_accept_partial_bio only from the map routine. It is * allowed for all bio types except REQ_PREFLUSH, REQ_OP_ZONE_* zone management
@@ -3027,6 +3053,7 @@ static const struct dax_operations dm_dax_ops = { .copy_from_iter = dm_dax_copy_from_iter, .copy_to_iter = dm_dax_copy_to_iter, .zero_page_range = dm_dax_zero_page_range, + .recovery_write = dm_dax_recovery_write, };/*
drivers/nvdimm/pmem.c +7 -0
@@ -319,11 +319,18 @@ static size_t pmem_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff,return _copy_mc_to_iter (addr, bytes, i); } +static size_t pmem_recovery_write (struct dax_device *dax_dev,pgoff_t pgoff, +void *addr,size_t bytes,struct iov_iter *i) +{ +return 0 ; +} +static const struct dax_operations pmem_dax_ops = { .direct_access = pmem_dax_direct_access, .copy_from_iter = pmem_copy_from_iter, .copy_to_iter = pmem_copy_to_iter, .zero_page_range = pmem_dax_zero_page_range, + .recovery_write = pmem_recovery_write, }; static ssize_t write_cache_show(struct device *dev,
include/linux/dax.h +6 -0
@@ -39,6 +39,9 @@ struct dax_operations { struct iov_iter *);/* zero_page_range: required operation. Zero page range */ int (*zero_page_range)(struct dax_device *,pgoff_t ,size_t ); +/* recovery_write: optional operation. */ +size_t (*recovery_write)(struct dax_device *,pgoff_t ,void *,size_t , +struct iov_iter *); };#if IS_ENABLED (CONFIG_DAX )
@@ -52,6 +55,7 @@ bool __dax_synchronous(struct dax_device *dax_dev);void set_dax_recovery (struct dax_device *dax_dev);bool dax_recovery_capable (struct dax_device *dax_dev);int dax_prep_recovery (struct dax_device *dax_dev,void **kaddr); +bool dax_recovery_started (struct dax_device *dax_dev,void **kaddr);static inline bool dax_synchronous (struct dax_device *dax_dev) {return __dax_synchronous (dax_dev);
@@ -203,6 +207,8 @@ size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i);int dax_zero_page_range (struct dax_device *dax_dev,pgoff_t pgoff,size_t nr_pages); +size_t dax_recovery_write (struct dax_device *dax_dev,pgoff_t pgoff,void *addr, +size_t bytes,struct iov_iter *i);void dax_flush (struct dax_device *dax_dev,void *addr,size_t size); ssize_t dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
include/linux/device-mapper.h +3 -0
@@ -151,6 +151,8 @@ typedef size_t (*dm_dax_copy_iter_fn)(struct dm_target *ti, pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i);typedef int (*dm_dax_zero_page_range_fn )(struct dm_target *ti,pgoff_t pgoff,size_t nr_pages); +typedef size_t (*dm_dax_recovery_write_fn )(struct dm_target *ti,pgoff_t pgoff, +void *addr,size_t bytes,struct iov_iter *i);void dm_error (const char *message);
@@ -203,6 +205,7 @@ struct target_type {dm_dax_copy_iter_fn dax_copy_from_iter;dm_dax_copy_iter_fn dax_copy_to_iter;dm_dax_zero_page_range_fn dax_zero_page_range; +dm_dax_recovery_write_fn dax_recovery_write;/* For internal device-mapper use. */ struct list_head list;
Signature · 1 line
2.18.4
PATCH v3 5/7 pmem: add pmem_recovery_write() dax op
In reply to Jane Chupmem_recovery_write() consists of clearing poison via DSM, clearing page HWPoison bit, re-state _PAGE_PRESENT bit, cacheflush, write, and finally clearing bad-block record.
A competing pread thread is held off during recovery write by the presence of bad-block record. A competing recovery_write thread is serialized by a lock.
--- drivers/nvdimm/pmem.c | 84 +++++++++++++++++++++++++++++++++++++++---- drivers/nvdimm/pmem.h | 1 + 2 files changed, 79 insertions(+), 6 deletions(-)
drivers/nvdimm/pmem.c +78 -6
@@ -69,6 +69,14 @@ static void hwpoison_clear(struct pmem_device *pmem, } } +static void pmem_clear_badblocks (struct pmem_device *pmem,sector_t sector, +long cleared_blks) +{ +badblocks_clear (&pmem->bb, sector, cleared_blks); +if (pmem->bb_state) +sysfs_notify_dirent (pmem->bb_state); +} +static blk_status_t pmem_clear_poison (struct pmem_device *pmem,phys_addr_t offset,unsigned int len) {
@@ -88,9 +96,7 @@ static blk_status_t pmem_clear_poison(struct pmem_device *pmem,dev_dbg (dev,"%#llx clear %ld sector%s\n" , (unsigned long long ) sector, cleared, cleared >1 ?"s" :"" ); -badblocks_clear (&pmem->bb, sector, cleared); -if (pmem->bb_state) -sysfs_notify_dirent (pmem->bb_state); +pmem_clear_badblocks (pmem, sector, cleared); }arch_invalidate_pmem (pmem->virt_addr + offset, len);
@@ -257,10 +263,15 @@ static int pmem_rw_page(struct block_device *bdev, sector_t sector,__weak long __pmem_direct_access (struct pmem_device *pmem,pgoff_t pgoff,long nr_pages,void **kaddr,pfn_t *pfn) { +bool bad_pmem; +bool do_recovery = false;resource_size_t offset =PFN_PHYS (pgoff) + pmem->data_offset; -if (unlikely (is_bad_pmem (&pmem->bb,PFN_PHYS (pgoff) /512 , -PFN_PHYS (nr_pages)))) + bad_pmem =is_bad_pmem (&pmem->bb,PFN_PHYS (pgoff) /512 , +PFN_PHYS (nr_pages)); +if (bad_pmem && kaddr) + do_recovery =dax_recovery_started (pmem->dax_dev, kaddr); +if (bad_pmem && !do_recovery)return -EIO ;if (kaddr)
@@ -319,10 +330,70 @@ static size_t pmem_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff,return _copy_mc_to_iter (addr, bytes, i); } +/* +* The recovery write thread started out as a normal pwrite thread and +* when the filesystem was told about potential media error in the +* range, filesystem turns the normal pwrite to a dax_recovery_write. +* +* The recovery write consists of clearing poison via DSM, clearing page +* HWPoison bit, reenable page-wide read-write permission, flush the +* caches and finally write. A competing pread thread needs to be held +* off during the recovery process since data read back might not be valid. +* And that's achieved by placing the badblock records clearing after +* the completion of the recovery write. +* +* Any competing recovery write thread needs to be serialized, and this is +* done via pmem device level lock .recovery_lock. +*/ static size_t pmem_recovery_write (struct dax_device *dax_dev,pgoff_t pgoff,void *addr,size_t bytes,struct iov_iter *i) { -return 0 ; +size_t rc, len, off; +phys_addr_t pmem_off; +struct pmem_device *pmem =dax_get_private (dax_dev); +struct device *dev = pmem->bb.dev; +sector_t sector; +long cleared, cleared_blk; + +mutex_lock (&pmem->recovery_lock); + +/* If no poison found in the range, go ahead with write */ + off = (unsigned long )addr & ~PAGE_MASK ; + len =PFN_PHYS (PFN_UP (off + bytes)); +if (!is_bad_pmem (&pmem->bb,PFN_PHYS (pgoff) /512 , len)) { + rc =_copy_from_iter_flushcache (addr, bytes, i); + goto write_done; + } + +/* Not page-aligned range cannot be recovered */ +if (off || !(PAGE_ALIGNED (bytes))) { +dev_warn (dev,"Found poison, but addr(%p) or bytes(%#lx) not page aligned\n" , + addr, bytes); + rc = (size_t ) -EIO ; + goto write_done; + } + + pmem_off =PFN_PHYS (pgoff) + pmem->data_offset; + sector = (pmem_off - pmem->data_offset) /512 ; + cleared =nvdimm_clear_poison (dev, pmem->phys_addr + pmem_off, len); + cleared_blk = cleared /512 ; +if (cleared_blk >0 ) { +hwpoison_clear (pmem, pmem->phys_addr + pmem_off, cleared); + }else { +dev_warn (dev,"pmem_recovery_write: cleared_blk: %ld\n" , + cleared_blk); + rc = (size_t ) -EIO ; + goto write_done; + } +arch_invalidate_pmem (pmem->virt_addr + pmem_off, bytes); + + rc =_copy_from_iter_flushcache (addr, bytes, i); + +pmem_clear_badblocks (pmem, sector, cleared_blk); + +write_done: +mutex_unlock (&pmem->recovery_lock); +return rc; }static const struct dax_operations pmem_dax_ops = {
@@ -514,6 +585,7 @@ static int pmem_attach_disk(struct device *dev, goto out_cleanup_dax;dax_write_cache (dax_dev,nvdimm_has_cache (nd_region));set_dax_recovery (dax_dev); +mutex_init (&pmem->recovery_lock); pmem->dax_dev = dax_dev; rc =device_add_disk (dev, disk, pmem_attribute_groups);
drivers/nvdimm/pmem.h +1 -0
@@ -24,6 +24,7 @@ struct pmem_device {struct dax_device *dax_dev;struct gendisk *disk;struct dev_pagemap pgmap; +struct mutex recovery_lock; }; long __pmem_direct_access(struct pmem_device *pmem, pgoff_t pgoff,
Signature · 1 line
2.18.4
PATCH v3 6/7 dax: add recovery_write to dax_iomap_iter in failure path
In reply to Jane Chudax_iomap_iter() fails if the destination range contains poison. Add recovery_write to the failure code path.
--- fs/dax.c | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-)
fs/dax.c +23 -2
@@ -1199,6 +1199,8 @@ int dax_truncate_page(struct inode *inode, loff_t pos, bool *did_zero, }EXPORT_SYMBOL_GPL (dax_truncate_page); +typedef size_t (*iter_func_t )(struct dax_device *dax_dev,pgoff_t pgoff, +void *addr,size_t bytes,struct iov_iter *i);static loff_t dax_iomap_iter (const struct iomap_iter *iomi,struct iov_iter *iter) {
@@ -1210,6 +1212,7 @@ static loff_t dax_iomap_iter(const struct iomap_iter *iomi,ssize_t ret =0 ;size_t xfer;int id; +iter_func_t write_func = dax_copy_from_iter;if (iov_iter_rw (iter) ==READ ) { end =min (end,i_size_read (iomi->inode));
@@ -1249,6 +1252,17 @@ static loff_t dax_iomap_iter(const struct iomap_iter *iomi, map_len =dax_direct_access (dax_dev, pgoff,PHYS_PFN (size), &kaddr,NULL ); +if ((map_len == -EIO ) && (iov_iter_rw (iter) ==WRITE )) { +if (dax_prep_recovery (dax_dev, &kaddr) <0 ) { + ret = map_len; +break ; + } + map_len =dax_direct_access (dax_dev, pgoff, +PHYS_PFN (size), &kaddr,NULL ); +if (map_len >0 ) + write_func = dax_recovery_write; + } +if (map_len <0 ) { ret = map_len;break ;
@@ -1264,14 +1278,21 @@ static loff_t dax_iomap_iter(const struct iomap_iter *iomi,* The userspace address for the memory copy has already been * validated via access_ok() in either vfs_read() or * vfs_write(), depending on which operation we are doing. +* xfer = dax_copy_from_iter(dax_dev, pgoff, kaddr, +* map_len, iter); */ if (iov_iter_rw (iter) ==WRITE ) - xfer =dax_copy_from_iter (dax_dev, pgoff, kaddr, - map_len, iter); + xfer =write_func (dax_dev, pgoff, kaddr, map_len, iter);else xfer =dax_copy_to_iter (dax_dev, pgoff, kaddr, map_len, iter); +if (xfer == (ssize_t ) -EIO ) { +pr_warn ("dax_ioma_iter: write_func returns-EIO\n" ); + ret = -EIO ; +break ; + } + pos += xfer; length -= xfer; done += xfer;
Signature · 1 line
2.18.4
PATCH v3 7/7 pmem: fix pmem_do_write() avoid writing to 'np' page
In reply to Jane ChuSince poisoned page is marked as not-present, the first of the two back-to-back write_pmem() calls can only be made when there is no poison in the range, otherwise kernel Oops.
--- drivers/nvdimm/pmem.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-)
drivers/nvdimm/pmem.c +8 -3
@@ -187,10 +187,15 @@ static blk_status_t pmem_do_write(struct pmem_device *pmem,* after clear poison. */ flush_dcache_page (page); -write_pmem (pmem_addr, page, page_off, len); -if (unlikely (bad_pmem)) { - rc =pmem_clear_poison (pmem, pmem_off, len); +if (!bad_pmem) {write_pmem (pmem_addr, page, page_off, len); + }else { + rc =pmem_clear_poison (pmem, pmem_off, len); +if (rc ==BLK_STS_OK ) +write_pmem (pmem_addr, page, page_off, len); +else +pr_warn ("%s: failed to clear poison\n" , + __func__); }return rc;
Signature · 1 line
2.18.4
Quoted 7 lines · Jane Chu
In v3, dax recovery code path is independent of that of normal write. Competing dax recovery threads are serialized, racing read threads are guaranteed not overlapping with the recovery process.
In this phase, the recovery granularity is page, future patch will explore recovery in finer granularity.
What tree is this against? I can't apply it to either 5.16 or Linus' current tree.
Quoted 11 lines · Christoph Hellwig
Quoted 7 lines · Jane Chu
In v3, dax recovery code path is independent of that of normal write. Competing dax recovery threads are serialized, racing read threads are guaranteed not overlapping with the recovery process.
In this phase, the recovery granularity is page, future patch will explore recovery in finer granularity.
What tree is this against? I can't apply it to either 5.16 or Linus' current tree.
It was based on your 'dax-block-cleanup' branch a while back.
thanks,
-jane
Quoted 4 lines · Jane Chu
Quoted 2 lines
What tree is this against? I can't apply it to either 5.16 or Linus' current tree.
It was based on your 'dax-block-cleanup' branch a while back.
Do you have a git tree with your patches included available somewhere?
Quoted 7 lines · Christoph Hellwig
Quoted 4 lines · Jane Chu
Quoted 2 lines
What tree is this against? I can't apply it to either 5.16 or Linus' current tree.
It was based on your 'dax-block-cleanup' branch a while back.
Do you have a git tree with your patches included available somewhere?
Sorry I don't have a git tree, so I rebased the series to v5.17-rc1-81-g0280e3c58f9, hope that helps.
-jane