libnvdimm/namespace: Make namespace size validation arch dependent
PATCH v3 1/6 libnvdimm/namespace: Make namespace size validation arch dependent
The page size used to map the namespace is arch dependent. For example architectures like ppc64 use 16MB page size for direct-mapping. If the namespace size is not aligned to the mapping page size, we can observe kernel crash during namespace init and destroy.
This is due to kernel doing partial map/unmap of the resource range
BUG: Unable to handle kernel data access at 0xc001000406000000 Faulting instruction address: 0xc000000000090790 NIP [c000000000090790] arch_add_memory+0xc0/0x130 LR [c000000000090744] arch_add_memory+0x74/0x130 Call Trace: arch_add_memory+0x74/0x130 (unreliable) memremap_pages+0x74c/0xa30 devm_memremap_pages+0x3c/0xa0 pmem_attach_disk+0x188/0x770 nvdimm_bus_probe+0xd8/0x470 really_probe+0x148/0x570 driver_probe_device+0x19c/0x1d0 device_driver_attach+0xcc/0x100 bind_store+0x134/0x1c0 drv_attr_store+0x44/0x60 sysfs_kf_write+0x74/0xc0 kernfs_fop_write+0x1b4/0x290 __vfs_write+0x3c/0x70 vfs_write+0xd0/0x260 ksys_write+0xdc/0x130 system_call+0x5c/0x68
Kernel should also ensure that namespace size is also mulitple of subsection size.
---
Changes from v2:
* Use SUBSECTION_SIZE instead of PAGE_SIZE. Namespace size should be multiple of SUBSECTION size.
arch/arm64/mm/flush.c | 6 ++++++ arch/powerpc/lib/pmem.c | 11 +++++++++++ arch/x86/mm/pageattr.c | 7 +++++++ include/linux/libnvdimm.h | 1 + 4 files changed, 25 insertions(+)
arch/arm64/mm/flush.c +6 -0
@@ -91,4 +91,10 @@ void arch_invalidate_pmem(void *addr, size_t size) __inval_dcache_area(addr, size); }EXPORT_SYMBOL_GPL (arch_invalidate_pmem); + +unsigned long arch_namespace_align_size (void ) +{ +return (1UL <<SUBSECTION_SHIFT ); +} +EXPORT_SYMBOL_GPL (arch_namespace_align_size);#endif
arch/powerpc/lib/pmem.c +11 -0
@@ -26,6 +26,17 @@ void arch_invalidate_pmem(void *addr, size_t size) }EXPORT_SYMBOL_GPL (arch_invalidate_pmem); +unsigned long arch_namespace_align_size (void ) +{ +unsigned long sub_section_size = (1UL <<SUBSECTION_SHIFT ); + +if (radix_enabled ()) +return sub_section_size; +return max (sub_section_size, (1UL << mmu_psize_defs[mmu_linear_psize].shift)); + +} +EXPORT_SYMBOL_GPL (arch_namespace_align_size); +/* * CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE symbols */
arch/x86/mm/pageattr.c +7 -0
@@ -310,6 +310,13 @@ void arch_invalidate_pmem(void *addr, size_t size) }EXPORT_SYMBOL_GPL (arch_invalidate_pmem); +unsigned long arch_namespace_align_size (void ) +{ +return (1UL <<SUBSECTION_SHIFT ); +} +EXPORT_SYMBOL_GPL (arch_namespace_align_size); + +static void __cpa_flush_all (void *arg) {unsigned long cache = (unsigned long )arg;
include/linux/libnvdimm.h +1 -0
@@ -284,4 +284,5 @@ static inline void arch_invalidate_pmem(void *addr, size_t size) }#endif +unsigned long arch_namespace_align_size (void );#endif /* __LIBNVDIMM_H__ */
Signature · 4 lines
2.24.1 _______________________________________________ Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
PATCH v3 2/6 libnvdimm/namespace: Validate namespace start addr and size
Make sure namespace start addr and size are properly aligned as per architecture restrictions. If the namespace is not aligned we mark the namespace 'disabled'
Architectures like ppc64 use different page size than PAGE_SIZE to map direct-map address range. The kernel needs to make sure the namespace size is aligned correctly for the direct-map page size.
kernel log will contain information as below.
[ 5.810939] nd_pmem namespace0.1: invalid size/SPA [ 5.810969] nd_pmem: probe of namespace0.1 failed with error -95
and the namespace will be marked 'disabled'
{
"dev":"namespace0.1",
"mode":"fsdax",
"map":"mem",
"size":1071644672,
"uuid":"25577a00-c012-421d-89ca-3ee189e08848",
"sector_size":512,
"state":"disabled"
},--- drivers/nvdimm/namespace_devs.c | 39 +++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+)
drivers/nvdimm/namespace_devs.c +39 -0
@@ -1113,6 +1113,40 @@ resource_size_t nvdimm_namespace_capacity(struct nd_namespace_common *ndns) }EXPORT_SYMBOL (nvdimm_namespace_capacity); +static bool nvdimm_valid_namespace (struct nd_namespace_common *ndns,resource_size_t size) +{ +struct device *dev = &ndns->dev; +struct nd_region *nd_region =to_nd_region (dev->parent); +unsigned long align_size =arch_namespace_align_size (); +struct resource *res; +u32 remainder; + +if (is_namespace_blk (dev)) +return true; + +div_u64_rem (size, align_size * nd_region->ndr_mappings, &remainder); +if (remainder) +return false; + +if (is_namespace_pmem (dev)) { +struct nd_namespace_pmem *nspm =to_nd_namespace_pmem (dev); + + res = &nspm->nsio.res; + }else if (is_namespace_io (dev)) { +struct nd_namespace_io *nsio =to_nd_namespace_io (dev); + + res = &nsio->res; + }else +/* cannot reach */ +return false; + +div_u64_rem (res->start, align_size * nd_region->ndr_mappings, &remainder); +if (remainder) +return false; + +return true; +} +bool nvdimm_namespace_locked (struct nd_namespace_common *ndns) {int i;
@@ -1739,6 +1773,11 @@ struct nd_namespace_common *nvdimm_namespace_common_probe(struct device *dev)return ERR_PTR (-ENODEV ); } +if (!nvdimm_valid_namespace (ndns, size)) { +dev_err (&ndns->dev,"invalid size/SPA" ); +return ERR_PTR (-EOPNOTSUPP ); + } +if (is_namespace_pmem (&ndns->dev)) {struct nd_namespace_pmem *nspm;
Signature · 4 lines
2.24.1 _______________________________________________ Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
PATCH v3 3/6 libnvdimm/namespace: Validate namespace size when creating new namespace.
In reply to Aneesh Kumar K.VArchitectures like ppc64 use different page size than PAGE_SIZE to map direct-map address range. The kernel needs to make sure the namespace size is aligned correctly for the direct-map page size.
kernel log will contain the below details
[ 939.620064] nd namespace0.3: 1071644672 is not 16384K aligned
--- drivers/nvdimm/namespace_devs.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-)
drivers/nvdimm/namespace_devs.c +16 -5
@@ -891,6 +891,17 @@ static int grow_dpa_allocation(struct nd_region *nd_region,return 0 ; } +static unsigned long nvdimm_validate_namespace_size (struct nd_region *nd_region, +unsigned long size,unsigned long align_size) +{ +u32 remainder; + +div_u64_rem (size, align_size * nd_region->ndr_mappings, &remainder); +if (remainder) +return align_size * nd_region->ndr_mappings; +return 0 ; +} +static void nd_namespace_pmem_set_resource (struct nd_region *nd_region,struct nd_namespace_pmem *nspm,resource_size_t size) {
@@ -949,7 +960,8 @@ static ssize_t __size_store(struct device *dev, unsigned long long val)struct nd_mapping *nd_mapping;struct nvdimm_drvdata *ndd;struct nd_label_id label_id; -u32 flags =0 , remainder; +unsigned long map_size; +u32 flags =0 ;int rc, i, id =-1 ;u8 *uuid =NULL ;
@@ -980,10 +992,9 @@ static ssize_t __size_store(struct device *dev, unsigned long long val)return -ENXIO ; } -div_u64_rem (val,PAGE_SIZE * nd_region->ndr_mappings, &remainder); -if (remainder) { -dev_dbg (dev,"%llu is not %ldK aligned\n" , val, - (PAGE_SIZE * nd_region->ndr_mappings) /SZ_1K ); + map_size =nvdimm_validate_namespace_size (nd_region, val,arch_namespace_align_size ()); +if (map_size) { +dev_err (dev,"%llu is not %ldK aligned\n" , val, map_size /SZ_1K );return -EINVAL ; }
Signature · 4 lines
2.24.1 _______________________________________________ Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
PATCH v3 4/6 libnvdimm/namespace: Add debug check while initializing namespace resource size.
In reply to Aneesh Kumar K.VThis should enable us to catch if we are initializing the namespace with a wrong size.
--- drivers/nvdimm/namespace_devs.c | 12 ++++++++++++ 1 file changed, 12 insertions(+)
drivers/nvdimm/namespace_devs.c +12 -0
@@ -941,6 +941,18 @@ static void nd_namespace_pmem_set_resource(struct nd_region *nd_region, out: res->start = nd_region->ndr_start + offset; res->end = res->start + size -1 ; +#ifdef CONFIG_DEBUG_VM +if (size) { +unsigned long map_size; + + map_size =nvdimm_validate_namespace_size (nd_region, size,arch_namespace_align_size ()); +WARN_ON (map_size); + + map_size =nvdimm_validate_namespace_size (nd_region, res->start,arch_namespace_align_size ()); +WARN_ON (map_size); + + } +#endif }static bool uuid_not_set (const u8 *uuid,struct device *dev,const char *where)
Signature · 4 lines
2.24.1 _______________________________________________ Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
PATCH v3 5/6 libnvdimm/namespace: Align DPA based on arch restrictions
In reply to Aneesh Kumar K.VWhen creating new namespace make sure DPA address are properly aligned based on arch restrictions.
--- drivers/nvdimm/namespace_devs.c | 2 ++ 1 file changed, 2 insertions(+)
drivers/nvdimm/namespace_devs.c +2 -0
@@ -563,6 +563,8 @@ static void space_valid(struct nd_region *nd_region, struct nvdimm_drvdata *ndd,return ; } + valid->start =ALIGN (valid->start,arch_namespace_align_size ()); +/* allocation needs to be contiguous, so this is all or nothing */ if (resource_size (valid) < n) goto invalid;
Signature · 4 lines
2.24.1 _______________________________________________ Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
PATCH v3 6/6 libnvdimm/namespace: Expose arch specific supported size align value
In reply to Aneesh Kumar K.VExpose supported size align as a namespace RO attribute. Usespace can use this to validate the size argument specified when creating a new namespace.
--- drivers/nvdimm/namespace_devs.c | 9 +++++++++ 1 file changed, 9 insertions(+)
drivers/nvdimm/namespace_devs.c +9 -0
@@ -1658,6 +1658,14 @@ static ssize_t force_raw_show(struct device *dev, }static DEVICE_ATTR_RW (force_raw); +static ssize_t supported_size_align_show (struct device *dev, +struct device_attribute *attr,char *buf) +{ +return sprintf (buf,"%ld\n" ,arch_namespace_align_size ()); +} +static DEVICE_ATTR_RO (supported_size_align); + +static struct attribute *nd_namespace_attributes[] = { &dev_attr_nstype.attr, &dev_attr_size.attr,
@@ -1670,6 +1678,7 @@ static struct attribute *nd_namespace_attributes[] = { &dev_attr_sector_size.attr, &dev_attr_dpa_extents.attr, &dev_attr_holder_class.attr, + &dev_attr_supported_size_align.attr, NULL, };
Signature · 4 lines
2.24.1 _______________________________________________ Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
Hi, Aneesh,
After applying this patch series, several of my namespaces no longer enumerate:
Before:
# ndctl list
[
{
"dev":"namespace0.2",
"mode":"sector",
"size":106541672960,
"uuid":"ea1122b2-c219-424c-b09c-38a6e94a1042",
"sector_size":512,
"blockdev":"pmem0.2s"
},
{
"dev":"namespace0.1",
"mode":"fsdax",
"map":"dev",
"size":10567548928,
"uuid":"68b6746f-481a-4ae6-80b5-71d62176606c",
"sector_size":512,
"align":4096,
"blockdev":"pmem0.1"
},
{
"dev":"namespace0.0",
"mode":"fsdax",
"map":"dev",
"size":52850327552,
"uuid":"6d3a0199-5d9a-4fed-830d-e25249b70571",
"sector_size":512,
"align":2097152,
"blockdev":"pmem0"
}
]After:
# ndctl list
[
{
"dev":"namespace0.0",
"mode":"fsdax",
"map":"dev",
"size":52850327552,
"uuid":"6d3a0199-5d9a-4fed-830d-e25249b70571",
"sector_size":512,
"align":2097152,
"blockdev":"pmem0"
}
]I won't have time to dig into it this week, but I wanted to mention it before Dan merged these patches.
I'll follow up next week with more information.
Cheers,
Jeff
_______________________________________________
Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
Quoted 56 lines · Jeff Moyer
Hi, Aneesh,
After applying this patch series, several of my namespaces no longer enumerate:
Before:
# ndctl list [ { "dev":"namespace0.2", "mode":"sector", "size":106541672960, "uuid":"ea1122b2-c219-424c-b09c-38a6e94a1042", "sector_size":512, "blockdev":"pmem0.2s" }, { "dev":"namespace0.1", "mode":"fsdax", "map":"dev", "size":10567548928, "uuid":"68b6746f-481a-4ae6-80b5-71d62176606c", "sector_size":512, "align":4096, "blockdev":"pmem0.1" }, { "dev":"namespace0.0", "mode":"fsdax", "map":"dev", "size":52850327552, "uuid":"6d3a0199-5d9a-4fed-830d-e25249b70571", "sector_size":512, "align":2097152, "blockdev":"pmem0" } ]After:
# ndctl list [ { "dev":"namespace0.0", "mode":"fsdax", "map":"dev", "size":52850327552, "uuid":"6d3a0199-5d9a-4fed-830d-e25249b70571", "sector_size":512, "align":2097152, "blockdev":"pmem0" } ]
Quoted 4 lines
I won't have time to dig into it this week, but I wanted to mention it before Dan merged these patches.
I'll follow up next week with more information.
Thanks Jeff, I hadn't had a chance to dig in on these yet.
Aneesh are you able to run the ndctl unit tests? Even if they don't run on powerpc you should be able to get them to run on x86 qemu just to catch the basics.
_______________________________________________
Linux-nvdimm mailing list -- [email protected]
To unsubscribe send an email to [email protected]
Quoted 61 lines · Dan Williams
Quoted 60 lines · Jeff Moyer
Hi, Aneesh,
After applying this patch series, several of my namespaces no longer enumerate:
Before:
# ndctl list [ { "dev":"namespace0.2", "mode":"sector", "size":106541672960, "uuid":"ea1122b2-c219-424c-b09c-38a6e94a1042", "sector_size":512, "blockdev":"pmem0.2s" }, { "dev":"namespace0.1", "mode":"fsdax", "map":"dev", "size":"uuid":"68b6746f-481a-4ae6-80b5-71d62176606c", "sector_size":512, "align":4096, "blockdev":"pmem0.1" }, { "dev":"namespace0.0", "mode":"fsdax", "map":"dev", "size":52850327552, "uuid":"6d3a0199-5d9a-4fed-830d-e25249b70571", "sector_size":512, "align":2097152, "blockdev":"pmem0"}
]After:
# ndctl list [ { "dev":"namespace0.0", "mode":"fsdax", "map":"dev", "size":52850327552, "uuid":"6d3a0199-5d9a-4fed-830d-e25249b70571", "sector_size":512, "align":2097152, "blockdev":"pmem0" } ]I won't have time to dig into it this week, but I wanted to mention it before Dan merged these patches.
Quoted 8 lines
Quoted 1 line
I'll follow up next week with more information.
Thanks Jeff, I hadn't had a chance to dig in on these yet.
Aneesh are you able to run the ndctl unit tests? Even if they don't run on powerpc you should be able to get them to run on x86 qemu just to catch the basics.
Thanks for the feedback. I will test the series with qemu.
-aneesh
_______________________________________________
Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
Quoted 55 lines · Jeff Moyer
Hi, Aneesh,
After applying this patch series, several of my namespaces no longer enumerate:
Before:
# ndctl list [ { "dev":"namespace0.2", "mode":"sector", "size":106541672960, "uuid":"ea1122b2-c219-424c-b09c-38a6e94a1042", "sector_size":512, "blockdev":"pmem0.2s" }, { "dev":"namespace0.1", "mode":"fsdax", "map":"dev", "size":10567548928, "uuid":"68b6746f-481a-4ae6-80b5-71d62176606c", "sector_size":512, "align":4096, "blockdev":"pmem0.1" }, { "dev":"namespace0.0", "mode":"fsdax", "map":"dev", "size":52850327552, "uuid":"6d3a0199-5d9a-4fed-830d-e25249b70571", "sector_size":512, "align":2097152, "blockdev":"pmem0" } ]After:
# ndctl list [ { "dev":"namespace0.0", "mode":"fsdax", "map":"dev", "size":52850327552, "uuid":"6d3a0199-5d9a-4fed-830d-e25249b70571", "sector_size":512, "align":2097152, "blockdev":"pmem0" } ]
Quoted 5 lines
I won't have time to dig into it this week, but I wanted to mention it before Dan merged these patches.
I'll follow up next week with more information.
dmesg should contain details like
[ 5.810939] nd_pmem namespace0.1: invalid size/SPA [ 5.810969] nd_pmem: probe of namespace0.1 failed with error -95
This is mostly due to the namespace start address not aligned to subsection size.
"namespace0.2" not having a 2MB aligned size which cause namespace 0.1 start addr to be not aligned. Hence both the namespace are marked disabled.
-aneesh
_______________________________________________
Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
Quoted 65 lines · Aneesh Kumar K.V
Quoted 55 lines · Jeff Moyer
Hi, Aneesh,
After applying this patch series, several of my namespaces no longer enumerate:
Before:
# ndctl list [ { "dev":"namespace0.2", "mode":"sector", "size":106541672960, "uuid":"ea1122b2-c219-424c-b09c-38a6e94a1042", "sector_size":512, "blockdev":"pmem0.2s" }, { "dev":"namespace0.1", "mode":"fsdax", "map":"dev", "size":10567548928, "uuid":"68b6746f-481a-4ae6-80b5-71d62176606c", "sector_size":512, "align":4096, "blockdev":"pmem0.1" }, { "dev":"namespace0.0", "mode":"fsdax", "map":"dev", "size":52850327552, "uuid":"6d3a0199-5d9a-4fed-830d-e25249b70571", "sector_size":512, "align":2097152, "blockdev":"pmem0" } ]After:
# ndctl list [ { "dev":"namespace0.0", "mode":"fsdax", "map":"dev", "size":52850327552, "uuid":"6d3a0199-5d9a-4fed-830d-e25249b70571", "sector_size":512, "align":2097152, "blockdev":"pmem0" } ]Quoted 5 lines
I won't have time to dig into it this week, but I wanted to mention it before Dan merged these patches.
I'll follow up next week with more information.
dmesg should contain details like
Quoted 9 lines
[ 5.810939] nd_pmem namespace0.1: invalid size/SPA [ 5.810969] nd_pmem: probe of namespace0.1 failed with error -95This is mostly due to the namespace start address not aligned to subsection size.
"namespace0.2" not having a 2MB aligned size which cause namespace 0.1 start addr to be not aligned. Hence both the namespace are marked disabled.
2 observations:
- It's ok if the namespace start address is not subsection aligned as long as the mapped portion for data access is subsection aligned, at least on x86.
- "sector" mode namespaces are not mapped by devm_memremap_pages() so there should be no restriction there. If powerpc can't map them that's a separate concern.
So, cross arch compatible namespaces is a goal, but not regressing existing namespaces takes precedence. I'd be happy if newly created namespaces tried to account for all the arch quirks, but if libnvdimm can enable a namespace it should try.
_______________________________________________
Linux-nvdimm mailing list -- [email protected]
To unsubscribe send an email to [email protected]
Quoted 12 lines · Dan Williams
Quoted 7 lines · Aneesh Kumar K.V
Quoted 1 line · Jeff Moyer
Hi, Aneesh,
"namespace0.2" not having a 2MB aligned size which cause namespace 0.1 start addr to be not aligned. Hence both the namespace are marked disabled.
2 observations:
Quoted 7 lines
- It's ok if the namespace start address is not subsection aligned as long as the mapped portion for data access is subsection aligned, at least on x86.
- "sector" mode namespaces are not mapped by
devm_memremap_pages()so there should be no restriction there. If powerpc can't map them that's a separate concern.
Does that mean the supported_size_align attribute should be a property of pfn and dax seed device? Considering we don't want to apply this restrictions for blk, raw namespace, and btt mode namespace should we make the attribute a seed device property rather than a namespace property?
Quoted 5 lines
So, cross arch compatible namespaces is a goal, but not regressing existing namespaces takes precedence. I'd be happy if newly created namespaces tried to account for all the arch quirks, but if libnvdimm can enable a namespace it should try.
Ok. So that means we apply the alignment rules when creating new namespaces irrespective of its type/mode. But when initializing via scan label we only apply them for devdax and fsdax namespace?
Something like
static bool nvdimm_valid_namespace(struct device *dev,
struct nd_namespace_common *ndns, resource_size_t size)
{
struct nd_region *nd_region = to_nd_region(ndns->dev.parent);
unsigned long align_size = arch_namespace_align_size();
struct resource *res;
u32 remainder;
/*
* Don't validate the start and size for blk namespace type
*/
if (is_namespace_blk(&ndns->dev))
return true;
/*
* For btt and raw namespace we use ioremap. Assume both can work
* with PAGE_SIZE alignment.
*/
if (is_nd_btt(dev) || is_namespace_io(dev))
return true;
div_u64_rem(size, align_size * nd_region->ndr_mappings, &remainder);
if (remainder)
return false;
if (is_namespace_pmem(&ndns->dev)) {
struct nd_namespace_pmem *nspm = to_nd_namespace_pmem(&ndns->dev);
res = &nspm->nsio.res;
} else
/* cannot reach */
return false;
div_u64_rem(res->start, align_size * nd_region->ndr_mappings, &remainder);
if (remainder)
return false;
return true;
}-aneesh
_______________________________________________
Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]