add pin memory method for checkout add restore
Subsystems (6)
PATCH 1/1 add pin memory method for checkout add restore
From: Jingxian He <[email protected]<mailto:[email protected]>>
Date: Thu, 10 Dec 2020 20:31:15 +0800
Subject: [PATCH] add pin memory method for checkout add restore
We can use the checkpoint and restore in userspace(criu) method to dump and restore tasks when updating the kernel. Currently, criu needs dump all memory data of tasks to files. When the memory size is very large(larger than 1G), the cost time of the dumping data will be very long(more than 1 min).
We can pin the memory data of tasks and collect the corresponding physical pages mapping info in checkpoint process, and remap the physical pages to restore tasks after kernel is updated.
The pin memory area info is saved in the reserve memblock named nvwa_res_first, which can keep usable in the kernel update process.
The pin memory driver provides the following ioctl command for criu:
1) SET_PIN_MEM_AREA: set pin memory area, which can be remap to the restore task.
2) CLEAR_PIN_MEM_AREA: clear the pin memory area info, which enable user reset the pin data.
3) REMAP_PIN_MEM_AREA: remap the pages of the pin memory to the restore task.
---
arch/arm64/kernel/setup.c | 7 + arch/arm64/mm/init.c | 62 +++- drivers/char/Kconfig | 7 + drivers/char/Makefile | 1 + drivers/char/pin_memory.c | 198 +++++++++++++ include/linux/crash_core.h | 5 + include/linux/pin_mem.h | 62 ++++ kernel/crash_core.c | 11 + mm/Kconfig | 6 + mm/Makefile | 1 + mm/huge_memory.c | 61 ++++ mm/memory.c | 68 +++++ mm/pin_mem.c | 691 +++++++++++++++++++++++++++++++++++++++++++++ 13 files changed, 1179 insertions(+), 1 deletion(-)
create mode 100644 drivers/char/pin_memory.c
create mode 100644 include/linux/pin_mem.h
create mode 100644 mm/pin_mem.c
arch/arm64/kernel/setup.c +0 -0
@@ -50,6 +50,9 @@
#include <asm/efi.h>
#include <asm/xen/hypervisor.h>
#include <asm/mmu_context.h>
+#ifdef CONFIG_PIN_MEMORY
+#include <linux/pin_memory.h>
+#endif
static int num_standard_resources;
static struct resource *standard_resources;
@@ -243,6 +246,10 @@ static void __init request_standard_resources(void)
crashk_res.end <= res->end)
request_resource(res, &crashk_res);#endif
+#ifdef CONFIG_PIN_MEMORY
+ if (pin_memory_resource.end)
+ insert_resource(&iomem_resource, &pin_memory_resource);
+#endif
} }
arch/arm64/mm/init.c +0 -0
@@ -41,7 +41,9 @@
#include <linux/sizes.h>
#include <asm/tlb.h>
#include <asm/alternative.h>
-
+#ifdef CONFIG_PIN_MEMORY
+#include <linux/pin_memory.h>
+#endif
#define ARM64_ZONE_DMA_BITS 30
/*
@@ -68,6 +70,16 @@phys_addr_t arm64_dma_phys_limit __ro_after_init; static phys_addr_t arm64_dma32_phys_limit __ro_after_init;
+#ifdef CONFIG_PIN_MEMORY
+struct resource pin_memory_resource = {
+ .name = "Pin memory maps", + .start = 0, + .end = 0, + .flags = IORESOURCE_MEM, + .desc = IORES_DESC_PIN_MEM_MAPS
+};
+#endif
+
#ifdef CONFIG_KEXEC_CORE
/*
* reserve_crashkernel() - reserves memory for crash kernel
@@ -129,6 +141,47 @@ static void __init reserve_crashkernel(void) }
#endif /* CONFIG_KEXEC_CORE */
+#ifdef CONFIG_PIN_MEMORY
+static void __init reserve_pin_memory_res(void)
+{
+ unsigned long long mem_start, mem_len;
+ int ret;
+
+ ret = parse_pin_memory(boot_command_line, memblock_phys_mem_size(),
+ &mem_len, &mem_start);
+ if (ret || !mem_len)
+ return;
+
+ mem_len = PAGE_ALIGN(mem_len);
+
+ if (!memblock_is_region_memory(mem_start, mem_len)) {
+ pr_warn("cannot reserve for pin memory: region is not memory!\n");
+ return;
+ }
+
+ if (memblock_is_region_reserved(mem_start, mem_len)) {
+ pr_warn("cannot reserve for pin memory: region overlaps reserved memory!\n");
+ return;
+ }
+
+ if (!IS_ALIGNED(mem_start, SZ_2M)) {
+ pr_warn("cannot reserve for pin memory: base address is not 2MB aligned\n");
+ return;
+ }
+
+ memblock_reserve(mem_start, mem_len);
+ pr_debug("pin memory resource reserved: 0x%016llx - 0x%016llx (%lld MB)\n",
+ mem_start, mem_start + mem_len, mem_len >> 20);
+
+ pin_memory_resource.start = mem_start;
+ pin_memory_resource.end = mem_start + mem_len - 1;
+}
+#else
+static void __init reserve_pin_memory_res(void)
+{
+}
+#endif /* CONFIG_PIN_MEMORY */
+
#ifdef CONFIG_CRASH_DUMP
static int __init early_init_dt_scan_elfcorehdr(unsigned long node,
const char *uname, int depth, void *data)
@@ -452,6 +505,8 @@ void __init arm64_memblock_init(void)
reserve_crashkernel();
+ reserve_pin_memory_res();
+
reserve_elfcorehdr();
high_memory = __va(memblock_end_of_DRAM() - 1) + 1;
@@ -573,6 +628,11 @@ void __init mem_init(void)
/* this will put all unused low memory onto the freelists */
memblock_free_all();+#ifdef CONFIG_PIN_MEMORY
+ /* pre alloc the pages for pin memory */
+ init_reserve_page_map((unsigned long)pin_memory_resource.start,
+ (unsigned long)(pin_memory_resource.end - pin_memory_resource.start));
+#endif
mem_init_print_info(NULL);
/*drivers/char/Kconfig +7 -0
@@ -560,3 +560,10 @@ config RANDOM_TRUST_BOOTLOADER booloader is trustworthy so it will be added to the kernel's entropy pool. Otherwise, say N here so it will be regarded as device input that only mixes the entropy pool. + +config PIN_MEMORY_DEV + bool "/dev/pinmem character device" + depends PIN_MEMORY + default n + help + pin memory driver
drivers/char/Makefile +0 -0
@@ -52,3 +52,4 @@ js-rtc-y = rtc.o
obj-$(CONFIG_XILLYBUS) += xillybus/
obj-$(CONFIG_POWERNV_OP_PANEL) += powernv-op-panel.o
obj-$(CONFIG_ADI) += adi.o
+obj-$(CONFIG_PIN_MEMORY_DEV) += pin_memory.o
drivers/char/pin_memory.c new file +198 -0
198 lines not on this page · raw message
include/linux/crash_core.h +0 -0
@@ -75,4 +75,9 @@ int parse_crashkernel_high(char *cmdline, unsigned long long system_ram,
int parse_crashkernel_low(char *cmdline, unsigned long long system_ram,
unsigned long long *crash_size, unsigned long long *crash_base);
+#ifdef CONFIG_PIN_MEMORY
+int __init parse_pin_memory(char *cmdline, unsigned long long system_ram,
+ unsigned long long *pin_size, unsigned long long *pin_base);
+#endif
+
#endif /* LINUX_CRASH_CORE_H */
include/linux/pin_mem.h new file +62 -0
62 lines not on this page · raw message
kernel/crash_core.c +0 -0
@@ -292,6 +292,17 @@ int __init parse_crashkernel_low(char *cmdline, "crashkernel=", suffix_tbl[SUFFIX_LOW]);
}
+#ifdef CONFIG_PIN_MEMORY
+int __init parse_pin_memory(char *cmdline,
+ unsigned long long system_ram, + unsigned long long *pin_size, + unsigned long long *pin_base)
+{
+ return __parse_crashkernel(cmdline, system_ram, pin_size, pin_base,
+ "pinmemory=", NULL);
+}
+#endif
+Elf_Word *append_elf_note(Elf_Word *buf, char *name, unsigned int type,
void *data, size_t data_len)
{
mm/Kconfig +0 -0
@@ -739,4 +739,10 @@ config ARCH_HAS_HUGEPD
config MAPPING_DIRTY_HELPERS
bool
+config PIN_MEMORY
+ bool "Support for pin memory" + depends on CHECKPOINT_RESTORE + help
+ Say y here to enable the pin memory feature for checkpoint
+ and restore.
endmenu
mm/Makefile +0 -0
@@ -108,3 +108,4 @@ obj-$(CONFIG_ZONE_DEVICE) += memremap.o
obj-$(CONFIG_HMM_MIRROR) += hmm.o obj-$(CONFIG_MEMFD_CREATE) += memfd.o obj-$(CONFIG_MAPPING_DIRTY_HELPERS) += mapping_dirty_helpers.o +obj-$(CONFIG_PIN_MEMORY) += pin_mem.o
mm/huge_memory.c +0 -0
@@ -3083,4 +3083,65 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new)mlock_vma_page (new);update_mmu_cache_pmd (vma, address, pvmw->pmd);
}
+
+#ifdef CONFIG_PIN_MEMORY
+vm_fault_t do_anon_huge_page_remap(struct vm_area_struct *vma, unsigned long address,
+ pmd_t *pmd, struct page *page)
+{
+ gfp_t gfp;
+ pgtable_t pgtable;
+ spinlock_t *ptl;
+ pmd_t entry;
+ vm_fault_t ret = 0;
+ struct mem_cgroup *memcg;
+
+ if (unlikely(anon_vma_prepare(vma)))
+ return VM_FAULT_OOM;
+ if (unlikely(khugepaged_enter(vma, vma->vm_flags)))
+ return VM_FAULT_OOM;
+ gfp = alloc_hugepage_direct_gfpmask(vma);
+ prep_transhuge_page(page);
+ if (mem_cgroup_try_charge_delay(page, vma->vm_mm, gfp, &memcg, true)) {
+ put_page(page);
+ count_vm_event(THP_FAULT_FALLBACK);
+ return VM_FAULT_FALLBACK;
+ }
+ pgtable = pte_alloc_one(vma->vm_mm, address);
+ if (unlikely(!pgtable)) {
+ ret = VM_FAULT_OOM;
+ goto release;
+ }
+ __SetPageUptodate(page);
+ ptl = pmd_lock(vma->vm_mm, pmd);
+ if (unlikely(!pmd_none(*pmd))) {+ goto unlock_release;
+ } else {
+ ret = check_stable_address_space(vma->vm_mm);
+ if (ret)
+ goto unlock_release;
+ entry = mk_huge_pmd(page, vma->vm_page_prot); + entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); + page_add_new_anon_rmap(page, vma, address, true); + mem_cgroup_commit_charge(page, memcg, false, true); + lru_cache_add_active_or_unevictable(page, vma); + pgtable_trans_huge_deposit(vma->vm_mm, pmd, pgtable); + set_pmd_at(vma->vm_mm, address, pmd, entry); + add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); + mm_inc_nr_ptes(vma->vm_mm); + spin_unlock(ptl); + count_vm_event(THP_FAULT_ALLOC);
+ }
+
+ return 0;
+unlock_release:
+ spin_unlock(ptl);
+release:
+ if (pgtable)
+ pte_free(vma->vm_mm, pgtable);
+ mem_cgroup_cancel_charge(page, memcg, true); + put_page(page); + return ret;
+}
+#endif
+
#endif
mm/memory.c +0 -0
@@ -4799,4 +4799,72 @@ void ptlock_free(struct page *page)
{
kmem_cache_free(page_ptl_cachep, page->ptl);
}+
+#ifdef CONFIG_PIN_MEMORY
+vm_fault_t do_anon_page_remap(struct vm_area_struct *vma, unsigned long address,
+ pmd_t *pmd, struct page *page)
+{
+ struct mem_cgroup *memcg; + pte_t entry; + spinlock_t *ptl; + pte_t *pte; + vm_fault_t ret = 0;
+
+ if (pte_alloc(vma->vm_mm, pmd, address))
+ return VM_FAULT_OOM;
+
+ /* See the comment in pte_alloc_one_map() */
+ if (unlikely(pmd_trans_unstable(pmd)))
+ return 0;
+
+ /* Allocate our own private page. */
+ if (unlikely(anon_vma_prepare(vma)))
+ goto oom;
+
+ if (mem_cgroup_try_charge_delay(page, vma->vm_mm, GFP_KERNEL, &memcg,
+ false))
+ goto oom_free_page;
+
+ /* + * The memory barrier inside __SetPageUptodate makes sure that + * preceeding stores to the page contents become visible before + * the set_pte_at() write. + */ + __SetPageUptodate(page);
+
+ entry = mk_pte(page, vma->vm_page_prot);
+ if (vma->vm_flags & VM_WRITE)
+ entry = pte_mkwrite(pte_mkdirty(entry));
+ pte = pte_offset_map_lock(vma->vm_mm, pmd, address,
+ &ptl);
+ if (!pte_none(*pte)) {
+ ret = VM_FAULT_FALLBACK;
+ goto release;
+ }
+
+ ret = check_stable_address_space(vma->vm_mm);
+ if (ret)
+ goto release;
+ inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES); + page_add_new_anon_rmap(page, vma, address, false); + mem_cgroup_commit_charge(page, memcg, false, false); + lru_cache_add_active_or_unevictable(page, vma); + + set_pte_at(vma->vm_mm, address, pte, entry); + /* No need to invalidate - it was non-present before */ + update_mmu_cache(vma, address, pte);
+unlock:
+ pte_unmap_unlock(pte, ptl);
+ return ret;
+release:
+ mem_cgroup_cancel_charge(page, memcg, false); + put_page(page); + goto unlock;
+oom_free_page:
+ put_page(page);
+oom:
+ return VM_FAULT_OOM;
+}
+#endif
+
#endif
mm/pin_mem.c new file +691 -0
691 lines not on this page · raw message
--
1.8.3.1
Hi,
drivers/char/Kconfig config RANDOM_TRUST_BOOTLOADER · line 566
Quoted 12 lines · hejingxian
drivers/char/Kconfig +4
-0@@ -560,3 +560,10 @@ config RANDOM_TRUST_BOOTLOADER booloader is trustworthy so it will be added to the kernel's entropy pool. Otherwise, say N here so it will be regarded as device input that only mixes the entropy pool. + +config PIN_MEMORY_DEV + bool "/dev/pinmem character device" + depends PIN_MEMORY
depends on
drivers/char/Kconfig config RANDOM_TRUST_BOOTLOADER · line 567
Quoted 1 line
drivers/char/Kconfig +1
-0+ default n
Don't need default n, it's already the default.
drivers/char/Kconfig config RANDOM_TRUST_BOOTLOADER · line 569
Quoted 2 lines
drivers/char/Kconfig +2
-0+ help + pin memory driver
Better help text, please. Also, it should be indented by 2 (more) spaces.
thanks.
Signature · 1 line
~Randy