Dirty Page Tracking & on-the-fly memory mirroring

This patch implements dirty page tracking and re-dirty detection. We use one of the programmer bits in the PTE to implement a "soft dirty" bit. In the VM subsystem, all tests for dirty pages check the logical OR of the hardware dirty bit and the "soft dirty" bit; if either one is set the page is considered "dirty" for VM purposes. To speed up the scanning passes this patch also implements a bitmapped side-list of dirty physical pages.

Kconfig                      |   11 ++
include/asm/hugetlb.h        |    3
include/asm/mm_track.h       |   98 ++++++++++++++++++++++++
include/asm/pgtable-2level.h |    4
include/asm/pgtable-3level.h |   12 ++
include/asm/pgtable.h        |    4
include/asm/pgtable_32.h     |    1
include/asm/pgtable_64.h     |    7 +
include/asm/pgtable_types.h  |    5 -
mm/Makefile                  |    1
mm/track.c                   |  174 +++++++++++++++++++++++++++++++++++++++++++
11 files changed, 317 insertions(+), 3 deletions(-)

diff -up linux-2.6-git-track/arch/x86/include/asm/hugetlb.h.track linux-2.6-git-track/arch/x86/include/asm/hugetlb.h
@@ -2,6 +2,7 @@

#define _ASM_X86_HUGETLB_H

#include <asm/page.h>
+#include <asm/mm_track.h>

static inline int is_hugepage_only_range(struct mm_struct *mm, @@ -39,12 +40,14 @@ static inline void hugetlb_free_pgd_rang static inline void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t pte)
{

+    mm_track_pmd((pmd_t *)ptep);
    set_pte_at(mm, addr, ptep, pte);
}

static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
unsigned long addr, pte_t *ptep)
{

+    mm_track_pmd((pmd_t *)ptep);
    return ptep_get_and_clear(mm, addr, ptep);
}

diff -up linux-2.6-git-track/arch/x86/include/asm/pgtable-2level.h.track linux-2.6-git-track/arch/x86/include/asm/pgtable-2level.h
@@ -13,11 +13,13 @@
 */

static inline void native_set_pte(pte_t *ptep , pte_t pte) {

+    mm_track_pte(ptep);
    *ptep = pte;
}

static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) {

+    mm_track_pmd(pmdp);
    *pmdp = pmd;
}

@@ -34,12 +36,14 @@ static inline void native_pmd_clear(pmd_ static inline void native_pte_clear(struct mm_struct *mm,
unsigned long addr, pte_t *xp)
{

+    mm_track_pte(xp);
    *xp = native_make_pte(0);
}

#ifdef CONFIG_SMP
static inline pte_t native_ptep_get_and_clear(pte_t *xp) {

+    mm_track_pte(xp);
    return __pte(xchg(&xp->pte_low, 0));
}

#else

diff -up linux-2.6-git-track/arch/x86/include/asm/pgtable_32.h.track linux-2.6-git-track/arch/x86/include/asm/pgtable_32.h
@@ -22,6 +22,7 @@

#include <linux/slab.h>
#include <linux/list.h>
#include <linux/spinlock.h>
+#include <asm/mm_track.h>

struct mm_struct;
struct vm_area_struct;

diff -up linux-2.6-git-track/arch/x86/include/asm/pgtable-3level.h.track linux-2.6-git-track/arch/x86/include/asm/pgtable-3level.h
@@ -26,6 +26,7 @@
 */

static inline void native_set_pte(pte_t *ptep, pte_t pte) {

+    mm_track_pte(ptep);
    ptep->pte_high = pte.pte_high;
    smp_wmb();
    ptep->pte_low = pte.pte_low;

@@ -33,16 +34,19 @@ static inline void native_set_pte(pte_t

static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte) {

+    mm_track_pte(ptep);
    set_64bit((unsigned long long *)(ptep), native_pte_val(pte));
}

static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) {

+    mm_track_pmd(pmdp);
    set_64bit((unsigned long long *)(pmdp), native_pmd_val(pmd));
}

static inline void native_set_pud(pud_t *pudp, pud_t pud) {

+    mm_track_pud(pudp);
    set_64bit((unsigned long long *)(pudp), native_pud_val(pud));
}

@@ -54,6 +58,7 @@ static inline void native_set_pud(pud_t
static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr,
pte_t *ptep)
{

+    mm_track_pte(ptep);
    ptep->pte_low = 0;
    smp_wmb();
    ptep->pte_high = 0;
@@ -62,6 +67,9 @@ static inline void native_pte_clear(stru
static inline void native_pmd_clear(pmd_t *pmd)
{
    u32 *tmp = (u32 *)pmd;
+
+    mm_track_pmd(pmd);
+
    *tmp = 0;
    smp_wmb();
    *(tmp + 1) = 0;
@@ -71,6 +79,8 @@ static inline void pud_clear(pud_t *pudp
{
    unsigned long pgd;

+ mm_track_pud(pudp);
+

    set_pud(pudp, __pud(0));

    /*
@@ -93,6 +103,8 @@ static inline pte_t native_ptep_get_and_
{
    pte_t res;

+ mm_track_pte(ptep);
+

    /* xchg acts as a barrier before the setting of the high bits */
    res.pte_low = xchg(&ptep->pte_low, 0);
    res.pte_high = ptep->pte_high;

diff -up linux-2.6-git-track/arch/x86/include/asm/pgtable_64.h.track linux-2.6-git-track/arch/x86/include/asm/pgtable_64.h
@@ -13,6 +13,7 @@

#include <asm/processor.h>
#include <linux/bitops.h>
#include <linux/threads.h>
+#include <asm/mm_track.h>

extern pud_t level3_kernel_pgt[512];
extern pud_t level3_ident_pgt[512];
@@ -46,11 +47,13 @@ void set_pte_vaddr_pud(pud_t *pud_page, static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr,
pte_t *ptep)
{

+    mm_track_pte(ptep);
    *ptep = native_make_pte(0);
}

static inline void native_set_pte(pte_t *ptep, pte_t pte) {

+    mm_track_pte(ptep);
    *ptep = pte;
}

@@ -61,6 +64,7 @@ static inline void native_set_pte_atomic

static inline pte_t native_ptep_get_and_clear(pte_t *xp) {
+ mm_track_pte(xp);

#ifdef CONFIG_SMP
    return native_make_pte(xchg(&xp->pte, 0));

#else
@@ -74,6 +78,7 @@ static inline pte_t native_ptep_get_and_

static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) {

+    mm_track_pmd(pmdp);
    *pmdp = pmd;
}

@@ -84,6 +89,7 @@ static inline void native_pmd_clear(pmd_

static inline void native_set_pud(pud_t *pudp, pud_t pud) {

+    mm_track_pud(pudp);
    *pudp = pud;
}

@@ -94,6 +100,7 @@ static inline void native_pud_clear(pud_

static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd) {

+    mm_track_pgd(pgdp);
    *pgdp = pgd;
}

diff -up linux-2.6-git-track/arch/x86/include/asm/pgtable.h.track linux-2.6-git-track/arch/x86/include/asm/pgtable.h
@@ -91,7 +91,7 @@ static inline void __init paravirt_paget
 */

static inline int pte_dirty(pte_t pte)
{
- return pte_flags(pte) & _PAGE_DIRTY;
+ return pte_flags(pte) & (_PAGE_DIRTY | _PAGE_SOFTDIRTY);
}

static inline int pte_young(pte_t pte)
@@ -158,7 +158,7 @@ static inline pte_t pte_clear_flags(pte_

static inline pte_t pte_mkclean(pte_t pte)
{
- return pte_clear_flags(pte, _PAGE_DIRTY);
+ return pte_clear_flags(pte, (_PAGE_DIRTY | _PAGE_SOFTDIRTY));
}

static inline pte_t pte_mkold(pte_t pte)

diff -up linux-2.6-git-track/arch/x86/include/asm/pgtable_types.h.track linux-2.6-git-track/arch/x86/include/asm/pgtable_types.h
@@ -22,6 +22,7 @@

#define _PAGE_BIT_PAT_LARGE 12 /* On 2MB or 1GB pages */
#define _PAGE_BIT_SPECIAL _PAGE_BIT_UNUSED1
#define _PAGE_BIT_CPA_TEST _PAGE_BIT_UNUSED1
+#define _PAGE_BIT_SOFTDIRTY _PAGE_BIT_HIDDEN
#define _PAGE_BIT_NX 63 /* No execute: only valid after cpuid check */

/* If _PAGE_BIT_PRESENT is clear, we use these: */
@@ -45,6 +46,7 @@
#define _PAGE_PAT_LARGE (_AT(pteval_t, 1) << _PAGE_BIT_PAT_LARGE)
#define _PAGE_SPECIAL (_AT(pteval_t, 1) << _PAGE_BIT_SPECIAL)
#define _PAGE_CPA_TEST (_AT(pteval_t, 1) << _PAGE_BIT_CPA_TEST)
+#define _PAGE_SOFTDIRTY (_AT(pteval_t, 1) << _PAGE_BIT_SOFTDIRTY)
#define __HAVE_ARCH_PTE_SPECIAL

#ifdef CONFIG_KMEMCHECK
@@ -69,7 +71,8 @@

/* Set of bits not changed in pte_modify */
#define _PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \

-             _PAGE_SPECIAL | _PAGE_ACCESSED | _PAGE_DIRTY)
+             _PAGE_SPECIAL | _PAGE_ACCESSED | _PAGE_DIRTY | \
+             _PAGE_SOFTDIRTY)

#define _PAGE_CACHE_MASK (_PAGE_PCD | _PAGE_PWT)
#define _PAGE_CACHE_WB (0)

diff -up linux-2.6-git-track/arch/x86/Kconfig.track linux-2.6-git-track/arch/x86/Kconfig
@@ -1128,6 +1128,17 @@ config DIRECT_GBPAGES
      support it. This can improve the kernel's performance a tiny bit by
      reducing TLB pressure. If in doubt, say "Y".
 
+config TRACK_DIRTY_PAGES
+    bool "Enable dirty page tracking"
+    default n
+    depends on !KMEMCHECK
+    ---help---
+      Turning this on allows third party modules to use a
+      kernel interface that can track dirty page generation
+      in the system.  This can be used to copy/mirror live
+      memory to another system or node.  Most users will
+      say n here.
+

# Common NUMA Features
config NUMA
bool "Numa Memory Allocation and Scheduler Support"

diff -up linux-2.6-git-track/arch/x86/mm/Makefile.track linux-2.6-git-track/arch/x86/mm/Makefile
@@ -19,5 +19,6 @@ obj-$(CONFIG_MMIOTRACE_TEST)    += testmmio

obj-$(CONFIG_NUMA) += numa.o numa_$(BITS).o
obj-$(CONFIG_K8_NUMA) += k8topology_64.o
obj-$(CONFIG_ACPI_NUMA) += srat_$(BITS).o
+obj-$(CONFIG_TRACK_DIRTY_PAGES) += track.o

obj-$(CONFIG_MEMTEST) += memtest.o

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Where is the actual consumer of this? You add a lot of infrastructure but no actualy code using it. Working ar Red Hat should should more than know that we don't add hooks for out of tree junk. Or maybe you should get a free bootcamp ticket from your manager..

Also your patch is extremly whitespace damaged, again quite sad for someone working at a Linux company.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>