arm64, numa: Add numa support for arm64 platforms.

Adding numa support for arm64 based platforms.
This patch adds by default the dummy numa node and
maps all memory and cpus to node 0.
using this patch, numa can be simulated on single node arm64 platforms.

Signed-off-by Ganapatrao Kulkarni <[email protected]>

---
 arch/arm64/Kconfig              |  26 ++
 arch/arm64/include/asm/mmzone.h |  32 +++
 arch/arm64/include/asm/numa.h   |  42 +++
 arch/arm64/kernel/setup.c       |   9 +
 arch/arm64/kernel/smp.c         |   2 +
 arch/arm64/mm/Makefile          |   1 +
 arch/arm64/mm/init.c            |  34 ++-
 arch/arm64/mm/numa.c            | 550 ++++++++++++++++++++++++++++++++++++++++
 8 files changed, 690 insertions(+), 6 deletions(-)
 create mode 100644 arch/arm64/include/asm/mmzone.h
 create mode 100644 arch/arm64/include/asm/numa.h
 create mode 100644 arch/arm64/mm/numa.c

arch/arm64/Kconfig +26 -0

index 43a0c26..fa37a5d 100644
@@ -72,6 +72,7 @@ config ARM64
        select HAVE_PERF_USER_STACK_DUMP
        select HAVE_RCU_TABLE_FREE
        select HAVE_SYSCALL_TRACEPOINTS
+       select HAVE_MEMBLOCK_NODE_MAP if NUMA
        select IRQ_DOMAIN
        select IRQ_FORCED_THREADING
        select MODULES_USE_ELF_RELA

@@ -559,6 +560,31 @@ config HOTPLUG_CPU
          Say Y here to experiment with turning CPUs off and on.  CPUs
          can be controlled through /sys/devices/system/cpu.
 
+# Common NUMA Features
+config NUMA
+       bool "Numa Memory Allocation and Scheduler Support"
+       depends on SMP
+       help
+         Enable NUMA (Non Uniform Memory Access) support.
+
+         The kernel will try to allocate memory used by a CPU on the
+         local memory controller of the CPU and add some more
+         NUMA awareness to the kernel.
+
+config NODES_SHIFT
+       int "Maximum NUMA Nodes (as a power of 2)"
+       range 1 10
+       default "2"
+       depends on NEED_MULTIPLE_NODES
+       help
+         Specify the maximum number of NUMA Nodes available on the target
+         system.  Increases memory reserved to accommodate various tables.
+
+config USE_PERCPU_NUMA_NODE_ID
+       def_bool y
+       depends on NUMA
+
+
 source kernel/Kconfig.preempt
 
 config UP_LATE_INIT

arch/arm64/include/asm/mmzone.h new file +32 -0

new file mode 100644
index 0000000..d27ee66
@@ -0,0 +1,32 @@
+#ifndef __ASM_ARM64_MMZONE_H_
+#define __ASM_ARM64_MMZONE_H_
+
+#ifdef CONFIG_NUMA
+
+#include <linux/mmdebug.h>
+#include <asm/smp.h>
+#include <linux/types.h>
+#include <asm/numa.h>
+
+extern struct pglist_data *node_data[];
+
+#define NODE_DATA(nid)         (node_data[nid])
+
+
+struct numa_memblk {
+       u64                     start;
+       u64                     end;
+       int                     nid;
+};
+
+struct numa_meminfo {
+       int                     nr_blks;
+       struct numa_memblk      blk[NR_NODE_MEMBLKS];
+};
+
+void __init numa_remove_memblk_from(int idx, struct numa_meminfo *mi);
+int __init numa_cleanup_meminfo(struct numa_meminfo *mi);
+void __init numa_reset_distance(void);
+
+#endif /* CONFIG_NUMA */
+#endif /* __ASM_ARM64_MMZONE_H_ */

arch/arm64/include/asm/numa.h new file +42 -0

new file mode 100644
index 0000000..59b834e
@@ -0,0 +1,42 @@
+#ifndef _ASM_NUMA_H
+#define _ASM_NUMA_H
+
+#include <linux/nodemask.h>
+#include <asm/topology.h>
+
+#ifdef CONFIG_NUMA
+
+#define NR_NODE_MEMBLKS                (MAX_NUMNODES * 2)
+#define ZONE_ALIGN (1UL << (MAX_ORDER + PAGE_SHIFT))
+
+/* currently, arm64 implements flat NUMA topology */
+#define parent_node(node)      (node)
+
+/* dummy definitions for pci functions */
+#define pcibus_to_node(node)   0
+#define cpumask_of_pcibus(bus) 0
+
+struct __node_cpu_hwid {
+       u32 node_id;    /* logical node containing this CPU */
+       u64 cpu_hwid;   /* MPIDR for this CPU */
+};
+
+extern struct __node_cpu_hwid node_cpu_hwid[NR_CPUS];
+extern nodemask_t numa_nodes_parsed __initdata;
+
+const struct cpumask *cpumask_of_node(int node);
+/* Mappings between node number and cpus on that node. */
+extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
+
+void __init arm64_numa_init(void);
+int __init numa_add_memblk(u32 nodeid, u64 start, u64 end);
+void numa_store_cpu_info(int cpu);
+void __init build_cpu_to_node_map(void);
+void __init numa_set_distance(int from, int to, int distance);
+#else  /* CONFIG_NUMA */
+static inline void numa_store_cpu_info(int cpu)                { }
+static inline void arm64_numa_init(void)               { }
+static inline void build_cpu_to_node_map(void) { }
+static inline void numa_set_distance(int from, int to, int distance) { }
+#endif /* CONFIG_NUMA */
+#endif /* _ASM_NUMA_H */

arch/arm64/kernel/setup.c +9 -0

index 4d4d7ce..6e101eb 100644
@@ -65,6 +65,7 @@
 #include <asm/efi.h>
 #include <asm/virt.h>
 #include <asm/xen/hypervisor.h>
+#include <asm/numa.h>
 
 unsigned long elf_hwcap __read_mostly;
 EXPORT_SYMBOL_GPL(elf_hwcap);

@@ -439,6 +440,9 @@ static int __init topology_init(void)
 {
        int i;
 
+       for_each_online_node(i)
+               register_one_node(i);
+
        for_each_possible_cpu(i) {
                struct cpu *cpu = &per_cpu(cpu_data.cpu, i);
                cpu->hotpluggable = 1;

@@ -511,7 +515,12 @@ static int c_show(struct seq_file *m, void *v)
                 * "processor".  Give glibc what it expects.
                 */
 #ifdef CONFIG_SMP
+       if (IS_ENABLED(CONFIG_NUMA)) {
+               seq_printf(m, "processor\t: %d", i);
+               seq_printf(m, " [nid: %d]\n", cpu_to_node(i));
+       } else {
                seq_printf(m, "processor\t: %d\n", i);
+       }
 #endif
 
                /*

arch/arm64/kernel/smp.c +2 -0

index 50fb469..ae3e02c 100644
@@ -52,6 +52,7 @@
 #include <asm/sections.h>
 #include <asm/tlbflush.h>
 #include <asm/ptrace.h>
+#include <asm/numa.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/ipi.h>

@@ -124,6 +125,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
 static void smp_store_cpu_info(unsigned int cpuid)
 {
        store_cpu_topology(cpuid);
+       numa_store_cpu_info(cpuid);
 }
 
 /*

arch/arm64/mm/Makefile +1 -0

index 773d37a..bb92d41 100644
@@ -4,3 +4,4 @@ obj-y                           := dma-mapping.o extable.o fault.o init.o \
                                   context.o proc.o pageattr.o
 obj-$(CONFIG_HUGETLB_PAGE)     += hugetlbpage.o
 obj-$(CONFIG_ARM64_PTDUMP)     += dump.o
+obj-$(CONFIG_NUMA)             += numa.o

arch/arm64/mm/init.c +28 -6

index 54da32e..cab384b 100644
@@ -42,6 +42,7 @@
 #include <asm/sizes.h>
 #include <asm/tlb.h>
 #include <asm/alternative.h>
+#include <asm/numa.h>
 
 #include "mm.h"
 

@@ -77,6 +78,20 @@ static phys_addr_t max_zone_dma_phys(void)
        return min(offset + (1ULL << 32), memblock_end_of_DRAM());
 }
 
+#ifdef CONFIG_NUMA
+static void __init zone_sizes_init(unsigned long min, unsigned long max)
+{
+       unsigned long max_zone_pfns[MAX_NR_ZONES];
+
+       memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
+       if (IS_ENABLED(CONFIG_ZONE_DMA))
+               max_zone_pfns[ZONE_DMA] = PFN_DOWN(max_zone_dma_phys());
+       max_zone_pfns[ZONE_NORMAL] = max;
+
+       free_area_init_nodes(max_zone_pfns);
+}
+
+#else
 static void __init zone_sizes_init(unsigned long min, unsigned long max)
 {
        struct memblock_region *reg;

@@ -115,6 +130,7 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max)
 
        free_area_init_node(0, zone_size, min, zhole_size);
 }
+#endif /* CONFIG_NUMA */
 
 #ifdef CONFIG_HAVE_ARCH_PFN_VALID
 int pfn_valid(unsigned long pfn)

@@ -132,10 +148,15 @@ static void arm64_memory_present(void)
 static void arm64_memory_present(void)
 {
        struct memblock_region *reg;
+       int nid = 0;
 
-       for_each_memblock(memory, reg)
-               memory_present(0, memblock_region_memory_base_pfn(reg),
-                              memblock_region_memory_end_pfn(reg));
+       for_each_memblock(memory, reg) {
+#ifdef CONFIG_NUMA
+               nid = reg->nid;
+#endif
+               memory_present(nid, memblock_region_memory_base_pfn(reg),
+                               memblock_region_memory_end_pfn(reg));
+       }
 }
 #endif
 

@@ -200,6 +221,10 @@ void __init bootmem_init(void)
        min = PFN_UP(memblock_start_of_DRAM());
        max = PFN_DOWN(memblock_end_of_DRAM());
 
+       high_memory = __va((max << PAGE_SHIFT) - 1) + 1;
+       max_pfn = max_low_pfn = max;
+
+       arm64_numa_init();
        /*
         * Sparsemem tries to allocate bootmem in memory_present(), so must be
         * done after the fixed reservations.

@@ -208,9 +233,6 @@ void __init bootmem_init(void)
 
        sparse_init();
        zone_sizes_init(min, max);
-
-       high_memory = __va((max << PAGE_SHIFT) - 1) + 1;
-       max_pfn = max_low_pfn = max;
 }
 
 #ifndef CONFIG_SPARSEMEM_VMEMMAP

arch/arm64/mm/numa.c new file +550 -1

new file mode 100644
index 0000000..2be83de
@@ -0,0 +1,550 @@
+/*
+ * NUMA support, based on the x86 implementation.
+ *
+ * Copyright (C) 2015 Cavium Inc.
+ * Author: Ganapatrao Kulkarni <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/string.h>
+#include <linux/init.h>
+#include <linux/bootmem.h>
+#include <linux/memblock.h>
+#include <linux/ctype.h>
+#include <linux/module.h>
+#include <linux/nodemask.h>
+#include <linux/sched.h>
+#include <linux/topology.h>
+#include <linux/mmzone.h>
+
+#include <asm/smp_plat.h>
+
+int __initdata numa_off;
+nodemask_t numa_nodes_parsed __initdata;
+static int numa_distance_cnt;
+static u8 *numa_distance;
+static u8 dummy_numa_enabled;
+
+struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;
+EXPORT_SYMBOL(node_data);
+
+struct __node_cpu_hwid node_cpu_hwid[NR_CPUS];
+static struct numa_meminfo numa_meminfo;
+
+static __init int numa_setup(char *opt)
+{
+       if (!opt)
+               return -EINVAL;
+       if (!strncmp(opt, "off", 3)) {
+               pr_info("%s\n", "NUMA turned off");
+               numa_off = 1;
+       }
+       return 0;
+}
+early_param("numa", numa_setup);
+
+cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
+EXPORT_SYMBOL(node_to_cpumask_map);
+
+int cpu_to_node_map[NR_CPUS];
+EXPORT_SYMBOL(cpu_to_node_map);
+
+/*
+ * Returns a pointer to the bitmask of CPUs on Node 'node'.
+ */
+const struct cpumask *cpumask_of_node(int node)
+{
+       if (node >= nr_node_ids) {
+               pr_warn("cpumask_of_node(%d): node > nr_node_ids(%d)\n",
+                       node, nr_node_ids);
+               dump_stack();
+               return cpu_none_mask;
+       }
+       if (node_to_cpumask_map[node] == NULL) {
+               pr_warn("cpumask_of_node(%d): no node_to_cpumask_map!\n",
+                       node);
+               dump_stack();
+               return cpu_online_mask;
+       }
+       return node_to_cpumask_map[node];
+}
+EXPORT_SYMBOL(cpumask_of_node);
+
+void numa_clear_node(int cpu)
+{
+       set_cpu_numa_node(cpu, NUMA_NO_NODE);
+}
+
+void map_cpu_to_node(int cpu, int nid)
+{
+       if (nid < 0) { /* just initialize by zero */
+               cpu_to_node_map[cpu] = 0;
+               return;
+       }
+
+       cpu_to_node_map[cpu] = nid;
+       cpumask_set_cpu(cpu, node_to_cpumask_map[nid]);
+       set_numa_node(nid);
+}
+
+/**
+ * build_cpu_to_node_map - setup cpu to node and node to cpumask arrays
+ *
+ * Build cpu to node mapping and initialize the per node cpu masks using
+ * info from the node_cpuid array handed to us by ACPI or DT.
+ */
+void __init build_cpu_to_node_map(void)
+{
+       int cpu, i, node;
+
+       for (node = 0; node < MAX_NUMNODES; node++)
+               cpumask_clear(node_to_cpumask_map[node]);
+
+       for_each_possible_cpu(cpu) {
+               node = NUMA_NO_NODE;
+               for_each_possible_cpu(i) {
+                       if (cpu_logical_map(cpu) == node_cpu_hwid[i].cpu_hwid) {
+                               node = node_cpu_hwid[i].node_id;
+                               break;
+                       }
+               }
+               map_cpu_to_node(cpu, node);
+       }
+}
+/*
+ * Allocate node_to_cpumask_map based on number of available nodes
+ * Requires node_possible_map to be valid.
+ *
+ * Note: cpumask_of_node() is not valid until after this is done.
+ * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.)
+ */
+void __init setup_node_to_cpumask_map(void)
+{
+       unsigned int node;
+
+       /* setup nr_node_ids if not done yet */
+       if (nr_node_ids == MAX_NUMNODES)
+               setup_nr_node_ids();
+
+       /* allocate the map */
+       for (node = 0; node < nr_node_ids; node++)
+               alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]);
+
+       /* cpumask_of_node() will now work */
+       pr_debug("Node to cpumask map for %d nodes\n", nr_node_ids);
+}
+
+/*
+ *  Set the cpu to node and mem mapping
+ */
+void numa_store_cpu_info(int cpu)
+{
+       if (dummy_numa_enabled) {
+               /* set to default */
+               node_cpu_hwid[cpu].node_id  =  0;
+               node_cpu_hwid[cpu].cpu_hwid = cpu_logical_map(cpu);
+       }
+       map_cpu_to_node(cpu, node_cpu_hwid[cpu].node_id);
+}
+
+/**
+ * numa_add_memblk_to - Add one numa_memblk to a numa_meminfo
+ */
+
+static int __init numa_add_memblk_to(int nid, u64 start, u64 end,
+                                    struct numa_meminfo *mi)
+{
+       /* ignore zero length blks */
+       if (start == end)
+               return 0;
+
+       /* whine about and ignore invalid blks */
+       if (start > end || nid < 0 || nid >= MAX_NUMNODES) {
+               pr_warn("NUMA: Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n",
+                               nid, start, end - 1);
+               return 0;
+       }
+
+       if (mi->nr_blks >= NR_NODE_MEMBLKS) {
+               pr_err("NUMA: too many memblk ranges\n");
+               return -EINVAL;
+       }
+
+       pr_info("NUMA: Adding memblock %d [0x%llx - 0x%llx] on node %d\n",
+                       mi->nr_blks, start, end, nid);
+       mi->blk[mi->nr_blks].start = start;
+       mi->blk[mi->nr_blks].end = end;
+       mi->blk[mi->nr_blks].nid = nid;
+       mi->nr_blks++;
+       return 0;
+}
+
+/**
+ * numa_add_memblk - Add one numa_memblk to numa_meminfo
+ * @nid: NUMA node ID of the new memblk
+ * @start: Start address of the new memblk
+ * @end: End address of the new memblk
+ *
+ * Add a new memblk to the default numa_meminfo.
+ *
+ * RETURNS:
+ * 0 on success, -errno on failure.
+ */
+#define MAX_PHYS_ADDR  ((phys_addr_t)~0)
+
+int __init numa_add_memblk(u32 nid, u64 base, u64 end)
+{
+       const u64 phys_offset = __pa(PAGE_OFFSET);
+
+       base &= PAGE_MASK;
+       end &= PAGE_MASK;
+
+       if (base > MAX_PHYS_ADDR) {
+               pr_warn("NUMA: Ignoring memory block 0x%llx - 0x%llx\n",
+                               base, base + end);
+               return -ENOMEM;
+       }
+
+       if (base + end > MAX_PHYS_ADDR) {
+               pr_info("NUMA: Ignoring memory range 0x%lx - 0x%llx\n",
+                               ULONG_MAX, base + end);
+               end = MAX_PHYS_ADDR - base;
+       }
+
+       if (base + end < phys_offset) {
+               pr_warn("NUMA: Ignoring memory block 0x%llx - 0x%llx\n",
+                          base, base + end);
+               return -ENOMEM;
+       }
+       if (base < phys_offset) {
+               pr_info("NUMA: Ignoring memory range 0x%llx - 0x%llx\n",
+                          base, phys_offset);
+               end -= phys_offset - base;
+               base = phys_offset;
+       }
+
+       return numa_add_memblk_to(nid, base, base + end, &numa_meminfo);
+}
+EXPORT_SYMBOL(numa_add_memblk);
+
+/* Initialize NODE_DATA for a node on the local memory */
+static void __init setup_node_data(int nid, u64 start, u64 end)
+{
+       const size_t nd_size = roundup(sizeof(pg_data_t), PAGE_SIZE);
+       u64 nd_pa;
+       void *nd;
+       int tnid;
+
+       start = roundup(start, ZONE_ALIGN);
+
+       pr_info("Initmem setup node %d [mem %#010Lx-%#010Lx]\n",
+              nid, start, end - 1);
+
+       /*
+        * Allocate node data.  Try node-local memory and then any node.
+        */
+       nd_pa = memblock_alloc_nid(nd_size, SMP_CACHE_BYTES, nid);
+       if (!nd_pa) {
+               nd_pa = __memblock_alloc_base(nd_size, SMP_CACHE_BYTES,
+                                             MEMBLOCK_ALLOC_ACCESSIBLE);
+               if (!nd_pa) {
+                       pr_err("Cannot find %zu bytes in node %d\n",
+                              nd_size, nid);
+                       return;
+               }
+       }
+       nd = __va(nd_pa);
+
+       /* report and initialize */
+       pr_info("  NODE_DATA [mem %#010Lx-%#010Lx]\n",
+              nd_pa, nd_pa + nd_size - 1);
+       tnid = early_pfn_to_nid(nd_pa >> PAGE_SHIFT);
+       if (tnid != nid)
+               pr_info("    NODE_DATA(%d) on node %d\n", nid, tnid);
+
+       node_data[nid] = nd;
+       memset(NODE_DATA(nid), 0, sizeof(pg_data_t));
+       NODE_DATA(nid)->node_id = nid;
+       NODE_DATA(nid)->node_start_pfn = start >> PAGE_SHIFT;
+       NODE_DATA(nid)->node_spanned_pages = (end - start) >> PAGE_SHIFT;
+
+       node_set_online(nid);
+}
+
+/*
+ * Set nodes, which have memory in @mi, in *@nodemask.
+ */
+static void __init numa_nodemask_from_meminfo(nodemask_t *nodemask,
+                                             const struct numa_meminfo *mi)
+{
+       int i;
+
+       for (i = 0; i < ARRAY_SIZE(mi->blk); i++)
+               if (mi->blk[i].start != mi->blk[i].end &&
+                   mi->blk[i].nid != NUMA_NO_NODE)
+                       node_set(mi->blk[i].nid, *nodemask);
+}
+
+/*
+ * Sanity check to catch more bad NUMA configurations (they are amazingly
+ * common).  Make sure the nodes cover all memory.
+ */
+static bool __init numa_meminfo_cover_memory(const struct numa_meminfo *mi)
+{
+       u64 numaram, totalram;
+       int i;
+
+       numaram = 0;
+       for (i = 0; i < mi->nr_blks; i++) {
+               u64 s = mi->blk[i].start >> PAGE_SHIFT;
+               u64 e = mi->blk[i].end >> PAGE_SHIFT;
+
+               numaram += e - s;
+               numaram -= __absent_pages_in_range(mi->blk[i].nid, s, e);
+               if ((s64)numaram < 0)
+                       numaram = 0;
+       }
+
+       totalram = max_pfn - absent_pages_in_range(0, max_pfn);
+
+       /* We seem to lose 3 pages somewhere. Allow 1M of slack. */
+       if ((s64)(totalram - numaram) >= (1 << (20 - PAGE_SHIFT))) {
+               pr_err("NUMA: nodes only cover %lluMB of your %lluMB Total RAM. Not used.\n",
+                      (numaram << PAGE_SHIFT) >> 20,
+                      (totalram << PAGE_SHIFT) >> 20);
+               return false;
+       }
+       return true;
+}
+
+/**
+ * numa_reset_distance - Reset NUMA distance table
+ *
+ * The current table is freed.  The next numa_set_distance() call will
+ * create a new one.
+ */
+void __init numa_reset_distance(void)
+{
+       size_t size = numa_distance_cnt * numa_distance_cnt *
+               sizeof(numa_distance[0]);
+
+       /* numa_distance could be 1LU marking allocation failure, test cnt */
+       if (numa_distance_cnt)
+               memblock_free(__pa(numa_distance), size);
+       numa_distance_cnt = 0;
+       numa_distance = NULL;   /* enable table creation */
+}
+
+static int __init numa_alloc_distance(void)
+{
+       nodemask_t nodes_parsed;
+       size_t size;
+       int i, j, cnt = 0;
+       u64 phys;
+
+       /* size the new table and allocate it */
+       nodes_parsed = numa_nodes_parsed;
+       numa_nodemask_from_meminfo(&nodes_parsed, &numa_meminfo);
+
+       for_each_node_mask(i, nodes_parsed)
+               cnt = i;
+       cnt++;
+       size = cnt * cnt * sizeof(numa_distance[0]);
+
+       phys = memblock_find_in_range(0, PFN_PHYS(max_pfn),
+                                     size, PAGE_SIZE);
+       if (!phys) {
+               pr_warn("NUMA: Warning: can't allocate distance table!\n");
+               /* don't retry until explicitly reset */
+               numa_distance = (void *)1LU;
+               return -ENOMEM;
+       }
+       memblock_reserve(phys, size);
+
+       numa_distance = __va(phys);
+       numa_distance_cnt = cnt;
+
+       /* fill with the default distances */
+       for (i = 0; i < cnt; i++)
+               for (j = 0; j < cnt; j++)
+                       numa_distance[i * cnt + j] = i == j ?
+                               LOCAL_DISTANCE : REMOTE_DISTANCE;
+       pr_debug("NUMA: Initialized distance table, cnt=%d\n", cnt);
+
+       return 0;
+}
+
+/**
+ * numa_set_distance - Set NUMA distance from one NUMA to another
+ * @from: the 'from' node to set distance
+ * @to: the 'to'  node to set distance
+ * @distance: NUMA distance
+ *
+ * Set the distance from node @from to @to to @distance.  If distance table
+ * doesn't exist, one which is large enough to accommodate all the currently
+ * known nodes will be created.
+ *
+ * If such table cannot be allocated, a warning is printed and further
+ * calls are ignored until the distance table is reset with
+ * numa_reset_distance().
+ *
+ * If @from or @to is higher than the highest known node or lower than zero
+ * at the time of table creation or @distance doesn't make sense, the call
+ * is ignored.
+ * This is to allow simplification of specific NUMA config implementations.
+ */
+void __init numa_set_distance(int from, int to, int distance)
+{
+       if (!numa_distance && numa_alloc_distance() < 0)
+               return;
+
+       if (from >= numa_distance_cnt || to >= numa_distance_cnt ||
+                       from < 0 || to < 0) {
+               pr_warn_once("NUMA: Warning: node ids are out of bound, from=%d to=%d distance=%d\n",
+                           from, to, distance);
+               return;
+       }
+
+       if ((u8)distance != distance ||
+           (from == to && distance != LOCAL_DISTANCE)) {
+               pr_warn_once("NUMA: Warning: invalid distance parameter, from=%d to=%d distance=%d\n",
+                            from, to, distance);
+               return;
+       }
+
+       numa_distance[from * numa_distance_cnt + to] = distance;
+}
+EXPORT_SYMBOL(numa_set_distance);
+
+int __node_distance(int from, int to)
+{
+       if (from >= numa_distance_cnt || to >= numa_distance_cnt)
+               return from == to ? LOCAL_DISTANCE : REMOTE_DISTANCE;
+       return numa_distance[from * numa_distance_cnt + to];
+}
+EXPORT_SYMBOL(__node_distance);
+
+static int __init numa_register_memblks(struct numa_meminfo *mi)
+{
+       unsigned long uninitialized_var(pfn_align);
+       int i, nid;
+
+       /* Account for nodes with cpus and no memory */
+       node_possible_map = numa_nodes_parsed;
+       numa_nodemask_from_meminfo(&node_possible_map, mi);
+       if (WARN_ON(nodes_empty(node_possible_map)))
+               return -EINVAL;
+
+       for (i = 0; i < mi->nr_blks; i++) {
+               struct numa_memblk *mb = &mi->blk[i];
+
+               memblock_set_node(mb->start, mb->end - mb->start,
+                                 &memblock.memory, mb->nid);
+       }
+
+       /*
+        * If sections array is gonna be used for pfn -> nid mapping, check
+        * whether its granularity is fine enough.
+        */
+#ifdef NODE_NOT_IN_PAGE_FLAGS
+       pfn_align = node_map_pfn_alignment();
+       if (pfn_align && pfn_align < PAGES_PER_SECTION) {
+               pr_warn("Node alignment %lluMB < min %lluMB, rejecting NUMA config\n",
+                      PFN_PHYS(pfn_align) >> 20,
+                      PFN_PHYS(PAGES_PER_SECTION) >> 20);
+               return -EINVAL;
+       }
+#endif
+       if (!numa_meminfo_cover_memory(mi))
+               return -EINVAL;
+
+       /* Finally register nodes. */
+       for_each_node_mask(nid, node_possible_map) {
+               u64 start = PFN_PHYS(max_pfn);
+               u64 end = 0;
+
+               for (i = 0; i < mi->nr_blks; i++) {
+                       if (nid != mi->blk[i].nid)
+                               continue;
+                       start = min(mi->blk[i].start, start);
+                       end = max(mi->blk[i].end, end);
+               }
+
+               if (start < end)
+                       setup_node_data(nid, start, end);
+       }
+
+       /* Dump memblock with node info and return. */
+       memblock_dump_all();
+       return 0;
+}
+
+static int __init numa_init(int (*init_func)(void))
+{
+       int ret, i;
+
+       nodes_clear(numa_nodes_parsed);
+       nodes_clear(node_possible_map);
+       nodes_clear(node_online_map);
+
+       ret = init_func();
+       if (ret < 0)
+               return ret;
+
+       ret = numa_register_memblks(&numa_meminfo);
+       if (ret < 0)
+               return ret;
+
+       for (i = 0; i < nr_cpu_ids; i++)
+               numa_clear_node(i);
+
+       setup_node_to_cpumask_map();
+       build_cpu_to_node_map();
+       return 0;
+}
+
+/**
+ * dummy_numa_init - Fallback dummy NUMA init
+ *
+ * Used if there's no underlying NUMA architecture, NUMA initialization
+ * fails, or NUMA is disabled on the command line.
+ *
+ * Must online at least one node and add memory blocks that cover all
+ * allowed memory.  This function must not fail.
+ */
+static int __init dummy_numa_init(void)
+{
+       pr_info("%s\n", "No NUMA configuration found");
+       pr_info("Faking a node at [mem %#018Lx-%#018Lx]\n",
+              0LLU, PFN_PHYS(max_pfn) - 1);
+       node_set(0, numa_nodes_parsed);
+       numa_add_memblk(0, 0, PFN_PHYS(max_pfn));
+       dummy_numa_enabled = 1;
+
+       return 0;
+}
+
+/**
+ * arm64_numa_init - Initialize NUMA
+ *
+ * Try each configured NUMA initialization method until one succeeds.  The
+ * last fallback is dummy single node config encomapssing whole memory and
+ * never fails.
+ */
+void __init arm64_numa_init(void)
+{
+       numa_init(dummy_numa_init);
+}
--

1.8.1.4

Quoted 6 lines · Ganapatrao Kulkarni

Adding numa support for arm64 based platforms.
This patch adds by default the dummy numa node and
maps all memory and cpus to node 0.
using this patch, numa can be simulated on single node arm64 platforms.

Signed-off-by Ganapatrao Kulkarni <[email protected]>

can you please review this patch.

Quoted 7 lines · Ganapatrao Kulkarni

Quoted 5 lines · Ganapatrao Kulkarni

Adding numa support for arm64 based platforms.
This patch adds by default the dummy numa node and
maps all memory and cpus to node 0.
using this patch, numa can be simulated on single node arm64 platforms.

Quoted 1 line

Quoted 1 line

Signed-off-by Ganapatrao Kulkarni <[email protected]>

[trimmed ~850 lines of context]

Quoted 1 line

can you please review this patch.

It's the middle of the merge window, please be patient.

Will

Adding numa support for arm64 based platforms.
This patch adds by default the dummy numa node and
maps all memory and cpus to node 0.
using this patch, numa can be simulated on single node arm64 platforms.

Signed-off-by Ganapatrao Kulkarni <[email protected]>

---
 arch/arm64/Kconfig              |  26 ++
 arch/arm64/include/asm/mmzone.h |  32 +++
 arch/arm64/include/asm/numa.h   |  42 +++
 arch/arm64/kernel/setup.c       |   9 +
 arch/arm64/kernel/smp.c         |   2 +
 arch/arm64/mm/Makefile          |   1 +
 arch/arm64/mm/init.c            |  34 ++-
 arch/arm64/mm/numa.c            | 550 ++++++++++++++++++++++++++++++++++++++++
 8 files changed, 690 insertions(+), 6 deletions(-)
 create mode 100644 arch/arm64/include/asm/mmzone.h
 create mode 100644 arch/arm64/include/asm/numa.h
 create mode 100644 arch/arm64/mm/numa.c

arch/arm64/Kconfig +26 -0

index 43a0c26..fa37a5d 100644
@@ -72,6 +72,7 @@ config ARM64
        select HAVE_PERF_USER_STACK_DUMP
        select HAVE_RCU_TABLE_FREE
        select HAVE_SYSCALL_TRACEPOINTS
+       select HAVE_MEMBLOCK_NODE_MAP if NUMA
        select IRQ_DOMAIN
        select IRQ_FORCED_THREADING
        select MODULES_USE_ELF_RELA

@@ -559,6 +560,31 @@ config HOTPLUG_CPU
          Say Y here to experiment with turning CPUs off and on.  CPUs
          can be controlled through /sys/devices/system/cpu.
 
+# Common NUMA Features
+config NUMA
+       bool "Numa Memory Allocation and Scheduler Support"
+       depends on SMP
+       help
+         Enable NUMA (Non Uniform Memory Access) support.
+
+         The kernel will try to allocate memory used by a CPU on the
+         local memory controller of the CPU and add some more
+         NUMA awareness to the kernel.
+
+config NODES_SHIFT
+       int "Maximum NUMA Nodes (as a power of 2)"
+       range 1 10
+       default "2"
+       depends on NEED_MULTIPLE_NODES
+       help
+         Specify the maximum number of NUMA Nodes available on the target
+         system.  Increases memory reserved to accommodate various tables.
+
+config USE_PERCPU_NUMA_NODE_ID
+       def_bool y
+       depends on NUMA
+
+
 source kernel/Kconfig.preempt
 
 config UP_LATE_INIT

arch/arm64/include/asm/mmzone.h new file +32 -0

new file mode 100644
index 0000000..d27ee66
@@ -0,0 +1,32 @@
+#ifndef __ASM_ARM64_MMZONE_H_
+#define __ASM_ARM64_MMZONE_H_
+
+#ifdef CONFIG_NUMA
+
+#include <linux/mmdebug.h>
+#include <asm/smp.h>
+#include <linux/types.h>
+#include <asm/numa.h>
+
+extern struct pglist_data *node_data[];
+
+#define NODE_DATA(nid)         (node_data[nid])
+
+
+struct numa_memblk {
+       u64                     start;
+       u64                     end;
+       int                     nid;
+};
+
+struct numa_meminfo {
+       int                     nr_blks;
+       struct numa_memblk      blk[NR_NODE_MEMBLKS];
+};
+
+void __init numa_remove_memblk_from(int idx, struct numa_meminfo *mi);
+int __init numa_cleanup_meminfo(struct numa_meminfo *mi);
+void __init numa_reset_distance(void);
+
+#endif /* CONFIG_NUMA */
+#endif /* __ASM_ARM64_MMZONE_H_ */

arch/arm64/include/asm/numa.h new file +42 -0

new file mode 100644
index 0000000..59b834e
@@ -0,0 +1,42 @@
+#ifndef _ASM_NUMA_H
+#define _ASM_NUMA_H
+
+#include <linux/nodemask.h>
+#include <asm/topology.h>
+
+#ifdef CONFIG_NUMA
+
+#define NR_NODE_MEMBLKS                (MAX_NUMNODES * 2)
+#define ZONE_ALIGN (1UL << (MAX_ORDER + PAGE_SHIFT))
+
+/* currently, arm64 implements flat NUMA topology */
+#define parent_node(node)      (node)
+
+/* dummy definitions for pci functions */
+#define pcibus_to_node(node)   0
+#define cpumask_of_pcibus(bus) 0
+
+struct __node_cpu_hwid {
+       u32 node_id;    /* logical node containing this CPU */
+       u64 cpu_hwid;   /* MPIDR for this CPU */
+};
+
+extern struct __node_cpu_hwid node_cpu_hwid[NR_CPUS];
+extern nodemask_t numa_nodes_parsed __initdata;
+
+const struct cpumask *cpumask_of_node(int node);
+/* Mappings between node number and cpus on that node. */
+extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
+
+void __init arm64_numa_init(void);
+int __init numa_add_memblk(u32 nodeid, u64 start, u64 end);
+void numa_store_cpu_info(int cpu);
+void __init build_cpu_to_node_map(void);
+void __init numa_set_distance(int from, int to, int distance);
+#else  /* CONFIG_NUMA */
+static inline void numa_store_cpu_info(int cpu)                { }
+static inline void arm64_numa_init(void)               { }
+static inline void build_cpu_to_node_map(void) { }
+static inline void numa_set_distance(int from, int to, int distance) { }
+#endif /* CONFIG_NUMA */
+#endif /* _ASM_NUMA_H */

arch/arm64/kernel/setup.c +9 -0

index 4d4d7ce..6e101eb 100644
@@ -65,6 +65,7 @@
 #include <asm/efi.h>
 #include <asm/virt.h>
 #include <asm/xen/hypervisor.h>
+#include <asm/numa.h>
 
 unsigned long elf_hwcap __read_mostly;
 EXPORT_SYMBOL_GPL(elf_hwcap);

@@ -439,6 +440,9 @@ static int __init topology_init(void)
 {
        int i;
 
+       for_each_online_node(i)
+               register_one_node(i);
+
        for_each_possible_cpu(i) {
                struct cpu *cpu = &per_cpu(cpu_data.cpu, i);
                cpu->hotpluggable = 1;

@@ -511,7 +515,12 @@ static int c_show(struct seq_file *m, void *v)
                 * "processor".  Give glibc what it expects.
                 */
 #ifdef CONFIG_SMP
+       if (IS_ENABLED(CONFIG_NUMA)) {
+               seq_printf(m, "processor\t: %d", i);
+               seq_printf(m, " [nid: %d]\n", cpu_to_node(i));
+       } else {
                seq_printf(m, "processor\t: %d\n", i);
+       }
 #endif
 
                /*

arch/arm64/kernel/smp.c +2 -0

index 50fb469..ae3e02c 100644
@@ -52,6 +52,7 @@
 #include <asm/sections.h>
 #include <asm/tlbflush.h>
 #include <asm/ptrace.h>
+#include <asm/numa.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/ipi.h>

@@ -124,6 +125,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
 static void smp_store_cpu_info(unsigned int cpuid)
 {
        store_cpu_topology(cpuid);
+       numa_store_cpu_info(cpuid);
 }
 
 /*

arch/arm64/mm/Makefile +1 -0

index 773d37a..bb92d41 100644
@@ -4,3 +4,4 @@ obj-y                           := dma-mapping.o extable.o fault.o init.o \
                                   context.o proc.o pageattr.o
 obj-$(CONFIG_HUGETLB_PAGE)     += hugetlbpage.o
 obj-$(CONFIG_ARM64_PTDUMP)     += dump.o
+obj-$(CONFIG_NUMA)             += numa.o

arch/arm64/mm/init.c +28 -6

index 54da32e..cab384b 100644
@@ -42,6 +42,7 @@
 #include <asm/sizes.h>
 #include <asm/tlb.h>
 #include <asm/alternative.h>
+#include <asm/numa.h>
 
 #include "mm.h"
 

@@ -77,6 +78,20 @@ static phys_addr_t max_zone_dma_phys(void)
        return min(offset + (1ULL << 32), memblock_end_of_DRAM());
 }
 
+#ifdef CONFIG_NUMA
+static void __init zone_sizes_init(unsigned long min, unsigned long max)
+{
+       unsigned long max_zone_pfns[MAX_NR_ZONES];
+
+       memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
+       if (IS_ENABLED(CONFIG_ZONE_DMA))
+               max_zone_pfns[ZONE_DMA] = PFN_DOWN(max_zone_dma_phys());
+       max_zone_pfns[ZONE_NORMAL] = max;
+
+       free_area_init_nodes(max_zone_pfns);
+}
+
+#else
 static void __init zone_sizes_init(unsigned long min, unsigned long max)
 {
        struct memblock_region *reg;

@@ -115,6 +130,7 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max)
 
        free_area_init_node(0, zone_size, min, zhole_size);
 }
+#endif /* CONFIG_NUMA */
 
 #ifdef CONFIG_HAVE_ARCH_PFN_VALID
 int pfn_valid(unsigned long pfn)

@@ -132,10 +148,15 @@ static void arm64_memory_present(void)
 static void arm64_memory_present(void)
 {
        struct memblock_region *reg;
+       int nid = 0;
 
-       for_each_memblock(memory, reg)
-               memory_present(0, memblock_region_memory_base_pfn(reg),
-                              memblock_region_memory_end_pfn(reg));
+       for_each_memblock(memory, reg) {
+#ifdef CONFIG_NUMA
+               nid = reg->nid;
+#endif
+               memory_present(nid, memblock_region_memory_base_pfn(reg),
+                               memblock_region_memory_end_pfn(reg));
+       }
 }
 #endif
 

@@ -200,6 +221,10 @@ void __init bootmem_init(void)
        min = PFN_UP(memblock_start_of_DRAM());
        max = PFN_DOWN(memblock_end_of_DRAM());
 
+       high_memory = __va((max << PAGE_SHIFT) - 1) + 1;
+       max_pfn = max_low_pfn = max;
+
+       arm64_numa_init();
        /*
         * Sparsemem tries to allocate bootmem in memory_present(), so must be
         * done after the fixed reservations.

@@ -208,9 +233,6 @@ void __init bootmem_init(void)
 
        sparse_init();
        zone_sizes_init(min, max);
-
-       high_memory = __va((max << PAGE_SHIFT) - 1) + 1;
-       max_pfn = max_low_pfn = max;
 }
 
 #ifndef CONFIG_SPARSEMEM_VMEMMAP

arch/arm64/mm/numa.c new file +550 -1

new file mode 100644
index 0000000..2be83de
@@ -0,0 +1,550 @@
+/*
+ * NUMA support, based on the x86 implementation.
+ *
+ * Copyright (C) 2015 Cavium Inc.
+ * Author: Ganapatrao Kulkarni <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/string.h>
+#include <linux/init.h>
+#include <linux/bootmem.h>
+#include <linux/memblock.h>
+#include <linux/ctype.h>
+#include <linux/module.h>
+#include <linux/nodemask.h>
+#include <linux/sched.h>
+#include <linux/topology.h>
+#include <linux/mmzone.h>
+
+#include <asm/smp_plat.h>
+
+int __initdata numa_off;
+nodemask_t numa_nodes_parsed __initdata;
+static int numa_distance_cnt;
+static u8 *numa_distance;
+static u8 dummy_numa_enabled;
+
+struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;
+EXPORT_SYMBOL(node_data);
+
+struct __node_cpu_hwid node_cpu_hwid[NR_CPUS];
+static struct numa_meminfo numa_meminfo;
+
+static __init int numa_setup(char *opt)
+{
+       if (!opt)
+               return -EINVAL;
+       if (!strncmp(opt, "off", 3)) {
+               pr_info("%s\n", "NUMA turned off");
+               numa_off = 1;
+       }
+       return 0;
+}
+early_param("numa", numa_setup);
+
+cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
+EXPORT_SYMBOL(node_to_cpumask_map);
+
+int cpu_to_node_map[NR_CPUS];
+EXPORT_SYMBOL(cpu_to_node_map);
+
+/*
+ * Returns a pointer to the bitmask of CPUs on Node 'node'.
+ */
+const struct cpumask *cpumask_of_node(int node)
+{
+       if (node >= nr_node_ids) {
+               pr_warn("cpumask_of_node(%d): node > nr_node_ids(%d)\n",
+                       node, nr_node_ids);
+               dump_stack();
+               return cpu_none_mask;
+       }
+       if (node_to_cpumask_map[node] == NULL) {
+               pr_warn("cpumask_of_node(%d): no node_to_cpumask_map!\n",
+                       node);
+               dump_stack();
+               return cpu_online_mask;
+       }
+       return node_to_cpumask_map[node];
+}
+EXPORT_SYMBOL(cpumask_of_node);
+
+void numa_clear_node(int cpu)
+{
+       set_cpu_numa_node(cpu, NUMA_NO_NODE);
+}
+
+void map_cpu_to_node(int cpu, int nid)
+{
+       if (nid < 0) { /* just initialize by zero */
+               cpu_to_node_map[cpu] = 0;
+               return;
+       }
+
+       cpu_to_node_map[cpu] = nid;
+       cpumask_set_cpu(cpu, node_to_cpumask_map[nid]);
+       set_numa_node(nid);
+}
+
+/**
+ * build_cpu_to_node_map - setup cpu to node and node to cpumask arrays
+ *
+ * Build cpu to node mapping and initialize the per node cpu masks using
+ * info from the node_cpuid array handed to us by ACPI or DT.
+ */
+void __init build_cpu_to_node_map(void)
+{
+       int cpu, i, node;
+
+       for (node = 0; node < MAX_NUMNODES; node++)
+               cpumask_clear(node_to_cpumask_map[node]);
+
+       for_each_possible_cpu(cpu) {
+               node = NUMA_NO_NODE;
+               for_each_possible_cpu(i) {
+                       if (cpu_logical_map(cpu) == node_cpu_hwid[i].cpu_hwid) {
+                               node = node_cpu_hwid[i].node_id;
+                               break;
+                       }
+               }
+               map_cpu_to_node(cpu, node);
+       }
+}
+/*
+ * Allocate node_to_cpumask_map based on number of available nodes
+ * Requires node_possible_map to be valid.
+ *
+ * Note: cpumask_of_node() is not valid until after this is done.
+ * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.)
+ */
+void __init setup_node_to_cpumask_map(void)
+{
+       unsigned int node;
+
+       /* setup nr_node_ids if not done yet */
+       if (nr_node_ids == MAX_NUMNODES)
+               setup_nr_node_ids();
+
+       /* allocate the map */
+       for (node = 0; node < nr_node_ids; node++)
+               alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]);
+
+       /* cpumask_of_node() will now work */
+       pr_debug("Node to cpumask map for %d nodes\n", nr_node_ids);
+}
+
+/*
+ *  Set the cpu to node and mem mapping
+ */
+void numa_store_cpu_info(int cpu)
+{
+       if (dummy_numa_enabled) {
+               /* set to default */
+               node_cpu_hwid[cpu].node_id  =  0;
+               node_cpu_hwid[cpu].cpu_hwid = cpu_logical_map(cpu);
+       }
+       map_cpu_to_node(cpu, node_cpu_hwid[cpu].node_id);
+}
+
+/**
+ * numa_add_memblk_to - Add one numa_memblk to a numa_meminfo
+ */
+
+static int __init numa_add_memblk_to(int nid, u64 start, u64 end,
+                                    struct numa_meminfo *mi)
+{
+       /* ignore zero length blks */
+       if (start == end)
+               return 0;
+
+       /* whine about and ignore invalid blks */
+       if (start > end || nid < 0 || nid >= MAX_NUMNODES) {
+               pr_warn("NUMA: Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n",
+                               nid, start, end - 1);
+               return 0;
+       }
+
+       if (mi->nr_blks >= NR_NODE_MEMBLKS) {
+               pr_err("NUMA: too many memblk ranges\n");
+               return -EINVAL;
+       }
+
+       pr_info("NUMA: Adding memblock %d [0x%llx - 0x%llx] on node %d\n",
+                       mi->nr_blks, start, end, nid);
+       mi->blk[mi->nr_blks].start = start;
+       mi->blk[mi->nr_blks].end = end;
+       mi->blk[mi->nr_blks].nid = nid;
+       mi->nr_blks++;
+       return 0;
+}
+
+/**
+ * numa_add_memblk - Add one numa_memblk to numa_meminfo
+ * @nid: NUMA node ID of the new memblk
+ * @start: Start address of the new memblk
+ * @end: End address of the new memblk
+ *
+ * Add a new memblk to the default numa_meminfo.
+ *
+ * RETURNS:
+ * 0 on success, -errno on failure.
+ */
+#define MAX_PHYS_ADDR  ((phys_addr_t)~0)
+
+int __init numa_add_memblk(u32 nid, u64 base, u64 end)
+{
+       const u64 phys_offset = __pa(PAGE_OFFSET);
+
+       base &= PAGE_MASK;
+       end &= PAGE_MASK;
+
+       if (base > MAX_PHYS_ADDR) {
+               pr_warn("NUMA: Ignoring memory block 0x%llx - 0x%llx\n",
+                               base, base + end);
+               return -ENOMEM;
+       }
+
+       if (base + end > MAX_PHYS_ADDR) {
+               pr_info("NUMA: Ignoring memory range 0x%lx - 0x%llx\n",
+                               ULONG_MAX, base + end);
+               end = MAX_PHYS_ADDR - base;
+       }
+
+       if (base + end < phys_offset) {
+               pr_warn("NUMA: Ignoring memory block 0x%llx - 0x%llx\n",
+                          base, base + end);
+               return -ENOMEM;
+       }
+       if (base < phys_offset) {
+               pr_info("NUMA: Ignoring memory range 0x%llx - 0x%llx\n",
+                          base, phys_offset);
+               end -= phys_offset - base;
+               base = phys_offset;
+       }
+
+       return numa_add_memblk_to(nid, base, base + end, &numa_meminfo);
+}
+EXPORT_SYMBOL(numa_add_memblk);
+
+/* Initialize NODE_DATA for a node on the local memory */
+static void __init setup_node_data(int nid, u64 start, u64 end)
+{
+       const size_t nd_size = roundup(sizeof(pg_data_t), PAGE_SIZE);
+       u64 nd_pa;
+       void *nd;
+       int tnid;
+
+       start = roundup(start, ZONE_ALIGN);
+
+       pr_info("Initmem setup node %d [mem %#010Lx-%#010Lx]\n",
+              nid, start, end - 1);
+
+       /*
+        * Allocate node data.  Try node-local memory and then any node.
+        */
+       nd_pa = memblock_alloc_nid(nd_size, SMP_CACHE_BYTES, nid);
+       if (!nd_pa) {
+               nd_pa = __memblock_alloc_base(nd_size, SMP_CACHE_BYTES,
+                                             MEMBLOCK_ALLOC_ACCESSIBLE);
+               if (!nd_pa) {
+                       pr_err("Cannot find %zu bytes in node %d\n",
+                              nd_size, nid);
+                       return;
+               }
+       }
+       nd = __va(nd_pa);
+
+       /* report and initialize */
+       pr_info("  NODE_DATA [mem %#010Lx-%#010Lx]\n",
+              nd_pa, nd_pa + nd_size - 1);
+       tnid = early_pfn_to_nid(nd_pa >> PAGE_SHIFT);
+       if (tnid != nid)
+               pr_info("    NODE_DATA(%d) on node %d\n", nid, tnid);
+
+       node_data[nid] = nd;
+       memset(NODE_DATA(nid), 0, sizeof(pg_data_t));
+       NODE_DATA(nid)->node_id = nid;
+       NODE_DATA(nid)->node_start_pfn = start >> PAGE_SHIFT;
+       NODE_DATA(nid)->node_spanned_pages = (end - start) >> PAGE_SHIFT;
+
+       node_set_online(nid);
+}
+
+/*
+ * Set nodes, which have memory in @mi, in *@nodemask.
+ */
+static void __init numa_nodemask_from_meminfo(nodemask_t *nodemask,
+                                             const struct numa_meminfo *mi)
+{
+       int i;
+
+       for (i = 0; i < ARRAY_SIZE(mi->blk); i++)
+               if (mi->blk[i].start != mi->blk[i].end &&
+                   mi->blk[i].nid != NUMA_NO_NODE)
+                       node_set(mi->blk[i].nid, *nodemask);
+}
+
+/*
+ * Sanity check to catch more bad NUMA configurations (they are amazingly
+ * common).  Make sure the nodes cover all memory.
+ */
+static bool __init numa_meminfo_cover_memory(const struct numa_meminfo *mi)
+{
+       u64 numaram, totalram;
+       int i;
+
+       numaram = 0;
+       for (i = 0; i < mi->nr_blks; i++) {
+               u64 s = mi->blk[i].start >> PAGE_SHIFT;
+               u64 e = mi->blk[i].end >> PAGE_SHIFT;
+
+               numaram += e - s;
+               numaram -= __absent_pages_in_range(mi->blk[i].nid, s, e);
+               if ((s64)numaram < 0)
+                       numaram = 0;
+       }
+
+       totalram = max_pfn - absent_pages_in_range(0, max_pfn);
+
+       /* We seem to lose 3 pages somewhere. Allow 1M of slack. */
+       if ((s64)(totalram - numaram) >= (1 << (20 - PAGE_SHIFT))) {
+               pr_err("NUMA: nodes only cover %lluMB of your %lluMB Total RAM. Not used.\n",
+                      (numaram << PAGE_SHIFT) >> 20,
+                      (totalram << PAGE_SHIFT) >> 20);
+               return false;
+       }
+       return true;
+}
+
+/**
+ * numa_reset_distance - Reset NUMA distance table
+ *
+ * The current table is freed.  The next numa_set_distance() call will
+ * create a new one.
+ */
+void __init numa_reset_distance(void)
+{
+       size_t size = numa_distance_cnt * numa_distance_cnt *
+               sizeof(numa_distance[0]);
+
+       /* numa_distance could be 1LU marking allocation failure, test cnt */
+       if (numa_distance_cnt)
+               memblock_free(__pa(numa_distance), size);
+       numa_distance_cnt = 0;
+       numa_distance = NULL;   /* enable table creation */
+}
+
+static int __init numa_alloc_distance(void)
+{
+       nodemask_t nodes_parsed;
+       size_t size;
+       int i, j, cnt = 0;
+       u64 phys;
+
+       /* size the new table and allocate it */
+       nodes_parsed = numa_nodes_parsed;
+       numa_nodemask_from_meminfo(&nodes_parsed, &numa_meminfo);
+
+       for_each_node_mask(i, nodes_parsed)
+               cnt = i;
+       cnt++;
+       size = cnt * cnt * sizeof(numa_distance[0]);
+
+       phys = memblock_find_in_range(0, PFN_PHYS(max_pfn),
+                                     size, PAGE_SIZE);
+       if (!phys) {
+               pr_warn("NUMA: Warning: can't allocate distance table!\n");
+               /* don't retry until explicitly reset */
+               numa_distance = (void *)1LU;
+               return -ENOMEM;
+       }
+       memblock_reserve(phys, size);
+
+       numa_distance = __va(phys);
+       numa_distance_cnt = cnt;
+
+       /* fill with the default distances */
+       for (i = 0; i < cnt; i++)
+               for (j = 0; j < cnt; j++)
+                       numa_distance[i * cnt + j] = i == j ?
+                               LOCAL_DISTANCE : REMOTE_DISTANCE;
+       pr_debug("NUMA: Initialized distance table, cnt=%d\n", cnt);
+
+       return 0;
+}
+
+/**
+ * numa_set_distance - Set NUMA distance from one NUMA to another
+ * @from: the 'from' node to set distance
+ * @to: the 'to'  node to set distance
+ * @distance: NUMA distance
+ *
+ * Set the distance from node @from to @to to @distance.  If distance table
+ * doesn't exist, one which is large enough to accommodate all the currently
+ * known nodes will be created.
+ *
+ * If such table cannot be allocated, a warning is printed and further
+ * calls are ignored until the distance table is reset with
+ * numa_reset_distance().
+ *
+ * If @from or @to is higher than the highest known node or lower than zero
+ * at the time of table creation or @distance doesn't make sense, the call
+ * is ignored.
+ * This is to allow simplification of specific NUMA config implementations.
+ */
+void __init numa_set_distance(int from, int to, int distance)
+{
+       if (!numa_distance && numa_alloc_distance() < 0)
+               return;
+
+       if (from >= numa_distance_cnt || to >= numa_distance_cnt ||
+                       from < 0 || to < 0) {
+               pr_warn_once("NUMA: Warning: node ids are out of bound, from=%d to=%d distance=%d\n",
+                           from, to, distance);
+               return;
+       }
+
+       if ((u8)distance != distance ||
+           (from == to && distance != LOCAL_DISTANCE)) {
+               pr_warn_once("NUMA: Warning: invalid distance parameter, from=%d to=%d distance=%d\n",
+                            from, to, distance);
+               return;
+       }
+
+       numa_distance[from * numa_distance_cnt + to] = distance;
+}
+EXPORT_SYMBOL(numa_set_distance);
+
+int __node_distance(int from, int to)
+{
+       if (from >= numa_distance_cnt || to >= numa_distance_cnt)
+               return from == to ? LOCAL_DISTANCE : REMOTE_DISTANCE;
+       return numa_distance[from * numa_distance_cnt + to];
+}
+EXPORT_SYMBOL(__node_distance);
+
+static int __init numa_register_memblks(struct numa_meminfo *mi)
+{
+       unsigned long uninitialized_var(pfn_align);
+       int i, nid;
+
+       /* Account for nodes with cpus and no memory */
+       node_possible_map = numa_nodes_parsed;
+       numa_nodemask_from_meminfo(&node_possible_map, mi);
+       if (WARN_ON(nodes_empty(node_possible_map)))
+               return -EINVAL;
+
+       for (i = 0; i < mi->nr_blks; i++) {
+               struct numa_memblk *mb = &mi->blk[i];
+
+               memblock_set_node(mb->start, mb->end - mb->start,
+                                 &memblock.memory, mb->nid);
+       }
+
+       /*
+        * If sections array is gonna be used for pfn -> nid mapping, check
+        * whether its granularity is fine enough.
+        */
+#ifdef NODE_NOT_IN_PAGE_FLAGS
+       pfn_align = node_map_pfn_alignment();
+       if (pfn_align && pfn_align < PAGES_PER_SECTION) {
+               pr_warn("Node alignment %lluMB < min %lluMB, rejecting NUMA config\n",
+                      PFN_PHYS(pfn_align) >> 20,
+                      PFN_PHYS(PAGES_PER_SECTION) >> 20);
+               return -EINVAL;
+       }
+#endif
+       if (!numa_meminfo_cover_memory(mi))
+               return -EINVAL;
+
+       /* Finally register nodes. */
+       for_each_node_mask(nid, node_possible_map) {
+               u64 start = PFN_PHYS(max_pfn);
+               u64 end = 0;
+
+               for (i = 0; i < mi->nr_blks; i++) {
+                       if (nid != mi->blk[i].nid)
+                               continue;
+                       start = min(mi->blk[i].start, start);
+                       end = max(mi->blk[i].end, end);
+               }
+
+               if (start < end)
+                       setup_node_data(nid, start, end);
+       }
+
+       /* Dump memblock with node info and return. */
+       memblock_dump_all();
+       return 0;
+}
+
+static int __init numa_init(int (*init_func)(void))
+{
+       int ret, i;
+
+       nodes_clear(numa_nodes_parsed);
+       nodes_clear(node_possible_map);
+       nodes_clear(node_online_map);
+
+       ret = init_func();
+       if (ret < 0)
+               return ret;
+
+       ret = numa_register_memblks(&numa_meminfo);
+       if (ret < 0)
+               return ret;
+
+       for (i = 0; i < nr_cpu_ids; i++)
+               numa_clear_node(i);
+
+       setup_node_to_cpumask_map();
+       build_cpu_to_node_map();
+       return 0;
+}
+
+/**
+ * dummy_numa_init - Fallback dummy NUMA init
+ *
+ * Used if there's no underlying NUMA architecture, NUMA initialization
+ * fails, or NUMA is disabled on the command line.
+ *
+ * Must online at least one node and add memory blocks that cover all
+ * allowed memory.  This function must not fail.
+ */
+static int __init dummy_numa_init(void)
+{
+       pr_info("%s\n", "No NUMA configuration found");
+       pr_info("Faking a node at [mem %#018Lx-%#018Lx]\n",
+              0LLU, PFN_PHYS(max_pfn) - 1);
+       node_set(0, numa_nodes_parsed);
+       numa_add_memblk(0, 0, PFN_PHYS(max_pfn));
+       dummy_numa_enabled = 1;
+
+       return 0;
+}
+
+/**
+ * arm64_numa_init - Initialize NUMA
+ *
+ * Try each configured NUMA initialization method until one succeeds.  The
+ * last fallback is dummy single node config encomapssing whole memory and
+ * never fails.
+ */
+void __init arm64_numa_init(void)
+{
+       numa_init(dummy_numa_init);
+}
--

1.8.1.4

DT bindings for numa map for memory, cores and IOs using arm,associativity device node property.

Signed-off-by Ganapatrao Kulkarni <[email protected]>

---
 Documentation/devicetree/bindings/arm/numa.txt | 212 +++++++++++++++++++++++++
 1 file changed, 212 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/arm/numa.txt

Documentation/devicetree/bindings/arm/numa.txt new file +212 -0

new file mode 100644
index 0000000..dc3ef86
@@ -0,0 +1,212 @@
+==============================================================================
+NUMA binding description.
+==============================================================================
+
+==============================================================================
+1 - Introduction
+==============================================================================
+
+Systems employing a Non Uniform Memory Access (NUMA) architecture contain
+collections of hardware resources including processors, memory, and I/O buses,
+that comprise what is commonly known as a NUMA node.
+Processor accesses to memory within the local NUMA node is generally faster
+than processor accesses to memory outside of the local NUMA node.
+DT defines interfaces that allow the platform to convey NUMA node
+topology information to OS.
+
+==============================================================================
+2 - arm,associativity
+==============================================================================
+The mapping is done using arm,associativity device property.
+this property needs to be present in every device node which needs to to be
+mapped to numa nodes.
+
+arm,associativity property is set of 32-bit integers which defines level of
+topology and boundary in the system at which a significant difference in
+performance can be measured between cross-device accesses within
+a single location and those spanning multiple locations.
+The first cell always contains the broadest subdivision within the system,
+while the last cell enumerates the individual devices, such as an SMT thread
+of a CPU, or a bus bridge within an SoC".
+
+ex:
+	/* board 0, socket 0, cluster 0, core 0  thread 0 */
+	arm,associativity = <0 0 0 0 0>;
+
+==============================================================================
+3 - arm,associativity-reference-points
+==============================================================================
+This property is a set of 32-bit integers, each representing an index into
+the arm,associativity nodes. The first integer is the most significant
+NUMA boundary and the following are progressively less significant boundaries.
+There can be more than one level of NUMA.
+
+Ex:
+	arm,associativity-reference-points = <0 1>;
+	The board Id(index 0) used first to calculate the associativity (node
+	distance), then follows the  socket id(index 1).
+
+	arm,associativity-reference-points = <1 0>;
+	The socket Id(index 1) used first to calculate the associativity,
+	then follows the board id(index 0).
+
+	arm,associativity-reference-points = <0>;
+	Only the board Id(index 0) used to calculate the associativity.
+
+	arm,associativity-reference-points = <1>;
+	Only socket Id(index 1) used to calculate the associativity.
+
+==============================================================================
+4 - Example dts
+==============================================================================
+
+Example: 2 Node system consists of 2 boards and each board having one socket
+and 8 core in each socket.
+
+	arm,associativity-reference-points = <0>;
+
+	memory at 00c00000 {
+		device_type = "memory";
+		reg = <0x0 0x00c00000 0x0 0x80000000>;
+		/* board 0, socket 0, no specific core */
+		arm,associativity = <0 0 0xffff>;
+	};
+
+	memory at 10000000000 {
+		device_type = "memory";
+		reg = <0x100 0x00000000 0x0 0x80000000>;
+		/* board 1, socket 0, no specific core */
+		arm,associativity = <1 0 0xffff>;
+	};
+
+	cpus {
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		cpu at 000 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x000>;
+			enable-method = "psci";
+			/* board 0, socket 0, core 0*/
+			arm,associativity = <0 0 0>;
+		};
+		cpu at 001 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x001>;
+			enable-method = "psci";
+			arm,associativity = <0 0 1>;
+		};
+		cpu at 002 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x002>;
+			enable-method = "psci";
+			arm,associativity = <0 0 2>;
+		};
+		cpu at 003 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x003>;
+			enable-method = "psci";
+			arm,associativity = <0 0 3>;
+		};
+		cpu at 004 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x004>;
+			enable-method = "psci";
+			arm,associativity = <0 0 4>;
+		};
+		cpu at 005 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x005>;
+			enable-method = "psci";
+			arm,associativity = <0 0 5>;
+		};
+		cpu at 006 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x006>;
+			enable-method = "psci";
+			arm,associativity = <0 0 6>;
+		};
+		cpu at 007 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x007>;
+			enable-method = "psci";
+			arm,associativity = <0 0 7>;
+		};
+		cpu at 008 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x008>;
+			enable-method = "psci";
+			/* board 1, socket 0, core 0*/
+			arm,associativity = <1 0 0>;
+		};
+		cpu at 009 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x009>;
+			enable-method = "psci";
+			arm,associativity = <1 0 1>;
+		};
+		cpu at 00a {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x00a>;
+			enable-method = "psci";
+			arm,associativity = <0 0 2>;
+		};
+		cpu at 00b {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x00b>;
+			enable-method = "psci";
+			arm,associativity = <1 0 3>;
+		};
+		cpu at 00c {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x00c>;
+			enable-method = "psci";
+			arm,associativity = <1 0 4>;
+		};
+		cpu at 00d {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x00d>;
+			enable-method = "psci";
+			arm,associativity = <1 0 5>;
+		};
+		cpu at 00e {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x00e>;
+			enable-method = "psci";
+			arm,associativity = <1 0 6>;
+		};
+		cpu at 00f {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x00f>;
+			enable-method = "psci";
+			arm,associativity = <1 0 7>;
+		};
+	};
+
+	pcie0: pcie0 at 0x8480,00000000 {
+		compatible = "arm,armv8";
+		device_type = "pci";
+		bus-range = <0 255>;
+		#size-cells = <2>;
+		#address-cells = <3>;
+		reg = <0x8480 0x00000000 0 0x10000000>;  /* Configuration space */
+		ranges = <0x03000000 0x8010 0x00000000 0x8010 0x00000000 0x70 0x00000000>; /* mem ranges */
+		/* board 0, socket 0, pci bus 0*/
+		arm,associativity = <0 0 0>;
+        };

Signature · 7 lines

1.8.1.4


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel at lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

DT bindings for numa map for memory, cores and IOs using arm,associativity device node property.

Signed-off-by Ganapatrao Kulkarni <[email protected]>

---
 Documentation/devicetree/bindings/arm/numa.txt | 212 +++++++++++++++++++++++++
 1 file changed, 212 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/arm/numa.txt

Documentation/devicetree/bindings/arm/numa.txt new file +212 -1

new file mode 100644
index 0000000..dc3ef86
@@ -0,0 +1,212 @@
+==============================================================================
+NUMA binding description.
+==============================================================================
+
+==============================================================================
+1 - Introduction
+==============================================================================
+
+Systems employing a Non Uniform Memory Access (NUMA) architecture contain
+collections of hardware resources including processors, memory, and I/O buses,
+that comprise what is commonly known as a NUMA node.
+Processor accesses to memory within the local NUMA node is generally faster
+than processor accesses to memory outside of the local NUMA node.
+DT defines interfaces that allow the platform to convey NUMA node
+topology information to OS.
+
+==============================================================================
+2 - arm,associativity
+==============================================================================
+The mapping is done using arm,associativity device property.
+this property needs to be present in every device node which needs to to be
+mapped to numa nodes.
+
+arm,associativity property is set of 32-bit integers which defines level of
+topology and boundary in the system at which a significant difference in
+performance can be measured between cross-device accesses within
+a single location and those spanning multiple locations.
+The first cell always contains the broadest subdivision within the system,
+while the last cell enumerates the individual devices, such as an SMT thread
+of a CPU, or a bus bridge within an SoC".
+
+ex:
+       /* board 0, socket 0, cluster 0, core 0  thread 0 */
+       arm,associativity = <0 0 0 0 0>;
+
+==============================================================================
+3 - arm,associativity-reference-points
+==============================================================================
+This property is a set of 32-bit integers, each representing an index into
+the arm,associativity nodes. The first integer is the most significant
+NUMA boundary and the following are progressively less significant boundaries.
+There can be more than one level of NUMA.
+
+Ex:
+       arm,associativity-reference-points = <0 1>;
+       The board Id(index 0) used first to calculate the associativity (node
+       distance), then follows the  socket id(index 1).
+
+       arm,associativity-reference-points = <1 0>;
+       The socket Id(index 1) used first to calculate the associativity,
+       then follows the board id(index 0).
+
+       arm,associativity-reference-points = <0>;
+       Only the board Id(index 0) used to calculate the associativity.
+
+       arm,associativity-reference-points = <1>;
+       Only socket Id(index 1) used to calculate the associativity.
+
+==============================================================================
+4 - Example dts
+==============================================================================
+
+Example: 2 Node system consists of 2 boards and each board having one socket
+and 8 core in each socket.
+
+       arm,associativity-reference-points = <0>;
+
+       memory at 00c00000 {
+               device_type = "memory";
+               reg = <0x0 0x00c00000 0x0 0x80000000>;
+               /* board 0, socket 0, no specific core */
+               arm,associativity = <0 0 0xffff>;
+       };
+
+       memory at 10000000000 {
+               device_type = "memory";
+               reg = <0x100 0x00000000 0x0 0x80000000>;
+               /* board 1, socket 0, no specific core */
+               arm,associativity = <1 0 0xffff>;
+       };
+
+       cpus {
+               #address-cells = <2>;
+               #size-cells = <0>;
+
+               cpu at 000 {
+                       device_type = "cpu";
+                       compatible =  "arm,armv8";
+                       reg = <0x0 0x000>;
+                       enable-method = "psci";
+                       /* board 0, socket 0, core 0*/
+                       arm,associativity = <0 0 0>;
+               };
+               cpu at 001 {
+                       device_type = "cpu";
+                       compatible =  "arm,armv8";
+                       reg = <0x0 0x001>;
+                       enable-method = "psci";
+                       arm,associativity = <0 0 1>;
+               };
+               cpu at 002 {
+                       device_type = "cpu";
+                       compatible =  "arm,armv8";
+                       reg = <0x0 0x002>;
+                       enable-method = "psci";
+                       arm,associativity = <0 0 2>;
+               };
+               cpu at 003 {
+                       device_type = "cpu";
+                       compatible =  "arm,armv8";
+                       reg = <0x0 0x003>;
+                       enable-method = "psci";
+                       arm,associativity = <0 0 3>;
+               };
+               cpu at 004 {
+                       device_type = "cpu";
+                       compatible =  "arm,armv8";
+                       reg = <0x0 0x004>;
+                       enable-method = "psci";
+                       arm,associativity = <0 0 4>;
+               };
+               cpu at 005 {
+                       device_type = "cpu";
+                       compatible =  "arm,armv8";
+                       reg = <0x0 0x005>;
+                       enable-method = "psci";
+                       arm,associativity = <0 0 5>;
+               };
+               cpu at 006 {
+                       device_type = "cpu";
+                       compatible =  "arm,armv8";
+                       reg = <0x0 0x006>;
+                       enable-method = "psci";
+                       arm,associativity = <0 0 6>;
+               };
+               cpu at 007 {
+                       device_type = "cpu";
+                       compatible =  "arm,armv8";
+                       reg = <0x0 0x007>;
+                       enable-method = "psci";
+                       arm,associativity = <0 0 7>;
+               };
+               cpu at 008 {
+                       device_type = "cpu";
+                       compatible =  "arm,armv8";
+                       reg = <0x0 0x008>;
+                       enable-method = "psci";
+                       /* board 1, socket 0, core 0*/
+                       arm,associativity = <1 0 0>;
+               };
+               cpu at 009 {
+                       device_type = "cpu";
+                       compatible =  "arm,armv8";
+                       reg = <0x0 0x009>;
+                       enable-method = "psci";
+                       arm,associativity = <1 0 1>;
+               };
+               cpu at 00a {
+                       device_type = "cpu";
+                       compatible =  "arm,armv8";
+                       reg = <0x0 0x00a>;
+                       enable-method = "psci";
+                       arm,associativity = <0 0 2>;
+               };
+               cpu at 00b {
+                       device_type = "cpu";
+                       compatible =  "arm,armv8";
+                       reg = <0x0 0x00b>;
+                       enable-method = "psci";
+                       arm,associativity = <1 0 3>;
+               };
+               cpu at 00c {
+                       device_type = "cpu";
+                       compatible =  "arm,armv8";
+                       reg = <0x0 0x00c>;
+                       enable-method = "psci";
+                       arm,associativity = <1 0 4>;
+               };
+               cpu at 00d {
+                       device_type = "cpu";
+                       compatible =  "arm,armv8";
+                       reg = <0x0 0x00d>;
+                       enable-method = "psci";
+                       arm,associativity = <1 0 5>;
+               };
+               cpu at 00e {
+                       device_type = "cpu";
+                       compatible =  "arm,armv8";
+                       reg = <0x0 0x00e>;
+                       enable-method = "psci";
+                       arm,associativity = <1 0 6>;
+               };
+               cpu at 00f {
+                       device_type = "cpu";
+                       compatible =  "arm,armv8";
+                       reg = <0x0 0x00f>;
+                       enable-method = "psci";
+                       arm,associativity = <1 0 7>;
+               };
+       };
+
+       pcie0: pcie0 at 0x8480,00000000 {
+               compatible = "arm,armv8";
+               device_type = "pci";
+               bus-range = <0 255>;
+               #size-cells = <2>;
+               #address-cells = <3>;
+               reg = <0x8480 0x00000000 0 0x10000000>;  /* Configuration space */
+               ranges = <0x03000000 0x8010 0x00000000 0x8010 0x00000000 0x70 0x00000000>; /* mem ranges */
+               /* board 0, socket 0, pci bus 0*/
+               arm,associativity = <0 0 0>;
+        };
--

1.8.1.4

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel at lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

DT bindings for numa map for memory, cores and IOs using arm,associativity device node property.

Signed-off-by Ganapatrao Kulkarni <[email protected]>

---
  Documentation/devicetree/bindings/arm/numa.txt | 212 +++++++++++++++++++++++++
  1 file changed, 212 insertions(+)

create mode 100644 Documentation/devicetree/bindings/arm/numa.txt

Documentation/devicetree/bindings/arm/numa.txt new file +153 -0

new file mode 100644
index 0000000..dc3ef86
@@ -0,0 +1,212 @@
+==============================================================================
+NUMA binding description.
+==============================================================================
+
+==============================================================================
+1 - Introduction
+==============================================================================
+
+Systems employing a Non Uniform Memory Access (NUMA) architecture contain
+collections of hardware resources including processors, memory, and I/O buses,
+that comprise what is commonly known as a NUMA node.
+Processor accesses to memory within the local NUMA node is generally faster
+than processor accesses to memory outside of the local NUMA node.
+DT defines interfaces that allow the platform to convey NUMA node
+topology information to OS.
+
+==============================================================================
+2 - arm,associativity
+==============================================================================
+The mapping is done using arm,associativity device property.
+this property needs to be present in every device node which needs to to be
+mapped to numa nodes.
+
+arm,associativity property is set of 32-bit integers which defines level of
+topology and boundary in the system at which a significant difference in
+performance can be measured between cross-device accesses within
+a single location and those spanning multiple locations.
+The first cell always contains the broadest subdivision within the system,
+while the last cell enumerates the individual devices, such as an SMT thread
+of a CPU, or a bus bridge within an SoC".
+
+ex:
+	/* board 0, socket 0, cluster 0, core 0  thread 0 */
+	arm,associativity = <0 0 0 0 0>;
+
+==============================================================================
+3 - arm,associativity-reference-points
+==============================================================================
+This property is a set of 32-bit integers, each representing an index into
+the arm,associativity nodes. The first integer is the most significant
+NUMA boundary and the following are progressively less significant boundaries.
+There can be more than one level of NUMA.
+
+Ex:
+	arm,associativity-reference-points = <0 1>;
+	The board Id(index 0) used first to calculate the associativity (node
+	distance), then follows the  socket id(index 1).
+
+	arm,associativity-reference-points = <1 0>;
+	The socket Id(index 1) used first to calculate the associativity,
+	then follows the board id(index 0).
+
+	arm,associativity-reference-points = <0>;
+	Only the board Id(index 0) used to calculate the associativity.
+
+	arm,associativity-reference-points = <1>;
+	Only socket Id(index 1) used to calculate the associativity.
+
+==============================================================================
+4 - Example dts
+==============================================================================
+
+Example: 2 Node system consists of 2 boards and each board having one socket
+and 8 core in each socket.
+
+	arm,associativity-reference-points = <0>;
+
+	memory at 00c00000 {
+		device_type = "memory";
+		reg = <0x0 0x00c00000 0x0 0x80000000>;
+		/* board 0, socket 0, no specific core */
+		arm,associativity = <0 0 0xffff>;
+	};
+
+	memory at 10000000000 {
+		device_type = "memory";
+		reg = <0x100 0x00000000 0x0 0x80000000>;
+		/* board 1, socket 0, no specific core */
+		arm,associativity = <1 0 0xffff>;
+	};
+
+	cpus {
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		cpu at 000 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x000>;
+			enable-method = "psci";
+			/* board 0, socket 0, core 0*/
+			arm,associativity = <0 0 0>;
+		};
+		cpu at 001 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x001>;
+			enable-method = "psci";
+			arm,associativity = <0 0 1>;
+		};
+		cpu at 002 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x002>;
+			enable-method = "psci";
+			arm,associativity = <0 0 2>;
+		};
+		cpu at 003 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x003>;
+			enable-method = "psci";
+			arm,associativity = <0 0 3>;
+		};
+		cpu at 004 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x004>;
+			enable-method = "psci";
+			arm,associativity = <0 0 4>;
+		};
+		cpu at 005 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x005>;
+			enable-method = "psci";
+			arm,associativity = <0 0 5>;
+		};
+		cpu at 006 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x006>;
+			enable-method = "psci";
+			arm,associativity = <0 0 6>;
+		};
+		cpu at 007 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x007>;
+			enable-method = "psci";
+			arm,associativity = <0 0 7>;
+		};
+		cpu at 008 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x008>;
+			enable-method = "psci";
+			/* board 1, socket 0, core 0*/
+			arm,associativity = <1 0 0>;
+		};
+		cpu at 009 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";

Documentation/devicetree/bindings/arm/numa.txt new file +49 -0

+		};
+		cpu at 00b {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x00b>;
+			enable-method = "psci";
+			arm,associativity = <1 0 3>;
+		};
+		cpu at 00c {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x00c>;
+			enable-method = "psci";
+			arm,associativity = <1 0 4>;
+		};
+		cpu at 00d {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x00d>;
+			enable-method = "psci";
+			arm,associativity = <1 0 5>;
+		};
+		cpu at 00e {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x00e>;
+			enable-method = "psci";
+			arm,associativity = <1 0 6>;
+		};
+		cpu at 00f {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x00f>;
+			enable-method = "psci";
+			arm,associativity = <1 0 7>;
+		};
+	};
+
+	pcie0: pcie0 at 0x8480,00000000 {
+		compatible = "arm,armv8";
+		device_type = "pci";
+		bus-range = <0 255>;
+		#size-cells = <2>;
+		#address-cells = <3>;
+		reg = <0x8480 0x00000000 0 0x10000000>;  /* Configuration space */
+		ranges = <0x03000000 0x8010 0x00000000 0x8010 0x00000000 0x70 0x00000000>; /* mem ranges */
+		/* board 0, socket 0, pci bus 0*/
+		arm,associativity = <0 0 0>;
+        };

DT bindings for numa map for memory, cores and IOs using arm,associativity device node property.

Signed-off-by Ganapatrao Kulkarni <[email protected]>

---
 Documentation/devicetree/bindings/arm/numa.txt | 212 +++++++++++++++++++++++++
 1 file changed, 212 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/arm/numa.txt

Documentation/devicetree/bindings/arm/numa.txt new file +12 -0

new file mode 100644
index 0000000..dc3ef86
@@ -0,0 +1,212 @@
+==============================================================================
+NUMA binding description.
+==============================================================================
+
+==============================================================================
+1 - Introduction
+==============================================================================
+
+Systems employing a Non Uniform Memory Access (NUMA) architecture contain
+collections of hardware resources including processors, memory, and I/O buses,
+that comprise what is commonly known as a NUMA node.
+Processor accesses to memory within the local NUMA node is generally faster

Documentation/devicetree/bindings/arm/numa.txt new file +38 -0

+       arm,associativity-reference-points = <0 1>;
+       The board Id(index 0) used first to calculate the associativity (node
+       distance), then follows the  socket id(index 1).
+
+       arm,associativity-reference-points = <1 0>;
+       The socket Id(index 1) used first to calculate the associativity,
+       then follows the board id(index 0).
+
+       arm,associativity-reference-points = <0>;
+       Only the board Id(index 0) used to calculate the associativity.
+
+       arm,associativity-reference-points = <1>;
+       Only socket Id(index 1) used to calculate the associativity.
+
+==============================================================================
+4 - Example dts
+==============================================================================
+
+Example: 2 Node system consists of 2 boards and each board having one socket
+and 8 core in each socket.
+
+       arm,associativity-reference-points = <0>;
+
+       memory at 00c00000 {
+               device_type = "memory";
+               reg = <0x0 0x00c00000 0x0 0x80000000>;
+               /* board 0, socket 0, no specific core */
+               arm,associativity = <0 0 0xffff>;
+       };
+
+       memory at 10000000000 {
+               device_type = "memory";
+               reg = <0x100 0x00000000 0x0 0x80000000>;
+               /* board 1, socket 0, no specific core */
+               arm,associativity = <1 0 0xffff>;
+       };
+
+       cpus {

Hi,

Quoted 2 lines · Ganapatrao Kulkarni

DT bindings for numa map for memory, cores and IOs using arm,associativity device node property.

Given this is just a copy of ibm,associativity, I'm not sure I see much point in renaming the properties.

However, (somewhat counter to that) I'm also concerned that this isn't sufficient for systems we're beginning to see today (more on that below), so I don't think a simple copy of ibm,associativity is good enough.

Quoted 30 lines

Signed-off-by Ganapatrao Kulkarni <[email protected]>

---
 Documentation/devicetree/bindings/arm/numa.txt | 212 +++++++++++++++++++++++++
 1 file changed, 212 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/arm/numa.txt

Documentation/devicetree/bindings/arm/numa.txt new file +17 -0

new file mode 100644
index 0000000..dc3ef86
@@ -0,0 +1,212 @@
+==============================================================================
+NUMA binding description.
+==============================================================================
+
+==============================================================================
+1 - Introduction
+==============================================================================
+
+Systems employing a Non Uniform Memory Access (NUMA) architecture contain
+collections of hardware resources including processors, memory, and I/O buses,
+that comprise what is commonly known as a NUMA node.
+Processor accesses to memory within the local NUMA node is generally faster
+than processor accesses to memory outside of the local NUMA node.
+DT defines interfaces that allow the platform to convey NUMA node
+topology information to OS.
+
+==============================================================================

Documentation/devicetree/bindings/arm/numa.txt line 22

Quoted 5 lines

Documentation/devicetree/bindings/arm/numa.txt new file +5 -0

+2 - arm,associativity
+==============================================================================
+The mapping is done using arm,associativity device property.
+this property needs to be present in every device node which needs to to be
+mapped to numa nodes.

Can't there be some inheritance? e.g. all devices on a bus with an arm,associativity property being assumed to share that value?

Documentation/devicetree/bindings/arm/numa.txt line 24

Quoted 3 lines

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+
+arm,associativity property is set of 32-bit integers which defines level of

s/set/list/ -- the order is important.

Documentation/devicetree/bindings/arm/numa.txt line 30

Quoted 6 lines

Documentation/devicetree/bindings/arm/numa.txt new file +6 -0

+topology and boundary in the system at which a significant difference in
+performance can be measured between cross-device accesses within
+a single location and those spanning multiple locations.
+The first cell always contains the broadest subdivision within the system,
+while the last cell enumerates the individual devices, such as an SMT thread
+of a CPU, or a bus bridge within an SoC".

While this gives us some hierarchy, this doesn't seem to encode relative distances at all. That seems like an oversight.

Additionally, I'm somewhat unclear on how what you'd be expected to provide for this property in cases like ring or mesh interconnects, where there isn't a strict hierarchy (see systems with ARM's own CCN, or Tilera's TILE-Mx), but there is some measure of closeness.

Must all of these have the same length? If so, why not have a #(whatever)-cells property in the root to describe the expected length? If not, how are they to be interpreted relative to each other?

So just keep the ibm? I'm okay with that. That would help move to common code. Alternatively, we could drop the vendor prefix and have common code just check for both.

All points that could be asked of the IBM binding. Perhaps Arnd or Ben can provide some insight or know who can?

Quoted 3 lines · Rob Herring

So just keep the ibm? I'm okay with that. That would help move to common code. Alternatively, we could drop the vendor prefix and have common code just check for both.

That wouldn't be the first time we go down that path and it makes sense imho.

Quoted 2 lines

All points that could be asked of the IBM binding. Perhaps Arnd or Ben can provide some insight or know who can?

They are part of the PAPR specification which we've been trying to get published for a while now but that hasn't happened yet. Beware that there are variants of the format based on some other property. There's also
"ibm,associativity-reference-points" which is used to calculate distances. I'll see if I can get you an excerpt of the PAPR chapter, or reword it, in the next few days (please poke me if I drop the ball next week).

Cheers,
Ben.

+benh

Quoted 6 lines · Mark Rutland

Hi,

Quoted 2 lines · Ganapatrao Kulkarni

DT bindings for numa map for memory, cores and IOs using arm,associativity device node property.

Quoted 2 lines

Given this is just a copy of ibm,associativity, I'm not sure I see much point in renaming the properties.

So just keep the ibm? I'm okay with that. That would help move to common code. Alternatively, we could drop the vendor prefix and have common code just check for both.

Why not copy the method of ACPI numa? There only three elements should be configured:
1) a cpu belong to which node
2) a memory block belong to which node
3) the distance of each two nodes

Quoted 5 lines · Leizhen (ThunderTown)

Why not copy the method of ACPI numa? There only three elements should be configured:
1) a cpu belong to which node
2) a memory block belong to which node
3) the distance of each two nodes

This means your are bolting into the DT representation the concept of "Node" which isn't necessarily very meaningful.

Your system is really a hierarchy of objects. You can have cores on a chip, already possibly sharing some level of cache or not, you can have chips on a module, modules linked at various distances, etc...

What is "a node" ?

For example, I have a P8 chip with 2 chips on a module (fast X-bus) and 2 modules (slightly slower A-bus). All 4 chips have 2 memory controllers each.

Is a "node" a chip or a module ?

The Linux concept of node is too restrictive. The associativity properties avoid this by allowing you to define as many "levels" of associativity as you wish. Also since it's right justified, a given component doesn't need to have all levels (a MC can stop at chip while cores can go down one more level for example).

The reference points property gives a hint as "interesting" levels can typically be used as a hint for chosing what Linux will use as a "node" at least until Linux gets smarter. It can also be used to calculate distances.

Cheers,
Ben.

.

Quoted 9 lines · Rob Herring

+benh

Quoted 6 lines · Mark Rutland

Hi,

Quoted 2 lines · Ganapatrao Kulkarni

DT bindings for numa map for memory, cores and IOs using arm,associativity device node property.

Quoted 7 lines

Quoted 2 lines

Given this is just a copy of ibm,associativity, I'm not sure I see much point in renaming the properties.

So just keep the ibm? I'm okay with that. That would help move to common code. Alternatively, we could drop the vendor prefix and have common code just check for both.

Hi all,

Why not copy the method of ACPI numa? There only three elements should be configured:
1) a cpu belong to which node
2) a memory block belong to which node
3) the distance of each two nodes

The devicetree nodes of numa can be like below:

/ {
        ...

        numa-nodes-info {
                node-name: node-description {
                        mem-ranges = <...>;
                        cpus-list = <...>;
                };

                nodes-distance {
                        distance-list = <...>;
                };
        };

        ...
};

Sorry, I don't think xxx,associativity is a good method, it's hard to config, and it seems hardware-dependent. Especially, when we want to support memory hot-add, it's too hard. Because xxx,associativity have no obvious information about it. Like powerpc, it use another
property: "/ibm,dynamic-reconfiguration-memory".

I spend almost a whole month to implement of_numa(configured by dt-nodes), base upon my opinion mentioned above. If somebody are interested in it, I can send my patchset to show it.

Regards,
Thunder.

Hi Thunder,

Quoted 42 lines · Leizhen (ThunderTown)

Quoted 9 lines · Rob Herring

+benh

Quoted 6 lines · Mark Rutland

Hi,

Quoted 2 lines · Ganapatrao Kulkarni

DT bindings for numa map for memory, cores and IOs using arm,associativity device node property.

Quoted 7 lines

Quoted 2 lines

Given this is just a copy of ibm,associativity, I'm not sure I see much point in renaming the properties.

So just keep the ibm? I'm okay with that. That would help move to common code. Alternatively, we could drop the vendor prefix and have common code just check for both.

Hi all,

Why not copy the method of ACPI numa? There only three elements should be configured:
1) a cpu belong to which node
2) a memory block belong to which node
3) the distance of each two nodes

The devicetree nodes of numa can be like below:

/ {
        ...

        numa-nodes-info {
                node-name: node-description {
                        mem-ranges = <...>;
                        cpus-list = <...>;
                };

                nodes-distance {
                        distance-list = <...>;
                };
        };

Quoted 3 lines

        ...
};

some what similar to what your are proposing is already implemented in my v2 patchset.
https://lwn.net/Articles/623920/
http://lists.infradead.org/pipermail/linux-arm-kernel/2014-November/305164.html

we have went to associativity property based implementation to keep it more generic.
i do have both acpi(using linaro/hanjun's patches) and associativity based implementations on our internal tree
and tested on thunderx platform.
i do see issue in creating numa mapping using ACPI for IOs(for example, i am not able to create numa mapping for ITS which is on each node, using ACPI tables), since ACPI spec (tables SRAT and SLIT) talks only about processor and memory.
however associativity is generic and you can apply on any dt node.

Quoted 11 lines

Sorry, I don't think xxx,associativity is a good method, it's hard to config, and it seems hardware-dependent. Especially, when we want to support memory hot-add, it's too hard. Because xxx,associativity have no obvious information about it. Like powerpc, it use another
property: "/ibm,dynamic-reconfiguration-memory".

I spend almost a whole month to implement of_numa(configured by dt-nodes), base upon my opinion mentioned above. If somebody are interested in it, I can send my patchset to show it.

Regards,
Thunder.

thanks
ganapat

.

Hi Thunder,

Quoted 28 lines · Leizhen (ThunderTown)

Quoted 9 lines · Rob Herring

+benh

Quoted 6 lines · Mark Rutland

Hi,

Quoted 2 lines · Ganapatrao Kulkarni

DT bindings for numa map for memory, cores and IOs using arm,associativity device node property.

Quoted 7 lines

Quoted 2 lines

Given this is just a copy of ibm,associativity, I'm not sure I see much point in renaming the properties.

So just keep the ibm? I'm okay with that. That would help move to common code. Alternatively, we could drop the vendor prefix and have common code just check for both.

Hi all,

Why not copy the method of ACPI numa? There only three elements should be configured:
1) a cpu belong to which node
2) a memory block belong to which node
3) the distance of each two nodes

The devicetree nodes of numa can be like below:

Quoted 17 lines

/ {
         ...

         numa-nodes-info {
                 node-name: node-description {
                         mem-ranges = <...>;
                         cpus-list = <...>;
                 };

                 nodes-distance {
                         distance-list = <...>;
                 };
         };

         ...
};

some what similar to what your are proposing is already implemented in my v2 patchset.
https://lwn.net/Articles/623920/
http://lists.infradead.org/pipermail/linux-arm-kernel/2014-November/305164.html
we have went to associativity property based implementation to keep it more generic.
i do have both acpi(using linaro/hanjun's patches) and associativity based implementations on our internal tree
and tested on thunderx platform.

i do see issue in creating numa mapping using ACPI for IOs(for example, i am not able to create numa mapping for ITS which is on each node, using ACPI tables), since ACPI spec (tables SRAT and SLIT) talks only about processor and memory.

Hi Ganapatrao,

Quoted 62 lines · Ganapatrao Kulkarni

Hi Thunder,

Quoted 36 lines · Leizhen (ThunderTown)

Quoted 14 lines · Rob Herring

+benh

Quoted 9 lines · Mark Rutland

Hi,

Quoted 3 lines · Ganapatrao Kulkarni

DT bindings for numa map for memory, cores and IOs using arm,associativity device node property.

Quoted 8 lines

Quoted 2 lines

Given this is just a copy of ibm,associativity, I'm not sure I see much point in renaming the properties.

So just keep the ibm? I'm okay with that. That would help move to common code. Alternatively, we could drop the vendor prefix and have common code just check for both.

Hi all,

Why not copy the method of ACPI numa? There only three elements should be configured:
1) a cpu belong to which node
2) a memory block belong to which node
3) the distance of each two nodes

The devicetree nodes of numa can be like below:

Quoted 17 lines

/ {
         ...

         numa-nodes-info {
                 node-name: node-description {
                         mem-ranges = <...>;
                         cpus-list = <...>;
                 };

                 nodes-distance {
                         distance-list = <...>;
                 };
         };

         ...
};

some what similar to what your are proposing is already implemented in my v2 patchset.
https://lwn.net/Articles/623920/

Quoted 6 lines

http://lists.infradead.org/pipermail/linux-arm-kernel/2014-November/305164.html
we have went to associativity property based implementation to keep it more generic.
i do have both acpi(using linaro/hanjun's patches) and associativity based implementations on our internal tree
and tested on thunderx platform.

Great thanks!

Quoted 4 lines

i do see issue in creating numa mapping using ACPI for IOs(for example, i am not able to create numa mapping for ITS which is on each node, using ACPI tables), since ACPI spec (tables SRAT and SLIT) talks only about processor and memory.

I'm not sure why the ITS needs to know the NUMA domain, for my understanding, the interrupt will route to the correct NUMA domain using setting the affinity, ITS will configured to route it to the right GICR(cpu), so I think the ITS don't need to know which NUMA node belonging to, correct me if I missed something.

Thanks
Hanjun

Hi Hanjun,

Quoted 43 lines · Hanjun Guo

Hi Ganapatrao,

Quoted 39 lines · Ganapatrao Kulkarni

Hi Thunder,

Quoted 34 lines · Leizhen (ThunderTown)

Quoted 14 lines · Rob Herring

+benh

Quoted 9 lines · Mark Rutland

Hi,

Quoted 3 lines · Ganapatrao Kulkarni

DT bindings for numa map for memory, cores and IOs using arm,associativity device node property.

Quoted 8 lines

Quoted 2 lines

Given this is just a copy of ibm,associativity, I'm not sure I see much point in renaming the properties.

So just keep the ibm? I'm okay with that. That would help move to common code. Alternatively, we could drop the vendor prefix and have common code just check for both.

Hi all,

Why not copy the method of ACPI numa? There only three elements should be configured:
1) a cpu belong to which node
2) a memory block belong to which node
3) the distance of each two nodes

Quoted 39 lines

Quoted 23 lines

Quoted 19 lines

The devicetree nodes of numa can be like below:

/ {
         ...

         numa-nodes-info {
                 node-name: node-description {
                         mem-ranges = <...>;
                         cpus-list = <...>;
                 };

                 nodes-distance {
                         distance-list = <...>;
                 };
         };

         ...
};

some what similar to what your are proposing is already implemented in my v2 patchset.
https://lwn.net/Articles/623920/

Quoted 6 lines

http://lists.infradead.org/pipermail/linux-arm-kernel/2014-November/305164.html
we have went to associativity property based implementation to keep it more generic.
i do have both acpi(using linaro/hanjun's patches) and associativity based implementations on our internal tree
and tested on thunderx platform.

Great thanks!

Quoted 4 lines

i do see issue in creating numa mapping using ACPI for IOs(for example, i am not able to create numa mapping for ITS which is on each node, using ACPI tables), since ACPI spec (tables SRAT and SLIT) talks only about processor and memory.

Quoted 5 lines

I'm not sure why the ITS needs to know the NUMA domain, for my understanding, the interrupt will route to the correct NUMA domain using setting the affinity, ITS will configured to route it to the right GICR(cpu), so I think the ITS don't need to know which NUMA node belonging to, correct me if I missed something.

IIUC, GICR/collection is per cpu and can be mapped to numa node using cpu to node mapping.
However there are multiple its in multi-socket platform(at-least one its per socket),
knowing its to numa node mapping will help in routing(optimal) the interrupts to any one of GICR/collections of that node Hence, we need to find which its belongs to which socket/node using dt. same applies to pci bus too.

Quoted 1 line

Quoted 2 lines

Thanks
Hanjun

thanks
Ganapat

Hi Thunder,

Quoted 46 lines · Ganapatrao Kulkarni

Hi Hanjun,

Quoted 43 lines · Hanjun Guo

Hi Ganapatrao,

Quoted 39 lines · Ganapatrao Kulkarni

Hi Thunder,

Quoted 34 lines · Leizhen (ThunderTown)

Quoted 14 lines · Rob Herring

+benh

Quoted 9 lines · Mark Rutland

Hi,

Quoted 3 lines · Ganapatrao Kulkarni

DT bindings for numa map for memory, cores and IOs using arm,associativity device node property.

Quoted 8 lines

Quoted 2 lines

Given this is just a copy of ibm,associativity, I'm not sure I see much point in renaming the properties.

So just keep the ibm? I'm okay with that. That would help move to common code. Alternatively, we could drop the vendor prefix and have common code just check for both.

Hi all,

Why not copy the method of ACPI numa? There only three elements should be configured:
1) a cpu belong to which node
2) a memory block belong to which node
3) the distance of each two nodes

I too thought acpi only defines mapping for cpu and memory to numa nodes and no specification to define for IOs.
however after going through the x86 implementation, i can see there is provision for mapping IOs to numa node in acpi.
in x86 code, function pci_acpi_scan_root calls acpi_get_node to get associated node for pci bus using _PXM object.
it imply there is entry in acpi tables to map pci bus for numa node(proximity domain).
so in dt also, we should have binding to define cpu, memory and IOs to node mapping.

Quoted 53 lines

Quoted 39 lines

Quoted 23 lines

Quoted 19 lines

The devicetree nodes of numa can be like below:

/ {
         ...

         numa-nodes-info {
                 node-name: node-description {
                         mem-ranges = <...>;
                         cpus-list = <...>;
                 };

                 nodes-distance {
                         distance-list = <...>;
                 };
         };

         ...
};

some what similar to what your are proposing is already implemented in my v2 patchset.
https://lwn.net/Articles/623920/

Quoted 6 lines

http://lists.infradead.org/pipermail/linux-arm-kernel/2014-November/305164.html
we have went to associativity property based implementation to keep it more generic.
i do have both acpi(using linaro/hanjun's patches) and associativity based implementations on our internal tree
and tested on thunderx platform.

Great thanks!

Quoted 4 lines

i do see issue in creating numa mapping using ACPI for IOs(for example, i am not able to create numa mapping for ITS which is on each node, using ACPI tables), since ACPI spec (tables SRAT and SLIT) talks only about processor and memory.

Quoted 5 lines

I'm not sure why the ITS needs to know the NUMA domain, for my understanding, the interrupt will route to the correct NUMA domain using setting the affinity, ITS will configured to route it to the right GICR(cpu), so I think the ITS don't need to know which NUMA node belonging to, correct me if I missed something.

IIUC, GICR/collection is per cpu and can be mapped to numa node using cpu to node mapping.
However there are multiple its in multi-socket platform(at-least one its per socket),
knowing its to numa node mapping will help in routing(optimal) the interrupts to any one of GICR/collections of that node Hence, we need to find which its belongs to which socket/node using dt. same applies to pci bus too.

Quoted 1 line

Quoted 5 lines

Quoted 2 lines

Thanks
Hanjun

thanks
Ganapat

thanks
Ganapat

.

Hi,

Quoted 2 lines · Ganapatrao Kulkarni

DT bindings for numa map for memory, cores and IOs using arm,associativity device node property.

Given this is just a copy of ibm,associativity, I'm not sure I see much point in renaming the properties.

However, (somewhat counter to that) I'm also concerned that this isn't sufficient for systems we're beginning to see today (more on that below), so I don't think a simple copy of ibm,associativity is good enough.

Quoted 5 lines

Signed-off-by Ganapatrao Kulkarni <[email protected]>

---
 Documentation/devicetree/bindings/arm/numa.txt | 212

+++++++++++++++++++++++++

Review

Quoted 4 lines

 1 file changed, 212 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/arm/numa.txt

(quoted diff) +0 -0

b/Documentation/devicetree/bindings/arm/numa.txt

Documentation/devicetree/bindings/arm/numa.txt

Quoted 6 lines

new file mode 100644
index 0000000..dc3ef86

Documentation/devicetree/bindings/arm/numa.txt new file +0 -0

@@ -0,0 +1,212 @@

+==============================================================================

Documentation/devicetree/bindings/arm/numa.txt line 2

Quoted 2 lines

Documentation/devicetree/bindings/arm/numa.txt new file +1 -0

+NUMA binding description.

+==============================================================================

Documentation/devicetree/bindings/arm/numa.txt line 2

Quoted 2 lines

Documentation/devicetree/bindings/arm/numa.txt new file +1 -0

+

+==============================================================================

Documentation/devicetree/bindings/arm/numa.txt line 6

Quoted 2 lines

Documentation/devicetree/bindings/arm/numa.txt new file +1 -0

+1 - Introduction

+==============================================================================

Documentation/devicetree/bindings/arm/numa.txt line 5

Quoted 2 lines

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+
+Systems employing a Non Uniform Memory Access (NUMA) architecture

contain

Documentation/devicetree/bindings/arm/numa.txt line 6

Quoted 1 line

Documentation/devicetree/bindings/arm/numa.txt new file +1 -0

+collections of hardware resources including processors, memory, and I/O

buses,

Documentation/devicetree/bindings/arm/numa.txt line 8

Quoted 2 lines

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+that comprise what is commonly known as a NUMA node.
+Processor accesses to memory within the local NUMA node is generally

faster

Documentation/devicetree/bindings/arm/numa.txt line 16

Quoted 5 lines

Documentation/devicetree/bindings/arm/numa.txt new file +4 -0

+than processor accesses to memory outside of the local NUMA node.
+DT defines interfaces that allow the platform to convey NUMA node
+topology information to OS.
+

+==============================================================================

Documentation/devicetree/bindings/arm/numa.txt line 18

Quoted 2 lines

Documentation/devicetree/bindings/arm/numa.txt new file +1 -0

+2 - arm,associativity

+==============================================================================

Documentation/devicetree/bindings/arm/numa.txt line 15

Quoted 2 lines

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+The mapping is done using arm,associativity device property.
+this property needs to be present in every device node which needs to

to be

Documentation/devicetree/bindings/arm/numa.txt line 22

Quoted 1 line

Documentation/devicetree/bindings/arm/numa.txt new file +1 -0

+mapped to numa nodes.

Can't there be some inheritance? e.g. all devices on a bus with an arm,associativity property being assumed to share that value?

Documentation/devicetree/bindings/arm/numa.txt line 18

Quoted 3 lines

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+
+arm,associativity property is set of 32-bit integers which defines

level of

s/set/list/ -- the order is important.

Documentation/devicetree/bindings/arm/numa.txt

Quoted 5 lines

Documentation/devicetree/bindings/arm/numa.txt new file +4 -0

+topology and boundary in the system at which a significant difference in
+performance can be measured between cross-device accesses within
+a single location and those spanning multiple locations.
+The first cell always contains the broadest subdivision within the

system,

Documentation/devicetree/bindings/arm/numa.txt line 23

Quoted 1 line

Documentation/devicetree/bindings/arm/numa.txt new file +1 -0

+while the last cell enumerates the individual devices, such as an SMT

thread

Documentation/devicetree/bindings/arm/numa.txt line 30

Quoted 1 line

Documentation/devicetree/bindings/arm/numa.txt new file +1 -0

+of a CPU, or a bus bridge within an SoC".

While this gives us some hierarchy, this doesn't seem to encode relative distances at all. That seems like an oversight.

Additionally, I'm somewhat unclear on how what you'd be expected to provide for this property in cases like ring or mesh interconnects, where there isn't a strict hierarchy (see systems with ARM's own CCN, or Tilera's TILE-Mx), but there is some measure of closeness.

#(whatever)-cells property in the root to describe the expected length? If not, how are they to be interpreted relative to each other?

Documentation/devicetree/bindings/arm/numa.txt line 26

Quoted 3 lines

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+
+ex:

s/ex/Example:/, please. There's no need to contract that.

Documentation/devicetree/bindings/arm/numa.txt line 35

Quoted 4 lines

Documentation/devicetree/bindings/arm/numa.txt new file +3 -0

+       /* board 0, socket 0, cluster 0, core 0  thread 0 */
+       arm,associativity = <0 0 0 0 0>;
+

+==============================================================================

Documentation/devicetree/bindings/arm/numa.txt line 37

Quoted 2 lines

Documentation/devicetree/bindings/arm/numa.txt new file +1 -0

+3 - arm,associativity-reference-points

+==============================================================================

Documentation/devicetree/bindings/arm/numa.txt line 31

Quoted 1 line

Documentation/devicetree/bindings/arm/numa.txt new file +1 -0

+This property is a set of 32-bit integers, each representing an index

into

Likeise, s/set/list/

Documentation/devicetree/bindings/arm/numa.txt line 33

Quoted 3 lines

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+the arm,associativity nodes. The first integer is the most significant
+NUMA boundary and the following are progressively less significant

boundaries.

Documentation/devicetree/bindings/arm/numa.txt line 42

Quoted 1 line

Documentation/devicetree/bindings/arm/numa.txt new file +1 -0

+There can be more than one level of NUMA.

I'm not clear on why this is necessary; the arm,associativity property is already ordered from most significant to least significant per its description.

What does this property achieve?

The description also doesn't describe where this property is expected to live. The example isn't sufficient to disambiguate that, especially as it seems like a trivial case.

Is this only expected at the root of the tree? Can it be re-defined in sub-nodes?

Documentation/devicetree/bindings/arm/numa.txt line 36

Quoted 3 lines

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+
+Ex:

s/Ex/Example:/, please

Documentation/devicetree/bindings/arm/numa.txt line 38

Quoted 3 lines

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+       arm,associativity-reference-points = <0 1>;
+       The board Id(index 0) used first to calculate the associativity

(node

Documentation/devicetree/bindings/arm/numa.txt line 58

Quoted 13 lines

Documentation/devicetree/bindings/arm/numa.txt new file +12 -0

+       distance), then follows the  socket id(index 1).
+
+       arm,associativity-reference-points = <1 0>;
+       The socket Id(index 1) used first to calculate the associativity,
+       then follows the board id(index 0).
+
+       arm,associativity-reference-points = <0>;
+       Only the board Id(index 0) used to calculate the associativity.
+
+       arm,associativity-reference-points = <1>;
+       Only socket Id(index 1) used to calculate the associativity.
+

+==============================================================================

Documentation/devicetree/bindings/arm/numa.txt line 60

Quoted 2 lines

Documentation/devicetree/bindings/arm/numa.txt new file +1 -0

+4 - Example dts

+==============================================================================

Documentation/devicetree/bindings/arm/numa.txt line 53

Quoted 2 lines

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+
+Example: 2 Node system consists of 2 boards and each board having one

socket

Quoted 26 lines

Documentation/devicetree/bindings/arm/numa.txt new file +26 -0

+and 8 core in each socket.
+
+       arm,associativity-reference-points = <0>;
+
+       memory@00c00000 {
+               device_type = "memory";
+               reg = <0x0 0x00c00000 0x0 0x80000000>;
+               /* board 0, socket 0, no specific core */
+               arm,associativity = <0 0 0xffff>;
+       };
+
+       memory@10000000000 {
+               device_type = "memory";
+               reg = <0x100 0x00000000 0x0 0x80000000>;
+               /* board 1, socket 0, no specific core */
+               arm,associativity = <1 0 0xffff>;
+       };
+
+       cpus {
+               #address-cells = <2>;
+               #size-cells = <0>;
+
+               cpu@000 {
+                       device_type = "cpu";
+                       compatible =  "arm,armv8";
+                       reg = <0x0 0x000>;

Documentation/devicetree/bindings/arm/numa.txt line 92

Quoted 3 lines

Documentation/devicetree/bindings/arm/numa.txt new file +3 -0

+                       enable-method = "psci";
+                       /* board 0, socket 0, core 0*/
+                       arm,associativity = <0 0 0>;

We should specify w.r.t. memory and CPUs how the property is expected to be used (e.g. in the CPU nodes rather than the cpu-map, with separate memory nodes, etc). The generic description of arm,associativity isn't sufficient to limit confusion there.

Thanks,
Mark.

Hi Mark,

I have tried to answer your comments, in the meantime we are waiting for Ben to share the details.

Quoted 7 lines · Mark Rutland

Hi,

Quoted 2 lines · Ganapatrao Kulkarni

DT bindings for numa map for memory, cores and IOs using arm,associativity device node property.

Quoted 7 lines

Given this is just a copy of ibm,associativity, I'm not sure I see much point in renaming the properties.

However, (somewhat counter to that) I'm also concerned that this isn't sufficient for systems we're beginning to see today (more on that below), so I don't think a simple copy of ibm,associativity is good enough.

it is just copy right now, however it can evolve when we come across more arm64 numa platforms

Quoted 44 lines

Quoted 44 lines

Signed-off-by Ganapatrao Kulkarni <[email protected]>

---
 Documentation/devicetree/bindings/arm/numa.txt | 212
+++++++++++++++++++++++++
 1 file changed, 212 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/arm/numa.txt

Documentation/devicetree/bindings/arm/numa.txt new file +9 -0

new file mode 100644
index 0000000..dc3ef86
@@ -0,0 +1,212 @@
 
+==============================================================================
+NUMA binding description.
 
+==============================================================================
+
 
+==============================================================================
+1 - Introduction
 
+==============================================================================
+
+Systems employing a Non Uniform Memory Access (NUMA) architecture

contain
+collections of hardware resources including processors, memory, and I/O buses,

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+that comprise what is commonly known as a NUMA node.
+Processor accesses to memory within the local NUMA node is generally

faster

Documentation/devicetree/bindings/arm/numa.txt new file +6 -0

+than processor accesses to memory outside of the local NUMA node.
+DT defines interfaces that allow the platform to convey NUMA node
+topology information to OS.
+
 
+==============================================================================
+2 - arm,associativity

Quoted 8 lines

Quoted 5 lines

+==============================================================================

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+The mapping is done using arm,associativity device property.
+this property needs to be present in every device node which needs to

to be
+mapped to numa nodes.

Can't there be some inheritance? e.g. all devices on a bus with an arm,associativity property being assumed to share that value?

yes there is inheritance and respective bus drivers should take care of it, like pci driver does at present.

Quoted 7 lines

Quoted 5 lines

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+
+arm,associativity property is set of 32-bit integers which defines

level of

s/set/list/ -- the order is important.

ok

Quoted 14 lines

Quoted 11 lines

+topology and boundary in the system at which a significant difference in

Documentation/devicetree/bindings/arm/numa.txt new file +3 -0

+performance can be measured between cross-device accesses within
+a single location and those spanning multiple locations.
+The first cell always contains the broadest subdivision within the

system,
+while the last cell enumerates the individual devices, such as an SMT thread
+of a CPU, or a bus bridge within an SoC".

While this gives us some hierarchy, this doesn't seem to encode relative distances at all. That seems like an oversight.

distance is computed, will add the details to document.
local nodes will have distance as 10(LOCAL_DISTANCE) and every level, the distance multiplies by 2.
for example, for level 1 numa topology, distance from local node to remote node will be 20.

Quoted 6 lines

Additionally, I'm somewhat unclear on how what you'd be expected to provide for this property in cases like ring or mesh interconnects, where there isn't a strict hierarchy (see systems with ARM's own CCN, or Tilera's TILE-Mx), but there is some measure of closeness.

IIUC, as per ARMs CCN architecture, all core/clusters are at equal distance of DDR, i dont see any NUMA topology.
however, if there are 2 SoC connected thorough the CCN, then it is very much similar to cavium topology.

Quoted 3 lines

Must all of these have the same length? If so, why not have a #(whatever)-cells property in the root to describe the expected length? If not, how are they to be interpreted relative to each other?

yes, all are of default size.
IMHO, there is no need to add cells property.

Quoted 11 lines

Documentation/devicetree/bindings/arm/numa.txt

Quoted 4 lines

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+
+ex:

s/ex/Example:/, please. There's no need to contract that.

Quoted 4 lines

Documentation/devicetree/bindings/arm/numa.txt new file +3 -0

+       /* board 0, socket 0, cluster 0, core 0  thread 0 */
+       arm,associativity = <0 0 0 0 0>;
+

Quoted 8 lines

Quoted 6 lines

+==============================================================================
+3 - arm,associativity-reference-points

+==============================================================================
+This property is a set of 32-bit integers, each representing an index into

Likeise, s/set/list/

ok

Quoted 10 lines

Quoted 6 lines

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+the arm,associativity nodes. The first integer is the most significant
+NUMA boundary and the following are progressively less significant

boundaries.
+There can be more than one level of NUMA.

I'm not clear on why this is necessary; the arm,associativity property is already ordered from most significant to least significant per its description.

first entry in arm,associativity-reference-points is used to find which entry in associativity defines node id.
also entries in arm,associativity-reference-points defines, how many entries(depth) in associativity can be used to calculate node distance
in both level 1 and multi level(hierarchical) numa topology.

Quoted 7 lines

What does this property achieve?

The description also doesn't describe where this property is expected to live. The example isn't sufficient to disambiguate that, especially as it seems like a trivial case.

sure, will add one more example to describe the
arm,associativity-reference-points

Quoted 4 lines

Is this only expected at the root of the tree? Can it be re-defined in sub-nodes?

yes it is defined only at the root.

Quoted 6 lines

Documentation/devicetree/bindings/arm/numa.txt

Quoted 4 lines

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+
+Ex:

s/Ex/Example:/, please

sure.

Quoted 52 lines

Quoted 52 lines

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+       arm,associativity-reference-points = <0 1>;
+       The board Id(index 0) used first to calculate the associativity

(node

Documentation/devicetree/bindings/arm/numa.txt new file +4 -0

+       distance), then follows the  socket id(index 1).
+
+       arm,associativity-reference-points = <1 0>;
+       The socket Id(index 1) used first to calculate the

associativity,

Documentation/devicetree/bindings/arm/numa.txt new file +13 -0

+       then follows the board id(index 0).
+
+       arm,associativity-reference-points = <0>;
+       Only the board Id(index 0) used to calculate the associativity.
+
+       arm,associativity-reference-points = <1>;
+       Only socket Id(index 1) used to calculate the associativity.
+
 
+==============================================================================
+4 - Example dts
 
+==============================================================================
+
+Example: 2 Node system consists of 2 boards and each board having one

socket

Documentation/devicetree/bindings/arm/numa.txt new file +26 -0

+and 8 core in each socket.
+
+       arm,associativity-reference-points = <0>;
+
+       memory at 00c00000 {
+               device_type = "memory";
+               reg = <0x0 0x00c00000 0x0 0x80000000>;
+               /* board 0, socket 0, no specific core */
+               arm,associativity = <0 0 0xffff>;
+       };
+
+       memory at 10000000000 {
+               device_type = "memory";
+               reg = <0x100 0x00000000 0x0 0x80000000>;
+               /* board 1, socket 0, no specific core */
+               arm,associativity = <1 0 0xffff>;
+       };
+
+       cpus {
+               #address-cells = <2>;
+               #size-cells = <0>;
+
+               cpu at 000 {
+                       device_type = "cpu";
+                       compatible =  "arm,armv8";
+                       reg = <0x0 0x000>;

Quoted 8 lines

Documentation/devicetree/bindings/arm/numa.txt line 92

Quoted 3 lines

Documentation/devicetree/bindings/arm/numa.txt new file +3 -0

+                       enable-method = "psci";
+                       /* board 0, socket 0, core 0*/
+                       arm,associativity = <0 0 0>;

We should specify w.r.t. memory and CPUs how the property is expected to be used (e.g. in the CPU nodes rather than the cpu-map, with separate memory nodes, etc). The generic description of arm,associativity isn't sufficient to limit confusion there.

ok, will add the details like which nodes can use this property.

Quoted 4 lines

Thanks,
Mark.

thanks
Ganapat

(sending again, by mistake it was set to html mode)

Quoted 322 lines · Ganapatrao Kulkarni

Hi Mark,

I have tried to answer your comments, in the meantime we are waiting for Ben
to share the details.

On Fri, Aug 28, 2015 at 6:02 PM, Mark Rutland <[email protected]

Quoted 13 lines

wrote:

Hi,

Quoted 2 lines · Ganapatrao Kulkarni

DT bindings for numa map for memory, cores and IOs using arm,associativity device node property.

Given this is just a copy of ibm,associativity, I'm not sure I see much
point in renaming the properties.

Quoted 6 lines

However, (somewhat counter to that) I'm also concerned that this isn't
sufficient for systems we're beginning to see today (more on that below), so I don't think a simple copy of ibm,associativity is good
enough.

it is just copy right now, however it can evolve when we come across more
arm64 numa platforms

Quoted 56 lines

Quoted 56 lines

Signed-off-by: Ganapatrao Kulkarni <
gkulkarni at caviumnetworks.com>

---
 Documentation/devicetree/bindings/arm/numa.txt | 212
+++++++++++++++++++++++++
 1 file changed, 212 insertions(+)
 create mode 100644

Documentation/devicetree/bindings/arm/numa.txt

Documentation/devicetree/bindings/arm/numa.txt new file +1 -0

new file mode 100644
index 0000000..dc3ef86
@@ -0,0 +1,212 @@
 
+==============================================================

================
+NUMA binding description.

+==============================================================
================
+

+==============================================================
================
+1 - Introduction

+==============================================================
================

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+
+Systems employing a Non Uniform Memory Access (NUMA)

architecture
contain
+collections of hardware resources including processors, memory, and I/O
buses,

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+that comprise what is commonly known as a NUMA node.
+Processor accesses to memory within the local NUMA node is

generally
faster
+than processor accesses to memory outside of the local NUMA node.
+DT defines interfaces that allow the platform to convey NUMA node

Documentation/devicetree/bindings/arm/numa.txt new file +3 -0

+topology information to OS.
+
 
+==============================================================

================
+2 - arm,associativity

Quoted 11 lines

Quoted 7 lines

+==============================================================
================

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+The mapping is done using arm,associativity device property.
+this property needs to be present in every device node which

needs to
to be
+mapped to numa nodes.

Can't there be some inheritance? e.g. all devices on a bus with an
arm,associativity property being assumed to share that value?

yes there is inheritance and respective bus drivers should take care of it,
like pci driver does at present.

Quoted 8 lines

Quoted 6 lines

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+
+arm,associativity property is set of 32-bit integers which

defines
level of

s/set/list/ -- the order is important.

ok

Quoted 11 lines

Quoted 11 lines

+topology and boundary in the system at which a significant difference
in
+performance can be measured between cross-device accesses within

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+a single location and those spanning multiple locations.
+The first cell always contains the broadest subdivision within

the
system,

Quoted 8 lines

Quoted 4 lines

+while the last cell enumerates the individual devices, such as an SMT
thread
+of a CPU, or a bus bridge within an SoC".

While this gives us some hierarchy, this doesn't seem to encode relative
distances at all. That seems like an oversight.

distance is computed, will add the details to document. local nodes will have distance as 10(LOCAL_DISTANCE) and every level, the
distance multiplies by 2.
for example, for level 1 numa topology, distance from local node to remote
node will be 20.

Quoted 9 lines

Additionally, I'm somewhat unclear on how what you'd be expected to
provide for this property in cases like ring or mesh interconnects,
where there isn't a strict hierarchy (see systems with ARM's own CCN, or
Tilera's TILE-Mx), but there is some measure of closeness.

IIUC, as per ARMs CCN architecture, all core/clusters are at equal distance
of DDR, i dont see any NUMA topology.
however, if there are 2 SoC connected thorough the CCN, then it is very much
similar to cavium topology.

Quoted 4 lines

Must all of these have the same length? If so, why not have a #(whatever)-cells property in the root to describe the expected length?
If not, how are they to be interpreted relative to each other?

yes, all are of default size.
IMHO, there is no need to add cells property.

Quoted 15 lines

Documentation/devicetree/bindings/arm/numa.txt

Quoted 4 lines

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+
+ex:

s/ex/Example:/, please. There's no need to contract that.

Quoted 8 lines

Documentation/devicetree/bindings/arm/numa.txt new file +4 -0

+       /* board 0, socket 0, cluster 0, core 0  thread 0 */
+       arm,associativity = <0 0 0 0 0>;
+
 
+==============================================================

================
+3 - arm,associativity-reference-points

Quoted 7 lines

Quoted 5 lines

+==============================================================
================
+This property is a set of 32-bit integers, each representing an index
into

Likeise, s/set/list/

ok

Quoted 14 lines

Quoted 8 lines

+the arm,associativity nodes. The first integer is the most significant
+NUMA boundary and the following are progressively less significant
boundaries.
+There can be more than one level of NUMA.

I'm not clear on why this is necessary; the arm,associativity property
is already ordered from most significant to least significant per its
description.

first entry in arm,associativity-reference-points is used to find which
entry in associativity defines node id.
also entries in arm,associativity-reference-points defines, how many entries(depth) in associativity can be used to calculate node
distance
in both level 1 and multi level(hierarchical) numa topology.

Quoted 9 lines

What does this property achieve?

The description also doesn't describe where this property is expected to
live. The example isn't sufficient to disambiguate that, especially as
it seems like a trivial case.

sure, will add one more example to describe the
arm,associativity-reference-points

Quoted 5 lines

Is this only expected at the root of the tree? Can it be re -defined in
sub-nodes?

yes it is defined only at the root.

Quoted 6 lines

Documentation/devicetree/bindings/arm/numa.txt

Quoted 4 lines

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+
+Ex:

s/Ex/Example:/, please

sure.

Quoted 62 lines

Quoted 51 lines

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+       arm,associativity-reference-points = <0 1>;
+       The board Id(index 0) used first to calculate the

associativity
(node

Documentation/devicetree/bindings/arm/numa.txt new file +4 -0

+       distance), then follows the  socket id(index 1).
+
+       arm,associativity-reference-points = <1 0>;
+       The socket Id(index 1) used first to calculate the

associativity,

Documentation/devicetree/bindings/arm/numa.txt new file +4 -0

+       then follows the board id(index 0).
+
+       arm,associativity-reference-points = <0>;
+       Only the board Id(index 0) used to calculate the

associativity.

Documentation/devicetree/bindings/arm/numa.txt new file +3 -0

+
+       arm,associativity-reference-points = <1>;
+       Only socket Id(index 1) used to calculate the

associativity.
+

+==============================================================
================
+4 - Example dts

+==============================================================
================

Documentation/devicetree/bindings/arm/numa.txt new file +2 -0

+
+Example: 2 Node system consists of 2 boards and each board

having one
socket

Documentation/devicetree/bindings/arm/numa.txt new file +19 -0

+and 8 core in each socket.
+
+       arm,associativity-reference-points = <0>;
+
+       memory at 00c00000 {
+               device_type = "memory";
+               reg = <0x0 0x00c00000 0x0 0x80000000>;
+               /* board 0, socket 0, no specific core */
+               arm,associativity = <0 0 0xffff>;
+       };
+
+       memory at 10000000000 {
+               device_type = "memory";
+               reg = <0x100 0x00000000 0x0 0x80000000>;
+               /* board 1, socket 0, no specific core */
+               arm,associativity = <1 0 0xffff>;
+       };
+
+       cpus {

Documentation/devicetree/bindings/arm/numa.txt line 92

Quoted 10 lines

Documentation/devicetree/bindings/arm/numa.txt new file +10 -0

+               #address-cells = <2>;
+               #size-cells = <0>;
+
+               cpu at 000 {
+                       device_type = "cpu";
+                       compatible =  "arm,armv8";
+                       reg = <0x0 0x000>;
+                       enable-method = "psci";
+                       /* board 0, socket 0, core 0*/
+                       arm,associativity = <0 0 0>;

Quoted 7 lines

We should specify w.r.t. memory and CPUs how the property is expected to
be used (e.g. in the CPU nodes rather than the cpu-map, with separate
memory nodes, etc). The generic description of arm,associativity isn't
sufficient to limit confusion there.

ok, will add the details like which nodes can use this property.

Quoted 4 lines

Thanks,
Mark.

thanks
Ganapat