x86/apic: limit irq affinity
Subsystems (2)
PATCH v6 1/1 x86/apic: limit irq affinity
This patch allows for hard numa restrictions to irq affinity on x86 systems.
Affinity is masked to allow only those cpus which the subarchitecture deems accessible by the given irq.
On some UV systems, this domain will be limited to the nodes accessible to the irq's node. Initially other X86 systems will not mask off any cpus so non-UV systems will remain unaffected.
---
Added apic functions for getting numa irq cpumasks. Systems other than UV now simply return the mask passed in. Added irq_desc_node for uniprocessor compatibility.
arch/x86/Kconfig | 1 arch/x86/include/asm/apic.h | 13 +++ arch/x86/include/asm/uv/uv_irq.h | 3 arch/x86/include/asm/uv/uv_mmrs.h | 25 +++++++ arch/x86/kernel/apic/apic_flat_64.c | 4 + arch/x86/kernel/apic/apic_noop.c | 2 arch/x86/kernel/apic/bigsmp_32.c | 2 arch/x86/kernel/apic/es7000_32.c | 4 + arch/x86/kernel/apic/io_apic.c | 102 +++++++++++++++++++++++------- arch/x86/kernel/apic/numaq_32.c | 2 arch/x86/kernel/apic/probe_32.c | 2 arch/x86/kernel/apic/summit_32.c | 2 arch/x86/kernel/apic/x2apic_cluster.c | 2 arch/x86/kernel/apic/x2apic_phys.c | 2 arch/x86/kernel/apic/x2apic_uv_x.c | 4 + arch/x86/kernel/uv_irq.c | 66 +++++++++++++++++++ 16 files changed, 214 insertions(+), 22 deletions(-)
arch/x86/kernel/apic/io_apic.c +80 -22
@@ -352,6 +352,18 @@ struct irq_cfg *irq_cfg(unsigned int irq#endif +#ifdef CONFIG_SMP +static int irq_desc_node (struct irq_desc *desc) +{ +return desc->node; +} +#else +static int irq_desc_node (struct irq_desc *desc) +{ +return 0 ; +} +#endif +struct io_apic {unsigned int index;unsigned int unused[3 ];
@@ -1427,6 +1439,7 @@ static void setup_IO_APIC_irq(int apic_i {struct irq_cfg *cfg;struct IO_APIC_route_entry entry; +struct cpumask *tmp_mask;unsigned int dest;if (!IO_APIC_IRQ (irq))
@@ -1434,10 +1447,15 @@ static void setup_IO_APIC_irq(int apic_i cfg = desc->chip_data; -if (assign_irq_vector (irq, cfg, apic->target_cpus ())) + tmp_mask = apic->get_restricted_mask (apic->target_cpus (), +irq_desc_node (desc)); +if (!tmp_mask)return ; - dest = apic->cpu_mask_to_apicid_and (cfg->domain, apic->target_cpus ()); +if (assign_irq_vector (irq, cfg, tmp_mask)) + goto error; + + dest = apic->cpu_mask_to_apicid_and (cfg->domain, tmp_mask);apic_printk (APIC_VERBOSE ,KERN_DEBUG "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> "
@@ -1451,7 +1469,7 @@ static void setup_IO_APIC_irq(int apic_iprintk ("Failed to setup ioapic entry for ioapic %d, pin %d\n" , mp_ioapics[apic_id].apicid, pin);__clear_irq_vector (irq, cfg); -return ; + goto error; }ioapic_register_intr (irq, desc, trigger);
@@ -1459,6 +1477,8 @@ static void setup_IO_APIC_irq(int apic_idisable_8259A_irq (irq);ioapic_write_entry (apic_id, pin, entry); +error: + apic->free_restricted_mask (tmp_mask); } static struct {
@@ -2278,18 +2298,30 @@ set_desc_affinity(struct irq_desc *desc, {struct irq_cfg *cfg;unsigned int irq; - -if (!cpumask_intersects (mask, cpu_online_mask)) -return BAD_APICID ; +struct cpumask *tmp_mask; irq = desc->irq; cfg = desc->chip_data; -if (assign_irq_vector (irq, cfg, mask)) + + tmp_mask = apic->get_restricted_mask (mask,irq_desc_node (desc)); +if (!tmp_mask)return BAD_APICID ; -cpumask_copy (desc->affinity, mask); +if (!cpumask_intersects (tmp_mask, cpu_online_mask)) + goto error; + +if (assign_irq_vector (irq, cfg, tmp_mask)) + goto error; + +cpumask_copy (desc->affinity, tmp_mask); + + apic->free_restricted_mask (tmp_mask);return apic->cpu_mask_to_apicid_and (desc->affinity, cfg->domain); + +error: + apic->free_restricted_mask (tmp_mask); +return BAD_APICID ; } static int
@@ -2345,22 +2377,29 @@ migrate_ioapic_irq_desc(struct irq_desc {struct irq_cfg *cfg;struct irte irte; +struct cpumask *tmp_mask;unsigned int dest;unsigned int irq;int ret =-1 ; -if (!cpumask_intersects (mask, cpu_online_mask)) + tmp_mask = apic->get_restricted_mask (mask,irq_desc_node (desc)); +if (!tmp_mask)return ret; +if (!cpumask_intersects (tmp_mask, cpu_online_mask)) + goto error; + irq = desc->irq;if (get_irte (irq, &irte)) -return ret; + goto error; cfg = desc->chip_data; -if (assign_irq_vector (irq, cfg, mask)) -return ret; +if (assign_irq_vector (irq, cfg, tmp_mask)) + goto error; + + ret =0 ; - dest = apic->cpu_mask_to_apicid_and (cfg->domain, mask); + dest = apic->cpu_mask_to_apicid_and (cfg->domain, tmp_mask); irte.vector = cfg->vector; irte.dest_id =IRTE_DEST (dest);
@@ -2373,9 +2412,10 @@ migrate_ioapic_irq_desc(struct irq_descif (cfg->move_in_progress)send_cleanup_vector (cfg); -cpumask_copy (desc->affinity, mask); - -return 0 ; +cpumask_copy (desc->affinity, tmp_mask); +error: + apic->free_restricted_mask (tmp_mask); +return ret; }/*
@@ -3244,18 +3284,27 @@ static int msi_compose_msg(struct pci_de struct msi_msg *msg, u8 hpet_id) {struct irq_cfg *cfg; +struct irq_desc *desc;int err;unsigned dest; +struct cpumask *tmp_mask;if (disable_apic)return -ENXIO ; cfg =irq_cfg (irq); - err =assign_irq_vector (irq, cfg, apic->target_cpus ()); + desc =irq_to_desc (irq); + + tmp_mask = apic->get_restricted_mask (apic->target_cpus (), +irq_desc_node (desc)); +if (!tmp_mask) +return -ENOMEM ; + + err =assign_irq_vector (irq, cfg, tmp_mask);if (err) -return err; + goto error; - dest = apic->cpu_mask_to_apicid_and (cfg->domain, apic->target_cpus ()); + dest = apic->cpu_mask_to_apicid_and (cfg->domain, tmp_mask);if (irq_remapped (irq)) {struct irte irte;
@@ -3313,6 +3362,8 @@ static int msi_compose_msg(struct pci_de MSI_DATA_DELIVERY_LOWPRI) |MSI_DATA_VECTOR (cfg->vector); } +error: + apic->free_restricted_mask (tmp_mask);return err; }
@@ -3730,19 +3781,25 @@ static struct irq_chip ht_irq_chip = {int arch_setup_ht_irq (unsigned int irq,struct pci_dev *dev) {struct irq_cfg *cfg; +struct cpumask *tmp_mask;int err;if (disable_apic)return -ENXIO ; cfg =irq_cfg (irq); - err =assign_irq_vector (irq, cfg, apic->target_cpus ()); + + tmp_mask = apic->get_restricted_mask (apic->target_cpus (), +dev_to_node (&dev->dev)); +if (!tmp_mask) +return -ENOMEM ; + + err =assign_irq_vector (irq, cfg, tmp_mask);if (!err) {struct ht_irq_msg msg;unsigned dest; - dest = apic->cpu_mask_to_apicid_and (cfg->domain, - apic->target_cpus ()); + dest = apic->cpu_mask_to_apicid_and (cfg->domain, tmp_mask); msg.address_hi =HT_IRQ_HIGH_DEST_ID (dest);
@@ -3766,6 +3823,7 @@ int arch_setup_ht_irq(unsigned int irq, dev_printk(KERN_DEBUG, &dev->dev, "irq %d for HT\n", irq); } + apic->free_restricted_mask (tmp_mask);return err; }#endif /* CONFIG_HT_IRQ */
arch/x86/include/asm/uv/uv_mmrs.h +25 -0
@@ -823,6 +823,31 @@ union uvh_lb_mcast_aoerr0_rpt_enable_u { };/* ========================================================================= */ +/* UVH_LB_SOCKET_DESTINATION_TABLE */ +/* ========================================================================= */ +#define UVH_LB_SOCKET_DESTINATION_TABLE 0x321000UL +#define UVH_LB_SOCKET_DESTINATION_TABLE_32 0x1800 +#define UVH_LB_SOCKET_DESTINATION_TABLE_DEPTH 128 + +#define UVH_LB_SOCKET_DESTINATION_TABLE_NODE_ID_SHFT 1 +#define UVH_LB_SOCKET_DESTINATION_TABLE_NODE_ID_MASK 0x0000000000007ffeUL +#define UVH_LB_SOCKET_DESTINATION_TABLE_CHIP_ID_SHFT 15 +#define UVH_LB_SOCKET_DESTINATION_TABLE_CHIP_ID_MASK 0x0000000000008000UL +#define UVH_LB_SOCKET_DESTINATION_TABLE_PARITY_SHFT 16 +#define UVH_LB_SOCKET_DESTINATION_TABLE_PARITY_MASK 0x0000000000010000UL + +union uvh_lb_socket_destination_table_u { +unsigned long v; +struct uvh_lb_socket_destination_table_s { +unsigned long rsvd_0 :1 ;/* */ +unsigned long node_id :14 ;/* RW */ +unsigned long chip_id :1 ;/* RW */ +unsigned long parity :1 ;/* RW */ +unsigned long rsvd_17_63:47 ;/* */ + } s; +}; + +/* ========================================================================= */ /* UVH_LOCAL_INT0_CONFIG */ /* ========================================================================= */ #define UVH_LOCAL_INT0_CONFIG 0x61000UL
arch/x86/Kconfig +1 -0
@@ -365,6 +365,7 @@ config X86_UV depends on X86_EXTENDED_PLATFORM depends on NUMA depends on X86_X2APIC + depends on NUMA_IRQ_DESC ---help--- This option is needed in order to support SGI Ultraviolet systems. If you don't have one of these, you should say N here.
arch/x86/kernel/uv_irq.c +66 -0
72 lines not on this page · raw message
arch/x86/kernel/apic/x2apic_uv_x.c +4 -0
@@ -23,6 +23,7 @@#include <asm/uv/uv_mmrs.h> #include <asm/uv/uv_hub.h> +#include <asm/uv/uv_irq.h> #include <asm/current.h> #include <asm/pgtable.h> #include <asm/uv/bios.h>
@@ -264,6 +265,8 @@ struct apic __refdata apic_x2apic_uv_x = .irq_dest_mode = 0, /* physical */ .target_cpus = uv_target_cpus, + .get_restricted_mask = uv_get_restricted_mask, + .free_restricted_mask = uv_free_restricted_mask, .disable_esr = 0, .dest_logical = APIC_DEST_LOGICAL, .check_apicid_used = NULL,
@@ -659,5 +662,6 @@ void __init uv_system_init(void)uv_cpu_init ();uv_scir_register_cpu_notifier (); +arch_init_uv_cfg_cpus_allowed ();proc_mkdir ("sgi_uv" ,NULL ); }
arch/x86/include/asm/uv/uv_irq.h +3 -0
@@ -31,6 +31,9 @@ enum { UV_AFFINITY_CPU }; +extern struct cpumask *uv_get_restricted_mask (const struct cpumask *,int ); +extern void uv_free_restricted_mask (struct cpumask *); +extern void arch_init_uv_cfg_cpus_allowed (void );extern int uv_irq_2_mmr_info (int ,unsigned long *,int *);extern int uv_setup_irq (char *,int ,int ,unsigned long ,int );extern void uv_teardown_irq (unsigned int );
arch/x86/include/asm/apic.h +13 -0
@@ -293,6 +293,9 @@ struct apic {u32 irq_dest_mode;const struct cpumask *(*target_cpus)(void ); +struct cpumask *(*get_restricted_mask)(const struct cpumask *mask, +int node); +void (*free_restricted_mask)(struct cpumask *mask);int disable_esr;
@@ -474,6 +477,16 @@ static inline const struct cpumask *defa#endif } +static inline struct cpumask * +default_get_restricted_mask (const struct cpumask *mask,int node) +{ +return (struct cpumask *)mask; +} + +static inline void default_free_restricted_mask (struct cpumask *mask) +{ +} +DECLARE_EARLY_PER_CPU (u16, x86_bios_cpu_apicid);
arch/x86/kernel/apic/apic_flat_64.c +4 -0
@@ -174,6 +174,8 @@ struct apic apic_flat = { .irq_dest_mode = 1, /* logical */ .target_cpus = flat_target_cpus, + .get_restricted_mask = default_get_restricted_mask, + .free_restricted_mask = default_free_restricted_mask, .disable_esr = 0, .dest_logical = APIC_DEST_LOGICAL, .check_apicid_used = NULL,
@@ -323,6 +325,8 @@ struct apic apic_physflat = { .irq_dest_mode = 0, /* physical */ .target_cpus = physflat_target_cpus, + .get_restricted_mask = default_get_restricted_mask, + .free_restricted_mask = default_free_restricted_mask, .disable_esr = 0, .dest_logical = 0, .check_apicid_used = NULL,
arch/x86/kernel/apic/apic_noop.c +2 -0
@@ -142,6 +142,8 @@ struct apic apic_noop = { .irq_dest_mode = 1, .target_cpus = noop_target_cpus, + .get_restricted_mask = default_get_restricted_mask, + .free_restricted_mask = default_free_restricted_mask, .disable_esr = 0, .dest_logical = APIC_DEST_LOGICAL, .check_apicid_used = noop_check_apicid_used,
arch/x86/kernel/apic/bigsmp_32.c +2 -0
@@ -211,6 +211,8 @@ struct apic apic_bigsmp = { .irq_dest_mode = 0, .target_cpus = bigsmp_target_cpus, + .get_restricted_mask = default_get_restricted_mask, + .free_restricted_mask = default_free_restricted_mask, .disable_esr = 1, .dest_logical = 0, .check_apicid_used = bigsmp_check_apicid_used,
arch/x86/kernel/apic/es7000_32.c +4 -0
@@ -661,6 +661,8 @@ struct apic __refdata apic_es7000_cluste .irq_dest_mode = 1, .target_cpus = target_cpus_cluster, + .get_restricted_mask = default_get_restricted_mask, + .free_restricted_mask = default_free_restricted_mask, .disable_esr = 1, .dest_logical = 0, .check_apicid_used = es7000_check_apicid_used,
@@ -726,6 +728,8 @@ struct apic __refdata apic_es7000 = { .irq_dest_mode = 0, .target_cpus = es7000_target_cpus, + .get_restricted_mask = default_get_restricted_mask, + .free_restricted_mask = default_free_restricted_mask, .disable_esr = 1, .dest_logical = 0, .check_apicid_used = es7000_check_apicid_used,
arch/x86/kernel/apic/numaq_32.c +2 -0
@@ -500,6 +500,8 @@ struct apic __refdata apic_numaq = { .irq_dest_mode = 0, .target_cpus = numaq_target_cpus, + .get_restricted_mask = default_get_restricted_mask, + .free_restricted_mask = default_free_restricted_mask, .disable_esr = 1, .dest_logical = APIC_DEST_LOGICAL, .check_apicid_used = numaq_check_apicid_used,
arch/x86/kernel/apic/probe_32.c +2 -0
@@ -94,6 +94,8 @@ struct apic apic_default = { .irq_dest_mode = 1, .target_cpus = default_target_cpus, + .get_restricted_mask = default_get_restricted_mask, + .free_restricted_mask = default_free_restricted_mask, .disable_esr = 0, .dest_logical = APIC_DEST_LOGICAL, .check_apicid_used = default_check_apicid_used,
arch/x86/kernel/apic/summit_32.c +2 -0
@@ -517,6 +517,8 @@ struct apic apic_summit = { .irq_dest_mode = 1, .target_cpus = summit_target_cpus, + .get_restricted_mask = default_get_restricted_mask, + .free_restricted_mask = default_free_restricted_mask, .disable_esr = 1, .dest_logical = APIC_DEST_LOGICAL, .check_apicid_used = summit_check_apicid_used,
arch/x86/kernel/apic/x2apic_cluster.c +2 -0
@@ -198,6 +198,8 @@ struct apic apic_x2apic_cluster = { .irq_dest_mode = 1, /* logical */ .target_cpus = x2apic_target_cpus, + .get_restricted_mask = default_get_restricted_mask, + .free_restricted_mask = default_free_restricted_mask, .disable_esr = 0, .dest_logical = APIC_DEST_LOGICAL, .check_apicid_used = NULL,
arch/x86/kernel/apic/x2apic_phys.c +2 -0
@@ -187,6 +187,8 @@ struct apic apic_x2apic_phys = { .irq_dest_mode = 0, /* physical */ .target_cpus = x2apic_target_cpus, + .get_restricted_mask = default_get_restricted_mask, + .free_restricted_mask = default_free_restricted_mask, .disable_esr = 0, .dest_logical = 0, .check_apicid_used = NULL,
Quoted 8 lines · Dimitri Sivanich
This patch allows for hard numa restrictions to irq affinity on x86 systems.
Affinity is masked to allow only those cpus which the subarchitecture deems accessible by the given irq.
On some UV systems, this domain will be limited to the nodes accessible to the irq's node. Initially other X86 systems will not mask off any cpus so non-UV systems will remain unaffected.
Is this a hardware restriction you are trying to model? If not this seems wrong.
Eric
Quoted 7 lines · Eric W. Biederman
Quoted 5 lines · Dimitri Sivanich
This patch allows for hard numa restrictions to irq affinity on x86 systems.
Affinity is masked to allow only those cpus which the subarchitecture deems accessible by the given irq.
Quoted 6 lines
Quoted 3 lines
On some UV systems, this domain will be limited to the nodes accessible to the irq's node. Initially other X86 systems will not mask off any cpus so non-UV systems will remain unaffected.
Is this a hardware restriction you are trying to model? If not this seems wrong.
Yes, it's a hardware restriction.
Quoted 8 lines · Dimitri Sivanich
Quoted 7 lines · Eric W. Biederman
Quoted 5 lines · Dimitri Sivanich
This patch allows for hard numa restrictions to irq affinity on x86 systems.
Affinity is masked to allow only those cpus which the subarchitecture deems accessible by the given irq.
Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
On some UV systems, this domain will be limited to the nodes accessible to the irq's node. Initially other X86 systems will not mask off any cpus so non-UV systems will remain unaffected.
Is this a hardware restriction you are trying to model? If not this seems wrong.
Yes, it's a hardware restriction.
Nevertheless I think that this is the wrong approach.
What we really want is a notion in the irq descriptor which tells us:
this interrupt is restricted to numa node N.
The solution in this patch is just restricted to x86 and hides that information deep in the arch code.
Further the patch adds code which should be in the generic interrupt management code as it is useful for other purposes as well:
Driver folks are looking for a way to restrict irq balancing to a given numa node when they have all the driver data allocated on that node. That's not a hardware restriction as in the UV case but requires a similar infrastructure.
One possible solution would be to have a new flag:
IRQF_NODE_BOUND - irq is bound to desc->node
When an interrupt is set up we would query with a new irq_chip function chip->get_node_affinity(irq) which would default to an empty implementation returning -1. The arch code can provide its own function to return the numa affinity which would express the hardware restriction.
The core code would restrict affinity settings to the cpumask of that node without any need for the arch code to check it further.
That same infrastructure could be used for the software restriction of interrupts to a node on which the device is bound.
Having it in the core code also allows us to expose this information to user space so that the irq balancer knows about it and does not try to randomly move the affinity to cpus which are not in the allowed set of the node.
Thanks,
tglx
Quoted 47 lines · Thomas Gleixner
Quoted 8 lines · Dimitri Sivanich
Quoted 7 lines · Eric W. Biederman
Quoted 5 lines · Dimitri Sivanich
This patch allows for hard numa restrictions to irq affinity on x86 systems.
Affinity is masked to allow only those cpus which the subarchitecture deems accessible by the given irq.
Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
On some UV systems, this domain will be limited to the nodes accessible to the irq's node. Initially other X86 systems will not mask off any cpus so non-UV systems will remain unaffected.
Is this a hardware restriction you are trying to model? If not this seems wrong.
Yes, it's a hardware restriction.
Nevertheless I think that this is the wrong approach.
What we really want is a notion in the irq descriptor which tells us:
this interrupt is restricted to numa node N.The solution in this patch is just restricted to x86 and hides that information deep in the arch code.
Further the patch adds code which should be in the generic interrupt management code as it is useful for other purposes as well:
Driver folks are looking for a way to restrict irq balancing to a given numa node when they have all the driver data allocated on that node. That's not a hardware restriction as in the UV case but requires a similar infrastructure.
One possible solution would be to have a new flag:
IRQF_NODE_BOUND- irq is bound todesc->nodeWhen an interrupt is set up we would query with a new
irq_chipfunctionchip->get_node_affinity(irq)which would default to an empty implementation returning -1. The arch code can provide its own function to return the numa affinity which would express the hardware restriction.The core code would restrict affinity settings to the cpumask of that node without any need for the arch code to check it further.
Quoted 7 lines
That same infrastructure could be used for the software restriction of interrupts to a node on which the device is bound.
Having it in the core code also allows us to expose this information to user space so that the irq balancer knows about it and does not try to randomly move the affinity to cpus which are not in the allowed set of the node.
I think we should not combine these two cases.
Node-bound devices simply prefer the IRQ to be routed to a cpu 'near' that node, hard-limiting them to that node is policy and is not something we should do.
Defaulting to the node-mask is debatable, but is, I think, something we could do. But I think we should allow user-space to write any mask as long as the hardware can indeed route the IRQ that way, even when clearly stupid.
Which is where the UV case comes in, they cannot route IRQs to every CPU, so it makes sense to limit the possible masks being written. I do however fully agree that that should be done in generic code, as I can quite imagine more hardware than UV having limitations in this regard.
Furthermore, the /sysfs topology information should include IRQ routing data in this case.
Quoted 58 lines · Peter Zijlstra
Quoted 47 lines · Thomas Gleixner
Quoted 8 lines · Dimitri Sivanich
Quoted 7 lines · Eric W. Biederman
Quoted 5 lines · Dimitri Sivanich
This patch allows for hard numa restrictions to irq affinity on x86 systems.
Affinity is masked to allow only those cpus which the subarchitecture deems accessible by the given irq.
Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
On some UV systems, this domain will be limited to the nodes accessible to the irq's node. Initially other X86 systems will not mask off any cpus so non-UV systems will remain unaffected.
Is this a hardware restriction you are trying to model? If not this seems wrong.
Yes, it's a hardware restriction.
Nevertheless I think that this is the wrong approach.
What we really want is a notion in the irq descriptor which tells us:
this interrupt is restricted to numa node N.The solution in this patch is just restricted to x86 and hides that information deep in the arch code.
Further the patch adds code which should be in the generic interrupt management code as it is useful for other purposes as well:
Driver folks are looking for a way to restrict irq balancing to a given numa node when they have all the driver data allocated on that node. That's not a hardware restriction as in the UV case but requires a similar infrastructure.
One possible solution would be to have a new flag:
IRQF_NODE_BOUND- irq is bound todesc->nodeWhen an interrupt is set up we would query with a new
irq_chipfunctionchip->get_node_affinity(irq)which would default to an empty implementation returning -1. The arch code can provide its own function to return the numa affinity which would express the hardware restriction.The core code would restrict affinity settings to the cpumask of that node without any need for the arch code to check it further.
Quoted 7 lines
That same infrastructure could be used for the software restriction of interrupts to a node on which the device is bound.
Having it in the core code also allows us to expose this information to user space so that the irq balancer knows about it and does not try to randomly move the affinity to cpus which are not in the allowed set of the node.
I think we should not combine these two cases.
Quoted 8 lines
Node-bound devices simply prefer the IRQ to be routed to a cpu 'near' that node, hard-limiting them to that node is policy and is not something we should do.
Defaulting to the node-mask is debatable, but is, I think, something we could do. But I think we should allow user-space to write any mask as long as the hardware can indeed route the IRQ that way, even when clearly stupid.
Fair enough, but I can imagine that we want a tunable know which prevents that. I'm not against giving sys admins enough rope to hang themself, but at least we want to give them a helping hand to fight off crappy user space applications which do not care about stupidity at all.
Quoted 4 lines
Which is where the UV case comes in, they cannot route IRQs to every CPU, so it makes sense to limit the possible masks being written. I do however fully agree that that should be done in generic code, as I can quite imagine more hardware than UV having limitations in this regard.
That's why I want to see it in the generic code.
Quoted 2 lines
Furthermore, the /sysfs topology information should include IRQ routing data in this case.
Hmm, not sure about that. You'd need to scan through all the nodes to find the set of CPUs where an irq can be routed to. I prefer to have the information exposed by the irq enumeration (which is currently in /proc/irq though).
Thanks,
tglx
On Tue, 24 Nov 2009 14:55:15 +0100 (CET)
Quoted 7 lines
Quoted 2 lines
Furthermore, the /sysfs topology information should include IRQ routing data in this case.
Hmm, not sure about that. You'd need to scan through all the nodes to find the set of CPUs where an irq can be routed to. I prefer to have the information exposed by the irq enumeration (which is currently in
/proc/irqthough).
yes please.
one device can have multiple irqs
one irq can be servicing multiple devices
expressing that in sysfs is a nightmare, while sticking it in /proc/irq *where the rest of the info is* is much nicer for apps like irqbalance
Signature · 3 lines
Arjan van de Ven Intel Open Source Technology Centre For development, discussion and tips for power savings, visit http://www.lesswatts.org
Quoted 9 lines · Arjan van de Ven
On Tue, 24 Nov 2009 14:55:15 +0100 (CET)
Quoted 7 lines
Quoted 2 lines
Furthermore, the /sysfs topology information should include IRQ routing data in this case.
Hmm, not sure about that. You'd need to scan through all the nodes to find the set of CPUs where an irq can be routed to. I prefer to have the information exposed by the irq enumeration (which is currently in
/proc/irqthough).
Quoted 8 lines
yes please.
one device can have multiple irqs
one irq can be servicing multiple devicesexpressing that in sysfs is a nightmare, while sticking it in
/proc/irq*where the rest of the info is* is much nicer for apps like irqbalance
Oii.
I don't think it is bad to export information to applications like irqbalance.
I think it pretty horrible that one of the standard ways I have heard to improve performance on 10G nics is to kill irqbalance.
Guys. Migrating an irq from one cpu to another while the device is running without dropping interrupts is hard.
At the point we start talking about limiting what a process with CAP_SYS_ADMIN can do because it makes bad decisions I think something is really broken.
Currently the irq code treats /proc/irq/N/smp_affinity as a strong hint on where we would like interrupts to be delivered, and we don't have good feedback from there to architecture specific code that knows what we really can do. It is going to take some effort and some work to make that happen.
I think the irq scheduler is the only scheduler (except for batch jobs) that we don't put in the kernel. It seems to me that if we are going to go to all of the trouble to rewrite the generic code to better support irqbalance because we are having serious irqbalance problems, it will be less effort to suck irqbalance into the kernel along with everything else.
I really think irqbalancing belongs in the kernel. It is hard to export all of the information we need to user space and the information that we need to export keeps changing. Until we master this new trend of exponentially increasing core counts that information is going to keep changing. Today we barely know how to balance flows across cpus. So because of the huge communication problem and the fact that there appears to be no benefit in keeping irqbalance in user space (there is no config file) if we are going to rework all of the interfaces let's pull irqbalance into the kernel.
As for the UV code, what we are looking at is a fundamental irq routing property. Those irqs cannot be routed to some cpus. That is something the code that sets up the routes needs to be aware of. Dimitri could you put your the extra code in assign_irq_vector instead of in the callers of assign_irq_vector? Since the probably is not likely to stay unique we probably want to put the information you base things on in struct irq_desc, but the logic I seems to live best in in assign_irq_vector.
Eric
Quoted 20 lines · Eric W. Biederman
Quoted 9 lines · Arjan van de Ven
On Tue, 24 Nov 2009 14:55:15 +0100 (CET)
Quoted 7 lines
Quoted 2 lines
Furthermore, the /sysfs topology information should include IRQ routing data in this case.
Hmm, not sure about that. You'd need to scan through all the nodes to find the set of CPUs where an irq can be routed to. I prefer to have the information exposed by the irq enumeration (which is currently in
/proc/irqthough).Quoted 8 lines
yes please.
one device can have multiple irqs
one irq can be servicing multiple devicesexpressing that in sysfs is a nightmare, while sticking it in
/proc/irq*where the rest of the info is* is much nicer for apps like irqbalance
Quoted 7 lines
Oii.
I don't think it is bad to export information to applications like irqbalance.
I think it pretty horrible that one of the standard ways I have heard to improve performance on 10G nics is to kill irqbalance.
This is something I'm actively trying to fix (see thread "irq: Add node_affinity CPU masks for smarter irqbalance hints"). That patch may not be the final answer, but whatever comes of it will fix the recommendation of "killall irqbalance" for performance boosts.
Cheers,
-PJ
Quoted 6 lines · Eric W. Biederman
Currently the irq code treats
/proc/irq/N/smp_affinityas a strong hint on where we would like interrupts to be delivered, and we don't have good feedback from there to architecture specific code that knows what we really can do. It is going to take some effort and some work to make that happen.
Quoted 8 lines
I think the irq scheduler is the only scheduler (except for batch jobs) that we don't put in the kernel. It seems to me that if we are going to go to all of the trouble to rewrite the generic code to better support irqbalance because we are having serious irqbalance problems, it will be less effort to suck irqbalance into the kernel along with everything else.
I really think irqbalancing belongs in the kernel. [...]
Interesting. I've yet to see a solution that is maintainable and works well, without putting too much policy into the kernel. Our previous solutions didnt really work all that well.
What would your model be, and can it be implemented reasonably?
Ingo
Quoted 15 lines · Ingo Molnar
Quoted 13 lines · Eric W. Biederman
Currently the irq code treats
/proc/irq/N/smp_affinityas a strong hint on where we would like interrupts to be delivered, and we don't have good feedback from there to architecture specific code that knows what we really can do. It is going to take some effort and some work to make that happen.I think the irq scheduler is the only scheduler (except for batch jobs) that we don't put in the kernel. It seems to me that if we are going to go to all of the trouble to rewrite the generic code to better support irqbalance because we are having serious irqbalance problems, it will be less effort to suck irqbalance into the kernel along with everything else.
Quoted 7 lines
Quoted 1 line
I really think irqbalancing belongs in the kernel. [...]
Interesting. I've yet to see a solution that is maintainable and works well, without putting too much policy into the kernel. Our previous solutions didnt really work all that well.
What would your model be, and can it be implemented reasonably?
we already have dev numa node, so could just make irqblance to some smart to use that device node corresponding for irq that is binding to the device.
YH
Quoted 2 lines · Yinghai Lu
we already have dev numa node, so could just make irqblance to some smart to use that device node corresponding for irq that is binding to the device.
The argument is that a single device can have multiple IRQs and therefore numa nodes.
Quoted 7 lines · Peter Zijlstra
Quoted 2 lines · Yinghai Lu
we already have dev numa node, so could just make irqblance to some smart to use that device node corresponding for irq that is binding to the device.
The argument is that a single device can have multiple IRQs and therefore numa nodes.
one node could have 8 cores or 16 threads, so should be right.
other case the intel new 4 sockets system will have two IOHs, and one for socket0 and socket 1 and one for socket2 and socet3
acpi root bus _PXM only can return one.
so wonder if ACPI spec need to be extended to support two nodes there.
after that, we need to extend numa_node to support that.
YH
Quoted 3 lines · Eric W. Biederman
As for the UV code, what we are looking at is a fundamental irq routing property. Those irqs cannot be routed to some cpus. That is something the code that sets up the routes needs to be aware of.
Correct. We can't allow an interrupt to be routed to an invalid node.
Quoted 5 lines
Dimitri could you put your the extra code in
assign_irq_vectorinstead of in the callers ofassign_irq_vector? Since the probably is not likely to stay unique we probably want to put the information you base things on in structirq_desc, but the logic I seems to live best in inassign_irq_vector.
So you're saying continue to use the node value in irq_desc, or add a cpumask there (which will add some size to that structure)? I'll have to take another look at assign_irq_vector, but as things are currently structured, we don't return any sort of valid cpumask that we'd need for further processing in the caller functions. One would need to pass that back or store that cpumask someplace, like irq_desc?
Quoted 13 lines · Dimitri Sivanich
Quoted 3 lines · Eric W. Biederman
As for the UV code, what we are looking at is a fundamental irq routing property. Those irqs cannot be routed to some cpus. That is something the code that sets up the routes needs to be aware of.
Correct. We can't allow an interrupt to be routed to an invalid node.
Quoted 5 lines
Dimitri could you put your the extra code in
assign_irq_vectorinstead of in the callers ofassign_irq_vector? Since the probably is not likely to stay unique we probably want to put the information you base things on in structirq_desc, but the logic I seems to live best in inassign_irq_vector.
Quoted 7 lines
So you're saying continue to use the node value in
irq_desc, or add a cpumask there (which will add some size to that structure)? I'll have to take another look atassign_irq_vector, but as things are currently structured, we don't return any sort of valid cpumask that we'd need for further processing in the caller functions. One would need to pass that back or store that cpumask someplace, likeirq_desc?
Please do not put anything complex into x86 code at all. Such designs are likely to happen on other architectures and as I said before we want to have
1) the decision function what's valid and not in the generic code
2) a way to expose that information as part of the irq interface to user space.
So what's wrong with a per irq_chip function which returns the cpumask which is valid for irq N ?
That function would be called to check the affinity mask in set_irq_affinity and to dump the mask to /proc/irq/N/possible_cpus or whatever name we agree on.
That way we don't have to worry about where in the x86 code the decision should reside as you simply would always get valid masks from the core code.
That just works and is neither restricted to UV nor to x86.
Thanks,
tglx
Quoted 5 lines · Thomas Gleixner
Please do not put anything complex into x86 code at all. Such designs are likely to happen on other architectures and as I said before we want to have
1) the decision function what's valid and not in the generic code
For the UV problem I don't have an issue. assign_irq_vector
enforces some rules that I don't see being able to expose to user space.
Quoted 2 lines
2) a way to expose that information as part of the irq interface to user space.
-EINVAL?
Quoted 2 lines
So what's wrong with a per
irq_chipfunction which returns the cpumask which is valid for irq N ?
I have no problems with a generic function to do that.
Quoted 7 lines
That function would be called to check the affinity mask in
set_irq_affinityand to dump the mask to/proc/irq/N/possible_cpusor whatever name we agree on.That way we don't have to worry about where in the x86 code the decision should reside as you simply would always get valid masks from the core code.
Impossible. assign_irq_vector is the only function that can tell you if a mask is valid or not. Currently we support roughly 240 irqs
per cpu. Currently we support more than 240 irqs. I don't see
how you can enforce that limit.
Furthermore irq migration on x86 is a very non-trivial exercise. We must wait until we get a new irq at the new location before we cleanup the irq state at the old location, to ensure that the state change has gone through. At which point again we can not know.
So Thomas the logical conclusion that you are requesting. An architecture specific interface for migrating irqs that does not need to return error codes because the generic code has enough information to avoid all problem cases is not going to happen. It is totally unreasonable.
Quoted 1 line
That just works and is neither restricted to UV nor to x86.
Doing it all in the core totally fails as it gets the initial irq assignment wrong.
Last I looked set_irq_affinity was a horribly broken interface. We can not return error codes to user space when they ask us to do the impossible. Right now irq->affinity is a hint that occasionally we ignore when what it requests is impossible.
....
Thomas my apologies for ranting but I am extremely sensitive about people placing demands on the irq code that would be very convenient and simple for the rest of the world, except that the hardware does not work the way people envision it should work. The worst offender is the cpu hotunplug logic that requests we perform the impossible when it comes to irq migration. In the case of UV I expect cpu hotplug is going to request we migrate irqs to another node.
Right now a lot of the generic irq code is living in a deluded fantasy and I really don't want to see more impossible requests from the irq code added to the pile.
...
The architecture specific function setup_irq_vector has all of the information available to it to make the decision. We use it consistently everywhere. For the case of UV it needs to know about another possible hardware limitation, to do it's job. I am happy if that information comes from an architecture agnostic source but making the decision somewhere else is just a guarantee that we will have more subtle breakage that occasionally fail for people but at too low a rate that people will care enough to fix.
Eric
Eric,
Quoted 11 lines · Eric W. Biederman
Quoted 5 lines · Thomas Gleixner
Please do not put anything complex into x86 code at all. Such designs are likely to happen on other architectures and as I said before we want to have
1) the decision function what's valid and not in the generic code
For the UV problem I don't have an issue.
assign_irq_vector
enforces some rules that I don't see being able to expose to user space.
How is that related to user space ? All I'm asking for is to move the decision to the generic code and not add more stuff to x86/*.
This UV problem is currently unique, but it's going to pop up all over the place.
Quoted 4 lines
Quoted 2 lines
2) a way to expose that information as part of the irq interface to user space.
-EINVAL?
Err ? What's your problem with telling user space that an irq is restricted to a certain set of CPUs ?
Quoted 9 lines
Quoted 2 lines
So what's wrong with a per
irq_chipfunction which returns the cpumask which is valid for irq N ?I have no problems with a generic function to do that.
Quoted 4 lines
That function would be called to check the affinity mask in
set_irq_affinityand to dump the mask to/proc/irq/N/possible_cpusor whatever name we agree on.
Quoted 7 lines
Quoted 3 lines
That way we don't have to worry about where in the x86 code the decision should reside as you simply would always get valid masks from the core code.
Impossible.
assign_irq_vectoris the only function that can tell you if a mask is valid or not. Currently we support roughly 240 irqs per cpu. Currently we support more than 240 irqs.
Interesting.
Quoted 1 line
I don't see how you can enforce that limit.
-ENOPARSE. Please excuse my limited ability to decode crypt_eric
Quoted 1 line
Furthermore irq migration on x86 is a very non-trivial exercise.
Thanks for the education, that's really new information to me.
Quoted 4 lines
We must wait until we get a new irq at the new location before we cleanup the irq state at the old location, to ensure that the state change has gone through. At which point again we can not know.
Know what?
Quoted 5 lines
So Thomas the logical conclusion that you are requesting. An architecture specific interface for migrating irqs that does not need to return error codes because the generic code has enough information to avoid all problem cases is not going to happen. It is totally unreasonable.
I did never request that at all. All I'm asking for is that we think more about having decision functions which are valuable for more than one esoteric use case like UV moved into generic code instead of hacking them into some architecture space.
Quoted 4 lines
Quoted 1 line
That just works and is neither restricted to UV nor to x86.
Doing it all in the core totally fails as it gets the initial irq assignment wrong.
That's fixable. And I did not say that these decision functions can only be used by the generic code, they can be used to do final or initial decisions in the low level code as well. I'm just fighting that you folks think that your problems are so unique that they need to be solved in special ways.
Quoted 4 lines
Last I looked
set_irq_affinitywas a horribly broken interface. We can not return error codes to user space when they ask us to do the impossible. Right nowirq->affinityis a hint that occasionally we ignore when what it requests is impossible.
And that is a perfect excuse to refuse to think about improvements or a reasonable replacement for that interface ?
Quoted 7 lines
Thomas my apologies for ranting but I am extremely sensitive about people placing demands on the irq code that would be very convenient and simple for the rest of the world, except that the hardware does not work the way people envision it should work. The worst offender is the cpu hotunplug logic that requests we perform the impossible when it comes to irq migration. In the case of UV I expect cpu hotplug is going to request we migrate irqs to another node.
Sure, and the proper way to fix that is the arch/x86 code, right ?
Quoted 3 lines
Right now a lot of the generic irq code is living in a deluded fantasy and I really don't want to see more impossible requests from the irq code added to the pile.
Sorry dude. _You_ are living in the delusion that your problems are unique and can only be solved by adding special crafted hackery to everything. You simply refuse to go the hard way of proving that it is impossible to solve for the benefit of everyone just on the base of handwaving and claiming that you are the only expert who understands all the details of interrupt handling.
Quoted 8 lines
The architecture specific function
setup_irq_vectorhas all of the information available to it to make the decision. We use it consistently everywhere. For the case of UV it needs to know about another possible hardware limitation, to do it's job. I am happy if that information comes from an architecture agnostic source but making the decision somewhere else is just a guarantee that we will have more subtle breakage that occasionally fail for people but at too low a rate that people will care enough to fix.
You are completely ignoring the fact, that the deluded people who started to work on a generic irq subsystem actually had and still have very good reasons to do so and do have a not so deluded insight into the general and also the x86 specific problem space.
I'm well aware that every generic interface has short comings, but we are in the kernel space where we can fix things if we _want_ and work together on doing so.
It's just impossible to do that with people who insist on their particular world view and I'm not talking about you alone. The same applies to the embedded crowd and driver folks who permanentely insist on implementing "workarounds" instead of talking to the people who are responsible and willing to fix things.
Thanks,
tglx
Quoted 13 lines · Dimitri Sivanich
Quoted 3 lines · Eric W. Biederman
As for the UV code, what we are looking at is a fundamental irq routing property. Those irqs cannot be routed to some cpus. That is something the code that sets up the routes needs to be aware of.
Correct. We can't allow an interrupt to be routed to an invalid node.
Quoted 5 lines
Dimitri could you put your the extra code in
assign_irq_vectorinstead of in the callers ofassign_irq_vector? Since the probably is not likely to stay unique we probably want to put the information you base things on in structirq_desc, but the logic I seems to live best in inassign_irq_vector.
Quoted 7 lines
So you're saying continue to use the node value in
irq_desc, or add a cpumask there (which will add some size to that structure)? I'll have to take another look atassign_irq_vector, but as things are currently structured, we don't return any sort of valid cpumask that we'd need for further processing in the caller functions. One would need to pass that back or store that cpumask someplace, likeirq_desc?
We have cfg->domain which is the set of cpus the irq was setup to work on. We don't need an extra return value.
All that is needed is an extra line or two up at the very top. Something like.
cpumask_and(requested_mask, mask, irq_possible_cpus_mask);
And then use requested_mask where we use cpumask_and today.
Simple and it doesn't require extra code all over the place.
Eric
Quoted 7 lines · On Tue, 24 Nov 2009 09:41:18 -0800 [email protected] (Eric W. Biederman)…
Oii.
I don't think it is bad to export information to applications like irqbalance.
I think it pretty horrible that one of the standard ways I have heard to improve performance on 10G nics is to kill irqbalance.
irqbalance does not move networking irqs; if it does there's something evil going on in the system. But thanks for the bugreport ;)
Quoted 2 lines
Guys. Migrating an irq from one cpu to another while the device is running without dropping interrupts is hard.
no it isn't; we've been doing it for years.
Quoted 6 lines
I think the irq scheduler is the only scheduler (except for batch jobs) that we don't put in the kernel. It seems to me that if we are going to go to all of the trouble to rewrite the generic code to better support irqbalance because we are having serious irqbalance problems, it will be less effort to suck irqbalance into the kernel along with everything else.
we had that; it didn't work.
what I'm asking for is for the kernel to expose the numa information; right now that is the piece that is missing.
Signature · 3 lines
Arjan van de Ven Intel Open Source Technology Centre For development, discussion and tips for power savings, visit http://www.lesswatts.org
Quoted 4 lines · Arjan van de Ven
Quoted 2 lines · On Tue, 24 Nov 2009 09:41:18 -0800 [email protected] (Eric W. Biederman)…
Oii.
Quoted 8 lines
Quoted 5 lines
I don't think it is bad to export information to applications like irqbalance.
I think it pretty horrible that one of the standard ways I have heard to improve performance on 10G nics is to kill irqbalance.
irqbalance does not move networking irqs; if it does there's something evil going on in the system. But thanks for the bugreport ;)
It does move networking irqs.
Quoted 5 lines
we had that; it didn't work.
what I'm asking for is for the kernel to expose the numa information; right now that is the piece that is missing.
I'm wondering if we should expose that numa information in the form of a node or the set of allowed cpus, or both?
I'm guessing 'both' is the correct answer, so that apps like irqbalance can make a qualitative decision based on the node (affinity to cpus on this node is better), but an absolute decision based on allowed cpus (I cannot change affinity to anything but this set of cpus).
Quoted 17 lines · Dimitri Sivanich
Quoted 4 lines · Arjan van de Ven
Quoted 2 lines · On Tue, 24 Nov 2009 09:41:18 -0800 [email protected] (Eric W. Biederman)…
Oii.
Quoted 8 lines
Quoted 5 lines
I don't think it is bad to export information to applications like irqbalance.
I think it pretty horrible that one of the standard ways I have heard to improve performance on 10G nics is to kill irqbalance.
irqbalance does not move networking irqs; if it does there's something evil going on in the system. But thanks for the bugreport ;)
It does move networking irqs.
Quoted 1 line
Quoted 8 lines
Quoted 4 lines
we had that; it didn't work.
what I'm asking for is for the kernel to expose the numa information; right now that is the piece that is missing.I'm wondering if we should expose that numa information in the form of a node or the set of allowed cpus, or both?
I'm guessing 'both' is the correct answer, so that apps like irqbalance can make a qualitative decision based on the node (affinity to cpus on this node is better), but an absolute decision based on allowed cpus (I cannot change affinity to anything but this set of cpus).
That's exactly what my patch in the thread "irq: Add node_affinity CPU masks for smarter irqbalance hints" is doing. I've also done the irqbalance changes based on that kernel patch, and Arjan currently has that patch.
Cheers,
-PJ
Quoted 24 lines · Peter P
Quoted 22 lines · Dimitri Sivanich
Quoted 4 lines · Arjan van de Ven
Quoted 2 lines · On Tue, 24 Nov 2009 09:41:18 -0800 [email protected] (Eric W. Biederman)…
Oii.
Quoted 8 lines
Quoted 5 lines
I don't think it is bad to export information to applications like irqbalance.
I think it pretty horrible that one of the standard ways I have heard to improve performance on 10G nics is to kill irqbalance.
irqbalance does not move networking irqs; if it does there's something evil going on in the system. But thanks for the bugreport ;)
It does move networking irqs.
Quoted 5 lines
we had that; it didn't work.
what I'm asking for is for the kernel to expose the numa information; right now that is the piece that is missing.
Quoted 8 lines
Quoted 3 lines
I'm wondering if we should expose that numa information in the form of a node or the set of allowed cpus, or both?
I'm guessing 'both' is the correct answer, so that apps like irqbalance can make a qualitative decision based on the node (affinity to cpus on this node is better), but an absolute decision based on allowed cpus (I cannot change affinity to anything but this set of cpus).
That's exactly what my patch in the thread "irq: Add
node_affinityCPU masks for smarter irqbalance hints" is doing. I've also done the irqbalance changes based on that kernel patch, and Arjan currently has that patch.
So if I understand correctly, you're patch takes care of the qualitative portion of it (we prefer to set affinity to these cpus, which may be on more than one node), but not the restrictive part of it (we cannot change affinity to anything but these cpus)?
Quoted 27 lines · Dimitri Sivanich
Quoted 26 lines · Peter P
Quoted 24 lines · Dimitri Sivanich
Quoted 4 lines · Arjan van de Ven
Quoted 2 lines · On Tue, 24 Nov 2009 09:41:18 -0800 [email protected] (Eric W. Biederman)…
Oii.
Quoted 8 lines
Quoted 5 lines
I don't think it is bad to export information to applications like irqbalance.
I think it pretty horrible that one of the standard ways I have heard to improve performance on 10G nics is to kill irqbalance.
irqbalance does not move networking irqs; if it does there's something evil going on in the system. But thanks for the bugreport ;)
It does move networking irqs.
Quoted 5 lines
we had that; it didn't work.
what I'm asking for is for the kernel to expose the numa information; right now that is the piece that is missing.I'm wondering if we should expose that numa information in the form of a node or the set of allowed cpus, or both?
Quoted 8 lines
Quoted 6 lines
Quoted 1 line
I'm guessing 'both' is the correct answer, so that apps like irqbalance can make a qualitative decision based on the node (affinity to cpus on this node is better), but an absolute decision based on allowed cpus (I cannot change affinity to anything but this set of cpus).
That's exactly what my patch in the thread "irq: Add
node_affinityCPU masks for smarter irqbalance hints" is doing. I've also done the irqbalance changes based on that kernel patch, and Arjan currently has that patch.So if I understand correctly, you're patch takes care of the qualitative portion of it (we prefer to set affinity to these cpus, which may be on more than one node), but not the restrictive part of it (we cannot change affinity to anything but these cpus)?
That is correct. The patch provides an interface to both the kernel (functions) and /proc for userspace to set a CPU mask. That is the preferred mask for the interrupt to be balanced on. Then irqbalance will make decisions on how to balance within that provided mask, if it in fact has been provided.
Cheers,
-PJ
Quoted 36 lines · Peter P
Quoted 34 lines · Dimitri Sivanich
Quoted 32 lines · Peter P
Quoted 17 lines · Dimitri Sivanich
Quoted 4 lines · Arjan van de Ven
Quoted 2 lines · On Tue, 24 Nov 2009 09:41:18 -0800 [email protected] (Eric W. Biederman)…
Oii.
Quoted 8 lines
Quoted 5 lines
I don't think it is bad to export information to applications like irqbalance.
I think it pretty horrible that one of the standard ways I have heard to improve performance on 10G nics is to kill irqbalance.
irqbalance does not move networking irqs; if it does there's something evil going on in the system. But thanks for the bugreport ;)
It does move networking irqs.
Quoted 1 line
Quoted 8 lines
Quoted 4 lines
we had that; it didn't work.
what I'm asking for is for the kernel to expose the numa information; right now that is the piece that is missing.I'm wondering if we should expose that numa information in the form of a node or the set of allowed cpus, or both?
I'm guessing 'both' is the correct answer, so that apps like irqbalance can make a qualitative decision based on the node (affinity to cpus on this node is better), but an absolute decision based on allowed cpus (I cannot change affinity to anything but this set of cpus).
That's exactly what my patch in the thread "irq: Add
node_affinityCPU masks for smarter irqbalance hints" is doing. I've also done the irqbalance changes based on that kernel patch, and Arjan currently has that patch.
Quoted 7 lines
Quoted 1 line
So if I understand correctly, you're patch takes care of the qualitative portion of it (we prefer to set affinity to these cpus, which may be on more than one node), but not the restrictive part of it (we cannot change affinity to anything but these cpus)?
That is correct. The patch provides an interface to both the kernel (functions) and /proc for userspace to set a CPU mask. That is the preferred mask for the interrupt to be balanced on. Then irqbalance will make decisions on how to balance within that provided mask, if it in fact has been provided.
What if it's not provided? Will irqbalance make decisions based on the numa_node of that irq (I would hope)?
Also, can we add a restricted mask as I mention above into this scheme? If we can't send an IRQ to some node, we don't want to bother attempting to change affinity to cpus on that node (hopefully code in the kernel will eventually restrict this).
As a matter of fact, driver's allocating rings, buffers, queues on other nodes should optimally be made aware of the restriction.
Quoted 39 lines · Dimitri Sivanich
Quoted 38 lines · Peter P
Quoted 35 lines · Dimitri Sivanich
Quoted 24 lines · Peter P
Quoted 22 lines · Dimitri Sivanich
Quoted 4 lines · Arjan van de Ven
Quoted 2 lines · On Tue, 24 Nov 2009 09:41:18 -0800 [email protected] (Eric W. Biederman)…
Oii.
Quoted 8 lines
Quoted 5 lines
I don't think it is bad to export information to applications like irqbalance.
I think it pretty horrible that one of the standard ways I have heard to improve performance on 10G nics is to kill irqbalance.
irqbalance does not move networking irqs; if it does there's something evil going on in the system. But thanks for the bugreport ;)
It does move networking irqs.
Quoted 5 lines
we had that; it didn't work.
what I'm asking for is for the kernel to expose the numa information; right now that is the piece that is missing.Quoted 8 lines
Quoted 3 lines
I'm wondering if we should expose that numa information in the form of a node or the set of allowed cpus, or both?
I'm guessing 'both' is the correct answer, so that apps like irqbalance can make a qualitative decision based on the node (affinity to cpus on this node is better), but an absolute decision based on allowed cpus (I cannot change affinity to anything but this set of cpus).
That's exactly what my patch in the thread "irq: Add
node_affinityCPU masks for smarter irqbalance hints" is doing. I've also done the irqbalance changes based on that kernel patch, and Arjan currently has that patch.So if I understand correctly, you're patch takes care of the qualitative portion of it (we prefer to set affinity to these cpus, which may be on more than one node), but not the restrictive part of it (we cannot change affinity to anything but these cpus)?
Quoted 7 lines
Quoted 5 lines
That is correct. The patch provides an interface to both the kernel (functions) and /proc for userspace to set a CPU mask. That is the preferred mask for the interrupt to be balanced on. Then irqbalance will make decisions on how to balance within that provided mask, if it in fact has been provided.
What if it's not provided? Will irqbalance make decisions based on the
numa_nodeof that irq (I would hope)?
If it's not provided, then irqbalance will continue to do what it does today. No changes.
Quoted 3 lines
Also, can we add a restricted mask as I mention above into this scheme? If we can't send an IRQ to some node, we don't want to bother attempting to change affinity to cpus on that node (hopefully code in the kernel will eventually restrict this).
The interface allows you to put in any CPU mask. The way it's written now, whatever mask you put in, irqbalance *only* balances within that mask. It won't ever try and go outside that mask.
Quoted 1 line
As a matter of fact, driver's allocating rings, buffers, queues on other nodes should optimally be made aware of the restriction.
The idea is that the driver will do its memory allocations for everything across nodes. When it does that, it will use the kernel interface (function call) to set the corresponding mask it wants for those queue resources. That is my end-goal for this code.
Cheers,
-PJ
Quoted 47 lines · Peter P
Quoted 45 lines · Dimitri Sivanich
Quoted 43 lines · Peter P
Quoted 27 lines · Dimitri Sivanich
Quoted 26 lines · Peter P
Quoted 24 lines · Dimitri Sivanich
Quoted 4 lines · Arjan van de Ven
Quoted 2 lines · On Tue, 24 Nov 2009 09:41:18 -0800 [email protected] (Eric W. Biederman)…
Oii.
Quoted 8 lines
Quoted 5 lines
I don't think it is bad to export information to applications like irqbalance.
I think it pretty horrible that one of the standard ways I have heard to improve performance on 10G nics is to kill irqbalance.
irqbalance does not move networking irqs; if it does there's something evil going on in the system. But thanks for the bugreport ;)
It does move networking irqs.
Quoted 5 lines
we had that; it didn't work.
what I'm asking for is for the kernel to expose the numa information; right now that is the piece that is missing.I'm wondering if we should expose that numa information in the form of a node or the set of allowed cpus, or both?
Quoted 8 lines
Quoted 6 lines
Quoted 1 line
I'm guessing 'both' is the correct answer, so that apps like irqbalance can make a qualitative decision based on the node (affinity to cpus on this node is better), but an absolute decision based on allowed cpus (I cannot change affinity to anything but this set of cpus).
That's exactly what my patch in the thread "irq: Add
node_affinityCPU masks for smarter irqbalance hints" is doing. I've also done the irqbalance changes based on that kernel patch, and Arjan currently has that patch.So if I understand correctly, you're patch takes care of the qualitative portion of it (we prefer to set affinity to these cpus, which may be on more than one node), but not the restrictive part of it (we cannot change affinity to anything but these cpus)?
That is correct. The patch provides an interface to both the kernel (functions) and /proc for userspace to set a CPU mask. That is the preferred mask for the interrupt to be balanced on. Then irqbalance will make decisions on how to balance within that provided mask, if it in fact has been provided.
Quoted 4 lines
Quoted 1 line
What if it's not provided? Will irqbalance make decisions based on the
numa_nodeof that irq (I would hope)?If it's not provided, then irqbalance will continue to do what it does today. No changes.
So no numa awareness for ethernet irqs.
I'm wondering what needs to be exposed in proc/irq at this point.
Do we expose separate cpu masks for everything? There are really 3 possible pieces of affinity information: your node_affinity (optionally selected by the driver according to allocations), my restricted_affinity (set by the specific arch), and numa_node affinity (the 'home' node of the device). Do we show cpumasks for all of these, or maybe show numa_node in place of a cpumask for the 'home' node of the device?
With all of that information apps like irqbalance should be able to make some good decisions, but things can get confusing if there are too many masks.
Also, if I manually change affinity, irqbalance can change that affinity from under me, correct? That's fine as long as it's stated that that's how things will work (turn off irqbalance or run oneshot mode for manual setting).
Quoted 8 lines
Quoted 4 lines
Also, can we add a restricted mask as I mention above into this scheme? If we can't send an IRQ to some node, we don't want to bother attempting to change affinity to cpus on that node (hopefully code in the kernel will eventually restrict this).
The interface allows you to put in any CPU mask. The way it's written now, whatever mask you put in, irqbalance *only* balances within that mask. It won't ever try and go outside that mask.
OK. Given that, it might be nice to combine the restricted cpus that I'm describing with your node_affinity mask, but we could expose them as separate masks (node_affinity and restricted_affinity, as I describe above).
Quoted 8 lines
Quoted 2 lines
As a matter of fact, driver's allocating rings, buffers, queues on other nodes should optimally be made aware of the restriction.
The idea is that the driver will do its memory allocations for everything across nodes. When it does that, it will use the kernel interface (function call) to set the corresponding mask it wants for those queue resources. That is my end-goal for this code.
OK, but we will eventually have to reject any irqbalance attempts to send irqs to restricted nodes.
Quoted 48 lines · Dimitri Sivanich
Quoted 47 lines · Peter P
Quoted 45 lines · Dimitri Sivanich
Quoted 43 lines · Peter P
Quoted 27 lines · Dimitri Sivanich
Quoted 26 lines · Peter P
Quoted 24 lines · Dimitri Sivanich
Quoted 4 lines · Arjan van de Ven
Quoted 2 lines · On Tue, 24 Nov 2009 09:41:18 -0800 [email protected] (Eric W. Biederman)…
Oii.
Quoted 8 lines
Quoted 5 lines
I don't think it is bad to export information to applications like irqbalance.
I think it pretty horrible that one of the standard ways I have heard to improve performance on 10G nics is to kill irqbalance.
irqbalance does not move networking irqs; if it does there's something evil going on in the system. But thanks for the bugreport ;)
It does move networking irqs.
Quoted 5 lines
we had that; it didn't work.
what I'm asking for is for the kernel to expose the numa information; right now that is the piece that is missing.I'm wondering if we should expose that numa information in the form of a node or the set of allowed cpus, or both?
Quoted 8 lines
Quoted 6 lines
Quoted 1 line
I'm guessing 'both' is the correct answer, so that apps like irqbalance can make a qualitative decision based on the node (affinity to cpus on this node is better), but an absolute decision based on allowed cpus (I cannot change affinity to anything but this set of cpus).
That's exactly what my patch in the thread "irq: Add
node_affinityCPU masks for smarter irqbalance hints" is doing. I've also done the irqbalance changes based on that kernel patch, and Arjan currently has that patch.So if I understand correctly, you're patch takes care of the qualitative portion of it (we prefer to set affinity to these cpus, which may be on more than one node), but not the restrictive part of it (we cannot change affinity to anything but these cpus)?
That is correct. The patch provides an interface to both the kernel (functions) and /proc for userspace to set a CPU mask. That is the preferred mask for the interrupt to be balanced on. Then irqbalance will make decisions on how to balance within that provided mask, if it in fact has been provided.
Quoted 7 lines
Quoted 4 lines
Quoted 1 line
What if it's not provided? Will irqbalance make decisions based on the
numa_nodeof that irq (I would hope)?If it's not provided, then irqbalance will continue to do what it does today. No changes.
So no numa awareness for ethernet irqs.
Nothing that makes sense, no.
Quoted 8 lines
I'm wondering what needs to be exposed in proc/irq at this point.
Do we expose separate cpu masks for everything? There are really 3 possible pieces of affinity information: your
node_affinity(optionally selected by the driver according to allocations), myrestricted_affinity(set by the specific arch), andnuma_nodeaffinity (the 'home' node of the device). Do we show cpumasks for all of these, or maybe shownuma_nodein place of a cpumask for the 'home' node of the device?With all of that information apps like irqbalance should be able to make some good decisions, but things can get confusing if there are too many masks.
Also, if I manually change affinity, irqbalance can change that affinity from under me, correct? That's fine as long as it's stated that that's how things will work (turn off irqbalance or run oneshot mode for manual setting).
If you change /proc/irq/<num>/smp_affinity, yes, irqbalance can change that after the fact.
Quoted 11 lines
Quoted 8 lines
Quoted 4 lines
Also, can we add a restricted mask as I mention above into this scheme? If we can't send an IRQ to some node, we don't want to bother attempting to change affinity to cpus on that node (hopefully code in the kernel will eventually restrict this).
The interface allows you to put in any CPU mask. The way it's written now, whatever mask you put in, irqbalance *only* balances within that mask. It won't ever try and go outside that mask.
OK. Given that, it might be nice to combine the restricted cpus that I'm describing with your
node_affinitymask, but we could expose them as separate masks (node_affinityandrestricted_affinity, as I describe above).
I think this might be getting too complicated. The only thing irqbalance is lacking today, in my mind, is the feedback mechanism, telling it what subset of CPU masks to balance within. There is a allowed_mask, but that is used for a different purpose. Hence why I added another. But I think your needs can be met 100% with what I have already, and we can come up with a different name that's more generic. The flows would be something like this:
Driver:
- Driver comes online, allocates memory in a sensible NUMA fashion
- Driver requests kernel for interrupts, ties them into handlers
- Driver now sets a NUMA-friendly affinity for each interrupt, to match with its initial memory allocation
- irqbalance balances interrupts within their new "hinted" affinities.
Other:
- System comes online
- In your case, interrupts must be kept away from certain CPUs.
- Some mechanism in your architecture init can set the "hinted" affinity mask for each interrupt.
- irqbalance will not move interrupts to the CPUs you left out of the "hinted" affinity.
Does this make more sense?
Quoted 10 lines
Quoted 8 lines
Quoted 2 lines
As a matter of fact, driver's allocating rings, buffers, queues on other nodes should optimally be made aware of the restriction.
The idea is that the driver will do its memory allocations for everything across nodes. When it does that, it will use the kernel interface (function call) to set the corresponding mask it wants for those queue resources. That is my end-goal for this code.
OK, but we will eventually have to reject any irqbalance attempts to send irqs to restricted nodes.
See above.
Cheers,
-PJ
Quoted 10 lines · Peter P Waskiewicz Jr
Quoted 10 lines
Quoted 9 lines
Quoted 5 lines
Also, can we add a restricted mask as I mention above into this scheme? If we can't send an IRQ to some node, we don't want to bother attempting to change affinity to cpus on that node (hopefully code in the kernel will eventually restrict this).
The interface allows you to put in any CPU mask. The way it's written now, whatever mask you put in, irqbalance *only* balances within that mask. It won't ever try and go outside that mask.
Quoted 6 lines
Quoted 2 lines
OK. Given that, it might be nice to combine the restricted cpus that I'm describing with your
node_affinitymask, but we could expose them as separate masks (node_affinityandrestricted_affinity, as I describe above).I think this might be getting too complicated. The only thing irqbalance is lacking today, in my mind, is the feedback mechanism, telling it what subset of CPU masks to balance within.
You mean besides knowing that devices can have more than one irq? You mean besides making good on it's promise not to move networking irqs? A policy of BALANCE_CORE sure doesn't look like a policy of don't touch.
You mean besides realizing that irqs can only be directed at one cpu on x86? At least when you have more than 8 logical cores in the system, the cases that matter.
Quoted 5 lines
There is a
allowed_mask, but that is used for a different purpose. Hence why I added another. But I think your needs can be met 100% with what I have already, and we can come up with a different name that's more generic. The flows would be something like this:
Two masks? You are asking the kernel to move irqs for you then?
Quoted 26 lines
Driver:
- Driver comes online, allocates memory in a sensible NUMA fashion
- Driver requests kernel for interrupts, ties them into handlers
- Driver now sets a NUMA-friendly affinity for each interrupt, to match with its initial memory allocation
- irqbalance balances interrupts within their new "hinted" affinities.Other:
- System comes online
- In your case, interrupts must be kept away from certain CPUs.
- Some mechanism in your architecture init can set the "hinted" affinity mask for each interrupt.
- irqbalance will not move interrupts to the CPUs you left out of the "hinted" affinity.Does this make more sense?
Quoted 8 lines
Quoted 7 lines
Quoted 1 line
As a matter of fact, driver's allocating rings, buffers, queues on other nodes should optimally be made aware of the restriction.
The idea is that the driver will do its memory allocations for everything across nodes. When it does that, it will use the kernel interface (function call) to set the corresponding mask it wants for those queue resources. That is my end-goal for this code.
Quoted 3 lines
Quoted 1 line
OK, but we will eventually have to reject any irqbalance attempts to send irqs to restricted nodes.
See above.
Either I am parsing this conversation wrong or there is a strong reality distortion field in place. It appears you are asking that we depend on a user space application to not attempt the physically impossible, when we could just as easily ignore or report -EINVAL to.
We really have two separate problems hear.
- How to avoid the impossible.
- How to deal with NUMA affinity.
Eric
Quoted 12 lines · Eric W. Biederman
Quoted 10 lines · Peter P Waskiewicz Jr
Quoted 10 lines
Quoted 9 lines
Quoted 5 lines
Also, can we add a restricted mask as I mention above into this scheme? If we can't send an IRQ to some node, we don't want to bother attempting to change affinity to cpus on that node (hopefully code in the kernel will eventually restrict this).
The interface allows you to put in any CPU mask. The way it's written now, whatever mask you put in, irqbalance *only* balances within that mask. It won't ever try and go outside that mask.
Quoted 8 lines
Quoted 6 lines
Quoted 2 lines
OK. Given that, it might be nice to combine the restricted cpus that I'm describing with your
node_affinitymask, but we could expose them as separate masks (node_affinityandrestricted_affinity, as I describe above).I think this might be getting too complicated. The only thing irqbalance is lacking today, in my mind, is the feedback mechanism, telling it what subset of CPU masks to balance within.
You mean besides knowing that devices can have more than one irq?
Why does it matter if it does or doesn't? The interrupts have to go somewhere.
Quoted 3 lines
You mean besides making good on it's promise not to move networking irqs? A policy of
BALANCE_COREsure doesn't look like a policy of don't touch.
Not moving network irqs is something Arjan said could be a bug, and he'd be happy to either look into it, or welcome a patch if it really is
broken. As for BALANCE_CORE, I have no idea what you're talking about.
Quoted 4 lines
You mean besides realizing that irqs can only be directed at one cpu on x86? At least when you have more than 8 logical cores in the system, the cases that matter.
Huh? I can have all of my interrupts directed to a single CPU on x86. Can you give me an example here?
Quoted 7 lines
Quoted 5 lines
There is a
allowed_mask, but that is used for a different purpose. Hence why I added another. But I think your needs can be met 100% with what I have already, and we can come up with a different name that's more generic. The flows would be something like this:Two masks? You are asking the kernel to move irqs for you then?
Absolutely not. Were you not following this thread earlier when this was being discussed with Thomas?
Quoted 26 lines
Quoted 26 lines
Driver:
- Driver comes online, allocates memory in a sensible NUMA fashion
- Driver requests kernel for interrupts, ties them into handlers
- Driver now sets a NUMA-friendly affinity for each interrupt, to match with its initial memory allocation
- irqbalance balances interrupts within their new "hinted" affinities.Other:
- System comes online
- In your case, interrupts must be kept away from certain CPUs.
- Some mechanism in your architecture init can set the "hinted" affinity mask for each interrupt.
- irqbalance will not move interrupts to the CPUs you left out of the "hinted" affinity.Does this make more sense?
Quoted 8 lines
Quoted 7 lines
Quoted 1 line
As a matter of fact, driver's allocating rings, buffers, queues on other nodes should optimally be made aware of the restriction.
The idea is that the driver will do its memory allocations for everything across nodes. When it does that, it will use the kernel interface (function call) to set the corresponding mask it wants for those queue resources. That is my end-goal for this code.
Quoted 9 lines
Quoted 3 lines
Quoted 1 line
OK, but we will eventually have to reject any irqbalance attempts to send irqs to restricted nodes.
See above.
Either I am parsing this conversation wrong or there is a strong reality distortion field in place. It appears you are asking that we depend on a user space application to not attempt the physically impossible, when we could just as easily ignore or report
-EINVALto.
You are parsing this conversation incorrectly. I also don't understand why you always have a very negative view of how impossible everything is. Do you think we get no work done in the kernel? We deal with countless issues across the kernel that are hard. Being hard doesn't mean they're impossible, it just means we may have to try something new and unknown.
What I'm asking is we make some mechanism for drivers to manage their interrupt affinities. Today drivers have no influence or control where their interrupts land. This is a limitation, plain and simple. We need a mechanism to allow a driver to say "hey, this interrupt needs to run only on these CPUs. Going elsewhere can severely impact performance of your network." Whatever listens and acts on that mechanism is irrelevant.
Quoted 2 lines
We really have two separate problems hear.
- How to avoid the impossible.
Really man, this type of view is neither helpful or useful. Either help people solve problems, or keep your negative views on proposed solutions to problems to yourself.
Quoted 1 line
- How to deal with NUMA affinity.
More generally, how to deal with a device's preferred affinity. That is the real issue I'm trying to solve.
Cheers,
-PJ
Quoted 14 lines · Eric W. Biederman
Quoted 11 lines · Peter P Waskiewicz Jr
Quoted 9 lines
Quoted 7 lines
Quoted 1 line
As a matter of fact, driver's allocating rings, buffers, queues on other nodes should optimally be made aware of the restriction.
The idea is that the driver will do its memory allocations for everything across nodes. When it does that, it will use the kernel interface (function call) to set the corresponding mask it wants for those queue resources. That is my end-goal for this code.
OK, but we will eventually have to reject any irqbalance attempts to send irqs to restricted nodes.
See above.
Quoted 7 lines
Either I am parsing this conversation wrong or there is a strong reality distortion field in place. It appears you are asking that we depend on a user space application to not attempt the physically impossible, when we could just as easily ignore or report
-EINVALto.We really have two separate problems hear.
- How to avoid the impossible.
The kernel does need to restrict attempts at the impossible. However I see nothing wrong with providing apps with information as to what the impossible actually is.
Quoted 1 line
- How to deal with NUMA affinity.
Quoted 51 lines · Peter P Waskiewicz Jr
Quoted 50 lines · Dimitri Sivanich
Quoted 49 lines · Peter P
Quoted 46 lines · Dimitri Sivanich
Quoted 36 lines · Peter P
Quoted 34 lines · Dimitri Sivanich
Quoted 32 lines · Peter P
Quoted 17 lines · Dimitri Sivanich
Quoted 4 lines · Arjan van de Ven
Quoted 2 lines · On Tue, 24 Nov 2009 09:41:18 -0800 [email protected] (Eric W. Biederman)…
Oii.
Quoted 8 lines
Quoted 5 lines
I don't think it is bad to export information to applications like irqbalance.
I think it pretty horrible that one of the standard ways I have heard to improve performance on 10G nics is to kill irqbalance.
irqbalance does not move networking irqs; if it does there's something evil going on in the system. But thanks for the bugreport ;)
It does move networking irqs.
Quoted 1 line
Quoted 8 lines
Quoted 4 lines
we had that; it didn't work.
what I'm asking for is for the kernel to expose the numa information; right now that is the piece that is missing.I'm wondering if we should expose that numa information in the form of a node or the set of allowed cpus, or both?
I'm guessing 'both' is the correct answer, so that apps like irqbalance can make a qualitative decision based on the node (affinity to cpus on this node is better), but an absolute decision based on allowed cpus (I cannot change affinity to anything but this set of cpus).
That's exactly what my patch in the thread "irq: Add
node_affinityCPU masks for smarter irqbalance hints" is doing. I've also done the irqbalance changes based on that kernel patch, and Arjan currently has that patch.Quoted 7 lines
Quoted 1 line
So if I understand correctly, you're patch takes care of the qualitative portion of it (we prefer to set affinity to these cpus, which may be on more than one node), but not the restrictive part of it (we cannot change affinity to anything but these cpus)?
That is correct. The patch provides an interface to both the kernel (functions) and /proc for userspace to set a CPU mask. That is the preferred mask for the interrupt to be balanced on. Then irqbalance will make decisions on how to balance within that provided mask, if it in fact has been provided.
What if it's not provided? Will irqbalance make decisions based on the
numa_nodeof that irq (I would hope)?
Quoted 7 lines
Quoted 5 lines
Quoted 2 lines
If it's not provided, then irqbalance will continue to do what it does today. No changes.
So no numa awareness for ethernet irqs.
Nothing that makes sense, no.
OK. See my note below concerning exposing the irq_desc 'node'.
Quoted 5 lines
Quoted 5 lines
I'm wondering what needs to be exposed in proc/irq at this point.
Do we expose separate cpu masks for everything? There are really 3 possible pieces of affinity information: your
node_affinity(optionally selected by the driver according to allocations), myrestricted_affinity(set by the specific arch), andnuma_nodeaffinity (the 'home' node of the device). Do we show cpumasks for all of these, or maybe shownuma_nodein place of a cpumask for the 'home' node of the device?
Quoted 7 lines
Quoted 4 lines
With all of that information apps like irqbalance should be able to make some good decisions, but things can get confusing if there are too many masks.
Also, if I manually change affinity, irqbalance can change that affinity from under me, correct? That's fine as long as it's stated that that's how things will work (turn off irqbalance or run oneshot mode for manual setting).
If you change /proc/irq/<num>/smp_affinity, yes, irqbalance can change that after the fact.
OK.
Quoted 36 lines
Quoted 12 lines
Quoted 9 lines
Quoted 5 lines
Also, can we add a restricted mask as I mention above into this scheme? If we can't send an IRQ to some node, we don't want to bother attempting to change affinity to cpus on that node (hopefully code in the kernel will eventually restrict this).
The interface allows you to put in any CPU mask. The way it's written now, whatever mask you put in, irqbalance *only* balances within that mask. It won't ever try and go outside that mask.
OK. Given that, it might be nice to combine the restricted cpus that I'm describing with your
node_affinitymask, but we could expose them as separate masks (node_affinityandrestricted_affinity, as I describe above).I think this might be getting too complicated. The only thing irqbalance is lacking today, in my mind, is the feedback mechanism, telling it what subset of CPU masks to balance within. There is a
allowed_mask, but that is used for a different purpose. Hence why I added another. But I think your needs can be met 100% with what I have already, and we can come up with a different name that's more generic. The flows would be something like this:Driver:
- Driver comes online, allocates memory in a sensible NUMA fashion
- Driver requests kernel for interrupts, ties them into handlers
- Driver now sets a NUMA-friendly affinity for each interrupt, to match with its initial memory allocation
- irqbalance balances interrupts within their new "hinted" affinities.Other:
- System comes online
- In your case, interrupts must be kept away from certain CPUs.
- Some mechanism in your architecture init can set the "hinted" affinity mask for each interrupt.
- irqbalance will not move interrupts to the CPUs you left out of the "hinted" affinity.
Quoted 1 line
Does this make more sense?
I agree that this was getting too complicated. What you have makes sense, but it -might- still be missing a couple of small things.
Here is a list of issues that I think we're trying to resolve here:
- Enforce hardware affinity restrictions in the kernel (don't allow irqs to be directed to where they absolutely cannot go). This will silently fail at the userspace level.
This is the issue I have been trying to resolve in this thread.
- Give userspace apps (irqbalance) better knowledge of where to redirect irqs.
- Driver provides direction (locality of queue/buffer allocations, etc..), which is the issue that you are attempting to resolve.
- I/O device provides direction (hint that hardware is on some node). I think we should expose the irq_desc->node (numa_node of the device). This is one of the two things we're missing.
Whether irqbalance uses it in the cases where the driver has not supplied a "hinted" affinity mask is up to you, but I think that information should be provided to userspace. Maybe we add this as a subsequent patch.
- provide hardware affinity restriction information to apps (irqbalance). This seems optional, although I think you've covered it in the "Other" flow above, but see my question below.
- provide hardware affinity restriction information to the driver for better decision making in allocation of resources.
This is the other thing we're missing.
This also seems optional, but -might- be desirable for what would eventually be a more complete solution to what you are trying to resolve. If the driver knows where irqs absolutely cannot be sent, it can better decide where to allocate it's resources.
I guess this leads to one more question: What happens if our hardware sets the "hinted" affinity mask, then a driver comes along and does the same thing? We need to make sure drivers and arches keep from stomping on each other.
Quoted 121 lines · Dimitri Sivanich
Quoted 51 lines · Peter P Waskiewicz Jr
Quoted 50 lines · Dimitri Sivanich
Quoted 49 lines · Peter P
Quoted 46 lines · Dimitri Sivanich
Quoted 36 lines · Peter P
Quoted 34 lines · Dimitri Sivanich
Quoted 32 lines · Peter P
Quoted 17 lines · Dimitri Sivanich
Quoted 4 lines · Arjan van de Ven
Quoted 2 lines · On Tue, 24 Nov 2009 09:41:18 -0800 [email protected] (Eric W. Biederman)…
Oii.
Quoted 8 lines
Quoted 5 lines
I don't think it is bad to export information to applications like irqbalance.
I think it pretty horrible that one of the standard ways I have heard to improve performance on 10G nics is to kill irqbalance.
irqbalance does not move networking irqs; if it does there's something evil going on in the system. But thanks for the bugreport ;)
It does move networking irqs.
Quoted 1 line
Quoted 8 lines
Quoted 4 lines
we had that; it didn't work.
what I'm asking for is for the kernel to expose the numa information; right now that is the piece that is missing.I'm wondering if we should expose that numa information in the form of a node or the set of allowed cpus, or both?
I'm guessing 'both' is the correct answer, so that apps like irqbalance can make a qualitative decision based on the node (affinity to cpus on this node is better), but an absolute decision based on allowed cpus (I cannot change affinity to anything but this set of cpus).
That's exactly what my patch in the thread "irq: Add
node_affinityCPU masks for smarter irqbalance hints" is doing. I've also done the irqbalance changes based on that kernel patch, and Arjan currently has that patch.Quoted 7 lines
Quoted 1 line
So if I understand correctly, you're patch takes care of the qualitative portion of it (we prefer to set affinity to these cpus, which may be on more than one node), but not the restrictive part of it (we cannot change affinity to anything but these cpus)?
That is correct. The patch provides an interface to both the kernel (functions) and /proc for userspace to set a CPU mask. That is the preferred mask for the interrupt to be balanced on. Then irqbalance will make decisions on how to balance within that provided mask, if it in fact has been provided.
What if it's not provided? Will irqbalance make decisions based on the
numa_nodeof that irq (I would hope)?Quoted 7 lines
Quoted 5 lines
Quoted 2 lines
If it's not provided, then irqbalance will continue to do what it does today. No changes.
So no numa awareness for ethernet irqs.
Nothing that makes sense, no.
OK. See my note below concerning exposing the
irq_desc'node'.Quoted 5 lines
Quoted 5 lines
I'm wondering what needs to be exposed in proc/irq at this point.
Do we expose separate cpu masks for everything? There are really 3 possible pieces of affinity information: your
node_affinity(optionally selected by the driver according to allocations), myrestricted_affinity(set by the specific arch), andnuma_nodeaffinity (the 'home' node of the device). Do we show cpumasks for all of these, or maybe shownuma_nodein place of a cpumask for the 'home' node of the device?Quoted 7 lines
Quoted 4 lines
With all of that information apps like irqbalance should be able to make some good decisions, but things can get confusing if there are too many masks.
Also, if I manually change affinity, irqbalance can change that affinity from under me, correct? That's fine as long as it's stated that that's how things will work (turn off irqbalance or run oneshot mode for manual setting).
If you change /proc/irq/<num>/smp_affinity, yes, irqbalance can change that after the fact.
OK.
Quoted 36 lines
Quoted 12 lines
Quoted 9 lines
Quoted 5 lines
Also, can we add a restricted mask as I mention above into this scheme? If we can't send an IRQ to some node, we don't want to bother attempting to change affinity to cpus on that node (hopefully code in the kernel will eventually restrict this).
The interface allows you to put in any CPU mask. The way it's written now, whatever mask you put in, irqbalance *only* balances within that mask. It won't ever try and go outside that mask.
OK. Given that, it might be nice to combine the restricted cpus that I'm describing with your
node_affinitymask, but we could expose them as separate masks (node_affinityandrestricted_affinity, as I describe above).I think this might be getting too complicated. The only thing irqbalance is lacking today, in my mind, is the feedback mechanism, telling it what subset of CPU masks to balance within. There is a
allowed_mask, but that is used for a different purpose. Hence why I added another. But I think your needs can be met 100% with what I have already, and we can come up with a different name that's more generic. The flows would be something like this:Driver:
- Driver comes online, allocates memory in a sensible NUMA fashion
- Driver requests kernel for interrupts, ties them into handlers
- Driver now sets a NUMA-friendly affinity for each interrupt, to match with its initial memory allocation
- irqbalance balances interrupts within their new "hinted" affinities.Other:
- System comes online
- In your case, interrupts must be kept away from certain CPUs.
- Some mechanism in your architecture init can set the "hinted" affinity mask for each interrupt.
- irqbalance will not move interrupts to the CPUs you left out of the "hinted" affinity.Quoted 1 line
Does this make more sense?
I agree that this was getting too complicated. What you have makes sense, but it -might- still be missing a couple of small things.
Here is a list of issues that I think we're trying to resolve here:
Quoted 8 lines
- Enforce hardware affinity restrictions in the kernel (don't allow irqs to be directed to where they absolutely cannot go). This will silently fail at the userspace level.
This is the issue I have been trying to resolve in this thread.- Give userspace apps (irqbalance) better knowledge of where to redirect irqs.
- Driver provides direction (locality of queue/buffer allocations, etc..), which is the issue that you are attempting to resolve.
Yes.
Quoted 6 lines
- I/O device provides direction (hint that hardware is on some node). I think we should expose the
irq_desc->node(numa_nodeof the device). This is one of the two things we're missing.
Whether irqbalance uses it in the cases where the driver has not supplied a "hinted" affinity mask is up to you, but I think that information should be provided to userspace. Maybe we add this as a subsequent patch.
Yes, we can either enforce that through the hinted mask, or we can give it the actual node. If we give it the node, irqbalance will take it a step farther and balance it down to the cache domain within that topology. Either way you'll get the same result.
Quoted 12 lines
- provide hardware affinity restriction information to apps (irqbalance). This seems optional, although I think you've covered it in the "Other" flow above, but see my question below.
- provide hardware affinity restriction information to the driver for better decision making in allocation of resources.
This is the other thing we're missing.
This also seems optional, but -might- be desirable for what would eventually be a more complete solution to what you are trying to resolve. If the driver knows where irqs absolutely cannot be sent, it can better decide where to allocate it's resources.
Quoted 3 lines
I guess this leads to one more question: What happens if our hardware sets the "hinted" affinity mask, then a driver comes along and does the same thing? We need to make sure drivers and arches keep from stomping on each other.
Agreed. We can add a bit or something in the irq_desc that the architecture code can set, indicating a driver shouldn't overload the hinted mask. The burden will still be on the driver to check that value and honor it.
I think we're on the same page. I hope Thomas can weigh in on this either way so I know where to go from here with my patchset. If I can lump other solutions into mine, I'm game for doing that work.
Cheers,
-PJ
Quoted 140 lines · Peter P Waskiewicz Jr
Quoted 121 lines · Dimitri Sivanich
Quoted 51 lines · Peter P Waskiewicz Jr
Quoted 50 lines · Dimitri Sivanich
Quoted 49 lines · Peter P
Quoted 46 lines · Dimitri Sivanich
Quoted 36 lines · Peter P
Quoted 34 lines · Dimitri Sivanich
Quoted 32 lines · Peter P
Quoted 17 lines · Dimitri Sivanich
Quoted 4 lines · Arjan van de Ven
Quoted 2 lines · On Tue, 24 Nov 2009 09:41:18 -0800 [email protected] (Eric W. Biederman)…
Oii.
Quoted 8 lines
Quoted 5 lines
I don't think it is bad to export information to applications like irqbalance.
I think it pretty horrible that one of the standard ways I have heard to improve performance on 10G nics is to kill irqbalance.
irqbalance does not move networking irqs; if it does there's something evil going on in the system. But thanks for the bugreport ;)
It does move networking irqs.
Quoted 1 line
Quoted 8 lines
Quoted 4 lines
we had that; it didn't work.
what I'm asking for is for the kernel to expose the numa information; right now that is the piece that is missing.I'm wondering if we should expose that numa information in the form of a node or the set of allowed cpus, or both?
I'm guessing 'both' is the correct answer, so that apps like irqbalance can make a qualitative decision based on the node (affinity to cpus on this node is better), but an absolute decision based on allowed cpus (I cannot change affinity to anything but this set of cpus).
That's exactly what my patch in the thread "irq: Add
node_affinityCPU masks for smarter irqbalance hints" is doing. I've also done the irqbalance changes based on that kernel patch, and Arjan currently has that patch.Quoted 7 lines
Quoted 1 line
So if I understand correctly, you're patch takes care of the qualitative portion of it (we prefer to set affinity to these cpus, which may be on more than one node), but not the restrictive part of it (we cannot change affinity to anything but these cpus)?
That is correct. The patch provides an interface to both the kernel (functions) and /proc for userspace to set a CPU mask. That is the preferred mask for the interrupt to be balanced on. Then irqbalance will make decisions on how to balance within that provided mask, if it in fact has been provided.
What if it's not provided? Will irqbalance make decisions based on the
numa_nodeof that irq (I would hope)?Quoted 7 lines
Quoted 5 lines
Quoted 2 lines
If it's not provided, then irqbalance will continue to do what it does today. No changes.
So no numa awareness for ethernet irqs.
Nothing that makes sense, no.
OK. See my note below concerning exposing the
irq_desc'node'.Quoted 5 lines
Quoted 5 lines
I'm wondering what needs to be exposed in proc/irq at this point.
Do we expose separate cpu masks for everything? There are really 3 possible pieces of affinity information: your
node_affinity(optionally selected by the driver according to allocations), myrestricted_affinity(set by the specific arch), andnuma_nodeaffinity (the 'home' node of the device). Do we show cpumasks for all of these, or maybe shownuma_nodein place of a cpumask for the 'home' node of the device?Quoted 7 lines
Quoted 4 lines
With all of that information apps like irqbalance should be able to make some good decisions, but things can get confusing if there are too many masks.
Also, if I manually change affinity, irqbalance can change that affinity from under me, correct? That's fine as long as it's stated that that's how things will work (turn off irqbalance or run oneshot mode for manual setting).
If you change /proc/irq/<num>/smp_affinity, yes, irqbalance can change that after the fact.
OK.
Quoted 36 lines
Quoted 12 lines
Quoted 9 lines
Quoted 5 lines
Also, can we add a restricted mask as I mention above into this scheme? If we can't send an IRQ to some node, we don't want to bother attempting to change affinity to cpus on that node (hopefully code in the kernel will eventually restrict this).
The interface allows you to put in any CPU mask. The way it's written now, whatever mask you put in, irqbalance *only* balances within that mask. It won't ever try and go outside that mask.
OK. Given that, it might be nice to combine the restricted cpus that I'm describing with your
node_affinitymask, but we could expose them as separate masks (node_affinityandrestricted_affinity, as I describe above).I think this might be getting too complicated. The only thing irqbalance is lacking today, in my mind, is the feedback mechanism, telling it what subset of CPU masks to balance within. There is a
allowed_mask, but that is used for a different purpose. Hence why I added another. But I think your needs can be met 100% with what I have already, and we can come up with a different name that's more generic. The flows would be something like this:Driver:
- Driver comes online, allocates memory in a sensible NUMA fashion
- Driver requests kernel for interrupts, ties them into handlers
- Driver now sets a NUMA-friendly affinity for each interrupt, to match with its initial memory allocation
- irqbalance balances interrupts within their new "hinted" affinities.Other:
- System comes online
- In your case, interrupts must be kept away from certain CPUs.
- Some mechanism in your architecture init can set the "hinted" affinity mask for each interrupt.
- irqbalance will not move interrupts to the CPUs you left out of the "hinted" affinity.Quoted 1 line
Does this make more sense?
I agree that this was getting too complicated. What you have makes sense, but it -might- still be missing a couple of small things.
Here is a list of issues that I think we're trying to resolve here:
Quoted 8 lines
- Enforce hardware affinity restrictions in the kernel (don't allow irqs to be directed to where they absolutely cannot go). This will silently fail at the userspace level.
This is the issue I have been trying to resolve in this thread.- Give userspace apps (irqbalance) better knowledge of where to redirect irqs.
- Driver provides direction (locality of queue/buffer allocations, etc..), which is the issue that you are attempting to resolve.Yes.
Quoted 6 lines
- I/O device provides direction (hint that hardware is on some node). I think we should expose the
irq_desc->node(numa_nodeof the device). This is one of the two things we're missing.
Whether irqbalance uses it in the cases where the driver has not supplied a "hinted" affinity mask is up to you, but I think that information should be provided to userspace. Maybe we add this as a subsequent patch.
Quoted 3 lines
Yes, we can either enforce that through the hinted mask, or we can give it the actual node. If we give it the node, irqbalance will take it a step farther and balance it down to the cache domain within that topology. Either way you'll get the same result.
So provide the irq_desc->node via /proc/irq/*/node for each irq, but also enforce through the hinted mask if so desired, right?
Quoted 13 lines
Quoted 13 lines
- provide hardware affinity restriction information to apps (irqbalance). This seems optional, although I think you've covered it in the "Other" flow above, but see my question below.
- provide hardware affinity restriction information to the driver for better decision making in allocation of resources.
This is the other thing we're missing.
This also seems optional, but -might- be desirable for what would eventually be a more complete solution to what you are trying to resolve. If the driver knows where irqs absolutely cannot be sent, it can better decide where to allocate it's resources.
Quoted 8 lines
Quoted 3 lines
I guess this leads to one more question: What happens if our hardware sets the "hinted" affinity mask, then a driver comes along and does the same thing? We need to make sure drivers and arches keep from stomping on each other.
Agreed. We can add a bit or something in the
irq_descthat the architecture code can set, indicating a driver shouldn't overload the hinted mask. The burden will still be on the driver to check that value and honor it.
OK. The driver could restrict the mask further (according to it's allocation), but not use cpus that were not set in the mask if the flag was set.
Quoted 7 lines
I think we're on the same page. I hope Thomas can weigh in on this either way so I know where to go from here with my patchset. If I can lump other solutions into mine, I'm game for doing that work.
Cheers,
-PJ
Quoted 10 lines · Arjan van de Ven
Quoted 6 lines
I think the irq scheduler is the only scheduler (except for batch jobs) that we don't put in the kernel. It seems to me that if we are going to go to all of the trouble to rewrite the generic code to better support irqbalance because we are having serious irqbalance problems, it will be less effort to suck irqbalance into the kernel along with everything else.
we had that; it didn't work.
what I'm asking for is for the kernel to expose the numa information; right now that is the piece that is missing.
but if there is only one userspace app, and that app has no user accessable configuration, what is the difference between telling everyone they must use that app and having the logic from that app in the kernel?
David Lang