cxl: Add generic MSI/MSI-X interrupt support
Davidlohr Bueso
maintainerPATCH 1/1 cxl: Add generic MSI/MSI-X interrupt support
Introduce a generic irq table for CXL components/features that can have standard irq support - DOE requires dynamic vector sizing and is not considered here.
Create an infrastructure to query the max vectors required for the CXL device.
--- drivers/cxl/pci.c | 63 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+)
drivers/cxl/pci.c +63 -0
@@ -428,6 +428,66 @@ static void devm_cxl_pci_create_doe(struct cxl_dev_state *cxlds) } } +/** +* struct cxl_irq_cap - CXL feature that is capable of receiving MSI/MSI-X irqs. +* +* @name: Name of the device generating this interrupt. +* @get_max_msgnum: Get the feature's largest interrupt message number. If the +* feature does not have the Interrupt Supported bit set, then +* return -1. +*/ +struct cxl_irq_cap { +const char *name; +int (*get_max_msgnum)(struct cxl_dev_state *cxlds); +}; + +static const struct cxl_irq_cap cxl_irq_cap_table[] = { + {"isolation" ,NULL }, + {"pmu_overflow" ,NULL }, + {"mailbox" ,NULL }, + {"event" ,NULL }, +}; + +static void cxl_pci_free_irq_vectors (void *data) +{ +pci_free_irq_vectors (data); +} + +static int cxl_pci_alloc_irq_vectors (struct cxl_dev_state *cxlds) +{ +struct device *dev = cxlds->dev; +struct pci_dev *pdev =to_pci_dev (dev); +int rc, i, vectors =-1 ; + +for (i =0 ; i <ARRAY_SIZE (cxl_irq_cap_table); i++) { +int irq; + +if (!cxl_irq_cap_table[i].get_max_msgnum) +continue ; + + irq = cxl_irq_cap_table[i].get_max_msgnum (cxlds); + vectors =max_t (int, irq, vectors); + } + +if (vectors ==-1 ) +return -EINVAL ;/* no irq support whatsoever */ + + vectors++; + rc =pci_alloc_irq_vectors (pdev, vectors, vectors, +PCI_IRQ_MSIX |PCI_IRQ_MSI ); +if (rc <0 ) +return rc; + +if (rc != vectors) { +dev_err (dev,"Not enough interrupts; use polling where supported\n" ); +/* Some got allocated; clean them up */ +cxl_pci_free_irq_vectors (pdev); +return -ENOSPC ; + } + +return devm_add_action_or_reset (dev, cxl_pci_free_irq_vectors, pdev); +} +static int cxl_pci_probe (struct pci_dev *pdev,const struct pci_device_id *id) {struct cxl_register_map map;
@@ -498,6 +558,9 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)if (IS_ERR (cxlmd))return PTR_ERR (cxlmd); +/* TODO: When there are users, this return value must be checked */ +cxl_pci_alloc_irq_vectors (cxlds); +if (resource_size (&cxlds->pmem_res) &&IS_ENABLED (CONFIG_CXL_PMEM )) rc =devm_cxl_add_nvdimm (&pdev->dev, cxlmd);
Signature · 1 line
2.37.3
Quoted 8 lines · Davidlohr Bueso
Introduce a generic irq table for CXL components/features that can have standard irq support - DOE requires dynamic vector sizing and is not considered here.
Create an infrastructure to query the max vectors required for the CXL device.
Signed-off-by Davidlohr Bueso <[email protected]>
LGTM. Although it would be nice to see the other half of the picture. i.e. having the mailbox consuming the vector via request_irq() as an example.
Quoted 85 lines
--- drivers/cxl/pci.c | 63 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+)drivers/cxl/pci.c +63
-0@@ -428,6 +428,66 @@ static void devm_cxl_pci_create_doe(struct cxl_dev_state *cxlds) } } +/** +* struct cxl_irq_cap - CXL feature that is capable of receiving MSI/MSI-X irqs. +* +* @name: Name of the device generating this interrupt. +* @get_max_msgnum: Get the feature's largest interrupt message number. If the +* feature does not have the Interrupt Supported bit set, then +* return -1. +*/ +struct cxl_irq_cap { +const char *name; +int (*get_max_msgnum)(struct cxl_dev_state *cxlds); +}; + +static const struct cxl_irq_cap cxl_irq_cap_table[] = { + {"isolation" ,NULL }, + {"pmu_overflow" ,NULL }, + {"mailbox" ,NULL }, + {"event" ,NULL }, +}; + +static void cxl_pci_free_irq_vectors (void *data) +{ +pci_free_irq_vectors (data); +} + +static int cxl_pci_alloc_irq_vectors (struct cxl_dev_state *cxlds) +{ +struct device *dev = cxlds->dev; +struct pci_dev *pdev =to_pci_dev (dev); +int rc, i, vectors =-1 ; + +for (i =0 ; i <ARRAY_SIZE (cxl_irq_cap_table); i++) { +int irq; + +if (!cxl_irq_cap_table[i].get_max_msgnum) +continue ; + + irq = cxl_irq_cap_table[i].get_max_msgnum (cxlds); + vectors =max_t (int, irq, vectors); + } + +if (vectors ==-1 ) +return -EINVAL ;/* no irq support whatsoever */ + + vectors++; + rc =pci_alloc_irq_vectors (pdev, vectors, vectors, +PCI_IRQ_MSIX |PCI_IRQ_MSI ); +if (rc <0 ) +return rc; + +if (rc != vectors) { +dev_err (dev,"Not enough interrupts; use polling where supported\n" ); +/* Some got allocated; clean them up */ +cxl_pci_free_irq_vectors (pdev); +return -ENOSPC ; + } + +return devm_add_action_or_reset (dev, cxl_pci_free_irq_vectors, pdev); +} +static int cxl_pci_probe (struct pci_dev *pdev,const struct pci_device_id *id) {struct cxl_register_map map;@@ -498,6 +558,9 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)if (IS_ERR (cxlmd))return PTR_ERR (cxlmd); +/* TODO: When there are users, this return value must be checked */ +cxl_pci_alloc_irq_vectors (cxlds); +if (resource_size (&cxlds->pmem_res) &&IS_ENABLED (CONFIG_CXL_PMEM )) rc =devm_cxl_add_nvdimm (&pdev->dev, cxlmd);
Davidlohr Bueso
maintainerQuoted 6 lines · Dave Jiang
Quoted 4 lines · Davidlohr Bueso
Introduce a generic irq table for CXL components/features that can have standard irq support - DOE requires dynamic vector sizing and is not considered here.
Quoted 8 lines
Quoted 4 lines
Create an infrastructure to query the max vectors required for the CXL device.
Signed-off-by Davidlohr Bueso <[email protected]>LGTM. Although it would be nice to see the other half of the picture. i.e. having the mailbox consuming the vector via
request_irq()as an example.
Well Ira has already implemented the irq support for the events[0], and my plan is to send my mbox irq patches along with the async handling stuff once this patch lands.
Quoted 2 lines
Reviewed-by Dave Jiang <[email protected]>
Thanks!
Quoted 8 lines · On Wed, 12 Oct 2022 11:04:32 -0700 Davidlohr Bueso <[email protected]> wrote:
Introduce a generic irq table for CXL components/features that can have standard irq support - DOE requires dynamic vector sizing and is not considered here.
Create an infrastructure to query the max vectors required for the CXL device.
Signed-off-by Davidlohr Bueso <[email protected]>
Hi Davidlohr,
Basically good, but a few comments inline.
I'll role this onto front of the v2 of the CPMU set as well.
Quoted 13 lines
--- drivers/cxl/pci.c | 63 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+)drivers/cxl/pci.c +1
-0@@ -428,6 +428,66 @@ static void devm_cxl_pci_create_doe(struct cxl_dev_state *cxlds) } } +/**
drivers/cxl/pci.c static void devm_cxl_pci_create_doe(struct cxl_dev_state *cxlds) · line 441
Quoted 10 lines
drivers/cxl/pci.c +10
-0+* struct cxl_irq_cap - CXL feature that is capable of receiving MSI/MSI-X irqs. +* +* @name: Name of the device generating this interrupt. +* @get_max_msgnum: Get the feature's largest interrupt message number. If the +* feature does not have the Interrupt Supported bit set, then +* return -1. +*/ +struct cxl_irq_cap { +const char *name; +int (*get_max_msgnum)(struct cxl_dev_state *cxlds);
For the CPMU case I need to walk the register locator dvsec block so need the callback to take the pci_dev not the cxl_dev_state.
Also need it later to map the resulting register blocks to go find the irq before then dropping them mappings so that the resulting CPMU device can grab them later.
drivers/cxl/pci.c static void devm_cxl_pci_create_doe(struct cxl_dev_state *cxlds) · line 448
Quoted 7 lines
drivers/cxl/pci.c +7
-0+}; + +static const struct cxl_irq_cap cxl_irq_cap_table[] = { + {"isolation" ,NULL }, + {"pmu_overflow" ,NULL }, + {"mailbox" ,NULL }, + {"event" ,NULL },
Fill these in as we provide them, not upfront. I'd rather see this attached to one (or possibly several) of the series that are coming along than stand alone. so start off with an empty table.
Quoted 15 lines
drivers/cxl/pci.c +15
-0+}; + +static void cxl_pci_free_irq_vectors (void *data) +{ +pci_free_irq_vectors (data); +} + +static int cxl_pci_alloc_irq_vectors (struct cxl_dev_state *cxlds) +{ +struct device *dev = cxlds->dev; +struct pci_dev *pdev =to_pci_dev (dev); +int rc, i, vectors =-1 ; + +for (i =0 ; i <ARRAY_SIZE (cxl_irq_cap_table); i++) { +int irq;
drivers/cxl/pci.c static void devm_cxl_pci_create_doe(struct cxl_dev_state *cxlds) · line 473
Quoted 10 lines
drivers/cxl/pci.c +10
-0+ +if (!cxl_irq_cap_table[i].get_max_msgnum) +continue ; + + irq = cxl_irq_cap_table[i].get_max_msgnum (cxlds); + vectors =max_t (int, irq, vectors); + } + +if (vectors ==-1 ) +return -EINVAL ;/* no irq support whatsoever */
return 0 in this case. No irqs present is a 'good' result if there aren't any. Will be up to the consumers of the interrupts to get their own interrupt vector numbers and they should get the same answers!
Quoted 21 lines
drivers/cxl/pci.c +17
-0+ + vectors++; + rc =pci_alloc_irq_vectors (pdev, vectors, vectors, +PCI_IRQ_MSIX |PCI_IRQ_MSI ); +if (rc <0 ) +return rc; + +if (rc != vectors) { +dev_err (dev,"Not enough interrupts; use polling where supported\n" ); +/* Some got allocated; clean them up */ +cxl_pci_free_irq_vectors (pdev); +return -ENOSPC ; + } + +return devm_add_action_or_reset (dev, cxl_pci_free_irq_vectors, pdev); +} +static int cxl_pci_probe (struct pci_dev *pdev,const struct pci_device_id *id) {struct cxl_register_map map;
drivers/cxl/pci.c static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) · line 563
Quoted 6 lines
drivers/cxl/pci.c +3
-0@@ -498,6 +558,9 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)if (IS_ERR (cxlmd))return PTR_ERR (cxlmd); +/* TODO: When there are users, this return value must be checked */ +cxl_pci_alloc_irq_vectors (cxlds); +
Gut feeling is this will end up moving ahead of any of the sub device creation because many of them end up needing interrupts.
Also check response from the start - can't see a reason to not do so as we won't be registering any at all if no callbacks provided.
So I'd move it above the devm_cxl_add_memdev() call.
Quoted 3 lines
drivers/cxl/pci.c +0
-0if (resource_size (&cxlds->pmem_res) &&IS_ENABLED (CONFIG_CXL_PMEM )) rc =devm_cxl_add_nvdimm (&pdev->dev, cxlmd);
Davidlohr Bueso
maintainerThanks for having a look.
Quoted 6 lines · Jonathan Cameron
drivers/cxl/pci.c static void devm_cxl_pci_create_doe(struct cxl_dev_state *cxlds) · line 441
Quoted 3 lines
drivers/cxl/pci.c +3
-0+struct cxl_irq_cap { +const char *name; +int (*get_max_msgnum)(struct cxl_dev_state *cxlds);For the CPMU case I need to walk the register locator dvsec block so need the callback to take the
pci_devnot thecxl_dev_state.
Hmm ok, however maybe I'm missing something, but given a pdev, do we have a way to get back to the cxlds?
...
Quoted 14 lines
drivers/cxl/pci.c static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) · line 563
Quoted 10 lines
static int
cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id){drivers/cxl/pci.c +3
-0struct cxl_register_map map;@@ -498,6 +558,9 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)if (IS_ERR (cxlmd))return PTR_ERR (cxlmd); +/* TODO: When there are users, this return value must be checked */ +cxl_pci_alloc_irq_vectors (cxlds); +Gut feeling is this will end up moving ahead of any of the sub device creation because many of them end up needing interrupts.
Quoted 4 lines
Also check response from the start - can't see a reason to not do so as we won't be registering any at all if no callbacks provided.
So I'd move it above the
devm_cxl_add_memdev()call.
Will do. In addition, are you ok with grouping the irq setup for each cxl feature/component, ie:
if (cxl_pci_alloc_irq_vectors(cxlds) > 0) {
cxl_setup_mbox_irq();
cxl_setup_events_irq();
cxl_setup_pmu_irq();
}
I ask mostly from the mailbox perspective, in that we already have a mbox setup call and can certainly understand if people would prefer it there, but I tend to prefer the above (logically wrt irqs).
Thanks,
Davidlohr
Davidlohr Bueso
maintainerQuoted 1 line · Davidlohr Bueso
if (
cxl_pci_alloc_irq_vectors(cxlds)> 0) {
bleh this should be == 0 of course.
Quoted 4 lines
cxl_setup_mbox_irq();
cxl_setup_events_irq();
cxl_setup_pmu_irq();
}
Quoted 11 lines · On Thu, 13 Oct 2022 10:37:03 -0700 Davidlohr Bueso <[email protected]> wrote:
Thanks for having a look.
Quoted 6 lines · Jonathan Cameron
drivers/cxl/pci.c static void devm_cxl_pci_create_doe(struct cxl_dev_state *cxlds) · line 441
Quoted 3 lines
drivers/cxl/pci.c +3
-0+struct cxl_irq_cap { +const char *name; +int (*get_max_msgnum)(struct cxl_dev_state *cxlds);For the CPMU case I need to walk the register locator dvsec block so need the callback to take the
pci_devnot thecxl_dev_state.
Quoted 2 lines
Hmm ok, however maybe I'm missing something, but given a pdev, do we have a way to get back to the cxlds?
I'd failed to register we can easily get from cxlds to pci dev. Had wrong mental model of what embedded what.
Quoted 31 lines
...
Quoted 14 lines
drivers/cxl/pci.c static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) · line 563
Quoted 10 lines
static int
cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id){drivers/cxl/pci.c +3
-0struct cxl_register_map map;@@ -498,6 +558,9 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)if (IS_ERR (cxlmd))return PTR_ERR (cxlmd); +/* TODO: When there are users, this return value must be checked */ +cxl_pci_alloc_irq_vectors (cxlds); +Gut feeling is this will end up moving ahead of any of the sub device creation because many of them end up needing interrupts.
Quoted 4 lines
Also check response from the start - can't see a reason to not do so as we won't be registering any at all if no callbacks provided.
So I'd move it above the
devm_cxl_add_memdev()call.Will do. In addition, are you ok with grouping the irq setup for each cxl feature/component, ie:
if (cxl_pci_alloc_irq_vectors(cxlds) > 0) { cxl_setup_mbox_irq(); cxl_setup_events_irq(); cxl_setup_pmu_irq(); }
Quoted 3 lines
I ask mostly from the mailbox perspective, in that we already have a mbox setup call and can certainly understand if people would prefer it there, but I tend to prefer the above (logically wrt irqs).
I'd rather see it in each of the setup calls. Out in pci.c we should just be doing minimum to find that max irq number. e.g. the CPMU driver will rewalk and find it's vector number directly from it's own register space.
Jonathan
Quoted 3 lines
Thanks,
Davidlohr
Quoted 10 lines · Jonathan Cameron
Quoted 7 lines · On Wed, 12 Oct 2022 11:04:32 -0700 Davidlohr Bueso <[email protected]> wrote:
Introduce a generic irq table for CXL components/features that can have standard irq support - DOE requires dynamic vector sizing and is not considered here.
Create an infrastructure to query the max vectors required for the CXL device.
Quoted 7 lines
Quoted 1 line
Signed-off-by Davidlohr Bueso <[email protected]>Hi Davidlohr,
Basically good, but a few comments inline.
I'll role this onto front of the v2 of the CPMU set as well.
And I don't mind this landing ahead of the event stuff. I'll take this in my series too but expect it to drop out when applied.
Ira
Quoted 119 lines
Quoted 14 lines
--- drivers/cxl/pci.c | 63 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+)drivers/cxl/pci.c +1
-0@@ -428,6 +428,66 @@ static void devm_cxl_pci_create_doe(struct cxl_dev_state *cxlds) } } +/** drivers/cxl/pci.c static void devm_cxl_pci_create_doe(struct cxl_dev_state *cxlds) · line 441
Quoted 10 lines
drivers/cxl/pci.c +10
-0+* struct cxl_irq_cap - CXL feature that is capable of receiving MSI/MSI-X irqs. +* +* @name: Name of the device generating this interrupt. +* @get_max_msgnum: Get the feature's largest interrupt message number. If the +* feature does not have the Interrupt Supported bit set, then +* return -1. +*/ +struct cxl_irq_cap { +const char *name; +int (*get_max_msgnum)(struct cxl_dev_state *cxlds);For the CPMU case I need to walk the register locator dvsec block so need the callback to take the
pci_devnot thecxl_dev_state.Also need it later to map the resulting register blocks to go find the irq before then dropping them mappings so that the resulting CPMU device can grab them later.
drivers/cxl/pci.c static void devm_cxl_pci_create_doe(struct cxl_dev_state *cxlds) · line 448
Quoted 7 lines
drivers/cxl/pci.c +7
-0+}; + +static const struct cxl_irq_cap cxl_irq_cap_table[] = { + {"isolation" ,NULL }, + {"pmu_overflow" ,NULL }, + {"mailbox" ,NULL }, + {"event" ,NULL },Fill these in as we provide them, not upfront. I'd rather see this attached to one (or possibly several) of the series that are coming along than stand alone. so start off with an empty table.
Quoted 15 lines
drivers/cxl/pci.c +15
-0+}; + +static void cxl_pci_free_irq_vectors (void *data) +{ +pci_free_irq_vectors (data); +} + +static int cxl_pci_alloc_irq_vectors (struct cxl_dev_state *cxlds) +{ +struct device *dev = cxlds->dev; +struct pci_dev *pdev =to_pci_dev (dev); +int rc, i, vectors =-1 ; + +for (i =0 ; i <ARRAY_SIZE (cxl_irq_cap_table); i++) { +int irq;drivers/cxl/pci.c static void devm_cxl_pci_create_doe(struct cxl_dev_state *cxlds) · line 473
Quoted 10 lines
drivers/cxl/pci.c +10
-0+ +if (!cxl_irq_cap_table[i].get_max_msgnum) +continue ; + + irq = cxl_irq_cap_table[i].get_max_msgnum (cxlds); + vectors =max_t (int, irq, vectors); + } + +if (vectors ==-1 ) +return -EINVAL ;/* no irq support whatsoever */ return 0 in this case. No irqs present is a 'good' result if there aren't any. Will be up to the consumers of the interrupts to get their own interrupt vector numbers and they should get the same answers!
Quoted 21 lines
drivers/cxl/pci.c +17
-0+ + vectors++; + rc =pci_alloc_irq_vectors (pdev, vectors, vectors, +PCI_IRQ_MSIX |PCI_IRQ_MSI ); +if (rc <0 ) +return rc; + +if (rc != vectors) { +dev_err (dev,"Not enough interrupts; use polling where supported\n" ); +/* Some got allocated; clean them up */ +cxl_pci_free_irq_vectors (pdev); +return -ENOSPC ; + } + +return devm_add_action_or_reset (dev, cxl_pci_free_irq_vectors, pdev); +} +static int cxl_pci_probe (struct pci_dev *pdev,const struct pci_device_id *id) {struct cxl_register_map map;drivers/cxl/pci.c static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) · line 563
Quoted 6 lines
drivers/cxl/pci.c +3
-0@@ -498,6 +558,9 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)if (IS_ERR (cxlmd))return PTR_ERR (cxlmd); +/* TODO: When there are users, this return value must be checked */ +cxl_pci_alloc_irq_vectors (cxlds); +Gut feeling is this will end up moving ahead of any of the sub device creation because many of them end up needing interrupts.
Also check response from the start - can't see a reason to not do so as we won't be registering any at all if no callbacks provided.
So I'd move it above the
devm_cxl_add_memdev()call.Quoted 3 lines
drivers/cxl/pci.c +0
-0if (resource_size (&cxlds->pmem_res) &&IS_ENABLED (CONFIG_CXL_PMEM )) rc =devm_cxl_add_nvdimm (&pdev->dev, cxlmd);