Adds pcitest tool support for MSI-X

Changes the pcie_raise_irq function signature, namely the interrupt_num variable type from u8 to u16 to accommodate the MSI-X maximum interrupts of 2048.

Implements a PCIe config space capability iterator function to search and save the MSI and MSI-X pointers. With this method the code becomes more generic and flexible.

Implements MSI-X set/get functions for sysfs interface in order to change the EP entries number.

Implements EP MSI-X interface for triggering interruptions.

Signed-off-by Gustavo Pimentel <[email protected]>

---
 drivers/pci/dwc/pci-dra7xx.c           |   2 +-
 drivers/pci/dwc/pcie-artpec6.c         |   2 +-
 drivers/pci/dwc/pcie-designware-ep.c   | 145 ++++++++++++++++++++++++++++++++-
 drivers/pci/dwc/pcie-designware-plat.c |   6 +-
 drivers/pci/dwc/pcie-designware.h      |  23 +++++-
 5 files changed, 173 insertions(+), 5 deletions(-)

drivers/pci/dwc/pci-dra7xx.c +1 -1

index ed8558d..5265725 100644
@@ -369,7 +369,7 @@ static void dra7xx_pcie_raise_msi_irq(struct dra7xx_pcie *dra7xx,
 }
 
 static int dra7xx_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
-				 enum pci_epc_irq_type type, u8 interrupt_num)
+				 enum pci_epc_irq_type type, u16 interrupt_num)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 	struct dra7xx_pcie *dra7xx = to_dra7xx_pcie(pci);

drivers/pci/dwc/pcie-artpec6.c +1 -1

index e66cede..96dc259 100644
@@ -428,7 +428,7 @@ static void artpec6_pcie_ep_init(struct dw_pcie_ep *ep)
 }
 
 static int artpec6_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
-				  enum pci_epc_irq_type type, u8 interrupt_num)
+				  enum pci_epc_irq_type type, u16 interrupt_num)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 

drivers/pci/dwc/pcie-designware-ep.c +0 -0

index 15b22a6..874d4c2 100644

Hi Gustavo,

Quoted 56 lines · Gustavo Pimentel

Changes the pcie_raise_irq function signature, namely the interrupt_num variable type from u8 to u16 to accommodate the MSI-X maximum interrupts of 2048.

Implements a PCIe config space capability iterator function to search and save the MSI and MSI-X pointers. With this method the code becomes more generic and flexible.

Implements MSI-X set/get functions for sysfs interface in order to change the EP entries number.

Implements EP MSI-X interface for triggering interruptions.

Signed-off-by Gustavo Pimentel <[email protected]>

---
 drivers/pci/dwc/pci-dra7xx.c           |   2 +-
 drivers/pci/dwc/pcie-artpec6.c         |   2 +-
 drivers/pci/dwc/pcie-designware-ep.c   | 145 ++++++++++++++++++++++++++++++++-
 drivers/pci/dwc/pcie-designware-plat.c |   6 +-
 drivers/pci/dwc/pcie-designware.h      |  23 +++++-
 5 files changed, 173 insertions(+), 5 deletions(-)

drivers/pci/dwc/pci-dra7xx.c +1 -1

index ed8558d..5265725 100644
@@ -369,7 +369,7 @@ static void dra7xx_pcie_raise_msi_irq(struct dra7xx_pcie *dra7xx,
 }
 
 static int dra7xx_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
-				 enum pci_epc_irq_type type, u8 interrupt_num)
+				 enum pci_epc_irq_type type, u16 interrupt_num)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 	struct dra7xx_pcie *dra7xx = to_dra7xx_pcie(pci);

drivers/pci/dwc/pcie-artpec6.c +1 -1

index e66cede..96dc259 100644
@@ -428,7 +428,7 @@ static void artpec6_pcie_ep_init(struct dw_pcie_ep *ep)
 }
 
 static int artpec6_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
-				  enum pci_epc_irq_type type, u8 interrupt_num)
+				  enum pci_epc_irq_type type, u16 interrupt_num)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 

drivers/pci/dwc/pcie-designware-ep.c +0 -0

index 15b22a6..874d4c2 100644
@@ -40,6 +40,44 @@ void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar)
 	__dw_pcie_ep_reset_bar(pci, bar, 0);
 }
 

drivers/pci/dwc/pcie-designware-ep.c void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar) · line 44

Quoted 2 lines

drivers/pci/dwc/pcie-designware-ep.c +2 -0

+void dw_pcie_ep_find_cap_addr(struct dw_pcie_ep *ep)
+{

This should be implemented in a generic way similar to pci_find_capability(). It'll be useful when we try to implement other capabilities as well.

Thanks
Kishon

Hi Kishon,

Quoted 59 lines · Kishon Vijay Abraham I

Hi Gustavo,

Quoted 56 lines · Gustavo Pimentel

Changes the pcie_raise_irq function signature, namely the interrupt_num variable type from u8 to u16 to accommodate the MSI-X maximum interrupts of 2048.

Implements a PCIe config space capability iterator function to search and save the MSI and MSI-X pointers. With this method the code becomes more generic and flexible.

Implements MSI-X set/get functions for sysfs interface in order to change the EP entries number.

Implements EP MSI-X interface for triggering interruptions.

Signed-off-by Gustavo Pimentel <[email protected]>

---
 drivers/pci/dwc/pci-dra7xx.c           |   2 +-
 drivers/pci/dwc/pcie-artpec6.c         |   2 +-
 drivers/pci/dwc/pcie-designware-ep.c   | 145 ++++++++++++++++++++++++++++++++-
 drivers/pci/dwc/pcie-designware-plat.c |   6 +-
 drivers/pci/dwc/pcie-designware.h      |  23 +++++-
 5 files changed, 173 insertions(+), 5 deletions(-)

drivers/pci/dwc/pci-dra7xx.c +1 -1

index ed8558d..5265725 100644
@@ -369,7 +369,7 @@ static void dra7xx_pcie_raise_msi_irq(struct dra7xx_pcie *dra7xx,
 }
 
 static int dra7xx_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
-				 enum pci_epc_irq_type type, u8 interrupt_num)
+				 enum pci_epc_irq_type type, u16 interrupt_num)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 	struct dra7xx_pcie *dra7xx = to_dra7xx_pcie(pci);

drivers/pci/dwc/pcie-artpec6.c +1 -1

index e66cede..96dc259 100644
@@ -428,7 +428,7 @@ static void artpec6_pcie_ep_init(struct dw_pcie_ep *ep)
 }
 
 static int artpec6_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
-				  enum pci_epc_irq_type type, u8 interrupt_num)
+				  enum pci_epc_irq_type type, u16 interrupt_num)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 

drivers/pci/dwc/pcie-designware-ep.c +0 -0

index 15b22a6..874d4c2 100644
@@ -40,6 +40,44 @@ void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar)
 	__dw_pcie_ep_reset_bar(pci, bar, 0);
 }
 

Quoted 5 lines

drivers/pci/dwc/pcie-designware-ep.c void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar) · line 44

Quoted 2 lines

drivers/pci/dwc/pcie-designware-ep.c +2 -0

+void dw_pcie_ep_find_cap_addr(struct dw_pcie_ep *ep)
+{

This should be implemented in a generic way similar to pci_find_capability(). It'll be useful when we try to implement other capabilities as well.

Hum, what you suggest? Something implemented on the pci-epf-core?

Hi,

Quoted 68 lines · Gustavo Pimentel

Hi Kishon,

Quoted 64 lines · Kishon Vijay Abraham I

Hi Gustavo,

Quoted 53 lines · Gustavo Pimentel

Changes the pcie_raise_irq function signature, namely the interrupt_num variable type from u8 to u16 to accommodate the MSI-X maximum interrupts of 2048.

Implements a PCIe config space capability iterator function to search and save the MSI and MSI-X pointers. With this method the code becomes more generic and flexible.

Implements MSI-X set/get functions for sysfs interface in order to change the EP entries number.

Implements EP MSI-X interface for triggering interruptions.

Signed-off-by Gustavo Pimentel <[email protected]>

---
 drivers/pci/dwc/pci-dra7xx.c           |   2 +-
 drivers/pci/dwc/pcie-artpec6.c         |   2 +-
 drivers/pci/dwc/pcie-designware-ep.c   | 145 ++++++++++++++++++++++++++++++++-
 drivers/pci/dwc/pcie-designware-plat.c |   6 +-
 drivers/pci/dwc/pcie-designware.h      |  23 +++++-
 5 files changed, 173 insertions(+), 5 deletions(-)

drivers/pci/dwc/pci-dra7xx.c +1 -1

index ed8558d..5265725 100644
@@ -369,7 +369,7 @@ static void dra7xx_pcie_raise_msi_irq(struct dra7xx_pcie *dra7xx,
 }
 
 static int dra7xx_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
-				 enum pci_epc_irq_type type, u8 interrupt_num)
+				 enum pci_epc_irq_type type, u16 interrupt_num)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 	struct dra7xx_pcie *dra7xx = to_dra7xx_pcie(pci);

drivers/pci/dwc/pcie-artpec6.c +1 -1

index e66cede..96dc259 100644
@@ -428,7 +428,7 @@ static void artpec6_pcie_ep_init(struct dw_pcie_ep *ep)
 }
 
 static int artpec6_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
-				  enum pci_epc_irq_type type, u8 interrupt_num)
+				  enum pci_epc_irq_type type, u16 interrupt_num)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 

drivers/pci/dwc/pcie-designware-ep.c +0 -0

index 15b22a6..874d4c2 100644

drivers/pci/dwc/pcie-designware-ep.c void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar) · line 44

Quoted 5 lines

drivers/pci/dwc/pcie-designware-ep.c +2 -0

@@ -40,6 +40,44 @@ void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar)
 	__dw_pcie_ep_reset_bar(pci, bar, 0);
 }
 
+void dw_pcie_ep_find_cap_addr(struct dw_pcie_ep *ep)
+{

This should be implemented in a generic way similar to pci_find_capability(). It'll be useful when we try to implement other capabilities as well.

Quoted 1 line

Hum, what you suggest? Something implemented on the pci-epf-core?

yeah, Initially thought it could be implemented as a helper function in pci-epc-core so that both designware and cadence can use it.

But do we really have to find the address like this? since all designware IP's will have a particular capability at a fixed address offset, why not follow use existing mechanism in dw_pcie_ep_get_msi?

Or is it possible for a particular capability to have address offsets for different vendors? How is it for cadence?

Thanks
Kishon

Hi Kishon,

Quoted 71 lines · Kishon Vijay Abraham I

Hi,

Quoted 68 lines · Gustavo Pimentel

Hi Kishon,

Quoted 64 lines · Kishon Vijay Abraham I

Hi Gustavo,

Quoted 53 lines · Gustavo Pimentel

Changes the pcie_raise_irq function signature, namely the interrupt_num variable type from u8 to u16 to accommodate the MSI-X maximum interrupts of 2048.

Implements a PCIe config space capability iterator function to search and save the MSI and MSI-X pointers. With this method the code becomes more generic and flexible.

Implements MSI-X set/get functions for sysfs interface in order to change the EP entries number.

Implements EP MSI-X interface for triggering interruptions.

Signed-off-by Gustavo Pimentel <[email protected]>

---
 drivers/pci/dwc/pci-dra7xx.c           |   2 +-
 drivers/pci/dwc/pcie-artpec6.c         |   2 +-
 drivers/pci/dwc/pcie-designware-ep.c   | 145 ++++++++++++++++++++++++++++++++-
 drivers/pci/dwc/pcie-designware-plat.c |   6 +-
 drivers/pci/dwc/pcie-designware.h      |  23 +++++-
 5 files changed, 173 insertions(+), 5 deletions(-)

drivers/pci/dwc/pci-dra7xx.c +1 -1

index ed8558d..5265725 100644
@@ -369,7 +369,7 @@ static void dra7xx_pcie_raise_msi_irq(struct dra7xx_pcie *dra7xx,
 }
 
 static int dra7xx_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
-				 enum pci_epc_irq_type type, u8 interrupt_num)
+				 enum pci_epc_irq_type type, u16 interrupt_num)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 	struct dra7xx_pcie *dra7xx = to_dra7xx_pcie(pci);

drivers/pci/dwc/pcie-artpec6.c +1 -1

index e66cede..96dc259 100644
@@ -428,7 +428,7 @@ static void artpec6_pcie_ep_init(struct dw_pcie_ep *ep)
 }
 
 static int artpec6_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
-				  enum pci_epc_irq_type type, u8 interrupt_num)
+				  enum pci_epc_irq_type type, u16 interrupt_num)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 

drivers/pci/dwc/pcie-designware-ep.c +0 -0

index 15b22a6..874d4c2 100644

drivers/pci/dwc/pcie-designware-ep.c void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar) · line 44

Quoted 5 lines

drivers/pci/dwc/pcie-designware-ep.c +2 -0

@@ -40,6 +40,44 @@ void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar)
 	__dw_pcie_ep_reset_bar(pci, bar, 0);
 }
 
+void dw_pcie_ep_find_cap_addr(struct dw_pcie_ep *ep)
+{

This should be implemented in a generic way similar to pci_find_capability(). It'll be useful when we try to implement other capabilities as well.

Quoted 4 lines

Quoted 1 line

Hum, what you suggest? Something implemented on the pci-epf-core?

yeah, Initially thought it could be implemented as a helper function in pci-epc-core so that both designware and cadence can use it.

That would be nice, however I couldn't find out how to access the config space, through the pci_epf or pci_epc structs.

So, I reworked the functions like this:

(on pcie-designware-ep.c)

u8 __dw_pcie_ep_find_next_cap(struct dw_pcie *pci, u8 cap_ptr,

                              u8 cap)
{
        u8 cap_id, next_cap_ptr;
        u16 reg;

        reg = dw_pcie_readw_dbi(pci, cap_ptr);
        next_cap_ptr = (reg & 0xff00) >> 8;
        cap_id = (reg & 0x00ff);

        if (!next_cap_ptr || cap_id > PCI_CAP_ID_MAX)
                return 0;

        if (cap_id == cap)
                return cap_ptr;

        return __dw_pcie_ep_find_next_cap(pci, next_cap_ptr, cap);
}

u8 dw_pcie_ep_find_capability(struct dw_pcie *pci, u8 cap)

{
        u8 next_cap_ptr;
        u16 reg;

        reg = dw_pcie_readw_dbi(pci, PCI_CAPABILITY_LIST);
        next_cap_ptr = (reg & 0x00ff);

        if (!next_cap_ptr)
                return 0;

        return __dw_pcie_ep_find_next_cap(pci, next_cap_ptr, cap);
}

int dw_pcie_ep_init(struct dw_pcie_ep *ep)
{
[...]

        ep->msi_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSI);
        ep->msix_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSIX);

[...]
}

Quoted 4 lines

But do we really have to find the address like this? since all designware IP's will have a particular capability at a fixed address offset, why not follow use existing mechanism in dw_pcie_ep_get_msi?

The capabilities are not fixed to a specific address offset by default they assume those values, but they can be easily change at design stage.

Quoted 3 lines

Or is it possible for a particular capability to have address offsets for different vendors? How is it for cadence?

Yes, it's possible to have different address offset for different vendors.

Quoted 4 lines

Thanks
Kishon

Thanks,
Gustavo

Hi,

Quoted 76 lines · Gustavo Pimentel

Hi Kishon,

Quoted 73 lines · Kishon Vijay Abraham I

Hi,

Quoted 69 lines · Gustavo Pimentel

Hi Kishon,

Quoted 59 lines · Kishon Vijay Abraham I

Hi Gustavo,

Quoted 56 lines · Gustavo Pimentel

Changes the pcie_raise_irq function signature, namely the interrupt_num variable type from u8 to u16 to accommodate the MSI-X maximum interrupts of 2048.

Implements a PCIe config space capability iterator function to search and save the MSI and MSI-X pointers. With this method the code becomes more generic and flexible.

Implements MSI-X set/get functions for sysfs interface in order to change the EP entries number.

Implements EP MSI-X interface for triggering interruptions.

Signed-off-by Gustavo Pimentel <[email protected]>

---
 drivers/pci/dwc/pci-dra7xx.c           |   2 +-
 drivers/pci/dwc/pcie-artpec6.c         |   2 +-
 drivers/pci/dwc/pcie-designware-ep.c   | 145 ++++++++++++++++++++++++++++++++-
 drivers/pci/dwc/pcie-designware-plat.c |   6 +-
 drivers/pci/dwc/pcie-designware.h      |  23 +++++-
 5 files changed, 173 insertions(+), 5 deletions(-)

drivers/pci/dwc/pci-dra7xx.c +1 -1

index ed8558d..5265725 100644
@@ -369,7 +369,7 @@ static void dra7xx_pcie_raise_msi_irq(struct dra7xx_pcie *dra7xx,
 }
 
 static int dra7xx_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
-				 enum pci_epc_irq_type type, u8 interrupt_num)
+				 enum pci_epc_irq_type type, u16 interrupt_num)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 	struct dra7xx_pcie *dra7xx = to_dra7xx_pcie(pci);

drivers/pci/dwc/pcie-artpec6.c +1 -1

index e66cede..96dc259 100644
@@ -428,7 +428,7 @@ static void artpec6_pcie_ep_init(struct dw_pcie_ep *ep)
 }
 
 static int artpec6_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
-				  enum pci_epc_irq_type type, u8 interrupt_num)
+				  enum pci_epc_irq_type type, u16 interrupt_num)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 

drivers/pci/dwc/pcie-designware-ep.c +0 -0

index 15b22a6..874d4c2 100644
@@ -40,6 +40,44 @@ void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar)
 	__dw_pcie_ep_reset_bar(pci, bar, 0);
 }
 

Quoted 5 lines

drivers/pci/dwc/pcie-designware-ep.c void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar) · line 44

Quoted 2 lines

drivers/pci/dwc/pcie-designware-ep.c +2 -0

+void dw_pcie_ep_find_cap_addr(struct dw_pcie_ep *ep)
+{

This should be implemented in a generic way similar to pci_find_capability(). It'll be useful when we try to implement other capabilities as well.

Hum, what you suggest? Something implemented on the pci-epf-core?

Quoted 5 lines

Quoted 2 lines

yeah, Initially thought it could be implemented as a helper function in pci-epc-core so that both designware and cadence can use it.

That would be nice, however I couldn't find out how to access the config space, through the pci_epf or pci_epc structs.

It's just a helper function so it can directly take the base address of the configuration space as argument (in our case, it should be dbi_base).

Thanks
Kishon

Quoted 68 lines

So, I reworked the functions like this:

(on pcie-designware-ep.c)

u8 __dw_pcie_ep_find_next_cap(struct dw_pcie *pci, u8 cap_ptr,

                              u8 cap)
{
        u8 cap_id, next_cap_ptr;
        u16 reg;

        reg = dw_pcie_readw_dbi(pci, cap_ptr);
        next_cap_ptr = (reg & 0xff00) >> 8;
        cap_id = (reg & 0x00ff);

        if (!next_cap_ptr || cap_id > PCI_CAP_ID_MAX)
                return 0;

        if (cap_id == cap)
                return cap_ptr;

        return __dw_pcie_ep_find_next_cap(pci, next_cap_ptr, cap);
}

u8 dw_pcie_ep_find_capability(struct dw_pcie *pci, u8 cap)

{
        u8 next_cap_ptr;
        u16 reg;

        reg = dw_pcie_readw_dbi(pci, PCI_CAPABILITY_LIST);
        next_cap_ptr = (reg & 0x00ff);

        if (!next_cap_ptr)
                return 0;

        return __dw_pcie_ep_find_next_cap(pci, next_cap_ptr, cap);
}

int dw_pcie_ep_init(struct dw_pcie_ep *ep)
{
[...]

        ep->msi_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSI);
        ep->msix_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSIX);

[...]
}

Quoted 4 lines

But do we really have to find the address like this? since all designware IP's will have a particular capability at a fixed address offset, why not follow use existing mechanism in dw_pcie_ep_get_msi?

The capabilities are not fixed to a specific address offset by default they assume those values, but they can be easily change at design stage.

Quoted 3 lines

Or is it possible for a particular capability to have address offsets for different vendors? How is it for cadence?

Yes, it's possible to have different address offset for different vendors.

Quoted 4 lines

Thanks
Kishon

Thanks,
Gustavo

Hi Kishon,
=20

Quoted 27 lines · Kishon Vijay Abraham I

Hi,

Quoted 24 lines · Gustavo Pimentel

Hi Kishon,

Quoted 21 lines · Kishon Vijay Abraham I

Hi Gustavo,

Quoted 18 lines · Gustavo Pimentel

Changes the pcie_raise_irq function signature, namely the interrupt_num variable type from u8 to u16 to accommodate the MSI-X maximum interrupts of 2048.

Implements a PCIe config space capability iterator function to search and save the MSI and MSI-X pointers. With this method the code becomes more generic and flexible.

Implements MSI-X set/get functions for sysfs interface in order to change the EP entries number.

Implements EP MSI-X interface for triggering interruptions.

Signed-off-by Gustavo Pimentel <[email protected]>

---
 drivers/pci/dwc/pci-dra7xx.c           |   2 +-
 drivers/pci/dwc/pcie-artpec6.c         |   2 +-
 drivers/pci/dwc/pcie-designware-ep.c   | 145

++++++++++++++++++++++++++++++++-

Quoted 11 lines

Quoted 11 lines

Quoted 11 lines

Quoted 11 lines

 drivers/pci/dwc/pcie-designware-plat.c |   6 +-
 drivers/pci/dwc/pcie-designware.h      |  23 +++++-
 5 files changed, 173 insertions(+), 5 deletions(-)

(quoted diff) +0 -0

b/drivers/pci/dwc/pci-dra7xx.c index ed8558d..5265725 100644

drivers/pci/dwc/pci-dra7xx.c +0 -0

@@ -369,7 +369,7 @@ static void dra7xx_pcie_raise_msi_irq(struct
 dra7xx_pcie *dra7xx,  }
 

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

drivers/pci/dwc/pci-dra7xx.c static void dra7xx_pcie_raise_msi_irq(struct · line 372

Quoted 2 lines

drivers/pci/dwc/pci-dra7xx.c +0 -1

 static int dra7xx_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
-				 enum pci_epc_irq_type type, u8

interrupt_num)

Quoted 1 line

Quoted 1 line

Quoted 1 line

drivers/pci/dwc/pci-dra7xx.c static void dra7xx_pcie_raise_msi_irq(struct · line 372

Quoted 1 line

drivers/pci/dwc/pci-dra7xx.c +1 -0

+				 enum pci_epc_irq_type type, u16

interrupt_num)

Quoted 10 lines

Quoted 10 lines

Quoted 10 lines

Quoted 10 lines

drivers/pci/dwc/pci-dra7xx.c +0 -0

 {
 	struct dw_pcie *pci =3D to_dw_pcie_from_ep(ep);
 	struct dra7xx_pcie *dra7xx =3D to_dra7xx_pcie(pci); diff --git

a/drivers/pci/dwc/pcie-artpec6.c b/drivers/pci/dwc/pcie-artpec6.c index e66cede..96dc259 100644

drivers/pci/dwc/pcie-artpec6.c +0 -0

@@ -428,7 +428,7 @@ static void artpec6_pcie_ep_init(struct
 dw_pcie_ep *ep)  }
 

Quoted 1 line

Quoted 1 line

Quoted 1 line

drivers/pci/dwc/pcie-artpec6.c static void artpec6_pcie_ep_init(struct · line 430

Quoted 1 line

drivers/pci/dwc/pcie-artpec6.c +0 -0

 static int artpec6_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no=

Quoted 1 line

Quoted 1 line

Quoted 1 line

drivers/pci/dwc/pcie-artpec6.c static void artpec6_pcie_ep_init(struct · line 431

Quoted 1 line

drivers/pci/dwc/pcie-artpec6.c +0 -1

-				  enum pci_epc_irq_type type, u8

interrupt_num)

Quoted 1 line

Quoted 1 line

Quoted 1 line

drivers/pci/dwc/pcie-artpec6.c static void artpec6_pcie_ep_init(struct · line 431

Quoted 1 line

drivers/pci/dwc/pcie-artpec6.c +1 -0

+				  enum pci_epc_irq_type type, u16

interrupt_num)

Quoted 9 lines

Quoted 9 lines

Quoted 9 lines

drivers/pci/dwc/pcie-designware-ep.c void dw_pcie_ep_reset_bar(struct dw_pcie

Quoted 9 lines

drivers/pci/dwc/pcie-artpec6.c +0 -0

 {
 	struct dw_pcie *pci =3D to_dw_pcie_from_ep(ep);
 

drivers/pci/dwc/pcie-designware-ep.c +0 -0

index 15b22a6..874d4c2 100644
@@ -40,6 +40,44 @@ void dw_pcie_ep_reset_bar(struct dw_pcie

*pci, enum pci_barno bar)

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

drivers/pci/dwc/pcie-designware-ep.c void dw_pcie_ep_reset_bar(struct dw_pcie · line 42

Quoted 3 lines

drivers/pci/dwc/pcie-designware-ep.c +1 -0

 	__dw_pcie_ep_reset_bar(pci, bar, 0);  }
 
+void dw_pcie_ep_find_cap_addr(struct dw_pcie_ep *ep) {

This should be implemented in a generic way similar to

pci_find_capability().

Quoted 6 lines

Quoted 3 lines

Quoted 1 line

It'll be useful when we try to implement other capabilities as well.

Hum, what you suggest? Something implemented on the pci-epf-core?

yeah, Initially thought it could be implemented as a helper function in pci-epc-core so that both designware and cadence can use it.

=20
That would be nice, however I couldn't find out how to access the config space, through the pci_epf or pci_epc structs.
=20
So, I reworked the functions like this:
=20
(on pcie-designware-ep.c)
=20
u8 __dw_pcie_ep_find_next_cap(struct dw_pcie *pci, u8 cap_ptr,

                              u8 cap)
{
        u8 cap_id, next_cap_ptr;
        u16 reg;
=20
        reg =3D dw_pcie_readw_dbi(pci, cap_ptr);
        next_cap_ptr =3D (reg & 0xff00) >> 8;
        cap_id =3D (reg & 0x00ff);
=20
        if (!next_cap_ptr || cap_id > PCI_CAP_ID_MAX)
                return 0;
=20
        if (cap_id =3D=3D cap)
                return cap_ptr;
=20
        return __dw_pcie_ep_find_next_cap(pci, next_cap_ptr, cap); }
=20
u8 dw_pcie_ep_find_capability(struct dw_pcie *pci, u8 cap) {
        u8 next_cap_ptr;
        u16 reg;
=20
        reg =3D dw_pcie_readw_dbi(pci, PCI_CAPABILITY_LIST);
        next_cap_ptr =3D (reg & 0x00ff);

=20

        if (!next_cap_ptr)
                return 0;
=20
        return __dw_pcie_ep_find_next_cap(pci, next_cap_ptr, cap); }
=20
int dw_pcie_ep_init(struct dw_pcie_ep *ep) { [...]
        ep->msi_cap =3D dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSI);
        ep->msix_cap =3D dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSIX)=

}
=20

Quoted 4 lines

But do we really have to find the address like this? since all designware IP's will have a particular capability at a fixed address offset, why not follow use existing mechanism in dw_pcie_ep_get_msi?

=20
The capabilities are not fixed to a specific address offset by default th=

assume those values, but they can be easily change at design stage. =20

Quoted 3 lines

Or is it possible for a particular capability to have address offsets for different vendors? How is it for cadence?

=20
Yes, it's possible to have different address offset for different vendors=

=20

Quoted 4 lines

Thanks
Kishon

=20
Thanks,
Gustavo

Changes the pcie_raise_irq function signature, namely the interrupt_num variable type from u8 to u16 to accommodate the MSI-X maximum interrupts of 2048.
=20
Implements a PCIe config space capability iterator function to search and=

the MSI and MSI-X pointers. With this method the code becomes more generic and flexible.
=20
Implements MSI-X set/get functions for sysfs interface in order to change=

ed8558d..5265725 100644

drivers/pci/dwc/pci-dra7xx.c +0 -0

@@ -369,7 +369,7 @@ static void dra7xx_pcie_raise_msi_irq(struct
 dra7xx_pcie *dra7xx,  }

=20
static int dra7xx_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,

drivers/pci/dwc/pci-dra7xx.c +0 -1

-				 enum pci_epc_irq_type type, u8

interrupt_num)

drivers/pci/dwc/pci-dra7xx.c +1 -0

+				 enum pci_epc_irq_type type, u16

interrupt_num)
{

drivers/pci/dwc/pci-dra7xx.c +0 -0

 	struct dw_pcie *pci =3D to_dw_pcie_from_ep(ep);
 	struct dra7xx_pcie *dra7xx =3D to_dra7xx_pcie(pci); diff --git

a/drivers/pci/dwc/pcie-artpec6.c b/drivers/pci/dwc/pcie-artpec6.c index e66cede..96dc259 100644

drivers/pci/dwc/pcie-artpec6.c +0 -0

@@ -428,7 +428,7 @@ static void artpec6_pcie_ep_init(struct dw_pcie_ep
 *ep)  }

=20
static int artpec6_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,

drivers/pci/dwc/pcie-artpec6.c +0 -1

-				  enum pci_epc_irq_type type, u8

interrupt_num)

drivers/pci/dwc/pcie-artpec6.c +1 -0

+				  enum pci_epc_irq_type type, u16

interrupt_num)
{
struct dw_pcie *pci =3D to_dw_pcie_from_ep(ep);
=20

drivers/pci/dwc/pcie- renamed from drivers/pci/dwc/pcie-designware-ep.c +0 -0

designware-ep.c
index 15b22a6..874d4c2 100644

drivers/pci/dwc/pcie-designware-ep.c +0 -0

@@ -40,6 +40,44 @@ void dw_pcie_ep_reset_bar(struct dw_pcie *pci,
 enum pci_barno bar)
 	__dw_pcie_ep_reset_bar(pci, bar, 0);
 }

=20

drivers/pci/dwc/pcie-designware-ep.c +37 -0

+void dw_pcie_ep_find_cap_addr(struct dw_pcie_ep *ep) {
+	struct dw_pcie *pci =3D to_dw_pcie_from_ep(ep);
+	u8 next_ptr, curr_ptr, cap_id;
+	u16 reg;
+
+	memset(&ep->cap_addr, 0, sizeof(ep->cap_addr));
+
+	reg =3D dw_pcie_readw_dbi(pci, PCI_STATUS);
+	if (!(reg & PCI_STATUS_CAP_LIST))
+		return;
+
+	reg =3D dw_pcie_readw_dbi(pci, PCI_CAPABILITY_LIST);
+	next_ptr =3D (reg & 0x00ff);
+	if (!next_ptr)
+		return;
+
+	reg =3D dw_pcie_readw_dbi(pci, next_ptr);
+	curr_ptr =3D next_ptr;
+	next_ptr =3D (reg & 0xff00) >> 8;
+	cap_id =3D (reg & 0x00ff);
+
+	while (next_ptr && (cap_id <=3D PCI_CAP_ID_MAX)) {
+		switch (cap_id) {
+		case PCI_CAP_ID_MSI:
+			ep->cap_addr.msi_addr =3D curr_ptr;
+			break;
+		case PCI_CAP_ID_MSIX:
+			ep->cap_addr.msix_addr =3D curr_ptr;
+			break;
+		}
+		reg =3D dw_pcie_readw_dbi(pci, next_ptr);
+		curr_ptr =3D next_ptr;
+		next_ptr =3D (reg & 0xff00) >> 8;
+		cap_id =3D (reg & 0x00ff);
+	}
+}
+
 static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no,
 				   struct pci_epf_header *hdr)
 {

@@ -241,8 +279,47 @@ static int dw_pcie_ep_set_msi(struct pci_epc *epc,
 u8 func_no, u8 encode_int)
 	return 0;
 }

=20

drivers/pci/dwc/pcie-designware-ep.c +38 -1

+static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no) {
+	struct dw_pcie_ep *ep =3D epc_get_drvdata(epc);
+	struct dw_pcie *pci =3D to_dw_pcie_from_ep(ep);
+	u32 val, reg;
+
+	if (ep->cap_addr.msix_addr =3D=3D 0)
+		return 0;
+
+	reg =3D ep->cap_addr.msix_addr + PCI_MSIX_FLAGS;
+	val =3D dw_pcie_readw_dbi(pci, reg);
+	if (!(val & PCI_MSIX_FLAGS_ENABLE))
+		return -EINVAL;
+
+	val &=3D PCI_MSIX_FLAGS_QSIZE;
+
+	return val;
+}
+
+static int dw_pcie_ep_set_msix(struct pci_epc *epc, u8 func_no, u16
+interrupts) {
+	struct dw_pcie_ep *ep =3D epc_get_drvdata(epc);
+	struct dw_pcie *pci =3D to_dw_pcie_from_ep(ep);
+	u32 val, reg;
+
+	if (ep->cap_addr.msix_addr =3D=3D 0)
+		return 0;
+
+	reg =3D ep->cap_addr.msix_addr + PCI_MSIX_FLAGS;
+	val =3D dw_pcie_readw_dbi(pci, reg);
+	val &=3D ~PCI_MSIX_FLAGS_QSIZE;
+	val |=3D interrupts;
+	dw_pcie_dbi_ro_wr_en(pci);
+	dw_pcie_writew_dbi(pci, reg, val);
+	dw_pcie_dbi_ro_wr_dis(pci);
+
+	return 0;
+}
+
 static int dw_pcie_ep_raise_irq(struct pci_epc *epc, u8 func_no,
-				enum pci_epc_irq_type type, u8

interrupt_num)

drivers/pci/dwc/pcie-designware-ep.c +1 -0

+				enum pci_epc_irq_type type, u16

interrupt_num)
{
struct dw_pcie_ep *ep =3D epc_get_drvdata(epc);
=20

drivers/pci/dwc/pcie-designware-ep.c +2 -0

@@ -282,6 +359,8 @@ static const struct pci_epc_ops epc_ops =3D {
 	.unmap_addr		=3D dw_pcie_ep_unmap_addr,
 	.set_msi		=3D dw_pcie_ep_set_msi,
 	.get_msi		=3D dw_pcie_ep_get_msi,
+	.set_msix		=3D dw_pcie_ep_set_msix,
+	.get_msix		=3D dw_pcie_ep_get_msix,
 	.raise_irq		=3D dw_pcie_ep_raise_irq,
 	.start			=3D dw_pcie_ep_start,
 	.stop			=3D dw_pcie_ep_stop,

@@ -322,6 +401,60 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep
 *ep, u8 func_no,
 	return 0;
 }

=20

drivers/pci/dwc/pcie-designware-ep.c +19 -0

+int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
+			     u16 interrupt_num)
+{
+	struct dw_pcie *pci =3D to_dw_pcie_from_ep(ep);
+	struct pci_epc *epc =3D ep->epc;
+	u16 tbl_offset, bir;
+	u32 bar_addr_upper, bar_addr_lower;
+	u32 msg_addr_upper, msg_addr_lower;
+	u32 reg, msg_data;
+	u64 tbl_addr, msg_addr, reg_u64;
+	void __iomem *msix_tbl;
+	int ret;
+
+	reg =3D ep->cap_addr.msix_addr + PCI_MSIX_TABLE;
+	tbl_offset =3D dw_pcie_readl_dbi(pci, reg);
+	bir =3D (tbl_offset & PCI_MSIX_TABLE_BIR);
+	tbl_offset &=3D PCI_MSIX_TABLE_OFFSET;
+	tbl_offset >>=3D 3;
+

Hi,

Quoted 199 lines · On 10 April 2018 18:15 Gustavo Pimentel wrote:

Changes the pcie_raise_irq function signature, namely the interrupt_num variable type from u8 to u16 to accommodate the MSI-X maximum interrupts of 2048.

Implements a PCIe config space capability iterator function to search and save the MSI and MSI-X pointers. With this method the code becomes more generic and flexible.

Implements MSI-X set/get functions for sysfs interface in order to change the EP entries number.

Implements EP MSI-X interface for triggering interruptions.

Signed-off-by Gustavo Pimentel <[email protected]>

---
 drivers/pci/dwc/pci-dra7xx.c           |   2 +-
 drivers/pci/dwc/pcie-artpec6.c         |   2 +-
 drivers/pci/dwc/pcie-designware-ep.c   | 145
++++++++++++++++++++++++++++++++-
 drivers/pci/dwc/pcie-designware-plat.c |   6 +-
 drivers/pci/dwc/pcie-designware.h      |  23 +++++-
 5 files changed, 173 insertions(+), 5 deletions(-)

drivers/pci/dwc/pci-dra7xx.c index renamed from drivers/pci/dwc/pci-dra7xx.c +0 -0

ed8558d..5265725 100644

drivers/pci/dwc/pci-dra7xx.c +1 -1

@@ -369,7 +369,7 @@ static void dra7xx_pcie_raise_msi_irq(struct
 dra7xx_pcie *dra7xx,  }
 
 static int dra7xx_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
-				 enum pci_epc_irq_type type, u8
 interrupt_num)
+				 enum pci_epc_irq_type type, u16
 interrupt_num)
 {

drivers/pci/dwc/pci-dra7xx.c +0 -0

 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 	struct dra7xx_pcie *dra7xx = to_dra7xx_pcie(pci); diff --git

a/drivers/pci/dwc/pcie-artpec6.c b/drivers/pci/dwc/pcie-artpec6.c index e66cede..96dc259 100644

drivers/pci/dwc/pcie-artpec6.c +1 -1

@@ -428,7 +428,7 @@ static void artpec6_pcie_ep_init(struct dw_pcie_ep
 *ep)  }
 
 static int artpec6_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
-				  enum pci_epc_irq_type type, u8
 interrupt_num)
+				  enum pci_epc_irq_type type, u16
 interrupt_num)
 {

drivers/pci/dwc/pcie-artpec6.c +0 -0

 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);

drivers/pci/dwc/pcie- renamed from drivers/pci/dwc/pcie-designware-ep.c +0 -0

designware-ep.c
index 15b22a6..874d4c2 100644

drivers/pci/dwc/pcie-designware-ep.c +109 -1

@@ -40,6 +40,44 @@ void dw_pcie_ep_reset_bar(struct dw_pcie *pci,
 enum pci_barno bar)
 	__dw_pcie_ep_reset_bar(pci, bar, 0);
 }
 
+void dw_pcie_ep_find_cap_addr(struct dw_pcie_ep *ep) {
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u8 next_ptr, curr_ptr, cap_id;
+	u16 reg;
+
+	memset(&ep->cap_addr, 0, sizeof(ep->cap_addr));
+
+	reg = dw_pcie_readw_dbi(pci, PCI_STATUS);
+	if (!(reg & PCI_STATUS_CAP_LIST))
+		return;
+
+	reg = dw_pcie_readw_dbi(pci, PCI_CAPABILITY_LIST);
+	next_ptr = (reg & 0x00ff);
+	if (!next_ptr)
+		return;
+
+	reg = dw_pcie_readw_dbi(pci, next_ptr);
+	curr_ptr = next_ptr;
+	next_ptr = (reg & 0xff00) >> 8;
+	cap_id = (reg & 0x00ff);
+
+	while (next_ptr && (cap_id <= PCI_CAP_ID_MAX)) {
+		switch (cap_id) {
+		case PCI_CAP_ID_MSI:
+			ep->cap_addr.msi_addr = curr_ptr;
+			break;
+		case PCI_CAP_ID_MSIX:
+			ep->cap_addr.msix_addr = curr_ptr;
+			break;
+		}
+		reg = dw_pcie_readw_dbi(pci, next_ptr);
+		curr_ptr = next_ptr;
+		next_ptr = (reg & 0xff00) >> 8;
+		cap_id = (reg & 0x00ff);
+	}
+}
+
 static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no,
 				   struct pci_epf_header *hdr)
 {

@@ -241,8 +279,47 @@ static int dw_pcie_ep_set_msi(struct pci_epc *epc,
 u8 func_no, u8 encode_int)
 	return 0;
 }
 
+static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no) {
+	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 val, reg;
+
+	if (ep->cap_addr.msix_addr == 0)
+		return 0;
+
+	reg = ep->cap_addr.msix_addr + PCI_MSIX_FLAGS;
+	val = dw_pcie_readw_dbi(pci, reg);
+	if (!(val & PCI_MSIX_FLAGS_ENABLE))
+		return -EINVAL;
+
+	val &= PCI_MSIX_FLAGS_QSIZE;
+
+	return val;
+}
+
+static int dw_pcie_ep_set_msix(struct pci_epc *epc, u8 func_no, u16
+interrupts) {
+	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 val, reg;
+
+	if (ep->cap_addr.msix_addr == 0)
+		return 0;
+
+	reg = ep->cap_addr.msix_addr + PCI_MSIX_FLAGS;
+	val = dw_pcie_readw_dbi(pci, reg);
+	val &= ~PCI_MSIX_FLAGS_QSIZE;
+	val |= interrupts;
+	dw_pcie_dbi_ro_wr_en(pci);
+	dw_pcie_writew_dbi(pci, reg, val);
+	dw_pcie_dbi_ro_wr_dis(pci);
+
+	return 0;
+}
+
 static int dw_pcie_ep_raise_irq(struct pci_epc *epc, u8 func_no,
-				enum pci_epc_irq_type type, u8
 interrupt_num)
+				enum pci_epc_irq_type type, u16
 interrupt_num)
 {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);

@@ -282,6 +359,8 @@ static const struct pci_epc_ops epc_ops = {
 	.unmap_addr		= dw_pcie_ep_unmap_addr,
 	.set_msi		= dw_pcie_ep_set_msi,
 	.get_msi		= dw_pcie_ep_get_msi,
+	.set_msix		= dw_pcie_ep_set_msix,
+	.get_msix		= dw_pcie_ep_get_msix,
 	.raise_irq		= dw_pcie_ep_raise_irq,
 	.start			= dw_pcie_ep_start,
 	.stop			= dw_pcie_ep_stop,

@@ -322,6 +401,60 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep
 *ep, u8 func_no,
 	return 0;
 }
 
+int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
+			     u16 interrupt_num)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	struct pci_epc *epc = ep->epc;
+	u16 tbl_offset, bir;
+	u32 bar_addr_upper, bar_addr_lower;
+	u32 msg_addr_upper, msg_addr_lower;
+	u32 reg, msg_data;
+	u64 tbl_addr, msg_addr, reg_u64;
+	void __iomem *msix_tbl;
+	int ret;
+
+	reg = ep->cap_addr.msix_addr + PCI_MSIX_TABLE;
+	tbl_offset = dw_pcie_readl_dbi(pci, reg);
+	bir = (tbl_offset & PCI_MSIX_TABLE_BIR);
+	tbl_offset &= PCI_MSIX_TABLE_OFFSET;
+	tbl_offset >>= 3;
+
+	reg = PCI_BASE_ADDRESS_0 + (4 * bir);
+	bar_addr_lower = dw_pcie_readl_dbi(pci, reg);
+	reg_u64 = (bar_addr_lower & PCI_BASE_ADDRESS_MEM_TYPE_MASK);
+	if (reg_u64 == PCI_BASE_ADDRESS_MEM_TYPE_64)
+		bar_addr_upper = dw_pcie_readl_dbi(pci, reg + 4);
+	else
+		bar_addr_upper = 0;
+
+	tbl_addr = ((u64) bar_addr_upper) << 32 | bar_addr_lower;
+	tbl_addr += (tbl_offset + ((interrupt_num - 1) * PCI_MSIX_ENTRY_SIZE));
+	tbl_addr &= PCI_BASE_ADDRESS_MEM_MASK;
+

drivers/pci/dwc/pcie-designware-ep.c int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, · line 438

Quoted 4 lines

drivers/pci/dwc/pcie-designware-ep.c +4 -0

+	msix_tbl = ioremap_nocache(ep->phys_base + tbl_addr, ep->addr_size);
+	if (!msix_tbl)
+		return -EINVAL;
+

I think you need to check the mask bit in vector control for the requested IRQ. You could set the pending bit if masked, but would then need some output signal to inform when the mask bit is cleared (or poll it) so the message can be sent later.

Also, do you need to check PCI_MSIX_FLAGS_ENABLE here as well, or is it checked earlier?

From what I could see the PCI_MSI_FLAGS_ENABLE/PCI_MSIX_FLAGS_ENABLE flags are

Regards,
Alan

Hi,

Quoted 196 lines · On 10 April 2018 18:15 Gustavo Pimentel wrote:

Changes the pcie_raise_irq function signature, namely the interrupt_num variable type from u8 to u16 to accommodate the MSI-X maximum interrupts of 2048.

Implements a PCIe config space capability iterator function to search and save the MSI and MSI-X pointers. With this method the code becomes more generic and flexible.

Implements MSI-X set/get functions for sysfs interface in order to change the EP entries number.

Implements EP MSI-X interface for triggering interruptions.

Signed-off-by Gustavo Pimentel <[email protected]>

---
 drivers/pci/dwc/pci-dra7xx.c           |   2 +-
 drivers/pci/dwc/pcie-artpec6.c         |   2 +-
 drivers/pci/dwc/pcie-designware-ep.c   | 145
++++++++++++++++++++++++++++++++-
 drivers/pci/dwc/pcie-designware-plat.c |   6 +-
 drivers/pci/dwc/pcie-designware.h      |  23 +++++-
 5 files changed, 173 insertions(+), 5 deletions(-)

drivers/pci/dwc/pci-dra7xx.c index renamed from drivers/pci/dwc/pci-dra7xx.c +0 -0

ed8558d..5265725 100644

drivers/pci/dwc/pci-dra7xx.c +1 -1

@@ -369,7 +369,7 @@ static void dra7xx_pcie_raise_msi_irq(struct
 dra7xx_pcie *dra7xx,  }
 
 static int dra7xx_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
-				 enum pci_epc_irq_type type, u8
 interrupt_num)
+				 enum pci_epc_irq_type type, u16
 interrupt_num)
 {

drivers/pci/dwc/pci-dra7xx.c +0 -0

 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 	struct dra7xx_pcie *dra7xx = to_dra7xx_pcie(pci); diff --git

a/drivers/pci/dwc/pcie-artpec6.c b/drivers/pci/dwc/pcie-artpec6.c index e66cede..96dc259 100644

drivers/pci/dwc/pcie-artpec6.c +1 -1

@@ -428,7 +428,7 @@ static void artpec6_pcie_ep_init(struct dw_pcie_ep
 *ep)  }
 
 static int artpec6_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
-				  enum pci_epc_irq_type type, u8
 interrupt_num)
+				  enum pci_epc_irq_type type, u16
 interrupt_num)
 {

drivers/pci/dwc/pcie-artpec6.c +0 -0

 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);

drivers/pci/dwc/pcie- renamed from drivers/pci/dwc/pcie-designware-ep.c +0 -0

designware-ep.c
index 15b22a6..874d4c2 100644

drivers/pci/dwc/pcie-designware-ep.c +106 -1

@@ -40,6 +40,44 @@ void dw_pcie_ep_reset_bar(struct dw_pcie *pci,
 enum pci_barno bar)
 	__dw_pcie_ep_reset_bar(pci, bar, 0);
 }
 
+void dw_pcie_ep_find_cap_addr(struct dw_pcie_ep *ep) {
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u8 next_ptr, curr_ptr, cap_id;
+	u16 reg;
+
+	memset(&ep->cap_addr, 0, sizeof(ep->cap_addr));
+
+	reg = dw_pcie_readw_dbi(pci, PCI_STATUS);
+	if (!(reg & PCI_STATUS_CAP_LIST))
+		return;
+
+	reg = dw_pcie_readw_dbi(pci, PCI_CAPABILITY_LIST);
+	next_ptr = (reg & 0x00ff);
+	if (!next_ptr)
+		return;
+
+	reg = dw_pcie_readw_dbi(pci, next_ptr);
+	curr_ptr = next_ptr;
+	next_ptr = (reg & 0xff00) >> 8;
+	cap_id = (reg & 0x00ff);
+
+	while (next_ptr && (cap_id <= PCI_CAP_ID_MAX)) {
+		switch (cap_id) {
+		case PCI_CAP_ID_MSI:
+			ep->cap_addr.msi_addr = curr_ptr;
+			break;
+		case PCI_CAP_ID_MSIX:
+			ep->cap_addr.msix_addr = curr_ptr;
+			break;
+		}
+		reg = dw_pcie_readw_dbi(pci, next_ptr);
+		curr_ptr = next_ptr;
+		next_ptr = (reg & 0xff00) >> 8;
+		cap_id = (reg & 0x00ff);
+	}
+}
+
 static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no,
 				   struct pci_epf_header *hdr)
 {

@@ -241,8 +279,47 @@ static int dw_pcie_ep_set_msi(struct pci_epc *epc,
 u8 func_no, u8 encode_int)
 	return 0;
 }
 
+static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no) {
+	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 val, reg;
+
+	if (ep->cap_addr.msix_addr == 0)
+		return 0;
+
+	reg = ep->cap_addr.msix_addr + PCI_MSIX_FLAGS;
+	val = dw_pcie_readw_dbi(pci, reg);
+	if (!(val & PCI_MSIX_FLAGS_ENABLE))
+		return -EINVAL;
+
+	val &= PCI_MSIX_FLAGS_QSIZE;
+
+	return val;
+}
+
+static int dw_pcie_ep_set_msix(struct pci_epc *epc, u8 func_no, u16
+interrupts) {
+	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 val, reg;
+
+	if (ep->cap_addr.msix_addr == 0)
+		return 0;
+
+	reg = ep->cap_addr.msix_addr + PCI_MSIX_FLAGS;
+	val = dw_pcie_readw_dbi(pci, reg);
+	val &= ~PCI_MSIX_FLAGS_QSIZE;
+	val |= interrupts;
+	dw_pcie_dbi_ro_wr_en(pci);
+	dw_pcie_writew_dbi(pci, reg, val);
+	dw_pcie_dbi_ro_wr_dis(pci);
+
+	return 0;
+}
+
 static int dw_pcie_ep_raise_irq(struct pci_epc *epc, u8 func_no,
-				enum pci_epc_irq_type type, u8
 interrupt_num)
+				enum pci_epc_irq_type type, u16
 interrupt_num)
 {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);

@@ -282,6 +359,8 @@ static const struct pci_epc_ops epc_ops = {
 	.unmap_addr		= dw_pcie_ep_unmap_addr,
 	.set_msi		= dw_pcie_ep_set_msi,
 	.get_msi		= dw_pcie_ep_get_msi,
+	.set_msix		= dw_pcie_ep_set_msix,
+	.get_msix		= dw_pcie_ep_get_msix,
 	.raise_irq		= dw_pcie_ep_raise_irq,
 	.start			= dw_pcie_ep_start,
 	.stop			= dw_pcie_ep_stop,

@@ -322,6 +401,60 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep
 *ep, u8 func_no,
 	return 0;
 }
 
+int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
+			     u16 interrupt_num)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	struct pci_epc *epc = ep->epc;
+	u16 tbl_offset, bir;
+	u32 bar_addr_upper, bar_addr_lower;
+	u32 msg_addr_upper, msg_addr_lower;
+	u32 reg, msg_data;
+	u64 tbl_addr, msg_addr, reg_u64;
+	void __iomem *msix_tbl;
+	int ret;
+
+	reg = ep->cap_addr.msix_addr + PCI_MSIX_TABLE;
+	tbl_offset = dw_pcie_readl_dbi(pci, reg);
+	bir = (tbl_offset & PCI_MSIX_TABLE_BIR);
+	tbl_offset &= PCI_MSIX_TABLE_OFFSET;
+	tbl_offset >>= 3;
+
+	reg = PCI_BASE_ADDRESS_0 + (4 * bir);
+	bar_addr_lower = dw_pcie_readl_dbi(pci, reg);
+	reg_u64 = (bar_addr_lower & PCI_BASE_ADDRESS_MEM_TYPE_MASK);
+	if (reg_u64 == PCI_BASE_ADDRESS_MEM_TYPE_64)
+		bar_addr_upper = dw_pcie_readl_dbi(pci, reg + 4);
+	else
+		bar_addr_upper = 0;
+
+	tbl_addr = ((u64) bar_addr_upper) << 32 | bar_addr_lower;

drivers/pci/dwc/pcie-designware-ep.c int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, · line 438

Quoted 7 lines

drivers/pci/dwc/pcie-designware-ep.c +7 -0

+	tbl_addr += (tbl_offset + ((interrupt_num - 1) * PCI_MSIX_ENTRY_SIZE));
+	tbl_addr &= PCI_BASE_ADDRESS_MEM_MASK;
+
+	msix_tbl = ioremap_nocache(ep->phys_base + tbl_addr, ep->addr_size);
+	if (!msix_tbl)
+		return -EINVAL;
+

I think you need to check the mask bit in vector control for the requested IRQ.

You could set the pending bit if masked, but would then need some output signal to inform when the mask bit is cleared (or poll it) so the message can be sent later.

Also, do you need to check PCI_MSIX_FLAGS_ENABLE here as well, or is it checked earlier?

Regards,
Alan

From: Gustavo Pimentel [mailto:[email protected]] Changes the cdns_pcie_ep_raise_irq function signature, namely the interrupt_num variable type from u8 to u16 to accommodate the MSI-X maximum interrupts of 2048.
=20

Signed-off-by Gustavo Pimentel <[email protected]>

---
 drivers/pci/cadence/pcie-cadence-ep.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

=20

drivers/pci/cadence/pcie-cadence-ep.c +0 -0

index 3d8283e..6d6322c 100644
@@ -363,7 +363,7 @@ static int cdns_pcie_ep_send_msi_irq(struct
 cdns_pcie_ep *ep, u8 fn,  }

=20
static int cdns_pcie_ep_raise_irq(struct pci_epc *epc, u8 fn,

drivers/pci/cadence/pcie-cadence-ep.c +0 -1

-				  enum pci_epc_irq_type type, u8

interrupt_num)

drivers/pci/cadence/pcie-cadence-ep.c +1 -0

+				  enum pci_epc_irq_type type, u16

interrupt_num)
{
struct cdns_pcie_ep *ep =3D epc_get_drvdata(epc);
=20
--
2.7.4
=20

Implements the generic method for calling the get/set callbacks.

Adds the PCI_EPC_IRQ_MSIX type.

Adds the MSI-X callbacks signatures to the ops structure.

Adds sysfs interface for altering the number of MSI-X entries.

Signed-off-by Gustavo Pimentel <[email protected]>

---
 drivers/pci/endpoint/pci-ep-cfs.c   | 24 ++++++++++++++++
 drivers/pci/endpoint/pci-epc-core.c | 57 +++++++++++++++++++++++++++++++++++++
 include/linux/pci-epc.h             | 11 ++++++-
 include/linux/pci-epf.h             |  1 +
 4 files changed, 92 insertions(+), 1 deletion(-)

drivers/pci/endpoint/pci-ep-cfs.c +24 -0

index 018ea34..d1288a0 100644
@@ -286,6 +286,28 @@ static ssize_t pci_epf_msi_interrupts_show(struct config_item *item,
 		       to_pci_epf_group(item)->epf->msi_interrupts);
 }
 
+static ssize_t pci_epf_msix_interrupts_store(struct config_item *item,
+					     const char *page, size_t len)
+{
+	u16 val;
+	int ret;
+
+	ret = kstrtou16(page, 0, &val);
+	if (ret)
+		return ret;
+
+	to_pci_epf_group(item)->epf->msix_interrupts = val;
+
+	return len;
+}
+
+static ssize_t pci_epf_msix_interrupts_show(struct config_item *item,
+					    char *page)
+{
+	return sprintf(page, "%d\n",
+		       to_pci_epf_group(item)->epf->msix_interrupts);
+}
+
 PCI_EPF_HEADER_R(vendorid)
 PCI_EPF_HEADER_W_u16(vendorid)
 

@@ -327,6 +349,7 @@ CONFIGFS_ATTR(pci_epf_, subsys_vendor_id);
 CONFIGFS_ATTR(pci_epf_, subsys_id);
 CONFIGFS_ATTR(pci_epf_, interrupt_pin);
 CONFIGFS_ATTR(pci_epf_, msi_interrupts);
+CONFIGFS_ATTR(pci_epf_, msix_interrupts);
 
 static struct configfs_attribute *pci_epf_attrs[] = {
 	&pci_epf_attr_vendorid,

@@ -340,6 +363,7 @@ static struct configfs_attribute *pci_epf_attrs[] = {
 	&pci_epf_attr_subsys_id,
 	&pci_epf_attr_interrupt_pin,
 	&pci_epf_attr_msi_interrupts,
+	&pci_epf_attr_msix_interrupts,
 	NULL,
 };
 

drivers/pci/endpoint/pci-epc-core.c +15 -0

index b0ee427..294a383 100644
@@ -218,6 +218,63 @@ int pci_epc_set_msi(struct pci_epc *epc, u8 func_no, u8 interrupts)
 EXPORT_SYMBOL_GPL(pci_epc_set_msi);
 
 /**
+ * pci_epc_get_msix() - get the number of MSI-X interrupt numbers allocated
+ * @epc: the EPC device to which MSI-X interrupts was requested
+ * @func_no: the endpoint function number in the EPC device
+ *
+ * Invoke to get the number of MSI-X interrupts allocated by the RC
+ */
+int pci_epc_get_msix(struct pci_epc *epc, u8 func_no)
+{
+	int interrupt;
+	unsigned long flags;
+
+	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions)
+		return 0;
+
+	if (!epc->ops->get_msix)

drivers/pci/endpoint/pci-epc-core.c +32 -0

+}
+EXPORT_SYMBOL_GPL(pci_epc_get_msix);
+
+/**
+ * pci_epc_set_msix() - set the number of MSI-X interrupt numbers required
+ * @epc: the EPC device on which MSI-X has to be configured
+ * @func_no: the endpoint function number in the EPC device
+ * @interrupts: number of MSI-X interrupts required by the EPF
+ *
+ * Invoke to set the required number of MSI-X interrupts.
+ */
+int pci_epc_set_msix(struct pci_epc *epc, u8 func_no, u16 interrupts)
+{
+	int ret;
+	unsigned long flags;
+
+	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions ||
+	    interrupts < 1 || interrupts > 2048)
+		return -EINVAL;
+
+	if (!epc->ops->set_msix)
+		return 0;
+
+	spin_lock_irqsave(&epc->lock, flags);
+	ret = epc->ops->set_msix(epc, func_no, interrupts - 1);
+	spin_unlock_irqrestore(&epc->lock, flags);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(pci_epc_set_msix);
+
+/**
  * pci_epc_unmap_addr() - unmap CPU address from PCI address
  * @epc: the EPC device on which address is allocated
  * @func_no: the endpoint function number in the EPC device

include/linux/pci-epc.h +5 -0

index af657ca..32e8961 100644
@@ -17,6 +17,7 @@ enum pci_epc_irq_type {
 	PCI_EPC_IRQ_UNKNOWN,
 	PCI_EPC_IRQ_LEGACY,
 	PCI_EPC_IRQ_MSI,
+	PCI_EPC_IRQ_MSIX,
 };
 
 /**

@@ -30,6 +31,10 @@ enum pci_epc_irq_type {
  *	     capability register
  * @get_msi: ops to get the number of MSI interrupts allocated by the RC from
  *	     the MSI capability register
+ * @set_msix: ops to set the requested number of MSI-X interrupts in the
+ *	     MSI-X capability register
+ * @get_msix: ops to get the number of MSI-X interrupts allocated by the RC
+ *	     from the MSI-X capability register
  * @raise_irq: ops to raise a legacy or MSI interrupt
  * @start: ops to start the PCI link
  * @stop: ops to stop the PCI link

Hi,

Quoted 92 lines · Gustavo Pimentel

Implements the generic method for calling the get/set callbacks.

Adds the PCI_EPC_IRQ_MSIX type.

Adds the MSI-X callbacks signatures to the ops structure.

Adds sysfs interface for altering the number of MSI-X entries.

Signed-off-by Gustavo Pimentel <[email protected]>

---
 drivers/pci/endpoint/pci-ep-cfs.c   | 24 ++++++++++++++++
 drivers/pci/endpoint/pci-epc-core.c | 57 +++++++++++++++++++++++++++++++++++++
 include/linux/pci-epc.h             | 11 ++++++-
 include/linux/pci-epf.h             |  1 +
 4 files changed, 92 insertions(+), 1 deletion(-)

drivers/pci/endpoint/pci-ep-cfs.c +24 -0

index 018ea34..d1288a0 100644
@@ -286,6 +286,28 @@ static ssize_t pci_epf_msi_interrupts_show(struct config_item *item,
 		       to_pci_epf_group(item)->epf->msi_interrupts);
 }
 
+static ssize_t pci_epf_msix_interrupts_store(struct config_item *item,
+					     const char *page, size_t len)
+{
+	u16 val;
+	int ret;
+
+	ret = kstrtou16(page, 0, &val);
+	if (ret)
+		return ret;
+
+	to_pci_epf_group(item)->epf->msix_interrupts = val;
+
+	return len;
+}
+
+static ssize_t pci_epf_msix_interrupts_show(struct config_item *item,
+					    char *page)
+{
+	return sprintf(page, "%d\n",
+		       to_pci_epf_group(item)->epf->msix_interrupts);
+}
+
 PCI_EPF_HEADER_R(vendorid)
 PCI_EPF_HEADER_W_u16(vendorid)
 

@@ -327,6 +349,7 @@ CONFIGFS_ATTR(pci_epf_, subsys_vendor_id);
 CONFIGFS_ATTR(pci_epf_, subsys_id);
 CONFIGFS_ATTR(pci_epf_, interrupt_pin);
 CONFIGFS_ATTR(pci_epf_, msi_interrupts);
+CONFIGFS_ATTR(pci_epf_, msix_interrupts);
 
 static struct configfs_attribute *pci_epf_attrs[] = {
 	&pci_epf_attr_vendorid,

@@ -340,6 +363,7 @@ static struct configfs_attribute *pci_epf_attrs[] = {
 	&pci_epf_attr_subsys_id,
 	&pci_epf_attr_interrupt_pin,
 	&pci_epf_attr_msi_interrupts,
+	&pci_epf_attr_msix_interrupts,
 	NULL,
 };
 

drivers/pci/endpoint/pci-epc-core.c +19 -0

index b0ee427..294a383 100644
@@ -218,6 +218,63 @@ int pci_epc_set_msi(struct pci_epc *epc, u8 func_no, u8 interrupts)
 EXPORT_SYMBOL_GPL(pci_epc_set_msi);
 
 /**
+ * pci_epc_get_msix() - get the number of MSI-X interrupt numbers allocated
+ * @epc: the EPC device to which MSI-X interrupts was requested
+ * @func_no: the endpoint function number in the EPC device
+ *
+ * Invoke to get the number of MSI-X interrupts allocated by the RC
+ */
+int pci_epc_get_msix(struct pci_epc *epc, u8 func_no)
+{
+	int interrupt;
+	unsigned long flags;
+
+	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions)
+		return 0;
+
+	if (!epc->ops->get_msix)
+		return 0;
+
+	spin_lock_irqsave(&epc->lock, flags);
+	interrupt = epc->ops->get_msix(epc, func_no);

drivers/pci/endpoint/pci-epc-core.c int pci_epc_set_msi(struct pci_epc *epc, u8 func_no, u8 interrupts) · line 245

Quoted 6 lines

drivers/pci/endpoint/pci-epc-core.c +6 -0

+	spin_unlock_irqrestore(&epc->lock, flags);
+
+	if (interrupt < 0)
+		return 0;
+
+	return interrupt++;

return interrupt + 1?

Quoted 59 lines

drivers/pci/endpoint/pci-epc-core.c +32 -0

+}
+EXPORT_SYMBOL_GPL(pci_epc_get_msix);
+
+/**
+ * pci_epc_set_msix() - set the number of MSI-X interrupt numbers required
+ * @epc: the EPC device on which MSI-X has to be configured
+ * @func_no: the endpoint function number in the EPC device
+ * @interrupts: number of MSI-X interrupts required by the EPF
+ *
+ * Invoke to set the required number of MSI-X interrupts.
+ */
+int pci_epc_set_msix(struct pci_epc *epc, u8 func_no, u16 interrupts)
+{
+	int ret;
+	unsigned long flags;
+
+	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions ||
+	    interrupts < 1 || interrupts > 2048)
+		return -EINVAL;
+
+	if (!epc->ops->set_msix)
+		return 0;
+
+	spin_lock_irqsave(&epc->lock, flags);
+	ret = epc->ops->set_msix(epc, func_no, interrupts - 1);
+	spin_unlock_irqrestore(&epc->lock, flags);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(pci_epc_set_msix);
+
+/**
  * pci_epc_unmap_addr() - unmap CPU address from PCI address
  * @epc: the EPC device on which address is allocated
  * @func_no: the endpoint function number in the EPC device

include/linux/pci-epc.h +5 -0

index af657ca..32e8961 100644
@@ -17,6 +17,7 @@ enum pci_epc_irq_type {
 	PCI_EPC_IRQ_UNKNOWN,
 	PCI_EPC_IRQ_LEGACY,
 	PCI_EPC_IRQ_MSI,
+	PCI_EPC_IRQ_MSIX,
 };
 
 /**

@@ -30,6 +31,10 @@ enum pci_epc_irq_type {
  *	     capability register
  * @get_msi: ops to get the number of MSI interrupts allocated by the RC from
  *	     the MSI capability register
+ * @set_msix: ops to set the requested number of MSI-X interrupts in the
+ *	     MSI-X capability register
+ * @get_msix: ops to get the number of MSI-X interrupts allocated by the RC
+ *	     from the MSI-X capability register
  * @raise_irq: ops to raise a legacy or MSI interrupt
  * @start: ops to start the PCI link
  * @stop: ops to stop the PCI link

@@ -48,8 +53,10 @@ struct pci_epc_ops {

include/linux/pci-epc.h struct pci_epc_ops { · line 57

Quoted 5 lines

include/linux/pci-epc.h +2 -0

 			      phys_addr_t addr);
 	int	(*set_msi)(struct pci_epc *epc, u8 func_no, u8 interrupts);
 	int	(*get_msi)(struct pci_epc *epc, u8 func_no);
+	int	(*set_msix)(struct pci_epc *epc, u8 func_no, u16 interrupts);
+	int	(*get_msix)(struct pci_epc *epc, u8 func_no);

The 1st patch in the series is already using get_msix, set_msix. This patch should precede the 1st patch.

Thanks
Kishon

Adds the MSI-X support and updates driver documentation accordingly.

Changes the driver parameter in order to allow the interruption type selection.

Signed-off-by Gustavo Pimentel <[email protected]>

---
 Documentation/misc-devices/pci-endpoint-test.txt |   3 +
 drivers/misc/pci_endpoint_test.c                 | 102 +++++++++++++++++------
 2 files changed, 79 insertions(+), 26 deletions(-)

Documentation/misc-devices/pci-endpoint-test.txt +3 -0

index 4ebc359..fdfa0f6 100644
@@ -10,6 +10,7 @@ The PCI driver for the test device performs the following tests
 	*) verifying addresses programmed in BAR
 	*) raise legacy IRQ
 	*) raise MSI IRQ
+	*) raise MSI-X IRQ
 	*) read data
 	*) write data
 	*) copy data

@@ -25,6 +26,8 @@ ioctl
  PCITEST_LEGACY_IRQ: Tests legacy IRQ
  PCITEST_MSI: Tests message signalled interrupts. The MSI number
 	      to be tested should be passed as argument.
+ PCITEST_MSIX: Tests message signalled interrupts. The MSI-X number
+	      to be tested should be passed as argument.
  PCITEST_WRITE: Perform write tests. The size of the buffer should be passed
 		as argument.
  PCITEST_READ: Perform read tests. The size of the buffer should be passed

drivers/misc/pci_endpoint_test.c +0 -1

index 37db0fc..a7d9354 100644
@@ -42,11 +42,16 @@
 #define PCI_ENDPOINT_TEST_COMMAND	0x4
 #define COMMAND_RAISE_LEGACY_IRQ	BIT(0)
 #define COMMAND_RAISE_MSI_IRQ		BIT(1)
-#define MSI_NUMBER_SHIFT		2

drivers/misc/pci_endpoint_test.c +19 -8

+static int irq_type = IRQ_TYPE_MSIX;
+module_param(irq_type, int, 0444);
+MODULE_PARM_DESC(irq_type, "IRQ mode selection in pci_endpoint_test (0 - Legacy, 1 - MSI, 2 - MSI-X)");
 
 enum pci_barno {
 	BAR_0,

@@ -103,7 +108,7 @@ struct pci_endpoint_test {
 struct pci_endpoint_test_data {
 	enum pci_barno test_reg_bar;
 	size_t alignment;
-	bool no_msi;
+	int irq_type;
 };
 
 static inline u32 pci_endpoint_test_readl(struct pci_endpoint_test *test,

@@ -177,10 +182,10 @@ static bool pci_endpoint_test_bar(struct pci_endpoint_test *test,
 
 static bool pci_endpoint_test_legacy_irq(struct pci_endpoint_test *test)
 {
-	u32 val;
+	u32 val = COMMAND_RAISE_LEGACY_IRQ;
 
-	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
-				 COMMAND_RAISE_LEGACY_IRQ);
+	val |= (IRQ_TYPE_LEGACY << IRQ_TYPE_SHIFT);
+	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND, val);
 	val = wait_for_completion_timeout(&test->irq_raised,
 					  msecs_to_jiffies(1000));
 	if (!val)

@@ -192,12 +197,12 @@ static bool pci_endpoint_test_legacy_irq(struct pci_endpoint_test *test)
 static bool pci_endpoint_test_msi_irq(struct pci_endpoint_test *test,
 				      u8 msi_num)
 {
-	u32 val;
+	u32 val = COMMAND_RAISE_MSI_IRQ;
 	struct pci_dev *pdev = test->pdev;
 
-	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
-				 msi_num << MSI_NUMBER_SHIFT |
-				 COMMAND_RAISE_MSI_IRQ);
+	val |= (msi_num << MSI_NUMBER_SHIFT);
+	val |= (IRQ_TYPE_MSI << IRQ_TYPE_SHIFT);
+	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND, val);
 	val = wait_for_completion_timeout(&test->irq_raised,
 					  msecs_to_jiffies(1000));
 	if (!val)

@@ -209,6 +214,26 @@ static bool pci_endpoint_test_msi_irq(struct pci_endpoint_test *test,
 	return false;
 }
 
+static bool pci_endpoint_test_msix_irq(struct pci_endpoint_test *test,
+				       u16 msix_num)
+{
+	u32 val = COMMAND_RAISE_MSIX_IRQ;
+	struct pci_dev *pdev = test->pdev;
+
+	val |= (msix_num << MSI_NUMBER_SHIFT);
+	val |= (IRQ_TYPE_MSIX << IRQ_TYPE_SHIFT);

Hi,

Quoted 46 lines · Gustavo Pimentel

Adds the MSI-X support and updates driver documentation accordingly.

Changes the driver parameter in order to allow the interruption type selection.

Signed-off-by Gustavo Pimentel <[email protected]>

---
 Documentation/misc-devices/pci-endpoint-test.txt |   3 +
 drivers/misc/pci_endpoint_test.c                 | 102 +++++++++++++++++------
 2 files changed, 79 insertions(+), 26 deletions(-)

Documentation/misc-devices/pci-endpoint-test.txt +3 -0

index 4ebc359..fdfa0f6 100644
@@ -10,6 +10,7 @@ The PCI driver for the test device performs the following tests
 	*) verifying addresses programmed in BAR
 	*) raise legacy IRQ
 	*) raise MSI IRQ
+	*) raise MSI-X IRQ
 	*) read data
 	*) write data
 	*) copy data

@@ -25,6 +26,8 @@ ioctl
  PCITEST_LEGACY_IRQ: Tests legacy IRQ
  PCITEST_MSI: Tests message signalled interrupts. The MSI number
 	      to be tested should be passed as argument.
+ PCITEST_MSIX: Tests message signalled interrupts. The MSI-X number
+	      to be tested should be passed as argument.
  PCITEST_WRITE: Perform write tests. The size of the buffer should be passed
 		as argument.
  PCITEST_READ: Perform read tests. The size of the buffer should be passed

drivers/misc/pci_endpoint_test.c +1 -5

index 37db0fc..a7d9354 100644
@@ -42,11 +42,16 @@
 #define PCI_ENDPOINT_TEST_COMMAND	0x4
 #define COMMAND_RAISE_LEGACY_IRQ	BIT(0)
 #define COMMAND_RAISE_MSI_IRQ		BIT(1)
-#define MSI_NUMBER_SHIFT		2
-/* 6 bits for MSI number */
-#define COMMAND_READ                    BIT(8)
-#define COMMAND_WRITE                   BIT(9)
-#define COMMAND_COPY                    BIT(10)
+#define COMMAND_RAISE_MSIX_IRQ		BIT(2)

drivers/misc/pci_endpoint_test.c line 50

Quoted 5 lines

drivers/misc/pci_endpoint_test.c +5 -0

+#define IRQ_TYPE_SHIFT			3
+#define IRQ_TYPE_LEGACY			0
+#define IRQ_TYPE_MSI			1
+#define IRQ_TYPE_MSIX			2
+#define MSI_NUMBER_SHIFT		5

Now that you are anyways fixing this, add a new register entry for MSI numbers. Let's not keep COMMAND and MSI's together.

drivers/misc/pci_endpoint_test.c line 54

Quoted 4 lines

drivers/misc/pci_endpoint_test.c +4 -0

+/* 12 bits for MSI number */
+#define COMMAND_READ                    BIT(17)
+#define COMMAND_WRITE                   BIT(18)
+#define COMMAND_COPY                    BIT(19)

This change should be done along with the pci-epf-test in a single patch.

Quoted 7 lines

drivers/misc/pci_endpoint_test.c +0 -0

 #define PCI_ENDPOINT_TEST_STATUS	0x8
 #define STATUS_READ_SUCCESS             BIT(0)

@@ -73,9 +78,9 @@ static DEFINE_IDA(pci_endpoint_test_ida);
 #define to_endpoint_test(priv) container_of((priv), struct pci_endpoint_test, \
 					    miscdev)
 

drivers/misc/pci_endpoint_test.c static DEFINE_IDA(pci_endpoint_test_ida); · line 78

Quoted 3 lines

drivers/misc/pci_endpoint_test.c +0 -3

-static bool no_msi;
-module_param(no_msi, bool, 0444);
-MODULE_PARM_DESC(no_msi, "Disable MSI interrupt in pci_endpoint_test");

Let's not remove this just to make sure existing users doesn't get affected.

Quoted 62 lines

drivers/misc/pci_endpoint_test.c +23 -8

+static int irq_type = IRQ_TYPE_MSIX;
+module_param(irq_type, int, 0444);
+MODULE_PARM_DESC(irq_type, "IRQ mode selection in pci_endpoint_test (0 - Legacy, 1 - MSI, 2 - MSI-X)");
 
 enum pci_barno {
 	BAR_0,

@@ -103,7 +108,7 @@ struct pci_endpoint_test {
 struct pci_endpoint_test_data {
 	enum pci_barno test_reg_bar;
 	size_t alignment;
-	bool no_msi;
+	int irq_type;
 };
 
 static inline u32 pci_endpoint_test_readl(struct pci_endpoint_test *test,

@@ -177,10 +182,10 @@ static bool pci_endpoint_test_bar(struct pci_endpoint_test *test,
 
 static bool pci_endpoint_test_legacy_irq(struct pci_endpoint_test *test)
 {
-	u32 val;
+	u32 val = COMMAND_RAISE_LEGACY_IRQ;
 
-	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
-				 COMMAND_RAISE_LEGACY_IRQ);
+	val |= (IRQ_TYPE_LEGACY << IRQ_TYPE_SHIFT);
+	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND, val);
 	val = wait_for_completion_timeout(&test->irq_raised,
 					  msecs_to_jiffies(1000));
 	if (!val)

@@ -192,12 +197,12 @@ static bool pci_endpoint_test_legacy_irq(struct pci_endpoint_test *test)
 static bool pci_endpoint_test_msi_irq(struct pci_endpoint_test *test,
 				      u8 msi_num)
 {
-	u32 val;
+	u32 val = COMMAND_RAISE_MSI_IRQ;
 	struct pci_dev *pdev = test->pdev;
 
-	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
-				 msi_num << MSI_NUMBER_SHIFT |
-				 COMMAND_RAISE_MSI_IRQ);
+	val |= (msi_num << MSI_NUMBER_SHIFT);
+	val |= (IRQ_TYPE_MSI << IRQ_TYPE_SHIFT);
+	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND, val);
 	val = wait_for_completion_timeout(&test->irq_raised,
 					  msecs_to_jiffies(1000));
 	if (!val)

@@ -209,6 +214,26 @@ static bool pci_endpoint_test_msi_irq(struct pci_endpoint_test *test,
 	return false;
 }
 
+static bool pci_endpoint_test_msix_irq(struct pci_endpoint_test *test,
+				       u16 msix_num)
+{
+	u32 val = COMMAND_RAISE_MSIX_IRQ;
+	struct pci_dev *pdev = test->pdev;
+
+	val |= (msix_num << MSI_NUMBER_SHIFT);
+	val |= (IRQ_TYPE_MSIX << IRQ_TYPE_SHIFT);
+	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND, val);
+	val = wait_for_completion_timeout(&test->irq_raised,
+					  msecs_to_jiffies(1000));
+	if (!val)

drivers/misc/pci_endpoint_test.c static bool pci_endpoint_test_msi_irq(struct pci_endpoint_test *test, · line 234

Quoted 6 lines

drivers/misc/pci_endpoint_test.c +6 -0

+		return false;
+
+	if (test->last_irq - pdev->irq == msix_num - 1)
+		return true;
+
+	return false;

I think you can have a single function for msix_irq and msi_irq.

Thanks
Kishon

Hi Kishon,

Quoted 49 lines · Kishon Vijay Abraham I

Hi,

Quoted 46 lines · Gustavo Pimentel

Adds the MSI-X support and updates driver documentation accordingly.

Changes the driver parameter in order to allow the interruption type selection.

Signed-off-by Gustavo Pimentel <[email protected]>

---
 Documentation/misc-devices/pci-endpoint-test.txt |   3 +
 drivers/misc/pci_endpoint_test.c                 | 102 +++++++++++++++++------
 2 files changed, 79 insertions(+), 26 deletions(-)

Documentation/misc-devices/pci-endpoint-test.txt +3 -0

index 4ebc359..fdfa0f6 100644
@@ -10,6 +10,7 @@ The PCI driver for the test device performs the following tests
 	*) verifying addresses programmed in BAR
 	*) raise legacy IRQ
 	*) raise MSI IRQ
+	*) raise MSI-X IRQ
 	*) read data
 	*) write data
 	*) copy data

@@ -25,6 +26,8 @@ ioctl
  PCITEST_LEGACY_IRQ: Tests legacy IRQ
  PCITEST_MSI: Tests message signalled interrupts. The MSI number
 	      to be tested should be passed as argument.
+ PCITEST_MSIX: Tests message signalled interrupts. The MSI-X number
+	      to be tested should be passed as argument.
  PCITEST_WRITE: Perform write tests. The size of the buffer should be passed
 		as argument.
  PCITEST_READ: Perform read tests. The size of the buffer should be passed

drivers/misc/pci_endpoint_test.c +1 -5

index 37db0fc..a7d9354 100644
@@ -42,11 +42,16 @@
 #define PCI_ENDPOINT_TEST_COMMAND	0x4
 #define COMMAND_RAISE_LEGACY_IRQ	BIT(0)
 #define COMMAND_RAISE_MSI_IRQ		BIT(1)
-#define MSI_NUMBER_SHIFT		2
-/* 6 bits for MSI number */
-#define COMMAND_READ                    BIT(8)
-#define COMMAND_WRITE                   BIT(9)
-#define COMMAND_COPY                    BIT(10)
+#define COMMAND_RAISE_MSIX_IRQ		BIT(2)

Quoted 8 lines

drivers/misc/pci_endpoint_test.c line 50

Quoted 5 lines

drivers/misc/pci_endpoint_test.c +5 -0

+#define IRQ_TYPE_SHIFT			3
+#define IRQ_TYPE_LEGACY			0
+#define IRQ_TYPE_MSI			1
+#define IRQ_TYPE_MSIX			2
+#define MSI_NUMBER_SHIFT		5

Now that you are anyways fixing this, add a new register entry for MSI numbers. Let's not keep COMMAND and MSI's together.

What you suggest?

Quoted 7 lines

drivers/misc/pci_endpoint_test.c line 54

Quoted 5 lines

drivers/misc/pci_endpoint_test.c +4 -0

+/* 12 bits for MSI number */
+#define COMMAND_READ                    BIT(17)
+#define COMMAND_WRITE                   BIT(18)
+#define COMMAND_COPY                    BIT(19)

This change should be done along with the pci-epf-test in a single patch.

To be clear, you're saying is this patch should be just be squashed into the patch number 8 [1], because there is a lot of dependencies namely the defines, that is used on the alter functions.

[1] -> https://patchwork.ozlabs.org/patch/896841/

Quoted 8 lines

Quoted 8 lines

drivers/misc/pci_endpoint_test.c +0 -0

 #define PCI_ENDPOINT_TEST_STATUS	0x8
 #define STATUS_READ_SUCCESS             BIT(0)

@@ -73,9 +78,9 @@ static DEFINE_IDA(pci_endpoint_test_ida);
 #define to_endpoint_test(priv) container_of((priv), struct pci_endpoint_test, \
 					    miscdev)
 

Quoted 5 lines

drivers/misc/pci_endpoint_test.c static DEFINE_IDA(pci_endpoint_test_ida); · line 78

Quoted 3 lines

drivers/misc/pci_endpoint_test.c +0 -3

-static bool no_msi;
-module_param(no_msi, bool, 0444);
-MODULE_PARM_DESC(no_msi, "Disable MSI interrupt in pci_endpoint_test");

Let's not remove this just to make sure existing users doesn't get affected.

Hum, by making an internal conversion? Like this
no_msi = false <=> irq_type = 1
no_msi = true <=> irq_type = 0

Hi,

Quoted 61 lines · Gustavo Pimentel

Hi Kishon,

Quoted 49 lines · Kishon Vijay Abraham I

Hi,

Quoted 46 lines · Gustavo Pimentel

Adds the MSI-X support and updates driver documentation accordingly.

Changes the driver parameter in order to allow the interruption type selection.

Signed-off-by Gustavo Pimentel <[email protected]>

---
 Documentation/misc-devices/pci-endpoint-test.txt |   3 +
 drivers/misc/pci_endpoint_test.c                 | 102 +++++++++++++++++------
 2 files changed, 79 insertions(+), 26 deletions(-)

Documentation/misc-devices/pci-endpoint-test.txt +3 -0

index 4ebc359..fdfa0f6 100644
@@ -10,6 +10,7 @@ The PCI driver for the test device performs the following tests
 	*) verifying addresses programmed in BAR
 	*) raise legacy IRQ
 	*) raise MSI IRQ
+	*) raise MSI-X IRQ
 	*) read data
 	*) write data
 	*) copy data

@@ -25,6 +26,8 @@ ioctl
  PCITEST_LEGACY_IRQ: Tests legacy IRQ
  PCITEST_MSI: Tests message signalled interrupts. The MSI number
 	      to be tested should be passed as argument.
+ PCITEST_MSIX: Tests message signalled interrupts. The MSI-X number
+	      to be tested should be passed as argument.
  PCITEST_WRITE: Perform write tests. The size of the buffer should be passed
 		as argument.
  PCITEST_READ: Perform read tests. The size of the buffer should be passed

drivers/misc/pci_endpoint_test.c +1 -5

index 37db0fc..a7d9354 100644
@@ -42,11 +42,16 @@
 #define PCI_ENDPOINT_TEST_COMMAND	0x4
 #define COMMAND_RAISE_LEGACY_IRQ	BIT(0)
 #define COMMAND_RAISE_MSI_IRQ		BIT(1)
-#define MSI_NUMBER_SHIFT		2
-/* 6 bits for MSI number */
-#define COMMAND_READ                    BIT(8)
-#define COMMAND_WRITE                   BIT(9)
-#define COMMAND_COPY                    BIT(10)
+#define COMMAND_RAISE_MSIX_IRQ		BIT(2)

Quoted 8 lines

drivers/misc/pci_endpoint_test.c line 50

Quoted 5 lines

drivers/misc/pci_endpoint_test.c +5 -0

+#define IRQ_TYPE_SHIFT			3
+#define IRQ_TYPE_LEGACY			0
+#define IRQ_TYPE_MSI			1
+#define IRQ_TYPE_MSIX			2
+#define MSI_NUMBER_SHIFT		5

Now that you are anyways fixing this, add a new register entry for MSI numbers. Let's not keep COMMAND and MSI's together.

Quoted 1 line

What you suggest?

#define PCI_ENDPOINT_TEST_COMMAND 0x4
#define COMMAND_RAISE_LEGACY_IRQ BIT(0)
#define COMMAND_RAISE_MSI_IRQ BIT(1)
#define COMMAND_RAISE_MSIX_IRQ BIT(2)

#define COMMAND_READ                    BIT(3)
#define COMMAND_WRITE                   BIT(4)
#define COMMAND_COPY                    BIT(5)

#define PCI_ENDPOINT_TEST_STATUS 0x8

#define STATUS_READ_SUCCESS             BIT(0)
#define STATUS_READ_FAIL                BIT(1)
#define STATUS_WRITE_SUCCESS            BIT(2)
#define STATUS_WRITE_FAIL               BIT(3)
#define STATUS_COPY_SUCCESS             BIT(4)
#define STATUS_COPY_FAIL                BIT(5)
#define STATUS_IRQ_RAISED               BIT(6)
#define STATUS_SRC_ADDR_INVALID         BIT(7)
#define STATUS_DST_ADDR_INVALID         BIT(8)

#define PCI_ENDPOINT_TEST_LOWER_SRC_ADDR 0xc
#define PCI_ENDPOINT_TEST_UPPER_SRC_ADDR 0x10

#define PCI_ENDPOINT_TEST_LOWER_DST_ADDR 0x14
#define PCI_ENDPOINT_TEST_UPPER_DST_ADDR 0x18

#define PCI_ENDPOINT_TEST_SIZE 0x1c
#define PCI_ENDPOINT_TEST_CHECKSUM 0x20

#define PCI_ENDPOINT_TEST_MSI_NUMBER 0x24

We should try not to modify either the existing register offsets or the bit fields within these registers in the future as EP and RC will be running on different systems and it is possible one of them might not have the updated kernel.

Quoted 8 lines

Quoted 7 lines

drivers/misc/pci_endpoint_test.c line 54

Quoted 5 lines

drivers/misc/pci_endpoint_test.c +4 -0

+/* 12 bits for MSI number */
+#define COMMAND_READ                    BIT(17)
+#define COMMAND_WRITE                   BIT(18)
+#define COMMAND_COPY                    BIT(19)

This change should be done along with the pci-epf-test in a single patch.

Quoted 5 lines

To be clear, you're saying is this patch should be just be squashed into the patch number 8 [1], because there is a lot of dependencies namely the defines, that is used on the alter functions.

[1] -> https://urldefense.proofpoint.com/v2/url?u=https-3A__patchwork.ozlabs.org_patch_896841_&d=DwIC-g&c=DPL6_X_6JkXFx7AXWqB0tg&r=bkWxpLoW-f-E3EdiDCCa0_h0PicsViasSlvIpzZvPxs&m=8urVwHCybXa1XMxsEbwHZAzzaEI_HJGXqmWgXpXb9TY&s=MRVr2YPY2Bk_WNFOxBfU4FGrFReTKdPhfzNDLiVxDbs&e=

yeah. We have to make sure git bisect doesn't break functionality.

Quoted 14 lines

Quoted 13 lines

drivers/misc/pci_endpoint_test.c static DEFINE_IDA(pci_endpoint_test_ida); · line 78

Quoted 11 lines

drivers/misc/pci_endpoint_test.c +0 -3

 #define PCI_ENDPOINT_TEST_STATUS	0x8
 #define STATUS_READ_SUCCESS             BIT(0)

@@ -73,9 +78,9 @@ static DEFINE_IDA(pci_endpoint_test_ida);
 #define to_endpoint_test(priv) container_of((priv), struct pci_endpoint_test, \
 					    miscdev)
 
-static bool no_msi;
-module_param(no_msi, bool, 0444);
-MODULE_PARM_DESC(no_msi, "Disable MSI interrupt in pci_endpoint_test");

Let's not remove this just to make sure existing users doesn't get affected.

Quoted 3 lines

Hum, by making an internal conversion? Like this
no_msi = false <=> irq_type = 1
no_msi = true <=> irq_type = 0

yeah..

Thanks
Kishon

Hi Kishon,

Quoted 88 lines · Kishon Vijay Abraham I

Hi,

Quoted 61 lines · Gustavo Pimentel

Hi Kishon,

Quoted 49 lines · Kishon Vijay Abraham I

Hi,

Quoted 46 lines · Gustavo Pimentel

Adds the MSI-X support and updates driver documentation accordingly.

Changes the driver parameter in order to allow the interruption type selection.

Signed-off-by Gustavo Pimentel <[email protected]>

---
 Documentation/misc-devices/pci-endpoint-test.txt |   3 +
 drivers/misc/pci_endpoint_test.c                 | 102 +++++++++++++++++------
 2 files changed, 79 insertions(+), 26 deletions(-)

Documentation/misc-devices/pci-endpoint-test.txt +3 -0

index 4ebc359..fdfa0f6 100644
@@ -10,6 +10,7 @@ The PCI driver for the test device performs the following tests
 	*) verifying addresses programmed in BAR
 	*) raise legacy IRQ
 	*) raise MSI IRQ
+	*) raise MSI-X IRQ
 	*) read data
 	*) write data
 	*) copy data

@@ -25,6 +26,8 @@ ioctl
  PCITEST_LEGACY_IRQ: Tests legacy IRQ
  PCITEST_MSI: Tests message signalled interrupts. The MSI number
 	      to be tested should be passed as argument.
+ PCITEST_MSIX: Tests message signalled interrupts. The MSI-X number
+	      to be tested should be passed as argument.
  PCITEST_WRITE: Perform write tests. The size of the buffer should be passed
 		as argument.
  PCITEST_READ: Perform read tests. The size of the buffer should be passed

drivers/misc/pci_endpoint_test.c +1 -5

index 37db0fc..a7d9354 100644
@@ -42,11 +42,16 @@
 #define PCI_ENDPOINT_TEST_COMMAND	0x4
 #define COMMAND_RAISE_LEGACY_IRQ	BIT(0)
 #define COMMAND_RAISE_MSI_IRQ		BIT(1)
-#define MSI_NUMBER_SHIFT		2
-/* 6 bits for MSI number */
-#define COMMAND_READ                    BIT(8)
-#define COMMAND_WRITE                   BIT(9)
-#define COMMAND_COPY                    BIT(10)
+#define COMMAND_RAISE_MSIX_IRQ		BIT(2)

Quoted 8 lines

drivers/misc/pci_endpoint_test.c line 50

Quoted 5 lines

drivers/misc/pci_endpoint_test.c +5 -0

+#define IRQ_TYPE_SHIFT			3
+#define IRQ_TYPE_LEGACY			0
+#define IRQ_TYPE_MSI			1
+#define IRQ_TYPE_MSIX			2
+#define MSI_NUMBER_SHIFT		5

Now that you are anyways fixing this, add a new register entry for MSI numbers. Let's not keep COMMAND and MSI's together.

Quoted 1 line

What you suggest?

#define PCI_ENDPOINT_TEST_COMMAND 0x4
#define COMMAND_RAISE_LEGACY_IRQ BIT(0)
#define COMMAND_RAISE_MSI_IRQ BIT(1)
#define COMMAND_RAISE_MSIX_IRQ BIT(2)

#define COMMAND_READ                    BIT(3)
#define COMMAND_WRITE                   BIT(4)
#define COMMAND_COPY                    BIT(5)

#define PCI_ENDPOINT_TEST_STATUS 0x8

#define STATUS_READ_SUCCESS             BIT(0)
#define STATUS_READ_FAIL                BIT(1)
#define STATUS_WRITE_SUCCESS            BIT(2)
#define STATUS_WRITE_FAIL               BIT(3)
#define STATUS_COPY_SUCCESS             BIT(4)
#define STATUS_COPY_FAIL                BIT(5)
#define STATUS_IRQ_RAISED               BIT(6)
#define STATUS_SRC_ADDR_INVALID         BIT(7)
#define STATUS_DST_ADDR_INVALID         BIT(8)

#define PCI_ENDPOINT_TEST_LOWER_SRC_ADDR 0xc
#define PCI_ENDPOINT_TEST_UPPER_SRC_ADDR 0x10

Quoted 7 lines

#define PCI_ENDPOINT_TEST_LOWER_DST_ADDR 0x14
#define PCI_ENDPOINT_TEST_UPPER_DST_ADDR 0x18

#define PCI_ENDPOINT_TEST_SIZE 0x1c
#define PCI_ENDPOINT_TEST_CHECKSUM 0x20

#define PCI_ENDPOINT_TEST_MSI_NUMBER 0x24

Ok. I will do it.

Quoted 5 lines

We should try not to modify either the existing register offsets or the bit fields within these registers in the future as EP and RC will be running on different systems and it is possible one of them might not have the updated kernel.

I totally agree.

Quoted 8 lines

Quoted 8 lines

Quoted 7 lines

drivers/misc/pci_endpoint_test.c line 54

Quoted 5 lines

drivers/misc/pci_endpoint_test.c +4 -0

+/* 12 bits for MSI number */
+#define COMMAND_READ                    BIT(17)
+#define COMMAND_WRITE                   BIT(18)
+#define COMMAND_COPY                    BIT(19)

This change should be done along with the pci-epf-test in a single patch.

Quoted 7 lines

Quoted 5 lines

To be clear, you're saying is this patch should be just be squashed into the patch number 8 [1], because there is a lot of dependencies namely the defines, that is used on the alter functions.

[1] -> https://urldefense.proofpoint.com/v2/url?u=https-3A__patchwork.ozlabs.org_patch_896841_&d=DwIC-g&c=DPL6_X_6JkXFx7AXWqB0tg&r=bkWxpLoW-f-E3EdiDCCa0_h0PicsViasSlvIpzZvPxs&m=8urVwHCybXa1XMxsEbwHZAzzaEI_HJGXqmWgXpXb9TY&s=MRVr2YPY2Bk_WNFOxBfU4FGrFReTKdPhfzNDLiVxDbs&e=

yeah. We have to make sure git bisect doesn't break functionality.

Ok, it'll be squashed.

Quoted 14 lines

Quoted 14 lines

Quoted 13 lines

drivers/misc/pci_endpoint_test.c static DEFINE_IDA(pci_endpoint_test_ida); · line 78

Quoted 11 lines

drivers/misc/pci_endpoint_test.c +0 -3

 #define PCI_ENDPOINT_TEST_STATUS	0x8
 #define STATUS_READ_SUCCESS             BIT(0)

@@ -73,9 +78,9 @@ static DEFINE_IDA(pci_endpoint_test_ida);
 #define to_endpoint_test(priv) container_of((priv), struct pci_endpoint_test, \
 					    miscdev)
 
-static bool no_msi;
-module_param(no_msi, bool, 0444);
-MODULE_PARM_DESC(no_msi, "Disable MSI interrupt in pci_endpoint_test");

Let's not remove this just to make sure existing users doesn't get affected.

Quoted 4 lines

Quoted 3 lines

Hum, by making an internal conversion? Like this
no_msi = false <=> irq_type = 1
no_msi = true <=> irq_type = 0

Disregard previous comment, it doesn't make sense. I don't know where my head was.

It will be like this on probe:

if (no_msi)

	irq_type = IRQ_TYPE_LEGACY;

However since we are breaking the compatibility on terms of MSI/MSI-X bits/registers (discussion on the top), it makes sense to keep the compatibility on this parameter?

Hi,

Quoted 152 lines · Gustavo Pimentel

Hi Kishon,

Quoted 88 lines · Kishon Vijay Abraham I

Hi,

Quoted 61 lines · Gustavo Pimentel

Hi Kishon,

Quoted 49 lines · Kishon Vijay Abraham I

Hi,

Quoted 46 lines · Gustavo Pimentel

Adds the MSI-X support and updates driver documentation accordingly.

Changes the driver parameter in order to allow the interruption type selection.

Signed-off-by Gustavo Pimentel <[email protected]>

---
 Documentation/misc-devices/pci-endpoint-test.txt |   3 +
 drivers/misc/pci_endpoint_test.c                 | 102 +++++++++++++++++------
 2 files changed, 79 insertions(+), 26 deletions(-)

Documentation/misc-devices/pci-endpoint-test.txt +3 -0

index 4ebc359..fdfa0f6 100644
@@ -10,6 +10,7 @@ The PCI driver for the test device performs the following tests
 	*) verifying addresses programmed in BAR
 	*) raise legacy IRQ
 	*) raise MSI IRQ
+	*) raise MSI-X IRQ
 	*) read data
 	*) write data
 	*) copy data

@@ -25,6 +26,8 @@ ioctl
  PCITEST_LEGACY_IRQ: Tests legacy IRQ
  PCITEST_MSI: Tests message signalled interrupts. The MSI number
 	      to be tested should be passed as argument.
+ PCITEST_MSIX: Tests message signalled interrupts. The MSI-X number
+	      to be tested should be passed as argument.
  PCITEST_WRITE: Perform write tests. The size of the buffer should be passed
 		as argument.
  PCITEST_READ: Perform read tests. The size of the buffer should be passed

drivers/misc/pci_endpoint_test.c +1 -5

index 37db0fc..a7d9354 100644
@@ -42,11 +42,16 @@
 #define PCI_ENDPOINT_TEST_COMMAND	0x4
 #define COMMAND_RAISE_LEGACY_IRQ	BIT(0)
 #define COMMAND_RAISE_MSI_IRQ		BIT(1)
-#define MSI_NUMBER_SHIFT		2
-/* 6 bits for MSI number */
-#define COMMAND_READ                    BIT(8)
-#define COMMAND_WRITE                   BIT(9)
-#define COMMAND_COPY                    BIT(10)
+#define COMMAND_RAISE_MSIX_IRQ		BIT(2)

Quoted 8 lines

drivers/misc/pci_endpoint_test.c line 50

Quoted 5 lines

drivers/misc/pci_endpoint_test.c +5 -0

+#define IRQ_TYPE_SHIFT			3
+#define IRQ_TYPE_LEGACY			0
+#define IRQ_TYPE_MSI			1
+#define IRQ_TYPE_MSIX			2
+#define MSI_NUMBER_SHIFT		5

Now that you are anyways fixing this, add a new register entry for MSI numbers. Let's not keep COMMAND and MSI's together.

Quoted 1 line

What you suggest?

#define PCI_ENDPOINT_TEST_COMMAND 0x4
#define COMMAND_RAISE_LEGACY_IRQ BIT(0)
#define COMMAND_RAISE_MSI_IRQ BIT(1)
#define COMMAND_RAISE_MSIX_IRQ BIT(2)

#define COMMAND_READ                    BIT(3)
#define COMMAND_WRITE                   BIT(4)
#define COMMAND_COPY                    BIT(5)

#define PCI_ENDPOINT_TEST_STATUS 0x8

#define STATUS_READ_SUCCESS             BIT(0)
#define STATUS_READ_FAIL                BIT(1)
#define STATUS_WRITE_SUCCESS            BIT(2)
#define STATUS_WRITE_FAIL               BIT(3)
#define STATUS_COPY_SUCCESS             BIT(4)
#define STATUS_COPY_FAIL                BIT(5)
#define STATUS_IRQ_RAISED               BIT(6)
#define STATUS_SRC_ADDR_INVALID         BIT(7)
#define STATUS_DST_ADDR_INVALID         BIT(8)

#define PCI_ENDPOINT_TEST_LOWER_SRC_ADDR 0xc
#define PCI_ENDPOINT_TEST_UPPER_SRC_ADDR 0x10

Quoted 7 lines

#define PCI_ENDPOINT_TEST_LOWER_DST_ADDR 0x14
#define PCI_ENDPOINT_TEST_UPPER_DST_ADDR 0x18

#define PCI_ENDPOINT_TEST_SIZE 0x1c
#define PCI_ENDPOINT_TEST_CHECKSUM 0x20

#define PCI_ENDPOINT_TEST_MSI_NUMBER 0x24

Ok. I will do it.

Quoted 5 lines

We should try not to modify either the existing register offsets or the bit fields within these registers in the future as EP and RC will be running on different systems and it is possible one of them might not have the updated kernel.

I totally agree.

Quoted 8 lines

Quoted 8 lines

Quoted 7 lines

drivers/misc/pci_endpoint_test.c line 54

Quoted 5 lines

drivers/misc/pci_endpoint_test.c +4 -0

+/* 12 bits for MSI number */
+#define COMMAND_READ                    BIT(17)
+#define COMMAND_WRITE                   BIT(18)
+#define COMMAND_COPY                    BIT(19)

This change should be done along with the pci-epf-test in a single patch.

Quoted 7 lines

Quoted 5 lines

To be clear, you're saying is this patch should be just be squashed into the patch number 8 [1], because there is a lot of dependencies namely the defines, that is used on the alter functions.

[1] -> https://urldefense.proofpoint.com/v2/url?u=https-3A__patchwork.ozlabs.org_patch_896841_&d=DwIC-g&c=DPL6_X_6JkXFx7AXWqB0tg&r=bkWxpLoW-f-E3EdiDCCa0_h0PicsViasSlvIpzZvPxs&m=8urVwHCybXa1XMxsEbwHZAzzaEI_HJGXqmWgXpXb9TY&s=MRVr2YPY2Bk_WNFOxBfU4FGrFReTKdPhfzNDLiVxDbs&e=

yeah. We have to make sure git bisect doesn't break functionality.

Ok, it'll be squashed.

Quoted 14 lines

Quoted 14 lines

Quoted 13 lines

drivers/misc/pci_endpoint_test.c static DEFINE_IDA(pci_endpoint_test_ida); · line 78

Quoted 11 lines

drivers/misc/pci_endpoint_test.c +0 -3

 #define PCI_ENDPOINT_TEST_STATUS	0x8
 #define STATUS_READ_SUCCESS             BIT(0)

@@ -73,9 +78,9 @@ static DEFINE_IDA(pci_endpoint_test_ida);
 #define to_endpoint_test(priv) container_of((priv), struct pci_endpoint_test, \
 					    miscdev)
 
-static bool no_msi;
-module_param(no_msi, bool, 0444);
-MODULE_PARM_DESC(no_msi, "Disable MSI interrupt in pci_endpoint_test");

Let's not remove this just to make sure existing users doesn't get affected.

Quoted 4 lines

Quoted 3 lines

Hum, by making an internal conversion? Like this
no_msi = false <=> irq_type = 1
no_msi = true <=> irq_type = 0

Disregard previous comment, it doesn't make sense. I don't know where my head was.

It will be like this on probe:

if (no_msi)

	irq_type = IRQ_TYPE_LEGACY;

Quoted 3 lines

However since we are breaking the compatibility on terms of MSI/MSI-X bits/registers (discussion on the top), it makes sense to keep the compatibility on this parameter?

This is userspace compatibility, so lets not break it.
Btw can we have a sysfs entry per device for defining irq_type. Having a sysfs entry might be helpful instead of insmod/rmmod with different irq_type values?

It will also help if a system has enumerated multiple PCI_ENDPOINT_TEST EP devices.

Thanks
Kishon

=20
Adds the MSI-X support and updates driver documentation accordingly. =20
Changes the driver parameter in order to allow the interruption type selection.
=20

Signed-off-by Gustavo Pimentel <[email protected]>

---
 Documentation/misc-devices/pci-endpoint-test.txt |   3 +

drivers/misc/pci_endpoint_test.c | 102 +++++++++++++++++=

 2 files changed, 79 insertions(+), 26 deletions(-)

=20

Documentation/misc-devices/pci-endpoint-test.txt +0 -0

index 4ebc359..fdfa0f6 100644
@@ -10,6 +10,7 @@ The PCI driver for the test device performs the

following tests

Documentation/misc-devices/pci-endpoint-test.txt +1 -0

 	*) verifying addresses programmed in BAR
 	*) raise legacy IRQ
 	*) raise MSI IRQ
+	*) raise MSI-X IRQ
 	*) read data
 	*) write data
 	*) copy data

drivers/misc/pci_endpoint_test.c +10 -5

index 37db0fc..a7d9354 100644
@@ -42,11 +42,16 @@
 #define PCI_ENDPOINT_TEST_COMMAND	0x4
 #define COMMAND_RAISE_LEGACY_IRQ	BIT(0)
 #define COMMAND_RAISE_MSI_IRQ		BIT(1)
-#define MSI_NUMBER_SHIFT		2
-/* 6 bits for MSI number */
-#define COMMAND_READ                    BIT(8)
-#define COMMAND_WRITE                   BIT(9)
-#define COMMAND_COPY                    BIT(10)
+#define COMMAND_RAISE_MSIX_IRQ		BIT(2)
+#define IRQ_TYPE_SHIFT			3
+#define IRQ_TYPE_LEGACY			0
+#define IRQ_TYPE_MSI			1
+#define IRQ_TYPE_MSIX			2
+#define MSI_NUMBER_SHIFT		5
+/* 12 bits for MSI number */
+#define COMMAND_READ                    BIT(17)
+#define COMMAND_WRITE                   BIT(18)
+#define COMMAND_COPY                    BIT(19)

=20
#define PCI_ENDPOINT_TEST_STATUS 0x8
#define STATUS_READ_SUCCESS BIT(0)

drivers/misc/pci_endpoint_test.c +0 -0

 \
 					    miscdev)

=20

drivers/misc/pci_endpoint_test.c +0 -3

-static bool no_msi;
-module_param(no_msi, bool, 0444);
-MODULE_PARM_DESC(no_msi, "Disable MSI interrupt in

pci_endpoint_test");

drivers/misc/pci_endpoint_test.c +3 -0

+static int irq_type =3D IRQ_TYPE_MSIX;
+module_param(irq_type, int, 0444);
+MODULE_PARM_DESC(irq_type, "IRQ mode selection in pci_endpoint_test

(0
+- Legacy, 1 - MSI, 2 - MSI-X)");
=20
enum pci_barno {

drivers/misc/pci_endpoint_test.c +1 -1

 	BAR_0,

@@ -103,7 +108,7 @@ struct pci_endpoint_test {  struct
 pci_endpoint_test_data {
 	enum pci_barno test_reg_bar;
 	size_t alignment;
-	bool no_msi;
+	int irq_type;
 };

=20
static inline u32 pci_endpoint_test_readl(struct pci_endpoint_test *test=

drivers/misc/pci_endpoint_test.c +0 -0

@@ -177,10 +182,10 @@ static bool pci_endpoint_test_bar(struct
 pci_endpoint_test *test,

=20
static bool pci_endpoint_test_legacy_irq(struct pci_endpoint_test *test)=

drivers/misc/pci_endpoint_test.c +1 -1

-	u32 val;
+	u32 val =3D COMMAND_RAISE_LEGACY_IRQ;

=20

drivers/misc/pci_endpoint_test.c +2 -2

-	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
-				 COMMAND_RAISE_LEGACY_IRQ);
+	val |=3D (IRQ_TYPE_LEGACY << IRQ_TYPE_SHIFT);
+	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND, val);
 	val =3D wait_for_completion_timeout(&test->irq_raised,
 					  msecs_to_jiffies(1000));
 	if (!val)

@@ -192,12 +197,12 @@ static bool pci_endpoint_test_legacy_irq(struct

pci_endpoint_test *test) static bool pci_endpoint_test_msi_irq(struct pci_endpoint_test *test,

drivers/misc/pci_endpoint_test.c +1 -1

 				      u8 msi_num)
 {
-	u32 val;
+	u32 val =3D COMMAND_RAISE_MSI_IRQ;
 	struct pci_dev *pdev =3D test->pdev;

=20

drivers/misc/pci_endpoint_test.c +3 -3

-	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
-				 msi_num << MSI_NUMBER_SHIFT |
-				 COMMAND_RAISE_MSI_IRQ);
+	val |=3D (msi_num << MSI_NUMBER_SHIFT);
+	val |=3D (IRQ_TYPE_MSI << IRQ_TYPE_SHIFT);
+	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND, val);
 	val =3D wait_for_completion_timeout(&test->irq_raised,
 					  msecs_to_jiffies(1000));
 	if (!val)

@@ -209,6 +214,26 @@ static bool pci_endpoint_test_msi_irq(struct
 pci_endpoint_test *test,
 	return false;
 }

=20

drivers/misc/pci_endpoint_test.c +6 -0

+static bool pci_endpoint_test_msix_irq(struct pci_endpoint_test *test,
+				       u16 msix_num)
+{
+	u32 val =3D COMMAND_RAISE_MSIX_IRQ;
+	struct pci_dev *pdev =3D test->pdev;
+

Hi Gustavo,

Quoted 131 lines · On 10 April 2018 18:15 Gustavo Pimentel wrote:

Adds the MSI-X support and updates driver documentation accordingly.

Changes the driver parameter in order to allow the interruption type selection.

Signed-off-by Gustavo Pimentel <[email protected]>

---
 Documentation/misc-devices/pci-endpoint-test.txt |   3 +
 drivers/misc/pci_endpoint_test.c                 | 102 +++++++++++++++++------
 2 files changed, 79 insertions(+), 26 deletions(-)

Documentation/misc-devices/pci-endpoint-test.txt +0 -0

index 4ebc359..fdfa0f6 100644
@@ -10,6 +10,7 @@ The PCI driver for the test device performs the

following tests

Documentation/misc-devices/pci-endpoint-test.txt +3 -0

 	*) verifying addresses programmed in BAR
 	*) raise legacy IRQ
 	*) raise MSI IRQ
+	*) raise MSI-X IRQ
 	*) read data
 	*) write data
 	*) copy data

@@ -25,6 +26,8 @@ ioctl
  PCITEST_LEGACY_IRQ: Tests legacy IRQ
  PCITEST_MSI: Tests message signalled interrupts. The MSI number
 	      to be tested should be passed as argument.
+ PCITEST_MSIX: Tests message signalled interrupts. The MSI-X number
+	      to be tested should be passed as argument.
  PCITEST_WRITE: Perform write tests. The size of the buffer should be passed
 		as argument.
  PCITEST_READ: Perform read tests. The size of the buffer should be passed

drivers/misc/pci_endpoint_test.c +13 -8

index 37db0fc..a7d9354 100644
@@ -42,11 +42,16 @@
 #define PCI_ENDPOINT_TEST_COMMAND	0x4
 #define COMMAND_RAISE_LEGACY_IRQ	BIT(0)
 #define COMMAND_RAISE_MSI_IRQ		BIT(1)
-#define MSI_NUMBER_SHIFT		2
-/* 6 bits for MSI number */
-#define COMMAND_READ                    BIT(8)
-#define COMMAND_WRITE                   BIT(9)
-#define COMMAND_COPY                    BIT(10)
+#define COMMAND_RAISE_MSIX_IRQ		BIT(2)
+#define IRQ_TYPE_SHIFT			3
+#define IRQ_TYPE_LEGACY			0
+#define IRQ_TYPE_MSI			1
+#define IRQ_TYPE_MSIX			2
+#define MSI_NUMBER_SHIFT		5
+/* 12 bits for MSI number */
+#define COMMAND_READ                    BIT(17)
+#define COMMAND_WRITE                   BIT(18)
+#define COMMAND_COPY                    BIT(19)
 
 #define PCI_ENDPOINT_TEST_STATUS	0x8
 #define STATUS_READ_SUCCESS             BIT(0)

@@ -73,9 +78,9 @@ static DEFINE_IDA(pci_endpoint_test_ida);
 #define to_endpoint_test(priv) container_of((priv), struct pci_endpoint_test,
 \
 					    miscdev)
 
-static bool no_msi;
-module_param(no_msi, bool, 0444);
-MODULE_PARM_DESC(no_msi, "Disable MSI interrupt in
 pci_endpoint_test");
+static int irq_type = IRQ_TYPE_MSIX;
+module_param(irq_type, int, 0444);
+MODULE_PARM_DESC(irq_type, "IRQ mode selection in pci_endpoint_test

(0
+- Legacy, 1 - MSI, 2 - MSI-X)");

enum pci_barno {

drivers/misc/pci_endpoint_test.c +4 -4

 	BAR_0,

@@ -103,7 +108,7 @@ struct pci_endpoint_test {  struct
 pci_endpoint_test_data {
 	enum pci_barno test_reg_bar;
 	size_t alignment;
-	bool no_msi;
+	int irq_type;
 };
 
 static inline u32 pci_endpoint_test_readl(struct pci_endpoint_test *test,

@@ -177,10 +182,10 @@ static bool pci_endpoint_test_bar(struct
 pci_endpoint_test *test,
 
 static bool pci_endpoint_test_legacy_irq(struct pci_endpoint_test *test)  {
-	u32 val;
+	u32 val = COMMAND_RAISE_LEGACY_IRQ;
 
-	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
-				 COMMAND_RAISE_LEGACY_IRQ);
+	val |= (IRQ_TYPE_LEGACY << IRQ_TYPE_SHIFT);
+	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND, val);
 	val = wait_for_completion_timeout(&test->irq_raised,
 					  msecs_to_jiffies(1000));
 	if (!val)

@@ -192,12 +197,12 @@ static bool pci_endpoint_test_legacy_irq(struct

pci_endpoint_test *test) static bool pci_endpoint_test_msi_irq(struct pci_endpoint_test *test,

drivers/misc/pci_endpoint_test.c +10 -4

 				      u8 msi_num)
 {
-	u32 val;
+	u32 val = COMMAND_RAISE_MSI_IRQ;
 	struct pci_dev *pdev = test->pdev;
 
-	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
-				 msi_num << MSI_NUMBER_SHIFT |
-				 COMMAND_RAISE_MSI_IRQ);
+	val |= (msi_num << MSI_NUMBER_SHIFT);
+	val |= (IRQ_TYPE_MSI << IRQ_TYPE_SHIFT);
+	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND, val);
 	val = wait_for_completion_timeout(&test->irq_raised,
 					  msecs_to_jiffies(1000));
 	if (!val)

@@ -209,6 +214,26 @@ static bool pci_endpoint_test_msi_irq(struct
 pci_endpoint_test *test,
 	return false;
 }
 
+static bool pci_endpoint_test_msix_irq(struct pci_endpoint_test *test,
+				       u16 msix_num)
+{
+	u32 val = COMMAND_RAISE_MSIX_IRQ;
+	struct pci_dev *pdev = test->pdev;
+

drivers/misc/pci_endpoint_test.c static bool pci_endpoint_test_msi_irq(struct · line 233

Quoted 10 lines

drivers/misc/pci_endpoint_test.c +10 -0

+	val |= (msix_num << MSI_NUMBER_SHIFT);
+	val |= (IRQ_TYPE_MSIX << IRQ_TYPE_SHIFT);
+	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND, val);
+	val = wait_for_completion_timeout(&test->irq_raised,
+					  msecs_to_jiffies(1000));
+	if (!val)
+		return false;
+
+	if (test->last_irq - pdev->irq == msix_num - 1)
+		return true;

I think we should use pci_irq_vector() to convert from device vector to Linux IRQ. It can be done in pci_endpoint_test_irqhandler()
(With MSI, pdev->irq will be updated during MSI init, but it is not for MSI-X.)

pci_irq_vector() should also be used for devm_request_irq() and devm_free_irq() so some changes required in probe and remove.

Regards,
Alan