PCI: assign bus numbers present in EA capability for bridges
PATCH v2 1/1 PCI: assign bus numbers present in EA capability for bridges
From: Subbaraya Sundeep <[email protected]>
As per the spec, bridges with EA capability work with fixed secondary and subordinate bus numbers. Hence assign bus numbers to bridges from EA if the capability exists.
---
Changes for v2:
No changes just added Sean Stalley who did EA support for BARs.
drivers/pci/probe.c | 58 ++++++++++++++++++++++++++++++++++++++++--- include/uapi/linux/pci_regs.h | 6 +++++ 2 files changed, 60 insertions(+), 4 deletions(-)
drivers/pci/probe.c +54 -4
@@ -1030,6 +1030,40 @@ static void pci_enable_crs(struct pci_dev *pdev)static unsigned int pci_scan_child_bus_extend (struct pci_bus *bus,unsigned int available_buses); +/* +* pci_ea_fixed_busnrs() - Read fixed Secondary and Subordinate bus +* numbers from EA capability. +* @dev: Bridge with EA +* @secondary: updated with secondary bus number in EA +* @subordinate: updated with subordinate bus number in EA +* +* If it is a bridge with EA capability then fixed bus numbers are +* read from EA capability list and secondary, subordinate reference +* variables will be updated. Otherwise secondary and subordinate reference +* variables will be zeroed. +*/ +static void pci_ea_fixed_busnrs (struct pci_dev *dev,u8 *secondary, +u8 *subordinate) +{ +int ea; +int offset; +u32 dw; + + *secondary = *subordinate =0 ; + +if (dev->hdr_type !=PCI_HEADER_TYPE_BRIDGE ) +return ; + +/* find PCI EA capability in list */ + ea =pci_find_capability (dev,PCI_CAP_ID_EA ); +if (!ea) +return ; + + offset = ea +PCI_EA_FIRST_ENT ; +pci_read_config_dword (dev, offset, &dw); + *secondary = dw &PCI_EA_SEC_BUS_MASK ; + *subordinate = (dw &PCI_EA_SUB_BUS_MASK ) >>PCI_EA_SUB_BUS_SHIFT ; +}/* * pci_scan_bridge_extend() - Scan buses behind a bridge
@@ -1064,6 +1098,8 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,u16 bctl;u8 primary, secondary, subordinate;int broken =0 ; +u8 fixed_sec, fixed_sub; +int next_busnr;/* * Make sure the bridge is powered on to be able to access config
@@ -1163,17 +1199,25 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,/* Clear errors */ pci_write_config_word (dev,PCI_STATUS ,0xffff ); +/* read bus numbers from EA */ +pci_ea_fixed_busnrs (dev, &fixed_sec, &fixed_sub); + + next_busnr = max +1 ; +/* Use secondary bus number in EA */ +if (fixed_sec) + next_busnr = fixed_sec; +/* * Prevent assigning a bus number that already exists. * This can happen when a bridge is hot-plugged, so in this * case we only re-scan this bus. */ - child =pci_find_bus (pci_domain_nr (bus), max+1 ); + child =pci_find_bus (pci_domain_nr (bus), next_busnr);if (!child) { - child =pci_add_new_bus (bus, dev, max+1 ); + child =pci_add_new_bus (bus, dev, next_busnr);if (!child) goto out; -pci_bus_insert_busn_res (child, max+1 , +pci_bus_insert_busn_res (child, next_busnr, bus->busn_res.end); } max++;
@@ -1234,7 +1278,13 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev, max += i; } -/* Set subordinate bus number to its real value */ +/* +* Set subordinate bus number to its real value. +* If fixed subordinate bus number exists from EA +* capability then use it. +*/ +if (fixed_sub) + max = fixed_sub;pci_bus_update_busn_res_end (child, max);pci_write_config_byte (dev,PCI_SUBORDINATE_BUS , max); }
include/uapi/linux/pci_regs.h +6 -0
@@ -372,6 +372,12 @@#define PCI_EA_FIRST_ENT_BRIDGE 8/* First EA Entry for Bridges */ #define PCI_EA_ES 0x00000007/* Entry Size */ #define PCI_EA_BEI 0x000000f0/* BAR Equivalent Indicator */ + +/* EA fixed Secondary and Subordinate bus numbers for Bridge */ +#define PCI_EA_SEC_BUS_MASK 0xff +#define PCI_EA_SUB_BUS_MASK 0xff00 +#define PCI_EA_SUB_BUS_SHIFT 8 +/* 0-5 map to BARs 0-5 respectively */ #define PCI_EA_BEI_BAR0 0#define PCI_EA_BEI_BAR5 5
Signature · 1 line
1.8.3.1
Hi Bjorn / Sean,
Any comments?
Thanks,
Sundeep
Quoted 135 lines · On Mon, Nov 19, 2018 at 6:44 PM <[email protected]> wrote:
From: Subbaraya Sundeep <[email protected]>
As per the spec, bridges with EA capability work with fixed secondary and subordinate bus numbers. Hence assign bus numbers to bridges from EA if the capability exists.
Signed-off-by Subbaraya Sundeep <[email protected]>---
Changes for v2:
No changes just added Sean Stalley who did EA support for BARs.drivers/pci/probe.c | 58 ++++++++++++++++++++++++++++++++++++++++--- include/uapi/linux/pci_regs.h | 6 +++++ 2 files changed, 60 insertions(+), 4 deletions(-)drivers/pci/probe.c +54
-4@@ -1030,6 +1030,40 @@ static void pci_enable_crs(struct pci_dev *pdev)static unsigned int pci_scan_child_bus_extend (struct pci_bus *bus,unsigned int available_buses); +/* +* pci_ea_fixed_busnrs() - Read fixed Secondary and Subordinate bus +* numbers from EA capability. +* @dev: Bridge with EA +* @secondary: updated with secondary bus number in EA +* @subordinate: updated with subordinate bus number in EA +* +* If it is a bridge with EA capability then fixed bus numbers are +* read from EA capability list and secondary, subordinate reference +* variables will be updated. Otherwise secondary and subordinate reference +* variables will be zeroed. +*/ +static void pci_ea_fixed_busnrs (struct pci_dev *dev,u8 *secondary, +u8 *subordinate) +{ +int ea; +int offset; +u32 dw; + + *secondary = *subordinate =0 ; + +if (dev->hdr_type !=PCI_HEADER_TYPE_BRIDGE ) +return ; + +/* find PCI EA capability in list */ + ea =pci_find_capability (dev,PCI_CAP_ID_EA ); +if (!ea) +return ; + + offset = ea +PCI_EA_FIRST_ENT ; +pci_read_config_dword (dev, offset, &dw); + *secondary = dw &PCI_EA_SEC_BUS_MASK ; + *subordinate = (dw &PCI_EA_SUB_BUS_MASK ) >>PCI_EA_SUB_BUS_SHIFT ; +}/* * pci_scan_bridge_extend() - Scan buses behind a bridge @@ -1064,6 +1098,8 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,u16 bctl;u8 primary, secondary, subordinate;int broken =0 ; +u8 fixed_sec, fixed_sub; +int next_busnr;/* * Make sure the bridge is powered on to be able to access config @@ -1163,17 +1199,25 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,/* Clear errors */ pci_write_config_word (dev,PCI_STATUS ,0xffff ); +/* read bus numbers from EA */ +pci_ea_fixed_busnrs (dev, &fixed_sec, &fixed_sub); + + next_busnr = max +1 ; +/* Use secondary bus number in EA */ +if (fixed_sec) + next_busnr = fixed_sec; +/* * Prevent assigning a bus number that already exists. * This can happen when a bridge is hot-plugged, so in this * case we only re-scan this bus. */ - child =pci_find_bus (pci_domain_nr (bus), max+1 ); + child =pci_find_bus (pci_domain_nr (bus), next_busnr);if (!child) { - child =pci_add_new_bus (bus, dev, max+1 ); + child =pci_add_new_bus (bus, dev, next_busnr);if (!child) goto out; -pci_bus_insert_busn_res (child, max+1 , +pci_bus_insert_busn_res (child, next_busnr, bus->busn_res.end); } max++;@@ -1234,7 +1278,13 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev, max += i; } -/* Set subordinate bus number to its real value */ +/* +* Set subordinate bus number to its real value. +* If fixed subordinate bus number exists from EA +* capability then use it. +*/ +if (fixed_sub) + max = fixed_sub;pci_bus_update_busn_res_end (child, max);pci_write_config_byte (dev,PCI_SUBORDINATE_BUS , max); }include/uapi/linux/pci_regs.h +6
-1@@ -372,6 +372,12 @@#define PCI_EA_FIRST_ENT_BRIDGE 8/* First EA Entry for Bridges */ #define PCI_EA_ES 0x00000007/* Entry Size */ #define PCI_EA_BEI 0x000000f0/* BAR Equivalent Indicator */ + +/* EA fixed Secondary and Subordinate bus numbers for Bridge */ +#define PCI_EA_SEC_BUS_MASK 0xff +#define PCI_EA_SUB_BUS_MASK 0xff00 +#define PCI_EA_SUB_BUS_SHIFT 8 +/* 0-5 map to BARs 0-5 respectively */ #define PCI_EA_BEI_BAR0 0#define PCI_EA_BEI_BAR5 5 --1.8.3.1
Quoted 6 lines · On Mon, Nov 19, 2018 at 06:44:32PM +0530, [email protected] wrote:
From: Subbaraya Sundeep <[email protected]>
As per the spec, bridges with EA capability work with fixed secondary and subordinate bus numbers. Hence assign bus numbers to bridges from EA if the capability exists.
A reference to the spec section would be good, i.e., PCIe r4.0, sec xxx.
Quoted 38 lines
Signed-off-by Subbaraya Sundeep <[email protected]>---
Changes for v2:
No changes just added Sean Stalley who did EA support for BARs.drivers/pci/probe.c | 58 ++++++++++++++++++++++++++++++++++++++++--- include/uapi/linux/pci_regs.h | 6 +++++ 2 files changed, 60 insertions(+), 4 deletions(-)drivers/pci/probe.c +21
-0@@ -1030,6 +1030,40 @@ static void pci_enable_crs(struct pci_dev *pdev)static unsigned int pci_scan_child_bus_extend (struct pci_bus *bus,unsigned int available_buses); +/* +* pci_ea_fixed_busnrs() - Read fixed Secondary and Subordinate bus +* numbers from EA capability. +* @dev: Bridge with EA +* @secondary: updated with secondary bus number in EA +* @subordinate: updated with subordinate bus number in EA +* +* If it is a bridge with EA capability then fixed bus numbers are +* read from EA capability list and secondary, subordinate reference +* variables will be updated. Otherwise secondary and subordinate reference +* variables will be zeroed. +*/ +static void pci_ea_fixed_busnrs (struct pci_dev *dev,u8 *secondary, +u8 *subordinate) +{ +int ea; +int offset; +u32 dw; + + *secondary = *subordinate =0 ; +
drivers/pci/probe.c static void pci_enable_crs(struct pci_dev *pdev) · line 1063
Quoted 10 lines
drivers/pci/probe.c +10
-0+if (dev->hdr_type !=PCI_HEADER_TYPE_BRIDGE ) +return ; + +/* find PCI EA capability in list */ + ea =pci_find_capability (dev,PCI_CAP_ID_EA ); +if (!ea) +return ; + + offset = ea +PCI_EA_FIRST_ENT ; +pci_read_config_dword (dev, offset, &dw);
"Num Entries" in the first DW of the capability is allowed to be zero, in which case this word (the second DW) is invalid. [See comments below; this code would be valid based on the 2014 ECN, but not per the 2017 PCIe r4.0 spec]
It would be much better if this function could be somehow incorporated into pci_ea_init(), which already knows how to parse much of the EA capability.
Quoted 64 lines
drivers/pci/probe.c +23
-4+ *secondary = dw &PCI_EA_SEC_BUS_MASK ; + *subordinate = (dw &PCI_EA_SUB_BUS_MASK ) >>PCI_EA_SUB_BUS_SHIFT ; +}/* * pci_scan_bridge_extend() - Scan buses behind a bridge @@ -1064,6 +1098,8 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,u16 bctl;u8 primary, secondary, subordinate;int broken =0 ; +u8 fixed_sec, fixed_sub; +int next_busnr;/* * Make sure the bridge is powered on to be able to access config @@ -1163,17 +1199,25 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,/* Clear errors */ pci_write_config_word (dev,PCI_STATUS ,0xffff ); +/* read bus numbers from EA */ +pci_ea_fixed_busnrs (dev, &fixed_sec, &fixed_sub); + + next_busnr = max +1 ; +/* Use secondary bus number in EA */ +if (fixed_sec) + next_busnr = fixed_sec; +/* * Prevent assigning a bus number that already exists. * This can happen when a bridge is hot-plugged, so in this * case we only re-scan this bus. */ - child =pci_find_bus (pci_domain_nr (bus), max+1 ); + child =pci_find_bus (pci_domain_nr (bus), next_busnr);if (!child) { - child =pci_add_new_bus (bus, dev, max+1 ); + child =pci_add_new_bus (bus, dev, next_busnr);if (!child) goto out; -pci_bus_insert_busn_res (child, max+1 , +pci_bus_insert_busn_res (child, next_busnr, bus->busn_res.end); } max++;@@ -1234,7 +1278,13 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev, max += i; } -/* Set subordinate bus number to its real value */ +/* +* Set subordinate bus number to its real value. +* If fixed subordinate bus number exists from EA +* capability then use it. +*/ +if (fixed_sub) + max = fixed_sub;pci_bus_update_busn_res_end (child, max);pci_write_config_byte (dev,PCI_SUBORDINATE_BUS , max); }include/uapi/linux/pci_regs.h +0
-0
include/uapi/linux/pci_regs.h line 372
Quoted 1 line
include/uapi/linux/pci_regs.h +0
-0@@ -372,6 +372,12 @@#define PCI_EA_FIRST_ENT_BRIDGE 8/* First EA Entry for Bridges */
You didn't add PCI_EA_FIRST_ENT_BRIDGE (Sean did with f80b0ba95964 ("PCI: Add Enhanced Allocation register entries")), but it is unused and I can't match it with anything in the spec (PCIe r4.0, sec 7.8.5).
It might be related to the code in pci_ea_init() that skips DWORD 2 for type 1 functions. But I still don't see that in the spec.
If it's obsolete, we should remove it (with a separate patch).
Hmm, I do see this in the ECN dated 23 Oct 2014, sec 6.9.1.2. But presumably that would be incorporated in PCIe r4.0, which is dated 27 Sep 2017.
I have no idea what to do with this. I can't merge this without some clarification here,
Quoted 14 lines
include/uapi/linux/pci_regs.h +6
-0#define PCI_EA_ES 0x00000007/* Entry Size */ #define PCI_EA_BEI 0x000000f0/* BAR Equivalent Indicator */ + +/* EA fixed Secondary and Subordinate bus numbers for Bridge */ +#define PCI_EA_SEC_BUS_MASK 0xff +#define PCI_EA_SUB_BUS_MASK 0xff00 +#define PCI_EA_SUB_BUS_SHIFT 8 +/* 0-5 map to BARs 0-5 respectively */ #define PCI_EA_BEI_BAR0 0#define PCI_EA_BEI_BAR5 5Signature · 1 line
1.8.3.1
Hi Bjorn,
Quoted 11 lines · Bjorn Helgaas
Quoted 6 lines · On Mon, Nov 19, 2018 at 06:44:32PM +0530, [email protected] wrote:
From: Subbaraya Sundeep <[email protected]>
As per the spec, bridges with EA capability work with fixed secondary and subordinate bus numbers. Hence assign bus numbers to bridges from EA if the capability exists.
A reference to the spec section would be good, i.e., PCIe r4.0, sec xxx.
Ok. I referred ECN 2014 section 6.9.1.2.
Quoted 45 lines
Quoted 45 lines
Signed-off-by Subbaraya Sundeep <[email protected]>---
Changes for v2:
No changes just added Sean Stalley who did EA support for BARs.drivers/pci/probe.c | 58 ++++++++++++++++++++++++++++++++++++++++--- include/uapi/linux/pci_regs.h | 6 +++++ 2 files changed, 60 insertions(+), 4 deletions(-)drivers/pci/probe.c +28
-0@@ -1030,6 +1030,40 @@ static void pci_enable_crs(struct pci_dev *pdev)static unsigned int pci_scan_child_bus_extend (struct pci_bus *bus,unsigned int available_buses); +/* +* pci_ea_fixed_busnrs() - Read fixed Secondary and Subordinate bus +* numbers from EA capability. +* @dev: Bridge with EA +* @secondary: updated with secondary bus number in EA +* @subordinate: updated with subordinate bus number in EA +* +* If it is a bridge with EA capability then fixed bus numbers are +* read from EA capability list and secondary, subordinate reference +* variables will be updated. Otherwise secondary and subordinate reference +* variables will be zeroed. +*/ +static void pci_ea_fixed_busnrs (struct pci_dev *dev,u8 *secondary, +u8 *subordinate) +{ +int ea; +int offset; +u32 dw; + + *secondary = *subordinate =0 ; + +if (dev->hdr_type !=PCI_HEADER_TYPE_BRIDGE ) +return ; + +/* find PCI EA capability in list */ + ea =pci_find_capability (dev,PCI_CAP_ID_EA ); +if (!ea) +return ;
Quoted 9 lines
drivers/pci/probe.c static void pci_enable_crs(struct pci_dev *pdev) · line 1063
Quoted 3 lines
drivers/pci/probe.c +3
-0+ + offset = ea +PCI_EA_FIRST_ENT ; +pci_read_config_dword (dev, offset, &dw);"Num Entries" in the first DW of the capability is allowed to be zero, in which case this word (the second DW) is invalid. [See comments below; this code would be valid based on the 2014 ECN, but not per the 2017 PCIe r4.0 spec]
Yes but Entries follow after first DW of EA capability for devices and after second
DW for bridges.
2014 ECN says for bridges DW2 of EA must be present:
"For Type 1 functions only, there is a second DW in the capability, preceding the first entry.
This second DW must be included in the Enhanced Allocation Capability whenever this
capability is implemented in a Type 1 Function"
So for normal device EA DW1, Entries(if any)
for bridges EA DW1,EA DW2, Entries(if any)
Quoted 4 lines
It would be much better if this function could be somehow incorporated into
pci_ea_init(), which already knows how to parse much of the EA capability.
I initially thought of this but didn't do it to avoid new members in pci_dev.
Quoted 70 lines
Quoted 64 lines
drivers/pci/probe.c +23
-4+ *secondary = dw &PCI_EA_SEC_BUS_MASK ; + *subordinate = (dw &PCI_EA_SUB_BUS_MASK ) >>PCI_EA_SUB_BUS_SHIFT ; +}/* * pci_scan_bridge_extend() - Scan buses behind a bridge @@ -1064,6 +1098,8 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,u16 bctl;u8 primary, secondary, subordinate;int broken =0 ; +u8 fixed_sec, fixed_sub; +int next_busnr;/* * Make sure the bridge is powered on to be able to access config @@ -1163,17 +1199,25 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,/* Clear errors */ pci_write_config_word (dev,PCI_STATUS ,0xffff ); +/* read bus numbers from EA */ +pci_ea_fixed_busnrs (dev, &fixed_sec, &fixed_sub); + + next_busnr = max +1 ; +/* Use secondary bus number in EA */ +if (fixed_sec) + next_busnr = fixed_sec; +/* * Prevent assigning a bus number that already exists. * This can happen when a bridge is hot-plugged, so in this * case we only re-scan this bus. */ - child =pci_find_bus (pci_domain_nr (bus), max+1 ); + child =pci_find_bus (pci_domain_nr (bus), next_busnr);if (!child) { - child =pci_add_new_bus (bus, dev, max+1 ); + child =pci_add_new_bus (bus, dev, next_busnr);if (!child) goto out; -pci_bus_insert_busn_res (child, max+1 , +pci_bus_insert_busn_res (child, next_busnr, bus->busn_res.end); } max++;@@ -1234,7 +1278,13 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev, max += i; } -/* Set subordinate bus number to its real value */ +/* +* Set subordinate bus number to its real value. +* If fixed subordinate bus number exists from EA +* capability then use it. +*/ +if (fixed_sub) + max = fixed_sub;pci_bus_update_busn_res_end (child, max);pci_write_config_byte (dev,PCI_SUBORDINATE_BUS , max); }include/uapi/linux/pci_regs.h +0
-0include/uapi/linux/pci_regs.h line 372
Quoted 1 line
include/uapi/linux/pci_regs.h +0
-0@@ -372,6 +372,12 @@#define PCI_EA_FIRST_ENT_BRIDGE 8/* First EA Entry for Bridges */ You didn't add
PCI_EA_FIRST_ENT_BRIDGE(Sean did withf80b0ba95964("PCI: Add Enhanced Allocation register entries")), but it is unused and I can't match it with anything in the spec (PCIe r4.0, sec 7.8.5).
Quoted 5 lines
It might be related to the code in
pci_ea_init()that skips DWORD 2 for type 1 functions. But I still don't see that in the spec.If it's obsolete, we should remove it (with a separate patch).
PCI_EA_FIRST_ENT_BRIDGE is not needed we can remove.
I will send a patch to remove it.
I will add new members for fixed secondary and subordinate bus numbers in pci_dev and make changes as below please let me know it is okay for you:
@@ -2909,6 +2909,7 @@ void pci_ea_init(struct pci_dev *dev)
u8 num_ent;
int offset;
int i;+ u32 dw;
/* find PCI EA capability in list */
ea = pci_find_capability(dev, PCI_CAP_ID_EA); @@ -2922,9
+2923,14 @@ void pci_ea_init(struct pci_dev *dev)
offset = ea + PCI_EA_FIRST_ENT;
- /* Skip DWORD 2 for type 1 functions */
- if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE)
+ /* Note fixed bus numbers for type 1 functions */
+ if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
+ pci_read_config_dword(dev, offset, &dw);
+ dev->fixed_sec_busnr = dw & PCI_EA_FIXED_SEC_BUS;
+ dev->fixed_sub_busnr = (dw & PCI_EA_FIXED_SUB_BUS) >>
+ PCI_EA_FIXED_SUB_SHIFT;
offset += 4;+ }
Quoted 7 lines
Hmm, I do see this in the ECN dated 23 Oct 2014, sec 6.9.1.2. But presumably that would be incorporated in PCIe r4.0, which is dated 27 Sep 2017.
I have no idea what to do with this. I can't merge this without some clarification here,
Yeah I see there is no mention of EA for bridges in r4.0.
Since we check whether the device is bridge or not before reading DW2 of EA I guess we are okay.
Thanks,
Sundeep
Quoted 14 lines
Quoted 14 lines
include/uapi/linux/pci_regs.h +6
-1#define PCI_EA_ES 0x00000007/* Entry Size */ #define PCI_EA_BEI 0x000000f0/* BAR Equivalent Indicator */ + +/* EA fixed Secondary and Subordinate bus numbers for Bridge */ +#define PCI_EA_SEC_BUS_MASK 0xff +#define PCI_EA_SUB_BUS_MASK 0xff00 +#define PCI_EA_SUB_BUS_SHIFT 8 +/* 0-5 map to BARs 0-5 respectively */ #define PCI_EA_BEI_BAR0 0#define PCI_EA_BEI_BAR5 5 --1.8.3.1
Quoted 189 lines · sundeep subbaraya
Quoted 10 lines · Bjorn Helgaas
Quoted 6 lines · On Mon, Nov 19, 2018 at 06:44:32PM +0530, [email protected] wrote:
From: Subbaraya Sundeep <[email protected]>
As per the spec, bridges with EA capability work with fixed secondary and subordinate bus numbers. Hence assign bus numbers to bridges from EA if the capability exists.
A reference to the spec section would be good, i.e., PCIe r4.0, sec xxx.
Ok. I referred ECN 2014 section 6.9.1.2.
Quoted 45 lines
Quoted 45 lines
Signed-off-by Subbaraya Sundeep <[email protected]>---
Changes for v2:
No changes just added Sean Stalley who did EA support for BARs.drivers/pci/probe.c | 58 ++++++++++++++++++++++++++++++++++++++++--- include/uapi/linux/pci_regs.h | 6 +++++ 2 files changed, 60 insertions(+), 4 deletions(-)drivers/pci/probe.c +28
-0@@ -1030,6 +1030,40 @@ static void pci_enable_crs(struct pci_dev *pdev)static unsigned int pci_scan_child_bus_extend (struct pci_bus *bus,unsigned int available_buses); +/* +* pci_ea_fixed_busnrs() - Read fixed Secondary and Subordinate bus +* numbers from EA capability. +* @dev: Bridge with EA +* @secondary: updated with secondary bus number in EA +* @subordinate: updated with subordinate bus number in EA +* +* If it is a bridge with EA capability then fixed bus numbers are +* read from EA capability list and secondary, subordinate reference +* variables will be updated. Otherwise secondary and subordinate reference +* variables will be zeroed. +*/ +static void pci_ea_fixed_busnrs (struct pci_dev *dev,u8 *secondary, +u8 *subordinate) +{ +int ea; +int offset; +u32 dw; + + *secondary = *subordinate =0 ; + +if (dev->hdr_type !=PCI_HEADER_TYPE_BRIDGE ) +return ; + +/* find PCI EA capability in list */ + ea =pci_find_capability (dev,PCI_CAP_ID_EA ); +if (!ea) +return ;Quoted 9 lines
drivers/pci/probe.c static void pci_enable_crs(struct pci_dev *pdev) · line 1063
Quoted 3 lines
drivers/pci/probe.c +3
-0+ + offset = ea +PCI_EA_FIRST_ENT ; +pci_read_config_dword (dev, offset, &dw);"Num Entries" in the first DW of the capability is allowed to be zero, in which case this word (the second DW) is invalid. [See comments below; this code would be valid based on the 2014 ECN, but not per the 2017 PCIe r4.0 spec]
Yes but Entries follow after first DW of EA capability for devices and after second
DW for bridges.
2014 ECN says for bridges DW2 of EA must be present:
"For Type 1 functions only, there is a second DW in the capability, preceding the first entry.
This second DW must be included in the Enhanced Allocation Capability whenever this
capability is implemented in a Type 1 Function"
So for normal device EA DW1, Entries(if any)
for bridges EA DW1,EA DW2, Entries(if any)Quoted 4 lines
It would be much better if this function could be somehow incorporated into
pci_ea_init(), which already knows how to parse much of the EA capability.I initially thought of this but didn't do it to avoid new members in
pci_dev.Quoted 70 lines
Quoted 64 lines
drivers/pci/probe.c +23
-4+ *secondary = dw &PCI_EA_SEC_BUS_MASK ; + *subordinate = (dw &PCI_EA_SUB_BUS_MASK ) >>PCI_EA_SUB_BUS_SHIFT ; +}/* * pci_scan_bridge_extend() - Scan buses behind a bridge @@ -1064,6 +1098,8 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,u16 bctl;u8 primary, secondary, subordinate;int broken =0 ; +u8 fixed_sec, fixed_sub; +int next_busnr;/* * Make sure the bridge is powered on to be able to access config @@ -1163,17 +1199,25 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,/* Clear errors */ pci_write_config_word (dev,PCI_STATUS ,0xffff ); +/* read bus numbers from EA */ +pci_ea_fixed_busnrs (dev, &fixed_sec, &fixed_sub); + + next_busnr = max +1 ; +/* Use secondary bus number in EA */ +if (fixed_sec) + next_busnr = fixed_sec; +/* * Prevent assigning a bus number that already exists. * This can happen when a bridge is hot-plugged, so in this * case we only re-scan this bus. */ - child =pci_find_bus (pci_domain_nr (bus), max+1 ); + child =pci_find_bus (pci_domain_nr (bus), next_busnr);if (!child) { - child =pci_add_new_bus (bus, dev, max+1 ); + child =pci_add_new_bus (bus, dev, next_busnr);if (!child) goto out; -pci_bus_insert_busn_res (child, max+1 , +pci_bus_insert_busn_res (child, next_busnr, bus->busn_res.end); } max++;@@ -1234,7 +1278,13 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev, max += i; } -/* Set subordinate bus number to its real value */ +/* +* Set subordinate bus number to its real value. +* If fixed subordinate bus number exists from EA +* capability then use it. +*/ +if (fixed_sub) + max = fixed_sub;pci_bus_update_busn_res_end (child, max);pci_write_config_byte (dev,PCI_SUBORDINATE_BUS , max); }include/uapi/linux/pci_regs.h +0
-0include/uapi/linux/pci_regs.h line 372
Quoted 1 line
include/uapi/linux/pci_regs.h +0
-0@@ -372,6 +372,12 @@#define PCI_EA_FIRST_ENT_BRIDGE 8/* First EA Entry for Bridges */ You didn't add
PCI_EA_FIRST_ENT_BRIDGE(Sean did withf80b0ba95964("PCI: Add Enhanced Allocation register entries")), but it is unused and I can't match it with anything in the spec (PCIe r4.0, sec 7.8.5).Quoted 5 lines
It might be related to the code in
pci_ea_init()that skips DWORD 2 for type 1 functions. But I still don't see that in the spec.If it's obsolete, we should remove it (with a separate patch).
PCI_EA_FIRST_ENT_BRIDGEis not needed we can remove.
I will send a patch to remove it.
I will add new members for fixed secondary and subordinate bus numbers inpci_devand make changes as below please let me know it is okay for you:(quoted diff) +9
-2@@ -2909,6 +2909,7 @@ void pci_ea_init(struct pci_dev *dev) u8 num_ent; int offset; int i; + u32 dw; /* find PCI EA capability in list */ ea = pci_find_capability(dev, PCI_CAP_ID_EA); @@ -2922,9 +2923,14 @@ void pci_ea_init(struct pci_dev *dev) offset = ea + PCI_EA_FIRST_ENT; - /* Skip DWORD 2 for type 1 functions */ - if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) + /* Note fixed bus numbers for type 1 functions */ + if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) { + pci_read_config_dword(dev, offset, &dw); + dev->fixed_sec_busnr = dw & PCI_EA_FIXED_SEC_BUS; + dev->fixed_sub_busnr = (dw & PCI_EA_FIXED_SUB_BUS) >> + PCI_EA_FIXED_SUB_SHIFT; offset += 4; + }Quoted 5 lines
Hmm, I do see this in the ECN dated 23 Oct 2014, sec 6.9.1.2. But presumably that would be incorporated in PCIe r4.0, which is dated 27 Sep 2017.
Quoted 6 lines
Quoted 3 lines
I have no idea what to do with this. I can't merge this without some clarification here,
Yeah I see there is no mention of EA for bridges in r4.0.
Since we check whether the device is bridge or not before reading DW2 of EA I guess we are okay.
I don't think so. We need at least some plan to clarify this in the spec. I think Sean is plugged into the PCI-SIG, so maybe he can help with that?
Bjorn
Bjorn Helgaas
AppliedQuoted 8 lines · On Mon, Nov 19, 2018 at 06:44:32PM +0530, [email protected] wrote:
From: Subbaraya Sundeep <[email protected]>
As per the spec, bridges with EA capability work with fixed secondary and subordinate bus numbers. Hence assign bus numbers to bridges from EA if the capability exists.
Signed-off-by Subbaraya Sundeep <[email protected]>
I applied this with minor revisions to pci/enumeration for v5.2, thanks!
Quoted 63 lines
---
Changes for v2:
No changes just added Sean Stalley who did EA support for BARs.drivers/pci/probe.c | 58 ++++++++++++++++++++++++++++++++++++++++--- include/uapi/linux/pci_regs.h | 6 +++++ 2 files changed, 60 insertions(+), 4 deletions(-)drivers/pci/probe.c +36
-0@@ -1030,6 +1030,40 @@ static void pci_enable_crs(struct pci_dev *pdev)static unsigned int pci_scan_child_bus_extend (struct pci_bus *bus,unsigned int available_buses); +/* +* pci_ea_fixed_busnrs() - Read fixed Secondary and Subordinate bus +* numbers from EA capability. +* @dev: Bridge with EA +* @secondary: updated with secondary bus number in EA +* @subordinate: updated with subordinate bus number in EA +* +* If it is a bridge with EA capability then fixed bus numbers are +* read from EA capability list and secondary, subordinate reference +* variables will be updated. Otherwise secondary and subordinate reference +* variables will be zeroed. +*/ +static void pci_ea_fixed_busnrs (struct pci_dev *dev,u8 *secondary, +u8 *subordinate) +{ +int ea; +int offset; +u32 dw; + + *secondary = *subordinate =0 ; + +if (dev->hdr_type !=PCI_HEADER_TYPE_BRIDGE ) +return ; + +/* find PCI EA capability in list */ + ea =pci_find_capability (dev,PCI_CAP_ID_EA ); +if (!ea) +return ; + + offset = ea +PCI_EA_FIRST_ENT ; +pci_read_config_dword (dev, offset, &dw); + *secondary = dw &PCI_EA_SEC_BUS_MASK ; + *subordinate = (dw &PCI_EA_SUB_BUS_MASK ) >>PCI_EA_SUB_BUS_SHIFT ; +}/* * pci_scan_bridge_extend() - Scan buses behind a bridge @@ -1064,6 +1098,8 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,u16 bctl;u8 primary, secondary, subordinate;int broken =0 ; +u8 fixed_sec, fixed_sub; +int next_busnr;/* * Make sure the bridge is powered on to be able to access config
drivers/pci/probe.c static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev, · line 1208
Quoted 10 lines
drivers/pci/probe.c +7
-0@@ -1163,17 +1199,25 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,/* Clear errors */ pci_write_config_word (dev,PCI_STATUS ,0xffff ); +/* read bus numbers from EA */ +pci_ea_fixed_busnrs (dev, &fixed_sec, &fixed_sub); + + next_busnr = max +1 ; +/* Use secondary bus number in EA */ +if (fixed_sec) + next_busnr = fixed_sec;
I initially thought this was not quite safe because EA could supply a secondary bus number of 0, in which case we would erroneously ignore it. But it's actually not possible for a PCI-to-PCI bridge to have a secondary bus number of 0, so it *is* safe.
But it's still a little subtle and since I've already done the work to add a bool return value from pci_ea_fixed_busnrs(), maybe it will be OK to keep that. The patch as applied is below, let me know if you have any comments.
Quoted 53 lines
drivers/pci/probe.c +11
-4+/* * Prevent assigning a bus number that already exists. * This can happen when a bridge is hot-plugged, so in this * case we only re-scan this bus. */ - child =pci_find_bus (pci_domain_nr (bus), max+1 ); + child =pci_find_bus (pci_domain_nr (bus), next_busnr);if (!child) { - child =pci_add_new_bus (bus, dev, max+1 ); + child =pci_add_new_bus (bus, dev, next_busnr);if (!child) goto out; -pci_bus_insert_busn_res (child, max+1 , +pci_bus_insert_busn_res (child, next_busnr, bus->busn_res.end); } max++;@@ -1234,7 +1278,13 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev, max += i; } -/* Set subordinate bus number to its real value */ +/* +* Set subordinate bus number to its real value. +* If fixed subordinate bus number exists from EA +* capability then use it. +*/ +if (fixed_sub) + max = fixed_sub;pci_bus_update_busn_res_end (child, max);pci_write_config_byte (dev,PCI_SUBORDINATE_BUS , max); }include/uapi/linux/pci_regs.h +6
-0@@ -372,6 +372,12 @@#define PCI_EA_FIRST_ENT_BRIDGE 8/* First EA Entry for Bridges */ #define PCI_EA_ES 0x00000007/* Entry Size */ #define PCI_EA_BEI 0x000000f0/* BAR Equivalent Indicator */ + +/* EA fixed Secondary and Subordinate bus numbers for Bridge */ +#define PCI_EA_SEC_BUS_MASK 0xff +#define PCI_EA_SUB_BUS_MASK 0xff00 +#define PCI_EA_SUB_BUS_SHIFT 8 +/* 0-5 map to BARs 0-5 respectively */ #define PCI_EA_BEI_BAR0 0#define PCI_EA_BEI_BAR5 5Signature · 1 line
1.8.3.1
commit 2dbce5901179
Author: Subbaraya Sundeep <[email protected]> Date: Mon Nov 19 18:44:32 2018 +0530
PCI: Assign bus numbers present in EA capability for bridges
The "Enhanced Allocation (EA) for Memory and I/O Resources" ECN, approved 23 October 2014, sec 6.9.1.2, specifies a second DW in the capability for type 1 (bridge) functions to describe fixed secondary and subordinate bus numbers. This ECN was included in the PCIe r4.0 spec, but sec 6.9.1.2 was omitted, presumably by mistake.
Read fixed bus numbers from the EA capability for bridges.
drivers/pci/probe.c +50 -4
@@ -1086,6 +1086,36 @@ static void pci_enable_crs(struct pci_dev *pdev)static unsigned int pci_scan_child_bus_extend (struct pci_bus *bus,unsigned int available_buses); +/** +* pci_ea_fixed_busnrs() - Read fixed Secondary and Subordinate bus +* numbers from EA capability. +* @dev: Bridge +* @sec: updated with secondary bus number from EA +* @sub: updated with subordinate bus number from EA +* +* If @dev is a bridge with EA capability, update @sec and @sub with +* fixed bus numbers from the capability and return true. Otherwise, +* return false. +*/ +static bool pci_ea_fixed_busnrs (struct pci_dev *dev,u8 *sec,u8 *sub) +{ +int ea, offset; +u32 dw; + +if (dev->hdr_type !=PCI_HEADER_TYPE_BRIDGE ) +return false; + +/* find PCI EA capability in list */ + ea =pci_find_capability (dev,PCI_CAP_ID_EA ); +if (!ea) +return false; + + offset = ea +PCI_EA_FIRST_ENT ; +pci_read_config_dword (dev, offset, &dw); + *sec = dw &PCI_EA_SEC_BUS_MASK ; + *sub = (dw &PCI_EA_SUB_BUS_MASK ) >>PCI_EA_SUB_BUS_SHIFT ; +return true; +}/* * pci_scan_bridge_extend() - Scan buses behind a bridge
@@ -1120,6 +1150,9 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,u16 bctl;u8 primary, secondary, subordinate;int broken =0 ; +bool fixed_buses; +u8 fixed_sec, fixed_sub; +int next_busnr;/* * Make sure the bridge is powered on to be able to access config
@@ -1219,17 +1252,24 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,/* Clear errors */ pci_write_config_word (dev,PCI_STATUS ,0xffff ); +/* Read bus numbers from EA Capability (if present) */ + fixed_buses =pci_ea_fixed_busnrs (dev, &fixed_sec, &fixed_sub); +if (fixed_buses) + next_busnr = fixed_sec; +else + next_busnr = max +1 ; +/* * Prevent assigning a bus number that already exists. * This can happen when a bridge is hot-plugged, so in this * case we only re-scan this bus. */ - child =pci_find_bus (pci_domain_nr (bus), max+1 ); + child =pci_find_bus (pci_domain_nr (bus), next_busnr);if (!child) { - child =pci_add_new_bus (bus, dev, max+1 ); + child =pci_add_new_bus (bus, dev, next_busnr);if (!child) goto out; -pci_bus_insert_busn_res (child, max+1 , +pci_bus_insert_busn_res (child, next_busnr, bus->busn_res.end); } max++;
@@ -1290,7 +1330,13 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev, max += i; } -/* Set subordinate bus number to its real value */ +/* +* Set subordinate bus number to its real value. +* If fixed subordinate bus number exists from EA +* capability then use it. +*/ +if (fixed_buses) + max = fixed_sub;pci_bus_update_busn_res_end (child, max);pci_write_config_byte (dev,PCI_SUBORDINATE_BUS , max); }
include/uapi/linux/pci_regs.h +6 -0
@@ -372,6 +372,12 @@#define PCI_EA_FIRST_ENT_BRIDGE 8/* First EA Entry for Bridges */ #define PCI_EA_ES 0x00000007/* Entry Size */ #define PCI_EA_BEI 0x000000f0/* BAR Equivalent Indicator */ + +/* EA fixed Secondary and Subordinate bus numbers for Bridge */ +#define PCI_EA_SEC_BUS_MASK 0xff +#define PCI_EA_SUB_BUS_MASK 0xff00 +#define PCI_EA_SUB_BUS_SHIFT 8 +/* 0-5 map to BARs 0-5 respectively */ #define PCI_EA_BEI_BAR0 0#define PCI_EA_BEI_BAR5 5
Hi Bjorn,
Quoted 93 lines · Bjorn Helgaas
Quoted 8 lines · On Mon, Nov 19, 2018 at 06:44:32PM +0530, [email protected] wrote:
From: Subbaraya Sundeep <[email protected]>
As per the spec, bridges with EA capability work with fixed secondary and subordinate bus numbers. Hence assign bus numbers to bridges from EA if the capability exists.
Signed-off-by Subbaraya Sundeep <[email protected]>I applied this with minor revisions to pci/enumeration for v5.2, thanks!
Quoted 63 lines
---
Changes for v2:
No changes just added Sean Stalley who did EA support for BARs.drivers/pci/probe.c | 58 ++++++++++++++++++++++++++++++++++++++++--- include/uapi/linux/pci_regs.h | 6 +++++ 2 files changed, 60 insertions(+), 4 deletions(-)drivers/pci/probe.c +36
-0@@ -1030,6 +1030,40 @@ static void pci_enable_crs(struct pci_dev *pdev)static unsigned int pci_scan_child_bus_extend (struct pci_bus *bus,unsigned int available_buses); +/* +* pci_ea_fixed_busnrs() - Read fixed Secondary and Subordinate bus +* numbers from EA capability. +* @dev: Bridge with EA +* @secondary: updated with secondary bus number in EA +* @subordinate: updated with subordinate bus number in EA +* +* If it is a bridge with EA capability then fixed bus numbers are +* read from EA capability list and secondary, subordinate reference +* variables will be updated. Otherwise secondary and subordinate reference +* variables will be zeroed. +*/ +static void pci_ea_fixed_busnrs (struct pci_dev *dev,u8 *secondary, +u8 *subordinate) +{ +int ea; +int offset; +u32 dw; + + *secondary = *subordinate =0 ; + +if (dev->hdr_type !=PCI_HEADER_TYPE_BRIDGE ) +return ; + +/* find PCI EA capability in list */ + ea =pci_find_capability (dev,PCI_CAP_ID_EA ); +if (!ea) +return ; + + offset = ea +PCI_EA_FIRST_ENT ; +pci_read_config_dword (dev, offset, &dw); + *secondary = dw &PCI_EA_SEC_BUS_MASK ; + *subordinate = (dw &PCI_EA_SUB_BUS_MASK ) >>PCI_EA_SUB_BUS_SHIFT ; +}/* * pci_scan_bridge_extend() - Scan buses behind a bridge @@ -1064,6 +1098,8 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,u16 bctl;u8 primary, secondary, subordinate;int broken =0 ; +u8 fixed_sec, fixed_sub; +int next_busnr;/* * Make sure the bridge is powered on to be able to access config drivers/pci/probe.c static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev, · line 1208
Quoted 10 lines
drivers/pci/probe.c +7
-0@@ -1163,17 +1199,25 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,/* Clear errors */ pci_write_config_word (dev,PCI_STATUS ,0xffff ); +/* read bus numbers from EA */ +pci_ea_fixed_busnrs (dev, &fixed_sec, &fixed_sub); + + next_busnr = max +1 ; +/* Use secondary bus number in EA */ +if (fixed_sec) + next_busnr = fixed_sec;I initially thought this was not quite safe because EA could supply a secondary bus number of 0, in which case we would erroneously ignore it. But it's actually not possible for a PCI-to-PCI bridge to have a secondary bus number of 0, so it *is* safe.
Quoted 5 lines
But it's still a little subtle and since I've already done the work to add a bool return value from
pci_ea_fixed_busnrs(), maybe it will be OK to keep that. The patch as applied is below, let me know if you have any comments.
Your changes looks good. Thanks for applying the patch.
Sundeep
Quoted 183 lines
Quoted 51 lines
drivers/pci/probe.c +11
-4+/* * Prevent assigning a bus number that already exists. * This can happen when a bridge is hot-plugged, so in this * case we only re-scan this bus. */ - child =pci_find_bus (pci_domain_nr (bus), max+1 ); + child =pci_find_bus (pci_domain_nr (bus), next_busnr);if (!child) { - child =pci_add_new_bus (bus, dev, max+1 ); + child =pci_add_new_bus (bus, dev, next_busnr);if (!child) goto out; -pci_bus_insert_busn_res (child, max+1 , +pci_bus_insert_busn_res (child, next_busnr, bus->busn_res.end); } max++;@@ -1234,7 +1278,13 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev, max += i; } -/* Set subordinate bus number to its real value */ +/* +* Set subordinate bus number to its real value. +* If fixed subordinate bus number exists from EA +* capability then use it. +*/ +if (fixed_sub) + max = fixed_sub;pci_bus_update_busn_res_end (child, max);pci_write_config_byte (dev,PCI_SUBORDINATE_BUS , max); }include/uapi/linux/pci_regs.h +6
-1@@ -372,6 +372,12 @@#define PCI_EA_FIRST_ENT_BRIDGE 8/* First EA Entry for Bridges */ #define PCI_EA_ES 0x00000007/* Entry Size */ #define PCI_EA_BEI 0x000000f0/* BAR Equivalent Indicator */ + +/* EA fixed Secondary and Subordinate bus numbers for Bridge */ +#define PCI_EA_SEC_BUS_MASK 0xff +#define PCI_EA_SUB_BUS_MASK 0xff00 +#define PCI_EA_SUB_BUS_SHIFT 8 +/* 0-5 map to BARs 0-5 respectively */ #define PCI_EA_BEI_BAR0 0#define PCI_EA_BEI_BAR5 5 --Quoted 2 lines
1.8.3.1
commit
2dbce5901179Author: Subbaraya Sundeep <[email protected]> Date: Mon Nov 19 18:44:32 2018 +0530PCI: Assign bus numbers present in EA capability for bridges
The "Enhanced Allocation (EA) for Memory and I/O Resources" ECN, approved 23 October 2014, sec 6.9.1.2, specifies a second DW in the capability for type 1 (bridge) functions to describe fixed secondary and subordinate bus numbers. This ECN was included in the PCIe r4.0 spec, but sec 6.9.1.2 was omitted, presumably by mistake.
Read fixed bus numbers from the EA capability for bridges.
Signed-off-by Subbaraya Sundeep <[email protected]>[bhelgaas: add pci_ea_fixed_busnrs() return value]Signed-off-by Bjorn Helgaas <[email protected]>drivers/pci/probe.c +50
-4@@ -1086,6 +1086,36 @@ static void pci_enable_crs(struct pci_dev *pdev)static unsigned int pci_scan_child_bus_extend (struct pci_bus *bus,unsigned int available_buses); +/** +* pci_ea_fixed_busnrs() - Read fixed Secondary and Subordinate bus +* numbers from EA capability. +* @dev: Bridge +* @sec: updated with secondary bus number from EA +* @sub: updated with subordinate bus number from EA +* +* If @dev is a bridge with EA capability, update @sec and @sub with +* fixed bus numbers from the capability and return true. Otherwise, +* return false. +*/ +static bool pci_ea_fixed_busnrs (struct pci_dev *dev,u8 *sec,u8 *sub) +{ +int ea, offset; +u32 dw; + +if (dev->hdr_type !=PCI_HEADER_TYPE_BRIDGE ) +return false; + +/* find PCI EA capability in list */ + ea =pci_find_capability (dev,PCI_CAP_ID_EA ); +if (!ea) +return false; + + offset = ea +PCI_EA_FIRST_ENT ; +pci_read_config_dword (dev, offset, &dw); + *sec = dw &PCI_EA_SEC_BUS_MASK ; + *sub = (dw &PCI_EA_SUB_BUS_MASK ) >>PCI_EA_SUB_BUS_SHIFT ; +return true; +}/* * pci_scan_bridge_extend() - Scan buses behind a bridge @@ -1120,6 +1150,9 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,u16 bctl;u8 primary, secondary, subordinate;int broken =0 ; +bool fixed_buses; +u8 fixed_sec, fixed_sub; +int next_busnr;/* * Make sure the bridge is powered on to be able to access config @@ -1219,17 +1252,24 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,/* Clear errors */ pci_write_config_word (dev,PCI_STATUS ,0xffff ); +/* Read bus numbers from EA Capability (if present) */ + fixed_buses =pci_ea_fixed_busnrs (dev, &fixed_sec, &fixed_sub); +if (fixed_buses) + next_busnr = fixed_sec; +else + next_busnr = max +1 ; +/* * Prevent assigning a bus number that already exists. * This can happen when a bridge is hot-plugged, so in this * case we only re-scan this bus. */ - child =pci_find_bus (pci_domain_nr (bus), max+1 ); + child =pci_find_bus (pci_domain_nr (bus), next_busnr);if (!child) { - child =pci_add_new_bus (bus, dev, max+1 ); + child =pci_add_new_bus (bus, dev, next_busnr);if (!child) goto out; -pci_bus_insert_busn_res (child, max+1 , +pci_bus_insert_busn_res (child, next_busnr, bus->busn_res.end); } max++;@@ -1290,7 +1330,13 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev, max += i; } -/* Set subordinate bus number to its real value */ +/* +* Set subordinate bus number to its real value. +* If fixed subordinate bus number exists from EA +* capability then use it. +*/ +if (fixed_buses) + max = fixed_sub;pci_bus_update_busn_res_end (child, max);pci_write_config_byte (dev,PCI_SUBORDINATE_BUS , max); }include/uapi/linux/pci_regs.h +6
-0@@ -372,6 +372,12 @@#define PCI_EA_FIRST_ENT_BRIDGE 8/* First EA Entry for Bridges */ #define PCI_EA_ES 0x00000007/* Entry Size */ #define PCI_EA_BEI 0x000000f0/* BAR Equivalent Indicator */ + +/* EA fixed Secondary and Subordinate bus numbers for Bridge */ +#define PCI_EA_SEC_BUS_MASK 0xff +#define PCI_EA_SUB_BUS_MASK 0xff00 +#define PCI_EA_SUB_BUS_SHIFT 8 +/* 0-5 map to BARs 0-5 respectively */ #define PCI_EA_BEI_BAR0 0#define PCI_EA_BEI_BAR5 5
Hi Bjorn,
Quoted 80 lines · Bjorn Helgaas
Quoted 8 lines · On Mon, Nov 19, 2018 at 06:44:32PM +0530, [email protected] wrote:
From: Subbaraya Sundeep <[email protected]>
As per the spec, bridges with EA capability work with fixed secondary and subordinate bus numbers. Hence assign bus numbers to bridges from EA if the capability exists.
Signed-off-by Subbaraya Sundeep <[email protected]>I applied this with minor revisions to pci/enumeration for v5.2, thanks!
Quoted 66 lines
---
Changes for v2:
No changes just added Sean Stalley who did EA support for BARs.drivers/pci/probe.c | 58 ++++++++++++++++++++++++++++++++++++++++--- include/uapi/linux/pci_regs.h | 6 +++++ 2 files changed, 60 insertions(+), 4 deletions(-)drivers/pci/probe.c +36
-0@@ -1030,6 +1030,40 @@ static void pci_enable_crs(struct pci_dev *pdev)static unsigned int pci_scan_child_bus_extend (struct pci_bus *bus,unsigned int available_buses); +/* +* pci_ea_fixed_busnrs() - Read fixed Secondary and Subordinate bus +* numbers from EA capability. +* @dev: Bridge with EA +* @secondary: updated with secondary bus number in EA +* @subordinate: updated with subordinate bus number in EA +* +* If it is a bridge with EA capability then fixed bus numbers are +* read from EA capability list and secondary, subordinate reference +* variables will be updated. Otherwise secondary and subordinate reference +* variables will be zeroed. +*/ +static void pci_ea_fixed_busnrs (struct pci_dev *dev,u8 *secondary, +u8 *subordinate) +{ +int ea; +int offset; +u32 dw; + + *secondary = *subordinate =0 ; + +if (dev->hdr_type !=PCI_HEADER_TYPE_BRIDGE ) +return ; + +/* find PCI EA capability in list */ + ea =pci_find_capability (dev,PCI_CAP_ID_EA ); +if (!ea) +return ; + + offset = ea +PCI_EA_FIRST_ENT ; +pci_read_config_dword (dev, offset, &dw); + *secondary = dw &PCI_EA_SEC_BUS_MASK ; + *subordinate = (dw &PCI_EA_SUB_BUS_MASK ) >>PCI_EA_SUB_BUS_SHIFT ; +}/* * pci_scan_bridge_extend() - Scan buses behind a bridge @@ -1064,6 +1098,8 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,u16 bctl;u8 primary, secondary, subordinate;int broken =0 ; +u8 fixed_sec, fixed_sub; +int next_busnr;/* * Make sure the bridge is powered on to be able to access config @@ -1163,17 +1199,25 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,/* Clear errors */ pci_write_config_word (dev,PCI_STATUS ,0xffff );
Quoted 11 lines
drivers/pci/probe.c static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev, · line 1208
Quoted 7 lines
drivers/pci/probe.c +7
-0+/* read bus numbers from EA */ +pci_ea_fixed_busnrs (dev, &fixed_sec, &fixed_sub); + + next_busnr = max +1 ; +/* Use secondary bus number in EA */ +if (fixed_sec) + next_busnr = fixed_sec;I initially thought this was not quite safe because EA could supply a secondary bus number of 0, in which case we would erroneously ignore it. But it's actually not possible for a PCI-to-PCI bridge to have a
When EA supplies secondary bus number as 0 then we should proceed with normal sequence where we scan with max + 1 as bus number for next round. Otherwise subordinate bus number will always be returning zero.
As per the spec, regd. Fixed Secondary Bus Number and Fixed Subordinate Bus Number:
"If at least one Function that uses EA is located behind this function,
then this field must be set to indicate the bus number of the PCI bus segment to which the secondary interface of this Function is connected. If no Function that uses EA is located behind this function, then this field must be set to all 0’s."
which implies that normal enumeration should take place behind this last EA bridge.
Shall I revert your changes which returns bool ? Or simply add a check if (fixed_buses && fixed_sec) ?
Sorry for the trouble.
Thanks,
Sundeep
Quoted 190 lines
secondary bus number of 0, so it *is* safe.
But it's still a little subtle and since I've already done the work to add a bool return value from
pci_ea_fixed_busnrs(), maybe it will be OK to keep that. The patch as applied is below, let me know if you have any comments.Quoted 51 lines
drivers/pci/probe.c +11
-4+/* * Prevent assigning a bus number that already exists. * This can happen when a bridge is hot-plugged, so in this * case we only re-scan this bus. */ - child =pci_find_bus (pci_domain_nr (bus), max+1 ); + child =pci_find_bus (pci_domain_nr (bus), next_busnr);if (!child) { - child =pci_add_new_bus (bus, dev, max+1 ); + child =pci_add_new_bus (bus, dev, next_busnr);if (!child) goto out; -pci_bus_insert_busn_res (child, max+1 , +pci_bus_insert_busn_res (child, next_busnr, bus->busn_res.end); } max++;@@ -1234,7 +1278,13 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev, max += i; } -/* Set subordinate bus number to its real value */ +/* +* Set subordinate bus number to its real value. +* If fixed subordinate bus number exists from EA +* capability then use it. +*/ +if (fixed_sub) + max = fixed_sub;pci_bus_update_busn_res_end (child, max);pci_write_config_byte (dev,PCI_SUBORDINATE_BUS , max); }include/uapi/linux/pci_regs.h +6
-1@@ -372,6 +372,12 @@#define PCI_EA_FIRST_ENT_BRIDGE 8/* First EA Entry for Bridges */ #define PCI_EA_ES 0x00000007/* Entry Size */ #define PCI_EA_BEI 0x000000f0/* BAR Equivalent Indicator */ + +/* EA fixed Secondary and Subordinate bus numbers for Bridge */ +#define PCI_EA_SEC_BUS_MASK 0xff +#define PCI_EA_SUB_BUS_MASK 0xff00 +#define PCI_EA_SUB_BUS_SHIFT 8 +/* 0-5 map to BARs 0-5 respectively */ #define PCI_EA_BEI_BAR0 0#define PCI_EA_BEI_BAR5 5 --Quoted 2 lines
1.8.3.1
commit
2dbce5901179Author: Subbaraya Sundeep <[email protected]> Date: Mon Nov 19 18:44:32 2018 +0530PCI: Assign bus numbers present in EA capability for bridges
The "Enhanced Allocation (EA) for Memory and I/O Resources" ECN, approved 23 October 2014, sec 6.9.1.2, specifies a second DW in the capability for type 1 (bridge) functions to describe fixed secondary and subordinate bus numbers. This ECN was included in the PCIe r4.0 spec, but sec 6.9.1.2 was omitted, presumably by mistake.
Read fixed bus numbers from the EA capability for bridges.
Signed-off-by Subbaraya Sundeep <[email protected]>[bhelgaas: add pci_ea_fixed_busnrs() return value]Signed-off-by Bjorn Helgaas <[email protected]>drivers/pci/probe.c +50
-4@@ -1086,6 +1086,36 @@ static void pci_enable_crs(struct pci_dev *pdev)static unsigned int pci_scan_child_bus_extend (struct pci_bus *bus,unsigned int available_buses); +/** +* pci_ea_fixed_busnrs() - Read fixed Secondary and Subordinate bus +* numbers from EA capability. +* @dev: Bridge +* @sec: updated with secondary bus number from EA +* @sub: updated with subordinate bus number from EA +* +* If @dev is a bridge with EA capability, update @sec and @sub with +* fixed bus numbers from the capability and return true. Otherwise, +* return false. +*/ +static bool pci_ea_fixed_busnrs (struct pci_dev *dev,u8 *sec,u8 *sub) +{ +int ea, offset; +u32 dw; + +if (dev->hdr_type !=PCI_HEADER_TYPE_BRIDGE ) +return false; + +/* find PCI EA capability in list */ + ea =pci_find_capability (dev,PCI_CAP_ID_EA ); +if (!ea) +return false; + + offset = ea +PCI_EA_FIRST_ENT ; +pci_read_config_dword (dev, offset, &dw); + *sec = dw &PCI_EA_SEC_BUS_MASK ; + *sub = (dw &PCI_EA_SUB_BUS_MASK ) >>PCI_EA_SUB_BUS_SHIFT ; +return true; +}/* * pci_scan_bridge_extend() - Scan buses behind a bridge @@ -1120,6 +1150,9 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,u16 bctl;u8 primary, secondary, subordinate;int broken =0 ; +bool fixed_buses; +u8 fixed_sec, fixed_sub; +int next_busnr;/* * Make sure the bridge is powered on to be able to access config @@ -1219,17 +1252,24 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,/* Clear errors */ pci_write_config_word (dev,PCI_STATUS ,0xffff ); +/* Read bus numbers from EA Capability (if present) */ + fixed_buses =pci_ea_fixed_busnrs (dev, &fixed_sec, &fixed_sub); +if (fixed_buses) + next_busnr = fixed_sec; +else + next_busnr = max +1 ; +/* * Prevent assigning a bus number that already exists. * This can happen when a bridge is hot-plugged, so in this * case we only re-scan this bus. */ - child =pci_find_bus (pci_domain_nr (bus), max+1 ); + child =pci_find_bus (pci_domain_nr (bus), next_busnr);if (!child) { - child =pci_add_new_bus (bus, dev, max+1 ); + child =pci_add_new_bus (bus, dev, next_busnr);if (!child) goto out; -pci_bus_insert_busn_res (child, max+1 , +pci_bus_insert_busn_res (child, next_busnr, bus->busn_res.end); } max++;@@ -1290,7 +1330,13 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev, max += i; } -/* Set subordinate bus number to its real value */ +/* +* Set subordinate bus number to its real value. +* If fixed subordinate bus number exists from EA +* capability then use it. +*/ +if (fixed_buses) + max = fixed_sub;pci_bus_update_busn_res_end (child, max);pci_write_config_byte (dev,PCI_SUBORDINATE_BUS , max); }include/uapi/linux/pci_regs.h +6
-0@@ -372,6 +372,12 @@#define PCI_EA_FIRST_ENT_BRIDGE 8/* First EA Entry for Bridges */ #define PCI_EA_ES 0x00000007/* Entry Size */ #define PCI_EA_BEI 0x000000f0/* BAR Equivalent Indicator */ + +/* EA fixed Secondary and Subordinate bus numbers for Bridge */ +#define PCI_EA_SEC_BUS_MASK 0xff +#define PCI_EA_SUB_BUS_MASK 0xff00 +#define PCI_EA_SUB_BUS_SHIFT 8 +/* 0-5 map to BARs 0-5 respectively */ #define PCI_EA_BEI_BAR0 0#define PCI_EA_BEI_BAR5 5