powerpc/eeh: Rework EEH initialisation
Subsystems (2)
Oliver O'Halloran
reviewerPATCH v2 1/9 powerpc/eeh: Rework EEH initialisation
Drop the EEH register / unregister ops thing and have the platform pass the ops structure into eeh_init() directly. This takes one initcall out of the EEH setup path and it means we're only doing EEH setup on the platforms which actually support it. It's also less code and generally easier to follow.
No functional changes.
--- arch/powerpc/include/asm/eeh.h | 3 +- arch/powerpc/kernel/eeh.c | 87 ++++---------------- arch/powerpc/platforms/powernv/eeh-powernv.c | 4 +- arch/powerpc/platforms/pseries/eeh_pseries.c | 5 +- 4 files changed, 21 insertions(+), 78 deletions(-)
arch/powerpc/include/asm/eeh.h +1 -2
@@ -295,8 +295,7 @@ const char *eeh_pe_loc_get(struct eeh_pe *pe);struct pci_bus *eeh_pe_bus_get (struct eeh_pe *pe);void eeh_show_enabled (void ); -int __init eeh_ops_register (struct eeh_ops *ops); -int __exit eeh_ops_unregister(const char *name); +int __init eeh_init (struct eeh_ops *ops);int eeh_check_failure (const volatile void __iomem *token);int eeh_dev_check_failure (struct eeh_dev *edev);void eeh_addr_cache_init (void );
arch/powerpc/kernel/eeh.c +16 -71
118 lines not on this page · raw message
arch/powerpc/platforms/powernv/eeh-powernv.c +2 -2
@@ -1699,7 +1699,7 @@ static int __init eeh_powernv_init(void) {int ret = -EINVAL ; - ret =eeh_ops_register (&pnv_eeh_ops); + ret =eeh_init (&pnv_eeh_ops);if (!ret)pr_info ("EEH: PowerNV platform initialized\n" );else
@@ -1707,4 +1707,4 @@ static int __init eeh_powernv_init(void)return ret; } -machine_early_initcall (powernv, eeh_powernv_init); +machine_core_initcall_sync (powernv, eeh_powernv_init);
arch/powerpc/platforms/pseries/eeh_pseries.c +2 -3
@@ -994,13 +994,12 @@ static int __init eeh_pseries_init(void) {int ret; - ret =eeh_ops_register (&pseries_eeh_ops); + ret =eeh_init (&pseries_eeh_ops);if (!ret)pr_info ("EEH: pSeries platform initialized\n" );else pr_info ("EEH: pSeries platform initialization failure (%d)\n" , ret); -return ret; } -machine_early_initcall (pseries, eeh_pseries_init); +machine_core_initcall_sync (pseries, eeh_pseries_init);
Signature · 1 line
2.26.2
Oliver O'Halloran
reviewerPATCH v2 2/9 powerpc/powernv: Stop using eeh_ops->init()
Fold pnv_eeh_init() into eeh_powernv_init() rather than having eeh_init() call it via eeh_ops->init(). It's simpler and it'll let us delete eeh_ops.init.
--- arch/powerpc/platforms/powernv/eeh-powernv.c | 94 ++++++++++---------- 1 file changed, 45 insertions(+), 49 deletions(-)
arch/powerpc/platforms/powernv/eeh-powernv.c +45 -49
@@ -44,54 +44,6 @@ void pnv_pcibios_bus_add_device(struct pci_dev *pdev)eeh_probe_device (pdev); } -static int pnv_eeh_init (void ) -{ -struct pci_controller *hose; -struct pnv_phb *phb; -int max_diag_size =PNV_PCI_DIAG_BUF_SIZE ; - -if (!firmware_has_feature (FW_FEATURE_OPAL )) { -pr_warn ("%s: OPAL is required !\n" , - __func__); -return -EINVAL ; - } - -/* Set probe mode */ -eeh_add_flag (EEH_PROBE_MODE_DEV ); - -/* -* P7IOC blocks PCI config access to frozen PE, but PHB3 -* doesn't do that. So we have to selectively enable I/O -* prior to collecting error log. -*/ -list_for_each_entry (hose, &hose_list, list_node) { - phb = hose->private_data; - -if (phb->model ==PNV_PHB_MODEL_P7IOC ) -eeh_add_flag (EEH_ENABLE_IO_FOR_LOG ); - -if (phb->diag_data_size > max_diag_size) - max_diag_size = phb->diag_data_size; - -/* -* PE#0 should be regarded as valid by EEH core -* if it's not the reserved one. Currently, we -* have the reserved PE#255 and PE#127 for PHB3 -* and P7IOC separately. So we should regard -* PE#0 as valid for PHB3 and P7IOC. -*/ -if (phb->ioda.reserved_pe_idx !=0 ) -eeh_add_flag (EEH_VALID_PE_ZERO ); - -break ; - } - -eeh_set_pe_aux_size (max_diag_size); - ppc_md.pcibios_bus_add_device = pnv_pcibios_bus_add_device; - -return 0 ; -} -static irqreturn_t pnv_eeh_event (int irq,void *data) {/*
@@ -1674,7 +1626,6 @@ static int pnv_eeh_restore_config(struct eeh_dev *edev)static struct eeh_ops pnv_eeh_ops = { .name ="powernv" , - .init = pnv_eeh_init, .probe = pnv_eeh_probe, .set_option = pnv_eeh_set_option, .get_state = pnv_eeh_get_state,
@@ -1697,8 +1648,53 @@ static struct eeh_ops pnv_eeh_ops = {*/ static int __init eeh_powernv_init (void ) { +int max_diag_size =PNV_PCI_DIAG_BUF_SIZE ; +struct pci_controller *hose; +struct pnv_phb *phb;int ret = -EINVAL ; +if (!firmware_has_feature (FW_FEATURE_OPAL )) { +pr_warn ("%s: OPAL is required !\n" , __func__); +return -EINVAL ; + } + +/* Set probe mode */ +eeh_add_flag (EEH_PROBE_MODE_DEV ); + +/* +* P7IOC blocks PCI config access to frozen PE, but PHB3 +* doesn't do that. So we have to selectively enable I/O +* prior to collecting error log. +*/ +list_for_each_entry (hose, &hose_list, list_node) { + phb = hose->private_data; + +if (phb->model ==PNV_PHB_MODEL_P7IOC ) +eeh_add_flag (EEH_ENABLE_IO_FOR_LOG ); + +if (phb->diag_data_size > max_diag_size) + max_diag_size = phb->diag_data_size; + +/* +* PE#0 should be regarded as valid by EEH core +* if it's not the reserved one. Currently, we +* have the reserved PE#255 and PE#127 for PHB3 +* and P7IOC separately. So we should regard +* PE#0 as valid for PHB3 and P7IOC. +*/ +if (phb->ioda.reserved_pe_idx !=0 ) +eeh_add_flag (EEH_VALID_PE_ZERO ); + +break ; + } + +/* +* eeh_init() allocates the eeh_pe and its aux data buf so the +* size needs to be set before calling eeh_init(). +*/ +eeh_set_pe_aux_size (max_diag_size); + ppc_md.pcibios_bus_add_device = pnv_pcibios_bus_add_device; + ret =eeh_init (&pnv_eeh_ops);if (!ret)pr_info ("EEH: PowerNV platform initialized\n" );
Signature · 1 line
2.26.2
Oliver O'Halloran
reviewerPATCH v2 3/9 powerpc/pseries: Stop using eeh_ops->init()
In reply to Oliver O'HalloranFold pseries_eeh_init() into eeh_pseries_init() rather than having eeh_init() call it via eeh_ops->init(). It's simpler and it'll let us delete eeh_ops.init.
--- arch/powerpc/platforms/pseries/eeh_pseries.c | 155 +++++++++---------- 1 file changed, 71 insertions(+), 84 deletions(-)
arch/powerpc/platforms/pseries/eeh_pseries.c +71 -84
@@ -239,88 +239,6 @@ static unsigned char slot_errbuf[RTAS_ERROR_LOG_MAX];static DEFINE_SPINLOCK (slot_errbuf_lock);static int eeh_error_buf_size; -/** -* pseries_eeh_init - EEH platform dependent initialization -* -* EEH platform dependent initialization on pseries. -*/ -static int pseries_eeh_init (void ) -{ -struct pci_controller *phb; -struct pci_dn *pdn; -int addr, config_addr; - -/* figure out EEH RTAS function call tokens */ - ibm_set_eeh_option =rtas_token ("ibm,set-eeh-option" ); - ibm_set_slot_reset =rtas_token ("ibm,set-slot-reset" ); - ibm_read_slot_reset_state2 =rtas_token ("ibm,read-slot-reset-state2" ); - ibm_read_slot_reset_state =rtas_token ("ibm,read-slot-reset-state" ); - ibm_slot_error_detail =rtas_token ("ibm,slot-error-detail" ); - ibm_get_config_addr_info2 =rtas_token ("ibm,get-config-addr-info2" ); - ibm_get_config_addr_info =rtas_token ("ibm,get-config-addr-info" ); - ibm_configure_pe =rtas_token ("ibm,configure-pe" ); - -/* -* ibm,configure-pe and ibm,configure-bridge have the same semantics, -* however ibm,configure-pe can be faster. If we can't find -* ibm,configure-pe then fall back to using ibm,configure-bridge. -*/ -if (ibm_configure_pe ==RTAS_UNKNOWN_SERVICE ) - ibm_configure_pe =rtas_token ("ibm,configure-bridge" ); - -/* -* Necessary sanity check. We needn't check "get-config-addr-info" -* and its variant since the old firmware probably support address -* of domain/bus/slot/function for EEH RTAS operations. -*/ -if (ibm_set_eeh_option ==RTAS_UNKNOWN_SERVICE || - ibm_set_slot_reset ==RTAS_UNKNOWN_SERVICE || - (ibm_read_slot_reset_state2 ==RTAS_UNKNOWN_SERVICE && - ibm_read_slot_reset_state ==RTAS_UNKNOWN_SERVICE ) || - ibm_slot_error_detail ==RTAS_UNKNOWN_SERVICE || - ibm_configure_pe ==RTAS_UNKNOWN_SERVICE ) { -pr_info ("EEH functionality not supported\n" ); -return -EINVAL ; - } - -/* Initialize error log lock and size */ -spin_lock_init (&slot_errbuf_lock); - eeh_error_buf_size =rtas_token ("rtas-error-log-max" ); -if (eeh_error_buf_size ==RTAS_UNKNOWN_SERVICE ) { -pr_info ("%s: unknown EEH error log size\n" , - __func__); - eeh_error_buf_size =1024 ; - }else if (eeh_error_buf_size >RTAS_ERROR_LOG_MAX ) { -pr_info ("%s: EEH error log size %d exceeds the maximal %d\n" , - __func__, eeh_error_buf_size,RTAS_ERROR_LOG_MAX ); - eeh_error_buf_size =RTAS_ERROR_LOG_MAX ; - } - -/* Set EEH probe mode */ -eeh_add_flag (EEH_PROBE_MODE_DEVTREE |EEH_ENABLE_IO_FOR_LOG ); - -/* Set EEH machine dependent code */ - ppc_md.pcibios_bus_add_device = pseries_pcibios_bus_add_device; - -if (is_kdump_kernel () || reset_devices) { -pr_info ("Issue PHB reset ...\n" ); -list_for_each_entry (phb, &hose_list, list_node) { - pdn = list_first_entry(&PCI_DN(phb->dn)->child_list, struct pci_dn, list); - addr = (pdn->busno <<16 ) | (pdn->devfn <<8 ); - config_addr =pseries_eeh_get_config_addr (phb, addr); -/* invalid PE config addr */ -if (config_addr ==0 ) -continue ; - -pseries_eeh_phb_reset (phb, config_addr,EEH_RESET_FUNDAMENTAL ); -pseries_eeh_phb_reset (phb, config_addr,EEH_RESET_DEACTIVATE ); -pseries_eeh_phb_configure_bridge (phb, config_addr); - } - } - -return 0 ; -} -static int pseries_eeh_cap_start (struct pci_dn *pdn) {u32 status;
@@ -967,7 +885,6 @@ static int pseries_notify_resume(struct eeh_dev *edev)static struct eeh_ops pseries_eeh_ops = { .name ="pseries" , - .init = pseries_eeh_init, .probe = pseries_eeh_probe, .set_option = pseries_eeh_set_option, .get_state = pseries_eeh_get_state,
@@ -992,7 +909,77 @@ static struct eeh_ops pseries_eeh_ops = {*/ static int __init eeh_pseries_init (void ) { -int ret; +struct pci_controller *phb; +struct pci_dn *pdn; +int ret, addr, config_addr; + +/* figure out EEH RTAS function call tokens */ + ibm_set_eeh_option =rtas_token ("ibm,set-eeh-option" ); + ibm_set_slot_reset =rtas_token ("ibm,set-slot-reset" ); + ibm_read_slot_reset_state2 =rtas_token ("ibm,read-slot-reset-state2" ); + ibm_read_slot_reset_state =rtas_token ("ibm,read-slot-reset-state" ); + ibm_slot_error_detail =rtas_token ("ibm,slot-error-detail" ); + ibm_get_config_addr_info2 =rtas_token ("ibm,get-config-addr-info2" ); + ibm_get_config_addr_info =rtas_token ("ibm,get-config-addr-info" ); + ibm_configure_pe =rtas_token ("ibm,configure-pe" ); + +/* +* ibm,configure-pe and ibm,configure-bridge have the same semantics, +* however ibm,configure-pe can be faster. If we can't find +* ibm,configure-pe then fall back to using ibm,configure-bridge. +*/ +if (ibm_configure_pe ==RTAS_UNKNOWN_SERVICE ) + ibm_configure_pe =rtas_token ("ibm,configure-bridge" ); + +/* +* Necessary sanity check. We needn't check "get-config-addr-info" +* and its variant since the old firmware probably support address +* of domain/bus/slot/function for EEH RTAS operations. +*/ +if (ibm_set_eeh_option ==RTAS_UNKNOWN_SERVICE || + ibm_set_slot_reset ==RTAS_UNKNOWN_SERVICE || + (ibm_read_slot_reset_state2 ==RTAS_UNKNOWN_SERVICE && + ibm_read_slot_reset_state ==RTAS_UNKNOWN_SERVICE ) || + ibm_slot_error_detail ==RTAS_UNKNOWN_SERVICE || + ibm_configure_pe ==RTAS_UNKNOWN_SERVICE ) { +pr_info ("EEH functionality not supported\n" ); +return -EINVAL ; + } + +/* Initialize error log lock and size */ +spin_lock_init (&slot_errbuf_lock); + eeh_error_buf_size =rtas_token ("rtas-error-log-max" ); +if (eeh_error_buf_size ==RTAS_UNKNOWN_SERVICE ) { +pr_info ("%s: unknown EEH error log size\n" , + __func__); + eeh_error_buf_size =1024 ; + }else if (eeh_error_buf_size >RTAS_ERROR_LOG_MAX ) { +pr_info ("%s: EEH error log size %d exceeds the maximal %d\n" , + __func__, eeh_error_buf_size,RTAS_ERROR_LOG_MAX ); + eeh_error_buf_size =RTAS_ERROR_LOG_MAX ; + } + +/* Set EEH probe mode */ +eeh_add_flag (EEH_PROBE_MODE_DEVTREE |EEH_ENABLE_IO_FOR_LOG ); + +/* Set EEH machine dependent code */ + ppc_md.pcibios_bus_add_device = pseries_pcibios_bus_add_device; + +if (is_kdump_kernel () || reset_devices) { +pr_info ("Issue PHB reset ...\n" ); +list_for_each_entry (phb, &hose_list, list_node) { + pdn = list_first_entry(&PCI_DN(phb->dn)->child_list, struct pci_dn, list); + addr = (pdn->busno <<16 ) | (pdn->devfn <<8 ); + config_addr =pseries_eeh_get_config_addr (phb, addr); +/* invalid PE config addr */ +if (config_addr ==0 ) +continue ; + +pseries_eeh_phb_reset (phb, config_addr,EEH_RESET_FUNDAMENTAL ); +pseries_eeh_phb_reset (phb, config_addr,EEH_RESET_DEACTIVATE ); +pseries_eeh_phb_configure_bridge (phb, config_addr); + } + } ret =eeh_init (&pseries_eeh_ops);if (!ret)
Signature · 1 line
2.26.2
Oliver O'Halloran
reviewerPATCH v2 4/9 powerpc/eeh: Delete eeh_ops->init
In reply to Oliver O'HalloranNo longer used since the platforms perform their EEH initialisation before calling eeh_init().
--- arch/powerpc/include/asm/eeh.h | 1 - arch/powerpc/kernel/eeh.c | 8 -------- 2 files changed, 9 deletions(-)
arch/powerpc/include/asm/eeh.h +0 -1
@@ -216,7 +216,6 @@ enum {struct eeh_ops {char *name; -int (*init)(void );struct eeh_dev *(*probe)(struct pci_dev *pdev);int (*set_option)(struct eeh_pe *pe,int option);int (*get_state)(struct eeh_pe *pe,int *delay);
arch/powerpc/kernel/eeh.c +0 -8
@@ -965,14 +965,6 @@ int eeh_init(struct eeh_ops *ops)return ret; } -if (eeh_ops->init) - ret = eeh_ops->init (); -if (ret) { -pr_warn ("%s: platform EEH init failed (%d)\n" , - __func__, ret); -return ret; - } -/* Initialize PHB PEs */ list_for_each_entry_safe (hose, tmp, &hose_list, list_node)eeh_phb_pe_create (hose);
Signature · 1 line
2.26.2
Oliver O'Halloran
reviewerPATCH v2 5/9 powerpc/eeh: Move EEH initialisation to an arch initcall
In reply to Oliver O'HalloranThe initialisation of EEH mostly happens in a core_initcall_sync initcall, followed by registering a bus notifier later on in an arch_initcall. Anything involving initcall dependecies is mostly incomprehensible unless you've spent a while staring at code so here's the full sequence:
ppc_md.setup_arch <-- pci_controllers are created here
...time passes...
core_initcall <-- pci_dns are created from DT nodes core_initcall_sync <-- platforms call eeh_init() postcore_initcall <-- PCI bus type is registered
postcore_initcall_syncarch_initcall <-- EEH pci_bus notifier registeredsubsys_initcall <-- PHBs are scanned here
There's no real requirement to do the EEH setup at the core_initcall_sync level. It just needs to be done after pci_dn's are created and before we start scanning PHBs. Simplify the flow a bit by moving the platform EEH inititalisation to an arch_initcall so we can fold the bus notifier registration into eeh_init().
--- arch/powerpc/kernel/eeh.c | 64 ++++++++++---------- arch/powerpc/platforms/powernv/eeh-powernv.c | 2 +- arch/powerpc/platforms/pseries/eeh_pseries.c | 2 +- 3 files changed, 34 insertions(+), 34 deletions(-)
arch/powerpc/kernel/eeh.c +32 -32
@@ -940,6 +940,30 @@ static struct notifier_block eeh_reboot_nb = { .notifier_call = eeh_reboot_notifier, }; +static int eeh_device_notifier (struct notifier_block *nb, +unsigned long action,void *data) +{ +struct device *dev = data; + +switch (action) { +/* +* Note: It's not possible to perform EEH device addition (i.e. +* {pseries,pnv}_pcibios_bus_add_device()) here because it depends on +* the device's resources, which have not yet been set up. +*/ +case BUS_NOTIFY_DEL_DEVICE : +eeh_remove_device (to_pci_dev (dev)); +break ; +default : +break ; + } +return NOTIFY_DONE ; +} + +static struct notifier_block eeh_device_nb = { + .notifier_call = eeh_device_notifier, +}; +/** * eeh_init - System wide EEH initialization *
@@ -960,7 +984,14 @@ int eeh_init(struct eeh_ops *ops)/* Register reboot notifier */ ret =register_reboot_notifier (&eeh_reboot_nb);if (ret) { -pr_warn ("%s: Failed to register notifier (%d)\n" , +pr_warn ("%s: Failed to register reboot notifier (%d)\n" , + __func__, ret); +return ret; + } + + ret =bus_register_notifier (&pci_bus_type, &eeh_device_nb); +if (ret) { +pr_warn ("%s: Failed to register bus notifier (%d)\n" , __func__, ret);return ret; }
@@ -975,37 +1006,6 @@ int eeh_init(struct eeh_ops *ops)return eeh_event_init (); } -static int eeh_device_notifier (struct notifier_block *nb, -unsigned long action,void *data) -{ -struct device *dev = data; - -switch (action) { -/* -* Note: It's not possible to perform EEH device addition (i.e. -* {pseries,pnv}_pcibios_bus_add_device()) here because it depends on -* the device's resources, which have not yet been set up. -*/ -case BUS_NOTIFY_DEL_DEVICE : -eeh_remove_device (to_pci_dev (dev)); -break ; -default : -break ; - } -return NOTIFY_DONE ; -} - -static struct notifier_block eeh_device_nb = { - .notifier_call = eeh_device_notifier, -}; - -static __init int eeh_set_bus_notifier (void ) -{ -bus_register_notifier (&pci_bus_type, &eeh_device_nb); -return 0 ; -} -arch_initcall (eeh_set_bus_notifier); -/** * eeh_probe_device() - Perform EEH initialization for the indicated pci device * @dev: pci device for which to set up EEH
arch/powerpc/platforms/powernv/eeh-powernv.c +1 -1
@@ -1703,4 +1703,4 @@ static int __init eeh_powernv_init(void)return ret; } -machine_core_initcall_sync (powernv, eeh_powernv_init); +machine_arch_initcall (powernv, eeh_powernv_init);
arch/powerpc/platforms/pseries/eeh_pseries.c +1 -1
@@ -989,4 +989,4 @@ static int __init eeh_pseries_init(void) ret);return ret; } -machine_core_initcall_sync (pseries, eeh_pseries_init); +machine_arch_initcall (pseries, eeh_pseries_init);
Signature · 1 line
2.26.2
Oliver O'Halloran
reviewerPATCH v2 6/9 powerpc/pseries/eeh: Clean up pe_config_addr lookups
In reply to Oliver O'HalloranDe-duplicate, and fix up the comments, and make the prototype just take a pci_dn since the job of the function is to return the pe_config_addr of the PE which contains a given device.
--- arch/powerpc/platforms/pseries/eeh_pseries.c | 80 +++----------------- 1 file changed, 11 insertions(+), 69 deletions(-)
arch/powerpc/platforms/pseries/eeh_pseries.c +11 -69
@@ -33,8 +33,6 @@#include <asm/ppc-pci.h> #include <asm/rtas.h> -static int pseries_eeh_get_pe_addr (struct pci_dn *pdn); -/* RTAS tokens */ static int ibm_set_eeh_option;static int ibm_set_slot_reset;
@@ -86,7 +84,8 @@ void pseries_pcibios_bus_add_device(struct pci_dev *pdev)/** -* pseries_eeh_get_config_addr - Retrieve config address +* pseries_eeh_get_pe_config_addr - Find the pe_config_addr for a device +* @pdn: pci_dn of the input device * * Retrieve the assocated config address. Actually, there're 2 RTAS * function calls dedicated for the purpose. We need implement
@@ -97,16 +96,17 @@ void pseries_pcibios_bus_add_device(struct pci_dev *pdev)* It's notable that zero'ed return value means invalid PE config * address. */ -static int pseries_eeh_get_config_addr (struct pci_controller *phb,int config_addr) +static int pseries_eeh_get_pe_config_addr (struct pci_dn *pdn) { +int config_addr =rtas_config_addr (pdn->busno, pdn->devfn,0 ); +struct pci_controller *phb = pdn->phb;int ret =0 ;int rets[3 ];if (ibm_get_config_addr_info2 !=RTAS_UNKNOWN_SERVICE ) {/* -* First of all, we need to make sure there has one PE -* associated with the device. Otherwise, PE address is -* meaningless. +* First of all, use function 1 to determine if this device is +* part of a PE or not. ret[0] being zero indicates it's not. */ ret = rtas_call(ibm_get_config_addr_info2, 4, 2, rets, config_addr, BUID_HI(phb->buid),
@@ -431,7 +431,7 @@ void pseries_eeh_init_edev(struct pci_dn *pdn)struct eeh_pe *parent;/* Retrieve PE address */ - edev->pe_config_addr =pseries_eeh_get_pe_addr (pdn); + edev->pe_config_addr =pseries_eeh_get_pe_config_addr (pdn); pe.addr = edev->pe_config_addr;/* Some older systems (Power4) allow the ibm,set-eeh-option
@@ -551,64 +551,6 @@ static int pseries_eeh_set_option(struct eeh_pe *pe, int option)return ret; } -/** -* pseries_eeh_get_pe_addr - Retrieve PE address -* @pe: EEH PE -* -* Retrieve the assocated PE address. Actually, there're 2 RTAS -* function calls dedicated for the purpose. We need implement -* it through the new function and then the old one. Besides, -* you should make sure the config address is figured out from -* FDT node before calling the function. -* -* It's notable that zero'ed return value means invalid PE config -* address. -*/ -static int pseries_eeh_get_pe_addr (struct pci_dn *pdn) -{ -int config_addr =rtas_config_addr (pdn->busno, pdn->devfn,0 ); -unsigned long buid = pdn->phb->buid; -int ret =0 ; -int rets[3 ]; - -if (ibm_get_config_addr_info2 !=RTAS_UNKNOWN_SERVICE ) { -/* -* First of all, we need to make sure there has one PE -* associated with the device. Otherwise, PE address is -* meaningless. -*/ - ret =rtas_call (ibm_get_config_addr_info2,4 ,2 , rets, - config_addr,BUID_HI (buid),BUID_LO (buid),1 ); -if (ret || (rets[0 ] ==0 )) -return 0 ; - -/* Retrieve the associated PE config address */ - ret =rtas_call (ibm_get_config_addr_info2,4 ,2 , rets, - config_addr,BUID_HI (buid),BUID_LO (buid),0 ); -if (ret) { -pr_warn ("%s: Failed to get address for PHB#%x-PE#%x\n" , - __func__, pdn->phb->global_number, config_addr); -return 0 ; - } - -return rets[0 ]; - } - -if (ibm_get_config_addr_info !=RTAS_UNKNOWN_SERVICE ) { - ret =rtas_call (ibm_get_config_addr_info,4 ,2 , rets, - config_addr,BUID_HI (buid),BUID_LO (buid),0 ); -if (ret) { -pr_warn ("%s: Failed to get address for PHB#%x-PE#%x\n" , - __func__, pdn->phb->global_number, config_addr); -return 0 ; - } - -return rets[0 ]; - } - -return ret; -} -/** * pseries_eeh_get_state - Retrieve PE state * @pe: EEH PE
@@ -911,7 +853,7 @@ static int __init eeh_pseries_init(void) {struct pci_controller *phb;struct pci_dn *pdn; -int ret, addr, config_addr; +int ret, config_addr;/* figure out EEH RTAS function call tokens */ ibm_set_eeh_option =rtas_token ("ibm,set-eeh-option" );
@@ -969,8 +911,8 @@ static int __init eeh_pseries_init(void)pr_info ("Issue PHB reset ...\n" );list_for_each_entry (phb, &hose_list, list_node) { pdn = list_first_entry(&PCI_DN(phb->dn)->child_list, struct pci_dn, list); - addr = (pdn->busno <<16 ) | (pdn->devfn <<8 ); - config_addr =pseries_eeh_get_config_addr (phb, addr); + config_addr =pseries_eeh_get_pe_config_addr (pdn); +/* invalid PE config addr */ if (config_addr ==0 )continue ;
Signature · 1 line
2.26.2
Oliver O'Halloran
reviewerPATCH v2 7/9 powerpc/pseries/eeh: Rework device EEH PE determination
In reply to Oliver O'HalloranThe process Linux uses for determining if a device supports EEH or not appears to be at odds with what PAPR says the OS should be doing. The current flow is something like:
1. Assume pe_config_addr is equal the the device's config_addr.
2. Attempt to enable EEH on that PE
3. Verify EEH was enabled (POWER4 bug workaround)
4. Try find the pe_config_addr using the ibm,get-config-addr-info2 RTAS call.
5. If that fails walk the pci_dn tree upwards trying to find a parent device with EEH support. If we find one then add the device to that PE.
The first major problem with this process is that we need the PE config address in step 2) since its needs to be passed to the ibm,set-eeh-option RTAS call when enabling EEH for th PE. We hack around this requirement in by making the assumption in 1) and delay finding the actual PE address until 4). This is fine if:
a) The PCI device is the 0th function, and
b) The device is on the PE's root bus.
Granted, the current sequence does appear to work on most systems even when these conditions are false. At a guess PowerVM's RTAS has workarounds to accommodate Linux's quirks or the RTAS call to enable EEH is treated as no-op on most platforms since EEH is usually enabled by default. However, what is currently implemented is a bit sketch and is downright confusing since it doesn't match up with what what PAPR suggests we should be doing.
This patch re-works how we handle EEH init so that we find the PE config address using the ibm,get-config-addr-info2 RTAS call first, then use the found address to finish the EEH init process. It also drops the Power4 workaround since as of commit 471d7ff8b51b ("powerpc/64s: Remove POWER4 support") the kernel does not support running on a Power4 CPU so there's no need to support the Power4 platform's quirks either. With the patch applied the sequence is now:
1. Find the pe_config_addr from the device using the RTAS call.
2. Enable the PE.
3. Insert the edev into the tree and create an eeh_pe if needed.
The other change made here is ignoring unsupported devices entirely. Currently the device's BARs are saved to the eeh_dev even if the device is not part of an EEH PE. Not being part of a PE means that an EEH recovery pass will never see that device so the saving the BARs is pointless.
--- arch/powerpc/platforms/pseries/eeh_pseries.c | 57 ++++++++------------ 1 file changed, 22 insertions(+), 35 deletions(-)
arch/powerpc/platforms/pseries/eeh_pseries.c +22 -35
@@ -357,10 +357,10 @@ static struct eeh_pe *pseries_eeh_pe_get_parent(struct eeh_dev *edev)*/ void pseries_eeh_init_edev (struct pci_dn *pdn) { +struct eeh_pe pe, *parent;struct eeh_dev *edev; -struct eeh_pe pe; +int addr;u32 pcie_flags; -int enable =0 ;int ret;if (WARN_ON_ONCE (!eeh_has_flag (EEH_PROBE_MODE_DEVTREE )))
@@ -417,51 +417,38 @@ void pseries_eeh_init_edev(struct pci_dn *pdn) } } -/* Initialize the fake PE */ +/* first up, find the pe_config_addr for the PE containing the device */ + addr =pseries_eeh_get_pe_config_addr (pdn); +if (addr ==0 ) { +eeh_edev_dbg (edev,"Unable to find pe_config_addr\n" ); + goto err; + } + +/* Try enable EEH on the fake PE */ memset (&pe,0 ,sizeof (struct eeh_pe )); pe.phb = pdn->phb; - pe.config_addr = (pdn->busno <<16 ) | (pdn->devfn <<8 ); + pe.addr = addr; -/* Enable EEH on the device */ eeh_edev_dbg (edev,"Enabling EEH on device\n" ); ret = eeh_ops->set_option (&pe,EEH_OPT_ENABLE );if (ret) {eeh_edev_dbg (edev,"EEH failed to enable on device (code %d)\n" , ret); - }else { -struct eeh_pe *parent; + goto err; + } -/* Retrieve PE address */ - edev->pe_config_addr =pseries_eeh_get_pe_config_addr (pdn); - pe.addr = edev->pe_config_addr; + edev->pe_config_addr = addr; -/* Some older systems (Power4) allow the ibm,set-eeh-option -* call to succeed even on nodes where EEH is not supported. -* Verify support explicitly. -*/ - ret = eeh_ops->get_state (&pe,NULL ); -if (ret >0 && ret !=EEH_STATE_NOT_SUPPORT ) - enable =1 ; +eeh_add_flag (EEH_ENABLED ); -/* -* This device doesn't support EEH, but it may have an -* EEH parent. In this case any error on the device will -* freeze the PE of it's upstream bridge, so added it to -* the upstream PE. -*/ - parent =pseries_eeh_pe_get_parent (edev); -if (parent && !enable) - edev->pe_config_addr = parent->addr; + parent =pseries_eeh_pe_get_parent (edev); +eeh_pe_tree_insert (edev, parent); +eeh_save_bars (edev); +eeh_edev_dbg (edev,"EEH enabled for device" ); -if (enable || parent) { -eeh_add_flag (EEH_ENABLED ); -eeh_pe_tree_insert (edev, parent); - } -eeh_edev_dbg (edev,"EEH is %s on device (code %d)\n" , - (enable ?"enabled" :"unsupported" ), ret); - } +return ; -/* Save memory bars */ -eeh_save_bars (edev); +err: +eeh_edev_dbg (edev,"EEH is unsupported on device (code = %d)\n" , ret); }static struct eeh_dev *pseries_eeh_probe (struct pci_dev *pdev)
Signature · 1 line
2.26.2
Oliver O'Halloran
reviewerPATCH v2 8/9 powerpc/pseries/eeh: Allow zero to be a valid PE configuration address
In reply to Oliver O'HalloranThere's no real reason why zero can't be a valid PE configuration address. Under qemu each sPAPR PHB (i.e. EEH supporting) has the passed-though devices on bus zero, so the PE address of bus <dddd>:00 should be zero.
However, all previous versions of Linux will reject that, so Qemu at least goes out of it's way to avoid it. The Qemu implementation of ibm,get-config-addr-info2 RTAS has the following comment:
Quoted 4 lines
/*
* We always have PE address of form "00BB0001". "BB"
* represents the bus number of PE's primary bus. */
So qemu puts a one into the register portion of the PE's config_addr to avoid it being zero. The whole is pretty silly considering that RTAS will return a negative error code if it can't map the device's config_addr to a PE.
This patch fixes Linux to treat zero as a valid PE address. This shouldn't have any real effects due to the Qemu hack mentioned above. And the fact that Linux EEH has worked historically on PowerVM means they never pass through devices on bus zero so we would never see the problem there either.
--- arch/powerpc/platforms/pseries/eeh_pseries.c | 38 +++++++++++--------- 1 file changed, 22 insertions(+), 16 deletions(-)
arch/powerpc/platforms/pseries/eeh_pseries.c +22 -16
@@ -87,21 +87,20 @@ void pseries_pcibios_bus_add_device(struct pci_dev *pdev)* pseries_eeh_get_pe_config_addr - Find the pe_config_addr for a device * @pdn: pci_dn of the input device * -* Retrieve the assocated config address. Actually, there're 2 RTAS -* function calls dedicated for the purpose. We need implement -* it through the new function and then the old one. Besides, -* you should make sure the config address is figured out from -* FDT node before calling the function. +* The EEH RTAS calls use a tuple consisting of: (buid_hi, buid_lo, +* pe_config_addr) as a handle to a given PE. This function finds the +* pe_config_addr based on the device's config addr. * -* It's notable that zero'ed return value means invalid PE config -* address. +* Keep in mind that the pe_config_addr *might* be numerically identical to the +* device's config addr, but the two are conceptually distinct. +* +* Returns the pe_config_addr, or a negative error code. */ static int pseries_eeh_get_pe_config_addr (struct pci_dn *pdn) {int config_addr =rtas_config_addr (pdn->busno, pdn->devfn,0 );struct pci_controller *phb = pdn->phb; -int ret =0 ; -int rets[3 ]; +int ret, rets[3 ];if (ibm_get_config_addr_info2 !=RTAS_UNKNOWN_SERVICE ) {/*
@@ -112,16 +111,16 @@ static int pseries_eeh_get_pe_config_addr(struct pci_dn *pdn) config_addr,BUID_HI (phb->buid), BUID_LO(phb->buid), 1);if (ret || (rets[0 ] ==0 )) -return 0 ; +return -ENOENT ; -/* Retrieve the associated PE config address */ +/* Retrieve the associated PE config address with function 0 */ ret =rtas_call (ibm_get_config_addr_info2,4 ,2 , rets, config_addr,BUID_HI (phb->buid),BUID_LO (phb->buid),0 );if (ret) {pr_warn ("%s: Failed to get address for PHB#%x-PE#%x\n" , __func__, phb->global_number, config_addr); -return 0 ; +return -ENXIO ; }return rets[0 ];
@@ -134,13 +133,20 @@ static int pseries_eeh_get_pe_config_addr(struct pci_dn *pdn)if (ret) {pr_warn ("%s: Failed to get address for PHB#%x-PE#%x\n" , __func__, phb->global_number, config_addr); -return 0 ; +return -ENXIO ; }return rets[0 ]; } -return ret; +/* +* PAPR does describe a process for finding the pe_config_addr that was +* used before the ibm,get-config-addr-info calls were added. However, +* I haven't found *any* systems that don't have that RTAS call +* implemented. If you happen to find one that needs the old DT based +* process, patches are welcome! +*/ +return -ENOENT ; }/**
@@ -419,7 +425,7 @@ void pseries_eeh_init_edev(struct pci_dn *pdn)/* first up, find the pe_config_addr for the PE containing the device */ addr =pseries_eeh_get_pe_config_addr (pdn); -if (addr ==0 ) { +if (addr <0 ) {eeh_edev_dbg (edev,"Unable to find pe_config_addr\n" ); goto err; }
@@ -901,7 +907,7 @@ static int __init eeh_pseries_init(void) config_addr =pseries_eeh_get_pe_config_addr (pdn);/* invalid PE config addr */ -if (config_addr ==0 ) +if (config_addr <0 )continue ;pseries_eeh_phb_reset (phb, config_addr,EEH_RESET_FUNDAMENTAL );
Signature · 1 line
2.26.2
Oliver O'Halloran
reviewerPATCH v2 9/9 powerpc/eeh: Clean up PE addressing
In reply to Oliver O'HalloranWhen support for EEH on PowerNV was added a lot of pseries specific code was made "generic" and some of the quirks of pseries EEH came along for the ride. One of the stranger quirks is eeh_pe containing two types of PE
address: pe->addr and pe->config_addr. There reason for this appears to be historical baggage rather than any real requirements.
On pseries EEH PEs are manipulated using RTAS calls. Each EEH RTAS call takes a "PE configuration address" as an input which is used to identify which EEH PE is being manipulated by the call. When initialising the EEH state for a device the first thing we need to do is determine the configuration address for the PE which contains the device so we can enable EEH on that PE. This process is outlined in PAPR which is the modern (i.e post-2003) FW specification for pseries. However, EEH support was first described in the pSeries RISC Platform Architecture (RPA) and although they are mostly compatible EEH is one of the areas where they are not.
The major difference is that RPA doesn't actually have the concept of a PE. On RPA systems the EEH RTAS calls are done on a per-device basis using the same config_addr that would be passed to the RTAS functions to access PCI config space (e.g. ibm,read-pci-config). The config_addr is not identical since the function and config register offsets of the config_addr must be set to zero. EEH operations being done on a per-device basis doesn't make a whole lot of sense when you consider how EEH was implemented on legacy PCI systems.
For legacy PCI(-X) systems EEH was implemented using special PCI-PCI bridges which contained logic to detect errors and freeze the secondary bus when one occurred. This means that the EEH enabled state is shared among all devices behind that EEH bridge. As a result there's no way to implement the per-device control required for the semantics specified by RPA. It can be made to work if we assume that a separate EEH bridge exists for each EEH capable PCI slot and there are no bridges behind those slots. However, RPA also specifies the ibm,configure-bridge RTAS call for re-initalising bridges behind EEH capable slots after they are reset due to an EEH event so that is probably not a valid assumption. This incoherence was fixed in later PAPR, which succeeded RPA. Unfortunately, since Linux EEH support seems to have been implemented based on the RPA spec some of the legacy assumptions were carried over (probably for POWER4 compatibility).
The fix made in PAPR was the introduction of the "PE" concept and redefining the EEH RTAS calls (set-eeh-option, reset-slot, etc) to operate on a per-PE basis so all devices behind an EEH bride would share the same EEH state. The "config_addr" argument to the EEH RTAS calls became the "PE_config_addr" and the OS was required to use the ibm,get-config-addr-info RTAS call to find the correct PE address for the device. When support for the new interfaces was added to Linux it was implemented using something like:
At probe time:
pdn->eeh_config_addr = rtas_config_addr(pdn); pdn->eeh_pe_config_addr = rtas_get_config_addr_info(pdn);
When performing an RTAS call:
config_addr = pdn->eeh_config_addr; if (pdn->eeh_pe_config_addr) config_addr = pdn->eeh_pe_config_addr; rtas_call(..., config_addr, ...);
In other words, if the ibm,get-config-addr-info RTAS call is implemented and returned a valid result we'd use that as the argument to the EEH RTAS calls. If not, Linux would fall back to using the device's config_addr. Over time these addresses have moved around going from pci_dn to eeh_dev and finally into eeh_pe. Today the users look like this:
config_addr = pe->config_addr; if (pe->addr) config_addr = pe->addr; rtas_call(..., config_addr, ...);
However, considering the EEH core always operates on a per-PE basis and even on pseries the only per-device operation is the initial call to ibm,set-eeh-option I'm not sure if any of this actually works on an RPA system today. It doesn't make much sense to have the fallback address in a generic structure either since the bulk of the code which reference it is in pseries anyway.
The EEH core makes a token effort to support looking up a PE using the config_addr by having two arguments to eeh_pe_get(). However, a survey of all the callers to eeh_pe_get() shows that all bar one have the config_addr argument hard-coded to zero.The only caller that doesn't is in eeh_pe_tree_insert() which has:
if (!eeh_has_flag(EEH_VALID_PE_ZERO) && !edev->pe_config_addr) return -EINVAL; pe = eeh_pe_get(hose, edev->pe_config_addr, edev->bdfn);
The third argument (config_addr) is only used if the second (pe->addr) argument is invalid. The preceding check ensures that the call to eeh_pe_get() will never happen if edev->pe_config_addr is invalid so there is no situation where eeh_pe_get() will search for a PE based on the 3rd argument. The check also means that we'll never insert a PE into the tree where pe_config_addr is zero since EEH_VALID_PE_ZERO is never set on pseries. All the users of the fallback address on pseries never actually use the fallback and all the only caller that supplies something for the config_addr argument to eeh_pe_get() never use it either. It's all dead code.
This patch removes the fallback address from eeh_pe since nothing uses it. Specificly, we do this by:
1) Removing pe->config_addr
2) Removing the EEH_VALID_PE_ZERO flag
3) Removing the fallback address argument to eeh_pe_get().
4) Removing all the checks for pe->addr being zero in the pseries EEH code.
This leaves us with PE's only being identified by what's in their pe->addr field and the EEH core relying on the platform to ensure that eeh_dev's are only inserted into the EEH tree if they're actually inside a PE.
No functional changes, I hope.
---
v2: re-wrote commit message
--- arch/powerpc/include/asm/eeh.h | 4 +- arch/powerpc/kernel/eeh.c | 2 +- arch/powerpc/kernel/eeh_pe.c | 46 +++----------------- arch/powerpc/platforms/powernv/eeh-powernv.c | 16 ++----- arch/powerpc/platforms/pseries/eeh_pseries.c | 42 +++--------------- 5 files changed, 17 insertions(+), 93 deletions(-)
arch/powerpc/include/asm/eeh.h +1 -3
@@ -27,7 +27,6 @@ struct pci_dn;#define EEH_FORCE_DISABLED 0x02/* EEH disabled */ #define EEH_PROBE_MODE_DEV 0x04/* From PCI device */ #define EEH_PROBE_MODE_DEVTREE 0x08/* From device tree */ -#define EEH_VALID_PE_ZERO 0x10/* PE#0 is valid */ #define EEH_ENABLE_IO_FOR_LOG 0x20/* Enable IO for log */ #define EEH_EARLY_DUMP_LOG 0x40/* Dump log immediately */
@@ -280,8 +279,7 @@ int eeh_phb_pe_create(struct pci_controller *phb);int eeh_wait_state (struct eeh_pe *pe,int max_wait);struct eeh_pe *eeh_phb_pe_get (struct pci_controller *phb);struct eeh_pe *eeh_pe_next (struct eeh_pe *pe,struct eeh_pe *root); -struct eeh_pe *eeh_pe_get (struct pci_controller *phb, -int pe_no,int config_addr); +struct eeh_pe *eeh_pe_get (struct pci_controller *phb,int pe_no);int eeh_pe_tree_insert (struct eeh_dev *edev,struct eeh_pe *new_pe_parent);int eeh_pe_tree_remove (struct eeh_dev *edev);void eeh_pe_update_time_stamp (struct eeh_pe *pe);
arch/powerpc/kernel/eeh.c +1 -1
@@ -1657,7 +1657,7 @@ static ssize_t eeh_force_recover_write(struct file *filp,return -ENODEV ;/* Retrieve PE */ - pe =eeh_pe_get (hose, pe_no,0 ); + pe =eeh_pe_get (hose, pe_no);if (!pe)return -ENODEV ;
arch/powerpc/kernel/eeh_pe.c +6 -40
91 lines not on this page · raw message
arch/powerpc/platforms/powernv/eeh-powernv.c +3 -13
@@ -87,7 +87,7 @@ static ssize_t pnv_eeh_ei_write(struct file *filp,return -EINVAL ;/* Retrieve PE */ - pe =eeh_pe_get (hose, pe_no,0 ); + pe =eeh_pe_get (hose, pe_no);if (!pe)return -ENODEV ;
@@ -306,7 +306,7 @@ static struct eeh_pe *pnv_eeh_get_upstream_pe(struct pci_dev *pdev)if (parent) {struct pnv_ioda_pe *ioda_pe =pnv_ioda_get_pe (parent); -return eeh_pe_get (phb->hose, ioda_pe->pe_number,0 ); +return eeh_pe_get (phb->hose, ioda_pe->pe_number); }return NULL ;
@@ -1358,7 +1358,7 @@ static int pnv_eeh_get_pe(struct pci_controller *hose, }/* Find the PE according to PE# */ - dev_pe =eeh_pe_get (hose, pe_no,0 ); + dev_pe =eeh_pe_get (hose, pe_no);if (!dev_pe)return -EEXIST ;
@@ -1675,16 +1675,6 @@ static int __init eeh_powernv_init(void)if (phb->diag_data_size > max_diag_size) max_diag_size = phb->diag_data_size; -/* -* PE#0 should be regarded as valid by EEH core -* if it's not the reserved one. Currently, we -* have the reserved PE#255 and PE#127 for PHB3 -* and P7IOC separately. So we should regard -* PE#0 as valid for PHB3 and P7IOC. -*/ -if (phb->ioda.reserved_pe_idx !=0 ) -eeh_add_flag (EEH_VALID_PE_ZERO ); -break ; }
arch/powerpc/platforms/pseries/eeh_pseries.c +6 -36
101 lines not on this page · raw message
Signature · 1 line
2.26.2
Michael Ellerman
AppliedQuoted 7 lines · Oliver O'Halloran
Drop the EEH register / unregister ops thing and have the platform pass the ops structure into
eeh_init()directly. This takes one initcall out of the EEH setup path and it means we're only doing EEH setup on the platforms which actually support it. It's also less code and generally easier to follow.No functional changes.
Applied to powerpc/next.
[1/9] powerpc/eeh: Rework EEH initialisation
https://git.kernel.org/powerpc/c/d125aedb404204de0579b16028096b2cc09e4deb [2/9] powerpc/powernv: Stop using eeh_ops->init() https://git.kernel.org/powerpc/c/82a1ea21f1bac42eb8e3f77d5d249201855f2c85 [3/9] powerpc/pseries: Stop using eeh_ops->init() https://git.kernel.org/powerpc/c/1f8fa0cd6a848ff072bffe0ee776554387128f60 [4/9] powerpc/eeh: Delete eeh_ops->init https://git.kernel.org/powerpc/c/5d69e46a2104050c0a458c6bf6abba5f58f56e4c [5/9] powerpc/eeh: Move EEH initialisation to an arch initcall https://git.kernel.org/powerpc/c/395ee2a2a15ba1c4c7c414db24dc3082ba8feab8 [6/9] powerpc/pseries/eeh: Clean up pe_config_addr lookups https://git.kernel.org/powerpc/c/f61c859feb5d19787c93d6b2b3d4beeca7260034 [7/9] powerpc/pseries/eeh: Rework device EEH PE determination https://git.kernel.org/powerpc/c/98ba956f6a3891b233466b8da064f17d16dc2090 [8/9] powerpc/pseries/eeh: Allow zero to be a valid PE configuration address https://git.kernel.org/powerpc/c/42de19d5ef71b91765266557705394e52954adb3 [9/9] powerpc/eeh: Clean up PE addressing https://git.kernel.org/powerpc/c/35d64734b64315f2c5716c5a0a380ed1ba8fbe4a
cheers