Skip to contentSkip to conversation

pci: Make DWORD accesses while saving / restoring LTR state

PATCH Applied Rajat Jain 3 participants 4 messages Dec 22, 2021 – Jan 11, 2022 PCI

lore
Share & tools
lore

Rajat Jain

author
#0 lore raw

PATCH 1/1 pci: Make DWORD accesses while saving / restoring LTR state

Some devices have an errata such that they only support DWORD accesses to some registers.

For e.g. this Bayhub O2 device ([VID:DID] = [0x1217:0x8621]) only supports DWORD accesses to LTR latency registers and L1 PM substates control registers:
https://github.com/rajatxjain/public_shared/blob/main/OZ711LV2_appnote.pdf

Since L1 PM substate control registers are DWORD sized, and hence their access in the kernel is already DWORD sized, so we don't need to do anything for them.

However, the LTR registers being WORD sized, are in need of a solution. This patch converts the WORD sized accesses to these registers, into DWORD sized accesses, while saving and restoring them.

---
 drivers/pci/pci.c       | 24 ++++++++++++++++--------
 drivers/pci/pcie/aspm.c |  1 +
 2 files changed, 17 insertions(+), 8 deletions(-)
drivers/pci/pci.c +16 -8
index 3d2fb394986a..efa8cd16827f 100644
@@ -1556,7 +1556,7 @@ static void pci_save_ltr_state(struct pci_dev *dev)
 {
 	int ltr;
 	struct pci_cap_saved_state *save_state;
-	u16 *cap;
+	u32 *cap;
 
 	if (!pci_is_pcie(dev))
 		return;
@@ -1571,25 +1571,33 @@ static void pci_save_ltr_state(struct pci_dev *dev)
 		return;
 	}
 
-	cap = (u16 *)&save_state->cap.data[0];
-	pci_read_config_word(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, cap++);
-	pci_read_config_word(dev, ltr + PCI_LTR_MAX_NOSNOOP_LAT, cap++);
+	/*
+	 * We deliberately do a dword access to save both PCI_LTR_MAX_SNOOP_LAT
+	 * and PCI_LTR_MAX_NOSNOOP_LAT together since some devices only support
+	 * dword accesses to these registers.
+	 */
+	cap = &save_state->cap.data[0];
+	pci_read_config_dword(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, cap);
 }
 
 static void pci_restore_ltr_state(struct pci_dev *dev)
 {
 	struct pci_cap_saved_state *save_state;
 	int ltr;
-	u16 *cap;
+	u32 *cap;
 
 	save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_LTR);
 	ltr = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_LTR);
 	if (!save_state || !ltr)
 		return;
 
-	cap = (u16 *)&save_state->cap.data[0];
-	pci_write_config_word(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, *cap++);
-	pci_write_config_word(dev, ltr + PCI_LTR_MAX_NOSNOOP_LAT, *cap++);
+	/*
+	 * We deliberately do a dword access to restore both
+	 * PCI_LTR_MAX_SNOOP_LAT and PCI_LTR_MAX_NOSNOOP_LAT together since
+	 * some devices only support dword accesses to these registers.
+	 */
+	cap = &save_state->cap.data[0];
+	pci_write_config_dword(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, *cap);
 }
 
 /**
drivers/pci/pcie/aspm.c +1 -0
index 52c74682601a..083f47a7b69b 100644
@@ -496,6 +496,7 @@ static void aspm_calc_l1ss_info(struct pcie_link_state *link,
 	encode_l12_threshold(l1_2_threshold, &scale, &value);
 	ctl1 |= t_common_mode << 8 | scale << 29 | value << 16;
 
+	/* Always make DWORD sized accesses to these registers */
 	pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, &pctl1);
 	pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, &pctl2);
 	pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, &cctl1);
Signature · 1 line
2.34.1.307.g9b7440fafd-goog
author
#1 lore raw

Hi Bjorn,

Sorry, I forgot to mention...

Quoted 13 lines · Rajat Jain

Some devices have an errata such that they only support DWORD accesses to some registers.

For e.g. this Bayhub O2 device ([VID:DID] = [0x1217:0x8621]) only supports DWORD accesses to LTR latency registers and L1 PM substates control registers:
https://github.com/rajatxjain/public_shared/blob/main/OZ711LV2_appnote.pdf

Since L1 PM substate control registers are DWORD sized, and hence their access in the kernel is already DWORD sized, so we don't need to do anything for them.

Quoted 5 lines

However, the LTR registers being WORD sized, are in need of a solution. This patch converts the WORD sized accesses to these registers, into DWORD sized accesses, while saving and restoring them.

... I have made a more generic proposal here:
https://lore.kernel.org/linux-pci/CACK8Z6GR9pR0Oc_G70q2MxTVgrk+PXWTFbWT1EjARnuYCRavLw@mail.gmail.com/T/#t

However that is a bigger effort in terms of implementation as well as reaching an agreement and acceptance with the community. So I'd appreciate it if this patch would please be included until a larger framework is put in place.

Thanks & Best Regards,

Rajat

Quoted 74 lines
---
 drivers/pci/pci.c       | 24 ++++++++++++++++--------
 drivers/pci/pcie/aspm.c |  1 +
 2 files changed, 17 insertions(+), 8 deletions(-)
drivers/pci/pci.c +16 -8
index 3d2fb394986a..efa8cd16827f 100644
@@ -1556,7 +1556,7 @@ static void pci_save_ltr_state(struct pci_dev *dev)
 {
        int ltr;
        struct pci_cap_saved_state *save_state;
-       u16 *cap;
+       u32 *cap;
 
        if (!pci_is_pcie(dev))
                return;
@@ -1571,25 +1571,33 @@ static void pci_save_ltr_state(struct pci_dev *dev)
                return;
        }
 
-       cap = (u16 *)&save_state->cap.data[0];
-       pci_read_config_word(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, cap++);
-       pci_read_config_word(dev, ltr + PCI_LTR_MAX_NOSNOOP_LAT, cap++);
+       /*
+        * We deliberately do a dword access to save both PCI_LTR_MAX_SNOOP_LAT
+        * and PCI_LTR_MAX_NOSNOOP_LAT together since some devices only support
+        * dword accesses to these registers.
+        */
+       cap = &save_state->cap.data[0];
+       pci_read_config_dword(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, cap);
 }
 
 static void pci_restore_ltr_state(struct pci_dev *dev)
 {
        struct pci_cap_saved_state *save_state;
        int ltr;
-       u16 *cap;
+       u32 *cap;
 
        save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_LTR);
        ltr = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_LTR);
        if (!save_state || !ltr)
                return;
 
-       cap = (u16 *)&save_state->cap.data[0];
-       pci_write_config_word(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, *cap++);
-       pci_write_config_word(dev, ltr + PCI_LTR_MAX_NOSNOOP_LAT, *cap++);
+       /*
+        * We deliberately do a dword access to restore both
+        * PCI_LTR_MAX_SNOOP_LAT and PCI_LTR_MAX_NOSNOOP_LAT together since
+        * some devices only support dword accesses to these registers.
+        */
+       cap = &save_state->cap.data[0];
+       pci_write_config_dword(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, *cap);
 }
 
 /**
drivers/pci/pcie/aspm.c +1 -1
index 52c74682601a..083f47a7b69b 100644
@@ -496,6 +496,7 @@ static void aspm_calc_l1ss_info(struct pcie_link_state *link,
        encode_l12_threshold(l1_2_threshold, &scale, &value);
        ctl1 |= t_common_mode << 8 | scale << 29 | value << 16;
 
+       /* Always make DWORD sized accesses to these registers */
        pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, &pctl1);
        pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, &pctl2);
        pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, &cctl1);
--

2.34.1.307.g9b7440fafd-goog

#2 lore raw
In reply to Rajat Jain
Quoted 12 lines · Rajat Jain

Some devices have an errata such that they only support DWORD accesses to some registers.

For e.g. this Bayhub O2 device ([VID:DID] = [0x1217:0x8621]) only supports DWORD accesses to LTR latency registers and L1 PM substates control registers:
https://github.com/rajatxjain/public_shared/blob/main/OZ711LV2_appnote.pdf

Since L1 PM substate control registers are DWORD sized, and hence their access in the kernel is already DWORD sized, so we don't need to do anything for them.

Quoted 5 lines

However, the LTR registers being WORD sized, are in need of a solution. This patch converts the WORD sized accesses to these registers, into DWORD sized accesses, while saving and restoring them.

Applied to pci/enumeration for v5.17, thanks, Rajat!

The app note suggests that this erratum only affects the registers at 0x234, 0x248, and 0x24c, i.e., the LTR snoop registers and the L1 SS control registers.

Can you confirm that is true? Byte and word accesses to other parts of config space work correctly?

I *assume* the other parts work correctly, because if byte and word accesses were broken, all sorts of things would not work, like PCI_COMMAND, PCI_STATUS, searching the capability list, etc.

Bjorn

Quoted 74 lines
---
 drivers/pci/pci.c       | 24 ++++++++++++++++--------
 drivers/pci/pcie/aspm.c |  1 +
 2 files changed, 17 insertions(+), 8 deletions(-)
drivers/pci/pci.c +16 -8
index 3d2fb394986a..efa8cd16827f 100644
@@ -1556,7 +1556,7 @@ static void pci_save_ltr_state(struct pci_dev *dev)
 {
 	int ltr;
 	struct pci_cap_saved_state *save_state;
-	u16 *cap;
+	u32 *cap;
 
 	if (!pci_is_pcie(dev))
 		return;
@@ -1571,25 +1571,33 @@ static void pci_save_ltr_state(struct pci_dev *dev)
 		return;
 	}
 
-	cap = (u16 *)&save_state->cap.data[0];
-	pci_read_config_word(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, cap++);
-	pci_read_config_word(dev, ltr + PCI_LTR_MAX_NOSNOOP_LAT, cap++);
+	/*
+	 * We deliberately do a dword access to save both PCI_LTR_MAX_SNOOP_LAT
+	 * and PCI_LTR_MAX_NOSNOOP_LAT together since some devices only support
+	 * dword accesses to these registers.
+	 */
+	cap = &save_state->cap.data[0];
+	pci_read_config_dword(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, cap);
 }
 
 static void pci_restore_ltr_state(struct pci_dev *dev)
 {
 	struct pci_cap_saved_state *save_state;
 	int ltr;
-	u16 *cap;
+	u32 *cap;
 
 	save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_LTR);
 	ltr = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_LTR);
 	if (!save_state || !ltr)
 		return;
 
-	cap = (u16 *)&save_state->cap.data[0];
-	pci_write_config_word(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, *cap++);
-	pci_write_config_word(dev, ltr + PCI_LTR_MAX_NOSNOOP_LAT, *cap++);
+	/*
+	 * We deliberately do a dword access to restore both
+	 * PCI_LTR_MAX_SNOOP_LAT and PCI_LTR_MAX_NOSNOOP_LAT together since
+	 * some devices only support dword accesses to these registers.
+	 */
+	cap = &save_state->cap.data[0];
+	pci_write_config_dword(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, *cap);
 }
 
 /**
drivers/pci/pcie/aspm.c +1 -0
index 52c74682601a..083f47a7b69b 100644
@@ -496,6 +496,7 @@ static void aspm_calc_l1ss_info(struct pcie_link_state *link,
 	encode_l12_threshold(l1_2_threshold, &scale, &value);
 	ctl1 |= t_common_mode << 8 | scale << 29 | value << 16;
 
+	/* Always make DWORD sized accesses to these registers */
 	pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, &pctl1);
 	pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, &pctl2);
 	pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, &cctl1);
Signature · 1 line
2.34.1.307.g9b7440fafd-goog
#3 lore raw

Hello Bjorn,

Quoted 14 lines · Bjorn Helgaas
Quoted 12 lines · Rajat Jain

Some devices have an errata such that they only support DWORD accesses to some registers.

For e.g. this Bayhub O2 device ([VID:DID] = [0x1217:0x8621]) only supports DWORD accesses to LTR latency registers and L1 PM substates control registers:
https://github.com/rajatxjain/public_shared/blob/main/OZ711LV2_appnote.pdf

Since L1 PM substate control registers are DWORD sized, and hence their access in the kernel is already DWORD sized, so we don't need to do anything for them.

Quoted 7 lines
Quoted 5 lines

However, the LTR registers being WORD sized, are in need of a solution. This patch converts the WORD sized accesses to these registers, into DWORD sized accesses, while saving and restoring them.

Applied to pci/enumeration for v5.17, thanks, Rajat!

Thank you.

Quoted 11 lines

The app note suggests that this erratum only affects the registers at 0x234, 0x248, and 0x24c, i.e., the LTR snoop registers and the L1 SS control registers.

Can you confirm that is true? Byte and word accesses to other parts of config space work correctly?

I *assume* the other parts work correctly, because if byte and word accesses were broken, all sorts of things would not work, like PCI_COMMAND, PCI_STATUS, searching the capability list, etc.

Yes, that is correct. The Bayhub SD controller works fine otherwise, so only these registers needed the quirk.

Thanks & Best Regards,

Rajat

Quoted 77 lines

Bjorn

Quoted 74 lines
---
 drivers/pci/pci.c       | 24 ++++++++++++++++--------
 drivers/pci/pcie/aspm.c |  1 +
 2 files changed, 17 insertions(+), 8 deletions(-)
drivers/pci/pci.c +16 -8
index 3d2fb394986a..efa8cd16827f 100644
@@ -1556,7 +1556,7 @@ static void pci_save_ltr_state(struct pci_dev *dev)
 {
      int ltr;
      struct pci_cap_saved_state *save_state;
-     u16 *cap;
+     u32 *cap;
 
      if (!pci_is_pcie(dev))
              return;
@@ -1571,25 +1571,33 @@ static void pci_save_ltr_state(struct pci_dev *dev)
              return;
      }
 
-     cap = (u16 *)&save_state->cap.data[0];
-     pci_read_config_word(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, cap++);
-     pci_read_config_word(dev, ltr + PCI_LTR_MAX_NOSNOOP_LAT, cap++);
+     /*
+      * We deliberately do a dword access to save both PCI_LTR_MAX_SNOOP_LAT
+      * and PCI_LTR_MAX_NOSNOOP_LAT together since some devices only support
+      * dword accesses to these registers.
+      */
+     cap = &save_state->cap.data[0];
+     pci_read_config_dword(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, cap);
 }
 
 static void pci_restore_ltr_state(struct pci_dev *dev)
 {
      struct pci_cap_saved_state *save_state;
      int ltr;
-     u16 *cap;
+     u32 *cap;
 
      save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_LTR);
      ltr = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_LTR);
      if (!save_state || !ltr)
              return;
 
-     cap = (u16 *)&save_state->cap.data[0];
-     pci_write_config_word(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, *cap++);
-     pci_write_config_word(dev, ltr + PCI_LTR_MAX_NOSNOOP_LAT, *cap++);
+     /*
+      * We deliberately do a dword access to restore both
+      * PCI_LTR_MAX_SNOOP_LAT and PCI_LTR_MAX_NOSNOOP_LAT together since
+      * some devices only support dword accesses to these registers.
+      */
+     cap = &save_state->cap.data[0];
+     pci_write_config_dword(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, *cap);
 }
 
 /**
drivers/pci/pcie/aspm.c +1 -1
index 52c74682601a..083f47a7b69b 100644
@@ -496,6 +496,7 @@ static void aspm_calc_l1ss_info(struct pcie_link_state *link,
      encode_l12_threshold(l1_2_threshold, &scale, &value);
      ctl1 |= t_common_mode << 8 | scale << 29 | value << 16;
 
+     /* Always make DWORD sized accesses to these registers */
      pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, &pctl1);
      pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, &pctl2);
      pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, &cctl1);
--

2.34.1.307.g9b7440fafd-goog