Skip to contentSkip to conversation

tegra: PCI Express support

PATCH Superseded Mike Rapoport 7 participants 26 messages Sep 16, 2010 – Sep 20, 2010

Subsystems (3)

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

cover tegra: PCI Express support

These patches enable PCI Express support on Tegra2.
The implementation is based on original NVidia code from (1), but it is heavily reworked to avoid custom PCI enumeration and make the code more Linux friendly.

This implementation assumes that the PCIe subsystem is fully powered and ungated by the bootloader.

[1] git://nv-tegra.nvidia.com/linux-2.6.git

The following changes since commit 9c03f1622af051004416dd3e24d8a0fa31e34178:
Linus Torvalds (1):
Merge ssh://master.kernel.org/home/hpa/tree/sec

Mike Rapoport (3):
[ARM] tegra: add PCI Express clocks
[ARM] tegra: add PCI Express support
[ARM] tegra: harmony: enable PCI Express

 arch/arm/mach-tegra/Kconfig                 |    4 +
 arch/arm/mach-tegra/Makefile                |    2 +
 arch/arm/mach-tegra/board-harmony-pcie.c    |   52 ++
 arch/arm/mach-tegra/board.h                 |    1 +
 arch/arm/mach-tegra/include/mach/hardware.h |    4 +
 arch/arm/mach-tegra/pcie.c                  |  889 +++++++++++++++++++++++++++
 arch/arm/mach-tegra/tegra2_clocks.c         |   76 +++-
 7 files changed, 1025 insertions(+), 3 deletions(-)
 create mode 100644 arch/arm/mach-tegra/board-harmony-pcie.c
 create mode 100644 arch/arm/mach-tegra/pcie.c
#1 lore raw

PATCH 1/3 tegra: add PCI Express clocks

---
 arch/arm/mach-tegra/tegra2_clocks.c |   76 +++++++++++++++++++++++++++++++++-
 1 files changed, 73 insertions(+), 3 deletions(-)
arch/arm/mach-tegra/tegra2_clocks.c +73 -3
index 4261632..44f1b81 100644
@@ -92,6 +92,8 @@
 #define PLLD_MISC_DIV_RST		(1<<23)
 #define PLLD_MISC_DCCON_SHIFT		12
 
+#define PLLE_MISC_READY			(1 << 15)
+
 #define PERIPH_CLK_TO_ENB_REG(c)	((c->clk_num / 32) * 4)
 #define PERIPH_CLK_TO_ENB_SET_REG(c)	((c->clk_num / 32) * 8)
 #define PERIPH_CLK_TO_ENB_BIT(c)	(1 << (c->clk_num % 32))
@@ -356,12 +358,12 @@ static unsigned long tegra2_pll_clk_recalculate_rate(struct clk *c)
 	return c->rate;
 }
 
-static int tegra2_pll_clk_wait_for_lock(struct clk *c)
+static int tegra2_pll_clk_wait_for_bit_set(struct clk *c, int bit)
 {
 	ktime_t before;
 
 	before = ktime_get();
-	while (!(clk_readl(c->reg + PLL_BASE) & PLL_BASE_LOCK)) {
+	while (!(clk_readl(c->reg + PLL_BASE) & bit)) {
 		if (ktime_us_delta(ktime_get(), before) > 5000) {
 			pr_err("Timed out waiting for lock bit on pll %s",
 				c->name);
@@ -414,7 +416,7 @@ static int tegra2_pll_clk_enable(struct clk *c)
 	val |= PLL_MISC_LOCK_ENABLE;
 	clk_writel(val, c->reg + PLL_MISC(c));
 
-	tegra2_pll_clk_wait_for_lock(c);
+	tegra2_pll_clk_wait_for_bit_set(c, PLL_BASE_LOCK);
 
 	return 0;
 }
@@ -754,6 +756,34 @@ static struct clk_ops tegra_clk_double_ops = {
 	.recalculate_rate	= &tegra2_clk_recalculate_rate,
 };
 
+/* PCI Express clock ops */
+static int tegra2_plle_clk_enable(struct clk *c)
+{
+	u32 val;
+
+	pr_debug("%s on clock %s\n", __func__, c->name);
+
+	if (tegra2_pll_clk_wait_for_bit_set(c, PLLE_MISC_READY))
+		return -EBUSY;
+
+	val = clk_readl(c->reg + PLL_BASE);
+	val |= PLL_BASE_ENABLE | PLL_BASE_BYPASS;
+	clk_writel(val, c->reg + PLL_BASE);
+
+	return 0;
+}
+
+static struct clk_ops tegra_plle_ops = {
+	.init       = tegra2_pll_clk_init,
+	.enable     = tegra2_plle_clk_enable,
+	.set_rate   = tegra2_pll_clk_set_rate,
+};
+
+static struct clk_ops tegra_pcie_clk_ops = {
+	.enable	    = tegra2_periph_clk_enable,
+	.disable    = tegra2_periph_clk_disable,
+};
+
 /* Clock definitions */
 static struct clk tegra_clk_32k = {
 	.name = "clk_32k",
@@ -1109,6 +1139,42 @@ static struct clk tegra_clk_pclk = {
 	.ops		= &tegra_bus_ops,
 };
 
+/* PCI Express clocks */
+static struct clk_pll_table tegra_pll_e_table[] = {
+       { 12000000, 100000000,  200,  24, 1, 0 },
+};
+
+static struct clk tegra_pll_e = {
+	.name      = "pll_e",
+	.flags	   = PLL_ALT_MISC_REG,
+	.ops       = &tegra_plle_ops,
+	.input_min = 12000000,
+	.input_max = 12000000,
+	.parent    = &tegra_clk_m,
+	.reg       = 0xe8,
+	.pll_table = tegra_pll_e_table,
+};
+
+static struct clk tegra_clk_pex = {
+	.name      = "pex",
+	.flags     = PERIPH_MANUAL_RESET,
+	.ops       = &tegra_pcie_clk_ops,
+	.clk_num   = 70,
+};
+
+static struct clk tegra_clk_afi = {
+	.name      = "afi",
+	.flags     = PERIPH_MANUAL_RESET,
+	.ops       = &tegra_pcie_clk_ops,
+	.clk_num   = 72,
+};
+
+/* the pcie_xclk is required for reset of PCIE subsystem */
+static struct clk tegra_clk_pcie_xclk = {
+	.name      = "pcie_xclk",
+	.clk_num   = 74,
+};
+
 static struct clk_mux_sel mux_pllm_pllc_pllp_plla[] = {
 	{ .input = &tegra_pll_m, .value = 0},
 	{ .input = &tegra_pll_c, .value = 1},
@@ -1315,11 +1381,15 @@ struct clk_lookup tegra_clk_lookups[] = {
 	CLK(NULL,	"pll_d_out0",	&tegra_pll_d_out0),
 	CLK(NULL,	"pll_u",	&tegra_pll_u),
 	CLK(NULL,	"pll_x",	&tegra_pll_x),
+	CLK(NULL,	"pll_e",	&tegra_pll_e),
 	CLK(NULL,	"cpu",		&tegra_clk_cpu),
 	CLK(NULL,	"sys",		&tegra_clk_sys),
 	CLK(NULL,	"hclk",		&tegra_clk_hclk),
 	CLK(NULL,	"pclk",		&tegra_clk_pclk),
 	CLK(NULL,	"clk_d",	&tegra_clk_d),
+	CLK(NULL,	"pcie_xclk",	&tegra_clk_pcie_xclk),
+	CLK(NULL,	"pex",		&tegra_clk_pex),
+	CLK(NULL,	"afi",		&tegra_clk_afi),
 };
 
 void __init tegra2_init_clocks(void)
Signature · 1 line
1.6.6.2
#2 lore raw

PATCH 1/3 tegra: add PCI Express clocks

Quoted 4 lines · Mike Rapoport

---
?arch/arm/mach-tegra/tegra2_clocks.c | ? 76 +++++++++++++++++++++++++++++++++- ?1 files changed, 73 insertions(+), 3 deletions(-)

<snip>

Quoted 12 lines
arch/arm/mach-tegra/tegra2_clocks.c +1 -1
-static int tegra2_pll_clk_wait_for_lock(struct clk *c)
+static int tegra2_pll_clk_wait_for_bit_set(struct clk *c, int bit)

?{
? ? ? ?ktime_t before;

? ? ? ?before = ktime_get();
- ? ? ? while (!(clk_readl(c->reg + PLL_BASE) & PLL_BASE_LOCK)) {
+ ? ? ? while (!(clk_readl(c->reg + PLL_BASE) & bit)) {
? ? ? ? ? ? ? ?if (ktime_us_delta(ktime_get(), before) > 5000) { ? ? ? ? ? ? ? ? ? ? ? ?pr_err("Timed out waiting for lock bit on pll %s", ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?c->name);

Review static int tegra2_pll_clk_enable(struct clk *c) · line 422

Quoted 8 lines
(quoted diff) +1 -1
@@ -414,7 +416,7 @@ static int tegra2_pll_clk_enable(struct clk *c)
 ? ? ? ?val |= PLL_MISC_LOCK_ENABLE;
 ? ? ? ?clk_writel(val, c->reg + PLL_MISC(c));
 
- ? ? ? tegra2_pll_clk_wait_for_lock(c);
+ ? ? ? tegra2_pll_clk_wait_for_bit_set(c, PLL_BASE_LOCK);
 
 ? ? ? ?return 0;
 ?}

I've been told by Nvidia that the pll lock bits are not reliable, and will be removing uses of tegra2_pll_clk_wait_for_lock and replacing them with delays selected by each pll. Can you confirm with Nvidia that PLLE_MISC_READY works?

<snip>

arch/arm/mach-tegra/tegra2_clocks.c static struct clk_ops tegra_clk_double_ops = { · line 785

Quoted 4 lines
arch/arm/mach-tegra/tegra2_clocks.c +4 -0
+static struct clk_ops tegra_pcie_clk_ops = {
+ ? ? ? .enable ? ? = tegra2_periph_clk_enable,
+ ? ? ? .disable ? ?= tegra2_periph_clk_disable,
+};

Why is this needed? Won't the regular periph ops work?

<snip>

Quoted 9 lines
(quoted diff) +9 -0
+static struct clk tegra_clk_pex = {
+ ? ? ? .name ? ? ?= "pex",
+ ? ? ? .flags ? ? = PERIPH_MANUAL_RESET,
+ ? ? ? .ops ? ? ? = &tegra_pcie_clk_ops,
+ ? ? ? .clk_num ? = 70,
+};
+
+static struct clk tegra_clk_afi = {
+ ? ? ? .name ? ? ?= "afi",

Review

Quoted 10 lines
(quoted diff) +10 -0
+ ? ? ? .flags ? ? = PERIPH_MANUAL_RESET,
+ ? ? ? .ops ? ? ? = &tegra_pcie_clk_ops,
+ ? ? ? .clk_num ? = 72,
+};
+
+/* the pcie_xclk is required for reset of PCIE subsystem */
+static struct clk tegra_clk_pcie_xclk = {
+ ? ? ? .name ? ? ?= "pcie_xclk",
+ ? ? ? .clk_num ? = 74,
+};

These should probably all be defined in the CLK_PERIPH table.

#3 lore raw

PATCH 1/3 tegra: add PCI Express clocks

Quoted 26 lines · Colin Cross
Quoted 4 lines · Mike Rapoport

---
?arch/arm/mach-tegra/tegra2_clocks.c | ? 76 +++++++++++++++++++++++++++++++++- ?1 files changed, 73 insertions(+), 3 deletions(-)

<snip>

Quoted 18 lines
arch/arm/mach-tegra/tegra2_clocks.c +1 -1
-static int tegra2_pll_clk_wait_for_lock(struct clk *c)
+static int tegra2_pll_clk_wait_for_bit_set(struct clk *c, int bit)

?{
? ? ? ?ktime_t before;

? ? ? ?before = ktime_get();
- ? ? ? while (!(clk_readl(c->reg + PLL_BASE) & PLL_BASE_LOCK)) {
+ ? ? ? while (!(clk_readl(c->reg + PLL_BASE) & bit)) {
? ? ? ? ? ? ? ?if (ktime_us_delta(ktime_get(), before) > 5000) { ? ? ? ? ? ? ? ? ? ? ? ?pr_err("Timed out waiting for lock bit on pll %s", ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?c->name);

(quoted diff) +1 -1
@@ -414,7 +416,7 @@ static int tegra2_pll_clk_enable(struct clk *c)
 ? ? ? ?val |= PLL_MISC_LOCK_ENABLE;
 ? ? ? ?clk_writel(val, c->reg + PLL_MISC(c));
 
- ? ? ? tegra2_pll_clk_wait_for_lock(c);
+ ? ? ? tegra2_pll_clk_wait_for_bit_set(c, PLL_BASE_LOCK);
 
Quoted 7 lines

Review static int tegra2_pll_clk_enable(struct clk *c) · line 422

Quoted 2 lines
(quoted diff) +0 -0
 ? ? ? ?return 0;
 ?}

I've been told by Nvidia that the pll lock bits are not reliable, and will be removing uses of tegra2_pll_clk_wait_for_lock and replacing them with delays selected by each pll. ?Can you confirm with Nvidia that PLLE_MISC_READY works?

I'll try to get a confirmation from NVidia....

#4 lore raw

PATCH 1/3 tegra: add PCI Express clocks

Quoted 2 lines · Colin Cross
Quoted 1 line · Mike Rapoport

<snip>

Quoted 9 lines
Quoted 4 lines

I've been told by Nvidia that the pll lock bits are not reliable, and will be removing uses of tegra2_pll_clk_wait_for_lock and replacing them with delays selected by each pll. Can you confirm with Nvidia that PLLE_MISC_READY works?

I'll try to get a confirmation from NVidia....
From what I've seen during testing the PLLE_MISC_READY works. When I used loops that counted udelay(1) I've noticed that the PLLE_MISC_READY is set after up to 3usecs.

All of the PLL lock bits in Tegra 2 are unreliable.

The safe way to ensure that PLLE is stable is to wait 1ms after changing it.

Quoted 11 lines
Quoted 7 lines

<snip>

arch/arm/mach-tegra/tegra2_clocks.c static struct clk_ops tegra_clk_double_ops = { · line 785

Quoted 4 lines
arch/arm/mach-tegra/tegra2_clocks.c +4 -0
+static struct clk_ops tegra_pcie_clk_ops = {
+       .enable     = tegra2_periph_clk_enable,
+       .disable    = tegra2_periph_clk_disable,
+};

Why is this needed? Won't the regular periph ops work?

They didn't. I haven't found anything about what feeds these clocks, can they change rate, what clock can be their parent and if there is any muxing options for these clocks.

You can follow the example used by the csi and isp clocks, which also only support enable and disable. give the PCI clocks a single-choice parent mux (like mux_clk_m), flags = 0, reg = 0, and a dummy max_rate.

- Gary

#5 lore raw

PATCH 1/3 tegra: add PCI Express clocks

Quoted 10 lines · Gary King
Quoted 2 lines · Colin Cross
Quoted 1 line · Mike Rapoport

<snip>

Quoted 4 lines
Quoted 4 lines

I've been told by Nvidia that the pll lock bits are not reliable, and will be removing uses of tegra2_pll_clk_wait_for_lock and replacing them with delays selected by each pll. Can you confirm with Nvidia that PLLE_MISC_READY works?

Quoted 8 lines
Quoted 4 lines

I'll try to get a confirmation from NVidia....
From what I've seen during testing the PLLE_MISC_READY works. When I used loops that counted udelay(1) I've noticed that the PLLE_MISC_READY is set after up to 3usecs.

All of the PLL lock bits in Tegra 2 are unreliable.

The safe way to ensure that PLLE is stable is to wait 1ms after changing it.

1 millisecond is a long time... Can you please elaborate about the unreliability of PLL lock bits?

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

<snip>

arch/arm/mach-tegra/tegra2_clocks.c static struct clk_ops tegra_clk_double_ops = { · line 785

Quoted 4 lines
arch/arm/mach-tegra/tegra2_clocks.c +4 -0
+static struct clk_ops tegra_pcie_clk_ops = {
+       .enable     = tegra2_periph_clk_enable,
+       .disable    = tegra2_periph_clk_disable,
+};

Why is this needed? Won't the regular periph ops work?

Quoted 7 lines
Quoted 3 lines

They didn't. I haven't found anything about what feeds these clocks, can they change rate, what clock can be their parent and if there is any muxing options for these clocks.

You can follow the example used by the csi and isp clocks, which also only support enable and disable. give the PCI clocks a single-choice parent mux (like mux_clk_m), flags = 0, reg = 0, and a dummy max_rate.

Ok, I'll try.

Quoted 4 lines

- Gary--
To unsubscribe from this list: send the line "unsubscribe linux-tegra" in the body of a message to majordomo at vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html

Signature · 2 lines
Sincerely yours,
Mike.
#6 lore raw

PATCH 1/3 tegra: add PCI Express clocks

In reply to Mike Rapoport

Good evening Mike Rapport,

Are there is some datasheet/TRM availability for Nvidia Tegra chip ?
Please say me if yes and where.

thanks

Quoted 145 lines · Mike Rapoport

---
?arch/arm/mach-tegra/tegra2_clocks.c | ? 76 +++++++++++++++++++++++++++++++++- ?1 files changed, 73 insertions(+), 3 deletions(-)

arch/arm/mach-tegra/tegra2_clocks.c +0 -0
index 4261632..44f1b81 100644
@@ -92,6 +92,8 @@
 ?#define PLLD_MISC_DIV_RST ? ? ? ? ? ? ?(1<<23)

?#define PLLD_MISC_DCCON_SHIFT ? ? ? ? ?12

arch/arm/mach-tegra/tegra2_clocks.c +2 -0
+#define PLLE_MISC_READY ? ? ? ? ? ? ? ? ? ? ? ?(1 << 15)
+

?#define PERIPH_CLK_TO_ENB_REG(c) ? ? ? ((c->clk_num / 32) * 4) ?#define PERIPH_CLK_TO_ENB_SET_REG(c) ? ((c->clk_num / 32) * 8) ?#define PERIPH_CLK_TO_ENB_BIT(c) ? ? ? (1 << (c->clk_num % 32))

arch/arm/mach-tegra/tegra2_clocks.c +31 -3
@@ -356,12 +358,12 @@ static unsigned long tegra2_pll_clk_recalculate_rate(struct clk *c)
 ? ? ? ?return c->rate;
 ?}
 
-static int tegra2_pll_clk_wait_for_lock(struct clk *c)
+static int tegra2_pll_clk_wait_for_bit_set(struct clk *c, int bit)
 ?{
 ? ? ? ?ktime_t before;
 
 ? ? ? ?before = ktime_get();
- ? ? ? while (!(clk_readl(c->reg + PLL_BASE) & PLL_BASE_LOCK)) {
+ ? ? ? while (!(clk_readl(c->reg + PLL_BASE) & bit)) {
 ? ? ? ? ? ? ? ?if (ktime_us_delta(ktime_get(), before) > 5000) {
 ? ? ? ? ? ? ? ? ? ? ? ?pr_err("Timed out waiting for lock bit on pll %s",
 ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?c->name);
@@ -414,7 +416,7 @@ static int tegra2_pll_clk_enable(struct clk *c)
 ? ? ? ?val |= PLL_MISC_LOCK_ENABLE;
 ? ? ? ?clk_writel(val, c->reg + PLL_MISC(c));
 
- ? ? ? tegra2_pll_clk_wait_for_lock(c);
+ ? ? ? tegra2_pll_clk_wait_for_bit_set(c, PLL_BASE_LOCK);
 
 ? ? ? ?return 0;
 ?}
@@ -754,6 +756,34 @@ static struct clk_ops tegra_clk_double_ops = {
 ? ? ? ?.recalculate_rate ? ? ? = &tegra2_clk_recalculate_rate,
 ?};
 
+/* PCI Express clock ops */
+static int tegra2_plle_clk_enable(struct clk *c)
+{
+ ? ? ? u32 val;
+
+ ? ? ? pr_debug("%s on clock %s\n", __func__, c->name);
+
+ ? ? ? if (tegra2_pll_clk_wait_for_bit_set(c, PLLE_MISC_READY))
+ ? ? ? ? ? ? ? return -EBUSY;
+
+ ? ? ? val = clk_readl(c->reg + PLL_BASE);
+ ? ? ? val |= PLL_BASE_ENABLE | PLL_BASE_BYPASS;
+ ? ? ? clk_writel(val, c->reg + PLL_BASE);
+
+ ? ? ? return 0;
+}
+
+static struct clk_ops tegra_plle_ops = {
+ ? ? ? .init ? ? ? = tegra2_pll_clk_init,
+ ? ? ? .enable ? ? = tegra2_plle_clk_enable,
+ ? ? ? .set_rate ? = tegra2_pll_clk_set_rate,
+};
+
+static struct clk_ops tegra_pcie_clk_ops = {
+ ? ? ? .enable ? ? = tegra2_periph_clk_enable,
+ ? ? ? .disable ? ?= tegra2_periph_clk_disable,
+};
+

?/* Clock definitions */
?static struct clk tegra_clk_32k = {
? ? ? ?.name = "clk_32k",

arch/arm/mach-tegra/tegra2_clocks.c +40 -1
@@ -1109,6 +1139,42 @@ static struct clk tegra_clk_pclk = {
 ? ? ? ?.ops ? ? ? ? ? ?= &tegra_bus_ops,
 ?};
 
+/* PCI Express clocks */
+static struct clk_pll_table tegra_pll_e_table[] = {
+ ? ? ? { 12000000, 100000000, ?200, ?24, 1, 0 },
+};
+
+static struct clk tegra_pll_e = {
+ ? ? ? .name ? ? ?= "pll_e",
+ ? ? ? .flags ? ? = PLL_ALT_MISC_REG,
+ ? ? ? .ops ? ? ? = &tegra_plle_ops,
+ ? ? ? .input_min = 12000000,
+ ? ? ? .input_max = 12000000,
+ ? ? ? .parent ? ?= &tegra_clk_m,
+ ? ? ? .reg ? ? ? = 0xe8,
+ ? ? ? .pll_table = tegra_pll_e_table,
+};
+
+static struct clk tegra_clk_pex = {
+ ? ? ? .name ? ? ?= "pex",
+ ? ? ? .flags ? ? = PERIPH_MANUAL_RESET,
+ ? ? ? .ops ? ? ? = &tegra_pcie_clk_ops,
+ ? ? ? .clk_num ? = 70,
+};
+
+static struct clk tegra_clk_afi = {
+ ? ? ? .name ? ? ?= "afi",
+ ? ? ? .flags ? ? = PERIPH_MANUAL_RESET,
+ ? ? ? .ops ? ? ? = &tegra_pcie_clk_ops,
+ ? ? ? .clk_num ? = 72,
+};
+
+/* the pcie_xclk is required for reset of PCIE subsystem */
+static struct clk tegra_clk_pcie_xclk = {
+ ? ? ? .name ? ? ?= "pcie_xclk",
+ ? ? ? .clk_num ? = 74,
+};
+
 ?static struct clk_mux_sel mux_pllm_pllc_pllp_plla[] = {
 ? ? ? ?{ .input = &tegra_pll_m, .value = 0},
 ? ? ? ?{ .input = &tegra_pll_c, .value = 1},
@@ -1315,11 +1381,15 @@ struct clk_lookup tegra_clk_lookups[] = {
 ? ? ? ?CLK(NULL, ? ? ? "pll_d_out0", ? &tegra_pll_d_out0),
 ? ? ? ?CLK(NULL, ? ? ? "pll_u", ? ? ? ?&tegra_pll_u),
 ? ? ? ?CLK(NULL, ? ? ? "pll_x", ? ? ? ?&tegra_pll_x),
+ ? ? ? CLK(NULL, ? ? ? "pll_e", ? ? ? ?&tegra_pll_e),
 ? ? ? ?CLK(NULL, ? ? ? "cpu", ? ? ? ? ?&tegra_clk_cpu),
 ? ? ? ?CLK(NULL, ? ? ? "sys", ? ? ? ? ?&tegra_clk_sys),
 ? ? ? ?CLK(NULL, ? ? ? "hclk", ? ? ? ? &tegra_clk_hclk),
 ? ? ? ?CLK(NULL, ? ? ? "pclk", ? ? ? ? &tegra_clk_pclk),
 ? ? ? ?CLK(NULL, ? ? ? "clk_d", ? ? ? ?&tegra_clk_d),
+ ? ? ? CLK(NULL, ? ? ? "pcie_xclk", ? ?&tegra_clk_pcie_xclk),
+ ? ? ? CLK(NULL, ? ? ? "pex", ? ? ? ? ?&tegra_clk_pex),
+ ? ? ? CLK(NULL, ? ? ? "afi", ? ? ? ? ?&tegra_clk_afi),
 ?};
 
 ?void __init tegra2_init_clocks(void)
--

1.6.6.2

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel at lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

#7 lore raw

PATCH 1/3 tegra: add PCI Express clocks

Quoted 4 lines · Mogambo Park

Good evening Mike Rapport,

Are there is some datasheet/TRM availability for Nvidia Tegra chip ?
Please say me if yes and where.

The Tegra docs are available only under NDA.

Quoted 149 lines

thanks

Quoted 145 lines · Mike Rapoport
---
 arch/arm/mach-tegra/tegra2_clocks.c |   76 +++++++++++++++++++++++++++++++++-
 1 files changed, 73 insertions(+), 3 deletions(-)
arch/arm/mach-tegra/tegra2_clocks.c +73 -4
index 4261632..44f1b81 100644
@@ -92,6 +92,8 @@
 #define PLLD_MISC_DIV_RST              (1<<23)
 #define PLLD_MISC_DCCON_SHIFT          12
 
+#define PLLE_MISC_READY                        (1 << 15)
+
 #define PERIPH_CLK_TO_ENB_REG(c)       ((c->clk_num / 32) * 4)
 #define PERIPH_CLK_TO_ENB_SET_REG(c)   ((c->clk_num / 32) * 8)
 #define PERIPH_CLK_TO_ENB_BIT(c)       (1 << (c->clk_num % 32))
@@ -356,12 +358,12 @@ static unsigned long tegra2_pll_clk_recalculate_rate(struct clk *c)
       return c->rate;
 }
 
-static int tegra2_pll_clk_wait_for_lock(struct clk *c)
+static int tegra2_pll_clk_wait_for_bit_set(struct clk *c, int bit)
 {
       ktime_t before;
 
       before = ktime_get();
-       while (!(clk_readl(c->reg + PLL_BASE) & PLL_BASE_LOCK)) {
+       while (!(clk_readl(c->reg + PLL_BASE) & bit)) {
               if (ktime_us_delta(ktime_get(), before) > 5000) {
                       pr_err("Timed out waiting for lock bit on pll %s",
                               c->name);
@@ -414,7 +416,7 @@ static int tegra2_pll_clk_enable(struct clk *c)
       val |= PLL_MISC_LOCK_ENABLE;
       clk_writel(val, c->reg + PLL_MISC(c));
 
-       tegra2_pll_clk_wait_for_lock(c);
+       tegra2_pll_clk_wait_for_bit_set(c, PLL_BASE_LOCK);
 
       return 0;
 }
@@ -754,6 +756,34 @@ static struct clk_ops tegra_clk_double_ops = {
       .recalculate_rate       = &tegra2_clk_recalculate_rate,
 };
 
+/* PCI Express clock ops */
+static int tegra2_plle_clk_enable(struct clk *c)
+{
+       u32 val;
+
+       pr_debug("%s on clock %s\n", __func__, c->name);
+
+       if (tegra2_pll_clk_wait_for_bit_set(c, PLLE_MISC_READY))
+               return -EBUSY;
+
+       val = clk_readl(c->reg + PLL_BASE);
+       val |= PLL_BASE_ENABLE | PLL_BASE_BYPASS;
+       clk_writel(val, c->reg + PLL_BASE);
+
+       return 0;
+}
+
+static struct clk_ops tegra_plle_ops = {
+       .init       = tegra2_pll_clk_init,
+       .enable     = tegra2_plle_clk_enable,
+       .set_rate   = tegra2_pll_clk_set_rate,
+};
+
+static struct clk_ops tegra_pcie_clk_ops = {
+       .enable     = tegra2_periph_clk_enable,
+       .disable    = tegra2_periph_clk_disable,
+};
+
 /* Clock definitions */
 static struct clk tegra_clk_32k = {
       .name = "clk_32k",
@@ -1109,6 +1139,42 @@ static struct clk tegra_clk_pclk = {
       .ops            = &tegra_bus_ops,
 };
 
+/* PCI Express clocks */
+static struct clk_pll_table tegra_pll_e_table[] = {
+       { 12000000, 100000000,  200,  24, 1, 0 },
+};
+
+static struct clk tegra_pll_e = {
+       .name      = "pll_e",
+       .flags     = PLL_ALT_MISC_REG,
+       .ops       = &tegra_plle_ops,
+       .input_min = 12000000,
+       .input_max = 12000000,
+       .parent    = &tegra_clk_m,
+       .reg       = 0xe8,
+       .pll_table = tegra_pll_e_table,
+};
+
+static struct clk tegra_clk_pex = {
+       .name      = "pex",
+       .flags     = PERIPH_MANUAL_RESET,
+       .ops       = &tegra_pcie_clk_ops,
+       .clk_num   = 70,
+};
+
+static struct clk tegra_clk_afi = {
+       .name      = "afi",
+       .flags     = PERIPH_MANUAL_RESET,
+       .ops       = &tegra_pcie_clk_ops,
+       .clk_num   = 72,
+};
+
+/* the pcie_xclk is required for reset of PCIE subsystem */
+static struct clk tegra_clk_pcie_xclk = {
+       .name      = "pcie_xclk",
+       .clk_num   = 74,
+};
+
 static struct clk_mux_sel mux_pllm_pllc_pllp_plla[] = {
       { .input = &tegra_pll_m, .value = 0},
       { .input = &tegra_pll_c, .value = 1},
@@ -1315,11 +1381,15 @@ struct clk_lookup tegra_clk_lookups[] = {
       CLK(NULL,       "pll_d_out0",   &tegra_pll_d_out0),
       CLK(NULL,       "pll_u",        &tegra_pll_u),
       CLK(NULL,       "pll_x",        &tegra_pll_x),
+       CLK(NULL,       "pll_e",        &tegra_pll_e),
       CLK(NULL,       "cpu",          &tegra_clk_cpu),
       CLK(NULL,       "sys",          &tegra_clk_sys),
       CLK(NULL,       "hclk",         &tegra_clk_hclk),
       CLK(NULL,       "pclk",         &tegra_clk_pclk),
       CLK(NULL,       "clk_d",        &tegra_clk_d),
+       CLK(NULL,       "pcie_xclk",    &tegra_clk_pcie_xclk),
+       CLK(NULL,       "pex",          &tegra_clk_pex),
+       CLK(NULL,       "afi",          &tegra_clk_afi),
 };
 
 void __init tegra2_init_clocks(void)
--

1.6.6.2

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel at lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Signature · 2 lines
Sincerely yours,
Mike.
#8 lore raw

PATCH 2/3 tegra: add PCI Express support

In reply to Mike Rapoport
---
 arch/arm/mach-tegra/Kconfig                 |    4 +
 arch/arm/mach-tegra/Makefile                |    1 +
 arch/arm/mach-tegra/board.h                 |    1 +
 arch/arm/mach-tegra/include/mach/hardware.h |    4 +
 arch/arm/mach-tegra/pcie.c                  |  889 +++++++++++++++++++++++++++
 5 files changed, 899 insertions(+), 0 deletions(-)
 create mode 100644 arch/arm/mach-tegra/pcie.c
arch/arm/mach-tegra/Kconfig +4 -0
index a57713c..5f09846 100644
@@ -16,6 +16,10 @@ config ARCH_TEGRA_2x_SOC
 
 endchoice
 
+config TEGRA_PCI
+	bool "PCI Express support"
+	select PCI
+
 comment "Tegra board type"
 
 config MACH_HARMONY
arch/arm/mach-tegra/Makefile +1 -0
index 51e9370..23c9600 100644
@@ -9,6 +9,7 @@ obj-$(CONFIG_ARCH_TEGRA_2x_SOC)         += clock.o
 obj-$(CONFIG_ARCH_TEGRA_2x_SOC)         += tegra2_clocks.o
 obj-$(CONFIG_SMP)                       += platsmp.o localtimer.o headsmp.o
 obj-$(CONFIG_HOTPLUG_CPU)               += hotplug.o
+obj-$(CONFIG_TEGRA_PCI)			+= pcie.o
 
 obj-${CONFIG_MACH_HARMONY}              += board-harmony.o
 obj-${CONFIG_MACH_HARMONY}              += board-harmony-pinmux.o
arch/arm/mach-tegra/board.h +1 -0
index 3d06354..0de565c 100644
@@ -27,6 +27,7 @@ void __init tegra_common_init(void);
 void __init tegra_map_common_io(void);
 void __init tegra_init_irq(void);
 void __init tegra_init_clock(void);
+int __init tegra_pcie_init(bool init_port0, bool init_port1);
 
 extern struct sys_timer tegra_timer;
 #endif
arch/arm/mach-tegra/include/mach/hardware.h +4 -0
index 6014edf..76f3478 100644
@@ -21,4 +21,8 @@
 #ifndef __MACH_TEGRA_HARDWARE_H
 #define __MACH_TEGRA_HARDWARE_H
 
+#define PCIBIOS_MIN_IO			0
+#define PCIBIOS_MIN_MEM			0
+#define pcibios_assign_all_busses()	1
+
 #endif
arch/arm/mach-tegra/pcie.c new file +889 -0

889 lines not on this page · raw message

Signature · 1 line
1.6.6.2
#9 lore raw

PATCH 2/3 tegra: add PCI Express support

Quoted 10 lines · Mike Rapoport

---

?arch/arm/mach-tegra/Kconfig ? ? ? ? ? ? ? ? | ? ?4 +
?arch/arm/mach-tegra/Makefile ? ? ? ? ? ? ? ?| ? ?1 +
?arch/arm/mach-tegra/board.h ? ? ? ? ? ? ? ? | ? ?1 +
?arch/arm/mach-tegra/include/mach/hardware.h | ? ?4 +
?arch/arm/mach-tegra/pcie.c ? ? ? ? ? ? ? ? ?| ?889 +++++++++++++++++++++++++++

?5 files changed, 899 insertions(+), 0 deletions(-) ?create mode 100644 arch/arm/mach-tegra/pcie.c

<snip>

Review

Quoted 3 lines
(quoted diff) +3 -0
+/* register definitions */
+#define AFI_OFSET ? ? ?0x3800
+#define PADS_OFSET ? ? 0x3000

OFFSET

Quoted 2 lines
(quoted diff) +2 -0
+#define RP0_OFFSET ? ? 0x0000
+#define RP1_OFFSET ? ? 0x1000

<snip>

arch/arm/mach-tegra/pcie.c line 675

Quoted 11 lines
arch/arm/mach-tegra/pcie.c new file +11 -0
+static void tegra_pcie_xclk_clamp(bool clamp)
+{
+ ? ? ? u32 reg;
+
+ ? ? ? reg = pmc_readl(PMC_SCRATCH42) & ~PMC_SCRATCH42_PCX_CLAMP;
+
+ ? ? ? if (clamp)
+ ? ? ? ? ? ? ? reg |= PMC_SCRATCH42_PCX_CLAMP;
+
+ ? ? ? pmc_writel(reg, PMC_SCRATCH42);
+}

Maybe add this to mach-tegra/powergate.c, to avoid having to set PMC registers directly?

Review

Quoted 12 lines
(quoted diff) +12 -0
+static int tegra_pcie_power_on(void)
+{
+ ? ? ? int err;
+
+ ? ? ? tegra_pcie_xclk_clamp(true);
+ ? ? ? tegra_periph_reset_assert(tegra_pcie.pcie_xclk);
+ ? ? ? tegra_pcie_xclk_clamp(false);
+
+ ? ? ? clk_enable(tegra_pcie.afi_clk);
+ ? ? ? clk_enable(tegra_pcie.pex_clk);
+ ? ? ? return clk_enable(tegra_pcie.pll_e);
+}

Any reason you can't use the powergate api here too?

arch/arm/mach-tegra/pcie.c line 697

Quoted 8 lines
arch/arm/mach-tegra/pcie.c new file +8 -0
+static void tegra_pcie_power_off(void)
+{
+ ? ? ? tegra_periph_reset_assert(tegra_pcie.pcie_xclk);
+ ? ? ? tegra_periph_reset_assert(tegra_pcie.afi_clk);
+ ? ? ? tegra_periph_reset_assert(tegra_pcie.pex_clk);
+
+ ? ? ? tegra_pcie_xclk_clamp(true);
+}

Can you power off the pcie partition?

#10 lore raw

PATCH 2/3 tegra: add PCI Express support

Quoted 11 lines · Colin Cross
Quoted 10 lines · Mike Rapoport

---

?arch/arm/mach-tegra/Kconfig ? ? ? ? ? ? ? ? | ? ?4 +
?arch/arm/mach-tegra/Makefile ? ? ? ? ? ? ? ?| ? ?1 +
?arch/arm/mach-tegra/board.h ? ? ? ? ? ? ? ? | ? ?1 +
?arch/arm/mach-tegra/include/mach/hardware.h | ? ?4 +
?arch/arm/mach-tegra/pcie.c ? ? ? ? ? ? ? ? ?| ?889 +++++++++++++++++++++++++++

?5 files changed, 899 insertions(+), 0 deletions(-) ?create mode 100644 arch/arm/mach-tegra/pcie.c

Quoted 6 lines

<snip>

Review

Quoted 3 lines
(quoted diff) +3 -0
+/* register definitions */
+#define AFI_OFSET ? ? ?0x3800
+#define PADS_OFSET ? ? 0x3000

OFFSET

thanks, will fix

Quoted 5 lines
Quoted 2 lines
(quoted diff) +2 -0
+#define RP0_OFFSET ? ? 0x0000
+#define RP1_OFFSET ? ? 0x1000

<snip>

Quoted 14 lines

arch/arm/mach-tegra/pcie.c line 675

Quoted 11 lines
arch/arm/mach-tegra/pcie.c new file +11 -0
+static void tegra_pcie_xclk_clamp(bool clamp)
+{
+ ? ? ? u32 reg;
+
+ ? ? ? reg = pmc_readl(PMC_SCRATCH42) & ~PMC_SCRATCH42_PCX_CLAMP;
+
+ ? ? ? if (clamp)
+ ? ? ? ? ? ? ? reg |= PMC_SCRATCH42_PCX_CLAMP;
+
+ ? ? ? pmc_writel(reg, PMC_SCRATCH42);
+}

Maybe add this to mach-tegra/powergate.c, to avoid having to set PMC registers directly?

The patch is aginst Linus' tree, that does not have powergate yet. I'd really like to see the PCIe in 2.6.37-rc1, so I would prefer not to depend on powergate here.
Besides, this particular bit is PCIe specific and does not fit into generic power gating framework.
As a side note, mach-tegra/irq.c and mach-tegra/suspend*.c also have direct access to the PMC, so probably it's worth adding mach-tegra/pmc.h with pmc_writel/pmc_readl definitions?

Quoted 5 lines
Quoted 5 lines
arch/arm/mach-tegra/pcie.c new file +5 -0
+static int tegra_pcie_power_on(void)
+{
+ ? ? ? int err;
+
+ ? ? ? tegra_pcie_xclk_clamp(true);
Quoted 8 lines

Review

Quoted 7 lines

+ ? ? ? tegra_periph_reset_assert(tegra_pcie.pcie_xclk);
+ ? ? ? tegra_pcie_xclk_clamp(false);

(quoted diff) +5 -0
+
+ ? ? ? clk_enable(tegra_pcie.afi_clk);
+ ? ? ? clk_enable(tegra_pcie.pex_clk);
+ ? ? ? return clk_enable(tegra_pcie.pll_e);
+}

Any reason you can't use the powergate api here too?

I cannot use the powergate api instead of what I have now, but it won't be a problem to add it on top of current implementation. The PCIe subsystem uses too many clocks, which leads to necessity to have combine powergate api with direct calls to clk_{en,dis}able and tegra_periph_reset_{de}assert.

Quoted 9 lines

arch/arm/mach-tegra/pcie.c line 697

Quoted 8 lines
arch/arm/mach-tegra/pcie.c new file +8 -0
+static void tegra_pcie_power_off(void)
+{
+ ? ? ? tegra_periph_reset_assert(tegra_pcie.pcie_xclk);
+ ? ? ? tegra_periph_reset_assert(tegra_pcie.afi_clk);
+ ? ? ? tegra_periph_reset_assert(tegra_pcie.pex_clk);
+
+ ? ? ? tegra_pcie_xclk_clamp(true);
+}

Can you power off the pcie partition?

Yes, but again, I prefer not to depend on powergate api. The power gating can be easily added afterwads.

Quoted 6 lines

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel at lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Signature · 2 lines
? ? Sincerely Yours,
? ? ? ? Mike.
#11 lore raw

PATCH 3/3 tegra: harmony: enable PCI Express

In reply to Mike Rapoport
---
 arch/arm/mach-tegra/Makefile             |    1 +
 arch/arm/mach-tegra/board-harmony-pcie.c |   52 ++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 0 deletions(-)
 create mode 100644 arch/arm/mach-tegra/board-harmony-pcie.c
arch/arm/mach-tegra/Makefile +1 -0
index 23c9600..60f73c7 100644
@@ -13,3 +13,4 @@ obj-$(CONFIG_TEGRA_PCI)			+= pcie.o
 
 obj-${CONFIG_MACH_HARMONY}              += board-harmony.o
 obj-${CONFIG_MACH_HARMONY}              += board-harmony-pinmux.o
+obj-${CONFIG_MACH_HARMONY}              += board-harmony-pcie.o
arch/arm/mach-tegra/board-harmony-pcie.c new file +52 -0

52 lines not on this page · raw message

Signature · 1 line
1.6.6.2
#12 lore raw

PATCH 3/3 tegra: harmony: enable PCI Express

Quoted 12 lines · Mike Rapoport

---
?arch/arm/mach-tegra/Makefile ? ? ? ? ? ? | ? ?1 + ?arch/arm/mach-tegra/board-harmony-pcie.c | ? 52 ++++++++++++++++++++++++++++++ ?2 files changed, 53 insertions(+), 0 deletions(-)
?create mode 100644 arch/arm/mach-tegra/board-harmony-pcie.c

arch/arm/mach-tegra/board-harmony-pcie.c new file +0 -0
new file mode 100644
index 0000000..824c0a1

<snip>

arch/arm/mach-tegra/board-harmony-pcie.c

Quoted 7 lines
arch/arm/mach-tegra/board-harmony-pcie.c new file +7 -0
+static int __init harmony_pcie_init(void)
+{
+ ? ? ? int err;
+
+ ? ? ? tegra_pinmux_set_tristate(TEGRA_PINGROUP_GPV, TEGRA_TRI_NORMAL);
+ ? ? ? tegra_pinmux_set_tristate(TEGRA_PINGROUP_SLXA, TEGRA_TRI_NORMAL);
+ ? ? ? tegra_pinmux_set_tristate(TEGRA_PINGROUP_SLXK, TEGRA_TRI_NORMAL);

Can these go in the main harmony pinmux table?

#13 lore raw

PATCH 3/3 tegra: harmony: enable PCI Express

Quoted 16 lines · Colin Cross
Quoted 12 lines · Mike Rapoport

---
?arch/arm/mach-tegra/Makefile ? ? ? ? ? ? | ? ?1 + ?arch/arm/mach-tegra/board-harmony-pcie.c | ? 52 ++++++++++++++++++++++++++++++ ?2 files changed, 53 insertions(+), 0 deletions(-)
?create mode 100644 arch/arm/mach-tegra/board-harmony-pcie.c

arch/arm/mach-tegra/board-harmony-pcie.c new file +0 -0
new file mode 100644
index 0000000..824c0a1

<snip>

Quoted 8 lines

arch/arm/mach-tegra/board-harmony-pcie.c

Quoted 7 lines
arch/arm/mach-tegra/board-harmony-pcie.c new file +7 -0
+static int __init harmony_pcie_init(void)
+{
+ ? ? ? int err;
+
+ ? ? ? tegra_pinmux_set_tristate(TEGRA_PINGROUP_GPV, TEGRA_TRI_NORMAL);
+ ? ? ? tegra_pinmux_set_tristate(TEGRA_PINGROUP_SLXA, TEGRA_TRI_NORMAL);
+ ? ? ? tegra_pinmux_set_tristate(TEGRA_PINGROUP_SLXK, TEGRA_TRI_NORMAL);

Can these go in the main harmony pinmux table?

The idea is to keep the pins in tristate unless PCIe is enabled explicitly.

Quoted 5 lines

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel at lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Signature · 2 lines
? ? Sincerely Yours,
? ? ? ? Mike.
#14 lore raw

cover tegra: PCI Express support

In reply to Mike Rapoport
Quoted 7 lines · On Thursday 16 September 2010 18:53:33 Mike Rapoport wrote:

These patches enable PCI Express support on Tegra2.
The implementation is based on original NVidia code from (1), but it is heavily reworked to avoid custom PCI enumeration and make the code more Linux friendly.

This implementation assumes that the PCIe subsystem is fully powered and ungated by the bootloader.

The patches look good to me, but it seems that I/O space accesses are still broken and need to be redirected to the PCI I/O range, like the (totally untested) patch below.

arch/arm/mach-tegra/include/mach/io.h +3 -2
index 35edfc3..d54e384 100644
@@ -21,7 +21,8 @@
 #ifndef __MACH_TEGRA_IO_H
 #define __MACH_TEGRA_IO_H
 
-#define IO_SPACE_LIMIT 0xffffffff
+/* Two 1MB windows */
+#define IO_SPACE_LIMIT (SZ_1M + SZ_1M - 1)
 
 /* On TEGRA, many peripherals are very closely packed in
  * two 256MB io windows (that actually only use about 64KB
@@ -69,7 +70,7 @@ void tegra_iounmap(volatile void __iomem *addr);
 
 static inline void __iomem *__io(unsigned long addr)
 {
-	return (void __iomem *)addr;
+	return addr + tegra_pcie.regs + SZ_4M;
 }
 #define __io(a)         __io(a)
 #define __mem_pci(a)    (a)
#15 lore raw

cover tegra: PCI Express support

Quoted 6 lines · Arnd Bergmann
Quoted 5 lines · On Thursday 16 September 2010 18:53:33 Mike Rapoport wrote:

These patches enable PCI Express support on Tegra2.
The implementation is based on original NVidia code from (1), but it is heavily reworked to avoid custom PCI enumeration and make the code more Linux friendly.

Quoted 6 lines
Quoted 2 lines

This implementation assumes that the PCIe subsystem is fully powered and ungated by the bootloader.

The patches look good to me, but it seems that I/O space accesses are still broken and need to be redirected to the PCI I/O range, like the (totally untested) patch below.

Thanks for catching this.

arch/arm/mach-tegra/include/mach/io.h line 25

Quoted 13 lines
arch/arm/mach-tegra/include/mach/io.h +2 -1
index 35edfc3..d54e384 100644
@@ -21,7 +21,8 @@
 #ifndef __MACH_TEGRA_IO_H
 #define __MACH_TEGRA_IO_H
 
-#define IO_SPACE_LIMIT 0xffffffff
+/* Two 1MB windows */
+#define IO_SPACE_LIMIT (SZ_1M + SZ_1M - 1)

This would limit ioport_resource to 2M, and request_resource(&ioport_resource, &res) will fail because ioport_resource does not take into account that IO can start somewhere else than at 0.

arch/arm/mach-tegra/include/mach/io.h void tegra_iounmap(volatile void __iomem *addr); · line 73

Quoted 9 lines
arch/arm/mach-tegra/include/mach/io.h +1 -1
 /* On TEGRA, many peripherals are very closely packed in
  * two 256MB io windows (that actually only use about 64KB
@@ -69,7 +70,7 @@ void tegra_iounmap(volatile void __iomem *addr);
 
 static inline void __iomem *__io(unsigned long addr)
 {
-	return (void __iomem *)addr;
+	return addr + tegra_pcie.regs + SZ_4M;

I wish things were that simple :)
As far as I understand, the IO space should be mapped prior to use and __io should return the virtual address.
I'll add it for the next re-spin.

Quoted 3 lines
arch/arm/mach-tegra/include/mach/io.h +0 -0
 }
 #define __io(a)         __io(a)
 #define __mem_pci(a)    (a)
Signature · 2 lines
Sincerely yours,
Mike.
#16 lore raw

cover tegra: PCI Express support

Quoted 8 lines · On Sunday 19 September 2010 16:07:02 Mike Rapoport wrote:
Quoted 8 lines
arch/arm/mach-tegra/include/mach/io.h +0 -0
index 35edfc3..d54e384 100644
@@ -21,7 +21,8 @@
 #ifndef __MACH_TEGRA_IO_H
 #define __MACH_TEGRA_IO_H
 
Quoted 7 lines

arch/arm/mach-tegra/include/mach/io.h line 25

Quoted 3 lines
arch/arm/mach-tegra/include/mach/io.h +2 -1
-#define IO_SPACE_LIMIT 0xffffffff
+/* Two 1MB windows */
+#define IO_SPACE_LIMIT (SZ_1M + SZ_1M - 1)

This would limit ioport_resource to 2M, and request_resource(&ioport_resource, &res) will fail because ioport_resource does not take into account that IO can start somewhere else than at 0.

Normally, the ioport_resource is limited to 65536 bytes in practice, because that's the most that many PCI cards decode.

The only reason to let the I/O window start at something other than 0 is to leave space for legacy ISA devices, so typically the available range is between 0x1000 and 0xffff.

I don't see that as a limitation here.

Quoted 4 lines
Quoted 4 lines
arch/arm/mach-tegra/include/mach/io.h +0 -0
 /* On TEGRA, many peripherals are very closely packed in
  * two 256MB io windows (that actually only use about 64KB
@@ -69,7 +70,7 @@ void tegra_iounmap(volatile void __iomem *addr);
 
Quoted 8 lines

arch/arm/mach-tegra/include/mach/io.h void tegra_iounmap(volatile void __iomem *addr); · line 73

Quoted 4 lines
arch/arm/mach-tegra/include/mach/io.h +1 -1
 static inline void __iomem *__io(unsigned long addr)
 {
-     return (void __iomem *)addr;
+     return addr + tegra_pcie.regs + SZ_4M;

I wish things were that simple :)
As far as I understand, the IO space should be mapped prior to use and __io should return the virtual address.

That's right. You already map all the PCI registers including the I/O port mapping at initialization time, but you must not attempt to access these during boot before that time.

You should probably mask the size as above, which I forgot:

static inline void __iomem *__io(unsigned long addr)

{
     return (addr & IO_SPACE_LIMIT) + (tegra_pcie.regs + SZ_4M);
}

Hopefully it's clearer that way, and certainly safer in case someone passes the physical address of the I/O window into __io rather than the offset.

Arnd

#17 lore raw

cover tegra: PCI Express support

Quoted 7 lines · Arnd Bergmann
Quoted 2 lines · On Sunday 19 September 2010 16:07:02 Mike Rapoport wrote:

As far as I understand, the IO space should be mapped prior to use and __io should return the virtual address.

That's right. You already map all the PCI registers including the I/O port mapping at initialization time, but you must not attempt to access these during boot before that time.

Yes. __io() takes the IO port address, and returns a virtual address for the inb() et.al. macros to deference to perform the actual access.

So, for example if you have a PCI serial card inserted which appears at 0x3f8, then inb(0x3f8) should access the first register on the PCI serial card. As inb(0x3f8) equates to __raw_readb(__io(0x3f8)), __io(0x3f8) must return the virtual address to access that port.

Eg, on DC21285 (footbridge) systems, the PCI IO window is at 0x7c000000 physical, mapped into 0xff000000 virtual. So __io(0x3f8) translates to 0xff0003f8 virtual, which hits 0x7c0003f8 physical, and 0x3f8 as an IO access on the PCI bus.

Things become a little more complicated when you have PCMCIA cards with separate IO regions, as on SA11x0 and PXA systems. These don't tend to have PCI, so we adopted there to have __io() do a 1:1 translation, and arrange for the "bus IO" address to be the actual virtual address.

#18 lore raw

cover tegra: PCI Express support

Quoted 9 lines · On Sunday 19 September 2010 17:02:43 Russell King - ARM Linux wrote:

Eg, on DC21285 (footbridge) systems, the PCI IO window is at 0x7c000000 physical, mapped into 0xff000000 virtual. So __io(0x3f8) translates to 0xff0003f8 virtual, which hits 0x7c0003f8 physical, and 0x3f8 as an IO access on the PCI bus.

Things become a little more complicated when you have PCMCIA cards with separate IO regions, as on SA11x0 and PXA systems. These don't tend to have PCI, so we adopted there to have __io() do a 1:1 translation, and arrange for the "bus IO" address to be the actual virtual address.

Such a mapping sounds dangerous when you have device drivers trying to access legacy ISA ports. Most of them are disabled on ARM, but some drivers are hard to disable.

More importantly, having PCI I/O port numbers above 65536 will confuse code like /dev/ioport, which then causes NULL pointer accesses when accessed by a user application. Also, I would guess that it breaks if you have PCI or PCMCIA cards that decode more than 16 bits of I/O port addresses, because then the PCI I/O BAR gets set to a high number that is not actually accessible inside the I/O space window.

The way we solved this on powerpc was to define a range of the virtual address space as the I/O port window and map each bus's physical I/O port window into a section of the virtual window.

Arnd

#19 lore raw

cover tegra: PCI Express support

Quoted 6 lines · Arnd Bergmann
Quoted 5 lines · On Sunday 19 September 2010 17:02:43 Russell King - ARM Linux wrote:

Eg, on DC21285 (footbridge) systems, the PCI IO window is at 0x7c000000 physical, mapped into 0xff000000 virtual. So __io(0x3f8) translates to 0xff0003f8 virtual, which hits 0x7c0003f8 physical, and 0x3f8 as an IO access on the PCI bus.

Quoted 8 lines
Quoted 4 lines

Things become a little more complicated when you have PCMCIA cards with separate IO regions, as on SA11x0 and PXA systems. These don't tend to have PCI, so we adopted there to have __io() do a 1:1 translation, and arrange for the "bus IO" address to be the actual virtual address.

Such a mapping sounds dangerous when you have device drivers trying to access legacy ISA ports. Most of them are disabled on ARM, but some drivers are hard to disable.

Seems reliable on SA11x0 and PXA platforms.

Quoted 6 lines

More importantly, having PCI I/O port numbers above 65536 will confuse code like /dev/ioport, which then causes NULL pointer accesses when accessed by a user application. Also, I would guess that it breaks if you have PCI or PCMCIA cards that decode more than 16 bits of I/O port addresses, because then the PCI I/O BAR gets set to a high number that is not actually accessible inside the I/O space window.

This machine I'm using to send this email has in /proc/ioports:

90000000-9000ffff : IOP3XX PCI I/O Space
90000000-900000ff : 0000:00:01.0

    90000000-900000ff : r8169
  90000400-900004ff : 0000:00:02.0
    90000400-900004ff : r8169
  90000800-9000081f : 0000:00:04.0
    90000800-9000081f : uhci_hcd
  90000820-9000083f : 0000:00:04.1
    90000820-9000083f : uhci_hcd

90000840-9000084f : 0000:00:03.0
90000850-90000857 : 0000:00:03.0
90000858-9000085f : 0000:00:03.0
90000860-90000863 : 0000:00:03.0
90000864-90000867 : 0000:00:03.0

which works fine - and yes, if you insert an ISA driver it probably will crash. But then you can crash an x86 PC by using setserial to define a serial port at a wrong address.

The answer to the latter issue is "don't do that then", and I'd suggest that also goes for the "don't insert an ISA driver when the platform doesn't support it".

But yes, in principle I agree that PCI IO space should be 0 - 0xffff rather than having addresses assigned at the memory mapped IO address. Unfortunately, broken hardware (bridges) sometimes gets in the way of that.

#20 lore raw

cover tegra: PCI Express support

Quoted 3 lines · On Sunday 19 September 2010 18:40:40 Russell King - ARM Linux wrote:

which works fine - and yes, if you insert an ISA driver it probably will crash. But then you can crash an x86 PC by using setserial to define a serial port at a wrong address.

Sure, if you build the kernel just for one machine, there is no reason to include code that tries to access fixed I/O ports.

What I'm more worried about is that as we're moving to multi-platform kernels you want to build for other hardware, e.g. one that has a PC-style keyboard controller and one that has a funny mapping like that.

Normally, the driver for the keyboard controller should just access the well-known port number and not find anything there, because PCI devices are mapped above port 0x1000. What you get instead is a kernel NULL pointer access.

Arnd

#21 lore raw

cover tegra: PCI Express support

In reply to Arnd Bergmann
Quoted 16 lines · Arnd Bergmann
Quoted 8 lines · On Sunday 19 September 2010 16:07:02 Mike Rapoport wrote:
Quoted 8 lines
arch/arm/mach-tegra/include/mach/io.h +0 -0
index 35edfc3..d54e384 100644
@@ -21,7 +21,8 @@
 #ifndef __MACH_TEGRA_IO_H
 #define __MACH_TEGRA_IO_H
 
Quoted 6 lines

arch/arm/mach-tegra/include/mach/io.h line 25

Quoted 3 lines
arch/arm/mach-tegra/include/mach/io.h +2 -1
-#define IO_SPACE_LIMIT 0xffffffff
+/* Two 1MB windows */
+#define IO_SPACE_LIMIT (SZ_1M + SZ_1M - 1)

This would limit ioport_resource to 2M, and request_resource(&ioport_resource, &res) will fail because ioport_resource does not take into account that IO can start somewhere else than at 0.

Quoted 8 lines

Normally, the ioport_resource is limited to 65536 bytes in practice, because that's the most that many PCI cards decode.

The only reason to let the I/O window start at something other than 0 is to leave space for legacy ISA devices, so typically the available range is between 0x1000 and 0xffff.

I don't see that as a limitation here.

Since ARM doesn't have special IO access instructions and all IO is memory mapped, from the CPU perspective IO window would be at some arbitrary physical address. For Tegra this address can be anywhere between 0x80004000 and 0x8fffffff. With sizes and offsets in my implementation the IO resources would be defined as follows:

struct tegra_pcie_io_res[] = {
	[0] = {
		.start = 0x80400000,
		.end   = 0x804fffff,
		.flags	= IORESOURCE_IO,
	},
	[1] = {
		.start = 0x80500000,
		.end   = 0x805fffff,
		.flags	= IORESOURCE_IO,
	},
}

With IO_SPACE_LIMIT set to 2M the ioport_resource defined in kernel/resource.c becomes

struct resource ioport_resource = {
	.name	= "PCI IO",
	.start = 0,
	.end = 0x1fffff,
	.flags	= IORESOURCE_IO,
};

And then a call to request_resource(&ioport_resource, &tegra_pcie_io_res) fails because Tegra IO resources do not fit into the global ioport_resource definition.

Quoted 18 lines
Quoted 11 lines

arch/arm/mach-tegra/include/mach/io.h void tegra_iounmap(volatile void __iomem *addr); · line 73

Quoted 8 lines
arch/arm/mach-tegra/include/mach/io.h +1 -1
 /* On TEGRA, many peripherals are very closely packed in
  * two 256MB io windows (that actually only use about 64KB
@@ -69,7 +70,7 @@ void tegra_iounmap(volatile void __iomem *addr);
 
 static inline void __iomem *__io(unsigned long addr)
 {
-     return (void __iomem *)addr;
+     return addr + tegra_pcie.regs + SZ_4M;

I wish things were that simple :)
As far as I understand, the IO space should be mapped prior to use and __io should return the virtual address.

That's right. You already map all the PCI registers including the I/O port mapping at initialization time, but you must not attempt to access these during boot before that time.

You should probably mask the size as above, which I forgot:

Quoted 8 lines

static inline void __iomem *__io(unsigned long addr)

{
     return (addr & IO_SPACE_LIMIT) + (tegra_pcie.regs + SZ_4M);
}

Hopefully it's clearer that way, and certainly safer in case someone passes the physical address of the I/O window into __io rather than the offset.

I haven't mapped the IO space originally, so the __io should be something like

static inline void __iomem *__io(unsigned long addr)

{
	return tegra_pcie.io_space + addr;
}

where tegra_pcie.io_space is mapped during PCIe initialization.

Quoted 5 lines

Arnd
--
To unsubscribe from this list: send the line "unsubscribe linux-tegra" in the body of a message to majordomo at vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html

Signature · 2 lines
Sincerely yours,
Mike.
#22 lore raw

cover tegra: PCI Express support

Quoted 5 lines · Mike Rapoport

Since ARM doesn't have special IO access instructions and all IO is memory mapped, from the CPU perspective IO window would be at some arbitrary physical address. For Tegra this address can be anywhere between 0x80004000 and 0x8fffffff. With sizes and offsets in my implementation the IO resources would be defined as follows:

Quoted 12 lines
struct tegra_pcie_io_res[] = {
	[0] = {
		.start = 0x80400000,
		.end   = 0x804fffff,
		.flags	= IORESOURCE_IO,
	},
	[1] = {
		.start = 0x80500000,
		.end   = 0x805fffff,
		.flags	= IORESOURCE_IO,
	},
}

These aren't IO resources - they're describing an area of physical host memory, so they should be IORESOURCE_MEM.

Quoted 3 lines

And then a call to request_resource(&ioport_resource, &tegra_pcie_io_res) fails because Tegra IO resources do not fit into the global ioport_resource definition.

And therefore they should not be registered against the ioport resource.

Think about it like this:

iomem_resource (CPU physical
address space view):

+------------+ 0x00000000
|  RAM etc   |
/            /
/            /
|            |                    ioport_resource
+------------+ 0x80400000 ------> +-----------------+ 0x00000000
|            |                    | PCI peripherals |
|            |                    |                 |
|            | 0x804003f8         +-----------------+ 0x000003f8
|            |                    | Eg, PCI serial  |
|PCI IO space| 0x804003ff         +-----------------+ 0x000003ff
|            |                    |                 |
|            |                    |                 |
|            |                    | PCI peripherals |
|            |                    |                 |
|            |                    |                 |
+------------+ 0x805fffff ------> +-----------------+ 0x001fffff
|            |
/other stuff /
/            /
|            |
+------------+ 0xffffffff

So the iomem resource tree is entirely separate from the ioport resource tree, and the two never overlap. The iomem resource tree represents the physical MMIO space, and just contains a reservation for the entire block of PCI IO space. The ioport resource represents the entirely separate PCI IO space resource tree.

To illustrate this better, on Footbridge, this is what you see in /proc/iomem:
00000000-07ffffff : System RAM
0002d000-002a9fff : Kernel text
002aa000-002e7c77 : Kernel data
42000160-4200017f : Footbridge UART
a0000000-bfffffff : Footbridge prefetch
a0000000-a001ffff : 0000:00:07.0
a0020000-a003ffff : 0000:00:08.0
a0040000-a004ffff : 0000:00:09.0
c0000000-ffffffff : Footbridge non-prefetch
c0000000-c3ffffff : 0000:00:09.0
c4000000-c4000fff : 0000:00:06.3
c4001000-c400107f : 0000:00:08.0

Note that 7c000000-7c00ffff is the address range used for PCI IO accesses, and isn't requested in the above (mainly because we never registered a separate resource for it.)

and /proc/ioports:
0000-000f : dma1
0020-003f : pic1
0060-006f : i8042
0070-0073 : rtc_cmos
0070-0073 : rtc0
0080-008f : dma low page
00a0-00bf : pic2
00c0-00df : dma2
01f0-01f7 : ide0
0213-0213 : ISAPnP
02f8-02ff : serial8250.0
02f8-02ff : serial
03c0-03df : vga+
03f6-03f6 : ide0
03f8-03ff : serial8250.0
03f8-03ff : serial
0480-048f : dma high page
0a79-0a79 : isapnp write
1000-107f : 0000:00:08.0
1080-108f : 0000:00:06.1
1080-1087 : ide0
1090-109f : 0000:00:07.0
1090-1097 : ide1
1098-109f : ide2
10a0-10a7 : 0000:00:07.0
10a0-10a7 : ide1
10a8-10af : 0000:00:07.0
10a8-10af : ide2
10b0-10b3 : 0000:00:07.0
10b2-10b2 : ide1
10b4-10b7 : 0000:00:07.0
10b6-10b6 : ide2
ff00-ff7f : Footbridge

Note that IO accesses correspond to 0x7c000000 + IO address in iomem space.

#23 lore raw

cover tegra: PCI Express support

Quoted 56 lines · Russell King - ARM Linux
Quoted 5 lines · Mike Rapoport

Since ARM doesn't have special IO access instructions and all IO is memory mapped, from the CPU perspective IO window would be at some arbitrary physical address. For Tegra this address can be anywhere between 0x80004000 and 0x8fffffff. With sizes and offsets in my implementation the IO resources would be defined as follows:

Quoted 12 lines
struct tegra_pcie_io_res[] = {
	[0] = {
		.start = 0x80400000,
		.end   = 0x804fffff,
		.flags	= IORESOURCE_IO,
	},
	[1] = {
		.start = 0x80500000,
		.end   = 0x805fffff,
		.flags	= IORESOURCE_IO,
	},
}

These aren't IO resources - they're describing an area of physical host memory, so they should be IORESOURCE_MEM.

Quoted 3 lines

And then a call to request_resource(&ioport_resource, &tegra_pcie_io_res) fails because Tegra IO resources do not fit into the global ioport_resource definition.

And therefore they should not be registered against the ioport resource.

Think about it like this:

iomem_resource (CPU physical
address space view):

+------------+ 0x00000000
|  RAM etc   |
/            /
/            /
|            |                    ioport_resource
+------------+ 0x80400000 ------> +-----------------+ 0x00000000
|            |                    | PCI peripherals |
|            |                    |                 |
|            | 0x804003f8         +-----------------+ 0x000003f8
|            |                    | Eg, PCI serial  |
|PCI IO space| 0x804003ff         +-----------------+ 0x000003ff
|            |                    |                 |
|            |                    |                 |
|            |                    | PCI peripherals |
|            |                    |                 |
|            |                    |                 |
+------------+ 0x805fffff ------> +-----------------+ 0x001fffff
|            |
/other stuff /
/            /
|            |
+------------+ 0xffffffff
Quoted 5 lines

So the iomem resource tree is entirely separate from the ioport resource tree, and the two never overlap. The iomem resource tree represents the physical MMIO space, and just contains a reservation for the entire block of PCI IO space. The ioport resource represents the entirely separate PCI IO space resource tree.

From what you are saying I understand that the region reservation should look like:

static struct resource res_mmio = {
	.name	= "PCI IO"
	.start	= 0x80400000,
	.end	= 0x80400000 + IO_SIZE,
	.flags	= IORESOURCE_MEM,
};
static struct resource pcie_res[] = {
	[0] = {
		.name	= "PCIe IO",
		.start	= 0x1000,
		.end	= 0x1000 + IO_SIZE - 1,
		.flags	= IORESOURCE_IO,
	},
	[1] = {
		.name	= "PCIe MEM",
		.start	= MEM_BASE,
		.end	= MEM_BASE + MEM_SIZE - 1,
		.flags	= IORESOURCE_MEM,
	},
};

static int tegra_pcie_setup(int nr, struct pci_sys_data *sys)

{
	request_region(&iomem_resource, &res_mmio);
	request_region(&iomem_resource, &pcie_res[1]);
	request_region(&ioport_resource, &pcie_res[0]);
	sys->resource[0] = &pcie_res[0];
	sys->resource[1] = &pcie_res[1];
}

I've used 0x1000 as IO resources start because having it 0 would cause pcibios_enable_device to fail.

Is the above setup reasonable enough or I'm missing something else?

Quoted 60 lines

To illustrate this better, on Footbridge, this is what you see in /proc/iomem:
00000000-07ffffff : System RAM
0002d000-002a9fff : Kernel text
002aa000-002e7c77 : Kernel data
42000160-4200017f : Footbridge UART
a0000000-bfffffff : Footbridge prefetch
a0000000-a001ffff : 0000:00:07.0
a0020000-a003ffff : 0000:00:08.0
a0040000-a004ffff : 0000:00:09.0
c0000000-ffffffff : Footbridge non-prefetch
c0000000-c3ffffff : 0000:00:09.0
c4000000-c4000fff : 0000:00:06.3
c4001000-c400107f : 0000:00:08.0

Note that 7c000000-7c00ffff is the address range used for PCI IO accesses, and isn't requested in the above (mainly because we never registered a separate resource for it.)

and /proc/ioports:
0000-000f : dma1
0020-003f : pic1
0060-006f : i8042
0070-0073 : rtc_cmos
0070-0073 : rtc0
0080-008f : dma low page
00a0-00bf : pic2
00c0-00df : dma2
01f0-01f7 : ide0
0213-0213 : ISAPnP
02f8-02ff : serial8250.0
02f8-02ff : serial
03c0-03df : vga+
03f6-03f6 : ide0
03f8-03ff : serial8250.0
03f8-03ff : serial
0480-048f : dma high page
0a79-0a79 : isapnp write
1000-107f : 0000:00:08.0
1080-108f : 0000:00:06.1
1080-1087 : ide0
1090-109f : 0000:00:07.0
1090-1097 : ide1
1098-109f : ide2
10a0-10a7 : 0000:00:07.0
10a0-10a7 : ide1
10a8-10af : 0000:00:07.0
10a8-10af : ide2
10b0-10b3 : 0000:00:07.0
10b2-10b2 : ide1
10b4-10b7 : 0000:00:07.0
10b6-10b6 : ide2
ff00-ff7f : Footbridge

Note that IO accesses correspond to 0x7c000000 + IO address in iomem space.

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel at lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Signature · 2 lines
Sincerely yours,
Mike.
#24 lore raw

cover tegra: PCI Express support

Quoted 7 lines · Mike Rapoport
static struct resource pcie_res[] = {
        [0] = {
                .name   = "PCIe IO",
                .start  = 0x1000,
                .end    = 0x1000 + IO_SIZE - 1,
                .flags  = IORESOURCE_IO,
        },

The end should be 'IO_SIZE - 1' or IO_SPACE_LIMIT (should be the same anyway), not '0x1000 + IO_SIZE - 1'. The reason for the 0x1000 is that the first 4096 ports are reserved for legacy ISA devices by setting PCIBIOS_MIN_IO.

The rest looks fine AFAICT.

Arnd

#25 lore raw

cover tegra: PCI Express support

In reply to Mike Rapoport
Quoted 34 lines · Mike Rapoport

From what you are saying I understand that the region reservation should look like:

static struct resource res_mmio = {
	.name	= "PCI IO"
	.start	= 0x80400000,
	.end	= 0x80400000 + IO_SIZE,
	.flags	= IORESOURCE_MEM,
};
static struct resource pcie_res[] = {
	[0] = {
		.name	= "PCIe IO",
		.start	= 0x1000,
		.end	= 0x1000 + IO_SIZE - 1,
		.flags	= IORESOURCE_IO,
	},
	[1] = {
		.name	= "PCIe MEM",
		.start	= MEM_BASE,
		.end	= MEM_BASE + MEM_SIZE - 1,
		.flags	= IORESOURCE_MEM,
	},
};

static int tegra_pcie_setup(int nr, struct pci_sys_data *sys)

{
	request_region(&iomem_resource, &res_mmio);
	request_region(&iomem_resource, &pcie_res[1]);
	request_region(&ioport_resource, &pcie_res[0]);
	sys->resource[0] = &pcie_res[0];
	sys->resource[1] = &pcie_res[1];
}
Quoted 2 lines

I've used 0x1000 as IO resources start because having it 0 would cause pcibios_enable_device to fail.

More or less. You can avoid the ioport resource (&pcie_res[0]) and replace it with &ioport_resource if you set PCIBIOS_MIN_IO to 0x1000. This will have the effect of preventing BARs being allocated below 0x1000.

Don't also forget to check the return value from request_region()...