arm64: runtime workaround for erratum #843419

index e80e232b730e..d9ce63386e33 100644
@@ -20,4 +20,15 @@
 
 #define MODULE_ARCH_VERMAGIC	"aarch64"
 
+#ifdef CONFIG_HAVE_MOD_ARCH_SPECIFIC
+struct mod_arch_specific {
+#ifdef CONFIG_ARM64_MOD_VENEERS
+	struct veneer_section {
+		int		sec_index;
+		unsigned int	sec_offset;
+	} core, init;
+#endif
+};
+#endif
+
 #endif /* __ASM_MODULE_H */

105 lines not on this page · raw message

new file mode 100644
index 000000000000..3cf36ce91549
@@ -0,0 +1,20 @@
+
+#include <linux/types.h>
+
+struct veneer_erratum_843419 {
+	u32	adrp;
+	u32	branch;
+};
+
+static inline bool erratum_843419_affects_adrp_insn(void *addr)
+{
+	/*
+	 * The workaround for erratum 843419 only needs to be
+	 * applied if the adrp instruction appears in either of
+	 * the last two instruction slots in the 4 KB page.
+	 */
+	return ((u64)addr % SZ_4K) >= (SZ_4K - 8);
+}
+
+void *emit_erratum_843419_veneer(struct module *mod, u32 *insn,
+				 Elf64_Shdr *sechdrs);

index 9649c4f305bd..7da300e59749 100644
@@ -10,6 +10,8 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 
+#include <asm/mod_veneers.h>
+
 static bool in_init(const struct module *mod, u64 addr)
 {
 	return addr - (u64)mod->module_init < mod->init_size;

@@ -32,6 +34,31 @@ static void *alloc_veneer(struct module *mod, u64 loc, int size,
 	return ret;
 }
 
+#ifdef CONFIG_ARM64_ERRATUM_843419
+void *emit_erratum_843419_veneer(struct module *mod, u32 *insn,
+				 Elf64_Shdr *sechdrs)
+{
+	struct veneer_erratum_843419 *veneer;
+
+	veneer = alloc_veneer(mod, (u64)insn, 2 * sizeof(*veneer), sechdrs);
+	if (erratum_843419_affects_adrp_insn(&veneer->adrp))
+		/*
+		 * We allocated a veneer that is susceptible to the same problem
+		 * as the original location. We allocated twice the space, so
+		 * just advance to the next slot.
+		 */
+		veneer++;
+
+	veneer->adrp = *insn;
+	veneer->branch = aarch64_insn_gen_branch_imm(
+				(unsigned long)&veneer->branch,
+				(unsigned long)(insn + 1),
+				AARCH64_INSN_BRANCH_NOLINK);
+
+	return veneer;
+}
+#endif
+
 /* estimate the maximum size of the veneer for this relocation */
 static unsigned long get_veneers_size(Elf64_Addr base, const Elf64_Rela *rel,
 				      int num)

@@ -41,6 +68,12 @@ static unsigned long get_veneers_size(Elf64_Addr base, const Elf64_Rela *rel,
 
 	for (i = 0; i < num; i++)
 		switch (ELF64_R_TYPE(rel[i].r_info)) {
+		case R_AARCH64_ADR_PREL_PG_HI21_NC:
+		case R_AARCH64_ADR_PREL_PG_HI21:
+#ifdef CONFIG_ARM64_ERRATUM_843419
+			ret += 2 * sizeof(struct veneer_erratum_843419);
+#endif
+			break;
 		}
 	return ret;
 }

index 67bf4107f6ef..5d46f63c0111 100644
@@ -27,6 +27,7 @@
 #include <linux/vmalloc.h>
 #include <asm/alternative.h>
 #include <asm/insn.h>
+#include <asm/mod_veneers.h>
 #include <asm/sections.h>
 
 #define	AARCH64_INSN_IMM_MOVNZ		AARCH64_INSN_IMM_MAX

@@ -335,6 +336,39 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
 		case R_AARCH64_ADR_PREL_PG_HI21_NC:
 			overflow_check = false;
 		case R_AARCH64_ADR_PREL_PG_HI21:
+#ifdef CONFIG_ARM64_ERRATUM_843419
+			/*
+			 * TODO check for presence of affected A53 cores
+			 */
+			if (erratum_843419_affects_adrp_insn(loc)) {
+				struct veneer_erratum_843419 *v;
+
+				/*
+				 * This adrp instruction appears at an offset
+				 * that may be problematic on older Cortex-A53
+				 * cores. So first, try to convert it into a
+				 * simple adr instruction.
+				 */
+				ovf = reloc_insn_imm(RELOC_OP_PREL, loc,
+						     val & ~(SZ_4K - 1), 0, 21,
+						     AARCH64_INSN_IMM_ADR);
+				if (ovf == 0) {
+					/* success! convert adrp -> adr */
+					*(u32 *)loc &= 0x7fffffff;
+					break;
+				}
+
+				/* symbol out of range for adr -> emit veneer */
+				v = emit_erratum_843419_veneer(me, loc,
+							       sechdrs);
+				*(u32 *)loc = aarch64_insn_gen_branch_imm(
+						(unsigned long)loc,
+						(unsigned long)v,
+						AARCH64_INSN_BRANCH_NOLINK);
+				loc = &v->adrp;
+				/* fall through */
+			}
+#endif
 			ovf = reloc_insn_imm(RELOC_OP_PAGE, loc, val, 12, 21,
 					     AARCH64_INSN_IMM_ADR);
 			break;