arm64: Add support for relocating the kernel with RELR relocations

[linux-2.6-block.git] / arch / arm64 / kernel / head.S

diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 2cdacd1c141b97295192e9685ffca9ef32765939..cc23302e9d95e0de75d545accc85706032e6056e 100644 (file)
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -102,6 +102,8 @@ pe_header:
         *  x23        stext() .. start_kernel()  physical misalignment/KASLR offset
         *  x28        __create_page_tables()     callee preserved temp register
         *  x19/x20    __primary_switch()         callee preserved temp registers
+        *  x24        __primary_switch() .. relocate_kernel()
+        *                                        current RELR displacement
         */
 ENTRY(stext)
        bl      preserve_boot_args
@@ -834,14 +836,93 @@ __relocate_kernel:
 
 0:     cmp     x9, x10
        b.hs    1f
-       ldp     x11, x12, [x9], #24
-       ldr     x13, [x9, #-8]
-       cmp     w12, #R_AARCH64_RELATIVE
+       ldp     x12, x13, [x9], #24
+       ldr     x14, [x9, #-8]
+       cmp     w13, #R_AARCH64_RELATIVE
        b.ne    0b
-       add     x13, x13, x23                   // relocate
-       str     x13, [x11, x23]
+       add     x14, x14, x23                   // relocate
+       str     x14, [x12, x23]
        b       0b
-1:     ret
+
+1:
+#ifdef CONFIG_RELR
+       /*
+        * Apply RELR relocations.
+        *
+        * RELR is a compressed format for storing relative relocations. The
+        * encoded sequence of entries looks like:
+        * [ AAAAAAAA BBBBBBB1 BBBBBBB1 ... AAAAAAAA BBBBBB1 ... ]
+        *
+        * i.e. start with an address, followed by any number of bitmaps. The
+        * address entry encodes 1 relocation. The subsequent bitmap entries
+        * encode up to 63 relocations each, at subsequent offsets following
+        * the last address entry.
+        *
+        * The bitmap entries must have 1 in the least significant bit. The
+        * assumption here is that an address cannot have 1 in lsb. Odd
+        * addresses are not supported. Any odd addresses are stored in the RELA
+        * section, which is handled above.
+        *
+        * Excluding the least significant bit in the bitmap, each non-zero
+        * bit in the bitmap represents a relocation to be applied to
+        * a corresponding machine word that follows the base address
+        * word. The second least significant bit represents the machine
+        * word immediately following the initial address, and each bit
+        * that follows represents the next word, in linear order. As such,
+        * a single bitmap can encode up to 63 relocations in a 64-bit object.
+        *
+        * In this implementation we store the address of the next RELR table
+        * entry in x9, the address being relocated by the current address or
+        * bitmap entry in x13 and the address being relocated by the current
+        * bit in x14.
+        *
+        * Because addends are stored in place in the binary, RELR relocations
+        * cannot be applied idempotently. We use x24 to keep track of the
+        * currently applied displacement so that we can correctly relocate if
+        * __relocate_kernel is called twice with non-zero displacements (i.e.
+        * if there is both a physical misalignment and a KASLR displacement).
+        */
+       ldr     w9, =__relr_offset              // offset to reloc table
+       ldr     w10, =__relr_size               // size of reloc table
+       add     x9, x9, x11                     // __va(.relr)
+       add     x10, x9, x10                    // __va(.relr) + sizeof(.relr)
+
+       sub     x15, x23, x24                   // delta from previous offset
+       cbz     x15, 7f                         // nothing to do if unchanged
+       mov     x24, x23                        // save new offset
+
+2:     cmp     x9, x10
+       b.hs    7f
+       ldr     x11, [x9], #8
+       tbnz    x11, #0, 3f                     // branch to handle bitmaps
+       add     x13, x11, x23
+       ldr     x12, [x13]                      // relocate address entry
+       add     x12, x12, x15
+       str     x12, [x13], #8                  // adjust to start of bitmap
+       b       2b
+
+3:     mov     x14, x13
+4:     lsr     x11, x11, #1
+       cbz     x11, 6f
+       tbz     x11, #0, 5f                     // skip bit if not set
+       ldr     x12, [x14]                      // relocate bit
+       add     x12, x12, x15
+       str     x12, [x14]
+
+5:     add     x14, x14, #8                    // move to next bit's address
+       b       4b
+
+6:     /*
+        * Move to the next bitmap's address. 8 is the word size, and 63 is the
+        * number of significant bits in a bitmap entry.
+        */
+       add     x13, x13, #(8 * 63)
+       b       2b
+
+7:
+#endif
+       ret
+
 ENDPROC(__relocate_kernel)
 #endif
 
@@ -854,6 +935,9 @@ __primary_switch:
        adrp    x1, init_pg_dir
        bl      __enable_mmu
 #ifdef CONFIG_RELOCATABLE
+#ifdef CONFIG_RELR
+       mov     x24, #0                         // no RELR displacement yet
+#endif
        bl      __relocate_kernel
 #ifdef CONFIG_RANDOMIZE_BASE
        ldr     x8, =__primary_switched