arch/powerpc/include/asm/head-64.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ASM_POWERPC_HEAD_64_H
   3 #define _ASM_POWERPC_HEAD_64_H
   4
   5 #include <asm/cache.h>
   6
   7 #ifdef __ASSEMBLY__
   8 /*
   9  * We can't do CPP stringification and concatination directly into the section
  10  * name for some reason, so these macros can do it for us.
  11  */
  12 .macro define_ftsec name
  13         .section ".head.text.\name\()","ax",@progbits
  14 .endm
  15 .macro define_data_ftsec name
  16         .section ".head.data.\name\()","a",@progbits
  17 .endm
  18 .macro use_ftsec name
  19         .section ".head.text.\name\()"
  20 .endm
  21
  22 /*
  23  * Fixed (location) sections are used by opening fixed sections and emitting
  24  * fixed section entries into them before closing them. Multiple fixed sections
  25  * can be open at any time.
  26  *
  27  * Each fixed section created in a .S file must have corresponding linkage
  28  * directives including location, added to  arch/powerpc/kernel/vmlinux.lds.S
  29  *
  30  * For each fixed section, code is generated into it in the order which it
  31  * appears in the source.  Fixed section entries can be placed at a fixed
  32  * location within the section using _LOCATION postifx variants. These must
  33  * be ordered according to their relative placements within the section.
  34  *
  35  * OPEN_FIXED_SECTION(section_name, start_address, end_address)
  36  * FIXED_SECTION_ENTRY_BEGIN(section_name, label1)
  37  *
  38  * USE_FIXED_SECTION(section_name)
  39  * label3:
  40  *     li  r10,128
  41  *     mv  r11,r10
  42
  43  * FIXED_SECTION_ENTRY_BEGIN_LOCATION(section_name, label2, start_address, size)
  44  * FIXED_SECTION_ENTRY_END_LOCATION(section_name, label2, start_address, size)
  45  * CLOSE_FIXED_SECTION(section_name)
  46  *
  47  * ZERO_FIXED_SECTION can be used to emit zeroed data.
  48  *
  49  * Troubleshooting:
  50  * - If the build dies with "Error: attempt to move .org backwards" at
  51  *   CLOSE_FIXED_SECTION() or elsewhere, there may be something
  52  *   unexpected being added there. Remove the '. = x_len' line, rebuild, and
  53  *   check what is pushing the section down.
  54  * - If the build dies in linking, check arch/powerpc/tools/head_check.sh
  55  *   comments.
  56  * - If the kernel crashes or hangs in very early boot, it could be linker
  57  *   stubs at the start of the main text.
  58  */
  59
  60 #define OPEN_FIXED_SECTION(sname, start, end)                   \
  61         sname##_start = (start);                                \
  62         sname##_end = (end);                                    \
  63         sname##_len = (end) - (start);                          \
  64         define_ftsec sname;                                     \
  65         . = 0x0;                                                \
  66 start_##sname:
  67
  68 /*
  69  * .linker_stub_catch section is used to catch linker stubs from being
  70  * inserted in our .text section, above the start_text label (which breaks
  71  * the ABS_ADDR calculation). See kernel/vmlinux.lds.S and tools/head_check.sh
  72  * for more details. We would prefer to just keep a cacheline (0x80), but
  73  * 0x100 seems to be how the linker aligns branch stub groups.
  74  */
  75 #ifdef CONFIG_LD_HEAD_STUB_CATCH
  76 #define OPEN_TEXT_SECTION(start)                                \
  77         .section ".linker_stub_catch","ax",@progbits;           \
  78 linker_stub_catch:                                              \
  79         . = 0x4;                                                \
  80         text_start = (start) + 0x100;                           \
  81         .section ".text","ax",@progbits;                        \
  82         .balign 0x100;                                          \
  83 start_text:
  84 #else
  85 #define OPEN_TEXT_SECTION(start)                                \
  86         text_start = (start);                                   \
  87         .section ".text","ax",@progbits;                        \
  88         . = 0x0;                                                \
  89 start_text:
  90 #endif
  91
  92 #define ZERO_FIXED_SECTION(sname, start, end)                   \
  93         sname##_start = (start);                                \
  94         sname##_end = (end);                                    \
  95         sname##_len = (end) - (start);                          \
  96         define_data_ftsec sname;                                \
  97         . = 0x0;                                                \
  98         . = sname##_len;
  99
 100 #define USE_FIXED_SECTION(sname)                                \
 101         fs_label = start_##sname;                               \
 102         fs_start = sname##_start;                               \
 103         use_ftsec sname;
 104
 105 #define USE_TEXT_SECTION()                                      \
 106         fs_label = start_text;                                  \
 107         fs_start = text_start;                                  \
 108         .text
 109
 110 #define CLOSE_FIXED_SECTION(sname)                              \
 111         USE_FIXED_SECTION(sname);                               \
 112         . = sname##_len;                                        \
 113 end_##sname:
 114
 115
 116 #define __FIXED_SECTION_ENTRY_BEGIN(sname, name, __align)       \
 117         USE_FIXED_SECTION(sname);                               \
 118         .balign __align;                                        \
 119         .global name;                                           \
 120 name:
 121
 122 #define FIXED_SECTION_ENTRY_BEGIN(sname, name)                  \
 123         __FIXED_SECTION_ENTRY_BEGIN(sname, name, IFETCH_ALIGN_BYTES)
 124
 125 #define FIXED_SECTION_ENTRY_BEGIN_LOCATION(sname, name, start, size) \
 126         USE_FIXED_SECTION(sname);                               \
 127         name##_start = (start);                                 \
 128         .if ((start) % (size) != 0);                            \
 129         .error "Fixed section exception vector misalignment";   \
 130         .endif;                                                 \
 131         .if ((size) != 0x20) && ((size) != 0x80) && ((size) != 0x100); \
 132         .error "Fixed section exception vector bad size";       \
 133         .endif;                                                 \
 134         .if (start) < sname##_start;                            \
 135         .error "Fixed section underflow";                       \
 136         .abort;                                                 \
 137         .endif;                                                 \
 138         . = (start) - sname##_start;                            \
 139         .global name;                                           \
 140 name:
 141
 142 #define FIXED_SECTION_ENTRY_END_LOCATION(sname, name, start, size) \
 143         .if (start) + (size) > sname##_end;                     \
 144         .error "Fixed section overflow";                        \
 145         .abort;                                                 \
 146         .endif;                                                 \
 147         .if (. - name > (start) + (size) - name##_start);       \
 148         .error "Fixed entry overflow";                          \
 149         .abort;                                                 \
 150         .endif;                                                 \
 151         . = ((start) + (size) - sname##_start);                 \
 152
 153
 154 /*
 155  * These macros are used to change symbols in other fixed sections to be
 156  * absolute or related to our current fixed section.
 157  *
 158  * - DEFINE_FIXED_SYMBOL / FIXED_SYMBOL_ABS_ADDR is used to find the
 159  *   absolute address of a symbol within a fixed section, from any section.
 160  *
 161  * - ABS_ADDR is used to find the absolute address of any symbol, from within
 162  *   a fixed section.
 163  */
 164 #define DEFINE_FIXED_SYMBOL(label)                              \
 165         label##_absolute = (label - fs_label + fs_start)
 166
 167 #define FIXED_SYMBOL_ABS_ADDR(label)                            \
 168         (label##_absolute)
 169
 170 #define ABS_ADDR(label) (label - fs_label + fs_start)
 171
 172 /*
 173  * Following are the BOOK3S exception handler helper macros.
 174  * Handlers come in a number of types, and each type has a number of varieties.
 175  *
 176  * EXC_REAL_*     - real, unrelocated exception vectors
 177  * EXC_VIRT_*     - virt (AIL), unrelocated exception vectors
 178  * TRAMP_REAL_*   - real, unrelocated helpers (virt can call these)
 179  * TRAMP_VIRT_*   - virt, unreloc helpers (in practice, real can use)
 180  * TRAMP_KVM      - KVM handlers that get put into real, unrelocated
 181  * EXC_COMMON     - virt, relocated common handlers
 182  *
 183  * The EXC handlers are given a name, and branch to name_common, or the
 184  * appropriate KVM or masking function. Vector handler verieties are as
 185  * follows:
 186  *
 187  * EXC_{REAL|VIRT}_BEGIN/END - used to open-code the exception
 188  *
 189  * EXC_{REAL|VIRT}  - standard exception
 190  *
 191  * EXC_{REAL|VIRT}_suffix
 192  *     where _suffix is:
 193  *   - _MASKABLE               - maskable exception
 194  *   - _OOL                    - out of line with trampoline to common handler
 195  *   - _HV                     - HV exception
 196  *
 197  * There can be combinations, e.g., EXC_VIRT_OOL_MASKABLE_HV
 198  *
 199  * The one unusual case is __EXC_REAL_OOL_HV_DIRECT, which is
 200  * an OOL vector that branches to a specified handler rather than the usual
 201  * trampoline that goes to common. It, and other underscore macros, should
 202  * be used with care.
 203  *
 204  * KVM handlers come in the following verieties:
 205  * TRAMP_KVM
 206  * TRAMP_KVM_SKIP
 207  * TRAMP_KVM_HV
 208  * TRAMP_KVM_HV_SKIP
 209  *
 210  * COMMON handlers come in the following verieties:
 211  * EXC_COMMON_BEGIN/END - used to open-code the handler
 212  * EXC_COMMON
 213  * EXC_COMMON_ASYNC
 214  *
 215  * TRAMP_REAL and TRAMP_VIRT can be used with BEGIN/END. KVM
 216  * and OOL handlers are implemented as types of TRAMP and TRAMP_VIRT handlers.
 217  */
 218
 219 #define EXC_REAL_BEGIN(name, start, size)                       \
 220         FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##name, start, size)
 221
 222 #define EXC_REAL_END(name, start, size)                         \
 223         FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##name, start, size)
 224
 225 #define EXC_VIRT_BEGIN(name, start, size)                       \
 226         FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size)
 227
 228 #define EXC_VIRT_END(name, start, size)                         \
 229         FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size)
 230
 231 #define EXC_COMMON_BEGIN(name)                                  \
 232         USE_TEXT_SECTION();                                     \
 233         .balign IFETCH_ALIGN_BYTES;                             \
 234         .global name;                                           \
 235         _ASM_NOKPROBE_SYMBOL(name);                             \
 236         DEFINE_FIXED_SYMBOL(name);                              \
 237 name:
 238
 239 #define TRAMP_REAL_BEGIN(name)                                  \
 240         FIXED_SECTION_ENTRY_BEGIN(real_trampolines, name)
 241
 242 #define TRAMP_VIRT_BEGIN(name)                                  \
 243         FIXED_SECTION_ENTRY_BEGIN(virt_trampolines, name)
 244
 245 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
 246 #define TRAMP_KVM_BEGIN(name)                                   \
 247         TRAMP_VIRT_BEGIN(name)
 248 #else
 249 #define TRAMP_KVM_BEGIN(name)
 250 #endif
 251
 252 #define EXC_REAL_NONE(start, size)                              \
 253         FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##unused, start, size); \
 254         FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##unused, start, size)
 255
 256 #define EXC_VIRT_NONE(start, size)                              \
 257         FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size); \
 258         FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size)
 259
 260
 261 #define EXC_REAL(name, start, size)                                     \
 262         EXC_REAL_BEGIN(name, start, size);                              \
 263         STD_EXCEPTION(start, name##_common);                            \
 264         EXC_REAL_END(name, start, size)
 265
 266 #define EXC_VIRT(name, start, size, realvec)                            \
 267         EXC_VIRT_BEGIN(name, start, size);                              \
 268         STD_RELON_EXCEPTION(start, realvec, name##_common);             \
 269         EXC_VIRT_END(name, start, size)
 270
 271 #define EXC_REAL_MASKABLE(name, start, size, bitmask)                   \
 272         EXC_REAL_BEGIN(name, start, size);                              \
 273         MASKABLE_EXCEPTION(start, name##_common, bitmask);              \
 274         EXC_REAL_END(name, start, size)
 275
 276 #define EXC_VIRT_MASKABLE(name, start, size, realvec, bitmask)          \
 277         EXC_VIRT_BEGIN(name, start, size);                              \
 278         MASKABLE_RELON_EXCEPTION(realvec, name##_common, bitmask);      \
 279         EXC_VIRT_END(name, start, size)
 280
 281 #define EXC_REAL_HV(name, start, size)                                  \
 282         EXC_REAL_BEGIN(name, start, size);                              \
 283         STD_EXCEPTION_HV(start, start, name##_common);                  \
 284         EXC_REAL_END(name, start, size)
 285
 286 #define EXC_VIRT_HV(name, start, size, realvec)                         \
 287         EXC_VIRT_BEGIN(name, start, size);                              \
 288         STD_RELON_EXCEPTION_HV(start, realvec, name##_common);          \
 289         EXC_VIRT_END(name, start, size)
 290
 291 #define __EXC_REAL_OOL(name, start, size)                               \
 292         EXC_REAL_BEGIN(name, start, size);                              \
 293         __OOL_EXCEPTION(start, label, tramp_real_##name);               \
 294         EXC_REAL_END(name, start, size)
 295
 296 #define __TRAMP_REAL_OOL(name, vec)                                     \
 297         TRAMP_REAL_BEGIN(tramp_real_##name);                            \
 298         STD_EXCEPTION_OOL(vec, name##_common)
 299
 300 #define EXC_REAL_OOL(name, start, size)                                 \
 301         __EXC_REAL_OOL(name, start, size);                              \
 302         __TRAMP_REAL_OOL(name, start)
 303
 304 #define __EXC_REAL_OOL_MASKABLE(name, start, size)                      \
 305         __EXC_REAL_OOL(name, start, size)
 306
 307 #define __TRAMP_REAL_OOL_MASKABLE(name, vec, bitmask)                   \
 308         TRAMP_REAL_BEGIN(tramp_real_##name);                            \
 309         MASKABLE_EXCEPTION_OOL(vec, name##_common, bitmask)
 310
 311 #define EXC_REAL_OOL_MASKABLE(name, start, size, bitmask)               \
 312         __EXC_REAL_OOL_MASKABLE(name, start, size);                     \
 313         __TRAMP_REAL_OOL_MASKABLE(name, start, bitmask)
 314
 315 #define __EXC_REAL_OOL_HV_DIRECT(name, start, size, handler)            \
 316         EXC_REAL_BEGIN(name, start, size);                              \
 317         __OOL_EXCEPTION(start, label, handler);                         \
 318         EXC_REAL_END(name, start, size)
 319
 320 #define __EXC_REAL_OOL_HV(name, start, size)                            \
 321         __EXC_REAL_OOL(name, start, size)
 322
 323 #define __TRAMP_REAL_OOL_HV(name, vec)                                  \
 324         TRAMP_REAL_BEGIN(tramp_real_##name);                            \
 325         STD_EXCEPTION_HV_OOL(vec, name##_common)
 326
 327 #define EXC_REAL_OOL_HV(name, start, size)                              \
 328         __EXC_REAL_OOL_HV(name, start, size);                           \
 329         __TRAMP_REAL_OOL_HV(name, start)
 330
 331 #define __EXC_REAL_OOL_MASKABLE_HV(name, start, size)                   \
 332         __EXC_REAL_OOL(name, start, size)
 333
 334 #define __TRAMP_REAL_OOL_MASKABLE_HV(name, vec, bitmask)                \
 335         TRAMP_REAL_BEGIN(tramp_real_##name);                            \
 336         MASKABLE_EXCEPTION_HV_OOL(vec, name##_common, bitmask)
 337
 338 #define EXC_REAL_OOL_MASKABLE_HV(name, start, size, bitmask)            \
 339         __EXC_REAL_OOL_MASKABLE_HV(name, start, size);                  \
 340         __TRAMP_REAL_OOL_MASKABLE_HV(name, start, bitmask)
 341
 342 #define __EXC_VIRT_OOL(name, start, size)                               \
 343         EXC_VIRT_BEGIN(name, start, size);                              \
 344         __OOL_EXCEPTION(start, label, tramp_virt_##name);               \
 345         EXC_VIRT_END(name, start, size)
 346
 347 #define __TRAMP_VIRT_OOL(name, realvec)                                 \
 348         TRAMP_VIRT_BEGIN(tramp_virt_##name);                            \
 349         STD_RELON_EXCEPTION_OOL(realvec, name##_common)
 350
 351 #define EXC_VIRT_OOL(name, start, size, realvec)                        \
 352         __EXC_VIRT_OOL(name, start, size);                              \
 353         __TRAMP_VIRT_OOL(name, realvec)
 354
 355 #define __EXC_VIRT_OOL_MASKABLE(name, start, size)                      \
 356         __EXC_VIRT_OOL(name, start, size)
 357
 358 #define __TRAMP_VIRT_OOL_MASKABLE(name, realvec, bitmask)               \
 359         TRAMP_VIRT_BEGIN(tramp_virt_##name);                            \
 360         MASKABLE_RELON_EXCEPTION_OOL(realvec, name##_common, bitmask)
 361
 362 #define EXC_VIRT_OOL_MASKABLE(name, start, size, realvec, bitmask)      \
 363         __EXC_VIRT_OOL_MASKABLE(name, start, size);                     \
 364         __TRAMP_VIRT_OOL_MASKABLE(name, realvec, bitmask)
 365
 366 #define __EXC_VIRT_OOL_HV(name, start, size)                            \
 367         __EXC_VIRT_OOL(name, start, size)
 368
 369 #define __TRAMP_VIRT_OOL_HV(name, realvec)                              \
 370         TRAMP_VIRT_BEGIN(tramp_virt_##name);                            \
 371         STD_RELON_EXCEPTION_HV_OOL(realvec, name##_common)
 372
 373 #define EXC_VIRT_OOL_HV(name, start, size, realvec)                     \
 374         __EXC_VIRT_OOL_HV(name, start, size);                           \
 375         __TRAMP_VIRT_OOL_HV(name, realvec)
 376
 377 #define __EXC_VIRT_OOL_MASKABLE_HV(name, start, size)                   \
 378         __EXC_VIRT_OOL(name, start, size)
 379
 380 #define __TRAMP_VIRT_OOL_MASKABLE_HV(name, realvec, bitmask)            \
 381         TRAMP_VIRT_BEGIN(tramp_virt_##name);                            \
 382         MASKABLE_RELON_EXCEPTION_HV_OOL(realvec, name##_common, bitmask)
 383
 384 #define EXC_VIRT_OOL_MASKABLE_HV(name, start, size, realvec, bitmask)   \
 385         __EXC_VIRT_OOL_MASKABLE_HV(name, start, size);                  \
 386         __TRAMP_VIRT_OOL_MASKABLE_HV(name, realvec, bitmask)
 387
 388 #define TRAMP_KVM(area, n)                                              \
 389         TRAMP_KVM_BEGIN(do_kvm_##n);                                    \
 390         KVM_HANDLER area, EXC_STD, n
 391
 392 #define TRAMP_KVM_SKIP(area, n)                                         \
 393         TRAMP_KVM_BEGIN(do_kvm_##n);                                    \
 394         KVM_HANDLER_SKIP area, EXC_STD, n
 395
 396 /*
 397  * HV variant exceptions get the 0x2 bit added to their trap number.
 398  */
 399 #define TRAMP_KVM_HV(area, n)                                           \
 400         TRAMP_KVM_BEGIN(do_kvm_H##n);                                   \
 401         KVM_HANDLER area, EXC_HV, n + 0x2
 402
 403 #define TRAMP_KVM_HV_SKIP(area, n)                                      \
 404         TRAMP_KVM_BEGIN(do_kvm_H##n);                                   \
 405         KVM_HANDLER_SKIP area, EXC_HV, n + 0x2
 406
 407 #define EXC_COMMON(name, realvec, hdlr)                                 \
 408         EXC_COMMON_BEGIN(name);                                         \
 409         STD_EXCEPTION_COMMON(realvec, name, hdlr)
 410
 411 #define EXC_COMMON_ASYNC(name, realvec, hdlr)                           \
 412         EXC_COMMON_BEGIN(name);                                         \
 413         STD_EXCEPTION_COMMON_ASYNC(realvec, name, hdlr)
 414
 415 #endif /* __ASSEMBLY__ */
 416
 417 #endif  /* _ASM_POWERPC_HEAD_64_H */