arch/x86/platform/efi/efi.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Common EFI (Extensible Firmware Interface) support functions
   4  * Based on Extensible Firmware Interface Specification version 1.0
   5  *
   6  * Copyright (C) 1999 VA Linux Systems
   7  * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
   8  * Copyright (C) 1999-2002 Hewlett-Packard Co.
   9  *      David Mosberger-Tang <davidm@hpl.hp.com>
  10  *      Stephane Eranian <eranian@hpl.hp.com>
  11  * Copyright (C) 2005-2008 Intel Co.
  12  *      Fenghua Yu <fenghua.yu@intel.com>
  13  *      Bibo Mao <bibo.mao@intel.com>
  14  *      Chandramouli Narayanan <mouli@linux.intel.com>
  15  *      Huang Ying <ying.huang@intel.com>
  16  * Copyright (C) 2013 SuSE Labs
  17  *      Borislav Petkov <bp@suse.de> - runtime services VA mapping
  18  *
  19  * Copied from efi_32.c to eliminate the duplicated code between EFI
  20  * 32/64 support code. --ying 2007-10-26
  21  *
  22  * All EFI Runtime Services are not implemented yet as EFI only
  23  * supports physical mode addressing on SoftSDV. This is to be fixed
  24  * in a future version.  --drummond 1999-07-20
  25  *
  26  * Implemented EFI runtime services and virtual mode calls.  --davidm
  27  *
  28  * Goutham Rao: <goutham.rao@intel.com>
  29  *      Skip non-WB memory and ignore empty memory ranges.
  30  */
  31
  32 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  33
  34 #include <linux/kernel.h>
  35 #include <linux/init.h>
  36 #include <linux/efi.h>
  37 #include <linux/efi-bgrt.h>
  38 #include <linux/export.h>
  39 #include <linux/bootmem.h>
  40 #include <linux/slab.h>
  41 #include <linux/memblock.h>
  42 #include <linux/spinlock.h>
  43 #include <linux/uaccess.h>
  44 #include <linux/time.h>
  45 #include <linux/io.h>
  46 #include <linux/reboot.h>
  47 #include <linux/bcd.h>
  48
  49 #include <asm/setup.h>
  50 #include <asm/efi.h>
  51 #include <asm/e820/api.h>
  52 #include <asm/time.h>
  53 #include <asm/set_memory.h>
  54 #include <asm/tlbflush.h>
  55 #include <asm/x86_init.h>
  56 #include <asm/uv/uv.h>
  57
  58 static struct efi efi_phys __initdata;
  59 static efi_system_table_t efi_systab __initdata;
  60
  61 static efi_config_table_type_t arch_tables[] __initdata = {
  62 #ifdef CONFIG_X86_UV
  63         {UV_SYSTEM_TABLE_GUID, "UVsystab", &efi.uv_systab},
  64 #endif
  65         {NULL_GUID, NULL, NULL},
  66 };
  67
  68 u64 efi_setup;          /* efi setup_data physical address */
  69
  70 static int add_efi_memmap __initdata;
  71 static int __init setup_add_efi_memmap(char *arg)
  72 {
  73         add_efi_memmap = 1;
  74         return 0;
  75 }
  76 early_param("add_efi_memmap", setup_add_efi_memmap);
  77
  78 static efi_status_t __init phys_efi_set_virtual_address_map(
  79         unsigned long memory_map_size,
  80         unsigned long descriptor_size,
  81         u32 descriptor_version,
  82         efi_memory_desc_t *virtual_map)
  83 {
  84         efi_status_t status;
  85         unsigned long flags;
  86         pgd_t *save_pgd;
  87
  88         save_pgd = efi_call_phys_prolog();
  89
  90         /* Disable interrupts around EFI calls: */
  91         local_irq_save(flags);
  92         status = efi_call_phys(efi_phys.set_virtual_address_map,
  93                                memory_map_size, descriptor_size,
  94                                descriptor_version, virtual_map);
  95         local_irq_restore(flags);
  96
  97         efi_call_phys_epilog(save_pgd);
  98
  99         return status;
 100 }
 101
 102 void __init efi_find_mirror(void)
 103 {
 104         efi_memory_desc_t *md;
 105         u64 mirror_size = 0, total_size = 0;
 106
 107         for_each_efi_memory_desc(md) {
 108                 unsigned long long start = md->phys_addr;
 109                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 110
 111                 total_size += size;
 112                 if (md->attribute & EFI_MEMORY_MORE_RELIABLE) {
 113                         memblock_mark_mirror(start, size);
 114                         mirror_size += size;
 115                 }
 116         }
 117         if (mirror_size)
 118                 pr_info("Memory: %lldM/%lldM mirrored memory\n",
 119                         mirror_size>>20, total_size>>20);
 120 }
 121
 122 /*
 123  * Tell the kernel about the EFI memory map.  This might include
 124  * more than the max 128 entries that can fit in the e820 legacy
 125  * (zeropage) memory map.
 126  */
 127
 128 static void __init do_add_efi_memmap(void)
 129 {
 130         efi_memory_desc_t *md;
 131
 132         for_each_efi_memory_desc(md) {
 133                 unsigned long long start = md->phys_addr;
 134                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 135                 int e820_type;
 136
 137                 switch (md->type) {
 138                 case EFI_LOADER_CODE:
 139                 case EFI_LOADER_DATA:
 140                 case EFI_BOOT_SERVICES_CODE:
 141                 case EFI_BOOT_SERVICES_DATA:
 142                 case EFI_CONVENTIONAL_MEMORY:
 143                         if (md->attribute & EFI_MEMORY_WB)
 144                                 e820_type = E820_TYPE_RAM;
 145                         else
 146                                 e820_type = E820_TYPE_RESERVED;
 147                         break;
 148                 case EFI_ACPI_RECLAIM_MEMORY:
 149                         e820_type = E820_TYPE_ACPI;
 150                         break;
 151                 case EFI_ACPI_MEMORY_NVS:
 152                         e820_type = E820_TYPE_NVS;
 153                         break;
 154                 case EFI_UNUSABLE_MEMORY:
 155                         e820_type = E820_TYPE_UNUSABLE;
 156                         break;
 157                 case EFI_PERSISTENT_MEMORY:
 158                         e820_type = E820_TYPE_PMEM;
 159                         break;
 160                 default:
 161                         /*
 162                          * EFI_RESERVED_TYPE EFI_RUNTIME_SERVICES_CODE
 163                          * EFI_RUNTIME_SERVICES_DATA EFI_MEMORY_MAPPED_IO
 164                          * EFI_MEMORY_MAPPED_IO_PORT_SPACE EFI_PAL_CODE
 165                          */
 166                         e820_type = E820_TYPE_RESERVED;
 167                         break;
 168                 }
 169                 e820__range_add(start, size, e820_type);
 170         }
 171         e820__update_table(e820_table);
 172 }
 173
 174 int __init efi_memblock_x86_reserve_range(void)
 175 {
 176         struct efi_info *e = &boot_params.efi_info;
 177         struct efi_memory_map_data data;
 178         phys_addr_t pmap;
 179         int rv;
 180
 181         if (efi_enabled(EFI_PARAVIRT))
 182                 return 0;
 183
 184 #ifdef CONFIG_X86_32
 185         /* Can't handle data above 4GB at this time */
 186         if (e->efi_memmap_hi) {
 187                 pr_err("Memory map is above 4GB, disabling EFI.\n");
 188                 return -EINVAL;
 189         }
 190         pmap =  e->efi_memmap;
 191 #else
 192         pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32));
 193 #endif
 194         data.phys_map           = pmap;
 195         data.size               = e->efi_memmap_size;
 196         data.desc_size          = e->efi_memdesc_size;
 197         data.desc_version       = e->efi_memdesc_version;
 198
 199         rv = efi_memmap_init_early(&data);
 200         if (rv)
 201                 return rv;
 202
 203         if (add_efi_memmap)
 204                 do_add_efi_memmap();
 205
 206         WARN(efi.memmap.desc_version != 1,
 207              "Unexpected EFI_MEMORY_DESCRIPTOR version %ld",
 208              efi.memmap.desc_version);
 209
 210         memblock_reserve(pmap, efi.memmap.nr_map * efi.memmap.desc_size);
 211
 212         return 0;
 213 }
 214
 215 #define OVERFLOW_ADDR_SHIFT     (64 - EFI_PAGE_SHIFT)
 216 #define OVERFLOW_ADDR_MASK      (U64_MAX << OVERFLOW_ADDR_SHIFT)
 217 #define U64_HIGH_BIT            (~(U64_MAX >> 1))
 218
 219 static bool __init efi_memmap_entry_valid(const efi_memory_desc_t *md, int i)
 220 {
 221         u64 end = (md->num_pages << EFI_PAGE_SHIFT) + md->phys_addr - 1;
 222         u64 end_hi = 0;
 223         char buf[64];
 224
 225         if (md->num_pages == 0) {
 226                 end = 0;
 227         } else if (md->num_pages > EFI_PAGES_MAX ||
 228                    EFI_PAGES_MAX - md->num_pages <
 229                    (md->phys_addr >> EFI_PAGE_SHIFT)) {
 230                 end_hi = (md->num_pages & OVERFLOW_ADDR_MASK)
 231                         >> OVERFLOW_ADDR_SHIFT;
 232
 233                 if ((md->phys_addr & U64_HIGH_BIT) && !(end & U64_HIGH_BIT))
 234                         end_hi += 1;
 235         } else {
 236                 return true;
 237         }
 238
 239         pr_warn_once(FW_BUG "Invalid EFI memory map entries:\n");
 240
 241         if (end_hi) {
 242                 pr_warn("mem%02u: %s range=[0x%016llx-0x%llx%016llx] (invalid)\n",
 243                         i, efi_md_typeattr_format(buf, sizeof(buf), md),
 244                         md->phys_addr, end_hi, end);
 245         } else {
 246                 pr_warn("mem%02u: %s range=[0x%016llx-0x%016llx] (invalid)\n",
 247                         i, efi_md_typeattr_format(buf, sizeof(buf), md),
 248                         md->phys_addr, end);
 249         }
 250         return false;
 251 }
 252
 253 static void __init efi_clean_memmap(void)
 254 {
 255         efi_memory_desc_t *out = efi.memmap.map;
 256         const efi_memory_desc_t *in = out;
 257         const efi_memory_desc_t *end = efi.memmap.map_end;
 258         int i, n_removal;
 259
 260         for (i = n_removal = 0; in < end; i++) {
 261                 if (efi_memmap_entry_valid(in, i)) {
 262                         if (out != in)
 263                                 memcpy(out, in, efi.memmap.desc_size);
 264                         out = (void *)out + efi.memmap.desc_size;
 265                 } else {
 266                         n_removal++;
 267                 }
 268                 in = (void *)in + efi.memmap.desc_size;
 269         }
 270
 271         if (n_removal > 0) {
 272                 u64 size = efi.memmap.nr_map - n_removal;
 273
 274                 pr_warn("Removing %d invalid memory map entries.\n", n_removal);
 275                 efi_memmap_install(efi.memmap.phys_map, size);
 276         }
 277 }
 278
 279 void __init efi_print_memmap(void)
 280 {
 281         efi_memory_desc_t *md;
 282         int i = 0;
 283
 284         for_each_efi_memory_desc(md) {
 285                 char buf[64];
 286
 287                 pr_info("mem%02u: %s range=[0x%016llx-0x%016llx] (%lluMB)\n",
 288                         i++, efi_md_typeattr_format(buf, sizeof(buf), md),
 289                         md->phys_addr,
 290                         md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT) - 1,
 291                         (md->num_pages >> (20 - EFI_PAGE_SHIFT)));
 292         }
 293 }
 294
 295 static int __init efi_systab_init(void *phys)
 296 {
 297         if (efi_enabled(EFI_64BIT)) {
 298                 efi_system_table_64_t *systab64;
 299                 struct efi_setup_data *data = NULL;
 300                 u64 tmp = 0;
 301
 302                 if (efi_setup) {
 303                         data = early_memremap(efi_setup, sizeof(*data));
 304                         if (!data)
 305                                 return -ENOMEM;
 306                 }
 307                 systab64 = early_memremap((unsigned long)phys,
 308                                          sizeof(*systab64));
 309                 if (systab64 == NULL) {
 310                         pr_err("Couldn't map the system table!\n");
 311                         if (data)
 312                                 early_memunmap(data, sizeof(*data));
 313                         return -ENOMEM;
 314                 }
 315
 316                 efi_systab.hdr = systab64->hdr;
 317                 efi_systab.fw_vendor = data ? (unsigned long)data->fw_vendor :
 318                                               systab64->fw_vendor;
 319                 tmp |= data ? data->fw_vendor : systab64->fw_vendor;
 320                 efi_systab.fw_revision = systab64->fw_revision;
 321                 efi_systab.con_in_handle = systab64->con_in_handle;
 322                 tmp |= systab64->con_in_handle;
 323                 efi_systab.con_in = systab64->con_in;
 324                 tmp |= systab64->con_in;
 325                 efi_systab.con_out_handle = systab64->con_out_handle;
 326                 tmp |= systab64->con_out_handle;
 327                 efi_systab.con_out = systab64->con_out;
 328                 tmp |= systab64->con_out;
 329                 efi_systab.stderr_handle = systab64->stderr_handle;
 330                 tmp |= systab64->stderr_handle;
 331                 efi_systab.stderr = systab64->stderr;
 332                 tmp |= systab64->stderr;
 333                 efi_systab.runtime = data ?
 334                                      (void *)(unsigned long)data->runtime :
 335                                      (void *)(unsigned long)systab64->runtime;
 336                 tmp |= data ? data->runtime : systab64->runtime;
 337                 efi_systab.boottime = (void *)(unsigned long)systab64->boottime;
 338                 tmp |= systab64->boottime;
 339                 efi_systab.nr_tables = systab64->nr_tables;
 340                 efi_systab.tables = data ? (unsigned long)data->tables :
 341                                            systab64->tables;
 342                 tmp |= data ? data->tables : systab64->tables;
 343
 344                 early_memunmap(systab64, sizeof(*systab64));
 345                 if (data)
 346                         early_memunmap(data, sizeof(*data));
 347 #ifdef CONFIG_X86_32
 348                 if (tmp >> 32) {
 349                         pr_err("EFI data located above 4GB, disabling EFI.\n");
 350                         return -EINVAL;
 351                 }
 352 #endif
 353         } else {
 354                 efi_system_table_32_t *systab32;
 355
 356                 systab32 = early_memremap((unsigned long)phys,
 357                                          sizeof(*systab32));
 358                 if (systab32 == NULL) {
 359                         pr_err("Couldn't map the system table!\n");
 360                         return -ENOMEM;
 361                 }
 362
 363                 efi_systab.hdr = systab32->hdr;
 364                 efi_systab.fw_vendor = systab32->fw_vendor;
 365                 efi_systab.fw_revision = systab32->fw_revision;
 366                 efi_systab.con_in_handle = systab32->con_in_handle;
 367                 efi_systab.con_in = systab32->con_in;
 368                 efi_systab.con_out_handle = systab32->con_out_handle;
 369                 efi_systab.con_out = systab32->con_out;
 370                 efi_systab.stderr_handle = systab32->stderr_handle;
 371                 efi_systab.stderr = systab32->stderr;
 372                 efi_systab.runtime = (void *)(unsigned long)systab32->runtime;
 373                 efi_systab.boottime = (void *)(unsigned long)systab32->boottime;
 374                 efi_systab.nr_tables = systab32->nr_tables;
 375                 efi_systab.tables = systab32->tables;
 376
 377                 early_memunmap(systab32, sizeof(*systab32));
 378         }
 379
 380         efi.systab = &efi_systab;
 381
 382         /*
 383          * Verify the EFI Table
 384          */
 385         if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) {
 386                 pr_err("System table signature incorrect!\n");
 387                 return -EINVAL;
 388         }
 389         if ((efi.systab->hdr.revision >> 16) == 0)
 390                 pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n",
 391                        efi.systab->hdr.revision >> 16,
 392                        efi.systab->hdr.revision & 0xffff);
 393
 394         return 0;
 395 }
 396
 397 static int __init efi_runtime_init32(void)
 398 {
 399         efi_runtime_services_32_t *runtime;
 400
 401         runtime = early_memremap((unsigned long)efi.systab->runtime,
 402                         sizeof(efi_runtime_services_32_t));
 403         if (!runtime) {
 404                 pr_err("Could not map the runtime service table!\n");
 405                 return -ENOMEM;
 406         }
 407
 408         /*
 409          * We will only need *early* access to the SetVirtualAddressMap
 410          * EFI runtime service. All other runtime services will be called
 411          * via the virtual mapping.
 412          */
 413         efi_phys.set_virtual_address_map =
 414                         (efi_set_virtual_address_map_t *)
 415                         (unsigned long)runtime->set_virtual_address_map;
 416         early_memunmap(runtime, sizeof(efi_runtime_services_32_t));
 417
 418         return 0;
 419 }
 420
 421 static int __init efi_runtime_init64(void)
 422 {
 423         efi_runtime_services_64_t *runtime;
 424
 425         runtime = early_memremap((unsigned long)efi.systab->runtime,
 426                         sizeof(efi_runtime_services_64_t));
 427         if (!runtime) {
 428                 pr_err("Could not map the runtime service table!\n");
 429                 return -ENOMEM;
 430         }
 431
 432         /*
 433          * We will only need *early* access to the SetVirtualAddressMap
 434          * EFI runtime service. All other runtime services will be called
 435          * via the virtual mapping.
 436          */
 437         efi_phys.set_virtual_address_map =
 438                         (efi_set_virtual_address_map_t *)
 439                         (unsigned long)runtime->set_virtual_address_map;
 440         early_memunmap(runtime, sizeof(efi_runtime_services_64_t));
 441
 442         return 0;
 443 }
 444
 445 static int __init efi_runtime_init(void)
 446 {
 447         int rv;
 448
 449         /*
 450          * Check out the runtime services table. We need to map
 451          * the runtime services table so that we can grab the physical
 452          * address of several of the EFI runtime functions, needed to
 453          * set the firmware into virtual mode.
 454          *
 455          * When EFI_PARAVIRT is in force then we could not map runtime
 456          * service memory region because we do not have direct access to it.
 457          * However, runtime services are available through proxy functions
 458          * (e.g. in case of Xen dom0 EFI implementation they call special
 459          * hypercall which executes relevant EFI functions) and that is why
 460          * they are always enabled.
 461          */
 462
 463         if (!efi_enabled(EFI_PARAVIRT)) {
 464                 if (efi_enabled(EFI_64BIT))
 465                         rv = efi_runtime_init64();
 466                 else
 467                         rv = efi_runtime_init32();
 468
 469                 if (rv)
 470                         return rv;
 471         }
 472
 473         set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 474
 475         return 0;
 476 }
 477
 478 void __init efi_init(void)
 479 {
 480         efi_char16_t *c16;
 481         char vendor[100] = "unknown";
 482         int i = 0;
 483         void *tmp;
 484
 485 #ifdef CONFIG_X86_32
 486         if (boot_params.efi_info.efi_systab_hi ||
 487             boot_params.efi_info.efi_memmap_hi) {
 488                 pr_info("Table located above 4GB, disabling EFI.\n");
 489                 return;
 490         }
 491         efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab;
 492 #else
 493         efi_phys.systab = (efi_system_table_t *)
 494                           (boot_params.efi_info.efi_systab |
 495                           ((__u64)boot_params.efi_info.efi_systab_hi<<32));
 496 #endif
 497
 498         if (efi_systab_init(efi_phys.systab))
 499                 return;
 500
 501         efi.config_table = (unsigned long)efi.systab->tables;
 502         efi.fw_vendor    = (unsigned long)efi.systab->fw_vendor;
 503         efi.runtime      = (unsigned long)efi.systab->runtime;
 504
 505         /*
 506          * Show what we know for posterity
 507          */
 508         c16 = tmp = early_memremap(efi.systab->fw_vendor, 2);
 509         if (c16) {
 510                 for (i = 0; i < sizeof(vendor) - 1 && *c16; ++i)
 511                         vendor[i] = *c16++;
 512                 vendor[i] = '\0';
 513         } else
 514                 pr_err("Could not map the firmware vendor!\n");
 515         early_memunmap(tmp, 2);
 516
 517         pr_info("EFI v%u.%.02u by %s\n",
 518                 efi.systab->hdr.revision >> 16,
 519                 efi.systab->hdr.revision & 0xffff, vendor);
 520
 521         if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables))
 522                 return;
 523
 524         if (efi_config_init(arch_tables))
 525                 return;
 526
 527         /*
 528          * Note: We currently don't support runtime services on an EFI
 529          * that doesn't match the kernel 32/64-bit mode.
 530          */
 531
 532         if (!efi_runtime_supported())
 533                 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n");
 534         else {
 535                 if (efi_runtime_disabled() || efi_runtime_init()) {
 536                         efi_memmap_unmap();
 537                         return;
 538                 }
 539         }
 540
 541         efi_clean_memmap();
 542
 543         if (efi_enabled(EFI_DBG))
 544                 efi_print_memmap();
 545 }
 546
 547 void __init efi_set_executable(efi_memory_desc_t *md, bool executable)
 548 {
 549         u64 addr, npages;
 550
 551         addr = md->virt_addr;
 552         npages = md->num_pages;
 553
 554         memrange_efi_to_native(&addr, &npages);
 555
 556         if (executable)
 557                 set_memory_x(addr, npages);
 558         else
 559                 set_memory_nx(addr, npages);
 560 }
 561
 562 void __init runtime_code_page_mkexec(void)
 563 {
 564         efi_memory_desc_t *md;
 565
 566         /* Make EFI runtime service code area executable */
 567         for_each_efi_memory_desc(md) {
 568                 if (md->type != EFI_RUNTIME_SERVICES_CODE)
 569                         continue;
 570
 571                 efi_set_executable(md, true);
 572         }
 573 }
 574
 575 void __init efi_memory_uc(u64 addr, unsigned long size)
 576 {
 577         unsigned long page_shift = 1UL << EFI_PAGE_SHIFT;
 578         u64 npages;
 579
 580         npages = round_up(size, page_shift) / page_shift;
 581         memrange_efi_to_native(&addr, &npages);
 582         set_memory_uc(addr, npages);
 583 }
 584
 585 void __init old_map_region(efi_memory_desc_t *md)
 586 {
 587         u64 start_pfn, end_pfn, end;
 588         unsigned long size;
 589         void *va;
 590
 591         start_pfn = PFN_DOWN(md->phys_addr);
 592         size      = md->num_pages << PAGE_SHIFT;
 593         end       = md->phys_addr + size;
 594         end_pfn   = PFN_UP(end);
 595
 596         if (pfn_range_is_mapped(start_pfn, end_pfn)) {
 597                 va = __va(md->phys_addr);
 598
 599                 if (!(md->attribute & EFI_MEMORY_WB))
 600                         efi_memory_uc((u64)(unsigned long)va, size);
 601         } else
 602                 va = efi_ioremap(md->phys_addr, size,
 603                                  md->type, md->attribute);
 604
 605         md->virt_addr = (u64) (unsigned long) va;
 606         if (!va)
 607                 pr_err("ioremap of 0x%llX failed!\n",
 608                        (unsigned long long)md->phys_addr);
 609 }
 610
 611 /* Merge contiguous regions of the same type and attribute */
 612 static void __init efi_merge_regions(void)
 613 {
 614         efi_memory_desc_t *md, *prev_md = NULL;
 615
 616         for_each_efi_memory_desc(md) {
 617                 u64 prev_size;
 618
 619                 if (!prev_md) {
 620                         prev_md = md;
 621                         continue;
 622                 }
 623
 624                 if (prev_md->type != md->type ||
 625                     prev_md->attribute != md->attribute) {
 626                         prev_md = md;
 627                         continue;
 628                 }
 629
 630                 prev_size = prev_md->num_pages << EFI_PAGE_SHIFT;
 631
 632                 if (md->phys_addr == (prev_md->phys_addr + prev_size)) {
 633                         prev_md->num_pages += md->num_pages;
 634                         md->type = EFI_RESERVED_TYPE;
 635                         md->attribute = 0;
 636                         continue;
 637                 }
 638                 prev_md = md;
 639         }
 640 }
 641
 642 static void __init get_systab_virt_addr(efi_memory_desc_t *md)
 643 {
 644         unsigned long size;
 645         u64 end, systab;
 646
 647         size = md->num_pages << EFI_PAGE_SHIFT;
 648         end = md->phys_addr + size;
 649         systab = (u64)(unsigned long)efi_phys.systab;
 650         if (md->phys_addr <= systab && systab < end) {
 651                 systab += md->virt_addr - md->phys_addr;
 652                 efi.systab = (efi_system_table_t *)(unsigned long)systab;
 653         }
 654 }
 655
 656 static void *realloc_pages(void *old_memmap, int old_shift)
 657 {
 658         void *ret;
 659
 660         ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1);
 661         if (!ret)
 662                 goto out;
 663
 664         /*
 665          * A first-time allocation doesn't have anything to copy.
 666          */
 667         if (!old_memmap)
 668                 return ret;
 669
 670         memcpy(ret, old_memmap, PAGE_SIZE << old_shift);
 671
 672 out:
 673         free_pages((unsigned long)old_memmap, old_shift);
 674         return ret;
 675 }
 676
 677 /*
 678  * Iterate the EFI memory map in reverse order because the regions
 679  * will be mapped top-down. The end result is the same as if we had
 680  * mapped things forward, but doesn't require us to change the
 681  * existing implementation of efi_map_region().
 682  */
 683 static inline void *efi_map_next_entry_reverse(void *entry)
 684 {
 685         /* Initial call */
 686         if (!entry)
 687                 return efi.memmap.map_end - efi.memmap.desc_size;
 688
 689         entry -= efi.memmap.desc_size;
 690         if (entry < efi.memmap.map)
 691                 return NULL;
 692
 693         return entry;
 694 }
 695
 696 /*
 697  * efi_map_next_entry - Return the next EFI memory map descriptor
 698  * @entry: Previous EFI memory map descriptor
 699  *
 700  * This is a helper function to iterate over the EFI memory map, which
 701  * we do in different orders depending on the current configuration.
 702  *
 703  * To begin traversing the memory map @entry must be %NULL.
 704  *
 705  * Returns %NULL when we reach the end of the memory map.
 706  */
 707 static void *efi_map_next_entry(void *entry)
 708 {
 709         if (!efi_enabled(EFI_OLD_MEMMAP) && efi_enabled(EFI_64BIT)) {
 710                 /*
 711                  * Starting in UEFI v2.5 the EFI_PROPERTIES_TABLE
 712                  * config table feature requires us to map all entries
 713                  * in the same order as they appear in the EFI memory
 714                  * map. That is to say, entry N must have a lower
 715                  * virtual address than entry N+1. This is because the
 716                  * firmware toolchain leaves relative references in
 717                  * the code/data sections, which are split and become
 718                  * separate EFI memory regions. Mapping things
 719                  * out-of-order leads to the firmware accessing
 720                  * unmapped addresses.
 721                  *
 722                  * Since we need to map things this way whether or not
 723                  * the kernel actually makes use of
 724                  * EFI_PROPERTIES_TABLE, let's just switch to this
 725                  * scheme by default for 64-bit.
 726                  */
 727                 return efi_map_next_entry_reverse(entry);
 728         }
 729
 730         /* Initial call */
 731         if (!entry)
 732                 return efi.memmap.map;
 733
 734         entry += efi.memmap.desc_size;
 735         if (entry >= efi.memmap.map_end)
 736                 return NULL;
 737
 738         return entry;
 739 }
 740
 741 static bool should_map_region(efi_memory_desc_t *md)
 742 {
 743         /*
 744          * Runtime regions always require runtime mappings (obviously).
 745          */
 746         if (md->attribute & EFI_MEMORY_RUNTIME)
 747                 return true;
 748
 749         /*
 750          * 32-bit EFI doesn't suffer from the bug that requires us to
 751          * reserve boot services regions, and mixed mode support
 752          * doesn't exist for 32-bit kernels.
 753          */
 754         if (IS_ENABLED(CONFIG_X86_32))
 755                 return false;
 756
 757         /*
 758          * Map all of RAM so that we can access arguments in the 1:1
 759          * mapping when making EFI runtime calls.
 760          */
 761         if (IS_ENABLED(CONFIG_EFI_MIXED) && !efi_is_native()) {
 762                 if (md->type == EFI_CONVENTIONAL_MEMORY ||
 763                     md->type == EFI_LOADER_DATA ||
 764                     md->type == EFI_LOADER_CODE)
 765                         return true;
 766         }
 767
 768         /*
 769          * Map boot services regions as a workaround for buggy
 770          * firmware that accesses them even when they shouldn't.
 771          *
 772          * See efi_{reserve,free}_boot_services().
 773          */
 774         if (md->type == EFI_BOOT_SERVICES_CODE ||
 775             md->type == EFI_BOOT_SERVICES_DATA)
 776                 return true;
 777
 778         return false;
 779 }
 780
 781 /*
 782  * Map the efi memory ranges of the runtime services and update new_mmap with
 783  * virtual addresses.
 784  */
 785 static void * __init efi_map_regions(int *count, int *pg_shift)
 786 {
 787         void *p, *new_memmap = NULL;
 788         unsigned long left = 0;
 789         unsigned long desc_size;
 790         efi_memory_desc_t *md;
 791
 792         desc_size = efi.memmap.desc_size;
 793
 794         p = NULL;
 795         while ((p = efi_map_next_entry(p))) {
 796                 md = p;
 797
 798                 if (!should_map_region(md))
 799                         continue;
 800
 801                 efi_map_region(md);
 802                 get_systab_virt_addr(md);
 803
 804                 if (left < desc_size) {
 805                         new_memmap = realloc_pages(new_memmap, *pg_shift);
 806                         if (!new_memmap)
 807                                 return NULL;
 808
 809                         left += PAGE_SIZE << *pg_shift;
 810                         (*pg_shift)++;
 811                 }
 812
 813                 memcpy(new_memmap + (*count * desc_size), md, desc_size);
 814
 815                 left -= desc_size;
 816                 (*count)++;
 817         }
 818
 819         return new_memmap;
 820 }
 821
 822 static void __init kexec_enter_virtual_mode(void)
 823 {
 824 #ifdef CONFIG_KEXEC_CORE
 825         efi_memory_desc_t *md;
 826         unsigned int num_pages;
 827
 828         efi.systab = NULL;
 829
 830         /*
 831          * We don't do virtual mode, since we don't do runtime services, on
 832          * non-native EFI. With efi=old_map, we don't do runtime services in
 833          * kexec kernel because in the initial boot something else might
 834          * have been mapped at these virtual addresses.
 835          */
 836         if (!efi_is_native() || efi_enabled(EFI_OLD_MEMMAP)) {
 837                 efi_memmap_unmap();
 838                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 839                 return;
 840         }
 841
 842         if (efi_alloc_page_tables()) {
 843                 pr_err("Failed to allocate EFI page tables\n");
 844                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 845                 return;
 846         }
 847
 848         /*
 849         * Map efi regions which were passed via setup_data. The virt_addr is a
 850         * fixed addr which was used in first kernel of a kexec boot.
 851         */
 852         for_each_efi_memory_desc(md) {
 853                 efi_map_region_fixed(md); /* FIXME: add error handling */
 854                 get_systab_virt_addr(md);
 855         }
 856
 857         /*
 858          * Unregister the early EFI memmap from efi_init() and install
 859          * the new EFI memory map.
 860          */
 861         efi_memmap_unmap();
 862
 863         if (efi_memmap_init_late(efi.memmap.phys_map,
 864                                  efi.memmap.desc_size * efi.memmap.nr_map)) {
 865                 pr_err("Failed to remap late EFI memory map\n");
 866                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 867                 return;
 868         }
 869
 870         BUG_ON(!efi.systab);
 871
 872         num_pages = ALIGN(efi.memmap.nr_map * efi.memmap.desc_size, PAGE_SIZE);
 873         num_pages >>= PAGE_SHIFT;
 874
 875         if (efi_setup_page_tables(efi.memmap.phys_map, num_pages)) {
 876                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 877                 return;
 878         }
 879
 880         efi_sync_low_kernel_mappings();
 881
 882         /*
 883          * Now that EFI is in virtual mode, update the function
 884          * pointers in the runtime service table to the new virtual addresses.
 885          *
 886          * Call EFI services through wrapper functions.
 887          */
 888         efi.runtime_version = efi_systab.hdr.revision;
 889
 890         efi_native_runtime_setup();
 891
 892         efi.set_virtual_address_map = NULL;
 893
 894         if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX))
 895                 runtime_code_page_mkexec();
 896
 897         /* clean DUMMY object */
 898         efi_delete_dummy_variable();
 899 #endif
 900 }
 901
 902 /*
 903  * This function will switch the EFI runtime services to virtual mode.
 904  * Essentially, we look through the EFI memmap and map every region that
 905  * has the runtime attribute bit set in its memory descriptor into the
 906  * efi_pgd page table.
 907  *
 908  * The old method which used to update that memory descriptor with the
 909  * virtual address obtained from ioremap() is still supported when the
 910  * kernel is booted with efi=old_map on its command line. Same old
 911  * method enabled the runtime services to be called without having to
 912  * thunk back into physical mode for every invocation.
 913  *
 914  * The new method does a pagetable switch in a preemption-safe manner
 915  * so that we're in a different address space when calling a runtime
 916  * function. For function arguments passing we do copy the PUDs of the
 917  * kernel page table into efi_pgd prior to each call.
 918  *
 919  * Specially for kexec boot, efi runtime maps in previous kernel should
 920  * be passed in via setup_data. In that case runtime ranges will be mapped
 921  * to the same virtual addresses as the first kernel, see
 922  * kexec_enter_virtual_mode().
 923  */
 924 static void __init __efi_enter_virtual_mode(void)
 925 {
 926         int count = 0, pg_shift = 0;
 927         void *new_memmap = NULL;
 928         efi_status_t status;
 929         unsigned long pa;
 930
 931         efi.systab = NULL;
 932
 933         if (efi_alloc_page_tables()) {
 934                 pr_err("Failed to allocate EFI page tables\n");
 935                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 936                 return;
 937         }
 938
 939         efi_merge_regions();
 940         new_memmap = efi_map_regions(&count, &pg_shift);
 941         if (!new_memmap) {
 942                 pr_err("Error reallocating memory, EFI runtime non-functional!\n");
 943                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 944                 return;
 945         }
 946
 947         pa = __pa(new_memmap);
 948
 949         /*
 950          * Unregister the early EFI memmap from efi_init() and install
 951          * the new EFI memory map that we are about to pass to the
 952          * firmware via SetVirtualAddressMap().
 953          */
 954         efi_memmap_unmap();
 955
 956         if (efi_memmap_init_late(pa, efi.memmap.desc_size * count)) {
 957                 pr_err("Failed to remap late EFI memory map\n");
 958                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 959                 return;
 960         }
 961
 962         if (efi_enabled(EFI_DBG)) {
 963                 pr_info("EFI runtime memory map:\n");
 964                 efi_print_memmap();
 965         }
 966
 967         BUG_ON(!efi.systab);
 968
 969         if (efi_setup_page_tables(pa, 1 << pg_shift)) {
 970                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 971                 return;
 972         }
 973
 974         efi_sync_low_kernel_mappings();
 975
 976         if (efi_is_native()) {
 977                 status = phys_efi_set_virtual_address_map(
 978                                 efi.memmap.desc_size * count,
 979                                 efi.memmap.desc_size,
 980                                 efi.memmap.desc_version,
 981                                 (efi_memory_desc_t *)pa);
 982         } else {
 983                 status = efi_thunk_set_virtual_address_map(
 984                                 efi_phys.set_virtual_address_map,
 985                                 efi.memmap.desc_size * count,
 986                                 efi.memmap.desc_size,
 987                                 efi.memmap.desc_version,
 988                                 (efi_memory_desc_t *)pa);
 989         }
 990
 991         if (status != EFI_SUCCESS) {
 992                 pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n",
 993                          status);
 994                 panic("EFI call to SetVirtualAddressMap() failed!");
 995         }
 996
 997         /*
 998          * Now that EFI is in virtual mode, update the function
 999          * pointers in the runtime service table to the new virtual addresses.
1000          *
1001          * Call EFI services through wrapper functions.
1002          */
1003         efi.runtime_version = efi_systab.hdr.revision;
1004
1005         if (efi_is_native())
1006                 efi_native_runtime_setup();
1007         else
1008                 efi_thunk_runtime_setup();
1009
1010         efi.set_virtual_address_map = NULL;
1011
1012         /*
1013          * Apply more restrictive page table mapping attributes now that
1014          * SVAM() has been called and the firmware has performed all
1015          * necessary relocation fixups for the new virtual addresses.
1016          */
1017         efi_runtime_update_mappings();
1018
1019         /* clean DUMMY object */
1020         efi_delete_dummy_variable();
1021 }
1022
1023 void __init efi_enter_virtual_mode(void)
1024 {
1025         if (efi_enabled(EFI_PARAVIRT))
1026                 return;
1027
1028         if (efi_setup)
1029                 kexec_enter_virtual_mode();
1030         else
1031                 __efi_enter_virtual_mode();
1032
1033         efi_dump_pagetable();
1034 }
1035
1036 static int __init arch_parse_efi_cmdline(char *str)
1037 {
1038         if (!str) {
1039                 pr_warn("need at least one option\n");
1040                 return -EINVAL;
1041         }
1042
1043         if (parse_option_str(str, "old_map"))
1044                 set_bit(EFI_OLD_MEMMAP, &efi.flags);
1045
1046         return 0;
1047 }
1048 early_param("efi", arch_parse_efi_cmdline);