arch/x86/platform/efi/efi.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Common EFI (Extensible Firmware Interface) support functions
   4  * Based on Extensible Firmware Interface Specification version 1.0
   5  *
   6  * Copyright (C) 1999 VA Linux Systems
   7  * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
   8  * Copyright (C) 1999-2002 Hewlett-Packard Co.
   9  *      David Mosberger-Tang <davidm@hpl.hp.com>
  10  *      Stephane Eranian <eranian@hpl.hp.com>
  11  * Copyright (C) 2005-2008 Intel Co.
  12  *      Fenghua Yu <fenghua.yu@intel.com>
  13  *      Bibo Mao <bibo.mao@intel.com>
  14  *      Chandramouli Narayanan <mouli@linux.intel.com>
  15  *      Huang Ying <ying.huang@intel.com>
  16  * Copyright (C) 2013 SuSE Labs
  17  *      Borislav Petkov <bp@suse.de> - runtime services VA mapping
  18  *
  19  * Copied from efi_32.c to eliminate the duplicated code between EFI
  20  * 32/64 support code. --ying 2007-10-26
  21  *
  22  * All EFI Runtime Services are not implemented yet as EFI only
  23  * supports physical mode addressing on SoftSDV. This is to be fixed
  24  * in a future version.  --drummond 1999-07-20
  25  *
  26  * Implemented EFI runtime services and virtual mode calls.  --davidm
  27  *
  28  * Goutham Rao: <goutham.rao@intel.com>
  29  *      Skip non-WB memory and ignore empty memory ranges.
  30  */
  31
  32 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  33
  34 #include <linux/kernel.h>
  35 #include <linux/init.h>
  36 #include <linux/efi.h>
  37 #include <linux/efi-bgrt.h>
  38 #include <linux/export.h>
  39 #include <linux/memblock.h>
  40 #include <linux/slab.h>
  41 #include <linux/spinlock.h>
  42 #include <linux/uaccess.h>
  43 #include <linux/time.h>
  44 #include <linux/io.h>
  45 #include <linux/reboot.h>
  46 #include <linux/bcd.h>
  47
  48 #include <asm/setup.h>
  49 #include <asm/efi.h>
  50 #include <asm/e820/api.h>
  51 #include <asm/time.h>
  52 #include <asm/set_memory.h>
  53 #include <asm/tlbflush.h>
  54 #include <asm/x86_init.h>
  55 #include <asm/uv/uv.h>
  56
  57 static struct efi efi_phys __initdata;
  58 static efi_system_table_t efi_systab __initdata;
  59
  60 static efi_config_table_type_t arch_tables[] __initdata = {
  61 #ifdef CONFIG_X86_UV
  62         {UV_SYSTEM_TABLE_GUID, "UVsystab", &efi.uv_systab},
  63 #endif
  64         {NULL_GUID, NULL, NULL},
  65 };
  66
  67 u64 efi_setup;          /* efi setup_data physical address */
  68
  69 static int add_efi_memmap __initdata;
  70 static int __init setup_add_efi_memmap(char *arg)
  71 {
  72         add_efi_memmap = 1;
  73         return 0;
  74 }
  75 early_param("add_efi_memmap", setup_add_efi_memmap);
  76
  77 static efi_status_t __init phys_efi_set_virtual_address_map(
  78         unsigned long memory_map_size,
  79         unsigned long descriptor_size,
  80         u32 descriptor_version,
  81         efi_memory_desc_t *virtual_map)
  82 {
  83         efi_status_t status;
  84         unsigned long flags;
  85         pgd_t *save_pgd;
  86
  87         save_pgd = efi_call_phys_prolog();
  88         if (!save_pgd)
  89                 return EFI_ABORTED;
  90
  91         /* Disable interrupts around EFI calls: */
  92         local_irq_save(flags);
  93         status = efi_call_phys(efi_phys.set_virtual_address_map,
  94                                memory_map_size, descriptor_size,
  95                                descriptor_version, virtual_map);
  96         local_irq_restore(flags);
  97
  98         efi_call_phys_epilog(save_pgd);
  99
 100         return status;
 101 }
 102
 103 void __init efi_find_mirror(void)
 104 {
 105         efi_memory_desc_t *md;
 106         u64 mirror_size = 0, total_size = 0;
 107
 108         for_each_efi_memory_desc(md) {
 109                 unsigned long long start = md->phys_addr;
 110                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 111
 112                 total_size += size;
 113                 if (md->attribute & EFI_MEMORY_MORE_RELIABLE) {
 114                         memblock_mark_mirror(start, size);
 115                         mirror_size += size;
 116                 }
 117         }
 118         if (mirror_size)
 119                 pr_info("Memory: %lldM/%lldM mirrored memory\n",
 120                         mirror_size>>20, total_size>>20);
 121 }
 122
 123 /*
 124  * Tell the kernel about the EFI memory map.  This might include
 125  * more than the max 128 entries that can fit in the e820 legacy
 126  * (zeropage) memory map.
 127  */
 128
 129 static void __init do_add_efi_memmap(void)
 130 {
 131         efi_memory_desc_t *md;
 132
 133         for_each_efi_memory_desc(md) {
 134                 unsigned long long start = md->phys_addr;
 135                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 136                 int e820_type;
 137
 138                 switch (md->type) {
 139                 case EFI_LOADER_CODE:
 140                 case EFI_LOADER_DATA:
 141                 case EFI_BOOT_SERVICES_CODE:
 142                 case EFI_BOOT_SERVICES_DATA:
 143                 case EFI_CONVENTIONAL_MEMORY:
 144                         if (md->attribute & EFI_MEMORY_WB)
 145                                 e820_type = E820_TYPE_RAM;
 146                         else
 147                                 e820_type = E820_TYPE_RESERVED;
 148                         break;
 149                 case EFI_ACPI_RECLAIM_MEMORY:
 150                         e820_type = E820_TYPE_ACPI;
 151                         break;
 152                 case EFI_ACPI_MEMORY_NVS:
 153                         e820_type = E820_TYPE_NVS;
 154                         break;
 155                 case EFI_UNUSABLE_MEMORY:
 156                         e820_type = E820_TYPE_UNUSABLE;
 157                         break;
 158                 case EFI_PERSISTENT_MEMORY:
 159                         e820_type = E820_TYPE_PMEM;
 160                         break;
 161                 default:
 162                         /*
 163                          * EFI_RESERVED_TYPE EFI_RUNTIME_SERVICES_CODE
 164                          * EFI_RUNTIME_SERVICES_DATA EFI_MEMORY_MAPPED_IO
 165                          * EFI_MEMORY_MAPPED_IO_PORT_SPACE EFI_PAL_CODE
 166                          */
 167                         e820_type = E820_TYPE_RESERVED;
 168                         break;
 169                 }
 170                 e820__range_add(start, size, e820_type);
 171         }
 172         e820__update_table(e820_table);
 173 }
 174
 175 int __init efi_memblock_x86_reserve_range(void)
 176 {
 177         struct efi_info *e = &boot_params.efi_info;
 178         struct efi_memory_map_data data;
 179         phys_addr_t pmap;
 180         int rv;
 181
 182         if (efi_enabled(EFI_PARAVIRT))
 183                 return 0;
 184
 185 #ifdef CONFIG_X86_32
 186         /* Can't handle data above 4GB at this time */
 187         if (e->efi_memmap_hi) {
 188                 pr_err("Memory map is above 4GB, disabling EFI.\n");
 189                 return -EINVAL;
 190         }
 191         pmap =  e->efi_memmap;
 192 #else
 193         pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32));
 194 #endif
 195         data.phys_map           = pmap;
 196         data.size               = e->efi_memmap_size;
 197         data.desc_size          = e->efi_memdesc_size;
 198         data.desc_version       = e->efi_memdesc_version;
 199
 200         rv = efi_memmap_init_early(&data);
 201         if (rv)
 202                 return rv;
 203
 204         if (add_efi_memmap)
 205                 do_add_efi_memmap();
 206
 207         WARN(efi.memmap.desc_version != 1,
 208              "Unexpected EFI_MEMORY_DESCRIPTOR version %ld",
 209              efi.memmap.desc_version);
 210
 211         memblock_reserve(pmap, efi.memmap.nr_map * efi.memmap.desc_size);
 212
 213         return 0;
 214 }
 215
 216 #define OVERFLOW_ADDR_SHIFT     (64 - EFI_PAGE_SHIFT)
 217 #define OVERFLOW_ADDR_MASK      (U64_MAX << OVERFLOW_ADDR_SHIFT)
 218 #define U64_HIGH_BIT            (~(U64_MAX >> 1))
 219
 220 static bool __init efi_memmap_entry_valid(const efi_memory_desc_t *md, int i)
 221 {
 222         u64 end = (md->num_pages << EFI_PAGE_SHIFT) + md->phys_addr - 1;
 223         u64 end_hi = 0;
 224         char buf[64];
 225
 226         if (md->num_pages == 0) {
 227                 end = 0;
 228         } else if (md->num_pages > EFI_PAGES_MAX ||
 229                    EFI_PAGES_MAX - md->num_pages <
 230                    (md->phys_addr >> EFI_PAGE_SHIFT)) {
 231                 end_hi = (md->num_pages & OVERFLOW_ADDR_MASK)
 232                         >> OVERFLOW_ADDR_SHIFT;
 233
 234                 if ((md->phys_addr & U64_HIGH_BIT) && !(end & U64_HIGH_BIT))
 235                         end_hi += 1;
 236         } else {
 237                 return true;
 238         }
 239
 240         pr_warn_once(FW_BUG "Invalid EFI memory map entries:\n");
 241
 242         if (end_hi) {
 243                 pr_warn("mem%02u: %s range=[0x%016llx-0x%llx%016llx] (invalid)\n",
 244                         i, efi_md_typeattr_format(buf, sizeof(buf), md),
 245                         md->phys_addr, end_hi, end);
 246         } else {
 247                 pr_warn("mem%02u: %s range=[0x%016llx-0x%016llx] (invalid)\n",
 248                         i, efi_md_typeattr_format(buf, sizeof(buf), md),
 249                         md->phys_addr, end);
 250         }
 251         return false;
 252 }
 253
 254 static void __init efi_clean_memmap(void)
 255 {
 256         efi_memory_desc_t *out = efi.memmap.map;
 257         const efi_memory_desc_t *in = out;
 258         const efi_memory_desc_t *end = efi.memmap.map_end;
 259         int i, n_removal;
 260
 261         for (i = n_removal = 0; in < end; i++) {
 262                 if (efi_memmap_entry_valid(in, i)) {
 263                         if (out != in)
 264                                 memcpy(out, in, efi.memmap.desc_size);
 265                         out = (void *)out + efi.memmap.desc_size;
 266                 } else {
 267                         n_removal++;
 268                 }
 269                 in = (void *)in + efi.memmap.desc_size;
 270         }
 271
 272         if (n_removal > 0) {
 273                 u64 size = efi.memmap.nr_map - n_removal;
 274
 275                 pr_warn("Removing %d invalid memory map entries.\n", n_removal);
 276                 efi_memmap_install(efi.memmap.phys_map, size);
 277         }
 278 }
 279
 280 void __init efi_print_memmap(void)
 281 {
 282         efi_memory_desc_t *md;
 283         int i = 0;
 284
 285         for_each_efi_memory_desc(md) {
 286                 char buf[64];
 287
 288                 pr_info("mem%02u: %s range=[0x%016llx-0x%016llx] (%lluMB)\n",
 289                         i++, efi_md_typeattr_format(buf, sizeof(buf), md),
 290                         md->phys_addr,
 291                         md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT) - 1,
 292                         (md->num_pages >> (20 - EFI_PAGE_SHIFT)));
 293         }
 294 }
 295
 296 static int __init efi_systab_init(void *phys)
 297 {
 298         if (efi_enabled(EFI_64BIT)) {
 299                 efi_system_table_64_t *systab64;
 300                 struct efi_setup_data *data = NULL;
 301                 u64 tmp = 0;
 302
 303                 if (efi_setup) {
 304                         data = early_memremap(efi_setup, sizeof(*data));
 305                         if (!data)
 306                                 return -ENOMEM;
 307                 }
 308                 systab64 = early_memremap((unsigned long)phys,
 309                                          sizeof(*systab64));
 310                 if (systab64 == NULL) {
 311                         pr_err("Couldn't map the system table!\n");
 312                         if (data)
 313                                 early_memunmap(data, sizeof(*data));
 314                         return -ENOMEM;
 315                 }
 316
 317                 efi_systab.hdr = systab64->hdr;
 318                 efi_systab.fw_vendor = data ? (unsigned long)data->fw_vendor :
 319                                               systab64->fw_vendor;
 320                 tmp |= data ? data->fw_vendor : systab64->fw_vendor;
 321                 efi_systab.fw_revision = systab64->fw_revision;
 322                 efi_systab.con_in_handle = systab64->con_in_handle;
 323                 tmp |= systab64->con_in_handle;
 324                 efi_systab.con_in = systab64->con_in;
 325                 tmp |= systab64->con_in;
 326                 efi_systab.con_out_handle = systab64->con_out_handle;
 327                 tmp |= systab64->con_out_handle;
 328                 efi_systab.con_out = systab64->con_out;
 329                 tmp |= systab64->con_out;
 330                 efi_systab.stderr_handle = systab64->stderr_handle;
 331                 tmp |= systab64->stderr_handle;
 332                 efi_systab.stderr = systab64->stderr;
 333                 tmp |= systab64->stderr;
 334                 efi_systab.runtime = data ?
 335                                      (void *)(unsigned long)data->runtime :
 336                                      (void *)(unsigned long)systab64->runtime;
 337                 tmp |= data ? data->runtime : systab64->runtime;
 338                 efi_systab.boottime = (void *)(unsigned long)systab64->boottime;
 339                 tmp |= systab64->boottime;
 340                 efi_systab.nr_tables = systab64->nr_tables;
 341                 efi_systab.tables = data ? (unsigned long)data->tables :
 342                                            systab64->tables;
 343                 tmp |= data ? data->tables : systab64->tables;
 344
 345                 early_memunmap(systab64, sizeof(*systab64));
 346                 if (data)
 347                         early_memunmap(data, sizeof(*data));
 348 #ifdef CONFIG_X86_32
 349                 if (tmp >> 32) {
 350                         pr_err("EFI data located above 4GB, disabling EFI.\n");
 351                         return -EINVAL;
 352                 }
 353 #endif
 354         } else {
 355                 efi_system_table_32_t *systab32;
 356
 357                 systab32 = early_memremap((unsigned long)phys,
 358                                          sizeof(*systab32));
 359                 if (systab32 == NULL) {
 360                         pr_err("Couldn't map the system table!\n");
 361                         return -ENOMEM;
 362                 }
 363
 364                 efi_systab.hdr = systab32->hdr;
 365                 efi_systab.fw_vendor = systab32->fw_vendor;
 366                 efi_systab.fw_revision = systab32->fw_revision;
 367                 efi_systab.con_in_handle = systab32->con_in_handle;
 368                 efi_systab.con_in = systab32->con_in;
 369                 efi_systab.con_out_handle = systab32->con_out_handle;
 370                 efi_systab.con_out = systab32->con_out;
 371                 efi_systab.stderr_handle = systab32->stderr_handle;
 372                 efi_systab.stderr = systab32->stderr;
 373                 efi_systab.runtime = (void *)(unsigned long)systab32->runtime;
 374                 efi_systab.boottime = (void *)(unsigned long)systab32->boottime;
 375                 efi_systab.nr_tables = systab32->nr_tables;
 376                 efi_systab.tables = systab32->tables;
 377
 378                 early_memunmap(systab32, sizeof(*systab32));
 379         }
 380
 381         efi.systab = &efi_systab;
 382
 383         /*
 384          * Verify the EFI Table
 385          */
 386         if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) {
 387                 pr_err("System table signature incorrect!\n");
 388                 return -EINVAL;
 389         }
 390         if ((efi.systab->hdr.revision >> 16) == 0)
 391                 pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n",
 392                        efi.systab->hdr.revision >> 16,
 393                        efi.systab->hdr.revision & 0xffff);
 394
 395         return 0;
 396 }
 397
 398 static int __init efi_runtime_init32(void)
 399 {
 400         efi_runtime_services_32_t *runtime;
 401
 402         runtime = early_memremap((unsigned long)efi.systab->runtime,
 403                         sizeof(efi_runtime_services_32_t));
 404         if (!runtime) {
 405                 pr_err("Could not map the runtime service table!\n");
 406                 return -ENOMEM;
 407         }
 408
 409         /*
 410          * We will only need *early* access to the SetVirtualAddressMap
 411          * EFI runtime service. All other runtime services will be called
 412          * via the virtual mapping.
 413          */
 414         efi_phys.set_virtual_address_map =
 415                         (efi_set_virtual_address_map_t *)
 416                         (unsigned long)runtime->set_virtual_address_map;
 417         early_memunmap(runtime, sizeof(efi_runtime_services_32_t));
 418
 419         return 0;
 420 }
 421
 422 static int __init efi_runtime_init64(void)
 423 {
 424         efi_runtime_services_64_t *runtime;
 425
 426         runtime = early_memremap((unsigned long)efi.systab->runtime,
 427                         sizeof(efi_runtime_services_64_t));
 428         if (!runtime) {
 429                 pr_err("Could not map the runtime service table!\n");
 430                 return -ENOMEM;
 431         }
 432
 433         /*
 434          * We will only need *early* access to the SetVirtualAddressMap
 435          * EFI runtime service. All other runtime services will be called
 436          * via the virtual mapping.
 437          */
 438         efi_phys.set_virtual_address_map =
 439                         (efi_set_virtual_address_map_t *)
 440                         (unsigned long)runtime->set_virtual_address_map;
 441         early_memunmap(runtime, sizeof(efi_runtime_services_64_t));
 442
 443         return 0;
 444 }
 445
 446 static int __init efi_runtime_init(void)
 447 {
 448         int rv;
 449
 450         /*
 451          * Check out the runtime services table. We need to map
 452          * the runtime services table so that we can grab the physical
 453          * address of several of the EFI runtime functions, needed to
 454          * set the firmware into virtual mode.
 455          *
 456          * When EFI_PARAVIRT is in force then we could not map runtime
 457          * service memory region because we do not have direct access to it.
 458          * However, runtime services are available through proxy functions
 459          * (e.g. in case of Xen dom0 EFI implementation they call special
 460          * hypercall which executes relevant EFI functions) and that is why
 461          * they are always enabled.
 462          */
 463
 464         if (!efi_enabled(EFI_PARAVIRT)) {
 465                 if (efi_enabled(EFI_64BIT))
 466                         rv = efi_runtime_init64();
 467                 else
 468                         rv = efi_runtime_init32();
 469
 470                 if (rv)
 471                         return rv;
 472         }
 473
 474         set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 475
 476         return 0;
 477 }
 478
 479 void __init efi_init(void)
 480 {
 481         efi_char16_t *c16;
 482         char vendor[100] = "unknown";
 483         int i = 0;
 484         void *tmp;
 485
 486 #ifdef CONFIG_X86_32
 487         if (boot_params.efi_info.efi_systab_hi ||
 488             boot_params.efi_info.efi_memmap_hi) {
 489                 pr_info("Table located above 4GB, disabling EFI.\n");
 490                 return;
 491         }
 492         efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab;
 493 #else
 494         efi_phys.systab = (efi_system_table_t *)
 495                           (boot_params.efi_info.efi_systab |
 496                           ((__u64)boot_params.efi_info.efi_systab_hi<<32));
 497 #endif
 498
 499         if (efi_systab_init(efi_phys.systab))
 500                 return;
 501
 502         efi.config_table = (unsigned long)efi.systab->tables;
 503         efi.fw_vendor    = (unsigned long)efi.systab->fw_vendor;
 504         efi.runtime      = (unsigned long)efi.systab->runtime;
 505
 506         /*
 507          * Show what we know for posterity
 508          */
 509         c16 = tmp = early_memremap(efi.systab->fw_vendor, 2);
 510         if (c16) {
 511                 for (i = 0; i < sizeof(vendor) - 1 && *c16; ++i)
 512                         vendor[i] = *c16++;
 513                 vendor[i] = '\0';
 514         } else
 515                 pr_err("Could not map the firmware vendor!\n");
 516         early_memunmap(tmp, 2);
 517
 518         pr_info("EFI v%u.%.02u by %s\n",
 519                 efi.systab->hdr.revision >> 16,
 520                 efi.systab->hdr.revision & 0xffff, vendor);
 521
 522         if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables))
 523                 return;
 524
 525         if (efi_config_init(arch_tables))
 526                 return;
 527
 528         /*
 529          * Note: We currently don't support runtime services on an EFI
 530          * that doesn't match the kernel 32/64-bit mode.
 531          */
 532
 533         if (!efi_runtime_supported())
 534                 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n");
 535         else {
 536                 if (efi_runtime_disabled() || efi_runtime_init()) {
 537                         efi_memmap_unmap();
 538                         return;
 539                 }
 540         }
 541
 542         efi_clean_memmap();
 543
 544         if (efi_enabled(EFI_DBG))
 545                 efi_print_memmap();
 546 }
 547
 548 void __init efi_set_executable(efi_memory_desc_t *md, bool executable)
 549 {
 550         u64 addr, npages;
 551
 552         addr = md->virt_addr;
 553         npages = md->num_pages;
 554
 555         memrange_efi_to_native(&addr, &npages);
 556
 557         if (executable)
 558                 set_memory_x(addr, npages);
 559         else
 560                 set_memory_nx(addr, npages);
 561 }
 562
 563 void __init runtime_code_page_mkexec(void)
 564 {
 565         efi_memory_desc_t *md;
 566
 567         /* Make EFI runtime service code area executable */
 568         for_each_efi_memory_desc(md) {
 569                 if (md->type != EFI_RUNTIME_SERVICES_CODE)
 570                         continue;
 571
 572                 efi_set_executable(md, true);
 573         }
 574 }
 575
 576 void __init efi_memory_uc(u64 addr, unsigned long size)
 577 {
 578         unsigned long page_shift = 1UL << EFI_PAGE_SHIFT;
 579         u64 npages;
 580
 581         npages = round_up(size, page_shift) / page_shift;
 582         memrange_efi_to_native(&addr, &npages);
 583         set_memory_uc(addr, npages);
 584 }
 585
 586 void __init old_map_region(efi_memory_desc_t *md)
 587 {
 588         u64 start_pfn, end_pfn, end;
 589         unsigned long size;
 590         void *va;
 591
 592         start_pfn = PFN_DOWN(md->phys_addr);
 593         size      = md->num_pages << PAGE_SHIFT;
 594         end       = md->phys_addr + size;
 595         end_pfn   = PFN_UP(end);
 596
 597         if (pfn_range_is_mapped(start_pfn, end_pfn)) {
 598                 va = __va(md->phys_addr);
 599
 600                 if (!(md->attribute & EFI_MEMORY_WB))
 601                         efi_memory_uc((u64)(unsigned long)va, size);
 602         } else
 603                 va = efi_ioremap(md->phys_addr, size,
 604                                  md->type, md->attribute);
 605
 606         md->virt_addr = (u64) (unsigned long) va;
 607         if (!va)
 608                 pr_err("ioremap of 0x%llX failed!\n",
 609                        (unsigned long long)md->phys_addr);
 610 }
 611
 612 /* Merge contiguous regions of the same type and attribute */
 613 static void __init efi_merge_regions(void)
 614 {
 615         efi_memory_desc_t *md, *prev_md = NULL;
 616
 617         for_each_efi_memory_desc(md) {
 618                 u64 prev_size;
 619
 620                 if (!prev_md) {
 621                         prev_md = md;
 622                         continue;
 623                 }
 624
 625                 if (prev_md->type != md->type ||
 626                     prev_md->attribute != md->attribute) {
 627                         prev_md = md;
 628                         continue;
 629                 }
 630
 631                 prev_size = prev_md->num_pages << EFI_PAGE_SHIFT;
 632
 633                 if (md->phys_addr == (prev_md->phys_addr + prev_size)) {
 634                         prev_md->num_pages += md->num_pages;
 635                         md->type = EFI_RESERVED_TYPE;
 636                         md->attribute = 0;
 637                         continue;
 638                 }
 639                 prev_md = md;
 640         }
 641 }
 642
 643 static void __init get_systab_virt_addr(efi_memory_desc_t *md)
 644 {
 645         unsigned long size;
 646         u64 end, systab;
 647
 648         size = md->num_pages << EFI_PAGE_SHIFT;
 649         end = md->phys_addr + size;
 650         systab = (u64)(unsigned long)efi_phys.systab;
 651         if (md->phys_addr <= systab && systab < end) {
 652                 systab += md->virt_addr - md->phys_addr;
 653                 efi.systab = (efi_system_table_t *)(unsigned long)systab;
 654         }
 655 }
 656
 657 static void *realloc_pages(void *old_memmap, int old_shift)
 658 {
 659         void *ret;
 660
 661         ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1);
 662         if (!ret)
 663                 goto out;
 664
 665         /*
 666          * A first-time allocation doesn't have anything to copy.
 667          */
 668         if (!old_memmap)
 669                 return ret;
 670
 671         memcpy(ret, old_memmap, PAGE_SIZE << old_shift);
 672
 673 out:
 674         free_pages((unsigned long)old_memmap, old_shift);
 675         return ret;
 676 }
 677
 678 /*
 679  * Iterate the EFI memory map in reverse order because the regions
 680  * will be mapped top-down. The end result is the same as if we had
 681  * mapped things forward, but doesn't require us to change the
 682  * existing implementation of efi_map_region().
 683  */
 684 static inline void *efi_map_next_entry_reverse(void *entry)
 685 {
 686         /* Initial call */
 687         if (!entry)
 688                 return efi.memmap.map_end - efi.memmap.desc_size;
 689
 690         entry -= efi.memmap.desc_size;
 691         if (entry < efi.memmap.map)
 692                 return NULL;
 693
 694         return entry;
 695 }
 696
 697 /*
 698  * efi_map_next_entry - Return the next EFI memory map descriptor
 699  * @entry: Previous EFI memory map descriptor
 700  *
 701  * This is a helper function to iterate over the EFI memory map, which
 702  * we do in different orders depending on the current configuration.
 703  *
 704  * To begin traversing the memory map @entry must be %NULL.
 705  *
 706  * Returns %NULL when we reach the end of the memory map.
 707  */
 708 static void *efi_map_next_entry(void *entry)
 709 {
 710         if (!efi_enabled(EFI_OLD_MEMMAP) && efi_enabled(EFI_64BIT)) {
 711                 /*
 712                  * Starting in UEFI v2.5 the EFI_PROPERTIES_TABLE
 713                  * config table feature requires us to map all entries
 714                  * in the same order as they appear in the EFI memory
 715                  * map. That is to say, entry N must have a lower
 716                  * virtual address than entry N+1. This is because the
 717                  * firmware toolchain leaves relative references in
 718                  * the code/data sections, which are split and become
 719                  * separate EFI memory regions. Mapping things
 720                  * out-of-order leads to the firmware accessing
 721                  * unmapped addresses.
 722                  *
 723                  * Since we need to map things this way whether or not
 724                  * the kernel actually makes use of
 725                  * EFI_PROPERTIES_TABLE, let's just switch to this
 726                  * scheme by default for 64-bit.
 727                  */
 728                 return efi_map_next_entry_reverse(entry);
 729         }
 730
 731         /* Initial call */
 732         if (!entry)
 733                 return efi.memmap.map;
 734
 735         entry += efi.memmap.desc_size;
 736         if (entry >= efi.memmap.map_end)
 737                 return NULL;
 738
 739         return entry;
 740 }
 741
 742 static bool should_map_region(efi_memory_desc_t *md)
 743 {
 744         /*
 745          * Runtime regions always require runtime mappings (obviously).
 746          */
 747         if (md->attribute & EFI_MEMORY_RUNTIME)
 748                 return true;
 749
 750         /*
 751          * 32-bit EFI doesn't suffer from the bug that requires us to
 752          * reserve boot services regions, and mixed mode support
 753          * doesn't exist for 32-bit kernels.
 754          */
 755         if (IS_ENABLED(CONFIG_X86_32))
 756                 return false;
 757
 758         /*
 759          * Map all of RAM so that we can access arguments in the 1:1
 760          * mapping when making EFI runtime calls.
 761          */
 762         if (IS_ENABLED(CONFIG_EFI_MIXED) && !efi_is_native()) {
 763                 if (md->type == EFI_CONVENTIONAL_MEMORY ||
 764                     md->type == EFI_LOADER_DATA ||
 765                     md->type == EFI_LOADER_CODE)
 766                         return true;
 767         }
 768
 769         /*
 770          * Map boot services regions as a workaround for buggy
 771          * firmware that accesses them even when they shouldn't.
 772          *
 773          * See efi_{reserve,free}_boot_services().
 774          */
 775         if (md->type == EFI_BOOT_SERVICES_CODE ||
 776             md->type == EFI_BOOT_SERVICES_DATA)
 777                 return true;
 778
 779         return false;
 780 }
 781
 782 /*
 783  * Map the efi memory ranges of the runtime services and update new_mmap with
 784  * virtual addresses.
 785  */
 786 static void * __init efi_map_regions(int *count, int *pg_shift)
 787 {
 788         void *p, *new_memmap = NULL;
 789         unsigned long left = 0;
 790         unsigned long desc_size;
 791         efi_memory_desc_t *md;
 792
 793         desc_size = efi.memmap.desc_size;
 794
 795         p = NULL;
 796         while ((p = efi_map_next_entry(p))) {
 797                 md = p;
 798
 799                 if (!should_map_region(md))
 800                         continue;
 801
 802                 efi_map_region(md);
 803                 get_systab_virt_addr(md);
 804
 805                 if (left < desc_size) {
 806                         new_memmap = realloc_pages(new_memmap, *pg_shift);
 807                         if (!new_memmap)
 808                                 return NULL;
 809
 810                         left += PAGE_SIZE << *pg_shift;
 811                         (*pg_shift)++;
 812                 }
 813
 814                 memcpy(new_memmap + (*count * desc_size), md, desc_size);
 815
 816                 left -= desc_size;
 817                 (*count)++;
 818         }
 819
 820         return new_memmap;
 821 }
 822
 823 static void __init kexec_enter_virtual_mode(void)
 824 {
 825 #ifdef CONFIG_KEXEC_CORE
 826         efi_memory_desc_t *md;
 827         unsigned int num_pages;
 828
 829         efi.systab = NULL;
 830
 831         /*
 832          * We don't do virtual mode, since we don't do runtime services, on
 833          * non-native EFI. With efi=old_map, we don't do runtime services in
 834          * kexec kernel because in the initial boot something else might
 835          * have been mapped at these virtual addresses.
 836          */
 837         if (!efi_is_native() || efi_enabled(EFI_OLD_MEMMAP)) {
 838                 efi_memmap_unmap();
 839                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 840                 return;
 841         }
 842
 843         if (efi_alloc_page_tables()) {
 844                 pr_err("Failed to allocate EFI page tables\n");
 845                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 846                 return;
 847         }
 848
 849         /*
 850         * Map efi regions which were passed via setup_data. The virt_addr is a
 851         * fixed addr which was used in first kernel of a kexec boot.
 852         */
 853         for_each_efi_memory_desc(md) {
 854                 efi_map_region_fixed(md); /* FIXME: add error handling */
 855                 get_systab_virt_addr(md);
 856         }
 857
 858         /*
 859          * Unregister the early EFI memmap from efi_init() and install
 860          * the new EFI memory map.
 861          */
 862         efi_memmap_unmap();
 863
 864         if (efi_memmap_init_late(efi.memmap.phys_map,
 865                                  efi.memmap.desc_size * efi.memmap.nr_map)) {
 866                 pr_err("Failed to remap late EFI memory map\n");
 867                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 868                 return;
 869         }
 870
 871         BUG_ON(!efi.systab);
 872
 873         num_pages = ALIGN(efi.memmap.nr_map * efi.memmap.desc_size, PAGE_SIZE);
 874         num_pages >>= PAGE_SHIFT;
 875
 876         if (efi_setup_page_tables(efi.memmap.phys_map, num_pages)) {
 877                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 878                 return;
 879         }
 880
 881         efi_sync_low_kernel_mappings();
 882
 883         /*
 884          * Now that EFI is in virtual mode, update the function
 885          * pointers in the runtime service table to the new virtual addresses.
 886          *
 887          * Call EFI services through wrapper functions.
 888          */
 889         efi.runtime_version = efi_systab.hdr.revision;
 890
 891         efi_native_runtime_setup();
 892
 893         efi.set_virtual_address_map = NULL;
 894
 895         if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX))
 896                 runtime_code_page_mkexec();
 897
 898         /* clean DUMMY object */
 899         efi_delete_dummy_variable();
 900 #endif
 901 }
 902
 903 /*
 904  * This function will switch the EFI runtime services to virtual mode.
 905  * Essentially, we look through the EFI memmap and map every region that
 906  * has the runtime attribute bit set in its memory descriptor into the
 907  * efi_pgd page table.
 908  *
 909  * The old method which used to update that memory descriptor with the
 910  * virtual address obtained from ioremap() is still supported when the
 911  * kernel is booted with efi=old_map on its command line. Same old
 912  * method enabled the runtime services to be called without having to
 913  * thunk back into physical mode for every invocation.
 914  *
 915  * The new method does a pagetable switch in a preemption-safe manner
 916  * so that we're in a different address space when calling a runtime
 917  * function. For function arguments passing we do copy the PUDs of the
 918  * kernel page table into efi_pgd prior to each call.
 919  *
 920  * Specially for kexec boot, efi runtime maps in previous kernel should
 921  * be passed in via setup_data. In that case runtime ranges will be mapped
 922  * to the same virtual addresses as the first kernel, see
 923  * kexec_enter_virtual_mode().
 924  */
 925 static void __init __efi_enter_virtual_mode(void)
 926 {
 927         int count = 0, pg_shift = 0;
 928         void *new_memmap = NULL;
 929         efi_status_t status;
 930         unsigned long pa;
 931
 932         efi.systab = NULL;
 933
 934         if (efi_alloc_page_tables()) {
 935                 pr_err("Failed to allocate EFI page tables\n");
 936                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 937                 return;
 938         }
 939
 940         efi_merge_regions();
 941         new_memmap = efi_map_regions(&count, &pg_shift);
 942         if (!new_memmap) {
 943                 pr_err("Error reallocating memory, EFI runtime non-functional!\n");
 944                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 945                 return;
 946         }
 947
 948         pa = __pa(new_memmap);
 949
 950         /*
 951          * Unregister the early EFI memmap from efi_init() and install
 952          * the new EFI memory map that we are about to pass to the
 953          * firmware via SetVirtualAddressMap().
 954          */
 955         efi_memmap_unmap();
 956
 957         if (efi_memmap_init_late(pa, efi.memmap.desc_size * count)) {
 958                 pr_err("Failed to remap late EFI memory map\n");
 959                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 960                 return;
 961         }
 962
 963         if (efi_enabled(EFI_DBG)) {
 964                 pr_info("EFI runtime memory map:\n");
 965                 efi_print_memmap();
 966         }
 967
 968         BUG_ON(!efi.systab);
 969
 970         if (efi_setup_page_tables(pa, 1 << pg_shift)) {
 971                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 972                 return;
 973         }
 974
 975         efi_sync_low_kernel_mappings();
 976
 977         if (efi_is_native()) {
 978                 status = phys_efi_set_virtual_address_map(
 979                                 efi.memmap.desc_size * count,
 980                                 efi.memmap.desc_size,
 981                                 efi.memmap.desc_version,
 982                                 (efi_memory_desc_t *)pa);
 983         } else {
 984                 status = efi_thunk_set_virtual_address_map(
 985                                 efi_phys.set_virtual_address_map,
 986                                 efi.memmap.desc_size * count,
 987                                 efi.memmap.desc_size,
 988                                 efi.memmap.desc_version,
 989                                 (efi_memory_desc_t *)pa);
 990         }
 991
 992         if (status != EFI_SUCCESS) {
 993                 pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n",
 994                          status);
 995                 panic("EFI call to SetVirtualAddressMap() failed!");
 996         }
 997
 998         efi_free_boot_services();
 999
1000         /*
1001          * Now that EFI is in virtual mode, update the function
1002          * pointers in the runtime service table to the new virtual addresses.
1003          *
1004          * Call EFI services through wrapper functions.
1005          */
1006         efi.runtime_version = efi_systab.hdr.revision;
1007
1008         if (efi_is_native())
1009                 efi_native_runtime_setup();
1010         else
1011                 efi_thunk_runtime_setup();
1012
1013         efi.set_virtual_address_map = NULL;
1014
1015         /*
1016          * Apply more restrictive page table mapping attributes now that
1017          * SVAM() has been called and the firmware has performed all
1018          * necessary relocation fixups for the new virtual addresses.
1019          */
1020         efi_runtime_update_mappings();
1021
1022         /* clean DUMMY object */
1023         efi_delete_dummy_variable();
1024 }
1025
1026 void __init efi_enter_virtual_mode(void)
1027 {
1028         if (efi_enabled(EFI_PARAVIRT))
1029                 return;
1030
1031         if (efi_setup)
1032                 kexec_enter_virtual_mode();
1033         else
1034                 __efi_enter_virtual_mode();
1035
1036         efi_dump_pagetable();
1037 }
1038
1039 static int __init arch_parse_efi_cmdline(char *str)
1040 {
1041         if (!str) {
1042                 pr_warn("need at least one option\n");
1043                 return -EINVAL;
1044         }
1045
1046         if (parse_option_str(str, "old_map"))
1047                 set_bit(EFI_OLD_MEMMAP, &efi.flags);
1048
1049         return 0;
1050 }
1051 early_param("efi", arch_parse_efi_cmdline);