[linux-2.6-block.git] / arch / x86 / kernel / head64.c

/*
 *  prepare to run common code
 *
 *  Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
 */

#define DISABLE_BRANCH_PROFILING
#include <linux/init.h>
#include <linux/linkage.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/percpu.h>
#include <linux/start_kernel.h>
#include <linux/io.h>
#include <linux/memblock.h>
#include <linux/mem_encrypt.h>

#include <asm/processor.h>
#include <asm/proto.h>
#include <asm/smp.h>
#include <asm/setup.h>
#include <asm/desc.h>
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
#include <asm/sections.h>
#include <asm/kdebug.h>
#include <asm/e820/api.h>
#include <asm/bios_ebda.h>
#include <asm/bootparam_utils.h>
#include <asm/microcode.h>
#include <asm/kasan.h>

/*
 * Manage page tables very early on.
 */
extern pgd_t early_top_pgt[PTRS_PER_PGD];
extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
static unsigned int __initdata next_early_pgt;
pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX);

#define __head	__section(.head.text)

static void __head *fixup_pointer(void *ptr, unsigned long physaddr)
{
	return ptr - (void *)_text + (void *)physaddr;
}

unsigned long __head __startup_64(unsigned long physaddr)
{
	unsigned long load_delta, *p;
	unsigned long pgtable_flags;
	pgdval_t *pgd;
	p4dval_t *p4d;
	pudval_t *pud;
	pmdval_t *pmd, pmd_entry;
	int i;

	/* Is the address too large? */
	if (physaddr >> MAX_PHYSMEM_BITS)
		for (;;);

	/*
	 * Compute the delta between the address I am compiled to run at
	 * and the address I am actually running at.
	 */
	load_delta = physaddr - (unsigned long)(_text - __START_KERNEL_map);

	/* Is the address not 2M aligned? */
	if (load_delta & ~PMD_PAGE_MASK)
		for (;;);

	/* Activate Secure Memory Encryption (SME) if supported and enabled */
	sme_enable();

	/* Include the SME encryption mask in the fixup value */
	load_delta += sme_get_me_mask();

	/* Fixup the physical addresses in the page table */

	pgd = fixup_pointer(&early_top_pgt, physaddr);
	pgd[pgd_index(__START_KERNEL_map)] += load_delta;

	if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
		p4d = fixup_pointer(&level4_kernel_pgt, physaddr);
		p4d[511] += load_delta;
	}

	pud = fixup_pointer(&level3_kernel_pgt, physaddr);
	pud[510] += load_delta;
	pud[511] += load_delta;

	pmd = fixup_pointer(level2_fixmap_pgt, physaddr);
	pmd[506] += load_delta;

	/*
	 * Set up the identity mapping for the switchover.  These
	 * entries should *NOT* have the global bit set!  This also
	 * creates a bunch of nonsense entries but that is fine --
	 * it avoids problems around wraparound.
	 */

	pud = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);
	pmd = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);
	pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask();

	if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
		p4d = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);

		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
		pgd[i + 0] = (pgdval_t)p4d + pgtable_flags;
		pgd[i + 1] = (pgdval_t)p4d + pgtable_flags;

		i = (physaddr >> P4D_SHIFT) % PTRS_PER_P4D;
		p4d[i + 0] = (pgdval_t)pud + pgtable_flags;
		p4d[i + 1] = (pgdval_t)pud + pgtable_flags;
	} else {
		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
		pgd[i + 0] = (pgdval_t)pud + pgtable_flags;
		pgd[i + 1] = (pgdval_t)pud + pgtable_flags;
	}

	i = (physaddr >> PUD_SHIFT) % PTRS_PER_PUD;
	pud[i + 0] = (pudval_t)pmd + pgtable_flags;
	pud[i + 1] = (pudval_t)pmd + pgtable_flags;

	pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
	pmd_entry += sme_get_me_mask();
	pmd_entry +=  physaddr;

	for (i = 0; i < DIV_ROUND_UP(_end - _text, PMD_SIZE); i++) {
		int idx = i + (physaddr >> PMD_SHIFT) % PTRS_PER_PMD;
		pmd[idx] = pmd_entry + i * PMD_SIZE;
	}

	/*
	 * Fixup the kernel text+data virtual addresses. Note that
	 * we might write invalid pmds, when the kernel is relocated
	 * cleanup_highmap() fixes this up along with the mappings
	 * beyond _end.
	 */

	pmd = fixup_pointer(level2_kernel_pgt, physaddr);
	for (i = 0; i < PTRS_PER_PMD; i++) {
		if (pmd[i] & _PAGE_PRESENT)
			pmd[i] += load_delta;
	}

	/*
	 * Fixup phys_base - remove the memory encryption mask to obtain
	 * the true physical address.
	 */
	p = fixup_pointer(&phys_base, physaddr);
	*p += load_delta - sme_get_me_mask();

	/* Encrypt the kernel (if SME is active) */
	sme_encrypt_kernel();

	/*
	 * Return the SME encryption mask (if SME is active) to be used as a
	 * modifier for the initial pgdir entry programmed into CR3.
	 */
	return sme_get_me_mask();
}

unsigned long __startup_secondary_64(void)
{
	/*
	 * Return the SME encryption mask (if SME is active) to be used as a
	 * modifier for the initial pgdir entry programmed into CR3.
	 */
	return sme_get_me_mask();
}

/* Wipe all early page tables except for the kernel symbol map */
static void __init reset_early_page_tables(void)
{
	memset(early_top_pgt, 0, sizeof(pgd_t)*(PTRS_PER_PGD-1));
	next_early_pgt = 0;
	write_cr3(__sme_pa_nodebug(early_top_pgt));
}

/* Create a new PMD entry */
int __init early_make_pgtable(unsigned long address)
{
	unsigned long physaddr = address - __PAGE_OFFSET;
	pgdval_t pgd, *pgd_p;
	p4dval_t p4d, *p4d_p;
	pudval_t pud, *pud_p;
	pmdval_t pmd, *pmd_p;

	/* Invalid address or early pgt is done ?  */
	if (physaddr >= MAXMEM || read_cr3_pa() != __pa_nodebug(early_top_pgt))
		return -1;

again:
	pgd_p = &early_top_pgt[pgd_index(address)].pgd;
	pgd = *pgd_p;

	/*
	 * The use of __START_KERNEL_map rather than __PAGE_OFFSET here is
	 * critical -- __PAGE_OFFSET would point us back into the dynamic
	 * range and we might end up looping forever...
	 */
	if (!IS_ENABLED(CONFIG_X86_5LEVEL))
		p4d_p = pgd_p;
	else if (pgd)
		p4d_p = (p4dval_t *)((pgd & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
	else {
		if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
			reset_early_page_tables();
			goto again;
		}

		p4d_p = (p4dval_t *)early_dynamic_pgts[next_early_pgt++];
		memset(p4d_p, 0, sizeof(*p4d_p) * PTRS_PER_P4D);
		*pgd_p = (pgdval_t)p4d_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
	}
	p4d_p += p4d_index(address);
	p4d = *p4d_p;

	if (p4d)
		pud_p = (pudval_t *)((p4d & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
	else {
		if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
			reset_early_page_tables();
			goto again;
		}

		pud_p = (pudval_t *)early_dynamic_pgts[next_early_pgt++];
		memset(pud_p, 0, sizeof(*pud_p) * PTRS_PER_PUD);
		*p4d_p = (p4dval_t)pud_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
	}
	pud_p += pud_index(address);
	pud = *pud_p;

	if (pud)
		pmd_p = (pmdval_t *)((pud & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
	else {
		if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
			reset_early_page_tables();
			goto again;
		}

		pmd_p = (pmdval_t *)early_dynamic_pgts[next_early_pgt++];
		memset(pmd_p, 0, sizeof(*pmd_p) * PTRS_PER_PMD);
		*pud_p = (pudval_t)pmd_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
	}
	pmd = (physaddr & PMD_MASK) + early_pmd_flags;
	pmd_p[pmd_index(address)] = pmd;

	return 0;
}

/* Don't add a printk in there. printk relies on the PDA which is not initialized 
   yet. */
static void __init clear_bss(void)
{
	memset(__bss_start, 0,
	       (unsigned long) __bss_stop - (unsigned long) __bss_start);
}

static unsigned long get_cmd_line_ptr(void)
{
	unsigned long cmd_line_ptr = boot_params.hdr.cmd_line_ptr;

	cmd_line_ptr |= (u64)boot_params.ext_cmd_line_ptr << 32;

	return cmd_line_ptr;
}

static void __init copy_bootdata(char *real_mode_data)
{
	char * command_line;
	unsigned long cmd_line_ptr;

	memcpy(&boot_params, real_mode_data, sizeof boot_params);
	sanitize_boot_params(&boot_params);
	cmd_line_ptr = get_cmd_line_ptr();
	if (cmd_line_ptr) {
		command_line = __va(cmd_line_ptr);
		memcpy(boot_command_line, command_line, COMMAND_LINE_SIZE);
	}
}

asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data)
{
	int i;

	/*
	 * Build-time sanity checks on the kernel image and module
	 * area mappings. (these are purely build-time and produce no code)
	 */
	BUILD_BUG_ON(MODULES_VADDR < __START_KERNEL_map);
	BUILD_BUG_ON(MODULES_VADDR - __START_KERNEL_map < KERNEL_IMAGE_SIZE);
	BUILD_BUG_ON(MODULES_LEN + KERNEL_IMAGE_SIZE > 2*PUD_SIZE);
	BUILD_BUG_ON((__START_KERNEL_map & ~PMD_MASK) != 0);
	BUILD_BUG_ON((MODULES_VADDR & ~PMD_MASK) != 0);
	BUILD_BUG_ON(!(MODULES_VADDR > __START_KERNEL));
	BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==
				(__START_KERNEL & PGDIR_MASK)));
	BUILD_BUG_ON(__fix_to_virt(__end_of_fixed_addresses) <= MODULES_END);

	cr4_init_shadow();

	/* Kill off the identity-map trampoline */
	reset_early_page_tables();

	clear_bss();

	clear_page(init_top_pgt);

	/*
	 * SME support may update early_pmd_flags to include the memory
	 * encryption mask, so it needs to be called before anything
	 * that may generate a page fault.
	 */
	sme_early_init();

	kasan_early_init();

	for (i = 0; i < NUM_EXCEPTION_VECTORS; i++)
		set_intr_gate(i, early_idt_handler_array[i]);
	load_idt((const struct desc_ptr *)&idt_descr);

	copy_bootdata(__va(real_mode_data));

	/*
	 * Load microcode early on BSP.
	 */
	load_ucode_bsp();

	/* set init_top_pgt kernel high mapping*/
	init_top_pgt[511] = early_top_pgt[511];

	x86_64_start_reservations(real_mode_data);
}

void __init x86_64_start_reservations(char *real_mode_data)
{
	/* version is always not zero if it is copied */
	if (!boot_params.hdr.version)
		copy_bootdata(__va(real_mode_data));

	x86_early_init_platform_quirks();

	switch (boot_params.hdr.hardware_subarch) {
	case X86_SUBARCH_INTEL_MID:
		x86_intel_mid_early_setup();
		break;
	default:
		break;
	}

	start_kernel();
}
Commit	Line	Data
1da177e4	1	/*
835c34a1	2	* prepare to run common code
1da177e4 LT	3	*
1da177e4 LT	4	* Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
1da177e4 LT	5	*/
1da177e4 LT	6
be3606ff	7	#define DISABLE_BRANCH_PROFILING
1da177e4 LT	8	#include <linux/init.h>
	9	#include <linux/linkage.h>
	10	#include <linux/types.h>
	11	#include <linux/kernel.h>
	12	#include <linux/string.h>
	13	#include <linux/percpu.h>
eaf76e8b	14	#include <linux/start_kernel.h>
8b664aa6	15	#include <linux/io.h>
72d7c3b3	16	#include <linux/memblock.h>
5868f365	17	#include <linux/mem_encrypt.h>
1da177e4 LT	18
	19	#include <asm/processor.h>
	20	#include <asm/proto.h>
	21	#include <asm/smp.h>
1da177e4 LT	22	#include <asm/setup.h>
1da177e4 LT	23	#include <asm/desc.h>
f6c2e333	24	#include <asm/pgtable.h>
cfd243d4	25	#include <asm/tlbflush.h>
2bc0414e	26	#include <asm/sections.h>
718fc13b	27	#include <asm/kdebug.h>
66441bd3	28	#include <asm/e820/api.h>
47a3d5da	29	#include <asm/bios_ebda.h>
5dcd14ec	30	#include <asm/bootparam_utils.h>
feddc9de	31	#include <asm/microcode.h>
ef7f0d6a	32	#include <asm/kasan.h>
1da177e4	33
8170e6be PA	34	/*
	35	* Manage page tables very early on.
	36	*/
65ade2f8	37	extern pgd_t early_top_pgt[PTRS_PER_PGD];
8170e6be	38	extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
c88d7150	39	static unsigned int __initdata next_early_pgt;
5e427ec2	40	pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL \| _PAGE_NX);
8170e6be	41
26179670 KS	42	#define __head __section(.head.text)
	43
	44	static void __head fixup_pointer(void ptr, unsigned long physaddr)
c88d7150 KS	45	{
	46	return ptr - (void )_text + (void )physaddr;
	47	}
	48
5868f365	49	unsigned long __head __startup_64(unsigned long physaddr)
c88d7150 KS	50	{
c88d7150 KS	51	unsigned long load_delta, *p;
5868f365	52	unsigned long pgtable_flags;
c88d7150	53	pgdval_t *pgd;
032370b9	54	p4dval_t *p4d;
c88d7150 KS	55	pudval_t *pud;
	56	pmdval_t *pmd, pmd_entry;
	57	int i;
	58
	59	/* Is the address too large? */
	60	if (physaddr >> MAX_PHYSMEM_BITS)
	61	for (;;);
	62
	63	/*
	64	* Compute the delta between the address I am compiled to run at
	65	* and the address I am actually running at.
	66	*/
	67	load_delta = physaddr - (unsigned long)(_text - __START_KERNEL_map);
	68
	69	/* Is the address not 2M aligned? */
	70	if (load_delta & ~PMD_PAGE_MASK)
	71	for (;;);
	72
5868f365 TL	73	/* Activate Secure Memory Encryption (SME) if supported and enabled */
	74	sme_enable();
	75
	76	/* Include the SME encryption mask in the fixup value */
	77	load_delta += sme_get_me_mask();
	78
c88d7150 KS	79	/* Fixup the physical addresses in the page table */
c88d7150 KS	80
65ade2f8	81	pgd = fixup_pointer(&early_top_pgt, physaddr);
c88d7150 KS	82	pgd[pgd_index(__START_KERNEL_map)] += load_delta;
c88d7150 KS	83
032370b9 KS	84	if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
	85	p4d = fixup_pointer(&level4_kernel_pgt, physaddr);
	86	p4d[511] += load_delta;
	87	}
	88
c88d7150 KS	89	pud = fixup_pointer(&level3_kernel_pgt, physaddr);
	90	pud[510] += load_delta;
	91	pud[511] += load_delta;
	92
	93	pmd = fixup_pointer(level2_fixmap_pgt, physaddr);
	94	pmd[506] += load_delta;
	95
	96	/*
	97	* Set up the identity mapping for the switchover. These
	98	* entries should NOT have the global bit set! This also
	99	* creates a bunch of nonsense entries but that is fine --
	100	* it avoids problems around wraparound.
	101	*/
	102
	103	pud = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);
	104	pmd = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);
21729f81	105	pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask();
c88d7150	106
032370b9 KS	107	if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
	108	p4d = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);
	109
	110	i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
5868f365 TL	111	pgd[i + 0] = (pgdval_t)p4d + pgtable_flags;
5868f365 TL	112	pgd[i + 1] = (pgdval_t)p4d + pgtable_flags;
032370b9 KS	113
032370b9 KS	114	i = (physaddr >> P4D_SHIFT) % PTRS_PER_P4D;
5868f365 TL	115	p4d[i + 0] = (pgdval_t)pud + pgtable_flags;
5868f365 TL	116	p4d[i + 1] = (pgdval_t)pud + pgtable_flags;
032370b9 KS	117	} else {
032370b9 KS	118	i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
5868f365 TL	119	pgd[i + 0] = (pgdval_t)pud + pgtable_flags;
5868f365 TL	120	pgd[i + 1] = (pgdval_t)pud + pgtable_flags;
032370b9	121	}
c88d7150 KS	122
c88d7150 KS	123	i = (physaddr >> PUD_SHIFT) % PTRS_PER_PUD;
5868f365 TL	124	pud[i + 0] = (pudval_t)pmd + pgtable_flags;
5868f365 TL	125	pud[i + 1] = (pudval_t)pmd + pgtable_flags;
c88d7150 KS	126
c88d7150 KS	127	pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
5868f365	128	pmd_entry += sme_get_me_mask();
c88d7150 KS	129	pmd_entry += physaddr;
	130
	131	for (i = 0; i < DIV_ROUND_UP(_end - _text, PMD_SIZE); i++) {
	132	int idx = i + (physaddr >> PMD_SHIFT) % PTRS_PER_PMD;
	133	pmd[idx] = pmd_entry + i * PMD_SIZE;
	134	}
	135
	136	/*
	137	* Fixup the kernel text+data virtual addresses. Note that
	138	* we might write invalid pmds, when the kernel is relocated
	139	* cleanup_highmap() fixes this up along with the mappings
	140	* beyond _end.
	141	*/
	142
	143	pmd = fixup_pointer(level2_kernel_pgt, physaddr);
	144	for (i = 0; i < PTRS_PER_PMD; i++) {
	145	if (pmd[i] & _PAGE_PRESENT)
	146	pmd[i] += load_delta;
	147	}
	148
5868f365 TL	149	/*
	150	* Fixup phys_base - remove the memory encryption mask to obtain
	151	* the true physical address.
	152	*/
c88d7150	153	p = fixup_pointer(&phys_base, physaddr);
5868f365 TL	154	*p += load_delta - sme_get_me_mask();
	155
	156	/* Encrypt the kernel (if SME is active) */
	157	sme_encrypt_kernel();
	158
	159	/*
	160	* Return the SME encryption mask (if SME is active) to be used as a
	161	* modifier for the initial pgdir entry programmed into CR3.
	162	*/
	163	return sme_get_me_mask();
	164	}
	165
	166	unsigned long __startup_secondary_64(void)
	167	{
	168	/*
	169	* Return the SME encryption mask (if SME is active) to be used as a
	170	* modifier for the initial pgdir entry programmed into CR3.
	171	*/
	172	return sme_get_me_mask();
c88d7150 KS	173	}
c88d7150 KS	174
8170e6be PA	175	/* Wipe all early page tables except for the kernel symbol map */
8170e6be PA	176	static void __init reset_early_page_tables(void)
cfd243d4	177	{
65ade2f8	178	memset(early_top_pgt, 0, sizeof(pgd_t)*(PTRS_PER_PGD-1));
8170e6be	179	next_early_pgt = 0;
21729f81	180	write_cr3(__sme_pa_nodebug(early_top_pgt));
8170e6be PA	181	}
	182
	183	/* Create a new PMD entry */
	184	int __init early_make_pgtable(unsigned long address)
	185	{
	186	unsigned long physaddr = address - __PAGE_OFFSET;
8170e6be	187	pgdval_t pgd, *pgd_p;
032370b9	188	p4dval_t p4d, *p4d_p;
6b9c75ac	189	pudval_t pud, *pud_p;
8170e6be PA	190	pmdval_t pmd, *pmd_p;
	191
	192	/* Invalid address or early pgt is done ? */
65ade2f8	193	if (physaddr >= MAXMEM \|\| read_cr3_pa() != __pa_nodebug(early_top_pgt))
8170e6be PA	194	return -1;
8170e6be PA	195
6b9c75ac	196	again:
65ade2f8	197	pgd_p = &early_top_pgt[pgd_index(address)].pgd;
8170e6be PA	198	pgd = *pgd_p;
	199
	200	/*
	201	* The use of __START_KERNEL_map rather than __PAGE_OFFSET here is
	202	* critical -- __PAGE_OFFSET would point us back into the dynamic
	203	* range and we might end up looping forever...
	204	*/
032370b9 KS	205	if (!IS_ENABLED(CONFIG_X86_5LEVEL))
	206	p4d_p = pgd_p;
	207	else if (pgd)
	208	p4d_p = (p4dval_t *)((pgd & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
	209	else {
	210	if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
	211	reset_early_page_tables();
	212	goto again;
	213	}
	214
	215	p4d_p = (p4dval_t *)early_dynamic_pgts[next_early_pgt++];
	216	memset(p4d_p, 0, sizeof(p4d_p) PTRS_PER_P4D);
	217	*pgd_p = (pgdval_t)p4d_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
	218	}
	219	p4d_p += p4d_index(address);
	220	p4d = *p4d_p;
	221
	222	if (p4d)
	223	pud_p = (pudval_t *)((p4d & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
6b9c75ac YL	224	else {
6b9c75ac YL	225	if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
8170e6be	226	reset_early_page_tables();
6b9c75ac YL	227	goto again;
6b9c75ac YL	228	}
8170e6be PA	229
8170e6be PA	230	pud_p = (pudval_t *)early_dynamic_pgts[next_early_pgt++];
a91bbe01	231	memset(pud_p, 0, sizeof(pud_p) PTRS_PER_PUD);
032370b9	232	*p4d_p = (p4dval_t)pud_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
8170e6be	233	}
6b9c75ac YL	234	pud_p += pud_index(address);
6b9c75ac YL	235	pud = *pud_p;
8170e6be	236
6b9c75ac YL	237	if (pud)
	238	pmd_p = (pmdval_t *)((pud & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
	239	else {
	240	if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
	241	reset_early_page_tables();
	242	goto again;
	243	}
	244
	245	pmd_p = (pmdval_t *)early_dynamic_pgts[next_early_pgt++];
a91bbe01	246	memset(pmd_p, 0, sizeof(pmd_p) PTRS_PER_PMD);
6b9c75ac YL	247	*pud_p = (pudval_t)pmd_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
6b9c75ac YL	248	}
78d77df7	249	pmd = (physaddr & PMD_MASK) + early_pmd_flags;
6b9c75ac	250	pmd_p[pmd_index(address)] = pmd;
8170e6be PA	251
8170e6be PA	252	return 0;
cfd243d4 VG	253	}
cfd243d4 VG	254
1da177e4 LT	255	/* Don't add a printk in there. printk relies on the PDA which is not initialized
	256	yet. */
	257	static void __init clear_bss(void)
	258	{
1da177e4	259	memset(__bss_start, 0,
2bc0414e	260	(unsigned long) __bss_stop - (unsigned long) __bss_start);
1da177e4 LT	261	}
1da177e4 LT	262
f1da834c YL	263	static unsigned long get_cmd_line_ptr(void)
	264	{
	265	unsigned long cmd_line_ptr = boot_params.hdr.cmd_line_ptr;
	266
ee92d815 YL	267	cmd_line_ptr \|= (u64)boot_params.ext_cmd_line_ptr << 32;
ee92d815 YL	268
f1da834c YL	269	return cmd_line_ptr;
	270	}
	271
1da177e4 LT	272	static void __init copy_bootdata(char *real_mode_data)
1da177e4 LT	273	{
1da177e4	274	char * command_line;
f1da834c	275	unsigned long cmd_line_ptr;
1da177e4	276
30c82645	277	memcpy(&boot_params, real_mode_data, sizeof boot_params);
5dcd14ec	278	sanitize_boot_params(&boot_params);
f1da834c YL	279	cmd_line_ptr = get_cmd_line_ptr();
	280	if (cmd_line_ptr) {
	281	command_line = __va(cmd_line_ptr);
30c82645	282	memcpy(boot_command_line, command_line, COMMAND_LINE_SIZE);
1da177e4	283	}
1da177e4 LT	284	}
1da177e4 LT	285
2605fc21	286	asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data)
1da177e4	287	{
1da177e4 LT	288	int i;
1da177e4 LT	289
b4e0409a IM	290	/*
	291	* Build-time sanity checks on the kernel image and module
	292	* area mappings. (these are purely build-time and produce no code)
	293	*/
8e3c2a8c BP	294	BUILD_BUG_ON(MODULES_VADDR < __START_KERNEL_map);
8e3c2a8c BP	295	BUILD_BUG_ON(MODULES_VADDR - __START_KERNEL_map < KERNEL_IMAGE_SIZE);
b4e0409a	296	BUILD_BUG_ON(MODULES_LEN + KERNEL_IMAGE_SIZE > 2*PUD_SIZE);
8e3c2a8c	297	BUILD_BUG_ON((__START_KERNEL_map & ~PMD_MASK) != 0);
b4e0409a IM	298	BUILD_BUG_ON((MODULES_VADDR & ~PMD_MASK) != 0);
	299	BUILD_BUG_ON(!(MODULES_VADDR > __START_KERNEL));
	300	BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==
	301	(__START_KERNEL & PGDIR_MASK)));
66d4bdf2	302	BUILD_BUG_ON(__fix_to_virt(__end_of_fixed_addresses) <= MODULES_END);
b4e0409a	303
1e02ce4c AL	304	cr4_init_shadow();
1e02ce4c AL	305
8170e6be PA	306	/* Kill off the identity-map trampoline */
	307	reset_early_page_tables();
	308
3df0af0e YL	309	clear_bss();
3df0af0e YL	310
65ade2f8	311	clear_page(init_top_pgt);
d0f77d4d	312
21729f81 TL	313	/*
	314	* SME support may update early_pmd_flags to include the memory
	315	* encryption mask, so it needs to be called before anything
	316	* that may generate a page fault.
	317	*/
	318	sme_early_init();
	319
5d5aa3cf AP	320	kasan_early_init();
5d5aa3cf AP	321
ac630dd9	322	for (i = 0; i < NUM_EXCEPTION_VECTORS; i++)
425be567	323	set_intr_gate(i, early_idt_handler_array[i]);
9d1c6e7c	324	load_idt((const struct desc_ptr *)&idt_descr);
f6c2e333	325
fa2bbce9 YL	326	copy_bootdata(__va(real_mode_data));
fa2bbce9 YL	327
feddc9de FY	328	/*
	329	* Load microcode early on BSP.
	330	*/
	331	load_ucode_bsp();
	332
65ade2f8 KS	333	/* set init_top_pgt kernel high mapping*/
65ade2f8 KS	334	init_top_pgt[511] = early_top_pgt[511];
8170e6be	335
f97013fd JF	336	x86_64_start_reservations(real_mode_data);
	337	}
	338
	339	void __init x86_64_start_reservations(char *real_mode_data)
	340	{
fa2bbce9 YL	341	/* version is always not zero if it is copied */
	342	if (!boot_params.hdr.version)
	343	copy_bootdata(__va(real_mode_data));
9de819fe	344
8d152e7a	345	x86_early_init_platform_quirks();
75175278	346
3fda5bb4 AS	347	switch (boot_params.hdr.hardware_subarch) {
	348	case X86_SUBARCH_INTEL_MID:
	349	x86_intel_mid_early_setup();
	350	break;
	351	default:
	352	break;
	353	}
	354
1da177e4 LT	355	start_kernel();
1da177e4 LT	356	}