[linux-block.git] / arch / s390 / boot / vmem.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/sched/task.h>
#include <linux/pgtable.h>
#include <linux/kasan.h>
#include <asm/pgalloc.h>
#include <asm/facility.h>
#include <asm/sections.h>
#include <asm/physmem_info.h>
#include <asm/maccess.h>
#include <asm/abs_lowcore.h>
#include "decompressor.h"
#include "boot.h"

unsigned long __bootdata_preserved(s390_invalid_asce);

#ifdef CONFIG_PROC_FS
atomic_long_t __bootdata_preserved(direct_pages_count[PG_DIRECT_MAP_MAX]);
#endif

#define init_mm			(*(struct mm_struct *)vmlinux.init_mm_off)
#define swapper_pg_dir		vmlinux.swapper_pg_dir_off
#define invalid_pg_dir		vmlinux.invalid_pg_dir_off

enum populate_mode {
	POPULATE_NONE,
	POPULATE_DIRECT,
	POPULATE_ABS_LOWCORE,
#ifdef CONFIG_KASAN
	POPULATE_KASAN_MAP_SHADOW,
	POPULATE_KASAN_ZERO_SHADOW,
	POPULATE_KASAN_SHALLOW
#endif
};

static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode);

#ifdef CONFIG_KASAN

#define kasan_early_shadow_page	vmlinux.kasan_early_shadow_page_off
#define kasan_early_shadow_pte	((pte_t *)vmlinux.kasan_early_shadow_pte_off)
#define kasan_early_shadow_pmd	((pmd_t *)vmlinux.kasan_early_shadow_pmd_off)
#define kasan_early_shadow_pud	((pud_t *)vmlinux.kasan_early_shadow_pud_off)
#define kasan_early_shadow_p4d	((p4d_t *)vmlinux.kasan_early_shadow_p4d_off)
#define __sha(x)		((unsigned long)kasan_mem_to_shadow((void *)x))

static pte_t pte_z;

static inline void kasan_populate(unsigned long start, unsigned long end, enum populate_mode mode)
{
	start = PAGE_ALIGN_DOWN(__sha(start));
	end = PAGE_ALIGN(__sha(end));
	pgtable_populate(start, end, mode);
}

static void kasan_populate_shadow(void)
{
	pmd_t pmd_z = __pmd(__pa(kasan_early_shadow_pte) | _SEGMENT_ENTRY);
	pud_t pud_z = __pud(__pa(kasan_early_shadow_pmd) | _REGION3_ENTRY);
	p4d_t p4d_z = __p4d(__pa(kasan_early_shadow_pud) | _REGION2_ENTRY);
	unsigned long memgap_start = 0;
	unsigned long untracked_end;
	unsigned long start, end;
	int i;

	pte_z = __pte(__pa(kasan_early_shadow_page) | pgprot_val(PAGE_KERNEL_RO));
	if (!machine.has_nx)
		pte_z = clear_pte_bit(pte_z, __pgprot(_PAGE_NOEXEC));
	crst_table_init((unsigned long *)kasan_early_shadow_p4d, p4d_val(p4d_z));
	crst_table_init((unsigned long *)kasan_early_shadow_pud, pud_val(pud_z));
	crst_table_init((unsigned long *)kasan_early_shadow_pmd, pmd_val(pmd_z));
	memset64((u64 *)kasan_early_shadow_pte, pte_val(pte_z), PTRS_PER_PTE);

	/*
	 * Current memory layout:
	 * +- 0 -------------+	       +- shadow start -+
	 * |1:1 ident mapping|	      /|1/8 of ident map|
	 * |		     |	     / |		|
	 * +-end of ident map+	    /  +----------------+
	 * | ... gap ...     |	   /   |    kasan	|
	 * |		     |	  /    |  zero page	|
	 * +- vmalloc area  -+	 /     |   mapping	|
	 * | vmalloc_size    |	/      | (untracked)	|
	 * +- modules vaddr -+ /       +----------------+
	 * | 2Gb	     |/        |    unmapped	| allocated per module
	 * +- shadow start  -+	       +----------------+
	 * | 1/8 addr space  |	       | zero pg mapping| (untracked)
	 * +- shadow end ----+---------+- shadow end ---+
	 *
	 * Current memory layout (KASAN_VMALLOC):
	 * +- 0 -------------+	       +- shadow start -+
	 * |1:1 ident mapping|	      /|1/8 of ident map|
	 * |		     |	     / |		|
	 * +-end of ident map+	    /  +----------------+
	 * | ... gap ...     |	   /   | kasan zero page| (untracked)
	 * |		     |	  /    | mapping	|
	 * +- vmalloc area  -+	 /     +----------------+
	 * | vmalloc_size    |	/      |shallow populate|
	 * +- modules vaddr -+ /       +----------------+
	 * | 2Gb	     |/        |shallow populate|
	 * +- shadow start  -+	       +----------------+
	 * | 1/8 addr space  |	       | zero pg mapping| (untracked)
	 * +- shadow end ----+---------+- shadow end ---+
	 */

	for_each_physmem_usable_range(i, &start, &end) {
		kasan_populate(start, end, POPULATE_KASAN_MAP_SHADOW);
		if (memgap_start && physmem_info.info_source == MEM_DETECT_DIAG260)
			kasan_populate(memgap_start, start, POPULATE_KASAN_ZERO_SHADOW);
		memgap_start = end;
	}
	if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
		untracked_end = VMALLOC_START;
		/* shallowly populate kasan shadow for vmalloc and modules */
		kasan_populate(VMALLOC_START, MODULES_END, POPULATE_KASAN_SHALLOW);
	} else {
		untracked_end = MODULES_VADDR;
	}
	/* populate kasan shadow for untracked memory */
	kasan_populate(ident_map_size, untracked_end, POPULATE_KASAN_ZERO_SHADOW);
	kasan_populate(MODULES_END, _REGION1_SIZE, POPULATE_KASAN_ZERO_SHADOW);
}

static bool kasan_pgd_populate_zero_shadow(pgd_t *pgd, unsigned long addr,
					   unsigned long end, enum populate_mode mode)
{
	if (mode == POPULATE_KASAN_ZERO_SHADOW &&
	    IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) {
		pgd_populate(&init_mm, pgd, kasan_early_shadow_p4d);
		return true;
	}
	return false;
}

static bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr,
					   unsigned long end, enum populate_mode mode)
{
	if (mode == POPULATE_KASAN_ZERO_SHADOW &&
	    IS_ALIGNED(addr, P4D_SIZE) && end - addr >= P4D_SIZE) {
		p4d_populate(&init_mm, p4d, kasan_early_shadow_pud);
		return true;
	}
	return false;
}

static bool kasan_pud_populate_zero_shadow(pud_t *pud, unsigned long addr,
					   unsigned long end, enum populate_mode mode)
{
	if (mode == POPULATE_KASAN_ZERO_SHADOW &&
	    IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) {
		pud_populate(&init_mm, pud, kasan_early_shadow_pmd);
		return true;
	}
	return false;
}

static bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr,
					   unsigned long end, enum populate_mode mode)
{
	if (mode == POPULATE_KASAN_ZERO_SHADOW &&
	    IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) {
		pmd_populate(&init_mm, pmd, kasan_early_shadow_pte);
		return true;
	}
	return false;
}

static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode)
{
	pte_t entry;

	if (mode == POPULATE_KASAN_ZERO_SHADOW) {
		set_pte(pte, pte_z);
		return true;
	}
	return false;
}
#else

static inline void kasan_populate_shadow(void) {}

static inline bool kasan_pgd_populate_zero_shadow(pgd_t *pgd, unsigned long addr,
						  unsigned long end, enum populate_mode mode)
{
	return false;
}

static inline bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr,
						  unsigned long end, enum populate_mode mode)
{
	return false;
}

static inline bool kasan_pud_populate_zero_shadow(pud_t *pud, unsigned long addr,
						  unsigned long end, enum populate_mode mode)
{
	return false;
}

static inline bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr,
						  unsigned long end, enum populate_mode mode)
{
	return false;
}

static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode)
{
	return false;
}

#endif

/*
 * Mimic virt_to_kpte() in lack of init_mm symbol. Skip pmd NULL check though.
 */
static inline pte_t *__virt_to_kpte(unsigned long va)
{
	return pte_offset_kernel(pmd_offset(pud_offset(p4d_offset(pgd_offset_k(va), va), va), va), va);
}

static void *boot_crst_alloc(unsigned long val)
{
	unsigned long size = PAGE_SIZE << CRST_ALLOC_ORDER;
	unsigned long *table;

	table = (unsigned long *)physmem_alloc_top_down(RR_VMEM, size, size);
	crst_table_init(table, val);
	return table;
}

static pte_t *boot_pte_alloc(void)
{
	static void *pte_leftover;
	pte_t *pte;

	/*
	 * handling pte_leftovers this way helps to avoid memory fragmentation
	 * during POPULATE_KASAN_MAP_SHADOW when EDAT is off
	 */
	if (!pte_leftover) {
		pte_leftover = (void *)physmem_alloc_top_down(RR_VMEM, PAGE_SIZE, PAGE_SIZE);
		pte = pte_leftover + _PAGE_TABLE_SIZE;
	} else {
		pte = pte_leftover;
		pte_leftover = NULL;
	}

	memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE);
	return pte;
}

static unsigned long _pa(unsigned long addr, unsigned long size, enum populate_mode mode)
{
	switch (mode) {
	case POPULATE_NONE:
		return -1;
	case POPULATE_DIRECT:
		return addr;
	case POPULATE_ABS_LOWCORE:
		return __abs_lowcore_pa(addr);
#ifdef CONFIG_KASAN
	case POPULATE_KASAN_MAP_SHADOW:
		addr = physmem_alloc_top_down(RR_VMEM, size, size);
		memset((void *)addr, 0, size);
		return addr;
#endif
	default:
		return -1;
	}
}

static bool can_large_pud(pud_t *pu_dir, unsigned long addr, unsigned long end)
{
	return machine.has_edat2 &&
	       IS_ALIGNED(addr, PUD_SIZE) && (end - addr) >= PUD_SIZE;
}

static bool can_large_pmd(pmd_t *pm_dir, unsigned long addr, unsigned long end)
{
	return machine.has_edat1 &&
	       IS_ALIGNED(addr, PMD_SIZE) && (end - addr) >= PMD_SIZE;
}

static void pgtable_pte_populate(pmd_t *pmd, unsigned long addr, unsigned long end,
				 enum populate_mode mode)
{
	unsigned long pages = 0;
	pte_t *pte, entry;

	pte = pte_offset_kernel(pmd, addr);
	for (; addr < end; addr += PAGE_SIZE, pte++) {
		if (pte_none(*pte)) {
			if (kasan_pte_populate_zero_shadow(pte, mode))
				continue;
			entry = __pte(_pa(addr, PAGE_SIZE, mode));
			entry = set_pte_bit(entry, PAGE_KERNEL);
			if (!machine.has_nx)
				entry = clear_pte_bit(entry, __pgprot(_PAGE_NOEXEC));
			set_pte(pte, entry);
			pages++;
		}
	}
	if (mode == POPULATE_DIRECT)
		update_page_count(PG_DIRECT_MAP_4K, pages);
}

static void pgtable_pmd_populate(pud_t *pud, unsigned long addr, unsigned long end,
				 enum populate_mode mode)
{
	unsigned long next, pages = 0;
	pmd_t *pmd, entry;
	pte_t *pte;

	pmd = pmd_offset(pud, addr);
	for (; addr < end; addr = next, pmd++) {
		next = pmd_addr_end(addr, end);
		if (pmd_none(*pmd)) {
			if (kasan_pmd_populate_zero_shadow(pmd, addr, next, mode))
				continue;
			if (can_large_pmd(pmd, addr, next)) {
				entry = __pmd(_pa(addr, _SEGMENT_SIZE, mode));
				entry = set_pmd_bit(entry, SEGMENT_KERNEL);
				if (!machine.has_nx)
					entry = clear_pmd_bit(entry, __pgprot(_SEGMENT_ENTRY_NOEXEC));
				set_pmd(pmd, entry);
				pages++;
				continue;
			}
			pte = boot_pte_alloc();
			pmd_populate(&init_mm, pmd, pte);
		} else if (pmd_large(*pmd)) {
			continue;
		}
		pgtable_pte_populate(pmd, addr, next, mode);
	}
	if (mode == POPULATE_DIRECT)
		update_page_count(PG_DIRECT_MAP_1M, pages);
}

static void pgtable_pud_populate(p4d_t *p4d, unsigned long addr, unsigned long end,
				 enum populate_mode mode)
{
	unsigned long next, pages = 0;
	pud_t *pud, entry;
	pmd_t *pmd;

	pud = pud_offset(p4d, addr);
	for (; addr < end; addr = next, pud++) {
		next = pud_addr_end(addr, end);
		if (pud_none(*pud)) {
			if (kasan_pud_populate_zero_shadow(pud, addr, next, mode))
				continue;
			if (can_large_pud(pud, addr, next)) {
				entry = __pud(_pa(addr, _REGION3_SIZE, mode));
				entry = set_pud_bit(entry, REGION3_KERNEL);
				if (!machine.has_nx)
					entry = clear_pud_bit(entry, __pgprot(_REGION_ENTRY_NOEXEC));
				set_pud(pud, entry);
				pages++;
				continue;
			}
			pmd = boot_crst_alloc(_SEGMENT_ENTRY_EMPTY);
			pud_populate(&init_mm, pud, pmd);
		} else if (pud_large(*pud)) {
			continue;
		}
		pgtable_pmd_populate(pud, addr, next, mode);
	}
	if (mode == POPULATE_DIRECT)
		update_page_count(PG_DIRECT_MAP_2G, pages);
}

static void pgtable_p4d_populate(pgd_t *pgd, unsigned long addr, unsigned long end,
				 enum populate_mode mode)
{
	unsigned long next;
	p4d_t *p4d;
	pud_t *pud;

	p4d = p4d_offset(pgd, addr);
	for (; addr < end; addr = next, p4d++) {
		next = p4d_addr_end(addr, end);
		if (p4d_none(*p4d)) {
			if (kasan_p4d_populate_zero_shadow(p4d, addr, next, mode))
				continue;
			pud = boot_crst_alloc(_REGION3_ENTRY_EMPTY);
			p4d_populate(&init_mm, p4d, pud);
		}
		pgtable_pud_populate(p4d, addr, next, mode);
	}
}

static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode)
{
	unsigned long next;
	pgd_t *pgd;
	p4d_t *p4d;

	pgd = pgd_offset(&init_mm, addr);
	for (; addr < end; addr = next, pgd++) {
		next = pgd_addr_end(addr, end);
		if (pgd_none(*pgd)) {
			if (kasan_pgd_populate_zero_shadow(pgd, addr, next, mode))
				continue;
			p4d = boot_crst_alloc(_REGION2_ENTRY_EMPTY);
			pgd_populate(&init_mm, pgd, p4d);
		}
#ifdef CONFIG_KASAN
		if (mode == POPULATE_KASAN_SHALLOW)
			continue;
#endif
		pgtable_p4d_populate(pgd, addr, next, mode);
	}
}

void setup_vmem(unsigned long asce_limit)
{
	unsigned long start, end;
	unsigned long asce_type;
	unsigned long asce_bits;
	int i;

	if (asce_limit == _REGION1_SIZE) {
		asce_type = _REGION2_ENTRY_EMPTY;
		asce_bits = _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH;
	} else {
		asce_type = _REGION3_ENTRY_EMPTY;
		asce_bits = _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH;
	}
	s390_invalid_asce = invalid_pg_dir | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH;

	crst_table_init((unsigned long *)swapper_pg_dir, asce_type);
	crst_table_init((unsigned long *)invalid_pg_dir, _REGION3_ENTRY_EMPTY);

	/*
	 * To allow prefixing the lowcore must be mapped with 4KB pages.
	 * To prevent creation of a large page at address 0 first map
	 * the lowcore and create the identity mapping only afterwards.
	 */
	pgtable_populate(0, sizeof(struct lowcore), POPULATE_DIRECT);
	for_each_physmem_usable_range(i, &start, &end)
		pgtable_populate(start, end, POPULATE_DIRECT);
	pgtable_populate(__abs_lowcore, __abs_lowcore + sizeof(struct lowcore),
			 POPULATE_ABS_LOWCORE);
	pgtable_populate(__memcpy_real_area, __memcpy_real_area + PAGE_SIZE,
			 POPULATE_NONE);
	memcpy_real_ptep = __virt_to_kpte(__memcpy_real_area);

	kasan_populate_shadow();

	S390_lowcore.kernel_asce = swapper_pg_dir | asce_bits;
	S390_lowcore.user_asce = s390_invalid_asce;

	__ctl_load(S390_lowcore.kernel_asce, 1, 1);
	__ctl_load(S390_lowcore.user_asce, 7, 7);
	__ctl_load(S390_lowcore.kernel_asce, 13, 13);

	init_mm.context.asce = S390_lowcore.kernel_asce;
}
Commit	Line	Data
bb1520d5 AG	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <linux/sched/task.h>
	3	#include <linux/pgtable.h>
557b1970	4	#include <linux/kasan.h>
bb1520d5 AG	5	#include <asm/pgalloc.h>
	6	#include <asm/facility.h>
	7	#include <asm/sections.h>
8c37cb7d	8	#include <asm/physmem_info.h>
8e9205d2	9	#include <asm/maccess.h>
2154e0b3	10	#include <asm/abs_lowcore.h>
bb1520d5 AG	11	#include "decompressor.h"
	12	#include "boot.h"
	13
f913a660 VG	14	unsigned long __bootdata_preserved(s390_invalid_asce);
f913a660 VG	15
81e84796 HC	16	#ifdef CONFIG_PROC_FS
	17	atomic_long_t __bootdata_preserved(direct_pages_count[PG_DIRECT_MAP_MAX]);
	18	#endif
	19
bb1520d5 AG	20	#define init_mm ((struct mm_struct )vmlinux.init_mm_off)
	21	#define swapper_pg_dir vmlinux.swapper_pg_dir_off
	22	#define invalid_pg_dir vmlinux.invalid_pg_dir_off
	23
557b1970 VG	24	enum populate_mode {
557b1970 VG	25	POPULATE_NONE,
07fdd662	26	POPULATE_DIRECT,
557b1970 VG	27	POPULATE_ABS_LOWCORE,
	28	#ifdef CONFIG_KASAN
	29	POPULATE_KASAN_MAP_SHADOW,
	30	POPULATE_KASAN_ZERO_SHADOW,
	31	POPULATE_KASAN_SHALLOW
	32	#endif
	33	};
	34
	35	static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode);
	36
	37	#ifdef CONFIG_KASAN
	38
	39	#define kasan_early_shadow_page vmlinux.kasan_early_shadow_page_off
	40	#define kasan_early_shadow_pte ((pte_t *)vmlinux.kasan_early_shadow_pte_off)
	41	#define kasan_early_shadow_pmd ((pmd_t *)vmlinux.kasan_early_shadow_pmd_off)
	42	#define kasan_early_shadow_pud ((pud_t *)vmlinux.kasan_early_shadow_pud_off)
	43	#define kasan_early_shadow_p4d ((p4d_t *)vmlinux.kasan_early_shadow_p4d_off)
	44	#define __sha(x) ((unsigned long)kasan_mem_to_shadow((void *)x))
	45
	46	static pte_t pte_z;
	47
3e826100 AG	48	static inline void kasan_populate(unsigned long start, unsigned long end, enum populate_mode mode)
	49	{
	50	start = PAGE_ALIGN_DOWN(__sha(start));
	51	end = PAGE_ALIGN(__sha(end));
	52	pgtable_populate(start, end, mode);
	53	}
	54
557b1970 VG	55	static void kasan_populate_shadow(void)
	56	{
	57	pmd_t pmd_z = __pmd(__pa(kasan_early_shadow_pte) \| _SEGMENT_ENTRY);
	58	pud_t pud_z = __pud(__pa(kasan_early_shadow_pmd) \| _REGION3_ENTRY);
	59	p4d_t p4d_z = __p4d(__pa(kasan_early_shadow_pud) \| _REGION2_ENTRY);
32789967	60	unsigned long memgap_start = 0;
557b1970 VG	61	unsigned long untracked_end;
	62	unsigned long start, end;
	63	int i;
	64
	65	pte_z = __pte(__pa(kasan_early_shadow_page) \| pgprot_val(PAGE_KERNEL_RO));
	66	if (!machine.has_nx)
	67	pte_z = clear_pte_bit(pte_z, __pgprot(_PAGE_NOEXEC));
	68	crst_table_init((unsigned long *)kasan_early_shadow_p4d, p4d_val(p4d_z));
	69	crst_table_init((unsigned long *)kasan_early_shadow_pud, pud_val(pud_z));
	70	crst_table_init((unsigned long *)kasan_early_shadow_pmd, pmd_val(pmd_z));
	71	memset64((u64 *)kasan_early_shadow_pte, pte_val(pte_z), PTRS_PER_PTE);
	72
	73	/*
	74	* Current memory layout:
	75	* +- 0 -------------+ +- shadow start -+
	76	* \|1:1 ident mapping\| /\|1/8 of ident map\|
	77	* \| \| / \| \|
	78	* +-end of ident map+ / +----------------+
	79	* \| ... gap ... \| / \| kasan \|
	80	* \| \| / \| zero page \|
	81	* +- vmalloc area -+ / \| mapping \|
	82	* \| vmalloc_size \| / \| (untracked) \|
	83	* +- modules vaddr -+ / +----------------+
	84	* \| 2Gb \|/ \| unmapped \| allocated per module
	85	* +- shadow start -+ +----------------+
	86	* \| 1/8 addr space \| \| zero pg mapping\| (untracked)
	87	* +- shadow end ----+---------+- shadow end ---+
	88	*
	89	* Current memory layout (KASAN_VMALLOC):
	90	* +- 0 -------------+ +- shadow start -+
	91	* \|1:1 ident mapping\| /\|1/8 of ident map\|
	92	* \| \| / \| \|
	93	* +-end of ident map+ / +----------------+
	94	* \| ... gap ... \| / \| kasan zero page\| (untracked)
	95	* \| \| / \| mapping \|
	96	* +- vmalloc area -+ / +----------------+
	97	* \| vmalloc_size \| / \|shallow populate\|
	98	* +- modules vaddr -+ / +----------------+
	99	* \| 2Gb \|/ \|shallow populate\|
	100	* +- shadow start -+ +----------------+
	101	* \| 1/8 addr space \| \| zero pg mapping\| (untracked)
	102	* +- shadow end ----+---------+- shadow end ---+
	103	*/
	104
32789967	105	for_each_physmem_usable_range(i, &start, &end) {
3e826100	106	kasan_populate(start, end, POPULATE_KASAN_MAP_SHADOW);
32789967 VG	107	if (memgap_start && physmem_info.info_source == MEM_DETECT_DIAG260)
	108	kasan_populate(memgap_start, start, POPULATE_KASAN_ZERO_SHADOW);
	109	memgap_start = end;
	110	}
557b1970 VG	111	if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
	112	untracked_end = VMALLOC_START;
	113	/* shallowly populate kasan shadow for vmalloc and modules */
3e826100	114	kasan_populate(VMALLOC_START, MODULES_END, POPULATE_KASAN_SHALLOW);
557b1970 VG	115	} else {
	116	untracked_end = MODULES_VADDR;
	117	}
	118	/* populate kasan shadow for untracked memory */
3e826100 AG	119	kasan_populate(ident_map_size, untracked_end, POPULATE_KASAN_ZERO_SHADOW);
3e826100 AG	120	kasan_populate(MODULES_END, _REGION1_SIZE, POPULATE_KASAN_ZERO_SHADOW);
557b1970 VG	121	}
	122
	123	static bool kasan_pgd_populate_zero_shadow(pgd_t *pgd, unsigned long addr,
	124	unsigned long end, enum populate_mode mode)
	125	{
	126	if (mode == POPULATE_KASAN_ZERO_SHADOW &&
	127	IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) {
	128	pgd_populate(&init_mm, pgd, kasan_early_shadow_p4d);
	129	return true;
	130	}
	131	return false;
	132	}
	133
	134	static bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr,
	135	unsigned long end, enum populate_mode mode)
	136	{
	137	if (mode == POPULATE_KASAN_ZERO_SHADOW &&
	138	IS_ALIGNED(addr, P4D_SIZE) && end - addr >= P4D_SIZE) {
	139	p4d_populate(&init_mm, p4d, kasan_early_shadow_pud);
	140	return true;
	141	}
	142	return false;
	143	}
	144
	145	static bool kasan_pud_populate_zero_shadow(pud_t *pud, unsigned long addr,
	146	unsigned long end, enum populate_mode mode)
	147	{
	148	if (mode == POPULATE_KASAN_ZERO_SHADOW &&
	149	IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) {
	150	pud_populate(&init_mm, pud, kasan_early_shadow_pmd);
	151	return true;
	152	}
	153	return false;
	154	}
	155
	156	static bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr,
	157	unsigned long end, enum populate_mode mode)
	158	{
	159	if (mode == POPULATE_KASAN_ZERO_SHADOW &&
	160	IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) {
	161	pmd_populate(&init_mm, pmd, kasan_early_shadow_pte);
	162	return true;
	163	}
	164	return false;
	165	}
	166
	167	static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode)
	168	{
	169	pte_t entry;
	170
	171	if (mode == POPULATE_KASAN_ZERO_SHADOW) {
	172	set_pte(pte, pte_z);
	173	return true;
	174	}
	175	return false;
	176	}
	177	#else
	178
	179	static inline void kasan_populate_shadow(void) {}
	180
	181	static inline bool kasan_pgd_populate_zero_shadow(pgd_t *pgd, unsigned long addr,
	182	unsigned long end, enum populate_mode mode)
	183	{
	184	return false;
185	}
186
187	static inline bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr,
188	unsigned long end, enum populate_mode mode)
189	{
190	return false;
191	}
192
193	static inline bool kasan_pud_populate_zero_shadow(pud_t *pud, unsigned long addr,
194	unsigned long end, enum populate_mode mode)
195	{
196	return false;
197	}
198
199	static inline bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr,
200	unsigned long end, enum populate_mode mode)
201	{
202	return false;
203	}
204
205	static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode)
206	{
207	return false;
208	}
209
210	#endif
211
8e9205d2 AG	212	/*
	213	* Mimic virt_to_kpte() in lack of init_mm symbol. Skip pmd NULL check though.
	214	*/
	215	static inline pte_t *__virt_to_kpte(unsigned long va)
	216	{
	217	return pte_offset_kernel(pmd_offset(pud_offset(p4d_offset(pgd_offset_k(va), va), va), va), va);
	218	}
	219
bb1520d5 AG	220	static void *boot_crst_alloc(unsigned long val)
bb1520d5 AG	221	{
f913a660	222	unsigned long size = PAGE_SIZE << CRST_ALLOC_ORDER;
bb1520d5 AG	223	unsigned long *table;
bb1520d5 AG	224
f913a660 VG	225	table = (unsigned long *)physmem_alloc_top_down(RR_VMEM, size, size);
f913a660 VG	226	crst_table_init(table, val);
bb1520d5 AG	227	return table;
	228	}
	229
	230	static pte_t *boot_pte_alloc(void)
	231	{
557b1970	232	static void *pte_leftover;
bb1520d5 AG	233	pte_t *pte;
bb1520d5 AG	234
557b1970 VG	235	/*
	236	* handling pte_leftovers this way helps to avoid memory fragmentation
	237	* during POPULATE_KASAN_MAP_SHADOW when EDAT is off
	238	*/
	239	if (!pte_leftover) {
	240	pte_leftover = (void *)physmem_alloc_top_down(RR_VMEM, PAGE_SIZE, PAGE_SIZE);
	241	pte = pte_leftover + _PAGE_TABLE_SIZE;
	242	} else {
	243	pte = pte_leftover;
	244	pte_leftover = NULL;
	245	}
	246
bb1520d5 AG	247	memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE);
	248	return pte;
	249	}
	250
557b1970	251	static unsigned long _pa(unsigned long addr, unsigned long size, enum populate_mode mode)
e0e0a87b AG	252	{
e0e0a87b AG	253	switch (mode) {
8e9205d2 AG	254	case POPULATE_NONE:
8e9205d2 AG	255	return -1;
07fdd662	256	case POPULATE_DIRECT:
e0e0a87b	257	return addr;
2154e0b3 AG	258	case POPULATE_ABS_LOWCORE:
2154e0b3 AG	259	return __abs_lowcore_pa(addr);
557b1970 VG	260	#ifdef CONFIG_KASAN
	261	case POPULATE_KASAN_MAP_SHADOW:
	262	addr = physmem_alloc_top_down(RR_VMEM, size, size);
	263	memset((void *)addr, 0, size);
	264	return addr;
	265	#endif
e0e0a87b AG	266	default:
	267	return -1;
	268	}
	269	}
	270
bb1520d5 AG	271	static bool can_large_pud(pud_t *pu_dir, unsigned long addr, unsigned long end)
	272	{
	273	return machine.has_edat2 &&
	274	IS_ALIGNED(addr, PUD_SIZE) && (end - addr) >= PUD_SIZE;
	275	}
	276
	277	static bool can_large_pmd(pmd_t *pm_dir, unsigned long addr, unsigned long end)
	278	{
	279	return machine.has_edat1 &&
	280	IS_ALIGNED(addr, PMD_SIZE) && (end - addr) >= PMD_SIZE;
	281	}
	282
e0e0a87b AG	283	static void pgtable_pte_populate(pmd_t *pmd, unsigned long addr, unsigned long end,
e0e0a87b AG	284	enum populate_mode mode)
bb1520d5	285	{
81e84796	286	unsigned long pages = 0;
bb1520d5 AG	287	pte_t *pte, entry;
	288
	289	pte = pte_offset_kernel(pmd, addr);
	290	for (; addr < end; addr += PAGE_SIZE, pte++) {
	291	if (pte_none(*pte)) {
557b1970 VG	292	if (kasan_pte_populate_zero_shadow(pte, mode))
	293	continue;
	294	entry = __pte(_pa(addr, PAGE_SIZE, mode));
c0f1d478 HC	295	entry = set_pte_bit(entry, PAGE_KERNEL);
	296	if (!machine.has_nx)
	297	entry = clear_pte_bit(entry, __pgprot(_PAGE_NOEXEC));
bb1520d5	298	set_pte(pte, entry);
81e84796	299	pages++;
bb1520d5 AG	300	}
bb1520d5 AG	301	}
81e84796 HC	302	if (mode == POPULATE_DIRECT)
81e84796 HC	303	update_page_count(PG_DIRECT_MAP_4K, pages);
bb1520d5 AG	304	}
bb1520d5 AG	305
e0e0a87b AG	306	static void pgtable_pmd_populate(pud_t *pud, unsigned long addr, unsigned long end,
e0e0a87b AG	307	enum populate_mode mode)
bb1520d5	308	{
81e84796	309	unsigned long next, pages = 0;
bb1520d5 AG	310	pmd_t *pmd, entry;
	311	pte_t *pte;
	312
	313	pmd = pmd_offset(pud, addr);
	314	for (; addr < end; addr = next, pmd++) {
	315	next = pmd_addr_end(addr, end);
	316	if (pmd_none(*pmd)) {
557b1970 VG	317	if (kasan_pmd_populate_zero_shadow(pmd, addr, next, mode))
557b1970 VG	318	continue;
bb1520d5	319	if (can_large_pmd(pmd, addr, next)) {
557b1970	320	entry = __pmd(_pa(addr, _SEGMENT_SIZE, mode));
c0f1d478 HC	321	entry = set_pmd_bit(entry, SEGMENT_KERNEL);
	322	if (!machine.has_nx)
	323	entry = clear_pmd_bit(entry, __pgprot(_SEGMENT_ENTRY_NOEXEC));
bb1520d5	324	set_pmd(pmd, entry);
81e84796	325	pages++;
bb1520d5 AG	326	continue;
	327	}
	328	pte = boot_pte_alloc();
	329	pmd_populate(&init_mm, pmd, pte);
	330	} else if (pmd_large(*pmd)) {
	331	continue;
	332	}
e0e0a87b	333	pgtable_pte_populate(pmd, addr, next, mode);
bb1520d5	334	}
81e84796 HC	335	if (mode == POPULATE_DIRECT)
81e84796 HC	336	update_page_count(PG_DIRECT_MAP_1M, pages);
bb1520d5 AG	337	}
bb1520d5 AG	338
e0e0a87b AG	339	static void pgtable_pud_populate(p4d_t *p4d, unsigned long addr, unsigned long end,
e0e0a87b AG	340	enum populate_mode mode)
bb1520d5	341	{
81e84796	342	unsigned long next, pages = 0;
bb1520d5 AG	343	pud_t *pud, entry;
	344	pmd_t *pmd;
	345
	346	pud = pud_offset(p4d, addr);
	347	for (; addr < end; addr = next, pud++) {
	348	next = pud_addr_end(addr, end);
	349	if (pud_none(*pud)) {
557b1970 VG	350	if (kasan_pud_populate_zero_shadow(pud, addr, next, mode))
557b1970 VG	351	continue;
bb1520d5	352	if (can_large_pud(pud, addr, next)) {
557b1970	353	entry = __pud(_pa(addr, _REGION3_SIZE, mode));
c0f1d478 HC	354	entry = set_pud_bit(entry, REGION3_KERNEL);
	355	if (!machine.has_nx)
	356	entry = clear_pud_bit(entry, __pgprot(_REGION_ENTRY_NOEXEC));
bb1520d5	357	set_pud(pud, entry);
81e84796	358	pages++;
bb1520d5 AG	359	continue;
	360	}
	361	pmd = boot_crst_alloc(_SEGMENT_ENTRY_EMPTY);
	362	pud_populate(&init_mm, pud, pmd);
	363	} else if (pud_large(*pud)) {
	364	continue;
	365	}
e0e0a87b	366	pgtable_pmd_populate(pud, addr, next, mode);
bb1520d5	367	}
81e84796 HC	368	if (mode == POPULATE_DIRECT)
81e84796 HC	369	update_page_count(PG_DIRECT_MAP_2G, pages);
bb1520d5 AG	370	}
bb1520d5 AG	371
e0e0a87b AG	372	static void pgtable_p4d_populate(pgd_t *pgd, unsigned long addr, unsigned long end,
e0e0a87b AG	373	enum populate_mode mode)
bb1520d5 AG	374	{
	375	unsigned long next;
	376	p4d_t *p4d;
	377	pud_t *pud;
	378
	379	p4d = p4d_offset(pgd, addr);
	380	for (; addr < end; addr = next, p4d++) {
	381	next = p4d_addr_end(addr, end);
	382	if (p4d_none(*p4d)) {
557b1970 VG	383	if (kasan_p4d_populate_zero_shadow(p4d, addr, next, mode))
557b1970 VG	384	continue;
bb1520d5 AG	385	pud = boot_crst_alloc(_REGION3_ENTRY_EMPTY);
	386	p4d_populate(&init_mm, p4d, pud);
	387	}
e0e0a87b	388	pgtable_pud_populate(p4d, addr, next, mode);
bb1520d5 AG	389	}
	390	}
	391
e0e0a87b	392	static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode)
bb1520d5 AG	393	{
	394	unsigned long next;
	395	pgd_t *pgd;
	396	p4d_t *p4d;
	397
	398	pgd = pgd_offset(&init_mm, addr);
	399	for (; addr < end; addr = next, pgd++) {
	400	next = pgd_addr_end(addr, end);
	401	if (pgd_none(*pgd)) {
557b1970 VG	402	if (kasan_pgd_populate_zero_shadow(pgd, addr, next, mode))
557b1970 VG	403	continue;
bb1520d5 AG	404	p4d = boot_crst_alloc(_REGION2_ENTRY_EMPTY);
	405	pgd_populate(&init_mm, pgd, p4d);
	406	}
557b1970 VG	407	#ifdef CONFIG_KASAN
	408	if (mode == POPULATE_KASAN_SHALLOW)
	409	continue;
	410	#endif
e0e0a87b	411	pgtable_p4d_populate(pgd, addr, next, mode);
bb1520d5 AG	412	}
	413	}
	414
bf64f051	415	void setup_vmem(unsigned long asce_limit)
bb1520d5	416	{
e966ccf8	417	unsigned long start, end;
bb1520d5 AG	418	unsigned long asce_type;
bb1520d5 AG	419	unsigned long asce_bits;
e966ccf8	420	int i;
bb1520d5 AG	421
	422	if (asce_limit == _REGION1_SIZE) {
	423	asce_type = _REGION2_ENTRY_EMPTY;
	424	asce_bits = _ASCE_TYPE_REGION2 \| _ASCE_TABLE_LENGTH;
	425	} else {
	426	asce_type = _REGION3_ENTRY_EMPTY;
	427	asce_bits = _ASCE_TYPE_REGION3 \| _ASCE_TABLE_LENGTH;
	428	}
	429	s390_invalid_asce = invalid_pg_dir \| _ASCE_TYPE_REGION3 \| _ASCE_TABLE_LENGTH;
	430
	431	crst_table_init((unsigned long *)swapper_pg_dir, asce_type);
	432	crst_table_init((unsigned long *)invalid_pg_dir, _REGION3_ENTRY_EMPTY);
	433
	434	/*
	435	* To allow prefixing the lowcore must be mapped with 4KB pages.
	436	* To prevent creation of a large page at address 0 first map
	437	* the lowcore and create the identity mapping only afterwards.
bb1520d5	438	*/
07fdd662	439	pgtable_populate(0, sizeof(struct lowcore), POPULATE_DIRECT);
8c37cb7d	440	for_each_physmem_usable_range(i, &start, &end)
07fdd662	441	pgtable_populate(start, end, POPULATE_DIRECT);
2154e0b3 AG	442	pgtable_populate(__abs_lowcore, __abs_lowcore + sizeof(struct lowcore),
2154e0b3 AG	443	POPULATE_ABS_LOWCORE);
8e9205d2 AG	444	pgtable_populate(__memcpy_real_area, __memcpy_real_area + PAGE_SIZE,
	445	POPULATE_NONE);
	446	memcpy_real_ptep = __virt_to_kpte(__memcpy_real_area);
bb1520d5	447
557b1970 VG	448	kasan_populate_shadow();
557b1970 VG	449
bb1520d5 AG	450	S390_lowcore.kernel_asce = swapper_pg_dir \| asce_bits;
	451	S390_lowcore.user_asce = s390_invalid_asce;
	452
	453	__ctl_load(S390_lowcore.kernel_asce, 1, 1);
	454	__ctl_load(S390_lowcore.user_asce, 7, 7);
	455	__ctl_load(S390_lowcore.kernel_asce, 13, 13);
	456
	457	init_mm.context.asce = S390_lowcore.kernel_asce;
	458	}