[linux-block.git] / arch / s390 / mm / kasan_init.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/kasan.h>
#include <linux/sched/task.h>
#include <linux/memblock.h>
#include <linux/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/kasan.h>
#include <asm/mem_detect.h>
#include <asm/processor.h>
#include <asm/sclp.h>
#include <asm/facility.h>
#include <asm/sections.h>
#include <asm/setup.h>
#include <asm/uv.h>

unsigned long kasan_vmax;
static unsigned long segment_pos __initdata;
static unsigned long segment_low __initdata;
static unsigned long pgalloc_pos __initdata;
static unsigned long pgalloc_low __initdata;
static unsigned long pgalloc_freeable __initdata;
static bool has_edat __initdata;
static bool has_nx __initdata;

#define __sha(x) ((unsigned long)kasan_mem_to_shadow((void *)x))

static pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE);

static void __init kasan_early_panic(const char *reason)
{
	sclp_early_printk("The Linux kernel failed to boot with the KernelAddressSanitizer:\n");
	sclp_early_printk(reason);
	disabled_wait();
}

static void * __init kasan_early_alloc_segment(void)
{
	segment_pos -= _SEGMENT_SIZE;

	if (segment_pos < segment_low)
		kasan_early_panic("out of memory during initialisation\n");

	return (void *)segment_pos;
}

static void * __init kasan_early_alloc_pages(unsigned int order)
{
	pgalloc_pos -= (PAGE_SIZE << order);

	if (pgalloc_pos < pgalloc_low)
		kasan_early_panic("out of memory during initialisation\n");

	return (void *)pgalloc_pos;
}

static void * __init kasan_early_crst_alloc(unsigned long val)
{
	unsigned long *table;

	table = kasan_early_alloc_pages(CRST_ALLOC_ORDER);
	if (table)
		crst_table_init(table, val);
	return table;
}

static pte_t * __init kasan_early_pte_alloc(void)
{
	static void *pte_leftover;
	pte_t *pte;

	BUILD_BUG_ON(_PAGE_TABLE_SIZE * 2 != PAGE_SIZE);

	if (!pte_leftover) {
		pte_leftover = kasan_early_alloc_pages(0);
		pte = pte_leftover + _PAGE_TABLE_SIZE;
	} else {
		pte = pte_leftover;
		pte_leftover = NULL;
	}
	memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE);
	return pte;
}

enum populate_mode {
	POPULATE_ONE2ONE,
	POPULATE_MAP,
	POPULATE_ZERO_SHADOW,
	POPULATE_SHALLOW
};
static void __init kasan_early_vmemmap_populate(unsigned long address,
						unsigned long end,
						enum populate_mode mode)
{
	unsigned long pgt_prot_zero, pgt_prot, sgt_prot;
	pgd_t *pg_dir;
	p4d_t *p4_dir;
	pud_t *pu_dir;
	pmd_t *pm_dir;
	pte_t *pt_dir;

	pgt_prot_zero = pgprot_val(PAGE_KERNEL_RO);
	if (!has_nx)
		pgt_prot_zero &= ~_PAGE_NOEXEC;
	pgt_prot = pgprot_val(PAGE_KERNEL);
	sgt_prot = pgprot_val(SEGMENT_KERNEL);
	if (!has_nx || mode == POPULATE_ONE2ONE) {
		pgt_prot &= ~_PAGE_NOEXEC;
		sgt_prot &= ~_SEGMENT_ENTRY_NOEXEC;
	}

	while (address < end) {
		pg_dir = pgd_offset_k(address);
		if (pgd_none(*pg_dir)) {
			if (mode == POPULATE_ZERO_SHADOW &&
			    IS_ALIGNED(address, PGDIR_SIZE) &&
			    end - address >= PGDIR_SIZE) {
				pgd_populate(&init_mm, pg_dir,
						kasan_early_shadow_p4d);
				address = (address + PGDIR_SIZE) & PGDIR_MASK;
				continue;
			}
			p4_dir = kasan_early_crst_alloc(_REGION2_ENTRY_EMPTY);
			pgd_populate(&init_mm, pg_dir, p4_dir);
		}

		if (IS_ENABLED(CONFIG_KASAN_S390_4_LEVEL_PAGING) &&
		    mode == POPULATE_SHALLOW) {
			address = (address + P4D_SIZE) & P4D_MASK;
			continue;
		}

		p4_dir = p4d_offset(pg_dir, address);
		if (p4d_none(*p4_dir)) {
			if (mode == POPULATE_ZERO_SHADOW &&
			    IS_ALIGNED(address, P4D_SIZE) &&
			    end - address >= P4D_SIZE) {
				p4d_populate(&init_mm, p4_dir,
						kasan_early_shadow_pud);
				address = (address + P4D_SIZE) & P4D_MASK;
				continue;
			}
			pu_dir = kasan_early_crst_alloc(_REGION3_ENTRY_EMPTY);
			p4d_populate(&init_mm, p4_dir, pu_dir);
		}

		if (!IS_ENABLED(CONFIG_KASAN_S390_4_LEVEL_PAGING) &&
		    mode == POPULATE_SHALLOW) {
			address = (address + PUD_SIZE) & PUD_MASK;
			continue;
		}

		pu_dir = pud_offset(p4_dir, address);
		if (pud_none(*pu_dir)) {
			if (mode == POPULATE_ZERO_SHADOW &&
			    IS_ALIGNED(address, PUD_SIZE) &&
			    end - address >= PUD_SIZE) {
				pud_populate(&init_mm, pu_dir,
						kasan_early_shadow_pmd);
				address = (address + PUD_SIZE) & PUD_MASK;
				continue;
			}
			pm_dir = kasan_early_crst_alloc(_SEGMENT_ENTRY_EMPTY);
			pud_populate(&init_mm, pu_dir, pm_dir);
		}

		pm_dir = pmd_offset(pu_dir, address);
		if (pmd_none(*pm_dir)) {
			if (mode == POPULATE_ZERO_SHADOW &&
			    IS_ALIGNED(address, PMD_SIZE) &&
			    end - address >= PMD_SIZE) {
				pmd_populate(&init_mm, pm_dir,
						kasan_early_shadow_pte);
				address = (address + PMD_SIZE) & PMD_MASK;
				continue;
			}
			/* the first megabyte of 1:1 is mapped with 4k pages */
			if (has_edat && address && end - address >= PMD_SIZE &&
			    mode != POPULATE_ZERO_SHADOW) {
				void *page;

				if (mode == POPULATE_ONE2ONE) {
					page = (void *)address;
				} else {
					page = kasan_early_alloc_segment();
					memset(page, 0, _SEGMENT_SIZE);
				}
				pmd_val(*pm_dir) = __pa(page) | sgt_prot;
				address = (address + PMD_SIZE) & PMD_MASK;
				continue;
			}

			pt_dir = kasan_early_pte_alloc();
			pmd_populate(&init_mm, pm_dir, pt_dir);
		} else if (pmd_large(*pm_dir)) {
			address = (address + PMD_SIZE) & PMD_MASK;
			continue;
		}

		pt_dir = pte_offset_kernel(pm_dir, address);
		if (pte_none(*pt_dir)) {
			void *page;

			switch (mode) {
			case POPULATE_ONE2ONE:
				page = (void *)address;
				pte_val(*pt_dir) = __pa(page) | pgt_prot;
				break;
			case POPULATE_MAP:
				page = kasan_early_alloc_pages(0);
				memset(page, 0, PAGE_SIZE);
				pte_val(*pt_dir) = __pa(page) | pgt_prot;
				break;
			case POPULATE_ZERO_SHADOW:
				page = kasan_early_shadow_page;
				pte_val(*pt_dir) = __pa(page) | pgt_prot_zero;
				break;
			case POPULATE_SHALLOW:
				/* should never happen */
				break;
			}
		}
		address += PAGE_SIZE;
	}
}

static void __init kasan_set_pgd(pgd_t *pgd, unsigned long asce_type)
{
	unsigned long asce_bits;

	asce_bits = asce_type | _ASCE_TABLE_LENGTH;
	S390_lowcore.kernel_asce = (__pa(pgd) & PAGE_MASK) | asce_bits;
	S390_lowcore.user_asce = S390_lowcore.kernel_asce;

	__ctl_load(S390_lowcore.kernel_asce, 1, 1);
	__ctl_load(S390_lowcore.kernel_asce, 7, 7);
	__ctl_load(S390_lowcore.kernel_asce, 13, 13);
}

static void __init kasan_enable_dat(void)
{
	psw_t psw;

	psw.mask = __extract_psw();
	psw_bits(psw).dat = 1;
	psw_bits(psw).as = PSW_BITS_AS_HOME;
	__load_psw_mask(psw.mask);
}

static void __init kasan_early_detect_facilities(void)
{
	if (test_facility(8)) {
		has_edat = true;
		__ctl_set_bit(0, 23);
	}
	if (!noexec_disabled && test_facility(130)) {
		has_nx = true;
		__ctl_set_bit(0, 20);
	}
}

static bool __init has_uv_sec_stor_limit(void)
{
	/*
	 * keep these conditions in line with setup_uv()
	 */
	if (!is_prot_virt_host())
		return false;

	if (is_prot_virt_guest())
		return false;

	if (!test_facility(158))
		return false;

	return !!uv_info.max_sec_stor_addr;
}

void __init kasan_early_init(void)
{
	unsigned long untracked_mem_end;
	unsigned long shadow_alloc_size;
	unsigned long vmax_unlimited;
	unsigned long initrd_end;
	unsigned long asce_type;
	unsigned long memsize;
	unsigned long pgt_prot = pgprot_val(PAGE_KERNEL_RO);
	pte_t pte_z;
	pmd_t pmd_z = __pmd(__pa(kasan_early_shadow_pte) | _SEGMENT_ENTRY);
	pud_t pud_z = __pud(__pa(kasan_early_shadow_pmd) | _REGION3_ENTRY);
	p4d_t p4d_z = __p4d(__pa(kasan_early_shadow_pud) | _REGION2_ENTRY);

	kasan_early_detect_facilities();
	if (!has_nx)
		pgt_prot &= ~_PAGE_NOEXEC;
	pte_z = __pte(__pa(kasan_early_shadow_page) | pgt_prot);

	memsize = get_mem_detect_end();
	if (!memsize)
		kasan_early_panic("cannot detect physical memory size\n");
	/* respect mem= cmdline parameter */
	if (memory_end_set && memsize > memory_end)
		memsize = memory_end;
	if (IS_ENABLED(CONFIG_CRASH_DUMP) && OLDMEM_BASE)
		memsize = min(memsize, OLDMEM_SIZE);
	memsize = min(memsize, KASAN_SHADOW_START);

	if (IS_ENABLED(CONFIG_KASAN_S390_4_LEVEL_PAGING)) {
		/* 4 level paging */
		BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, P4D_SIZE));
		BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, P4D_SIZE));
		crst_table_init((unsigned long *)early_pg_dir,
				_REGION2_ENTRY_EMPTY);
		untracked_mem_end = kasan_vmax = vmax_unlimited = _REGION1_SIZE;
		if (has_uv_sec_stor_limit())
			kasan_vmax = min(vmax_unlimited, uv_info.max_sec_stor_addr);
		asce_type = _ASCE_TYPE_REGION2;
	} else {
		/* 3 level paging */
		BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, PUD_SIZE));
		BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, PUD_SIZE));
		crst_table_init((unsigned long *)early_pg_dir,
				_REGION3_ENTRY_EMPTY);
		untracked_mem_end = kasan_vmax = vmax_unlimited = _REGION2_SIZE;
		asce_type = _ASCE_TYPE_REGION3;
	}

	/* init kasan zero shadow */
	crst_table_init((unsigned long *)kasan_early_shadow_p4d,
				p4d_val(p4d_z));
	crst_table_init((unsigned long *)kasan_early_shadow_pud,
				pud_val(pud_z));
	crst_table_init((unsigned long *)kasan_early_shadow_pmd,
				pmd_val(pmd_z));
	memset64((u64 *)kasan_early_shadow_pte, pte_val(pte_z), PTRS_PER_PTE);

	shadow_alloc_size = memsize >> KASAN_SHADOW_SCALE_SHIFT;
	pgalloc_low = round_up((unsigned long)_end, _SEGMENT_SIZE);
	if (IS_ENABLED(CONFIG_BLK_DEV_INITRD)) {
		initrd_end =
		    round_up(INITRD_START + INITRD_SIZE, _SEGMENT_SIZE);
		pgalloc_low = max(pgalloc_low, initrd_end);
	}

	if (pgalloc_low + shadow_alloc_size > memsize)
		kasan_early_panic("out of memory during initialisation\n");

	if (has_edat) {
		segment_pos = round_down(memsize, _SEGMENT_SIZE);
		segment_low = segment_pos - shadow_alloc_size;
		pgalloc_pos = segment_low;
	} else {
		pgalloc_pos = memsize;
	}
	init_mm.pgd = early_pg_dir;
	/*
	 * Current memory layout:
	 * +- 0 -------------+	   +- shadow start -+
	 * | 1:1 ram mapping |	  /| 1/8 ram	    |
	 * |		     |	 / |		    |
	 * +- end of ram ----+	/  +----------------+
	 * | ... gap ...     | /   |		    |
	 * |		     |/    |	kasan	    |
	 * +- shadow start --+	   |	zero	    |
	 * | 1/8 addr space  |	   |	page	    |
	 * +- shadow end    -+	   |	mapping	    |
	 * | ... gap ...     |\    |  (untracked)   |
	 * +- vmalloc area  -+ \   |		    |
	 * | vmalloc_size    |	\  |		    |
	 * +- modules vaddr -+	 \ +----------------+
	 * | 2Gb	     |	  \|	  unmapped  | allocated per module
	 * +-----------------+	   +- shadow end ---+
	 *
	 * Current memory layout (KASAN_VMALLOC):
	 * +- 0 -------------+	   +- shadow start -+
	 * | 1:1 ram mapping |	  /| 1/8 ram	    |
	 * |		     |	 / |		    |
	 * +- end of ram ----+	/  +----------------+
	 * | ... gap ...     | /   |	kasan	    |
	 * |		     |/    |	zero	    |
	 * +- shadow start --+	   |	page	    |
	 * | 1/8 addr space  |	   |	mapping     |
	 * +- shadow end    -+	   |  (untracked)   |
	 * | ... gap ...     |\    |		    |
	 * +- vmalloc area  -+ \   +- vmalloc area -+
	 * | vmalloc_size    |	\  |shallow populate|
	 * +- modules vaddr -+	 \ +- modules area -+
	 * | 2Gb	     |	  \|shallow populate|
	 * +-----------------+	   +- shadow end ---+
	 */
	/* populate kasan shadow (for identity mapping and zero page mapping) */
	kasan_early_vmemmap_populate(__sha(0), __sha(memsize), POPULATE_MAP);
	if (IS_ENABLED(CONFIG_MODULES))
		untracked_mem_end = kasan_vmax - MODULES_LEN;
	if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
		untracked_mem_end = kasan_vmax - vmalloc_size - MODULES_LEN;
		/* shallowly populate kasan shadow for vmalloc and modules */
		kasan_early_vmemmap_populate(__sha(untracked_mem_end),
					     __sha(kasan_vmax), POPULATE_SHALLOW);
	}
	/* populate kasan shadow for untracked memory */
	kasan_early_vmemmap_populate(__sha(max_physmem_end),
				     __sha(untracked_mem_end),
				     POPULATE_ZERO_SHADOW);
	kasan_early_vmemmap_populate(__sha(kasan_vmax),
				     __sha(vmax_unlimited),
				     POPULATE_ZERO_SHADOW);
	/* memory allocated for identity mapping structs will be freed later */
	pgalloc_freeable = pgalloc_pos;
	/* populate identity mapping */
	kasan_early_vmemmap_populate(0, memsize, POPULATE_ONE2ONE);
	kasan_set_pgd(early_pg_dir, asce_type);
	kasan_enable_dat();
	/* enable kasan */
	init_task.kasan_depth = 0;
	memblock_reserve(pgalloc_pos, memsize - pgalloc_pos);
	sclp_early_printk("KernelAddressSanitizer initialized\n");
}

void __init kasan_copy_shadow(pgd_t *pg_dir)
{
	/*
	 * At this point we are still running on early pages setup early_pg_dir,
	 * while swapper_pg_dir has just been initialized with identity mapping.
	 * Carry over shadow memory region from early_pg_dir to swapper_pg_dir.
	 */

	pgd_t *pg_dir_src;
	pgd_t *pg_dir_dst;
	p4d_t *p4_dir_src;
	p4d_t *p4_dir_dst;
	pud_t *pu_dir_src;
	pud_t *pu_dir_dst;

	pg_dir_src = pgd_offset_raw(early_pg_dir, KASAN_SHADOW_START);
	pg_dir_dst = pgd_offset_raw(pg_dir, KASAN_SHADOW_START);
	p4_dir_src = p4d_offset(pg_dir_src, KASAN_SHADOW_START);
	p4_dir_dst = p4d_offset(pg_dir_dst, KASAN_SHADOW_START);
	if (!p4d_folded(*p4_dir_src)) {
		/* 4 level paging */
		memcpy(p4_dir_dst, p4_dir_src,
		       (KASAN_SHADOW_SIZE >> P4D_SHIFT) * sizeof(p4d_t));
		return;
	}
	/* 3 level paging */
	pu_dir_src = pud_offset(p4_dir_src, KASAN_SHADOW_START);
	pu_dir_dst = pud_offset(p4_dir_dst, KASAN_SHADOW_START);
	memcpy(pu_dir_dst, pu_dir_src,
	       (KASAN_SHADOW_SIZE >> PUD_SHIFT) * sizeof(pud_t));
}

void __init kasan_free_early_identity(void)
{
	memblock_free(pgalloc_pos, pgalloc_freeable - pgalloc_pos);
}
Commit	Line	Data
42db5ed8 VG	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <linux/kasan.h>
	3	#include <linux/sched/task.h>
	4	#include <linux/memblock.h>
ca5999fd	5	#include <linux/pgtable.h>
65fddcfc	6	#include <asm/pgalloc.h>
42db5ed8	7	#include <asm/kasan.h>
29635239	8	#include <asm/mem_detect.h>
42db5ed8 VG	9	#include <asm/processor.h>
42db5ed8 VG	10	#include <asm/sclp.h>
d58106c3	11	#include <asm/facility.h>
42db5ed8 VG	12	#include <asm/sections.h>
42db5ed8 VG	13	#include <asm/setup.h>
c360c9a2	14	#include <asm/uv.h>
42db5ed8	15
c360c9a2	16	unsigned long kasan_vmax;
d58106c3 VG	17	static unsigned long segment_pos __initdata;
d58106c3 VG	18	static unsigned long segment_low __initdata;
42db5ed8 VG	19	static unsigned long pgalloc_pos __initdata;
42db5ed8 VG	20	static unsigned long pgalloc_low __initdata;
135ff163	21	static unsigned long pgalloc_freeable __initdata;
d58106c3 VG	22	static bool has_edat __initdata;
d58106c3 VG	23	static bool has_nx __initdata;
42db5ed8 VG	24
	25	#define __sha(x) ((unsigned long)kasan_mem_to_shadow((void *)x))
	26
	27	static pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE);
	28
	29	static void __init kasan_early_panic(const char *reason)
	30	{
	31	sclp_early_printk("The Linux kernel failed to boot with the KernelAddressSanitizer:\n");
	32	sclp_early_printk(reason);
184c460f	33	disabled_wait();
42db5ed8 VG	34	}
42db5ed8 VG	35
d58106c3 VG	36	static void * __init kasan_early_alloc_segment(void)
	37	{
	38	segment_pos -= _SEGMENT_SIZE;
	39
	40	if (segment_pos < segment_low)
	41	kasan_early_panic("out of memory during initialisation\n");
	42
	43	return (void *)segment_pos;
	44	}
	45
42db5ed8 VG	46	static void * __init kasan_early_alloc_pages(unsigned int order)
	47	{
	48	pgalloc_pos -= (PAGE_SIZE << order);
	49
	50	if (pgalloc_pos < pgalloc_low)
	51	kasan_early_panic("out of memory during initialisation\n");
	52
	53	return (void *)pgalloc_pos;
	54	}
	55
	56	static void * __init kasan_early_crst_alloc(unsigned long val)
	57	{
	58	unsigned long *table;
	59
	60	table = kasan_early_alloc_pages(CRST_ALLOC_ORDER);
	61	if (table)
	62	crst_table_init(table, val);
	63	return table;
	64	}
	65
	66	static pte_t * __init kasan_early_pte_alloc(void)
	67	{
	68	static void *pte_leftover;
	69	pte_t *pte;
	70
	71	BUILD_BUG_ON(_PAGE_TABLE_SIZE * 2 != PAGE_SIZE);
	72
	73	if (!pte_leftover) {
	74	pte_leftover = kasan_early_alloc_pages(0);
	75	pte = pte_leftover + _PAGE_TABLE_SIZE;
	76	} else {
	77	pte = pte_leftover;
	78	pte_leftover = NULL;
	79	}
	80	memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE);
	81	return pte;
	82	}
	83
	84	enum populate_mode {
	85	POPULATE_ONE2ONE,
	86	POPULATE_MAP,
3e39ce26 VG	87	POPULATE_ZERO_SHADOW,
3e39ce26 VG	88	POPULATE_SHALLOW
42db5ed8 VG	89	};
	90	static void __init kasan_early_vmemmap_populate(unsigned long address,
	91	unsigned long end,
	92	enum populate_mode mode)
	93	{
d58106c3	94	unsigned long pgt_prot_zero, pgt_prot, sgt_prot;
42db5ed8 VG	95	pgd_t *pg_dir;
	96	p4d_t *p4_dir;
	97	pud_t *pu_dir;
	98	pmd_t *pm_dir;
	99	pte_t *pt_dir;
	100
	101	pgt_prot_zero = pgprot_val(PAGE_KERNEL_RO);
d58106c3 VG	102	if (!has_nx)
d58106c3 VG	103	pgt_prot_zero &= ~_PAGE_NOEXEC;
d411e3c6 VG	104	pgt_prot = pgprot_val(PAGE_KERNEL);
	105	sgt_prot = pgprot_val(SEGMENT_KERNEL);
	106	if (!has_nx \|\| mode == POPULATE_ONE2ONE) {
	107	pgt_prot &= ~_PAGE_NOEXEC;
	108	sgt_prot &= ~_SEGMENT_ENTRY_NOEXEC;
	109	}
42db5ed8 VG	110
	111	while (address < end) {
	112	pg_dir = pgd_offset_k(address);
	113	if (pgd_none(*pg_dir)) {
	114	if (mode == POPULATE_ZERO_SHADOW &&
	115	IS_ALIGNED(address, PGDIR_SIZE) &&
	116	end - address >= PGDIR_SIZE) {
9577dd74 AK	117	pgd_populate(&init_mm, pg_dir,
9577dd74 AK	118	kasan_early_shadow_p4d);
42db5ed8 VG	119	address = (address + PGDIR_SIZE) & PGDIR_MASK;
	120	continue;
	121	}
	122	p4_dir = kasan_early_crst_alloc(_REGION2_ENTRY_EMPTY);
	123	pgd_populate(&init_mm, pg_dir, p4_dir);
	124	}
	125
3e39ce26 VG	126	if (IS_ENABLED(CONFIG_KASAN_S390_4_LEVEL_PAGING) &&
	127	mode == POPULATE_SHALLOW) {
	128	address = (address + P4D_SIZE) & P4D_MASK;
	129	continue;
	130	}
	131
42db5ed8 VG	132	p4_dir = p4d_offset(pg_dir, address);
	133	if (p4d_none(*p4_dir)) {
	134	if (mode == POPULATE_ZERO_SHADOW &&
	135	IS_ALIGNED(address, P4D_SIZE) &&
	136	end - address >= P4D_SIZE) {
9577dd74 AK	137	p4d_populate(&init_mm, p4_dir,
9577dd74 AK	138	kasan_early_shadow_pud);
42db5ed8 VG	139	address = (address + P4D_SIZE) & P4D_MASK;
	140	continue;
	141	}
	142	pu_dir = kasan_early_crst_alloc(_REGION3_ENTRY_EMPTY);
	143	p4d_populate(&init_mm, p4_dir, pu_dir);
	144	}
	145
3e39ce26 VG	146	if (!IS_ENABLED(CONFIG_KASAN_S390_4_LEVEL_PAGING) &&
	147	mode == POPULATE_SHALLOW) {
	148	address = (address + PUD_SIZE) & PUD_MASK;
	149	continue;
	150	}
	151
42db5ed8 VG	152	pu_dir = pud_offset(p4_dir, address);
	153	if (pud_none(*pu_dir)) {
	154	if (mode == POPULATE_ZERO_SHADOW &&
	155	IS_ALIGNED(address, PUD_SIZE) &&
	156	end - address >= PUD_SIZE) {
9577dd74 AK	157	pud_populate(&init_mm, pu_dir,
9577dd74 AK	158	kasan_early_shadow_pmd);
42db5ed8 VG	159	address = (address + PUD_SIZE) & PUD_MASK;
	160	continue;
	161	}
	162	pm_dir = kasan_early_crst_alloc(_SEGMENT_ENTRY_EMPTY);
	163	pud_populate(&init_mm, pu_dir, pm_dir);
	164	}
	165
	166	pm_dir = pmd_offset(pu_dir, address);
	167	if (pmd_none(*pm_dir)) {
	168	if (mode == POPULATE_ZERO_SHADOW &&
	169	IS_ALIGNED(address, PMD_SIZE) &&
	170	end - address >= PMD_SIZE) {
9577dd74 AK	171	pmd_populate(&init_mm, pm_dir,
9577dd74 AK	172	kasan_early_shadow_pte);
42db5ed8 VG	173	address = (address + PMD_SIZE) & PMD_MASK;
	174	continue;
	175	}
d58106c3 VG	176	/* the first megabyte of 1:1 is mapped with 4k pages */
	177	if (has_edat && address && end - address >= PMD_SIZE &&
	178	mode != POPULATE_ZERO_SHADOW) {
	179	void *page;
	180
	181	if (mode == POPULATE_ONE2ONE) {
	182	page = (void *)address;
	183	} else {
	184	page = kasan_early_alloc_segment();
	185	memset(page, 0, _SEGMENT_SIZE);
	186	}
	187	pmd_val(*pm_dir) = __pa(page) \| sgt_prot;
	188	address = (address + PMD_SIZE) & PMD_MASK;
	189	continue;
	190	}
	191
42db5ed8 VG	192	pt_dir = kasan_early_pte_alloc();
42db5ed8 VG	193	pmd_populate(&init_mm, pm_dir, pt_dir);
d58106c3 VG	194	} else if (pmd_large(*pm_dir)) {
	195	address = (address + PMD_SIZE) & PMD_MASK;
	196	continue;
42db5ed8 VG	197	}
	198
	199	pt_dir = pte_offset_kernel(pm_dir, address);
	200	if (pte_none(*pt_dir)) {
	201	void *page;
	202
	203	switch (mode) {
	204	case POPULATE_ONE2ONE:
	205	page = (void *)address;
	206	pte_val(*pt_dir) = __pa(page) \| pgt_prot;
	207	break;
	208	case POPULATE_MAP:
	209	page = kasan_early_alloc_pages(0);
	210	memset(page, 0, PAGE_SIZE);
	211	pte_val(*pt_dir) = __pa(page) \| pgt_prot;
	212	break;
	213	case POPULATE_ZERO_SHADOW:
9577dd74	214	page = kasan_early_shadow_page;
42db5ed8 VG	215	pte_val(*pt_dir) = __pa(page) \| pgt_prot_zero;
42db5ed8 VG	216	break;
3e39ce26 VG	217	case POPULATE_SHALLOW:
	218	/* should never happen */
	219	break;
42db5ed8 VG	220	}
	221	}
	222	address += PAGE_SIZE;
	223	}
	224	}
	225
	226	static void __init kasan_set_pgd(pgd_t *pgd, unsigned long asce_type)
	227	{
	228	unsigned long asce_bits;
	229
	230	asce_bits = asce_type \| _ASCE_TABLE_LENGTH;
	231	S390_lowcore.kernel_asce = (__pa(pgd) & PAGE_MASK) \| asce_bits;
	232	S390_lowcore.user_asce = S390_lowcore.kernel_asce;
	233
	234	__ctl_load(S390_lowcore.kernel_asce, 1, 1);
	235	__ctl_load(S390_lowcore.kernel_asce, 7, 7);
	236	__ctl_load(S390_lowcore.kernel_asce, 13, 13);
	237	}
	238
	239	static void __init kasan_enable_dat(void)
	240	{
	241	psw_t psw;
	242
	243	psw.mask = __extract_psw();
	244	psw_bits(psw).dat = 1;
	245	psw_bits(psw).as = PSW_BITS_AS_HOME;
	246	__load_psw_mask(psw.mask);
	247	}
	248
d58106c3 VG	249	static void __init kasan_early_detect_facilities(void)
d58106c3 VG	250	{
d58106c3 VG	251	if (test_facility(8)) {
	252	has_edat = true;
	253	__ctl_set_bit(0, 23);
	254	}
	255	if (!noexec_disabled && test_facility(130)) {
	256	has_nx = true;
	257	__ctl_set_bit(0, 20);
	258	}
	259	}
	260
c360c9a2 VG	261	static bool __init has_uv_sec_stor_limit(void)
	262	{
	263	/*
	264	* keep these conditions in line with setup_uv()
	265	*/
	266	if (!is_prot_virt_host())
	267	return false;
	268
	269	if (is_prot_virt_guest())
	270	return false;
	271
	272	if (!test_facility(158))
	273	return false;
	274
	275	return !!uv_info.max_sec_stor_addr;
	276	}
	277
42db5ed8 VG	278	void __init kasan_early_init(void)
	279	{
	280	unsigned long untracked_mem_end;
	281	unsigned long shadow_alloc_size;
c360c9a2	282	unsigned long vmax_unlimited;
42db5ed8 VG	283	unsigned long initrd_end;
	284	unsigned long asce_type;
	285	unsigned long memsize;
42db5ed8 VG	286	unsigned long pgt_prot = pgprot_val(PAGE_KERNEL_RO);
42db5ed8 VG	287	pte_t pte_z;
9577dd74 AK	288	pmd_t pmd_z = __pmd(__pa(kasan_early_shadow_pte) \| _SEGMENT_ENTRY);
	289	pud_t pud_z = __pud(__pa(kasan_early_shadow_pmd) \| _REGION3_ENTRY);
	290	p4d_t p4d_z = __p4d(__pa(kasan_early_shadow_pud) \| _REGION2_ENTRY);
42db5ed8	291
d58106c3 VG	292	kasan_early_detect_facilities();
	293	if (!has_nx)
	294	pgt_prot &= ~_PAGE_NOEXEC;
9577dd74	295	pte_z = __pte(__pa(kasan_early_shadow_page) \| pgt_prot);
42db5ed8	296
29635239 VG	297	memsize = get_mem_detect_end();
	298	if (!memsize)
	299	kasan_early_panic("cannot detect physical memory size\n");
78333d1f VG	300	/* respect mem= cmdline parameter */
	301	if (memory_end_set && memsize > memory_end)
	302	memsize = memory_end;
042c1d29 VG	303	if (IS_ENABLED(CONFIG_CRASH_DUMP) && OLDMEM_BASE)
042c1d29 VG	304	memsize = min(memsize, OLDMEM_SIZE);
29635239 VG	305	memsize = min(memsize, KASAN_SHADOW_START);
29635239 VG	306
5dff0381 VG	307	if (IS_ENABLED(CONFIG_KASAN_S390_4_LEVEL_PAGING)) {
	308	/* 4 level paging */
	309	BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, P4D_SIZE));
	310	BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, P4D_SIZE));
	311	crst_table_init((unsigned long *)early_pg_dir,
	312	_REGION2_ENTRY_EMPTY);
c360c9a2 VG	313	untracked_mem_end = kasan_vmax = vmax_unlimited = _REGION1_SIZE;
	314	if (has_uv_sec_stor_limit())
	315	kasan_vmax = min(vmax_unlimited, uv_info.max_sec_stor_addr);
5dff0381 VG	316	asce_type = _ASCE_TYPE_REGION2;
	317	} else {
	318	/* 3 level paging */
	319	BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, PUD_SIZE));
	320	BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, PUD_SIZE));
	321	crst_table_init((unsigned long *)early_pg_dir,
	322	_REGION3_ENTRY_EMPTY);
c360c9a2	323	untracked_mem_end = kasan_vmax = vmax_unlimited = _REGION2_SIZE;
5dff0381 VG	324	asce_type = _ASCE_TYPE_REGION3;
5dff0381 VG	325	}
42db5ed8 VG	326
42db5ed8 VG	327	/* init kasan zero shadow */
9577dd74 AK	328	crst_table_init((unsigned long *)kasan_early_shadow_p4d,
	329	p4d_val(p4d_z));
	330	crst_table_init((unsigned long *)kasan_early_shadow_pud,
	331	pud_val(pud_z));
	332	crst_table_init((unsigned long *)kasan_early_shadow_pmd,
	333	pmd_val(pmd_z));
	334	memset64((u64 *)kasan_early_shadow_pte, pte_val(pte_z), PTRS_PER_PTE);
42db5ed8	335
42db5ed8	336	shadow_alloc_size = memsize >> KASAN_SHADOW_SCALE_SHIFT;
42db5ed8 VG	337	pgalloc_low = round_up((unsigned long)_end, _SEGMENT_SIZE);
	338	if (IS_ENABLED(CONFIG_BLK_DEV_INITRD)) {
	339	initrd_end =
	340	round_up(INITRD_START + INITRD_SIZE, _SEGMENT_SIZE);
	341	pgalloc_low = max(pgalloc_low, initrd_end);
	342	}
	343
	344	if (pgalloc_low + shadow_alloc_size > memsize)
	345	kasan_early_panic("out of memory during initialisation\n");
	346
d58106c3 VG	347	if (has_edat) {
	348	segment_pos = round_down(memsize, _SEGMENT_SIZE);
	349	segment_low = segment_pos - shadow_alloc_size;
	350	pgalloc_pos = segment_low;
	351	} else {
	352	pgalloc_pos = memsize;
	353	}
42db5ed8 VG	354	init_mm.pgd = early_pg_dir;
	355	/*
	356	* Current memory layout:
3e39ce26 VG	357	* +- 0 -------------+ +- shadow start -+
	358	* \| 1:1 ram mapping \| /\| 1/8 ram \|
	359	* \| \| / \| \|
	360	* +- end of ram ----+ / +----------------+
	361	* \| ... gap ... \| / \| \|
	362	* \| \|/ \| kasan \|
	363	* +- shadow start --+ \| zero \|
	364	* \| 1/8 addr space \| \| page \|
	365	* +- shadow end -+ \| mapping \|
	366	* \| ... gap ... \|\ \| (untracked) \|
	367	* +- vmalloc area -+ \ \| \|
	368	* \| vmalloc_size \| \ \| \|
	369	* +- modules vaddr -+ \ +----------------+
	370	* \| 2Gb \| \\| unmapped \| allocated per module
	371	* +-----------------+ +- shadow end ---+
	372	*
	373	* Current memory layout (KASAN_VMALLOC):
	374	* +- 0 -------------+ +- shadow start -+
	375	* \| 1:1 ram mapping \| /\| 1/8 ram \|
	376	* \| \| / \| \|
	377	* +- end of ram ----+ / +----------------+
	378	* \| ... gap ... \| / \| kasan \|
	379	* \| \|/ \| zero \|
	380	* +- shadow start --+ \| page \|
	381	* \| 1/8 addr space \| \| mapping \|
	382	* +- shadow end -+ \| (untracked) \|
	383	* \| ... gap ... \|\ \| \|
	384	* +- vmalloc area -+ \ +- vmalloc area -+
	385	* \| vmalloc_size \| \ \|shallow populate\|
	386	* +- modules vaddr -+ \ +- modules area -+
	387	* \| 2Gb \| \\|shallow populate\|
	388	* +-----------------+ +- shadow end ---+
42db5ed8	389	*/
793213a8	390	/* populate kasan shadow (for identity mapping and zero page mapping) */
42db5ed8	391	kasan_early_vmemmap_populate(__sha(0), __sha(memsize), POPULATE_MAP);
793213a8	392	if (IS_ENABLED(CONFIG_MODULES))
c360c9a2	393	untracked_mem_end = kasan_vmax - MODULES_LEN;
3e39ce26	394	if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
c360c9a2	395	untracked_mem_end = kasan_vmax - vmalloc_size - MODULES_LEN;
3e39ce26 VG	396	/* shallowly populate kasan shadow for vmalloc and modules */
3e39ce26 VG	397	kasan_early_vmemmap_populate(__sha(untracked_mem_end),
c360c9a2	398	__sha(kasan_vmax), POPULATE_SHALLOW);
3e39ce26 VG	399	}
3e39ce26 VG	400	/* populate kasan shadow for untracked memory */
12e55fa1 VG	401	kasan_early_vmemmap_populate(__sha(max_physmem_end),
12e55fa1 VG	402	__sha(untracked_mem_end),
42db5ed8	403	POPULATE_ZERO_SHADOW);
c360c9a2 VG	404	kasan_early_vmemmap_populate(__sha(kasan_vmax),
	405	__sha(vmax_unlimited),
	406	POPULATE_ZERO_SHADOW);
135ff163 VG	407	/* memory allocated for identity mapping structs will be freed later */
	408	pgalloc_freeable = pgalloc_pos;
	409	/* populate identity mapping */
	410	kasan_early_vmemmap_populate(0, memsize, POPULATE_ONE2ONE);
42db5ed8 VG	411	kasan_set_pgd(early_pg_dir, asce_type);
	412	kasan_enable_dat();
	413	/* enable kasan */
	414	init_task.kasan_depth = 0;
	415	memblock_reserve(pgalloc_pos, memsize - pgalloc_pos);
	416	sclp_early_printk("KernelAddressSanitizer initialized\n");
	417	}
	418
	419	void __init kasan_copy_shadow(pgd_t *pg_dir)
	420	{
	421	/*
	422	* At this point we are still running on early pages setup early_pg_dir,
	423	* while swapper_pg_dir has just been initialized with identity mapping.
	424	* Carry over shadow memory region from early_pg_dir to swapper_pg_dir.
	425	*/
	426
	427	pgd_t *pg_dir_src;
	428	pgd_t *pg_dir_dst;
	429	p4d_t *p4_dir_src;
	430	p4d_t *p4_dir_dst;
	431	pud_t *pu_dir_src;
	432	pud_t *pu_dir_dst;
	433
	434	pg_dir_src = pgd_offset_raw(early_pg_dir, KASAN_SHADOW_START);
	435	pg_dir_dst = pgd_offset_raw(pg_dir, KASAN_SHADOW_START);
	436	p4_dir_src = p4d_offset(pg_dir_src, KASAN_SHADOW_START);
	437	p4_dir_dst = p4d_offset(pg_dir_dst, KASAN_SHADOW_START);
	438	if (!p4d_folded(*p4_dir_src)) {
	439	/* 4 level paging */
	440	memcpy(p4_dir_dst, p4_dir_src,
	441	(KASAN_SHADOW_SIZE >> P4D_SHIFT) * sizeof(p4d_t));
	442	return;
	443	}
	444	/* 3 level paging */
	445	pu_dir_src = pud_offset(p4_dir_src, KASAN_SHADOW_START);
	446	pu_dir_dst = pud_offset(p4_dir_dst, KASAN_SHADOW_START);
	447	memcpy(pu_dir_dst, pu_dir_src,
	448	(KASAN_SHADOW_SIZE >> PUD_SHIFT) * sizeof(pud_t));
	449	}
135ff163 VG	450
	451	void __init kasan_free_early_identity(void)
	452	{
	453	memblock_free(pgalloc_pos, pgalloc_freeable - pgalloc_pos);
	454	}