[linux-block.git] / arch / riscv / kernel / elf_kexec.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Load ELF vmlinux file for the kexec_file_load syscall.
 *
 * Copyright (C) 2021 Huawei Technologies Co, Ltd.
 *
 * Author: Liao Chang (liaochang1@huawei.com)
 *
 * Based on kexec-tools' kexec-elf-riscv.c, heavily modified
 * for kernel.
 */

#define pr_fmt(fmt)	"kexec_image: " fmt

#include <linux/elf.h>
#include <linux/kexec.h>
#include <linux/slab.h>
#include <linux/of.h>
#include <linux/libfdt.h>
#include <linux/types.h>
#include <linux/memblock.h>
#include <linux/vmalloc.h>
#include <asm/setup.h>

int arch_kimage_file_post_load_cleanup(struct kimage *image)
{
	kvfree(image->arch.fdt);
	image->arch.fdt = NULL;

	vfree(image->elf_headers);
	image->elf_headers = NULL;
	image->elf_headers_sz = 0;

	return kexec_image_post_load_cleanup_default(image);
}

static int riscv_kexec_elf_load(struct kimage *image, struct elfhdr *ehdr,
				struct kexec_elf_info *elf_info, unsigned long old_pbase,
				unsigned long new_pbase)
{
	int i;
	int ret = 0;
	size_t size;
	struct kexec_buf kbuf;
	const struct elf_phdr *phdr;

	kbuf.image = image;

	for (i = 0; i < ehdr->e_phnum; i++) {
		phdr = &elf_info->proghdrs[i];
		if (phdr->p_type != PT_LOAD)
			continue;

		size = phdr->p_filesz;
		if (size > phdr->p_memsz)
			size = phdr->p_memsz;

		kbuf.buffer = (void *) elf_info->buffer + phdr->p_offset;
		kbuf.bufsz = size;
		kbuf.buf_align = phdr->p_align;
		kbuf.mem = phdr->p_paddr - old_pbase + new_pbase;
		kbuf.memsz = phdr->p_memsz;
		kbuf.top_down = false;
		ret = kexec_add_buffer(&kbuf);
		if (ret)
			break;
	}

	return ret;
}

/*
 * Go through the available phsyical memory regions and find one that hold
 * an image of the specified size.
 */
static int elf_find_pbase(struct kimage *image, unsigned long kernel_len,
			  struct elfhdr *ehdr, struct kexec_elf_info *elf_info,
			  unsigned long *old_pbase, unsigned long *new_pbase)
{
	int i;
	int ret;
	struct kexec_buf kbuf;
	const struct elf_phdr *phdr;
	unsigned long lowest_paddr = ULONG_MAX;
	unsigned long lowest_vaddr = ULONG_MAX;

	for (i = 0; i < ehdr->e_phnum; i++) {
		phdr = &elf_info->proghdrs[i];
		if (phdr->p_type != PT_LOAD)
			continue;

		if (lowest_paddr > phdr->p_paddr)
			lowest_paddr = phdr->p_paddr;

		if (lowest_vaddr > phdr->p_vaddr)
			lowest_vaddr = phdr->p_vaddr;
	}

	kbuf.image = image;
	kbuf.buf_min = lowest_paddr;
	kbuf.buf_max = ULONG_MAX;

	/*
	 * Current riscv boot protocol requires 2MB alignment for
	 * RV64 and 4MB alignment for RV32
	 *
	 */
	kbuf.buf_align = PMD_SIZE;
	kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
	kbuf.memsz = ALIGN(kernel_len, PAGE_SIZE);
	kbuf.top_down = false;
	ret = arch_kexec_locate_mem_hole(&kbuf);
	if (!ret) {
		*old_pbase = lowest_paddr;
		*new_pbase = kbuf.mem;
		image->start = ehdr->e_entry - lowest_vaddr + kbuf.mem;
	}
	return ret;
}

#ifdef CONFIG_CRASH_DUMP
static int get_nr_ram_ranges_callback(struct resource *res, void *arg)
{
	unsigned int *nr_ranges = arg;

	(*nr_ranges)++;
	return 0;
}

static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg)
{
	struct crash_mem *cmem = arg;

	cmem->ranges[cmem->nr_ranges].start = res->start;
	cmem->ranges[cmem->nr_ranges].end = res->end;
	cmem->nr_ranges++;

	return 0;
}

static int prepare_elf_headers(void **addr, unsigned long *sz)
{
	struct crash_mem *cmem;
	unsigned int nr_ranges;
	int ret;

	nr_ranges = 1; /* For exclusion of crashkernel region */
	walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback);

	cmem = kmalloc(struct_size(cmem, ranges, nr_ranges), GFP_KERNEL);
	if (!cmem)
		return -ENOMEM;

	cmem->max_nr_ranges = nr_ranges;
	cmem->nr_ranges = 0;
	ret = walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback);
	if (ret)
		goto out;

	/* Exclude crashkernel region */
	ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end);
	if (!ret)
		ret = crash_prepare_elf64_headers(cmem, true, addr, sz);

out:
	kfree(cmem);
	return ret;
}

static char *setup_kdump_cmdline(struct kimage *image, char *cmdline,
				 unsigned long cmdline_len)
{
	int elfcorehdr_strlen;
	char *cmdline_ptr;

	cmdline_ptr = kzalloc(COMMAND_LINE_SIZE, GFP_KERNEL);
	if (!cmdline_ptr)
		return NULL;

	elfcorehdr_strlen = sprintf(cmdline_ptr, "elfcorehdr=0x%lx ",
		image->elf_load_addr);

	if (elfcorehdr_strlen + cmdline_len > COMMAND_LINE_SIZE) {
		pr_err("Appending elfcorehdr=<addr> exceeds cmdline size\n");
		kfree(cmdline_ptr);
		return NULL;
	}

	memcpy(cmdline_ptr + elfcorehdr_strlen, cmdline, cmdline_len);
	/* Ensure it's nul terminated */
	cmdline_ptr[COMMAND_LINE_SIZE - 1] = '\0';
	return cmdline_ptr;
}
#endif

static void *elf_kexec_load(struct kimage *image, char *kernel_buf,
			    unsigned long kernel_len, char *initrd,
			    unsigned long initrd_len, char *cmdline,
			    unsigned long cmdline_len)
{
	int ret;
	void *fdt;
	unsigned long old_kernel_pbase = ULONG_MAX;
	unsigned long new_kernel_pbase = 0UL;
	unsigned long initrd_pbase = 0UL;
	unsigned long kernel_start;
	struct elfhdr ehdr;
	struct kexec_buf kbuf;
	struct kexec_elf_info elf_info;
	char *modified_cmdline = NULL;

	ret = kexec_build_elf_info(kernel_buf, kernel_len, &ehdr, &elf_info);
	if (ret)
		return ERR_PTR(ret);

	ret = elf_find_pbase(image, kernel_len, &ehdr, &elf_info,
			     &old_kernel_pbase, &new_kernel_pbase);
	if (ret)
		goto out;
	kernel_start = image->start;

	/* Add the kernel binary to the image */
	ret = riscv_kexec_elf_load(image, &ehdr, &elf_info,
				   old_kernel_pbase, new_kernel_pbase);
	if (ret)
		goto out;

	kbuf.image = image;
	kbuf.buf_min = new_kernel_pbase + kernel_len;
	kbuf.buf_max = ULONG_MAX;

#ifdef CONFIG_CRASH_DUMP
	/* Add elfcorehdr */
	if (image->type == KEXEC_TYPE_CRASH) {
		void *headers;
		unsigned long headers_sz;
		ret = prepare_elf_headers(&headers, &headers_sz);
		if (ret) {
			pr_err("Preparing elf core header failed\n");
			goto out;
		}

		kbuf.buffer = headers;
		kbuf.bufsz = headers_sz;
		kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
		kbuf.memsz = headers_sz;
		kbuf.buf_align = ELF_CORE_HEADER_ALIGN;
		kbuf.top_down = true;

		ret = kexec_add_buffer(&kbuf);
		if (ret) {
			vfree(headers);
			goto out;
		}
		image->elf_headers = headers;
		image->elf_load_addr = kbuf.mem;
		image->elf_headers_sz = headers_sz;

		kexec_dprintk("Loaded elf core header at 0x%lx bufsz=0x%lx memsz=0x%lx\n",
			      image->elf_load_addr, kbuf.bufsz, kbuf.memsz);

		/* Setup cmdline for kdump kernel case */
		modified_cmdline = setup_kdump_cmdline(image, cmdline,
						       cmdline_len);
		if (!modified_cmdline) {
			pr_err("Setting up cmdline for kdump kernel failed\n");
			ret = -EINVAL;
			goto out;
		}
		cmdline = modified_cmdline;
	}
#endif

#ifdef CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY
	/* Add purgatory to the image */
	kbuf.top_down = true;
	kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
	ret = kexec_load_purgatory(image, &kbuf);
	if (ret) {
		pr_err("Error loading purgatory ret=%d\n", ret);
		goto out;
	}
	kexec_dprintk("Loaded purgatory at 0x%lx\n", kbuf.mem);

	ret = kexec_purgatory_get_set_symbol(image, "riscv_kernel_entry",
					     &kernel_start,
					     sizeof(kernel_start), 0);
	if (ret)
		pr_err("Error update purgatory ret=%d\n", ret);
#endif /* CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY */

	/* Add the initrd to the image */
	if (initrd != NULL) {
		kbuf.buffer = initrd;
		kbuf.bufsz = kbuf.memsz = initrd_len;
		kbuf.buf_align = PAGE_SIZE;
		kbuf.top_down = true;
		kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
		ret = kexec_add_buffer(&kbuf);
		if (ret)
			goto out;
		initrd_pbase = kbuf.mem;
		kexec_dprintk("Loaded initrd at 0x%lx\n", initrd_pbase);
	}

	/* Add the DTB to the image */
	fdt = of_kexec_alloc_and_setup_fdt(image, initrd_pbase,
					   initrd_len, cmdline, 0);
	if (!fdt) {
		pr_err("Error setting up the new device tree.\n");
		ret = -EINVAL;
		goto out;
	}

	fdt_pack(fdt);
	kbuf.buffer = fdt;
	kbuf.bufsz = kbuf.memsz = fdt_totalsize(fdt);
	kbuf.buf_align = PAGE_SIZE;
	kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
	kbuf.top_down = true;
	ret = kexec_add_buffer(&kbuf);
	if (ret) {
		pr_err("Error add DTB kbuf ret=%d\n", ret);
		goto out_free_fdt;
	}
	/* Cache the fdt buffer address for memory cleanup */
	image->arch.fdt = fdt;
	kexec_dprintk("Loaded device tree at 0x%lx\n", kbuf.mem);
	goto out;

out_free_fdt:
	kvfree(fdt);
out:
	kfree(modified_cmdline);
	kexec_free_elf_info(&elf_info);
	return ret ? ERR_PTR(ret) : NULL;
}

#define RV_X(x, s, n)  (((x) >> (s)) & ((1 << (n)) - 1))
#define RISCV_IMM_BITS 12
#define RISCV_IMM_REACH (1LL << RISCV_IMM_BITS)
#define RISCV_CONST_HIGH_PART(x) \
	(((x) + (RISCV_IMM_REACH >> 1)) & ~(RISCV_IMM_REACH - 1))
#define RISCV_CONST_LOW_PART(x) ((x) - RISCV_CONST_HIGH_PART(x))

#define ENCODE_ITYPE_IMM(x) \
	(RV_X(x, 0, 12) << 20)
#define ENCODE_BTYPE_IMM(x) \
	((RV_X(x, 1, 4) << 8) | (RV_X(x, 5, 6) << 25) | \
	(RV_X(x, 11, 1) << 7) | (RV_X(x, 12, 1) << 31))
#define ENCODE_UTYPE_IMM(x) \
	(RV_X(x, 12, 20) << 12)
#define ENCODE_JTYPE_IMM(x) \
	((RV_X(x, 1, 10) << 21) | (RV_X(x, 11, 1) << 20) | \
	(RV_X(x, 12, 8) << 12) | (RV_X(x, 20, 1) << 31))
#define ENCODE_CBTYPE_IMM(x) \
	((RV_X(x, 1, 2) << 3) | (RV_X(x, 3, 2) << 10) | (RV_X(x, 5, 1) << 2) | \
	(RV_X(x, 6, 2) << 5) | (RV_X(x, 8, 1) << 12))
#define ENCODE_CJTYPE_IMM(x) \
	((RV_X(x, 1, 3) << 3) | (RV_X(x, 4, 1) << 11) | (RV_X(x, 5, 1) << 2) | \
	(RV_X(x, 6, 1) << 7) | (RV_X(x, 7, 1) << 6) | (RV_X(x, 8, 2) << 9) | \
	(RV_X(x, 10, 1) << 8) | (RV_X(x, 11, 1) << 12))
#define ENCODE_UJTYPE_IMM(x) \
	(ENCODE_UTYPE_IMM(RISCV_CONST_HIGH_PART(x)) | \
	(ENCODE_ITYPE_IMM(RISCV_CONST_LOW_PART(x)) << 32))
#define ENCODE_UITYPE_IMM(x) \
	(ENCODE_UTYPE_IMM(x) | (ENCODE_ITYPE_IMM(x) << 32))

#define CLEAN_IMM(type, x) \
	((~ENCODE_##type##_IMM((uint64_t)(-1))) & (x))

int arch_kexec_apply_relocations_add(struct purgatory_info *pi,
				     Elf_Shdr *section,
				     const Elf_Shdr *relsec,
				     const Elf_Shdr *symtab)
{
	const char *strtab, *name, *shstrtab;
	const Elf_Shdr *sechdrs;
	Elf64_Rela *relas;
	int i, r_type;

	/* String & section header string table */
	sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff;
	strtab = (char *)pi->ehdr + sechdrs[symtab->sh_link].sh_offset;
	shstrtab = (char *)pi->ehdr + sechdrs[pi->ehdr->e_shstrndx].sh_offset;

	relas = (void *)pi->ehdr + relsec->sh_offset;

	for (i = 0; i < relsec->sh_size / sizeof(*relas); i++) {
		const Elf_Sym *sym;	/* symbol to relocate */
		unsigned long addr;	/* final location after relocation */
		unsigned long val;	/* relocated symbol value */
		unsigned long sec_base;	/* relocated symbol value */
		void *loc;		/* tmp location to modify */

		sym = (void *)pi->ehdr + symtab->sh_offset;
		sym += ELF64_R_SYM(relas[i].r_info);

		if (sym->st_name)
			name = strtab + sym->st_name;
		else
			name = shstrtab + sechdrs[sym->st_shndx].sh_name;

		loc = pi->purgatory_buf;
		loc += section->sh_offset;
		loc += relas[i].r_offset;

		if (sym->st_shndx == SHN_ABS)
			sec_base = 0;
		else if (sym->st_shndx >= pi->ehdr->e_shnum) {
			pr_err("Invalid section %d for symbol %s\n",
			       sym->st_shndx, name);
			return -ENOEXEC;
		} else
			sec_base = pi->sechdrs[sym->st_shndx].sh_addr;

		val = sym->st_value;
		val += sec_base;
		val += relas[i].r_addend;

		addr = section->sh_addr + relas[i].r_offset;

		r_type = ELF64_R_TYPE(relas[i].r_info);

		switch (r_type) {
		case R_RISCV_BRANCH:
			*(u32 *)loc = CLEAN_IMM(BTYPE, *(u32 *)loc) |
				 ENCODE_BTYPE_IMM(val - addr);
			break;
		case R_RISCV_JAL:
			*(u32 *)loc = CLEAN_IMM(JTYPE, *(u32 *)loc) |
				 ENCODE_JTYPE_IMM(val - addr);
			break;
		/*
		 * With no R_RISCV_PCREL_LO12_S, R_RISCV_PCREL_LO12_I
		 * sym is expected to be next to R_RISCV_PCREL_HI20
		 * in purgatory relsec. Handle it like R_RISCV_CALL
		 * sym, instead of searching the whole relsec.
		 */
		case R_RISCV_PCREL_HI20:
		case R_RISCV_CALL_PLT:
		case R_RISCV_CALL:
			*(u64 *)loc = CLEAN_IMM(UITYPE, *(u64 *)loc) |
				 ENCODE_UJTYPE_IMM(val - addr);
			break;
		case R_RISCV_RVC_BRANCH:
			*(u32 *)loc = CLEAN_IMM(CBTYPE, *(u32 *)loc) |
				 ENCODE_CBTYPE_IMM(val - addr);
			break;
		case R_RISCV_RVC_JUMP:
			*(u32 *)loc = CLEAN_IMM(CJTYPE, *(u32 *)loc) |
				 ENCODE_CJTYPE_IMM(val - addr);
			break;
		case R_RISCV_ADD32:
			*(u32 *)loc += val;
			break;
		case R_RISCV_SUB32:
			*(u32 *)loc -= val;
			break;
		/* It has been applied by R_RISCV_PCREL_HI20 sym */
		case R_RISCV_PCREL_LO12_I:
		case R_RISCV_ALIGN:
		case R_RISCV_RELAX:
			break;
		default:
			pr_err("Unknown rela relocation: %d\n", r_type);
			return -ENOEXEC;
		}
	}
	return 0;
}

const struct kexec_file_ops elf_kexec_ops = {
	.probe = kexec_elf_probe,
	.load  = elf_kexec_load,
};
Commit	Line	Data
6261586e LC	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Load ELF vmlinux file for the kexec_file_load syscall.
	4	*
	5	* Copyright (C) 2021 Huawei Technologies Co, Ltd.
	6	*
	7	* Author: Liao Chang (liaochang1@huawei.com)
	8	*
	9	* Based on kexec-tools' kexec-elf-riscv.c, heavily modified
	10	* for kernel.
	11	*/
	12
	13	#define pr_fmt(fmt) "kexec_image: " fmt
	14
	15	#include <linux/elf.h>
	16	#include <linux/kexec.h>
	17	#include <linux/slab.h>
	18	#include <linux/of.h>
	19	#include <linux/libfdt.h>
	20	#include <linux/types.h>
8acea455	21	#include <linux/memblock.h>
0069455b	22	#include <linux/vmalloc.h>
8acea455	23	#include <asm/setup.h>
6261586e	24
96df59b1 LH	25	int arch_kimage_file_post_load_cleanup(struct kimage *image)
	26	{
	27	kvfree(image->arch.fdt);
	28	image->arch.fdt = NULL;
	29
cbc32023 LH	30	vfree(image->elf_headers);
	31	image->elf_headers = NULL;
	32	image->elf_headers_sz = 0;
	33
96df59b1 LH	34	return kexec_image_post_load_cleanup_default(image);
	35	}
	36
6261586e LC	37	static int riscv_kexec_elf_load(struct kimage image, struct elfhdr ehdr,
	38	struct kexec_elf_info *elf_info, unsigned long old_pbase,
	39	unsigned long new_pbase)
	40	{
	41	int i;
	42	int ret = 0;
	43	size_t size;
	44	struct kexec_buf kbuf;
	45	const struct elf_phdr *phdr;
	46
	47	kbuf.image = image;
	48
	49	for (i = 0; i < ehdr->e_phnum; i++) {
	50	phdr = &elf_info->proghdrs[i];
	51	if (phdr->p_type != PT_LOAD)
	52	continue;
	53
	54	size = phdr->p_filesz;
	55	if (size > phdr->p_memsz)
	56	size = phdr->p_memsz;
	57
	58	kbuf.buffer = (void *) elf_info->buffer + phdr->p_offset;
	59	kbuf.bufsz = size;
	60	kbuf.buf_align = phdr->p_align;
	61	kbuf.mem = phdr->p_paddr - old_pbase + new_pbase;
	62	kbuf.memsz = phdr->p_memsz;
	63	kbuf.top_down = false;
	64	ret = kexec_add_buffer(&kbuf);
	65	if (ret)
	66	break;
	67	}
	68
	69	return ret;
	70	}
	71
	72	/*
	73	* Go through the available phsyical memory regions and find one that hold
	74	* an image of the specified size.
	75	*/
	76	static int elf_find_pbase(struct kimage *image, unsigned long kernel_len,
	77	struct elfhdr ehdr, struct kexec_elf_info elf_info,
	78	unsigned long old_pbase, unsigned long new_pbase)
	79	{
	80	int i;
	81	int ret;
	82	struct kexec_buf kbuf;
	83	const struct elf_phdr *phdr;
	84	unsigned long lowest_paddr = ULONG_MAX;
	85	unsigned long lowest_vaddr = ULONG_MAX;
	86
	87	for (i = 0; i < ehdr->e_phnum; i++) {
	88	phdr = &elf_info->proghdrs[i];
	89	if (phdr->p_type != PT_LOAD)
	90	continue;
	91
	92	if (lowest_paddr > phdr->p_paddr)
	93	lowest_paddr = phdr->p_paddr;
	94
	95	if (lowest_vaddr > phdr->p_vaddr)
	96	lowest_vaddr = phdr->p_vaddr;
	97	}
	98
	99	kbuf.image = image;
	100	kbuf.buf_min = lowest_paddr;
101	kbuf.buf_max = ULONG_MAX;
1bfb2b61 SS	102
	103	/*
	104	* Current riscv boot protocol requires 2MB alignment for
	105	* RV64 and 4MB alignment for RV32
	106	*
	107	*/
	108	kbuf.buf_align = PMD_SIZE;
6261586e LC	109	kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
	110	kbuf.memsz = ALIGN(kernel_len, PAGE_SIZE);
	111	kbuf.top_down = false;
	112	ret = arch_kexec_locate_mem_hole(&kbuf);
	113	if (!ret) {
	114	*old_pbase = lowest_paddr;
	115	*new_pbase = kbuf.mem;
	116	image->start = ehdr->e_entry - lowest_vaddr + kbuf.mem;
	117	}
	118	return ret;
	119	}
	120
0978a63f	121	#ifdef CONFIG_CRASH_DUMP
8acea455 LZ	122	static int get_nr_ram_ranges_callback(struct resource res, void arg)
	123	{
	124	unsigned int *nr_ranges = arg;
	125
	126	(*nr_ranges)++;
	127	return 0;
	128	}
	129
	130	static int prepare_elf64_ram_headers_callback(struct resource res, void arg)
	131	{
	132	struct crash_mem *cmem = arg;
	133
	134	cmem->ranges[cmem->nr_ranges].start = res->start;
	135	cmem->ranges[cmem->nr_ranges].end = res->end;
	136	cmem->nr_ranges++;
	137
	138	return 0;
	139	}
	140
	141	static int prepare_elf_headers(void *addr, unsigned long sz)
	142	{
	143	struct crash_mem *cmem;
	144	unsigned int nr_ranges;
	145	int ret;
	146
	147	nr_ranges = 1; /* For exclusion of crashkernel region */
	148	walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback);
	149
	150	cmem = kmalloc(struct_size(cmem, ranges, nr_ranges), GFP_KERNEL);
	151	if (!cmem)
	152	return -ENOMEM;
	153
	154	cmem->max_nr_ranges = nr_ranges;
	155	cmem->nr_ranges = 0;
	156	ret = walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback);
	157	if (ret)
	158	goto out;
	159
	160	/* Exclude crashkernel region */
	161	ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end);
	162	if (!ret)
	163	ret = crash_prepare_elf64_headers(cmem, true, addr, sz);
	164
	165	out:
	166	kfree(cmem);
	167	return ret;
	168	}
	169
	170	static char setup_kdump_cmdline(struct kimage image, char *cmdline,
	171	unsigned long cmdline_len)
	172	{
	173	int elfcorehdr_strlen;
	174	char *cmdline_ptr;
	175
	176	cmdline_ptr = kzalloc(COMMAND_LINE_SIZE, GFP_KERNEL);
	177	if (!cmdline_ptr)
	178	return NULL;
	179
	180	elfcorehdr_strlen = sprintf(cmdline_ptr, "elfcorehdr=0x%lx ",
	181	image->elf_load_addr);
	182
	183	if (elfcorehdr_strlen + cmdline_len > COMMAND_LINE_SIZE) {
	184	pr_err("Appending elfcorehdr=<addr> exceeds cmdline size\n");
	185	kfree(cmdline_ptr);
186	return NULL;
187	}
188
189	memcpy(cmdline_ptr + elfcorehdr_strlen, cmdline, cmdline_len);
190	/* Ensure it's nul terminated */
191	cmdline_ptr[COMMAND_LINE_SIZE - 1] = '\0';
192	return cmdline_ptr;
193	}
0978a63f	194	#endif
8acea455	195
6261586e LC	196	static void elf_kexec_load(struct kimage image, char *kernel_buf,
	197	unsigned long kernel_len, char *initrd,
	198	unsigned long initrd_len, char *cmdline,
	199	unsigned long cmdline_len)
	200	{
	201	int ret;
0978a63f	202	void *fdt;
6261586e LC	203	unsigned long old_kernel_pbase = ULONG_MAX;
	204	unsigned long new_kernel_pbase = 0UL;
	205	unsigned long initrd_pbase = 0UL;
838b3e28	206	unsigned long kernel_start;
6261586e LC	207	struct elfhdr ehdr;
	208	struct kexec_buf kbuf;
	209	struct kexec_elf_info elf_info;
8acea455	210	char *modified_cmdline = NULL;
6261586e LC	211
	212	ret = kexec_build_elf_info(kernel_buf, kernel_len, &ehdr, &elf_info);
	213	if (ret)
	214	return ERR_PTR(ret);
	215
	216	ret = elf_find_pbase(image, kernel_len, &ehdr, &elf_info,
	217	&old_kernel_pbase, &new_kernel_pbase);
	218	if (ret)
	219	goto out;
838b3e28	220	kernel_start = image->start;
6261586e LC	221
	222	/* Add the kernel binary to the image */
	223	ret = riscv_kexec_elf_load(image, &ehdr, &elf_info,
	224	old_kernel_pbase, new_kernel_pbase);
	225	if (ret)
	226	goto out;
	227
	228	kbuf.image = image;
	229	kbuf.buf_min = new_kernel_pbase + kernel_len;
	230	kbuf.buf_max = ULONG_MAX;
8acea455	231
0978a63f	232	#ifdef CONFIG_CRASH_DUMP
8acea455 LZ	233	/* Add elfcorehdr */
8acea455 LZ	234	if (image->type == KEXEC_TYPE_CRASH) {
0978a63f BH	235	void *headers;
0978a63f BH	236	unsigned long headers_sz;
8acea455 LZ	237	ret = prepare_elf_headers(&headers, &headers_sz);
	238	if (ret) {
	239	pr_err("Preparing elf core header failed\n");
	240	goto out;
	241	}
	242
	243	kbuf.buffer = headers;
	244	kbuf.bufsz = headers_sz;
	245	kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
	246	kbuf.memsz = headers_sz;
	247	kbuf.buf_align = ELF_CORE_HEADER_ALIGN;
	248	kbuf.top_down = true;
	249
	250	ret = kexec_add_buffer(&kbuf);
	251	if (ret) {
	252	vfree(headers);
	253	goto out;
	254	}
	255	image->elf_headers = headers;
	256	image->elf_load_addr = kbuf.mem;
	257	image->elf_headers_sz = headers_sz;
	258
eb7622d9 BH	259	kexec_dprintk("Loaded elf core header at 0x%lx bufsz=0x%lx memsz=0x%lx\n",
eb7622d9 BH	260	image->elf_load_addr, kbuf.bufsz, kbuf.memsz);
8acea455 LZ	261
	262	/* Setup cmdline for kdump kernel case */
	263	modified_cmdline = setup_kdump_cmdline(image, cmdline,
	264	cmdline_len);
	265	if (!modified_cmdline) {
	266	pr_err("Setting up cmdline for kdump kernel failed\n");
	267	ret = -EINVAL;
	268	goto out;
	269	}
	270	cmdline = modified_cmdline;
	271	}
0978a63f	272	#endif
8acea455	273
e6265fe7	274	#ifdef CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY
838b3e28 LZ	275	/* Add purgatory to the image */
	276	kbuf.top_down = true;
	277	kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
	278	ret = kexec_load_purgatory(image, &kbuf);
	279	if (ret) {
	280	pr_err("Error loading purgatory ret=%d\n", ret);
	281	goto out;
	282	}
eb7622d9 BH	283	kexec_dprintk("Loaded purgatory at 0x%lx\n", kbuf.mem);
eb7622d9 BH	284
838b3e28 LZ	285	ret = kexec_purgatory_get_set_symbol(image, "riscv_kernel_entry",
	286	&kernel_start,
	287	sizeof(kernel_start), 0);
	288	if (ret)
	289	pr_err("Error update purgatory ret=%d\n", ret);
e6265fe7	290	#endif /* CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY */
838b3e28	291
6261586e LC	292	/* Add the initrd to the image */
	293	if (initrd != NULL) {
	294	kbuf.buffer = initrd;
	295	kbuf.bufsz = kbuf.memsz = initrd_len;
	296	kbuf.buf_align = PAGE_SIZE;
49af7a2c	297	kbuf.top_down = true;
6261586e LC	298	kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
	299	ret = kexec_add_buffer(&kbuf);
	300	if (ret)
	301	goto out;
	302	initrd_pbase = kbuf.mem;
eb7622d9	303	kexec_dprintk("Loaded initrd at 0x%lx\n", initrd_pbase);
6261586e LC	304	}
	305
	306	/* Add the DTB to the image */
	307	fdt = of_kexec_alloc_and_setup_fdt(image, initrd_pbase,
	308	initrd_len, cmdline, 0);
	309	if (!fdt) {
	310	pr_err("Error setting up the new device tree.\n");
	311	ret = -EINVAL;
	312	goto out;
	313	}
	314
	315	fdt_pack(fdt);
	316	kbuf.buffer = fdt;
	317	kbuf.bufsz = kbuf.memsz = fdt_totalsize(fdt);
	318	kbuf.buf_align = PAGE_SIZE;
	319	kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
	320	kbuf.top_down = true;
	321	ret = kexec_add_buffer(&kbuf);
	322	if (ret) {
	323	pr_err("Error add DTB kbuf ret=%d\n", ret);
	324	goto out_free_fdt;
	325	}
96df59b1 LH	326	/* Cache the fdt buffer address for memory cleanup */
96df59b1 LH	327	image->arch.fdt = fdt;
eb7622d9	328	kexec_dprintk("Loaded device tree at 0x%lx\n", kbuf.mem);
6261586e LC	329	goto out;
	330
	331	out_free_fdt:
	332	kvfree(fdt);
	333	out:
8acea455	334	kfree(modified_cmdline);
6261586e LC	335	kexec_free_elf_info(&elf_info);
	336	return ret ? ERR_PTR(ret) : NULL;
	337	}
	338
838b3e28 LZ	339	#define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1))
	340	#define RISCV_IMM_BITS 12
	341	#define RISCV_IMM_REACH (1LL << RISCV_IMM_BITS)
	342	#define RISCV_CONST_HIGH_PART(x) \
	343	(((x) + (RISCV_IMM_REACH >> 1)) & ~(RISCV_IMM_REACH - 1))
	344	#define RISCV_CONST_LOW_PART(x) ((x) - RISCV_CONST_HIGH_PART(x))
	345
	346	#define ENCODE_ITYPE_IMM(x) \
	347	(RV_X(x, 0, 12) << 20)
	348	#define ENCODE_BTYPE_IMM(x) \
	349	((RV_X(x, 1, 4) << 8) \| (RV_X(x, 5, 6) << 25) \| \
	350	(RV_X(x, 11, 1) << 7) \| (RV_X(x, 12, 1) << 31))
	351	#define ENCODE_UTYPE_IMM(x) \
	352	(RV_X(x, 12, 20) << 12)
	353	#define ENCODE_JTYPE_IMM(x) \
	354	((RV_X(x, 1, 10) << 21) \| (RV_X(x, 11, 1) << 20) \| \
	355	(RV_X(x, 12, 8) << 12) \| (RV_X(x, 20, 1) << 31))
	356	#define ENCODE_CBTYPE_IMM(x) \
	357	((RV_X(x, 1, 2) << 3) \| (RV_X(x, 3, 2) << 10) \| (RV_X(x, 5, 1) << 2) \| \
	358	(RV_X(x, 6, 2) << 5) \| (RV_X(x, 8, 1) << 12))
	359	#define ENCODE_CJTYPE_IMM(x) \
	360	((RV_X(x, 1, 3) << 3) \| (RV_X(x, 4, 1) << 11) \| (RV_X(x, 5, 1) << 2) \| \
	361	(RV_X(x, 6, 1) << 7) \| (RV_X(x, 7, 1) << 6) \| (RV_X(x, 8, 2) << 9) \| \
	362	(RV_X(x, 10, 1) << 8) \| (RV_X(x, 11, 1) << 12))
	363	#define ENCODE_UJTYPE_IMM(x) \
	364	(ENCODE_UTYPE_IMM(RISCV_CONST_HIGH_PART(x)) \| \
	365	(ENCODE_ITYPE_IMM(RISCV_CONST_LOW_PART(x)) << 32))
	366	#define ENCODE_UITYPE_IMM(x) \
	367	(ENCODE_UTYPE_IMM(x) \| (ENCODE_ITYPE_IMM(x) << 32))
	368
	369	#define CLEAN_IMM(type, x) \
	370	((~ENCODE_##type##_IMM((uint64_t)(-1))) & (x))
	371
	372	int arch_kexec_apply_relocations_add(struct purgatory_info *pi,
	373	Elf_Shdr *section,
	374	const Elf_Shdr *relsec,
	375	const Elf_Shdr *symtab)
	376	{
	377	const char strtab, name, *shstrtab;
	378	const Elf_Shdr *sechdrs;
a927444a	379	Elf64_Rela *relas;
838b3e28 LZ	380	int i, r_type;
	381
	382	/* String & section header string table */
	383	sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff;
	384	strtab = (char *)pi->ehdr + sechdrs[symtab->sh_link].sh_offset;
	385	shstrtab = (char *)pi->ehdr + sechdrs[pi->ehdr->e_shstrndx].sh_offset;
	386
	387	relas = (void *)pi->ehdr + relsec->sh_offset;
	388
	389	for (i = 0; i < relsec->sh_size / sizeof(*relas); i++) {
	390	const Elf_Sym sym; / symbol to relocate */
	391	unsigned long addr; /* final location after relocation */
	392	unsigned long val; /* relocated symbol value */
	393	unsigned long sec_base; /* relocated symbol value */
	394	void loc; / tmp location to modify */
	395
	396	sym = (void *)pi->ehdr + symtab->sh_offset;
	397	sym += ELF64_R_SYM(relas[i].r_info);
	398
	399	if (sym->st_name)
	400	name = strtab + sym->st_name;
	401	else
	402	name = shstrtab + sechdrs[sym->st_shndx].sh_name;
	403
	404	loc = pi->purgatory_buf;
	405	loc += section->sh_offset;
	406	loc += relas[i].r_offset;
	407
	408	if (sym->st_shndx == SHN_ABS)
	409	sec_base = 0;
	410	else if (sym->st_shndx >= pi->ehdr->e_shnum) {
	411	pr_err("Invalid section %d for symbol %s\n",
	412	sym->st_shndx, name);
	413	return -ENOEXEC;
	414	} else
	415	sec_base = pi->sechdrs[sym->st_shndx].sh_addr;
	416
	417	val = sym->st_value;
	418	val += sec_base;
	419	val += relas[i].r_addend;
	420
	421	addr = section->sh_addr + relas[i].r_offset;
	422
	423	r_type = ELF64_R_TYPE(relas[i].r_info);
	424
	425	switch (r_type) {
	426	case R_RISCV_BRANCH:
	427	(u32 )loc = CLEAN_IMM(BTYPE, (u32 )loc) \|
	428	ENCODE_BTYPE_IMM(val - addr);
	429	break;
	430	case R_RISCV_JAL:
	431	(u32 )loc = CLEAN_IMM(JTYPE, (u32 )loc) \|
	432	ENCODE_JTYPE_IMM(val - addr);
	433	break;
	434	/*
	435	* With no R_RISCV_PCREL_LO12_S, R_RISCV_PCREL_LO12_I
	436	* sym is expected to be next to R_RISCV_PCREL_HI20
	437	* in purgatory relsec. Handle it like R_RISCV_CALL
	438	* sym, instead of searching the whole relsec.
	439	*/
	440	case R_RISCV_PCREL_HI20:
d0b4f95a	441	case R_RISCV_CALL_PLT:
838b3e28 LZ	442	case R_RISCV_CALL:
	443	(u64 )loc = CLEAN_IMM(UITYPE, (u64 )loc) \|
	444	ENCODE_UJTYPE_IMM(val - addr);
	445	break;
	446	case R_RISCV_RVC_BRANCH:
	447	(u32 )loc = CLEAN_IMM(CBTYPE, (u32 )loc) \|
	448	ENCODE_CBTYPE_IMM(val - addr);
	449	break;
	450	case R_RISCV_RVC_JUMP:
	451	(u32 )loc = CLEAN_IMM(CJTYPE, (u32 )loc) \|
	452	ENCODE_CJTYPE_IMM(val - addr);
	453	break;
	454	case R_RISCV_ADD32:
	455	(u32 )loc += val;
	456	break;
	457	case R_RISCV_SUB32:
	458	(u32 )loc -= val;
	459	break;
	460	/* It has been applied by R_RISCV_PCREL_HI20 sym */
	461	case R_RISCV_PCREL_LO12_I:
	462	case R_RISCV_ALIGN:
	463	case R_RISCV_RELAX:
	464	break;
	465	default:
	466	pr_err("Unknown rela relocation: %d\n", r_type);
	467	return -ENOEXEC;
	468	}
	469	}
	470	return 0;
	471	}
	472
6261586e LC	473	const struct kexec_file_ops elf_kexec_ops = {
	474	.probe = kexec_elf_probe,
	475	.load = elf_kexec_load,
	476	};