[linux-block.git] / arch / riscv / kernel / module.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *
 *  Copyright (C) 2017 Zihao Yu
 */

#include <linux/elf.h>
#include <linux/err.h>
#include <linux/errno.h>
#include <linux/moduleloader.h>
#include <linux/vmalloc.h>
#include <linux/sizes.h>
#include <asm/pgtable.h>
#include <asm/sections.h>

static int apply_r_riscv_32_rela(struct module *me, u32 *location, Elf_Addr v)
{
	if (v != (u32)v) {
		pr_err("%s: value %016llx out of range for 32-bit field\n",
		       me->name, (long long)v);
		return -EINVAL;
	}
	*location = v;
	return 0;
}

static int apply_r_riscv_64_rela(struct module *me, u32 *location, Elf_Addr v)
{
	*(u64 *)location = v;
	return 0;
}

static int apply_r_riscv_branch_rela(struct module *me, u32 *location,
				     Elf_Addr v)
{
	ptrdiff_t offset = (void *)v - (void *)location;
	u32 imm12 = (offset & 0x1000) << (31 - 12);
	u32 imm11 = (offset & 0x800) >> (11 - 7);
	u32 imm10_5 = (offset & 0x7e0) << (30 - 10);
	u32 imm4_1 = (offset & 0x1e) << (11 - 4);

	*location = (*location & 0x1fff07f) | imm12 | imm11 | imm10_5 | imm4_1;
	return 0;
}

static int apply_r_riscv_jal_rela(struct module *me, u32 *location,
				  Elf_Addr v)
{
	ptrdiff_t offset = (void *)v - (void *)location;
	u32 imm20 = (offset & 0x100000) << (31 - 20);
	u32 imm19_12 = (offset & 0xff000);
	u32 imm11 = (offset & 0x800) << (20 - 11);
	u32 imm10_1 = (offset & 0x7fe) << (30 - 10);

	*location = (*location & 0xfff) | imm20 | imm19_12 | imm11 | imm10_1;
	return 0;
}

static int apply_r_riscv_rcv_branch_rela(struct module *me, u32 *location,
					 Elf_Addr v)
{
	ptrdiff_t offset = (void *)v - (void *)location;
	u16 imm8 = (offset & 0x100) << (12 - 8);
	u16 imm7_6 = (offset & 0xc0) >> (6 - 5);
	u16 imm5 = (offset & 0x20) >> (5 - 2);
	u16 imm4_3 = (offset & 0x18) << (12 - 5);
	u16 imm2_1 = (offset & 0x6) << (12 - 10);

	*(u16 *)location = (*(u16 *)location & 0xe383) |
		    imm8 | imm7_6 | imm5 | imm4_3 | imm2_1;
	return 0;
}

static int apply_r_riscv_rvc_jump_rela(struct module *me, u32 *location,
				       Elf_Addr v)
{
	ptrdiff_t offset = (void *)v - (void *)location;
	u16 imm11 = (offset & 0x800) << (12 - 11);
	u16 imm10 = (offset & 0x400) >> (10 - 8);
	u16 imm9_8 = (offset & 0x300) << (12 - 11);
	u16 imm7 = (offset & 0x80) >> (7 - 6);
	u16 imm6 = (offset & 0x40) << (12 - 11);
	u16 imm5 = (offset & 0x20) >> (5 - 2);
	u16 imm4 = (offset & 0x10) << (12 - 5);
	u16 imm3_1 = (offset & 0xe) << (12 - 10);

	*(u16 *)location = (*(u16 *)location & 0xe003) |
		    imm11 | imm10 | imm9_8 | imm7 | imm6 | imm5 | imm4 | imm3_1;
	return 0;
}

static int apply_r_riscv_pcrel_hi20_rela(struct module *me, u32 *location,
					 Elf_Addr v)
{
	ptrdiff_t offset = (void *)v - (void *)location;
	s32 hi20;

	if (offset != (s32)offset) {
		pr_err(
		  "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n",
		  me->name, (long long)v, location);
		return -EINVAL;
	}

	hi20 = (offset + 0x800) & 0xfffff000;
	*location = (*location & 0xfff) | hi20;
	return 0;
}

static int apply_r_riscv_pcrel_lo12_i_rela(struct module *me, u32 *location,
					   Elf_Addr v)
{
	/*
	 * v is the lo12 value to fill. It is calculated before calling this
	 * handler.
	 */
	*location = (*location & 0xfffff) | ((v & 0xfff) << 20);
	return 0;
}

static int apply_r_riscv_pcrel_lo12_s_rela(struct module *me, u32 *location,
					   Elf_Addr v)
{
	/*
	 * v is the lo12 value to fill. It is calculated before calling this
	 * handler.
	 */
	u32 imm11_5 = (v & 0xfe0) << (31 - 11);
	u32 imm4_0 = (v & 0x1f) << (11 - 4);

	*location = (*location & 0x1fff07f) | imm11_5 | imm4_0;
	return 0;
}

static int apply_r_riscv_hi20_rela(struct module *me, u32 *location,
				   Elf_Addr v)
{
	s32 hi20;

	if (IS_ENABLED(CONFIG_CMODEL_MEDLOW)) {
		pr_err(
		  "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n",
		  me->name, (long long)v, location);
		return -EINVAL;
	}

	hi20 = ((s32)v + 0x800) & 0xfffff000;
	*location = (*location & 0xfff) | hi20;
	return 0;
}

static int apply_r_riscv_lo12_i_rela(struct module *me, u32 *location,
				     Elf_Addr v)
{
	/* Skip medlow checking because of filtering by HI20 already */
	s32 hi20 = ((s32)v + 0x800) & 0xfffff000;
	s32 lo12 = ((s32)v - hi20);
	*location = (*location & 0xfffff) | ((lo12 & 0xfff) << 20);
	return 0;
}

static int apply_r_riscv_lo12_s_rela(struct module *me, u32 *location,
				     Elf_Addr v)
{
	/* Skip medlow checking because of filtering by HI20 already */
	s32 hi20 = ((s32)v + 0x800) & 0xfffff000;
	s32 lo12 = ((s32)v - hi20);
	u32 imm11_5 = (lo12 & 0xfe0) << (31 - 11);
	u32 imm4_0 = (lo12 & 0x1f) << (11 - 4);
	*location = (*location & 0x1fff07f) | imm11_5 | imm4_0;
	return 0;
}

static int apply_r_riscv_got_hi20_rela(struct module *me, u32 *location,
				       Elf_Addr v)
{
	ptrdiff_t offset = (void *)v - (void *)location;
	s32 hi20;

	/* Always emit the got entry */
	if (IS_ENABLED(CONFIG_MODULE_SECTIONS)) {
		offset = module_emit_got_entry(me, v);
		offset = (void *)offset - (void *)location;
	} else {
		pr_err(
		  "%s: can not generate the GOT entry for symbol = %016llx from PC = %p\n",
		  me->name, (long long)v, location);
		return -EINVAL;
	}

	hi20 = (offset + 0x800) & 0xfffff000;
	*location = (*location & 0xfff) | hi20;
	return 0;
}

static int apply_r_riscv_call_plt_rela(struct module *me, u32 *location,
				       Elf_Addr v)
{
	ptrdiff_t offset = (void *)v - (void *)location;
	s32 fill_v = offset;
	u32 hi20, lo12;

	if (offset != fill_v) {
		/* Only emit the plt entry if offset over 32-bit range */
		if (IS_ENABLED(CONFIG_MODULE_SECTIONS)) {
			offset = module_emit_plt_entry(me, v);
			offset = (void *)offset - (void *)location;
		} else {
			pr_err(
			  "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n",
			  me->name, (long long)v, location);
			return -EINVAL;
		}
	}

	hi20 = (offset + 0x800) & 0xfffff000;
	lo12 = (offset - hi20) & 0xfff;
	*location = (*location & 0xfff) | hi20;
	*(location + 1) = (*(location + 1) & 0xfffff) | (lo12 << 20);
	return 0;
}

static int apply_r_riscv_call_rela(struct module *me, u32 *location,
				   Elf_Addr v)
{
	ptrdiff_t offset = (void *)v - (void *)location;
	s32 fill_v = offset;
	u32 hi20, lo12;

	if (offset != fill_v) {
		pr_err(
		  "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n",
		  me->name, (long long)v, location);
		return -EINVAL;
	}

	hi20 = (offset + 0x800) & 0xfffff000;
	lo12 = (offset - hi20) & 0xfff;
	*location = (*location & 0xfff) | hi20;
	*(location + 1) = (*(location + 1) & 0xfffff) | (lo12 << 20);
	return 0;
}

static int apply_r_riscv_relax_rela(struct module *me, u32 *location,
				    Elf_Addr v)
{
	return 0;
}

static int apply_r_riscv_align_rela(struct module *me, u32 *location,
				    Elf_Addr v)
{
	pr_err(
	  "%s: The unexpected relocation type 'R_RISCV_ALIGN' from PC = %p\n",
	  me->name, location);
	return -EINVAL;
}

static int apply_r_riscv_add32_rela(struct module *me, u32 *location,
				    Elf_Addr v)
{
	*(u32 *)location += (u32)v;
	return 0;
}

static int apply_r_riscv_sub32_rela(struct module *me, u32 *location,
				    Elf_Addr v)
{
	*(u32 *)location -= (u32)v;
	return 0;
}

static int (*reloc_handlers_rela[]) (struct module *me, u32 *location,
				Elf_Addr v) = {
	[R_RISCV_32]			= apply_r_riscv_32_rela,
	[R_RISCV_64]			= apply_r_riscv_64_rela,
	[R_RISCV_BRANCH]		= apply_r_riscv_branch_rela,
	[R_RISCV_JAL]			= apply_r_riscv_jal_rela,
	[R_RISCV_RVC_BRANCH]		= apply_r_riscv_rcv_branch_rela,
	[R_RISCV_RVC_JUMP]		= apply_r_riscv_rvc_jump_rela,
	[R_RISCV_PCREL_HI20]		= apply_r_riscv_pcrel_hi20_rela,
	[R_RISCV_PCREL_LO12_I]		= apply_r_riscv_pcrel_lo12_i_rela,
	[R_RISCV_PCREL_LO12_S]		= apply_r_riscv_pcrel_lo12_s_rela,
	[R_RISCV_HI20]			= apply_r_riscv_hi20_rela,
	[R_RISCV_LO12_I]		= apply_r_riscv_lo12_i_rela,
	[R_RISCV_LO12_S]		= apply_r_riscv_lo12_s_rela,
	[R_RISCV_GOT_HI20]		= apply_r_riscv_got_hi20_rela,
	[R_RISCV_CALL_PLT]		= apply_r_riscv_call_plt_rela,
	[R_RISCV_CALL]			= apply_r_riscv_call_rela,
	[R_RISCV_RELAX]			= apply_r_riscv_relax_rela,
	[R_RISCV_ALIGN]			= apply_r_riscv_align_rela,
	[R_RISCV_ADD32]			= apply_r_riscv_add32_rela,
	[R_RISCV_SUB32]			= apply_r_riscv_sub32_rela,
};

int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
		       unsigned int symindex, unsigned int relsec,
		       struct module *me)
{
	Elf_Rela *rel = (void *) sechdrs[relsec].sh_addr;
	int (*handler)(struct module *me, u32 *location, Elf_Addr v);
	Elf_Sym *sym;
	u32 *location;
	unsigned int i, type;
	Elf_Addr v;
	int res;

	pr_debug("Applying relocate section %u to %u\n", relsec,
	       sechdrs[relsec].sh_info);

	for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) {
		/* This is where to make the change */
		location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr
			+ rel[i].r_offset;
		/* This is the symbol it is referring to */
		sym = (Elf_Sym *)sechdrs[symindex].sh_addr
			+ ELF_RISCV_R_SYM(rel[i].r_info);
		if (IS_ERR_VALUE(sym->st_value)) {
			/* Ignore unresolved weak symbol */
			if (ELF_ST_BIND(sym->st_info) == STB_WEAK)
				continue;
			pr_warn("%s: Unknown symbol %s\n",
				me->name, strtab + sym->st_name);
			return -ENOENT;
		}

		type = ELF_RISCV_R_TYPE(rel[i].r_info);

		if (type < ARRAY_SIZE(reloc_handlers_rela))
			handler = reloc_handlers_rela[type];
		else
			handler = NULL;

		if (!handler) {
			pr_err("%s: Unknown relocation type %u\n",
			       me->name, type);
			return -EINVAL;
		}

		v = sym->st_value + rel[i].r_addend;

		if (type == R_RISCV_PCREL_LO12_I || type == R_RISCV_PCREL_LO12_S) {
			unsigned int j;

			for (j = 0; j < sechdrs[relsec].sh_size / sizeof(*rel); j++) {
				unsigned long hi20_loc =
					sechdrs[sechdrs[relsec].sh_info].sh_addr
					+ rel[j].r_offset;
				u32 hi20_type = ELF_RISCV_R_TYPE(rel[j].r_info);

				/* Find the corresponding HI20 relocation entry */
				if (hi20_loc == sym->st_value
				    && (hi20_type == R_RISCV_PCREL_HI20
					|| hi20_type == R_RISCV_GOT_HI20)) {
					s32 hi20, lo12;
					Elf_Sym *hi20_sym =
						(Elf_Sym *)sechdrs[symindex].sh_addr
						+ ELF_RISCV_R_SYM(rel[j].r_info);
					unsigned long hi20_sym_val =
						hi20_sym->st_value
						+ rel[j].r_addend;

					/* Calculate lo12 */
					size_t offset = hi20_sym_val - hi20_loc;
					if (IS_ENABLED(CONFIG_MODULE_SECTIONS)
					    && hi20_type == R_RISCV_GOT_HI20) {
						offset = module_emit_got_entry(
							 me, hi20_sym_val);
						offset = offset - hi20_loc;
					}
					hi20 = (offset + 0x800) & 0xfffff000;
					lo12 = offset - hi20;
					v = lo12;

					break;
				}
			}
			if (j == sechdrs[relsec].sh_size / sizeof(*rel)) {
				pr_err(
				  "%s: Can not find HI20 relocation information\n",
				  me->name);
				return -EINVAL;
			}
		}

		res = handler(me, location, v);
		if (res)
			return res;
	}

	return 0;
}

#if defined(CONFIG_MMU) && defined(CONFIG_64BIT)
#define VMALLOC_MODULE_START \
	 max(PFN_ALIGN((unsigned long)&_end - SZ_2G), VMALLOC_START)
void *module_alloc(unsigned long size)
{
	return __vmalloc_node_range(size, 1, VMALLOC_MODULE_START,
				    VMALLOC_END, GFP_KERNEL,
				    PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE,
				    __builtin_return_address(0));
}
#endif
Commit	Line	Data
c942fddf	1	// SPDX-License-Identifier: GPL-2.0-or-later
e2c0cdfb	2	/*
e2c0cdfb PD	3	*
	4	* Copyright (C) 2017 Zihao Yu
	5	*/
	6
	7	#include <linux/elf.h>
	8	#include <linux/err.h>
	9	#include <linux/errno.h>
	10	#include <linux/moduleloader.h>
0cff8bff VC	11	#include <linux/vmalloc.h>
	12	#include <linux/sizes.h>
	13	#include <asm/pgtable.h>
	14	#include <asm/sections.h>
e2c0cdfb	15
77aa85de AS	16	static int apply_r_riscv_32_rela(struct module me, u32 location, Elf_Addr v)
	17	{
	18	if (v != (u32)v) {
	19	pr_err("%s: value %016llx out of range for 32-bit field\n",
ef3a6140	20	me->name, (long long)v);
77aa85de AS	21	return -EINVAL;
	22	}
	23	*location = v;
	24	return 0;
	25	}
	26
e2c0cdfb PD	27	static int apply_r_riscv_64_rela(struct module me, u32 location, Elf_Addr v)
	28	{
	29	(u64 )location = v;
	30	return 0;
	31	}
	32
	33	static int apply_r_riscv_branch_rela(struct module me, u32 location,
	34	Elf_Addr v)
	35	{
7df85002	36	ptrdiff_t offset = (void )v - (void )location;
e2c0cdfb PD	37	u32 imm12 = (offset & 0x1000) << (31 - 12);
	38	u32 imm11 = (offset & 0x800) >> (11 - 7);
	39	u32 imm10_5 = (offset & 0x7e0) << (30 - 10);
	40	u32 imm4_1 = (offset & 0x1e) << (11 - 4);
	41
	42	location = (location & 0x1fff07f) \| imm12 \| imm11 \| imm10_5 \| imm4_1;
	43	return 0;
	44	}
	45
	46	static int apply_r_riscv_jal_rela(struct module me, u32 location,
	47	Elf_Addr v)
	48	{
7df85002	49	ptrdiff_t offset = (void )v - (void )location;
e2c0cdfb PD	50	u32 imm20 = (offset & 0x100000) << (31 - 20);
	51	u32 imm19_12 = (offset & 0xff000);
	52	u32 imm11 = (offset & 0x800) << (20 - 11);
	53	u32 imm10_1 = (offset & 0x7fe) << (30 - 10);
	54
	55	location = (location & 0xfff) \| imm20 \| imm19_12 \| imm11 \| imm10_1;
	56	return 0;
	57	}
	58
56ea45ae ZL	59	static int apply_r_riscv_rcv_branch_rela(struct module me, u32 location,
	60	Elf_Addr v)
	61	{
7df85002	62	ptrdiff_t offset = (void )v - (void )location;
56ea45ae ZL	63	u16 imm8 = (offset & 0x100) << (12 - 8);
	64	u16 imm7_6 = (offset & 0xc0) >> (6 - 5);
	65	u16 imm5 = (offset & 0x20) >> (5 - 2);
	66	u16 imm4_3 = (offset & 0x18) << (12 - 5);
	67	u16 imm2_1 = (offset & 0x6) << (12 - 10);
	68
	69	(u16 )location = ((u16 )location & 0xe383) \|
	70	imm8 \| imm7_6 \| imm5 \| imm4_3 \| imm2_1;
	71	return 0;
	72	}
	73
	74	static int apply_r_riscv_rvc_jump_rela(struct module me, u32 location,
	75	Elf_Addr v)
	76	{
7df85002	77	ptrdiff_t offset = (void )v - (void )location;
56ea45ae ZL	78	u16 imm11 = (offset & 0x800) << (12 - 11);
	79	u16 imm10 = (offset & 0x400) >> (10 - 8);
	80	u16 imm9_8 = (offset & 0x300) << (12 - 11);
	81	u16 imm7 = (offset & 0x80) >> (7 - 6);
	82	u16 imm6 = (offset & 0x40) << (12 - 11);
	83	u16 imm5 = (offset & 0x20) >> (5 - 2);
	84	u16 imm4 = (offset & 0x10) << (12 - 5);
	85	u16 imm3_1 = (offset & 0xe) << (12 - 10);
	86
	87	(u16 )location = ((u16 )location & 0xe003) \|
	88	imm11 \| imm10 \| imm9_8 \| imm7 \| imm6 \| imm5 \| imm4 \| imm3_1;
	89	return 0;
	90	}
	91
e2c0cdfb PD	92	static int apply_r_riscv_pcrel_hi20_rela(struct module me, u32 location,
	93	Elf_Addr v)
	94	{
7df85002	95	ptrdiff_t offset = (void )v - (void )location;
e2c0cdfb PD	96	s32 hi20;
	97
	98	if (offset != (s32)offset) {
	99	pr_err(
	100	"%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n",
ef3a6140	101	me->name, (long long)v, location);
e2c0cdfb PD	102	return -EINVAL;
	103	}
	104
	105	hi20 = (offset + 0x800) & 0xfffff000;
	106	location = (location & 0xfff) \| hi20;
	107	return 0;
	108	}
	109
	110	static int apply_r_riscv_pcrel_lo12_i_rela(struct module me, u32 location,
	111	Elf_Addr v)
	112	{
	113	/*
	114	* v is the lo12 value to fill. It is calculated before calling this
	115	* handler.
	116	*/
	117	location = (location & 0xfffff) \| ((v & 0xfff) << 20);
	118	return 0;
	119	}
	120
	121	static int apply_r_riscv_pcrel_lo12_s_rela(struct module me, u32 location,
	122	Elf_Addr v)
	123	{
	124	/*
	125	* v is the lo12 value to fill. It is calculated before calling this
	126	* handler.
	127	*/
	128	u32 imm11_5 = (v & 0xfe0) << (31 - 11);
	129	u32 imm4_0 = (v & 0x1f) << (11 - 4);
	130
	131	location = (location & 0x1fff07f) \| imm11_5 \| imm4_0;
	132	return 0;
	133	}
	134
e7456e69 ZL	135	static int apply_r_riscv_hi20_rela(struct module me, u32 location,
	136	Elf_Addr v)
	137	{
	138	s32 hi20;
	139
da4ed378	140	if (IS_ENABLED(CONFIG_CMODEL_MEDLOW)) {
e7456e69 ZL	141	pr_err(
e7456e69 ZL	142	"%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n",
ef3a6140	143	me->name, (long long)v, location);
e7456e69 ZL	144	return -EINVAL;
	145	}
	146
	147	hi20 = ((s32)v + 0x800) & 0xfffff000;
	148	location = (location & 0xfff) \| hi20;
	149	return 0;
	150	}
	151
	152	static int apply_r_riscv_lo12_i_rela(struct module me, u32 location,
	153	Elf_Addr v)
	154	{
	155	/* Skip medlow checking because of filtering by HI20 already */
	156	s32 hi20 = ((s32)v + 0x800) & 0xfffff000;
	157	s32 lo12 = ((s32)v - hi20);
	158	location = (location & 0xfffff) \| ((lo12 & 0xfff) << 20);
	159	return 0;
	160	}
	161
	162	static int apply_r_riscv_lo12_s_rela(struct module me, u32 location,
	163	Elf_Addr v)
	164	{
	165	/* Skip medlow checking because of filtering by HI20 already */
	166	s32 hi20 = ((s32)v + 0x800) & 0xfffff000;
	167	s32 lo12 = ((s32)v - hi20);
	168	u32 imm11_5 = (lo12 & 0xfe0) << (31 - 11);
	169	u32 imm4_0 = (lo12 & 0x1f) << (11 - 4);
	170	location = (location & 0x1fff07f) \| imm11_5 \| imm4_0;
	171	return 0;
	172	}
	173
da975dd4 ZL	174	static int apply_r_riscv_got_hi20_rela(struct module me, u32 location,
	175	Elf_Addr v)
	176	{
7df85002	177	ptrdiff_t offset = (void )v - (void )location;
da975dd4 ZL	178	s32 hi20;
	179
	180	/* Always emit the got entry */
	181	if (IS_ENABLED(CONFIG_MODULE_SECTIONS)) {
	182	offset = module_emit_got_entry(me, v);
	183	offset = (void )offset - (void )location;
	184	} else {
	185	pr_err(
	186	"%s: can not generate the GOT entry for symbol = %016llx from PC = %p\n",
ef3a6140	187	me->name, (long long)v, location);
da975dd4 ZL	188	return -EINVAL;
	189	}
	190
	191	hi20 = (offset + 0x800) & 0xfffff000;
	192	location = (location & 0xfff) \| hi20;
	193	return 0;
	194	}
	195
e2c0cdfb PD	196	static int apply_r_riscv_call_plt_rela(struct module me, u32 location,
	197	Elf_Addr v)
	198	{
7df85002	199	ptrdiff_t offset = (void )v - (void )location;
e2c0cdfb PD	200	s32 fill_v = offset;
	201	u32 hi20, lo12;
	202
	203	if (offset != fill_v) {
da975dd4 ZL	204	/* Only emit the plt entry if offset over 32-bit range */
	205	if (IS_ENABLED(CONFIG_MODULE_SECTIONS)) {
	206	offset = module_emit_plt_entry(me, v);
	207	offset = (void )offset - (void )location;
	208	} else {
	209	pr_err(
	210	"%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n",
ef3a6140	211	me->name, (long long)v, location);
da975dd4 ZL	212	return -EINVAL;
da975dd4 ZL	213	}
e2c0cdfb PD	214	}
	215
	216	hi20 = (offset + 0x800) & 0xfffff000;
	217	lo12 = (offset - hi20) & 0xfff;
	218	location = (location & 0xfff) \| hi20;
	219	(location + 1) = ((location + 1) & 0xfffff) \| (lo12 << 20);
	220	return 0;
	221	}
	222
e1910c72 ZL	223	static int apply_r_riscv_call_rela(struct module me, u32 location,
	224	Elf_Addr v)
	225	{
7df85002	226	ptrdiff_t offset = (void )v - (void )location;
e1910c72 ZL	227	s32 fill_v = offset;
	228	u32 hi20, lo12;
	229
	230	if (offset != fill_v) {
	231	pr_err(
	232	"%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n",
ef3a6140	233	me->name, (long long)v, location);
e1910c72 ZL	234	return -EINVAL;
	235	}
	236
	237	hi20 = (offset + 0x800) & 0xfffff000;
	238	lo12 = (offset - hi20) & 0xfff;
	239	location = (location & 0xfff) \| hi20;
	240	(location + 1) = ((location + 1) & 0xfffff) \| (lo12 << 20);
	241	return 0;
	242	}
	243
e2c0cdfb PD	244	static int apply_r_riscv_relax_rela(struct module me, u32 location,
	245	Elf_Addr v)
	246	{
	247	return 0;
	248	}
	249
29e405cd ZL	250	static int apply_r_riscv_align_rela(struct module me, u32 location,
	251	Elf_Addr v)
	252	{
	253	pr_err(
	254	"%s: The unexpected relocation type 'R_RISCV_ALIGN' from PC = %p\n",
	255	me->name, location);
	256	return -EINVAL;
	257	}
	258
8e691b16 ZL	259	static int apply_r_riscv_add32_rela(struct module me, u32 location,
	260	Elf_Addr v)
	261	{
781c8fe2	262	(u32 )location += (u32)v;
8e691b16 ZL	263	return 0;
	264	}
	265
4aad074c ZL	266	static int apply_r_riscv_sub32_rela(struct module me, u32 location,
	267	Elf_Addr v)
	268	{
781c8fe2	269	(u32 )location -= (u32)v;
4aad074c ZL	270	return 0;
	271	}
	272
e2c0cdfb PD	273	static int (reloc_handlers_rela[]) (struct module me, u32 *location,
e2c0cdfb PD	274	Elf_Addr v) = {
77aa85de	275	[R_RISCV_32] = apply_r_riscv_32_rela,
e2c0cdfb PD	276	[R_RISCV_64] = apply_r_riscv_64_rela,
	277	[R_RISCV_BRANCH] = apply_r_riscv_branch_rela,
	278	[R_RISCV_JAL] = apply_r_riscv_jal_rela,
56ea45ae ZL	279	[R_RISCV_RVC_BRANCH] = apply_r_riscv_rcv_branch_rela,
56ea45ae ZL	280	[R_RISCV_RVC_JUMP] = apply_r_riscv_rvc_jump_rela,
e2c0cdfb PD	281	[R_RISCV_PCREL_HI20] = apply_r_riscv_pcrel_hi20_rela,
	282	[R_RISCV_PCREL_LO12_I] = apply_r_riscv_pcrel_lo12_i_rela,
	283	[R_RISCV_PCREL_LO12_S] = apply_r_riscv_pcrel_lo12_s_rela,
e7456e69 ZL	284	[R_RISCV_HI20] = apply_r_riscv_hi20_rela,
	285	[R_RISCV_LO12_I] = apply_r_riscv_lo12_i_rela,
	286	[R_RISCV_LO12_S] = apply_r_riscv_lo12_s_rela,
da975dd4	287	[R_RISCV_GOT_HI20] = apply_r_riscv_got_hi20_rela,
e2c0cdfb	288	[R_RISCV_CALL_PLT] = apply_r_riscv_call_plt_rela,
e1910c72	289	[R_RISCV_CALL] = apply_r_riscv_call_rela,
e2c0cdfb	290	[R_RISCV_RELAX] = apply_r_riscv_relax_rela,
29e405cd	291	[R_RISCV_ALIGN] = apply_r_riscv_align_rela,
8e691b16	292	[R_RISCV_ADD32] = apply_r_riscv_add32_rela,
4aad074c	293	[R_RISCV_SUB32] = apply_r_riscv_sub32_rela,
e2c0cdfb PD	294	};
	295
	296	int apply_relocate_add(Elf_Shdr sechdrs, const char strtab,
	297	unsigned int symindex, unsigned int relsec,
	298	struct module *me)
	299	{
	300	Elf_Rela rel = (void ) sechdrs[relsec].sh_addr;
	301	int (handler)(struct module me, u32 *location, Elf_Addr v);
	302	Elf_Sym *sym;
	303	u32 *location;
	304	unsigned int i, type;
	305	Elf_Addr v;
	306	int res;
	307
	308	pr_debug("Applying relocate section %u to %u\n", relsec,
	309	sechdrs[relsec].sh_info);
	310
	311	for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) {
	312	/* This is where to make the change */
	313	location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr
	314	+ rel[i].r_offset;
	315	/* This is the symbol it is referring to */
	316	sym = (Elf_Sym *)sechdrs[symindex].sh_addr
	317	+ ELF_RISCV_R_SYM(rel[i].r_info);
	318	if (IS_ERR_VALUE(sym->st_value)) {
	319	/* Ignore unresolved weak symbol */
	320	if (ELF_ST_BIND(sym->st_info) == STB_WEAK)
	321	continue;
04ce8d3f KW	322	pr_warn("%s: Unknown symbol %s\n",
04ce8d3f KW	323	me->name, strtab + sym->st_name);
e2c0cdfb PD	324	return -ENOENT;
	325	}
	326
	327	type = ELF_RISCV_R_TYPE(rel[i].r_info);
	328
	329	if (type < ARRAY_SIZE(reloc_handlers_rela))
	330	handler = reloc_handlers_rela[type];
	331	else
	332	handler = NULL;
	333
	334	if (!handler) {
	335	pr_err("%s: Unknown relocation type %u\n",
	336	me->name, type);
	337	return -EINVAL;
	338	}
	339
	340	v = sym->st_value + rel[i].r_addend;
	341
	342	if (type == R_RISCV_PCREL_LO12_I \|\| type == R_RISCV_PCREL_LO12_S) {
	343	unsigned int j;
	344
	345	for (j = 0; j < sechdrs[relsec].sh_size / sizeof(*rel); j++) {
7df85002	346	unsigned long hi20_loc =
e2c0cdfb PD	347	sechdrs[sechdrs[relsec].sh_info].sh_addr
e2c0cdfb PD	348	+ rel[j].r_offset;
da975dd4 ZL	349	u32 hi20_type = ELF_RISCV_R_TYPE(rel[j].r_info);
	350
	351	/* Find the corresponding HI20 relocation entry */
	352	if (hi20_loc == sym->st_value
	353	&& (hi20_type == R_RISCV_PCREL_HI20
	354	\|\| hi20_type == R_RISCV_GOT_HI20)) {
	355	s32 hi20, lo12;
e2c0cdfb PD	356	Elf_Sym *hi20_sym =
	357	(Elf_Sym *)sechdrs[symindex].sh_addr
	358	+ ELF_RISCV_R_SYM(rel[j].r_info);
7df85002	359	unsigned long hi20_sym_val =
e2c0cdfb PD	360	hi20_sym->st_value
e2c0cdfb PD	361	+ rel[j].r_addend;
da975dd4	362
e2c0cdfb	363	/* Calculate lo12 */
7df85002	364	size_t offset = hi20_sym_val - hi20_loc;
da975dd4 ZL	365	if (IS_ENABLED(CONFIG_MODULE_SECTIONS)
	366	&& hi20_type == R_RISCV_GOT_HI20) {
	367	offset = module_emit_got_entry(
	368	me, hi20_sym_val);
	369	offset = offset - hi20_loc;
	370	}
	371	hi20 = (offset + 0x800) & 0xfffff000;
	372	lo12 = offset - hi20;
e2c0cdfb	373	v = lo12;
da975dd4	374
e2c0cdfb PD	375	break;
	376	}
	377	}
	378	if (j == sechdrs[relsec].sh_size / sizeof(*rel)) {
	379	pr_err(
da975dd4	380	"%s: Can not find HI20 relocation information\n",
e2c0cdfb PD	381	me->name);
	382	return -EINVAL;
	383	}
	384	}
	385
	386	res = handler(me, location, v);
	387	if (res)
	388	return res;
	389	}
	390
	391	return 0;
	392	}
0cff8bff VC	393
	394	#if defined(CONFIG_MMU) && defined(CONFIG_64BIT)
	395	#define VMALLOC_MODULE_START \
	396	max(PFN_ALIGN((unsigned long)&_end - SZ_2G), VMALLOC_START)
	397	void *module_alloc(unsigned long size)
	398	{
	399	return __vmalloc_node_range(size, 1, VMALLOC_MODULE_START,
	400	VMALLOC_END, GFP_KERNEL,
	401	PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE,
	402	__builtin_return_address(0));
	403	}
	404	#endif