[linux-block.git] / arch / s390 / kernel / module.c

// SPDX-License-Identifier: GPL-2.0+
/*
 *  Kernel module help for s390.
 *
 *  S390 version
 *    Copyright IBM Corp. 2002, 2003
 *    Author(s): Arnd Bergmann (arndb@de.ibm.com)
 *		 Martin Schwidefsky (schwidefsky@de.ibm.com)
 *
 *  based on i386 version
 *    Copyright (C) 2001 Rusty Russell.
 */
#include <linux/module.h>
#include <linux/elf.h>
#include <linux/vmalloc.h>
#include <linux/fs.h>
#include <linux/ftrace.h>
#include <linux/string.h>
#include <linux/kernel.h>
#include <linux/kasan.h>
#include <linux/moduleloader.h>
#include <linux/bug.h>
#include <linux/memory.h>
#include <asm/alternative.h>
#include <asm/nospec-branch.h>
#include <asm/facility.h>
#include <asm/ftrace.lds.h>
#include <asm/set_memory.h>
#include <asm/setup.h>

#if 0
#define DEBUGP printk
#else
#define DEBUGP(fmt , ...)
#endif

#define PLT_ENTRY_SIZE 22

static unsigned long get_module_load_offset(void)
{
	static DEFINE_MUTEX(module_kaslr_mutex);
	static unsigned long module_load_offset;

	if (!kaslr_enabled())
		return 0;
	/*
	 * Calculate the module_load_offset the first time this code
	 * is called. Once calculated it stays the same until reboot.
	 */
	mutex_lock(&module_kaslr_mutex);
	if (!module_load_offset)
		module_load_offset = get_random_u32_inclusive(1, 1024) * PAGE_SIZE;
	mutex_unlock(&module_kaslr_mutex);
	return module_load_offset;
}

void *module_alloc(unsigned long size)
{
	gfp_t gfp_mask = GFP_KERNEL;
	void *p;

	if (PAGE_ALIGN(size) > MODULES_LEN)
		return NULL;
	p = __vmalloc_node_range(size, MODULE_ALIGN,
				 MODULES_VADDR + get_module_load_offset(),
				 MODULES_END, gfp_mask, PAGE_KERNEL,
				 VM_FLUSH_RESET_PERMS | VM_DEFER_KMEMLEAK,
				 NUMA_NO_NODE, __builtin_return_address(0));
	if (p && (kasan_alloc_module_shadow(p, size, gfp_mask) < 0)) {
		vfree(p);
		return NULL;
	}
	return p;
}

#ifdef CONFIG_FUNCTION_TRACER
void module_arch_cleanup(struct module *mod)
{
	module_memfree(mod->arch.trampolines_start);
}
#endif

void module_arch_freeing_init(struct module *mod)
{
	if (is_livepatch_module(mod) &&
	    mod->state == MODULE_STATE_LIVE)
		return;

	vfree(mod->arch.syminfo);
	mod->arch.syminfo = NULL;
}

static void check_rela(Elf_Rela *rela, struct module *me)
{
	struct mod_arch_syminfo *info;

	info = me->arch.syminfo + ELF_R_SYM (rela->r_info);
	switch (ELF_R_TYPE (rela->r_info)) {
	case R_390_GOT12:	/* 12 bit GOT offset.  */
	case R_390_GOT16:	/* 16 bit GOT offset.  */
	case R_390_GOT20:	/* 20 bit GOT offset.  */
	case R_390_GOT32:	/* 32 bit GOT offset.  */
	case R_390_GOT64:	/* 64 bit GOT offset.  */
	case R_390_GOTENT:	/* 32 bit PC rel. to GOT entry shifted by 1. */
	case R_390_GOTPLT12:	/* 12 bit offset to jump slot.	*/
	case R_390_GOTPLT16:	/* 16 bit offset to jump slot.  */
	case R_390_GOTPLT20:	/* 20 bit offset to jump slot.  */
	case R_390_GOTPLT32:	/* 32 bit offset to jump slot.  */
	case R_390_GOTPLT64:	/* 64 bit offset to jump slot.	*/
	case R_390_GOTPLTENT:	/* 32 bit rel. offset to jump slot >> 1. */
		if (info->got_offset == -1UL) {
			info->got_offset = me->arch.got_size;
			me->arch.got_size += sizeof(void*);
		}
		break;
	case R_390_PLT16DBL:	/* 16 bit PC rel. PLT shifted by 1.  */
	case R_390_PLT32DBL:	/* 32 bit PC rel. PLT shifted by 1.  */
	case R_390_PLT32:	/* 32 bit PC relative PLT address.  */
	case R_390_PLT64:	/* 64 bit PC relative PLT address.  */
	case R_390_PLTOFF16:	/* 16 bit offset from GOT to PLT. */
	case R_390_PLTOFF32:	/* 32 bit offset from GOT to PLT. */
	case R_390_PLTOFF64:	/* 16 bit offset from GOT to PLT. */
		if (info->plt_offset == -1UL) {
			info->plt_offset = me->arch.plt_size;
			me->arch.plt_size += PLT_ENTRY_SIZE;
		}
		break;
	case R_390_COPY:
	case R_390_GLOB_DAT:
	case R_390_JMP_SLOT:
	case R_390_RELATIVE:
		/* Only needed if we want to support loading of 
		   modules linked with -shared. */
		break;
	}
}

/*
 * Account for GOT and PLT relocations. We can't add sections for
 * got and plt but we can increase the core module size.
 */
int module_frob_arch_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs,
			      char *secstrings, struct module *me)
{
	Elf_Shdr *symtab;
	Elf_Sym *symbols;
	Elf_Rela *rela;
	char *strings;
	int nrela, i, j;
	struct module_memory *mod_mem;

	/* Find symbol table and string table. */
	symtab = NULL;
	for (i = 0; i < hdr->e_shnum; i++)
		switch (sechdrs[i].sh_type) {
		case SHT_SYMTAB:
			symtab = sechdrs + i;
			break;
		}
	if (!symtab) {
		printk(KERN_ERR "module %s: no symbol table\n", me->name);
		return -ENOEXEC;
	}

	/* Allocate one syminfo structure per symbol. */
	me->arch.nsyms = symtab->sh_size / sizeof(Elf_Sym);
	me->arch.syminfo = vmalloc(array_size(sizeof(struct mod_arch_syminfo),
					      me->arch.nsyms));
	if (!me->arch.syminfo)
		return -ENOMEM;
	symbols = (void *) hdr + symtab->sh_offset;
	strings = (void *) hdr + sechdrs[symtab->sh_link].sh_offset;
	for (i = 0; i < me->arch.nsyms; i++) {
		if (symbols[i].st_shndx == SHN_UNDEF &&
		    strcmp(strings + symbols[i].st_name,
			   "_GLOBAL_OFFSET_TABLE_") == 0)
			/* "Define" it as absolute. */
			symbols[i].st_shndx = SHN_ABS;
		me->arch.syminfo[i].got_offset = -1UL;
		me->arch.syminfo[i].plt_offset = -1UL;
		me->arch.syminfo[i].got_initialized = 0;
		me->arch.syminfo[i].plt_initialized = 0;
	}

	/* Search for got/plt relocations. */
	me->arch.got_size = me->arch.plt_size = 0;
	for (i = 0; i < hdr->e_shnum; i++) {
		if (sechdrs[i].sh_type != SHT_RELA)
			continue;
		nrela = sechdrs[i].sh_size / sizeof(Elf_Rela);
		rela = (void *) hdr + sechdrs[i].sh_offset;
		for (j = 0; j < nrela; j++)
			check_rela(rela + j, me);
	}

	/* Increase core size by size of got & plt and set start
	   offsets for got and plt. */
	mod_mem = &me->mem[MOD_TEXT];
	mod_mem->size = ALIGN(mod_mem->size, 4);
	me->arch.got_offset = mod_mem->size;
	mod_mem->size += me->arch.got_size;
	me->arch.plt_offset = mod_mem->size;
	if (me->arch.plt_size) {
		if (IS_ENABLED(CONFIG_EXPOLINE) && !nospec_disable)
			me->arch.plt_size += PLT_ENTRY_SIZE;
		mod_mem->size += me->arch.plt_size;
	}
	return 0;
}

static int apply_rela_bits(Elf_Addr loc, Elf_Addr val,
			   int sign, int bits, int shift,
			   void *(*write)(void *dest, const void *src, size_t len))
{
	unsigned long umax;
	long min, max;
	void *dest = (void *)loc;

	if (val & ((1UL << shift) - 1))
		return -ENOEXEC;
	if (sign) {
		val = (Elf_Addr)(((long) val) >> shift);
		min = -(1L << (bits - 1));
		max = (1L << (bits - 1)) - 1;
		if ((long) val < min || (long) val > max)
			return -ENOEXEC;
	} else {
		val >>= shift;
		umax = ((1UL << (bits - 1)) << 1) - 1;
		if ((unsigned long) val > umax)
			return -ENOEXEC;
	}

	if (bits == 8) {
		unsigned char tmp = val;
		write(dest, &tmp, 1);
	} else if (bits == 12) {
		unsigned short tmp = (val & 0xfff) |
			(*(unsigned short *) loc & 0xf000);
		write(dest, &tmp, 2);
	} else if (bits == 16) {
		unsigned short tmp = val;
		write(dest, &tmp, 2);
	} else if (bits == 20) {
		unsigned int tmp = (val & 0xfff) << 16 |
			(val & 0xff000) >> 4 | (*(unsigned int *) loc & 0xf00000ff);
		write(dest, &tmp, 4);
	} else if (bits == 32) {
		unsigned int tmp = val;
		write(dest, &tmp, 4);
	} else if (bits == 64) {
		unsigned long tmp = val;
		write(dest, &tmp, 8);
	}
	return 0;
}

static int apply_rela(Elf_Rela *rela, Elf_Addr base, Elf_Sym *symtab,
		      const char *strtab, struct module *me,
		      void *(*write)(void *dest, const void *src, size_t len))
{
	struct mod_arch_syminfo *info;
	Elf_Addr loc, val;
	int r_type, r_sym;
	int rc = -ENOEXEC;

	/* This is where to make the change */
	loc = base + rela->r_offset;
	/* This is the symbol it is referring to.  Note that all
	   undefined symbols have been resolved.  */
	r_sym = ELF_R_SYM(rela->r_info);
	r_type = ELF_R_TYPE(rela->r_info);
	info = me->arch.syminfo + r_sym;
	val = symtab[r_sym].st_value;

	switch (r_type) {
	case R_390_NONE:	/* No relocation.  */
		rc = 0;
		break;
	case R_390_8:		/* Direct 8 bit.   */
	case R_390_12:		/* Direct 12 bit.  */
	case R_390_16:		/* Direct 16 bit.  */
	case R_390_20:		/* Direct 20 bit.  */
	case R_390_32:		/* Direct 32 bit.  */
	case R_390_64:		/* Direct 64 bit.  */
		val += rela->r_addend;
		if (r_type == R_390_8)
			rc = apply_rela_bits(loc, val, 0, 8, 0, write);
		else if (r_type == R_390_12)
			rc = apply_rela_bits(loc, val, 0, 12, 0, write);
		else if (r_type == R_390_16)
			rc = apply_rela_bits(loc, val, 0, 16, 0, write);
		else if (r_type == R_390_20)
			rc = apply_rela_bits(loc, val, 1, 20, 0, write);
		else if (r_type == R_390_32)
			rc = apply_rela_bits(loc, val, 0, 32, 0, write);
		else if (r_type == R_390_64)
			rc = apply_rela_bits(loc, val, 0, 64, 0, write);
		break;
	case R_390_PC16:	/* PC relative 16 bit.  */
	case R_390_PC16DBL:	/* PC relative 16 bit shifted by 1.  */
	case R_390_PC32DBL:	/* PC relative 32 bit shifted by 1.  */
	case R_390_PC32:	/* PC relative 32 bit.  */
	case R_390_PC64:	/* PC relative 64 bit.	*/
		val += rela->r_addend - loc;
		if (r_type == R_390_PC16)
			rc = apply_rela_bits(loc, val, 1, 16, 0, write);
		else if (r_type == R_390_PC16DBL)
			rc = apply_rela_bits(loc, val, 1, 16, 1, write);
		else if (r_type == R_390_PC32DBL)
			rc = apply_rela_bits(loc, val, 1, 32, 1, write);
		else if (r_type == R_390_PC32)
			rc = apply_rela_bits(loc, val, 1, 32, 0, write);
		else if (r_type == R_390_PC64)
			rc = apply_rela_bits(loc, val, 1, 64, 0, write);
		break;
	case R_390_GOT12:	/* 12 bit GOT offset.  */
	case R_390_GOT16:	/* 16 bit GOT offset.  */
	case R_390_GOT20:	/* 20 bit GOT offset.  */
	case R_390_GOT32:	/* 32 bit GOT offset.  */
	case R_390_GOT64:	/* 64 bit GOT offset.  */
	case R_390_GOTENT:	/* 32 bit PC rel. to GOT entry shifted by 1. */
	case R_390_GOTPLT12:	/* 12 bit offset to jump slot.	*/
	case R_390_GOTPLT20:	/* 20 bit offset to jump slot.  */
	case R_390_GOTPLT16:	/* 16 bit offset to jump slot.  */
	case R_390_GOTPLT32:	/* 32 bit offset to jump slot.  */
	case R_390_GOTPLT64:	/* 64 bit offset to jump slot.	*/
	case R_390_GOTPLTENT:	/* 32 bit rel. offset to jump slot >> 1. */
		if (info->got_initialized == 0) {
			Elf_Addr *gotent = me->mem[MOD_TEXT].base +
					   me->arch.got_offset +
					   info->got_offset;

			write(gotent, &val, sizeof(*gotent));
			info->got_initialized = 1;
		}
		val = info->got_offset + rela->r_addend;
		if (r_type == R_390_GOT12 ||
		    r_type == R_390_GOTPLT12)
			rc = apply_rela_bits(loc, val, 0, 12, 0, write);
		else if (r_type == R_390_GOT16 ||
			 r_type == R_390_GOTPLT16)
			rc = apply_rela_bits(loc, val, 0, 16, 0, write);
		else if (r_type == R_390_GOT20 ||
			 r_type == R_390_GOTPLT20)
			rc = apply_rela_bits(loc, val, 1, 20, 0, write);
		else if (r_type == R_390_GOT32 ||
			 r_type == R_390_GOTPLT32)
			rc = apply_rela_bits(loc, val, 0, 32, 0, write);
		else if (r_type == R_390_GOT64 ||
			 r_type == R_390_GOTPLT64)
			rc = apply_rela_bits(loc, val, 0, 64, 0, write);
		else if (r_type == R_390_GOTENT ||
			 r_type == R_390_GOTPLTENT) {
			val += (Elf_Addr)me->mem[MOD_TEXT].base +
				me->arch.got_offset - loc;
			rc = apply_rela_bits(loc, val, 1, 32, 1, write);
		}
		break;
	case R_390_PLT16DBL:	/* 16 bit PC rel. PLT shifted by 1.  */
	case R_390_PLT32DBL:	/* 32 bit PC rel. PLT shifted by 1.  */
	case R_390_PLT32:	/* 32 bit PC relative PLT address.  */
	case R_390_PLT64:	/* 64 bit PC relative PLT address.  */
	case R_390_PLTOFF16:	/* 16 bit offset from GOT to PLT. */
	case R_390_PLTOFF32:	/* 32 bit offset from GOT to PLT. */
	case R_390_PLTOFF64:	/* 16 bit offset from GOT to PLT. */
		if (info->plt_initialized == 0) {
			unsigned char insn[PLT_ENTRY_SIZE];
			char *plt_base;
			char *ip;

			plt_base = me->mem[MOD_TEXT].base + me->arch.plt_offset;
			ip = plt_base + info->plt_offset;
			*(int *)insn = 0x0d10e310;	/* basr 1,0  */
			*(int *)&insn[4] = 0x100c0004;	/* lg	1,12(1) */
			if (IS_ENABLED(CONFIG_EXPOLINE) && !nospec_disable) {
				char *jump_r1;

				jump_r1 = plt_base + me->arch.plt_size -
					PLT_ENTRY_SIZE;
				/* brcl	0xf,__jump_r1 */
				*(short *)&insn[8] = 0xc0f4;
				*(int *)&insn[10] = (jump_r1 - (ip + 8)) / 2;
			} else {
				*(int *)&insn[8] = 0x07f10000;	/* br %r1 */
			}
			*(long *)&insn[14] = val;

			write(ip, insn, sizeof(insn));
			info->plt_initialized = 1;
		}
		if (r_type == R_390_PLTOFF16 ||
		    r_type == R_390_PLTOFF32 ||
		    r_type == R_390_PLTOFF64)
			val = me->arch.plt_offset - me->arch.got_offset +
				info->plt_offset + rela->r_addend;
		else {
			if (!((r_type == R_390_PLT16DBL &&
			       val - loc + 0xffffUL < 0x1ffffeUL) ||
			      (r_type == R_390_PLT32DBL &&
			       val - loc + 0xffffffffULL < 0x1fffffffeULL)))
				val = (Elf_Addr) me->mem[MOD_TEXT].base +
					me->arch.plt_offset +
					info->plt_offset;
			val += rela->r_addend - loc;
		}
		if (r_type == R_390_PLT16DBL)
			rc = apply_rela_bits(loc, val, 1, 16, 1, write);
		else if (r_type == R_390_PLTOFF16)
			rc = apply_rela_bits(loc, val, 0, 16, 0, write);
		else if (r_type == R_390_PLT32DBL)
			rc = apply_rela_bits(loc, val, 1, 32, 1, write);
		else if (r_type == R_390_PLT32 ||
			 r_type == R_390_PLTOFF32)
			rc = apply_rela_bits(loc, val, 0, 32, 0, write);
		else if (r_type == R_390_PLT64 ||
			 r_type == R_390_PLTOFF64)
			rc = apply_rela_bits(loc, val, 0, 64, 0, write);
		break;
	case R_390_GOTOFF16:	/* 16 bit offset to GOT.  */
	case R_390_GOTOFF32:	/* 32 bit offset to GOT.  */
	case R_390_GOTOFF64:	/* 64 bit offset to GOT. */
		val = val + rela->r_addend -
			((Elf_Addr) me->mem[MOD_TEXT].base + me->arch.got_offset);
		if (r_type == R_390_GOTOFF16)
			rc = apply_rela_bits(loc, val, 0, 16, 0, write);
		else if (r_type == R_390_GOTOFF32)
			rc = apply_rela_bits(loc, val, 0, 32, 0, write);
		else if (r_type == R_390_GOTOFF64)
			rc = apply_rela_bits(loc, val, 0, 64, 0, write);
		break;
	case R_390_GOTPC:	/* 32 bit PC relative offset to GOT. */
	case R_390_GOTPCDBL:	/* 32 bit PC rel. off. to GOT shifted by 1. */
		val = (Elf_Addr) me->mem[MOD_TEXT].base + me->arch.got_offset +
			rela->r_addend - loc;
		if (r_type == R_390_GOTPC)
			rc = apply_rela_bits(loc, val, 1, 32, 0, write);
		else if (r_type == R_390_GOTPCDBL)
			rc = apply_rela_bits(loc, val, 1, 32, 1, write);
		break;
	case R_390_COPY:
	case R_390_GLOB_DAT:	/* Create GOT entry.  */
	case R_390_JMP_SLOT:	/* Create PLT entry.  */
	case R_390_RELATIVE:	/* Adjust by program base.  */
		/* Only needed if we want to support loading of 
		   modules linked with -shared. */
		return -ENOEXEC;
	default:
		printk(KERN_ERR "module %s: unknown relocation: %u\n",
		       me->name, r_type);
		return -ENOEXEC;
	}
	if (rc) {
		printk(KERN_ERR "module %s: relocation error for symbol %s "
		       "(r_type %i, value 0x%lx)\n",
		       me->name, strtab + symtab[r_sym].st_name,
		       r_type, (unsigned long) val);
		return rc;
	}
	return 0;
}

static int __apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
		       unsigned int symindex, unsigned int relsec,
		       struct module *me,
		       void *(*write)(void *dest, const void *src, size_t len))
{
	Elf_Addr base;
	Elf_Sym *symtab;
	Elf_Rela *rela;
	unsigned long i, n;
	int rc;

	DEBUGP("Applying relocate section %u to %u\n",
	       relsec, sechdrs[relsec].sh_info);
	base = sechdrs[sechdrs[relsec].sh_info].sh_addr;
	symtab = (Elf_Sym *) sechdrs[symindex].sh_addr;
	rela = (Elf_Rela *) sechdrs[relsec].sh_addr;
	n = sechdrs[relsec].sh_size / sizeof(Elf_Rela);

	for (i = 0; i < n; i++, rela++) {
		rc = apply_rela(rela, base, symtab, strtab, me, write);
		if (rc)
			return rc;
	}
	return 0;
}

int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
		       unsigned int symindex, unsigned int relsec,
		       struct module *me)
{
	bool early = me->state == MODULE_STATE_UNFORMED;
	void *(*write)(void *, const void *, size_t) = memcpy;

	if (!early)
		write = s390_kernel_write;

	return __apply_relocate_add(sechdrs, strtab, symindex, relsec, me,
				    write);
}

#ifdef CONFIG_FUNCTION_TRACER
static int module_alloc_ftrace_hotpatch_trampolines(struct module *me,
						    const Elf_Shdr *s)
{
	char *start, *end;
	int numpages;
	size_t size;

	size = FTRACE_HOTPATCH_TRAMPOLINES_SIZE(s->sh_size);
	numpages = DIV_ROUND_UP(size, PAGE_SIZE);
	start = module_alloc(numpages * PAGE_SIZE);
	if (!start)
		return -ENOMEM;
	set_memory_rox((unsigned long)start, numpages);
	end = start + size;

	me->arch.trampolines_start = (struct ftrace_hotpatch_trampoline *)start;
	me->arch.trampolines_end = (struct ftrace_hotpatch_trampoline *)end;
	me->arch.next_trampoline = me->arch.trampolines_start;

	return 0;
}
#endif /* CONFIG_FUNCTION_TRACER */

int module_finalize(const Elf_Ehdr *hdr,
		    const Elf_Shdr *sechdrs,
		    struct module *me)
{
	const Elf_Shdr *s;
	char *secstrings, *secname;
	void *aseg;
#ifdef CONFIG_FUNCTION_TRACER
	int ret;
#endif

	if (IS_ENABLED(CONFIG_EXPOLINE) &&
	    !nospec_disable && me->arch.plt_size) {
		unsigned int *ij;

		ij = me->mem[MOD_TEXT].base + me->arch.plt_offset +
			me->arch.plt_size - PLT_ENTRY_SIZE;
		ij[0] = 0xc6000000;	/* exrl	%r0,.+10	*/
		ij[1] = 0x0005a7f4;	/* j	.		*/
		ij[2] = 0x000007f1;	/* br	%r1		*/
	}

	secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
	for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
		aseg = (void *) s->sh_addr;
		secname = secstrings + s->sh_name;

		if (!strcmp(".altinstructions", secname))
			/* patch .altinstructions */
			apply_alternatives(aseg, aseg + s->sh_size);

		if (IS_ENABLED(CONFIG_EXPOLINE) &&
		    (str_has_prefix(secname, ".s390_indirect")))
			nospec_revert(aseg, aseg + s->sh_size);

		if (IS_ENABLED(CONFIG_EXPOLINE) &&
		    (str_has_prefix(secname, ".s390_return")))
			nospec_revert(aseg, aseg + s->sh_size);

#ifdef CONFIG_FUNCTION_TRACER
		if (!strcmp(FTRACE_CALLSITE_SECTION, secname)) {
			ret = module_alloc_ftrace_hotpatch_trampolines(me, s);
			if (ret < 0)
				return ret;
		}
#endif /* CONFIG_FUNCTION_TRACER */
	}

	return 0;
}
Commit	Line	Data
a17ae4c3	1	// SPDX-License-Identifier: GPL-2.0+
1da177e4	2	/*
a53c8fab	3	* Kernel module help for s390.
1da177e4 LT	4	*
1da177e4 LT	5	* S390 version
a53c8fab	6	* Copyright IBM Corp. 2002, 2003
1da177e4 LT	7	* Author(s): Arnd Bergmann (arndb@de.ibm.com)
	8	* Martin Schwidefsky (schwidefsky@de.ibm.com)
	9	*
	10	* based on i386 version
	11	* Copyright (C) 2001 Rusty Russell.
1da177e4 LT	12	*/
	13	#include <linux/module.h>
	14	#include <linux/elf.h>
	15	#include <linux/vmalloc.h>
	16	#include <linux/fs.h>
de5012b4	17	#include <linux/ftrace.h>
1da177e4 LT	18	#include <linux/string.h>
1da177e4 LT	19	#include <linux/kernel.h>
793213a8	20	#include <linux/kasan.h>
2b67fc46	21	#include <linux/moduleloader.h>
c0007f1a	22	#include <linux/bug.h>
be242261	23	#include <linux/memory.h>
686140a1	24	#include <asm/alternative.h>
f19fbd5e MS	25	#include <asm/nospec-branch.h>
f19fbd5e MS	26	#include <asm/facility.h>
de5012b4 IL	27	#include <asm/ftrace.lds.h>
de5012b4 IL	28	#include <asm/set_memory.h>
34644cc2	29	#include <asm/setup.h>
1da177e4 LT	30
	31	#if 0
	32	#define DEBUGP printk
	33	#else
	34	#define DEBUGP(fmt , ...)
	35	#endif
	36
f3b7e73b	37	#define PLT_ENTRY_SIZE 22
1da177e4	38
34644cc2 HC	39	static unsigned long get_module_load_offset(void)
	40	{
	41	static DEFINE_MUTEX(module_kaslr_mutex);
	42	static unsigned long module_load_offset;
	43
	44	if (!kaslr_enabled())
	45	return 0;
	46	/*
	47	* Calculate the module_load_offset the first time this code
	48	* is called. Once calculated it stays the same until reboot.
	49	*/
	50	mutex_lock(&module_kaslr_mutex);
	51	if (!module_load_offset)
	52	module_load_offset = get_random_u32_inclusive(1, 1024) * PAGE_SIZE;
	53	mutex_unlock(&module_kaslr_mutex);
	54	return module_load_offset;
	55	}
	56
c972cc60 HC	57	void *module_alloc(unsigned long size)
c972cc60 HC	58	{
60115fa5	59	gfp_t gfp_mask = GFP_KERNEL;
793213a8 VG	60	void *p;
793213a8 VG	61
c972cc60 HC	62	if (PAGE_ALIGN(size) > MODULES_LEN)
c972cc60 HC	63	return NULL;
34644cc2	64	p = __vmalloc_node_range(size, MODULE_ALIGN,
34e4c79f HC	65	MODULES_VADDR + get_module_load_offset(),
	66	MODULES_END, gfp_mask, PAGE_KERNEL,
	67	VM_FLUSH_RESET_PERMS \| VM_DEFER_KMEMLEAK,
	68	NUMA_NO_NODE, __builtin_return_address(0));
63840de2	69	if (p && (kasan_alloc_module_shadow(p, size, gfp_mask) < 0)) {
793213a8 VG	70	vfree(p);
	71	return NULL;
	72	}
	73	return p;
c972cc60	74	}
c972cc60	75
de5012b4 IL	76	#ifdef CONFIG_FUNCTION_TRACER
	77	void module_arch_cleanup(struct module *mod)
	78	{
	79	module_memfree(mod->arch.trampolines_start);
	80	}
	81	#endif
	82
d453cded	83	void module_arch_freeing_init(struct module *mod)
1da177e4	84	{
f31e0960 JY	85	if (is_livepatch_module(mod) &&
	86	mod->state == MODULE_STATE_LIVE)
	87	return;
	88
d453cded RR	89	vfree(mod->arch.syminfo);
d453cded RR	90	mod->arch.syminfo = NULL;
1da177e4 LT	91	}
1da177e4 LT	92
083e14c0	93	static void check_rela(Elf_Rela rela, struct module me)
1da177e4 LT	94	{
	95	struct mod_arch_syminfo *info;
	96
	97	info = me->arch.syminfo + ELF_R_SYM (rela->r_info);
	98	switch (ELF_R_TYPE (rela->r_info)) {
	99	case R_390_GOT12: /* 12 bit GOT offset. */
	100	case R_390_GOT16: /* 16 bit GOT offset. */
	101	case R_390_GOT20: /* 20 bit GOT offset. */
	102	case R_390_GOT32: /* 32 bit GOT offset. */
	103	case R_390_GOT64: /* 64 bit GOT offset. */
	104	case R_390_GOTENT: /* 32 bit PC rel. to GOT entry shifted by 1. */
	105	case R_390_GOTPLT12: /* 12 bit offset to jump slot. */
	106	case R_390_GOTPLT16: /* 16 bit offset to jump slot. */
	107	case R_390_GOTPLT20: /* 20 bit offset to jump slot. */
	108	case R_390_GOTPLT32: /* 32 bit offset to jump slot. */
	109	case R_390_GOTPLT64: /* 64 bit offset to jump slot. */
	110	case R_390_GOTPLTENT: /* 32 bit rel. offset to jump slot >> 1. */
	111	if (info->got_offset == -1UL) {
	112	info->got_offset = me->arch.got_size;
	113	me->arch.got_size += sizeof(void*);
	114	}
	115	break;
	116	case R_390_PLT16DBL: /* 16 bit PC rel. PLT shifted by 1. */
	117	case R_390_PLT32DBL: /* 32 bit PC rel. PLT shifted by 1. */
	118	case R_390_PLT32: /* 32 bit PC relative PLT address. */
	119	case R_390_PLT64: /* 64 bit PC relative PLT address. */
	120	case R_390_PLTOFF16: /* 16 bit offset from GOT to PLT. */
	121	case R_390_PLTOFF32: /* 32 bit offset from GOT to PLT. */
	122	case R_390_PLTOFF64: /* 16 bit offset from GOT to PLT. */
	123	if (info->plt_offset == -1UL) {
	124	info->plt_offset = me->arch.plt_size;
	125	me->arch.plt_size += PLT_ENTRY_SIZE;
	126	}
	127	break;
	128	case R_390_COPY:
	129	case R_390_GLOB_DAT:
	130	case R_390_JMP_SLOT:
	131	case R_390_RELATIVE:
	132	/* Only needed if we want to support loading of
	133	modules linked with -shared. */
	134	break;
	135	}
	136	}
	137
	138	/*
	139	* Account for GOT and PLT relocations. We can't add sections for
	140	* got and plt but we can increase the core module size.
	141	*/
083e14c0 MS	142	int module_frob_arch_sections(Elf_Ehdr hdr, Elf_Shdr sechdrs,
083e14c0 MS	143	char secstrings, struct module me)
1da177e4 LT	144	{
	145	Elf_Shdr *symtab;
	146	Elf_Sym *symbols;
	147	Elf_Rela *rela;
	148	char *strings;
	149	int nrela, i, j;
ac3b4328	150	struct module_memory *mod_mem;
1da177e4 LT	151
1da177e4 LT	152	/* Find symbol table and string table. */
d2c993d8	153	symtab = NULL;
1da177e4 LT	154	for (i = 0; i < hdr->e_shnum; i++)
	155	switch (sechdrs[i].sh_type) {
	156	case SHT_SYMTAB:
	157	symtab = sechdrs + i;
	158	break;
	159	}
	160	if (!symtab) {
	161	printk(KERN_ERR "module %s: no symbol table\n", me->name);
	162	return -ENOEXEC;
	163	}
	164
	165	/* Allocate one syminfo structure per symbol. */
	166	me->arch.nsyms = symtab->sh_size / sizeof(Elf_Sym);
42bc47b3 KC	167	me->arch.syminfo = vmalloc(array_size(sizeof(struct mod_arch_syminfo),
42bc47b3 KC	168	me->arch.nsyms));
1da177e4 LT	169	if (!me->arch.syminfo)
	170	return -ENOMEM;
	171	symbols = (void *) hdr + symtab->sh_offset;
	172	strings = (void *) hdr + sechdrs[symtab->sh_link].sh_offset;
	173	for (i = 0; i < me->arch.nsyms; i++) {
	174	if (symbols[i].st_shndx == SHN_UNDEF &&
	175	strcmp(strings + symbols[i].st_name,
	176	"_GLOBAL_OFFSET_TABLE_") == 0)
	177	/* "Define" it as absolute. */
	178	symbols[i].st_shndx = SHN_ABS;
	179	me->arch.syminfo[i].got_offset = -1UL;
	180	me->arch.syminfo[i].plt_offset = -1UL;
	181	me->arch.syminfo[i].got_initialized = 0;
	182	me->arch.syminfo[i].plt_initialized = 0;
	183	}
	184
	185	/* Search for got/plt relocations. */
	186	me->arch.got_size = me->arch.plt_size = 0;
	187	for (i = 0; i < hdr->e_shnum; i++) {
	188	if (sechdrs[i].sh_type != SHT_RELA)
	189	continue;
	190	nrela = sechdrs[i].sh_size / sizeof(Elf_Rela);
	191	rela = (void *) hdr + sechdrs[i].sh_offset;
	192	for (j = 0; j < nrela; j++)
	193	check_rela(rela + j, me);
	194	}
	195
	196	/* Increase core size by size of got & plt and set start
	197	offsets for got and plt. */
ac3b4328 SL	198	mod_mem = &me->mem[MOD_TEXT];
	199	mod_mem->size = ALIGN(mod_mem->size, 4);
	200	me->arch.got_offset = mod_mem->size;
	201	mod_mem->size += me->arch.got_size;
	202	me->arch.plt_offset = mod_mem->size;
f19fbd5e	203	if (me->arch.plt_size) {
6e179d64	204	if (IS_ENABLED(CONFIG_EXPOLINE) && !nospec_disable)
f19fbd5e	205	me->arch.plt_size += PLT_ENTRY_SIZE;
ac3b4328	206	mod_mem->size += me->arch.plt_size;
f19fbd5e	207	}
1da177e4 LT	208	return 0;
	209	}
	210
083e14c0	211	static int apply_rela_bits(Elf_Addr loc, Elf_Addr val,
be242261 PZ	212	int sign, int bits, int shift,
be242261 PZ	213	void (write)(void dest, const void src, size_t len))
083e14c0 MS	214	{
	215	unsigned long umax;
	216	long min, max;
be242261	217	void dest = (void )loc;
083e14c0 MS	218
	219	if (val & ((1UL << shift) - 1))
	220	return -ENOEXEC;
	221	if (sign) {
	222	val = (Elf_Addr)(((long) val) >> shift);
	223	min = -(1L << (bits - 1));
	224	max = (1L << (bits - 1)) - 1;
	225	if ((long) val < min \|\| (long) val > max)
	226	return -ENOEXEC;
	227	} else {
	228	val >>= shift;
	229	umax = ((1UL << (bits - 1)) << 1) - 1;
	230	if ((unsigned long) val > umax)
	231	return -ENOEXEC;
	232	}
	233
be242261 PZ	234	if (bits == 8) {
	235	unsigned char tmp = val;
	236	write(dest, &tmp, 1);
	237	} else if (bits == 12) {
	238	unsigned short tmp = (val & 0xfff) \|
083e14c0	239	((unsigned short ) loc & 0xf000);
be242261 PZ	240	write(dest, &tmp, 2);
	241	} else if (bits == 16) {
	242	unsigned short tmp = val;
	243	write(dest, &tmp, 2);
	244	} else if (bits == 20) {
	245	unsigned int tmp = (val & 0xfff) << 16 \|
	246	(val & 0xff000) >> 4 \| ((unsigned int ) loc & 0xf00000ff);
	247	write(dest, &tmp, 4);
	248	} else if (bits == 32) {
	249	unsigned int tmp = val;
	250	write(dest, &tmp, 4);
	251	} else if (bits == 64) {
	252	unsigned long tmp = val;
	253	write(dest, &tmp, 8);
	254	}
083e14c0 MS	255	return 0;
	256	}
	257
	258	static int apply_rela(Elf_Rela rela, Elf_Addr base, Elf_Sym symtab,
be242261 PZ	259	const char strtab, struct module me,
be242261 PZ	260	void (write)(void dest, const void src, size_t len))
1da177e4 LT	261	{
	262	struct mod_arch_syminfo *info;
	263	Elf_Addr loc, val;
	264	int r_type, r_sym;
72a6b43e	265	int rc = -ENOEXEC;
1da177e4 LT	266
	267	/* This is where to make the change */
	268	loc = base + rela->r_offset;
	269	/* This is the symbol it is referring to. Note that all
	270	undefined symbols have been resolved. */
	271	r_sym = ELF_R_SYM(rela->r_info);
	272	r_type = ELF_R_TYPE(rela->r_info);
	273	info = me->arch.syminfo + r_sym;
	274	val = symtab[r_sym].st_value;
	275
	276	switch (r_type) {
e80cfc31 HB	277	case R_390_NONE: /* No relocation. */
	278	rc = 0;
	279	break;
1da177e4 LT	280	case R_390_8: /* Direct 8 bit. */
	281	case R_390_12: /* Direct 12 bit. */
	282	case R_390_16: /* Direct 16 bit. */
	283	case R_390_20: /* Direct 20 bit. */
	284	case R_390_32: /* Direct 32 bit. */
	285	case R_390_64: /* Direct 64 bit. */
	286	val += rela->r_addend;
	287	if (r_type == R_390_8)
be242261	288	rc = apply_rela_bits(loc, val, 0, 8, 0, write);
1da177e4	289	else if (r_type == R_390_12)
be242261	290	rc = apply_rela_bits(loc, val, 0, 12, 0, write);
1da177e4	291	else if (r_type == R_390_16)
be242261	292	rc = apply_rela_bits(loc, val, 0, 16, 0, write);
1da177e4	293	else if (r_type == R_390_20)
be242261	294	rc = apply_rela_bits(loc, val, 1, 20, 0, write);
1da177e4	295	else if (r_type == R_390_32)
be242261	296	rc = apply_rela_bits(loc, val, 0, 32, 0, write);
1da177e4	297	else if (r_type == R_390_64)
be242261	298	rc = apply_rela_bits(loc, val, 0, 64, 0, write);
1da177e4 LT	299	break;
	300	case R_390_PC16: /* PC relative 16 bit. */
	301	case R_390_PC16DBL: /* PC relative 16 bit shifted by 1. */
	302	case R_390_PC32DBL: /* PC relative 32 bit shifted by 1. */
	303	case R_390_PC32: /* PC relative 32 bit. */
	304	case R_390_PC64: /* PC relative 64 bit. */
	305	val += rela->r_addend - loc;
	306	if (r_type == R_390_PC16)
be242261	307	rc = apply_rela_bits(loc, val, 1, 16, 0, write);
1da177e4	308	else if (r_type == R_390_PC16DBL)
be242261	309	rc = apply_rela_bits(loc, val, 1, 16, 1, write);
1da177e4	310	else if (r_type == R_390_PC32DBL)
be242261	311	rc = apply_rela_bits(loc, val, 1, 32, 1, write);
1da177e4	312	else if (r_type == R_390_PC32)
be242261	313	rc = apply_rela_bits(loc, val, 1, 32, 0, write);
1da177e4	314	else if (r_type == R_390_PC64)
be242261	315	rc = apply_rela_bits(loc, val, 1, 64, 0, write);
1da177e4 LT	316	break;
	317	case R_390_GOT12: /* 12 bit GOT offset. */
	318	case R_390_GOT16: /* 16 bit GOT offset. */
	319	case R_390_GOT20: /* 20 bit GOT offset. */
	320	case R_390_GOT32: /* 32 bit GOT offset. */
	321	case R_390_GOT64: /* 64 bit GOT offset. */
	322	case R_390_GOTENT: /* 32 bit PC rel. to GOT entry shifted by 1. */
	323	case R_390_GOTPLT12: /* 12 bit offset to jump slot. */
	324	case R_390_GOTPLT20: /* 20 bit offset to jump slot. */
	325	case R_390_GOTPLT16: /* 16 bit offset to jump slot. */
	326	case R_390_GOTPLT32: /* 32 bit offset to jump slot. */
	327	case R_390_GOTPLT64: /* 64 bit offset to jump slot. */
	328	case R_390_GOTPLTENT: /* 32 bit rel. offset to jump slot >> 1. */
	329	if (info->got_initialized == 0) {
ac3b4328	330	Elf_Addr *gotent = me->mem[MOD_TEXT].base +
be242261 PZ	331	me->arch.got_offset +
be242261 PZ	332	info->got_offset;
1da177e4	333
be242261	334	write(gotent, &val, sizeof(*gotent));
1da177e4 LT	335	info->got_initialized = 1;
	336	}
	337	val = info->got_offset + rela->r_addend;
	338	if (r_type == R_390_GOT12 \|\|
	339	r_type == R_390_GOTPLT12)
be242261	340	rc = apply_rela_bits(loc, val, 0, 12, 0, write);
1da177e4 LT	341	else if (r_type == R_390_GOT16 \|\|
1da177e4 LT	342	r_type == R_390_GOTPLT16)
be242261	343	rc = apply_rela_bits(loc, val, 0, 16, 0, write);
1da177e4 LT	344	else if (r_type == R_390_GOT20 \|\|
1da177e4 LT	345	r_type == R_390_GOTPLT20)
be242261	346	rc = apply_rela_bits(loc, val, 1, 20, 0, write);
1da177e4 LT	347	else if (r_type == R_390_GOT32 \|\|
1da177e4 LT	348	r_type == R_390_GOTPLT32)
be242261	349	rc = apply_rela_bits(loc, val, 0, 32, 0, write);
1da177e4 LT	350	else if (r_type == R_390_GOT64 \|\|
1da177e4 LT	351	r_type == R_390_GOTPLT64)
be242261	352	rc = apply_rela_bits(loc, val, 0, 64, 0, write);
083e14c0 MS	353	else if (r_type == R_390_GOTENT \|\|
083e14c0 MS	354	r_type == R_390_GOTPLTENT) {
11458e2b SK	355	val += (Elf_Addr)me->mem[MOD_TEXT].base +
11458e2b SK	356	me->arch.got_offset - loc;
be242261	357	rc = apply_rela_bits(loc, val, 1, 32, 1, write);
083e14c0	358	}
1da177e4 LT	359	break;
	360	case R_390_PLT16DBL: /* 16 bit PC rel. PLT shifted by 1. */
	361	case R_390_PLT32DBL: /* 32 bit PC rel. PLT shifted by 1. */
	362	case R_390_PLT32: /* 32 bit PC relative PLT address. */
	363	case R_390_PLT64: /* 64 bit PC relative PLT address. */
	364	case R_390_PLTOFF16: /* 16 bit offset from GOT to PLT. */
	365	case R_390_PLTOFF32: /* 32 bit offset from GOT to PLT. */
	366	case R_390_PLTOFF64: /* 16 bit offset from GOT to PLT. */
	367	if (info->plt_initialized == 0) {
f3b7e73b IL	368	unsigned char insn[PLT_ENTRY_SIZE];
	369	char *plt_base;
	370	char *ip;
be242261	371
ac3b4328	372	plt_base = me->mem[MOD_TEXT].base + me->arch.plt_offset;
f3b7e73b IL	373	ip = plt_base + info->plt_offset;
	374	(int )insn = 0x0d10e310; /* basr 1,0 */
	375	(int )&insn[4] = 0x100c0004; /* lg 1,12(1) */
6e179d64	376	if (IS_ENABLED(CONFIG_EXPOLINE) && !nospec_disable) {
f3b7e73b IL	377	char *jump_r1;
	378
	379	jump_r1 = plt_base + me->arch.plt_size -
	380	PLT_ENTRY_SIZE;
	381	/* brcl 0xf,__jump_r1 */
	382	(short )&insn[8] = 0xc0f4;
	383	(int )&insn[10] = (jump_r1 - (ip + 8)) / 2;
f19fbd5e	384	} else {
f3b7e73b	385	(int )&insn[8] = 0x07f10000; /* br %r1 */
f19fbd5e	386	}
f3b7e73b	387	(long )&insn[14] = val;
be242261 PZ	388
be242261 PZ	389	write(ip, insn, sizeof(insn));
1da177e4 LT	390	info->plt_initialized = 1;
	391	}
	392	if (r_type == R_390_PLTOFF16 \|\|
504665a9 MS	393	r_type == R_390_PLTOFF32 \|\|
504665a9 MS	394	r_type == R_390_PLTOFF64)
1da177e4 LT	395	val = me->arch.plt_offset - me->arch.got_offset +
1da177e4 LT	396	info->plt_offset + rela->r_addend;
504665a9 MS	397	else {
	398	if (!((r_type == R_390_PLT16DBL &&
	399	val - loc + 0xffffUL < 0x1ffffeUL) \|\|
	400	(r_type == R_390_PLT32DBL &&
	401	val - loc + 0xffffffffULL < 0x1fffffffeULL)))
ac3b4328	402	val = (Elf_Addr) me->mem[MOD_TEXT].base +
504665a9 MS	403	me->arch.plt_offset +
	404	info->plt_offset;
	405	val += rela->r_addend - loc;
	406	}
1da177e4	407	if (r_type == R_390_PLT16DBL)
be242261	408	rc = apply_rela_bits(loc, val, 1, 16, 1, write);
1da177e4	409	else if (r_type == R_390_PLTOFF16)
be242261	410	rc = apply_rela_bits(loc, val, 0, 16, 0, write);
1da177e4	411	else if (r_type == R_390_PLT32DBL)
be242261	412	rc = apply_rela_bits(loc, val, 1, 32, 1, write);
1da177e4 LT	413	else if (r_type == R_390_PLT32 \|\|
1da177e4 LT	414	r_type == R_390_PLTOFF32)
be242261	415	rc = apply_rela_bits(loc, val, 0, 32, 0, write);
1da177e4 LT	416	else if (r_type == R_390_PLT64 \|\|
1da177e4 LT	417	r_type == R_390_PLTOFF64)
be242261	418	rc = apply_rela_bits(loc, val, 0, 64, 0, write);
1da177e4 LT	419	break;
	420	case R_390_GOTOFF16: /* 16 bit offset to GOT. */
	421	case R_390_GOTOFF32: /* 32 bit offset to GOT. */
	422	case R_390_GOTOFF64: /* 64 bit offset to GOT. */
	423	val = val + rela->r_addend -
ac3b4328	424	((Elf_Addr) me->mem[MOD_TEXT].base + me->arch.got_offset);
1da177e4	425	if (r_type == R_390_GOTOFF16)
be242261	426	rc = apply_rela_bits(loc, val, 0, 16, 0, write);
1da177e4	427	else if (r_type == R_390_GOTOFF32)
be242261	428	rc = apply_rela_bits(loc, val, 0, 32, 0, write);
1da177e4	429	else if (r_type == R_390_GOTOFF64)
be242261	430	rc = apply_rela_bits(loc, val, 0, 64, 0, write);
1da177e4 LT	431	break;
	432	case R_390_GOTPC: /* 32 bit PC relative offset to GOT. */
	433	case R_390_GOTPCDBL: /* 32 bit PC rel. off. to GOT shifted by 1. */
ac3b4328	434	val = (Elf_Addr) me->mem[MOD_TEXT].base + me->arch.got_offset +
1da177e4 LT	435	rela->r_addend - loc;
1da177e4 LT	436	if (r_type == R_390_GOTPC)
be242261	437	rc = apply_rela_bits(loc, val, 1, 32, 0, write);
1da177e4	438	else if (r_type == R_390_GOTPCDBL)
be242261	439	rc = apply_rela_bits(loc, val, 1, 32, 1, write);
1da177e4 LT	440	break;
	441	case R_390_COPY:
	442	case R_390_GLOB_DAT: /* Create GOT entry. */
	443	case R_390_JMP_SLOT: /* Create PLT entry. */
	444	case R_390_RELATIVE: /* Adjust by program base. */
	445	/* Only needed if we want to support loading of
	446	modules linked with -shared. */
083e14c0	447	return -ENOEXEC;
1da177e4	448	default:
083e14c0	449	printk(KERN_ERR "module %s: unknown relocation: %u\n",
1da177e4 LT	450	me->name, r_type);
	451	return -ENOEXEC;
	452	}
083e14c0 MS	453	if (rc) {
	454	printk(KERN_ERR "module %s: relocation error for symbol %s "
	455	"(r_type %i, value 0x%lx)\n",
	456	me->name, strtab + symtab[r_sym].st_name,
	457	r_type, (unsigned long) val);
	458	return rc;
	459	}
1da177e4 LT	460	return 0;
	461	}
	462
be242261	463	static int __apply_relocate_add(Elf_Shdr sechdrs, const char strtab,
083e14c0	464	unsigned int symindex, unsigned int relsec,
be242261 PZ	465	struct module *me,
be242261 PZ	466	void (write)(void dest, const void src, size_t len))
1da177e4 LT	467	{
	468	Elf_Addr base;
	469	Elf_Sym *symtab;
	470	Elf_Rela *rela;
	471	unsigned long i, n;
	472	int rc;
	473
	474	DEBUGP("Applying relocate section %u to %u\n",
	475	relsec, sechdrs[relsec].sh_info);
	476	base = sechdrs[sechdrs[relsec].sh_info].sh_addr;
	477	symtab = (Elf_Sym *) sechdrs[symindex].sh_addr;
	478	rela = (Elf_Rela *) sechdrs[relsec].sh_addr;
	479	n = sechdrs[relsec].sh_size / sizeof(Elf_Rela);
	480
	481	for (i = 0; i < n; i++, rela++) {
be242261	482	rc = apply_rela(rela, base, symtab, strtab, me, write);
1da177e4 LT	483	if (rc)
	484	return rc;
	485	}
	486	return 0;
	487	}
	488
be242261 PZ	489	int apply_relocate_add(Elf_Shdr sechdrs, const char strtab,
	490	unsigned int symindex, unsigned int relsec,
	491	struct module *me)
	492	{
	493	bool early = me->state == MODULE_STATE_UNFORMED;
	494	void (write)(void , const void , size_t) = memcpy;
	495
	496	if (!early)
	497	write = s390_kernel_write;
	498
	499	return __apply_relocate_add(sechdrs, strtab, symindex, relsec, me,
	500	write);
	501	}
	502
de5012b4 IL	503	#ifdef CONFIG_FUNCTION_TRACER
	504	static int module_alloc_ftrace_hotpatch_trampolines(struct module *me,
	505	const Elf_Shdr *s)
	506	{
	507	char start, end;
	508	int numpages;
	509	size_t size;
	510
	511	size = FTRACE_HOTPATCH_TRAMPOLINES_SIZE(s->sh_size);
	512	numpages = DIV_ROUND_UP(size, PAGE_SIZE);
	513	start = module_alloc(numpages * PAGE_SIZE);
	514	if (!start)
	515	return -ENOMEM;
7c7ab788	516	set_memory_rox((unsigned long)start, numpages);
de5012b4 IL	517	end = start + size;
	518
	519	me->arch.trampolines_start = (struct ftrace_hotpatch_trampoline *)start;
	520	me->arch.trampolines_end = (struct ftrace_hotpatch_trampoline *)end;
	521	me->arch.next_trampoline = me->arch.trampolines_start;
	522
	523	return 0;
	524	}
	525	#endif /* CONFIG_FUNCTION_TRACER */
	526
1da177e4 LT	527	int module_finalize(const Elf_Ehdr *hdr,
	528	const Elf_Shdr *sechdrs,
	529	struct module *me)
	530	{
686140a1	531	const Elf_Shdr *s;
f19fbd5e MS	532	char secstrings, secname;
f19fbd5e MS	533	void *aseg;
de5012b4 IL	534	#ifdef CONFIG_FUNCTION_TRACER
	535	int ret;
	536	#endif
f19fbd5e MS	537
f19fbd5e MS	538	if (IS_ENABLED(CONFIG_EXPOLINE) &&
6e179d64	539	!nospec_disable && me->arch.plt_size) {
f19fbd5e MS	540	unsigned int *ij;
f19fbd5e MS	541
ac3b4328	542	ij = me->mem[MOD_TEXT].base + me->arch.plt_offset +
f19fbd5e	543	me->arch.plt_size - PLT_ENTRY_SIZE;
4efd417f VG	544	ij[0] = 0xc6000000; /* exrl %r0,.+10 */
	545	ij[1] = 0x0005a7f4; /* j . */
	546	ij[2] = 0x000007f1; /* br %r1 */
f19fbd5e	547	}
686140a1	548
049a2c2d HC	549	secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
049a2c2d HC	550	for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
f19fbd5e MS	551	aseg = (void *) s->sh_addr;
f19fbd5e MS	552	secname = secstrings + s->sh_name;
686140a1	553
f19fbd5e MS	554	if (!strcmp(".altinstructions", secname))
f19fbd5e MS	555	/* patch .altinstructions */
049a2c2d	556	apply_alternatives(aseg, aseg + s->sh_size);
f19fbd5e MS	557
f19fbd5e MS	558	if (IS_ENABLED(CONFIG_EXPOLINE) &&
b29cd7c4	559	(str_has_prefix(secname, ".s390_indirect")))
6e179d64	560	nospec_revert(aseg, aseg + s->sh_size);
f19fbd5e MS	561
f19fbd5e MS	562	if (IS_ENABLED(CONFIG_EXPOLINE) &&
b29cd7c4	563	(str_has_prefix(secname, ".s390_return")))
6e179d64	564	nospec_revert(aseg, aseg + s->sh_size);
de5012b4 IL	565
	566	#ifdef CONFIG_FUNCTION_TRACER
	567	if (!strcmp(FTRACE_CALLSITE_SECTION, secname)) {
	568	ret = module_alloc_ftrace_hotpatch_trampolines(me, s);
	569	if (ret < 0)
	570	return ret;
	571	}
	572	#endif /* CONFIG_FUNCTION_TRACER */
686140a1 VG	573	}
686140a1 VG	574
5336377d	575	return 0;
1da177e4	576	}