[linux-2.6-block.git] / arch / x86 / include / asm / desc.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_DESC_H
#define _ASM_X86_DESC_H

#include <asm/desc_defs.h>
#include <asm/ldt.h>
#include <asm/mmu.h>
#include <asm/fixmap.h>
#include <asm/irq_vectors.h>
#include <asm/cpu_entry_area.h>

#include <linux/debug_locks.h>
#include <linux/smp.h>
#include <linux/percpu.h>

static inline void fill_ldt(struct desc_struct *desc, const struct user_desc *info)
{
	desc->limit0		= info->limit & 0x0ffff;

	desc->base0		= (info->base_addr & 0x0000ffff);
	desc->base1		= (info->base_addr & 0x00ff0000) >> 16;

	desc->type		= (info->read_exec_only ^ 1) << 1;
	desc->type	       |= info->contents << 2;
	/* Set the ACCESS bit so it can be mapped RO */
	desc->type	       |= 1;

	desc->s			= 1;
	desc->dpl		= 0x3;
	desc->p			= info->seg_not_present ^ 1;
	desc->limit1		= (info->limit & 0xf0000) >> 16;
	desc->avl		= info->useable;
	desc->d			= info->seg_32bit;
	desc->g			= info->limit_in_pages;

	desc->base2		= (info->base_addr & 0xff000000) >> 24;
	/*
	 * Don't allow setting of the lm bit. It would confuse
	 * user_64bit_mode and would get overridden by sysret anyway.
	 */
	desc->l			= 0;
}

struct gdt_page {
	struct desc_struct gdt[GDT_ENTRIES];
} __attribute__((aligned(PAGE_SIZE)));

DECLARE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page);

/* Provide the original GDT */
static inline struct desc_struct *get_cpu_gdt_rw(unsigned int cpu)
{
	return per_cpu(gdt_page, cpu).gdt;
}

/* Provide the current original GDT */
static inline struct desc_struct *get_current_gdt_rw(void)
{
	return this_cpu_ptr(&gdt_page)->gdt;
}

/* Provide the fixmap address of the remapped GDT */
static inline struct desc_struct *get_cpu_gdt_ro(int cpu)
{
	return (struct desc_struct *)&get_cpu_entry_area(cpu)->gdt;
}

/* Provide the current read-only GDT */
static inline struct desc_struct *get_current_gdt_ro(void)
{
	return get_cpu_gdt_ro(smp_processor_id());
}

/* Provide the physical address of the GDT page. */
static inline phys_addr_t get_cpu_gdt_paddr(unsigned int cpu)
{
	return per_cpu_ptr_to_phys(get_cpu_gdt_rw(cpu));
}

static inline void pack_gate(gate_desc *gate, unsigned type, unsigned long func,
			     unsigned dpl, unsigned ist, unsigned seg)
{
	gate->offset_low	= (u16) func;
	gate->bits.p		= 1;
	gate->bits.dpl		= dpl;
	gate->bits.zero		= 0;
	gate->bits.type		= type;
	gate->offset_middle	= (u16) (func >> 16);
#ifdef CONFIG_X86_64
	gate->segment		= __KERNEL_CS;
	gate->bits.ist		= ist;
	gate->reserved		= 0;
	gate->offset_high	= (u32) (func >> 32);
#else
	gate->segment		= seg;
	gate->bits.ist		= 0;
#endif
}

static inline int desc_empty(const void *ptr)
{
	const u32 *desc = ptr;

	return !(desc[0] | desc[1]);
}

#ifdef CONFIG_PARAVIRT_XXL
#include <asm/paravirt.h>
#else
#define load_TR_desc()				native_load_tr_desc()
#define load_gdt(dtr)				native_load_gdt(dtr)
#define load_idt(dtr)				native_load_idt(dtr)
#define load_tr(tr)				asm volatile("ltr %0"::"m" (tr))
#define load_ldt(ldt)				asm volatile("lldt %0"::"m" (ldt))

#define store_gdt(dtr)				native_store_gdt(dtr)
#define store_tr(tr)				(tr = native_store_tr())

#define load_TLS(t, cpu)			native_load_tls(t, cpu)
#define set_ldt					native_set_ldt

#define write_ldt_entry(dt, entry, desc)	native_write_ldt_entry(dt, entry, desc)
#define write_gdt_entry(dt, entry, desc, type)	native_write_gdt_entry(dt, entry, desc, type)
#define write_idt_entry(dt, entry, g)		native_write_idt_entry(dt, entry, g)

static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries)
{
}

static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries)
{
}
#endif	/* CONFIG_PARAVIRT_XXL */

#define store_ldt(ldt) asm("sldt %0" : "=m"(ldt))

static inline void native_write_idt_entry(gate_desc *idt, int entry, const gate_desc *gate)
{
	memcpy(&idt[entry], gate, sizeof(*gate));
}

static inline void native_write_ldt_entry(struct desc_struct *ldt, int entry, const void *desc)
{
	memcpy(&ldt[entry], desc, 8);
}

static inline void
native_write_gdt_entry(struct desc_struct *gdt, int entry, const void *desc, int type)
{
	unsigned int size;

	switch (type) {
	case DESC_TSS:	size = sizeof(tss_desc);	break;
	case DESC_LDT:	size = sizeof(ldt_desc);	break;
	default:	size = sizeof(*gdt);		break;
	}

	memcpy(&gdt[entry], desc, size);
}

static inline void set_tssldt_descriptor(void *d, unsigned long addr,
					 unsigned type, unsigned size)
{
	struct ldttss_desc *desc = d;

	memset(desc, 0, sizeof(*desc));

	desc->limit0		= (u16) size;
	desc->base0		= (u16) addr;
	desc->base1		= (addr >> 16) & 0xFF;
	desc->type		= type;
	desc->p			= 1;
	desc->limit1		= (size >> 16) & 0xF;
	desc->base2		= (addr >> 24) & 0xFF;
#ifdef CONFIG_X86_64
	desc->base3		= (u32) (addr >> 32);
#endif
}

static inline void __set_tss_desc(unsigned cpu, unsigned int entry, struct x86_hw_tss *addr)
{
	struct desc_struct *d = get_cpu_gdt_rw(cpu);
	tss_desc tss;

	set_tssldt_descriptor(&tss, (unsigned long)addr, DESC_TSS,
			      __KERNEL_TSS_LIMIT);
	write_gdt_entry(d, entry, &tss, DESC_TSS);
}

#define set_tss_desc(cpu, addr) __set_tss_desc(cpu, GDT_ENTRY_TSS, addr)

static inline void native_set_ldt(const void *addr, unsigned int entries)
{
	if (likely(entries == 0))
		asm volatile("lldt %w0"::"q" (0));
	else {
		unsigned cpu = smp_processor_id();
		ldt_desc ldt;

		set_tssldt_descriptor(&ldt, (unsigned long)addr, DESC_LDT,
				      entries * LDT_ENTRY_SIZE - 1);
		write_gdt_entry(get_cpu_gdt_rw(cpu), GDT_ENTRY_LDT,
				&ldt, DESC_LDT);
		asm volatile("lldt %w0"::"q" (GDT_ENTRY_LDT*8));
	}
}

static inline void native_load_gdt(const struct desc_ptr *dtr)
{
	asm volatile("lgdt %0"::"m" (*dtr));
}

static __always_inline void native_load_idt(const struct desc_ptr *dtr)
{
	asm volatile("lidt %0"::"m" (*dtr));
}

static inline void native_store_gdt(struct desc_ptr *dtr)
{
	asm volatile("sgdt %0":"=m" (*dtr));
}

static inline void store_idt(struct desc_ptr *dtr)
{
	asm volatile("sidt %0":"=m" (*dtr));
}

static inline void native_gdt_invalidate(void)
{
	const struct desc_ptr invalid_gdt = {
		.address = 0,
		.size = 0
	};

	native_load_gdt(&invalid_gdt);
}

static inline void native_idt_invalidate(void)
{
	const struct desc_ptr invalid_idt = {
		.address = 0,
		.size = 0
	};

	native_load_idt(&invalid_idt);
}

/*
 * The LTR instruction marks the TSS GDT entry as busy. On 64-bit, the GDT is
 * a read-only remapping. To prevent a page fault, the GDT is switched to the
 * original writeable version when needed.
 */
#ifdef CONFIG_X86_64
static inline void native_load_tr_desc(void)
{
	struct desc_ptr gdt;
	int cpu = raw_smp_processor_id();
	bool restore = 0;
	struct desc_struct *fixmap_gdt;

	native_store_gdt(&gdt);
	fixmap_gdt = get_cpu_gdt_ro(cpu);

	/*
	 * If the current GDT is the read-only fixmap, swap to the original
	 * writeable version. Swap back at the end.
	 */
	if (gdt.address == (unsigned long)fixmap_gdt) {
		load_direct_gdt(cpu);
		restore = 1;
	}
	asm volatile("ltr %w0"::"q" (GDT_ENTRY_TSS*8));
	if (restore)
		load_fixmap_gdt(cpu);
}
#else
static inline void native_load_tr_desc(void)
{
	asm volatile("ltr %w0"::"q" (GDT_ENTRY_TSS*8));
}
#endif

static inline unsigned long native_store_tr(void)
{
	unsigned long tr;

	asm volatile("str %0":"=r" (tr));

	return tr;
}

static inline void native_load_tls(struct thread_struct *t, unsigned int cpu)
{
	struct desc_struct *gdt = get_cpu_gdt_rw(cpu);
	unsigned int i;

	for (i = 0; i < GDT_ENTRY_TLS_ENTRIES; i++)
		gdt[GDT_ENTRY_TLS_MIN + i] = t->tls_array[i];
}

DECLARE_PER_CPU(bool, __tss_limit_invalid);

static inline void force_reload_TR(void)
{
	struct desc_struct *d = get_current_gdt_rw();
	tss_desc tss;

	memcpy(&tss, &d[GDT_ENTRY_TSS], sizeof(tss_desc));

	/*
	 * LTR requires an available TSS, and the TSS is currently
	 * busy.  Make it be available so that LTR will work.
	 */
	tss.type = DESC_TSS;
	write_gdt_entry(d, GDT_ENTRY_TSS, &tss, DESC_TSS);

	load_TR_desc();
	this_cpu_write(__tss_limit_invalid, false);
}

/*
 * Call this if you need the TSS limit to be correct, which should be the case
 * if and only if you have TIF_IO_BITMAP set or you're switching to a task
 * with TIF_IO_BITMAP set.
 */
static inline void refresh_tss_limit(void)
{
	DEBUG_LOCKS_WARN_ON(preemptible());

	if (unlikely(this_cpu_read(__tss_limit_invalid)))
		force_reload_TR();
}

/*
 * If you do something evil that corrupts the cached TSS limit (I'm looking
 * at you, VMX exits), call this function.
 *
 * The optimization here is that the TSS limit only matters for Linux if the
 * IO bitmap is in use.  If the TSS limit gets forced to its minimum value,
 * everything works except that IO bitmap will be ignored and all CPL 3 IO
 * instructions will #GP, which is exactly what we want for normal tasks.
 */
static inline void invalidate_tss_limit(void)
{
	DEBUG_LOCKS_WARN_ON(preemptible());

	if (unlikely(test_thread_flag(TIF_IO_BITMAP)))
		force_reload_TR();
	else
		this_cpu_write(__tss_limit_invalid, true);
}

/* This intentionally ignores lm, since 32-bit apps don't have that field. */
#define LDT_empty(info)					\
	((info)->base_addr		== 0	&&	\
	 (info)->limit			== 0	&&	\
	 (info)->contents		== 0	&&	\
	 (info)->read_exec_only		== 1	&&	\
	 (info)->seg_32bit		== 0	&&	\
	 (info)->limit_in_pages		== 0	&&	\
	 (info)->seg_not_present	== 1	&&	\
	 (info)->useable		== 0)

/* Lots of programs expect an all-zero user_desc to mean "no segment at all". */
static inline bool LDT_zero(const struct user_desc *info)
{
	return (info->base_addr		== 0 &&
		info->limit		== 0 &&
		info->contents		== 0 &&
		info->read_exec_only	== 0 &&
		info->seg_32bit		== 0 &&
		info->limit_in_pages	== 0 &&
		info->seg_not_present	== 0 &&
		info->useable		== 0);
}

static inline void clear_LDT(void)
{
	set_ldt(NULL, 0);
}

static inline unsigned long get_desc_base(const struct desc_struct *desc)
{
	return (unsigned)(desc->base0 | ((desc->base1) << 16) | ((desc->base2) << 24));
}

static inline void set_desc_base(struct desc_struct *desc, unsigned long base)
{
	desc->base0 = base & 0xffff;
	desc->base1 = (base >> 16) & 0xff;
	desc->base2 = (base >> 24) & 0xff;
}

static inline unsigned long get_desc_limit(const struct desc_struct *desc)
{
	return desc->limit0 | (desc->limit1 << 16);
}

static inline void set_desc_limit(struct desc_struct *desc, unsigned long limit)
{
	desc->limit0 = limit & 0xffff;
	desc->limit1 = (limit >> 16) & 0xf;
}

void alloc_intr_gate(unsigned int n, const void *addr);

static inline void init_idt_data(struct idt_data *data, unsigned int n,
				 const void *addr)
{
	BUG_ON(n > 0xFF);

	memset(data, 0, sizeof(*data));
	data->vector	= n;
	data->addr	= addr;
	data->segment	= __KERNEL_CS;
	data->bits.type	= GATE_INTERRUPT;
	data->bits.p	= 1;
}

static inline void idt_init_desc(gate_desc *gate, const struct idt_data *d)
{
	unsigned long addr = (unsigned long) d->addr;

	gate->offset_low	= (u16) addr;
	gate->segment		= (u16) d->segment;
	gate->bits		= d->bits;
	gate->offset_middle	= (u16) (addr >> 16);
#ifdef CONFIG_X86_64
	gate->offset_high	= (u32) (addr >> 32);
	gate->reserved		= 0;
#endif
}

extern unsigned long system_vectors[];

extern void load_current_idt(void);
extern void idt_setup_early_handler(void);
extern void idt_setup_early_traps(void);
extern void idt_setup_traps(void);
extern void idt_setup_apic_and_irq_gates(void);
extern bool idt_is_f00f_address(unsigned long address);

#ifdef CONFIG_X86_64
extern void idt_setup_early_pf(void);
#else
static inline void idt_setup_early_pf(void) { }
#endif

extern void idt_invalidate(void);

#endif /* _ASM_X86_DESC_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
1965aae3 PA	2	#ifndef _ASM_X86_DESC_H
1965aae3 PA	3	#define _ASM_X86_DESC_H
80fbb69a	4
80fbb69a GOC	5	#include <asm/desc_defs.h>
80fbb69a GOC	6	#include <asm/ldt.h>
881c2975	7	#include <asm/mmu.h>
69218e47	8	#include <asm/fixmap.h>
05161b9c	9	#include <asm/irq_vectors.h>
92a0f81d	10	#include <asm/cpu_entry_area.h>
9a3865b1	11
f39650de	12	#include <linux/debug_locks.h>
54cd0eac	13	#include <linux/smp.h>
c6ae41e7	14	#include <linux/percpu.h>
80fbb69a	15
9a3865b1 IM	16	static inline void fill_ldt(struct desc_struct desc, const struct user_desc info)
	17	{
	18	desc->limit0 = info->limit & 0x0ffff;
	19
	20	desc->base0 = (info->base_addr & 0x0000ffff);
	21	desc->base1 = (info->base_addr & 0x00ff0000) >> 16;
	22
	23	desc->type = (info->read_exec_only ^ 1) << 1;
	24	desc->type \|= info->contents << 2;
9f5cb6b3 TG	25	/* Set the ACCESS bit so it can be mapped RO */
9f5cb6b3 TG	26	desc->type \|= 1;
9a3865b1 IM	27
	28	desc->s = 1;
	29	desc->dpl = 0x3;
	30	desc->p = info->seg_not_present ^ 1;
38e9e81f	31	desc->limit1 = (info->limit & 0xf0000) >> 16;
9a3865b1 IM	32	desc->avl = info->useable;
	33	desc->d = info->seg_32bit;
	34	desc->g = info->limit_in_pages;
	35
	36	desc->base2 = (info->base_addr & 0xff000000) >> 24;
64f53a04	37	/*
318f5a2a AL	38	* Don't allow setting of the lm bit. It would confuse
318f5a2a AL	39	* user_64bit_mode and would get overridden by sysret anyway.
64f53a04	40	*/
9a3865b1	41	desc->l = 0;
80fbb69a GOC	42	}
80fbb69a GOC	43
a939098a GC	44	struct gdt_page {
	45	struct desc_struct gdt[GDT_ENTRIES];
	46	} __attribute__((aligned(PAGE_SIZE)));
9a3865b1	47
9b8de747	48	DECLARE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page);
a939098a	49
69218e47 TG	50	/* Provide the original GDT */
69218e47 TG	51	static inline struct desc_struct *get_cpu_gdt_rw(unsigned int cpu)
a939098a GC	52	{
	53	return per_cpu(gdt_page, cpu).gdt;
	54	}
	55
69218e47 TG	56	/* Provide the current original GDT */
	57	static inline struct desc_struct *get_current_gdt_rw(void)
	58	{
	59	return this_cpu_ptr(&gdt_page)->gdt;
	60	}
	61
69218e47 TG	62	/* Provide the fixmap address of the remapped GDT */
	63	static inline struct desc_struct *get_cpu_gdt_ro(int cpu)
	64	{
ef8813ab	65	return (struct desc_struct *)&get_cpu_entry_area(cpu)->gdt;
69218e47 TG	66	}
69218e47 TG	67
69218e47 TG	68	/* Provide the current read-only GDT */
	69	static inline struct desc_struct *get_current_gdt_ro(void)
	70	{
	71	return get_cpu_gdt_ro(smp_processor_id());
	72	}
	73
aa4ea675 AL	74	/* Provide the physical address of the GDT page. */
	75	static inline phys_addr_t get_cpu_gdt_paddr(unsigned int cpu)
	76	{
	77	return per_cpu_ptr_to_phys(get_cpu_gdt_rw(cpu));
	78	}
	79
507f90c9 GOC	80	static inline void pack_gate(gate_desc *gate, unsigned type, unsigned long func,
	81	unsigned dpl, unsigned ist, unsigned seg)
	82	{
64b163fa TG	83	gate->offset_low = (u16) func;
	84	gate->bits.p = 1;
	85	gate->bits.dpl = dpl;
	86	gate->bits.zero = 0;
	87	gate->bits.type = type;
	88	gate->offset_middle = (u16) (func >> 16);
	89	#ifdef CONFIG_X86_64
9a3865b1	90	gate->segment = __KERNEL_CS;
64b163fa TG	91	gate->bits.ist = ist;
	92	gate->reserved = 0;
	93	gate->offset_high = (u32) (func >> 32);
54cd0eac	94	#else
64b163fa TG	95	gate->segment = seg;
64b163fa TG	96	gate->bits.ist = 0;
54cd0eac	97	#endif
64b163fa	98	}
54cd0eac	99
746ff60f GOC	100	static inline int desc_empty(const void *ptr)
	101	{
	102	const u32 *desc = ptr;
9a3865b1	103
746ff60f GOC	104	return !(desc[0] \| desc[1]);
	105	}
	106
9bad5658	107	#ifdef CONFIG_PARAVIRT_XXL
54cd0eac GOC	108	#include <asm/paravirt.h>
54cd0eac GOC	109	#else
9a3865b1 IM	110	#define load_TR_desc() native_load_tr_desc()
	111	#define load_gdt(dtr) native_load_gdt(dtr)
	112	#define load_idt(dtr) native_load_idt(dtr)
	113	#define load_tr(tr) asm volatile("ltr %0"::"m" (tr))
	114	#define load_ldt(ldt) asm volatile("lldt %0"::"m" (ldt))
	115
	116	#define store_gdt(dtr) native_store_gdt(dtr)
9a3865b1 IM	117	#define store_tr(tr) (tr = native_store_tr())
	118
	119	#define load_TLS(t, cpu) native_load_tls(t, cpu)
	120	#define set_ldt native_set_ldt
	121
	122	#define write_ldt_entry(dt, entry, desc) native_write_ldt_entry(dt, entry, desc)
	123	#define write_gdt_entry(dt, entry, desc, type) native_write_gdt_entry(dt, entry, desc, type)
	124	#define write_idt_entry(dt, entry, g) native_write_idt_entry(dt, entry, g)
38ffbe66 JF	125
	126	static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries)
	127	{
	128	}
	129
	130	static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries)
	131	{
	132	}
9bad5658	133	#endif /* CONFIG_PARAVIRT_XXL */
54cd0eac	134
8229d754 JSR	135	#define store_ldt(ldt) asm("sldt %0" : "=m"(ldt))
8229d754 JSR	136
9a3865b1	137	static inline void native_write_idt_entry(gate_desc idt, int entry, const gate_desc gate)
54cd0eac GOC	138	{
	139	memcpy(&idt[entry], gate, sizeof(*gate));
	140	}
	141
9a3865b1	142	static inline void native_write_ldt_entry(struct desc_struct ldt, int entry, const void desc)
54cd0eac GOC	143	{
	144	memcpy(&ldt[entry], desc, 8);
	145	}
	146
9a3865b1 IM	147	static inline void
9a3865b1 IM	148	native_write_gdt_entry(struct desc_struct gdt, int entry, const void desc, int type)
54cd0eac GOC	149	{
54cd0eac GOC	150	unsigned int size;
9a3865b1	151
54cd0eac	152	switch (type) {
9a3865b1 IM	153	case DESC_TSS: size = sizeof(tss_desc); break;
	154	case DESC_LDT: size = sizeof(ldt_desc); break;
	155	default: size = sizeof(*gdt); break;
54cd0eac	156	}
9a3865b1	157
54cd0eac GOC	158	memcpy(&gdt[entry], desc, size);
	159	}
	160
64b163fa TG	161	static inline void set_tssldt_descriptor(void *d, unsigned long addr,
64b163fa TG	162	unsigned type, unsigned size)
c81c6ca4	163	{
87cc0376	164	struct ldttss_desc *desc = d;
9a3865b1	165
f6e0eba1	166	memset(desc, 0, sizeof(*desc));
9a3865b1	167
87cc0376	168	desc->limit0 = (u16) size;
64b163fa TG	169	desc->base0 = (u16) addr;
64b163fa TG	170	desc->base1 = (addr >> 16) & 0xFF;
9a3865b1 IM	171	desc->type = type;
	172	desc->p = 1;
	173	desc->limit1 = (size >> 16) & 0xF;
64b163fa	174	desc->base2 = (addr >> 24) & 0xFF;
87cc0376	175	#ifdef CONFIG_X86_64
64b163fa	176	desc->base3 = (u32) (addr >> 32);
c81c6ca4 GOC	177	#endif
	178	}
	179
7fb983b4	180	static inline void __set_tss_desc(unsigned cpu, unsigned int entry, struct x86_hw_tss *addr)
c81c6ca4	181	{
69218e47	182	struct desc_struct *d = get_cpu_gdt_rw(cpu);
c81c6ca4 GOC	183	tss_desc tss;
c81c6ca4 GOC	184
f6e0eba1	185	set_tssldt_descriptor(&tss, (unsigned long)addr, DESC_TSS,
4f53ab14	186	__KERNEL_TSS_LIMIT);
c81c6ca4 GOC	187	write_gdt_entry(d, entry, &tss, DESC_TSS);
	188	}
	189
	190	#define set_tss_desc(cpu, addr) __set_tss_desc(cpu, GDT_ENTRY_TSS, addr)
	191
54cd0eac GOC	192	static inline void native_set_ldt(const void *addr, unsigned int entries)
	193	{
	194	if (likely(entries == 0))
c1773a16	195	asm volatile("lldt %w0"::"q" (0));
54cd0eac GOC	196	else {
	197	unsigned cpu = smp_processor_id();
	198	ldt_desc ldt;
	199
5ac37f87 MK	200	set_tssldt_descriptor(&ldt, (unsigned long)addr, DESC_LDT,
5ac37f87 MK	201	entries * LDT_ENTRY_SIZE - 1);
69218e47	202	write_gdt_entry(get_cpu_gdt_rw(cpu), GDT_ENTRY_LDT,
54cd0eac	203	&ldt, DESC_LDT);
c1773a16	204	asm volatile("lldt %w0"::"q" (GDT_ENTRY_LDT*8));
54cd0eac GOC	205	}
	206	}
	207
45fc8757 TG	208	static inline void native_load_gdt(const struct desc_ptr *dtr)
	209	{
	210	asm volatile("lgdt %0"::"m" (*dtr));
	211	}
	212
f051f697	213	static __always_inline void native_load_idt(const struct desc_ptr *dtr)
45fc8757 TG	214	{
	215	asm volatile("lidt %0"::"m" (*dtr));
	216	}
	217
	218	static inline void native_store_gdt(struct desc_ptr *dtr)
	219	{
	220	asm volatile("sgdt %0":"=m" (*dtr));
	221	}
	222
87930019	223	static inline void store_idt(struct desc_ptr *dtr)
45fc8757 TG	224	{
	225	asm volatile("sidt %0":"=m" (*dtr));
	226	}
	227
283fa3b6 PAI	228	static inline void native_gdt_invalidate(void)
	229	{
	230	const struct desc_ptr invalid_gdt = {
	231	.address = 0,
	232	.size = 0
	233	};
	234
	235	native_load_gdt(&invalid_gdt);
	236	}
	237
	238	static inline void native_idt_invalidate(void)
	239	{
	240	const struct desc_ptr invalid_idt = {
	241	.address = 0,
	242	.size = 0
	243	};
	244
	245	native_load_idt(&invalid_idt);
	246	}
	247
45fc8757 TG	248	/*
	249	* The LTR instruction marks the TSS GDT entry as busy. On 64-bit, the GDT is
	250	* a read-only remapping. To prevent a page fault, the GDT is switched to the
	251	* original writeable version when needed.
	252	*/
	253	#ifdef CONFIG_X86_64
54cd0eac GOC	254	static inline void native_load_tr_desc(void)
54cd0eac GOC	255	{
45fc8757 TG	256	struct desc_ptr gdt;
	257	int cpu = raw_smp_processor_id();
	258	bool restore = 0;
	259	struct desc_struct *fixmap_gdt;
	260
	261	native_store_gdt(&gdt);
	262	fixmap_gdt = get_cpu_gdt_ro(cpu);
	263
	264	/*
	265	* If the current GDT is the read-only fixmap, swap to the original
	266	* writeable version. Swap back at the end.
	267	*/
	268	if (gdt.address == (unsigned long)fixmap_gdt) {
	269	load_direct_gdt(cpu);
	270	restore = 1;
	271	}
54cd0eac	272	asm volatile("ltr %w0"::"q" (GDT_ENTRY_TSS*8));
45fc8757 TG	273	if (restore)
	274	load_fixmap_gdt(cpu);
	275	}
	276	#else
	277	static inline void native_load_tr_desc(void)
	278	{
	279	asm volatile("ltr %w0"::"q" (GDT_ENTRY_TSS*8));
	280	}
	281	#endif
	282
	283	static inline unsigned long native_store_tr(void)
	284	{
	285	unsigned long tr;
	286
	287	asm volatile("str %0":"=r" (tr));
	288
	289	return tr;
	290	}
	291
	292	static inline void native_load_tls(struct thread_struct *t, unsigned int cpu)
	293	{
	294	struct desc_struct *gdt = get_cpu_gdt_rw(cpu);
	295	unsigned int i;
	296
	297	for (i = 0; i < GDT_ENTRY_TLS_ENTRIES; i++)
	298	gdt[GDT_ENTRY_TLS_MIN + i] = t->tls_array[i];
54cd0eac GOC	299	}
54cd0eac GOC	300
b7ceaec1 AL	301	DECLARE_PER_CPU(bool, __tss_limit_invalid);
b7ceaec1 AL	302
b7ffc44d AL	303	static inline void force_reload_TR(void)
b7ffc44d AL	304	{
69218e47	305	struct desc_struct *d = get_current_gdt_rw();
b7ffc44d AL	306	tss_desc tss;
	307
	308	memcpy(&tss, &d[GDT_ENTRY_TSS], sizeof(tss_desc));
	309
	310	/*
	311	* LTR requires an available TSS, and the TSS is currently
	312	* busy. Make it be available so that LTR will work.
	313	*/
	314	tss.type = DESC_TSS;
	315	write_gdt_entry(d, GDT_ENTRY_TSS, &tss, DESC_TSS);
	316
	317	load_TR_desc();
b7ceaec1	318	this_cpu_write(__tss_limit_invalid, false);
b7ffc44d AL	319	}
b7ffc44d AL	320
b7ceaec1 AL	321	/*
	322	* Call this if you need the TSS limit to be correct, which should be the case
	323	* if and only if you have TIF_IO_BITMAP set or you're switching to a task
	324	* with TIF_IO_BITMAP set.
	325	*/
	326	static inline void refresh_tss_limit(void)
b7ffc44d AL	327	{
	328	DEBUG_LOCKS_WARN_ON(preemptible());
	329
b7ceaec1	330	if (unlikely(this_cpu_read(__tss_limit_invalid)))
b7ffc44d	331	force_reload_TR();
b7ffc44d AL	332	}
	333
	334	/*
	335	* If you do something evil that corrupts the cached TSS limit (I'm looking
	336	* at you, VMX exits), call this function.
	337	*
	338	* The optimization here is that the TSS limit only matters for Linux if the
	339	* IO bitmap is in use. If the TSS limit gets forced to its minimum value,
	340	* everything works except that IO bitmap will be ignored and all CPL 3 IO
	341	* instructions will #GP, which is exactly what we want for normal tasks.
	342	*/
	343	static inline void invalidate_tss_limit(void)
	344	{
	345	DEBUG_LOCKS_WARN_ON(preemptible());
	346
	347	if (unlikely(test_thread_flag(TIF_IO_BITMAP)))
	348	force_reload_TR();
	349	else
b7ceaec1	350	this_cpu_write(__tss_limit_invalid, true);
b7ffc44d AL	351	}
b7ffc44d AL	352
e30ab185 AL	353	/* This intentionally ignores lm, since 32-bit apps don't have that field. */
e30ab185 AL	354	#define LDT_empty(info) \
c1773a16 JP	355	((info)->base_addr == 0 && \
	356	(info)->limit == 0 && \
	357	(info)->contents == 0 && \
	358	(info)->read_exec_only == 1 && \
	359	(info)->seg_32bit == 0 && \
	360	(info)->limit_in_pages == 0 && \
	361	(info)->seg_not_present == 1 && \
	362	(info)->useable == 0)
881c2975	363
3669ef9f AL	364	/* Lots of programs expect an all-zero user_desc to mean "no segment at all". */
	365	static inline bool LDT_zero(const struct user_desc *info)
	366	{
	367	return (info->base_addr == 0 &&
	368	info->limit == 0 &&
	369	info->contents == 0 &&
	370	info->read_exec_only == 0 &&
	371	info->seg_32bit == 0 &&
	372	info->limit_in_pages == 0 &&
	373	info->seg_not_present == 0 &&
	374	info->useable == 0);
	375	}
	376
881c2975 GOC	377	static inline void clear_LDT(void)
	378	{
	379	set_ldt(NULL, 0);
	380	}
	381
1bd5718c	382	static inline unsigned long get_desc_base(const struct desc_struct *desc)
cc697852	383	{
2c75910f	384	return (unsigned)(desc->base0 \| ((desc->base1) << 16) \| ((desc->base2) << 24));
cc697852	385	}
1bd5718c	386
57594742 AM	387	static inline void set_desc_base(struct desc_struct *desc, unsigned long base)
	388	{
	389	desc->base0 = base & 0xffff;
	390	desc->base1 = (base >> 16) & 0xff;
	391	desc->base2 = (base >> 24) & 0xff;
	392	}
	393
1bd5718c RM	394	static inline unsigned long get_desc_limit(const struct desc_struct *desc)
1bd5718c RM	395	{
38e9e81f	396	return desc->limit0 \| (desc->limit1 << 16);
1bd5718c RM	397	}
1bd5718c RM	398
57594742 AM	399	static inline void set_desc_limit(struct desc_struct *desc, unsigned long limit)
	400	{
	401	desc->limit0 = limit & 0xffff;
38e9e81f	402	desc->limit1 = (limit >> 16) & 0xf;
57594742 AM	403	}
57594742 AM	404
db18da78	405	void alloc_intr_gate(unsigned int n, const void *addr);
507f90c9	406
097ee5b7 JR	407	static inline void init_idt_data(struct idt_data *data, unsigned int n,
	408	const void *addr)
	409	{
	410	BUG_ON(n > 0xFF);
	411
	412	memset(data, 0, sizeof(*data));
	413	data->vector = n;
	414	data->addr = addr;
	415	data->segment = __KERNEL_CS;
	416	data->bits.type = GATE_INTERRUPT;
	417	data->bits.p = 1;
	418	}
	419
	420	static inline void idt_init_desc(gate_desc gate, const struct idt_data d)
	421	{
	422	unsigned long addr = (unsigned long) d->addr;
	423
	424	gate->offset_low = (u16) addr;
	425	gate->segment = (u16) d->segment;
	426	gate->bits = d->bits;
	427	gate->offset_middle = (u16) (addr >> 16);
	428	#ifdef CONFIG_X86_64
	429	gate->offset_high = (u32) (addr >> 32);
	430	gate->reserved = 0;
	431	#endif
	432	}
	433
7854f822	434	extern unsigned long system_vectors[];
305b92a2	435
3e77abda	436	extern void load_current_idt(void);
588787fd TG	437	extern void idt_setup_early_handler(void);
588787fd TG	438	extern void idt_setup_early_traps(void);
b70543a0	439	extern void idt_setup_traps(void);
636a7598	440	extern void idt_setup_apic_and_irq_gates(void);
3e77abda	441	extern bool idt_is_f00f_address(unsigned long address);
588787fd TG	442
	443	#ifdef CONFIG_X86_64
	444	extern void idt_setup_early_pf(void);
	445	#else
	446	static inline void idt_setup_early_pf(void) { }
	447	#endif
	448
8ec9069a	449	extern void idt_invalidate(void);
e802a51e	450
1965aae3	451	#endif /* _ASM_X86_DESC_H */