[linux-2.6-block.git] / arch / s390 / mm / hugetlbpage.c

/*
 *  IBM System z Huge TLB Page Support for Kernel.
 *
 *    Copyright IBM Corp. 2007,2016
 *    Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
 */

#define KMSG_COMPONENT "hugetlb"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <linux/mm.h>
#include <linux/hugetlb.h>

static inline unsigned long __pte_to_rste(pte_t pte)
{
	unsigned long rste;

	/*
	 * Convert encoding		  pte bits	pmd / pud bits
	 *				lIR.uswrdy.p	dy..R...I...wr
	 * empty			010.000000.0 -> 00..0...1...00
	 * prot-none, clean, old	111.000000.1 -> 00..1...1...00
	 * prot-none, clean, young	111.000001.1 -> 01..1...1...00
	 * prot-none, dirty, old	111.000010.1 -> 10..1...1...00
	 * prot-none, dirty, young	111.000011.1 -> 11..1...1...00
	 * read-only, clean, old	111.000100.1 -> 00..1...1...01
	 * read-only, clean, young	101.000101.1 -> 01..1...0...01
	 * read-only, dirty, old	111.000110.1 -> 10..1...1...01
	 * read-only, dirty, young	101.000111.1 -> 11..1...0...01
	 * read-write, clean, old	111.001100.1 -> 00..1...1...11
	 * read-write, clean, young	101.001101.1 -> 01..1...0...11
	 * read-write, dirty, old	110.001110.1 -> 10..0...1...11
	 * read-write, dirty, young	100.001111.1 -> 11..0...0...11
	 * HW-bits: R read-only, I invalid
	 * SW-bits: p present, y young, d dirty, r read, w write, s special,
	 *	    u unused, l large
	 */
	if (pte_present(pte)) {
		rste = pte_val(pte) & PAGE_MASK;
		rste |= (pte_val(pte) & _PAGE_READ) >> 4;
		rste |= (pte_val(pte) & _PAGE_WRITE) >> 4;
		rste |= (pte_val(pte) & _PAGE_INVALID) >> 5;
		rste |= (pte_val(pte) & _PAGE_PROTECT);
		rste |= (pte_val(pte) & _PAGE_DIRTY) << 10;
		rste |= (pte_val(pte) & _PAGE_YOUNG) << 10;
		rste |= (pte_val(pte) & _PAGE_SOFT_DIRTY) << 13;
	} else
		rste = _SEGMENT_ENTRY_INVALID;
	return rste;
}

static inline pte_t __rste_to_pte(unsigned long rste)
{
	int present;
	pte_t pte;

	if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
		present = pud_present(__pud(rste));
	else
		present = pmd_present(__pmd(rste));

	/*
	 * Convert encoding		pmd / pud bits	    pte bits
	 *				dy..R...I...wr	  lIR.uswrdy.p
	 * empty			00..0...1...00 -> 010.000000.0
	 * prot-none, clean, old	00..1...1...00 -> 111.000000.1
	 * prot-none, clean, young	01..1...1...00 -> 111.000001.1
	 * prot-none, dirty, old	10..1...1...00 -> 111.000010.1
	 * prot-none, dirty, young	11..1...1...00 -> 111.000011.1
	 * read-only, clean, old	00..1...1...01 -> 111.000100.1
	 * read-only, clean, young	01..1...0...01 -> 101.000101.1
	 * read-only, dirty, old	10..1...1...01 -> 111.000110.1
	 * read-only, dirty, young	11..1...0...01 -> 101.000111.1
	 * read-write, clean, old	00..1...1...11 -> 111.001100.1
	 * read-write, clean, young	01..1...0...11 -> 101.001101.1
	 * read-write, dirty, old	10..0...1...11 -> 110.001110.1
	 * read-write, dirty, young	11..0...0...11 -> 100.001111.1
	 * HW-bits: R read-only, I invalid
	 * SW-bits: p present, y young, d dirty, r read, w write, s special,
	 *	    u unused, l large
	 */
	if (present) {
		pte_val(pte) = rste & _SEGMENT_ENTRY_ORIGIN_LARGE;
		pte_val(pte) |= _PAGE_LARGE | _PAGE_PRESENT;
		pte_val(pte) |= (rste & _SEGMENT_ENTRY_READ) << 4;
		pte_val(pte) |= (rste & _SEGMENT_ENTRY_WRITE) << 4;
		pte_val(pte) |= (rste & _SEGMENT_ENTRY_INVALID) << 5;
		pte_val(pte) |= (rste & _SEGMENT_ENTRY_PROTECT);
		pte_val(pte) |= (rste & _SEGMENT_ENTRY_DIRTY) >> 10;
		pte_val(pte) |= (rste & _SEGMENT_ENTRY_YOUNG) >> 10;
		pte_val(pte) |= (rste & _SEGMENT_ENTRY_SOFT_DIRTY) >> 13;
	} else
		pte_val(pte) = _PAGE_INVALID;
	return pte;
}

void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
		     pte_t *ptep, pte_t pte)
{
	unsigned long rste = __pte_to_rste(pte);

	/* Set correct table type for 2G hugepages */
	if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
		rste |= _REGION_ENTRY_TYPE_R3 | _REGION3_ENTRY_LARGE;
	else
		rste |= _SEGMENT_ENTRY_LARGE;
	pte_val(*ptep) = rste;
}

pte_t huge_ptep_get(pte_t *ptep)
{
	return __rste_to_pte(pte_val(*ptep));
}

pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
			      unsigned long addr, pte_t *ptep)
{
	pte_t pte = huge_ptep_get(ptep);
	pmd_t *pmdp = (pmd_t *) ptep;
	pud_t *pudp = (pud_t *) ptep;

	if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
		pudp_xchg_direct(mm, addr, pudp, __pud(_REGION3_ENTRY_EMPTY));
	else
		pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
	return pte;
}

pte_t *huge_pte_alloc(struct mm_struct *mm,
			unsigned long addr, unsigned long sz)
{
	pgd_t *pgdp;
	pud_t *pudp;
	pmd_t *pmdp = NULL;

	pgdp = pgd_offset(mm, addr);
	pudp = pud_alloc(mm, pgdp, addr);
	if (pudp) {
		if (sz == PUD_SIZE)
			return (pte_t *) pudp;
		else if (sz == PMD_SIZE)
			pmdp = pmd_alloc(mm, pudp, addr);
	}
	return (pte_t *) pmdp;
}

pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
{
	pgd_t *pgdp;
	pud_t *pudp;
	pmd_t *pmdp = NULL;

	pgdp = pgd_offset(mm, addr);
	if (pgd_present(*pgdp)) {
		pudp = pud_offset(pgdp, addr);
		if (pud_present(*pudp)) {
			if (pud_large(*pudp))
				return (pte_t *) pudp;
			pmdp = pmd_offset(pudp, addr);
		}
	}
	return (pte_t *) pmdp;
}

int pmd_huge(pmd_t pmd)
{
	return pmd_large(pmd);
}

int pud_huge(pud_t pud)
{
	return pud_large(pud);
}

struct page *
follow_huge_pud(struct mm_struct *mm, unsigned long address,
		pud_t *pud, int flags)
{
	if (flags & FOLL_GET)
		return NULL;

	return pud_page(*pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
}

static __init int setup_hugepagesz(char *opt)
{
	unsigned long size;
	char *string = opt;

	size = memparse(opt, &opt);
	if (MACHINE_HAS_EDAT1 && size == PMD_SIZE) {
		hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
	} else if (MACHINE_HAS_EDAT2 && size == PUD_SIZE) {
		hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
	} else {
		pr_err("hugepagesz= specifies an unsupported page size %s\n",
			string);
		return 0;
	}
	return 1;
}
__setup("hugepagesz=", setup_hugepagesz);
Commit	Line	Data
53492b1d GS	1	/*
	2	* IBM System z Huge TLB Page Support for Kernel.
	3	*
d08de8e2	4	* Copyright IBM Corp. 2007,2016
53492b1d GS	5	* Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
	6	*/
	7
d08de8e2 GS	8	#define KMSG_COMPONENT "hugetlb"
	9	#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
	10
53492b1d GS	11	#include <linux/mm.h>
	12	#include <linux/hugetlb.h>
	13
d08de8e2	14	static inline unsigned long __pte_to_rste(pte_t pte)
e5098611	15	{
d08de8e2	16	unsigned long rste;
e5098611 MS	17
e5098611 MS	18	/*
d08de8e2	19	* Convert encoding pte bits pmd / pud bits
a1c843b8 MS	20	* lIR.uswrdy.p dy..R...I...wr
	21	* empty 010.000000.0 -> 00..0...1...00
	22	* prot-none, clean, old 111.000000.1 -> 00..1...1...00
	23	* prot-none, clean, young 111.000001.1 -> 01..1...1...00
	24	* prot-none, dirty, old 111.000010.1 -> 10..1...1...00
	25	* prot-none, dirty, young 111.000011.1 -> 11..1...1...00
	26	* read-only, clean, old 111.000100.1 -> 00..1...1...01
	27	* read-only, clean, young 101.000101.1 -> 01..1...0...01
	28	* read-only, dirty, old 111.000110.1 -> 10..1...1...01
	29	* read-only, dirty, young 101.000111.1 -> 11..1...0...01
	30	* read-write, clean, old 111.001100.1 -> 00..1...1...11
	31	* read-write, clean, young 101.001101.1 -> 01..1...0...11
	32	* read-write, dirty, old 110.001110.1 -> 10..0...1...11
	33	* read-write, dirty, young 100.001111.1 -> 11..0...0...11
	34	* HW-bits: R read-only, I invalid
	35	* SW-bits: p present, y young, d dirty, r read, w write, s special,
	36	* u unused, l large
e5098611 MS	37	*/
e5098611 MS	38	if (pte_present(pte)) {
d08de8e2 GS	39	rste = pte_val(pte) & PAGE_MASK;
	40	rste \|= (pte_val(pte) & _PAGE_READ) >> 4;
	41	rste \|= (pte_val(pte) & _PAGE_WRITE) >> 4;
	42	rste \|= (pte_val(pte) & _PAGE_INVALID) >> 5;
	43	rste \|= (pte_val(pte) & _PAGE_PROTECT);
	44	rste \|= (pte_val(pte) & _PAGE_DIRTY) << 10;
	45	rste \|= (pte_val(pte) & _PAGE_YOUNG) << 10;
	46	rste \|= (pte_val(pte) & _PAGE_SOFT_DIRTY) << 13;
e5098611	47	} else
d08de8e2 GS	48	rste = _SEGMENT_ENTRY_INVALID;
d08de8e2 GS	49	return rste;
e5098611 MS	50	}
e5098611 MS	51
d08de8e2	52	static inline pte_t __rste_to_pte(unsigned long rste)
e5098611	53	{
d08de8e2	54	int present;
e5098611 MS	55	pte_t pte;
e5098611 MS	56
d08de8e2 GS	57	if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
	58	present = pud_present(__pud(rste));
	59	else
	60	present = pmd_present(__pmd(rste));
	61
e5098611	62	/*
d08de8e2	63	* Convert encoding pmd / pud bits pte bits
a1c843b8 MS	64	* dy..R...I...wr lIR.uswrdy.p
	65	* empty 00..0...1...00 -> 010.000000.0
	66	* prot-none, clean, old 00..1...1...00 -> 111.000000.1
	67	* prot-none, clean, young 01..1...1...00 -> 111.000001.1
	68	* prot-none, dirty, old 10..1...1...00 -> 111.000010.1
	69	* prot-none, dirty, young 11..1...1...00 -> 111.000011.1
	70	* read-only, clean, old 00..1...1...01 -> 111.000100.1
	71	* read-only, clean, young 01..1...0...01 -> 101.000101.1
	72	* read-only, dirty, old 10..1...1...01 -> 111.000110.1
	73	* read-only, dirty, young 11..1...0...01 -> 101.000111.1
	74	* read-write, clean, old 00..1...1...11 -> 111.001100.1
	75	* read-write, clean, young 01..1...0...11 -> 101.001101.1
	76	* read-write, dirty, old 10..0...1...11 -> 110.001110.1
	77	* read-write, dirty, young 11..0...0...11 -> 100.001111.1
	78	* HW-bits: R read-only, I invalid
	79	* SW-bits: p present, y young, d dirty, r read, w write, s special,
	80	* u unused, l large
e5098611	81	*/
d08de8e2 GS	82	if (present) {
d08de8e2 GS	83	pte_val(pte) = rste & _SEGMENT_ENTRY_ORIGIN_LARGE;
152125b7	84	pte_val(pte) \|= _PAGE_LARGE \| _PAGE_PRESENT;
d08de8e2 GS	85	pte_val(pte) \|= (rste & _SEGMENT_ENTRY_READ) << 4;
	86	pte_val(pte) \|= (rste & _SEGMENT_ENTRY_WRITE) << 4;
	87	pte_val(pte) \|= (rste & _SEGMENT_ENTRY_INVALID) << 5;
	88	pte_val(pte) \|= (rste & _SEGMENT_ENTRY_PROTECT);
	89	pte_val(pte) \|= (rste & _SEGMENT_ENTRY_DIRTY) >> 10;
	90	pte_val(pte) \|= (rste & _SEGMENT_ENTRY_YOUNG) >> 10;
	91	pte_val(pte) \|= (rste & _SEGMENT_ENTRY_SOFT_DIRTY) >> 13;
e5098611 MS	92	} else
	93	pte_val(pte) = _PAGE_INVALID;
	94	return pte;
	95	}
53492b1d GS	96
53492b1d GS	97	void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
e5098611	98	pte_t *ptep, pte_t pte)
53492b1d	99	{
d08de8e2 GS	100	unsigned long rste = __pte_to_rste(pte);
	101
	102	/* Set correct table type for 2G hugepages */
	103	if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
	104	rste \|= _REGION_ENTRY_TYPE_R3 \| _REGION3_ENTRY_LARGE;
	105	else
	106	rste \|= _SEGMENT_ENTRY_LARGE;
	107	pte_val(*ptep) = rste;
e5098611 MS	108	}
	109
	110	pte_t huge_ptep_get(pte_t *ptep)
	111	{
d08de8e2	112	return __rste_to_pte(pte_val(*ptep));
e5098611 MS	113	}
	114
	115	pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
	116	unsigned long addr, pte_t *ptep)
	117	{
d08de8e2	118	pte_t pte = huge_ptep_get(ptep);
e5098611	119	pmd_t pmdp = (pmd_t ) ptep;
d08de8e2	120	pud_t pudp = (pud_t ) ptep;
53492b1d	121
d08de8e2 GS	122	if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
	123	pudp_xchg_direct(mm, addr, pudp, __pud(_REGION3_ENTRY_EMPTY));
	124	else
	125	pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
	126	return pte;
53492b1d GS	127	}
53492b1d GS	128
a5516438 AK	129	pte_t huge_pte_alloc(struct mm_struct mm,
a5516438 AK	130	unsigned long addr, unsigned long sz)
53492b1d GS	131	{
	132	pgd_t *pgdp;
	133	pud_t *pudp;
	134	pmd_t *pmdp = NULL;
	135
	136	pgdp = pgd_offset(mm, addr);
	137	pudp = pud_alloc(mm, pgdp, addr);
d08de8e2 GS	138	if (pudp) {
	139	if (sz == PUD_SIZE)
	140	return (pte_t *) pudp;
	141	else if (sz == PMD_SIZE)
	142	pmdp = pmd_alloc(mm, pudp, addr);
	143	}
53492b1d GS	144	return (pte_t *) pmdp;
	145	}
	146
	147	pte_t huge_pte_offset(struct mm_struct mm, unsigned long addr)
	148	{
	149	pgd_t *pgdp;
	150	pud_t *pudp;
	151	pmd_t *pmdp = NULL;
	152
	153	pgdp = pgd_offset(mm, addr);
	154	if (pgd_present(*pgdp)) {
	155	pudp = pud_offset(pgdp, addr);
d08de8e2 GS	156	if (pud_present(*pudp)) {
	157	if (pud_large(*pudp))
	158	return (pte_t *) pudp;
53492b1d	159	pmdp = pmd_offset(pudp, addr);
d08de8e2	160	}
53492b1d GS	161	}
	162	return (pte_t *) pmdp;
	163	}
	164
53492b1d GS	165	int pmd_huge(pmd_t pmd)
53492b1d GS	166	{
cbd7d9c2	167	return pmd_large(pmd);
53492b1d GS	168	}
53492b1d GS	169
ceb86879 AK	170	int pud_huge(pud_t pud)
ceb86879 AK	171	{
d08de8e2 GS	172	return pud_large(pud);
	173	}
	174
	175	struct page *
	176	follow_huge_pud(struct mm_struct *mm, unsigned long address,
	177	pud_t *pud, int flags)
	178	{
	179	if (flags & FOLL_GET)
	180	return NULL;
	181
	182	return pud_page(*pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
	183	}
	184
	185	static __init int setup_hugepagesz(char *opt)
	186	{
	187	unsigned long size;
	188	char *string = opt;
	189
	190	size = memparse(opt, &opt);
	191	if (MACHINE_HAS_EDAT1 && size == PMD_SIZE) {
	192	hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
	193	} else if (MACHINE_HAS_EDAT2 && size == PUD_SIZE) {
	194	hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
	195	} else {
	196	pr_err("hugepagesz= specifies an unsupported page size %s\n",
	197	string);
	198	return 0;
	199	}
	200	return 1;
ceb86879	201	}
d08de8e2	202	__setup("hugepagesz=", setup_hugepagesz);