[linux-2.6-block.git] / arch / s390 / mm / hugetlbpage.c

/*
 *  IBM System z Huge TLB Page Support for Kernel.
 *
 *    Copyright IBM Corp. 2007,2016
 *    Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
 */

#define KMSG_COMPONENT "hugetlb"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <linux/mm.h>
#include <linux/hugetlb.h>

/*
 * If the bit selected by single-bit bitmask "a" is set within "x", move
 * it to the position indicated by single-bit bitmask "b".
 */
#define move_set_bit(x, a, b)	(((x) & (a)) >> ilog2(a) << ilog2(b))

static inline unsigned long __pte_to_rste(pte_t pte)
{
	unsigned long rste;

	/*
	 * Convert encoding		  pte bits	pmd / pud bits
	 *				lIR.uswrdy.p	dy..R...I...wr
	 * empty			010.000000.0 -> 00..0...1...00
	 * prot-none, clean, old	111.000000.1 -> 00..1...1...00
	 * prot-none, clean, young	111.000001.1 -> 01..1...1...00
	 * prot-none, dirty, old	111.000010.1 -> 10..1...1...00
	 * prot-none, dirty, young	111.000011.1 -> 11..1...1...00
	 * read-only, clean, old	111.000100.1 -> 00..1...1...01
	 * read-only, clean, young	101.000101.1 -> 01..1...0...01
	 * read-only, dirty, old	111.000110.1 -> 10..1...1...01
	 * read-only, dirty, young	101.000111.1 -> 11..1...0...01
	 * read-write, clean, old	111.001100.1 -> 00..1...1...11
	 * read-write, clean, young	101.001101.1 -> 01..1...0...11
	 * read-write, dirty, old	110.001110.1 -> 10..0...1...11
	 * read-write, dirty, young	100.001111.1 -> 11..0...0...11
	 * HW-bits: R read-only, I invalid
	 * SW-bits: p present, y young, d dirty, r read, w write, s special,
	 *	    u unused, l large
	 */
	if (pte_present(pte)) {
		rste = pte_val(pte) & PAGE_MASK;
		rste |= move_set_bit(pte_val(pte), _PAGE_READ,
				     _SEGMENT_ENTRY_READ);
		rste |= move_set_bit(pte_val(pte), _PAGE_WRITE,
				     _SEGMENT_ENTRY_WRITE);
		rste |= move_set_bit(pte_val(pte), _PAGE_INVALID,
				     _SEGMENT_ENTRY_INVALID);
		rste |= move_set_bit(pte_val(pte), _PAGE_PROTECT,
				     _SEGMENT_ENTRY_PROTECT);
		rste |= move_set_bit(pte_val(pte), _PAGE_DIRTY,
				     _SEGMENT_ENTRY_DIRTY);
		rste |= move_set_bit(pte_val(pte), _PAGE_YOUNG,
				     _SEGMENT_ENTRY_YOUNG);
#ifdef CONFIG_MEM_SOFT_DIRTY
		rste |= move_set_bit(pte_val(pte), _PAGE_SOFT_DIRTY,
				     _SEGMENT_ENTRY_SOFT_DIRTY);
#endif
	} else
		rste = _SEGMENT_ENTRY_INVALID;
	return rste;
}

static inline pte_t __rste_to_pte(unsigned long rste)
{
	int present;
	pte_t pte;

	if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
		present = pud_present(__pud(rste));
	else
		present = pmd_present(__pmd(rste));

	/*
	 * Convert encoding		pmd / pud bits	    pte bits
	 *				dy..R...I...wr	  lIR.uswrdy.p
	 * empty			00..0...1...00 -> 010.000000.0
	 * prot-none, clean, old	00..1...1...00 -> 111.000000.1
	 * prot-none, clean, young	01..1...1...00 -> 111.000001.1
	 * prot-none, dirty, old	10..1...1...00 -> 111.000010.1
	 * prot-none, dirty, young	11..1...1...00 -> 111.000011.1
	 * read-only, clean, old	00..1...1...01 -> 111.000100.1
	 * read-only, clean, young	01..1...0...01 -> 101.000101.1
	 * read-only, dirty, old	10..1...1...01 -> 111.000110.1
	 * read-only, dirty, young	11..1...0...01 -> 101.000111.1
	 * read-write, clean, old	00..1...1...11 -> 111.001100.1
	 * read-write, clean, young	01..1...0...11 -> 101.001101.1
	 * read-write, dirty, old	10..0...1...11 -> 110.001110.1
	 * read-write, dirty, young	11..0...0...11 -> 100.001111.1
	 * HW-bits: R read-only, I invalid
	 * SW-bits: p present, y young, d dirty, r read, w write, s special,
	 *	    u unused, l large
	 */
	if (present) {
		pte_val(pte) = rste & _SEGMENT_ENTRY_ORIGIN_LARGE;
		pte_val(pte) |= _PAGE_LARGE | _PAGE_PRESENT;
		pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_READ,
					     _PAGE_READ);
		pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_WRITE,
					     _PAGE_WRITE);
		pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_INVALID,
					     _PAGE_INVALID);
		pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_PROTECT,
					     _PAGE_PROTECT);
		pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_DIRTY,
					     _PAGE_DIRTY);
		pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_YOUNG,
					     _PAGE_YOUNG);
#ifdef CONFIG_MEM_SOFT_DIRTY
		pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_SOFT_DIRTY,
					     _PAGE_DIRTY);
#endif
	} else
		pte_val(pte) = _PAGE_INVALID;
	return pte;
}

void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
		     pte_t *ptep, pte_t pte)
{
	unsigned long rste = __pte_to_rste(pte);

	/* Set correct table type for 2G hugepages */
	if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
		rste |= _REGION_ENTRY_TYPE_R3 | _REGION3_ENTRY_LARGE;
	else
		rste |= _SEGMENT_ENTRY_LARGE;
	pte_val(*ptep) = rste;
}

pte_t huge_ptep_get(pte_t *ptep)
{
	return __rste_to_pte(pte_val(*ptep));
}

pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
			      unsigned long addr, pte_t *ptep)
{
	pte_t pte = huge_ptep_get(ptep);
	pmd_t *pmdp = (pmd_t *) ptep;
	pud_t *pudp = (pud_t *) ptep;

	if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
		pudp_xchg_direct(mm, addr, pudp, __pud(_REGION3_ENTRY_EMPTY));
	else
		pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
	return pte;
}

pte_t *huge_pte_alloc(struct mm_struct *mm,
			unsigned long addr, unsigned long sz)
{
	pgd_t *pgdp;
	pud_t *pudp;
	pmd_t *pmdp = NULL;

	pgdp = pgd_offset(mm, addr);
	pudp = pud_alloc(mm, pgdp, addr);
	if (pudp) {
		if (sz == PUD_SIZE)
			return (pte_t *) pudp;
		else if (sz == PMD_SIZE)
			pmdp = pmd_alloc(mm, pudp, addr);
	}
	return (pte_t *) pmdp;
}

pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
{
	pgd_t *pgdp;
	pud_t *pudp;
	pmd_t *pmdp = NULL;

	pgdp = pgd_offset(mm, addr);
	if (pgd_present(*pgdp)) {
		pudp = pud_offset(pgdp, addr);
		if (pud_present(*pudp)) {
			if (pud_large(*pudp))
				return (pte_t *) pudp;
			pmdp = pmd_offset(pudp, addr);
		}
	}
	return (pte_t *) pmdp;
}

int pmd_huge(pmd_t pmd)
{
	return pmd_large(pmd);
}

int pud_huge(pud_t pud)
{
	return pud_large(pud);
}

struct page *
follow_huge_pud(struct mm_struct *mm, unsigned long address,
		pud_t *pud, int flags)
{
	if (flags & FOLL_GET)
		return NULL;

	return pud_page(*pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
}

static __init int setup_hugepagesz(char *opt)
{
	unsigned long size;
	char *string = opt;

	size = memparse(opt, &opt);
	if (MACHINE_HAS_EDAT1 && size == PMD_SIZE) {
		hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
	} else if (MACHINE_HAS_EDAT2 && size == PUD_SIZE) {
		hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
	} else {
		pr_err("hugepagesz= specifies an unsupported page size %s\n",
			string);
		return 0;
	}
	return 1;
}
__setup("hugepagesz=", setup_hugepagesz);
Commit	Line	Data
53492b1d GS	1	/*
	2	* IBM System z Huge TLB Page Support for Kernel.
	3	*
d08de8e2	4	* Copyright IBM Corp. 2007,2016
53492b1d GS	5	* Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
	6	*/
	7
d08de8e2 GS	8	#define KMSG_COMPONENT "hugetlb"
	9	#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
	10
53492b1d GS	11	#include <linux/mm.h>
	12	#include <linux/hugetlb.h>
	13
bc29b7ac GS	14	/*
	15	* If the bit selected by single-bit bitmask "a" is set within "x", move
	16	* it to the position indicated by single-bit bitmask "b".
	17	*/
	18	#define move_set_bit(x, a, b) (((x) & (a)) >> ilog2(a) << ilog2(b))
	19
d08de8e2	20	static inline unsigned long __pte_to_rste(pte_t pte)
e5098611	21	{
d08de8e2	22	unsigned long rste;
e5098611 MS	23
e5098611 MS	24	/*
d08de8e2	25	* Convert encoding pte bits pmd / pud bits
a1c843b8 MS	26	* lIR.uswrdy.p dy..R...I...wr
	27	* empty 010.000000.0 -> 00..0...1...00
	28	* prot-none, clean, old 111.000000.1 -> 00..1...1...00
	29	* prot-none, clean, young 111.000001.1 -> 01..1...1...00
	30	* prot-none, dirty, old 111.000010.1 -> 10..1...1...00
	31	* prot-none, dirty, young 111.000011.1 -> 11..1...1...00
	32	* read-only, clean, old 111.000100.1 -> 00..1...1...01
	33	* read-only, clean, young 101.000101.1 -> 01..1...0...01
	34	* read-only, dirty, old 111.000110.1 -> 10..1...1...01
	35	* read-only, dirty, young 101.000111.1 -> 11..1...0...01
	36	* read-write, clean, old 111.001100.1 -> 00..1...1...11
	37	* read-write, clean, young 101.001101.1 -> 01..1...0...11
	38	* read-write, dirty, old 110.001110.1 -> 10..0...1...11
	39	* read-write, dirty, young 100.001111.1 -> 11..0...0...11
	40	* HW-bits: R read-only, I invalid
	41	* SW-bits: p present, y young, d dirty, r read, w write, s special,
	42	* u unused, l large
e5098611 MS	43	*/
e5098611 MS	44	if (pte_present(pte)) {
d08de8e2	45	rste = pte_val(pte) & PAGE_MASK;
bc29b7ac GS	46	rste \|= move_set_bit(pte_val(pte), _PAGE_READ,
	47	_SEGMENT_ENTRY_READ);
	48	rste \|= move_set_bit(pte_val(pte), _PAGE_WRITE,
	49	_SEGMENT_ENTRY_WRITE);
	50	rste \|= move_set_bit(pte_val(pte), _PAGE_INVALID,
	51	_SEGMENT_ENTRY_INVALID);
	52	rste \|= move_set_bit(pte_val(pte), _PAGE_PROTECT,
	53	_SEGMENT_ENTRY_PROTECT);
	54	rste \|= move_set_bit(pte_val(pte), _PAGE_DIRTY,
	55	_SEGMENT_ENTRY_DIRTY);
	56	rste \|= move_set_bit(pte_val(pte), _PAGE_YOUNG,
	57	_SEGMENT_ENTRY_YOUNG);
	58	#ifdef CONFIG_MEM_SOFT_DIRTY
	59	rste \|= move_set_bit(pte_val(pte), _PAGE_SOFT_DIRTY,
	60	_SEGMENT_ENTRY_SOFT_DIRTY);
	61	#endif
e5098611	62	} else
d08de8e2 GS	63	rste = _SEGMENT_ENTRY_INVALID;
d08de8e2 GS	64	return rste;
e5098611 MS	65	}
e5098611 MS	66
d08de8e2	67	static inline pte_t __rste_to_pte(unsigned long rste)
e5098611	68	{
d08de8e2	69	int present;
e5098611 MS	70	pte_t pte;
e5098611 MS	71
d08de8e2 GS	72	if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
	73	present = pud_present(__pud(rste));
	74	else
	75	present = pmd_present(__pmd(rste));
	76
e5098611	77	/*
d08de8e2	78	* Convert encoding pmd / pud bits pte bits
a1c843b8 MS	79	* dy..R...I...wr lIR.uswrdy.p
	80	* empty 00..0...1...00 -> 010.000000.0
	81	* prot-none, clean, old 00..1...1...00 -> 111.000000.1
	82	* prot-none, clean, young 01..1...1...00 -> 111.000001.1
	83	* prot-none, dirty, old 10..1...1...00 -> 111.000010.1
	84	* prot-none, dirty, young 11..1...1...00 -> 111.000011.1
	85	* read-only, clean, old 00..1...1...01 -> 111.000100.1
	86	* read-only, clean, young 01..1...0...01 -> 101.000101.1
	87	* read-only, dirty, old 10..1...1...01 -> 111.000110.1
	88	* read-only, dirty, young 11..1...0...01 -> 101.000111.1
	89	* read-write, clean, old 00..1...1...11 -> 111.001100.1
	90	* read-write, clean, young 01..1...0...11 -> 101.001101.1
	91	* read-write, dirty, old 10..0...1...11 -> 110.001110.1
	92	* read-write, dirty, young 11..0...0...11 -> 100.001111.1
	93	* HW-bits: R read-only, I invalid
	94	* SW-bits: p present, y young, d dirty, r read, w write, s special,
	95	* u unused, l large
e5098611	96	*/
d08de8e2 GS	97	if (present) {
d08de8e2 GS	98	pte_val(pte) = rste & _SEGMENT_ENTRY_ORIGIN_LARGE;
152125b7	99	pte_val(pte) \|= _PAGE_LARGE \| _PAGE_PRESENT;
bc29b7ac GS	100	pte_val(pte) \|= move_set_bit(rste, _SEGMENT_ENTRY_READ,
	101	_PAGE_READ);
	102	pte_val(pte) \|= move_set_bit(rste, _SEGMENT_ENTRY_WRITE,
	103	_PAGE_WRITE);
	104	pte_val(pte) \|= move_set_bit(rste, _SEGMENT_ENTRY_INVALID,
	105	_PAGE_INVALID);
	106	pte_val(pte) \|= move_set_bit(rste, _SEGMENT_ENTRY_PROTECT,
	107	_PAGE_PROTECT);
	108	pte_val(pte) \|= move_set_bit(rste, _SEGMENT_ENTRY_DIRTY,
	109	_PAGE_DIRTY);
	110	pte_val(pte) \|= move_set_bit(rste, _SEGMENT_ENTRY_YOUNG,
	111	_PAGE_YOUNG);
	112	#ifdef CONFIG_MEM_SOFT_DIRTY
	113	pte_val(pte) \|= move_set_bit(rste, _SEGMENT_ENTRY_SOFT_DIRTY,
	114	_PAGE_DIRTY);
	115	#endif
e5098611 MS	116	} else
	117	pte_val(pte) = _PAGE_INVALID;
	118	return pte;
	119	}
53492b1d GS	120
53492b1d GS	121	void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
e5098611	122	pte_t *ptep, pte_t pte)
53492b1d	123	{
d08de8e2 GS	124	unsigned long rste = __pte_to_rste(pte);
	125
	126	/* Set correct table type for 2G hugepages */
	127	if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
	128	rste \|= _REGION_ENTRY_TYPE_R3 \| _REGION3_ENTRY_LARGE;
	129	else
	130	rste \|= _SEGMENT_ENTRY_LARGE;
	131	pte_val(*ptep) = rste;
e5098611 MS	132	}
	133
	134	pte_t huge_ptep_get(pte_t *ptep)
	135	{
d08de8e2	136	return __rste_to_pte(pte_val(*ptep));
e5098611 MS	137	}
	138
	139	pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
	140	unsigned long addr, pte_t *ptep)
	141	{
d08de8e2	142	pte_t pte = huge_ptep_get(ptep);
e5098611	143	pmd_t pmdp = (pmd_t ) ptep;
d08de8e2	144	pud_t pudp = (pud_t ) ptep;
53492b1d	145
d08de8e2 GS	146	if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
	147	pudp_xchg_direct(mm, addr, pudp, __pud(_REGION3_ENTRY_EMPTY));
	148	else
	149	pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
	150	return pte;
53492b1d GS	151	}
53492b1d GS	152
a5516438 AK	153	pte_t huge_pte_alloc(struct mm_struct mm,
a5516438 AK	154	unsigned long addr, unsigned long sz)
53492b1d GS	155	{
	156	pgd_t *pgdp;
	157	pud_t *pudp;
	158	pmd_t *pmdp = NULL;
	159
	160	pgdp = pgd_offset(mm, addr);
	161	pudp = pud_alloc(mm, pgdp, addr);
d08de8e2 GS	162	if (pudp) {
	163	if (sz == PUD_SIZE)
	164	return (pte_t *) pudp;
	165	else if (sz == PMD_SIZE)
	166	pmdp = pmd_alloc(mm, pudp, addr);
	167	}
53492b1d GS	168	return (pte_t *) pmdp;
	169	}
	170
	171	pte_t huge_pte_offset(struct mm_struct mm, unsigned long addr)
	172	{
	173	pgd_t *pgdp;
	174	pud_t *pudp;
	175	pmd_t *pmdp = NULL;
	176
	177	pgdp = pgd_offset(mm, addr);
	178	if (pgd_present(*pgdp)) {
	179	pudp = pud_offset(pgdp, addr);
d08de8e2 GS	180	if (pud_present(*pudp)) {
	181	if (pud_large(*pudp))
	182	return (pte_t *) pudp;
53492b1d	183	pmdp = pmd_offset(pudp, addr);
d08de8e2	184	}
53492b1d GS	185	}
	186	return (pte_t *) pmdp;
	187	}
	188
53492b1d GS	189	int pmd_huge(pmd_t pmd)
53492b1d GS	190	{
cbd7d9c2	191	return pmd_large(pmd);
53492b1d GS	192	}
53492b1d GS	193
ceb86879 AK	194	int pud_huge(pud_t pud)
ceb86879 AK	195	{
d08de8e2 GS	196	return pud_large(pud);
	197	}
	198
	199	struct page *
	200	follow_huge_pud(struct mm_struct *mm, unsigned long address,
	201	pud_t *pud, int flags)
	202	{
	203	if (flags & FOLL_GET)
	204	return NULL;
	205
	206	return pud_page(*pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
	207	}
	208
	209	static __init int setup_hugepagesz(char *opt)
	210	{
	211	unsigned long size;
	212	char *string = opt;
	213
	214	size = memparse(opt, &opt);
	215	if (MACHINE_HAS_EDAT1 && size == PMD_SIZE) {
	216	hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
	217	} else if (MACHINE_HAS_EDAT2 && size == PUD_SIZE) {
	218	hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
	219	} else {
	220	pr_err("hugepagesz= specifies an unsupported page size %s\n",
	221	string);
	222	return 0;
	223	}
	224	return 1;
ceb86879	225	}
d08de8e2	226	__setup("hugepagesz=", setup_hugepagesz);