[linux-block.git] / mm / page_table_check.c

// SPDX-License-Identifier: GPL-2.0

/*
 * Copyright (c) 2021, Google LLC.
 * Pasha Tatashin <pasha.tatashin@soleen.com>
 */
#include <linux/mm.h>
#include <linux/page_table_check.h>

#undef pr_fmt
#define pr_fmt(fmt)	"page_table_check: " fmt

struct page_table_check {
	atomic_t anon_map_count;
	atomic_t file_map_count;
};

static bool __page_table_check_enabled __initdata =
				IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED);

DEFINE_STATIC_KEY_TRUE(page_table_check_disabled);
EXPORT_SYMBOL(page_table_check_disabled);

static int __init early_page_table_check_param(char *buf)
{
	if (!buf)
		return -EINVAL;

	if (strcmp(buf, "on") == 0)
		__page_table_check_enabled = true;
	else if (strcmp(buf, "off") == 0)
		__page_table_check_enabled = false;

	return 0;
}

early_param("page_table_check", early_page_table_check_param);

static bool __init need_page_table_check(void)
{
	return __page_table_check_enabled;
}

static void __init init_page_table_check(void)
{
	if (!__page_table_check_enabled)
		return;
	static_branch_disable(&page_table_check_disabled);
}

struct page_ext_operations page_table_check_ops = {
	.size = sizeof(struct page_table_check),
	.need = need_page_table_check,
	.init = init_page_table_check,
};

static struct page_table_check *get_page_table_check(struct page_ext *page_ext)
{
	BUG_ON(!page_ext);
	return (void *)(page_ext) + page_table_check_ops.offset;
}

static inline bool pte_user_accessible_page(pte_t pte)
{
	return (pte_val(pte) & _PAGE_PRESENT) && (pte_val(pte) & _PAGE_USER);
}

static inline bool pmd_user_accessible_page(pmd_t pmd)
{
	return pmd_leaf(pmd) && (pmd_val(pmd) & _PAGE_PRESENT) &&
		(pmd_val(pmd) & _PAGE_USER);
}

static inline bool pud_user_accessible_page(pud_t pud)
{
	return pud_leaf(pud) && (pud_val(pud) & _PAGE_PRESENT) &&
		(pud_val(pud) & _PAGE_USER);
}

/*
 * An enty is removed from the page table, decrement the counters for that page
 * verify that it is of correct type and counters do not become negative.
 */
static void page_table_check_clear(struct mm_struct *mm, unsigned long addr,
				   unsigned long pfn, unsigned long pgcnt)
{
	struct page_ext *page_ext;
	struct page *page;
	unsigned long i;
	bool anon;

	if (!pfn_valid(pfn))
		return;

	page = pfn_to_page(pfn);
	page_ext = lookup_page_ext(page);
	anon = PageAnon(page);

	for (i = 0; i < pgcnt; i++) {
		struct page_table_check *ptc = get_page_table_check(page_ext);

		if (anon) {
			BUG_ON(atomic_read(&ptc->file_map_count));
			BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0);
		} else {
			BUG_ON(atomic_read(&ptc->anon_map_count));
			BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0);
		}
		page_ext = page_ext_next(page_ext);
	}
}

/*
 * A new enty is added to the page table, increment the counters for that page
 * verify that it is of correct type and is not being mapped with a different
 * type to a different process.
 */
static void page_table_check_set(struct mm_struct *mm, unsigned long addr,
				 unsigned long pfn, unsigned long pgcnt,
				 bool rw)
{
	struct page_ext *page_ext;
	struct page *page;
	unsigned long i;
	bool anon;

	if (!pfn_valid(pfn))
		return;

	page = pfn_to_page(pfn);
	page_ext = lookup_page_ext(page);
	anon = PageAnon(page);

	for (i = 0; i < pgcnt; i++) {
		struct page_table_check *ptc = get_page_table_check(page_ext);

		if (anon) {
			BUG_ON(atomic_read(&ptc->file_map_count));
			BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw);
		} else {
			BUG_ON(atomic_read(&ptc->anon_map_count));
			BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0);
		}
		page_ext = page_ext_next(page_ext);
	}
}

/*
 * page is on free list, or is being allocated, verify that counters are zeroes
 * crash if they are not.
 */
void __page_table_check_zero(struct page *page, unsigned int order)
{
	struct page_ext *page_ext = lookup_page_ext(page);
	unsigned long i;

	BUG_ON(!page_ext);
	for (i = 0; i < (1ul << order); i++) {
		struct page_table_check *ptc = get_page_table_check(page_ext);

		BUG_ON(atomic_read(&ptc->anon_map_count));
		BUG_ON(atomic_read(&ptc->file_map_count));
		page_ext = page_ext_next(page_ext);
	}
}

void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr,
				  pte_t pte)
{
	if (&init_mm == mm)
		return;

	if (pte_user_accessible_page(pte)) {
		page_table_check_clear(mm, addr, pte_pfn(pte),
				       PAGE_SIZE >> PAGE_SHIFT);
	}
}
EXPORT_SYMBOL(__page_table_check_pte_clear);

void __page_table_check_pmd_clear(struct mm_struct *mm, unsigned long addr,
				  pmd_t pmd)
{
	if (&init_mm == mm)
		return;

	if (pmd_user_accessible_page(pmd)) {
		page_table_check_clear(mm, addr, pmd_pfn(pmd),
				       PMD_PAGE_SIZE >> PAGE_SHIFT);
	}
}
EXPORT_SYMBOL(__page_table_check_pmd_clear);

void __page_table_check_pud_clear(struct mm_struct *mm, unsigned long addr,
				  pud_t pud)
{
	if (&init_mm == mm)
		return;

	if (pud_user_accessible_page(pud)) {
		page_table_check_clear(mm, addr, pud_pfn(pud),
				       PUD_PAGE_SIZE >> PAGE_SHIFT);
	}
}
EXPORT_SYMBOL(__page_table_check_pud_clear);

void __page_table_check_pte_set(struct mm_struct *mm, unsigned long addr,
				pte_t *ptep, pte_t pte)
{
	if (&init_mm == mm)
		return;

	__page_table_check_pte_clear(mm, addr, *ptep);
	if (pte_user_accessible_page(pte)) {
		page_table_check_set(mm, addr, pte_pfn(pte),
				     PAGE_SIZE >> PAGE_SHIFT,
				     pte_write(pte));
	}
}
EXPORT_SYMBOL(__page_table_check_pte_set);

void __page_table_check_pmd_set(struct mm_struct *mm, unsigned long addr,
				pmd_t *pmdp, pmd_t pmd)
{
	if (&init_mm == mm)
		return;

	__page_table_check_pmd_clear(mm, addr, *pmdp);
	if (pmd_user_accessible_page(pmd)) {
		page_table_check_set(mm, addr, pmd_pfn(pmd),
				     PMD_PAGE_SIZE >> PAGE_SHIFT,
				     pmd_write(pmd));
	}
}
EXPORT_SYMBOL(__page_table_check_pmd_set);

void __page_table_check_pud_set(struct mm_struct *mm, unsigned long addr,
				pud_t *pudp, pud_t pud)
{
	if (&init_mm == mm)
		return;

	__page_table_check_pud_clear(mm, addr, *pudp);
	if (pud_user_accessible_page(pud)) {
		page_table_check_set(mm, addr, pud_pfn(pud),
				     PUD_PAGE_SIZE >> PAGE_SHIFT,
				     pud_write(pud));
	}
}
EXPORT_SYMBOL(__page_table_check_pud_set);

void __page_table_check_pte_clear_range(struct mm_struct *mm,
					unsigned long addr,
					pmd_t pmd)
{
	if (&init_mm == mm)
		return;

	if (!pmd_bad(pmd) && !pmd_leaf(pmd)) {
		pte_t *ptep = pte_offset_map(&pmd, addr);
		unsigned long i;

		pte_unmap(ptep);
		for (i = 0; i < PTRS_PER_PTE; i++) {
			__page_table_check_pte_clear(mm, addr, *ptep);
			addr += PAGE_SIZE;
			ptep++;
		}
	}
}
Commit	Line	Data
df4e817b PT	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	/*
	4	* Copyright (c) 2021, Google LLC.
	5	* Pasha Tatashin <pasha.tatashin@soleen.com>
	6	*/
	7	#include <linux/mm.h>
	8	#include <linux/page_table_check.h>
	9
	10	#undef pr_fmt
	11	#define pr_fmt(fmt) "page_table_check: " fmt
	12
	13	struct page_table_check {
	14	atomic_t anon_map_count;
	15	atomic_t file_map_count;
	16	};
	17
	18	static bool __page_table_check_enabled __initdata =
	19	IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED);
	20
	21	DEFINE_STATIC_KEY_TRUE(page_table_check_disabled);
	22	EXPORT_SYMBOL(page_table_check_disabled);
	23
	24	static int __init early_page_table_check_param(char *buf)
	25	{
	26	if (!buf)
	27	return -EINVAL;
	28
	29	if (strcmp(buf, "on") == 0)
	30	__page_table_check_enabled = true;
	31	else if (strcmp(buf, "off") == 0)
	32	__page_table_check_enabled = false;
	33
	34	return 0;
	35	}
	36
	37	early_param("page_table_check", early_page_table_check_param);
	38
	39	static bool __init need_page_table_check(void)
	40	{
	41	return __page_table_check_enabled;
	42	}
	43
	44	static void __init init_page_table_check(void)
	45	{
	46	if (!__page_table_check_enabled)
	47	return;
	48	static_branch_disable(&page_table_check_disabled);
	49	}
	50
	51	struct page_ext_operations page_table_check_ops = {
	52	.size = sizeof(struct page_table_check),
	53	.need = need_page_table_check,
	54	.init = init_page_table_check,
	55	};
	56
	57	static struct page_table_check get_page_table_check(struct page_ext page_ext)
	58	{
	59	BUG_ON(!page_ext);
	60	return (void *)(page_ext) + page_table_check_ops.offset;
	61	}
	62
	63	static inline bool pte_user_accessible_page(pte_t pte)
	64	{
65	return (pte_val(pte) & _PAGE_PRESENT) && (pte_val(pte) & _PAGE_USER);
66	}
67
68	static inline bool pmd_user_accessible_page(pmd_t pmd)
69	{
70	return pmd_leaf(pmd) && (pmd_val(pmd) & _PAGE_PRESENT) &&
71	(pmd_val(pmd) & _PAGE_USER);
72	}
73
74	static inline bool pud_user_accessible_page(pud_t pud)
75	{
76	return pud_leaf(pud) && (pud_val(pud) & _PAGE_PRESENT) &&
77	(pud_val(pud) & _PAGE_USER);
78	}
79
80	/*
81	* An enty is removed from the page table, decrement the counters for that page
82	* verify that it is of correct type and counters do not become negative.
83	*/
84	static void page_table_check_clear(struct mm_struct *mm, unsigned long addr,
85	unsigned long pfn, unsigned long pgcnt)
86	{
87	struct page_ext *page_ext;
88	struct page *page;
64d8b9e1	89	unsigned long i;
df4e817b	90	bool anon;
df4e817b PT	91
	92	if (!pfn_valid(pfn))
	93	return;
	94
	95	page = pfn_to_page(pfn);
	96	page_ext = lookup_page_ext(page);
	97	anon = PageAnon(page);
	98
	99	for (i = 0; i < pgcnt; i++) {
	100	struct page_table_check *ptc = get_page_table_check(page_ext);
	101
	102	if (anon) {
	103	BUG_ON(atomic_read(&ptc->file_map_count));
	104	BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0);
	105	} else {
	106	BUG_ON(atomic_read(&ptc->anon_map_count));
	107	BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0);
	108	}
	109	page_ext = page_ext_next(page_ext);
	110	}
	111	}
	112
	113	/*
	114	* A new enty is added to the page table, increment the counters for that page
	115	* verify that it is of correct type and is not being mapped with a different
	116	* type to a different process.
	117	*/
	118	static void page_table_check_set(struct mm_struct *mm, unsigned long addr,
	119	unsigned long pfn, unsigned long pgcnt,
	120	bool rw)
	121	{
	122	struct page_ext *page_ext;
	123	struct page *page;
64d8b9e1	124	unsigned long i;
df4e817b	125	bool anon;
df4e817b PT	126
	127	if (!pfn_valid(pfn))
	128	return;
	129
	130	page = pfn_to_page(pfn);
	131	page_ext = lookup_page_ext(page);
	132	anon = PageAnon(page);
	133
	134	for (i = 0; i < pgcnt; i++) {
	135	struct page_table_check *ptc = get_page_table_check(page_ext);
	136
	137	if (anon) {
	138	BUG_ON(atomic_read(&ptc->file_map_count));
	139	BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw);
	140	} else {
	141	BUG_ON(atomic_read(&ptc->anon_map_count));
	142	BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0);
	143	}
	144	page_ext = page_ext_next(page_ext);
	145	}
	146	}
	147
	148	/*
	149	* page is on free list, or is being allocated, verify that counters are zeroes
	150	* crash if they are not.
	151	*/
	152	void __page_table_check_zero(struct page *page, unsigned int order)
	153	{
	154	struct page_ext *page_ext = lookup_page_ext(page);
64d8b9e1	155	unsigned long i;
df4e817b PT	156
df4e817b PT	157	BUG_ON(!page_ext);
64d8b9e1	158	for (i = 0; i < (1ul << order); i++) {
df4e817b PT	159	struct page_table_check *ptc = get_page_table_check(page_ext);
	160
	161	BUG_ON(atomic_read(&ptc->anon_map_count));
	162	BUG_ON(atomic_read(&ptc->file_map_count));
	163	page_ext = page_ext_next(page_ext);
	164	}
	165	}
	166
	167	void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr,
	168	pte_t pte)
	169	{
	170	if (&init_mm == mm)
	171	return;
	172
	173	if (pte_user_accessible_page(pte)) {
	174	page_table_check_clear(mm, addr, pte_pfn(pte),
	175	PAGE_SIZE >> PAGE_SHIFT);
	176	}
	177	}
	178	EXPORT_SYMBOL(__page_table_check_pte_clear);
	179
	180	void __page_table_check_pmd_clear(struct mm_struct *mm, unsigned long addr,
	181	pmd_t pmd)
	182	{
	183	if (&init_mm == mm)
	184	return;
	185
	186	if (pmd_user_accessible_page(pmd)) {
	187	page_table_check_clear(mm, addr, pmd_pfn(pmd),
	188	PMD_PAGE_SIZE >> PAGE_SHIFT);
	189	}
	190	}
	191	EXPORT_SYMBOL(__page_table_check_pmd_clear);
	192
	193	void __page_table_check_pud_clear(struct mm_struct *mm, unsigned long addr,
	194	pud_t pud)
	195	{
	196	if (&init_mm == mm)
	197	return;
	198
	199	if (pud_user_accessible_page(pud)) {
	200	page_table_check_clear(mm, addr, pud_pfn(pud),
	201	PUD_PAGE_SIZE >> PAGE_SHIFT);
	202	}
	203	}
	204	EXPORT_SYMBOL(__page_table_check_pud_clear);
	205
	206	void __page_table_check_pte_set(struct mm_struct *mm, unsigned long addr,
	207	pte_t *ptep, pte_t pte)
	208	{
df4e817b PT	209	if (&init_mm == mm)
	210	return;
	211
64d8b9e1	212	__page_table_check_pte_clear(mm, addr, *ptep);
df4e817b PT	213	if (pte_user_accessible_page(pte)) {
	214	page_table_check_set(mm, addr, pte_pfn(pte),
	215	PAGE_SIZE >> PAGE_SHIFT,
	216	pte_write(pte));
	217	}
	218	}
	219	EXPORT_SYMBOL(__page_table_check_pte_set);
	220
	221	void __page_table_check_pmd_set(struct mm_struct *mm, unsigned long addr,
	222	pmd_t *pmdp, pmd_t pmd)
	223	{
df4e817b PT	224	if (&init_mm == mm)
	225	return;
	226
64d8b9e1	227	__page_table_check_pmd_clear(mm, addr, *pmdp);
df4e817b PT	228	if (pmd_user_accessible_page(pmd)) {
	229	page_table_check_set(mm, addr, pmd_pfn(pmd),
	230	PMD_PAGE_SIZE >> PAGE_SHIFT,
	231	pmd_write(pmd));
	232	}
	233	}
	234	EXPORT_SYMBOL(__page_table_check_pmd_set);
	235
	236	void __page_table_check_pud_set(struct mm_struct *mm, unsigned long addr,
	237	pud_t *pudp, pud_t pud)
	238	{
df4e817b PT	239	if (&init_mm == mm)
	240	return;
	241
64d8b9e1	242	__page_table_check_pud_clear(mm, addr, *pudp);
df4e817b PT	243	if (pud_user_accessible_page(pud)) {
	244	page_table_check_set(mm, addr, pud_pfn(pud),
	245	PUD_PAGE_SIZE >> PAGE_SHIFT,
	246	pud_write(pud));
	247	}
	248	}
	249	EXPORT_SYMBOL(__page_table_check_pud_set);
80110bbf PT	250
	251	void __page_table_check_pte_clear_range(struct mm_struct *mm,
	252	unsigned long addr,
	253	pmd_t pmd)
	254	{
	255	if (&init_mm == mm)
	256	return;
	257
	258	if (!pmd_bad(pmd) && !pmd_leaf(pmd)) {
	259	pte_t *ptep = pte_offset_map(&pmd, addr);
	260	unsigned long i;
	261
	262	pte_unmap(ptep);
	263	for (i = 0; i < PTRS_PER_PTE; i++) {
	264	__page_table_check_pte_clear(mm, addr, *ptep);
	265	addr += PAGE_SIZE;
	266	ptep++;
	267	}
	268	}
	269	}