[linux-block.git] / mm / kfence / kfence.h

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Kernel Electric-Fence (KFENCE). For more info please see
 * Documentation/dev-tools/kfence.rst.
 *
 * Copyright (C) 2020, Google LLC.
 */

#ifndef MM_KFENCE_KFENCE_H
#define MM_KFENCE_KFENCE_H

#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/types.h>

#include "../slab.h" /* for struct kmem_cache */

/*
 * Get the canary byte pattern for @addr. Use a pattern that varies based on the
 * lower 3 bits of the address, to detect memory corruptions with higher
 * probability, where similar constants are used.
 */
#define KFENCE_CANARY_PATTERN(addr) ((u8)0xaa ^ (u8)((unsigned long)(addr) & 0x7))

/* Maximum stack depth for reports. */
#define KFENCE_STACK_DEPTH 64

/* KFENCE object states. */
enum kfence_object_state {
	KFENCE_OBJECT_UNUSED,		/* Object is unused. */
	KFENCE_OBJECT_ALLOCATED,	/* Object is currently allocated. */
	KFENCE_OBJECT_FREED,		/* Object was allocated, and then freed. */
};

/* Alloc/free tracking information. */
struct kfence_track {
	pid_t pid;
	int cpu;
	u64 ts_nsec;
	int num_stack_entries;
	unsigned long stack_entries[KFENCE_STACK_DEPTH];
};

/* KFENCE metadata per guarded allocation. */
struct kfence_metadata {
	struct list_head list;		/* Freelist node; access under kfence_freelist_lock. */
	struct rcu_head rcu_head;	/* For delayed freeing. */

	/*
	 * Lock protecting below data; to ensure consistency of the below data,
	 * since the following may execute concurrently: __kfence_alloc(),
	 * __kfence_free(), kfence_handle_page_fault(). However, note that we
	 * cannot grab the same metadata off the freelist twice, and multiple
	 * __kfence_alloc() cannot run concurrently on the same metadata.
	 */
	raw_spinlock_t lock;

	/* The current state of the object; see above. */
	enum kfence_object_state state;

	/*
	 * Allocated object address; cannot be calculated from size, because of
	 * alignment requirements.
	 *
	 * Invariant: ALIGN_DOWN(addr, PAGE_SIZE) is constant.
	 */
	unsigned long addr;

	/*
	 * The size of the original allocation.
	 */
	size_t size;

	/*
	 * The kmem_cache cache of the last allocation; NULL if never allocated
	 * or the cache has already been destroyed.
	 */
	struct kmem_cache *cache;

	/*
	 * In case of an invalid access, the page that was unprotected; we
	 * optimistically only store one address.
	 */
	unsigned long unprotected_page;

	/* Allocation and free stack information. */
	struct kfence_track alloc_track;
	struct kfence_track free_track;
	/* For updating alloc_covered on frees. */
	u32 alloc_stack_hash;
#ifdef CONFIG_MEMCG
	struct obj_cgroup *objcg;
#endif
};

extern struct kfence_metadata kfence_metadata[CONFIG_KFENCE_NUM_OBJECTS];

static inline struct kfence_metadata *addr_to_metadata(unsigned long addr)
{
	long index;

	/* The checks do not affect performance; only called from slow-paths. */

	if (!is_kfence_address((void *)addr))
		return NULL;

	/*
	 * May be an invalid index if called with an address at the edge of
	 * __kfence_pool, in which case we would report an "invalid access"
	 * error.
	 */
	index = (addr - (unsigned long)__kfence_pool) / (PAGE_SIZE * 2) - 1;
	if (index < 0 || index >= CONFIG_KFENCE_NUM_OBJECTS)
		return NULL;

	return &kfence_metadata[index];
}

/* KFENCE error types for report generation. */
enum kfence_error_type {
	KFENCE_ERROR_OOB,		/* Detected a out-of-bounds access. */
	KFENCE_ERROR_UAF,		/* Detected a use-after-free access. */
	KFENCE_ERROR_CORRUPTION,	/* Detected a memory corruption on free. */
	KFENCE_ERROR_INVALID,		/* Invalid access of unknown type. */
	KFENCE_ERROR_INVALID_FREE,	/* Invalid free. */
};

void kfence_report_error(unsigned long address, bool is_write, struct pt_regs *regs,
			 const struct kfence_metadata *meta, enum kfence_error_type type);

void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *meta);

#endif /* MM_KFENCE_KFENCE_H */
Commit	Line	Data
0ce20dd8 AP	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	/*
	3	* Kernel Electric-Fence (KFENCE). For more info please see
	4	* Documentation/dev-tools/kfence.rst.
	5	*
	6	* Copyright (C) 2020, Google LLC.
	7	*/
	8
	9	#ifndef MM_KFENCE_KFENCE_H
	10	#define MM_KFENCE_KFENCE_H
	11
	12	#include <linux/mm.h>
	13	#include <linux/slab.h>
	14	#include <linux/spinlock.h>
	15	#include <linux/types.h>
	16
	17	#include "../slab.h" /* for struct kmem_cache */
	18
0ce20dd8 AP	19	/*
	20	* Get the canary byte pattern for @addr. Use a pattern that varies based on the
	21	* lower 3 bits of the address, to detect memory corruptions with higher
	22	* probability, where similar constants are used.
	23	*/
	24	#define KFENCE_CANARY_PATTERN(addr) ((u8)0xaa ^ (u8)((unsigned long)(addr) & 0x7))
	25
	26	/* Maximum stack depth for reports. */
	27	#define KFENCE_STACK_DEPTH 64
	28
	29	/* KFENCE object states. */
	30	enum kfence_object_state {
	31	KFENCE_OBJECT_UNUSED, /* Object is unused. */
	32	KFENCE_OBJECT_ALLOCATED, /* Object is currently allocated. */
	33	KFENCE_OBJECT_FREED, /* Object was allocated, and then freed. */
	34	};
	35
	36	/* Alloc/free tracking information. */
	37	struct kfence_track {
	38	pid_t pid;
4bbf04aa ME	39	int cpu;
4bbf04aa ME	40	u64 ts_nsec;
0ce20dd8 AP	41	int num_stack_entries;
	42	unsigned long stack_entries[KFENCE_STACK_DEPTH];
	43	};
	44
	45	/* KFENCE metadata per guarded allocation. */
	46	struct kfence_metadata {
	47	struct list_head list; /* Freelist node; access under kfence_freelist_lock. */
	48	struct rcu_head rcu_head; /* For delayed freeing. */
	49
	50	/*
	51	* Lock protecting below data; to ensure consistency of the below data,
	52	* since the following may execute concurrently: __kfence_alloc(),
	53	* __kfence_free(), kfence_handle_page_fault(). However, note that we
	54	* cannot grab the same metadata off the freelist twice, and multiple
	55	* __kfence_alloc() cannot run concurrently on the same metadata.
	56	*/
	57	raw_spinlock_t lock;
	58
	59	/* The current state of the object; see above. */
	60	enum kfence_object_state state;
	61
	62	/*
	63	* Allocated object address; cannot be calculated from size, because of
	64	* alignment requirements.
	65	*
	66	* Invariant: ALIGN_DOWN(addr, PAGE_SIZE) is constant.
	67	*/
	68	unsigned long addr;
	69
	70	/*
	71	* The size of the original allocation.
	72	*/
	73	size_t size;
	74
	75	/*
	76	* The kmem_cache cache of the last allocation; NULL if never allocated
	77	* or the cache has already been destroyed.
	78	*/
	79	struct kmem_cache *cache;
	80
	81	/*
	82	* In case of an invalid access, the page that was unprotected; we
	83	* optimistically only store one address.
	84	*/
	85	unsigned long unprotected_page;
	86
	87	/* Allocation and free stack information. */
	88	struct kfence_track alloc_track;
	89	struct kfence_track free_track;
08f6b106 ME	90	/* For updating alloc_covered on frees. */
08f6b106 ME	91	u32 alloc_stack_hash;
8f0b3649 MS	92	#ifdef CONFIG_MEMCG
	93	struct obj_cgroup *objcg;
	94	#endif
0ce20dd8 AP	95	};
	96
	97	extern struct kfence_metadata kfence_metadata[CONFIG_KFENCE_NUM_OBJECTS];
	98
2dfe63e6 ME	99	static inline struct kfence_metadata *addr_to_metadata(unsigned long addr)
	100	{
	101	long index;
	102
	103	/* The checks do not affect performance; only called from slow-paths. */
	104
	105	if (!is_kfence_address((void *)addr))
	106	return NULL;
	107
	108	/*
	109	* May be an invalid index if called with an address at the edge of
	110	* __kfence_pool, in which case we would report an "invalid access"
	111	* error.
	112	*/
	113	index = (addr - (unsigned long)__kfence_pool) / (PAGE_SIZE * 2) - 1;
	114	if (index < 0 \|\| index >= CONFIG_KFENCE_NUM_OBJECTS)
	115	return NULL;
	116
	117	return &kfence_metadata[index];
	118	}
	119
0ce20dd8 AP	120	/* KFENCE error types for report generation. */
	121	enum kfence_error_type {
	122	KFENCE_ERROR_OOB, /* Detected a out-of-bounds access. */
	123	KFENCE_ERROR_UAF, /* Detected a use-after-free access. */
	124	KFENCE_ERROR_CORRUPTION, /* Detected a memory corruption on free. */
	125	KFENCE_ERROR_INVALID, /* Invalid access of unknown type. */
	126	KFENCE_ERROR_INVALID_FREE, /* Invalid free. */
	127	};
	128
bc8fbc5f	129	void kfence_report_error(unsigned long address, bool is_write, struct pt_regs *regs,
d438fabc	130	const struct kfence_metadata *meta, enum kfence_error_type type);
0ce20dd8 AP	131
	132	void kfence_print_object(struct seq_file seq, const struct kfence_metadata meta);
	133
	134	#endif /* MM_KFENCE_KFENCE_H */