[linux-block.git] / mm / kasan / tags.c

// SPDX-License-Identifier: GPL-2.0
/*
 * This file contains common tag-based KASAN code.
 *
 * Copyright (c) 2018 Google, Inc.
 * Copyright (c) 2020 Google, Inc.
 */

#include <linux/atomic.h>
#include <linux/init.h>
#include <linux/kasan.h>
#include <linux/kernel.h>
#include <linux/memblock.h>
#include <linux/memory.h>
#include <linux/mm.h>
#include <linux/static_key.h>
#include <linux/string.h>
#include <linux/types.h>

#include "kasan.h"
#include "../slab.h"

#define KASAN_STACK_RING_SIZE_DEFAULT (32 << 10)

enum kasan_arg_stacktrace {
	KASAN_ARG_STACKTRACE_DEFAULT,
	KASAN_ARG_STACKTRACE_OFF,
	KASAN_ARG_STACKTRACE_ON,
};

static enum kasan_arg_stacktrace kasan_arg_stacktrace __initdata;

/* Whether to collect alloc/free stack traces. */
DEFINE_STATIC_KEY_TRUE(kasan_flag_stacktrace);

/* Non-zero, as initial pointer values are 0. */
#define STACK_RING_BUSY_PTR ((void *)1)

struct kasan_stack_ring stack_ring = {
	.lock = __RW_LOCK_UNLOCKED(stack_ring.lock)
};

/* kasan.stacktrace=off/on */
static int __init early_kasan_flag_stacktrace(char *arg)
{
	if (!arg)
		return -EINVAL;

	if (!strcmp(arg, "off"))
		kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_OFF;
	else if (!strcmp(arg, "on"))
		kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_ON;
	else
		return -EINVAL;

	return 0;
}
early_param("kasan.stacktrace", early_kasan_flag_stacktrace);

/* kasan.stack_ring_size=<number of entries> */
static int __init early_kasan_flag_stack_ring_size(char *arg)
{
	if (!arg)
		return -EINVAL;

	return kstrtoul(arg, 0, &stack_ring.size);
}
early_param("kasan.stack_ring_size", early_kasan_flag_stack_ring_size);

void __init kasan_init_tags(void)
{
	switch (kasan_arg_stacktrace) {
	case KASAN_ARG_STACKTRACE_DEFAULT:
		/* Default is specified by kasan_flag_stacktrace definition. */
		break;
	case KASAN_ARG_STACKTRACE_OFF:
		static_branch_disable(&kasan_flag_stacktrace);
		break;
	case KASAN_ARG_STACKTRACE_ON:
		static_branch_enable(&kasan_flag_stacktrace);
		break;
	}

	if (kasan_stack_collection_enabled()) {
		if (!stack_ring.size)
			stack_ring.size = KASAN_STACK_RING_SIZE_DEFAULT;
		stack_ring.entries = memblock_alloc(
			sizeof(stack_ring.entries[0]) * stack_ring.size,
			SMP_CACHE_BYTES);
		if (WARN_ON(!stack_ring.entries))
			static_branch_disable(&kasan_flag_stacktrace);
	}
}

static void save_stack_info(struct kmem_cache *cache, void *object,
			gfp_t gfp_flags, bool is_free)
{
	unsigned long flags;
	depot_stack_handle_t stack;
	u64 pos;
	struct kasan_stack_ring_entry *entry;
	void *old_ptr;

	stack = kasan_save_stack(gfp_flags, true);

	/*
	 * Prevent save_stack_info() from modifying stack ring
	 * when kasan_complete_mode_report_info() is walking it.
	 */
	read_lock_irqsave(&stack_ring.lock, flags);

next:
	pos = atomic64_fetch_add(1, &stack_ring.pos);
	entry = &stack_ring.entries[pos % stack_ring.size];

	/* Detect stack ring entry slots that are being written to. */
	old_ptr = READ_ONCE(entry->ptr);
	if (old_ptr == STACK_RING_BUSY_PTR)
		goto next; /* Busy slot. */
	if (!try_cmpxchg(&entry->ptr, &old_ptr, STACK_RING_BUSY_PTR))
		goto next; /* Busy slot. */

	WRITE_ONCE(entry->size, cache->object_size);
	WRITE_ONCE(entry->pid, current->pid);
	WRITE_ONCE(entry->stack, stack);
	WRITE_ONCE(entry->is_free, is_free);

	/*
	 * Paired with smp_load_acquire() in kasan_complete_mode_report_info().
	 */
	smp_store_release(&entry->ptr, (s64)object);

	read_unlock_irqrestore(&stack_ring.lock, flags);
}

void kasan_save_alloc_info(struct kmem_cache *cache, void *object, gfp_t flags)
{
	save_stack_info(cache, object, flags, false);
}

void kasan_save_free_info(struct kmem_cache *cache, void *object)
{
	save_stack_info(cache, object, GFP_NOWAIT, true);
}
Commit	Line	Data
a0503b8a KYL	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* This file contains common tag-based KASAN code.
	4	*
	5	* Copyright (c) 2018 Google, Inc.
	6	* Copyright (c) 2020 Google, Inc.
	7	*/
	8
7bc0584e	9	#include <linux/atomic.h>
a0503b8a KYL	10	#include <linux/init.h>
	11	#include <linux/kasan.h>
	12	#include <linux/kernel.h>
80b92bfe	13	#include <linux/memblock.h>
a0503b8a KYL	14	#include <linux/memory.h>
	15	#include <linux/mm.h>
	16	#include <linux/static_key.h>
	17	#include <linux/string.h>
	18	#include <linux/types.h>
	19
	20	#include "kasan.h"
7bc0584e AK	21	#include "../slab.h"
7bc0584e AK	22
80b92bfe AK	23	#define KASAN_STACK_RING_SIZE_DEFAULT (32 << 10)
80b92bfe AK	24
7ebfce33 AK	25	enum kasan_arg_stacktrace {
	26	KASAN_ARG_STACKTRACE_DEFAULT,
	27	KASAN_ARG_STACKTRACE_OFF,
	28	KASAN_ARG_STACKTRACE_ON,
	29	};
	30
	31	static enum kasan_arg_stacktrace kasan_arg_stacktrace __initdata;
	32
	33	/* Whether to collect alloc/free stack traces. */
	34	DEFINE_STATIC_KEY_TRUE(kasan_flag_stacktrace);
	35
7bc0584e AK	36	/* Non-zero, as initial pointer values are 0. */
	37	#define STACK_RING_BUSY_PTR ((void *)1)
	38
	39	struct kasan_stack_ring stack_ring = {
	40	.lock = __RW_LOCK_UNLOCKED(stack_ring.lock)
	41	};
	42
7ebfce33 AK	43	/* kasan.stacktrace=off/on */
	44	static int __init early_kasan_flag_stacktrace(char *arg)
	45	{
	46	if (!arg)
	47	return -EINVAL;
	48
	49	if (!strcmp(arg, "off"))
	50	kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_OFF;
	51	else if (!strcmp(arg, "on"))
	52	kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_ON;
	53	else
	54	return -EINVAL;
	55
	56	return 0;
	57	}
	58	early_param("kasan.stacktrace", early_kasan_flag_stacktrace);
	59
80b92bfe AK	60	/* kasan.stack_ring_size=<number of entries> */
	61	static int __init early_kasan_flag_stack_ring_size(char *arg)
	62	{
	63	if (!arg)
	64	return -EINVAL;
	65
	66	return kstrtoul(arg, 0, &stack_ring.size);
	67	}
	68	early_param("kasan.stack_ring_size", early_kasan_flag_stack_ring_size);
	69
7ebfce33 AK	70	void __init kasan_init_tags(void)
	71	{
	72	switch (kasan_arg_stacktrace) {
	73	case KASAN_ARG_STACKTRACE_DEFAULT:
	74	/* Default is specified by kasan_flag_stacktrace definition. */
	75	break;
	76	case KASAN_ARG_STACKTRACE_OFF:
	77	static_branch_disable(&kasan_flag_stacktrace);
	78	break;
	79	case KASAN_ARG_STACKTRACE_ON:
	80	static_branch_enable(&kasan_flag_stacktrace);
	81	break;
	82	}
80b92bfe AK	83
	84	if (kasan_stack_collection_enabled()) {
	85	if (!stack_ring.size)
	86	stack_ring.size = KASAN_STACK_RING_SIZE_DEFAULT;
	87	stack_ring.entries = memblock_alloc(
	88	sizeof(stack_ring.entries[0]) * stack_ring.size,
	89	SMP_CACHE_BYTES);
	90	if (WARN_ON(!stack_ring.entries))
	91	static_branch_disable(&kasan_flag_stacktrace);
	92	}
7ebfce33 AK	93	}
7ebfce33 AK	94
7bc0584e AK	95	static void save_stack_info(struct kmem_cache cache, void object,
	96	gfp_t gfp_flags, bool is_free)
	97	{
	98	unsigned long flags;
	99	depot_stack_handle_t stack;
	100	u64 pos;
	101	struct kasan_stack_ring_entry *entry;
	102	void *old_ptr;
	103
	104	stack = kasan_save_stack(gfp_flags, true);
	105
	106	/*
	107	* Prevent save_stack_info() from modifying stack ring
	108	* when kasan_complete_mode_report_info() is walking it.
	109	*/
	110	read_lock_irqsave(&stack_ring.lock, flags);
	111
	112	next:
	113	pos = atomic64_fetch_add(1, &stack_ring.pos);
80b92bfe	114	entry = &stack_ring.entries[pos % stack_ring.size];
7bc0584e AK	115
	116	/* Detect stack ring entry slots that are being written to. */
	117	old_ptr = READ_ONCE(entry->ptr);
	118	if (old_ptr == STACK_RING_BUSY_PTR)
	119	goto next; /* Busy slot. */
	120	if (!try_cmpxchg(&entry->ptr, &old_ptr, STACK_RING_BUSY_PTR))
	121	goto next; /* Busy slot. */
	122
	123	WRITE_ONCE(entry->size, cache->object_size);
	124	WRITE_ONCE(entry->pid, current->pid);
	125	WRITE_ONCE(entry->stack, stack);
	126	WRITE_ONCE(entry->is_free, is_free);
	127
	128	/*
	129	* Paired with smp_load_acquire() in kasan_complete_mode_report_info().
	130	*/
	131	smp_store_release(&entry->ptr, (s64)object);
	132
	133	read_unlock_irqrestore(&stack_ring.lock, flags);
	134	}
a0503b8a	135
ccf643e6 AK	136	void kasan_save_alloc_info(struct kmem_cache cache, void object, gfp_t flags)
ccf643e6 AK	137	{
7bc0584e	138	save_stack_info(cache, object, flags, false);
ccf643e6 AK	139	}
ccf643e6 AK	140
6b074349	141	void kasan_save_free_info(struct kmem_cache cache, void object)
a0503b8a	142	{
7bc0584e	143	save_stack_info(cache, object, GFP_NOWAIT, true);
a0503b8a	144	}