[linux-2.6-block.git] / kernel / trace / trace_preemptirq.c

// SPDX-License-Identifier: GPL-2.0
/*
 * preemptoff and irqoff tracepoints
 *
 * Copyright (C) Joel Fernandes (Google) <joel@joelfernandes.org>
 */

#include <linux/kallsyms.h>
#include <linux/uaccess.h>
#include <linux/module.h>
#include <linux/ftrace.h>
#include <linux/kprobes.h>
#include "trace.h"

#define CREATE_TRACE_POINTS
#include <trace/events/preemptirq.h>

#ifdef CONFIG_TRACE_IRQFLAGS
/* Per-cpu variable to prevent redundant calls when IRQs already off */
static DEFINE_PER_CPU(int, tracing_irq_cpu);

/*
 * Like trace_hardirqs_on() but without the lockdep invocation. This is
 * used in the low level entry code where the ordering vs. RCU is important
 * and lockdep uses a staged approach which splits the lockdep hardirq
 * tracking into a RCU on and a RCU off section.
 */
void trace_hardirqs_on_prepare(void)
{
	if (this_cpu_read(tracing_irq_cpu)) {
		if (!in_nmi())
			trace_irq_enable(CALLER_ADDR0, CALLER_ADDR1);
		tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
		this_cpu_write(tracing_irq_cpu, 0);
	}
}
EXPORT_SYMBOL(trace_hardirqs_on_prepare);
NOKPROBE_SYMBOL(trace_hardirqs_on_prepare);

void trace_hardirqs_on(void)
{
	if (this_cpu_read(tracing_irq_cpu)) {
		if (!in_nmi())
			trace_irq_enable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
		tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
		this_cpu_write(tracing_irq_cpu, 0);
	}

	lockdep_hardirqs_on_prepare(CALLER_ADDR0);
	lockdep_hardirqs_on(CALLER_ADDR0);
}
EXPORT_SYMBOL(trace_hardirqs_on);
NOKPROBE_SYMBOL(trace_hardirqs_on);

/*
 * Like trace_hardirqs_off() but without the lockdep invocation. This is
 * used in the low level entry code where the ordering vs. RCU is important
 * and lockdep uses a staged approach which splits the lockdep hardirq
 * tracking into a RCU on and a RCU off section.
 */
void trace_hardirqs_off_finish(void)
{
	if (!this_cpu_read(tracing_irq_cpu)) {
		this_cpu_write(tracing_irq_cpu, 1);
		tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
		if (!in_nmi())
			trace_irq_disable(CALLER_ADDR0, CALLER_ADDR1);
	}

}
EXPORT_SYMBOL(trace_hardirqs_off_finish);
NOKPROBE_SYMBOL(trace_hardirqs_off_finish);

void trace_hardirqs_off(void)
{
	lockdep_hardirqs_off(CALLER_ADDR0);

	if (!this_cpu_read(tracing_irq_cpu)) {
		this_cpu_write(tracing_irq_cpu, 1);
		tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
		if (!in_nmi())
			trace_irq_disable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
	}
}
EXPORT_SYMBOL(trace_hardirqs_off);
NOKPROBE_SYMBOL(trace_hardirqs_off);

__visible void trace_hardirqs_on_caller(unsigned long caller_addr)
{
	if (this_cpu_read(tracing_irq_cpu)) {
		if (!in_nmi())
			trace_irq_enable_rcuidle(CALLER_ADDR0, caller_addr);
		tracer_hardirqs_on(CALLER_ADDR0, caller_addr);
		this_cpu_write(tracing_irq_cpu, 0);
	}

	lockdep_hardirqs_on_prepare(CALLER_ADDR0);
	lockdep_hardirqs_on(CALLER_ADDR0);
}
EXPORT_SYMBOL(trace_hardirqs_on_caller);
NOKPROBE_SYMBOL(trace_hardirqs_on_caller);

__visible void trace_hardirqs_off_caller(unsigned long caller_addr)
{
	lockdep_hardirqs_off(CALLER_ADDR0);

	if (!this_cpu_read(tracing_irq_cpu)) {
		this_cpu_write(tracing_irq_cpu, 1);
		tracer_hardirqs_off(CALLER_ADDR0, caller_addr);
		if (!in_nmi())
			trace_irq_disable_rcuidle(CALLER_ADDR0, caller_addr);
	}
}
EXPORT_SYMBOL(trace_hardirqs_off_caller);
NOKPROBE_SYMBOL(trace_hardirqs_off_caller);
#endif /* CONFIG_TRACE_IRQFLAGS */

#ifdef CONFIG_TRACE_PREEMPT_TOGGLE

void trace_preempt_on(unsigned long a0, unsigned long a1)
{
	if (!in_nmi())
		trace_preempt_enable_rcuidle(a0, a1);
	tracer_preempt_on(a0, a1);
}

void trace_preempt_off(unsigned long a0, unsigned long a1)
{
	if (!in_nmi())
		trace_preempt_disable_rcuidle(a0, a1);
	tracer_preempt_off(a0, a1);
}
#endif
Commit	Line	Data
c3bc8fd6 JFG	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* preemptoff and irqoff tracepoints
	4	*
	5	* Copyright (C) Joel Fernandes (Google) <joel@joelfernandes.org>
	6	*/
	7
	8	#include <linux/kallsyms.h>
	9	#include <linux/uaccess.h>
	10	#include <linux/module.h>
	11	#include <linux/ftrace.h>
eeeb080b	12	#include <linux/kprobes.h>
3f1756dc	13	#include "trace.h"
c3bc8fd6 JFG	14
	15	#define CREATE_TRACE_POINTS
	16	#include <trace/events/preemptirq.h>
	17
	18	#ifdef CONFIG_TRACE_IRQFLAGS
	19	/* Per-cpu variable to prevent redundant calls when IRQs already off */
	20	static DEFINE_PER_CPU(int, tracing_irq_cpu);
	21
0995a5df TG	22	/*
	23	* Like trace_hardirqs_on() but without the lockdep invocation. This is
	24	* used in the low level entry code where the ordering vs. RCU is important
	25	* and lockdep uses a staged approach which splits the lockdep hardirq
	26	* tracking into a RCU on and a RCU off section.
	27	*/
	28	void trace_hardirqs_on_prepare(void)
	29	{
	30	if (this_cpu_read(tracing_irq_cpu)) {
	31	if (!in_nmi())
	32	trace_irq_enable(CALLER_ADDR0, CALLER_ADDR1);
	33	tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
	34	this_cpu_write(tracing_irq_cpu, 0);
	35	}
	36	}
	37	EXPORT_SYMBOL(trace_hardirqs_on_prepare);
	38	NOKPROBE_SYMBOL(trace_hardirqs_on_prepare);
	39
c3bc8fd6 JFG	40	void trace_hardirqs_on(void)
c3bc8fd6 JFG	41	{
bff1b208	42	if (this_cpu_read(tracing_irq_cpu)) {
3f1756dc SRV	43	if (!in_nmi())
	44	trace_irq_enable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
	45	tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
bff1b208 SRV	46	this_cpu_write(tracing_irq_cpu, 0);
bff1b208 SRV	47	}
c3bc8fd6	48
c86e9b98	49	lockdep_hardirqs_on_prepare(CALLER_ADDR0);
bff1b208	50	lockdep_hardirqs_on(CALLER_ADDR0);
c3bc8fd6 JFG	51	}
c3bc8fd6 JFG	52	EXPORT_SYMBOL(trace_hardirqs_on);
eeeb080b	53	NOKPROBE_SYMBOL(trace_hardirqs_on);
c3bc8fd6	54
0995a5df TG	55	/*
	56	* Like trace_hardirqs_off() but without the lockdep invocation. This is
	57	* used in the low level entry code where the ordering vs. RCU is important
	58	* and lockdep uses a staged approach which splits the lockdep hardirq
	59	* tracking into a RCU on and a RCU off section.
	60	*/
bf2b3008	61	void trace_hardirqs_off_finish(void)
0995a5df TG	62	{
	63	if (!this_cpu_read(tracing_irq_cpu)) {
	64	this_cpu_write(tracing_irq_cpu, 1);
	65	tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
	66	if (!in_nmi())
	67	trace_irq_disable(CALLER_ADDR0, CALLER_ADDR1);
	68	}
	69
	70	}
bf2b3008 PZ	71	EXPORT_SYMBOL(trace_hardirqs_off_finish);
bf2b3008 PZ	72	NOKPROBE_SYMBOL(trace_hardirqs_off_finish);
0995a5df	73
c3bc8fd6 JFG	74	void trace_hardirqs_off(void)
c3bc8fd6 JFG	75	{
bf2b3008 PZ	76	lockdep_hardirqs_off(CALLER_ADDR0);
bf2b3008 PZ	77
bff1b208 SRV	78	if (!this_cpu_read(tracing_irq_cpu)) {
bff1b208 SRV	79	this_cpu_write(tracing_irq_cpu, 1);
3f1756dc SRV	80	tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
	81	if (!in_nmi())
	82	trace_irq_disable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
bff1b208	83	}
c3bc8fd6 JFG	84	}
c3bc8fd6 JFG	85	EXPORT_SYMBOL(trace_hardirqs_off);
eeeb080b	86	NOKPROBE_SYMBOL(trace_hardirqs_off);
c3bc8fd6 JFG	87
	88	__visible void trace_hardirqs_on_caller(unsigned long caller_addr)
	89	{
bff1b208	90	if (this_cpu_read(tracing_irq_cpu)) {
3f1756dc SRV	91	if (!in_nmi())
	92	trace_irq_enable_rcuidle(CALLER_ADDR0, caller_addr);
	93	tracer_hardirqs_on(CALLER_ADDR0, caller_addr);
bff1b208 SRV	94	this_cpu_write(tracing_irq_cpu, 0);
bff1b208 SRV	95	}
c3bc8fd6	96
c86e9b98	97	lockdep_hardirqs_on_prepare(CALLER_ADDR0);
bff1b208	98	lockdep_hardirqs_on(CALLER_ADDR0);
c3bc8fd6 JFG	99	}
c3bc8fd6 JFG	100	EXPORT_SYMBOL(trace_hardirqs_on_caller);
eeeb080b	101	NOKPROBE_SYMBOL(trace_hardirqs_on_caller);
c3bc8fd6 JFG	102
	103	__visible void trace_hardirqs_off_caller(unsigned long caller_addr)
	104	{
73ac74c7 SS	105	lockdep_hardirqs_off(CALLER_ADDR0);
73ac74c7 SS	106
bff1b208 SRV	107	if (!this_cpu_read(tracing_irq_cpu)) {
bff1b208 SRV	108	this_cpu_write(tracing_irq_cpu, 1);
3f1756dc SRV	109	tracer_hardirqs_off(CALLER_ADDR0, caller_addr);
	110	if (!in_nmi())
	111	trace_irq_disable_rcuidle(CALLER_ADDR0, caller_addr);
bff1b208	112	}
c3bc8fd6 JFG	113	}
c3bc8fd6 JFG	114	EXPORT_SYMBOL(trace_hardirqs_off_caller);
eeeb080b	115	NOKPROBE_SYMBOL(trace_hardirqs_off_caller);
c3bc8fd6 JFG	116	#endif /* CONFIG_TRACE_IRQFLAGS */
	117
	118	#ifdef CONFIG_TRACE_PREEMPT_TOGGLE
	119
	120	void trace_preempt_on(unsigned long a0, unsigned long a1)
	121	{
3f1756dc SRV	122	if (!in_nmi())
	123	trace_preempt_enable_rcuidle(a0, a1);
	124	tracer_preempt_on(a0, a1);
c3bc8fd6 JFG	125	}
	126
	127	void trace_preempt_off(unsigned long a0, unsigned long a1)
	128	{
3f1756dc SRV	129	if (!in_nmi())
	130	trace_preempt_disable_rcuidle(a0, a1);
	131	tracer_preempt_off(a0, a1);
c3bc8fd6 JFG	132	}
c3bc8fd6 JFG	133	#endif