[linux-2.6-block.git] / kernel / trace / ring_buffer_benchmark.c

// SPDX-License-Identifier: GPL-2.0
/*
 * ring buffer tester and benchmark
 *
 * Copyright (C) 2009 Steven Rostedt <srostedt@redhat.com>
 */
#include <linux/ring_buffer.h>
#include <linux/completion.h>
#include <linux/kthread.h>
#include <uapi/linux/sched/types.h>
#include <linux/module.h>
#include <linux/ktime.h>
#include <asm/local.h>

struct rb_page {
	u64		ts;
	local_t		commit;
	char		data[4080];
};

/* run time and sleep time in seconds */
#define RUN_TIME	10ULL
#define SLEEP_TIME	10

/* number of events for writer to wake up the reader */
static int wakeup_interval = 100;

static int reader_finish;
static DECLARE_COMPLETION(read_start);
static DECLARE_COMPLETION(read_done);

static struct trace_buffer *buffer;
static struct task_struct *producer;
static struct task_struct *consumer;
static unsigned long read;

static unsigned int disable_reader;
module_param(disable_reader, uint, 0644);
MODULE_PARM_DESC(disable_reader, "only run producer");

static unsigned int write_iteration = 50;
module_param(write_iteration, uint, 0644);
MODULE_PARM_DESC(write_iteration, "# of writes between timestamp readings");

static int producer_nice = MAX_NICE;
static int consumer_nice = MAX_NICE;

static int producer_fifo;
static int consumer_fifo;

module_param(producer_nice, int, 0644);
MODULE_PARM_DESC(producer_nice, "nice prio for producer");

module_param(consumer_nice, int, 0644);
MODULE_PARM_DESC(consumer_nice, "nice prio for consumer");

module_param(producer_fifo, int, 0644);
MODULE_PARM_DESC(producer_fifo, "use fifo for producer: 0 - disabled, 1 - low prio, 2 - fifo");

module_param(consumer_fifo, int, 0644);
MODULE_PARM_DESC(consumer_fifo, "use fifo for consumer: 0 - disabled, 1 - low prio, 2 - fifo");

static int read_events;

static int test_error;

#define TEST_ERROR()				\
	do {					\
		if (!test_error) {		\
			test_error = 1;		\
			WARN_ON(1);		\
		}				\
	} while (0)

enum event_status {
	EVENT_FOUND,
	EVENT_DROPPED,
};

static bool break_test(void)
{
	return test_error || kthread_should_stop();
}

static enum event_status read_event(int cpu)
{
	struct ring_buffer_event *event;
	int *entry;
	u64 ts;

	event = ring_buffer_consume(buffer, cpu, &ts, NULL);
	if (!event)
		return EVENT_DROPPED;

	entry = ring_buffer_event_data(event);
	if (*entry != cpu) {
		TEST_ERROR();
		return EVENT_DROPPED;
	}

	read++;
	return EVENT_FOUND;
}

static enum event_status read_page(int cpu)
{
	struct buffer_data_read_page *bpage;
	struct ring_buffer_event *event;
	struct rb_page *rpage;
	unsigned long commit;
	int page_size;
	int *entry;
	int ret;
	int inc;
	int i;

	bpage = ring_buffer_alloc_read_page(buffer, cpu);
	if (IS_ERR(bpage))
		return EVENT_DROPPED;

	page_size = ring_buffer_subbuf_size_get(buffer);
	ret = ring_buffer_read_page(buffer, bpage, page_size, cpu, 1);
	if (ret >= 0) {
		rpage = ring_buffer_read_page_data(bpage);
		/* The commit may have missed event flags set, clear them */
		commit = local_read(&rpage->commit) & 0xfffff;
		for (i = 0; i < commit && !test_error ; i += inc) {

			if (i >= (page_size - offsetof(struct rb_page, data))) {
				TEST_ERROR();
				break;
			}

			inc = -1;
			event = (void *)&rpage->data[i];
			switch (event->type_len) {
			case RINGBUF_TYPE_PADDING:
				/* failed writes may be discarded events */
				if (!event->time_delta)
					TEST_ERROR();
				inc = event->array[0] + 4;
				break;
			case RINGBUF_TYPE_TIME_EXTEND:
				inc = 8;
				break;
			case 0:
				entry = ring_buffer_event_data(event);
				if (*entry != cpu) {
					TEST_ERROR();
					break;
				}
				read++;
				if (!event->array[0]) {
					TEST_ERROR();
					break;
				}
				inc = event->array[0] + 4;
				break;
			default:
				entry = ring_buffer_event_data(event);
				if (*entry != cpu) {
					TEST_ERROR();
					break;
				}
				read++;
				inc = ((event->type_len + 1) * 4);
			}
			if (test_error)
				break;

			if (inc <= 0) {
				TEST_ERROR();
				break;
			}
		}
	}
	ring_buffer_free_read_page(buffer, cpu, bpage);

	if (ret < 0)
		return EVENT_DROPPED;
	return EVENT_FOUND;
}

static void ring_buffer_consumer(void)
{
	/* toggle between reading pages and events */
	read_events ^= 1;

	read = 0;
	/*
	 * Continue running until the producer specifically asks to stop
	 * and is ready for the completion.
	 */
	while (!READ_ONCE(reader_finish)) {
		int found = 1;

		while (found && !test_error) {
			int cpu;

			found = 0;
			for_each_online_cpu(cpu) {
				enum event_status stat;

				if (read_events)
					stat = read_event(cpu);
				else
					stat = read_page(cpu);

				if (test_error)
					break;

				if (stat == EVENT_FOUND)
					found = 1;

			}
		}

		/* Wait till the producer wakes us up when there is more data
		 * available or when the producer wants us to finish reading.
		 */
		set_current_state(TASK_INTERRUPTIBLE);
		if (reader_finish)
			break;

		schedule();
	}
	__set_current_state(TASK_RUNNING);
	reader_finish = 0;
	complete(&read_done);
}

static void ring_buffer_producer(void)
{
	ktime_t start_time, end_time, timeout;
	unsigned long long time;
	unsigned long long entries;
	unsigned long long overruns;
	unsigned long missed = 0;
	unsigned long hit = 0;
	unsigned long avg;
	int cnt = 0;

	/*
	 * Hammer the buffer for 10 secs (this may
	 * make the system stall)
	 */
	trace_printk("Starting ring buffer hammer\n");
	start_time = ktime_get();
	timeout = ktime_add_ns(start_time, RUN_TIME * NSEC_PER_SEC);
	do {
		struct ring_buffer_event *event;
		int *entry;
		int i;

		for (i = 0; i < write_iteration; i++) {
			event = ring_buffer_lock_reserve(buffer, 10);
			if (!event) {
				missed++;
			} else {
				hit++;
				entry = ring_buffer_event_data(event);
				*entry = smp_processor_id();
				ring_buffer_unlock_commit(buffer);
			}
		}
		end_time = ktime_get();

		cnt++;
		if (consumer && !(cnt % wakeup_interval))
			wake_up_process(consumer);

#ifndef CONFIG_PREEMPTION
		/*
		 * If we are a non preempt kernel, the 10 seconds run will
		 * stop everything while it runs. Instead, we will call
		 * cond_resched and also add any time that was lost by a
		 * reschedule.
		 *
		 * Do a cond resched at the same frequency we would wake up
		 * the reader.
		 */
		if (cnt % wakeup_interval)
			cond_resched();
#endif
	} while (ktime_before(end_time, timeout) && !break_test());
	trace_printk("End ring buffer hammer\n");

	if (consumer) {
		/* Init both completions here to avoid races */
		init_completion(&read_start);
		init_completion(&read_done);
		/* the completions must be visible before the finish var */
		smp_wmb();
		reader_finish = 1;
		wake_up_process(consumer);
		wait_for_completion(&read_done);
	}

	time = ktime_us_delta(end_time, start_time);

	entries = ring_buffer_entries(buffer);
	overruns = ring_buffer_overruns(buffer);

	if (test_error)
		trace_printk("ERROR!\n");

	if (!disable_reader) {
		if (consumer_fifo)
			trace_printk("Running Consumer at SCHED_FIFO %s\n",
				     consumer_fifo == 1 ? "low" : "high");
		else
			trace_printk("Running Consumer at nice: %d\n",
				     consumer_nice);
	}
	if (producer_fifo)
		trace_printk("Running Producer at SCHED_FIFO %s\n",
			     producer_fifo == 1 ? "low" : "high");
	else
		trace_printk("Running Producer at nice: %d\n",
			     producer_nice);

	/* Let the user know that the test is running at low priority */
	if (!producer_fifo && !consumer_fifo &&
	    producer_nice == MAX_NICE && consumer_nice == MAX_NICE)
		trace_printk("WARNING!!! This test is running at lowest priority.\n");

	trace_printk("Time:     %lld (usecs)\n", time);
	trace_printk("Overruns: %lld\n", overruns);
	if (disable_reader)
		trace_printk("Read:     (reader disabled)\n");
	else
		trace_printk("Read:     %ld  (by %s)\n", read,
			read_events ? "events" : "pages");
	trace_printk("Entries:  %lld\n", entries);
	trace_printk("Total:    %lld\n", entries + overruns + read);
	trace_printk("Missed:   %ld\n", missed);
	trace_printk("Hit:      %ld\n", hit);

	/* Convert time from usecs to millisecs */
	do_div(time, USEC_PER_MSEC);
	if (time)
		hit /= (long)time;
	else
		trace_printk("TIME IS ZERO??\n");

	trace_printk("Entries per millisec: %ld\n", hit);

	if (hit) {
		/* Calculate the average time in nanosecs */
		avg = NSEC_PER_MSEC / hit;
		trace_printk("%ld ns per entry\n", avg);
	}

	if (missed) {
		if (time)
			missed /= (long)time;

		trace_printk("Total iterations per millisec: %ld\n",
			     hit + missed);

		/* it is possible that hit + missed will overflow and be zero */
		if (!(hit + missed)) {
			trace_printk("hit + missed overflowed and totalled zero!\n");
			hit--; /* make it non zero */
		}

		/* Calculate the average time in nanosecs */
		avg = NSEC_PER_MSEC / (hit + missed);
		trace_printk("%ld ns per entry\n", avg);
	}
}

static void wait_to_die(void)
{
	set_current_state(TASK_INTERRUPTIBLE);
	while (!kthread_should_stop()) {
		schedule();
		set_current_state(TASK_INTERRUPTIBLE);
	}
	__set_current_state(TASK_RUNNING);
}

static int ring_buffer_consumer_thread(void *arg)
{
	while (!break_test()) {
		complete(&read_start);

		ring_buffer_consumer();

		set_current_state(TASK_INTERRUPTIBLE);
		if (break_test())
			break;
		schedule();
	}
	__set_current_state(TASK_RUNNING);

	if (!kthread_should_stop())
		wait_to_die();

	return 0;
}

static int ring_buffer_producer_thread(void *arg)
{
	while (!break_test()) {
		ring_buffer_reset(buffer);

		if (consumer) {
			wake_up_process(consumer);
			wait_for_completion(&read_start);
		}

		ring_buffer_producer();
		if (break_test())
			goto out_kill;

		trace_printk("Sleeping for 10 secs\n");
		set_current_state(TASK_INTERRUPTIBLE);
		if (break_test())
			goto out_kill;
		schedule_timeout(HZ * SLEEP_TIME);
	}

out_kill:
	__set_current_state(TASK_RUNNING);
	if (!kthread_should_stop())
		wait_to_die();

	return 0;
}

static int __init ring_buffer_benchmark_init(void)
{
	int ret;

	/* make a one meg buffer in overwite mode */
	buffer = ring_buffer_alloc(1000000, RB_FL_OVERWRITE);
	if (!buffer)
		return -ENOMEM;

	if (!disable_reader) {
		consumer = kthread_create(ring_buffer_consumer_thread,
					  NULL, "rb_consumer");
		ret = PTR_ERR(consumer);
		if (IS_ERR(consumer))
			goto out_fail;
	}

	producer = kthread_run(ring_buffer_producer_thread,
			       NULL, "rb_producer");
	ret = PTR_ERR(producer);

	if (IS_ERR(producer))
		goto out_kill;

	/*
	 * Run them as low-prio background tasks by default:
	 */
	if (!disable_reader) {
		if (consumer_fifo >= 2)
			sched_set_fifo(consumer);
		else if (consumer_fifo == 1)
			sched_set_fifo_low(consumer);
		else
			set_user_nice(consumer, consumer_nice);
	}

	if (producer_fifo >= 2)
		sched_set_fifo(producer);
	else if (producer_fifo == 1)
		sched_set_fifo_low(producer);
	else
		set_user_nice(producer, producer_nice);

	return 0;

 out_kill:
	if (consumer)
		kthread_stop(consumer);

 out_fail:
	ring_buffer_free(buffer);
	return ret;
}

static void __exit ring_buffer_benchmark_exit(void)
{
	kthread_stop(producer);
	if (consumer)
		kthread_stop(consumer);
	ring_buffer_free(buffer);
}

module_init(ring_buffer_benchmark_init);
module_exit(ring_buffer_benchmark_exit);

MODULE_AUTHOR("Steven Rostedt");
MODULE_DESCRIPTION("ring_buffer_benchmark");
MODULE_LICENSE("GPL");
Commit	Line	Data
bcea3f96	1	// SPDX-License-Identifier: GPL-2.0
5092dbc9 SR	2	/*
	3	* ring buffer tester and benchmark
	4	*
	5	* Copyright (C) 2009 Steven Rostedt <srostedt@redhat.com>
	6	*/
	7	#include <linux/ring_buffer.h>
	8	#include <linux/completion.h>
	9	#include <linux/kthread.h>
ae7e81c0	10	#include <uapi/linux/sched/types.h>
5092dbc9	11	#include <linux/module.h>
da194930	12	#include <linux/ktime.h>
79615760	13	#include <asm/local.h>
5092dbc9 SR	14
	15	struct rb_page {
	16	u64 ts;
	17	local_t commit;
	18	char data[4080];
	19	};
	20
	21	/* run time and sleep time in seconds */
da194930	22	#define RUN_TIME 10ULL
5092dbc9 SR	23	#define SLEEP_TIME 10
	24
	25	/* number of events for writer to wake up the reader */
	26	static int wakeup_interval = 100;
	27
	28	static int reader_finish;
8b46ff69 PM	29	static DECLARE_COMPLETION(read_start);
8b46ff69 PM	30	static DECLARE_COMPLETION(read_done);
5092dbc9	31
13292494	32	static struct trace_buffer *buffer;
5092dbc9 SR	33	static struct task_struct *producer;
	34	static struct task_struct *consumer;
	35	static unsigned long read;
	36
33d657d1	37	static unsigned int disable_reader;
5092dbc9 SR	38	module_param(disable_reader, uint, 0644);
	39	MODULE_PARM_DESC(disable_reader, "only run producer");
	40
33d657d1	41	static unsigned int write_iteration = 50;
a6f0eb6a SR	42	module_param(write_iteration, uint, 0644);
	43	MODULE_PARM_DESC(write_iteration, "# of writes between timestamp readings");
	44
2b3942e4 DY	45	static int producer_nice = MAX_NICE;
2b3942e4 DY	46	static int consumer_nice = MAX_NICE;
7ac07434	47
4fd5750a PZ	48	static int producer_fifo;
4fd5750a PZ	49	static int consumer_fifo;
7ac07434	50
7364e865	51	module_param(producer_nice, int, 0644);
7ac07434 SR	52	MODULE_PARM_DESC(producer_nice, "nice prio for producer");
7ac07434 SR	53
7364e865	54	module_param(consumer_nice, int, 0644);
7ac07434 SR	55	MODULE_PARM_DESC(consumer_nice, "nice prio for consumer");
7ac07434 SR	56
7364e865	57	module_param(producer_fifo, int, 0644);
4fd5750a	58	MODULE_PARM_DESC(producer_fifo, "use fifo for producer: 0 - disabled, 1 - low prio, 2 - fifo");
7ac07434	59
7364e865	60	module_param(consumer_fifo, int, 0644);
4fd5750a	61	MODULE_PARM_DESC(consumer_fifo, "use fifo for consumer: 0 - disabled, 1 - low prio, 2 - fifo");
7ac07434	62
5092dbc9 SR	63	static int read_events;
5092dbc9 SR	64
f47cb66d	65	static int test_error;
5092dbc9	66
f47cb66d	67	#define TEST_ERROR() \
5092dbc9	68	do { \
f47cb66d PM	69	if (!test_error) { \
f47cb66d PM	70	test_error = 1; \
5092dbc9 SR	71	WARN_ON(1); \
	72	} \
	73	} while (0)
	74
	75	enum event_status {
	76	EVENT_FOUND,
	77	EVENT_DROPPED,
	78	};
	79
f47cb66d PM	80	static bool break_test(void)
	81	{
	82	return test_error \|\| kthread_should_stop();
	83	}
	84
5092dbc9 SR	85	static enum event_status read_event(int cpu)
	86	{
	87	struct ring_buffer_event *event;
	88	int *entry;
	89	u64 ts;
	90
66a8cb95	91	event = ring_buffer_consume(buffer, cpu, &ts, NULL);
5092dbc9 SR	92	if (!event)
	93	return EVENT_DROPPED;
	94
	95	entry = ring_buffer_event_data(event);
	96	if (*entry != cpu) {
f47cb66d	97	TEST_ERROR();
5092dbc9 SR	98	return EVENT_DROPPED;
	99	}
	100
	101	read++;
	102	return EVENT_FOUND;
	103	}
	104
	105	static enum event_status read_page(int cpu)
	106	{
bce761d7	107	struct buffer_data_read_page *bpage;
5092dbc9 SR	108	struct ring_buffer_event *event;
	109	struct rb_page *rpage;
	110	unsigned long commit;
bce761d7	111	int page_size;
5092dbc9 SR	112	int *entry;
	113	int ret;
	114	int inc;
	115	int i;
	116
7ea59064	117	bpage = ring_buffer_alloc_read_page(buffer, cpu);
a7e52ad7	118	if (IS_ERR(bpage))
00c81a58 SR	119	return EVENT_DROPPED;
00c81a58 SR	120
bce761d7 TSV	121	page_size = ring_buffer_subbuf_size_get(buffer);
bce761d7 TSV	122	ret = ring_buffer_read_page(buffer, bpage, page_size, cpu, 1);
5092dbc9	123	if (ret >= 0) {
bce761d7	124	rpage = ring_buffer_read_page_data(bpage);
a838b2e6 SR	125	/* The commit may have missed event flags set, clear them */
a838b2e6 SR	126	commit = local_read(&rpage->commit) & 0xfffff;
f47cb66d	127	for (i = 0; i < commit && !test_error ; i += inc) {
5092dbc9	128
bce761d7	129	if (i >= (page_size - offsetof(struct rb_page, data))) {
f47cb66d	130	TEST_ERROR();
5092dbc9 SR	131	break;
	132	}
	133
	134	inc = -1;
	135	event = (void *)&rpage->data[i];
	136	switch (event->type_len) {
	137	case RINGBUF_TYPE_PADDING:
9086c7b9 SR	138	/* failed writes may be discarded events */
9086c7b9 SR	139	if (!event->time_delta)
f47cb66d	140	TEST_ERROR();
9086c7b9	141	inc = event->array[0] + 4;
5092dbc9 SR	142	break;
	143	case RINGBUF_TYPE_TIME_EXTEND:
	144	inc = 8;
	145	break;
	146	case 0:
	147	entry = ring_buffer_event_data(event);
	148	if (*entry != cpu) {
f47cb66d	149	TEST_ERROR();
5092dbc9 SR	150	break;
	151	}
	152	read++;
	153	if (!event->array[0]) {
f47cb66d	154	TEST_ERROR();
5092dbc9 SR	155	break;
5092dbc9 SR	156	}
9086c7b9	157	inc = event->array[0] + 4;
5092dbc9 SR	158	break;
	159	default:
	160	entry = ring_buffer_event_data(event);
	161	if (*entry != cpu) {
f47cb66d	162	TEST_ERROR();
5092dbc9 SR	163	break;
	164	}
	165	read++;
	166	inc = ((event->type_len + 1) * 4);
	167	}
f47cb66d	168	if (test_error)
5092dbc9 SR	169	break;
	170
	171	if (inc <= 0) {
f47cb66d	172	TEST_ERROR();
5092dbc9 SR	173	break;
	174	}
	175	}
	176	}
73a757e6	177	ring_buffer_free_read_page(buffer, cpu, bpage);
5092dbc9 SR	178
	179	if (ret < 0)
	180	return EVENT_DROPPED;
	181	return EVENT_FOUND;
	182	}
	183
	184	static void ring_buffer_consumer(void)
	185	{
	186	/* toggle between reading pages and events */
	187	read_events ^= 1;
	188
	189	read = 0;
8b46ff69 PM	190	/*
	191	* Continue running until the producer specifically asks to stop
	192	* and is ready for the completion.
	193	*/
	194	while (!READ_ONCE(reader_finish)) {
	195	int found = 1;
5092dbc9	196
f47cb66d	197	while (found && !test_error) {
5092dbc9 SR	198	int cpu;
	199
	200	found = 0;
	201	for_each_online_cpu(cpu) {
	202	enum event_status stat;
	203
	204	if (read_events)
	205	stat = read_event(cpu);
	206	else
	207	stat = read_page(cpu);
	208
f47cb66d	209	if (test_error)
5092dbc9	210	break;
8b46ff69	211
5092dbc9 SR	212	if (stat == EVENT_FOUND)
5092dbc9 SR	213	found = 1;
8b46ff69	214
5092dbc9	215	}
8b46ff69	216	}
5092dbc9	217
8b46ff69 PM	218	/* Wait till the producer wakes us up when there is more data
	219	* available or when the producer wants us to finish reading.
	220	*/
5092dbc9 SR	221	set_current_state(TASK_INTERRUPTIBLE);
	222	if (reader_finish)
	223	break;
	224
	225	schedule();
5092dbc9	226	}
8b46ff69	227	__set_current_state(TASK_RUNNING);
5092dbc9 SR	228	reader_finish = 0;
	229	complete(&read_done);
	230	}
	231
	232	static void ring_buffer_producer(void)
	233	{
da194930	234	ktime_t start_time, end_time, timeout;
5092dbc9 SR	235	unsigned long long time;
	236	unsigned long long entries;
	237	unsigned long long overruns;
	238	unsigned long missed = 0;
	239	unsigned long hit = 0;
	240	unsigned long avg;
	241	int cnt = 0;
	242
	243	/*
	244	* Hammer the buffer for 10 secs (this may
	245	* make the system stall)
	246	*/
4b221f03	247	trace_printk("Starting ring buffer hammer\n");
da194930 TR	248	start_time = ktime_get();
da194930 TR	249	timeout = ktime_add_ns(start_time, RUN_TIME * NSEC_PER_SEC);
5092dbc9 SR	250	do {
	251	struct ring_buffer_event *event;
	252	int *entry;
a6f0eb6a SR	253	int i;
	254
	255	for (i = 0; i < write_iteration; i++) {
	256	event = ring_buffer_lock_reserve(buffer, 10);
	257	if (!event) {
	258	missed++;
	259	} else {
	260	hit++;
	261	entry = ring_buffer_event_data(event);
	262	*entry = smp_processor_id();
04aabc32	263	ring_buffer_unlock_commit(buffer);
a6f0eb6a	264	}
5092dbc9	265	}
da194930	266	end_time = ktime_get();
5092dbc9	267
0574ea42 SR	268	cnt++;
0574ea42 SR	269	if (consumer && !(cnt % wakeup_interval))
5092dbc9 SR	270	wake_up_process(consumer);
5092dbc9 SR	271
30c93704	272	#ifndef CONFIG_PREEMPTION
29c8000e	273	/*
a82a4804	274	* If we are a non preempt kernel, the 10 seconds run will
29c8000e SR	275	* stop everything while it runs. Instead, we will call
29c8000e SR	276	* cond_resched and also add any time that was lost by a
a82a4804	277	* reschedule.
0574ea42 SR	278	*
	279	* Do a cond resched at the same frequency we would wake up
	280	* the reader.
29c8000e	281	*/
0574ea42 SR	282	if (cnt % wakeup_interval)
	283	cond_resched();
	284	#endif
f47cb66d	285	} while (ktime_before(end_time, timeout) && !break_test());
4b221f03	286	trace_printk("End ring buffer hammer\n");
5092dbc9 SR	287
	288	if (consumer) {
	289	/* Init both completions here to avoid races */
	290	init_completion(&read_start);
	291	init_completion(&read_done);
	292	/* the completions must be visible before the finish var */
	293	smp_wmb();
	294	reader_finish = 1;
5092dbc9 SR	295	wake_up_process(consumer);
	296	wait_for_completion(&read_done);
	297	}
	298
da194930	299	time = ktime_us_delta(end_time, start_time);
5092dbc9 SR	300
	301	entries = ring_buffer_entries(buffer);
	302	overruns = ring_buffer_overruns(buffer);
	303
f47cb66d	304	if (test_error)
4b221f03	305	trace_printk("ERROR!\n");
7ac07434 SR	306
7ac07434 SR	307	if (!disable_reader) {
4fd5750a PZ	308	if (consumer_fifo)
	309	trace_printk("Running Consumer at SCHED_FIFO %s\n",
	310	consumer_fifo == 1 ? "low" : "high");
	311	else
7ac07434 SR	312	trace_printk("Running Consumer at nice: %d\n",
7ac07434 SR	313	consumer_nice);
7ac07434	314	}
4fd5750a PZ	315	if (producer_fifo)
	316	trace_printk("Running Producer at SCHED_FIFO %s\n",
	317	producer_fifo == 1 ? "low" : "high");
	318	else
7ac07434 SR	319	trace_printk("Running Producer at nice: %d\n",
7ac07434 SR	320	producer_nice);
7ac07434 SR	321
7ac07434 SR	322	/* Let the user know that the test is running at low priority */
4fd5750a	323	if (!producer_fifo && !consumer_fifo &&
2b3942e4	324	producer_nice == MAX_NICE && consumer_nice == MAX_NICE)
7ac07434 SR	325	trace_printk("WARNING!!! This test is running at lowest priority.\n");
7ac07434 SR	326
4b221f03 SR	327	trace_printk("Time: %lld (usecs)\n", time);
4b221f03 SR	328	trace_printk("Overruns: %lld\n", overruns);
5092dbc9	329	if (disable_reader)
4b221f03	330	trace_printk("Read: (reader disabled)\n");
5092dbc9	331	else
4b221f03	332	trace_printk("Read: %ld (by %s)\n", read,
5092dbc9	333	read_events ? "events" : "pages");
4b221f03 SR	334	trace_printk("Entries: %lld\n", entries);
	335	trace_printk("Total: %lld\n", entries + overruns + read);
	336	trace_printk("Missed: %ld\n", missed);
	337	trace_printk("Hit: %ld\n", hit);
5092dbc9	338
5a772b2b SR	339	/* Convert time from usecs to millisecs */
5a772b2b SR	340	do_div(time, USEC_PER_MSEC);
5092dbc9 SR	341	if (time)
	342	hit /= (long)time;
	343	else
4b221f03	344	trace_printk("TIME IS ZERO??\n");
5092dbc9	345
4b221f03	346	trace_printk("Entries per millisec: %ld\n", hit);
5092dbc9 SR	347
5092dbc9 SR	348	if (hit) {
5a772b2b SR	349	/* Calculate the average time in nanosecs */
5a772b2b SR	350	avg = NSEC_PER_MSEC / hit;
4b221f03	351	trace_printk("%ld ns per entry\n", avg);
5092dbc9	352	}
7da3046d	353
7da3046d SR	354	if (missed) {
	355	if (time)
	356	missed /= (long)time;
	357
4b221f03 SR	358	trace_printk("Total iterations per millisec: %ld\n",
4b221f03 SR	359	hit + missed);
7da3046d	360
d988ff94 SR	361	/* it is possible that hit + missed will overflow and be zero */
d988ff94 SR	362	if (!(hit + missed)) {
4b221f03	363	trace_printk("hit + missed overflowed and totalled zero!\n");
d988ff94 SR	364	hit--; /* make it non zero */
	365	}
	366
5c173bed	367	/* Calculate the average time in nanosecs */
5a772b2b	368	avg = NSEC_PER_MSEC / (hit + missed);
4b221f03	369	trace_printk("%ld ns per entry\n", avg);
7da3046d	370	}
5092dbc9 SR	371	}
	372
	373	static void wait_to_die(void)
	374	{
	375	set_current_state(TASK_INTERRUPTIBLE);
	376	while (!kthread_should_stop()) {
	377	schedule();
	378	set_current_state(TASK_INTERRUPTIBLE);
	379	}
	380	__set_current_state(TASK_RUNNING);
	381	}
	382
	383	static int ring_buffer_consumer_thread(void *arg)
	384	{
f47cb66d	385	while (!break_test()) {
5092dbc9 SR	386	complete(&read_start);
	387
	388	ring_buffer_consumer();
	389
	390	set_current_state(TASK_INTERRUPTIBLE);
f47cb66d	391	if (break_test())
5092dbc9	392	break;
5092dbc9	393	schedule();
5092dbc9 SR	394	}
	395	__set_current_state(TASK_RUNNING);
	396
b44754d8	397	if (!kthread_should_stop())
5092dbc9 SR	398	wait_to_die();
	399
	400	return 0;
	401	}
	402
	403	static int ring_buffer_producer_thread(void *arg)
	404	{
f47cb66d	405	while (!break_test()) {
5092dbc9 SR	406	ring_buffer_reset(buffer);
	407
	408	if (consumer) {
5092dbc9 SR	409	wake_up_process(consumer);
	410	wait_for_completion(&read_start);
	411	}
	412
	413	ring_buffer_producer();
f47cb66d	414	if (break_test())
b44754d8	415	goto out_kill;
5092dbc9	416
4b221f03	417	trace_printk("Sleeping for 10 secs\n");
5092dbc9	418	set_current_state(TASK_INTERRUPTIBLE);
f47cb66d PM	419	if (break_test())
f47cb66d PM	420	goto out_kill;
5092dbc9	421	schedule_timeout(HZ * SLEEP_TIME);
5092dbc9 SR	422	}
5092dbc9 SR	423
b44754d8	424	out_kill:
f47cb66d	425	__set_current_state(TASK_RUNNING);
b44754d8	426	if (!kthread_should_stop())
5092dbc9 SR	427	wait_to_die();
	428
	429	return 0;
	430	}
	431
	432	static int __init ring_buffer_benchmark_init(void)
	433	{
	434	int ret;
	435
	436	/* make a one meg buffer in overwite mode */
	437	buffer = ring_buffer_alloc(1000000, RB_FL_OVERWRITE);
	438	if (!buffer)
	439	return -ENOMEM;
	440
	441	if (!disable_reader) {
	442	consumer = kthread_create(ring_buffer_consumer_thread,
	443	NULL, "rb_consumer");
	444	ret = PTR_ERR(consumer);
	445	if (IS_ERR(consumer))
	446	goto out_fail;
	447	}
	448
	449	producer = kthread_run(ring_buffer_producer_thread,
	450	NULL, "rb_producer");
	451	ret = PTR_ERR(producer);
	452
	453	if (IS_ERR(producer))
	454	goto out_kill;
	455
98e4833b IM	456	/*
	457	* Run them as low-prio background tasks by default:
	458	*/
7ac07434	459	if (!disable_reader) {
4fd5750a PZ	460	if (consumer_fifo >= 2)
	461	sched_set_fifo(consumer);
	462	else if (consumer_fifo == 1)
	463	sched_set_fifo_low(consumer);
	464	else
7ac07434 SR	465	set_user_nice(consumer, consumer_nice);
	466	}
	467
4fd5750a PZ	468	if (producer_fifo >= 2)
	469	sched_set_fifo(producer);
	470	else if (producer_fifo == 1)
	471	sched_set_fifo_low(producer);
	472	else
7ac07434	473	set_user_nice(producer, producer_nice);
98e4833b	474
5092dbc9 SR	475	return 0;
	476
	477	out_kill:
	478	if (consumer)
	479	kthread_stop(consumer);
	480
	481	out_fail:
	482	ring_buffer_free(buffer);
	483	return ret;
	484	}
	485
	486	static void __exit ring_buffer_benchmark_exit(void)
	487	{
	488	kthread_stop(producer);
	489	if (consumer)
	490	kthread_stop(consumer);
	491	ring_buffer_free(buffer);
	492	}
	493
	494	module_init(ring_buffer_benchmark_init);
	495	module_exit(ring_buffer_benchmark_exit);
	496
	497	MODULE_AUTHOR("Steven Rostedt");
	498	MODULE_DESCRIPTION("ring_buffer_benchmark");
	499	MODULE_LICENSE("GPL");