[linux-2.6-block.git] / drivers / powercap / idle_inject.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright 2018 Linaro Limited
 *
 * Author: Daniel Lezcano <daniel.lezcano@linaro.org>
 *
 * The idle injection framework provides a way to force CPUs to enter idle
 * states for a specified fraction of time over a specified period.
 *
 * It relies on the smpboot kthreads feature providing common code for CPU
 * hotplug and thread [un]parking.
 *
 * All of the kthreads used for idle injection are created at init time.
 *
 * Next, the users of the idle injection framework provide a cpumask via
 * its register function. The kthreads will be synchronized with respect to
 * this cpumask.
 *
 * The idle + run duration is specified via separate helpers and that allows
 * idle injection to be started.
 *
 * The idle injection kthreads will call play_idle_precise() with the idle
 * duration and max allowed latency specified as per the above.
 *
 * After all of them have been woken up, a timer is set to start the next idle
 * injection cycle.
 *
 * The timer interrupt handler will wake up the idle injection kthreads for
 * all of the CPUs in the cpumask provided by the user.
 *
 * Idle injection is stopped synchronously and no leftover idle injection
 * kthread activity after its completion is guaranteed.
 *
 * It is up to the user of this framework to provide a lock for higher-level
 * synchronization to prevent race conditions like starting idle injection
 * while unregistering from the framework.
 */
#define pr_fmt(fmt) "ii_dev: " fmt

#include <linux/cpu.h>
#include <linux/hrtimer.h>
#include <linux/kthread.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/smpboot.h>
#include <linux/idle_inject.h>

#include <uapi/linux/sched/types.h>

/**
 * struct idle_inject_thread - task on/off switch structure
 * @tsk: task injecting the idle cycles
 * @should_run: whether or not to run the task (for the smpboot kthread API)
 */
struct idle_inject_thread {
	struct task_struct *tsk;
	int should_run;
};

/**
 * struct idle_inject_device - idle injection data
 * @timer: idle injection period timer
 * @idle_duration_us: duration of CPU idle time to inject
 * @run_duration_us: duration of CPU run time to allow
 * @latency_us: max allowed latency
 * @cpumask: mask of CPUs affected by idle injection
 */
struct idle_inject_device {
	struct hrtimer timer;
	unsigned int idle_duration_us;
	unsigned int run_duration_us;
	unsigned int latency_us;
	unsigned long cpumask[];
};

static DEFINE_PER_CPU(struct idle_inject_thread, idle_inject_thread);
static DEFINE_PER_CPU(struct idle_inject_device *, idle_inject_device);

/**
 * idle_inject_wakeup - Wake up idle injection threads
 * @ii_dev: target idle injection device
 *
 * Every idle injection task associated with the given idle injection device
 * and running on an online CPU will be woken up.
 */
static void idle_inject_wakeup(struct idle_inject_device *ii_dev)
{
	struct idle_inject_thread *iit;
	unsigned int cpu;

	for_each_cpu_and(cpu, to_cpumask(ii_dev->cpumask), cpu_online_mask) {
		iit = per_cpu_ptr(&idle_inject_thread, cpu);
		iit->should_run = 1;
		wake_up_process(iit->tsk);
	}
}

/**
 * idle_inject_timer_fn - idle injection timer function
 * @timer: idle injection hrtimer
 *
 * This function is called when the idle injection timer expires.  It wakes up
 * idle injection tasks associated with the timer and they, in turn, invoke
 * play_idle_precise() to inject a specified amount of CPU idle time.
 *
 * Return: HRTIMER_RESTART.
 */
static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer)
{
	unsigned int duration_us;
	struct idle_inject_device *ii_dev =
		container_of(timer, struct idle_inject_device, timer);

	duration_us = READ_ONCE(ii_dev->run_duration_us);
	duration_us += READ_ONCE(ii_dev->idle_duration_us);

	idle_inject_wakeup(ii_dev);

	hrtimer_forward_now(timer, ns_to_ktime(duration_us * NSEC_PER_USEC));

	return HRTIMER_RESTART;
}

/**
 * idle_inject_fn - idle injection work function
 * @cpu: the CPU owning the task
 *
 * This function calls play_idle_precise() to inject a specified amount of CPU
 * idle time.
 */
static void idle_inject_fn(unsigned int cpu)
{
	struct idle_inject_device *ii_dev;
	struct idle_inject_thread *iit;

	ii_dev = per_cpu(idle_inject_device, cpu);
	iit = per_cpu_ptr(&idle_inject_thread, cpu);

	/*
	 * Let the smpboot main loop know that the task should not run again.
	 */
	iit->should_run = 0;

	play_idle_precise(READ_ONCE(ii_dev->idle_duration_us) * NSEC_PER_USEC,
			  READ_ONCE(ii_dev->latency_us) * NSEC_PER_USEC);
}

/**
 * idle_inject_set_duration - idle and run duration update helper
 * @ii_dev: idle injection control device structure
 * @run_duration_us: CPU run time to allow in microseconds
 * @idle_duration_us: CPU idle time to inject in microseconds
 */
void idle_inject_set_duration(struct idle_inject_device *ii_dev,
			      unsigned int run_duration_us,
			      unsigned int idle_duration_us)
{
	if (run_duration_us && idle_duration_us) {
		WRITE_ONCE(ii_dev->run_duration_us, run_duration_us);
		WRITE_ONCE(ii_dev->idle_duration_us, idle_duration_us);
	}
}
EXPORT_SYMBOL_NS_GPL(idle_inject_set_duration, IDLE_INJECT);

/**
 * idle_inject_get_duration - idle and run duration retrieval helper
 * @ii_dev: idle injection control device structure
 * @run_duration_us: memory location to store the current CPU run time
 * @idle_duration_us: memory location to store the current CPU idle time
 */
void idle_inject_get_duration(struct idle_inject_device *ii_dev,
			      unsigned int *run_duration_us,
			      unsigned int *idle_duration_us)
{
	*run_duration_us = READ_ONCE(ii_dev->run_duration_us);
	*idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
}
EXPORT_SYMBOL_NS_GPL(idle_inject_get_duration, IDLE_INJECT);

/**
 * idle_inject_set_latency - set the maximum latency allowed
 * @ii_dev: idle injection control device structure
 * @latency_us: set the latency requirement for the idle state
 */
void idle_inject_set_latency(struct idle_inject_device *ii_dev,
			     unsigned int latency_us)
{
	WRITE_ONCE(ii_dev->latency_us, latency_us);
}
EXPORT_SYMBOL_NS_GPL(idle_inject_set_latency, IDLE_INJECT);

/**
 * idle_inject_start - start idle injections
 * @ii_dev: idle injection control device structure
 *
 * The function starts idle injection by first waking up all of the idle
 * injection kthreads associated with @ii_dev to let them inject CPU idle time
 * sets up a timer to start the next idle injection period.
 *
 * Return: -EINVAL if the CPU idle or CPU run time is not set or 0 on success.
 */
int idle_inject_start(struct idle_inject_device *ii_dev)
{
	unsigned int idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
	unsigned int run_duration_us = READ_ONCE(ii_dev->run_duration_us);

	if (!idle_duration_us || !run_duration_us)
		return -EINVAL;

	pr_debug("Starting injecting idle cycles on CPUs '%*pbl'\n",
		 cpumask_pr_args(to_cpumask(ii_dev->cpumask)));

	idle_inject_wakeup(ii_dev);

	hrtimer_start(&ii_dev->timer,
		      ns_to_ktime((idle_duration_us + run_duration_us) *
				  NSEC_PER_USEC),
		      HRTIMER_MODE_REL);

	return 0;
}
EXPORT_SYMBOL_NS_GPL(idle_inject_start, IDLE_INJECT);

/**
 * idle_inject_stop - stops idle injections
 * @ii_dev: idle injection control device structure
 *
 * The function stops idle injection and waits for the threads to finish work.
 * If CPU idle time is being injected when this function runs, then it will
 * wait until the end of the cycle.
 *
 * When it returns, there is no more idle injection kthread activity.  The
 * kthreads are scheduled out and the periodic timer is off.
 */
void idle_inject_stop(struct idle_inject_device *ii_dev)
{
	struct idle_inject_thread *iit;
	unsigned int cpu;

	pr_debug("Stopping idle injection on CPUs '%*pbl'\n",
		 cpumask_pr_args(to_cpumask(ii_dev->cpumask)));

	hrtimer_cancel(&ii_dev->timer);

	/*
	 * Stopping idle injection requires all of the idle injection kthreads
	 * associated with the given cpumask to be parked and stay that way, so
	 * prevent CPUs from going online at this point.  Any CPUs going online
	 * after the loop below will be covered by clearing the should_run flag
	 * that will cause the smpboot main loop to schedule them out.
	 */
	cpu_hotplug_disable();

	/*
	 * Iterate over all (online + offline) CPUs here in case one of them
	 * goes offline with the should_run flag set so as to prevent its idle
	 * injection kthread from running when the CPU goes online again after
	 * the ii_dev has been freed.
	 */
	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
		iit = per_cpu_ptr(&idle_inject_thread, cpu);
		iit->should_run = 0;

		wait_task_inactive(iit->tsk, TASK_ANY);
	}

	cpu_hotplug_enable();
}
EXPORT_SYMBOL_NS_GPL(idle_inject_stop, IDLE_INJECT);

/**
 * idle_inject_setup - prepare the current task for idle injection
 * @cpu: not used
 *
 * Called once, this function is in charge of setting the current task's
 * scheduler parameters to make it an RT task.
 */
static void idle_inject_setup(unsigned int cpu)
{
	sched_set_fifo(current);
}

/**
 * idle_inject_should_run - function helper for the smpboot API
 * @cpu: CPU the kthread is running on
 *
 * Return: whether or not the thread can run.
 */
static int idle_inject_should_run(unsigned int cpu)
{
	struct idle_inject_thread *iit =
		per_cpu_ptr(&idle_inject_thread, cpu);

	return iit->should_run;
}

/**
 * idle_inject_register - initialize idle injection on a set of CPUs
 * @cpumask: CPUs to be affected by idle injection
 *
 * This function creates an idle injection control device structure for the
 * given set of CPUs and initializes the timer associated with it.  It does not
 * start any injection cycles.
 *
 * Return: NULL if memory allocation fails, idle injection control device
 * pointer on success.
 */
struct idle_inject_device *idle_inject_register(struct cpumask *cpumask)
{
	struct idle_inject_device *ii_dev;
	int cpu, cpu_rb;

	ii_dev = kzalloc(sizeof(*ii_dev) + cpumask_size(), GFP_KERNEL);
	if (!ii_dev)
		return NULL;

	cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask);
	hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	ii_dev->timer.function = idle_inject_timer_fn;
	ii_dev->latency_us = UINT_MAX;

	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {

		if (per_cpu(idle_inject_device, cpu)) {
			pr_err("cpu%d is already registered\n", cpu);
			goto out_rollback;
		}

		per_cpu(idle_inject_device, cpu) = ii_dev;
	}

	return ii_dev;

out_rollback:
	for_each_cpu(cpu_rb, to_cpumask(ii_dev->cpumask)) {
		if (cpu == cpu_rb)
			break;
		per_cpu(idle_inject_device, cpu_rb) = NULL;
	}

	kfree(ii_dev);

	return NULL;
}
EXPORT_SYMBOL_NS_GPL(idle_inject_register, IDLE_INJECT);

/**
 * idle_inject_unregister - unregister idle injection control device
 * @ii_dev: idle injection control device to unregister
 *
 * The function stops idle injection for the given control device,
 * unregisters its kthreads and frees memory allocated when that device was
 * created.
 */
void idle_inject_unregister(struct idle_inject_device *ii_dev)
{
	unsigned int cpu;

	idle_inject_stop(ii_dev);

	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask))
		per_cpu(idle_inject_device, cpu) = NULL;

	kfree(ii_dev);
}
EXPORT_SYMBOL_NS_GPL(idle_inject_unregister, IDLE_INJECT);

static struct smp_hotplug_thread idle_inject_threads = {
	.store = &idle_inject_thread.tsk,
	.setup = idle_inject_setup,
	.thread_fn = idle_inject_fn,
	.thread_comm = "idle_inject/%u",
	.thread_should_run = idle_inject_should_run,
};

static int __init idle_inject_init(void)
{
	return smpboot_register_percpu_thread(&idle_inject_threads);
}
early_initcall(idle_inject_init);
Commit	Line	Data
88763a5c DL	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Copyright 2018 Linaro Limited
	4	*
	5	* Author: Daniel Lezcano <daniel.lezcano@linaro.org>
	6	*
	7	* The idle injection framework provides a way to force CPUs to enter idle
	8	* states for a specified fraction of time over a specified period.
	9	*
	10	* It relies on the smpboot kthreads feature providing common code for CPU
	11	* hotplug and thread [un]parking.
	12	*
	13	* All of the kthreads used for idle injection are created at init time.
	14	*
86ffed3d	15	* Next, the users of the idle injection framework provide a cpumask via
88763a5c DL	16	* its register function. The kthreads will be synchronized with respect to
	17	* this cpumask.
	18	*
	19	* The idle + run duration is specified via separate helpers and that allows
	20	* idle injection to be started.
	21	*
0735069c YL	22	* The idle injection kthreads will call play_idle_precise() with the idle
0735069c YL	23	* duration and max allowed latency specified as per the above.
88763a5c DL	24	*
	25	* After all of them have been woken up, a timer is set to start the next idle
	26	* injection cycle.
	27	*
	28	* The timer interrupt handler will wake up the idle injection kthreads for
	29	* all of the CPUs in the cpumask provided by the user.
	30	*
	31	* Idle injection is stopped synchronously and no leftover idle injection
	32	* kthread activity after its completion is guaranteed.
	33	*
	34	* It is up to the user of this framework to provide a lock for higher-level
	35	* synchronization to prevent race conditions like starting idle injection
	36	* while unregistering from the framework.
	37	*/
	38	#define pr_fmt(fmt) "ii_dev: " fmt
	39
	40	#include <linux/cpu.h>
	41	#include <linux/hrtimer.h>
	42	#include <linux/kthread.h>
	43	#include <linux/sched.h>
	44	#include <linux/slab.h>
	45	#include <linux/smpboot.h>
00610935	46	#include <linux/idle_inject.h>
88763a5c DL	47
	48	#include <uapi/linux/sched/types.h>
	49
	50	/**
	51	* struct idle_inject_thread - task on/off switch structure
	52	* @tsk: task injecting the idle cycles
	53	* @should_run: whether or not to run the task (for the smpboot kthread API)
	54	*/
	55	struct idle_inject_thread {
	56	struct task_struct *tsk;
	57	int should_run;
	58	};
	59
	60	/**
	61	* struct idle_inject_device - idle injection data
	62	* @timer: idle injection period timer
cd4c0763 DL	63	* @idle_duration_us: duration of CPU idle time to inject
cd4c0763 DL	64	* @run_duration_us: duration of CPU run time to allow
333cff6c	65	* @latency_us: max allowed latency
88763a5c DL	66	* @cpumask: mask of CPUs affected by idle injection
	67	*/
	68	struct idle_inject_device {
	69	struct hrtimer timer;
cd4c0763 DL	70	unsigned int idle_duration_us;
cd4c0763 DL	71	unsigned int run_duration_us;
333cff6c	72	unsigned int latency_us;
27565c9e	73	unsigned long cpumask[];
88763a5c DL	74	};
	75
	76	static DEFINE_PER_CPU(struct idle_inject_thread, idle_inject_thread);
	77	static DEFINE_PER_CPU(struct idle_inject_device *, idle_inject_device);
	78
	79	/**
	80	* idle_inject_wakeup - Wake up idle injection threads
	81	* @ii_dev: target idle injection device
	82	*
	83	* Every idle injection task associated with the given idle injection device
	84	* and running on an online CPU will be woken up.
	85	*/
	86	static void idle_inject_wakeup(struct idle_inject_device *ii_dev)
	87	{
	88	struct idle_inject_thread *iit;
	89	unsigned int cpu;
	90
	91	for_each_cpu_and(cpu, to_cpumask(ii_dev->cpumask), cpu_online_mask) {
	92	iit = per_cpu_ptr(&idle_inject_thread, cpu);
	93	iit->should_run = 1;
	94	wake_up_process(iit->tsk);
	95	}
	96	}
	97
	98	/**
	99	* idle_inject_timer_fn - idle injection timer function
	100	* @timer: idle injection hrtimer
	101	*
	102	* This function is called when the idle injection timer expires. It wakes up
	103	* idle injection tasks associated with the timer and they, in turn, invoke
0735069c	104	* play_idle_precise() to inject a specified amount of CPU idle time.
88763a5c DL	105	*
	106	* Return: HRTIMER_RESTART.
	107	*/
	108	static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer)
	109	{
cd4c0763	110	unsigned int duration_us;
88763a5c DL	111	struct idle_inject_device *ii_dev =
	112	container_of(timer, struct idle_inject_device, timer);
	113
cd4c0763 DL	114	duration_us = READ_ONCE(ii_dev->run_duration_us);
cd4c0763 DL	115	duration_us += READ_ONCE(ii_dev->idle_duration_us);
88763a5c DL	116
	117	idle_inject_wakeup(ii_dev);
	118
cd4c0763	119	hrtimer_forward_now(timer, ns_to_ktime(duration_us * NSEC_PER_USEC));
88763a5c DL	120
	121	return HRTIMER_RESTART;
	122	}
	123
	124	/**
	125	* idle_inject_fn - idle injection work function
	126	* @cpu: the CPU owning the task
	127	*
0735069c YL	128	* This function calls play_idle_precise() to inject a specified amount of CPU
0735069c YL	129	* idle time.
88763a5c DL	130	*/
	131	static void idle_inject_fn(unsigned int cpu)
	132	{
	133	struct idle_inject_device *ii_dev;
	134	struct idle_inject_thread *iit;
	135
	136	ii_dev = per_cpu(idle_inject_device, cpu);
	137	iit = per_cpu_ptr(&idle_inject_thread, cpu);
	138
	139	/*
	140	* Let the smpboot main loop know that the task should not run again.
	141	*/
	142	iit->should_run = 0;
	143
333cff6c DL	144	play_idle_precise(READ_ONCE(ii_dev->idle_duration_us) * NSEC_PER_USEC,
333cff6c DL	145	READ_ONCE(ii_dev->latency_us) * NSEC_PER_USEC);
88763a5c DL	146	}
	147
	148	/**
	149	* idle_inject_set_duration - idle and run duration update helper
98e596fc	150	* @ii_dev: idle injection control device structure
cd4c0763 DL	151	* @run_duration_us: CPU run time to allow in microseconds
cd4c0763 DL	152	* @idle_duration_us: CPU idle time to inject in microseconds
88763a5c DL	153	*/
88763a5c DL	154	void idle_inject_set_duration(struct idle_inject_device *ii_dev,
cd4c0763 DL	155	unsigned int run_duration_us,
cd4c0763 DL	156	unsigned int idle_duration_us)
88763a5c	157	{
cd4c0763 DL	158	if (run_duration_us && idle_duration_us) {
	159	WRITE_ONCE(ii_dev->run_duration_us, run_duration_us);
	160	WRITE_ONCE(ii_dev->idle_duration_us, idle_duration_us);
88763a5c DL	161	}
88763a5c DL	162	}
bbfc3349	163	EXPORT_SYMBOL_NS_GPL(idle_inject_set_duration, IDLE_INJECT);
88763a5c DL	164
	165	/**
	166	* idle_inject_get_duration - idle and run duration retrieval helper
98e596fc	167	* @ii_dev: idle injection control device structure
cd4c0763 DL	168	* @run_duration_us: memory location to store the current CPU run time
cd4c0763 DL	169	* @idle_duration_us: memory location to store the current CPU idle time
88763a5c DL	170	*/
88763a5c DL	171	void idle_inject_get_duration(struct idle_inject_device *ii_dev,
cd4c0763 DL	172	unsigned int *run_duration_us,
cd4c0763 DL	173	unsigned int *idle_duration_us)
88763a5c	174	{
cd4c0763 DL	175	*run_duration_us = READ_ONCE(ii_dev->run_duration_us);
cd4c0763 DL	176	*idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
88763a5c	177	}
bbfc3349	178	EXPORT_SYMBOL_NS_GPL(idle_inject_get_duration, IDLE_INJECT);
88763a5c	179
333cff6c DL	180	/**
333cff6c DL	181	* idle_inject_set_latency - set the maximum latency allowed
98e596fc	182	* @ii_dev: idle injection control device structure
333cff6c DL	183	* @latency_us: set the latency requirement for the idle state
	184	*/
	185	void idle_inject_set_latency(struct idle_inject_device *ii_dev,
	186	unsigned int latency_us)
	187	{
	188	WRITE_ONCE(ii_dev->latency_us, latency_us);
	189	}
bbfc3349	190	EXPORT_SYMBOL_NS_GPL(idle_inject_set_latency, IDLE_INJECT);
333cff6c	191
88763a5c DL	192	/**
	193	* idle_inject_start - start idle injections
	194	* @ii_dev: idle injection control device structure
	195	*
	196	* The function starts idle injection by first waking up all of the idle
	197	* injection kthreads associated with @ii_dev to let them inject CPU idle time
	198	* sets up a timer to start the next idle injection period.
	199	*
	200	* Return: -EINVAL if the CPU idle or CPU run time is not set or 0 on success.
	201	*/
	202	int idle_inject_start(struct idle_inject_device *ii_dev)
	203	{
cd4c0763 DL	204	unsigned int idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
cd4c0763 DL	205	unsigned int run_duration_us = READ_ONCE(ii_dev->run_duration_us);
88763a5c	206
cd4c0763	207	if (!idle_duration_us \|\| !run_duration_us)
88763a5c DL	208	return -EINVAL;
	209
	210	pr_debug("Starting injecting idle cycles on CPUs '%*pbl'\n",
	211	cpumask_pr_args(to_cpumask(ii_dev->cpumask)));
	212
	213	idle_inject_wakeup(ii_dev);
	214
	215	hrtimer_start(&ii_dev->timer,
cd4c0763 DL	216	ns_to_ktime((idle_duration_us + run_duration_us) *
cd4c0763 DL	217	NSEC_PER_USEC),
88763a5c DL	218	HRTIMER_MODE_REL);
	219
	220	return 0;
	221	}
bbfc3349	222	EXPORT_SYMBOL_NS_GPL(idle_inject_start, IDLE_INJECT);
88763a5c DL	223
	224	/**
	225	* idle_inject_stop - stops idle injections
	226	* @ii_dev: idle injection control device structure
	227	*
	228	* The function stops idle injection and waits for the threads to finish work.
	229	* If CPU idle time is being injected when this function runs, then it will
	230	* wait until the end of the cycle.
	231	*
	232	* When it returns, there is no more idle injection kthread activity. The
	233	* kthreads are scheduled out and the periodic timer is off.
	234	*/
	235	void idle_inject_stop(struct idle_inject_device *ii_dev)
	236	{
	237	struct idle_inject_thread *iit;
	238	unsigned int cpu;
	239
	240	pr_debug("Stopping idle injection on CPUs '%*pbl'\n",
	241	cpumask_pr_args(to_cpumask(ii_dev->cpumask)));
	242
	243	hrtimer_cancel(&ii_dev->timer);
	244
	245	/*
	246	* Stopping idle injection requires all of the idle injection kthreads
	247	* associated with the given cpumask to be parked and stay that way, so
	248	* prevent CPUs from going online at this point. Any CPUs going online
	249	* after the loop below will be covered by clearing the should_run flag
	250	* that will cause the smpboot main loop to schedule them out.
	251	*/
	252	cpu_hotplug_disable();
	253
	254	/*
	255	* Iterate over all (online + offline) CPUs here in case one of them
	256	* goes offline with the should_run flag set so as to prevent its idle
	257	* injection kthread from running when the CPU goes online again after
	258	* the ii_dev has been freed.
	259	*/
	260	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
	261	iit = per_cpu_ptr(&idle_inject_thread, cpu);
	262	iit->should_run = 0;
	263
f9fc8cad	264	wait_task_inactive(iit->tsk, TASK_ANY);
88763a5c DL	265	}
	266
	267	cpu_hotplug_enable();
	268	}
bbfc3349	269	EXPORT_SYMBOL_NS_GPL(idle_inject_stop, IDLE_INJECT);
88763a5c DL	270
	271	/**
	272	* idle_inject_setup - prepare the current task for idle injection
	273	* @cpu: not used
	274	*
	275	* Called once, this function is in charge of setting the current task's
	276	* scheduler parameters to make it an RT task.
	277	*/
	278	static void idle_inject_setup(unsigned int cpu)
	279	{
c3f47cf9	280	sched_set_fifo(current);
88763a5c DL	281	}
	282
	283	/**
	284	* idle_inject_should_run - function helper for the smpboot API
	285	* @cpu: CPU the kthread is running on
	286	*
	287	* Return: whether or not the thread can run.
	288	*/
	289	static int idle_inject_should_run(unsigned int cpu)
	290	{
	291	struct idle_inject_thread *iit =
	292	per_cpu_ptr(&idle_inject_thread, cpu);
	293
	294	return iit->should_run;
	295	}
	296
	297	/**
	298	* idle_inject_register - initialize idle injection on a set of CPUs
	299	* @cpumask: CPUs to be affected by idle injection
	300	*
	301	* This function creates an idle injection control device structure for the
	302	* given set of CPUs and initializes the timer associated with it. It does not
	303	* start any injection cycles.
	304	*
	305	* Return: NULL if memory allocation fails, idle injection control device
	306	* pointer on success.
	307	*/
	308	struct idle_inject_device idle_inject_register(struct cpumask cpumask)
	309	{
	310	struct idle_inject_device *ii_dev;
	311	int cpu, cpu_rb;
	312
	313	ii_dev = kzalloc(sizeof(*ii_dev) + cpumask_size(), GFP_KERNEL);
	314	if (!ii_dev)
	315	return NULL;
	316
	317	cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask);
	318	hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	319	ii_dev->timer.function = idle_inject_timer_fn;
333cff6c	320	ii_dev->latency_us = UINT_MAX;
88763a5c DL	321
	322	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
	323
	324	if (per_cpu(idle_inject_device, cpu)) {
	325	pr_err("cpu%d is already registered\n", cpu);
	326	goto out_rollback;
	327	}
	328
	329	per_cpu(idle_inject_device, cpu) = ii_dev;
	330	}
	331
	332	return ii_dev;
	333
	334	out_rollback:
	335	for_each_cpu(cpu_rb, to_cpumask(ii_dev->cpumask)) {
	336	if (cpu == cpu_rb)
	337	break;
	338	per_cpu(idle_inject_device, cpu_rb) = NULL;
	339	}
	340
	341	kfree(ii_dev);
	342
	343	return NULL;
	344	}
bbfc3349	345	EXPORT_SYMBOL_NS_GPL(idle_inject_register, IDLE_INJECT);
88763a5c DL	346
	347	/**
	348	* idle_inject_unregister - unregister idle injection control device
	349	* @ii_dev: idle injection control device to unregister
	350	*
	351	* The function stops idle injection for the given control device,
	352	* unregisters its kthreads and frees memory allocated when that device was
	353	* created.
	354	*/
	355	void idle_inject_unregister(struct idle_inject_device *ii_dev)
	356	{
	357	unsigned int cpu;
	358
	359	idle_inject_stop(ii_dev);
	360
	361	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask))
	362	per_cpu(idle_inject_device, cpu) = NULL;
	363
	364	kfree(ii_dev);
	365	}
bbfc3349	366	EXPORT_SYMBOL_NS_GPL(idle_inject_unregister, IDLE_INJECT);
88763a5c DL	367
	368	static struct smp_hotplug_thread idle_inject_threads = {
	369	.store = &idle_inject_thread.tsk,
	370	.setup = idle_inject_setup,
	371	.thread_fn = idle_inject_fn,
	372	.thread_comm = "idle_inject/%u",
	373	.thread_should_run = idle_inject_should_run,
	374	};
	375
	376	static int __init idle_inject_init(void)
	377	{
	378	return smpboot_register_percpu_thread(&idle_inject_threads);
	379	}
	380	early_initcall(idle_inject_init);