[linux-block.git] / include / linux / rcutree.h

/* SPDX-License-Identifier: GPL-2.0+ */
/*
 * Read-Copy Update mechanism for mutual exclusion (tree-based version)
 *
 * Copyright IBM Corporation, 2008
 *
 * Author: Dipankar Sarma <dipankar@in.ibm.com>
 *	   Paul E. McKenney <paulmck@linux.ibm.com> Hierarchical algorithm
 *
 * Based on the original work by Paul McKenney <paulmck@linux.ibm.com>
 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
 *
 * For detailed explanation of Read-Copy Update mechanism see -
 *	Documentation/RCU
 */

#ifndef __LINUX_RCUTREE_H
#define __LINUX_RCUTREE_H

void rcu_softirq_qs(void);
void rcu_note_context_switch(bool preempt);
int rcu_needs_cpu(void);
void rcu_cpu_stall_reset(void);
void rcu_request_urgent_qs_task(struct task_struct *t);

/*
 * Note a virtualization-based context switch.  This is simply a
 * wrapper around rcu_note_context_switch(), which allows TINY_RCU
 * to save a few bytes. The caller must have disabled interrupts.
 */
static inline void rcu_virt_note_context_switch(void)
{
	rcu_note_context_switch(false);
}

void synchronize_rcu_expedited(void);

void rcu_barrier(void);
void rcu_momentary_eqs(void);

struct rcu_gp_oldstate {
	unsigned long rgos_norm;
	unsigned long rgos_exp;
};

// Maximum number of rcu_gp_oldstate values corresponding to
// not-yet-completed RCU grace periods.
#define NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE 4

/**
 * same_state_synchronize_rcu_full - Are two old-state values identical?
 * @rgosp1: First old-state value.
 * @rgosp2: Second old-state value.
 *
 * The two old-state values must have been obtained from either
 * get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(),
 * or get_completed_synchronize_rcu_full().  Returns @true if the two
 * values are identical and @false otherwise.  This allows structures
 * whose lifetimes are tracked by old-state values to push these values
 * to a list header, allowing those structures to be slightly smaller.
 *
 * Note that equality is judged on a bitwise basis, so that an
 * @rcu_gp_oldstate structure with an already-completed state in one field
 * will compare not-equal to a structure with an already-completed state
 * in the other field.  After all, the @rcu_gp_oldstate structure is opaque
 * so how did such a situation come to pass in the first place?
 */
static inline bool same_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp1,
						   struct rcu_gp_oldstate *rgosp2)
{
	return rgosp1->rgos_norm == rgosp2->rgos_norm && rgosp1->rgos_exp == rgosp2->rgos_exp;
}

unsigned long start_poll_synchronize_rcu_expedited(void);
void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp);
void cond_synchronize_rcu_expedited(unsigned long oldstate);
void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp);
unsigned long get_state_synchronize_rcu(void);
void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);
unsigned long start_poll_synchronize_rcu(void);
void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);
bool poll_state_synchronize_rcu(unsigned long oldstate);
bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);
void cond_synchronize_rcu(unsigned long oldstate);
void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);

#ifdef CONFIG_PROVE_RCU
void rcu_irq_exit_check_preempt(void);
#else
static inline void rcu_irq_exit_check_preempt(void) { }
#endif

struct task_struct;
void rcu_preempt_deferred_qs(struct task_struct *t);

void exit_rcu(void);

void rcu_scheduler_starting(void);
extern int rcu_scheduler_active;
void rcu_end_inkernel_boot(void);
bool rcu_inkernel_boot_has_ended(void);
bool rcu_is_watching(void);
#ifndef CONFIG_PREEMPT_RCU
void rcu_all_qs(void);
#endif

/* RCUtree hotplug events */
int rcutree_prepare_cpu(unsigned int cpu);
int rcutree_online_cpu(unsigned int cpu);
void rcutree_report_cpu_starting(unsigned int cpu);

#ifdef CONFIG_HOTPLUG_CPU
int rcutree_dead_cpu(unsigned int cpu);
int rcutree_dying_cpu(unsigned int cpu);
int rcutree_offline_cpu(unsigned int cpu);
#else
#define rcutree_dead_cpu NULL
#define rcutree_dying_cpu NULL
#define rcutree_offline_cpu NULL
#endif

void rcutree_migrate_callbacks(int cpu);

/* Called from hotplug and also arm64 early secondary boot failure */
void rcutree_report_cpu_dead(void);

#endif /* __LINUX_RCUTREE_H */
Commit	Line	Data
	1	/* SPDX-License-Identifier: GPL-2.0+ */
	2	/*
	3	* Read-Copy Update mechanism for mutual exclusion (tree-based version)
	4	*
	5	* Copyright IBM Corporation, 2008
	6	*
	7	* Author: Dipankar Sarma <dipankar@in.ibm.com>
	8	* Paul E. McKenney <paulmck@linux.ibm.com> Hierarchical algorithm
	9	*
	10	* Based on the original work by Paul McKenney <paulmck@linux.ibm.com>
	11	* and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
	12	*
	13	* For detailed explanation of Read-Copy Update mechanism see -
	14	* Documentation/RCU
	15	*/
	16
	17	#ifndef __LINUX_RCUTREE_H
	18	#define __LINUX_RCUTREE_H
	19
	20	void rcu_softirq_qs(void);
	21	void rcu_note_context_switch(bool preempt);
	22	int rcu_needs_cpu(void);
	23	void rcu_cpu_stall_reset(void);
	24	void rcu_request_urgent_qs_task(struct task_struct *t);
	25
	26	/*
	27	* Note a virtualization-based context switch. This is simply a
	28	* wrapper around rcu_note_context_switch(), which allows TINY_RCU
	29	* to save a few bytes. The caller must have disabled interrupts.
	30	*/
	31	static inline void rcu_virt_note_context_switch(void)
	32	{
	33	rcu_note_context_switch(false);
	34	}
	35
	36	void synchronize_rcu_expedited(void);
	37
	38	void rcu_barrier(void);
	39	void rcu_momentary_eqs(void);
	40
	41	struct rcu_gp_oldstate {
	42	unsigned long rgos_norm;
	43	unsigned long rgos_exp;
	44	};
	45
	46	// Maximum number of rcu_gp_oldstate values corresponding to
	47	// not-yet-completed RCU grace periods.
	48	#define NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE 4
	49
	50	/**
	51	* same_state_synchronize_rcu_full - Are two old-state values identical?
	52	* @rgosp1: First old-state value.
	53	* @rgosp2: Second old-state value.
	54	*
	55	* The two old-state values must have been obtained from either
	56	* get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(),
	57	* or get_completed_synchronize_rcu_full(). Returns @true if the two
	58	* values are identical and @false otherwise. This allows structures
	59	* whose lifetimes are tracked by old-state values to push these values
	60	* to a list header, allowing those structures to be slightly smaller.
	61	*
	62	* Note that equality is judged on a bitwise basis, so that an
	63	* @rcu_gp_oldstate structure with an already-completed state in one field
	64	* will compare not-equal to a structure with an already-completed state
	65	* in the other field. After all, the @rcu_gp_oldstate structure is opaque
	66	* so how did such a situation come to pass in the first place?
	67	*/
	68	static inline bool same_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp1,
	69	struct rcu_gp_oldstate *rgosp2)
	70	{
	71	return rgosp1->rgos_norm == rgosp2->rgos_norm && rgosp1->rgos_exp == rgosp2->rgos_exp;
	72	}
	73
	74	unsigned long start_poll_synchronize_rcu_expedited(void);
	75	void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp);
	76	void cond_synchronize_rcu_expedited(unsigned long oldstate);
	77	void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp);
	78	unsigned long get_state_synchronize_rcu(void);
	79	void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);
	80	unsigned long start_poll_synchronize_rcu(void);
	81	void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);
	82	bool poll_state_synchronize_rcu(unsigned long oldstate);
	83	bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);
	84	void cond_synchronize_rcu(unsigned long oldstate);
	85	void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);
	86
	87	#ifdef CONFIG_PROVE_RCU
	88	void rcu_irq_exit_check_preempt(void);
	89	#else
	90	static inline void rcu_irq_exit_check_preempt(void) { }
	91	#endif
	92
	93	struct task_struct;
	94	void rcu_preempt_deferred_qs(struct task_struct *t);
	95
	96	void exit_rcu(void);
	97
	98	void rcu_scheduler_starting(void);
	99	extern int rcu_scheduler_active;
	100	void rcu_end_inkernel_boot(void);
	101	bool rcu_inkernel_boot_has_ended(void);
	102	bool rcu_is_watching(void);
	103	#ifndef CONFIG_PREEMPT_RCU
	104	void rcu_all_qs(void);
	105	#endif
	106
	107	/* RCUtree hotplug events */
	108	int rcutree_prepare_cpu(unsigned int cpu);
	109	int rcutree_online_cpu(unsigned int cpu);
	110	void rcutree_report_cpu_starting(unsigned int cpu);
	111
	112	#ifdef CONFIG_HOTPLUG_CPU
	113	int rcutree_dead_cpu(unsigned int cpu);
	114	int rcutree_dying_cpu(unsigned int cpu);
	115	int rcutree_offline_cpu(unsigned int cpu);
	116	#else
	117	#define rcutree_dead_cpu NULL
	118	#define rcutree_dying_cpu NULL
	119	#define rcutree_offline_cpu NULL
	120	#endif
	121
	122	void rcutree_migrate_callbacks(int cpu);
	123
	124	/* Called from hotplug and also arm64 early secondary boot failure */
	125	void rcutree_report_cpu_dead(void);
	126
	127	#endif /* __LINUX_RCUTREE_H */