[linux-block.git] / include / linux / vmstat.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_VMSTAT_H
#define _LINUX_VMSTAT_H

#include <linux/types.h>
#include <linux/percpu.h>
#include <linux/mmzone.h>
#include <linux/vm_event_item.h>
#include <linux/atomic.h>
#include <linux/static_key.h>
#include <linux/mmdebug.h>

extern int sysctl_stat_interval;

#ifdef CONFIG_NUMA
#define ENABLE_NUMA_STAT   1
#define DISABLE_NUMA_STAT   0
extern int sysctl_vm_numa_stat;
DECLARE_STATIC_KEY_TRUE(vm_numa_stat_key);
int sysctl_vm_numa_stat_handler(struct ctl_table *table, int write,
		void *buffer, size_t *length, loff_t *ppos);
#endif

struct reclaim_stat {
	unsigned nr_dirty;
	unsigned nr_unqueued_dirty;
	unsigned nr_congested;
	unsigned nr_writeback;
	unsigned nr_immediate;
	unsigned nr_pageout;
	unsigned nr_activate[ANON_AND_FILE];
	unsigned nr_ref_keep;
	unsigned nr_unmap_fail;
	unsigned nr_lazyfree_fail;
};

enum writeback_stat_item {
	NR_DIRTY_THRESHOLD,
	NR_DIRTY_BG_THRESHOLD,
	NR_VM_WRITEBACK_STAT_ITEMS,
};

#ifdef CONFIG_VM_EVENT_COUNTERS
/*
 * Light weight per cpu counter implementation.
 *
 * Counters should only be incremented and no critical kernel component
 * should rely on the counter values.
 *
 * Counters are handled completely inline. On many platforms the code
 * generated will simply be the increment of a global address.
 */

struct vm_event_state {
	unsigned long event[NR_VM_EVENT_ITEMS];
};

DECLARE_PER_CPU(struct vm_event_state, vm_event_states);

/*
 * vm counters are allowed to be racy. Use raw_cpu_ops to avoid the
 * local_irq_disable overhead.
 */
static inline void __count_vm_event(enum vm_event_item item)
{
	raw_cpu_inc(vm_event_states.event[item]);
}

static inline void count_vm_event(enum vm_event_item item)
{
	this_cpu_inc(vm_event_states.event[item]);
}

static inline void __count_vm_events(enum vm_event_item item, long delta)
{
	raw_cpu_add(vm_event_states.event[item], delta);
}

static inline void count_vm_events(enum vm_event_item item, long delta)
{
	this_cpu_add(vm_event_states.event[item], delta);
}

extern void all_vm_events(unsigned long *);

extern void vm_events_fold_cpu(int cpu);

#else

/* Disable counters */
static inline void count_vm_event(enum vm_event_item item)
{
}
static inline void count_vm_events(enum vm_event_item item, long delta)
{
}
static inline void __count_vm_event(enum vm_event_item item)
{
}
static inline void __count_vm_events(enum vm_event_item item, long delta)
{
}
static inline void all_vm_events(unsigned long *ret)
{
}
static inline void vm_events_fold_cpu(int cpu)
{
}

#endif /* CONFIG_VM_EVENT_COUNTERS */

#ifdef CONFIG_NUMA_BALANCING
#define count_vm_numa_event(x)     count_vm_event(x)
#define count_vm_numa_events(x, y) count_vm_events(x, y)
#else
#define count_vm_numa_event(x) do {} while (0)
#define count_vm_numa_events(x, y) do { (void)(y); } while (0)
#endif /* CONFIG_NUMA_BALANCING */

#ifdef CONFIG_DEBUG_TLBFLUSH
#define count_vm_tlb_event(x)	   count_vm_event(x)
#define count_vm_tlb_events(x, y)  count_vm_events(x, y)
#else
#define count_vm_tlb_event(x)     do {} while (0)
#define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
#endif

#ifdef CONFIG_DEBUG_VM_VMACACHE
#define count_vm_vmacache_event(x) count_vm_event(x)
#else
#define count_vm_vmacache_event(x) do {} while (0)
#endif

#define __count_zid_vm_events(item, zid, delta) \
	__count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)

/*
 * Zone and node-based page accounting with per cpu differentials.
 */
extern atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS];
extern atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS];
extern atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS];

#ifdef CONFIG_NUMA
static inline void zone_numa_event_add(long x, struct zone *zone,
				enum numa_stat_item item)
{
	atomic_long_add(x, &zone->vm_numa_event[item]);
	atomic_long_add(x, &vm_numa_event[item]);
}

static inline unsigned long zone_numa_event_state(struct zone *zone,
					enum numa_stat_item item)
{
	return atomic_long_read(&zone->vm_numa_event[item]);
}

static inline unsigned long
global_numa_event_state(enum numa_stat_item item)
{
	return atomic_long_read(&vm_numa_event[item]);
}
#endif /* CONFIG_NUMA */

static inline void zone_page_state_add(long x, struct zone *zone,
				 enum zone_stat_item item)
{
	atomic_long_add(x, &zone->vm_stat[item]);
	atomic_long_add(x, &vm_zone_stat[item]);
}

static inline void node_page_state_add(long x, struct pglist_data *pgdat,
				 enum node_stat_item item)
{
	atomic_long_add(x, &pgdat->vm_stat[item]);
	atomic_long_add(x, &vm_node_stat[item]);
}

static inline unsigned long global_zone_page_state(enum zone_stat_item item)
{
	long x = atomic_long_read(&vm_zone_stat[item]);
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
}

static inline
unsigned long global_node_page_state_pages(enum node_stat_item item)
{
	long x = atomic_long_read(&vm_node_stat[item]);
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
}

static inline unsigned long global_node_page_state(enum node_stat_item item)
{
	VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));

	return global_node_page_state_pages(item);
}

static inline unsigned long zone_page_state(struct zone *zone,
					enum zone_stat_item item)
{
	long x = atomic_long_read(&zone->vm_stat[item]);
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
}

/*
 * More accurate version that also considers the currently pending
 * deltas. For that we need to loop over all cpus to find the current
 * deltas. There is no synchronization so the result cannot be
 * exactly accurate either.
 */
static inline unsigned long zone_page_state_snapshot(struct zone *zone,
					enum zone_stat_item item)
{
	long x = atomic_long_read(&zone->vm_stat[item]);

#ifdef CONFIG_SMP
	int cpu;
	for_each_online_cpu(cpu)
		x += per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_stat_diff[item];

	if (x < 0)
		x = 0;
#endif
	return x;
}

#ifdef CONFIG_NUMA
/* See __count_vm_event comment on why raw_cpu_inc is used. */
static inline void
__count_numa_event(struct zone *zone, enum numa_stat_item item)
{
	struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;

	raw_cpu_inc(pzstats->vm_numa_event[item]);
}

static inline void
__count_numa_events(struct zone *zone, enum numa_stat_item item, long delta)
{
	struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;

	raw_cpu_add(pzstats->vm_numa_event[item], delta);
}

extern unsigned long sum_zone_node_page_state(int node,
					      enum zone_stat_item item);
extern unsigned long sum_zone_numa_event_state(int node, enum numa_stat_item item);
extern unsigned long node_page_state(struct pglist_data *pgdat,
						enum node_stat_item item);
extern unsigned long node_page_state_pages(struct pglist_data *pgdat,
					   enum node_stat_item item);
extern void fold_vm_numa_events(void);
#else
#define sum_zone_node_page_state(node, item) global_zone_page_state(item)
#define node_page_state(node, item) global_node_page_state(item)
#define node_page_state_pages(node, item) global_node_page_state_pages(item)
static inline void fold_vm_numa_events(void)
{
}
#endif /* CONFIG_NUMA */

#ifdef CONFIG_SMP
void __mod_zone_page_state(struct zone *, enum zone_stat_item item, long);
void __inc_zone_page_state(struct page *, enum zone_stat_item);
void __dec_zone_page_state(struct page *, enum zone_stat_item);

void __mod_node_page_state(struct pglist_data *, enum node_stat_item item, long);
void __inc_node_page_state(struct page *, enum node_stat_item);
void __dec_node_page_state(struct page *, enum node_stat_item);

void mod_zone_page_state(struct zone *, enum zone_stat_item, long);
void inc_zone_page_state(struct page *, enum zone_stat_item);
void dec_zone_page_state(struct page *, enum zone_stat_item);

void mod_node_page_state(struct pglist_data *, enum node_stat_item, long);
void inc_node_page_state(struct page *, enum node_stat_item);
void dec_node_page_state(struct page *, enum node_stat_item);

extern void inc_node_state(struct pglist_data *, enum node_stat_item);
extern void __inc_zone_state(struct zone *, enum zone_stat_item);
extern void __inc_node_state(struct pglist_data *, enum node_stat_item);
extern void dec_zone_state(struct zone *, enum zone_stat_item);
extern void __dec_zone_state(struct zone *, enum zone_stat_item);
extern void __dec_node_state(struct pglist_data *, enum node_stat_item);

void quiet_vmstat(void);
void cpu_vm_stats_fold(int cpu);
void refresh_zone_stat_thresholds(void);

struct ctl_table;
int vmstat_refresh(struct ctl_table *, int write, void *buffer, size_t *lenp,
		loff_t *ppos);

void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *);

int calculate_pressure_threshold(struct zone *zone);
int calculate_normal_threshold(struct zone *zone);
void set_pgdat_percpu_threshold(pg_data_t *pgdat,
				int (*calculate_pressure)(struct zone *));
#else /* CONFIG_SMP */

/*
 * We do not maintain differentials in a single processor configuration.
 * The functions directly modify the zone and global counters.
 */
static inline void __mod_zone_page_state(struct zone *zone,
			enum zone_stat_item item, long delta)
{
	zone_page_state_add(delta, zone, item);
}

static inline void __mod_node_page_state(struct pglist_data *pgdat,
			enum node_stat_item item, int delta)
{
	if (vmstat_item_in_bytes(item)) {
		/*
		 * Only cgroups use subpage accounting right now; at
		 * the global level, these items still change in
		 * multiples of whole pages. Store them as pages
		 * internally to keep the per-cpu counters compact.
		 */
		VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1));
		delta >>= PAGE_SHIFT;
	}

	node_page_state_add(delta, pgdat, item);
}

static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
{
	atomic_long_inc(&zone->vm_stat[item]);
	atomic_long_inc(&vm_zone_stat[item]);
}

static inline void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
{
	atomic_long_inc(&pgdat->vm_stat[item]);
	atomic_long_inc(&vm_node_stat[item]);
}

static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
{
	atomic_long_dec(&zone->vm_stat[item]);
	atomic_long_dec(&vm_zone_stat[item]);
}

static inline void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
{
	atomic_long_dec(&pgdat->vm_stat[item]);
	atomic_long_dec(&vm_node_stat[item]);
}

static inline void __inc_zone_page_state(struct page *page,
			enum zone_stat_item item)
{
	__inc_zone_state(page_zone(page), item);
}

static inline void __inc_node_page_state(struct page *page,
			enum node_stat_item item)
{
	__inc_node_state(page_pgdat(page), item);
}


static inline void __dec_zone_page_state(struct page *page,
			enum zone_stat_item item)
{
	__dec_zone_state(page_zone(page), item);
}

static inline void __dec_node_page_state(struct page *page,
			enum node_stat_item item)
{
	__dec_node_state(page_pgdat(page), item);
}


/*
 * We only use atomic operations to update counters. So there is no need to
 * disable interrupts.
 */
#define inc_zone_page_state __inc_zone_page_state
#define dec_zone_page_state __dec_zone_page_state
#define mod_zone_page_state __mod_zone_page_state

#define inc_node_page_state __inc_node_page_state
#define dec_node_page_state __dec_node_page_state
#define mod_node_page_state __mod_node_page_state

#define inc_zone_state __inc_zone_state
#define inc_node_state __inc_node_state
#define dec_zone_state __dec_zone_state

#define set_pgdat_percpu_threshold(pgdat, callback) { }

static inline void refresh_zone_stat_thresholds(void) { }
static inline void cpu_vm_stats_fold(int cpu) { }
static inline void quiet_vmstat(void) { }

static inline void drain_zonestat(struct zone *zone,
			struct per_cpu_zonestat *pzstats) { }
#endif		/* CONFIG_SMP */

static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages,
					     int migratetype)
{
	__mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
	if (is_migrate_cma(migratetype))
		__mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages);
}

extern const char * const vmstat_text[];

static inline const char *zone_stat_name(enum zone_stat_item item)
{
	return vmstat_text[item];
}

#ifdef CONFIG_NUMA
static inline const char *numa_stat_name(enum numa_stat_item item)
{
	return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
			   item];
}
#endif /* CONFIG_NUMA */

static inline const char *node_stat_name(enum node_stat_item item)
{
	return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
			   NR_VM_NUMA_EVENT_ITEMS +
			   item];
}

static inline const char *lru_list_name(enum lru_list lru)
{
	return node_stat_name(NR_LRU_BASE + lru) + 3; // skip "nr_"
}

static inline const char *writeback_stat_name(enum writeback_stat_item item)
{
	return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
			   NR_VM_NUMA_EVENT_ITEMS +
			   NR_VM_NODE_STAT_ITEMS +
			   item];
}

#if defined(CONFIG_VM_EVENT_COUNTERS) || defined(CONFIG_MEMCG)
static inline const char *vm_event_name(enum vm_event_item item)
{
	return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
			   NR_VM_NUMA_EVENT_ITEMS +
			   NR_VM_NODE_STAT_ITEMS +
			   NR_VM_WRITEBACK_STAT_ITEMS +
			   item];
}
#endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_MEMCG */

#ifdef CONFIG_MEMCG

void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
			int val);

static inline void mod_lruvec_state(struct lruvec *lruvec,
				    enum node_stat_item idx, int val)
{
	unsigned long flags;

	local_irq_save(flags);
	__mod_lruvec_state(lruvec, idx, val);
	local_irq_restore(flags);
}

void __mod_lruvec_page_state(struct page *page,
			     enum node_stat_item idx, int val);

static inline void mod_lruvec_page_state(struct page *page,
					 enum node_stat_item idx, int val)
{
	unsigned long flags;

	local_irq_save(flags);
	__mod_lruvec_page_state(page, idx, val);
	local_irq_restore(flags);
}

#else

static inline void __mod_lruvec_state(struct lruvec *lruvec,
				      enum node_stat_item idx, int val)
{
	__mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
}

static inline void mod_lruvec_state(struct lruvec *lruvec,
				    enum node_stat_item idx, int val)
{
	mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
}

static inline void __mod_lruvec_page_state(struct page *page,
					   enum node_stat_item idx, int val)
{
	__mod_node_page_state(page_pgdat(page), idx, val);
}

static inline void mod_lruvec_page_state(struct page *page,
					 enum node_stat_item idx, int val)
{
	mod_node_page_state(page_pgdat(page), idx, val);
}

#endif /* CONFIG_MEMCG */

static inline void inc_lruvec_state(struct lruvec *lruvec,
				    enum node_stat_item idx)
{
	mod_lruvec_state(lruvec, idx, 1);
}

static inline void __inc_lruvec_page_state(struct page *page,
					   enum node_stat_item idx)
{
	__mod_lruvec_page_state(page, idx, 1);
}

static inline void __dec_lruvec_page_state(struct page *page,
					   enum node_stat_item idx)
{
	__mod_lruvec_page_state(page, idx, -1);
}

static inline void inc_lruvec_page_state(struct page *page,
					 enum node_stat_item idx)
{
	mod_lruvec_page_state(page, idx, 1);
}

static inline void dec_lruvec_page_state(struct page *page,
					 enum node_stat_item idx)
{
	mod_lruvec_page_state(page, idx, -1);
}

#endif /* _LINUX_VMSTAT_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
f6ac2354 CL	2	#ifndef _LINUX_VMSTAT_H
	3	#define _LINUX_VMSTAT_H
	4
	5	#include <linux/types.h>
	6	#include <linux/percpu.h>
2244b95a	7	#include <linux/mmzone.h>
f042e707	8	#include <linux/vm_event_item.h>
60063497	9	#include <linux/atomic.h>
4518085e	10	#include <linux/static_key.h>
ea426c2a	11	#include <linux/mmdebug.h>
f6ac2354	12
c748e134 AB	13	extern int sysctl_stat_interval;
c748e134 AB	14
4518085e KW	15	#ifdef CONFIG_NUMA
	16	#define ENABLE_NUMA_STAT 1
	17	#define DISABLE_NUMA_STAT 0
	18	extern int sysctl_vm_numa_stat;
	19	DECLARE_STATIC_KEY_TRUE(vm_numa_stat_key);
32927393 CH	20	int sysctl_vm_numa_stat_handler(struct ctl_table *table, int write,
32927393 CH	21	void buffer, size_t length, loff_t *ppos);
4518085e KW	22	#endif
4518085e KW	23
d51d1e64 SR	24	struct reclaim_stat {
	25	unsigned nr_dirty;
	26	unsigned nr_unqueued_dirty;
	27	unsigned nr_congested;
	28	unsigned nr_writeback;
	29	unsigned nr_immediate;
96f8bf4f	30	unsigned nr_pageout;
ed017373	31	unsigned nr_activate[ANON_AND_FILE];
d51d1e64 SR	32	unsigned nr_ref_keep;
d51d1e64 SR	33	unsigned nr_unmap_fail;
1f318a9b	34	unsigned nr_lazyfree_fail;
d51d1e64 SR	35	};
d51d1e64 SR	36
9d7ea9a2 KK	37	enum writeback_stat_item {
	38	NR_DIRTY_THRESHOLD,
	39	NR_DIRTY_BG_THRESHOLD,
	40	NR_VM_WRITEBACK_STAT_ITEMS,
	41	};
	42
780a0656 AM	43	#ifdef CONFIG_VM_EVENT_COUNTERS
	44	/*
	45	* Light weight per cpu counter implementation.
	46	*
	47	* Counters should only be incremented and no critical kernel component
	48	* should rely on the counter values.
	49	*
	50	* Counters are handled completely inline. On many platforms the code
	51	* generated will simply be the increment of a global address.
	52	*/
	53
f8891e5e CL	54	struct vm_event_state {
f8891e5e CL	55	unsigned long event[NR_VM_EVENT_ITEMS];
f6ac2354 CL	56	};
f6ac2354 CL	57
f8891e5e CL	58	DECLARE_PER_CPU(struct vm_event_state, vm_event_states);
f8891e5e CL	59
293b6a4c CL	60	/*
	61	* vm counters are allowed to be racy. Use raw_cpu_ops to avoid the
	62	* local_irq_disable overhead.
	63	*/
f8891e5e CL	64	static inline void __count_vm_event(enum vm_event_item item)
f8891e5e CL	65	{
293b6a4c	66	raw_cpu_inc(vm_event_states.event[item]);
f8891e5e CL	67	}
	68
	69	static inline void count_vm_event(enum vm_event_item item)
	70	{
dd17c8f7	71	this_cpu_inc(vm_event_states.event[item]);
f8891e5e CL	72	}
	73
	74	static inline void __count_vm_events(enum vm_event_item item, long delta)
	75	{
293b6a4c	76	raw_cpu_add(vm_event_states.event[item], delta);
f8891e5e CL	77	}
	78
	79	static inline void count_vm_events(enum vm_event_item item, long delta)
	80	{
dd17c8f7	81	this_cpu_add(vm_event_states.event[item], delta);
f8891e5e CL	82	}
	83
	84	extern void all_vm_events(unsigned long *);
f1cb0879	85
f8891e5e CL	86	extern void vm_events_fold_cpu(int cpu);
	87
	88	#else
	89
	90	/* Disable counters */
780a0656 AM	91	static inline void count_vm_event(enum vm_event_item item)
	92	{
	93	}
	94	static inline void count_vm_events(enum vm_event_item item, long delta)
	95	{
	96	}
	97	static inline void __count_vm_event(enum vm_event_item item)
	98	{
	99	}
	100	static inline void __count_vm_events(enum vm_event_item item, long delta)
	101	{
	102	}
	103	static inline void all_vm_events(unsigned long *ret)
	104	{
	105	}
	106	static inline void vm_events_fold_cpu(int cpu)
	107	{
	108	}
f8891e5e CL	109
	110	#endif /* CONFIG_VM_EVENT_COUNTERS */
	111
03c5a6e1 MG	112	#ifdef CONFIG_NUMA_BALANCING
	113	#define count_vm_numa_event(x) count_vm_event(x)
	114	#define count_vm_numa_events(x, y) count_vm_events(x, y)
	115	#else
	116	#define count_vm_numa_event(x) do {} while (0)
3c0ff468	117	#define count_vm_numa_events(x, y) do { (void)(y); } while (0)
03c5a6e1 MG	118	#endif /* CONFIG_NUMA_BALANCING */
03c5a6e1 MG	119
ec659934 MG	120	#ifdef CONFIG_DEBUG_TLBFLUSH
	121	#define count_vm_tlb_event(x) count_vm_event(x)
	122	#define count_vm_tlb_events(x, y) count_vm_events(x, y)
	123	#else
	124	#define count_vm_tlb_event(x) do {} while (0)
	125	#define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
	126	#endif
	127
4f115147 DB	128	#ifdef CONFIG_DEBUG_VM_VMACACHE
	129	#define count_vm_vmacache_event(x) count_vm_event(x)
	130	#else
	131	#define count_vm_vmacache_event(x) do {} while (0)
	132	#endif
	133
16709d1d MG	134	#define __count_zid_vm_events(item, zid, delta) \
16709d1d MG	135	__count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)
f6ac2354	136
2244b95a	137	/*
75ef7184	138	* Zone and node-based page accounting with per cpu differentials.
2244b95a	139	*/
75ef7184 MG	140	extern atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS];
75ef7184 MG	141	extern atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS];
f19298b9	142	extern atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS];
2244b95a	143
3a321d2a	144	#ifdef CONFIG_NUMA
f19298b9 MG	145	static inline void zone_numa_event_add(long x, struct zone *zone,
f19298b9 MG	146	enum numa_stat_item item)
3a321d2a	147	{
f19298b9 MG	148	atomic_long_add(x, &zone->vm_numa_event[item]);
f19298b9 MG	149	atomic_long_add(x, &vm_numa_event[item]);
3a321d2a KW	150	}
3a321d2a KW	151
f19298b9 MG	152	static inline unsigned long zone_numa_event_state(struct zone *zone,
f19298b9 MG	153	enum numa_stat_item item)
3a321d2a	154	{
f19298b9	155	return atomic_long_read(&zone->vm_numa_event[item]);
3a321d2a KW	156	}
3a321d2a KW	157
f19298b9 MG	158	static inline unsigned long
f19298b9 MG	159	global_numa_event_state(enum numa_stat_item item)
3a321d2a	160	{
f19298b9	161	return atomic_long_read(&vm_numa_event[item]);
3a321d2a KW	162	}
	163	#endif /* CONFIG_NUMA */
	164
2244b95a CL	165	static inline void zone_page_state_add(long x, struct zone *zone,
	166	enum zone_stat_item item)
	167	{
	168	atomic_long_add(x, &zone->vm_stat[item]);
75ef7184 MG	169	atomic_long_add(x, &vm_zone_stat[item]);
	170	}
	171
	172	static inline void node_page_state_add(long x, struct pglist_data *pgdat,
	173	enum node_stat_item item)
	174	{
	175	atomic_long_add(x, &pgdat->vm_stat[item]);
	176	atomic_long_add(x, &vm_node_stat[item]);
2244b95a CL	177	}
2244b95a CL	178
c41f012a	179	static inline unsigned long global_zone_page_state(enum zone_stat_item item)
2244b95a	180	{
75ef7184 MG	181	long x = atomic_long_read(&vm_zone_stat[item]);
	182	#ifdef CONFIG_SMP
	183	if (x < 0)
	184	x = 0;
	185	#endif
	186	return x;
	187	}
	188
ea426c2a RG	189	static inline
ea426c2a RG	190	unsigned long global_node_page_state_pages(enum node_stat_item item)
75ef7184 MG	191	{
75ef7184 MG	192	long x = atomic_long_read(&vm_node_stat[item]);
2244b95a CL	193	#ifdef CONFIG_SMP
	194	if (x < 0)
	195	x = 0;
	196	#endif
	197	return x;
	198	}
	199
ea426c2a RG	200	static inline unsigned long global_node_page_state(enum node_stat_item item)
	201	{
	202	VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
	203
	204	return global_node_page_state_pages(item);
	205	}
	206
2244b95a CL	207	static inline unsigned long zone_page_state(struct zone *zone,
	208	enum zone_stat_item item)
	209	{
	210	long x = atomic_long_read(&zone->vm_stat[item]);
	211	#ifdef CONFIG_SMP
	212	if (x < 0)
	213	x = 0;
	214	#endif
	215	return x;
	216	}
	217
aa454840 CL	218	/*
	219	* More accurate version that also considers the currently pending
	220	* deltas. For that we need to loop over all cpus to find the current
	221	* deltas. There is no synchronization so the result cannot be
	222	* exactly accurate either.
	223	*/
	224	static inline unsigned long zone_page_state_snapshot(struct zone *zone,
	225	enum zone_stat_item item)
	226	{
	227	long x = atomic_long_read(&zone->vm_stat[item]);
	228
	229	#ifdef CONFIG_SMP
	230	int cpu;
	231	for_each_online_cpu(cpu)
28f836b6	232	x += per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_stat_diff[item];
aa454840 CL	233
	234	if (x < 0)
	235	x = 0;
	236	#endif
	237	return x;
	238	}
	239
2244b95a	240	#ifdef CONFIG_NUMA
3ac44a34 MG	241	/* See __count_vm_event comment on why raw_cpu_inc is used. */
	242	static inline void
	243	__count_numa_event(struct zone *zone, enum numa_stat_item item)
	244	{
	245	struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;
	246
	247	raw_cpu_inc(pzstats->vm_numa_event[item]);
	248	}
	249
3e23060b MG	250	static inline void
	251	__count_numa_events(struct zone *zone, enum numa_stat_item item, long delta)
	252	{
	253	struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;
	254
	255	raw_cpu_add(pzstats->vm_numa_event[item], delta);
	256	}
	257
75ef7184	258	extern unsigned long sum_zone_node_page_state(int node,
3a321d2a	259	enum zone_stat_item item);
f19298b9	260	extern unsigned long sum_zone_numa_event_state(int node, enum numa_stat_item item);
75ef7184 MG	261	extern unsigned long node_page_state(struct pglist_data *pgdat,
75ef7184 MG	262	enum node_stat_item item);
ea426c2a RG	263	extern unsigned long node_page_state_pages(struct pglist_data *pgdat,
ea426c2a RG	264	enum node_stat_item item);
f19298b9	265	extern void fold_vm_numa_events(void);
2244b95a	266	#else
c41f012a	267	#define sum_zone_node_page_state(node, item) global_zone_page_state(item)
75ef7184	268	#define node_page_state(node, item) global_node_page_state(item)
ea426c2a	269	#define node_page_state_pages(node, item) global_node_page_state_pages(item)
f19298b9 MG	270	static inline void fold_vm_numa_events(void)
	271	{
	272	}
ca889e6c	273	#endif /* CONFIG_NUMA */
2244b95a	274
2244b95a	275	#ifdef CONFIG_SMP
6cdb18ad	276	void __mod_zone_page_state(struct zone *, enum zone_stat_item item, long);
2244b95a CL	277	void __inc_zone_page_state(struct page *, enum zone_stat_item);
2244b95a CL	278	void __dec_zone_page_state(struct page *, enum zone_stat_item);
f6ac2354	279
75ef7184 MG	280	void __mod_node_page_state(struct pglist_data *, enum node_stat_item item, long);
	281	void __inc_node_page_state(struct page *, enum node_stat_item);
	282	void __dec_node_page_state(struct page *, enum node_stat_item);
	283
6cdb18ad	284	void mod_zone_page_state(struct zone *, enum zone_stat_item, long);
2244b95a CL	285	void inc_zone_page_state(struct page *, enum zone_stat_item);
	286	void dec_zone_page_state(struct page *, enum zone_stat_item);
	287
75ef7184 MG	288	void mod_node_page_state(struct pglist_data *, enum node_stat_item, long);
	289	void inc_node_page_state(struct page *, enum node_stat_item);
	290	void dec_node_page_state(struct page *, enum node_stat_item);
	291
75ef7184	292	extern void inc_node_state(struct pglist_data *, enum node_stat_item);
c8785385	293	extern void __inc_zone_state(struct zone *, enum zone_stat_item);
75ef7184	294	extern void __inc_node_state(struct pglist_data *, enum node_stat_item);
c8785385 CL	295	extern void dec_zone_state(struct zone *, enum zone_stat_item);
c8785385 CL	296	extern void __dec_zone_state(struct zone *, enum zone_stat_item);
75ef7184	297	extern void __dec_node_state(struct pglist_data *, enum node_stat_item);
2244b95a	298
0eb77e98	299	void quiet_vmstat(void);
2bb921e5	300	void cpu_vm_stats_fold(int cpu);
a6cccdc3	301	void refresh_zone_stat_thresholds(void);
b44129b3	302
52b6f46b	303	struct ctl_table;
32927393 CH	304	int vmstat_refresh(struct ctl_table , int write, void buffer, size_t *lenp,
32927393 CH	305	loff_t *ppos);
52b6f46b	306
28f836b6	307	void drain_zonestat(struct zone zone, struct per_cpu_zonestat );
5a883813	308
b44129b3 MG	309	int calculate_pressure_threshold(struct zone *zone);
	310	int calculate_normal_threshold(struct zone *zone);
	311	void set_pgdat_percpu_threshold(pg_data_t *pgdat,
	312	int (calculate_pressure)(struct zone ));
2244b95a CL	313	#else /* CONFIG_SMP */
	314
	315	/*
	316	* We do not maintain differentials in a single processor configuration.
	317	* The functions directly modify the zone and global counters.
	318	*/
	319	static inline void __mod_zone_page_state(struct zone *zone,
6cdb18ad	320	enum zone_stat_item item, long delta)
2244b95a CL	321	{
	322	zone_page_state_add(delta, zone, item);
	323	}
	324
75ef7184 MG	325	static inline void __mod_node_page_state(struct pglist_data *pgdat,
	326	enum node_stat_item item, int delta)
	327	{
be458311	328	if (vmstat_item_in_bytes(item)) {
629484ae JW	329	/*
	330	* Only cgroups use subpage accounting right now; at
	331	* the global level, these items still change in
	332	* multiples of whole pages. Store them as pages
	333	* internally to keep the per-cpu counters compact.
	334	*/
be458311 RG	335	VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1));
	336	delta >>= PAGE_SHIFT;
	337	}
	338
75ef7184 MG	339	node_page_state_add(delta, pgdat, item);
	340	}
	341
7f4599e9 CL	342	static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
	343	{
	344	atomic_long_inc(&zone->vm_stat[item]);
75ef7184 MG	345	atomic_long_inc(&vm_zone_stat[item]);
	346	}
	347
	348	static inline void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
	349	{
	350	atomic_long_inc(&pgdat->vm_stat[item]);
	351	atomic_long_inc(&vm_node_stat[item]);
7f4599e9 CL	352	}
7f4599e9 CL	353
c8785385 CL	354	static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
	355	{
	356	atomic_long_dec(&zone->vm_stat[item]);
75ef7184 MG	357	atomic_long_dec(&vm_zone_stat[item]);
	358	}
	359
	360	static inline void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
	361	{
	362	atomic_long_dec(&pgdat->vm_stat[item]);
	363	atomic_long_dec(&vm_node_stat[item]);
c8785385 CL	364	}
c8785385 CL	365
6a3ed212 JW	366	static inline void __inc_zone_page_state(struct page *page,
	367	enum zone_stat_item item)
	368	{
	369	__inc_zone_state(page_zone(page), item);
	370	}
	371
75ef7184 MG	372	static inline void __inc_node_page_state(struct page *page,
	373	enum node_stat_item item)
	374	{
	375	__inc_node_state(page_pgdat(page), item);
	376	}
	377
	378
2244b95a CL	379	static inline void __dec_zone_page_state(struct page *page,
	380	enum zone_stat_item item)
	381	{
57ce36fe	382	__dec_zone_state(page_zone(page), item);
2244b95a CL	383	}
2244b95a CL	384
75ef7184 MG	385	static inline void __dec_node_page_state(struct page *page,
	386	enum node_stat_item item)
	387	{
	388	__dec_node_state(page_pgdat(page), item);
	389	}
	390
	391
2244b95a CL	392	/*
	393	* We only use atomic operations to update counters. So there is no need to
	394	* disable interrupts.
	395	*/
	396	#define inc_zone_page_state __inc_zone_page_state
	397	#define dec_zone_page_state __dec_zone_page_state
	398	#define mod_zone_page_state __mod_zone_page_state
	399
75ef7184 MG	400	#define inc_node_page_state __inc_node_page_state
	401	#define dec_node_page_state __dec_node_page_state
	402	#define mod_node_page_state __mod_node_page_state
	403
6a3ed212	404	#define inc_zone_state __inc_zone_state
75ef7184	405	#define inc_node_state __inc_node_state
6a3ed212 JW	406	#define dec_zone_state __dec_zone_state
6a3ed212 JW	407
b44129b3	408	#define set_pgdat_percpu_threshold(pgdat, callback) { }
88f5acf8	409
a6cccdc3	410	static inline void refresh_zone_stat_thresholds(void) { }
2bb921e5	411	static inline void cpu_vm_stats_fold(int cpu) { }
0eb77e98	412	static inline void quiet_vmstat(void) { }
a6cccdc3	413
5a883813	414	static inline void drain_zonestat(struct zone *zone,
28f836b6	415	struct per_cpu_zonestat *pzstats) { }
fa25c503 KM	416	#endif /* CONFIG_SMP */
fa25c503 KM	417
d1ce749a BZ	418	static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages,
	419	int migratetype)
	420	{
	421	__mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
	422	if (is_migrate_cma(migratetype))
	423	__mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages);
	424	}
	425
fa25c503	426	extern const char * const vmstat_text[];
2244b95a	427
9d7ea9a2 KK	428	static inline const char *zone_stat_name(enum zone_stat_item item)
	429	{
	430	return vmstat_text[item];
	431	}
	432
	433	#ifdef CONFIG_NUMA
	434	static inline const char *numa_stat_name(enum numa_stat_item item)
	435	{
	436	return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
	437	item];
	438	}
	439	#endif /* CONFIG_NUMA */
	440
	441	static inline const char *node_stat_name(enum node_stat_item item)
	442	{
	443	return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
f19298b9	444	NR_VM_NUMA_EVENT_ITEMS +
9d7ea9a2 KK	445	item];
	446	}
	447
	448	static inline const char *lru_list_name(enum lru_list lru)
	449	{
	450	return node_stat_name(NR_LRU_BASE + lru) + 3; // skip "nr_"
	451	}
	452
	453	static inline const char *writeback_stat_name(enum writeback_stat_item item)
	454	{
	455	return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
f19298b9	456	NR_VM_NUMA_EVENT_ITEMS +
9d7ea9a2 KK	457	NR_VM_NODE_STAT_ITEMS +
	458	item];
	459	}
	460
ebc5d83d	461	#if defined(CONFIG_VM_EVENT_COUNTERS) \|\| defined(CONFIG_MEMCG)
9d7ea9a2 KK	462	static inline const char *vm_event_name(enum vm_event_item item)
	463	{
	464	return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
f19298b9	465	NR_VM_NUMA_EVENT_ITEMS +
9d7ea9a2 KK	466	NR_VM_NODE_STAT_ITEMS +
	467	NR_VM_WRITEBACK_STAT_ITEMS +
	468	item];
	469	}
ebc5d83d	470	#endif /* CONFIG_VM_EVENT_COUNTERS \|\| CONFIG_MEMCG */
9d7ea9a2	471
c47d5032 SB	472	#ifdef CONFIG_MEMCG
	473
	474	void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
	475	int val);
	476
	477	static inline void mod_lruvec_state(struct lruvec *lruvec,
	478	enum node_stat_item idx, int val)
	479	{
	480	unsigned long flags;
	481
	482	local_irq_save(flags);
	483	__mod_lruvec_state(lruvec, idx, val);
	484	local_irq_restore(flags);
	485	}
	486
	487	void __mod_lruvec_page_state(struct page *page,
	488	enum node_stat_item idx, int val);
	489
	490	static inline void mod_lruvec_page_state(struct page *page,
	491	enum node_stat_item idx, int val)
	492	{
	493	unsigned long flags;
	494
	495	local_irq_save(flags);
	496	__mod_lruvec_page_state(page, idx, val);
	497	local_irq_restore(flags);
	498	}
	499
	500	#else
	501
	502	static inline void __mod_lruvec_state(struct lruvec *lruvec,
	503	enum node_stat_item idx, int val)
	504	{
	505	__mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
	506	}
	507
	508	static inline void mod_lruvec_state(struct lruvec *lruvec,
	509	enum node_stat_item idx, int val)
	510	{
	511	mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
	512	}
	513
	514	static inline void __mod_lruvec_page_state(struct page *page,
	515	enum node_stat_item idx, int val)
	516	{
	517	__mod_node_page_state(page_pgdat(page), idx, val);
	518	}
	519
	520	static inline void mod_lruvec_page_state(struct page *page,
	521	enum node_stat_item idx, int val)
	522	{
	523	mod_node_page_state(page_pgdat(page), idx, val);
	524	}
	525
	526	#endif /* CONFIG_MEMCG */
	527
1c824a68 JW	528	static inline void inc_lruvec_state(struct lruvec *lruvec,
1c824a68 JW	529	enum node_stat_item idx)
c47d5032	530	{
1c824a68	531	mod_lruvec_state(lruvec, idx, 1);
c47d5032 SB	532	}
	533
	534	static inline void __inc_lruvec_page_state(struct page *page,
	535	enum node_stat_item idx)
	536	{
	537	__mod_lruvec_page_state(page, idx, 1);
	538	}
	539
	540	static inline void __dec_lruvec_page_state(struct page *page,
	541	enum node_stat_item idx)
	542	{
	543	__mod_lruvec_page_state(page, idx, -1);
	544	}
	545
c47d5032 SB	546	static inline void inc_lruvec_page_state(struct page *page,
	547	enum node_stat_item idx)
	548	{
	549	mod_lruvec_page_state(page, idx, 1);
	550	}
	551
	552	static inline void dec_lruvec_page_state(struct page *page,
	553	enum node_stat_item idx)
	554	{
	555	mod_lruvec_page_state(page, idx, -1);
	556	}
	557
2244b95a	558	#endif /* _LINUX_VMSTAT_H */