[linux-2.6-block.git] / include / linux / vmstat.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_VMSTAT_H
#define _LINUX_VMSTAT_H

#include <linux/types.h>
#include <linux/percpu.h>
#include <linux/mmzone.h>
#include <linux/vm_event_item.h>
#include <linux/atomic.h>
#include <linux/static_key.h>

extern int sysctl_stat_interval;

#ifdef CONFIG_NUMA
#define ENABLE_NUMA_STAT   1
#define DISABLE_NUMA_STAT   0
extern int sysctl_vm_numa_stat;
DECLARE_STATIC_KEY_TRUE(vm_numa_stat_key);
int sysctl_vm_numa_stat_handler(struct ctl_table *table, int write,
		void *buffer, size_t *length, loff_t *ppos);
#endif

struct reclaim_stat {
	unsigned nr_dirty;
	unsigned nr_unqueued_dirty;
	unsigned nr_congested;
	unsigned nr_writeback;
	unsigned nr_immediate;
	unsigned nr_activate[2];
	unsigned nr_ref_keep;
	unsigned nr_unmap_fail;
};

enum writeback_stat_item {
	NR_DIRTY_THRESHOLD,
	NR_DIRTY_BG_THRESHOLD,
	NR_VM_WRITEBACK_STAT_ITEMS,
};

#ifdef CONFIG_VM_EVENT_COUNTERS
/*
 * Light weight per cpu counter implementation.
 *
 * Counters should only be incremented and no critical kernel component
 * should rely on the counter values.
 *
 * Counters are handled completely inline. On many platforms the code
 * generated will simply be the increment of a global address.
 */

struct vm_event_state {
	unsigned long event[NR_VM_EVENT_ITEMS];
};

DECLARE_PER_CPU(struct vm_event_state, vm_event_states);

/*
 * vm counters are allowed to be racy. Use raw_cpu_ops to avoid the
 * local_irq_disable overhead.
 */
static inline void __count_vm_event(enum vm_event_item item)
{
	raw_cpu_inc(vm_event_states.event[item]);
}

static inline void count_vm_event(enum vm_event_item item)
{
	this_cpu_inc(vm_event_states.event[item]);
}

static inline void __count_vm_events(enum vm_event_item item, long delta)
{
	raw_cpu_add(vm_event_states.event[item], delta);
}

static inline void count_vm_events(enum vm_event_item item, long delta)
{
	this_cpu_add(vm_event_states.event[item], delta);
}

extern void all_vm_events(unsigned long *);

extern void vm_events_fold_cpu(int cpu);

#else

/* Disable counters */
static inline void count_vm_event(enum vm_event_item item)
{
}
static inline void count_vm_events(enum vm_event_item item, long delta)
{
}
static inline void __count_vm_event(enum vm_event_item item)
{
}
static inline void __count_vm_events(enum vm_event_item item, long delta)
{
}
static inline void all_vm_events(unsigned long *ret)
{
}
static inline void vm_events_fold_cpu(int cpu)
{
}

#endif /* CONFIG_VM_EVENT_COUNTERS */

#ifdef CONFIG_NUMA_BALANCING
#define count_vm_numa_event(x)     count_vm_event(x)
#define count_vm_numa_events(x, y) count_vm_events(x, y)
#else
#define count_vm_numa_event(x) do {} while (0)
#define count_vm_numa_events(x, y) do { (void)(y); } while (0)
#endif /* CONFIG_NUMA_BALANCING */

#ifdef CONFIG_DEBUG_TLBFLUSH
#define count_vm_tlb_event(x)	   count_vm_event(x)
#define count_vm_tlb_events(x, y)  count_vm_events(x, y)
#else
#define count_vm_tlb_event(x)     do {} while (0)
#define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
#endif

#ifdef CONFIG_DEBUG_VM_VMACACHE
#define count_vm_vmacache_event(x) count_vm_event(x)
#else
#define count_vm_vmacache_event(x) do {} while (0)
#endif

#define __count_zid_vm_events(item, zid, delta) \
	__count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)

/*
 * Zone and node-based page accounting with per cpu differentials.
 */
extern atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS];
extern atomic_long_t vm_numa_stat[NR_VM_NUMA_STAT_ITEMS];
extern atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS];

#ifdef CONFIG_NUMA
static inline void zone_numa_state_add(long x, struct zone *zone,
				 enum numa_stat_item item)
{
	atomic_long_add(x, &zone->vm_numa_stat[item]);
	atomic_long_add(x, &vm_numa_stat[item]);
}

static inline unsigned long global_numa_state(enum numa_stat_item item)
{
	long x = atomic_long_read(&vm_numa_stat[item]);

	return x;
}

static inline unsigned long zone_numa_state_snapshot(struct zone *zone,
					enum numa_stat_item item)
{
	long x = atomic_long_read(&zone->vm_numa_stat[item]);
	int cpu;

	for_each_online_cpu(cpu)
		x += per_cpu_ptr(zone->pageset, cpu)->vm_numa_stat_diff[item];

	return x;
}
#endif /* CONFIG_NUMA */

static inline void zone_page_state_add(long x, struct zone *zone,
				 enum zone_stat_item item)
{
	atomic_long_add(x, &zone->vm_stat[item]);
	atomic_long_add(x, &vm_zone_stat[item]);
}

static inline void node_page_state_add(long x, struct pglist_data *pgdat,
				 enum node_stat_item item)
{
	atomic_long_add(x, &pgdat->vm_stat[item]);
	atomic_long_add(x, &vm_node_stat[item]);
}

static inline unsigned long global_zone_page_state(enum zone_stat_item item)
{
	long x = atomic_long_read(&vm_zone_stat[item]);
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
}

static inline unsigned long global_node_page_state(enum node_stat_item item)
{
	long x = atomic_long_read(&vm_node_stat[item]);
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
}

static inline unsigned long zone_page_state(struct zone *zone,
					enum zone_stat_item item)
{
	long x = atomic_long_read(&zone->vm_stat[item]);
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
}

/*
 * More accurate version that also considers the currently pending
 * deltas. For that we need to loop over all cpus to find the current
 * deltas. There is no synchronization so the result cannot be
 * exactly accurate either.
 */
static inline unsigned long zone_page_state_snapshot(struct zone *zone,
					enum zone_stat_item item)
{
	long x = atomic_long_read(&zone->vm_stat[item]);

#ifdef CONFIG_SMP
	int cpu;
	for_each_online_cpu(cpu)
		x += per_cpu_ptr(zone->pageset, cpu)->vm_stat_diff[item];

	if (x < 0)
		x = 0;
#endif
	return x;
}

#ifdef CONFIG_NUMA
extern void __inc_numa_state(struct zone *zone, enum numa_stat_item item);
extern unsigned long sum_zone_node_page_state(int node,
					      enum zone_stat_item item);
extern unsigned long sum_zone_numa_state(int node, enum numa_stat_item item);
extern unsigned long node_page_state(struct pglist_data *pgdat,
						enum node_stat_item item);
#else
#define sum_zone_node_page_state(node, item) global_zone_page_state(item)
#define node_page_state(node, item) global_node_page_state(item)
#endif /* CONFIG_NUMA */

#ifdef CONFIG_SMP
void __mod_zone_page_state(struct zone *, enum zone_stat_item item, long);
void __inc_zone_page_state(struct page *, enum zone_stat_item);
void __dec_zone_page_state(struct page *, enum zone_stat_item);

void __mod_node_page_state(struct pglist_data *, enum node_stat_item item, long);
void __inc_node_page_state(struct page *, enum node_stat_item);
void __dec_node_page_state(struct page *, enum node_stat_item);

void mod_zone_page_state(struct zone *, enum zone_stat_item, long);
void inc_zone_page_state(struct page *, enum zone_stat_item);
void dec_zone_page_state(struct page *, enum zone_stat_item);

void mod_node_page_state(struct pglist_data *, enum node_stat_item, long);
void inc_node_page_state(struct page *, enum node_stat_item);
void dec_node_page_state(struct page *, enum node_stat_item);

extern void inc_node_state(struct pglist_data *, enum node_stat_item);
extern void __inc_zone_state(struct zone *, enum zone_stat_item);
extern void __inc_node_state(struct pglist_data *, enum node_stat_item);
extern void dec_zone_state(struct zone *, enum zone_stat_item);
extern void __dec_zone_state(struct zone *, enum zone_stat_item);
extern void __dec_node_state(struct pglist_data *, enum node_stat_item);

void quiet_vmstat(void);
void cpu_vm_stats_fold(int cpu);
void refresh_zone_stat_thresholds(void);

struct ctl_table;
int vmstat_refresh(struct ctl_table *, int write, void *buffer, size_t *lenp,
		loff_t *ppos);

void drain_zonestat(struct zone *zone, struct per_cpu_pageset *);

int calculate_pressure_threshold(struct zone *zone);
int calculate_normal_threshold(struct zone *zone);
void set_pgdat_percpu_threshold(pg_data_t *pgdat,
				int (*calculate_pressure)(struct zone *));
#else /* CONFIG_SMP */

/*
 * We do not maintain differentials in a single processor configuration.
 * The functions directly modify the zone and global counters.
 */
static inline void __mod_zone_page_state(struct zone *zone,
			enum zone_stat_item item, long delta)
{
	zone_page_state_add(delta, zone, item);
}

static inline void __mod_node_page_state(struct pglist_data *pgdat,
			enum node_stat_item item, int delta)
{
	node_page_state_add(delta, pgdat, item);
}

static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
{
	atomic_long_inc(&zone->vm_stat[item]);
	atomic_long_inc(&vm_zone_stat[item]);
}

static inline void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
{
	atomic_long_inc(&pgdat->vm_stat[item]);
	atomic_long_inc(&vm_node_stat[item]);
}

static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
{
	atomic_long_dec(&zone->vm_stat[item]);
	atomic_long_dec(&vm_zone_stat[item]);
}

static inline void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
{
	atomic_long_dec(&pgdat->vm_stat[item]);
	atomic_long_dec(&vm_node_stat[item]);
}

static inline void __inc_zone_page_state(struct page *page,
			enum zone_stat_item item)
{
	__inc_zone_state(page_zone(page), item);
}

static inline void __inc_node_page_state(struct page *page,
			enum node_stat_item item)
{
	__inc_node_state(page_pgdat(page), item);
}


static inline void __dec_zone_page_state(struct page *page,
			enum zone_stat_item item)
{
	__dec_zone_state(page_zone(page), item);
}

static inline void __dec_node_page_state(struct page *page,
			enum node_stat_item item)
{
	__dec_node_state(page_pgdat(page), item);
}


/*
 * We only use atomic operations to update counters. So there is no need to
 * disable interrupts.
 */
#define inc_zone_page_state __inc_zone_page_state
#define dec_zone_page_state __dec_zone_page_state
#define mod_zone_page_state __mod_zone_page_state

#define inc_node_page_state __inc_node_page_state
#define dec_node_page_state __dec_node_page_state
#define mod_node_page_state __mod_node_page_state

#define inc_zone_state __inc_zone_state
#define inc_node_state __inc_node_state
#define dec_zone_state __dec_zone_state

#define set_pgdat_percpu_threshold(pgdat, callback) { }

static inline void refresh_zone_stat_thresholds(void) { }
static inline void cpu_vm_stats_fold(int cpu) { }
static inline void quiet_vmstat(void) { }

static inline void drain_zonestat(struct zone *zone,
			struct per_cpu_pageset *pset) { }
#endif		/* CONFIG_SMP */

static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages,
					     int migratetype)
{
	__mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
	if (is_migrate_cma(migratetype))
		__mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages);
}

extern const char * const vmstat_text[];

static inline const char *zone_stat_name(enum zone_stat_item item)
{
	return vmstat_text[item];
}

#ifdef CONFIG_NUMA
static inline const char *numa_stat_name(enum numa_stat_item item)
{
	return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
			   item];
}
#endif /* CONFIG_NUMA */

static inline const char *node_stat_name(enum node_stat_item item)
{
	return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
			   NR_VM_NUMA_STAT_ITEMS +
			   item];
}

static inline const char *lru_list_name(enum lru_list lru)
{
	return node_stat_name(NR_LRU_BASE + lru) + 3; // skip "nr_"
}

static inline const char *writeback_stat_name(enum writeback_stat_item item)
{
	return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
			   NR_VM_NUMA_STAT_ITEMS +
			   NR_VM_NODE_STAT_ITEMS +
			   item];
}

#if defined(CONFIG_VM_EVENT_COUNTERS) || defined(CONFIG_MEMCG)
static inline const char *vm_event_name(enum vm_event_item item)
{
	return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
			   NR_VM_NUMA_STAT_ITEMS +
			   NR_VM_NODE_STAT_ITEMS +
			   NR_VM_WRITEBACK_STAT_ITEMS +
			   item];
}
#endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_MEMCG */

#endif /* _LINUX_VMSTAT_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
f6ac2354 CL	2	#ifndef _LINUX_VMSTAT_H
	3	#define _LINUX_VMSTAT_H
	4
	5	#include <linux/types.h>
	6	#include <linux/percpu.h>
2244b95a	7	#include <linux/mmzone.h>
f042e707	8	#include <linux/vm_event_item.h>
60063497	9	#include <linux/atomic.h>
4518085e	10	#include <linux/static_key.h>
f6ac2354	11
c748e134 AB	12	extern int sysctl_stat_interval;
c748e134 AB	13
4518085e KW	14	#ifdef CONFIG_NUMA
	15	#define ENABLE_NUMA_STAT 1
	16	#define DISABLE_NUMA_STAT 0
	17	extern int sysctl_vm_numa_stat;
	18	DECLARE_STATIC_KEY_TRUE(vm_numa_stat_key);
32927393 CH	19	int sysctl_vm_numa_stat_handler(struct ctl_table *table, int write,
32927393 CH	20	void buffer, size_t length, loff_t *ppos);
4518085e KW	21	#endif
4518085e KW	22
d51d1e64 SR	23	struct reclaim_stat {
	24	unsigned nr_dirty;
	25	unsigned nr_unqueued_dirty;
	26	unsigned nr_congested;
	27	unsigned nr_writeback;
	28	unsigned nr_immediate;
886cf190	29	unsigned nr_activate[2];
d51d1e64 SR	30	unsigned nr_ref_keep;
	31	unsigned nr_unmap_fail;
	32	};
	33
9d7ea9a2 KK	34	enum writeback_stat_item {
	35	NR_DIRTY_THRESHOLD,
	36	NR_DIRTY_BG_THRESHOLD,
	37	NR_VM_WRITEBACK_STAT_ITEMS,
	38	};
	39
780a0656 AM	40	#ifdef CONFIG_VM_EVENT_COUNTERS
	41	/*
	42	* Light weight per cpu counter implementation.
	43	*
	44	* Counters should only be incremented and no critical kernel component
	45	* should rely on the counter values.
	46	*
	47	* Counters are handled completely inline. On many platforms the code
	48	* generated will simply be the increment of a global address.
	49	*/
	50
f8891e5e CL	51	struct vm_event_state {
f8891e5e CL	52	unsigned long event[NR_VM_EVENT_ITEMS];
f6ac2354 CL	53	};
f6ac2354 CL	54
f8891e5e CL	55	DECLARE_PER_CPU(struct vm_event_state, vm_event_states);
f8891e5e CL	56
293b6a4c CL	57	/*
	58	* vm counters are allowed to be racy. Use raw_cpu_ops to avoid the
	59	* local_irq_disable overhead.
	60	*/
f8891e5e CL	61	static inline void __count_vm_event(enum vm_event_item item)
f8891e5e CL	62	{
293b6a4c	63	raw_cpu_inc(vm_event_states.event[item]);
f8891e5e CL	64	}
	65
	66	static inline void count_vm_event(enum vm_event_item item)
	67	{
dd17c8f7	68	this_cpu_inc(vm_event_states.event[item]);
f8891e5e CL	69	}
	70
	71	static inline void __count_vm_events(enum vm_event_item item, long delta)
	72	{
293b6a4c	73	raw_cpu_add(vm_event_states.event[item], delta);
f8891e5e CL	74	}
	75
	76	static inline void count_vm_events(enum vm_event_item item, long delta)
	77	{
dd17c8f7	78	this_cpu_add(vm_event_states.event[item], delta);
f8891e5e CL	79	}
	80
	81	extern void all_vm_events(unsigned long *);
f1cb0879	82
f8891e5e CL	83	extern void vm_events_fold_cpu(int cpu);
	84
	85	#else
	86
	87	/* Disable counters */
780a0656 AM	88	static inline void count_vm_event(enum vm_event_item item)
	89	{
	90	}
	91	static inline void count_vm_events(enum vm_event_item item, long delta)
	92	{
	93	}
	94	static inline void __count_vm_event(enum vm_event_item item)
	95	{
	96	}
	97	static inline void __count_vm_events(enum vm_event_item item, long delta)
	98	{
	99	}
	100	static inline void all_vm_events(unsigned long *ret)
	101	{
	102	}
	103	static inline void vm_events_fold_cpu(int cpu)
	104	{
	105	}
f8891e5e CL	106
	107	#endif /* CONFIG_VM_EVENT_COUNTERS */
	108
03c5a6e1 MG	109	#ifdef CONFIG_NUMA_BALANCING
	110	#define count_vm_numa_event(x) count_vm_event(x)
	111	#define count_vm_numa_events(x, y) count_vm_events(x, y)
	112	#else
	113	#define count_vm_numa_event(x) do {} while (0)
3c0ff468	114	#define count_vm_numa_events(x, y) do { (void)(y); } while (0)
03c5a6e1 MG	115	#endif /* CONFIG_NUMA_BALANCING */
03c5a6e1 MG	116
ec659934 MG	117	#ifdef CONFIG_DEBUG_TLBFLUSH
	118	#define count_vm_tlb_event(x) count_vm_event(x)
	119	#define count_vm_tlb_events(x, y) count_vm_events(x, y)
	120	#else
	121	#define count_vm_tlb_event(x) do {} while (0)
	122	#define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
	123	#endif
	124
4f115147 DB	125	#ifdef CONFIG_DEBUG_VM_VMACACHE
	126	#define count_vm_vmacache_event(x) count_vm_event(x)
	127	#else
	128	#define count_vm_vmacache_event(x) do {} while (0)
	129	#endif
	130
16709d1d MG	131	#define __count_zid_vm_events(item, zid, delta) \
16709d1d MG	132	__count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)
f6ac2354	133
2244b95a	134	/*
75ef7184	135	* Zone and node-based page accounting with per cpu differentials.
2244b95a	136	*/
75ef7184	137	extern atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS];
3a321d2a	138	extern atomic_long_t vm_numa_stat[NR_VM_NUMA_STAT_ITEMS];
75ef7184	139	extern atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS];
2244b95a	140
3a321d2a KW	141	#ifdef CONFIG_NUMA
	142	static inline void zone_numa_state_add(long x, struct zone *zone,
	143	enum numa_stat_item item)
	144	{
	145	atomic_long_add(x, &zone->vm_numa_stat[item]);
	146	atomic_long_add(x, &vm_numa_stat[item]);
	147	}
	148
	149	static inline unsigned long global_numa_state(enum numa_stat_item item)
	150	{
	151	long x = atomic_long_read(&vm_numa_stat[item]);
	152
	153	return x;
	154	}
	155
63803222	156	static inline unsigned long zone_numa_state_snapshot(struct zone *zone,
3a321d2a KW	157	enum numa_stat_item item)
	158	{
	159	long x = atomic_long_read(&zone->vm_numa_stat[item]);
63803222 KW	160	int cpu;
	161
	162	for_each_online_cpu(cpu)
	163	x += per_cpu_ptr(zone->pageset, cpu)->vm_numa_stat_diff[item];
3a321d2a KW	164
	165	return x;
	166	}
	167	#endif /* CONFIG_NUMA */
	168
2244b95a CL	169	static inline void zone_page_state_add(long x, struct zone *zone,
	170	enum zone_stat_item item)
	171	{
	172	atomic_long_add(x, &zone->vm_stat[item]);
75ef7184 MG	173	atomic_long_add(x, &vm_zone_stat[item]);
	174	}
	175
	176	static inline void node_page_state_add(long x, struct pglist_data *pgdat,
	177	enum node_stat_item item)
	178	{
	179	atomic_long_add(x, &pgdat->vm_stat[item]);
	180	atomic_long_add(x, &vm_node_stat[item]);
2244b95a CL	181	}
2244b95a CL	182
c41f012a	183	static inline unsigned long global_zone_page_state(enum zone_stat_item item)
2244b95a	184	{
75ef7184 MG	185	long x = atomic_long_read(&vm_zone_stat[item]);
	186	#ifdef CONFIG_SMP
	187	if (x < 0)
	188	x = 0;
	189	#endif
	190	return x;
	191	}
	192
	193	static inline unsigned long global_node_page_state(enum node_stat_item item)
	194	{
	195	long x = atomic_long_read(&vm_node_stat[item]);
2244b95a CL	196	#ifdef CONFIG_SMP
	197	if (x < 0)
	198	x = 0;
	199	#endif
	200	return x;
	201	}
	202
	203	static inline unsigned long zone_page_state(struct zone *zone,
	204	enum zone_stat_item item)
	205	{
	206	long x = atomic_long_read(&zone->vm_stat[item]);
	207	#ifdef CONFIG_SMP
	208	if (x < 0)
	209	x = 0;
	210	#endif
	211	return x;
	212	}
	213
aa454840 CL	214	/*
	215	* More accurate version that also considers the currently pending
	216	* deltas. For that we need to loop over all cpus to find the current
	217	* deltas. There is no synchronization so the result cannot be
	218	* exactly accurate either.
	219	*/
	220	static inline unsigned long zone_page_state_snapshot(struct zone *zone,
	221	enum zone_stat_item item)
	222	{
	223	long x = atomic_long_read(&zone->vm_stat[item]);
	224
	225	#ifdef CONFIG_SMP
	226	int cpu;
	227	for_each_online_cpu(cpu)
	228	x += per_cpu_ptr(zone->pageset, cpu)->vm_stat_diff[item];
	229
	230	if (x < 0)
	231	x = 0;
	232	#endif
	233	return x;
	234	}
	235
2244b95a	236	#ifdef CONFIG_NUMA
3a321d2a	237	extern void __inc_numa_state(struct zone *zone, enum numa_stat_item item);
75ef7184	238	extern unsigned long sum_zone_node_page_state(int node,
3a321d2a KW	239	enum zone_stat_item item);
3a321d2a KW	240	extern unsigned long sum_zone_numa_state(int node, enum numa_stat_item item);
75ef7184 MG	241	extern unsigned long node_page_state(struct pglist_data *pgdat,
75ef7184 MG	242	enum node_stat_item item);
2244b95a	243	#else
c41f012a	244	#define sum_zone_node_page_state(node, item) global_zone_page_state(item)
75ef7184	245	#define node_page_state(node, item) global_node_page_state(item)
ca889e6c	246	#endif /* CONFIG_NUMA */
2244b95a	247
2244b95a	248	#ifdef CONFIG_SMP
6cdb18ad	249	void __mod_zone_page_state(struct zone *, enum zone_stat_item item, long);
2244b95a CL	250	void __inc_zone_page_state(struct page *, enum zone_stat_item);
2244b95a CL	251	void __dec_zone_page_state(struct page *, enum zone_stat_item);
f6ac2354	252
75ef7184 MG	253	void __mod_node_page_state(struct pglist_data *, enum node_stat_item item, long);
	254	void __inc_node_page_state(struct page *, enum node_stat_item);
	255	void __dec_node_page_state(struct page *, enum node_stat_item);
	256
6cdb18ad	257	void mod_zone_page_state(struct zone *, enum zone_stat_item, long);
2244b95a CL	258	void inc_zone_page_state(struct page *, enum zone_stat_item);
	259	void dec_zone_page_state(struct page *, enum zone_stat_item);
	260
75ef7184 MG	261	void mod_node_page_state(struct pglist_data *, enum node_stat_item, long);
	262	void inc_node_page_state(struct page *, enum node_stat_item);
	263	void dec_node_page_state(struct page *, enum node_stat_item);
	264
75ef7184	265	extern void inc_node_state(struct pglist_data *, enum node_stat_item);
c8785385	266	extern void __inc_zone_state(struct zone *, enum zone_stat_item);
75ef7184	267	extern void __inc_node_state(struct pglist_data *, enum node_stat_item);
c8785385 CL	268	extern void dec_zone_state(struct zone *, enum zone_stat_item);
c8785385 CL	269	extern void __dec_zone_state(struct zone *, enum zone_stat_item);
75ef7184	270	extern void __dec_node_state(struct pglist_data *, enum node_stat_item);
2244b95a	271
0eb77e98	272	void quiet_vmstat(void);
2bb921e5	273	void cpu_vm_stats_fold(int cpu);
a6cccdc3	274	void refresh_zone_stat_thresholds(void);
b44129b3	275
52b6f46b	276	struct ctl_table;
32927393 CH	277	int vmstat_refresh(struct ctl_table , int write, void buffer, size_t *lenp,
32927393 CH	278	loff_t *ppos);
52b6f46b	279
5a883813 MK	280	void drain_zonestat(struct zone zone, struct per_cpu_pageset );
5a883813 MK	281
b44129b3 MG	282	int calculate_pressure_threshold(struct zone *zone);
	283	int calculate_normal_threshold(struct zone *zone);
	284	void set_pgdat_percpu_threshold(pg_data_t *pgdat,
	285	int (calculate_pressure)(struct zone ));
2244b95a CL	286	#else /* CONFIG_SMP */
	287
	288	/*
	289	* We do not maintain differentials in a single processor configuration.
	290	* The functions directly modify the zone and global counters.
	291	*/
	292	static inline void __mod_zone_page_state(struct zone *zone,
6cdb18ad	293	enum zone_stat_item item, long delta)
2244b95a CL	294	{
	295	zone_page_state_add(delta, zone, item);
	296	}
	297
75ef7184 MG	298	static inline void __mod_node_page_state(struct pglist_data *pgdat,
	299	enum node_stat_item item, int delta)
	300	{
	301	node_page_state_add(delta, pgdat, item);
	302	}
	303
7f4599e9 CL	304	static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
	305	{
	306	atomic_long_inc(&zone->vm_stat[item]);
75ef7184 MG	307	atomic_long_inc(&vm_zone_stat[item]);
	308	}
	309
	310	static inline void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
	311	{
	312	atomic_long_inc(&pgdat->vm_stat[item]);
	313	atomic_long_inc(&vm_node_stat[item]);
7f4599e9 CL	314	}
7f4599e9 CL	315
c8785385 CL	316	static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
	317	{
	318	atomic_long_dec(&zone->vm_stat[item]);
75ef7184 MG	319	atomic_long_dec(&vm_zone_stat[item]);
	320	}
	321
	322	static inline void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
	323	{
	324	atomic_long_dec(&pgdat->vm_stat[item]);
	325	atomic_long_dec(&vm_node_stat[item]);
c8785385 CL	326	}
c8785385 CL	327
6a3ed212 JW	328	static inline void __inc_zone_page_state(struct page *page,
	329	enum zone_stat_item item)
	330	{
	331	__inc_zone_state(page_zone(page), item);
	332	}
	333
75ef7184 MG	334	static inline void __inc_node_page_state(struct page *page,
	335	enum node_stat_item item)
	336	{
	337	__inc_node_state(page_pgdat(page), item);
	338	}
	339
	340
2244b95a CL	341	static inline void __dec_zone_page_state(struct page *page,
	342	enum zone_stat_item item)
	343	{
57ce36fe	344	__dec_zone_state(page_zone(page), item);
2244b95a CL	345	}
2244b95a CL	346
75ef7184 MG	347	static inline void __dec_node_page_state(struct page *page,
	348	enum node_stat_item item)
	349	{
	350	__dec_node_state(page_pgdat(page), item);
	351	}
	352
	353
2244b95a CL	354	/*
	355	* We only use atomic operations to update counters. So there is no need to
	356	* disable interrupts.
	357	*/
	358	#define inc_zone_page_state __inc_zone_page_state
	359	#define dec_zone_page_state __dec_zone_page_state
	360	#define mod_zone_page_state __mod_zone_page_state
	361
75ef7184 MG	362	#define inc_node_page_state __inc_node_page_state
	363	#define dec_node_page_state __dec_node_page_state
	364	#define mod_node_page_state __mod_node_page_state
	365
6a3ed212	366	#define inc_zone_state __inc_zone_state
75ef7184	367	#define inc_node_state __inc_node_state
6a3ed212 JW	368	#define dec_zone_state __dec_zone_state
6a3ed212 JW	369
b44129b3	370	#define set_pgdat_percpu_threshold(pgdat, callback) { }
88f5acf8	371
a6cccdc3	372	static inline void refresh_zone_stat_thresholds(void) { }
2bb921e5	373	static inline void cpu_vm_stats_fold(int cpu) { }
0eb77e98	374	static inline void quiet_vmstat(void) { }
a6cccdc3	375
5a883813 MK	376	static inline void drain_zonestat(struct zone *zone,
5a883813 MK	377	struct per_cpu_pageset *pset) { }
fa25c503 KM	378	#endif /* CONFIG_SMP */
fa25c503 KM	379
d1ce749a BZ	380	static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages,
	381	int migratetype)
	382	{
	383	__mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
	384	if (is_migrate_cma(migratetype))
	385	__mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages);
	386	}
	387
fa25c503	388	extern const char * const vmstat_text[];
2244b95a	389
9d7ea9a2 KK	390	static inline const char *zone_stat_name(enum zone_stat_item item)
	391	{
	392	return vmstat_text[item];
	393	}
	394
	395	#ifdef CONFIG_NUMA
	396	static inline const char *numa_stat_name(enum numa_stat_item item)
	397	{
	398	return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
	399	item];
	400	}
	401	#endif /* CONFIG_NUMA */
	402
	403	static inline const char *node_stat_name(enum node_stat_item item)
	404	{
	405	return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
	406	NR_VM_NUMA_STAT_ITEMS +
	407	item];
	408	}
	409
	410	static inline const char *lru_list_name(enum lru_list lru)
	411	{
	412	return node_stat_name(NR_LRU_BASE + lru) + 3; // skip "nr_"
	413	}
	414
	415	static inline const char *writeback_stat_name(enum writeback_stat_item item)
	416	{
	417	return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
	418	NR_VM_NUMA_STAT_ITEMS +
	419	NR_VM_NODE_STAT_ITEMS +
	420	item];
	421	}
	422
ebc5d83d	423	#if defined(CONFIG_VM_EVENT_COUNTERS) \|\| defined(CONFIG_MEMCG)
9d7ea9a2 KK	424	static inline const char *vm_event_name(enum vm_event_item item)
	425	{
	426	return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
	427	NR_VM_NUMA_STAT_ITEMS +
	428	NR_VM_NODE_STAT_ITEMS +
	429	NR_VM_WRITEBACK_STAT_ITEMS +
	430	item];
	431	}
ebc5d83d	432	#endif /* CONFIG_VM_EVENT_COUNTERS \|\| CONFIG_MEMCG */
9d7ea9a2	433
2244b95a	434	#endif /* _LINUX_VMSTAT_H */