[linux-block.git] / include / linux / page_counter.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_PAGE_COUNTER_H
#define _LINUX_PAGE_COUNTER_H

#include <linux/atomic.h>
#include <linux/cache.h>
#include <linux/kernel.h>
#include <asm/page.h>

struct page_counter {
	/*
	 * Make sure 'usage' does not share cacheline with any other field. The
	 * memcg->memory.usage is a hot member of struct mem_cgroup.
	 */
	atomic_long_t usage;
	CACHELINE_PADDING(_pad1_);

	/* effective memory.min and memory.min usage tracking */
	unsigned long emin;
	atomic_long_t min_usage;
	atomic_long_t children_min_usage;

	/* effective memory.low and memory.low usage tracking */
	unsigned long elow;
	atomic_long_t low_usage;
	atomic_long_t children_low_usage;

	unsigned long watermark;
	unsigned long failcnt;

	/* Keep all the read most fields in a separete cacheline. */
	CACHELINE_PADDING(_pad2_);

	unsigned long min;
	unsigned long low;
	unsigned long high;
	unsigned long max;
	struct page_counter *parent;
} ____cacheline_internodealigned_in_smp;

#if BITS_PER_LONG == 32
#define PAGE_COUNTER_MAX LONG_MAX
#else
#define PAGE_COUNTER_MAX (LONG_MAX / PAGE_SIZE)
#endif

static inline void page_counter_init(struct page_counter *counter,
				     struct page_counter *parent)
{
	atomic_long_set(&counter->usage, 0);
	counter->max = PAGE_COUNTER_MAX;
	counter->parent = parent;
}

static inline unsigned long page_counter_read(struct page_counter *counter)
{
	return atomic_long_read(&counter->usage);
}

void page_counter_cancel(struct page_counter *counter, unsigned long nr_pages);
void page_counter_charge(struct page_counter *counter, unsigned long nr_pages);
bool page_counter_try_charge(struct page_counter *counter,
			     unsigned long nr_pages,
			     struct page_counter **fail);
void page_counter_uncharge(struct page_counter *counter, unsigned long nr_pages);
void page_counter_set_min(struct page_counter *counter, unsigned long nr_pages);
void page_counter_set_low(struct page_counter *counter, unsigned long nr_pages);

static inline void page_counter_set_high(struct page_counter *counter,
					 unsigned long nr_pages)
{
	WRITE_ONCE(counter->high, nr_pages);
}

int page_counter_set_max(struct page_counter *counter, unsigned long nr_pages);
int page_counter_memparse(const char *buf, const char *max,
			  unsigned long *nr_pages);

static inline void page_counter_reset_watermark(struct page_counter *counter)
{
	counter->watermark = page_counter_read(counter);
}

#endif /* _LINUX_PAGE_COUNTER_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
3e32cb2e JW	2	#ifndef _LINUX_PAGE_COUNTER_H
	3	#define _LINUX_PAGE_COUNTER_H
	4
	5	#include <linux/atomic.h>
408587ba	6	#include <linux/cache.h>
3e32cb2e JW	7	#include <linux/kernel.h>
	8	#include <asm/page.h>
	9
	10	struct page_counter {
408587ba SB	11	/*
	12	* Make sure 'usage' does not share cacheline with any other field. The
	13	* memcg->memory.usage is a hot member of struct mem_cgroup.
	14	*/
bbec2e15	15	atomic_long_t usage;
e6ad640b	16	CACHELINE_PADDING(_pad1_);
3e32cb2e	17
bf8d5d52 RG	18	/* effective memory.min and memory.min usage tracking */
	19	unsigned long emin;
	20	atomic_long_t min_usage;
	21	atomic_long_t children_min_usage;
	22
23067153 RG	23	/* effective memory.low and memory.low usage tracking */
	24	unsigned long elow;
	25	atomic_long_t low_usage;
	26	atomic_long_t children_low_usage;
	27
3e32cb2e JW	28	unsigned long watermark;
3e32cb2e JW	29	unsigned long failcnt;
802f1d52	30
408587ba	31	/* Keep all the read most fields in a separete cacheline. */
e6ad640b	32	CACHELINE_PADDING(_pad2_);
408587ba SB	33
	34	unsigned long min;
	35	unsigned long low;
	36	unsigned long high;
	37	unsigned long max;
802f1d52	38	struct page_counter *parent;
408587ba	39	} ____cacheline_internodealigned_in_smp;
3e32cb2e JW	40
	41	#if BITS_PER_LONG == 32
	42	#define PAGE_COUNTER_MAX LONG_MAX
	43	#else
	44	#define PAGE_COUNTER_MAX (LONG_MAX / PAGE_SIZE)
	45	#endif
	46
	47	static inline void page_counter_init(struct page_counter *counter,
	48	struct page_counter *parent)
	49	{
bbec2e15 RG	50	atomic_long_set(&counter->usage, 0);
bbec2e15 RG	51	counter->max = PAGE_COUNTER_MAX;
3e32cb2e JW	52	counter->parent = parent;
	53	}
	54
	55	static inline unsigned long page_counter_read(struct page_counter *counter)
	56	{
bbec2e15	57	return atomic_long_read(&counter->usage);
3e32cb2e JW	58	}
3e32cb2e JW	59
64f21993	60	void page_counter_cancel(struct page_counter *counter, unsigned long nr_pages);
3e32cb2e	61	void page_counter_charge(struct page_counter *counter, unsigned long nr_pages);
6071ca52 JW	62	bool page_counter_try_charge(struct page_counter *counter,
	63	unsigned long nr_pages,
	64	struct page_counter **fail);
64f21993	65	void page_counter_uncharge(struct page_counter *counter, unsigned long nr_pages);
bf8d5d52	66	void page_counter_set_min(struct page_counter *counter, unsigned long nr_pages);
23067153	67	void page_counter_set_low(struct page_counter *counter, unsigned long nr_pages);
d1663a90 JK	68
	69	static inline void page_counter_set_high(struct page_counter *counter,
	70	unsigned long nr_pages)
	71	{
	72	WRITE_ONCE(counter->high, nr_pages);
	73	}
	74
bf8d5d52	75	int page_counter_set_max(struct page_counter *counter, unsigned long nr_pages);
650c5e56 JW	76	int page_counter_memparse(const char buf, const char max,
650c5e56 JW	77	unsigned long *nr_pages);
3e32cb2e JW	78
	79	static inline void page_counter_reset_watermark(struct page_counter *counter)
	80	{
	81	counter->watermark = page_counter_read(counter);
	82	}
	83
	84	#endif /* _LINUX_PAGE_COUNTER_H */