[linux-block.git] / include / linux / page-flags-layout.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef PAGE_FLAGS_LAYOUT_H
#define PAGE_FLAGS_LAYOUT_H

#include <linux/numa.h>
#include <generated/bounds.h>

/*
 * When a memory allocation must conform to specific limitations (such
 * as being suitable for DMA) the caller will pass in hints to the
 * allocator in the gfp_mask, in the zone modifier bits.  These bits
 * are used to select a priority ordered list of memory zones which
 * match the requested limits. See gfp_zone() in include/linux/gfp.h
 */
#if MAX_NR_ZONES < 2
#define ZONES_SHIFT 0
#elif MAX_NR_ZONES <= 2
#define ZONES_SHIFT 1
#elif MAX_NR_ZONES <= 4
#define ZONES_SHIFT 2
#elif MAX_NR_ZONES <= 8
#define ZONES_SHIFT 3
#else
#error ZONES_SHIFT "Too many zones configured"
#endif

#define ZONES_WIDTH		ZONES_SHIFT

#ifdef CONFIG_SPARSEMEM
#include <asm/sparsemem.h>
#define SECTIONS_SHIFT	(MAX_PHYSMEM_BITS - SECTION_SIZE_BITS)
#else
#define SECTIONS_SHIFT	0
#endif

#ifndef BUILD_VDSO32_64
/*
 * page->flags layout:
 *
 * There are five possibilities for how page->flags get laid out.  The first
 * pair is for the normal case without sparsemem. The second pair is for
 * sparsemem when there is plenty of space for node and section information.
 * The last is when there is insufficient space in page->flags and a separate
 * lookup is necessary.
 *
 * No sparsemem or sparsemem vmemmap: |       NODE     | ZONE |             ... | FLAGS |
 *      " plus space for last_cpupid: |       NODE     | ZONE | LAST_CPUPID ... | FLAGS |
 * classic sparse with space for node:| SECTION | NODE | ZONE |             ... | FLAGS |
 *      " plus space for last_cpupid: | SECTION | NODE | ZONE | LAST_CPUPID ... | FLAGS |
 * classic sparse no space for node:  | SECTION |     ZONE    | ... | FLAGS |
 */
#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
#define SECTIONS_WIDTH		SECTIONS_SHIFT
#else
#define SECTIONS_WIDTH		0
#endif

#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_SHIFT \
	<= BITS_PER_LONG - NR_PAGEFLAGS
#define NODES_WIDTH		NODES_SHIFT
#elif defined(CONFIG_SPARSEMEM_VMEMMAP)
#error "Vmemmap: No space for nodes field in page flags"
#else
#define NODES_WIDTH		0
#endif

/*
 * Note that this #define MUST have a value so that it can be tested with
 * the IS_ENABLED() macro.
 */
#if NODES_SHIFT != 0 && NODES_WIDTH == 0
#define NODE_NOT_IN_PAGE_FLAGS	1
#endif

#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
#define KASAN_TAG_WIDTH 8
#else
#define KASAN_TAG_WIDTH 0
#endif

#ifdef CONFIG_NUMA_BALANCING
#define LAST__PID_SHIFT 8
#define LAST__PID_MASK  ((1 << LAST__PID_SHIFT)-1)

#define LAST__CPU_SHIFT NR_CPUS_BITS
#define LAST__CPU_MASK  ((1 << LAST__CPU_SHIFT)-1)

#define LAST_CPUPID_SHIFT (LAST__PID_SHIFT+LAST__CPU_SHIFT)
#else
#define LAST_CPUPID_SHIFT 0
#endif

#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \
	KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
#define LAST_CPUPID_WIDTH LAST_CPUPID_SHIFT
#else
#define LAST_CPUPID_WIDTH 0
#endif

#if LAST_CPUPID_SHIFT != 0 && LAST_CPUPID_WIDTH == 0
#define LAST_CPUPID_NOT_IN_PAGE_FLAGS
#endif

#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \
	KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
#error "Not enough bits in page flags"
#endif

/* see the comment on MAX_NR_TIERS */
#define LRU_REFS_WIDTH	min(__LRU_REFS_WIDTH, BITS_PER_LONG - NR_PAGEFLAGS - \
			    ZONES_WIDTH - LRU_GEN_WIDTH - SECTIONS_WIDTH - \
			    NODES_WIDTH - KASAN_TAG_WIDTH - LAST_CPUPID_WIDTH)

#endif
#endif /* _LINUX_PAGE_FLAGS_LAYOUT */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
bbeae5b0 PZ	2	#ifndef PAGE_FLAGS_LAYOUT_H
	3	#define PAGE_FLAGS_LAYOUT_H
	4
	5	#include <linux/numa.h>
	6	#include <generated/bounds.h>
	7
	8	/*
	9	* When a memory allocation must conform to specific limitations (such
	10	* as being suitable for DMA) the caller will pass in hints to the
	11	* allocator in the gfp_mask, in the zone modifier bits. These bits
	12	* are used to select a priority ordered list of memory zones which
	13	* match the requested limits. See gfp_zone() in include/linux/gfp.h
	14	*/
	15	#if MAX_NR_ZONES < 2
	16	#define ZONES_SHIFT 0
	17	#elif MAX_NR_ZONES <= 2
	18	#define ZONES_SHIFT 1
	19	#elif MAX_NR_ZONES <= 4
	20	#define ZONES_SHIFT 2
b11a7b94 DW	21	#elif MAX_NR_ZONES <= 8
b11a7b94 DW	22	#define ZONES_SHIFT 3
bbeae5b0	23	#else
1587db62	24	#error ZONES_SHIFT "Too many zones configured"
bbeae5b0 PZ	25	#endif
bbeae5b0 PZ	26
1587db62 YZ	27	#define ZONES_WIDTH ZONES_SHIFT
1587db62 YZ	28
bbeae5b0 PZ	29	#ifdef CONFIG_SPARSEMEM
bbeae5b0 PZ	30	#include <asm/sparsemem.h>
bbeae5b0	31	#define SECTIONS_SHIFT (MAX_PHYSMEM_BITS - SECTION_SIZE_BITS)
1587db62 YZ	32	#else
	33	#define SECTIONS_SHIFT 0
	34	#endif
bbeae5b0	35
ee38d94a	36	#ifndef BUILD_VDSO32_64
bbeae5b0 PZ	37	/*
	38	* page->flags layout:
	39	*
75980e97 PZ	40	* There are five possibilities for how page->flags get laid out. The first
	41	* pair is for the normal case without sparsemem. The second pair is for
	42	* sparsemem when there is plenty of space for node and section information.
	43	* The last is when there is insufficient space in page->flags and a separate
	44	* lookup is necessary.
bbeae5b0	45	*
b795854b	46	* No sparsemem or sparsemem vmemmap: \| NODE \| ZONE \| ... \| FLAGS \|
90572890	47	* " plus space for last_cpupid: \| NODE \| ZONE \| LAST_CPUPID ... \| FLAGS \|
b795854b	48	* classic sparse with space for node:\| SECTION \| NODE \| ZONE \| ... \| FLAGS \|
90572890	49	* " plus space for last_cpupid: \| SECTION \| NODE \| ZONE \| LAST_CPUPID ... \| FLAGS \|
bbeae5b0 PZ	50	* classic sparse no space for node: \| SECTION \| ZONE \| ... \| FLAGS \|
	51	*/
	52	#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
	53	#define SECTIONS_WIDTH SECTIONS_SHIFT
	54	#else
	55	#define SECTIONS_WIDTH 0
	56	#endif
	57
ec1c86b2 YZ	58	#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_SHIFT \
ec1c86b2 YZ	59	<= BITS_PER_LONG - NR_PAGEFLAGS
bbeae5b0	60	#define NODES_WIDTH NODES_SHIFT
1587db62	61	#elif defined(CONFIG_SPARSEMEM_VMEMMAP)
bbeae5b0	62	#error "Vmemmap: No space for nodes field in page flags"
1587db62	63	#else
bbeae5b0 PZ	64	#define NODES_WIDTH 0
	65	#endif
	66
1587db62 YZ	67	/*
	68	* Note that this #define MUST have a value so that it can be tested with
	69	* the IS_ENABLED() macro.
	70	*/
	71	#if NODES_SHIFT != 0 && NODES_WIDTH == 0
	72	#define NODE_NOT_IN_PAGE_FLAGS 1
	73	#endif
	74
	75	#if defined(CONFIG_KASAN_SW_TAGS) \|\| defined(CONFIG_KASAN_HW_TAGS)
	76	#define KASAN_TAG_WIDTH 8
	77	#else
	78	#define KASAN_TAG_WIDTH 0
	79	#endif
	80
75980e97	81	#ifdef CONFIG_NUMA_BALANCING
b795854b MG	82	#define LAST__PID_SHIFT 8
	83	#define LAST__PID_MASK ((1 << LAST__PID_SHIFT)-1)
	84
90572890 PZ	85	#define LAST__CPU_SHIFT NR_CPUS_BITS
90572890 PZ	86	#define LAST__CPU_MASK ((1 << LAST__CPU_SHIFT)-1)
b795854b	87
90572890	88	#define LAST_CPUPID_SHIFT (LAST__PID_SHIFT+LAST__CPU_SHIFT)
75980e97	89	#else
90572890	90	#define LAST_CPUPID_SHIFT 0
75980e97 PZ	91	#endif
75980e97 PZ	92
ec1c86b2 YZ	93	#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \
ec1c86b2 YZ	94	KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
90572890	95	#define LAST_CPUPID_WIDTH LAST_CPUPID_SHIFT
75980e97	96	#else
90572890	97	#define LAST_CPUPID_WIDTH 0
75980e97 PZ	98	#endif
75980e97 PZ	99
1587db62 YZ	100	#if LAST_CPUPID_SHIFT != 0 && LAST_CPUPID_WIDTH == 0
1587db62 YZ	101	#define LAST_CPUPID_NOT_IN_PAGE_FLAGS
bbeae5b0 PZ	102	#endif
bbeae5b0 PZ	103
ec1c86b2 YZ	104	#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \
ec1c86b2 YZ	105	KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
1587db62	106	#error "Not enough bits in page flags"
75980e97 PZ	107	#endif
75980e97 PZ	108
ac35a490 YZ	109	/* see the comment on MAX_NR_TIERS */
	110	#define LRU_REFS_WIDTH min(__LRU_REFS_WIDTH, BITS_PER_LONG - NR_PAGEFLAGS - \
	111	ZONES_WIDTH - LRU_GEN_WIDTH - SECTIONS_WIDTH - \
	112	NODES_WIDTH - KASAN_TAG_WIDTH - LAST_CPUPID_WIDTH)
ec1c86b2	113
ee38d94a	114	#endif
bbeae5b0	115	#endif /* _LINUX_PAGE_FLAGS_LAYOUT */