Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | #ifndef __LINUX_PERCPU_H |
2 | #define __LINUX_PERCPU_H | |
7ff6f082 | 3 | |
0a3021f4 | 4 | #include <linux/preempt.h> |
1da177e4 LT |
5 | #include <linux/slab.h> /* For kmalloc() */ |
6 | #include <linux/smp.h> | |
7ff6f082 MP |
7 | #include <linux/cpumask.h> |
8 | ||
1da177e4 LT |
9 | #include <asm/percpu.h> |
10 | ||
d3770449 | 11 | #ifndef PER_CPU_BASE_SECTION |
5280e004 | 12 | #ifdef CONFIG_SMP |
0bd74fa8 | 13 | #define PER_CPU_BASE_SECTION ".data.percpu" |
d3770449 BG |
14 | #else |
15 | #define PER_CPU_BASE_SECTION ".data" | |
16 | #endif | |
17 | #endif | |
18 | ||
19 | #ifdef CONFIG_SMP | |
5280e004 | 20 | |
44c81433 | 21 | #ifdef MODULE |
0bd74fa8 | 22 | #define PER_CPU_SHARED_ALIGNED_SECTION "" |
44c81433 | 23 | #else |
0bd74fa8 | 24 | #define PER_CPU_SHARED_ALIGNED_SECTION ".shared_aligned" |
44c81433 | 25 | #endif |
0bd74fa8 | 26 | #define PER_CPU_FIRST_SECTION ".first" |
44c81433 | 27 | |
0bd74fa8 BG |
28 | #else |
29 | ||
0bd74fa8 BG |
30 | #define PER_CPU_SHARED_ALIGNED_SECTION "" |
31 | #define PER_CPU_FIRST_SECTION "" | |
32 | ||
33 | #endif | |
63cc8c75 | 34 | |
0bd74fa8 BG |
35 | #define DEFINE_PER_CPU_SECTION(type, name, section) \ |
36 | __attribute__((__section__(PER_CPU_BASE_SECTION section))) \ | |
63cc8c75 | 37 | PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name |
0bd74fa8 | 38 | |
5280e004 | 39 | #define DEFINE_PER_CPU(type, name) \ |
0bd74fa8 | 40 | DEFINE_PER_CPU_SECTION(type, name, "") |
5280e004 | 41 | |
0bd74fa8 BG |
42 | #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ |
43 | DEFINE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \ | |
44 | ____cacheline_aligned_in_smp | |
63cc8c75 | 45 | |
0bd74fa8 BG |
46 | #define DEFINE_PER_CPU_PAGE_ALIGNED(type, name) \ |
47 | DEFINE_PER_CPU_SECTION(type, name, ".page_aligned") | |
48 | ||
49 | #define DEFINE_PER_CPU_FIRST(type, name) \ | |
50 | DEFINE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION) | |
5280e004 | 51 | |
52 | #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var) | |
53 | #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var) | |
54 | ||
1da177e4 LT |
55 | /* Enough to cover all DEFINE_PER_CPUs in kernel, including modules. */ |
56 | #ifndef PERCPU_ENOUGH_ROOM | |
b00742d3 JF |
57 | #ifdef CONFIG_MODULES |
58 | #define PERCPU_MODULE_RESERVE 8192 | |
59 | #else | |
60 | #define PERCPU_MODULE_RESERVE 0 | |
1da177e4 LT |
61 | #endif |
62 | ||
b00742d3 JF |
63 | #define PERCPU_ENOUGH_ROOM \ |
64 | (__per_cpu_end - __per_cpu_start + PERCPU_MODULE_RESERVE) | |
65 | #endif /* PERCPU_ENOUGH_ROOM */ | |
66 | ||
632bbfee JB |
67 | /* |
68 | * Must be an lvalue. Since @var must be a simple identifier, | |
69 | * we force a syntax error here if it isn't. | |
70 | */ | |
71 | #define get_cpu_var(var) (*({ \ | |
a666ecfb | 72 | extern int simple_identifier_##var(void); \ |
632bbfee JB |
73 | preempt_disable(); \ |
74 | &__get_cpu_var(var); })) | |
1da177e4 LT |
75 | #define put_cpu_var(var) preempt_enable() |
76 | ||
77 | #ifdef CONFIG_SMP | |
78 | ||
79 | struct percpu_data { | |
b3242151 | 80 | void *ptrs[1]; |
1da177e4 LT |
81 | }; |
82 | ||
7ff6f082 | 83 | #define __percpu_disguise(pdata) (struct percpu_data *)~(unsigned long)(pdata) |
1da177e4 | 84 | |
f2a8205c TH |
85 | /* |
86 | * Use this to get to a cpu's version of the per-cpu object | |
87 | * dynamically allocated. Non-atomic access to the current CPU's | |
88 | * version should probably be combined with get_cpu()/put_cpu(). | |
89 | */ | |
90 | #define per_cpu_ptr(ptr, cpu) \ | |
91 | ({ \ | |
92 | struct percpu_data *__p = __percpu_disguise(ptr); \ | |
93 | (__typeof__(ptr))__p->ptrs[(cpu)]; \ | |
94 | }) | |
95 | ||
96 | extern void *__alloc_percpu(size_t size, size_t align); | |
97 | extern void free_percpu(void *__pdata); | |
1da177e4 LT |
98 | |
99 | #else /* CONFIG_SMP */ | |
100 | ||
b36128c8 | 101 | #define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); }) |
7ff6f082 | 102 | |
f2a8205c | 103 | static inline void *__alloc_percpu(size_t size, size_t align) |
7ff6f082 | 104 | { |
f2a8205c TH |
105 | /* |
106 | * Can't easily make larger alignment work with kmalloc. WARN | |
107 | * on it. Larger alignment should only be used for module | |
108 | * percpu sections on SMP for which this path isn't used. | |
109 | */ | |
110 | WARN_ON_ONCE(align > __alignof__(unsigned long long)); | |
7ff6f082 MP |
111 | return kzalloc(size, gfp); |
112 | } | |
113 | ||
f2a8205c | 114 | static inline void free_percpu(void *p) |
7ff6f082 | 115 | { |
f2a8205c | 116 | kfree(p); |
1da177e4 LT |
117 | } |
118 | ||
119 | #endif /* CONFIG_SMP */ | |
120 | ||
313e458f RR |
121 | #define alloc_percpu(type) (type *)__alloc_percpu(sizeof(type), \ |
122 | __alignof__(type)) | |
1da177e4 LT |
123 | |
124 | #endif /* __LINUX_PERCPU_H */ |