Make oslib/linux-dev-lookup.c a stand-alone library
[fio.git] / memory.c
CommitLineData
2f9ade3c
JA
1/*
2 * Memory helpers
3 */
5921e80c
JA
4#include <sys/types.h>
5#include <sys/stat.h>
6#include <fcntl.h>
2f9ade3c 7#include <unistd.h>
2f9ade3c
JA
8#include <sys/mman.h>
9
10#include "fio.h"
a5e0ee11
O
11#ifndef FIO_NO_HAVE_SHM_H
12#include <sys/shm.h>
13#endif
2f9ade3c 14
1b79a070 15void fio_unpin_memory(struct thread_data *td)
2f9ade3c 16{
1b79a070
JA
17 if (td->pinned_mem) {
18 dprint(FD_MEM, "unpinning %llu bytes\n", td->o.lockmem);
19 if (munlock(td->pinned_mem, td->o.lockmem) < 0)
2f9ade3c 20 perror("munlock");
1b79a070
JA
21 munmap(td->pinned_mem, td->o.lockmem);
22 td->pinned_mem = NULL;
2f9ade3c
JA
23 }
24}
25
1b79a070 26int fio_pin_memory(struct thread_data *td)
2f9ade3c
JA
27{
28 unsigned long long phys_mem;
29
1b79a070 30 if (!td->o.lockmem)
2f9ade3c
JA
31 return 0;
32
1b79a070 33 dprint(FD_MEM, "pinning %llu bytes\n", td->o.lockmem);
ee56ad50 34
2f9ade3c 35 /*
420b104a 36 * Don't allow mlock of more than real_mem-128MiB
2f9ade3c
JA
37 */
38 phys_mem = os_phys_mem();
39 if (phys_mem) {
1b79a070
JA
40 if ((td->o.lockmem + 128 * 1024 * 1024) > phys_mem) {
41 td->o.lockmem = phys_mem - 128 * 1024 * 1024;
420b104a 42 log_info("fio: limiting mlocked memory to %lluMiB\n",
1b79a070 43 td->o.lockmem >> 20);
2f9ade3c
JA
44 }
45 }
46
1b79a070 47 td->pinned_mem = mmap(NULL, td->o.lockmem, PROT_READ | PROT_WRITE,
a55820db 48 MAP_PRIVATE | OS_MAP_ANON, -1, 0);
1b79a070 49 if (td->pinned_mem == MAP_FAILED) {
2f9ade3c 50 perror("malloc locked mem");
1b79a070 51 td->pinned_mem = NULL;
2f9ade3c
JA
52 return 1;
53 }
1b79a070 54 if (mlock(td->pinned_mem, td->o.lockmem) < 0) {
2f9ade3c 55 perror("mlock");
1b79a070
JA
56 munmap(td->pinned_mem, td->o.lockmem);
57 td->pinned_mem = NULL;
2f9ade3c
JA
58 return 1;
59 }
60
61 return 0;
62}
63
829a602c 64static int alloc_mem_shm(struct thread_data *td, unsigned int total_mem)
2f9ade3c 65{
91e47529 66#ifndef CONFIG_NO_SHM
03e20d68 67 int flags = IPC_CREAT | S_IRUSR | S_IWUSR;
b6f9676e 68
a1242a20
JA
69 if (td->o.mem_type == MEM_SHMHUGE) {
70 unsigned long mask = td->o.hugepage_size - 1;
71
b6f9676e 72 flags |= SHM_HUGETLB;
a1242a20
JA
73 total_mem = (total_mem + mask) & ~mask;
74 }
b6f9676e 75
829a602c
JA
76 td->shm_id = shmget(IPC_PRIVATE, total_mem, flags);
77 dprint(FD_MEM, "shmget %u, %d\n", total_mem, td->shm_id);
b6f9676e
JA
78 if (td->shm_id < 0) {
79 td_verror(td, errno, "shmget");
da7d79b0 80 if (geteuid() != 0 && (errno == ENOMEM || errno == EPERM))
b6f9676e 81 log_err("fio: you may need to run this job as root\n");
886b878a 82 if (td->o.mem_type == MEM_SHMHUGE) {
5ec10eaa
JA
83 if (errno == EINVAL) {
84 log_err("fio: check that you have free huge"
85 " pages and that hugepage-size is"
86 " correct.\n");
87 } else if (errno == ENOSYS) {
88 log_err("fio: your system does not appear to"
89 " support huge pages.\n");
90 } else if (errno == ENOMEM) {
91 log_err("fio: no huge pages available, do you"
dd117eb6 92 " need to allocate some? See HOWTO.\n");
5ec10eaa 93 }
d8602dd0 94 }
5ec10eaa 95
b6f9676e
JA
96 return 1;
97 }
2f9ade3c 98
b6f9676e 99 td->orig_buffer = shmat(td->shm_id, NULL, 0);
ee56ad50 100 dprint(FD_MEM, "shmat %d, %p\n", td->shm_id, td->orig_buffer);
b6f9676e
JA
101 if (td->orig_buffer == (void *) -1) {
102 td_verror(td, errno, "shmat");
103 td->orig_buffer = NULL;
104 return 1;
105 }
106
107 return 0;
91e47529
JA
108#else
109 log_err("fio: shm not supported\n");
110 return 1;
111#endif
b6f9676e
JA
112}
113
829a602c
JA
114static void free_mem_shm(struct thread_data *td)
115{
91e47529 116#ifndef CONFIG_NO_SHM
829a602c
JA
117 struct shmid_ds sbuf;
118
119 dprint(FD_MEM, "shmdt/ctl %d %p\n", td->shm_id, td->orig_buffer);
120 shmdt(td->orig_buffer);
121 shmctl(td->shm_id, IPC_RMID, &sbuf);
91e47529 122#endif
829a602c
JA
123}
124
0f805c00 125static int alloc_mem_mmap(struct thread_data *td, size_t total_mem)
b6f9676e 126{
d9759b1e 127 int flags = 0;
b6f9676e 128
4a995dda 129 td->mmapfd = -1;
b6f9676e 130
d6dc02fb
JA
131 if (td->o.mem_type == MEM_MMAPHUGE) {
132 unsigned long mask = td->o.hugepage_size - 1;
133
d9759b1e 134 /* TODO: make sure the file is a real hugetlbfs file */
836fcc0f 135 if (!td->o.mmapfile)
d9759b1e 136 flags |= MAP_HUGETLB;
d6dc02fb
JA
137 total_mem = (total_mem + mask) & ~mask;
138 }
139
83ea422a
JA
140 if (td->o.mmapfile) {
141 td->mmapfd = open(td->o.mmapfile, O_RDWR|O_CREAT, 0644);
b6f9676e
JA
142
143 if (td->mmapfd < 0) {
144 td_verror(td, errno, "open mmap file");
2f9ade3c
JA
145 td->orig_buffer = NULL;
146 return 1;
147 }
d9759b1e 148 if (td->o.mem_type != MEM_MMAPHUGE &&
217b0f1d 149 td->o.mem_type != MEM_MMAPSHARED &&
d9759b1e 150 ftruncate(td->mmapfd, total_mem) < 0) {
b6f9676e 151 td_verror(td, errno, "truncate mmap file");
2f9ade3c
JA
152 td->orig_buffer = NULL;
153 return 1;
154 }
217b0f1d
LG
155 if (td->o.mem_type == MEM_MMAPHUGE ||
156 td->o.mem_type == MEM_MMAPSHARED)
d9759b1e
SL
157 flags |= MAP_SHARED;
158 else
159 flags |= MAP_PRIVATE;
b6f9676e 160 } else
d9759b1e 161 flags |= OS_MAP_ANON | MAP_PRIVATE;
b6f9676e 162
829a602c
JA
163 td->orig_buffer = mmap(NULL, total_mem, PROT_READ | PROT_WRITE, flags,
164 td->mmapfd, 0);
4b91ee8f
JA
165 dprint(FD_MEM, "mmap %llu/%d %p\n", (unsigned long long) total_mem,
166 td->mmapfd, td->orig_buffer);
b6f9676e
JA
167 if (td->orig_buffer == MAP_FAILED) {
168 td_verror(td, errno, "mmap");
169 td->orig_buffer = NULL;
9ce94349 170 if (td->mmapfd != 1 && td->mmapfd != -1) {
b6f9676e 171 close(td->mmapfd);
b3493a7a
JA
172 if (td->o.mmapfile)
173 unlink(td->o.mmapfile);
b6f9676e 174 }
5ec10eaa 175
b6f9676e 176 return 1;
2f9ade3c
JA
177 }
178
179 return 0;
180}
181
0f805c00 182static void free_mem_mmap(struct thread_data *td, size_t total_mem)
b6f9676e 183{
4b91ee8f
JA
184 dprint(FD_MEM, "munmap %llu %p\n", (unsigned long long) total_mem,
185 td->orig_buffer);
829a602c 186 munmap(td->orig_buffer, td->orig_buffer_size);
83ea422a 187 if (td->o.mmapfile) {
4a995dda
JA
188 if (td->mmapfd != -1)
189 close(td->mmapfd);
83ea422a
JA
190 unlink(td->o.mmapfile);
191 free(td->o.mmapfile);
829a602c
JA
192 }
193}
d87612ac 194
0f805c00 195static int alloc_mem_malloc(struct thread_data *td, size_t total_mem)
829a602c
JA
196{
197 td->orig_buffer = malloc(total_mem);
4b91ee8f
JA
198 dprint(FD_MEM, "malloc %llu %p\n", (unsigned long long) total_mem,
199 td->orig_buffer);
5ec10eaa 200
829a602c
JA
201 return td->orig_buffer == NULL;
202}
b6f9676e 203
829a602c
JA
204static void free_mem_malloc(struct thread_data *td)
205{
206 dprint(FD_MEM, "free malloc mem %p\n", td->orig_buffer);
207 free(td->orig_buffer);
b6f9676e
JA
208}
209
210/*
03e20d68 211 * Set up the buffer area we need for io.
b6f9676e
JA
212 */
213int allocate_io_mem(struct thread_data *td)
214{
0f805c00 215 size_t total_mem;
b6f9676e
JA
216 int ret = 0;
217
9b87f09b 218 if (td_ioengine_flagged(td, FIO_NOIO))
b4c5e1ac
JA
219 return 0;
220
829a602c 221 total_mem = td->orig_buffer_size;
d529ee19 222
d01612f3 223 if (td->o.odirect || td->o.mem_align || td->o.oatomic ||
9b87f09b 224 td_ioengine_flagged(td, FIO_MEMALIGN)) {
829a602c 225 total_mem += page_mask;
d529ee19
JA
226 if (td->o.mem_align && td->o.mem_align > page_size)
227 total_mem += td->o.mem_align - page_size;
228 }
829a602c 229
4b91ee8f 230 dprint(FD_MEM, "Alloc %llu for buffers\n", (unsigned long long) total_mem);
0f805c00 231
a6cb85e2
JA
232 /*
233 * If the IO engine has hooks to allocate/free memory, use those. But
234 * error out if the user explicitly asked for something else.
235 */
236 if (td->io_ops->iomem_alloc) {
237 if (fio_option_is_set(&td->o, mem_type)) {
238 log_err("fio: option 'mem/iomem' conflicts with specified IO engine\n");
239 ret = 1;
240 } else
241 ret = td->io_ops->iomem_alloc(td, total_mem);
242 } else if (td->o.mem_type == MEM_MALLOC)
829a602c 243 ret = alloc_mem_malloc(td, total_mem);
b6f9676e 244 else if (td->o.mem_type == MEM_SHM || td->o.mem_type == MEM_SHMHUGE)
829a602c 245 ret = alloc_mem_shm(td, total_mem);
217b0f1d
LG
246 else if (td->o.mem_type == MEM_MMAP || td->o.mem_type == MEM_MMAPHUGE ||
247 td->o.mem_type == MEM_MMAPSHARED)
829a602c 248 ret = alloc_mem_mmap(td, total_mem);
b6f9676e
JA
249 else {
250 log_err("fio: bad mem type: %d\n", td->o.mem_type);
251 ret = 1;
252 }
253
3deb3101
JA
254 if (ret)
255 td_verror(td, ENOMEM, "iomem allocation");
256
b6f9676e
JA
257 return ret;
258}
259
2f9ade3c
JA
260void free_io_mem(struct thread_data *td)
261{
829a602c
JA
262 unsigned int total_mem;
263
264 total_mem = td->orig_buffer_size;
d01612f3 265 if (td->o.odirect || td->o.oatomic)
829a602c
JA
266 total_mem += page_mask;
267
a6cb85e2
JA
268 if (td->io_ops->iomem_alloc) {
269 if (td->io_ops->iomem_free)
270 td->io_ops->iomem_free(td);
271 } else if (td->o.mem_type == MEM_MALLOC)
829a602c
JA
272 free_mem_malloc(td);
273 else if (td->o.mem_type == MEM_SHM || td->o.mem_type == MEM_SHMHUGE)
274 free_mem_shm(td);
217b0f1d
LG
275 else if (td->o.mem_type == MEM_MMAP || td->o.mem_type == MEM_MMAPHUGE ||
276 td->o.mem_type == MEM_MMAPSHARED)
829a602c
JA
277 free_mem_mmap(td, total_mem);
278 else
2dc1bbeb 279 log_err("Bad memory type %u\n", td->o.mem_type);
2f9ade3c
JA
280
281 td->orig_buffer = NULL;
829a602c 282 td->orig_buffer_size = 0;
2f9ade3c 283}