document changes to --client syntax and behavior
[fio.git] / memory.c
CommitLineData
2f9ade3c
JA
1/*
2 * Memory helpers
3 */
5921e80c
JA
4#include <sys/types.h>
5#include <sys/stat.h>
6#include <fcntl.h>
2f9ade3c 7#include <unistd.h>
2f9ade3c
JA
8#include <sys/mman.h>
9
10#include "fio.h"
a5e0ee11
O
11#ifndef FIO_NO_HAVE_SHM_H
12#include <sys/shm.h>
13#endif
2f9ade3c 14
1b79a070 15void fio_unpin_memory(struct thread_data *td)
2f9ade3c 16{
1b79a070
JA
17 if (td->pinned_mem) {
18 dprint(FD_MEM, "unpinning %llu bytes\n", td->o.lockmem);
19 if (munlock(td->pinned_mem, td->o.lockmem) < 0)
2f9ade3c 20 perror("munlock");
1b79a070
JA
21 munmap(td->pinned_mem, td->o.lockmem);
22 td->pinned_mem = NULL;
2f9ade3c
JA
23 }
24}
25
1b79a070 26int fio_pin_memory(struct thread_data *td)
2f9ade3c
JA
27{
28 unsigned long long phys_mem;
29
1b79a070 30 if (!td->o.lockmem)
2f9ade3c
JA
31 return 0;
32
1b79a070 33 dprint(FD_MEM, "pinning %llu bytes\n", td->o.lockmem);
ee56ad50 34
2f9ade3c
JA
35 /*
36 * Don't allow mlock of more than real_mem-128MB
37 */
38 phys_mem = os_phys_mem();
39 if (phys_mem) {
1b79a070
JA
40 if ((td->o.lockmem + 128 * 1024 * 1024) > phys_mem) {
41 td->o.lockmem = phys_mem - 128 * 1024 * 1024;
b22989b9 42 log_info("fio: limiting mlocked memory to %lluMB\n",
1b79a070 43 td->o.lockmem >> 20);
2f9ade3c
JA
44 }
45 }
46
1b79a070 47 td->pinned_mem = mmap(NULL, td->o.lockmem, PROT_READ | PROT_WRITE,
a55820db 48 MAP_PRIVATE | OS_MAP_ANON, -1, 0);
1b79a070 49 if (td->pinned_mem == MAP_FAILED) {
2f9ade3c 50 perror("malloc locked mem");
1b79a070 51 td->pinned_mem = NULL;
2f9ade3c
JA
52 return 1;
53 }
1b79a070 54 if (mlock(td->pinned_mem, td->o.lockmem) < 0) {
2f9ade3c 55 perror("mlock");
1b79a070
JA
56 munmap(td->pinned_mem, td->o.lockmem);
57 td->pinned_mem = NULL;
2f9ade3c
JA
58 return 1;
59 }
60
61 return 0;
62}
63
829a602c 64static int alloc_mem_shm(struct thread_data *td, unsigned int total_mem)
2f9ade3c 65{
91e47529 66#ifndef CONFIG_NO_SHM
03e20d68 67 int flags = IPC_CREAT | S_IRUSR | S_IWUSR;
b6f9676e 68
a1242a20
JA
69 if (td->o.mem_type == MEM_SHMHUGE) {
70 unsigned long mask = td->o.hugepage_size - 1;
71
b6f9676e 72 flags |= SHM_HUGETLB;
a1242a20
JA
73 total_mem = (total_mem + mask) & ~mask;
74 }
b6f9676e 75
829a602c
JA
76 td->shm_id = shmget(IPC_PRIVATE, total_mem, flags);
77 dprint(FD_MEM, "shmget %u, %d\n", total_mem, td->shm_id);
b6f9676e
JA
78 if (td->shm_id < 0) {
79 td_verror(td, errno, "shmget");
da7d79b0 80 if (geteuid() != 0 && (errno == ENOMEM || errno == EPERM))
b6f9676e 81 log_err("fio: you may need to run this job as root\n");
886b878a 82 if (td->o.mem_type == MEM_SHMHUGE) {
5ec10eaa
JA
83 if (errno == EINVAL) {
84 log_err("fio: check that you have free huge"
85 " pages and that hugepage-size is"
86 " correct.\n");
87 } else if (errno == ENOSYS) {
88 log_err("fio: your system does not appear to"
89 " support huge pages.\n");
90 } else if (errno == ENOMEM) {
91 log_err("fio: no huge pages available, do you"
92 " need to alocate some? See HOWTO.\n");
93 }
d8602dd0 94 }
5ec10eaa 95
b6f9676e
JA
96 return 1;
97 }
2f9ade3c 98
b6f9676e 99 td->orig_buffer = shmat(td->shm_id, NULL, 0);
ee56ad50 100 dprint(FD_MEM, "shmat %d, %p\n", td->shm_id, td->orig_buffer);
b6f9676e
JA
101 if (td->orig_buffer == (void *) -1) {
102 td_verror(td, errno, "shmat");
103 td->orig_buffer = NULL;
104 return 1;
105 }
106
107 return 0;
91e47529
JA
108#else
109 log_err("fio: shm not supported\n");
110 return 1;
111#endif
b6f9676e
JA
112}
113
829a602c
JA
114static void free_mem_shm(struct thread_data *td)
115{
91e47529 116#ifndef CONFIG_NO_SHM
829a602c
JA
117 struct shmid_ds sbuf;
118
119 dprint(FD_MEM, "shmdt/ctl %d %p\n", td->shm_id, td->orig_buffer);
120 shmdt(td->orig_buffer);
121 shmctl(td->shm_id, IPC_RMID, &sbuf);
91e47529 122#endif
829a602c
JA
123}
124
0f805c00 125static int alloc_mem_mmap(struct thread_data *td, size_t total_mem)
b6f9676e 126{
d9759b1e 127 int flags = 0;
b6f9676e 128
4a995dda 129 td->mmapfd = -1;
b6f9676e 130
d6dc02fb
JA
131 if (td->o.mem_type == MEM_MMAPHUGE) {
132 unsigned long mask = td->o.hugepage_size - 1;
133
d9759b1e 134 /* TODO: make sure the file is a real hugetlbfs file */
836fcc0f 135 if (!td->o.mmapfile)
d9759b1e 136 flags |= MAP_HUGETLB;
d6dc02fb
JA
137 total_mem = (total_mem + mask) & ~mask;
138 }
139
83ea422a
JA
140 if (td->o.mmapfile) {
141 td->mmapfd = open(td->o.mmapfile, O_RDWR|O_CREAT, 0644);
b6f9676e
JA
142
143 if (td->mmapfd < 0) {
144 td_verror(td, errno, "open mmap file");
2f9ade3c
JA
145 td->orig_buffer = NULL;
146 return 1;
147 }
d9759b1e
SL
148 if (td->o.mem_type != MEM_MMAPHUGE &&
149 ftruncate(td->mmapfd, total_mem) < 0) {
b6f9676e 150 td_verror(td, errno, "truncate mmap file");
2f9ade3c
JA
151 td->orig_buffer = NULL;
152 return 1;
153 }
d9759b1e
SL
154 if (td->o.mem_type == MEM_MMAPHUGE)
155 flags |= MAP_SHARED;
156 else
157 flags |= MAP_PRIVATE;
b6f9676e 158 } else
d9759b1e 159 flags |= OS_MAP_ANON | MAP_PRIVATE;
b6f9676e 160
829a602c
JA
161 td->orig_buffer = mmap(NULL, total_mem, PROT_READ | PROT_WRITE, flags,
162 td->mmapfd, 0);
4b91ee8f
JA
163 dprint(FD_MEM, "mmap %llu/%d %p\n", (unsigned long long) total_mem,
164 td->mmapfd, td->orig_buffer);
b6f9676e
JA
165 if (td->orig_buffer == MAP_FAILED) {
166 td_verror(td, errno, "mmap");
167 td->orig_buffer = NULL;
9ce94349 168 if (td->mmapfd != 1 && td->mmapfd != -1) {
b6f9676e 169 close(td->mmapfd);
b3493a7a
JA
170 if (td->o.mmapfile)
171 unlink(td->o.mmapfile);
b6f9676e 172 }
5ec10eaa 173
b6f9676e 174 return 1;
2f9ade3c
JA
175 }
176
177 return 0;
178}
179
0f805c00 180static void free_mem_mmap(struct thread_data *td, size_t total_mem)
b6f9676e 181{
4b91ee8f
JA
182 dprint(FD_MEM, "munmap %llu %p\n", (unsigned long long) total_mem,
183 td->orig_buffer);
829a602c 184 munmap(td->orig_buffer, td->orig_buffer_size);
83ea422a 185 if (td->o.mmapfile) {
4a995dda
JA
186 if (td->mmapfd != -1)
187 close(td->mmapfd);
83ea422a
JA
188 unlink(td->o.mmapfile);
189 free(td->o.mmapfile);
829a602c
JA
190 }
191}
d87612ac 192
0f805c00 193static int alloc_mem_malloc(struct thread_data *td, size_t total_mem)
829a602c
JA
194{
195 td->orig_buffer = malloc(total_mem);
4b91ee8f
JA
196 dprint(FD_MEM, "malloc %llu %p\n", (unsigned long long) total_mem,
197 td->orig_buffer);
5ec10eaa 198
829a602c
JA
199 return td->orig_buffer == NULL;
200}
b6f9676e 201
829a602c
JA
202static void free_mem_malloc(struct thread_data *td)
203{
204 dprint(FD_MEM, "free malloc mem %p\n", td->orig_buffer);
205 free(td->orig_buffer);
b6f9676e
JA
206}
207
208/*
03e20d68 209 * Set up the buffer area we need for io.
b6f9676e
JA
210 */
211int allocate_io_mem(struct thread_data *td)
212{
0f805c00 213 size_t total_mem;
b6f9676e
JA
214 int ret = 0;
215
b4c5e1ac
JA
216 if (td->io_ops->flags & FIO_NOIO)
217 return 0;
218
829a602c 219 total_mem = td->orig_buffer_size;
d529ee19 220
d01612f3 221 if (td->o.odirect || td->o.mem_align || td->o.oatomic ||
ca7e0ddb 222 (td->io_ops->flags & FIO_MEMALIGN)) {
829a602c 223 total_mem += page_mask;
d529ee19
JA
224 if (td->o.mem_align && td->o.mem_align > page_size)
225 total_mem += td->o.mem_align - page_size;
226 }
829a602c 227
4b91ee8f 228 dprint(FD_MEM, "Alloc %llu for buffers\n", (unsigned long long) total_mem);
0f805c00 229
b6f9676e 230 if (td->o.mem_type == MEM_MALLOC)
829a602c 231 ret = alloc_mem_malloc(td, total_mem);
b6f9676e 232 else if (td->o.mem_type == MEM_SHM || td->o.mem_type == MEM_SHMHUGE)
829a602c 233 ret = alloc_mem_shm(td, total_mem);
b6f9676e 234 else if (td->o.mem_type == MEM_MMAP || td->o.mem_type == MEM_MMAPHUGE)
829a602c 235 ret = alloc_mem_mmap(td, total_mem);
b6f9676e
JA
236 else {
237 log_err("fio: bad mem type: %d\n", td->o.mem_type);
238 ret = 1;
239 }
240
3deb3101
JA
241 if (ret)
242 td_verror(td, ENOMEM, "iomem allocation");
243
b6f9676e
JA
244 return ret;
245}
246
2f9ade3c
JA
247void free_io_mem(struct thread_data *td)
248{
829a602c
JA
249 unsigned int total_mem;
250
251 total_mem = td->orig_buffer_size;
d01612f3 252 if (td->o.odirect || td->o.oatomic)
829a602c
JA
253 total_mem += page_mask;
254
255 if (td->o.mem_type == MEM_MALLOC)
256 free_mem_malloc(td);
257 else if (td->o.mem_type == MEM_SHM || td->o.mem_type == MEM_SHMHUGE)
258 free_mem_shm(td);
259 else if (td->o.mem_type == MEM_MMAP || td->o.mem_type == MEM_MMAPHUGE)
260 free_mem_mmap(td, total_mem);
261 else
2dc1bbeb 262 log_err("Bad memory type %u\n", td->o.mem_type);
2f9ade3c
JA
263
264 td->orig_buffer = NULL;
829a602c 265 td->orig_buffer_size = 0;
2f9ade3c 266}