genfio: Splitting gen_template in 2 parts
[fio.git] / memory.c
CommitLineData
2f9ade3c
JA
1/*
2 * Memory helpers
3 */
5921e80c
JA
4#include <sys/types.h>
5#include <sys/stat.h>
6#include <fcntl.h>
2f9ade3c 7#include <unistd.h>
2f9ade3c
JA
8#include <sys/mman.h>
9
10#include "fio.h"
a5e0ee11
O
11#ifndef FIO_NO_HAVE_SHM_H
12#include <sys/shm.h>
13#endif
2f9ade3c 14
1b79a070 15void fio_unpin_memory(struct thread_data *td)
2f9ade3c 16{
1b79a070
JA
17 if (td->pinned_mem) {
18 dprint(FD_MEM, "unpinning %llu bytes\n", td->o.lockmem);
19 if (munlock(td->pinned_mem, td->o.lockmem) < 0)
2f9ade3c 20 perror("munlock");
1b79a070
JA
21 munmap(td->pinned_mem, td->o.lockmem);
22 td->pinned_mem = NULL;
2f9ade3c
JA
23 }
24}
25
1b79a070 26int fio_pin_memory(struct thread_data *td)
2f9ade3c
JA
27{
28 unsigned long long phys_mem;
29
1b79a070 30 if (!td->o.lockmem)
2f9ade3c
JA
31 return 0;
32
1b79a070 33 dprint(FD_MEM, "pinning %llu bytes\n", td->o.lockmem);
ee56ad50 34
2f9ade3c
JA
35 /*
36 * Don't allow mlock of more than real_mem-128MB
37 */
38 phys_mem = os_phys_mem();
39 if (phys_mem) {
1b79a070
JA
40 if ((td->o.lockmem + 128 * 1024 * 1024) > phys_mem) {
41 td->o.lockmem = phys_mem - 128 * 1024 * 1024;
b22989b9 42 log_info("fio: limiting mlocked memory to %lluMB\n",
1b79a070 43 td->o.lockmem >> 20);
2f9ade3c
JA
44 }
45 }
46
1b79a070 47 td->pinned_mem = mmap(NULL, td->o.lockmem, PROT_READ | PROT_WRITE,
a55820db 48 MAP_PRIVATE | OS_MAP_ANON, -1, 0);
1b79a070 49 if (td->pinned_mem == MAP_FAILED) {
2f9ade3c 50 perror("malloc locked mem");
1b79a070 51 td->pinned_mem = NULL;
2f9ade3c
JA
52 return 1;
53 }
1b79a070 54 if (mlock(td->pinned_mem, td->o.lockmem) < 0) {
2f9ade3c 55 perror("mlock");
1b79a070
JA
56 munmap(td->pinned_mem, td->o.lockmem);
57 td->pinned_mem = NULL;
2f9ade3c
JA
58 return 1;
59 }
60
61 return 0;
62}
63
829a602c 64static int alloc_mem_shm(struct thread_data *td, unsigned int total_mem)
2f9ade3c 65{
03e20d68 66 int flags = IPC_CREAT | S_IRUSR | S_IWUSR;
b6f9676e 67
a1242a20
JA
68 if (td->o.mem_type == MEM_SHMHUGE) {
69 unsigned long mask = td->o.hugepage_size - 1;
70
b6f9676e 71 flags |= SHM_HUGETLB;
a1242a20
JA
72 total_mem = (total_mem + mask) & ~mask;
73 }
b6f9676e 74
829a602c
JA
75 td->shm_id = shmget(IPC_PRIVATE, total_mem, flags);
76 dprint(FD_MEM, "shmget %u, %d\n", total_mem, td->shm_id);
b6f9676e
JA
77 if (td->shm_id < 0) {
78 td_verror(td, errno, "shmget");
da7d79b0 79 if (geteuid() != 0 && (errno == ENOMEM || errno == EPERM))
b6f9676e 80 log_err("fio: you may need to run this job as root\n");
886b878a 81 if (td->o.mem_type == MEM_SHMHUGE) {
5ec10eaa
JA
82 if (errno == EINVAL) {
83 log_err("fio: check that you have free huge"
84 " pages and that hugepage-size is"
85 " correct.\n");
86 } else if (errno == ENOSYS) {
87 log_err("fio: your system does not appear to"
88 " support huge pages.\n");
89 } else if (errno == ENOMEM) {
90 log_err("fio: no huge pages available, do you"
91 " need to alocate some? See HOWTO.\n");
92 }
d8602dd0 93 }
5ec10eaa 94
b6f9676e
JA
95 return 1;
96 }
2f9ade3c 97
b6f9676e 98 td->orig_buffer = shmat(td->shm_id, NULL, 0);
ee56ad50 99 dprint(FD_MEM, "shmat %d, %p\n", td->shm_id, td->orig_buffer);
b6f9676e
JA
100 if (td->orig_buffer == (void *) -1) {
101 td_verror(td, errno, "shmat");
102 td->orig_buffer = NULL;
103 return 1;
104 }
105
106 return 0;
107}
108
829a602c
JA
109static void free_mem_shm(struct thread_data *td)
110{
111 struct shmid_ds sbuf;
112
113 dprint(FD_MEM, "shmdt/ctl %d %p\n", td->shm_id, td->orig_buffer);
114 shmdt(td->orig_buffer);
115 shmctl(td->shm_id, IPC_RMID, &sbuf);
116}
117
0f805c00 118static int alloc_mem_mmap(struct thread_data *td, size_t total_mem)
b6f9676e 119{
d9759b1e 120 int flags = 0;
b6f9676e 121
a55820db 122 td->mmapfd = 1;
b6f9676e 123
d6dc02fb
JA
124 if (td->o.mem_type == MEM_MMAPHUGE) {
125 unsigned long mask = td->o.hugepage_size - 1;
126
d9759b1e 127 /* TODO: make sure the file is a real hugetlbfs file */
836fcc0f 128 if (!td->o.mmapfile)
d9759b1e 129 flags |= MAP_HUGETLB;
d6dc02fb
JA
130 total_mem = (total_mem + mask) & ~mask;
131 }
132
83ea422a
JA
133 if (td->o.mmapfile) {
134 td->mmapfd = open(td->o.mmapfile, O_RDWR|O_CREAT, 0644);
b6f9676e
JA
135
136 if (td->mmapfd < 0) {
137 td_verror(td, errno, "open mmap file");
2f9ade3c
JA
138 td->orig_buffer = NULL;
139 return 1;
140 }
d9759b1e
SL
141 if (td->o.mem_type != MEM_MMAPHUGE &&
142 ftruncate(td->mmapfd, total_mem) < 0) {
b6f9676e 143 td_verror(td, errno, "truncate mmap file");
2f9ade3c
JA
144 td->orig_buffer = NULL;
145 return 1;
146 }
d9759b1e
SL
147 if (td->o.mem_type == MEM_MMAPHUGE)
148 flags |= MAP_SHARED;
149 else
150 flags |= MAP_PRIVATE;
b6f9676e 151 } else
d9759b1e 152 flags |= OS_MAP_ANON | MAP_PRIVATE;
b6f9676e 153
829a602c
JA
154 td->orig_buffer = mmap(NULL, total_mem, PROT_READ | PROT_WRITE, flags,
155 td->mmapfd, 0);
4b91ee8f
JA
156 dprint(FD_MEM, "mmap %llu/%d %p\n", (unsigned long long) total_mem,
157 td->mmapfd, td->orig_buffer);
b6f9676e
JA
158 if (td->orig_buffer == MAP_FAILED) {
159 td_verror(td, errno, "mmap");
160 td->orig_buffer = NULL;
161 if (td->mmapfd) {
162 close(td->mmapfd);
83ea422a 163 unlink(td->o.mmapfile);
b6f9676e 164 }
5ec10eaa 165
b6f9676e 166 return 1;
2f9ade3c
JA
167 }
168
169 return 0;
170}
171
0f805c00 172static void free_mem_mmap(struct thread_data *td, size_t total_mem)
b6f9676e 173{
4b91ee8f
JA
174 dprint(FD_MEM, "munmap %llu %p\n", (unsigned long long) total_mem,
175 td->orig_buffer);
829a602c 176 munmap(td->orig_buffer, td->orig_buffer_size);
83ea422a 177 if (td->o.mmapfile) {
829a602c 178 close(td->mmapfd);
83ea422a
JA
179 unlink(td->o.mmapfile);
180 free(td->o.mmapfile);
829a602c
JA
181 }
182}
d87612ac 183
0f805c00 184static int alloc_mem_malloc(struct thread_data *td, size_t total_mem)
829a602c
JA
185{
186 td->orig_buffer = malloc(total_mem);
4b91ee8f
JA
187 dprint(FD_MEM, "malloc %llu %p\n", (unsigned long long) total_mem,
188 td->orig_buffer);
5ec10eaa 189
829a602c
JA
190 return td->orig_buffer == NULL;
191}
b6f9676e 192
829a602c
JA
193static void free_mem_malloc(struct thread_data *td)
194{
195 dprint(FD_MEM, "free malloc mem %p\n", td->orig_buffer);
196 free(td->orig_buffer);
b6f9676e
JA
197}
198
199/*
03e20d68 200 * Set up the buffer area we need for io.
b6f9676e
JA
201 */
202int allocate_io_mem(struct thread_data *td)
203{
0f805c00 204 size_t total_mem;
b6f9676e
JA
205 int ret = 0;
206
b4c5e1ac
JA
207 if (td->io_ops->flags & FIO_NOIO)
208 return 0;
209
829a602c 210 total_mem = td->orig_buffer_size;
d529ee19 211
ca7e0ddb
JA
212 if (td->o.odirect || td->o.mem_align ||
213 (td->io_ops->flags & FIO_MEMALIGN)) {
829a602c 214 total_mem += page_mask;
d529ee19
JA
215 if (td->o.mem_align && td->o.mem_align > page_size)
216 total_mem += td->o.mem_align - page_size;
217 }
829a602c 218
4b91ee8f 219 dprint(FD_MEM, "Alloc %llu for buffers\n", (unsigned long long) total_mem);
0f805c00 220
b6f9676e 221 if (td->o.mem_type == MEM_MALLOC)
829a602c 222 ret = alloc_mem_malloc(td, total_mem);
b6f9676e 223 else if (td->o.mem_type == MEM_SHM || td->o.mem_type == MEM_SHMHUGE)
829a602c 224 ret = alloc_mem_shm(td, total_mem);
b6f9676e 225 else if (td->o.mem_type == MEM_MMAP || td->o.mem_type == MEM_MMAPHUGE)
829a602c 226 ret = alloc_mem_mmap(td, total_mem);
b6f9676e
JA
227 else {
228 log_err("fio: bad mem type: %d\n", td->o.mem_type);
229 ret = 1;
230 }
231
3deb3101
JA
232 if (ret)
233 td_verror(td, ENOMEM, "iomem allocation");
234
b6f9676e
JA
235 return ret;
236}
237
2f9ade3c
JA
238void free_io_mem(struct thread_data *td)
239{
829a602c
JA
240 unsigned int total_mem;
241
242 total_mem = td->orig_buffer_size;
243 if (td->o.odirect)
244 total_mem += page_mask;
245
246 if (td->o.mem_type == MEM_MALLOC)
247 free_mem_malloc(td);
248 else if (td->o.mem_type == MEM_SHM || td->o.mem_type == MEM_SHMHUGE)
249 free_mem_shm(td);
250 else if (td->o.mem_type == MEM_MMAP || td->o.mem_type == MEM_MMAPHUGE)
251 free_mem_mmap(td, total_mem);
252 else
2dc1bbeb 253 log_err("Bad memory type %u\n", td->o.mem_type);
2f9ade3c
JA
254
255 td->orig_buffer = NULL;
829a602c 256 td->orig_buffer_size = 0;
2f9ade3c 257}