projects
/
fio.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Add regression test for recent offload locking bug
[fio.git]
/
io_u.c
diff --git
a/io_u.c
b/io_u.c
index 9e9dad5a79f15c6f405b26ad16736388149d257c..a3540d14714ef45f59f7fcf22744bae507f6b491 100644
(file)
--- a/
io_u.c
+++ b/
io_u.c
@@
-10,6
+10,7
@@
#include "err.h"
#include "lib/pow2.h"
#include "minmax.h"
#include "err.h"
#include "lib/pow2.h"
#include "minmax.h"
+#include "zbd.h"
struct io_completion_data {
int nr; /* input */
struct io_completion_data {
int nr; /* input */
@@
-31,21
+32,27
@@
static bool random_map_free(struct fio_file *f, const uint64_t block)
/*
* Mark a given offset as used in the map.
*/
/*
* Mark a given offset as used in the map.
*/
-static void mark_random_map(struct thread_data *td, struct io_u *io_u)
+static uint64_t mark_random_map(struct thread_data *td, struct io_u *io_u,
+ uint64_t offset, uint64_t buflen)
{
{
- unsigned
int
min_bs = td->o.min_bs[io_u->ddir];
+ unsigned
long long
min_bs = td->o.min_bs[io_u->ddir];
struct fio_file *f = io_u->file;
struct fio_file *f = io_u->file;
- unsigned
int
nr_blocks;
+ unsigned
long long
nr_blocks;
uint64_t block;
uint64_t block;
- block = (io_u->offset - f->file_offset) / (uint64_t) min_bs;
- nr_blocks = (io_u->buflen + min_bs - 1) / min_bs;
+ block = (offset - f->file_offset) / (uint64_t) min_bs;
+ nr_blocks = (buflen + min_bs - 1) / min_bs;
+ assert(nr_blocks > 0);
- if (!(io_u->flags & IO_U_F_BUSY_OK))
+ if (!(io_u->flags & IO_U_F_BUSY_OK))
{
nr_blocks = axmap_set_nr(f->io_axmap, block, nr_blocks);
nr_blocks = axmap_set_nr(f->io_axmap, block, nr_blocks);
+ assert(nr_blocks > 0);
+ }
+
+ if ((nr_blocks * min_bs) < buflen)
+ buflen = nr_blocks * min_bs;
- if ((nr_blocks * min_bs) < io_u->buflen)
- io_u->buflen = nr_blocks * min_bs;
+ return buflen;
}
static uint64_t last_block(struct thread_data *td, struct fio_file *f,
}
static uint64_t last_block(struct thread_data *td, struct fio_file *f,
@@
-64,7
+71,7
@@
static uint64_t last_block(struct thread_data *td, struct fio_file *f,
if (max_size > f->real_file_size)
max_size = f->real_file_size;
if (max_size > f->real_file_size)
max_size = f->real_file_size;
- if (td->o.zone_range)
+ if (td->o.zone_
mode == ZONE_MODE_STRIDED && td->o.zone_
range)
max_size = td->o.zone_range;
if (td->o.min_bs[ddir] > td->o.ba[ddir])
max_size = td->o.zone_range;
if (td->o.min_bs[ddir] > td->o.ba[ddir])
@@
-503,19
+510,19
@@
static int get_next_offset(struct thread_data *td, struct io_u *io_u,
}
static inline bool io_u_fits(struct thread_data *td, struct io_u *io_u,
}
static inline bool io_u_fits(struct thread_data *td, struct io_u *io_u,
- unsigned
int
buflen)
+ unsigned
long long
buflen)
{
struct fio_file *f = io_u->file;
return io_u->offset + buflen <= f->io_size + get_start_offset(td, f);
}
{
struct fio_file *f = io_u->file;
return io_u->offset + buflen <= f->io_size + get_start_offset(td, f);
}
-static unsigned
int
get_next_buflen(struct thread_data *td, struct io_u *io_u,
+static unsigned
long long
get_next_buflen(struct thread_data *td, struct io_u *io_u,
bool is_random)
{
int ddir = io_u->ddir;
bool is_random)
{
int ddir = io_u->ddir;
- unsigned
int
buflen = 0;
- unsigned
int
minbs, maxbs;
+ unsigned
long long
buflen = 0;
+ unsigned
long long
minbs, maxbs;
uint64_t frand_max, r;
bool power_2;
uint64_t frand_max, r;
bool power_2;
@@
-541,10
+548,8
@@
static unsigned int get_next_buflen(struct thread_data *td, struct io_u *io_u,
r = __rand(&td->bsrange_state[ddir]);
if (!td->o.bssplit_nr[ddir]) {
r = __rand(&td->bsrange_state[ddir]);
if (!td->o.bssplit_nr[ddir]) {
- buflen =
1 + (unsigned int
) ((double) maxbs *
+ buflen =
minbs + (unsigned long long
) ((double) maxbs *
(r / (frand_max + 1.0)));
(r / (frand_max + 1.0)));
- if (buflen < minbs)
- buflen = minbs;
} else {
long long perc = 0;
unsigned int i;
} else {
long long perc = 0;
unsigned int i;
@@
-763,10
+768,18
@@
void put_file_log(struct thread_data *td, struct fio_file *f)
void put_io_u(struct thread_data *td, struct io_u *io_u)
{
void put_io_u(struct thread_data *td, struct io_u *io_u)
{
+ const bool needs_lock = td_async_processing(td);
+
+ if (io_u->post_submit) {
+ io_u->post_submit(io_u, io_u->error == 0);
+ io_u->post_submit = NULL;
+ }
+
if (td->parent)
td = td->parent;
if (td->parent)
td = td->parent;
- td_io_u_lock(td);
+ if (needs_lock)
+ __td_io_u_lock(td);
if (io_u->file && !(io_u->flags & IO_U_F_NO_FILE_PUT))
put_file_log(td, io_u->file);
if (io_u->file && !(io_u->flags & IO_U_F_NO_FILE_PUT))
put_file_log(td, io_u->file);
@@
-780,7
+793,9
@@
void put_io_u(struct thread_data *td, struct io_u *io_u)
}
io_u_qpush(&td->io_u_freelist, io_u);
td_io_u_free_notify(td);
}
io_u_qpush(&td->io_u_freelist, io_u);
td_io_u_free_notify(td);
- td_io_u_unlock(td);
+
+ if (needs_lock)
+ __td_io_u_unlock(td);
}
void clear_io_u(struct thread_data *td, struct io_u *io_u)
}
void clear_io_u(struct thread_data *td, struct io_u *io_u)
@@
-791,6
+806,7
@@
void clear_io_u(struct thread_data *td, struct io_u *io_u)
void requeue_io_u(struct thread_data *td, struct io_u **io_u)
{
void requeue_io_u(struct thread_data *td, struct io_u **io_u)
{
+ const bool needs_lock = td_async_processing(td);
struct io_u *__io_u = *io_u;
enum fio_ddir ddir = acct_ddir(__io_u);
struct io_u *__io_u = *io_u;
enum fio_ddir ddir = acct_ddir(__io_u);
@@
-799,7
+815,8
@@
void requeue_io_u(struct thread_data *td, struct io_u **io_u)
if (td->parent)
td = td->parent;
if (td->parent)
td = td->parent;
- td_io_u_lock(td);
+ if (needs_lock)
+ __td_io_u_lock(td);
io_u_set(td, __io_u, IO_U_F_FREE);
if ((__io_u->flags & IO_U_F_FLIGHT) && ddir_rw(ddir))
io_u_set(td, __io_u, IO_U_F_FREE);
if ((__io_u->flags & IO_U_F_FLIGHT) && ddir_rw(ddir))
@@
-813,14
+830,21
@@
void requeue_io_u(struct thread_data *td, struct io_u **io_u)
io_u_rpush(&td->io_u_requeues, __io_u);
td_io_u_free_notify(td);
io_u_rpush(&td->io_u_requeues, __io_u);
td_io_u_free_notify(td);
- td_io_u_unlock(td);
+
+ if (needs_lock)
+ __td_io_u_unlock(td);
+
*io_u = NULL;
}
*io_u = NULL;
}
-static void
__fill_io_u_zon
e(struct thread_data *td, struct io_u *io_u)
+static void
setup_strided_zone_mod
e(struct thread_data *td, struct io_u *io_u)
{
struct fio_file *f = io_u->file;
{
struct fio_file *f = io_u->file;
+ assert(td->o.zone_mode == ZONE_MODE_STRIDED);
+ assert(td->o.zone_size);
+ assert(td->o.zone_range);
+
/*
* See if it's time to switch to a new zone
*/
/*
* See if it's time to switch to a new zone
*/
@@
-859,6
+883,8
@@
static void __fill_io_u_zone(struct thread_data *td, struct io_u *io_u)
static int fill_io_u(struct thread_data *td, struct io_u *io_u)
{
bool is_random;
static int fill_io_u(struct thread_data *td, struct io_u *io_u)
{
bool is_random;
+ uint64_t offset;
+ enum io_u_action ret;
if (td_ioengine_flagged(td, FIO_NOIO))
goto out;
if (td_ioengine_flagged(td, FIO_NOIO))
goto out;
@@
-871,11
+897,8
@@
static int fill_io_u(struct thread_data *td, struct io_u *io_u)
if (!ddir_rw(io_u->ddir))
goto out;
if (!ddir_rw(io_u->ddir))
goto out;
- /*
- * When file is zoned zone_range is always positive
- */
- if (td->o.zone_range)
- __fill_io_u_zone(td, io_u);
+ if (td->o.zone_mode == ZONE_MODE_STRIDED)
+ setup_strided_zone_mode(td, io_u);
/*
* No log, let the seq/rand engine retrieve the next buflen and
/*
* No log, let the seq/rand engine retrieve the next buflen and
@@
-892,8
+915,15
@@
static int fill_io_u(struct thread_data *td, struct io_u *io_u)
return 1;
}
return 1;
}
+ offset = io_u->offset;
+ if (td->o.zone_mode == ZONE_MODE_ZBD) {
+ ret = zbd_adjust_block(td, io_u);
+ if (ret == io_u_eof)
+ return 1;
+ }
+
if (io_u->offset + io_u->buflen > io_u->file->real_file_size) {
if (io_u->offset + io_u->buflen > io_u->file->real_file_size) {
- dprint(FD_IO, "io_u %p, off=0x%llx + len=0x%lx exceeds file size=0x%llx\n",
+ dprint(FD_IO, "io_u %p, off=0x%llx + len=0x%l
l
x exceeds file size=0x%llx\n",
io_u,
(unsigned long long) io_u->offset, io_u->buflen,
(unsigned long long) io_u->file->real_file_size);
io_u,
(unsigned long long) io_u->offset, io_u->buflen,
(unsigned long long) io_u->file->real_file_size);
@@
-904,7
+934,7
@@
static int fill_io_u(struct thread_data *td, struct io_u *io_u)
* mark entry before potentially trimming io_u
*/
if (td_random(td) && file_randommap(td, io_u->file))
* mark entry before potentially trimming io_u
*/
if (td_random(td) && file_randommap(td, io_u->file))
-
mark_random_map(td, io_u
);
+
io_u->buflen = mark_random_map(td, io_u, offset, io_u->buflen
);
out:
dprint_io_u(io_u, "fill");
out:
dprint_io_u(io_u, "fill");
@@
-1305,6
+1335,11
@@
static long set_io_u_file(struct thread_data *td, struct io_u *io_u)
if (!fill_io_u(td, io_u))
break;
if (!fill_io_u(td, io_u))
break;
+ if (io_u->post_submit) {
+ io_u->post_submit(io_u, false);
+ io_u->post_submit = NULL;
+ }
+
put_file_log(td, f);
td_io_close_file(td, f);
io_u->file = NULL;
put_file_log(td, f);
td_io_close_file(td, f);
io_u->file = NULL;
@@
-1479,13
+1514,15
@@
bool queue_full(const struct thread_data *td)
struct io_u *__get_io_u(struct thread_data *td)
{
struct io_u *__get_io_u(struct thread_data *td)
{
+ const bool needs_lock = td_async_processing(td);
struct io_u *io_u = NULL;
int ret;
if (td->stop_io)
return NULL;
struct io_u *io_u = NULL;
int ret;
if (td->stop_io)
return NULL;
- td_io_u_lock(td);
+ if (needs_lock)
+ __td_io_u_lock(td);
again:
if (!io_u_rempty(&td->io_u_requeues))
again:
if (!io_u_rempty(&td->io_u_requeues))
@@
-1522,7
+1559,9
@@
again:
goto again;
}
goto again;
}
- td_io_u_unlock(td);
+ if (needs_lock)
+ __td_io_u_unlock(td);
+
return io_u;
}
return io_u;
}
@@
-1584,7
+1623,7
@@
static bool check_get_verify(struct thread_data *td, struct io_u *io_u)
*/
static void small_content_scramble(struct io_u *io_u)
{
*/
static void small_content_scramble(struct io_u *io_u)
{
- unsigned
int
i, nr_blocks = io_u->buflen >> 9;
+ unsigned
long long
i, nr_blocks = io_u->buflen >> 9;
unsigned int offset;
uint64_t boffset, *iptr;
char *p;
unsigned int offset;
uint64_t boffset, *iptr;
char *p;
@@
-1728,7
+1767,7
@@
static void __io_u_log_error(struct thread_data *td, struct io_u *io_u)
if (td_non_fatal_error(td, eb, io_u->error) && !td->o.error_dump)
return;
if (td_non_fatal_error(td, eb, io_u->error) && !td->o.error_dump)
return;
- log_err("fio: io_u error%s%s: %s: %s offset=%llu, buflen=%lu\n",
+ log_err("fio: io_u error%s%s: %s: %s offset=%llu, buflen=%l
l
u\n",
io_u->file ? " on file " : "",
io_u->file ? io_u->file->file_name : "",
strerror(io_u->error),
io_u->file ? " on file " : "",
io_u->file ? io_u->file->file_name : "",
strerror(io_u->error),
@@
-1894,7
+1933,7
@@
static void io_completed(struct thread_data *td, struct io_u **io_u_ptr,
td->last_ddir = ddir;
if (!io_u->error && ddir_rw(ddir)) {
td->last_ddir = ddir;
if (!io_u->error && ddir_rw(ddir)) {
- unsigned
int
bytes = io_u->buflen - io_u->resid;
+ unsigned
long long
bytes = io_u->buflen - io_u->resid;
int ret;
td->io_blocks[ddir]++;
int ret;
td->io_blocks[ddir]++;
@@
-2084,8
+2123,8
@@
static void save_buf_state(struct thread_data *td, struct frand_state *rs)
frand_copy(&td->buf_state_prev, rs);
}
frand_copy(&td->buf_state_prev, rs);
}
-void fill_io_buffer(struct thread_data *td, void *buf, unsigned
int
min_write,
- unsigned
int
max_bs)
+void fill_io_buffer(struct thread_data *td, void *buf, unsigned
long long
min_write,
+ unsigned
long long
max_bs)
{
struct thread_options *o = &td->o;
{
struct thread_options *o = &td->o;
@@
-2095,8
+2134,8
@@
void fill_io_buffer(struct thread_data *td, void *buf, unsigned int min_write,
if (o->compress_percentage || o->dedupe_percentage) {
unsigned int perc = td->o.compress_percentage;
struct frand_state *rs;
if (o->compress_percentage || o->dedupe_percentage) {
unsigned int perc = td->o.compress_percentage;
struct frand_state *rs;
- unsigned
int
left = max_bs;
- unsigned
int
this_write;
+ unsigned
long long
left = max_bs;
+ unsigned
long long
this_write;
do {
rs = get_buf_state(td);
do {
rs = get_buf_state(td);
@@
-2105,7
+2144,7
@@
void fill_io_buffer(struct thread_data *td, void *buf, unsigned int min_write,
if (perc) {
this_write = min_not_zero(min_write,
if (perc) {
this_write = min_not_zero(min_write,
- td->o.compress_chunk);
+
(unsigned long long)
td->o.compress_chunk);
fill_random_buf_percentage(rs, buf, perc,
this_write, this_write,
fill_random_buf_percentage(rs, buf, perc,
this_write, this_write,
@@
-2132,7
+2171,7
@@
void fill_io_buffer(struct thread_data *td, void *buf, unsigned int min_write,
* "randomly" fill the buffer contents
*/
void io_u_fill_buffer(struct thread_data *td, struct io_u *io_u,
* "randomly" fill the buffer contents
*/
void io_u_fill_buffer(struct thread_data *td, struct io_u *io_u,
- unsigned
int min_write, unsigned int
max_bs)
+ unsigned
long long min_write, unsigned long long
max_bs)
{
io_u->buf_filled_len = 0;
fill_io_buffer(td, io_u->buf, min_write, max_bs);
{
io_u->buf_filled_len = 0;
fill_io_buffer(td, io_u->buf, min_write, max_bs);