zbd: introduce per job maximum open zones limit

It is not possible to maintain sustained per-thread iodepth in ZBD mode.
The way code is written, "max_open_zones" acts as a global limit, and
once one or few threads open all "max_open_zones" zones, other threads
can't open anything and _exit_ prematurely.

This config is guaranteed to make equal number of zone resets/IO now:
each thread generates identical pattern and doesn't intersect with other
threads:

	zonemode=zbd
	zonesize=...
	rw=write

	numjobs=N
	offset_increment=M*zonesize

	[j]
	size=M*zonesize

Patch introduces "job_max_open_zones" which is per-thread/process limit.
"max_open_zones" remains per file/device limit. Both limits are checked
for each open zone so one thread can't kick out others.

Signed-off-by: Alexey Dobriyan (SK hynix) <adobriyan@gmail.com>
Reviewed-by: Damien Le Moal <damien.lemoal@wdc.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fio.1 b/fio.1
index c905e9ae6a5fc921e708c3286d65ad07083164d0..47bc15920831804ed3bba30ddc42685889f6d785 100644 (file)
--- a/fio.1
+++ b/fio.1
@@ -804,7 +804,11 @@ so. Default: false.
 When running a random write test across an entire drive many more zones will be
 open than in a typical application workload. Hence this command line option
 that allows to limit the number of open zones. The number of open zones is
-defined as the number of zones to which write commands are issued.
+defined as the number of zones to which write commands are issued by all
+threads/processes.
+.TP
+.BI job_max_open_zones \fR=\fPint
+Limit on the number of simultaneously opened zones per single thread/process.
 .TP
 .BI zone_reset_threshold \fR=\fPfloat
 A number between zero and one that indicates the ratio of logical blocks with

diff --git a/fio.h b/fio.h
index d496e7f27c4367206afaf5e9499d10ac5f8c6b5a..8045c32f67fa7400cc8fd37ebab781353f009cc0 100644 (file)
--- a/fio.h
+++ b/fio.h
@@ -260,6 +260,7 @@ struct thread_data {
        struct frand_state prio_state;
 
        struct zone_split_index **zone_state_index;
+       unsigned int num_open_zones;
 
        unsigned int verify_batch;
        unsigned int trim_batch;

diff --git a/options.c b/options.c
index da401aed118c1b2b6e5da25be4ebab4268e3a242..f2d98fa6bb59f4425dfdc7a155e2f55e725351d0 100644 (file)
--- a/options.c
+++ b/options.c
@@ -3360,12 +3360,22 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
        },
        {
                .name   = "max_open_zones",
-               .lname  = "Maximum number of open zones",
+               .lname  = "Per device/file maximum number of open zones",
                .type   = FIO_OPT_INT,
                .off1   = offsetof(struct thread_options, max_open_zones),
                .maxval = ZBD_MAX_OPEN_ZONES,
-               .help   = "Limit random writes to SMR drives to the specified"
-                         " number of sequential zones",
+               .help   = "Limit on the number of simultaneously opened sequential write zones with zonemode=zbd",
+               .def    = "0",
+               .category = FIO_OPT_C_IO,
+               .group  = FIO_OPT_G_INVALID,
+       },
+       {
+               .name   = "job_max_open_zones",
+               .lname  = "Job maximum number of open zones",
+               .type   = FIO_OPT_INT,
+               .off1   = offsetof(struct thread_options, job_max_open_zones),
+               .maxval = ZBD_MAX_OPEN_ZONES,
+               .help   = "Limit on the number of simultaneously opened sequential write zones with zonemode=zbd by one thread/process",
                .def    = "0",
                .category = FIO_OPT_C_IO,
                .group  = FIO_OPT_G_INVALID,

diff --git a/thread_options.h b/thread_options.h
index 09ccd5b2259fc126812aa17b9eb23ad0deb99847..968ea0abc0b15a91bbc55fa4d04a231f5e3a3474 100644 (file)
--- a/thread_options.h
+++ b/thread_options.h
@@ -343,6 +343,7 @@ struct thread_options {
        /* Parameters that affect zonemode=zbd */
        unsigned int read_beyond_wp;
        int max_open_zones;
+       unsigned int job_max_open_zones;
        fio_fp64_t zrt;
        fio_fp64_t zrf;
 };

diff --git a/zbd.c b/zbd.c
index 7185d85caa84ffca0c339047b86cf52b108a2e4b..61c685a1ec4881887d134f1e3a6f45a41f892474 100644 (file)
--- a/zbd.c
+++ b/zbd.c
@@ -548,8 +548,10 @@ static int zbd_create_zone_info(struct thread_data *td, struct fio_file *f)
                return -EINVAL;
        }
 
-       if (ret == 0)
+       if (ret == 0) {
                f->zbd_info->model = zbd_model;
+               f->zbd_info->max_open_zones = td->o.max_open_zones;
+       }
        return ret;
 }
 
@@ -623,6 +625,25 @@ int zbd_setup_files(struct thread_data *td)
        if (!zbd_verify_bs())
                return 1;
 
+       for_each_file(td, f, i) {
+               struct zoned_block_device_info *zbd = f->zbd_info;
+
+               if (!zbd)
+                       continue;
+
+               zbd->max_open_zones = zbd->max_open_zones ?: ZBD_MAX_OPEN_ZONES;
+
+               if (td->o.max_open_zones > 0 &&
+                   zbd->max_open_zones != td->o.max_open_zones) {
+                       log_err("Different 'max_open_zones' values\n");
+                       return 1;
+               }
+               if (zbd->max_open_zones > ZBD_MAX_OPEN_ZONES) {
+                       log_err("'max_open_zones' value is limited by %u\n", ZBD_MAX_OPEN_ZONES);
+                       return 1;
+               }
+       }
+
        return 0;
 }
 
@@ -710,6 +731,7 @@ static void zbd_close_zone(struct thread_data *td, const struct fio_file *f,
                (ZBD_MAX_OPEN_ZONES - (open_zone_idx + 1)) *
                sizeof(f->zbd_info->open_zones[0]));
        f->zbd_info->num_open_zones--;
+       td->num_open_zones--;
        f->zbd_info->zone_info[zone_idx].open = 0;
 }
 
@@ -887,8 +909,9 @@ static bool is_zone_open(const struct thread_data *td, const struct fio_file *f,
        struct zoned_block_device_info *zbdi = f->zbd_info;
        int i;
 
-       assert(td->o.max_open_zones <= ARRAY_SIZE(zbdi->open_zones));
-       assert(zbdi->num_open_zones <= td->o.max_open_zones);
+       assert(td->o.job_max_open_zones == 0 || td->num_open_zones <= td->o.job_max_open_zones);
+       assert(td->o.job_max_open_zones <= zbdi->max_open_zones);
+       assert(zbdi->num_open_zones <= zbdi->max_open_zones);
 
        for (i = 0; i < zbdi->num_open_zones; i++)
                if (zbdi->open_zones[i] == zone_idx)
@@ -921,18 +944,19 @@ static bool zbd_open_zone(struct thread_data *td, const struct io_u *io_u,
        if (td->o.verify != VERIFY_NONE && zbd_zone_full(f, z, min_bs))
                return false;
 
-       /* Zero means no limit */
-       if (!td->o.max_open_zones)
-               return true;
-
        pthread_mutex_lock(&f->zbd_info->mutex);
        if (is_zone_open(td, f, zone_idx))
                goto out;
        res = false;
-       if (f->zbd_info->num_open_zones >= td->o.max_open_zones)
+       /* Zero means no limit */
+       if (td->o.job_max_open_zones > 0 &&
+           td->num_open_zones >= td->o.job_max_open_zones)
+               goto out;
+       if (f->zbd_info->num_open_zones >= f->zbd_info->max_open_zones)
                goto out;
        dprint(FD_ZBD, "%s: opening zone %d\n", f->file_name, zone_idx);
        f->zbd_info->open_zones[f->zbd_info->num_open_zones++] = zone_idx;
+       td->num_open_zones++;
        z->open = 1;
        res = true;
 
@@ -967,7 +991,7 @@ static struct fio_zone_info *zbd_convert_to_open_zone(struct thread_data *td,
 
        assert(is_valid_offset(f, io_u->offset));
 
-       if (td->o.max_open_zones) {
+       if (td->o.job_max_open_zones) {
                /*
                 * This statement accesses f->zbd_info->open_zones[] on purpose
                 * without locking.
@@ -996,7 +1020,7 @@ static struct fio_zone_info *zbd_convert_to_open_zone(struct thread_data *td,
 
                zone_lock(td, f, z);
                pthread_mutex_lock(&f->zbd_info->mutex);
-               if (td->o.max_open_zones == 0)
+               if (td->o.job_max_open_zones == 0)
                        goto examine_zone;
                if (f->zbd_info->num_open_zones == 0) {
                        pthread_mutex_unlock(&f->zbd_info->mutex);
@@ -1052,7 +1076,7 @@ examine_zone:
        }
        dprint(FD_ZBD, "%s(%s): closing zone %d\n", __func__, f->file_name,
               zone_idx);
-       if (td->o.max_open_zones)
+       if (td->o.job_max_open_zones)
                zbd_close_zone(td, f, open_zone_idx);
        pthread_mutex_unlock(&f->zbd_info->mutex);
 

diff --git a/zbd.h b/zbd.h
index e8dd3d6d5d3610c740a9f7da8baf695c05c0834e..9c447af40f707fd65cbe652674e5a374a34d4878 100644 (file)
--- a/zbd.h
+++ b/zbd.h
@@ -45,6 +45,8 @@ struct fio_zone_info {
 /**
  * zoned_block_device_info - zoned block device characteristics
  * @model: Device model.
+ * @max_open_zones: global limit on the number of simultaneously opened
+ *     sequential write zones.
  * @mutex: Protects the modifiable members in this structure (refcount and
  *             num_open_zones).
  * @zone_size: size of a single zone in units of 512 bytes
@@ -65,6 +67,7 @@ struct fio_zone_info {
  */
 struct zoned_block_device_info {
        enum zbd_zoned_model    model;
+       uint32_t                max_open_zones;
        pthread_mutex_t         mutex;
        uint64_t                zone_size;
        uint64_t                sectors_with_data;