#ifdef FIO_HAVE_SGIO
+#ifndef SGV4_FLAG_HIPRI
+#define SGV4_FLAG_HIPRI 0x800
+#endif
+
enum {
FIO_SG_WRITE = 1,
FIO_SG_WRITE_VERIFY = 2,
struct sg_options {
void *pad;
+ unsigned int hipri;
unsigned int readfua;
unsigned int writefua;
unsigned int write_mode;
};
static struct fio_option options[] = {
+ {
+ .name = "hipri",
+ .lname = "High Priority",
+ .type = FIO_OPT_STR_SET,
+ .off1 = offsetof(struct sg_options, hipri),
+ .help = "Use polled IO completions",
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_SG,
+ },
{
.name = "readfua",
.lname = "sg engine read fua flag support",
};
struct sgio_trim {
- char *unmap_param;
+ uint8_t *unmap_param;
unsigned int unmap_range_count;
struct io_u **trim_io_us;
};
#endif
};
+static inline uint32_t sgio_get_be32(uint8_t *buf)
+{
+ return be32_to_cpu(*((uint32_t *) buf));
+}
+
+static inline uint64_t sgio_get_be64(uint8_t *buf)
+{
+ return be64_to_cpu(*((uint64_t *) buf));
+}
+
+static inline void sgio_set_be16(uint16_t val, uint8_t *buf)
+{
+ uint16_t t = cpu_to_be16(val);
+
+ memcpy(buf, &t, sizeof(uint16_t));
+}
+
+static inline void sgio_set_be32(uint32_t val, uint8_t *buf)
+{
+ uint32_t t = cpu_to_be32(val);
+
+ memcpy(buf, &t, sizeof(uint32_t));
+}
+
+static inline void sgio_set_be64(uint64_t val, uint8_t *buf)
+{
+ uint64_t t = cpu_to_be64(val);
+
+ memcpy(buf, &t, sizeof(uint64_t));
+}
+
static inline bool sgio_unbuffered(struct thread_data *td)
{
return (td->o.odirect || td->o.sync_io);
return FIO_Q_COMPLETED;
}
-static enum fio_q_status fio_sgio_rw_doio(struct fio_file *f,
+static enum fio_q_status fio_sgio_rw_doio(struct thread_data *td,
+ struct fio_file *f,
struct io_u *io_u, int do_sync)
{
struct sg_io_hdr *hdr = &io_u->hdr;
return ret;
if (do_sync) {
- ret = read(f->fd, hdr, sizeof(*hdr));
- if (ret < 0)
- return ret;
+ /*
+ * We can't just read back the first command that completes
+ * and assume it's the one we need, it could be any command
+ * that is inflight.
+ */
+ do {
+ struct io_u *__io_u;
- /* record if an io error occurred */
- if (hdr->info & SG_INFO_CHECK)
- io_u->error = EIO;
+ ret = read(f->fd, hdr, sizeof(*hdr));
+ if (ret < 0)
+ return ret;
+
+ __io_u = hdr->usr_ptr;
+
+ /* record if an io error occurred */
+ if (hdr->info & SG_INFO_CHECK)
+ __io_u->error = EIO;
+
+ if (__io_u == io_u)
+ break;
+
+ if (io_u_sync_complete(td, __io_u)) {
+ ret = -1;
+ break;
+ }
+ } while (1);
return FIO_Q_COMPLETED;
}
if (f->filetype == FIO_TYPE_BLOCK) {
ret = fio_sgio_ioctl_doio(td, f, io_u);
- td_verror(td, io_u->error, __func__);
+ if (io_u->error)
+ td_verror(td, io_u->error, __func__);
} else {
- ret = fio_sgio_rw_doio(f, io_u, do_sync);
- if (do_sync)
+ ret = fio_sgio_rw_doio(td, f, io_u, do_sync);
+ if (io_u->error && do_sync)
td_verror(td, io_u->error, __func__);
}
unsigned long long nr_blocks)
{
if (lba < MAX_10B_LBA) {
- hdr->cmdp[2] = (unsigned char) ((lba >> 24) & 0xff);
- hdr->cmdp[3] = (unsigned char) ((lba >> 16) & 0xff);
- hdr->cmdp[4] = (unsigned char) ((lba >> 8) & 0xff);
- hdr->cmdp[5] = (unsigned char) (lba & 0xff);
- hdr->cmdp[7] = (unsigned char) ((nr_blocks >> 8) & 0xff);
- hdr->cmdp[8] = (unsigned char) (nr_blocks & 0xff);
+ sgio_set_be32((uint32_t) lba, &hdr->cmdp[2]);
+ sgio_set_be16((uint16_t) nr_blocks, &hdr->cmdp[7]);
} else {
- hdr->cmdp[2] = (unsigned char) ((lba >> 56) & 0xff);
- hdr->cmdp[3] = (unsigned char) ((lba >> 48) & 0xff);
- hdr->cmdp[4] = (unsigned char) ((lba >> 40) & 0xff);
- hdr->cmdp[5] = (unsigned char) ((lba >> 32) & 0xff);
- hdr->cmdp[6] = (unsigned char) ((lba >> 24) & 0xff);
- hdr->cmdp[7] = (unsigned char) ((lba >> 16) & 0xff);
- hdr->cmdp[8] = (unsigned char) ((lba >> 8) & 0xff);
- hdr->cmdp[9] = (unsigned char) (lba & 0xff);
- hdr->cmdp[10] = (unsigned char) ((nr_blocks >> 32) & 0xff);
- hdr->cmdp[11] = (unsigned char) ((nr_blocks >> 16) & 0xff);
- hdr->cmdp[12] = (unsigned char) ((nr_blocks >> 8) & 0xff);
- hdr->cmdp[13] = (unsigned char) (nr_blocks & 0xff);
+ sgio_set_be64(lba, &hdr->cmdp[2]);
+ sgio_set_be32((uint32_t) nr_blocks, &hdr->cmdp[10]);
}
return;
else
hdr->cmdp[0] = 0x88; // read(16)
+ if (o->hipri)
+ hdr->flags |= SGV4_FLAG_HIPRI;
if (o->readfua)
hdr->cmdp[1] |= 0x08;
hdr->cmdp[0] = 0x2a; // write(10)
else
hdr->cmdp[0] = 0x8a; // write(16)
+ if (o->hipri)
+ hdr->flags |= SGV4_FLAG_HIPRI;
if (o->writefua)
hdr->cmdp[1] |= 0x08;
break;
#endif
offset = 8 + 16 * st->unmap_range_count;
- st->unmap_param[offset] = (unsigned char) ((lba >> 56) & 0xff);
- st->unmap_param[offset+1] = (unsigned char) ((lba >> 48) & 0xff);
- st->unmap_param[offset+2] = (unsigned char) ((lba >> 40) & 0xff);
- st->unmap_param[offset+3] = (unsigned char) ((lba >> 32) & 0xff);
- st->unmap_param[offset+4] = (unsigned char) ((lba >> 24) & 0xff);
- st->unmap_param[offset+5] = (unsigned char) ((lba >> 16) & 0xff);
- st->unmap_param[offset+6] = (unsigned char) ((lba >> 8) & 0xff);
- st->unmap_param[offset+7] = (unsigned char) (lba & 0xff);
- st->unmap_param[offset+8] = (unsigned char) ((nr_blocks >> 32) & 0xff);
- st->unmap_param[offset+9] = (unsigned char) ((nr_blocks >> 16) & 0xff);
- st->unmap_param[offset+10] = (unsigned char) ((nr_blocks >> 8) & 0xff);
- st->unmap_param[offset+11] = (unsigned char) (nr_blocks & 0xff);
+ sgio_set_be64(lba, &st->unmap_param[offset]);
+ sgio_set_be32((uint32_t) nr_blocks, &st->unmap_param[offset + 8]);
st->unmap_range_count++;
static void fio_sgio_unmap_setup(struct sg_io_hdr *hdr, struct sgio_trim *st)
{
- hdr->dxfer_len = st->unmap_range_count * 16 + 8;
- hdr->cmdp[7] = (unsigned char) (((st->unmap_range_count * 16 + 8) >> 8) & 0xff);
- hdr->cmdp[8] = (unsigned char) ((st->unmap_range_count * 16 + 8) & 0xff);
+ uint16_t cnt = st->unmap_range_count * 16;
- st->unmap_param[0] = (unsigned char) (((16 * st->unmap_range_count + 6) >> 8) & 0xff);
- st->unmap_param[1] = (unsigned char) ((16 * st->unmap_range_count + 6) & 0xff);
- st->unmap_param[2] = (unsigned char) (((16 * st->unmap_range_count) >> 8) & 0xff);
- st->unmap_param[3] = (unsigned char) ((16 * st->unmap_range_count) & 0xff);
+ hdr->dxfer_len = cnt + 8;
+ sgio_set_be16(cnt + 8, &hdr->cmdp[7]);
+ sgio_set_be16(cnt + 6, st->unmap_param);
+ sgio_set_be16(cnt, &st->unmap_param[2]);
return;
}
sd->current_queue = -1;
- ret = fio_sgio_rw_doio(io_u->file, io_u, 0);
+ ret = fio_sgio_rw_doio(td, io_u->file, io_u, 0);
- if (ret < 0)
- for (i = 0; i < st->unmap_range_count; i++)
- st->trim_io_us[i]->error = errno;
- else if (hdr->status)
- for (i = 0; i < st->unmap_range_count; i++) {
- st->trim_io_us[i]->resid = hdr->resid;
- st->trim_io_us[i]->error = EIO;
+ if (ret < 0 || hdr->status) {
+ int error;
+
+ if (ret < 0)
+ error = errno;
+ else {
+ error = EIO;
+ ret = -EIO;
}
- else {
- if (fio_fill_issue_time(td)) {
- fio_gettime(&now, NULL);
- for (i = 0; i < st->unmap_range_count; i++) {
- struct io_u *io_u = st->trim_io_us[i];
-
- memcpy(&io_u->issue_time, &now, sizeof(now));
- io_u_queued(td, io_u);
- }
+
+ for (i = 0; i < st->unmap_range_count; i++) {
+ st->trim_io_us[i]->error = error;
+ clear_io_u(td, st->trim_io_us[i]);
+ if (hdr->status)
+ st->trim_io_us[i]->resid = hdr->resid;
}
- io_u_mark_submit(td, st->unmap_range_count);
+
+ td_verror(td, error, "xfer");
+ return ret;
}
- if (io_u->error) {
- td_verror(td, io_u->error, "xfer");
- return 0;
+ if (fio_fill_issue_time(td)) {
+ fio_gettime(&now, NULL);
+ for (i = 0; i < st->unmap_range_count; i++) {
+ memcpy(&st->trim_io_us[i]->issue_time, &now, sizeof(now));
+ io_u_queued(td, io_u);
+ }
}
+ io_u_mark_submit(td, st->unmap_range_count);
- if (ret == FIO_Q_QUEUED)
- return 0;
- else
- return ret;
+ return 0;
}
static struct io_u *fio_sgio_event(struct thread_data *td, int event)
* io_u structures, which are not initialized until later.
*/
struct sg_io_hdr hdr;
+ unsigned long long hlba;
+ unsigned int blksz = 0;
unsigned char cmd[16];
unsigned char sb[64];
unsigned char buf[32]; // read capacity return
return ret;
}
- *bs = ((unsigned long) buf[4] << 24) | ((unsigned long) buf[5] << 16) |
- ((unsigned long) buf[6] << 8) | (unsigned long) buf[7];
- *max_lba = ((unsigned long) buf[0] << 24) | ((unsigned long) buf[1] << 16) |
- ((unsigned long) buf[2] << 8) | (unsigned long) buf[3];
+ if (hdr.info & SG_INFO_CHECK) {
+ /* RCAP(10) might be unsupported by device. Force RCAP(16) */
+ hlba = MAX_10B_LBA;
+ } else {
+ blksz = sgio_get_be32(&buf[4]);
+ hlba = sgio_get_be32(buf);
+ }
/*
* If max lba masked by MAX_10B_LBA equals MAX_10B_LBA,
* then need to retry with 16 byte Read Capacity command.
*/
- if (*max_lba == MAX_10B_LBA) {
+ if (hlba == MAX_10B_LBA) {
hdr.cmd_len = 16;
hdr.cmdp[0] = 0x9e; // service action
hdr.cmdp[1] = 0x10; // Read Capacity(16)
- hdr.cmdp[10] = (unsigned char) ((sizeof(buf) >> 24) & 0xff);
- hdr.cmdp[11] = (unsigned char) ((sizeof(buf) >> 16) & 0xff);
- hdr.cmdp[12] = (unsigned char) ((sizeof(buf) >> 8) & 0xff);
- hdr.cmdp[13] = (unsigned char) (sizeof(buf) & 0xff);
+ sgio_set_be32(sizeof(buf), &hdr.cmdp[10]);
hdr.dxfer_direction = SG_DXFER_FROM_DEV;
hdr.dxferp = buf;
if (hdr.info & SG_INFO_CHECK)
td_verror(td, EIO, "fio_sgio_read_capacity");
- *bs = (buf[8] << 24) | (buf[9] << 16) | (buf[10] << 8) | buf[11];
- *max_lba = ((unsigned long long)buf[0] << 56) |
- ((unsigned long long)buf[1] << 48) |
- ((unsigned long long)buf[2] << 40) |
- ((unsigned long long)buf[3] << 32) |
- ((unsigned long long)buf[4] << 24) |
- ((unsigned long long)buf[5] << 16) |
- ((unsigned long long)buf[6] << 8) |
- (unsigned long long)buf[7];
+ blksz = sgio_get_be32(&buf[8]);
+ hlba = sgio_get_be64(buf);
+ }
+
+ if (blksz) {
+ *bs = blksz;
+ *max_lba = hlba;
+ ret = 0;
+ } else {
+ ret = EIO;
}
close(fd);
- return 0;
+ return ret;
}
static void fio_sgio_cleanup(struct thread_data *td)
{
struct sgio_data *sd;
struct sgio_trim *st;
+ struct sg_io_hdr *h3p;
int i;
sd = calloc(1, sizeof(*sd));
#ifdef FIO_SGIO_DEBUG
sd->trim_queue_map = calloc(td->o.iodepth, sizeof(int));
#endif
- for (i = 0; i < td->o.iodepth; i++) {
+ for (i = 0, h3p = sd->sgbuf; i < td->o.iodepth; i++, ++h3p) {
sd->trim_queues[i] = calloc(1, sizeof(struct sgio_trim));
st = sd->trim_queues[i];
st->unmap_param = calloc(td->o.iodepth + 1, sizeof(char[16]));
st->unmap_range_count = 0;
st->trim_io_us = calloc(td->o.iodepth, sizeof(struct io_u *));
+ h3p->interface_id = 'S';
}
td->io_ops_data = sd;
}
if (!(hdr->info & SG_INFO_CHECK) && !strlen(msg))
- strncpy(msg, "SG Driver did not report a Host, Driver or Device check",
- MAXERRDETAIL - 1);
+ snprintf(msg, MAXERRDETAIL, "%s",
+ "SG Driver did not report a Host, Driver or Device check");
return msg;
}
projects / fio.git / blobdiff