#define MAX_10B_LBA 0xFFFFFFFFULL
#define SCSI_TIMEOUT_MS 30000 // 30 second timeout; currently no method to override
#define MAX_SB 64 // sense block maximum return size
+/*
+#define FIO_SGIO_DEBUG
+*/
struct sgio_cmd {
unsigned char cdb[16]; // enhanced from 10 to support 16 byte commands
};
struct sgio_trim {
- char *unmap_param;
+ uint8_t *unmap_param;
unsigned int unmap_range_count;
struct io_u **trim_io_us;
};
int type_checked;
struct sgio_trim **trim_queues;
int current_queue;
+#ifdef FIO_SGIO_DEBUG
unsigned int *trim_queue_map;
+#endif
};
+static inline uint32_t sgio_get_be32(uint8_t *buf)
+{
+ return be32_to_cpu(*((uint32_t *) buf));
+}
+
+static inline uint64_t sgio_get_be64(uint8_t *buf)
+{
+ return be64_to_cpu(*((uint64_t *) buf));
+}
+
+static inline void sgio_set_be16(uint16_t val, uint8_t *buf)
+{
+ uint16_t t = cpu_to_be16(val);
+
+ memcpy(buf, &t, sizeof(uint16_t));
+}
+
+static inline void sgio_set_be32(uint32_t val, uint8_t *buf)
+{
+ uint32_t t = cpu_to_be32(val);
+
+ memcpy(buf, &t, sizeof(uint32_t));
+}
+
+static inline void sgio_set_be64(uint64_t val, uint8_t *buf)
+{
+ uint64_t t = cpu_to_be64(val);
+
+ memcpy(buf, &t, sizeof(uint64_t));
+}
+
static inline bool sgio_unbuffered(struct thread_data *td)
{
return (td->o.odirect || td->o.sync_io);
if (io_u->ddir == DDIR_TRIM) {
struct sgio_trim *st = sd->trim_queues[io_u->index];
+#ifdef FIO_SGIO_DEBUG
assert(st->trim_io_us[0] == io_u);
+ assert(sd->trim_queue_map[io_u->index] == io_u->index);
dprint(FD_IO, "sgio_getevents: reaping %d io_us from trim queue %d\n", st->unmap_range_count, io_u->index);
dprint(FD_IO, "sgio_getevents: reaped io_u %d and stored in events[%d]\n", io_u->index, i+trims);
+#endif
for (j = 1; j < st->unmap_range_count; j++) {
++trims;
sd->events[i + trims] = st->trim_io_us[j];
+#ifdef FIO_SGIO_DEBUG
dprint(FD_IO, "sgio_getevents: reaped io_u %d and stored in events[%d]\n", st->trim_io_us[j]->index, i+trims);
+ assert(sd->trim_queue_map[st->trim_io_us[j]->index] == io_u->index);
+#endif
if (hdr->info & SG_INFO_CHECK) {
/* record if an io error occurred, ignore resid */
memcpy(&st->trim_io_us[j]->hdr, hdr, sizeof(struct sg_io_hdr));
return FIO_Q_COMPLETED;
}
-static enum fio_q_status fio_sgio_rw_doio(struct fio_file *f,
+static enum fio_q_status fio_sgio_rw_doio(struct thread_data *td,
+ struct fio_file *f,
struct io_u *io_u, int do_sync)
{
struct sg_io_hdr *hdr = &io_u->hdr;
return ret;
if (do_sync) {
- ret = read(f->fd, hdr, sizeof(*hdr));
- if (ret < 0)
- return ret;
+ /*
+ * We can't just read back the first command that completes
+ * and assume it's the one we need, it could be any command
+ * that is inflight.
+ */
+ do {
+ struct io_u *__io_u;
- /* record if an io error occurred */
- if (hdr->info & SG_INFO_CHECK)
- io_u->error = EIO;
+ ret = read(f->fd, hdr, sizeof(*hdr));
+ if (ret < 0)
+ return ret;
+
+ __io_u = hdr->usr_ptr;
+
+ /* record if an io error occurred */
+ if (hdr->info & SG_INFO_CHECK)
+ __io_u->error = EIO;
+
+ if (__io_u == io_u)
+ break;
+
+ if (io_u_sync_complete(td, __io_u)) {
+ ret = -1;
+ break;
+ }
+ } while (1);
return FIO_Q_COMPLETED;
}
if (f->filetype == FIO_TYPE_BLOCK) {
ret = fio_sgio_ioctl_doio(td, f, io_u);
- td_verror(td, io_u->error, __func__);
+ if (io_u->error)
+ td_verror(td, io_u->error, __func__);
} else {
- ret = fio_sgio_rw_doio(f, io_u, do_sync);
- if (do_sync)
+ ret = fio_sgio_rw_doio(td, f, io_u, do_sync);
+ if (io_u->error && do_sync)
td_verror(td, io_u->error, __func__);
}
unsigned long long nr_blocks)
{
if (lba < MAX_10B_LBA) {
- hdr->cmdp[2] = (unsigned char) ((lba >> 24) & 0xff);
- hdr->cmdp[3] = (unsigned char) ((lba >> 16) & 0xff);
- hdr->cmdp[4] = (unsigned char) ((lba >> 8) & 0xff);
- hdr->cmdp[5] = (unsigned char) (lba & 0xff);
- hdr->cmdp[7] = (unsigned char) ((nr_blocks >> 8) & 0xff);
- hdr->cmdp[8] = (unsigned char) (nr_blocks & 0xff);
+ sgio_set_be32((uint32_t) lba, &hdr->cmdp[2]);
+ sgio_set_be16((uint16_t) nr_blocks, &hdr->cmdp[7]);
} else {
- hdr->cmdp[2] = (unsigned char) ((lba >> 56) & 0xff);
- hdr->cmdp[3] = (unsigned char) ((lba >> 48) & 0xff);
- hdr->cmdp[4] = (unsigned char) ((lba >> 40) & 0xff);
- hdr->cmdp[5] = (unsigned char) ((lba >> 32) & 0xff);
- hdr->cmdp[6] = (unsigned char) ((lba >> 24) & 0xff);
- hdr->cmdp[7] = (unsigned char) ((lba >> 16) & 0xff);
- hdr->cmdp[8] = (unsigned char) ((lba >> 8) & 0xff);
- hdr->cmdp[9] = (unsigned char) (lba & 0xff);
- hdr->cmdp[10] = (unsigned char) ((nr_blocks >> 32) & 0xff);
- hdr->cmdp[11] = (unsigned char) ((nr_blocks >> 16) & 0xff);
- hdr->cmdp[12] = (unsigned char) ((nr_blocks >> 8) & 0xff);
- hdr->cmdp[13] = (unsigned char) (nr_blocks & 0xff);
+ sgio_set_be64(lba, &hdr->cmdp[2]);
+ sgio_set_be32((uint32_t) nr_blocks, &hdr->cmdp[10]);
}
return;
sd->current_queue = io_u->index;
st = sd->trim_queues[sd->current_queue];
hdr->dxferp = st->unmap_param;
+#ifdef FIO_SGIO_DEBUG
assert(sd->trim_queues[io_u->index]->unmap_range_count == 0);
dprint(FD_IO, "sg: creating new queue based on io_u %d\n", io_u->index);
+#endif
}
else
st = sd->trim_queues[sd->current_queue];
dprint(FD_IO, "sg: adding io_u %d to trim queue %d\n", io_u->index, sd->current_queue);
st->trim_io_us[st->unmap_range_count] = io_u;
+#ifdef FIO_SGIO_DEBUG
sd->trim_queue_map[io_u->index] = sd->current_queue;
+#endif
offset = 8 + 16 * st->unmap_range_count;
- st->unmap_param[offset] = (unsigned char) ((lba >> 56) & 0xff);
- st->unmap_param[offset+1] = (unsigned char) ((lba >> 48) & 0xff);
- st->unmap_param[offset+2] = (unsigned char) ((lba >> 40) & 0xff);
- st->unmap_param[offset+3] = (unsigned char) ((lba >> 32) & 0xff);
- st->unmap_param[offset+4] = (unsigned char) ((lba >> 24) & 0xff);
- st->unmap_param[offset+5] = (unsigned char) ((lba >> 16) & 0xff);
- st->unmap_param[offset+6] = (unsigned char) ((lba >> 8) & 0xff);
- st->unmap_param[offset+7] = (unsigned char) (lba & 0xff);
- st->unmap_param[offset+8] = (unsigned char) ((nr_blocks >> 32) & 0xff);
- st->unmap_param[offset+9] = (unsigned char) ((nr_blocks >> 16) & 0xff);
- st->unmap_param[offset+10] = (unsigned char) ((nr_blocks >> 8) & 0xff);
- st->unmap_param[offset+11] = (unsigned char) (nr_blocks & 0xff);
+ sgio_set_be64(lba, &st->unmap_param[offset]);
+ sgio_set_be32((uint32_t) nr_blocks, &st->unmap_param[offset + 8]);
st->unmap_range_count++;
static void fio_sgio_unmap_setup(struct sg_io_hdr *hdr, struct sgio_trim *st)
{
- hdr->dxfer_len = st->unmap_range_count * 16 + 8;
- hdr->cmdp[7] = (unsigned char) (((st->unmap_range_count * 16 + 8) >> 8) & 0xff);
- hdr->cmdp[8] = (unsigned char) ((st->unmap_range_count * 16 + 8) & 0xff);
+ uint16_t cnt = st->unmap_range_count * 16;
- st->unmap_param[0] = (unsigned char) (((16 * st->unmap_range_count + 6) >> 8) & 0xff);
- st->unmap_param[1] = (unsigned char) ((16 * st->unmap_range_count + 6) & 0xff);
- st->unmap_param[2] = (unsigned char) (((16 * st->unmap_range_count) >> 8) & 0xff);
- st->unmap_param[3] = (unsigned char) ((16 * st->unmap_range_count) & 0xff);
+ hdr->dxfer_len = cnt + 8;
+ sgio_set_be16(cnt + 8, &hdr->cmdp[7]);
+ sgio_set_be16(cnt + 6, st->unmap_param);
+ sgio_set_be16(cnt, &st->unmap_param[2]);
return;
}
/* finish cdb setup for unmap because we are
** doing unmap commands synchronously */
+#ifdef FIO_SGIO_DEBUG
assert(st->unmap_range_count == 1);
assert(io_u == st->trim_io_us[0]);
+#endif
hdr = &io_u->hdr;
fio_sgio_unmap_setup(hdr, st);
sd->current_queue = -1;
- ret = fio_sgio_rw_doio(io_u->file, io_u, 0);
+ ret = fio_sgio_rw_doio(td, io_u->file, io_u, 0);
- if (ret < 0)
- for (i = 0; i < st->unmap_range_count; i++)
- st->trim_io_us[i]->error = errno;
- else if (hdr->status)
- for (i = 0; i < st->unmap_range_count; i++) {
- st->trim_io_us[i]->resid = hdr->resid;
- st->trim_io_us[i]->error = EIO;
+ if (ret < 0 || hdr->status) {
+ int error;
+
+ if (ret < 0)
+ error = errno;
+ else {
+ error = EIO;
+ ret = -EIO;
}
- else {
- if (fio_fill_issue_time(td)) {
- fio_gettime(&now, NULL);
- for (i = 0; i < st->unmap_range_count; i++) {
- struct io_u *io_u = st->trim_io_us[i];
-
- memcpy(&io_u->issue_time, &now, sizeof(now));
- io_u_queued(td, io_u);
- }
+
+ for (i = 0; i < st->unmap_range_count; i++) {
+ st->trim_io_us[i]->error = error;
+ clear_io_u(td, st->trim_io_us[i]);
+ if (hdr->status)
+ st->trim_io_us[i]->resid = hdr->resid;
}
- io_u_mark_submit(td, st->unmap_range_count);
+
+ td_verror(td, error, "xfer");
+ return ret;
}
- if (io_u->error) {
- td_verror(td, io_u->error, "xfer");
- return 0;
+ if (fio_fill_issue_time(td)) {
+ fio_gettime(&now, NULL);
+ for (i = 0; i < st->unmap_range_count; i++) {
+ memcpy(&st->trim_io_us[i]->issue_time, &now, sizeof(now));
+ io_u_queued(td, io_u);
+ }
}
+ io_u_mark_submit(td, st->unmap_range_count);
- if (ret == FIO_Q_QUEUED)
- return 0;
- else
- return ret;
+ return 0;
}
static struct io_u *fio_sgio_event(struct thread_data *td, int event)
* io_u structures, which are not initialized until later.
*/
struct sg_io_hdr hdr;
+ unsigned long long hlba;
+ unsigned int blksz = 0;
unsigned char cmd[16];
unsigned char sb[64];
unsigned char buf[32]; // read capacity return
return ret;
}
- *bs = ((unsigned long) buf[4] << 24) | ((unsigned long) buf[5] << 16) |
- ((unsigned long) buf[6] << 8) | (unsigned long) buf[7];
- *max_lba = ((unsigned long) buf[0] << 24) | ((unsigned long) buf[1] << 16) |
- ((unsigned long) buf[2] << 8) | (unsigned long) buf[3];
+ if (hdr.info & SG_INFO_CHECK) {
+ /* RCAP(10) might be unsupported by device. Force RCAP(16) */
+ hlba = MAX_10B_LBA;
+ } else {
+ blksz = sgio_get_be32(&buf[4]);
+ hlba = sgio_get_be32(buf);
+ }
/*
* If max lba masked by MAX_10B_LBA equals MAX_10B_LBA,
* then need to retry with 16 byte Read Capacity command.
*/
- if (*max_lba == MAX_10B_LBA) {
+ if (hlba == MAX_10B_LBA) {
hdr.cmd_len = 16;
hdr.cmdp[0] = 0x9e; // service action
hdr.cmdp[1] = 0x10; // Read Capacity(16)
- hdr.cmdp[10] = (unsigned char) ((sizeof(buf) >> 24) & 0xff);
- hdr.cmdp[11] = (unsigned char) ((sizeof(buf) >> 16) & 0xff);
- hdr.cmdp[12] = (unsigned char) ((sizeof(buf) >> 8) & 0xff);
- hdr.cmdp[13] = (unsigned char) (sizeof(buf) & 0xff);
+ sgio_set_be32(sizeof(buf), &hdr.cmdp[10]);
hdr.dxfer_direction = SG_DXFER_FROM_DEV;
hdr.dxferp = buf;
if (hdr.info & SG_INFO_CHECK)
td_verror(td, EIO, "fio_sgio_read_capacity");
- *bs = (buf[8] << 24) | (buf[9] << 16) | (buf[10] << 8) | buf[11];
- *max_lba = ((unsigned long long)buf[0] << 56) |
- ((unsigned long long)buf[1] << 48) |
- ((unsigned long long)buf[2] << 40) |
- ((unsigned long long)buf[3] << 32) |
- ((unsigned long long)buf[4] << 24) |
- ((unsigned long long)buf[5] << 16) |
- ((unsigned long long)buf[6] << 8) |
- (unsigned long long)buf[7];
+ blksz = sgio_get_be32(&buf[8]);
+ hlba = sgio_get_be64(buf);
+ }
+
+ if (blksz) {
+ *bs = blksz;
+ *max_lba = hlba;
+ ret = 0;
+ } else {
+ ret = EIO;
}
close(fd);
- return 0;
+ return ret;
}
static void fio_sgio_cleanup(struct thread_data *td)
free(sd->fd_flags);
free(sd->pfds);
free(sd->sgbuf);
+#ifdef FIO_SGIO_DEBUG
free(sd->trim_queue_map);
+#endif
for (i = 0; i < td->o.iodepth; i++) {
free(sd->trim_queues[i]->unmap_param);
sd->trim_queues = calloc(td->o.iodepth, sizeof(struct sgio_trim *));
sd->current_queue = -1;
+#ifdef FIO_SGIO_DEBUG
sd->trim_queue_map = calloc(td->o.iodepth, sizeof(int));
+#endif
for (i = 0; i < td->o.iodepth; i++) {
sd->trim_queues[i] = calloc(1, sizeof(struct sgio_trim));
st = sd->trim_queues[i];
snprintf(msgchunk, MAXMSGCHUNK, "SG Driver: %d bytes out of %d not transferred. ", hdr->resid, hdr->dxfer_len);
strlcat(msg, msgchunk, MAXERRDETAIL);
}
+ if (hdr->cmdp) {
+ strlcat(msg, "cdb:", MAXERRDETAIL);
+ for (i = 0; i < hdr->cmd_len; i++) {
+ snprintf(msgchunk, MAXMSGCHUNK, " %02x", hdr->cmdp[i]);
+ strlcat(msg, msgchunk, MAXERRDETAIL);
+ }
+ strlcat(msg, ". ", MAXERRDETAIL);
+ if (io_u->ddir == DDIR_TRIM) {
+ unsigned char *param_list = hdr->dxferp;
+ strlcat(msg, "dxferp:", MAXERRDETAIL);
+ for (i = 0; i < hdr->dxfer_len; i++) {
+ snprintf(msgchunk, MAXMSGCHUNK, " %02x", param_list[i]);
+ strlcat(msg, msgchunk, MAXERRDETAIL);
+ }
+ strlcat(msg, ". ", MAXERRDETAIL);
+ }
+ }
}
if (!(hdr->info & SG_INFO_CHECK) && !strlen(msg))