#include <assert.h>
#include <time.h>
#include <sys/mman.h>
+#include <sys/poll.h>
#include "fio.h"
#include "os.h"
#ifdef FIO_HAVE_SGIO
-struct sgio_data {
- struct io_u *last_io_u;
+struct sgio_cmd {
unsigned char cdb[10];
+ int nr;
+};
+
+struct sgio_data {
+ struct sgio_cmd *cmds;
+ struct io_u **events;
unsigned int bs;
};
-static inline void sgio_hdr_init(struct sgio_data *sd, struct sg_io_hdr *hdr,
- struct io_u *io_u)
+static void sgio_hdr_init(struct sgio_data *sd, struct sg_io_hdr *hdr,
+ struct io_u *io_u, int fs)
{
+ struct sgio_cmd *sc = &sd->cmds[io_u->index];
+
memset(hdr, 0, sizeof(*hdr));
- memset(sd->cdb, 0, sizeof(sd->cdb));
+ memset(sc->cdb, 0, sizeof(sc->cdb));
hdr->interface_id = 'S';
- hdr->cmdp = sd->cdb;
- hdr->cmd_len = sizeof(sd->cdb);
+ hdr->cmdp = sc->cdb;
+ hdr->cmd_len = sizeof(sc->cdb);
+ hdr->pack_id = io_u->index;
+ hdr->usr_ptr = io_u;
- if (io_u) {
+ if (fs) {
hdr->dxferp = io_u->buf;
hdr->dxfer_len = io_u->buflen;
}
}
+static int fio_sgio_getevents(struct thread_data *td, int min, int max,
+ struct timespec *t)
+{
+ struct sgio_data *sd = td->io_data;
+ struct pollfd pfd = { .fd = td->fd, .events = POLLIN };
+ void *buf = malloc(max * sizeof(struct sg_io_hdr));
+ int left = max, ret, events, i, r = 0, fl = 0;
+
+ /*
+ * don't block for !events
+ */
+ if (!min) {
+ fl = fcntl(td->fd, F_GETFL);
+ fcntl(td->fd, F_SETFL, fl | O_NONBLOCK);
+ }
+
+ while (left) {
+ do {
+ if (!min)
+ break;
+ poll(&pfd, 1, -1);
+ if (pfd.revents & POLLIN)
+ break;
+ } while (1);
+
+ ret = read(td->fd, buf, left * sizeof(struct sg_io_hdr));
+ if (ret < 0) {
+ if (errno == EAGAIN)
+ break;
+ td_verror(td, errno);
+ r = -1;
+ break;
+ } else if (!ret)
+ break;
+
+ events = ret / sizeof(struct sg_io_hdr);
+ left -= events;
+ r += events;
+
+ for (i = 0; i < events; i++) {
+ struct sg_io_hdr *hdr = (struct sg_io_hdr *) buf + i;
+
+ sd->events[i] = hdr->usr_ptr;
+ }
+ }
+
+ if (!min)
+ fcntl(td->fd, F_SETFL, fl);
+
+ free(buf);
+ return r;
+}
+
+static int fio_sgio_ioctl_doio(struct thread_data *td, struct io_u *io_u)
+{
+ struct sgio_data *sd = td->io_data;
+ struct sg_io_hdr *hdr = &io_u->hdr;
+
+ sd->events[0] = io_u;
+
+ return ioctl(td->fd, SG_IO, hdr);
+}
+
+static int fio_sgio_rw_doio(struct thread_data *td, struct io_u *io_u, int sync)
+{
+ struct sg_io_hdr *hdr = &io_u->hdr;
+ int ret;
+
+ ret = write(td->fd, hdr, sizeof(*hdr));
+ if (ret < 0)
+ return errno;
+
+ if (sync) {
+ ret = read(td->fd, hdr, sizeof(*hdr));
+ if (ret < 0)
+ return errno;
+ }
+
+ return 0;
+}
+
+static int fio_sgio_doio(struct thread_data *td, struct io_u *io_u, int sync)
+{
+ if (td->filetype == FIO_TYPE_BD)
+ return fio_sgio_ioctl_doio(td, io_u);
+
+ return fio_sgio_rw_doio(td, io_u, sync);
+}
+
static int fio_sgio_sync(struct thread_data *td)
{
struct sgio_data *sd = td->io_data;
- struct sg_io_hdr hdr;
+ struct sg_io_hdr *hdr;
+ struct io_u *io_u;
+ int ret;
+
+ io_u = __get_io_u(td);
+ if (!io_u)
+ return ENOMEM;
- sgio_hdr_init(sd, &hdr, NULL);
- hdr.dxfer_direction = SG_DXFER_NONE;
+ hdr = &io_u->hdr;
+ sgio_hdr_init(sd, hdr, io_u, 0);
+ hdr->dxfer_direction = SG_DXFER_NONE;
- hdr.cmdp[0] = 0x35;
+ hdr->cmdp[0] = 0x35;
- return ioctl(td->fd, SG_IO, &hdr);
+ ret = fio_sgio_doio(td, io_u, 1);
+ put_io_u(td, io_u);
+ return ret;
}
static int fio_sgio_prep(struct thread_data *td, struct io_u *io_u)
return EINVAL;
}
- sgio_hdr_init(sd, hdr, io_u);
+ sgio_hdr_init(sd, hdr, io_u, 1);
if (io_u->ddir == DDIR_READ) {
hdr->dxfer_direction = SG_DXFER_FROM_DEV;
static int fio_sgio_queue(struct thread_data *td, struct io_u *io_u)
{
struct sg_io_hdr *hdr = &io_u->hdr;
- struct sgio_data *sd = td->io_data;
int ret;
- ret = ioctl(td->fd, SG_IO, hdr);
+ ret = fio_sgio_doio(td, io_u, 0);
+
if (ret < 0)
io_u->error = errno;
else if (hdr->status) {
io_u->error = EIO;
}
- if (!io_u->error)
- sd->last_io_u = io_u;
-
return io_u->error;
}
{
struct sgio_data *sd = td->io_data;
- assert(event == 0);
+ return sd->events[event];
+}
- return sd->last_io_u;
+static int fio_sgio_get_bs(struct thread_data *td, unsigned int *bs)
+{
+ struct sgio_data *sd = td->io_data;
+ struct io_u *io_u;
+ struct sg_io_hdr *hdr;
+ unsigned char buf[8];
+ int ret;
+
+ io_u = __get_io_u(td);
+ assert(io_u);
+
+ hdr = &io_u->hdr;
+ sgio_hdr_init(sd, hdr, io_u, 0);
+ memset(buf, 0, sizeof(buf));
+
+ hdr->cmdp[0] = 0x25;
+ hdr->dxfer_direction = SG_DXFER_FROM_DEV;
+ hdr->dxferp = buf;
+ hdr->dxfer_len = sizeof(buf);
+
+ ret = fio_sgio_doio(td, io_u, 1);
+ if (ret) {
+ put_io_u(td, io_u);
+ return ret;
+ }
+
+ *bs = (buf[4] << 24) | (buf[5] << 16) | (buf[6] << 8) | buf[7];
+ put_io_u(td, io_u);
+ return 0;
}
int fio_sgio_init(struct thread_data *td)
{
struct sgio_data *sd;
- int bs;
+ unsigned int bs;
+ int ret;
- if (td->filetype != FIO_TYPE_BD) {
- fprintf(stderr, "ioengine sgio only works on block devices\n");
- return 1;
- }
+ sd = malloc(sizeof(*sd));
+ sd->cmds = malloc(td->iodepth * sizeof(struct sgio_cmd));
+ sd->events = malloc(td->iodepth * sizeof(struct io_u *));
+ td->io_data = sd;
- if (ioctl(td->fd, BLKSSZGET, &bs) < 0) {
- td_verror(td, errno);
+ if (td->filetype == FIO_TYPE_BD) {
+ if (ioctl(td->fd, BLKSSZGET, &bs) < 0) {
+ td_verror(td, errno);
+ return 1;
+ }
+ } else if (td->filetype == FIO_TYPE_CHAR) {
+ int version;
+
+ if (ioctl(td->fd, SG_GET_VERSION_NUM, &version) < 0) {
+ td_verror(td, errno);
+ return 1;
+ }
+
+ ret = fio_sgio_get_bs(td, &bs);
+ if (ret)
+ return ret;
+ } else {
+ fprintf(stderr, "ioengine sgio only works on block devices\n");
return 1;
}
- sd = malloc(sizeof(*sd));
sd->bs = bs;
td->io_prep = fio_sgio_prep;
td->io_queue = fio_sgio_queue;
- td->io_getevents = fio_syncio_getevents;
+
+ if (td->filetype == FIO_TYPE_BD)
+ td->io_getevents = fio_syncio_getevents;
+ else
+ td->io_getevents = fio_sgio_getevents;
+
td->io_event = fio_sgio_event;
td->io_cancel = NULL;
td->io_cleanup = fio_syncio_cleanup;
* we want to do it, regardless of whether odirect is set or not
*/
td->override_sync = 1;
-
- sd->last_io_u = NULL;
- td->io_data = sd;
return 0;
}
}
#endif /* FIO_HAVE_SGIO */
+
+#ifdef FIO_HAVE_SPLICE
+struct spliceio_data {
+ struct io_u *last_io_u;
+ int pipe[2];
+};
+
+static struct io_u *fio_spliceio_event(struct thread_data *td, int event)
+{
+ struct spliceio_data *sd = td->io_data;
+
+ assert(event == 0);
+
+ return sd->last_io_u;
+}
+
+/*
+ * For splice reading, we unfortunately cannot (yet) vmsplice the other way.
+ * So just splice the data from the file into the pipe, and use regular
+ * read to fill the buffer. Doesn't make a lot of sense, but...
+ */
+static int fio_splice_read(struct thread_data *td, struct io_u *io_u)
+{
+ struct spliceio_data *sd = td->io_data;
+ int ret, ret2, buflen;
+ void *p;
+
+ buflen = io_u->buflen;
+ p = io_u->buf;
+ while (buflen) {
+ off_t off = io_u->offset;
+
+ ret = splice(td->fd, &off, sd->pipe[1], NULL, buflen, 0);
+ if (ret < 0)
+ return errno;
+
+ buflen -= ret;
+
+ while (ret) {
+ ret2 = read(sd->pipe[0], p, ret);
+ if (ret2 < 0)
+ return errno;
+
+ ret -= ret2;
+ p += ret2;
+ }
+ }
+
+ return io_u->buflen;
+}
+
+/*
+ * For splice writing, we can vmsplice our data buffer directly into a
+ * pipe and then splice that to a file.
+ */
+static int fio_splice_write(struct thread_data *td, struct io_u *io_u)
+{
+ struct spliceio_data *sd = td->io_data;
+ struct iovec iov[1] = {
+ {
+ .iov_base = io_u->buf,
+ .iov_len = io_u->buflen,
+ }
+ };
+ struct pollfd pfd = { .fd = sd->pipe[1], .events = POLLOUT, };
+ int ret, ret2;
+
+ while (iov[0].iov_len) {
+ if (poll(&pfd, 1, -1) < 0)
+ return errno;
+
+ ret = vmsplice(sd->pipe[1], iov, 1, SPLICE_F_NONBLOCK);
+ if (ret < 0)
+ return errno;
+
+ iov[0].iov_len -= ret;
+ iov[0].iov_base += ret;
+
+ while (ret) {
+ off_t off = io_u->offset;
+
+ ret2 = splice(sd->pipe[0], NULL, td->fd, &off, ret, 0);
+ if (ret2 < 0)
+ return errno;
+
+ ret -= ret2;
+ }
+ }
+
+ return io_u->buflen;
+}
+
+static int fio_spliceio_queue(struct thread_data *td, struct io_u *io_u)
+{
+ struct spliceio_data *sd = td->io_data;
+ int ret;
+
+ if (io_u->ddir == DDIR_READ)
+ ret = fio_splice_read(td, io_u);
+ else
+ ret = fio_splice_write(td, io_u);
+
+ if ((unsigned int) ret != io_u->buflen) {
+ if (ret > 0) {
+ io_u->resid = io_u->buflen - ret;
+ io_u->error = ENODATA;
+ } else
+ io_u->error = errno;
+ }
+
+ if (!io_u->error)
+ sd->last_io_u = io_u;
+
+ return io_u->error;
+}
+
+static void fio_spliceio_cleanup(struct thread_data *td)
+{
+ struct spliceio_data *sd = td->io_data;
+
+ if (sd) {
+ close(sd->pipe[0]);
+ close(sd->pipe[1]);
+ free(sd);
+ td->io_data = NULL;
+ }
+}
+
+int fio_spliceio_init(struct thread_data *td)
+{
+ struct spliceio_data *sd = malloc(sizeof(*sd));
+
+ td->io_queue = fio_spliceio_queue;
+ td->io_getevents = fio_syncio_getevents;
+ td->io_event = fio_spliceio_event;
+ td->io_cancel = NULL;
+ td->io_cleanup = fio_spliceio_cleanup;
+ td->io_sync = fio_io_sync;
+
+ sd->last_io_u = NULL;
+ if (pipe(sd->pipe) < 0) {
+ td_verror(td, errno);
+ free(sd);
+ return 1;
+ }
+
+ td->io_data = sd;
+ return 0;
+}
+
+#else /* FIO_HAVE_SPLICE */
+
+int fio_spliceio_init(struct thread_data *td)
+{
+ return EINVAL;
+}
+
+#endif /* FIO_HAVE_SPLICE */
projects / fio.git / blobdiff