#include <asm/unistd.h>
#include "../fio.h"
-#include "../indirect.h"
-#include "../syslet.h"
+#include "../lib/fls.h"
#ifdef FIO_HAVE_SYSLET
unsigned int nr_events;
struct syslet_ring *ring;
+ unsigned int ring_mask;
void *stack;
};
-static void fio_syslet_complete(struct thread_data *td, struct io_u *io_u)
+static void fio_syslet_add_event(struct thread_data *td, struct io_u *io_u)
{
struct syslet_data *sd = td->io_ops->data;
sd->events[sd->nr_events++] = io_u;
}
-static void syslet_complete_nr(struct thread_data *td, unsigned int nr)
+static void fio_syslet_add_events(struct thread_data *td, unsigned int nr)
{
struct syslet_data *sd = td->io_ops->data;
- unsigned int i;
+ unsigned int i, uidx;
+
+ uidx = sd->ring->user_tail;
+ read_barrier();
for (i = 0; i < nr; i++) {
- unsigned int idx = (i + sd->ring->user_tail) % td->o.iodepth;
+ unsigned int idx = (i + uidx) & sd->ring_mask;
struct syslet_completion *comp = &sd->ring->comp[idx];
struct io_u *io_u = (struct io_u *) (long) comp->caller_data;
+ long ret;
+
+ ret = comp->status;
+ if (ret <= 0) {
+ io_u->resid = io_u->xfer_buflen;
+ io_u->error = -ret;
+ } else {
+ io_u->resid = io_u->xfer_buflen - ret;
+ io_u->error = 0;
+ }
- io_u->resid = io_u->xfer_buflen - comp->status;
- fio_syslet_complete(td, io_u);
+ fio_syslet_add_event(td, io_u);
}
}
-
static void fio_syslet_wait_for_events(struct thread_data *td)
{
struct syslet_data *sd = td->io_ops->data;
struct syslet_ring *ring = sd->ring;
- unsigned int events;
- events = 0;
do {
unsigned int kh = ring->kernel_head;
int ret;
if (ring->user_tail != kh) {
unsigned int nr = kh - ring->user_tail;
- syslet_complete_nr(td, nr);
- events += nr;
+ fio_syslet_add_events(td, nr);
ring->user_tail = kh;
- continue;
+ break;
}
/*
*/
ret = syscall(__NR_syslet_ring_wait, ring, ring->user_tail);
assert(!ret);
- } while (!events);
+ } while (1);
}
-static int fio_syslet_getevents(struct thread_data *td, int min,
- int fio_unused max,
+static int fio_syslet_getevents(struct thread_data *td, unsigned int min,
+ unsigned int fio_unused max,
struct timespec fio_unused *t)
{
struct syslet_data *sd = td->io_ops->data;
long ret;
- do {
- /*
- * do we have enough immediate completions?
- */
- if (sd->nr_events >= (unsigned int) min)
- break;
-
+ /*
+ * While we have less events than requested, block waiting for them
+ * (if we have to, there may already be more completed events ready
+ * for us - see fio_syslet_wait_for_events()
+ */
+ while (sd->nr_events < min)
fio_syslet_wait_for_events(td);
- } while (1);
ret = sd->nr_events;
sd->nr_events = 0;
if (sd) {
free(sd->events);
free(sd->ring);
- free(sd->stack);
free(sd);
- td->io_ops->data = NULL;
}
}
static int fio_syslet_init(struct thread_data *td)
{
struct syslet_data *sd;
- void *ring, *stack;
+ void *ring = NULL, *stack = NULL;
+ unsigned int ring_size, ring_nr;
sd = malloc(sizeof(*sd));
memset(sd, 0, sizeof(*sd));
sd->events = malloc(sizeof(struct io_u *) * td->o.iodepth);
memset(sd->events, 0, sizeof(struct io_u *) * td->o.iodepth);
- if (posix_memalign(&ring, sizeof(uint64_t), sizeof(struct syslet_ring)))
- return 1;
+
+ /*
+ * The ring needs to be a power-of-2, so round it up if we have to
+ */
+ ring_nr = td->o.iodepth;
+ if (ring_nr & (ring_nr - 1))
+ ring_nr = 1 << __fls(ring_nr);
+
+ ring_size = sizeof(struct syslet_ring) +
+ ring_nr * sizeof(struct syslet_completion);
+ if (posix_memalign(&ring, sizeof(uint64_t), ring_size))
+ goto err_mem;
if (posix_memalign(&stack, page_size, page_size))
- return 1;
+ goto err_mem;
sd->ring = ring;
+ sd->ring_mask = ring_nr - 1;
sd->stack = stack;
- memset(sd->ring, 0, sizeof(*sd->ring));
- sd->ring->elements = td->o.iodepth;
+ memset(sd->ring, 0, ring_size);
+ sd->ring->elements = ring_nr;
- if (check_syslet_support(sd)) {
- log_err("fio: syslets do not appear to work\n");
- free(sd->events);
- free(sd->ring);
- free(sd->stack);
- free(sd);
- return 1;
+ if (!check_syslet_support(sd)) {
+ td->io_ops->data = sd;
+ return 0;
}
- td->io_ops->data = sd;
- return 0;
+ log_err("fio: syslets do not appear to work\n");
+err_mem:
+ free(sd->events);
+ if (ring)
+ free(ring);
+ if (stack)
+ free(stack);
+ free(sd);
+ return 1;
}
static struct ioengine_ops ioengine = {
.cleanup = fio_syslet_cleanup,
.open_file = generic_open_file,
.close_file = generic_close_file,
+ .get_file_size = generic_get_file_size,
};
#else /* FIO_HAVE_SYSLET */
projects / fio.git / blobdiff