Improve async verify offload efficiency

Signed-off-by: Jens Axboe <jens.axboe@oracle.com>

diff --git a/flist.h b/flist.h
index e62f4b295e7f843dd565ec426328fefb7470ff5d..95023c75aa1799732ea8f2ea99e64c3df4027644 100644 (file)
--- a/flist.h
+++ b/flist.h
@@ -117,6 +117,36 @@ static inline int flist_empty(const struct flist_head *head)
        return head->next == head;
 }
 
+static inline void __flist_splice(const struct flist_head *list,
+                                 struct flist_head *prev,
+                                 struct flist_head *next)
+{
+       struct flist_head *first = list->next;
+       struct flist_head *last = list->prev;
+
+       first->prev = prev;
+       prev->next = first;
+
+       last->next = next;
+       next->prev = last;
+}
+
+static inline void flist_splice(const struct flist_head *list,
+                               struct flist_head *head)
+{
+       if (!flist_empty(list))
+               __flist_splice(list, head, head->next);
+}
+
+static inline void flist_splice_init(struct flist_head *list,
+                                   struct flist_head *head)
+{
+       if (!flist_empty(list)) {
+               __flist_splice(list, head, head->next);
+               INIT_FLIST_HEAD(list);
+       }
+}
+
 /**
  * flist_entry - get the struct for this entry
  * @ptr:       the &struct flist_head pointer.

diff --git a/verify.c b/verify.c
index 5dd9ee339b3858ebe630c67ad4c11e80248b683f..978e77b836683c81f37061f8656a57b0193c0928 100644 (file)
--- a/verify.c
+++ b/verify.c
@@ -756,6 +756,8 @@ static void *verify_async_thread(void *data)
        }
 
        do {
+               FLIST_HEAD(list);
+
                read_barrier();
                if (td->verify_thread_exit)
                        break;
@@ -771,17 +773,19 @@ static void *verify_async_thread(void *data)
                        }
                }
 
-               if (flist_empty(&td->verify_list)) {
-                       pthread_mutex_unlock(&td->io_u_lock);
+               flist_splice_init(&td->verify_list, &list);
+               pthread_mutex_unlock(&td->io_u_lock);
+
+               if (flist_empty(&list))
                        continue;
-               }
 
-               io_u = flist_entry(td->verify_list.next, struct io_u, list);
-               flist_del_init(&io_u->list);
-               pthread_mutex_unlock(&td->io_u_lock);
+               while (!flist_empty(&list)) {
+                       io_u = flist_entry(list.next, struct io_u, list);
+                       flist_del_init(&io_u->list);
 
-               ret = verify_io_u(td, io_u);
-               put_io_u(td, io_u);
+                       ret |= verify_io_u(td, io_u);
+                       put_io_u(td, io_u);
+               }
        } while (!ret);
 
 done: