Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma

[linux-block.git] / io_uring / alloc_cache.h

diff --git a/io_uring/alloc_cache.h b/io_uring/alloc_cache.h
index c2cde88aeed53f6e8d31438a83c10677a70d43f3..241245cb54a6b1c53306e36a4dd45ccaa3325adb 100644 (file)
--- a/io_uring/alloc_cache.h
+++ b/io_uring/alloc_cache.h
@@ -7,47 +7,60 @@
 #define IO_ALLOC_CACHE_MAX     512
 
 struct io_cache_entry {
-       struct hlist_node       node;
+       struct io_wq_work_node node;
 };
 
 static inline bool io_alloc_cache_put(struct io_alloc_cache *cache,
                                      struct io_cache_entry *entry)
 {
-       if (cache->nr_cached < IO_ALLOC_CACHE_MAX) {
+       if (cache->nr_cached < cache->max_cached) {
                cache->nr_cached++;
-               hlist_add_head(&entry->node, &cache->list);
+               wq_stack_add_head(&entry->node, &cache->list);
+               /* KASAN poisons object */
+               kasan_slab_free_mempool(entry);
                return true;
        }
        return false;
 }
 
+static inline bool io_alloc_cache_empty(struct io_alloc_cache *cache)
+{
+       return !cache->list.next;
+}
+
 static inline struct io_cache_entry *io_alloc_cache_get(struct io_alloc_cache *cache)
 {
-       if (!hlist_empty(&cache->list)) {
-               struct hlist_node *node = cache->list.first;
+       if (cache->list.next) {
+               struct io_cache_entry *entry;
 
-               hlist_del(node);
+               entry = container_of(cache->list.next, struct io_cache_entry, node);
+               kasan_unpoison_range(entry, cache->elem_size);
+               cache->list.next = cache->list.next->next;
                cache->nr_cached--;
-               return container_of(node, struct io_cache_entry, node);
+               return entry;
        }
 
        return NULL;
 }
 
-static inline void io_alloc_cache_init(struct io_alloc_cache *cache)
+static inline void io_alloc_cache_init(struct io_alloc_cache *cache,
+                                      unsigned max_nr, size_t size)
 {
-       INIT_HLIST_HEAD(&cache->list);
+       cache->list.next = NULL;
        cache->nr_cached = 0;
+       cache->max_cached = max_nr;
+       cache->elem_size = size;
 }
 
 static inline void io_alloc_cache_free(struct io_alloc_cache *cache,
                                        void (*free)(struct io_cache_entry *))
 {
-       while (!hlist_empty(&cache->list)) {
-               struct hlist_node *node = cache->list.first;
+       while (1) {
+               struct io_cache_entry *entry = io_alloc_cache_get(cache);
 
-               hlist_del(node);
-               free(container_of(node, struct io_cache_entry, node));
+               if (!entry)
+                       break;
+               free(entry);
        }
        cache->nr_cached = 0;
 }