Merge tag 'linux-kselftest-next-6.3-rc1' of git://git.kernel.org/pub/scm/linux/kernel...

[linux-block.git] / lib / sbitmap.c

diff --git a/lib/sbitmap.c b/lib/sbitmap.c
index 1fcede228fa25a9e5e72b1e1247988cfc1d45da2..eff4e42c425a4b150e13cf1aefbab66f22f6ca28 100644 (file)
--- a/lib/sbitmap.c
+++ b/lib/sbitmap.c
@@ -167,15 +167,16 @@ static int __sbitmap_get_word(unsigned long *word, unsigned long depth,
        return nr;
 }
 
-static int sbitmap_find_bit_in_index(struct sbitmap *sb, int index,
-                                    unsigned int alloc_hint)
+static int sbitmap_find_bit_in_word(struct sbitmap_word *map,
+                                   unsigned int depth,
+                                   unsigned int alloc_hint,
+                                   bool wrap)
 {
-       struct sbitmap_word *map = &sb->map[index];
        int nr;
 
        do {
-               nr = __sbitmap_get_word(&map->word, __map_depth(sb, index),
-                                       alloc_hint, !sb->round_robin);
+               nr = __sbitmap_get_word(&map->word, depth,
+                                       alloc_hint, wrap);
                if (nr != -1)
                        break;
                if (!sbitmap_deferred_clear(map))
@@ -185,25 +186,22 @@ static int sbitmap_find_bit_in_index(struct sbitmap *sb, int index,
        return nr;
 }
 
-static int __sbitmap_get(struct sbitmap *sb, unsigned int alloc_hint)
+static int sbitmap_find_bit(struct sbitmap *sb,
+                           unsigned int depth,
+                           unsigned int index,
+                           unsigned int alloc_hint,
+                           bool wrap)
 {
-       unsigned int i, index;
+       unsigned int i;
        int nr = -1;
 
-       index = SB_NR_TO_INDEX(sb, alloc_hint);
-
-       /*
-        * Unless we're doing round robin tag allocation, just use the
-        * alloc_hint to find the right word index. No point in looping
-        * twice in find_next_zero_bit() for that case.
-        */
-       if (sb->round_robin)
-               alloc_hint = SB_NR_TO_BIT(sb, alloc_hint);
-       else
-               alloc_hint = 0;
-
        for (i = 0; i < sb->map_nr; i++) {
-               nr = sbitmap_find_bit_in_index(sb, index, alloc_hint);
+               nr = sbitmap_find_bit_in_word(&sb->map[index],
+                                             min_t(unsigned int,
+                                                   __map_depth(sb, index),
+                                                   depth),
+                                             alloc_hint, wrap);
+
                if (nr != -1) {
                        nr += index << sb->shift;
                        break;
@@ -218,6 +216,26 @@ static int __sbitmap_get(struct sbitmap *sb, unsigned int alloc_hint)
        return nr;
 }
 
+static int __sbitmap_get(struct sbitmap *sb, unsigned int alloc_hint)
+{
+       unsigned int index;
+
+       index = SB_NR_TO_INDEX(sb, alloc_hint);
+
+       /*
+        * Unless we're doing round robin tag allocation, just use the
+        * alloc_hint to find the right word index. No point in looping
+        * twice in find_next_zero_bit() for that case.
+        */
+       if (sb->round_robin)
+               alloc_hint = SB_NR_TO_BIT(sb, alloc_hint);
+       else
+               alloc_hint = 0;
+
+       return sbitmap_find_bit(sb, UINT_MAX, index, alloc_hint,
+                               !sb->round_robin);
+}
+
 int sbitmap_get(struct sbitmap *sb)
 {
        int nr;
@@ -239,37 +257,12 @@ static int __sbitmap_get_shallow(struct sbitmap *sb,
                                 unsigned int alloc_hint,
                                 unsigned long shallow_depth)
 {
-       unsigned int i, index;
-       int nr = -1;
+       unsigned int index;
 
        index = SB_NR_TO_INDEX(sb, alloc_hint);
+       alloc_hint = SB_NR_TO_BIT(sb, alloc_hint);
 
-       for (i = 0; i < sb->map_nr; i++) {
-again:
-               nr = __sbitmap_get_word(&sb->map[index].word,
-                                       min_t(unsigned int,
-                                             __map_depth(sb, index),
-                                             shallow_depth),
-                                       SB_NR_TO_BIT(sb, alloc_hint), true);
-               if (nr != -1) {
-                       nr += index << sb->shift;
-                       break;
-               }
-
-               if (sbitmap_deferred_clear(&sb->map[index]))
-                       goto again;
-
-               /* Jump to next index. */
-               index++;
-               alloc_hint = index << sb->shift;
-
-               if (index >= sb->map_nr) {
-                       index = 0;
-                       alloc_hint = 0;
-               }
-       }
-
-       return nr;
+       return sbitmap_find_bit(sb, shallow_depth, index, alloc_hint, true);
 }
 
 int sbitmap_get_shallow(struct sbitmap *sb, unsigned long shallow_depth)
@@ -464,13 +457,10 @@ void sbitmap_queue_recalculate_wake_batch(struct sbitmap_queue *sbq,
                                            unsigned int users)
 {
        unsigned int wake_batch;
-       unsigned int min_batch;
        unsigned int depth = (sbq->sb.depth + users - 1) / users;
 
-       min_batch = sbq->sb.depth >= (4 * SBQ_WAIT_QUEUES) ? 4 : 1;
-
        wake_batch = clamp_val(depth / SBQ_WAIT_QUEUES,
-                       min_batch, SBQ_WAKE_BATCH);
+                       1, SBQ_WAKE_BATCH);
 
        WRITE_ONCE(sbq->wake_batch, wake_batch);
 }
@@ -521,11 +511,9 @@ unsigned long __sbitmap_queue_get_batch(struct sbitmap_queue *sbq, int nr_tags,
 
                        get_mask = ((1UL << nr_tags) - 1) << nr;
                        val = READ_ONCE(map->word);
-                       do {
-                               if ((val & ~get_mask) != val)
-                                       goto next;
-                       } while (!atomic_long_try_cmpxchg(ptr, &val,
-                                                         get_mask | val));
+                       while (!atomic_long_try_cmpxchg(ptr, &val,
+                                                         get_mask | val))
+                               ;
                        get_mask = (get_mask & ~val) >> nr;
                        if (get_mask) {
                                *offset = nr + (index << sb->shift);