lib/find_bit.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /* bit search implementation
   3  *
   4  * Copyright (C) 2004 Red Hat, Inc. All Rights Reserved.
   5  * Written by David Howells (dhowells@redhat.com)
   6  *
   7  * Copyright (C) 2008 IBM Corporation
   8  * 'find_last_bit' is written by Rusty Russell <rusty@rustcorp.com.au>
   9  * (Inspired by David Howell's find_next_bit implementation)
  10  *
  11  * Rewritten by Yury Norov <yury.norov@gmail.com> to decrease
  12  * size and improve performance, 2015.
  13  */
  14
  15 #include <linux/bitops.h>
  16 #include <linux/bitmap.h>
  17 #include <linux/export.h>
  18 #include <linux/math.h>
  19 #include <linux/minmax.h>
  20 #include <linux/swab.h>
  21
  22 /*
  23  * Common helper for find_bit() function family
  24  * @FETCH: The expression that fetches and pre-processes each word of bitmap(s)
  25  * @MUNGE: The expression that post-processes a word containing found bit (may be empty)
  26  * @size: The bitmap size in bits
  27  */
  28 #define FIND_FIRST_BIT(FETCH, MUNGE, size)                                      \
  29 ({                                                                              \
  30         unsigned long idx, val, sz = (size);                                    \
  31                                                                                 \
  32         for (idx = 0; idx * BITS_PER_LONG < sz; idx++) {                        \
  33                 val = (FETCH);                                                  \
  34                 if (val) {                                                      \
  35                         sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(val)), sz);  \
  36                         break;                                                  \
  37                 }                                                               \
  38         }                                                                       \
  39                                                                                 \
  40         sz;                                                                     \
  41 })
  42
  43 /*
  44  * Common helper for find_next_bit() function family
  45  * @FETCH: The expression that fetches and pre-processes each word of bitmap(s)
  46  * @MUNGE: The expression that post-processes a word containing found bit (may be empty)
  47  * @size: The bitmap size in bits
  48  * @start: The bitnumber to start searching at
  49  */
  50 #define FIND_NEXT_BIT(FETCH, MUNGE, size, start)                                \
  51 ({                                                                              \
  52         unsigned long mask, idx, tmp, sz = (size), __start = (start);           \
  53                                                                                 \
  54         if (unlikely(__start >= sz))                                            \
  55                 goto out;                                                       \
  56                                                                                 \
  57         mask = MUNGE(BITMAP_FIRST_WORD_MASK(__start));                          \
  58         idx = __start / BITS_PER_LONG;                                          \
  59                                                                                 \
  60         for (tmp = (FETCH) & mask; !tmp; tmp = (FETCH)) {                       \
  61                 if ((idx + 1) * BITS_PER_LONG >= sz)                            \
  62                         goto out;                                               \
  63                 idx++;                                                          \
  64         }                                                                       \
  65                                                                                 \
  66         sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(tmp)), sz);                  \
  67 out:                                                                            \
  68         sz;                                                                     \
  69 })
  70
  71 #define FIND_NTH_BIT(FETCH, size, num)                                          \
  72 ({                                                                              \
  73         unsigned long sz = (size), nr = (num), idx, w, tmp;                     \
  74                                                                                 \
  75         for (idx = 0; (idx + 1) * BITS_PER_LONG <= sz; idx++) {                 \
  76                 if (idx * BITS_PER_LONG + nr >= sz)                             \
  77                         goto out;                                               \
  78                                                                                 \
  79                 tmp = (FETCH);                                                  \
  80                 w = hweight_long(tmp);                                          \
  81                 if (w > nr)                                                     \
  82                         goto found;                                             \
  83                                                                                 \
  84                 nr -= w;                                                        \
  85         }                                                                       \
  86                                                                                 \
  87         if (sz % BITS_PER_LONG)                                                 \
  88                 tmp = (FETCH) & BITMAP_LAST_WORD_MASK(sz);                      \
  89 found:                                                                          \
  90         sz = idx * BITS_PER_LONG + fns(tmp, nr);                                \
  91 out:                                                                            \
  92         sz;                                                                     \
  93 })
  94
  95 #ifndef find_first_bit
  96 /*
  97  * Find the first set bit in a memory region.
  98  */
  99 unsigned long _find_first_bit(const unsigned long *addr, unsigned long size)
 100 {
 101         return FIND_FIRST_BIT(addr[idx], /* nop */, size);
 102 }
 103 EXPORT_SYMBOL(_find_first_bit);
 104 #endif
 105
 106 #ifndef find_first_and_bit
 107 /*
 108  * Find the first set bit in two memory regions.
 109  */
 110 unsigned long _find_first_and_bit(const unsigned long *addr1,
 111                                   const unsigned long *addr2,
 112                                   unsigned long size)
 113 {
 114         return FIND_FIRST_BIT(addr1[idx] & addr2[idx], /* nop */, size);
 115 }
 116 EXPORT_SYMBOL(_find_first_and_bit);
 117 #endif
 118
 119 /*
 120  * Find the first bit set in 1st memory region and unset in 2nd.
 121  */
 122 unsigned long _find_first_andnot_bit(const unsigned long *addr1,
 123                                   const unsigned long *addr2,
 124                                   unsigned long size)
 125 {
 126         return FIND_FIRST_BIT(addr1[idx] & ~addr2[idx], /* nop */, size);
 127 }
 128 EXPORT_SYMBOL(_find_first_andnot_bit);
 129
 130 /*
 131  * Find the first set bit in three memory regions.
 132  */
 133 unsigned long _find_first_and_and_bit(const unsigned long *addr1,
 134                                       const unsigned long *addr2,
 135                                       const unsigned long *addr3,
 136                                       unsigned long size)
 137 {
 138         return FIND_FIRST_BIT(addr1[idx] & addr2[idx] & addr3[idx], /* nop */, size);
 139 }
 140 EXPORT_SYMBOL(_find_first_and_and_bit);
 141
 142 #ifndef find_first_zero_bit
 143 /*
 144  * Find the first cleared bit in a memory region.
 145  */
 146 unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size)
 147 {
 148         return FIND_FIRST_BIT(~addr[idx], /* nop */, size);
 149 }
 150 EXPORT_SYMBOL(_find_first_zero_bit);
 151 #endif
 152
 153 #ifndef find_next_bit
 154 unsigned long _find_next_bit(const unsigned long *addr, unsigned long nbits, unsigned long start)
 155 {
 156         return FIND_NEXT_BIT(addr[idx], /* nop */, nbits, start);
 157 }
 158 EXPORT_SYMBOL(_find_next_bit);
 159 #endif
 160
 161 unsigned long __find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n)
 162 {
 163         return FIND_NTH_BIT(addr[idx], size, n);
 164 }
 165 EXPORT_SYMBOL(__find_nth_bit);
 166
 167 unsigned long __find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2,
 168                                  unsigned long size, unsigned long n)
 169 {
 170         return FIND_NTH_BIT(addr1[idx] & addr2[idx], size, n);
 171 }
 172 EXPORT_SYMBOL(__find_nth_and_bit);
 173
 174 unsigned long __find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
 175                                  unsigned long size, unsigned long n)
 176 {
 177         return FIND_NTH_BIT(addr1[idx] & ~addr2[idx], size, n);
 178 }
 179 EXPORT_SYMBOL(__find_nth_andnot_bit);
 180
 181 unsigned long __find_nth_and_andnot_bit(const unsigned long *addr1,
 182                                         const unsigned long *addr2,
 183                                         const unsigned long *addr3,
 184                                         unsigned long size, unsigned long n)
 185 {
 186         return FIND_NTH_BIT(addr1[idx] & addr2[idx] & ~addr3[idx], size, n);
 187 }
 188 EXPORT_SYMBOL(__find_nth_and_andnot_bit);
 189
 190 #ifndef find_next_and_bit
 191 unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2,
 192                                         unsigned long nbits, unsigned long start)
 193 {
 194         return FIND_NEXT_BIT(addr1[idx] & addr2[idx], /* nop */, nbits, start);
 195 }
 196 EXPORT_SYMBOL(_find_next_and_bit);
 197 #endif
 198
 199 #ifndef find_next_andnot_bit
 200 unsigned long _find_next_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
 201                                         unsigned long nbits, unsigned long start)
 202 {
 203         return FIND_NEXT_BIT(addr1[idx] & ~addr2[idx], /* nop */, nbits, start);
 204 }
 205 EXPORT_SYMBOL(_find_next_andnot_bit);
 206 #endif
 207
 208 #ifndef find_next_or_bit
 209 unsigned long _find_next_or_bit(const unsigned long *addr1, const unsigned long *addr2,
 210                                         unsigned long nbits, unsigned long start)
 211 {
 212         return FIND_NEXT_BIT(addr1[idx] | addr2[idx], /* nop */, nbits, start);
 213 }
 214 EXPORT_SYMBOL(_find_next_or_bit);
 215 #endif
 216
 217 #ifndef find_next_zero_bit
 218 unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits,
 219                                          unsigned long start)
 220 {
 221         return FIND_NEXT_BIT(~addr[idx], /* nop */, nbits, start);
 222 }
 223 EXPORT_SYMBOL(_find_next_zero_bit);
 224 #endif
 225
 226 #ifndef find_last_bit
 227 unsigned long _find_last_bit(const unsigned long *addr, unsigned long size)
 228 {
 229         if (size) {
 230                 unsigned long val = BITMAP_LAST_WORD_MASK(size);
 231                 unsigned long idx = (size-1) / BITS_PER_LONG;
 232
 233                 do {
 234                         val &= addr[idx];
 235                         if (val)
 236                                 return idx * BITS_PER_LONG + __fls(val);
 237
 238                         val = ~0ul;
 239                 } while (idx--);
 240         }
 241         return size;
 242 }
 243 EXPORT_SYMBOL(_find_last_bit);
 244 #endif
 245
 246 unsigned long find_next_clump8(unsigned long *clump, const unsigned long *addr,
 247                                unsigned long size, unsigned long offset)
 248 {
 249         offset = find_next_bit(addr, size, offset);
 250         if (offset == size)
 251                 return size;
 252
 253         offset = round_down(offset, 8);
 254         *clump = bitmap_get_value8(addr, offset);
 255
 256         return offset;
 257 }
 258 EXPORT_SYMBOL(find_next_clump8);
 259
 260 #ifdef __BIG_ENDIAN
 261
 262 #ifndef find_first_zero_bit_le
 263 /*
 264  * Find the first cleared bit in an LE memory region.
 265  */
 266 unsigned long _find_first_zero_bit_le(const unsigned long *addr, unsigned long size)
 267 {
 268         return FIND_FIRST_BIT(~addr[idx], swab, size);
 269 }
 270 EXPORT_SYMBOL(_find_first_zero_bit_le);
 271
 272 #endif
 273
 274 #ifndef find_next_zero_bit_le
 275 unsigned long _find_next_zero_bit_le(const unsigned long *addr,
 276                                         unsigned long size, unsigned long offset)
 277 {
 278         return FIND_NEXT_BIT(~addr[idx], swab, size, offset);
 279 }
 280 EXPORT_SYMBOL(_find_next_zero_bit_le);
 281 #endif
 282
 283 #ifndef find_next_bit_le
 284 unsigned long _find_next_bit_le(const unsigned long *addr,
 285                                 unsigned long size, unsigned long offset)
 286 {
 287         return FIND_NEXT_BIT(addr[idx], swab, size, offset);
 288 }
 289 EXPORT_SYMBOL(_find_next_bit_le);
 290
 291 #endif
 292
 293 #endif /* __BIG_ENDIAN */