lib: add find_nth{,_and,_andnot}_bit()

Kernel lacks for a function that searches for Nth bit in a bitmap.
Usually people do it like this:
	for_each_set_bit(bit, mask, size)
		if (n-- == 0)
			return bit;

We can do it more efficiently, if we:
1. find a word containing Nth bit, using hweight(); and
2. find the bit, using a helper fns(), that works similarly to
   __ffs() and ffz().

fns() is implemented as a simple loop. For x86_64, there's PDEP instruction
to do that: ret = clz(pdep(1 << idx, num)). However, for large bitmaps the
most of improvement comes from using hweight(), so I kept fns() simple.

New find_nth_bit() is ~70 times faster on x86_64/kvm in find_bit benchmark:
find_nth_bit:                  7154190 ns,  16411 iterations
for_each_bit:                505493126 ns,  16315 iterations

With all that, a family of 3 new functions is added, and used where
appropriate in the following patches.

Signed-off-by: Yury Norov <yury.norov@gmail.com>

diff --git a/include/linux/bitops.h b/include/linux/bitops.h
index 3b89c64bcfd8f029593f9fa2da7e6366467868f6..d7dd83fafebaa499bf39613775a16d0063341a1e 100644 (file)
--- a/include/linux/bitops.h
+++ b/include/linux/bitops.h
@@ -247,6 +247,25 @@ static inline unsigned long __ffs64(u64 word)
        return __ffs((unsigned long)word);
 }
 
+/**
+ * fns - find N'th set bit in a word
+ * @word: The word to search
+ * @n: Bit to find
+ */
+static inline unsigned long fns(unsigned long word, unsigned int n)
+{
+       unsigned int bit;
+
+       while (word) {
+               bit = __ffs(word);
+               if (n-- == 0)
+                       return bit;
+               __clear_bit(bit, &word);
+       }
+
+       return BITS_PER_LONG;
+}
+
 /**
  * assign_bit - Assign value to a bit in memory
  * @nr: the bit to set

diff --git a/include/linux/find.h b/include/linux/find.h
index dead6f53a97bcb9040a719897f8e3210e7bab07f..b100944daba0661bd77e7601b119956e6650e0d0 100644 (file)
--- a/include/linux/find.h
+++ b/include/linux/find.h
@@ -15,6 +15,11 @@ unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long
 unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits,
                                         unsigned long start);
 extern unsigned long _find_first_bit(const unsigned long *addr, unsigned long size);
+unsigned long __find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n);
+unsigned long __find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2,
+                               unsigned long size, unsigned long n);
+unsigned long __find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
+                                       unsigned long size, unsigned long n);
 extern unsigned long _find_first_and_bit(const unsigned long *addr1,
                                         const unsigned long *addr2, unsigned long size);
 extern unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size);
@@ -136,6 +141,87 @@ unsigned long find_first_bit(const unsigned long *addr, unsigned long size)
 }
 #endif
 
+/**
+ * find_nth_bit - find N'th set bit in a memory region
+ * @addr: The address to start the search at
+ * @size: The maximum number of bits to search
+ * @n: The number of set bit, which position is needed, counting from 0
+ *
+ * The following is semantically equivalent:
+ *      idx = find_nth_bit(addr, size, 0);
+ *      idx = find_first_bit(addr, size);
+ *
+ * Returns the bit number of the N'th set bit.
+ * If no such, returns @size.
+ */
+static inline
+unsigned long find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n)
+{
+       if (n >= size)
+               return size;
+
+       if (small_const_nbits(size)) {
+               unsigned long val =  *addr & GENMASK(size - 1, 0);
+
+               return val ? fns(val, n) : size;
+       }
+
+       return __find_nth_bit(addr, size, n);
+}
+
+/**
+ * find_nth_and_bit - find N'th set bit in 2 memory regions
+ * @addr1: The 1st address to start the search at
+ * @addr2: The 2nd address to start the search at
+ * @size: The maximum number of bits to search
+ * @n: The number of set bit, which position is needed, counting from 0
+ *
+ * Returns the bit number of the N'th set bit.
+ * If no such, returns @size.
+ */
+static inline
+unsigned long find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2,
+                               unsigned long size, unsigned long n)
+{
+       if (n >= size)
+               return size;
+
+       if (small_const_nbits(size)) {
+               unsigned long val =  *addr1 & *addr2 & GENMASK(size - 1, 0);
+
+               return val ? fns(val, n) : size;
+       }
+
+       return __find_nth_and_bit(addr1, addr2, size, n);
+}
+
+/**
+ * find_nth_andnot_bit - find N'th set bit in 2 memory regions,
+ *                      flipping bits in 2nd region
+ * @addr1: The 1st address to start the search at
+ * @addr2: The 2nd address to start the search at
+ * @size: The maximum number of bits to search
+ * @n: The number of set bit, which position is needed, counting from 0
+ *
+ * Returns the bit number of the N'th set bit.
+ * If no such, returns @size.
+ */
+static inline
+unsigned long find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
+                               unsigned long size, unsigned long n)
+{
+       if (n >= size)
+               return size;
+
+       if (small_const_nbits(size)) {
+               unsigned long val =  *addr1 & (~*addr2) & GENMASK(size - 1, 0);
+
+               return val ? fns(val, n) : size;
+       }
+
+       return __find_nth_andnot_bit(addr1, addr2, size, n);
+}
+
 #ifndef find_first_and_bit
 /**
  * find_first_and_bit - find the first set bit in both memory regions

diff --git a/lib/find_bit.c b/lib/find_bit.c
index d00ee23ab65736a1b33da458e95d0b220bbeca17..25609974cbe477151fc6981a8e7cc0fb9724c6fe 100644 (file)
--- a/lib/find_bit.c
+++ b/lib/find_bit.c
@@ -68,6 +68,30 @@ out:                                                                         \
        sz;                                                                     \
 })
 
+#define FIND_NTH_BIT(FETCH, size, num)                                         \
+({                                                                             \
+       unsigned long sz = (size), nr = (num), idx, w, tmp;                     \
+                                                                               \
+       for (idx = 0; (idx + 1) * BITS_PER_LONG <= sz; idx++) {                 \
+               if (idx * BITS_PER_LONG + nr >= sz)                             \
+                       goto out;                                               \
+                                                                               \
+               tmp = (FETCH);                                                  \
+               w = hweight_long(tmp);                                          \
+               if (w > nr)                                                     \
+                       goto found;                                             \
+                                                                               \
+               nr -= w;                                                        \
+       }                                                                       \
+                                                                               \
+       if (sz % BITS_PER_LONG)                                                 \
+               tmp = (FETCH) & BITMAP_LAST_WORD_MASK(sz);                      \
+found:                                                                         \
+       sz = min(idx * BITS_PER_LONG + fns(tmp, nr), sz);                       \
+out:                                                                           \
+       sz;                                                                     \
+})
+
 #ifndef find_first_bit
 /*
  * Find the first set bit in a memory region.
@@ -111,6 +135,26 @@ unsigned long _find_next_bit(const unsigned long *addr, unsigned long nbits, uns
 EXPORT_SYMBOL(_find_next_bit);
 #endif
 
+unsigned long __find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n)
+{
+       return FIND_NTH_BIT(addr[idx], size, n);
+}
+EXPORT_SYMBOL(__find_nth_bit);
+
+unsigned long __find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2,
+                                unsigned long size, unsigned long n)
+{
+       return FIND_NTH_BIT(addr1[idx] & addr2[idx], size, n);
+}
+EXPORT_SYMBOL(__find_nth_and_bit);
+
+unsigned long __find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
+                                unsigned long size, unsigned long n)
+{
+       return FIND_NTH_BIT(addr1[idx] & ~addr2[idx], size, n);
+}
+EXPORT_SYMBOL(__find_nth_andnot_bit);
+
 #ifndef find_next_and_bit
 unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2,
                                        unsigned long nbits, unsigned long start)