7f184275aa
This version of the gen_pool memory allocator supports lockless operation. This makes it safe to use in NMI handlers and other special unblockable contexts that could otherwise deadlock on locks. This is implemented by using atomic operations and retries on any conflicts. The disadvantage is that there may be livelocks in extreme cases. For better scalability, one gen_pool allocator can be used for each CPU. The lockless operation only works if there is enough memory available. If new memory is added to the pool a lock has to be still taken. So any user relying on locklessness has to ensure that sufficient memory is preallocated. The basic atomic operation of this allocator is cmpxchg on long. On architectures that don't have NMI-safe cmpxchg implementation, the allocator can NOT be used in NMI handler. So code uses the allocator in NMI handler should depend on CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG. Signed-off-by: Huang Ying <ying.huang@intel.com> Reviewed-by: Andi Kleen <ak@linux.intel.com> Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com> Cc: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Len Brown <len.brown@intel.com>
308 lines
11 KiB
C
308 lines
11 KiB
C
#ifndef __LINUX_BITMAP_H
|
|
#define __LINUX_BITMAP_H
|
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
#include <linux/types.h>
|
|
#include <linux/bitops.h>
|
|
#include <linux/string.h>
|
|
#include <linux/kernel.h>
|
|
|
|
/*
|
|
* bitmaps provide bit arrays that consume one or more unsigned
|
|
* longs. The bitmap interface and available operations are listed
|
|
* here, in bitmap.h
|
|
*
|
|
* Function implementations generic to all architectures are in
|
|
* lib/bitmap.c. Functions implementations that are architecture
|
|
* specific are in various include/asm-<arch>/bitops.h headers
|
|
* and other arch/<arch> specific files.
|
|
*
|
|
* See lib/bitmap.c for more details.
|
|
*/
|
|
|
|
/*
|
|
* The available bitmap operations and their rough meaning in the
|
|
* case that the bitmap is a single unsigned long are thus:
|
|
*
|
|
* Note that nbits should be always a compile time evaluable constant.
|
|
* Otherwise many inlines will generate horrible code.
|
|
*
|
|
* bitmap_zero(dst, nbits) *dst = 0UL
|
|
* bitmap_fill(dst, nbits) *dst = ~0UL
|
|
* bitmap_copy(dst, src, nbits) *dst = *src
|
|
* bitmap_and(dst, src1, src2, nbits) *dst = *src1 & *src2
|
|
* bitmap_or(dst, src1, src2, nbits) *dst = *src1 | *src2
|
|
* bitmap_xor(dst, src1, src2, nbits) *dst = *src1 ^ *src2
|
|
* bitmap_andnot(dst, src1, src2, nbits) *dst = *src1 & ~(*src2)
|
|
* bitmap_complement(dst, src, nbits) *dst = ~(*src)
|
|
* bitmap_equal(src1, src2, nbits) Are *src1 and *src2 equal?
|
|
* bitmap_intersects(src1, src2, nbits) Do *src1 and *src2 overlap?
|
|
* bitmap_subset(src1, src2, nbits) Is *src1 a subset of *src2?
|
|
* bitmap_empty(src, nbits) Are all bits zero in *src?
|
|
* bitmap_full(src, nbits) Are all bits set in *src?
|
|
* bitmap_weight(src, nbits) Hamming Weight: number set bits
|
|
* bitmap_set(dst, pos, nbits) Set specified bit area
|
|
* bitmap_clear(dst, pos, nbits) Clear specified bit area
|
|
* bitmap_find_next_zero_area(buf, len, pos, n, mask) Find bit free area
|
|
* bitmap_shift_right(dst, src, n, nbits) *dst = *src >> n
|
|
* bitmap_shift_left(dst, src, n, nbits) *dst = *src << n
|
|
* bitmap_remap(dst, src, old, new, nbits) *dst = map(old, new)(src)
|
|
* bitmap_bitremap(oldbit, old, new, nbits) newbit = map(old, new)(oldbit)
|
|
* bitmap_onto(dst, orig, relmap, nbits) *dst = orig relative to relmap
|
|
* bitmap_fold(dst, orig, sz, nbits) dst bits = orig bits mod sz
|
|
* bitmap_scnprintf(buf, len, src, nbits) Print bitmap src to buf
|
|
* bitmap_parse(buf, buflen, dst, nbits) Parse bitmap dst from kernel buf
|
|
* bitmap_parse_user(ubuf, ulen, dst, nbits) Parse bitmap dst from user buf
|
|
* bitmap_scnlistprintf(buf, len, src, nbits) Print bitmap src as list to buf
|
|
* bitmap_parselist(buf, dst, nbits) Parse bitmap dst from kernel buf
|
|
* bitmap_parselist_user(buf, dst, nbits) Parse bitmap dst from user buf
|
|
* bitmap_find_free_region(bitmap, bits, order) Find and allocate bit region
|
|
* bitmap_release_region(bitmap, pos, order) Free specified bit region
|
|
* bitmap_allocate_region(bitmap, pos, order) Allocate specified bit region
|
|
*/
|
|
|
|
/*
|
|
* Also the following operations in asm/bitops.h apply to bitmaps.
|
|
*
|
|
* set_bit(bit, addr) *addr |= bit
|
|
* clear_bit(bit, addr) *addr &= ~bit
|
|
* change_bit(bit, addr) *addr ^= bit
|
|
* test_bit(bit, addr) Is bit set in *addr?
|
|
* test_and_set_bit(bit, addr) Set bit and return old value
|
|
* test_and_clear_bit(bit, addr) Clear bit and return old value
|
|
* test_and_change_bit(bit, addr) Change bit and return old value
|
|
* find_first_zero_bit(addr, nbits) Position first zero bit in *addr
|
|
* find_first_bit(addr, nbits) Position first set bit in *addr
|
|
* find_next_zero_bit(addr, nbits, bit) Position next zero bit in *addr >= bit
|
|
* find_next_bit(addr, nbits, bit) Position next set bit in *addr >= bit
|
|
*/
|
|
|
|
/*
|
|
* The DECLARE_BITMAP(name,bits) macro, in linux/types.h, can be used
|
|
* to declare an array named 'name' of just enough unsigned longs to
|
|
* contain all bit positions from 0 to 'bits' - 1.
|
|
*/
|
|
|
|
/*
|
|
* lib/bitmap.c provides these functions:
|
|
*/
|
|
|
|
extern int __bitmap_empty(const unsigned long *bitmap, int bits);
|
|
extern int __bitmap_full(const unsigned long *bitmap, int bits);
|
|
extern int __bitmap_equal(const unsigned long *bitmap1,
|
|
const unsigned long *bitmap2, int bits);
|
|
extern void __bitmap_complement(unsigned long *dst, const unsigned long *src,
|
|
int bits);
|
|
extern void __bitmap_shift_right(unsigned long *dst,
|
|
const unsigned long *src, int shift, int bits);
|
|
extern void __bitmap_shift_left(unsigned long *dst,
|
|
const unsigned long *src, int shift, int bits);
|
|
extern int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1,
|
|
const unsigned long *bitmap2, int bits);
|
|
extern void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1,
|
|
const unsigned long *bitmap2, int bits);
|
|
extern void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1,
|
|
const unsigned long *bitmap2, int bits);
|
|
extern int __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1,
|
|
const unsigned long *bitmap2, int bits);
|
|
extern int __bitmap_intersects(const unsigned long *bitmap1,
|
|
const unsigned long *bitmap2, int bits);
|
|
extern int __bitmap_subset(const unsigned long *bitmap1,
|
|
const unsigned long *bitmap2, int bits);
|
|
extern int __bitmap_weight(const unsigned long *bitmap, int bits);
|
|
|
|
extern void bitmap_set(unsigned long *map, int i, int len);
|
|
extern void bitmap_clear(unsigned long *map, int start, int nr);
|
|
extern unsigned long bitmap_find_next_zero_area(unsigned long *map,
|
|
unsigned long size,
|
|
unsigned long start,
|
|
unsigned int nr,
|
|
unsigned long align_mask);
|
|
|
|
extern int bitmap_scnprintf(char *buf, unsigned int len,
|
|
const unsigned long *src, int nbits);
|
|
extern int __bitmap_parse(const char *buf, unsigned int buflen, int is_user,
|
|
unsigned long *dst, int nbits);
|
|
extern int bitmap_parse_user(const char __user *ubuf, unsigned int ulen,
|
|
unsigned long *dst, int nbits);
|
|
extern int bitmap_scnlistprintf(char *buf, unsigned int len,
|
|
const unsigned long *src, int nbits);
|
|
extern int bitmap_parselist(const char *buf, unsigned long *maskp,
|
|
int nmaskbits);
|
|
extern int bitmap_parselist_user(const char __user *ubuf, unsigned int ulen,
|
|
unsigned long *dst, int nbits);
|
|
extern void bitmap_remap(unsigned long *dst, const unsigned long *src,
|
|
const unsigned long *old, const unsigned long *new, int bits);
|
|
extern int bitmap_bitremap(int oldbit,
|
|
const unsigned long *old, const unsigned long *new, int bits);
|
|
extern void bitmap_onto(unsigned long *dst, const unsigned long *orig,
|
|
const unsigned long *relmap, int bits);
|
|
extern void bitmap_fold(unsigned long *dst, const unsigned long *orig,
|
|
int sz, int bits);
|
|
extern int bitmap_find_free_region(unsigned long *bitmap, int bits, int order);
|
|
extern void bitmap_release_region(unsigned long *bitmap, int pos, int order);
|
|
extern int bitmap_allocate_region(unsigned long *bitmap, int pos, int order);
|
|
extern void bitmap_copy_le(void *dst, const unsigned long *src, int nbits);
|
|
|
|
#define BITMAP_FIRST_WORD_MASK(start) (~0UL << ((start) % BITS_PER_LONG))
|
|
#define BITMAP_LAST_WORD_MASK(nbits) \
|
|
( \
|
|
((nbits) % BITS_PER_LONG) ? \
|
|
(1UL<<((nbits) % BITS_PER_LONG))-1 : ~0UL \
|
|
)
|
|
|
|
#define small_const_nbits(nbits) \
|
|
(__builtin_constant_p(nbits) && (nbits) <= BITS_PER_LONG)
|
|
|
|
static inline void bitmap_zero(unsigned long *dst, int nbits)
|
|
{
|
|
if (small_const_nbits(nbits))
|
|
*dst = 0UL;
|
|
else {
|
|
int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
|
|
memset(dst, 0, len);
|
|
}
|
|
}
|
|
|
|
static inline void bitmap_fill(unsigned long *dst, int nbits)
|
|
{
|
|
size_t nlongs = BITS_TO_LONGS(nbits);
|
|
if (!small_const_nbits(nbits)) {
|
|
int len = (nlongs - 1) * sizeof(unsigned long);
|
|
memset(dst, 0xff, len);
|
|
}
|
|
dst[nlongs - 1] = BITMAP_LAST_WORD_MASK(nbits);
|
|
}
|
|
|
|
static inline void bitmap_copy(unsigned long *dst, const unsigned long *src,
|
|
int nbits)
|
|
{
|
|
if (small_const_nbits(nbits))
|
|
*dst = *src;
|
|
else {
|
|
int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
|
|
memcpy(dst, src, len);
|
|
}
|
|
}
|
|
|
|
static inline int bitmap_and(unsigned long *dst, const unsigned long *src1,
|
|
const unsigned long *src2, int nbits)
|
|
{
|
|
if (small_const_nbits(nbits))
|
|
return (*dst = *src1 & *src2) != 0;
|
|
return __bitmap_and(dst, src1, src2, nbits);
|
|
}
|
|
|
|
static inline void bitmap_or(unsigned long *dst, const unsigned long *src1,
|
|
const unsigned long *src2, int nbits)
|
|
{
|
|
if (small_const_nbits(nbits))
|
|
*dst = *src1 | *src2;
|
|
else
|
|
__bitmap_or(dst, src1, src2, nbits);
|
|
}
|
|
|
|
static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1,
|
|
const unsigned long *src2, int nbits)
|
|
{
|
|
if (small_const_nbits(nbits))
|
|
*dst = *src1 ^ *src2;
|
|
else
|
|
__bitmap_xor(dst, src1, src2, nbits);
|
|
}
|
|
|
|
static inline int bitmap_andnot(unsigned long *dst, const unsigned long *src1,
|
|
const unsigned long *src2, int nbits)
|
|
{
|
|
if (small_const_nbits(nbits))
|
|
return (*dst = *src1 & ~(*src2)) != 0;
|
|
return __bitmap_andnot(dst, src1, src2, nbits);
|
|
}
|
|
|
|
static inline void bitmap_complement(unsigned long *dst, const unsigned long *src,
|
|
int nbits)
|
|
{
|
|
if (small_const_nbits(nbits))
|
|
*dst = ~(*src) & BITMAP_LAST_WORD_MASK(nbits);
|
|
else
|
|
__bitmap_complement(dst, src, nbits);
|
|
}
|
|
|
|
static inline int bitmap_equal(const unsigned long *src1,
|
|
const unsigned long *src2, int nbits)
|
|
{
|
|
if (small_const_nbits(nbits))
|
|
return ! ((*src1 ^ *src2) & BITMAP_LAST_WORD_MASK(nbits));
|
|
else
|
|
return __bitmap_equal(src1, src2, nbits);
|
|
}
|
|
|
|
static inline int bitmap_intersects(const unsigned long *src1,
|
|
const unsigned long *src2, int nbits)
|
|
{
|
|
if (small_const_nbits(nbits))
|
|
return ((*src1 & *src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;
|
|
else
|
|
return __bitmap_intersects(src1, src2, nbits);
|
|
}
|
|
|
|
static inline int bitmap_subset(const unsigned long *src1,
|
|
const unsigned long *src2, int nbits)
|
|
{
|
|
if (small_const_nbits(nbits))
|
|
return ! ((*src1 & ~(*src2)) & BITMAP_LAST_WORD_MASK(nbits));
|
|
else
|
|
return __bitmap_subset(src1, src2, nbits);
|
|
}
|
|
|
|
static inline int bitmap_empty(const unsigned long *src, int nbits)
|
|
{
|
|
if (small_const_nbits(nbits))
|
|
return ! (*src & BITMAP_LAST_WORD_MASK(nbits));
|
|
else
|
|
return __bitmap_empty(src, nbits);
|
|
}
|
|
|
|
static inline int bitmap_full(const unsigned long *src, int nbits)
|
|
{
|
|
if (small_const_nbits(nbits))
|
|
return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits));
|
|
else
|
|
return __bitmap_full(src, nbits);
|
|
}
|
|
|
|
static inline int bitmap_weight(const unsigned long *src, int nbits)
|
|
{
|
|
if (small_const_nbits(nbits))
|
|
return hweight_long(*src & BITMAP_LAST_WORD_MASK(nbits));
|
|
return __bitmap_weight(src, nbits);
|
|
}
|
|
|
|
static inline void bitmap_shift_right(unsigned long *dst,
|
|
const unsigned long *src, int n, int nbits)
|
|
{
|
|
if (small_const_nbits(nbits))
|
|
*dst = *src >> n;
|
|
else
|
|
__bitmap_shift_right(dst, src, n, nbits);
|
|
}
|
|
|
|
static inline void bitmap_shift_left(unsigned long *dst,
|
|
const unsigned long *src, int n, int nbits)
|
|
{
|
|
if (small_const_nbits(nbits))
|
|
*dst = (*src << n) & BITMAP_LAST_WORD_MASK(nbits);
|
|
else
|
|
__bitmap_shift_left(dst, src, n, nbits);
|
|
}
|
|
|
|
static inline int bitmap_parse(const char *buf, unsigned int buflen,
|
|
unsigned long *maskp, int nmaskbits)
|
|
{
|
|
return __bitmap_parse(buf, buflen, 0, maskp, nmaskbits);
|
|
}
|
|
|
|
#endif /* __ASSEMBLY__ */
|
|
|
|
#endif /* __LINUX_BITMAP_H */
|