diff options
Diffstat (limited to 'include/linux/slab.h')
| -rw-r--r-- | include/linux/slab.h | 251 |
1 files changed, 206 insertions, 45 deletions
diff --git a/include/linux/slab.h b/include/linux/slab.h index d5a8ab98035c..15a60b501b95 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -12,6 +12,7 @@ #ifndef _LINUX_SLAB_H #define _LINUX_SLAB_H +#include <linux/bug.h> #include <linux/cache.h> #include <linux/gfp.h> #include <linux/overflow.h> @@ -57,8 +58,9 @@ enum _slab_flag_bits { #endif _SLAB_OBJECT_POISON, _SLAB_CMPXCHG_DOUBLE, -#ifdef CONFIG_SLAB_OBJ_EXT _SLAB_NO_OBJ_EXT, +#if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT) + _SLAB_OBJ_EXT_IN_OBJ, #endif _SLAB_FLAGS_LAST_BIT }; @@ -238,10 +240,12 @@ enum _slab_flag_bits { #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */ /* Slab created using create_boot_cache */ -#ifdef CONFIG_SLAB_OBJ_EXT #define SLAB_NO_OBJ_EXT __SLAB_FLAG_BIT(_SLAB_NO_OBJ_EXT) + +#if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT) +#define SLAB_OBJ_EXT_IN_OBJ __SLAB_FLAG_BIT(_SLAB_OBJ_EXT_IN_OBJ) #else -#define SLAB_NO_OBJ_EXT __SLAB_FLAG_UNUSED +#define SLAB_OBJ_EXT_IN_OBJ __SLAB_FLAG_UNUSED #endif /* @@ -299,24 +303,26 @@ struct kmem_cache_args { unsigned int usersize; /** * @freeptr_offset: Custom offset for the free pointer - * in &SLAB_TYPESAFE_BY_RCU caches + * in caches with &SLAB_TYPESAFE_BY_RCU or @ctor * - * By default &SLAB_TYPESAFE_BY_RCU caches place the free pointer - * outside of the object. This might cause the object to grow in size. - * Cache creators that have a reason to avoid this can specify a custom - * free pointer offset in their struct where the free pointer will be - * placed. + * By default, &SLAB_TYPESAFE_BY_RCU and @ctor caches place the free + * pointer outside of the object. This might cause the object to grow + * in size. Cache creators that have a reason to avoid this can specify + * a custom free pointer offset in their data structure where the free + * pointer will be placed. * - * Note that placing the free pointer inside the object requires the - * caller to ensure that no fields are invalidated that are required to - * guard against object recycling (See &SLAB_TYPESAFE_BY_RCU for - * details). + * For caches with &SLAB_TYPESAFE_BY_RCU, the caller must ensure that + * the free pointer does not overlay fields required to guard against + * object recycling (See &SLAB_TYPESAFE_BY_RCU for details). * - * Using %0 as a value for @freeptr_offset is valid. If @freeptr_offset - * is specified, %use_freeptr_offset must be set %true. + * For caches with @ctor, the caller must ensure that the free pointer + * does not overlay fields initialized by the constructor. * - * Note that @ctor currently isn't supported with custom free pointers - * as a @ctor requires an external free pointer. + * Currently, only caches with &SLAB_TYPESAFE_BY_RCU or @ctor + * may specify @freeptr_offset. + * + * Using %0 as a value for @freeptr_offset is valid. If @freeptr_offset + * is specified, @use_freeptr_offset must be set %true. */ unsigned int freeptr_offset; /** @@ -335,6 +341,37 @@ struct kmem_cache_args { * %NULL means no constructor. */ void (*ctor)(void *); + /** + * @sheaf_capacity: Enable sheaves of given capacity for the cache. + * + * With a non-zero value, allocations from the cache go through caching + * arrays called sheaves. Each cpu has a main sheaf that's always + * present, and a spare sheaf that may be not present. When both become + * empty, there's an attempt to replace an empty sheaf with a full sheaf + * from the per-node barn. + * + * When no full sheaf is available, and gfp flags allow blocking, a + * sheaf is allocated and filled from slab(s) using bulk allocation. + * Otherwise the allocation falls back to the normal operation + * allocating a single object from a slab. + * + * Analogically when freeing and both percpu sheaves are full, the barn + * may replace it with an empty sheaf, unless it's over capacity. In + * that case a sheaf is bulk freed to slab pages. + * + * The sheaves do not enforce NUMA placement of objects, so allocations + * via kmem_cache_alloc_node() with a node specified other than + * NUMA_NO_NODE will bypass them. + * + * Bulk allocation and free operations also try to use the cpu sheaves + * and barn, but fallback to using slab pages directly. + * + * When slub_debug is enabled for the cache, the sheaf_capacity argument + * is ignored. + * + * %0 means no sheaves will be created. + */ + unsigned int sheaf_capacity; }; struct kmem_cache *__kmem_cache_create_args(const char *name, @@ -465,29 +502,21 @@ int kmem_cache_shrink(struct kmem_cache *s); /* * Common kmalloc functions provided by all allocators */ -void * __must_check krealloc_noprof(const void *objp, size_t new_size, - gfp_t flags) __realloc_size(2); -#define krealloc(...) alloc_hooks(krealloc_noprof(__VA_ARGS__)) +void * __must_check krealloc_node_align_noprof(const void *objp, size_t new_size, + unsigned long align, + gfp_t flags, int nid) __realloc_size(2); +#define krealloc_noprof(_o, _s, _f) krealloc_node_align_noprof(_o, _s, 1, _f, NUMA_NO_NODE) +#define krealloc_node_align(...) alloc_hooks(krealloc_node_align_noprof(__VA_ARGS__)) +#define krealloc_node(_o, _s, _f, _n) krealloc_node_align(_o, _s, 1, _f, _n) +#define krealloc(...) krealloc_node(__VA_ARGS__, NUMA_NO_NODE) void kfree(const void *objp); +void kfree_nolock(const void *objp); void kfree_sensitive(const void *objp); -size_t __ksize(const void *objp); DEFINE_FREE(kfree, void *, if (!IS_ERR_OR_NULL(_T)) kfree(_T)) DEFINE_FREE(kfree_sensitive, void *, if (_T) kfree_sensitive(_T)) -/** - * ksize - Report actual allocation size of associated object - * - * @objp: Pointer returned from a prior kmalloc()-family allocation. - * - * This should not be used for writing beyond the originally requested - * allocation size. Either use krealloc() or round up the allocation size - * with kmalloc_size_roundup() prior to allocation. If this is used to - * access beyond the originally requested allocation size, UBSAN_BOUNDS - * and/or FORTIFY_SOURCE may trip, since they only know about the - * originally allocated size via the __alloc_size attribute. - */ size_t ksize(const void *objp); #ifdef CONFIG_PRINTK @@ -798,6 +827,22 @@ void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t flags, int node) __assume_slab_alignment __malloc; #define kmem_cache_alloc_node(...) alloc_hooks(kmem_cache_alloc_node_noprof(__VA_ARGS__)) +struct slab_sheaf * +kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size); + +int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp, + struct slab_sheaf **sheafp, unsigned int size); + +void kmem_cache_return_sheaf(struct kmem_cache *s, gfp_t gfp, + struct slab_sheaf *sheaf); + +void *kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *cachep, gfp_t gfp, + struct slab_sheaf *sheaf) __assume_slab_alignment __malloc; +#define kmem_cache_alloc_from_sheaf(...) \ + alloc_hooks(kmem_cache_alloc_from_sheaf_noprof(__VA_ARGS__)) + +unsigned int kmem_cache_sheaf_size(struct slab_sheaf *sheaf); + /* * These macros allow declaring a kmem_buckets * parameter alongside size, which * can be compiled out with CONFIG_SLAB_BUCKETS=n so that a large number of call @@ -910,6 +955,110 @@ static __always_inline __alloc_size(1) void *kmalloc_noprof(size_t size, gfp_t f } #define kmalloc(...) alloc_hooks(kmalloc_noprof(__VA_ARGS__)) +void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node); +#define kmalloc_nolock(...) alloc_hooks(kmalloc_nolock_noprof(__VA_ARGS__)) + +/** + * __alloc_objs - Allocate objects of a given type using + * @KMALLOC: which size-based kmalloc wrapper to allocate with. + * @GFP: GFP flags for the allocation. + * @TYPE: type to allocate space for. + * @COUNT: how many @TYPE objects to allocate. + * + * Returns: Newly allocated pointer to (first) @TYPE of @COUNT-many + * allocated @TYPE objects, or NULL on failure. + */ +#define __alloc_objs(KMALLOC, GFP, TYPE, COUNT) \ +({ \ + const size_t __obj_size = size_mul(sizeof(TYPE), COUNT); \ + (TYPE *)KMALLOC(__obj_size, GFP); \ +}) + +/** + * __alloc_flex - Allocate an object that has a trailing flexible array + * @KMALLOC: kmalloc wrapper function to use for allocation. + * @GFP: GFP flags for the allocation. + * @TYPE: type of structure to allocate space for. + * @FAM: The name of the flexible array member of @TYPE structure. + * @COUNT: how many @FAM elements to allocate space for. + * + * Returns: Newly allocated pointer to @TYPE with @COUNT-many trailing + * @FAM elements, or NULL on failure or if @COUNT cannot be represented + * by the member of @TYPE that counts the @FAM elements (annotated via + * __counted_by()). + */ +#define __alloc_flex(KMALLOC, GFP, TYPE, FAM, COUNT) \ +({ \ + const size_t __count = (COUNT); \ + const size_t __obj_size = struct_size_t(TYPE, FAM, __count); \ + TYPE *__obj_ptr = KMALLOC(__obj_size, GFP); \ + if (__obj_ptr) \ + __set_flex_counter(__obj_ptr->FAM, __count); \ + __obj_ptr; \ +}) + +/** + * kmalloc_obj - Allocate a single instance of the given type + * @VAR_OR_TYPE: Variable or type to allocate. + * @GFP: GFP flags for the allocation. + * + * Returns: newly allocated pointer to a @VAR_OR_TYPE on success, or NULL + * on failure. + */ +#define kmalloc_obj(VAR_OR_TYPE, ...) \ + __alloc_objs(kmalloc, default_gfp(__VA_ARGS__), typeof(VAR_OR_TYPE), 1) + +/** + * kmalloc_objs - Allocate an array of the given type + * @VAR_OR_TYPE: Variable or type to allocate an array of. + * @COUNT: How many elements in the array. + * @GFP: GFP flags for the allocation. + * + * Returns: newly allocated pointer to array of @VAR_OR_TYPE on success, + * or NULL on failure. + */ +#define kmalloc_objs(VAR_OR_TYPE, COUNT, ...) \ + __alloc_objs(kmalloc, default_gfp(__VA_ARGS__), typeof(VAR_OR_TYPE), COUNT) + +/** + * kmalloc_flex - Allocate a single instance of the given flexible structure + * @VAR_OR_TYPE: Variable or type to allocate (with its flex array). + * @FAM: The name of the flexible array member of the structure. + * @COUNT: How many flexible array member elements are desired. + * @GFP: GFP flags for the allocation. + * + * Returns: newly allocated pointer to @VAR_OR_TYPE on success, NULL on + * failure. If @FAM has been annotated with __counted_by(), the allocation + * will immediately fail if @COUNT is larger than what the type of the + * struct's counter variable can represent. + */ +#define kmalloc_flex(VAR_OR_TYPE, FAM, COUNT, ...) \ + __alloc_flex(kmalloc, default_gfp(__VA_ARGS__), typeof(VAR_OR_TYPE), FAM, COUNT) + +/* All kzalloc aliases for kmalloc_(obj|objs|flex). */ +#define kzalloc_obj(P, ...) \ + __alloc_objs(kzalloc, default_gfp(__VA_ARGS__), typeof(P), 1) +#define kzalloc_objs(P, COUNT, ...) \ + __alloc_objs(kzalloc, default_gfp(__VA_ARGS__), typeof(P), COUNT) +#define kzalloc_flex(P, FAM, COUNT, ...) \ + __alloc_flex(kzalloc, default_gfp(__VA_ARGS__), typeof(P), FAM, COUNT) + +/* All kvmalloc aliases for kmalloc_(obj|objs|flex). */ +#define kvmalloc_obj(P, ...) \ + __alloc_objs(kvmalloc, default_gfp(__VA_ARGS__), typeof(P), 1) +#define kvmalloc_objs(P, COUNT, ...) \ + __alloc_objs(kvmalloc, default_gfp(__VA_ARGS__), typeof(P), COUNT) +#define kvmalloc_flex(P, FAM, COUNT, ...) \ + __alloc_flex(kvmalloc, default_gfp(__VA_ARGS__), typeof(P), FAM, COUNT) + +/* All kvzalloc aliases for kmalloc_(obj|objs|flex). */ +#define kvzalloc_obj(P, ...) \ + __alloc_objs(kvzalloc, default_gfp(__VA_ARGS__), typeof(P), 1) +#define kvzalloc_objs(P, COUNT, ...) \ + __alloc_objs(kvzalloc, default_gfp(__VA_ARGS__), typeof(P), COUNT) +#define kvzalloc_flex(P, FAM, COUNT, ...) \ + __alloc_flex(kvzalloc, default_gfp(__VA_ARGS__), typeof(P), FAM, COUNT) + #define kmem_buckets_alloc(_b, _size, _flags) \ alloc_hooks(__kmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags, NUMA_NO_NODE)) @@ -1041,18 +1190,20 @@ static inline __alloc_size(1) void *kzalloc_noprof(size_t size, gfp_t flags) #define kzalloc(...) alloc_hooks(kzalloc_noprof(__VA_ARGS__)) #define kzalloc_node(_size, _flags, _node) kmalloc_node(_size, (_flags)|__GFP_ZERO, _node) -void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node) __alloc_size(1); -#define kvmalloc_node_noprof(size, flags, node) \ - __kvmalloc_node_noprof(PASS_BUCKET_PARAMS(size, NULL), flags, node) -#define kvmalloc_node(...) alloc_hooks(kvmalloc_node_noprof(__VA_ARGS__)) - -#define kvmalloc(_size, _flags) kvmalloc_node(_size, _flags, NUMA_NO_NODE) -#define kvmalloc_noprof(_size, _flags) kvmalloc_node_noprof(_size, _flags, NUMA_NO_NODE) +void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), unsigned long align, + gfp_t flags, int node) __alloc_size(1); +#define kvmalloc_node_align_noprof(_size, _align, _flags, _node) \ + __kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, NULL), _align, _flags, _node) +#define kvmalloc_node_align(...) \ + alloc_hooks(kvmalloc_node_align_noprof(__VA_ARGS__)) +#define kvmalloc_node(_s, _f, _n) kvmalloc_node_align(_s, 1, _f, _n) +#define kvmalloc(...) kvmalloc_node(__VA_ARGS__, NUMA_NO_NODE) #define kvzalloc(_size, _flags) kvmalloc(_size, (_flags)|__GFP_ZERO) #define kvzalloc_node(_size, _flags, _node) kvmalloc_node(_size, (_flags)|__GFP_ZERO, _node) + #define kmem_buckets_valloc(_b, _size, _flags) \ - alloc_hooks(__kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags, NUMA_NO_NODE)) + alloc_hooks(__kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), 1, _flags, NUMA_NO_NODE)) static inline __alloc_size(1, 2) void * kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node) @@ -1062,7 +1213,7 @@ kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node) if (unlikely(check_mul_overflow(n, size, &bytes))) return NULL; - return kvmalloc_node_noprof(bytes, flags, node); + return kvmalloc_node_align_noprof(bytes, 1, flags, node); } #define kvmalloc_array_noprof(...) kvmalloc_array_node_noprof(__VA_ARGS__, NUMA_NO_NODE) @@ -1073,9 +1224,12 @@ kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node) #define kvcalloc_node(...) alloc_hooks(kvcalloc_node_noprof(__VA_ARGS__)) #define kvcalloc(...) alloc_hooks(kvcalloc_noprof(__VA_ARGS__)) -void *kvrealloc_noprof(const void *p, size_t size, gfp_t flags) - __realloc_size(2); -#define kvrealloc(...) alloc_hooks(kvrealloc_noprof(__VA_ARGS__)) +void *kvrealloc_node_align_noprof(const void *p, size_t size, unsigned long align, + gfp_t flags, int nid) __realloc_size(2); +#define kvrealloc_node_align(...) \ + alloc_hooks(kvrealloc_node_align_noprof(__VA_ARGS__)) +#define kvrealloc_node(_p, _s, _f, _n) kvrealloc_node_align(_p, _s, 1, _f, _n) +#define kvrealloc(...) kvrealloc_node(__VA_ARGS__, NUMA_NO_NODE) extern void kvfree(const void *addr); DEFINE_FREE(kvfree, void *, if (!IS_ERR_OR_NULL(_T)) kvfree(_T)) @@ -1090,10 +1244,17 @@ static inline void kvfree_rcu_barrier(void) rcu_barrier(); } +static inline void kvfree_rcu_barrier_on_cache(struct kmem_cache *s) +{ + rcu_barrier(); +} + static inline void kfree_rcu_scheduler_running(void) { } #else void kvfree_rcu_barrier(void); +void kvfree_rcu_barrier_on_cache(struct kmem_cache *s); + void kfree_rcu_scheduler_running(void); #endif |