diff options
| -rw-r--r-- | include/linux/kmalloc_sizes.h | 45 | ||||
| -rw-r--r-- | include/linux/slab_def.h | 47 | ||||
| -rw-r--r-- | mm/slab.c | 169 |
3 files changed, 88 insertions, 173 deletions
diff --git a/include/linux/kmalloc_sizes.h b/include/linux/kmalloc_sizes.h deleted file mode 100644 index e576b848ce10..000000000000 --- a/include/linux/kmalloc_sizes.h +++ /dev/null @@ -1,45 +0,0 @@ -#if (PAGE_SIZE == 4096) - CACHE(32) -#endif - CACHE(64) -#if L1_CACHE_BYTES < 64 - CACHE(96) -#endif - CACHE(128) -#if L1_CACHE_BYTES < 128 - CACHE(192) -#endif - CACHE(256) - CACHE(512) - CACHE(1024) - CACHE(2048) - CACHE(4096) - CACHE(8192) - CACHE(16384) - CACHE(32768) - CACHE(65536) - CACHE(131072) -#if KMALLOC_MAX_SIZE >= 262144 - CACHE(262144) -#endif -#if KMALLOC_MAX_SIZE >= 524288 - CACHE(524288) -#endif -#if KMALLOC_MAX_SIZE >= 1048576 - CACHE(1048576) -#endif -#if KMALLOC_MAX_SIZE >= 2097152 - CACHE(2097152) -#endif -#if KMALLOC_MAX_SIZE >= 4194304 - CACHE(4194304) -#endif -#if KMALLOC_MAX_SIZE >= 8388608 - CACHE(8388608) -#endif -#if KMALLOC_MAX_SIZE >= 16777216 - CACHE(16777216) -#endif -#if KMALLOC_MAX_SIZE >= 33554432 - CACHE(33554432) -#endif diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h index 8bb6e0eaf3c6..e0f30ef9525d 100644 --- a/include/linux/slab_def.h +++ b/include/linux/slab_def.h @@ -11,8 +11,6 @@ */ #include <linux/init.h> -#include <asm/page.h> /* kmalloc_sizes.h needs PAGE_SIZE */ -#include <asm/cache.h> /* kmalloc_sizes.h needs L1_CACHE_BYTES */ #include <linux/compiler.h> /* @@ -104,15 +102,8 @@ struct kmem_cache { */ }; -/* Size description struct for general caches. */ -struct cache_sizes { - size_t cs_size; - struct kmem_cache *cs_cachep; -#ifdef CONFIG_ZONE_DMA - struct kmem_cache *cs_dmacachep; -#endif -}; -extern struct cache_sizes malloc_sizes[]; +extern struct kmem_cache *kmalloc_caches[PAGE_SHIFT + MAX_ORDER]; +extern struct kmem_cache *kmalloc_dma_caches[PAGE_SHIFT + MAX_ORDER]; void *kmem_cache_alloc(struct kmem_cache *, gfp_t); void *__kmalloc(size_t size, gfp_t flags); @@ -133,26 +124,19 @@ static __always_inline void *kmalloc(size_t size, gfp_t flags) void *ret; if (__builtin_constant_p(size)) { - int i = 0; + int i; if (!size) return ZERO_SIZE_PTR; -#define CACHE(x) \ - if (size <= x) \ - goto found; \ - else \ - i++; -#include <linux/kmalloc_sizes.h> -#undef CACHE - return NULL; -found: + i = kmalloc_index(size); + #ifdef CONFIG_ZONE_DMA if (flags & GFP_DMA) - cachep = malloc_sizes[i].cs_dmacachep; + cachep = kmalloc_dma_caches[i]; else #endif - cachep = malloc_sizes[i].cs_cachep; + cachep = kmalloc_caches[i]; ret = kmem_cache_alloc_trace(cachep, flags, size); @@ -186,26 +170,19 @@ static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) struct kmem_cache *cachep; if (__builtin_constant_p(size)) { - int i = 0; + int i; if (!size) return ZERO_SIZE_PTR; -#define CACHE(x) \ - if (size <= x) \ - goto found; \ - else \ - i++; -#include <linux/kmalloc_sizes.h> -#undef CACHE - return NULL; -found: + i = kmalloc_index(size); + #ifdef CONFIG_ZONE_DMA if (flags & GFP_DMA) - cachep = malloc_sizes[i].cs_dmacachep; + cachep = kmalloc_dma_caches[i]; else #endif - cachep = malloc_sizes[i].cs_cachep; + cachep = kmalloc_caches[i]; return kmem_cache_alloc_node_trace(cachep, flags, node, size); } diff --git a/mm/slab.c b/mm/slab.c index e7667a3584bc..2a7132ec4ff6 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -318,34 +318,18 @@ static void free_block(struct kmem_cache *cachep, void **objpp, int len, static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp); static void cache_reap(struct work_struct *unused); -/* - * This function must be completely optimized away if a constant is passed to - * it. Mostly the same as what is in linux/slab.h except it returns an index. - */ -static __always_inline int index_of(const size_t size) -{ - extern void __bad_size(void); - - if (__builtin_constant_p(size)) { - int i = 0; +struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1]; +EXPORT_SYMBOL(kmalloc_caches); -#define CACHE(x) \ - if (size <=x) \ - return i; \ - else \ - i++; -#include <linux/kmalloc_sizes.h> -#undef CACHE - __bad_size(); - } else - __bad_size(); - return 0; -} +#ifdef CONFIG_ZONE_DMA +struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1]; +EXPORT_SYMBOL(kmalloc_dma_caches); +#endif static int slab_early_init = 1; -#define INDEX_AC index_of(sizeof(struct arraycache_init)) -#define INDEX_L3 index_of(sizeof(struct kmem_list3)) +#define INDEX_AC kmalloc_index(sizeof(struct arraycache_init)) +#define INDEX_L3 kmalloc_index(sizeof(struct kmem_list3)) static void kmem_list3_init(struct kmem_list3 *parent) { @@ -524,30 +508,6 @@ static inline unsigned int obj_to_index(const struct kmem_cache *cache, return reciprocal_divide(offset, cache->reciprocal_buffer_size); } -/* - * These are the default caches for kmalloc. Custom caches can have other sizes. - */ -struct cache_sizes malloc_sizes[] = { -#define CACHE(x) { .cs_size = (x) }, -#include <linux/kmalloc_sizes.h> - CACHE(ULONG_MAX) -#undef CACHE -}; -EXPORT_SYMBOL(malloc_sizes); - -/* Must match cache_sizes above. Out of line to keep cache footprint low. */ -struct cache_names { - char *name; - char *name_dma; -}; - -static struct cache_names __initdata cache_names[] = { -#define CACHE(x) { .name = "size-" #x, .name_dma = "size-" #x "(DMA)" }, -#include <linux/kmalloc_sizes.h> - {NULL,} -#undef CACHE -}; - static struct arraycache_init initarray_generic = { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} }; @@ -625,19 +585,23 @@ static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep) static void init_node_lock_keys(int q) { - struct cache_sizes *s = malloc_sizes; + int i; if (slab_state < UP) return; - for (s = malloc_sizes; s->cs_size != ULONG_MAX; s++) { + for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) { struct kmem_list3 *l3; + struct kmem_cache *cache = kmalloc_caches[i]; + + if (!cache) + continue; - l3 = s->cs_cachep->nodelists[q]; - if (!l3 || OFF_SLAB(s->cs_cachep)) + l3 = cache->nodelists[q]; + if (!l3 || OFF_SLAB(cache)) continue; - slab_set_lock_classes(s->cs_cachep, &on_slab_l3_key, + slab_set_lock_classes(cache, &on_slab_l3_key, &on_slab_alc_key, q); } } @@ -705,20 +669,19 @@ static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep) static inline struct kmem_cache *__find_general_cachep(size_t size, gfp_t gfpflags) { - struct cache_sizes *csizep = malloc_sizes; + int i; #if DEBUG /* This happens if someone tries to call * kmem_cache_create(), or __kmalloc(), before * the generic caches are initialized. */ - BUG_ON(malloc_sizes[INDEX_AC].cs_cachep == NULL); + BUG_ON(kmalloc_caches[INDEX_AC] == NULL); #endif if (!size) return ZERO_SIZE_PTR; - while (size > csizep->cs_size) - csizep++; + i = kmalloc_index(size); /* * Really subtle: The last entry with cs->cs_size==ULONG_MAX @@ -727,9 +690,9 @@ static inline struct kmem_cache *__find_general_cachep(size_t size, */ #ifdef CONFIG_ZONE_DMA if (unlikely(gfpflags & GFP_DMA)) - return csizep->cs_dmacachep; + return kmalloc_dma_caches[i]; #endif - return csizep->cs_cachep; + return kmalloc_caches[i]; } static struct kmem_cache *kmem_find_general_cachep(size_t size, gfp_t gfpflags) @@ -1602,8 +1565,6 @@ static void setup_nodelists_pointer(struct kmem_cache *cachep) */ void __init kmem_cache_init(void) { - struct cache_sizes *sizes; - struct cache_names *names; int i; kmem_cache = &kmem_cache_boot; @@ -1657,8 +1618,6 @@ void __init kmem_cache_init(void) list_add(&kmem_cache->list, &slab_caches); /* 2+3) create the kmalloc caches */ - sizes = malloc_sizes; - names = cache_names; /* * Initialize the caches that provide memory for the array cache and the @@ -1666,35 +1625,39 @@ void __init kmem_cache_init(void) * bug. */ - sizes[INDEX_AC].cs_cachep = create_kmalloc_cache(names[INDEX_AC].name, - sizes[INDEX_AC].cs_size, ARCH_KMALLOC_FLAGS); + kmalloc_caches[INDEX_AC] = create_kmalloc_cache("kmalloc-ac", + kmalloc_size(INDEX_AC), ARCH_KMALLOC_FLAGS); if (INDEX_AC != INDEX_L3) - sizes[INDEX_L3].cs_cachep = - create_kmalloc_cache(names[INDEX_L3].name, - sizes[INDEX_L3].cs_size, ARCH_KMALLOC_FLAGS); + kmalloc_caches[INDEX_L3] = + create_kmalloc_cache("kmalloc-l3", + kmalloc_size(INDEX_L3), ARCH_KMALLOC_FLAGS); slab_early_init = 0; - while (sizes->cs_size != ULONG_MAX) { - /* - * For performance, all the general caches are L1 aligned. - * This should be particularly beneficial on SMP boxes, as it - * eliminates "false sharing". - * Note for systems short on memory removing the alignment will - * allow tighter packing of the smaller caches. - */ - if (!sizes->cs_cachep) - sizes->cs_cachep = create_kmalloc_cache(names->name, - sizes->cs_size, ARCH_KMALLOC_FLAGS); + for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) { + size_t cs_size = kmalloc_size(i); + + if (cs_size < KMALLOC_MIN_SIZE) + continue; + + if (!kmalloc_caches[i]) { + /* + * For performance, all the general caches are L1 aligned. + * This should be particularly beneficial on SMP boxes, as it + * eliminates "false sharing". + * Note for systems short on memory removing the alignment will + * allow tighter packing of the smaller caches. + */ + kmalloc_caches[i] = create_kmalloc_cache("kmalloc", + cs_size, ARCH_KMALLOC_FLAGS); + } #ifdef CONFIG_ZONE_DMA - sizes->cs_dmacachep = create_kmalloc_cache( - names->name_dma, sizes->cs_size, + kmalloc_dma_caches[i] = create_kmalloc_cache( + "kmalloc-dma", cs_size, SLAB_CACHE_DMA|ARCH_KMALLOC_FLAGS); #endif - sizes++; - names++; } /* 4) Replace the bootstrap head arrays */ { @@ -1713,17 +1676,16 @@ void __init kmem_cache_init(void) ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT); - BUG_ON(cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep) + BUG_ON(cpu_cache_get(kmalloc_caches[INDEX_AC]) != &initarray_generic.cache); - memcpy(ptr, cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep), + memcpy(ptr, cpu_cache_get(kmalloc_caches[INDEX_AC]), sizeof(struct arraycache_init)); /* * Do not assume that spinlocks can be initialized via memcpy: */ spin_lock_init(&ptr->lock); - malloc_sizes[INDEX_AC].cs_cachep->array[smp_processor_id()] = - ptr; + kmalloc_caches[INDEX_AC]->array[smp_processor_id()] = ptr; } /* 5) Replace the bootstrap kmem_list3's */ { @@ -1732,17 +1694,39 @@ void __init kmem_cache_init(void) for_each_online_node(nid) { init_list(kmem_cache, &initkmem_list3[CACHE_CACHE + nid], nid); - init_list(malloc_sizes[INDEX_AC].cs_cachep, + init_list(kmalloc_caches[INDEX_AC], &initkmem_list3[SIZE_AC + nid], nid); if (INDEX_AC != INDEX_L3) { - init_list(malloc_sizes[INDEX_L3].cs_cachep, + init_list(kmalloc_caches[INDEX_L3], &initkmem_list3[SIZE_L3 + nid], nid); } } } slab_state = UP; + + /* Create the proper names */ + for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) { + char *s; + struct kmem_cache *c = kmalloc_caches[i]; + + if (!c) + continue; + + s = kasprintf(GFP_NOWAIT, "kmalloc-%d", kmalloc_size(i)); + + BUG_ON(!s); + c->name = s; + +#ifdef CONFIG_ZONE_DMA + c = kmalloc_dma_caches[i]; + BUG_ON(!c); + s = kasprintf(GFP_NOWAIT, "dma-kmalloc-%d", kmalloc_size(i)); + BUG_ON(!s); + c->name = s; +#endif + } } void __init kmem_cache_init_late(void) @@ -2428,10 +2412,9 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags) size += BYTES_PER_WORD; } #if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC) - if (size >= malloc_sizes[INDEX_L3 + 1].cs_size - && cachep->object_size > cache_line_size() - && ALIGN(size, cachep->align) < PAGE_SIZE) { - cachep->obj_offset += PAGE_SIZE - ALIGN(size, cachep->align); + if (size >= kmalloc_size(INDEX_L3 + 1) + && cachep->object_size > cache_line_size() && ALIGN(size, align) < PAGE_SIZE) { + cachep->obj_offset += PAGE_SIZE - ALIGN(size, align); size = PAGE_SIZE; } #endif |