diff options
Diffstat (limited to 'drivers/staging/zcache/zcache-main.c')
-rw-r--r-- | drivers/staging/zcache/zcache-main.c | 178 |
1 files changed, 86 insertions, 92 deletions
diff --git a/drivers/staging/zcache/zcache-main.c b/drivers/staging/zcache/zcache-main.c index 784c796b9848..c214977b4ab4 100644 --- a/drivers/staging/zcache/zcache-main.c +++ b/drivers/staging/zcache/zcache-main.c @@ -31,13 +31,11 @@ #include <linux/math64.h> #include <linux/crypto.h> #include <linux/string.h> +#include <linux/idr.h> #include "tmem.h" #include "../zsmalloc/zsmalloc.h" -#if (!defined(CONFIG_CLEANCACHE) && !defined(CONFIG_FRONTSWAP)) -#error "zcache is useless without CONFIG_CLEANCACHE or CONFIG_FRONTSWAP" -#endif #ifdef CONFIG_CLEANCACHE #include <linux/cleancache.h> #endif @@ -53,15 +51,13 @@ (__GFP_FS | __GFP_NORETRY | __GFP_NOWARN | __GFP_NOMEMALLOC) #endif -#define MAX_POOLS_PER_CLIENT 16 - #define MAX_CLIENTS 16 #define LOCAL_CLIENT ((uint16_t)-1) MODULE_LICENSE("GPL"); struct zcache_client { - struct tmem_pool *tmem_pools[MAX_POOLS_PER_CLIENT]; + struct idr tmem_pools; struct zs_pool *zspool; bool allocated; atomic_t refcount; @@ -78,6 +74,17 @@ static inline uint16_t get_client_id_from_client(struct zcache_client *cli) return cli - &zcache_clients[0]; } +static struct zcache_client *get_zcache_client(uint16_t cli_id) +{ + if (cli_id == LOCAL_CLIENT) + return &zcache_host; + + if ((unsigned int)cli_id < MAX_CLIENTS) + return &zcache_clients[cli_id]; + + return NULL; +} + static inline bool is_local_client(struct zcache_client *cli) { return cli == &zcache_host; @@ -110,6 +117,8 @@ static inline int zcache_comp_op(enum comp_op op, case ZCACHE_COMPOP_DECOMPRESS: ret = crypto_comp_decompress(tfm, src, slen, dst, dlen); break; + default: + ret = -EINVAL; } put_cpu(); return ret; @@ -602,16 +611,14 @@ out: return; } -static void zbud_init(void) +static void __init zbud_init(void) { int i; INIT_LIST_HEAD(&zbud_buddied_list); - zcache_zbud_buddied_count = 0; - for (i = 0; i < NCHUNKS; i++) { + + for (i = 0; i < NCHUNKS; i++) INIT_LIST_HEAD(&zbud_unbuddied[i].list); - zbud_unbuddied[i].count = 0; - } } #ifdef CONFIG_SYSFS @@ -693,14 +700,14 @@ static unsigned int zv_max_mean_zsize = (PAGE_SIZE / 8) * 5; static atomic_t zv_curr_dist_counts[NCHUNKS]; static atomic_t zv_cumul_dist_counts[NCHUNKS]; -static struct zv_hdr *zv_create(struct zs_pool *pool, uint32_t pool_id, +static unsigned long zv_create(struct zs_pool *pool, uint32_t pool_id, struct tmem_oid *oid, uint32_t index, void *cdata, unsigned clen) { struct zv_hdr *zv; u32 size = clen + sizeof(struct zv_hdr); int chunks = (size + (CHUNK_SIZE - 1)) >> CHUNK_SHIFT; - void *handle = NULL; + unsigned long handle = 0; BUG_ON(!irqs_disabled()); BUG_ON(chunks >= NCHUNKS); @@ -709,7 +716,7 @@ static struct zv_hdr *zv_create(struct zs_pool *pool, uint32_t pool_id, goto out; atomic_inc(&zv_curr_dist_counts[chunks]); atomic_inc(&zv_cumul_dist_counts[chunks]); - zv = zs_map_object(pool, handle); + zv = zs_map_object(pool, handle, ZS_MM_WO); zv->index = index; zv->oid = *oid; zv->pool_id = pool_id; @@ -721,14 +728,14 @@ out: return handle; } -static void zv_free(struct zs_pool *pool, void *handle) +static void zv_free(struct zs_pool *pool, unsigned long handle) { unsigned long flags; struct zv_hdr *zv; uint16_t size; int chunks; - zv = zs_map_object(pool, handle); + zv = zs_map_object(pool, handle, ZS_MM_RW); ASSERT_SENTINEL(zv, ZVH); size = zv->size + sizeof(struct zv_hdr); INVERT_SENTINEL(zv, ZVH); @@ -743,14 +750,14 @@ static void zv_free(struct zs_pool *pool, void *handle) local_irq_restore(flags); } -static void zv_decompress(struct page *page, void *handle) +static void zv_decompress(struct page *page, unsigned long handle) { unsigned int clen = PAGE_SIZE; char *to_va; int ret; struct zv_hdr *zv; - zv = zs_map_object(zcache_host.zspool, handle); + zv = zs_map_object(zcache_host.zspool, handle, ZS_MM_RO); BUG_ON(zv->size == 0); ASSERT_SENTINEL(zv, ZVH); to_va = kmap_atomic(page); @@ -939,21 +946,14 @@ static struct tmem_pool *zcache_get_pool_by_id(uint16_t cli_id, uint16_t poolid) struct tmem_pool *pool = NULL; struct zcache_client *cli = NULL; - if (cli_id == LOCAL_CLIENT) - cli = &zcache_host; - else { - if (cli_id >= MAX_CLIENTS) - goto out; - cli = &zcache_clients[cli_id]; - if (cli == NULL) - goto out; - atomic_inc(&cli->refcount); - } - if (poolid < MAX_POOLS_PER_CLIENT) { - pool = cli->tmem_pools[poolid]; - if (pool != NULL) - atomic_inc(&pool->refcount); - } + cli = get_zcache_client(cli_id); + if (!cli) + goto out; + + atomic_inc(&cli->refcount); + pool = idr_find(&cli->tmem_pools, poolid); + if (pool != NULL) + atomic_inc(&pool->refcount); out: return pool; } @@ -971,13 +971,11 @@ static void zcache_put_pool(struct tmem_pool *pool) int zcache_new_client(uint16_t cli_id) { - struct zcache_client *cli = NULL; + struct zcache_client *cli; int ret = -1; - if (cli_id == LOCAL_CLIENT) - cli = &zcache_host; - else if ((unsigned int)cli_id < MAX_CLIENTS) - cli = &zcache_clients[cli_id]; + cli = get_zcache_client(cli_id); + if (cli == NULL) goto out; if (cli->allocated) @@ -987,6 +985,7 @@ int zcache_new_client(uint16_t cli_id) cli->zspool = zs_create_pool("zcache", ZCACHE_GFP_MASK); if (cli->zspool == NULL) goto out; + idr_init(&cli->tmem_pools); #endif ret = 0; out: @@ -1035,45 +1034,38 @@ static int zcache_do_preload(struct tmem_pool *pool) goto out; if (unlikely(zcache_obj_cache == NULL)) goto out; - preempt_disable(); + + /* IRQ has already been disabled. */ kp = &__get_cpu_var(zcache_preloads); while (kp->nr < ARRAY_SIZE(kp->objnodes)) { - preempt_enable_no_resched(); objnode = kmem_cache_alloc(zcache_objnode_cache, ZCACHE_GFP_MASK); if (unlikely(objnode == NULL)) { zcache_failed_alloc++; goto out; } - preempt_disable(); - kp = &__get_cpu_var(zcache_preloads); - if (kp->nr < ARRAY_SIZE(kp->objnodes)) - kp->objnodes[kp->nr++] = objnode; - else - kmem_cache_free(zcache_objnode_cache, objnode); + + kp->objnodes[kp->nr++] = objnode; } - preempt_enable_no_resched(); - obj = kmem_cache_alloc(zcache_obj_cache, ZCACHE_GFP_MASK); - if (unlikely(obj == NULL)) { - zcache_failed_alloc++; - goto out; + + if (!kp->obj) { + obj = kmem_cache_alloc(zcache_obj_cache, ZCACHE_GFP_MASK); + if (unlikely(obj == NULL)) { + zcache_failed_alloc++; + goto out; + } + kp->obj = obj; } - page = (void *)__get_free_page(ZCACHE_GFP_MASK); - if (unlikely(page == NULL)) { - zcache_failed_get_free_pages++; - kmem_cache_free(zcache_obj_cache, obj); - goto out; + + if (!kp->page) { + page = (void *)__get_free_page(ZCACHE_GFP_MASK); + if (unlikely(page == NULL)) { + zcache_failed_get_free_pages++; + goto out; + } + kp->page = page; } - preempt_disable(); - kp = &__get_cpu_var(zcache_preloads); - if (kp->obj == NULL) - kp->obj = obj; - else - kmem_cache_free(zcache_obj_cache, obj); - if (kp->page == NULL) - kp->page = page; - else - free_page((unsigned long)page); + ret = 0; out: return ret; @@ -1247,7 +1239,7 @@ static int zcache_pampd_get_data(char *data, size_t *bufsize, bool raw, int ret = 0; BUG_ON(is_ephemeral(pool)); - zv_decompress((struct page *)(data), pampd); + zv_decompress((struct page *)(data), (unsigned long)pampd); return ret; } @@ -1282,7 +1274,7 @@ static void zcache_pampd_free(void *pampd, struct tmem_pool *pool, atomic_dec(&zcache_curr_eph_pampd_count); BUG_ON(atomic_read(&zcache_curr_eph_pampd_count) < 0); } else { - zv_free(cli->zspool, pampd); + zv_free(cli->zspool, (unsigned long)pampd); atomic_dec(&zcache_curr_pers_pampd_count); BUG_ON(atomic_read(&zcache_curr_pers_pampd_count) < 0); } @@ -1583,15 +1575,14 @@ static int zcache_put_page(int cli_id, int pool_id, struct tmem_oid *oidp, else zcache_failed_pers_puts++; } - zcache_put_pool(pool); - preempt_enable_no_resched(); } else { zcache_put_to_flush++; if (atomic_read(&pool->obj_count) > 0) /* the put fails whether the flush succeeds or not */ (void)tmem_flush_page(pool, oidp, index); - zcache_put_pool(pool); } + + zcache_put_pool(pool); out: return ret; } @@ -1661,22 +1652,21 @@ static int zcache_flush_object(int cli_id, int pool_id, static int zcache_destroy_pool(int cli_id, int pool_id) { struct tmem_pool *pool = NULL; - struct zcache_client *cli = NULL; + struct zcache_client *cli; int ret = -1; if (pool_id < 0) goto out; - if (cli_id == LOCAL_CLIENT) - cli = &zcache_host; - else if ((unsigned int)cli_id < MAX_CLIENTS) - cli = &zcache_clients[cli_id]; + + cli = get_zcache_client(cli_id); if (cli == NULL) goto out; + atomic_inc(&cli->refcount); - pool = cli->tmem_pools[pool_id]; + pool = idr_find(&cli->tmem_pools, pool_id); if (pool == NULL) goto out; - cli->tmem_pools[pool_id] = NULL; + idr_remove(&cli->tmem_pools, pool_id); /* wait for pool activity on other cpus to quiesce */ while (atomic_read(&pool->refcount) != 0) ; @@ -1696,13 +1686,12 @@ static int zcache_new_pool(uint16_t cli_id, uint32_t flags) int poolid = -1; struct tmem_pool *pool; struct zcache_client *cli = NULL; + int r; - if (cli_id == LOCAL_CLIENT) - cli = &zcache_host; - else if ((unsigned int)cli_id < MAX_CLIENTS) - cli = &zcache_clients[cli_id]; + cli = get_zcache_client(cli_id); if (cli == NULL) goto out; + atomic_inc(&cli->refcount); pool = kmalloc(sizeof(struct tmem_pool), GFP_ATOMIC); if (pool == NULL) { @@ -1710,20 +1699,25 @@ static int zcache_new_pool(uint16_t cli_id, uint32_t flags) goto out; } - for (poolid = 0; poolid < MAX_POOLS_PER_CLIENT; poolid++) - if (cli->tmem_pools[poolid] == NULL) - break; - if (poolid >= MAX_POOLS_PER_CLIENT) { - pr_info("zcache: pool creation failed: max exceeded\n"); + do { + r = idr_pre_get(&cli->tmem_pools, GFP_ATOMIC); + if (r != 1) { + kfree(pool); + pr_info("zcache: pool creation failed: out of memory\n"); + goto out; + } + r = idr_get_new(&cli->tmem_pools, pool, &poolid); + } while (r == -EAGAIN); + if (r) { + pr_info("zcache: pool creation failed: error %d\n", r); kfree(pool); - poolid = -1; goto out; } + atomic_set(&pool->refcount, 0); pool->client = cli; pool->pool_id = poolid; tmem_new_pool(pool, flags); - cli->tmem_pools[poolid] = pool; pr_info("zcache: created %s tmem pool, id=%d, client=%d\n", flags & TMEM_POOL_PERSIST ? "persistent" : "ephemeral", poolid, cli_id); @@ -1981,7 +1975,7 @@ static int __init enable_zcache_compressor(char *s) __setup("zcache=", enable_zcache_compressor); -static int zcache_comp_init(void) +static int __init zcache_comp_init(void) { int ret = 0; @@ -2021,7 +2015,7 @@ static int __init zcache_init(void) goto out; } #endif /* CONFIG_SYSFS */ -#if defined(CONFIG_CLEANCACHE) || defined(CONFIG_FRONTSWAP) + if (zcache_enabled) { unsigned int cpu; @@ -2052,7 +2046,7 @@ static int __init zcache_init(void) pr_err("zcache: can't create client\n"); goto out; } -#endif + #ifdef CONFIG_CLEANCACHE if (zcache_enabled && use_cleancache) { struct cleancache_ops old_ops; |