diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2015-09-10 18:19:42 -0700 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2015-09-10 18:19:42 -0700 |
commit | 33e247c7e58d335d70ecb84fd869091e2e4b8dcb (patch) | |
tree | e8561e1993dff03f8e56d10a5795fe9d379a3390 /mm | |
parent | d71fc239b6915a8b750e9a447311029ff45b6580 (diff) | |
parent | 452e06af1f0149b01201f94264d452cd7a95db7a (diff) | |
download | linux-33e247c7e58d335d70ecb84fd869091e2e4b8dcb.tar.gz linux-33e247c7e58d335d70ecb84fd869091e2e4b8dcb.tar.bz2 linux-33e247c7e58d335d70ecb84fd869091e2e4b8dcb.zip |
Merge branch 'akpm' (patches from Andrew)
Merge third patch-bomb from Andrew Morton:
- even more of the rest of MM
- lib/ updates
- checkpatch updates
- small changes to a few scruffy filesystems
- kmod fixes/cleanups
- kexec updates
- a dma-mapping cleanup series from hch
* emailed patches from Andrew Morton <akpm@linux-foundation.org>: (81 commits)
dma-mapping: consolidate dma_set_mask
dma-mapping: consolidate dma_supported
dma-mapping: cosolidate dma_mapping_error
dma-mapping: consolidate dma_{alloc,free}_noncoherent
dma-mapping: consolidate dma_{alloc,free}_{attrs,coherent}
mm: use vma_is_anonymous() in create_huge_pmd() and wp_huge_pmd()
mm: make sure all file VMAs have ->vm_ops set
mm, mpx: add "vm_flags_t vm_flags" arg to do_mmap_pgoff()
mm: mark most vm_operations_struct const
namei: fix warning while make xmldocs caused by namei.c
ipc: convert invalid scenarios to use WARN_ON
zlib_deflate/deftree: remove bi_reverse()
lib/decompress_unlzma: Do a NULL check for pointer
lib/decompressors: use real out buf size for gunzip with kernel
fs/affs: make root lookup from blkdev logical size
sysctl: fix int -> unsigned long assignments in INT_MIN case
kexec: export KERNEL_IMAGE_SIZE to vmcoreinfo
kexec: align crash_notes allocation to make it be inside one physical page
kexec: remove unnecessary test in kimage_alloc_crash_control_pages()
kexec: split kexec_load syscall from kexec core code
...
Diffstat (limited to 'mm')
-rw-r--r-- | mm/Kconfig | 12 | ||||
-rw-r--r-- | mm/Makefile | 1 | ||||
-rw-r--r-- | mm/debug.c | 4 | ||||
-rw-r--r-- | mm/huge_memory.c | 12 | ||||
-rw-r--r-- | mm/hwpoison-inject.c | 5 | ||||
-rw-r--r-- | mm/kmemleak.c | 21 | ||||
-rw-r--r-- | mm/memcontrol.c | 76 | ||||
-rw-r--r-- | mm/memory-failure.c | 16 | ||||
-rw-r--r-- | mm/memory.c | 4 | ||||
-rw-r--r-- | mm/migrate.c | 6 | ||||
-rw-r--r-- | mm/mmap.c | 18 | ||||
-rw-r--r-- | mm/mmu_notifier.c | 17 | ||||
-rw-r--r-- | mm/nommu.c | 19 | ||||
-rw-r--r-- | mm/page_ext.c | 4 | ||||
-rw-r--r-- | mm/page_idle.c | 232 | ||||
-rw-r--r-- | mm/rmap.c | 6 | ||||
-rw-r--r-- | mm/swap.c | 3 | ||||
-rw-r--r-- | mm/zpool.c | 33 | ||||
-rw-r--r-- | mm/zswap.c | 688 |
19 files changed, 934 insertions, 243 deletions
diff --git a/mm/Kconfig b/mm/Kconfig index 3a4070f5ab79..6413d027c0b2 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -649,6 +649,18 @@ config DEFERRED_STRUCT_PAGE_INIT processes running early in the lifetime of the systemm until kswapd finishes the initialisation. +config IDLE_PAGE_TRACKING + bool "Enable idle page tracking" + depends on SYSFS && MMU + select PAGE_EXTENSION if !64BIT + help + This feature allows to estimate the amount of user pages that have + not been touched during a given period of time. This information can + be useful to tune memory cgroup limits and/or for job placement + within a compute cluster. + + See Documentation/vm/idle_page_tracking.txt for more details. + config ZONE_DEVICE bool "Device memory (pmem, etc...) hotplug support" if EXPERT default !ZONE_DMA diff --git a/mm/Makefile b/mm/Makefile index b424d5e5b6ff..56f8eed73f1a 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -79,3 +79,4 @@ obj-$(CONFIG_MEMORY_BALLOON) += balloon_compaction.o obj-$(CONFIG_PAGE_EXTENSION) += page_ext.o obj-$(CONFIG_CMA_DEBUGFS) += cma_debug.o obj-$(CONFIG_USERFAULTFD) += userfaultfd.o +obj-$(CONFIG_IDLE_PAGE_TRACKING) += page_idle.o diff --git a/mm/debug.c b/mm/debug.c index 76089ddf99ea..6c1b3ea61bfd 100644 --- a/mm/debug.c +++ b/mm/debug.c @@ -48,6 +48,10 @@ static const struct trace_print_flags pageflag_names[] = { #ifdef CONFIG_TRANSPARENT_HUGEPAGE {1UL << PG_compound_lock, "compound_lock" }, #endif +#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT) + {1UL << PG_young, "young" }, + {1UL << PG_idle, "idle" }, +#endif }; static void dump_flags(unsigned long flags, diff --git a/mm/huge_memory.c b/mm/huge_memory.c index b16279cbd91d..4b06b8db9df2 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -25,6 +25,7 @@ #include <linux/migrate.h> #include <linux/hashtable.h> #include <linux/userfaultfd_k.h> +#include <linux/page_idle.h> #include <asm/tlb.h> #include <asm/pgalloc.h> @@ -1757,6 +1758,11 @@ static void __split_huge_page_refcount(struct page *page, /* clear PageTail before overwriting first_page */ smp_wmb(); + if (page_is_young(page)) + set_page_young(page_tail); + if (page_is_idle(page)) + set_page_idle(page_tail); + /* * __split_huge_page_splitting() already set the * splitting bit in all pmd that could map this @@ -2262,7 +2268,8 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, VM_BUG_ON_PAGE(PageLRU(page), page); /* If there is no mapped pte young don't collapse the page */ - if (pte_young(pteval) || PageReferenced(page) || + if (pte_young(pteval) || + page_is_young(page) || PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm, address)) referenced = true; } @@ -2693,7 +2700,8 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, */ if (page_count(page) != 1 + !!PageSwapCache(page)) goto out_unmap; - if (pte_young(pteval) || PageReferenced(page) || + if (pte_young(pteval) || + page_is_young(page) || PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm, address)) referenced = true; } diff --git a/mm/hwpoison-inject.c b/mm/hwpoison-inject.c index aeba0edd6e44..9d26fd9fefe4 100644 --- a/mm/hwpoison-inject.c +++ b/mm/hwpoison-inject.c @@ -45,12 +45,9 @@ static int hwpoison_inject(void *data, u64 val) /* * do a racy check with elevated page count, to make sure PG_hwpoison * will only be set for the targeted owner (or on a free page). - * We temporarily take page lock for try_get_mem_cgroup_from_page(). * memory_failure() will redo the check reliably inside page lock. */ - lock_page(hpage); err = hwpoison_filter(hpage); - unlock_page(hpage); if (err) goto put_out; @@ -126,7 +123,7 @@ static int pfn_inject_init(void) if (!dentry) goto fail; -#ifdef CONFIG_MEMCG_SWAP +#ifdef CONFIG_MEMCG dentry = debugfs_create_u64("corrupt-filter-memcg", 0600, hwpoison_dir, &hwpoison_filter_memcg); if (!dentry) diff --git a/mm/kmemleak.c b/mm/kmemleak.c index f532f6a37b55..77191eccdc6f 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -302,23 +302,14 @@ static void hex_dump_object(struct seq_file *seq, struct kmemleak_object *object) { const u8 *ptr = (const u8 *)object->pointer; - int i, len, remaining; - unsigned char linebuf[HEX_ROW_SIZE * 5]; + size_t len; /* limit the number of lines to HEX_MAX_LINES */ - remaining = len = - min(object->size, (size_t)(HEX_MAX_LINES * HEX_ROW_SIZE)); - - seq_printf(seq, " hex dump (first %d bytes):\n", len); - for (i = 0; i < len; i += HEX_ROW_SIZE) { - int linelen = min(remaining, HEX_ROW_SIZE); - - remaining -= HEX_ROW_SIZE; - hex_dump_to_buffer(ptr + i, linelen, HEX_ROW_SIZE, - HEX_GROUP_SIZE, linebuf, sizeof(linebuf), - HEX_ASCII); - seq_printf(seq, " %s\n", linebuf); - } + len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE); + + seq_printf(seq, " hex dump (first %zu bytes):\n", len); + seq_hex_dump(seq, " ", DUMP_PREFIX_NONE, HEX_ROW_SIZE, + HEX_GROUP_SIZE, ptr, len, HEX_ASCII); } /* diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 1742a2db89c7..6ddaeba34e09 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -441,6 +441,34 @@ struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page) return &memcg->css; } +/** + * page_cgroup_ino - return inode number of the memcg a page is charged to + * @page: the page + * + * Look up the closest online ancestor of the memory cgroup @page is charged to + * and return its inode number or 0 if @page is not charged to any cgroup. It + * is safe to call this function without holding a reference to @page. + * + * Note, this function is inherently racy, because there is nothing to prevent + * the cgroup inode from getting torn down and potentially reallocated a moment + * after page_cgroup_ino() returns, so it only should be used by callers that + * do not care (such as procfs interfaces). + */ +ino_t page_cgroup_ino(struct page *page) +{ + struct mem_cgroup *memcg; + unsigned long ino = 0; + + rcu_read_lock(); + memcg = READ_ONCE(page->mem_cgroup); + while (memcg && !(memcg->css.flags & CSS_ONLINE)) + memcg = parent_mem_cgroup(memcg); + if (memcg) + ino = cgroup_ino(memcg->css.cgroup); + rcu_read_unlock(); + return ino; +} + static struct mem_cgroup_per_zone * mem_cgroup_page_zoneinfo(struct mem_cgroup *memcg, struct page *page) { @@ -2071,40 +2099,6 @@ static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages) css_put_many(&memcg->css, nr_pages); } -/* - * try_get_mem_cgroup_from_page - look up page's memcg association - * @page: the page - * - * Look up, get a css reference, and return the memcg that owns @page. - * - * The page must be locked to prevent racing with swap-in and page - * cache charges. If coming from an unlocked page table, the caller - * must ensure the page is on the LRU or this can race with charging. - */ -struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page) -{ - struct mem_cgroup *memcg; - unsigned short id; - swp_entry_t ent; - - VM_BUG_ON_PAGE(!PageLocked(page), page); - - memcg = page->mem_cgroup; - if (memcg) { - if (!css_tryget_online(&memcg->css)) - memcg = NULL; - } else if (PageSwapCache(page)) { - ent.val = page_private(page); - id = lookup_swap_cgroup_id(ent); - rcu_read_lock(); - memcg = mem_cgroup_from_id(id); - if (memcg && !css_tryget_online(&memcg->css)) - memcg = NULL; - rcu_read_unlock(); - } - return memcg; -} - static void lock_page_lru(struct page *page, int *isolated) { struct zone *zone = page_zone(page); @@ -5301,8 +5295,20 @@ int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm, * the page lock, which serializes swap cache removal, which * in turn serializes uncharging. */ + VM_BUG_ON_PAGE(!PageLocked(page), page); if (page->mem_cgroup) goto out; + + if (do_swap_account) { + swp_entry_t ent = { .val = page_private(page), }; + unsigned short id = lookup_swap_cgroup_id(ent); + + rcu_read_lock(); + memcg = mem_cgroup_from_id(id); + if (memcg && !css_tryget_online(&memcg->css)) + memcg = NULL; + rcu_read_unlock(); + } } if (PageTransHuge(page)) { @@ -5310,8 +5316,6 @@ int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm, VM_BUG_ON_PAGE(!PageTransHuge(page), page); } - if (do_swap_account && PageSwapCache(page)) - memcg = try_get_mem_cgroup_from_page(page); if (!memcg) memcg = get_mem_cgroup_from_mm(mm); diff --git a/mm/memory-failure.c b/mm/memory-failure.c index eeda6485e76c..95882692e747 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -130,27 +130,15 @@ static int hwpoison_filter_flags(struct page *p) * can only guarantee that the page either belongs to the memcg tasks, or is * a freed page. */ -#ifdef CONFIG_MEMCG_SWAP +#ifdef CONFIG_MEMCG u64 hwpoison_filter_memcg; EXPORT_SYMBOL_GPL(hwpoison_filter_memcg); static int hwpoison_filter_task(struct page *p) { - struct mem_cgroup *mem; - struct cgroup_subsys_state *css; - unsigned long ino; - if (!hwpoison_filter_memcg) return 0; - mem = try_get_mem_cgroup_from_page(p); - if (!mem) - return -EINVAL; - - css = &mem->css; - ino = cgroup_ino(css->cgroup); - css_put(css); - - if (ino != hwpoison_filter_memcg) + if (page_cgroup_ino(p) != hwpoison_filter_memcg) return -EINVAL; return 0; diff --git a/mm/memory.c b/mm/memory.c index 6cd0b2160401..9cb27470fee9 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3233,7 +3233,7 @@ out: static int create_huge_pmd(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, pmd_t *pmd, unsigned int flags) { - if (!vma->vm_ops) + if (vma_is_anonymous(vma)) return do_huge_pmd_anonymous_page(mm, vma, address, pmd, flags); if (vma->vm_ops->pmd_fault) return vma->vm_ops->pmd_fault(vma, address, pmd, flags); @@ -3244,7 +3244,7 @@ static int wp_huge_pmd(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, pmd_t *pmd, pmd_t orig_pmd, unsigned int flags) { - if (!vma->vm_ops) + if (vma_is_anonymous(vma)) return do_huge_pmd_wp_page(mm, vma, address, pmd, orig_pmd); if (vma->vm_ops->pmd_fault) return vma->vm_ops->pmd_fault(vma, address, pmd, flags); diff --git a/mm/migrate.c b/mm/migrate.c index 02ce25df16c2..c3cb566af3e2 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -37,6 +37,7 @@ #include <linux/gfp.h> #include <linux/balloon_compaction.h> #include <linux/mmu_notifier.h> +#include <linux/page_idle.h> #include <asm/tlbflush.h> @@ -524,6 +525,11 @@ void migrate_page_copy(struct page *newpage, struct page *page) __set_page_dirty_nobuffers(newpage); } + if (page_is_young(page)) + set_page_young(newpage); + if (page_is_idle(page)) + set_page_idle(newpage); + /* * Copy NUMA information to the new page, to prevent over-eager * future migrations of this same page. diff --git a/mm/mmap.c b/mm/mmap.c index b6be3249f0a9..971dd2cb77d2 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -612,6 +612,8 @@ static unsigned long count_vma_pages_range(struct mm_struct *mm, void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma, struct rb_node **rb_link, struct rb_node *rb_parent) { + WARN_ONCE(vma->vm_file && !vma->vm_ops, "missing vma->vm_ops"); + /* Update tracking information for the gap following the new vma. */ if (vma->vm_next) vma_gap_update(vma->vm_next); @@ -1260,14 +1262,12 @@ static inline int mlock_future_check(struct mm_struct *mm, /* * The caller must hold down_write(¤t->mm->mmap_sem). */ - -unsigned long do_mmap_pgoff(struct file *file, unsigned long addr, +unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, - unsigned long flags, unsigned long pgoff, - unsigned long *populate) + unsigned long flags, vm_flags_t vm_flags, + unsigned long pgoff, unsigned long *populate) { struct mm_struct *mm = current->mm; - vm_flags_t vm_flags; *populate = 0; @@ -1311,7 +1311,7 @@ unsigned long do_mmap_pgoff(struct file *file, unsigned long addr, * to. we assume access permissions have been handled by the open * of the memory object, so we don't do any here. */ - vm_flags = calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags) | + vm_flags |= calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags) | mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; if (flags & MAP_LOCKED) @@ -1638,6 +1638,12 @@ unsigned long mmap_region(struct file *file, unsigned long addr, */ WARN_ON_ONCE(addr != vma->vm_start); + /* All file mapping must have ->vm_ops set */ + if (!vma->vm_ops) { + static const struct vm_operations_struct dummy_ops = {}; + vma->vm_ops = &dummy_ops; + } + addr = vma->vm_start; vm_flags = vma->vm_flags; } else if (vm_flags & VM_SHARED) { diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c index 3b9b3d0741b2..5fbdd367bbed 100644 --- a/mm/mmu_notifier.c +++ b/mm/mmu_notifier.c @@ -123,6 +123,23 @@ int __mmu_notifier_clear_flush_young(struct mm_struct *mm, return young; } +int __mmu_notifier_clear_young(struct mm_struct *mm, + unsigned long start, + unsigned long end) +{ + struct mmu_notifier *mn; + int young = 0, id; + + id = srcu_read_lock(&srcu); + hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) { + if (mn->ops->clear_young) + young |= mn->ops->clear_young(mn, mm, start, end); + } + srcu_read_unlock(&srcu, id); + + return young; +} + int __mmu_notifier_test_young(struct mm_struct *mm, unsigned long address) { diff --git a/mm/nommu.c b/mm/nommu.c index 1cc0709fcaa5..ab14a2014dea 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -1233,18 +1233,19 @@ enomem: /* * handle mapping creation for uClinux */ -unsigned long do_mmap_pgoff(struct file *file, - unsigned long addr, - unsigned long len, - unsigned long prot, - unsigned long flags, - unsigned long pgoff, - unsigned long *populate) +unsigned long do_mmap(struct file *file, + unsigned long addr, + unsigned long len, + unsigned long prot, + unsigned long flags, + vm_flags_t vm_flags, + unsigned long pgoff, + unsigned long *populate) { struct vm_area_struct *vma; struct vm_region *region; struct rb_node *rb; - unsigned long capabilities, vm_flags, result; + unsigned long capabilities, result; int ret; *populate = 0; @@ -1262,7 +1263,7 @@ unsigned long do_mmap_pgoff(struct file *file, /* we've determined that we can make the mapping, now translate what we * now know into VMA flags */ - vm_flags = determine_vm_flags(file, prot, flags, capabilities); + vm_flags |= determine_vm_flags(file, prot, flags, capabilities); /* we're going to need to record the mapping */ region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL); diff --git a/mm/page_ext.c b/mm/page_ext.c index d86fd2f5353f..292ca7b8debd 100644 --- a/mm/page_ext.c +++ b/mm/page_ext.c @@ -6,6 +6,7 @@ #include <linux/vmalloc.h> #include <linux/kmemleak.h> #include <linux/page_owner.h> +#include <linux/page_idle.h> /* * struct page extension @@ -59,6 +60,9 @@ static struct page_ext_operations *page_ext_ops[] = { #ifdef CONFIG_PAGE_OWNER &page_owner_ops, #endif +#if defined(CONFIG_IDLE_PAGE_TRACKING) && !defined(CONFIG_64BIT) + &page_idle_ops, +#endif }; static unsigned long total_usage; diff --git a/mm/page_idle.c b/mm/page_idle.c new file mode 100644 index 000000000000..d5dd79041484 --- /dev/null +++ b/mm/page_idle.c @@ -0,0 +1,232 @@ +#include <linux/init.h> +#include <linux/bootmem.h> +#include <linux/fs.h> +#include <linux/sysfs.h> +#include <linux/kobject.h> +#include <linux/mm.h> +#include <linux/mmzone.h> +#include <linux/pagemap.h> +#include <linux/rmap.h> +#include <linux/mmu_notifier.h> +#include <linux/page_ext.h> +#include <linux/page_idle.h> + +#define BITMAP_CHUNK_SIZE sizeof(u64) +#define BITMAP_CHUNK_BITS (BITMAP_CHUNK_SIZE * BITS_PER_BYTE) + +/* + * Idle page tracking only considers user memory pages, for other types of + * pages the idle flag is always unset and an attempt to set it is silently + * ignored. + * + * We treat a page as a user memory page if it is on an LRU list, because it is + * always safe to pass such a page to rmap_walk(), which is essential for idle + * page tracking. With such an indicator of user pages we can skip isolated + * pages, but since there are not usually many of them, it will hardly affect + * the overall result. + * + * This function tries to get a user memory page by pfn as described above. + */ +static struct page *page_idle_get_page(unsigned long pfn) +{ + struct page *page; + struct zone *zone; + + if (!pfn_valid(pfn)) + return NULL; + + page = pfn_to_page(pfn); + if (!page || !PageLRU(page) || + !get_page_unless_zero(page)) + return NULL; + + zone = page_zone(page); + spin_lock_irq(&zone->lru_lock); + if (unlikely(!PageLRU(page))) { + put_page(page); + page = NULL; + } + spin_unlock_irq(&zone->lru_lock); + return page; +} + +static int page_idle_clear_pte_refs_one(struct page *page, + struct vm_area_struct *vma, + unsigned long addr, void *arg) +{ + struct mm_struct *mm = vma->vm_mm; + spinlock_t *ptl; + pmd_t *pmd; + pte_t *pte; + bool referenced = false; + + if (unlikely(PageTransHuge(page))) { + pmd = page_check_address_pmd(page, mm, addr, + PAGE_CHECK_ADDRESS_PMD_FLAG, &ptl); + if (pmd) { + referenced = pmdp_clear_young_notify(vma, addr, pmd); + spin_unlock(ptl); + } + } else { + pte = page_check_address(page, mm, addr, &ptl, 0); + if (pte) { + referenced = ptep_clear_young_notify(vma, addr, pte); + pte_unmap_unlock(pte, ptl); + } + } + if (referenced) { + clear_page_idle(page); + /* + * We cleared the referenced bit in a mapping to this page. To + * avoid interference with page reclaim, mark it young so that + * page_referenced() will return > 0. + */ + set_page_young(page); + } + return SWAP_AGAIN; +} + +static void page_idle_clear_pte_refs(struct page *page) +{ + /* + * Since rwc.arg is unused, rwc is effectively immutable, so we + * can make it static const to save some cycles and stack. + */ + static const struct rmap_walk_control rwc = { + .rmap_one = page_idle_clear_pte_refs_one, + .anon_lock = page_lock_anon_vma_read, + }; + bool need_lock; + + if (!page_mapped(page) || + !page_rmapping(page)) + return; + + need_lock = !PageAnon(page) || PageKsm(page); + if (need_lock && !trylock_page(page)) + return; + + rmap_walk(page, (struct rmap_walk_control *)&rwc); + + if (need_lock) + unlock_page(page); +} + +static ssize_t page_idle_bitmap_read(struct file *file, struct kobject *kobj, + struct bin_attribute *attr, char *buf, + loff_t pos, size_t count) +{ + u64 *out = (u64 *)buf; + struct page *page; + unsigned long pfn, end_pfn; + int bit; + + if (pos % BITMAP_CHUNK_SIZE || count % BITMAP_CHUNK_SIZE) + return -EINVAL; + + pfn = pos * BITS_PER_BYTE; + if (pfn >= max_pfn) + return 0; + + end_pfn = pfn + count * BITS_PER_BYTE; + if (end_pfn > max_pfn) + end_pfn = ALIGN(max_pfn, BITMAP_CHUNK_BITS); + + for (; pfn < end_pfn; pfn++) { + bit = pfn % BITMAP_CHUNK_BITS; + if (!bit) + *out = 0ULL; + page = page_idle_get_page(pfn); + if (page) { + if (page_is_idle(page)) { + /* + * The page might have been referenced via a + * pte, in which case it is not idle. Clear + * refs and recheck. + */ + page_idle_clear_pte_refs(page); + if (page_is_idle(page)) + *out |= 1ULL << bit; + } + put_page(page); + } + if (bit == BITMAP_CHUNK_BITS - 1) + out++; + cond_resched(); + } + return (char *)out - buf; +} + +static ssize_t page_idle_bitmap_write(struct file *file, struct kobject *kobj, + struct bin_attribute *attr, char *buf, + loff_t pos, size_t count) +{ + const u64 *in = (u64 *)buf; + struct page *page; + unsigned long pfn, end_pfn; + int bit; + + if (pos % BITMAP_CHUNK_SIZE || count % BITMAP_CHUNK_SIZE) + return -EINVAL; + + pfn = pos * BITS_PER_BYTE; + if (pfn >= max_pfn) + return -ENXIO; + + end_pfn = pfn + count * BITS_PER_BYTE; + if (end_pfn > max_pfn) + end_pfn = ALIGN(max_pfn, BITMAP_CHUNK_BITS); + + for (; pfn < end_pfn; pfn++) { + bit = pfn % BITMAP_CHUNK_BITS; + if ((*in >> bit) & 1) { + page = page_idle_get_page(pfn); + if (page) { + page_idle_clear_pte_refs(page); + set_page_idle(page); + put_page(page); + } + } + if (bit == BITMAP_CHUNK_BITS - 1) + in++; + cond_resched(); + } + return (char *)in - buf; +} + +static struct bin_attribute page_idle_bitmap_attr = + __BIN_ATTR(bitmap, S_IRUSR | S_IWUSR, + page_idle_bitmap_read, page_idle_bitmap_write, 0); + +static struct bin_attribute *page_idle_bin_attrs[] = { + &page_idle_bitmap_attr, + NULL, +}; + +static struct attribute_group page_idle_attr_group = { + .bin_attrs = page_idle_bin_attrs, + .name = "page_idle", +}; + +#ifndef CONFIG_64BIT +static bool need_page_idle(void) +{ + return true; +} +struct page_ext_operations page_idle_ops = { + .need = need_page_idle, +}; +#endif + +static int __init page_idle_init(void) +{ + int err; + + err = sysfs_create_group(mm_kobj, &page_idle_attr_group); + if (err) { + pr_err("page_idle: register sysfs failed\n"); + return err; + } + return 0; +} +subsys_initcall(page_idle_init); diff --git a/mm/rmap.c b/mm/rmap.c index 0db38e7d0a72..f5b5c1f3dcd7 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -59,6 +59,7 @@ #include <linux/migrate.h> #include <linux/hugetlb.h> #include <linux/backing-dev.h> +#include <linux/page_idle.h> #include <asm/tlbflush.h> @@ -886,6 +887,11 @@ static int page_referenced_one(struct page *page, struct vm_area_struct *vma, pte_unmap_unlock(pte, ptl); } + if (referenced) + clear_page_idle(page); + if (test_and_clear_page_young(page)) + referenced++; + if (referenced) { pra->referenced++; pra->vm_flags |= vma->vm_flags; diff --git a/mm/swap.c b/mm/swap.c index a3a0a2f1f7c3..983f692a47fd 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -32,6 +32,7 @@ #include <linux/gfp.h> #include <linux/uio.h> #include <linux/hugetlb.h> +#include <linux/page_idle.h> #include "internal.h" @@ -622,6 +623,8 @@ void mark_page_accessed(struct page *page) } else if (!PageReferenced(page)) { SetPageReferenced(page); } + if (page_is_idle(page)) + clear_page_idle(page); } EXPORT_SYMBOL(mark_page_accessed); diff --git a/mm/zpool.c b/mm/zpool.c index 68d2dd8ed2d8..8f670d3e8706 100644 --- a/mm/zpool.c +++ b/mm/zpool.c @@ -100,6 +100,39 @@ static void zpool_put_driver(struct zpool_driver *driver) } /** + * zpool_has_pool() - Check if the pool driver is available + * @type The type of the zpool to check (e.g. zbud, zsmalloc) + * + * This checks if the @type pool driver is available. This will try to load + * the requested module, if needed, but there is no guarantee the module will + * still be loaded and available immediately after calling. If this returns + * true, the caller should assume the pool is available, but must be prepared + * to handle the @zpool_create_pool() returning failure. However if this + * returns false, the caller should assume the requested pool type is not + * available; either the requested pool type module does not exist, or could + * not be loaded, and calling @zpool_create_pool() with the pool type will + * fail. + * + * Returns: true if @type pool is available, false if not + */ +bool zpool_has_pool(char *type) +{ + struct zpool_driver *driver = zpool_get_driver(type); + + if (!driver) { + request_module("zpool-%s", type); + driver = zpool_get_driver(type); + } + + if (!driver) + return false; + + zpool_put_driver(driver); + return true; +} +EXPORT_SYMBOL(zpool_has_pool); + +/** * zpool_create_pool() - Create a new zpool * @type The type of the zpool to create (e.g. zbud, zsmalloc) * @name The name of the zpool (e.g. zram0, zswap) diff --git a/mm/zswap.c b/mm/zswap.c index 48a1d081e2a5..4043df7c672f 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -80,85 +80,54 @@ static u64 zswap_duplicate_entry; static bool zswap_enabled; module_param_named(enabled, zswap_enabled, bool, 0644); -/* Compressor to be used by zswap (fixed at boot for now) */ +/* Crypto compressor to use */ #define ZSWAP_COMPRESSOR_DEFAULT "lzo" -static char *zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT; -module_param_named(compressor, zswap_compressor, charp, 0444); - -/* The maximum percentage of memory that the compressed pool can occupy */ -static unsigned int zswap_max_pool_percent = 20; -module_param_named(max_pool_percent, - zswap_max_pool_percent, uint, 0644); +static char zswap_compressor[CRYPTO_MAX_ALG_NAME] = ZSWAP_COMPRESSOR_DEFAULT; +static struct kparam_string zswap_compressor_kparam = { + .string = zswap_compressor, + .maxlen = sizeof(zswap_compressor), +}; +static int zswap_compressor_param_set(const char *, + const struct kernel_param *); +static struct kernel_param_ops zswap_compressor_param_ops = { + .set = zswap_compressor_param_set, + .get = param_get_string, +}; +module_param_cb(compressor, &zswap_compressor_param_ops, + &zswap_compressor_kparam, 0644); -/* Compressed storage to use */ +/* Compressed storage zpool to use */ #define ZSWAP_ZPOOL_DEFAULT "zbud" -static char *zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT; -module_param_named(zpool, zswap_zpool_type, charp, 0444); +static char zswap_zpool_type[32 /* arbitrary */] = ZSWAP_ZPOOL_DEFAULT; +static struct kparam_string zswap_zpool_kparam = { + .string = zswap_zpool_type, + .maxlen = sizeof(zswap_zpool_type), +}; +static int zswap_zpool_param_set(const char *, const struct kernel_param *); +static struct kernel_param_ops zswap_zpool_param_ops = { + .set = zswap_zpool_param_set, + .get = param_get_string, +}; +module_param_cb(zpool, &zswap_zpool_param_ops, &zswap_zpool_kparam, 0644); -/* zpool is shared by all of zswap backend */ -static struct zpool *zswap_pool; +/* The maximum percentage of memory that the compressed pool can occupy */ +static unsigned int zswap_max_pool_percent = 20; +module_param_named(max_pool_percent, zswap_max_pool_percent, uint, 0644); /********************************* -* compression functions +* data structures **********************************/ -/* per-cpu compression transforms */ -static struct crypto_comp * __percpu *zswap_comp_pcpu_tfms; -enum comp_op { - ZSWAP_COMPOP_COMPRESS, - ZSWAP_COMPOP_DECOMPRESS +struct zswap_pool { + struct zpool *zpool; + struct crypto_comp * __percpu *tfm; + struct kref kref; + struct list_head list; + struct rcu_head rcu_head; + struct notifier_block notifier; + char tfm_name[CRYPTO_MAX_ALG_NAME]; }; -static int zswap_comp_op(enum comp_op op, const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen) -{ - struct crypto_comp *tfm; - int ret; - - tfm = *per_cpu_ptr(zswap_comp_pcpu_tfms, get_cpu()); - switch (op) { - case ZSWAP_COMPOP_COMPRESS: - ret = crypto_comp_compress(tfm, src, slen, dst, dlen); - break; - case ZSWAP_COMPOP_DECOMPRESS: - ret = crypto_comp_decompress(tfm, src, slen, dst, dlen); - break; - default: - ret = -EINVAL; - } - - put_cpu(); - return ret; -} - -static int __init zswap_comp_init(void) -{ - if (!crypto_has_comp(zswap_compressor, 0, 0)) { - pr_info("%s compressor not available\n", zswap_compressor); - /* fall back to default compressor */ - zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT; - if (!crypto_has_comp(zswap_compressor, 0, 0)) - /* can't even load the default compressor */ - return -ENODEV; - } - pr_info("using %s compressor\n", zswap_compressor); - - /* alloc percpu transforms */ - zswap_comp_pcpu_tfms = alloc_percpu(struct crypto_comp *); - if (!zswap_comp_pcpu_tfms) - return -ENOMEM; - return 0; -} - -static void __init zswap_comp_exit(void) -{ - /* free percpu transforms */ - free_percpu(zswap_comp_pcpu_tfms); -} - -/********************************* -* data structures -**********************************/ /* * struct zswap_entry * @@ -166,22 +135,24 @@ static void __init zswap_comp_exit(void) * page within zswap. * * rbnode - links the entry into red-black tree for the appropriate swap type + * offset - the swap offset for the entry. Index into the red-black tree. * refcount - the number of outstanding reference to the entry. This is needed * to protect against premature freeing of the entry by code * concurrent calls to load, invalidate, and writeback. The lock * for the zswap_tree structure that contains the entry must * be held while changing the refcount. Since the lock must * be held, there is no reason to also make refcount atomic. - * offset - the swap offset for the entry. Index into the red-black tree. - * handle - zpool allocation handle that stores the compressed page data * length - the length in bytes of the compressed page data. Needed during * decompression + * pool - the zswap_pool the entry's data is in + * handle - zpool allocation handle that stores the compressed page data */ struct zswap_entry { struct rb_node rbnode; pgoff_t offset; int refcount; unsigned int length; + struct zswap_pool *pool; unsigned long handle; }; @@ -201,6 +172,51 @@ struct zswap_tree { static struct zswap_tree *zswap_trees[MAX_SWAPFILES]; +/* RCU-protected iteration */ +static LIST_HEAD(zswap_pools); +/* protects zswap_pools list modification */ +static DEFINE_SPINLOCK(zswap_pools_lock); + +/* used by param callback function */ +static bool zswap_init_started; + +/********************************* +* helpers and fwd declarations +**********************************/ + +#define zswap_pool_debug(msg, p) \ + pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name, \ + zpool_get_type((p)->zpool)) + +static int zswap_writeback_entry(struct zpool *pool, unsigned long handle); +static int zswap_pool_get(struct zswap_pool *pool); +static void zswap_pool_put(struct zswap_pool *pool); + +static const struct zpool_ops zswap_zpool_ops = { + .evict = zswap_writeback_entry +}; + +static bool zswap_is_full(void) +{ + return totalram_pages * zswap_max_pool_percent / 100 < + DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE); +} + +static void zswap_update_total_size(void) +{ + struct zswap_pool *pool; + u64 total = 0; + + rcu_read_lock(); + + list_for_each_entry_rcu(pool, &zswap_pools, list) + total += zpool_get_total_size(pool->zpool); + + rcu_read_unlock(); + + zswap_pool_total_size = total; +} + /********************************* * zswap entry functions **********************************/ @@ -294,10 +310,11 @@ static void zswap_rb_erase(struct rb_root *root, struct zswap_entry *entry) */ static void zswap_free_entry(struct zswap_entry *entry) { - zpool_free(zswap_pool, entry->handle); + zpool_free(entry->pool->zpool, entry->handle); + zswap_pool_put(entry->pool); zswap_entry_cache_free(entry); atomic_dec(&zswap_stored_pages); - zswap_pool_total_size = zpool_get_total_size(zswap_pool); + zswap_update_total_size(); } /* caller must hold the tree lock */ @@ -339,35 +356,21 @@ static struct zswap_entry *zswap_entry_find_get(struct rb_root *root, **********************************/ static DEFINE_PER_CPU(u8 *, zswap_dstmem); -static int __zswap_cpu_notifier(unsigned long action, unsigned long cpu) +static int __zswap_cpu_dstmem_notifier(unsigned long action, unsigned long cpu) { - struct crypto_comp *tfm; u8 *dst; switch (action) { case CPU_UP_PREPARE: - tfm = crypto_alloc_comp(zswap_compressor, 0, 0); - if (IS_ERR(tfm)) { - pr_err("can't allocate compressor transform\n"); - return NOTIFY_BAD; - } - *per_cpu_ptr(zswap_comp_pcpu_tfms, cpu) = tfm; dst = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu)); if (!dst) { pr_err("can't allocate compressor buffer\n"); - crypto_free_comp(tfm); - *per_cpu_ptr(zswap_comp_pcpu_tfms, cpu) = NULL; return NOTIFY_BAD; } per_cpu(zswap_dstmem, cpu) = dst; break; case CPU_DEAD: case CPU_UP_CANCELED: - tfm = *per_cpu_ptr(zswap_comp_pcpu_tfms, cpu); - if (tfm) { - crypto_free_comp(tfm); - *per_cpu_ptr(zswap_comp_pcpu_tfms, cpu) = NULL; - } dst = per_cpu(zswap_dstmem, cpu); kfree(dst); per_cpu(zswap_dstmem, cpu) = NULL; @@ -378,43 +381,398 @@ static int __zswap_cpu_notifier(unsigned long action, unsigned long cpu) return NOTIFY_OK; } -static int zswap_cpu_notifier(struct notifier_block *nb, - unsigned long action, void *pcpu) +static int zswap_cpu_dstmem_notifier(struct notifier_block *nb, + unsigned long action, void *pcpu) { - unsigned long cpu = (unsigned long)pcpu; - return __zswap_cpu_notifier(action, cpu); + return __zswap_cpu_dstmem_notifier(action, (unsigned long)pcpu); } -static struct notifier_block zswap_cpu_notifier_block = { - .notifier_call = zswap_cpu_notifier +static struct notifier_block zswap_dstmem_notifier = { + .notifier_call = zswap_cpu_dstmem_notifier, }; -static int __init zswap_cpu_init(void) +static int __init zswap_cpu_dstmem_init(void) +{ + unsigned long cpu; + + cpu_notifier_register_begin(); + for_each_online_cpu(cpu) + if (__zswap_cpu_dstmem_notifier(CPU_UP_PREPARE, cpu) == + NOTIFY_BAD) + goto cleanup; + __register_cpu_notifier(&zswap_dstmem_notifier); + cpu_notifier_register_done(); + return 0; + +cleanup: + for_each_online_cpu(cpu) + __zswap_cpu_dstmem_notifier(CPU_UP_CANCELED, cpu); + cpu_notifier_register_done(); + return -ENOMEM; +} + +static void zswap_cpu_dstmem_destroy(void) +{ + unsigned long cpu; + + cpu_notifier_register_begin(); + for_each_online_cpu(cpu) + __zswap_cpu_dstmem_notifier(CPU_UP_CANCELED, cpu); + __unregister_cpu_notifier(&zswap_dstmem_notifier); + cpu_notifier_register_done(); +} + +static int __zswap_cpu_comp_notifier(struct zswap_pool *pool, + unsigned long action, unsigned long cpu) +{ + struct crypto_comp *tfm; + + switch (action) { + case CPU_UP_PREPARE: + if (WARN_ON(*per_cpu_ptr(pool->tfm, cpu))) + break; + tfm = crypto_alloc_comp(pool->tfm_name, 0, 0); + if (IS_ERR_OR_NULL(tfm)) { + pr_err("could not alloc crypto comp %s : %ld\n", + pool->tfm_name, PTR_ERR(tfm)); + return NOTIFY_BAD; + } + *per_cpu_ptr(pool->tfm, cpu) = tfm; + break; + case CPU_DEAD: + case CPU_UP_CANCELED: + tfm = *per_cpu_ptr(pool->tfm, cpu); + if (!IS_ERR_OR_NULL(tfm)) + crypto_free_comp(tfm); + *per_cpu_ptr(pool->tfm, cpu) = NULL; + break; + default: + break; + } + return NOTIFY_OK; +} + +static int zswap_cpu_comp_notifier(struct notifier_block *nb, + unsigned long action, void *pcpu) +{ + unsigned long cpu = (unsigned long)pcpu; + struct zswap_pool *pool = container_of(nb, typeof(*pool), notifier); + + return __zswap_cpu_comp_notifier(pool, action, cpu); +} + +static int zswap_cpu_comp_init(struct zswap_pool *pool) { unsigned long cpu; + memset(&pool->notifier, 0, sizeof(pool->notifier)); + pool->notifier.notifier_call = zswap_cpu_comp_notifier; + cpu_notifier_register_begin(); for_each_online_cpu(cpu) - if (__zswap_cpu_notifier(CPU_UP_PREPARE, cpu) != NOTIFY_OK) + if (__zswap_cpu_comp_notifier(pool, CPU_UP_PREPARE, cpu) == + NOTIFY_BAD) goto cleanup; - __register_cpu_notifier(&zswap_cpu_notifier_block); + __register_cpu_notifier(&pool->notifier); cpu_notifier_register_done(); return 0; cleanup: for_each_online_cpu(cpu) - __zswap_cpu_notifier(CPU_UP_CANCELED, cpu); + __zswap_cpu_comp_notifier(pool, CPU_UP_CANCELED, cpu); cpu_notifier_register_done(); return -ENOMEM; } +static void zswap_cpu_comp_destroy(struct zswap_pool *pool) +{ + unsigned long cpu; + + cpu_notifier_register_begin(); + for_each_online_cpu(cpu) + __zswap_cpu_comp_notifier(pool, CPU_UP_CANCELED, cpu); + __unregister_cpu_notifier(&pool->notifier); + cpu_notifier_register_done(); +} + /********************************* -* helpers +* pool functions **********************************/ -static bool zswap_is_full(void) + +static struct zswap_pool *__zswap_pool_current(void) { - return totalram_pages * zswap_max_pool_percent / 100 < - DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE); + struct zswap_pool *pool; + + pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list); + WARN_ON(!pool); + + return pool; +} + +static struct zswap_pool *zswap_pool_current(void) +{ + assert_spin_locked(&zswap_pools_lock); + + return __zswap_pool_current(); +} + +static struct zswap_pool *zswap_pool_current_get(void) +{ + struct zswap_pool *pool; + + rcu_read_lock(); + + pool = __zswap_pool_current(); + if (!pool || !zswap_pool_get(pool)) + pool = NULL; + + rcu_read_unlock(); + + return pool; +} + +static struct zswap_pool *zswap_pool_last_get(void) +{ + struct zswap_pool *pool, *last = NULL; + + rcu_read_lock(); + + list_for_each_entry_rcu(pool, &zswap_pools, list) + last = pool; + if (!WARN_ON(!last) && !zswap_pool_get(last)) + last = NULL; + + rcu_read_unlock(); + + return last; +} + +static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor) +{ + struct zswap_pool *pool; + + assert_spin_locked(&zswap_pools_lock); + + list_for_each_entry_rcu(pool, &zswap_pools, list) { + if (strncmp(pool->tfm_name, compressor, sizeof(pool->tfm_name))) + continue; + if (strncmp(zpool_get_type(pool->zpool), type, + sizeof(zswap_zpool_type))) + continue; + /* if we can't get it, it's about to be destroyed */ + if (!zswap_pool_get(pool)) + continue; + return pool; + } + + return NULL; +} + +static struct zswap_pool *zswap_pool_create(char *type, char *compressor) +{ + struct zswap_pool *pool; + gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN; + + pool = kzalloc(sizeof(*pool), GFP_KERNEL); + if (!pool) { + pr_err("pool alloc failed\n"); + return NULL; + } + + pool->zpool = zpool_create_pool(type, "zswap", gfp, &zswap_zpool_ops); + if (!pool->zpool) { + pr_err("%s zpool not available\n", type); + goto error; + } + pr_debug("using %s zpool\n", zpool_get_type(pool->zpool)); + + strlcpy(pool->tfm_name, compressor, sizeof(pool->tfm_name)); + pool->tfm = alloc_percpu(struct crypto_comp *); + if (!pool->tfm) { + pr_err("percpu alloc failed\n"); + goto error; + } + + if (zswap_cpu_comp_init(pool)) + goto error; + pr_debug("using %s compressor\n", pool->tfm_name); + + /* being the current pool takes 1 ref; this func expects the + * caller to always add the new pool as the current pool + */ + kref_init(&pool->kref); + INIT_LIST_HEAD(&pool->list); + + zswap_pool_debug("created", pool); + + return pool; + +error: + free_percpu(pool->tfm); + if (pool->zpool) + zpool_destroy_pool(pool->zpool); + kfree(pool); + return NULL; +} + +static struct zswap_pool *__zswap_pool_create_fallback(void) +{ + if (!crypto_has_comp(zswap_compressor, 0, 0)) { + pr_err("compressor %s not available, using default %s\n", + zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT); + strncpy(zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT, + sizeof(zswap_compressor)); + } + if (!zpool_has_pool(zswap_zpool_type)) { + pr_err("zpool %s not available, using default %s\n", + zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT); + strncpy(zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT, + sizeof(zswap_zpool_type)); + } + + return zswap_pool_create(zswap_zpool_type, zswap_compressor); +} + +static void zswap_pool_destroy(struct zswap_pool *pool) +{ + zswap_pool_debug("destroying", pool); + + zswap_cpu_comp_destroy(pool); + free_percpu(pool->tfm); + zpool_destroy_pool(pool->zpool); + kfree(pool); +} + +static int __must_check zswap_pool_get(struct zswap_pool *pool) +{ + return kref_get_unless_zero(&pool->kref); +} + +static void __zswap_pool_release(struct rcu_head *head) +{ + struct zswap_pool *pool = container_of(head, typeof(*pool), rcu_head); + + /* nobody should have been able to get a kref... */ + WARN_ON(kref_get_unless_zero(&pool->kref)); + + /* pool is now off zswap_pools list and has no references. */ + zswap_pool_destroy(pool); +} + +static void __zswap_pool_empty(struct kref *kref) +{ + struct zswap_pool *pool; + + pool = container_of(kref, typeof(*pool), kref); + + spin_lock(&zswap_pools_lock); + + WARN_ON(pool == zswap_pool_current()); + + list_del_rcu(&pool->list); + call_rcu(&pool->rcu_head, __zswap_pool_release); + + spin_unlock(&zswap_pools_lock); +} + +static void zswap_pool_put(struct zswap_pool *pool) +{ + kref_put(&pool->kref, __zswap_pool_empty); +} + +/********************************* +* param callbacks +**********************************/ + +static int __zswap_param_set(const char *val, const struct kernel_param *kp, + char *type, char *compressor) +{ + struct zswap_pool *pool, *put_pool = NULL; + char str[kp->str->maxlen], *s; + int ret; + + /* + * kp is either zswap_zpool_kparam or zswap_compressor_kparam, defined + * at the top of this file, so maxlen is CRYPTO_MAX_ALG_NAME (64) or + * 32 (arbitrary). + */ + strlcpy(str, val, kp->str->maxlen); + s = strim(str); + + /* if this is load-time (pre-init) param setting, + * don't create a pool; that's done during init. + */ + if (!zswap_init_started) + return param_set_copystring(s, kp); + + /* no change required */ + if (!strncmp(kp->str->string, s, kp->str->maxlen)) + return 0; + + if (!type) { + type = s; + if (!zpool_has_pool(type)) { + pr_err("zpool %s not available\n", type); + return -ENOENT; + } + } else if (!compressor) { + compressor = s; + if (!crypto_has_comp(compressor, 0, 0)) { + pr_err("compressor %s not available\n", compressor); + return -ENOENT; + } + } + + spin_lock(&zswap_pools_lock); + + pool = zswap_pool_find_get(type, compressor); + if (pool) { + zswap_pool_debug("using existing", pool); + list_del_rcu(&pool->list); + } else { + spin_unlock(&zswap_pools_lock); + pool = zswap_pool_create(type, compressor); + spin_lock(&zswap_pools_lock); + } + + if (pool) + ret = param_set_copystring(s, kp); + else + ret = -EINVAL; + + if (!ret) { + put_pool = zswap_pool_current(); + list_add_rcu(&pool->list, &zswap_pools); + } else if (pool) { + /* add the possibly pre-existing pool to the end of the pools + * list; if it's new (and empty) then it'll be removed and + * destroyed by the put after we drop the lock + */ + list_add_tail_rcu(&pool->list, &zswap_pools); + put_pool = pool; + } + + spin_unlock(&zswap_pools_lock); + + /* drop the ref from either the old current pool, + * or the new pool we failed to add + */ + if (put_pool) + zswap_pool_put(put_pool); + + return ret; +} + +static int zswap_compressor_param_set(const char *val, + const struct kernel_param *kp) +{ + return __zswap_param_set(val, kp, zswap_zpool_type, NULL); +} + +static int zswap_zpool_param_set(const char *val, + const struct kernel_param *kp) +{ + return __zswap_param_set(val, kp, NULL, zswap_compressor); } /********************************* @@ -477,6 +835,7 @@ static int zswap_writeback_entry(struct zpool *pool, unsigned long handle) pgoff_t offset; struct zswap_entry *entry; struct page *page; + struct crypto_comp *tfm; u8 *src, *dst; unsigned int dlen; int ret; @@ -517,13 +876,15 @@ static int zswap_writeback_entry(struct zpool *pool, unsigned long handle) case ZSWAP_SWAPCACHE_NEW: /* page is locked */ /* decompress */ dlen = PAGE_SIZE; - src = (u8 *)zpool_map_handle(zswap_pool, entry->handle, + src = (u8 *)zpool_map_handle(entry->pool->zpool, entry->handle, ZPOOL_MM_RO) + sizeof(struct zswap_header); dst = kmap_atomic(page); - ret = zswap_comp_op(ZSWAP_COMPOP_DECOMPRESS, src, - entry->length, dst, &dlen); + tfm = *get_cpu_ptr(entry->pool->tfm); + ret = crypto_comp_decompress(tfm, src, entry->length, + dst, &dlen); + put_cpu_ptr(entry->pool->tfm); kunmap_atomic(dst); - zpool_unmap_handle(zswap_pool, entry->handle); + zpool_unmap_handle(entry->pool->zpool, entry->handle); BUG_ON(ret); BUG_ON(dlen != PAGE_SIZE); @@ -572,6 +933,22 @@ end: return ret; } +static int zswap_shrink(void) +{ + struct zswap_pool *pool; + int ret; + + pool = zswap_pool_last_get(); + if (!pool) + return -ENOENT; + + ret = zpool_shrink(pool->zpool, 1, NULL); + + zswap_pool_put(pool); + + return ret; +} + /********************************* * frontswap hooks **********************************/ @@ -581,6 +958,7 @@ static int zswap_frontswap_store(unsigned type, pgoff_t offset, { struct zswap_tree *tree = zswap_trees[type]; struct zswap_entry *entry, *dupentry; + struct crypto_comp *tfm; int ret; unsigned int dlen = PAGE_SIZE, len; unsigned long handle; @@ -596,7 +974,7 @@ static int zswap_frontswap_store(unsigned type, pgoff_t offset, /* reclaim space if needed */ if (zswap_is_full()) { zswap_pool_limit_hit++; - if (zpool_shrink(zswap_pool, 1, NULL)) { + if (zswap_shrink()) { zswap_reject_reclaim_fail++; ret = -ENOMEM; goto reject; @@ -611,33 +989,42 @@ static int zswap_frontswap_store(unsigned type, pgoff_t offset, goto reject; } + /* if entry is successfully added, it keeps the reference */ + entry->pool = zswap_pool_current_get(); + if (!entry->pool) { + ret = -EINVAL; + goto freepage; + } + /* compress */ dst = get_cpu_var(zswap_dstmem); + tfm = *get_cpu_ptr(entry->pool->tfm); src = kmap_atomic(page); - ret = zswap_comp_op(ZSWAP_COMPOP_COMPRESS, src, PAGE_SIZE, dst, &dlen); + ret = crypto_comp_compress(tfm, src, PAGE_SIZE, dst, &dlen); kunmap_atomic(src); + put_cpu_ptr(entry->pool->tfm); if (ret) { ret = -EINVAL; - goto freepage; + goto put_dstmem; } /* store */ len = dlen + sizeof(struct zswap_header); - ret = zpool_malloc(zswap_pool, len, __GFP_NORETRY | __GFP_NOWARN, - &handle); + ret = zpool_malloc(entry->pool->zpool, len, + __GFP_NORETRY | __GFP_NOWARN, &handle); if (ret == -ENOSPC) { zswap_reject_compress_poor++; - goto freepage; + goto put_dstmem; } if (ret) { zswap_reject_alloc_fail++; - goto freepage; + goto put_dstmem; } - zhdr = zpool_map_handle(zswap_pool, handle, ZPOOL_MM_RW); + zhdr = zpool_map_handle(entry->pool->zpool, handle, ZPOOL_MM_RW); zhdr->swpentry = swp_entry(type, offset); buf = (u8 *)(zhdr + 1); memcpy(buf, dst, dlen); - zpool_unmap_handle(zswap_pool, handle); + zpool_unmap_handle(entry->pool->zpool, handle); put_cpu_var(zswap_dstmem); /* populate entry */ @@ -660,12 +1047,14 @@ static int zswap_frontswap_store(unsigned type, pgoff_t offset, /* update stats */ atomic_inc(&zswap_stored_pages); - zswap_pool_total_size = zpool_get_total_size(zswap_pool); + zswap_update_total_size(); return 0; -freepage: +put_dstmem: put_cpu_var(zswap_dstmem); + zswap_pool_put(entry->pool); +freepage: zswap_entry_cache_free(entry); reject: return ret; @@ -680,6 +1069,7 @@ static int zswap_frontswap_load(unsigned type, pgoff_t offset, { struct zswap_tree *tree = zswap_trees[type]; struct zswap_entry *entry; + struct crypto_comp *tfm; u8 *src, *dst; unsigned int dlen; int ret; @@ -696,13 +1086,14 @@ static int zswap_frontswap_load(unsigned type, pgoff_t offset, /* decompress */ dlen = PAGE_SIZE; - src = (u8 *)zpool_map_handle(zswap_pool, entry->handle, + src = (u8 *)zpool_map_handle(entry->pool->zpool, entry->handle, ZPOOL_MM_RO) + sizeof(struct zswap_header); dst = kmap_atomic(page); - ret = zswap_comp_op(ZSWAP_COMPOP_DECOMPRESS, src, entry->length, - dst, &dlen); + tfm = *get_cpu_ptr(entry->pool->tfm); + ret = crypto_comp_decompress(tfm, src, entry->length, dst, &dlen); + put_cpu_ptr(entry->pool->tfm); kunmap_atomic(dst); - zpool_unmap_handle(zswap_pool, entry->handle); + zpool_unmap_handle(entry->pool->zpool, entry->handle); BUG_ON(ret); spin_lock(&tree->lock); @@ -755,10 +1146,6 @@ static void zswap_frontswap_invalidate_area(unsigned type) zswap_trees[type] = NULL; } -static const struct zpool_ops zswap_zpool_ops = { - .evict = zswap_writeback_entry -}; - static void zswap_frontswap_init(unsigned type) { struct zswap_tree *tree; @@ -839,49 +1226,40 @@ static void __exit zswap_debugfs_exit(void) { } **********************************/ static int __init init_zswap(void) { - gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN; + struct zswap_pool *pool; - pr_info("loading zswap\n"); - - zswap_pool = zpool_create_pool(zswap_zpool_type, "zswap", gfp, - &zswap_zpool_ops); - if (!zswap_pool && strcmp(zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT)) { - pr_info("%s zpool not available\n", zswap_zpool_type); - zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT; - zswap_pool = zpool_create_pool(zswap_zpool_type, "zswap", gfp, - &zswap_zpool_ops); - } - if (!zswap_pool) { - pr_err("%s zpool not available\n", zswap_zpool_type); - pr_err("zpool creation failed\n"); - goto error; - } - pr_info("using %s pool\n", zswap_zpool_type); + zswap_init_started = true; if (zswap_entry_cache_create()) { pr_err("entry cache creation failed\n"); - goto cachefail; + goto cache_fail; } - if (zswap_comp_init()) { - pr_err("compressor initialization failed\n"); - goto compfail; + + if (zswap_cpu_dstmem_init()) { + pr_err("dstmem alloc failed\n"); + goto dstmem_fail; } - if (zswap_cpu_init()) { - pr_err("per-cpu initialization failed\n"); - goto pcpufail; + + pool = __zswap_pool_create_fallback(); + if (!pool) { + pr_err("pool creation failed\n"); + goto pool_fail; } + pr_info("loaded using pool %s/%s\n", pool->tfm_name, + zpool_get_type(pool->zpool)); + + list_add(&pool->list, &zswap_pools); frontswap_register_ops(&zswap_frontswap_ops); if (zswap_debugfs_init()) pr_warn("debugfs initialization failed\n"); return 0; -pcpufail: - zswap_comp_exit(); -compfail: + +pool_fail: + zswap_cpu_dstmem_destroy(); +dstmem_fail: zswap_entry_cache_destroy(); -cachefail: - zpool_destroy_pool(zswap_pool); -error: +cache_fail: return -ENOMEM; } /* must be late so crypto has time to come up */ |