csky: MMU and page table management

This patch adds files related to memory management and here is our
memory-layout:

   Fixmap       : 0xffc02000 – 0xfffff000       (4 MB - 12KB)
   Pkmap        : 0xff800000 – 0xffc00000       (4 MB)
   Vmalloc      : 0xf0200000 – 0xff000000       (238 MB)
   Lowmem       : 0x80000000 – 0xc0000000       (1GB)

abiv1 CPU (CK610) is VIPT cache and it doesn't support highmem.
abiv2 CPUs are all PIPT cache and they could support highmem.

Lowmem is directly mapped by msa0 & msa1 reg, and we needn't setup
memory page table for it.

Link:https://lore.kernel.org/lkml/20180518215548.GH17671@n2100.armlinux.org.uk/
Signed-off-by: Guo Ren <ren_guo@c-sky.com>
Cc: Christoph Hellwig <hch@infradead.org>
Reviewed-by: Arnd Bergmann <arnd@arndb.de>

4 files changed, 622 insertions, 0 deletions

diff --git a/arch/csky/mm/dma-mapping.c b/arch/csky/mm/dma-mapping.c
new file mode 100644
index 000000000000..85437b21e045
--- /dev/null
+++ b/arch/csky/mm/dma-mapping.c
@@ -0,0 +1,254 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd.
+
+#include <linux/cache.h>
+#include <linux/dma-mapping.h>
+#include <linux/dma-contiguous.h>
+#include <linux/dma-noncoherent.h>
+#include <linux/genalloc.h>
+#include <linux/highmem.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/scatterlist.h>
+#include <linux/types.h>
+#include <linux/version.h>
+#include <asm/cache.h>
+
+static struct gen_pool *atomic_pool;
+static size_t atomic_pool_size __initdata = SZ_256K;
+
+static int __init early_coherent_pool(char *p)
+{
+	atomic_pool_size = memparse(p, &p);
+	return 0;
+}
+early_param("coherent_pool", early_coherent_pool);
+
+static int __init atomic_pool_init(void)
+{
+	struct page *page;
+	size_t size = atomic_pool_size;
+	void *ptr;
+	int ret;
+
+	atomic_pool = gen_pool_create(PAGE_SHIFT, -1);
+	if (!atomic_pool)
+		BUG();
+
+	page = alloc_pages(GFP_KERNEL | GFP_DMA, get_order(size));
+	if (!page)
+		BUG();
+
+	ptr = dma_common_contiguous_remap(page, size, VM_ALLOC,
+					  pgprot_noncached(PAGE_KERNEL),
+					  __builtin_return_address(0));
+	if (!ptr)
+		BUG();
+
+	ret = gen_pool_add_virt(atomic_pool, (unsigned long)ptr,
+				page_to_phys(page), atomic_pool_size, -1);
+	if (ret)
+		BUG();
+
+	gen_pool_set_algo(atomic_pool, gen_pool_first_fit_order_align, NULL);
+
+	pr_info("DMA: preallocated %zu KiB pool for atomic coherent pool\n",
+		atomic_pool_size / 1024);
+
+	pr_info("DMA: vaddr: 0x%x phy: 0x%lx,\n", (unsigned int)ptr,
+		page_to_phys(page));
+
+	return 0;
+}
+postcore_initcall(atomic_pool_init);
+
+static void *csky_dma_alloc_atomic(struct device *dev, size_t size,
+				   dma_addr_t *dma_handle)
+{
+	unsigned long addr;
+
+	addr = gen_pool_alloc(atomic_pool, size);
+	if (addr)
+		*dma_handle = gen_pool_virt_to_phys(atomic_pool, addr);
+
+	return (void *)addr;
+}
+
+static void csky_dma_free_atomic(struct device *dev, size_t size, void *vaddr,
+				 dma_addr_t dma_handle, unsigned long attrs)
+{
+	gen_pool_free(atomic_pool, (unsigned long)vaddr, size);
+}
+
+static void __dma_clear_buffer(struct page *page, size_t size)
+{
+	if (PageHighMem(page)) {
+		unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
+
+		do {
+			void *ptr = kmap_atomic(page);
+			size_t _size = (size < PAGE_SIZE) ? size : PAGE_SIZE;
+
+			memset(ptr, 0, _size);
+			dma_wbinv_range((unsigned long)ptr,
+					(unsigned long)ptr + _size);
+
+			kunmap_atomic(ptr);
+
+			page++;
+			size -= PAGE_SIZE;
+			count--;
+		} while (count);
+	} else {
+		void *ptr = page_address(page);
+
+		memset(ptr, 0, size);
+		dma_wbinv_range((unsigned long)ptr, (unsigned long)ptr + size);
+	}
+}
+
+static void *csky_dma_alloc_nonatomic(struct device *dev, size_t size,
+				      dma_addr_t *dma_handle, gfp_t gfp,
+				      unsigned long attrs)
+{
+	void  *vaddr;
+	struct page *page;
+	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
+
+	if (DMA_ATTR_NON_CONSISTENT & attrs) {
+		pr_err("csky %s can't support DMA_ATTR_NON_CONSISTENT.\n", __func__);
+		return NULL;
+	}
+
+	if (IS_ENABLED(CONFIG_DMA_CMA))
+		page = dma_alloc_from_contiguous(dev, count, get_order(size),
+						 gfp);
+	else
+		page = alloc_pages(gfp, get_order(size));
+
+	if (!page) {
+		pr_err("csky %s no more free pages.\n", __func__);
+		return NULL;
+	}
+
+	*dma_handle = page_to_phys(page);
+
+	__dma_clear_buffer(page, size);
+
+	if (attrs & DMA_ATTR_NO_KERNEL_MAPPING)
+		return page;
+
+	vaddr = dma_common_contiguous_remap(page, PAGE_ALIGN(size), VM_USERMAP,
+		pgprot_noncached(PAGE_KERNEL), __builtin_return_address(0));
+	if (!vaddr)
+		BUG();
+
+	return vaddr;
+}
+
+static void csky_dma_free_nonatomic(
+	struct device *dev,
+	size_t size,
+	void *vaddr,
+	dma_addr_t dma_handle,
+	unsigned long attrs
+	)
+{
+	struct page *page = phys_to_page(dma_handle);
+	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
+
+	if ((unsigned int)vaddr >= VMALLOC_START)
+		dma_common_free_remap(vaddr, size, VM_USERMAP);
+
+	if (IS_ENABLED(CONFIG_DMA_CMA))
+		dma_release_from_contiguous(dev, page, count);
+	else
+		__free_pages(page, get_order(size));
+}
+
+void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
+		     gfp_t gfp, unsigned long attrs)
+{
+	if (gfpflags_allow_blocking(gfp))
+		return csky_dma_alloc_nonatomic(dev, size, dma_handle, gfp,
+						attrs);
+	else
+		return csky_dma_alloc_atomic(dev, size, dma_handle);
+}
+
+void arch_dma_free(struct device *dev, size_t size, void *vaddr,
+		   dma_addr_t dma_handle, unsigned long attrs)
+{
+	if (!addr_in_gen_pool(atomic_pool, (unsigned int) vaddr, size))
+		csky_dma_free_nonatomic(dev, size, vaddr, dma_handle, attrs);
+	else
+		csky_dma_free_atomic(dev, size, vaddr, dma_handle, attrs);
+}
+
+static inline void cache_op(phys_addr_t paddr, size_t size,
+			    void (*fn)(unsigned long start, unsigned long end))
+{
+	struct page *page = pfn_to_page(paddr >> PAGE_SHIFT);
+	unsigned int offset = paddr & ~PAGE_MASK;
+	size_t left = size;
+	unsigned long start;
+
+	do {
+		size_t len = left;
+
+		if (PageHighMem(page)) {
+			void *addr;
+
+			if (offset + len > PAGE_SIZE) {
+				if (offset >= PAGE_SIZE) {
+					page += offset >> PAGE_SHIFT;
+					offset &= ~PAGE_MASK;
+				}
+				len = PAGE_SIZE - offset;
+			}
+
+			addr = kmap_atomic(page);
+			start = (unsigned long)(addr + offset);
+			fn(start, start + len);
+			kunmap_atomic(addr);
+		} else {
+			start = (unsigned long)phys_to_virt(paddr);
+			fn(start, start + size);
+		}
+		offset = 0;
+		page++;
+		left -= len;
+	} while (left);
+}
+
+void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr,
+			      size_t size, enum dma_data_direction dir)
+{
+	switch (dir) {
+	case DMA_TO_DEVICE:
+		cache_op(paddr, size, dma_wb_range);
+		break;
+	case DMA_FROM_DEVICE:
+	case DMA_BIDIRECTIONAL:
+		cache_op(paddr, size, dma_wbinv_range);
+		break;
+	default:
+		BUG();
+	}
+}
+
+void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr,
+			   size_t size, enum dma_data_direction dir)
+{
+	switch (dir) {
+	case DMA_TO_DEVICE:
+		cache_op(paddr, size, dma_wb_range);
+		break;
+	case DMA_FROM_DEVICE:
+	case DMA_BIDIRECTIONAL:
+		cache_op(paddr, size, dma_wbinv_range);
+		break;
+	default:
+		BUG();
+	}
+}
diff --git a/arch/csky/mm/highmem.c b/arch/csky/mm/highmem.c
new file mode 100644
index 000000000000..e168ac087ccb
--- /dev/null
+++ b/arch/csky/mm/highmem.c
@@ -0,0 +1,198 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd.
+
+#include <linux/module.h>
+#include <linux/highmem.h>
+#include <linux/smp.h>
+#include <linux/bootmem.h>
+#include <asm/fixmap.h>
+#include <asm/tlbflush.h>
+#include <asm/cacheflush.h>
+
+static pte_t *kmap_pte;
+
+unsigned long highstart_pfn, highend_pfn;
+
+void *kmap(struct page *page)
+{
+	void *addr;
+
+	might_sleep();
+	if (!PageHighMem(page))
+		return page_address(page);
+	addr = kmap_high(page);
+	flush_tlb_one((unsigned long)addr);
+
+	return addr;
+}
+EXPORT_SYMBOL(kmap);
+
+void kunmap(struct page *page)
+{
+	BUG_ON(in_interrupt());
+	if (!PageHighMem(page))
+		return;
+	kunmap_high(page);
+}
+EXPORT_SYMBOL(kunmap);
+
+void *kmap_atomic(struct page *page)
+{
+	unsigned long vaddr;
+	int idx, type;
+
+	preempt_disable();
+	pagefault_disable();
+	if (!PageHighMem(page))
+		return page_address(page);
+
+	type = kmap_atomic_idx_push();
+	idx = type + KM_TYPE_NR*smp_processor_id();
+	vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
+#ifdef CONFIG_DEBUG_HIGHMEM
+	BUG_ON(!pte_none(*(kmap_pte - idx)));
+#endif
+	set_pte(kmap_pte-idx, mk_pte(page, PAGE_KERNEL));
+	flush_tlb_one((unsigned long)vaddr);
+
+	return (void *)vaddr;
+}
+EXPORT_SYMBOL(kmap_atomic);
+
+void __kunmap_atomic(void *kvaddr)
+{
+	unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK;
+	int idx;
+
+	if (vaddr < FIXADDR_START)
+		goto out;
+
+#ifdef CONFIG_DEBUG_HIGHMEM
+	idx = KM_TYPE_NR*smp_processor_id() + kmap_atomic_idx();
+
+	BUG_ON(vaddr != __fix_to_virt(FIX_KMAP_BEGIN + idx));
+
+	pte_clear(&init_mm, vaddr, kmap_pte - idx);
+	flush_tlb_one(vaddr);
+#else
+	(void) idx; /* to kill a warning */
+#endif
+	kmap_atomic_idx_pop();
+out:
+	pagefault_enable();
+	preempt_enable();
+}
+EXPORT_SYMBOL(__kunmap_atomic);
+
+/*
+ * This is the same as kmap_atomic() but can map memory that doesn't
+ * have a struct page associated with it.
+ */
+void *kmap_atomic_pfn(unsigned long pfn)
+{
+	unsigned long vaddr;
+	int idx, type;
+
+	pagefault_disable();
+
+	type = kmap_atomic_idx_push();
+	idx = type + KM_TYPE_NR*smp_processor_id();
+	vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
+	set_pte(kmap_pte-idx, pfn_pte(pfn, PAGE_KERNEL));
+	flush_tlb_one(vaddr);
+
+	return (void *) vaddr;
+}
+
+struct page *kmap_atomic_to_page(void *ptr)
+{
+	unsigned long idx, vaddr = (unsigned long)ptr;
+	pte_t *pte;
+
+	if (vaddr < FIXADDR_START)
+		return virt_to_page(ptr);
+
+	idx = virt_to_fix(vaddr);
+	pte = kmap_pte - (idx - FIX_KMAP_BEGIN);
+	return pte_page(*pte);
+}
+
+static void __init fixrange_init(unsigned long start, unsigned long end,
+				pgd_t *pgd_base)
+{
+#ifdef CONFIG_HIGHMEM
+	pgd_t *pgd;
+	pud_t *pud;
+	pmd_t *pmd;
+	pte_t *pte;
+	int i, j, k;
+	unsigned long vaddr;
+
+	vaddr = start;
+	i = __pgd_offset(vaddr);
+	j = __pud_offset(vaddr);
+	k = __pmd_offset(vaddr);
+	pgd = pgd_base + i;
+
+	for ( ; (i < PTRS_PER_PGD) && (vaddr != end); pgd++, i++) {
+		pud = (pud_t *)pgd;
+		for ( ; (j < PTRS_PER_PUD) && (vaddr != end); pud++, j++) {
+			pmd = (pmd_t *)pud;
+			for (; (k < PTRS_PER_PMD) && (vaddr != end); pmd++, k++) {
+				if (pmd_none(*pmd)) {
+					pte = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE);
+					set_pmd(pmd, __pmd(__pa(pte)));
+					BUG_ON(pte != pte_offset_kernel(pmd, 0));
+				}
+				vaddr += PMD_SIZE;
+			}
+			k = 0;
+		}
+		j = 0;
+	}
+#endif
+}
+
+void __init fixaddr_kmap_pages_init(void)
+{
+	unsigned long vaddr;
+	pgd_t *pgd_base;
+#ifdef CONFIG_HIGHMEM
+	pgd_t *pgd;
+	pmd_t *pmd;
+	pud_t *pud;
+	pte_t *pte;
+#endif
+	pgd_base = swapper_pg_dir;
+
+	/*
+	 * Fixed mappings:
+	 */
+	vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
+	fixrange_init(vaddr, 0, pgd_base);
+
+#ifdef CONFIG_HIGHMEM
+	/*
+	 * Permanent kmaps:
+	 */
+	vaddr = PKMAP_BASE;
+	fixrange_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base);
+
+	pgd = swapper_pg_dir + __pgd_offset(vaddr);
+	pud = (pud_t *)pgd;
+	pmd = pmd_offset(pud, vaddr);
+	pte = pte_offset_kernel(pmd, vaddr);
+	pkmap_page_table = pte;
+#endif
+}
+
+void __init kmap_init(void)
+{
+	unsigned long vaddr;
+
+	fixaddr_kmap_pages_init();
+
+	vaddr = __fix_to_virt(FIX_KMAP_BEGIN);
+
+	kmap_pte = pte_offset_kernel((pmd_t *)pgd_offset_k(vaddr), vaddr);
+}
diff --git a/arch/csky/mm/init.c b/arch/csky/mm/init.c
new file mode 100644
index 000000000000..ce2711e050ad
--- /dev/null
+++ b/arch/csky/mm/init.c
@@ -0,0 +1,122 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd.
+
+#include <linux/bug.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/signal.h>
+#include <linux/sched.h>
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/string.h>
+#include <linux/types.h>
+#include <linux/pagemap.h>
+#include <linux/ptrace.h>
+#include <linux/mman.h>
+#include <linux/mm.h>
+#include <linux/bootmem.h>
+#include <linux/highmem.h>
+#include <linux/memblock.h>
+#include <linux/swap.h>
+#include <linux/proc_fs.h>
+#include <linux/pfn.h>
+
+#include <asm/setup.h>
+#include <asm/cachectl.h>
+#include <asm/dma.h>
+#include <asm/pgtable.h>
+#include <asm/pgalloc.h>
+#include <asm/mmu_context.h>
+#include <asm/sections.h>
+#include <asm/tlb.h>
+
+pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss;
+pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss;
+unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]
+						__page_aligned_bss;
+EXPORT_SYMBOL(empty_zero_page);
+
+void __init mem_init(void)
+{
+#ifdef CONFIG_HIGHMEM
+	unsigned long tmp;
+
+	max_mapnr = highend_pfn;
+#else
+	max_mapnr = max_low_pfn;
+#endif
+	high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT);
+
+	free_all_bootmem();
+
+#ifdef CONFIG_HIGHMEM
+	for (tmp = highstart_pfn; tmp < highend_pfn; tmp++) {
+		struct page *page = pfn_to_page(tmp);
+
+		/* FIXME not sure about */
+		if (!memblock_is_reserved(tmp << PAGE_SHIFT))
+			free_highmem_page(page);
+	}
+#endif
+	mem_init_print_info(NULL);
+}
+
+#ifdef CONFIG_BLK_DEV_INITRD
+void free_initrd_mem(unsigned long start, unsigned long end)
+{
+	if (start < end)
+		pr_info("Freeing initrd memory: %ldk freed\n",
+			(end - start) >> 10);
+
+	for (; start < end; start += PAGE_SIZE) {
+		ClearPageReserved(virt_to_page(start));
+		init_page_count(virt_to_page(start));
+		free_page(start);
+		totalram_pages++;
+	}
+}
+#endif
+
+extern char __init_begin[], __init_end[];
+
+void free_initmem(void)
+{
+	unsigned long addr;
+
+	addr = (unsigned long) &__init_begin;
+
+	while (addr < (unsigned long) &__init_end) {
+		ClearPageReserved(virt_to_page(addr));
+		init_page_count(virt_to_page(addr));
+		free_page(addr);
+		totalram_pages++;
+		addr += PAGE_SIZE;
+	}
+
+	pr_info("Freeing unused kernel memory: %dk freed\n",
+	((unsigned int)&__init_end - (unsigned int)&__init_begin) >> 10);
+}
+
+void pgd_init(unsigned long *p)
+{
+	int i;
+
+	for (i = 0; i < PTRS_PER_PGD; i++)
+		p[i] = __pa(invalid_pte_table);
+}
+
+void __init pre_mmu_init(void)
+{
+	/*
+	 * Setup page-table and enable TLB-hardrefill
+	 */
+	flush_tlb_all();
+	pgd_init((unsigned long *)swapper_pg_dir);
+	TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir);
+	TLBMISS_HANDLER_SETUP_PGD_KERNEL(swapper_pg_dir);
+
+	asid_cache(smp_processor_id()) = ASID_FIRST_VERSION;
+
+	/* Setup page mask to 4k */
+	write_mmu_pagemask(0);
+}
diff --git a/arch/csky/mm/ioremap.c b/arch/csky/mm/ioremap.c
new file mode 100644
index 000000000000..7ad3ff103f4a
--- /dev/null
+++ b/arch/csky/mm/ioremap.c
@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd.
+
+#include <linux/export.h>
+#include <linux/mm.h>
+#include <linux/vmalloc.h>
+#include <linux/io.h>
+
+#include <asm/pgtable.h>
+
+void __iomem *ioremap(phys_addr_t addr, size_t size)
+{
+	phys_addr_t last_addr;
+	unsigned long offset, vaddr;
+	struct vm_struct *area;
+	pgprot_t prot;
+
+	last_addr = addr + size - 1;
+	if (!size || last_addr < addr)
+		return NULL;
+
+	offset = addr & (~PAGE_MASK);
+	addr &= PAGE_MASK;
+	size = PAGE_ALIGN(size + offset);
+
+	area = get_vm_area_caller(size, VM_ALLOC, __builtin_return_address(0));
+	if (!area)
+		return NULL;
+
+	vaddr = (unsigned long)area->addr;
+
+	prot = __pgprot(_PAGE_PRESENT | __READABLE | __WRITEABLE |
+			_PAGE_GLOBAL | _CACHE_UNCACHED);
+
+	if (ioremap_page_range(vaddr, vaddr + size, addr, prot)) {
+		free_vm_area(area);
+		return NULL;
+	}
+
+	return (void __iomem *)(vaddr + offset);
+}
+EXPORT_SYMBOL(ioremap);
+
+void iounmap(void __iomem *addr)
+{
+	vunmap((void *)((unsigned long)addr & PAGE_MASK));
+}
+EXPORT_SYMBOL(iounmap);