From: Christoph Lameter The zeroing of a page of a arbitrary order in page_alloc.c and in hugetlb.c may benefit from a clear_page that is capable of zeroing multiple pages at once. The following patch adds a function "clear_pages" that is capable of clearing multiple continuous pages at once. Signed-off-by: Christoph Lameter Signed-off-by: Andrew Morton --- 25-akpm/arch/sparc64/Kconfig | 4 ++++ 25-akpm/arch/sparc64/lib/clear_page.S | 11 +++++++---- 25-akpm/arch/x86_64/Kconfig | 4 ++++ 25-akpm/arch/x86_64/kernel/x8664_ksyms.c | 2 +- 25-akpm/arch/x86_64/lib/clear_page.S | 29 ++++++++++++++++++----------- 25-akpm/include/asm-i386/mmx.h | 2 +- 25-akpm/include/asm-i386/page.h | 5 +++-- 25-akpm/include/asm-ia64/page.h | 3 ++- 25-akpm/include/asm-sparc64/page.h | 6 ++++-- 25-akpm/include/asm-x86_64/page.h | 3 ++- 25-akpm/include/linux/gfp.h | 1 + 25-akpm/mm/hugetlb.c | 4 +--- 25-akpm/mm/page_alloc.c | 11 ++++++++++- arch/i386/Kconfig | 0 arch/i386/lib/mmx.c | 0 arch/ia64/Kconfig | 0 arch/ia64/kernel/ia64_ksyms.c | 0 arch/ia64/lib/clear_page.S | 0 18 files changed, 58 insertions(+), 27 deletions(-) diff -puN arch/i386/Kconfig~add-a-clear_pages-function-to-clear-pages-of-higher arch/i386/Kconfig diff -puN arch/i386/lib/mmx.c~add-a-clear_pages-function-to-clear-pages-of-higher arch/i386/lib/mmx.c diff -puN arch/ia64/Kconfig~add-a-clear_pages-function-to-clear-pages-of-higher arch/ia64/Kconfig diff -puN arch/ia64/kernel/ia64_ksyms.c~add-a-clear_pages-function-to-clear-pages-of-higher arch/ia64/kernel/ia64_ksyms.c diff -puN arch/ia64/lib/clear_page.S~add-a-clear_pages-function-to-clear-pages-of-higher arch/ia64/lib/clear_page.S diff -puN arch/sparc64/Kconfig~add-a-clear_pages-function-to-clear-pages-of-higher arch/sparc64/Kconfig --- 25/arch/sparc64/Kconfig~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700 +++ 25-akpm/arch/sparc64/Kconfig 2005-04-04 01:54:48.000000000 -0700 @@ -43,6 +43,10 @@ config SPARC64_PAGE_SIZE_4MB endchoice +config CLEAR_PAGES + bool + default y + source "init/Kconfig" config SYSVIPC_COMPAT diff -puN arch/sparc64/lib/clear_page.S~add-a-clear_pages-function-to-clear-pages-of-higher arch/sparc64/lib/clear_page.S --- 25/arch/sparc64/lib/clear_page.S~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700 +++ 25-akpm/arch/sparc64/lib/clear_page.S 2005-04-04 01:54:48.000000000 -0700 @@ -28,9 +28,12 @@ .text .globl _clear_page -_clear_page: /* %o0=dest */ +_clear_page: /* %o0=dest, %o1=order */ + sethi %hi(PAGE_SIZE/64), %o2 + clr %o4 + or %o2, %lo(PAGE_SIZE/64), %o2 ba,pt %xcc, clear_page_common - clr %o4 + sllx %o2, %o1, %o1 /* This thing is pretty important, it shows up * on the profiles via do_anonymous_page(). @@ -69,16 +72,16 @@ clear_user_page: /* %o0=dest, %o1=vaddr flush %g6 wrpr %o4, 0x0, %pstate + sethi %hi(PAGE_SIZE/64), %o1 mov 1, %o4 + or %o1, %lo(PAGE_SIZE/64), %o1 clear_page_common: VISEntryHalf membar #StoreLoad | #StoreStore | #LoadStore fzero %f0 - sethi %hi(PAGE_SIZE/64), %o1 mov %o0, %g1 ! remember vaddr for tlbflush fzero %f2 - or %o1, %lo(PAGE_SIZE/64), %o1 faddd %f0, %f2, %f4 fmuld %f0, %f2, %f6 faddd %f0, %f2, %f8 diff -puN arch/x86_64/Kconfig~add-a-clear_pages-function-to-clear-pages-of-higher arch/x86_64/Kconfig --- 25/arch/x86_64/Kconfig~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700 +++ 25-akpm/arch/x86_64/Kconfig 2005-04-04 01:54:48.000000000 -0700 @@ -61,6 +61,10 @@ config GENERIC_IOMAP bool default y +config CLEAR_PAGES + bool + default y + source "init/Kconfig" diff -puN arch/x86_64/kernel/x8664_ksyms.c~add-a-clear_pages-function-to-clear-pages-of-higher arch/x86_64/kernel/x8664_ksyms.c --- 25/arch/x86_64/kernel/x8664_ksyms.c~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700 +++ 25-akpm/arch/x86_64/kernel/x8664_ksyms.c 2005-04-04 01:54:48.000000000 -0700 @@ -107,7 +107,7 @@ EXPORT_SYMBOL(pci_mem_start); #endif EXPORT_SYMBOL(copy_page); -EXPORT_SYMBOL(clear_page); +EXPORT_SYMBOL(clear_pages); EXPORT_SYMBOL(cpu_pda); #ifdef CONFIG_SMP diff -puN arch/x86_64/lib/clear_page.S~add-a-clear_pages-function-to-clear-pages-of-higher arch/x86_64/lib/clear_page.S --- 25/arch/x86_64/lib/clear_page.S~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700 +++ 25-akpm/arch/x86_64/lib/clear_page.S 2005-04-04 01:54:48.000000000 -0700 @@ -1,12 +1,16 @@ /* * Zero a page. * rdi page + * rsi order */ - .globl clear_page + .globl clear_pages .p2align 4 -clear_page: +clear_pages: + movl $4096/64,%eax + movl %esi, %ecx + shll %cl, %eax + movl %eax, %ecx xorl %eax,%eax - movl $4096/64,%ecx .p2align 4 .Lloop: decl %ecx @@ -23,7 +27,7 @@ clear_page: jnz .Lloop nop ret -clear_page_end: +clear_pages_end: /* C stepping K8 run faster using the string instructions. It is also a lot simpler. Use this when possible */ @@ -32,19 +36,22 @@ clear_page_end: .section .altinstructions,"a" .align 8 - .quad clear_page - .quad clear_page_c + .quad clear_pages + .quad clear_pages_c .byte X86_FEATURE_K8_C - .byte clear_page_end-clear_page - .byte clear_page_c_end-clear_page_c + .byte clear_pages_end-clear_pages + .byte clear_pages_c_end-clear_pages_c .previous .section .altinstr_replacement,"ax" -clear_page_c: - movl $4096/8,%ecx +clear_pages_c: + movl $4096/8,%eax + movl %esi, %ecx + shll %cl, %eax + movl %eax, %ecx xorl %eax,%eax rep stosq ret -clear_page_c_end: +clear_pages_c_end: .previous diff -puN include/asm-i386/mmx.h~add-a-clear_pages-function-to-clear-pages-of-higher include/asm-i386/mmx.h --- 25/include/asm-i386/mmx.h~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700 +++ 25-akpm/include/asm-i386/mmx.h 2005-04-04 01:54:48.000000000 -0700 @@ -8,7 +8,7 @@ #include extern void *_mmx_memcpy(void *to, const void *from, size_t size); -extern void mmx_clear_page(void *page); +extern void mmx_clear_page(void *page, int order); extern void mmx_copy_page(void *to, void *from); #endif diff -puN include/asm-i386/page.h~add-a-clear_pages-function-to-clear-pages-of-higher include/asm-i386/page.h --- 25/include/asm-i386/page.h~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700 +++ 25-akpm/include/asm-i386/page.h 2005-04-04 01:54:48.000000000 -0700 @@ -18,7 +18,7 @@ #include -#define clear_page(page) mmx_clear_page((void *)(page)) +#define clear_pages(page, order) mmx_clear_page((void *)(page),order) #define copy_page(to,from) mmx_copy_page(to,from) #else @@ -28,11 +28,12 @@ * Maybe the K6-III ? */ -#define clear_page(page) memset((void *)(page), 0, PAGE_SIZE) +#define clear_pages(page, order) memset((void *)(page), 0, PAGE_SIZE << (order)) #define copy_page(to,from) memcpy((void *)(to), (void *)(from), PAGE_SIZE) #endif +#define clear_page(page) clear_pages(page, 0) #define clear_user_page(page, vaddr, pg) clear_page(page) #define copy_user_page(to, from, vaddr, pg) copy_page(to, from) diff -puN include/asm-ia64/page.h~add-a-clear_pages-function-to-clear-pages-of-higher include/asm-ia64/page.h --- 25/include/asm-ia64/page.h~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700 +++ 25-akpm/include/asm-ia64/page.h 2005-04-04 01:54:48.000000000 -0700 @@ -56,8 +56,9 @@ # ifdef __KERNEL__ # define STRICT_MM_TYPECHECKS -extern void clear_page (void *page); +extern void clear_pages (void *page, int order); extern void copy_page (void *to, void *from); +#define clear_page(__page) clear_pages(__page, 0) /* * clear_user_page() and copy_user_page() can't be inline functions because diff -puN include/asm-sparc64/page.h~add-a-clear_pages-function-to-clear-pages-of-higher include/asm-sparc64/page.h --- 25/include/asm-sparc64/page.h~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700 +++ 25-akpm/include/asm-sparc64/page.h 2005-04-04 01:54:48.000000000 -0700 @@ -25,8 +25,10 @@ #ifndef __ASSEMBLY__ -extern void _clear_page(void *page); -#define clear_page(X) _clear_page((void *)(X)) +extern void _clear_page(void *page, int order); +#define clear_page(X) _clear_page((void *)(X), 0) +#define clear_pages _clear_page + struct page; extern void clear_user_page(void *addr, unsigned long vaddr, struct page *page); #define copy_page(X,Y) memcpy((void *)(X), (void *)(Y), PAGE_SIZE) diff -puN include/asm-x86_64/page.h~add-a-clear_pages-function-to-clear-pages-of-higher include/asm-x86_64/page.h --- 25/include/asm-x86_64/page.h~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700 +++ 25-akpm/include/asm-x86_64/page.h 2005-04-04 01:54:48.000000000 -0700 @@ -32,8 +32,9 @@ #ifdef __KERNEL__ #ifndef __ASSEMBLY__ -void clear_page(void *); +void clear_pages(void *, int); void copy_page(void *, void *); +#define clear_page(__page) clear_pages(__page, 0) #define clear_user_page(page, vaddr, pg) clear_page(page) #define copy_user_page(to, from, vaddr, pg) copy_page(to, from) diff -puN include/linux/gfp.h~add-a-clear_pages-function-to-clear-pages-of-higher include/linux/gfp.h --- 25/include/linux/gfp.h~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700 +++ 25-akpm/include/linux/gfp.h 2005-04-04 01:54:48.000000000 -0700 @@ -131,4 +131,5 @@ extern void FASTCALL(free_cold_page(stru void page_alloc_init(void); +void prep_zero_page(struct page *, int order, unsigned int __nocast gfp_flags); #endif /* __LINUX_GFP_H */ diff -puN mm/hugetlb.c~add-a-clear_pages-function-to-clear-pages-of-higher mm/hugetlb.c --- 25/mm/hugetlb.c~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700 +++ 25-akpm/mm/hugetlb.c 2005-04-04 01:54:48.000000000 -0700 @@ -78,7 +78,6 @@ void free_huge_page(struct page *page) struct page *alloc_huge_page(void) { struct page *page; - int i; spin_lock(&hugetlb_lock); page = dequeue_huge_page(); @@ -89,8 +88,7 @@ struct page *alloc_huge_page(void) spin_unlock(&hugetlb_lock); set_page_count(page, 1); page[1].mapping = (void *)free_huge_page; - for (i = 0; i < (HPAGE_SIZE/PAGE_SIZE); ++i) - clear_highpage(&page[i]); + prep_zero_page(page, HUGETLB_PAGE_ORDER, GFP_HIGHUSER); return page; } diff -puN mm/page_alloc.c~add-a-clear_pages-function-to-clear-pages-of-higher mm/page_alloc.c --- 25/mm/page_alloc.c~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700 +++ 25-akpm/mm/page_alloc.c 2005-04-04 01:54:48.000000000 -0700 @@ -633,11 +633,20 @@ void fastcall free_cold_page(struct page free_hot_cold_page(page, 1); } -static inline void prep_zero_page(struct page *page, int order, unsigned int __nocast gfp_flags) +void prep_zero_page(struct page *page, int order, + unsigned int __nocast gfp_flags) { int i; BUG_ON((gfp_flags & (__GFP_WAIT | __GFP_HIGHMEM)) == __GFP_HIGHMEM); + +#ifdef CONFIG_CLEAR_PAGES + if (!PageHighMem(page)) { + clear_pages(page_address(page), order); + return; + } +#endif + for(i = 0; i < (1 << order); i++) clear_highpage(page + i); } _