From 7ff00a49a2329ec3339da5059b36c6c1cbe6a3cc Mon Sep 17 00:00:00 2001 From: Heesub Shin Date: Sun, 6 Jan 2013 18:10:00 -0800 Subject: [PATCH] ANDROID: cma: redirect page allocation to CMA CMA pages are designed to be used as fallback for movable allocations and cannot be used for non-movable allocations. If CMA pages are utilized poorly, non-movable allocations may end up getting starved if all regular movable pages are allocated and the only pages left are CMA. Always using CMA pages first creates unacceptable performance problems. As a midway alternative, use CMA pages for certain userspace allocations. The userspace pages can be migrated or dropped quickly which giving decent utilization. Additionally, add a fall-backs for failed CMA allocations in rmqueue() and __rmqueue_pcplist() (the latter addition being driven by a report by the kernel test robot); these fallbacks were dealt with differently in the original version of the patch as the rmqueue() call chain has changed). Bug: 158645321 Link: https://lore.kernel.org/lkml/cover.1604282969.git.cgoldswo@codeaurora.org/ Reported-by: kernel test robot Signed-off-by: Kyungmin Park Signed-off-by: Heesub Shin Signed-off-by: Vinayak Menon [cgoldswo@codeaurora.org: Place in bugfixes; remove cma_alloc zone flag] Signed-off-by: Chris Goldsworthy Change-Id: Ibca5eedfc5eacd44542ad483851d741166715f84 --- include/linux/gfp.h | 15 ++++++++ include/linux/highmem.h | 4 +- include/linux/mmzone.h | 2 + mm/page_alloc.c | 83 ++++++++++++++++++++++++++--------------- 4 files changed, 73 insertions(+), 31 deletions(-) diff --git a/include/linux/gfp.h b/include/linux/gfp.h index c603237e006c..e80b7d2f5b38 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -39,11 +39,21 @@ struct vm_area_struct; #define ___GFP_HARDWALL 0x100000u #define ___GFP_THISNODE 0x200000u #define ___GFP_ACCOUNT 0x400000u +#ifdef CONFIG_CMA +#define ___GFP_CMA 0x800000u +#else +#define ___GFP_CMA 0 +#endif #ifdef CONFIG_LOCKDEP +#ifdef CONFIG_CMA +#define ___GFP_NOLOCKDEP 0x1000000u +#else #define ___GFP_NOLOCKDEP 0x800000u +#endif #else #define ___GFP_NOLOCKDEP 0 #endif + /* If the above are modified, __GFP_BITS_SHIFT may need updating */ /* @@ -57,6 +67,7 @@ struct vm_area_struct; #define __GFP_HIGHMEM ((__force gfp_t)___GFP_HIGHMEM) #define __GFP_DMA32 ((__force gfp_t)___GFP_DMA32) #define __GFP_MOVABLE ((__force gfp_t)___GFP_MOVABLE) /* ZONE_MOVABLE allowed */ +#define __GFP_CMA ((__force gfp_t)___GFP_CMA) #define GFP_ZONEMASK (__GFP_DMA|__GFP_HIGHMEM|__GFP_DMA32|__GFP_MOVABLE) /** @@ -224,7 +235,11 @@ struct vm_area_struct; #define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP) /* Room for N __GFP_FOO bits */ +#ifdef CONFIG_CMA +#define __GFP_BITS_SHIFT (24 + IS_ENABLED(CONFIG_LOCKDEP)) +#else #define __GFP_BITS_SHIFT (23 + IS_ENABLED(CONFIG_LOCKDEP)) +#endif #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) /** diff --git a/include/linux/highmem.h b/include/linux/highmem.h index 14e6202ce47f..35f052babdba 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -274,7 +274,9 @@ static inline struct page * alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma, unsigned long vaddr) { - return __alloc_zeroed_user_highpage(__GFP_MOVABLE, vma, vaddr); + return __alloc_zeroed_user_highpage( + __GFP_MOVABLE|__GFP_CMA, vma, + vaddr); } static inline void clear_highpage(struct page *page) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index a140d856e354..278c8f47259e 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -72,9 +72,11 @@ extern const char * const migratetype_names[MIGRATE_TYPES]; #ifdef CONFIG_CMA # define is_migrate_cma(migratetype) unlikely((migratetype) == MIGRATE_CMA) # define is_migrate_cma_page(_page) (get_pageblock_migratetype(_page) == MIGRATE_CMA) +# define get_cma_migrate_type() MIGRATE_CMA #else # define is_migrate_cma(migratetype) false # define is_migrate_cma_page(_page) false +# define get_cma_migrate_type() MIGRATE_MOVABLE #endif static inline bool is_migrate_movable(int mt) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 2f38f57b1ed2..ca8156111b43 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2855,35 +2855,35 @@ __rmqueue(struct zone *zone, unsigned int order, int migratetype, { struct page *page; -#ifdef CONFIG_CMA - /* - * Balance movable allocations between regular and CMA areas by - * allocating from CMA when over half of the zone's free memory - * is in the CMA area. - */ - if (alloc_flags & ALLOC_CMA && - zone_page_state(zone, NR_FREE_CMA_PAGES) > - zone_page_state(zone, NR_FREE_PAGES) / 2) { - page = __rmqueue_cma_fallback(zone, order); - if (page) - return page; - } -#endif retry: page = __rmqueue_smallest(zone, order, migratetype); - if (unlikely(!page)) { - if (alloc_flags & ALLOC_CMA) - page = __rmqueue_cma_fallback(zone, order); - if (!page && __rmqueue_fallback(zone, order, migratetype, - alloc_flags)) - goto retry; - } + if (unlikely(!page) && __rmqueue_fallback(zone, order, migratetype, + alloc_flags)) + goto retry; trace_mm_page_alloc_zone_locked(page, order, migratetype); return page; } +#ifdef CONFIG_CMA +static struct page *__rmqueue_cma(struct zone *zone, unsigned int order, + int migratetype, + unsigned int alloc_flags) +{ + struct page *page = __rmqueue_cma_fallback(zone, order); + trace_mm_page_alloc_zone_locked(page, order, MIGRATE_CMA); + return page; +} +#else +static inline struct page *__rmqueue_cma(struct zone *zone, unsigned int order, + int migratetype, + unsigned int alloc_flags) +{ + return NULL; +} +#endif + /* * Obtain a specified number of elements from the buddy allocator, all under * a single hold of the lock, for efficiency. Add them to the supplied list. @@ -2897,8 +2897,14 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, spin_lock(&zone->lock); for (i = 0; i < count; ++i) { - struct page *page = __rmqueue(zone, order, migratetype, - alloc_flags); + struct page *page; + + if (is_migrate_cma(migratetype)) + page = __rmqueue_cma(zone, order, migratetype, + alloc_flags); + else + page = __rmqueue(zone, order, migratetype, alloc_flags); + if (unlikely(page == NULL)) break; @@ -3383,15 +3389,24 @@ static inline void zone_statistics(struct zone *preferred_zone, struct zone *z) static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype, unsigned int alloc_flags, struct per_cpu_pages *pcp, - struct list_head *list) + struct list_head *list, + gfp_t gfp_flags) { struct page *page; do { - if (list_empty(list)) { + if (list_empty(list) && migratetype == MIGRATE_MOVABLE && + gfp_flags & __GFP_CMA) { + pcp->count += rmqueue_bulk(zone, 0, + pcp->batch, list, + get_cma_migrate_type(), alloc_flags); + } + + if (unlikely(list_empty(list))) { pcp->count += rmqueue_bulk(zone, 0, pcp->batch, list, migratetype, alloc_flags); + if (unlikely(list_empty(list))) return NULL; } @@ -3417,7 +3432,8 @@ static struct page *rmqueue_pcplist(struct zone *preferred_zone, local_irq_save(flags); pcp = &this_cpu_ptr(zone->pageset)->pcp; list = &pcp->lists[migratetype]; - page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, list); + page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, list, + gfp_flags); if (page) { __count_zid_vm_events(PGALLOC, page_zonenum(page), 1); zone_statistics(preferred_zone, zone); @@ -3443,7 +3459,7 @@ struct page *rmqueue(struct zone *preferred_zone, * MIGRATE_MOVABLE pcplist could have the pages on CMA area and * we need to skip it when CMA area isn't allowed. */ - if (!IS_ENABLED(CONFIG_CMA) || alloc_flags & ALLOC_CMA || + if (!IS_ENABLED(CONFIG_CMA) || gfp_flags & __GFP_CMA || migratetype != MIGRATE_MOVABLE) { page = rmqueue_pcplist(preferred_zone, zone, gfp_flags, migratetype, alloc_flags); @@ -3471,8 +3487,14 @@ struct page *rmqueue(struct zone *preferred_zone, if (page) trace_mm_page_alloc_zone_locked(page, order, migratetype); } - if (!page) - page = __rmqueue(zone, order, migratetype, alloc_flags); + if (!page) { + if (gfp_flags & __GFP_CMA && migratetype == MIGRATE_MOVABLE) + page = __rmqueue_cma(zone, order, migratetype, + alloc_flags); + if (!page) + page = __rmqueue(zone, order, migratetype, + alloc_flags); + } } while (page && check_new_pages(page, order)); spin_unlock(&zone->lock); if (!page) @@ -3785,7 +3807,8 @@ static inline unsigned int current_alloc_flags(gfp_t gfp_mask, unsigned int pflags = current->flags; if (!(pflags & PF_MEMALLOC_NOCMA) && - gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) + gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE && + gfp_mask & __GFP_CMA) alloc_flags |= ALLOC_CMA; #endif