diff --git a/include/linux/gfp.h b/include/linux/gfp.h index c603237e006c..e80b7d2f5b38 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -39,11 +39,21 @@ struct vm_area_struct; #define ___GFP_HARDWALL 0x100000u #define ___GFP_THISNODE 0x200000u #define ___GFP_ACCOUNT 0x400000u +#ifdef CONFIG_CMA +#define ___GFP_CMA 0x800000u +#else +#define ___GFP_CMA 0 +#endif #ifdef CONFIG_LOCKDEP +#ifdef CONFIG_CMA +#define ___GFP_NOLOCKDEP 0x1000000u +#else #define ___GFP_NOLOCKDEP 0x800000u +#endif #else #define ___GFP_NOLOCKDEP 0 #endif + /* If the above are modified, __GFP_BITS_SHIFT may need updating */ /* @@ -57,6 +67,7 @@ struct vm_area_struct; #define __GFP_HIGHMEM ((__force gfp_t)___GFP_HIGHMEM) #define __GFP_DMA32 ((__force gfp_t)___GFP_DMA32) #define __GFP_MOVABLE ((__force gfp_t)___GFP_MOVABLE) /* ZONE_MOVABLE allowed */ +#define __GFP_CMA ((__force gfp_t)___GFP_CMA) #define GFP_ZONEMASK (__GFP_DMA|__GFP_HIGHMEM|__GFP_DMA32|__GFP_MOVABLE) /** @@ -224,7 +235,11 @@ struct vm_area_struct; #define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP) /* Room for N __GFP_FOO bits */ +#ifdef CONFIG_CMA +#define __GFP_BITS_SHIFT (24 + IS_ENABLED(CONFIG_LOCKDEP)) +#else #define __GFP_BITS_SHIFT (23 + IS_ENABLED(CONFIG_LOCKDEP)) +#endif #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) /** diff --git a/include/linux/highmem.h b/include/linux/highmem.h index 14e6202ce47f..35f052babdba 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -274,7 +274,9 @@ static inline struct page * alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma, unsigned long vaddr) { - return __alloc_zeroed_user_highpage(__GFP_MOVABLE, vma, vaddr); + return __alloc_zeroed_user_highpage( + __GFP_MOVABLE|__GFP_CMA, vma, + vaddr); } static inline void clear_highpage(struct page *page) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index a140d856e354..278c8f47259e 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -72,9 +72,11 @@ extern const char * const migratetype_names[MIGRATE_TYPES]; #ifdef CONFIG_CMA # define is_migrate_cma(migratetype) unlikely((migratetype) == MIGRATE_CMA) # define is_migrate_cma_page(_page) (get_pageblock_migratetype(_page) == MIGRATE_CMA) +# define get_cma_migrate_type() MIGRATE_CMA #else # define is_migrate_cma(migratetype) false # define is_migrate_cma_page(_page) false +# define get_cma_migrate_type() MIGRATE_MOVABLE #endif static inline bool is_migrate_movable(int mt) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 2f38f57b1ed2..ca8156111b43 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2855,35 +2855,35 @@ __rmqueue(struct zone *zone, unsigned int order, int migratetype, { struct page *page; -#ifdef CONFIG_CMA - /* - * Balance movable allocations between regular and CMA areas by - * allocating from CMA when over half of the zone's free memory - * is in the CMA area. - */ - if (alloc_flags & ALLOC_CMA && - zone_page_state(zone, NR_FREE_CMA_PAGES) > - zone_page_state(zone, NR_FREE_PAGES) / 2) { - page = __rmqueue_cma_fallback(zone, order); - if (page) - return page; - } -#endif retry: page = __rmqueue_smallest(zone, order, migratetype); - if (unlikely(!page)) { - if (alloc_flags & ALLOC_CMA) - page = __rmqueue_cma_fallback(zone, order); - if (!page && __rmqueue_fallback(zone, order, migratetype, - alloc_flags)) - goto retry; - } + if (unlikely(!page) && __rmqueue_fallback(zone, order, migratetype, + alloc_flags)) + goto retry; trace_mm_page_alloc_zone_locked(page, order, migratetype); return page; } +#ifdef CONFIG_CMA +static struct page *__rmqueue_cma(struct zone *zone, unsigned int order, + int migratetype, + unsigned int alloc_flags) +{ + struct page *page = __rmqueue_cma_fallback(zone, order); + trace_mm_page_alloc_zone_locked(page, order, MIGRATE_CMA); + return page; +} +#else +static inline struct page *__rmqueue_cma(struct zone *zone, unsigned int order, + int migratetype, + unsigned int alloc_flags) +{ + return NULL; +} +#endif + /* * Obtain a specified number of elements from the buddy allocator, all under * a single hold of the lock, for efficiency. Add them to the supplied list. @@ -2897,8 +2897,14 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, spin_lock(&zone->lock); for (i = 0; i < count; ++i) { - struct page *page = __rmqueue(zone, order, migratetype, - alloc_flags); + struct page *page; + + if (is_migrate_cma(migratetype)) + page = __rmqueue_cma(zone, order, migratetype, + alloc_flags); + else + page = __rmqueue(zone, order, migratetype, alloc_flags); + if (unlikely(page == NULL)) break; @@ -3383,15 +3389,24 @@ static inline void zone_statistics(struct zone *preferred_zone, struct zone *z) static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype, unsigned int alloc_flags, struct per_cpu_pages *pcp, - struct list_head *list) + struct list_head *list, + gfp_t gfp_flags) { struct page *page; do { - if (list_empty(list)) { + if (list_empty(list) && migratetype == MIGRATE_MOVABLE && + gfp_flags & __GFP_CMA) { + pcp->count += rmqueue_bulk(zone, 0, + pcp->batch, list, + get_cma_migrate_type(), alloc_flags); + } + + if (unlikely(list_empty(list))) { pcp->count += rmqueue_bulk(zone, 0, pcp->batch, list, migratetype, alloc_flags); + if (unlikely(list_empty(list))) return NULL; } @@ -3417,7 +3432,8 @@ static struct page *rmqueue_pcplist(struct zone *preferred_zone, local_irq_save(flags); pcp = &this_cpu_ptr(zone->pageset)->pcp; list = &pcp->lists[migratetype]; - page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, list); + page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, list, + gfp_flags); if (page) { __count_zid_vm_events(PGALLOC, page_zonenum(page), 1); zone_statistics(preferred_zone, zone); @@ -3443,7 +3459,7 @@ struct page *rmqueue(struct zone *preferred_zone, * MIGRATE_MOVABLE pcplist could have the pages on CMA area and * we need to skip it when CMA area isn't allowed. */ - if (!IS_ENABLED(CONFIG_CMA) || alloc_flags & ALLOC_CMA || + if (!IS_ENABLED(CONFIG_CMA) || gfp_flags & __GFP_CMA || migratetype != MIGRATE_MOVABLE) { page = rmqueue_pcplist(preferred_zone, zone, gfp_flags, migratetype, alloc_flags); @@ -3471,8 +3487,14 @@ struct page *rmqueue(struct zone *preferred_zone, if (page) trace_mm_page_alloc_zone_locked(page, order, migratetype); } - if (!page) - page = __rmqueue(zone, order, migratetype, alloc_flags); + if (!page) { + if (gfp_flags & __GFP_CMA && migratetype == MIGRATE_MOVABLE) + page = __rmqueue_cma(zone, order, migratetype, + alloc_flags); + if (!page) + page = __rmqueue(zone, order, migratetype, + alloc_flags); + } } while (page && check_new_pages(page, order)); spin_unlock(&zone->lock); if (!page) @@ -3785,7 +3807,8 @@ static inline unsigned int current_alloc_flags(gfp_t gfp_mask, unsigned int pflags = current->flags; if (!(pflags & PF_MEMALLOC_NOCMA) && - gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) + gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE && + gfp_mask & __GFP_CMA) alloc_flags |= ALLOC_CMA; #endif