From 1f8f6d59a298c4fbd10603e681852c1670ab1b1c Mon Sep 17 00:00:00 2001 From: Peifeng Li Date: Sun, 24 Jul 2022 13:47:14 +0800 Subject: [PATCH] ANDROID: vendor_hook: Add hook to not be stuck ro rmap lock in kswapd or direct_reclaim Add hooks to support trylock in rmaplock when reclaiming in kswapd or direct_reclaim, in order to avoid wait lock for a long time. - android_vh_handle_failed_page_trylock - android_vh_page_trylock_set - android_vh_page_trylock_clear - android_vh_page_trylock_get_result - android_vh_do_page_trylock Bug: 240003372 Signed-off-by: Peifeng Li Change-Id: I0f605b35ae41f15b3ca7bc72cd5f003175c318a5 --- drivers/android/vendor_hooks.c | 5 +++++ include/trace/hooks/mm.h | 5 +++++ include/trace/hooks/vmscan.h | 12 ++++++++++++ mm/rmap.c | 20 +++++++++++++++++--- mm/vmscan.c | 18 ++++++++++++++---- 5 files changed, 53 insertions(+), 7 deletions(-) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index d7fab5cb876d..128f63a76938 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -304,6 +304,11 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_logbuf_pr_cont); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_tune_scan_type); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_tune_swappiness); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_shrink_slab_bypass); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_handle_failed_page_trylock); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_page_trylock_set); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_page_trylock_clear); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_page_trylock_get_result); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_do_page_trylock); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_page_referenced_check_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_drain_all_pages_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cma_drain_all_pages_bypass); diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index 495f66d0f926..8b0225ce7ced 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -22,6 +22,7 @@ #include #include #include +#include #ifdef __GENKSYMS__ struct slabinfo; @@ -148,6 +149,10 @@ DECLARE_HOOK(android_vh_mmap_region, DECLARE_HOOK(android_vh_try_to_unmap_one, TP_PROTO(struct vm_area_struct *vma, struct page *page, unsigned long addr, bool ret), TP_ARGS(vma, page, addr, ret)); +DECLARE_HOOK(android_vh_do_page_trylock, + TP_PROTO(struct page *page, struct rw_semaphore *sem, + bool *got_lock, bool *success), + TP_ARGS(page, sem, got_lock, success)); DECLARE_HOOK(android_vh_drain_all_pages_bypass, TP_PROTO(gfp_t gfp_mask, unsigned int order, unsigned long alloc_flags, int migratetype, unsigned long did_some_progress, diff --git a/include/trace/hooks/vmscan.h b/include/trace/hooks/vmscan.h index 4f1d6ab9f498..a175232c89f6 100644 --- a/include/trace/hooks/vmscan.h +++ b/include/trace/hooks/vmscan.h @@ -28,6 +28,18 @@ DECLARE_RESTRICTED_HOOK(android_rvh_set_balance_anon_file_reclaim, DECLARE_HOOK(android_vh_page_referenced_check_bypass, TP_PROTO(struct page *page, unsigned long nr_to_scan, int lru, bool *bypass), TP_ARGS(page, nr_to_scan, lru, bypass)); +DECLARE_HOOK(android_vh_page_trylock_get_result, + TP_PROTO(struct page *page, bool *trylock_fail), + TP_ARGS(page, trylock_fail)); +DECLARE_HOOK(android_vh_handle_failed_page_trylock, + TP_PROTO(struct list_head *page_list), + TP_ARGS(page_list)); +DECLARE_HOOK(android_vh_page_trylock_set, + TP_PROTO(struct page *page), + TP_ARGS(page)); +DECLARE_HOOK(android_vh_page_trylock_clear, + TP_PROTO(struct page *page), + TP_ARGS(page)); DECLARE_HOOK(android_vh_shrink_node_memcgs, TP_PROTO(struct mem_cgroup *memcg, bool *skip), TP_ARGS(memcg, skip)); diff --git a/mm/rmap.c b/mm/rmap.c index ea1fd71b03df..2e62c1aa5139 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -525,6 +525,7 @@ struct anon_vma *page_lock_anon_vma_read(struct page *page) struct anon_vma *anon_vma = NULL; struct anon_vma *root_anon_vma; unsigned long anon_mapping; + bool success = false; rcu_read_lock(); anon_mapping = (unsigned long)READ_ONCE(page->mapping); @@ -547,7 +548,11 @@ struct anon_vma *page_lock_anon_vma_read(struct page *page) } goto out; } - + trace_android_vh_do_page_trylock(page, NULL, NULL, &success); + if (success) { + anon_vma = NULL; + goto out; + } /* trylock failed, we got to sleep */ if (!atomic_inc_not_zero(&anon_vma->refcount)) { anon_vma = NULL; @@ -1981,6 +1986,7 @@ static void rmap_walk_file(struct page *page, struct rmap_walk_control *rwc, struct address_space *mapping = page_mapping(page); pgoff_t pgoff_start, pgoff_end; struct vm_area_struct *vma; + bool got_lock = false, success = false; /* * The page lock not only makes sure that page->mapping cannot @@ -1995,8 +2001,16 @@ static void rmap_walk_file(struct page *page, struct rmap_walk_control *rwc, pgoff_start = page_to_pgoff(page); pgoff_end = pgoff_start + thp_nr_pages(page) - 1; - if (!locked) - i_mmap_lock_read(mapping); + if (!locked) { + trace_android_vh_do_page_trylock(page, + &mapping->i_mmap_rwsem, &got_lock, &success); + if (success) { + if (!got_lock) + return; + } else { + i_mmap_lock_read(mapping); + } + } vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff_start, pgoff_end) { unsigned long address = vma_address(page, vma); diff --git a/mm/vmscan.c b/mm/vmscan.c index 22ab1afed87f..7899d6905b6e 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1021,15 +1021,19 @@ static enum page_references page_check_references(struct page *page, int referenced_ptes, referenced_page; unsigned long vm_flags; bool should_protect = false; + bool trylock_fail = false; trace_android_vh_page_should_be_protected(page, &should_protect); if (unlikely(should_protect)) return PAGEREF_ACTIVATE; + trace_android_vh_page_trylock_set(page); referenced_ptes = page_referenced(page, 1, sc->target_mem_cgroup, &vm_flags); referenced_page = TestClearPageReferenced(page); - + trace_android_vh_page_trylock_get_result(page, &trylock_fail); + if (trylock_fail) + return PAGEREF_KEEP; /* * Mlock lost the isolation race with us. Let try_to_unmap() * move the page to the unevictable list. @@ -1341,6 +1345,7 @@ static unsigned int shrink_page_list(struct list_head *page_list, if (unlikely(PageTransHuge(page))) flags |= TTU_SPLIT_HUGE_PMD; + trace_android_vh_page_trylock_set(page); if (!try_to_unmap(page, flags)) { stat->nr_unmap_fail += nr_pages; if (!was_swapbacked && PageSwapBacked(page)) @@ -1451,6 +1456,7 @@ static unsigned int shrink_page_list(struct list_head *page_list, * increment nr_reclaimed here (and * leave it off the LRU). */ + trace_android_vh_page_trylock_clear(page); nr_reclaimed++; continue; } @@ -1486,8 +1492,10 @@ free_it: */ if (unlikely(PageTransHuge(page))) destroy_compound_page(page); - else + else { + trace_android_vh_page_trylock_clear(page); list_add(&page->lru, &free_pages); + } continue; activate_locked_split: @@ -1999,6 +2007,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, return 0; nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, &stat, false); + trace_android_vh_handle_failed_page_trylock(&page_list); spin_lock_irq(&pgdat->lru_lock); @@ -2011,7 +2020,6 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, __count_vm_events(item, nr_reclaimed); __count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed); __count_vm_events(PGSTEAL_ANON + file, nr_reclaimed); - spin_unlock_irq(&pgdat->lru_lock); mem_cgroup_uncharge_list(&page_list); @@ -2107,7 +2115,7 @@ static void shrink_active_list(unsigned long nr_to_scan, trace_android_vh_page_referenced_check_bypass(page, nr_to_scan, lru, &bypass); if (bypass) goto skip_page_referenced; - + trace_android_vh_page_trylock_set(page); if (page_referenced(page, 0, sc->target_mem_cgroup, &vm_flags)) { /* @@ -2120,11 +2128,13 @@ static void shrink_active_list(unsigned long nr_to_scan, * so we ignore them here. */ if ((vm_flags & VM_EXEC) && page_is_file_lru(page)) { + trace_android_vh_page_trylock_clear(page); nr_rotated += thp_nr_pages(page); list_add(&page->lru, &l_active); continue; } } + trace_android_vh_page_trylock_clear(page); skip_page_referenced: ClearPageActive(page); /* we are de-activating */ SetPageWorkingset(page);