From 41f9ce4a5980e9133095b6ff8029b7b299fdf00a Mon Sep 17 00:00:00 2001 From: Joonsoo Kim Date: Tue, 11 Aug 2020 18:30:47 -0700 Subject: [PATCH] BACKPORT: mm/swapcache: support to handle the shadow entries Workingset detection for anonymous page will be implemented in the following patch and it requires to store the shadow entries into the swapcache. This patch implements an infrastructure to store the shadow entry in the swapcache. Moto-CRs-Fixed:(CR) Signed-off-by: Joonsoo Kim Signed-off-by: Andrew Morton Acked-by: Johannes Weiner Cc: Hugh Dickins Cc: Matthew Wilcox Cc: Mel Gorman Cc: Michal Hocko Cc: Minchan Kim Cc: Vlastimil Babka Link: http://lkml.kernel.org/r/1595490560-15117-5-git-send-email-iamjoonsoo.kim@lge.com Signed-off-by: Linus Torvalds (cherry picked from commit 3852f6768ede542ed48b9077bedf482c7ecb6327) Bug: 228114874 Signed-off-by: Yu Zhao Change-Id: Ie95a3e7ef0bd487746bfdb08e279c3f7ab5ca17a Reviewed-on: https://gerrit.mot.com/2268684 SME-Granted: SME Approvals Granted SLTApproved: Slta Waiver Tested-by: Jira Key Reviewed-by: Xiangpo Zhao Submit-Approved: Jira Key --- include/linux/swap.h | 14 ++++++-- mm/swap_state.c | 80 +++++++++++++++++++++++++++++++++++++++----- mm/swapfile.c | 2 ++ mm/vmscan.c | 2 +- 4 files changed, 86 insertions(+), 12 deletions(-) diff --git a/include/linux/swap.h b/include/linux/swap.h index ba8541f7a245..4956bfb03034 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -421,9 +421,12 @@ extern unsigned long total_swapcache_pages(void); extern void show_swap_cache_info(void); extern int add_to_swap(struct page *page); extern int add_to_swap_cache(struct page *, swp_entry_t, gfp_t); -extern int __add_to_swap_cache(struct page *page, swp_entry_t entry); -extern void __delete_from_swap_cache(struct page *); +extern int __add_to_swap_cache(struct page *page, swp_entry_t entry, + void **shadowp); +extern void __delete_from_swap_cache(struct page *page, void *shadow); extern void delete_from_swap_cache(struct page *); +extern void clear_shadow_from_swap_cache(int type, unsigned long begin, + unsigned long end); extern void free_page_and_swap_cache(struct page *); extern void free_pages_and_swap_cache(struct page **, int); extern struct page *lookup_swap_cache(swp_entry_t entry, @@ -576,7 +579,7 @@ static inline int add_to_swap_cache(struct page *page, swp_entry_t entry, return -1; } -static inline void __delete_from_swap_cache(struct page *page) +static inline void __delete_from_swap_cache(struct page *page, void *shadow) { } @@ -584,6 +587,11 @@ static inline void delete_from_swap_cache(struct page *page) { } +static inline void clear_shadow_from_swap_cache(int type, unsigned long begin, + unsigned long end) +{ +} + static inline int page_swapcount(struct page *page) { return 0; diff --git a/mm/swap_state.c b/mm/swap_state.c index 246e02815a6e..d44637a7b27a 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -111,7 +111,7 @@ void show_swap_cache_info(void) * __add_to_swap_cache resembles add_to_page_cache_locked on swapper_space, * but sets SwapCache flag and private instead of mapping and index. */ -int __add_to_swap_cache(struct page *page, swp_entry_t entry) +int __add_to_swap_cache(struct page *page, swp_entry_t entry, void **shadowp) { int error, i, nr = hpage_nr_pages(page); struct address_space *address_space; @@ -127,11 +127,30 @@ int __add_to_swap_cache(struct page *page, swp_entry_t entry) address_space = swap_address_space(entry); xa_lock_irq(&address_space->i_pages); for (i = 0; i < nr; i++) { + void *item; + void __rcu **slot; + struct radix_tree_node *node; + set_page_private(page + i, entry.val + i); - error = radix_tree_insert(&address_space->i_pages, - idx + i, page + i); + error = __radix_tree_create(&address_space->i_pages, + idx + i, 0, &node, &slot); if (unlikely(error)) break; + + item = radix_tree_deref_slot_protected(slot, + &address_space->i_pages.xa_lock); + if (WARN_ON_ONCE(item && !radix_tree_exceptional_entry(item))) { + error = -EEXIST; + break; + } + + __radix_tree_replace(&address_space->i_pages, node, slot, + page + i, NULL); + + if (shadowp) { + VM_BUG_ON(i); + *shadowp = item; + } } if (likely(!error)) { address_space->nrpages += nr; @@ -164,7 +183,7 @@ int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp_mask) error = radix_tree_maybe_preload_order(gfp_mask, compound_order(page)); if (!error) { - error = __add_to_swap_cache(page, entry); + error = __add_to_swap_cache(page, entry, NULL); radix_tree_preload_end(); } return error; @@ -174,7 +193,7 @@ int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp_mask) * This must be called only on pages that have * been verified to be in the swap cache. */ -void __delete_from_swap_cache(struct page *page) +void __delete_from_swap_cache(struct page *page, void *shadow) { struct address_space *address_space; int i, nr = hpage_nr_pages(page); @@ -184,12 +203,23 @@ void __delete_from_swap_cache(struct page *page) VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(!PageSwapCache(page), page); VM_BUG_ON_PAGE(PageWriteback(page), page); + VM_BUG_ON(shadow && !radix_tree_exceptional_entry(shadow)); entry.val = page_private(page); address_space = swap_address_space(entry); idx = swp_offset(entry); for (i = 0; i < nr; i++) { - radix_tree_delete(&address_space->i_pages, idx + i); + void *item; + void __rcu **slot; + struct radix_tree_node *node; + + item = __radix_tree_lookup(&address_space->i_pages, + idx + i, &node, &slot); + if (WARN_ON_ONCE(item != page + i)) + continue; + + __radix_tree_replace(&address_space->i_pages, + node, slot, shadow, NULL); set_page_private(page + i, 0); } ClearPageSwapCache(page); @@ -271,13 +301,47 @@ void delete_from_swap_cache(struct page *page) address_space = swap_address_space(entry); xa_lock_irq(&address_space->i_pages); - __delete_from_swap_cache(page); + __delete_from_swap_cache(page, NULL); xa_unlock_irq(&address_space->i_pages); put_swap_page(page, entry); page_ref_sub(page, hpage_nr_pages(page)); } +void clear_shadow_from_swap_cache(int type, unsigned long begin, + unsigned long end) +{ + unsigned long curr = begin; + + for (;;) { + void *item; + void __rcu **slot; + struct radix_tree_iter iter; + swp_entry_t entry = swp_entry(type, curr); + struct address_space *address_space = swap_address_space(entry); + + xa_lock_irq(&address_space->i_pages); + radix_tree_for_each_slot(slot, &address_space->i_pages, + &iter, curr) { + item = radix_tree_deref_slot_protected(slot, + &address_space->i_pages.xa_lock); + if (radix_tree_exceptional_entry(item)) + radix_tree_iter_delete(&address_space->i_pages, + &iter, slot); + if (iter.next_index > end) + break; + } + xa_unlock_irq(&address_space->i_pages); + + /* search the next swapcache until we meet end */ + curr >>= SWAP_ADDRESS_SPACE_SHIFT; + curr++; + curr <<= SWAP_ADDRESS_SPACE_SHIFT; + if (curr > end) + break; + } +} + /* * If we are the only user, then try to free up the swap cache. * @@ -443,7 +507,7 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, /* May fail (-ENOMEM) if radix-tree node allocation failed. */ __SetPageLocked(new_page); __SetPageSwapBacked(new_page); - err = __add_to_swap_cache(new_page, entry); + err = __add_to_swap_cache(new_page, entry, NULL); if (likely(!err)) { radix_tree_preload_end(); /* diff --git a/mm/swapfile.c b/mm/swapfile.c index c1eb8bdc8300..40bc80cb94ee 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -663,6 +663,7 @@ static void add_to_avail_list(struct swap_info_struct *p) static void swap_range_free(struct swap_info_struct *si, unsigned long offset, unsigned int nr_entries) { + unsigned long begin = offset; unsigned long end = offset + nr_entries - 1; void (*swap_slot_free_notify)(struct block_device *, unsigned long); @@ -688,6 +689,7 @@ static void swap_range_free(struct swap_info_struct *si, unsigned long offset, swap_slot_free_notify(si->bdev, offset); offset++; } + clear_shadow_from_swap_cache(si->type, begin, end); } static int scan_swap_map_slots(struct swap_info_struct *si, diff --git a/mm/vmscan.c b/mm/vmscan.c index 3a29d2759154..14e3079bb008 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -950,7 +950,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page, if (PageSwapCache(page)) { swp_entry_t swap = { .val = page_private(page) }; mem_cgroup_swapout(page, swap); - __delete_from_swap_cache(page); + __delete_from_swap_cache(page, NULL); xa_unlock_irqrestore(&mapping->i_pages, flags); put_swap_page(page, swap); } else {