diff --git a/include/linux/page-flags-layout.h b/include/linux/page-flags-layout.h index f6fcd45a30e9..c9cb85defac0 100644 --- a/include/linux/page-flags-layout.h +++ b/include/linux/page-flags-layout.h @@ -77,23 +77,27 @@ #define LAST_CPUPID_SHIFT 0 #endif -#ifdef CONFIG_KASAN_SW_TAGS -#define KASAN_TAG_WIDTH 8 -#else -#define KASAN_TAG_WIDTH 0 -#endif - -#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_WIDTH+LRU_GEN_WIDTH+LRU_REFS_WIDTH+ \ - KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS +#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_WIDTH+LAST_CPUPID_SHIFT+ \ + LRU_GEN_WIDTH+LRU_REFS_WIDTH <= BITS_PER_LONG - NR_PAGEFLAGS #define LAST_CPUPID_WIDTH LAST_CPUPID_SHIFT #else #define LAST_CPUPID_WIDTH 0 #endif -#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_WIDTH+LRU_GEN_WIDTH+LRU_REFS_WIDTH+ \ - KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS +#ifdef CONFIG_KASAN_SW_TAGS +#define KASAN_TAG_WIDTH 8 +#if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH+LAST_CPUPID_WIDTH+KASAN_TAG_WIDTH \ + > BITS_PER_LONG - NR_PAGEFLAGS #error "KASAN: not enough bits in page flags for tag" #endif +#else +#define KASAN_TAG_WIDTH 0 +#endif + +#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_WIDTH+LAST_CPUPID_WIDTH+KASAN_TAG_WIDTH+ \ + LRU_GEN_WIDTH+LRU_REFS_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS +#error "Not enough bits in page flags" +#endif /* * We are going to use the flags for the page to node mapping if its in diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h index beeafcf33c60..2f9e7699f6c8 100644 --- a/include/linux/vm_event_item.h +++ b/include/linux/vm_event_item.h @@ -35,6 +35,15 @@ enum vm_event_item { PGPGIN, PGPGOUT, PGPGOUTCLEAN, PSWPIN, PSWPOUT, PGSCAN_KSWAPD, PGSCAN_DIRECT, PGSCAN_DIRECT_THROTTLE, + LRU_KSWAPD_NO_PROGRESS, + LRU_KSWAPD_ANON, + LRU_KSWAPD_FILE, + LRU_DIRECT_ANON, + LRU_DIRECT_FILE, + LRU_KSWAPD_SWAP_FULL, + LRU_DIRECT_SWAP_FULL, + LRU_NO_GFP_IO, + LRU_NO_WRITEPAGE, #ifdef CONFIG_NUMA PGSCAN_ZONE_RECLAIM_FAILED, #endif diff --git a/mm/swap.c b/mm/swap.c index 49c9e2062058..5e7539467894 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -501,9 +501,10 @@ void __lru_cache_add_active_or_unevictable(struct page *page, { VM_BUG_ON_PAGE(PageLRU(page), page); - if (likely((vma_flags & (VM_LOCKED | VM_SPECIAL)) != VM_LOCKED)) - SetPageActive(page); - else if (!TestSetPageMlocked(page)) { + if (likely((vma_flags & (VM_LOCKED | VM_SPECIAL)) != VM_LOCKED)) { + if (!lru_gen_enabled()) + SetPageActive(page); + } else if (!TestSetPageMlocked(page)) { /* * We use the irq-unsafe __mod_zone_page_stat because this * counter is not modified from interrupt context, and the pte diff --git a/mm/vmscan.c b/mm/vmscan.c index 4aecda263c4d..af81bef23682 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -2630,8 +2630,11 @@ static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc) { struct mem_cgroup *memcg = lruvec_memcg(lruvec); - if (mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH) + if (mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH) { + count_vm_event(current_is_kswapd() ? LRU_KSWAPD_SWAP_FULL: + LRU_DIRECT_SWAP_FULL); return 0; + } return mem_cgroup_swappiness(memcg); } @@ -3857,9 +3860,9 @@ static long get_nr_evictable(struct lruvec *lruvec, unsigned long max_seq, * from the producer's POV, the aging only cares about the upper bound * of hot pages, i.e., 1/MIN_NR_GENS. */ - if (min_seq[LRU_GEN_FILE] + MIN_NR_GENS > max_seq) + if (min_seq[!can_swap] + MIN_NR_GENS > max_seq) *need_aging = true; - else if (min_seq[LRU_GEN_FILE] + MIN_NR_GENS < max_seq) + else if (min_seq[!can_swap] + MIN_NR_GENS < max_seq) *need_aging = false; else if (young * MIN_NR_GENS > total) *need_aging = true; @@ -3954,10 +3957,9 @@ static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) * younger than min_ttl. However, another theoretical possibility is all * memcgs are either below min or empty. */ - if (!success && mutex_trylock(&oom_lock)) { + if (!success && !sc->order && mutex_trylock(&oom_lock)) { struct oom_control oc = { .gfp_mask = sc->gfp_mask, - .order = sc->order, }; out_of_memory(&oc); @@ -3997,7 +3999,7 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) return; start = max(pvmw->address & PMD_MASK, pvmw->vma->vm_start); - end = pmd_addr_end(pvmw->address, pvmw->vma->vm_end); + end = min(pvmw->address | ~PMD_MASK, pvmw->vma->vm_end - 1) + 1; if (end - start > MIN_LRU_BATCH * PAGE_SIZE) { if (pvmw->address - start < MIN_LRU_BATCH * PAGE_SIZE / 2) @@ -4169,8 +4171,13 @@ static bool isolate_page(struct lruvec *lruvec, struct page *page, struct scan_c return false; if (!(sc->may_writepage && (sc->gfp_mask & __GFP_IO)) && - (PageDirty(page) || (PageAnon(page) && !PageSwapCache(page)))) + (PageDirty(page) || (PageAnon(page) && !PageSwapCache(page)))) { + if (!sc->may_writepage) + __count_vm_event(LRU_NO_WRITEPAGE); + if (!(sc->gfp_mask & __GFP_IO)) + __count_vm_event(LRU_NO_GFP_IO); return false; + } if (!get_page_unless_zero(page)) return false; @@ -4372,7 +4379,7 @@ static int evict_pages(struct lruvec *lruvec, struct scan_control *sc, int swapp if (try_to_inc_min_seq(lruvec, swappiness)) scanned++; - if (get_nr_gens(lruvec, LRU_GEN_FILE) == MIN_NR_GENS) + if (get_nr_gens(lruvec, !swappiness) == MIN_NR_GENS) scanned = 0; spin_unlock_irq(&pgdat->lru_lock); @@ -4415,6 +4422,10 @@ static int evict_pages(struct lruvec *lruvec, struct scan_control *sc, int swapp free_unref_page_list(&list); sc->nr_reclaimed += reclaimed; + if (!reclaimed) { + item = current_is_kswapd() ? LRU_KSWAPD_ANON : LRU_DIRECT_ANON; + count_vm_event(item + type); + } if (type == LRU_GEN_ANON && swapped) *swapped = true; @@ -4422,15 +4433,15 @@ static int evict_pages(struct lruvec *lruvec, struct scan_control *sc, int swapp return scanned; } -static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, bool can_swap) +static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, bool can_swap, + bool *need_aging) { - bool need_aging; long nr_to_scan; struct mem_cgroup *memcg = lruvec_memcg(lruvec); DEFINE_MAX_SEQ(lruvec); DEFINE_MIN_SEQ(lruvec); - nr_to_scan = get_nr_evictable(lruvec, max_seq, min_seq, can_swap, &need_aging); + nr_to_scan = get_nr_evictable(lruvec, max_seq, min_seq, can_swap, need_aging); if (!nr_to_scan) return 0; @@ -4443,27 +4454,29 @@ static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, bool if (!nr_to_scan) return 0; - if (!need_aging) { - sc->memcgs_need_aging = false; + if (!*need_aging) return nr_to_scan; - } /* leave the work to lru_gen_age_node() */ if (current_is_kswapd()) return 0; + /* try other memcgs before going to the aging path */ + if (sc->priority == DEF_PRIORITY) + return 0; + if (try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, false)) return nr_to_scan; - return min_seq[LRU_GEN_FILE] + MIN_NR_GENS <= max_seq ? nr_to_scan : 0; + return min_seq[!can_swap] + MIN_NR_GENS <= max_seq ? nr_to_scan : 0; } static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) { struct blk_plug plug; long scanned = 0; + bool need_aging = false; bool swapped = false; - unsigned long reclaimed = sc->nr_reclaimed; struct pglist_data *pgdat = lruvec_pgdat(lruvec); lru_add_drain(); @@ -4485,27 +4498,29 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc else swappiness = 0; - nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness); + nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness, &need_aging); if (!nr_to_scan) - break; + goto done; delta = evict_pages(lruvec, sc, swappiness, &swapped); if (!delta) - break; + goto done; if (sc->memcgs_avoid_swapping && swappiness < 200 && swapped) break; scanned += delta; - if (scanned >= nr_to_scan) { - if (!swapped && sc->nr_reclaimed - reclaimed >= MIN_LRU_BATCH) - sc->memcgs_need_swapping = false; + if (scanned >= nr_to_scan) break; - } cond_resched(); } + if (!need_aging) + sc->memcgs_need_aging = false; + if (!swapped) + sc->memcgs_need_swapping = false; +done: if (current_is_kswapd()) current->reclaim_state->mm_walk = NULL; @@ -6414,8 +6429,11 @@ restart: sc.priority--; } while (sc.priority >= 1); - if (!sc.nr_reclaimed) - pgdat->kswapd_failures++; + if (!sc.nr_reclaimed) { + count_vm_event(LRU_KSWAPD_NO_PROGRESS); + if (!lru_gen_enabled()) + pgdat->kswapd_failures++; + } out: /* If reclaim was boosted, account for the reclaim done in this pass */ diff --git a/mm/vmstat.c b/mm/vmstat.c index 6c202df93035..5df342d5f3f7 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1203,6 +1203,15 @@ const char * const vmstat_text[] = { "pgscan_kswapd", "pgscan_direct", "pgscan_direct_throttle", + "lru_kswapd_no_progress", + "lru_kswapd_anon", + "lru_kswapd_file", + "lru_direct_anon", + "lru_direct_file", + "lru_kswapd_swap_full", + "lru_direct_swap_full", + "lru_no_gfp_io", + "lru_no_writepage", #ifdef CONFIG_NUMA "zone_reclaim_failed",