ich777/linux_media
1
0
Fork 0
mirror of https://github.com/tbsdtv/linux_media.git synced 2025-07-23 04:33:26 +02:00
Code Issues Packages Projects Releases Wiki Activity

mm: compaction: convert to use a folio in isolate_migratepages_block()

Browse Source 
Directly use a folio instead of page_folio() when page successfully
isolated (hugepage and movable page) and after folio_get_nontail_page(),
which removes several calls to compound_head().

Link: https://lkml.kernel.org/r/20230619110718.65679-1-wangkefeng.wang@huawei.com
Signed-off-by: Kefeng Wang <wangkefeng.wang@huawei.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: James Gowans <jgowans@amazon.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Yu Zhao <yuzhao@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Kefeng Wang
2023-06-19 19:07:17 +08:00
committed by Andrew Morton
parent 18a937076c
commit 56ae0bb349
1 changed files with 44 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
mm/compaction.c
View File

@@ -822,6 +822,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
struct lruvec *lruvec; struct lruvec *lruvec;
unsigned long flags = 0; unsigned long flags = 0;
struct lruvec *locked = NULL; struct lruvec *locked = NULL;
struct folio *folio = NULL;
struct page *page = NULL, *valid_page = NULL; struct page *page = NULL, *valid_page = NULL;
struct address_space *mapping; struct address_space *mapping;
unsigned long start_pfn = low_pfn; unsigned long start_pfn = low_pfn;
@@ -918,7 +919,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
if (!valid_page && pageblock_aligned(low_pfn)) { if (!valid_page && pageblock_aligned(low_pfn)) {
if (!isolation_suitable(cc, page)) { if (!isolation_suitable(cc, page)) {
low_pfn = end_pfn; low_pfn = end_pfn;
page = NULL; folio = NULL;
goto isolate_abort; goto isolate_abort;
} }
valid_page = page; valid_page = page;
@@ -950,7 +951,8 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
* Hugepage was successfully isolated and placed * Hugepage was successfully isolated and placed
* on the cc->migratepages list. * on the cc->migratepages list.
*/ */
low_pfn += compound_nr(page) - 1; folio = page_folio(page);
low_pfn += folio_nr_pages(folio) - 1;
goto isolate_success_no_list; goto isolate_success_no_list;
} }
@@ -1018,8 +1020,10 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
locked = NULL; locked = NULL;
} }
if (isolate_movable_page(page, mode)) if (isolate_movable_page(page, mode)) {
folio = page_folio(page);
goto isolate_success; goto isolate_success;
}
} }
goto isolate_fail; goto isolate_fail;
@@ -1030,7 +1034,8 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
* sure the page is not being freed elsewhere -- the * sure the page is not being freed elsewhere -- the
* page release code relies on it. * page release code relies on it.
*/ */
if (unlikely(!get_page_unless_zero(page))) folio = folio_get_nontail_page(page);
if (unlikely(!folio))
goto isolate_fail; goto isolate_fail;
/* /*
@@ -1038,8 +1043,8 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
* so avoid taking lru_lock and isolating it unnecessarily in an * so avoid taking lru_lock and isolating it unnecessarily in an
* admittedly racy check. * admittedly racy check.
*/ */
mapping = page_mapping(page); mapping = folio_mapping(folio);
if (!mapping && (page_count(page) - 1) > total_mapcount(page)) if (!mapping && (folio_ref_count(folio) - 1) > folio_mapcount(folio))
goto isolate_fail_put; goto isolate_fail_put;
/* /*
@@ -1050,11 +1055,11 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
goto isolate_fail_put; goto isolate_fail_put;
/* Only take pages on LRU: a check now makes later tests safe */ /* Only take pages on LRU: a check now makes later tests safe */
if (!PageLRU(page)) if (!folio_test_lru(folio))
goto isolate_fail_put; goto isolate_fail_put;
/* Compaction might skip unevictable pages but CMA takes them */ /* Compaction might skip unevictable pages but CMA takes them */
if (!(mode & ISOLATE_UNEVICTABLE) && PageUnevictable(page)) if (!(mode & ISOLATE_UNEVICTABLE) && folio_test_unevictable(folio))
goto isolate_fail_put; goto isolate_fail_put;
/* /*
@@ -1063,10 +1068,10 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
* it will be able to migrate without blocking - clean pages * it will be able to migrate without blocking - clean pages
* for the most part. PageWriteback would require blocking. * for the most part. PageWriteback would require blocking.
*/ */
if ((mode & ISOLATE_ASYNC_MIGRATE) && PageWriteback(page)) if ((mode & ISOLATE_ASYNC_MIGRATE) && folio_test_writeback(folio))
goto isolate_fail_put; goto isolate_fail_put;
if ((mode & ISOLATE_ASYNC_MIGRATE) && PageDirty(page)) { if ((mode & ISOLATE_ASYNC_MIGRATE) && folio_test_dirty(folio)) {
bool migrate_dirty; bool migrate_dirty;
/* /*
@@ -1078,22 +1083,22 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
* the page lock until after the page is removed * the page lock until after the page is removed
* from the page cache. * from the page cache.
*/ */
if (!trylock_page(page)) if (!folio_trylock(folio))
goto isolate_fail_put; goto isolate_fail_put;
mapping = page_mapping(page); mapping = folio_mapping(folio);
migrate_dirty = !mapping || migrate_dirty = !mapping ||
mapping->a_ops->migrate_folio; mapping->a_ops->migrate_folio;
unlock_page(page); folio_unlock(folio);
if (!migrate_dirty) if (!migrate_dirty)
goto isolate_fail_put; goto isolate_fail_put;
} }
/* Try isolate the page */ /* Try isolate the folio */
if (!TestClearPageLRU(page)) if (!folio_test_clear_lru(folio))
goto isolate_fail_put; goto isolate_fail_put;
lruvec = folio_lruvec(page_folio(page)); lruvec = folio_lruvec(folio);
/* If we already hold the lock, we can skip some rechecking */ /* If we already hold the lock, we can skip some rechecking */
if (lruvec != locked) { if (lruvec != locked) {
@@ -1103,7 +1108,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
compact_lock_irqsave(&lruvec->lru_lock, &flags, cc); compact_lock_irqsave(&lruvec->lru_lock, &flags, cc);
locked = lruvec; locked = lruvec;
lruvec_memcg_debug(lruvec, page_folio(page)); lruvec_memcg_debug(lruvec, folio);
/* /*
* Try get exclusive access under lock. If marked for * Try get exclusive access under lock. If marked for
@@ -1119,34 +1124,33 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
} }
/* /*
* Page become compound since the non-locked check, * folio become large since the non-locked check,
* and it's on LRU. It can only be a THP so the order * and it's on LRU.
* is safe to read and it's 0 for tail pages.
*/ */
if (unlikely(PageCompound(page) && !cc->alloc_contig)) { if (unlikely(folio_test_large(folio) && !cc->alloc_contig)) {
low_pfn += compound_nr(page) - 1; low_pfn += folio_nr_pages(folio) - 1;
nr_scanned += compound_nr(page) - 1; nr_scanned += folio_nr_pages(folio) - 1;
SetPageLRU(page); folio_set_lru(folio);
goto isolate_fail_put; goto isolate_fail_put;
} }
} }
/* The whole page is taken off the LRU; skip the tail pages. */ /* The folio is taken off the LRU */
if (PageCompound(page)) if (folio_test_large(folio))
low_pfn += compound_nr(page) - 1; low_pfn += folio_nr_pages(folio) - 1;
/* Successfully isolated */ /* Successfully isolated */
del_page_from_lru_list(page, lruvec); lruvec_del_folio(lruvec, folio);
mod_node_page_state(page_pgdat(page), node_stat_mod_folio(folio,
NR_ISOLATED_ANON + page_is_file_lru(page), NR_ISOLATED_ANON + folio_is_file_lru(folio),
thp_nr_pages(page)); folio_nr_pages(folio));
isolate_success: isolate_success:
list_add(&page->lru, &cc->migratepages); list_add(&folio->lru, &cc->migratepages);
isolate_success_no_list: isolate_success_no_list:
cc->nr_migratepages += compound_nr(page); cc->nr_migratepages += folio_nr_pages(folio);
nr_isolated += compound_nr(page); nr_isolated += folio_nr_pages(folio);
nr_scanned += compound_nr(page) - 1; nr_scanned += folio_nr_pages(folio) - 1;
/* /*
* Avoid isolating too much unless this block is being * Avoid isolating too much unless this block is being
@@ -1168,7 +1172,7 @@ isolate_fail_put:
unlock_page_lruvec_irqrestore(locked, flags); unlock_page_lruvec_irqrestore(locked, flags);
locked = NULL; locked = NULL;
} }
put_page(page); folio_put(folio);
isolate_fail: isolate_fail:
if (!skip_on_failure && ret != -ENOMEM) if (!skip_on_failure && ret != -ENOMEM)
@@ -1209,14 +1213,14 @@ isolate_fail:
if (unlikely(low_pfn > end_pfn)) if (unlikely(low_pfn > end_pfn))
low_pfn = end_pfn; low_pfn = end_pfn;
page = NULL; folio = NULL;
isolate_abort: isolate_abort:
if (locked) if (locked)
unlock_page_lruvec_irqrestore(locked, flags); unlock_page_lruvec_irqrestore(locked, flags);
if (page) { if (folio) {
SetPageLRU(page); folio_set_lru(folio);
put_page(page); folio_put(folio);
} }
/* /*