mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-10-01 14:44:12 +00:00
fa8c919dac
With the fix in place to atomically test and clear young on ptes and pmds, simplify the code to handle the clearing for both the primary mmu and the mmu notifier with a single API call. Link: https://lkml.kernel.org/r/20230602092949.545577-4-ryan.roberts@arm.com Signed-off-by: Ryan Roberts <ryan.roberts@arm.com> Acked-by: Yu Zhao <yuzhao@google.com> Reviewed-by: SeongJae Park <sj@kernel.org> Cc: Christoph Hellwig <hch@lst.de> Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com> Cc: Lorenzo Stoakes <lstoakes@gmail.com> Cc: Matthew Wilcox (Oracle) <willy@infradead.org> Cc: Mike Rapoport (IBM) <rppt@kernel.org> Cc: Uladzislau Rezki (Sony) <urezki@gmail.com> Cc: Zi Yan <ziy@nvidia.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
122 lines
3 KiB
C
122 lines
3 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* Common Primitives for Data Access Monitoring
|
|
*
|
|
* Author: SeongJae Park <sj@kernel.org>
|
|
*/
|
|
|
|
#include <linux/mmu_notifier.h>
|
|
#include <linux/page_idle.h>
|
|
#include <linux/pagemap.h>
|
|
#include <linux/rmap.h>
|
|
|
|
#include "ops-common.h"
|
|
|
|
/*
|
|
* Get an online page for a pfn if it's in the LRU list. Otherwise, returns
|
|
* NULL.
|
|
*
|
|
* The body of this function is stolen from the 'page_idle_get_folio()'. We
|
|
* steal rather than reuse it because the code is quite simple.
|
|
*/
|
|
struct folio *damon_get_folio(unsigned long pfn)
|
|
{
|
|
struct page *page = pfn_to_online_page(pfn);
|
|
struct folio *folio;
|
|
|
|
if (!page || PageTail(page))
|
|
return NULL;
|
|
|
|
folio = page_folio(page);
|
|
if (!folio_test_lru(folio) || !folio_try_get(folio))
|
|
return NULL;
|
|
if (unlikely(page_folio(page) != folio || !folio_test_lru(folio))) {
|
|
folio_put(folio);
|
|
folio = NULL;
|
|
}
|
|
return folio;
|
|
}
|
|
|
|
void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr)
|
|
{
|
|
struct folio *folio = damon_get_folio(pte_pfn(*pte));
|
|
|
|
if (!folio)
|
|
return;
|
|
|
|
if (ptep_clear_young_notify(vma, addr, pte))
|
|
folio_set_young(folio);
|
|
|
|
folio_set_idle(folio);
|
|
folio_put(folio);
|
|
}
|
|
|
|
void damon_pmdp_mkold(pmd_t *pmd, struct vm_area_struct *vma, unsigned long addr)
|
|
{
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
struct folio *folio = damon_get_folio(pmd_pfn(*pmd));
|
|
|
|
if (!folio)
|
|
return;
|
|
|
|
if (pmdp_clear_young_notify(vma, addr, pmd))
|
|
folio_set_young(folio);
|
|
|
|
folio_set_idle(folio);
|
|
folio_put(folio);
|
|
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
|
|
}
|
|
|
|
#define DAMON_MAX_SUBSCORE (100)
|
|
#define DAMON_MAX_AGE_IN_LOG (32)
|
|
|
|
int damon_hot_score(struct damon_ctx *c, struct damon_region *r,
|
|
struct damos *s)
|
|
{
|
|
unsigned int max_nr_accesses;
|
|
int freq_subscore;
|
|
unsigned int age_in_sec;
|
|
int age_in_log, age_subscore;
|
|
unsigned int freq_weight = s->quota.weight_nr_accesses;
|
|
unsigned int age_weight = s->quota.weight_age;
|
|
int hotness;
|
|
|
|
max_nr_accesses = c->attrs.aggr_interval / c->attrs.sample_interval;
|
|
freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / max_nr_accesses;
|
|
|
|
age_in_sec = (unsigned long)r->age * c->attrs.aggr_interval / 1000000;
|
|
for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec;
|
|
age_in_log++, age_in_sec >>= 1)
|
|
;
|
|
|
|
/* If frequency is 0, higher age means it's colder */
|
|
if (freq_subscore == 0)
|
|
age_in_log *= -1;
|
|
|
|
/*
|
|
* Now age_in_log is in [-DAMON_MAX_AGE_IN_LOG, DAMON_MAX_AGE_IN_LOG].
|
|
* Scale it to be in [0, 100] and set it as age subscore.
|
|
*/
|
|
age_in_log += DAMON_MAX_AGE_IN_LOG;
|
|
age_subscore = age_in_log * DAMON_MAX_SUBSCORE /
|
|
DAMON_MAX_AGE_IN_LOG / 2;
|
|
|
|
hotness = (freq_weight * freq_subscore + age_weight * age_subscore);
|
|
if (freq_weight + age_weight)
|
|
hotness /= freq_weight + age_weight;
|
|
/*
|
|
* Transform it to fit in [0, DAMOS_MAX_SCORE]
|
|
*/
|
|
hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE;
|
|
|
|
return hotness;
|
|
}
|
|
|
|
int damon_cold_score(struct damon_ctx *c, struct damon_region *r,
|
|
struct damos *s)
|
|
{
|
|
int hotness = damon_hot_score(c, r, s);
|
|
|
|
/* Return coldness of the region */
|
|
return DAMOS_MAX_SCORE - hotness;
|
|
}
|