projects / linux-block.git / blame
| Commit | Line | Data |
|---|---|---|
| 46c3a0ac SP |
1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | /* | |
| 3 | * Common Primitives for Data Access Monitoring | |
| 4 | * | |
| 5 | * Author: SeongJae Park <sj@kernel.org> | |
| 6 | */ | |
| 7 | ||
| 8 | #include <linux/mmu_notifier.h> | |
| 9 | #include <linux/page_idle.h> | |
| 10 | #include <linux/pagemap.h> | |
| 11 | #include <linux/rmap.h> | |
| 12 | ||
| f7d911c3 | 13 | #include "ops-common.h" |
| 46c3a0ac SP |
14 | |
| 15 | /* | |
| 16 | * Get an online page for a pfn if it's in the LRU list. Otherwise, returns | |
| 17 | * NULL. | |
| 18 | * | |
| 19 | * The body of this function is stolen from the 'page_idle_get_page()'. We | |
| 20 | * steal rather than reuse it because the code is quite simple. | |
| 21 | */ | |
| 22 | struct page *damon_get_page(unsigned long pfn) | |
| 23 | { | |
| 24 | struct page *page = pfn_to_online_page(pfn); | |
| 25 | ||
| 26 | if (!page || !PageLRU(page) || !get_page_unless_zero(page)) | |
| 27 | return NULL; | |
| 28 | ||
| 29 | if (unlikely(!PageLRU(page))) { | |
| 30 | put_page(page); | |
| 31 | page = NULL; | |
| 32 | } | |
| 33 | return page; | |
| 34 | } | |
| 35 | ||
| 36 | void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr) | |
| 37 | { | |
| 38 | bool referenced = false; | |
| 39 | struct page *page = damon_get_page(pte_pfn(*pte)); | |
| 40 | ||
| 41 | if (!page) | |
| 42 | return; | |
| 43 | ||
| 44 | if (pte_young(*pte)) { | |
| 45 | referenced = true; | |
| 46 | *pte = pte_mkold(*pte); | |
| 47 | } | |
| 48 | ||
| 49 | #ifdef CONFIG_MMU_NOTIFIER | |
| 50 | if (mmu_notifier_clear_young(mm, addr, addr + PAGE_SIZE)) | |
| 51 | referenced = true; | |
| 52 | #endif /* CONFIG_MMU_NOTIFIER */ | |
| 53 | ||
| 54 | if (referenced) | |
| 55 | set_page_young(page); | |
| 56 | ||
| 57 | set_page_idle(page); | |
| 58 | put_page(page); | |
| 59 | } | |
| 60 | ||
| 61 | void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr) | |
| 62 | { | |
| 63 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | |
| 64 | bool referenced = false; | |
| 65 | struct page *page = damon_get_page(pmd_pfn(*pmd)); | |
| 66 | ||
| 67 | if (!page) | |
| 68 | return; | |
| 69 | ||
| 70 | if (pmd_young(*pmd)) { | |
| 71 | referenced = true; | |
| 72 | *pmd = pmd_mkold(*pmd); | |
| 73 | } | |
| 74 | ||
| 75 | #ifdef CONFIG_MMU_NOTIFIER | |
| 02e34fff | 76 | if (mmu_notifier_clear_young(mm, addr, addr + HPAGE_PMD_SIZE)) |
| 46c3a0ac SP |
77 | referenced = true; |
| 78 | #endif /* CONFIG_MMU_NOTIFIER */ | |
| 79 | ||
| 80 | if (referenced) | |
| 81 | set_page_young(page); | |
| 82 | ||
| 83 | set_page_idle(page); | |
| 84 | put_page(page); | |
| 85 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ | |
| 86 | } | |
| 198f0f4c SP |
87 | |
| 88 | #define DAMON_MAX_SUBSCORE (100) | |
| 89 | #define DAMON_MAX_AGE_IN_LOG (32) | |
| 90 | ||
| 91 | int damon_pageout_score(struct damon_ctx *c, struct damon_region *r, | |
| 92 | struct damos *s) | |
| 93 | { | |
| 94 | unsigned int max_nr_accesses; | |
| 95 | int freq_subscore; | |
| 96 | unsigned int age_in_sec; | |
| 97 | int age_in_log, age_subscore; | |
| 98 | unsigned int freq_weight = s->quota.weight_nr_accesses; | |
| 99 | unsigned int age_weight = s->quota.weight_age; | |
| 100 | int hotness; | |
| 101 | ||
| 102 | max_nr_accesses = c->aggr_interval / c->sample_interval; | |
| 103 | freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / max_nr_accesses; | |
| 104 | ||
| 105 | age_in_sec = (unsigned long)r->age * c->aggr_interval / 1000000; | |
| 106 | for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec; | |
| 107 | age_in_log++, age_in_sec >>= 1) | |
| 108 | ; | |
| 109 | ||
| 110 | /* If frequency is 0, higher age means it's colder */ | |
| 111 | if (freq_subscore == 0) | |
| 112 | age_in_log *= -1; | |
| 113 | ||
| 114 | /* | |
| 115 | * Now age_in_log is in [-DAMON_MAX_AGE_IN_LOG, DAMON_MAX_AGE_IN_LOG]. | |
| 116 | * Scale it to be in [0, 100] and set it as age subscore. | |
| 117 | */ | |
| 118 | age_in_log += DAMON_MAX_AGE_IN_LOG; | |
| 119 | age_subscore = age_in_log * DAMON_MAX_SUBSCORE / | |
| 120 | DAMON_MAX_AGE_IN_LOG / 2; | |
| 121 | ||
| 122 | hotness = (freq_weight * freq_subscore + age_weight * age_subscore); | |
| 123 | if (freq_weight + age_weight) | |
| 124 | hotness /= freq_weight + age_weight; | |
| 125 | /* | |
| 126 | * Transform it to fit in [0, DAMOS_MAX_SCORE] | |
| 127 | */ | |
| 128 | hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE; | |
| 129 | ||
| 130 | /* Return coldness of the region */ | |
| 131 | return DAMOS_MAX_SCORE - hotness; | |
| 132 | } |