Merge tag 'sched-psi-2022-10-14' of git://git.kernel.org/pub/scm/linux/kernel/git...

[linux-block.git] / kernel / sched / core.c

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 8cd1b5a8f61331a642db3c4b56c2383cd7e4fca3..5800b0623ff30687cf60b24e5109fc40e5ee9229 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -357,10 +357,7 @@ static void __sched_core_flip(bool enabled)
        /*
         * Toggle the offline CPUs.
         */
-       cpumask_copy(&sched_core_mask, cpu_possible_mask);
-       cpumask_andnot(&sched_core_mask, &sched_core_mask, cpu_online_mask);
-
-       for_each_cpu(cpu, &sched_core_mask)
+       for_each_cpu_andnot(cpu, cpu_possible_mask, cpu_online_mask)
                cpu_rq(cpu)->core_enabled = enabled;
 
        cpus_read_unlock();
@@ -704,6 +701,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta)
 
        rq->prev_irq_time += irq_delta;
        delta -= irq_delta;
+       psi_account_irqtime(rq->curr, irq_delta);
 #endif
 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
        if (static_key_false((&paravirt_steal_rq_enabled))) {
@@ -4392,6 +4390,17 @@ void set_numabalancing_state(bool enabled)
 }
 
 #ifdef CONFIG_PROC_SYSCTL
+static void reset_memory_tiering(void)
+{
+       struct pglist_data *pgdat;
+
+       for_each_online_pgdat(pgdat) {
+               pgdat->nbp_threshold = 0;
+               pgdat->nbp_th_nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE);
+               pgdat->nbp_th_start = jiffies_to_msecs(jiffies);
+       }
+}
+
 int sysctl_numa_balancing(struct ctl_table *table, int write,
                          void *buffer, size_t *lenp, loff_t *ppos)
 {
@@ -4408,6 +4417,9 @@ int sysctl_numa_balancing(struct ctl_table *table, int write,
        if (err < 0)
                return err;
        if (write) {
+               if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&
+                   (state & NUMA_BALANCING_MEMORY_TIERING))
+                       reset_memory_tiering();
                sysctl_numa_balancing_mode = state;
                __set_numabalancing_state(state);
        }
@@ -5162,6 +5174,7 @@ context_switch(struct rq *rq, struct task_struct *prev,
                 * finish_task_switch()'s mmdrop().
                 */
                switch_mm_irqs_off(prev->active_mm, next->mm, next);
+               lru_gen_use_mm(next->mm);
 
                if (!prev->mm) {                        // from kernel
                        /* will mmdrop() in finish_task_switch(). */