dm: dm-zoned: use __bio_add_page for adding single metadata page

[linux-block.git] / mm / compaction.c

diff --git a/mm/compaction.c b/mm/compaction.c
index 5a9501e0ae0174f3093d764b3e339d0dd0d40b7f..c8bcdea15f5f918ac514d9f81f726ffdb4a12efa 100644 (file)
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -583,9 +583,10 @@ static unsigned long isolate_freepages_block(struct compact_control *cc,
                if (PageCompound(page)) {
                        const unsigned int order = compound_order(page);
 
-                       if (likely(order < MAX_ORDER)) {
+                       if (likely(order <= MAX_ORDER)) {
                                blockpfn += (1UL << order) - 1;
                                cursor += (1UL << order) - 1;
+                               nr_scanned += (1UL << order) - 1;
                        }
                        goto isolate_fail;
                }
@@ -893,6 +894,11 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                }
 
                if (PageHuge(page) && cc->alloc_contig) {
+                       if (locked) {
+                               unlock_page_lruvec_irqrestore(locked, flags);
+                               locked = NULL;
+                       }
+
                        ret = isolate_or_dissolve_huge_page(page, &cc->migratepages);
 
                        /*
@@ -904,6 +910,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                                if (ret == -EBUSY)
                                        ret = 0;
                                low_pfn += compound_nr(page) - 1;
+                               nr_scanned += compound_nr(page) - 1;
                                goto isolate_fail;
                        }
 
@@ -938,8 +945,10 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                         * a valid page order. Consider only values in the
                         * valid order range to prevent low_pfn overflow.
                         */
-                       if (freepage_order > 0 && freepage_order < MAX_ORDER)
+                       if (freepage_order > 0 && freepage_order <= MAX_ORDER) {
                                low_pfn += (1UL << freepage_order) - 1;
+                               nr_scanned += (1UL << freepage_order) - 1;
+                       }
                        continue;
                }
 
@@ -954,8 +963,10 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                if (PageCompound(page) && !cc->alloc_contig) {
                        const unsigned int order = compound_order(page);
 
-                       if (likely(order < MAX_ORDER))
+                       if (likely(order <= MAX_ORDER)) {
                                low_pfn += (1UL << order) - 1;
+                               nr_scanned += (1UL << order) - 1;
+                       }
                        goto isolate_fail;
                }
 
@@ -1077,6 +1088,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                         */
                        if (unlikely(PageCompound(page) && !cc->alloc_contig)) {
                                low_pfn += compound_nr(page) - 1;
+                               nr_scanned += compound_nr(page) - 1;
                                SetPageLRU(page);
                                goto isolate_fail_put;
                        }
@@ -1716,7 +1728,14 @@ typedef enum {
  * Allow userspace to control policy on scanning the unevictable LRU for
  * compactable pages.
  */
-int sysctl_compact_unevictable_allowed __read_mostly = CONFIG_COMPACT_UNEVICTABLE_DEFAULT;
+static int sysctl_compact_unevictable_allowed __read_mostly = CONFIG_COMPACT_UNEVICTABLE_DEFAULT;
+/*
+ * Tunable for proactive compaction. It determines how
+ * aggressively the kernel should compact memory in the
+ * background. It takes values in the range [0, 100].
+ */
+static unsigned int __read_mostly sysctl_compaction_proactiveness = 20;
+static int sysctl_extfrag_threshold = 500;
 
 static inline void
 update_fast_start_pfn(struct compact_control *cc, unsigned long pfn)
@@ -2124,7 +2143,7 @@ static enum compact_result __compact_finished(struct compact_control *cc)
 
        /* Direct compactor: Is a suitable page free? */
        ret = COMPACT_NO_SUITABLE_PAGE;
-       for (order = cc->order; order < MAX_ORDER; order++) {
+       for (order = cc->order; order <= MAX_ORDER; order++) {
                struct free_area *area = &cc->zone->free_area[order];
                bool can_steal;
 
@@ -2572,8 +2591,6 @@ static enum compact_result compact_zone_order(struct zone *zone, int order,
        return ret;
 }
 
-int sysctl_extfrag_threshold = 500;
-
 /**
  * try_to_compact_pages - Direct compact to satisfy a high-order allocation
  * @gfp_mask: The GFP mask of the current allocation
@@ -2730,14 +2747,7 @@ static void compact_nodes(void)
                compact_node(nid);
 }
 
-/*
- * Tunable for proactive compaction. It determines how
- * aggressively the kernel should compact memory in the
- * background. It takes values in the range [0, 100].
- */
-unsigned int __read_mostly sysctl_compaction_proactiveness = 20;
-
-int compaction_proactiveness_sysctl_handler(struct ctl_table *table, int write,
+static int compaction_proactiveness_sysctl_handler(struct ctl_table *table, int write,
                void *buffer, size_t *length, loff_t *ppos)
 {
        int rc, nid;
@@ -2767,7 +2777,7 @@ int compaction_proactiveness_sysctl_handler(struct ctl_table *table, int write,
  * This is the entry point for compacting all nodes via
  * /proc/sys/vm/compact_memory
  */
-int sysctl_compaction_handler(struct ctl_table *table, int write,
+static int sysctl_compaction_handler(struct ctl_table *table, int write,
                        void *buffer, size_t *length, loff_t *ppos)
 {
        if (write)
@@ -3063,6 +3073,63 @@ static int kcompactd_cpu_online(unsigned int cpu)
        return 0;
 }
 
+static int proc_dointvec_minmax_warn_RT_change(struct ctl_table *table,
+               int write, void *buffer, size_t *lenp, loff_t *ppos)
+{
+       int ret, old;
+
+       if (!IS_ENABLED(CONFIG_PREEMPT_RT) || !write)
+               return proc_dointvec_minmax(table, write, buffer, lenp, ppos);
+
+       old = *(int *)table->data;
+       ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
+       if (ret)
+               return ret;
+       if (old != *(int *)table->data)
+               pr_warn_once("sysctl attribute %s changed by %s[%d]\n",
+                            table->procname, current->comm,
+                            task_pid_nr(current));
+       return ret;
+}
+
+static struct ctl_table vm_compaction[] = {
+       {
+               .procname       = "compact_memory",
+               .data           = NULL,
+               .maxlen         = sizeof(int),
+               .mode           = 0200,
+               .proc_handler   = sysctl_compaction_handler,
+       },
+       {
+               .procname       = "compaction_proactiveness",
+               .data           = &sysctl_compaction_proactiveness,
+               .maxlen         = sizeof(sysctl_compaction_proactiveness),
+               .mode           = 0644,
+               .proc_handler   = compaction_proactiveness_sysctl_handler,
+               .extra1         = SYSCTL_ZERO,
+               .extra2         = SYSCTL_ONE_HUNDRED,
+       },
+       {
+               .procname       = "extfrag_threshold",
+               .data           = &sysctl_extfrag_threshold,
+               .maxlen         = sizeof(int),
+               .mode           = 0644,
+               .proc_handler   = proc_dointvec_minmax,
+               .extra1         = SYSCTL_ZERO,
+               .extra2         = SYSCTL_ONE_THOUSAND,
+       },
+       {
+               .procname       = "compact_unevictable_allowed",
+               .data           = &sysctl_compact_unevictable_allowed,
+               .maxlen         = sizeof(int),
+               .mode           = 0644,
+               .proc_handler   = proc_dointvec_minmax_warn_RT_change,
+               .extra1         = SYSCTL_ZERO,
+               .extra2         = SYSCTL_ONE,
+       },
+       { }
+};
+
 static int __init kcompactd_init(void)
 {
        int nid;
@@ -3078,6 +3145,7 @@ static int __init kcompactd_init(void)
 
        for_each_node_state(nid, N_MEMORY)
                kcompactd_run(nid);
+       register_sysctl_init("vm", vm_compaction);
        return 0;
 }
 subsys_initcall(kcompactd_init)