| 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | #include <linux/swap_cgroup.h> |
| 3 | #include <linux/vmalloc.h> |
| 4 | #include <linux/mm.h> |
| 5 | |
| 6 | #include <linux/swapops.h> /* depends on mm.h include */ |
| 7 | |
| 8 | static DEFINE_MUTEX(swap_cgroup_mutex); |
| 9 | struct swap_cgroup_ctrl { |
| 10 | struct page **map; |
| 11 | unsigned long length; |
| 12 | spinlock_t lock; |
| 13 | }; |
| 14 | |
| 15 | static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES]; |
| 16 | |
| 17 | struct swap_cgroup { |
| 18 | unsigned short id; |
| 19 | }; |
| 20 | #define SC_PER_PAGE (PAGE_SIZE/sizeof(struct swap_cgroup)) |
| 21 | |
| 22 | /* |
| 23 | * SwapCgroup implements "lookup" and "exchange" operations. |
| 24 | * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge |
| 25 | * against SwapCache. At swap_free(), this is accessed directly from swap. |
| 26 | * |
| 27 | * This means, |
| 28 | * - we have no race in "exchange" when we're accessed via SwapCache because |
| 29 | * SwapCache(and its swp_entry) is under lock. |
| 30 | * - When called via swap_free(), there is no user of this entry and no race. |
| 31 | * Then, we don't need lock around "exchange". |
| 32 | * |
| 33 | * TODO: we can push these buffers out to HIGHMEM. |
| 34 | */ |
| 35 | |
| 36 | /* |
| 37 | * allocate buffer for swap_cgroup. |
| 38 | */ |
| 39 | static int swap_cgroup_prepare(int type) |
| 40 | { |
| 41 | struct page *page; |
| 42 | struct swap_cgroup_ctrl *ctrl; |
| 43 | unsigned long idx, max; |
| 44 | |
| 45 | ctrl = &swap_cgroup_ctrl[type]; |
| 46 | |
| 47 | for (idx = 0; idx < ctrl->length; idx++) { |
| 48 | page = alloc_page(GFP_KERNEL | __GFP_ZERO); |
| 49 | if (!page) |
| 50 | goto not_enough_page; |
| 51 | ctrl->map[idx] = page; |
| 52 | |
| 53 | if (!(idx % SWAP_CLUSTER_MAX)) |
| 54 | cond_resched(); |
| 55 | } |
| 56 | return 0; |
| 57 | not_enough_page: |
| 58 | max = idx; |
| 59 | for (idx = 0; idx < max; idx++) |
| 60 | __free_page(ctrl->map[idx]); |
| 61 | |
| 62 | return -ENOMEM; |
| 63 | } |
| 64 | |
| 65 | static struct swap_cgroup *__lookup_swap_cgroup(struct swap_cgroup_ctrl *ctrl, |
| 66 | pgoff_t offset) |
| 67 | { |
| 68 | struct page *mappage; |
| 69 | struct swap_cgroup *sc; |
| 70 | |
| 71 | mappage = ctrl->map[offset / SC_PER_PAGE]; |
| 72 | sc = page_address(mappage); |
| 73 | return sc + offset % SC_PER_PAGE; |
| 74 | } |
| 75 | |
| 76 | static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent, |
| 77 | struct swap_cgroup_ctrl **ctrlp) |
| 78 | { |
| 79 | pgoff_t offset = swp_offset(ent); |
| 80 | struct swap_cgroup_ctrl *ctrl; |
| 81 | |
| 82 | ctrl = &swap_cgroup_ctrl[swp_type(ent)]; |
| 83 | if (ctrlp) |
| 84 | *ctrlp = ctrl; |
| 85 | return __lookup_swap_cgroup(ctrl, offset); |
| 86 | } |
| 87 | |
| 88 | /** |
| 89 | * swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry. |
| 90 | * @ent: swap entry to be cmpxchged |
| 91 | * @old: old id |
| 92 | * @new: new id |
| 93 | * |
| 94 | * Returns old id at success, 0 at failure. |
| 95 | * (There is no mem_cgroup using 0 as its id) |
| 96 | */ |
| 97 | unsigned short swap_cgroup_cmpxchg(swp_entry_t ent, |
| 98 | unsigned short old, unsigned short new) |
| 99 | { |
| 100 | struct swap_cgroup_ctrl *ctrl; |
| 101 | struct swap_cgroup *sc; |
| 102 | unsigned long flags; |
| 103 | unsigned short retval; |
| 104 | |
| 105 | sc = lookup_swap_cgroup(ent, &ctrl); |
| 106 | |
| 107 | spin_lock_irqsave(&ctrl->lock, flags); |
| 108 | retval = sc->id; |
| 109 | if (retval == old) |
| 110 | sc->id = new; |
| 111 | else |
| 112 | retval = 0; |
| 113 | spin_unlock_irqrestore(&ctrl->lock, flags); |
| 114 | return retval; |
| 115 | } |
| 116 | |
| 117 | /** |
| 118 | * swap_cgroup_record - record mem_cgroup for a set of swap entries |
| 119 | * @ent: the first swap entry to be recorded into |
| 120 | * @id: mem_cgroup to be recorded |
| 121 | * @nr_ents: number of swap entries to be recorded |
| 122 | * |
| 123 | * Returns old value at success, 0 at failure. |
| 124 | * (Of course, old value can be 0.) |
| 125 | */ |
| 126 | unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id, |
| 127 | unsigned int nr_ents) |
| 128 | { |
| 129 | struct swap_cgroup_ctrl *ctrl; |
| 130 | struct swap_cgroup *sc; |
| 131 | unsigned short old; |
| 132 | unsigned long flags; |
| 133 | pgoff_t offset = swp_offset(ent); |
| 134 | pgoff_t end = offset + nr_ents; |
| 135 | |
| 136 | sc = lookup_swap_cgroup(ent, &ctrl); |
| 137 | |
| 138 | spin_lock_irqsave(&ctrl->lock, flags); |
| 139 | old = sc->id; |
| 140 | for (;;) { |
| 141 | VM_BUG_ON(sc->id != old); |
| 142 | sc->id = id; |
| 143 | offset++; |
| 144 | if (offset == end) |
| 145 | break; |
| 146 | if (offset % SC_PER_PAGE) |
| 147 | sc++; |
| 148 | else |
| 149 | sc = __lookup_swap_cgroup(ctrl, offset); |
| 150 | } |
| 151 | spin_unlock_irqrestore(&ctrl->lock, flags); |
| 152 | |
| 153 | return old; |
| 154 | } |
| 155 | |
| 156 | /** |
| 157 | * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry |
| 158 | * @ent: swap entry to be looked up. |
| 159 | * |
| 160 | * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID) |
| 161 | */ |
| 162 | unsigned short lookup_swap_cgroup_id(swp_entry_t ent) |
| 163 | { |
| 164 | return lookup_swap_cgroup(ent, NULL)->id; |
| 165 | } |
| 166 | |
| 167 | int swap_cgroup_swapon(int type, unsigned long max_pages) |
| 168 | { |
| 169 | void *array; |
| 170 | unsigned long length; |
| 171 | struct swap_cgroup_ctrl *ctrl; |
| 172 | |
| 173 | if (mem_cgroup_disabled()) |
| 174 | return 0; |
| 175 | |
| 176 | length = DIV_ROUND_UP(max_pages, SC_PER_PAGE); |
| 177 | |
| 178 | array = vcalloc(length, sizeof(void *)); |
| 179 | if (!array) |
| 180 | goto nomem; |
| 181 | |
| 182 | ctrl = &swap_cgroup_ctrl[type]; |
| 183 | mutex_lock(&swap_cgroup_mutex); |
| 184 | ctrl->length = length; |
| 185 | ctrl->map = array; |
| 186 | spin_lock_init(&ctrl->lock); |
| 187 | if (swap_cgroup_prepare(type)) { |
| 188 | /* memory shortage */ |
| 189 | ctrl->map = NULL; |
| 190 | ctrl->length = 0; |
| 191 | mutex_unlock(&swap_cgroup_mutex); |
| 192 | vfree(array); |
| 193 | goto nomem; |
| 194 | } |
| 195 | mutex_unlock(&swap_cgroup_mutex); |
| 196 | |
| 197 | return 0; |
| 198 | nomem: |
| 199 | pr_info("couldn't allocate enough memory for swap_cgroup\n"); |
| 200 | pr_info("swap_cgroup can be disabled by swapaccount=0 boot option\n"); |
| 201 | return -ENOMEM; |
| 202 | } |
| 203 | |
| 204 | void swap_cgroup_swapoff(int type) |
| 205 | { |
| 206 | struct page **map; |
| 207 | unsigned long i, length; |
| 208 | struct swap_cgroup_ctrl *ctrl; |
| 209 | |
| 210 | if (mem_cgroup_disabled()) |
| 211 | return; |
| 212 | |
| 213 | mutex_lock(&swap_cgroup_mutex); |
| 214 | ctrl = &swap_cgroup_ctrl[type]; |
| 215 | map = ctrl->map; |
| 216 | length = ctrl->length; |
| 217 | ctrl->map = NULL; |
| 218 | ctrl->length = 0; |
| 219 | mutex_unlock(&swap_cgroup_mutex); |
| 220 | |
| 221 | if (map) { |
| 222 | for (i = 0; i < length; i++) { |
| 223 | struct page *page = map[i]; |
| 224 | if (page) |
| 225 | __free_page(page); |
| 226 | if (!(i % SWAP_CLUSTER_MAX)) |
| 227 | cond_resched(); |
| 228 | } |
| 229 | vfree(map); |
| 230 | } |
| 231 | } |