mm/damon/sysfs-schemes.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * DAMON sysfs Interface
   4  *
   5  * Copyright (c) 2022 SeongJae Park <sj@kernel.org>
   6  */
   7
   8 #include <linux/slab.h>
   9
  10 #include "sysfs-common.h"
  11
  12 /*
  13  * scheme region directory
  14  */
  15
  16 struct damon_sysfs_scheme_region {
  17         struct kobject kobj;
  18         struct damon_addr_range ar;
  19         unsigned int nr_accesses;
  20         unsigned int age;
  21         struct list_head list;
  22 };
  23
  24 static struct damon_sysfs_scheme_region *damon_sysfs_scheme_region_alloc(
  25                 struct damon_region *region)
  26 {
  27         struct damon_sysfs_scheme_region *sysfs_region = kmalloc(
  28                         sizeof(*sysfs_region), GFP_KERNEL);
  29
  30         if (!sysfs_region)
  31                 return NULL;
  32         sysfs_region->kobj = (struct kobject){};
  33         sysfs_region->ar = region->ar;
  34         sysfs_region->nr_accesses = region->nr_accesses_bp / 10000;
  35         sysfs_region->age = region->age;
  36         INIT_LIST_HEAD(&sysfs_region->list);
  37         return sysfs_region;
  38 }
  39
  40 static ssize_t start_show(struct kobject *kobj, struct kobj_attribute *attr,
  41                 char *buf)
  42 {
  43         struct damon_sysfs_scheme_region *region = container_of(kobj,
  44                         struct damon_sysfs_scheme_region, kobj);
  45
  46         return sysfs_emit(buf, "%lu\n", region->ar.start);
  47 }
  48
  49 static ssize_t end_show(struct kobject *kobj, struct kobj_attribute *attr,
  50                 char *buf)
  51 {
  52         struct damon_sysfs_scheme_region *region = container_of(kobj,
  53                         struct damon_sysfs_scheme_region, kobj);
  54
  55         return sysfs_emit(buf, "%lu\n", region->ar.end);
  56 }
  57
  58 static ssize_t nr_accesses_show(struct kobject *kobj,
  59                 struct kobj_attribute *attr, char *buf)
  60 {
  61         struct damon_sysfs_scheme_region *region = container_of(kobj,
  62                         struct damon_sysfs_scheme_region, kobj);
  63
  64         return sysfs_emit(buf, "%u\n", region->nr_accesses);
  65 }
  66
  67 static ssize_t age_show(struct kobject *kobj, struct kobj_attribute *attr,
  68                 char *buf)
  69 {
  70         struct damon_sysfs_scheme_region *region = container_of(kobj,
  71                         struct damon_sysfs_scheme_region, kobj);
  72
  73         return sysfs_emit(buf, "%u\n", region->age);
  74 }
  75
  76 static void damon_sysfs_scheme_region_release(struct kobject *kobj)
  77 {
  78         struct damon_sysfs_scheme_region *region = container_of(kobj,
  79                         struct damon_sysfs_scheme_region, kobj);
  80
  81         list_del(&region->list);
  82         kfree(region);
  83 }
  84
  85 static struct kobj_attribute damon_sysfs_scheme_region_start_attr =
  86                 __ATTR_RO_MODE(start, 0400);
  87
  88 static struct kobj_attribute damon_sysfs_scheme_region_end_attr =
  89                 __ATTR_RO_MODE(end, 0400);
  90
  91 static struct kobj_attribute damon_sysfs_scheme_region_nr_accesses_attr =
  92                 __ATTR_RO_MODE(nr_accesses, 0400);
  93
  94 static struct kobj_attribute damon_sysfs_scheme_region_age_attr =
  95                 __ATTR_RO_MODE(age, 0400);
  96
  97 static struct attribute *damon_sysfs_scheme_region_attrs[] = {
  98         &damon_sysfs_scheme_region_start_attr.attr,
  99         &damon_sysfs_scheme_region_end_attr.attr,
 100         &damon_sysfs_scheme_region_nr_accesses_attr.attr,
 101         &damon_sysfs_scheme_region_age_attr.attr,
 102         NULL,
 103 };
 104 ATTRIBUTE_GROUPS(damon_sysfs_scheme_region);
 105
 106 static const struct kobj_type damon_sysfs_scheme_region_ktype = {
 107         .release = damon_sysfs_scheme_region_release,
 108         .sysfs_ops = &kobj_sysfs_ops,
 109         .default_groups = damon_sysfs_scheme_region_groups,
 110 };
 111
 112 /*
 113  * scheme regions directory
 114  */
 115
 116 /*
 117  * enum damos_sysfs_regions_upd_status - Represent DAMOS tried regions update
 118  *                                       status
 119  * @DAMOS_TRIED_REGIONS_UPD_IDLE:               Waiting for next request.
 120  * @DAMOS_TRIED_REGIONS_UPD_STARTED:            Update started.
 121  * @DAMOS_TRIED_REGIONS_UPD_FINISHED:   Update finished.
 122  *
 123  * Each DAMON-based operation scheme (&struct damos) has its own apply
 124  * interval, and we need to expose the scheme tried regions based on only
 125  * single snapshot.  For this, we keep the tried regions update status for each
 126  * scheme.  The status becomes 'idle' at the beginning.
 127  *
 128  * Once the tried regions update request is received, the request handling
 129  * start function (damon_sysfs_scheme_update_regions_start()) sets the status
 130  * of all schemes as 'idle' again, and register ->before_damos_apply() and
 131  * ->after_sampling() callbacks.
 132  *
 133  * Then, the first followup ->before_damos_apply() callback
 134  * (damon_sysfs_before_damos_apply()) sets the status 'started'.  The first
 135  * ->after_sampling() callback (damon_sysfs_after_sampling()) after the call
 136  * is called only after the scheme is completely applied
 137  * to the given snapshot.  Hence the callback knows the situation by showing
 138  * 'started' status, and sets the status as 'finished'.  Then,
 139  * damon_sysfs_before_damos_apply() understands the situation by showing the
 140  * 'finished' status and do nothing.
 141  *
 142  * If DAMOS is not applied to any region due to any reasons including the
 143  * access pattern, the watermarks, the quotas, and the filters,
 144  * ->before_damos_apply() will not be called back.  Until the situation is
 145  * changed, the update will not be finished.  To avoid this,
 146  * damon_sysfs_after_sampling() set the status as 'finished' if more than two
 147  * apply intervals of the scheme is passed while the state is 'idle'.
 148  *
 149  *  Finally, the tried regions request handling finisher function
 150  *  (damon_sysfs_schemes_update_regions_stop()) unregisters the callbacks.
 151  */
 152 enum damos_sysfs_regions_upd_status {
 153         DAMOS_TRIED_REGIONS_UPD_IDLE,
 154         DAMOS_TRIED_REGIONS_UPD_STARTED,
 155         DAMOS_TRIED_REGIONS_UPD_FINISHED,
 156 };
 157
 158 struct damon_sysfs_scheme_regions {
 159         struct kobject kobj;
 160         struct list_head regions_list;
 161         int nr_regions;
 162         unsigned long total_bytes;
 163         enum damos_sysfs_regions_upd_status upd_status;
 164         unsigned long upd_timeout_jiffies;
 165 };
 166
 167 static struct damon_sysfs_scheme_regions *
 168 damon_sysfs_scheme_regions_alloc(void)
 169 {
 170         struct damon_sysfs_scheme_regions *regions = kmalloc(sizeof(*regions),
 171                         GFP_KERNEL);
 172
 173         if (!regions)
 174                 return NULL;
 175
 176         regions->kobj = (struct kobject){};
 177         INIT_LIST_HEAD(&regions->regions_list);
 178         regions->nr_regions = 0;
 179         regions->total_bytes = 0;
 180         regions->upd_status = DAMOS_TRIED_REGIONS_UPD_IDLE;
 181         return regions;
 182 }
 183
 184 static ssize_t total_bytes_show(struct kobject *kobj,
 185                 struct kobj_attribute *attr, char *buf)
 186 {
 187         struct damon_sysfs_scheme_regions *regions = container_of(kobj,
 188                         struct damon_sysfs_scheme_regions, kobj);
 189
 190         return sysfs_emit(buf, "%lu\n", regions->total_bytes);
 191 }
 192
 193 static void damon_sysfs_scheme_regions_rm_dirs(
 194                 struct damon_sysfs_scheme_regions *regions)
 195 {
 196         struct damon_sysfs_scheme_region *r, *next;
 197
 198         list_for_each_entry_safe(r, next, &regions->regions_list, list) {
 199                 /* release function deletes it from the list */
 200                 kobject_put(&r->kobj);
 201                 regions->nr_regions--;
 202         }
 203 }
 204
 205 static void damon_sysfs_scheme_regions_release(struct kobject *kobj)
 206 {
 207         kfree(container_of(kobj, struct damon_sysfs_scheme_regions, kobj));
 208 }
 209
 210 static struct kobj_attribute damon_sysfs_scheme_regions_total_bytes_attr =
 211                 __ATTR_RO_MODE(total_bytes, 0400);
 212
 213 static struct attribute *damon_sysfs_scheme_regions_attrs[] = {
 214         &damon_sysfs_scheme_regions_total_bytes_attr.attr,
 215         NULL,
 216 };
 217 ATTRIBUTE_GROUPS(damon_sysfs_scheme_regions);
 218
 219 static const struct kobj_type damon_sysfs_scheme_regions_ktype = {
 220         .release = damon_sysfs_scheme_regions_release,
 221         .sysfs_ops = &kobj_sysfs_ops,
 222         .default_groups = damon_sysfs_scheme_regions_groups,
 223 };
 224
 225 /*
 226  * schemes/stats directory
 227  */
 228
 229 struct damon_sysfs_stats {
 230         struct kobject kobj;
 231         unsigned long nr_tried;
 232         unsigned long sz_tried;
 233         unsigned long nr_applied;
 234         unsigned long sz_applied;
 235         unsigned long qt_exceeds;
 236 };
 237
 238 static struct damon_sysfs_stats *damon_sysfs_stats_alloc(void)
 239 {
 240         return kzalloc(sizeof(struct damon_sysfs_stats), GFP_KERNEL);
 241 }
 242
 243 static ssize_t nr_tried_show(struct kobject *kobj, struct kobj_attribute *attr,
 244                 char *buf)
 245 {
 246         struct damon_sysfs_stats *stats = container_of(kobj,
 247                         struct damon_sysfs_stats, kobj);
 248
 249         return sysfs_emit(buf, "%lu\n", stats->nr_tried);
 250 }
 251
 252 static ssize_t sz_tried_show(struct kobject *kobj, struct kobj_attribute *attr,
 253                 char *buf)
 254 {
 255         struct damon_sysfs_stats *stats = container_of(kobj,
 256                         struct damon_sysfs_stats, kobj);
 257
 258         return sysfs_emit(buf, "%lu\n", stats->sz_tried);
 259 }
 260
 261 static ssize_t nr_applied_show(struct kobject *kobj,
 262                 struct kobj_attribute *attr, char *buf)
 263 {
 264         struct damon_sysfs_stats *stats = container_of(kobj,
 265                         struct damon_sysfs_stats, kobj);
 266
 267         return sysfs_emit(buf, "%lu\n", stats->nr_applied);
 268 }
 269
 270 static ssize_t sz_applied_show(struct kobject *kobj,
 271                 struct kobj_attribute *attr, char *buf)
 272 {
 273         struct damon_sysfs_stats *stats = container_of(kobj,
 274                         struct damon_sysfs_stats, kobj);
 275
 276         return sysfs_emit(buf, "%lu\n", stats->sz_applied);
 277 }
 278
 279 static ssize_t qt_exceeds_show(struct kobject *kobj,
 280                 struct kobj_attribute *attr, char *buf)
 281 {
 282         struct damon_sysfs_stats *stats = container_of(kobj,
 283                         struct damon_sysfs_stats, kobj);
 284
 285         return sysfs_emit(buf, "%lu\n", stats->qt_exceeds);
 286 }
 287
 288 static void damon_sysfs_stats_release(struct kobject *kobj)
 289 {
 290         kfree(container_of(kobj, struct damon_sysfs_stats, kobj));
 291 }
 292
 293 static struct kobj_attribute damon_sysfs_stats_nr_tried_attr =
 294                 __ATTR_RO_MODE(nr_tried, 0400);
 295
 296 static struct kobj_attribute damon_sysfs_stats_sz_tried_attr =
 297                 __ATTR_RO_MODE(sz_tried, 0400);
 298
 299 static struct kobj_attribute damon_sysfs_stats_nr_applied_attr =
 300                 __ATTR_RO_MODE(nr_applied, 0400);
 301
 302 static struct kobj_attribute damon_sysfs_stats_sz_applied_attr =
 303                 __ATTR_RO_MODE(sz_applied, 0400);
 304
 305 static struct kobj_attribute damon_sysfs_stats_qt_exceeds_attr =
 306                 __ATTR_RO_MODE(qt_exceeds, 0400);
 307
 308 static struct attribute *damon_sysfs_stats_attrs[] = {
 309         &damon_sysfs_stats_nr_tried_attr.attr,
 310         &damon_sysfs_stats_sz_tried_attr.attr,
 311         &damon_sysfs_stats_nr_applied_attr.attr,
 312         &damon_sysfs_stats_sz_applied_attr.attr,
 313         &damon_sysfs_stats_qt_exceeds_attr.attr,
 314         NULL,
 315 };
 316 ATTRIBUTE_GROUPS(damon_sysfs_stats);
 317
 318 static const struct kobj_type damon_sysfs_stats_ktype = {
 319         .release = damon_sysfs_stats_release,
 320         .sysfs_ops = &kobj_sysfs_ops,
 321         .default_groups = damon_sysfs_stats_groups,
 322 };
 323
 324 /*
 325  * filter directory
 326  */
 327
 328 struct damon_sysfs_scheme_filter {
 329         struct kobject kobj;
 330         enum damos_filter_type type;
 331         bool matching;
 332         char *memcg_path;
 333         struct damon_addr_range addr_range;
 334         int target_idx;
 335 };
 336
 337 static struct damon_sysfs_scheme_filter *damon_sysfs_scheme_filter_alloc(void)
 338 {
 339         return kzalloc(sizeof(struct damon_sysfs_scheme_filter), GFP_KERNEL);
 340 }
 341
 342 /* Should match with enum damos_filter_type */
 343 static const char * const damon_sysfs_scheme_filter_type_strs[] = {
 344         "anon",
 345         "memcg",
 346         "addr",
 347         "target",
 348 };
 349
 350 static ssize_t type_show(struct kobject *kobj,
 351                 struct kobj_attribute *attr, char *buf)
 352 {
 353         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 354                         struct damon_sysfs_scheme_filter, kobj);
 355
 356         return sysfs_emit(buf, "%s\n",
 357                         damon_sysfs_scheme_filter_type_strs[filter->type]);
 358 }
 359
 360 static ssize_t type_store(struct kobject *kobj,
 361                 struct kobj_attribute *attr, const char *buf, size_t count)
 362 {
 363         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 364                         struct damon_sysfs_scheme_filter, kobj);
 365         enum damos_filter_type type;
 366         ssize_t ret = -EINVAL;
 367
 368         for (type = 0; type < NR_DAMOS_FILTER_TYPES; type++) {
 369                 if (sysfs_streq(buf, damon_sysfs_scheme_filter_type_strs[
 370                                         type])) {
 371                         filter->type = type;
 372                         ret = count;
 373                         break;
 374                 }
 375         }
 376         return ret;
 377 }
 378
 379 static ssize_t matching_show(struct kobject *kobj,
 380                 struct kobj_attribute *attr, char *buf)
 381 {
 382         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 383                         struct damon_sysfs_scheme_filter, kobj);
 384
 385         return sysfs_emit(buf, "%c\n", filter->matching ? 'Y' : 'N');
 386 }
 387
 388 static ssize_t matching_store(struct kobject *kobj,
 389                 struct kobj_attribute *attr, const char *buf, size_t count)
 390 {
 391         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 392                         struct damon_sysfs_scheme_filter, kobj);
 393         bool matching;
 394         int err = kstrtobool(buf, &matching);
 395
 396         if (err)
 397                 return err;
 398
 399         filter->matching = matching;
 400         return count;
 401 }
 402
 403 static ssize_t memcg_path_show(struct kobject *kobj,
 404                 struct kobj_attribute *attr, char *buf)
 405 {
 406         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 407                         struct damon_sysfs_scheme_filter, kobj);
 408
 409         return sysfs_emit(buf, "%s\n",
 410                         filter->memcg_path ? filter->memcg_path : "");
 411 }
 412
 413 static ssize_t memcg_path_store(struct kobject *kobj,
 414                 struct kobj_attribute *attr, const char *buf, size_t count)
 415 {
 416         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 417                         struct damon_sysfs_scheme_filter, kobj);
 418         char *path = kmalloc(sizeof(*path) * (count + 1), GFP_KERNEL);
 419
 420         if (!path)
 421                 return -ENOMEM;
 422
 423         strscpy(path, buf, count + 1);
 424         filter->memcg_path = path;
 425         return count;
 426 }
 427
 428 static ssize_t addr_start_show(struct kobject *kobj,
 429                 struct kobj_attribute *attr, char *buf)
 430 {
 431         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 432                         struct damon_sysfs_scheme_filter, kobj);
 433
 434         return sysfs_emit(buf, "%lu\n", filter->addr_range.start);
 435 }
 436
 437 static ssize_t addr_start_store(struct kobject *kobj,
 438                 struct kobj_attribute *attr, const char *buf, size_t count)
 439 {
 440         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 441                         struct damon_sysfs_scheme_filter, kobj);
 442         int err = kstrtoul(buf, 0, &filter->addr_range.start);
 443
 444         return err ? err : count;
 445 }
 446
 447 static ssize_t addr_end_show(struct kobject *kobj,
 448                 struct kobj_attribute *attr, char *buf)
 449 {
 450         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 451                         struct damon_sysfs_scheme_filter, kobj);
 452
 453         return sysfs_emit(buf, "%lu\n", filter->addr_range.end);
 454 }
 455
 456 static ssize_t addr_end_store(struct kobject *kobj,
 457                 struct kobj_attribute *attr, const char *buf, size_t count)
 458 {
 459         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 460                         struct damon_sysfs_scheme_filter, kobj);
 461         int err = kstrtoul(buf, 0, &filter->addr_range.end);
 462
 463         return err ? err : count;
 464 }
 465
 466 static ssize_t damon_target_idx_show(struct kobject *kobj,
 467                 struct kobj_attribute *attr, char *buf)
 468 {
 469         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 470                         struct damon_sysfs_scheme_filter, kobj);
 471
 472         return sysfs_emit(buf, "%d\n", filter->target_idx);
 473 }
 474
 475 static ssize_t damon_target_idx_store(struct kobject *kobj,
 476                 struct kobj_attribute *attr, const char *buf, size_t count)
 477 {
 478         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 479                         struct damon_sysfs_scheme_filter, kobj);
 480         int err = kstrtoint(buf, 0, &filter->target_idx);
 481
 482         return err ? err : count;
 483 }
 484
 485 static void damon_sysfs_scheme_filter_release(struct kobject *kobj)
 486 {
 487         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 488                         struct damon_sysfs_scheme_filter, kobj);
 489
 490         kfree(filter->memcg_path);
 491         kfree(filter);
 492 }
 493
 494 static struct kobj_attribute damon_sysfs_scheme_filter_type_attr =
 495                 __ATTR_RW_MODE(type, 0600);
 496
 497 static struct kobj_attribute damon_sysfs_scheme_filter_matching_attr =
 498                 __ATTR_RW_MODE(matching, 0600);
 499
 500 static struct kobj_attribute damon_sysfs_scheme_filter_memcg_path_attr =
 501                 __ATTR_RW_MODE(memcg_path, 0600);
 502
 503 static struct kobj_attribute damon_sysfs_scheme_filter_addr_start_attr =
 504                 __ATTR_RW_MODE(addr_start, 0600);
 505
 506 static struct kobj_attribute damon_sysfs_scheme_filter_addr_end_attr =
 507                 __ATTR_RW_MODE(addr_end, 0600);
 508
 509 static struct kobj_attribute damon_sysfs_scheme_filter_damon_target_idx_attr =
 510                 __ATTR_RW_MODE(damon_target_idx, 0600);
 511
 512 static struct attribute *damon_sysfs_scheme_filter_attrs[] = {
 513         &damon_sysfs_scheme_filter_type_attr.attr,
 514         &damon_sysfs_scheme_filter_matching_attr.attr,
 515         &damon_sysfs_scheme_filter_memcg_path_attr.attr,
 516         &damon_sysfs_scheme_filter_addr_start_attr.attr,
 517         &damon_sysfs_scheme_filter_addr_end_attr.attr,
 518         &damon_sysfs_scheme_filter_damon_target_idx_attr.attr,
 519         NULL,
 520 };
 521 ATTRIBUTE_GROUPS(damon_sysfs_scheme_filter);
 522
 523 static const struct kobj_type damon_sysfs_scheme_filter_ktype = {
 524         .release = damon_sysfs_scheme_filter_release,
 525         .sysfs_ops = &kobj_sysfs_ops,
 526         .default_groups = damon_sysfs_scheme_filter_groups,
 527 };
 528
 529 /*
 530  * filters directory
 531  */
 532
 533 struct damon_sysfs_scheme_filters {
 534         struct kobject kobj;
 535         struct damon_sysfs_scheme_filter **filters_arr;
 536         int nr;
 537 };
 538
 539 static struct damon_sysfs_scheme_filters *
 540 damon_sysfs_scheme_filters_alloc(void)
 541 {
 542         return kzalloc(sizeof(struct damon_sysfs_scheme_filters), GFP_KERNEL);
 543 }
 544
 545 static void damon_sysfs_scheme_filters_rm_dirs(
 546                 struct damon_sysfs_scheme_filters *filters)
 547 {
 548         struct damon_sysfs_scheme_filter **filters_arr = filters->filters_arr;
 549         int i;
 550
 551         for (i = 0; i < filters->nr; i++)
 552                 kobject_put(&filters_arr[i]->kobj);
 553         filters->nr = 0;
 554         kfree(filters_arr);
 555         filters->filters_arr = NULL;
 556 }
 557
 558 static int damon_sysfs_scheme_filters_add_dirs(
 559                 struct damon_sysfs_scheme_filters *filters, int nr_filters)
 560 {
 561         struct damon_sysfs_scheme_filter **filters_arr, *filter;
 562         int err, i;
 563
 564         damon_sysfs_scheme_filters_rm_dirs(filters);
 565         if (!nr_filters)
 566                 return 0;
 567
 568         filters_arr = kmalloc_array(nr_filters, sizeof(*filters_arr),
 569                         GFP_KERNEL | __GFP_NOWARN);
 570         if (!filters_arr)
 571                 return -ENOMEM;
 572         filters->filters_arr = filters_arr;
 573
 574         for (i = 0; i < nr_filters; i++) {
 575                 filter = damon_sysfs_scheme_filter_alloc();
 576                 if (!filter) {
 577                         damon_sysfs_scheme_filters_rm_dirs(filters);
 578                         return -ENOMEM;
 579                 }
 580
 581                 err = kobject_init_and_add(&filter->kobj,
 582                                 &damon_sysfs_scheme_filter_ktype,
 583                                 &filters->kobj, "%d", i);
 584                 if (err) {
 585                         kobject_put(&filter->kobj);
 586                         damon_sysfs_scheme_filters_rm_dirs(filters);
 587                         return err;
 588                 }
 589
 590                 filters_arr[i] = filter;
 591                 filters->nr++;
 592         }
 593         return 0;
 594 }
 595
 596 static ssize_t nr_filters_show(struct kobject *kobj,
 597                 struct kobj_attribute *attr, char *buf)
 598 {
 599         struct damon_sysfs_scheme_filters *filters = container_of(kobj,
 600                         struct damon_sysfs_scheme_filters, kobj);
 601
 602         return sysfs_emit(buf, "%d\n", filters->nr);
 603 }
 604
 605 static ssize_t nr_filters_store(struct kobject *kobj,
 606                 struct kobj_attribute *attr, const char *buf, size_t count)
 607 {
 608         struct damon_sysfs_scheme_filters *filters;
 609         int nr, err = kstrtoint(buf, 0, &nr);
 610
 611         if (err)
 612                 return err;
 613         if (nr < 0)
 614                 return -EINVAL;
 615
 616         filters = container_of(kobj, struct damon_sysfs_scheme_filters, kobj);
 617
 618         if (!mutex_trylock(&damon_sysfs_lock))
 619                 return -EBUSY;
 620         err = damon_sysfs_scheme_filters_add_dirs(filters, nr);
 621         mutex_unlock(&damon_sysfs_lock);
 622         if (err)
 623                 return err;
 624
 625         return count;
 626 }
 627
 628 static void damon_sysfs_scheme_filters_release(struct kobject *kobj)
 629 {
 630         kfree(container_of(kobj, struct damon_sysfs_scheme_filters, kobj));
 631 }
 632
 633 static struct kobj_attribute damon_sysfs_scheme_filters_nr_attr =
 634                 __ATTR_RW_MODE(nr_filters, 0600);
 635
 636 static struct attribute *damon_sysfs_scheme_filters_attrs[] = {
 637         &damon_sysfs_scheme_filters_nr_attr.attr,
 638         NULL,
 639 };
 640 ATTRIBUTE_GROUPS(damon_sysfs_scheme_filters);
 641
 642 static const struct kobj_type damon_sysfs_scheme_filters_ktype = {
 643         .release = damon_sysfs_scheme_filters_release,
 644         .sysfs_ops = &kobj_sysfs_ops,
 645         .default_groups = damon_sysfs_scheme_filters_groups,
 646 };
 647
 648 /*
 649  * watermarks directory
 650  */
 651
 652 struct damon_sysfs_watermarks {
 653         struct kobject kobj;
 654         enum damos_wmark_metric metric;
 655         unsigned long interval_us;
 656         unsigned long high;
 657         unsigned long mid;
 658         unsigned long low;
 659 };
 660
 661 static struct damon_sysfs_watermarks *damon_sysfs_watermarks_alloc(
 662                 enum damos_wmark_metric metric, unsigned long interval_us,
 663                 unsigned long high, unsigned long mid, unsigned long low)
 664 {
 665         struct damon_sysfs_watermarks *watermarks = kmalloc(
 666                         sizeof(*watermarks), GFP_KERNEL);
 667
 668         if (!watermarks)
 669                 return NULL;
 670         watermarks->kobj = (struct kobject){};
 671         watermarks->metric = metric;
 672         watermarks->interval_us = interval_us;
 673         watermarks->high = high;
 674         watermarks->mid = mid;
 675         watermarks->low = low;
 676         return watermarks;
 677 }
 678
 679 /* Should match with enum damos_wmark_metric */
 680 static const char * const damon_sysfs_wmark_metric_strs[] = {
 681         "none",
 682         "free_mem_rate",
 683 };
 684
 685 static ssize_t metric_show(struct kobject *kobj, struct kobj_attribute *attr,
 686                 char *buf)
 687 {
 688         struct damon_sysfs_watermarks *watermarks = container_of(kobj,
 689                         struct damon_sysfs_watermarks, kobj);
 690
 691         return sysfs_emit(buf, "%s\n",
 692                         damon_sysfs_wmark_metric_strs[watermarks->metric]);
 693 }
 694
 695 static ssize_t metric_store(struct kobject *kobj, struct kobj_attribute *attr,
 696                 const char *buf, size_t count)
 697 {
 698         struct damon_sysfs_watermarks *watermarks = container_of(kobj,
 699                         struct damon_sysfs_watermarks, kobj);
 700         enum damos_wmark_metric metric;
 701
 702         for (metric = 0; metric < NR_DAMOS_WMARK_METRICS; metric++) {
 703                 if (sysfs_streq(buf, damon_sysfs_wmark_metric_strs[metric])) {
 704                         watermarks->metric = metric;
 705                         return count;
 706                 }
 707         }
 708         return -EINVAL;
 709 }
 710
 711 static ssize_t interval_us_show(struct kobject *kobj,
 712                 struct kobj_attribute *attr, char *buf)
 713 {
 714         struct damon_sysfs_watermarks *watermarks = container_of(kobj,
 715                         struct damon_sysfs_watermarks, kobj);
 716
 717         return sysfs_emit(buf, "%lu\n", watermarks->interval_us);
 718 }
 719
 720 static ssize_t interval_us_store(struct kobject *kobj,
 721                 struct kobj_attribute *attr, const char *buf, size_t count)
 722 {
 723         struct damon_sysfs_watermarks *watermarks = container_of(kobj,
 724                         struct damon_sysfs_watermarks, kobj);
 725         int err = kstrtoul(buf, 0, &watermarks->interval_us);
 726
 727         return err ? err : count;
 728 }
 729
 730 static ssize_t high_show(struct kobject *kobj,
 731                 struct kobj_attribute *attr, char *buf)
 732 {
 733         struct damon_sysfs_watermarks *watermarks = container_of(kobj,
 734                         struct damon_sysfs_watermarks, kobj);
 735
 736         return sysfs_emit(buf, "%lu\n", watermarks->high);
 737 }
 738
 739 static ssize_t high_store(struct kobject *kobj,
 740                 struct kobj_attribute *attr, const char *buf, size_t count)
 741 {
 742         struct damon_sysfs_watermarks *watermarks = container_of(kobj,
 743                         struct damon_sysfs_watermarks, kobj);
 744         int err = kstrtoul(buf, 0, &watermarks->high);
 745
 746         return err ? err : count;
 747 }
 748
 749 static ssize_t mid_show(struct kobject *kobj,
 750                 struct kobj_attribute *attr, char *buf)
 751 {
 752         struct damon_sysfs_watermarks *watermarks = container_of(kobj,
 753                         struct damon_sysfs_watermarks, kobj);
 754
 755         return sysfs_emit(buf, "%lu\n", watermarks->mid);
 756 }
 757
 758 static ssize_t mid_store(struct kobject *kobj,
 759                 struct kobj_attribute *attr, const char *buf, size_t count)
 760 {
 761         struct damon_sysfs_watermarks *watermarks = container_of(kobj,
 762                         struct damon_sysfs_watermarks, kobj);
 763         int err = kstrtoul(buf, 0, &watermarks->mid);
 764
 765         return err ? err : count;
 766 }
 767
 768 static ssize_t low_show(struct kobject *kobj,
 769                 struct kobj_attribute *attr, char *buf)
 770 {
 771         struct damon_sysfs_watermarks *watermarks = container_of(kobj,
 772                         struct damon_sysfs_watermarks, kobj);
 773
 774         return sysfs_emit(buf, "%lu\n", watermarks->low);
 775 }
 776
 777 static ssize_t low_store(struct kobject *kobj,
 778                 struct kobj_attribute *attr, const char *buf, size_t count)
 779 {
 780         struct damon_sysfs_watermarks *watermarks = container_of(kobj,
 781                         struct damon_sysfs_watermarks, kobj);
 782         int err = kstrtoul(buf, 0, &watermarks->low);
 783
 784         return err ? err : count;
 785 }
 786
 787 static void damon_sysfs_watermarks_release(struct kobject *kobj)
 788 {
 789         kfree(container_of(kobj, struct damon_sysfs_watermarks, kobj));
 790 }
 791
 792 static struct kobj_attribute damon_sysfs_watermarks_metric_attr =
 793                 __ATTR_RW_MODE(metric, 0600);
 794
 795 static struct kobj_attribute damon_sysfs_watermarks_interval_us_attr =
 796                 __ATTR_RW_MODE(interval_us, 0600);
 797
 798 static struct kobj_attribute damon_sysfs_watermarks_high_attr =
 799                 __ATTR_RW_MODE(high, 0600);
 800
 801 static struct kobj_attribute damon_sysfs_watermarks_mid_attr =
 802                 __ATTR_RW_MODE(mid, 0600);
 803
 804 static struct kobj_attribute damon_sysfs_watermarks_low_attr =
 805                 __ATTR_RW_MODE(low, 0600);
 806
 807 static struct attribute *damon_sysfs_watermarks_attrs[] = {
 808         &damon_sysfs_watermarks_metric_attr.attr,
 809         &damon_sysfs_watermarks_interval_us_attr.attr,
 810         &damon_sysfs_watermarks_high_attr.attr,
 811         &damon_sysfs_watermarks_mid_attr.attr,
 812         &damon_sysfs_watermarks_low_attr.attr,
 813         NULL,
 814 };
 815 ATTRIBUTE_GROUPS(damon_sysfs_watermarks);
 816
 817 static const struct kobj_type damon_sysfs_watermarks_ktype = {
 818         .release = damon_sysfs_watermarks_release,
 819         .sysfs_ops = &kobj_sysfs_ops,
 820         .default_groups = damon_sysfs_watermarks_groups,
 821 };
 822
 823 /*
 824  * quota goal directory
 825  */
 826
 827 struct damos_sysfs_quota_goal {
 828         struct kobject kobj;
 829         unsigned long target_value;
 830         unsigned long current_value;
 831 };
 832
 833 static struct damos_sysfs_quota_goal *damos_sysfs_quota_goal_alloc(void)
 834 {
 835         return kzalloc(sizeof(struct damos_sysfs_quota_goal), GFP_KERNEL);
 836 }
 837
 838 static ssize_t target_value_show(struct kobject *kobj,
 839                 struct kobj_attribute *attr, char *buf)
 840 {
 841         struct damos_sysfs_quota_goal *goal = container_of(kobj, struct
 842                         damos_sysfs_quota_goal, kobj);
 843
 844         return sysfs_emit(buf, "%lu\n", goal->target_value);
 845 }
 846
 847 static ssize_t target_value_store(struct kobject *kobj,
 848                 struct kobj_attribute *attr, const char *buf, size_t count)
 849 {
 850         struct damos_sysfs_quota_goal *goal = container_of(kobj, struct
 851                         damos_sysfs_quota_goal, kobj);
 852         int err = kstrtoul(buf, 0, &goal->target_value);
 853
 854         return err ? err : count;
 855 }
 856
 857 static ssize_t current_value_show(struct kobject *kobj,
 858                 struct kobj_attribute *attr, char *buf)
 859 {
 860         struct damos_sysfs_quota_goal *goal = container_of(kobj, struct
 861                         damos_sysfs_quota_goal, kobj);
 862
 863         return sysfs_emit(buf, "%lu\n", goal->current_value);
 864 }
 865
 866 static ssize_t current_value_store(struct kobject *kobj,
 867                 struct kobj_attribute *attr, const char *buf, size_t count)
 868 {
 869         struct damos_sysfs_quota_goal *goal = container_of(kobj, struct
 870                         damos_sysfs_quota_goal, kobj);
 871         int err = kstrtoul(buf, 0, &goal->current_value);
 872
 873         /* feed callback should check existence of this file and read value */
 874         return err ? err : count;
 875 }
 876
 877 static void damos_sysfs_quota_goal_release(struct kobject *kobj)
 878 {
 879         /* or, notify this release to the feed callback */
 880         kfree(container_of(kobj, struct damos_sysfs_quota_goal, kobj));
 881 }
 882
 883 static struct kobj_attribute damos_sysfs_quota_goal_target_value_attr =
 884                 __ATTR_RW_MODE(target_value, 0600);
 885
 886 static struct kobj_attribute damos_sysfs_quota_goal_current_value_attr =
 887                 __ATTR_RW_MODE(current_value, 0600);
 888
 889 static struct attribute *damos_sysfs_quota_goal_attrs[] = {
 890         &damos_sysfs_quota_goal_target_value_attr.attr,
 891         &damos_sysfs_quota_goal_current_value_attr.attr,
 892         NULL,
 893 };
 894 ATTRIBUTE_GROUPS(damos_sysfs_quota_goal);
 895
 896 static const struct kobj_type damos_sysfs_quota_goal_ktype = {
 897         .release = damos_sysfs_quota_goal_release,
 898         .sysfs_ops = &kobj_sysfs_ops,
 899         .default_groups = damos_sysfs_quota_goal_groups,
 900 };
 901
 902 /*
 903  * quota goals directory
 904  */
 905
 906 struct damos_sysfs_quota_goals {
 907         struct kobject kobj;
 908         struct damos_sysfs_quota_goal **goals_arr;      /* counted by nr */
 909         int nr;
 910 };
 911
 912 static struct damos_sysfs_quota_goals *damos_sysfs_quota_goals_alloc(void)
 913 {
 914         return kzalloc(sizeof(struct damos_sysfs_quota_goals), GFP_KERNEL);
 915 }
 916
 917 static void damos_sysfs_quota_goals_rm_dirs(
 918                 struct damos_sysfs_quota_goals *goals)
 919 {
 920         struct damos_sysfs_quota_goal **goals_arr = goals->goals_arr;
 921         int i;
 922
 923         for (i = 0; i < goals->nr; i++)
 924                 kobject_put(&goals_arr[i]->kobj);
 925         goals->nr = 0;
 926         kfree(goals_arr);
 927         goals->goals_arr = NULL;
 928 }
 929
 930 static int damos_sysfs_quota_goals_add_dirs(
 931                 struct damos_sysfs_quota_goals *goals, int nr_goals)
 932 {
 933         struct damos_sysfs_quota_goal **goals_arr, *goal;
 934         int err, i;
 935
 936         damos_sysfs_quota_goals_rm_dirs(goals);
 937         if (!nr_goals)
 938                 return 0;
 939
 940         goals_arr = kmalloc_array(nr_goals, sizeof(*goals_arr),
 941                         GFP_KERNEL | __GFP_NOWARN);
 942         if (!goals_arr)
 943                 return -ENOMEM;
 944         goals->goals_arr = goals_arr;
 945
 946         for (i = 0; i < nr_goals; i++) {
 947                 goal = damos_sysfs_quota_goal_alloc();
 948                 if (!goal) {
 949                         damos_sysfs_quota_goals_rm_dirs(goals);
 950                         return -ENOMEM;
 951                 }
 952
 953                 err = kobject_init_and_add(&goal->kobj,
 954                                 &damos_sysfs_quota_goal_ktype, &goals->kobj,
 955                                 "%d", i);
 956                 if (err) {
 957                         kobject_put(&goal->kobj);
 958                         damos_sysfs_quota_goals_rm_dirs(goals);
 959                         return err;
 960                 }
 961
 962                 goals_arr[i] = goal;
 963                 goals->nr++;
 964         }
 965         return 0;
 966 }
 967
 968 static ssize_t nr_goals_show(struct kobject *kobj,
 969                 struct kobj_attribute *attr, char *buf)
 970 {
 971         struct damos_sysfs_quota_goals *goals = container_of(kobj,
 972                         struct damos_sysfs_quota_goals, kobj);
 973
 974         return sysfs_emit(buf, "%d\n", goals->nr);
 975 }
 976
 977 static ssize_t nr_goals_store(struct kobject *kobj,
 978                 struct kobj_attribute *attr, const char *buf, size_t count)
 979 {
 980         struct damos_sysfs_quota_goals *goals;
 981         int nr, err = kstrtoint(buf, 0, &nr);
 982
 983         if (err)
 984                 return err;
 985         if (nr < 0)
 986                 return -EINVAL;
 987
 988         goals = container_of(kobj, struct damos_sysfs_quota_goals, kobj);
 989
 990         if (!mutex_trylock(&damon_sysfs_lock))
 991                 return -EBUSY;
 992         err = damos_sysfs_quota_goals_add_dirs(goals, nr);
 993         mutex_unlock(&damon_sysfs_lock);
 994         if (err)
 995                 return err;
 996
 997         return count;
 998 }
 999
1000 static void damos_sysfs_quota_goals_release(struct kobject *kobj)
1001 {
1002         kfree(container_of(kobj, struct damos_sysfs_quota_goals, kobj));
1003 }
1004
1005 static struct kobj_attribute damos_sysfs_quota_goals_nr_attr =
1006                 __ATTR_RW_MODE(nr_goals, 0600);
1007
1008 static struct attribute *damos_sysfs_quota_goals_attrs[] = {
1009         &damos_sysfs_quota_goals_nr_attr.attr,
1010         NULL,
1011 };
1012 ATTRIBUTE_GROUPS(damos_sysfs_quota_goals);
1013
1014 static const struct kobj_type damos_sysfs_quota_goals_ktype = {
1015         .release = damos_sysfs_quota_goals_release,
1016         .sysfs_ops = &kobj_sysfs_ops,
1017         .default_groups = damos_sysfs_quota_goals_groups,
1018 };
1019
1020 /*
1021  * scheme/weights directory
1022  */
1023
1024 struct damon_sysfs_weights {
1025         struct kobject kobj;
1026         unsigned int sz;
1027         unsigned int nr_accesses;
1028         unsigned int age;
1029 };
1030
1031 static struct damon_sysfs_weights *damon_sysfs_weights_alloc(unsigned int sz,
1032                 unsigned int nr_accesses, unsigned int age)
1033 {
1034         struct damon_sysfs_weights *weights = kmalloc(sizeof(*weights),
1035                         GFP_KERNEL);
1036
1037         if (!weights)
1038                 return NULL;
1039         weights->kobj = (struct kobject){};
1040         weights->sz = sz;
1041         weights->nr_accesses = nr_accesses;
1042         weights->age = age;
1043         return weights;
1044 }
1045
1046 static ssize_t sz_permil_show(struct kobject *kobj,
1047                 struct kobj_attribute *attr, char *buf)
1048 {
1049         struct damon_sysfs_weights *weights = container_of(kobj,
1050                         struct damon_sysfs_weights, kobj);
1051
1052         return sysfs_emit(buf, "%u\n", weights->sz);
1053 }
1054
1055 static ssize_t sz_permil_store(struct kobject *kobj,
1056                 struct kobj_attribute *attr, const char *buf, size_t count)
1057 {
1058         struct damon_sysfs_weights *weights = container_of(kobj,
1059                         struct damon_sysfs_weights, kobj);
1060         int err = kstrtouint(buf, 0, &weights->sz);
1061
1062         return err ? err : count;
1063 }
1064
1065 static ssize_t nr_accesses_permil_show(struct kobject *kobj,
1066                 struct kobj_attribute *attr, char *buf)
1067 {
1068         struct damon_sysfs_weights *weights = container_of(kobj,
1069                         struct damon_sysfs_weights, kobj);
1070
1071         return sysfs_emit(buf, "%u\n", weights->nr_accesses);
1072 }
1073
1074 static ssize_t nr_accesses_permil_store(struct kobject *kobj,
1075                 struct kobj_attribute *attr, const char *buf, size_t count)
1076 {
1077         struct damon_sysfs_weights *weights = container_of(kobj,
1078                         struct damon_sysfs_weights, kobj);
1079         int err = kstrtouint(buf, 0, &weights->nr_accesses);
1080
1081         return err ? err : count;
1082 }
1083
1084 static ssize_t age_permil_show(struct kobject *kobj,
1085                 struct kobj_attribute *attr, char *buf)
1086 {
1087         struct damon_sysfs_weights *weights = container_of(kobj,
1088                         struct damon_sysfs_weights, kobj);
1089
1090         return sysfs_emit(buf, "%u\n", weights->age);
1091 }
1092
1093 static ssize_t age_permil_store(struct kobject *kobj,
1094                 struct kobj_attribute *attr, const char *buf, size_t count)
1095 {
1096         struct damon_sysfs_weights *weights = container_of(kobj,
1097                         struct damon_sysfs_weights, kobj);
1098         int err = kstrtouint(buf, 0, &weights->age);
1099
1100         return err ? err : count;
1101 }
1102
1103 static void damon_sysfs_weights_release(struct kobject *kobj)
1104 {
1105         kfree(container_of(kobj, struct damon_sysfs_weights, kobj));
1106 }
1107
1108 static struct kobj_attribute damon_sysfs_weights_sz_attr =
1109                 __ATTR_RW_MODE(sz_permil, 0600);
1110
1111 static struct kobj_attribute damon_sysfs_weights_nr_accesses_attr =
1112                 __ATTR_RW_MODE(nr_accesses_permil, 0600);
1113
1114 static struct kobj_attribute damon_sysfs_weights_age_attr =
1115                 __ATTR_RW_MODE(age_permil, 0600);
1116
1117 static struct attribute *damon_sysfs_weights_attrs[] = {
1118         &damon_sysfs_weights_sz_attr.attr,
1119         &damon_sysfs_weights_nr_accesses_attr.attr,
1120         &damon_sysfs_weights_age_attr.attr,
1121         NULL,
1122 };
1123 ATTRIBUTE_GROUPS(damon_sysfs_weights);
1124
1125 static const struct kobj_type damon_sysfs_weights_ktype = {
1126         .release = damon_sysfs_weights_release,
1127         .sysfs_ops = &kobj_sysfs_ops,
1128         .default_groups = damon_sysfs_weights_groups,
1129 };
1130
1131 /*
1132  * quotas directory
1133  */
1134
1135 struct damon_sysfs_quotas {
1136         struct kobject kobj;
1137         struct damon_sysfs_weights *weights;
1138         struct damos_sysfs_quota_goals *goals;
1139         unsigned long ms;
1140         unsigned long sz;
1141         unsigned long reset_interval_ms;
1142 };
1143
1144 static struct damon_sysfs_quotas *damon_sysfs_quotas_alloc(void)
1145 {
1146         return kzalloc(sizeof(struct damon_sysfs_quotas), GFP_KERNEL);
1147 }
1148
1149 static int damon_sysfs_quotas_add_dirs(struct damon_sysfs_quotas *quotas)
1150 {
1151         struct damon_sysfs_weights *weights;
1152         struct damos_sysfs_quota_goals *goals;
1153         int err;
1154
1155         weights = damon_sysfs_weights_alloc(0, 0, 0);
1156         if (!weights)
1157                 return -ENOMEM;
1158
1159         err = kobject_init_and_add(&weights->kobj, &damon_sysfs_weights_ktype,
1160                         &quotas->kobj, "weights");
1161         if (err) {
1162                 kobject_put(&weights->kobj);
1163                 return err;
1164         }
1165         quotas->weights = weights;
1166
1167         goals = damos_sysfs_quota_goals_alloc();
1168         if (!goals) {
1169                 kobject_put(&weights->kobj);
1170                 return -ENOMEM;
1171         }
1172         err = kobject_init_and_add(&goals->kobj,
1173                         &damos_sysfs_quota_goals_ktype, &quotas->kobj,
1174                         "goals");
1175         if (err) {
1176                 kobject_put(&weights->kobj);
1177                 kobject_put(&goals->kobj);
1178         } else {
1179                 quotas->goals = goals;
1180         }
1181
1182         return err;
1183 }
1184
1185 static void damon_sysfs_quotas_rm_dirs(struct damon_sysfs_quotas *quotas)
1186 {
1187         kobject_put(&quotas->weights->kobj);
1188         damos_sysfs_quota_goals_rm_dirs(quotas->goals);
1189         kobject_put(&quotas->goals->kobj);
1190 }
1191
1192 static ssize_t ms_show(struct kobject *kobj, struct kobj_attribute *attr,
1193                 char *buf)
1194 {
1195         struct damon_sysfs_quotas *quotas = container_of(kobj,
1196                         struct damon_sysfs_quotas, kobj);
1197
1198         return sysfs_emit(buf, "%lu\n", quotas->ms);
1199 }
1200
1201 static ssize_t ms_store(struct kobject *kobj, struct kobj_attribute *attr,
1202                 const char *buf, size_t count)
1203 {
1204         struct damon_sysfs_quotas *quotas = container_of(kobj,
1205                         struct damon_sysfs_quotas, kobj);
1206         int err = kstrtoul(buf, 0, &quotas->ms);
1207
1208         if (err)
1209                 return -EINVAL;
1210         return count;
1211 }
1212
1213 static ssize_t bytes_show(struct kobject *kobj, struct kobj_attribute *attr,
1214                 char *buf)
1215 {
1216         struct damon_sysfs_quotas *quotas = container_of(kobj,
1217                         struct damon_sysfs_quotas, kobj);
1218
1219         return sysfs_emit(buf, "%lu\n", quotas->sz);
1220 }
1221
1222 static ssize_t bytes_store(struct kobject *kobj,
1223                 struct kobj_attribute *attr, const char *buf, size_t count)
1224 {
1225         struct damon_sysfs_quotas *quotas = container_of(kobj,
1226                         struct damon_sysfs_quotas, kobj);
1227         int err = kstrtoul(buf, 0, &quotas->sz);
1228
1229         if (err)
1230                 return -EINVAL;
1231         return count;
1232 }
1233
1234 static ssize_t reset_interval_ms_show(struct kobject *kobj,
1235                 struct kobj_attribute *attr, char *buf)
1236 {
1237         struct damon_sysfs_quotas *quotas = container_of(kobj,
1238                         struct damon_sysfs_quotas, kobj);
1239
1240         return sysfs_emit(buf, "%lu\n", quotas->reset_interval_ms);
1241 }
1242
1243 static ssize_t reset_interval_ms_store(struct kobject *kobj,
1244                 struct kobj_attribute *attr, const char *buf, size_t count)
1245 {
1246         struct damon_sysfs_quotas *quotas = container_of(kobj,
1247                         struct damon_sysfs_quotas, kobj);
1248         int err = kstrtoul(buf, 0, &quotas->reset_interval_ms);
1249
1250         if (err)
1251                 return -EINVAL;
1252         return count;
1253 }
1254
1255 static void damon_sysfs_quotas_release(struct kobject *kobj)
1256 {
1257         kfree(container_of(kobj, struct damon_sysfs_quotas, kobj));
1258 }
1259
1260 static struct kobj_attribute damon_sysfs_quotas_ms_attr =
1261                 __ATTR_RW_MODE(ms, 0600);
1262
1263 static struct kobj_attribute damon_sysfs_quotas_sz_attr =
1264                 __ATTR_RW_MODE(bytes, 0600);
1265
1266 static struct kobj_attribute damon_sysfs_quotas_reset_interval_ms_attr =
1267                 __ATTR_RW_MODE(reset_interval_ms, 0600);
1268
1269 static struct attribute *damon_sysfs_quotas_attrs[] = {
1270         &damon_sysfs_quotas_ms_attr.attr,
1271         &damon_sysfs_quotas_sz_attr.attr,
1272         &damon_sysfs_quotas_reset_interval_ms_attr.attr,
1273         NULL,
1274 };
1275 ATTRIBUTE_GROUPS(damon_sysfs_quotas);
1276
1277 static const struct kobj_type damon_sysfs_quotas_ktype = {
1278         .release = damon_sysfs_quotas_release,
1279         .sysfs_ops = &kobj_sysfs_ops,
1280         .default_groups = damon_sysfs_quotas_groups,
1281 };
1282
1283 /*
1284  * access_pattern directory
1285  */
1286
1287 struct damon_sysfs_access_pattern {
1288         struct kobject kobj;
1289         struct damon_sysfs_ul_range *sz;
1290         struct damon_sysfs_ul_range *nr_accesses;
1291         struct damon_sysfs_ul_range *age;
1292 };
1293
1294 static
1295 struct damon_sysfs_access_pattern *damon_sysfs_access_pattern_alloc(void)
1296 {
1297         struct damon_sysfs_access_pattern *access_pattern =
1298                 kmalloc(sizeof(*access_pattern), GFP_KERNEL);
1299
1300         if (!access_pattern)
1301                 return NULL;
1302         access_pattern->kobj = (struct kobject){};
1303         return access_pattern;
1304 }
1305
1306 static int damon_sysfs_access_pattern_add_range_dir(
1307                 struct damon_sysfs_access_pattern *access_pattern,
1308                 struct damon_sysfs_ul_range **range_dir_ptr,
1309                 char *name)
1310 {
1311         struct damon_sysfs_ul_range *range = damon_sysfs_ul_range_alloc(0, 0);
1312         int err;
1313
1314         if (!range)
1315                 return -ENOMEM;
1316         err = kobject_init_and_add(&range->kobj, &damon_sysfs_ul_range_ktype,
1317                         &access_pattern->kobj, name);
1318         if (err)
1319                 kobject_put(&range->kobj);
1320         else
1321                 *range_dir_ptr = range;
1322         return err;
1323 }
1324
1325 static int damon_sysfs_access_pattern_add_dirs(
1326                 struct damon_sysfs_access_pattern *access_pattern)
1327 {
1328         int err;
1329
1330         err = damon_sysfs_access_pattern_add_range_dir(access_pattern,
1331                         &access_pattern->sz, "sz");
1332         if (err)
1333                 goto put_sz_out;
1334
1335         err = damon_sysfs_access_pattern_add_range_dir(access_pattern,
1336                         &access_pattern->nr_accesses, "nr_accesses");
1337         if (err)
1338                 goto put_nr_accesses_sz_out;
1339
1340         err = damon_sysfs_access_pattern_add_range_dir(access_pattern,
1341                         &access_pattern->age, "age");
1342         if (err)
1343                 goto put_age_nr_accesses_sz_out;
1344         return 0;
1345
1346 put_age_nr_accesses_sz_out:
1347         kobject_put(&access_pattern->age->kobj);
1348         access_pattern->age = NULL;
1349 put_nr_accesses_sz_out:
1350         kobject_put(&access_pattern->nr_accesses->kobj);
1351         access_pattern->nr_accesses = NULL;
1352 put_sz_out:
1353         kobject_put(&access_pattern->sz->kobj);
1354         access_pattern->sz = NULL;
1355         return err;
1356 }
1357
1358 static void damon_sysfs_access_pattern_rm_dirs(
1359                 struct damon_sysfs_access_pattern *access_pattern)
1360 {
1361         kobject_put(&access_pattern->sz->kobj);
1362         kobject_put(&access_pattern->nr_accesses->kobj);
1363         kobject_put(&access_pattern->age->kobj);
1364 }
1365
1366 static void damon_sysfs_access_pattern_release(struct kobject *kobj)
1367 {
1368         kfree(container_of(kobj, struct damon_sysfs_access_pattern, kobj));
1369 }
1370
1371 static struct attribute *damon_sysfs_access_pattern_attrs[] = {
1372         NULL,
1373 };
1374 ATTRIBUTE_GROUPS(damon_sysfs_access_pattern);
1375
1376 static const struct kobj_type damon_sysfs_access_pattern_ktype = {
1377         .release = damon_sysfs_access_pattern_release,
1378         .sysfs_ops = &kobj_sysfs_ops,
1379         .default_groups = damon_sysfs_access_pattern_groups,
1380 };
1381
1382 /*
1383  * scheme directory
1384  */
1385
1386 struct damon_sysfs_scheme {
1387         struct kobject kobj;
1388         enum damos_action action;
1389         struct damon_sysfs_access_pattern *access_pattern;
1390         unsigned long apply_interval_us;
1391         struct damon_sysfs_quotas *quotas;
1392         struct damon_sysfs_watermarks *watermarks;
1393         struct damon_sysfs_scheme_filters *filters;
1394         struct damon_sysfs_stats *stats;
1395         struct damon_sysfs_scheme_regions *tried_regions;
1396 };
1397
1398 /* This should match with enum damos_action */
1399 static const char * const damon_sysfs_damos_action_strs[] = {
1400         "willneed",
1401         "cold",
1402         "pageout",
1403         "hugepage",
1404         "nohugepage",
1405         "lru_prio",
1406         "lru_deprio",
1407         "stat",
1408 };
1409
1410 static struct damon_sysfs_scheme *damon_sysfs_scheme_alloc(
1411                 enum damos_action action, unsigned long apply_interval_us)
1412 {
1413         struct damon_sysfs_scheme *scheme = kmalloc(sizeof(*scheme),
1414                                 GFP_KERNEL);
1415
1416         if (!scheme)
1417                 return NULL;
1418         scheme->kobj = (struct kobject){};
1419         scheme->action = action;
1420         scheme->apply_interval_us = apply_interval_us;
1421         return scheme;
1422 }
1423
1424 static int damon_sysfs_scheme_set_access_pattern(
1425                 struct damon_sysfs_scheme *scheme)
1426 {
1427         struct damon_sysfs_access_pattern *access_pattern;
1428         int err;
1429
1430         access_pattern = damon_sysfs_access_pattern_alloc();
1431         if (!access_pattern)
1432                 return -ENOMEM;
1433         err = kobject_init_and_add(&access_pattern->kobj,
1434                         &damon_sysfs_access_pattern_ktype, &scheme->kobj,
1435                         "access_pattern");
1436         if (err)
1437                 goto out;
1438         err = damon_sysfs_access_pattern_add_dirs(access_pattern);
1439         if (err)
1440                 goto out;
1441         scheme->access_pattern = access_pattern;
1442         return 0;
1443
1444 out:
1445         kobject_put(&access_pattern->kobj);
1446         return err;
1447 }
1448
1449 static int damon_sysfs_scheme_set_quotas(struct damon_sysfs_scheme *scheme)
1450 {
1451         struct damon_sysfs_quotas *quotas = damon_sysfs_quotas_alloc();
1452         int err;
1453
1454         if (!quotas)
1455                 return -ENOMEM;
1456         err = kobject_init_and_add(&quotas->kobj, &damon_sysfs_quotas_ktype,
1457                         &scheme->kobj, "quotas");
1458         if (err)
1459                 goto out;
1460         err = damon_sysfs_quotas_add_dirs(quotas);
1461         if (err)
1462                 goto out;
1463         scheme->quotas = quotas;
1464         return 0;
1465
1466 out:
1467         kobject_put(&quotas->kobj);
1468         return err;
1469 }
1470
1471 static int damon_sysfs_scheme_set_watermarks(struct damon_sysfs_scheme *scheme)
1472 {
1473         struct damon_sysfs_watermarks *watermarks =
1474                 damon_sysfs_watermarks_alloc(DAMOS_WMARK_NONE, 0, 0, 0, 0);
1475         int err;
1476
1477         if (!watermarks)
1478                 return -ENOMEM;
1479         err = kobject_init_and_add(&watermarks->kobj,
1480                         &damon_sysfs_watermarks_ktype, &scheme->kobj,
1481                         "watermarks");
1482         if (err)
1483                 kobject_put(&watermarks->kobj);
1484         else
1485                 scheme->watermarks = watermarks;
1486         return err;
1487 }
1488
1489 static int damon_sysfs_scheme_set_filters(struct damon_sysfs_scheme *scheme)
1490 {
1491         struct damon_sysfs_scheme_filters *filters =
1492                 damon_sysfs_scheme_filters_alloc();
1493         int err;
1494
1495         if (!filters)
1496                 return -ENOMEM;
1497         err = kobject_init_and_add(&filters->kobj,
1498                         &damon_sysfs_scheme_filters_ktype, &scheme->kobj,
1499                         "filters");
1500         if (err)
1501                 kobject_put(&filters->kobj);
1502         else
1503                 scheme->filters = filters;
1504         return err;
1505 }
1506
1507 static int damon_sysfs_scheme_set_stats(struct damon_sysfs_scheme *scheme)
1508 {
1509         struct damon_sysfs_stats *stats = damon_sysfs_stats_alloc();
1510         int err;
1511
1512         if (!stats)
1513                 return -ENOMEM;
1514         err = kobject_init_and_add(&stats->kobj, &damon_sysfs_stats_ktype,
1515                         &scheme->kobj, "stats");
1516         if (err)
1517                 kobject_put(&stats->kobj);
1518         else
1519                 scheme->stats = stats;
1520         return err;
1521 }
1522
1523 static int damon_sysfs_scheme_set_tried_regions(
1524                 struct damon_sysfs_scheme *scheme)
1525 {
1526         struct damon_sysfs_scheme_regions *tried_regions =
1527                 damon_sysfs_scheme_regions_alloc();
1528         int err;
1529
1530         if (!tried_regions)
1531                 return -ENOMEM;
1532         err = kobject_init_and_add(&tried_regions->kobj,
1533                         &damon_sysfs_scheme_regions_ktype, &scheme->kobj,
1534                         "tried_regions");
1535         if (err)
1536                 kobject_put(&tried_regions->kobj);
1537         else
1538                 scheme->tried_regions = tried_regions;
1539         return err;
1540 }
1541
1542 static int damon_sysfs_scheme_add_dirs(struct damon_sysfs_scheme *scheme)
1543 {
1544         int err;
1545
1546         err = damon_sysfs_scheme_set_access_pattern(scheme);
1547         if (err)
1548                 return err;
1549         err = damon_sysfs_scheme_set_quotas(scheme);
1550         if (err)
1551                 goto put_access_pattern_out;
1552         err = damon_sysfs_scheme_set_watermarks(scheme);
1553         if (err)
1554                 goto put_quotas_access_pattern_out;
1555         err = damon_sysfs_scheme_set_filters(scheme);
1556         if (err)
1557                 goto put_watermarks_quotas_access_pattern_out;
1558         err = damon_sysfs_scheme_set_stats(scheme);
1559         if (err)
1560                 goto put_filters_watermarks_quotas_access_pattern_out;
1561         err = damon_sysfs_scheme_set_tried_regions(scheme);
1562         if (err)
1563                 goto put_tried_regions_out;
1564         return 0;
1565
1566 put_tried_regions_out:
1567         kobject_put(&scheme->tried_regions->kobj);
1568         scheme->tried_regions = NULL;
1569 put_filters_watermarks_quotas_access_pattern_out:
1570         kobject_put(&scheme->filters->kobj);
1571         scheme->filters = NULL;
1572 put_watermarks_quotas_access_pattern_out:
1573         kobject_put(&scheme->watermarks->kobj);
1574         scheme->watermarks = NULL;
1575 put_quotas_access_pattern_out:
1576         kobject_put(&scheme->quotas->kobj);
1577         scheme->quotas = NULL;
1578 put_access_pattern_out:
1579         kobject_put(&scheme->access_pattern->kobj);
1580         scheme->access_pattern = NULL;
1581         return err;
1582 }
1583
1584 static void damon_sysfs_scheme_rm_dirs(struct damon_sysfs_scheme *scheme)
1585 {
1586         damon_sysfs_access_pattern_rm_dirs(scheme->access_pattern);
1587         kobject_put(&scheme->access_pattern->kobj);
1588         damon_sysfs_quotas_rm_dirs(scheme->quotas);
1589         kobject_put(&scheme->quotas->kobj);
1590         kobject_put(&scheme->watermarks->kobj);
1591         damon_sysfs_scheme_filters_rm_dirs(scheme->filters);
1592         kobject_put(&scheme->filters->kobj);
1593         kobject_put(&scheme->stats->kobj);
1594         damon_sysfs_scheme_regions_rm_dirs(scheme->tried_regions);
1595         kobject_put(&scheme->tried_regions->kobj);
1596 }
1597
1598 static ssize_t action_show(struct kobject *kobj, struct kobj_attribute *attr,
1599                 char *buf)
1600 {
1601         struct damon_sysfs_scheme *scheme = container_of(kobj,
1602                         struct damon_sysfs_scheme, kobj);
1603
1604         return sysfs_emit(buf, "%s\n",
1605                         damon_sysfs_damos_action_strs[scheme->action]);
1606 }
1607
1608 static ssize_t action_store(struct kobject *kobj, struct kobj_attribute *attr,
1609                 const char *buf, size_t count)
1610 {
1611         struct damon_sysfs_scheme *scheme = container_of(kobj,
1612                         struct damon_sysfs_scheme, kobj);
1613         enum damos_action action;
1614
1615         for (action = 0; action < NR_DAMOS_ACTIONS; action++) {
1616                 if (sysfs_streq(buf, damon_sysfs_damos_action_strs[action])) {
1617                         scheme->action = action;
1618                         return count;
1619                 }
1620         }
1621         return -EINVAL;
1622 }
1623
1624 static ssize_t apply_interval_us_show(struct kobject *kobj,
1625                 struct kobj_attribute *attr, char *buf)
1626 {
1627         struct damon_sysfs_scheme *scheme = container_of(kobj,
1628                         struct damon_sysfs_scheme, kobj);
1629
1630         return sysfs_emit(buf, "%lu\n", scheme->apply_interval_us);
1631 }
1632
1633 static ssize_t apply_interval_us_store(struct kobject *kobj,
1634                 struct kobj_attribute *attr, const char *buf, size_t count)
1635 {
1636         struct damon_sysfs_scheme *scheme = container_of(kobj,
1637                         struct damon_sysfs_scheme, kobj);
1638         int err = kstrtoul(buf, 0, &scheme->apply_interval_us);
1639
1640         return err ? err : count;
1641 }
1642
1643 static void damon_sysfs_scheme_release(struct kobject *kobj)
1644 {
1645         kfree(container_of(kobj, struct damon_sysfs_scheme, kobj));
1646 }
1647
1648 static struct kobj_attribute damon_sysfs_scheme_action_attr =
1649                 __ATTR_RW_MODE(action, 0600);
1650
1651 static struct kobj_attribute damon_sysfs_scheme_apply_interval_us_attr =
1652                 __ATTR_RW_MODE(apply_interval_us, 0600);
1653
1654 static struct attribute *damon_sysfs_scheme_attrs[] = {
1655         &damon_sysfs_scheme_action_attr.attr,
1656         &damon_sysfs_scheme_apply_interval_us_attr.attr,
1657         NULL,
1658 };
1659 ATTRIBUTE_GROUPS(damon_sysfs_scheme);
1660
1661 static const struct kobj_type damon_sysfs_scheme_ktype = {
1662         .release = damon_sysfs_scheme_release,
1663         .sysfs_ops = &kobj_sysfs_ops,
1664         .default_groups = damon_sysfs_scheme_groups,
1665 };
1666
1667 /*
1668  * schemes directory
1669  */
1670
1671 struct damon_sysfs_schemes *damon_sysfs_schemes_alloc(void)
1672 {
1673         return kzalloc(sizeof(struct damon_sysfs_schemes), GFP_KERNEL);
1674 }
1675
1676 void damon_sysfs_schemes_rm_dirs(struct damon_sysfs_schemes *schemes)
1677 {
1678         struct damon_sysfs_scheme **schemes_arr = schemes->schemes_arr;
1679         int i;
1680
1681         for (i = 0; i < schemes->nr; i++) {
1682                 damon_sysfs_scheme_rm_dirs(schemes_arr[i]);
1683                 kobject_put(&schemes_arr[i]->kobj);
1684         }
1685         schemes->nr = 0;
1686         kfree(schemes_arr);
1687         schemes->schemes_arr = NULL;
1688 }
1689
1690 static int damon_sysfs_schemes_add_dirs(struct damon_sysfs_schemes *schemes,
1691                 int nr_schemes)
1692 {
1693         struct damon_sysfs_scheme **schemes_arr, *scheme;
1694         int err, i;
1695
1696         damon_sysfs_schemes_rm_dirs(schemes);
1697         if (!nr_schemes)
1698                 return 0;
1699
1700         schemes_arr = kmalloc_array(nr_schemes, sizeof(*schemes_arr),
1701                         GFP_KERNEL | __GFP_NOWARN);
1702         if (!schemes_arr)
1703                 return -ENOMEM;
1704         schemes->schemes_arr = schemes_arr;
1705
1706         for (i = 0; i < nr_schemes; i++) {
1707                 /*
1708                  * apply_interval_us as 0 means same to aggregation interval
1709                  * (same to before-apply_interval behavior)
1710                  */
1711                 scheme = damon_sysfs_scheme_alloc(DAMOS_STAT, 0);
1712                 if (!scheme) {
1713                         damon_sysfs_schemes_rm_dirs(schemes);
1714                         return -ENOMEM;
1715                 }
1716
1717                 err = kobject_init_and_add(&scheme->kobj,
1718                                 &damon_sysfs_scheme_ktype, &schemes->kobj,
1719                                 "%d", i);
1720                 if (err)
1721                         goto out;
1722                 err = damon_sysfs_scheme_add_dirs(scheme);
1723                 if (err)
1724                         goto out;
1725
1726                 schemes_arr[i] = scheme;
1727                 schemes->nr++;
1728         }
1729         return 0;
1730
1731 out:
1732         damon_sysfs_schemes_rm_dirs(schemes);
1733         kobject_put(&scheme->kobj);
1734         return err;
1735 }
1736
1737 static ssize_t nr_schemes_show(struct kobject *kobj,
1738                 struct kobj_attribute *attr, char *buf)
1739 {
1740         struct damon_sysfs_schemes *schemes = container_of(kobj,
1741                         struct damon_sysfs_schemes, kobj);
1742
1743         return sysfs_emit(buf, "%d\n", schemes->nr);
1744 }
1745
1746 static ssize_t nr_schemes_store(struct kobject *kobj,
1747                 struct kobj_attribute *attr, const char *buf, size_t count)
1748 {
1749         struct damon_sysfs_schemes *schemes;
1750         int nr, err = kstrtoint(buf, 0, &nr);
1751
1752         if (err)
1753                 return err;
1754         if (nr < 0)
1755                 return -EINVAL;
1756
1757         schemes = container_of(kobj, struct damon_sysfs_schemes, kobj);
1758
1759         if (!mutex_trylock(&damon_sysfs_lock))
1760                 return -EBUSY;
1761         err = damon_sysfs_schemes_add_dirs(schemes, nr);
1762         mutex_unlock(&damon_sysfs_lock);
1763         if (err)
1764                 return err;
1765         return count;
1766 }
1767
1768 static void damon_sysfs_schemes_release(struct kobject *kobj)
1769 {
1770         kfree(container_of(kobj, struct damon_sysfs_schemes, kobj));
1771 }
1772
1773 static struct kobj_attribute damon_sysfs_schemes_nr_attr =
1774                 __ATTR_RW_MODE(nr_schemes, 0600);
1775
1776 static struct attribute *damon_sysfs_schemes_attrs[] = {
1777         &damon_sysfs_schemes_nr_attr.attr,
1778         NULL,
1779 };
1780 ATTRIBUTE_GROUPS(damon_sysfs_schemes);
1781
1782 const struct kobj_type damon_sysfs_schemes_ktype = {
1783         .release = damon_sysfs_schemes_release,
1784         .sysfs_ops = &kobj_sysfs_ops,
1785         .default_groups = damon_sysfs_schemes_groups,
1786 };
1787
1788 static bool damon_sysfs_memcg_path_eq(struct mem_cgroup *memcg,
1789                 char *memcg_path_buf, char *path)
1790 {
1791 #ifdef CONFIG_MEMCG
1792         cgroup_path(memcg->css.cgroup, memcg_path_buf, PATH_MAX);
1793         if (sysfs_streq(memcg_path_buf, path))
1794                 return true;
1795 #endif /* CONFIG_MEMCG */
1796         return false;
1797 }
1798
1799 static int damon_sysfs_memcg_path_to_id(char *memcg_path, unsigned short *id)
1800 {
1801         struct mem_cgroup *memcg;
1802         char *path;
1803         bool found = false;
1804
1805         if (!memcg_path)
1806                 return -EINVAL;
1807
1808         path = kmalloc(sizeof(*path) * PATH_MAX, GFP_KERNEL);
1809         if (!path)
1810                 return -ENOMEM;
1811
1812         for (memcg = mem_cgroup_iter(NULL, NULL, NULL); memcg;
1813                         memcg = mem_cgroup_iter(NULL, memcg, NULL)) {
1814                 /* skip removed memcg */
1815                 if (!mem_cgroup_id(memcg))
1816                         continue;
1817                 if (damon_sysfs_memcg_path_eq(memcg, path, memcg_path)) {
1818                         *id = mem_cgroup_id(memcg);
1819                         found = true;
1820                         break;
1821                 }
1822         }
1823
1824         kfree(path);
1825         return found ? 0 : -EINVAL;
1826 }
1827
1828 static int damon_sysfs_set_scheme_filters(struct damos *scheme,
1829                 struct damon_sysfs_scheme_filters *sysfs_filters)
1830 {
1831         int i;
1832         struct damos_filter *filter, *next;
1833
1834         damos_for_each_filter_safe(filter, next, scheme)
1835                 damos_destroy_filter(filter);
1836
1837         for (i = 0; i < sysfs_filters->nr; i++) {
1838                 struct damon_sysfs_scheme_filter *sysfs_filter =
1839                         sysfs_filters->filters_arr[i];
1840                 struct damos_filter *filter =
1841                         damos_new_filter(sysfs_filter->type,
1842                                         sysfs_filter->matching);
1843                 int err;
1844
1845                 if (!filter)
1846                         return -ENOMEM;
1847                 if (filter->type == DAMOS_FILTER_TYPE_MEMCG) {
1848                         err = damon_sysfs_memcg_path_to_id(
1849                                         sysfs_filter->memcg_path,
1850                                         &filter->memcg_id);
1851                         if (err) {
1852                                 damos_destroy_filter(filter);
1853                                 return err;
1854                         }
1855                 } else if (filter->type == DAMOS_FILTER_TYPE_ADDR) {
1856                         if (sysfs_filter->addr_range.end <
1857                                         sysfs_filter->addr_range.start) {
1858                                 damos_destroy_filter(filter);
1859                                 return -EINVAL;
1860                         }
1861                         filter->addr_range = sysfs_filter->addr_range;
1862                 } else if (filter->type == DAMOS_FILTER_TYPE_TARGET) {
1863                         filter->target_idx = sysfs_filter->target_idx;
1864                 }
1865
1866                 damos_add_filter(scheme, filter);
1867         }
1868         return 0;
1869 }
1870
1871 static unsigned long damos_sysfs_get_quota_score(void *arg)
1872 {
1873         return (unsigned long)arg;
1874 }
1875
1876 static void damos_sysfs_set_quota_score(
1877                 struct damos_sysfs_quota_goals *sysfs_goals,
1878                 struct damos_quota *quota)
1879 {
1880         struct damos_sysfs_quota_goal *sysfs_goal;
1881         int i;
1882
1883         quota->get_score = NULL;
1884         quota->get_score_arg = (void *)0;
1885         for (i = 0; i < sysfs_goals->nr; i++) {
1886                 sysfs_goal = sysfs_goals->goals_arr[i];
1887                 if (!sysfs_goal->target_value)
1888                         continue;
1889
1890                 /* Higher score makes scheme less aggressive */
1891                 quota->get_score_arg = (void *)max(
1892                                 (unsigned long)quota->get_score_arg,
1893                                 sysfs_goal->current_value * 10000 /
1894                                 sysfs_goal->target_value);
1895                 quota->get_score = damos_sysfs_get_quota_score;
1896         }
1897 }
1898
1899 void damos_sysfs_set_quota_scores(struct damon_sysfs_schemes *sysfs_schemes,
1900                 struct damon_ctx *ctx)
1901 {
1902         struct damos *scheme;
1903         int i = 0;
1904
1905         damon_for_each_scheme(scheme, ctx) {
1906                 struct damon_sysfs_scheme *sysfs_scheme;
1907
1908                 sysfs_scheme = sysfs_schemes->schemes_arr[i];
1909                 damos_sysfs_set_quota_score(sysfs_scheme->quotas->goals,
1910                                 &scheme->quota);
1911                 i++;
1912         }
1913 }
1914
1915 static struct damos *damon_sysfs_mk_scheme(
1916                 struct damon_sysfs_scheme *sysfs_scheme)
1917 {
1918         struct damon_sysfs_access_pattern *access_pattern =
1919                 sysfs_scheme->access_pattern;
1920         struct damon_sysfs_quotas *sysfs_quotas = sysfs_scheme->quotas;
1921         struct damon_sysfs_weights *sysfs_weights = sysfs_quotas->weights;
1922         struct damon_sysfs_watermarks *sysfs_wmarks = sysfs_scheme->watermarks;
1923         struct damon_sysfs_scheme_filters *sysfs_filters =
1924                 sysfs_scheme->filters;
1925         struct damos *scheme;
1926         int err;
1927
1928         struct damos_access_pattern pattern = {
1929                 .min_sz_region = access_pattern->sz->min,
1930                 .max_sz_region = access_pattern->sz->max,
1931                 .min_nr_accesses = access_pattern->nr_accesses->min,
1932                 .max_nr_accesses = access_pattern->nr_accesses->max,
1933                 .min_age_region = access_pattern->age->min,
1934                 .max_age_region = access_pattern->age->max,
1935         };
1936         struct damos_quota quota = {
1937                 .ms = sysfs_quotas->ms,
1938                 .sz = sysfs_quotas->sz,
1939                 .reset_interval = sysfs_quotas->reset_interval_ms,
1940                 .weight_sz = sysfs_weights->sz,
1941                 .weight_nr_accesses = sysfs_weights->nr_accesses,
1942                 .weight_age = sysfs_weights->age,
1943         };
1944         struct damos_watermarks wmarks = {
1945                 .metric = sysfs_wmarks->metric,
1946                 .interval = sysfs_wmarks->interval_us,
1947                 .high = sysfs_wmarks->high,
1948                 .mid = sysfs_wmarks->mid,
1949                 .low = sysfs_wmarks->low,
1950         };
1951
1952         damos_sysfs_set_quota_score(sysfs_quotas->goals, &quota);
1953
1954         scheme = damon_new_scheme(&pattern, sysfs_scheme->action,
1955                         sysfs_scheme->apply_interval_us, &quota, &wmarks);
1956         if (!scheme)
1957                 return NULL;
1958
1959         err = damon_sysfs_set_scheme_filters(scheme, sysfs_filters);
1960         if (err) {
1961                 damon_destroy_scheme(scheme);
1962                 return NULL;
1963         }
1964         return scheme;
1965 }
1966
1967 static void damon_sysfs_update_scheme(struct damos *scheme,
1968                 struct damon_sysfs_scheme *sysfs_scheme)
1969 {
1970         struct damon_sysfs_access_pattern *access_pattern =
1971                 sysfs_scheme->access_pattern;
1972         struct damon_sysfs_quotas *sysfs_quotas = sysfs_scheme->quotas;
1973         struct damon_sysfs_weights *sysfs_weights = sysfs_quotas->weights;
1974         struct damon_sysfs_watermarks *sysfs_wmarks = sysfs_scheme->watermarks;
1975         int err;
1976
1977         scheme->pattern.min_sz_region = access_pattern->sz->min;
1978         scheme->pattern.max_sz_region = access_pattern->sz->max;
1979         scheme->pattern.min_nr_accesses = access_pattern->nr_accesses->min;
1980         scheme->pattern.max_nr_accesses = access_pattern->nr_accesses->max;
1981         scheme->pattern.min_age_region = access_pattern->age->min;
1982         scheme->pattern.max_age_region = access_pattern->age->max;
1983
1984         scheme->action = sysfs_scheme->action;
1985         scheme->apply_interval_us = sysfs_scheme->apply_interval_us;
1986
1987         scheme->quota.ms = sysfs_quotas->ms;
1988         scheme->quota.sz = sysfs_quotas->sz;
1989         scheme->quota.reset_interval = sysfs_quotas->reset_interval_ms;
1990         scheme->quota.weight_sz = sysfs_weights->sz;
1991         scheme->quota.weight_nr_accesses = sysfs_weights->nr_accesses;
1992         scheme->quota.weight_age = sysfs_weights->age;
1993
1994         damos_sysfs_set_quota_score(sysfs_quotas->goals, &scheme->quota);
1995
1996         scheme->wmarks.metric = sysfs_wmarks->metric;
1997         scheme->wmarks.interval = sysfs_wmarks->interval_us;
1998         scheme->wmarks.high = sysfs_wmarks->high;
1999         scheme->wmarks.mid = sysfs_wmarks->mid;
2000         scheme->wmarks.low = sysfs_wmarks->low;
2001
2002         err = damon_sysfs_set_scheme_filters(scheme, sysfs_scheme->filters);
2003         if (err)
2004                 damon_destroy_scheme(scheme);
2005 }
2006
2007 int damon_sysfs_set_schemes(struct damon_ctx *ctx,
2008                 struct damon_sysfs_schemes *sysfs_schemes)
2009 {
2010         struct damos *scheme, *next;
2011         int i = 0;
2012
2013         damon_for_each_scheme_safe(scheme, next, ctx) {
2014                 if (i < sysfs_schemes->nr)
2015                         damon_sysfs_update_scheme(scheme,
2016                                         sysfs_schemes->schemes_arr[i]);
2017                 else
2018                         damon_destroy_scheme(scheme);
2019                 i++;
2020         }
2021
2022         for (; i < sysfs_schemes->nr; i++) {
2023                 struct damos *scheme, *next;
2024
2025                 scheme = damon_sysfs_mk_scheme(sysfs_schemes->schemes_arr[i]);
2026                 if (!scheme) {
2027                         damon_for_each_scheme_safe(scheme, next, ctx)
2028                                 damon_destroy_scheme(scheme);
2029                         return -ENOMEM;
2030                 }
2031                 damon_add_scheme(ctx, scheme);
2032         }
2033         return 0;
2034 }
2035
2036 void damon_sysfs_schemes_update_stats(
2037                 struct damon_sysfs_schemes *sysfs_schemes,
2038                 struct damon_ctx *ctx)
2039 {
2040         struct damos *scheme;
2041         int schemes_idx = 0;
2042
2043         damon_for_each_scheme(scheme, ctx) {
2044                 struct damon_sysfs_stats *sysfs_stats;
2045
2046                 /* user could have removed the scheme sysfs dir */
2047                 if (schemes_idx >= sysfs_schemes->nr)
2048                         break;
2049
2050                 sysfs_stats = sysfs_schemes->schemes_arr[schemes_idx++]->stats;
2051                 sysfs_stats->nr_tried = scheme->stat.nr_tried;
2052                 sysfs_stats->sz_tried = scheme->stat.sz_tried;
2053                 sysfs_stats->nr_applied = scheme->stat.nr_applied;
2054                 sysfs_stats->sz_applied = scheme->stat.sz_applied;
2055                 sysfs_stats->qt_exceeds = scheme->stat.qt_exceeds;
2056         }
2057 }
2058
2059 /*
2060  * damon_sysfs_schemes that need to update its schemes regions dir.  Protected
2061  * by damon_sysfs_lock
2062  */
2063 static struct damon_sysfs_schemes *damon_sysfs_schemes_for_damos_callback;
2064 static int damon_sysfs_schemes_region_idx;
2065 static bool damos_regions_upd_total_bytes_only;
2066
2067 /*
2068  * DAMON callback that called before damos apply.  While this callback is
2069  * registered, damon_sysfs_lock should be held to ensure the regions
2070  * directories exist.
2071  */
2072 static int damon_sysfs_before_damos_apply(struct damon_ctx *ctx,
2073                 struct damon_target *t, struct damon_region *r,
2074                 struct damos *s)
2075 {
2076         struct damos *scheme;
2077         struct damon_sysfs_scheme_regions *sysfs_regions;
2078         struct damon_sysfs_scheme_region *region;
2079         struct damon_sysfs_schemes *sysfs_schemes =
2080                 damon_sysfs_schemes_for_damos_callback;
2081         int schemes_idx = 0;
2082
2083         damon_for_each_scheme(scheme, ctx) {
2084                 if (scheme == s)
2085                         break;
2086                 schemes_idx++;
2087         }
2088
2089         /* user could have removed the scheme sysfs dir */
2090         if (schemes_idx >= sysfs_schemes->nr)
2091                 return 0;
2092
2093         sysfs_regions = sysfs_schemes->schemes_arr[schemes_idx]->tried_regions;
2094         if (sysfs_regions->upd_status == DAMOS_TRIED_REGIONS_UPD_FINISHED)
2095                 return 0;
2096         if (sysfs_regions->upd_status == DAMOS_TRIED_REGIONS_UPD_IDLE)
2097                 sysfs_regions->upd_status = DAMOS_TRIED_REGIONS_UPD_STARTED;
2098         sysfs_regions->total_bytes += r->ar.end - r->ar.start;
2099         if (damos_regions_upd_total_bytes_only)
2100                 return 0;
2101
2102         region = damon_sysfs_scheme_region_alloc(r);
2103         if (!region)
2104                 return 0;
2105         list_add_tail(&region->list, &sysfs_regions->regions_list);
2106         sysfs_regions->nr_regions++;
2107         if (kobject_init_and_add(&region->kobj,
2108                                 &damon_sysfs_scheme_region_ktype,
2109                                 &sysfs_regions->kobj, "%d",
2110                                 damon_sysfs_schemes_region_idx++)) {
2111                 kobject_put(&region->kobj);
2112         }
2113         return 0;
2114 }
2115
2116 /*
2117  * DAMON callback that called after each accesses sampling.  While this
2118  * callback is registered, damon_sysfs_lock should be held to ensure the
2119  * regions directories exist.
2120  */
2121 static int damon_sysfs_after_sampling(struct damon_ctx *ctx)
2122 {
2123         struct damon_sysfs_schemes *sysfs_schemes =
2124                 damon_sysfs_schemes_for_damos_callback;
2125         struct damon_sysfs_scheme_regions *sysfs_regions;
2126         int i;
2127
2128         for (i = 0; i < sysfs_schemes->nr; i++) {
2129                 sysfs_regions = sysfs_schemes->schemes_arr[i]->tried_regions;
2130                 if (sysfs_regions->upd_status ==
2131                                 DAMOS_TRIED_REGIONS_UPD_STARTED ||
2132                                 time_after(jiffies,
2133                                         sysfs_regions->upd_timeout_jiffies))
2134                         sysfs_regions->upd_status =
2135                                 DAMOS_TRIED_REGIONS_UPD_FINISHED;
2136         }
2137
2138         return 0;
2139 }
2140
2141 /* Called from damon_sysfs_cmd_request_callback under damon_sysfs_lock */
2142 int damon_sysfs_schemes_clear_regions(
2143                 struct damon_sysfs_schemes *sysfs_schemes,
2144                 struct damon_ctx *ctx)
2145 {
2146         struct damos *scheme;
2147         int schemes_idx = 0;
2148
2149         damon_for_each_scheme(scheme, ctx) {
2150                 struct damon_sysfs_scheme *sysfs_scheme;
2151
2152                 /* user could have removed the scheme sysfs dir */
2153                 if (schemes_idx >= sysfs_schemes->nr)
2154                         break;
2155
2156                 sysfs_scheme = sysfs_schemes->schemes_arr[schemes_idx++];
2157                 damon_sysfs_scheme_regions_rm_dirs(
2158                                 sysfs_scheme->tried_regions);
2159                 sysfs_scheme->tried_regions->total_bytes = 0;
2160         }
2161         return 0;
2162 }
2163
2164 static struct damos *damos_sysfs_nth_scheme(int n, struct damon_ctx *ctx)
2165 {
2166         struct damos *scheme;
2167         int i = 0;
2168
2169         damon_for_each_scheme(scheme, ctx) {
2170                 if (i == n)
2171                         return scheme;
2172                 i++;
2173         }
2174         return NULL;
2175 }
2176
2177 static void damos_tried_regions_init_upd_status(
2178                 struct damon_sysfs_schemes *sysfs_schemes,
2179                 struct damon_ctx *ctx)
2180 {
2181         int i;
2182         struct damos *scheme;
2183         struct damon_sysfs_scheme_regions *sysfs_regions;
2184
2185         for (i = 0; i < sysfs_schemes->nr; i++) {
2186                 sysfs_regions = sysfs_schemes->schemes_arr[i]->tried_regions;
2187                 scheme = damos_sysfs_nth_scheme(i, ctx);
2188                 if (!scheme) {
2189                         sysfs_regions->upd_status =
2190                                 DAMOS_TRIED_REGIONS_UPD_FINISHED;
2191                         continue;
2192                 }
2193                 sysfs_regions->upd_status = DAMOS_TRIED_REGIONS_UPD_IDLE;
2194                 sysfs_regions->upd_timeout_jiffies = jiffies +
2195                         2 * usecs_to_jiffies(scheme->apply_interval_us ?
2196                                         scheme->apply_interval_us :
2197                                         ctx->attrs.sample_interval);
2198         }
2199 }
2200
2201 /* Called from damon_sysfs_cmd_request_callback under damon_sysfs_lock */
2202 int damon_sysfs_schemes_update_regions_start(
2203                 struct damon_sysfs_schemes *sysfs_schemes,
2204                 struct damon_ctx *ctx, bool total_bytes_only)
2205 {
2206         damon_sysfs_schemes_clear_regions(sysfs_schemes, ctx);
2207         damon_sysfs_schemes_for_damos_callback = sysfs_schemes;
2208         damos_tried_regions_init_upd_status(sysfs_schemes, ctx);
2209         damos_regions_upd_total_bytes_only = total_bytes_only;
2210         ctx->callback.before_damos_apply = damon_sysfs_before_damos_apply;
2211         ctx->callback.after_sampling = damon_sysfs_after_sampling;
2212         return 0;
2213 }
2214
2215 bool damos_sysfs_regions_upd_done(void)
2216 {
2217         struct damon_sysfs_schemes *sysfs_schemes =
2218                 damon_sysfs_schemes_for_damos_callback;
2219         struct damon_sysfs_scheme_regions *sysfs_regions;
2220         int i;
2221
2222         for (i = 0; i < sysfs_schemes->nr; i++) {
2223                 sysfs_regions = sysfs_schemes->schemes_arr[i]->tried_regions;
2224                 if (sysfs_regions->upd_status !=
2225                                 DAMOS_TRIED_REGIONS_UPD_FINISHED)
2226                         return false;
2227         }
2228         return true;
2229 }
2230
2231 /*
2232  * Called from damon_sysfs_cmd_request_callback under damon_sysfs_lock.  Caller
2233  * should unlock damon_sysfs_lock which held before
2234  * damon_sysfs_schemes_update_regions_start()
2235  */
2236 int damon_sysfs_schemes_update_regions_stop(struct damon_ctx *ctx)
2237 {
2238         damon_sysfs_schemes_for_damos_callback = NULL;
2239         ctx->callback.before_damos_apply = NULL;
2240         ctx->callback.after_sampling = NULL;
2241         damon_sysfs_schemes_region_idx = 0;
2242         return 0;
2243 }