tools/testing/selftests/bpf/bench.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2020 Facebook */
   3 #define _GNU_SOURCE
   4 #include <argp.h>
   5 #include <linux/compiler.h>
   6 #include <sys/time.h>
   7 #include <sched.h>
   8 #include <fcntl.h>
   9 #include <pthread.h>
  10 #include <sys/sysinfo.h>
  11 #include <signal.h>
  12 #include "bench.h"
  13 #include "testing_helpers.h"
  14
  15 struct env env = {
  16         .warmup_sec = 1,
  17         .duration_sec = 5,
  18         .affinity = false,
  19         .quiet = false,
  20         .consumer_cnt = 0,
  21         .producer_cnt = 1,
  22 };
  23
  24 static int libbpf_print_fn(enum libbpf_print_level level,
  25                     const char *format, va_list args)
  26 {
  27         if (level == LIBBPF_DEBUG && !env.verbose)
  28                 return 0;
  29         return vfprintf(stderr, format, args);
  30 }
  31
  32 void setup_libbpf(void)
  33 {
  34         libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
  35         libbpf_set_print(libbpf_print_fn);
  36 }
  37
  38 void false_hits_report_progress(int iter, struct bench_res *res, long delta_ns)
  39 {
  40         long total = res->false_hits  + res->hits + res->drops;
  41
  42         printf("Iter %3d (%7.3lfus): ",
  43                iter, (delta_ns - 1000000000) / 1000.0);
  44
  45         printf("%ld false hits of %ld total operations. Percentage = %2.2f %%\n",
  46                res->false_hits, total, ((float)res->false_hits / total) * 100);
  47 }
  48
  49 void false_hits_report_final(struct bench_res res[], int res_cnt)
  50 {
  51         long total_hits = 0, total_drops = 0, total_false_hits = 0, total_ops = 0;
  52         int i;
  53
  54         for (i = 0; i < res_cnt; i++) {
  55                 total_hits += res[i].hits;
  56                 total_false_hits += res[i].false_hits;
  57                 total_drops += res[i].drops;
  58         }
  59         total_ops = total_hits + total_false_hits + total_drops;
  60
  61         printf("Summary: %ld false hits of %ld total operations. ",
  62                total_false_hits, total_ops);
  63         printf("Percentage =  %2.2f %%\n",
  64                ((float)total_false_hits / total_ops) * 100);
  65 }
  66
  67 void hits_drops_report_progress(int iter, struct bench_res *res, long delta_ns)
  68 {
  69         double hits_per_sec, drops_per_sec;
  70         double hits_per_prod;
  71
  72         hits_per_sec = res->hits / 1000000.0 / (delta_ns / 1000000000.0);
  73         hits_per_prod = hits_per_sec / env.producer_cnt;
  74         drops_per_sec = res->drops / 1000000.0 / (delta_ns / 1000000000.0);
  75
  76         printf("Iter %3d (%7.3lfus): ",
  77                iter, (delta_ns - 1000000000) / 1000.0);
  78
  79         printf("hits %8.3lfM/s (%7.3lfM/prod), drops %8.3lfM/s, total operations %8.3lfM/s\n",
  80                hits_per_sec, hits_per_prod, drops_per_sec, hits_per_sec + drops_per_sec);
  81 }
  82
  83 void
  84 grace_period_latency_basic_stats(struct bench_res res[], int res_cnt, struct basic_stats *gp_stat)
  85 {
  86         int i;
  87
  88         memset(gp_stat, 0, sizeof(struct basic_stats));
  89
  90         for (i = 0; i < res_cnt; i++)
  91                 gp_stat->mean += res[i].gp_ns / 1000.0 / (double)res[i].gp_ct / (0.0 + res_cnt);
  92
  93 #define IT_MEAN_DIFF (res[i].gp_ns / 1000.0 / (double)res[i].gp_ct - gp_stat->mean)
  94         if (res_cnt > 1) {
  95                 for (i = 0; i < res_cnt; i++)
  96                         gp_stat->stddev += (IT_MEAN_DIFF * IT_MEAN_DIFF) / (res_cnt - 1.0);
  97         }
  98         gp_stat->stddev = sqrt(gp_stat->stddev);
  99 #undef IT_MEAN_DIFF
 100 }
 101
 102 void
 103 grace_period_ticks_basic_stats(struct bench_res res[], int res_cnt, struct basic_stats *gp_stat)
 104 {
 105         int i;
 106
 107         memset(gp_stat, 0, sizeof(struct basic_stats));
 108         for (i = 0; i < res_cnt; i++)
 109                 gp_stat->mean += res[i].stime / (double)res[i].gp_ct / (0.0 + res_cnt);
 110
 111 #define IT_MEAN_DIFF (res[i].stime / (double)res[i].gp_ct - gp_stat->mean)
 112         if (res_cnt > 1) {
 113                 for (i = 0; i < res_cnt; i++)
 114                         gp_stat->stddev += (IT_MEAN_DIFF * IT_MEAN_DIFF) / (res_cnt - 1.0);
 115         }
 116         gp_stat->stddev = sqrt(gp_stat->stddev);
 117 #undef IT_MEAN_DIFF
 118 }
 119
 120 void hits_drops_report_final(struct bench_res res[], int res_cnt)
 121 {
 122         int i;
 123         double hits_mean = 0.0, drops_mean = 0.0, total_ops_mean = 0.0;
 124         double hits_stddev = 0.0, drops_stddev = 0.0, total_ops_stddev = 0.0;
 125         double total_ops;
 126
 127         for (i = 0; i < res_cnt; i++) {
 128                 hits_mean += res[i].hits / 1000000.0 / (0.0 + res_cnt);
 129                 drops_mean += res[i].drops / 1000000.0 / (0.0 + res_cnt);
 130         }
 131         total_ops_mean = hits_mean + drops_mean;
 132
 133         if (res_cnt > 1)  {
 134                 for (i = 0; i < res_cnt; i++) {
 135                         hits_stddev += (hits_mean - res[i].hits / 1000000.0) *
 136                                        (hits_mean - res[i].hits / 1000000.0) /
 137                                        (res_cnt - 1.0);
 138                         drops_stddev += (drops_mean - res[i].drops / 1000000.0) *
 139                                         (drops_mean - res[i].drops / 1000000.0) /
 140                                         (res_cnt - 1.0);
 141                         total_ops = res[i].hits + res[i].drops;
 142                         total_ops_stddev += (total_ops_mean - total_ops / 1000000.0) *
 143                                         (total_ops_mean - total_ops / 1000000.0) /
 144                                         (res_cnt - 1.0);
 145                 }
 146                 hits_stddev = sqrt(hits_stddev);
 147                 drops_stddev = sqrt(drops_stddev);
 148                 total_ops_stddev = sqrt(total_ops_stddev);
 149         }
 150         printf("Summary: hits %8.3lf \u00B1 %5.3lfM/s (%7.3lfM/prod), ",
 151                hits_mean, hits_stddev, hits_mean / env.producer_cnt);
 152         printf("drops %8.3lf \u00B1 %5.3lfM/s, ",
 153                drops_mean, drops_stddev);
 154         printf("total operations %8.3lf \u00B1 %5.3lfM/s\n",
 155                total_ops_mean, total_ops_stddev);
 156 }
 157
 158 void ops_report_progress(int iter, struct bench_res *res, long delta_ns)
 159 {
 160         double hits_per_sec, hits_per_prod;
 161
 162         hits_per_sec = res->hits / 1000000.0 / (delta_ns / 1000000000.0);
 163         hits_per_prod = hits_per_sec / env.producer_cnt;
 164
 165         printf("Iter %3d (%7.3lfus): ", iter, (delta_ns - 1000000000) / 1000.0);
 166
 167         printf("hits %8.3lfM/s (%7.3lfM/prod)\n", hits_per_sec, hits_per_prod);
 168 }
 169
 170 void ops_report_final(struct bench_res res[], int res_cnt)
 171 {
 172         double hits_mean = 0.0, hits_stddev = 0.0;
 173         int i;
 174
 175         for (i = 0; i < res_cnt; i++)
 176                 hits_mean += res[i].hits / 1000000.0 / (0.0 + res_cnt);
 177
 178         if (res_cnt > 1)  {
 179                 for (i = 0; i < res_cnt; i++)
 180                         hits_stddev += (hits_mean - res[i].hits / 1000000.0) *
 181                                        (hits_mean - res[i].hits / 1000000.0) /
 182                                        (res_cnt - 1.0);
 183
 184                 hits_stddev = sqrt(hits_stddev);
 185         }
 186         printf("Summary: throughput %8.3lf \u00B1 %5.3lf M ops/s (%7.3lfM ops/prod), ",
 187                hits_mean, hits_stddev, hits_mean / env.producer_cnt);
 188         printf("latency %8.3lf ns/op\n", 1000.0 / hits_mean * env.producer_cnt);
 189 }
 190
 191 void local_storage_report_progress(int iter, struct bench_res *res,
 192                                    long delta_ns)
 193 {
 194         double important_hits_per_sec, hits_per_sec;
 195         double delta_sec = delta_ns / 1000000000.0;
 196
 197         hits_per_sec = res->hits / 1000000.0 / delta_sec;
 198         important_hits_per_sec = res->important_hits / 1000000.0 / delta_sec;
 199
 200         printf("Iter %3d (%7.3lfus): ", iter, (delta_ns - 1000000000) / 1000.0);
 201
 202         printf("hits %8.3lfM/s ", hits_per_sec);
 203         printf("important_hits %8.3lfM/s\n", important_hits_per_sec);
 204 }
 205
 206 void local_storage_report_final(struct bench_res res[], int res_cnt)
 207 {
 208         double important_hits_mean = 0.0, important_hits_stddev = 0.0;
 209         double hits_mean = 0.0, hits_stddev = 0.0;
 210         int i;
 211
 212         for (i = 0; i < res_cnt; i++) {
 213                 hits_mean += res[i].hits / 1000000.0 / (0.0 + res_cnt);
 214                 important_hits_mean += res[i].important_hits / 1000000.0 / (0.0 + res_cnt);
 215         }
 216
 217         if (res_cnt > 1)  {
 218                 for (i = 0; i < res_cnt; i++) {
 219                         hits_stddev += (hits_mean - res[i].hits / 1000000.0) *
 220                                        (hits_mean - res[i].hits / 1000000.0) /
 221                                        (res_cnt - 1.0);
 222                         important_hits_stddev +=
 223                                        (important_hits_mean - res[i].important_hits / 1000000.0) *
 224                                        (important_hits_mean - res[i].important_hits / 1000000.0) /
 225                                        (res_cnt - 1.0);
 226                 }
 227
 228                 hits_stddev = sqrt(hits_stddev);
 229                 important_hits_stddev = sqrt(important_hits_stddev);
 230         }
 231         printf("Summary: hits throughput %8.3lf \u00B1 %5.3lf M ops/s, ",
 232                hits_mean, hits_stddev);
 233         printf("hits latency %8.3lf ns/op, ", 1000.0 / hits_mean);
 234         printf("important_hits throughput %8.3lf \u00B1 %5.3lf M ops/s\n",
 235                important_hits_mean, important_hits_stddev);
 236 }
 237
 238 const char *argp_program_version = "benchmark";
 239 const char *argp_program_bug_address = "<bpf@vger.kernel.org>";
 240 const char argp_program_doc[] =
 241 "benchmark    Generic benchmarking framework.\n"
 242 "\n"
 243 "This tool runs benchmarks.\n"
 244 "\n"
 245 "USAGE: benchmark <bench-name>\n"
 246 "\n"
 247 "EXAMPLES:\n"
 248 "    # run 'count-local' benchmark with 1 producer and 1 consumer\n"
 249 "    benchmark count-local\n"
 250 "    # run 'count-local' with 16 producer and 8 consumer thread, pinned to CPUs\n"
 251 "    benchmark -p16 -c8 -a count-local\n";
 252
 253 enum {
 254         ARG_PROD_AFFINITY_SET = 1000,
 255         ARG_CONS_AFFINITY_SET = 1001,
 256 };
 257
 258 static const struct argp_option opts[] = {
 259         { "list", 'l', NULL, 0, "List available benchmarks"},
 260         { "duration", 'd', "SEC", 0, "Duration of benchmark, seconds"},
 261         { "warmup", 'w', "SEC", 0, "Warm-up period, seconds"},
 262         { "producers", 'p', "NUM", 0, "Number of producer threads"},
 263         { "consumers", 'c', "NUM", 0, "Number of consumer threads"},
 264         { "verbose", 'v', NULL, 0, "Verbose debug output"},
 265         { "affinity", 'a', NULL, 0, "Set consumer/producer thread affinity"},
 266         { "quiet", 'q', NULL, 0, "Be more quiet"},
 267         { "prod-affinity", ARG_PROD_AFFINITY_SET, "CPUSET", 0,
 268           "Set of CPUs for producer threads; implies --affinity"},
 269         { "cons-affinity", ARG_CONS_AFFINITY_SET, "CPUSET", 0,
 270           "Set of CPUs for consumer threads; implies --affinity"},
 271         {},
 272 };
 273
 274 extern struct argp bench_ringbufs_argp;
 275 extern struct argp bench_bloom_map_argp;
 276 extern struct argp bench_bpf_loop_argp;
 277 extern struct argp bench_local_storage_argp;
 278 extern struct argp bench_local_storage_rcu_tasks_trace_argp;
 279 extern struct argp bench_strncmp_argp;
 280 extern struct argp bench_hashmap_lookup_argp;
 281 extern struct argp bench_local_storage_create_argp;
 282 extern struct argp bench_htab_mem_argp;
 283 extern struct argp bench_trigger_batch_argp;
 284 extern struct argp bench_crypto_argp;
 285
 286 static const struct argp_child bench_parsers[] = {
 287         { &bench_ringbufs_argp, 0, "Ring buffers benchmark", 0 },
 288         { &bench_bloom_map_argp, 0, "Bloom filter map benchmark", 0 },
 289         { &bench_bpf_loop_argp, 0, "bpf_loop helper benchmark", 0 },
 290         { &bench_local_storage_argp, 0, "local_storage benchmark", 0 },
 291         { &bench_strncmp_argp, 0, "bpf_strncmp helper benchmark", 0 },
 292         { &bench_local_storage_rcu_tasks_trace_argp, 0,
 293                 "local_storage RCU Tasks Trace slowdown benchmark", 0 },
 294         { &bench_hashmap_lookup_argp, 0, "Hashmap lookup benchmark", 0 },
 295         { &bench_local_storage_create_argp, 0, "local-storage-create benchmark", 0 },
 296         { &bench_htab_mem_argp, 0, "hash map memory benchmark", 0 },
 297         { &bench_trigger_batch_argp, 0, "BPF triggering benchmark", 0 },
 298         { &bench_crypto_argp, 0, "bpf crypto benchmark", 0 },
 299         {},
 300 };
 301
 302 /* Make pos_args global, so that we can run argp_parse twice, if necessary */
 303 static int pos_args;
 304
 305 static error_t parse_arg(int key, char *arg, struct argp_state *state)
 306 {
 307         switch (key) {
 308         case 'v':
 309                 env.verbose = true;
 310                 break;
 311         case 'l':
 312                 env.list = true;
 313                 break;
 314         case 'd':
 315                 env.duration_sec = strtol(arg, NULL, 10);
 316                 if (env.duration_sec <= 0) {
 317                         fprintf(stderr, "Invalid duration: %s\n", arg);
 318                         argp_usage(state);
 319                 }
 320                 break;
 321         case 'w':
 322                 env.warmup_sec = strtol(arg, NULL, 10);
 323                 if (env.warmup_sec <= 0) {
 324                         fprintf(stderr, "Invalid warm-up duration: %s\n", arg);
 325                         argp_usage(state);
 326                 }
 327                 break;
 328         case 'p':
 329                 env.producer_cnt = strtol(arg, NULL, 10);
 330                 if (env.producer_cnt < 0) {
 331                         fprintf(stderr, "Invalid producer count: %s\n", arg);
 332                         argp_usage(state);
 333                 }
 334                 break;
 335         case 'c':
 336                 env.consumer_cnt = strtol(arg, NULL, 10);
 337                 if (env.consumer_cnt < 0) {
 338                         fprintf(stderr, "Invalid consumer count: %s\n", arg);
 339                         argp_usage(state);
 340                 }
 341                 break;
 342         case 'a':
 343                 env.affinity = true;
 344                 break;
 345         case 'q':
 346                 env.quiet = true;
 347                 break;
 348         case ARG_PROD_AFFINITY_SET:
 349                 env.affinity = true;
 350                 if (parse_num_list(arg, &env.prod_cpus.cpus,
 351                                    &env.prod_cpus.cpus_len)) {
 352                         fprintf(stderr, "Invalid format of CPU set for producers.");
 353                         argp_usage(state);
 354                 }
 355                 break;
 356         case ARG_CONS_AFFINITY_SET:
 357                 env.affinity = true;
 358                 if (parse_num_list(arg, &env.cons_cpus.cpus,
 359                                    &env.cons_cpus.cpus_len)) {
 360                         fprintf(stderr, "Invalid format of CPU set for consumers.");
 361                         argp_usage(state);
 362                 }
 363                 break;
 364         case ARGP_KEY_ARG:
 365                 if (pos_args++) {
 366                         fprintf(stderr,
 367                                 "Unrecognized positional argument: %s\n", arg);
 368                         argp_usage(state);
 369                 }
 370                 env.bench_name = strdup(arg);
 371                 break;
 372         default:
 373                 return ARGP_ERR_UNKNOWN;
 374         }
 375         return 0;
 376 }
 377
 378 static void parse_cmdline_args_init(int argc, char **argv)
 379 {
 380         static const struct argp argp = {
 381                 .options = opts,
 382                 .parser = parse_arg,
 383                 .doc = argp_program_doc,
 384                 .children = bench_parsers,
 385         };
 386         if (argp_parse(&argp, argc, argv, 0, NULL, NULL))
 387                 exit(1);
 388 }
 389
 390 static void parse_cmdline_args_final(int argc, char **argv)
 391 {
 392         struct argp_child bench_parsers[2] = {};
 393         const struct argp argp = {
 394                 .options = opts,
 395                 .parser = parse_arg,
 396                 .doc = argp_program_doc,
 397                 .children = bench_parsers,
 398         };
 399
 400         /* Parse arguments the second time with the correct set of parsers */
 401         if (bench->argp) {
 402                 bench_parsers[0].argp = bench->argp;
 403                 bench_parsers[0].header = bench->name;
 404                 pos_args = 0;
 405                 if (argp_parse(&argp, argc, argv, 0, NULL, NULL))
 406                         exit(1);
 407         }
 408 }
 409
 410 static void collect_measurements(long delta_ns);
 411
 412 static __u64 last_time_ns;
 413 static void sigalarm_handler(int signo)
 414 {
 415         long new_time_ns = get_time_ns();
 416         long delta_ns = new_time_ns - last_time_ns;
 417
 418         collect_measurements(delta_ns);
 419
 420         last_time_ns = new_time_ns;
 421 }
 422
 423 /* set up periodic 1-second timer */
 424 static void setup_timer()
 425 {
 426         static struct sigaction sigalarm_action = {
 427                 .sa_handler = sigalarm_handler,
 428         };
 429         struct itimerval timer_settings = {};
 430         int err;
 431
 432         last_time_ns = get_time_ns();
 433         err = sigaction(SIGALRM, &sigalarm_action, NULL);
 434         if (err < 0) {
 435                 fprintf(stderr, "failed to install SIGALRM handler: %d\n", -errno);
 436                 exit(1);
 437         }
 438         timer_settings.it_interval.tv_sec = 1;
 439         timer_settings.it_value.tv_sec = 1;
 440         err = setitimer(ITIMER_REAL, &timer_settings, NULL);
 441         if (err < 0) {
 442                 fprintf(stderr, "failed to arm interval timer: %d\n", -errno);
 443                 exit(1);
 444         }
 445 }
 446
 447 static void set_thread_affinity(pthread_t thread, int cpu)
 448 {
 449         cpu_set_t cpuset;
 450         int err;
 451
 452         CPU_ZERO(&cpuset);
 453         CPU_SET(cpu, &cpuset);
 454         err = pthread_setaffinity_np(thread, sizeof(cpuset), &cpuset);
 455         if (err) {
 456                 fprintf(stderr, "setting affinity to CPU #%d failed: %d\n",
 457                         cpu, -err);
 458                 exit(1);
 459         }
 460 }
 461
 462 static int next_cpu(struct cpu_set *cpu_set)
 463 {
 464         if (cpu_set->cpus) {
 465                 int i;
 466
 467                 /* find next available CPU */
 468                 for (i = cpu_set->next_cpu; i < cpu_set->cpus_len; i++) {
 469                         if (cpu_set->cpus[i]) {
 470                                 cpu_set->next_cpu = i + 1;
 471                                 return i;
 472                         }
 473                 }
 474                 fprintf(stderr, "Not enough CPUs specified, need CPU #%d or higher.\n", i);
 475                 exit(1);
 476         }
 477
 478         return cpu_set->next_cpu++ % env.nr_cpus;
 479 }
 480
 481 static struct bench_state {
 482         int res_cnt;
 483         struct bench_res *results;
 484         pthread_t *consumers;
 485         pthread_t *producers;
 486 } state;
 487
 488 const struct bench *bench = NULL;
 489
 490 extern const struct bench bench_count_global;
 491 extern const struct bench bench_count_local;
 492 extern const struct bench bench_rename_base;
 493 extern const struct bench bench_rename_kprobe;
 494 extern const struct bench bench_rename_kretprobe;
 495 extern const struct bench bench_rename_rawtp;
 496 extern const struct bench bench_rename_fentry;
 497 extern const struct bench bench_rename_fexit;
 498
 499 /* pure counting benchmarks to establish theoretical lmits */
 500 extern const struct bench bench_trig_usermode_count;
 501 extern const struct bench bench_trig_syscall_count;
 502 extern const struct bench bench_trig_kernel_count;
 503
 504 /* batched, staying mostly in-kernel benchmarks */
 505 extern const struct bench bench_trig_kprobe;
 506 extern const struct bench bench_trig_kretprobe;
 507 extern const struct bench bench_trig_kprobe_multi;
 508 extern const struct bench bench_trig_kretprobe_multi;
 509 extern const struct bench bench_trig_fentry;
 510 extern const struct bench bench_trig_fexit;
 511 extern const struct bench bench_trig_fmodret;
 512 extern const struct bench bench_trig_tp;
 513 extern const struct bench bench_trig_rawtp;
 514
 515 /* uprobe/uretprobe benchmarks */
 516 extern const struct bench bench_trig_uprobe_nop;
 517 extern const struct bench bench_trig_uretprobe_nop;
 518 extern const struct bench bench_trig_uprobe_push;
 519 extern const struct bench bench_trig_uretprobe_push;
 520 extern const struct bench bench_trig_uprobe_ret;
 521 extern const struct bench bench_trig_uretprobe_ret;
 522
 523 extern const struct bench bench_rb_libbpf;
 524 extern const struct bench bench_rb_custom;
 525 extern const struct bench bench_pb_libbpf;
 526 extern const struct bench bench_pb_custom;
 527 extern const struct bench bench_bloom_lookup;
 528 extern const struct bench bench_bloom_update;
 529 extern const struct bench bench_bloom_false_positive;
 530 extern const struct bench bench_hashmap_without_bloom;
 531 extern const struct bench bench_hashmap_with_bloom;
 532 extern const struct bench bench_bpf_loop;
 533 extern const struct bench bench_strncmp_no_helper;
 534 extern const struct bench bench_strncmp_helper;
 535 extern const struct bench bench_bpf_hashmap_full_update;
 536 extern const struct bench bench_local_storage_cache_seq_get;
 537 extern const struct bench bench_local_storage_cache_interleaved_get;
 538 extern const struct bench bench_local_storage_cache_hashmap_control;
 539 extern const struct bench bench_local_storage_tasks_trace;
 540 extern const struct bench bench_bpf_hashmap_lookup;
 541 extern const struct bench bench_local_storage_create;
 542 extern const struct bench bench_htab_mem;
 543 extern const struct bench bench_crypto_encrypt;
 544 extern const struct bench bench_crypto_decrypt;
 545
 546 static const struct bench *benchs[] = {
 547         &bench_count_global,
 548         &bench_count_local,
 549         &bench_rename_base,
 550         &bench_rename_kprobe,
 551         &bench_rename_kretprobe,
 552         &bench_rename_rawtp,
 553         &bench_rename_fentry,
 554         &bench_rename_fexit,
 555         /* pure counting benchmarks for establishing theoretical limits */
 556         &bench_trig_usermode_count,
 557         &bench_trig_kernel_count,
 558         &bench_trig_syscall_count,
 559         /* batched, staying mostly in-kernel triggers */
 560         &bench_trig_kprobe,
 561         &bench_trig_kretprobe,
 562         &bench_trig_kprobe_multi,
 563         &bench_trig_kretprobe_multi,
 564         &bench_trig_fentry,
 565         &bench_trig_fexit,
 566         &bench_trig_fmodret,
 567         &bench_trig_tp,
 568         &bench_trig_rawtp,
 569         /* uprobes */
 570         &bench_trig_uprobe_nop,
 571         &bench_trig_uretprobe_nop,
 572         &bench_trig_uprobe_push,
 573         &bench_trig_uretprobe_push,
 574         &bench_trig_uprobe_ret,
 575         &bench_trig_uretprobe_ret,
 576         /* ringbuf/perfbuf benchmarks */
 577         &bench_rb_libbpf,
 578         &bench_rb_custom,
 579         &bench_pb_libbpf,
 580         &bench_pb_custom,
 581         &bench_bloom_lookup,
 582         &bench_bloom_update,
 583         &bench_bloom_false_positive,
 584         &bench_hashmap_without_bloom,
 585         &bench_hashmap_with_bloom,
 586         &bench_bpf_loop,
 587         &bench_strncmp_no_helper,
 588         &bench_strncmp_helper,
 589         &bench_bpf_hashmap_full_update,
 590         &bench_local_storage_cache_seq_get,
 591         &bench_local_storage_cache_interleaved_get,
 592         &bench_local_storage_cache_hashmap_control,
 593         &bench_local_storage_tasks_trace,
 594         &bench_bpf_hashmap_lookup,
 595         &bench_local_storage_create,
 596         &bench_htab_mem,
 597         &bench_crypto_encrypt,
 598         &bench_crypto_decrypt,
 599 };
 600
 601 static void find_benchmark(void)
 602 {
 603         int i;
 604
 605         if (!env.bench_name) {
 606                 fprintf(stderr, "benchmark name is not specified\n");
 607                 exit(1);
 608         }
 609         for (i = 0; i < ARRAY_SIZE(benchs); i++) {
 610                 if (strcmp(benchs[i]->name, env.bench_name) == 0) {
 611                         bench = benchs[i];
 612                         break;
 613                 }
 614         }
 615         if (!bench) {
 616                 fprintf(stderr, "benchmark '%s' not found\n", env.bench_name);
 617                 exit(1);
 618         }
 619 }
 620
 621 static void setup_benchmark(void)
 622 {
 623         int i, err;
 624
 625         if (!env.quiet)
 626                 printf("Setting up benchmark '%s'...\n", bench->name);
 627
 628         state.producers = calloc(env.producer_cnt, sizeof(*state.producers));
 629         state.consumers = calloc(env.consumer_cnt, sizeof(*state.consumers));
 630         state.results = calloc(env.duration_sec + env.warmup_sec + 2,
 631                                sizeof(*state.results));
 632         if (!state.producers || !state.consumers || !state.results)
 633                 exit(1);
 634
 635         if (bench->validate)
 636                 bench->validate();
 637         if (bench->setup)
 638                 bench->setup();
 639
 640         for (i = 0; i < env.consumer_cnt; i++) {
 641                 if (!bench->consumer_thread) {
 642                         fprintf(stderr, "benchmark doesn't support consumers!\n");
 643                         exit(1);
 644                 }
 645                 err = pthread_create(&state.consumers[i], NULL,
 646                                      bench->consumer_thread, (void *)(long)i);
 647                 if (err) {
 648                         fprintf(stderr, "failed to create consumer thread #%d: %d\n",
 649                                 i, -err);
 650                         exit(1);
 651                 }
 652                 if (env.affinity)
 653                         set_thread_affinity(state.consumers[i],
 654                                             next_cpu(&env.cons_cpus));
 655         }
 656
 657         /* unless explicit producer CPU list is specified, continue after
 658          * last consumer CPU
 659          */
 660         if (!env.prod_cpus.cpus)
 661                 env.prod_cpus.next_cpu = env.cons_cpus.next_cpu;
 662
 663         for (i = 0; i < env.producer_cnt; i++) {
 664                 if (!bench->producer_thread) {
 665                         fprintf(stderr, "benchmark doesn't support producers!\n");
 666                         exit(1);
 667                 }
 668                 err = pthread_create(&state.producers[i], NULL,
 669                                      bench->producer_thread, (void *)(long)i);
 670                 if (err) {
 671                         fprintf(stderr, "failed to create producer thread #%d: %d\n",
 672                                 i, -err);
 673                         exit(1);
 674                 }
 675                 if (env.affinity)
 676                         set_thread_affinity(state.producers[i],
 677                                             next_cpu(&env.prod_cpus));
 678         }
 679
 680         if (!env.quiet)
 681                 printf("Benchmark '%s' started.\n", bench->name);
 682 }
 683
 684 static pthread_mutex_t bench_done_mtx = PTHREAD_MUTEX_INITIALIZER;
 685 static pthread_cond_t bench_done = PTHREAD_COND_INITIALIZER;
 686
 687 static void collect_measurements(long delta_ns) {
 688         int iter = state.res_cnt++;
 689         struct bench_res *res = &state.results[iter];
 690
 691         bench->measure(res);
 692
 693         if (bench->report_progress)
 694                 bench->report_progress(iter, res, delta_ns);
 695
 696         if (iter == env.duration_sec + env.warmup_sec) {
 697                 pthread_mutex_lock(&bench_done_mtx);
 698                 pthread_cond_signal(&bench_done);
 699                 pthread_mutex_unlock(&bench_done_mtx);
 700         }
 701 }
 702
 703 int main(int argc, char **argv)
 704 {
 705         env.nr_cpus = get_nprocs();
 706         parse_cmdline_args_init(argc, argv);
 707
 708         if (env.list) {
 709                 int i;
 710
 711                 printf("Available benchmarks:\n");
 712                 for (i = 0; i < ARRAY_SIZE(benchs); i++) {
 713                         printf("- %s\n", benchs[i]->name);
 714                 }
 715                 return 0;
 716         }
 717
 718         find_benchmark();
 719         parse_cmdline_args_final(argc, argv);
 720
 721         setup_benchmark();
 722
 723         setup_timer();
 724
 725         pthread_mutex_lock(&bench_done_mtx);
 726         pthread_cond_wait(&bench_done, &bench_done_mtx);
 727         pthread_mutex_unlock(&bench_done_mtx);
 728
 729         if (bench->report_final)
 730                 /* skip first sample */
 731                 bench->report_final(state.results + env.warmup_sec,
 732                                     state.res_cnt - env.warmup_sec);
 733
 734         return 0;
 735 }