kernel/time/tick-broadcast.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * This file contains functions which emulate a local clock-event
   4  * device via a broadcast event source.
   5  *
   6  * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
   7  * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
   8  * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
   9  */
  10 #include <linux/cpu.h>
  11 #include <linux/err.h>
  12 #include <linux/hrtimer.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/percpu.h>
  15 #include <linux/profile.h>
  16 #include <linux/sched.h>
  17 #include <linux/smp.h>
  18 #include <linux/module.h>
  19
  20 #include "tick-internal.h"
  21
  22 /*
  23  * Broadcast support for broken x86 hardware, where the local apic
  24  * timer stops in C3 state.
  25  */
  26
  27 static struct tick_device tick_broadcast_device;
  28 static cpumask_var_t tick_broadcast_mask __cpumask_var_read_mostly;
  29 static cpumask_var_t tick_broadcast_on __cpumask_var_read_mostly;
  30 static cpumask_var_t tmpmask __cpumask_var_read_mostly;
  31 static int tick_broadcast_forced;
  32
  33 static __cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(tick_broadcast_lock);
  34
  35 #ifdef CONFIG_TICK_ONESHOT
  36 static void tick_broadcast_setup_oneshot(struct clock_event_device *bc);
  37 static void tick_broadcast_clear_oneshot(int cpu);
  38 static void tick_resume_broadcast_oneshot(struct clock_event_device *bc);
  39 static void tick_broadcast_oneshot_offline(unsigned int cpu);
  40 #else
  41 static inline void tick_broadcast_setup_oneshot(struct clock_event_device *bc) { BUG(); }
  42 static inline void tick_broadcast_clear_oneshot(int cpu) { }
  43 static inline void tick_resume_broadcast_oneshot(struct clock_event_device *bc) { }
  44 static inline void tick_broadcast_oneshot_offline(unsigned int cpu) { }
  45 #endif
  46
  47 /*
  48  * Debugging: see timer_list.c
  49  */
  50 struct tick_device *tick_get_broadcast_device(void)
  51 {
  52         return &tick_broadcast_device;
  53 }
  54
  55 struct cpumask *tick_get_broadcast_mask(void)
  56 {
  57         return tick_broadcast_mask;
  58 }
  59
  60 /*
  61  * Start the device in periodic mode
  62  */
  63 static void tick_broadcast_start_periodic(struct clock_event_device *bc)
  64 {
  65         if (bc)
  66                 tick_setup_periodic(bc, 1);
  67 }
  68
  69 /*
  70  * Check, if the device can be utilized as broadcast device:
  71  */
  72 static bool tick_check_broadcast_device(struct clock_event_device *curdev,
  73                                         struct clock_event_device *newdev)
  74 {
  75         if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) ||
  76             (newdev->features & CLOCK_EVT_FEAT_PERCPU) ||
  77             (newdev->features & CLOCK_EVT_FEAT_C3STOP))
  78                 return false;
  79
  80         if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT &&
  81             !(newdev->features & CLOCK_EVT_FEAT_ONESHOT))
  82                 return false;
  83
  84         return !curdev || newdev->rating > curdev->rating;
  85 }
  86
  87 /*
  88  * Conditionally install/replace broadcast device
  89  */
  90 void tick_install_broadcast_device(struct clock_event_device *dev)
  91 {
  92         struct clock_event_device *cur = tick_broadcast_device.evtdev;
  93
  94         if (!tick_check_broadcast_device(cur, dev))
  95                 return;
  96
  97         if (!try_module_get(dev->owner))
  98                 return;
  99
 100         clockevents_exchange_device(cur, dev);
 101         if (cur)
 102                 cur->event_handler = clockevents_handle_noop;
 103         tick_broadcast_device.evtdev = dev;
 104         if (!cpumask_empty(tick_broadcast_mask))
 105                 tick_broadcast_start_periodic(dev);
 106         /*
 107          * Inform all cpus about this. We might be in a situation
 108          * where we did not switch to oneshot mode because the per cpu
 109          * devices are affected by CLOCK_EVT_FEAT_C3STOP and the lack
 110          * of a oneshot capable broadcast device. Without that
 111          * notification the systems stays stuck in periodic mode
 112          * forever.
 113          */
 114         if (dev->features & CLOCK_EVT_FEAT_ONESHOT)
 115                 tick_clock_notify();
 116 }
 117
 118 /*
 119  * Check, if the device is the broadcast device
 120  */
 121 int tick_is_broadcast_device(struct clock_event_device *dev)
 122 {
 123         return (dev && tick_broadcast_device.evtdev == dev);
 124 }
 125
 126 int tick_broadcast_update_freq(struct clock_event_device *dev, u32 freq)
 127 {
 128         int ret = -ENODEV;
 129
 130         if (tick_is_broadcast_device(dev)) {
 131                 raw_spin_lock(&tick_broadcast_lock);
 132                 ret = __clockevents_update_freq(dev, freq);
 133                 raw_spin_unlock(&tick_broadcast_lock);
 134         }
 135         return ret;
 136 }
 137
 138
 139 static void err_broadcast(const struct cpumask *mask)
 140 {
 141         pr_crit_once("Failed to broadcast timer tick. Some CPUs may be unresponsive.\n");
 142 }
 143
 144 static void tick_device_setup_broadcast_func(struct clock_event_device *dev)
 145 {
 146         if (!dev->broadcast)
 147                 dev->broadcast = tick_broadcast;
 148         if (!dev->broadcast) {
 149                 pr_warn_once("%s depends on broadcast, but no broadcast function available\n",
 150                              dev->name);
 151                 dev->broadcast = err_broadcast;
 152         }
 153 }
 154
 155 /*
 156  * Check, if the device is disfunctional and a place holder, which
 157  * needs to be handled by the broadcast device.
 158  */
 159 int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
 160 {
 161         struct clock_event_device *bc = tick_broadcast_device.evtdev;
 162         unsigned long flags;
 163         int ret = 0;
 164
 165         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 166
 167         /*
 168          * Devices might be registered with both periodic and oneshot
 169          * mode disabled. This signals, that the device needs to be
 170          * operated from the broadcast device and is a placeholder for
 171          * the cpu local device.
 172          */
 173         if (!tick_device_is_functional(dev)) {
 174                 dev->event_handler = tick_handle_periodic;
 175                 tick_device_setup_broadcast_func(dev);
 176                 cpumask_set_cpu(cpu, tick_broadcast_mask);
 177                 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
 178                         tick_broadcast_start_periodic(bc);
 179                 else
 180                         tick_broadcast_setup_oneshot(bc);
 181                 ret = 1;
 182         } else {
 183                 /*
 184                  * Clear the broadcast bit for this cpu if the
 185                  * device is not power state affected.
 186                  */
 187                 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
 188                         cpumask_clear_cpu(cpu, tick_broadcast_mask);
 189                 else
 190                         tick_device_setup_broadcast_func(dev);
 191
 192                 /*
 193                  * Clear the broadcast bit if the CPU is not in
 194                  * periodic broadcast on state.
 195                  */
 196                 if (!cpumask_test_cpu(cpu, tick_broadcast_on))
 197                         cpumask_clear_cpu(cpu, tick_broadcast_mask);
 198
 199                 switch (tick_broadcast_device.mode) {
 200                 case TICKDEV_MODE_ONESHOT:
 201                         /*
 202                          * If the system is in oneshot mode we can
 203                          * unconditionally clear the oneshot mask bit,
 204                          * because the CPU is running and therefore
 205                          * not in an idle state which causes the power
 206                          * state affected device to stop. Let the
 207                          * caller initialize the device.
 208                          */
 209                         tick_broadcast_clear_oneshot(cpu);
 210                         ret = 0;
 211                         break;
 212
 213                 case TICKDEV_MODE_PERIODIC:
 214                         /*
 215                          * If the system is in periodic mode, check
 216                          * whether the broadcast device can be
 217                          * switched off now.
 218                          */
 219                         if (cpumask_empty(tick_broadcast_mask) && bc)
 220                                 clockevents_shutdown(bc);
 221                         /*
 222                          * If we kept the cpu in the broadcast mask,
 223                          * tell the caller to leave the per cpu device
 224                          * in shutdown state. The periodic interrupt
 225                          * is delivered by the broadcast device, if
 226                          * the broadcast device exists and is not
 227                          * hrtimer based.
 228                          */
 229                         if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER))
 230                                 ret = cpumask_test_cpu(cpu, tick_broadcast_mask);
 231                         break;
 232                 default:
 233                         break;
 234                 }
 235         }
 236         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 237         return ret;
 238 }
 239
 240 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
 241 int tick_receive_broadcast(void)
 242 {
 243         struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
 244         struct clock_event_device *evt = td->evtdev;
 245
 246         if (!evt)
 247                 return -ENODEV;
 248
 249         if (!evt->event_handler)
 250                 return -EINVAL;
 251
 252         evt->event_handler(evt);
 253         return 0;
 254 }
 255 #endif
 256
 257 /*
 258  * Broadcast the event to the cpus, which are set in the mask (mangled).
 259  */
 260 static bool tick_do_broadcast(struct cpumask *mask)
 261 {
 262         int cpu = smp_processor_id();
 263         struct tick_device *td;
 264         bool local = false;
 265
 266         /*
 267          * Check, if the current cpu is in the mask
 268          */
 269         if (cpumask_test_cpu(cpu, mask)) {
 270                 struct clock_event_device *bc = tick_broadcast_device.evtdev;
 271
 272                 cpumask_clear_cpu(cpu, mask);
 273                 /*
 274                  * We only run the local handler, if the broadcast
 275                  * device is not hrtimer based. Otherwise we run into
 276                  * a hrtimer recursion.
 277                  *
 278                  * local timer_interrupt()
 279                  *   local_handler()
 280                  *     expire_hrtimers()
 281                  *       bc_handler()
 282                  *         local_handler()
 283                  *           expire_hrtimers()
 284                  */
 285                 local = !(bc->features & CLOCK_EVT_FEAT_HRTIMER);
 286         }
 287
 288         if (!cpumask_empty(mask)) {
 289                 /*
 290                  * It might be necessary to actually check whether the devices
 291                  * have different broadcast functions. For now, just use the
 292                  * one of the first device. This works as long as we have this
 293                  * misfeature only on x86 (lapic)
 294                  */
 295                 td = &per_cpu(tick_cpu_device, cpumask_first(mask));
 296                 td->evtdev->broadcast(mask);
 297         }
 298         return local;
 299 }
 300
 301 /*
 302  * Periodic broadcast:
 303  * - invoke the broadcast handlers
 304  */
 305 static bool tick_do_periodic_broadcast(void)
 306 {
 307         cpumask_and(tmpmask, cpu_online_mask, tick_broadcast_mask);
 308         return tick_do_broadcast(tmpmask);
 309 }
 310
 311 /*
 312  * Event handler for periodic broadcast ticks
 313  */
 314 static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
 315 {
 316         struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
 317         bool bc_local;
 318
 319         raw_spin_lock(&tick_broadcast_lock);
 320
 321         /* Handle spurious interrupts gracefully */
 322         if (clockevent_state_shutdown(tick_broadcast_device.evtdev)) {
 323                 raw_spin_unlock(&tick_broadcast_lock);
 324                 return;
 325         }
 326
 327         bc_local = tick_do_periodic_broadcast();
 328
 329         if (clockevent_state_oneshot(dev)) {
 330                 ktime_t next = ktime_add(dev->next_event, tick_period);
 331
 332                 clockevents_program_event(dev, next, true);
 333         }
 334         raw_spin_unlock(&tick_broadcast_lock);
 335
 336         /*
 337          * We run the handler of the local cpu after dropping
 338          * tick_broadcast_lock because the handler might deadlock when
 339          * trying to switch to oneshot mode.
 340          */
 341         if (bc_local)
 342                 td->evtdev->event_handler(td->evtdev);
 343 }
 344
 345 /**
 346  * tick_broadcast_control - Enable/disable or force broadcast mode
 347  * @mode:       The selected broadcast mode
 348  *
 349  * Called when the system enters a state where affected tick devices
 350  * might stop. Note: TICK_BROADCAST_FORCE cannot be undone.
 351  */
 352 void tick_broadcast_control(enum tick_broadcast_mode mode)
 353 {
 354         struct clock_event_device *bc, *dev;
 355         struct tick_device *td;
 356         int cpu, bc_stopped;
 357         unsigned long flags;
 358
 359         /* Protects also the local clockevent device. */
 360         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 361         td = this_cpu_ptr(&tick_cpu_device);
 362         dev = td->evtdev;
 363
 364         /*
 365          * Is the device not affected by the powerstate ?
 366          */
 367         if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP))
 368                 goto out;
 369
 370         if (!tick_device_is_functional(dev))
 371                 goto out;
 372
 373         cpu = smp_processor_id();
 374         bc = tick_broadcast_device.evtdev;
 375         bc_stopped = cpumask_empty(tick_broadcast_mask);
 376
 377         switch (mode) {
 378         case TICK_BROADCAST_FORCE:
 379                 tick_broadcast_forced = 1;
 380                 /* fall through */
 381         case TICK_BROADCAST_ON:
 382                 cpumask_set_cpu(cpu, tick_broadcast_on);
 383                 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_mask)) {
 384                         /*
 385                          * Only shutdown the cpu local device, if:
 386                          *
 387                          * - the broadcast device exists
 388                          * - the broadcast device is not a hrtimer based one
 389                          * - the broadcast device is in periodic mode to
 390                          *   avoid a hickup during switch to oneshot mode
 391                          */
 392                         if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER) &&
 393                             tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
 394                                 clockevents_shutdown(dev);
 395                 }
 396                 break;
 397
 398         case TICK_BROADCAST_OFF:
 399                 if (tick_broadcast_forced)
 400                         break;
 401                 cpumask_clear_cpu(cpu, tick_broadcast_on);
 402                 if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_mask)) {
 403                         if (tick_broadcast_device.mode ==
 404                             TICKDEV_MODE_PERIODIC)
 405                                 tick_setup_periodic(dev, 0);
 406                 }
 407                 break;
 408         }
 409
 410         if (bc) {
 411                 if (cpumask_empty(tick_broadcast_mask)) {
 412                         if (!bc_stopped)
 413                                 clockevents_shutdown(bc);
 414                 } else if (bc_stopped) {
 415                         if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
 416                                 tick_broadcast_start_periodic(bc);
 417                         else
 418                                 tick_broadcast_setup_oneshot(bc);
 419                 }
 420         }
 421 out:
 422         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 423 }
 424 EXPORT_SYMBOL_GPL(tick_broadcast_control);
 425
 426 /*
 427  * Set the periodic handler depending on broadcast on/off
 428  */
 429 void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
 430 {
 431         if (!broadcast)
 432                 dev->event_handler = tick_handle_periodic;
 433         else
 434                 dev->event_handler = tick_handle_periodic_broadcast;
 435 }
 436
 437 #ifdef CONFIG_HOTPLUG_CPU
 438 static void tick_shutdown_broadcast(void)
 439 {
 440         struct clock_event_device *bc = tick_broadcast_device.evtdev;
 441
 442         if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
 443                 if (bc && cpumask_empty(tick_broadcast_mask))
 444                         clockevents_shutdown(bc);
 445         }
 446 }
 447
 448 /*
 449  * Remove a CPU from broadcasting
 450  */
 451 void tick_broadcast_offline(unsigned int cpu)
 452 {
 453         raw_spin_lock(&tick_broadcast_lock);
 454         cpumask_clear_cpu(cpu, tick_broadcast_mask);
 455         cpumask_clear_cpu(cpu, tick_broadcast_on);
 456         tick_broadcast_oneshot_offline(cpu);
 457         tick_shutdown_broadcast();
 458         raw_spin_unlock(&tick_broadcast_lock);
 459 }
 460
 461 #endif
 462
 463 void tick_suspend_broadcast(void)
 464 {
 465         struct clock_event_device *bc;
 466         unsigned long flags;
 467
 468         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 469
 470         bc = tick_broadcast_device.evtdev;
 471         if (bc)
 472                 clockevents_shutdown(bc);
 473
 474         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 475 }
 476
 477 /*
 478  * This is called from tick_resume_local() on a resuming CPU. That's
 479  * called from the core resume function, tick_unfreeze() and the magic XEN
 480  * resume hackery.
 481  *
 482  * In none of these cases the broadcast device mode can change and the
 483  * bit of the resuming CPU in the broadcast mask is safe as well.
 484  */
 485 bool tick_resume_check_broadcast(void)
 486 {
 487         if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT)
 488                 return false;
 489         else
 490                 return cpumask_test_cpu(smp_processor_id(), tick_broadcast_mask);
 491 }
 492
 493 void tick_resume_broadcast(void)
 494 {
 495         struct clock_event_device *bc;
 496         unsigned long flags;
 497
 498         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 499
 500         bc = tick_broadcast_device.evtdev;
 501
 502         if (bc) {
 503                 clockevents_tick_resume(bc);
 504
 505                 switch (tick_broadcast_device.mode) {
 506                 case TICKDEV_MODE_PERIODIC:
 507                         if (!cpumask_empty(tick_broadcast_mask))
 508                                 tick_broadcast_start_periodic(bc);
 509                         break;
 510                 case TICKDEV_MODE_ONESHOT:
 511                         if (!cpumask_empty(tick_broadcast_mask))
 512                                 tick_resume_broadcast_oneshot(bc);
 513                         break;
 514                 }
 515         }
 516         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 517 }
 518
 519 #ifdef CONFIG_TICK_ONESHOT
 520
 521 static cpumask_var_t tick_broadcast_oneshot_mask __cpumask_var_read_mostly;
 522 static cpumask_var_t tick_broadcast_pending_mask __cpumask_var_read_mostly;
 523 static cpumask_var_t tick_broadcast_force_mask __cpumask_var_read_mostly;
 524
 525 /*
 526  * Exposed for debugging: see timer_list.c
 527  */
 528 struct cpumask *tick_get_broadcast_oneshot_mask(void)
 529 {
 530         return tick_broadcast_oneshot_mask;
 531 }
 532
 533 /*
 534  * Called before going idle with interrupts disabled. Checks whether a
 535  * broadcast event from the other core is about to happen. We detected
 536  * that in tick_broadcast_oneshot_control(). The callsite can use this
 537  * to avoid a deep idle transition as we are about to get the
 538  * broadcast IPI right away.
 539  */
 540 int tick_check_broadcast_expired(void)
 541 {
 542         return cpumask_test_cpu(smp_processor_id(), tick_broadcast_force_mask);
 543 }
 544
 545 /*
 546  * Set broadcast interrupt affinity
 547  */
 548 static void tick_broadcast_set_affinity(struct clock_event_device *bc,
 549                                         const struct cpumask *cpumask)
 550 {
 551         if (!(bc->features & CLOCK_EVT_FEAT_DYNIRQ))
 552                 return;
 553
 554         if (cpumask_equal(bc->cpumask, cpumask))
 555                 return;
 556
 557         bc->cpumask = cpumask;
 558         irq_set_affinity(bc->irq, bc->cpumask);
 559 }
 560
 561 static void tick_broadcast_set_event(struct clock_event_device *bc, int cpu,
 562                                      ktime_t expires)
 563 {
 564         if (!clockevent_state_oneshot(bc))
 565                 clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);
 566
 567         clockevents_program_event(bc, expires, 1);
 568         tick_broadcast_set_affinity(bc, cpumask_of(cpu));
 569 }
 570
 571 static void tick_resume_broadcast_oneshot(struct clock_event_device *bc)
 572 {
 573         clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);
 574 }
 575
 576 /*
 577  * Called from irq_enter() when idle was interrupted to reenable the
 578  * per cpu device.
 579  */
 580 void tick_check_oneshot_broadcast_this_cpu(void)
 581 {
 582         if (cpumask_test_cpu(smp_processor_id(), tick_broadcast_oneshot_mask)) {
 583                 struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
 584
 585                 /*
 586                  * We might be in the middle of switching over from
 587                  * periodic to oneshot. If the CPU has not yet
 588                  * switched over, leave the device alone.
 589                  */
 590                 if (td->mode == TICKDEV_MODE_ONESHOT) {
 591                         clockevents_switch_state(td->evtdev,
 592                                               CLOCK_EVT_STATE_ONESHOT);
 593                 }
 594         }
 595 }
 596
 597 /*
 598  * Handle oneshot mode broadcasting
 599  */
 600 static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
 601 {
 602         struct tick_device *td;
 603         ktime_t now, next_event;
 604         int cpu, next_cpu = 0;
 605         bool bc_local;
 606
 607         raw_spin_lock(&tick_broadcast_lock);
 608         dev->next_event = KTIME_MAX;
 609         next_event = KTIME_MAX;
 610         cpumask_clear(tmpmask);
 611         now = ktime_get();
 612         /* Find all expired events */
 613         for_each_cpu(cpu, tick_broadcast_oneshot_mask) {
 614                 /*
 615                  * Required for !SMP because for_each_cpu() reports
 616                  * unconditionally CPU0 as set on UP kernels.
 617                  */
 618                 if (!IS_ENABLED(CONFIG_SMP) &&
 619                     cpumask_empty(tick_broadcast_oneshot_mask))
 620                         break;
 621
 622                 td = &per_cpu(tick_cpu_device, cpu);
 623                 if (td->evtdev->next_event <= now) {
 624                         cpumask_set_cpu(cpu, tmpmask);
 625                         /*
 626                          * Mark the remote cpu in the pending mask, so
 627                          * it can avoid reprogramming the cpu local
 628                          * timer in tick_broadcast_oneshot_control().
 629                          */
 630                         cpumask_set_cpu(cpu, tick_broadcast_pending_mask);
 631                 } else if (td->evtdev->next_event < next_event) {
 632                         next_event = td->evtdev->next_event;
 633                         next_cpu = cpu;
 634                 }
 635         }
 636
 637         /*
 638          * Remove the current cpu from the pending mask. The event is
 639          * delivered immediately in tick_do_broadcast() !
 640          */
 641         cpumask_clear_cpu(smp_processor_id(), tick_broadcast_pending_mask);
 642
 643         /* Take care of enforced broadcast requests */
 644         cpumask_or(tmpmask, tmpmask, tick_broadcast_force_mask);
 645         cpumask_clear(tick_broadcast_force_mask);
 646
 647         /*
 648          * Sanity check. Catch the case where we try to broadcast to
 649          * offline cpus.
 650          */
 651         if (WARN_ON_ONCE(!cpumask_subset(tmpmask, cpu_online_mask)))
 652                 cpumask_and(tmpmask, tmpmask, cpu_online_mask);
 653
 654         /*
 655          * Wakeup the cpus which have an expired event.
 656          */
 657         bc_local = tick_do_broadcast(tmpmask);
 658
 659         /*
 660          * Two reasons for reprogram:
 661          *
 662          * - The global event did not expire any CPU local
 663          * events. This happens in dyntick mode, as the maximum PIT
 664          * delta is quite small.
 665          *
 666          * - There are pending events on sleeping CPUs which were not
 667          * in the event mask
 668          */
 669         if (next_event != KTIME_MAX)
 670                 tick_broadcast_set_event(dev, next_cpu, next_event);
 671
 672         raw_spin_unlock(&tick_broadcast_lock);
 673
 674         if (bc_local) {
 675                 td = this_cpu_ptr(&tick_cpu_device);
 676                 td->evtdev->event_handler(td->evtdev);
 677         }
 678 }
 679
 680 static int broadcast_needs_cpu(struct clock_event_device *bc, int cpu)
 681 {
 682         if (!(bc->features & CLOCK_EVT_FEAT_HRTIMER))
 683                 return 0;
 684         if (bc->next_event == KTIME_MAX)
 685                 return 0;
 686         return bc->bound_on == cpu ? -EBUSY : 0;
 687 }
 688
 689 static void broadcast_shutdown_local(struct clock_event_device *bc,
 690                                      struct clock_event_device *dev)
 691 {
 692         /*
 693          * For hrtimer based broadcasting we cannot shutdown the cpu
 694          * local device if our own event is the first one to expire or
 695          * if we own the broadcast timer.
 696          */
 697         if (bc->features & CLOCK_EVT_FEAT_HRTIMER) {
 698                 if (broadcast_needs_cpu(bc, smp_processor_id()))
 699                         return;
 700                 if (dev->next_event < bc->next_event)
 701                         return;
 702         }
 703         clockevents_switch_state(dev, CLOCK_EVT_STATE_SHUTDOWN);
 704 }
 705
 706 int __tick_broadcast_oneshot_control(enum tick_broadcast_state state)
 707 {
 708         struct clock_event_device *bc, *dev;
 709         int cpu, ret = 0;
 710         ktime_t now;
 711
 712         /*
 713          * If there is no broadcast device, tell the caller not to go
 714          * into deep idle.
 715          */
 716         if (!tick_broadcast_device.evtdev)
 717                 return -EBUSY;
 718
 719         dev = this_cpu_ptr(&tick_cpu_device)->evtdev;
 720
 721         raw_spin_lock(&tick_broadcast_lock);
 722         bc = tick_broadcast_device.evtdev;
 723         cpu = smp_processor_id();
 724
 725         if (state == TICK_BROADCAST_ENTER) {
 726                 /*
 727                  * If the current CPU owns the hrtimer broadcast
 728                  * mechanism, it cannot go deep idle and we do not add
 729                  * the CPU to the broadcast mask. We don't have to go
 730                  * through the EXIT path as the local timer is not
 731                  * shutdown.
 732                  */
 733                 ret = broadcast_needs_cpu(bc, cpu);
 734                 if (ret)
 735                         goto out;
 736
 737                 /*
 738                  * If the broadcast device is in periodic mode, we
 739                  * return.
 740                  */
 741                 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
 742                         /* If it is a hrtimer based broadcast, return busy */
 743                         if (bc->features & CLOCK_EVT_FEAT_HRTIMER)
 744                                 ret = -EBUSY;
 745                         goto out;
 746                 }
 747
 748                 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_oneshot_mask)) {
 749                         WARN_ON_ONCE(cpumask_test_cpu(cpu, tick_broadcast_pending_mask));
 750
 751                         /* Conditionally shut down the local timer. */
 752                         broadcast_shutdown_local(bc, dev);
 753
 754                         /*
 755                          * We only reprogram the broadcast timer if we
 756                          * did not mark ourself in the force mask and
 757                          * if the cpu local event is earlier than the
 758                          * broadcast event. If the current CPU is in
 759                          * the force mask, then we are going to be
 760                          * woken by the IPI right away; we return
 761                          * busy, so the CPU does not try to go deep
 762                          * idle.
 763                          */
 764                         if (cpumask_test_cpu(cpu, tick_broadcast_force_mask)) {
 765                                 ret = -EBUSY;
 766                         } else if (dev->next_event < bc->next_event) {
 767                                 tick_broadcast_set_event(bc, cpu, dev->next_event);
 768                                 /*
 769                                  * In case of hrtimer broadcasts the
 770                                  * programming might have moved the
 771                                  * timer to this cpu. If yes, remove
 772                                  * us from the broadcast mask and
 773                                  * return busy.
 774                                  */
 775                                 ret = broadcast_needs_cpu(bc, cpu);
 776                                 if (ret) {
 777                                         cpumask_clear_cpu(cpu,
 778                                                 tick_broadcast_oneshot_mask);
 779                                 }
 780                         }
 781                 }
 782         } else {
 783                 if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_oneshot_mask)) {
 784                         clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT);
 785                         /*
 786                          * The cpu which was handling the broadcast
 787                          * timer marked this cpu in the broadcast
 788                          * pending mask and fired the broadcast
 789                          * IPI. So we are going to handle the expired
 790                          * event anyway via the broadcast IPI
 791                          * handler. No need to reprogram the timer
 792                          * with an already expired event.
 793                          */
 794                         if (cpumask_test_and_clear_cpu(cpu,
 795                                        tick_broadcast_pending_mask))
 796                                 goto out;
 797
 798                         /*
 799                          * Bail out if there is no next event.
 800                          */
 801                         if (dev->next_event == KTIME_MAX)
 802                                 goto out;
 803                         /*
 804                          * If the pending bit is not set, then we are
 805                          * either the CPU handling the broadcast
 806                          * interrupt or we got woken by something else.
 807                          *
 808                          * We are not longer in the broadcast mask, so
 809                          * if the cpu local expiry time is already
 810                          * reached, we would reprogram the cpu local
 811                          * timer with an already expired event.
 812                          *
 813                          * This can lead to a ping-pong when we return
 814                          * to idle and therefor rearm the broadcast
 815                          * timer before the cpu local timer was able
 816                          * to fire. This happens because the forced
 817                          * reprogramming makes sure that the event
 818                          * will happen in the future and depending on
 819                          * the min_delta setting this might be far
 820                          * enough out that the ping-pong starts.
 821                          *
 822                          * If the cpu local next_event has expired
 823                          * then we know that the broadcast timer
 824                          * next_event has expired as well and
 825                          * broadcast is about to be handled. So we
 826                          * avoid reprogramming and enforce that the
 827                          * broadcast handler, which did not run yet,
 828                          * will invoke the cpu local handler.
 829                          *
 830                          * We cannot call the handler directly from
 831                          * here, because we might be in a NOHZ phase
 832                          * and we did not go through the irq_enter()
 833                          * nohz fixups.
 834                          */
 835                         now = ktime_get();
 836                         if (dev->next_event <= now) {
 837                                 cpumask_set_cpu(cpu, tick_broadcast_force_mask);
 838                                 goto out;
 839                         }
 840                         /*
 841                          * We got woken by something else. Reprogram
 842                          * the cpu local timer device.
 843                          */
 844                         tick_program_event(dev->next_event, 1);
 845                 }
 846         }
 847 out:
 848         raw_spin_unlock(&tick_broadcast_lock);
 849         return ret;
 850 }
 851
 852 /*
 853  * Reset the one shot broadcast for a cpu
 854  *
 855  * Called with tick_broadcast_lock held
 856  */
 857 static void tick_broadcast_clear_oneshot(int cpu)
 858 {
 859         cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
 860         cpumask_clear_cpu(cpu, tick_broadcast_pending_mask);
 861 }
 862
 863 static void tick_broadcast_init_next_event(struct cpumask *mask,
 864                                            ktime_t expires)
 865 {
 866         struct tick_device *td;
 867         int cpu;
 868
 869         for_each_cpu(cpu, mask) {
 870                 td = &per_cpu(tick_cpu_device, cpu);
 871                 if (td->evtdev)
 872                         td->evtdev->next_event = expires;
 873         }
 874 }
 875
 876 /**
 877  * tick_broadcast_setup_oneshot - setup the broadcast device
 878  */
 879 static void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
 880 {
 881         int cpu = smp_processor_id();
 882
 883         if (!bc)
 884                 return;
 885
 886         /* Set it up only once ! */
 887         if (bc->event_handler != tick_handle_oneshot_broadcast) {
 888                 int was_periodic = clockevent_state_periodic(bc);
 889
 890                 bc->event_handler = tick_handle_oneshot_broadcast;
 891
 892                 /*
 893                  * We must be careful here. There might be other CPUs
 894                  * waiting for periodic broadcast. We need to set the
 895                  * oneshot_mask bits for those and program the
 896                  * broadcast device to fire.
 897                  */
 898                 cpumask_copy(tmpmask, tick_broadcast_mask);
 899                 cpumask_clear_cpu(cpu, tmpmask);
 900                 cpumask_or(tick_broadcast_oneshot_mask,
 901                            tick_broadcast_oneshot_mask, tmpmask);
 902
 903                 if (was_periodic && !cpumask_empty(tmpmask)) {
 904                         clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);
 905                         tick_broadcast_init_next_event(tmpmask,
 906                                                        tick_next_period);
 907                         tick_broadcast_set_event(bc, cpu, tick_next_period);
 908                 } else
 909                         bc->next_event = KTIME_MAX;
 910         } else {
 911                 /*
 912                  * The first cpu which switches to oneshot mode sets
 913                  * the bit for all other cpus which are in the general
 914                  * (periodic) broadcast mask. So the bit is set and
 915                  * would prevent the first broadcast enter after this
 916                  * to program the bc device.
 917                  */
 918                 tick_broadcast_clear_oneshot(cpu);
 919         }
 920 }
 921
 922 /*
 923  * Select oneshot operating mode for the broadcast device
 924  */
 925 void tick_broadcast_switch_to_oneshot(void)
 926 {
 927         struct clock_event_device *bc;
 928         unsigned long flags;
 929
 930         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 931
 932         tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
 933         bc = tick_broadcast_device.evtdev;
 934         if (bc)
 935                 tick_broadcast_setup_oneshot(bc);
 936
 937         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 938 }
 939
 940 #ifdef CONFIG_HOTPLUG_CPU
 941 void hotplug_cpu__broadcast_tick_pull(int deadcpu)
 942 {
 943         struct clock_event_device *bc;
 944         unsigned long flags;
 945
 946         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 947         bc = tick_broadcast_device.evtdev;
 948
 949         if (bc && broadcast_needs_cpu(bc, deadcpu)) {
 950                 /* This moves the broadcast assignment to this CPU: */
 951                 clockevents_program_event(bc, bc->next_event, 1);
 952         }
 953         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 954 }
 955
 956 /*
 957  * Remove a dying CPU from broadcasting
 958  */
 959 static void tick_broadcast_oneshot_offline(unsigned int cpu)
 960 {
 961         /*
 962          * Clear the broadcast masks for the dead cpu, but do not stop
 963          * the broadcast device!
 964          */
 965         cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
 966         cpumask_clear_cpu(cpu, tick_broadcast_pending_mask);
 967         cpumask_clear_cpu(cpu, tick_broadcast_force_mask);
 968 }
 969 #endif
 970
 971 /*
 972  * Check, whether the broadcast device is in one shot mode
 973  */
 974 int tick_broadcast_oneshot_active(void)
 975 {
 976         return tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT;
 977 }
 978
 979 /*
 980  * Check whether the broadcast device supports oneshot.
 981  */
 982 bool tick_broadcast_oneshot_available(void)
 983 {
 984         struct clock_event_device *bc = tick_broadcast_device.evtdev;
 985
 986         return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false;
 987 }
 988
 989 #else
 990 int __tick_broadcast_oneshot_control(enum tick_broadcast_state state)
 991 {
 992         struct clock_event_device *bc = tick_broadcast_device.evtdev;
 993
 994         if (!bc || (bc->features & CLOCK_EVT_FEAT_HRTIMER))
 995                 return -EBUSY;
 996
 997         return 0;
 998 }
 999 #endif
1000
1001 void __init tick_broadcast_init(void)
1002 {
1003         zalloc_cpumask_var(&tick_broadcast_mask, GFP_NOWAIT);
1004         zalloc_cpumask_var(&tick_broadcast_on, GFP_NOWAIT);
1005         zalloc_cpumask_var(&tmpmask, GFP_NOWAIT);
1006 #ifdef CONFIG_TICK_ONESHOT
1007         zalloc_cpumask_var(&tick_broadcast_oneshot_mask, GFP_NOWAIT);
1008         zalloc_cpumask_var(&tick_broadcast_pending_mask, GFP_NOWAIT);
1009         zalloc_cpumask_var(&tick_broadcast_force_mask, GFP_NOWAIT);
1010 #endif
1011 }