kernel/power/hibernate.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * kernel/power/hibernate.c - Hibernation (a.k.a suspend-to-disk) support.
   4  *
   5  * Copyright (c) 2003 Patrick Mochel
   6  * Copyright (c) 2003 Open Source Development Lab
   7  * Copyright (c) 2004 Pavel Machek <pavel@ucw.cz>
   8  * Copyright (c) 2009 Rafael J. Wysocki, Novell Inc.
   9  * Copyright (C) 2012 Bojan Smojver <bojan@rexursive.com>
  10  */
  11
  12 #define pr_fmt(fmt) "PM: hibernation: " fmt
  13
  14 #include <crypto/acompress.h>
  15 #include <linux/blkdev.h>
  16 #include <linux/export.h>
  17 #include <linux/suspend.h>
  18 #include <linux/reboot.h>
  19 #include <linux/string.h>
  20 #include <linux/device.h>
  21 #include <linux/async.h>
  22 #include <linux/delay.h>
  23 #include <linux/fs.h>
  24 #include <linux/mount.h>
  25 #include <linux/pm.h>
  26 #include <linux/nmi.h>
  27 #include <linux/console.h>
  28 #include <linux/cpu.h>
  29 #include <linux/freezer.h>
  30 #include <linux/gfp.h>
  31 #include <linux/syscore_ops.h>
  32 #include <linux/ctype.h>
  33 #include <linux/ktime.h>
  34 #include <linux/security.h>
  35 #include <linux/secretmem.h>
  36 #include <trace/events/power.h>
  37
  38 #include "power.h"
  39
  40
  41 static int nocompress;
  42 static int noresume;
  43 static int nohibernate;
  44 static int resume_wait;
  45 static unsigned int resume_delay;
  46 static char resume_file[256] = CONFIG_PM_STD_PARTITION;
  47 dev_t swsusp_resume_device;
  48 sector_t swsusp_resume_block;
  49 __visible int in_suspend __nosavedata;
  50
  51 static char hibernate_compressor[CRYPTO_MAX_ALG_NAME] = CONFIG_HIBERNATION_DEF_COMP;
  52
  53 /*
  54  * Compression/decompression algorithm to be used while saving/loading
  55  * image to/from disk. This would later be used in 'kernel/power/swap.c'
  56  * to allocate comp streams.
  57  */
  58 char hib_comp_algo[CRYPTO_MAX_ALG_NAME];
  59
  60 enum {
  61         HIBERNATION_INVALID,
  62         HIBERNATION_PLATFORM,
  63         HIBERNATION_SHUTDOWN,
  64         HIBERNATION_REBOOT,
  65 #ifdef CONFIG_SUSPEND
  66         HIBERNATION_SUSPEND,
  67 #endif
  68         HIBERNATION_TEST_RESUME,
  69         /* keep last */
  70         __HIBERNATION_AFTER_LAST
  71 };
  72 #define HIBERNATION_MAX (__HIBERNATION_AFTER_LAST-1)
  73 #define HIBERNATION_FIRST (HIBERNATION_INVALID + 1)
  74
  75 static int hibernation_mode = HIBERNATION_SHUTDOWN;
  76
  77 bool freezer_test_done;
  78
  79 static const struct platform_hibernation_ops *hibernation_ops;
  80
  81 static atomic_t hibernate_atomic = ATOMIC_INIT(1);
  82
  83 bool hibernate_acquire(void)
  84 {
  85         return atomic_add_unless(&hibernate_atomic, -1, 0);
  86 }
  87
  88 void hibernate_release(void)
  89 {
  90         atomic_inc(&hibernate_atomic);
  91 }
  92
  93 bool hibernation_in_progress(void)
  94 {
  95         return !atomic_read(&hibernate_atomic);
  96 }
  97
  98 bool hibernation_available(void)
  99 {
 100         return nohibernate == 0 &&
 101                 !security_locked_down(LOCKDOWN_HIBERNATION) &&
 102                 !secretmem_active() && !cxl_mem_active();
 103 }
 104
 105 /**
 106  * hibernation_set_ops - Set the global hibernate operations.
 107  * @ops: Hibernation operations to use in subsequent hibernation transitions.
 108  */
 109 void hibernation_set_ops(const struct platform_hibernation_ops *ops)
 110 {
 111         unsigned int sleep_flags;
 112
 113         if (ops && !(ops->begin && ops->end &&  ops->pre_snapshot
 114             && ops->prepare && ops->finish && ops->enter && ops->pre_restore
 115             && ops->restore_cleanup && ops->leave)) {
 116                 WARN_ON(1);
 117                 return;
 118         }
 119
 120         sleep_flags = lock_system_sleep();
 121
 122         hibernation_ops = ops;
 123         if (ops)
 124                 hibernation_mode = HIBERNATION_PLATFORM;
 125         else if (hibernation_mode == HIBERNATION_PLATFORM)
 126                 hibernation_mode = HIBERNATION_SHUTDOWN;
 127
 128         unlock_system_sleep(sleep_flags);
 129 }
 130 EXPORT_SYMBOL_GPL(hibernation_set_ops);
 131
 132 static bool entering_platform_hibernation;
 133
 134 bool system_entering_hibernation(void)
 135 {
 136         return entering_platform_hibernation;
 137 }
 138 EXPORT_SYMBOL(system_entering_hibernation);
 139
 140 #ifdef CONFIG_PM_DEBUG
 141 static unsigned int pm_test_delay = 5;
 142 module_param(pm_test_delay, uint, 0644);
 143 MODULE_PARM_DESC(pm_test_delay,
 144                  "Number of seconds to wait before resuming from hibernation test");
 145 static void hibernation_debug_sleep(void)
 146 {
 147         pr_info("hibernation debug: Waiting for %d second(s).\n",
 148                 pm_test_delay);
 149         mdelay(pm_test_delay * 1000);
 150 }
 151
 152 static int hibernation_test(int level)
 153 {
 154         if (pm_test_level == level) {
 155                 hibernation_debug_sleep();
 156                 return 1;
 157         }
 158         return 0;
 159 }
 160 #else /* !CONFIG_PM_DEBUG */
 161 static int hibernation_test(int level) { return 0; }
 162 #endif /* !CONFIG_PM_DEBUG */
 163
 164 /**
 165  * platform_begin - Call platform to start hibernation.
 166  * @platform_mode: Whether or not to use the platform driver.
 167  */
 168 static int platform_begin(int platform_mode)
 169 {
 170         return (platform_mode && hibernation_ops) ?
 171                 hibernation_ops->begin(PMSG_FREEZE) : 0;
 172 }
 173
 174 /**
 175  * platform_end - Call platform to finish transition to the working state.
 176  * @platform_mode: Whether or not to use the platform driver.
 177  */
 178 static void platform_end(int platform_mode)
 179 {
 180         if (platform_mode && hibernation_ops)
 181                 hibernation_ops->end();
 182 }
 183
 184 /**
 185  * platform_pre_snapshot - Call platform to prepare the machine for hibernation.
 186  * @platform_mode: Whether or not to use the platform driver.
 187  *
 188  * Use the platform driver to prepare the system for creating a hibernate image,
 189  * if so configured, and return an error code if that fails.
 190  */
 191
 192 static int platform_pre_snapshot(int platform_mode)
 193 {
 194         return (platform_mode && hibernation_ops) ?
 195                 hibernation_ops->pre_snapshot() : 0;
 196 }
 197
 198 /**
 199  * platform_leave - Call platform to prepare a transition to the working state.
 200  * @platform_mode: Whether or not to use the platform driver.
 201  *
 202  * Use the platform driver prepare to prepare the machine for switching to the
 203  * normal mode of operation.
 204  *
 205  * This routine is called on one CPU with interrupts disabled.
 206  */
 207 static void platform_leave(int platform_mode)
 208 {
 209         if (platform_mode && hibernation_ops)
 210                 hibernation_ops->leave();
 211 }
 212
 213 /**
 214  * platform_finish - Call platform to switch the system to the working state.
 215  * @platform_mode: Whether or not to use the platform driver.
 216  *
 217  * Use the platform driver to switch the machine to the normal mode of
 218  * operation.
 219  *
 220  * This routine must be called after platform_prepare().
 221  */
 222 static void platform_finish(int platform_mode)
 223 {
 224         if (platform_mode && hibernation_ops)
 225                 hibernation_ops->finish();
 226 }
 227
 228 /**
 229  * platform_pre_restore - Prepare for hibernate image restoration.
 230  * @platform_mode: Whether or not to use the platform driver.
 231  *
 232  * Use the platform driver to prepare the system for resume from a hibernation
 233  * image.
 234  *
 235  * If the restore fails after this function has been called,
 236  * platform_restore_cleanup() must be called.
 237  */
 238 static int platform_pre_restore(int platform_mode)
 239 {
 240         return (platform_mode && hibernation_ops) ?
 241                 hibernation_ops->pre_restore() : 0;
 242 }
 243
 244 /**
 245  * platform_restore_cleanup - Switch to the working state after failing restore.
 246  * @platform_mode: Whether or not to use the platform driver.
 247  *
 248  * Use the platform driver to switch the system to the normal mode of operation
 249  * after a failing restore.
 250  *
 251  * If platform_pre_restore() has been called before the failing restore, this
 252  * function must be called too, regardless of the result of
 253  * platform_pre_restore().
 254  */
 255 static void platform_restore_cleanup(int platform_mode)
 256 {
 257         if (platform_mode && hibernation_ops)
 258                 hibernation_ops->restore_cleanup();
 259 }
 260
 261 /**
 262  * platform_recover - Recover from a failure to suspend devices.
 263  * @platform_mode: Whether or not to use the platform driver.
 264  */
 265 static void platform_recover(int platform_mode)
 266 {
 267         if (platform_mode && hibernation_ops && hibernation_ops->recover)
 268                 hibernation_ops->recover();
 269 }
 270
 271 /**
 272  * swsusp_show_speed - Print time elapsed between two events during hibernation.
 273  * @start: Starting event.
 274  * @stop: Final event.
 275  * @nr_pages: Number of memory pages processed between @start and @stop.
 276  * @msg: Additional diagnostic message to print.
 277  */
 278 void swsusp_show_speed(ktime_t start, ktime_t stop,
 279                       unsigned nr_pages, char *msg)
 280 {
 281         ktime_t diff;
 282         u64 elapsed_centisecs64;
 283         unsigned int centisecs;
 284         unsigned int k;
 285         unsigned int kps;
 286
 287         diff = ktime_sub(stop, start);
 288         elapsed_centisecs64 = ktime_divns(diff, 10*NSEC_PER_MSEC);
 289         centisecs = elapsed_centisecs64;
 290         if (centisecs == 0)
 291                 centisecs = 1;  /* avoid div-by-zero */
 292         k = nr_pages * (PAGE_SIZE / 1024);
 293         kps = (k * 100) / centisecs;
 294         pr_info("%s %u kbytes in %u.%02u seconds (%u.%02u MB/s)\n",
 295                 msg, k, centisecs / 100, centisecs % 100, kps / 1000,
 296                 (kps % 1000) / 10);
 297 }
 298
 299 __weak int arch_resume_nosmt(void)
 300 {
 301         return 0;
 302 }
 303
 304 /**
 305  * create_image - Create a hibernation image.
 306  * @platform_mode: Whether or not to use the platform driver.
 307  *
 308  * Execute device drivers' "late" and "noirq" freeze callbacks, create a
 309  * hibernation image and run the drivers' "noirq" and "early" thaw callbacks.
 310  *
 311  * Control reappears in this routine after the subsequent restore.
 312  */
 313 static int create_image(int platform_mode)
 314 {
 315         int error;
 316
 317         error = dpm_suspend_end(PMSG_FREEZE);
 318         if (error) {
 319                 pr_err("Some devices failed to power down, aborting\n");
 320                 return error;
 321         }
 322
 323         error = platform_pre_snapshot(platform_mode);
 324         if (error || hibernation_test(TEST_PLATFORM))
 325                 goto Platform_finish;
 326
 327         error = pm_sleep_disable_secondary_cpus();
 328         if (error || hibernation_test(TEST_CPUS))
 329                 goto Enable_cpus;
 330
 331         local_irq_disable();
 332
 333         system_state = SYSTEM_SUSPEND;
 334
 335         error = syscore_suspend();
 336         if (error) {
 337                 pr_err("Some system devices failed to power down, aborting\n");
 338                 goto Enable_irqs;
 339         }
 340
 341         if (hibernation_test(TEST_CORE) || pm_wakeup_pending())
 342                 goto Power_up;
 343
 344         in_suspend = 1;
 345         save_processor_state();
 346         trace_suspend_resume(TPS("machine_suspend"), PM_EVENT_HIBERNATE, true);
 347         error = swsusp_arch_suspend();
 348         /* Restore control flow magically appears here */
 349         restore_processor_state();
 350         trace_suspend_resume(TPS("machine_suspend"), PM_EVENT_HIBERNATE, false);
 351         if (error)
 352                 pr_err("Error %d creating image\n", error);
 353
 354         if (!in_suspend) {
 355                 events_check_enabled = false;
 356                 clear_or_poison_free_pages();
 357         }
 358
 359         platform_leave(platform_mode);
 360
 361  Power_up:
 362         syscore_resume();
 363
 364  Enable_irqs:
 365         system_state = SYSTEM_RUNNING;
 366         local_irq_enable();
 367
 368  Enable_cpus:
 369         pm_sleep_enable_secondary_cpus();
 370
 371         /* Allow architectures to do nosmt-specific post-resume dances */
 372         if (!in_suspend)
 373                 error = arch_resume_nosmt();
 374
 375  Platform_finish:
 376         platform_finish(platform_mode);
 377
 378         dpm_resume_start(in_suspend ?
 379                 (error ? PMSG_RECOVER : PMSG_THAW) : PMSG_RESTORE);
 380
 381         return error;
 382 }
 383
 384 /**
 385  * hibernation_snapshot - Quiesce devices and create a hibernation image.
 386  * @platform_mode: If set, use platform driver to prepare for the transition.
 387  *
 388  * This routine must be called with system_transition_mutex held.
 389  */
 390 int hibernation_snapshot(int platform_mode)
 391 {
 392         pm_message_t msg;
 393         int error;
 394
 395         pm_suspend_clear_flags();
 396         error = platform_begin(platform_mode);
 397         if (error)
 398                 goto Close;
 399
 400         /* Preallocate image memory before shutting down devices. */
 401         error = hibernate_preallocate_memory();
 402         if (error)
 403                 goto Close;
 404
 405         error = freeze_kernel_threads();
 406         if (error)
 407                 goto Cleanup;
 408
 409         if (hibernation_test(TEST_FREEZER)) {
 410
 411                 /*
 412                  * Indicate to the caller that we are returning due to a
 413                  * successful freezer test.
 414                  */
 415                 freezer_test_done = true;
 416                 goto Thaw;
 417         }
 418
 419         error = dpm_prepare(PMSG_FREEZE);
 420         if (error) {
 421                 dpm_complete(PMSG_RECOVER);
 422                 goto Thaw;
 423         }
 424
 425         console_suspend_all();
 426
 427         error = dpm_suspend(PMSG_FREEZE);
 428
 429         if (error || hibernation_test(TEST_DEVICES))
 430                 platform_recover(platform_mode);
 431         else
 432                 error = create_image(platform_mode);
 433
 434         /*
 435          * In the case that we call create_image() above, the control
 436          * returns here (1) after the image has been created or the
 437          * image creation has failed and (2) after a successful restore.
 438          */
 439
 440         /* We may need to release the preallocated image pages here. */
 441         if (error || !in_suspend)
 442                 swsusp_free();
 443
 444         msg = in_suspend ? (error ? PMSG_RECOVER : PMSG_THAW) : PMSG_RESTORE;
 445         dpm_resume(msg);
 446
 447         if (error || !in_suspend)
 448                 pm_restore_gfp_mask();
 449
 450         console_resume_all();
 451         dpm_complete(msg);
 452
 453  Close:
 454         platform_end(platform_mode);
 455         return error;
 456
 457  Thaw:
 458         thaw_kernel_threads();
 459  Cleanup:
 460         swsusp_free();
 461         goto Close;
 462 }
 463
 464 int __weak hibernate_resume_nonboot_cpu_disable(void)
 465 {
 466         return suspend_disable_secondary_cpus();
 467 }
 468
 469 /**
 470  * resume_target_kernel - Restore system state from a hibernation image.
 471  * @platform_mode: Whether or not to use the platform driver.
 472  *
 473  * Execute device drivers' "noirq" and "late" freeze callbacks, restore the
 474  * contents of highmem that have not been restored yet from the image and run
 475  * the low-level code that will restore the remaining contents of memory and
 476  * switch to the just restored target kernel.
 477  */
 478 static int resume_target_kernel(bool platform_mode)
 479 {
 480         int error;
 481
 482         error = dpm_suspend_end(PMSG_QUIESCE);
 483         if (error) {
 484                 pr_err("Some devices failed to power down, aborting resume\n");
 485                 return error;
 486         }
 487
 488         error = platform_pre_restore(platform_mode);
 489         if (error)
 490                 goto Cleanup;
 491
 492         cpuidle_pause();
 493
 494         error = hibernate_resume_nonboot_cpu_disable();
 495         if (error)
 496                 goto Enable_cpus;
 497
 498         local_irq_disable();
 499         system_state = SYSTEM_SUSPEND;
 500
 501         error = syscore_suspend();
 502         if (error)
 503                 goto Enable_irqs;
 504
 505         save_processor_state();
 506         error = restore_highmem();
 507         if (!error) {
 508                 error = swsusp_arch_resume();
 509                 /*
 510                  * The code below is only ever reached in case of a failure.
 511                  * Otherwise, execution continues at the place where
 512                  * swsusp_arch_suspend() was called.
 513                  */
 514                 BUG_ON(!error);
 515                 /*
 516                  * This call to restore_highmem() reverts the changes made by
 517                  * the previous one.
 518                  */
 519                 restore_highmem();
 520         }
 521         /*
 522          * The only reason why swsusp_arch_resume() can fail is memory being
 523          * very tight, so we have to free it as soon as we can to avoid
 524          * subsequent failures.
 525          */
 526         swsusp_free();
 527         restore_processor_state();
 528         touch_softlockup_watchdog();
 529
 530         syscore_resume();
 531
 532  Enable_irqs:
 533         system_state = SYSTEM_RUNNING;
 534         local_irq_enable();
 535
 536  Enable_cpus:
 537         pm_sleep_enable_secondary_cpus();
 538
 539  Cleanup:
 540         platform_restore_cleanup(platform_mode);
 541
 542         dpm_resume_start(PMSG_RECOVER);
 543
 544         return error;
 545 }
 546
 547 /**
 548  * hibernation_restore - Quiesce devices and restore from a hibernation image.
 549  * @platform_mode: If set, use platform driver to prepare for the transition.
 550  *
 551  * This routine must be called with system_transition_mutex held.  If it is
 552  * successful, control reappears in the restored target kernel in
 553  * hibernation_snapshot().
 554  */
 555 int hibernation_restore(int platform_mode)
 556 {
 557         int error;
 558
 559         pm_prepare_console();
 560         console_suspend_all();
 561         error = dpm_suspend_start(PMSG_QUIESCE);
 562         if (!error) {
 563                 error = resume_target_kernel(platform_mode);
 564                 /*
 565                  * The above should either succeed and jump to the new kernel,
 566                  * or return with an error. Otherwise things are just
 567                  * undefined, so let's be paranoid.
 568                  */
 569                 BUG_ON(!error);
 570         }
 571         dpm_resume_end(PMSG_RECOVER);
 572         console_resume_all();
 573         pm_restore_console();
 574         return error;
 575 }
 576
 577 /**
 578  * hibernation_platform_enter - Power off the system using the platform driver.
 579  */
 580 int hibernation_platform_enter(void)
 581 {
 582         int error;
 583
 584         if (!hibernation_ops)
 585                 return -ENOSYS;
 586
 587         /*
 588          * We have cancelled the power transition by running
 589          * hibernation_ops->finish() before saving the image, so we should let
 590          * the firmware know that we're going to enter the sleep state after all
 591          */
 592         error = hibernation_ops->begin(PMSG_HIBERNATE);
 593         if (error)
 594                 goto Close;
 595
 596         entering_platform_hibernation = true;
 597         console_suspend_all();
 598         error = dpm_suspend_start(PMSG_HIBERNATE);
 599         if (error) {
 600                 if (hibernation_ops->recover)
 601                         hibernation_ops->recover();
 602                 goto Resume_devices;
 603         }
 604
 605         error = dpm_suspend_end(PMSG_HIBERNATE);
 606         if (error)
 607                 goto Resume_devices;
 608
 609         error = hibernation_ops->prepare();
 610         if (error)
 611                 goto Platform_finish;
 612
 613         error = pm_sleep_disable_secondary_cpus();
 614         if (error)
 615                 goto Enable_cpus;
 616
 617         local_irq_disable();
 618         system_state = SYSTEM_SUSPEND;
 619
 620         error = syscore_suspend();
 621         if (error)
 622                 goto Enable_irqs;
 623
 624         if (pm_wakeup_pending()) {
 625                 error = -EAGAIN;
 626                 goto Power_up;
 627         }
 628
 629         hibernation_ops->enter();
 630         /* We should never get here */
 631         while (1);
 632
 633  Power_up:
 634         syscore_resume();
 635  Enable_irqs:
 636         system_state = SYSTEM_RUNNING;
 637         local_irq_enable();
 638
 639  Enable_cpus:
 640         pm_sleep_enable_secondary_cpus();
 641
 642  Platform_finish:
 643         hibernation_ops->finish();
 644
 645         dpm_resume_start(PMSG_RESTORE);
 646
 647  Resume_devices:
 648         entering_platform_hibernation = false;
 649         dpm_resume_end(PMSG_RESTORE);
 650         console_resume_all();
 651
 652  Close:
 653         hibernation_ops->end();
 654
 655         return error;
 656 }
 657
 658 /**
 659  * power_down - Shut the machine down for hibernation.
 660  *
 661  * Use the platform driver, if configured, to put the system into the sleep
 662  * state corresponding to hibernation, or try to power it off or reboot,
 663  * depending on the value of hibernation_mode.
 664  */
 665 static void power_down(void)
 666 {
 667         int error;
 668
 669 #ifdef CONFIG_SUSPEND
 670         if (hibernation_mode == HIBERNATION_SUSPEND) {
 671                 error = suspend_devices_and_enter(mem_sleep_current);
 672                 if (error) {
 673                         hibernation_mode = hibernation_ops ?
 674                                                 HIBERNATION_PLATFORM :
 675                                                 HIBERNATION_SHUTDOWN;
 676                 } else {
 677                         /* Restore swap signature. */
 678                         error = swsusp_unmark();
 679                         if (error)
 680                                 pr_err("Swap will be unusable! Try swapon -a.\n");
 681
 682                         return;
 683                 }
 684         }
 685 #endif
 686
 687         switch (hibernation_mode) {
 688         case HIBERNATION_REBOOT:
 689                 kernel_restart(NULL);
 690                 break;
 691         case HIBERNATION_PLATFORM:
 692                 error = hibernation_platform_enter();
 693                 if (error == -EAGAIN || error == -EBUSY) {
 694                         swsusp_unmark();
 695                         events_check_enabled = false;
 696                         pr_info("Wakeup event detected during hibernation, rolling back.\n");
 697                         return;
 698                 }
 699                 fallthrough;
 700         case HIBERNATION_SHUTDOWN:
 701                 if (kernel_can_power_off()) {
 702                         entering_platform_hibernation = true;
 703                         kernel_power_off();
 704                         entering_platform_hibernation = false;
 705                 }
 706                 break;
 707         }
 708         kernel_halt();
 709         /*
 710          * Valid image is on the disk, if we continue we risk serious data
 711          * corruption after resume.
 712          */
 713         pr_crit("Power down manually\n");
 714         while (1)
 715                 cpu_relax();
 716 }
 717
 718 static int load_image_and_restore(void)
 719 {
 720         int error;
 721         unsigned int flags;
 722
 723         pm_pr_dbg("Loading hibernation image.\n");
 724
 725         lock_device_hotplug();
 726         error = create_basic_memory_bitmaps();
 727         if (error) {
 728                 swsusp_close();
 729                 goto Unlock;
 730         }
 731
 732         error = swsusp_read(&flags);
 733         swsusp_close();
 734         if (!error)
 735                 error = hibernation_restore(flags & SF_PLATFORM_MODE);
 736
 737         pr_err("Failed to load image, recovering.\n");
 738         swsusp_free();
 739         free_basic_memory_bitmaps();
 740  Unlock:
 741         unlock_device_hotplug();
 742
 743         return error;
 744 }
 745
 746 #define COMPRESSION_ALGO_LZO "lzo"
 747 #define COMPRESSION_ALGO_LZ4 "lz4"
 748
 749 /**
 750  * hibernate - Carry out system hibernation, including saving the image.
 751  */
 752 int hibernate(void)
 753 {
 754         bool snapshot_test = false;
 755         unsigned int sleep_flags;
 756         int error;
 757
 758         if (!hibernation_available()) {
 759                 pm_pr_dbg("Hibernation not available.\n");
 760                 return -EPERM;
 761         }
 762
 763         /*
 764          * Query for the compression algorithm support if compression is enabled.
 765          */
 766         if (!nocompress) {
 767                 strscpy(hib_comp_algo, hibernate_compressor);
 768                 if (!crypto_has_acomp(hib_comp_algo, 0, CRYPTO_ALG_ASYNC)) {
 769                         pr_err("%s compression is not available\n", hib_comp_algo);
 770                         return -EOPNOTSUPP;
 771                 }
 772         }
 773
 774         sleep_flags = lock_system_sleep();
 775         /* The snapshot device should not be opened while we're running */
 776         if (!hibernate_acquire()) {
 777                 error = -EBUSY;
 778                 goto Unlock;
 779         }
 780
 781         pr_info("hibernation entry\n");
 782         pm_prepare_console();
 783         error = pm_notifier_call_chain_robust(PM_HIBERNATION_PREPARE, PM_POST_HIBERNATION);
 784         if (error)
 785                 goto Restore;
 786
 787         ksys_sync_helper();
 788         if (filesystem_freeze_enabled)
 789                 filesystems_freeze();
 790
 791         error = freeze_processes();
 792         if (error)
 793                 goto Exit;
 794
 795         lock_device_hotplug();
 796         /* Allocate memory management structures */
 797         error = create_basic_memory_bitmaps();
 798         if (error)
 799                 goto Thaw;
 800
 801         error = hibernation_snapshot(hibernation_mode == HIBERNATION_PLATFORM);
 802         if (error || freezer_test_done)
 803                 goto Free_bitmaps;
 804
 805         if (in_suspend) {
 806                 unsigned int flags = 0;
 807
 808                 if (hibernation_mode == HIBERNATION_PLATFORM)
 809                         flags |= SF_PLATFORM_MODE;
 810                 if (nocompress) {
 811                         flags |= SF_NOCOMPRESS_MODE;
 812                 } else {
 813                         flags |= SF_CRC32_MODE;
 814
 815                         /*
 816                          * By default, LZO compression is enabled. Use SF_COMPRESSION_ALG_LZ4
 817                          * to override this behaviour and use LZ4.
 818                          *
 819                          * Refer kernel/power/power.h for more details
 820                          */
 821
 822                         if (!strcmp(hib_comp_algo, COMPRESSION_ALGO_LZ4))
 823                                 flags |= SF_COMPRESSION_ALG_LZ4;
 824                         else
 825                                 flags |= SF_COMPRESSION_ALG_LZO;
 826                 }
 827
 828                 pm_pr_dbg("Writing hibernation image.\n");
 829                 error = swsusp_write(flags);
 830                 swsusp_free();
 831                 if (!error) {
 832                         if (hibernation_mode == HIBERNATION_TEST_RESUME)
 833                                 snapshot_test = true;
 834                         else
 835                                 power_down();
 836                 }
 837                 in_suspend = 0;
 838                 pm_restore_gfp_mask();
 839         } else {
 840                 pm_pr_dbg("Hibernation image restored successfully.\n");
 841         }
 842
 843  Free_bitmaps:
 844         free_basic_memory_bitmaps();
 845  Thaw:
 846         unlock_device_hotplug();
 847         if (snapshot_test) {
 848                 pm_pr_dbg("Checking hibernation image\n");
 849                 error = swsusp_check(false);
 850                 if (!error)
 851                         error = load_image_and_restore();
 852         }
 853         thaw_processes();
 854
 855         /* Don't bother checking whether freezer_test_done is true */
 856         freezer_test_done = false;
 857  Exit:
 858         filesystems_thaw();
 859         pm_notifier_call_chain(PM_POST_HIBERNATION);
 860  Restore:
 861         pm_restore_console();
 862         hibernate_release();
 863  Unlock:
 864         unlock_system_sleep(sleep_flags);
 865         pr_info("hibernation exit\n");
 866
 867         return error;
 868 }
 869
 870 /**
 871  * hibernate_quiet_exec - Execute a function with all devices frozen.
 872  * @func: Function to execute.
 873  * @data: Data pointer to pass to @func.
 874  *
 875  * Return the @func return value or an error code if it cannot be executed.
 876  */
 877 int hibernate_quiet_exec(int (*func)(void *data), void *data)
 878 {
 879         unsigned int sleep_flags;
 880         int error;
 881
 882         sleep_flags = lock_system_sleep();
 883
 884         if (!hibernate_acquire()) {
 885                 error = -EBUSY;
 886                 goto unlock;
 887         }
 888
 889         pm_prepare_console();
 890
 891         error = pm_notifier_call_chain_robust(PM_HIBERNATION_PREPARE, PM_POST_HIBERNATION);
 892         if (error)
 893                 goto restore;
 894
 895         if (filesystem_freeze_enabled)
 896                 filesystems_freeze();
 897
 898         error = freeze_processes();
 899         if (error)
 900                 goto exit;
 901
 902         lock_device_hotplug();
 903
 904         pm_suspend_clear_flags();
 905
 906         error = platform_begin(true);
 907         if (error)
 908                 goto thaw;
 909
 910         error = freeze_kernel_threads();
 911         if (error)
 912                 goto thaw;
 913
 914         error = dpm_prepare(PMSG_FREEZE);
 915         if (error)
 916                 goto dpm_complete;
 917
 918         console_suspend_all();
 919
 920         error = dpm_suspend(PMSG_FREEZE);
 921         if (error)
 922                 goto dpm_resume;
 923
 924         error = dpm_suspend_end(PMSG_FREEZE);
 925         if (error)
 926                 goto dpm_resume;
 927
 928         error = platform_pre_snapshot(true);
 929         if (error)
 930                 goto skip;
 931
 932         error = func(data);
 933
 934 skip:
 935         platform_finish(true);
 936
 937         dpm_resume_start(PMSG_THAW);
 938
 939 dpm_resume:
 940         dpm_resume(PMSG_THAW);
 941
 942         console_resume_all();
 943
 944 dpm_complete:
 945         dpm_complete(PMSG_THAW);
 946
 947         thaw_kernel_threads();
 948
 949 thaw:
 950         platform_end(true);
 951
 952         unlock_device_hotplug();
 953
 954         thaw_processes();
 955
 956 exit:
 957         filesystems_thaw();
 958         pm_notifier_call_chain(PM_POST_HIBERNATION);
 959
 960 restore:
 961         pm_restore_console();
 962
 963         hibernate_release();
 964
 965 unlock:
 966         unlock_system_sleep(sleep_flags);
 967
 968         return error;
 969 }
 970 EXPORT_SYMBOL_GPL(hibernate_quiet_exec);
 971
 972 static int __init find_resume_device(void)
 973 {
 974         if (!strlen(resume_file))
 975                 return -ENOENT;
 976
 977         pm_pr_dbg("Checking hibernation image partition %s\n", resume_file);
 978
 979         if (resume_delay) {
 980                 pr_info("Waiting %dsec before reading resume device ...\n",
 981                         resume_delay);
 982                 ssleep(resume_delay);
 983         }
 984
 985         /* Check if the device is there */
 986         if (!early_lookup_bdev(resume_file, &swsusp_resume_device))
 987                 return 0;
 988
 989         /*
 990          * Some device discovery might still be in progress; we need to wait for
 991          * this to finish.
 992          */
 993         wait_for_device_probe();
 994         if (resume_wait) {
 995                 while (early_lookup_bdev(resume_file, &swsusp_resume_device))
 996                         msleep(10);
 997                 async_synchronize_full();
 998         }
 999
1000         return early_lookup_bdev(resume_file, &swsusp_resume_device);
1001 }
1002
1003 static int software_resume(void)
1004 {
1005         int error;
1006
1007         pm_pr_dbg("Hibernation image partition %d:%d present\n",
1008                 MAJOR(swsusp_resume_device), MINOR(swsusp_resume_device));
1009
1010         pm_pr_dbg("Looking for hibernation image.\n");
1011
1012         mutex_lock(&system_transition_mutex);
1013         error = swsusp_check(true);
1014         if (error)
1015                 goto Unlock;
1016
1017         /*
1018          * Check if the hibernation image is compressed. If so, query for
1019          * the algorithm support.
1020          */
1021         if (!(swsusp_header_flags & SF_NOCOMPRESS_MODE)) {
1022                 if (swsusp_header_flags & SF_COMPRESSION_ALG_LZ4)
1023                         strscpy(hib_comp_algo, COMPRESSION_ALGO_LZ4);
1024                 else
1025                         strscpy(hib_comp_algo, COMPRESSION_ALGO_LZO);
1026                 if (!crypto_has_acomp(hib_comp_algo, 0, CRYPTO_ALG_ASYNC)) {
1027                         pr_err("%s compression is not available\n", hib_comp_algo);
1028                         error = -EOPNOTSUPP;
1029                         goto Unlock;
1030                 }
1031         }
1032
1033         /* The snapshot device should not be opened while we're running */
1034         if (!hibernate_acquire()) {
1035                 error = -EBUSY;
1036                 swsusp_close();
1037                 goto Unlock;
1038         }
1039
1040         pr_info("resume from hibernation\n");
1041         pm_prepare_console();
1042         error = pm_notifier_call_chain_robust(PM_RESTORE_PREPARE, PM_POST_RESTORE);
1043         if (error)
1044                 goto Restore;
1045
1046         if (filesystem_freeze_enabled)
1047                 filesystems_freeze();
1048
1049         pm_pr_dbg("Preparing processes for hibernation restore.\n");
1050         error = freeze_processes();
1051         if (error) {
1052                 filesystems_thaw();
1053                 goto Close_Finish;
1054         }
1055
1056         error = freeze_kernel_threads();
1057         if (error) {
1058                 thaw_processes();
1059                 filesystems_thaw();
1060                 goto Close_Finish;
1061         }
1062
1063         error = load_image_and_restore();
1064         thaw_processes();
1065         filesystems_thaw();
1066  Finish:
1067         pm_notifier_call_chain(PM_POST_RESTORE);
1068  Restore:
1069         pm_restore_console();
1070         pr_info("resume failed (%d)\n", error);
1071         hibernate_release();
1072         /* For success case, the suspend path will release the lock */
1073  Unlock:
1074         mutex_unlock(&system_transition_mutex);
1075         pm_pr_dbg("Hibernation image not present or could not be loaded.\n");
1076         return error;
1077  Close_Finish:
1078         swsusp_close();
1079         goto Finish;
1080 }
1081
1082 /**
1083  * software_resume_initcall - Resume from a saved hibernation image.
1084  *
1085  * This routine is called as a late initcall, when all devices have been
1086  * discovered and initialized already.
1087  *
1088  * The image reading code is called to see if there is a hibernation image
1089  * available for reading.  If that is the case, devices are quiesced and the
1090  * contents of memory is restored from the saved image.
1091  *
1092  * If this is successful, control reappears in the restored target kernel in
1093  * hibernation_snapshot() which returns to hibernate().  Otherwise, the routine
1094  * attempts to recover gracefully and make the kernel return to the normal mode
1095  * of operation.
1096  */
1097 static int __init software_resume_initcall(void)
1098 {
1099         /*
1100          * If the user said "noresume".. bail out early.
1101          */
1102         if (noresume || !hibernation_available())
1103                 return 0;
1104
1105         if (!swsusp_resume_device) {
1106                 int error = find_resume_device();
1107
1108                 if (error)
1109                         return error;
1110         }
1111
1112         return software_resume();
1113 }
1114 late_initcall_sync(software_resume_initcall);
1115
1116
1117 static const char * const hibernation_modes[] = {
1118         [HIBERNATION_PLATFORM]  = "platform",
1119         [HIBERNATION_SHUTDOWN]  = "shutdown",
1120         [HIBERNATION_REBOOT]    = "reboot",
1121 #ifdef CONFIG_SUSPEND
1122         [HIBERNATION_SUSPEND]   = "suspend",
1123 #endif
1124         [HIBERNATION_TEST_RESUME]       = "test_resume",
1125 };
1126
1127 /*
1128  * /sys/power/disk - Control hibernation mode.
1129  *
1130  * Hibernation can be handled in several ways.  There are a few different ways
1131  * to put the system into the sleep state: using the platform driver (e.g. ACPI
1132  * or other hibernation_ops), powering it off or rebooting it (for testing
1133  * mostly).
1134  *
1135  * The sysfs file /sys/power/disk provides an interface for selecting the
1136  * hibernation mode to use.  Reading from this file causes the available modes
1137  * to be printed.  There are 3 modes that can be supported:
1138  *
1139  *      'platform'
1140  *      'shutdown'
1141  *      'reboot'
1142  *
1143  * If a platform hibernation driver is in use, 'platform' will be supported
1144  * and will be used by default.  Otherwise, 'shutdown' will be used by default.
1145  * The selected option (i.e. the one corresponding to the current value of
1146  * hibernation_mode) is enclosed by a square bracket.
1147  *
1148  * To select a given hibernation mode it is necessary to write the mode's
1149  * string representation (as returned by reading from /sys/power/disk) back
1150  * into /sys/power/disk.
1151  */
1152
1153 static ssize_t disk_show(struct kobject *kobj, struct kobj_attribute *attr,
1154                          char *buf)
1155 {
1156         ssize_t count = 0;
1157         int i;
1158
1159         if (!hibernation_available())
1160                 return sysfs_emit(buf, "[disabled]\n");
1161
1162         for (i = HIBERNATION_FIRST; i <= HIBERNATION_MAX; i++) {
1163                 if (!hibernation_modes[i])
1164                         continue;
1165                 switch (i) {
1166                 case HIBERNATION_SHUTDOWN:
1167                 case HIBERNATION_REBOOT:
1168 #ifdef CONFIG_SUSPEND
1169                 case HIBERNATION_SUSPEND:
1170 #endif
1171                 case HIBERNATION_TEST_RESUME:
1172                         break;
1173                 case HIBERNATION_PLATFORM:
1174                         if (hibernation_ops)
1175                                 break;
1176                         /* not a valid mode, continue with loop */
1177                         continue;
1178                 }
1179                 if (i == hibernation_mode)
1180                         count += sysfs_emit_at(buf, count, "[%s] ", hibernation_modes[i]);
1181                 else
1182                         count += sysfs_emit_at(buf, count, "%s ", hibernation_modes[i]);
1183         }
1184
1185         /* Convert the last space to a newline if needed. */
1186         if (count > 0)
1187                 buf[count - 1] = '\n';
1188
1189         return count;
1190 }
1191
1192 static ssize_t disk_store(struct kobject *kobj, struct kobj_attribute *attr,
1193                           const char *buf, size_t n)
1194 {
1195         int mode = HIBERNATION_INVALID;
1196         unsigned int sleep_flags;
1197         int error = 0;
1198         int len;
1199         char *p;
1200         int i;
1201
1202         if (!hibernation_available())
1203                 return -EPERM;
1204
1205         p = memchr(buf, '\n', n);
1206         len = p ? p - buf : n;
1207
1208         sleep_flags = lock_system_sleep();
1209         for (i = HIBERNATION_FIRST; i <= HIBERNATION_MAX; i++) {
1210                 if (len == strlen(hibernation_modes[i])
1211                     && !strncmp(buf, hibernation_modes[i], len)) {
1212                         mode = i;
1213                         break;
1214                 }
1215         }
1216         if (mode != HIBERNATION_INVALID) {
1217                 switch (mode) {
1218                 case HIBERNATION_SHUTDOWN:
1219                 case HIBERNATION_REBOOT:
1220 #ifdef CONFIG_SUSPEND
1221                 case HIBERNATION_SUSPEND:
1222 #endif
1223                 case HIBERNATION_TEST_RESUME:
1224                         hibernation_mode = mode;
1225                         break;
1226                 case HIBERNATION_PLATFORM:
1227                         if (hibernation_ops)
1228                                 hibernation_mode = mode;
1229                         else
1230                                 error = -EINVAL;
1231                 }
1232         } else
1233                 error = -EINVAL;
1234
1235         if (!error)
1236                 pm_pr_dbg("Hibernation mode set to '%s'\n",
1237                                hibernation_modes[mode]);
1238         unlock_system_sleep(sleep_flags);
1239         return error ? error : n;
1240 }
1241
1242 power_attr(disk);
1243
1244 static ssize_t resume_show(struct kobject *kobj, struct kobj_attribute *attr,
1245                            char *buf)
1246 {
1247         return sysfs_emit(buf, "%d:%d\n", MAJOR(swsusp_resume_device),
1248                           MINOR(swsusp_resume_device));
1249 }
1250
1251 static ssize_t resume_store(struct kobject *kobj, struct kobj_attribute *attr,
1252                             const char *buf, size_t n)
1253 {
1254         unsigned int sleep_flags;
1255         int len = n;
1256         char *name;
1257         dev_t dev;
1258         int error;
1259
1260         if (!hibernation_available())
1261                 return n;
1262
1263         if (len && buf[len-1] == '\n')
1264                 len--;
1265         name = kstrndup(buf, len, GFP_KERNEL);
1266         if (!name)
1267                 return -ENOMEM;
1268
1269         error = lookup_bdev(name, &dev);
1270         if (error) {
1271                 unsigned maj, min, offset;
1272                 char *p, dummy;
1273
1274                 error = 0;
1275                 if (sscanf(name, "%u:%u%c", &maj, &min, &dummy) == 2 ||
1276                     sscanf(name, "%u:%u:%u:%c", &maj, &min, &offset,
1277                                 &dummy) == 3) {
1278                         dev = MKDEV(maj, min);
1279                         if (maj != MAJOR(dev) || min != MINOR(dev))
1280                                 error = -EINVAL;
1281                 } else {
1282                         dev = new_decode_dev(simple_strtoul(name, &p, 16));
1283                         if (*p)
1284                                 error = -EINVAL;
1285                 }
1286         }
1287         kfree(name);
1288         if (error)
1289                 return error;
1290
1291         sleep_flags = lock_system_sleep();
1292         swsusp_resume_device = dev;
1293         unlock_system_sleep(sleep_flags);
1294
1295         pm_pr_dbg("Configured hibernation resume from disk to %u\n",
1296                   swsusp_resume_device);
1297         noresume = 0;
1298         software_resume();
1299         return n;
1300 }
1301
1302 power_attr(resume);
1303
1304 static ssize_t resume_offset_show(struct kobject *kobj,
1305                                   struct kobj_attribute *attr, char *buf)
1306 {
1307         return sysfs_emit(buf, "%llu\n", (unsigned long long)swsusp_resume_block);
1308 }
1309
1310 static ssize_t resume_offset_store(struct kobject *kobj,
1311                                    struct kobj_attribute *attr, const char *buf,
1312                                    size_t n)
1313 {
1314         unsigned long long offset;
1315         int rc;
1316
1317         rc = kstrtoull(buf, 0, &offset);
1318         if (rc)
1319                 return rc;
1320         swsusp_resume_block = offset;
1321
1322         return n;
1323 }
1324
1325 power_attr(resume_offset);
1326
1327 static ssize_t image_size_show(struct kobject *kobj, struct kobj_attribute *attr,
1328                                char *buf)
1329 {
1330         return sysfs_emit(buf, "%lu\n", image_size);
1331 }
1332
1333 static ssize_t image_size_store(struct kobject *kobj, struct kobj_attribute *attr,
1334                                 const char *buf, size_t n)
1335 {
1336         unsigned long size;
1337
1338         if (sscanf(buf, "%lu", &size) == 1) {
1339                 image_size = size;
1340                 return n;
1341         }
1342
1343         return -EINVAL;
1344 }
1345
1346 power_attr(image_size);
1347
1348 static ssize_t reserved_size_show(struct kobject *kobj,
1349                                   struct kobj_attribute *attr, char *buf)
1350 {
1351         return sysfs_emit(buf, "%lu\n", reserved_size);
1352 }
1353
1354 static ssize_t reserved_size_store(struct kobject *kobj,
1355                                    struct kobj_attribute *attr,
1356                                    const char *buf, size_t n)
1357 {
1358         unsigned long size;
1359
1360         if (sscanf(buf, "%lu", &size) == 1) {
1361                 reserved_size = size;
1362                 return n;
1363         }
1364
1365         return -EINVAL;
1366 }
1367
1368 power_attr(reserved_size);
1369
1370 static struct attribute *g[] = {
1371         &disk_attr.attr,
1372         &resume_offset_attr.attr,
1373         &resume_attr.attr,
1374         &image_size_attr.attr,
1375         &reserved_size_attr.attr,
1376         NULL,
1377 };
1378
1379
1380 static const struct attribute_group attr_group = {
1381         .attrs = g,
1382 };
1383
1384
1385 static int __init pm_disk_init(void)
1386 {
1387         return sysfs_create_group(power_kobj, &attr_group);
1388 }
1389
1390 core_initcall(pm_disk_init);
1391
1392
1393 static int __init resume_setup(char *str)
1394 {
1395         if (noresume)
1396                 return 1;
1397
1398         strscpy(resume_file, str);
1399         return 1;
1400 }
1401
1402 static int __init resume_offset_setup(char *str)
1403 {
1404         unsigned long long offset;
1405
1406         if (noresume)
1407                 return 1;
1408
1409         if (sscanf(str, "%llu", &offset) == 1)
1410                 swsusp_resume_block = offset;
1411
1412         return 1;
1413 }
1414
1415 static int __init hibernate_setup(char *str)
1416 {
1417         if (!strncmp(str, "noresume", 8)) {
1418                 noresume = 1;
1419         } else if (!strncmp(str, "nocompress", 10)) {
1420                 nocompress = 1;
1421         } else if (!strncmp(str, "no", 2)) {
1422                 noresume = 1;
1423                 nohibernate = 1;
1424         } else if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)
1425                    && !strncmp(str, "protect_image", 13)) {
1426                 enable_restore_image_protection();
1427         }
1428         return 1;
1429 }
1430
1431 static int __init noresume_setup(char *str)
1432 {
1433         noresume = 1;
1434         return 1;
1435 }
1436
1437 static int __init resumewait_setup(char *str)
1438 {
1439         resume_wait = 1;
1440         return 1;
1441 }
1442
1443 static int __init resumedelay_setup(char *str)
1444 {
1445         int rc = kstrtouint(str, 0, &resume_delay);
1446
1447         if (rc)
1448                 pr_warn("resumedelay: bad option string '%s'\n", str);
1449         return 1;
1450 }
1451
1452 static int __init nohibernate_setup(char *str)
1453 {
1454         noresume = 1;
1455         nohibernate = 1;
1456         return 1;
1457 }
1458
1459 static const char * const comp_alg_enabled[] = {
1460 #if IS_ENABLED(CONFIG_CRYPTO_LZO)
1461         COMPRESSION_ALGO_LZO,
1462 #endif
1463 #if IS_ENABLED(CONFIG_CRYPTO_LZ4)
1464         COMPRESSION_ALGO_LZ4,
1465 #endif
1466 };
1467
1468 static int hibernate_compressor_param_set(const char *compressor,
1469                 const struct kernel_param *kp)
1470 {
1471         int index, ret;
1472
1473         if (!mutex_trylock(&system_transition_mutex))
1474                 return -EBUSY;
1475
1476         index = sysfs_match_string(comp_alg_enabled, compressor);
1477         if (index >= 0) {
1478                 ret = param_set_copystring(comp_alg_enabled[index], kp);
1479                 if (!ret)
1480                         strscpy(hib_comp_algo, comp_alg_enabled[index]);
1481         } else {
1482                 ret = index;
1483         }
1484
1485         mutex_unlock(&system_transition_mutex);
1486
1487         if (ret)
1488                 pr_debug("Cannot set specified compressor %s\n",
1489                          compressor);
1490
1491         return ret;
1492 }
1493
1494 static const struct kernel_param_ops hibernate_compressor_param_ops = {
1495         .set    = hibernate_compressor_param_set,
1496         .get    = param_get_string,
1497 };
1498
1499 static struct kparam_string hibernate_compressor_param_string = {
1500         .maxlen = sizeof(hibernate_compressor),
1501         .string = hibernate_compressor,
1502 };
1503
1504 module_param_cb(compressor, &hibernate_compressor_param_ops,
1505                 &hibernate_compressor_param_string, 0644);
1506 MODULE_PARM_DESC(compressor,
1507                  "Compression algorithm to be used with hibernation");
1508
1509 __setup("noresume", noresume_setup);
1510 __setup("resume_offset=", resume_offset_setup);
1511 __setup("resume=", resume_setup);
1512 __setup("hibernate=", hibernate_setup);
1513 __setup("resumewait", resumewait_setup);
1514 __setup("resumedelay=", resumedelay_setup);
1515 __setup("nohibernate", nohibernate_setup);