arch, ftrace: for KASAN put hard/soft IRQ entries into separate sections

KASAN needs to know whether the allocation happens in an IRQ handler.
This lets us strip everything below the IRQ entry point to reduce the
number of unique stack traces needed to be stored.

Move the definition of __irq_entry to <linux/interrupt.h> so that the
users don't need to pull in <linux/ftrace.h>.  Also introduce the
__softirq_entry macro which is similar to __irq_entry, but puts the
corresponding functions to the .softirqentry.text section.

Signed-off-by: Alexander Potapenko <glider@google.com>
Acked-by: Steven Rostedt <rostedt@goodmis.org>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Andrey Konovalov <adech.fo@gmail.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com>
Cc: Konstantin Serebryany <kcc@google.com>
Cc: Dmitry Chernenkov <dmitryc@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/arch/arm/include/asm/exception.h b/arch/arm/include/asm/exception.h
index 5abaf5bbd985f1049c65e6368c3e868399bb609e..bf1991263d2df2128f4564aaee76d1c1560fe141 100644 (file)
--- a/arch/arm/include/asm/exception.h
+++ b/arch/arm/include/asm/exception.h
@@ -7,7 +7,7 @@
 #ifndef __ASM_ARM_EXCEPTION_H
 #define __ASM_ARM_EXCEPTION_H
 
-#include <linux/ftrace.h>
+#include <linux/interrupt.h>
 
 #define __exception    __attribute__((section(".exception.text")))
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER

diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S
index 1fab979daeaf1c490b1215b44190bb498157124b..e2c6da096cefa491669ae8b4464b89f6df5f0a75 100644 (file)
--- a/arch/arm/kernel/vmlinux.lds.S
+++ b/arch/arm/kernel/vmlinux.lds.S
@@ -108,6 +108,7 @@ SECTIONS
                        *(.exception.text)
                        __exception_text_end = .;
                        IRQENTRY_TEXT
+                       SOFTIRQENTRY_TEXT
                        TEXT_TEXT
                        SCHED_TEXT
                        LOCK_TEXT

diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h
index 6cb7e1a6bc02a54237d7475ae20406039b14ed4b..0c2eec490abf65933b2d965270dfaf89523f3417 100644 (file)
--- a/arch/arm64/include/asm/exception.h
+++ b/arch/arm64/include/asm/exception.h
@@ -18,7 +18,7 @@
 #ifndef __ASM_EXCEPTION_H
 #define __ASM_EXCEPTION_H
 
-#include <linux/ftrace.h>
+#include <linux/interrupt.h>
 
 #define __exception    __attribute__((section(".exception.text")))
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER

diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 37f624df68fad01dad3c9b0be798602f14800f03..5a1939a74ff3588bb388c73741499d6061594a5d 100644 (file)
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -103,6 +103,7 @@ SECTIONS
                        *(.exception.text)
                        __exception_text_end = .;
                        IRQENTRY_TEXT
+                       SOFTIRQENTRY_TEXT
                        TEXT_TEXT
                        SCHED_TEXT
                        LOCK_TEXT

diff --git a/arch/blackfin/kernel/vmlinux.lds.S b/arch/blackfin/kernel/vmlinux.lds.S
index c9eec84aa258628d1c9d70fc545420dde30c8bc1..d920b959ff3a4b8cb3d6a28d67d40ebf6f7c457b 100644 (file)
--- a/arch/blackfin/kernel/vmlinux.lds.S
+++ b/arch/blackfin/kernel/vmlinux.lds.S
@@ -35,6 +35,7 @@ SECTIONS
 #endif
                LOCK_TEXT
                IRQENTRY_TEXT
+               SOFTIRQENTRY_TEXT
                KPROBES_TEXT
 #ifdef CONFIG_ROMKERNEL
                __sinittext = .;

diff --git a/arch/c6x/kernel/vmlinux.lds.S b/arch/c6x/kernel/vmlinux.lds.S
index 5a6e141d1641bced3bcabc94e66849bd337cbe3d..50bc10f97bcb4a234bff525d9e120842aa663c99 100644 (file)
--- a/arch/c6x/kernel/vmlinux.lds.S
+++ b/arch/c6x/kernel/vmlinux.lds.S
@@ -72,6 +72,7 @@ SECTIONS
                SCHED_TEXT
                LOCK_TEXT
                IRQENTRY_TEXT
+               SOFTIRQENTRY_TEXT
                KPROBES_TEXT
                *(.fixup)
                *(.gnu.warning)

diff --git a/arch/metag/kernel/vmlinux.lds.S b/arch/metag/kernel/vmlinux.lds.S
index e12055e88bfe5e55b6092dbf3cf95dd63eb08cdd..150ace92c7ade9046c9c22b114a979356e7395ac 100644 (file)
--- a/arch/metag/kernel/vmlinux.lds.S
+++ b/arch/metag/kernel/vmlinux.lds.S
@@ -24,6 +24,7 @@ SECTIONS
        LOCK_TEXT
        KPROBES_TEXT
        IRQENTRY_TEXT
+       SOFTIRQENTRY_TEXT
        *(.text.*)
        *(.gnu.warning)
        }

diff --git a/arch/microblaze/kernel/vmlinux.lds.S b/arch/microblaze/kernel/vmlinux.lds.S
index be9488d697347c330f9b226d1a5a5c9ba2d97714..0a47f041055416a90d34a64be3208e3fd74b2b6c 100644 (file)
--- a/arch/microblaze/kernel/vmlinux.lds.S
+++ b/arch/microblaze/kernel/vmlinux.lds.S
@@ -36,6 +36,7 @@ SECTIONS {
                LOCK_TEXT
                KPROBES_TEXT
                IRQENTRY_TEXT
+               SOFTIRQENTRY_TEXT
                . = ALIGN (4) ;
                _etext = . ;
        }

diff --git a/arch/mips/kernel/vmlinux.lds.S b/arch/mips/kernel/vmlinux.lds.S
index 0a93e83cd01479a4abc952e2e41a853dbf4159b9..54d653ee17e1a8f3e2887bf76b1849fcaf73c27d 100644 (file)
--- a/arch/mips/kernel/vmlinux.lds.S
+++ b/arch/mips/kernel/vmlinux.lds.S
@@ -58,6 +58,7 @@ SECTIONS
                LOCK_TEXT
                KPROBES_TEXT
                IRQENTRY_TEXT
+               SOFTIRQENTRY_TEXT
                *(.text.*)
                *(.fixup)
                *(.gnu.warning)

diff --git a/arch/nios2/kernel/vmlinux.lds.S b/arch/nios2/kernel/vmlinux.lds.S
index 326fab40a9de0c05cff1649b2080c4e8e493a2f4..e23e89539967713c301c4f452f802f878bb7562f 100644 (file)
--- a/arch/nios2/kernel/vmlinux.lds.S
+++ b/arch/nios2/kernel/vmlinux.lds.S
@@ -39,6 +39,7 @@ SECTIONS
                SCHED_TEXT
                LOCK_TEXT
                IRQENTRY_TEXT
+               SOFTIRQENTRY_TEXT
                KPROBES_TEXT
        } =0
        _etext = .;

diff --git a/arch/openrisc/kernel/vmlinux.lds.S b/arch/openrisc/kernel/vmlinux.lds.S
index 2d69a853b742e9ff3d7608b2b5ecfda1cf780941..d936de4c07cadd73f00a0700a3781243b8f7852c 100644 (file)
--- a/arch/openrisc/kernel/vmlinux.lds.S
+++ b/arch/openrisc/kernel/vmlinux.lds.S
@@ -50,6 +50,7 @@ SECTIONS
          LOCK_TEXT
          KPROBES_TEXT
          IRQENTRY_TEXT
+         SOFTIRQENTRY_TEXT
          *(.fixup)
          *(.text.__*)
          _etext = .;

diff --git a/arch/parisc/kernel/vmlinux.lds.S b/arch/parisc/kernel/vmlinux.lds.S
index 308f29081d461b0a9b0a0d8eec0d54e0a1936ff3..f3ead0b6ce461501508c351fdc3246f67f18e218 100644 (file)
--- a/arch/parisc/kernel/vmlinux.lds.S
+++ b/arch/parisc/kernel/vmlinux.lds.S
@@ -72,6 +72,7 @@ SECTIONS
                LOCK_TEXT
                KPROBES_TEXT
                IRQENTRY_TEXT
+               SOFTIRQENTRY_TEXT
                *(.text.do_softirq)
                *(.text.sys_exit)
                *(.text.do_sigaltstack)

diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S
index d41fd0af89807c5b3b5bb4e3e713bd09d4b94eaf..2dd91f79de05a729f2aca86f4467365dba4d35de 100644 (file)
--- a/arch/powerpc/kernel/vmlinux.lds.S
+++ b/arch/powerpc/kernel/vmlinux.lds.S
@@ -55,6 +55,7 @@ SECTIONS
                LOCK_TEXT
                KPROBES_TEXT
                IRQENTRY_TEXT
+               SOFTIRQENTRY_TEXT
 
 #ifdef CONFIG_PPC32
                *(.got1)

diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S
index 445657fe658cc62b4308f7fda4a23360dadfbb26..0f41a8286378503a3708d78d79255ef41bde65fe 100644 (file)
--- a/arch/s390/kernel/vmlinux.lds.S
+++ b/arch/s390/kernel/vmlinux.lds.S
@@ -28,6 +28,7 @@ SECTIONS
                LOCK_TEXT
                KPROBES_TEXT
                IRQENTRY_TEXT
+               SOFTIRQENTRY_TEXT
                *(.fixup)
                *(.gnu.warning)
        } :text = 0x0700

diff --git a/arch/sh/kernel/vmlinux.lds.S b/arch/sh/kernel/vmlinux.lds.S
index db88cbf9eafdc815d76d72784c18e4df42b361e5..235a4101999fe5a4462bd5ec1d78cba49eb49b90 100644 (file)
--- a/arch/sh/kernel/vmlinux.lds.S
+++ b/arch/sh/kernel/vmlinux.lds.S
@@ -39,6 +39,7 @@ SECTIONS
                LOCK_TEXT
                KPROBES_TEXT
                IRQENTRY_TEXT
+               SOFTIRQENTRY_TEXT
                *(.fixup)
                *(.gnu.warning)
                _etext = .;             /* End of text section */

diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S
index f1a2f688b28a31fc47d2232f3ed10e9d95930223..aadd321aa05db983af75a13ee14f2693ad75ed5c 100644 (file)
--- a/arch/sparc/kernel/vmlinux.lds.S
+++ b/arch/sparc/kernel/vmlinux.lds.S
@@ -48,6 +48,7 @@ SECTIONS
                LOCK_TEXT
                KPROBES_TEXT
                IRQENTRY_TEXT
+               SOFTIRQENTRY_TEXT
                *(.gnu.warning)
        } = 0
        _etext = .;

diff --git a/arch/tile/kernel/vmlinux.lds.S b/arch/tile/kernel/vmlinux.lds.S
index 0e059a0101ea3097527968bd2632a2b91436bb0d..378f5d8d1ec8aa8ca77cd34848e34ea3d962dc5b 100644 (file)
--- a/arch/tile/kernel/vmlinux.lds.S
+++ b/arch/tile/kernel/vmlinux.lds.S
@@ -45,6 +45,7 @@ SECTIONS
     LOCK_TEXT
     KPROBES_TEXT
     IRQENTRY_TEXT
+    SOFTIRQENTRY_TEXT
     __fix_text_end = .;   /* tile-cpack won't rearrange before this */
     ALIGN_FUNCTION();
     *(.hottext*)

diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index d239639e0c1dd4f31649d05a150068152ad7835a..4c941f88d405fbe339d9194e845d9d748fd72604 100644 (file)
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -101,6 +101,7 @@ SECTIONS
                KPROBES_TEXT
                ENTRY_TEXT
                IRQENTRY_TEXT
+               SOFTIRQENTRY_TEXT
                *(.fixup)
                *(.gnu.warning)
                /* End of text section */

diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index 8f5a12ab2f2b9a00947add5f325fafc67f6e918a..339125bb4d2cf919669677c5d8edc900157bcc50 100644 (file)
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -456,7 +456,7 @@
                *(.entry.text)                                          \
                VMLINUX_SYMBOL(__entry_text_end) = .;
 
-#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+#if defined(CONFIG_FUNCTION_GRAPH_TRACER) || defined(CONFIG_KASAN)
 #define IRQENTRY_TEXT                                                  \
                ALIGN_FUNCTION();                                       \
                VMLINUX_SYMBOL(__irqentry_text_start) = .;              \
@@ -466,6 +466,16 @@
 #define IRQENTRY_TEXT
 #endif
 
+#if defined(CONFIG_FUNCTION_GRAPH_TRACER) || defined(CONFIG_KASAN)
+#define SOFTIRQENTRY_TEXT                                              \
+               ALIGN_FUNCTION();                                       \
+               VMLINUX_SYMBOL(__softirqentry_text_start) = .;          \
+               *(.softirqentry.text)                                   \
+               VMLINUX_SYMBOL(__softirqentry_text_end) = .;
+#else
+#define SOFTIRQENTRY_TEXT
+#endif
+
 /* Section used for early init (in .S files) */
 #define HEAD_TEXT  *(.head.text)
 

diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index 6d9df3f7e334c3a942728237dca471f7bb602f98..dea12a6e413bcf49711fa101ca22abed1f446d78 100644 (file)
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -811,16 +811,6 @@ ftrace_push_return_trace(unsigned long ret, unsigned long func, int *depth,
  */
 #define __notrace_funcgraph            notrace
 
-/*
- * We want to which function is an entrypoint of a hardirq.
- * That will help us to put a signal on output.
- */
-#define __irq_entry             __attribute__((__section__(".irqentry.text")))
-
-/* Limits of hardirq entrypoints */
-extern char __irqentry_text_start[];
-extern char __irqentry_text_end[];
-
 #define FTRACE_NOTRACE_DEPTH 65536
 #define FTRACE_RETFUNC_DEPTH 50
 #define FTRACE_RETSTACK_ALLOC_SIZE 32
@@ -857,7 +847,6 @@ static inline void unpause_graph_tracing(void)
 #else /* !CONFIG_FUNCTION_GRAPH_TRACER */
 
 #define __notrace_funcgraph
-#define __irq_entry
 #define INIT_FTRACE_GRAPH
 
 static inline void ftrace_graph_init_task(struct task_struct *t) { }

diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 358076eda364ec929d5a5181c4c37663e4d78c40..9fcabeb07787b75cef51e3c9118754b29ff41e03 100644 (file)
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -683,4 +683,24 @@ extern int early_irq_init(void);
 extern int arch_probe_nr_irqs(void);
 extern int arch_early_irq_init(void);
 
+#if defined(CONFIG_FUNCTION_GRAPH_TRACER) || defined(CONFIG_KASAN)
+/*
+ * We want to know which function is an entrypoint of a hardirq or a softirq.
+ */
+#define __irq_entry             __attribute__((__section__(".irqentry.text")))
+#define __softirq_entry  \
+       __attribute__((__section__(".softirqentry.text")))
+
+/* Limits of hardirq entrypoints */
+extern char __irqentry_text_start[];
+extern char __irqentry_text_end[];
+/* Limits of softirq entrypoints */
+extern char __softirqentry_text_start[];
+extern char __softirqentry_text_end[];
+
+#else
+#define __irq_entry
+#define __softirq_entry
+#endif
+
 #endif

diff --git a/kernel/softirq.c b/kernel/softirq.c
index 8aae49dd7da862955f9ec49c65b0d80ad8effcd6..17caf4b63342d7839528f367b283a386413b0362 100644 (file)
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -227,7 +227,7 @@ static inline bool lockdep_softirq_start(void) { return false; }
 static inline void lockdep_softirq_end(bool in_hardirq) { }
 #endif
 
-asmlinkage __visible void __do_softirq(void)
+asmlinkage __visible void __softirq_entry __do_softirq(void)
 {
        unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
        unsigned long old_flags = current->flags;

diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index 91d6a63a2ea7601feab8b09f5ef0c7da28eb4da3..3a0244ff7ea8df27680c1eeca9caf177d8e083bd 100644 (file)
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -8,6 +8,7 @@
  */
 #include <linux/uaccess.h>
 #include <linux/ftrace.h>
+#include <linux/interrupt.h>
 #include <linux/slab.h>
 #include <linux/fs.h>