[PATCH] reduce sizeof(struct file)
[linux-2.6-block.git] / fs / exec.c
CommitLineData
1da177e4
LT
1/*
2 * linux/fs/exec.c
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 */
6
7/*
8 * #!-checking implemented by tytso.
9 */
10/*
11 * Demand-loading implemented 01.12.91 - no need to read anything but
12 * the header into memory. The inode of the executable is put into
13 * "current->executable", and page faults do the actual loading. Clean.
14 *
15 * Once more I can proudly say that linux stood up to being changed: it
16 * was less than 2 hours work to get demand-loading completely implemented.
17 *
18 * Demand loading changed July 1993 by Eric Youngdale. Use mmap instead,
19 * current->executable is only used by the procfs. This allows a dispatch
20 * table to check for several different types of binary formats. We keep
21 * trying until we recognize the file or we run out of supported binary
22 * formats.
23 */
24
25#include <linux/config.h>
26#include <linux/slab.h>
27#include <linux/file.h>
28#include <linux/mman.h>
29#include <linux/a.out.h>
30#include <linux/stat.h>
31#include <linux/fcntl.h>
32#include <linux/smp_lock.h>
33#include <linux/init.h>
34#include <linux/pagemap.h>
35#include <linux/highmem.h>
36#include <linux/spinlock.h>
37#include <linux/key.h>
38#include <linux/personality.h>
39#include <linux/binfmts.h>
40#include <linux/swap.h>
41#include <linux/utsname.h>
42#include <linux/module.h>
43#include <linux/namei.h>
44#include <linux/proc_fs.h>
45#include <linux/ptrace.h>
46#include <linux/mount.h>
47#include <linux/security.h>
48#include <linux/syscalls.h>
49#include <linux/rmap.h>
50#include <linux/acct.h>
51
52#include <asm/uaccess.h>
53#include <asm/mmu_context.h>
54
55#ifdef CONFIG_KMOD
56#include <linux/kmod.h>
57#endif
58
59int core_uses_pid;
60char core_pattern[65] = "core";
d6e71144
AC
61int suid_dumpable = 0;
62
63EXPORT_SYMBOL(suid_dumpable);
1da177e4
LT
64/* The maximal length of core_pattern is also specified in sysctl.c */
65
66static struct linux_binfmt *formats;
67static DEFINE_RWLOCK(binfmt_lock);
68
69int register_binfmt(struct linux_binfmt * fmt)
70{
71 struct linux_binfmt ** tmp = &formats;
72
73 if (!fmt)
74 return -EINVAL;
75 if (fmt->next)
76 return -EBUSY;
77 write_lock(&binfmt_lock);
78 while (*tmp) {
79 if (fmt == *tmp) {
80 write_unlock(&binfmt_lock);
81 return -EBUSY;
82 }
83 tmp = &(*tmp)->next;
84 }
85 fmt->next = formats;
86 formats = fmt;
87 write_unlock(&binfmt_lock);
88 return 0;
89}
90
91EXPORT_SYMBOL(register_binfmt);
92
93int unregister_binfmt(struct linux_binfmt * fmt)
94{
95 struct linux_binfmt ** tmp = &formats;
96
97 write_lock(&binfmt_lock);
98 while (*tmp) {
99 if (fmt == *tmp) {
100 *tmp = fmt->next;
101 write_unlock(&binfmt_lock);
102 return 0;
103 }
104 tmp = &(*tmp)->next;
105 }
106 write_unlock(&binfmt_lock);
107 return -EINVAL;
108}
109
110EXPORT_SYMBOL(unregister_binfmt);
111
112static inline void put_binfmt(struct linux_binfmt * fmt)
113{
114 module_put(fmt->module);
115}
116
117/*
118 * Note that a shared library must be both readable and executable due to
119 * security reasons.
120 *
121 * Also note that we take the address to load from from the file itself.
122 */
123asmlinkage long sys_uselib(const char __user * library)
124{
125 struct file * file;
126 struct nameidata nd;
127 int error;
128
834f2a4a 129 error = __user_path_lookup_open(library, LOOKUP_FOLLOW, &nd, FMODE_READ);
1da177e4
LT
130 if (error)
131 goto out;
132
133 error = -EINVAL;
134 if (!S_ISREG(nd.dentry->d_inode->i_mode))
135 goto exit;
136
137 error = permission(nd.dentry->d_inode, MAY_READ | MAY_EXEC, &nd);
138 if (error)
139 goto exit;
140
834f2a4a 141 file = nameidata_to_filp(&nd, O_RDONLY);
1da177e4
LT
142 error = PTR_ERR(file);
143 if (IS_ERR(file))
144 goto out;
145
146 error = -ENOEXEC;
147 if(file->f_op) {
148 struct linux_binfmt * fmt;
149
150 read_lock(&binfmt_lock);
151 for (fmt = formats ; fmt ; fmt = fmt->next) {
152 if (!fmt->load_shlib)
153 continue;
154 if (!try_module_get(fmt->module))
155 continue;
156 read_unlock(&binfmt_lock);
157 error = fmt->load_shlib(file);
158 read_lock(&binfmt_lock);
159 put_binfmt(fmt);
160 if (error != -ENOEXEC)
161 break;
162 }
163 read_unlock(&binfmt_lock);
164 }
165 fput(file);
166out:
167 return error;
168exit:
834f2a4a 169 release_open_intent(&nd);
1da177e4
LT
170 path_release(&nd);
171 goto out;
172}
173
174/*
175 * count() counts the number of strings in array ARGV.
176 */
177static int count(char __user * __user * argv, int max)
178{
179 int i = 0;
180
181 if (argv != NULL) {
182 for (;;) {
183 char __user * p;
184
185 if (get_user(p, argv))
186 return -EFAULT;
187 if (!p)
188 break;
189 argv++;
190 if(++i > max)
191 return -E2BIG;
192 cond_resched();
193 }
194 }
195 return i;
196}
197
198/*
199 * 'copy_strings()' copies argument/environment strings from user
200 * memory to free pages in kernel mem. These are in a format ready
201 * to be put directly into the top of new user memory.
202 */
75c96f85
AB
203static int copy_strings(int argc, char __user * __user * argv,
204 struct linux_binprm *bprm)
1da177e4
LT
205{
206 struct page *kmapped_page = NULL;
207 char *kaddr = NULL;
208 int ret;
209
210 while (argc-- > 0) {
211 char __user *str;
212 int len;
213 unsigned long pos;
214
215 if (get_user(str, argv+argc) ||
216 !(len = strnlen_user(str, bprm->p))) {
217 ret = -EFAULT;
218 goto out;
219 }
220
221 if (bprm->p < len) {
222 ret = -E2BIG;
223 goto out;
224 }
225
226 bprm->p -= len;
227 /* XXX: add architecture specific overflow check here. */
228 pos = bprm->p;
229
230 while (len > 0) {
231 int i, new, err;
232 int offset, bytes_to_copy;
233 struct page *page;
234
235 offset = pos % PAGE_SIZE;
236 i = pos/PAGE_SIZE;
237 page = bprm->page[i];
238 new = 0;
239 if (!page) {
240 page = alloc_page(GFP_HIGHUSER);
241 bprm->page[i] = page;
242 if (!page) {
243 ret = -ENOMEM;
244 goto out;
245 }
246 new = 1;
247 }
248
249 if (page != kmapped_page) {
250 if (kmapped_page)
251 kunmap(kmapped_page);
252 kmapped_page = page;
253 kaddr = kmap(kmapped_page);
254 }
255 if (new && offset)
256 memset(kaddr, 0, offset);
257 bytes_to_copy = PAGE_SIZE - offset;
258 if (bytes_to_copy > len) {
259 bytes_to_copy = len;
260 if (new)
261 memset(kaddr+offset+len, 0,
262 PAGE_SIZE-offset-len);
263 }
264 err = copy_from_user(kaddr+offset, str, bytes_to_copy);
265 if (err) {
266 ret = -EFAULT;
267 goto out;
268 }
269
270 pos += bytes_to_copy;
271 str += bytes_to_copy;
272 len -= bytes_to_copy;
273 }
274 }
275 ret = 0;
276out:
277 if (kmapped_page)
278 kunmap(kmapped_page);
279 return ret;
280}
281
282/*
283 * Like copy_strings, but get argv and its values from kernel memory.
284 */
285int copy_strings_kernel(int argc,char ** argv, struct linux_binprm *bprm)
286{
287 int r;
288 mm_segment_t oldfs = get_fs();
289 set_fs(KERNEL_DS);
290 r = copy_strings(argc, (char __user * __user *)argv, bprm);
291 set_fs(oldfs);
292 return r;
293}
294
295EXPORT_SYMBOL(copy_strings_kernel);
296
297#ifdef CONFIG_MMU
298/*
299 * This routine is used to map in a page into an address space: needed by
300 * execve() for the initial stack and environment pages.
301 *
302 * vma->vm_mm->mmap_sem is held for writing.
303 */
304void install_arg_page(struct vm_area_struct *vma,
305 struct page *page, unsigned long address)
306{
307 struct mm_struct *mm = vma->vm_mm;
308 pgd_t * pgd;
309 pud_t * pud;
310 pmd_t * pmd;
311 pte_t * pte;
c74df32c 312 spinlock_t *ptl;
1da177e4
LT
313
314 if (unlikely(anon_vma_prepare(vma)))
c74df32c 315 goto out;
1da177e4
LT
316
317 flush_dcache_page(page);
318 pgd = pgd_offset(mm, address);
1da177e4
LT
319 pud = pud_alloc(mm, pgd, address);
320 if (!pud)
321 goto out;
322 pmd = pmd_alloc(mm, pud, address);
323 if (!pmd)
324 goto out;
c74df32c 325 pte = pte_alloc_map_lock(mm, pmd, address, &ptl);
1da177e4
LT
326 if (!pte)
327 goto out;
328 if (!pte_none(*pte)) {
c74df32c 329 pte_unmap_unlock(pte, ptl);
1da177e4
LT
330 goto out;
331 }
4294621f 332 inc_mm_counter(mm, anon_rss);
1da177e4
LT
333 lru_cache_add_active(page);
334 set_pte_at(mm, address, pte, pte_mkdirty(pte_mkwrite(mk_pte(
335 page, vma->vm_page_prot))));
336 page_add_anon_rmap(page, vma, address);
c74df32c 337 pte_unmap_unlock(pte, ptl);
1da177e4
LT
338
339 /* no need for flush_tlb */
340 return;
341out:
1da177e4
LT
342 __free_page(page);
343 force_sig(SIGKILL, current);
344}
345
346#define EXTRA_STACK_VM_PAGES 20 /* random */
347
348int setup_arg_pages(struct linux_binprm *bprm,
349 unsigned long stack_top,
350 int executable_stack)
351{
352 unsigned long stack_base;
353 struct vm_area_struct *mpnt;
354 struct mm_struct *mm = current->mm;
355 int i, ret;
356 long arg_size;
357
358#ifdef CONFIG_STACK_GROWSUP
359 /* Move the argument and environment strings to the bottom of the
360 * stack space.
361 */
362 int offset, j;
363 char *to, *from;
364
365 /* Start by shifting all the pages down */
366 i = 0;
367 for (j = 0; j < MAX_ARG_PAGES; j++) {
368 struct page *page = bprm->page[j];
369 if (!page)
370 continue;
371 bprm->page[i++] = page;
372 }
373
374 /* Now move them within their pages */
375 offset = bprm->p % PAGE_SIZE;
376 to = kmap(bprm->page[0]);
377 for (j = 1; j < i; j++) {
378 memmove(to, to + offset, PAGE_SIZE - offset);
379 from = kmap(bprm->page[j]);
380 memcpy(to + PAGE_SIZE - offset, from, offset);
381 kunmap(bprm->page[j - 1]);
382 to = from;
383 }
384 memmove(to, to + offset, PAGE_SIZE - offset);
385 kunmap(bprm->page[j - 1]);
386
387 /* Limit stack size to 1GB */
388 stack_base = current->signal->rlim[RLIMIT_STACK].rlim_max;
389 if (stack_base > (1 << 30))
390 stack_base = 1 << 30;
391 stack_base = PAGE_ALIGN(stack_top - stack_base);
392
393 /* Adjust bprm->p to point to the end of the strings. */
394 bprm->p = stack_base + PAGE_SIZE * i - offset;
395
396 mm->arg_start = stack_base;
397 arg_size = i << PAGE_SHIFT;
398
399 /* zero pages that were copied above */
400 while (i < MAX_ARG_PAGES)
401 bprm->page[i++] = NULL;
402#else
403 stack_base = arch_align_stack(stack_top - MAX_ARG_PAGES*PAGE_SIZE);
404 stack_base = PAGE_ALIGN(stack_base);
405 bprm->p += stack_base;
406 mm->arg_start = bprm->p;
407 arg_size = stack_top - (PAGE_MASK & (unsigned long) mm->arg_start);
408#endif
409
410 arg_size += EXTRA_STACK_VM_PAGES * PAGE_SIZE;
411
412 if (bprm->loader)
413 bprm->loader += stack_base;
414 bprm->exec += stack_base;
415
416 mpnt = kmem_cache_alloc(vm_area_cachep, SLAB_KERNEL);
417 if (!mpnt)
418 return -ENOMEM;
419
1da177e4
LT
420 memset(mpnt, 0, sizeof(*mpnt));
421
422 down_write(&mm->mmap_sem);
423 {
424 mpnt->vm_mm = mm;
425#ifdef CONFIG_STACK_GROWSUP
426 mpnt->vm_start = stack_base;
427 mpnt->vm_end = stack_base + arg_size;
428#else
429 mpnt->vm_end = stack_top;
430 mpnt->vm_start = mpnt->vm_end - arg_size;
431#endif
432 /* Adjust stack execute permissions; explicitly enable
433 * for EXSTACK_ENABLE_X, disable for EXSTACK_DISABLE_X
434 * and leave alone (arch default) otherwise. */
435 if (unlikely(executable_stack == EXSTACK_ENABLE_X))
436 mpnt->vm_flags = VM_STACK_FLAGS | VM_EXEC;
437 else if (executable_stack == EXSTACK_DISABLE_X)
438 mpnt->vm_flags = VM_STACK_FLAGS & ~VM_EXEC;
439 else
440 mpnt->vm_flags = VM_STACK_FLAGS;
441 mpnt->vm_flags |= mm->def_flags;
442 mpnt->vm_page_prot = protection_map[mpnt->vm_flags & 0x7];
443 if ((ret = insert_vm_struct(mm, mpnt))) {
444 up_write(&mm->mmap_sem);
445 kmem_cache_free(vm_area_cachep, mpnt);
446 return ret;
447 }
448 mm->stack_vm = mm->total_vm = vma_pages(mpnt);
449 }
450
451 for (i = 0 ; i < MAX_ARG_PAGES ; i++) {
452 struct page *page = bprm->page[i];
453 if (page) {
454 bprm->page[i] = NULL;
455 install_arg_page(mpnt, page, stack_base);
456 }
457 stack_base += PAGE_SIZE;
458 }
459 up_write(&mm->mmap_sem);
460
461 return 0;
462}
463
464EXPORT_SYMBOL(setup_arg_pages);
465
466#define free_arg_pages(bprm) do { } while (0)
467
468#else
469
470static inline void free_arg_pages(struct linux_binprm *bprm)
471{
472 int i;
473
474 for (i = 0; i < MAX_ARG_PAGES; i++) {
475 if (bprm->page[i])
476 __free_page(bprm->page[i]);
477 bprm->page[i] = NULL;
478 }
479}
480
481#endif /* CONFIG_MMU */
482
483struct file *open_exec(const char *name)
484{
485 struct nameidata nd;
486 int err;
487 struct file *file;
488
834f2a4a 489 err = path_lookup_open(name, LOOKUP_FOLLOW, &nd, FMODE_READ);
1da177e4
LT
490 file = ERR_PTR(err);
491
492 if (!err) {
493 struct inode *inode = nd.dentry->d_inode;
494 file = ERR_PTR(-EACCES);
495 if (!(nd.mnt->mnt_flags & MNT_NOEXEC) &&
496 S_ISREG(inode->i_mode)) {
497 int err = permission(inode, MAY_EXEC, &nd);
498 if (!err && !(inode->i_mode & 0111))
499 err = -EACCES;
500 file = ERR_PTR(err);
501 if (!err) {
834f2a4a 502 file = nameidata_to_filp(&nd, O_RDONLY);
1da177e4
LT
503 if (!IS_ERR(file)) {
504 err = deny_write_access(file);
505 if (err) {
506 fput(file);
507 file = ERR_PTR(err);
508 }
509 }
510out:
511 return file;
512 }
513 }
834f2a4a 514 release_open_intent(&nd);
1da177e4
LT
515 path_release(&nd);
516 }
517 goto out;
518}
519
520EXPORT_SYMBOL(open_exec);
521
522int kernel_read(struct file *file, unsigned long offset,
523 char *addr, unsigned long count)
524{
525 mm_segment_t old_fs;
526 loff_t pos = offset;
527 int result;
528
529 old_fs = get_fs();
530 set_fs(get_ds());
531 /* The cast to a user pointer is valid due to the set_fs() */
532 result = vfs_read(file, (void __user *)addr, count, &pos);
533 set_fs(old_fs);
534 return result;
535}
536
537EXPORT_SYMBOL(kernel_read);
538
539static int exec_mmap(struct mm_struct *mm)
540{
541 struct task_struct *tsk;
542 struct mm_struct * old_mm, *active_mm;
543
544 /* Notify parent that we're no longer interested in the old VM */
545 tsk = current;
546 old_mm = current->mm;
547 mm_release(tsk, old_mm);
548
549 if (old_mm) {
550 /*
551 * Make sure that if there is a core dump in progress
552 * for the old mm, we get out and die instead of going
553 * through with the exec. We must hold mmap_sem around
554 * checking core_waiters and changing tsk->mm. The
555 * core-inducing thread will increment core_waiters for
556 * each thread whose ->mm == old_mm.
557 */
558 down_read(&old_mm->mmap_sem);
559 if (unlikely(old_mm->core_waiters)) {
560 up_read(&old_mm->mmap_sem);
561 return -EINTR;
562 }
563 }
564 task_lock(tsk);
565 active_mm = tsk->active_mm;
566 tsk->mm = mm;
567 tsk->active_mm = mm;
568 activate_mm(active_mm, mm);
569 task_unlock(tsk);
570 arch_pick_mmap_layout(mm);
571 if (old_mm) {
572 up_read(&old_mm->mmap_sem);
573 if (active_mm != old_mm) BUG();
574 mmput(old_mm);
575 return 0;
576 }
577 mmdrop(active_mm);
578 return 0;
579}
580
581/*
582 * This function makes sure the current process has its own signal table,
583 * so that flush_signal_handlers can later reset the handlers without
584 * disturbing other processes. (Other processes might share the signal
585 * table via the CLONE_SIGHAND option to clone().)
586 */
587static inline int de_thread(struct task_struct *tsk)
588{
589 struct signal_struct *sig = tsk->signal;
590 struct sighand_struct *newsighand, *oldsighand = tsk->sighand;
591 spinlock_t *lock = &oldsighand->siglock;
592 int count;
593
594 /*
595 * If we don't share sighandlers, then we aren't sharing anything
596 * and we can just re-use it all.
597 */
598 if (atomic_read(&oldsighand->count) <= 1) {
599 BUG_ON(atomic_read(&sig->count) != 1);
600 exit_itimers(sig);
601 return 0;
602 }
603
604 newsighand = kmem_cache_alloc(sighand_cachep, GFP_KERNEL);
605 if (!newsighand)
606 return -ENOMEM;
607
608 if (thread_group_empty(current))
609 goto no_thread_group;
610
611 /*
612 * Kill all other threads in the thread group.
613 * We must hold tasklist_lock to call zap_other_threads.
614 */
615 read_lock(&tasklist_lock);
616 spin_lock_irq(lock);
617 if (sig->flags & SIGNAL_GROUP_EXIT) {
618 /*
619 * Another group action in progress, just
620 * return so that the signal is processed.
621 */
622 spin_unlock_irq(lock);
623 read_unlock(&tasklist_lock);
624 kmem_cache_free(sighand_cachep, newsighand);
625 return -EAGAIN;
626 }
627 zap_other_threads(current);
628 read_unlock(&tasklist_lock);
629
630 /*
631 * Account for the thread group leader hanging around:
632 */
9e4e23bc
ON
633 count = 1;
634 if (!thread_group_leader(current)) {
635 count = 2;
53231250
RM
636 /*
637 * The SIGALRM timer survives the exec, but needs to point
638 * at us as the new group leader now. We have a race with
639 * a timer firing now getting the old leader, so we need to
640 * synchronize with any firing (by calling del_timer_sync)
641 * before we can safely let the old group leader die.
642 */
643 sig->real_timer.data = (unsigned long)current;
644 if (del_timer_sync(&sig->real_timer))
645 add_timer(&sig->real_timer);
646 }
1da177e4
LT
647 while (atomic_read(&sig->count) > count) {
648 sig->group_exit_task = current;
649 sig->notify_count = count;
650 __set_current_state(TASK_UNINTERRUPTIBLE);
651 spin_unlock_irq(lock);
652 schedule();
653 spin_lock_irq(lock);
654 }
655 sig->group_exit_task = NULL;
656 sig->notify_count = 0;
c2a0f594 657 sig->real_timer.data = (unsigned long)current;
1da177e4
LT
658 spin_unlock_irq(lock);
659
660 /*
661 * At this point all other threads have exited, all we have to
662 * do is to wait for the thread group leader to become inactive,
663 * and to assume its PID:
664 */
665 if (!thread_group_leader(current)) {
666 struct task_struct *leader = current->group_leader, *parent;
667 struct dentry *proc_dentry1, *proc_dentry2;
668 unsigned long exit_state, ptrace;
669
670 /*
671 * Wait for the thread group leader to be a zombie.
672 * It should already be zombie at this point, most
673 * of the time.
674 */
675 while (leader->exit_state != EXIT_ZOMBIE)
676 yield();
677
678 spin_lock(&leader->proc_lock);
679 spin_lock(&current->proc_lock);
680 proc_dentry1 = proc_pid_unhash(current);
681 proc_dentry2 = proc_pid_unhash(leader);
682 write_lock_irq(&tasklist_lock);
683
c2a0f594
LT
684 BUG_ON(leader->tgid != current->tgid);
685 BUG_ON(current->pid == current->tgid);
1da177e4
LT
686 /*
687 * An exec() starts a new thread group with the
688 * TGID of the previous thread group. Rehash the
689 * two threads with a switched PID, and release
690 * the former thread group leader:
691 */
692 ptrace = leader->ptrace;
693 parent = leader->parent;
694 if (unlikely(ptrace) && unlikely(parent == current)) {
695 /*
696 * Joker was ptracing his own group leader,
697 * and now he wants to be his own parent!
698 * We can't have that.
699 */
700 ptrace = 0;
701 }
702
703 ptrace_unlink(current);
704 ptrace_unlink(leader);
705 remove_parent(current);
706 remove_parent(leader);
707
708 switch_exec_pids(leader, current);
709
710 current->parent = current->real_parent = leader->real_parent;
711 leader->parent = leader->real_parent = child_reaper;
712 current->group_leader = current;
713 leader->group_leader = leader;
714
715 add_parent(current, current->parent);
716 add_parent(leader, leader->parent);
717 if (ptrace) {
718 current->ptrace = ptrace;
719 __ptrace_link(current, parent);
720 }
721
722 list_del(&current->tasks);
723 list_add_tail(&current->tasks, &init_task.tasks);
724 current->exit_signal = SIGCHLD;
725 exit_state = leader->exit_state;
726
727 write_unlock_irq(&tasklist_lock);
728 spin_unlock(&leader->proc_lock);
729 spin_unlock(&current->proc_lock);
730 proc_pid_flush(proc_dentry1);
731 proc_pid_flush(proc_dentry2);
732
c2a0f594 733 BUG_ON(exit_state != EXIT_ZOMBIE);
1da177e4
LT
734 release_task(leader);
735 }
736
737 /*
fb085cf1
AN
738 * There may be one thread left which is just exiting,
739 * but it's safe to stop telling the group to kill themselves.
1da177e4
LT
740 */
741 sig->flags = 0;
742
743no_thread_group:
744 BUG_ON(atomic_read(&sig->count) != 1);
745 exit_itimers(sig);
746
747 if (atomic_read(&oldsighand->count) == 1) {
748 /*
749 * Now that we nuked the rest of the thread group,
750 * it turns out we are not sharing sighand any more either.
751 * So we can just keep it.
752 */
753 kmem_cache_free(sighand_cachep, newsighand);
754 } else {
755 /*
756 * Move our state over to newsighand and switch it in.
757 */
758 spin_lock_init(&newsighand->siglock);
759 atomic_set(&newsighand->count, 1);
760 memcpy(newsighand->action, oldsighand->action,
761 sizeof(newsighand->action));
762
763 write_lock_irq(&tasklist_lock);
764 spin_lock(&oldsighand->siglock);
765 spin_lock(&newsighand->siglock);
766
767 current->sighand = newsighand;
768 recalc_sigpending();
769
770 spin_unlock(&newsighand->siglock);
771 spin_unlock(&oldsighand->siglock);
772 write_unlock_irq(&tasklist_lock);
773
774 if (atomic_dec_and_test(&oldsighand->count))
775 kmem_cache_free(sighand_cachep, oldsighand);
776 }
777
c2a0f594 778 BUG_ON(!thread_group_leader(current));
1da177e4
LT
779 return 0;
780}
781
782/*
783 * These functions flushes out all traces of the currently running executable
784 * so that a new one can be started
785 */
786
787static inline void flush_old_files(struct files_struct * files)
788{
789 long j = -1;
badf1662 790 struct fdtable *fdt;
1da177e4
LT
791
792 spin_lock(&files->file_lock);
793 for (;;) {
794 unsigned long set, i;
795
796 j++;
797 i = j * __NFDBITS;
badf1662
DS
798 fdt = files_fdtable(files);
799 if (i >= fdt->max_fds || i >= fdt->max_fdset)
1da177e4 800 break;
badf1662 801 set = fdt->close_on_exec->fds_bits[j];
1da177e4
LT
802 if (!set)
803 continue;
badf1662 804 fdt->close_on_exec->fds_bits[j] = 0;
1da177e4
LT
805 spin_unlock(&files->file_lock);
806 for ( ; set ; i++,set >>= 1) {
807 if (set & 1) {
808 sys_close(i);
809 }
810 }
811 spin_lock(&files->file_lock);
812
813 }
814 spin_unlock(&files->file_lock);
815}
816
817void get_task_comm(char *buf, struct task_struct *tsk)
818{
819 /* buf must be at least sizeof(tsk->comm) in size */
820 task_lock(tsk);
821 strncpy(buf, tsk->comm, sizeof(tsk->comm));
822 task_unlock(tsk);
823}
824
825void set_task_comm(struct task_struct *tsk, char *buf)
826{
827 task_lock(tsk);
828 strlcpy(tsk->comm, buf, sizeof(tsk->comm));
829 task_unlock(tsk);
830}
831
832int flush_old_exec(struct linux_binprm * bprm)
833{
834 char * name;
835 int i, ch, retval;
836 struct files_struct *files;
837 char tcomm[sizeof(current->comm)];
838
839 /*
840 * Make sure we have a private signal table and that
841 * we are unassociated from the previous thread group.
842 */
843 retval = de_thread(current);
844 if (retval)
845 goto out;
846
847 /*
848 * Make sure we have private file handles. Ask the
849 * fork helper to do the work for us and the exit
850 * helper to do the cleanup of the old one.
851 */
852 files = current->files; /* refcounted so safe to hold */
853 retval = unshare_files();
854 if (retval)
855 goto out;
856 /*
857 * Release all of the old mmap stuff
858 */
859 retval = exec_mmap(bprm->mm);
860 if (retval)
861 goto mmap_failed;
862
863 bprm->mm = NULL; /* We're using it now */
864
865 /* This is the point of no return */
866 steal_locks(files);
867 put_files_struct(files);
868
869 current->sas_ss_sp = current->sas_ss_size = 0;
870
871 if (current->euid == current->uid && current->egid == current->gid)
872 current->mm->dumpable = 1;
d6e71144
AC
873 else
874 current->mm->dumpable = suid_dumpable;
875
1da177e4 876 name = bprm->filename;
36772092
PBG
877
878 /* Copies the binary name from after last slash */
1da177e4
LT
879 for (i=0; (ch = *(name++)) != '\0';) {
880 if (ch == '/')
36772092 881 i = 0; /* overwrite what we wrote */
1da177e4
LT
882 else
883 if (i < (sizeof(tcomm) - 1))
884 tcomm[i++] = ch;
885 }
886 tcomm[i] = '\0';
887 set_task_comm(current, tcomm);
888
889 current->flags &= ~PF_RANDOMIZE;
890 flush_thread();
891
892 if (bprm->e_uid != current->euid || bprm->e_gid != current->egid ||
893 permission(bprm->file->f_dentry->d_inode,MAY_READ, NULL) ||
894 (bprm->interp_flags & BINPRM_FLAGS_ENFORCE_NONDUMP)) {
895 suid_keys(current);
d6e71144 896 current->mm->dumpable = suid_dumpable;
1da177e4
LT
897 }
898
899 /* An exec changes our domain. We are no longer part of the thread
900 group */
901
902 current->self_exec_id++;
903
904 flush_signal_handlers(current, 0);
905 flush_old_files(current->files);
906
907 return 0;
908
909mmap_failed:
910 put_files_struct(current->files);
911 current->files = files;
912out:
913 return retval;
914}
915
916EXPORT_SYMBOL(flush_old_exec);
917
918/*
919 * Fill the binprm structure from the inode.
920 * Check permissions, then read the first 128 (BINPRM_BUF_SIZE) bytes
921 */
922int prepare_binprm(struct linux_binprm *bprm)
923{
924 int mode;
925 struct inode * inode = bprm->file->f_dentry->d_inode;
926 int retval;
927
928 mode = inode->i_mode;
929 /*
930 * Check execute perms again - if the caller has CAP_DAC_OVERRIDE,
931 * generic_permission lets a non-executable through
932 */
933 if (!(mode & 0111)) /* with at least _one_ execute bit set */
934 return -EACCES;
935 if (bprm->file->f_op == NULL)
936 return -EACCES;
937
938 bprm->e_uid = current->euid;
939 bprm->e_gid = current->egid;
940
941 if(!(bprm->file->f_vfsmnt->mnt_flags & MNT_NOSUID)) {
942 /* Set-uid? */
943 if (mode & S_ISUID) {
944 current->personality &= ~PER_CLEAR_ON_SETID;
945 bprm->e_uid = inode->i_uid;
946 }
947
948 /* Set-gid? */
949 /*
950 * If setgid is set but no group execute bit then this
951 * is a candidate for mandatory locking, not a setgid
952 * executable.
953 */
954 if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP)) {
955 current->personality &= ~PER_CLEAR_ON_SETID;
956 bprm->e_gid = inode->i_gid;
957 }
958 }
959
960 /* fill in binprm security blob */
961 retval = security_bprm_set(bprm);
962 if (retval)
963 return retval;
964
965 memset(bprm->buf,0,BINPRM_BUF_SIZE);
966 return kernel_read(bprm->file,0,bprm->buf,BINPRM_BUF_SIZE);
967}
968
969EXPORT_SYMBOL(prepare_binprm);
970
971static inline int unsafe_exec(struct task_struct *p)
972{
973 int unsafe = 0;
974 if (p->ptrace & PT_PTRACED) {
975 if (p->ptrace & PT_PTRACE_CAP)
976 unsafe |= LSM_UNSAFE_PTRACE_CAP;
977 else
978 unsafe |= LSM_UNSAFE_PTRACE;
979 }
980 if (atomic_read(&p->fs->count) > 1 ||
981 atomic_read(&p->files->count) > 1 ||
982 atomic_read(&p->sighand->count) > 1)
983 unsafe |= LSM_UNSAFE_SHARE;
984
985 return unsafe;
986}
987
988void compute_creds(struct linux_binprm *bprm)
989{
990 int unsafe;
991
992 if (bprm->e_uid != current->uid)
993 suid_keys(current);
994 exec_keys(current);
995
996 task_lock(current);
997 unsafe = unsafe_exec(current);
998 security_bprm_apply_creds(bprm, unsafe);
999 task_unlock(current);
1000 security_bprm_post_apply_creds(bprm);
1001}
1002
1003EXPORT_SYMBOL(compute_creds);
1004
1005void remove_arg_zero(struct linux_binprm *bprm)
1006{
1007 if (bprm->argc) {
1008 unsigned long offset;
1009 char * kaddr;
1010 struct page *page;
1011
1012 offset = bprm->p % PAGE_SIZE;
1013 goto inside;
1014
1015 while (bprm->p++, *(kaddr+offset++)) {
1016 if (offset != PAGE_SIZE)
1017 continue;
1018 offset = 0;
1019 kunmap_atomic(kaddr, KM_USER0);
1020inside:
1021 page = bprm->page[bprm->p/PAGE_SIZE];
1022 kaddr = kmap_atomic(page, KM_USER0);
1023 }
1024 kunmap_atomic(kaddr, KM_USER0);
1025 bprm->argc--;
1026 }
1027}
1028
1029EXPORT_SYMBOL(remove_arg_zero);
1030
1031/*
1032 * cycle the list of binary formats handler, until one recognizes the image
1033 */
1034int search_binary_handler(struct linux_binprm *bprm,struct pt_regs *regs)
1035{
1036 int try,retval;
1037 struct linux_binfmt *fmt;
1038#ifdef __alpha__
1039 /* handle /sbin/loader.. */
1040 {
1041 struct exec * eh = (struct exec *) bprm->buf;
1042
1043 if (!bprm->loader && eh->fh.f_magic == 0x183 &&
1044 (eh->fh.f_flags & 0x3000) == 0x3000)
1045 {
1046 struct file * file;
1047 unsigned long loader;
1048
1049 allow_write_access(bprm->file);
1050 fput(bprm->file);
1051 bprm->file = NULL;
1052
1053 loader = PAGE_SIZE*MAX_ARG_PAGES-sizeof(void *);
1054
1055 file = open_exec("/sbin/loader");
1056 retval = PTR_ERR(file);
1057 if (IS_ERR(file))
1058 return retval;
1059
1060 /* Remember if the application is TASO. */
1061 bprm->sh_bang = eh->ah.entry < 0x100000000UL;
1062
1063 bprm->file = file;
1064 bprm->loader = loader;
1065 retval = prepare_binprm(bprm);
1066 if (retval<0)
1067 return retval;
1068 /* should call search_binary_handler recursively here,
1069 but it does not matter */
1070 }
1071 }
1072#endif
1073 retval = security_bprm_check(bprm);
1074 if (retval)
1075 return retval;
1076
1077 /* kernel module loader fixup */
1078 /* so we don't try to load run modprobe in kernel space. */
1079 set_fs(USER_DS);
1080 retval = -ENOENT;
1081 for (try=0; try<2; try++) {
1082 read_lock(&binfmt_lock);
1083 for (fmt = formats ; fmt ; fmt = fmt->next) {
1084 int (*fn)(struct linux_binprm *, struct pt_regs *) = fmt->load_binary;
1085 if (!fn)
1086 continue;
1087 if (!try_module_get(fmt->module))
1088 continue;
1089 read_unlock(&binfmt_lock);
1090 retval = fn(bprm, regs);
1091 if (retval >= 0) {
1092 put_binfmt(fmt);
1093 allow_write_access(bprm->file);
1094 if (bprm->file)
1095 fput(bprm->file);
1096 bprm->file = NULL;
1097 current->did_exec = 1;
1098 return retval;
1099 }
1100 read_lock(&binfmt_lock);
1101 put_binfmt(fmt);
1102 if (retval != -ENOEXEC || bprm->mm == NULL)
1103 break;
1104 if (!bprm->file) {
1105 read_unlock(&binfmt_lock);
1106 return retval;
1107 }
1108 }
1109 read_unlock(&binfmt_lock);
1110 if (retval != -ENOEXEC || bprm->mm == NULL) {
1111 break;
1112#ifdef CONFIG_KMOD
1113 }else{
1114#define printable(c) (((c)=='\t') || ((c)=='\n') || (0x20<=(c) && (c)<=0x7e))
1115 if (printable(bprm->buf[0]) &&
1116 printable(bprm->buf[1]) &&
1117 printable(bprm->buf[2]) &&
1118 printable(bprm->buf[3]))
1119 break; /* -ENOEXEC */
1120 request_module("binfmt-%04x", *(unsigned short *)(&bprm->buf[2]));
1121#endif
1122 }
1123 }
1124 return retval;
1125}
1126
1127EXPORT_SYMBOL(search_binary_handler);
1128
1129/*
1130 * sys_execve() executes a new program.
1131 */
1132int do_execve(char * filename,
1133 char __user *__user *argv,
1134 char __user *__user *envp,
1135 struct pt_regs * regs)
1136{
1137 struct linux_binprm *bprm;
1138 struct file *file;
1139 int retval;
1140 int i;
1141
1142 retval = -ENOMEM;
1143 bprm = kmalloc(sizeof(*bprm), GFP_KERNEL);
1144 if (!bprm)
1145 goto out_ret;
1146 memset(bprm, 0, sizeof(*bprm));
1147
1148 file = open_exec(filename);
1149 retval = PTR_ERR(file);
1150 if (IS_ERR(file))
1151 goto out_kfree;
1152
1153 sched_exec();
1154
1155 bprm->p = PAGE_SIZE*MAX_ARG_PAGES-sizeof(void *);
1156
1157 bprm->file = file;
1158 bprm->filename = filename;
1159 bprm->interp = filename;
1160 bprm->mm = mm_alloc();
1161 retval = -ENOMEM;
1162 if (!bprm->mm)
1163 goto out_file;
1164
1165 retval = init_new_context(current, bprm->mm);
1166 if (retval < 0)
1167 goto out_mm;
1168
1169 bprm->argc = count(argv, bprm->p / sizeof(void *));
1170 if ((retval = bprm->argc) < 0)
1171 goto out_mm;
1172
1173 bprm->envc = count(envp, bprm->p / sizeof(void *));
1174 if ((retval = bprm->envc) < 0)
1175 goto out_mm;
1176
1177 retval = security_bprm_alloc(bprm);
1178 if (retval)
1179 goto out;
1180
1181 retval = prepare_binprm(bprm);
1182 if (retval < 0)
1183 goto out;
1184
1185 retval = copy_strings_kernel(1, &bprm->filename, bprm);
1186 if (retval < 0)
1187 goto out;
1188
1189 bprm->exec = bprm->p;
1190 retval = copy_strings(bprm->envc, envp, bprm);
1191 if (retval < 0)
1192 goto out;
1193
1194 retval = copy_strings(bprm->argc, argv, bprm);
1195 if (retval < 0)
1196 goto out;
1197
1198 retval = search_binary_handler(bprm,regs);
1199 if (retval >= 0) {
1200 free_arg_pages(bprm);
1201
1202 /* execve success */
1203 security_bprm_free(bprm);
1204 acct_update_integrals(current);
1da177e4
LT
1205 kfree(bprm);
1206 return retval;
1207 }
1208
1209out:
1210 /* Something went wrong, return the inode and free the argument pages*/
1211 for (i = 0 ; i < MAX_ARG_PAGES ; i++) {
1212 struct page * page = bprm->page[i];
1213 if (page)
1214 __free_page(page);
1215 }
1216
1217 if (bprm->security)
1218 security_bprm_free(bprm);
1219
1220out_mm:
1221 if (bprm->mm)
1222 mmdrop(bprm->mm);
1223
1224out_file:
1225 if (bprm->file) {
1226 allow_write_access(bprm->file);
1227 fput(bprm->file);
1228 }
1229
1230out_kfree:
1231 kfree(bprm);
1232
1233out_ret:
1234 return retval;
1235}
1236
1237int set_binfmt(struct linux_binfmt *new)
1238{
1239 struct linux_binfmt *old = current->binfmt;
1240
1241 if (new) {
1242 if (!try_module_get(new->module))
1243 return -1;
1244 }
1245 current->binfmt = new;
1246 if (old)
1247 module_put(old->module);
1248 return 0;
1249}
1250
1251EXPORT_SYMBOL(set_binfmt);
1252
1253#define CORENAME_MAX_SIZE 64
1254
1255/* format_corename will inspect the pattern parameter, and output a
1256 * name into corename, which must have space for at least
1257 * CORENAME_MAX_SIZE bytes plus one byte for the zero terminator.
1258 */
1259static void format_corename(char *corename, const char *pattern, long signr)
1260{
1261 const char *pat_ptr = pattern;
1262 char *out_ptr = corename;
1263 char *const out_end = corename + CORENAME_MAX_SIZE;
1264 int rc;
1265 int pid_in_pattern = 0;
1266
1267 /* Repeat as long as we have more pattern to process and more output
1268 space */
1269 while (*pat_ptr) {
1270 if (*pat_ptr != '%') {
1271 if (out_ptr == out_end)
1272 goto out;
1273 *out_ptr++ = *pat_ptr++;
1274 } else {
1275 switch (*++pat_ptr) {
1276 case 0:
1277 goto out;
1278 /* Double percent, output one percent */
1279 case '%':
1280 if (out_ptr == out_end)
1281 goto out;
1282 *out_ptr++ = '%';
1283 break;
1284 /* pid */
1285 case 'p':
1286 pid_in_pattern = 1;
1287 rc = snprintf(out_ptr, out_end - out_ptr,
1288 "%d", current->tgid);
1289 if (rc > out_end - out_ptr)
1290 goto out;
1291 out_ptr += rc;
1292 break;
1293 /* uid */
1294 case 'u':
1295 rc = snprintf(out_ptr, out_end - out_ptr,
1296 "%d", current->uid);
1297 if (rc > out_end - out_ptr)
1298 goto out;
1299 out_ptr += rc;
1300 break;
1301 /* gid */
1302 case 'g':
1303 rc = snprintf(out_ptr, out_end - out_ptr,
1304 "%d", current->gid);
1305 if (rc > out_end - out_ptr)
1306 goto out;
1307 out_ptr += rc;
1308 break;
1309 /* signal that caused the coredump */
1310 case 's':
1311 rc = snprintf(out_ptr, out_end - out_ptr,
1312 "%ld", signr);
1313 if (rc > out_end - out_ptr)
1314 goto out;
1315 out_ptr += rc;
1316 break;
1317 /* UNIX time of coredump */
1318 case 't': {
1319 struct timeval tv;
1320 do_gettimeofday(&tv);
1321 rc = snprintf(out_ptr, out_end - out_ptr,
1322 "%lu", tv.tv_sec);
1323 if (rc > out_end - out_ptr)
1324 goto out;
1325 out_ptr += rc;
1326 break;
1327 }
1328 /* hostname */
1329 case 'h':
1330 down_read(&uts_sem);
1331 rc = snprintf(out_ptr, out_end - out_ptr,
1332 "%s", system_utsname.nodename);
1333 up_read(&uts_sem);
1334 if (rc > out_end - out_ptr)
1335 goto out;
1336 out_ptr += rc;
1337 break;
1338 /* executable */
1339 case 'e':
1340 rc = snprintf(out_ptr, out_end - out_ptr,
1341 "%s", current->comm);
1342 if (rc > out_end - out_ptr)
1343 goto out;
1344 out_ptr += rc;
1345 break;
1346 default:
1347 break;
1348 }
1349 ++pat_ptr;
1350 }
1351 }
1352 /* Backward compatibility with core_uses_pid:
1353 *
1354 * If core_pattern does not include a %p (as is the default)
1355 * and core_uses_pid is set, then .%pid will be appended to
1356 * the filename */
1357 if (!pid_in_pattern
1358 && (core_uses_pid || atomic_read(&current->mm->mm_users) != 1)) {
1359 rc = snprintf(out_ptr, out_end - out_ptr,
1360 ".%d", current->tgid);
1361 if (rc > out_end - out_ptr)
1362 goto out;
1363 out_ptr += rc;
1364 }
1365 out:
1366 *out_ptr = 0;
1367}
1368
1369static void zap_threads (struct mm_struct *mm)
1370{
1371 struct task_struct *g, *p;
1372 struct task_struct *tsk = current;
1373 struct completion *vfork_done = tsk->vfork_done;
1374 int traced = 0;
1375
1376 /*
1377 * Make sure nobody is waiting for us to release the VM,
1378 * otherwise we can deadlock when we wait on each other
1379 */
1380 if (vfork_done) {
1381 tsk->vfork_done = NULL;
1382 complete(vfork_done);
1383 }
1384
1385 read_lock(&tasklist_lock);
1386 do_each_thread(g,p)
1387 if (mm == p->mm && p != tsk) {
1388 force_sig_specific(SIGKILL, p);
1389 mm->core_waiters++;
1390 if (unlikely(p->ptrace) &&
1391 unlikely(p->parent->mm == mm))
1392 traced = 1;
1393 }
1394 while_each_thread(g,p);
1395
1396 read_unlock(&tasklist_lock);
1397
1398 if (unlikely(traced)) {
1399 /*
1400 * We are zapping a thread and the thread it ptraces.
1401 * If the tracee went into a ptrace stop for exit tracing,
1402 * we could deadlock since the tracer is waiting for this
1403 * coredump to finish. Detach them so they can both die.
1404 */
1405 write_lock_irq(&tasklist_lock);
1406 do_each_thread(g,p) {
1407 if (mm == p->mm && p != tsk &&
1408 p->ptrace && p->parent->mm == mm) {
1409 __ptrace_unlink(p);
1410 }
1411 } while_each_thread(g,p);
1412 write_unlock_irq(&tasklist_lock);
1413 }
1414}
1415
1416static void coredump_wait(struct mm_struct *mm)
1417{
1418 DECLARE_COMPLETION(startup_done);
1419
1420 mm->core_waiters++; /* let other threads block */
1421 mm->core_startup_done = &startup_done;
1422
1423 /* give other threads a chance to run: */
1424 yield();
1425
1426 zap_threads(mm);
1427 if (--mm->core_waiters) {
1428 up_write(&mm->mmap_sem);
1429 wait_for_completion(&startup_done);
1430 } else
1431 up_write(&mm->mmap_sem);
1432 BUG_ON(mm->core_waiters);
1433}
1434
1435int do_coredump(long signr, int exit_code, struct pt_regs * regs)
1436{
1437 char corename[CORENAME_MAX_SIZE + 1];
1438 struct mm_struct *mm = current->mm;
1439 struct linux_binfmt * binfmt;
1440 struct inode * inode;
1441 struct file * file;
1442 int retval = 0;
d6e71144
AC
1443 int fsuid = current->fsuid;
1444 int flag = 0;
1da177e4
LT
1445
1446 binfmt = current->binfmt;
1447 if (!binfmt || !binfmt->core_dump)
1448 goto fail;
1449 down_write(&mm->mmap_sem);
1450 if (!mm->dumpable) {
1451 up_write(&mm->mmap_sem);
1452 goto fail;
1453 }
d6e71144
AC
1454
1455 /*
1456 * We cannot trust fsuid as being the "true" uid of the
1457 * process nor do we know its entire history. We only know it
1458 * was tainted so we dump it as root in mode 2.
1459 */
1460 if (mm->dumpable == 2) { /* Setuid core dump mode */
1461 flag = O_EXCL; /* Stop rewrite attacks */
1462 current->fsuid = 0; /* Dump root private */
1463 }
1da177e4
LT
1464 mm->dumpable = 0;
1465 init_completion(&mm->core_done);
1466 spin_lock_irq(&current->sighand->siglock);
1467 current->signal->flags = SIGNAL_GROUP_EXIT;
1468 current->signal->group_exit_code = exit_code;
1469 spin_unlock_irq(&current->sighand->siglock);
1470 coredump_wait(mm);
1471
1472 /*
1473 * Clear any false indication of pending signals that might
1474 * be seen by the filesystem code called to write the core file.
1475 */
1476 current->signal->group_stop_count = 0;
1477 clear_thread_flag(TIF_SIGPENDING);
1478
1479 if (current->signal->rlim[RLIMIT_CORE].rlim_cur < binfmt->min_coredump)
1480 goto fail_unlock;
1481
1482 /*
1483 * lock_kernel() because format_corename() is controlled by sysctl, which
1484 * uses lock_kernel()
1485 */
1486 lock_kernel();
1487 format_corename(corename, core_pattern, signr);
1488 unlock_kernel();
d6e71144 1489 file = filp_open(corename, O_CREAT | 2 | O_NOFOLLOW | O_LARGEFILE | flag, 0600);
1da177e4
LT
1490 if (IS_ERR(file))
1491 goto fail_unlock;
1492 inode = file->f_dentry->d_inode;
1493 if (inode->i_nlink > 1)
1494 goto close_fail; /* multiple links - don't dump */
1495 if (d_unhashed(file->f_dentry))
1496 goto close_fail;
1497
1498 if (!S_ISREG(inode->i_mode))
1499 goto close_fail;
1500 if (!file->f_op)
1501 goto close_fail;
1502 if (!file->f_op->write)
1503 goto close_fail;
1504 if (do_truncate(file->f_dentry, 0) != 0)
1505 goto close_fail;
1506
1507 retval = binfmt->core_dump(signr, regs, file);
1508
1509 if (retval)
1510 current->signal->group_exit_code |= 0x80;
1511close_fail:
1512 filp_close(file, NULL);
1513fail_unlock:
d6e71144 1514 current->fsuid = fsuid;
1da177e4
LT
1515 complete_all(&mm->core_done);
1516fail:
1517 return retval;
1518}