mirror of
https://github.com/tbsdtv/linux_media.git
synced 2025-07-23 04:33:26 +02:00
Patch series "mm: Page fault accounting cleanups", v5.
This is v5 of the pf accounting cleanup series. It originates from Gerald
Schaefer's report on an issue a week ago regarding to incorrect page fault
accountings for retried page fault after commit 4064b98270
("mm: allow
VM_FAULT_RETRY for multiple times"):
https://lore.kernel.org/lkml/20200610174811.44b94525@thinkpad/
What this series did:
- Correct page fault accounting: we do accounting for a page fault
(no matter whether it's from #PF handling, or gup, or anything else)
only with the one that completed the fault. For example, page fault
retries should not be counted in page fault counters. Same to the
perf events.
- Unify definition of PERF_COUNT_SW_PAGE_FAULTS: currently this perf
event is used in an adhoc way across different archs.
Case (1): for many archs it's done at the entry of a page fault
handler, so that it will also cover e.g. errornous faults.
Case (2): for some other archs, it is only accounted when the page
fault is resolved successfully.
Case (3): there're still quite some archs that have not enabled
this perf event.
Since this series will touch merely all the archs, we unify this
perf event to always follow case (1), which is the one that makes most
sense. And since we moved the accounting into handle_mm_fault, the
other two MAJ/MIN perf events are well taken care of naturally.
- Unify definition of "major faults": the definition of "major
fault" is slightly changed when used in accounting (not
VM_FAULT_MAJOR). More information in patch 1.
- Always account the page fault onto the one that triggered the page
fault. This does not matter much for #PF handlings, but mostly for
gup. More information on this in patch 25.
Patchset layout:
Patch 1: Introduced the accounting in handle_mm_fault(), not enabled.
Patch 2-23: Enable the new accounting for arch #PF handlers one by one.
Patch 24: Enable the new accounting for the rest outliers (gup, iommu, etc.)
Patch 25: Cleanup GUP task_struct pointer since it's not needed any more
This patch (of 25):
This is a preparation patch to move page fault accountings into the
general code in handle_mm_fault(). This includes both the per task
flt_maj/flt_min counters, and the major/minor page fault perf events. To
do this, the pt_regs pointer is passed into handle_mm_fault().
PERF_COUNT_SW_PAGE_FAULTS should still be kept in per-arch page fault
handlers.
So far, all the pt_regs pointer that passed into handle_mm_fault() is
NULL, which means this patch should have no intented functional change.
Suggested-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Peter Xu <peterx@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Cain <bcain@codeaurora.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christian Borntraeger <borntraeger@de.ibm.com>
Cc: Chris Zankel <chris@zankel.net>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: David S. Miller <davem@davemloft.net>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Cc: Gerald Schaefer <gerald.schaefer@de.ibm.com>
Cc: Greentime Hu <green.hu@gmail.com>
Cc: Guo Ren <guoren@kernel.org>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Helge Deller <deller@gmx.de>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Ivan Kokshaysky <ink@jurassic.park.msu.ru>
Cc: James E.J. Bottomley <James.Bottomley@HansenPartnership.com>
Cc: John Hubbard <jhubbard@nvidia.com>
Cc: Jonas Bonn <jonas@southpole.se>
Cc: Ley Foon Tan <ley.foon.tan@intel.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Matt Turner <mattst88@gmail.com>
Cc: Max Filippov <jcmvbkbc@gmail.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Michal Simek <monstr@monstr.eu>
Cc: Nick Hu <nickhu@andestech.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Paul Walmsley <paul.walmsley@sifive.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Richard Henderson <rth@twiddle.net>
Cc: Rich Felker <dalias@libc.org>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Stafford Horne <shorne@gmail.com>
Cc: Stefan Kristiansson <stefan.kristiansson@saunalahti.fi>
Cc: Thomas Bogendoerfer <tsbogend@alpha.franken.de>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Vincent Chen <deanbo422@gmail.com>
Cc: Vineet Gupta <vgupta@synopsys.com>
Cc: Will Deacon <will@kernel.org>
Cc: Yoshinori Sato <ysato@users.sourceforge.jp>
Link: http://lkml.kernel.org/r/20200707225021.200906-1-peterx@redhat.com
Link: http://lkml.kernel.org/r/20200707225021.200906-2-peterx@redhat.com
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
253 lines
6.0 KiB
C
253 lines
6.0 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* linux/arch/alpha/mm/fault.c
|
|
*
|
|
* Copyright (C) 1995 Linus Torvalds
|
|
*/
|
|
|
|
#include <linux/sched/signal.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/mm.h>
|
|
#include <asm/io.h>
|
|
|
|
#define __EXTERN_INLINE inline
|
|
#include <asm/mmu_context.h>
|
|
#include <asm/tlbflush.h>
|
|
#undef __EXTERN_INLINE
|
|
|
|
#include <linux/signal.h>
|
|
#include <linux/errno.h>
|
|
#include <linux/string.h>
|
|
#include <linux/types.h>
|
|
#include <linux/ptrace.h>
|
|
#include <linux/mman.h>
|
|
#include <linux/smp.h>
|
|
#include <linux/interrupt.h>
|
|
#include <linux/extable.h>
|
|
#include <linux/uaccess.h>
|
|
|
|
extern void die_if_kernel(char *,struct pt_regs *,long, unsigned long *);
|
|
|
|
|
|
/*
|
|
* Force a new ASN for a task.
|
|
*/
|
|
|
|
#ifndef CONFIG_SMP
|
|
unsigned long last_asn = ASN_FIRST_VERSION;
|
|
#endif
|
|
|
|
void
|
|
__load_new_mm_context(struct mm_struct *next_mm)
|
|
{
|
|
unsigned long mmc;
|
|
struct pcb_struct *pcb;
|
|
|
|
mmc = __get_new_mm_context(next_mm, smp_processor_id());
|
|
next_mm->context[smp_processor_id()] = mmc;
|
|
|
|
pcb = ¤t_thread_info()->pcb;
|
|
pcb->asn = mmc & HARDWARE_ASN_MASK;
|
|
pcb->ptbr = ((unsigned long) next_mm->pgd - IDENT_ADDR) >> PAGE_SHIFT;
|
|
|
|
__reload_thread(pcb);
|
|
}
|
|
|
|
|
|
/*
|
|
* This routine handles page faults. It determines the address,
|
|
* and the problem, and then passes it off to handle_mm_fault().
|
|
*
|
|
* mmcsr:
|
|
* 0 = translation not valid
|
|
* 1 = access violation
|
|
* 2 = fault-on-read
|
|
* 3 = fault-on-execute
|
|
* 4 = fault-on-write
|
|
*
|
|
* cause:
|
|
* -1 = instruction fetch
|
|
* 0 = load
|
|
* 1 = store
|
|
*
|
|
* Registers $9 through $15 are saved in a block just prior to `regs' and
|
|
* are saved and restored around the call to allow exception code to
|
|
* modify them.
|
|
*/
|
|
|
|
/* Macro for exception fixup code to access integer registers. */
|
|
#define dpf_reg(r) \
|
|
(((unsigned long *)regs)[(r) <= 8 ? (r) : (r) <= 15 ? (r)-16 : \
|
|
(r) <= 18 ? (r)+10 : (r)-10])
|
|
|
|
asmlinkage void
|
|
do_page_fault(unsigned long address, unsigned long mmcsr,
|
|
long cause, struct pt_regs *regs)
|
|
{
|
|
struct vm_area_struct * vma;
|
|
struct mm_struct *mm = current->mm;
|
|
const struct exception_table_entry *fixup;
|
|
int si_code = SEGV_MAPERR;
|
|
vm_fault_t fault;
|
|
unsigned int flags = FAULT_FLAG_DEFAULT;
|
|
|
|
/* As of EV6, a load into $31/$f31 is a prefetch, and never faults
|
|
(or is suppressed by the PALcode). Support that for older CPUs
|
|
by ignoring such an instruction. */
|
|
if (cause == 0) {
|
|
unsigned int insn;
|
|
__get_user(insn, (unsigned int __user *)regs->pc);
|
|
if ((insn >> 21 & 0x1f) == 0x1f &&
|
|
/* ldq ldl ldt lds ldg ldf ldwu ldbu */
|
|
(1ul << (insn >> 26) & 0x30f00001400ul)) {
|
|
regs->pc += 4;
|
|
return;
|
|
}
|
|
}
|
|
|
|
/* If we're in an interrupt context, or have no user context,
|
|
we must not take the fault. */
|
|
if (!mm || faulthandler_disabled())
|
|
goto no_context;
|
|
|
|
#ifdef CONFIG_ALPHA_LARGE_VMALLOC
|
|
if (address >= TASK_SIZE)
|
|
goto vmalloc_fault;
|
|
#endif
|
|
if (user_mode(regs))
|
|
flags |= FAULT_FLAG_USER;
|
|
retry:
|
|
mmap_read_lock(mm);
|
|
vma = find_vma(mm, address);
|
|
if (!vma)
|
|
goto bad_area;
|
|
if (vma->vm_start <= address)
|
|
goto good_area;
|
|
if (!(vma->vm_flags & VM_GROWSDOWN))
|
|
goto bad_area;
|
|
if (expand_stack(vma, address))
|
|
goto bad_area;
|
|
|
|
/* Ok, we have a good vm_area for this memory access, so
|
|
we can handle it. */
|
|
good_area:
|
|
si_code = SEGV_ACCERR;
|
|
if (cause < 0) {
|
|
if (!(vma->vm_flags & VM_EXEC))
|
|
goto bad_area;
|
|
} else if (!cause) {
|
|
/* Allow reads even for write-only mappings */
|
|
if (!(vma->vm_flags & (VM_READ | VM_WRITE)))
|
|
goto bad_area;
|
|
} else {
|
|
if (!(vma->vm_flags & VM_WRITE))
|
|
goto bad_area;
|
|
flags |= FAULT_FLAG_WRITE;
|
|
}
|
|
|
|
/* If for any reason at all we couldn't handle the fault,
|
|
make sure we exit gracefully rather than endlessly redo
|
|
the fault. */
|
|
fault = handle_mm_fault(vma, address, flags, NULL);
|
|
|
|
if (fault_signal_pending(fault, regs))
|
|
return;
|
|
|
|
if (unlikely(fault & VM_FAULT_ERROR)) {
|
|
if (fault & VM_FAULT_OOM)
|
|
goto out_of_memory;
|
|
else if (fault & VM_FAULT_SIGSEGV)
|
|
goto bad_area;
|
|
else if (fault & VM_FAULT_SIGBUS)
|
|
goto do_sigbus;
|
|
BUG();
|
|
}
|
|
|
|
if (flags & FAULT_FLAG_ALLOW_RETRY) {
|
|
if (fault & VM_FAULT_MAJOR)
|
|
current->maj_flt++;
|
|
else
|
|
current->min_flt++;
|
|
if (fault & VM_FAULT_RETRY) {
|
|
flags |= FAULT_FLAG_TRIED;
|
|
|
|
/* No need to mmap_read_unlock(mm) as we would
|
|
* have already released it in __lock_page_or_retry
|
|
* in mm/filemap.c.
|
|
*/
|
|
|
|
goto retry;
|
|
}
|
|
}
|
|
|
|
mmap_read_unlock(mm);
|
|
|
|
return;
|
|
|
|
/* Something tried to access memory that isn't in our memory map.
|
|
Fix it, but check if it's kernel or user first. */
|
|
bad_area:
|
|
mmap_read_unlock(mm);
|
|
|
|
if (user_mode(regs))
|
|
goto do_sigsegv;
|
|
|
|
no_context:
|
|
/* Are we prepared to handle this fault as an exception? */
|
|
if ((fixup = search_exception_tables(regs->pc)) != 0) {
|
|
unsigned long newpc;
|
|
newpc = fixup_exception(dpf_reg, fixup, regs->pc);
|
|
regs->pc = newpc;
|
|
return;
|
|
}
|
|
|
|
/* Oops. The kernel tried to access some bad page. We'll have to
|
|
terminate things with extreme prejudice. */
|
|
printk(KERN_ALERT "Unable to handle kernel paging request at "
|
|
"virtual address %016lx\n", address);
|
|
die_if_kernel("Oops", regs, cause, (unsigned long*)regs - 16);
|
|
do_exit(SIGKILL);
|
|
|
|
/* We ran out of memory, or some other thing happened to us that
|
|
made us unable to handle the page fault gracefully. */
|
|
out_of_memory:
|
|
mmap_read_unlock(mm);
|
|
if (!user_mode(regs))
|
|
goto no_context;
|
|
pagefault_out_of_memory();
|
|
return;
|
|
|
|
do_sigbus:
|
|
mmap_read_unlock(mm);
|
|
/* Send a sigbus, regardless of whether we were in kernel
|
|
or user mode. */
|
|
force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *) address, 0);
|
|
if (!user_mode(regs))
|
|
goto no_context;
|
|
return;
|
|
|
|
do_sigsegv:
|
|
force_sig_fault(SIGSEGV, si_code, (void __user *) address, 0);
|
|
return;
|
|
|
|
#ifdef CONFIG_ALPHA_LARGE_VMALLOC
|
|
vmalloc_fault:
|
|
if (user_mode(regs))
|
|
goto do_sigsegv;
|
|
else {
|
|
/* Synchronize this task's top level page-table
|
|
with the "reference" page table from init. */
|
|
long index = pgd_index(address);
|
|
pgd_t *pgd, *pgd_k;
|
|
|
|
pgd = current->active_mm->pgd + index;
|
|
pgd_k = swapper_pg_dir + index;
|
|
if (!pgd_present(*pgd) && pgd_present(*pgd_k)) {
|
|
pgd_val(*pgd) = pgd_val(*pgd_k);
|
|
return;
|
|
}
|
|
goto no_context;
|
|
}
|
|
#endif
|
|
}
|