From e680cd14cc5e98ff1a3cd074d86eb8d929feb54c Mon Sep 17 00:00:00 2001 From: Michel Lespinasse Date: Mon, 8 Jun 2020 21:33:25 -0700 Subject: [PATCH] BACKPORT: mmap locking API: use coccinelle to convert mmap_sem rwsem call sites This change converts the existing mmap_sem rwsem calls to use the new mmap locking API instead. The change is generated using coccinelle with the following rule: // spatch --sp-file mmap_lock_api.cocci --in-place --include-headers --dir . @@ expression mm; @@ ( -init_rwsem +mmap_init_lock | -down_write +mmap_write_lock | -down_write_killable +mmap_write_lock_killable | -down_write_trylock +mmap_write_trylock | -up_write +mmap_write_unlock | -downgrade_write +mmap_write_downgrade | -down_read +mmap_read_lock | -down_read_killable +mmap_read_lock_killable | -down_read_trylock +mmap_read_trylock | -up_read +mmap_read_unlock ) -(&mm->mmap_sem) +(mm) Change-Id: If729000ea8cedab7079ccc1350db26ed71f0df92 Signed-off-by: Michel Lespinasse Signed-off-by: Andrew Morton Reviewed-by: Daniel Jordan Reviewed-by: Laurent Dufour Reviewed-by: Vlastimil Babka Cc: Davidlohr Bueso Cc: David Rientjes Cc: Hugh Dickins Cc: Jason Gunthorpe Cc: Jerome Glisse Cc: John Hubbard Cc: Liam Howlett Cc: Matthew Wilcox Cc: Peter Zijlstra Cc: Ying Han Link: http://lkml.kernel.org/r/20200520052908.204642-5-walken@google.com Signed-off-by: Linus Torvalds --- arch/alpha/kernel/traps.c | 4 +- arch/alpha/mm/fault.c | 10 ++--- arch/arc/kernel/process.c | 4 +- arch/arc/kernel/troubleshoot.c | 4 +- arch/arc/mm/fault.c | 10 ++--- arch/arm/kernel/process.c | 4 +- arch/arm/kernel/swp_emulate.c | 4 +- arch/arm/lib/uaccess_with_memcpy.c | 16 ++++---- arch/arm/mm/fault.c | 6 +-- arch/arm64/kernel/traps.c | 4 +- arch/arm64/kernel/vdso.c | 8 ++-- arch/arm64/mm/fault.c | 6 +-- arch/hexagon/kernel/vdso.c | 4 +- arch/hexagon/mm/vm_fault.c | 8 ++-- arch/ia64/kernel/perfmon.c | 8 ++-- arch/ia64/mm/fault.c | 8 ++-- arch/ia64/mm/init.c | 12 +++--- arch/m68k/kernel/sys_m68k.c | 16 ++++---- arch/m68k/mm/fault.c | 8 ++-- arch/microblaze/mm/fault.c | 12 +++--- arch/mips/kernel/traps.c | 4 +- arch/mips/kernel/vdso.c | 4 +- arch/mips/mm/c-octeon.c | 4 +- arch/mips/mm/c-r4k.c | 4 +- arch/nds32/kernel/vdso.c | 6 +-- arch/nds32/mm/fault.c | 12 +++--- arch/nios2/mm/fault.c | 12 +++--- arch/nios2/mm/init.c | 4 +- arch/openrisc/mm/fault.c | 10 ++--- arch/parisc/kernel/traps.c | 6 +-- arch/parisc/mm/fault.c | 8 ++-- arch/powerpc/kernel/vdso.c | 6 +-- arch/powerpc/kvm/book3s_64_mmu_hv.c | 4 +- arch/powerpc/kvm/book3s_64_vio.c | 4 +- arch/powerpc/kvm/book3s_hv.c | 6 +-- arch/powerpc/kvm/e500_mmu_host.c | 4 +- arch/powerpc/mm/copro_fault.c | 4 +- arch/powerpc/mm/fault.c | 12 +++--- arch/powerpc/mm/mmu_context_iommu.c | 4 +- arch/powerpc/mm/subpage-prot.c | 12 +++--- arch/powerpc/oprofile/cell/spu_task_sync.c | 6 +-- arch/powerpc/platforms/cell/spufs/file.c | 4 +- arch/riscv/kernel/vdso.c | 4 +- arch/riscv/mm/fault.c | 10 ++--- arch/s390/kernel/vdso.c | 4 +- arch/s390/kvm/gaccess.c | 4 +- arch/s390/kvm/kvm-s390.c | 24 +++++------ arch/s390/kvm/priv.c | 32 +++++++-------- arch/s390/mm/fault.c | 6 +-- arch/s390/mm/gmap.c | 40 +++++++++---------- arch/s390/pci/pci_mmio.c | 4 +- arch/sh/kernel/sys_sh.c | 6 +-- arch/sh/kernel/vsyscall/vsyscall.c | 4 +- arch/sh/mm/fault.c | 14 +++---- arch/sparc/mm/fault_32.c | 18 ++++----- arch/sparc/mm/fault_64.c | 12 +++--- arch/sparc/vdso/vma.c | 4 +- arch/um/include/asm/mmu_context.h | 2 +- arch/um/kernel/tlb.c | 2 +- arch/um/kernel/trap.c | 6 +-- arch/unicore32/mm/fault.c | 6 +-- arch/x86/entry/vdso/vma.c | 10 ++--- arch/x86/kernel/vm86_32.c | 4 +- arch/x86/mm/debug_pagetables.c | 8 ++-- arch/x86/mm/fault.c | 8 ++-- arch/x86/mm/mpx.c | 12 +++--- arch/x86/um/vdso/vma.c | 4 +- arch/xtensa/mm/fault.c | 10 ++--- drivers/android/binder_alloc.c | 10 ++--- drivers/char/adsprpc.c | 6 +-- drivers/fpga/dfl-afu-dma-region.c | 4 +- .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 4 +- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c | 6 +-- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 8 ++-- drivers/gpu/drm/amd/amdkfd/kfd_events.c | 4 +- drivers/gpu/drm/i915/i915_gem.c | 4 +- drivers/gpu/drm/i915/i915_gem_userptr.c | 8 ++-- drivers/gpu/drm/radeon/radeon_cs.c | 4 +- drivers/gpu/drm/radeon/radeon_gem.c | 6 +-- drivers/gpu/drm/radeon/radeon_mn.c | 6 +-- drivers/gpu/drm/ttm/ttm_bo_vm.c | 4 +- drivers/gpu/msm/kgsl.c | 22 +++++----- drivers/infiniband/core/rdma_core.c | 4 +- drivers/infiniband/core/umem.c | 26 ++++++------ drivers/infiniband/core/umem_odp.c | 10 ++--- drivers/infiniband/hw/hfi1/user_pages.c | 12 +++--- drivers/infiniband/hw/i40iw/i40iw_verbs.c | 4 +- drivers/infiniband/hw/mlx4/mr.c | 4 +- drivers/infiniband/hw/qib/qib_user_pages.c | 8 ++-- drivers/infiniband/hw/usnic/usnic_uiom.c | 14 +++---- drivers/iommu/amd_iommu_v2.c | 4 +- drivers/iommu/intel-svm.c | 4 +- drivers/media/v4l2-core/videobuf-core.c | 4 +- drivers/media/v4l2-core/videobuf-dma-contig.c | 4 +- drivers/media/v4l2-core/videobuf-dma-sg.c | 4 +- drivers/misc/cxl/cxllib.c | 4 +- drivers/misc/cxl/fault.c | 4 +- drivers/misc/mic/scif/scif_rma.c | 12 +++--- drivers/misc/sgi-gru/grufault.c | 16 ++++---- drivers/misc/sgi-gru/grufile.c | 4 +- drivers/misc/uid_sys_stats.c | 4 +- drivers/oprofile/buffer_sync.c | 10 ++--- drivers/tee/optee/call.c | 4 +- drivers/vfio/pci/vfio_pci.c | 8 ++-- drivers/vfio/vfio_iommu_spapr_tce.c | 8 ++-- drivers/vfio/vfio_iommu_type1.c | 12 +++--- drivers/xen/gntdev.c | 4 +- drivers/xen/privcmd.c | 14 +++---- fs/aio.c | 4 +- fs/coredump.c | 4 +- fs/exec.c | 16 ++++---- fs/proc/base.c | 18 ++++----- fs/proc/task_mmu.c | 40 +++++++++---------- fs/proc/task_nommu.c | 18 ++++----- fs/userfaultfd.c | 20 +++++----- ipc/shm.c | 8 ++-- kernel/acct.c | 4 +- kernel/bpf/stackmap.c | 4 +- kernel/events/core.c | 4 +- kernel/events/uprobes.c | 16 ++++---- kernel/exit.c | 8 ++-- kernel/fork.c | 12 +++--- kernel/futex.c | 4 +- kernel/sched/core.c | 4 +- kernel/sched/fair.c | 4 +- kernel/sys.c | 22 +++++----- kernel/trace/trace_output.c | 4 +- mm/filemap.c | 6 +-- mm/frame_vector.c | 4 +- mm/gup.c | 14 +++---- mm/internal.h | 2 +- mm/khugepaged.c | 32 +++++++-------- mm/ksm.c | 34 ++++++++-------- mm/madvise.c | 14 +++---- mm/memcontrol.c | 8 ++-- mm/memory.c | 10 ++--- mm/mempolicy.c | 20 +++++----- mm/migrate.c | 8 ++-- mm/mincore.c | 4 +- mm/mlock.c | 16 ++++---- mm/mmap.c | 30 +++++++------- mm/mmu_notifier.c | 4 +- mm/mprotect.c | 12 +++--- mm/mremap.c | 4 +- mm/msync.c | 8 ++-- mm/nommu.c | 20 +++++----- mm/oom_kill.c | 4 +- mm/process_vm_access.c | 4 +- mm/swapfile.c | 6 +-- mm/userfaultfd.c | 14 +++---- mm/util.c | 8 ++-- net/ipv4/tcp.c | 6 +-- net/xdp/xdp_umem.c | 4 +- virt/kvm/arm/mmu.c | 14 +++---- virt/kvm/async_pf.c | 4 +- virt/kvm/kvm_main.c | 8 ++-- 156 files changed, 696 insertions(+), 696 deletions(-) diff --git a/arch/alpha/kernel/traps.c b/arch/alpha/kernel/traps.c index 6bcef15aa4a6..aa3d74ec3f83 100644 --- a/arch/alpha/kernel/traps.c +++ b/arch/alpha/kernel/traps.c @@ -957,12 +957,12 @@ give_sigsegv: si_code = SEGV_ACCERR; else { struct mm_struct *mm = current->mm; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); if (find_vma(mm, (unsigned long)va)) si_code = SEGV_ACCERR; else si_code = SEGV_MAPERR; - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); } send_sig_fault(SIGSEGV, si_code, va, 0, current); return; diff --git a/arch/alpha/mm/fault.c b/arch/alpha/mm/fault.c index 10207c9193ec..1504c6c961ac 100644 --- a/arch/alpha/mm/fault.c +++ b/arch/alpha/mm/fault.c @@ -117,7 +117,7 @@ do_page_fault(unsigned long address, unsigned long mmcsr, if (user_mode(regs)) flags |= FAULT_FLAG_USER; retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma(mm, address); if (!vma) goto bad_area; @@ -180,14 +180,14 @@ retry: } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return; /* Something tried to access memory that isn't in our memory map. Fix it, but check if it's kernel or user first. */ bad_area: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (user_mode(regs)) goto do_sigsegv; @@ -211,14 +211,14 @@ retry: /* We ran out of memory, or some other thing happened to us that made us unable to handle the page fault gracefully. */ out_of_memory: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (!user_mode(regs)) goto no_context; pagefault_out_of_memory(); return; do_sigbus: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* Send a sigbus, regardless of whether we were in kernel or user mode. */ force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *) address, 0, current); diff --git a/arch/arc/kernel/process.c b/arch/arc/kernel/process.c index 8ce6e7235915..bafa725fb92a 100644 --- a/arch/arc/kernel/process.c +++ b/arch/arc/kernel/process.c @@ -91,10 +91,10 @@ fault: if (unlikely(ret != -EFAULT)) goto fail; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); ret = fixup_user_fault(current, current->mm, (unsigned long) uaddr, FAULT_FLAG_WRITE, NULL); - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); if (likely(!ret)) goto again; diff --git a/arch/arc/kernel/troubleshoot.c b/arch/arc/kernel/troubleshoot.c index 215f515442e0..fa070484a2e2 100644 --- a/arch/arc/kernel/troubleshoot.c +++ b/arch/arc/kernel/troubleshoot.c @@ -91,7 +91,7 @@ static void show_faulting_vma(unsigned long address) /* can't use print_vma_addr() yet as it doesn't check for * non-inclusive vma */ - down_read(&active_mm->mmap_sem); + mmap_read_lock(active_mm); vma = find_vma(active_mm, address); /* check against the find_vma( ) behaviour which returns the next VMA @@ -114,7 +114,7 @@ static void show_faulting_vma(unsigned long address) } else pr_info(" @No matching VMA found\n"); - up_read(&active_mm->mmap_sem); + mmap_read_unlock(active_mm); } static void show_ecr_verbose(struct pt_regs *regs) diff --git a/arch/arc/mm/fault.c b/arch/arc/mm/fault.c index 4e8143de32e7..5ecf6673cb6c 100644 --- a/arch/arc/mm/fault.c +++ b/arch/arc/mm/fault.c @@ -99,7 +99,7 @@ void do_page_fault(unsigned long address, struct pt_regs *regs) if (user_mode(regs)) flags |= FAULT_FLAG_USER; retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma(mm, address); if (!vma) goto bad_area; @@ -175,7 +175,7 @@ good_area: } /* Fault Handled Gracefully */ - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return; } @@ -194,7 +194,7 @@ good_area: * Fix it, but check if it's kernel or user first.. */ bad_area: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* User mode accesses just cause a SIGSEGV */ if (user_mode(regs)) { @@ -218,7 +218,7 @@ no_context: die("Oops", regs, address); out_of_memory: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (user_mode(regs)) { pagefault_out_of_memory(); @@ -228,7 +228,7 @@ out_of_memory: goto no_context; do_sigbus: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (!user_mode(regs)) goto no_context; diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index 45e5492a460e..832a562028cb 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -435,7 +435,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) npages = 1; /* for sigpage */ npages += vdso_total_pages; - if (down_write_killable(&mm->mmap_sem)) + if (mmap_write_lock_killable(mm)) return -EINTR; hint = sigpage_addr(mm, npages); addr = get_unmapped_area(NULL, hint, npages << PAGE_SHIFT, 0, 0); @@ -462,7 +462,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) arm_install_vdso(mm, addr + PAGE_SIZE); up_fail: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return ret; } #endif diff --git a/arch/arm/kernel/swp_emulate.c b/arch/arm/kernel/swp_emulate.c index 80517f293eb9..1466eb568675 100644 --- a/arch/arm/kernel/swp_emulate.c +++ b/arch/arm/kernel/swp_emulate.c @@ -101,12 +101,12 @@ static void set_segfault(struct pt_regs *regs, unsigned long addr) siginfo_t info; clear_siginfo(&info); - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); if (find_vma(current->mm, addr) == NULL) info.si_code = SEGV_MAPERR; else info.si_code = SEGV_ACCERR; - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); info.si_signo = SIGSEGV; info.si_errno = 0; diff --git a/arch/arm/lib/uaccess_with_memcpy.c b/arch/arm/lib/uaccess_with_memcpy.c index 73dc7360cbdd..81ca60e9b6a5 100644 --- a/arch/arm/lib/uaccess_with_memcpy.c +++ b/arch/arm/lib/uaccess_with_memcpy.c @@ -99,7 +99,7 @@ __copy_to_user_memcpy(void __user *to, const void *from, unsigned long n) atomic = faulthandler_disabled(); if (!atomic) - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); while (n) { pte_t *pte; spinlock_t *ptl; @@ -107,11 +107,11 @@ __copy_to_user_memcpy(void __user *to, const void *from, unsigned long n) while (!pin_page_for_write(to, &pte, &ptl)) { if (!atomic) - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); if (__put_user(0, (char __user *)to)) goto out; if (!atomic) - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); } tocopy = (~(unsigned long)to & ~PAGE_MASK) + 1; @@ -131,7 +131,7 @@ __copy_to_user_memcpy(void __user *to, const void *from, unsigned long n) spin_unlock(ptl); } if (!atomic) - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); out: return n; @@ -168,17 +168,17 @@ __clear_user_memset(void __user *addr, unsigned long n) return 0; } - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); while (n) { pte_t *pte; spinlock_t *ptl; int tocopy; while (!pin_page_for_write(addr, &pte, &ptl)) { - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); if (__put_user(0, (char __user *)addr)) goto out; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); } tocopy = (~(unsigned long)addr & ~PAGE_MASK) + 1; @@ -196,7 +196,7 @@ __clear_user_memset(void __user *addr, unsigned long n) else spin_unlock(ptl); } - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); out: return n; diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index a089fe9e3a17..6a244339b93f 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -302,11 +302,11 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) * validly references user space from well defined areas of the code, * we can bug out early if this is from code which shouldn't. */ - if (!down_read_trylock(&mm->mmap_sem)) { + if (!mmap_read_trylock(mm)) { if (!user_mode(regs) && !search_exception_tables(regs->ARM_pc)) goto no_context; retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); } else { /* * The above down_read_trylock() might have succeeded in @@ -359,7 +359,7 @@ retry: } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* * Handle the "normal" case first - VM_FAULT_MAJOR diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index fa7e9ad44823..32333f9590bb 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -417,12 +417,12 @@ void arm64_notify_segfault(unsigned long addr) { int code; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); if (find_vma(current->mm, addr) == NULL) code = SEGV_MAPERR; else code = SEGV_ACCERR; - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); force_signal_inject(SIGSEGV, code, addr); } diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c index 3c90b381ac21..d0b9cde567e2 100644 --- a/arch/arm64/kernel/vdso.c +++ b/arch/arm64/kernel/vdso.c @@ -357,7 +357,7 @@ int aarch32_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) struct mm_struct *mm = current->mm; int ret; - if (down_write_killable(&mm->mmap_sem)) + if (mmap_write_lock_killable(mm)) return -EINTR; ret = aarch32_kuser_helpers_setup(mm); @@ -374,7 +374,7 @@ int aarch32_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) #endif /* CONFIG_COMPAT_VDSO */ out: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return ret; } #endif /* CONFIG_COMPAT */ @@ -418,7 +418,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, struct mm_struct *mm = current->mm; int ret; - if (down_write_killable(&mm->mmap_sem)) + if (mmap_write_lock_killable(mm)) return -EINTR; ret = __setup_additional_pages(ARM64_VDSO, @@ -426,7 +426,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, bprm, uses_interp); - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return ret; } diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index d7c777f9fbbe..29a85a4de9f9 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -490,11 +490,11 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, * validly references user space from well defined areas of the code, * we can bug out early if this is from code which shouldn't. */ - if (!down_read_trylock(&mm->mmap_sem)) { + if (!mmap_read_trylock(mm)) { if (!user_mode(regs) && !search_exception_tables(regs->pc)) goto no_context; retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); } else { /* * The above down_read_trylock() might have succeeded in which @@ -523,7 +523,7 @@ retry: goto retry; } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* * Handle the "normal" (no error) case first. diff --git a/arch/hexagon/kernel/vdso.c b/arch/hexagon/kernel/vdso.c index 3ea968415539..7c1f8cbffadb 100644 --- a/arch/hexagon/kernel/vdso.c +++ b/arch/hexagon/kernel/vdso.c @@ -65,7 +65,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) unsigned long vdso_base; struct mm_struct *mm = current->mm; - if (down_write_killable(&mm->mmap_sem)) + if (mmap_write_lock_killable(mm)) return -EINTR; /* Try to get it loaded right near ld.so/glibc. */ @@ -89,7 +89,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) mm->context.vdso = (void *)vdso_base; up_fail: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return ret; } diff --git a/arch/hexagon/mm/vm_fault.c b/arch/hexagon/mm/vm_fault.c index fcf0fe73f36e..afec4323e977 100644 --- a/arch/hexagon/mm/vm_fault.c +++ b/arch/hexagon/mm/vm_fault.c @@ -68,7 +68,7 @@ void do_page_fault(unsigned long address, long cause, struct pt_regs *regs) if (user_mode(regs)) flags |= FAULT_FLAG_USER; retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma(mm, address); if (!vma) goto bad_area; @@ -121,11 +121,11 @@ good_area: } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return; } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* Handle copyin/out exception cases */ if (!user_mode(regs)) @@ -152,7 +152,7 @@ good_area: return; bad_area: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (user_mode(regs)) { force_sig_fault(SIGSEGV, si_code, (void __user *)address, current); diff --git a/arch/ia64/kernel/perfmon.c b/arch/ia64/kernel/perfmon.c index 8b52d25eb877..c450d990c191 100644 --- a/arch/ia64/kernel/perfmon.c +++ b/arch/ia64/kernel/perfmon.c @@ -2308,13 +2308,13 @@ pfm_smpl_buffer_alloc(struct task_struct *task, struct file *filp, pfm_context_t * now we atomically find some area in the address space and * remap the buffer in it. */ - down_write(&task->mm->mmap_sem); + mmap_write_lock(task->mm); /* find some free area in address space, must have mmap sem held */ vma->vm_start = get_unmapped_area(NULL, 0, size, 0, MAP_PRIVATE|MAP_ANONYMOUS); if (IS_ERR_VALUE(vma->vm_start)) { DPRINT(("Cannot find unmapped area for size %ld\n", size)); - up_write(&task->mm->mmap_sem); + mmap_write_unlock(task->mm); goto error; } vma->vm_end = vma->vm_start + size; @@ -2325,7 +2325,7 @@ pfm_smpl_buffer_alloc(struct task_struct *task, struct file *filp, pfm_context_t /* can only be applied to current task, need to have the mm semaphore held when called */ if (pfm_remap_buffer(vma, (unsigned long)smpl_buf, vma->vm_start, size)) { DPRINT(("Can't remap buffer\n")); - up_write(&task->mm->mmap_sem); + mmap_write_unlock(task->mm); goto error; } @@ -2336,7 +2336,7 @@ pfm_smpl_buffer_alloc(struct task_struct *task, struct file *filp, pfm_context_t insert_vm_struct(mm, vma); vm_stat_account(vma->vm_mm, vma->vm_flags, vma_pages(vma)); - up_write(&task->mm->mmap_sem); + mmap_write_unlock(task->mm); /* * keep track of user level virtual address diff --git a/arch/ia64/mm/fault.c b/arch/ia64/mm/fault.c index 2bd90a64f29b..4995916f8a34 100644 --- a/arch/ia64/mm/fault.c +++ b/arch/ia64/mm/fault.c @@ -124,7 +124,7 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re if (mask & VM_WRITE) flags |= FAULT_FLAG_WRITE; retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma_prev(mm, address, &prev_vma); if (!vma && !prev_vma ) @@ -201,7 +201,7 @@ retry: } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return; check_expansion: @@ -232,7 +232,7 @@ retry: goto good_area; bad_area: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); #ifdef CONFIG_VIRTUAL_MEM_MAP bad_area_no_up: #endif @@ -306,7 +306,7 @@ retry: return; out_of_memory: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (!user_mode(regs)) goto no_context; pagefault_out_of_memory(); diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c index 561e2573bd34..e677b6038418 100644 --- a/arch/ia64/mm/init.c +++ b/arch/ia64/mm/init.c @@ -121,13 +121,13 @@ ia64_init_addr_space (void) vma->vm_end = vma->vm_start + PAGE_SIZE; vma->vm_flags = VM_DATA_DEFAULT_FLAGS|VM_GROWSUP|VM_ACCOUNT; vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); - down_write(¤t->mm->mmap_sem); + mmap_write_lock(current->mm); if (insert_vm_struct(current->mm, vma)) { - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); vm_area_free(vma); return; } - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); } /* map NaT-page at address zero to speed up speculative dereferencing of NULL: */ @@ -139,13 +139,13 @@ ia64_init_addr_space (void) vma->vm_page_prot = __pgprot(pgprot_val(PAGE_READONLY) | _PAGE_MA_NAT); vma->vm_flags = VM_READ | VM_MAYREAD | VM_IO | VM_DONTEXPAND | VM_DONTDUMP; - down_write(¤t->mm->mmap_sem); + mmap_write_lock(current->mm); if (insert_vm_struct(current->mm, vma)) { - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); vm_area_free(vma); return; } - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); } } } diff --git a/arch/m68k/kernel/sys_m68k.c b/arch/m68k/kernel/sys_m68k.c index 38dcc1a2097d..ee357851361b 100644 --- a/arch/m68k/kernel/sys_m68k.c +++ b/arch/m68k/kernel/sys_m68k.c @@ -389,7 +389,7 @@ sys_cacheflush (unsigned long addr, int scope, int cache, unsigned long len) if (!capable(CAP_SYS_ADMIN)) goto out; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); } else { struct vm_area_struct *vma; @@ -401,7 +401,7 @@ sys_cacheflush (unsigned long addr, int scope, int cache, unsigned long len) * Verify that the specified address region actually belongs * to this process. */ - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); vma = find_vma(current->mm, addr); if (!vma || addr < vma->vm_start || addr + len > vma->vm_end) goto out_unlock; @@ -452,7 +452,7 @@ sys_cacheflush (unsigned long addr, int scope, int cache, unsigned long len) } } out_unlock: - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); out: return ret; } @@ -472,7 +472,7 @@ sys_atomic_cmpxchg_32(unsigned long newval, int oldval, int d3, int d4, int d5, spinlock_t *ptl; unsigned long mem_value; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); pgd = pgd_offset(mm, (unsigned long)mem); if (!pgd_present(*pgd)) goto bad_access; @@ -495,11 +495,11 @@ sys_atomic_cmpxchg_32(unsigned long newval, int oldval, int d3, int d4, int d5, __put_user(newval, mem); pte_unmap_unlock(pte, ptl); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return mem_value; bad_access: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* This is not necessarily a bad access, we can get here if a memory we're trying to write to should be copied-on-write. Make the kernel do the necessary page stuff, then re-iterate. @@ -539,13 +539,13 @@ sys_atomic_cmpxchg_32(unsigned long newval, int oldval, int d3, int d4, int d5, struct mm_struct *mm = current->mm; unsigned long mem_value; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); mem_value = *mem; if (mem_value == oldval) *mem = newval; - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return mem_value; } diff --git a/arch/m68k/mm/fault.c b/arch/m68k/mm/fault.c index c80191c795f6..87f6f0563b44 100644 --- a/arch/m68k/mm/fault.c +++ b/arch/m68k/mm/fault.c @@ -86,7 +86,7 @@ int do_page_fault(struct pt_regs *regs, unsigned long address, if (user_mode(regs)) flags |= FAULT_FLAG_USER; retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma(mm, address); if (!vma) @@ -177,7 +177,7 @@ good_area: } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return 0; /* @@ -185,7 +185,7 @@ good_area: * us unable to handle the page fault gracefully. */ out_of_memory: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (!user_mode(regs)) goto no_context; pagefault_out_of_memory(); @@ -214,6 +214,6 @@ acc_err: current->thread.faddr = address; send_sig: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return send_fault_sig(regs); } diff --git a/arch/microblaze/mm/fault.c b/arch/microblaze/mm/fault.c index b88d87e5d6ca..81a14352bd6f 100644 --- a/arch/microblaze/mm/fault.c +++ b/arch/microblaze/mm/fault.c @@ -137,12 +137,12 @@ void do_page_fault(struct pt_regs *regs, unsigned long address, * source. If this is invalid we can skip the address space check, * thus avoiding the deadlock. */ - if (unlikely(!down_read_trylock(&mm->mmap_sem))) { + if (unlikely(!mmap_read_trylock(mm))) { if (kernel_mode(regs) && !search_exception_tables(regs->pc)) goto bad_area_nosemaphore; retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); } vma = find_vma(mm, address); @@ -249,7 +249,7 @@ good_area: } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* * keep track of tlb+htab misses that are good addrs but @@ -260,7 +260,7 @@ good_area: return; bad_area: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); bad_area_nosemaphore: pte_errors++; @@ -279,7 +279,7 @@ bad_area_nosemaphore: * us unable to handle the page fault gracefully. */ out_of_memory: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (!user_mode(regs)) bad_page_fault(regs, address, SIGKILL); else @@ -287,7 +287,7 @@ out_of_memory: return; do_sigbus: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (user_mode(regs)) { force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address, current); return; diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index 01c85d37c47a..5a8890a51077 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -749,13 +749,13 @@ int process_fpemu_return(int sig, void __user *fault_addr, unsigned long fcr31) return 1; case SIGSEGV: - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); vma = find_vma(current->mm, (unsigned long)fault_addr); if (vma && (vma->vm_start <= (unsigned long)fault_addr)) si_code = SEGV_ACCERR; else si_code = SEGV_MAPERR; - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); force_sig_fault(SIGSEGV, si_code, fault_addr, current); return 1; diff --git a/arch/mips/kernel/vdso.c b/arch/mips/kernel/vdso.c index 157ee8045035..7b3d87460407 100644 --- a/arch/mips/kernel/vdso.c +++ b/arch/mips/kernel/vdso.c @@ -96,7 +96,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) struct vm_area_struct *vma; int ret; - if (down_write_killable(&mm->mmap_sem)) + if (mmap_write_lock_killable(mm)) return -EINTR; /* Map delay slot emulation page */ @@ -187,6 +187,6 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) ret = 0; out: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return ret; } diff --git a/arch/mips/mm/c-octeon.c b/arch/mips/mm/c-octeon.c index 0e45b061e514..4d0cf2097b78 100644 --- a/arch/mips/mm/c-octeon.c +++ b/arch/mips/mm/c-octeon.c @@ -137,10 +137,10 @@ static void octeon_flush_cache_sigtramp(unsigned long addr) { struct vm_area_struct *vma; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); vma = find_vma(current->mm, addr); octeon_flush_icache_all_cores(vma); - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); } diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index 60fe72170856..5431f1279072 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -1034,7 +1034,7 @@ static void r4k_flush_cache_sigtramp(unsigned long addr) struct flush_cache_sigtramp_args args; int npages; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); npages = get_user_pages_fast(addr, 1, 0, &args.page); if (npages < 1) @@ -1047,7 +1047,7 @@ static void r4k_flush_cache_sigtramp(unsigned long addr) put_page(args.page); out: - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); } static void r4k_flush_icache_all(void) diff --git a/arch/nds32/kernel/vdso.c b/arch/nds32/kernel/vdso.c index 90bcae6f8554..e16009a07971 100644 --- a/arch/nds32/kernel/vdso.c +++ b/arch/nds32/kernel/vdso.c @@ -130,7 +130,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) vdso_mapping_len += L1_cache_info[DCACHE].aliasing_num - 1; #endif - if (down_write_killable(&mm->mmap_sem)) + if (mmap_write_lock_killable(mm)) return -EINTR; addr = vdso_random_addr(vdso_mapping_len); @@ -185,12 +185,12 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) goto up_fail; } - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return 0; up_fail: mm->context.vdso = NULL; - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return ret; } diff --git a/arch/nds32/mm/fault.c b/arch/nds32/mm/fault.c index 0a69dcf91ae2..89e4086ff028 100644 --- a/arch/nds32/mm/fault.c +++ b/arch/nds32/mm/fault.c @@ -122,12 +122,12 @@ void do_page_fault(unsigned long entry, unsigned long addr, * validly references user space from well defined areas of the code, * we can bug out early if this is from code which shouldn't. */ - if (unlikely(!down_read_trylock(&mm->mmap_sem))) { + if (unlikely(!mmap_read_trylock(mm))) { if (!user_mode(regs) && !search_exception_tables(instruction_pointer(regs))) goto no_context; retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); } else { /* * The above down_read_trylock() might have succeeded in which @@ -248,7 +248,7 @@ good_area: } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return; /* @@ -256,7 +256,7 @@ good_area: * Fix it, but check if it's kernel or user first.. */ bad_area: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); bad_area_nosemaphore: @@ -316,14 +316,14 @@ no_context: */ out_of_memory: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (!user_mode(regs)) goto no_context; pagefault_out_of_memory(); return; do_sigbus: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* Kernel mode? Handle exceptions or die */ if (!user_mode(regs)) diff --git a/arch/nios2/mm/fault.c b/arch/nios2/mm/fault.c index a1f2af87b614..ecba152870e1 100644 --- a/arch/nios2/mm/fault.c +++ b/arch/nios2/mm/fault.c @@ -84,11 +84,11 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long cause, if (user_mode(regs)) flags |= FAULT_FLAG_USER; - if (!down_read_trylock(&mm->mmap_sem)) { + if (!mmap_read_trylock(mm)) { if (!user_mode(regs) && !search_exception_tables(regs->ea)) goto bad_area_nosemaphore; retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); } vma = find_vma(mm, address); @@ -173,7 +173,7 @@ good_area: } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return; /* @@ -181,7 +181,7 @@ good_area: * Fix it, but check if it's kernel or user first.. */ bad_area: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); bad_area_nosemaphore: /* User mode accesses just cause a SIGSEGV */ @@ -219,14 +219,14 @@ no_context: * us unable to handle the page fault gracefully. */ out_of_memory: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (!user_mode(regs)) goto no_context; pagefault_out_of_memory(); return; do_sigbus: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* Kernel mode? Handle exceptions or die */ if (!user_mode(regs)) diff --git a/arch/nios2/mm/init.c b/arch/nios2/mm/init.c index c92fe4234009..9c6c33e5dcff 100644 --- a/arch/nios2/mm/init.c +++ b/arch/nios2/mm/init.c @@ -124,14 +124,14 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) struct mm_struct *mm = current->mm; int ret; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); /* Map kuser helpers to user space address */ ret = install_special_mapping(mm, KUSER_BASE, KUSER_SIZE, VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYEXEC, kuser_page); - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return ret; } diff --git a/arch/openrisc/mm/fault.c b/arch/openrisc/mm/fault.c index f640745ecf37..357788b02e68 100644 --- a/arch/openrisc/mm/fault.c +++ b/arch/openrisc/mm/fault.c @@ -108,7 +108,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long address, goto no_context; retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma(mm, address); if (!vma) @@ -197,7 +197,7 @@ good_area: } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return; /* @@ -206,7 +206,7 @@ good_area: */ bad_area: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); bad_area_nosemaphore: @@ -265,14 +265,14 @@ out_of_memory: __asm__ __volatile__("l.nop 42"); __asm__ __volatile__("l.nop 1"); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (!user_mode(regs)) goto no_context; pagefault_out_of_memory(); return; do_sigbus: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* * Send a sigbus, regardless of whether we were in kernel diff --git a/arch/parisc/kernel/traps.c b/arch/parisc/kernel/traps.c index 5d9cf726e4ec..9ba3f5cc6279 100644 --- a/arch/parisc/kernel/traps.c +++ b/arch/parisc/kernel/traps.c @@ -684,7 +684,7 @@ void notrace handle_interruption(int code, struct pt_regs *regs) if (user_mode(regs)) { struct vm_area_struct *vma; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); vma = find_vma(current->mm,regs->iaoq[0]); if (vma && (regs->iaoq[0] >= vma->vm_start) && (vma->vm_flags & VM_EXEC)) { @@ -692,10 +692,10 @@ void notrace handle_interruption(int code, struct pt_regs *regs) fault_address = regs->iaoq[0]; fault_space = regs->iasq[0]; - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); break; /* call do_page_fault() */ } - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); } /* Fall Through */ case 27: diff --git a/arch/parisc/mm/fault.c b/arch/parisc/mm/fault.c index f1725498f102..7d712aafcf45 100644 --- a/arch/parisc/mm/fault.c +++ b/arch/parisc/mm/fault.c @@ -281,7 +281,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long code, if (acc_type & VM_WRITE) flags |= FAULT_FLAG_WRITE; retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma_prev(mm, address, &prev_vma); if (!vma || address < vma->vm_start) goto check_expansion; @@ -338,7 +338,7 @@ good_area: goto retry; } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return; check_expansion: @@ -350,7 +350,7 @@ check_expansion: * Something tried to access memory that isn't in our memory map.. */ bad_area: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (user_mode(regs)) { int signo, si_code; @@ -422,7 +422,7 @@ no_context: parisc_terminate("Bad Address (null pointer deref?)", regs, code, address); out_of_memory: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (!user_mode(regs)) goto no_context; pagefault_out_of_memory(); diff --git a/arch/powerpc/kernel/vdso.c b/arch/powerpc/kernel/vdso.c index 31ab6eb61e26..11e38488d60f 100644 --- a/arch/powerpc/kernel/vdso.c +++ b/arch/powerpc/kernel/vdso.c @@ -197,7 +197,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) * and end up putting it elsewhere. * Add enough to the size so that the result can be aligned. */ - if (down_write_killable(&mm->mmap_sem)) + if (mmap_write_lock_killable(mm)) return -EINTR; vdso_base = get_unmapped_area(NULL, vdso_base, (vdso_pages << PAGE_SHIFT) + @@ -237,11 +237,11 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) goto fail_mmapsem; } - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return 0; fail_mmapsem: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return rc; } diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c index a488c105b923..9da8970e3ebf 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_hv.c +++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c @@ -586,7 +586,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, npages = get_user_pages_fast(hva, 1, writing, pages); if (npages < 1) { /* Check if it's an I/O mapping */ - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); vma = find_vma(current->mm, hva); if (vma && vma->vm_start <= hva && hva + psize <= vma->vm_end && (vma->vm_flags & VM_PFNMAP)) { @@ -596,7 +596,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, is_ci = pte_ci(__pte((pgprot_val(vma->vm_page_prot)))); write_ok = vma->vm_flags & VM_WRITE; } - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); if (!pfn) goto out_put; } else { diff --git a/arch/powerpc/kvm/book3s_64_vio.c b/arch/powerpc/kvm/book3s_64_vio.c index 26b03af71abd..06b24dea42f4 100644 --- a/arch/powerpc/kvm/book3s_64_vio.c +++ b/arch/powerpc/kvm/book3s_64_vio.c @@ -63,7 +63,7 @@ static long kvmppc_account_memlimit(unsigned long stt_pages, bool inc) if (!current || !current->mm) return ret; /* process exited */ - down_write(¤t->mm->mmap_sem); + mmap_write_lock(current->mm); if (inc) { unsigned long locked, lock_limit; @@ -88,7 +88,7 @@ static long kvmppc_account_memlimit(unsigned long stt_pages, bool inc) rlimit(RLIMIT_MEMLOCK), ret ? " - exceeded" : ""); - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); return ret; } diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index 5dc592fb4f5f..088b70cc91bb 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -3781,14 +3781,14 @@ static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu) /* Look up the VMA for the start of this memory slot */ hva = memslot->userspace_addr; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); vma = find_vma(current->mm, hva); if (!vma || vma->vm_start > hva || (vma->vm_flags & VM_IO)) goto up_out; psize = vma_kernel_pagesize(vma); - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); /* We can handle 4k, 64k or 16M pages in the VRMA */ if (psize >= 0x1000000) @@ -3821,7 +3821,7 @@ static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu) return err; up_out: - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); goto out_srcu; } diff --git a/arch/powerpc/kvm/e500_mmu_host.c b/arch/powerpc/kvm/e500_mmu_host.c index bbb02195dc53..2773f7e328ba 100644 --- a/arch/powerpc/kvm/e500_mmu_host.c +++ b/arch/powerpc/kvm/e500_mmu_host.c @@ -358,7 +358,7 @@ static inline int kvmppc_e500_shadow_map(struct kvmppc_vcpu_e500 *vcpu_e500, if (tlbsel == 1) { struct vm_area_struct *vma; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); vma = find_vma(current->mm, hva); if (vma && hva >= vma->vm_start && @@ -444,7 +444,7 @@ static inline int kvmppc_e500_shadow_map(struct kvmppc_vcpu_e500 *vcpu_e500, tsize = max(BOOK3E_PAGESZ_4K, tsize & ~1); } - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); } if (likely(!pfnmap)) { diff --git a/arch/powerpc/mm/copro_fault.c b/arch/powerpc/mm/copro_fault.c index c8da352e8686..87edde9d31ab 100644 --- a/arch/powerpc/mm/copro_fault.c +++ b/arch/powerpc/mm/copro_fault.c @@ -46,7 +46,7 @@ int copro_handle_mm_fault(struct mm_struct *mm, unsigned long ea, if (mm->pgd == NULL) return -EFAULT; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); ret = -EFAULT; vma = find_vma(mm, ea); if (!vma) @@ -95,7 +95,7 @@ int copro_handle_mm_fault(struct mm_struct *mm, unsigned long ea, current->min_flt++; out_unlock: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return ret; } EXPORT_SYMBOL_GPL(copro_handle_mm_fault); diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index eb5252177b66..12eb5551654f 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -133,7 +133,7 @@ static int __bad_area(struct pt_regs *regs, unsigned long address, int si_code, * Something tried to access memory that isn't in our memory map.. * Fix it, but check if it's kernel or user first.. */ - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return __bad_area_nosemaphore(regs, address, si_code, pkey); } @@ -494,12 +494,12 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address, * source. If this is invalid we can skip the address space check, * thus avoiding the deadlock. */ - if (unlikely(!down_read_trylock(&mm->mmap_sem))) { + if (unlikely(!mmap_read_trylock(mm))) { if (!is_user && !search_exception_tables(regs->nip)) return bad_area_nosemaphore(regs, address); retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); } else { /* * The above down_read_trylock() might have succeeded in @@ -523,7 +523,7 @@ retry: if (!must_retry) return bad_area(regs, address); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (fault_in_pages_readable((const char __user *)regs->nip, sizeof(unsigned int))) return bad_area_nosemaphore(regs, address); @@ -555,7 +555,7 @@ good_area: int pkey = vma_pkey(vma); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return bad_key_fault_exception(regs, address, pkey); } #endif /* CONFIG_PPC_MEM_KEYS */ @@ -586,7 +586,7 @@ good_area: return is_user ? 0 : SIGBUS; } - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); if (unlikely(fault & VM_FAULT_ERROR)) return mm_fault_error(regs, address, fault); diff --git a/arch/powerpc/mm/mmu_context_iommu.c b/arch/powerpc/mm/mmu_context_iommu.c index 56c2234cc6ae..35023fe05254 100644 --- a/arch/powerpc/mm/mmu_context_iommu.c +++ b/arch/powerpc/mm/mmu_context_iommu.c @@ -46,7 +46,7 @@ static long mm_iommu_adjust_locked_vm(struct mm_struct *mm, if (!npages) return 0; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); if (incr) { locked = mm->locked_vm + npages; @@ -67,7 +67,7 @@ static long mm_iommu_adjust_locked_vm(struct mm_struct *mm, npages << PAGE_SHIFT, mm->locked_vm << PAGE_SHIFT, rlimit(RLIMIT_MEMLOCK)); - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return ret; } diff --git a/arch/powerpc/mm/subpage-prot.c b/arch/powerpc/mm/subpage-prot.c index 3327551c8b47..d733f0c31a71 100644 --- a/arch/powerpc/mm/subpage-prot.c +++ b/arch/powerpc/mm/subpage-prot.c @@ -99,7 +99,7 @@ static void subpage_prot_clear(unsigned long addr, unsigned long len) size_t nw; unsigned long next, limit; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); limit = addr + len; if (limit > spt->maxaddr) limit = spt->maxaddr; @@ -127,7 +127,7 @@ static void subpage_prot_clear(unsigned long addr, unsigned long len) /* now flush any existing HPTEs for the range */ hpte_flush_range(mm, addr, nw); } - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); } #ifdef CONFIG_TRANSPARENT_HUGEPAGE @@ -217,7 +217,7 @@ SYSCALL_DEFINE3(subpage_prot, unsigned long, addr, if (!access_ok(VERIFY_READ, map, (len >> PAGE_SHIFT) * sizeof(u32))) return -EFAULT; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); subpage_mark_vma_nohuge(mm, addr, len); for (limit = addr + len; addr < limit; addr = next) { next = pmd_addr_end(addr, limit); @@ -252,11 +252,11 @@ SYSCALL_DEFINE3(subpage_prot, unsigned long, addr, if (addr + (nw << PAGE_SHIFT) > next) nw = (next - addr) >> PAGE_SHIFT; - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); if (__copy_from_user(spp, map, nw * sizeof(u32))) return -EFAULT; map += nw; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); /* now flush any existing HPTEs for the range */ hpte_flush_range(mm, addr, nw); @@ -265,6 +265,6 @@ SYSCALL_DEFINE3(subpage_prot, unsigned long, addr, spt->maxaddr = limit; err = 0; out: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return err; } diff --git a/arch/powerpc/oprofile/cell/spu_task_sync.c b/arch/powerpc/oprofile/cell/spu_task_sync.c index 2668cc414e4e..c2ea772fbaab 100644 --- a/arch/powerpc/oprofile/cell/spu_task_sync.c +++ b/arch/powerpc/oprofile/cell/spu_task_sync.c @@ -336,7 +336,7 @@ get_exec_dcookie_and_offset(struct spu *spu, unsigned int *offsetp, fput(exe_file); } - down_read(&mm->mmap_sem); + mmap_read_lock(mm); for (vma = mm->mmap; vma; vma = vma->vm_next) { if (vma->vm_start > spu_ref || vma->vm_end <= spu_ref) continue; @@ -353,13 +353,13 @@ get_exec_dcookie_and_offset(struct spu *spu, unsigned int *offsetp, *spu_bin_dcookie = fast_get_dcookie(&vma->vm_file->f_path); pr_debug("got dcookie for %pD\n", vma->vm_file); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); out: return app_cookie; fail_no_image_cookie: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); printk(KERN_ERR "SPU_PROF: " "%s, line %d: Cannot find dcookie for SPU binary\n", diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c index d16adcd93921..0f4d5dbbd97a 100644 --- a/arch/powerpc/platforms/cell/spufs/file.c +++ b/arch/powerpc/platforms/cell/spufs/file.c @@ -349,11 +349,11 @@ static vm_fault_t spufs_ps_fault(struct vm_fault *vmf, goto refault; if (ctx->state == SPU_STATE_SAVED) { - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); spu_context_nospu_trace(spufs_ps_fault__sleep, ctx); err = spufs_wait(ctx->run_wq, ctx->state == SPU_STATE_RUNNABLE); spu_context_trace(spufs_ps_fault__wake, ctx, ctx->spu); - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); } else { area = ctx->spu->problem_phys + ps_offs; ret = vmf_insert_pfn(vmf->vma, vmf->address, diff --git a/arch/riscv/kernel/vdso.c b/arch/riscv/kernel/vdso.c index 582cb153eb24..dda3f623390f 100644 --- a/arch/riscv/kernel/vdso.c +++ b/arch/riscv/kernel/vdso.c @@ -72,7 +72,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, vdso_len = (vdso_pages + 1) << PAGE_SHIFT; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); vdso_base = get_unmapped_area(NULL, 0, vdso_len, 0, 0); if (IS_ERR_VALUE(vdso_base)) { ret = vdso_base; @@ -94,7 +94,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, mm->context.vdso = NULL; end: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return ret; } diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c index ac5d7af9708b..98b821e79470 100644 --- a/arch/riscv/mm/fault.c +++ b/arch/riscv/mm/fault.c @@ -80,7 +80,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs) perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma(mm, addr); if (unlikely(!vma)) goto bad_area; @@ -171,7 +171,7 @@ good_area: } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return; /* @@ -179,7 +179,7 @@ good_area: * Fix it, but check if it's kernel or user first. */ bad_area: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* User mode accesses just cause a SIGSEGV */ if (user_mode(regs)) { do_trap(regs, SIGSEGV, code, addr, tsk); @@ -207,14 +207,14 @@ no_context: * (which will retry the fault, or kill us if we got oom-killed). */ out_of_memory: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (!user_mode(regs)) goto no_context; pagefault_out_of_memory(); return; do_sigbus: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* Kernel mode? Handle exceptions or die */ if (!user_mode(regs)) goto no_context; diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c index 7ab7d256d1eb..89d55bae282f 100644 --- a/arch/s390/kernel/vdso.c +++ b/arch/s390/kernel/vdso.c @@ -240,7 +240,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) * it at vdso_base which is the "natural" base for it, but we might * fail and end up putting it elsewhere. */ - if (down_write_killable(&mm->mmap_sem)) + if (mmap_write_lock_killable(mm)) return -EINTR; vdso_base = get_unmapped_area(NULL, 0, vdso_pages << PAGE_SHIFT, 0, 0); if (IS_ERR_VALUE(vdso_base)) { @@ -271,7 +271,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) rc = 0; out_up: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return rc; } diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c index 11ddac5e3e92..b5b67c6db980 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/gaccess.c @@ -1200,7 +1200,7 @@ int kvm_s390_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg, int dat_protection, fake; int rc; - down_read(&sg->mm->mmap_sem); + mmap_read_lock(sg->mm); /* * We don't want any guest-2 tables to change - so the parent * tables/pointers we read stay valid - unshadowing is however @@ -1229,6 +1229,6 @@ shadow_page: if (!rc) rc = gmap_shadow_page(sg, saddr, __pte(pte.val)); ipte_unlock(vcpu); - up_read(&sg->mm->mmap_sem); + mmap_read_unlock(sg->mm); return rc; } diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index bc700cb9fc53..e5c59e09ac48 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -707,9 +707,9 @@ static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) r = -EINVAL; else { r = 0; - down_write(&kvm->mm->mmap_sem); + mmap_write_lock(kvm->mm); kvm->mm->context.allow_gmap_hpage_1m = 1; - up_write(&kvm->mm->mmap_sem); + mmap_write_unlock(kvm->mm); /* * We might have to create fake 4k page * tables. To avoid that the hardware works on @@ -1542,7 +1542,7 @@ static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) if (!keys) return -ENOMEM; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); srcu_idx = srcu_read_lock(&kvm->srcu); for (i = 0; i < args->count; i++) { hva = gfn_to_hva(kvm, args->start_gfn + i); @@ -1556,7 +1556,7 @@ static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) break; } srcu_read_unlock(&kvm->srcu, srcu_idx); - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); if (!r) { r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys, @@ -1600,7 +1600,7 @@ static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) goto out; i = 0; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); srcu_idx = srcu_read_lock(&kvm->srcu); while (i < args->count) { unlocked = false; @@ -1627,7 +1627,7 @@ static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) i++; } srcu_read_unlock(&kvm->srcu, srcu_idx); - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); out: kvfree(keys); return r; @@ -1817,14 +1817,14 @@ static int kvm_s390_get_cmma_bits(struct kvm *kvm, if (!values) return -ENOMEM; - down_read(&kvm->mm->mmap_sem); + mmap_read_lock(kvm->mm); srcu_idx = srcu_read_lock(&kvm->srcu); if (peek) ret = kvm_s390_peek_cmma(kvm, args, values, bufsize); else ret = kvm_s390_get_cmma(kvm, args, values, bufsize); srcu_read_unlock(&kvm->srcu, srcu_idx); - up_read(&kvm->mm->mmap_sem); + mmap_read_unlock(kvm->mm); if (kvm->arch.migration_mode) args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages); @@ -1874,7 +1874,7 @@ static int kvm_s390_set_cmma_bits(struct kvm *kvm, goto out; } - down_read(&kvm->mm->mmap_sem); + mmap_read_lock(kvm->mm); srcu_idx = srcu_read_lock(&kvm->srcu); for (i = 0; i < args->count; i++) { hva = gfn_to_hva(kvm, args->start_gfn + i); @@ -1889,12 +1889,12 @@ static int kvm_s390_set_cmma_bits(struct kvm *kvm, set_pgste_bits(kvm->mm, hva, mask, pgstev); } srcu_read_unlock(&kvm->srcu, srcu_idx); - up_read(&kvm->mm->mmap_sem); + mmap_read_unlock(kvm->mm); if (!kvm->mm->context.uses_cmm) { - down_write(&kvm->mm->mmap_sem); + mmap_write_lock(kvm->mm); kvm->mm->context.uses_cmm = 1; - up_write(&kvm->mm->mmap_sem); + mmap_write_unlock(kvm->mm); } out: vfree(bits); diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c index 9abdfb8b1a1e..4cdca4402bda 100644 --- a/arch/s390/kvm/priv.c +++ b/arch/s390/kvm/priv.c @@ -269,18 +269,18 @@ static int handle_iske(struct kvm_vcpu *vcpu) return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); retry: unlocked = false; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); rc = get_guest_storage_key(current->mm, vmaddr, &key); if (rc) { rc = fixup_user_fault(current, current->mm, vmaddr, FAULT_FLAG_WRITE, &unlocked); if (!rc) { - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); goto retry; } } - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); if (rc == -EFAULT) return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); if (rc < 0) @@ -316,17 +316,17 @@ static int handle_rrbe(struct kvm_vcpu *vcpu) return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); retry: unlocked = false; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); rc = reset_guest_reference_bit(current->mm, vmaddr); if (rc < 0) { rc = fixup_user_fault(current, current->mm, vmaddr, FAULT_FLAG_WRITE, &unlocked); if (!rc) { - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); goto retry; } } - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); if (rc == -EFAULT) return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); if (rc < 0) @@ -384,7 +384,7 @@ static int handle_sske(struct kvm_vcpu *vcpu) if (kvm_is_error_hva(vmaddr)) return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); rc = cond_set_guest_storage_key(current->mm, vmaddr, key, &oldkey, m3 & SSKE_NQ, m3 & SSKE_MR, m3 & SSKE_MC); @@ -394,7 +394,7 @@ static int handle_sske(struct kvm_vcpu *vcpu) FAULT_FLAG_WRITE, &unlocked); rc = !rc ? -EAGAIN : rc; } - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); if (rc == -EFAULT) return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); if (rc == -EAGAIN) @@ -1000,7 +1000,7 @@ static int handle_pfmf(struct kvm_vcpu *vcpu) if (rc) return rc; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); rc = cond_set_guest_storage_key(current->mm, vmaddr, key, NULL, nq, mr, mc); if (rc < 0) { @@ -1008,7 +1008,7 @@ static int handle_pfmf(struct kvm_vcpu *vcpu) FAULT_FLAG_WRITE, &unlocked); rc = !rc ? -EAGAIN : rc; } - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); if (rc == -EFAULT) return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); if (rc == -EAGAIN) @@ -1129,9 +1129,9 @@ static int handle_essa(struct kvm_vcpu *vcpu) * already correct, we do nothing and avoid the lock. */ if (vcpu->kvm->mm->context.uses_cmm == 0) { - down_write(&vcpu->kvm->mm->mmap_sem); + mmap_write_lock(vcpu->kvm->mm); vcpu->kvm->mm->context.uses_cmm = 1; - up_write(&vcpu->kvm->mm->mmap_sem); + mmap_write_unlock(vcpu->kvm->mm); } /* * If we are here, we are supposed to have CMMA enabled in @@ -1148,11 +1148,11 @@ static int handle_essa(struct kvm_vcpu *vcpu) } else { int srcu_idx; - down_read(&vcpu->kvm->mm->mmap_sem); + mmap_read_lock(vcpu->kvm->mm); srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); i = __do_essa(vcpu, orc); srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx); - up_read(&vcpu->kvm->mm->mmap_sem); + mmap_read_unlock(vcpu->kvm->mm); if (i < 0) return i; /* Account for the possible extra cbrl entry */ @@ -1160,10 +1160,10 @@ static int handle_essa(struct kvm_vcpu *vcpu) } vcpu->arch.sie_block->cbrlo &= PAGE_MASK; /* reset nceo */ cbrlo = phys_to_virt(vcpu->arch.sie_block->cbrlo); - down_read(&gmap->mm->mmap_sem); + mmap_read_lock(gmap->mm); for (i = 0; i < entries; ++i) __gmap_zap(gmap, cbrlo[i]); - up_read(&gmap->mm->mmap_sem); + mmap_read_unlock(gmap->mm); return 0; } diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 1d8d12efc265..44230c266729 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -459,7 +459,7 @@ static inline vm_fault_t do_exception(struct pt_regs *regs, int access) access = VM_WRITE; if (access == VM_WRITE) flags |= FAULT_FLAG_WRITE; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); gmap = NULL; if (IS_ENABLED(CONFIG_PGSTE) && type == GMAP_FAULT) { @@ -543,7 +543,7 @@ retry: flags &= ~(FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT); flags |= FAULT_FLAG_TRIED; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); goto retry; } } @@ -561,7 +561,7 @@ retry: } fault = 0; out_up: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); out: return fault; } diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index 3c1217d34109..2c1720965ba4 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -405,10 +405,10 @@ int gmap_unmap_segment(struct gmap *gmap, unsigned long to, unsigned long len) return -EINVAL; flush = 0; - down_write(&gmap->mm->mmap_sem); + mmap_write_lock(gmap->mm); for (off = 0; off < len; off += PMD_SIZE) flush |= __gmap_unmap_by_gaddr(gmap, to + off); - up_write(&gmap->mm->mmap_sem); + mmap_write_unlock(gmap->mm); if (flush) gmap_flush_tlb(gmap); return 0; @@ -438,7 +438,7 @@ int gmap_map_segment(struct gmap *gmap, unsigned long from, return -EINVAL; flush = 0; - down_write(&gmap->mm->mmap_sem); + mmap_write_lock(gmap->mm); for (off = 0; off < len; off += PMD_SIZE) { /* Remove old translation */ flush |= __gmap_unmap_by_gaddr(gmap, to + off); @@ -448,7 +448,7 @@ int gmap_map_segment(struct gmap *gmap, unsigned long from, (void *) from + off)) break; } - up_write(&gmap->mm->mmap_sem); + mmap_write_unlock(gmap->mm); if (flush) gmap_flush_tlb(gmap); if (off >= len) @@ -495,9 +495,9 @@ unsigned long gmap_translate(struct gmap *gmap, unsigned long gaddr) { unsigned long rc; - down_read(&gmap->mm->mmap_sem); + mmap_read_lock(gmap->mm); rc = __gmap_translate(gmap, gaddr); - up_read(&gmap->mm->mmap_sem); + mmap_read_unlock(gmap->mm); return rc; } EXPORT_SYMBOL_GPL(gmap_translate); @@ -640,7 +640,7 @@ int gmap_fault(struct gmap *gmap, unsigned long gaddr, int rc; bool unlocked; - down_read(&gmap->mm->mmap_sem); + mmap_read_lock(gmap->mm); retry: unlocked = false; @@ -663,7 +663,7 @@ retry: rc = __gmap_link(gmap, gaddr, vmaddr); out_up: - up_read(&gmap->mm->mmap_sem); + mmap_read_unlock(gmap->mm); return rc; } EXPORT_SYMBOL_GPL(gmap_fault); @@ -697,7 +697,7 @@ void gmap_discard(struct gmap *gmap, unsigned long from, unsigned long to) unsigned long gaddr, vmaddr, size; struct vm_area_struct *vma; - down_read(&gmap->mm->mmap_sem); + mmap_read_lock(gmap->mm); for (gaddr = from; gaddr < to; gaddr = (gaddr + PMD_SIZE) & PMD_MASK) { /* Find the vm address for the guest address */ @@ -720,7 +720,7 @@ void gmap_discard(struct gmap *gmap, unsigned long from, unsigned long to) size = min(to - gaddr, PMD_SIZE - (gaddr & ~PMD_MASK)); zap_page_range(vma, vmaddr, size); } - up_read(&gmap->mm->mmap_sem); + mmap_read_unlock(gmap->mm); } EXPORT_SYMBOL_GPL(gmap_discard); @@ -1101,9 +1101,9 @@ int gmap_mprotect_notify(struct gmap *gmap, unsigned long gaddr, return -EINVAL; if (!MACHINE_HAS_ESOP && prot == PROT_READ) return -EINVAL; - down_read(&gmap->mm->mmap_sem); + mmap_read_lock(gmap->mm); rc = gmap_protect_range(gmap, gaddr, len, prot, GMAP_NOTIFY_MPROT); - up_read(&gmap->mm->mmap_sem); + mmap_read_unlock(gmap->mm); return rc; } EXPORT_SYMBOL_GPL(gmap_mprotect_notify); @@ -1691,11 +1691,11 @@ struct gmap *gmap_shadow(struct gmap *parent, unsigned long asce, } spin_unlock(&parent->shadow_lock); /* protect after insertion, so it will get properly invalidated */ - down_read(&parent->mm->mmap_sem); + mmap_read_lock(parent->mm); rc = gmap_protect_range(parent, asce & _ASCE_ORIGIN, ((asce & _ASCE_TABLE_LENGTH) + 1) * PAGE_SIZE, PROT_READ, GMAP_NOTIFY_SHADOW); - up_read(&parent->mm->mmap_sem); + mmap_read_unlock(parent->mm); spin_lock(&parent->shadow_lock); new->initialized = true; if (rc) { @@ -2542,12 +2542,12 @@ int s390_enable_sie(void) /* Fail if the page tables are 2K */ if (!mm_alloc_pgste(mm)) return -EINVAL; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); mm->context.has_pgste = 1; /* split thp mappings and disable thp for future mappings */ thp_split_mm(mm); zap_zero_pages(mm); - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return 0; } EXPORT_SYMBOL_GPL(s390_enable_sie); @@ -2599,7 +2599,7 @@ int s390_enable_skey(void) struct vm_area_struct *vma; int rc = 0; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); if (mm_uses_skeys(mm)) goto out_up; @@ -2618,7 +2618,7 @@ int s390_enable_skey(void) walk_page_range(0, TASK_SIZE, &walk); out_up: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return rc; } EXPORT_SYMBOL_GPL(s390_enable_skey); @@ -2637,9 +2637,9 @@ void s390_reset_cmma(struct mm_struct *mm) { struct mm_walk walk = { .pte_entry = __s390_reset_cmma }; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); walk.mm = mm; walk_page_range(0, TASK_SIZE, &walk); - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); } EXPORT_SYMBOL_GPL(s390_reset_cmma); diff --git a/arch/s390/pci/pci_mmio.c b/arch/s390/pci/pci_mmio.c index 7d42a8794f10..2090f71cf435 100644 --- a/arch/s390/pci/pci_mmio.c +++ b/arch/s390/pci/pci_mmio.c @@ -18,7 +18,7 @@ static long get_pfn(unsigned long user_addr, unsigned long access, struct vm_area_struct *vma; long ret; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); ret = -EINVAL; vma = find_vma(current->mm, user_addr); if (!vma) @@ -28,7 +28,7 @@ static long get_pfn(unsigned long user_addr, unsigned long access, goto out; ret = follow_pfn(vma, user_addr, pfn); out: - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); return ret; } diff --git a/arch/sh/kernel/sys_sh.c b/arch/sh/kernel/sys_sh.c index f8afc014e084..a5a7b33ed81a 100644 --- a/arch/sh/kernel/sys_sh.c +++ b/arch/sh/kernel/sys_sh.c @@ -69,10 +69,10 @@ asmlinkage int sys_cacheflush(unsigned long addr, unsigned long len, int op) if (addr + len < addr) return -EFAULT; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); vma = find_vma (current->mm, addr); if (vma == NULL || addr < vma->vm_start || addr + len > vma->vm_end) { - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); return -EFAULT; } @@ -91,6 +91,6 @@ asmlinkage int sys_cacheflush(unsigned long addr, unsigned long len, int op) if (op & CACHEFLUSH_I) flush_icache_range(addr, addr+len); - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); return 0; } diff --git a/arch/sh/kernel/vsyscall/vsyscall.c b/arch/sh/kernel/vsyscall/vsyscall.c index cc0cc5b4ff18..aed9a1fea0c7 100644 --- a/arch/sh/kernel/vsyscall/vsyscall.c +++ b/arch/sh/kernel/vsyscall/vsyscall.c @@ -64,7 +64,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) unsigned long addr; int ret; - if (down_write_killable(&mm->mmap_sem)) + if (mmap_write_lock_killable(mm)) return -EINTR; addr = get_unmapped_area(NULL, 0, PAGE_SIZE, 0, 0); @@ -83,7 +83,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) current->mm->context.vdso = (void *)addr; up_fail: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return ret; } diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault.c index 6defd2c6d9b1..9de75e6ab924 100644 --- a/arch/sh/mm/fault.c +++ b/arch/sh/mm/fault.c @@ -278,7 +278,7 @@ __bad_area(struct pt_regs *regs, unsigned long error_code, * Something tried to access memory that isn't in our memory map.. * Fix it, but check if it's kernel or user first.. */ - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); __bad_area_nosemaphore(regs, error_code, address, si_code); } @@ -302,7 +302,7 @@ do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address) struct task_struct *tsk = current; struct mm_struct *mm = tsk->mm; - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* Kernel mode? Handle exceptions or die: */ if (!user_mode(regs)) @@ -321,7 +321,7 @@ mm_fault_error(struct pt_regs *regs, unsigned long error_code, */ if (fatal_signal_pending(current)) { if (!(fault & VM_FAULT_RETRY)) - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); if (!user_mode(regs)) no_context(regs, error_code, address); return 1; @@ -333,11 +333,11 @@ mm_fault_error(struct pt_regs *regs, unsigned long error_code, if (fault & VM_FAULT_OOM) { /* Kernel mode? Handle exceptions or die: */ if (!user_mode(regs)) { - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); no_context(regs, error_code, address); return 1; } - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); /* * We ran out of memory, call the OOM killer, and return the @@ -441,7 +441,7 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, } retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma(mm, address); if (unlikely(!vma)) { @@ -510,5 +510,5 @@ good_area: } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); } diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c index 909820508c78..b0919cd90cea 100644 --- a/arch/sparc/mm/fault_32.c +++ b/arch/sparc/mm/fault_32.c @@ -196,7 +196,7 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); if (!from_user && address >= PAGE_OFFSET) goto bad_area; @@ -273,7 +273,7 @@ good_area: } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return; /* @@ -281,7 +281,7 @@ good_area: * Fix it, but check if it's kernel or user first.. */ bad_area: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); bad_area_nosemaphore: /* User mode accesses just cause a SIGSEGV */ @@ -330,7 +330,7 @@ no_context: * us unable to handle the page fault gracefully. */ out_of_memory: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (from_user) { pagefault_out_of_memory(); return; @@ -338,7 +338,7 @@ out_of_memory: goto no_context; do_sigbus: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); do_fault_siginfo(BUS_ADRERR, SIGBUS, regs, text_fault); if (!from_user) goto no_context; @@ -385,7 +385,7 @@ static void force_user_fault(unsigned long address, int write) code = SEGV_MAPERR; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma(mm, address); if (!vma) goto bad_area; @@ -410,15 +410,15 @@ good_area: case VM_FAULT_OOM: goto do_sigbus; } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return; bad_area: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); __do_fault_siginfo(code, SIGSEGV, tsk->thread.kregs, address); return; do_sigbus: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); __do_fault_siginfo(BUS_ADRERR, SIGBUS, tsk->thread.kregs, address); } diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c index 58947e0aaa29..a1571efe515f 100644 --- a/arch/sparc/mm/fault_64.c +++ b/arch/sparc/mm/fault_64.c @@ -329,7 +329,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); - if (!down_read_trylock(&mm->mmap_sem)) { + if (!mmap_read_trylock(mm)) { if ((regs->tstate & TSTATE_PRIV) && !search_exception_tables(regs->tpc)) { insn = get_fault_insn(regs, insn); @@ -337,7 +337,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) } retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); } if (fault_code & FAULT_CODE_BAD_RA) @@ -470,7 +470,7 @@ good_area: goto retry; } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); mm_rss = get_mm_rss(mm); #if defined(CONFIG_TRANSPARENT_HUGEPAGE) @@ -501,7 +501,7 @@ exit_exception: */ bad_area: insn = get_fault_insn(regs, insn); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); handle_kernel_fault: do_kernel_fault(regs, si_code, fault_code, insn, address); @@ -513,7 +513,7 @@ handle_kernel_fault: */ out_of_memory: insn = get_fault_insn(regs, insn); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (!(regs->tstate & TSTATE_PRIV)) { pagefault_out_of_memory(); goto exit_exception; @@ -526,7 +526,7 @@ intr_or_no_mm: do_sigbus: insn = get_fault_insn(regs, insn); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* * Send a sigbus, regardless of whether we were in kernel diff --git a/arch/sparc/vdso/vma.c b/arch/sparc/vdso/vma.c index 1869e4b36db2..84251ae485bc 100644 --- a/arch/sparc/vdso/vma.c +++ b/arch/sparc/vdso/vma.c @@ -179,7 +179,7 @@ static int map_vdso(const struct vdso_image *image, unsigned long text_start, addr = 0; int ret = 0; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); /* * First, get an unmapped region: then randomize it, and make sure that @@ -235,7 +235,7 @@ up_fail: if (ret) current->mm->context.vdso = NULL; - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return ret; } diff --git a/arch/um/include/asm/mmu_context.h b/arch/um/include/asm/mmu_context.h index 129fb1d1f1c5..ca2c1a809da5 100644 --- a/arch/um/include/asm/mmu_context.h +++ b/arch/um/include/asm/mmu_context.h @@ -55,7 +55,7 @@ static inline void activate_mm(struct mm_struct *old, struct mm_struct *new) __switch_mm(&new->context.id); down_write_nested(&new->mmap_sem, 1); uml_setup_stubs(new); - up_write(&new->mmap_sem); + mmap_write_unlock(new); } static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, diff --git a/arch/um/kernel/tlb.c b/arch/um/kernel/tlb.c index 37508b190106..b195ee9f2b27 100644 --- a/arch/um/kernel/tlb.c +++ b/arch/um/kernel/tlb.c @@ -300,7 +300,7 @@ void fix_range_common(struct mm_struct *mm, unsigned long start_addr, printk(KERN_ERR "fix_range_common: failed, killing current " "process: %d\n", task_tgid_vnr(current)); /* We are under mmap_sem, release it such that current can terminate */ - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); force_sig(SIGKILL, current); do_signal(¤t->thread.regs); } diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c index cced82946042..68e6c845372f 100644 --- a/arch/um/kernel/trap.c +++ b/arch/um/kernel/trap.c @@ -46,7 +46,7 @@ int handle_page_fault(unsigned long address, unsigned long ip, if (is_user) flags |= FAULT_FLAG_USER; retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma(mm, address); if (!vma) goto out; @@ -122,7 +122,7 @@ good_area: #endif flush_tlb_page(vma, address); out: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); out_nosemaphore: return err; @@ -131,7 +131,7 @@ out_of_memory: * We ran out of memory, call the OOM killer, and return the userspace * (which will retry the fault, or kill us if we got oom-killed). */ - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (!is_user) goto out_nosemaphore; pagefault_out_of_memory(); diff --git a/arch/unicore32/mm/fault.c b/arch/unicore32/mm/fault.c index 5a01f6cffab3..48ea7e6c9357 100644 --- a/arch/unicore32/mm/fault.c +++ b/arch/unicore32/mm/fault.c @@ -233,12 +233,12 @@ static int do_pf(unsigned long addr, unsigned int fsr, struct pt_regs *regs) * validly references user space from well defined areas of the code, * we can bug out early if this is from code which shouldn't. */ - if (!down_read_trylock(&mm->mmap_sem)) { + if (!mmap_read_trylock(mm)) { if (!user_mode(regs) && !search_exception_tables(regs->UCreg_pc)) goto no_context; retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); } else { /* * The above down_read_trylock() might have succeeded in @@ -275,7 +275,7 @@ retry: } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* * Handle the "normal" case first - VM_FAULT_MAJOR diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c index a6ff2ebc8aca..b1f0887a9039 100644 --- a/arch/x86/entry/vdso/vma.c +++ b/arch/x86/entry/vdso/vma.c @@ -156,7 +156,7 @@ static int map_vdso(const struct vdso_image *image, unsigned long addr) unsigned long text_start; int ret = 0; - if (down_write_killable(&mm->mmap_sem)) + if (mmap_write_lock_killable(mm)) return -EINTR; addr = get_unmapped_area(NULL, addr, @@ -199,7 +199,7 @@ static int map_vdso(const struct vdso_image *image, unsigned long addr) } up_fail: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return ret; } @@ -261,7 +261,7 @@ int map_vdso_once(const struct vdso_image *image, unsigned long addr) struct mm_struct *mm = current->mm; struct vm_area_struct *vma; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); /* * Check if we have already mapped vdso blob - fail to prevent * abusing from userspace install_speciall_mapping, which may @@ -272,11 +272,11 @@ int map_vdso_once(const struct vdso_image *image, unsigned long addr) for (vma = mm->mmap; vma; vma = vma->vm_next) { if (vma_is_special_mapping(vma, &vdso_mapping) || vma_is_special_mapping(vma, &vvar_mapping)) { - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return -EEXIST; } } - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return map_vdso(image, addr); } diff --git a/arch/x86/kernel/vm86_32.c b/arch/x86/kernel/vm86_32.c index c2fd39752da8..2abc0047f729 100644 --- a/arch/x86/kernel/vm86_32.c +++ b/arch/x86/kernel/vm86_32.c @@ -172,7 +172,7 @@ static void mark_screen_rdonly(struct mm_struct *mm) pte_t *pte; int i; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); pgd = pgd_offset(mm, 0xA0000); if (pgd_none_or_clear_bad(pgd)) goto out; @@ -198,7 +198,7 @@ static void mark_screen_rdonly(struct mm_struct *mm) } pte_unmap_unlock(pte, ptl); out: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); flush_tlb_mm_range(mm, 0xA0000, 0xA0000 + 32*PAGE_SIZE, PAGE_SHIFT, false); } diff --git a/arch/x86/mm/debug_pagetables.c b/arch/x86/mm/debug_pagetables.c index 225fe2f0bfec..46b34ee82aad 100644 --- a/arch/x86/mm/debug_pagetables.c +++ b/arch/x86/mm/debug_pagetables.c @@ -26,9 +26,9 @@ static const struct file_operations ptdump_fops = { static int ptdump_show_curknl(struct seq_file *m, void *v) { if (current->mm->pgd) { - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); ptdump_walk_pgd_level_debugfs(m, current->mm->pgd, false); - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); } return 0; } @@ -52,9 +52,9 @@ static struct dentry *pe_curusr; static int ptdump_show_curusr(struct seq_file *m, void *v) { if (current->mm->pgd) { - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); ptdump_walk_pgd_level_debugfs(m, current->mm->pgd, true); - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); } return 0; } diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 0253dc184d33..a044b2ca8e94 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -933,7 +933,7 @@ __bad_area(struct pt_regs *regs, unsigned long error_code, * Something tried to access memory that isn't in our memory map.. * Fix it, but check if it's kernel or user first.. */ - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); __bad_area_nosemaphore(regs, error_code, address, (vma) ? &pkey : NULL, si_code); @@ -1338,14 +1338,14 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code, * validate the source. If this is invalid we can skip the address * space check, thus avoiding the deadlock: */ - if (unlikely(!down_read_trylock(&mm->mmap_sem))) { + if (unlikely(!mmap_read_trylock(mm))) { if (!(error_code & X86_PF_USER) && !search_exception_tables(regs->ip)) { bad_area_nosemaphore(regs, error_code, address, NULL); return; } retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); } else { /* * The above down_read_trylock() might have succeeded in @@ -1435,7 +1435,7 @@ good_area: return; } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (unlikely(fault & VM_FAULT_ERROR)) { mm_fault_error(regs, error_code, address, &pkey, fault); return; diff --git a/arch/x86/mm/mpx.c b/arch/x86/mm/mpx.c index e500949bae24..af39e29ab3fd 100644 --- a/arch/x86/mm/mpx.c +++ b/arch/x86/mm/mpx.c @@ -52,10 +52,10 @@ static unsigned long mpx_mmap(unsigned long len) if (len != mpx_bt_size_bytes(mm)) return -EINVAL; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); addr = do_mmap(NULL, 0, len, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, VM_MPX, 0, &populate, NULL); - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); if (populate) mm_populate(addr, populate); @@ -239,7 +239,7 @@ int mpx_enable_management(void) * unmap path; we can just use mm->context.bd_addr instead. */ bd_base = mpx_get_bounds_dir(); - down_write(&mm->mmap_sem); + mmap_write_lock(mm); /* MPX doesn't support addresses above 47 bits yet. */ if (find_vma(mm, DEFAULT_MAP_WINDOW)) { @@ -253,7 +253,7 @@ int mpx_enable_management(void) if (mm->context.bd_addr == MPX_INVALID_BOUNDS_DIR) ret = -ENXIO; out: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return ret; } @@ -264,9 +264,9 @@ int mpx_disable_management(void) if (!cpu_feature_enabled(X86_FEATURE_MPX)) return -ENXIO; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); mm->context.bd_addr = MPX_INVALID_BOUNDS_DIR; - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return 0; } diff --git a/arch/x86/um/vdso/vma.c b/arch/x86/um/vdso/vma.c index 6be22f991b59..091297e3b247 100644 --- a/arch/x86/um/vdso/vma.c +++ b/arch/x86/um/vdso/vma.c @@ -61,7 +61,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) if (!vdso_enabled) return 0; - if (down_write_killable(&mm->mmap_sem)) + if (mmap_write_lock_killable(mm)) return -EINTR; err = install_special_mapping(mm, um_vdso_addr, PAGE_SIZE, @@ -69,7 +69,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, vdsop); - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return err; } diff --git a/arch/xtensa/mm/fault.c b/arch/xtensa/mm/fault.c index 038e2213db9b..49de543c9997 100644 --- a/arch/xtensa/mm/fault.c +++ b/arch/xtensa/mm/fault.c @@ -74,7 +74,7 @@ void do_page_fault(struct pt_regs *regs) if (user_mode(regs)) flags |= FAULT_FLAG_USER; retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma(mm, address); if (!vma) @@ -140,7 +140,7 @@ good_area: } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); if (flags & VM_FAULT_MAJOR) perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address); @@ -153,7 +153,7 @@ good_area: * Fix it, but check if it's kernel or user first.. */ bad_area: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (user_mode(regs)) { current->thread.bad_vaddr = address; current->thread.error_code = is_write; @@ -168,7 +168,7 @@ bad_area: * us unable to handle the page fault gracefully. */ out_of_memory: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (!user_mode(regs)) bad_page_fault(regs, address, SIGKILL); else @@ -176,7 +176,7 @@ out_of_memory: return; do_sigbus: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* Send a sigbus, regardless of whether we were in kernel * or user mode. diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c index bb94b76d6372..0e79bf7c5bbc 100644 --- a/drivers/android/binder_alloc.c +++ b/drivers/android/binder_alloc.c @@ -220,7 +220,7 @@ static int binder_update_page_range(struct binder_alloc *alloc, int allocate, mm = alloc->vma_vm_mm; if (mm) { - down_write(&mm->mmap_sem); + mmap_write_lock(mm); vma = alloc->vma; } @@ -279,7 +279,7 @@ static int binder_update_page_range(struct binder_alloc *alloc, int allocate, /* vm_insert_page does not seem to increment the refcount */ } if (mm) { - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); mmput_async(mm); } return 0; @@ -312,7 +312,7 @@ err_page_ptr_cleared: } err_no_vma: if (mm) { - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); mmput_async(mm); } return vma ? -ENOMEM : -ESRCH; @@ -1013,7 +1013,7 @@ enum lru_status binder_alloc_free_page(struct list_head *item, mm = alloc->vma_vm_mm; if (!mmget_not_zero(mm)) goto err_mmget; - if (!down_read_trylock(&mm->mmap_sem)) + if (!mmap_read_trylock(mm)) goto err_down_read_mmap_sem_failed; vma = binder_alloc_get_vma(alloc); @@ -1027,7 +1027,7 @@ enum lru_status binder_alloc_free_page(struct list_head *item, trace_binder_unmap_user_end(alloc, index); } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); mmput_async(mm); trace_binder_unmap_kernel_start(alloc, index); diff --git a/drivers/char/adsprpc.c b/drivers/char/adsprpc.c index 0942c969d1f4..0c0bf5e03c98 100644 --- a/drivers/char/adsprpc.c +++ b/drivers/char/adsprpc.c @@ -2185,12 +2185,12 @@ static void inv_args(struct smq_invoke_ctx *ctx) uint64_t inv_len; struct vm_area_struct *vma; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); VERIFY(err, NULL != (vma = find_vma( current->mm, rpra[over].buf.pv))); if (err) { - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); goto bail; } if (ctx->overps[i]->do_cmo) { @@ -2205,7 +2205,7 @@ static void inv_args(struct smq_invoke_ctx *ctx) ctx->overps[i]->mend - ctx->overps[i]->mstart; } - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); dma_buf_begin_cpu_access_partial( map->buf, DMA_TO_DEVICE, offset, inv_len); diff --git a/drivers/fpga/dfl-afu-dma-region.c b/drivers/fpga/dfl-afu-dma-region.c index e056965ef97b..dab5349d184f 100644 --- a/drivers/fpga/dfl-afu-dma-region.c +++ b/drivers/fpga/dfl-afu-dma-region.c @@ -51,7 +51,7 @@ static int afu_dma_adjust_locked_vm(struct device *dev, long npages, bool incr) if (!current->mm) return 0; - down_write(¤t->mm->mmap_sem); + mmap_write_lock(current->mm); if (incr) { locked = current->mm->locked_vm + npages; @@ -72,7 +72,7 @@ static int afu_dma_adjust_locked_vm(struct device *dev, long npages, bool incr) current->mm->locked_vm << PAGE_SHIFT, rlimit(RLIMIT_MEMLOCK), ret ? "- execeeded" : ""); - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); return ret; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index 604381075903..91ccacc261cc 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -1385,9 +1385,9 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( * concurrently and the queues are actually stopped */ if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { - down_write(¤t->mm->mmap_sem); + mmap_write_lock(current->mm); is_invalid_userptr = atomic_read(&mem->invalid); - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); } mutex_lock(&mem->lock); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c index e55508b39496..4aed3f7bce20 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c @@ -389,7 +389,7 @@ struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev, int r; mutex_lock(&adev->mn_lock); - if (down_write_killable(&mm->mmap_sem)) { + if (mmap_write_lock_killable(mm)) { mutex_unlock(&adev->mn_lock); return ERR_PTR(-EINTR); } @@ -420,13 +420,13 @@ struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev, hash_add(adev->mn_hash, &amn->node, AMDGPU_MN_KEY(mm, type)); release_locks: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); mutex_unlock(&adev->mn_lock); return amn; free_amn: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); mutex_unlock(&adev->mn_lock); kfree(amn); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 50807d621eca..db4042191a7e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -817,7 +817,7 @@ int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages) if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY)) flags |= FOLL_WRITE; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); if (gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) { /* @@ -829,7 +829,7 @@ int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages) vma = find_vma(mm, gtt->userptr); if (!vma || vma->vm_file || vma->vm_end < end) { - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return -EPERM; } } @@ -864,12 +864,12 @@ int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages) } while (pinned < ttm->num_pages); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return 0; release_pages: release_pages(pages, pinned); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return r; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c index 8f23192b6709..a52e7c62a871 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c @@ -901,7 +901,7 @@ void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid, memset(&memory_exception_data, 0, sizeof(memory_exception_data)); - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma(mm, address); memory_exception_data.gpu_id = dev->id; @@ -924,7 +924,7 @@ void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid, memory_exception_data.failure.NoExecute = 0; } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); mmput(mm); pr_debug("notpresent %d, noexecute %d, readonly %d\n", diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 478d989a2369..5d9ed84c7a4a 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -1901,7 +1901,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data, struct mm_struct *mm = current->mm; struct vm_area_struct *vma; - if (down_write_killable(&mm->mmap_sem)) { + if (mmap_write_lock_killable(mm)) { addr = -EINTR; goto err; } @@ -1911,7 +1911,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data, pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); else addr = -ENOMEM; - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); if (IS_ERR_VALUE(addr)) goto err; diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c index 961abb6ea18e..24a638968ad8 100644 --- a/drivers/gpu/drm/i915/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/i915_gem_userptr.c @@ -221,7 +221,7 @@ i915_mmu_notifier_find(struct i915_mm_struct *mm) if (IS_ERR(mn)) err = PTR_ERR(mn); - down_write(&mm->mm->mmap_sem); + mmap_write_lock(mm->mm); mutex_lock(&mm->i915->mm_lock); if (mm->mn == NULL && !err) { /* Protected by mmap_sem (write-lock) */ @@ -238,7 +238,7 @@ i915_mmu_notifier_find(struct i915_mm_struct *mm) err = 0; } mutex_unlock(&mm->i915->mm_lock); - up_write(&mm->mm->mmap_sem); + mmap_write_unlock(mm->mm); if (mn && !IS_ERR(mn)) { destroy_workqueue(mn->wq); @@ -519,7 +519,7 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work) ret = -EFAULT; if (mmget_not_zero(mm)) { - down_read(&mm->mmap_sem); + mmap_read_lock(mm); while (pinned < npages) { ret = get_user_pages_remote (work->task, mm, @@ -532,7 +532,7 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work) pinned += ret; } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); mmput(mm); } } diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c index 5e61abb3dce5..67683b8a364f 100644 --- a/drivers/gpu/drm/radeon/radeon_cs.c +++ b/drivers/gpu/drm/radeon/radeon_cs.c @@ -190,12 +190,12 @@ static int radeon_cs_parser_relocs(struct radeon_cs_parser *p) p->vm_bos = radeon_vm_get_bos(p->rdev, p->ib.vm, &p->validated); if (need_mmap_lock) - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); r = radeon_bo_list_validate(p->rdev, &p->ticket, &p->validated, p->ring); if (need_mmap_lock) - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); return r; } diff --git a/drivers/gpu/drm/radeon/radeon_gem.c b/drivers/gpu/drm/radeon/radeon_gem.c index 4efbc230c3f3..86cea0446e4e 100644 --- a/drivers/gpu/drm/radeon/radeon_gem.c +++ b/drivers/gpu/drm/radeon/radeon_gem.c @@ -336,17 +336,17 @@ int radeon_gem_userptr_ioctl(struct drm_device *dev, void *data, } if (args->flags & RADEON_GEM_USERPTR_VALIDATE) { - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); r = radeon_bo_reserve(bo, true); if (r) { - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); goto release_object; } radeon_ttm_placement_from_domain(bo, RADEON_GEM_DOMAIN_GTT); r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); radeon_bo_unreserve(bo); - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); if (r) goto release_object; } diff --git a/drivers/gpu/drm/radeon/radeon_mn.c b/drivers/gpu/drm/radeon/radeon_mn.c index f8b35df44c60..d567b1a6b9ec 100644 --- a/drivers/gpu/drm/radeon/radeon_mn.c +++ b/drivers/gpu/drm/radeon/radeon_mn.c @@ -203,7 +203,7 @@ static struct radeon_mn *radeon_mn_get(struct radeon_device *rdev) struct radeon_mn *rmn; int r; - if (down_write_killable(&mm->mmap_sem)) + if (mmap_write_lock_killable(mm)) return ERR_PTR(-EINTR); mutex_lock(&rdev->mn_lock); @@ -232,13 +232,13 @@ static struct radeon_mn *radeon_mn_get(struct radeon_device *rdev) release_locks: mutex_unlock(&rdev->mn_lock); - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return rmn; free_rmn: mutex_unlock(&rdev->mn_lock); - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); kfree(rmn); return ERR_PTR(r); diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index 2119ae715962..071f43d2f7be 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -70,7 +70,7 @@ static vm_fault_t ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, goto out_unlock; ttm_bo_get(bo); - up_read(&vmf->vma->vm_mm->mmap_sem); + mmap_read_unlock(vmf->vma->vm_mm); (void) dma_fence_wait(bo->moving, true); ttm_bo_unreserve(bo); ttm_bo_put(bo); @@ -145,7 +145,7 @@ static vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) if (fault_flag_allow_retry_first(vmf->flags)) { if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { ttm_bo_get(bo); - up_read(&vmf->vma->vm_mm->mmap_sem); + mmap_read_unlock(vmf->vma->vm_mm); (void) ttm_bo_wait_unreserved(bo); ttm_bo_put(bo); } diff --git a/drivers/gpu/msm/kgsl.c b/drivers/gpu/msm/kgsl.c index 1f3f73a0359f..a17d164c07a0 100644 --- a/drivers/gpu/msm/kgsl.c +++ b/drivers/gpu/msm/kgsl.c @@ -2599,15 +2599,15 @@ static int memdesc_sg_virt(struct kgsl_memdesc *memdesc, unsigned long useraddr) goto out; } - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); if (!check_vma(useraddr, memdesc->size)) { - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); ret = -EFAULT; goto out; } npages = get_user_pages(useraddr, sglen, write, pages, NULL); - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); ret = (npages < 0) ? (int)npages : 0; if (ret) @@ -2729,7 +2729,7 @@ static int kgsl_setup_dmabuf_useraddr(struct kgsl_device *device, * Find the VMA containing this pointer and figure out if it * is a dma-buf. */ - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); vma = find_vma(current->mm, hostptr); if (vma && vma->vm_file) { @@ -2737,7 +2737,7 @@ static int kgsl_setup_dmabuf_useraddr(struct kgsl_device *device, ret = check_vma_flags(vma, entry->memdesc.flags); if (ret) { - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); return ret; } @@ -2746,7 +2746,7 @@ static int kgsl_setup_dmabuf_useraddr(struct kgsl_device *device, * already mapped */ if (vma->vm_ops == &kgsl_gpumem_vm_ops) { - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); return -EFAULT; } @@ -2755,7 +2755,7 @@ static int kgsl_setup_dmabuf_useraddr(struct kgsl_device *device, if (fd) { dmabuf = dma_buf_get(fd - 1); if (IS_ERR(dmabuf)) { - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); return PTR_ERR(dmabuf); } /* @@ -2767,21 +2767,21 @@ static int kgsl_setup_dmabuf_useraddr(struct kgsl_device *device, */ if (dmabuf != vma->vm_file->private_data) { dma_buf_put(dmabuf); - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); return -EBADF; } } } if (IS_ERR_OR_NULL(dmabuf)) { - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); return dmabuf ? PTR_ERR(dmabuf) : -ENODEV; } ret = kgsl_setup_dma_buf(device, pagetable, entry, dmabuf); if (ret) { dma_buf_put(dmabuf); - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); return ret; } @@ -2793,7 +2793,7 @@ static int kgsl_setup_dmabuf_useraddr(struct kgsl_device *device, else entry->memdesc.flags &= ~((u64) KGSL_MEMFLAGS_IOCOHERENT); - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); return 0; } #else diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c index 5819a2fb027d..406926cf4910 100644 --- a/drivers/infiniband/core/rdma_core.c +++ b/drivers/infiniband/core/rdma_core.c @@ -832,9 +832,9 @@ static void ufile_disassociate_ucontext(struct ib_ucontext *ibcontext) } } - down_write(&owning_mm->mmap_sem); + mmap_write_lock(owning_mm); ib_dev->disassociate_ucontext(ibcontext); - up_write(&owning_mm->mmap_sem); + mmap_write_unlock(owning_mm); mmput(owning_mm); put_task_struct(owning_process); } diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index a41792dbae1f..77569ccfcc27 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -147,14 +147,14 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr, lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; - down_write(¤t->mm->mmap_sem); + mmap_write_lock(current->mm); current->mm->pinned_vm += npages; if ((current->mm->pinned_vm > lock_limit) && !capable(CAP_IPC_LOCK)) { - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); ret = -ENOMEM; goto vma; } - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); cur_base = addr & PAGE_MASK; @@ -172,14 +172,14 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr, sg_list_start = umem->sg_head.sgl; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); while (npages) { ret = get_user_pages_longterm(cur_base, min_t(unsigned long, npages, PAGE_SIZE / sizeof (struct page *)), gup_flags, page_list, vma_list); if (ret < 0) { - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); goto umem_release; } @@ -197,7 +197,7 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr, /* preparing for next loop */ sg_list_start = sg; } - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); umem->nmap = ib_dma_map_sg_attrs(context->device, umem->sg_head.sgl, @@ -216,9 +216,9 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr, umem_release: __ib_umem_release(context->device, umem, 0); vma: - down_write(¤t->mm->mmap_sem); + mmap_write_lock(current->mm); current->mm->pinned_vm -= ib_umem_num_pages(umem); - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); out: if (vma_list) free_page((unsigned long) vma_list); @@ -234,9 +234,9 @@ static void ib_umem_account(struct work_struct *work) { struct ib_umem *umem = container_of(work, struct ib_umem, work); - down_write(&umem->mm->mmap_sem); + mmap_write_lock(umem->mm); umem->mm->pinned_vm -= umem->diff; - up_write(&umem->mm->mmap_sem); + mmap_write_unlock(umem->mm); mmput(umem->mm); kfree(umem); } @@ -278,7 +278,7 @@ void ib_umem_release(struct ib_umem *umem) * we defer the vm_locked accounting to the system workqueue. */ if (context->closing) { - if (!down_write_trylock(&mm->mmap_sem)) { + if (!mmap_write_trylock(mm)) { INIT_WORK(&umem->work, ib_umem_account); umem->mm = mm; umem->diff = diff; @@ -287,10 +287,10 @@ void ib_umem_release(struct ib_umem *umem) return; } } else - down_write(&mm->mmap_sem); + mmap_write_lock(mm); mm->pinned_vm -= diff; - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); mmput(mm); out: kfree(umem); diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c index fd6ec56c17f9..5b776057d832 100644 --- a/drivers/infiniband/core/umem_odp.c +++ b/drivers/infiniband/core/umem_odp.c @@ -352,16 +352,16 @@ int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem, struct vm_area_struct *vma; struct hstate *h; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma(mm, ib_umem_start(umem)); if (!vma || !is_vm_hugetlb_page(vma)) { - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); ret_val = -EINVAL; goto out_mm; } h = hstate_vma(vma); umem->page_shift = huge_page_shift(h); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); umem->hugetlb = 1; } else { umem->hugetlb = 0; @@ -693,7 +693,7 @@ int ib_umem_odp_map_dma_pages(struct ib_umem *umem, u64 user_virt, u64 bcnt, ALIGN(bcnt, PAGE_SIZE) / PAGE_SIZE, PAGE_SIZE / sizeof(struct page *)); - down_read(&owning_mm->mmap_sem); + mmap_read_lock(owning_mm); /* * Note: this might result in redundent page getting. We can * avoid this by checking dma_list to be 0 before calling @@ -704,7 +704,7 @@ int ib_umem_odp_map_dma_pages(struct ib_umem *umem, u64 user_virt, u64 bcnt, npages = get_user_pages_remote(owning_process, owning_mm, user_virt, gup_num_pages, flags, local_page_list, NULL, NULL); - up_read(&owning_mm->mmap_sem); + mmap_read_unlock(owning_mm); if (npages < 0) break; diff --git a/drivers/infiniband/hw/hfi1/user_pages.c b/drivers/infiniband/hw/hfi1/user_pages.c index e341e6dcc388..ac3415e3aa6a 100644 --- a/drivers/infiniband/hw/hfi1/user_pages.c +++ b/drivers/infiniband/hw/hfi1/user_pages.c @@ -91,9 +91,9 @@ bool hfi1_can_pin_pages(struct hfi1_devdata *dd, struct mm_struct *mm, /* Convert to number of pages */ size = DIV_ROUND_UP(size, PAGE_SIZE); - down_read(&mm->mmap_sem); + mmap_read_lock(mm); pinned = mm->pinned_vm; - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* First, check the absolute limit against all pinned pages. */ if (pinned + npages >= ulimit && !can_lock) @@ -111,9 +111,9 @@ int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr, size_t np if (ret < 0) return ret; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); mm->pinned_vm += ret; - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return ret; } @@ -130,8 +130,8 @@ void hfi1_release_user_pages(struct mm_struct *mm, struct page **p, } if (mm) { /* during close after signal, mm can be NULL */ - down_write(&mm->mmap_sem); + mmap_write_lock(mm); mm->pinned_vm -= npages; - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); } } diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c index 6fde373300f5..7cef4733b473 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c +++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c @@ -1390,7 +1390,7 @@ static void i40iw_set_hugetlb_values(u64 addr, struct i40iw_mr *iwmr) struct vm_area_struct *vma; struct hstate *h; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); vma = find_vma(current->mm, addr); if (vma && is_vm_hugetlb_page(vma)) { h = hstate_vma(vma); @@ -1399,7 +1399,7 @@ static void i40iw_set_hugetlb_values(u64 addr, struct i40iw_mr *iwmr) iwmr->page_msk = huge_page_mask(h); } } - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); } /** diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c index c7c85c22e4e3..b8898ce82e65 100644 --- a/drivers/infiniband/hw/mlx4/mr.c +++ b/drivers/infiniband/hw/mlx4/mr.c @@ -380,7 +380,7 @@ static struct ib_umem *mlx4_get_umem_mr(struct ib_ucontext *context, u64 start, if (!ib_access_writable(access_flags)) { struct vm_area_struct *vma; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); /* * FIXME: Ideally this would iterate over all the vmas that * cover the memory, but for now it requires a single vma to @@ -395,7 +395,7 @@ static struct ib_umem *mlx4_get_umem_mr(struct ib_ucontext *context, u64 start, access_flags |= IB_ACCESS_LOCAL_WRITE; } - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); } return ib_umem_get(context, start, length, access_flags, 0); diff --git a/drivers/infiniband/hw/qib/qib_user_pages.c b/drivers/infiniband/hw/qib/qib_user_pages.c index 16543d5e80c3..339b53026712 100644 --- a/drivers/infiniband/hw/qib/qib_user_pages.c +++ b/drivers/infiniband/hw/qib/qib_user_pages.c @@ -139,11 +139,11 @@ int qib_get_user_pages(unsigned long start_page, size_t num_pages, { int ret; - down_write(¤t->mm->mmap_sem); + mmap_write_lock(current->mm); ret = __qib_get_user_pages(start_page, num_pages, p); - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); return ret; } @@ -151,12 +151,12 @@ int qib_get_user_pages(unsigned long start_page, size_t num_pages, void qib_release_user_pages(struct page **p, size_t num_pages) { if (current->mm) /* during close after signal, mm can be NULL */ - down_write(¤t->mm->mmap_sem); + mmap_write_lock(current->mm); __qib_release_user_pages(p, num_pages, 1); if (current->mm) { current->mm->pinned_vm -= num_pages; - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); } } diff --git a/drivers/infiniband/hw/usnic/usnic_uiom.c b/drivers/infiniband/hw/usnic/usnic_uiom.c index 9dd39daa602b..90da54e6fc54 100644 --- a/drivers/infiniband/hw/usnic/usnic_uiom.c +++ b/drivers/infiniband/hw/usnic/usnic_uiom.c @@ -59,9 +59,9 @@ static void usnic_uiom_reg_account(struct work_struct *work) struct usnic_uiom_reg *umem = container_of(work, struct usnic_uiom_reg, work); - down_write(&umem->mm->mmap_sem); + mmap_write_lock(umem->mm); umem->mm->locked_vm -= umem->diff; - up_write(&umem->mm->mmap_sem); + mmap_write_unlock(umem->mm); mmput(umem->mm); kfree(umem); } @@ -136,7 +136,7 @@ static int usnic_uiom_get_pages(unsigned long addr, size_t size, int writable, npages = PAGE_ALIGN(size + (addr & ~PAGE_MASK)) >> PAGE_SHIFT; - down_write(¤t->mm->mmap_sem); + mmap_write_lock(current->mm); locked = npages + current->mm->pinned_vm; lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; @@ -199,7 +199,7 @@ out: else current->mm->pinned_vm = locked; - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); free_page((unsigned long) page_list); return ret; } @@ -459,7 +459,7 @@ void usnic_uiom_reg_release(struct usnic_uiom_reg *uiomr, * we defer the vm_locked accounting to the system workqueue. */ if (ucontext->closing) { - if (!down_write_trylock(&mm->mmap_sem)) { + if (!mmap_write_trylock(mm)) { INIT_WORK(&uiomr->work, usnic_uiom_reg_account); uiomr->mm = mm; uiomr->diff = diff; @@ -468,10 +468,10 @@ void usnic_uiom_reg_release(struct usnic_uiom_reg *uiomr, return; } } else - down_write(&mm->mmap_sem); + mmap_write_lock(mm); mm->pinned_vm -= diff; - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); mmput(mm); out: kfree(uiomr); diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c index 387b72b7e7b3..b9b10a611978 100644 --- a/drivers/iommu/amd_iommu_v2.c +++ b/drivers/iommu/amd_iommu_v2.c @@ -522,7 +522,7 @@ static void do_fault(struct work_struct *work) flags |= FAULT_FLAG_WRITE; flags |= FAULT_FLAG_REMOTE; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_extend_vma(mm, address); if (!vma || address < vma->vm_start) /* failed to get a vma in the right range */ @@ -534,7 +534,7 @@ static void do_fault(struct work_struct *work) ret = handle_mm_fault(vma, address, flags); out: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (ret & VM_FAULT_ERROR) /* failed to service fault */ diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c index ef3aadec980e..c11135e826d0 100644 --- a/drivers/iommu/intel-svm.c +++ b/drivers/iommu/intel-svm.c @@ -629,7 +629,7 @@ static irqreturn_t prq_event_thread(int irq, void *d) if (!mmget_not_zero(svm->mm)) goto bad_req; - down_read(&svm->mm->mmap_sem); + mmap_read_lock(svm->mm); vma = find_extend_vma(svm->mm, address); if (!vma || address < vma->vm_start) goto invalid; @@ -644,7 +644,7 @@ static irqreturn_t prq_event_thread(int irq, void *d) result = QI_RESP_SUCCESS; invalid: - up_read(&svm->mm->mmap_sem); + mmap_read_unlock(svm->mm); mmput(svm->mm); bad_req: /* Accounting for major/minor faults? */ diff --git a/drivers/media/v4l2-core/videobuf-core.c b/drivers/media/v4l2-core/videobuf-core.c index 7491b337002c..d06b40993e78 100644 --- a/drivers/media/v4l2-core/videobuf-core.c +++ b/drivers/media/v4l2-core/videobuf-core.c @@ -537,7 +537,7 @@ int videobuf_qbuf(struct videobuf_queue *q, struct v4l2_buffer *b) MAGIC_CHECK(q->int_ops->magic, MAGIC_QTYPE_OPS); if (b->memory == V4L2_MEMORY_MMAP) - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); videobuf_queue_lock(q); retval = -EBUSY; @@ -624,7 +624,7 @@ done: videobuf_queue_unlock(q); if (b->memory == V4L2_MEMORY_MMAP) - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); return retval; } diff --git a/drivers/media/v4l2-core/videobuf-dma-contig.c b/drivers/media/v4l2-core/videobuf-dma-contig.c index a7a9bcc54343..bb5d96915bad 100644 --- a/drivers/media/v4l2-core/videobuf-dma-contig.c +++ b/drivers/media/v4l2-core/videobuf-dma-contig.c @@ -172,7 +172,7 @@ static int videobuf_dma_contig_user_get(struct videobuf_dma_contig_memory *mem, mem->size = PAGE_ALIGN(vb->size + offset); ret = -EINVAL; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma(mm, untagged_baddr); if (!vma) @@ -204,7 +204,7 @@ static int videobuf_dma_contig_user_get(struct videobuf_dma_contig_memory *mem, } out_up: - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); return ret; } diff --git a/drivers/media/v4l2-core/videobuf-dma-sg.c b/drivers/media/v4l2-core/videobuf-dma-sg.c index 4c396e0defa4..c4033061c9ae 100644 --- a/drivers/media/v4l2-core/videobuf-dma-sg.c +++ b/drivers/media/v4l2-core/videobuf-dma-sg.c @@ -203,9 +203,9 @@ static int videobuf_dma_init_user(struct videobuf_dmabuf *dma, int direction, { int ret; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); ret = videobuf_dma_init_user_locked(dma, direction, data, size); - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); return ret; } diff --git a/drivers/misc/cxl/cxllib.c b/drivers/misc/cxl/cxllib.c index 5a3f91255258..7c48b4e417de 100644 --- a/drivers/misc/cxl/cxllib.c +++ b/drivers/misc/cxl/cxllib.c @@ -211,7 +211,7 @@ static int get_vma_info(struct mm_struct *mm, u64 addr, struct vm_area_struct *vma = NULL; int rc = 0; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma(mm, addr); if (!vma) { @@ -222,7 +222,7 @@ static int get_vma_info(struct mm_struct *mm, u64 addr, *vma_start = vma->vm_start; *vma_end = vma->vm_end; out: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return rc; } diff --git a/drivers/misc/cxl/fault.c b/drivers/misc/cxl/fault.c index dc7b34174f85..96e183c5af86 100644 --- a/drivers/misc/cxl/fault.c +++ b/drivers/misc/cxl/fault.c @@ -325,7 +325,7 @@ static void cxl_prefault_vma(struct cxl_context *ctx) return; } - down_read(&mm->mmap_sem); + mmap_read_lock(mm); for (vma = mm->mmap; vma; vma = vma->vm_next) { for (ea = vma->vm_start; ea < vma->vm_end; ea = next_segment(ea, slb.vsid)) { @@ -340,7 +340,7 @@ static void cxl_prefault_vma(struct cxl_context *ctx) last_esid = slb.esid; } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); mmput(mm); } diff --git a/drivers/misc/mic/scif/scif_rma.c b/drivers/misc/mic/scif/scif_rma.c index e1f59b17715d..88b99c11ed36 100644 --- a/drivers/misc/mic/scif/scif_rma.c +++ b/drivers/misc/mic/scif/scif_rma.c @@ -277,16 +277,16 @@ __scif_dec_pinned_vm_lock(struct mm_struct *mm, if (!mm || !nr_pages || !scif_ulimit_check) return 0; if (try_lock) { - if (!down_write_trylock(&mm->mmap_sem)) { + if (!mmap_write_trylock(mm)) { dev_err(scif_info.mdev.this_device, "%s %d err\n", __func__, __LINE__); return -1; } } else { - down_write(&mm->mmap_sem); + mmap_write_lock(mm); } mm->pinned_vm -= nr_pages; - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return 0; } @@ -1385,11 +1385,11 @@ int __scif_pin_pages(void *addr, size_t len, int *out_prot, prot |= SCIF_PROT_WRITE; retry: mm = current->mm; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); if (ulimit) { err = __scif_check_inc_pinned_vm(mm, nr_pages); if (err) { - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); pinned_pages->nr_pages = 0; goto error_unmap; } @@ -1401,7 +1401,7 @@ retry: (prot & SCIF_PROT_WRITE) ? FOLL_WRITE : 0, pinned_pages->pages, NULL); - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); if (nr_pages != pinned_pages->nr_pages) { if (pinned_pages->nr_pages < 0) pinned_pages->nr_pages = 0; diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index 16df731e63c5..7cf2311ab079 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -82,14 +82,14 @@ static struct gru_thread_state *gru_find_lock_gts(unsigned long vaddr) struct vm_area_struct *vma; struct gru_thread_state *gts = NULL; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = gru_find_vma(vaddr); if (vma) gts = gru_find_thread_state(vma, TSID(vaddr, vma)); if (gts) mutex_lock(>s->ts_ctxlock); else - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return gts; } @@ -99,7 +99,7 @@ static struct gru_thread_state *gru_alloc_locked_gts(unsigned long vaddr) struct vm_area_struct *vma; struct gru_thread_state *gts = ERR_PTR(-EINVAL); - down_write(&mm->mmap_sem); + mmap_write_lock(mm); vma = gru_find_vma(vaddr); if (!vma) goto err; @@ -108,11 +108,11 @@ static struct gru_thread_state *gru_alloc_locked_gts(unsigned long vaddr) if (IS_ERR(gts)) goto err; mutex_lock(>s->ts_ctxlock); - downgrade_write(&mm->mmap_sem); + mmap_write_downgrade(mm); return gts; err: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return gts; } @@ -122,7 +122,7 @@ err: static void gru_unlock_gts(struct gru_thread_state *gts) { mutex_unlock(>s->ts_ctxlock); - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); } /* @@ -588,9 +588,9 @@ static irqreturn_t gru_intr(int chiplet, int blade) */ gts->ustats.fmm_tlbmiss++; if (!gts->ts_force_cch_reload && - down_read_trylock(>s->ts_mm->mmap_sem)) { + mmap_read_trylock(gts->ts_mm)) { gru_try_dropin(gru, gts, tfh, NULL); - up_read(>s->ts_mm->mmap_sem); + mmap_read_unlock(gts->ts_mm); } else { tfh_user_polling_mode(tfh); STAT(intr_mm_lock_failed); diff --git a/drivers/misc/sgi-gru/grufile.c b/drivers/misc/sgi-gru/grufile.c index 104a05f6b738..12f68d3f143e 100644 --- a/drivers/misc/sgi-gru/grufile.c +++ b/drivers/misc/sgi-gru/grufile.c @@ -148,7 +148,7 @@ static int gru_create_new_context(unsigned long arg) if (!(req.options & GRU_OPT_MISS_MASK)) req.options |= GRU_OPT_MISS_FMM_INTR; - down_write(¤t->mm->mmap_sem); + mmap_write_lock(current->mm); vma = gru_find_vma(req.gseg); if (vma) { vdata = vma->vm_private_data; @@ -159,7 +159,7 @@ static int gru_create_new_context(unsigned long arg) vdata->vd_tlb_preload_count = req.tlb_preload_count; ret = 0; } - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); return ret; } diff --git a/drivers/misc/uid_sys_stats.c b/drivers/misc/uid_sys_stats.c index cb027ed1f8c3..260392919116 100644 --- a/drivers/misc/uid_sys_stats.c +++ b/drivers/misc/uid_sys_stats.c @@ -137,7 +137,7 @@ static void get_full_task_comm(struct task_entry *task_entry, /* next the executable file name */ if (mm) { - down_read(&mm->mmap_sem); + mmap_read_lock(mm); if (mm->exe_file) { char *pathname = d_path(&mm->exe_file->f_path, buf, unused_len); @@ -150,7 +150,7 @@ static void get_full_task_comm(struct task_entry *task_entry, unused_len--; } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); } unused_len -= len; diff --git a/drivers/oprofile/buffer_sync.c b/drivers/oprofile/buffer_sync.c index 4f61b1b45e0d..a95489d6aa35 100644 --- a/drivers/oprofile/buffer_sync.c +++ b/drivers/oprofile/buffer_sync.c @@ -91,11 +91,11 @@ munmap_notify(struct notifier_block *self, unsigned long val, void *data) struct mm_struct *mm = current->mm; struct vm_area_struct *mpnt; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); mpnt = find_vma(mm, addr); if (mpnt && mpnt->vm_file && (mpnt->vm_flags & VM_EXEC)) { - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* To avoid latency problems, we only process the current CPU, * hoping that most samples for the task are on this CPU */ @@ -103,7 +103,7 @@ munmap_notify(struct notifier_block *self, unsigned long val, void *data) return 0; } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return 0; } @@ -256,7 +256,7 @@ lookup_dcookie(struct mm_struct *mm, unsigned long addr, off_t *offset) unsigned long cookie = NO_COOKIE; struct vm_area_struct *vma; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) { if (addr < vma->vm_start || addr >= vma->vm_end) @@ -276,7 +276,7 @@ lookup_dcookie(struct mm_struct *mm, unsigned long addr, off_t *offset) if (!vma) cookie = INVALID_COOKIE; - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return cookie; } diff --git a/drivers/tee/optee/call.c b/drivers/tee/optee/call.c index 4f4a7e2b122b..16f316e46b5d 100644 --- a/drivers/tee/optee/call.c +++ b/drivers/tee/optee/call.c @@ -593,10 +593,10 @@ static int check_mem_type(unsigned long start, size_t num_pages) struct mm_struct *mm = current->mm; int rc; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); rc = __check_mem_type(find_vma(mm, start), start + num_pages * PAGE_SIZE); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return rc; } diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c index 51b791c750f1..bcd490a182b8 100644 --- a/drivers/vfio/pci/vfio_pci.c +++ b/drivers/vfio/pci/vfio_pci.c @@ -1251,17 +1251,17 @@ static int vfio_pci_zap_and_vma_lock(struct vfio_pci_device *vdev, bool try) mutex_unlock(&vdev->vma_lock); if (try) { - if (!down_read_trylock(&mm->mmap_sem)) { + if (!mmap_read_trylock(mm)) { mmput(mm); return 0; } } else { - down_read(&mm->mmap_sem); + mmap_read_lock(mm); } if (mmget_still_valid(mm)) { if (try) { if (!mutex_trylock(&vdev->vma_lock)) { - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); mmput(mm); return 0; } @@ -1283,7 +1283,7 @@ static int vfio_pci_zap_and_vma_lock(struct vfio_pci_device *vdev, bool try) } mutex_unlock(&vdev->vma_lock); } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); mmput(mm); } } diff --git a/drivers/vfio/vfio_iommu_spapr_tce.c b/drivers/vfio/vfio_iommu_spapr_tce.c index ec53310f1613..5a5bd0e21444 100644 --- a/drivers/vfio/vfio_iommu_spapr_tce.c +++ b/drivers/vfio/vfio_iommu_spapr_tce.c @@ -44,7 +44,7 @@ static long try_increment_locked_vm(struct mm_struct *mm, long npages) if (!npages) return 0; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); locked = mm->locked_vm + npages; lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; if (locked > lock_limit && !capable(CAP_IPC_LOCK)) @@ -58,7 +58,7 @@ static long try_increment_locked_vm(struct mm_struct *mm, long npages) rlimit(RLIMIT_MEMLOCK), ret ? " - exceeded" : ""); - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return ret; } @@ -68,7 +68,7 @@ static void decrement_locked_vm(struct mm_struct *mm, long npages) if (!mm || !npages) return; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); if (WARN_ON_ONCE(npages > mm->locked_vm)) npages = mm->locked_vm; mm->locked_vm -= npages; @@ -76,7 +76,7 @@ static void decrement_locked_vm(struct mm_struct *mm, long npages) npages << PAGE_SHIFT, mm->locked_vm << PAGE_SHIFT, rlimit(RLIMIT_MEMLOCK)); - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); } /* diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 3f20cc205666..6d9fe627e8aa 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -272,7 +272,7 @@ static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async) if (!mm) return -ESRCH; /* process exited */ - ret = down_write_killable(&mm->mmap_sem); + ret = mmap_write_lock_killable(mm); if (!ret) { if (npage > 0) { if (!dma->lock_cap) { @@ -289,7 +289,7 @@ static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async) if (!ret) mm->locked_vm += npage; - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); } if (async) @@ -381,7 +381,7 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr, if (prot & IOMMU_WRITE) flags |= FOLL_WRITE; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); if (mm == current->mm) { ret = get_user_pages_longterm(vaddr, 1, flags, page, vmas); } else { @@ -399,14 +399,14 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr, put_page(page[0]); } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (ret == 1) { *pfn = page_to_pfn(page[0]); return 0; } - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vaddr = untagged_addr(vaddr); @@ -422,7 +422,7 @@ retry: ret = -EFAULT; } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return ret; } diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c index e5d4a1e1411d..06cd04ed7b3c 100644 --- a/drivers/xen/gntdev.c +++ b/drivers/xen/gntdev.c @@ -823,7 +823,7 @@ static long gntdev_ioctl_get_offset_for_vaddr(struct gntdev_priv *priv, return -EFAULT; pr_debug("priv %p, offset for vaddr %lx\n", priv, (unsigned long)op.vaddr); - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); vma = find_vma(current->mm, op.vaddr); if (!vma || vma->vm_ops != &gntdev_vmops) goto out_unlock; @@ -837,7 +837,7 @@ static long gntdev_ioctl_get_offset_for_vaddr(struct gntdev_priv *priv, rv = 0; out_unlock: - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); if (rv == 0 && copy_to_user(u, &op, sizeof(op)) != 0) return -EFAULT; diff --git a/drivers/xen/privcmd.c b/drivers/xen/privcmd.c index 15ece1041d17..12c965f83dcf 100644 --- a/drivers/xen/privcmd.c +++ b/drivers/xen/privcmd.c @@ -277,7 +277,7 @@ static long privcmd_ioctl_mmap(struct file *file, void __user *udata) if (rc || list_empty(&pagelist)) goto out; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); { struct page *page = list_first_entry(&pagelist, @@ -302,7 +302,7 @@ static long privcmd_ioctl_mmap(struct file *file, void __user *udata) out_up: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); out: free_page_list(&pagelist); @@ -498,7 +498,7 @@ static long privcmd_ioctl_mmap_batch( } } - down_write(&mm->mmap_sem); + mmap_write_lock(mm); vma = find_vma(mm, m.addr); if (!vma || @@ -554,7 +554,7 @@ static long privcmd_ioctl_mmap_batch( BUG_ON(traverse_pages_block(m.num, sizeof(xen_pfn_t), &pagelist, mmap_batch_fn, &state)); - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); if (state.global_error) { /* Write back errors in second pass. */ @@ -575,7 +575,7 @@ out: return ret; out_unlock: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); goto out; } @@ -777,7 +777,7 @@ static long privcmd_ioctl_mmap_resource(struct file *file, return __put_user(xdata.nr_frames, &udata->num); } - down_write(&mm->mmap_sem); + mmap_write_lock(mm); vma = find_vma(mm, kdata.addr); if (!vma || vma->vm_ops != &privcmd_vm_ops) { @@ -859,7 +859,7 @@ static long privcmd_ioctl_mmap_resource(struct file *file, } out: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); kfree(pfns); return rc; diff --git a/fs/aio.c b/fs/aio.c index a37d9a450600..b9eaef6b4f38 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -519,7 +519,7 @@ static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events) ctx->mmap_size = nr_pages * PAGE_SIZE; pr_debug("attempting mmap of %lu bytes\n", ctx->mmap_size); - if (down_write_killable(&mm->mmap_sem)) { + if (mmap_write_lock_killable(mm)) { ctx->mmap_size = 0; aio_free_ring(ctx); return -EINTR; @@ -528,7 +528,7 @@ static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events) ctx->mmap_base = do_mmap_pgoff(ctx->aio_ring_file, 0, ctx->mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED, 0, &unused, NULL); - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); if (IS_ERR((void *)ctx->mmap_base)) { ctx->mmap_size = 0; aio_free_ring(ctx); diff --git a/fs/coredump.c b/fs/coredump.c index 506650c85a83..109b4e200f5a 100644 --- a/fs/coredump.c +++ b/fs/coredump.c @@ -417,12 +417,12 @@ static int coredump_wait(int exit_code, struct core_state *core_state) core_state->dumper.task = tsk; core_state->dumper.next = NULL; - if (down_write_killable(&mm->mmap_sem)) + if (mmap_write_lock_killable(mm)) return -EINTR; if (!mm->core_state) core_waiters = zap_threads(tsk, mm, core_state, exit_code); - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); if (core_waiters > 0) { struct core_thread *ptr; diff --git a/fs/exec.c b/fs/exec.c index b91811b0f3f9..9a70a2f943d8 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -295,7 +295,7 @@ static int __bprm_mm_init(struct linux_binprm *bprm) return -ENOMEM; vma_set_anonymous(vma); - if (down_write_killable(&mm->mmap_sem)) { + if (mmap_write_lock_killable(mm)) { err = -EINTR; goto err_free; } @@ -318,11 +318,11 @@ static int __bprm_mm_init(struct linux_binprm *bprm) mm->stack_vm = mm->total_vm = 1; arch_bprm_mm_init(mm, vma); - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); bprm->p = vma->vm_end - sizeof(void *); return 0; err: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); err_free: bprm->vma = NULL; vm_area_free(vma); @@ -735,7 +735,7 @@ int setup_arg_pages(struct linux_binprm *bprm, bprm->loader -= stack_shift; bprm->exec -= stack_shift; - if (down_write_killable(&mm->mmap_sem)) + if (mmap_write_lock_killable(mm)) return -EINTR; vm_flags = VM_STACK_FLAGS; @@ -792,7 +792,7 @@ int setup_arg_pages(struct linux_binprm *bprm, ret = -EFAULT; out_unlock: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return ret; } EXPORT_SYMBOL(setup_arg_pages); @@ -1022,9 +1022,9 @@ static int exec_mmap(struct mm_struct *mm) * through with the exec. We must hold mmap_sem around * checking core_state and changing tsk->mm. */ - down_read(&old_mm->mmap_sem); + mmap_read_lock(old_mm); if (unlikely(old_mm->core_state)) { - up_read(&old_mm->mmap_sem); + mmap_read_unlock(old_mm); return -EINTR; } } @@ -1050,7 +1050,7 @@ static int exec_mmap(struct mm_struct *mm) vmacache_flush(tsk); task_unlock(tsk); if (old_mm) { - up_read(&old_mm->mmap_sem); + mmap_read_unlock(old_mm); BUG_ON(active_mm != old_mm); setmax_mm_hiwater_rss(&tsk->signal->maxrss, old_mm); mm_update_next_owner(old_mm); diff --git a/fs/proc/base.c b/fs/proc/base.c index 703cda9c34aa..6c0b18ae383d 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -2232,11 +2232,11 @@ static int map_files_d_revalidate(struct dentry *dentry, unsigned int flags) goto out; if (!dname_to_vma_addr(dentry, &vm_start, &vm_end)) { - status = down_read_killable(&mm->mmap_sem); + status = mmap_read_lock_killable(mm); if (!status) { exact_vma_exists = !!find_exact_vma(mm, vm_start, vm_end); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); } } @@ -2283,7 +2283,7 @@ static int map_files_get_link(struct dentry *dentry, struct path *path) if (rc) goto out_mmput; - rc = down_read_killable(&mm->mmap_sem); + rc = mmap_read_lock_killable(mm); if (rc) goto out_mmput; @@ -2294,7 +2294,7 @@ static int map_files_get_link(struct dentry *dentry, struct path *path) path_get(path); rc = 0; } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); out_mmput: mmput(mm); @@ -2384,7 +2384,7 @@ static struct dentry *proc_map_files_lookup(struct inode *dir, goto out_put_task; result = ERR_PTR(-EINTR); - if (down_read_killable(&mm->mmap_sem)) + if (mmap_read_lock_killable(mm)) goto out_put_mm; result = ERR_PTR(-ENOENT); @@ -2397,7 +2397,7 @@ static struct dentry *proc_map_files_lookup(struct inode *dir, (void *)(unsigned long)vma->vm_file->f_mode); out_no_vma: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); out_put_mm: mmput(mm); out_put_task: @@ -2441,7 +2441,7 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) if (!mm) goto out_put_task; - ret = down_read_killable(&mm->mmap_sem); + ret = mmap_read_lock_killable(mm); if (ret) { mmput(mm); goto out_put_task; @@ -2472,7 +2472,7 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) ret = -ENOMEM; if (fa) flex_array_free(fa); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); mmput(mm); goto out_put_task; } @@ -2490,7 +2490,7 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) BUG(); } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); mmput(mm); for (i = 0; i < nr_files; i++) { diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 1801f9591814..fb7138e6a40a 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -180,7 +180,7 @@ static void vma_stop(struct proc_maps_private *priv) struct mm_struct *mm = priv->mm; release_task_mempolicy(priv); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); mmput(mm); } @@ -218,7 +218,7 @@ static void *m_start(struct seq_file *m, loff_t *ppos) if (!mm || !mmget_not_zero(mm)) return NULL; - if (down_read_killable(&mm->mmap_sem)) { + if (mmap_read_lock_killable(mm)) { mmput(mm); return ERR_PTR(-EINTR); } @@ -897,7 +897,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) memset(&mss, 0, sizeof(mss)); - ret = down_read_killable(&mm->mmap_sem); + ret = mmap_read_lock_killable(mm); if (ret) goto out_put_mm; @@ -916,7 +916,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) __show_smap(m, &mss); release_task_mempolicy(priv); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); out_put_mm: mmput(mm); @@ -1188,7 +1188,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, }; if (type == CLEAR_REFS_MM_HIWATER_RSS) { - if (down_write_killable(&mm->mmap_sem)) { + if (mmap_write_lock_killable(mm)) { count = -EINTR; goto out_mm; } @@ -1198,11 +1198,11 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, * resident set size to this mm's current rss value. */ reset_mm_hiwater_rss(mm); - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); goto out_mm; } - if (down_read_killable(&mm->mmap_sem)) { + if (mmap_read_lock_killable(mm)) { count = -EINTR; goto out_mm; } @@ -1211,8 +1211,8 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, for (vma = mm->mmap; vma; vma = vma->vm_next) { if (!(vma->vm_flags & VM_SOFTDIRTY)) continue; - up_read(&mm->mmap_sem); - if (down_write_killable(&mm->mmap_sem)) { + mmap_read_unlock(mm); + if (mmap_write_lock_killable(mm)) { count = -EINTR; goto out_mm; } @@ -1231,14 +1231,14 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, * failed like if * get_proc_task() fails? */ - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); goto out_mm; } for (vma = mm->mmap; vma; vma = vma->vm_next) { vma->vm_flags &= ~VM_SOFTDIRTY; vma_set_page_prot(vma); } - downgrade_write(&mm->mmap_sem); + mmap_write_downgrade(mm); break; } mmu_notifier_invalidate_range_start(mm, 0, -1); @@ -1247,7 +1247,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, if (type == CLEAR_REFS_SOFT_DIRTY) mmu_notifier_invalidate_range_end(mm, 0, -1); tlb_finish_mmu(&tlb, 0, -1); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); out_mm: mmput(mm); } @@ -1609,11 +1609,11 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, /* overflow ? */ if (end < start_vaddr || end > end_vaddr) end = end_vaddr; - ret = down_read_killable(&mm->mmap_sem); + ret = mmap_read_lock_killable(mm); if (ret) goto out_free; ret = walk_page_range(start_vaddr, end, &pagemap_walk); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); start_vaddr = end; len = min(count, PM_ENTRY_BYTES * pm.pos); @@ -1817,11 +1817,11 @@ struct reclaim_param reclaim_task_nomap(struct task_struct *task, mm = get_task_mm(task); if (!mm) goto out; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); proc_reclaim_notify((unsigned long)task_pid(task), (void *)&rp); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); mmput(mm); out: put_task_struct(task); @@ -1848,7 +1848,7 @@ struct reclaim_param reclaim_task_anon(struct task_struct *task, reclaim_walk.private = &rp; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); for (vma = mm->mmap; vma; vma = vma->vm_next) { if (is_vm_hugetlb_page(vma)) continue; @@ -1865,7 +1865,7 @@ struct reclaim_param reclaim_task_anon(struct task_struct *task, } flush_tlb_mm(mm); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); mmput(mm); out: put_task_struct(task); @@ -1951,7 +1951,7 @@ static ssize_t reclaim_write(struct file *file, const char __user *buf, rp.nr_reclaimed = 0; reclaim_walk.private = &rp; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); if (type == RECLAIM_RANGE) { vma = find_vma(mm, start); while (vma) { @@ -1984,7 +1984,7 @@ static ssize_t reclaim_write(struct file *file, const char __user *buf, } flush_tlb_mm(mm); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); mmput(mm); out: put_task_struct(task); diff --git a/fs/proc/task_nommu.c b/fs/proc/task_nommu.c index 5161894a6d62..44b43e8849ad 100644 --- a/fs/proc/task_nommu.c +++ b/fs/proc/task_nommu.c @@ -25,7 +25,7 @@ void task_mem(struct seq_file *m, struct mm_struct *mm) struct rb_node *p; unsigned long bytes = 0, sbytes = 0, slack = 0, size; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) { vma = rb_entry(p, struct vm_area_struct, vm_rb); @@ -77,7 +77,7 @@ void task_mem(struct seq_file *m, struct mm_struct *mm) "Shared:\t%8lu bytes\n", bytes, slack, sbytes); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); } unsigned long task_vsize(struct mm_struct *mm) @@ -86,12 +86,12 @@ unsigned long task_vsize(struct mm_struct *mm) struct rb_node *p; unsigned long vsize = 0; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) { vma = rb_entry(p, struct vm_area_struct, vm_rb); vsize += vma->vm_end - vma->vm_start; } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return vsize; } @@ -104,7 +104,7 @@ unsigned long task_statm(struct mm_struct *mm, struct rb_node *p; unsigned long size = kobjsize(mm); - down_read(&mm->mmap_sem); + mmap_read_lock(mm); for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) { vma = rb_entry(p, struct vm_area_struct, vm_rb); size += kobjsize(vma); @@ -119,7 +119,7 @@ unsigned long task_statm(struct mm_struct *mm, >> PAGE_SHIFT; *data = (PAGE_ALIGN(mm->start_stack) - (mm->start_data & PAGE_MASK)) >> PAGE_SHIFT; - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); size >>= PAGE_SHIFT; size += *text + *data; *resident = size; @@ -211,7 +211,7 @@ static void *m_start(struct seq_file *m, loff_t *pos) if (!mm || !mmget_not_zero(mm)) return NULL; - if (down_read_killable(&mm->mmap_sem)) { + if (mmap_read_lock_killable(mm)) { mmput(mm); return ERR_PTR(-EINTR); } @@ -221,7 +221,7 @@ static void *m_start(struct seq_file *m, loff_t *pos) if (n-- == 0) return p; - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); mmput(mm); return NULL; } @@ -231,7 +231,7 @@ static void m_stop(struct seq_file *m, void *_vml) struct proc_maps_private *priv = m->private; if (!IS_ERR_OR_NULL(_vml)) { - up_read(&priv->mm->mmap_sem); + mmap_read_unlock(priv->mm); mmput(priv->mm); } if (priv->task) { diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 76de58f5b18a..97e10307fbfd 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -519,7 +519,7 @@ vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason) must_wait = userfaultfd_huge_must_wait(ctx, vmf->vma, vmf->address, vmf->flags, reason); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (likely(must_wait && !READ_ONCE(ctx->released) && !userfaultfd_signal_pending(vmf->flags))) { @@ -642,7 +642,7 @@ static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx, struct mm_struct *mm = release_new_ctx->mm; /* the various vma->vm_userfaultfd_ctx still points to it */ - down_write(&mm->mmap_sem); + mmap_write_lock(mm); /* no task can run (and in turn coredump) yet */ VM_WARN_ON(!mmget_still_valid(mm)); for (vma = mm->mmap; vma; vma = vma->vm_next) @@ -650,7 +650,7 @@ static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx, vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; vma->vm_flags &= ~__VM_UFFD_FLAGS; } - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); userfaultfd_ctx_put(release_new_ctx); } @@ -803,7 +803,7 @@ bool userfaultfd_remove(struct vm_area_struct *vma, userfaultfd_ctx_get(ctx); WRITE_ONCE(ctx->mmap_changing, true); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); msg_init(&ewq.msg); @@ -898,7 +898,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file) * it's critical that released is set to true (above), before * taking the mmap_sem for writing. */ - down_write(&mm->mmap_sem); + mmap_write_lock(mm); still_valid = mmget_still_valid(mm); prev = NULL; for (vma = mm->mmap; vma; vma = vma->vm_next) { @@ -925,7 +925,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file) vma->vm_flags = new_flags; vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; } - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); mmput(mm); wakeup: /* @@ -1359,7 +1359,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, if (!mmget_not_zero(mm)) goto out; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); if (!mmget_still_valid(mm)) goto out_unlock; vma = find_vma_prev(mm, start, &prev); @@ -1507,7 +1507,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, vma = vma->vm_next; } while (vma && vma->vm_start < end); out_unlock: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); mmput(mm); if (!ret) { __u64 ioctls_out; @@ -1559,7 +1559,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, if (!mmget_not_zero(mm)) goto out; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); if (!mmget_still_valid(mm)) goto out_unlock; vma = find_vma_prev(mm, start, &prev); @@ -1677,7 +1677,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, vma = vma->vm_next; } while (vma && vma->vm_start < end); out_unlock: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); mmput(mm); out: return ret; diff --git a/ipc/shm.c b/ipc/shm.c index 14282b7f9800..8356778591e9 100644 --- a/ipc/shm.c +++ b/ipc/shm.c @@ -1617,7 +1617,7 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, if (err) goto out_fput; - if (down_write_killable(¤t->mm->mmap_sem)) { + if (mmap_write_lock_killable(current->mm)) { err = -EINTR; goto out_fput; } @@ -1637,7 +1637,7 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, if (IS_ERR_VALUE(addr)) err = (long)addr; invalid: - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); if (populate) mm_populate(addr, populate); @@ -1712,7 +1712,7 @@ long ksys_shmdt(char __user *shmaddr) if (addr & ~PAGE_MASK) return retval; - if (down_write_killable(&mm->mmap_sem)) + if (mmap_write_lock_killable(mm)) return -EINTR; /* @@ -1800,7 +1800,7 @@ long ksys_shmdt(char __user *shmaddr) #endif - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return retval; } diff --git a/kernel/acct.c b/kernel/acct.c index bddb31472f9e..7f4dce59cf7c 100644 --- a/kernel/acct.c +++ b/kernel/acct.c @@ -572,13 +572,13 @@ void acct_collect(long exitcode, int group_dead) if (group_dead && current->mm) { struct vm_area_struct *vma; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); vma = current->mm->mmap; while (vma) { vsize += vma->vm_end - vma->vm_start; vma = vma->vm_next; } - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); } spin_lock_irq(¤t->sighand->siglock); diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 5be1f521317a..3cc807e5325b 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -331,7 +331,7 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, * with build_id. */ if (!user || !current || !current->mm || irq_work_busy || - down_read_trylock(¤t->mm->mmap_sem) == 0) { + mmap_read_trylock(current->mm) == 0) { /* cannot access current->mm, fall back to ips */ for (i = 0; i < trace_nr; i++) { id_offs[i].status = BPF_STACK_BUILD_ID_IP; @@ -356,7 +356,7 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, } if (!work) { - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); } else { work->sem = ¤t->mm->mmap_sem; irq_work_queue(&work->irq_work); diff --git a/kernel/events/core.c b/kernel/events/core.c index c66ee4aa1f70..20997ecb8b75 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -9389,7 +9389,7 @@ static void perf_event_addr_filters_apply(struct perf_event *event) if (!mm) goto restart; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); } raw_spin_lock_irqsave(&ifh->lock, flags); @@ -9415,7 +9415,7 @@ static void perf_event_addr_filters_apply(struct perf_event *event) raw_spin_unlock_irqrestore(&ifh->lock, flags); if (ifh->nr_file_filters) { - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); mmput(mm); } diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 3ca91daddc9f..fb296d3e7c5b 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -805,7 +805,7 @@ register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new) if (err && is_register) goto free; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); vma = find_vma(mm, info->vaddr); if (!vma || !valid_vma(vma, is_register) || file_inode(vma->vm_file) != uprobe->inode) @@ -827,7 +827,7 @@ register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new) } unlock: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); free: mmput(mm); info = free_map_info(info); @@ -979,7 +979,7 @@ static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) struct vm_area_struct *vma; int err = 0; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); for (vma = mm->mmap; vma; vma = vma->vm_next) { unsigned long vaddr; loff_t offset; @@ -996,7 +996,7 @@ static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) vaddr = offset_to_vaddr(vma, uprobe->offset); err |= remove_breakpoint(uprobe, mm, vaddr); } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return err; } @@ -1146,7 +1146,7 @@ static int xol_add_vma(struct mm_struct *mm, struct xol_area *area) struct vm_area_struct *vma; int ret; - if (down_write_killable(&mm->mmap_sem)) + if (mmap_write_lock_killable(mm)) return -EINTR; if (mm->uprobes_state.xol_area) { @@ -1176,7 +1176,7 @@ static int xol_add_vma(struct mm_struct *mm, struct xol_area *area) /* pairs with get_xol_area() */ smp_store_release(&mm->uprobes_state.xol_area, area); /* ^^^ */ fail: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return ret; } @@ -1744,7 +1744,7 @@ static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp) struct uprobe *uprobe = NULL; struct vm_area_struct *vma; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma(mm, bp_vaddr); if (vma && vma->vm_start <= bp_vaddr) { if (valid_vma(vma, false)) { @@ -1762,7 +1762,7 @@ static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp) if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags)) mmf_recalc_uprobes(mm); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return uprobe; } diff --git a/kernel/exit.c b/kernel/exit.c index 14e6093a7555..b1a784f17374 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -564,12 +564,12 @@ static void exit_mm(void) * will increment ->nr_threads for each thread in the * group with ->mm != NULL. */ - down_read(&mm->mmap_sem); + mmap_read_lock(mm); core_state = mm->core_state; if (core_state) { struct core_thread self; - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); self.task = current; if (self.task->flags & PF_SIGNALED) @@ -590,14 +590,14 @@ static void exit_mm(void) freezable_schedule(); } __set_current_state(TASK_RUNNING); - down_read(&mm->mmap_sem); + mmap_read_lock(mm); } mmgrab(mm); BUG_ON(mm != current->active_mm); /* more a memory barrier than a real lock */ task_lock(current); current->mm = NULL; - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); enter_lazy_tlb(mm, current); task_unlock(current); mm_update_next_owner(mm); diff --git a/kernel/fork.c b/kernel/fork.c index 1308fd5e22eb..0f61e1bf45a9 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -440,7 +440,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, LIST_HEAD(uf); uprobe_start_dup_mmap(); - if (down_write_killable(&oldmm->mmap_sem)) { + if (mmap_write_lock_killable(oldmm)) { retval = -EINTR; goto fail_uprobe_end; } @@ -564,9 +564,9 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, /* a new mm has just been created */ retval = arch_dup_mmap(oldmm, mm); out: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); flush_tlb_mm(oldmm); - up_write(&oldmm->mmap_sem); + mmap_write_unlock(oldmm); dup_userfaultfd_complete(&uf); fail_uprobe_end: uprobe_end_dup_mmap(); @@ -596,9 +596,9 @@ static inline void mm_free_pgd(struct mm_struct *mm) #else static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) { - down_write(&oldmm->mmap_sem); + mmap_write_lock(oldmm); RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm)); - up_write(&oldmm->mmap_sem); + mmap_write_unlock(oldmm); return 0; } #define mm_alloc_pgd(mm) (0) @@ -964,7 +964,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, mm->vmacache_seqnum = 0; atomic_set(&mm->mm_users, 1); atomic_set(&mm->mm_count, 1); - init_rwsem(&mm->mmap_sem); + mmap_init_lock(mm); INIT_LIST_HEAD(&mm->mmlist); mm->core_state = NULL; mm_pgtables_bytes_init(mm); diff --git a/kernel/futex.c b/kernel/futex.c index ca2a2a894839..c1b775eace71 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -752,10 +752,10 @@ static int fault_in_user_writeable(u32 __user *uaddr) struct mm_struct *mm = current->mm; int ret; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); ret = fixup_user_fault(current, mm, (unsigned long)uaddr, FAULT_FLAG_WRITE, NULL); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return ret < 0 ? ret : 0; } diff --git a/kernel/sched/core.c b/kernel/sched/core.c index f42e7402fac2..396125964271 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5834,7 +5834,7 @@ bool is_sched_lib_based_app(pid_t pid) if (!mm) goto put_task_struct; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); for (vma = mm->mmap; vma ; vma = vma->vm_next) { if (vma->vm_file && vma->vm_flags & VM_EXEC) { name = d_path(&vma->vm_file->f_path, @@ -5856,7 +5856,7 @@ bool is_sched_lib_based_app(pid_t pid) } release_sem: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); mmput(mm); put_task_struct: put_task_struct(p); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index f4d5cdf5373d..4936c32411b0 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -2654,7 +2654,7 @@ void task_numa_work(struct callback_head *work) return; - if (!down_read_trylock(&mm->mmap_sem)) + if (!mmap_read_trylock(mm)) return; vma = find_vma(mm, start); if (!vma) { @@ -2722,7 +2722,7 @@ out: mm->numa_scan_offset = start; else reset_ptenuma_scan(p); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* * Make sure tasks use at least 32x as much time to run other code diff --git a/kernel/sys.c b/kernel/sys.c index 6c6b671033f7..a19ddda6818f 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -1873,7 +1873,7 @@ static int prctl_set_mm_exe_file(struct mm_struct *mm, unsigned int fd) if (exe_file) { struct vm_area_struct *vma; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); for (vma = mm->mmap; vma; vma = vma->vm_next) { if (!vma->vm_file) continue; @@ -1882,7 +1882,7 @@ static int prctl_set_mm_exe_file(struct mm_struct *mm, unsigned int fd) goto exit_err; } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); fput(exe_file); } @@ -1896,7 +1896,7 @@ exit: fdput(exe); return err; exit_err: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); fput(exe_file); goto exit; } @@ -2032,7 +2032,7 @@ static int prctl_set_mm_map(int opt, const void __user *addr, unsigned long data * arg_lock protects concurent updates but we still need mmap_sem for * read to exclude races with sys_brk. */ - down_read(&mm->mmap_sem); + mmap_read_lock(mm); /* * We don't validate if these members are pointing to @@ -2071,7 +2071,7 @@ static int prctl_set_mm_map(int opt, const void __user *addr, unsigned long data if (prctl_map.auxv_size) memcpy(mm->saved_auxv, user_auxv, sizeof(user_auxv)); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return 0; } #endif /* CONFIG_CHECKPOINT_RESTORE */ @@ -2138,7 +2138,7 @@ static int prctl_set_mm(int opt, unsigned long addr, error = -EINVAL; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); vma = find_vma(mm, addr); prctl_map.start_code = mm->start_code; @@ -2231,7 +2231,7 @@ static int prctl_set_mm(int opt, unsigned long addr, error = 0; out: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return error; } @@ -2401,7 +2401,7 @@ static int prctl_set_vma(unsigned long opt, unsigned long start, if (end == start) return 0; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); switch (opt) { case PR_SET_VMA_ANON_NAME: @@ -2411,7 +2411,7 @@ static int prctl_set_vma(unsigned long opt, unsigned long start, error = -EINVAL; } - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return error; } @@ -2601,13 +2601,13 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, case PR_SET_THP_DISABLE: if (arg3 || arg4 || arg5) return -EINVAL; - if (down_write_killable(&me->mm->mmap_sem)) + if (mmap_write_lock_killable(me->mm)) return -EINTR; if (arg2) set_bit(MMF_DISABLE_THP, &me->mm->flags); else clear_bit(MMF_DISABLE_THP, &me->mm->flags); - up_write(&me->mm->mmap_sem); + mmap_write_unlock(me->mm); break; case PR_MPX_ENABLE_MANAGEMENT: if (arg2 || arg3 || arg4 || arg5) diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c index 62015d62dd6f..0319e1d870ae 100644 --- a/kernel/trace/trace_output.c +++ b/kernel/trace/trace_output.c @@ -397,7 +397,7 @@ static int seq_print_user_ip(struct trace_seq *s, struct mm_struct *mm, if (mm) { const struct vm_area_struct *vma; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma(mm, ip); if (vma) { file = vma->vm_file; @@ -409,7 +409,7 @@ static int seq_print_user_ip(struct trace_seq *s, struct mm_struct *mm, trace_seq_printf(s, "[+0x%lx]", ip - vmstart); } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); } if (ret && ((sym_flags & TRACE_ITER_SYM_ADDR) || !file)) trace_seq_printf(s, " <" IP_FMT ">", ip); diff --git a/mm/filemap.c b/mm/filemap.c index bde5b8ded7b1..a1ead44a24ab 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1407,7 +1407,7 @@ int __sched __lock_page_or_retry(struct page *page, struct mm_struct *mm, if (flags & FAULT_FLAG_RETRY_NOWAIT) return 0; - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (flags & FAULT_FLAG_KILLABLE) wait_on_page_locked_killable(page); else @@ -1419,7 +1419,7 @@ int __sched __lock_page_or_retry(struct page *page, struct mm_struct *mm, ret = __lock_page_killable(page); if (ret) { - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return 0; } } else @@ -2515,7 +2515,7 @@ static int lock_page_maybe_drop_mmap(struct vm_fault *vmf, struct page *page, * mmap_sem here and return 0 if we don't have a fpin. */ if (*fpin == NULL) - up_read(&vmf->vma->vm_mm->mmap_sem); + mmap_read_unlock(vmf->vma->vm_mm); return 0; } } else diff --git a/mm/frame_vector.c b/mm/frame_vector.c index 5779c1bc6f44..37c6710b1eb6 100644 --- a/mm/frame_vector.c +++ b/mm/frame_vector.c @@ -47,7 +47,7 @@ int get_vaddr_frames(unsigned long start, unsigned int nr_frames, start = untagged_addr(start); - down_read(&mm->mmap_sem); + mmap_read_lock(mm); locked = 1; vma = find_vma_intersection(mm, start, start + 1); if (!vma) { @@ -83,7 +83,7 @@ int get_vaddr_frames(unsigned long start, unsigned int nr_frames, out: if (locked) - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (!ret) ret = -EFAULT; if (ret > 0) diff --git a/mm/gup.c b/mm/gup.c index 82dbe924b7c1..53875e393220 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -916,7 +916,7 @@ retry: } if (ret & VM_FAULT_RETRY) { - down_read(&mm->mmap_sem); + mmap_read_lock(mm); *unlocked = true; fault_flags |= FAULT_FLAG_TRIED; goto retry; @@ -1012,7 +1012,7 @@ retry: break; } - ret = down_read_killable(&mm->mmap_sem); + ret = mmap_read_lock_killable(mm); if (ret) { BUG_ON(ret > 0); if (!pages_done) @@ -1046,7 +1046,7 @@ retry: * We must let the caller know we temporarily dropped the lock * and so the critical section protected by it was lost. */ - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); *locked = 0; } return pages_done; @@ -1105,11 +1105,11 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, int locked = 1; long ret; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); ret = __get_user_pages_locked(current, mm, start, nr_pages, pages, NULL, &locked, gup_flags | FOLL_TOUCH); if (locked) - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return ret; } EXPORT_SYMBOL(get_user_pages_unlocked); @@ -1344,7 +1344,7 @@ int __mm_populate(unsigned long start, unsigned long len, int ignore_errors) */ if (!locked) { locked = 1; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma(mm, nstart); } else if (nstart >= vma->vm_end) vma = vma->vm_next; @@ -1376,7 +1376,7 @@ int __mm_populate(unsigned long start, unsigned long len, int ignore_errors) ret = 0; } if (locked) - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return ret; /* 0 or negative error code */ } diff --git a/mm/internal.h b/mm/internal.h index a0e59cfa14b8..3366665699df 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -407,7 +407,7 @@ static inline struct file *maybe_unlock_mmap_for_io(struct vm_fault *vmf, if (fault_flag_allow_retry_first(flags) && !(flags & FAULT_FLAG_RETRY_NOWAIT)) { fpin = get_file(vmf->vma->vm_file); - up_read(&vmf->vma->vm_mm->mmap_sem); + mmap_read_unlock(vmf->vma->vm_mm); } return fpin; } diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 60f7df987567..2d2901d9c9f1 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -512,8 +512,8 @@ void __khugepaged_exit(struct mm_struct *mm) * khugepaged has finished working on the pagetables * under the mmap_sem. */ - down_write(&mm->mmap_sem); - up_write(&mm->mmap_sem); + mmap_write_lock(mm); + mmap_write_unlock(mm); } } @@ -934,7 +934,7 @@ static bool __collapse_huge_page_swapin(struct mm_struct *mm, /* do_swap_page returns VM_FAULT_RETRY with released mmap_sem */ if (ret & VM_FAULT_RETRY) { - down_read(&mm->mmap_sem); + mmap_read_lock(mm); if (hugepage_vma_revalidate(mm, address, &vmf.vma)) { /* vma is no longer available, don't continue to swapin */ trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0); @@ -987,7 +987,7 @@ static void collapse_huge_page(struct mm_struct *mm, * sync compaction, and we do not need to hold the mmap_sem during * that. We will recheck the vma after taking it again in write mode. */ - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); new_page = khugepaged_alloc_page(hpage, gfp, node); if (!new_page) { result = SCAN_ALLOC_HUGE_PAGE_FAIL; @@ -999,11 +999,11 @@ static void collapse_huge_page(struct mm_struct *mm, goto out_nolock; } - down_read(&mm->mmap_sem); + mmap_read_lock(mm); result = hugepage_vma_revalidate(mm, address, &vma); if (result) { mem_cgroup_cancel_charge(new_page, memcg, true); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); goto out_nolock; } @@ -1011,7 +1011,7 @@ static void collapse_huge_page(struct mm_struct *mm, if (!pmd) { result = SCAN_PMD_NULL; mem_cgroup_cancel_charge(new_page, memcg, true); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); goto out_nolock; } @@ -1022,17 +1022,17 @@ static void collapse_huge_page(struct mm_struct *mm, */ if (!__collapse_huge_page_swapin(mm, vma, address, pmd, referenced)) { mem_cgroup_cancel_charge(new_page, memcg, true); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); goto out_nolock; } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* * Prevent all access to pagetables with the exception of * gup_fast later handled by the ptep_clear_flush and the VM * handled by the anon_vma lock + PG_lock. */ - down_write(&mm->mmap_sem); + mmap_write_lock(mm); result = hugepage_vma_revalidate(mm, address, &vma); if (result) goto out; @@ -1116,7 +1116,7 @@ static void collapse_huge_page(struct mm_struct *mm, khugepaged_pages_collapsed++; result = SCAN_SUCCEED; out_up_write: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); out_nolock: trace_mm_collapse_huge_page(mm, isolated, result); return; @@ -1301,7 +1301,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) * re-fault. Not ideal, but it's more important to not disturb * the system too much. */ - if (down_write_trylock(&mm->mmap_sem)) { + if (mmap_write_trylock(mm)) { if (!khugepaged_test_exit(mm)) { spinlock_t *ptl; unsigned long end = addr + HPAGE_PMD_SIZE; @@ -1318,7 +1318,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) mmu_notifier_invalidate_range_end(mm, addr, end); } - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); } } i_mmap_unlock_write(mapping); @@ -1741,7 +1741,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, * the next mm on the list. */ vma = NULL; - if (unlikely(!down_read_trylock(&mm->mmap_sem))) + if (unlikely(!mmap_read_trylock(mm))) goto breakouterloop_mmap_sem; if (likely(!khugepaged_test_exit(mm))) vma = find_vma(mm, khugepaged_scan.address); @@ -1786,7 +1786,7 @@ skip: if (!shmem_huge_enabled(vma)) goto skip; file = get_file(vma->vm_file); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); ret = 1; khugepaged_scan_shmem(mm, file->f_mapping, pgoff, hpage); @@ -1807,7 +1807,7 @@ skip: } } breakouterloop: - up_read(&mm->mmap_sem); /* exit_mmap will destroy ptes after this */ + mmap_read_unlock(mm); /* exit_mmap will destroy ptes after this */ breakouterloop_mmap_sem: spin_lock(&khugepaged_mm_lock); diff --git a/mm/ksm.c b/mm/ksm.c index 5d7dbfcc7e34..af9cb791add1 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -542,11 +542,11 @@ static void break_cow(struct rmap_item *rmap_item) */ put_anon_vma(rmap_item->anon_vma); - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_mergeable_vma(mm, addr); if (vma) break_ksm(vma, addr); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); } static struct page *get_mergeable_page(struct rmap_item *rmap_item) @@ -556,7 +556,7 @@ static struct page *get_mergeable_page(struct rmap_item *rmap_item) struct vm_area_struct *vma; struct page *page; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_mergeable_vma(mm, addr); if (!vma) goto out; @@ -572,7 +572,7 @@ static struct page *get_mergeable_page(struct rmap_item *rmap_item) out: page = NULL; } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return page; } @@ -963,7 +963,7 @@ static int unmerge_and_remove_all_rmap_items(void) for (mm_slot = ksm_scan.mm_slot; mm_slot != &ksm_mm_head; mm_slot = ksm_scan.mm_slot) { mm = mm_slot->mm; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); for (vma = mm->mmap; vma; vma = vma->vm_next) { if (ksm_test_exit(mm)) break; @@ -976,7 +976,7 @@ static int unmerge_and_remove_all_rmap_items(void) } remove_trailing_rmap_items(mm_slot, &mm_slot->rmap_list); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); spin_lock(&ksm_mmlist_lock); ksm_scan.mm_slot = list_entry(mm_slot->mm_list.next, @@ -999,7 +999,7 @@ static int unmerge_and_remove_all_rmap_items(void) return 0; error: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); spin_lock(&ksm_mmlist_lock); ksm_scan.mm_slot = &ksm_mm_head; spin_unlock(&ksm_mmlist_lock); @@ -1286,7 +1286,7 @@ static int try_to_merge_with_ksm_page(struct rmap_item *rmap_item, struct vm_area_struct *vma; int err = -EFAULT; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_mergeable_vma(mm, rmap_item->address); if (!vma) goto out; @@ -1302,7 +1302,7 @@ static int try_to_merge_with_ksm_page(struct rmap_item *rmap_item, rmap_item->anon_vma = vma->anon_vma; get_anon_vma(vma->anon_vma); out: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return err; } @@ -2106,7 +2106,7 @@ static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item) if (ksm_use_zero_pages && (checksum == zero_checksum)) { struct vm_area_struct *vma; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_mergeable_vma(mm, rmap_item->address); if (vma) { err = try_to_merge_one_page(vma, page, @@ -2118,7 +2118,7 @@ static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item) */ err = 0; } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* * In case of failure, the page was not really empty, so we * need to continue. Otherwise we're done. @@ -2280,7 +2280,7 @@ next_mm: } mm = slot->mm; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); if (ksm_test_exit(mm)) vma = NULL; else @@ -2314,7 +2314,7 @@ next_mm: ksm_scan.address += PAGE_SIZE; } else put_page(*page); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return rmap_item; } put_page(*page); @@ -2352,10 +2352,10 @@ next_mm: free_mm_slot(slot); clear_bit(MMF_VM_MERGEABLE, &mm->flags); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); mmdrop(mm); } else { - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* * up_read(&mm->mmap_sem) first because after * spin_unlock(&ksm_mmlist_lock) run, the "mm" may @@ -2550,8 +2550,8 @@ void __ksm_exit(struct mm_struct *mm) clear_bit(MMF_VM_MERGEABLE, &mm->flags); mmdrop(mm); } else if (mm_slot) { - down_write(&mm->mmap_sem); - up_write(&mm->mmap_sem); + mmap_write_lock(mm); + mmap_write_unlock(mm); } } diff --git a/mm/madvise.c b/mm/madvise.c index 524e76300157..7af39554abc6 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -523,7 +523,7 @@ static long madvise_dontneed_free(struct vm_area_struct *vma, if (!userfaultfd_remove(vma, start, end)) { *prev = NULL; /* mmap_sem has been dropped, prev is stale */ - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); vma = find_vma(current->mm, start); if (!vma) return -ENOMEM; @@ -605,13 +605,13 @@ static long madvise_remove(struct vm_area_struct *vma, get_file(f); if (userfaultfd_remove(vma, start, end)) { /* mmap_sem was not released by userfaultfd_remove() */ - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); } error = vfs_fallocate(f, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, offset, end - start); fput(f); - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); return error; } @@ -836,10 +836,10 @@ SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior) write = madvise_need_mmap_write(behavior); if (write) { - if (down_write_killable(¤t->mm->mmap_sem)) + if (mmap_write_lock_killable(current->mm)) return -EINTR; } else { - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); } /* @@ -889,9 +889,9 @@ SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior) out: blk_finish_plug(&plug); if (write) - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); else - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); return error; } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 2aa1c978c85b..c77146d9583c 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5117,10 +5117,10 @@ static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm) .pmd_entry = mem_cgroup_count_precharge_pte_range, .mm = mm, }; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); walk_page_range(0, mm->highest_vm_end, &mem_cgroup_count_precharge_walk); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); precharge = mc.precharge; mc.precharge = 0; @@ -5404,7 +5404,7 @@ static void mem_cgroup_move_charge(void) atomic_inc(&mc.from->moving_account); synchronize_rcu(); retry: - if (unlikely(!down_read_trylock(&mc.mm->mmap_sem))) { + if (unlikely(!mmap_read_trylock(mc.mm))) { /* * Someone who are holding the mmap_sem might be waiting in * waitq. So we cancel all extra charges, wake up all waiters, @@ -5422,7 +5422,7 @@ retry: */ walk_page_range(0, mc.mm->highest_vm_end, &mem_cgroup_move_charge_walk); - up_read(&mc.mm->mmap_sem); + mmap_read_unlock(mc.mm); atomic_dec(&mc.from->moving_account); } diff --git a/mm/memory.c b/mm/memory.c index b4be287376a2..6ab06d82c780 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1573,7 +1573,7 @@ int vm_insert_page(struct vm_area_struct *vma, unsigned long addr, if (!page_count(page)) return -EINVAL; if (!(vma->vm_flags & VM_MIXEDMAP)) { - BUG_ON(down_read_trylock(&vma->vm_mm->mmap_sem)); + BUG_ON(mmap_read_trylock(vma->vm_mm)); BUG_ON(vma->vm_flags & VM_PFNMAP); vma->vm_flags |= VM_MIXEDMAP; } @@ -4470,7 +4470,7 @@ int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, void *old_buf = buf; int write = gup_flags & FOLL_WRITE; - if (down_read_killable(&mm->mmap_sem)) + if (mmap_read_lock_killable(mm)) return 0; /* ignore errors, just check how much was successfully transferred */ @@ -4521,7 +4521,7 @@ int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, buf += bytes; addr += bytes; } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return buf - old_buf; } @@ -4576,7 +4576,7 @@ void print_vma_addr(char *prefix, unsigned long ip) /* * we might be running from an atomic context so we cannot sleep */ - if (!down_read_trylock(&mm->mmap_sem)) + if (!mmap_read_trylock(mm)) return; vma = find_vma(mm, ip); @@ -4595,7 +4595,7 @@ void print_vma_addr(char *prefix, unsigned long ip) free_page((unsigned long)buf); } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); } #if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP) diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 49cf0d637deb..1fa93c3559cb 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -379,10 +379,10 @@ void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new) { struct vm_area_struct *vma; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); for (vma = mm->mmap; vma; vma = vma->vm_next) mpol_rebind_policy(vma->vm_policy, new); - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); } static const struct mempolicy_operations mpol_ops[MPOL_MAX] = { @@ -890,10 +890,10 @@ static long do_get_mempolicy(int *policy, nodemask_t *nmask, * vma/shared policy at addr is NULL. We * want to return MPOL_DEFAULT in this case. */ - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma_intersection(mm, addr, addr+1); if (!vma) { - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return -EFAULT; } if (vma->vm_ops && vma->vm_ops->get_policy) @@ -943,7 +943,7 @@ static long do_get_mempolicy(int *policy, nodemask_t *nmask, out: mpol_cond_put(pol); if (vma) - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); return err; } @@ -1050,7 +1050,7 @@ int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from, if (err) return err; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); /* * Find a 'source' bit set in 'tmp' whose corresponding 'dest' @@ -1131,7 +1131,7 @@ int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from, if (err < 0) break; } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (err < 0) return err; return busy; @@ -1254,12 +1254,12 @@ static long do_mbind(unsigned long start, unsigned long len, { NODEMASK_SCRATCH(scratch); if (scratch) { - down_write(&mm->mmap_sem); + mmap_write_lock(mm); task_lock(current); err = mpol_set_nodemask(new, nmask, scratch); task_unlock(current); if (err) - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); } else err = -ENOMEM; NODEMASK_SCRATCH_FREE(scratch); @@ -1296,7 +1296,7 @@ up_out: putback_movable_pages(&pagelist); } - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); mpol_out: mpol_put(new); return err; diff --git a/mm/migrate.c b/mm/migrate.c index 3197153ad4d1..64649229738f 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -1534,7 +1534,7 @@ static int add_page_for_migration(struct mm_struct *mm, unsigned long addr, unsigned int follflags; int err; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); err = -EFAULT; vma = find_vma(mm, addr); if (!vma || addr < vma->vm_start || !vma_migratable(vma)) @@ -1587,7 +1587,7 @@ out_putpage: */ put_page(page); out: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return err; } @@ -1719,7 +1719,7 @@ static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages, { unsigned long i; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); for (i = 0; i < nr_pages; i++) { unsigned long addr = (unsigned long)(*pages); @@ -1746,7 +1746,7 @@ set_status: status++; } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); } /* diff --git a/mm/mincore.c b/mm/mincore.c index 7eb149191550..7f4dc897142d 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -276,9 +276,9 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, * Do at most PAGE_SIZE entries per iteration, due to * the temporary buffer size. */ - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); retval = do_mincore(start, min(pages, PAGE_SIZE), tmp); - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); if (retval <= 0) break; diff --git a/mm/mlock.c b/mm/mlock.c index ef2abafbc18f..1bce8d057fb1 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -686,7 +686,7 @@ static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t fla lock_limit >>= PAGE_SHIFT; locked = len >> PAGE_SHIFT; - if (down_write_killable(¤t->mm->mmap_sem)) + if (mmap_write_lock_killable(current->mm)) return -EINTR; locked += current->mm->locked_vm; @@ -705,7 +705,7 @@ static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t fla if ((locked <= lock_limit) || capable(CAP_IPC_LOCK)) error = apply_vma_lock_flags(start, len, flags); - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); if (error) return error; @@ -742,10 +742,10 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) len = PAGE_ALIGN(len + (offset_in_page(start))); start &= PAGE_MASK; - if (down_write_killable(¤t->mm->mmap_sem)) + if (mmap_write_lock_killable(current->mm)) return -EINTR; ret = apply_vma_lock_flags(start, len, 0); - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); return ret; } @@ -810,14 +810,14 @@ SYSCALL_DEFINE1(mlockall, int, flags) lock_limit = rlimit(RLIMIT_MEMLOCK); lock_limit >>= PAGE_SHIFT; - if (down_write_killable(¤t->mm->mmap_sem)) + if (mmap_write_lock_killable(current->mm)) return -EINTR; ret = -ENOMEM; if (!(flags & MCL_CURRENT) || (current->mm->total_vm <= lock_limit) || capable(CAP_IPC_LOCK)) ret = apply_mlockall_flags(flags); - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); if (!ret && (flags & MCL_CURRENT)) mm_populate(0, TASK_SIZE); @@ -828,10 +828,10 @@ SYSCALL_DEFINE0(munlockall) { int ret; - if (down_write_killable(¤t->mm->mmap_sem)) + if (mmap_write_lock_killable(current->mm)) return -EINTR; ret = apply_mlockall_flags(0); - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); return ret; } diff --git a/mm/mmap.c b/mm/mmap.c index ec3874bbbe2c..6fbeca7262ef 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -193,7 +193,7 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) bool populate; LIST_HEAD(uf); - if (down_write_killable(&mm->mmap_sem)) + if (mmap_write_lock_killable(mm)) return -EINTR; #ifdef CONFIG_COMPAT_BRK @@ -246,7 +246,7 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) set_brk: mm->brk = brk; populate = newbrk > oldbrk && (mm->def_flags & VM_LOCKED) != 0; - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); userfaultfd_unmap_complete(mm, &uf); if (populate) mm_populate(oldbrk, newbrk - oldbrk); @@ -254,7 +254,7 @@ set_brk: out: retval = mm->brk; - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return retval; } @@ -2829,11 +2829,11 @@ int vm_munmap(unsigned long start, size_t len) struct mm_struct *mm = current->mm; LIST_HEAD(uf); - if (down_write_killable(&mm->mmap_sem)) + if (mmap_write_lock_killable(mm)) return -EINTR; ret = do_munmap(mm, start, len, &uf); - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); userfaultfd_unmap_complete(mm, &uf); return ret; } @@ -2875,7 +2875,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, if (pgoff + (size >> PAGE_SHIFT) < pgoff) return ret; - if (down_write_killable(&mm->mmap_sem)) + if (mmap_write_lock_killable(mm)) return -EINTR; vma = find_vma(mm, start); @@ -2938,7 +2938,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, prot, flags, pgoff, &populate, NULL); fput(file); out: - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); if (populate) mm_populate(ret, populate); if (!IS_ERR_VALUE(ret)) @@ -2949,9 +2949,9 @@ out: static inline void verify_mm_writelocked(struct mm_struct *mm) { #ifdef CONFIG_DEBUG_VM - if (unlikely(down_read_trylock(&mm->mmap_sem))) { + if (unlikely(mmap_read_trylock(mm))) { WARN_ON(1); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); } #endif } @@ -3053,12 +3053,12 @@ int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags) if (!len) return 0; - if (down_write_killable(&mm->mmap_sem)) + if (mmap_write_lock_killable(mm)) return -EINTR; ret = do_brk_flags(addr, len, flags, &uf); populate = ((mm->def_flags & VM_LOCKED) != 0); - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); userfaultfd_unmap_complete(mm, &uf); if (populate && !ret) mm_populate(addr, len); @@ -3102,8 +3102,8 @@ void exit_mmap(struct mm_struct *mm) (void)__oom_reap_task_mm(mm); set_bit(MMF_OOM_SKIP, &mm->flags); - down_write(&mm->mmap_sem); - up_write(&mm->mmap_sem); + mmap_write_lock(mm); + mmap_write_unlock(mm); } if (mm->locked_vm) { @@ -3528,7 +3528,7 @@ int mm_take_all_locks(struct mm_struct *mm) struct vm_area_struct *vma; struct anon_vma_chain *avc; - BUG_ON(down_read_trylock(&mm->mmap_sem)); + BUG_ON(mmap_read_trylock(mm)); mutex_lock(&mm_all_locks_mutex); @@ -3608,7 +3608,7 @@ void mm_drop_all_locks(struct mm_struct *mm) struct vm_area_struct *vma; struct anon_vma_chain *avc; - BUG_ON(down_read_trylock(&mm->mmap_sem)); + BUG_ON(mmap_read_trylock(mm)); BUG_ON(!mutex_is_locked(&mm_all_locks_mutex)); for (vma = mm->mmap; vma; vma = vma->vm_next) { diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c index 06dedb175572..5244f2a0c40f 100644 --- a/mm/mmu_notifier.c +++ b/mm/mmu_notifier.c @@ -293,7 +293,7 @@ static int do_mmu_notifier_register(struct mmu_notifier *mn, goto out; if (take_mmap_sem) - down_write(&mm->mmap_sem); + mmap_write_lock(mm); ret = mm_take_all_locks(mm); if (unlikely(ret)) goto out_clean; @@ -322,7 +322,7 @@ static int do_mmu_notifier_register(struct mmu_notifier *mn, mm_drop_all_locks(mm); out_clean: if (take_mmap_sem) - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); kfree(mmu_notifier_mm); out: BUG_ON(atomic_read(&mm->mm_users) <= 0); diff --git a/mm/mprotect.c b/mm/mprotect.c index 9b6160f284c5..c55987fb82ad 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -516,7 +516,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, reqprot = prot; - if (down_write_killable(¤t->mm->mmap_sem)) + if (mmap_write_lock_killable(current->mm)) return -EINTR; /* @@ -606,7 +606,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, prot = reqprot; } out: - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); return error; } @@ -636,7 +636,7 @@ SYSCALL_DEFINE2(pkey_alloc, unsigned long, flags, unsigned long, init_val) if (init_val & ~PKEY_ACCESS_MASK) return -EINVAL; - down_write(¤t->mm->mmap_sem); + mmap_write_lock(current->mm); pkey = mm_pkey_alloc(current->mm); ret = -ENOSPC; @@ -650,7 +650,7 @@ SYSCALL_DEFINE2(pkey_alloc, unsigned long, flags, unsigned long, init_val) } ret = pkey; out: - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); return ret; } @@ -658,9 +658,9 @@ SYSCALL_DEFINE1(pkey_free, int, pkey) { int ret; - down_write(¤t->mm->mmap_sem); + mmap_write_lock(current->mm); ret = mm_pkey_free(current->mm, pkey); - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); /* * We could provie warnings or errors if any VMA still diff --git a/mm/mremap.c b/mm/mremap.c index 59ecf383f27f..a0bc41867fd3 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -833,7 +833,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, if (!new_len) return ret; - if (down_write_killable(¤t->mm->mmap_sem)) + if (mmap_write_lock_killable(current->mm)) return -EINTR; if (flags & (MREMAP_FIXED | MREMAP_DONTUNMAP)) { @@ -922,7 +922,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, out: if (offset_in_page(ret)) locked = 0; - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); if (locked && new_len > old_len) mm_populate(new_addr + old_len, new_len - old_len); userfaultfd_unmap_complete(mm, &uf_unmap_early); diff --git a/mm/msync.c b/mm/msync.c index c3bd3e75f687..69c6d2029531 100644 --- a/mm/msync.c +++ b/mm/msync.c @@ -57,7 +57,7 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags) * If the interval [start,end) covers some unmapped address ranges, * just ignore them, but return -ENOMEM at the end. */ - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma(mm, start); for (;;) { struct file *file; @@ -88,12 +88,12 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags) if ((flags & MS_SYNC) && file && (vma->vm_flags & VM_SHARED)) { get_file(file); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); error = vfs_fsync_range(file, fstart, fend, 1); fput(file); if (error || start >= end) goto out; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); vma = find_vma(mm, start); } else { if (start >= end) { @@ -104,7 +104,7 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags) } } out_unlock: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); out: return error ? : unmapped_error; } diff --git a/mm/nommu.c b/mm/nommu.c index 4dc1b5648872..a68fffa536f5 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -183,10 +183,10 @@ static long __get_user_pages_unlocked(struct task_struct *tsk, unsigned int gup_flags) { long ret; - down_read(&mm->mmap_sem); + mmap_read_lock(mm); ret = __get_user_pages(tsk, mm, start, nr_pages, gup_flags, pages, NULL, NULL); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return ret; } @@ -259,11 +259,11 @@ void *vmalloc_user(unsigned long size) if (ret) { struct vm_area_struct *vma; - down_write(¤t->mm->mmap_sem); + mmap_write_lock(current->mm); vma = find_vma(current->mm, (unsigned long)ret); if (vma) vma->vm_flags |= VM_USERMAP; - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); } return ret; @@ -1626,9 +1626,9 @@ int vm_munmap(unsigned long addr, size_t len) struct mm_struct *mm = current->mm; int ret; - down_write(&mm->mmap_sem); + mmap_write_lock(mm); ret = do_munmap(mm, addr, len, NULL); - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); return ret; } EXPORT_SYMBOL(vm_munmap); @@ -1715,9 +1715,9 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, { unsigned long ret; - down_write(¤t->mm->mmap_sem); + mmap_write_lock(current->mm); ret = do_mremap(addr, old_len, new_len, flags, new_addr); - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); return ret; } @@ -1789,7 +1789,7 @@ int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, struct vm_area_struct *vma; int write = gup_flags & FOLL_WRITE; - if (down_read_killable(&mm->mmap_sem)) + if (mmap_read_lock_killable(mm)) return 0; /* the access must start within one of the target process's mappings */ @@ -1812,7 +1812,7 @@ int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, len = 0; } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return len; } diff --git a/mm/oom_kill.c b/mm/oom_kill.c index d3b29dbcab95..7fa63851e620 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -732,7 +732,7 @@ static bool oom_reap_task_mm(struct task_struct *tsk, struct mm_struct *mm) { bool ret = true; - if (!down_read_trylock(&mm->mmap_sem)) { + if (!mmap_read_trylock(mm)) { trace_skip_task_reaping(tsk->pid); return false; } @@ -763,7 +763,7 @@ static bool oom_reap_task_mm(struct task_struct *tsk, struct mm_struct *mm) out_finish: trace_finish_task_reaping(tsk->pid); out_unlock: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return ret; } diff --git a/mm/process_vm_access.c b/mm/process_vm_access.c index a447092d4635..77e0f5f174bf 100644 --- a/mm/process_vm_access.c +++ b/mm/process_vm_access.c @@ -109,11 +109,11 @@ static int process_vm_rw_single_vec(unsigned long addr, * access remotely because task/mm might not * current/current->mm */ - down_read(&mm->mmap_sem); + mmap_read_lock(mm); pages = get_user_pages_remote(task, mm, pa, pages, flags, process_pages, NULL, &locked); if (locked) - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (pages <= 0) return -EFAULT; diff --git a/mm/swapfile.c b/mm/swapfile.c index 7993fb952d06..ddf3d400d251 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1976,14 +1976,14 @@ static int unuse_mm(struct mm_struct *mm, struct vm_area_struct *vma; int ret = 0; - if (!down_read_trylock(&mm->mmap_sem)) { + if (!mmap_read_trylock(mm)) { /* * Activate page so shrink_inactive_list is unlikely to unmap * its ptes while lock is dropped, so swapoff can make progress. */ activate_page(page); unlock_page(page); - down_read(&mm->mmap_sem); + mmap_read_lock(mm); lock_page(page); } for (vma = mm->mmap; vma; vma = vma->vm_next) { @@ -1991,7 +1991,7 @@ static int unuse_mm(struct mm_struct *mm, break; cond_resched(); } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return (ret < 0)? ret: 0; } diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index cc602336eb8e..c15c41c50607 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -274,7 +274,7 @@ static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm, * feature is not supported. */ if (mode == MCOPY_ATOMIC_ZEROPAGE) { - up_read(&dst_mm->mmap_sem); + mmap_read_unlock(dst_mm); return -EINVAL; } @@ -370,7 +370,7 @@ retry: cond_resched(); if (unlikely(err == -ENOENT)) { - up_read(&dst_mm->mmap_sem); + mmap_read_unlock(dst_mm); BUG_ON(!page); err = copy_huge_page_from_user(page, @@ -380,7 +380,7 @@ retry: err = -EFAULT; goto out; } - down_read(&dst_mm->mmap_sem); + mmap_read_lock(dst_mm); /* * If memory mappings are changing because of non-cooperative * operation (e.g. mremap) running in parallel, bail out and @@ -409,7 +409,7 @@ retry: } out_unlock: - up_read(&dst_mm->mmap_sem); + mmap_read_unlock(dst_mm); out: if (page) { /* @@ -544,7 +544,7 @@ static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm, copied = 0; page = NULL; retry: - down_read(&dst_mm->mmap_sem); + mmap_read_lock(dst_mm); /* * If memory mappings are changing because of non-cooperative @@ -648,7 +648,7 @@ retry: if (unlikely(err == -ENOENT)) { void *page_kaddr; - up_read(&dst_mm->mmap_sem); + mmap_read_unlock(dst_mm); BUG_ON(!page); page_kaddr = kmap(page); @@ -678,7 +678,7 @@ retry: } out_unlock: - up_read(&dst_mm->mmap_sem); + mmap_read_unlock(dst_mm); out: if (page) put_page(page); diff --git a/mm/util.c b/mm/util.c index a4e375c46788..457892211157 100644 --- a/mm/util.c +++ b/mm/util.c @@ -385,11 +385,11 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, ret = security_mmap_file(file, prot, flag); if (!ret) { - if (down_write_killable(&mm->mmap_sem)) + if (mmap_write_lock_killable(mm)) return -EINTR; ret = do_mmap_pgoff(file, addr, len, prot, flag, pgoff, &populate, &uf); - up_write(&mm->mmap_sem); + mmap_write_unlock(mm); userfaultfd_unmap_complete(mm, &uf); if (populate) mm_populate(ret, populate); @@ -799,12 +799,12 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen) if (!mm->arg_end) goto out_mm; /* Shh! No looking before we're done */ - down_read(&mm->mmap_sem); + mmap_read_lock(mm); arg_start = mm->arg_start; arg_end = mm->arg_end; env_start = mm->env_start; env_end = mm->env_end; - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); len = arg_end - arg_start; diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 220b878c9075..52be9c72c500 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1784,11 +1784,11 @@ static int tcp_zerocopy_receive(struct sock *sk, sock_rps_record_flow(sk); - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); vma = find_vma(current->mm, address); if (!vma || vma->vm_start > address || vma->vm_ops != &tcp_vm_ops) { - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); return -EINVAL; } zc->length = min_t(unsigned long, zc->length, vma->vm_end - address); @@ -1835,7 +1835,7 @@ static int tcp_zerocopy_receive(struct sock *sk, frags++; } out: - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); if (length) { WRITE_ONCE(tp->copied_seq, seq); tcp_rcv_space_adjust(sk); diff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c index d8a27944d4c7..7ae9f969f0d4 100644 --- a/net/xdp/xdp_umem.c +++ b/net/xdp/xdp_umem.c @@ -244,10 +244,10 @@ static int xdp_umem_pin_pages(struct xdp_umem *umem) if (!umem->pgs) return -ENOMEM; - down_write(¤t->mm->mmap_sem); + mmap_write_lock(current->mm); npgs = get_user_pages(umem->address, umem->npgs, gup_flags, &umem->pgs[0], NULL); - up_write(¤t->mm->mmap_sem); + mmap_write_unlock(current->mm); if (npgs != umem->npgs) { if (npgs >= 0) { diff --git a/virt/kvm/arm/mmu.c b/virt/kvm/arm/mmu.c index 11103b75c596..b6c5338d4220 100644 --- a/virt/kvm/arm/mmu.c +++ b/virt/kvm/arm/mmu.c @@ -967,7 +967,7 @@ void stage2_unmap_vm(struct kvm *kvm) int idx; idx = srcu_read_lock(&kvm->srcu); - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); spin_lock(&kvm->mmu_lock); slots = kvm_memslots(kvm); @@ -975,7 +975,7 @@ void stage2_unmap_vm(struct kvm *kvm) stage2_unmap_memslot(kvm, memslot); spin_unlock(&kvm->mmu_lock); - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); srcu_read_unlock(&kvm->srcu, idx); } @@ -1508,11 +1508,11 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, } /* Let's check if we will get back a huge page backed by hugetlbfs */ - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); vma = find_vma_intersection(current->mm, hva, hva + 1); if (unlikely(!vma)) { kvm_err("Failed to find VMA for hva 0x%lx\n", hva); - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); return -EFAULT; } @@ -1533,7 +1533,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, ((memslot->base_gfn << PAGE_SHIFT) & ~PMD_MASK)) force_pte = true; } - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); /* We need minimum second+third level pages */ ret = mmu_topup_memory_cache(memcache, KVM_MMU_CACHE_MIN_PAGES, @@ -2084,7 +2084,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, (KVM_PHYS_SIZE >> PAGE_SHIFT)) return -EFAULT; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); /* * A memory region could potentially cover multiple VMAs, and any holes * between them, so iterate over all of them to find out if we can map @@ -2152,7 +2152,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, stage2_flush_memslot(kvm, memslot); spin_unlock(&kvm->mmu_lock); out: - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); return ret; } diff --git a/virt/kvm/async_pf.c b/virt/kvm/async_pf.c index 3e17f8c7432c..4ecef47e13c3 100644 --- a/virt/kvm/async_pf.c +++ b/virt/kvm/async_pf.c @@ -86,11 +86,11 @@ static void async_pf_execute(struct work_struct *work) * mm and might be done in another context, so we must * access remotely. */ - down_read(&mm->mmap_sem); + mmap_read_lock(mm); get_user_pages_remote(NULL, mm, addr, 1, FOLL_WRITE, NULL, NULL, &locked); if (locked) - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); kvm_async_page_present_sync(vcpu, apf); diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 92a998a57772..1a3610bb1a94 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1326,7 +1326,7 @@ unsigned long kvm_host_page_size(struct kvm_vcpu *vcpu, gfn_t gfn) if (kvm_is_error_hva(addr)) return PAGE_SIZE; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); vma = find_vma(current->mm, addr); if (!vma) goto out; @@ -1334,7 +1334,7 @@ unsigned long kvm_host_page_size(struct kvm_vcpu *vcpu, gfn_t gfn) size = vma_kernel_pagesize(vma); out: - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); return size; } @@ -1609,7 +1609,7 @@ static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async, if (npages == 1) return pfn; - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); if (npages == -EHWPOISON || (!async && check_user_page_hwpoison(addr))) { pfn = KVM_PFN_ERR_HWPOISON; @@ -1633,7 +1633,7 @@ retry: pfn = KVM_PFN_ERR_FAULT; } exit: - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); return pfn; }