From c2bad05b7b749f1f9b5924fe52f42c56c6c88bc3 Mon Sep 17 00:00:00 2001 From: Michel Lespinasse Date: Mon, 8 Jun 2020 21:33:47 -0700 Subject: [PATCH] BACKPORT: mmap locking API: rename mmap_sem to mmap_lock Rename the mmap_sem field to mmap_lock. Any new uses of this lock should now go through the new mmap locking api. The mmap_lock is still implemented as a rwsem, though this could change in the future. [akpm@linux-foundation.org: fix it for mm-gup-might_lock_readmmap_sem-in-get_user_pages_fast.patch] Change-Id: I4d550763e380f418e10b6a80e2c88b4cc4caaeea Signed-off-by: Michel Lespinasse Signed-off-by: Andrew Morton Reviewed-by: Vlastimil Babka Reviewed-by: Davidlohr Bueso Reviewed-by: Daniel Jordan Cc: David Rientjes Cc: Hugh Dickins Cc: Jason Gunthorpe Cc: Jerome Glisse Cc: John Hubbard Cc: Laurent Dufour Cc: Liam Howlett Cc: Matthew Wilcox Cc: Peter Zijlstra Cc: Ying Han Link: http://lkml.kernel.org/r/20200520052908.204642-11-walken@google.com Signed-off-by: Linus Torvalds --- arch/ia64/mm/fault.c | 4 ++-- arch/mips/mm/fault.c | 12 +++++------ arch/powerpc/platforms/powernv/npu-dma.c | 2 +- arch/x86/events/core.c | 2 +- arch/x86/mm/fault.c | 2 +- drivers/char/adsprpc.c | 12 +++++------ drivers/gpu/drm/etnaviv/etnaviv_gem.c | 2 +- drivers/misc/okl4-link-shbuf.c | 2 +- fs/userfaultfd.c | 6 +++--- include/linux/mm_types.h | 2 +- include/linux/mmap_lock.h | 26 ++++++++++++------------ kernel/fork.c | 2 +- mm/gup.c | 2 +- mm/init-mm.c | 2 +- mm/khugepaged.c | 2 +- mm/memory.c | 4 ++-- mm/mmap.c | 4 ++-- mm/mmu_notifier.c | 2 +- mm/pagewalk.c | 4 ++-- 19 files changed, 47 insertions(+), 47 deletions(-) diff --git a/arch/ia64/mm/fault.c b/arch/ia64/mm/fault.c index c77e9ee3693e..57346e8482eb 100644 --- a/arch/ia64/mm/fault.c +++ b/arch/ia64/mm/fault.c @@ -92,8 +92,8 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re mask = ((((isr >> IA64_ISR_X_BIT) & 1UL) << VM_EXEC_BIT) | (((isr >> IA64_ISR_W_BIT) & 1UL) << VM_WRITE_BIT)); - /* mmap_sem is performance critical.... */ - prefetchw(&mm->mmap_sem); + /* mmap_lock is performance critical.... */ + prefetchw(&mm->mmap_lock); /* * If we're in an interrupt or have no user context, we must not take the fault.. diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c index 158080ec672b..df55efef4366 100644 --- a/arch/mips/mm/fault.c +++ b/arch/mips/mm/fault.c @@ -97,7 +97,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write, if (user_mode(regs)) flags |= FAULT_FLAG_USER; retry: - down_read(&mm->mmap_sem); + down_read(&mm->mmap_lock); vma = find_vma(mm, address); if (!vma) goto bad_area; @@ -182,7 +182,7 @@ good_area: flags |= FAULT_FLAG_TRIED; /* - * No need to up_read(&mm->mmap_sem) as we would + * No need to up_read(&mm->mmap_lock) as we would * have already released it in __lock_page_or_retry * in mm/filemap.c. */ @@ -191,7 +191,7 @@ good_area: } } - up_read(&mm->mmap_sem); + up_read(&mm->mmap_lock); return; /* @@ -199,7 +199,7 @@ good_area: * Fix it, but check if it's kernel or user first.. */ bad_area: - up_read(&mm->mmap_sem); + up_read(&mm->mmap_lock); bad_area_nosemaphore: /* User mode accesses just cause a SIGSEGV */ @@ -251,14 +251,14 @@ out_of_memory: * We ran out of memory, call the OOM killer, and return the userspace * (which will retry the fault, or kill us if we got oom-killed). */ - up_read(&mm->mmap_sem); + up_read(&mm->mmap_lock); if (!user_mode(regs)) goto no_context; pagefault_out_of_memory(); return; do_sigbus: - up_read(&mm->mmap_sem); + up_read(&mm->mmap_lock); /* Kernel mode? Handle exceptions or die */ if (!user_mode(regs)) diff --git a/arch/powerpc/platforms/powernv/npu-dma.c b/arch/powerpc/platforms/powernv/npu-dma.c index fd8166ffbffa..eb242cd5d08b 100644 --- a/arch/powerpc/platforms/powernv/npu-dma.c +++ b/arch/powerpc/platforms/powernv/npu-dma.c @@ -938,7 +938,7 @@ int pnv_npu2_handle_fault(struct npu_context *context, uintptr_t *ea, if (!firmware_has_feature(FW_FEATURE_OPAL)) return -ENODEV; - WARN_ON(!rwsem_is_locked(&mm->mmap_sem)); + WARN_ON(!rwsem_is_locked(&mm->mmap_lock)); for (i = 0; i < count; i++) { is_write = flags[i] & NPU2_WRITE; diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index f612eb1cc818..160bd127914e 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -2126,7 +2126,7 @@ static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm) * For now, this can't happen because all callers hold mmap_sem * for write. If this changes, we'll need a different solution. */ - lockdep_assert_held_exclusive(&mm->mmap_sem); + lockdep_assert_held_exclusive(&mm->mmap_lock); if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1) on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1); diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index e9ee4bd5d44a..e4a2b4c30fc9 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -1244,7 +1244,7 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code, tsk = current; mm = tsk->mm; - prefetchw(&mm->mmap_sem); + prefetchw(&mm->mmap_lock); if (unlikely(kmmio_fault(regs, address))) return; diff --git a/drivers/char/adsprpc.c b/drivers/char/adsprpc.c index 0c0bf5e03c98..410f2d61cfba 100644 --- a/drivers/char/adsprpc.c +++ b/drivers/char/adsprpc.c @@ -1890,15 +1890,15 @@ static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx) if (map->attr & FASTRPC_ATTR_NOVA) { offset = 0; } else { - down_read(¤t->mm->mmap_sem); + down_read(¤t->mm->mmap_lock); VERIFY(err, NULL != (vma = find_vma(current->mm, map->va))); if (err) { - up_read(¤t->mm->mmap_sem); + up_read(¤t->mm->mmap_lock); goto bail; } offset = buf_page_start(buf) - vma->vm_start; - up_read(¤t->mm->mmap_sem); + up_read(¤t->mm->mmap_lock); VERIFY(err, offset + len <= (uintptr_t)map->size); if (err) @@ -2022,11 +2022,11 @@ static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx) uint64_t flush_len; struct vm_area_struct *vma; - down_read(¤t->mm->mmap_sem); + down_read(¤t->mm->mmap_lock); VERIFY(err, NULL != (vma = find_vma( current->mm, rpra[i].buf.pv))); if (err) { - up_read(¤t->mm->mmap_sem); + up_read(¤t->mm->mmap_lock); goto bail; } if (ctx->overps[oix]->do_cmo) { @@ -2041,7 +2041,7 @@ static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx) ctx->overps[oix]->mend - ctx->overps[oix]->mstart; } - up_read(¤t->mm->mmap_sem); + up_read(¤t->mm->mmap_lock); dma_buf_begin_cpu_access_partial( map->buf, DMA_TO_DEVICE, offset, flush_len); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c index 753875767c3a..1eb60f9669cd 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c @@ -695,7 +695,7 @@ static int etnaviv_gem_userptr_get_pages(struct etnaviv_gem_object *etnaviv_obj) struct etnaviv_gem_userptr *userptr = &etnaviv_obj->userptr; int ret, pinned = 0, npages = etnaviv_obj->base.size >> PAGE_SHIFT; - might_lock_read(¤t->mm->mmap_sem); + might_lock_read(¤t->mm->mmap_lock); if (userptr->mm != current->mm) return -EPERM; diff --git a/drivers/misc/okl4-link-shbuf.c b/drivers/misc/okl4-link-shbuf.c index 88f13e3cb7b9..62b6a8ffd461 100644 --- a/drivers/misc/okl4-link-shbuf.c +++ b/drivers/misc/okl4-link-shbuf.c @@ -259,7 +259,7 @@ static int link_shbuf_mmap(struct file *file, struct vm_area_struct *vma) pgprot_t prot; /* Our caller should have taken the MM semaphore. */ - if (WARN_ON(!rwsem_is_locked(&vma->vm_mm->mmap_sem))) + if (WARN_ON(!rwsem_is_locked(&vma->vm_mm->mmap_lock))) return -EINVAL; /* diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 037093bf1f0d..cae752a25969 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -235,7 +235,7 @@ static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx, pte_t *ptep, pte; bool ret = true; - VM_BUG_ON(!rwsem_is_locked(&mm->mmap_sem)); + VM_BUG_ON(!rwsem_is_locked(&mm->mmap_locked)); ptep = huge_pte_offset(mm, address, vma_mmu_pagesize(vma)); @@ -287,7 +287,7 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, pte_t *pte; bool ret = true; - VM_BUG_ON(!rwsem_is_locked(&mm->mmap_sem)); + VM_BUG_ON(!rwsem_is_locked(&mm->mmap_lock)); pgd = pgd_offset(mm, address); if (!pgd_present(*pgd)) @@ -401,7 +401,7 @@ vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason) * Coredumping runs without mmap_sem so we can only check that * the mmap_sem is held, if PF_DUMPCORE was not set. */ - WARN_ON_ONCE(!rwsem_is_locked(&mm->mmap_sem)); + WARN_ON_ONCE(!rwsem_is_locked(&mm->mmap_lock)); ctx = vmf->vma->vm_userfaultfd_ctx.ctx; if (!ctx) diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index fab389a48afb..7efc7a0083f8 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -410,7 +410,7 @@ struct mm_struct { spinlock_t page_table_lock; /* Protects page tables and some * counters */ - struct rw_semaphore mmap_sem; + struct rw_semaphore mmap_lock; struct list_head mmlist; /* List of maybe swapped mm's. These * are globally strung together off diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h index 424a43c06c0d..b9f933696c3f 100644 --- a/include/linux/mmap_lock.h +++ b/include/linux/mmap_lock.h @@ -3,58 +3,58 @@ static inline void mmap_init_lock(struct mm_struct *mm) { - init_rwsem(&mm->mmap_sem); + init_rwsem(&mm->mmap_lock); } static inline void mmap_write_lock(struct mm_struct *mm) { - down_write(&mm->mmap_sem); + down_write(&mm->mmap_lock); } static inline int mmap_write_lock_killable(struct mm_struct *mm) { - return down_write_killable(&mm->mmap_sem); + return down_write_killable(&mm->mmap_lock); } static inline bool mmap_write_trylock(struct mm_struct *mm) { - return down_write_trylock(&mm->mmap_sem) != 0; + return down_write_trylock(&mm->mmap_lock) != 0; } static inline void mmap_write_unlock(struct mm_struct *mm) { - up_write(&mm->mmap_sem); + up_write(&mm->mmap_lock); } static inline void mmap_write_downgrade(struct mm_struct *mm) { - downgrade_write(&mm->mmap_sem); + downgrade_write(&mm->mmap_lock); } static inline void mmap_read_lock(struct mm_struct *mm) { - down_read(&mm->mmap_sem); + down_read(&mm->mmap_lock); } static inline int mmap_read_lock_killable(struct mm_struct *mm) { - return down_read_killable(&mm->mmap_sem); + return down_read_killable(&mm->mmap_lock); } static inline bool mmap_read_trylock(struct mm_struct *mm) { - return down_read_trylock(&mm->mmap_sem) != 0; + return down_read_trylock(&mm->mmap_lock) != 0; } static inline void mmap_read_unlock(struct mm_struct *mm) { - up_read(&mm->mmap_sem); + up_read(&mm->mmap_lock); } static inline bool mmap_read_trylock_non_owner(struct mm_struct *mm) { - if (down_read_trylock(&mm->mmap_sem)) { - rwsem_release(&mm->mmap_sem.dep_map, _RET_IP_); + if (down_read_trylock(&mm->mmap_lock)) { + rwsem_release(&mm->mmap_lock.dep_map, _RET_IP_); return true; } return false; @@ -62,7 +62,7 @@ static inline bool mmap_read_trylock_non_owner(struct mm_struct *mm) static inline void mmap_read_unlock_non_owner(struct mm_struct *mm) { - up_read_non_owner(&mm->mmap_sem); + up_read_non_owner(&mm->mmap_lock); } #endif /* _LINUX_MMAP_LOCK_H */ diff --git a/kernel/fork.c b/kernel/fork.c index ca047565ae7c..858f0e0d26f4 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -449,7 +449,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, /* * Not linked in yet - no deadlock potential: */ - down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING); + down_write_nested(&mm->mmap_lock, SINGLE_DEPTH_NESTING); /* No ordering required: file already has been exposed. */ RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm)); diff --git a/mm/gup.c b/mm/gup.c index cf3ae8a09760..008c7308da88 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -1292,7 +1292,7 @@ long populate_vma_page_range(struct vm_area_struct *vma, VM_BUG_ON(end & ~PAGE_MASK); VM_BUG_ON_VMA(start < vma->vm_start, vma); VM_BUG_ON_VMA(end > vma->vm_end, vma); - VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_sem), mm); + VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm); gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK; if (vma->vm_flags & VM_LOCKONFAULT) diff --git a/mm/init-mm.c b/mm/init-mm.c index b50f61e57708..f524690f00f2 100644 --- a/mm/init-mm.c +++ b/mm/init-mm.c @@ -30,7 +30,7 @@ struct mm_struct init_mm = { .pgd = swapper_pg_dir, .mm_users = ATOMIC_INIT(2), .mm_count = ATOMIC_INIT(1), - .mmap_sem = __RWSEM_INITIALIZER(init_mm.mmap_sem), + .mmap_lock = __RWSEM_INITIALIZER(init_mm.mmap_lock), .page_table_lock = __SPIN_LOCK_UNLOCKED(init_mm.page_table_lock), .arg_lock = __SPIN_LOCK_UNLOCKED(init_mm.arg_lock), .mmlist = LIST_HEAD_INIT(init_mm.mmlist), diff --git a/mm/khugepaged.c b/mm/khugepaged.c index bb4e727901d9..23fee6e866f8 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1316,7 +1316,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) * mmap_sem in read mode. */ if (vma->anon_vma) { - up_write(&mm->mmap_sem); + up_write(&mm->mmap_lock); continue; } mmu_notifier_invalidate_range_start(mm, addr, diff --git a/mm/memory.c b/mm/memory.c index a8cef6c9dc5e..f3103265c849 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1272,7 +1272,7 @@ static inline unsigned long zap_pud_range(struct mmu_gather *tlb, next = pud_addr_end(addr, end); if (pud_trans_huge(*pud) || pud_devmap(*pud)) { if (next - addr != HPAGE_PUD_SIZE) { - VM_BUG_ON_VMA(!rwsem_is_locked(&tlb->mm->mmap_sem), vma); + VM_BUG_ON_VMA(!rwsem_is_locked(&tlb->mm->mmap_lock), vma); split_huge_pud(vma, pud, addr); } else if (zap_huge_pud(tlb, vma, pud, addr)) goto next; @@ -4628,7 +4628,7 @@ void __might_fault(const char *file, int line) __might_sleep(file, line, 0); #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) if (current->mm) - might_lock_read(¤t->mm->mmap_sem); + might_lock_read(¤t->mm->mmap_lock); #endif } EXPORT_SYMBOL(__might_fault); diff --git a/mm/mmap.c b/mm/mmap.c index 6fbeca7262ef..c0f9e4a125aa 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -3452,7 +3452,7 @@ static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma) * The LSB of head.next can't change from under us * because we hold the mm_all_locks_mutex. */ - down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_sem); + down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_lock); /* * We can safely modify head.next after taking the * anon_vma->root->rwsem. If some other vma in this mm shares @@ -3482,7 +3482,7 @@ static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping) */ if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags)) BUG(); - down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_sem); + down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_lock); } } diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c index 3f9cfed78be6..a4b35cf0a144 100644 --- a/mm/mmu_notifier.c +++ b/mm/mmu_notifier.c @@ -255,7 +255,7 @@ bool mm_has_blockable_invalidate_notifiers(struct mm_struct *mm) int id; bool ret = false; - WARN_ON_ONCE(!rwsem_is_locked(&mm->mmap_sem)); + WARN_ON_ONCE(!rwsem_is_locked(&mm->mmap_lock)); if (!mm_has_notifiers(mm)) return ret; diff --git a/mm/pagewalk.c b/mm/pagewalk.c index 3c0930d94a29..329b9b98fab0 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -303,7 +303,7 @@ int walk_page_range(unsigned long start, unsigned long end, if (!walk->mm) return -EINVAL; - VM_BUG_ON_MM(!rwsem_is_locked(&walk->mm->mmap_sem), walk->mm); + VM_BUG_ON_MM(!rwsem_is_locked(&walk->mm->mmap_lock), walk->mm); vma = find_vma(walk->mm, start); do { @@ -346,7 +346,7 @@ int walk_page_vma(struct vm_area_struct *vma, struct mm_walk *walk) if (!walk->mm) return -EINVAL; - VM_BUG_ON(!rwsem_is_locked(&walk->mm->mmap_sem)); + VM_BUG_ON(!rwsem_is_locked(&walk->mm->mmap_lock)); VM_BUG_ON(!vma); walk->vma = vma; err = walk_page_test(vma->vm_start, vma->vm_end, walk);