mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-10-30 08:02:30 +00:00
2b5067a814
The goal of these tracepoints is to be able to debug lock contention issues. This lock is acquired on most (all?) mmap / munmap / page fault operations, so a multi-threaded process which does a lot of these can experience significant contention. We trace just before we start acquisition, when the acquisition returns (whether it succeeded or not), and when the lock is released (or downgraded). The events are broken out by lock type (read / write). The events are also broken out by memcg path. For container-based workloads, users often think of several processes in a memcg as a single logical "task", so collecting statistics at this level is useful. The end goal is to get latency information. This isn't directly included in the trace events. Instead, users are expected to compute the time between "start locking" and "acquire returned", using e.g. synthetic events or BPF. The benefit we get from this is simpler code. Because we use tracepoint_enabled() to decide whether or not to trace, this patch has effectively no overhead unless tracepoints are enabled at runtime. If tracepoints are enabled, there is a performance impact, but how much depends on exactly what e.g. the BPF program does. [axelrasmussen@google.com: fix use-after-free race and css ref leak in tracepoints] Link: https://lkml.kernel.org/r/20201130233504.3725241-1-axelrasmussen@google.com [axelrasmussen@google.com: v3] Link: https://lkml.kernel.org/r/20201207213358.573750-1-axelrasmussen@google.com [rostedt@goodmis.org: in-depth examples of tracepoint_enabled() usage, and per-cpu-per-context buffer design] Link: https://lkml.kernel.org/r/20201105211739.568279-2-axelrasmussen@google.com Signed-off-by: Axel Rasmussen <axelrasmussen@google.com> Acked-by: Vlastimil Babka <vbabka@suse.cz> Cc: Steven Rostedt <rostedt@goodmis.org> Cc: Ingo Molnar <mingo@redhat.com> Cc: Michel Lespinasse <walken@google.com> Cc: Daniel Jordan <daniel.m.jordan@oracle.com> Cc: Jann Horn <jannh@google.com> Cc: Chinwen Chang <chinwen.chang@mediatek.com> Cc: Davidlohr Bueso <dbueso@suse.de> Cc: David Rientjes <rientjes@google.com> Cc: Laurent Dufour <ldufour@linux.ibm.com> Cc: Yafang Shao <laoar.shao@gmail.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
179 lines
4.4 KiB
C
179 lines
4.4 KiB
C
#ifndef _LINUX_MMAP_LOCK_H
|
|
#define _LINUX_MMAP_LOCK_H
|
|
|
|
#include <linux/lockdep.h>
|
|
#include <linux/mm_types.h>
|
|
#include <linux/mmdebug.h>
|
|
#include <linux/rwsem.h>
|
|
#include <linux/tracepoint-defs.h>
|
|
#include <linux/types.h>
|
|
|
|
#define MMAP_LOCK_INITIALIZER(name) \
|
|
.mmap_lock = __RWSEM_INITIALIZER((name).mmap_lock),
|
|
|
|
DECLARE_TRACEPOINT(mmap_lock_start_locking);
|
|
DECLARE_TRACEPOINT(mmap_lock_acquire_returned);
|
|
DECLARE_TRACEPOINT(mmap_lock_released);
|
|
|
|
#ifdef CONFIG_TRACING
|
|
|
|
void __mmap_lock_do_trace_start_locking(struct mm_struct *mm, bool write);
|
|
void __mmap_lock_do_trace_acquire_returned(struct mm_struct *mm, bool write,
|
|
bool success);
|
|
void __mmap_lock_do_trace_released(struct mm_struct *mm, bool write);
|
|
|
|
static inline void __mmap_lock_trace_start_locking(struct mm_struct *mm,
|
|
bool write)
|
|
{
|
|
if (tracepoint_enabled(mmap_lock_start_locking))
|
|
__mmap_lock_do_trace_start_locking(mm, write);
|
|
}
|
|
|
|
static inline void __mmap_lock_trace_acquire_returned(struct mm_struct *mm,
|
|
bool write, bool success)
|
|
{
|
|
if (tracepoint_enabled(mmap_lock_acquire_returned))
|
|
__mmap_lock_do_trace_acquire_returned(mm, write, success);
|
|
}
|
|
|
|
static inline void __mmap_lock_trace_released(struct mm_struct *mm, bool write)
|
|
{
|
|
if (tracepoint_enabled(mmap_lock_released))
|
|
__mmap_lock_do_trace_released(mm, write);
|
|
}
|
|
|
|
#else /* !CONFIG_TRACING */
|
|
|
|
static inline void __mmap_lock_trace_start_locking(struct mm_struct *mm,
|
|
bool write)
|
|
{
|
|
}
|
|
|
|
static inline void __mmap_lock_trace_acquire_returned(struct mm_struct *mm,
|
|
bool write, bool success)
|
|
{
|
|
}
|
|
|
|
static inline void __mmap_lock_trace_released(struct mm_struct *mm, bool write)
|
|
{
|
|
}
|
|
|
|
#endif /* CONFIG_TRACING */
|
|
|
|
static inline void mmap_init_lock(struct mm_struct *mm)
|
|
{
|
|
init_rwsem(&mm->mmap_lock);
|
|
}
|
|
|
|
static inline void mmap_write_lock(struct mm_struct *mm)
|
|
{
|
|
__mmap_lock_trace_start_locking(mm, true);
|
|
down_write(&mm->mmap_lock);
|
|
__mmap_lock_trace_acquire_returned(mm, true, true);
|
|
}
|
|
|
|
static inline void mmap_write_lock_nested(struct mm_struct *mm, int subclass)
|
|
{
|
|
__mmap_lock_trace_start_locking(mm, true);
|
|
down_write_nested(&mm->mmap_lock, subclass);
|
|
__mmap_lock_trace_acquire_returned(mm, true, true);
|
|
}
|
|
|
|
static inline int mmap_write_lock_killable(struct mm_struct *mm)
|
|
{
|
|
int ret;
|
|
|
|
__mmap_lock_trace_start_locking(mm, true);
|
|
ret = down_write_killable(&mm->mmap_lock);
|
|
__mmap_lock_trace_acquire_returned(mm, true, ret == 0);
|
|
return ret;
|
|
}
|
|
|
|
static inline bool mmap_write_trylock(struct mm_struct *mm)
|
|
{
|
|
bool ret;
|
|
|
|
__mmap_lock_trace_start_locking(mm, true);
|
|
ret = down_write_trylock(&mm->mmap_lock) != 0;
|
|
__mmap_lock_trace_acquire_returned(mm, true, ret);
|
|
return ret;
|
|
}
|
|
|
|
static inline void mmap_write_unlock(struct mm_struct *mm)
|
|
{
|
|
up_write(&mm->mmap_lock);
|
|
__mmap_lock_trace_released(mm, true);
|
|
}
|
|
|
|
static inline void mmap_write_downgrade(struct mm_struct *mm)
|
|
{
|
|
downgrade_write(&mm->mmap_lock);
|
|
__mmap_lock_trace_acquire_returned(mm, false, true);
|
|
}
|
|
|
|
static inline void mmap_read_lock(struct mm_struct *mm)
|
|
{
|
|
__mmap_lock_trace_start_locking(mm, false);
|
|
down_read(&mm->mmap_lock);
|
|
__mmap_lock_trace_acquire_returned(mm, false, true);
|
|
}
|
|
|
|
static inline int mmap_read_lock_killable(struct mm_struct *mm)
|
|
{
|
|
int ret;
|
|
|
|
__mmap_lock_trace_start_locking(mm, false);
|
|
ret = down_read_killable(&mm->mmap_lock);
|
|
__mmap_lock_trace_acquire_returned(mm, false, ret == 0);
|
|
return ret;
|
|
}
|
|
|
|
static inline bool mmap_read_trylock(struct mm_struct *mm)
|
|
{
|
|
bool ret;
|
|
|
|
__mmap_lock_trace_start_locking(mm, false);
|
|
ret = down_read_trylock(&mm->mmap_lock) != 0;
|
|
__mmap_lock_trace_acquire_returned(mm, false, ret);
|
|
return ret;
|
|
}
|
|
|
|
static inline void mmap_read_unlock(struct mm_struct *mm)
|
|
{
|
|
up_read(&mm->mmap_lock);
|
|
__mmap_lock_trace_released(mm, false);
|
|
}
|
|
|
|
static inline bool mmap_read_trylock_non_owner(struct mm_struct *mm)
|
|
{
|
|
if (mmap_read_trylock(mm)) {
|
|
rwsem_release(&mm->mmap_lock.dep_map, _RET_IP_);
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
static inline void mmap_read_unlock_non_owner(struct mm_struct *mm)
|
|
{
|
|
up_read_non_owner(&mm->mmap_lock);
|
|
__mmap_lock_trace_released(mm, false);
|
|
}
|
|
|
|
static inline void mmap_assert_locked(struct mm_struct *mm)
|
|
{
|
|
lockdep_assert_held(&mm->mmap_lock);
|
|
VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm);
|
|
}
|
|
|
|
static inline void mmap_assert_write_locked(struct mm_struct *mm)
|
|
{
|
|
lockdep_assert_held_write(&mm->mmap_lock);
|
|
VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm);
|
|
}
|
|
|
|
static inline int mmap_lock_is_contended(struct mm_struct *mm)
|
|
{
|
|
return rwsem_is_contended(&mm->mmap_lock);
|
|
}
|
|
|
|
#endif /* _LINUX_MMAP_LOCK_H */
|