mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-04 18:09:57 +00:00
802f1d522d
When checking a memory cgroup related performance regression [1], from the perf c2c profiling data, we found high false sharing for accessing 'usage' and 'parent'. On 64 bit system, the 'usage' and 'parent' are close to each other, and easy to be in one cacheline (for cacheline size == 64+ B). 'usage' is usally written, while 'parent' is usually read as the cgroup's hierarchical counting nature. So move the 'parent' to the end of the structure to make sure they are in different cache lines. Following are some performance data with the patch, against v5.11-rc1. [ In the data, A means a platform with 2 sockets 48C/96T, B is a platform of 4 sockests 72C/144T, and if a %stddev will be shown bigger than 2%, P100/P50 means number of test tasks equals to 100%/50% of nr_cpu] will-it-scale/malloc1 --------------------- v5.11-rc1 v5.11-rc1+patch A-P100 15782 ± 2% -0.1% 15765 ± 3% will-it-scale.per_process_ops A-P50 21511 +8.9% 23432 will-it-scale.per_process_ops B-P100 9155 +2.2% 9357 will-it-scale.per_process_ops B-P50 10967 +7.1% 11751 ± 2% will-it-scale.per_process_ops will-it-scale/pagefault2 ------------------------ v5.11-rc1 v5.11-rc1+patch A-P100 79028 +3.0% 81411 will-it-scale.per_process_ops A-P50 183960 ± 2% +4.4% 192078 ± 2% will-it-scale.per_process_ops B-P100 85966 +9.9% 94467 ± 3% will-it-scale.per_process_ops B-P50 198195 +9.8% 217526 will-it-scale.per_process_ops fio (4k/1M is block size) ------------------------- v5.11-rc1 v5.11-rc1+patch A-P50-r-4k 16881 ± 2% +1.2% 17081 ± 2% fio.read_bw_MBps A-P50-w-4k 3931 +4.5% 4111 ± 2% fio.write_bw_MBps A-P50-r-1M 15178 -0.2% 15154 fio.read_bw_MBps A-P50-w-1M 3924 +0.1% 3929 fio.write_bw_MBps [1].https://lore.kernel.org/lkml/20201102091543.GM31092@shao2-debian/ Link: https://lkml.kernel.org/r/1611040814-33449-1-git-send-email-feng.tang@intel.com Signed-off-by: Feng Tang <feng.tang@intel.com> Reviewed-by: Roman Gushchin <guro@fb.com> Reviewed-by: Shakeel Butt <shakeelb@google.com> Acked-by: Johannes Weiner <hannes@cmpxchg.org> Acked-by: Michal Hocko <mhocko@suse.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
82 lines
2.3 KiB
C
82 lines
2.3 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _LINUX_PAGE_COUNTER_H
|
|
#define _LINUX_PAGE_COUNTER_H
|
|
|
|
#include <linux/atomic.h>
|
|
#include <linux/kernel.h>
|
|
#include <asm/page.h>
|
|
|
|
struct page_counter {
|
|
atomic_long_t usage;
|
|
unsigned long min;
|
|
unsigned long low;
|
|
unsigned long high;
|
|
unsigned long max;
|
|
|
|
/* effective memory.min and memory.min usage tracking */
|
|
unsigned long emin;
|
|
atomic_long_t min_usage;
|
|
atomic_long_t children_min_usage;
|
|
|
|
/* effective memory.low and memory.low usage tracking */
|
|
unsigned long elow;
|
|
atomic_long_t low_usage;
|
|
atomic_long_t children_low_usage;
|
|
|
|
/* legacy */
|
|
unsigned long watermark;
|
|
unsigned long failcnt;
|
|
|
|
/*
|
|
* 'parent' is placed here to be far from 'usage' to reduce
|
|
* cache false sharing, as 'usage' is written mostly while
|
|
* parent is frequently read for cgroup's hierarchical
|
|
* counting nature.
|
|
*/
|
|
struct page_counter *parent;
|
|
};
|
|
|
|
#if BITS_PER_LONG == 32
|
|
#define PAGE_COUNTER_MAX LONG_MAX
|
|
#else
|
|
#define PAGE_COUNTER_MAX (LONG_MAX / PAGE_SIZE)
|
|
#endif
|
|
|
|
static inline void page_counter_init(struct page_counter *counter,
|
|
struct page_counter *parent)
|
|
{
|
|
atomic_long_set(&counter->usage, 0);
|
|
counter->max = PAGE_COUNTER_MAX;
|
|
counter->parent = parent;
|
|
}
|
|
|
|
static inline unsigned long page_counter_read(struct page_counter *counter)
|
|
{
|
|
return atomic_long_read(&counter->usage);
|
|
}
|
|
|
|
void page_counter_cancel(struct page_counter *counter, unsigned long nr_pages);
|
|
void page_counter_charge(struct page_counter *counter, unsigned long nr_pages);
|
|
bool page_counter_try_charge(struct page_counter *counter,
|
|
unsigned long nr_pages,
|
|
struct page_counter **fail);
|
|
void page_counter_uncharge(struct page_counter *counter, unsigned long nr_pages);
|
|
void page_counter_set_min(struct page_counter *counter, unsigned long nr_pages);
|
|
void page_counter_set_low(struct page_counter *counter, unsigned long nr_pages);
|
|
|
|
static inline void page_counter_set_high(struct page_counter *counter,
|
|
unsigned long nr_pages)
|
|
{
|
|
WRITE_ONCE(counter->high, nr_pages);
|
|
}
|
|
|
|
int page_counter_set_max(struct page_counter *counter, unsigned long nr_pages);
|
|
int page_counter_memparse(const char *buf, const char *max,
|
|
unsigned long *nr_pages);
|
|
|
|
static inline void page_counter_reset_watermark(struct page_counter *counter)
|
|
{
|
|
counter->watermark = page_counter_read(counter);
|
|
}
|
|
|
|
#endif /* _LINUX_PAGE_COUNTER_H */
|