mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-12 05:48:40 +00:00
![Yang Jihong](/assets/img/avatar_default.png)
Implements workqueue trace bpf function. Test cases: # perf kwork -k workqueue lat -b Starting trace, Hit <Ctrl+C> to stop and report ^C Kwork Name | Cpu | Avg delay | Count | Max delay | Max delay start | Max delay end | -------------------------------------------------------------------------------------------------------------------------------- (w)addrconf_verify_work | 0002 | 5.856 ms | 1 | 5.856 ms | 111994.634313 s | 111994.640169 s | (w)vmstat_update | 0001 | 1.247 ms | 1 | 1.247 ms | 111996.462651 s | 111996.463899 s | (w)neigh_periodic_work | 0001 | 1.183 ms | 1 | 1.183 ms | 111996.462789 s | 111996.463973 s | (w)neigh_managed_work | 0001 | 0.989 ms | 2 | 1.635 ms | 111996.462820 s | 111996.464455 s | (w)wb_workfn | 0000 | 0.667 ms | 1 | 0.667 ms | 111996.384273 s | 111996.384940 s | (w)bpf_prog_free_deferred | 0001 | 0.495 ms | 1 | 0.495 ms | 111986.314201 s | 111986.314696 s | (w)mix_interrupt_randomness | 0002 | 0.421 ms | 6 | 0.749 ms | 111995.927750 s | 111995.928499 s | (w)vmstat_shepherd | 0000 | 0.374 ms | 2 | 0.385 ms | 111991.265242 s | 111991.265627 s | (w)e1000_watchdog | 0002 | 0.356 ms | 5 | 0.390 ms | 111994.528380 s | 111994.528770 s | (w)vmstat_update | 0000 | 0.231 ms | 2 | 0.365 ms | 111996.384407 s | 111996.384772 s | (w)flush_to_ldisc | 0006 | 0.165 ms | 1 | 0.165 ms | 111995.930606 s | 111995.930771 s | (w)flush_to_ldisc | 0000 | 0.094 ms | 2 | 0.095 ms | 111996.460453 s | 111996.460548 s | -------------------------------------------------------------------------------------------------------------------------------- # perf kwork -k workqueue rep -b Starting trace, Hit <Ctrl+C> to stop and report ^C Kwork Name | Cpu | Total Runtime | Count | Max runtime | Max runtime start | Max runtime end | -------------------------------------------------------------------------------------------------------------------------------- (w)e1000_watchdog | 0002 | 0.627 ms | 2 | 0.324 ms | 112002.720665 s | 112002.720989 s | (w)flush_to_ldisc | 0007 | 0.598 ms | 2 | 0.534 ms | 112000.875226 s | 112000.875761 s | (w)wq_barrier_func | 0007 | 0.492 ms | 1 | 0.492 ms | 112000.876981 s | 112000.877473 s | (w)flush_to_ldisc | 0007 | 0.281 ms | 1 | 0.281 ms | 112005.826882 s | 112005.827163 s | (w)mix_interrupt_randomness | 0002 | 0.229 ms | 3 | 0.102 ms | 112005.825671 s | 112005.825774 s | (w)vmstat_shepherd | 0000 | 0.202 ms | 1 | 0.202 ms | 112001.504511 s | 112001.504713 s | (w)bpf_prog_free_deferred | 0001 | 0.181 ms | 1 | 0.181 ms | 112000.883251 s | 112000.883432 s | (w)wb_workfn | 0007 | 0.130 ms | 1 | 0.130 ms | 112001.505195 s | 112001.505325 s | (w)vmstat_update | 0000 | 0.053 ms | 1 | 0.053 ms | 112001.504763 s | 112001.504815 s | -------------------------------------------------------------------------------------------------------------------------------- Signed-off-by: Yang Jihong <yangjihong1@huawei.com> Tested-by: Arnaldo Carvalho de Melo <acme@redhat.com> Cc: Alexander Shishkin <alexander.shishkin@linux.intel.com> Cc: Ingo Molnar <mingo@redhat.com> Cc: Jiri Olsa <jolsa@kernel.org> Cc: Mark Rutland <mark.rutland@arm.com> Cc: Namhyung Kim <namhyung@kernel.org> Cc: Paul Clarke <pc@us.ibm.com> Cc: Peter Zijlstra <peterz@infradead.org> Link: https://lore.kernel.org/r/20220709015033.38326-18-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
383 lines
8.8 KiB
C
383 lines
8.8 KiB
C
// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
|
|
// Copyright (c) 2022, Huawei
|
|
|
|
#include "vmlinux.h"
|
|
#include <bpf/bpf_helpers.h>
|
|
#include <bpf/bpf_tracing.h>
|
|
|
|
#define KWORK_COUNT 100
|
|
#define MAX_KWORKNAME 128
|
|
|
|
/*
|
|
* This should be in sync with "util/kwork.h"
|
|
*/
|
|
enum kwork_class_type {
|
|
KWORK_CLASS_IRQ,
|
|
KWORK_CLASS_SOFTIRQ,
|
|
KWORK_CLASS_WORKQUEUE,
|
|
KWORK_CLASS_MAX,
|
|
};
|
|
|
|
struct work_key {
|
|
__u32 type;
|
|
__u32 cpu;
|
|
__u64 id;
|
|
};
|
|
|
|
struct report_data {
|
|
__u64 nr;
|
|
__u64 total_time;
|
|
__u64 max_time;
|
|
__u64 max_time_start;
|
|
__u64 max_time_end;
|
|
};
|
|
|
|
struct {
|
|
__uint(type, BPF_MAP_TYPE_HASH);
|
|
__uint(key_size, sizeof(struct work_key));
|
|
__uint(value_size, MAX_KWORKNAME);
|
|
__uint(max_entries, KWORK_COUNT);
|
|
} perf_kwork_names SEC(".maps");
|
|
|
|
struct {
|
|
__uint(type, BPF_MAP_TYPE_HASH);
|
|
__uint(key_size, sizeof(struct work_key));
|
|
__uint(value_size, sizeof(__u64));
|
|
__uint(max_entries, KWORK_COUNT);
|
|
} perf_kwork_time SEC(".maps");
|
|
|
|
struct {
|
|
__uint(type, BPF_MAP_TYPE_HASH);
|
|
__uint(key_size, sizeof(struct work_key));
|
|
__uint(value_size, sizeof(struct report_data));
|
|
__uint(max_entries, KWORK_COUNT);
|
|
} perf_kwork_report SEC(".maps");
|
|
|
|
struct {
|
|
__uint(type, BPF_MAP_TYPE_HASH);
|
|
__uint(key_size, sizeof(__u32));
|
|
__uint(value_size, sizeof(__u8));
|
|
__uint(max_entries, 1);
|
|
} perf_kwork_cpu_filter SEC(".maps");
|
|
|
|
struct {
|
|
__uint(type, BPF_MAP_TYPE_ARRAY);
|
|
__uint(key_size, sizeof(__u32));
|
|
__uint(value_size, MAX_KWORKNAME);
|
|
__uint(max_entries, 1);
|
|
} perf_kwork_name_filter SEC(".maps");
|
|
|
|
int enabled = 0;
|
|
int has_cpu_filter = 0;
|
|
int has_name_filter = 0;
|
|
|
|
static __always_inline int local_strncmp(const char *s1,
|
|
unsigned int sz, const char *s2)
|
|
{
|
|
int ret = 0;
|
|
unsigned int i;
|
|
|
|
for (i = 0; i < sz; i++) {
|
|
ret = (unsigned char)s1[i] - (unsigned char)s2[i];
|
|
if (ret || !s1[i] || !s2[i])
|
|
break;
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
static __always_inline int trace_event_match(struct work_key *key, char *name)
|
|
{
|
|
__u8 *cpu_val;
|
|
char *name_val;
|
|
__u32 zero = 0;
|
|
__u32 cpu = bpf_get_smp_processor_id();
|
|
|
|
if (!enabled)
|
|
return 0;
|
|
|
|
if (has_cpu_filter) {
|
|
cpu_val = bpf_map_lookup_elem(&perf_kwork_cpu_filter, &cpu);
|
|
if (!cpu_val)
|
|
return 0;
|
|
}
|
|
|
|
if (has_name_filter && (name != NULL)) {
|
|
name_val = bpf_map_lookup_elem(&perf_kwork_name_filter, &zero);
|
|
if (name_val &&
|
|
(local_strncmp(name_val, MAX_KWORKNAME, name) != 0)) {
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
return 1;
|
|
}
|
|
|
|
static __always_inline void do_update_time(void *map, struct work_key *key,
|
|
__u64 time_start, __u64 time_end)
|
|
{
|
|
struct report_data zero, *data;
|
|
__s64 delta = time_end - time_start;
|
|
|
|
if (delta < 0)
|
|
return;
|
|
|
|
data = bpf_map_lookup_elem(map, key);
|
|
if (!data) {
|
|
__builtin_memset(&zero, 0, sizeof(zero));
|
|
bpf_map_update_elem(map, key, &zero, BPF_NOEXIST);
|
|
data = bpf_map_lookup_elem(map, key);
|
|
if (!data)
|
|
return;
|
|
}
|
|
|
|
if ((delta > data->max_time) ||
|
|
(data->max_time == 0)) {
|
|
data->max_time = delta;
|
|
data->max_time_start = time_start;
|
|
data->max_time_end = time_end;
|
|
}
|
|
|
|
data->total_time += delta;
|
|
data->nr++;
|
|
}
|
|
|
|
static __always_inline void do_update_timestart(void *map, struct work_key *key)
|
|
{
|
|
__u64 ts = bpf_ktime_get_ns();
|
|
|
|
bpf_map_update_elem(map, key, &ts, BPF_ANY);
|
|
}
|
|
|
|
static __always_inline void do_update_timeend(void *report_map, void *time_map,
|
|
struct work_key *key)
|
|
{
|
|
__u64 *time = bpf_map_lookup_elem(time_map, key);
|
|
|
|
if (time) {
|
|
bpf_map_delete_elem(time_map, key);
|
|
do_update_time(report_map, key, *time, bpf_ktime_get_ns());
|
|
}
|
|
}
|
|
|
|
static __always_inline void do_update_name(void *map,
|
|
struct work_key *key, char *name)
|
|
{
|
|
if (!bpf_map_lookup_elem(map, key))
|
|
bpf_map_update_elem(map, key, name, BPF_ANY);
|
|
}
|
|
|
|
static __always_inline int update_timestart(void *map, struct work_key *key)
|
|
{
|
|
if (!trace_event_match(key, NULL))
|
|
return 0;
|
|
|
|
do_update_timestart(map, key);
|
|
return 0;
|
|
}
|
|
|
|
static __always_inline int update_timestart_and_name(void *time_map,
|
|
void *names_map,
|
|
struct work_key *key,
|
|
char *name)
|
|
{
|
|
if (!trace_event_match(key, name))
|
|
return 0;
|
|
|
|
do_update_timestart(time_map, key);
|
|
do_update_name(names_map, key, name);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static __always_inline int update_timeend(void *report_map,
|
|
void *time_map, struct work_key *key)
|
|
{
|
|
if (!trace_event_match(key, NULL))
|
|
return 0;
|
|
|
|
do_update_timeend(report_map, time_map, key);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static __always_inline int update_timeend_and_name(void *report_map,
|
|
void *time_map,
|
|
void *names_map,
|
|
struct work_key *key,
|
|
char *name)
|
|
{
|
|
if (!trace_event_match(key, name))
|
|
return 0;
|
|
|
|
do_update_timeend(report_map, time_map, key);
|
|
do_update_name(names_map, key, name);
|
|
|
|
return 0;
|
|
}
|
|
|
|
SEC("tracepoint/irq/irq_handler_entry")
|
|
int report_irq_handler_entry(struct trace_event_raw_irq_handler_entry *ctx)
|
|
{
|
|
char name[MAX_KWORKNAME];
|
|
struct work_key key = {
|
|
.type = KWORK_CLASS_IRQ,
|
|
.cpu = bpf_get_smp_processor_id(),
|
|
.id = (__u64)ctx->irq,
|
|
};
|
|
void *name_addr = (void *)ctx + (ctx->__data_loc_name & 0xffff);
|
|
|
|
bpf_probe_read_kernel_str(name, sizeof(name), name_addr);
|
|
|
|
return update_timestart_and_name(&perf_kwork_time,
|
|
&perf_kwork_names, &key, name);
|
|
}
|
|
|
|
SEC("tracepoint/irq/irq_handler_exit")
|
|
int report_irq_handler_exit(struct trace_event_raw_irq_handler_exit *ctx)
|
|
{
|
|
struct work_key key = {
|
|
.type = KWORK_CLASS_IRQ,
|
|
.cpu = bpf_get_smp_processor_id(),
|
|
.id = (__u64)ctx->irq,
|
|
};
|
|
|
|
return update_timeend(&perf_kwork_report, &perf_kwork_time, &key);
|
|
}
|
|
|
|
static char softirq_name_list[NR_SOFTIRQS][MAX_KWORKNAME] = {
|
|
{ "HI" },
|
|
{ "TIMER" },
|
|
{ "NET_TX" },
|
|
{ "NET_RX" },
|
|
{ "BLOCK" },
|
|
{ "IRQ_POLL" },
|
|
{ "TASKLET" },
|
|
{ "SCHED" },
|
|
{ "HRTIMER" },
|
|
{ "RCU" },
|
|
};
|
|
|
|
SEC("tracepoint/irq/softirq_entry")
|
|
int report_softirq_entry(struct trace_event_raw_softirq *ctx)
|
|
{
|
|
unsigned int vec = ctx->vec;
|
|
struct work_key key = {
|
|
.type = KWORK_CLASS_SOFTIRQ,
|
|
.cpu = bpf_get_smp_processor_id(),
|
|
.id = (__u64)vec,
|
|
};
|
|
|
|
if (vec < NR_SOFTIRQS) {
|
|
return update_timestart_and_name(&perf_kwork_time,
|
|
&perf_kwork_names, &key,
|
|
softirq_name_list[vec]);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
SEC("tracepoint/irq/softirq_exit")
|
|
int report_softirq_exit(struct trace_event_raw_softirq *ctx)
|
|
{
|
|
struct work_key key = {
|
|
.type = KWORK_CLASS_SOFTIRQ,
|
|
.cpu = bpf_get_smp_processor_id(),
|
|
.id = (__u64)ctx->vec,
|
|
};
|
|
|
|
return update_timeend(&perf_kwork_report, &perf_kwork_time, &key);
|
|
}
|
|
|
|
SEC("tracepoint/irq/softirq_raise")
|
|
int latency_softirq_raise(struct trace_event_raw_softirq *ctx)
|
|
{
|
|
unsigned int vec = ctx->vec;
|
|
struct work_key key = {
|
|
.type = KWORK_CLASS_SOFTIRQ,
|
|
.cpu = bpf_get_smp_processor_id(),
|
|
.id = (__u64)vec,
|
|
};
|
|
|
|
if (vec < NR_SOFTIRQS) {
|
|
return update_timestart_and_name(&perf_kwork_time,
|
|
&perf_kwork_names, &key,
|
|
softirq_name_list[vec]);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
SEC("tracepoint/irq/softirq_entry")
|
|
int latency_softirq_entry(struct trace_event_raw_softirq *ctx)
|
|
{
|
|
struct work_key key = {
|
|
.type = KWORK_CLASS_SOFTIRQ,
|
|
.cpu = bpf_get_smp_processor_id(),
|
|
.id = (__u64)ctx->vec,
|
|
};
|
|
|
|
return update_timeend(&perf_kwork_report, &perf_kwork_time, &key);
|
|
}
|
|
|
|
SEC("tracepoint/workqueue/workqueue_execute_start")
|
|
int report_workqueue_execute_start(struct trace_event_raw_workqueue_execute_start *ctx)
|
|
{
|
|
struct work_key key = {
|
|
.type = KWORK_CLASS_WORKQUEUE,
|
|
.cpu = bpf_get_smp_processor_id(),
|
|
.id = (__u64)ctx->work,
|
|
};
|
|
|
|
return update_timestart(&perf_kwork_time, &key);
|
|
}
|
|
|
|
SEC("tracepoint/workqueue/workqueue_execute_end")
|
|
int report_workqueue_execute_end(struct trace_event_raw_workqueue_execute_end *ctx)
|
|
{
|
|
char name[MAX_KWORKNAME];
|
|
struct work_key key = {
|
|
.type = KWORK_CLASS_WORKQUEUE,
|
|
.cpu = bpf_get_smp_processor_id(),
|
|
.id = (__u64)ctx->work,
|
|
};
|
|
unsigned long long func_addr = (unsigned long long)ctx->function;
|
|
|
|
__builtin_memset(name, 0, sizeof(name));
|
|
bpf_snprintf(name, sizeof(name), "%ps", &func_addr, sizeof(func_addr));
|
|
|
|
return update_timeend_and_name(&perf_kwork_report, &perf_kwork_time,
|
|
&perf_kwork_names, &key, name);
|
|
}
|
|
|
|
SEC("tracepoint/workqueue/workqueue_activate_work")
|
|
int latency_workqueue_activate_work(struct trace_event_raw_workqueue_activate_work *ctx)
|
|
{
|
|
struct work_key key = {
|
|
.type = KWORK_CLASS_WORKQUEUE,
|
|
.cpu = bpf_get_smp_processor_id(),
|
|
.id = (__u64)ctx->work,
|
|
};
|
|
|
|
return update_timestart(&perf_kwork_time, &key);
|
|
}
|
|
|
|
SEC("tracepoint/workqueue/workqueue_execute_start")
|
|
int latency_workqueue_execute_start(struct trace_event_raw_workqueue_execute_start *ctx)
|
|
{
|
|
char name[MAX_KWORKNAME];
|
|
struct work_key key = {
|
|
.type = KWORK_CLASS_WORKQUEUE,
|
|
.cpu = bpf_get_smp_processor_id(),
|
|
.id = (__u64)ctx->work,
|
|
};
|
|
unsigned long long func_addr = (unsigned long long)ctx->function;
|
|
|
|
__builtin_memset(name, 0, sizeof(name));
|
|
bpf_snprintf(name, sizeof(name), "%ps", &func_addr, sizeof(func_addr));
|
|
|
|
return update_timeend_and_name(&perf_kwork_report, &perf_kwork_time,
|
|
&perf_kwork_names, &key, name);
|
|
}
|
|
|
|
char LICENSE[] SEC("license") = "Dual BSD/GPL";
|