linux-stable/tools/testing/selftests/bpf/test_progs.h
Stanislav Fomichev 9cacf81f81 bpf: Remove extra lock_sock for TCP_ZEROCOPY_RECEIVE
Add custom implementation of getsockopt hook for TCP_ZEROCOPY_RECEIVE.
We skip generic hooks for TCP_ZEROCOPY_RECEIVE and have a custom
call in do_tcp_getsockopt using the on-stack data. This removes
3% overhead for locking/unlocking the socket.

Without this patch:
     3.38%     0.07%  tcp_mmap  [kernel.kallsyms]  [k] __cgroup_bpf_run_filter_getsockopt
            |
             --3.30%--__cgroup_bpf_run_filter_getsockopt
                       |
                        --0.81%--__kmalloc

With the patch applied:
     0.52%     0.12%  tcp_mmap  [kernel.kallsyms]  [k] __cgroup_bpf_run_filter_getsockopt_kern

Note, exporting uapi/tcp.h requires removing netinet/tcp.h
from test_progs.h because those headers have confliciting
definitions.

Signed-off-by: Stanislav Fomichev <sdf@google.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Acked-by: Martin KaFai Lau <kafai@fb.com>
Link: https://lore.kernel.org/bpf/20210115163501.805133-2-sdf@google.com
2021-01-20 14:23:00 -08:00

229 lines
5.9 KiB
C

/* SPDX-License-Identifier: GPL-2.0 */
#include <stdio.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <assert.h>
#include <stdlib.h>
#include <stdarg.h>
#include <time.h>
#include <signal.h>
#include <linux/types.h>
typedef __u16 __sum16;
#include <arpa/inet.h>
#include <linux/if_ether.h>
#include <linux/if_packet.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/filter.h>
#include <linux/perf_event.h>
#include <linux/socket.h>
#include <linux/unistd.h>
#include <sys/ioctl.h>
#include <sys/wait.h>
#include <sys/types.h>
#include <sys/time.h>
#include <fcntl.h>
#include <pthread.h>
#include <linux/bpf.h>
#include <linux/err.h>
#include <bpf/bpf.h>
#include <bpf/libbpf.h>
#include "test_iptunnel_common.h"
#include "bpf_util.h"
#include <bpf/bpf_endian.h>
#include "trace_helpers.h"
#include "testing_helpers.h"
#include "flow_dissector_load.h"
enum verbosity {
VERBOSE_NONE,
VERBOSE_NORMAL,
VERBOSE_VERY,
VERBOSE_SUPER,
};
struct str_set {
const char **strs;
int cnt;
};
struct test_selector {
struct str_set whitelist;
struct str_set blacklist;
bool *num_set;
int num_set_len;
};
struct test_env {
struct test_selector test_selector;
struct test_selector subtest_selector;
bool verifier_stats;
enum verbosity verbosity;
bool jit_enabled;
bool has_testmod;
bool get_test_cnt;
bool list_test_names;
struct prog_test_def *test;
FILE *stdout;
FILE *stderr;
char *log_buf;
size_t log_cnt;
int nr_cpus;
int succ_cnt; /* successful tests */
int sub_succ_cnt; /* successful sub-tests */
int fail_cnt; /* total failed tests + sub-tests */
int skip_cnt; /* skipped tests */
int saved_netns_fd;
};
extern struct test_env env;
extern void test__force_log();
extern bool test__start_subtest(const char *name);
extern void test__skip(void);
extern void test__fail(void);
extern int test__join_cgroup(const char *path);
#define PRINT_FAIL(format...) \
({ \
test__fail(); \
fprintf(stdout, "%s:FAIL:%d ", __func__, __LINE__); \
fprintf(stdout, ##format); \
})
#define _CHECK(condition, tag, duration, format...) ({ \
int __ret = !!(condition); \
int __save_errno = errno; \
if (__ret) { \
test__fail(); \
fprintf(stdout, "%s:FAIL:%s ", __func__, tag); \
fprintf(stdout, ##format); \
} else { \
fprintf(stdout, "%s:PASS:%s %d nsec\n", \
__func__, tag, duration); \
} \
errno = __save_errno; \
__ret; \
})
#define CHECK_FAIL(condition) ({ \
int __ret = !!(condition); \
int __save_errno = errno; \
if (__ret) { \
test__fail(); \
fprintf(stdout, "%s:FAIL:%d\n", __func__, __LINE__); \
} \
errno = __save_errno; \
__ret; \
})
#define CHECK(condition, tag, format...) \
_CHECK(condition, tag, duration, format)
#define CHECK_ATTR(condition, tag, format...) \
_CHECK(condition, tag, tattr.duration, format)
#define ASSERT_EQ(actual, expected, name) ({ \
static int duration = 0; \
typeof(actual) ___act = (actual); \
typeof(expected) ___exp = (expected); \
bool ___ok = ___act == ___exp; \
CHECK(!___ok, (name), \
"unexpected %s: actual %lld != expected %lld\n", \
(name), (long long)(___act), (long long)(___exp)); \
___ok; \
})
#define ASSERT_NEQ(actual, expected, name) ({ \
static int duration = 0; \
typeof(actual) ___act = (actual); \
typeof(expected) ___exp = (expected); \
bool ___ok = ___act != ___exp; \
CHECK(!___ok, (name), \
"unexpected %s: actual %lld == expected %lld\n", \
(name), (long long)(___act), (long long)(___exp)); \
___ok; \
})
#define ASSERT_STREQ(actual, expected, name) ({ \
static int duration = 0; \
const char *___act = actual; \
const char *___exp = expected; \
bool ___ok = strcmp(___act, ___exp) == 0; \
CHECK(!___ok, (name), \
"unexpected %s: actual '%s' != expected '%s'\n", \
(name), ___act, ___exp); \
___ok; \
})
#define ASSERT_OK(res, name) ({ \
static int duration = 0; \
long long ___res = (res); \
bool ___ok = ___res == 0; \
CHECK(!___ok, (name), "unexpected error: %lld\n", ___res); \
___ok; \
})
#define ASSERT_ERR(res, name) ({ \
static int duration = 0; \
long long ___res = (res); \
bool ___ok = ___res < 0; \
CHECK(!___ok, (name), "unexpected success: %lld\n", ___res); \
___ok; \
})
#define ASSERT_NULL(ptr, name) ({ \
static int duration = 0; \
const void *___res = (ptr); \
bool ___ok = !___res; \
CHECK(!___ok, (name), "unexpected pointer: %p\n", ___res); \
___ok; \
})
#define ASSERT_OK_PTR(ptr, name) ({ \
static int duration = 0; \
const void *___res = (ptr); \
bool ___ok = !IS_ERR_OR_NULL(___res); \
CHECK(!___ok, (name), \
"unexpected error: %ld\n", PTR_ERR(___res)); \
___ok; \
})
#define ASSERT_ERR_PTR(ptr, name) ({ \
static int duration = 0; \
const void *___res = (ptr); \
bool ___ok = IS_ERR(___res) \
CHECK(!___ok, (name), "unexpected pointer: %p\n", ___res); \
___ok; \
})
static inline __u64 ptr_to_u64(const void *ptr)
{
return (__u64) (unsigned long) ptr;
}
static inline void *u64_to_ptr(__u64 ptr)
{
return (void *) (unsigned long) ptr;
}
int bpf_find_map(const char *test, struct bpf_object *obj, const char *name);
int compare_map_keys(int map1_fd, int map2_fd);
int compare_stack_ips(int smap_fd, int amap_fd, int stack_trace_len);
int extract_build_id(char *build_id, size_t size);
int kern_sync_rcu(void);
#ifdef __x86_64__
#define SYS_NANOSLEEP_KPROBE_NAME "__x64_sys_nanosleep"
#elif defined(__s390x__)
#define SYS_NANOSLEEP_KPROBE_NAME "__s390x_sys_nanosleep"
#else
#define SYS_NANOSLEEP_KPROBE_NAME "sys_nanosleep"
#endif