mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-07-06 11:18:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improve the affinity system calls

Browse source
This commit is contained in:
Justine Tunney 2022-10-06 15:08:29 -07:00
parent 60b68d7152
commit 59ac141e49
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
19 changed files with 231 additions and 123 deletions
Download patch file Download diff file Expand all files Collapse all files

16
libc/calls/sched-sysv.internal.h
View file

@ -1,13 +1,19 @@
#ifndef COSMOPOLITAN_LIBC_CALLS_SCHED_SYSV_INTERNAL_H_
#define COSMOPOLITAN_LIBC_CALLS_SCHED_SYSV_INTERNAL_H_
#include "libc/calls/struct/sched_param.h"
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
#define MAXCPUS_NETBSD 256
#define MAXCPUS_FREEBSD 256
#define MAXCPUS_OPENBSD 64
#define P_ALL_LWPS 0 /* for effect on all threads in pid */
#define CPU_LEVEL_WHICH 3
#define CPU_WHICH_TID 1
#define CPU_WHICH_PID 2
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
int sys_sched_get_priority_max(int);
int sys_sched_get_priority_min(int);
int sys_sched_getparam(int, struct sched_param *);
@ -28,6 +34,12 @@ int sys_sched_setaffinity_netbsd(int, int, size_t, const void *) //
int sys_sched_getaffinity_netbsd(int, int, size_t, void *) //
asm("sys_sched_setaffinity");
int sys_sched_setaffinity_freebsd(
int level, int which, int id, size_t setsize,
const void *mask) asm("sys_sched_setaffinity");
int sys_sched_getaffinity_freebsd(int level, int which, int id, size_t setsize,
void *mask) asm("sys_sched_getaffinity");
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_CALLS_SCHED_SYSV_INTERNAL_H_ */

72
libc/calls/sched_getaffinity.c
View file

@ -17,59 +17,79 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/calls/internal.h"
#include "libc/calls/sched-sysv.internal.h"
#include "libc/calls/struct/cpuset.h"
#include "libc/calls/syscall_support-nt.internal.h"
#include "libc/dce.h"
#include "libc/intrin/strace.internal.h"
#include "libc/macros.internal.h"
#include "libc/nt/enum/processaccess.h"
#include "libc/nt/process.h"
#include "libc/nt/runtime.h"
#include "libc/nt/thread.h"
#include "libc/str/str.h"
#include "libc/sysv/errfuns.h"
static textwindows int sys_sched_getaffinity_nt(int tid, size_t size,
cpu_set_t *bitset) {
uint64_t ProcessAffinityMask, SystemAffinityMask;
if (GetProcessAffinityMask(GetCurrentProcess(), &ProcessAffinityMask,
&SystemAffinityMask)) {
bzero(bitset, size);
bitset->__bits[0] = ProcessAffinityMask;
return 0;
static dontinline textwindows int sys_sched_getaffinity_nt(int pid, size_t size,
cpu_set_t *bitset) {
int rc;
int64_t h, closeme = -1;
uint64_t SystemAffinityMask;
if (!pid || pid == getpid()) {
h = GetCurrentProcess();
} else if (__isfdkind(pid, kFdProcess)) {
h = g_fds.p[pid].handle;
} else {
return __winerr();
h = OpenProcess(kNtProcessQueryInformation, false, pid);
if (!h) return __winerr();
closeme = h;
}
if (GetProcessAffinityMask(h, bitset->__bits, &SystemAffinityMask)) {
rc = 8;
} else {
rc = __winerr();
}
if (closeme != -1) {
CloseHandle(closeme);
}
return rc;
}
/**
* Gets CPU affinity for thread.
* Gets CPU affinity for process.
*
* While Windows allows us to change the thread affinity mask, it's only
* possible to read the process affinity mask. Therefore this function
* won't reflect the changes made by sched_setaffinity() on Windows.
*
* @param pid is the process or thread id (or 0 for caller)
* @param size is byte length of bitset, which should 128
* @param pid is the process id (or 0 for caller)
* @param size is bytes in bitset, which should be `sizeof(cpuset_t)`
* @param bitset receives bitset and should be uint64_t[16] in order to
* work on older versions of Linux
* @return 0 on success, or -1 w/ errno
* @raise ENOSYS if not Linux, NetBSD, or Windows
* @raise ENOSYS if not Linux, FreeBSD, NetBSD, or Windows
* @see pthread_getaffinity_np() for threads
*/
int sched_getaffinity(int tid, size_t size, cpu_set_t *bitset) {
int sched_getaffinity(int pid, size_t size, cpu_set_t *bitset) {
int rc;
if (size != 128) {
if (size != sizeof(cpu_set_t)) {
rc = einval();
} else if (IsWindows()) {
rc = sys_sched_getaffinity_nt(tid, size, bitset);
rc = sys_sched_getaffinity_nt(pid, size, bitset);
} else if (IsFreebsd()) {
if (!sys_sched_getaffinity_freebsd(CPU_LEVEL_WHICH, CPU_WHICH_PID, pid, 32,
bitset)) {
rc = 32;
} else {
rc = -1;
}
} else if (IsNetbsd()) {
if (!sys_sched_getaffinity_netbsd(0, tid, MIN(size, 32), bitset)) {
rc = MIN(size, 32);
if (!sys_sched_getaffinity_netbsd(P_ALL_LWPS, pid, 32, bitset)) {
rc = 32;
} else {
rc = -1;
}
} else {
rc = sys_sched_getaffinity(tid, size, bitset);
rc = sys_sched_getaffinity(pid, size, bitset);
}
if (rc > 0) {
if (rc < size) {
@ -77,6 +97,6 @@ int sched_getaffinity(int tid, size_t size, cpu_set_t *bitset) {
}
rc = 0;
}
STRACE("sched_getaffinity(%d, %'zu, %p) → %d% m", tid, size, bitset, rc);
STRACE("sched_getaffinity(%d, %'zu, %p) → %d% m", pid, size, bitset, rc);
return rc;
}

84
libc/calls/sched_setaffinity.c
View file

@ -16,75 +16,69 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/calls/internal.h"
#include "libc/calls/sched-sysv.internal.h"
#include "libc/calls/struct/cpuset.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/calls/syscall_support-nt.internal.h"
#include "libc/dce.h"
#include "libc/intrin/safemacros.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/limits.h"
#include "libc/macros.internal.h"
#include "libc/nt/enum/processaccess.h"
#include "libc/nt/enum/threadaccess.h"
#include "libc/nt/process.h"
#include "libc/nt/runtime.h"
#include "libc/nt/thread.h"
#include "libc/str/str.h"
#include "libc/sysv/errfuns.h"
static textwindows dontinline int sys_sched_setaffinity_nt(int pid,
uint64_t size,
const void *bitset) {
static dontinline textwindows int sys_sched_setaffinity_nt(
int pid, uint64_t size, const cpu_set_t *bitset) {
int rc;
int64_t handle;
uintptr_t mask;
typeof(SetThreadAffinityMask) *SetAffinityMask = SetThreadAffinityMask;
mask = 0;
memcpy(&mask, bitset, min(size, sizeof(uintptr_t)));
handle = 0;
if (!pid) pid = GetCurrentThreadId();
if (0 < pid && pid <= UINT32_MAX) {
if (pid == GetCurrentProcessId()) {
pid = GetCurrentProcess();
SetAffinityMask = SetProcessAffinityMask;
} else if (pid == GetCurrentThreadId()) {
pid = GetCurrentThread();
} else {
handle = OpenThread(kNtThreadSetInformation | kNtThreadQueryInformation,
false, pid);
if (!handle) {
handle = OpenProcess(
kNtProcessSetInformation | kNtProcessQueryInformation, false, pid);
SetAffinityMask = SetProcessAffinityMask;
}
}
int64_t h, closeme = -1;
if (!pid /* || pid == getpid() */) {
h = GetCurrentProcess();
} else if (__isfdkind(pid, kFdProcess)) {
h = g_fds.p[pid].handle;
} else {
h = OpenProcess(kNtProcessSetInformation | kNtProcessQueryInformation,
false, pid);
if (!h) return __winerr();
closeme = h;
}
rc = SetAffinityMask(handle ? handle : pid, mask) ? 0 : __winerr();
if (handle) CloseHandle(handle);
if (SetProcessAffinityMask(h, bitset->__bits[0])) {
rc = 0;
} else {
rc = __winerr();
}
if (closeme != -1) {
CloseHandle(closeme);
}
return rc;
}
/**
* Asks kernel to only schedule thread on particular CPUs.
* Asks kernel to only schedule process on particular CPUs.
*
* @param tid is the process or thread id (or 0 for caller)
* @param size is byte length of bitset, which should be 128
* @param pid is the process or process id (or 0 for caller)
* @param size is bytes in bitset, which should be `sizeof(cpuset_t)`
* @return 0 on success, or -1 w/ errno
* @raise ENOSYS if not Linux, NetBSD, or Windows
* @raise ENOSYS if not Linux, FreeBSD, NetBSD, or Windows
* @see pthread_getaffinity_np() for threads
*/
int sched_setaffinity(int tid, size_t size, const cpu_set_t *bitset) {
int sched_setaffinity(int pid, size_t size, const cpu_set_t *bitset) {
int rc;
if (size != 128) {
if (size != sizeof(cpu_set_t)) {
rc = einval();
} else if (IsWindows()) {
rc = sys_sched_setaffinity_nt(tid, size, bitset);
rc = sys_sched_setaffinity_nt(pid, size, bitset);
} else if (IsFreebsd()) {
rc = sys_sched_setaffinity_freebsd(CPU_LEVEL_WHICH, CPU_WHICH_PID, pid, 32,
bitset);
} else if (IsNetbsd()) {
rc = sys_sched_setaffinity_netbsd(0, tid, MIN(size, 32), bitset);
rc = sys_sched_setaffinity_netbsd(P_ALL_LWPS, pid, 32, bitset);
} else {
rc = sys_sched_setaffinity(tid, size, bitset);
rc = sys_sched_setaffinity(pid, size, bitset);
}
STRACE("sched_setaffinity(%d, %'zu, %p) → %d% m", tid, size, bitset, rc);
STRACE("sched_setaffinity(%d, %'zu, %p) → %d% m", pid, size, bitset, rc);
return rc;
}

2
libc/intrin/describefutexresult.c → libc/intrin/describeerrnoresult.c
View file

@ -20,7 +20,7 @@
#include "libc/intrin/describeflags.internal.h"
#include "libc/str/str.h"
const char *(DescribeFutexResult)(char buf[12], int ax) {
const char *(DescribeErrnoResult)(char buf[12], int ax) {
const char *s;
if (ax > -4095u && (s = _strerrno(-ax))) {
return s;

4
libc/intrin/describeflags.internal.h
View file

@ -16,9 +16,9 @@ const char *DescribeArchPrctlCode(char[12], int);
const char *DescribeCapability(char[20], int);
const char *DescribeClockName(char[32], int);
const char *DescribeDirfd(char[12], int);
const char *DescribeErrnoResult(char[12], int);
const char *DescribeFrame(char[32], int);
const char *DescribeFutexOp(char[64], int);
const char *DescribeFutexResult(char[12], int);
const char *DescribeHow(char[12], int);
const char *DescribeInOutInt64(char[23], ssize_t, int64_t *);
const char *DescribeMapFlags(char[64], int);
@ -62,9 +62,9 @@ const char *DescribeWhence(char[12], int);
#define DescribeCapability(x) DescribeCapability(alloca(20), x)
#define DescribeClockName(x) DescribeClockName(alloca(32), x)
#define DescribeDirfd(x) DescribeDirfd(alloca(12), x)
#define DescribeErrnoResult(x) DescribeErrnoResult(alloca(12), x)
#define DescribeFrame(x) DescribeFrame(alloca(32), x)
#define DescribeFutexOp(x) DescribeFutexOp(alloca(64), x)
#define DescribeFutexResult(x) DescribeFutexResult(alloca(12), x)
#define DescribeHow(x) DescribeHow(alloca(12), x)
#define DescribeInOutInt64(rc, x) DescribeInOutInt64(alloca(23), rc, x)
#define DescribeMapFlags(x) DescribeMapFlags(alloca(64), x)

2
libc/nt/process.h
View file

@ -61,7 +61,7 @@ int32_t SetEnvironmentStrings(char16_t *NewEnvironment);
bool32 GetProcessAffinityMask(int64_t hProcess, uint64_t *lpProcessAffinityMask,
uint64_t *lpSystemAffinityMask);
uint64_t /*bool32*/ SetProcessAffinityMask(int64_t hProcess,
uintptr_t dwProcessAffinityMask);
uint64_t dwProcessAffinityMask);
/* e.g. kNtAboveNormalPriorityClass, kNtHighPriorityClass */
uint32_t GetPriorityClass(int64_t hProcess);

9
libc/runtime/fork.c
View file

@ -18,6 +18,7 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/calls/struct/sigset.h"
#include "libc/calls/struct/sigset.internal.h"
#include "libc/calls/syscall-nt.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
@ -39,9 +40,9 @@ int fork(void) {
axdx_t ad;
sigset_t old, all;
int ax, dx, parent;
sigfillset(&all);
sigprocmask(SIG_BLOCK, &all, &old);
if (!IsWindows()) {
sigfillset(&all);
sys_sigprocmask(SIG_BLOCK, &all, &old);
ad = sys_fork();
ax = ad.ax;
dx = ad.dx;
@ -64,10 +65,10 @@ int fork(void) {
if (__tls_enabled) {
__get_tls()->tib_tid = IsLinux() ? dx : sys_gettid();
}
sigprocmask(SIG_SETMASK, &old, 0);
if (!IsWindows()) sys_sigprocmask(SIG_SETMASK, &old, 0);
STRACE("fork() → 0 (child of %d)", parent);
} else {
sigprocmask(SIG_SETMASK, &old, 0);
if (!IsWindows()) sys_sigprocmask(SIG_SETMASK, &old, 0);
STRACE("fork() → %d% m", ax);
}
return ax;

2
libc/sysv/calls/sys_cpuset_getaffinity.s
View file

@ -1,2 +0,0 @@
.include "o/libc/sysv/macros.internal.inc"
.scall sys_cpuset_getaffinity,0xffffff1e7fffffff,globl

2
libc/sysv/calls/sys_cpuset_setaffinity.s
View file

@ -1,2 +0,0 @@
.include "o/libc/sysv/macros.internal.inc"
.scall sys_cpuset_setaffinity,0xffffff1e8fffffff,globl

2
libc/sysv/calls/sys_sched_getaffinity.s
View file

@ -1,2 +1,2 @@
.include "o/libc/sysv/macros.internal.inc"
.scall sys_sched_getaffinity,0x15dffffffffff0cc,globl,hidden
.scall sys_sched_getaffinity,0x15dfff1e7ffff0cc,globl,hidden

2
libc/sysv/calls/sys_sched_setaffinity.s
View file

@ -1,2 +1,2 @@
.include "o/libc/sysv/macros.internal.inc"
.scall sys_sched_setaffinity,0x15cffffffffff0cb,globl,hidden
.scall sys_sched_setaffinity,0x15cfff1e8ffff0cb,globl,hidden

6
libc/sysv/syscalls.sh
View file

@ -202,12 +202,10 @@ scall sys_sched_setparam 0x15afff147ffff08e globl hidden
scall sys_sched_getparam 0x15bfff148ffff08f globl hidden
scall sys_sched_setscheduler 0xffffff149ffff090 globl hidden
scall sys_sched_getscheduler 0xffffff14affff091 globl hidden
scall sys_sched_setaffinity 0x15cffffffffff0cb globl hidden
scall sys_sched_getaffinity 0x15dffffffffff0cc globl hidden # returns bytes written on success. we polyfill bad posix designs like nice() returning 0, but we won't polyfill a bad unilateral redesign that's just glibc
scall sys_sched_setaffinity 0x15cfff1e8ffff0cb globl hidden # hairy; cpuset_setaffinity on FreeBSD
scall sys_sched_getaffinity 0x15dfff1e7ffff0cc globl hidden # hairy; cpuset_getaffinity on FreeBSD
scall sys_sched_get_priority_max 0xffffff14cffff092 globl hidden
scall sys_sched_get_priority_min 0xffffff14dffff093 globl hidden
scall sys_cpuset_getaffinity 0xffffff1e7fffffff globl # no wrapper
scall sys_cpuset_setaffinity 0xffffff1e8fffffff globl # no wrapper
scall sys_sched_rr_get_interval 0xffffff14effff094 globl hidden
scall sys_vhangup 0xfffffffffffff099 globl # no wrapper
scall sys_modify_ldt 0xfffffffffffff09a globl # no wrapper

55
libc/thread/pthread_getaffinity_np.c
View file

@ -16,30 +16,57 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/sched-sysv.internal.h"
#include "libc/calls/struct/cpuset.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/str/str.h"
#include "libc/sysv/errfuns.h"
#include "libc/thread/posixthread.internal.h"
/**
* Gets CPU affinity for thread.
*
* While Windows allows us to change the thread affinity mask, it's only
* possible to read the process affinity mask. Therefore this function
* won't reflect the changes made by psched_setaffinity_np() on Windows.
*
* @param bitsetsize is byte length of bitset, which should be 128
* @param size is bytes in bitset, which should be `sizeof(cpu_set_t)`
* @return 0 on success, or errno on error
* @raise ENOSYS if not Linux or Windows
* @raise EINVAL if `size` or `bitset` is invalid
* @raise ENOSYS if not Linux, FreeBSD, or NetBSD
*/
errno_t pthread_getaffinity_np(pthread_t thread, size_t bitsetsize,
errno_t pthread_getaffinity_np(pthread_t thread, size_t size,
cpu_set_t *bitset) {
int rc, e = errno;
struct PosixThread *pt = (struct PosixThread *)thread;
if (!sched_getaffinity(pt->tid, bitsetsize, bitset)) {
return 0;
int tid, rc, e = errno;
tid = ((struct PosixThread *)thread)->tid;
if (size != sizeof(cpu_set_t)) {
rc = einval();
} else if (IsWindows() || IsMetal() || IsOpenbsd()) {
rc = enosys();
} else if (IsFreebsd()) {
if (!sys_sched_getaffinity_freebsd(CPU_LEVEL_WHICH, CPU_WHICH_TID, tid, 32,
bitset)) {
rc = 32;
} else {
rc = -1;
}
} else if (IsNetbsd()) {
if (!sys_sched_getaffinity_netbsd(tid, 0, 32, bitset)) {
rc = 32;
} else {
rc = -1;
}
} else {
rc = errno;
errno = e;
return rc;
rc = sys_sched_getaffinity(tid, size, bitset);
}
if (rc > 0) {
if (rc < size) {
bzero((char *)bitset + rc, size - rc);
}
rc = 0;
}
STRACE("pthread_getaffinity_np(%d, %'zu, %p) → %s", tid, size, bitset,
DescribeErrnoResult(rc));
return rc;
}

51
libc/thread/pthread_setaffinity_np.c
View file

@ -16,26 +16,61 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/sched-sysv.internal.h"
#include "libc/calls/struct/cpuset.h"
#include "libc/calls/syscall_support-nt.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/nt/enum/threadaccess.h"
#include "libc/nt/runtime.h"
#include "libc/nt/thread.h"
#include "libc/sysv/errfuns.h"
#include "libc/thread/posixthread.internal.h"
static dontinline textwindows int sys_pthread_setaffinity_nt(
int tid, uint64_t size, const cpu_set_t *bitset) {
int rc;
int64_t h;
h = OpenThread(kNtThreadSetInformation | kNtThreadQueryInformation, false,
tid);
if (!h) return __winerr();
rc = SetThreadAffinityMask(h, bitset->__bits[0]) ? 0 : __winerr();
CloseHandle(h);
return rc;
}
/**
* Asks kernel to only schedule thread on particular CPUs.
*
* @param bitsetsize is byte length of bitset, which should be 128
* @param size is bytes in bitset, which should be `sizeof(cpu_set_t)`
* @return 0 on success, or errno on error
* @raise ENOSYS if not Linux or Windows
* @raise EINVAL if `size` or `bitset` is invalid
* @raise ENOSYS if not Linux, FreeBSD, NetBSD, or Windows
* @see sched_setaffinity() for processes
*/
errno_t pthread_setaffinity_np(pthread_t thread, size_t bitsetsize,
errno_t pthread_setaffinity_np(pthread_t thread, size_t size,
const cpu_set_t *bitset) {
int rc, e = errno;
struct PosixThread *pt = (struct PosixThread *)thread;
if (!sched_setaffinity(pt->tid, bitsetsize, bitset)) {
return 0;
int tid, rc, e = errno;
tid = ((struct PosixThread *)thread)->tid;
if (size != sizeof(cpu_set_t)) {
rc = einval();
} else if (IsWindows()) {
rc = sys_pthread_setaffinity_nt(tid, size, bitset);
} else if (IsFreebsd()) {
rc = sys_sched_setaffinity_freebsd(CPU_LEVEL_WHICH, CPU_WHICH_TID, tid, 32,
bitset);
} else if (IsNetbsd()) {
rc = sys_sched_setaffinity_netbsd(tid, 0, 32, bitset);
} else {
rc = sys_sched_setaffinity(tid, size, bitset);
}
if (rc == -1) {
rc = errno;
errno = e;
return rc;
}
STRACE("pthread_setaffinity_np(%d, %'zu, %p) → %s", tid, size, bitset,
DescribeErrnoResult(rc));
return rc;
}

2
libc/thread/wait0.c
View file

@ -74,7 +74,7 @@ static void _wait0_futex(const atomic_int *a, int e) {
}
}
STRACE("futex(%t, %s, %d, %s) → %s", a, DescribeFutexOp(op), e, "NULL",
DescribeFutexResult(rc));
DescribeErrnoResult(rc));
_npassert(rc == 0 || //
rc == -EINTR || //
rc == -ETIMEDOUT || //