Greatly expand system() shell code features

The cosmopolitan command interpreter now has 13 builtin commands,
variable support, support for ; / && / || syntax, asynchronous support,
and plenty of unit tests with bug fixes.

This change fixes a bug in posix_spawn() with null envp arg. strace
logging now uses atomic writes for scatter functions. Breaking change
renaming GetCpuCount() to _getcpucount(). TurfWar is now updated to use
the new token bucket algorithm. WIN32 affinity masks now inherit across
fork() and execve().
This commit is contained in:
Justine Tunney 2022-10-11 21:06:27 -07:00
parent e7329b7cba
commit b41f91c658
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
80 changed files with 1370 additions and 344 deletions

View file

@ -16,15 +16,26 @@
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/calls/struct/stat.h"
#include "libc/calls/struct/timespec.h"
#include "libc/errno.h"
#include "libc/fmt/conv.h"
#include "libc/fmt/itoa.h"
#include "libc/fmt/magnumstrs.internal.h"
#include "libc/intrin/safemacros.internal.h"
#include "libc/macros.internal.h"
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/append.h"
#include "libc/stdio/stdio.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/o.h"
#include "libc/sysv/consts/s.h"
#include "libc/sysv/consts/sig.h"
#include "libc/x/x.h"
#include "third_party/double-conversion/wrapper.h"
/**
* @fileoverview Cosmopolitan Command Interpreter
@ -33,27 +44,53 @@
* enough shell script language support to support our build config.
*/
#define STATE_SHELL 0
#define STATE_STR 1
#define STATE_QUO 2
#define STATE_CMD 0
#define STATE_VAR 1
#define STATE_SINGLE 2
#define STATE_QUOTED 3
#define STATE_QUOTED_VAR 4
#define STATE_WHITESPACE 5
static char *p;
static char *q;
static int vari;
static size_t n;
static char *cmd;
static char var[32];
static int lastchild;
static int exitstatus;
static char *args[8192];
static const char *prog;
static char errbuf[512];
static char argbuf[ARG_MAX];
static bool unsupported[256];
static ssize_t Write(int fd, const char *s) {
return write(fd, s, strlen(s));
}
static void Log(const char *s, ...) {
va_list va;
va_start(va, s);
errbuf[0] = 0;
do {
strlcat(errbuf, s, sizeof(argbuf));
} while ((s = va_arg(va, const char *)));
strlcat(errbuf, "\n", sizeof(argbuf));
Write(2, errbuf);
va_end(va);
}
static wontreturn void Wexit(int rc, const char *s, ...) {
va_list va;
va_start(va, s);
errbuf[0] = 0;
do {
write(2, s, strlen(s));
strlcat(errbuf, s, sizeof(argbuf));
} while ((s = va_arg(va, const char *)));
Write(2, errbuf);
va_end(va);
exit(rc);
_Exit(rc);
}
static wontreturn void UnsupportedSyntax(unsigned char c) {
@ -86,86 +123,391 @@ static void Open(const char *path, int fd, int flags) {
}
static wontreturn void Exec(void) {
const char *s;
if (!n) {
Wexit(5, prog, ": error: too few args\n", 0);
}
_npassert(args[0][0]);
if (!n) Wexit(5, prog, ": error: too few args\n", 0);
execvp(args[0], args);
SysExit(127, "execve", args[0]);
}
static int GetSignalByName(const char *s) {
for (int i = 0; kSignalNames[i].x != MAGNUM_TERMINATOR; ++i) {
if (!strcmp(s, MAGNUM_STRING(kSignalNames, i) + 3)) {
return MAGNUM_NUMBER(kSignalNames, i);
}
}
return 0;
}
static int True(void) {
return 0;
}
static int False(void) {
return 1;
}
static wontreturn void Exit(void) {
_Exit(n > 1 ? atoi(args[1]) : 0);
}
static int Wait(void) {
char ibuf[12];
int e, rc, ws, pid;
if (n > 1) {
if (waitpid(atoi(args[1]), &ws, 0) == -1) {
SysExit(22, "waitpid", prog);
}
rc = WIFEXITED(ws) ? WEXITSTATUS(ws) : 128 + WTERMSIG(ws);
exitstatus = rc;
} else {
for (e = errno;;) {
if (waitpid(-1, &ws, 0) == -1) {
if (errno == ECHILD) {
errno = e;
break;
}
SysExit(22, "waitpid", prog);
}
}
rc = 0;
}
return rc;
}
static int Echo(void) {
int i = 1;
bool once = false;
const char *l = " ";
if (i < n && !strcmp(args[i], "-l")) {
++i, l = "\n";
}
for (; i < n; ++i) {
if (once) {
Write(1, l);
} else {
once = true;
}
Write(1, args[i]);
}
Write(1, "\n");
return 0;
}
static int Read(void) {
char *b = 0;
unsigned char c;
int a = 1, rc = 1;
if (n >= 3 && !strcmp(args[1], "-p")) {
Write(1, args[2]);
a = 3;
}
appendr(&b, 0);
while (read(0, &c, 1) > 0) {
if (c == '\n') {
rc = 0;
break;
}
appendw(&b, c);
}
if (a < n) {
setenv(args[1], b, true);
}
free(b);
return rc;
}
static int Cd(void) {
const char *s = n > 1 ? args[1] : getenv("HOME");
if (s) {
if (!chdir(s)) {
return 0;
} else {
Log("chdir: ", s, ": ", _strerdoc(errno), 0);
return 1;
}
} else {
Log("chdir: missing argument", 0);
return 1;
}
}
static int Mkdir(void) {
int i = 1;
int (*f)(const char *, unsigned) = mkdir;
if (n >= 3 && !strcmp(args[1], "-p")) ++i, f = makedirs;
for (; i < n; ++i) {
if (f(args[i], 0755)) {
Log("mkdir: ", args[i], ": ", _strerdoc(errno), 0);
return errno;
}
}
return 0;
}
static int Kill(void) {
int sig, rc = 0, i = 1;
if (i < n && args[i][0] == '-') {
sig = GetSignalByName(args[i++] + 1);
if (!sig) return 1;
} else {
sig = SIGTERM;
}
for (; i < n; ++i) {
if (kill(atoi(args[i]), sig)) {
Log("kill: ", args[i], ": ", _strerdoc(errno), 0);
rc = 1;
}
}
return rc;
}
static int Toupper(void) {
int i, n;
char b[512];
while ((n = read(0, b, 512)) > 0) {
for (i = 0; i < n; ++i) {
b[i] = toupper(b[i]);
}
write(1, b, n);
}
return 0;
}
static int Usleep(void) {
struct timespec t, *p = 0;
if (n > 1) {
t = _timespec_frommicros(atoi(args[1]));
p = &t;
}
return clock_nanosleep(0, 0, p, 0);
}
static int Test(void) {
struct stat st;
if (n && !strcmp(args[n - 1], "]")) --n;
if (n == 4 && !strcmp(args[2], "=")) {
return !!strcmp(args[1], args[3]);
} else if (n == 4 && !strcmp(args[2], "!=")) {
return !strcmp(args[1], args[3]);
} else if (n == 3 && !strcmp(args[1], "-n")) {
return !(strlen(args[2]) > 0);
} else if (n == 3 && !strcmp(args[1], "-z")) {
return !(strlen(args[2]) == 0);
} else if (n == 3 && !strcmp(args[1], "-e")) {
return !!stat(args[2], &st);
} else if (n == 3 && !strcmp(args[1], "-f")) {
return !stat(args[2], &st) && S_ISREG(st.st_mode);
} else if (n == 3 && !strcmp(args[1], "-d")) {
return !stat(args[2], &st) && S_ISDIR(st.st_mode);
} else if (n == 3 && !strcmp(args[1], "-h")) {
return !stat(args[2], &st) && S_ISLNK(st.st_mode);
} else {
return 1;
}
}
static int TryBuiltin(void) {
if (!n) return 0;
if (!strcmp(args[0], "exit")) Exit();
if (!strcmp(args[0], "cd")) return Cd();
if (!strcmp(args[0], "[")) return Test();
if (!strcmp(args[0], "wait")) return Wait();
if (!strcmp(args[0], "echo")) return Echo();
if (!strcmp(args[0], "read")) return Read();
if (!strcmp(args[0], "true")) return True();
if (!strcmp(args[0], "test")) return Test();
if (!strcmp(args[0], "kill")) return Kill();
if (!strcmp(args[0], "mkdir")) return Mkdir();
if (!strcmp(args[0], "false")) return False();
if (!strcmp(args[0], "usleep")) return Usleep();
if (!strcmp(args[0], "toupper")) return Toupper();
return -1;
}
static wontreturn void Launch(void) {
int rc;
if ((rc = TryBuiltin()) != -1) _Exit(rc);
Exec();
}
static void Pipe(void) {
int pid, pfds[2];
if (pipe2(pfds, O_CLOEXEC)) {
SysExit(8, "pipe2", prog);
}
if ((pid = vfork()) == -1) {
SysExit(9, "vfork", prog);
}
if (pipe2(pfds, O_CLOEXEC)) SysExit(8, "pipe2", prog);
if ((pid = fork()) == -1) SysExit(9, "vfork", prog);
if (!pid) {
dup2(pfds[1], 1);
Exec();
_unassert(dup2(pfds[1], 1) == 1);
// we can't rely on cloexec because builtins
if (pfds[0] != 1) _unassert(!close(pfds[0]));
if (pfds[1] != 1) _unassert(!close(pfds[1]));
Launch();
}
dup2(pfds[0], 0);
_unassert(!dup2(pfds[0], 0));
if (pfds[1]) _unassert(!close(pfds[1]));
n = 0;
}
static int Run(void) {
int exitstatus, ws, pid;
if ((exitstatus = TryBuiltin()) == -1) {
if ((pid = vfork()) == -1) SysExit(21, "vfork", prog);
if (!pid) Exec();
if (waitpid(pid, &ws, 0) == -1) SysExit(22, "waitpid", prog);
exitstatus = WIFEXITED(ws) ? WEXITSTATUS(ws) : 128 + WTERMSIG(ws);
}
n = 0;
return exitstatus;
}
static void Async(void) {
if ((lastchild = fork()) == -1) SysExit(21, "vfork", prog);
if (!lastchild) Launch();
n = 0;
}
static const char *IntToStr(int x) {
static char ibuf[12];
FormatInt32(ibuf, x);
return ibuf;
}
static const char *GetEnv(const char *key) {
if (key[0] == '$' && !key[1]) {
return IntToStr(getpid());
} else if (key[0] == '!' && !key[1]) {
return IntToStr(lastchild);
} else if (key[0] == '?' && !key[1]) {
return IntToStr(exitstatus);
} else {
return getenv(key);
}
}
static bool IsVarName(int c) {
return isalnum(c) || c == '_' ||
(!vari && (c == '?' || c == '!' || c == '$'));
}
static inline void Append(int c) {
_unassert(q + 1 < argbuf + sizeof(argbuf));
*q++ = c;
}
static char *Tokenize(void) {
char *r;
int c, t;
while (*p == ' ' || *p == '\t' || *p == '\n' ||
(p[0] == '\\' && p[1] == '\n')) {
++p;
}
if (!*p) return 0;
t = STATE_SHELL;
for (r = q;; ++p) {
const char *s;
int c, t, j, k, rc;
for (r = q, t = STATE_WHITESPACE;; ++p) {
switch (t) {
case STATE_SHELL:
case STATE_WHITESPACE:
if (!*p) return 0;
if (*p == ' ' || *p == '\t' || *p == '\n' ||
(p[0] == '\\' && p[1] == '\n')) {
continue;
}
t = STATE_CMD;
// fallthrough
case STATE_CMD:
if (unsupported[*p & 255]) {
UnsupportedSyntax(*p);
}
if (!*p || *p == ' ' || *p == '\t') {
*q++ = 0;
Append(0);
return r;
} else if (*p == '"') {
t = STATE_QUO;
t = STATE_QUOTED;
} else if (*p == '\'') {
t = STATE_STR;
t = STATE_SINGLE;
} else if (*p == '$') {
t = STATE_VAR;
var[(vari = 0)] = 0;
} else if (*p == '\\') {
if (!p[1]) UnsupportedSyntax(*p);
*q++ = *++p;
Append(*++p);
} else if (*p == '|') {
if (q > r) {
*q = 0;
Append(0);
return r;
} else if (p[1] == '|') {
rc = Run();
if (!rc) {
_Exit(0);
} else {
++p;
t = STATE_WHITESPACE;
}
} else {
Pipe();
++p;
t = STATE_WHITESPACE;
}
} else if (*p == ';') {
if (q > r) {
Append(0);
return r;
} else {
Run();
t = STATE_WHITESPACE;
}
} else if (*p == '&') {
if (q > r) {
Append(0);
return r;
} else if (p[1] == '&') {
rc = Run();
if (!rc) {
++p;
t = STATE_WHITESPACE;
} else {
_Exit(rc);
}
} else {
Async();
t = STATE_WHITESPACE;
}
} else {
*q++ = *p;
Append(*p);
}
break;
case STATE_STR:
if (!*p) {
Wexit(6, "cmd: error: unterminated single string\n", 0);
case STATE_VAR:
if (IsVarName(*p)) {
_unassert(vari + 1 < sizeof(var));
var[vari++] = *p;
var[vari] = 0;
} else {
// XXX: we need to find a simple elegant way to break up
// unquoted variable expansions into multiple args.
if ((s = GetEnv(var))) {
if ((j = strlen(s))) {
_unassert(q + j < argbuf + sizeof(argbuf));
q = mempcpy(q, s, j);
}
}
--p;
t = STATE_CMD;
}
break;
case STATE_SINGLE:
if (!*p) goto UnterminatedString;
if (*p == '\'') {
t = STATE_SHELL;
t = STATE_CMD;
} else {
*q++ = *p;
}
break;
case STATE_QUO:
if (!*p) {
Wexit(6, "cmd: error: unterminated quoted string\n", 0);
}
UnterminatedString:
Wexit(6, "cmd: error: unterminated string\n", 0);
case STATE_QUOTED:
if (!*p) goto UnterminatedString;
if (*p == '"') {
t = STATE_SHELL;
t = STATE_CMD;
} else if (p[0] == '$') {
t = STATE_QUOTED_VAR;
var[(vari = 0)] = 0;
} else if (p[0] == '\\') {
switch ((c = *++p)) {
case 0:
@ -187,6 +529,24 @@ static char *Tokenize(void) {
}
break;
case STATE_QUOTED_VAR:
if (!*p) goto UnterminatedString;
if (IsVarName(*p)) {
_unassert(vari + 1 < sizeof(var));
var[vari++] = *p;
var[vari] = 0;
} else {
if ((s = GetEnv(var))) {
if ((j = strlen(s))) {
_unassert(q + j < argbuf + sizeof(argbuf));
q = mempcpy(q, s, j);
}
}
--p;
t = STATE_QUOTED;
}
break;
default:
unreachable;
}
@ -219,13 +579,9 @@ int cocmd(int argc, char *argv[]) {
unsupported['*'] = true;
unsupported['('] = true;
unsupported[')'] = true;
unsupported['['] = true;
unsupported[']'] = true;
unsupported['{'] = true;
unsupported['}'] = true;
unsupported[';'] = true;
unsupported['?'] = true;
unsupported['!'] = true;
if (argc != 3) {
Wexit(10, prog, ": error: wrong number of args\n", 0);
@ -270,5 +626,5 @@ int cocmd(int argc, char *argv[]) {
}
}
Exec();
Launch();
}

74
libc/stdio/makedirs.c Normal file
View file

@ -0,0 +1,74 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi
Copyright 2021 Justine Alexandra Roberts Tunney
Permission to use, copy, modify, and/or distribute this software for
any purpose with or without fee is hereby granted, provided that the
above copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/calls/calls.h"
#include "libc/errno.h"
#include "libc/fmt/conv.h"
#include "libc/mem/mem.h"
#include "libc/str/str.h"
static char *DirName(const char *path) {
char *dirp;
if (!(path = strdup(path))) return 0;
dirp = strdup(dirname(path));
free(path);
return dirp;
}
static int MakeDirs(const char *path, unsigned mode, int e) {
int rc;
char *dir;
if (!mkdir(path, mode) || errno == EEXIST) {
errno = e;
return 0;
}
if (errno != ENOENT) {
return -1;
}
if (!(dir = DirName(path))) {
return -1;
}
if (strcmp(dir, path)) {
rc = MakeDirs(dir, mode, e);
} else {
rc = -1;
}
free(dir);
if (rc == -1) return -1;
errno = e;
if (!mkdir(path, mode) || errno == EEXIST) {
errno = e;
return 0;
} else {
return -1;
}
}
/**
* Recursively creates directory a.k.a. folder.
*
* This function won't fail if the directory already exists.
*
* @param path is a UTF-8 string, preferably relative w/ forward slashes
* @param mode can be, for example, 0755
* @return 0 on success or -1 w/ errno
* @threadsafe
*/
int makedirs(const char *path, unsigned mode) {
return MakeDirs(path, mode, errno);
}

View file

@ -16,6 +16,7 @@
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/errno.h"
#include "libc/paths.h"
@ -31,14 +32,8 @@
/**
* Spawns subprocess and returns pipe stream.
*
* This embeds the cocmd.com shell interpreter which supports a limited
* subset of the bourne shell that's significantly faster:
*
* - pipelines
* - single quotes
* - double quotes
* - input redirection, e.g. `<path`
* - output redirection, e.g. `>path`, `>>append`, `2>err.txt, `2>&1`
* This embeds the Cosmopolitan Command Interpreter which provides
* Bourne-like syntax on all platforms including Windows.
*
* @see pclose()
*/
@ -54,28 +49,30 @@ FILE *popen(const char *cmdline, const char *mode) {
einval();
return NULL;
}
if (pipe(pipefds) == -1) return NULL;
fcntl(pipefds[dir], F_SETFD, FD_CLOEXEC);
if (pipe2(pipefds, O_CLOEXEC) == -1) return NULL;
if ((f = fdopen(pipefds[dir], mode))) {
switch ((pid = fork())) {
case 0:
dup2(pipefds[!dir], !dir);
_unassert(dup2(pipefds[!dir], !dir) == !dir);
// we can't rely on cloexec because cocmd builtins don't execev
if (pipefds[0] != !dir) _unassert(!close(pipefds[0]));
if (pipefds[1] != !dir) _unassert(!close(pipefds[1]));
_Exit(cocmd(3, (char *[]){"popen", "-c", cmdline, 0}));
default:
f->pid = pid;
close(pipefds[!dir]);
_unassert(!close(pipefds[!dir]));
return f;
case -1:
e = errno;
fclose(f);
close(pipefds[!dir]);
_unassert(!fclose(f));
_unassert(!close(pipefds[!dir]));
errno = e;
return NULL;
}
} else {
e = errno;
close(pipefds[0]);
close(pipefds[1]);
_unassert(!close(pipefds[0]));
_unassert(!close(pipefds[1]));
errno = e;
return NULL;
}

View file

@ -16,11 +16,14 @@
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/calls/struct/sched_param.h"
#include "libc/calls/struct/sigaction.h"
#include "libc/errno.h"
#include "libc/fmt/fmt.h"
#include "libc/intrin/kprintf.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/spawn.h"
#include "libc/stdio/spawna.internal.h"
#include "libc/str/str.h"
@ -47,7 +50,7 @@ int posix_spawn(int *pid, const char *path,
if (setpgid(0, attrp->posix_attr_pgroup)) _Exit(127);
}
if (attrp->posix_attr_flags & POSIX_SPAWN_SETSIGMASK) {
sigprocmask(SIG_SETMASK, &attrp->posix_attr_sigmask, NULL);
sigprocmask(SIG_SETMASK, &attrp->posix_attr_sigmask, 0);
}
if (attrp->posix_attr_flags & POSIX_SPAWN_RESETIDS) {
setuid(getuid());
@ -59,7 +62,7 @@ int posix_spawn(int *pid, const char *path,
sigfillset(&allsigs);
for (s = 0; sigismember(&allsigs, s); s++) {
if (sigismember(&attrp->posix_attr_sigdefault, s)) {
if (sigaction(s, &dfl, NULL) == -1) _Exit(127);
if (sigaction(s, &dfl, 0) == -1) _Exit(127);
}
}
}
@ -96,15 +99,16 @@ int posix_spawn(int *pid, const char *path,
if (attrp->posix_attr_flags & POSIX_SPAWN_SETSCHEDULER) {
if (sched_setscheduler(0, attrp->posix_attr_schedpolicy,
&attrp->posix_attr_schedparam) == -1) {
_Exit(127);
if (errno != ENOSYS) _Exit(127);
}
}
if (attrp->posix_attr_flags & POSIX_SPAWN_SETSCHEDPARAM) {
if (sched_setparam(0, &attrp->posix_attr_schedparam) == -1) {
_Exit(127);
if (errno != ENOSYS) _Exit(127);
}
}
}
if (!envp) envp = environ;
execve(path, argv, envp);
_Exit(127);
} else {

View file

@ -86,6 +86,7 @@ int setvbuf(FILE *, char *, int, size_t);
int pclose(FILE *);
char *ctermid(char *);
void perror(const char *) relegated;
int makedirs(const char *, unsigned);
typedef uint64_t fpos_t;
char *gets(char *) paramsnonnull();

View file

@ -33,14 +33,8 @@
/**
* Launches program with system command interpreter.
*
* This embeds the cocmd.com shell interpreter which supports a limited
* subset of the bourne shell that's significantly faster:
*
* - pipelines
* - single quotes
* - double quotes
* - input redirection, e.g. `<path`
* - output redirection, e.g. `>path`, `>>append`, `2>err.txt, `2>&1`
* This embeds the Cosmopolitan Command Interpreter which provides
* Bourne-like syntax on all platforms including Windows.
*
* @param cmdline is an interpreted Turing-complete command
* @return -1 if child process couldn't be created, otherwise a wait