mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-07-02 17:28:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Prove that Makefile is fully defined

Browse source 
The whole repository is now buildable with GNU Make Landlock sandboxing.
This proves that no Makefile targets exist which touch files other than
their declared prerequisites. In order to do this, we had to:

  1. Stop code morphing GCC output in package.com and instead run a
     newly introduced FIXUPOBJ.COM command after GCC invocations.

  2. Disable all the crumby Python unit tests that do things like create
     files in the current directory, or rename() files between folders.
     This ended up being a lot of tests, but most of them are still ok.

  3. Introduce an .UNSANDBOXED variable to GNU Make to disable Landlock.
     We currently only do this for things like `make tags`.

  4. This change deletes some GNU Make code that was preventing the
     execve() optimization from working. This means it should no longer
     be necessary in most cases for command invocations to be indirected
     through the cocmd interpreter.

  5. Missing dependencies had to be declared in certain places, in cases
     where they couldn't be automatically determined by MKDEPS.COM

  6. The libcxx header situation has finally been tamed. One of the
     things that makes this difficult is MKDEPS.COM only wants to
     consider the first 64kb of a file, in order to go fast. But libcxx
     likes to have #include lines buried after huge documentation.

  7. An .UNVEIL variable has been introduced to GNU Make just in case
     we ever wish to explicitly specify additional things that need to
     be whitelisted which aren't strictly prerequisites. This works in
     a manner similar to the recently introduced .EXTRA_PREREQS feature.

There's now a new build/bootstrap/make.com prebuilt binary available. It
should no longer be possible to write invalid Makefile code.
This commit is contained in:
Justine Tunney 2022-08-06 03:51:50 -07:00
parent acdf591833
commit cf93ecbbb2
181 changed files with 1902 additions and 1986 deletions
Download patch file Download diff file Expand all files Collapse all files

271
tool/build/package.c
View file

@ -18,41 +18,23 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/alg/alg.h"
#include "libc/alg/arraylist.internal.h"
#include "libc/alg/bisect.internal.h"
#include "libc/alg/bisectcarleft.internal.h"
#include "libc/assert.h"
#include "libc/bits/bswap.h"
#include "libc/bits/safemacros.internal.h"
#include "libc/calls/calls.h"
#include "libc/calls/struct/stat.h"
#include "libc/dce.h"
#include "libc/elf/def.h"
#include "libc/elf/elf.h"
#include "libc/elf/struct/rela.h"
#include "libc/errno.h"
#include "libc/fmt/conv.h"
#include "libc/elf/struct/shdr.h"
#include "libc/elf/struct/sym.h"
#include "libc/log/check.h"
#include "libc/log/log.h"
#include "libc/macros.internal.h"
#include "libc/mem/mem.h"
#include "libc/nexgen32e/bsr.h"
#include "libc/nexgen32e/kompressor.h"
#include "libc/nt/enum/fileflagandattributes.h"
#include "libc/runtime/gc.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/sock/sock.h"
#include "libc/stdio/stdio.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/msync.h"
#include "libc/sysv/consts/o.h"
#include "libc/sysv/consts/prot.h"
#include "libc/time/time.h"
#include "libc/x/x.h"
#include "third_party/getopt/getopt.h"
#include "third_party/xed/x86.h"
#include "third_party/zlib/zlib.h"
#include "tool/build/lib/elfwriter.h"
#include "tool/build/lib/getargs.h"
#include "tool/build/lib/persist.h"
@ -82,33 +64,6 @@
* These rules help keep the structure of large codebases easy to
* understand. More importantly, it allows us to further optimize
* compiled objects very cheaply as the build progresses.
*
* SECOND PURPOSE
*
* Compress read-only data sections of particularly low entropy, using
* the most appropriate directly-linked algorithm and then inject code
* into _init() that calls it. If the data is extremely low energy, we
* will inject code for merging page table entries too. The overcommit
* here is limitless.
*
* POSSIBLE PURPOSE
*
* It might be nice to have all storage to be thread-local storage. So
* we change RIP-relative instructions to be RBX-relative, only when
* they reference sections in the binary mutable after initialization.
*
* This is basically what the Go language does to implement its fiber
* multiprocessing model. We can have this in C by appropriating all the
* work folks put into enriching GNU C with WIN32 and ASLR lool.
*
* CAVEATS
*
* This tool monkey patches `.o` files as a side-effect since we're not
* able to modify the GCC source code. Therefore it's VERY IMPORTANT to
* have Makefile rules which build `.a` or `.com.dbg` *depend* upon the
* `.pkg` rule. That way they happen in the right order. Otherwise they
* might build binaries with compromised profiling nops at the start of
* functions, which will almost certainly result in SIGILL.
*/
#define PACKAGE_MAGIC bswap_32(0xBEEFBEEFu)
@ -149,15 +104,6 @@ struct Packages {
kPiroBss,
kBss,
} kind;
struct Ops {
size_t i, n;
struct Op {
int32_t offset;
uint8_t length;
uint8_t pos_disp;
uint16_t __pad;
} * p;
} ops;
} * p;
} sections; // not persisted
} * p; // persisted as pkg+RVA
@ -273,7 +219,6 @@ void GetOpts(struct Package *pkg, struct Packages *deps, int argc,
void IndexSections(struct Object *obj) {
size_t i;
struct Op op;
const char *name;
const uint8_t *code;
struct Section sect;
@ -306,23 +251,6 @@ void IndexSections(struct Object *obj) {
} else {
sect.kind = kUndef; /* should always and only be section #0 */
}
if (shdr->sh_flags & SHF_EXECINSTR) {
CHECK_NOTNULL((code = GetElfSectionAddress(obj->elf, obj->size, shdr)));
for (op.offset = 0; op.offset < shdr->sh_size; op.offset += op.length) {
if (xed_instruction_length_decode(
xed_decoded_inst_zero_set_mode(&xedd, XED_MACHINE_MODE_LONG_64),
&code[op.offset],
min(shdr->sh_size - op.offset, XED_MAX_INSTRUCTION_BYTES)) ==
XED_ERROR_NONE) {
op.length = xedd.length;
op.pos_disp = xedd.op.pos_disp;
} else {
op.length = 1;
op.pos_disp = 0;
}
CHECK_NE(-1, append(&sect.ops, &op));
}
}
CHECK_NE(-1, append(&obj->sections, &sect));
}
}
@ -384,9 +312,6 @@ void OpenObject(struct Package *pkg, struct Object *obj, int mode, int prot,
}
void CloseObject(struct Object *obj) {
if ((obj->mode & O_ACCMODE) != O_RDONLY) {
CHECK_NE(-1, msync(obj->elf, obj->size, MS_ASYNC | MS_INVALIDATE));
}
CHECK_NE(-1, munmap(obj->elf, obj->size));
}
@ -475,214 +400,22 @@ forceinline uint8_t ChangeRipToRbx(uint8_t modrm) {
return (modrm & 0b00111000) | 0b10000011;
}
void OptimizeRelocations(struct Package *pkg, struct Packages *deps,
struct Object *obj) {
Elf64_Half i;
struct Op *op;
Elf64_Rela *rela;
struct Symbol *refsym;
struct Package *refpkg;
unsigned char *code, *p;
Elf64_Shdr *shdr, *shdrcode;
for (i = 0; i < obj->elf->e_shnum; ++i) {
shdr = GetElfSectionHeaderAddress(obj->elf, obj->size, i);
if (shdr->sh_type == SHT_RELA) {
CHECK_EQ(sizeof(struct Elf64_Rela), shdr->sh_entsize);
CHECK_NOTNULL((shdrcode = GetElfSectionHeaderAddress(obj->elf, obj->size,
shdr->sh_info)));
if (!(shdrcode->sh_flags & SHF_EXECINSTR)) continue;
CHECK_NOTNULL(
(code = GetElfSectionAddress(obj->elf, obj->size, shdrcode)));
for (rela = GetElfSectionAddress(obj->elf, obj->size, shdr);
((uintptr_t)rela + shdr->sh_entsize <=
min((uintptr_t)obj->elf + obj->size,
(uintptr_t)obj->elf + shdr->sh_offset + shdr->sh_size));
++rela) {
CHECK_LT(ELF64_R_SYM(rela->r_info), obj->symcount);
#if 0
/*
* Change (%rip) to (%rbx) on program instructions that
* reference memory, if and only if the memory location is a
* global variable that's mutable after initialization. The
* displacement is also updated to be relative to the image
* base, rather than relative to the program counter.
*/
if ((ELF64_R_TYPE(rela->r_info) == R_X86_64_PC32 ||
ELF64_R_TYPE(rela->r_info) == R_X86_64_GOTPCREL) &&
FindSymbol(
GetElfString(obj->elf, obj->size, obj->strs,
obj->syms[ELF64_R_SYM(rela->r_info)].st_name),
pkg, deps, &refpkg, &refsym) &&
(refsym->kind == kData || refsym->kind == kBss) &&
IsRipRelativeModrm(code[rela->r_offset - 1])) {
op = &obj->sections.p[shdr->sh_info].ops.p[bisectcarleft(
(const int32_t(*)[2])obj->sections.p[shdr->sh_info].ops.p,
obj->sections.p[shdr->sh_info].ops.i, rela->r_offset)];
CHECK_GT(op->length, 4);
CHECK_GT(op->pos_disp, 0);
rela->r_info = ELF64_R_INFO(ELF64_R_SYM(rela->r_info), R_X86_64_32S);
rela->r_addend = -IMAGE_BASE_VIRTUAL + rela->r_addend +
(op->length - op->pos_disp);
code[rela->r_offset - 1] = ChangeRipToRbx(code[rela->r_offset - 1]);
}
#endif
/*
* GCC isn't capable of -mnop-mcount when using -fpie.
* Let's fix that. It saves ~14 cycles per function call.
* Then libc/runtime/ftrace.greg.c morphs it back at runtime.
*/
if (ELF64_R_TYPE(rela->r_info) == R_X86_64_GOTPCRELX &&
strcmp(GetElfString(obj->elf, obj->size, obj->strs,
obj->syms[ELF64_R_SYM(rela->r_info)].st_name),
"mcount") == 0) {
rela->r_info = R_X86_64_NONE;
p = &code[rela->r_offset - 2];
p[0] = 0x66; /* nopw 0x00(%rax,%rax,1) */
p[1] = 0x0f;
p[2] = 0x1f;
p[3] = 0x44;
p[4] = 0x00;
p[5] = 0x00;
}
/*
* Let's just try to nop mcount calls in general due to the above.
*/
if ((ELF64_R_TYPE(rela->r_info) == R_X86_64_PC32 ||
ELF64_R_TYPE(rela->r_info) == R_X86_64_PLT32) &&
strcmp(GetElfString(obj->elf, obj->size, obj->strs,
obj->syms[ELF64_R_SYM(rela->r_info)].st_name),
"mcount") == 0) {
rela->r_info = R_X86_64_NONE;
p = &code[rela->r_offset - 1];
p[0] = 0x0f; /* nopl 0x00(%rax,%rax,1) */
p[1] = 0x1f;
p[2] = 0x44;
p[3] = 0x00;
p[4] = 0x00;
}
}
}
}
}
bool IsSymbolDirectlyReachable(struct Package *pkg, struct Packages *deps,
const char *symbol) {
return FindSymbol(symbol, pkg, deps, NULL, NULL);
}
struct RlEncoder {
size_t i, n;
struct RlDecode *p;
};
ssize_t rlencode_extend(struct RlEncoder *rle, size_t n) {
size_t n2;
struct RlDecode *p2;
n2 = rle->n;
if (!n2) n2 = 512;
while (n > n2) n2 += n2 >> 1;
if (!(p2 = realloc(rle->p, n2 * sizeof(rle->p[0])))) return -1;
rle->p = p2;
rle->n = n2;
return n2;
}
void rlencode_encode(struct RlEncoder *rle, const unsigned char *data,
size_t size) {
size_t i, j;
for (i = 0; i < size; i += j) {
for (j = 1; j < 255 && i + j < size; ++j) {
if (data[i] != data[i + j]) break;
}
rle->p[rle->i].repititions = j;
rle->p[rle->i].byte = data[i];
rle->i++;
}
rle->p[rle->i].repititions = 0;
rle->p[rle->i].byte = 0;
rle->i++;
}
ssize_t rlencode(struct RlEncoder *rle, const unsigned char *data,
size_t size) {
if (size + 1 > rle->n && rlencode_extend(rle, size + 1) == -1) return -1;
rlencode_encode(rle, data, size);
assert(rle->i <= rle->n);
return rle->i;
}
void CompressLowEntropyReadOnlyDataSections(struct Package *pkg,
struct Packages *deps,
struct Object *obj) {
Elf64_Half i;
const char *name;
unsigned char *p;
Elf64_Shdr *shdr;
struct RlEncoder rle;
bool haverldecode, isprofitable;
bzero(&rle, sizeof(rle));
haverldecode = IsSymbolDirectlyReachable(pkg, deps, "rldecode");
for (i = 0; i < obj->elf->e_shnum; ++i) {
if ((shdr = GetElfSectionHeaderAddress(obj->elf, obj->size, i)) &&
shdr->sh_size >= 256 &&
(shdr->sh_type == SHT_PROGBITS &&
!(shdr->sh_flags &
(SHF_WRITE | SHF_MERGE | SHF_STRINGS | SHF_COMPRESSED))) &&
(p = GetElfSectionAddress(obj->elf, obj->size, shdr)) &&
startswith((name = GetElfSectionName(obj->elf, obj->size, shdr)),
".rodata") &&
rlencode(&rle, p, shdr->sh_size) != -1) {
isprofitable = rle.i * sizeof(rle.p[0]) <= shdr->sh_size / 2;
INFOF("%s(%s): rlencode()%s on %s is%s profitable (%,zu → %,zu bytes)",
&pkg->strings.p[pkg->path], &pkg->strings.p[obj->path],
haverldecode ? "" : " [NOT LINKED]", name,
isprofitable ? "" : " NOT", shdr->sh_size,
rle.i * sizeof(rle.p[0]));
}
}
free(rle.p);
}
void RewriteObjects(struct Package *pkg, struct Packages *deps) {
size_t i;
struct Object *obj;
#if 0
struct ElfWriter *elf;
elf = elfwriter_open(gc(xstrcat(&pkg->strings.p[pkg->path], ".o")), 0644);
elfwriter_cargoculting(elf);
#endif
for (i = 0; i < pkg->objects.i; ++i) {
obj = &pkg->objects.p[i];
OpenObject(pkg, obj, O_RDWR, PROT_READ | PROT_WRITE, MAP_SHARED);
OptimizeRelocations(pkg, deps, obj);
#if 0
CompressLowEntropyReadOnlyDataSections(pkg, deps, obj);
#endif
CloseObject(obj);
}
#if 0
elfwriter_close(elf);
#endif
}
void Package(int argc, char *argv[], struct Package *pkg,
struct Packages *deps) {
size_t i, j;
GetOpts(pkg, deps, argc, argv);
LoadObjects(pkg);
CheckStrictDeps(pkg, deps);
RewriteObjects(pkg, deps);
WritePackage(pkg);
for (i = 0; i < deps->i; ++i) {
CHECK_NE(-1, munmap(deps->p[i]->addr, deps->p[i]->size));
}
for (i = 0; i < pkg->objects.i; ++i) {
for (j = 0; j < pkg->objects.p[i].sections.i; ++j) {
free(pkg->objects.p[i].sections.p[j].ops.p);
}
free(pkg->objects.p[i].sections.p);
}
free(pkg->strings.p);