diff --git a/ape/ape.S b/ape/ape.S index 412a03626..81bcff31e 100644 --- a/ape/ape.S +++ b/ape/ape.S @@ -1147,10 +1147,10 @@ sconf: .short 1843200/*hz*/ / 16/*wut*/ / 9600/*baud*/ // Global Descriptor Table .align 8 -_gdtrphy: +_gdtrlo: .short 2f-1f-1 # table byte length - .long REAL(1f) # table address (physical space) - .endobj _gdtrphy,global,hidden + .long REAL(1f) # table address (base memory space) + .endobj _gdtrlo,global,hidden _gdtr: .short 2f-1f-1 # table byte length .quad 1f # table address (final virtual space) @@ -1263,6 +1263,7 @@ longmodeloader: call lcheck call a20 call e820 + call cpyhi call pinit call golong .endfn longmodeloader @@ -1405,6 +1406,32 @@ a20: cli 5: ret .endfn a20 +// Copies program pages loaded into base memory, to extended memory. +cpyhi: push %ds + push %es + cli + lgdt REAL(_gdtrlo) + mov %cr0,%eax + or $CR0_PE,%al + mov %eax,%cr0 + jmp 0f +0: pushpop GDT_LEGACY_DATA,%ds + push %ds + pop %es + mov $IMAGE_BASE_REAL,%esi + mov $IMAGE_BASE_PHYSICAL,%edi + mov $v_ape_realdwords,%ecx + cld + rep movsl %ds:(%esi),%es:(%edi) + and $~CR0_PE,%al + mov %eax,%cr0 + jmp 1f +1: sti + pop %es + pop %ds + ret + .endfn cpyhi + // Initializes long mode paging. pinit: push %ds push %es @@ -1449,7 +1476,7 @@ golong: cli rdmsr or $EFER_LME|EFER_SCE|EFER_NXE,%eax wrmsr - lgdt REAL(_gdtrphy) + lgdt REAL(_gdtrlo) mov %cr0,%eax or $CR0_PE|CR0_PG|CR0_MP,%eax and $~CR0_EM,%eax @@ -1459,7 +1486,9 @@ golong: cli // Long mode is long. .code64 -long: xor %eax,%eax +long: movabs $BANE+PHYSICAL(0f),%rax + jmp *%rax +0: xor %eax,%eax mov $GDT_LONG_DATA,%al mov %eax,%ds mov %eax,%ss @@ -1478,7 +1507,8 @@ long: xor %eax,%eax push %rbp mov $mm,%rdi mov %cr3,%rsi - mov $IMAGE_BASE_REAL,%edx + mov $IMAGE_BASE_PHYSICAL,%edx + lea v_ape_realbytes(%rdx),%ecx call __map_phdrs push $0x037f fldcw (%rsp) @@ -1609,6 +1639,7 @@ kernel: movabs $ape_stack_vaddr,%rsp .ldsvar _end .ldsvar _etext .ldsvar v_ape_realsectors + .ldsvar v_ape_realbytes .ldsvar v_ape_highsectors .ldsvar ape_idata_ro .ldsvar ape_pad_rodata diff --git a/ape/ape.lds b/ape/ape.lds index 15db9913e..6c6109805 100644 --- a/ape/ape.lds +++ b/ape/ape.lds @@ -566,7 +566,8 @@ SHSTUB2(ape_loader_dd_count, HIDDEN(v_ape_realsectors = MIN(0x70000 - IMAGE_BASE_REAL, ROUNDUP(RVA(_tdata_end), 512)) / 512); -HIDDEN(v_ape_realpages = v_ape_realsectors / (4096 / 512)); +HIDDEN(v_ape_realbytes = v_ape_realsectors * 512); +HIDDEN(v_ape_realdwords = v_ape_realsectors * (512 / 4)); HIDDEN(v_ape_highsectors = (ROUNDUP(RVA(_tdata_end), 512) / 512) - v_ape_realsectors); TSSDESCSTUB2(_tss, _tss, _tss_end ? _tss_end - _tss - 1 : 0); diff --git a/libc/calls/metalfile.c b/libc/calls/metalfile.c index efa9292db..cc9b5dc56 100644 --- a/libc/calls/metalfile.c +++ b/libc/calls/metalfile.c @@ -68,7 +68,7 @@ textstartup noasan void InitializeMetalFile(void) { MAP_SHARED_linux | MAP_ANONYMOUS_linux, -1, 0); copied_base = dm.addr; _npassert(copied_base != (void *)-1); - memcpy(copied_base, (void *)(BANE + IMAGE_BASE_REAL), size); + memcpy(copied_base, (void *)(BANE + IMAGE_BASE_PHYSICAL), size); __ape_com_base = copied_base; __ape_com_size = size; } diff --git a/libc/intrin/mman.greg.c b/libc/intrin/mman.greg.c index 88574445a..642418409 100644 --- a/libc/intrin/mman.greg.c +++ b/libc/intrin/mman.greg.c @@ -42,6 +42,8 @@ #include "libc/runtime/pc.internal.h" #include "libc/runtime/runtime.h" +#define INVERT(x) (BANE + PHYSICAL(x)) + /** * Allocates new page of physical memory. */ @@ -53,7 +55,7 @@ noasan texthead uint64_t __new_page(struct mman *mm) { } while (mm->pdp >= mm->e820[mm->pdpi].addr + mm->e820[mm->pdpi].size) { if (++mm->pdpi == mm->e820n) return 0; - mm->pdp = mm->e820[mm->pdpi].addr; + mm->pdp = MAX(mm->pdp, mm->e820[mm->pdpi].addr); } p = mm->pdp; mm->pdp += 4096; @@ -84,7 +86,7 @@ noasan textreal uint64_t *__get_virtual(struct mman *mm, uint64_t *t, /** * Sorts, rounds, and filters BIOS memory map. */ -static noasan textreal void __normalize_e820(struct mman *mm) { +static noasan textreal void __normalize_e820(struct mman *mm, uint64_t top) { uint64_t a, b; uint64_t x, y; unsigned i, j, n; @@ -107,7 +109,8 @@ static noasan textreal void __normalize_e820(struct mman *mm) { } mm->e820[j] = mm->e820[i]; } - mm->pdp = MAX(0x80000, mm->e820[0].addr); + top = ROUNDUP(top, 4096); + mm->pdp = MAX(top, mm->e820[0].addr); mm->pdpi = 0; mm->e820n = n; } @@ -155,7 +158,8 @@ static noasan textreal void __invert_memory(struct mman *mm, uint64_t *pml4t) { : "i"(offsetof(type, member))); \ } while (0) -noasan textreal void __setup_mman(struct mman *mm, uint64_t *pml4t) { +noasan textreal void __setup_mman(struct mman *mm, uint64_t *pml4t, + uint64_t top) { export_offsetof(struct mman, pc_drive_base_table); export_offsetof(struct mman, pc_drive_last_sector); export_offsetof(struct mman, pc_drive_last_head); @@ -170,21 +174,22 @@ noasan textreal void __setup_mman(struct mman *mm, uint64_t *pml4t) { export_offsetof(struct mman, pc_video_framebuffer_size); export_offsetof(struct mman, pc_video_curs_info); export_offsetof(struct mman, pc_video_char_height); - __normalize_e820(mm); + __normalize_e820(mm, top); __invert_memory(mm, pml4t); } /** * Maps APE-defined ELF program headers into memory and clears BSS. */ -noasan textreal void __map_phdrs(struct mman *mm, uint64_t *pml4t, uint64_t b) { +noasan textreal void __map_phdrs(struct mman *mm, uint64_t *pml4t, uint64_t b, + uint64_t top) { struct Elf64_Phdr *p; uint64_t i, f, v, m, *e; extern char ape_phdrs[] __attribute__((__weak__)); extern char ape_phdrs_end[] __attribute__((__weak__)); - __setup_mman(mm, pml4t); - for (p = (struct Elf64_Phdr *)REAL(ape_phdrs), m = 0; - p < (struct Elf64_Phdr *)REAL(ape_phdrs_end); ++p) { + __setup_mman(mm, pml4t, top); + for (p = (struct Elf64_Phdr *)INVERT(ape_phdrs), m = 0; + p < (struct Elf64_Phdr *)INVERT(ape_phdrs_end); ++p) { if (p->p_type == PT_LOAD || p->p_type == PT_GNU_STACK) { f = PAGE_RSRV | PAGE_U; if (p->p_flags & PF_W) diff --git a/libc/runtime/efimain.greg.c b/libc/runtime/efimain.greg.c index b64b21d0b..5601c896e 100644 --- a/libc/runtime/efimain.greg.c +++ b/libc/runtime/efimain.greg.c @@ -135,7 +135,7 @@ __msabi noasan EFI_STATUS EfiMain(EFI_HANDLE ImageHandle, pdpt2[0] = (intptr_t)pdt2 + PAGE_V + PAGE_RW; pml4t[0] = (intptr_t)pdpt1 + PAGE_V + PAGE_RW; pml4t[256] = (intptr_t)pdpt2 + PAGE_V + PAGE_RW; - __map_phdrs(mm, pml4t, 1024 * 1024); + __map_phdrs(mm, pml4t, 1024 * 1024, 1024 * 1024 + (_end - _base)); /* * Asks UEFI to handover control? diff --git a/libc/runtime/pc.internal.h b/libc/runtime/pc.internal.h index 24c552c3a..64904dcba 100644 --- a/libc/runtime/pc.internal.h +++ b/libc/runtime/pc.internal.h @@ -188,7 +188,7 @@ uint64_t __clear_page(uint64_t); uint64_t __new_page(struct mman *); void __invert_memory_area(struct mman *, uint64_t *, uint64_t, uint64_t, uint64_t); -void __map_phdrs(struct mman *, uint64_t *, uint64_t); +void __map_phdrs(struct mman *, uint64_t *, uint64_t, uint64_t); forceinline unsigned char inb(unsigned short port) { unsigned char al;