[metal] Copy program pages to extended memory at startup

ape/ape.S

@@ -1147,10 +1147,10 @@ sconf:	.short	1843200/*hz*/ / 16/*wut*/ / 9600/*baud*/
 
 //	Global Descriptor Table
 	.align	8
-_gdtrphy:
+_gdtrlo:
 	.short	2f-1f-1		# table byte length
-	.long	REAL(1f)	# table address (physical space)
+	.long	REAL(1f)	# table address (base memory space)
-	.endobj	_gdtrphy,global,hidden
+	.endobj	_gdtrlo,global,hidden
 _gdtr:
 	.short	2f-1f-1		# table byte length
 	.quad	1f		# table address (final virtual space)
@@ -1263,6 +1263,7 @@ longmodeloader:
 	call	lcheck
 	call	a20
 	call	e820
+	call	cpyhi
 	call	pinit
 	call	golong
 	.endfn	longmodeloader
@@ -1405,6 +1406,32 @@ a20:	cli
 5:	ret
 	.endfn	a20
 
+//	Copies program pages loaded into base memory, to extended memory.
+cpyhi:	push	%ds
+	push	%es
+	cli
+	lgdt	REAL(_gdtrlo)
+	mov	%cr0,%eax
+	or	$CR0_PE,%al
+	mov	%eax,%cr0
+	jmp	0f
+0:	pushpop	GDT_LEGACY_DATA,%ds
+	push	%ds
+	pop	%es
+	mov	$IMAGE_BASE_REAL,%esi
+	mov	$IMAGE_BASE_PHYSICAL,%edi
+	mov	$v_ape_realdwords,%ecx
+	cld
+	rep movsl %ds:(%esi),%es:(%edi)
+	and	$~CR0_PE,%al
+	mov	%eax,%cr0
+	jmp	1f
+1:	sti
+	pop	%es
+	pop	%ds
+	ret
+	.endfn	cpyhi
+
 //	Initializes long mode paging.
 pinit:	push	%ds
 	push	%es
@@ -1449,7 +1476,7 @@ golong:	cli
 	rdmsr
 	or	$EFER_LME|EFER_SCE|EFER_NXE,%eax
 	wrmsr
-	lgdt	REAL(_gdtrphy)
+	lgdt	REAL(_gdtrlo)
 	mov	%cr0,%eax
 	or	$CR0_PE|CR0_PG|CR0_MP,%eax
 	and	$~CR0_EM,%eax
@@ -1459,7 +1486,9 @@ golong:	cli
 
 //	Long mode is long.
 	.code64
-long:	xor	%eax,%eax
+long:	movabs	$BANE+PHYSICAL(0f),%rax
+	jmp	*%rax
+0:	xor	%eax,%eax
 	mov	$GDT_LONG_DATA,%al
 	mov	%eax,%ds
 	mov	%eax,%ss
@@ -1478,7 +1507,8 @@ long:	xor	%eax,%eax
 	push	%rbp
 	mov	$mm,%rdi
 	mov	%cr3,%rsi
-	mov	$IMAGE_BASE_REAL,%edx
+	mov	$IMAGE_BASE_PHYSICAL,%edx
+	lea	v_ape_realbytes(%rdx),%ecx
 	call	__map_phdrs
 	push	$0x037f
 	fldcw	(%rsp)
@@ -1609,6 +1639,7 @@ kernel:	movabs	$ape_stack_vaddr,%rsp
 	.ldsvar	_end
 	.ldsvar	_etext
 	.ldsvar	v_ape_realsectors
+	.ldsvar	v_ape_realbytes
 	.ldsvar	v_ape_highsectors
 	.ldsvar	ape_idata_ro
 	.ldsvar	ape_pad_rodata

ape/ape.lds

@@ -566,7 +566,8 @@ SHSTUB2(ape_loader_dd_count,
 HIDDEN(v_ape_realsectors =
            MIN(0x70000 - IMAGE_BASE_REAL,
                ROUNDUP(RVA(_tdata_end), 512)) / 512);
-HIDDEN(v_ape_realpages = v_ape_realsectors / (4096 / 512));
+HIDDEN(v_ape_realbytes = v_ape_realsectors * 512);
+HIDDEN(v_ape_realdwords = v_ape_realsectors * (512 / 4));
 HIDDEN(v_ape_highsectors =
            (ROUNDUP(RVA(_tdata_end), 512) / 512) - v_ape_realsectors);
 TSSDESCSTUB2(_tss, _tss, _tss_end ? _tss_end - _tss - 1 : 0);

libc/calls/metalfile.c

@@ -68,7 +68,7 @@ textstartup noasan void InitializeMetalFile(void) {
                         MAP_SHARED_linux | MAP_ANONYMOUS_linux, -1, 0);
     copied_base = dm.addr;
     _npassert(copied_base != (void *)-1);
-    memcpy(copied_base, (void *)(BANE + IMAGE_BASE_REAL), size);
+    memcpy(copied_base, (void *)(BANE + IMAGE_BASE_PHYSICAL), size);
     __ape_com_base = copied_base;
     __ape_com_size = size;
   }

libc/intrin/mman.greg.c

@@ -42,6 +42,8 @@
 #include "libc/runtime/pc.internal.h"
 #include "libc/runtime/runtime.h"
 
+#define INVERT(x) (BANE + PHYSICAL(x))
+
 /**
  * Allocates new page of physical memory.
  */
@@ -53,7 +55,7 @@ noasan texthead uint64_t __new_page(struct mman *mm) {
   }
   while (mm->pdp >= mm->e820[mm->pdpi].addr + mm->e820[mm->pdpi].size) {
     if (++mm->pdpi == mm->e820n) return 0;
-    mm->pdp = mm->e820[mm->pdpi].addr;
+    mm->pdp = MAX(mm->pdp, mm->e820[mm->pdpi].addr);
   }
   p = mm->pdp;
   mm->pdp += 4096;
@@ -84,7 +86,7 @@ noasan textreal uint64_t *__get_virtual(struct mman *mm, uint64_t *t,
 /**
  * Sorts, rounds, and filters BIOS memory map.
  */
-static noasan textreal void __normalize_e820(struct mman *mm) {
+static noasan textreal void __normalize_e820(struct mman *mm, uint64_t top) {
   uint64_t a, b;
   uint64_t x, y;
   unsigned i, j, n;
@@ -107,7 +109,8 @@ static noasan textreal void __normalize_e820(struct mman *mm) {
     }
     mm->e820[j] = mm->e820[i];
   }
-  mm->pdp = MAX(0x80000, mm->e820[0].addr);
+  top = ROUNDUP(top, 4096);
+  mm->pdp = MAX(top, mm->e820[0].addr);
   mm->pdpi = 0;
   mm->e820n = n;
 }
@@ -155,7 +158,8 @@ static noasan textreal void __invert_memory(struct mman *mm, uint64_t *pml4t) {
                  : "i"(offsetof(type, member)));         \
   } while (0)
 
-noasan textreal void __setup_mman(struct mman *mm, uint64_t *pml4t) {
+noasan textreal void __setup_mman(struct mman *mm, uint64_t *pml4t,
+                                  uint64_t top) {
   export_offsetof(struct mman, pc_drive_base_table);
   export_offsetof(struct mman, pc_drive_last_sector);
   export_offsetof(struct mman, pc_drive_last_head);
@@ -170,21 +174,22 @@ noasan textreal void __setup_mman(struct mman *mm, uint64_t *pml4t) {
   export_offsetof(struct mman, pc_video_framebuffer_size);
   export_offsetof(struct mman, pc_video_curs_info);
   export_offsetof(struct mman, pc_video_char_height);
-  __normalize_e820(mm);
+  __normalize_e820(mm, top);
   __invert_memory(mm, pml4t);
 }
 
 /**
  * Maps APE-defined ELF program headers into memory and clears BSS.
  */
-noasan textreal void __map_phdrs(struct mman *mm, uint64_t *pml4t, uint64_t b) {
+noasan textreal void __map_phdrs(struct mman *mm, uint64_t *pml4t, uint64_t b,
+                                 uint64_t top) {
   struct Elf64_Phdr *p;
   uint64_t i, f, v, m, *e;
   extern char ape_phdrs[] __attribute__((__weak__));
   extern char ape_phdrs_end[] __attribute__((__weak__));
-  __setup_mman(mm, pml4t);
+  __setup_mman(mm, pml4t, top);
-  for (p = (struct Elf64_Phdr *)REAL(ape_phdrs), m = 0;
+  for (p = (struct Elf64_Phdr *)INVERT(ape_phdrs), m = 0;
-       p < (struct Elf64_Phdr *)REAL(ape_phdrs_end); ++p) {
+       p < (struct Elf64_Phdr *)INVERT(ape_phdrs_end); ++p) {
     if (p->p_type == PT_LOAD || p->p_type == PT_GNU_STACK) {
       f = PAGE_RSRV | PAGE_U;
       if (p->p_flags & PF_W)

libc/runtime/efimain.greg.c

@@ -135,7 +135,7 @@ __msabi noasan EFI_STATUS EfiMain(EFI_HANDLE ImageHandle,
   pdpt2[0] = (intptr_t)pdt2 + PAGE_V + PAGE_RW;
   pml4t[0] = (intptr_t)pdpt1 + PAGE_V + PAGE_RW;
   pml4t[256] = (intptr_t)pdpt2 + PAGE_V + PAGE_RW;
-  __map_phdrs(mm, pml4t, 1024 * 1024);
+  __map_phdrs(mm, pml4t, 1024 * 1024, 1024 * 1024 + (_end - _base));
 
   /*
    * Asks UEFI to handover control?

libc/runtime/pc.internal.h

@@ -188,7 +188,7 @@ uint64_t __clear_page(uint64_t);
 uint64_t __new_page(struct mman *);
 void __invert_memory_area(struct mman *, uint64_t *, uint64_t, uint64_t,
                           uint64_t);
-void __map_phdrs(struct mman *, uint64_t *, uint64_t);
+void __map_phdrs(struct mman *, uint64_t *, uint64_t, uint64_t);
 
 forceinline unsigned char inb(unsigned short port) {
   unsigned char al;