Generate ZIP files the same way as Windows

ape/ape.lds

@@ -364,13 +364,18 @@ SECTIONS {
     PROVIDE_HIDDEN(edata = .);
   } :Ram
 
+/*END:   file content that's loaded by o/s */
+/*BEGIN: bss memory void */
+
   .zip . : {
     KEEP(*(SORT_BY_NAME(.zip.*)))
-    . = ALIGN(PAGESIZE);
+    HIDDEN(_ezip = .);
-    HIDDEN(_efile = .);
   }
 
-  .bss . : {
+/*END:   file content */
+/*BEGIN: bss memory that's addressable */
+
+  .bss ALIGN(__SIZEOF_POINTER__) : {
     KEEP(*(SORT_BY_NAME(.piro.bss.init.*)))
     *(.piro.bss)
     KEEP(*(SORT_BY_NAME(.piro.bss.sort.*)))
@@ -540,8 +545,9 @@ HIDDEN(v_ape_highsectors =
 #define ZIPCONST(NAME, VAL) HIDDEN(NAME = DEFINED(__zip_start) ? VAL : 0);
 ZIPCONST(v_zip_cdoffset, __zip_start - IMAGE_BASE_VIRTUAL);
 ZIPCONST(v_zip_cdirsize, __zip_end - __zip_start);
+ASSERT(v_zip_cdirsize % kZipCdirHdrLinkableSize == 0, "bad zip cdir");
 ZIPCONST(v_zip_records, v_zip_cdirsize / kZipCdirHdrLinkableSize);
-ZIPCONST(v_zip_commentsize, _efile - __zip_end - kZipCdirHdrMinSize);
+ZIPCONST(v_zip_commentsize, _ezip - __zip_end - kZipCdirHdrMinSize);
 
 #if SupportsXnu()
 /* Generates deterministic ID. */

libc/nexgen32e/zip.S

@@ -25,19 +25,17 @@
 	.hidden	__zip_start
 	.globl	__zip_start
 	.type	__zip_start,@object
-	.align	kZipCdirAlign
 __zip_start:
 	.previous/*
 	...
 	decentralized content
 	...
 	*/.section .zip.5,"",@progbits
-	.align	kZipCdirAlign
 __zip_end:
 	.long	kZipCdirHdrMagic	# magic
 	.short	0			# disk
 	.short	0			# starting disk
-	.short	v_zip_records		# records on disk
+	.short	v_zip_records		# number of records on disk
 	.short	v_zip_records		# records
 	.long	v_zip_cdirsize		# size of central directory
 	.long	RVA(__zip_start)	# central directory offset

libc/str/zipfindcentraldir.c

@@ -18,16 +18,34 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/zip.h"
 
-uint8_t *zipfindcentraldir(const uint8_t *map, size_t mapsize) {
+/**
-  const uint8_t *p, *pe;
+ * Locates End Of Central Directory record in ZIP file.
-  for (p = map + mapsize - kZipCdirHdrMinSize,
+ *
-      pe = mapsize > 65536 + kZipCdirHdrMinSize
+ * We search backwards for the End Of Central Directory Record magnum.
-               ? map + mapsize - 65536 - kZipCdirHdrMinSize
+ * The ZIP spec says this can be anywhere in the last 64kb. We go all
-               : map;
+ * the way since .com.dbg binaries will have lots of DWARF stuff after
-       p >= pe; --p) {
+ * the embedded .com binary. As such, we sanity check the other fields
-    if (ZIP_CDIR_MAGIC(p) == kZipCdirHdrMagic) {
+ * too to make sure the record seems legit and it's not just a random
-      return (/*unconst*/ uint8_t *)p;
+ * occurrence of the magnum.
+ *
+ * @param p points to file memory
+ * @param n is byte size of file
+ */
+uint8_t *zipfindcentraldir(const uint8_t *p, size_t n) {
+  size_t i;
+  if (n >= kZipCdirHdrMinSize) {
+    i = n - kZipCdirHdrMinSize;
+    do {
+      if (ZIP_CDIR_MAGIC(p + i) == kZipCdirHdrMagic &&
+          i + ZIP_CDIR_HDRSIZE(p + i) <= n &&
+          ZIP_CDIR_DISK(p + i) == ZIP_CDIR_STARTINGDISK(p + i) &&
+          ZIP_CDIR_RECORDSONDISK(p + i) == ZIP_CDIR_RECORDS(p + i) &&
+          ZIP_CDIR_RECORDS(p + i) * kZipCdirHdrMinSize <=
+              ZIP_CDIR_SIZE(p + i) &&
+          ZIP_CDIR_OFFSET(p + i) + ZIP_CDIR_SIZE(p + i) <= i) {
+        return (/*unconst*/ uint8_t *)(p + i);
       }
+    } while (i--);
   }
   return NULL;
 }

libc/zip.h

@@ -10,7 +10,7 @@
 
 #define kZipAlign 2
 
-#define kZipCosmopolitanVersion 1
+#define kZipCosmopolitanVersion 20
 
 #define kZipOsDos         0
 #define kZipOsAmiga       1
@@ -45,7 +45,7 @@
 
 #define kZipCdirHdrMagic   0x06054b50 /* PK♣♠ "PK\5\6" */
 #define kZipCdirHdrMinSize 22
-#define kZipCdirAlign      64 /* our choice; actually 2 */
+#define kZipCdirAlign      kZipAlign
 #define kZipCdirHdrLinkableSize \
   ROUNDUP(kZipCfileHdrMinSize + PATH_MAX, kZipCdirAlign)
 

libc/zipos/find.c

@@ -29,6 +29,7 @@ ssize_t __zipos_find(struct Zipos *zipos, const struct ZiposUri *name) {
   for (i = 0, cf = ZIP_CDIR_OFFSET(zipos->cdir);
        i < ZIP_CDIR_RECORDS(zipos->cdir);
        ++i, cf += ZIP_CFILE_HDRSIZE(zipos->map + cf)) {
+    if (ZIP_CFILE_MAGIC(zipos->map + cf) != kZipCfileHdrMagic) DebugBreak();
     assert(ZIP_CFILE_MAGIC(zipos->map + cf) == kZipCfileHdrMagic);
     if (name->len == ZIP_CFILE_NAMESIZE(zipos->map + cf) &&
         memcmp(name->path, ZIP_CFILE_NAME(zipos->map + cf), name->len) == 0) {

libc/zipos/get.c

@@ -21,6 +21,7 @@
 #include "libc/calls/struct/stat.h"
 #include "libc/limits.h"
 #include "libc/runtime/runtime.h"
+#include "libc/str/str.h"
 #include "libc/sysv/consts/auxv.h"
 #include "libc/sysv/consts/map.h"
 #include "libc/sysv/consts/o.h"
@@ -36,9 +37,8 @@ struct Zipos *__zipos_get(void) {
   static bool once;
   const char *exe, *dir;
   char path[PATH_MAX];
-  size_t mapsize;
+  size_t size;
-  uint8_t *cdir;
+  uint8_t *map, *base, *cdir;
-  void *map;
   if (!once) {
     dir = nulltoempty(getenv("PWD")); /* suboptimal */
     exe = (const char *)getauxval(AT_EXECFN);
@@ -47,14 +47,23 @@ struct Zipos *__zipos_get(void) {
       exe = path;
     }
     if ((zipos.fd = open(exe, O_RDONLY)) != -1) {
-      if ((mapsize = getfiledescriptorsize(zipos.fd)) != SIZE_MAX &&
+      if ((size = getfiledescriptorsize(zipos.fd)) != SIZE_MAX &&
-          (map = mmap(NULL, mapsize, PROT_READ, MAP_SHARED, zipos.fd, 0)) !=
+          (map = mmap(NULL, size, PROT_READ, MAP_SHARED, zipos.fd, 0)) !=
               MAP_FAILED) {
-        if ((cdir = zipfindcentraldir(map, mapsize))) {
+        if (endswith(exe, ".com.dbg")) {
-          zipos.map = map;
+          if ((base = memmem(map, size, "MZqFpD", 6))) {
+            size -= base - map;
+          } else {
+            base = map;
+          }
+        } else {
+          base = map;
+        }
+        if ((cdir = zipfindcentraldir(base, size))) {
+          zipos.map = base;
           zipos.cdir = cdir;
         } else {
-          munmap(map, mapsize);
+          munmap(map, size);
         }
       }
     }

test/libc/release/emulate.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-if [ $MODE = dbg ]; then
+if [ "$MODE" = dbg ]; then
   exit  # TODO
 fi
 

test/libc/release/metal.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-if [ $MODE = dbg ]; then
+if [ "$MODE" = dbg ]; then
   exit  # TODO
 fi
 

test/libc/str/undeflate_test.c

@@ -93,7 +93,6 @@ TEST(undeflate, testEmbeddedCompressedZipFile_theHardWay) {
   ASSERT_GE(ZIP_CDIR_RECORDS(cd), 1);
   for (i = 0, cf = map + ZIP_CDIR_OFFSET(cd); i < ZIP_CDIR_RECORDS(cd);
        ++i, cf += ZIP_CFILE_HDRSIZE(cf)) {
-    fprintf(stderr, "%.*s\n", ZIP_CFILE_NAMESIZE(cf), ZIP_CFILE_NAME(cf));
     if (strncmp("libc/testlib/hyperion.txt", ZIP_CFILE_NAME(cf),
                 ZIP_CFILE_NAMESIZE(cf)) == 0) {
       lf = map + ZIP_CFILE_OFFSET(cf);

tool/build/zipobj.c

@@ -60,6 +60,10 @@
 #define PUT32(P, V)                                                \
   P[0] = V & 0xff, P[1] = V >> 010 & 0xff, P[2] = V >> 020 & 0xff, \
   P[3] = V >> 030 & 0xff, P += 4
+#define PUT64(P, V)                                                       \
+  P[0] = V & 0xff, P[1] = V >> 010 & 0xff, P[2] = V >> 020 & 0xff,        \
+  P[3] = V >> 030 & 0xff, P[4] = V >> 040 & 0xff, P[5] = V >> 050 & 0xff, \
+  P[6] = V >> 060 & 0xff, P[7] = V >> 070 & 0xff, P += 8
 
 #define DOS_DATE(YEAR, MONTH_IDX1, DAY_IDX1) \
   (((YEAR)-1980) << 9 | (MONTH_IDX1) << 5 | (DAY_IDX1))
@@ -149,7 +153,7 @@ static unsigned char *EmitZipLfileHdr(unsigned char *op, const void *name,
                                       uint16_t mdate, size_t compsize,
                                       size_t uncompsize) {
   PUT32(op, kZipLfileHdrMagic);
-  PUT8(op, era);
+  PUT8(op, kZipEra1993);
   PUT8(op, kZipOsDos);
   PUT16(op, gflags);
   PUT16(op, method);
@@ -168,28 +172,60 @@ static void EmitZipCdirHdr(unsigned char *op, const void *name, size_t namesize,
                            uint16_t method, uint16_t mtime, uint16_t mdate,
                            uint16_t iattrs, uint16_t dosmode, uint16_t unixmode,
                            size_t compsize, size_t uncompsize,
-                           size_t commentsize) {
+                           size_t commentsize, struct stat *st) {
+  uint64_t mt, at, ct;
   PUT32(op, kZipCfileHdrMagic);
-  PUT8(op, kZipCosmopolitanVersion);
+  PUT8(op, 20);
-  PUT8(op, kZipOsUnix);
+  PUT8(op, kZipOsDos);
-  PUT8(op, era);
+  PUT8(op, kZipEra1993);
   PUT8(op, kZipOsDos);
   PUT16(op, gflags);
   PUT16(op, method);
   PUT16(op, mtime);
   PUT16(op, mdate);
+  /* 16 */
   PUT32(op, crc);
   PUT32(op, compsize);
   PUT32(op, uncompsize);
   PUT16(op, namesize);
+#if 0
+#define CFILE_HDR_SIZE kZipCfileHdrMinSize
   PUT16(op, 0); /* extra size */
+  /* 32 */
   PUT16(op, commentsize);
   PUT16(op, 0); /* disk */
   PUT16(op, iattrs);
   PUT16(op, dosmode);
   PUT16(op, unixmode);
   PUT32(op, 0); /* RELOCATE ME (kZipCfileOffsetOffset) */
+  /* 46 */
   memcpy(op, name, namesize);
+#else
+#define CFILE_HDR_SIZE (kZipCfileHdrMinSize + 36)
+  PUT16(op, 36); /* extra size */
+  /* 32 */
+  PUT16(op, commentsize);
+  PUT16(op, 0); /* disk */
+  PUT16(op, iattrs);
+  PUT32(op, dosmode);
+  PUT32(op, 0); /* RELOCATE ME (kZipCfileOffsetOffset) */
+  /* 46 */
+  memcpy(op, name, namesize);
+  op += namesize;
+  PUT16(op, kZipExtraNtfs);
+  PUT16(op, 32);
+  PUT32(op, 0);
+  PUT16(op, 1);
+  PUT16(op, 24);
+#define NTTIME(t) \
+  (t.tv_sec + MODERNITYSECONDS) * HECTONANOSECONDS + t.tv_nsec / 100
+  mt = NTTIME(st->st_mtim);
+  at = NTTIME(st->st_atim);
+  ct = NTTIME(st->st_ctim);
+  PUT64(op, mt);
+  PUT64(op, at);
+  PUT64(op, ct);
+#endif
 }
 
 void EmitZip(struct ElfWriter *elf, const char *name, size_t namesize,
@@ -209,7 +245,7 @@ void EmitZip(struct ElfWriter *elf, const char *name, size_t namesize,
   crc = crc32_z(0, data, uncompsize);
   GetDosLocalTime(st->st_mtim.tv_sec, &mtime, &mdate);
   gflags = IsPureAscii(name, namesize) ? 0 : kZipGflagUtf8;
-  commentsize = kZipCdirHdrLinkableSize - (kZipCfileHdrMinSize + namesize);
+  commentsize = kZipCdirHdrLinkableSize - (CFILE_HDR_SIZE + namesize);
   iattrs = IsPureAscii(data, st->st_size) ? kZipIattrAscii : kZipIattrBinary;
   dosmode = !(st->st_mode & 0200) ? kNtFileAttributeReadonly : 0;
   method = (st->st_size >= kMinCompressSize && ShouldCompress(name, namesize))
@@ -217,9 +253,10 @@ void EmitZip(struct ElfWriter *elf, const char *name, size_t namesize,
                : kZipCompressionNone;
 
   /* emit embedded file content w/ pkzip local file header */
-  elfwriter_align(elf, kZipCdirAlign, 0);
+  elfwriter_align(elf, 1, 0);
-  elfwriter_startsection(
+  elfwriter_startsection(elf,
-      elf, gc(xasprintf("%s%s", ZIP_LOCALFILE_SECTION, name)), SHT_PROGBITS, 0);
+                         gc(xasprintf("%s%s", ZIP_LOCALFILE_SECTION, name)),
+                         SHT_PROGBITS, SHF_ALLOC);
   if (method == kZipCompressionDeflate) {
     CHECK_EQ(Z_OK, deflateInit2(memset(&zs, 0, sizeof(zs)),
                                 Z_DEFAULT_COMPRESSION, Z_DEFLATED, -MAX_WBITS,
@@ -255,12 +292,13 @@ void EmitZip(struct ElfWriter *elf, const char *name, size_t namesize,
   elfwriter_finishsection(elf);
 
   /* emit central directory record */
-  elfwriter_align(elf, kZipCdirAlign, 0);
+  elfwriter_align(elf, 1, 0);
-  elfwriter_startsection(
+  elfwriter_startsection(elf,
-      elf, gc(xasprintf("%s%s", ZIP_DIRECTORY_SECTION, name)), SHT_PROGBITS, 0);
+                         gc(xasprintf("%s%s", ZIP_DIRECTORY_SECTION, name)),
+                         SHT_PROGBITS, SHF_ALLOC);
   EmitZipCdirHdr((cfile = elfwriter_reserve(elf, kZipCdirHdrLinkableSize)),
                  name, namesize, crc, era, gflags, method, mtime, mdate, iattrs,
-                 dosmode, st->st_mode, compsize, uncompsize, commentsize);
+                 dosmode, st->st_mode, compsize, uncompsize, commentsize, st);
   elfwriter_appendsym(elf, gc(xasprintf("%s%s", "zip+cdir:", name)),
                       ELF64_ST_INFO(STB_LOCAL, STT_OBJECT), STV_DEFAULT, 0,
                       kZipCdirHdrLinkableSize);