Fix fread() with 2gb+ sizes

libc/calls/struct/iovec.internal.h

@@ -5,7 +5,6 @@
 #include "libc/mem/alloca.h"
 COSMOPOLITAN_C_START_
 
-ssize_t __robust_writev(int, struct iovec *, int);
 int64_t sys_preadv(int, struct iovec *, int, int64_t, int64_t);
 int64_t sys_pwritev(int, const struct iovec *, int, int64_t, int64_t);
 int64_t sys_readv(int32_t, const struct iovec *, int32_t);

libc/calls/writevuninterruptible.c

@@ -1,45 +0,0 @@
-/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
-│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
-╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2021 Justine Alexandra Roberts Tunney                              │
-│                                                                              │
-│ Permission to use, copy, modify, and/or distribute this software for         │
-│ any purpose with or without fee is hereby granted, provided that the         │
-│ above copyright notice and this permission notice appear in all copies.      │
-│                                                                              │
-│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
-│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
-│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
-│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
-│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
-│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
-│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
-│ PERFORMANCE OF THIS SOFTWARE.                                                │
-╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/calls/struct/iovec.h"
-#include "libc/calls/struct/iovec.internal.h"
-#include "libc/errno.h"
-
-ssize_t __robust_writev(int fd, struct iovec *iov, int iovlen) {
-  ssize_t rc;
-  size_t wrote;
-  do {
-    if ((rc = writev(fd, iov, iovlen)) != -1) {
-      wrote = rc;
-      do {
-        if (wrote >= iov->iov_len) {
-          wrote -= iov->iov_len;
-          ++iov;
-          --iovlen;
-        } else {
-          iov->iov_base = (char *)iov->iov_base + wrote;
-          iov->iov_len -= wrote;
-          wrote = 0;
-        }
-      } while (wrote);
-    } else if (errno != EINTR) {
-      return -1;
-    }
-  } while (iovlen);
-  return 0;
-}

libc/stdio/fread_unlocked.c

@@ -16,6 +16,7 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/assert.h"
 #include "libc/calls/calls.h"
 #include "libc/calls/struct/iovec.h"
 #include "libc/errno.h"
@@ -25,6 +26,41 @@
 #include "libc/stdio/stdio.h"
 #include "libc/str/str.h"
 #include "libc/sysv/consts/o.h"
+#include "libc/sysv/errfuns.h"
+
+static ssize_t readvall(int fd, struct iovec *iov, int iovlen) {
+  int olde;
+  ssize_t rc;
+  size_t got, toto;
+  toto = 0;
+  olde = errno;
+  do {
+    if ((rc = readv(fd, iov, iovlen)) == -1) {
+      if (toto && errno == EINTR) {
+        errno = olde;
+        continue;
+      }
+      return -1;
+    }
+    got = rc;
+    toto += got;
+    for (;;) {
+      if (!iov->iov_len) {
+        --iovlen;
+        ++iov;
+      } else if (got >= iov->iov_len) {
+        got -= iov->iov_len;
+        --iovlen;
+        ++iov;
+      } else {
+        iov->iov_base += got;
+        iov->iov_len -= got;
+        break;
+      }
+    }
+  } while (got && iovlen);
+  return toto;
+}
 
 /**
  * Reads data from stream.
@@ -36,11 +72,10 @@
 size_t fread_unlocked(void *buf, size_t stride, size_t count, FILE *f) {
   char *p;
   ssize_t rc;
-  size_t n, m;
   struct iovec iov[2];
-  if (!stride) {
+  size_t n, m, got, need;
-    return 0;
+
-  }
+  // check state and parameters
   if ((f->iomode & O_ACCMODE) == O_WRONLY) {
     f->state = errno = EBADF;
     return 0;
@@ -53,52 +88,74 @@ size_t fread_unlocked(void *buf, size_t stride, size_t count, FILE *f) {
     f->state = errno = EOVERFLOW;
     return 0;
   }
+  if (!n)
+    return 0;
+
+  // try to fulfill request from buffer if possible
   p = buf;
   m = f->end - f->beg;
-  if (MIN(n, m)) {
+  if (n <= m) {
-    memcpy(p, f->buf + f->beg, MIN(n, m));
+    memcpy(p, f->buf + f->beg, n);
-  }
+    if ((f->beg += n) == f->end) {
-  if (n < m) {
+      f->beg = 0;
-    f->beg += n;
+      f->end = 0;
-    return count;
+    }
-  }
-  if (n == m) {
-    f->beg = f->end = 0;
     return count;
   }
+
+  // handle end-of-file condition in fileless mode
   if (f->fd == -1) {
-    f->beg = 0;
+    m /= stride;
-    f->end = 0;
+    m *= stride;
-    f->state = -1;
+    if (m)
+      memcpy(p, f->buf + f->beg, m);
+    if ((f->beg += m) == f->end) {
+      f->state = EOF;
+      f->beg = 0;
+      f->end = 0;
+    }
     return m / stride;
   }
+
+  // `n` is number of bytes requested by caller
+  // `m` is how much of `n` came from existing buffer
+  // `iov[0]` reads remainder of the caller request
+  // `iov[1]` reads ahead extra content into buffer
+  if (m)
+    memcpy(p, f->buf + f->beg, m);
   iov[0].iov_base = p + m;
-  iov[0].iov_len = n - m;
+  iov[0].iov_len = need = n - m;
   if (f->bufmode != _IONBF && n < f->size) {
     iov[1].iov_base = f->buf;
-    if (f->size > PUSHBACK) {
+    if (f->size > PUSHBACK)
       iov[1].iov_len = f->size - PUSHBACK;
-    } else {
+    else
       iov[1].iov_len = f->size;
-    }
   } else {
     iov[1].iov_base = NULL;
     iov[1].iov_len = 0;
   }
-  if ((rc = readv(f->fd, iov, 2)) == -1) {
+  if ((rc = readvall(f->fd, iov, 2)) == -1) {
     f->state = errno;
     return 0;
   }
-  n = rc;
+  got = rc;
-  f->beg = 0;
+
-  f->end = 0;
+  // handle partial fulfillment
-  if (n > iov[0].iov_len) {
+  if (got < need) {
-    f->end += n - iov[0].iov_len;
+    got += m;
-    return count;
+    if (got % stride) {
-  } else {
+      f->state = eio();
-    n = (m + n) / stride;
+      return 0;
-    if (n < count)
+    }
-      f->state = -1;
+    f->beg = 0;
-    return n;
+    f->end = 0;
+    f->state = EOF;
+    return got / stride;
   }
+
+  // handle overfulfillment
+  f->beg = 0;
+  f->end = got - need;
+  return count;
 }

libc/stdio/fwrite_unlocked.c

@@ -27,6 +27,40 @@
 #include "libc/str/str.h"
 #include "libc/sysv/consts/o.h"
 
+static ssize_t writevall(int fd, struct iovec *iov, int iovlen) {
+  int olde;
+  ssize_t rc;
+  size_t got, toto;
+  toto = 0;
+  olde = errno;
+  do {
+    if ((rc = writev(fd, iov, iovlen)) == -1) {
+      if (toto && errno == EINTR) {
+        errno = olde;
+        continue;
+      }
+      return -1;
+    }
+    got = rc;
+    toto += got;
+    for (;;) {
+      if (!iov->iov_len) {
+        --iovlen;
+        ++iov;
+      } else if (got >= iov->iov_len) {
+        got -= iov->iov_len;
+        --iovlen;
+        ++iov;
+      } else {
+        iov->iov_base += got;
+        iov->iov_len -= got;
+        break;
+      }
+    }
+  } while (got && iovlen);
+  return toto;
+}
+
 /**
  * Writes data to stream.
  *
@@ -104,7 +138,7 @@ size_t fwrite_unlocked(const void *data, size_t stride, size_t count, FILE *f) {
   iov[1].iov_base = (void *)data;
   iov[1].iov_len = n;
   n += f->beg;
-  if (__robust_writev(f->fd, iov, 2) == -1) {
+  if ((rc = writevall(f->fd, iov, 2)) == -1) {
     f->state = errno;
     return 0;
   }

libc/stdio/vdprintf.c

@@ -16,7 +16,7 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/calls/struct/iovec.internal.h"
+#include "libc/calls/struct/iovec.h"
 #include "libc/dce.h"
 #include "libc/fmt/internal.h"
 #include "libc/limits.h"
@@ -42,7 +42,7 @@ static int vdprintf_putc(const char *s, struct VdprintfState *t, size_t n) {
       iov[0].iov_len = t->n;
       iov[1].iov_base = (void *)s;
       iov[1].iov_len = n;
-      if (__robust_writev(t->fd, iov, 2) == -1) {
+      if (writev(t->fd, iov, 2) == -1) {
         return -1;
       }
       t->t += t->n;
@@ -68,7 +68,7 @@ int vdprintf(int fd, const char *fmt, va_list va) {
   if (t.n) {
     iov[0].iov_base = t.b;
     iov[0].iov_len = t.n;
-    if (__robust_writev(t.fd, iov, 1) == -1) {
+    if (writev(t.fd, iov, 1) == -1) {
       return -1;
     }
     t.t += t.n;

test/libc/stdio/freopen_test.c

@@ -16,53 +16,67 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/stdio/stdio.h"
+#include <limits.h>
-#include "libc/testlib/testlib.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
 
 /*
  * This test was contributed by @ahgamut
  * https://github.com/jart/cosmopolitan/issues/61#issuecomment-792214575
  */
 
-void SetUpOnce(void) {
+char path[PATH_MAX];
-  testlib_enable_tmp_setup_teardown();
+
+void teardown(void) {
+  unlink(path);
 }
 
-int writefile(const char* filename) {
+int setup(void) {
-  int stat = 0;
+  int fd;
-  FILE* fp = fopen(filename, "w");
+  const char* tmpdir;
-  stat = fputs("cosmopolitan libc\n", fp);
+  if ((tmpdir = getenv("TMPDIR")))
-  fclose(fp);
+    strlcpy(path, tmpdir, sizeof(path));
-  return stat;
+  else
-}
+    strlcpy(path, "/tmp", sizeof(path));
-
+  strlcat(path, "/freopen_test.XXXXXX", sizeof(path));
-int readfile(const char* filename) {
+  if ((fd = mkstemp(path)) == -1)
-  int stat = 0;
-  char buf1[30];
-  char buf2[30];
-  FILE *fp1, *fp2;
-  fp1 = fopen(filename, "r");
-  if (!fp1) {
-    printf("failed to read %s in r\n", filename);
     return 1;
-  }
+  if (write(fd, "cosmopolitan libc\n", 18) != 18)
-  buf1[0] = fgetc(fp1);
+    return 2;
-  buf1[1] = '\0';
+  if (close(fd))
-  fp2 = freopen(filename, "rb", fp1);
+    return 3;
-  if (!fp2) {
-    printf("failed to read %s in rb\n", filename);
-    return 1;
-  }
-  stat = fread(buf2, sizeof(buf2[0]), 20, fp2);
-  ASSERT_EQ(18, stat);
-  buf2[stat] = '\0';
-  fclose(fp2);
-  ASSERT_STREQ("c", buf1);
-  ASSERT_STREQ("cosmopolitan libc\n", buf2);
   return 0;
 }
 
-TEST(freopen, test) {
+int test(void) {
-  writefile("file.txt");
+  FILE* fp;
-  readfile("file.txt");
+  char buf[20];
+  if (!(fp = fopen(path, "r")))
+    return 4;
+  if (fgetc(fp) != 'c')
+    return 5;
+  if (!(fp = freopen(path, "rb", fp)))
+    return 6;
+  if (fread(buf, 1, 20, fp) != 18)
+    return 7;
+  if (memcmp(buf, "cosmopolitan libc\n", 18))
+    return 8;
+  if (fclose(fp))
+    return 9;
+  return 0;
+}
+
+int main(int argc, char* argv[]) {
+  int rc;
+  if ((rc = setup())) {
+    perror(path);
+    teardown();
+    return rc;
+  }
+  rc = test();
+  teardown();
+  return rc;
 }

test/posix/BUILD.mk

@@ -52,6 +52,9 @@ o/$(MODE)/test/posix/%.dbg:			\
 		$(APE_NO_MODIFY_SELF)
 	@$(APELINK)
 
+o/$(MODE)/test/posix/fread3gb_test.runs:	\
+		private QUOTA += -F5gb -M5gb
+
 .PHONY: o/$(MODE)/test/posix
 o/$(MODE)/test/posix:				\
 		$(TEST_POSIX_BINS)		\

test/posix/fread3gb_test.c

@@ -0,0 +1,73 @@
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#define SIZE ((size_t)2 * 1024 * 1024 * 1024 + 13)
+
+char path[PATH_MAX];
+
+void teardown(void) {
+  unlink(path);
+}
+
+int setup(void) {
+  int fd;
+  struct stat st;
+  const char *tmpdir;
+  if (!stat("/dev/shm", &st))
+    strlcpy(path, "/dev/shm", sizeof(path));
+  else if ((tmpdir = getenv("TMPDIR")))
+    strlcpy(path, tmpdir, sizeof(path));
+  else
+    strlcpy(path, "/tmp", sizeof(path));
+  strlcat(path, "/fread3gb.XXXXXX", sizeof(path));
+  if ((fd = mkstemp(path)) == -1)
+    return 1;
+  if (ftruncate(fd, SIZE))
+    return 2;
+  if (pwrite(fd, "a", 1, 0) != 1)
+    return 3;
+  if (pwrite(fd, "z", 1, SIZE - 1) != 1)
+    return 4;
+  if (close(fd))
+    return 5;
+  return 0;
+}
+
+int test(void) {
+  FILE *f;
+  char *buf;
+  size_t rc;
+  if (!(f = fopen(path, "r")))
+    return 6;
+  if (!(buf = malloc(SIZE)))
+    return 7;
+  if ((rc = fread(buf, SIZE, 1, f)) != 1) {
+    fprintf(stderr, "tell = %zu\n", ftello(f));
+    fprintf(stderr, "rc = %zu\n", rc);
+    perror("fread");
+    return 8;
+  }
+  if (buf[0] != 'a')
+    return 9;
+  if (buf[SIZE - 1] != 'z')
+    return 10;
+  if (fclose(f))
+    return 11;
+  return 0;
+}
+
+int main(int argc, char *argv[]) {
+  int rc;
+  if ((rc = setup())) {
+    perror(path);
+    teardown();
+    return rc;
+  }
+  rc = test();
+  teardown();
+  return rc;
+}