Set errno when out of range in strtoimax (#111)

libc/fmt/strtoimax.c

@@ -17,6 +17,7 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/fmt/conv.h"
+#include "libc/errno.h"
 #include "libc/limits.h"
 #include "libc/nexgen32e/bsr.h"
 #include "libc/str/str.h"
@@ -85,7 +86,7 @@ intmax_t strtoimax(const char *s, char **endptr, int base) {
     diglet = kBase36[*s & 0xff];
     if (!diglet || diglet > base) break;
     diglet -= 1;
-    if (!diglet || !x || (bits = bsr(diglet) + bsrmax(x)) < 127) {
+    if (!x || (bits = (diglet ? bsr(diglet) : 0) + bsrmax(x * base)) < 127) {
       s++;
       x *= base;
       x += diglet;
@@ -96,9 +97,11 @@ intmax_t strtoimax(const char *s, char **endptr, int base) {
         if (x == INTMAX_MIN) s++;
       }
       x = INTMAX_MIN;
+      errno = ERANGE;
       break;
     } else {
       x = INTMAX_MAX;
+      errno = ERANGE;
       break;
     }
   }

test/libc/fmt/strtoimax_test.c

@@ -53,9 +53,17 @@ TEST(strtoimax, testLimits) {
       strtoimax("0x7fffffffffffffffffffffffffffffff", NULL, 0));
 }
 
-TEST(strtoimax, testTwosBane) {
-  EXPECT_EQ(((uintmax_t)0x8000000000000000) << 64 | 0x0000000000000000,
-            strtoimax("0x80000000000000000000000000000000", NULL, 0));
+TEST(strtoimax, testOutsideLimit) {
+  errno = 0;
+  EXPECT_EQ(
+      ((uintmax_t)0x7fffffffffffffff) << 64 | (uintmax_t)0xffffffffffffffff,
+      strtoimax("0x80000000000000000000000000000000", NULL, 0));
+  EXPECT_EQ(ERANGE, errno);
+  errno = 0;
+  EXPECT_EQ(
+      ((uintmax_t)0x8000000000000000) << 64 | 0x0000000000000000,
+      strtoimax("-0x80000000000000000000000000000001", NULL, 0));
+  EXPECT_EQ(ERANGE, errno);
 }
 
 TEST(strtoul, neghex) {

test/libc/fmt/strtoumax_test.c

@@ -46,3 +46,8 @@ TEST(strtoumax, testMaximum) {
   EXPECT_EQ(UINTMAX_MAX,
             strtoumax("0xffffffffffffffffffffffffffffffff", NULL, 0));
 }
+
+TEST(strtoumax, testTwosBane) {
+  EXPECT_EQ(((uintmax_t)0x8000000000000000) << 64 | 0x0000000000000000,
+            strtoumax("0x80000000000000000000000000000000", NULL, 0));
+}

tool/build/calculator.c

@@ -443,7 +443,7 @@ bool ConsumeLiteral(const char *literal) {
   char *e;
   struct Value x;
   x.t = kInt;
-  x.i = strtoimax(literal, &e, 0);
+  x.i = *literal == '-' ? strtoimax(literal, &e, 0) : strtoumax(literal, &e, 0);
   if (!e || *e) {
     x.t = kFloat;
     x.f = strtod(literal, &e);

tool/build/calculator.ctest

@@ -81,3 +81,11 @@ false false || ! assert
 1 -1 min -1 = assert
 1 2 min 1 = assert
 rand64 rand64 rand64 rand64 != != && assert
+
+# HEX SIGN
+ -0x80000000 -2147483648 = assert
+  0x80000000  2147483648 = assert
+  0x80000001  2147483649 = assert
+  0xffffffff  4294967295 = assert
+ 0x100000000  4294967296 = assert
+-0x100000000 -4294967296 = assert