Detect more x86 features

libc/intrin/x86.c

@@ -381,6 +381,13 @@ static const char *getAMDProcessorTypeAndSubtype(unsigned Family,
   const char *CPU = 0;
 
   switch (Family) {
+    case 15:
+      if (testFeature(FEATURE_SSE3)) {
+        CPU = "k8-sse3";
+        break;
+      }
+      CPU = "k8";
+      break;
     case 16:
       CPU = "amdfam10";
       *Type = AMDFAM10H;
@@ -535,13 +542,15 @@ static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf,
     setFeature(FEATURE_AES);
   if ((ECX >> 29) & 1)
     setFeature(FEATURE_F16C);
+  if ((ECX >> 30) & 1)
+    setFeature(FEATURE_RDRND);
 
   // If CPUID indicates support for XSAVE, XRESTORE and AVX, and XGETBV
   // indicates that the AVX registers will be saved and restored on context
   // switch, then we have full AVX support.
   const unsigned AVXBits = (1 << 27) | (1 << 28);
-  bool HasAVX = ((ECX & AVXBits) == AVXBits) && !getX86XCR0(&EAX, &EDX) &&
+  bool HasAVXSave = ((ECX & AVXBits) == AVXBits) && !getX86XCR0(&EAX, &EDX) &&
-                ((EAX & 0x6) == 0x6);
+                    ((EAX & 0x6) == 0x6);
 #if defined(__APPLE__)
   // Darwin lazily saves the AVX512 context on first use: trust that the OS will
   // save the AVX512 context if we use AVX512 instructions, even the bit is not
@@ -549,71 +558,174 @@ static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf,
   bool HasAVX512Save = true;
 #else
   // AVX512 requires additional context to be saved by the OS.
-  bool HasAVX512Save = HasAVX && ((EAX & 0xe0) == 0xe0);
+  bool HasAVX512Save = HasAVXSave && ((EAX & 0xe0) == 0xe0);
 #endif
+  // AMX requires additional context to be saved by the OS.
+  const unsigned AMXBits = (1 << 17) | (1 << 18);
+  bool HasXSave = ((ECX >> 27) & 1) && !getX86XCR0(&EAX, &EDX);
+  bool HasAMXSave = HasXSave && ((EAX & AMXBits) == AMXBits);
 
-  if (HasAVX)
+  if (HasAVXSave)
     setFeature(FEATURE_AVX);
 
   bool HasLeaf7 =
       MaxLeaf >= 0x7 && !getX86CpuIDAndInfoEx(0x7, 0x0, &EAX, &EBX, &ECX, &EDX);
 
-  if (HasLeaf7) {
+  if (HasLeaf7 && ((EBX >> 0) & 1))
-    if ((EBX >> 3) & 1)
+    setFeature(FEATURE_FSGSBASE);
-      setFeature(FEATURE_BMI);
+  if (HasLeaf7 && ((EBX >> 2) & 1))
-    if (((EBX >> 5) & 1) && HasAVX)
+    setFeature(FEATURE_SGX);
-      setFeature(FEATURE_AVX2);
+  if (HasLeaf7 && ((EBX >> 3) & 1))
-    if ((EBX >> 8) & 1)
+    setFeature(FEATURE_BMI);
-      setFeature(FEATURE_BMI2);
+  if (HasLeaf7 && ((EBX >> 5) & 1) && HasAVXSave)
-    if (HasAVX512Save) {
+    setFeature(FEATURE_AVX2);
-      if ((EBX >> 16) & 1)
+  if (HasLeaf7 && ((EBX >> 8) & 1))
-        setFeature(FEATURE_AVX512F);
+    setFeature(FEATURE_BMI2);
-      if ((EBX >> 17) & 1)
+  if (HasLeaf7 && ((EBX >> 11) & 1))
-        setFeature(FEATURE_AVX512DQ);
+    setFeature(FEATURE_RTM);
-      if ((EBX >> 21) & 1)
+  if (HasLeaf7 && ((EBX >> 16) & 1) && HasAVX512Save)
-        setFeature(FEATURE_AVX512IFMA);
+    setFeature(FEATURE_AVX512F);
-      if ((EBX >> 26) & 1)
+  if (HasLeaf7 && ((EBX >> 17) & 1) && HasAVX512Save)
-        setFeature(FEATURE_AVX512PF);
+    setFeature(FEATURE_AVX512DQ);
-      if ((EBX >> 27) & 1)
+  if (HasLeaf7 && ((EBX >> 18) & 1))
-        setFeature(FEATURE_AVX512ER);
+    setFeature(FEATURE_RDSEED);
-      if ((EBX >> 28) & 1)
+  if (HasLeaf7 && ((EBX >> 19) & 1))
-        setFeature(FEATURE_AVX512CD);
+    setFeature(FEATURE_ADX);
-      if ((EBX >> 30) & 1)
+  if (HasLeaf7 && ((EBX >> 21) & 1) && HasAVX512Save)
-        setFeature(FEATURE_AVX512BW);
+    setFeature(FEATURE_AVX512IFMA);
-      if ((EBX >> 31) & 1)
+  if (HasLeaf7 && ((EBX >> 24) & 1))
-        setFeature(FEATURE_AVX512VL);
+    setFeature(FEATURE_CLWB);
-      if ((ECX >> 1) & 1)
+  if (HasLeaf7 && ((EBX >> 26) & 1) && HasAVX512Save)
-        setFeature(FEATURE_AVX512VBMI);
+    setFeature(FEATURE_AVX512PF);
-      if ((ECX >> 6) & 1)
+  if (HasLeaf7 && ((EBX >> 27) & 1) && HasAVX512Save)
-        setFeature(FEATURE_AVX512VBMI2);
+    setFeature(FEATURE_AVX512ER);
-      if ((ECX >> 11) & 1)
+  if (HasLeaf7 && ((EBX >> 28) & 1) && HasAVX512Save)
-        setFeature(FEATURE_AVX512VNNI);
+    setFeature(FEATURE_AVX512CD);
-      if ((ECX >> 12) & 1)
+  if (HasLeaf7 && ((EBX >> 29) & 1))
-        setFeature(FEATURE_AVX512BITALG);
+    setFeature(FEATURE_SHA);
-      if ((ECX >> 14) & 1)
+  if (HasLeaf7 && ((EBX >> 30) & 1) && HasAVX512Save)
-        setFeature(FEATURE_AVX512VPOPCNTDQ);
+    setFeature(FEATURE_AVX512BW);
-      if ((EDX >> 2) & 1)
+  if (HasLeaf7 && ((EBX >> 31) & 1) && HasAVX512Save)
-        setFeature(FEATURE_AVX5124VNNIW);
+    setFeature(FEATURE_AVX512VL);
-      if ((EDX >> 3) & 1)
+
-        setFeature(FEATURE_AVX5124FMAPS);
+  if (HasLeaf7 && ((ECX >> 0) & 1))
-      if ((EDX >> 8) & 1)
+    setFeature(FEATURE_PREFETCHWT1);
-        setFeature(FEATURE_AVX512VP2INTERSECT);
+  if (HasLeaf7 && ((ECX >> 1) & 1) && HasAVX512Save)
-      if ((EDX >> 23) & 1)
+    setFeature(FEATURE_AVX512VBMI);
-        setFeature(FEATURE_AVX512FP16);
+  if (HasLeaf7 && ((ECX >> 4) & 1))
-    }
+    setFeature(FEATURE_PKU);
-    if ((ECX >> 8) & 1)
+  if (HasLeaf7 && ((ECX >> 5) & 1))
-      setFeature(FEATURE_GFNI);
+    setFeature(FEATURE_WAITPKG);
-    if (((ECX >> 10) & 1) && HasAVX)
+  if (HasLeaf7 && ((ECX >> 6) & 1) && HasAVX512Save)
-      setFeature(FEATURE_VPCLMULQDQ);
+    setFeature(FEATURE_AVX512VBMI2);
-  }
+  if (HasLeaf7 && ((ECX >> 7) & 1))
+    setFeature(FEATURE_SHSTK);
+  if (HasLeaf7 && ((ECX >> 8) & 1))
+    setFeature(FEATURE_GFNI);
+  if (HasLeaf7 && ((ECX >> 9) & 1) && HasAVXSave)
+    setFeature(FEATURE_VAES);
+  if (HasLeaf7 && ((ECX >> 10) & 1) && HasAVXSave)
+    setFeature(FEATURE_VPCLMULQDQ);
+  if (HasLeaf7 && ((ECX >> 11) & 1) && HasAVX512Save)
+    setFeature(FEATURE_AVX512VNNI);
+  if (HasLeaf7 && ((ECX >> 12) & 1) && HasAVX512Save)
+    setFeature(FEATURE_AVX512BITALG);
+  if (HasLeaf7 && ((ECX >> 14) & 1) && HasAVX512Save)
+    setFeature(FEATURE_AVX512VPOPCNTDQ);
+  if (HasLeaf7 && ((ECX >> 22) & 1))
+    setFeature(FEATURE_RDPID);
+  if (HasLeaf7 && ((ECX >> 23) & 1))
+    setFeature(FEATURE_KL);
+  if (HasLeaf7 && ((ECX >> 25) & 1))
+    setFeature(FEATURE_CLDEMOTE);
+  if (HasLeaf7 && ((ECX >> 27) & 1))
+    setFeature(FEATURE_MOVDIRI);
+  if (HasLeaf7 && ((ECX >> 28) & 1))
+    setFeature(FEATURE_MOVDIR64B);
+  if (HasLeaf7 && ((ECX >> 29) & 1))
+    setFeature(FEATURE_ENQCMD);
+
+  if (HasLeaf7 && ((EDX >> 2) & 1) && HasAVX512Save)
+    setFeature(FEATURE_AVX5124VNNIW);
+  if (HasLeaf7 && ((EDX >> 3) & 1) && HasAVX512Save)
+    setFeature(FEATURE_AVX5124FMAPS);
+  if (HasLeaf7 && ((EDX >> 5) & 1))
+    setFeature(FEATURE_UINTR);
+  if (HasLeaf7 && ((EDX >> 8) & 1) && HasAVX512Save)
+    setFeature(FEATURE_AVX512VP2INTERSECT);
+  if (HasLeaf7 && ((EDX >> 14) & 1))
+    setFeature(FEATURE_SERIALIZE);
+  if (HasLeaf7 && ((EDX >> 16) & 1))
+    setFeature(FEATURE_TSXLDTRK);
+  if (HasLeaf7 && ((EDX >> 18) & 1))
+    setFeature(FEATURE_PCONFIG);
+  if (HasLeaf7 && ((EDX >> 22) & 1) && HasAMXSave)
+    setFeature(FEATURE_AMX_BF16);
+  if (HasLeaf7 && ((EDX >> 23) & 1) && HasAVX512Save)
+    setFeature(FEATURE_AVX512FP16);
+  if (HasLeaf7 && ((EDX >> 24) & 1) && HasAMXSave)
+    setFeature(FEATURE_AMX_TILE);
+  if (HasLeaf7 && ((EDX >> 25) & 1) && HasAMXSave)
+    setFeature(FEATURE_AMX_INT8);
 
   // EAX from subleaf 0 is the maximum subleaf supported. Some CPUs don't
   // return all 0s for invalid subleaves so check the limit.
   bool HasLeaf7Subleaf1 =
       HasLeaf7 && EAX >= 1 &&
       !getX86CpuIDAndInfoEx(0x7, 0x1, &EAX, &EBX, &ECX, &EDX);
+  if (HasLeaf7Subleaf1 && ((EAX >> 0) & 1))
+    setFeature(FEATURE_SHA512);
+  if (HasLeaf7Subleaf1 && ((EAX >> 1) & 1))
+    setFeature(FEATURE_SM3);
+  if (HasLeaf7Subleaf1 && ((EAX >> 2) & 1))
+    setFeature(FEATURE_SM4);
+  if (HasLeaf7Subleaf1 && ((EAX >> 3) & 1))
+    setFeature(FEATURE_RAOINT);
+  if (HasLeaf7Subleaf1 && ((EAX >> 4) & 1) && HasAVXSave)
+    setFeature(FEATURE_AVXVNNI);
   if (HasLeaf7Subleaf1 && ((EAX >> 5) & 1) && HasAVX512Save)
     setFeature(FEATURE_AVX512BF16);
+  if (HasLeaf7Subleaf1 && ((EAX >> 7) & 1))
+    setFeature(FEATURE_CMPCCXADD);
+  if (HasLeaf7Subleaf1 && ((EAX >> 21) & 1) && HasAMXSave)
+    setFeature(FEATURE_AMX_FP16);
+  if (HasLeaf7Subleaf1 && ((EAX >> 22) & 1))
+    setFeature(FEATURE_HRESET);
+  if (HasLeaf7Subleaf1 && ((EAX >> 23) & 1) && HasAVXSave)
+    setFeature(FEATURE_AVXIFMA);
+
+  if (HasLeaf7Subleaf1 && ((EDX >> 4) & 1) && HasAVXSave)
+    setFeature(FEATURE_AVXVNNIINT8);
+  if (HasLeaf7Subleaf1 && ((EDX >> 5) & 1) && HasAVXSave)
+    setFeature(FEATURE_AVXNECONVERT);
+  if (HasLeaf7Subleaf1 && ((EDX >> 8) & 1) && HasAMXSave)
+    setFeature(FEATURE_AMX_COMPLEX);
+  if (HasLeaf7Subleaf1 && ((EDX >> 10) & 1) && HasAVXSave)
+    setFeature(FEATURE_AVXVNNIINT16);
+  if (HasLeaf7Subleaf1 && ((EDX >> 14) & 1))
+    setFeature(FEATURE_PREFETCHI);
+  if (HasLeaf7Subleaf1 && ((EDX >> 15) & 1))
+    setFeature(FEATURE_USERMSR);
+  if (HasLeaf7Subleaf1 && ((EDX >> 19) & 1))
+    setFeature(FEATURE_AVX10_1_256);
+  if (HasLeaf7Subleaf1 && ((EDX >> 21) & 1))
+    setFeature(FEATURE_APXF);
+
+  unsigned MaxLevel;
+  getX86CpuIDAndInfo(0, &MaxLevel, &EBX, &ECX, &EDX);
+  bool HasLeafD = MaxLevel >= 0xd &&
+                  !getX86CpuIDAndInfoEx(0xd, 0x1, &EAX, &EBX, &ECX, &EDX);
+  if (HasLeafD && ((EAX >> 0) & 1) && HasAVXSave)
+    setFeature(FEATURE_XSAVEOPT);
+  if (HasLeafD && ((EAX >> 1) & 1) && HasAVXSave)
+    setFeature(FEATURE_XSAVEC);
+  if (HasLeafD && ((EAX >> 3) & 1) && HasAVXSave)
+    setFeature(FEATURE_XSAVES);
+
+  bool HasLeaf24 =
+      MaxLevel >= 0x24 && !getX86CpuIDAndInfo(0x24, &EAX, &EBX, &ECX, &EDX);
+  if (HasLeaf7Subleaf1 && ((EDX >> 19) & 1) && HasLeaf24 && ((EBX >> 18) & 1))
+    setFeature(FEATURE_AVX10_1_512);
 
   unsigned MaxExtLevel;
   getX86CpuIDAndInfo(0x80000000, &MaxExtLevel, &EBX, &ECX, &EDX);
@@ -627,14 +739,40 @@ static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf,
       setFeature(FEATURE_LZCNT);
     if (((ECX >> 6) & 1))
       setFeature(FEATURE_SSE4_A);
+    if (((ECX >> 8) & 1))
+      setFeature(FEATURE_PRFCHW);
     if (((ECX >> 11) & 1))
       setFeature(FEATURE_XOP);
+    if (((ECX >> 15) & 1))
+      setFeature(FEATURE_LWP);
     if (((ECX >> 16) & 1))
       setFeature(FEATURE_FMA4);
+    if (((ECX >> 21) & 1))
+      setFeature(FEATURE_TBM);
+    if (((ECX >> 29) & 1))
+      setFeature(FEATURE_MWAITX);
+
     if (((EDX >> 29) & 1))
       setFeature(FEATURE_LM);
   }
 
+  bool HasExtLeaf8 = MaxExtLevel >= 0x80000008 &&
+                     !getX86CpuIDAndInfo(0x80000008, &EAX, &EBX, &ECX, &EDX);
+  if (HasExtLeaf8 && ((EBX >> 0) & 1))
+    setFeature(FEATURE_CLZERO);
+  if (HasExtLeaf8 && ((EBX >> 9) & 1))
+    setFeature(FEATURE_WBNOINVD);
+
+  bool HasLeaf14 = MaxLevel >= 0x14 &&
+                   !getX86CpuIDAndInfoEx(0x14, 0x0, &EAX, &EBX, &ECX, &EDX);
+  if (HasLeaf14 && ((EBX >> 4) & 1))
+    setFeature(FEATURE_PTWRITE);
+
+  bool HasLeaf19 =
+      MaxLevel >= 0x19 && !getX86CpuIDAndInfo(0x19, &EAX, &EBX, &ECX, &EDX);
+  if (HasLeaf7 && HasLeaf19 && ((EBX >> 2) & 1))
+    setFeature(FEATURE_WIDEKL);
+
   if (hasFeature(FEATURE_LM) && hasFeature(FEATURE_SSE2)) {
     setFeature(FEATURE_X86_64_BASELINE);
     if (hasFeature(FEATURE_CMPXCHG16B) && hasFeature(FEATURE_POPCNT) &&

libc/intrin/x86.h

@@ -114,20 +114,88 @@ enum ProcessorFeatures {
   FEATURE_AVX512BITALG,
   FEATURE_AVX512BF16,
   FEATURE_AVX512VP2INTERSECT,
-
+  // FIXME: Below Features has some missings comparing to gcc, it's because gcc
-  FEATURE_CMPXCHG16B = 46,
+  // has some not one-to-one mapped in llvm.
-  FEATURE_F16C = 49,
+  // FEATURE_3DNOW,
+  // FEATURE_3DNOWP,
+  FEATURE_ADX = 40,
+  // FEATURE_ABM,
+  FEATURE_CLDEMOTE = 42,
+  FEATURE_CLFLUSHOPT,
+  FEATURE_CLWB,
+  FEATURE_CLZERO,
+  FEATURE_CMPXCHG16B,
+  // FIXME: Not adding FEATURE_CMPXCHG8B is a workaround to make 'generic' as
+  // a cpu string with no X86_FEATURE_COMPAT features, which is required in
+  // current implementantion of cpu_specific/cpu_dispatch FMV feature.
+  // FEATURE_CMPXCHG8B,
+  FEATURE_ENQCMD = 48,
+  FEATURE_F16C,
+  FEATURE_FSGSBASE,
+  // FEATURE_FXSAVE,
+  // FEATURE_HLE,
+  // FEATURE_IBT,
   FEATURE_LAHF_LM = 54,
   FEATURE_LM,
-  FEATURE_WP,
+  FEATURE_LWP,
   FEATURE_LZCNT,
   FEATURE_MOVBE,
-
+  FEATURE_MOVDIR64B,
-  FEATURE_AVX512FP16 = 94,
+  FEATURE_MOVDIRI,
+  FEATURE_MWAITX,
+  // FEATURE_OSXSAVE,
+  FEATURE_PCONFIG = 63,
+  FEATURE_PKU,
+  FEATURE_PREFETCHWT1,
+  FEATURE_PRFCHW,
+  FEATURE_PTWRITE,
+  FEATURE_RDPID,
+  FEATURE_RDRND,
+  FEATURE_RDSEED,
+  FEATURE_RTM,
+  FEATURE_SERIALIZE,
+  FEATURE_SGX,
+  FEATURE_SHA,
+  FEATURE_SHSTK,
+  FEATURE_TBM,
+  FEATURE_TSXLDTRK,
+  FEATURE_VAES,
+  FEATURE_WAITPKG,
+  FEATURE_WBNOINVD,
+  FEATURE_XSAVE,
+  FEATURE_XSAVEC,
+  FEATURE_XSAVEOPT,
+  FEATURE_XSAVES,
+  FEATURE_AMX_TILE,
+  FEATURE_AMX_INT8,
+  FEATURE_AMX_BF16,
+  FEATURE_UINTR,
+  FEATURE_HRESET,
+  FEATURE_KL,
+  // FEATURE_AESKLE,
+  FEATURE_WIDEKL = 92,
+  FEATURE_AVXVNNI,
+  FEATURE_AVX512FP16,
   FEATURE_X86_64_BASELINE,
   FEATURE_X86_64_V2,
   FEATURE_X86_64_V3,
   FEATURE_X86_64_V4,
+  FEATURE_AVXIFMA,
+  FEATURE_AVXVNNIINT8,
+  FEATURE_AVXNECONVERT,
+  FEATURE_CMPCCXADD,
+  FEATURE_AMX_FP16,
+  FEATURE_PREFETCHI,
+  FEATURE_RAOINT,
+  FEATURE_AMX_COMPLEX,
+  FEATURE_AVXVNNIINT16,
+  FEATURE_SM3,
+  FEATURE_SHA512,
+  FEATURE_SM4,
+  FEATURE_APXF,
+  FEATURE_USERMSR,
+  FEATURE_AVX10_1_256,
+  FEATURE_AVX10_1_512,
   CPU_FEATURE_MAX
 };