cpu_id.cc 10 KB
Newer Older
mikhal@webrtc.org's avatar
mikhal@webrtc.org committed
1
/*
2
 *  Copyright 2011 The LibYuv Project Authors. All rights reserved.
mikhal@webrtc.org's avatar
mikhal@webrtc.org committed
3 4 5 6
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS. All contributing project authors may
mikhal@webrtc.org's avatar
mikhal@webrtc.org committed
8 9 10
 *  be found in the AUTHORS file in the root of the source tree.
 */

11
#include "libyuv/cpu_id.h"
mikhal@webrtc.org's avatar
mikhal@webrtc.org committed
12

13
#if defined(_MSC_VER)
14
#include <intrin.h>  // For __cpuidex()
15
#endif
Frank Barchard's avatar
Frank Barchard committed
16
#if !defined(__pnacl__) && !defined(__CLR_VER) &&                           \
17
    !defined(__native_client__) && (defined(_M_IX86) || defined(_M_X64)) && \
18
    defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 160040219)
19 20
#include <immintrin.h>  // For _xgetbv()
#endif
21

22
#if !defined(__native_client__)
23
#include <stdlib.h>  // For getenv()
24
#endif
25 26 27 28

// For ArmCpuCaps() but unittested on all platforms
#include <stdio.h>
#include <string.h>
mikhal@webrtc.org's avatar
mikhal@webrtc.org committed
29

30
#include "libyuv/basic_types.h"  // For CPU_X86
frkoenig@google.com's avatar
frkoenig@google.com committed
31

32
#ifdef __cplusplus
mikhal@webrtc.org's avatar
mikhal@webrtc.org committed
33
namespace libyuv {
34 35
extern "C" {
#endif
mikhal@webrtc.org's avatar
mikhal@webrtc.org committed
36

37
// For functions that use the stack and have runtime checks for overflow,
38
// use SAFEBUFFERS to avoid additional check.
39 40
#if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 160040219) && \
    !defined(__clang__)
41 42 43 44 45
#define SAFEBUFFERS __declspec(safebuffers)
#else
#define SAFEBUFFERS
#endif

46
// Low level cpuid for X86.
Frank Barchard's avatar
Frank Barchard committed
47 48
#if (defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || \
     defined(__x86_64__)) &&                                     \
49
    !defined(__pnacl__) && !defined(__CLR_VER)
50
LIBYUV_API
51
void CpuId(uint32 info_eax, uint32 info_ecx, uint32* cpu_info) {
52
#if defined(_MSC_VER)
53
// Visual C version uses intrinsic or inline x86 assembly.
54
#if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 160040219)
55
  __cpuidex((int*)(cpu_info), info_eax, info_ecx);
56
#elif defined(_M_IX86)
57 58 59 60 61 62 63 64 65 66
  __asm {
    mov        eax, info_eax
    mov        ecx, info_ecx
    mov        edi, cpu_info
    cpuid
    mov        [edi], eax
    mov        [edi + 4], ebx
    mov        [edi + 8], ecx
    mov        [edi + 12], edx
  }
67
#else  // Visual C but not x86
68
  if (info_ecx == 0) {
69
    __cpuid((int*)(cpu_info), info_eax);
70
  } else {
71
    cpu_info[3] = cpu_info[2] = cpu_info[1] = cpu_info[0] = 0u;
72 73
  }
#endif
74
// GCC version uses inline x86 assembly.
75
#else  // defined(_MSC_VER)
76
  uint32 info_ebx, info_edx;
Frank Barchard's avatar
Frank Barchard committed
77 78 79 80 81 82 83
  asm volatile(
#if defined(__i386__) && defined(__PIC__)
      // Preserve ebx for fpic 32 bit.
      "mov %%ebx, %%edi                          \n"
      "cpuid                                     \n"
      "xchg %%edi, %%ebx                         \n"
      : "=D"(info_ebx),
84
#else
Frank Barchard's avatar
Frank Barchard committed
85 86
      "cpuid                                     \n"
      : "=b"(info_ebx),
87
#endif  //  defined( __i386__) && defined(__PIC__)
Frank Barchard's avatar
Frank Barchard committed
88
        "+a"(info_eax), "+c"(info_ecx), "=d"(info_edx));
89 90 91 92
  cpu_info[0] = info_eax;
  cpu_info[1] = info_ebx;
  cpu_info[2] = info_ecx;
  cpu_info[3] = info_edx;
93
#endif  // defined(_MSC_VER)
94 95 96 97
}
#else  // (defined(_M_IX86) || defined(_M_X64) ...
LIBYUV_API
void CpuId(uint32 eax, uint32 ecx, uint32* cpu_info) {
98 99
  (void)eax;
  (void)ecx;
100
  cpu_info[0] = cpu_info[1] = cpu_info[2] = cpu_info[3] = 0;
101
}
102
#endif
103

104 105 106 107 108 109 110 111 112 113 114 115
// For VS2010 and earlier emit can be used:
//   _asm _emit 0x0f _asm _emit 0x01 _asm _emit 0xd0  // For VS2010 and earlier.
//  __asm {
//    xor        ecx, ecx    // xcr 0
//    xgetbv
//    mov        xcr0, eax
//  }
// For VS2013 and earlier 32 bit, the _xgetbv(0) optimizer produces bad code.
// https://code.google.com/p/libyuv/issues/detail?id=529
#if defined(_M_IX86) && (_MSC_VER < 1900)
#pragma optimize("g", off)
#endif
Frank Barchard's avatar
Frank Barchard committed
116 117
#if (defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || \
     defined(__x86_64__)) &&                                     \
118
    !defined(__pnacl__) && !defined(__CLR_VER) && !defined(__native_client__)
119
// X86 CPUs have xgetbv to detect OS saves high parts of ymm registers.
120
int GetXCR0() {
121
  uint32 xcr0 = 0u;
122
#if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 160040219)
123
  xcr0 = (uint32)(_xgetbv(0));  // VS2010 SP1 required.
124
#elif defined(__i386__) || defined(__x86_64__)
Frank Barchard's avatar
Frank Barchard committed
125
  asm(".byte 0x0f, 0x01, 0xd0" : "=a"(xcr0) : "c"(0) : "%edx");
126
#endif  // defined(__i386__) || defined(__x86_64__)
127
  return xcr0;
128
}
129 130 131
#else
// xgetbv unavailable to query for OSSave support.  Return 0.
#define GetXCR0() 0
132
#endif  // defined(_M_IX86) || defined(_M_X64) ..
133 134 135 136
// Return optimization to previous setting.
#if defined(_M_IX86) && (_MSC_VER < 1900)
#pragma optimize("g", on)
#endif
137

138
// based on libvpx arm_cpudetect.c
139
// For Arm, but public to allow testing on any CPU
Frank Barchard's avatar
Frank Barchard committed
140
LIBYUV_API SAFEBUFFERS int ArmCpuCaps(const char* cpuinfo_name) {
141
  char cpuinfo_line[512];
142
  FILE* f = fopen(cpuinfo_name, "r");
143
  if (!f) {
144 145 146
    // Assume Neon if /proc/cpuinfo is unavailable.
    // This will occur for Chrome sandbox for Pepper or Render process.
    return kCpuHasNEON;
147 148 149 150 151 152 153
  }
  while (fgets(cpuinfo_line, sizeof(cpuinfo_line) - 1, f)) {
    if (memcmp(cpuinfo_line, "Features", 8) == 0) {
      char* p = strstr(cpuinfo_line, " neon");
      if (p && (p[5] == ' ' || p[5] == '\n')) {
        fclose(f);
        return kCpuHasNEON;
154
      }
155 156 157 158 159 160
      // aarch64 uses asimd for Neon.
      p = strstr(cpuinfo_line, " asimd");
      if (p && (p[6] == ' ' || p[6] == '\n')) {
        fclose(f);
        return kCpuHasNEON;
      }
161 162
    }
  }
163
  fclose(f);
164
  return 0;
165 166
}

Frank Barchard's avatar
Frank Barchard committed
167 168
LIBYUV_API SAFEBUFFERS int MipsCpuCaps(const char* cpuinfo_name,
                                       const char ase[]) {
169
  char cpuinfo_line[512];
170
  int len = (int)strlen(ase);
171 172 173
  FILE* f = fopen(cpuinfo_name, "r");
  if (!f) {
    // ase enabled if /proc/cpuinfo is unavailable.
174
    if (strcmp(ase, " msa") == 0) {
175 176
      return kCpuHasMSA;
    }
177
    return kCpuHasDSPR2;
178 179 180 181 182 183
  }
  while (fgets(cpuinfo_line, sizeof(cpuinfo_line) - 1, f)) {
    if (memcmp(cpuinfo_line, "ASEs implemented", 16) == 0) {
      char* p = strstr(cpuinfo_line, ase);
      if (p && (p[len] == ' ' || p[len] == '\n')) {
        fclose(f);
184
        if (strcmp(ase, " msa") == 0) {
185 186
          return kCpuHasMSA;
        }
187
        return kCpuHasDSPR2;
188 189 190 191 192 193 194
      }
    }
  }
  fclose(f);
  return 0;
}

mikhal@webrtc.org's avatar
mikhal@webrtc.org committed
195
// CPU detect function for SIMD instruction sets.
196
LIBYUV_API
197
int cpu_info_ = 0;  // cpu_info is not initialized yet.
mikhal@webrtc.org's avatar
mikhal@webrtc.org committed
198

199 200
// Test environment variable for disabling CPU features. Any non-zero value
// to disable. Zero ignored to make it easy to set the variable on/off.
201 202
#if !defined(__native_client__) && !defined(_M_ARM)

203
static LIBYUV_BOOL TestEnv(const char* name) {
204 205 206
  const char* var = getenv(name);
  if (var) {
    if (var[0] != '0') {
207
      return LIBYUV_TRUE;
208 209
    }
  }
210
  return LIBYUV_FALSE;
211
}
212
#else  // nacl does not support getenv().
213 214
static LIBYUV_BOOL TestEnv(const char*) {
  return LIBYUV_FALSE;
215 216
}
#endif
217

Frank Barchard's avatar
Frank Barchard committed
218
LIBYUV_API SAFEBUFFERS int InitCpuFlags(void) {
219
  int cpu_info = 0;
220
#if !defined(__pnacl__) && !defined(__CLR_VER) && defined(CPU_X86)
Frank Barchard's avatar
Frank Barchard committed
221 222 223
  uint32 cpu_info0[4] = {0, 0, 0, 0};
  uint32 cpu_info1[4] = {0, 0, 0, 0};
  uint32 cpu_info7[4] = {0, 0, 0, 0};
224
  CpuId(0, 0, cpu_info0);
225
  CpuId(1, 0, cpu_info1);
226 227 228
  if (cpu_info0[0] >= 7) {
    CpuId(7, 0, cpu_info7);
  }
Frank Barchard's avatar
Frank Barchard committed
229
  cpu_info = kCpuHasX86 | ((cpu_info1[3] & 0x04000000) ? kCpuHasSSE2 : 0) |
230 231 232
             ((cpu_info1[2] & 0x00000200) ? kCpuHasSSSE3 : 0) |
             ((cpu_info1[2] & 0x00080000) ? kCpuHasSSE41 : 0) |
             ((cpu_info1[2] & 0x00100000) ? kCpuHasSSE42 : 0) |
233
             ((cpu_info7[1] & 0x00000200) ? kCpuHasERMS : 0);
234

235
  // AVX requires OS saves YMM registers.
236 237
  if (((cpu_info1[2] & 0x1c000000) == 0x1c000000) &&  // AVX and OSXSave
      ((GetXCR0() & 6) == 6)) {  // Test OS saves YMM registers
Frank Barchard's avatar
Frank Barchard committed
238
    cpu_info |= kCpuHasAVX | ((cpu_info7[1] & 0x00000020) ? kCpuHasAVX2 : 0) |
239 240
                ((cpu_info1[2] & 0x00001000) ? kCpuHasFMA3 : 0) |
                ((cpu_info1[2] & 0x20000000) ? kCpuHasF16C : 0);
241 242 243 244 245

    // Detect AVX512bw
    if ((GetXCR0() & 0xe0) == 0xe0) {
      cpu_info |= (cpu_info7[1] & 0x40000000) ? kCpuHasAVX3 : 0;
    }
246
  }
247

248
  // Environment variable overrides for testing.
249
  if (TestEnv("LIBYUV_DISABLE_X86")) {
250
    cpu_info &= ~kCpuHasX86;
251
  }
252
  if (TestEnv("LIBYUV_DISABLE_SSE2")) {
253
    cpu_info &= ~kCpuHasSSE2;
254
  }
255
  if (TestEnv("LIBYUV_DISABLE_SSSE3")) {
256
    cpu_info &= ~kCpuHasSSSE3;
257
  }
258
  if (TestEnv("LIBYUV_DISABLE_SSE41")) {
259
    cpu_info &= ~kCpuHasSSE41;
260
  }
261
  if (TestEnv("LIBYUV_DISABLE_SSE42")) {
262
    cpu_info &= ~kCpuHasSSE42;
263
  }
264 265 266
  if (TestEnv("LIBYUV_DISABLE_AVX")) {
    cpu_info &= ~kCpuHasAVX;
  }
267
  if (TestEnv("LIBYUV_DISABLE_AVX2")) {
268
    cpu_info &= ~kCpuHasAVX2;
269
  }
fbarchard@google.com's avatar
fbarchard@google.com committed
270
  if (TestEnv("LIBYUV_DISABLE_ERMS")) {
271
    cpu_info &= ~kCpuHasERMS;
fbarchard@google.com's avatar
fbarchard@google.com committed
272
  }
273
  if (TestEnv("LIBYUV_DISABLE_FMA3")) {
274
    cpu_info &= ~kCpuHasFMA3;
275
  }
276 277 278
  if (TestEnv("LIBYUV_DISABLE_AVX3")) {
    cpu_info &= ~kCpuHasAVX3;
  }
279 280 281 282
  if (TestEnv("LIBYUV_DISABLE_F16C")) {
    cpu_info &= ~kCpuHasF16C;
  }

283 284
#endif
#if defined(__mips__) && defined(__linux__)
285
#if defined(__mips_dspr2)
286
  cpu_info |= kCpuHasDSPR2;
287 288 289
#endif
#if defined(__mips_msa)
  cpu_info = MipsCpuCaps("/proc/cpuinfo", " msa");
290
#endif
291
  cpu_info |= kCpuHasMIPS;
292 293
  if (getenv("LIBYUV_DISABLE_DSPR2")) {
    cpu_info &= ~kCpuHasDSPR2;
294
  }
295 296 297
  if (getenv("LIBYUV_DISABLE_MSA")) {
    cpu_info &= ~kCpuHasMSA;
  }
298 299
#endif
#if defined(__arm__) || defined(__aarch64__)
300 301 302
// gcc -mfpu=neon defines __ARM_NEON__
// __ARM_NEON__ generates code that requires Neon.  NaCL also requires Neon.
// For Linux, /proc/cpuinfo can be tested but without that assume Neon.
303
#if defined(__ARM_NEON__) || defined(__native_client__) || !defined(__linux__)
304
  cpu_info = kCpuHasNEON;
305 306 307
// For aarch64(arm64), /proc/cpuinfo's feature is not complete, e.g. no neon
// flag in it.
// So for aarch64, neon enabling is hard coded here.
308 309
#endif
#if defined(__aarch64__)
310
  cpu_info = kCpuHasNEON;
311 312
#else
  // Linux arm parse text file for neon detect.
313
  cpu_info = ArmCpuCaps("/proc/cpuinfo");
314
#endif
315
  cpu_info |= kCpuHasARM;
316
  if (TestEnv("LIBYUV_DISABLE_NEON")) {
317
    cpu_info &= ~kCpuHasNEON;
318
  }
319
#endif  // __arm__
320
  if (TestEnv("LIBYUV_DISABLE_ASM")) {
321
    cpu_info = 0;
322
  }
Frank Barchard's avatar
Frank Barchard committed
323
  cpu_info |= kCpuInitialized;
324
  cpu_info_ = cpu_info;
325
  return cpu_info;
mikhal@webrtc.org's avatar
mikhal@webrtc.org committed
326 327
}

328
// Note that use of this function is not thread safe.
329
LIBYUV_API
330
void MaskCpuFlags(int enable_flags) {
331
  cpu_info_ = InitCpuFlags() & enable_flags;
mikhal@webrtc.org's avatar
mikhal@webrtc.org committed
332 333
}

334 335
#ifdef __cplusplus
}  // extern "C"
mikhal@webrtc.org's avatar
mikhal@webrtc.org committed
336
}  // namespace libyuv
337
#endif