rotate.cc 47.5 KB
Newer Older
1
/*
2
 *  Copyright 2011 The LibYuv Project Authors. All rights reserved.
3 4 5 6
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS. All contributing project authors may
8 9 10
 *  be found in the AUTHORS file in the root of the source tree.
 */

11
#include "libyuv/rotate.h"
12

13
#include "libyuv/cpu_id.h"
14
#include "libyuv/convert.h"
frkoenig@google.com's avatar
frkoenig@google.com committed
15
#include "libyuv/planar_functions.h"
16
#include "libyuv/row.h"
17

18
#ifdef __cplusplus
19
namespace libyuv {
20 21
extern "C" {
#endif
22

23
#if !defined(LIBYUV_DISABLE_X86) && \
24
    (defined(_M_IX86) || defined(__x86_64__) || defined(__i386__))
25
#if defined(__APPLE__) && defined(__i386__)
26 27
#define DECLARE_FUNCTION(name)                                                 \
    ".text                                     \n"                             \
28 29
    ".private_extern _" #name "                \n"                             \
    ".align 4,0x90                             \n"                             \
30
"_" #name ":                                   \n"
31
#elif defined(__MINGW32__) || defined(__CYGWIN__) && defined(__i386__)
32 33 34 35
#define DECLARE_FUNCTION(name)                                                 \
    ".text                                     \n"                             \
    ".align 4,0x90                             \n"                             \
"_" #name ":                                   \n"
36 37 38
#else
#define DECLARE_FUNCTION(name)                                                 \
    ".text                                     \n"                             \
39
    ".align 4,0x90                             \n"                             \
40
#name ":                                       \n"
41
#endif
42 43
#endif

44 45
#if !defined(LIBYUV_DISABLE_NEON) && \
    (defined(__ARM_NEON__) || defined(LIBYUV_NEON))
46 47 48
#define HAS_MIRRORROW_NEON
void MirrorRow_NEON(const uint8* src, uint8* dst, int width);
#define HAS_MIRRORROW_UV_NEON
49
void MirrorUVRow_NEON(const uint8* src, uint8* dst_a, uint8* dst_b, int width);
50
#define HAS_TRANSPOSE_WX8_NEON
51 52
void TransposeWx8_NEON(const uint8* src, int src_stride,
                       uint8* dst, int dst_stride, int width);
53
#define HAS_TRANSPOSE_UVWX8_NEON
54 55 56 57
void TransposeUVWx8_NEON(const uint8* src, int src_stride,
                         uint8* dst_a, int dst_stride_a,
                         uint8* dst_b, int dst_stride_b,
                         int width);
58
#endif  // defined(__ARM_NEON__)
59

60 61
#if !defined(LIBYUV_DISABLE_MIPS) && defined(__mips__) && \
    defined(__mips_dsp) && (__mips_dsp_rev >= 2)
62 63 64 65 66 67 68 69 70 71 72
#define HAS_TRANSPOSE_WX8_MIPS_DSPR2
void TransposeWx8_MIPS_DSPR2(const uint8* src, int src_stride,
                             uint8* dst, int dst_stride, int width);

void TransposeWx8_FAST_MIPS_DSPR2(const uint8* src, int src_stride,
                                  uint8* dst, int dst_stride, int width);
#define HAS_TRANSPOSE_UVWx8_MIPS_DSPR2
void TransposeUVWx8_MIPS_DSPR2(const uint8* src, int src_stride,
                               uint8* dst_a, int dst_stride_a,
                               uint8* dst_b, int dst_stride_b,
                               int width);
73
#endif  // defined(__mips__)
74

75
#if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86)
76
#define HAS_TRANSPOSE_WX8_SSSE3
77
__declspec(naked) __declspec(align(16))
78 79
static void TransposeWx8_SSSE3(const uint8* src, int src_stride,
                               uint8* dst, int dst_stride, int width) {
80
  __asm {
81 82 83 84 85 86 87 88
    push      edi
    push      esi
    push      ebp
    mov       eax, [esp + 12 + 4]   // src
    mov       edi, [esp + 12 + 8]   // src_stride
    mov       edx, [esp + 12 + 12]  // dst
    mov       esi, [esp + 12 + 16]  // dst_stride
    mov       ecx, [esp + 12 + 20]  // width
89

90 91
    // Read in the data from the source pointer.
    // First round of bit swap.
92 93
    align      16
 convertloop:
94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
    movq      xmm0, qword ptr [eax]
    lea       ebp, [eax + 8]
    movq      xmm1, qword ptr [eax + edi]
    lea       eax, [eax + 2 * edi]
    punpcklbw xmm0, xmm1
    movq      xmm2, qword ptr [eax]
    movdqa    xmm1, xmm0
    palignr   xmm1, xmm1, 8
    movq      xmm3, qword ptr [eax + edi]
    lea       eax, [eax + 2 * edi]
    punpcklbw xmm2, xmm3
    movdqa    xmm3, xmm2
    movq      xmm4, qword ptr [eax]
    palignr   xmm3, xmm3, 8
    movq      xmm5, qword ptr [eax + edi]
    punpcklbw xmm4, xmm5
    lea       eax, [eax + 2 * edi]
    movdqa    xmm5, xmm4
    movq      xmm6, qword ptr [eax]
    palignr   xmm5, xmm5, 8
    movq      xmm7, qword ptr [eax + edi]
    punpcklbw xmm6, xmm7
    mov       eax, ebp
    movdqa    xmm7, xmm6
    palignr   xmm7, xmm7, 8
    // Second round of bit swap.
    punpcklwd xmm0, xmm2
    punpcklwd xmm1, xmm3
    movdqa    xmm2, xmm0
    movdqa    xmm3, xmm1
    palignr   xmm2, xmm2, 8
    palignr   xmm3, xmm3, 8
    punpcklwd xmm4, xmm6
    punpcklwd xmm5, xmm7
    movdqa    xmm6, xmm4
    movdqa    xmm7, xmm5
    palignr   xmm6, xmm6, 8
    palignr   xmm7, xmm7, 8
    // Third round of bit swap.
133
    // Write to the destination pointer.
134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
    punpckldq xmm0, xmm4
    movq      qword ptr [edx], xmm0
    movdqa    xmm4, xmm0
    palignr   xmm4, xmm4, 8
    movq      qword ptr [edx + esi], xmm4
    lea       edx, [edx + 2 * esi]
    punpckldq xmm2, xmm6
    movdqa    xmm6, xmm2
    palignr   xmm6, xmm6, 8
    movq      qword ptr [edx], xmm2
    punpckldq xmm1, xmm5
    movq      qword ptr [edx + esi], xmm6
    lea       edx, [edx + 2 * esi]
    movdqa    xmm5, xmm1
    movq      qword ptr [edx], xmm1
    palignr   xmm5, xmm5, 8
    punpckldq xmm3, xmm7
    movq      qword ptr [edx + esi], xmm5
    lea       edx, [edx + 2 * esi]
    movq      qword ptr [edx], xmm3
    movdqa    xmm7, xmm3
    palignr   xmm7, xmm7, 8
156
    sub       ecx, 8
157 158
    movq      qword ptr [edx + esi], xmm7
    lea       edx, [edx + 2 * esi]
159
    jg        convertloop
160

161 162 163 164 165 166 167
    pop       ebp
    pop       esi
    pop       edi
    ret
  }
}

168
#define HAS_TRANSPOSE_UVWX8_SSE2
169
__declspec(naked) __declspec(align(16))
170 171 172 173
static void TransposeUVWx8_SSE2(const uint8* src, int src_stride,
                                uint8* dst_a, int dst_stride_a,
                                uint8* dst_b, int dst_stride_b,
                                int w) {
174
  __asm {
175 176 177 178 179 180 181 182 183 184 185 186 187 188 189
    push      ebx
    push      esi
    push      edi
    push      ebp
    mov       eax, [esp + 16 + 4]   // src
    mov       edi, [esp + 16 + 8]   // src_stride
    mov       edx, [esp + 16 + 12]  // dst_a
    mov       esi, [esp + 16 + 16]  // dst_stride_a
    mov       ebx, [esp + 16 + 20]  // dst_b
    mov       ebp, [esp + 16 + 24]  // dst_stride_b
    mov       ecx, esp
    sub       esp, 4 + 16
    and       esp, ~15
    mov       [esp + 16], ecx
    mov       ecx, [ecx + 16 + 28]  // w
190 191

    align      16
192
 convertloop:
193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281
    // Read in the data from the source pointer.
    // First round of bit swap.
    movdqa    xmm0, [eax]
    movdqa    xmm1, [eax + edi]
    lea       eax, [eax + 2 * edi]
    movdqa    xmm7, xmm0  // use xmm7 as temp register.
    punpcklbw xmm0, xmm1
    punpckhbw xmm7, xmm1
    movdqa    xmm1, xmm7
    movdqa    xmm2, [eax]
    movdqa    xmm3, [eax + edi]
    lea       eax, [eax + 2 * edi]
    movdqa    xmm7, xmm2
    punpcklbw xmm2, xmm3
    punpckhbw xmm7, xmm3
    movdqa    xmm3, xmm7
    movdqa    xmm4, [eax]
    movdqa    xmm5, [eax + edi]
    lea       eax, [eax + 2 * edi]
    movdqa    xmm7, xmm4
    punpcklbw xmm4, xmm5
    punpckhbw xmm7, xmm5
    movdqa    xmm5, xmm7
    movdqa    xmm6, [eax]
    movdqa    xmm7, [eax + edi]
    lea       eax, [eax + 2 * edi]
    movdqa    [esp], xmm5  // backup xmm5
    neg       edi
    movdqa    xmm5, xmm6   // use xmm5 as temp register.
    punpcklbw xmm6, xmm7
    punpckhbw xmm5, xmm7
    movdqa    xmm7, xmm5
    lea       eax, [eax + 8 * edi + 16]
    neg       edi
    // Second round of bit swap.
    movdqa    xmm5, xmm0
    punpcklwd xmm0, xmm2
    punpckhwd xmm5, xmm2
    movdqa    xmm2, xmm5
    movdqa    xmm5, xmm1
    punpcklwd xmm1, xmm3
    punpckhwd xmm5, xmm3
    movdqa    xmm3, xmm5
    movdqa    xmm5, xmm4
    punpcklwd xmm4, xmm6
    punpckhwd xmm5, xmm6
    movdqa    xmm6, xmm5
    movdqa    xmm5, [esp]  // restore xmm5
    movdqa    [esp], xmm6  // backup xmm6
    movdqa    xmm6, xmm5    // use xmm6 as temp register.
    punpcklwd xmm5, xmm7
    punpckhwd xmm6, xmm7
    movdqa    xmm7, xmm6
    // Third round of bit swap.
    // Write to the destination pointer.
    movdqa    xmm6, xmm0
    punpckldq xmm0, xmm4
    punpckhdq xmm6, xmm4
    movdqa    xmm4, xmm6
    movdqa    xmm6, [esp]  // restore xmm6
    movlpd    qword ptr [edx], xmm0
    movhpd    qword ptr [ebx], xmm0
    movlpd    qword ptr [edx + esi], xmm4
    lea       edx, [edx + 2 * esi]
    movhpd    qword ptr [ebx + ebp], xmm4
    lea       ebx, [ebx + 2 * ebp]
    movdqa    xmm0, xmm2   // use xmm0 as the temp register.
    punpckldq xmm2, xmm6
    movlpd    qword ptr [edx], xmm2
    movhpd    qword ptr [ebx], xmm2
    punpckhdq xmm0, xmm6
    movlpd    qword ptr [edx + esi], xmm0
    lea       edx, [edx + 2 * esi]
    movhpd    qword ptr [ebx + ebp], xmm0
    lea       ebx, [ebx + 2 * ebp]
    movdqa    xmm0, xmm1   // use xmm0 as the temp register.
    punpckldq xmm1, xmm5
    movlpd    qword ptr [edx], xmm1
    movhpd    qword ptr [ebx], xmm1
    punpckhdq xmm0, xmm5
    movlpd    qword ptr [edx + esi], xmm0
    lea       edx, [edx + 2 * esi]
    movhpd    qword ptr [ebx + ebp], xmm0
    lea       ebx, [ebx + 2 * ebp]
    movdqa    xmm0, xmm3   // use xmm0 as the temp register.
    punpckldq xmm3, xmm7
    movlpd    qword ptr [edx], xmm3
    movhpd    qword ptr [ebx], xmm3
    punpckhdq xmm0, xmm7
282
    sub       ecx, 8
283 284 285 286
    movlpd    qword ptr [edx + esi], xmm0
    lea       edx, [edx + 2 * esi]
    movhpd    qword ptr [ebx + ebp], xmm0
    lea       ebx, [ebx + 2 * ebp]
287
    jg        convertloop
288 289 290 291 292 293 294 295 296

    mov       esp, [esp + 16]
    pop       ebp
    pop       edi
    pop       esi
    pop       ebx
    ret
  }
}
297
#elif !defined(LIBYUV_DISABLE_X86) && (defined(__i386__) || defined(__x86_64__))
298 299 300
#define HAS_TRANSPOSE_WX8_SSSE3
static void TransposeWx8_SSSE3(const uint8* src, int src_stride,
                               uint8* dst, int dst_stride, int width) {
301
  asm volatile (
302 303
    // Read in the data from the source pointer.
    // First round of bit swap.
304
    ".p2align  4                                 \n"
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372
  "1:                                            \n"
    "movq       (%0),%%xmm0                      \n"
    "movq       (%0,%3),%%xmm1                   \n"
    "lea        (%0,%3,2),%0                     \n"
    "punpcklbw  %%xmm1,%%xmm0                    \n"
    "movq       (%0),%%xmm2                      \n"
    "movdqa     %%xmm0,%%xmm1                    \n"
    "palignr    $0x8,%%xmm1,%%xmm1               \n"
    "movq       (%0,%3),%%xmm3                   \n"
    "lea        (%0,%3,2),%0                     \n"
    "punpcklbw  %%xmm3,%%xmm2                    \n"
    "movdqa     %%xmm2,%%xmm3                    \n"
    "movq       (%0),%%xmm4                      \n"
    "palignr    $0x8,%%xmm3,%%xmm3               \n"
    "movq       (%0,%3),%%xmm5                   \n"
    "lea        (%0,%3,2),%0                     \n"
    "punpcklbw  %%xmm5,%%xmm4                    \n"
    "movdqa     %%xmm4,%%xmm5                    \n"
    "movq       (%0),%%xmm6                      \n"
    "palignr    $0x8,%%xmm5,%%xmm5               \n"
    "movq       (%0,%3),%%xmm7                   \n"
    "lea        (%0,%3,2),%0                     \n"
    "punpcklbw  %%xmm7,%%xmm6                    \n"
    "neg        %3                               \n"
    "movdqa     %%xmm6,%%xmm7                    \n"
    "lea        0x8(%0,%3,8),%0                  \n"
    "palignr    $0x8,%%xmm7,%%xmm7               \n"
    "neg        %3                               \n"
     // Second round of bit swap.
    "punpcklwd  %%xmm2,%%xmm0                    \n"
    "punpcklwd  %%xmm3,%%xmm1                    \n"
    "movdqa     %%xmm0,%%xmm2                    \n"
    "movdqa     %%xmm1,%%xmm3                    \n"
    "palignr    $0x8,%%xmm2,%%xmm2               \n"
    "palignr    $0x8,%%xmm3,%%xmm3               \n"
    "punpcklwd  %%xmm6,%%xmm4                    \n"
    "punpcklwd  %%xmm7,%%xmm5                    \n"
    "movdqa     %%xmm4,%%xmm6                    \n"
    "movdqa     %%xmm5,%%xmm7                    \n"
    "palignr    $0x8,%%xmm6,%%xmm6               \n"
    "palignr    $0x8,%%xmm7,%%xmm7               \n"
    // Third round of bit swap.
    // Write to the destination pointer.
    "punpckldq  %%xmm4,%%xmm0                    \n"
    "movq       %%xmm0,(%1)                      \n"
    "movdqa     %%xmm0,%%xmm4                    \n"
    "palignr    $0x8,%%xmm4,%%xmm4               \n"
    "movq       %%xmm4,(%1,%4)                   \n"
    "lea        (%1,%4,2),%1                     \n"
    "punpckldq  %%xmm6,%%xmm2                    \n"
    "movdqa     %%xmm2,%%xmm6                    \n"
    "movq       %%xmm2,(%1)                      \n"
    "palignr    $0x8,%%xmm6,%%xmm6               \n"
    "punpckldq  %%xmm5,%%xmm1                    \n"
    "movq       %%xmm6,(%1,%4)                   \n"
    "lea        (%1,%4,2),%1                     \n"
    "movdqa     %%xmm1,%%xmm5                    \n"
    "movq       %%xmm1,(%1)                      \n"
    "palignr    $0x8,%%xmm5,%%xmm5               \n"
    "movq       %%xmm5,(%1,%4)                   \n"
    "lea        (%1,%4,2),%1                     \n"
    "punpckldq  %%xmm7,%%xmm3                    \n"
    "movq       %%xmm3,(%1)                      \n"
    "movdqa     %%xmm3,%%xmm7                    \n"
    "palignr    $0x8,%%xmm7,%%xmm7               \n"
    "sub        $0x8,%2                          \n"
    "movq       %%xmm7,(%1,%4)                   \n"
    "lea        (%1,%4,2),%1                     \n"
373
    "jg         1b                               \n"
374 375 376 377 378 379 380 381 382 383
    : "+r"(src),    // %0
      "+r"(dst),    // %1
      "+r"(width)   // %2
    : "r"(static_cast<intptr_t>(src_stride)),  // %3
      "r"(static_cast<intptr_t>(dst_stride))   // %4
    : "memory", "cc"
  #if defined(__SSE2__)
      , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"
  #endif
  );
384 385
}

386
#if !defined(LIBYUV_DISABLE_X86) && defined (__i386__)
387
#define HAS_TRANSPOSE_UVWX8_SSE2
arphaxad.cy@gmail.com's avatar
arphaxad.cy@gmail.com committed
388 389 390 391
extern "C" void TransposeUVWx8_SSE2(const uint8* src, int src_stride,
                                    uint8* dst_a, int dst_stride_a,
                                    uint8* dst_b, int dst_stride_b,
                                    int w);
frkoenig@google.com's avatar
frkoenig@google.com committed
392
  asm (
393
    DECLARE_FUNCTION(TransposeUVWx8_SSE2)
394 395 396 397 398 399 400 401 402 403 404 405 406 407 408
    "push   %ebx                               \n"
    "push   %esi                               \n"
    "push   %edi                               \n"
    "push   %ebp                               \n"
    "mov    0x14(%esp),%eax                    \n"
    "mov    0x18(%esp),%edi                    \n"
    "mov    0x1c(%esp),%edx                    \n"
    "mov    0x20(%esp),%esi                    \n"
    "mov    0x24(%esp),%ebx                    \n"
    "mov    0x28(%esp),%ebp                    \n"
    "mov    %esp,%ecx                          \n"
    "sub    $0x14,%esp                         \n"
    "and    $0xfffffff0,%esp                   \n"
    "mov    %ecx,0x10(%esp)                    \n"
    "mov    0x2c(%ecx),%ecx                    \n"
arphaxad.cy@gmail.com's avatar
arphaxad.cy@gmail.com committed
409

410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494
"1:                                            \n"
    "movdqa (%eax),%xmm0                       \n"
    "movdqa (%eax,%edi,1),%xmm1                \n"
    "lea    (%eax,%edi,2),%eax                 \n"
    "movdqa %xmm0,%xmm7                        \n"
    "punpcklbw %xmm1,%xmm0                     \n"
    "punpckhbw %xmm1,%xmm7                     \n"
    "movdqa %xmm7,%xmm1                        \n"
    "movdqa (%eax),%xmm2                       \n"
    "movdqa (%eax,%edi,1),%xmm3                \n"
    "lea    (%eax,%edi,2),%eax                 \n"
    "movdqa %xmm2,%xmm7                        \n"
    "punpcklbw %xmm3,%xmm2                     \n"
    "punpckhbw %xmm3,%xmm7                     \n"
    "movdqa %xmm7,%xmm3                        \n"
    "movdqa (%eax),%xmm4                       \n"
    "movdqa (%eax,%edi,1),%xmm5                \n"
    "lea    (%eax,%edi,2),%eax                 \n"
    "movdqa %xmm4,%xmm7                        \n"
    "punpcklbw %xmm5,%xmm4                     \n"
    "punpckhbw %xmm5,%xmm7                     \n"
    "movdqa %xmm7,%xmm5                        \n"
    "movdqa (%eax),%xmm6                       \n"
    "movdqa (%eax,%edi,1),%xmm7                \n"
    "lea    (%eax,%edi,2),%eax                 \n"
    "movdqa %xmm5,(%esp)                       \n"
    "neg    %edi                               \n"
    "movdqa %xmm6,%xmm5                        \n"
    "punpcklbw %xmm7,%xmm6                     \n"
    "punpckhbw %xmm7,%xmm5                     \n"
    "movdqa %xmm5,%xmm7                        \n"
    "lea    0x10(%eax,%edi,8),%eax             \n"
    "neg    %edi                               \n"
    "movdqa %xmm0,%xmm5                        \n"
    "punpcklwd %xmm2,%xmm0                     \n"
    "punpckhwd %xmm2,%xmm5                     \n"
    "movdqa %xmm5,%xmm2                        \n"
    "movdqa %xmm1,%xmm5                        \n"
    "punpcklwd %xmm3,%xmm1                     \n"
    "punpckhwd %xmm3,%xmm5                     \n"
    "movdqa %xmm5,%xmm3                        \n"
    "movdqa %xmm4,%xmm5                        \n"
    "punpcklwd %xmm6,%xmm4                     \n"
    "punpckhwd %xmm6,%xmm5                     \n"
    "movdqa %xmm5,%xmm6                        \n"
    "movdqa (%esp),%xmm5                       \n"
    "movdqa %xmm6,(%esp)                       \n"
    "movdqa %xmm5,%xmm6                        \n"
    "punpcklwd %xmm7,%xmm5                     \n"
    "punpckhwd %xmm7,%xmm6                     \n"
    "movdqa %xmm6,%xmm7                        \n"
    "movdqa %xmm0,%xmm6                        \n"
    "punpckldq %xmm4,%xmm0                     \n"
    "punpckhdq %xmm4,%xmm6                     \n"
    "movdqa %xmm6,%xmm4                        \n"
    "movdqa (%esp),%xmm6                       \n"
    "movlpd %xmm0,(%edx)                       \n"
    "movhpd %xmm0,(%ebx)                       \n"
    "movlpd %xmm4,(%edx,%esi,1)                \n"
    "lea    (%edx,%esi,2),%edx                 \n"
    "movhpd %xmm4,(%ebx,%ebp,1)                \n"
    "lea    (%ebx,%ebp,2),%ebx                 \n"
    "movdqa %xmm2,%xmm0                        \n"
    "punpckldq %xmm6,%xmm2                     \n"
    "movlpd %xmm2,(%edx)                       \n"
    "movhpd %xmm2,(%ebx)                       \n"
    "punpckhdq %xmm6,%xmm0                     \n"
    "movlpd %xmm0,(%edx,%esi,1)                \n"
    "lea    (%edx,%esi,2),%edx                 \n"
    "movhpd %xmm0,(%ebx,%ebp,1)                \n"
    "lea    (%ebx,%ebp,2),%ebx                 \n"
    "movdqa %xmm1,%xmm0                        \n"
    "punpckldq %xmm5,%xmm1                     \n"
    "movlpd %xmm1,(%edx)                       \n"
    "movhpd %xmm1,(%ebx)                       \n"
    "punpckhdq %xmm5,%xmm0                     \n"
    "movlpd %xmm0,(%edx,%esi,1)                \n"
    "lea    (%edx,%esi,2),%edx                 \n"
    "movhpd %xmm0,(%ebx,%ebp,1)                \n"
    "lea    (%ebx,%ebp,2),%ebx                 \n"
    "movdqa %xmm3,%xmm0                        \n"
    "punpckldq %xmm7,%xmm3                     \n"
    "movlpd %xmm3,(%edx)                       \n"
    "movhpd %xmm3,(%ebx)                       \n"
    "punpckhdq %xmm7,%xmm0                     \n"
495
    "sub    $0x8,%ecx                          \n"
496 497 498 499
    "movlpd %xmm0,(%edx,%esi,1)                \n"
    "lea    (%edx,%esi,2),%edx                 \n"
    "movhpd %xmm0,(%ebx,%ebp,1)                \n"
    "lea    (%ebx,%ebp,2),%ebx                 \n"
500
    "jg     1b                                 \n"
501 502 503 504 505 506
    "mov    0x10(%esp),%esp                    \n"
    "pop    %ebp                               \n"
    "pop    %edi                               \n"
    "pop    %esi                               \n"
    "pop    %ebx                               \n"
    "ret                                       \n"
arphaxad.cy@gmail.com's avatar
arphaxad.cy@gmail.com committed
507
);
508
#elif !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__)
509 510 511 512
// 64 bit version has enough registers to do 16x8 to 8x16 at a time.
#define HAS_TRANSPOSE_WX8_FAST_SSSE3
static void TransposeWx8_FAST_SSSE3(const uint8* src, int src_stride,
                                    uint8* dst, int dst_stride, int width) {
513
  asm volatile (
514 515
  // Read in the data from the source pointer.
  // First round of bit swap.
516
  ".p2align  4                                 \n"
517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560
"1:                                            \n"
  "movdqa     (%0),%%xmm0                      \n"
  "movdqa     (%0,%3),%%xmm1                   \n"
  "lea        (%0,%3,2),%0                     \n"
  "movdqa     %%xmm0,%%xmm8                    \n"
  "punpcklbw  %%xmm1,%%xmm0                    \n"
  "punpckhbw  %%xmm1,%%xmm8                    \n"
  "movdqa     (%0),%%xmm2                      \n"
  "movdqa     %%xmm0,%%xmm1                    \n"
  "movdqa     %%xmm8,%%xmm9                    \n"
  "palignr    $0x8,%%xmm1,%%xmm1               \n"
  "palignr    $0x8,%%xmm9,%%xmm9               \n"
  "movdqa     (%0,%3),%%xmm3                   \n"
  "lea        (%0,%3,2),%0                     \n"
  "movdqa     %%xmm2,%%xmm10                   \n"
  "punpcklbw  %%xmm3,%%xmm2                    \n"
  "punpckhbw  %%xmm3,%%xmm10                   \n"
  "movdqa     %%xmm2,%%xmm3                    \n"
  "movdqa     %%xmm10,%%xmm11                  \n"
  "movdqa     (%0),%%xmm4                      \n"
  "palignr    $0x8,%%xmm3,%%xmm3               \n"
  "palignr    $0x8,%%xmm11,%%xmm11             \n"
  "movdqa     (%0,%3),%%xmm5                   \n"
  "lea        (%0,%3,2),%0                     \n"
  "movdqa     %%xmm4,%%xmm12                   \n"
  "punpcklbw  %%xmm5,%%xmm4                    \n"
  "punpckhbw  %%xmm5,%%xmm12                   \n"
  "movdqa     %%xmm4,%%xmm5                    \n"
  "movdqa     %%xmm12,%%xmm13                  \n"
  "movdqa     (%0),%%xmm6                      \n"
  "palignr    $0x8,%%xmm5,%%xmm5               \n"
  "palignr    $0x8,%%xmm13,%%xmm13             \n"
  "movdqa     (%0,%3),%%xmm7                   \n"
  "lea        (%0,%3,2),%0                     \n"
  "movdqa     %%xmm6,%%xmm14                   \n"
  "punpcklbw  %%xmm7,%%xmm6                    \n"
  "punpckhbw  %%xmm7,%%xmm14                   \n"
  "neg        %3                               \n"
  "movdqa     %%xmm6,%%xmm7                    \n"
  "movdqa     %%xmm14,%%xmm15                  \n"
  "lea        0x10(%0,%3,8),%0                 \n"
  "palignr    $0x8,%%xmm7,%%xmm7               \n"
  "palignr    $0x8,%%xmm15,%%xmm15             \n"
  "neg        %3                               \n"
561
   // Second round of bit swap.
562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585
  "punpcklwd  %%xmm2,%%xmm0                    \n"
  "punpcklwd  %%xmm3,%%xmm1                    \n"
  "movdqa     %%xmm0,%%xmm2                    \n"
  "movdqa     %%xmm1,%%xmm3                    \n"
  "palignr    $0x8,%%xmm2,%%xmm2               \n"
  "palignr    $0x8,%%xmm3,%%xmm3               \n"
  "punpcklwd  %%xmm6,%%xmm4                    \n"
  "punpcklwd  %%xmm7,%%xmm5                    \n"
  "movdqa     %%xmm4,%%xmm6                    \n"
  "movdqa     %%xmm5,%%xmm7                    \n"
  "palignr    $0x8,%%xmm6,%%xmm6               \n"
  "palignr    $0x8,%%xmm7,%%xmm7               \n"
  "punpcklwd  %%xmm10,%%xmm8                   \n"
  "punpcklwd  %%xmm11,%%xmm9                   \n"
  "movdqa     %%xmm8,%%xmm10                   \n"
  "movdqa     %%xmm9,%%xmm11                   \n"
  "palignr    $0x8,%%xmm10,%%xmm10             \n"
  "palignr    $0x8,%%xmm11,%%xmm11             \n"
  "punpcklwd  %%xmm14,%%xmm12                  \n"
  "punpcklwd  %%xmm15,%%xmm13                  \n"
  "movdqa     %%xmm12,%%xmm14                  \n"
  "movdqa     %%xmm13,%%xmm15                  \n"
  "palignr    $0x8,%%xmm14,%%xmm14             \n"
  "palignr    $0x8,%%xmm15,%%xmm15             \n"
586 587
  // Third round of bit swap.
  // Write to the destination pointer.
588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633
  "punpckldq  %%xmm4,%%xmm0                    \n"
  "movq       %%xmm0,(%1)                      \n"
  "movdqa     %%xmm0,%%xmm4                    \n"
  "palignr    $0x8,%%xmm4,%%xmm4               \n"
  "movq       %%xmm4,(%1,%4)                   \n"
  "lea        (%1,%4,2),%1                     \n"
  "punpckldq  %%xmm6,%%xmm2                    \n"
  "movdqa     %%xmm2,%%xmm6                    \n"
  "movq       %%xmm2,(%1)                      \n"
  "palignr    $0x8,%%xmm6,%%xmm6               \n"
  "punpckldq  %%xmm5,%%xmm1                    \n"
  "movq       %%xmm6,(%1,%4)                   \n"
  "lea        (%1,%4,2),%1                     \n"
  "movdqa     %%xmm1,%%xmm5                    \n"
  "movq       %%xmm1,(%1)                      \n"
  "palignr    $0x8,%%xmm5,%%xmm5               \n"
  "movq       %%xmm5,(%1,%4)                   \n"
  "lea        (%1,%4,2),%1                     \n"
  "punpckldq  %%xmm7,%%xmm3                    \n"
  "movq       %%xmm3,(%1)                      \n"
  "movdqa     %%xmm3,%%xmm7                    \n"
  "palignr    $0x8,%%xmm7,%%xmm7               \n"
  "movq       %%xmm7,(%1,%4)                   \n"
  "lea        (%1,%4,2),%1                     \n"
  "punpckldq  %%xmm12,%%xmm8                   \n"
  "movq       %%xmm8,(%1)                      \n"
  "movdqa     %%xmm8,%%xmm12                   \n"
  "palignr    $0x8,%%xmm12,%%xmm12             \n"
  "movq       %%xmm12,(%1,%4)                  \n"
  "lea        (%1,%4,2),%1                     \n"
  "punpckldq  %%xmm14,%%xmm10                  \n"
  "movdqa     %%xmm10,%%xmm14                  \n"
  "movq       %%xmm10,(%1)                     \n"
  "palignr    $0x8,%%xmm14,%%xmm14             \n"
  "punpckldq  %%xmm13,%%xmm9                   \n"
  "movq       %%xmm14,(%1,%4)                  \n"
  "lea        (%1,%4,2),%1                     \n"
  "movdqa     %%xmm9,%%xmm13                   \n"
  "movq       %%xmm9,(%1)                      \n"
  "palignr    $0x8,%%xmm13,%%xmm13             \n"
  "movq       %%xmm13,(%1,%4)                  \n"
  "lea        (%1,%4,2),%1                     \n"
  "punpckldq  %%xmm15,%%xmm11                  \n"
  "movq       %%xmm11,(%1)                     \n"
  "movdqa     %%xmm11,%%xmm15                  \n"
  "palignr    $0x8,%%xmm15,%%xmm15             \n"
634
  "sub        $0x10,%2                         \n"
635 636
  "movq       %%xmm15,(%1,%4)                  \n"
  "lea        (%1,%4,2),%1                     \n"
637
  "jg         1b                               \n"
638 639 640 641 642
  : "+r"(src),    // %0
    "+r"(dst),    // %1
    "+r"(width)   // %2
  : "r"(static_cast<intptr_t>(src_stride)),  // %3
    "r"(static_cast<intptr_t>(dst_stride))   // %4
643 644 645
  : "memory", "cc",
    "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7",
    "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13",  "xmm14",  "xmm15"
646 647 648
);
}

649 650 651 652 653
#define HAS_TRANSPOSE_UVWX8_SSE2
static void TransposeUVWx8_SSE2(const uint8* src, int src_stride,
                                uint8* dst_a, int dst_stride_a,
                                uint8* dst_b, int dst_stride_b,
                                int w) {
654
  asm volatile (
655 656
  // Read in the data from the source pointer.
  // First round of bit swap.
657
  ".p2align  4                                 \n"
658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689
"1:                                            \n"
  "movdqa     (%0),%%xmm0                      \n"
  "movdqa     (%0,%4),%%xmm1                   \n"
  "lea        (%0,%4,2),%0                     \n"
  "movdqa     %%xmm0,%%xmm8                    \n"
  "punpcklbw  %%xmm1,%%xmm0                    \n"
  "punpckhbw  %%xmm1,%%xmm8                    \n"
  "movdqa     %%xmm8,%%xmm1                    \n"
  "movdqa     (%0),%%xmm2                      \n"
  "movdqa     (%0,%4),%%xmm3                   \n"
  "lea        (%0,%4,2),%0                     \n"
  "movdqa     %%xmm2,%%xmm8                    \n"
  "punpcklbw  %%xmm3,%%xmm2                    \n"
  "punpckhbw  %%xmm3,%%xmm8                    \n"
  "movdqa     %%xmm8,%%xmm3                    \n"
  "movdqa     (%0),%%xmm4                      \n"
  "movdqa     (%0,%4),%%xmm5                   \n"
  "lea        (%0,%4,2),%0                     \n"
  "movdqa     %%xmm4,%%xmm8                    \n"
  "punpcklbw  %%xmm5,%%xmm4                    \n"
  "punpckhbw  %%xmm5,%%xmm8                    \n"
  "movdqa     %%xmm8,%%xmm5                    \n"
  "movdqa     (%0),%%xmm6                      \n"
  "movdqa     (%0,%4),%%xmm7                   \n"
  "lea        (%0,%4,2),%0                     \n"
  "movdqa     %%xmm6,%%xmm8                    \n"
  "punpcklbw  %%xmm7,%%xmm6                    \n"
  "neg        %4                               \n"
  "lea        0x10(%0,%4,8),%0                 \n"
  "punpckhbw  %%xmm7,%%xmm8                    \n"
  "movdqa     %%xmm8,%%xmm7                    \n"
  "neg        %4                               \n"
690
   // Second round of bit swap.
691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706
  "movdqa     %%xmm0,%%xmm8                    \n"
  "movdqa     %%xmm1,%%xmm9                    \n"
  "punpckhwd  %%xmm2,%%xmm8                    \n"
  "punpckhwd  %%xmm3,%%xmm9                    \n"
  "punpcklwd  %%xmm2,%%xmm0                    \n"
  "punpcklwd  %%xmm3,%%xmm1                    \n"
  "movdqa     %%xmm8,%%xmm2                    \n"
  "movdqa     %%xmm9,%%xmm3                    \n"
  "movdqa     %%xmm4,%%xmm8                    \n"
  "movdqa     %%xmm5,%%xmm9                    \n"
  "punpckhwd  %%xmm6,%%xmm8                    \n"
  "punpckhwd  %%xmm7,%%xmm9                    \n"
  "punpcklwd  %%xmm6,%%xmm4                    \n"
  "punpcklwd  %%xmm7,%%xmm5                    \n"
  "movdqa     %%xmm8,%%xmm6                    \n"
  "movdqa     %%xmm9,%%xmm7                    \n"
707 708
  // Third round of bit swap.
  // Write to the destination pointer.
709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740
  "movdqa     %%xmm0,%%xmm8                    \n"
  "punpckldq  %%xmm4,%%xmm0                    \n"
  "movlpd     %%xmm0,(%1)                      \n"  // Write back U channel
  "movhpd     %%xmm0,(%2)                      \n"  // Write back V channel
  "punpckhdq  %%xmm4,%%xmm8                    \n"
  "movlpd     %%xmm8,(%1,%5)                   \n"
  "lea        (%1,%5,2),%1                     \n"
  "movhpd     %%xmm8,(%2,%6)                   \n"
  "lea        (%2,%6,2),%2                     \n"
  "movdqa     %%xmm2,%%xmm8                    \n"
  "punpckldq  %%xmm6,%%xmm2                    \n"
  "movlpd     %%xmm2,(%1)                      \n"
  "movhpd     %%xmm2,(%2)                      \n"
  "punpckhdq  %%xmm6,%%xmm8                    \n"
  "movlpd     %%xmm8,(%1,%5)                   \n"
  "lea        (%1,%5,2),%1                     \n"
  "movhpd     %%xmm8,(%2,%6)                   \n"
  "lea        (%2,%6,2),%2                     \n"
  "movdqa     %%xmm1,%%xmm8                    \n"
  "punpckldq  %%xmm5,%%xmm1                    \n"
  "movlpd     %%xmm1,(%1)                      \n"
  "movhpd     %%xmm1,(%2)                      \n"
  "punpckhdq  %%xmm5,%%xmm8                    \n"
  "movlpd     %%xmm8,(%1,%5)                   \n"
  "lea        (%1,%5,2),%1                     \n"
  "movhpd     %%xmm8,(%2,%6)                   \n"
  "lea        (%2,%6,2),%2                     \n"
  "movdqa     %%xmm3,%%xmm8                    \n"
  "punpckldq  %%xmm7,%%xmm3                    \n"
  "movlpd     %%xmm3,(%1)                      \n"
  "movhpd     %%xmm3,(%2)                      \n"
  "punpckhdq  %%xmm7,%%xmm8                    \n"
741
  "sub        $0x8,%3                          \n"
742 743 744 745
  "movlpd     %%xmm8,(%1,%5)                   \n"
  "lea        (%1,%5,2),%1                     \n"
  "movhpd     %%xmm8,(%2,%6)                   \n"
  "lea        (%2,%6,2),%2                     \n"
746
  "jg         1b                               \n"
747 748 749 750 751 752 753
  : "+r"(src),    // %0
    "+r"(dst_a),  // %1
    "+r"(dst_b),  // %2
    "+r"(w)   // %3
  : "r"(static_cast<intptr_t>(src_stride)),    // %4
    "r"(static_cast<intptr_t>(dst_stride_a)),  // %5
    "r"(static_cast<intptr_t>(dst_stride_b))   // %6
754 755 756
  : "memory", "cc",
    "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7",
    "xmm8", "xmm9"
757 758 759
);
}
#endif
760 761
#endif

762 763
static void TransposeWx8_C(const uint8* src, int src_stride,
                           uint8* dst, int dst_stride,
764 765
                           int width) {
  for (int i = 0; i < width; ++i) {
766 767 768 769 770 771 772 773 774 775 776
    dst[0] = src[0 * src_stride];
    dst[1] = src[1 * src_stride];
    dst[2] = src[2 * src_stride];
    dst[3] = src[3 * src_stride];
    dst[4] = src[4 * src_stride];
    dst[5] = src[5 * src_stride];
    dst[6] = src[6 * src_stride];
    dst[7] = src[7 * src_stride];
    ++src;
    dst += dst_stride;
  }
777 778
}

779 780 781
static void TransposeWxH_C(const uint8* src, int src_stride,
                           uint8* dst, int dst_stride,
                           int width, int height) {
782 783
  for (int i = 0; i < width; ++i) {
    for (int j = 0; j < height; ++j) {
784
      dst[i * dst_stride + j] = src[j * src_stride + i];
785 786
    }
  }
787 788
}

789
LIBYUV_API
790 791 792
void TransposePlane(const uint8* src, int src_stride,
                    uint8* dst, int dst_stride,
                    int width, int height) {
793 794 795
  void (*TransposeWx8)(const uint8* src, int src_stride,
                       uint8* dst, int dst_stride,
                       int width) = TransposeWx8_C;
796
#if defined(HAS_TRANSPOSE_WX8_NEON)
mikhal@webrtc.org's avatar
mikhal@webrtc.org committed
797
  if (TestCpuFlag(kCpuHasNEON)) {
798
    TransposeWx8 = TransposeWx8_NEON;
799 800 801 802 803 804
  }
#endif
#if defined(HAS_TRANSPOSE_WX8_SSSE3)
  if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 8)) {
    TransposeWx8 = TransposeWx8_SSSE3;
  }
805
#endif
806
#if defined(HAS_TRANSPOSE_WX8_FAST_SSSE3)
mikhal@webrtc.org's avatar
mikhal@webrtc.org committed
807
  if (TestCpuFlag(kCpuHasSSSE3) &&
808
      IS_ALIGNED(width, 16) &&
809
      IS_ALIGNED(src, 16) && IS_ALIGNED(src_stride, 16)) {
810
    TransposeWx8 = TransposeWx8_FAST_SSSE3;
811
  }
812
#endif
813 814 815 816 817 818 819 820 821 822
#if defined(HAS_TRANSPOSE_WX8_MIPS_DSPR2)
  if (TestCpuFlag(kCpuHasMIPS_DSPR2)) {
    if (IS_ALIGNED(width, 4) &&
        IS_ALIGNED(src, 4) && IS_ALIGNED(src_stride, 4)) {
      TransposeWx8 = TransposeWx8_FAST_MIPS_DSPR2;
    } else {
      TransposeWx8 = TransposeWx8_MIPS_DSPR2;
    }
  }
#endif
823

824 825
  // Work across the source in 8x8 tiles
  int i = height;
826 827
  while (i >= 8) {
    TransposeWx8(src, src_stride, dst, dst_stride, width);
828 829 830
    src += 8 * src_stride;    // Go down 8 rows.
    dst += 8;                 // Move over 8 columns.
    i -= 8;
831
  }
832

833
  TransposeWxH_C(src, src_stride, dst, dst_stride, width, i);
834 835
}

836
LIBYUV_API
837 838 839 840
void RotatePlane90(const uint8* src, int src_stride,
                   uint8* dst, int dst_stride,
                   int width, int height) {
  // Rotate by 90 is a transpose with the source read
841
  // from bottom to top. So set the source pointer to the end
842 843
  // of the buffer and flip the sign of the source stride.
  src += src_stride * (height - 1);
844
  src_stride = -src_stride;
845
  TransposePlane(src, src_stride, dst, dst_stride, width, height);
846 847
}

848
LIBYUV_API
849 850 851 852
void RotatePlane270(const uint8* src, int src_stride,
                    uint8* dst, int dst_stride,
                    int width, int height) {
  // Rotate by 270 is a transpose with the destination written
853
  // from bottom to top. So set the destination pointer to the end
854 855
  // of the buffer and flip the sign of the destination stride.
  dst += dst_stride * (width - 1);
856
  dst_stride = -dst_stride;
857
  TransposePlane(src, src_stride, dst, dst_stride, width, height);
858 859
}

860
LIBYUV_API
861 862 863
void RotatePlane180(const uint8* src, int src_stride,
                    uint8* dst, int dst_stride,
                    int width, int height) {
864
  void (*MirrorRow)(const uint8* src, uint8* dst, int width) = MirrorRow_C;
865
#if defined(HAS_MIRRORROW_NEON)
866
  if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 16)) {
867
    MirrorRow = MirrorRow_NEON;
868
  }
869
#endif
870
#if defined(HAS_MIRRORROW_SSE2)
871
  if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(width, 16) &&
872 873
      IS_ALIGNED(src, 16) && IS_ALIGNED(src_stride, 16) &&
      IS_ALIGNED(dst, 16) && IS_ALIGNED(dst_stride, 16)) {
874
    MirrorRow = MirrorRow_SSE2;
875
  }
876
#endif
877
#if defined(HAS_MIRRORROW_SSSE3)
878
  if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 16) &&
879 880 881
      IS_ALIGNED(src, 16) && IS_ALIGNED(src_stride, 16) &&
      IS_ALIGNED(dst, 16) && IS_ALIGNED(dst_stride, 16)) {
    MirrorRow = MirrorRow_SSSE3;
882
  }
883
#endif
fbarchard@google.com's avatar
fbarchard@google.com committed
884 885 886 887 888 889 890
#if defined(HAS_MIRRORROW_AVX2)
  bool clear = false;
  if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(width, 32)) {
    clear = true;
    MirrorRow = MirrorRow_AVX2;
  }
#endif
891 892 893 894 895 896
#if defined(HAS_MIRRORROW_MIPS_DSPR2)
  if (TestCpuFlag(kCpuHasMIPS_DSPR2) &&
      IS_ALIGNED(src, 4) && IS_ALIGNED(src_stride, 4) &&
      IS_ALIGNED(dst, 4) && IS_ALIGNED(dst_stride, 4)) {
    MirrorRow = MirrorRow_MIPS_DSPR2;
  }
897
#endif
898 899
  void (*CopyRow)(const uint8* src, uint8* dst, int width) = CopyRow_C;
#if defined(HAS_COPYROW_NEON)
900
  if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 32)) {
901 902
    CopyRow = CopyRow_NEON;
  }
903 904 905
#endif
#if defined(HAS_COPYROW_X86)
  if (TestCpuFlag(kCpuHasX86) && IS_ALIGNED(width, 4)) {
906
    CopyRow = CopyRow_X86;
907
  }
908
#endif
909 910 911 912 913
#if defined(HAS_COPYROW_SSE2)
  if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(width, 32) &&
      IS_ALIGNED(src, 16) && IS_ALIGNED(src_stride, 16) &&
      IS_ALIGNED(dst, 16) && IS_ALIGNED(dst_stride, 16)) {
    CopyRow = CopyRow_SSE2;
914
  }
915
#endif
fbarchard@google.com's avatar
fbarchard@google.com committed
916 917 918 919 920 921
#if defined(HAS_COPYROW_AVX2)
  // TODO(fbarchard): Detect Fast String support.
  if (TestCpuFlag(kCpuHasAVX2)) {
    CopyRow = CopyRow_AVX2;
  }
#endif
922 923 924 925
#if defined(HAS_COPYROW_MIPS)
  if (TestCpuFlag(kCpuHasMIPS)) {
    CopyRow = CopyRow_MIPS;
  }
926 927 928 929
#endif
  if (width > kMaxStride) {
    return;
  }
930
  // Swap first and last row and mirror the content. Uses a temporary row.
931 932 933 934 935 936 937 938 939
  SIMD_ALIGNED(uint8 row[kMaxStride]);
  const uint8* src_bot = src + src_stride * (height - 1);
  uint8* dst_bot = dst + dst_stride * (height - 1);
  int half_height = (height + 1) >> 1;
  // Odd height will harmlessly mirror the middle row twice.
  for (int y = 0; y < half_height; ++y) {
    MirrorRow(src, row, width);  // Mirror first row into a buffer
    src += src_stride;
    MirrorRow(src_bot, dst, width);  // Mirror last row into first row
940
    dst += dst_stride;
941 942 943
    CopyRow(row, dst_bot, width);  // Copy first mirrored row into last
    src_bot -= src_stride;
    dst_bot -= dst_stride;
944
  }
fbarchard@google.com's avatar
fbarchard@google.com committed
945 946 947 948 949
#if defined(HAS_MIRRORROW_AVX2)
  if (clear) {
    __asm vzeroupper;
  }
#endif
950 951
}

952 953 954
static void TransposeUVWx8_C(const uint8* src, int src_stride,
                             uint8* dst_a, int dst_stride_a,
                             uint8* dst_b, int dst_stride_b,
955
                             int width) {
956
  for (int i = 0; i < width; ++i) {
957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976
    dst_a[0] = src[0 * src_stride + 0];
    dst_b[0] = src[0 * src_stride + 1];
    dst_a[1] = src[1 * src_stride + 0];
    dst_b[1] = src[1 * src_stride + 1];
    dst_a[2] = src[2 * src_stride + 0];
    dst_b[2] = src[2 * src_stride + 1];
    dst_a[3] = src[3 * src_stride + 0];
    dst_b[3] = src[3 * src_stride + 1];
    dst_a[4] = src[4 * src_stride + 0];
    dst_b[4] = src[4 * src_stride + 1];
    dst_a[5] = src[5 * src_stride + 0];
    dst_b[5] = src[5 * src_stride + 1];
    dst_a[6] = src[6 * src_stride + 0];
    dst_b[6] = src[6 * src_stride + 1];
    dst_a[7] = src[7 * src_stride + 0];
    dst_b[7] = src[7 * src_stride + 1];
    src += 2;
    dst_a += dst_stride_a;
    dst_b += dst_stride_b;
  }
977 978 979 980 981
}

static void TransposeUVWxH_C(const uint8* src, int src_stride,
                             uint8* dst_a, int dst_stride_a,
                             uint8* dst_b, int dst_stride_b,
982
                             int width, int height) {
983 984
  for (int i = 0; i < width * 2; i += 2)
    for (int j = 0; j < height; ++j) {
985 986 987 988 989
      dst_a[j + ((i >> 1) * dst_stride_a)] = src[i + (j * src_stride)];
      dst_b[j + ((i >> 1) * dst_stride_b)] = src[i + (j * src_stride) + 1];
    }
}

990
LIBYUV_API
991 992 993 994
void TransposeUV(const uint8* src, int src_stride,
                 uint8* dst_a, int dst_stride_a,
                 uint8* dst_b, int dst_stride_b,
                 int width, int height) {
995 996 997 998
  void (*TransposeUVWx8)(const uint8* src, int src_stride,
                         uint8* dst_a, int dst_stride_a,
                         uint8* dst_b, int dst_stride_b,
                         int width) = TransposeUVWx8_C;
999
#if defined(HAS_TRANSPOSE_UVWX8_NEON)
mikhal@webrtc.org's avatar
mikhal@webrtc.org committed
1000
  if (TestCpuFlag(kCpuHasNEON)) {
1001 1002 1003
    TransposeUVWx8 = TransposeUVWx8_NEON;
  }
#elif defined(HAS_TRANSPOSE_UVWX8_SSE2)
mikhal@webrtc.org's avatar
mikhal@webrtc.org committed
1004
  if (TestCpuFlag(kCpuHasSSE2) &&
1005
      IS_ALIGNED(width, 8) &&
1006
      IS_ALIGNED(src, 16) && IS_ALIGNED(src_stride, 16)) {
1007
    TransposeUVWx8 = TransposeUVWx8_SSE2;
1008
  }
1009 1010 1011 1012 1013
#elif defined(HAS_TRANSPOSE_UVWx8_MIPS_DSPR2)
  if (TestCpuFlag(kCpuHasMIPS_DSPR2) && IS_ALIGNED(width, 2) &&
      IS_ALIGNED(src, 4) && IS_ALIGNED(src_stride, 4)) {
    TransposeUVWx8 = TransposeUVWx8_MIPS_DSPR2;
  }
1014
#endif
1015

1016 1017
  // Work through the source in 8x8 tiles.
  int i = height;
1018
  while (i >= 8) {
1019 1020 1021 1022 1023 1024 1025 1026
    TransposeUVWx8(src, src_stride,
                   dst_a, dst_stride_a,
                   dst_b, dst_stride_b,
                   width);
    src += 8 * src_stride;    // Go down 8 rows.
    dst_a += 8;               // Move over 8 columns.
    dst_b += 8;               // Move over 8 columns.
    i -= 8;
1027 1028
  }

1029 1030 1031 1032
  TransposeUVWxH_C(src, src_stride,
                   dst_a, dst_stride_a,
                   dst_b, dst_stride_b,
                   width, i);
1033 1034
}

1035
LIBYUV_API
1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048
void RotateUV90(const uint8* src, int src_stride,
                uint8* dst_a, int dst_stride_a,
                uint8* dst_b, int dst_stride_b,
                int width, int height) {
  src += src_stride * (height - 1);
  src_stride = -src_stride;

  TransposeUV(src, src_stride,
              dst_a, dst_stride_a,
              dst_b, dst_stride_b,
              width, height);
}

1049
LIBYUV_API
1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064
void RotateUV270(const uint8* src, int src_stride,
                 uint8* dst_a, int dst_stride_a,
                 uint8* dst_b, int dst_stride_b,
                 int width, int height) {
  dst_a += dst_stride_a * (width - 1);
  dst_b += dst_stride_b * (width - 1);
  dst_stride_a = -dst_stride_a;
  dst_stride_b = -dst_stride_b;

  TransposeUV(src, src_stride,
              dst_a, dst_stride_a,
              dst_b, dst_stride_b,
              width, height);
}

1065
// Rotate 180 is a horizontal and vertical flip.
1066
LIBYUV_API
1067 1068 1069 1070
void RotateUV180(const uint8* src, int src_stride,
                 uint8* dst_a, int dst_stride_a,
                 uint8* dst_b, int dst_stride_b,
                 int width, int height) {
1071
  void (*MirrorRowUV)(const uint8* src, uint8* dst_u, uint8* dst_v, int width) =
1072
      MirrorUVRow_C;
1073 1074
#if defined(HAS_MIRRORUVROW_NEON)
  if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) {
1075
    MirrorRowUV = MirrorUVRow_NEON;
1076 1077
  }
#elif defined(HAS_MIRRORROW_UV_SSSE3)
1078
  if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 16) &&
1079
      IS_ALIGNED(src, 16) && IS_ALIGNED(src_stride, 16)) {
1080
    MirrorRowUV = MirrorUVRow_SSSE3;
1081
  }
1082
#elif defined(HAS_MIRRORUVROW_MIPS_DSPR2)
1083 1084
  if (TestCpuFlag(kCpuHasMIPS_DSPR2) &&
      IS_ALIGNED(src, 4) && IS_ALIGNED(src_stride, 4)) {
1085
    MirrorRowUV = MirrorUVRow_MIPS_DSPR2;
1086
  }
1087
#endif
1088 1089 1090 1091

  dst_a += dst_stride_a * (height - 1);
  dst_b += dst_stride_b * (height - 1);

1092
  for (int i = 0; i < height; ++i) {
1093
    MirrorRowUV(src, dst_a, dst_b, width);
1094 1095 1096
    src += src_stride;
    dst_a -= dst_stride_a;
    dst_b -= dst_stride_b;
1097 1098 1099
  }
}

1100
LIBYUV_API
1101 1102 1103 1104 1105 1106 1107 1108
int I420Rotate(const uint8* src_y, int src_stride_y,
               const uint8* src_u, int src_stride_u,
               const uint8* src_v, int src_stride_v,
               uint8* dst_y, int dst_stride_y,
               uint8* dst_u, int dst_stride_u,
               uint8* dst_v, int dst_stride_v,
               int width, int height,
               RotationMode mode) {
1109 1110 1111 1112
  if (!src_y || !src_u || !src_v || width <= 0 || height == 0 ||
      !dst_y || !dst_u || !dst_v) {
    return -1;
  }
1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128
  int halfwidth = (width + 1) >> 1;
  int halfheight = (height + 1) >> 1;

  // Negative height means invert the image.
  if (height < 0) {
    height = -height;
    halfheight = (height + 1) >> 1;
    src_y = src_y + (height - 1) * src_stride_y;
    src_u = src_u + (halfheight - 1) * src_stride_u;
    src_v = src_v + (halfheight - 1) * src_stride_v;
    src_stride_y = -src_stride_y;
    src_stride_u = -src_stride_u;
    src_stride_v = -src_stride_v;
  }

  switch (mode) {
1129
    case kRotate0:
1130 1131 1132 1133 1134 1135 1136 1137
      // copy frame
      return I420Copy(src_y, src_stride_y,
                      src_u, src_stride_u,
                      src_v, src_stride_v,
                      dst_y, dst_stride_y,
                      dst_u, dst_stride_u,
                      dst_v, dst_stride_v,
                      width, height);
1138
    case kRotate90:
1139 1140 1141 1142 1143 1144 1145 1146 1147 1148
      RotatePlane90(src_y, src_stride_y,
                    dst_y, dst_stride_y,
                    width, height);
      RotatePlane90(src_u, src_stride_u,
                    dst_u, dst_stride_u,
                    halfwidth, halfheight);
      RotatePlane90(src_v, src_stride_v,
                    dst_v, dst_stride_v,
                    halfwidth, halfheight);
      return 0;
1149
    case kRotate270:
1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176
      RotatePlane270(src_y, src_stride_y,
                     dst_y, dst_stride_y,
                     width, height);
      RotatePlane270(src_u, src_stride_u,
                     dst_u, dst_stride_u,
                     halfwidth, halfheight);
      RotatePlane270(src_v, src_stride_v,
                     dst_v, dst_stride_v,
                     halfwidth, halfheight);
      return 0;
    case kRotate180:
      RotatePlane180(src_y, src_stride_y,
                     dst_y, dst_stride_y,
                     width, height);
      RotatePlane180(src_u, src_stride_u,
                     dst_u, dst_stride_u,
                     halfwidth, halfheight);
      RotatePlane180(src_v, src_stride_v,
                     dst_v, dst_stride_v,
                     halfwidth, halfheight);
      return 0;
    default:
      break;
  }
  return -1;
}

1177
LIBYUV_API
1178 1179 1180 1181 1182 1183 1184
int NV12ToI420Rotate(const uint8* src_y, int src_stride_y,
                     const uint8* src_uv, int src_stride_uv,
                     uint8* dst_y, int dst_stride_y,
                     uint8* dst_u, int dst_stride_u,
                     uint8* dst_v, int dst_stride_v,
                     int width, int height,
                     RotationMode mode) {
1185 1186 1187 1188
  if (!src_y || !src_uv || width <= 0 || height == 0 ||
      !dst_y || !dst_u || !dst_v) {
    return -1;
  }
1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202
  int halfwidth = (width + 1) >> 1;
  int halfheight = (height + 1) >> 1;

  // Negative height means invert the image.
  if (height < 0) {
    height = -height;
    halfheight = (height + 1) >> 1;
    src_y = src_y + (height - 1) * src_stride_y;
    src_uv = src_uv + (halfheight - 1) * src_stride_uv;
    src_stride_y = -src_stride_y;
    src_stride_uv = -src_stride_uv;
  }

  switch (mode) {
1203
    case kRotate0:
1204
      // copy frame
1205 1206
      return NV12ToI420(src_y, src_stride_y,
                        src_uv, src_stride_uv,
1207 1208 1209 1210
                        dst_y, dst_stride_y,
                        dst_u, dst_stride_u,
                        dst_v, dst_stride_v,
                        width, height);
1211
    case kRotate90:
1212 1213 1214 1215 1216 1217 1218 1219
      RotatePlane90(src_y, src_stride_y,
                    dst_y, dst_stride_y,
                    width, height);
      RotateUV90(src_uv, src_stride_uv,
                 dst_u, dst_stride_u,
                 dst_v, dst_stride_v,
                 halfwidth, halfheight);
      return 0;
1220
    case kRotate270:
1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243
      RotatePlane270(src_y, src_stride_y,
                     dst_y, dst_stride_y,
                     width, height);
      RotateUV270(src_uv, src_stride_uv,
                  dst_u, dst_stride_u,
                  dst_v, dst_stride_v,
                  halfwidth, halfheight);
      return 0;
    case kRotate180:
      RotatePlane180(src_y, src_stride_y,
                     dst_y, dst_stride_y,
                     width, height);
      RotateUV180(src_uv, src_stride_uv,
                  dst_u, dst_stride_u,
                  dst_v, dst_stride_v,
                  halfwidth, halfheight);
      return 0;
    default:
      break;
  }
  return -1;
}

1244 1245
#ifdef __cplusplus
}  // extern "C"
1246
}  // namespace libyuv
1247
#endif