scale_argb.cc 43 KB
Newer Older
1
/*
2
 *  Copyright 2011 The LibYuv Project Authors. All rights reserved.
3 4 5 6
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS. All contributing project authors may
8 9 10 11 12 13 14 15 16 17 18
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "libyuv/scale.h"

#include <assert.h>
#include <string.h>
#include <stdlib.h>  // For getenv()

#include "libyuv/cpu_id.h"
#include "libyuv/planar_functions.h"  // For CopyARGB
19
#include "libyuv/row.h"
20 21 22 23 24 25

#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif

26
// ARGB scaling uses bilinear or point, but not box filter.
27

28 29
#if !defined(LIBYUV_DISABLE_NEON) && \
    (defined(__ARM_NEON__) || defined(LIBYUV_NEON))
30
#define HAS_SCALEARGBROWDOWNEVEN_NEON
31 32
#define HAS_SCALEARGBROWDOWN2_NEON
#define HAS_SCALEARGBFILTERROWS_NEON
33 34 35 36 37 38
void ScaleARGBRowDownEven_NEON(const uint8* src_argb, int src_stride,
                               int src_stepx,
                               uint8* dst_argb, int dst_width);
void ScaleARGBRowDownEvenInt_NEON(const uint8* src_argb, int src_stride,
                                  int src_stepx,
                                  uint8* dst_argb, int dst_width);
fbarchard@google.com's avatar
fbarchard@google.com committed
39 40 41 42
void ScaleARGBRowDown2_NEON(const uint8* src_ptr, ptrdiff_t /* src_stride */,
                            uint8* dst, int dst_width);
void ScaleARGBRowDown2Int_NEON(const uint8* src_ptr, ptrdiff_t src_stride,
                               uint8* dst, int dst_width);
43 44 45
void ScaleARGBFilterRows_NEON(uint8* dst_ptr,
                              const uint8* src_ptr, ptrdiff_t src_stride,
                              int dst_width, int source_y_fraction);
46 47
#endif

48
#if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86)
49 50
#define HAS_SCALEARGBROWDOWN2_SSE2
// Reads 8 pixels, throws half away and writes 4 even pixels (0, 2, 4, 6)
51
// Alignment requirement: src_argb 16 byte aligned, dst_argb 16 byte aligned.
52
__declspec(naked) __declspec(align(16))
53
static void ScaleARGBRowDown2_SSE2(const uint8* src_argb,
54
                                   ptrdiff_t /* src_stride */,
55
                                   uint8* dst_argb, int dst_width) {
56
  __asm {
57
    mov        eax, [esp + 4]        // src_argb
58
                                     // src_stride ignored
59
    mov        edx, [esp + 12]       // dst_argb
60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77
    mov        ecx, [esp + 16]       // dst_width

    align      16
  wloop:
    movdqa     xmm0, [eax]
    movdqa     xmm1, [eax + 16]
    lea        eax,  [eax + 32]
    shufps     xmm0, xmm1, 0x88
    sub        ecx, 4
    movdqa     [edx], xmm0
    lea        edx, [edx + 16]
    jg         wloop

    ret
  }
}

// Blends 8x2 rectangle to 4x1.
78
// Alignment requirement: src_argb 16 byte aligned, dst_argb 16 byte aligned.
79
__declspec(naked) __declspec(align(16))
80
static void ScaleARGBRowDown2Int_SSE2(const uint8* src_argb,
81
                                      ptrdiff_t src_stride,
82
                                      uint8* dst_argb, int dst_width) {
83 84
  __asm {
    push       esi
85
    mov        eax, [esp + 4 + 4]    // src_argb
86
    mov        esi, [esp + 4 + 8]    // src_stride
87
    mov        edx, [esp + 4 + 12]   // dst_argb
88 89 90 91 92 93 94 95 96 97 98
    mov        ecx, [esp + 4 + 16]   // dst_width

    align      16
  wloop:
    movdqa     xmm0, [eax]
    movdqa     xmm1, [eax + 16]
    movdqa     xmm2, [eax + esi]
    movdqa     xmm3, [eax + esi + 16]
    lea        eax,  [eax + 32]
    pavgb      xmm0, xmm2            // average rows
    pavgb      xmm1, xmm3
fbarchard@google.com's avatar
fbarchard@google.com committed
99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114
    movdqa     xmm2, xmm0            // average columns (8 to 4 pixels)
    shufps     xmm0, xmm1, 0x88      // even pixels
    shufps     xmm2, xmm1, 0xdd      // odd pixels
    pavgb      xmm0, xmm2
    sub        ecx, 4
    movdqa     [edx], xmm0
    lea        edx, [edx + 16]
    jg         wloop

    pop        esi
    ret
  }
}

#define HAS_SCALEARGBROWDOWNEVEN_SSE2
// Reads 4 pixels at a time.
115
// Alignment requirement: dst_argb 16 byte aligned.
fbarchard@google.com's avatar
fbarchard@google.com committed
116
__declspec(naked) __declspec(align(16))
117
void ScaleARGBRowDownEven_SSE2(const uint8* src_argb, ptrdiff_t src_stride,
118
                               int src_stepx,
119
                               uint8* dst_argb, int dst_width) {
fbarchard@google.com's avatar
fbarchard@google.com committed
120 121 122
  __asm {
    push       ebx
    push       edi
123
    mov        eax, [esp + 8 + 4]    // src_argb
fbarchard@google.com's avatar
fbarchard@google.com committed
124 125
                                     // src_stride ignored
    mov        ebx, [esp + 8 + 12]   // src_stepx
126
    mov        edx, [esp + 8 + 16]   // dst_argb
fbarchard@google.com's avatar
fbarchard@google.com committed
127 128 129
    mov        ecx, [esp + 8 + 20]   // dst_width
    lea        ebx, [ebx * 4]
    lea        edi, [ebx + ebx * 2]
130

fbarchard@google.com's avatar
fbarchard@google.com committed
131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152
    align      16
  wloop:
    movd       xmm0, [eax]
    movd       xmm1, [eax + ebx]
    punpckldq  xmm0, xmm1
    movd       xmm2, [eax + ebx * 2]
    movd       xmm3, [eax + edi]
    lea        eax,  [eax + ebx * 4]
    punpckldq  xmm2, xmm3
    punpcklqdq xmm0, xmm2
    sub        ecx, 4
    movdqa     [edx], xmm0
    lea        edx, [edx + 16]
    jg         wloop

    pop        edi
    pop        ebx
    ret
  }
}

// Blends four 2x2 to 4x1.
153
// Alignment requirement: dst_argb 16 byte aligned.
fbarchard@google.com's avatar
fbarchard@google.com committed
154
__declspec(naked) __declspec(align(16))
155
static void ScaleARGBRowDownEvenInt_SSE2(const uint8* src_argb,
156
                                         ptrdiff_t src_stride,
fbarchard@google.com's avatar
fbarchard@google.com committed
157
                                         int src_stepx,
158
                                         uint8* dst_argb, int dst_width) {
fbarchard@google.com's avatar
fbarchard@google.com committed
159 160 161 162
  __asm {
    push       ebx
    push       esi
    push       edi
163
    mov        eax, [esp + 12 + 4]    // src_argb
fbarchard@google.com's avatar
fbarchard@google.com committed
164 165
    mov        esi, [esp + 12 + 8]    // src_stride
    mov        ebx, [esp + 12 + 12]   // src_stepx
166
    mov        edx, [esp + 12 + 16]   // dst_argb
fbarchard@google.com's avatar
fbarchard@google.com committed
167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186
    mov        ecx, [esp + 12 + 20]   // dst_width
    lea        esi, [eax + esi]      // row1 pointer
    lea        ebx, [ebx * 4]
    lea        edi, [ebx + ebx * 2]

    align      16
  wloop:
    movq       xmm0, qword ptr [eax] // row0 4 pairs
    movhps     xmm0, qword ptr [eax + ebx]
    movq       xmm1, qword ptr [eax + ebx * 2]
    movhps     xmm1, qword ptr [eax + edi]
    lea        eax,  [eax + ebx * 4]
    movq       xmm2, qword ptr [esi] // row1 4 pairs
    movhps     xmm2, qword ptr [esi + ebx]
    movq       xmm3, qword ptr [esi + ebx * 2]
    movhps     xmm3, qword ptr [esi + edi]
    lea        esi,  [esi + ebx * 4]
    pavgb      xmm0, xmm2            // average rows
    pavgb      xmm1, xmm3
    movdqa     xmm2, xmm0            // average columns (8 to 4 pixels)
187 188 189 190 191 192 193 194
    shufps     xmm0, xmm1, 0x88      // even pixels
    shufps     xmm2, xmm1, 0xdd      // odd pixels
    pavgb      xmm0, xmm2
    sub        ecx, 4
    movdqa     [edx], xmm0
    lea        edx, [edx + 16]
    jg         wloop

fbarchard@google.com's avatar
fbarchard@google.com committed
195
    pop        edi
196
    pop        esi
fbarchard@google.com's avatar
fbarchard@google.com committed
197
    pop        ebx
198 199 200 201 202
    ret
  }
}

// Bilinear row filtering combines 4x2 -> 4x1. SSE2 version.
203
#define HAS_SCALEARGBFILTERROWS_SSE2
204
__declspec(naked) __declspec(align(16))
205
void ScaleARGBFilterRows_SSE2(uint8* dst_argb, const uint8* src_argb,
206
                              ptrdiff_t src_stride, int dst_width,
207
                              int source_y_fraction) {
208 209 210
  __asm {
    push       esi
    push       edi
211 212
    mov        edi, [esp + 8 + 4]   // dst_argb
    mov        esi, [esp + 8 + 8]   // src_argb
213 214 215 216
    mov        edx, [esp + 8 + 12]  // src_stride
    mov        ecx, [esp + 8 + 16]  // dst_width
    mov        eax, [esp + 8 + 20]  // source_y_fraction (0..255)
    sub        edi, esi
217
    // Dispatch to specialized filters if applicable.
218
    cmp        eax, 0
219 220 221
    je         xloop100  // 0 / 256.  Blend 100 / 0.
    cmp        eax, 64
    je         xloop75   // 64 / 256 is 0.25.  Blend 75 / 25.
222
    cmp        eax, 128
223 224 225
    je         xloop50   // 128 / 256 is 0.50.  Blend 50 / 50.
    cmp        eax, 192
    je         xloop25   // 192 / 256 is 0.75.  Blend 25 / 75.
226

227 228
    movd       xmm5, eax            // xmm5 = y fraction
    punpcklbw  xmm5, xmm5
229
    psrlw      xmm5, 1
230
    punpcklwd  xmm5, xmm5
231 232
    punpckldq  xmm5, xmm5
    punpcklqdq xmm5, xmm5
233
    pxor       xmm4, xmm4
234 235 236

    align      16
  xloop:
237 238
    movdqa     xmm0, [esi]  // row0
    movdqa     xmm2, [esi + edx]  // row1
239 240
    movdqa     xmm1, xmm0
    movdqa     xmm3, xmm2
241 242 243 244 245 246
    punpcklbw  xmm2, xmm4
    punpckhbw  xmm3, xmm4
    punpcklbw  xmm0, xmm4
    punpckhbw  xmm1, xmm4
    psubw      xmm2, xmm0  // row1 - row0
    psubw      xmm3, xmm1
247 248
    paddw      xmm2, xmm2  // 9 bits * 15 bits = 8.16
    paddw      xmm3, xmm3
249 250 251 252
    pmulhw     xmm2, xmm5  // scale diff
    pmulhw     xmm3, xmm5
    paddw      xmm0, xmm2  // sum rows
    paddw      xmm1, xmm3
253 254 255 256 257
    packuswb   xmm0, xmm1
    sub        ecx, 4
    movdqa     [esi + edi], xmm0
    lea        esi, [esi + 16]
    jg         xloop
258
    jmp        xloop99
259

260 261 262 263 264 265 266 267 268 269 270 271
    // Blend 25 / 75.
    align      16
  xloop25:
    movdqa     xmm0, [esi]
    movdqa     xmm1, [esi + edx]
    pavgb      xmm0, xmm1
    pavgb      xmm0, xmm1
    sub        ecx, 4
    movdqa     [esi + edi], xmm0
    lea        esi, [esi + 16]
    jg         xloop25
    jmp        xloop99
272

273
    // Blend 50 / 50.
274
    align      16
275
  xloop50:
276
    movdqa     xmm0, [esi]
277 278
    movdqa     xmm1, [esi + edx]
    pavgb      xmm0, xmm1
279 280 281
    sub        ecx, 4
    movdqa     [esi + edi], xmm0
    lea        esi, [esi + 16]
282 283
    jg         xloop50
    jmp        xloop99
284

285 286 287 288 289 290 291 292
    // Blend 75 / 25.
    align      16
  xloop75:
    movdqa     xmm1, [esi]
    movdqa     xmm0, [esi + edx]
    pavgb      xmm0, xmm1
    pavgb      xmm0, xmm1
    sub        ecx, 4
293
    movdqa     [esi + edi], xmm0
294 295 296
    lea        esi, [esi + 16]
    jg         xloop75
    jmp        xloop99
297

298
    // Blend 100 / 0 - Copy row unchanged.
299
    align      16
300
  xloop100:
301 302 303 304
    movdqa     xmm0, [esi]
    sub        ecx, 4
    movdqa     [esi + edi], xmm0
    lea        esi, [esi + 16]
305
    jg         xloop100
306

307
  xloop99:
308
    shufps     xmm0, xmm0, 0xff
309
    movdqa     [esi + edi], xmm0    // duplicate last pixel for filtering
310 311 312 313 314 315 316 317 318
    pop        edi
    pop        esi
    ret
  }
}

// Bilinear row filtering combines 4x2 -> 4x1. SSSE3 version.
#define HAS_SCALEARGBFILTERROWS_SSSE3
__declspec(naked) __declspec(align(16))
319
void ScaleARGBFilterRows_SSSE3(uint8* dst_argb, const uint8* src_argb,
320
                               ptrdiff_t src_stride, int dst_width,
321
                               int source_y_fraction) {
322 323 324
  __asm {
    push       esi
    push       edi
325 326
    mov        edi, [esp + 8 + 4]   // dst_argb
    mov        esi, [esp + 8 + 8]   // src_argb
327 328 329
    mov        edx, [esp + 8 + 12]  // src_stride
    mov        ecx, [esp + 8 + 16]  // dst_width
    mov        eax, [esp + 8 + 20]  // source_y_fraction (0..255)
330
    sub        edi, esi
331
    shr        eax, 1
332 333 334 335
    cmp        eax, 0  // dispatch to specialized filters if applicable.
    je         xloop100
    cmp        eax, 32
    je         xloop75
336
    cmp        eax, 64
337 338 339 340
    je         xloop50
    cmp        eax, 96
    je         xloop25

341 342 343 344 345
    movd       xmm0, eax  // high fraction 0..127
    neg        eax
    add        eax, 128
    movd       xmm5, eax  // low fraction 128..1
    punpcklbw  xmm5, xmm0
346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364
    punpcklwd  xmm5, xmm5
    pshufd     xmm5, xmm5, 0

    align      16
  xloop:
    movdqa     xmm0, [esi]
    movdqa     xmm2, [esi + edx]
    movdqa     xmm1, xmm0
    punpcklbw  xmm0, xmm2
    punpckhbw  xmm1, xmm2
    pmaddubsw  xmm0, xmm5
    pmaddubsw  xmm1, xmm5
    psrlw      xmm0, 7
    psrlw      xmm1, 7
    packuswb   xmm0, xmm1
    sub        ecx, 4
    movdqa     [esi + edi], xmm0
    lea        esi, [esi + 16]
    jg         xloop
365
    jmp        xloop99
366

367 368 369 370 371 372 373 374 375 376 377 378
    // Blend 25 / 75.
    align      16
  xloop25:
    movdqa     xmm0, [esi]
    movdqa     xmm1, [esi + edx]
    pavgb      xmm0, xmm1
    pavgb      xmm0, xmm1
    sub        ecx, 4
    movdqa     [esi + edi], xmm0
    lea        esi, [esi + 16]
    jg         xloop25
    jmp        xloop99
379

380
    // Blend 50 / 50.
381
    align      16
382
  xloop50:
383
    movdqa     xmm0, [esi]
384 385
    movdqa     xmm1, [esi + edx]
    pavgb      xmm0, xmm1
386 387 388
    sub        ecx, 4
    movdqa     [esi + edi], xmm0
    lea        esi, [esi + 16]
389 390
    jg         xloop50
    jmp        xloop99
391

392 393 394 395 396 397 398 399
    // Blend 75 / 25.
    align      16
  xloop75:
    movdqa     xmm1, [esi]
    movdqa     xmm0, [esi + edx]
    pavgb      xmm0, xmm1
    pavgb      xmm0, xmm1
    sub        ecx, 4
400
    movdqa     [esi + edi], xmm0
401 402 403
    lea        esi, [esi + 16]
    jg         xloop75
    jmp        xloop99
404

405
    // Blend 100 / 0 - Copy row unchanged.
406
    align      16
407
  xloop100:
408 409 410 411
    movdqa     xmm0, [esi]
    sub        ecx, 4
    movdqa     [esi + edi], xmm0
    lea        esi, [esi + 16]
412
    jg         xloop100
413

414 415
    // Extrude last pixel.
  xloop99:
416 417 418 419 420 421 422 423
    shufps     xmm0, xmm0, 0xff
    movdqa     [esi + edi], xmm0
    pop        edi
    pop        esi
    ret
  }
}

424
#elif !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) || defined(__i386__))
425 426 427 428
// GCC versions of row functions are verbatim conversions from Visual C.
// Generated using gcc disassembly on Visual C object file:
// objdump -D yuvscaler.obj >yuvscaler.txt
#define HAS_SCALEARGBROWDOWN2_SSE2
429
static void ScaleARGBRowDown2_SSE2(const uint8* src_argb,
430
                                   ptrdiff_t /* src_stride */,
431
                                   uint8* dst_argb, int dst_width) {
432 433 434 435 436 437 438 439 440 441 442
  asm volatile (
    ".p2align  4                               \n"
  "1:                                          \n"
    "movdqa    (%0),%%xmm0                     \n"
    "movdqa    0x10(%0),%%xmm1                 \n"
    "lea       0x20(%0),%0                     \n"
    "shufps    $0x88,%%xmm1,%%xmm0             \n"
    "sub       $0x4,%2                         \n"
    "movdqa    %%xmm0,(%1)                     \n"
    "lea       0x10(%1),%1                     \n"
    "jg        1b                              \n"
443 444
  : "+r"(src_argb),   // %0
    "+r"(dst_argb),   // %1
445 446 447 448 449 450 451 452 453
    "+r"(dst_width)  // %2
  :
  : "memory", "cc"
#if defined(__SSE2__)
    , "xmm0", "xmm1"
#endif
  );
}

454
static void ScaleARGBRowDown2Int_SSE2(const uint8* src_argb,
455
                                      ptrdiff_t src_stride,
456
                                      uint8* dst_argb, int dst_width) {
457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474
  asm volatile (
    ".p2align  4                               \n"
  "1:                                          \n"
    "movdqa    (%0),%%xmm0                     \n"
    "movdqa    0x10(%0),%%xmm1                 \n"
    "movdqa    (%0,%3,1),%%xmm2                \n"
    "movdqa    0x10(%0,%3,1),%%xmm3            \n"
    "lea       0x20(%0),%0                     \n"
    "pavgb     %%xmm2,%%xmm0                   \n"
    "pavgb     %%xmm3,%%xmm1                   \n"
    "movdqa    %%xmm0,%%xmm2                   \n"
    "shufps    $0x88,%%xmm1,%%xmm0             \n"
    "shufps    $0xdd,%%xmm1,%%xmm2             \n"
    "pavgb     %%xmm2,%%xmm0                   \n"
    "sub       $0x4,%2                         \n"
    "movdqa    %%xmm0,(%1)                     \n"
    "lea       0x10(%1),%1                     \n"
    "jg        1b                              \n"
475 476
  : "+r"(src_argb),    // %0
    "+r"(dst_argb),    // %1
477 478 479
    "+r"(dst_width)   // %2
  : "r"(static_cast<intptr_t>(src_stride))   // %3
  : "memory", "cc"
480 481 482 483 484 485 486 487
#if defined(__SSE2__)
    , "xmm0", "xmm1", "xmm2", "xmm3"
#endif
  );
}

#define HAS_SCALEARGBROWDOWNEVEN_SSE2
// Reads 4 pixels at a time.
488 489
// Alignment requirement: dst_argb 16 byte aligned.
void ScaleARGBRowDownEven_SSE2(const uint8* src_argb, ptrdiff_t src_stride,
490
                               int src_stepx,
491
                               uint8* dst_argb, int dst_width) {
492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510
  intptr_t src_stepx_x4 = static_cast<intptr_t>(src_stepx);
  intptr_t src_stepx_x12 = 0;
  asm volatile (
    "lea       0x0(,%1,4),%1                   \n"
    "lea       (%1,%1,2),%4                    \n"
    ".p2align  4                               \n"
  "1:                                          \n"
    "movd      (%0),%%xmm0                     \n"
    "movd      (%0,%1,1),%%xmm1                \n"
    "punpckldq %%xmm1,%%xmm0                   \n"
    "movd      (%0,%1,2),%%xmm2                \n"
    "movd      (%0,%4,1),%%xmm3                \n"
    "lea       (%0,%1,4),%0                    \n"
    "punpckldq %%xmm3,%%xmm2                   \n"
    "punpcklqdq %%xmm2,%%xmm0                  \n"
    "sub       $0x4,%3                         \n"
    "movdqa    %%xmm0,(%2)                     \n"
    "lea       0x10(%2),%2                     \n"
    "jg        1b                              \n"
511
  : "+r"(src_argb),       // %0
512
    "+r"(src_stepx_x4),  // %1
513
    "+r"(dst_argb),       // %2
514 515 516 517 518 519 520 521 522 523 524
    "+r"(dst_width),     // %3
    "+r"(src_stepx_x12)  // %4
  :
  : "memory", "cc"
#if defined(__SSE2__)
    , "xmm0", "xmm1", "xmm2", "xmm3"
#endif
  );
}

// Blends four 2x2 to 4x1.
525 526
// Alignment requirement: dst_argb 16 byte aligned.
static void ScaleARGBRowDownEvenInt_SSE2(const uint8* src_argb,
527
                                         ptrdiff_t src_stride, int src_stepx,
528
                                         uint8* dst_argb, int dst_width) {
529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557
  intptr_t src_stepx_x4 = static_cast<intptr_t>(src_stepx);
  intptr_t src_stepx_x12 = 0;
  intptr_t row1 = static_cast<intptr_t>(src_stride);
  asm volatile (
    "lea       0x0(,%1,4),%1                   \n"
    "lea       (%1,%1,2),%4                    \n"
    "lea       (%0,%5,1),%5                    \n"
    ".p2align  4                               \n"
  "1:                                          \n"
    "movq      (%0),%%xmm0                     \n"
    "movhps    (%0,%1,1),%%xmm0                \n"
    "movq      (%0,%1,2),%%xmm1                \n"
    "movhps    (%0,%4,1),%%xmm1                \n"
    "lea       (%0,%1,4),%0                    \n"
    "movq      (%5),%%xmm2                     \n"
    "movhps    (%5,%1,1),%%xmm2                \n"
    "movq      (%5,%1,2),%%xmm3                \n"
    "movhps    (%5,%4,1),%%xmm3                \n"
    "lea       (%5,%1,4),%5                    \n"
    "pavgb     %%xmm2,%%xmm0                   \n"
    "pavgb     %%xmm3,%%xmm1                   \n"
    "movdqa    %%xmm0,%%xmm2                   \n"
    "shufps    $0x88,%%xmm1,%%xmm0             \n"
    "shufps    $0xdd,%%xmm1,%%xmm2             \n"
    "pavgb     %%xmm2,%%xmm0                   \n"
    "sub       $0x4,%3                         \n"
    "movdqa    %%xmm0,(%2)                     \n"
    "lea       0x10(%2),%2                     \n"
    "jg        1b                              \n"
558
  : "+r"(src_argb),        // %0
559
    "+r"(src_stepx_x4),   // %1
560
    "+r"(dst_argb),        // %2
561 562 563 564 565
    "+rm"(dst_width),     // %3
    "+r"(src_stepx_x12),  // %4
    "+r"(row1)            // %5
  :
  : "memory", "cc"
566 567 568
#if defined(__SSE2__)
    , "xmm0", "xmm1", "xmm2", "xmm3"
#endif
569
  );
570 571 572
}

// Bilinear row filtering combines 4x2 -> 4x1. SSE2 version
573
#define HAS_SCALEARGBFILTERROWS_SSE2
574
void ScaleARGBFilterRows_SSE2(uint8* dst_argb, const uint8* src_argb,
575
                              ptrdiff_t src_stride, int dst_width,
576
                              int source_y_fraction) {
577 578
  asm volatile (
    "sub       %1,%0                           \n"
579
    "shr       %3                              \n"
580
    "cmp       $0x0,%3                         \n"
581 582 583 584 585 586 587 588 589 590 591
    "je        100f                            \n"
    "cmp       $0x20,%3                        \n"
    "je        75f                             \n"
    "cmp       $0x40,%3                        \n"
    "je        50f                             \n"
    "cmp       $0x60,%3                        \n"
    "je        25f                             \n"

    "movd      %3,%%xmm0                       \n"
    "neg       %3                              \n"
    "add       $0x80,%3                        \n"
592
    "movd      %3,%%xmm5                       \n"
593
    "punpcklbw %%xmm0,%%xmm5                   \n"
594 595 596
    "punpcklwd %%xmm5,%%xmm5                   \n"
    "pshufd    $0x0,%%xmm5,%%xmm5              \n"
    "pxor      %%xmm4,%%xmm4                   \n"
597 598

    // General purpose row blend.
599 600 601 602 603 604 605 606 607 608 609 610
    ".p2align  4                               \n"
  "1:                                          \n"
    "movdqa    (%1),%%xmm0                     \n"
    "movdqa    (%1,%4,1),%%xmm2                \n"
    "movdqa    %%xmm0,%%xmm1                   \n"
    "movdqa    %%xmm2,%%xmm3                   \n"
    "punpcklbw %%xmm4,%%xmm2                   \n"
    "punpckhbw %%xmm4,%%xmm3                   \n"
    "punpcklbw %%xmm4,%%xmm0                   \n"
    "punpckhbw %%xmm4,%%xmm1                   \n"
    "psubw     %%xmm0,%%xmm2                   \n"
    "psubw     %%xmm1,%%xmm3                   \n"
611 612
    "paddw     %%xmm2,%%xmm2                   \n"
    "paddw     %%xmm3,%%xmm3                   \n"
613 614 615 616 617 618 619 620 621
    "pmulhw    %%xmm5,%%xmm2                   \n"
    "pmulhw    %%xmm5,%%xmm3                   \n"
    "paddw     %%xmm2,%%xmm0                   \n"
    "paddw     %%xmm3,%%xmm1                   \n"
    "packuswb  %%xmm1,%%xmm0                   \n"
    "sub       $0x4,%2                         \n"
    "movdqa    %%xmm0,(%1,%0,1)                \n"
    "lea       0x10(%1),%1                     \n"
    "jg        1b                              \n"
622 623 624
    "jmp       99f                             \n"

    // Blend 25 / 75.
625
    ".p2align  4                               \n"
626
  "25:                                         \n"
627
    "movdqa    (%1),%%xmm0                     \n"
628 629 630
    "movdqa    (%1,%4,1),%%xmm1                \n"
    "pavgb     %%xmm1,%%xmm0                   \n"
    "pavgb     %%xmm1,%%xmm0                   \n"
631 632 633
    "sub       $0x4,%2                         \n"
    "movdqa    %%xmm0,(%1,%0,1)                \n"
    "lea       0x10(%1),%1                     \n"
634 635 636 637
    "jg        25b                             \n"
    "jmp       99f                             \n"

    // Blend 50 / 50.
638
    ".p2align  4                               \n"
639
  "50:                                         \n"
640
    "movdqa    (%1),%%xmm0                     \n"
641 642
    "movdqa    (%1,%4,1),%%xmm1                \n"
    "pavgb     %%xmm1,%%xmm0                   \n"
643 644 645
    "sub       $0x4,%2                         \n"
    "movdqa    %%xmm0,(%1,%0,1)                \n"
    "lea       0x10(%1),%1                     \n"
646 647 648 649 650 651 652 653 654 655 656 657
    "jg        50b                             \n"
    "jmp       99f                             \n"

    // Blend 75 / 25.
    ".p2align  4                               \n"
  "75:                                         \n"
    "movdqa    (%1),%%xmm1                     \n"
    "movdqa    (%1,%4,1),%%xmm0                \n"
    "pavgb     %%xmm1,%%xmm0                   \n"
    "pavgb     %%xmm1,%%xmm0                   \n"
    "sub       $0x4,%2                         \n"
    "movdqa    %%xmm0,(%1,%0,1)                \n"
658
    "lea       0x10(%1),%1                     \n"
659 660 661 662
    "jg        75b                             \n"
    "jmp       99f                             \n"

    // Blend 100 / 0 - Copy row unchanged.
663
    ".p2align  4                               \n"
664 665 666 667 668 669 670 671
  "100:                                        \n"
    "movdqa    (%1),%%xmm0                     \n"
    "sub       $0x4,%2                         \n"
    "movdqa    %%xmm0,(%1,%0,1)                \n"
    "lea       0x10(%1),%1                     \n"
    "jg        100b                            \n"

  "99:                                         \n"
672 673
    "shufps    $0xff,%%xmm0,%%xmm0             \n"
    "movdqa    %%xmm0,(%1,%0,1)                \n"
674 675 676
  : "+r"(dst_argb),   // %0
    "+r"(src_argb),   // %1
    "+r"(dst_width),  // %2
677 678 679
    "+r"(source_y_fraction)  // %3
  : "r"(static_cast<intptr_t>(src_stride))  // %4
  : "memory", "cc"
680
#if defined(__SSE2__)
681
    , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
682
#endif
683
  );
684 685 686 687
}

// Bilinear row filtering combines 4x2 -> 4x1. SSSE3 version
#define HAS_SCALEARGBFILTERROWS_SSSE3
688
void ScaleARGBFilterRows_SSSE3(uint8* dst_argb, const uint8* src_argb,
689
                               ptrdiff_t src_stride, int dst_width,
690
                               int source_y_fraction) {
691
  asm volatile (
692
    "sub       %1,%0                           \n"
693 694
    "shr       %3                              \n"
    "cmp       $0x0,%3                         \n"
695 696 697
    "je        100f                            \n"
    "cmp       $0x20,%3                        \n"
    "je        75f                             \n"
698
    "cmp       $0x40,%3                        \n"
699 700 701 702
    "je        50f                             \n"
    "cmp       $0x60,%3                        \n"
    "je        25f                             \n"

703 704 705 706 707 708 709
    "movd      %3,%%xmm0                       \n"
    "neg       %3                              \n"
    "add       $0x80,%3                        \n"
    "movd      %3,%%xmm5                       \n"
    "punpcklbw %%xmm0,%%xmm5                   \n"
    "punpcklwd %%xmm5,%%xmm5                   \n"
    "pshufd    $0x0,%%xmm5,%%xmm5              \n"
710 711

    // General purpose row blend.
712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727
    ".p2align  4                               \n"
  "1:                                          \n"
    "movdqa    (%1),%%xmm0                     \n"
    "movdqa    (%1,%4,1),%%xmm2                \n"
    "movdqa    %%xmm0,%%xmm1                   \n"
    "punpcklbw %%xmm2,%%xmm0                   \n"
    "punpckhbw %%xmm2,%%xmm1                   \n"
    "pmaddubsw %%xmm5,%%xmm0                   \n"
    "pmaddubsw %%xmm5,%%xmm1                   \n"
    "psrlw     $0x7,%%xmm0                     \n"
    "psrlw     $0x7,%%xmm1                     \n"
    "packuswb  %%xmm1,%%xmm0                   \n"
    "sub       $0x4,%2                         \n"
    "movdqa    %%xmm0,(%1,%0,1)                \n"
    "lea       0x10(%1),%1                     \n"
    "jg        1b                              \n"
728 729 730
    "jmp       99f                             \n"

    // Blend 25 / 75.
731
    ".p2align  4                               \n"
732
  "25:                                         \n"
733
    "movdqa    (%1),%%xmm0                     \n"
734 735 736
    "movdqa    (%1,%4,1),%%xmm1                \n"
    "pavgb     %%xmm1,%%xmm0                   \n"
    "pavgb     %%xmm1,%%xmm0                   \n"
737 738 739
    "sub       $0x4,%2                         \n"
    "movdqa    %%xmm0,(%1,%0,1)                \n"
    "lea       0x10(%1),%1                     \n"
740 741 742 743
    "jg        25b                             \n"
    "jmp       99f                             \n"

    // Blend 50 / 50.
744
    ".p2align  4                               \n"
745
  "50:                                         \n"
746
    "movdqa    (%1),%%xmm0                     \n"
747 748
    "movdqa    (%1,%4,1),%%xmm1                \n"
    "pavgb     %%xmm1,%%xmm0                   \n"
749 750 751
    "sub       $0x4,%2                         \n"
    "movdqa    %%xmm0,(%1,%0,1)                \n"
    "lea       0x10(%1),%1                     \n"
752 753 754 755
    "jg        50b                             \n"
    "jmp       99f                             \n"

    // Blend 75 / 25.
756
    ".p2align  4                               \n"
757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778
  "75:                                         \n"
    "movdqa    (%1),%%xmm1                     \n"
    "movdqa    (%1,%4,1),%%xmm0                \n"
    "pavgb     %%xmm1,%%xmm0                   \n"
    "pavgb     %%xmm1,%%xmm0                   \n"
    "sub       $0x4,%2                         \n"
    "movdqa    %%xmm0,(%1,%0,1)                \n"
    "lea       0x10(%1),%1                     \n"
    "jg        75b                             \n"
    "jmp       99f                             \n"

    // Blend 100 / 0 - Copy row unchanged.
    ".p2align  4                               \n"
  "100:                                        \n"
    "movdqa    (%1),%%xmm0                     \n"
    "sub       $0x4,%2                         \n"
    "movdqa    %%xmm0,(%1,%0,1)                \n"
    "lea       0x10(%1),%1                     \n"
    "jg        100b                            \n"

    // Extrude last pixel.
  "99:                                         \n"
779 780
    "shufps    $0xff,%%xmm0,%%xmm0             \n"
    "movdqa    %%xmm0,(%1,%0,1)                \n"
781 782
  : "+r"(dst_argb),    // %0
    "+r"(src_argb),    // %1
783
    "+r"(dst_width),  // %2
784 785 786
    "+r"(source_y_fraction)  // %3
  : "r"(static_cast<intptr_t>(src_stride))  // %4
  : "memory", "cc"
787
#if defined(__SSE2__)
788
    , "xmm0", "xmm1", "xmm2", "xmm5"
789
#endif
790
  );
791
}
792
#endif  // defined(__x86_64__) || defined(__i386__)
793

794
static void ScaleARGBRowDown2_C(const uint8* src_argb,
795
                                ptrdiff_t /* src_stride */,
796 797 798
                                uint8* dst_argb, int dst_width) {
  const uint32* src = reinterpret_cast<const uint32*>(src_argb);
  uint32* dst = reinterpret_cast<uint32*>(dst_argb);
799 800 801 802 803

  for (int x = 0; x < dst_width - 1; x += 2) {
    dst[0] = src[0];
    dst[1] = src[2];
    src += 4;
fbarchard@google.com's avatar
fbarchard@google.com committed
804
    dst += 2;
805 806 807 808 809 810
  }
  if (dst_width & 1) {
    dst[0] = src[0];
  }
}

811 812
static void ScaleARGBRowDown2Int_C(const uint8* src_argb, ptrdiff_t src_stride,
                                   uint8* dst_argb, int dst_width) {
813
  for (int x = 0; x < dst_width; ++x) {
814 815 816 817 818 819 820 821 822 823
    dst_argb[0] = (src_argb[0] + src_argb[4] +
                  src_argb[src_stride] + src_argb[src_stride + 4] + 2) >> 2;
    dst_argb[1] = (src_argb[1] + src_argb[5] +
                  src_argb[src_stride + 1] + src_argb[src_stride + 5] + 2) >> 2;
    dst_argb[2] = (src_argb[2] + src_argb[6] +
                  src_argb[src_stride + 2] + src_argb[src_stride + 6] + 2) >> 2;
    dst_argb[3] = (src_argb[3] + src_argb[7] +
                  src_argb[src_stride + 3] + src_argb[src_stride + 7] + 2) >> 2;
    src_argb += 8;
    dst_argb += 4;
fbarchard@google.com's avatar
fbarchard@google.com committed
824 825 826
  }
}

827
void ScaleARGBRowDownEven_C(const uint8* src_argb, ptrdiff_t /* src_stride */,
828
                            int src_stepx,
829 830 831
                            uint8* dst_argb, int dst_width) {
  const uint32* src = reinterpret_cast<const uint32*>(src_argb);
  uint32* dst = reinterpret_cast<uint32*>(dst_argb);
fbarchard@google.com's avatar
fbarchard@google.com committed
832 833 834 835 836 837 838 839 840 841 842 843

  for (int x = 0; x < dst_width - 1; x += 2) {
    dst[0] = src[0];
    dst[1] = src[src_stepx];
    src += src_stepx * 2;
    dst += 2;
  }
  if (dst_width & 1) {
    dst[0] = src[0];
  }
}

844
static void ScaleARGBRowDownEvenInt_C(const uint8* src_argb,
845
                                      ptrdiff_t src_stride,
fbarchard@google.com's avatar
fbarchard@google.com committed
846
                                      int src_stepx,
847
                                      uint8* dst_argb, int dst_width) {
fbarchard@google.com's avatar
fbarchard@google.com committed
848
  for (int x = 0; x < dst_width; ++x) {
849 850 851 852 853 854 855 856 857 858
    dst_argb[0] = (src_argb[0] + src_argb[4] +
                  src_argb[src_stride] + src_argb[src_stride + 4] + 2) >> 2;
    dst_argb[1] = (src_argb[1] + src_argb[5] +
                  src_argb[src_stride + 1] + src_argb[src_stride + 5] + 2) >> 2;
    dst_argb[2] = (src_argb[2] + src_argb[6] +
                  src_argb[src_stride + 2] + src_argb[src_stride + 6] + 2) >> 2;
    dst_argb[3] = (src_argb[3] + src_argb[7] +
                  src_argb[src_stride + 3] + src_argb[src_stride + 7] + 2) >> 2;
    src_argb += src_stepx * 4;
    dst_argb += 4;
859 860 861 862 863 864 865 866 867 868 869 870 871 872 873
  }
}

// (1-f)a + fb can be replaced with a + f(b-a)

#define BLENDER1(a, b, f) (static_cast<int>(a) + \
    ((f) * (static_cast<int>(b) - static_cast<int>(a)) >> 16))

#define BLENDERC(a, b, f, s) static_cast<uint32>( \
    BLENDER1(((a) >> s) & 255, ((b) >> s) & 255, f) << s)

#define BLENDER(a, b, f) \
    BLENDERC(a, b, f, 24) | BLENDERC(a, b, f, 16) | \
    BLENDERC(a, b, f, 8) | BLENDERC(a, b, f, 0)

874
static void ScaleARGBFilterCols_C(uint8* dst_argb, const uint8* src_argb,
875
                                  int dst_width, int x, int dx) {
876 877
  const uint32* src = reinterpret_cast<const uint32*>(src_argb);
  uint32* dst = reinterpret_cast<uint32*>(dst_argb);
878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899
  for (int j = 0; j < dst_width - 1; j += 2) {
    int xi = x >> 16;
    uint32 a = src[xi];
    uint32 b = src[xi + 1];
    dst[0] = BLENDER(a, b, x & 0xffff);
    x += dx;
    xi = x >> 16;
    a = src[xi];
    b = src[xi + 1];
    dst[1] = BLENDER(a, b, x & 0xffff);
    x += dx;
    dst += 2;
  }
  if (dst_width & 1) {
    int xi = x >> 16;
    uint32 a = src[xi];
    uint32 b = src[xi + 1];
    dst[0] = BLENDER(a, b, x & 0xffff);
  }
}

// C version 2x2 -> 2x1
900
void ScaleARGBFilterRows_C(uint8* dst_argb, const uint8* src_argb,
901
                           ptrdiff_t src_stride,
902
                           int dst_width, int source_y_fraction) {
903
  assert(dst_width > 0);
904 905 906 907 908 909 910 911 912 913
  // Specialized case for 100% first row.  Helps avoid reading beyond last row.
  if (source_y_fraction == 0) {
    memcpy(dst_argb, src_argb, dst_width * 4);
    dst_argb += dst_width * 4;
    dst_argb[0] = dst_argb[-4];
    dst_argb[1] = dst_argb[-3];
    dst_argb[2] = dst_argb[-2];
    dst_argb[3] = dst_argb[-1];
    return;
  }
914 915
  int y1_fraction = source_y_fraction;
  int y0_fraction = 256 - y1_fraction;
916
  const uint8* src_ptr1 = src_argb + src_stride;
917
  for (int x = 0; x < dst_width - 1; x += 2) {
918 919 920 921 922 923 924 925 926
    dst_argb[0] = (src_argb[0] * y0_fraction + src_ptr1[0] * y1_fraction) >> 8;
    dst_argb[1] = (src_argb[1] * y0_fraction + src_ptr1[1] * y1_fraction) >> 8;
    dst_argb[2] = (src_argb[2] * y0_fraction + src_ptr1[2] * y1_fraction) >> 8;
    dst_argb[3] = (src_argb[3] * y0_fraction + src_ptr1[3] * y1_fraction) >> 8;
    dst_argb[4] = (src_argb[4] * y0_fraction + src_ptr1[4] * y1_fraction) >> 8;
    dst_argb[5] = (src_argb[5] * y0_fraction + src_ptr1[5] * y1_fraction) >> 8;
    dst_argb[6] = (src_argb[6] * y0_fraction + src_ptr1[6] * y1_fraction) >> 8;
    dst_argb[7] = (src_argb[7] * y0_fraction + src_ptr1[7] * y1_fraction) >> 8;
    src_argb += 8;
927
    src_ptr1 += 8;
928
    dst_argb += 8;
929 930 931 932 933 934 935 936
  }
  if (dst_width & 1) {
    dst_argb[0] = (src_argb[0] * y0_fraction + src_ptr1[0] * y1_fraction) >> 8;
    dst_argb[1] = (src_argb[1] * y0_fraction + src_ptr1[1] * y1_fraction) >> 8;
    dst_argb[2] = (src_argb[2] * y0_fraction + src_ptr1[2] * y1_fraction) >> 8;
    dst_argb[3] = (src_argb[3] * y0_fraction + src_ptr1[3] * y1_fraction) >> 8;
    dst_argb += 4;
  }
937
  // Duplicate the last pixel (4 bytes) for filtering.
938 939 940 941
  dst_argb[0] = dst_argb[-4];
  dst_argb[1] = dst_argb[-3];
  dst_argb[2] = dst_argb[-2];
  dst_argb[3] = dst_argb[-1];
942 943
}

fbarchard@google.com's avatar
fbarchard@google.com committed
944 945 946 947
// ScaleARGB ARGB, 1/2
// This is an optimized version for scaling down a ARGB to 1/2 of
// its original size.

948
static void ScaleARGBDown2(int /* src_width */, int /* src_height */,
949 950
                           int dst_width, int dst_height,
                           int src_stride, int dst_stride,
951
                           const uint8* src_argb, uint8* dst_argb,
952
                           FilterMode filtering) {
953 954
  void (*ScaleARGBRowDown2)(const uint8* src_argb, ptrdiff_t src_stride,
                            uint8* dst_argb, int dst_width) =
955 956
      filtering ? ScaleARGBRowDown2Int_C : ScaleARGBRowDown2_C;
#if defined(HAS_SCALEARGBROWDOWN2_SSE2)
fbarchard@google.com's avatar
fbarchard@google.com committed
957
  if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 4) &&
958 959
      IS_ALIGNED(src_argb, 16) && IS_ALIGNED(src_stride, 16) &&
      IS_ALIGNED(dst_argb, 16) && IS_ALIGNED(dst_stride, 16)) {
960 961
    ScaleARGBRowDown2 = filtering ? ScaleARGBRowDown2Int_SSE2 :
        ScaleARGBRowDown2_SSE2;
962
  }
fbarchard@google.com's avatar
fbarchard@google.com committed
963 964 965 966 967 968
#elif defined(HAS_SCALEARGBROWDOWN2_NEON)
  if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(dst_width, 8) &&
      IS_ALIGNED(src_argb, 4) && IS_ALIGNED(src_stride, 4)) {
    ScaleARGBRowDown2 = filtering ? ScaleARGBRowDown2Int_NEON :
        ScaleARGBRowDown2_NEON;
  }
969 970 971 972
#endif

  // TODO(fbarchard): Loop through source height to allow odd height.
  for (int y = 0; y < dst_height; ++y) {
973 974 975
    ScaleARGBRowDown2(src_argb, src_stride, dst_argb, dst_width);
    src_argb += (src_stride << 1);
    dst_argb += dst_stride;
976 977 978
  }
}

fbarchard@google.com's avatar
fbarchard@google.com committed
979 980 981 982
// ScaleARGB ARGB Even
// This is an optimized version for scaling down a ARGB to even
// multiple of its original size.

fbarchard@google.com's avatar
fbarchard@google.com committed
983 984 985
static void ScaleARGBDownEven(int src_width, int src_height,
                              int dst_width, int dst_height,
                              int src_stride, int dst_stride,
986
                              const uint8* src_argb, uint8* dst_argb,
fbarchard@google.com's avatar
fbarchard@google.com committed
987 988 989
                              FilterMode filtering) {
  assert(IS_ALIGNED(src_width, 2));
  assert(IS_ALIGNED(src_height, 2));
990 991
  void (*ScaleARGBRowDownEven)(const uint8* src_argb, ptrdiff_t src_stride,
                               int src_step, uint8* dst_argb, int dst_width) =
fbarchard@google.com's avatar
fbarchard@google.com committed
992 993
      filtering ? ScaleARGBRowDownEvenInt_C : ScaleARGBRowDownEven_C;
#if defined(HAS_SCALEARGBROWDOWNEVEN_SSE2)
994 995
  if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 4) &&
      IS_ALIGNED(dst_argb, 16) && IS_ALIGNED(dst_stride, 16)) {
fbarchard@google.com's avatar
fbarchard@google.com committed
996 997 998
    ScaleARGBRowDownEven = filtering ? ScaleARGBRowDownEvenInt_SSE2 :
        ScaleARGBRowDownEven_SSE2;
  }
999 1000 1001 1002 1003 1004
#elif defined(HAS_SCALEARGBROWDOWNEVEN_NEON)
  if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(dst_width, 4) &&
      IS_ALIGNED(src_argb, 4)) {
    ScaleARGBRowDownEven = filtering ? ScaleARGBRowDownEvenInt_NEON :
        ScaleARGBRowDownEven_NEON;
  }
fbarchard@google.com's avatar
fbarchard@google.com committed
1005 1006 1007 1008 1009
#endif
  int src_step = src_width / dst_width;
  // Adjust to point to center of box.
  int row_step = src_height / dst_height;
  int row_stride = row_step * src_stride;
1010
  src_argb += ((row_step >> 1) - 1) * src_stride + ((src_step >> 1) - 1) * 4;
fbarchard@google.com's avatar
fbarchard@google.com committed
1011
  for (int y = 0; y < dst_height; ++y) {
1012 1013 1014
    ScaleARGBRowDownEven(src_argb, src_stride, src_step, dst_argb, dst_width);
    src_argb += row_stride;
    dst_argb += dst_stride;
fbarchard@google.com's avatar
fbarchard@google.com committed
1015 1016
  }
}
fbarchard@google.com's avatar
fbarchard@google.com committed
1017 1018 1019

// ScaleARGB ARGB to/from any dimensions, with bilinear
// interpolation.
1020

1021 1022
// Maximum width handled by 2 pass Bilinear.
static const int kMaxInputWidth = 2560;
fbarchard@google.com's avatar
fbarchard@google.com committed
1023 1024 1025
static void ScaleARGBBilinear(int src_width, int src_height,
                              int dst_width, int dst_height,
                              int src_stride, int dst_stride,
1026
                              const uint8* src_argb, uint8* dst_argb) {
1027 1028 1029
  assert(dst_width > 0);
  assert(dst_height > 0);
  assert(src_width <= kMaxInputWidth);
1030
  SIMD_ALIGNED(uint8 row[kMaxInputWidth * 4 + 16]);
1031
  void (*ScaleARGBFilterRows)(uint8* dst_argb, const uint8* src_argb,
1032
                              ptrdiff_t src_stride,
1033 1034 1035
                              int dst_width, int source_y_fraction) =
      ScaleARGBFilterRows_C;
#if defined(HAS_SCALEARGBFILTERROWS_SSE2)
1036 1037
  if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(src_width, 4) &&
      IS_ALIGNED(src_argb, 16) && IS_ALIGNED(src_stride, 16)) {
1038 1039 1040 1041
    ScaleARGBFilterRows = ScaleARGBFilterRows_SSE2;
  }
#endif
#if defined(HAS_SCALEARGBFILTERROWS_SSSE3)
1042 1043
  if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(src_width, 4) &&
      IS_ALIGNED(src_argb, 16) && IS_ALIGNED(src_stride, 16)) {
1044 1045
    ScaleARGBFilterRows = ScaleARGBFilterRows_SSSE3;
  }
1046 1047
#endif
#if defined(HAS_SCALEARGBFILTERROWS_NEON)
1048
  if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(src_width, 4)) {
1049 1050
    ScaleARGBFilterRows = ScaleARGBFilterRows_NEON;
  }
1051 1052 1053 1054 1055 1056 1057
#endif
  int dx = (src_width << 16) / dst_width;
  int dy = (src_height << 16) / dst_height;
  int x = (dx >= 65536) ? ((dx >> 1) - 32768) : (dx >> 1);
  int y = (dy >= 65536) ? ((dy >> 1) - 32768) : (dy >> 1);
  int maxy = (src_height > 1) ? ((src_height - 1) << 16) - 1 : 0;
  for (int j = 0; j < dst_height; ++j) {
1058 1059 1060
    if (y > maxy) {
      y = maxy;
    }
1061 1062
    int yi = y >> 16;
    int yf = (y >> 8) & 255;
1063
    const uint8* src = src_argb + yi * src_stride;
1064
    ScaleARGBFilterRows(row, src, src_stride, src_width, yf);
1065 1066
    ScaleARGBFilterCols_C(dst_argb, row, dst_width, x, dx);
    dst_argb += dst_stride;
1067 1068 1069 1070 1071 1072 1073
    y += dy;
  }
}

// Scales a single row of pixels using point sampling.
// Code is adapted from libyuv bilinear yuv scaling, but with bilinear
//     interpolation off, and argb pixels instead of yuv.
1074
static void ScaleARGBCols(uint8* dst_argb, const uint8* src_argb,
1075
                          int dst_width, int x, int dx) {
1076 1077
  const uint32* src = reinterpret_cast<const uint32*>(src_argb);
  uint32* dst = reinterpret_cast<uint32*>(dst_argb);
1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089
  for (int j = 0; j < dst_width - 1; j += 2) {
    dst[0] = src[x >> 16];
    x += dx;
    dst[1] = src[x >> 16];
    x += dx;
    dst += 2;
  }
  if (dst_width & 1) {
    dst[0] = src[x >> 16];
  }
}

fbarchard@google.com's avatar
fbarchard@google.com committed
1090 1091 1092 1093 1094

// ScaleARGB ARGB to/from any dimensions, without interpolation.
// Fixed point math is used for performance: The upper 16 bits
// of x and dx is the integer part of the source position and
// the lower 16 bits are the fixed decimal part.
1095 1096 1097 1098

static void ScaleARGBSimple(int src_width, int src_height,
                            int dst_width, int dst_height,
                            int src_stride, int dst_stride,
1099
                            const uint8* src_argb, uint8* dst_argb) {
1100 1101 1102 1103 1104
  int dx = (src_width << 16) / dst_width;
  int dy = (src_height << 16) / dst_height;
  int x = (dx >= 65536) ? ((dx >> 1) - 32768) : (dx >> 1);
  int y = (dy >= 65536) ? ((dy >> 1) - 32768) : (dy >> 1);
  for (int i = 0; i < dst_height; ++i) {
1105 1106
    ScaleARGBCols(dst_argb, src_argb + (y >> 16) * src_stride, dst_width, x,
                  dx);
1107
    dst_argb += dst_stride;
1108 1109 1110 1111
    y += dy;
  }
}

fbarchard@google.com's avatar
fbarchard@google.com committed
1112 1113
// ScaleARGB ARGB to/from any dimensions.

1114 1115 1116
static void ScaleARGBAnySize(int src_width, int src_height,
                             int dst_width, int dst_height,
                             int src_stride, int dst_stride,
1117
                             const uint8* src_argb, uint8* dst_argb,
1118 1119 1120
                             FilterMode filtering) {
  if (!filtering || (src_width > kMaxInputWidth)) {
    ScaleARGBSimple(src_width, src_height, dst_width, dst_height,
1121
                    src_stride, dst_stride, src_argb, dst_argb);
1122 1123
  } else {
    ScaleARGBBilinear(src_width, src_height, dst_width, dst_height,
1124
                      src_stride, dst_stride, src_argb, dst_argb);
1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150
  }
}

// ScaleARGB a ARGB.
//
// This function in turn calls a scaling function
// suitable for handling the desired resolutions.

static void ScaleARGB(const uint8* src, int src_stride,
                      int src_width, int src_height,
                      uint8* dst, int dst_stride,
                      int dst_width, int dst_height,
                      FilterMode filtering) {
#ifdef CPU_X86
  // environment variable overrides for testing.
  char *filter_override = getenv("LIBYUV_FILTER");
  if (filter_override) {
    filtering = (FilterMode)atoi(filter_override);  // NOLINT
  }
#endif
  if (dst_width == src_width && dst_height == src_height) {
    // Straight copy.
    ARGBCopy(src, src_stride, dst, dst_stride, dst_width, dst_height);
    return;
  }
  if (2 * dst_width == src_width && 2 * dst_height == src_height) {
fbarchard@google.com's avatar
fbarchard@google.com committed
1151
    // Optimized 1/2.
1152 1153 1154 1155
    ScaleARGBDown2(src_width, src_height, dst_width, dst_height,
                   src_stride, dst_stride, src, dst, filtering);
    return;
  }
fbarchard@google.com's avatar
fbarchard@google.com committed
1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169
  int scale_down_x = src_width / dst_width;
  int scale_down_y = src_height / dst_height;
  if (dst_width * scale_down_x == src_width &&
      dst_height * scale_down_y == src_height) {
    if (!(scale_down_x & 1) && !(scale_down_y & 1)) {
      // Optimized even scale down. ie 4, 6, 8, 10x
      ScaleARGBDownEven(src_width, src_height, dst_width, dst_height,
                        src_stride, dst_stride, src, dst, filtering);
      return;
    }
    if ((scale_down_x & 1) && (scale_down_y & 1)) {
      filtering = kFilterNone;
    }
  }
1170 1171 1172 1173 1174 1175
  // Arbitrary scale up and/or down.
  ScaleARGBAnySize(src_width, src_height, dst_width, dst_height,
                   src_stride, dst_stride, src, dst, filtering);
}

// ScaleARGB an ARGB image.
1176
LIBYUV_API
1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201
int ARGBScale(const uint8* src_argb, int src_stride_argb,
             int src_width, int src_height,
             uint8* dst_argb, int dst_stride_argb,
             int dst_width, int dst_height,
             FilterMode filtering) {
  if (!src_argb || src_width <= 0 || src_height == 0 ||
      !dst_argb || dst_width <= 0 || dst_height <= 0) {
    return -1;
  }
  // Negative height means invert the image.
  if (src_height < 0) {
    src_height = -src_height;
    src_argb = src_argb + (src_height - 1) * src_stride_argb;
    src_stride_argb = -src_stride_argb;
  }
  ScaleARGB(src_argb, src_stride_argb, src_width, src_height,
            dst_argb, dst_stride_argb, dst_width, dst_height,
            filtering);
  return 0;
}

#ifdef __cplusplus
}  // extern "C"
}  // namespace libyuv
#endif