scale_win.cc 44 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
/*
 *  Copyright 2013 The LibYuv Project Authors. All rights reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS. All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "libyuv/row.h"
12
#include "libyuv/scale_row.h"
13 14 15 16 17 18

#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif

Frank Barchard's avatar
Frank Barchard committed
19
// This module is for 32 bit Visual C x86 and clangcl
20
#if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) && defined(_MSC_VER)
21 22

// Offsets for source bytes 0 to 9
23 24
static const uvec8 kShuf0 = {0,   1,   3,   4,   5,   7,   8,   9,
                             128, 128, 128, 128, 128, 128, 128, 128};
25 26

// Offsets for source bytes 11 to 20 with 8 subtracted = 3 to 12.
27 28
static const uvec8 kShuf1 = {3,   4,   5,   7,   8,   9,   11,  12,
                             128, 128, 128, 128, 128, 128, 128, 128};
29 30

// Offsets for source bytes 21 to 31 with 16 subtracted = 5 to 31.
31 32
static const uvec8 kShuf2 = {5,   7,   8,   9,   11,  12,  13,  15,
                             128, 128, 128, 128, 128, 128, 128, 128};
33 34

// Offsets for source bytes 0 to 10
35
static const uvec8 kShuf01 = {0, 1, 1, 2, 2, 3, 4, 5, 5, 6, 6, 7, 8, 9, 9, 10};
36 37

// Offsets for source bytes 10 to 21 with 8 subtracted = 3 to 13.
38 39
static const uvec8 kShuf11 = {2, 3, 4, 5,  5,  6,  6,  7,
                              8, 9, 9, 10, 10, 11, 12, 13};
40 41

// Offsets for source bytes 21 to 31 with 16 subtracted = 5 to 31.
42 43
static const uvec8 kShuf21 = {5,  6,  6,  7,  8,  9,  9,  10,
                              10, 11, 12, 13, 13, 14, 14, 15};
44 45

// Coefficients for source bytes 0 to 10
46
static const uvec8 kMadd01 = {3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2};
47 48

// Coefficients for source bytes 10 to 21
49
static const uvec8 kMadd11 = {1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1};
50 51

// Coefficients for source bytes 21 to 31
52
static const uvec8 kMadd21 = {2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3};
53 54

// Coefficients for source bytes 21 to 31
55
static const vec16 kRound34 = {2, 2, 2, 2, 2, 2, 2, 2};
56

57 58
static const uvec8 kShuf38a = {0,   3,   6,   8,   11,  14,  128, 128,
                               128, 128, 128, 128, 128, 128, 128, 128};
59

60 61
static const uvec8 kShuf38b = {128, 128, 128, 128, 128, 128, 0,   3,
                               6,   8,   11,  14,  128, 128, 128, 128};
62 63

// Arrange words 0,3,6 into 0,1,2
64 65
static const uvec8 kShufAc = {0,   1,   6,   7,   12,  13,  128, 128,
                              128, 128, 128, 128, 128, 128, 128, 128};
66 67

// Arrange words 0,3,6 into 3,4,5
68 69
static const uvec8 kShufAc3 = {128, 128, 128, 128, 128, 128, 0,   1,
                               6,   7,   12,  13,  128, 128, 128, 128};
70 71

// Scaling values for boxes of 3x3 and 2x3
72 73
static const uvec16 kScaleAc33 = {65536 / 9, 65536 / 9, 65536 / 6, 65536 / 9,
                                  65536 / 9, 65536 / 6, 0,         0};
74 75

// Arrange first value for pixels 0,1,2,3,4,5
76 77
static const uvec8 kShufAb0 = {0,  128, 3,  128, 6,   128, 8,   128,
                               11, 128, 14, 128, 128, 128, 128, 128};
78 79

// Arrange second value for pixels 0,1,2,3,4,5
80 81
static const uvec8 kShufAb1 = {1,  128, 4,  128, 7,   128, 9,   128,
                               12, 128, 15, 128, 128, 128, 128, 128};
82 83

// Arrange third value for pixels 0,1,2,3,4,5
84 85
static const uvec8 kShufAb2 = {2,  128, 5,   128, 128, 128, 10,  128,
                               13, 128, 128, 128, 128, 128, 128, 128};
86 87

// Scaling values for boxes of 3x2 and 2x2
88 89
static const uvec16 kScaleAb2 = {65536 / 3, 65536 / 3, 65536 / 2, 65536 / 3,
                                 65536 / 3, 65536 / 2, 0,         0};
90 91

// Reads 32 pixels, throws half away and writes 16 pixels.
Frank Barchard's avatar
Frank Barchard committed
92
__declspec(naked) void ScaleRowDown2_SSSE3(const uint8_t* src_ptr,
Frank Barchard's avatar
Frank Barchard committed
93
                                           ptrdiff_t src_stride,
Frank Barchard's avatar
Frank Barchard committed
94
                                           uint8_t* dst_ptr,
Frank Barchard's avatar
Frank Barchard committed
95
                                           int dst_width) {
96
  __asm {
Frank Barchard's avatar
Frank Barchard committed
97 98 99 100
    mov        eax, [esp + 4]  // src_ptr
    // src_stride ignored
    mov        edx, [esp + 12]  // dst_ptr
    mov        ecx, [esp + 16]  // dst_width
101 102 103 104 105

  wloop:
    movdqu     xmm0, [eax]
    movdqu     xmm1, [eax + 16]
    lea        eax,  [eax + 32]
Frank Barchard's avatar
Frank Barchard committed
106
    psrlw      xmm0, 8          // isolate odd pixels.
107 108 109 110
    psrlw      xmm1, 8
    packuswb   xmm0, xmm1
    movdqu     [edx], xmm0
    lea        edx, [edx + 16]
111
    sub        ecx, 16
112 113 114 115 116 117 118
    jg         wloop

    ret
  }
}

// Blends 32x1 rectangle to 16x1.
Frank Barchard's avatar
Frank Barchard committed
119
__declspec(naked) void ScaleRowDown2Linear_SSSE3(const uint8_t* src_ptr,
Frank Barchard's avatar
Frank Barchard committed
120
                                                 ptrdiff_t src_stride,
Frank Barchard's avatar
Frank Barchard committed
121
                                                 uint8_t* dst_ptr,
Frank Barchard's avatar
Frank Barchard committed
122
                                                 int dst_width) {
123
  __asm {
Frank Barchard's avatar
Frank Barchard committed
124 125 126 127
    mov        eax, [esp + 4]  // src_ptr
    // src_stride
    mov        edx, [esp + 12]  // dst_ptr
    mov        ecx, [esp + 16]  // dst_width
128

Frank Barchard's avatar
Frank Barchard committed
129
    pcmpeqb    xmm4, xmm4  // constant 0x0101
130 131
    psrlw      xmm4, 15
    packuswb   xmm4, xmm4
Frank Barchard's avatar
Frank Barchard committed
132
    pxor       xmm5, xmm5  // constant 0
133 134 135 136 137

  wloop:
    movdqu     xmm0, [eax]
    movdqu     xmm1, [eax + 16]
    lea        eax,  [eax + 32]
Frank Barchard's avatar
Frank Barchard committed
138
    pmaddubsw  xmm0, xmm4  // horizontal add
139
    pmaddubsw  xmm1, xmm4
Frank Barchard's avatar
Frank Barchard committed
140
    pavgw      xmm0, xmm5       // (x + 1) / 2
141
    pavgw      xmm1, xmm5
142 143 144
    packuswb   xmm0, xmm1
    movdqu     [edx], xmm0
    lea        edx, [edx + 16]
145
    sub        ecx, 16
146 147 148 149 150 151 152
    jg         wloop

    ret
  }
}

// Blends 32x2 rectangle to 16x1.
Frank Barchard's avatar
Frank Barchard committed
153
__declspec(naked) void ScaleRowDown2Box_SSSE3(const uint8_t* src_ptr,
Frank Barchard's avatar
Frank Barchard committed
154
                                              ptrdiff_t src_stride,
Frank Barchard's avatar
Frank Barchard committed
155
                                              uint8_t* dst_ptr,
Frank Barchard's avatar
Frank Barchard committed
156
                                              int dst_width) {
157 158
  __asm {
    push       esi
Frank Barchard's avatar
Frank Barchard committed
159 160 161 162
    mov        eax, [esp + 4 + 4]  // src_ptr
    mov        esi, [esp + 4 + 8]  // src_stride
    mov        edx, [esp + 4 + 12]  // dst_ptr
    mov        ecx, [esp + 4 + 16]  // dst_width
163

Frank Barchard's avatar
Frank Barchard committed
164
    pcmpeqb    xmm4, xmm4  // constant 0x0101
165 166
    psrlw      xmm4, 15
    packuswb   xmm4, xmm4
Frank Barchard's avatar
Frank Barchard committed
167
    pxor       xmm5, xmm5  // constant 0
168 169 170 171 172 173 174

  wloop:
    movdqu     xmm0, [eax]
    movdqu     xmm1, [eax + 16]
    movdqu     xmm2, [eax + esi]
    movdqu     xmm3, [eax + esi + 16]
    lea        eax,  [eax + 32]
Frank Barchard's avatar
Frank Barchard committed
175
    pmaddubsw  xmm0, xmm4  // horizontal add
176 177 178
    pmaddubsw  xmm1, xmm4
    pmaddubsw  xmm2, xmm4
    pmaddubsw  xmm3, xmm4
Frank Barchard's avatar
Frank Barchard committed
179
    paddw      xmm0, xmm2  // vertical add
180 181 182
    paddw      xmm1, xmm3
    psrlw      xmm0, 1
    psrlw      xmm1, 1
Frank Barchard's avatar
Frank Barchard committed
183
    pavgw      xmm0, xmm5  // (x + 1) / 2
184
    pavgw      xmm1, xmm5
185 186 187
    packuswb   xmm0, xmm1
    movdqu     [edx], xmm0
    lea        edx, [edx + 16]
188
    sub        ecx, 16
189 190 191 192 193 194 195
    jg         wloop

    pop        esi
    ret
  }
}

196
#ifdef HAS_SCALEROWDOWN2_AVX2
197
// Reads 64 pixels, throws half away and writes 32 pixels.
Frank Barchard's avatar
Frank Barchard committed
198
__declspec(naked) void ScaleRowDown2_AVX2(const uint8_t* src_ptr,
Frank Barchard's avatar
Frank Barchard committed
199
                                          ptrdiff_t src_stride,
Frank Barchard's avatar
Frank Barchard committed
200
                                          uint8_t* dst_ptr,
Frank Barchard's avatar
Frank Barchard committed
201
                                          int dst_width) {
202
  __asm {
Frank Barchard's avatar
Frank Barchard committed
203 204 205 206
    mov        eax, [esp + 4]  // src_ptr
    // src_stride ignored
    mov        edx, [esp + 12]  // dst_ptr
    mov        ecx, [esp + 16]  // dst_width
207 208 209 210 211

  wloop:
    vmovdqu     ymm0, [eax]
    vmovdqu     ymm1, [eax + 32]
    lea         eax,  [eax + 64]
Frank Barchard's avatar
Frank Barchard committed
212
    vpsrlw      ymm0, ymm0, 8  // isolate odd pixels.
213 214
    vpsrlw      ymm1, ymm1, 8
    vpackuswb   ymm0, ymm0, ymm1
Frank Barchard's avatar
Frank Barchard committed
215
    vpermq      ymm0, ymm0, 0xd8       // unmutate vpackuswb
216 217 218 219 220 221 222 223 224 225 226
    vmovdqu     [edx], ymm0
    lea         edx, [edx + 32]
    sub         ecx, 32
    jg          wloop

    vzeroupper
    ret
  }
}

// Blends 64x1 rectangle to 32x1.
Frank Barchard's avatar
Frank Barchard committed
227
__declspec(naked) void ScaleRowDown2Linear_AVX2(const uint8_t* src_ptr,
Frank Barchard's avatar
Frank Barchard committed
228
                                                ptrdiff_t src_stride,
Frank Barchard's avatar
Frank Barchard committed
229
                                                uint8_t* dst_ptr,
Frank Barchard's avatar
Frank Barchard committed
230
                                                int dst_width) {
231
  __asm {
Frank Barchard's avatar
Frank Barchard committed
232 233 234 235
    mov         eax, [esp + 4]  // src_ptr
    // src_stride
    mov         edx, [esp + 12]  // dst_ptr
    mov         ecx, [esp + 16]  // dst_width
236

Frank Barchard's avatar
Frank Barchard committed
237
    vpcmpeqb    ymm4, ymm4, ymm4  // '1' constant, 8b
238 239
    vpsrlw      ymm4, ymm4, 15
    vpackuswb   ymm4, ymm4, ymm4
Frank Barchard's avatar
Frank Barchard committed
240
    vpxor       ymm5, ymm5, ymm5  // constant 0
241 242 243 244 245

  wloop:
    vmovdqu     ymm0, [eax]
    vmovdqu     ymm1, [eax + 32]
    lea         eax,  [eax + 64]
Frank Barchard's avatar
Frank Barchard committed
246
    vpmaddubsw  ymm0, ymm0, ymm4  // horizontal add
247
    vpmaddubsw  ymm1, ymm1, ymm4
Frank Barchard's avatar
Frank Barchard committed
248
    vpavgw      ymm0, ymm0, ymm5  // (x + 1) / 2
249
    vpavgw      ymm1, ymm1, ymm5
250
    vpackuswb   ymm0, ymm0, ymm1
Frank Barchard's avatar
Frank Barchard committed
251
    vpermq      ymm0, ymm0, 0xd8       // unmutate vpackuswb
252 253 254 255 256 257 258 259 260 261
    vmovdqu     [edx], ymm0
    lea         edx, [edx + 32]
    sub         ecx, 32
    jg          wloop

    vzeroupper
    ret
  }
}

262 263
// For rounding, average = (sum + 2) / 4
// becomes average((sum >> 1), 0)
264
// Blends 64x2 rectangle to 32x1.
Frank Barchard's avatar
Frank Barchard committed
265
__declspec(naked) void ScaleRowDown2Box_AVX2(const uint8_t* src_ptr,
Frank Barchard's avatar
Frank Barchard committed
266
                                             ptrdiff_t src_stride,
Frank Barchard's avatar
Frank Barchard committed
267
                                             uint8_t* dst_ptr,
Frank Barchard's avatar
Frank Barchard committed
268
                                             int dst_width) {
269 270
  __asm {
    push        esi
Frank Barchard's avatar
Frank Barchard committed
271 272 273 274
    mov         eax, [esp + 4 + 4]  // src_ptr
    mov         esi, [esp + 4 + 8]  // src_stride
    mov         edx, [esp + 4 + 12]  // dst_ptr
    mov         ecx, [esp + 4 + 16]  // dst_width
275

Frank Barchard's avatar
Frank Barchard committed
276
    vpcmpeqb    ymm4, ymm4, ymm4  // '1' constant, 8b
277 278
    vpsrlw      ymm4, ymm4, 15
    vpackuswb   ymm4, ymm4, ymm4
Frank Barchard's avatar
Frank Barchard committed
279
    vpxor       ymm5, ymm5, ymm5  // constant 0
280 281

  wloop:
282
    vmovdqu     ymm0, [eax]
283
    vmovdqu     ymm1, [eax + 32]
284 285
    vmovdqu     ymm2, [eax + esi]
    vmovdqu     ymm3, [eax + esi + 32]
286
    lea         eax,  [eax + 64]
Frank Barchard's avatar
Frank Barchard committed
287
    vpmaddubsw  ymm0, ymm0, ymm4  // horizontal add
288
    vpmaddubsw  ymm1, ymm1, ymm4
289 290
    vpmaddubsw  ymm2, ymm2, ymm4
    vpmaddubsw  ymm3, ymm3, ymm4
Frank Barchard's avatar
Frank Barchard committed
291
    vpaddw      ymm0, ymm0, ymm2  // vertical add
292
    vpaddw      ymm1, ymm1, ymm3
Frank Barchard's avatar
Frank Barchard committed
293
    vpsrlw      ymm0, ymm0, 1  // (x + 2) / 4 = (x / 2 + 1) / 2
294
    vpsrlw      ymm1, ymm1, 1
Frank Barchard's avatar
Frank Barchard committed
295
    vpavgw      ymm0, ymm0, ymm5  // (x + 1) / 2
296
    vpavgw      ymm1, ymm1, ymm5
297
    vpackuswb   ymm0, ymm0, ymm1
Frank Barchard's avatar
Frank Barchard committed
298
    vpermq      ymm0, ymm0, 0xd8  // unmutate vpackuswb
299 300 301 302 303 304 305 306 307 308
    vmovdqu     [edx], ymm0
    lea         edx, [edx + 32]
    sub         ecx, 32
    jg          wloop

    pop         esi
    vzeroupper
    ret
  }
}
309
#endif  // HAS_SCALEROWDOWN2_AVX2
310

311
// Point samples 32 pixels to 8 pixels.
Frank Barchard's avatar
Frank Barchard committed
312
__declspec(naked) void ScaleRowDown4_SSSE3(const uint8_t* src_ptr,
Frank Barchard's avatar
Frank Barchard committed
313
                                           ptrdiff_t src_stride,
Frank Barchard's avatar
Frank Barchard committed
314
                                           uint8_t* dst_ptr,
Frank Barchard's avatar
Frank Barchard committed
315
                                           int dst_width) {
316
  __asm {
Frank Barchard's avatar
Frank Barchard committed
317 318 319 320 321
    mov        eax, [esp + 4]  // src_ptr
    // src_stride ignored
    mov        edx, [esp + 12]  // dst_ptr
    mov        ecx, [esp + 16]  // dst_width
    pcmpeqb    xmm5, xmm5       // generate mask 0x00ff0000
322 323 324 325
    psrld      xmm5, 24
    pslld      xmm5, 16

  wloop:
326 327
    movdqu     xmm0, [eax]
    movdqu     xmm1, [eax + 16]
328 329 330 331 332 333 334 335
    lea        eax,  [eax + 32]
    pand       xmm0, xmm5
    pand       xmm1, xmm5
    packuswb   xmm0, xmm1
    psrlw      xmm0, 8
    packuswb   xmm0, xmm0
    movq       qword ptr [edx], xmm0
    lea        edx, [edx + 8]
336
    sub        ecx, 8
337 338 339 340 341 342 343
    jg         wloop

    ret
  }
}

// Blends 32x4 rectangle to 8x1.
Frank Barchard's avatar
Frank Barchard committed
344
__declspec(naked) void ScaleRowDown4Box_SSSE3(const uint8_t* src_ptr,
Frank Barchard's avatar
Frank Barchard committed
345
                                              ptrdiff_t src_stride,
Frank Barchard's avatar
Frank Barchard committed
346
                                              uint8_t* dst_ptr,
Frank Barchard's avatar
Frank Barchard committed
347
                                              int dst_width) {
348 349 350
  __asm {
    push       esi
    push       edi
Frank Barchard's avatar
Frank Barchard committed
351 352 353 354
    mov        eax, [esp + 8 + 4]  // src_ptr
    mov        esi, [esp + 8 + 8]  // src_stride
    mov        edx, [esp + 8 + 12]  // dst_ptr
    mov        ecx, [esp + 8 + 16]  // dst_width
355
    lea        edi, [esi + esi * 2]  // src_stride * 3
Frank Barchard's avatar
Frank Barchard committed
356
    pcmpeqb    xmm4, xmm4  // constant 0x0101
357 358 359
    psrlw      xmm4, 15
    movdqa     xmm5, xmm4
    packuswb   xmm4, xmm4
Frank Barchard's avatar
Frank Barchard committed
360
    psllw      xmm5, 3  // constant 0x0008
361 362

  wloop:
Frank Barchard's avatar
Frank Barchard committed
363
    movdqu     xmm0, [eax]  // average rows
364 365 366
    movdqu     xmm1, [eax + 16]
    movdqu     xmm2, [eax + esi]
    movdqu     xmm3, [eax + esi + 16]
Frank Barchard's avatar
Frank Barchard committed
367
    pmaddubsw  xmm0, xmm4  // horizontal add
368 369 370
    pmaddubsw  xmm1, xmm4
    pmaddubsw  xmm2, xmm4
    pmaddubsw  xmm3, xmm4
Frank Barchard's avatar
Frank Barchard committed
371
    paddw      xmm0, xmm2  // vertical add rows 0, 1
372
    paddw      xmm1, xmm3
373 374
    movdqu     xmm2, [eax + esi * 2]
    movdqu     xmm3, [eax + esi * 2 + 16]
375 376
    pmaddubsw  xmm2, xmm4
    pmaddubsw  xmm3, xmm4
Frank Barchard's avatar
Frank Barchard committed
377
    paddw      xmm0, xmm2  // add row 2
378 379 380
    paddw      xmm1, xmm3
    movdqu     xmm2, [eax + edi]
    movdqu     xmm3, [eax + edi + 16]
381
    lea        eax, [eax + 32]
382 383
    pmaddubsw  xmm2, xmm4
    pmaddubsw  xmm3, xmm4
Frank Barchard's avatar
Frank Barchard committed
384
    paddw      xmm0, xmm2  // add row 3
385 386
    paddw      xmm1, xmm3
    phaddw     xmm0, xmm1
Frank Barchard's avatar
Frank Barchard committed
387 388
    paddw      xmm0, xmm5  // + 8 for round
    psrlw      xmm0, 4  // /16 for average of 4 * 4
389 390 391
    packuswb   xmm0, xmm0
    movq       qword ptr [edx], xmm0
    lea        edx, [edx + 8]
392
    sub        ecx, 8
393 394 395 396 397 398 399 400
    jg         wloop

    pop        edi
    pop        esi
    ret
  }
}

401 402
#ifdef HAS_SCALEROWDOWN4_AVX2
// Point samples 64 pixels to 16 pixels.
Frank Barchard's avatar
Frank Barchard committed
403
__declspec(naked) void ScaleRowDown4_AVX2(const uint8_t* src_ptr,
Frank Barchard's avatar
Frank Barchard committed
404
                                          ptrdiff_t src_stride,
Frank Barchard's avatar
Frank Barchard committed
405
                                          uint8_t* dst_ptr,
Frank Barchard's avatar
Frank Barchard committed
406
                                          int dst_width) {
407
  __asm {
Frank Barchard's avatar
Frank Barchard committed
408 409 410 411 412
    mov         eax, [esp + 4]  // src_ptr
    // src_stride ignored
    mov         edx, [esp + 12]  // dst_ptr
    mov         ecx, [esp + 16]  // dst_width
    vpcmpeqb    ymm5, ymm5, ymm5  // generate mask 0x00ff0000
413 414 415 416 417 418 419 420 421 422
    vpsrld      ymm5, ymm5, 24
    vpslld      ymm5, ymm5, 16

  wloop:
    vmovdqu     ymm0, [eax]
    vmovdqu     ymm1, [eax + 32]
    lea         eax,  [eax + 64]
    vpand       ymm0, ymm0, ymm5
    vpand       ymm1, ymm1, ymm5
    vpackuswb   ymm0, ymm0, ymm1
Frank Barchard's avatar
Frank Barchard committed
423
    vpermq      ymm0, ymm0, 0xd8  // unmutate vpackuswb
424 425
    vpsrlw      ymm0, ymm0, 8
    vpackuswb   ymm0, ymm0, ymm0
Frank Barchard's avatar
Frank Barchard committed
426
    vpermq      ymm0, ymm0, 0xd8       // unmutate vpackuswb
427 428 429 430 431 432 433 434 435 436 437
    vmovdqu     [edx], xmm0
    lea         edx, [edx + 16]
    sub         ecx, 16
    jg          wloop

    vzeroupper
    ret
  }
}

// Blends 64x4 rectangle to 16x1.
Frank Barchard's avatar
Frank Barchard committed
438
__declspec(naked) void ScaleRowDown4Box_AVX2(const uint8_t* src_ptr,
Frank Barchard's avatar
Frank Barchard committed
439
                                             ptrdiff_t src_stride,
Frank Barchard's avatar
Frank Barchard committed
440
                                             uint8_t* dst_ptr,
Frank Barchard's avatar
Frank Barchard committed
441
                                             int dst_width) {
442 443 444
  __asm {
    push        esi
    push        edi
Frank Barchard's avatar
Frank Barchard committed
445 446 447 448
    mov         eax, [esp + 8 + 4]  // src_ptr
    mov         esi, [esp + 8 + 8]  // src_stride
    mov         edx, [esp + 8 + 12]  // dst_ptr
    mov         ecx, [esp + 8 + 16]  // dst_width
449
    lea         edi, [esi + esi * 2]  // src_stride * 3
Frank Barchard's avatar
Frank Barchard committed
450
    vpcmpeqb    ymm4, ymm4, ymm4  // constant 0x0101
451
    vpsrlw      ymm4, ymm4, 15
Frank Barchard's avatar
Frank Barchard committed
452
    vpsllw      ymm5, ymm4, 3  // constant 0x0008
453
    vpackuswb   ymm4, ymm4, ymm4
454 455

  wloop:
Frank Barchard's avatar
Frank Barchard committed
456
    vmovdqu     ymm0, [eax]  // average rows
457
    vmovdqu     ymm1, [eax + 32]
458 459
    vmovdqu     ymm2, [eax + esi]
    vmovdqu     ymm3, [eax + esi + 32]
Frank Barchard's avatar
Frank Barchard committed
460
    vpmaddubsw  ymm0, ymm0, ymm4  // horizontal add
461 462 463
    vpmaddubsw  ymm1, ymm1, ymm4
    vpmaddubsw  ymm2, ymm2, ymm4
    vpmaddubsw  ymm3, ymm3, ymm4
Frank Barchard's avatar
Frank Barchard committed
464
    vpaddw      ymm0, ymm0, ymm2  // vertical add rows 0, 1
465
    vpaddw      ymm1, ymm1, ymm3
466 467
    vmovdqu     ymm2, [eax + esi * 2]
    vmovdqu     ymm3, [eax + esi * 2 + 32]
468 469
    vpmaddubsw  ymm2, ymm2, ymm4
    vpmaddubsw  ymm3, ymm3, ymm4
Frank Barchard's avatar
Frank Barchard committed
470
    vpaddw      ymm0, ymm0, ymm2  // add row 2
471 472 473 474 475 476
    vpaddw      ymm1, ymm1, ymm3
    vmovdqu     ymm2, [eax + edi]
    vmovdqu     ymm3, [eax + edi + 32]
    lea         eax,  [eax + 64]
    vpmaddubsw  ymm2, ymm2, ymm4
    vpmaddubsw  ymm3, ymm3, ymm4
Frank Barchard's avatar
Frank Barchard committed
477
    vpaddw      ymm0, ymm0, ymm2  // add row 3
478
    vpaddw      ymm1, ymm1, ymm3
Frank Barchard's avatar
Frank Barchard committed
479 480 481 482
    vphaddw     ymm0, ymm0, ymm1  // mutates
    vpermq      ymm0, ymm0, 0xd8  // unmutate vphaddw
    vpaddw      ymm0, ymm0, ymm5  // + 8 for round
    vpsrlw      ymm0, ymm0, 4  // /32 for average of 4 * 4
483
    vpackuswb   ymm0, ymm0, ymm0
Frank Barchard's avatar
Frank Barchard committed
484
    vpermq      ymm0, ymm0, 0xd8  // unmutate vpackuswb
485 486 487 488 489 490 491 492 493 494 495 496 497
    vmovdqu     [edx], xmm0
    lea         edx, [edx + 16]
    sub         ecx, 16
    jg          wloop

    pop        edi
    pop        esi
    vzeroupper
    ret
  }
}
#endif  // HAS_SCALEROWDOWN4_AVX2

498 499 500 501
// Point samples 32 pixels to 24 pixels.
// Produces three 8 byte values. For each 8 bytes, 16 bytes are read.
// Then shuffled to do the scaling.

Frank Barchard's avatar
Frank Barchard committed
502
__declspec(naked) void ScaleRowDown34_SSSE3(const uint8_t* src_ptr,
Frank Barchard's avatar
Frank Barchard committed
503
                                            ptrdiff_t src_stride,
Frank Barchard's avatar
Frank Barchard committed
504
                                            uint8_t* dst_ptr,
Frank Barchard's avatar
Frank Barchard committed
505
                                            int dst_width) {
506
  __asm {
Frank Barchard's avatar
Frank Barchard committed
507 508 509 510
    mov        eax, [esp + 4]   // src_ptr
    // src_stride ignored
    mov        edx, [esp + 12]  // dst_ptr
    mov        ecx, [esp + 16]  // dst_width
Frank Barchard's avatar
Frank Barchard committed
511 512 513
    movdqa     xmm3, xmmword ptr kShuf0
    movdqa     xmm4, xmmword ptr kShuf1
    movdqa     xmm5, xmmword ptr kShuf2
514 515

  wloop:
516 517
    movdqu     xmm0, [eax]
    movdqu     xmm1, [eax + 16]
518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549
    lea        eax,  [eax + 32]
    movdqa     xmm2, xmm1
    palignr    xmm1, xmm0, 8
    pshufb     xmm0, xmm3
    pshufb     xmm1, xmm4
    pshufb     xmm2, xmm5
    movq       qword ptr [edx], xmm0
    movq       qword ptr [edx + 8], xmm1
    movq       qword ptr [edx + 16], xmm2
    lea        edx, [edx + 24]
    sub        ecx, 24
    jg         wloop

    ret
  }
}

// Blends 32x2 rectangle to 24x1
// Produces three 8 byte values. For each 8 bytes, 16 bytes are read.
// Then shuffled to do the scaling.

// Register usage:
// xmm0 src_row 0
// xmm1 src_row 1
// xmm2 shuf 0
// xmm3 shuf 1
// xmm4 shuf 2
// xmm5 madd 0
// xmm6 madd 1
// xmm7 kRound34

// Note that movdqa+palign may be better than movdqu.
Frank Barchard's avatar
Frank Barchard committed
550
__declspec(naked) void ScaleRowDown34_1_Box_SSSE3(const uint8_t* src_ptr,
Frank Barchard's avatar
Frank Barchard committed
551
                                                  ptrdiff_t src_stride,
Frank Barchard's avatar
Frank Barchard committed
552
                                                  uint8_t* dst_ptr,
Frank Barchard's avatar
Frank Barchard committed
553
                                                  int dst_width) {
554 555
  __asm {
    push       esi
Frank Barchard's avatar
Frank Barchard committed
556 557 558 559
    mov        eax, [esp + 4 + 4]  // src_ptr
    mov        esi, [esp + 4 + 8]  // src_stride
    mov        edx, [esp + 4 + 12]  // dst_ptr
    mov        ecx, [esp + 4 + 16]  // dst_width
Frank Barchard's avatar
Frank Barchard committed
560 561 562 563 564 565
    movdqa     xmm2, xmmword ptr kShuf01
    movdqa     xmm3, xmmword ptr kShuf11
    movdqa     xmm4, xmmword ptr kShuf21
    movdqa     xmm5, xmmword ptr kMadd01
    movdqa     xmm6, xmmword ptr kMadd11
    movdqa     xmm7, xmmword ptr kRound34
566 567

  wloop:
Frank Barchard's avatar
Frank Barchard committed
568
    movdqu     xmm0, [eax]  // pixels 0..7
569
    movdqu     xmm1, [eax + esi]
570 571 572 573 574 575 576
    pavgb      xmm0, xmm1
    pshufb     xmm0, xmm2
    pmaddubsw  xmm0, xmm5
    paddsw     xmm0, xmm7
    psrlw      xmm0, 2
    packuswb   xmm0, xmm0
    movq       qword ptr [edx], xmm0
Frank Barchard's avatar
Frank Barchard committed
577
    movdqu     xmm0, [eax + 8]  // pixels 8..15
578 579 580 581 582 583 584 585
    movdqu     xmm1, [eax + esi + 8]
    pavgb      xmm0, xmm1
    pshufb     xmm0, xmm3
    pmaddubsw  xmm0, xmm6
    paddsw     xmm0, xmm7
    psrlw      xmm0, 2
    packuswb   xmm0, xmm0
    movq       qword ptr [edx + 8], xmm0
Frank Barchard's avatar
Frank Barchard committed
586
    movdqu     xmm0, [eax + 16]  // pixels 16..23
587
    movdqu     xmm1, [eax + esi + 16]
588 589 590
    lea        eax, [eax + 32]
    pavgb      xmm0, xmm1
    pshufb     xmm0, xmm4
Frank Barchard's avatar
Frank Barchard committed
591
    movdqa     xmm1, xmmword ptr kMadd21
592 593 594 595 596 597
    pmaddubsw  xmm0, xmm1
    paddsw     xmm0, xmm7
    psrlw      xmm0, 2
    packuswb   xmm0, xmm0
    movq       qword ptr [edx + 16], xmm0
    lea        edx, [edx + 24]
598
    sub        ecx, 24
599 600 601 602 603 604 605 606
    jg         wloop

    pop        esi
    ret
  }
}

// Note that movdqa+palign may be better than movdqu.
Frank Barchard's avatar
Frank Barchard committed
607
__declspec(naked) void ScaleRowDown34_0_Box_SSSE3(const uint8_t* src_ptr,
Frank Barchard's avatar
Frank Barchard committed
608
                                                  ptrdiff_t src_stride,
Frank Barchard's avatar
Frank Barchard committed
609
                                                  uint8_t* dst_ptr,
Frank Barchard's avatar
Frank Barchard committed
610
                                                  int dst_width) {
611 612
  __asm {
    push       esi
Frank Barchard's avatar
Frank Barchard committed
613 614 615 616
    mov        eax, [esp + 4 + 4]  // src_ptr
    mov        esi, [esp + 4 + 8]  // src_stride
    mov        edx, [esp + 4 + 12]  // dst_ptr
    mov        ecx, [esp + 4 + 16]  // dst_width
Frank Barchard's avatar
Frank Barchard committed
617 618 619 620 621 622
    movdqa     xmm2, xmmword ptr kShuf01
    movdqa     xmm3, xmmword ptr kShuf11
    movdqa     xmm4, xmmword ptr kShuf21
    movdqa     xmm5, xmmword ptr kMadd01
    movdqa     xmm6, xmmword ptr kMadd11
    movdqa     xmm7, xmmword ptr kRound34
623 624

  wloop:
Frank Barchard's avatar
Frank Barchard committed
625
    movdqu     xmm0, [eax]  // pixels 0..7
626
    movdqu     xmm1, [eax + esi]
627 628 629 630 631 632 633 634
    pavgb      xmm1, xmm0
    pavgb      xmm0, xmm1
    pshufb     xmm0, xmm2
    pmaddubsw  xmm0, xmm5
    paddsw     xmm0, xmm7
    psrlw      xmm0, 2
    packuswb   xmm0, xmm0
    movq       qword ptr [edx], xmm0
Frank Barchard's avatar
Frank Barchard committed
635
    movdqu     xmm0, [eax + 8]  // pixels 8..15
636 637 638 639 640 641 642 643 644
    movdqu     xmm1, [eax + esi + 8]
    pavgb      xmm1, xmm0
    pavgb      xmm0, xmm1
    pshufb     xmm0, xmm3
    pmaddubsw  xmm0, xmm6
    paddsw     xmm0, xmm7
    psrlw      xmm0, 2
    packuswb   xmm0, xmm0
    movq       qword ptr [edx + 8], xmm0
Frank Barchard's avatar
Frank Barchard committed
645
    movdqu     xmm0, [eax + 16]  // pixels 16..23
646
    movdqu     xmm1, [eax + esi + 16]
647 648 649 650
    lea        eax, [eax + 32]
    pavgb      xmm1, xmm0
    pavgb      xmm0, xmm1
    pshufb     xmm0, xmm4
Frank Barchard's avatar
Frank Barchard committed
651
    movdqa     xmm1, xmmword ptr kMadd21
652 653 654 655 656 657
    pmaddubsw  xmm0, xmm1
    paddsw     xmm0, xmm7
    psrlw      xmm0, 2
    packuswb   xmm0, xmm0
    movq       qword ptr [edx + 16], xmm0
    lea        edx, [edx+24]
658
    sub        ecx, 24
659 660 661 662 663 664 665 666 667 668
    jg         wloop

    pop        esi
    ret
  }
}

// 3/8 point sampler

// Scale 32 pixels to 12
Frank Barchard's avatar
Frank Barchard committed
669
__declspec(naked) void ScaleRowDown38_SSSE3(const uint8_t* src_ptr,
Frank Barchard's avatar
Frank Barchard committed
670
                                            ptrdiff_t src_stride,
Frank Barchard's avatar
Frank Barchard committed
671
                                            uint8_t* dst_ptr,
Frank Barchard's avatar
Frank Barchard committed
672
                                            int dst_width) {
673
  __asm {
Frank Barchard's avatar
Frank Barchard committed
674 675 676 677
    mov        eax, [esp + 4]  // src_ptr
    // src_stride ignored
    mov        edx, [esp + 12]  // dst_ptr
    mov        ecx, [esp + 16]  // dst_width
Frank Barchard's avatar
Frank Barchard committed
678 679
    movdqa     xmm4, xmmword ptr kShuf38a
    movdqa     xmm5, xmmword ptr kShuf38b
680 681

  xloop:
Frank Barchard's avatar
Frank Barchard committed
682 683
    movdqu     xmm0, [eax]  // 16 pixels -> 0,1,2,3,4,5
    movdqu     xmm1, [eax + 16]  // 16 pixels -> 6,7,8,9,10,11
684 685 686 687 688
    lea        eax, [eax + 32]
    pshufb     xmm0, xmm4
    pshufb     xmm1, xmm5
    paddusb    xmm0, xmm1

Frank Barchard's avatar
Frank Barchard committed
689
    movq       qword ptr [edx], xmm0       // write 12 pixels
690 691 692
    movhlps    xmm1, xmm0
    movd       [edx + 8], xmm1
    lea        edx, [edx + 12]
693
    sub        ecx, 12
694 695 696 697 698 699 700
    jg         xloop

    ret
  }
}

// Scale 16x3 pixels to 6x1 with interpolation
Frank Barchard's avatar
Frank Barchard committed
701
__declspec(naked) void ScaleRowDown38_3_Box_SSSE3(const uint8_t* src_ptr,
Frank Barchard's avatar
Frank Barchard committed
702
                                                  ptrdiff_t src_stride,
Frank Barchard's avatar
Frank Barchard committed
703
                                                  uint8_t* dst_ptr,
Frank Barchard's avatar
Frank Barchard committed
704
                                                  int dst_width) {
705 706
  __asm {
    push       esi
Frank Barchard's avatar
Frank Barchard committed
707 708 709 710
    mov        eax, [esp + 4 + 4]  // src_ptr
    mov        esi, [esp + 4 + 8]  // src_stride
    mov        edx, [esp + 4 + 12]  // dst_ptr
    mov        ecx, [esp + 4 + 16]  // dst_width
Frank Barchard's avatar
Frank Barchard committed
711 712 713
    movdqa     xmm2, xmmword ptr kShufAc
    movdqa     xmm3, xmmword ptr kShufAc3
    movdqa     xmm4, xmmword ptr kScaleAc33
714 715 716
    pxor       xmm5, xmm5

  xloop:
Frank Barchard's avatar
Frank Barchard committed
717
    movdqu     xmm0, [eax]  // sum up 3 rows into xmm0/1
718
    movdqu     xmm6, [eax + esi]
719 720 721 722 723 724 725 726
    movhlps    xmm1, xmm0
    movhlps    xmm7, xmm6
    punpcklbw  xmm0, xmm5
    punpcklbw  xmm1, xmm5
    punpcklbw  xmm6, xmm5
    punpcklbw  xmm7, xmm5
    paddusw    xmm0, xmm6
    paddusw    xmm1, xmm7
727
    movdqu     xmm6, [eax + esi * 2]
728 729 730 731 732 733 734
    lea        eax, [eax + 16]
    movhlps    xmm7, xmm6
    punpcklbw  xmm6, xmm5
    punpcklbw  xmm7, xmm5
    paddusw    xmm0, xmm6
    paddusw    xmm1, xmm7

Frank Barchard's avatar
Frank Barchard committed
735
    movdqa     xmm6, xmm0  // 8 pixels -> 0,1,2 of xmm6
736 737 738 739 740 741
    psrldq     xmm0, 2
    paddusw    xmm6, xmm0
    psrldq     xmm0, 2
    paddusw    xmm6, xmm0
    pshufb     xmm6, xmm2

Frank Barchard's avatar
Frank Barchard committed
742
    movdqa     xmm7, xmm1  // 8 pixels -> 3,4,5 of xmm6
743 744 745 746 747 748 749
    psrldq     xmm1, 2
    paddusw    xmm7, xmm1
    psrldq     xmm1, 2
    paddusw    xmm7, xmm1
    pshufb     xmm7, xmm3
    paddusw    xmm6, xmm7

Frank Barchard's avatar
Frank Barchard committed
750
    pmulhuw    xmm6, xmm4  // divide by 9,9,6, 9,9,6
751 752
    packuswb   xmm6, xmm6

Frank Barchard's avatar
Frank Barchard committed
753
    movd       [edx], xmm6  // write 6 pixels
754 755 756
    psrlq      xmm6, 16
    movd       [edx + 2], xmm6
    lea        edx, [edx + 6]
757
    sub        ecx, 6
758 759 760 761 762 763 764 765
    jg         xloop

    pop        esi
    ret
  }
}

// Scale 16x2 pixels to 6x1 with interpolation
Frank Barchard's avatar
Frank Barchard committed
766
__declspec(naked) void ScaleRowDown38_2_Box_SSSE3(const uint8_t* src_ptr,
Frank Barchard's avatar
Frank Barchard committed
767
                                                  ptrdiff_t src_stride,
Frank Barchard's avatar
Frank Barchard committed
768
                                                  uint8_t* dst_ptr,
Frank Barchard's avatar
Frank Barchard committed
769
                                                  int dst_width) {
770 771
  __asm {
    push       esi
Frank Barchard's avatar
Frank Barchard committed
772 773 774 775
    mov        eax, [esp + 4 + 4]  // src_ptr
    mov        esi, [esp + 4 + 8]  // src_stride
    mov        edx, [esp + 4 + 12]  // dst_ptr
    mov        ecx, [esp + 4 + 16]  // dst_width
Frank Barchard's avatar
Frank Barchard committed
776 777 778 779
    movdqa     xmm2, xmmword ptr kShufAb0
    movdqa     xmm3, xmmword ptr kShufAb1
    movdqa     xmm4, xmmword ptr kShufAb2
    movdqa     xmm5, xmmword ptr kScaleAb2
780 781

  xloop:
Frank Barchard's avatar
Frank Barchard committed
782
    movdqu     xmm0, [eax]  // average 2 rows into xmm0
783
    movdqu     xmm1, [eax + esi]
784
    lea        eax, [eax + 16]
785
    pavgb      xmm0, xmm1
786

Frank Barchard's avatar
Frank Barchard committed
787
    movdqa     xmm1, xmm0  // 16 pixels -> 0,1,2,3,4,5 of xmm1
788 789 790 791 792 793 794
    pshufb     xmm1, xmm2
    movdqa     xmm6, xmm0
    pshufb     xmm6, xmm3
    paddusw    xmm1, xmm6
    pshufb     xmm0, xmm4
    paddusw    xmm1, xmm0

Frank Barchard's avatar
Frank Barchard committed
795
    pmulhuw    xmm1, xmm5  // divide by 3,3,2, 3,3,2
796 797
    packuswb   xmm1, xmm1

Frank Barchard's avatar
Frank Barchard committed
798
    movd       [edx], xmm1  // write 6 pixels
799 800 801
    psrlq      xmm1, 16
    movd       [edx + 2], xmm1
    lea        edx, [edx + 6]
802
    sub        ecx, 6
803 804 805 806 807 808 809
    jg         xloop

    pop        esi
    ret
  }
}

810
// Reads 16 bytes and accumulates to 16 shorts at a time.
Frank Barchard's avatar
Frank Barchard committed
811 812
__declspec(naked) void ScaleAddRow_SSE2(const uint8_t* src_ptr,
                                        uint16_t* dst_ptr,
Frank Barchard's avatar
Frank Barchard committed
813
                                        int src_width) {
814
  __asm {
Frank Barchard's avatar
Frank Barchard committed
815 816
    mov        eax, [esp + 4]  // src_ptr
    mov        edx, [esp + 8]  // dst_ptr
817 818
    mov        ecx, [esp + 12]  // src_width
    pxor       xmm5, xmm5
819

820
        // sum rows
821
  xloop:
Frank Barchard's avatar
Frank Barchard committed
822
    movdqu     xmm3, [eax]  // read 16 bytes
823
    lea        eax, [eax + 16]
Frank Barchard's avatar
Frank Barchard committed
824
    movdqu     xmm0, [edx]  // read 16 words from destination
825 826 827 828
    movdqu     xmm1, [edx + 16]
    movdqa     xmm2, xmm3
    punpcklbw  xmm2, xmm5
    punpckhbw  xmm3, xmm5
Frank Barchard's avatar
Frank Barchard committed
829
    paddusw    xmm0, xmm2  // sum 16 words
830
    paddusw    xmm1, xmm3
Frank Barchard's avatar
Frank Barchard committed
831
    movdqu     [edx], xmm0  // write 16 words to destination
832 833
    movdqu     [edx + 16], xmm1
    lea        edx, [edx + 32]
834 835 836 837 838 839
    sub        ecx, 16
    jg         xloop
    ret
  }
}

840
#ifdef HAS_SCALEADDROW_AVX2
841
// Reads 32 bytes and accumulates to 32 shorts at a time.
Frank Barchard's avatar
Frank Barchard committed
842 843
__declspec(naked) void ScaleAddRow_AVX2(const uint8_t* src_ptr,
                                        uint16_t* dst_ptr,
Frank Barchard's avatar
Frank Barchard committed
844
                                        int src_width) {
845
  __asm {
Frank Barchard's avatar
Frank Barchard committed
846 847
    mov         eax, [esp + 4]  // src_ptr
    mov         edx, [esp + 8]  // dst_ptr
848 849
    mov         ecx, [esp + 12]  // src_width
    vpxor       ymm5, ymm5, ymm5
850

851
        // sum rows
852
  xloop:
Frank Barchard's avatar
Frank Barchard committed
853
    vmovdqu     ymm3, [eax]  // read 32 bytes
854
    lea         eax, [eax + 32]
855
    vpermq      ymm3, ymm3, 0xd8  // unmutate for vpunpck
856 857
    vpunpcklbw  ymm2, ymm3, ymm5
    vpunpckhbw  ymm3, ymm3, ymm5
Frank Barchard's avatar
Frank Barchard committed
858
    vpaddusw    ymm0, ymm2, [edx]  // sum 16 words
859
    vpaddusw    ymm1, ymm3, [edx + 32]
Frank Barchard's avatar
Frank Barchard committed
860
    vmovdqu     [edx], ymm0  // write 32 words to destination
861
    vmovdqu     [edx + 32], ymm1
862
    lea         edx, [edx + 64]
863 864
    sub         ecx, 32
    jg          xloop
865

866 867 868 869
    vzeroupper
    ret
  }
}
870
#endif  // HAS_SCALEADDROW_AVX2
871

872 873
// Constant for making pixels signed to avoid pmaddubsw
// saturation.
874 875
static const uvec8 kFsub80 = {0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
                              0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80};
876 877

// Constant for making pixels unsigned and adding .5 for rounding.
878 879
static const uvec16 kFadd40 = {0x4040, 0x4040, 0x4040, 0x4040,
                               0x4040, 0x4040, 0x4040, 0x4040};
880

881
// Bilinear column filtering. SSSE3 version.
Frank Barchard's avatar
Frank Barchard committed
882 883
__declspec(naked) void ScaleFilterCols_SSSE3(uint8_t* dst_ptr,
                                             const uint8_t* src_ptr,
Frank Barchard's avatar
Frank Barchard committed
884 885 886
                                             int dst_width,
                                             int x,
                                             int dx) {
887 888 889 890
  __asm {
    push       ebx
    push       esi
    push       edi
Frank Barchard's avatar
Frank Barchard committed
891 892 893
    mov        edi, [esp + 12 + 4]  // dst_ptr
    mov        esi, [esp + 12 + 8]  // src_ptr
    mov        ecx, [esp + 12 + 12]  // dst_width
894 895
    movd       xmm2, [esp + 12 + 16]  // x
    movd       xmm3, [esp + 12 + 20]  // dx
Frank Barchard's avatar
Frank Barchard committed
896
    mov        eax, 0x04040000  // shuffle to line up fractions with pixel.
897
    movd       xmm5, eax
Frank Barchard's avatar
Frank Barchard committed
898
    pcmpeqb    xmm6, xmm6  // generate 0x007f for inverting fraction.
899
    psrlw      xmm6, 9
Frank Barchard's avatar
Frank Barchard committed
900
    pcmpeqb    xmm7, xmm7  // generate 0x0001
901
    psrlw      xmm7, 15
Frank Barchard's avatar
Frank Barchard committed
902
    pextrw     eax, xmm2, 1  // get x0 integer. preroll
903 904 905
    sub        ecx, 2
    jl         xloop29

Frank Barchard's avatar
Frank Barchard committed
906
    movdqa     xmm0, xmm2  // x1 = x0 + dx
907
    paddd      xmm0, xmm3
Frank Barchard's avatar
Frank Barchard committed
908 909 910 911
    punpckldq  xmm2, xmm0  // x0 x1
    punpckldq  xmm3, xmm3  // dx dx
    paddd      xmm3, xmm3  // dx * 2, dx * 2
    pextrw     edx, xmm2, 3  // get x1 integer. preroll
912 913 914

    // 2 Pixel loop.
  xloop2:
Frank Barchard's avatar
Frank Barchard committed
915 916
    movdqa     xmm1, xmm2  // x0, x1 fractions.
    paddd      xmm2, xmm3  // x += dx
917
    movzx      ebx, word ptr [esi + eax]  // 2 source x0 pixels
918
    movd       xmm0, ebx
Frank Barchard's avatar
Frank Barchard committed
919
    psrlw      xmm1, 9  // 7 bit fractions.
920
    movzx      ebx, word ptr [esi + edx]  // 2 source x1 pixels
921
    movd       xmm4, ebx
Frank Barchard's avatar
Frank Barchard committed
922
    pshufb     xmm1, xmm5  // 0011
923
    punpcklwd  xmm0, xmm4
924
    psubb      xmm0, xmmword ptr kFsub80  // make pixels signed.
Frank Barchard's avatar
Frank Barchard committed
925 926 927 928 929
    pxor       xmm1, xmm6  // 0..7f and 7f..0
    paddusb    xmm1, xmm7  // +1 so 0..7f and 80..1
    pmaddubsw  xmm1, xmm0  // 16 bit, 2 pixels.
    pextrw     eax, xmm2, 1  // get x0 integer. next iteration.
    pextrw     edx, xmm2, 3  // get x1 integer. next iteration.
930
    paddw      xmm1, xmmword ptr kFadd40  // make pixels unsigned and round.
Frank Barchard's avatar
Frank Barchard committed
931 932
    psrlw      xmm1, 7  // 8.7 fixed point to low 8 bits.
    packuswb   xmm1, xmm1  // 8 bits, 2 pixels.
933
    movd       ebx, xmm1
934 935
    mov        [edi], bx
    lea        edi, [edi + 2]
Frank Barchard's avatar
Frank Barchard committed
936
    sub        ecx, 2  // 2 pixels
937 938 939 940 941 942
    jge        xloop2

 xloop29:
    add        ecx, 2 - 1
    jl         xloop99

943
            // 1 pixel remainder
944
    movzx      ebx, word ptr [esi + eax]  // 2 source x0 pixels
945
    movd       xmm0, ebx
Frank Barchard's avatar
Frank Barchard committed
946 947
    psrlw      xmm2, 9  // 7 bit fractions.
    pshufb     xmm2, xmm5  // 0011
948
    psubb      xmm0, xmmword ptr kFsub80  // make pixels signed.
Frank Barchard's avatar
Frank Barchard committed
949 950 951
    pxor       xmm2, xmm6  // 0..7f and 7f..0
    paddusb    xmm2, xmm7  // +1 so 0..7f and 80..1
    pmaddubsw  xmm2, xmm0  // 16 bit
952
    paddw      xmm2, xmmword ptr kFadd40  // make pixels unsigned and round.
Frank Barchard's avatar
Frank Barchard committed
953 954
    psrlw      xmm2, 7  // 8.7 fixed point to low 8 bits.
    packuswb   xmm2, xmm2  // 8 bits
955
    movd       ebx, xmm2
956 957 958 959 960 961 962 963 964 965 966 967
    mov        [edi], bl

 xloop99:

    pop        edi
    pop        esi
    pop        ebx
    ret
  }
}

// Reads 16 pixels, duplicates them and writes 32 pixels.
Frank Barchard's avatar
Frank Barchard committed
968 969
__declspec(naked) void ScaleColsUp2_SSE2(uint8_t* dst_ptr,
                                         const uint8_t* src_ptr,
Frank Barchard's avatar
Frank Barchard committed
970 971 972
                                         int dst_width,
                                         int x,
                                         int dx) {
973
  __asm {
Frank Barchard's avatar
Frank Barchard committed
974 975 976
    mov        edx, [esp + 4]  // dst_ptr
    mov        eax, [esp + 8]  // src_ptr
    mov        ecx, [esp + 12]  // dst_width
977 978

  wloop:
979
    movdqu     xmm0, [eax]
980 981 982 983
    lea        eax,  [eax + 16]
    movdqa     xmm1, xmm0
    punpcklbw  xmm0, xmm0
    punpckhbw  xmm1, xmm1
984 985
    movdqu     [edx], xmm0
    movdqu     [edx + 16], xmm1
986
    lea        edx, [edx + 32]
987
    sub        ecx, 32
988 989 990 991 992 993 994
    jg         wloop

    ret
  }
}

// Reads 8 pixels, throws half away and writes 4 even pixels (0, 2, 4, 6)
Frank Barchard's avatar
Frank Barchard committed
995
__declspec(naked) void ScaleARGBRowDown2_SSE2(const uint8_t* src_argb,
Frank Barchard's avatar
Frank Barchard committed
996
                                              ptrdiff_t src_stride,
Frank Barchard's avatar
Frank Barchard committed
997
                                              uint8_t* dst_argb,
Frank Barchard's avatar
Frank Barchard committed
998
                                              int dst_width) {
999
  __asm {
Frank Barchard's avatar
Frank Barchard committed
1000 1001 1002 1003
    mov        eax, [esp + 4]   // src_argb
    // src_stride ignored
    mov        edx, [esp + 12]  // dst_argb
    mov        ecx, [esp + 16]  // dst_width
1004 1005

  wloop:
1006 1007
    movdqu     xmm0, [eax]
    movdqu     xmm1, [eax + 16]
1008 1009
    lea        eax,  [eax + 32]
    shufps     xmm0, xmm1, 0xdd
1010
    movdqu     [edx], xmm0
1011
    lea        edx, [edx + 16]
1012
    sub        ecx, 4
1013 1014 1015 1016 1017 1018 1019
    jg         wloop

    ret
  }
}

// Blends 8x1 rectangle to 4x1.
Frank Barchard's avatar
Frank Barchard committed
1020
__declspec(naked) void ScaleARGBRowDown2Linear_SSE2(const uint8_t* src_argb,
Frank Barchard's avatar
Frank Barchard committed
1021
                                                    ptrdiff_t src_stride,
Frank Barchard's avatar
Frank Barchard committed
1022
                                                    uint8_t* dst_argb,
Frank Barchard's avatar
Frank Barchard committed
1023
                                                    int dst_width) {
1024
  __asm {
Frank Barchard's avatar
Frank Barchard committed
1025 1026 1027 1028
    mov        eax, [esp + 4]  // src_argb
    // src_stride ignored
    mov        edx, [esp + 12]  // dst_argb
    mov        ecx, [esp + 16]  // dst_width
1029 1030

  wloop:
1031 1032
    movdqu     xmm0, [eax]
    movdqu     xmm1, [eax + 16]
1033 1034
    lea        eax,  [eax + 32]
    movdqa     xmm2, xmm0
Frank Barchard's avatar
Frank Barchard committed
1035 1036
    shufps     xmm0, xmm1, 0x88  // even pixels
    shufps     xmm2, xmm1, 0xdd       // odd pixels
1037
    pavgb      xmm0, xmm2
1038
    movdqu     [edx], xmm0
1039
    lea        edx, [edx + 16]
1040
    sub        ecx, 4
1041 1042 1043 1044 1045 1046 1047
    jg         wloop

    ret
  }
}

// Blends 8x2 rectangle to 4x1.
Frank Barchard's avatar
Frank Barchard committed
1048
__declspec(naked) void ScaleARGBRowDown2Box_SSE2(const uint8_t* src_argb,
Frank Barchard's avatar
Frank Barchard committed
1049
                                                 ptrdiff_t src_stride,
Frank Barchard's avatar
Frank Barchard committed
1050
                                                 uint8_t* dst_argb,
Frank Barchard's avatar
Frank Barchard committed
1051
                                                 int dst_width) {
1052 1053
  __asm {
    push       esi
Frank Barchard's avatar
Frank Barchard committed
1054 1055 1056 1057
    mov        eax, [esp + 4 + 4]  // src_argb
    mov        esi, [esp + 4 + 8]  // src_stride
    mov        edx, [esp + 4 + 12]  // dst_argb
    mov        ecx, [esp + 4 + 16]  // dst_width
1058 1059

  wloop:
1060 1061 1062 1063
    movdqu     xmm0, [eax]
    movdqu     xmm1, [eax + 16]
    movdqu     xmm2, [eax + esi]
    movdqu     xmm3, [eax + esi + 16]
1064
    lea        eax,  [eax + 32]
Frank Barchard's avatar
Frank Barchard committed
1065
    pavgb      xmm0, xmm2  // average rows
1066
    pavgb      xmm1, xmm3
Frank Barchard's avatar
Frank Barchard committed
1067 1068 1069
    movdqa     xmm2, xmm0  // average columns (8 to 4 pixels)
    shufps     xmm0, xmm1, 0x88  // even pixels
    shufps     xmm2, xmm1, 0xdd  // odd pixels
1070
    pavgb      xmm0, xmm2
1071
    movdqu     [edx], xmm0
1072
    lea        edx, [edx + 16]
1073
    sub        ecx, 4
1074 1075 1076 1077 1078 1079 1080 1081
    jg         wloop

    pop        esi
    ret
  }
}

// Reads 4 pixels at a time.
Frank Barchard's avatar
Frank Barchard committed
1082
__declspec(naked) void ScaleARGBRowDownEven_SSE2(const uint8_t* src_argb,
Frank Barchard's avatar
Frank Barchard committed
1083 1084
                                                 ptrdiff_t src_stride,
                                                 int src_stepx,
Frank Barchard's avatar
Frank Barchard committed
1085
                                                 uint8_t* dst_argb,
Frank Barchard's avatar
Frank Barchard committed
1086
                                                 int dst_width) {
1087 1088 1089
  __asm {
    push       ebx
    push       edi
Frank Barchard's avatar
Frank Barchard committed
1090 1091 1092 1093 1094
    mov        eax, [esp + 8 + 4]   // src_argb
    // src_stride ignored
    mov        ebx, [esp + 8 + 12]  // src_stepx
    mov        edx, [esp + 8 + 16]  // dst_argb
    mov        ecx, [esp + 8 + 20]  // dst_width
1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106
    lea        ebx, [ebx * 4]
    lea        edi, [ebx + ebx * 2]

  wloop:
    movd       xmm0, [eax]
    movd       xmm1, [eax + ebx]
    punpckldq  xmm0, xmm1
    movd       xmm2, [eax + ebx * 2]
    movd       xmm3, [eax + edi]
    lea        eax,  [eax + ebx * 4]
    punpckldq  xmm2, xmm3
    punpcklqdq xmm0, xmm2
1107
    movdqu     [edx], xmm0
1108
    lea        edx, [edx + 16]
1109
    sub        ecx, 4
1110 1111 1112 1113 1114 1115 1116 1117 1118
    jg         wloop

    pop        edi
    pop        ebx
    ret
  }
}

// Blends four 2x2 to 4x1.
Frank Barchard's avatar
Frank Barchard committed
1119
__declspec(naked) void ScaleARGBRowDownEvenBox_SSE2(const uint8_t* src_argb,
Frank Barchard's avatar
Frank Barchard committed
1120 1121
                                                    ptrdiff_t src_stride,
                                                    int src_stepx,
Frank Barchard's avatar
Frank Barchard committed
1122
                                                    uint8_t* dst_argb,
Frank Barchard's avatar
Frank Barchard committed
1123
                                                    int dst_width) {
1124 1125 1126 1127
  __asm {
    push       ebx
    push       esi
    push       edi
Frank Barchard's avatar
Frank Barchard committed
1128 1129 1130 1131 1132 1133
    mov        eax, [esp + 12 + 4]  // src_argb
    mov        esi, [esp + 12 + 8]  // src_stride
    mov        ebx, [esp + 12 + 12]  // src_stepx
    mov        edx, [esp + 12 + 16]  // dst_argb
    mov        ecx, [esp + 12 + 20]  // dst_width
    lea        esi, [eax + esi]  // row1 pointer
1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147
    lea        ebx, [ebx * 4]
    lea        edi, [ebx + ebx * 2]

  wloop:
    movq       xmm0, qword ptr [eax]  // row0 4 pairs
    movhps     xmm0, qword ptr [eax + ebx]
    movq       xmm1, qword ptr [eax + ebx * 2]
    movhps     xmm1, qword ptr [eax + edi]
    lea        eax,  [eax + ebx * 4]
    movq       xmm2, qword ptr [esi]  // row1 4 pairs
    movhps     xmm2, qword ptr [esi + ebx]
    movq       xmm3, qword ptr [esi + ebx * 2]
    movhps     xmm3, qword ptr [esi + edi]
    lea        esi,  [esi + ebx * 4]
Frank Barchard's avatar
Frank Barchard committed
1148
    pavgb      xmm0, xmm2  // average rows
1149
    pavgb      xmm1, xmm3
Frank Barchard's avatar
Frank Barchard committed
1150 1151 1152
    movdqa     xmm2, xmm0  // average columns (8 to 4 pixels)
    shufps     xmm0, xmm1, 0x88  // even pixels
    shufps     xmm2, xmm1, 0xdd  // odd pixels
1153
    pavgb      xmm0, xmm2
1154
    movdqu     [edx], xmm0
1155
    lea        edx, [edx + 16]
1156
    sub        ecx, 4
1157 1158 1159 1160 1161 1162 1163 1164 1165 1166
    jg         wloop

    pop        edi
    pop        esi
    pop        ebx
    ret
  }
}

// Column scaling unfiltered. SSE2 version.
Frank Barchard's avatar
Frank Barchard committed
1167 1168
__declspec(naked) void ScaleARGBCols_SSE2(uint8_t* dst_argb,
                                          const uint8_t* src_argb,
Frank Barchard's avatar
Frank Barchard committed
1169 1170 1171
                                          int dst_width,
                                          int x,
                                          int dx) {
1172 1173 1174
  __asm {
    push       edi
    push       esi
Frank Barchard's avatar
Frank Barchard committed
1175 1176 1177
    mov        edi, [esp + 8 + 4]  // dst_argb
    mov        esi, [esp + 8 + 8]  // src_argb
    mov        ecx, [esp + 8 + 12]  // dst_width
1178 1179 1180
    movd       xmm2, [esp + 8 + 16]  // x
    movd       xmm3, [esp + 8 + 20]  // dx

Frank Barchard's avatar
Frank Barchard committed
1181 1182
    pshufd     xmm2, xmm2, 0  // x0 x0 x0 x0
    pshufd     xmm0, xmm3, 0x11  // dx  0 dx  0
1183
    paddd      xmm2, xmm0
Frank Barchard's avatar
Frank Barchard committed
1184 1185 1186 1187 1188
    paddd      xmm3, xmm3  // 0, 0, 0,  dx * 2
    pshufd     xmm0, xmm3, 0x05  // dx * 2, dx * 2, 0, 0
    paddd      xmm2, xmm0  // x3 x2 x1 x0
    paddd      xmm3, xmm3  // 0, 0, 0,  dx * 4
    pshufd     xmm3, xmm3, 0  // dx * 4, dx * 4, dx * 4, dx * 4
1189

Frank Barchard's avatar
Frank Barchard committed
1190 1191
    pextrw     eax, xmm2, 1  // get x0 integer.
    pextrw     edx, xmm2, 3  // get x1 integer.
1192 1193 1194 1195 1196 1197

    cmp        ecx, 0
    jle        xloop99
    sub        ecx, 4
    jl         xloop49

1198
        // 4 Pixel loop.
1199 1200 1201
 xloop4:
    movd       xmm0, [esi + eax * 4]  // 1 source x0 pixels
    movd       xmm1, [esi + edx * 4]  // 1 source x1 pixels
Frank Barchard's avatar
Frank Barchard committed
1202 1203 1204 1205
    pextrw     eax, xmm2, 5  // get x2 integer.
    pextrw     edx, xmm2, 7  // get x3 integer.
    paddd      xmm2, xmm3  // x += dx
    punpckldq  xmm0, xmm1  // x0 x1
1206 1207 1208

    movd       xmm1, [esi + eax * 4]  // 1 source x2 pixels
    movd       xmm4, [esi + edx * 4]  // 1 source x3 pixels
Frank Barchard's avatar
Frank Barchard committed
1209 1210 1211 1212
    pextrw     eax, xmm2, 1  // get x0 integer. next iteration.
    pextrw     edx, xmm2, 3  // get x1 integer. next iteration.
    punpckldq  xmm1, xmm4  // x2 x3
    punpcklqdq xmm0, xmm1  // x0 x1 x2 x3
1213 1214
    movdqu     [edi], xmm0
    lea        edi, [edi + 16]
Frank Barchard's avatar
Frank Barchard committed
1215
    sub        ecx, 4  // 4 pixels
1216 1217 1218 1219 1220 1221
    jge        xloop4

 xloop49:
    test       ecx, 2
    je         xloop29

1222
        // 2 Pixels.
1223 1224
    movd       xmm0, [esi + eax * 4]  // 1 source x0 pixels
    movd       xmm1, [esi + edx * 4]  // 1 source x1 pixels
Frank Barchard's avatar
Frank Barchard committed
1225 1226
    pextrw     eax, xmm2, 5  // get x2 integer.
    punpckldq  xmm0, xmm1  // x0 x1
1227 1228 1229 1230 1231 1232 1233 1234

    movq       qword ptr [edi], xmm0
    lea        edi, [edi + 8]

 xloop29:
    test       ecx, 1
    je         xloop99

1235
        // 1 Pixels.
1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249
    movd       xmm0, [esi + eax * 4]  // 1 source x2 pixels
    movd       dword ptr [edi], xmm0
 xloop99:

    pop        esi
    pop        edi
    ret
  }
}

// Bilinear row filtering combines 2x1 -> 1x1. SSSE3 version.
// TODO(fbarchard): Port to Neon

// Shuffle table for arranging 2 pixels into pairs for pmaddubsw
1250
static const uvec8 kShuffleColARGB = {
Frank Barchard's avatar
Frank Barchard committed
1251 1252
    0u, 4u,  1u, 5u,  2u,  6u,  3u,  7u,  // bbggrraa 1st pixel
    8u, 12u, 9u, 13u, 10u, 14u, 11u, 15u  // bbggrraa 2nd pixel
1253 1254 1255
};

// Shuffle table for duplicating 2 fractions into 8 bytes each
1256
static const uvec8 kShuffleFractions = {
Frank Barchard's avatar
Frank Barchard committed
1257
    0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, 4u,
1258 1259
};

Frank Barchard's avatar
Frank Barchard committed
1260 1261
__declspec(naked) void ScaleARGBFilterCols_SSSE3(uint8_t* dst_argb,
                                                 const uint8_t* src_argb,
Frank Barchard's avatar
Frank Barchard committed
1262 1263 1264
                                                 int dst_width,
                                                 int x,
                                                 int dx) {
1265 1266 1267
  __asm {
    push       esi
    push       edi
Frank Barchard's avatar
Frank Barchard committed
1268 1269 1270
    mov        edi, [esp + 8 + 4]  // dst_argb
    mov        esi, [esp + 8 + 8]  // src_argb
    mov        ecx, [esp + 8 + 12]  // dst_width
1271 1272
    movd       xmm2, [esp + 8 + 16]  // x
    movd       xmm3, [esp + 8 + 20]  // dx
Frank Barchard's avatar
Frank Barchard committed
1273 1274
    movdqa     xmm4, xmmword ptr kShuffleColARGB
    movdqa     xmm5, xmmword ptr kShuffleFractions
Frank Barchard's avatar
Frank Barchard committed
1275
    pcmpeqb    xmm6, xmm6  // generate 0x007f for inverting fraction.
1276
    psrlw      xmm6, 9
Frank Barchard's avatar
Frank Barchard committed
1277
    pextrw     eax, xmm2, 1  // get x0 integer. preroll
1278 1279 1280
    sub        ecx, 2
    jl         xloop29

Frank Barchard's avatar
Frank Barchard committed
1281
    movdqa     xmm0, xmm2  // x1 = x0 + dx
1282
    paddd      xmm0, xmm3
Frank Barchard's avatar
Frank Barchard committed
1283 1284 1285 1286
    punpckldq  xmm2, xmm0  // x0 x1
    punpckldq  xmm3, xmm3  // dx dx
    paddd      xmm3, xmm3  // dx * 2, dx * 2
    pextrw     edx, xmm2, 3  // get x1 integer. preroll
1287 1288 1289

    // 2 Pixel loop.
  xloop2:
Frank Barchard's avatar
Frank Barchard committed
1290 1291
    movdqa     xmm1, xmm2  // x0, x1 fractions.
    paddd      xmm2, xmm3  // x += dx
1292
    movq       xmm0, qword ptr [esi + eax * 4]  // 2 source x0 pixels
Frank Barchard's avatar
Frank Barchard committed
1293
    psrlw      xmm1, 9  // 7 bit fractions.
1294
    movhps     xmm0, qword ptr [esi + edx * 4]  // 2 source x1 pixels
Frank Barchard's avatar
Frank Barchard committed
1295 1296 1297 1298 1299 1300 1301 1302
    pshufb     xmm1, xmm5  // 0000000011111111
    pshufb     xmm0, xmm4  // arrange pixels into pairs
    pxor       xmm1, xmm6  // 0..7f and 7f..0
    pmaddubsw  xmm0, xmm1  // argb_argb 16 bit, 2 pixels.
    pextrw     eax, xmm2, 1  // get x0 integer. next iteration.
    pextrw     edx, xmm2, 3  // get x1 integer. next iteration.
    psrlw      xmm0, 7  // argb 8.7 fixed point to low 8 bits.
    packuswb   xmm0, xmm0  // argb_argb 8 bits, 2 pixels.
1303 1304
    movq       qword ptr [edi], xmm0
    lea        edi, [edi + 8]
Frank Barchard's avatar
Frank Barchard committed
1305
    sub        ecx, 2  // 2 pixels
1306 1307 1308 1309 1310 1311 1312
    jge        xloop2

 xloop29:

    add        ecx, 2 - 1
    jl         xloop99

1313
            // 1 pixel remainder
Frank Barchard's avatar
Frank Barchard committed
1314
    psrlw      xmm2, 9  // 7 bit fractions.
1315
    movq       xmm0, qword ptr [esi + eax * 4]  // 2 source x0 pixels
Frank Barchard's avatar
Frank Barchard committed
1316 1317 1318 1319
    pshufb     xmm2, xmm5  // 00000000
    pshufb     xmm0, xmm4  // arrange pixels into pairs
    pxor       xmm2, xmm6  // 0..7f and 7f..0
    pmaddubsw  xmm0, xmm2  // argb 16 bit, 1 pixel.
1320
    psrlw      xmm0, 7
Frank Barchard's avatar
Frank Barchard committed
1321
    packuswb   xmm0, xmm0  // argb 8 bits, 1 pixel.
1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332
    movd       [edi], xmm0

 xloop99:

    pop        edi
    pop        esi
    ret
  }
}

// Reads 4 pixels, duplicates them and writes 8 pixels.
Frank Barchard's avatar
Frank Barchard committed
1333 1334
__declspec(naked) void ScaleARGBColsUp2_SSE2(uint8_t* dst_argb,
                                             const uint8_t* src_argb,
Frank Barchard's avatar
Frank Barchard committed
1335 1336 1337
                                             int dst_width,
                                             int x,
                                             int dx) {
1338
  __asm {
Frank Barchard's avatar
Frank Barchard committed
1339 1340 1341
    mov        edx, [esp + 4]  // dst_argb
    mov        eax, [esp + 8]  // src_argb
    mov        ecx, [esp + 12]  // dst_width
1342 1343

  wloop:
1344
    movdqu     xmm0, [eax]
1345 1346 1347 1348
    lea        eax,  [eax + 16]
    movdqa     xmm1, xmm0
    punpckldq  xmm0, xmm0
    punpckhdq  xmm1, xmm1
1349 1350
    movdqu     [edx], xmm0
    movdqu     [edx + 16], xmm1
1351
    lea        edx, [edx + 32]
1352
    sub        ecx, 8
1353 1354 1355 1356 1357 1358
    jg         wloop

    ret
  }
}

1359
// Divide num by div and return as 16.16 fixed point result.
Frank Barchard's avatar
Frank Barchard committed
1360
__declspec(naked) int FixedDiv_X86(int num, int div) {
1361
  __asm {
Frank Barchard's avatar
Frank Barchard committed
1362 1363 1364
    mov        eax, [esp + 4]  // num
    cdq  // extend num to 64 bits
    shld       edx, eax, 16  // 32.16
1365 1366 1367 1368 1369 1370 1371
    shl        eax, 16
    idiv       dword ptr [esp + 8]
    ret
  }
}

// Divide num by div and return as 16.16 fixed point result.
Frank Barchard's avatar
Frank Barchard committed
1372
__declspec(naked) int FixedDiv1_X86(int num, int div) {
1373
  __asm {
Frank Barchard's avatar
Frank Barchard committed
1374 1375 1376 1377
    mov        eax, [esp + 4]  // num
    mov        ecx, [esp + 8]  // denom
    cdq  // extend num to 64 bits
    shld       edx, eax, 16  // 32.16
1378 1379 1380 1381 1382 1383 1384 1385
    shl        eax, 16
    sub        eax, 0x00010001
    sbb        edx, 0
    sub        ecx, 1
    idiv       ecx
    ret
  }
}
1386
#endif  // !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86)
1387 1388 1389 1390 1391

#ifdef __cplusplus
}  // extern "C"
}  // namespace libyuv
#endif