| 1 | ;***************************************************************************** |
| 2 | ;* x86inc.asm: x264asm abstraction layer |
| 3 | ;***************************************************************************** |
| 4 | ;* Copyright (C) 2005-2013 x264 project |
| 5 | ;* |
| 6 | ;* Authors: Loren Merritt <lorenm@u.washington.edu> |
| 7 | ;* Anton Mitrofanov <BugMaster@narod.ru> |
| 8 | ;* Fiona Glaser <fiona@x264.com> |
| 9 | ;* Henrik Gramner <henrik@gramner.com> |
| 10 | ;* |
| 11 | ;* Permission to use, copy, modify, and/or distribute this software for any |
| 12 | ;* purpose with or without fee is hereby granted, provided that the above |
| 13 | ;* copyright notice and this permission notice appear in all copies. |
| 14 | ;* |
| 15 | ;* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES |
| 16 | ;* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF |
| 17 | ;* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR |
| 18 | ;* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
| 19 | ;* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN |
| 20 | ;* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF |
| 21 | ;* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. |
| 22 | ;***************************************************************************** |
| 23 | |
| 24 | ; This is a header file for the x264ASM assembly language, which uses |
| 25 | ; NASM/YASM syntax combined with a large number of macros to provide easy |
| 26 | ; abstraction between different calling conventions (x86_32, win64, linux64). |
| 27 | ; It also has various other useful features to simplify writing the kind of |
| 28 | ; DSP functions that are most often used in x264. |
| 29 | |
| 30 | ; Unlike the rest of x264, this file is available under an ISC license, as it |
| 31 | ; has significant usefulness outside of x264 and we want it to be available |
| 32 | ; to the largest audience possible. Of course, if you modify it for your own |
| 33 | ; purposes to add a new feature, we strongly encourage contributing a patch |
| 34 | ; as this feature might be useful for others as well. Send patches or ideas |
| 35 | ; to x264-devel@videolan.org . |
| 36 | |
| 37 | %ifndef private_prefix |
| 38 | %define private_prefix x264 |
| 39 | %endif |
| 40 | |
| 41 | %ifndef public_prefix |
| 42 | %define public_prefix private_prefix |
| 43 | %endif |
| 44 | |
| 45 | %define WIN64 0 |
| 46 | %define UNIX64 0 |
| 47 | %if ARCH_X86_64 |
| 48 | %ifidn __OUTPUT_FORMAT__,win32 |
| 49 | %define WIN64 1 |
| 50 | %elifidn __OUTPUT_FORMAT__,win64 |
| 51 | %define WIN64 1 |
| 52 | %elifidn __OUTPUT_FORMAT__,x64 |
| 53 | %define WIN64 1 |
| 54 | %else |
| 55 | %define UNIX64 1 |
| 56 | %endif |
| 57 | %endif |
| 58 | |
| 59 | %ifdef PREFIX |
| 60 | %define mangle(x) _ %+ x |
| 61 | %else |
| 62 | %define mangle(x) x |
| 63 | %endif |
| 64 | |
| 65 | ; aout does not support align= |
| 66 | ; NOTE: This section is out of sync with x264, in order to |
| 67 | ; keep supporting OS/2. |
| 68 | %macro SECTION_RODATA 0-1 16 |
| 69 | %ifidn __OUTPUT_FORMAT__,aout |
| 70 | section .text |
| 71 | %else |
| 72 | SECTION .rodata align=%1 |
| 73 | %endif |
| 74 | %endmacro |
| 75 | |
| 76 | %macro SECTION_TEXT 0-1 16 |
| 77 | %ifidn __OUTPUT_FORMAT__,aout |
| 78 | SECTION .text |
| 79 | %else |
| 80 | SECTION .text align=%1 |
| 81 | %endif |
| 82 | %endmacro |
| 83 | |
| 84 | %if WIN64 |
| 85 | %define PIC |
| 86 | %elif ARCH_X86_64 == 0 |
| 87 | ; x86_32 doesn't require PIC. |
| 88 | ; Some distros prefer shared objects to be PIC, but nothing breaks if |
| 89 | ; the code contains a few textrels, so we'll skip that complexity. |
| 90 | %undef PIC |
| 91 | %endif |
| 92 | %ifdef PIC |
| 93 | default rel |
| 94 | %endif |
| 95 | |
| 96 | %macro CPUNOP 1 |
| 97 | %if HAVE_CPUNOP |
| 98 | CPU %1 |
| 99 | %endif |
| 100 | %endmacro |
| 101 | |
| 102 | ; Macros to eliminate most code duplication between x86_32 and x86_64: |
| 103 | ; Currently this works only for leaf functions which load all their arguments |
| 104 | ; into registers at the start, and make no other use of the stack. Luckily that |
| 105 | ; covers most of x264's asm. |
| 106 | |
| 107 | ; PROLOGUE: |
| 108 | ; %1 = number of arguments. loads them from stack if needed. |
| 109 | ; %2 = number of registers used. pushes callee-saved regs if needed. |
| 110 | ; %3 = number of xmm registers used. pushes callee-saved xmm regs if needed. |
| 111 | ; %4 = (optional) stack size to be allocated. If not aligned (x86-32 ICC 10.x, |
| 112 | ; MSVC or YMM), the stack will be manually aligned (to 16 or 32 bytes), |
| 113 | ; and an extra register will be allocated to hold the original stack |
| 114 | ; pointer (to not invalidate r0m etc.). To prevent the use of an extra |
| 115 | ; register as stack pointer, request a negative stack size. |
| 116 | ; %4+/%5+ = list of names to define to registers |
| 117 | ; PROLOGUE can also be invoked by adding the same options to cglobal |
| 118 | |
| 119 | ; e.g. |
| 120 | ; cglobal foo, 2,3,0, dst, src, tmp |
| 121 | ; declares a function (foo), taking two args (dst and src) and one local variable (tmp) |
| 122 | |
| 123 | ; TODO Some functions can use some args directly from the stack. If they're the |
| 124 | ; last args then you can just not declare them, but if they're in the middle |
| 125 | ; we need more flexible macro. |
| 126 | |
| 127 | ; RET: |
| 128 | ; Pops anything that was pushed by PROLOGUE, and returns. |
| 129 | |
| 130 | ; REP_RET: |
| 131 | ; Use this instead of RET if it's a branch target. |
| 132 | |
| 133 | ; registers: |
| 134 | ; rN and rNq are the native-size register holding function argument N |
| 135 | ; rNd, rNw, rNb are dword, word, and byte size |
| 136 | ; rNh is the high 8 bits of the word size |
| 137 | ; rNm is the original location of arg N (a register or on the stack), dword |
| 138 | ; rNmp is native size |
| 139 | |
| 140 | %macro DECLARE_REG 2-3 |
| 141 | %define r%1q %2 |
| 142 | %define r%1d %2d |
| 143 | %define r%1w %2w |
| 144 | %define r%1b %2b |
| 145 | %define r%1h %2h |
| 146 | %define %2q %2 |
| 147 | %if %0 == 2 |
| 148 | %define r%1m %2d |
| 149 | %define r%1mp %2 |
| 150 | %elif ARCH_X86_64 ; memory |
| 151 | %define r%1m [rstk + stack_offset + %3] |
| 152 | %define r%1mp qword r %+ %1 %+ m |
| 153 | %else |
| 154 | %define r%1m [rstk + stack_offset + %3] |
| 155 | %define r%1mp dword r %+ %1 %+ m |
| 156 | %endif |
| 157 | %define r%1 %2 |
| 158 | %endmacro |
| 159 | |
| 160 | %macro DECLARE_REG_SIZE 3 |
| 161 | %define r%1q r%1 |
| 162 | %define e%1q r%1 |
| 163 | %define r%1d e%1 |
| 164 | %define e%1d e%1 |
| 165 | %define r%1w %1 |
| 166 | %define e%1w %1 |
| 167 | %define r%1h %3 |
| 168 | %define e%1h %3 |
| 169 | %define r%1b %2 |
| 170 | %define e%1b %2 |
| 171 | %if ARCH_X86_64 == 0 |
| 172 | %define r%1 e%1 |
| 173 | %endif |
| 174 | %endmacro |
| 175 | |
| 176 | DECLARE_REG_SIZE ax, al, ah |
| 177 | DECLARE_REG_SIZE bx, bl, bh |
| 178 | DECLARE_REG_SIZE cx, cl, ch |
| 179 | DECLARE_REG_SIZE dx, dl, dh |
| 180 | DECLARE_REG_SIZE si, sil, null |
| 181 | DECLARE_REG_SIZE di, dil, null |
| 182 | DECLARE_REG_SIZE bp, bpl, null |
| 183 | |
| 184 | ; t# defines for when per-arch register allocation is more complex than just function arguments |
| 185 | |
| 186 | %macro DECLARE_REG_TMP 1-* |
| 187 | %assign %%i 0 |
| 188 | %rep %0 |
| 189 | CAT_XDEFINE t, %%i, r%1 |
| 190 | %assign %%i %%i+1 |
| 191 | %rotate 1 |
| 192 | %endrep |
| 193 | %endmacro |
| 194 | |
| 195 | %macro DECLARE_REG_TMP_SIZE 0-* |
| 196 | %rep %0 |
| 197 | %define t%1q t%1 %+ q |
| 198 | %define t%1d t%1 %+ d |
| 199 | %define t%1w t%1 %+ w |
| 200 | %define t%1h t%1 %+ h |
| 201 | %define t%1b t%1 %+ b |
| 202 | %rotate 1 |
| 203 | %endrep |
| 204 | %endmacro |
| 205 | |
| 206 | DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 |
| 207 | |
| 208 | %if ARCH_X86_64 |
| 209 | %define gprsize 8 |
| 210 | %else |
| 211 | %define gprsize 4 |
| 212 | %endif |
| 213 | |
| 214 | %macro PUSH 1 |
| 215 | push %1 |
| 216 | %ifidn rstk, rsp |
| 217 | %assign stack_offset stack_offset+gprsize |
| 218 | %endif |
| 219 | %endmacro |
| 220 | |
| 221 | %macro POP 1 |
| 222 | pop %1 |
| 223 | %ifidn rstk, rsp |
| 224 | %assign stack_offset stack_offset-gprsize |
| 225 | %endif |
| 226 | %endmacro |
| 227 | |
| 228 | %macro PUSH_IF_USED 1-* |
| 229 | %rep %0 |
| 230 | %if %1 < regs_used |
| 231 | PUSH r%1 |
| 232 | %endif |
| 233 | %rotate 1 |
| 234 | %endrep |
| 235 | %endmacro |
| 236 | |
| 237 | %macro POP_IF_USED 1-* |
| 238 | %rep %0 |
| 239 | %if %1 < regs_used |
| 240 | pop r%1 |
| 241 | %endif |
| 242 | %rotate 1 |
| 243 | %endrep |
| 244 | %endmacro |
| 245 | |
| 246 | %macro LOAD_IF_USED 1-* |
| 247 | %rep %0 |
| 248 | %if %1 < num_args |
| 249 | mov r%1, r %+ %1 %+ mp |
| 250 | %endif |
| 251 | %rotate 1 |
| 252 | %endrep |
| 253 | %endmacro |
| 254 | |
| 255 | %macro SUB 2 |
| 256 | sub %1, %2 |
| 257 | %ifidn %1, rstk |
| 258 | %assign stack_offset stack_offset+(%2) |
| 259 | %endif |
| 260 | %endmacro |
| 261 | |
| 262 | %macro ADD 2 |
| 263 | add %1, %2 |
| 264 | %ifidn %1, rstk |
| 265 | %assign stack_offset stack_offset-(%2) |
| 266 | %endif |
| 267 | %endmacro |
| 268 | |
| 269 | %macro movifnidn 2 |
| 270 | %ifnidn %1, %2 |
| 271 | mov %1, %2 |
| 272 | %endif |
| 273 | %endmacro |
| 274 | |
| 275 | %macro movsxdifnidn 2 |
| 276 | %ifnidn %1, %2 |
| 277 | movsxd %1, %2 |
| 278 | %endif |
| 279 | %endmacro |
| 280 | |
| 281 | %macro ASSERT 1 |
| 282 | %if (%1) == 0 |
| 283 | %error assert failed |
| 284 | %endif |
| 285 | %endmacro |
| 286 | |
| 287 | %macro DEFINE_ARGS 0-* |
| 288 | %ifdef n_arg_names |
| 289 | %assign %%i 0 |
| 290 | %rep n_arg_names |
| 291 | CAT_UNDEF arg_name %+ %%i, q |
| 292 | CAT_UNDEF arg_name %+ %%i, d |
| 293 | CAT_UNDEF arg_name %+ %%i, w |
| 294 | CAT_UNDEF arg_name %+ %%i, h |
| 295 | CAT_UNDEF arg_name %+ %%i, b |
| 296 | CAT_UNDEF arg_name %+ %%i, m |
| 297 | CAT_UNDEF arg_name %+ %%i, mp |
| 298 | CAT_UNDEF arg_name, %%i |
| 299 | %assign %%i %%i+1 |
| 300 | %endrep |
| 301 | %endif |
| 302 | |
| 303 | %xdefine %%stack_offset stack_offset |
| 304 | %undef stack_offset ; so that the current value of stack_offset doesn't get baked in by xdefine |
| 305 | %assign %%i 0 |
| 306 | %rep %0 |
| 307 | %xdefine %1q r %+ %%i %+ q |
| 308 | %xdefine %1d r %+ %%i %+ d |
| 309 | %xdefine %1w r %+ %%i %+ w |
| 310 | %xdefine %1h r %+ %%i %+ h |
| 311 | %xdefine %1b r %+ %%i %+ b |
| 312 | %xdefine %1m r %+ %%i %+ m |
| 313 | %xdefine %1mp r %+ %%i %+ mp |
| 314 | CAT_XDEFINE arg_name, %%i, %1 |
| 315 | %assign %%i %%i+1 |
| 316 | %rotate 1 |
| 317 | %endrep |
| 318 | %xdefine stack_offset %%stack_offset |
| 319 | %assign n_arg_names %0 |
| 320 | %endmacro |
| 321 | |
| 322 | %macro ALLOC_STACK 1-2 0 ; stack_size, n_xmm_regs (for win64 only) |
| 323 | %ifnum %1 |
| 324 | %if %1 != 0 |
| 325 | %assign %%stack_alignment ((mmsize + 15) & ~15) |
| 326 | %assign stack_size %1 |
| 327 | %if stack_size < 0 |
| 328 | %assign stack_size -stack_size |
| 329 | %endif |
| 330 | %assign stack_size_padded stack_size |
| 331 | %if WIN64 |
| 332 | %assign stack_size_padded stack_size_padded + 32 ; reserve 32 bytes for shadow space |
| 333 | %if mmsize != 8 |
| 334 | %assign xmm_regs_used %2 |
| 335 | %if xmm_regs_used > 8 |
| 336 | %assign stack_size_padded stack_size_padded + (xmm_regs_used-8)*16 |
| 337 | %endif |
| 338 | %endif |
| 339 | %endif |
| 340 | %if mmsize <= 16 && HAVE_ALIGNED_STACK |
| 341 | %assign stack_size_padded stack_size_padded + %%stack_alignment - gprsize - (stack_offset & (%%stack_alignment - 1)) |
| 342 | SUB rsp, stack_size_padded |
| 343 | %else |
| 344 | %assign %%reg_num (regs_used - 1) |
| 345 | %xdefine rstk r %+ %%reg_num |
| 346 | ; align stack, and save original stack location directly above |
| 347 | ; it, i.e. in [rsp+stack_size_padded], so we can restore the |
| 348 | ; stack in a single instruction (i.e. mov rsp, rstk or mov |
| 349 | ; rsp, [rsp+stack_size_padded]) |
| 350 | mov rstk, rsp |
| 351 | %if %1 < 0 ; need to store rsp on stack |
| 352 | sub rsp, gprsize+stack_size_padded |
| 353 | and rsp, ~(%%stack_alignment-1) |
| 354 | %xdefine rstkm [rsp+stack_size_padded] |
| 355 | mov rstkm, rstk |
| 356 | %else ; can keep rsp in rstk during whole function |
| 357 | sub rsp, stack_size_padded |
| 358 | and rsp, ~(%%stack_alignment-1) |
| 359 | %xdefine rstkm rstk |
| 360 | %endif |
| 361 | %endif |
| 362 | WIN64_PUSH_XMM |
| 363 | %endif |
| 364 | %endif |
| 365 | %endmacro |
| 366 | |
| 367 | %macro SETUP_STACK_POINTER 1 |
| 368 | %ifnum %1 |
| 369 | %if %1 != 0 && (HAVE_ALIGNED_STACK == 0 || mmsize == 32) |
| 370 | %if %1 > 0 |
| 371 | %assign regs_used (regs_used + 1) |
| 372 | %elif ARCH_X86_64 && regs_used == num_args && num_args <= 4 + UNIX64 * 2 |
| 373 | %warning "Stack pointer will overwrite register argument" |
| 374 | %endif |
| 375 | %endif |
| 376 | %endif |
| 377 | %endmacro |
| 378 | |
| 379 | %macro DEFINE_ARGS_INTERNAL 3+ |
| 380 | %ifnum %2 |
| 381 | DEFINE_ARGS %3 |
| 382 | %elif %1 == 4 |
| 383 | DEFINE_ARGS %2 |
| 384 | %elif %1 > 4 |
| 385 | DEFINE_ARGS %2, %3 |
| 386 | %endif |
| 387 | %endmacro |
| 388 | |
| 389 | %if WIN64 ; Windows x64 ;================================================= |
| 390 | |
| 391 | DECLARE_REG 0, rcx |
| 392 | DECLARE_REG 1, rdx |
| 393 | DECLARE_REG 2, R8 |
| 394 | DECLARE_REG 3, R9 |
| 395 | DECLARE_REG 4, R10, 40 |
| 396 | DECLARE_REG 5, R11, 48 |
| 397 | DECLARE_REG 6, rax, 56 |
| 398 | DECLARE_REG 7, rdi, 64 |
| 399 | DECLARE_REG 8, rsi, 72 |
| 400 | DECLARE_REG 9, rbx, 80 |
| 401 | DECLARE_REG 10, rbp, 88 |
| 402 | DECLARE_REG 11, R12, 96 |
| 403 | DECLARE_REG 12, R13, 104 |
| 404 | DECLARE_REG 13, R14, 112 |
| 405 | DECLARE_REG 14, R15, 120 |
| 406 | |
| 407 | %macro PROLOGUE 2-5+ 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names... |
| 408 | %assign num_args %1 |
| 409 | %assign regs_used %2 |
| 410 | ASSERT regs_used >= num_args |
| 411 | SETUP_STACK_POINTER %4 |
| 412 | ASSERT regs_used <= 15 |
| 413 | PUSH_IF_USED 7, 8, 9, 10, 11, 12, 13, 14 |
| 414 | ALLOC_STACK %4, %3 |
| 415 | %if mmsize != 8 && stack_size == 0 |
| 416 | WIN64_SPILL_XMM %3 |
| 417 | %endif |
| 418 | LOAD_IF_USED 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 |
| 419 | DEFINE_ARGS_INTERNAL %0, %4, %5 |
| 420 | %endmacro |
| 421 | |
| 422 | %macro WIN64_PUSH_XMM 0 |
| 423 | ; Use the shadow space to store XMM6 and XMM7, the rest needs stack space allocated. |
| 424 | %if xmm_regs_used > 6 |
| 425 | movaps [rstk + stack_offset + 8], xmm6 |
| 426 | %endif |
| 427 | %if xmm_regs_used > 7 |
| 428 | movaps [rstk + stack_offset + 24], xmm7 |
| 429 | %endif |
| 430 | %if xmm_regs_used > 8 |
| 431 | %assign %%i 8 |
| 432 | %rep xmm_regs_used-8 |
| 433 | movaps [rsp + (%%i-8)*16 + stack_size + 32], xmm %+ %%i |
| 434 | %assign %%i %%i+1 |
| 435 | %endrep |
| 436 | %endif |
| 437 | %endmacro |
| 438 | |
| 439 | %macro WIN64_SPILL_XMM 1 |
| 440 | %assign xmm_regs_used %1 |
| 441 | ASSERT xmm_regs_used <= 16 |
| 442 | %if xmm_regs_used > 8 |
| 443 | %assign stack_size_padded (xmm_regs_used-8)*16 + (~stack_offset&8) + 32 |
| 444 | SUB rsp, stack_size_padded |
| 445 | %endif |
| 446 | WIN64_PUSH_XMM |
| 447 | %endmacro |
| 448 | |
| 449 | %macro WIN64_RESTORE_XMM_INTERNAL 1 |
| 450 | %assign %%pad_size 0 |
| 451 | %if xmm_regs_used > 8 |
| 452 | %assign %%i xmm_regs_used |
| 453 | %rep xmm_regs_used-8 |
| 454 | %assign %%i %%i-1 |
| 455 | movaps xmm %+ %%i, [%1 + (%%i-8)*16 + stack_size + 32] |
| 456 | %endrep |
| 457 | %endif |
| 458 | %if stack_size_padded > 0 |
| 459 | %if stack_size > 0 && (mmsize == 32 || HAVE_ALIGNED_STACK == 0) |
| 460 | mov rsp, rstkm |
| 461 | %else |
| 462 | add %1, stack_size_padded |
| 463 | %assign %%pad_size stack_size_padded |
| 464 | %endif |
| 465 | %endif |
| 466 | %if xmm_regs_used > 7 |
| 467 | movaps xmm7, [%1 + stack_offset - %%pad_size + 24] |
| 468 | %endif |
| 469 | %if xmm_regs_used > 6 |
| 470 | movaps xmm6, [%1 + stack_offset - %%pad_size + 8] |
| 471 | %endif |
| 472 | %endmacro |
| 473 | |
| 474 | %macro WIN64_RESTORE_XMM 1 |
| 475 | WIN64_RESTORE_XMM_INTERNAL %1 |
| 476 | %assign stack_offset (stack_offset-stack_size_padded) |
| 477 | %assign xmm_regs_used 0 |
| 478 | %endmacro |
| 479 | |
| 480 | %define has_epilogue regs_used > 7 || xmm_regs_used > 6 || mmsize == 32 || stack_size > 0 |
| 481 | |
| 482 | %macro RET 0 |
| 483 | WIN64_RESTORE_XMM_INTERNAL rsp |
| 484 | POP_IF_USED 14, 13, 12, 11, 10, 9, 8, 7 |
| 485 | %if mmsize == 32 |
| 486 | vzeroupper |
| 487 | %endif |
| 488 | AUTO_REP_RET |
| 489 | %endmacro |
| 490 | |
| 491 | %elif ARCH_X86_64 ; *nix x64 ;============================================= |
| 492 | |
| 493 | DECLARE_REG 0, rdi |
| 494 | DECLARE_REG 1, rsi |
| 495 | DECLARE_REG 2, rdx |
| 496 | DECLARE_REG 3, rcx |
| 497 | DECLARE_REG 4, R8 |
| 498 | DECLARE_REG 5, R9 |
| 499 | DECLARE_REG 6, rax, 8 |
| 500 | DECLARE_REG 7, R10, 16 |
| 501 | DECLARE_REG 8, R11, 24 |
| 502 | DECLARE_REG 9, rbx, 32 |
| 503 | DECLARE_REG 10, rbp, 40 |
| 504 | DECLARE_REG 11, R12, 48 |
| 505 | DECLARE_REG 12, R13, 56 |
| 506 | DECLARE_REG 13, R14, 64 |
| 507 | DECLARE_REG 14, R15, 72 |
| 508 | |
| 509 | %macro PROLOGUE 2-5+ ; #args, #regs, #xmm_regs, [stack_size,] arg_names... |
| 510 | %assign num_args %1 |
| 511 | %assign regs_used %2 |
| 512 | ASSERT regs_used >= num_args |
| 513 | SETUP_STACK_POINTER %4 |
| 514 | ASSERT regs_used <= 15 |
| 515 | PUSH_IF_USED 9, 10, 11, 12, 13, 14 |
| 516 | ALLOC_STACK %4 |
| 517 | LOAD_IF_USED 6, 7, 8, 9, 10, 11, 12, 13, 14 |
| 518 | DEFINE_ARGS_INTERNAL %0, %4, %5 |
| 519 | %endmacro |
| 520 | |
| 521 | %define has_epilogue regs_used > 9 || mmsize == 32 || stack_size > 0 |
| 522 | |
| 523 | %macro RET 0 |
| 524 | %if stack_size_padded > 0 |
| 525 | %if mmsize == 32 || HAVE_ALIGNED_STACK == 0 |
| 526 | mov rsp, rstkm |
| 527 | %else |
| 528 | add rsp, stack_size_padded |
| 529 | %endif |
| 530 | %endif |
| 531 | POP_IF_USED 14, 13, 12, 11, 10, 9 |
| 532 | %if mmsize == 32 |
| 533 | vzeroupper |
| 534 | %endif |
| 535 | AUTO_REP_RET |
| 536 | %endmacro |
| 537 | |
| 538 | %else ; X86_32 ;============================================================== |
| 539 | |
| 540 | DECLARE_REG 0, eax, 4 |
| 541 | DECLARE_REG 1, ecx, 8 |
| 542 | DECLARE_REG 2, edx, 12 |
| 543 | DECLARE_REG 3, ebx, 16 |
| 544 | DECLARE_REG 4, esi, 20 |
| 545 | DECLARE_REG 5, edi, 24 |
| 546 | DECLARE_REG 6, ebp, 28 |
| 547 | %define rsp esp |
| 548 | |
| 549 | %macro DECLARE_ARG 1-* |
| 550 | %rep %0 |
| 551 | %define r%1m [rstk + stack_offset + 4*%1 + 4] |
| 552 | %define r%1mp dword r%1m |
| 553 | %rotate 1 |
| 554 | %endrep |
| 555 | %endmacro |
| 556 | |
| 557 | DECLARE_ARG 7, 8, 9, 10, 11, 12, 13, 14 |
| 558 | |
| 559 | %macro PROLOGUE 2-5+ ; #args, #regs, #xmm_regs, [stack_size,] arg_names... |
| 560 | %assign num_args %1 |
| 561 | %assign regs_used %2 |
| 562 | ASSERT regs_used >= num_args |
| 563 | %if num_args > 7 |
| 564 | %assign num_args 7 |
| 565 | %endif |
| 566 | %if regs_used > 7 |
| 567 | %assign regs_used 7 |
| 568 | %endif |
| 569 | SETUP_STACK_POINTER %4 |
| 570 | ASSERT regs_used <= 7 |
| 571 | PUSH_IF_USED 3, 4, 5, 6 |
| 572 | ALLOC_STACK %4 |
| 573 | LOAD_IF_USED 0, 1, 2, 3, 4, 5, 6 |
| 574 | DEFINE_ARGS_INTERNAL %0, %4, %5 |
| 575 | %endmacro |
| 576 | |
| 577 | %define has_epilogue regs_used > 3 || mmsize == 32 || stack_size > 0 |
| 578 | |
| 579 | %macro RET 0 |
| 580 | %if stack_size_padded > 0 |
| 581 | %if mmsize == 32 || HAVE_ALIGNED_STACK == 0 |
| 582 | mov rsp, rstkm |
| 583 | %else |
| 584 | add rsp, stack_size_padded |
| 585 | %endif |
| 586 | %endif |
| 587 | POP_IF_USED 6, 5, 4, 3 |
| 588 | %if mmsize == 32 |
| 589 | vzeroupper |
| 590 | %endif |
| 591 | AUTO_REP_RET |
| 592 | %endmacro |
| 593 | |
| 594 | %endif ;====================================================================== |
| 595 | |
| 596 | %if WIN64 == 0 |
| 597 | %macro WIN64_SPILL_XMM 1 |
| 598 | %endmacro |
| 599 | %macro WIN64_RESTORE_XMM 1 |
| 600 | %endmacro |
| 601 | %macro WIN64_PUSH_XMM 0 |
| 602 | %endmacro |
| 603 | %endif |
| 604 | |
| 605 | ; On AMD cpus <=K10, an ordinary ret is slow if it immediately follows either |
| 606 | ; a branch or a branch target. So switch to a 2-byte form of ret in that case. |
| 607 | ; We can automatically detect "follows a branch", but not a branch target. |
| 608 | ; (SSSE3 is a sufficient condition to know that your cpu doesn't have this problem.) |
| 609 | %macro REP_RET 0 |
| 610 | %if has_epilogue |
| 611 | RET |
| 612 | %else |
| 613 | rep ret |
| 614 | %endif |
| 615 | %endmacro |
| 616 | |
| 617 | %define last_branch_adr $$ |
| 618 | %macro AUTO_REP_RET 0 |
| 619 | %ifndef cpuflags |
| 620 | times ((last_branch_adr-$)>>31)+1 rep ; times 1 iff $ != last_branch_adr. |
| 621 | %elif notcpuflag(ssse3) |
| 622 | times ((last_branch_adr-$)>>31)+1 rep |
| 623 | %endif |
| 624 | ret |
| 625 | %endmacro |
| 626 | |
| 627 | %macro BRANCH_INSTR 0-* |
| 628 | %rep %0 |
| 629 | %macro %1 1-2 %1 |
| 630 | %2 %1 |
| 631 | %%branch_instr: |
| 632 | %xdefine last_branch_adr %%branch_instr |
| 633 | %endmacro |
| 634 | %rotate 1 |
| 635 | %endrep |
| 636 | %endmacro |
| 637 | |
| 638 | BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, jna, jnae, jb, jbe, jnb, jnbe, jc, jnc, js, jns, jo, jno, jp, jnp |
| 639 | |
| 640 | %macro TAIL_CALL 2 ; callee, is_nonadjacent |
| 641 | %if has_epilogue |
| 642 | call %1 |
| 643 | RET |
| 644 | %elif %2 |
| 645 | jmp %1 |
| 646 | %endif |
| 647 | %endmacro |
| 648 | |
| 649 | ;============================================================================= |
| 650 | ; arch-independent part |
| 651 | ;============================================================================= |
| 652 | |
| 653 | %assign function_align 16 |
| 654 | |
| 655 | ; Begin a function. |
| 656 | ; Applies any symbol mangling needed for C linkage, and sets up a define such that |
| 657 | ; subsequent uses of the function name automatically refer to the mangled version. |
| 658 | ; Appends cpuflags to the function name if cpuflags has been specified. |
| 659 | ; The "" empty default parameter is a workaround for nasm, which fails if SUFFIX |
| 660 | ; is empty and we call cglobal_internal with just %1 %+ SUFFIX (without %2). |
| 661 | %macro cglobal 1-2+ "" ; name, [PROLOGUE args] |
| 662 | cglobal_internal 1, %1 %+ SUFFIX, %2 |
| 663 | %endmacro |
| 664 | %macro cvisible 1-2+ "" ; name, [PROLOGUE args] |
| 665 | cglobal_internal 0, %1 %+ SUFFIX, %2 |
| 666 | %endmacro |
| 667 | %macro cglobal_internal 2-3+ |
| 668 | %if %1 |
| 669 | %xdefine %%FUNCTION_PREFIX private_prefix |
| 670 | %xdefine %%VISIBILITY hidden |
| 671 | %else |
| 672 | %xdefine %%FUNCTION_PREFIX public_prefix |
| 673 | %xdefine %%VISIBILITY |
| 674 | %endif |
| 675 | %ifndef cglobaled_%2 |
| 676 | %xdefine %2 mangle(%%FUNCTION_PREFIX %+ _ %+ %2) |
| 677 | %xdefine %2.skip_prologue %2 %+ .skip_prologue |
| 678 | CAT_XDEFINE cglobaled_, %2, 1 |
| 679 | %endif |
| 680 | %xdefine current_function %2 |
| 681 | %ifidn __OUTPUT_FORMAT__,elf |
| 682 | global %2:function %%VISIBILITY |
| 683 | %else |
| 684 | global %2 |
| 685 | %endif |
| 686 | align function_align |
| 687 | %2: |
| 688 | RESET_MM_PERMUTATION ; needed for x86-64, also makes disassembly somewhat nicer |
| 689 | %xdefine rstk rsp ; copy of the original stack pointer, used when greater alignment than the known stack alignment is required |
| 690 | %assign stack_offset 0 ; stack pointer offset relative to the return address |
| 691 | %assign stack_size 0 ; amount of stack space that can be freely used inside a function |
| 692 | %assign stack_size_padded 0 ; total amount of allocated stack space, including space for callee-saved xmm registers on WIN64 and alignment padding |
| 693 | %assign xmm_regs_used 0 ; number of XMM registers requested, used for dealing with callee-saved registers on WIN64 |
| 694 | %ifnidn %3, "" |
| 695 | PROLOGUE %3 |
| 696 | %endif |
| 697 | %endmacro |
| 698 | |
| 699 | %macro cextern 1 |
| 700 | %xdefine %1 mangle(private_prefix %+ _ %+ %1) |
| 701 | CAT_XDEFINE cglobaled_, %1, 1 |
| 702 | extern %1 |
| 703 | %endmacro |
| 704 | |
| 705 | ; like cextern, but without the prefix |
| 706 | %macro cextern_naked 1 |
| 707 | %xdefine %1 mangle(%1) |
| 708 | CAT_XDEFINE cglobaled_, %1, 1 |
| 709 | extern %1 |
| 710 | %endmacro |
| 711 | |
| 712 | %macro const 1-2+ |
| 713 | %xdefine %1 mangle(private_prefix %+ _ %+ %1) |
| 714 | %ifidn __OUTPUT_FORMAT__,elf |
| 715 | global %1:data hidden |
| 716 | %else |
| 717 | global %1 |
| 718 | %endif |
| 719 | %1: %2 |
| 720 | %endmacro |
| 721 | |
| 722 | ; This is needed for ELF, otherwise the GNU linker assumes the stack is |
| 723 | ; executable by default. |
| 724 | %ifidn __OUTPUT_FORMAT__,elf |
| 725 | SECTION .note.GNU-stack noalloc noexec nowrite progbits |
| 726 | %endif |
| 727 | |
| 728 | ; cpuflags |
| 729 | |
| 730 | %assign cpuflags_mmx (1<<0) |
| 731 | %assign cpuflags_mmx2 (1<<1) | cpuflags_mmx |
| 732 | %assign cpuflags_3dnow (1<<2) | cpuflags_mmx |
| 733 | %assign cpuflags_3dnowext (1<<3) | cpuflags_3dnow |
| 734 | %assign cpuflags_sse (1<<4) | cpuflags_mmx2 |
| 735 | %assign cpuflags_sse2 (1<<5) | cpuflags_sse |
| 736 | %assign cpuflags_sse2slow (1<<6) | cpuflags_sse2 |
| 737 | %assign cpuflags_sse3 (1<<7) | cpuflags_sse2 |
| 738 | %assign cpuflags_ssse3 (1<<8) | cpuflags_sse3 |
| 739 | %assign cpuflags_sse4 (1<<9) | cpuflags_ssse3 |
| 740 | %assign cpuflags_sse42 (1<<10)| cpuflags_sse4 |
| 741 | %assign cpuflags_avx (1<<11)| cpuflags_sse42 |
| 742 | %assign cpuflags_xop (1<<12)| cpuflags_avx |
| 743 | %assign cpuflags_fma4 (1<<13)| cpuflags_avx |
| 744 | %assign cpuflags_avx2 (1<<14)| cpuflags_avx |
| 745 | %assign cpuflags_fma3 (1<<15)| cpuflags_avx |
| 746 | |
| 747 | %assign cpuflags_cache32 (1<<16) |
| 748 | %assign cpuflags_cache64 (1<<17) |
| 749 | %assign cpuflags_slowctz (1<<18) |
| 750 | %assign cpuflags_lzcnt (1<<19) |
| 751 | %assign cpuflags_aligned (1<<20) ; not a cpu feature, but a function variant |
| 752 | %assign cpuflags_atom (1<<21) |
| 753 | %assign cpuflags_bmi1 (1<<22)|cpuflags_lzcnt |
| 754 | %assign cpuflags_bmi2 (1<<23)|cpuflags_bmi1 |
| 755 | |
| 756 | %define cpuflag(x) ((cpuflags & (cpuflags_ %+ x)) == (cpuflags_ %+ x)) |
| 757 | %define notcpuflag(x) ((cpuflags & (cpuflags_ %+ x)) != (cpuflags_ %+ x)) |
| 758 | |
| 759 | ; Takes an arbitrary number of cpuflags from the above list. |
| 760 | ; All subsequent functions (up to the next INIT_CPUFLAGS) is built for the specified cpu. |
| 761 | ; You shouldn't need to invoke this macro directly, it's a subroutine for INIT_MMX &co. |
| 762 | %macro INIT_CPUFLAGS 0-* |
| 763 | %xdefine SUFFIX |
| 764 | %undef cpuname |
| 765 | %assign cpuflags 0 |
| 766 | |
| 767 | %if %0 >= 1 |
| 768 | %rep %0 |
| 769 | %ifdef cpuname |
| 770 | %xdefine cpuname cpuname %+ _%1 |
| 771 | %else |
| 772 | %xdefine cpuname %1 |
| 773 | %endif |
| 774 | %assign cpuflags cpuflags | cpuflags_%1 |
| 775 | %rotate 1 |
| 776 | %endrep |
| 777 | %xdefine SUFFIX _ %+ cpuname |
| 778 | |
| 779 | %if cpuflag(avx) |
| 780 | %assign avx_enabled 1 |
| 781 | %endif |
| 782 | %if (mmsize == 16 && notcpuflag(sse2)) || (mmsize == 32 && notcpuflag(avx2)) |
| 783 | %define mova movaps |
| 784 | %define movu movups |
| 785 | %define movnta movntps |
| 786 | %endif |
| 787 | %if cpuflag(aligned) |
| 788 | %define movu mova |
| 789 | %elif cpuflag(sse3) && notcpuflag(ssse3) |
| 790 | %define movu lddqu |
| 791 | %endif |
| 792 | %endif |
| 793 | |
| 794 | %if cpuflag(sse2) |
| 795 | CPUNOP amdnop |
| 796 | %else |
| 797 | CPUNOP basicnop |
| 798 | %endif |
| 799 | %endmacro |
| 800 | |
| 801 | ; Merge mmx and sse* |
| 802 | ; m# is a simd register of the currently selected size |
| 803 | ; xm# is the corresponding xmm register if mmsize >= 16, otherwise the same as m# |
| 804 | ; ym# is the corresponding ymm register if mmsize >= 32, otherwise the same as m# |
| 805 | ; (All 3 remain in sync through SWAP.) |
| 806 | |
| 807 | %macro CAT_XDEFINE 3 |
| 808 | %xdefine %1%2 %3 |
| 809 | %endmacro |
| 810 | |
| 811 | %macro CAT_UNDEF 2 |
| 812 | %undef %1%2 |
| 813 | %endmacro |
| 814 | |
| 815 | %macro INIT_MMX 0-1+ |
| 816 | %assign avx_enabled 0 |
| 817 | %define RESET_MM_PERMUTATION INIT_MMX %1 |
| 818 | %define mmsize 8 |
| 819 | %define num_mmregs 8 |
| 820 | %define mova movq |
| 821 | %define movu movq |
| 822 | %define movh movd |
| 823 | %define movnta movntq |
| 824 | %assign %%i 0 |
| 825 | %rep 8 |
| 826 | CAT_XDEFINE m, %%i, mm %+ %%i |
| 827 | CAT_XDEFINE nnmm, %%i, %%i |
| 828 | %assign %%i %%i+1 |
| 829 | %endrep |
| 830 | %rep 8 |
| 831 | CAT_UNDEF m, %%i |
| 832 | CAT_UNDEF nnmm, %%i |
| 833 | %assign %%i %%i+1 |
| 834 | %endrep |
| 835 | INIT_CPUFLAGS %1 |
| 836 | %endmacro |
| 837 | |
| 838 | %macro INIT_XMM 0-1+ |
| 839 | %assign avx_enabled 0 |
| 840 | %define RESET_MM_PERMUTATION INIT_XMM %1 |
| 841 | %define mmsize 16 |
| 842 | %define num_mmregs 8 |
| 843 | %if ARCH_X86_64 |
| 844 | %define num_mmregs 16 |
| 845 | %endif |
| 846 | %define mova movdqa |
| 847 | %define movu movdqu |
| 848 | %define movh movq |
| 849 | %define movnta movntdq |
| 850 | %assign %%i 0 |
| 851 | %rep num_mmregs |
| 852 | CAT_XDEFINE m, %%i, xmm %+ %%i |
| 853 | CAT_XDEFINE nnxmm, %%i, %%i |
| 854 | %assign %%i %%i+1 |
| 855 | %endrep |
| 856 | INIT_CPUFLAGS %1 |
| 857 | %endmacro |
| 858 | |
| 859 | ; FIXME: INIT_AVX can be replaced by INIT_XMM avx |
| 860 | %macro INIT_AVX 0 |
| 861 | INIT_XMM |
| 862 | %assign avx_enabled 1 |
| 863 | %define PALIGNR PALIGNR_SSSE3 |
| 864 | %define RESET_MM_PERMUTATION INIT_AVX |
| 865 | %endmacro |
| 866 | |
| 867 | %macro INIT_YMM 0-1+ |
| 868 | %assign avx_enabled 1 |
| 869 | %define RESET_MM_PERMUTATION INIT_YMM %1 |
| 870 | %define mmsize 32 |
| 871 | %define num_mmregs 8 |
| 872 | %if ARCH_X86_64 |
| 873 | %define num_mmregs 16 |
| 874 | %endif |
| 875 | %define mova movdqa |
| 876 | %define movu movdqu |
| 877 | %undef movh |
| 878 | %define movnta movntdq |
| 879 | %assign %%i 0 |
| 880 | %rep num_mmregs |
| 881 | CAT_XDEFINE m, %%i, ymm %+ %%i |
| 882 | CAT_XDEFINE nymm, %%i, %%i |
| 883 | %assign %%i %%i+1 |
| 884 | %endrep |
| 885 | INIT_CPUFLAGS %1 |
| 886 | %endmacro |
| 887 | |
| 888 | INIT_XMM |
| 889 | |
| 890 | %macro DECLARE_MMCAST 1 |
| 891 | %define mmmm%1 mm%1 |
| 892 | %define mmxmm%1 mm%1 |
| 893 | %define mmymm%1 mm%1 |
| 894 | %define xmmmm%1 mm%1 |
| 895 | %define xmmxmm%1 xmm%1 |
| 896 | %define xmmymm%1 xmm%1 |
| 897 | %define ymmmm%1 mm%1 |
| 898 | %define ymmxmm%1 xmm%1 |
| 899 | %define ymmymm%1 ymm%1 |
| 900 | %define xm%1 xmm %+ m%1 |
| 901 | %define ym%1 ymm %+ m%1 |
| 902 | %endmacro |
| 903 | |
| 904 | %assign i 0 |
| 905 | %rep 16 |
| 906 | DECLARE_MMCAST i |
| 907 | %assign i i+1 |
| 908 | %endrep |
| 909 | |
| 910 | ; I often want to use macros that permute their arguments. e.g. there's no |
| 911 | ; efficient way to implement butterfly or transpose or dct without swapping some |
| 912 | ; arguments. |
| 913 | ; |
| 914 | ; I would like to not have to manually keep track of the permutations: |
| 915 | ; If I insert a permutation in the middle of a function, it should automatically |
| 916 | ; change everything that follows. For more complex macros I may also have multiple |
| 917 | ; implementations, e.g. the SSE2 and SSSE3 versions may have different permutations. |
| 918 | ; |
| 919 | ; Hence these macros. Insert a PERMUTE or some SWAPs at the end of a macro that |
| 920 | ; permutes its arguments. It's equivalent to exchanging the contents of the |
| 921 | ; registers, except that this way you exchange the register names instead, so it |
| 922 | ; doesn't cost any cycles. |
| 923 | |
| 924 | %macro PERMUTE 2-* ; takes a list of pairs to swap |
| 925 | %rep %0/2 |
| 926 | %xdefine %%tmp%2 m%2 |
| 927 | %rotate 2 |
| 928 | %endrep |
| 929 | %rep %0/2 |
| 930 | %xdefine m%1 %%tmp%2 |
| 931 | CAT_XDEFINE nn, m%1, %1 |
| 932 | %rotate 2 |
| 933 | %endrep |
| 934 | %endmacro |
| 935 | |
| 936 | %macro SWAP 2+ ; swaps a single chain (sometimes more concise than pairs) |
| 937 | %ifnum %1 ; SWAP 0, 1, ... |
| 938 | SWAP_INTERNAL_NUM %1, %2 |
| 939 | %else ; SWAP m0, m1, ... |
| 940 | SWAP_INTERNAL_NAME %1, %2 |
| 941 | %endif |
| 942 | %endmacro |
| 943 | |
| 944 | %macro SWAP_INTERNAL_NUM 2-* |
| 945 | %rep %0-1 |
| 946 | %xdefine %%tmp m%1 |
| 947 | %xdefine m%1 m%2 |
| 948 | %xdefine m%2 %%tmp |
| 949 | CAT_XDEFINE nn, m%1, %1 |
| 950 | CAT_XDEFINE nn, m%2, %2 |
| 951 | %rotate 1 |
| 952 | %endrep |
| 953 | %endmacro |
| 954 | |
| 955 | %macro SWAP_INTERNAL_NAME 2-* |
| 956 | %xdefine %%args nn %+ %1 |
| 957 | %rep %0-1 |
| 958 | %xdefine %%args %%args, nn %+ %2 |
| 959 | %rotate 1 |
| 960 | %endrep |
| 961 | SWAP_INTERNAL_NUM %%args |
| 962 | %endmacro |
| 963 | |
| 964 | ; If SAVE_MM_PERMUTATION is placed at the end of a function, then any later |
| 965 | ; calls to that function will automatically load the permutation, so values can |
| 966 | ; be returned in mmregs. |
| 967 | %macro SAVE_MM_PERMUTATION 0-1 |
| 968 | %if %0 |
| 969 | %xdefine %%f %1_m |
| 970 | %else |
| 971 | %xdefine %%f current_function %+ _m |
| 972 | %endif |
| 973 | %assign %%i 0 |
| 974 | %rep num_mmregs |
| 975 | CAT_XDEFINE %%f, %%i, m %+ %%i |
| 976 | %assign %%i %%i+1 |
| 977 | %endrep |
| 978 | %endmacro |
| 979 | |
| 980 | %macro LOAD_MM_PERMUTATION 1 ; name to load from |
| 981 | %ifdef %1_m0 |
| 982 | %assign %%i 0 |
| 983 | %rep num_mmregs |
| 984 | CAT_XDEFINE m, %%i, %1_m %+ %%i |
| 985 | CAT_XDEFINE nn, m %+ %%i, %%i |
| 986 | %assign %%i %%i+1 |
| 987 | %endrep |
| 988 | %endif |
| 989 | %endmacro |
| 990 | |
| 991 | ; Append cpuflags to the callee's name iff the appended name is known and the plain name isn't |
| 992 | %macro call 1 |
| 993 | call_internal %1 %+ SUFFIX, %1 |
| 994 | %endmacro |
| 995 | %macro call_internal 2 |
| 996 | %xdefine %%i %2 |
| 997 | %ifndef cglobaled_%2 |
| 998 | %ifdef cglobaled_%1 |
| 999 | %xdefine %%i %1 |
| 1000 | %endif |
| 1001 | %endif |
| 1002 | call %%i |
| 1003 | LOAD_MM_PERMUTATION %%i |
| 1004 | %endmacro |
| 1005 | |
| 1006 | ; Substitutions that reduce instruction size but are functionally equivalent |
| 1007 | %macro add 2 |
| 1008 | %ifnum %2 |
| 1009 | %if %2==128 |
| 1010 | sub %1, -128 |
| 1011 | %else |
| 1012 | add %1, %2 |
| 1013 | %endif |
| 1014 | %else |
| 1015 | add %1, %2 |
| 1016 | %endif |
| 1017 | %endmacro |
| 1018 | |
| 1019 | %macro sub 2 |
| 1020 | %ifnum %2 |
| 1021 | %if %2==128 |
| 1022 | add %1, -128 |
| 1023 | %else |
| 1024 | sub %1, %2 |
| 1025 | %endif |
| 1026 | %else |
| 1027 | sub %1, %2 |
| 1028 | %endif |
| 1029 | %endmacro |
| 1030 | |
| 1031 | ;============================================================================= |
| 1032 | ; AVX abstraction layer |
| 1033 | ;============================================================================= |
| 1034 | |
| 1035 | %assign i 0 |
| 1036 | %rep 16 |
| 1037 | %if i < 8 |
| 1038 | CAT_XDEFINE sizeofmm, i, 8 |
| 1039 | %endif |
| 1040 | CAT_XDEFINE sizeofxmm, i, 16 |
| 1041 | CAT_XDEFINE sizeofymm, i, 32 |
| 1042 | %assign i i+1 |
| 1043 | %endrep |
| 1044 | %undef i |
| 1045 | |
| 1046 | %macro CHECK_AVX_INSTR_EMU 3-* |
| 1047 | %xdefine %%opcode %1 |
| 1048 | %xdefine %%dst %2 |
| 1049 | %rep %0-2 |
| 1050 | %ifidn %%dst, %3 |
| 1051 | %error non-avx emulation of ``%%opcode'' is not supported |
| 1052 | %endif |
| 1053 | %rotate 1 |
| 1054 | %endrep |
| 1055 | %endmacro |
| 1056 | |
| 1057 | ;%1 == instruction |
| 1058 | ;%2 == 1 if float, 0 if int |
| 1059 | ;%3 == 1 if non-destructive or 4-operand (xmm, xmm, xmm, imm), 0 otherwise |
| 1060 | ;%4 == 1 if commutative (i.e. doesn't matter which src arg is which), 0 if not |
| 1061 | ;%5+: operands |
| 1062 | %macro RUN_AVX_INSTR 5-8+ |
| 1063 | %ifnum sizeof%6 |
| 1064 | %assign __sizeofreg sizeof%6 |
| 1065 | %elifnum sizeof%5 |
| 1066 | %assign __sizeofreg sizeof%5 |
| 1067 | %else |
| 1068 | %assign __sizeofreg mmsize |
| 1069 | %endif |
| 1070 | %assign __emulate_avx 0 |
| 1071 | %if avx_enabled && __sizeofreg >= 16 |
| 1072 | %xdefine __instr v%1 |
| 1073 | %else |
| 1074 | %xdefine __instr %1 |
| 1075 | %if %0 >= 7+%3 |
| 1076 | %assign __emulate_avx 1 |
| 1077 | %endif |
| 1078 | %endif |
| 1079 | |
| 1080 | %if __emulate_avx |
| 1081 | %xdefine __src1 %6 |
| 1082 | %xdefine __src2 %7 |
| 1083 | %ifnidn %5, %6 |
| 1084 | %if %0 >= 8 |
| 1085 | CHECK_AVX_INSTR_EMU {%1 %5, %6, %7, %8}, %5, %7, %8 |
| 1086 | %else |
| 1087 | CHECK_AVX_INSTR_EMU {%1 %5, %6, %7}, %5, %7 |
| 1088 | %endif |
| 1089 | %if %4 && %3 == 0 |
| 1090 | %ifnid %7 |
| 1091 | ; 3-operand AVX instructions with a memory arg can only have it in src2, |
| 1092 | ; whereas SSE emulation prefers to have it in src1 (i.e. the mov). |
| 1093 | ; So, if the instruction is commutative with a memory arg, swap them. |
| 1094 | %xdefine __src1 %7 |
| 1095 | %xdefine __src2 %6 |
| 1096 | %endif |
| 1097 | %endif |
| 1098 | %if __sizeofreg == 8 |
| 1099 | MOVQ %5, __src1 |
| 1100 | %elif %2 |
| 1101 | MOVAPS %5, __src1 |
| 1102 | %else |
| 1103 | MOVDQA %5, __src1 |
| 1104 | %endif |
| 1105 | %endif |
| 1106 | %if %0 >= 8 |
| 1107 | %1 %5, __src2, %8 |
| 1108 | %else |
| 1109 | %1 %5, __src2 |
| 1110 | %endif |
| 1111 | %elif %0 >= 8 |
| 1112 | __instr %5, %6, %7, %8 |
| 1113 | %elif %0 == 7 |
| 1114 | __instr %5, %6, %7 |
| 1115 | %elif %0 == 6 |
| 1116 | __instr %5, %6 |
| 1117 | %else |
| 1118 | __instr %5 |
| 1119 | %endif |
| 1120 | %endmacro |
| 1121 | |
| 1122 | ;%1 == instruction |
| 1123 | ;%2 == 1 if float, 0 if int |
| 1124 | ;%3 == 1 if non-destructive or 4-operand (xmm, xmm, xmm, imm), 0 otherwise |
| 1125 | ;%4 == 1 if commutative (i.e. doesn't matter which src arg is which), 0 if not |
| 1126 | %macro AVX_INSTR 1-4 0, 1, 0 |
| 1127 | %macro %1 1-9 fnord, fnord, fnord, fnord, %1, %2, %3, %4 |
| 1128 | %ifidn %2, fnord |
| 1129 | RUN_AVX_INSTR %6, %7, %8, %9, %1 |
| 1130 | %elifidn %3, fnord |
| 1131 | RUN_AVX_INSTR %6, %7, %8, %9, %1, %2 |
| 1132 | %elifidn %4, fnord |
| 1133 | RUN_AVX_INSTR %6, %7, %8, %9, %1, %2, %3 |
| 1134 | %elifidn %5, fnord |
| 1135 | RUN_AVX_INSTR %6, %7, %8, %9, %1, %2, %3, %4 |
| 1136 | %else |
| 1137 | RUN_AVX_INSTR %6, %7, %8, %9, %1, %2, %3, %4, %5 |
| 1138 | %endif |
| 1139 | %endmacro |
| 1140 | %endmacro |
| 1141 | |
| 1142 | ; Instructions with both VEX and non-VEX encodings |
| 1143 | ; Non-destructive instructions are written without parameters |
| 1144 | AVX_INSTR addpd, 1, 0, 1 |
| 1145 | AVX_INSTR addps, 1, 0, 1 |
| 1146 | AVX_INSTR addsd, 1, 0, 1 |
| 1147 | AVX_INSTR addss, 1, 0, 1 |
| 1148 | AVX_INSTR addsubpd, 1, 0, 0 |
| 1149 | AVX_INSTR addsubps, 1, 0, 0 |
| 1150 | AVX_INSTR aesdec, 0, 0, 0 |
| 1151 | AVX_INSTR aesdeclast, 0, 0, 0 |
| 1152 | AVX_INSTR aesenc, 0, 0, 0 |
| 1153 | AVX_INSTR aesenclast, 0, 0, 0 |
| 1154 | AVX_INSTR aesimc |
| 1155 | AVX_INSTR aeskeygenassist |
| 1156 | AVX_INSTR andnpd, 1, 0, 0 |
| 1157 | AVX_INSTR andnps, 1, 0, 0 |
| 1158 | AVX_INSTR andpd, 1, 0, 1 |
| 1159 | AVX_INSTR andps, 1, 0, 1 |
| 1160 | AVX_INSTR blendpd, 1, 0, 0 |
| 1161 | AVX_INSTR blendps, 1, 0, 0 |
| 1162 | AVX_INSTR blendvpd, 1, 0, 0 |
| 1163 | AVX_INSTR blendvps, 1, 0, 0 |
| 1164 | AVX_INSTR cmppd, 1, 1, 0 |
| 1165 | AVX_INSTR cmpps, 1, 1, 0 |
| 1166 | AVX_INSTR cmpsd, 1, 1, 0 |
| 1167 | AVX_INSTR cmpss, 1, 1, 0 |
| 1168 | AVX_INSTR comisd |
| 1169 | AVX_INSTR comiss |
| 1170 | AVX_INSTR cvtdq2pd |
| 1171 | AVX_INSTR cvtdq2ps |
| 1172 | AVX_INSTR cvtpd2dq |
| 1173 | AVX_INSTR cvtpd2ps |
| 1174 | AVX_INSTR cvtps2dq |
| 1175 | AVX_INSTR cvtps2pd |
| 1176 | AVX_INSTR cvtsd2si |
| 1177 | AVX_INSTR cvtsd2ss |
| 1178 | AVX_INSTR cvtsi2sd |
| 1179 | AVX_INSTR cvtsi2ss |
| 1180 | AVX_INSTR cvtss2sd |
| 1181 | AVX_INSTR cvtss2si |
| 1182 | AVX_INSTR cvttpd2dq |
| 1183 | AVX_INSTR cvttps2dq |
| 1184 | AVX_INSTR cvttsd2si |
| 1185 | AVX_INSTR cvttss2si |
| 1186 | AVX_INSTR divpd, 1, 0, 0 |
| 1187 | AVX_INSTR divps, 1, 0, 0 |
| 1188 | AVX_INSTR divsd, 1, 0, 0 |
| 1189 | AVX_INSTR divss, 1, 0, 0 |
| 1190 | AVX_INSTR dppd, 1, 1, 0 |
| 1191 | AVX_INSTR dpps, 1, 1, 0 |
| 1192 | AVX_INSTR extractps |
| 1193 | AVX_INSTR haddpd, 1, 0, 0 |
| 1194 | AVX_INSTR haddps, 1, 0, 0 |
| 1195 | AVX_INSTR hsubpd, 1, 0, 0 |
| 1196 | AVX_INSTR hsubps, 1, 0, 0 |
| 1197 | AVX_INSTR insertps, 1, 1, 0 |
| 1198 | AVX_INSTR lddqu |
| 1199 | AVX_INSTR ldmxcsr |
| 1200 | AVX_INSTR maskmovdqu |
| 1201 | AVX_INSTR maxpd, 1, 0, 1 |
| 1202 | AVX_INSTR maxps, 1, 0, 1 |
| 1203 | AVX_INSTR maxsd, 1, 0, 1 |
| 1204 | AVX_INSTR maxss, 1, 0, 1 |
| 1205 | AVX_INSTR minpd, 1, 0, 1 |
| 1206 | AVX_INSTR minps, 1, 0, 1 |
| 1207 | AVX_INSTR minsd, 1, 0, 1 |
| 1208 | AVX_INSTR minss, 1, 0, 1 |
| 1209 | AVX_INSTR movapd |
| 1210 | AVX_INSTR movaps |
| 1211 | AVX_INSTR movd |
| 1212 | AVX_INSTR movddup |
| 1213 | AVX_INSTR movdqa |
| 1214 | AVX_INSTR movdqu |
| 1215 | AVX_INSTR movhlps, 1, 0, 0 |
| 1216 | AVX_INSTR movhpd, 1, 0, 0 |
| 1217 | AVX_INSTR movhps, 1, 0, 0 |
| 1218 | AVX_INSTR movlhps, 1, 0, 0 |
| 1219 | AVX_INSTR movlpd, 1, 0, 0 |
| 1220 | AVX_INSTR movlps, 1, 0, 0 |
| 1221 | AVX_INSTR movmskpd |
| 1222 | AVX_INSTR movmskps |
| 1223 | AVX_INSTR movntdq |
| 1224 | AVX_INSTR movntdqa |
| 1225 | AVX_INSTR movntpd |
| 1226 | AVX_INSTR movntps |
| 1227 | AVX_INSTR movq |
| 1228 | AVX_INSTR movsd, 1, 0, 0 |
| 1229 | AVX_INSTR movshdup |
| 1230 | AVX_INSTR movsldup |
| 1231 | AVX_INSTR movss, 1, 0, 0 |
| 1232 | AVX_INSTR movupd |
| 1233 | AVX_INSTR movups |
| 1234 | AVX_INSTR mpsadbw, 0, 1, 0 |
| 1235 | AVX_INSTR mulpd, 1, 0, 1 |
| 1236 | AVX_INSTR mulps, 1, 0, 1 |
| 1237 | AVX_INSTR mulsd, 1, 0, 1 |
| 1238 | AVX_INSTR mulss, 1, 0, 1 |
| 1239 | AVX_INSTR orpd, 1, 0, 1 |
| 1240 | AVX_INSTR orps, 1, 0, 1 |
| 1241 | AVX_INSTR pabsb |
| 1242 | AVX_INSTR pabsd |
| 1243 | AVX_INSTR pabsw |
| 1244 | AVX_INSTR packsswb, 0, 0, 0 |
| 1245 | AVX_INSTR packssdw, 0, 0, 0 |
| 1246 | AVX_INSTR packuswb, 0, 0, 0 |
| 1247 | AVX_INSTR packusdw, 0, 0, 0 |
| 1248 | AVX_INSTR paddb, 0, 0, 1 |
| 1249 | AVX_INSTR paddw, 0, 0, 1 |
| 1250 | AVX_INSTR paddd, 0, 0, 1 |
| 1251 | AVX_INSTR paddq, 0, 0, 1 |
| 1252 | AVX_INSTR paddsb, 0, 0, 1 |
| 1253 | AVX_INSTR paddsw, 0, 0, 1 |
| 1254 | AVX_INSTR paddusb, 0, 0, 1 |
| 1255 | AVX_INSTR paddusw, 0, 0, 1 |
| 1256 | AVX_INSTR palignr, 0, 1, 0 |
| 1257 | AVX_INSTR pand, 0, 0, 1 |
| 1258 | AVX_INSTR pandn, 0, 0, 0 |
| 1259 | AVX_INSTR pavgb, 0, 0, 1 |
| 1260 | AVX_INSTR pavgw, 0, 0, 1 |
| 1261 | AVX_INSTR pblendvb, 0, 0, 0 |
| 1262 | AVX_INSTR pblendw, 0, 1, 0 |
| 1263 | AVX_INSTR pclmulqdq, 0, 1, 0 |
| 1264 | AVX_INSTR pcmpestri |
| 1265 | AVX_INSTR pcmpestrm |
| 1266 | AVX_INSTR pcmpistri |
| 1267 | AVX_INSTR pcmpistrm |
| 1268 | AVX_INSTR pcmpeqb, 0, 0, 1 |
| 1269 | AVX_INSTR pcmpeqw, 0, 0, 1 |
| 1270 | AVX_INSTR pcmpeqd, 0, 0, 1 |
| 1271 | AVX_INSTR pcmpeqq, 0, 0, 1 |
| 1272 | AVX_INSTR pcmpgtb, 0, 0, 0 |
| 1273 | AVX_INSTR pcmpgtw, 0, 0, 0 |
| 1274 | AVX_INSTR pcmpgtd, 0, 0, 0 |
| 1275 | AVX_INSTR pcmpgtq, 0, 0, 0 |
| 1276 | AVX_INSTR pextrb |
| 1277 | AVX_INSTR pextrd |
| 1278 | AVX_INSTR pextrq |
| 1279 | AVX_INSTR pextrw |
| 1280 | AVX_INSTR phaddw, 0, 0, 0 |
| 1281 | AVX_INSTR phaddd, 0, 0, 0 |
| 1282 | AVX_INSTR phaddsw, 0, 0, 0 |
| 1283 | AVX_INSTR phminposuw |
| 1284 | AVX_INSTR phsubw, 0, 0, 0 |
| 1285 | AVX_INSTR phsubd, 0, 0, 0 |
| 1286 | AVX_INSTR phsubsw, 0, 0, 0 |
| 1287 | AVX_INSTR pinsrb, 0, 1, 0 |
| 1288 | AVX_INSTR pinsrd, 0, 1, 0 |
| 1289 | AVX_INSTR pinsrq, 0, 1, 0 |
| 1290 | AVX_INSTR pinsrw, 0, 1, 0 |
| 1291 | AVX_INSTR pmaddwd, 0, 0, 1 |
| 1292 | AVX_INSTR pmaddubsw, 0, 0, 0 |
| 1293 | AVX_INSTR pmaxsb, 0, 0, 1 |
| 1294 | AVX_INSTR pmaxsw, 0, 0, 1 |
| 1295 | AVX_INSTR pmaxsd, 0, 0, 1 |
| 1296 | AVX_INSTR pmaxub, 0, 0, 1 |
| 1297 | AVX_INSTR pmaxuw, 0, 0, 1 |
| 1298 | AVX_INSTR pmaxud, 0, 0, 1 |
| 1299 | AVX_INSTR pminsb, 0, 0, 1 |
| 1300 | AVX_INSTR pminsw, 0, 0, 1 |
| 1301 | AVX_INSTR pminsd, 0, 0, 1 |
| 1302 | AVX_INSTR pminub, 0, 0, 1 |
| 1303 | AVX_INSTR pminuw, 0, 0, 1 |
| 1304 | AVX_INSTR pminud, 0, 0, 1 |
| 1305 | AVX_INSTR pmovmskb |
| 1306 | AVX_INSTR pmovsxbw |
| 1307 | AVX_INSTR pmovsxbd |
| 1308 | AVX_INSTR pmovsxbq |
| 1309 | AVX_INSTR pmovsxwd |
| 1310 | AVX_INSTR pmovsxwq |
| 1311 | AVX_INSTR pmovsxdq |
| 1312 | AVX_INSTR pmovzxbw |
| 1313 | AVX_INSTR pmovzxbd |
| 1314 | AVX_INSTR pmovzxbq |
| 1315 | AVX_INSTR pmovzxwd |
| 1316 | AVX_INSTR pmovzxwq |
| 1317 | AVX_INSTR pmovzxdq |
| 1318 | AVX_INSTR pmuldq, 0, 0, 1 |
| 1319 | AVX_INSTR pmulhrsw, 0, 0, 1 |
| 1320 | AVX_INSTR pmulhuw, 0, 0, 1 |
| 1321 | AVX_INSTR pmulhw, 0, 0, 1 |
| 1322 | AVX_INSTR pmullw, 0, 0, 1 |
| 1323 | AVX_INSTR pmulld, 0, 0, 1 |
| 1324 | AVX_INSTR pmuludq, 0, 0, 1 |
| 1325 | AVX_INSTR por, 0, 0, 1 |
| 1326 | AVX_INSTR psadbw, 0, 0, 1 |
| 1327 | AVX_INSTR pshufb, 0, 0, 0 |
| 1328 | AVX_INSTR pshufd |
| 1329 | AVX_INSTR pshufhw |
| 1330 | AVX_INSTR pshuflw |
| 1331 | AVX_INSTR psignb, 0, 0, 0 |
| 1332 | AVX_INSTR psignw, 0, 0, 0 |
| 1333 | AVX_INSTR psignd, 0, 0, 0 |
| 1334 | AVX_INSTR psllw, 0, 0, 0 |
| 1335 | AVX_INSTR pslld, 0, 0, 0 |
| 1336 | AVX_INSTR psllq, 0, 0, 0 |
| 1337 | AVX_INSTR pslldq, 0, 0, 0 |
| 1338 | AVX_INSTR psraw, 0, 0, 0 |
| 1339 | AVX_INSTR psrad, 0, 0, 0 |
| 1340 | AVX_INSTR psrlw, 0, 0, 0 |
| 1341 | AVX_INSTR psrld, 0, 0, 0 |
| 1342 | AVX_INSTR psrlq, 0, 0, 0 |
| 1343 | AVX_INSTR psrldq, 0, 0, 0 |
| 1344 | AVX_INSTR psubb, 0, 0, 0 |
| 1345 | AVX_INSTR psubw, 0, 0, 0 |
| 1346 | AVX_INSTR psubd, 0, 0, 0 |
| 1347 | AVX_INSTR psubq, 0, 0, 0 |
| 1348 | AVX_INSTR psubsb, 0, 0, 0 |
| 1349 | AVX_INSTR psubsw, 0, 0, 0 |
| 1350 | AVX_INSTR psubusb, 0, 0, 0 |
| 1351 | AVX_INSTR psubusw, 0, 0, 0 |
| 1352 | AVX_INSTR ptest |
| 1353 | AVX_INSTR punpckhbw, 0, 0, 0 |
| 1354 | AVX_INSTR punpckhwd, 0, 0, 0 |
| 1355 | AVX_INSTR punpckhdq, 0, 0, 0 |
| 1356 | AVX_INSTR punpckhqdq, 0, 0, 0 |
| 1357 | AVX_INSTR punpcklbw, 0, 0, 0 |
| 1358 | AVX_INSTR punpcklwd, 0, 0, 0 |
| 1359 | AVX_INSTR punpckldq, 0, 0, 0 |
| 1360 | AVX_INSTR punpcklqdq, 0, 0, 0 |
| 1361 | AVX_INSTR pxor, 0, 0, 1 |
| 1362 | AVX_INSTR rcpps, 1, 0, 0 |
| 1363 | AVX_INSTR rcpss, 1, 0, 0 |
| 1364 | AVX_INSTR roundpd |
| 1365 | AVX_INSTR roundps |
| 1366 | AVX_INSTR roundsd |
| 1367 | AVX_INSTR roundss |
| 1368 | AVX_INSTR rsqrtps, 1, 0, 0 |
| 1369 | AVX_INSTR rsqrtss, 1, 0, 0 |
| 1370 | AVX_INSTR shufpd, 1, 1, 0 |
| 1371 | AVX_INSTR shufps, 1, 1, 0 |
| 1372 | AVX_INSTR sqrtpd, 1, 0, 0 |
| 1373 | AVX_INSTR sqrtps, 1, 0, 0 |
| 1374 | AVX_INSTR sqrtsd, 1, 0, 0 |
| 1375 | AVX_INSTR sqrtss, 1, 0, 0 |
| 1376 | AVX_INSTR stmxcsr |
| 1377 | AVX_INSTR subpd, 1, 0, 0 |
| 1378 | AVX_INSTR subps, 1, 0, 0 |
| 1379 | AVX_INSTR subsd, 1, 0, 0 |
| 1380 | AVX_INSTR subss, 1, 0, 0 |
| 1381 | AVX_INSTR ucomisd |
| 1382 | AVX_INSTR ucomiss |
| 1383 | AVX_INSTR unpckhpd, 1, 0, 0 |
| 1384 | AVX_INSTR unpckhps, 1, 0, 0 |
| 1385 | AVX_INSTR unpcklpd, 1, 0, 0 |
| 1386 | AVX_INSTR unpcklps, 1, 0, 0 |
| 1387 | AVX_INSTR xorpd, 1, 0, 1 |
| 1388 | AVX_INSTR xorps, 1, 0, 1 |
| 1389 | |
| 1390 | ; 3DNow instructions, for sharing code between AVX, SSE and 3DN |
| 1391 | AVX_INSTR pfadd, 1, 0, 1 |
| 1392 | AVX_INSTR pfsub, 1, 0, 0 |
| 1393 | AVX_INSTR pfmul, 1, 0, 1 |
| 1394 | |
| 1395 | ; base-4 constants for shuffles |
| 1396 | %assign i 0 |
| 1397 | %rep 256 |
| 1398 | %assign j ((i>>6)&3)*1000 + ((i>>4)&3)*100 + ((i>>2)&3)*10 + (i&3) |
| 1399 | %if j < 10 |
| 1400 | CAT_XDEFINE q000, j, i |
| 1401 | %elif j < 100 |
| 1402 | CAT_XDEFINE q00, j, i |
| 1403 | %elif j < 1000 |
| 1404 | CAT_XDEFINE q0, j, i |
| 1405 | %else |
| 1406 | CAT_XDEFINE q, j, i |
| 1407 | %endif |
| 1408 | %assign i i+1 |
| 1409 | %endrep |
| 1410 | %undef i |
| 1411 | %undef j |
| 1412 | |
| 1413 | ; tzcnt is equivalent to "rep bsf" and is backwards-compatible with bsf. |
| 1414 | ; This lets us use tzcnt without bumping the yasm version requirement yet. |
| 1415 | %define tzcnt rep bsf |
| 1416 | |
| 1417 | ; convert FMA4 to FMA3 if possible |
| 1418 | %macro FMA4_INSTR 4 |
| 1419 | %macro %1 4-8 %1, %2, %3, %4 |
| 1420 | %if cpuflag(fma4) |
| 1421 | v%5 %1, %2, %3, %4 |
| 1422 | %elifidn %1, %2 |
| 1423 | v%6 %1, %4, %3 ; %1 = %1 * %3 + %4 |
| 1424 | %elifidn %1, %3 |
| 1425 | v%7 %1, %2, %4 ; %1 = %2 * %1 + %4 |
| 1426 | %elifidn %1, %4 |
| 1427 | v%8 %1, %2, %3 ; %1 = %2 * %3 + %1 |
| 1428 | %else |
| 1429 | %error fma3 emulation of ``%5 %1, %2, %3, %4'' is not supported |
| 1430 | %endif |
| 1431 | %endmacro |
| 1432 | %endmacro |
| 1433 | |
| 1434 | FMA4_INSTR fmaddpd, fmadd132pd, fmadd213pd, fmadd231pd |
| 1435 | FMA4_INSTR fmaddps, fmadd132ps, fmadd213ps, fmadd231ps |
| 1436 | FMA4_INSTR fmaddsd, fmadd132sd, fmadd213sd, fmadd231sd |
| 1437 | FMA4_INSTR fmaddss, fmadd132ss, fmadd213ss, fmadd231ss |
| 1438 | |
| 1439 | FMA4_INSTR fmaddsubpd, fmaddsub132pd, fmaddsub213pd, fmaddsub231pd |
| 1440 | FMA4_INSTR fmaddsubps, fmaddsub132ps, fmaddsub213ps, fmaddsub231ps |
| 1441 | FMA4_INSTR fmsubaddpd, fmsubadd132pd, fmsubadd213pd, fmsubadd231pd |
| 1442 | FMA4_INSTR fmsubaddps, fmsubadd132ps, fmsubadd213ps, fmsubadd231ps |
| 1443 | |
| 1444 | FMA4_INSTR fmsubpd, fmsub132pd, fmsub213pd, fmsub231pd |
| 1445 | FMA4_INSTR fmsubps, fmsub132ps, fmsub213ps, fmsub231ps |
| 1446 | FMA4_INSTR fmsubsd, fmsub132sd, fmsub213sd, fmsub231sd |
| 1447 | FMA4_INSTR fmsubss, fmsub132ss, fmsub213ss, fmsub231ss |
| 1448 | |
| 1449 | FMA4_INSTR fnmaddpd, fnmadd132pd, fnmadd213pd, fnmadd231pd |
| 1450 | FMA4_INSTR fnmaddps, fnmadd132ps, fnmadd213ps, fnmadd231ps |
| 1451 | FMA4_INSTR fnmaddsd, fnmadd132sd, fnmadd213sd, fnmadd231sd |
| 1452 | FMA4_INSTR fnmaddss, fnmadd132ss, fnmadd213ss, fnmadd231ss |
| 1453 | |
| 1454 | FMA4_INSTR fnmsubpd, fnmsub132pd, fnmsub213pd, fnmsub231pd |
| 1455 | FMA4_INSTR fnmsubps, fnmsub132ps, fnmsub213ps, fnmsub231ps |
| 1456 | FMA4_INSTR fnmsubsd, fnmsub132sd, fnmsub213sd, fnmsub231sd |
| 1457 | FMA4_INSTR fnmsubss, fnmsub132ss, fnmsub213ss, fnmsub231ss |
| 1458 | |
| 1459 | ; workaround: vpbroadcastq is broken in x86_32 due to a yasm bug |
| 1460 | %if ARCH_X86_64 == 0 |
| 1461 | %macro vpbroadcastq 2 |
| 1462 | %if sizeof%1 == 16 |
| 1463 | movddup %1, %2 |
| 1464 | %else |
| 1465 | vbroadcastsd %1, %2 |
| 1466 | %endif |
| 1467 | %endmacro |
| 1468 | %endif |