Debian Codegraph @

.cargo/vendor/av-scenechange/src/asm/x86/x86inc.asm

raw
1 ;*****************************************************************************
2 ;* x86inc.asm: x86 abstraction layer
3 ;*****************************************************************************
4 ;* Copyright (C) 2005-2022 x264 project
5 ;*
6 ;* Authors: Loren Merritt <[email protected]>
7 ;* Henrik Gramner <[email protected]>
8 ;* Anton Mitrofanov <[email protected]>
9 ;* Fiona Glaser <[email protected]>
10 ;*
11 ;* Permission to use, copy, modify, and/or distribute this software for any
12 ;* purpose with or without fee is hereby granted, provided that the above
13 ;* copyright notice and this permission notice appear in all copies.
14 ;*
15 ;* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
16 ;* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
17 ;* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
18 ;* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
19 ;* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
20 ;* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
21 ;* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
22 ;*****************************************************************************
23
24 ; This is a header file for the x86inc.asm assembly language, which uses
25 ; NASM/YASM syntax combined with a large number of macros to provide easy
26 ; abstraction between different calling conventions (x86_32, win64, linux64).
27 ; It also has various other useful features to simplify writing the kind of
28 ; DSP functions that are most often used.
29
30 %ifndef private_prefix
31 %error private_prefix not defined
32 %endif
33
34 %ifndef public_prefix
35 %define public_prefix private_prefix
36 %endif
37
38 %ifndef STACK_ALIGNMENT
39 %if ARCH_X86_64
40 %define STACK_ALIGNMENT 16
41 %else
42 %define STACK_ALIGNMENT 4
43 %endif
44 %endif
45
46 %define WIN64 0
47 %define UNIX64 0
48 %if ARCH_X86_64
49 %ifidn __OUTPUT_FORMAT__,win32
50 %define WIN64 1
51 %elifidn __OUTPUT_FORMAT__,win64
52 %define WIN64 1
53 %elifidn __OUTPUT_FORMAT__,x64
54 %define WIN64 1
55 %else
56 %define UNIX64 1
57 %endif
58 %endif
59
60 %define FORMAT_ELF 0
61 %define FORMAT_MACHO 0
62 %ifidn __OUTPUT_FORMAT__,elf
63 %define FORMAT_ELF 1
64 %elifidn __OUTPUT_FORMAT__,elf32
65 %define FORMAT_ELF 1
66 %elifidn __OUTPUT_FORMAT__,elf64
67 %define FORMAT_ELF 1
68 %elifidn __OUTPUT_FORMAT__,macho
69 %define FORMAT_MACHO 1
70 %elifidn __OUTPUT_FORMAT__,macho32
71 %define FORMAT_MACHO 1
72 %elifidn __OUTPUT_FORMAT__,macho64
73 %define FORMAT_MACHO 1
74 %endif
75
76 %ifdef PREFIX
77 %define mangle(x) _ %+ x
78 %else
79 %define mangle(x) x
80 %endif
81
82 ; Use VEX-encoding even in non-AVX functions
83 %ifndef FORCE_VEX_ENCODING
84 %define FORCE_VEX_ENCODING 0
85 %endif
86
87 %macro SECTION_RODATA 0-1 16
88 %ifidn __OUTPUT_FORMAT__,win32
89 SECTION .rdata align=%1
90 %elif WIN64
91 SECTION .rdata align=%1
92 %else
93 SECTION .rodata align=%1
94 %endif
95 %endmacro
96
97 %if ARCH_X86_64
98 %define PIC 1 ; always use PIC on x86-64
99 default rel
100 %elifidn __OUTPUT_FORMAT__,win32
101 %define PIC 0 ; PIC isn't used on 32-bit Windows
102 %elifndef PIC
103 %define PIC 0
104 %endif
105
106 %define HAVE_PRIVATE_EXTERN 1
107 %ifdef __NASM_VER__
108 %use smartalign
109 %if __NASM_VERSION_ID__ < 0x020e0000 ; 2.14
110 %define HAVE_PRIVATE_EXTERN 0
111 %endif
112 %endif
113
114 ; Macros to eliminate most code duplication between x86_32 and x86_64:
115 ; Currently this works only for leaf functions which load all their arguments
116 ; into registers at the start, and make no other use of the stack. Luckily that
117 ; covers most use cases.
118
119 ; PROLOGUE:
120 ; %1 = number of arguments. loads them from stack if needed.
121 ; %2 = number of registers used. pushes callee-saved regs if needed.
122 ; %3 = number of xmm registers used. pushes callee-saved xmm regs if needed.
123 ; %4 = (optional) stack size to be allocated. The stack will be aligned before
124 ; allocating the specified stack size. If the required stack alignment is
125 ; larger than the known stack alignment the stack will be manually aligned
126 ; and an extra register will be allocated to hold the original stack
127 ; pointer (to not invalidate r0m etc.). To prevent the use of an extra
128 ; register as stack pointer, request a negative stack size.
129 ; %4+/%5+ = list of names to define to registers
130 ; PROLOGUE can also be invoked by adding the same options to cglobal
131
132 ; e.g.
133 ; cglobal foo, 2,3,7,0x40, dst, src, tmp
134 ; declares a function (foo) that automatically loads two arguments (dst and
135 ; src) into registers, uses one additional register (tmp) plus 7 vector
136 ; registers (m0-m6) and allocates 0x40 bytes of stack space.
137
138 ; TODO Some functions can use some args directly from the stack. If they're the
139 ; last args then you can just not declare them, but if they're in the middle
140 ; we need more flexible macro.
141
142 ; RET:
143 ; Pops anything that was pushed by PROLOGUE, and returns.
144
145 ; REP_RET:
146 ; Use this instead of RET if it's a branch target.
147
148 ; registers:
149 ; rN and rNq are the native-size register holding function argument N
150 ; rNd, rNw, rNb are dword, word, and byte size
151 ; rNh is the high 8 bits of the word size
152 ; rNm is the original location of arg N (a register or on the stack), dword
153 ; rNmp is native size
154
155 %macro DECLARE_REG 2-3
156 %define r%1q %2
157 %define r%1d %2d
158 %define r%1w %2w
159 %define r%1b %2b
160 %define r%1h %2h
161 %define %2q %2
162 %if %0 == 2
163 %define r%1m %2d
164 %define r%1mp %2
165 %elif ARCH_X86_64 ; memory
166 %define r%1m [rstk + stack_offset + %3]
167 %define r%1mp qword r %+ %1 %+ m
168 %else
169 %define r%1m [rstk + stack_offset + %3]
170 %define r%1mp dword r %+ %1 %+ m
171 %endif
172 %define r%1 %2
173 %endmacro
174
175 %macro DECLARE_REG_SIZE 3
176 %define r%1q r%1
177 %define e%1q r%1
178 %define r%1d e%1
179 %define e%1d e%1
180 %define r%1w %1
181 %define e%1w %1
182 %define r%1h %3
183 %define e%1h %3
184 %define r%1b %2
185 %define e%1b %2
186 %if ARCH_X86_64 == 0
187 %define r%1 e%1
188 %endif
189 %endmacro
190
191 DECLARE_REG_SIZE ax, al, ah
192 DECLARE_REG_SIZE bx, bl, bh
193 DECLARE_REG_SIZE cx, cl, ch
194 DECLARE_REG_SIZE dx, dl, dh
195 DECLARE_REG_SIZE si, sil, null
196 DECLARE_REG_SIZE di, dil, null
197 DECLARE_REG_SIZE bp, bpl, null
198
199 ; t# defines for when per-arch register allocation is more complex than just function arguments
200
201 %macro DECLARE_REG_TMP 1-*
202 %assign %%i 0
203 %rep %0
204 CAT_XDEFINE t, %%i, r%1
205 %assign %%i %%i+1
206 %rotate 1
207 %endrep
208 %endmacro
209
210 %macro DECLARE_REG_TMP_SIZE 0-*
211 %rep %0
212 %define t%1q t%1 %+ q
213 %define t%1d t%1 %+ d
214 %define t%1w t%1 %+ w
215 %define t%1h t%1 %+ h
216 %define t%1b t%1 %+ b
217 %rotate 1
218 %endrep
219 %endmacro
220
221 DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14
222
223 %if ARCH_X86_64
224 %define gprsize 8
225 %else
226 %define gprsize 4
227 %endif
228
229 %macro LEA 2
230 %if ARCH_X86_64
231 lea %1, [%2]
232 %elif PIC
233 call $+5 ; special-cased to not affect the RSB on most CPU:s
234 pop %1
235 add %1, (%2)-$+1
236 %else
237 mov %1, %2
238 %endif
239 %endmacro
240
241 ; Repeats an instruction/operation for multiple arguments.
242 ; Example usage: "REPX {psrlw x, 8}, m0, m1, m2, m3"
243 %macro REPX 2-* ; operation, args
244 %xdefine %%f(x) %1
245 %rep %0 - 1
246 %rotate 1
247 %%f(%1)
248 %endrep
249 %endmacro
250
251 %macro PUSH 1
252 push %1
253 %ifidn rstk, rsp
254 %assign stack_offset stack_offset+gprsize
255 %endif
256 %endmacro
257
258 %macro POP 1
259 pop %1
260 %ifidn rstk, rsp
261 %assign stack_offset stack_offset-gprsize
262 %endif
263 %endmacro
264
265 %macro PUSH_IF_USED 1-*
266 %rep %0
267 %if %1 < regs_used
268 PUSH r%1
269 %endif
270 %rotate 1
271 %endrep
272 %endmacro
273
274 %macro POP_IF_USED 1-*
275 %rep %0
276 %if %1 < regs_used
277 pop r%1
278 %endif
279 %rotate 1
280 %endrep
281 %endmacro
282
283 %macro LOAD_IF_USED 1-*
284 %rep %0
285 %if %1 < num_args
286 mov r%1, r %+ %1 %+ mp
287 %endif
288 %rotate 1
289 %endrep
290 %endmacro
291
292 %macro SUB 2
293 sub %1, %2
294 %ifidn %1, rstk
295 %assign stack_offset stack_offset+(%2)
296 %endif
297 %endmacro
298
299 %macro ADD 2
300 add %1, %2
301 %ifidn %1, rstk
302 %assign stack_offset stack_offset-(%2)
303 %endif
304 %endmacro
305
306 %macro movifnidn 2
307 %ifnidn %1, %2
308 mov %1, %2
309 %endif
310 %endmacro
311
312 %if ARCH_X86_64 == 0
313 %define movsxd movifnidn
314 %endif
315
316 %macro movsxdifnidn 2
317 %ifnidn %1, %2
318 movsxd %1, %2
319 %endif
320 %endmacro
321
322 %macro ASSERT 1
323 %if (%1) == 0
324 %error assertion ``%1'' failed
325 %endif
326 %endmacro
327
328 %macro DEFINE_ARGS 0-*
329 %ifdef n_arg_names
330 %assign %%i 0
331 %rep n_arg_names
332 CAT_UNDEF arg_name %+ %%i, q
333 CAT_UNDEF arg_name %+ %%i, d
334 CAT_UNDEF arg_name %+ %%i, w
335 CAT_UNDEF arg_name %+ %%i, h
336 CAT_UNDEF arg_name %+ %%i, b
337 CAT_UNDEF arg_name %+ %%i, m
338 CAT_UNDEF arg_name %+ %%i, mp
339 CAT_UNDEF arg_name, %%i
340 %assign %%i %%i+1
341 %endrep
342 %endif
343
344 %xdefine %%stack_offset stack_offset
345 %undef stack_offset ; so that the current value of stack_offset doesn't get baked in by xdefine
346 %assign %%i 0
347 %rep %0
348 %xdefine %1q r %+ %%i %+ q
349 %xdefine %1d r %+ %%i %+ d
350 %xdefine %1w r %+ %%i %+ w
351 %xdefine %1h r %+ %%i %+ h
352 %xdefine %1b r %+ %%i %+ b
353 %xdefine %1m r %+ %%i %+ m
354 %xdefine %1mp r %+ %%i %+ mp
355 CAT_XDEFINE arg_name, %%i, %1
356 %assign %%i %%i+1
357 %rotate 1
358 %endrep
359 %xdefine stack_offset %%stack_offset
360 %assign n_arg_names %0
361 %endmacro
362
363 %define required_stack_alignment ((mmsize + 15) & ~15)
364 %define vzeroupper_required (mmsize > 16 && (ARCH_X86_64 == 0 || xmm_regs_used > 16 || notcpuflag(avx512)))
365 %define high_mm_regs (16*cpuflag(avx512))
366
367 ; Large stack allocations on Windows need to use stack probing in order
368 ; to guarantee that all stack memory is committed before accessing it.
369 ; This is done by ensuring that the guard page(s) at the end of the
370 ; currently committed pages are touched prior to any pages beyond that.
371 %if WIN64
372 %assign STACK_PROBE_SIZE 8192
373 %elifidn __OUTPUT_FORMAT__, win32
374 %assign STACK_PROBE_SIZE 4096
375 %else
376 %assign STACK_PROBE_SIZE 0
377 %endif
378
379 %macro PROBE_STACK 1 ; stack_size
380 %if STACK_PROBE_SIZE
381 %assign %%i STACK_PROBE_SIZE
382 %rep %1 / STACK_PROBE_SIZE
383 mov eax, [rsp-%%i]
384 %assign %%i %%i+STACK_PROBE_SIZE
385 %endrep
386 %endif
387 %endmacro
388
389 %macro ALLOC_STACK 0-2 0, 0 ; stack_size, n_xmm_regs (for win64 only)
390 %ifnum %1
391 %if %1 != 0
392 %assign %%pad 0
393 %assign stack_size %1
394 %if stack_size < 0
395 %assign stack_size -stack_size
396 %endif
397 %if WIN64
398 %assign %%pad %%pad + 32 ; shadow space
399 %if mmsize != 8
400 %assign xmm_regs_used %2
401 %if xmm_regs_used > 8
402 %assign %%pad %%pad + (xmm_regs_used-8)*16 ; callee-saved xmm registers
403 %endif
404 %endif
405 %endif
406 %if required_stack_alignment <= STACK_ALIGNMENT
407 ; maintain the current stack alignment
408 %assign stack_size_padded stack_size + %%pad + ((-%%pad-stack_offset-gprsize) & (STACK_ALIGNMENT-1))
409 PROBE_STACK stack_size_padded
410 SUB rsp, stack_size_padded
411 %else
412 %assign %%reg_num (regs_used - 1)
413 %xdefine rstk r %+ %%reg_num
414 ; align stack, and save original stack location directly above
415 ; it, i.e. in [rsp+stack_size_padded], so we can restore the
416 ; stack in a single instruction (i.e. mov rsp, rstk or mov
417 ; rsp, [rsp+stack_size_padded])
418 %if %1 < 0 ; need to store rsp on stack
419 %xdefine rstkm [rsp + stack_size + %%pad]
420 %assign %%pad %%pad + gprsize
421 %else ; can keep rsp in rstk during whole function
422 %xdefine rstkm rstk
423 %endif
424 %assign stack_size_padded stack_size + ((%%pad + required_stack_alignment-1) & ~(required_stack_alignment-1))
425 PROBE_STACK stack_size_padded
426 mov rstk, rsp
427 and rsp, ~(required_stack_alignment-1)
428 sub rsp, stack_size_padded
429 movifnidn rstkm, rstk
430 %endif
431 WIN64_PUSH_XMM
432 %endif
433 %endif
434 %endmacro
435
436 %macro SETUP_STACK_POINTER 0-1 0
437 %ifnum %1
438 %if %1 != 0 && required_stack_alignment > STACK_ALIGNMENT
439 %if %1 > 0
440 ; Reserve an additional register for storing the original stack pointer, but avoid using
441 ; eax/rax for this purpose since it can potentially get overwritten as a return value.
442 %assign regs_used (regs_used + 1)
443 %if ARCH_X86_64 && regs_used == 7
444 %assign regs_used 8
445 %elif ARCH_X86_64 == 0 && regs_used == 1
446 %assign regs_used 2
447 %endif
448 %endif
449 %if ARCH_X86_64 && regs_used < 5 + UNIX64 * 3
450 ; Ensure that we don't clobber any registers containing arguments. For UNIX64 we also preserve r6 (rax)
451 ; since it's used as a hidden argument in vararg functions to specify the number of vector registers used.
452 %assign regs_used 5 + UNIX64 * 3
453 %endif
454 %endif
455 %endif
456 %endmacro
457
458 %if WIN64 ; Windows x64 ;=================================================
459
460 DECLARE_REG 0, rcx
461 DECLARE_REG 1, rdx
462 DECLARE_REG 2, R8
463 DECLARE_REG 3, R9
464 DECLARE_REG 4, R10, 40
465 DECLARE_REG 5, R11, 48
466 DECLARE_REG 6, rax, 56
467 DECLARE_REG 7, rdi, 64
468 DECLARE_REG 8, rsi, 72
469 DECLARE_REG 9, rbx, 80
470 DECLARE_REG 10, rbp, 88
471 DECLARE_REG 11, R14, 96
472 DECLARE_REG 12, R15, 104
473 DECLARE_REG 13, R12, 112
474 DECLARE_REG 14, R13, 120
475
476 %macro PROLOGUE 2-5+ 0, 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names...
477 %assign num_args %1
478 %assign regs_used %2
479 ASSERT regs_used >= num_args
480 SETUP_STACK_POINTER %4
481 ASSERT regs_used <= 15
482 PUSH_IF_USED 7, 8, 9, 10, 11, 12, 13, 14
483 ALLOC_STACK %4, %3
484 %if mmsize != 8 && stack_size == 0
485 WIN64_SPILL_XMM %3
486 %endif
487 LOAD_IF_USED 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14
488 %if %0 > 4
489 %ifnum %4
490 DEFINE_ARGS %5
491 %else
492 DEFINE_ARGS %4, %5
493 %endif
494 %elifnnum %4
495 DEFINE_ARGS %4
496 %endif
497 %endmacro
498
499 %macro WIN64_PUSH_XMM 0
500 ; Use the shadow space to store XMM6 and XMM7, the rest needs stack space allocated.
501 %if xmm_regs_used > 6 + high_mm_regs
502 movaps [rstk + stack_offset + 8], xmm6
503 %endif
504 %if xmm_regs_used > 7 + high_mm_regs
505 movaps [rstk + stack_offset + 24], xmm7
506 %endif
507 %assign %%xmm_regs_on_stack xmm_regs_used - high_mm_regs - 8
508 %if %%xmm_regs_on_stack > 0
509 %assign %%i 8
510 %rep %%xmm_regs_on_stack
511 movaps [rsp + (%%i-8)*16 + stack_size + 32], xmm %+ %%i
512 %assign %%i %%i+1
513 %endrep
514 %endif
515 %endmacro
516
517 %macro WIN64_SPILL_XMM 1
518 %assign xmm_regs_used %1
519 ASSERT xmm_regs_used <= 16 + high_mm_regs
520 %assign %%xmm_regs_on_stack xmm_regs_used - high_mm_regs - 8
521 %if %%xmm_regs_on_stack > 0
522 ; Allocate stack space for callee-saved xmm registers plus shadow space and align the stack.
523 %assign %%pad %%xmm_regs_on_stack*16 + 32
524 %assign stack_size_padded %%pad + ((-%%pad-stack_offset-gprsize) & (STACK_ALIGNMENT-1))
525 SUB rsp, stack_size_padded
526 %endif
527 WIN64_PUSH_XMM
528 %endmacro
529
530 %macro WIN64_RESTORE_XMM_INTERNAL 0
531 %assign %%pad_size 0
532 %assign %%xmm_regs_on_stack xmm_regs_used - high_mm_regs - 8
533 %if %%xmm_regs_on_stack > 0
534 %assign %%i xmm_regs_used - high_mm_regs
535 %rep %%xmm_regs_on_stack
536 %assign %%i %%i-1
537 movaps xmm %+ %%i, [rsp + (%%i-8)*16 + stack_size + 32]
538 %endrep
539 %endif
540 %if stack_size_padded > 0
541 %if stack_size > 0 && required_stack_alignment > STACK_ALIGNMENT
542 mov rsp, rstkm
543 %else
544 add rsp, stack_size_padded
545 %assign %%pad_size stack_size_padded
546 %endif
547 %endif
548 %if xmm_regs_used > 7 + high_mm_regs
549 movaps xmm7, [rsp + stack_offset - %%pad_size + 24]
550 %endif
551 %if xmm_regs_used > 6 + high_mm_regs
552 movaps xmm6, [rsp + stack_offset - %%pad_size + 8]
553 %endif
554 %endmacro
555
556 %macro WIN64_RESTORE_XMM 0
557 WIN64_RESTORE_XMM_INTERNAL
558 %assign stack_offset (stack_offset-stack_size_padded)
559 %assign stack_size_padded 0
560 %assign xmm_regs_used 0
561 %endmacro
562
563 %define has_epilogue regs_used > 7 || stack_size > 0 || vzeroupper_required || xmm_regs_used > 6+high_mm_regs
564
565 %macro RET 0
566 WIN64_RESTORE_XMM_INTERNAL
567 POP_IF_USED 14, 13, 12, 11, 10, 9, 8, 7
568 %if vzeroupper_required
569 vzeroupper
570 %endif
571 AUTO_REP_RET
572 %endmacro
573
574 %elif ARCH_X86_64 ; *nix x64 ;=============================================
575
576 DECLARE_REG 0, rdi
577 DECLARE_REG 1, rsi
578 DECLARE_REG 2, rdx
579 DECLARE_REG 3, rcx
580 DECLARE_REG 4, R8
581 DECLARE_REG 5, R9
582 DECLARE_REG 6, rax, 8
583 DECLARE_REG 7, R10, 16
584 DECLARE_REG 8, R11, 24
585 DECLARE_REG 9, rbx, 32
586 DECLARE_REG 10, rbp, 40
587 DECLARE_REG 11, R14, 48
588 DECLARE_REG 12, R15, 56
589 DECLARE_REG 13, R12, 64
590 DECLARE_REG 14, R13, 72
591
592 %macro PROLOGUE 2-5+ 0, 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names...
593 %assign num_args %1
594 %assign regs_used %2
595 %assign xmm_regs_used %3
596 ASSERT regs_used >= num_args
597 SETUP_STACK_POINTER %4
598 ASSERT regs_used <= 15
599 PUSH_IF_USED 9, 10, 11, 12, 13, 14
600 ALLOC_STACK %4
601 LOAD_IF_USED 6, 7, 8, 9, 10, 11, 12, 13, 14
602 %if %0 > 4
603 %ifnum %4
604 DEFINE_ARGS %5
605 %else
606 DEFINE_ARGS %4, %5
607 %endif
608 %elifnnum %4
609 DEFINE_ARGS %4
610 %endif
611 %endmacro
612
613 %define has_epilogue regs_used > 9 || stack_size > 0 || vzeroupper_required
614
615 %macro RET 0
616 %if stack_size_padded > 0
617 %if required_stack_alignment > STACK_ALIGNMENT
618 mov rsp, rstkm
619 %else
620 add rsp, stack_size_padded
621 %endif
622 %endif
623 POP_IF_USED 14, 13, 12, 11, 10, 9
624 %if vzeroupper_required
625 vzeroupper
626 %endif
627 AUTO_REP_RET
628 %endmacro
629
630 %else ; X86_32 ;==============================================================
631
632 DECLARE_REG 0, eax, 4
633 DECLARE_REG 1, ecx, 8
634 DECLARE_REG 2, edx, 12
635 DECLARE_REG 3, ebx, 16
636 DECLARE_REG 4, esi, 20
637 DECLARE_REG 5, edi, 24
638 DECLARE_REG 6, ebp, 28
639 %define rsp esp
640
641 %macro DECLARE_ARG 1-*
642 %rep %0
643 %define r%1m [rstk + stack_offset + 4*%1 + 4]
644 %define r%1mp dword r%1m
645 %rotate 1
646 %endrep
647 %endmacro
648
649 DECLARE_ARG 7, 8, 9, 10, 11, 12, 13, 14
650
651 %macro PROLOGUE 2-5+ 0, 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names...
652 %assign num_args %1
653 %assign regs_used %2
654 ASSERT regs_used >= num_args
655 %if num_args > 7
656 %assign num_args 7
657 %endif
658 %if regs_used > 7
659 %assign regs_used 7
660 %endif
661 SETUP_STACK_POINTER %4
662 ASSERT regs_used <= 7
663 PUSH_IF_USED 3, 4, 5, 6
664 ALLOC_STACK %4
665 LOAD_IF_USED 0, 1, 2, 3, 4, 5, 6
666 %if %0 > 4
667 %ifnum %4
668 DEFINE_ARGS %5
669 %else
670 DEFINE_ARGS %4, %5
671 %endif
672 %elifnnum %4
673 DEFINE_ARGS %4
674 %endif
675 %endmacro
676
677 %define has_epilogue regs_used > 3 || stack_size > 0 || vzeroupper_required
678
679 %macro RET 0
680 %if stack_size_padded > 0
681 %if required_stack_alignment > STACK_ALIGNMENT
682 mov rsp, rstkm
683 %else
684 add rsp, stack_size_padded
685 %endif
686 %endif
687 POP_IF_USED 6, 5, 4, 3
688 %if vzeroupper_required
689 vzeroupper
690 %endif
691 AUTO_REP_RET
692 %endmacro
693
694 %endif ;======================================================================
695
696 %if WIN64 == 0
697 %macro WIN64_SPILL_XMM 1
698 %assign xmm_regs_used %1
699 %endmacro
700 %macro WIN64_RESTORE_XMM 0
701 %assign xmm_regs_used 0
702 %endmacro
703 %macro WIN64_PUSH_XMM 0
704 %endmacro
705 %endif
706
707 ; On AMD cpus <=K10, an ordinary ret is slow if it immediately follows either
708 ; a branch or a branch target. So switch to a 2-byte form of ret in that case.
709 ; We can automatically detect "follows a branch", but not a branch target.
710 ; (SSSE3 is a sufficient condition to know that your cpu doesn't have this problem.)
711 %macro REP_RET 0
712 %if has_epilogue || cpuflag(ssse3)
713 RET
714 %else
715 rep ret
716 %endif
717 annotate_function_size
718 %endmacro
719
720 %define last_branch_adr $$
721 %macro AUTO_REP_RET 0
722 %if notcpuflag(ssse3)
723 times ((last_branch_adr-$)>>31)+1 rep ; times 1 iff $ == last_branch_adr.
724 %endif
725 ret
726 annotate_function_size
727 %endmacro
728
729 %macro BRANCH_INSTR 0-*
730 %rep %0
731 %macro %1 1-2 %1
732 %2 %1
733 %if notcpuflag(ssse3)
734 %%branch_instr equ $
735 %xdefine last_branch_adr %%branch_instr
736 %endif
737 %endmacro
738 %rotate 1
739 %endrep
740 %endmacro
741
742 BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, jna, jnae, jb, jbe, jnb, jnbe, jc, jnc, js, jns, jo, jno, jp, jnp
743
744 %macro TAIL_CALL 1-2 1 ; callee, is_nonadjacent
745 %if has_epilogue
746 call %1
747 RET
748 %elif %2
749 jmp %1
750 %endif
751 annotate_function_size
752 %endmacro
753
754 ;=============================================================================
755 ; arch-independent part
756 ;=============================================================================
757
758 %assign function_align 16
759
760 ; Begin a function.
761 ; Applies any symbol mangling needed for C linkage, and sets up a define such that
762 ; subsequent uses of the function name automatically refer to the mangled version.
763 ; Appends cpuflags to the function name if cpuflags has been specified.
764 ; The "" empty default parameter is a workaround for nasm, which fails if SUFFIX
765 ; is empty and we call cglobal_internal with just %1 %+ SUFFIX (without %2).
766 %macro cglobal 1-2+ "" ; name, [PROLOGUE args]
767 cglobal_internal 1, %1 %+ SUFFIX, %2
768 %endmacro
769 %macro cvisible 1-2+ "" ; name, [PROLOGUE args]
770 cglobal_internal 0, %1 %+ SUFFIX, %2
771 %endmacro
772 %macro cglobal_internal 2-3+
773 annotate_function_size
774 %ifndef cglobaled_%2
775 %if %1
776 %xdefine %2 mangle(private_prefix %+ _ %+ %2)
777 %else
778 %xdefine %2 mangle(public_prefix %+ _ %+ %2)
779 %endif
780 %xdefine %2.skip_prologue %2 %+ .skip_prologue
781 CAT_XDEFINE cglobaled_, %2, 1
782 %endif
783 %xdefine current_function %2
784 %xdefine current_function_section __SECT__
785 %if FORMAT_ELF
786 %if %1
787 global %2:function hidden
788 %else
789 global %2:function
790 %endif
791 %elif FORMAT_MACHO && HAVE_PRIVATE_EXTERN && %1
792 global %2:private_extern
793 %else
794 global %2
795 %endif
796 align function_align
797 %2:
798 RESET_MM_PERMUTATION ; needed for x86-64, also makes disassembly somewhat nicer
799 %xdefine rstk rsp ; copy of the original stack pointer, used when greater alignment than the known stack alignment is required
800 %assign stack_offset 0 ; stack pointer offset relative to the return address
801 %assign stack_size 0 ; amount of stack space that can be freely used inside a function
802 %assign stack_size_padded 0 ; total amount of allocated stack space, including space for callee-saved xmm registers on WIN64 and alignment padding
803 %assign xmm_regs_used 0 ; number of XMM registers requested, used for dealing with callee-saved registers on WIN64 and vzeroupper
804 %ifnidn %3, ""
805 PROLOGUE %3
806 %endif
807 %endmacro
808
809 ; Create a global symbol from a local label with the correct name mangling and type
810 %macro cglobal_label 1
811 %if FORMAT_ELF
812 global current_function %+ %1:function hidden
813 %elif FORMAT_MACHO && HAVE_PRIVATE_EXTERN
814 global current_function %+ %1:private_extern
815 %else
816 global current_function %+ %1
817 %endif
818 %1:
819 %endmacro
820
821 %macro cextern 1
822 %xdefine %1 mangle(private_prefix %+ _ %+ %1)
823 CAT_XDEFINE cglobaled_, %1, 1
824 extern %1
825 %endmacro
826
827 ; like cextern, but without the prefix
828 %macro cextern_naked 1
829 %ifdef PREFIX
830 %xdefine %1 mangle(%1)
831 %endif
832 CAT_XDEFINE cglobaled_, %1, 1
833 extern %1
834 %endmacro
835
836 %macro const 1-2+
837 %xdefine %1 mangle(private_prefix %+ _ %+ %1)
838 %if FORMAT_ELF
839 global %1:data hidden
840 %elif FORMAT_MACHO && HAVE_PRIVATE_EXTERN
841 global %1:private_extern
842 %else
843 global %1
844 %endif
845 %1: %2
846 %endmacro
847
848 ; This is needed for ELF, otherwise the GNU linker assumes the stack is executable by default.
849 %if FORMAT_ELF
850 [SECTION .note.GNU-stack noalloc noexec nowrite progbits]
851 %endif
852
853 ; Tell debuggers how large the function was.
854 ; This may be invoked multiple times per function; we rely on later instances overriding earlier ones.
855 ; This is invoked by RET and similar macros, and also cglobal does it for the previous function,
856 ; but if the last function in a source file doesn't use any of the standard macros for its epilogue,
857 ; then its size might be unspecified.
858 %macro annotate_function_size 0
859 %ifdef __YASM_VER__
860 %ifdef current_function
861 %if FORMAT_ELF
862 current_function_section
863 %%ecf equ $
864 size current_function %%ecf - current_function
865 __SECT__
866 %endif
867 %endif
868 %endif
869 %endmacro
870
871 ; cpuflags
872
873 %assign cpuflags_mmx (1<<0)
874 %assign cpuflags_mmx2 (1<<1) | cpuflags_mmx
875 %assign cpuflags_3dnow (1<<2) | cpuflags_mmx
876 %assign cpuflags_3dnowext (1<<3) | cpuflags_3dnow
877 %assign cpuflags_sse (1<<4) | cpuflags_mmx2
878 %assign cpuflags_sse2 (1<<5) | cpuflags_sse
879 %assign cpuflags_sse2slow (1<<6) | cpuflags_sse2
880 %assign cpuflags_lzcnt (1<<7) | cpuflags_sse2
881 %assign cpuflags_sse3 (1<<8) | cpuflags_sse2
882 %assign cpuflags_ssse3 (1<<9) | cpuflags_sse3
883 %assign cpuflags_sse4 (1<<10) | cpuflags_ssse3
884 %assign cpuflags_sse42 (1<<11) | cpuflags_sse4
885 %assign cpuflags_aesni (1<<12) | cpuflags_sse42
886 %assign cpuflags_gfni (1<<13) | cpuflags_sse42
887 %assign cpuflags_avx (1<<14) | cpuflags_sse42
888 %assign cpuflags_xop (1<<15) | cpuflags_avx
889 %assign cpuflags_fma4 (1<<16) | cpuflags_avx
890 %assign cpuflags_fma3 (1<<17) | cpuflags_avx
891 %assign cpuflags_bmi1 (1<<18) | cpuflags_avx|cpuflags_lzcnt
892 %assign cpuflags_bmi2 (1<<19) | cpuflags_bmi1
893 %assign cpuflags_avx2 (1<<20) | cpuflags_fma3|cpuflags_bmi2
894 %assign cpuflags_avx512 (1<<21) | cpuflags_avx2 ; F, CD, BW, DQ, VL
895 %assign cpuflags_avx512icl (1<<22) | cpuflags_avx512|cpuflags_gfni ; VNNI, IFMA, VBMI, VBMI2, VPOPCNTDQ, BITALG, VAES, VPCLMULQDQ
896
897 %assign cpuflags_cache32 (1<<23)
898 %assign cpuflags_cache64 (1<<24)
899 %assign cpuflags_aligned (1<<25) ; not a cpu feature, but a function variant
900 %assign cpuflags_atom (1<<26)
901
902 ; Returns a boolean value expressing whether or not the specified cpuflag is enabled.
903 %define cpuflag(x) (((((cpuflags & (cpuflags_ %+ x)) ^ (cpuflags_ %+ x)) - 1) >> 31) & 1)
904 %define notcpuflag(x) (cpuflag(x) ^ 1)
905
906 ; Takes an arbitrary number of cpuflags from the above list.
907 ; All subsequent functions (up to the next INIT_CPUFLAGS) is built for the specified cpu.
908 ; You shouldn't need to invoke this macro directly, it's a subroutine for INIT_MMX &co.
909 %macro INIT_CPUFLAGS 0-*
910 %xdefine SUFFIX
911 %undef cpuname
912 %assign cpuflags 0
913
914 %if %0 >= 1
915 %rep %0
916 %ifdef cpuname
917 %xdefine cpuname cpuname %+ _%1
918 %else
919 %xdefine cpuname %1
920 %endif
921 %assign cpuflags cpuflags | cpuflags_%1
922 %rotate 1
923 %endrep
924 %xdefine SUFFIX _ %+ cpuname
925
926 %if cpuflag(avx)
927 %assign avx_enabled 1
928 %endif
929 %if (mmsize == 16 && notcpuflag(sse2)) || (mmsize == 32 && notcpuflag(avx2))
930 %define mova movaps
931 %define movu movups
932 %define movnta movntps
933 %endif
934 %if cpuflag(aligned)
935 %define movu mova
936 %elif cpuflag(sse3) && notcpuflag(ssse3)
937 %define movu lddqu
938 %endif
939 %endif
940
941 %if ARCH_X86_64 || cpuflag(sse2)
942 %ifdef __NASM_VER__
943 ALIGNMODE p6
944 %else
945 CPU amdnop
946 %endif
947 %else
948 %ifdef __NASM_VER__
949 ALIGNMODE nop
950 %else
951 CPU basicnop
952 %endif
953 %endif
954 %endmacro
955
956 ; Merge mmx, sse*, and avx*
957 ; m# is a simd register of the currently selected size
958 ; xm# is the corresponding xmm register if mmsize >= 16, otherwise the same as m#
959 ; ym# is the corresponding ymm register if mmsize >= 32, otherwise the same as m#
960 ; zm# is the corresponding zmm register if mmsize >= 64, otherwise the same as m#
961 ; (All 4 remain in sync through SWAP.)
962
963 %macro CAT_XDEFINE 3
964 %xdefine %1%2 %3
965 %endmacro
966
967 %macro CAT_UNDEF 2
968 %undef %1%2
969 %endmacro
970
971 %macro DEFINE_MMREGS 1 ; mmtype
972 %assign %%prev_mmregs 0
973 %ifdef num_mmregs
974 %assign %%prev_mmregs num_mmregs
975 %endif
976
977 %assign num_mmregs 8
978 %if ARCH_X86_64 && mmsize >= 16
979 %assign num_mmregs 16
980 %if cpuflag(avx512) || mmsize == 64
981 %assign num_mmregs 32
982 %endif
983 %endif
984
985 %assign %%i 0
986 %rep num_mmregs
987 CAT_XDEFINE m, %%i, %1 %+ %%i
988 CAT_XDEFINE nn%1, %%i, %%i
989 %assign %%i %%i+1
990 %endrep
991 %if %%prev_mmregs > num_mmregs
992 %rep %%prev_mmregs - num_mmregs
993 CAT_UNDEF m, %%i
994 CAT_UNDEF nn %+ mmtype, %%i
995 %assign %%i %%i+1
996 %endrep
997 %endif
998 %xdefine mmtype %1
999 %endmacro
1000
1001 ; Prefer registers 16-31 over 0-15 to avoid having to use vzeroupper
1002 %macro AVX512_MM_PERMUTATION 0-1 0 ; start_reg
1003 %if ARCH_X86_64 && cpuflag(avx512)
1004 %assign %%i %1
1005 %rep 16-%1
1006 %assign %%i_high %%i+16
1007 SWAP %%i, %%i_high
1008 %assign %%i %%i+1
1009 %endrep
1010 %endif
1011 %endmacro
1012
1013 %macro INIT_MMX 0-1+
1014 %assign avx_enabled 0
1015 %define RESET_MM_PERMUTATION INIT_MMX %1
1016 %define mmsize 8
1017 %define mova movq
1018 %define movu movq
1019 %define movh movd
1020 %define movnta movntq
1021 INIT_CPUFLAGS %1
1022 DEFINE_MMREGS mm
1023 %endmacro
1024
1025 %macro INIT_XMM 0-1+
1026 %assign avx_enabled FORCE_VEX_ENCODING
1027 %define RESET_MM_PERMUTATION INIT_XMM %1
1028 %define mmsize 16
1029 %define mova movdqa
1030 %define movu movdqu
1031 %define movh movq
1032 %define movnta movntdq
1033 INIT_CPUFLAGS %1
1034 DEFINE_MMREGS xmm
1035 %if WIN64
1036 AVX512_MM_PERMUTATION 6 ; Swap callee-saved registers with volatile registers
1037 %endif
1038 %xdefine bcstd 1to4
1039 %xdefine bcstq 1to2
1040 %endmacro
1041
1042 %macro INIT_YMM 0-1+
1043 %assign avx_enabled 1
1044 %define RESET_MM_PERMUTATION INIT_YMM %1
1045 %define mmsize 32
1046 %define mova movdqa
1047 %define movu movdqu
1048 %undef movh
1049 %define movnta movntdq
1050 INIT_CPUFLAGS %1
1051 DEFINE_MMREGS ymm
1052 AVX512_MM_PERMUTATION
1053 %xdefine bcstd 1to8
1054 %xdefine bcstq 1to4
1055 %endmacro
1056
1057 %macro INIT_ZMM 0-1+
1058 %assign avx_enabled 1
1059 %define RESET_MM_PERMUTATION INIT_ZMM %1
1060 %define mmsize 64
1061 %define mova movdqa
1062 %define movu movdqu
1063 %undef movh
1064 %define movnta movntdq
1065 INIT_CPUFLAGS %1
1066 DEFINE_MMREGS zmm
1067 AVX512_MM_PERMUTATION
1068 %xdefine bcstd 1to16
1069 %xdefine bcstq 1to8
1070 %endmacro
1071
1072 INIT_XMM
1073
1074 %macro DECLARE_MMCAST 1
1075 %define mmmm%1 mm%1
1076 %define mmxmm%1 mm%1
1077 %define mmymm%1 mm%1
1078 %define mmzmm%1 mm%1
1079 %define xmmmm%1 mm%1
1080 %define xmmxmm%1 xmm%1
1081 %define xmmymm%1 xmm%1
1082 %define xmmzmm%1 xmm%1
1083 %define ymmmm%1 mm%1
1084 %define ymmxmm%1 xmm%1
1085 %define ymmymm%1 ymm%1
1086 %define ymmzmm%1 ymm%1
1087 %define zmmmm%1 mm%1
1088 %define zmmxmm%1 xmm%1
1089 %define zmmymm%1 ymm%1
1090 %define zmmzmm%1 zmm%1
1091 %define xm%1 xmm %+ m%1
1092 %define ym%1 ymm %+ m%1
1093 %define zm%1 zmm %+ m%1
1094 %endmacro
1095
1096 %assign i 0
1097 %rep 32
1098 DECLARE_MMCAST i
1099 %assign i i+1
1100 %endrep
1101
1102 ; I often want to use macros that permute their arguments. e.g. there's no
1103 ; efficient way to implement butterfly or transpose or dct without swapping some
1104 ; arguments.
1105 ;
1106 ; I would like to not have to manually keep track of the permutations:
1107 ; If I insert a permutation in the middle of a function, it should automatically
1108 ; change everything that follows. For more complex macros I may also have multiple
1109 ; implementations, e.g. the SSE2 and SSSE3 versions may have different permutations.
1110 ;
1111 ; Hence these macros. Insert a PERMUTE or some SWAPs at the end of a macro that
1112 ; permutes its arguments. It's equivalent to exchanging the contents of the
1113 ; registers, except that this way you exchange the register names instead, so it
1114 ; doesn't cost any cycles.
1115
1116 %macro PERMUTE 2-* ; takes a list of pairs to swap
1117 %rep %0/2
1118 %xdefine %%tmp%2 m%2
1119 %rotate 2
1120 %endrep
1121 %rep %0/2
1122 %xdefine m%1 %%tmp%2
1123 CAT_XDEFINE nn, m%1, %1
1124 %rotate 2
1125 %endrep
1126 %endmacro
1127
1128 %macro SWAP 2+ ; swaps a single chain (sometimes more concise than pairs)
1129 %ifnum %1 ; SWAP 0, 1, ...
1130 SWAP_INTERNAL_NUM %1, %2
1131 %else ; SWAP m0, m1, ...
1132 SWAP_INTERNAL_NAME %1, %2
1133 %endif
1134 %endmacro
1135
1136 %macro SWAP_INTERNAL_NUM 2-*
1137 %rep %0-1
1138 %xdefine %%tmp m%1
1139 %xdefine m%1 m%2
1140 %xdefine m%2 %%tmp
1141 CAT_XDEFINE nn, m%1, %1
1142 CAT_XDEFINE nn, m%2, %2
1143 %rotate 1
1144 %endrep
1145 %endmacro
1146
1147 %macro SWAP_INTERNAL_NAME 2-*
1148 %xdefine %%args nn %+ %1
1149 %rep %0-1
1150 %xdefine %%args %%args, nn %+ %2
1151 %rotate 1
1152 %endrep
1153 SWAP_INTERNAL_NUM %%args
1154 %endmacro
1155
1156 ; If SAVE_MM_PERMUTATION is placed at the end of a function, then any later
1157 ; calls to that function will automatically load the permutation, so values can
1158 ; be returned in mmregs.
1159 %macro SAVE_MM_PERMUTATION 0-1
1160 %if %0
1161 %xdefine %%f %1_m
1162 %else
1163 %xdefine %%f current_function %+ _m
1164 %endif
1165 %assign %%i 0
1166 %rep num_mmregs
1167 %xdefine %%tmp m %+ %%i
1168 CAT_XDEFINE %%f, %%i, regnumof %+ %%tmp
1169 %assign %%i %%i+1
1170 %endrep
1171 %endmacro
1172
1173 %macro LOAD_MM_PERMUTATION 0-1 ; name to load from
1174 %if %0
1175 %xdefine %%f %1_m
1176 %else
1177 %xdefine %%f current_function %+ _m
1178 %endif
1179 %xdefine %%tmp %%f %+ 0
1180 %ifnum %%tmp
1181 DEFINE_MMREGS mmtype
1182 %assign %%i 0
1183 %rep num_mmregs
1184 %xdefine %%tmp %%f %+ %%i
1185 CAT_XDEFINE %%m, %%i, m %+ %%tmp
1186 %assign %%i %%i+1
1187 %endrep
1188 %rep num_mmregs
1189 %assign %%i %%i-1
1190 CAT_XDEFINE m, %%i, %%m %+ %%i
1191 CAT_XDEFINE nn, m %+ %%i, %%i
1192 %endrep
1193 %endif
1194 %endmacro
1195
1196 ; Append cpuflags to the callee's name iff the appended name is known and the plain name isn't
1197 %macro call 1
1198 %ifid %1
1199 call_internal %1 %+ SUFFIX, %1
1200 %else
1201 call %1
1202 %endif
1203 %endmacro
1204 %macro call_internal 2
1205 %xdefine %%i %2
1206 %ifndef cglobaled_%2
1207 %ifdef cglobaled_%1
1208 %xdefine %%i %1
1209 %endif
1210 %endif
1211 call %%i
1212 LOAD_MM_PERMUTATION %%i
1213 %endmacro
1214
1215 ; Substitutions that reduce instruction size but are functionally equivalent
1216 %macro add 2
1217 %ifnum %2
1218 %if %2==128
1219 sub %1, -128
1220 %else
1221 add %1, %2
1222 %endif
1223 %else
1224 add %1, %2
1225 %endif
1226 %endmacro
1227
1228 %macro sub 2
1229 %ifnum %2
1230 %if %2==128
1231 add %1, -128
1232 %else
1233 sub %1, %2
1234 %endif
1235 %else
1236 sub %1, %2
1237 %endif
1238 %endmacro
1239
1240 ;=============================================================================
1241 ; AVX abstraction layer
1242 ;=============================================================================
1243
1244 %assign i 0
1245 %rep 32
1246 %if i < 8
1247 CAT_XDEFINE sizeofmm, i, 8
1248 CAT_XDEFINE regnumofmm, i, i
1249 %endif
1250 CAT_XDEFINE sizeofxmm, i, 16
1251 CAT_XDEFINE sizeofymm, i, 32
1252 CAT_XDEFINE sizeofzmm, i, 64
1253 CAT_XDEFINE regnumofxmm, i, i
1254 CAT_XDEFINE regnumofymm, i, i
1255 CAT_XDEFINE regnumofzmm, i, i
1256 %assign i i+1
1257 %endrep
1258 %undef i
1259
1260 %macro CHECK_AVX_INSTR_EMU 3-*
1261 %xdefine %%opcode %1
1262 %xdefine %%dst %2
1263 %rep %0-2
1264 %ifidn %%dst, %3
1265 %error non-avx emulation of ``%%opcode'' is not supported
1266 %endif
1267 %rotate 1
1268 %endrep
1269 %endmacro
1270
1271 ;%1 == instruction
1272 ;%2 == minimal instruction set
1273 ;%3 == 1 if float, 0 if int
1274 ;%4 == 1 if 4-operand emulation, 0 if 3-operand emulation, 255 otherwise (no emulation)
1275 ;%5 == 1 if commutative (i.e. doesn't matter which src arg is which), 0 if not
1276 ;%6+: operands
1277 %macro RUN_AVX_INSTR 6-9+
1278 %ifnum sizeof%7
1279 %assign __sizeofreg sizeof%7
1280 %elifnum sizeof%6
1281 %assign __sizeofreg sizeof%6
1282 %else
1283 %assign __sizeofreg mmsize
1284 %endif
1285 %assign __emulate_avx 0
1286 %if avx_enabled && __sizeofreg >= 16
1287 %xdefine __instr v%1
1288 %else
1289 %xdefine __instr %1
1290 %if %0 >= 8+%4
1291 %assign __emulate_avx 1
1292 %endif
1293 %endif
1294 %ifnidn %2, fnord
1295 %ifdef cpuname
1296 %if notcpuflag(%2)
1297 %error use of ``%1'' %2 instruction in cpuname function: current_function
1298 %elif %3 == 0 && __sizeofreg == 16 && notcpuflag(sse2)
1299 %error use of ``%1'' sse2 instruction in cpuname function: current_function
1300 %elif %3 == 0 && __sizeofreg == 32 && notcpuflag(avx2)
1301 %error use of ``%1'' avx2 instruction in cpuname function: current_function
1302 %elif __sizeofreg == 16 && notcpuflag(sse)
1303 %error use of ``%1'' sse instruction in cpuname function: current_function
1304 %elif __sizeofreg == 32 && notcpuflag(avx)
1305 %error use of ``%1'' avx instruction in cpuname function: current_function
1306 %elif __sizeofreg == 64 && notcpuflag(avx512)
1307 %error use of ``%1'' avx512 instruction in cpuname function: current_function
1308 %elifidn %1, pextrw ; special case because the base instruction is mmx2,
1309 %ifnid %6 ; but sse4 is required for memory operands
1310 %if notcpuflag(sse4)
1311 %error use of ``%1'' sse4 instruction in cpuname function: current_function
1312 %endif
1313 %endif
1314 %endif
1315 %endif
1316 %endif
1317
1318 %if __emulate_avx
1319 %xdefine __src1 %7
1320 %xdefine __src2 %8
1321 %if %5 && %4 == 0
1322 %ifnidn %6, %7
1323 %ifidn %6, %8
1324 %xdefine __src1 %8
1325 %xdefine __src2 %7
1326 %elifnnum sizeof%8
1327 ; 3-operand AVX instructions with a memory arg can only have it in src2,
1328 ; whereas SSE emulation prefers to have it in src1 (i.e. the mov).
1329 ; So, if the instruction is commutative with a memory arg, swap them.
1330 %xdefine __src1 %8
1331 %xdefine __src2 %7
1332 %endif
1333 %endif
1334 %endif
1335 %ifnidn %6, __src1
1336 %if %0 >= 9
1337 CHECK_AVX_INSTR_EMU {%1 %6, %7, %8, %9}, %6, __src2, %9
1338 %else
1339 CHECK_AVX_INSTR_EMU {%1 %6, %7, %8}, %6, __src2
1340 %endif
1341 %if __sizeofreg == 8
1342 MOVQ %6, __src1
1343 %elif %3
1344 MOVAPS %6, __src1
1345 %else
1346 MOVDQA %6, __src1
1347 %endif
1348 %endif
1349 %if %0 >= 9
1350 %1 %6, __src2, %9
1351 %else
1352 %1 %6, __src2
1353 %endif
1354 %elif %0 >= 9
1355 %if avx_enabled && __sizeofreg >= 16 && %4 == 1
1356 %ifnnum regnumof%7
1357 %if %3
1358 vmovaps %6, %7
1359 %else
1360 vmovdqa %6, %7
1361 %endif
1362 __instr %6, %6, %8, %9
1363 %else
1364 __instr %6, %7, %8, %9
1365 %endif
1366 %else
1367 __instr %6, %7, %8, %9
1368 %endif
1369 %elif %0 == 8
1370 %if avx_enabled && __sizeofreg >= 16 && %4 == 0
1371 %xdefine __src1 %7
1372 %xdefine __src2 %8
1373 %if %5
1374 %ifnum regnumof%7
1375 %ifnum regnumof%8
1376 %if regnumof%7 < 8 && regnumof%8 >= 8 && regnumof%8 < 16 && sizeof%8 <= 32
1377 ; Most VEX-encoded instructions require an additional byte to encode when
1378 ; src2 is a high register (e.g. m8..15). If the instruction is commutative
1379 ; we can swap src1 and src2 when doing so reduces the instruction length.
1380 %xdefine __src1 %8
1381 %xdefine __src2 %7
1382 %endif
1383 %endif
1384 %elifnum regnumof%8 ; put memory operands in src2 when possible
1385 %xdefine __src1 %8
1386 %xdefine __src2 %7
1387 %else
1388 %assign __emulate_avx 1
1389 %endif
1390 %elifnnum regnumof%7
1391 ; EVEX allows imm8 shift instructions to be used with memory operands,
1392 ; but VEX does not. This handles those special cases.
1393 %ifnnum %8
1394 %assign __emulate_avx 1
1395 %elif notcpuflag(avx512)
1396 %assign __emulate_avx 1
1397 %endif
1398 %endif
1399 %if __emulate_avx ; a separate load is required
1400 %if %3
1401 vmovaps %6, %7
1402 %else
1403 vmovdqa %6, %7
1404 %endif
1405 __instr %6, %6, %8
1406 %else
1407 __instr %6, __src1, __src2
1408 %endif
1409 %else
1410 __instr %6, %7, %8
1411 %endif
1412 %elif %0 == 7
1413 %if avx_enabled && __sizeofreg >= 16 && %5
1414 %xdefine __src1 %6
1415 %xdefine __src2 %7
1416 %ifnum regnumof%6
1417 %ifnum regnumof%7
1418 %if regnumof%6 < 8 && regnumof%7 >= 8 && regnumof%7 < 16 && sizeof%7 <= 32
1419 %xdefine __src1 %7
1420 %xdefine __src2 %6
1421 %endif
1422 %endif
1423 %endif
1424 __instr %6, __src1, __src2
1425 %else
1426 __instr %6, %7
1427 %endif
1428 %else
1429 __instr %6
1430 %endif
1431 %endmacro
1432
1433 ;%1 == instruction
1434 ;%2 == minimal instruction set
1435 ;%3 == 1 if float, 0 if int
1436 ;%4 == 1 if 4-operand emulation, 0 if 3-operand emulation, 255 otherwise (no emulation)
1437 ;%5 == 1 if commutative (i.e. doesn't matter which src arg is which), 0 if not
1438 %macro AVX_INSTR 1-5 fnord, 0, 255, 0
1439 %macro %1 1-10 fnord, fnord, fnord, fnord, %1, %2, %3, %4, %5
1440 %ifidn %2, fnord
1441 RUN_AVX_INSTR %6, %7, %8, %9, %10, %1
1442 %elifidn %3, fnord
1443 RUN_AVX_INSTR %6, %7, %8, %9, %10, %1, %2
1444 %elifidn %4, fnord
1445 RUN_AVX_INSTR %6, %7, %8, %9, %10, %1, %2, %3
1446 %elifidn %5, fnord
1447 RUN_AVX_INSTR %6, %7, %8, %9, %10, %1, %2, %3, %4
1448 %else
1449 RUN_AVX_INSTR %6, %7, %8, %9, %10, %1, %2, %3, %4, %5
1450 %endif
1451 %endmacro
1452 %endmacro
1453
1454 ; Instructions with both VEX/EVEX and legacy encodings
1455 ; Non-destructive instructions are written without parameters
1456 AVX_INSTR addpd, sse2, 1, 0, 1
1457 AVX_INSTR addps, sse, 1, 0, 1
1458 AVX_INSTR addsd, sse2, 1, 0, 0
1459 AVX_INSTR addss, sse, 1, 0, 0
1460 AVX_INSTR addsubpd, sse3, 1, 0, 0
1461 AVX_INSTR addsubps, sse3, 1, 0, 0
1462 AVX_INSTR aesdec, aesni, 0, 0, 0
1463 AVX_INSTR aesdeclast, aesni, 0, 0, 0
1464 AVX_INSTR aesenc, aesni, 0, 0, 0
1465 AVX_INSTR aesenclast, aesni, 0, 0, 0
1466 AVX_INSTR aesimc, aesni
1467 AVX_INSTR aeskeygenassist, aesni
1468 AVX_INSTR andnpd, sse2, 1, 0, 0
1469 AVX_INSTR andnps, sse, 1, 0, 0
1470 AVX_INSTR andpd, sse2, 1, 0, 1
1471 AVX_INSTR andps, sse, 1, 0, 1
1472 AVX_INSTR blendpd, sse4, 1, 1, 0
1473 AVX_INSTR blendps, sse4, 1, 1, 0
1474 AVX_INSTR blendvpd, sse4, 1, 1, 0 ; last operand must be xmm0 with legacy encoding
1475 AVX_INSTR blendvps, sse4, 1, 1, 0 ; last operand must be xmm0 with legacy encoding
1476 AVX_INSTR cmpeqpd, sse2, 1, 0, 1
1477 AVX_INSTR cmpeqps, sse, 1, 0, 1
1478 AVX_INSTR cmpeqsd, sse2, 1, 0, 0
1479 AVX_INSTR cmpeqss, sse, 1, 0, 0
1480 AVX_INSTR cmplepd, sse2, 1, 0, 0
1481 AVX_INSTR cmpleps, sse, 1, 0, 0
1482 AVX_INSTR cmplesd, sse2, 1, 0, 0
1483 AVX_INSTR cmpless, sse, 1, 0, 0
1484 AVX_INSTR cmpltpd, sse2, 1, 0, 0
1485 AVX_INSTR cmpltps, sse, 1, 0, 0
1486 AVX_INSTR cmpltsd, sse2, 1, 0, 0
1487 AVX_INSTR cmpltss, sse, 1, 0, 0
1488 AVX_INSTR cmpneqpd, sse2, 1, 0, 1
1489 AVX_INSTR cmpneqps, sse, 1, 0, 1
1490 AVX_INSTR cmpneqsd, sse2, 1, 0, 0
1491 AVX_INSTR cmpneqss, sse, 1, 0, 0
1492 AVX_INSTR cmpnlepd, sse2, 1, 0, 0
1493 AVX_INSTR cmpnleps, sse, 1, 0, 0
1494 AVX_INSTR cmpnlesd, sse2, 1, 0, 0
1495 AVX_INSTR cmpnless, sse, 1, 0, 0
1496 AVX_INSTR cmpnltpd, sse2, 1, 0, 0
1497 AVX_INSTR cmpnltps, sse, 1, 0, 0
1498 AVX_INSTR cmpnltsd, sse2, 1, 0, 0
1499 AVX_INSTR cmpnltss, sse, 1, 0, 0
1500 AVX_INSTR cmpordpd, sse2 1, 0, 1
1501 AVX_INSTR cmpordps, sse 1, 0, 1
1502 AVX_INSTR cmpordsd, sse2 1, 0, 0
1503 AVX_INSTR cmpordss, sse 1, 0, 0
1504 AVX_INSTR cmppd, sse2, 1, 1, 0
1505 AVX_INSTR cmpps, sse, 1, 1, 0
1506 AVX_INSTR cmpsd, sse2, 1, 1, 0
1507 AVX_INSTR cmpss, sse, 1, 1, 0
1508 AVX_INSTR cmpunordpd, sse2, 1, 0, 1
1509 AVX_INSTR cmpunordps, sse, 1, 0, 1
1510 AVX_INSTR cmpunordsd, sse2, 1, 0, 0
1511 AVX_INSTR cmpunordss, sse, 1, 0, 0
1512 AVX_INSTR comisd, sse2, 1
1513 AVX_INSTR comiss, sse, 1
1514 AVX_INSTR cvtdq2pd, sse2, 1
1515 AVX_INSTR cvtdq2ps, sse2, 1
1516 AVX_INSTR cvtpd2dq, sse2, 1
1517 AVX_INSTR cvtpd2ps, sse2, 1
1518 AVX_INSTR cvtps2dq, sse2, 1
1519 AVX_INSTR cvtps2pd, sse2, 1
1520 AVX_INSTR cvtsd2si, sse2, 1
1521 AVX_INSTR cvtsd2ss, sse2, 1, 0, 0
1522 AVX_INSTR cvtsi2sd, sse2, 1, 0, 0
1523 AVX_INSTR cvtsi2ss, sse, 1, 0, 0
1524 AVX_INSTR cvtss2sd, sse2, 1, 0, 0
1525 AVX_INSTR cvtss2si, sse, 1
1526 AVX_INSTR cvttpd2dq, sse2, 1
1527 AVX_INSTR cvttps2dq, sse2, 1
1528 AVX_INSTR cvttsd2si, sse2, 1
1529 AVX_INSTR cvttss2si, sse, 1
1530 AVX_INSTR divpd, sse2, 1, 0, 0
1531 AVX_INSTR divps, sse, 1, 0, 0
1532 AVX_INSTR divsd, sse2, 1, 0, 0
1533 AVX_INSTR divss, sse, 1, 0, 0
1534 AVX_INSTR dppd, sse4, 1, 1, 0
1535 AVX_INSTR dpps, sse4, 1, 1, 0
1536 AVX_INSTR extractps, sse4, 1
1537 AVX_INSTR gf2p8affineinvqb, gfni, 0, 1, 0
1538 AVX_INSTR gf2p8affineqb, gfni, 0, 1, 0
1539 AVX_INSTR gf2p8mulb, gfni, 0, 0, 0
1540 AVX_INSTR haddpd, sse3, 1, 0, 0
1541 AVX_INSTR haddps, sse3, 1, 0, 0
1542 AVX_INSTR hsubpd, sse3, 1, 0, 0
1543 AVX_INSTR hsubps, sse3, 1, 0, 0
1544 AVX_INSTR insertps, sse4, 1, 1, 0
1545 AVX_INSTR lddqu, sse3
1546 AVX_INSTR ldmxcsr, sse, 1
1547 AVX_INSTR maskmovdqu, sse2
1548 AVX_INSTR maxpd, sse2, 1, 0, 1
1549 AVX_INSTR maxps, sse, 1, 0, 1
1550 AVX_INSTR maxsd, sse2, 1, 0, 0
1551 AVX_INSTR maxss, sse, 1, 0, 0
1552 AVX_INSTR minpd, sse2, 1, 0, 1
1553 AVX_INSTR minps, sse, 1, 0, 1
1554 AVX_INSTR minsd, sse2, 1, 0, 0
1555 AVX_INSTR minss, sse, 1, 0, 0
1556 AVX_INSTR movapd, sse2, 1
1557 AVX_INSTR movaps, sse, 1
1558 AVX_INSTR movd, mmx
1559 AVX_INSTR movddup, sse3, 1
1560 AVX_INSTR movdqa, sse2
1561 AVX_INSTR movdqu, sse2
1562 AVX_INSTR movhlps, sse, 1, 0, 0
1563 AVX_INSTR movhpd, sse2, 1, 0, 0
1564 AVX_INSTR movhps, sse, 1, 0, 0
1565 AVX_INSTR movlhps, sse, 1, 0, 0
1566 AVX_INSTR movlpd, sse2, 1, 0, 0
1567 AVX_INSTR movlps, sse, 1, 0, 0
1568 AVX_INSTR movmskpd, sse2, 1
1569 AVX_INSTR movmskps, sse, 1
1570 AVX_INSTR movntdq, sse2
1571 AVX_INSTR movntdqa, sse4
1572 AVX_INSTR movntpd, sse2, 1
1573 AVX_INSTR movntps, sse, 1
1574 AVX_INSTR movq, mmx
1575 AVX_INSTR movsd, sse2, 1, 0, 0
1576 AVX_INSTR movshdup, sse3, 1
1577 AVX_INSTR movsldup, sse3, 1
1578 AVX_INSTR movss, sse, 1, 0, 0
1579 AVX_INSTR movupd, sse2, 1
1580 AVX_INSTR movups, sse, 1
1581 AVX_INSTR mpsadbw, sse4, 0, 1, 0
1582 AVX_INSTR mulpd, sse2, 1, 0, 1
1583 AVX_INSTR mulps, sse, 1, 0, 1
1584 AVX_INSTR mulsd, sse2, 1, 0, 0
1585 AVX_INSTR mulss, sse, 1, 0, 0
1586 AVX_INSTR orpd, sse2, 1, 0, 1
1587 AVX_INSTR orps, sse, 1, 0, 1
1588 AVX_INSTR pabsb, ssse3
1589 AVX_INSTR pabsd, ssse3
1590 AVX_INSTR pabsw, ssse3
1591 AVX_INSTR packssdw, mmx, 0, 0, 0
1592 AVX_INSTR packsswb, mmx, 0, 0, 0
1593 AVX_INSTR packusdw, sse4, 0, 0, 0
1594 AVX_INSTR packuswb, mmx, 0, 0, 0
1595 AVX_INSTR paddb, mmx, 0, 0, 1
1596 AVX_INSTR paddd, mmx, 0, 0, 1
1597 AVX_INSTR paddq, sse2, 0, 0, 1
1598 AVX_INSTR paddsb, mmx, 0, 0, 1
1599 AVX_INSTR paddsw, mmx, 0, 0, 1
1600 AVX_INSTR paddusb, mmx, 0, 0, 1
1601 AVX_INSTR paddusw, mmx, 0, 0, 1
1602 AVX_INSTR paddw, mmx, 0, 0, 1
1603 AVX_INSTR palignr, ssse3, 0, 1, 0
1604 AVX_INSTR pand, mmx, 0, 0, 1
1605 AVX_INSTR pandn, mmx, 0, 0, 0
1606 AVX_INSTR pavgb, mmx2, 0, 0, 1
1607 AVX_INSTR pavgw, mmx2, 0, 0, 1
1608 AVX_INSTR pblendvb, sse4, 0, 1, 0 ; last operand must be xmm0 with legacy encoding
1609 AVX_INSTR pblendw, sse4, 0, 1, 0
1610 AVX_INSTR pclmulhqhqdq, fnord, 0, 0, 0
1611 AVX_INSTR pclmulhqlqdq, fnord, 0, 0, 0
1612 AVX_INSTR pclmullqhqdq, fnord, 0, 0, 0
1613 AVX_INSTR pclmullqlqdq, fnord, 0, 0, 0
1614 AVX_INSTR pclmulqdq, fnord, 0, 1, 0
1615 AVX_INSTR pcmpeqb, mmx, 0, 0, 1
1616 AVX_INSTR pcmpeqd, mmx, 0, 0, 1
1617 AVX_INSTR pcmpeqq, sse4, 0, 0, 1
1618 AVX_INSTR pcmpeqw, mmx, 0, 0, 1
1619 AVX_INSTR pcmpestri, sse42
1620 AVX_INSTR pcmpestrm, sse42
1621 AVX_INSTR pcmpgtb, mmx, 0, 0, 0
1622 AVX_INSTR pcmpgtd, mmx, 0, 0, 0
1623 AVX_INSTR pcmpgtq, sse42, 0, 0, 0
1624 AVX_INSTR pcmpgtw, mmx, 0, 0, 0
1625 AVX_INSTR pcmpistri, sse42
1626 AVX_INSTR pcmpistrm, sse42
1627 AVX_INSTR pextrb, sse4
1628 AVX_INSTR pextrd, sse4
1629 AVX_INSTR pextrq, sse4
1630 AVX_INSTR pextrw, mmx2
1631 AVX_INSTR phaddd, ssse3, 0, 0, 0
1632 AVX_INSTR phaddsw, ssse3, 0, 0, 0
1633 AVX_INSTR phaddw, ssse3, 0, 0, 0
1634 AVX_INSTR phminposuw, sse4
1635 AVX_INSTR phsubd, ssse3, 0, 0, 0
1636 AVX_INSTR phsubsw, ssse3, 0, 0, 0
1637 AVX_INSTR phsubw, ssse3, 0, 0, 0
1638 AVX_INSTR pinsrb, sse4, 0, 1, 0
1639 AVX_INSTR pinsrd, sse4, 0, 1, 0
1640 AVX_INSTR pinsrq, sse4, 0, 1, 0
1641 AVX_INSTR pinsrw, mmx2, 0, 1, 0
1642 AVX_INSTR pmaddubsw, ssse3, 0, 0, 0
1643 AVX_INSTR pmaddwd, mmx, 0, 0, 1
1644 AVX_INSTR pmaxsb, sse4, 0, 0, 1
1645 AVX_INSTR pmaxsd, sse4, 0, 0, 1
1646 AVX_INSTR pmaxsw, mmx2, 0, 0, 1
1647 AVX_INSTR pmaxub, mmx2, 0, 0, 1
1648 AVX_INSTR pmaxud, sse4, 0, 0, 1
1649 AVX_INSTR pmaxuw, sse4, 0, 0, 1
1650 AVX_INSTR pminsb, sse4, 0, 0, 1
1651 AVX_INSTR pminsd, sse4, 0, 0, 1
1652 AVX_INSTR pminsw, mmx2, 0, 0, 1
1653 AVX_INSTR pminub, mmx2, 0, 0, 1
1654 AVX_INSTR pminud, sse4, 0, 0, 1
1655 AVX_INSTR pminuw, sse4, 0, 0, 1
1656 AVX_INSTR pmovmskb, mmx2
1657 AVX_INSTR pmovsxbd, sse4
1658 AVX_INSTR pmovsxbq, sse4
1659 AVX_INSTR pmovsxbw, sse4
1660 AVX_INSTR pmovsxdq, sse4
1661 AVX_INSTR pmovsxwd, sse4
1662 AVX_INSTR pmovsxwq, sse4
1663 AVX_INSTR pmovzxbd, sse4
1664 AVX_INSTR pmovzxbq, sse4
1665 AVX_INSTR pmovzxbw, sse4
1666 AVX_INSTR pmovzxdq, sse4
1667 AVX_INSTR pmovzxwd, sse4
1668 AVX_INSTR pmovzxwq, sse4
1669 AVX_INSTR pmuldq, sse4, 0, 0, 1
1670 AVX_INSTR pmulhrsw, ssse3, 0, 0, 1
1671 AVX_INSTR pmulhuw, mmx2, 0, 0, 1
1672 AVX_INSTR pmulhw, mmx, 0, 0, 1
1673 AVX_INSTR pmulld, sse4, 0, 0, 1
1674 AVX_INSTR pmullw, mmx, 0, 0, 1
1675 AVX_INSTR pmuludq, sse2, 0, 0, 1
1676 AVX_INSTR por, mmx, 0, 0, 1
1677 AVX_INSTR psadbw, mmx2, 0, 0, 1
1678 AVX_INSTR pshufb, ssse3, 0, 0, 0
1679 AVX_INSTR pshufd, sse2
1680 AVX_INSTR pshufhw, sse2
1681 AVX_INSTR pshuflw, sse2
1682 AVX_INSTR psignb, ssse3, 0, 0, 0
1683 AVX_INSTR psignd, ssse3, 0, 0, 0
1684 AVX_INSTR psignw, ssse3, 0, 0, 0
1685 AVX_INSTR pslld, mmx, 0, 0, 0
1686 AVX_INSTR pslldq, sse2, 0, 0, 0
1687 AVX_INSTR psllq, mmx, 0, 0, 0
1688 AVX_INSTR psllw, mmx, 0, 0, 0
1689 AVX_INSTR psrad, mmx, 0, 0, 0
1690 AVX_INSTR psraw, mmx, 0, 0, 0
1691 AVX_INSTR psrld, mmx, 0, 0, 0
1692 AVX_INSTR psrldq, sse2, 0, 0, 0
1693 AVX_INSTR psrlq, mmx, 0, 0, 0
1694 AVX_INSTR psrlw, mmx, 0, 0, 0
1695 AVX_INSTR psubb, mmx, 0, 0, 0
1696 AVX_INSTR psubd, mmx, 0, 0, 0
1697 AVX_INSTR psubq, sse2, 0, 0, 0
1698 AVX_INSTR psubsb, mmx, 0, 0, 0
1699 AVX_INSTR psubsw, mmx, 0, 0, 0
1700 AVX_INSTR psubusb, mmx, 0, 0, 0
1701 AVX_INSTR psubusw, mmx, 0, 0, 0
1702 AVX_INSTR psubw, mmx, 0, 0, 0
1703 AVX_INSTR ptest, sse4
1704 AVX_INSTR punpckhbw, mmx, 0, 0, 0
1705 AVX_INSTR punpckhdq, mmx, 0, 0, 0
1706 AVX_INSTR punpckhqdq, sse2, 0, 0, 0
1707 AVX_INSTR punpckhwd, mmx, 0, 0, 0
1708 AVX_INSTR punpcklbw, mmx, 0, 0, 0
1709 AVX_INSTR punpckldq, mmx, 0, 0, 0
1710 AVX_INSTR punpcklqdq, sse2, 0, 0, 0
1711 AVX_INSTR punpcklwd, mmx, 0, 0, 0
1712 AVX_INSTR pxor, mmx, 0, 0, 1
1713 AVX_INSTR rcpps, sse, 1
1714 AVX_INSTR rcpss, sse, 1, 0, 0
1715 AVX_INSTR roundpd, sse4, 1
1716 AVX_INSTR roundps, sse4, 1
1717 AVX_INSTR roundsd, sse4, 1, 1, 0
1718 AVX_INSTR roundss, sse4, 1, 1, 0
1719 AVX_INSTR rsqrtps, sse, 1
1720 AVX_INSTR rsqrtss, sse, 1, 0, 0
1721 AVX_INSTR shufpd, sse2, 1, 1, 0
1722 AVX_INSTR shufps, sse, 1, 1, 0
1723 AVX_INSTR sqrtpd, sse2, 1
1724 AVX_INSTR sqrtps, sse, 1
1725 AVX_INSTR sqrtsd, sse2, 1, 0, 0
1726 AVX_INSTR sqrtss, sse, 1, 0, 0
1727 AVX_INSTR stmxcsr, sse, 1
1728 AVX_INSTR subpd, sse2, 1, 0, 0
1729 AVX_INSTR subps, sse, 1, 0, 0
1730 AVX_INSTR subsd, sse2, 1, 0, 0
1731 AVX_INSTR subss, sse, 1, 0, 0
1732 AVX_INSTR ucomisd, sse2, 1
1733 AVX_INSTR ucomiss, sse, 1
1734 AVX_INSTR unpckhpd, sse2, 1, 0, 0
1735 AVX_INSTR unpckhps, sse, 1, 0, 0
1736 AVX_INSTR unpcklpd, sse2, 1, 0, 0
1737 AVX_INSTR unpcklps, sse, 1, 0, 0
1738 AVX_INSTR xorpd, sse2, 1, 0, 1
1739 AVX_INSTR xorps, sse, 1, 0, 1
1740
1741 ; 3DNow instructions, for sharing code between AVX, SSE and 3DN
1742 AVX_INSTR pfadd, 3dnow, 1, 0, 1
1743 AVX_INSTR pfmul, 3dnow, 1, 0, 1
1744 AVX_INSTR pfsub, 3dnow, 1, 0, 0
1745
1746 ;%1 == instruction
1747 ;%2 == minimal instruction set
1748 %macro GPR_INSTR 2
1749 %macro %1 2-5 fnord, %1, %2
1750 %ifdef cpuname
1751 %if notcpuflag(%5)
1752 %error use of ``%4'' %5 instruction in cpuname function: current_function
1753 %endif
1754 %endif
1755 %ifidn %3, fnord
1756 %4 %1, %2
1757 %else
1758 %4 %1, %2, %3
1759 %endif
1760 %endmacro
1761 %endmacro
1762
1763 GPR_INSTR andn, bmi1
1764 GPR_INSTR bextr, bmi1
1765 GPR_INSTR blsi, bmi1
1766 GPR_INSTR blsmsk, bmi1
1767 GPR_INSTR blsr, bmi1
1768 GPR_INSTR bzhi, bmi2
1769 GPR_INSTR mulx, bmi2
1770 GPR_INSTR pdep, bmi2
1771 GPR_INSTR pext, bmi2
1772 GPR_INSTR popcnt, sse42
1773 GPR_INSTR rorx, bmi2
1774 GPR_INSTR sarx, bmi2
1775 GPR_INSTR shlx, bmi2
1776 GPR_INSTR shrx, bmi2
1777
1778 ; base-4 constants for shuffles
1779 %assign i 0
1780 %rep 256
1781 %assign j ((i>>6)&3)*1000 + ((i>>4)&3)*100 + ((i>>2)&3)*10 + (i&3)
1782 %if j < 10
1783 CAT_XDEFINE q000, j, i
1784 %elif j < 100
1785 CAT_XDEFINE q00, j, i
1786 %elif j < 1000
1787 CAT_XDEFINE q0, j, i
1788 %else
1789 CAT_XDEFINE q, j, i
1790 %endif
1791 %assign i i+1
1792 %endrep
1793 %undef i
1794 %undef j
1795
1796 %macro FMA_INSTR 3
1797 %macro %1 4-7 %1, %2, %3
1798 %if cpuflag(xop)
1799 v%5 %1, %2, %3, %4
1800 %elifnidn %1, %4
1801 %6 %1, %2, %3
1802 %7 %1, %4
1803 %else
1804 %error non-xop emulation of ``%5 %1, %2, %3, %4'' is not supported
1805 %endif
1806 %endmacro
1807 %endmacro
1808
1809 FMA_INSTR pmacsdd, pmulld, paddd ; sse4 emulation
1810 FMA_INSTR pmacsdql, pmuldq, paddq ; sse4 emulation
1811 FMA_INSTR pmacsww, pmullw, paddw
1812 FMA_INSTR pmadcswd, pmaddwd, paddd
1813
1814 ; Macros for consolidating FMA3 and FMA4 using 4-operand (dst, src1, src2, src3) syntax.
1815 ; FMA3 is only possible if dst is the same as one of the src registers.
1816 ; Either src2 or src3 can be a memory operand.
1817 %macro FMA4_INSTR 2-*
1818 %push fma4_instr
1819 %xdefine %$prefix %1
1820 %rep %0 - 1
1821 %macro %$prefix%2 4-6 %$prefix, %2
1822 %if notcpuflag(fma3) && notcpuflag(fma4)
1823 %error use of ``%5%6'' fma instruction in cpuname function: current_function
1824 %elif cpuflag(fma4)
1825 v%5%6 %1, %2, %3, %4
1826 %elifidn %1, %2
1827 ; If %3 or %4 is a memory operand it needs to be encoded as the last operand.
1828 %ifnum sizeof%3
1829 v%{5}213%6 %2, %3, %4
1830 %else
1831 v%{5}132%6 %2, %4, %3
1832 %endif
1833 %elifidn %1, %3
1834 v%{5}213%6 %3, %2, %4
1835 %elifidn %1, %4
1836 v%{5}231%6 %4, %2, %3
1837 %else
1838 %error fma3 emulation of ``%5%6 %1, %2, %3, %4'' is not supported
1839 %endif
1840 %endmacro
1841 %rotate 1
1842 %endrep
1843 %pop
1844 %endmacro
1845
1846 FMA4_INSTR fmadd, pd, ps, sd, ss
1847 FMA4_INSTR fmaddsub, pd, ps
1848 FMA4_INSTR fmsub, pd, ps, sd, ss
1849 FMA4_INSTR fmsubadd, pd, ps
1850 FMA4_INSTR fnmadd, pd, ps, sd, ss
1851 FMA4_INSTR fnmsub, pd, ps, sd, ss
1852
1853 ; Macros for converting VEX instructions to equivalent EVEX ones.
1854 %macro EVEX_INSTR 2-3 0 ; vex, evex, prefer_evex
1855 %macro %1 2-7 fnord, fnord, %1, %2, %3
1856 %ifidn %3, fnord
1857 %define %%args %1, %2
1858 %elifidn %4, fnord
1859 %define %%args %1, %2, %3
1860 %else
1861 %define %%args %1, %2, %3, %4
1862 %endif
1863 %assign %%evex_required cpuflag(avx512) & %7
1864 %ifnum regnumof%1
1865 %if regnumof%1 >= 16 || sizeof%1 > 32
1866 %assign %%evex_required 1
1867 %endif
1868 %endif
1869 %ifnum regnumof%2
1870 %if regnumof%2 >= 16 || sizeof%2 > 32
1871 %assign %%evex_required 1
1872 %endif
1873 %endif
1874 %ifnum regnumof%3
1875 %if regnumof%3 >= 16 || sizeof%3 > 32
1876 %assign %%evex_required 1
1877 %endif
1878 %endif
1879 %if %%evex_required
1880 %6 %%args
1881 %else
1882 %5 %%args ; Prefer VEX over EVEX due to shorter instruction length
1883 %endif
1884 %endmacro
1885 %endmacro
1886
1887 EVEX_INSTR vbroadcastf128, vbroadcastf32x4
1888 EVEX_INSTR vbroadcasti128, vbroadcasti32x4
1889 EVEX_INSTR vextractf128, vextractf32x4
1890 EVEX_INSTR vextracti128, vextracti32x4
1891 EVEX_INSTR vinsertf128, vinsertf32x4
1892 EVEX_INSTR vinserti128, vinserti32x4
1893 EVEX_INSTR vmovdqa, vmovdqa32
1894 EVEX_INSTR vmovdqu, vmovdqu32
1895 EVEX_INSTR vpand, vpandd
1896 EVEX_INSTR vpandn, vpandnd
1897 EVEX_INSTR vpor, vpord
1898 EVEX_INSTR vpxor, vpxord
1899 EVEX_INSTR vrcpps, vrcp14ps, 1 ; EVEX versions have higher precision
1900 EVEX_INSTR vrcpss, vrcp14ss, 1
1901 EVEX_INSTR vrsqrtps, vrsqrt14ps, 1
1902 EVEX_INSTR vrsqrtss, vrsqrt14ss, 1