| 1 |
; This file is generated from a similarly-named Perl script in the BoringSSL |
| 2 |
; source tree. Do not edit by hand. |
| 3 |
|
| 4 |
%ifidn __OUTPUT_FORMAT__, win64 |
| 5 |
default rel |
| 6 |
%define XMMWORD |
| 7 |
%define YMMWORD |
| 8 |
%define ZMMWORD |
| 9 |
|
| 10 |
%ifdef BORINGSSL_PREFIX |
| 11 |
%include "boringssl_prefix_symbols_nasm.inc" |
| 12 |
%endif |
| 13 |
section .text code align=64 |
| 14 |
|
| 15 |
|
| 16 |
global rsaz_1024_sqr_avx2 |
| 17 |
|
| 18 |
ALIGN 64 |
| 19 |
rsaz_1024_sqr_avx2: |
| 20 |
mov QWORD[8+rsp],rdi ;WIN64 prologue |
| 21 |
mov QWORD[16+rsp],rsi |
| 22 |
mov rax,rsp |
| 23 |
$L$SEH_begin_rsaz_1024_sqr_avx2: |
| 24 |
mov rdi,rcx |
| 25 |
mov rsi,rdx |
| 26 |
mov rdx,r8 |
| 27 |
mov rcx,r9 |
| 28 |
mov r8,QWORD[40+rsp] |
| 29 |
|
| 30 |
|
| 31 |
|
| 32 |
lea rax,[rsp] |
| 33 |
|
| 34 |
push rbx |
| 35 |
|
| 36 |
push rbp |
| 37 |
|
| 38 |
push r12 |
| 39 |
|
| 40 |
push r13 |
| 41 |
|
| 42 |
push r14 |
| 43 |
|
| 44 |
push r15 |
| 45 |
|
| 46 |
vzeroupper |
| 47 |
lea rsp,[((-168))+rsp] |
| 48 |
vmovaps XMMWORD[(-216)+rax],xmm6 |
| 49 |
vmovaps XMMWORD[(-200)+rax],xmm7 |
| 50 |
vmovaps XMMWORD[(-184)+rax],xmm8 |
| 51 |
vmovaps XMMWORD[(-168)+rax],xmm9 |
| 52 |
vmovaps XMMWORD[(-152)+rax],xmm10 |
| 53 |
vmovaps XMMWORD[(-136)+rax],xmm11 |
| 54 |
vmovaps XMMWORD[(-120)+rax],xmm12 |
| 55 |
vmovaps XMMWORD[(-104)+rax],xmm13 |
| 56 |
vmovaps XMMWORD[(-88)+rax],xmm14 |
| 57 |
vmovaps XMMWORD[(-72)+rax],xmm15 |
| 58 |
$L$sqr_1024_body: |
| 59 |
mov rbp,rax |
| 60 |
|
| 61 |
mov r13,rdx |
| 62 |
sub rsp,832 |
| 63 |
mov r15,r13 |
| 64 |
sub rdi,-128 |
| 65 |
sub rsi,-128 |
| 66 |
sub r13,-128 |
| 67 |
|
| 68 |
and r15,4095 |
| 69 |
add r15,32*10 |
| 70 |
shr r15,12 |
| 71 |
vpxor ymm9,ymm9,ymm9 |
| 72 |
jz NEAR $L$sqr_1024_no_n_copy |
| 73 |
|
| 74 |
|
| 75 |
|
| 76 |
|
| 77 |
|
| 78 |
sub rsp,32*10 |
| 79 |
vmovdqu ymm0,YMMWORD[((0-128))+r13] |
| 80 |
and rsp,-2048 |
| 81 |
vmovdqu ymm1,YMMWORD[((32-128))+r13] |
| 82 |
vmovdqu ymm2,YMMWORD[((64-128))+r13] |
| 83 |
vmovdqu ymm3,YMMWORD[((96-128))+r13] |
| 84 |
vmovdqu ymm4,YMMWORD[((128-128))+r13] |
| 85 |
vmovdqu ymm5,YMMWORD[((160-128))+r13] |
| 86 |
vmovdqu ymm6,YMMWORD[((192-128))+r13] |
| 87 |
vmovdqu ymm7,YMMWORD[((224-128))+r13] |
| 88 |
vmovdqu ymm8,YMMWORD[((256-128))+r13] |
| 89 |
lea r13,[((832+128))+rsp] |
| 90 |
vmovdqu YMMWORD[(0-128)+r13],ymm0 |
| 91 |
vmovdqu YMMWORD[(32-128)+r13],ymm1 |
| 92 |
vmovdqu YMMWORD[(64-128)+r13],ymm2 |
| 93 |
vmovdqu YMMWORD[(96-128)+r13],ymm3 |
| 94 |
vmovdqu YMMWORD[(128-128)+r13],ymm4 |
| 95 |
vmovdqu YMMWORD[(160-128)+r13],ymm5 |
| 96 |
vmovdqu YMMWORD[(192-128)+r13],ymm6 |
| 97 |
vmovdqu YMMWORD[(224-128)+r13],ymm7 |
| 98 |
vmovdqu YMMWORD[(256-128)+r13],ymm8 |
| 99 |
vmovdqu YMMWORD[(288-128)+r13],ymm9 |
| 100 |
|
| 101 |
$L$sqr_1024_no_n_copy: |
| 102 |
and rsp,-1024 |
| 103 |
|
| 104 |
vmovdqu ymm1,YMMWORD[((32-128))+rsi] |
| 105 |
vmovdqu ymm2,YMMWORD[((64-128))+rsi] |
| 106 |
vmovdqu ymm3,YMMWORD[((96-128))+rsi] |
| 107 |
vmovdqu ymm4,YMMWORD[((128-128))+rsi] |
| 108 |
vmovdqu ymm5,YMMWORD[((160-128))+rsi] |
| 109 |
vmovdqu ymm6,YMMWORD[((192-128))+rsi] |
| 110 |
vmovdqu ymm7,YMMWORD[((224-128))+rsi] |
| 111 |
vmovdqu ymm8,YMMWORD[((256-128))+rsi] |
| 112 |
|
| 113 |
lea rbx,[192+rsp] |
| 114 |
vmovdqu ymm15,YMMWORD[$L$and_mask] |
| 115 |
jmp NEAR $L$OOP_GRANDE_SQR_1024 |
| 116 |
|
| 117 |
ALIGN 32 |
| 118 |
$L$OOP_GRANDE_SQR_1024: |
| 119 |
lea r9,[((576+128))+rsp] |
| 120 |
lea r12,[448+rsp] |
| 121 |
|
| 122 |
|
| 123 |
|
| 124 |
|
| 125 |
vpaddq ymm1,ymm1,ymm1 |
| 126 |
vpbroadcastq ymm10,QWORD[((0-128))+rsi] |
| 127 |
vpaddq ymm2,ymm2,ymm2 |
| 128 |
vmovdqa YMMWORD[(0-128)+r9],ymm1 |
| 129 |
vpaddq ymm3,ymm3,ymm3 |
| 130 |
vmovdqa YMMWORD[(32-128)+r9],ymm2 |
| 131 |
vpaddq ymm4,ymm4,ymm4 |
| 132 |
vmovdqa YMMWORD[(64-128)+r9],ymm3 |
| 133 |
vpaddq ymm5,ymm5,ymm5 |
| 134 |
vmovdqa YMMWORD[(96-128)+r9],ymm4 |
| 135 |
vpaddq ymm6,ymm6,ymm6 |
| 136 |
vmovdqa YMMWORD[(128-128)+r9],ymm5 |
| 137 |
vpaddq ymm7,ymm7,ymm7 |
| 138 |
vmovdqa YMMWORD[(160-128)+r9],ymm6 |
| 139 |
vpaddq ymm8,ymm8,ymm8 |
| 140 |
vmovdqa YMMWORD[(192-128)+r9],ymm7 |
| 141 |
vpxor ymm9,ymm9,ymm9 |
| 142 |
vmovdqa YMMWORD[(224-128)+r9],ymm8 |
| 143 |
|
| 144 |
vpmuludq ymm0,ymm10,YMMWORD[((0-128))+rsi] |
| 145 |
vpbroadcastq ymm11,QWORD[((32-128))+rsi] |
| 146 |
vmovdqu YMMWORD[(288-192)+rbx],ymm9 |
| 147 |
vpmuludq ymm1,ymm1,ymm10 |
| 148 |
vmovdqu YMMWORD[(320-448)+r12],ymm9 |
| 149 |
vpmuludq ymm2,ymm2,ymm10 |
| 150 |
vmovdqu YMMWORD[(352-448)+r12],ymm9 |
| 151 |
vpmuludq ymm3,ymm3,ymm10 |
| 152 |
vmovdqu YMMWORD[(384-448)+r12],ymm9 |
| 153 |
vpmuludq ymm4,ymm4,ymm10 |
| 154 |
vmovdqu YMMWORD[(416-448)+r12],ymm9 |
| 155 |
vpmuludq ymm5,ymm5,ymm10 |
| 156 |
vmovdqu YMMWORD[(448-448)+r12],ymm9 |
| 157 |
vpmuludq ymm6,ymm6,ymm10 |
| 158 |
vmovdqu YMMWORD[(480-448)+r12],ymm9 |
| 159 |
vpmuludq ymm7,ymm7,ymm10 |
| 160 |
vmovdqu YMMWORD[(512-448)+r12],ymm9 |
| 161 |
vpmuludq ymm8,ymm8,ymm10 |
| 162 |
vpbroadcastq ymm10,QWORD[((64-128))+rsi] |
| 163 |
vmovdqu YMMWORD[(544-448)+r12],ymm9 |
| 164 |
|
| 165 |
mov r15,rsi |
| 166 |
mov r14d,4 |
| 167 |
jmp NEAR $L$sqr_entry_1024 |
| 168 |
ALIGN 32 |
| 169 |
$L$OOP_SQR_1024: |
| 170 |
vpbroadcastq ymm11,QWORD[((32-128))+r15] |
| 171 |
vpmuludq ymm0,ymm10,YMMWORD[((0-128))+rsi] |
| 172 |
vpaddq ymm0,ymm0,YMMWORD[((0-192))+rbx] |
| 173 |
vpmuludq ymm1,ymm10,YMMWORD[((0-128))+r9] |
| 174 |
vpaddq ymm1,ymm1,YMMWORD[((32-192))+rbx] |
| 175 |
vpmuludq ymm2,ymm10,YMMWORD[((32-128))+r9] |
| 176 |
vpaddq ymm2,ymm2,YMMWORD[((64-192))+rbx] |
| 177 |
vpmuludq ymm3,ymm10,YMMWORD[((64-128))+r9] |
| 178 |
vpaddq ymm3,ymm3,YMMWORD[((96-192))+rbx] |
| 179 |
vpmuludq ymm4,ymm10,YMMWORD[((96-128))+r9] |
| 180 |
vpaddq ymm4,ymm4,YMMWORD[((128-192))+rbx] |
| 181 |
vpmuludq ymm5,ymm10,YMMWORD[((128-128))+r9] |
| 182 |
vpaddq ymm5,ymm5,YMMWORD[((160-192))+rbx] |
| 183 |
vpmuludq ymm6,ymm10,YMMWORD[((160-128))+r9] |
| 184 |
vpaddq ymm6,ymm6,YMMWORD[((192-192))+rbx] |
| 185 |
vpmuludq ymm7,ymm10,YMMWORD[((192-128))+r9] |
| 186 |
vpaddq ymm7,ymm7,YMMWORD[((224-192))+rbx] |
| 187 |
vpmuludq ymm8,ymm10,YMMWORD[((224-128))+r9] |
| 188 |
vpbroadcastq ymm10,QWORD[((64-128))+r15] |
| 189 |
vpaddq ymm8,ymm8,YMMWORD[((256-192))+rbx] |
| 190 |
$L$sqr_entry_1024: |
| 191 |
vmovdqu YMMWORD[(0-192)+rbx],ymm0 |
| 192 |
vmovdqu YMMWORD[(32-192)+rbx],ymm1 |
| 193 |
|
| 194 |
vpmuludq ymm12,ymm11,YMMWORD[((32-128))+rsi] |
| 195 |
vpaddq ymm2,ymm2,ymm12 |
| 196 |
vpmuludq ymm14,ymm11,YMMWORD[((32-128))+r9] |
| 197 |
vpaddq ymm3,ymm3,ymm14 |
| 198 |
vpmuludq ymm13,ymm11,YMMWORD[((64-128))+r9] |
| 199 |
vpaddq ymm4,ymm4,ymm13 |
| 200 |
vpmuludq ymm12,ymm11,YMMWORD[((96-128))+r9] |
| 201 |
vpaddq ymm5,ymm5,ymm12 |
| 202 |
vpmuludq ymm14,ymm11,YMMWORD[((128-128))+r9] |
| 203 |
vpaddq ymm6,ymm6,ymm14 |
| 204 |
vpmuludq ymm13,ymm11,YMMWORD[((160-128))+r9] |
| 205 |
vpaddq ymm7,ymm7,ymm13 |
| 206 |
vpmuludq ymm12,ymm11,YMMWORD[((192-128))+r9] |
| 207 |
vpaddq ymm8,ymm8,ymm12 |
| 208 |
vpmuludq ymm0,ymm11,YMMWORD[((224-128))+r9] |
| 209 |
vpbroadcastq ymm11,QWORD[((96-128))+r15] |
| 210 |
vpaddq ymm0,ymm0,YMMWORD[((288-192))+rbx] |
| 211 |
|
| 212 |
vmovdqu YMMWORD[(64-192)+rbx],ymm2 |
| 213 |
vmovdqu YMMWORD[(96-192)+rbx],ymm3 |
| 214 |
|
| 215 |
vpmuludq ymm13,ymm10,YMMWORD[((64-128))+rsi] |
| 216 |
vpaddq ymm4,ymm4,ymm13 |
| 217 |
vpmuludq ymm12,ymm10,YMMWORD[((64-128))+r9] |
| 218 |
vpaddq ymm5,ymm5,ymm12 |
| 219 |
vpmuludq ymm14,ymm10,YMMWORD[((96-128))+r9] |
| 220 |
vpaddq ymm6,ymm6,ymm14 |
| 221 |
vpmuludq ymm13,ymm10,YMMWORD[((128-128))+r9] |
| 222 |
vpaddq ymm7,ymm7,ymm13 |
| 223 |
vpmuludq ymm12,ymm10,YMMWORD[((160-128))+r9] |
| 224 |
vpaddq ymm8,ymm8,ymm12 |
| 225 |
vpmuludq ymm14,ymm10,YMMWORD[((192-128))+r9] |
| 226 |
vpaddq ymm0,ymm0,ymm14 |
| 227 |
vpmuludq ymm1,ymm10,YMMWORD[((224-128))+r9] |
| 228 |
vpbroadcastq ymm10,QWORD[((128-128))+r15] |
| 229 |
vpaddq ymm1,ymm1,YMMWORD[((320-448))+r12] |
| 230 |
|
| 231 |
vmovdqu YMMWORD[(128-192)+rbx],ymm4 |
| 232 |
vmovdqu YMMWORD[(160-192)+rbx],ymm5 |
| 233 |
|
| 234 |
vpmuludq ymm12,ymm11,YMMWORD[((96-128))+rsi] |
| 235 |
vpaddq ymm6,ymm6,ymm12 |
| 236 |
vpmuludq ymm14,ymm11,YMMWORD[((96-128))+r9] |
| 237 |
vpaddq ymm7,ymm7,ymm14 |
| 238 |
vpmuludq ymm13,ymm11,YMMWORD[((128-128))+r9] |
| 239 |
vpaddq ymm8,ymm8,ymm13 |
| 240 |
vpmuludq ymm12,ymm11,YMMWORD[((160-128))+r9] |
| 241 |
vpaddq ymm0,ymm0,ymm12 |
| 242 |
vpmuludq ymm14,ymm11,YMMWORD[((192-128))+r9] |
| 243 |
vpaddq ymm1,ymm1,ymm14 |
| 244 |
vpmuludq ymm2,ymm11,YMMWORD[((224-128))+r9] |
| 245 |
vpbroadcastq ymm11,QWORD[((160-128))+r15] |
| 246 |
vpaddq ymm2,ymm2,YMMWORD[((352-448))+r12] |
| 247 |
|
| 248 |
vmovdqu YMMWORD[(192-192)+rbx],ymm6 |
| 249 |
vmovdqu YMMWORD[(224-192)+rbx],ymm7 |
| 250 |
|
| 251 |
vpmuludq ymm12,ymm10,YMMWORD[((128-128))+rsi] |
| 252 |
vpaddq ymm8,ymm8,ymm12 |
| 253 |
vpmuludq ymm14,ymm10,YMMWORD[((128-128))+r9] |
| 254 |
vpaddq ymm0,ymm0,ymm14 |
| 255 |
vpmuludq ymm13,ymm10,YMMWORD[((160-128))+r9] |
| 256 |
vpaddq ymm1,ymm1,ymm13 |
| 257 |
vpmuludq ymm12,ymm10,YMMWORD[((192-128))+r9] |
| 258 |
vpaddq ymm2,ymm2,ymm12 |
| 259 |
vpmuludq ymm3,ymm10,YMMWORD[((224-128))+r9] |
| 260 |
vpbroadcastq ymm10,QWORD[((192-128))+r15] |
| 261 |
vpaddq ymm3,ymm3,YMMWORD[((384-448))+r12] |
| 262 |
|
| 263 |
vmovdqu YMMWORD[(256-192)+rbx],ymm8 |
| 264 |
vmovdqu YMMWORD[(288-192)+rbx],ymm0 |
| 265 |
lea rbx,[8+rbx] |
| 266 |
|
| 267 |
vpmuludq ymm13,ymm11,YMMWORD[((160-128))+rsi] |
| 268 |
vpaddq ymm1,ymm1,ymm13 |
| 269 |
vpmuludq ymm12,ymm11,YMMWORD[((160-128))+r9] |
| 270 |
vpaddq ymm2,ymm2,ymm12 |
| 271 |
vpmuludq ymm14,ymm11,YMMWORD[((192-128))+r9] |
| 272 |
vpaddq ymm3,ymm3,ymm14 |
| 273 |
vpmuludq ymm4,ymm11,YMMWORD[((224-128))+r9] |
| 274 |
vpbroadcastq ymm11,QWORD[((224-128))+r15] |
| 275 |
vpaddq ymm4,ymm4,YMMWORD[((416-448))+r12] |
| 276 |
|
| 277 |
vmovdqu YMMWORD[(320-448)+r12],ymm1 |
| 278 |
vmovdqu YMMWORD[(352-448)+r12],ymm2 |
| 279 |
|
| 280 |
vpmuludq ymm12,ymm10,YMMWORD[((192-128))+rsi] |
| 281 |
vpaddq ymm3,ymm3,ymm12 |
| 282 |
vpmuludq ymm14,ymm10,YMMWORD[((192-128))+r9] |
| 283 |
vpbroadcastq ymm0,QWORD[((256-128))+r15] |
| 284 |
vpaddq ymm4,ymm4,ymm14 |
| 285 |
vpmuludq ymm5,ymm10,YMMWORD[((224-128))+r9] |
| 286 |
vpbroadcastq ymm10,QWORD[((0+8-128))+r15] |
| 287 |
vpaddq ymm5,ymm5,YMMWORD[((448-448))+r12] |
| 288 |
|
| 289 |
vmovdqu YMMWORD[(384-448)+r12],ymm3 |
| 290 |
vmovdqu YMMWORD[(416-448)+r12],ymm4 |
| 291 |
lea r15,[8+r15] |
| 292 |
|
| 293 |
vpmuludq ymm12,ymm11,YMMWORD[((224-128))+rsi] |
| 294 |
vpaddq ymm5,ymm5,ymm12 |
| 295 |
vpmuludq ymm6,ymm11,YMMWORD[((224-128))+r9] |
| 296 |
vpaddq ymm6,ymm6,YMMWORD[((480-448))+r12] |
| 297 |
|
| 298 |
vpmuludq ymm7,ymm0,YMMWORD[((256-128))+rsi] |
| 299 |
vmovdqu YMMWORD[(448-448)+r12],ymm5 |
| 300 |
vpaddq ymm7,ymm7,YMMWORD[((512-448))+r12] |
| 301 |
vmovdqu YMMWORD[(480-448)+r12],ymm6 |
| 302 |
vmovdqu YMMWORD[(512-448)+r12],ymm7 |
| 303 |
lea r12,[8+r12] |
| 304 |
|
| 305 |
dec r14d |
| 306 |
jnz NEAR $L$OOP_SQR_1024 |
| 307 |
|
| 308 |
vmovdqu ymm8,YMMWORD[256+rsp] |
| 309 |
vmovdqu ymm1,YMMWORD[288+rsp] |
| 310 |
vmovdqu ymm2,YMMWORD[320+rsp] |
| 311 |
lea rbx,[192+rsp] |
| 312 |
|
| 313 |
vpsrlq ymm14,ymm8,29 |
| 314 |
vpand ymm8,ymm8,ymm15 |
| 315 |
vpsrlq ymm11,ymm1,29 |
| 316 |
vpand ymm1,ymm1,ymm15 |
| 317 |
|
| 318 |
vpermq ymm14,ymm14,0x93 |
| 319 |
vpxor ymm9,ymm9,ymm9 |
| 320 |
vpermq ymm11,ymm11,0x93 |
| 321 |
|
| 322 |
vpblendd ymm10,ymm14,ymm9,3 |
| 323 |
vpblendd ymm14,ymm11,ymm14,3 |
| 324 |
vpaddq ymm8,ymm8,ymm10 |
| 325 |
vpblendd ymm11,ymm9,ymm11,3 |
| 326 |
vpaddq ymm1,ymm1,ymm14 |
| 327 |
vpaddq ymm2,ymm2,ymm11 |
| 328 |
vmovdqu YMMWORD[(288-192)+rbx],ymm1 |
| 329 |
vmovdqu YMMWORD[(320-192)+rbx],ymm2 |
| 330 |
|
| 331 |
mov rax,QWORD[rsp] |
| 332 |
mov r10,QWORD[8+rsp] |
| 333 |
mov r11,QWORD[16+rsp] |
| 334 |
mov r12,QWORD[24+rsp] |
| 335 |
vmovdqu ymm1,YMMWORD[32+rsp] |
| 336 |
vmovdqu ymm2,YMMWORD[((64-192))+rbx] |
| 337 |
vmovdqu ymm3,YMMWORD[((96-192))+rbx] |
| 338 |
vmovdqu ymm4,YMMWORD[((128-192))+rbx] |
| 339 |
vmovdqu ymm5,YMMWORD[((160-192))+rbx] |
| 340 |
vmovdqu ymm6,YMMWORD[((192-192))+rbx] |
| 341 |
vmovdqu ymm7,YMMWORD[((224-192))+rbx] |
| 342 |
|
| 343 |
mov r9,rax |
| 344 |
imul eax,ecx |
| 345 |
and eax,0x1fffffff |
| 346 |
vmovd xmm12,eax |
| 347 |
|
| 348 |
mov rdx,rax |
| 349 |
imul rax,QWORD[((-128))+r13] |
| 350 |
vpbroadcastq ymm12,xmm12 |
| 351 |
add r9,rax |
| 352 |
mov rax,rdx |
| 353 |
imul rax,QWORD[((8-128))+r13] |
| 354 |
shr r9,29 |
| 355 |
add r10,rax |
| 356 |
mov rax,rdx |
| 357 |
imul rax,QWORD[((16-128))+r13] |
| 358 |
add r10,r9 |
| 359 |
add r11,rax |
| 360 |
imul rdx,QWORD[((24-128))+r13] |
| 361 |
add r12,rdx |
| 362 |
|
| 363 |
mov rax,r10 |
| 364 |
imul eax,ecx |
| 365 |
and eax,0x1fffffff |
| 366 |
|
| 367 |
mov r14d,9 |
| 368 |
jmp NEAR $L$OOP_REDUCE_1024 |
| 369 |
|
| 370 |
ALIGN 32 |
| 371 |
$L$OOP_REDUCE_1024: |
| 372 |
vmovd xmm13,eax |
| 373 |
vpbroadcastq ymm13,xmm13 |
| 374 |
|
| 375 |
vpmuludq ymm10,ymm12,YMMWORD[((32-128))+r13] |
| 376 |
mov rdx,rax |
| 377 |
imul rax,QWORD[((-128))+r13] |
| 378 |
vpaddq ymm1,ymm1,ymm10 |
| 379 |
add r10,rax |
| 380 |
vpmuludq ymm14,ymm12,YMMWORD[((64-128))+r13] |
| 381 |
mov rax,rdx |
| 382 |
imul rax,QWORD[((8-128))+r13] |
| 383 |
vpaddq ymm2,ymm2,ymm14 |
| 384 |
vpmuludq ymm11,ymm12,YMMWORD[((96-128))+r13] |
| 385 |
DB 0x67 |
| 386 |
add r11,rax |
| 387 |
DB 0x67 |
| 388 |
mov rax,rdx |
| 389 |
imul rax,QWORD[((16-128))+r13] |
| 390 |
shr r10,29 |
| 391 |
vpaddq ymm3,ymm3,ymm11 |
| 392 |
vpmuludq ymm10,ymm12,YMMWORD[((128-128))+r13] |
| 393 |
add r12,rax |
| 394 |
add r11,r10 |
| 395 |
vpaddq ymm4,ymm4,ymm10 |
| 396 |
vpmuludq ymm14,ymm12,YMMWORD[((160-128))+r13] |
| 397 |
mov rax,r11 |
| 398 |
imul eax,ecx |
| 399 |
vpaddq ymm5,ymm5,ymm14 |
| 400 |
vpmuludq ymm11,ymm12,YMMWORD[((192-128))+r13] |
| 401 |
and eax,0x1fffffff |
| 402 |
vpaddq ymm6,ymm6,ymm11 |
| 403 |
vpmuludq ymm10,ymm12,YMMWORD[((224-128))+r13] |
| 404 |
vpaddq ymm7,ymm7,ymm10 |
| 405 |
vpmuludq ymm14,ymm12,YMMWORD[((256-128))+r13] |
| 406 |
vmovd xmm12,eax |
| 407 |
|
| 408 |
vpaddq ymm8,ymm8,ymm14 |
| 409 |
|
| 410 |
vpbroadcastq ymm12,xmm12 |
| 411 |
|
| 412 |
vpmuludq ymm11,ymm13,YMMWORD[((32-8-128))+r13] |
| 413 |
vmovdqu ymm14,YMMWORD[((96-8-128))+r13] |
| 414 |
mov rdx,rax |
| 415 |
imul rax,QWORD[((-128))+r13] |
| 416 |
vpaddq ymm1,ymm1,ymm11 |
| 417 |
vpmuludq ymm10,ymm13,YMMWORD[((64-8-128))+r13] |
| 418 |
vmovdqu ymm11,YMMWORD[((128-8-128))+r13] |
| 419 |
add r11,rax |
| 420 |
mov rax,rdx |
| 421 |
imul rax,QWORD[((8-128))+r13] |
| 422 |
vpaddq ymm2,ymm2,ymm10 |
| 423 |
add rax,r12 |
| 424 |
shr r11,29 |
| 425 |
vpmuludq ymm14,ymm14,ymm13 |
| 426 |
vmovdqu ymm10,YMMWORD[((160-8-128))+r13] |
| 427 |
add rax,r11 |
| 428 |
vpaddq ymm3,ymm3,ymm14 |
| 429 |
vpmuludq ymm11,ymm11,ymm13 |
| 430 |
vmovdqu ymm14,YMMWORD[((192-8-128))+r13] |
| 431 |
DB 0x67 |
| 432 |
mov r12,rax |
| 433 |
imul eax,ecx |
| 434 |
vpaddq ymm4,ymm4,ymm11 |
| 435 |
vpmuludq ymm10,ymm10,ymm13 |
| 436 |
DB 0xc4,0x41,0x7e,0x6f,0x9d,0x58,0x00,0x00,0x00 |
| 437 |
and eax,0x1fffffff |
| 438 |
vpaddq ymm5,ymm5,ymm10 |
| 439 |
vpmuludq ymm14,ymm14,ymm13 |
| 440 |
vmovdqu ymm10,YMMWORD[((256-8-128))+r13] |
| 441 |
vpaddq ymm6,ymm6,ymm14 |
| 442 |
vpmuludq ymm11,ymm11,ymm13 |
| 443 |
vmovdqu ymm9,YMMWORD[((288-8-128))+r13] |
| 444 |
vmovd xmm0,eax |
| 445 |
imul rax,QWORD[((-128))+r13] |
| 446 |
vpaddq ymm7,ymm7,ymm11 |
| 447 |
vpmuludq ymm10,ymm10,ymm13 |
| 448 |
vmovdqu ymm14,YMMWORD[((32-16-128))+r13] |
| 449 |
vpbroadcastq ymm0,xmm0 |
| 450 |
vpaddq ymm8,ymm8,ymm10 |
| 451 |
vpmuludq ymm9,ymm9,ymm13 |
| 452 |
vmovdqu ymm11,YMMWORD[((64-16-128))+r13] |
| 453 |
add r12,rax |
| 454 |
|
| 455 |
vmovdqu ymm13,YMMWORD[((32-24-128))+r13] |
| 456 |
vpmuludq ymm14,ymm14,ymm12 |
| 457 |
vmovdqu ymm10,YMMWORD[((96-16-128))+r13] |
| 458 |
vpaddq ymm1,ymm1,ymm14 |
| 459 |
vpmuludq ymm13,ymm13,ymm0 |
| 460 |
vpmuludq ymm11,ymm11,ymm12 |
| 461 |
DB 0xc4,0x41,0x7e,0x6f,0xb5,0xf0,0xff,0xff,0xff |
| 462 |
vpaddq ymm13,ymm13,ymm1 |
| 463 |
vpaddq ymm2,ymm2,ymm11 |
| 464 |
vpmuludq ymm10,ymm10,ymm12 |
| 465 |
vmovdqu ymm11,YMMWORD[((160-16-128))+r13] |
| 466 |
DB 0x67 |
| 467 |
vmovq rax,xmm13 |
| 468 |
vmovdqu YMMWORD[rsp],ymm13 |
| 469 |
vpaddq ymm3,ymm3,ymm10 |
| 470 |
vpmuludq ymm14,ymm14,ymm12 |
| 471 |
vmovdqu ymm10,YMMWORD[((192-16-128))+r13] |
| 472 |
vpaddq ymm4,ymm4,ymm14 |
| 473 |
vpmuludq ymm11,ymm11,ymm12 |
| 474 |
vmovdqu ymm14,YMMWORD[((224-16-128))+r13] |
| 475 |
vpaddq ymm5,ymm5,ymm11 |
| 476 |
vpmuludq ymm10,ymm10,ymm12 |
| 477 |
vmovdqu ymm11,YMMWORD[((256-16-128))+r13] |
| 478 |
vpaddq ymm6,ymm6,ymm10 |
| 479 |
vpmuludq ymm14,ymm14,ymm12 |
| 480 |
shr r12,29 |
| 481 |
vmovdqu ymm10,YMMWORD[((288-16-128))+r13] |
| 482 |
add rax,r12 |
| 483 |
vpaddq ymm7,ymm7,ymm14 |
| 484 |
vpmuludq ymm11,ymm11,ymm12 |
| 485 |
|
| 486 |
mov r9,rax |
| 487 |
imul eax,ecx |
| 488 |
vpaddq ymm8,ymm8,ymm11 |
| 489 |
vpmuludq ymm10,ymm10,ymm12 |
| 490 |
and eax,0x1fffffff |
| 491 |
vmovd xmm12,eax |
| 492 |
vmovdqu ymm11,YMMWORD[((96-24-128))+r13] |
| 493 |
DB 0x67 |
| 494 |
vpaddq ymm9,ymm9,ymm10 |
| 495 |
vpbroadcastq ymm12,xmm12 |
| 496 |
|
| 497 |
vpmuludq ymm14,ymm0,YMMWORD[((64-24-128))+r13] |
| 498 |
vmovdqu ymm10,YMMWORD[((128-24-128))+r13] |
| 499 |
mov rdx,rax |
| 500 |
imul rax,QWORD[((-128))+r13] |
| 501 |
mov r10,QWORD[8+rsp] |
| 502 |
vpaddq ymm1,ymm2,ymm14 |
| 503 |
vpmuludq ymm11,ymm11,ymm0 |
| 504 |
vmovdqu ymm14,YMMWORD[((160-24-128))+r13] |
| 505 |
add r9,rax |
| 506 |
mov rax,rdx |
| 507 |
imul rax,QWORD[((8-128))+r13] |
| 508 |
DB 0x67 |
| 509 |
shr r9,29 |
| 510 |
mov r11,QWORD[16+rsp] |
| 511 |
vpaddq ymm2,ymm3,ymm11 |
| 512 |
vpmuludq ymm10,ymm10,ymm0 |
| 513 |
vmovdqu ymm11,YMMWORD[((192-24-128))+r13] |
| 514 |
add r10,rax |
| 515 |
mov rax,rdx |
| 516 |
imul rax,QWORD[((16-128))+r13] |
| 517 |
vpaddq ymm3,ymm4,ymm10 |
| 518 |
vpmuludq ymm14,ymm14,ymm0 |
| 519 |
vmovdqu ymm10,YMMWORD[((224-24-128))+r13] |
| 520 |
imul rdx,QWORD[((24-128))+r13] |
| 521 |
add r11,rax |
| 522 |
lea rax,[r10*1+r9] |
| 523 |
vpaddq ymm4,ymm5,ymm14 |
| 524 |
vpmuludq ymm11,ymm11,ymm0 |
| 525 |
vmovdqu ymm14,YMMWORD[((256-24-128))+r13] |
| 526 |
mov r10,rax |
| 527 |
imul eax,ecx |
| 528 |
vpmuludq ymm10,ymm10,ymm0 |
| 529 |
vpaddq ymm5,ymm6,ymm11 |
| 530 |
vmovdqu ymm11,YMMWORD[((288-24-128))+r13] |
| 531 |
and eax,0x1fffffff |
| 532 |
vpaddq ymm6,ymm7,ymm10 |
| 533 |
vpmuludq ymm14,ymm14,ymm0 |
| 534 |
add rdx,QWORD[24+rsp] |
| 535 |
vpaddq ymm7,ymm8,ymm14 |
| 536 |
vpmuludq ymm11,ymm11,ymm0 |
| 537 |
vpaddq ymm8,ymm9,ymm11 |
| 538 |
vmovq xmm9,r12 |
| 539 |
mov r12,rdx |
| 540 |
|
| 541 |
dec r14d |
| 542 |
jnz NEAR $L$OOP_REDUCE_1024 |
| 543 |
lea r12,[448+rsp] |
| 544 |
vpaddq ymm0,ymm13,ymm9 |
| 545 |
vpxor ymm9,ymm9,ymm9 |
| 546 |
|
| 547 |
vpaddq ymm0,ymm0,YMMWORD[((288-192))+rbx] |
| 548 |
vpaddq ymm1,ymm1,YMMWORD[((320-448))+r12] |
| 549 |
vpaddq ymm2,ymm2,YMMWORD[((352-448))+r12] |
| 550 |
vpaddq ymm3,ymm3,YMMWORD[((384-448))+r12] |
| 551 |
vpaddq ymm4,ymm4,YMMWORD[((416-448))+r12] |
| 552 |
vpaddq ymm5,ymm5,YMMWORD[((448-448))+r12] |
| 553 |
vpaddq ymm6,ymm6,YMMWORD[((480-448))+r12] |
| 554 |
vpaddq ymm7,ymm7,YMMWORD[((512-448))+r12] |
| 555 |
vpaddq ymm8,ymm8,YMMWORD[((544-448))+r12] |
| 556 |
|
| 557 |
vpsrlq ymm14,ymm0,29 |
| 558 |
vpand ymm0,ymm0,ymm15 |
| 559 |
vpsrlq ymm11,ymm1,29 |
| 560 |
vpand ymm1,ymm1,ymm15 |
| 561 |
vpsrlq ymm12,ymm2,29 |
| 562 |
vpermq ymm14,ymm14,0x93 |
| 563 |
vpand ymm2,ymm2,ymm15 |
| 564 |
vpsrlq ymm13,ymm3,29 |
| 565 |
vpermq ymm11,ymm11,0x93 |
| 566 |
vpand ymm3,ymm3,ymm15 |
| 567 |
vpermq ymm12,ymm12,0x93 |
| 568 |
|
| 569 |
vpblendd ymm10,ymm14,ymm9,3 |
| 570 |
vpermq ymm13,ymm13,0x93 |
| 571 |
vpblendd ymm14,ymm11,ymm14,3 |
| 572 |
vpaddq ymm0,ymm0,ymm10 |
| 573 |
vpblendd ymm11,ymm12,ymm11,3 |
| 574 |
vpaddq ymm1,ymm1,ymm14 |
| 575 |
vpblendd ymm12,ymm13,ymm12,3 |
| 576 |
vpaddq ymm2,ymm2,ymm11 |
| 577 |
vpblendd ymm13,ymm9,ymm13,3 |
| 578 |
vpaddq ymm3,ymm3,ymm12 |
| 579 |
vpaddq ymm4,ymm4,ymm13 |
| 580 |
|
| 581 |
vpsrlq ymm14,ymm0,29 |
| 582 |
vpand ymm0,ymm0,ymm15 |
| 583 |
vpsrlq ymm11,ymm1,29 |
| 584 |
vpand ymm1,ymm1,ymm15 |
| 585 |
vpsrlq ymm12,ymm2,29 |
| 586 |
vpermq ymm14,ymm14,0x93 |
| 587 |
vpand ymm2,ymm2,ymm15 |
| 588 |
vpsrlq ymm13,ymm3,29 |
| 589 |
vpermq ymm11,ymm11,0x93 |
| 590 |
vpand ymm3,ymm3,ymm15 |
| 591 |
vpermq ymm12,ymm12,0x93 |
| 592 |
|
| 593 |
vpblendd ymm10,ymm14,ymm9,3 |
| 594 |
vpermq ymm13,ymm13,0x93 |
| 595 |
vpblendd ymm14,ymm11,ymm14,3 |
| 596 |
vpaddq ymm0,ymm0,ymm10 |
| 597 |
vpblendd ymm11,ymm12,ymm11,3 |
| 598 |
vpaddq ymm1,ymm1,ymm14 |
| 599 |
vmovdqu YMMWORD[(0-128)+rdi],ymm0 |
| 600 |
vpblendd ymm12,ymm13,ymm12,3 |
| 601 |
vpaddq ymm2,ymm2,ymm11 |
| 602 |
vmovdqu YMMWORD[(32-128)+rdi],ymm1 |
| 603 |
vpblendd ymm13,ymm9,ymm13,3 |
| 604 |
vpaddq ymm3,ymm3,ymm12 |
| 605 |
vmovdqu YMMWORD[(64-128)+rdi],ymm2 |
| 606 |
vpaddq ymm4,ymm4,ymm13 |
| 607 |
vmovdqu YMMWORD[(96-128)+rdi],ymm3 |
| 608 |
vpsrlq ymm14,ymm4,29 |
| 609 |
vpand ymm4,ymm4,ymm15 |
| 610 |
vpsrlq ymm11,ymm5,29 |
| 611 |
vpand ymm5,ymm5,ymm15 |
| 612 |
vpsrlq ymm12,ymm6,29 |
| 613 |
vpermq ymm14,ymm14,0x93 |
| 614 |
vpand ymm6,ymm6,ymm15 |
| 615 |
vpsrlq ymm13,ymm7,29 |
| 616 |
vpermq ymm11,ymm11,0x93 |
| 617 |
vpand ymm7,ymm7,ymm15 |
| 618 |
vpsrlq ymm0,ymm8,29 |
| 619 |
vpermq ymm12,ymm12,0x93 |
| 620 |
vpand ymm8,ymm8,ymm15 |
| 621 |
vpermq ymm13,ymm13,0x93 |
| 622 |
|
| 623 |
vpblendd ymm10,ymm14,ymm9,3 |
| 624 |
vpermq ymm0,ymm0,0x93 |
| 625 |
vpblendd ymm14,ymm11,ymm14,3 |
| 626 |
vpaddq ymm4,ymm4,ymm10 |
| 627 |
vpblendd ymm11,ymm12,ymm11,3 |
| 628 |
vpaddq ymm5,ymm5,ymm14 |
| 629 |
vpblendd ymm12,ymm13,ymm12,3 |
| 630 |
vpaddq ymm6,ymm6,ymm11 |
| 631 |
vpblendd ymm13,ymm0,ymm13,3 |
| 632 |
vpaddq ymm7,ymm7,ymm12 |
| 633 |
vpaddq ymm8,ymm8,ymm13 |
| 634 |
|
| 635 |
vpsrlq ymm14,ymm4,29 |
| 636 |
vpand ymm4,ymm4,ymm15 |
| 637 |
vpsrlq ymm11,ymm5,29 |
| 638 |
vpand ymm5,ymm5,ymm15 |
| 639 |
vpsrlq ymm12,ymm6,29 |
| 640 |
vpermq ymm14,ymm14,0x93 |
| 641 |
vpand ymm6,ymm6,ymm15 |
| 642 |
vpsrlq ymm13,ymm7,29 |
| 643 |
vpermq ymm11,ymm11,0x93 |
| 644 |
vpand ymm7,ymm7,ymm15 |
| 645 |
vpsrlq ymm0,ymm8,29 |
| 646 |
vpermq ymm12,ymm12,0x93 |
| 647 |
vpand ymm8,ymm8,ymm15 |
| 648 |
vpermq ymm13,ymm13,0x93 |
| 649 |
|
| 650 |
vpblendd ymm10,ymm14,ymm9,3 |
| 651 |
vpermq ymm0,ymm0,0x93 |
| 652 |
vpblendd ymm14,ymm11,ymm14,3 |
| 653 |
vpaddq ymm4,ymm4,ymm10 |
| 654 |
vpblendd ymm11,ymm12,ymm11,3 |
| 655 |
vpaddq ymm5,ymm5,ymm14 |
| 656 |
vmovdqu YMMWORD[(128-128)+rdi],ymm4 |
| 657 |
vpblendd ymm12,ymm13,ymm12,3 |
| 658 |
vpaddq ymm6,ymm6,ymm11 |
| 659 |
vmovdqu YMMWORD[(160-128)+rdi],ymm5 |
| 660 |
vpblendd ymm13,ymm0,ymm13,3 |
| 661 |
vpaddq ymm7,ymm7,ymm12 |
| 662 |
vmovdqu YMMWORD[(192-128)+rdi],ymm6 |
| 663 |
vpaddq ymm8,ymm8,ymm13 |
| 664 |
vmovdqu YMMWORD[(224-128)+rdi],ymm7 |
| 665 |
vmovdqu YMMWORD[(256-128)+rdi],ymm8 |
| 666 |
|
| 667 |
mov rsi,rdi |
| 668 |
dec r8d |
| 669 |
jne NEAR $L$OOP_GRANDE_SQR_1024 |
| 670 |
|
| 671 |
vzeroall |
| 672 |
mov rax,rbp |
| 673 |
|
| 674 |
$L$sqr_1024_in_tail: |
| 675 |
movaps xmm6,XMMWORD[((-216))+rax] |
| 676 |
movaps xmm7,XMMWORD[((-200))+rax] |
| 677 |
movaps xmm8,XMMWORD[((-184))+rax] |
| 678 |
movaps xmm9,XMMWORD[((-168))+rax] |
| 679 |
movaps xmm10,XMMWORD[((-152))+rax] |
| 680 |
movaps xmm11,XMMWORD[((-136))+rax] |
| 681 |
movaps xmm12,XMMWORD[((-120))+rax] |
| 682 |
movaps xmm13,XMMWORD[((-104))+rax] |
| 683 |
movaps xmm14,XMMWORD[((-88))+rax] |
| 684 |
movaps xmm15,XMMWORD[((-72))+rax] |
| 685 |
mov r15,QWORD[((-48))+rax] |
| 686 |
|
| 687 |
mov r14,QWORD[((-40))+rax] |
| 688 |
|
| 689 |
mov r13,QWORD[((-32))+rax] |
| 690 |
|
| 691 |
mov r12,QWORD[((-24))+rax] |
| 692 |
|
| 693 |
mov rbp,QWORD[((-16))+rax] |
| 694 |
|
| 695 |
mov rbx,QWORD[((-8))+rax] |
| 696 |
|
| 697 |
lea rsp,[rax] |
| 698 |
|
| 699 |
$L$sqr_1024_epilogue: |
| 700 |
mov rdi,QWORD[8+rsp] ;WIN64 epilogue |
| 701 |
mov rsi,QWORD[16+rsp] |
| 702 |
DB 0F3h,0C3h ;repret |
| 703 |
|
| 704 |
$L$SEH_end_rsaz_1024_sqr_avx2: |
| 705 |
global rsaz_1024_mul_avx2 |
| 706 |
|
| 707 |
ALIGN 64 |
| 708 |
rsaz_1024_mul_avx2: |
| 709 |
mov QWORD[8+rsp],rdi ;WIN64 prologue |
| 710 |
mov QWORD[16+rsp],rsi |
| 711 |
mov rax,rsp |
| 712 |
$L$SEH_begin_rsaz_1024_mul_avx2: |
| 713 |
mov rdi,rcx |
| 714 |
mov rsi,rdx |
| 715 |
mov rdx,r8 |
| 716 |
mov rcx,r9 |
| 717 |
mov r8,QWORD[40+rsp] |
| 718 |
|
| 719 |
|
| 720 |
|
| 721 |
lea rax,[rsp] |
| 722 |
|
| 723 |
push rbx |
| 724 |
|
| 725 |
push rbp |
| 726 |
|
| 727 |
push r12 |
| 728 |
|
| 729 |
push r13 |
| 730 |
|
| 731 |
push r14 |
| 732 |
|
| 733 |
push r15 |
| 734 |
|
| 735 |
vzeroupper |
| 736 |
lea rsp,[((-168))+rsp] |
| 737 |
vmovaps XMMWORD[(-216)+rax],xmm6 |
| 738 |
vmovaps XMMWORD[(-200)+rax],xmm7 |
| 739 |
vmovaps XMMWORD[(-184)+rax],xmm8 |
| 740 |
vmovaps XMMWORD[(-168)+rax],xmm9 |
| 741 |
vmovaps XMMWORD[(-152)+rax],xmm10 |
| 742 |
vmovaps XMMWORD[(-136)+rax],xmm11 |
| 743 |
vmovaps XMMWORD[(-120)+rax],xmm12 |
| 744 |
vmovaps XMMWORD[(-104)+rax],xmm13 |
| 745 |
vmovaps XMMWORD[(-88)+rax],xmm14 |
| 746 |
vmovaps XMMWORD[(-72)+rax],xmm15 |
| 747 |
$L$mul_1024_body: |
| 748 |
mov rbp,rax |
| 749 |
|
| 750 |
vzeroall |
| 751 |
mov r13,rdx |
| 752 |
sub rsp,64 |
| 753 |
|
| 754 |
|
| 755 |
|
| 756 |
|
| 757 |
|
| 758 |
|
| 759 |
DB 0x67,0x67 |
| 760 |
mov r15,rsi |
| 761 |
and r15,4095 |
| 762 |
add r15,32*10 |
| 763 |
shr r15,12 |
| 764 |
mov r15,rsi |
| 765 |
cmovnz rsi,r13 |
| 766 |
cmovnz r13,r15 |
| 767 |
|
| 768 |
mov r15,rcx |
| 769 |
sub rsi,-128 |
| 770 |
sub rcx,-128 |
| 771 |
sub rdi,-128 |
| 772 |
|
| 773 |
and r15,4095 |
| 774 |
add r15,32*10 |
| 775 |
DB 0x67,0x67 |
| 776 |
shr r15,12 |
| 777 |
jz NEAR $L$mul_1024_no_n_copy |
| 778 |
|
| 779 |
|
| 780 |
|
| 781 |
|
| 782 |
|
| 783 |
sub rsp,32*10 |
| 784 |
vmovdqu ymm0,YMMWORD[((0-128))+rcx] |
| 785 |
and rsp,-512 |
| 786 |
vmovdqu ymm1,YMMWORD[((32-128))+rcx] |
| 787 |
vmovdqu ymm2,YMMWORD[((64-128))+rcx] |
| 788 |
vmovdqu ymm3,YMMWORD[((96-128))+rcx] |
| 789 |
vmovdqu ymm4,YMMWORD[((128-128))+rcx] |
| 790 |
vmovdqu ymm5,YMMWORD[((160-128))+rcx] |
| 791 |
vmovdqu ymm6,YMMWORD[((192-128))+rcx] |
| 792 |
vmovdqu ymm7,YMMWORD[((224-128))+rcx] |
| 793 |
vmovdqu ymm8,YMMWORD[((256-128))+rcx] |
| 794 |
lea rcx,[((64+128))+rsp] |
| 795 |
vmovdqu YMMWORD[(0-128)+rcx],ymm0 |
| 796 |
vpxor ymm0,ymm0,ymm0 |
| 797 |
vmovdqu YMMWORD[(32-128)+rcx],ymm1 |
| 798 |
vpxor ymm1,ymm1,ymm1 |
| 799 |
vmovdqu YMMWORD[(64-128)+rcx],ymm2 |
| 800 |
vpxor ymm2,ymm2,ymm2 |
| 801 |
vmovdqu YMMWORD[(96-128)+rcx],ymm3 |
| 802 |
vpxor ymm3,ymm3,ymm3 |
| 803 |
vmovdqu YMMWORD[(128-128)+rcx],ymm4 |
| 804 |
vpxor ymm4,ymm4,ymm4 |
| 805 |
vmovdqu YMMWORD[(160-128)+rcx],ymm5 |
| 806 |
vpxor ymm5,ymm5,ymm5 |
| 807 |
vmovdqu YMMWORD[(192-128)+rcx],ymm6 |
| 808 |
vpxor ymm6,ymm6,ymm6 |
| 809 |
vmovdqu YMMWORD[(224-128)+rcx],ymm7 |
| 810 |
vpxor ymm7,ymm7,ymm7 |
| 811 |
vmovdqu YMMWORD[(256-128)+rcx],ymm8 |
| 812 |
vmovdqa ymm8,ymm0 |
| 813 |
vmovdqu YMMWORD[(288-128)+rcx],ymm9 |
| 814 |
$L$mul_1024_no_n_copy: |
| 815 |
and rsp,-64 |
| 816 |
|
| 817 |
mov rbx,QWORD[r13] |
| 818 |
vpbroadcastq ymm10,QWORD[r13] |
| 819 |
vmovdqu YMMWORD[rsp],ymm0 |
| 820 |
xor r9,r9 |
| 821 |
DB 0x67 |
| 822 |
xor r10,r10 |
| 823 |
xor r11,r11 |
| 824 |
xor r12,r12 |
| 825 |
|
| 826 |
vmovdqu ymm15,YMMWORD[$L$and_mask] |
| 827 |
mov r14d,9 |
| 828 |
vmovdqu YMMWORD[(288-128)+rdi],ymm9 |
| 829 |
jmp NEAR $L$oop_mul_1024 |
| 830 |
|
| 831 |
ALIGN 32 |
| 832 |
$L$oop_mul_1024: |
| 833 |
vpsrlq ymm9,ymm3,29 |
| 834 |
mov rax,rbx |
| 835 |
imul rax,QWORD[((-128))+rsi] |
| 836 |
add rax,r9 |
| 837 |
mov r10,rbx |
| 838 |
imul r10,QWORD[((8-128))+rsi] |
| 839 |
add r10,QWORD[8+rsp] |
| 840 |
|
| 841 |
mov r9,rax |
| 842 |
imul eax,r8d |
| 843 |
and eax,0x1fffffff |
| 844 |
|
| 845 |
mov r11,rbx |
| 846 |
imul r11,QWORD[((16-128))+rsi] |
| 847 |
add r11,QWORD[16+rsp] |
| 848 |
|
| 849 |
mov r12,rbx |
| 850 |
imul r12,QWORD[((24-128))+rsi] |
| 851 |
add r12,QWORD[24+rsp] |
| 852 |
vpmuludq ymm0,ymm10,YMMWORD[((32-128))+rsi] |
| 853 |
vmovd xmm11,eax |
| 854 |
vpaddq ymm1,ymm1,ymm0 |
| 855 |
vpmuludq ymm12,ymm10,YMMWORD[((64-128))+rsi] |
| 856 |
vpbroadcastq ymm11,xmm11 |
| 857 |
vpaddq ymm2,ymm2,ymm12 |
| 858 |
vpmuludq ymm13,ymm10,YMMWORD[((96-128))+rsi] |
| 859 |
vpand ymm3,ymm3,ymm15 |
| 860 |
vpaddq ymm3,ymm3,ymm13 |
| 861 |
vpmuludq ymm0,ymm10,YMMWORD[((128-128))+rsi] |
| 862 |
vpaddq ymm4,ymm4,ymm0 |
| 863 |
vpmuludq ymm12,ymm10,YMMWORD[((160-128))+rsi] |
| 864 |
vpaddq ymm5,ymm5,ymm12 |
| 865 |
vpmuludq ymm13,ymm10,YMMWORD[((192-128))+rsi] |
| 866 |
vpaddq ymm6,ymm6,ymm13 |
| 867 |
vpmuludq ymm0,ymm10,YMMWORD[((224-128))+rsi] |
| 868 |
vpermq ymm9,ymm9,0x93 |
| 869 |
vpaddq ymm7,ymm7,ymm0 |
| 870 |
vpmuludq ymm12,ymm10,YMMWORD[((256-128))+rsi] |
| 871 |
vpbroadcastq ymm10,QWORD[8+r13] |
| 872 |
vpaddq ymm8,ymm8,ymm12 |
| 873 |
|
| 874 |
mov rdx,rax |
| 875 |
imul rax,QWORD[((-128))+rcx] |
| 876 |
add r9,rax |
| 877 |
mov rax,rdx |
| 878 |
imul rax,QWORD[((8-128))+rcx] |
| 879 |
add r10,rax |
| 880 |
mov rax,rdx |
| 881 |
imul rax,QWORD[((16-128))+rcx] |
| 882 |
add r11,rax |
| 883 |
shr r9,29 |
| 884 |
imul rdx,QWORD[((24-128))+rcx] |
| 885 |
add r12,rdx |
| 886 |
add r10,r9 |
| 887 |
|
| 888 |
vpmuludq ymm13,ymm11,YMMWORD[((32-128))+rcx] |
| 889 |
vmovq rbx,xmm10 |
| 890 |
vpaddq ymm1,ymm1,ymm13 |
| 891 |
vpmuludq ymm0,ymm11,YMMWORD[((64-128))+rcx] |
| 892 |
vpaddq ymm2,ymm2,ymm0 |
| 893 |
vpmuludq ymm12,ymm11,YMMWORD[((96-128))+rcx] |
| 894 |
vpaddq ymm3,ymm3,ymm12 |
| 895 |
vpmuludq ymm13,ymm11,YMMWORD[((128-128))+rcx] |
| 896 |
vpaddq ymm4,ymm4,ymm13 |
| 897 |
vpmuludq ymm0,ymm11,YMMWORD[((160-128))+rcx] |
| 898 |
vpaddq ymm5,ymm5,ymm0 |
| 899 |
vpmuludq ymm12,ymm11,YMMWORD[((192-128))+rcx] |
| 900 |
vpaddq ymm6,ymm6,ymm12 |
| 901 |
vpmuludq ymm13,ymm11,YMMWORD[((224-128))+rcx] |
| 902 |
vpblendd ymm12,ymm9,ymm14,3 |
| 903 |
vpaddq ymm7,ymm7,ymm13 |
| 904 |
vpmuludq ymm0,ymm11,YMMWORD[((256-128))+rcx] |
| 905 |
vpaddq ymm3,ymm3,ymm12 |
| 906 |
vpaddq ymm8,ymm8,ymm0 |
| 907 |
|
| 908 |
mov rax,rbx |
| 909 |
imul rax,QWORD[((-128))+rsi] |
| 910 |
add r10,rax |
| 911 |
vmovdqu ymm12,YMMWORD[((-8+32-128))+rsi] |
| 912 |
mov rax,rbx |
| 913 |
imul rax,QWORD[((8-128))+rsi] |
| 914 |
add r11,rax |
| 915 |
vmovdqu ymm13,YMMWORD[((-8+64-128))+rsi] |
| 916 |
|
| 917 |
mov rax,r10 |
| 918 |
vpblendd ymm9,ymm9,ymm14,0xfc |
| 919 |
imul eax,r8d |
| 920 |
vpaddq ymm4,ymm4,ymm9 |
| 921 |
and eax,0x1fffffff |
| 922 |
|
| 923 |
imul rbx,QWORD[((16-128))+rsi] |
| 924 |
add r12,rbx |
| 925 |
vpmuludq ymm12,ymm12,ymm10 |
| 926 |
vmovd xmm11,eax |
| 927 |
vmovdqu ymm0,YMMWORD[((-8+96-128))+rsi] |
| 928 |
vpaddq ymm1,ymm1,ymm12 |
| 929 |
vpmuludq ymm13,ymm13,ymm10 |
| 930 |
vpbroadcastq ymm11,xmm11 |
| 931 |
vmovdqu ymm12,YMMWORD[((-8+128-128))+rsi] |
| 932 |
vpaddq ymm2,ymm2,ymm13 |
| 933 |
vpmuludq ymm0,ymm0,ymm10 |
| 934 |
vmovdqu ymm13,YMMWORD[((-8+160-128))+rsi] |
| 935 |
vpaddq ymm3,ymm3,ymm0 |
| 936 |
vpmuludq ymm12,ymm12,ymm10 |
| 937 |
vmovdqu ymm0,YMMWORD[((-8+192-128))+rsi] |
| 938 |
vpaddq ymm4,ymm4,ymm12 |
| 939 |
vpmuludq ymm13,ymm13,ymm10 |
| 940 |
vmovdqu ymm12,YMMWORD[((-8+224-128))+rsi] |
| 941 |
vpaddq ymm5,ymm5,ymm13 |
| 942 |
vpmuludq ymm0,ymm0,ymm10 |
| 943 |
vmovdqu ymm13,YMMWORD[((-8+256-128))+rsi] |
| 944 |
vpaddq ymm6,ymm6,ymm0 |
| 945 |
vpmuludq ymm12,ymm12,ymm10 |
| 946 |
vmovdqu ymm9,YMMWORD[((-8+288-128))+rsi] |
| 947 |
vpaddq ymm7,ymm7,ymm12 |
| 948 |
vpmuludq ymm13,ymm13,ymm10 |
| 949 |
vpaddq ymm8,ymm8,ymm13 |
| 950 |
vpmuludq ymm9,ymm9,ymm10 |
| 951 |
vpbroadcastq ymm10,QWORD[16+r13] |
| 952 |
|
| 953 |
mov rdx,rax |
| 954 |
imul rax,QWORD[((-128))+rcx] |
| 955 |
add r10,rax |
| 956 |
vmovdqu ymm0,YMMWORD[((-8+32-128))+rcx] |
| 957 |
mov rax,rdx |
| 958 |
imul rax,QWORD[((8-128))+rcx] |
| 959 |
add r11,rax |
| 960 |
vmovdqu ymm12,YMMWORD[((-8+64-128))+rcx] |
| 961 |
shr r10,29 |
| 962 |
imul rdx,QWORD[((16-128))+rcx] |
| 963 |
add r12,rdx |
| 964 |
add r11,r10 |
| 965 |
|
| 966 |
vpmuludq ymm0,ymm0,ymm11 |
| 967 |
vmovq rbx,xmm10 |
| 968 |
vmovdqu ymm13,YMMWORD[((-8+96-128))+rcx] |
| 969 |
vpaddq ymm1,ymm1,ymm0 |
| 970 |
vpmuludq ymm12,ymm12,ymm11 |
| 971 |
vmovdqu ymm0,YMMWORD[((-8+128-128))+rcx] |
| 972 |
vpaddq ymm2,ymm2,ymm12 |
| 973 |
vpmuludq ymm13,ymm13,ymm11 |
| 974 |
vmovdqu ymm12,YMMWORD[((-8+160-128))+rcx] |
| 975 |
vpaddq ymm3,ymm3,ymm13 |
| 976 |
vpmuludq ymm0,ymm0,ymm11 |
| 977 |
vmovdqu ymm13,YMMWORD[((-8+192-128))+rcx] |
| 978 |
vpaddq ymm4,ymm4,ymm0 |
| 979 |
vpmuludq ymm12,ymm12,ymm11 |
| 980 |
vmovdqu ymm0,YMMWORD[((-8+224-128))+rcx] |
| 981 |
vpaddq ymm5,ymm5,ymm12 |
| 982 |
vpmuludq ymm13,ymm13,ymm11 |
| 983 |
vmovdqu ymm12,YMMWORD[((-8+256-128))+rcx] |
| 984 |
vpaddq ymm6,ymm6,ymm13 |
| 985 |
vpmuludq ymm0,ymm0,ymm11 |
| 986 |
vmovdqu ymm13,YMMWORD[((-8+288-128))+rcx] |
| 987 |
vpaddq ymm7,ymm7,ymm0 |
| 988 |
vpmuludq ymm12,ymm12,ymm11 |
| 989 |
vpaddq ymm8,ymm8,ymm12 |
| 990 |
vpmuludq ymm13,ymm13,ymm11 |
| 991 |
vpaddq ymm9,ymm9,ymm13 |
| 992 |
|
| 993 |
vmovdqu ymm0,YMMWORD[((-16+32-128))+rsi] |
| 994 |
mov rax,rbx |
| 995 |
imul rax,QWORD[((-128))+rsi] |
| 996 |
add rax,r11 |
| 997 |
|
| 998 |
vmovdqu ymm12,YMMWORD[((-16+64-128))+rsi] |
| 999 |
mov r11,rax |
| 1000 |
imul eax,r8d |
| 1001 |
and eax,0x1fffffff |
| 1002 |
|
| 1003 |
imul rbx,QWORD[((8-128))+rsi] |
| 1004 |
add r12,rbx |
| 1005 |
vpmuludq ymm0,ymm0,ymm10 |
| 1006 |
vmovd xmm11,eax |
| 1007 |
vmovdqu ymm13,YMMWORD[((-16+96-128))+rsi] |
| 1008 |
vpaddq ymm1,ymm1,ymm0 |
| 1009 |
vpmuludq ymm12,ymm12,ymm10 |
| 1010 |
vpbroadcastq ymm11,xmm11 |
| 1011 |
vmovdqu ymm0,YMMWORD[((-16+128-128))+rsi] |
| 1012 |
vpaddq ymm2,ymm2,ymm12 |
| 1013 |
vpmuludq ymm13,ymm13,ymm10 |
| 1014 |
vmovdqu ymm12,YMMWORD[((-16+160-128))+rsi] |
| 1015 |
vpaddq ymm3,ymm3,ymm13 |
| 1016 |
vpmuludq ymm0,ymm0,ymm10 |
| 1017 |
vmovdqu ymm13,YMMWORD[((-16+192-128))+rsi] |
| 1018 |
vpaddq ymm4,ymm4,ymm0 |
| 1019 |
vpmuludq ymm12,ymm12,ymm10 |
| 1020 |
vmovdqu ymm0,YMMWORD[((-16+224-128))+rsi] |
| 1021 |
vpaddq ymm5,ymm5,ymm12 |
| 1022 |
vpmuludq ymm13,ymm13,ymm10 |
| 1023 |
vmovdqu ymm12,YMMWORD[((-16+256-128))+rsi] |
| 1024 |
vpaddq ymm6,ymm6,ymm13 |
| 1025 |
vpmuludq ymm0,ymm0,ymm10 |
| 1026 |
vmovdqu ymm13,YMMWORD[((-16+288-128))+rsi] |
| 1027 |
vpaddq ymm7,ymm7,ymm0 |
| 1028 |
vpmuludq ymm12,ymm12,ymm10 |
| 1029 |
vpaddq ymm8,ymm8,ymm12 |
| 1030 |
vpmuludq ymm13,ymm13,ymm10 |
| 1031 |
vpbroadcastq ymm10,QWORD[24+r13] |
| 1032 |
vpaddq ymm9,ymm9,ymm13 |
| 1033 |
|
| 1034 |
vmovdqu ymm0,YMMWORD[((-16+32-128))+rcx] |
| 1035 |
mov rdx,rax |
| 1036 |
imul rax,QWORD[((-128))+rcx] |
| 1037 |
add r11,rax |
| 1038 |
vmovdqu ymm12,YMMWORD[((-16+64-128))+rcx] |
| 1039 |
imul rdx,QWORD[((8-128))+rcx] |
| 1040 |
add r12,rdx |
| 1041 |
shr r11,29 |
| 1042 |
|
| 1043 |
vpmuludq ymm0,ymm0,ymm11 |
| 1044 |
vmovq rbx,xmm10 |
| 1045 |
vmovdqu ymm13,YMMWORD[((-16+96-128))+rcx] |
| 1046 |
vpaddq ymm1,ymm1,ymm0 |
| 1047 |
vpmuludq ymm12,ymm12,ymm11 |
| 1048 |
vmovdqu ymm0,YMMWORD[((-16+128-128))+rcx] |
| 1049 |
vpaddq ymm2,ymm2,ymm12 |
| 1050 |
vpmuludq ymm13,ymm13,ymm11 |
| 1051 |
vmovdqu ymm12,YMMWORD[((-16+160-128))+rcx] |
| 1052 |
vpaddq ymm3,ymm3,ymm13 |
| 1053 |
vpmuludq ymm0,ymm0,ymm11 |
| 1054 |
vmovdqu ymm13,YMMWORD[((-16+192-128))+rcx] |
| 1055 |
vpaddq ymm4,ymm4,ymm0 |
| 1056 |
vpmuludq ymm12,ymm12,ymm11 |
| 1057 |
vmovdqu ymm0,YMMWORD[((-16+224-128))+rcx] |
| 1058 |
vpaddq ymm5,ymm5,ymm12 |
| 1059 |
vpmuludq ymm13,ymm13,ymm11 |
| 1060 |
vmovdqu ymm12,YMMWORD[((-16+256-128))+rcx] |
| 1061 |
vpaddq ymm6,ymm6,ymm13 |
| 1062 |
vpmuludq ymm0,ymm0,ymm11 |
| 1063 |
vmovdqu ymm13,YMMWORD[((-16+288-128))+rcx] |
| 1064 |
vpaddq ymm7,ymm7,ymm0 |
| 1065 |
vpmuludq ymm12,ymm12,ymm11 |
| 1066 |
vmovdqu ymm0,YMMWORD[((-24+32-128))+rsi] |
| 1067 |
vpaddq ymm8,ymm8,ymm12 |
| 1068 |
vpmuludq ymm13,ymm13,ymm11 |
| 1069 |
vmovdqu ymm12,YMMWORD[((-24+64-128))+rsi] |
| 1070 |
vpaddq ymm9,ymm9,ymm13 |
| 1071 |
|
| 1072 |
add r12,r11 |
| 1073 |
imul rbx,QWORD[((-128))+rsi] |
| 1074 |
add r12,rbx |
| 1075 |
|
| 1076 |
mov rax,r12 |
| 1077 |
imul eax,r8d |
| 1078 |
and eax,0x1fffffff |
| 1079 |
|
| 1080 |
vpmuludq ymm0,ymm0,ymm10 |
| 1081 |
vmovd xmm11,eax |
| 1082 |
vmovdqu ymm13,YMMWORD[((-24+96-128))+rsi] |
| 1083 |
vpaddq ymm1,ymm1,ymm0 |
| 1084 |
vpmuludq ymm12,ymm12,ymm10 |
| 1085 |
vpbroadcastq ymm11,xmm11 |
| 1086 |
vmovdqu ymm0,YMMWORD[((-24+128-128))+rsi] |
| 1087 |
vpaddq ymm2,ymm2,ymm12 |
| 1088 |
vpmuludq ymm13,ymm13,ymm10 |
| 1089 |
vmovdqu ymm12,YMMWORD[((-24+160-128))+rsi] |
| 1090 |
vpaddq ymm3,ymm3,ymm13 |
| 1091 |
vpmuludq ymm0,ymm0,ymm10 |
| 1092 |
vmovdqu ymm13,YMMWORD[((-24+192-128))+rsi] |
| 1093 |
vpaddq ymm4,ymm4,ymm0 |
| 1094 |
vpmuludq ymm12,ymm12,ymm10 |
| 1095 |
vmovdqu ymm0,YMMWORD[((-24+224-128))+rsi] |
| 1096 |
vpaddq ymm5,ymm5,ymm12 |
| 1097 |
vpmuludq ymm13,ymm13,ymm10 |
| 1098 |
vmovdqu ymm12,YMMWORD[((-24+256-128))+rsi] |
| 1099 |
vpaddq ymm6,ymm6,ymm13 |
| 1100 |
vpmuludq ymm0,ymm0,ymm10 |
| 1101 |
vmovdqu ymm13,YMMWORD[((-24+288-128))+rsi] |
| 1102 |
vpaddq ymm7,ymm7,ymm0 |
| 1103 |
vpmuludq ymm12,ymm12,ymm10 |
| 1104 |
vpaddq ymm8,ymm8,ymm12 |
| 1105 |
vpmuludq ymm13,ymm13,ymm10 |
| 1106 |
vpbroadcastq ymm10,QWORD[32+r13] |
| 1107 |
vpaddq ymm9,ymm9,ymm13 |
| 1108 |
add r13,32 |
| 1109 |
|
| 1110 |
vmovdqu ymm0,YMMWORD[((-24+32-128))+rcx] |
| 1111 |
imul rax,QWORD[((-128))+rcx] |
| 1112 |
add r12,rax |
| 1113 |
shr r12,29 |
| 1114 |
|
| 1115 |
vmovdqu ymm12,YMMWORD[((-24+64-128))+rcx] |
| 1116 |
vpmuludq ymm0,ymm0,ymm11 |
| 1117 |
vmovq rbx,xmm10 |
| 1118 |
vmovdqu ymm13,YMMWORD[((-24+96-128))+rcx] |
| 1119 |
vpaddq ymm0,ymm1,ymm0 |
| 1120 |
vpmuludq ymm12,ymm12,ymm11 |
| 1121 |
vmovdqu YMMWORD[rsp],ymm0 |
| 1122 |
vpaddq ymm1,ymm2,ymm12 |
| 1123 |
vmovdqu ymm0,YMMWORD[((-24+128-128))+rcx] |
| 1124 |
vpmuludq ymm13,ymm13,ymm11 |
| 1125 |
vmovdqu ymm12,YMMWORD[((-24+160-128))+rcx] |
| 1126 |
vpaddq ymm2,ymm3,ymm13 |
| 1127 |
vpmuludq ymm0,ymm0,ymm11 |
| 1128 |
vmovdqu ymm13,YMMWORD[((-24+192-128))+rcx] |
| 1129 |
vpaddq ymm3,ymm4,ymm0 |
| 1130 |
vpmuludq ymm12,ymm12,ymm11 |
| 1131 |
vmovdqu ymm0,YMMWORD[((-24+224-128))+rcx] |
| 1132 |
vpaddq ymm4,ymm5,ymm12 |
| 1133 |
vpmuludq ymm13,ymm13,ymm11 |
| 1134 |
vmovdqu ymm12,YMMWORD[((-24+256-128))+rcx] |
| 1135 |
vpaddq ymm5,ymm6,ymm13 |
| 1136 |
vpmuludq ymm0,ymm0,ymm11 |
| 1137 |
vmovdqu ymm13,YMMWORD[((-24+288-128))+rcx] |
| 1138 |
mov r9,r12 |
| 1139 |
vpaddq ymm6,ymm7,ymm0 |
| 1140 |
vpmuludq ymm12,ymm12,ymm11 |
| 1141 |
add r9,QWORD[rsp] |
| 1142 |
vpaddq ymm7,ymm8,ymm12 |
| 1143 |
vpmuludq ymm13,ymm13,ymm11 |
| 1144 |
vmovq xmm12,r12 |
| 1145 |
vpaddq ymm8,ymm9,ymm13 |
| 1146 |
|
| 1147 |
dec r14d |
| 1148 |
jnz NEAR $L$oop_mul_1024 |
| 1149 |
vpaddq ymm0,ymm12,YMMWORD[rsp] |
| 1150 |
|
| 1151 |
vpsrlq ymm12,ymm0,29 |
| 1152 |
vpand ymm0,ymm0,ymm15 |
| 1153 |
vpsrlq ymm13,ymm1,29 |
| 1154 |
vpand ymm1,ymm1,ymm15 |
| 1155 |
vpsrlq ymm10,ymm2,29 |
| 1156 |
vpermq ymm12,ymm12,0x93 |
| 1157 |
vpand ymm2,ymm2,ymm15 |
| 1158 |
vpsrlq ymm11,ymm3,29 |
| 1159 |
vpermq ymm13,ymm13,0x93 |
| 1160 |
vpand ymm3,ymm3,ymm15 |
| 1161 |
|
| 1162 |
vpblendd ymm9,ymm12,ymm14,3 |
| 1163 |
vpermq ymm10,ymm10,0x93 |
| 1164 |
vpblendd ymm12,ymm13,ymm12,3 |
| 1165 |
vpermq ymm11,ymm11,0x93 |
| 1166 |
vpaddq ymm0,ymm0,ymm9 |
| 1167 |
vpblendd ymm13,ymm10,ymm13,3 |
| 1168 |
vpaddq ymm1,ymm1,ymm12 |
| 1169 |
vpblendd ymm10,ymm11,ymm10,3 |
| 1170 |
vpaddq ymm2,ymm2,ymm13 |
| 1171 |
vpblendd ymm11,ymm14,ymm11,3 |
| 1172 |
vpaddq ymm3,ymm3,ymm10 |
| 1173 |
vpaddq ymm4,ymm4,ymm11 |
| 1174 |
|
| 1175 |
vpsrlq ymm12,ymm0,29 |
| 1176 |
vpand ymm0,ymm0,ymm15 |
| 1177 |
vpsrlq ymm13,ymm1,29 |
| 1178 |
vpand ymm1,ymm1,ymm15 |
| 1179 |
vpsrlq ymm10,ymm2,29 |
| 1180 |
vpermq ymm12,ymm12,0x93 |
| 1181 |
vpand ymm2,ymm2,ymm15 |
| 1182 |
vpsrlq ymm11,ymm3,29 |
| 1183 |
vpermq ymm13,ymm13,0x93 |
| 1184 |
vpand ymm3,ymm3,ymm15 |
| 1185 |
vpermq ymm10,ymm10,0x93 |
| 1186 |
|
| 1187 |
vpblendd ymm9,ymm12,ymm14,3 |
| 1188 |
vpermq ymm11,ymm11,0x93 |
| 1189 |
vpblendd ymm12,ymm13,ymm12,3 |
| 1190 |
vpaddq ymm0,ymm0,ymm9 |
| 1191 |
vpblendd ymm13,ymm10,ymm13,3 |
| 1192 |
vpaddq ymm1,ymm1,ymm12 |
| 1193 |
vpblendd ymm10,ymm11,ymm10,3 |
| 1194 |
vpaddq ymm2,ymm2,ymm13 |
| 1195 |
vpblendd ymm11,ymm14,ymm11,3 |
| 1196 |
vpaddq ymm3,ymm3,ymm10 |
| 1197 |
vpaddq ymm4,ymm4,ymm11 |
| 1198 |
|
| 1199 |
vmovdqu YMMWORD[(0-128)+rdi],ymm0 |
| 1200 |
vmovdqu YMMWORD[(32-128)+rdi],ymm1 |
| 1201 |
vmovdqu YMMWORD[(64-128)+rdi],ymm2 |
| 1202 |
vmovdqu YMMWORD[(96-128)+rdi],ymm3 |
| 1203 |
vpsrlq ymm12,ymm4,29 |
| 1204 |
vpand ymm4,ymm4,ymm15 |
| 1205 |
vpsrlq ymm13,ymm5,29 |
| 1206 |
vpand ymm5,ymm5,ymm15 |
| 1207 |
vpsrlq ymm10,ymm6,29 |
| 1208 |
vpermq ymm12,ymm12,0x93 |
| 1209 |
vpand ymm6,ymm6,ymm15 |
| 1210 |
vpsrlq ymm11,ymm7,29 |
| 1211 |
vpermq ymm13,ymm13,0x93 |
| 1212 |
vpand ymm7,ymm7,ymm15 |
| 1213 |
vpsrlq ymm0,ymm8,29 |
| 1214 |
vpermq ymm10,ymm10,0x93 |
| 1215 |
vpand ymm8,ymm8,ymm15 |
| 1216 |
vpermq ymm11,ymm11,0x93 |
| 1217 |
|
| 1218 |
vpblendd ymm9,ymm12,ymm14,3 |
| 1219 |
vpermq ymm0,ymm0,0x93 |
| 1220 |
vpblendd ymm12,ymm13,ymm12,3 |
| 1221 |
vpaddq ymm4,ymm4,ymm9 |
| 1222 |
vpblendd ymm13,ymm10,ymm13,3 |
| 1223 |
vpaddq ymm5,ymm5,ymm12 |
| 1224 |
vpblendd ymm10,ymm11,ymm10,3 |
| 1225 |
vpaddq ymm6,ymm6,ymm13 |
| 1226 |
vpblendd ymm11,ymm0,ymm11,3 |
| 1227 |
vpaddq ymm7,ymm7,ymm10 |
| 1228 |
vpaddq ymm8,ymm8,ymm11 |
| 1229 |
|
| 1230 |
vpsrlq ymm12,ymm4,29 |
| 1231 |
vpand ymm4,ymm4,ymm15 |
| 1232 |
vpsrlq ymm13,ymm5,29 |
| 1233 |
vpand ymm5,ymm5,ymm15 |
| 1234 |
vpsrlq ymm10,ymm6,29 |
| 1235 |
vpermq ymm12,ymm12,0x93 |
| 1236 |
vpand ymm6,ymm6,ymm15 |
| 1237 |
vpsrlq ymm11,ymm7,29 |
| 1238 |
vpermq ymm13,ymm13,0x93 |
| 1239 |
vpand ymm7,ymm7,ymm15 |
| 1240 |
vpsrlq ymm0,ymm8,29 |
| 1241 |
vpermq ymm10,ymm10,0x93 |
| 1242 |
vpand ymm8,ymm8,ymm15 |
| 1243 |
vpermq ymm11,ymm11,0x93 |
| 1244 |
|
| 1245 |
vpblendd ymm9,ymm12,ymm14,3 |
| 1246 |
vpermq ymm0,ymm0,0x93 |
| 1247 |
vpblendd ymm12,ymm13,ymm12,3 |
| 1248 |
vpaddq ymm4,ymm4,ymm9 |
| 1249 |
vpblendd ymm13,ymm10,ymm13,3 |
| 1250 |
vpaddq ymm5,ymm5,ymm12 |
| 1251 |
vpblendd ymm10,ymm11,ymm10,3 |
| 1252 |
vpaddq ymm6,ymm6,ymm13 |
| 1253 |
vpblendd ymm11,ymm0,ymm11,3 |
| 1254 |
vpaddq ymm7,ymm7,ymm10 |
| 1255 |
vpaddq ymm8,ymm8,ymm11 |
| 1256 |
|
| 1257 |
vmovdqu YMMWORD[(128-128)+rdi],ymm4 |
| 1258 |
vmovdqu YMMWORD[(160-128)+rdi],ymm5 |
| 1259 |
vmovdqu YMMWORD[(192-128)+rdi],ymm6 |
| 1260 |
vmovdqu YMMWORD[(224-128)+rdi],ymm7 |
| 1261 |
vmovdqu YMMWORD[(256-128)+rdi],ymm8 |
| 1262 |
vzeroupper |
| 1263 |
|
| 1264 |
mov rax,rbp |
| 1265 |
|
| 1266 |
$L$mul_1024_in_tail: |
| 1267 |
movaps xmm6,XMMWORD[((-216))+rax] |
| 1268 |
movaps xmm7,XMMWORD[((-200))+rax] |
| 1269 |
movaps xmm8,XMMWORD[((-184))+rax] |
| 1270 |
movaps xmm9,XMMWORD[((-168))+rax] |
| 1271 |
movaps xmm10,XMMWORD[((-152))+rax] |
| 1272 |
movaps xmm11,XMMWORD[((-136))+rax] |
| 1273 |
movaps xmm12,XMMWORD[((-120))+rax] |
| 1274 |
movaps xmm13,XMMWORD[((-104))+rax] |
| 1275 |
movaps xmm14,XMMWORD[((-88))+rax] |
| 1276 |
movaps xmm15,XMMWORD[((-72))+rax] |
| 1277 |
mov r15,QWORD[((-48))+rax] |
| 1278 |
|
| 1279 |
mov r14,QWORD[((-40))+rax] |
| 1280 |
|
| 1281 |
mov r13,QWORD[((-32))+rax] |
| 1282 |
|
| 1283 |
mov r12,QWORD[((-24))+rax] |
| 1284 |
|
| 1285 |
mov rbp,QWORD[((-16))+rax] |
| 1286 |
|
| 1287 |
mov rbx,QWORD[((-8))+rax] |
| 1288 |
|
| 1289 |
lea rsp,[rax] |
| 1290 |
|
| 1291 |
$L$mul_1024_epilogue: |
| 1292 |
mov rdi,QWORD[8+rsp] ;WIN64 epilogue |
| 1293 |
mov rsi,QWORD[16+rsp] |
| 1294 |
DB 0F3h,0C3h ;repret |
| 1295 |
|
| 1296 |
$L$SEH_end_rsaz_1024_mul_avx2: |
| 1297 |
global rsaz_1024_red2norm_avx2 |
| 1298 |
|
| 1299 |
ALIGN 32 |
| 1300 |
rsaz_1024_red2norm_avx2: |
| 1301 |
|
| 1302 |
sub rdx,-128 |
| 1303 |
xor rax,rax |
| 1304 |
mov r8,QWORD[((-128))+rdx] |
| 1305 |
mov r9,QWORD[((-120))+rdx] |
| 1306 |
mov r10,QWORD[((-112))+rdx] |
| 1307 |
shl r8,0 |
| 1308 |
shl r9,29 |
| 1309 |
mov r11,r10 |
| 1310 |
shl r10,58 |
| 1311 |
shr r11,6 |
| 1312 |
add rax,r8 |
| 1313 |
add rax,r9 |
| 1314 |
add rax,r10 |
| 1315 |
adc r11,0 |
| 1316 |
mov QWORD[rcx],rax |
| 1317 |
mov rax,r11 |
| 1318 |
mov r8,QWORD[((-104))+rdx] |
| 1319 |
mov r9,QWORD[((-96))+rdx] |
| 1320 |
shl r8,23 |
| 1321 |
mov r10,r9 |
| 1322 |
shl r9,52 |
| 1323 |
shr r10,12 |
| 1324 |
add rax,r8 |
| 1325 |
add rax,r9 |
| 1326 |
adc r10,0 |
| 1327 |
mov QWORD[8+rcx],rax |
| 1328 |
mov rax,r10 |
| 1329 |
mov r11,QWORD[((-88))+rdx] |
| 1330 |
mov r8,QWORD[((-80))+rdx] |
| 1331 |
shl r11,17 |
| 1332 |
mov r9,r8 |
| 1333 |
shl r8,46 |
| 1334 |
shr r9,18 |
| 1335 |
add rax,r11 |
| 1336 |
add rax,r8 |
| 1337 |
adc r9,0 |
| 1338 |
mov QWORD[16+rcx],rax |
| 1339 |
mov rax,r9 |
| 1340 |
mov r10,QWORD[((-72))+rdx] |
| 1341 |
mov r11,QWORD[((-64))+rdx] |
| 1342 |
shl r10,11 |
| 1343 |
mov r8,r11 |
| 1344 |
shl r11,40 |
| 1345 |
shr r8,24 |
| 1346 |
add rax,r10 |
| 1347 |
add rax,r11 |
| 1348 |
adc r8,0 |
| 1349 |
mov QWORD[24+rcx],rax |
| 1350 |
mov rax,r8 |
| 1351 |
mov r9,QWORD[((-56))+rdx] |
| 1352 |
mov r10,QWORD[((-48))+rdx] |
| 1353 |
mov r11,QWORD[((-40))+rdx] |
| 1354 |
shl r9,5 |
| 1355 |
shl r10,34 |
| 1356 |
mov r8,r11 |
| 1357 |
shl r11,63 |
| 1358 |
shr r8,1 |
| 1359 |
add rax,r9 |
| 1360 |
add rax,r10 |
| 1361 |
add rax,r11 |
| 1362 |
adc r8,0 |
| 1363 |
mov QWORD[32+rcx],rax |
| 1364 |
mov rax,r8 |
| 1365 |
mov r9,QWORD[((-32))+rdx] |
| 1366 |
mov r10,QWORD[((-24))+rdx] |
| 1367 |
shl r9,28 |
| 1368 |
mov r11,r10 |
| 1369 |
shl r10,57 |
| 1370 |
shr r11,7 |
| 1371 |
add rax,r9 |
| 1372 |
add rax,r10 |
| 1373 |
adc r11,0 |
| 1374 |
mov QWORD[40+rcx],rax |
| 1375 |
mov rax,r11 |
| 1376 |
mov r8,QWORD[((-16))+rdx] |
| 1377 |
mov r9,QWORD[((-8))+rdx] |
| 1378 |
shl r8,22 |
| 1379 |
mov r10,r9 |
| 1380 |
shl r9,51 |
| 1381 |
shr r10,13 |
| 1382 |
add rax,r8 |
| 1383 |
add rax,r9 |
| 1384 |
adc r10,0 |
| 1385 |
mov QWORD[48+rcx],rax |
| 1386 |
mov rax,r10 |
| 1387 |
mov r11,QWORD[rdx] |
| 1388 |
mov r8,QWORD[8+rdx] |
| 1389 |
shl r11,16 |
| 1390 |
mov r9,r8 |
| 1391 |
shl r8,45 |
| 1392 |
shr r9,19 |
| 1393 |
add rax,r11 |
| 1394 |
add rax,r8 |
| 1395 |
adc r9,0 |
| 1396 |
mov QWORD[56+rcx],rax |
| 1397 |
mov rax,r9 |
| 1398 |
mov r10,QWORD[16+rdx] |
| 1399 |
mov r11,QWORD[24+rdx] |
| 1400 |
shl r10,10 |
| 1401 |
mov r8,r11 |
| 1402 |
shl r11,39 |
| 1403 |
shr r8,25 |
| 1404 |
add rax,r10 |
| 1405 |
add rax,r11 |
| 1406 |
adc r8,0 |
| 1407 |
mov QWORD[64+rcx],rax |
| 1408 |
mov rax,r8 |
| 1409 |
mov r9,QWORD[32+rdx] |
| 1410 |
mov r10,QWORD[40+rdx] |
| 1411 |
mov r11,QWORD[48+rdx] |
| 1412 |
shl r9,4 |
| 1413 |
shl r10,33 |
| 1414 |
mov r8,r11 |
| 1415 |
shl r11,62 |
| 1416 |
shr r8,2 |
| 1417 |
add rax,r9 |
| 1418 |
add rax,r10 |
| 1419 |
add rax,r11 |
| 1420 |
adc r8,0 |
| 1421 |
mov QWORD[72+rcx],rax |
| 1422 |
mov rax,r8 |
| 1423 |
mov r9,QWORD[56+rdx] |
| 1424 |
mov r10,QWORD[64+rdx] |
| 1425 |
shl r9,27 |
| 1426 |
mov r11,r10 |
| 1427 |
shl r10,56 |
| 1428 |
shr r11,8 |
| 1429 |
add rax,r9 |
| 1430 |
add rax,r10 |
| 1431 |
adc r11,0 |
| 1432 |
mov QWORD[80+rcx],rax |
| 1433 |
mov rax,r11 |
| 1434 |
mov r8,QWORD[72+rdx] |
| 1435 |
mov r9,QWORD[80+rdx] |
| 1436 |
shl r8,21 |
| 1437 |
mov r10,r9 |
| 1438 |
shl r9,50 |
| 1439 |
shr r10,14 |
| 1440 |
add rax,r8 |
| 1441 |
add rax,r9 |
| 1442 |
adc r10,0 |
| 1443 |
mov QWORD[88+rcx],rax |
| 1444 |
mov rax,r10 |
| 1445 |
mov r11,QWORD[88+rdx] |
| 1446 |
mov r8,QWORD[96+rdx] |
| 1447 |
shl r11,15 |
| 1448 |
mov r9,r8 |
| 1449 |
shl r8,44 |
| 1450 |
shr r9,20 |
| 1451 |
add rax,r11 |
| 1452 |
add rax,r8 |
| 1453 |
adc r9,0 |
| 1454 |
mov QWORD[96+rcx],rax |
| 1455 |
mov rax,r9 |
| 1456 |
mov r10,QWORD[104+rdx] |
| 1457 |
mov r11,QWORD[112+rdx] |
| 1458 |
shl r10,9 |
| 1459 |
mov r8,r11 |
| 1460 |
shl r11,38 |
| 1461 |
shr r8,26 |
| 1462 |
add rax,r10 |
| 1463 |
add rax,r11 |
| 1464 |
adc r8,0 |
| 1465 |
mov QWORD[104+rcx],rax |
| 1466 |
mov rax,r8 |
| 1467 |
mov r9,QWORD[120+rdx] |
| 1468 |
mov r10,QWORD[128+rdx] |
| 1469 |
mov r11,QWORD[136+rdx] |
| 1470 |
shl r9,3 |
| 1471 |
shl r10,32 |
| 1472 |
mov r8,r11 |
| 1473 |
shl r11,61 |
| 1474 |
shr r8,3 |
| 1475 |
add rax,r9 |
| 1476 |
add rax,r10 |
| 1477 |
add rax,r11 |
| 1478 |
adc r8,0 |
| 1479 |
mov QWORD[112+rcx],rax |
| 1480 |
mov rax,r8 |
| 1481 |
mov r9,QWORD[144+rdx] |
| 1482 |
mov r10,QWORD[152+rdx] |
| 1483 |
shl r9,26 |
| 1484 |
mov r11,r10 |
| 1485 |
shl r10,55 |
| 1486 |
shr r11,9 |
| 1487 |
add rax,r9 |
| 1488 |
add rax,r10 |
| 1489 |
adc r11,0 |
| 1490 |
mov QWORD[120+rcx],rax |
| 1491 |
mov rax,r11 |
| 1492 |
DB 0F3h,0C3h ;repret |
| 1493 |
|
| 1494 |
|
| 1495 |
|
| 1496 |
global rsaz_1024_norm2red_avx2 |
| 1497 |
|
| 1498 |
ALIGN 32 |
| 1499 |
rsaz_1024_norm2red_avx2: |
| 1500 |
|
| 1501 |
sub rcx,-128 |
| 1502 |
mov r8,QWORD[rdx] |
| 1503 |
mov eax,0x1fffffff |
| 1504 |
mov r9,QWORD[8+rdx] |
| 1505 |
mov r11,r8 |
| 1506 |
shr r11,0 |
| 1507 |
and r11,rax |
| 1508 |
mov QWORD[((-128))+rcx],r11 |
| 1509 |
mov r10,r8 |
| 1510 |
shr r10,29 |
| 1511 |
and r10,rax |
| 1512 |
mov QWORD[((-120))+rcx],r10 |
| 1513 |
shrd r8,r9,58 |
| 1514 |
and r8,rax |
| 1515 |
mov QWORD[((-112))+rcx],r8 |
| 1516 |
mov r10,QWORD[16+rdx] |
| 1517 |
mov r8,r9 |
| 1518 |
shr r8,23 |
| 1519 |
and r8,rax |
| 1520 |
mov QWORD[((-104))+rcx],r8 |
| 1521 |
shrd r9,r10,52 |
| 1522 |
and r9,rax |
| 1523 |
mov QWORD[((-96))+rcx],r9 |
| 1524 |
mov r11,QWORD[24+rdx] |
| 1525 |
mov r9,r10 |
| 1526 |
shr r9,17 |
| 1527 |
and r9,rax |
| 1528 |
mov QWORD[((-88))+rcx],r9 |
| 1529 |
shrd r10,r11,46 |
| 1530 |
and r10,rax |
| 1531 |
mov QWORD[((-80))+rcx],r10 |
| 1532 |
mov r8,QWORD[32+rdx] |
| 1533 |
mov r10,r11 |
| 1534 |
shr r10,11 |
| 1535 |
and r10,rax |
| 1536 |
mov QWORD[((-72))+rcx],r10 |
| 1537 |
shrd r11,r8,40 |
| 1538 |
and r11,rax |
| 1539 |
mov QWORD[((-64))+rcx],r11 |
| 1540 |
mov r9,QWORD[40+rdx] |
| 1541 |
mov r11,r8 |
| 1542 |
shr r11,5 |
| 1543 |
and r11,rax |
| 1544 |
mov QWORD[((-56))+rcx],r11 |
| 1545 |
mov r10,r8 |
| 1546 |
shr r10,34 |
| 1547 |
and r10,rax |
| 1548 |
mov QWORD[((-48))+rcx],r10 |
| 1549 |
shrd r8,r9,63 |
| 1550 |
and r8,rax |
| 1551 |
mov QWORD[((-40))+rcx],r8 |
| 1552 |
mov r10,QWORD[48+rdx] |
| 1553 |
mov r8,r9 |
| 1554 |
shr r8,28 |
| 1555 |
and r8,rax |
| 1556 |
mov QWORD[((-32))+rcx],r8 |
| 1557 |
shrd r9,r10,57 |
| 1558 |
and r9,rax |
| 1559 |
mov QWORD[((-24))+rcx],r9 |
| 1560 |
mov r11,QWORD[56+rdx] |
| 1561 |
mov r9,r10 |
| 1562 |
shr r9,22 |
| 1563 |
and r9,rax |
| 1564 |
mov QWORD[((-16))+rcx],r9 |
| 1565 |
shrd r10,r11,51 |
| 1566 |
and r10,rax |
| 1567 |
mov QWORD[((-8))+rcx],r10 |
| 1568 |
mov r8,QWORD[64+rdx] |
| 1569 |
mov r10,r11 |
| 1570 |
shr r10,16 |
| 1571 |
and r10,rax |
| 1572 |
mov QWORD[rcx],r10 |
| 1573 |
shrd r11,r8,45 |
| 1574 |
and r11,rax |
| 1575 |
mov QWORD[8+rcx],r11 |
| 1576 |
mov r9,QWORD[72+rdx] |
| 1577 |
mov r11,r8 |
| 1578 |
shr r11,10 |
| 1579 |
and r11,rax |
| 1580 |
mov QWORD[16+rcx],r11 |
| 1581 |
shrd r8,r9,39 |
| 1582 |
and r8,rax |
| 1583 |
mov QWORD[24+rcx],r8 |
| 1584 |
mov r10,QWORD[80+rdx] |
| 1585 |
mov r8,r9 |
| 1586 |
shr r8,4 |
| 1587 |
and r8,rax |
| 1588 |
mov QWORD[32+rcx],r8 |
| 1589 |
mov r11,r9 |
| 1590 |
shr r11,33 |
| 1591 |
and r11,rax |
| 1592 |
mov QWORD[40+rcx],r11 |
| 1593 |
shrd r9,r10,62 |
| 1594 |
and r9,rax |
| 1595 |
mov QWORD[48+rcx],r9 |
| 1596 |
mov r11,QWORD[88+rdx] |
| 1597 |
mov r9,r10 |
| 1598 |
shr r9,27 |
| 1599 |
and r9,rax |
| 1600 |
mov QWORD[56+rcx],r9 |
| 1601 |
shrd r10,r11,56 |
| 1602 |
and r10,rax |
| 1603 |
mov QWORD[64+rcx],r10 |
| 1604 |
mov r8,QWORD[96+rdx] |
| 1605 |
mov r10,r11 |
| 1606 |
shr r10,21 |
| 1607 |
and r10,rax |
| 1608 |
mov QWORD[72+rcx],r10 |
| 1609 |
shrd r11,r8,50 |
| 1610 |
and r11,rax |
| 1611 |
mov QWORD[80+rcx],r11 |
| 1612 |
mov r9,QWORD[104+rdx] |
| 1613 |
mov r11,r8 |
| 1614 |
shr r11,15 |
| 1615 |
and r11,rax |
| 1616 |
mov QWORD[88+rcx],r11 |
| 1617 |
shrd r8,r9,44 |
| 1618 |
and r8,rax |
| 1619 |
mov QWORD[96+rcx],r8 |
| 1620 |
mov r10,QWORD[112+rdx] |
| 1621 |
mov r8,r9 |
| 1622 |
shr r8,9 |
| 1623 |
and r8,rax |
| 1624 |
mov QWORD[104+rcx],r8 |
| 1625 |
shrd r9,r10,38 |
| 1626 |
and r9,rax |
| 1627 |
mov QWORD[112+rcx],r9 |
| 1628 |
mov r11,QWORD[120+rdx] |
| 1629 |
mov r9,r10 |
| 1630 |
shr r9,3 |
| 1631 |
and r9,rax |
| 1632 |
mov QWORD[120+rcx],r9 |
| 1633 |
mov r8,r10 |
| 1634 |
shr r8,32 |
| 1635 |
and r8,rax |
| 1636 |
mov QWORD[128+rcx],r8 |
| 1637 |
shrd r10,r11,61 |
| 1638 |
and r10,rax |
| 1639 |
mov QWORD[136+rcx],r10 |
| 1640 |
xor r8,r8 |
| 1641 |
mov r10,r11 |
| 1642 |
shr r10,26 |
| 1643 |
and r10,rax |
| 1644 |
mov QWORD[144+rcx],r10 |
| 1645 |
shrd r11,r8,55 |
| 1646 |
and r11,rax |
| 1647 |
mov QWORD[152+rcx],r11 |
| 1648 |
mov QWORD[160+rcx],r8 |
| 1649 |
mov QWORD[168+rcx],r8 |
| 1650 |
mov QWORD[176+rcx],r8 |
| 1651 |
mov QWORD[184+rcx],r8 |
| 1652 |
DB 0F3h,0C3h ;repret |
| 1653 |
|
| 1654 |
|
| 1655 |
global rsaz_1024_scatter5_avx2 |
| 1656 |
|
| 1657 |
ALIGN 32 |
| 1658 |
rsaz_1024_scatter5_avx2: |
| 1659 |
|
| 1660 |
vzeroupper |
| 1661 |
vmovdqu ymm5,YMMWORD[$L$scatter_permd] |
| 1662 |
shl r8d,4 |
| 1663 |
lea rcx,[r8*1+rcx] |
| 1664 |
mov eax,9 |
| 1665 |
jmp NEAR $L$oop_scatter_1024 |
| 1666 |
|
| 1667 |
ALIGN 32 |
| 1668 |
$L$oop_scatter_1024: |
| 1669 |
vmovdqu ymm0,YMMWORD[rdx] |
| 1670 |
lea rdx,[32+rdx] |
| 1671 |
vpermd ymm0,ymm5,ymm0 |
| 1672 |
vmovdqu XMMWORD[rcx],xmm0 |
| 1673 |
lea rcx,[512+rcx] |
| 1674 |
dec eax |
| 1675 |
jnz NEAR $L$oop_scatter_1024 |
| 1676 |
|
| 1677 |
vzeroupper |
| 1678 |
DB 0F3h,0C3h ;repret |
| 1679 |
|
| 1680 |
|
| 1681 |
|
| 1682 |
global rsaz_1024_gather5_avx2 |
| 1683 |
|
| 1684 |
ALIGN 32 |
| 1685 |
rsaz_1024_gather5_avx2: |
| 1686 |
|
| 1687 |
vzeroupper |
| 1688 |
mov r11,rsp |
| 1689 |
|
| 1690 |
lea rax,[((-136))+rsp] |
| 1691 |
$L$SEH_begin_rsaz_1024_gather5: |
| 1692 |
|
| 1693 |
DB 0x48,0x8d,0x60,0xe0 |
| 1694 |
DB 0xc5,0xf8,0x29,0x70,0xe0 |
| 1695 |
DB 0xc5,0xf8,0x29,0x78,0xf0 |
| 1696 |
DB 0xc5,0x78,0x29,0x40,0x00 |
| 1697 |
DB 0xc5,0x78,0x29,0x48,0x10 |
| 1698 |
DB 0xc5,0x78,0x29,0x50,0x20 |
| 1699 |
DB 0xc5,0x78,0x29,0x58,0x30 |
| 1700 |
DB 0xc5,0x78,0x29,0x60,0x40 |
| 1701 |
DB 0xc5,0x78,0x29,0x68,0x50 |
| 1702 |
DB 0xc5,0x78,0x29,0x70,0x60 |
| 1703 |
DB 0xc5,0x78,0x29,0x78,0x70 |
| 1704 |
lea rsp,[((-256))+rsp] |
| 1705 |
and rsp,-32 |
| 1706 |
lea r10,[$L$inc] |
| 1707 |
lea rax,[((-128))+rsp] |
| 1708 |
|
| 1709 |
vmovd xmm4,r8d |
| 1710 |
vmovdqa ymm0,YMMWORD[r10] |
| 1711 |
vmovdqa ymm1,YMMWORD[32+r10] |
| 1712 |
vmovdqa ymm5,YMMWORD[64+r10] |
| 1713 |
vpbroadcastd ymm4,xmm4 |
| 1714 |
|
| 1715 |
vpaddd ymm2,ymm0,ymm5 |
| 1716 |
vpcmpeqd ymm0,ymm0,ymm4 |
| 1717 |
vpaddd ymm3,ymm1,ymm5 |
| 1718 |
vpcmpeqd ymm1,ymm1,ymm4 |
| 1719 |
vmovdqa YMMWORD[(0+128)+rax],ymm0 |
| 1720 |
vpaddd ymm0,ymm2,ymm5 |
| 1721 |
vpcmpeqd ymm2,ymm2,ymm4 |
| 1722 |
vmovdqa YMMWORD[(32+128)+rax],ymm1 |
| 1723 |
vpaddd ymm1,ymm3,ymm5 |
| 1724 |
vpcmpeqd ymm3,ymm3,ymm4 |
| 1725 |
vmovdqa YMMWORD[(64+128)+rax],ymm2 |
| 1726 |
vpaddd ymm2,ymm0,ymm5 |
| 1727 |
vpcmpeqd ymm0,ymm0,ymm4 |
| 1728 |
vmovdqa YMMWORD[(96+128)+rax],ymm3 |
| 1729 |
vpaddd ymm3,ymm1,ymm5 |
| 1730 |
vpcmpeqd ymm1,ymm1,ymm4 |
| 1731 |
vmovdqa YMMWORD[(128+128)+rax],ymm0 |
| 1732 |
vpaddd ymm8,ymm2,ymm5 |
| 1733 |
vpcmpeqd ymm2,ymm2,ymm4 |
| 1734 |
vmovdqa YMMWORD[(160+128)+rax],ymm1 |
| 1735 |
vpaddd ymm9,ymm3,ymm5 |
| 1736 |
vpcmpeqd ymm3,ymm3,ymm4 |
| 1737 |
vmovdqa YMMWORD[(192+128)+rax],ymm2 |
| 1738 |
vpaddd ymm10,ymm8,ymm5 |
| 1739 |
vpcmpeqd ymm8,ymm8,ymm4 |
| 1740 |
vmovdqa YMMWORD[(224+128)+rax],ymm3 |
| 1741 |
vpaddd ymm11,ymm9,ymm5 |
| 1742 |
vpcmpeqd ymm9,ymm9,ymm4 |
| 1743 |
vpaddd ymm12,ymm10,ymm5 |
| 1744 |
vpcmpeqd ymm10,ymm10,ymm4 |
| 1745 |
vpaddd ymm13,ymm11,ymm5 |
| 1746 |
vpcmpeqd ymm11,ymm11,ymm4 |
| 1747 |
vpaddd ymm14,ymm12,ymm5 |
| 1748 |
vpcmpeqd ymm12,ymm12,ymm4 |
| 1749 |
vpaddd ymm15,ymm13,ymm5 |
| 1750 |
vpcmpeqd ymm13,ymm13,ymm4 |
| 1751 |
vpcmpeqd ymm14,ymm14,ymm4 |
| 1752 |
vpcmpeqd ymm15,ymm15,ymm4 |
| 1753 |
|
| 1754 |
vmovdqa ymm7,YMMWORD[((-32))+r10] |
| 1755 |
lea rdx,[128+rdx] |
| 1756 |
mov r8d,9 |
| 1757 |
|
| 1758 |
$L$oop_gather_1024: |
| 1759 |
vmovdqa ymm0,YMMWORD[((0-128))+rdx] |
| 1760 |
vmovdqa ymm1,YMMWORD[((32-128))+rdx] |
| 1761 |
vmovdqa ymm2,YMMWORD[((64-128))+rdx] |
| 1762 |
vmovdqa ymm3,YMMWORD[((96-128))+rdx] |
| 1763 |
vpand ymm0,ymm0,YMMWORD[((0+128))+rax] |
| 1764 |
vpand ymm1,ymm1,YMMWORD[((32+128))+rax] |
| 1765 |
vpand ymm2,ymm2,YMMWORD[((64+128))+rax] |
| 1766 |
vpor ymm4,ymm1,ymm0 |
| 1767 |
vpand ymm3,ymm3,YMMWORD[((96+128))+rax] |
| 1768 |
vmovdqa ymm0,YMMWORD[((128-128))+rdx] |
| 1769 |
vmovdqa ymm1,YMMWORD[((160-128))+rdx] |
| 1770 |
vpor ymm5,ymm3,ymm2 |
| 1771 |
vmovdqa ymm2,YMMWORD[((192-128))+rdx] |
| 1772 |
vmovdqa ymm3,YMMWORD[((224-128))+rdx] |
| 1773 |
vpand ymm0,ymm0,YMMWORD[((128+128))+rax] |
| 1774 |
vpand ymm1,ymm1,YMMWORD[((160+128))+rax] |
| 1775 |
vpand ymm2,ymm2,YMMWORD[((192+128))+rax] |
| 1776 |
vpor ymm4,ymm4,ymm0 |
| 1777 |
vpand ymm3,ymm3,YMMWORD[((224+128))+rax] |
| 1778 |
vpand ymm0,ymm8,YMMWORD[((256-128))+rdx] |
| 1779 |
vpor ymm5,ymm5,ymm1 |
| 1780 |
vpand ymm1,ymm9,YMMWORD[((288-128))+rdx] |
| 1781 |
vpor ymm4,ymm4,ymm2 |
| 1782 |
vpand ymm2,ymm10,YMMWORD[((320-128))+rdx] |
| 1783 |
vpor ymm5,ymm5,ymm3 |
| 1784 |
vpand ymm3,ymm11,YMMWORD[((352-128))+rdx] |
| 1785 |
vpor ymm4,ymm4,ymm0 |
| 1786 |
vpand ymm0,ymm12,YMMWORD[((384-128))+rdx] |
| 1787 |
vpor ymm5,ymm5,ymm1 |
| 1788 |
vpand ymm1,ymm13,YMMWORD[((416-128))+rdx] |
| 1789 |
vpor ymm4,ymm4,ymm2 |
| 1790 |
vpand ymm2,ymm14,YMMWORD[((448-128))+rdx] |
| 1791 |
vpor ymm5,ymm5,ymm3 |
| 1792 |
vpand ymm3,ymm15,YMMWORD[((480-128))+rdx] |
| 1793 |
lea rdx,[512+rdx] |
| 1794 |
vpor ymm4,ymm4,ymm0 |
| 1795 |
vpor ymm5,ymm5,ymm1 |
| 1796 |
vpor ymm4,ymm4,ymm2 |
| 1797 |
vpor ymm5,ymm5,ymm3 |
| 1798 |
|
| 1799 |
vpor ymm4,ymm4,ymm5 |
| 1800 |
vextracti128 xmm5,ymm4,1 |
| 1801 |
vpor xmm5,xmm5,xmm4 |
| 1802 |
vpermd ymm5,ymm7,ymm5 |
| 1803 |
vmovdqu YMMWORD[rcx],ymm5 |
| 1804 |
lea rcx,[32+rcx] |
| 1805 |
dec r8d |
| 1806 |
jnz NEAR $L$oop_gather_1024 |
| 1807 |
|
| 1808 |
vpxor ymm0,ymm0,ymm0 |
| 1809 |
vmovdqu YMMWORD[rcx],ymm0 |
| 1810 |
vzeroupper |
| 1811 |
movaps xmm6,XMMWORD[((-168))+r11] |
| 1812 |
movaps xmm7,XMMWORD[((-152))+r11] |
| 1813 |
movaps xmm8,XMMWORD[((-136))+r11] |
| 1814 |
movaps xmm9,XMMWORD[((-120))+r11] |
| 1815 |
movaps xmm10,XMMWORD[((-104))+r11] |
| 1816 |
movaps xmm11,XMMWORD[((-88))+r11] |
| 1817 |
movaps xmm12,XMMWORD[((-72))+r11] |
| 1818 |
movaps xmm13,XMMWORD[((-56))+r11] |
| 1819 |
movaps xmm14,XMMWORD[((-40))+r11] |
| 1820 |
movaps xmm15,XMMWORD[((-24))+r11] |
| 1821 |
lea rsp,[r11] |
| 1822 |
|
| 1823 |
DB 0F3h,0C3h ;repret |
| 1824 |
|
| 1825 |
$L$SEH_end_rsaz_1024_gather5: |
| 1826 |
|
| 1827 |
section .rdata rdata align=8 |
| 1828 |
ALIGN 64 |
| 1829 |
$L$and_mask: |
| 1830 |
DQ 0x1fffffff,0x1fffffff,0x1fffffff,0x1fffffff |
| 1831 |
$L$scatter_permd: |
| 1832 |
DD 0,2,4,6,7,7,7,7 |
| 1833 |
$L$gather_permd: |
| 1834 |
DD 0,7,1,7,2,7,3,7 |
| 1835 |
$L$inc: |
| 1836 |
DD 0,0,0,0,1,1,1,1 |
| 1837 |
DD 2,2,2,2,3,3,3,3 |
| 1838 |
DD 4,4,4,4,4,4,4,4 |
| 1839 |
ALIGN 64 |
| 1840 |
section .text |
| 1841 |
|
| 1842 |
EXTERN __imp_RtlVirtualUnwind |
| 1843 |
|
| 1844 |
ALIGN 16 |
| 1845 |
rsaz_se_handler: |
| 1846 |
push rsi |
| 1847 |
push rdi |
| 1848 |
push rbx |
| 1849 |
push rbp |
| 1850 |
push r12 |
| 1851 |
push r13 |
| 1852 |
push r14 |
| 1853 |
push r15 |
| 1854 |
pushfq |
| 1855 |
sub rsp,64 |
| 1856 |
|
| 1857 |
mov rax,QWORD[120+r8] |
| 1858 |
mov rbx,QWORD[248+r8] |
| 1859 |
|
| 1860 |
mov rsi,QWORD[8+r9] |
| 1861 |
mov r11,QWORD[56+r9] |
| 1862 |
|
| 1863 |
mov r10d,DWORD[r11] |
| 1864 |
lea r10,[r10*1+rsi] |
| 1865 |
cmp rbx,r10 |
| 1866 |
jb NEAR $L$common_seh_tail |
| 1867 |
|
| 1868 |
mov r10d,DWORD[4+r11] |
| 1869 |
lea r10,[r10*1+rsi] |
| 1870 |
cmp rbx,r10 |
| 1871 |
jae NEAR $L$common_seh_tail |
| 1872 |
|
| 1873 |
mov rbp,QWORD[160+r8] |
| 1874 |
|
| 1875 |
mov r10d,DWORD[8+r11] |
| 1876 |
lea r10,[r10*1+rsi] |
| 1877 |
cmp rbx,r10 |
| 1878 |
cmovc rax,rbp |
| 1879 |
|
| 1880 |
mov r15,QWORD[((-48))+rax] |
| 1881 |
mov r14,QWORD[((-40))+rax] |
| 1882 |
mov r13,QWORD[((-32))+rax] |
| 1883 |
mov r12,QWORD[((-24))+rax] |
| 1884 |
mov rbp,QWORD[((-16))+rax] |
| 1885 |
mov rbx,QWORD[((-8))+rax] |
| 1886 |
mov QWORD[240+r8],r15 |
| 1887 |
mov QWORD[232+r8],r14 |
| 1888 |
mov QWORD[224+r8],r13 |
| 1889 |
mov QWORD[216+r8],r12 |
| 1890 |
mov QWORD[160+r8],rbp |
| 1891 |
mov QWORD[144+r8],rbx |
| 1892 |
|
| 1893 |
lea rsi,[((-216))+rax] |
| 1894 |
lea rdi,[512+r8] |
| 1895 |
mov ecx,20 |
| 1896 |
DD 0xa548f3fc |
| 1897 |
|
| 1898 |
$L$common_seh_tail: |
| 1899 |
mov rdi,QWORD[8+rax] |
| 1900 |
mov rsi,QWORD[16+rax] |
| 1901 |
mov QWORD[152+r8],rax |
| 1902 |
mov QWORD[168+r8],rsi |
| 1903 |
mov QWORD[176+r8],rdi |
| 1904 |
|
| 1905 |
mov rdi,QWORD[40+r9] |
| 1906 |
mov rsi,r8 |
| 1907 |
mov ecx,154 |
| 1908 |
DD 0xa548f3fc |
| 1909 |
|
| 1910 |
mov rsi,r9 |
| 1911 |
xor rcx,rcx |
| 1912 |
mov rdx,QWORD[8+rsi] |
| 1913 |
mov r8,QWORD[rsi] |
| 1914 |
mov r9,QWORD[16+rsi] |
| 1915 |
mov r10,QWORD[40+rsi] |
| 1916 |
lea r11,[56+rsi] |
| 1917 |
lea r12,[24+rsi] |
| 1918 |
mov QWORD[32+rsp],r10 |
| 1919 |
mov QWORD[40+rsp],r11 |
| 1920 |
mov QWORD[48+rsp],r12 |
| 1921 |
mov QWORD[56+rsp],rcx |
| 1922 |
call QWORD[__imp_RtlVirtualUnwind] |
| 1923 |
|
| 1924 |
mov eax,1 |
| 1925 |
add rsp,64 |
| 1926 |
popfq |
| 1927 |
pop r15 |
| 1928 |
pop r14 |
| 1929 |
pop r13 |
| 1930 |
pop r12 |
| 1931 |
pop rbp |
| 1932 |
pop rbx |
| 1933 |
pop rdi |
| 1934 |
pop rsi |
| 1935 |
DB 0F3h,0C3h ;repret |
| 1936 |
|
| 1937 |
|
| 1938 |
section .pdata rdata align=4 |
| 1939 |
ALIGN 4 |
| 1940 |
DD $L$SEH_begin_rsaz_1024_sqr_avx2 wrt ..imagebase |
| 1941 |
DD $L$SEH_end_rsaz_1024_sqr_avx2 wrt ..imagebase |
| 1942 |
DD $L$SEH_info_rsaz_1024_sqr_avx2 wrt ..imagebase |
| 1943 |
|
| 1944 |
DD $L$SEH_begin_rsaz_1024_mul_avx2 wrt ..imagebase |
| 1945 |
DD $L$SEH_end_rsaz_1024_mul_avx2 wrt ..imagebase |
| 1946 |
DD $L$SEH_info_rsaz_1024_mul_avx2 wrt ..imagebase |
| 1947 |
|
| 1948 |
DD $L$SEH_begin_rsaz_1024_gather5 wrt ..imagebase |
| 1949 |
DD $L$SEH_end_rsaz_1024_gather5 wrt ..imagebase |
| 1950 |
DD $L$SEH_info_rsaz_1024_gather5 wrt ..imagebase |
| 1951 |
section .xdata rdata align=8 |
| 1952 |
ALIGN 8 |
| 1953 |
$L$SEH_info_rsaz_1024_sqr_avx2: |
| 1954 |
DB 9,0,0,0 |
| 1955 |
DD rsaz_se_handler wrt ..imagebase |
| 1956 |
DD $L$sqr_1024_body wrt ..imagebase,$L$sqr_1024_epilogue wrt ..imagebase,$L$sqr_1024_in_tail wrt ..imagebase |
| 1957 |
DD 0 |
| 1958 |
$L$SEH_info_rsaz_1024_mul_avx2: |
| 1959 |
DB 9,0,0,0 |
| 1960 |
DD rsaz_se_handler wrt ..imagebase |
| 1961 |
DD $L$mul_1024_body wrt ..imagebase,$L$mul_1024_epilogue wrt ..imagebase,$L$mul_1024_in_tail wrt ..imagebase |
| 1962 |
DD 0 |
| 1963 |
$L$SEH_info_rsaz_1024_gather5: |
| 1964 |
DB 0x01,0x36,0x17,0x0b |
| 1965 |
DB 0x36,0xf8,0x09,0x00 |
| 1966 |
DB 0x31,0xe8,0x08,0x00 |
| 1967 |
DB 0x2c,0xd8,0x07,0x00 |
| 1968 |
DB 0x27,0xc8,0x06,0x00 |
| 1969 |
DB 0x22,0xb8,0x05,0x00 |
| 1970 |
DB 0x1d,0xa8,0x04,0x00 |
| 1971 |
DB 0x18,0x98,0x03,0x00 |
| 1972 |
DB 0x13,0x88,0x02,0x00 |
| 1973 |
DB 0x0e,0x78,0x01,0x00 |
| 1974 |
DB 0x09,0x68,0x00,0x00 |
| 1975 |
DB 0x04,0x01,0x15,0x00 |
| 1976 |
DB 0x00,0xb3,0x00,0x00 |
| 1977 |
%else |
| 1978 |
; Work around https://bugzilla.nasm.us/show_bug.cgi?id=3392738 |
| 1979 |
ret |
| 1980 |
%endif |