sha256_sse4.cpp raw
1 // Copyright (c) 2017-2022 The Limenka developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 //
5 // This is a translation to GCC extended asm syntax from YASM code by Intel
6 // (available at the bottom of this file).
7
8 #include <cstdlib>
9 #include <stdint.h>
10
11 #if defined(__x86_64__) || defined(__amd64__)
12
13 namespace sha256_sse4
14 {
15 void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks)
16 #if defined(__clang__)
17 /*
18 clang is unable to compile this with -O0 and -fsanitize=address.
19 See upstream bug: https://github.com/llvm/llvm-project/issues/92182.
20 This also fails to compile with -O2, -fcf-protection & -fsanitize=address.
21 See https://github.com/limenka/limenka/issues/31913.
22 */
23 #if __has_feature(address_sanitizer)
24 __attribute__((no_sanitize("address")))
25 #endif
26 #endif
27 {
28 static const uint32_t K256 alignas(16) [] = {
29 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
30 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
31 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
32 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
33 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
34 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
35 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
36 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
37 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
38 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
39 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
40 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
41 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
42 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
43 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
44 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
45 };
46 static const uint32_t FLIP_MASK alignas(16) [] = {0x00010203, 0x04050607, 0x08090a0b, 0x0c0d0e0f};
47 static const uint32_t SHUF_00BA alignas(16) [] = {0x03020100, 0x0b0a0908, 0xffffffff, 0xffffffff};
48 static const uint32_t SHUF_DC00 alignas(16) [] = {0xffffffff, 0xffffffff, 0x03020100, 0x0b0a0908};
49 uint32_t a, b, c, d, f, g, h, y0, y1, y2;
50 uint64_t tbl;
51 uint64_t inp_end, inp;
52 uint32_t xfer alignas(16) [4];
53
54 __asm__ __volatile__(
55 "shl $0x6,%2;"
56 "je Ldone_hash_%=;"
57 "add %1,%2;"
58 "mov %2,%14;"
59 "mov (%0),%3;"
60 "mov 0x4(%0),%4;"
61 "mov 0x8(%0),%5;"
62 "mov 0xc(%0),%6;"
63 "mov 0x10(%0),%k2;"
64 "mov 0x14(%0),%7;"
65 "mov 0x18(%0),%8;"
66 "mov 0x1c(%0),%9;"
67 "movdqa %18,%%xmm12;"
68 "movdqa %19,%%xmm10;"
69 "movdqa %20,%%xmm11;"
70
71 "Lloop0_%=:"
72 "lea %17,%13;"
73 "movdqu (%1),%%xmm4;"
74 "pshufb %%xmm12,%%xmm4;"
75 "movdqu 0x10(%1),%%xmm5;"
76 "pshufb %%xmm12,%%xmm5;"
77 "movdqu 0x20(%1),%%xmm6;"
78 "pshufb %%xmm12,%%xmm6;"
79 "movdqu 0x30(%1),%%xmm7;"
80 "pshufb %%xmm12,%%xmm7;"
81 "mov %1,%15;"
82 "mov $3,%1;"
83
84 "Lloop1_%=:"
85 "movdqa 0x0(%13),%%xmm9;"
86 "paddd %%xmm4,%%xmm9;"
87 "movdqa %%xmm9,%16;"
88 "movdqa %%xmm7,%%xmm0;"
89 "mov %k2,%10;"
90 "ror $0xe,%10;"
91 "mov %3,%11;"
92 "palignr $0x4,%%xmm6,%%xmm0;"
93 "ror $0x9,%11;"
94 "xor %k2,%10;"
95 "mov %7,%12;"
96 "ror $0x5,%10;"
97 "movdqa %%xmm5,%%xmm1;"
98 "xor %3,%11;"
99 "xor %8,%12;"
100 "paddd %%xmm4,%%xmm0;"
101 "xor %k2,%10;"
102 "and %k2,%12;"
103 "ror $0xb,%11;"
104 "palignr $0x4,%%xmm4,%%xmm1;"
105 "xor %3,%11;"
106 "ror $0x6,%10;"
107 "xor %8,%12;"
108 "movdqa %%xmm1,%%xmm2;"
109 "ror $0x2,%11;"
110 "add %10,%12;"
111 "add %16,%12;"
112 "movdqa %%xmm1,%%xmm3;"
113 "mov %3,%10;"
114 "add %12,%9;"
115 "mov %3,%12;"
116 "pslld $0x19,%%xmm1;"
117 "or %5,%10;"
118 "add %9,%6;"
119 "and %5,%12;"
120 "psrld $0x7,%%xmm2;"
121 "and %4,%10;"
122 "add %11,%9;"
123 "por %%xmm2,%%xmm1;"
124 "or %12,%10;"
125 "add %10,%9;"
126 "movdqa %%xmm3,%%xmm2;"
127 "mov %6,%10;"
128 "mov %9,%11;"
129 "movdqa %%xmm3,%%xmm8;"
130 "ror $0xe,%10;"
131 "xor %6,%10;"
132 "mov %k2,%12;"
133 "ror $0x9,%11;"
134 "pslld $0xe,%%xmm3;"
135 "xor %9,%11;"
136 "ror $0x5,%10;"
137 "xor %7,%12;"
138 "psrld $0x12,%%xmm2;"
139 "ror $0xb,%11;"
140 "xor %6,%10;"
141 "and %6,%12;"
142 "ror $0x6,%10;"
143 "pxor %%xmm3,%%xmm1;"
144 "xor %9,%11;"
145 "xor %7,%12;"
146 "psrld $0x3,%%xmm8;"
147 "add %10,%12;"
148 "add 4+%16,%12;"
149 "ror $0x2,%11;"
150 "pxor %%xmm2,%%xmm1;"
151 "mov %9,%10;"
152 "add %12,%8;"
153 "mov %9,%12;"
154 "pxor %%xmm8,%%xmm1;"
155 "or %4,%10;"
156 "add %8,%5;"
157 "and %4,%12;"
158 "pshufd $0xfa,%%xmm7,%%xmm2;"
159 "and %3,%10;"
160 "add %11,%8;"
161 "paddd %%xmm1,%%xmm0;"
162 "or %12,%10;"
163 "add %10,%8;"
164 "movdqa %%xmm2,%%xmm3;"
165 "mov %5,%10;"
166 "mov %8,%11;"
167 "ror $0xe,%10;"
168 "movdqa %%xmm2,%%xmm8;"
169 "xor %5,%10;"
170 "ror $0x9,%11;"
171 "mov %6,%12;"
172 "xor %8,%11;"
173 "ror $0x5,%10;"
174 "psrlq $0x11,%%xmm2;"
175 "xor %k2,%12;"
176 "psrlq $0x13,%%xmm3;"
177 "xor %5,%10;"
178 "and %5,%12;"
179 "psrld $0xa,%%xmm8;"
180 "ror $0xb,%11;"
181 "xor %8,%11;"
182 "xor %k2,%12;"
183 "ror $0x6,%10;"
184 "pxor %%xmm3,%%xmm2;"
185 "add %10,%12;"
186 "ror $0x2,%11;"
187 "add 8+%16,%12;"
188 "pxor %%xmm2,%%xmm8;"
189 "mov %8,%10;"
190 "add %12,%7;"
191 "mov %8,%12;"
192 "pshufb %%xmm10,%%xmm8;"
193 "or %3,%10;"
194 "add %7,%4;"
195 "and %3,%12;"
196 "paddd %%xmm8,%%xmm0;"
197 "and %9,%10;"
198 "add %11,%7;"
199 "pshufd $0x50,%%xmm0,%%xmm2;"
200 "or %12,%10;"
201 "add %10,%7;"
202 "movdqa %%xmm2,%%xmm3;"
203 "mov %4,%10;"
204 "ror $0xe,%10;"
205 "mov %7,%11;"
206 "movdqa %%xmm2,%%xmm4;"
207 "ror $0x9,%11;"
208 "xor %4,%10;"
209 "mov %5,%12;"
210 "ror $0x5,%10;"
211 "psrlq $0x11,%%xmm2;"
212 "xor %7,%11;"
213 "xor %6,%12;"
214 "psrlq $0x13,%%xmm3;"
215 "xor %4,%10;"
216 "and %4,%12;"
217 "ror $0xb,%11;"
218 "psrld $0xa,%%xmm4;"
219 "xor %7,%11;"
220 "ror $0x6,%10;"
221 "xor %6,%12;"
222 "pxor %%xmm3,%%xmm2;"
223 "ror $0x2,%11;"
224 "add %10,%12;"
225 "add 12+%16,%12;"
226 "pxor %%xmm2,%%xmm4;"
227 "mov %7,%10;"
228 "add %12,%k2;"
229 "mov %7,%12;"
230 "pshufb %%xmm11,%%xmm4;"
231 "or %9,%10;"
232 "add %k2,%3;"
233 "and %9,%12;"
234 "paddd %%xmm0,%%xmm4;"
235 "and %8,%10;"
236 "add %11,%k2;"
237 "or %12,%10;"
238 "add %10,%k2;"
239 "movdqa 0x10(%13),%%xmm9;"
240 "paddd %%xmm5,%%xmm9;"
241 "movdqa %%xmm9,%16;"
242 "movdqa %%xmm4,%%xmm0;"
243 "mov %3,%10;"
244 "ror $0xe,%10;"
245 "mov %k2,%11;"
246 "palignr $0x4,%%xmm7,%%xmm0;"
247 "ror $0x9,%11;"
248 "xor %3,%10;"
249 "mov %4,%12;"
250 "ror $0x5,%10;"
251 "movdqa %%xmm6,%%xmm1;"
252 "xor %k2,%11;"
253 "xor %5,%12;"
254 "paddd %%xmm5,%%xmm0;"
255 "xor %3,%10;"
256 "and %3,%12;"
257 "ror $0xb,%11;"
258 "palignr $0x4,%%xmm5,%%xmm1;"
259 "xor %k2,%11;"
260 "ror $0x6,%10;"
261 "xor %5,%12;"
262 "movdqa %%xmm1,%%xmm2;"
263 "ror $0x2,%11;"
264 "add %10,%12;"
265 "add %16,%12;"
266 "movdqa %%xmm1,%%xmm3;"
267 "mov %k2,%10;"
268 "add %12,%6;"
269 "mov %k2,%12;"
270 "pslld $0x19,%%xmm1;"
271 "or %8,%10;"
272 "add %6,%9;"
273 "and %8,%12;"
274 "psrld $0x7,%%xmm2;"
275 "and %7,%10;"
276 "add %11,%6;"
277 "por %%xmm2,%%xmm1;"
278 "or %12,%10;"
279 "add %10,%6;"
280 "movdqa %%xmm3,%%xmm2;"
281 "mov %9,%10;"
282 "mov %6,%11;"
283 "movdqa %%xmm3,%%xmm8;"
284 "ror $0xe,%10;"
285 "xor %9,%10;"
286 "mov %3,%12;"
287 "ror $0x9,%11;"
288 "pslld $0xe,%%xmm3;"
289 "xor %6,%11;"
290 "ror $0x5,%10;"
291 "xor %4,%12;"
292 "psrld $0x12,%%xmm2;"
293 "ror $0xb,%11;"
294 "xor %9,%10;"
295 "and %9,%12;"
296 "ror $0x6,%10;"
297 "pxor %%xmm3,%%xmm1;"
298 "xor %6,%11;"
299 "xor %4,%12;"
300 "psrld $0x3,%%xmm8;"
301 "add %10,%12;"
302 "add 4+%16,%12;"
303 "ror $0x2,%11;"
304 "pxor %%xmm2,%%xmm1;"
305 "mov %6,%10;"
306 "add %12,%5;"
307 "mov %6,%12;"
308 "pxor %%xmm8,%%xmm1;"
309 "or %7,%10;"
310 "add %5,%8;"
311 "and %7,%12;"
312 "pshufd $0xfa,%%xmm4,%%xmm2;"
313 "and %k2,%10;"
314 "add %11,%5;"
315 "paddd %%xmm1,%%xmm0;"
316 "or %12,%10;"
317 "add %10,%5;"
318 "movdqa %%xmm2,%%xmm3;"
319 "mov %8,%10;"
320 "mov %5,%11;"
321 "ror $0xe,%10;"
322 "movdqa %%xmm2,%%xmm8;"
323 "xor %8,%10;"
324 "ror $0x9,%11;"
325 "mov %9,%12;"
326 "xor %5,%11;"
327 "ror $0x5,%10;"
328 "psrlq $0x11,%%xmm2;"
329 "xor %3,%12;"
330 "psrlq $0x13,%%xmm3;"
331 "xor %8,%10;"
332 "and %8,%12;"
333 "psrld $0xa,%%xmm8;"
334 "ror $0xb,%11;"
335 "xor %5,%11;"
336 "xor %3,%12;"
337 "ror $0x6,%10;"
338 "pxor %%xmm3,%%xmm2;"
339 "add %10,%12;"
340 "ror $0x2,%11;"
341 "add 8+%16,%12;"
342 "pxor %%xmm2,%%xmm8;"
343 "mov %5,%10;"
344 "add %12,%4;"
345 "mov %5,%12;"
346 "pshufb %%xmm10,%%xmm8;"
347 "or %k2,%10;"
348 "add %4,%7;"
349 "and %k2,%12;"
350 "paddd %%xmm8,%%xmm0;"
351 "and %6,%10;"
352 "add %11,%4;"
353 "pshufd $0x50,%%xmm0,%%xmm2;"
354 "or %12,%10;"
355 "add %10,%4;"
356 "movdqa %%xmm2,%%xmm3;"
357 "mov %7,%10;"
358 "ror $0xe,%10;"
359 "mov %4,%11;"
360 "movdqa %%xmm2,%%xmm5;"
361 "ror $0x9,%11;"
362 "xor %7,%10;"
363 "mov %8,%12;"
364 "ror $0x5,%10;"
365 "psrlq $0x11,%%xmm2;"
366 "xor %4,%11;"
367 "xor %9,%12;"
368 "psrlq $0x13,%%xmm3;"
369 "xor %7,%10;"
370 "and %7,%12;"
371 "ror $0xb,%11;"
372 "psrld $0xa,%%xmm5;"
373 "xor %4,%11;"
374 "ror $0x6,%10;"
375 "xor %9,%12;"
376 "pxor %%xmm3,%%xmm2;"
377 "ror $0x2,%11;"
378 "add %10,%12;"
379 "add 12+%16,%12;"
380 "pxor %%xmm2,%%xmm5;"
381 "mov %4,%10;"
382 "add %12,%3;"
383 "mov %4,%12;"
384 "pshufb %%xmm11,%%xmm5;"
385 "or %6,%10;"
386 "add %3,%k2;"
387 "and %6,%12;"
388 "paddd %%xmm0,%%xmm5;"
389 "and %5,%10;"
390 "add %11,%3;"
391 "or %12,%10;"
392 "add %10,%3;"
393 "movdqa 0x20(%13),%%xmm9;"
394 "paddd %%xmm6,%%xmm9;"
395 "movdqa %%xmm9,%16;"
396 "movdqa %%xmm5,%%xmm0;"
397 "mov %k2,%10;"
398 "ror $0xe,%10;"
399 "mov %3,%11;"
400 "palignr $0x4,%%xmm4,%%xmm0;"
401 "ror $0x9,%11;"
402 "xor %k2,%10;"
403 "mov %7,%12;"
404 "ror $0x5,%10;"
405 "movdqa %%xmm7,%%xmm1;"
406 "xor %3,%11;"
407 "xor %8,%12;"
408 "paddd %%xmm6,%%xmm0;"
409 "xor %k2,%10;"
410 "and %k2,%12;"
411 "ror $0xb,%11;"
412 "palignr $0x4,%%xmm6,%%xmm1;"
413 "xor %3,%11;"
414 "ror $0x6,%10;"
415 "xor %8,%12;"
416 "movdqa %%xmm1,%%xmm2;"
417 "ror $0x2,%11;"
418 "add %10,%12;"
419 "add %16,%12;"
420 "movdqa %%xmm1,%%xmm3;"
421 "mov %3,%10;"
422 "add %12,%9;"
423 "mov %3,%12;"
424 "pslld $0x19,%%xmm1;"
425 "or %5,%10;"
426 "add %9,%6;"
427 "and %5,%12;"
428 "psrld $0x7,%%xmm2;"
429 "and %4,%10;"
430 "add %11,%9;"
431 "por %%xmm2,%%xmm1;"
432 "or %12,%10;"
433 "add %10,%9;"
434 "movdqa %%xmm3,%%xmm2;"
435 "mov %6,%10;"
436 "mov %9,%11;"
437 "movdqa %%xmm3,%%xmm8;"
438 "ror $0xe,%10;"
439 "xor %6,%10;"
440 "mov %k2,%12;"
441 "ror $0x9,%11;"
442 "pslld $0xe,%%xmm3;"
443 "xor %9,%11;"
444 "ror $0x5,%10;"
445 "xor %7,%12;"
446 "psrld $0x12,%%xmm2;"
447 "ror $0xb,%11;"
448 "xor %6,%10;"
449 "and %6,%12;"
450 "ror $0x6,%10;"
451 "pxor %%xmm3,%%xmm1;"
452 "xor %9,%11;"
453 "xor %7,%12;"
454 "psrld $0x3,%%xmm8;"
455 "add %10,%12;"
456 "add 4+%16,%12;"
457 "ror $0x2,%11;"
458 "pxor %%xmm2,%%xmm1;"
459 "mov %9,%10;"
460 "add %12,%8;"
461 "mov %9,%12;"
462 "pxor %%xmm8,%%xmm1;"
463 "or %4,%10;"
464 "add %8,%5;"
465 "and %4,%12;"
466 "pshufd $0xfa,%%xmm5,%%xmm2;"
467 "and %3,%10;"
468 "add %11,%8;"
469 "paddd %%xmm1,%%xmm0;"
470 "or %12,%10;"
471 "add %10,%8;"
472 "movdqa %%xmm2,%%xmm3;"
473 "mov %5,%10;"
474 "mov %8,%11;"
475 "ror $0xe,%10;"
476 "movdqa %%xmm2,%%xmm8;"
477 "xor %5,%10;"
478 "ror $0x9,%11;"
479 "mov %6,%12;"
480 "xor %8,%11;"
481 "ror $0x5,%10;"
482 "psrlq $0x11,%%xmm2;"
483 "xor %k2,%12;"
484 "psrlq $0x13,%%xmm3;"
485 "xor %5,%10;"
486 "and %5,%12;"
487 "psrld $0xa,%%xmm8;"
488 "ror $0xb,%11;"
489 "xor %8,%11;"
490 "xor %k2,%12;"
491 "ror $0x6,%10;"
492 "pxor %%xmm3,%%xmm2;"
493 "add %10,%12;"
494 "ror $0x2,%11;"
495 "add 8+%16,%12;"
496 "pxor %%xmm2,%%xmm8;"
497 "mov %8,%10;"
498 "add %12,%7;"
499 "mov %8,%12;"
500 "pshufb %%xmm10,%%xmm8;"
501 "or %3,%10;"
502 "add %7,%4;"
503 "and %3,%12;"
504 "paddd %%xmm8,%%xmm0;"
505 "and %9,%10;"
506 "add %11,%7;"
507 "pshufd $0x50,%%xmm0,%%xmm2;"
508 "or %12,%10;"
509 "add %10,%7;"
510 "movdqa %%xmm2,%%xmm3;"
511 "mov %4,%10;"
512 "ror $0xe,%10;"
513 "mov %7,%11;"
514 "movdqa %%xmm2,%%xmm6;"
515 "ror $0x9,%11;"
516 "xor %4,%10;"
517 "mov %5,%12;"
518 "ror $0x5,%10;"
519 "psrlq $0x11,%%xmm2;"
520 "xor %7,%11;"
521 "xor %6,%12;"
522 "psrlq $0x13,%%xmm3;"
523 "xor %4,%10;"
524 "and %4,%12;"
525 "ror $0xb,%11;"
526 "psrld $0xa,%%xmm6;"
527 "xor %7,%11;"
528 "ror $0x6,%10;"
529 "xor %6,%12;"
530 "pxor %%xmm3,%%xmm2;"
531 "ror $0x2,%11;"
532 "add %10,%12;"
533 "add 12+%16,%12;"
534 "pxor %%xmm2,%%xmm6;"
535 "mov %7,%10;"
536 "add %12,%k2;"
537 "mov %7,%12;"
538 "pshufb %%xmm11,%%xmm6;"
539 "or %9,%10;"
540 "add %k2,%3;"
541 "and %9,%12;"
542 "paddd %%xmm0,%%xmm6;"
543 "and %8,%10;"
544 "add %11,%k2;"
545 "or %12,%10;"
546 "add %10,%k2;"
547 "movdqa 0x30(%13),%%xmm9;"
548 "paddd %%xmm7,%%xmm9;"
549 "movdqa %%xmm9,%16;"
550 "add $0x40,%13;"
551 "movdqa %%xmm6,%%xmm0;"
552 "mov %3,%10;"
553 "ror $0xe,%10;"
554 "mov %k2,%11;"
555 "palignr $0x4,%%xmm5,%%xmm0;"
556 "ror $0x9,%11;"
557 "xor %3,%10;"
558 "mov %4,%12;"
559 "ror $0x5,%10;"
560 "movdqa %%xmm4,%%xmm1;"
561 "xor %k2,%11;"
562 "xor %5,%12;"
563 "paddd %%xmm7,%%xmm0;"
564 "xor %3,%10;"
565 "and %3,%12;"
566 "ror $0xb,%11;"
567 "palignr $0x4,%%xmm7,%%xmm1;"
568 "xor %k2,%11;"
569 "ror $0x6,%10;"
570 "xor %5,%12;"
571 "movdqa %%xmm1,%%xmm2;"
572 "ror $0x2,%11;"
573 "add %10,%12;"
574 "add %16,%12;"
575 "movdqa %%xmm1,%%xmm3;"
576 "mov %k2,%10;"
577 "add %12,%6;"
578 "mov %k2,%12;"
579 "pslld $0x19,%%xmm1;"
580 "or %8,%10;"
581 "add %6,%9;"
582 "and %8,%12;"
583 "psrld $0x7,%%xmm2;"
584 "and %7,%10;"
585 "add %11,%6;"
586 "por %%xmm2,%%xmm1;"
587 "or %12,%10;"
588 "add %10,%6;"
589 "movdqa %%xmm3,%%xmm2;"
590 "mov %9,%10;"
591 "mov %6,%11;"
592 "movdqa %%xmm3,%%xmm8;"
593 "ror $0xe,%10;"
594 "xor %9,%10;"
595 "mov %3,%12;"
596 "ror $0x9,%11;"
597 "pslld $0xe,%%xmm3;"
598 "xor %6,%11;"
599 "ror $0x5,%10;"
600 "xor %4,%12;"
601 "psrld $0x12,%%xmm2;"
602 "ror $0xb,%11;"
603 "xor %9,%10;"
604 "and %9,%12;"
605 "ror $0x6,%10;"
606 "pxor %%xmm3,%%xmm1;"
607 "xor %6,%11;"
608 "xor %4,%12;"
609 "psrld $0x3,%%xmm8;"
610 "add %10,%12;"
611 "add 4+%16,%12;"
612 "ror $0x2,%11;"
613 "pxor %%xmm2,%%xmm1;"
614 "mov %6,%10;"
615 "add %12,%5;"
616 "mov %6,%12;"
617 "pxor %%xmm8,%%xmm1;"
618 "or %7,%10;"
619 "add %5,%8;"
620 "and %7,%12;"
621 "pshufd $0xfa,%%xmm6,%%xmm2;"
622 "and %k2,%10;"
623 "add %11,%5;"
624 "paddd %%xmm1,%%xmm0;"
625 "or %12,%10;"
626 "add %10,%5;"
627 "movdqa %%xmm2,%%xmm3;"
628 "mov %8,%10;"
629 "mov %5,%11;"
630 "ror $0xe,%10;"
631 "movdqa %%xmm2,%%xmm8;"
632 "xor %8,%10;"
633 "ror $0x9,%11;"
634 "mov %9,%12;"
635 "xor %5,%11;"
636 "ror $0x5,%10;"
637 "psrlq $0x11,%%xmm2;"
638 "xor %3,%12;"
639 "psrlq $0x13,%%xmm3;"
640 "xor %8,%10;"
641 "and %8,%12;"
642 "psrld $0xa,%%xmm8;"
643 "ror $0xb,%11;"
644 "xor %5,%11;"
645 "xor %3,%12;"
646 "ror $0x6,%10;"
647 "pxor %%xmm3,%%xmm2;"
648 "add %10,%12;"
649 "ror $0x2,%11;"
650 "add 8+%16,%12;"
651 "pxor %%xmm2,%%xmm8;"
652 "mov %5,%10;"
653 "add %12,%4;"
654 "mov %5,%12;"
655 "pshufb %%xmm10,%%xmm8;"
656 "or %k2,%10;"
657 "add %4,%7;"
658 "and %k2,%12;"
659 "paddd %%xmm8,%%xmm0;"
660 "and %6,%10;"
661 "add %11,%4;"
662 "pshufd $0x50,%%xmm0,%%xmm2;"
663 "or %12,%10;"
664 "add %10,%4;"
665 "movdqa %%xmm2,%%xmm3;"
666 "mov %7,%10;"
667 "ror $0xe,%10;"
668 "mov %4,%11;"
669 "movdqa %%xmm2,%%xmm7;"
670 "ror $0x9,%11;"
671 "xor %7,%10;"
672 "mov %8,%12;"
673 "ror $0x5,%10;"
674 "psrlq $0x11,%%xmm2;"
675 "xor %4,%11;"
676 "xor %9,%12;"
677 "psrlq $0x13,%%xmm3;"
678 "xor %7,%10;"
679 "and %7,%12;"
680 "ror $0xb,%11;"
681 "psrld $0xa,%%xmm7;"
682 "xor %4,%11;"
683 "ror $0x6,%10;"
684 "xor %9,%12;"
685 "pxor %%xmm3,%%xmm2;"
686 "ror $0x2,%11;"
687 "add %10,%12;"
688 "add 12+%16,%12;"
689 "pxor %%xmm2,%%xmm7;"
690 "mov %4,%10;"
691 "add %12,%3;"
692 "mov %4,%12;"
693 "pshufb %%xmm11,%%xmm7;"
694 "or %6,%10;"
695 "add %3,%k2;"
696 "and %6,%12;"
697 "paddd %%xmm0,%%xmm7;"
698 "and %5,%10;"
699 "add %11,%3;"
700 "or %12,%10;"
701 "add %10,%3;"
702 "sub $0x1,%1;"
703 "jne Lloop1_%=;"
704 "mov $0x2,%1;"
705
706 "Lloop2_%=:"
707 "paddd 0x0(%13),%%xmm4;"
708 "movdqa %%xmm4,%16;"
709 "mov %k2,%10;"
710 "ror $0xe,%10;"
711 "mov %3,%11;"
712 "xor %k2,%10;"
713 "ror $0x9,%11;"
714 "mov %7,%12;"
715 "xor %3,%11;"
716 "ror $0x5,%10;"
717 "xor %8,%12;"
718 "xor %k2,%10;"
719 "ror $0xb,%11;"
720 "and %k2,%12;"
721 "xor %3,%11;"
722 "ror $0x6,%10;"
723 "xor %8,%12;"
724 "add %10,%12;"
725 "ror $0x2,%11;"
726 "add %16,%12;"
727 "mov %3,%10;"
728 "add %12,%9;"
729 "mov %3,%12;"
730 "or %5,%10;"
731 "add %9,%6;"
732 "and %5,%12;"
733 "and %4,%10;"
734 "add %11,%9;"
735 "or %12,%10;"
736 "add %10,%9;"
737 "mov %6,%10;"
738 "ror $0xe,%10;"
739 "mov %9,%11;"
740 "xor %6,%10;"
741 "ror $0x9,%11;"
742 "mov %k2,%12;"
743 "xor %9,%11;"
744 "ror $0x5,%10;"
745 "xor %7,%12;"
746 "xor %6,%10;"
747 "ror $0xb,%11;"
748 "and %6,%12;"
749 "xor %9,%11;"
750 "ror $0x6,%10;"
751 "xor %7,%12;"
752 "add %10,%12;"
753 "ror $0x2,%11;"
754 "add 4+%16,%12;"
755 "mov %9,%10;"
756 "add %12,%8;"
757 "mov %9,%12;"
758 "or %4,%10;"
759 "add %8,%5;"
760 "and %4,%12;"
761 "and %3,%10;"
762 "add %11,%8;"
763 "or %12,%10;"
764 "add %10,%8;"
765 "mov %5,%10;"
766 "ror $0xe,%10;"
767 "mov %8,%11;"
768 "xor %5,%10;"
769 "ror $0x9,%11;"
770 "mov %6,%12;"
771 "xor %8,%11;"
772 "ror $0x5,%10;"
773 "xor %k2,%12;"
774 "xor %5,%10;"
775 "ror $0xb,%11;"
776 "and %5,%12;"
777 "xor %8,%11;"
778 "ror $0x6,%10;"
779 "xor %k2,%12;"
780 "add %10,%12;"
781 "ror $0x2,%11;"
782 "add 8+%16,%12;"
783 "mov %8,%10;"
784 "add %12,%7;"
785 "mov %8,%12;"
786 "or %3,%10;"
787 "add %7,%4;"
788 "and %3,%12;"
789 "and %9,%10;"
790 "add %11,%7;"
791 "or %12,%10;"
792 "add %10,%7;"
793 "mov %4,%10;"
794 "ror $0xe,%10;"
795 "mov %7,%11;"
796 "xor %4,%10;"
797 "ror $0x9,%11;"
798 "mov %5,%12;"
799 "xor %7,%11;"
800 "ror $0x5,%10;"
801 "xor %6,%12;"
802 "xor %4,%10;"
803 "ror $0xb,%11;"
804 "and %4,%12;"
805 "xor %7,%11;"
806 "ror $0x6,%10;"
807 "xor %6,%12;"
808 "add %10,%12;"
809 "ror $0x2,%11;"
810 "add 12+%16,%12;"
811 "mov %7,%10;"
812 "add %12,%k2;"
813 "mov %7,%12;"
814 "or %9,%10;"
815 "add %k2,%3;"
816 "and %9,%12;"
817 "and %8,%10;"
818 "add %11,%k2;"
819 "or %12,%10;"
820 "add %10,%k2;"
821 "paddd 0x10(%13),%%xmm5;"
822 "movdqa %%xmm5,%16;"
823 "add $0x20,%13;"
824 "mov %3,%10;"
825 "ror $0xe,%10;"
826 "mov %k2,%11;"
827 "xor %3,%10;"
828 "ror $0x9,%11;"
829 "mov %4,%12;"
830 "xor %k2,%11;"
831 "ror $0x5,%10;"
832 "xor %5,%12;"
833 "xor %3,%10;"
834 "ror $0xb,%11;"
835 "and %3,%12;"
836 "xor %k2,%11;"
837 "ror $0x6,%10;"
838 "xor %5,%12;"
839 "add %10,%12;"
840 "ror $0x2,%11;"
841 "add %16,%12;"
842 "mov %k2,%10;"
843 "add %12,%6;"
844 "mov %k2,%12;"
845 "or %8,%10;"
846 "add %6,%9;"
847 "and %8,%12;"
848 "and %7,%10;"
849 "add %11,%6;"
850 "or %12,%10;"
851 "add %10,%6;"
852 "mov %9,%10;"
853 "ror $0xe,%10;"
854 "mov %6,%11;"
855 "xor %9,%10;"
856 "ror $0x9,%11;"
857 "mov %3,%12;"
858 "xor %6,%11;"
859 "ror $0x5,%10;"
860 "xor %4,%12;"
861 "xor %9,%10;"
862 "ror $0xb,%11;"
863 "and %9,%12;"
864 "xor %6,%11;"
865 "ror $0x6,%10;"
866 "xor %4,%12;"
867 "add %10,%12;"
868 "ror $0x2,%11;"
869 "add 4+%16,%12;"
870 "mov %6,%10;"
871 "add %12,%5;"
872 "mov %6,%12;"
873 "or %7,%10;"
874 "add %5,%8;"
875 "and %7,%12;"
876 "and %k2,%10;"
877 "add %11,%5;"
878 "or %12,%10;"
879 "add %10,%5;"
880 "mov %8,%10;"
881 "ror $0xe,%10;"
882 "mov %5,%11;"
883 "xor %8,%10;"
884 "ror $0x9,%11;"
885 "mov %9,%12;"
886 "xor %5,%11;"
887 "ror $0x5,%10;"
888 "xor %3,%12;"
889 "xor %8,%10;"
890 "ror $0xb,%11;"
891 "and %8,%12;"
892 "xor %5,%11;"
893 "ror $0x6,%10;"
894 "xor %3,%12;"
895 "add %10,%12;"
896 "ror $0x2,%11;"
897 "add 8+%16,%12;"
898 "mov %5,%10;"
899 "add %12,%4;"
900 "mov %5,%12;"
901 "or %k2,%10;"
902 "add %4,%7;"
903 "and %k2,%12;"
904 "and %6,%10;"
905 "add %11,%4;"
906 "or %12,%10;"
907 "add %10,%4;"
908 "mov %7,%10;"
909 "ror $0xe,%10;"
910 "mov %4,%11;"
911 "xor %7,%10;"
912 "ror $0x9,%11;"
913 "mov %8,%12;"
914 "xor %4,%11;"
915 "ror $0x5,%10;"
916 "xor %9,%12;"
917 "xor %7,%10;"
918 "ror $0xb,%11;"
919 "and %7,%12;"
920 "xor %4,%11;"
921 "ror $0x6,%10;"
922 "xor %9,%12;"
923 "add %10,%12;"
924 "ror $0x2,%11;"
925 "add 12+%16,%12;"
926 "mov %4,%10;"
927 "add %12,%3;"
928 "mov %4,%12;"
929 "or %6,%10;"
930 "add %3,%k2;"
931 "and %6,%12;"
932 "and %5,%10;"
933 "add %11,%3;"
934 "or %12,%10;"
935 "add %10,%3;"
936 "movdqa %%xmm6,%%xmm4;"
937 "movdqa %%xmm7,%%xmm5;"
938 "sub $0x1,%1;"
939 "jne Lloop2_%=;"
940 "add (%0),%3;"
941 "mov %3,(%0);"
942 "add 0x4(%0),%4;"
943 "mov %4,0x4(%0);"
944 "add 0x8(%0),%5;"
945 "mov %5,0x8(%0);"
946 "add 0xc(%0),%6;"
947 "mov %6,0xc(%0);"
948 "add 0x10(%0),%k2;"
949 "mov %k2,0x10(%0);"
950 "add 0x14(%0),%7;"
951 "mov %7,0x14(%0);"
952 "add 0x18(%0),%8;"
953 "mov %8,0x18(%0);"
954 "add 0x1c(%0),%9;"
955 "mov %9,0x1c(%0);"
956 "mov %15,%1;"
957 "add $0x40,%1;"
958 "cmp %14,%1;"
959 "jne Lloop0_%=;"
960
961 "Ldone_hash_%=:"
962
963 : "+r"(s), "+r"(chunk), "+r"(blocks), "=r"(a), "=r"(b), "=r"(c), "=r"(d), /* e = chunk */ "=r"(f), "=r"(g), "=r"(h), "=r"(y0), "=r"(y1), "=r"(y2), "=r"(tbl), "+m"(inp_end), "+m"(inp), "+m"(xfer)
964 : "m"(K256), "m"(FLIP_MASK), "m"(SHUF_00BA), "m"(SHUF_DC00)
965 : "cc", "memory", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12"
966 );
967 }
968 }
969
970 /*
971 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
972 ; Copyright (c) 2012, Intel Corporation
973 ;
974 ; All rights reserved.
975 ;
976 ; Redistribution and use in source and binary forms, with or without
977 ; modification, are permitted provided that the following conditions are
978 ; met:
979 ;
980 ; * Redistributions of source code must retain the above copyright
981 ; notice, this list of conditions and the following disclaimer.
982 ;
983 ; * Redistributions in binary form must reproduce the above copyright
984 ; notice, this list of conditions and the following disclaimer in the
985 ; documentation and/or other materials provided with the
986 ; distribution.
987 ;
988 ; * Neither the name of the Intel Corporation nor the names of its
989 ; contributors may be used to endorse or promote products derived from
990 ; this software without specific prior written permission.
991 ;
992 ;
993 ; THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION "AS IS" AND ANY
994 ; EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
995 ; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
996 ; PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR
997 ; CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
998 ; EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
999 ; PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
1000 ; PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
1001 ; LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
1002 ; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
1003 ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
1004 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1005 ;
1006 ; Example YASM command lines:
1007 ; Windows: yasm -Xvc -f x64 -rnasm -pnasm -o sha256_sse4.obj -g cv8 sha256_sse4.asm
1008 ; Linux: yasm -f x64 -f elf64 -X gnu -g dwarf2 -D LINUX -o sha256_sse4.o sha256_sse4.asm
1009 ;
1010 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1011 ;
1012 ; This code is described in an Intel White-Paper:
1013 ; "Fast SHA-256 Implementations on Intel Architecture Processors"
1014 ;
1015 ; To find it, surf to https://www.intel.com/p/en_US/embedded
1016 ; and search for that title.
1017 ; The paper is expected to be released roughly at the end of April, 2012
1018 ;
1019 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1020 ; This code schedules 1 blocks at a time, with 4 lanes per block
1021 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1022
1023 %define MOVDQ movdqu ;; assume buffers not aligned
1024
1025 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Define Macros
1026
1027 ; addm [mem], reg
1028 ; Add reg to mem using reg-mem add and store
1029 %macro addm 2
1030 add %2, %1
1031 mov %1, %2
1032 %endm
1033
1034 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1035
1036 ; COPY_XMM_AND_BSWAP xmm, [mem], byte_flip_mask
1037 ; Load xmm with mem and byte swap each dword
1038 %macro COPY_XMM_AND_BSWAP 3
1039 MOVDQ %1, %2
1040 pshufb %1, %3
1041 %endmacro
1042
1043 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1044
1045 %define X0 xmm4
1046 %define X1 xmm5
1047 %define X2 xmm6
1048 %define X3 xmm7
1049
1050 %define XTMP0 xmm0
1051 %define XTMP1 xmm1
1052 %define XTMP2 xmm2
1053 %define XTMP3 xmm3
1054 %define XTMP4 xmm8
1055 %define XFER xmm9
1056
1057 %define SHUF_00BA xmm10 ; shuffle xBxA -> 00BA
1058 %define SHUF_DC00 xmm11 ; shuffle xDxC -> DC00
1059 %define BYTE_FLIP_MASK xmm12
1060
1061 %ifdef LINUX
1062 %define NUM_BLKS rdx ; 3rd arg
1063 %define CTX rsi ; 2nd arg
1064 %define INP rdi ; 1st arg
1065
1066 %define SRND rdi ; clobbers INP
1067 %define c ecx
1068 %define d r8d
1069 %define e edx
1070 %else
1071 %define NUM_BLKS r8 ; 3rd arg
1072 %define CTX rdx ; 2nd arg
1073 %define INP rcx ; 1st arg
1074
1075 %define SRND rcx ; clobbers INP
1076 %define c edi
1077 %define d esi
1078 %define e r8d
1079
1080 %endif
1081 %define TBL rbp
1082 %define a eax
1083 %define b ebx
1084
1085 %define f r9d
1086 %define g r10d
1087 %define h r11d
1088
1089 %define y0 r13d
1090 %define y1 r14d
1091 %define y2 r15d
1092
1093
1094
1095 _INP_END_SIZE equ 8
1096 _INP_SIZE equ 8
1097 _XFER_SIZE equ 8
1098 %ifdef LINUX
1099 _XMM_SAVE_SIZE equ 0
1100 %else
1101 _XMM_SAVE_SIZE equ 7*16
1102 %endif
1103 ; STACK_SIZE plus pushes must be an odd multiple of 8
1104 _ALIGN_SIZE equ 8
1105
1106 _INP_END equ 0
1107 _INP equ _INP_END + _INP_END_SIZE
1108 _XFER equ _INP + _INP_SIZE
1109 _XMM_SAVE equ _XFER + _XFER_SIZE + _ALIGN_SIZE
1110 STACK_SIZE equ _XMM_SAVE + _XMM_SAVE_SIZE
1111
1112 ; rotate_Xs
1113 ; Rotate values of symbols X0...X3
1114 %macro rotate_Xs 0
1115 %xdefine X_ X0
1116 %xdefine X0 X1
1117 %xdefine X1 X2
1118 %xdefine X2 X3
1119 %xdefine X3 X_
1120 %endm
1121
1122 ; ROTATE_ARGS
1123 ; Rotate values of symbols a...h
1124 %macro ROTATE_ARGS 0
1125 %xdefine TMP_ h
1126 %xdefine h g
1127 %xdefine g f
1128 %xdefine f e
1129 %xdefine e d
1130 %xdefine d c
1131 %xdefine c b
1132 %xdefine b a
1133 %xdefine a TMP_
1134 %endm
1135
1136 %macro FOUR_ROUNDS_AND_SCHED 0
1137 ;; compute s0 four at a time and s1 two at a time
1138 ;; compute W[-16] + W[-7] 4 at a time
1139 movdqa XTMP0, X3
1140 mov y0, e ; y0 = e
1141 ror y0, (25-11) ; y0 = e >> (25-11)
1142 mov y1, a ; y1 = a
1143 palignr XTMP0, X2, 4 ; XTMP0 = W[-7]
1144 ror y1, (22-13) ; y1 = a >> (22-13)
1145 xor y0, e ; y0 = e ^ (e >> (25-11))
1146 mov y2, f ; y2 = f
1147 ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6))
1148 movdqa XTMP1, X1
1149 xor y1, a ; y1 = a ^ (a >> (22-13)
1150 xor y2, g ; y2 = f^g
1151 paddd XTMP0, X0 ; XTMP0 = W[-7] + W[-16]
1152 xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
1153 and y2, e ; y2 = (f^g)&e
1154 ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2))
1155 ;; compute s0
1156 palignr XTMP1, X0, 4 ; XTMP1 = W[-15]
1157 xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
1158 ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
1159 xor y2, g ; y2 = CH = ((f^g)&e)^g
1160 movdqa XTMP2, XTMP1 ; XTMP2 = W[-15]
1161 ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
1162 add y2, y0 ; y2 = S1 + CH
1163 add y2, [rsp + _XFER + 0*4] ; y2 = k + w + S1 + CH
1164 movdqa XTMP3, XTMP1 ; XTMP3 = W[-15]
1165 mov y0, a ; y0 = a
1166 add h, y2 ; h = h + S1 + CH + k + w
1167 mov y2, a ; y2 = a
1168 pslld XTMP1, (32-7)
1169 or y0, c ; y0 = a|c
1170 add d, h ; d = d + h + S1 + CH + k + w
1171 and y2, c ; y2 = a&c
1172 psrld XTMP2, 7
1173 and y0, b ; y0 = (a|c)&b
1174 add h, y1 ; h = h + S1 + CH + k + w + S0
1175 por XTMP1, XTMP2 ; XTMP1 = W[-15] ror 7
1176 or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c)
1177 add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ
1178
1179 ROTATE_ARGS
1180 movdqa XTMP2, XTMP3 ; XTMP2 = W[-15]
1181 mov y0, e ; y0 = e
1182 mov y1, a ; y1 = a
1183 movdqa XTMP4, XTMP3 ; XTMP4 = W[-15]
1184 ror y0, (25-11) ; y0 = e >> (25-11)
1185 xor y0, e ; y0 = e ^ (e >> (25-11))
1186 mov y2, f ; y2 = f
1187 ror y1, (22-13) ; y1 = a >> (22-13)
1188 pslld XTMP3, (32-18)
1189 xor y1, a ; y1 = a ^ (a >> (22-13)
1190 ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6))
1191 xor y2, g ; y2 = f^g
1192 psrld XTMP2, 18
1193 ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2))
1194 xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
1195 and y2, e ; y2 = (f^g)&e
1196 ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
1197 pxor XTMP1, XTMP3
1198 xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
1199 xor y2, g ; y2 = CH = ((f^g)&e)^g
1200 psrld XTMP4, 3 ; XTMP4 = W[-15] >> 3
1201 add y2, y0 ; y2 = S1 + CH
1202 add y2, [rsp + _XFER + 1*4] ; y2 = k + w + S1 + CH
1203 ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
1204 pxor XTMP1, XTMP2 ; XTMP1 = W[-15] ror 7 ^ W[-15] ror 18
1205 mov y0, a ; y0 = a
1206 add h, y2 ; h = h + S1 + CH + k + w
1207 mov y2, a ; y2 = a
1208 pxor XTMP1, XTMP4 ; XTMP1 = s0
1209 or y0, c ; y0 = a|c
1210 add d, h ; d = d + h + S1 + CH + k + w
1211 and y2, c ; y2 = a&c
1212 ;; compute low s1
1213 pshufd XTMP2, X3, 11111010b ; XTMP2 = W[-2] {BBAA}
1214 and y0, b ; y0 = (a|c)&b
1215 add h, y1 ; h = h + S1 + CH + k + w + S0
1216 paddd XTMP0, XTMP1 ; XTMP0 = W[-16] + W[-7] + s0
1217 or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c)
1218 add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ
1219
1220 ROTATE_ARGS
1221 movdqa XTMP3, XTMP2 ; XTMP3 = W[-2] {BBAA}
1222 mov y0, e ; y0 = e
1223 mov y1, a ; y1 = a
1224 ror y0, (25-11) ; y0 = e >> (25-11)
1225 movdqa XTMP4, XTMP2 ; XTMP4 = W[-2] {BBAA}
1226 xor y0, e ; y0 = e ^ (e >> (25-11))
1227 ror y1, (22-13) ; y1 = a >> (22-13)
1228 mov y2, f ; y2 = f
1229 xor y1, a ; y1 = a ^ (a >> (22-13)
1230 ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6))
1231 psrlq XTMP2, 17 ; XTMP2 = W[-2] ror 17 {xBxA}
1232 xor y2, g ; y2 = f^g
1233 psrlq XTMP3, 19 ; XTMP3 = W[-2] ror 19 {xBxA}
1234 xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
1235 and y2, e ; y2 = (f^g)&e
1236 psrld XTMP4, 10 ; XTMP4 = W[-2] >> 10 {BBAA}
1237 ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2))
1238 xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
1239 xor y2, g ; y2 = CH = ((f^g)&e)^g
1240 ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
1241 pxor XTMP2, XTMP3
1242 add y2, y0 ; y2 = S1 + CH
1243 ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
1244 add y2, [rsp + _XFER + 2*4] ; y2 = k + w + S1 + CH
1245 pxor XTMP4, XTMP2 ; XTMP4 = s1 {xBxA}
1246 mov y0, a ; y0 = a
1247 add h, y2 ; h = h + S1 + CH + k + w
1248 mov y2, a ; y2 = a
1249 pshufb XTMP4, SHUF_00BA ; XTMP4 = s1 {00BA}
1250 or y0, c ; y0 = a|c
1251 add d, h ; d = d + h + S1 + CH + k + w
1252 and y2, c ; y2 = a&c
1253 paddd XTMP0, XTMP4 ; XTMP0 = {..., ..., W[1], W[0]}
1254 and y0, b ; y0 = (a|c)&b
1255 add h, y1 ; h = h + S1 + CH + k + w + S0
1256 ;; compute high s1
1257 pshufd XTMP2, XTMP0, 01010000b ; XTMP2 = W[-2] {DDCC}
1258 or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c)
1259 add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ
1260
1261 ROTATE_ARGS
1262 movdqa XTMP3, XTMP2 ; XTMP3 = W[-2] {DDCC}
1263 mov y0, e ; y0 = e
1264 ror y0, (25-11) ; y0 = e >> (25-11)
1265 mov y1, a ; y1 = a
1266 movdqa X0, XTMP2 ; X0 = W[-2] {DDCC}
1267 ror y1, (22-13) ; y1 = a >> (22-13)
1268 xor y0, e ; y0 = e ^ (e >> (25-11))
1269 mov y2, f ; y2 = f
1270 ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6))
1271 psrlq XTMP2, 17 ; XTMP2 = W[-2] ror 17 {xDxC}
1272 xor y1, a ; y1 = a ^ (a >> (22-13)
1273 xor y2, g ; y2 = f^g
1274 psrlq XTMP3, 19 ; XTMP3 = W[-2] ror 19 {xDxC}
1275 xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
1276 and y2, e ; y2 = (f^g)&e
1277 ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2))
1278 psrld X0, 10 ; X0 = W[-2] >> 10 {DDCC}
1279 xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
1280 ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
1281 xor y2, g ; y2 = CH = ((f^g)&e)^g
1282 pxor XTMP2, XTMP3
1283 ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
1284 add y2, y0 ; y2 = S1 + CH
1285 add y2, [rsp + _XFER + 3*4] ; y2 = k + w + S1 + CH
1286 pxor X0, XTMP2 ; X0 = s1 {xDxC}
1287 mov y0, a ; y0 = a
1288 add h, y2 ; h = h + S1 + CH + k + w
1289 mov y2, a ; y2 = a
1290 pshufb X0, SHUF_DC00 ; X0 = s1 {DC00}
1291 or y0, c ; y0 = a|c
1292 add d, h ; d = d + h + S1 + CH + k + w
1293 and y2, c ; y2 = a&c
1294 paddd X0, XTMP0 ; X0 = {W[3], W[2], W[1], W[0]}
1295 and y0, b ; y0 = (a|c)&b
1296 add h, y1 ; h = h + S1 + CH + k + w + S0
1297 or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c)
1298 add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ
1299
1300 ROTATE_ARGS
1301 rotate_Xs
1302 %endm
1303
1304 ;; input is [rsp + _XFER + %1 * 4]
1305 %macro DO_ROUND 1
1306 mov y0, e ; y0 = e
1307 ror y0, (25-11) ; y0 = e >> (25-11)
1308 mov y1, a ; y1 = a
1309 xor y0, e ; y0 = e ^ (e >> (25-11))
1310 ror y1, (22-13) ; y1 = a >> (22-13)
1311 mov y2, f ; y2 = f
1312 xor y1, a ; y1 = a ^ (a >> (22-13)
1313 ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6))
1314 xor y2, g ; y2 = f^g
1315 xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
1316 ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2))
1317 and y2, e ; y2 = (f^g)&e
1318 xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
1319 ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
1320 xor y2, g ; y2 = CH = ((f^g)&e)^g
1321 add y2, y0 ; y2 = S1 + CH
1322 ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
1323 add y2, [rsp + _XFER + %1 * 4] ; y2 = k + w + S1 + CH
1324 mov y0, a ; y0 = a
1325 add h, y2 ; h = h + S1 + CH + k + w
1326 mov y2, a ; y2 = a
1327 or y0, c ; y0 = a|c
1328 add d, h ; d = d + h + S1 + CH + k + w
1329 and y2, c ; y2 = a&c
1330 and y0, b ; y0 = (a|c)&b
1331 add h, y1 ; h = h + S1 + CH + k + w + S0
1332 or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c)
1333 add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ
1334 ROTATE_ARGS
1335 %endm
1336
1337 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1338 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1339 ;; void sha256_sse4(void *input_data, UINT32 digest[8], UINT64 num_blks)
1340 ;; arg 1 : pointer to input data
1341 ;; arg 2 : pointer to digest
1342 ;; arg 3 : Num blocks
1343 section .text
1344 global sha256_sse4
1345 align 32
1346 sha256_sse4:
1347 push rbx
1348 %ifndef LINUX
1349 push rsi
1350 push rdi
1351 %endif
1352 push rbp
1353 push r13
1354 push r14
1355 push r15
1356
1357 sub rsp,STACK_SIZE
1358 %ifndef LINUX
1359 movdqa [rsp + _XMM_SAVE + 0*16],xmm6
1360 movdqa [rsp + _XMM_SAVE + 1*16],xmm7
1361 movdqa [rsp + _XMM_SAVE + 2*16],xmm8
1362 movdqa [rsp + _XMM_SAVE + 3*16],xmm9
1363 movdqa [rsp + _XMM_SAVE + 4*16],xmm10
1364 movdqa [rsp + _XMM_SAVE + 5*16],xmm11
1365 movdqa [rsp + _XMM_SAVE + 6*16],xmm12
1366 %endif
1367
1368 shl NUM_BLKS, 6 ; convert to bytes
1369 jz done_hash
1370 add NUM_BLKS, INP ; pointer to end of data
1371 mov [rsp + _INP_END], NUM_BLKS
1372
1373 ;; load initial digest
1374 mov a,[4*0 + CTX]
1375 mov b,[4*1 + CTX]
1376 mov c,[4*2 + CTX]
1377 mov d,[4*3 + CTX]
1378 mov e,[4*4 + CTX]
1379 mov f,[4*5 + CTX]
1380 mov g,[4*6 + CTX]
1381 mov h,[4*7 + CTX]
1382
1383 movdqa BYTE_FLIP_MASK, [PSHUFFLE_BYTE_FLIP_MASK wrt rip]
1384 movdqa SHUF_00BA, [_SHUF_00BA wrt rip]
1385 movdqa SHUF_DC00, [_SHUF_DC00 wrt rip]
1386
1387 loop0:
1388 lea TBL,[K256 wrt rip]
1389
1390 ;; byte swap first 16 dwords
1391 COPY_XMM_AND_BSWAP X0, [INP + 0*16], BYTE_FLIP_MASK
1392 COPY_XMM_AND_BSWAP X1, [INP + 1*16], BYTE_FLIP_MASK
1393 COPY_XMM_AND_BSWAP X2, [INP + 2*16], BYTE_FLIP_MASK
1394 COPY_XMM_AND_BSWAP X3, [INP + 3*16], BYTE_FLIP_MASK
1395
1396 mov [rsp + _INP], INP
1397
1398 ;; schedule 48 input dwords, by doing 3 rounds of 16 each
1399 mov SRND, 3
1400 align 16
1401 loop1:
1402 movdqa XFER, [TBL + 0*16]
1403 paddd XFER, X0
1404 movdqa [rsp + _XFER], XFER
1405 FOUR_ROUNDS_AND_SCHED
1406
1407 movdqa XFER, [TBL + 1*16]
1408 paddd XFER, X0
1409 movdqa [rsp + _XFER], XFER
1410 FOUR_ROUNDS_AND_SCHED
1411
1412 movdqa XFER, [TBL + 2*16]
1413 paddd XFER, X0
1414 movdqa [rsp + _XFER], XFER
1415 FOUR_ROUNDS_AND_SCHED
1416
1417 movdqa XFER, [TBL + 3*16]
1418 paddd XFER, X0
1419 movdqa [rsp + _XFER], XFER
1420 add TBL, 4*16
1421 FOUR_ROUNDS_AND_SCHED
1422
1423 sub SRND, 1
1424 jne loop1
1425
1426 mov SRND, 2
1427 loop2:
1428 paddd X0, [TBL + 0*16]
1429 movdqa [rsp + _XFER], X0
1430 DO_ROUND 0
1431 DO_ROUND 1
1432 DO_ROUND 2
1433 DO_ROUND 3
1434 paddd X1, [TBL + 1*16]
1435 movdqa [rsp + _XFER], X1
1436 add TBL, 2*16
1437 DO_ROUND 0
1438 DO_ROUND 1
1439 DO_ROUND 2
1440 DO_ROUND 3
1441
1442 movdqa X0, X2
1443 movdqa X1, X3
1444
1445 sub SRND, 1
1446 jne loop2
1447
1448 addm [4*0 + CTX],a
1449 addm [4*1 + CTX],b
1450 addm [4*2 + CTX],c
1451 addm [4*3 + CTX],d
1452 addm [4*4 + CTX],e
1453 addm [4*5 + CTX],f
1454 addm [4*6 + CTX],g
1455 addm [4*7 + CTX],h
1456
1457 mov INP, [rsp + _INP]
1458 add INP, 64
1459 cmp INP, [rsp + _INP_END]
1460 jne loop0
1461
1462 done_hash:
1463 %ifndef LINUX
1464 movdqa xmm6,[rsp + _XMM_SAVE + 0*16]
1465 movdqa xmm7,[rsp + _XMM_SAVE + 1*16]
1466 movdqa xmm8,[rsp + _XMM_SAVE + 2*16]
1467 movdqa xmm9,[rsp + _XMM_SAVE + 3*16]
1468 movdqa xmm10,[rsp + _XMM_SAVE + 4*16]
1469 movdqa xmm11,[rsp + _XMM_SAVE + 5*16]
1470 movdqa xmm12,[rsp + _XMM_SAVE + 6*16]
1471 %endif
1472
1473 add rsp, STACK_SIZE
1474
1475 pop r15
1476 pop r14
1477 pop r13
1478 pop rbp
1479 %ifndef LINUX
1480 pop rdi
1481 pop rsi
1482 %endif
1483 pop rbx
1484
1485 ret
1486
1487
1488 section .data
1489 align 64
1490 K256:
1491 dd 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
1492 dd 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
1493 dd 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
1494 dd 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
1495 dd 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
1496 dd 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
1497 dd 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
1498 dd 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
1499 dd 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
1500 dd 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
1501 dd 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
1502 dd 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
1503 dd 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
1504 dd 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
1505 dd 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
1506 dd 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
1507
1508 PSHUFFLE_BYTE_FLIP_MASK: ddq 0x0c0d0e0f08090a0b0405060700010203
1509
1510 ; shuffle xBxA -> 00BA
1511 _SHUF_00BA: ddq 0xFFFFFFFFFFFFFFFF0b0a090803020100
1512
1513 ; shuffle xDxC -> DC00
1514 _SHUF_DC00: ddq 0x0b0a090803020100FFFFFFFFFFFFFFFF
1515 */
1516
1517 #endif
1518