sha256_x86_shani.cpp raw
1 // Copyright (c) 2018-2022 The Limenka developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 //
5 // Based on https://github.com/noloader/SHA-Intrinsics/blob/master/sha256-x86.c,
6 // Written and placed in public domain by Jeffrey Walton.
7 // Based on code from Intel, and by Sean Gulley for the miTLS project.
8
9 #if defined(ENABLE_SSE41) && defined(ENABLE_X86_SHANI)
10
11 #include <stdint.h>
12 #include <immintrin.h>
13
14 #include <attributes.h>
15
16 #if defined(__clang__)
17 #pragma clang attribute push(__attribute__((__target__("sse4,sse4.1,sha"))), apply_to = function)
18 #elif defined(__GNUC__)
19 #pragma GCC target ("sse4,sse4.1,sha")
20 #endif
21
22 namespace {
23
24 alignas(__m128i) const uint8_t MASK[16] = {0x03, 0x02, 0x01, 0x00, 0x07, 0x06, 0x05, 0x04, 0x0b, 0x0a, 0x09, 0x08, 0x0f, 0x0e, 0x0d, 0x0c};
25 alignas(__m128i) const uint8_t INIT0[16] = {0x8c, 0x68, 0x05, 0x9b, 0x7f, 0x52, 0x0e, 0x51, 0x85, 0xae, 0x67, 0xbb, 0x67, 0xe6, 0x09, 0x6a};
26 alignas(__m128i) const uint8_t INIT1[16] = {0x19, 0xcd, 0xe0, 0x5b, 0xab, 0xd9, 0x83, 0x1f, 0x3a, 0xf5, 0x4f, 0xa5, 0x72, 0xf3, 0x6e, 0x3c};
27
28 void ALWAYS_INLINE QuadRound(__m128i& state0, __m128i& state1, uint64_t k1, uint64_t k0)
29 {
30 const __m128i msg = _mm_set_epi64x(k1, k0);
31 state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
32 state0 = _mm_sha256rnds2_epu32(state0, state1, _mm_shuffle_epi32(msg, 0x0e));
33 }
34
35 void ALWAYS_INLINE QuadRound(__m128i& state0, __m128i& state1, __m128i m, uint64_t k1, uint64_t k0)
36 {
37 const __m128i msg = _mm_add_epi32(m, _mm_set_epi64x(k1, k0));
38 state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
39 state0 = _mm_sha256rnds2_epu32(state0, state1, _mm_shuffle_epi32(msg, 0x0e));
40 }
41
42 void ALWAYS_INLINE ShiftMessageA(__m128i& m0, __m128i m1)
43 {
44 m0 = _mm_sha256msg1_epu32(m0, m1);
45 }
46
47 void ALWAYS_INLINE ShiftMessageC(__m128i& m0, __m128i m1, __m128i& m2)
48 {
49 m2 = _mm_sha256msg2_epu32(_mm_add_epi32(m2, _mm_alignr_epi8(m1, m0, 4)), m1);
50 }
51
52 void ALWAYS_INLINE ShiftMessageB(__m128i& m0, __m128i m1, __m128i& m2)
53 {
54 ShiftMessageC(m0, m1, m2);
55 ShiftMessageA(m0, m1);
56 }
57
58 void ALWAYS_INLINE Shuffle(__m128i& s0, __m128i& s1)
59 {
60 const __m128i t1 = _mm_shuffle_epi32(s0, 0xB1);
61 const __m128i t2 = _mm_shuffle_epi32(s1, 0x1B);
62 s0 = _mm_alignr_epi8(t1, t2, 0x08);
63 s1 = _mm_blend_epi16(t2, t1, 0xF0);
64 }
65
66 void ALWAYS_INLINE Unshuffle(__m128i& s0, __m128i& s1)
67 {
68 const __m128i t1 = _mm_shuffle_epi32(s0, 0x1B);
69 const __m128i t2 = _mm_shuffle_epi32(s1, 0xB1);
70 s0 = _mm_blend_epi16(t1, t2, 0xF0);
71 s1 = _mm_alignr_epi8(t2, t1, 0x08);
72 }
73
74 __m128i ALWAYS_INLINE Load(const unsigned char* in)
75 {
76 return _mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)in), _mm_load_si128((const __m128i*)MASK));
77 }
78
79 void ALWAYS_INLINE Save(unsigned char* out, __m128i s)
80 {
81 _mm_storeu_si128((__m128i*)out, _mm_shuffle_epi8(s, _mm_load_si128((const __m128i*)MASK)));
82 }
83 }
84
85 namespace sha256_x86_shani {
86 void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks)
87 {
88 __m128i m0, m1, m2, m3, s0, s1, so0, so1;
89
90 /* Load state */
91 s0 = _mm_loadu_si128((const __m128i*)s);
92 s1 = _mm_loadu_si128((const __m128i*)(s + 4));
93 Shuffle(s0, s1);
94
95 while (blocks--) {
96 /* Remember old state */
97 so0 = s0;
98 so1 = s1;
99
100 /* Load data and transform */
101 m0 = Load(chunk);
102 QuadRound(s0, s1, m0, 0xe9b5dba5b5c0fbcfull, 0x71374491428a2f98ull);
103 m1 = Load(chunk + 16);
104 QuadRound(s0, s1, m1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull);
105 ShiftMessageA(m0, m1);
106 m2 = Load(chunk + 32);
107 QuadRound(s0, s1, m2, 0x550c7dc3243185beull, 0x12835b01d807aa98ull);
108 ShiftMessageA(m1, m2);
109 m3 = Load(chunk + 48);
110 QuadRound(s0, s1, m3, 0xc19bf1749bdc06a7ull, 0x80deb1fe72be5d74ull);
111 ShiftMessageB(m2, m3, m0);
112 QuadRound(s0, s1, m0, 0x240ca1cc0fc19dc6ull, 0xefbe4786E49b69c1ull);
113 ShiftMessageB(m3, m0, m1);
114 QuadRound(s0, s1, m1, 0x76f988da5cb0a9dcull, 0x4a7484aa2de92c6full);
115 ShiftMessageB(m0, m1, m2);
116 QuadRound(s0, s1, m2, 0xbf597fc7b00327c8ull, 0xa831c66d983e5152ull);
117 ShiftMessageB(m1, m2, m3);
118 QuadRound(s0, s1, m3, 0x1429296706ca6351ull, 0xd5a79147c6e00bf3ull);
119 ShiftMessageB(m2, m3, m0);
120 QuadRound(s0, s1, m0, 0x53380d134d2c6dfcull, 0x2e1b213827b70a85ull);
121 ShiftMessageB(m3, m0, m1);
122 QuadRound(s0, s1, m1, 0x92722c8581c2c92eull, 0x766a0abb650a7354ull);
123 ShiftMessageB(m0, m1, m2);
124 QuadRound(s0, s1, m2, 0xc76c51A3c24b8b70ull, 0xa81a664ba2bfe8a1ull);
125 ShiftMessageB(m1, m2, m3);
126 QuadRound(s0, s1, m3, 0x106aa070f40e3585ull, 0xd6990624d192e819ull);
127 ShiftMessageB(m2, m3, m0);
128 QuadRound(s0, s1, m0, 0x34b0bcb52748774cull, 0x1e376c0819a4c116ull);
129 ShiftMessageB(m3, m0, m1);
130 QuadRound(s0, s1, m1, 0x682e6ff35b9cca4full, 0x4ed8aa4a391c0cb3ull);
131 ShiftMessageC(m0, m1, m2);
132 QuadRound(s0, s1, m2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull);
133 ShiftMessageC(m1, m2, m3);
134 QuadRound(s0, s1, m3, 0xc67178f2bef9A3f7ull, 0xa4506ceb90befffaull);
135
136 /* Combine with old state */
137 s0 = _mm_add_epi32(s0, so0);
138 s1 = _mm_add_epi32(s1, so1);
139
140 /* Advance */
141 chunk += 64;
142 }
143
144 Unshuffle(s0, s1);
145 _mm_storeu_si128((__m128i*)s, s0);
146 _mm_storeu_si128((__m128i*)(s + 4), s1);
147 }
148 }
149
150 namespace sha256d64_x86_shani {
151
152 void Transform_2way(unsigned char* out, const unsigned char* in)
153 {
154 __m128i am0, am1, am2, am3, as0, as1, aso0, aso1;
155 __m128i bm0, bm1, bm2, bm3, bs0, bs1, bso0, bso1;
156
157 /* Transform 1 */
158 bs0 = as0 = _mm_load_si128((const __m128i*)INIT0);
159 bs1 = as1 = _mm_load_si128((const __m128i*)INIT1);
160 am0 = Load(in);
161 bm0 = Load(in + 64);
162 QuadRound(as0, as1, am0, 0xe9b5dba5b5c0fbcfull, 0x71374491428a2f98ull);
163 QuadRound(bs0, bs1, bm0, 0xe9b5dba5b5c0fbcfull, 0x71374491428a2f98ull);
164 am1 = Load(in + 16);
165 bm1 = Load(in + 80);
166 QuadRound(as0, as1, am1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull);
167 QuadRound(bs0, bs1, bm1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull);
168 ShiftMessageA(am0, am1);
169 ShiftMessageA(bm0, bm1);
170 am2 = Load(in + 32);
171 bm2 = Load(in + 96);
172 QuadRound(as0, as1, am2, 0x550c7dc3243185beull, 0x12835b01d807aa98ull);
173 QuadRound(bs0, bs1, bm2, 0x550c7dc3243185beull, 0x12835b01d807aa98ull);
174 ShiftMessageA(am1, am2);
175 ShiftMessageA(bm1, bm2);
176 am3 = Load(in + 48);
177 bm3 = Load(in + 112);
178 QuadRound(as0, as1, am3, 0xc19bf1749bdc06a7ull, 0x80deb1fe72be5d74ull);
179 QuadRound(bs0, bs1, bm3, 0xc19bf1749bdc06a7ull, 0x80deb1fe72be5d74ull);
180 ShiftMessageB(am2, am3, am0);
181 ShiftMessageB(bm2, bm3, bm0);
182 QuadRound(as0, as1, am0, 0x240ca1cc0fc19dc6ull, 0xefbe4786E49b69c1ull);
183 QuadRound(bs0, bs1, bm0, 0x240ca1cc0fc19dc6ull, 0xefbe4786E49b69c1ull);
184 ShiftMessageB(am3, am0, am1);
185 ShiftMessageB(bm3, bm0, bm1);
186 QuadRound(as0, as1, am1, 0x76f988da5cb0a9dcull, 0x4a7484aa2de92c6full);
187 QuadRound(bs0, bs1, bm1, 0x76f988da5cb0a9dcull, 0x4a7484aa2de92c6full);
188 ShiftMessageB(am0, am1, am2);
189 ShiftMessageB(bm0, bm1, bm2);
190 QuadRound(as0, as1, am2, 0xbf597fc7b00327c8ull, 0xa831c66d983e5152ull);
191 QuadRound(bs0, bs1, bm2, 0xbf597fc7b00327c8ull, 0xa831c66d983e5152ull);
192 ShiftMessageB(am1, am2, am3);
193 ShiftMessageB(bm1, bm2, bm3);
194 QuadRound(as0, as1, am3, 0x1429296706ca6351ull, 0xd5a79147c6e00bf3ull);
195 QuadRound(bs0, bs1, bm3, 0x1429296706ca6351ull, 0xd5a79147c6e00bf3ull);
196 ShiftMessageB(am2, am3, am0);
197 ShiftMessageB(bm2, bm3, bm0);
198 QuadRound(as0, as1, am0, 0x53380d134d2c6dfcull, 0x2e1b213827b70a85ull);
199 QuadRound(bs0, bs1, bm0, 0x53380d134d2c6dfcull, 0x2e1b213827b70a85ull);
200 ShiftMessageB(am3, am0, am1);
201 ShiftMessageB(bm3, bm0, bm1);
202 QuadRound(as0, as1, am1, 0x92722c8581c2c92eull, 0x766a0abb650a7354ull);
203 QuadRound(bs0, bs1, bm1, 0x92722c8581c2c92eull, 0x766a0abb650a7354ull);
204 ShiftMessageB(am0, am1, am2);
205 ShiftMessageB(bm0, bm1, bm2);
206 QuadRound(as0, as1, am2, 0xc76c51A3c24b8b70ull, 0xa81a664ba2bfe8a1ull);
207 QuadRound(bs0, bs1, bm2, 0xc76c51A3c24b8b70ull, 0xa81a664ba2bfe8a1ull);
208 ShiftMessageB(am1, am2, am3);
209 ShiftMessageB(bm1, bm2, bm3);
210 QuadRound(as0, as1, am3, 0x106aa070f40e3585ull, 0xd6990624d192e819ull);
211 QuadRound(bs0, bs1, bm3, 0x106aa070f40e3585ull, 0xd6990624d192e819ull);
212 ShiftMessageB(am2, am3, am0);
213 ShiftMessageB(bm2, bm3, bm0);
214 QuadRound(as0, as1, am0, 0x34b0bcb52748774cull, 0x1e376c0819a4c116ull);
215 QuadRound(bs0, bs1, bm0, 0x34b0bcb52748774cull, 0x1e376c0819a4c116ull);
216 ShiftMessageB(am3, am0, am1);
217 ShiftMessageB(bm3, bm0, bm1);
218 QuadRound(as0, as1, am1, 0x682e6ff35b9cca4full, 0x4ed8aa4a391c0cb3ull);
219 QuadRound(bs0, bs1, bm1, 0x682e6ff35b9cca4full, 0x4ed8aa4a391c0cb3ull);
220 ShiftMessageC(am0, am1, am2);
221 ShiftMessageC(bm0, bm1, bm2);
222 QuadRound(as0, as1, am2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull);
223 QuadRound(bs0, bs1, bm2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull);
224 ShiftMessageC(am1, am2, am3);
225 ShiftMessageC(bm1, bm2, bm3);
226 QuadRound(as0, as1, am3, 0xc67178f2bef9A3f7ull, 0xa4506ceb90befffaull);
227 QuadRound(bs0, bs1, bm3, 0xc67178f2bef9A3f7ull, 0xa4506ceb90befffaull);
228 as0 = _mm_add_epi32(as0, _mm_load_si128((const __m128i*)INIT0));
229 bs0 = _mm_add_epi32(bs0, _mm_load_si128((const __m128i*)INIT0));
230 as1 = _mm_add_epi32(as1, _mm_load_si128((const __m128i*)INIT1));
231 bs1 = _mm_add_epi32(bs1, _mm_load_si128((const __m128i*)INIT1));
232
233 /* Transform 2 */
234 aso0 = as0;
235 bso0 = bs0;
236 aso1 = as1;
237 bso1 = bs1;
238 QuadRound(as0, as1, 0xe9b5dba5b5c0fbcfull, 0x71374491c28a2f98ull);
239 QuadRound(bs0, bs1, 0xe9b5dba5b5c0fbcfull, 0x71374491c28a2f98ull);
240 QuadRound(as0, as1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull);
241 QuadRound(bs0, bs1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull);
242 QuadRound(as0, as1, 0x550c7dc3243185beull, 0x12835b01d807aa98ull);
243 QuadRound(bs0, bs1, 0x550c7dc3243185beull, 0x12835b01d807aa98ull);
244 QuadRound(as0, as1, 0xc19bf3749bdc06a7ull, 0x80deb1fe72be5d74ull);
245 QuadRound(bs0, bs1, 0xc19bf3749bdc06a7ull, 0x80deb1fe72be5d74ull);
246 QuadRound(as0, as1, 0x240cf2540fe1edc6ull, 0xf0fe4786649b69c1ull);
247 QuadRound(bs0, bs1, 0x240cf2540fe1edc6ull, 0xf0fe4786649b69c1ull);
248 QuadRound(as0, as1, 0x16f988fa61b9411eull, 0x6cc984be4fe9346full);
249 QuadRound(bs0, bs1, 0x16f988fa61b9411eull, 0x6cc984be4fe9346full);
250 QuadRound(as0, as1, 0xb9d99ec7b019fc65ull, 0xa88e5a6df2c65152ull);
251 QuadRound(bs0, bs1, 0xb9d99ec7b019fc65ull, 0xa88e5a6df2c65152ull);
252 QuadRound(as0, as1, 0xc7353eb0fdb1232bull, 0xe70eeaa09a1231c3ull);
253 QuadRound(bs0, bs1, 0xc7353eb0fdb1232bull, 0xe70eeaa09a1231c3ull);
254 QuadRound(as0, as1, 0xdc1eeefd5a0f118full, 0xcb976d5f3069bad5ull);
255 QuadRound(bs0, bs1, 0xdc1eeefd5a0f118full, 0xcb976d5f3069bad5ull);
256 QuadRound(as0, as1, 0xe15d5b1658f4ca9dull, 0xde0b7a040a35b689ull);
257 QuadRound(bs0, bs1, 0xe15d5b1658f4ca9dull, 0xde0b7a040a35b689ull);
258 QuadRound(as0, as1, 0x6fab9537a507ea32ull, 0x37088980007f3e86ull);
259 QuadRound(bs0, bs1, 0x6fab9537a507ea32ull, 0x37088980007f3e86ull);
260 QuadRound(as0, as1, 0xc0bbbe37cdaa3b6dull, 0x0d8cd6f117406110ull);
261 QuadRound(bs0, bs1, 0xc0bbbe37cdaa3b6dull, 0x0d8cd6f117406110ull);
262 QuadRound(as0, as1, 0x6fd15ca70b02e931ull, 0xdb48a36383613bdaull);
263 QuadRound(bs0, bs1, 0x6fd15ca70b02e931ull, 0xdb48a36383613bdaull);
264 QuadRound(as0, as1, 0x6d4378906ed41a95ull, 0x31338431521afacaull);
265 QuadRound(bs0, bs1, 0x6d4378906ed41a95ull, 0x31338431521afacaull);
266 QuadRound(as0, as1, 0x532fb63cb5c9a0e6ull, 0x9eccabbdc39c91f2ull);
267 QuadRound(bs0, bs1, 0x532fb63cb5c9a0e6ull, 0x9eccabbdc39c91f2ull);
268 QuadRound(as0, as1, 0x4c191d76a4954b68ull, 0x07237ea3d2c741c6ull);
269 QuadRound(bs0, bs1, 0x4c191d76a4954b68ull, 0x07237ea3d2c741c6ull);
270 as0 = _mm_add_epi32(as0, aso0);
271 bs0 = _mm_add_epi32(bs0, bso0);
272 as1 = _mm_add_epi32(as1, aso1);
273 bs1 = _mm_add_epi32(bs1, bso1);
274
275 /* Extract hash */
276 Unshuffle(as0, as1);
277 Unshuffle(bs0, bs1);
278 am0 = as0;
279 bm0 = bs0;
280 am1 = as1;
281 bm1 = bs1;
282
283 /* Transform 3 */
284 bs0 = as0 = _mm_load_si128((const __m128i*)INIT0);
285 bs1 = as1 = _mm_load_si128((const __m128i*)INIT1);
286 QuadRound(as0, as1, am0, 0xe9b5dba5B5c0fbcfull, 0x71374491428a2f98ull);
287 QuadRound(bs0, bs1, bm0, 0xe9b5dba5B5c0fbcfull, 0x71374491428a2f98ull);
288 QuadRound(as0, as1, am1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull);
289 QuadRound(bs0, bs1, bm1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull);
290 ShiftMessageA(am0, am1);
291 ShiftMessageA(bm0, bm1);
292 bm2 = am2 = _mm_set_epi64x(0x0ull, 0x80000000ull);
293 QuadRound(as0, as1, 0x550c7dc3243185beull, 0x12835b015807aa98ull);
294 QuadRound(bs0, bs1, 0x550c7dc3243185beull, 0x12835b015807aa98ull);
295 ShiftMessageA(am1, am2);
296 ShiftMessageA(bm1, bm2);
297 bm3 = am3 = _mm_set_epi64x(0x10000000000ull, 0x0ull);
298 QuadRound(as0, as1, 0xc19bf2749bdc06a7ull, 0x80deb1fe72be5d74ull);
299 QuadRound(bs0, bs1, 0xc19bf2749bdc06a7ull, 0x80deb1fe72be5d74ull);
300 ShiftMessageB(am2, am3, am0);
301 ShiftMessageB(bm2, bm3, bm0);
302 QuadRound(as0, as1, am0, 0x240ca1cc0fc19dc6ull, 0xefbe4786e49b69c1ull);
303 QuadRound(bs0, bs1, bm0, 0x240ca1cc0fc19dc6ull, 0xefbe4786e49b69c1ull);
304 ShiftMessageB(am3, am0, am1);
305 ShiftMessageB(bm3, bm0, bm1);
306 QuadRound(as0, as1, am1, 0x76f988da5cb0a9dcull, 0x4a7484aa2de92c6full);
307 QuadRound(bs0, bs1, bm1, 0x76f988da5cb0a9dcull, 0x4a7484aa2de92c6full);
308 ShiftMessageB(am0, am1, am2);
309 ShiftMessageB(bm0, bm1, bm2);
310 QuadRound(as0, as1, am2, 0xbf597fc7b00327c8ull, 0xa831c66d983e5152ull);
311 QuadRound(bs0, bs1, bm2, 0xbf597fc7b00327c8ull, 0xa831c66d983e5152ull);
312 ShiftMessageB(am1, am2, am3);
313 ShiftMessageB(bm1, bm2, bm3);
314 QuadRound(as0, as1, am3, 0x1429296706ca6351ull, 0xd5a79147c6e00bf3ull);
315 QuadRound(bs0, bs1, bm3, 0x1429296706ca6351ull, 0xd5a79147c6e00bf3ull);
316 ShiftMessageB(am2, am3, am0);
317 ShiftMessageB(bm2, bm3, bm0);
318 QuadRound(as0, as1, am0, 0x53380d134d2c6dfcull, 0x2e1b213827b70a85ull);
319 QuadRound(bs0, bs1, bm0, 0x53380d134d2c6dfcull, 0x2e1b213827b70a85ull);
320 ShiftMessageB(am3, am0, am1);
321 ShiftMessageB(bm3, bm0, bm1);
322 QuadRound(as0, as1, am1, 0x92722c8581c2c92eull, 0x766a0abb650a7354ull);
323 QuadRound(bs0, bs1, bm1, 0x92722c8581c2c92eull, 0x766a0abb650a7354ull);
324 ShiftMessageB(am0, am1, am2);
325 ShiftMessageB(bm0, bm1, bm2);
326 QuadRound(as0, as1, am2, 0xc76c51a3c24b8b70ull, 0xa81a664ba2bfe8A1ull);
327 QuadRound(bs0, bs1, bm2, 0xc76c51a3c24b8b70ull, 0xa81a664ba2bfe8A1ull);
328 ShiftMessageB(am1, am2, am3);
329 ShiftMessageB(bm1, bm2, bm3);
330 QuadRound(as0, as1, am3, 0x106aa070f40e3585ull, 0xd6990624d192e819ull);
331 QuadRound(bs0, bs1, bm3, 0x106aa070f40e3585ull, 0xd6990624d192e819ull);
332 ShiftMessageB(am2, am3, am0);
333 ShiftMessageB(bm2, bm3, bm0);
334 QuadRound(as0, as1, am0, 0x34b0bcb52748774cull, 0x1e376c0819a4c116ull);
335 QuadRound(bs0, bs1, bm0, 0x34b0bcb52748774cull, 0x1e376c0819a4c116ull);
336 ShiftMessageB(am3, am0, am1);
337 ShiftMessageB(bm3, bm0, bm1);
338 QuadRound(as0, as1, am1, 0x682e6ff35b9cca4full, 0x4ed8aa4a391c0cb3ull);
339 QuadRound(bs0, bs1, bm1, 0x682e6ff35b9cca4full, 0x4ed8aa4a391c0cb3ull);
340 ShiftMessageC(am0, am1, am2);
341 ShiftMessageC(bm0, bm1, bm2);
342 QuadRound(as0, as1, am2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull);
343 QuadRound(bs0, bs1, bm2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull);
344 ShiftMessageC(am1, am2, am3);
345 ShiftMessageC(bm1, bm2, bm3);
346 QuadRound(as0, as1, am3, 0xc67178f2bef9a3f7ull, 0xa4506ceb90befffaull);
347 QuadRound(bs0, bs1, bm3, 0xc67178f2bef9a3f7ull, 0xa4506ceb90befffaull);
348 as0 = _mm_add_epi32(as0, _mm_load_si128((const __m128i*)INIT0));
349 bs0 = _mm_add_epi32(bs0, _mm_load_si128((const __m128i*)INIT0));
350 as1 = _mm_add_epi32(as1, _mm_load_si128((const __m128i*)INIT1));
351 bs1 = _mm_add_epi32(bs1, _mm_load_si128((const __m128i*)INIT1));
352
353 /* Extract hash into out */
354 Unshuffle(as0, as1);
355 Unshuffle(bs0, bs1);
356 Save(out, as0);
357 Save(out + 16, as1);
358 Save(out + 32, bs0);
359 Save(out + 48, bs1);
360 }
361
362 }
363
364 #if defined(__clang__)
365 #pragma clang attribute pop
366 #endif
367
368 #endif
369