sha256_power8.cpp raw
1 // Copyright (c) 2017 The Limenka developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 //
5 // This is a translation to GCC extended asm syntax from YASM code by Intel
6
7 #include <stdint.h>
8
9 #include <altivec.h>
10
11 namespace sha256_power8
12 {
13
14 typedef __vector uint32_t uint32x4_p8;
15 typedef __vector uint8_t uint8x16_p8;
16
17 //! Gets the first uin32_t from a, b, c, d, converts from BE to host endian, and returns them concatenated
18 template<uint8_t OFFS> static inline uint32x4_p8 pack_bytes
19 (const uint8x16_p8 a, const uint8x16_p8 b, const uint8x16_p8 c, const uint8x16_p8 d) {
20 uint8x16_p8 perm1 = {0+OFFS,1+OFFS,2+OFFS,3+OFFS, 16+OFFS,17+OFFS,18+OFFS,19+OFFS, 0,0,0,0, 0,0,0,0};
21 #ifdef WORDS_BIGENDIAN
22 uint8x16_p8 perm2 = {0,1,2,3, 4,5,6,7, 16,17,18,19, 20,21,22,23};
23 #else
24 uint8x16_p8 perm2 = {3,2,1,0, 7,6,5,4, 19,18,17,16, 23,22,21,20};
25 #endif
26 return (uint32x4_p8)vec_perm(vec_perm((uint8x16_p8)a, (uint8x16_p8)b, perm1), vec_perm((uint8x16_p8)c, (uint8x16_p8)d, perm1), perm2);
27 }
28
29 static const __attribute__((aligned(16))) uint32_t K[] = {
30 0x428A2F98, 0x71374491, 0xB5C0FBCF, 0xE9B5DBA5,
31 0x3956C25B, 0x59F111F1, 0x923F82A4, 0xAB1C5ED5,
32 0xD807AA98, 0x12835B01, 0x243185BE, 0x550C7DC3,
33 0x72BE5D74, 0x80DEB1FE, 0x9BDC06A7, 0xC19BF174,
34 0xE49B69C1, 0xEFBE4786, 0x0FC19DC6, 0x240CA1CC,
35 0x2DE92C6F, 0x4A7484AA, 0x5CB0A9DC, 0x76F988DA,
36 0x983E5152, 0xA831C66D, 0xB00327C8, 0xBF597FC7,
37 0xC6E00BF3, 0xD5A79147, 0x06CA6351, 0x14292967,
38 0x27B70A85, 0x2E1B2138, 0x4D2C6DFC, 0x53380D13,
39 0x650A7354, 0x766A0ABB, 0x81C2C92E, 0x92722C85,
40 0xA2BFE8A1, 0xA81A664B, 0xC24B8B70, 0xC76C51A3,
41 0xD192E819, 0xD6990624, 0xF40E3585, 0x106AA070,
42 0x19A4C116, 0x1E376C08, 0x2748774C, 0x34B0BCB5,
43 0x391C0CB3, 0x4ED8AA4A, 0x5B9CCA4F, 0x682E6FF3,
44 0x748F82EE, 0x78A5636F, 0x84C87814, 0x8CC70208,
45 0x90BEFFFA, 0xA4506CEB, 0xBEF9A3F7, 0xC67178F2
46 };
47
48 #define Ch(x, y, z) vec_sel((z), (y), (x))
49 #define Maj(x, y, z) vec_sel((y), (z), vec_xor((x), (y)))
50
51 #define KRound(a, b, c, d, e, f, g, h, k, w) \
52 do { \
53 uint32x4_p8 t1 = h + Ch(e, f, g) + __builtin_crypto_vshasigmaw(e, 1, 0xf) + k + w; \
54 uint32x4_p8 t2 = Maj(a, b, c) + __builtin_crypto_vshasigmaw(a, 1, 0); \
55 d += t1; \
56 h = t1 + t2; \
57 } while(false);
58
59 #define Round(a, b, c, d, e, f, g, h, k, w) \
60 do { \
61 uint32x4_p8 kay = {k, k, k, k}; \
62 uint32x4_p8 t1 = h + Ch(e, f, g) + __builtin_crypto_vshasigmaw(e, 1, 0xf) + kay + w; \
63 uint32x4_p8 t2 = Maj(a, b, c) + __builtin_crypto_vshasigmaw(a, 1, 0); \
64 d += t1; \
65 h = t1 + t2; \
66 } while(false);
67
68 #define KRound2(a, b, c, d, e, f, g, h, k) \
69 do { \
70 uint32x4_p8 t1 = h + Ch(e, f, g) + __builtin_crypto_vshasigmaw(e, 1, 0xf) + k; \
71 uint32x4_p8 t2 = Maj(a, b, c) + __builtin_crypto_vshasigmaw(a, 1, 0); \
72 d += t1; \
73 h = t1 + t2; \
74 } while(false);
75
76 #define Round2(a, b, c, d, e, f, g, h, k) \
77 do { \
78 uint32x4_p8 kay = {k, k, k, k}; \
79 uint32x4_p8 t1 = h + Ch(e, f, g) + __builtin_crypto_vshasigmaw(e, 1, 0xf) + kay; \
80 uint32x4_p8 t2 = Maj(a, b, c) + __builtin_crypto_vshasigmaw(a, 1, 0); \
81 d += t1; \
82 h = t1 + t2; \
83 } while(false);
84
85 #define sigma0(w) __builtin_crypto_vshasigmaw(w, 0, 0)
86 #define sigma1(w) __builtin_crypto_vshasigmaw(w, 0, 0xf)
87
88 /** Perform a 4 double-SHA-256 64-byte updates at once*/
89 void Transform_4way(unsigned char* out, const unsigned char* in)
90 {
91 uint32x4_p8 a = {0x6a09e667ul, 0x6a09e667ul, 0x6a09e667ul, 0x6a09e667ul};
92 uint32x4_p8 b = {0xbb67ae85ul, 0xbb67ae85ul, 0xbb67ae85ul, 0xbb67ae85ul};
93 uint32x4_p8 c = {0x3c6ef372ul, 0x3c6ef372ul, 0x3c6ef372ul, 0x3c6ef372ul};
94 uint32x4_p8 d = {0xa54ff53aul, 0xa54ff53aul, 0xa54ff53aul, 0xa54ff53aul};
95 uint32x4_p8 e = {0x510e527ful, 0x510e527ful, 0x510e527ful, 0x510e527ful};
96 uint32x4_p8 f = {0x9b05688cul, 0x9b05688cul, 0x9b05688cul, 0x9b05688cul};
97 uint32x4_p8 g = {0x1f83d9abul, 0x1f83d9abul, 0x1f83d9abul, 0x1f83d9abul};
98 uint32x4_p8 h = {0x5be0cd19ul, 0x5be0cd19ul, 0x5be0cd19ul, 0x5be0cd19ul};
99
100 uint8x16_p8 w0123_0 = vec_vsx_ld(0 *16 + 0 , in);
101 uint8x16_p8 w4567_0 = vec_vsx_ld(1 *16 + 0 , in);
102 uint8x16_p8 w8901_0 = vec_vsx_ld(2 *16 + 0 , in);
103 uint8x16_p8 w2345_0 = vec_vsx_ld(3 *16 + 0 , in);
104
105 uint8x16_p8 w0123_1 = vec_vsx_ld(0 *16 + 64 , in);
106 uint8x16_p8 w4567_1 = vec_vsx_ld(1 *16 + 64 , in);
107 uint8x16_p8 w8901_1 = vec_vsx_ld(2 *16 + 64 , in);
108 uint8x16_p8 w2345_1 = vec_vsx_ld(3 *16 + 64 , in);
109
110 uint8x16_p8 w0123_2 = vec_vsx_ld(0 *16 + 128, in);
111 uint8x16_p8 w4567_2 = vec_vsx_ld(1 *16 + 128, in);
112 uint8x16_p8 w8901_2 = vec_vsx_ld(2 *16 + 128, in);
113 uint8x16_p8 w2345_2 = vec_vsx_ld(3 *16 + 128, in);
114
115 uint8x16_p8 w0123_3 = vec_vsx_ld(0 *16 + 192, in);
116 uint8x16_p8 w4567_3 = vec_vsx_ld(1 *16 + 192, in);
117 uint8x16_p8 w8901_3 = vec_vsx_ld(2 *16 + 192, in);
118 uint8x16_p8 w2345_3 = vec_vsx_ld(3 *16 + 192, in);
119
120 uint32x4_p8 w0 = pack_bytes<0 >(w0123_0, w0123_1, w0123_2, w0123_3);
121 uint32x4_p8 w1 = pack_bytes<4 >(w0123_0, w0123_1, w0123_2, w0123_3);
122 uint32x4_p8 w2 = pack_bytes<8 >(w0123_0, w0123_1, w0123_2, w0123_3);
123 uint32x4_p8 w3 = pack_bytes<12>(w0123_0, w0123_1, w0123_2, w0123_3);
124
125 uint32x4_p8 w4 = pack_bytes<0 >(w4567_0, w4567_1, w4567_2, w4567_3);
126 uint32x4_p8 w5 = pack_bytes<4 >(w4567_0, w4567_1, w4567_2, w4567_3);
127 uint32x4_p8 w6 = pack_bytes<8 >(w4567_0, w4567_1, w4567_2, w4567_3);
128 uint32x4_p8 w7 = pack_bytes<12>(w4567_0, w4567_1, w4567_2, w4567_3);
129
130 uint32x4_p8 w8 = pack_bytes<0 >(w8901_0, w8901_1, w8901_2, w8901_3);
131 uint32x4_p8 w9 = pack_bytes<4 >(w8901_0, w8901_1, w8901_2, w8901_3);
132 uint32x4_p8 w10 = pack_bytes<8 >(w8901_0, w8901_1, w8901_2, w8901_3);
133 uint32x4_p8 w11 = pack_bytes<12>(w8901_0, w8901_1, w8901_2, w8901_3);
134
135 uint32x4_p8 w12 = pack_bytes<0 >(w2345_0, w2345_1, w2345_2, w2345_3);
136 uint32x4_p8 w13 = pack_bytes<4 >(w2345_0, w2345_1, w2345_2, w2345_3);
137 uint32x4_p8 w14 = pack_bytes<8 >(w2345_0, w2345_1, w2345_2, w2345_3);
138 uint32x4_p8 w15 = pack_bytes<12>(w2345_0, w2345_1, w2345_2, w2345_3);
139
140 uint32x4_p8 k = (uint32x4_p8)vec_ld(0, K);
141 KRound(a, b, c, d, e, f, g, h, vec_splat(k, 0), w0);
142 KRound(h, a, b, c, d, e, f, g, vec_splat(k, 1), w1);
143 KRound(g, h, a, b, c, d, e, f, vec_splat(k, 2), w2);
144 KRound(f, g, h, a, b, c, d, e, vec_splat(k, 3), w3);
145 k = (uint32x4_p8)vec_ld(1*16, K);
146 KRound(e, f, g, h, a, b, c, d, vec_splat(k, 0), w4);
147 KRound(d, e, f, g, h, a, b, c, vec_splat(k, 1), w5);
148 KRound(c, d, e, f, g, h, a, b, vec_splat(k, 2), w6);
149 KRound(b, c, d, e, f, g, h, a, vec_splat(k, 3), w7);
150 k = (uint32x4_p8)vec_ld(2*16, K);
151 KRound(a, b, c, d, e, f, g, h, vec_splat(k, 0), w8);
152 KRound(h, a, b, c, d, e, f, g, vec_splat(k, 1), w9);
153 KRound(g, h, a, b, c, d, e, f, vec_splat(k, 2), w10);
154 KRound(f, g, h, a, b, c, d, e, vec_splat(k, 3), w11);
155 k = (uint32x4_p8)vec_ld(3*16, K);
156 KRound(e, f, g, h, a, b, c, d, vec_splat(k, 0), w12);
157 KRound(d, e, f, g, h, a, b, c, vec_splat(k, 1), w13);
158 KRound(c, d, e, f, g, h, a, b, vec_splat(k, 2), w14);
159 KRound(b, c, d, e, f, g, h, a, vec_splat(k, 3), w15);
160
161 for (int i = 0; i < 3; i++) {
162 k = (uint32x4_p8)vec_ld((4+4*i)*16, K);
163 KRound(a, b, c, d, e, f, g, h, vec_splat(k, 0), (w0 += sigma1(w14) + w9 + sigma0(w1)));
164 KRound(h, a, b, c, d, e, f, g, vec_splat(k, 1), (w1 += sigma1(w15) + w10 + sigma0(w2)));
165 KRound(g, h, a, b, c, d, e, f, vec_splat(k, 2), (w2 += sigma1(w0) + w11 + sigma0(w3)));
166 KRound(f, g, h, a, b, c, d, e, vec_splat(k, 3), (w3 += sigma1(w1) + w12 + sigma0(w4)));
167 k = (uint32x4_p8)vec_ld((5+4*i)*16, K);
168 KRound(e, f, g, h, a, b, c, d, vec_splat(k, 0), (w4 += sigma1(w2) + w13 + sigma0(w5)));
169 KRound(d, e, f, g, h, a, b, c, vec_splat(k, 1), (w5 += sigma1(w3) + w14 + sigma0(w6)));
170 KRound(c, d, e, f, g, h, a, b, vec_splat(k, 2), (w6 += sigma1(w4) + w15 + sigma0(w7)));
171 KRound(b, c, d, e, f, g, h, a, vec_splat(k, 3), (w7 += sigma1(w5) + w0 + sigma0(w8)));
172 k = (uint32x4_p8)vec_ld((6+4*i)*16, K);
173 KRound(a, b, c, d, e, f, g, h, vec_splat(k, 0), (w8 += sigma1(w6) + w1 + sigma0(w9)));
174 KRound(h, a, b, c, d, e, f, g, vec_splat(k, 1), (w9 += sigma1(w7) + w2 + sigma0(w10)));
175 KRound(g, h, a, b, c, d, e, f, vec_splat(k, 2), (w10 += sigma1(w8) + w3 + sigma0(w11)));
176 KRound(f, g, h, a, b, c, d, e, vec_splat(k, 3), (w11 += sigma1(w9) + w4 + sigma0(w12)));
177 k = (uint32x4_p8)vec_ld((7+4*i)*16, K);
178 KRound(e, f, g, h, a, b, c, d, vec_splat(k, 0), (w12 += sigma1(w10) + w5 + sigma0(w13)));
179 KRound(d, e, f, g, h, a, b, c, vec_splat(k, 1), (w13 += sigma1(w11) + w6 + sigma0(w14)));
180 KRound(c, d, e, f, g, h, a, b, vec_splat(k, 2), (w14 += sigma1(w12) + w7 + sigma0(w15)));
181 KRound(b, c, d, e, f, g, h, a, vec_splat(k, 3), (w15 += sigma1(w13) + w8 + sigma0(w0)));
182 }
183
184 a += uint32x4_p8{0x6a09e667ul, 0x6a09e667ul, 0x6a09e667ul, 0x6a09e667ul};
185 b += uint32x4_p8{0xbb67ae85ul, 0xbb67ae85ul, 0xbb67ae85ul, 0xbb67ae85ul};
186 c += uint32x4_p8{0x3c6ef372ul, 0x3c6ef372ul, 0x3c6ef372ul, 0x3c6ef372ul};
187 d += uint32x4_p8{0xa54ff53aul, 0xa54ff53aul, 0xa54ff53aul, 0xa54ff53aul};
188 e += uint32x4_p8{0x510e527ful, 0x510e527ful, 0x510e527ful, 0x510e527ful};
189 f += uint32x4_p8{0x9b05688cul, 0x9b05688cul, 0x9b05688cul, 0x9b05688cul};
190 g += uint32x4_p8{0x1f83d9abul, 0x1f83d9abul, 0x1f83d9abul, 0x1f83d9abul};
191 h += uint32x4_p8{0x5be0cd19ul, 0x5be0cd19ul, 0x5be0cd19ul, 0x5be0cd19ul};
192
193 uint32x4_p8 t0 = a;
194 uint32x4_p8 t1 = b;
195 uint32x4_p8 t2 = c;
196 uint32x4_p8 t3 = d;
197 uint32x4_p8 t4 = e;
198 uint32x4_p8 t5 = f;
199 uint32x4_p8 t6 = g;
200 uint32x4_p8 t7 = h;
201
202 KRound2(a, b, c, d, e, f, g, h, 0xc28a2f98);
203 KRound2(h, a, b, c, d, e, f, g, 0x71374491);
204 KRound2(g, h, a, b, c, d, e, f, 0xb5c0fbcf);
205 KRound2(f, g, h, a, b, c, d, e, 0xe9b5dba5);
206 KRound2(e, f, g, h, a, b, c, d, 0x3956c25b);
207 KRound2(d, e, f, g, h, a, b, c, 0x59f111f1);
208 KRound2(c, d, e, f, g, h, a, b, 0x923f82a4);
209 KRound2(b, c, d, e, f, g, h, a, 0xab1c5ed5);
210 KRound2(a, b, c, d, e, f, g, h, 0xd807aa98);
211 KRound2(h, a, b, c, d, e, f, g, 0x12835b01);
212 KRound2(g, h, a, b, c, d, e, f, 0x243185be);
213 KRound2(f, g, h, a, b, c, d, e, 0x550c7dc3);
214 KRound2(e, f, g, h, a, b, c, d, 0x72be5d74);
215 KRound2(d, e, f, g, h, a, b, c, 0x80deb1fe);
216 KRound2(c, d, e, f, g, h, a, b, 0x9bdc06a7);
217 KRound2(b, c, d, e, f, g, h, a, 0xc19bf374);
218 KRound2(a, b, c, d, e, f, g, h, 0x649b69c1);
219 KRound2(h, a, b, c, d, e, f, g, 0xf0fe4786);
220 KRound2(g, h, a, b, c, d, e, f, 0x0fe1edc6);
221 KRound2(f, g, h, a, b, c, d, e, 0x240cf254);
222 KRound2(e, f, g, h, a, b, c, d, 0x4fe9346f);
223 KRound2(d, e, f, g, h, a, b, c, 0x6cc984be);
224 KRound2(c, d, e, f, g, h, a, b, 0x61b9411e);
225 KRound2(b, c, d, e, f, g, h, a, 0x16f988fa);
226 KRound2(a, b, c, d, e, f, g, h, 0xf2c65152);
227 KRound2(h, a, b, c, d, e, f, g, 0xa88e5a6d);
228 KRound2(g, h, a, b, c, d, e, f, 0xb019fc65);
229 KRound2(f, g, h, a, b, c, d, e, 0xb9d99ec7);
230 KRound2(e, f, g, h, a, b, c, d, 0x9a1231c3);
231 KRound2(d, e, f, g, h, a, b, c, 0xe70eeaa0);
232 KRound2(c, d, e, f, g, h, a, b, 0xfdb1232b);
233 KRound2(b, c, d, e, f, g, h, a, 0xc7353eb0);
234 KRound2(a, b, c, d, e, f, g, h, 0x3069bad5);
235 KRound2(h, a, b, c, d, e, f, g, 0xcb976d5f);
236 KRound2(g, h, a, b, c, d, e, f, 0x5a0f118f);
237 KRound2(f, g, h, a, b, c, d, e, 0xdc1eeefd);
238 KRound2(e, f, g, h, a, b, c, d, 0x0a35b689);
239 KRound2(d, e, f, g, h, a, b, c, 0xde0b7a04);
240 KRound2(c, d, e, f, g, h, a, b, 0x58f4ca9d);
241 KRound2(b, c, d, e, f, g, h, a, 0xe15d5b16);
242 KRound2(a, b, c, d, e, f, g, h, 0x007f3e86);
243 KRound2(h, a, b, c, d, e, f, g, 0x37088980);
244 KRound2(g, h, a, b, c, d, e, f, 0xa507ea32);
245 KRound2(f, g, h, a, b, c, d, e, 0x6fab9537);
246 KRound2(e, f, g, h, a, b, c, d, 0x17406110);
247 KRound2(d, e, f, g, h, a, b, c, 0x0d8cd6f1);
248 KRound2(c, d, e, f, g, h, a, b, 0xcdaa3b6d);
249 KRound2(b, c, d, e, f, g, h, a, 0xc0bbbe37);
250 KRound2(a, b, c, d, e, f, g, h, 0x83613bda);
251 KRound2(h, a, b, c, d, e, f, g, 0xdb48a363);
252 KRound2(g, h, a, b, c, d, e, f, 0x0b02e931);
253 KRound2(f, g, h, a, b, c, d, e, 0x6fd15ca7);
254 KRound2(e, f, g, h, a, b, c, d, 0x521afaca);
255 KRound2(d, e, f, g, h, a, b, c, 0x31338431);
256 KRound2(c, d, e, f, g, h, a, b, 0x6ed41a95);
257 KRound2(b, c, d, e, f, g, h, a, 0x6d437890);
258 KRound2(a, b, c, d, e, f, g, h, 0xc39c91f2);
259 KRound2(h, a, b, c, d, e, f, g, 0x9eccabbd);
260 KRound2(g, h, a, b, c, d, e, f, 0xb5c9a0e6);
261 KRound2(f, g, h, a, b, c, d, e, 0x532fb63c);
262 KRound2(e, f, g, h, a, b, c, d, 0xd2c741c6);
263 KRound2(d, e, f, g, h, a, b, c, 0x07237ea3);
264 KRound2(c, d, e, f, g, h, a, b, 0xa4954b68);
265 KRound2(b, c, d, e, f, g, h, a, 0x4c191d76);
266
267
268 w0 = t0 + a;
269 w1 = t1 + b;
270 w2 = t2 + c;
271 w3 = t3 + d;
272 w4 = t4 + e;
273 w5 = t5 + f;
274 w6 = t6 + g;
275 w7 = t7 + h;
276
277 a = uint32x4_p8{0x6a09e667ul, 0x6a09e667ul, 0x6a09e667ul, 0x6a09e667ul};
278 b = uint32x4_p8{0xbb67ae85ul, 0xbb67ae85ul, 0xbb67ae85ul, 0xbb67ae85ul};
279 c = uint32x4_p8{0x3c6ef372ul, 0x3c6ef372ul, 0x3c6ef372ul, 0x3c6ef372ul};
280 d = uint32x4_p8{0xa54ff53aul, 0xa54ff53aul, 0xa54ff53aul, 0xa54ff53aul};
281 e = uint32x4_p8{0x510e527ful, 0x510e527ful, 0x510e527ful, 0x510e527ful};
282 f = uint32x4_p8{0x9b05688cul, 0x9b05688cul, 0x9b05688cul, 0x9b05688cul};
283 g = uint32x4_p8{0x1f83d9abul, 0x1f83d9abul, 0x1f83d9abul, 0x1f83d9abul};
284 h = uint32x4_p8{0x5be0cd19ul, 0x5be0cd19ul, 0x5be0cd19ul, 0x5be0cd19ul};
285
286 Round(a, b, c, d, e, f, g, h, 0x428a2f98, w0);
287 Round(h, a, b, c, d, e, f, g, 0x71374491, w1);
288 Round(g, h, a, b, c, d, e, f, 0xb5c0fbcf, w2);
289 Round(f, g, h, a, b, c, d, e, 0xe9b5dba5, w3);
290 Round(e, f, g, h, a, b, c, d, 0x3956c25b, w4);
291 Round(d, e, f, g, h, a, b, c, 0x59f111f1, w5);
292 Round(c, d, e, f, g, h, a, b, 0x923f82a4, w6);
293 Round(b, c, d, e, f, g, h, a, 0xab1c5ed5, w7);
294 Round2(a, b, c, d, e, f, g, h, 0x5807aa98);
295 Round2(h, a, b, c, d, e, f, g, 0x12835b01);
296 Round2(g, h, a, b, c, d, e, f, 0x243185be);
297 Round2(f, g, h, a, b, c, d, e, 0x550c7dc3);
298 Round2(e, f, g, h, a, b, c, d, 0x72be5d74);
299 Round2(d, e, f, g, h, a, b, c, 0x80deb1fe);
300 Round2(c, d, e, f, g, h, a, b, 0x9bdc06a7);
301 Round2(b, c, d, e, f, g, h, a, 0xc19bf274);
302 Round(a, b, c, d, e, f, g, h, 0xe49b69c1, (w0 += sigma0(w1)));
303 w1 += uint32x4_p8{0xa00000, 0xa00000, 0xa00000, 0xa00000};
304 Round(h, a, b, c, d, e, f, g, 0xefbe4786, (w1 += sigma0(w2)));
305 Round(g, h, a, b, c, d, e, f, 0x0fc19dc6, (w2 += sigma1(w0) + sigma0(w3)));
306 Round(f, g, h, a, b, c, d, e, 0x240ca1cc, (w3 += sigma1(w1) + sigma0(w4)));
307 Round(e, f, g, h, a, b, c, d, 0x2de92c6f, (w4 += sigma1(w2) + sigma0(w5)));
308 Round(d, e, f, g, h, a, b, c, 0x4a7484aa, (w5 += sigma1(w3) + sigma0(w6)));
309 w6 += uint32x4_p8{0x100, 0x100, 0x100, 0x100};
310 Round(c, d, e, f, g, h, a, b, 0x5cb0a9dc, (w6 += sigma1(w4) + sigma0(w7)));
311 w7 += uint32x4_p8{0x11002000, 0x11002000, 0x11002000, 0x11002000};
312 Round(b, c, d, e, f, g, h, a, 0x76f988da, (w7 += sigma1(w5) + w0));
313 w8 = uint32x4_p8{0x80000000, 0x80000000, 0x80000000, 0x80000000};
314 Round(a, b, c, d, e, f, g, h, 0x983e5152, (w8 += sigma1(w6) + w1));
315 Round(h, a, b, c, d, e, f, g, 0xa831c66d, (w9 = sigma1(w7) + w2));
316 Round(g, h, a, b, c, d, e, f, 0xb00327c8, (w10 = sigma1(w8) + w3));
317 Round(f, g, h, a, b, c, d, e, 0xbf597fc7, (w11 = sigma1(w9) + w4));
318 Round(e, f, g, h, a, b, c, d, 0xc6e00bf3, (w12 = sigma1(w10) + w5));
319 Round(d, e, f, g, h, a, b, c, 0xd5a79147, (w13 = sigma1(w11) + w6));
320 w14 = uint32x4_p8{0x400022, 0x400022, 0x400022, 0x400022};
321 Round(c, d, e, f, g, h, a, b, 0x06ca6351, (w14 += sigma1(w12) + w7));
322 w15 = uint32x4_p8{0x100, 0x100, 0x100, 0x100};
323 Round(b, c, d, e, f, g, h, a, 0x14292967, (w15 += sigma1(w13) + w8 + sigma0(w0)));
324 Round(a, b, c, d, e, f, g, h, 0x27b70a85, (w0 += sigma1(w14) + w9 + sigma0(w1)));
325 Round(h, a, b, c, d, e, f, g, 0x2e1b2138, (w1 += sigma1(w15) + w10 + sigma0(w2)));
326 Round(g, h, a, b, c, d, e, f, 0x4d2c6dfc, (w2 += sigma1(w0) + w11 + sigma0(w3)));
327 Round(f, g, h, a, b, c, d, e, 0x53380d13, (w3 += sigma1(w1) + w12 + sigma0(w4)));
328 Round(e, f, g, h, a, b, c, d, 0x650a7354, (w4 += sigma1(w2) + w13 + sigma0(w5)));
329 Round(d, e, f, g, h, a, b, c, 0x766a0abb, (w5 += sigma1(w3) + w14 + sigma0(w6)));
330 Round(c, d, e, f, g, h, a, b, 0x81c2c92e, (w6 += sigma1(w4) + w15 + sigma0(w7)));
331 Round(b, c, d, e, f, g, h, a, 0x92722c85, (w7 += sigma1(w5) + w0 + sigma0(w8)));
332 Round(a, b, c, d, e, f, g, h, 0xa2bfe8a1, (w8 += sigma1(w6) + w1 + sigma0(w9)));
333 Round(h, a, b, c, d, e, f, g, 0xa81a664b, (w9 += sigma1(w7) + w2 + sigma0(w10)));
334 Round(g, h, a, b, c, d, e, f, 0xc24b8b70, (w10 += sigma1(w8) + w3 + sigma0(w11)));
335 Round(f, g, h, a, b, c, d, e, 0xc76c51a3, (w11 += sigma1(w9) + w4 + sigma0(w12)));
336 Round(e, f, g, h, a, b, c, d, 0xd192e819, (w12 += sigma1(w10) + w5 + sigma0(w13)));
337 Round(d, e, f, g, h, a, b, c, 0xd6990624, (w13 += sigma1(w11) + w6 + sigma0(w14)));
338 Round(c, d, e, f, g, h, a, b, 0xf40e3585, (w14 += sigma1(w12) + w7 + sigma0(w15)));
339 Round(b, c, d, e, f, g, h, a, 0x106aa070, (w15 += sigma1(w13) + w8 + sigma0(w0)));
340 Round(a, b, c, d, e, f, g, h, 0x19a4c116, (w0 += sigma1(w14) + w9 + sigma0(w1)));
341 Round(h, a, b, c, d, e, f, g, 0x1e376c08, (w1 += sigma1(w15) + w10 + sigma0(w2)));
342 Round(g, h, a, b, c, d, e, f, 0x2748774c, (w2 += sigma1(w0) + w11 + sigma0(w3)));
343 Round(f, g, h, a, b, c, d, e, 0x34b0bcb5, (w3 += sigma1(w1) + w12 + sigma0(w4)));
344 Round(e, f, g, h, a, b, c, d, 0x391c0cb3, (w4 += sigma1(w2) + w13 + sigma0(w5)));
345 Round(d, e, f, g, h, a, b, c, 0x4ed8aa4a, (w5 += sigma1(w3) + w14 + sigma0(w6)));
346 Round(c, d, e, f, g, h, a, b, 0x5b9cca4f, (w6 += sigma1(w4) + w15 + sigma0(w7)));
347 Round(b, c, d, e, f, g, h, a, 0x682e6ff3, (w7 += sigma1(w5) + w0 + sigma0(w8)));
348 Round(a, b, c, d, e, f, g, h, 0x748f82ee, (w8 += sigma1(w6) + w1 + sigma0(w9)));
349 Round(h, a, b, c, d, e, f, g, 0x78a5636f, (w9 += sigma1(w7) + w2 + sigma0(w10)));
350 Round(g, h, a, b, c, d, e, f, 0x84c87814, (w10 += sigma1(w8) + w3 + sigma0(w11)));
351 Round(f, g, h, a, b, c, d, e, 0x8cc70208, (w11 += sigma1(w9) + w4 + sigma0(w12)));
352 Round(e, f, g, h, a, b, c, d, 0x90befffa, (w12 += sigma1(w10) + w5 + sigma0(w13)));
353 Round(d, e, f, g, h, a, b, c, 0xa4506ceb, (w13 += sigma1(w11) + w6 + sigma0(w14)));
354 Round(c, d, e, f, g, h, a, b, 0xbef9a3f7, (w14 + sigma1(w12) + w7 + sigma0(w15)));
355 Round(b, c, d, e, f, g, h, a, 0xc67178f2, (w15 + sigma1(w13) + w8 + sigma0(w0)));
356
357 a += uint32x4_p8{0x6a09e667ul, 0x6a09e667ul, 0x6a09e667ul, 0x6a09e667ul};
358 b += uint32x4_p8{0xbb67ae85ul, 0xbb67ae85ul, 0xbb67ae85ul, 0xbb67ae85ul};
359 c += uint32x4_p8{0x3c6ef372ul, 0x3c6ef372ul, 0x3c6ef372ul, 0x3c6ef372ul};
360 d += uint32x4_p8{0xa54ff53aul, 0xa54ff53aul, 0xa54ff53aul, 0xa54ff53aul};
361 e += uint32x4_p8{0x510e527ful, 0x510e527ful, 0x510e527ful, 0x510e527ful};
362 f += uint32x4_p8{0x9b05688cul, 0x9b05688cul, 0x9b05688cul, 0x9b05688cul};
363 g += uint32x4_p8{0x1f83d9abul, 0x1f83d9abul, 0x1f83d9abul, 0x1f83d9abul};
364 h += uint32x4_p8{0x5be0cd19ul, 0x5be0cd19ul, 0x5be0cd19ul, 0x5be0cd19ul};
365
366 w0123_0 = (uint8x16_p8)pack_bytes<0 >((uint8x16_p8)a, (uint8x16_p8)b, (uint8x16_p8)c, (uint8x16_p8)d);
367 w4567_0 = (uint8x16_p8)pack_bytes<0 >((uint8x16_p8)e, (uint8x16_p8)f, (uint8x16_p8)g, (uint8x16_p8)h);
368
369 w0123_1 = (uint8x16_p8)pack_bytes<4 >((uint8x16_p8)a, (uint8x16_p8)b, (uint8x16_p8)c, (uint8x16_p8)d);
370 w4567_1 = (uint8x16_p8)pack_bytes<4 >((uint8x16_p8)e, (uint8x16_p8)f, (uint8x16_p8)g, (uint8x16_p8)h);
371
372 w0123_2 = (uint8x16_p8)pack_bytes<8 >((uint8x16_p8)a, (uint8x16_p8)b, (uint8x16_p8)c, (uint8x16_p8)d);
373 w4567_2 = (uint8x16_p8)pack_bytes<8 >((uint8x16_p8)e, (uint8x16_p8)f, (uint8x16_p8)g, (uint8x16_p8)h);
374
375 w0123_3 = (uint8x16_p8)pack_bytes<12>((uint8x16_p8)a, (uint8x16_p8)b, (uint8x16_p8)c, (uint8x16_p8)d);
376 w4567_3 = (uint8x16_p8)pack_bytes<12>((uint8x16_p8)e, (uint8x16_p8)f, (uint8x16_p8)g, (uint8x16_p8)h);
377
378 vec_vsx_st(w0123_0, 0 *16 + 0 , out);
379 vec_vsx_st(w4567_0, 1 *16 + 0 , out);
380
381 vec_vsx_st(w0123_1, 0 *16 + 32, out);
382 vec_vsx_st(w4567_1, 1 *16 + 32, out);
383
384 vec_vsx_st(w0123_2, 0 *16 + 64, out);
385 vec_vsx_st(w4567_2, 1 *16 + 64, out);
386
387 vec_vsx_st(w0123_3, 0 *16 + 96, out);
388 vec_vsx_st(w4567_3, 1 *16 + 96, out);
389 }
390 }
391