1 // Copyright 2016 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4 5 //go:build !purego
6 7 package gcm
8 9 import (
10 "crypto/internal/fips140/aes"
11 "crypto/internal/fips140/subtle"
12 "crypto/internal/fips140deps/byteorder"
13 "crypto/internal/fips140deps/cpu"
14 "crypto/internal/impl"
15 )
16 17 // This file contains two implementations of AES-GCM. The first implementation
18 // (useGHASH) uses the KMCTR instruction to encrypt using AES in counter mode
19 // and the KIMD instruction for GHASH. The second implementation (useGCM) uses
20 // the newer KMA instruction which performs both operations (but still requires
21 // KIMD to hash large nonces).
22 23 // Keep in sync with crypto/tls.hasAESGCMHardwareSupport.
24 var useGHASH = cpu.S390XHasAES && cpu.S390XHasAESCTR && cpu.S390XHasGHASH
25 var useGCM = useGHASH && cpu.S390XHasAESGCM
26 27 func init() {
28 impl.Register("gcm", "CPACF/KIMD", &useGHASH)
29 impl.Register("gcm", "CPACF/KMA", &useGCM)
30 }
31 32 func checkGenericIsExpected() {
33 if useGHASH || useGCM {
34 panic("gcm: internal error: using generic implementation despite hardware support")
35 }
36 }
37 38 // gcmLengths writes len0 || len1 as big-endian values to a 16-byte array.
39 func gcmLengths(len0, len1 uint64) [16]byte {
40 v := [16]byte{}
41 byteorder.BEPutUint64(v[0:], len0)
42 byteorder.BEPutUint64(v[8:], len1)
43 return v
44 }
45 46 // gcmHashKey represents the 16-byte hash key required by the GHASH algorithm.
47 type gcmHashKey [16]byte
48 49 type gcmPlatformData struct {
50 hashKey gcmHashKey
51 }
52 53 func initGCM(g *GCM) {
54 if !useGCM && !useGHASH {
55 return
56 }
57 // Note that hashKey is also used in the KMA codepath to hash large nonces.
58 aes.EncryptBlockInternal(&g.cipher, g.hashKey[:], g.hashKey[:])
59 }
60 61 // ghashAsm uses the GHASH algorithm to hash data with the given key. The initial
62 // hash value is given by hash which will be updated with the new hash value.
63 // The length of data must be a multiple of 16-bytes.
64 //
65 //go:noescape
66 func ghashAsm(key *gcmHashKey, hash *[16]byte, data []byte)
67 68 // paddedGHASH pads data with zeroes until its length is a multiple of
69 // 16-bytes. It then calculates a new value for hash using the GHASH algorithm.
70 func paddedGHASH(hashKey *gcmHashKey, hash *[16]byte, data []byte) {
71 siz := len(data) &^ 0xf // align size to 16-bytes
72 if siz > 0 {
73 ghashAsm(hashKey, hash, data[:siz])
74 data = data[siz:]
75 }
76 if len(data) > 0 {
77 var s [16]byte
78 copy(s[:], data)
79 ghashAsm(hashKey, hash, s[:])
80 }
81 }
82 83 // cryptBlocksGCM encrypts src using AES in counter mode using the given
84 // function code and key. The rightmost 32-bits of the counter are incremented
85 // between each block as required by the GCM spec. The initial counter value
86 // is given by cnt, which is updated with the value of the next counter value
87 // to use.
88 //
89 // The lengths of both dst and buf must be greater than or equal to the length
90 // of src. buf may be partially or completely overwritten during the execution
91 // of the function.
92 //
93 //go:noescape
94 func cryptBlocksGCM(fn int, key, dst, src, buf []byte, cnt *[gcmBlockSize]byte)
95 96 // counterCrypt encrypts src using AES in counter mode and places the result
97 // into dst. cnt is the initial count value and will be updated with the next
98 // count value. The length of dst must be greater than or equal to the length
99 // of src.
100 func counterCrypt(g *GCM, dst, src []byte, cnt *[gcmBlockSize]byte) {
101 // Copying src into a buffer improves performance on some models when
102 // src and dst point to the same underlying array. We also need a
103 // buffer for counter values.
104 var ctrbuf, srcbuf [2048]byte
105 for len(src) >= 16 {
106 siz := len(src)
107 if len(src) > len(ctrbuf) {
108 siz = len(ctrbuf)
109 }
110 siz &^= 0xf // align siz to 16-bytes
111 copy(srcbuf[:], src[:siz])
112 cryptBlocksGCM(aes.BlockFunction(&g.cipher), aes.BlockKey(&g.cipher), dst[:siz], srcbuf[:siz], ctrbuf[:], cnt)
113 src = src[siz:]
114 dst = dst[siz:]
115 }
116 if len(src) > 0 {
117 var x [16]byte
118 aes.EncryptBlockInternal(&g.cipher, x[:], cnt[:])
119 for i := range src {
120 dst[i] = src[i] ^ x[i]
121 }
122 gcmInc32(cnt)
123 }
124 }
125 126 // deriveCounter computes the initial GCM counter state from the given nonce.
127 // See NIST SP 800-38D, section 7.1 and deriveCounterGeneric in gcm_generic.go.
128 func deriveCounter(H *gcmHashKey, counter *[gcmBlockSize]byte, nonce []byte) {
129 if len(nonce) == gcmStandardNonceSize {
130 copy(counter[:], nonce)
131 counter[gcmBlockSize-1] = 1
132 } else {
133 var hash [16]byte
134 paddedGHASH(H, &hash, nonce)
135 lens := gcmLengths(0, uint64(len(nonce))*8)
136 paddedGHASH(H, &hash, lens[:])
137 copy(counter[:], hash[:])
138 }
139 }
140 141 // gcmAuth calculates GHASH(additionalData, ciphertext), masks the result
142 // with tagMask and writes the result to out.
143 func gcmAuth(out []byte, H *gcmHashKey, tagMask *[gcmBlockSize]byte, ciphertext, additionalData []byte) {
144 var hash [16]byte
145 paddedGHASH(H, &hash, additionalData)
146 paddedGHASH(H, &hash, ciphertext)
147 lens := gcmLengths(uint64(len(additionalData))*8, uint64(len(ciphertext))*8)
148 paddedGHASH(H, &hash, lens[:])
149 150 copy(out, hash[:])
151 for i := range out {
152 out[i] ^= tagMask[i]
153 }
154 }
155 156 func seal(out []byte, g *GCM, nonce, plaintext, data []byte) {
157 switch {
158 case useGCM:
159 sealKMA(out, g, nonce, plaintext, data)
160 case useGHASH:
161 sealAsm(out, g, nonce, plaintext, data)
162 default:
163 sealGeneric(out, g, nonce, plaintext, data)
164 }
165 }
166 167 func sealAsm(out []byte, g *GCM, nonce, plaintext, additionalData []byte) {
168 var counter, tagMask [gcmBlockSize]byte
169 deriveCounter(&g.hashKey, &counter, nonce)
170 counterCrypt(g, tagMask[:], tagMask[:], &counter)
171 172 counterCrypt(g, out, plaintext, &counter)
173 174 var tag [gcmTagSize]byte
175 gcmAuth(tag[:], &g.hashKey, &tagMask, out[:len(plaintext)], additionalData)
176 copy(out[len(plaintext):], tag[:])
177 }
178 179 func open(out []byte, g *GCM, nonce, ciphertext, data []byte) error {
180 switch {
181 case useGCM:
182 return openKMA(out, g, nonce, ciphertext, data)
183 case useGHASH:
184 return openAsm(out, g, nonce, ciphertext, data)
185 default:
186 return openGeneric(out, g, nonce, ciphertext, data)
187 }
188 }
189 190 func openAsm(out []byte, g *GCM, nonce, ciphertext, additionalData []byte) error {
191 var counter, tagMask [gcmBlockSize]byte
192 deriveCounter(&g.hashKey, &counter, nonce)
193 counterCrypt(g, tagMask[:], tagMask[:], &counter)
194 195 tag := ciphertext[len(ciphertext)-g.tagSize:]
196 ciphertext = ciphertext[:len(ciphertext)-g.tagSize]
197 198 var expectedTag [gcmTagSize]byte
199 gcmAuth(expectedTag[:], &g.hashKey, &tagMask, ciphertext, additionalData)
200 if subtle.ConstantTimeCompare(expectedTag[:g.tagSize], tag) != 1 {
201 return errOpen
202 }
203 204 counterCrypt(g, out, ciphertext, &counter)
205 206 return nil
207 }
208 209 // flags for the KMA instruction
210 const (
211 kmaHS = 1 << 10 // hash subkey supplied
212 kmaLAAD = 1 << 9 // last series of additional authenticated data
213 kmaLPC = 1 << 8 // last series of plaintext or ciphertext blocks
214 kmaDecrypt = 1 << 7 // decrypt
215 )
216 217 // kmaGCM executes the encryption or decryption operation given by fn. The tag
218 // will be calculated and written to tag. cnt should contain the current
219 // counter state and will be overwritten with the updated counter state.
220 // TODO(mundaym): could pass in hash subkey
221 //
222 //go:noescape
223 func kmaGCM(fn int, key, dst, src, aad []byte, tag *[16]byte, cnt *[gcmBlockSize]byte)
224 225 func sealKMA(out []byte, g *GCM, nonce, plaintext, data []byte) {
226 var counter [gcmBlockSize]byte
227 deriveCounter(&g.hashKey, &counter, nonce)
228 fc := aes.BlockFunction(&g.cipher) | kmaLAAD | kmaLPC
229 230 var tag [gcmTagSize]byte
231 kmaGCM(fc, aes.BlockKey(&g.cipher), out[:len(plaintext)], plaintext, data, &tag, &counter)
232 copy(out[len(plaintext):], tag[:])
233 }
234 235 func openKMA(out []byte, g *GCM, nonce, ciphertext, data []byte) error {
236 tag := ciphertext[len(ciphertext)-g.tagSize:]
237 ciphertext = ciphertext[:len(ciphertext)-g.tagSize]
238 239 var counter [gcmBlockSize]byte
240 deriveCounter(&g.hashKey, &counter, nonce)
241 fc := aes.BlockFunction(&g.cipher) | kmaLAAD | kmaLPC | kmaDecrypt
242 243 var expectedTag [gcmTagSize]byte
244 kmaGCM(fc, aes.BlockKey(&g.cipher), out[:len(ciphertext)], ciphertext, data, &expectedTag, &counter)
245 246 if subtle.ConstantTimeCompare(expectedTag[:g.tagSize], tag) != 1 {
247 return errOpen
248 }
249 250 return nil
251 }
252