gcm_ppc64x.mx raw
1 // Copyright 2019 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 //go:build (ppc64le || ppc64) && !purego
6
7 package gcm
8
9 import (
10 "crypto/internal/fips140/aes"
11 "crypto/internal/fips140/subtle"
12 "crypto/internal/fips140deps/byteorder"
13 "crypto/internal/fips140deps/godebug"
14 "crypto/internal/impl"
15 "runtime"
16 )
17
18 // This file implements GCM using an optimized GHASH function.
19
20 //go:noescape
21 func gcmInit(productTable *[256]byte, h []byte)
22
23 //go:noescape
24 func gcmHash(output []byte, productTable *[256]byte, inp []byte, len int)
25
26 func counterCryptASM(nr int, out, in []byte, counter *[gcmBlockSize]byte, key *uint32)
27
28 // The POWER architecture doesn't have a way to turn off AES-GCM support
29 // at runtime with GODEBUG=cpu.something=off, so introduce a new GODEBUG
30 // knob for that. It's intentionally only checked at init() time, to
31 // avoid the performance overhead of checking it every time.
32 var supportsAESGCM = godebug.Value("#ppc64gcm") != "off"
33
34 func init() {
35 impl.Register("gcm", "POWER8", &supportsAESGCM)
36 }
37
38 func checkGenericIsExpected() {
39 if supportsAESGCM {
40 panic("gcm: internal error: using generic implementation despite hardware support")
41 }
42 }
43
44 type gcmPlatformData struct {
45 productTable [256]byte
46 }
47
48 func initGCM(g *GCM) {
49 if !supportsAESGCM {
50 return
51 }
52
53 hle := []byte{:gcmBlockSize}
54 aes.EncryptBlockInternal(&g.cipher, hle, hle)
55
56 // Reverse the bytes in each 8 byte chunk
57 // Load little endian, store big endian
58 var h1, h2 uint64
59 if runtime.GOARCH == "ppc64le" {
60 h1 = byteorder.LEUint64(hle[:8])
61 h2 = byteorder.LEUint64(hle[8:])
62 } else {
63 h1 = byteorder.BEUint64(hle[:8])
64 h2 = byteorder.BEUint64(hle[8:])
65 }
66 byteorder.BEPutUint64(hle[:8], h1)
67 byteorder.BEPutUint64(hle[8:], h2)
68 gcmInit(&g.productTable, hle)
69 }
70
71 // deriveCounter computes the initial GCM counter state from the given nonce.
72 func deriveCounter(counter *[gcmBlockSize]byte, nonce []byte, productTable *[256]byte) {
73 if len(nonce) == gcmStandardNonceSize {
74 copy(counter[:], nonce)
75 counter[gcmBlockSize-1] = 1
76 } else {
77 var hash [16]byte
78 paddedGHASH(&hash, nonce, productTable)
79 lens := gcmLengths(0, uint64(len(nonce))*8)
80 paddedGHASH(&hash, lens[:], productTable)
81 copy(counter[:], hash[:])
82 }
83 }
84
85 // counterCrypt encrypts in using AES in counter mode and places the result
86 // into out. counter is the initial count value and will be updated with the next
87 // count value. The length of out must be greater than or equal to the length
88 // of in.
89 // counterCryptASM implements counterCrypt which then allows the loop to
90 // be unrolled and optimized.
91 func counterCrypt(b *aes.Block, out, in []byte, counter *[gcmBlockSize]byte) {
92 enc := aes.EncryptionKeySchedule(b)
93 rounds := len(enc)/4 - 1
94 counterCryptASM(rounds, out, in, counter, &enc[0])
95 }
96
97 // paddedGHASH pads data with zeroes until its length is a multiple of
98 // 16-bytes. It then calculates a new value for hash using the ghash
99 // algorithm.
100 func paddedGHASH(hash *[16]byte, data []byte, productTable *[256]byte) {
101 if siz := len(data) - (len(data) % gcmBlockSize); siz > 0 {
102 gcmHash(hash[:], productTable, data[:], siz)
103 data = data[siz:]
104 }
105 if len(data) > 0 {
106 var s [16]byte
107 copy(s[:], data)
108 gcmHash(hash[:], productTable, s[:], len(s))
109 }
110 }
111
112 // auth calculates GHASH(ciphertext, additionalData), masks the result with
113 // tagMask and writes the result to out.
114 func auth(out, ciphertext, aad []byte, tagMask *[gcmTagSize]byte, productTable *[256]byte) {
115 var hash [16]byte
116 paddedGHASH(&hash, aad, productTable)
117 paddedGHASH(&hash, ciphertext, productTable)
118 lens := gcmLengths(uint64(len(aad))*8, uint64(len(ciphertext))*8)
119 paddedGHASH(&hash, lens[:], productTable)
120
121 copy(out, hash[:])
122 for i := range out {
123 out[i] ^= tagMask[i]
124 }
125 }
126
127 func seal(out []byte, g *GCM, nonce, plaintext, data []byte) {
128 if !supportsAESGCM {
129 sealGeneric(out, g, nonce, plaintext, data)
130 return
131 }
132
133 var counter, tagMask [gcmBlockSize]byte
134 deriveCounter(&counter, nonce, &g.productTable)
135
136 aes.EncryptBlockInternal(&g.cipher, tagMask[:], counter[:])
137 gcmInc32(&counter)
138
139 counterCrypt(&g.cipher, out, plaintext, &counter)
140 auth(out[len(plaintext):], out[:len(plaintext)], data, &tagMask, &g.productTable)
141 }
142
143 func open(out []byte, g *GCM, nonce, ciphertext, data []byte) error {
144 if !supportsAESGCM {
145 return openGeneric(out, g, nonce, ciphertext, data)
146 }
147
148 tag := ciphertext[len(ciphertext)-g.tagSize:]
149 ciphertext = ciphertext[:len(ciphertext)-g.tagSize]
150
151 var counter, tagMask [gcmBlockSize]byte
152 deriveCounter(&counter, nonce, &g.productTable)
153
154 aes.EncryptBlockInternal(&g.cipher, tagMask[:], counter[:])
155 gcmInc32(&counter)
156
157 var expectedTag [gcmTagSize]byte
158 auth(expectedTag[:], ciphertext, data, &tagMask, &g.productTable)
159
160 if subtle.ConstantTimeCompare(expectedTag[:g.tagSize], tag) != 1 {
161 return errOpen
162 }
163
164 counterCrypt(&g.cipher, out, ciphertext, &counter)
165 return nil
166 }
167
168 func gcmLengths(len0, len1 uint64) [16]byte {
169 return [16]byte{
170 byte(len0 >> 56),
171 byte(len0 >> 48),
172 byte(len0 >> 40),
173 byte(len0 >> 32),
174 byte(len0 >> 24),
175 byte(len0 >> 16),
176 byte(len0 >> 8),
177 byte(len0),
178 byte(len1 >> 56),
179 byte(len1 >> 48),
180 byte(len1 >> 40),
181 byte(len1 >> 32),
182 byte(len1 >> 24),
183 byte(len1 >> 16),
184 byte(len1 >> 8),
185 byte(len1),
186 }
187 }
188