gcm_asm.mx raw

   1  // Copyright 2015 The Go Authors. All rights reserved.
   2  // Use of this source code is governed by a BSD-style
   3  // license that can be found in the LICENSE file.
   4  
   5  //go:build (amd64 || arm64) && !purego
   6  
   7  package gcm
   8  
   9  import (
  10  	"crypto/internal/fips140/aes"
  11  	"crypto/internal/fips140/subtle"
  12  	"crypto/internal/fips140deps/cpu"
  13  	"crypto/internal/impl"
  14  )
  15  
  16  // The following functions are defined in gcm_*.s.
  17  
  18  //go:noescape
  19  func gcmAesInit(productTable *[256]byte, ks []uint32)
  20  
  21  //go:noescape
  22  func gcmAesData(productTable *[256]byte, data []byte, T *[16]byte)
  23  
  24  //go:noescape
  25  func gcmAesEnc(productTable *[256]byte, dst, src []byte, ctr, T *[16]byte, ks []uint32)
  26  
  27  //go:noescape
  28  func gcmAesDec(productTable *[256]byte, dst, src []byte, ctr, T *[16]byte, ks []uint32)
  29  
  30  //go:noescape
  31  func gcmAesFinish(productTable *[256]byte, tagMask, T *[16]byte, pLen, dLen uint64)
  32  
  33  // Keep in sync with crypto/tls.hasAESGCMHardwareSupport.
  34  var supportsAESGCM = cpu.X86HasAES && cpu.X86HasPCLMULQDQ && cpu.X86HasSSE41 && cpu.X86HasSSSE3 ||
  35  	cpu.ARM64HasAES && cpu.ARM64HasPMULL
  36  
  37  func init() {
  38  	if cpu.AMD64 {
  39  		impl.Register("gcm", "AES-NI", &supportsAESGCM)
  40  	}
  41  	if cpu.ARM64 {
  42  		impl.Register("gcm", "Armv8.0", &supportsAESGCM)
  43  	}
  44  }
  45  
  46  // checkGenericIsExpected is called by the variable-time implementation to make
  47  // sure it is not used when hardware support is available. It shouldn't happen,
  48  // but this way it's more evidently correct.
  49  func checkGenericIsExpected() {
  50  	if supportsAESGCM {
  51  		panic("gcm: internal error: using generic implementation despite hardware support")
  52  	}
  53  }
  54  
  55  type gcmPlatformData struct {
  56  	productTable [256]byte
  57  }
  58  
  59  func initGCM(g *GCM) {
  60  	if !supportsAESGCM {
  61  		return
  62  	}
  63  	gcmAesInit(&g.productTable, aes.EncryptionKeySchedule(&g.cipher))
  64  }
  65  
  66  func seal(out []byte, g *GCM, nonce, plaintext, data []byte) {
  67  	if !supportsAESGCM {
  68  		sealGeneric(out, g, nonce, plaintext, data)
  69  		return
  70  	}
  71  
  72  	var counter, tagMask [gcmBlockSize]byte
  73  
  74  	if len(nonce) == gcmStandardNonceSize {
  75  		// Init counter to nonce||1
  76  		copy(counter[:], nonce)
  77  		counter[gcmBlockSize-1] = 1
  78  	} else {
  79  		// Otherwise counter = GHASH(nonce)
  80  		gcmAesData(&g.productTable, nonce, &counter)
  81  		gcmAesFinish(&g.productTable, &tagMask, &counter, uint64(len(nonce)), uint64(0))
  82  	}
  83  
  84  	aes.EncryptBlockInternal(&g.cipher, tagMask[:], counter[:])
  85  
  86  	var tagOut [gcmTagSize]byte
  87  	gcmAesData(&g.productTable, data, &tagOut)
  88  
  89  	if len(plaintext) > 0 {
  90  		gcmAesEnc(&g.productTable, out, plaintext, &counter, &tagOut, aes.EncryptionKeySchedule(&g.cipher))
  91  	}
  92  	gcmAesFinish(&g.productTable, &tagMask, &tagOut, uint64(len(plaintext)), uint64(len(data)))
  93  	copy(out[len(plaintext):], tagOut[:])
  94  }
  95  
  96  func open(out []byte, g *GCM, nonce, ciphertext, data []byte) error {
  97  	if !supportsAESGCM {
  98  		return openGeneric(out, g, nonce, ciphertext, data)
  99  	}
 100  
 101  	tag := ciphertext[len(ciphertext)-g.tagSize:]
 102  	ciphertext = ciphertext[:len(ciphertext)-g.tagSize]
 103  
 104  	// See GCM spec, section 7.1.
 105  	var counter, tagMask [gcmBlockSize]byte
 106  
 107  	if len(nonce) == gcmStandardNonceSize {
 108  		// Init counter to nonce||1
 109  		copy(counter[:], nonce)
 110  		counter[gcmBlockSize-1] = 1
 111  	} else {
 112  		// Otherwise counter = GHASH(nonce)
 113  		gcmAesData(&g.productTable, nonce, &counter)
 114  		gcmAesFinish(&g.productTable, &tagMask, &counter, uint64(len(nonce)), uint64(0))
 115  	}
 116  
 117  	aes.EncryptBlockInternal(&g.cipher, tagMask[:], counter[:])
 118  
 119  	var expectedTag [gcmTagSize]byte
 120  	gcmAesData(&g.productTable, data, &expectedTag)
 121  
 122  	if len(ciphertext) > 0 {
 123  		gcmAesDec(&g.productTable, out, ciphertext, &counter, &expectedTag, aes.EncryptionKeySchedule(&g.cipher))
 124  	}
 125  	gcmAesFinish(&g.productTable, &tagMask, &expectedTag, uint64(len(ciphertext)), uint64(len(data)))
 126  
 127  	if subtle.ConstantTimeCompare(expectedTag[:g.tagSize], tag) != 1 {
 128  		return errOpen
 129  	}
 130  	return nil
 131  }
 132