evie/android_kernel_oneplus_msm8998
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

crypto: aesni - Convert rfc4106 to new AEAD interface

Browse source 
This patch converts the low-level __gcm-aes-aesni algorithm to
the new AEAD interface.

Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
This commit is contained in:
Herbert Xu 2015-06-01 15:53:06 +08:00
parent 3e648cbeb3
commit b7c89d9e2f
1 changed files with 83 additions and 167 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

248
arch/x86/crypto/aesni-intel_glue.c
View file

@ -44,13 +44,18 @@
#endif #endif
#define AESNI_ALIGN 16
#define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1))
#define RFC4106_HASH_SUBKEY_SIZE 16
/* This data is stored at the end of the crypto_tfm struct. /* This data is stored at the end of the crypto_tfm struct.
* It's a type of per "session" data storage location. * It's a type of per "session" data storage location.
* This needs to be 16 byte aligned. * This needs to be 16 byte aligned.
*/ */
struct aesni_rfc4106_gcm_ctx { struct aesni_rfc4106_gcm_ctx {
u8 hash_subkey[16]; u8 hash_subkey[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
struct crypto_aes_ctx aes_key_expanded; struct crypto_aes_ctx aes_key_expanded
__attribute__ ((__aligned__(AESNI_ALIGN)));
u8 nonce[4]; u8 nonce[4];
}; };
@ -65,10 +70,6 @@ struct aesni_hash_subkey_req_data {
struct scatterlist sg; struct scatterlist sg;
}; };
#define AESNI_ALIGN (16)
#define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1))
#define RFC4106_HASH_SUBKEY_SIZE 16
struct aesni_lrw_ctx { struct aesni_lrw_ctx {
struct lrw_table_ctx lrw_table; struct lrw_table_ctx lrw_table;
u8 raw_aes_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; u8 raw_aes_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
@ -282,10 +283,11 @@ static void (*aesni_gcm_dec_tfm)(void *ctx, u8 *out,
static inline struct static inline struct
aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm) aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
{ {
return unsigned long align = AESNI_ALIGN;
(struct aesni_rfc4106_gcm_ctx *)
PTR_ALIGN((u8 *) if (align <= crypto_tfm_ctx_alignment())
crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN); align = 1;
return PTR_ALIGN(crypto_aead_ctx(tfm), align);
} }
#endif #endif
@ -838,8 +840,6 @@ rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
if (IS_ERR(ctr_tfm)) if (IS_ERR(ctr_tfm))
return PTR_ERR(ctr_tfm); return PTR_ERR(ctr_tfm);
crypto_ablkcipher_clear_flags(ctr_tfm, ~0);
ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len); ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);
if (ret) if (ret)
goto out_free_ablkcipher; goto out_free_ablkcipher;
@ -888,56 +888,20 @@ out_free_ablkcipher:
static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key, static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
unsigned int key_len) unsigned int key_len)
{ {
int ret = 0;
struct crypto_tfm *tfm = crypto_aead_tfm(aead);
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead); struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
u8 *new_key_align, *new_key_mem = NULL;
if (key_len < 4) { if (key_len < 4) {
crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL; return -EINVAL;
} }
/*Account for 4 byte nonce at the end.*/ /*Account for 4 byte nonce at the end.*/
key_len -= 4; key_len -= 4;
if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
key_len != AES_KEYSIZE_256) {
crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce)); memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
/*This must be on a 16 byte boundary!*/
if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN)
return -EINVAL;
if ((unsigned long)key % AESNI_ALIGN) { return aes_set_key_common(crypto_aead_tfm(aead),
/*key is not aligned: use an auxuliar aligned pointer*/ &ctx->aes_key_expanded, key, key_len) ?:
new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL); rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
if (!new_key_mem)
return -ENOMEM;
new_key_align = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
memcpy(new_key_align, key, key_len);
key = new_key_align;
}
if (!irq_fpu_usable())
ret = crypto_aes_expand_key(&(ctx->aes_key_expanded),
key, key_len);
else {
kernel_fpu_begin();
ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len);
kernel_fpu_end();
}
/*This must be on a 16 byte boundary!*/
if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) {
ret = -EINVAL;
goto exit;
}
ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
exit:
kfree(new_key_mem);
return ret;
} }
static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key, static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
@ -960,7 +924,7 @@ static int common_rfc4106_set_authsize(struct crypto_aead *aead,
default: default:
return -EINVAL; return -EINVAL;
} }
crypto_aead_crt(aead)->authsize = authsize;
return 0; return 0;
} }
@ -975,20 +939,17 @@ static int rfc4106_set_authsize(struct crypto_aead *parent,
return crypto_aead_setauthsize(&cryptd_tfm->base, authsize); return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
} }
static int __driver_rfc4106_encrypt(struct aead_request *req) static int helper_rfc4106_encrypt(struct aead_request *req)
{ {
u8 one_entry_in_sg = 0; u8 one_entry_in_sg = 0;
u8 *src, *dst, *assoc; u8 *src, *dst, *assoc;
__be32 counter = cpu_to_be32(1); __be32 counter = cpu_to_be32(1);
struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
u32 key_len = ctx->aes_key_expanded.key_length;
void *aes_ctx = &(ctx->aes_key_expanded); void *aes_ctx = &(ctx->aes_key_expanded);
unsigned long auth_tag_len = crypto_aead_authsize(tfm); unsigned long auth_tag_len = crypto_aead_authsize(tfm);
u8 iv_tab[16+AESNI_ALIGN]; u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN);
struct scatter_walk src_sg_walk; struct scatter_walk src_sg_walk;
struct scatter_walk assoc_sg_walk;
struct scatter_walk dst_sg_walk; struct scatter_walk dst_sg_walk;
unsigned int i; unsigned int i;
@ -997,12 +958,6 @@ static int __driver_rfc4106_encrypt(struct aead_request *req)
/* to 8 or 12 bytes */ /* to 8 or 12 bytes */
if (unlikely(req->assoclen != 8 && req->assoclen != 12)) if (unlikely(req->assoclen != 8 && req->assoclen != 12))
return -EINVAL; return -EINVAL;
if (unlikely(auth_tag_len != 8 && auth_tag_len != 12 && auth_tag_len != 16))
return -EINVAL;
if (unlikely(key_len != AES_KEYSIZE_128 &&
key_len != AES_KEYSIZE_192 &&
key_len != AES_KEYSIZE_256))
return -EINVAL;
/* IV below built */ /* IV below built */
for (i = 0; i < 4; i++) for (i = 0; i < 4; i++)
@ -1011,55 +966,57 @@ static int __driver_rfc4106_encrypt(struct aead_request *req)
*(iv+4+i) = req->iv[i]; *(iv+4+i) = req->iv[i];
*((__be32 *)(iv+12)) = counter; *((__be32 *)(iv+12)) = counter;
if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) { if (sg_is_last(req->src) &&
req->src->offset + req->src->length <= PAGE_SIZE &&
sg_is_last(req->dst) &&
req->dst->offset + req->dst->length <= PAGE_SIZE) {
one_entry_in_sg = 1; one_entry_in_sg = 1;
scatterwalk_start(&src_sg_walk, req->src); scatterwalk_start(&src_sg_walk, req->src);
scatterwalk_start(&assoc_sg_walk, req->assoc); assoc = scatterwalk_map(&src_sg_walk);
src = scatterwalk_map(&src_sg_walk); src = assoc + req->assoclen;
assoc = scatterwalk_map(&assoc_sg_walk);
dst = src; dst = src;
if (unlikely(req->src != req->dst)) { if (unlikely(req->src != req->dst)) {
scatterwalk_start(&dst_sg_walk, req->dst); scatterwalk_start(&dst_sg_walk, req->dst);
dst = scatterwalk_map(&dst_sg_walk); dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
} }
} else { } else {
/* Allocate memory for src, dst, assoc */ /* Allocate memory for src, dst, assoc */
src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen, assoc = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
GFP_ATOMIC); GFP_ATOMIC);
if (unlikely(!src)) if (unlikely(!assoc))
return -ENOMEM; return -ENOMEM;
assoc = (src + req->cryptlen + auth_tag_len); scatterwalk_map_and_copy(assoc, req->src, 0,
scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0); req->assoclen + req->cryptlen, 0);
scatterwalk_map_and_copy(assoc, req->assoc, 0, src = assoc + req->assoclen;
req->assoclen, 0);
dst = src; dst = src;
} }
kernel_fpu_begin();
aesni_gcm_enc_tfm(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv, aesni_gcm_enc_tfm(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv,
ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst
+ ((unsigned long)req->cryptlen), auth_tag_len); + ((unsigned long)req->cryptlen), auth_tag_len);
kernel_fpu_end();
/* The authTag (aka the Integrity Check Value) needs to be written /* The authTag (aka the Integrity Check Value) needs to be written
* back to the packet. */ * back to the packet. */
if (one_entry_in_sg) { if (one_entry_in_sg) {
if (unlikely(req->src != req->dst)) { if (unlikely(req->src != req->dst)) {
scatterwalk_unmap(dst); scatterwalk_unmap(dst - req->assoclen);
scatterwalk_done(&dst_sg_walk, 0, 0); scatterwalk_advance(&dst_sg_walk, req->dst->length);
scatterwalk_done(&dst_sg_walk, 1, 0);
} }
scatterwalk_unmap(src);
scatterwalk_unmap(assoc); scatterwalk_unmap(assoc);
scatterwalk_done(&src_sg_walk, 0, 0); scatterwalk_advance(&src_sg_walk, req->src->length);
scatterwalk_done(&assoc_sg_walk, 0, 0); scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);
} else { } else {
scatterwalk_map_and_copy(dst, req->dst, 0, scatterwalk_map_and_copy(dst, req->dst, req->assoclen,
req->cryptlen + auth_tag_len, 1); req->cryptlen + auth_tag_len, 1);
kfree(src); kfree(assoc);
} }
return 0; return 0;
} }
static int __driver_rfc4106_decrypt(struct aead_request *req) static int helper_rfc4106_decrypt(struct aead_request *req)
{ {
u8 one_entry_in_sg = 0; u8 one_entry_in_sg = 0;
u8 *src, *dst, *assoc; u8 *src, *dst, *assoc;
@ -1068,25 +1025,15 @@ static int __driver_rfc4106_decrypt(struct aead_request *req)
int retval = 0; int retval = 0;
struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
u32 key_len = ctx->aes_key_expanded.key_length;
void *aes_ctx = &(ctx->aes_key_expanded); void *aes_ctx = &(ctx->aes_key_expanded);
unsigned long auth_tag_len = crypto_aead_authsize(tfm); unsigned long auth_tag_len = crypto_aead_authsize(tfm);
u8 iv_and_authTag[32+AESNI_ALIGN]; u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN); u8 authTag[16];
u8 *authTag = iv + 16;
struct scatter_walk src_sg_walk; struct scatter_walk src_sg_walk;
struct scatter_walk assoc_sg_walk;
struct scatter_walk dst_sg_walk; struct scatter_walk dst_sg_walk;
unsigned int i; unsigned int i;
if (unlikely((req->cryptlen < auth_tag_len) || if (unlikely(req->assoclen != 8 && req->assoclen != 12))
(req->assoclen != 8 && req->assoclen != 12)))
return -EINVAL;
if (unlikely(auth_tag_len != 8 && auth_tag_len != 12 && auth_tag_len != 16))
return -EINVAL;
if (unlikely(key_len != AES_KEYSIZE_128 &&
key_len != AES_KEYSIZE_192 &&
key_len != AES_KEYSIZE_256))
return -EINVAL; return -EINVAL;
/* Assuming we are supporting rfc4106 64-bit extended */ /* Assuming we are supporting rfc4106 64-bit extended */
@ -1101,33 +1048,36 @@ static int __driver_rfc4106_decrypt(struct aead_request *req)
*(iv+4+i) = req->iv[i]; *(iv+4+i) = req->iv[i];
*((__be32 *)(iv+12)) = counter; *((__be32 *)(iv+12)) = counter;
if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) { if (sg_is_last(req->src) &&
req->src->offset + req->src->length <= PAGE_SIZE &&
sg_is_last(req->dst) &&
req->dst->offset + req->dst->length <= PAGE_SIZE) {
one_entry_in_sg = 1; one_entry_in_sg = 1;
scatterwalk_start(&src_sg_walk, req->src); scatterwalk_start(&src_sg_walk, req->src);
scatterwalk_start(&assoc_sg_walk, req->assoc); assoc = scatterwalk_map(&src_sg_walk);
src = scatterwalk_map(&src_sg_walk); src = assoc + req->assoclen;
assoc = scatterwalk_map(&assoc_sg_walk);
dst = src; dst = src;
if (unlikely(req->src != req->dst)) { if (unlikely(req->src != req->dst)) {
scatterwalk_start(&dst_sg_walk, req->dst); scatterwalk_start(&dst_sg_walk, req->dst);
dst = scatterwalk_map(&dst_sg_walk); dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
} }
} else { } else {
/* Allocate memory for src, dst, assoc */ /* Allocate memory for src, dst, assoc */
src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC); assoc = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
if (!src) if (!assoc)
return -ENOMEM; return -ENOMEM;
assoc = (src + req->cryptlen); scatterwalk_map_and_copy(assoc, req->src, 0,
scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0); req->assoclen + req->cryptlen, 0);
scatterwalk_map_and_copy(assoc, req->assoc, 0, src = assoc + req->assoclen;
req->assoclen, 0);
dst = src; dst = src;
} }
kernel_fpu_begin();
aesni_gcm_dec_tfm(aes_ctx, dst, src, tempCipherLen, iv, aesni_gcm_dec_tfm(aes_ctx, dst, src, tempCipherLen, iv,
ctx->hash_subkey, assoc, (unsigned long)req->assoclen, ctx->hash_subkey, assoc, (unsigned long)req->assoclen,
authTag, auth_tag_len); authTag, auth_tag_len);
kernel_fpu_end();
/* Compare generated tag with passed in tag. */ /* Compare generated tag with passed in tag. */
retval = crypto_memneq(src + tempCipherLen, authTag, auth_tag_len) ? retval = crypto_memneq(src + tempCipherLen, authTag, auth_tag_len) ?
@ -1135,16 +1085,17 @@ static int __driver_rfc4106_decrypt(struct aead_request *req)
if (one_entry_in_sg) { if (one_entry_in_sg) {
if (unlikely(req->src != req->dst)) { if (unlikely(req->src != req->dst)) {
scatterwalk_unmap(dst); scatterwalk_unmap(dst - req->assoclen);
scatterwalk_done(&dst_sg_walk, 0, 0); scatterwalk_advance(&dst_sg_walk, req->dst->length);
scatterwalk_done(&dst_sg_walk, 1, 0);
} }
scatterwalk_unmap(src);
scatterwalk_unmap(assoc); scatterwalk_unmap(assoc);
scatterwalk_done(&src_sg_walk, 0, 0); scatterwalk_advance(&src_sg_walk, req->src->length);
scatterwalk_done(&assoc_sg_walk, 0, 0); scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);
} else { } else {
scatterwalk_map_and_copy(dst, req->dst, 0, tempCipherLen, 1); scatterwalk_map_and_copy(dst, req->dst, req->assoclen,
kfree(src); tempCipherLen, 1);
kfree(assoc);
} }
return retval; return retval;
} }
@ -1188,36 +1139,6 @@ static int rfc4106_decrypt(struct aead_request *req)
return crypto_aead_decrypt(subreq); return crypto_aead_decrypt(subreq);
} }
static int helper_rfc4106_encrypt(struct aead_request *req)
{
int ret;
if (unlikely(!irq_fpu_usable())) {
WARN_ONCE(1, "__gcm-aes-aesni alg used in invalid context");
ret = -EINVAL;
} else {
kernel_fpu_begin();
ret = __driver_rfc4106_encrypt(req);
kernel_fpu_end();
}
return ret;
}
static int helper_rfc4106_decrypt(struct aead_request *req)
{
int ret;
if (unlikely(!irq_fpu_usable())) {
WARN_ONCE(1, "__gcm-aes-aesni alg used in invalid context");
ret = -EINVAL;
} else {
kernel_fpu_begin();
ret = __driver_rfc4106_decrypt(req);
kernel_fpu_end();
}
return ret;
}
#endif #endif
static struct crypto_alg aesni_algs[] = { { static struct crypto_alg aesni_algs[] = { {
@ -1389,27 +1310,6 @@ static struct crypto_alg aesni_algs[] = { {
.geniv = "chainiv", .geniv = "chainiv",
}, },
}, },
}, {
.cra_name = "__gcm-aes-aesni",
.cra_driver_name = "__driver-gcm-aes-aesni",
.cra_priority = 0,
.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_INTERNAL,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) +
AESNI_ALIGN,
.cra_alignmask = 0,
.cra_type = &crypto_aead_type,
.cra_module = THIS_MODULE,
.cra_u = {
.aead = {
.setkey = common_rfc4106_set_key,
.setauthsize = common_rfc4106_set_authsize,
.encrypt = helper_rfc4106_encrypt,
.decrypt = helper_rfc4106_decrypt,
.ivsize = 8,
.maxauthsize = 16,
},
},
#endif #endif
#if IS_ENABLED(CONFIG_CRYPTO_PCBC) #if IS_ENABLED(CONFIG_CRYPTO_PCBC)
}, { }, {
@ -1526,6 +1426,22 @@ static struct crypto_alg aesni_algs[] = { {
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64
static struct aead_alg aesni_aead_algs[] = { { static struct aead_alg aesni_aead_algs[] = { {
.setkey = common_rfc4106_set_key,
.setauthsize = common_rfc4106_set_authsize,
.encrypt = helper_rfc4106_encrypt,
.decrypt = helper_rfc4106_decrypt,
.ivsize = 8,
.maxauthsize = 16,
.base = {
.cra_name = "__gcm-aes-aesni",
.cra_driver_name = "__driver-gcm-aes-aesni",
.cra_flags = CRYPTO_ALG_INTERNAL,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx),
.cra_alignmask = AESNI_ALIGN - 1,
.cra_module = THIS_MODULE,
},
}, {
.init = rfc4106_init, .init = rfc4106_init,
.exit = rfc4106_exit, .exit = rfc4106_exit,
.setkey = rfc4106_set_key, .setkey = rfc4106_set_key,