Revert "BACKPORT, FROMLIST: crypto: arm64/speck - add NEON-accelerated implementation of Speck-XTS"
This reverts commit 37343fde0c.
Bug: 116008047
Change-Id: Ic47509910c162a35f6fba10a196f4369299451ac
Signed-off-by: Alistair Strachan <astrachan@google.com>
This commit is contained in:
Alistair Strachan
parent
076c36fce1
commit
08400aec8d
4 changed files with 0 additions and 670 deletions
|
|
@ -58,11 +58,4 @@ config CRYPTO_CRC32_ARM64
|
||||||
tristate "CRC32 and CRC32C using optional ARMv8 instructions"
|
tristate "CRC32 and CRC32C using optional ARMv8 instructions"
|
||||||
depends on ARM64
|
depends on ARM64
|
||||||
select CRYPTO_HASH
|
select CRYPTO_HASH
|
||||||
|
|
||||||
config CRYPTO_SPECK_NEON
|
|
||||||
tristate "NEON accelerated Speck cipher algorithms"
|
|
||||||
depends on KERNEL_MODE_NEON
|
|
||||||
select CRYPTO_BLKCIPHER
|
|
||||||
select CRYPTO_GF128MUL
|
|
||||||
select CRYPTO_SPECK
|
|
||||||
endif
|
endif
|
||||||
|
|
|
||||||
|
|
@ -32,9 +32,6 @@ aes-ce-blk-y := aes-glue-ce.o aes-ce.o
|
||||||
obj-$(CONFIG_CRYPTO_AES_ARM64_NEON_BLK) += aes-neon-blk.o
|
obj-$(CONFIG_CRYPTO_AES_ARM64_NEON_BLK) += aes-neon-blk.o
|
||||||
aes-neon-blk-y := aes-glue-neon.o aes-neon.o
|
aes-neon-blk-y := aes-glue-neon.o aes-neon.o
|
||||||
|
|
||||||
obj-$(CONFIG_CRYPTO_SPECK_NEON) += speck-neon.o
|
|
||||||
speck-neon-y := speck-neon-core.o speck-neon-glue.o
|
|
||||||
|
|
||||||
AFLAGS_aes-ce.o := -DINTERLEAVE=4
|
AFLAGS_aes-ce.o := -DINTERLEAVE=4
|
||||||
AFLAGS_aes-neon.o := -DINTERLEAVE=4
|
AFLAGS_aes-neon.o := -DINTERLEAVE=4
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,352 +0,0 @@
|
||||||
// SPDX-License-Identifier: GPL-2.0
|
|
||||||
/*
|
|
||||||
* ARM64 NEON-accelerated implementation of Speck128-XTS and Speck64-XTS
|
|
||||||
*
|
|
||||||
* Copyright (c) 2018 Google, Inc
|
|
||||||
*
|
|
||||||
* Author: Eric Biggers <ebiggers@google.com>
|
|
||||||
*/
|
|
||||||
|
|
||||||
#include <linux/linkage.h>
|
|
||||||
|
|
||||||
.text
|
|
||||||
|
|
||||||
// arguments
|
|
||||||
ROUND_KEYS .req x0 // const {u64,u32} *round_keys
|
|
||||||
NROUNDS .req w1 // int nrounds
|
|
||||||
NROUNDS_X .req x1
|
|
||||||
DST .req x2 // void *dst
|
|
||||||
SRC .req x3 // const void *src
|
|
||||||
NBYTES .req w4 // unsigned int nbytes
|
|
||||||
TWEAK .req x5 // void *tweak
|
|
||||||
|
|
||||||
// registers which hold the data being encrypted/decrypted
|
|
||||||
// (underscores avoid a naming collision with ARM64 registers x0-x3)
|
|
||||||
X_0 .req v0
|
|
||||||
Y_0 .req v1
|
|
||||||
X_1 .req v2
|
|
||||||
Y_1 .req v3
|
|
||||||
X_2 .req v4
|
|
||||||
Y_2 .req v5
|
|
||||||
X_3 .req v6
|
|
||||||
Y_3 .req v7
|
|
||||||
|
|
||||||
// the round key, duplicated in all lanes
|
|
||||||
ROUND_KEY .req v8
|
|
||||||
|
|
||||||
// index vector for tbl-based 8-bit rotates
|
|
||||||
ROTATE_TABLE .req v9
|
|
||||||
ROTATE_TABLE_Q .req q9
|
|
||||||
|
|
||||||
// temporary registers
|
|
||||||
TMP0 .req v10
|
|
||||||
TMP1 .req v11
|
|
||||||
TMP2 .req v12
|
|
||||||
TMP3 .req v13
|
|
||||||
|
|
||||||
// multiplication table for updating XTS tweaks
|
|
||||||
GFMUL_TABLE .req v14
|
|
||||||
GFMUL_TABLE_Q .req q14
|
|
||||||
|
|
||||||
// next XTS tweak value(s)
|
|
||||||
TWEAKV_NEXT .req v15
|
|
||||||
|
|
||||||
// XTS tweaks for the blocks currently being encrypted/decrypted
|
|
||||||
TWEAKV0 .req v16
|
|
||||||
TWEAKV1 .req v17
|
|
||||||
TWEAKV2 .req v18
|
|
||||||
TWEAKV3 .req v19
|
|
||||||
TWEAKV4 .req v20
|
|
||||||
TWEAKV5 .req v21
|
|
||||||
TWEAKV6 .req v22
|
|
||||||
TWEAKV7 .req v23
|
|
||||||
|
|
||||||
.align 4
|
|
||||||
.Lror64_8_table:
|
|
||||||
.octa 0x080f0e0d0c0b0a090007060504030201
|
|
||||||
.Lror32_8_table:
|
|
||||||
.octa 0x0c0f0e0d080b0a090407060500030201
|
|
||||||
.Lrol64_8_table:
|
|
||||||
.octa 0x0e0d0c0b0a09080f0605040302010007
|
|
||||||
.Lrol32_8_table:
|
|
||||||
.octa 0x0e0d0c0f0a09080b0605040702010003
|
|
||||||
.Lgf128mul_table:
|
|
||||||
.octa 0x00000000000000870000000000000001
|
|
||||||
.Lgf64mul_table:
|
|
||||||
.octa 0x0000000000000000000000002d361b00
|
|
||||||
|
|
||||||
/*
|
|
||||||
* _speck_round_128bytes() - Speck encryption round on 128 bytes at a time
|
|
||||||
*
|
|
||||||
* Do one Speck encryption round on the 128 bytes (8 blocks for Speck128, 16 for
|
|
||||||
* Speck64) stored in X0-X3 and Y0-Y3, using the round key stored in all lanes
|
|
||||||
* of ROUND_KEY. 'n' is the lane size: 64 for Speck128, or 32 for Speck64.
|
|
||||||
* 'lanes' is the lane specifier: "2d" for Speck128 or "4s" for Speck64.
|
|
||||||
*/
|
|
||||||
.macro _speck_round_128bytes n, lanes
|
|
||||||
|
|
||||||
// x = ror(x, 8)
|
|
||||||
tbl X_0.16b, {X_0.16b}, ROTATE_TABLE.16b
|
|
||||||
tbl X_1.16b, {X_1.16b}, ROTATE_TABLE.16b
|
|
||||||
tbl X_2.16b, {X_2.16b}, ROTATE_TABLE.16b
|
|
||||||
tbl X_3.16b, {X_3.16b}, ROTATE_TABLE.16b
|
|
||||||
|
|
||||||
// x += y
|
|
||||||
add X_0.\lanes, X_0.\lanes, Y_0.\lanes
|
|
||||||
add X_1.\lanes, X_1.\lanes, Y_1.\lanes
|
|
||||||
add X_2.\lanes, X_2.\lanes, Y_2.\lanes
|
|
||||||
add X_3.\lanes, X_3.\lanes, Y_3.\lanes
|
|
||||||
|
|
||||||
// x ^= k
|
|
||||||
eor X_0.16b, X_0.16b, ROUND_KEY.16b
|
|
||||||
eor X_1.16b, X_1.16b, ROUND_KEY.16b
|
|
||||||
eor X_2.16b, X_2.16b, ROUND_KEY.16b
|
|
||||||
eor X_3.16b, X_3.16b, ROUND_KEY.16b
|
|
||||||
|
|
||||||
// y = rol(y, 3)
|
|
||||||
shl TMP0.\lanes, Y_0.\lanes, #3
|
|
||||||
shl TMP1.\lanes, Y_1.\lanes, #3
|
|
||||||
shl TMP2.\lanes, Y_2.\lanes, #3
|
|
||||||
shl TMP3.\lanes, Y_3.\lanes, #3
|
|
||||||
sri TMP0.\lanes, Y_0.\lanes, #(\n - 3)
|
|
||||||
sri TMP1.\lanes, Y_1.\lanes, #(\n - 3)
|
|
||||||
sri TMP2.\lanes, Y_2.\lanes, #(\n - 3)
|
|
||||||
sri TMP3.\lanes, Y_3.\lanes, #(\n - 3)
|
|
||||||
|
|
||||||
// y ^= x
|
|
||||||
eor Y_0.16b, TMP0.16b, X_0.16b
|
|
||||||
eor Y_1.16b, TMP1.16b, X_1.16b
|
|
||||||
eor Y_2.16b, TMP2.16b, X_2.16b
|
|
||||||
eor Y_3.16b, TMP3.16b, X_3.16b
|
|
||||||
.endm
|
|
||||||
|
|
||||||
/*
|
|
||||||
* _speck_unround_128bytes() - Speck decryption round on 128 bytes at a time
|
|
||||||
*
|
|
||||||
* This is the inverse of _speck_round_128bytes().
|
|
||||||
*/
|
|
||||||
.macro _speck_unround_128bytes n, lanes
|
|
||||||
|
|
||||||
// y ^= x
|
|
||||||
eor TMP0.16b, Y_0.16b, X_0.16b
|
|
||||||
eor TMP1.16b, Y_1.16b, X_1.16b
|
|
||||||
eor TMP2.16b, Y_2.16b, X_2.16b
|
|
||||||
eor TMP3.16b, Y_3.16b, X_3.16b
|
|
||||||
|
|
||||||
// y = ror(y, 3)
|
|
||||||
ushr Y_0.\lanes, TMP0.\lanes, #3
|
|
||||||
ushr Y_1.\lanes, TMP1.\lanes, #3
|
|
||||||
ushr Y_2.\lanes, TMP2.\lanes, #3
|
|
||||||
ushr Y_3.\lanes, TMP3.\lanes, #3
|
|
||||||
sli Y_0.\lanes, TMP0.\lanes, #(\n - 3)
|
|
||||||
sli Y_1.\lanes, TMP1.\lanes, #(\n - 3)
|
|
||||||
sli Y_2.\lanes, TMP2.\lanes, #(\n - 3)
|
|
||||||
sli Y_3.\lanes, TMP3.\lanes, #(\n - 3)
|
|
||||||
|
|
||||||
// x ^= k
|
|
||||||
eor X_0.16b, X_0.16b, ROUND_KEY.16b
|
|
||||||
eor X_1.16b, X_1.16b, ROUND_KEY.16b
|
|
||||||
eor X_2.16b, X_2.16b, ROUND_KEY.16b
|
|
||||||
eor X_3.16b, X_3.16b, ROUND_KEY.16b
|
|
||||||
|
|
||||||
// x -= y
|
|
||||||
sub X_0.\lanes, X_0.\lanes, Y_0.\lanes
|
|
||||||
sub X_1.\lanes, X_1.\lanes, Y_1.\lanes
|
|
||||||
sub X_2.\lanes, X_2.\lanes, Y_2.\lanes
|
|
||||||
sub X_3.\lanes, X_3.\lanes, Y_3.\lanes
|
|
||||||
|
|
||||||
// x = rol(x, 8)
|
|
||||||
tbl X_0.16b, {X_0.16b}, ROTATE_TABLE.16b
|
|
||||||
tbl X_1.16b, {X_1.16b}, ROTATE_TABLE.16b
|
|
||||||
tbl X_2.16b, {X_2.16b}, ROTATE_TABLE.16b
|
|
||||||
tbl X_3.16b, {X_3.16b}, ROTATE_TABLE.16b
|
|
||||||
.endm
|
|
||||||
|
|
||||||
.macro _next_xts_tweak next, cur, tmp, n
|
|
||||||
.if \n == 64
|
|
||||||
/*
|
|
||||||
* Calculate the next tweak by multiplying the current one by x,
|
|
||||||
* modulo p(x) = x^128 + x^7 + x^2 + x + 1.
|
|
||||||
*/
|
|
||||||
sshr \tmp\().2d, \cur\().2d, #63
|
|
||||||
and \tmp\().16b, \tmp\().16b, GFMUL_TABLE.16b
|
|
||||||
shl \next\().2d, \cur\().2d, #1
|
|
||||||
ext \tmp\().16b, \tmp\().16b, \tmp\().16b, #8
|
|
||||||
eor \next\().16b, \next\().16b, \tmp\().16b
|
|
||||||
.else
|
|
||||||
/*
|
|
||||||
* Calculate the next two tweaks by multiplying the current ones by x^2,
|
|
||||||
* modulo p(x) = x^64 + x^4 + x^3 + x + 1.
|
|
||||||
*/
|
|
||||||
ushr \tmp\().2d, \cur\().2d, #62
|
|
||||||
shl \next\().2d, \cur\().2d, #2
|
|
||||||
tbl \tmp\().16b, {GFMUL_TABLE.16b}, \tmp\().16b
|
|
||||||
eor \next\().16b, \next\().16b, \tmp\().16b
|
|
||||||
.endif
|
|
||||||
.endm
|
|
||||||
|
|
||||||
/*
|
|
||||||
* _speck_xts_crypt() - Speck-XTS encryption/decryption
|
|
||||||
*
|
|
||||||
* Encrypt or decrypt NBYTES bytes of data from the SRC buffer to the DST buffer
|
|
||||||
* using Speck-XTS, specifically the variant with a block size of '2n' and round
|
|
||||||
* count given by NROUNDS. The expanded round keys are given in ROUND_KEYS, and
|
|
||||||
* the current XTS tweak value is given in TWEAK. It's assumed that NBYTES is a
|
|
||||||
* nonzero multiple of 128.
|
|
||||||
*/
|
|
||||||
.macro _speck_xts_crypt n, lanes, decrypting
|
|
||||||
|
|
||||||
/*
|
|
||||||
* If decrypting, modify the ROUND_KEYS parameter to point to the last
|
|
||||||
* round key rather than the first, since for decryption the round keys
|
|
||||||
* are used in reverse order.
|
|
||||||
*/
|
|
||||||
.if \decrypting
|
|
||||||
mov NROUNDS, NROUNDS /* zero the high 32 bits */
|
|
||||||
.if \n == 64
|
|
||||||
add ROUND_KEYS, ROUND_KEYS, NROUNDS_X, lsl #3
|
|
||||||
sub ROUND_KEYS, ROUND_KEYS, #8
|
|
||||||
.else
|
|
||||||
add ROUND_KEYS, ROUND_KEYS, NROUNDS_X, lsl #2
|
|
||||||
sub ROUND_KEYS, ROUND_KEYS, #4
|
|
||||||
.endif
|
|
||||||
.endif
|
|
||||||
|
|
||||||
// Load the index vector for tbl-based 8-bit rotates
|
|
||||||
.if \decrypting
|
|
||||||
ldr ROTATE_TABLE_Q, .Lrol\n\()_8_table
|
|
||||||
.else
|
|
||||||
ldr ROTATE_TABLE_Q, .Lror\n\()_8_table
|
|
||||||
.endif
|
|
||||||
|
|
||||||
// One-time XTS preparation
|
|
||||||
.if \n == 64
|
|
||||||
// Load first tweak
|
|
||||||
ld1 {TWEAKV0.16b}, [TWEAK]
|
|
||||||
|
|
||||||
// Load GF(2^128) multiplication table
|
|
||||||
ldr GFMUL_TABLE_Q, .Lgf128mul_table
|
|
||||||
.else
|
|
||||||
// Load first tweak
|
|
||||||
ld1 {TWEAKV0.8b}, [TWEAK]
|
|
||||||
|
|
||||||
// Load GF(2^64) multiplication table
|
|
||||||
ldr GFMUL_TABLE_Q, .Lgf64mul_table
|
|
||||||
|
|
||||||
// Calculate second tweak, packing it together with the first
|
|
||||||
ushr TMP0.2d, TWEAKV0.2d, #63
|
|
||||||
shl TMP1.2d, TWEAKV0.2d, #1
|
|
||||||
tbl TMP0.8b, {GFMUL_TABLE.16b}, TMP0.8b
|
|
||||||
eor TMP0.8b, TMP0.8b, TMP1.8b
|
|
||||||
mov TWEAKV0.d[1], TMP0.d[0]
|
|
||||||
.endif
|
|
||||||
|
|
||||||
.Lnext_128bytes_\@:
|
|
||||||
|
|
||||||
// Calculate XTS tweaks for next 128 bytes
|
|
||||||
_next_xts_tweak TWEAKV1, TWEAKV0, TMP0, \n
|
|
||||||
_next_xts_tweak TWEAKV2, TWEAKV1, TMP0, \n
|
|
||||||
_next_xts_tweak TWEAKV3, TWEAKV2, TMP0, \n
|
|
||||||
_next_xts_tweak TWEAKV4, TWEAKV3, TMP0, \n
|
|
||||||
_next_xts_tweak TWEAKV5, TWEAKV4, TMP0, \n
|
|
||||||
_next_xts_tweak TWEAKV6, TWEAKV5, TMP0, \n
|
|
||||||
_next_xts_tweak TWEAKV7, TWEAKV6, TMP0, \n
|
|
||||||
_next_xts_tweak TWEAKV_NEXT, TWEAKV7, TMP0, \n
|
|
||||||
|
|
||||||
// Load the next source blocks into {X,Y}[0-3]
|
|
||||||
ld1 {X_0.16b-Y_1.16b}, [SRC], #64
|
|
||||||
ld1 {X_2.16b-Y_3.16b}, [SRC], #64
|
|
||||||
|
|
||||||
// XOR the source blocks with their XTS tweaks
|
|
||||||
eor TMP0.16b, X_0.16b, TWEAKV0.16b
|
|
||||||
eor Y_0.16b, Y_0.16b, TWEAKV1.16b
|
|
||||||
eor TMP1.16b, X_1.16b, TWEAKV2.16b
|
|
||||||
eor Y_1.16b, Y_1.16b, TWEAKV3.16b
|
|
||||||
eor TMP2.16b, X_2.16b, TWEAKV4.16b
|
|
||||||
eor Y_2.16b, Y_2.16b, TWEAKV5.16b
|
|
||||||
eor TMP3.16b, X_3.16b, TWEAKV6.16b
|
|
||||||
eor Y_3.16b, Y_3.16b, TWEAKV7.16b
|
|
||||||
|
|
||||||
/*
|
|
||||||
* De-interleave the 'x' and 'y' elements of each block, i.e. make it so
|
|
||||||
* that the X[0-3] registers contain only the second halves of blocks,
|
|
||||||
* and the Y[0-3] registers contain only the first halves of blocks.
|
|
||||||
* (Speck uses the order (y, x) rather than the more intuitive (x, y).)
|
|
||||||
*/
|
|
||||||
uzp2 X_0.\lanes, TMP0.\lanes, Y_0.\lanes
|
|
||||||
uzp1 Y_0.\lanes, TMP0.\lanes, Y_0.\lanes
|
|
||||||
uzp2 X_1.\lanes, TMP1.\lanes, Y_1.\lanes
|
|
||||||
uzp1 Y_1.\lanes, TMP1.\lanes, Y_1.\lanes
|
|
||||||
uzp2 X_2.\lanes, TMP2.\lanes, Y_2.\lanes
|
|
||||||
uzp1 Y_2.\lanes, TMP2.\lanes, Y_2.\lanes
|
|
||||||
uzp2 X_3.\lanes, TMP3.\lanes, Y_3.\lanes
|
|
||||||
uzp1 Y_3.\lanes, TMP3.\lanes, Y_3.\lanes
|
|
||||||
|
|
||||||
// Do the cipher rounds
|
|
||||||
mov x6, ROUND_KEYS
|
|
||||||
mov w7, NROUNDS
|
|
||||||
.Lnext_round_\@:
|
|
||||||
.if \decrypting
|
|
||||||
ld1r {ROUND_KEY.\lanes}, [x6]
|
|
||||||
sub x6, x6, #( \n / 8 )
|
|
||||||
_speck_unround_128bytes \n, \lanes
|
|
||||||
.else
|
|
||||||
ld1r {ROUND_KEY.\lanes}, [x6], #( \n / 8 )
|
|
||||||
_speck_round_128bytes \n, \lanes
|
|
||||||
.endif
|
|
||||||
subs w7, w7, #1
|
|
||||||
bne .Lnext_round_\@
|
|
||||||
|
|
||||||
// Re-interleave the 'x' and 'y' elements of each block
|
|
||||||
zip1 TMP0.\lanes, Y_0.\lanes, X_0.\lanes
|
|
||||||
zip2 Y_0.\lanes, Y_0.\lanes, X_0.\lanes
|
|
||||||
zip1 TMP1.\lanes, Y_1.\lanes, X_1.\lanes
|
|
||||||
zip2 Y_1.\lanes, Y_1.\lanes, X_1.\lanes
|
|
||||||
zip1 TMP2.\lanes, Y_2.\lanes, X_2.\lanes
|
|
||||||
zip2 Y_2.\lanes, Y_2.\lanes, X_2.\lanes
|
|
||||||
zip1 TMP3.\lanes, Y_3.\lanes, X_3.\lanes
|
|
||||||
zip2 Y_3.\lanes, Y_3.\lanes, X_3.\lanes
|
|
||||||
|
|
||||||
// XOR the encrypted/decrypted blocks with the tweaks calculated earlier
|
|
||||||
eor X_0.16b, TMP0.16b, TWEAKV0.16b
|
|
||||||
eor Y_0.16b, Y_0.16b, TWEAKV1.16b
|
|
||||||
eor X_1.16b, TMP1.16b, TWEAKV2.16b
|
|
||||||
eor Y_1.16b, Y_1.16b, TWEAKV3.16b
|
|
||||||
eor X_2.16b, TMP2.16b, TWEAKV4.16b
|
|
||||||
eor Y_2.16b, Y_2.16b, TWEAKV5.16b
|
|
||||||
eor X_3.16b, TMP3.16b, TWEAKV6.16b
|
|
||||||
eor Y_3.16b, Y_3.16b, TWEAKV7.16b
|
|
||||||
mov TWEAKV0.16b, TWEAKV_NEXT.16b
|
|
||||||
|
|
||||||
// Store the ciphertext in the destination buffer
|
|
||||||
st1 {X_0.16b-Y_1.16b}, [DST], #64
|
|
||||||
st1 {X_2.16b-Y_3.16b}, [DST], #64
|
|
||||||
|
|
||||||
// Continue if there are more 128-byte chunks remaining
|
|
||||||
subs NBYTES, NBYTES, #128
|
|
||||||
bne .Lnext_128bytes_\@
|
|
||||||
|
|
||||||
// Store the next tweak and return
|
|
||||||
.if \n == 64
|
|
||||||
st1 {TWEAKV_NEXT.16b}, [TWEAK]
|
|
||||||
.else
|
|
||||||
st1 {TWEAKV_NEXT.8b}, [TWEAK]
|
|
||||||
.endif
|
|
||||||
ret
|
|
||||||
.endm
|
|
||||||
|
|
||||||
ENTRY(speck128_xts_encrypt_neon)
|
|
||||||
_speck_xts_crypt n=64, lanes=2d, decrypting=0
|
|
||||||
ENDPROC(speck128_xts_encrypt_neon)
|
|
||||||
|
|
||||||
ENTRY(speck128_xts_decrypt_neon)
|
|
||||||
_speck_xts_crypt n=64, lanes=2d, decrypting=1
|
|
||||||
ENDPROC(speck128_xts_decrypt_neon)
|
|
||||||
|
|
||||||
ENTRY(speck64_xts_encrypt_neon)
|
|
||||||
_speck_xts_crypt n=32, lanes=4s, decrypting=0
|
|
||||||
ENDPROC(speck64_xts_encrypt_neon)
|
|
||||||
|
|
||||||
ENTRY(speck64_xts_decrypt_neon)
|
|
||||||
_speck_xts_crypt n=32, lanes=4s, decrypting=1
|
|
||||||
ENDPROC(speck64_xts_decrypt_neon)
|
|
||||||
|
|
@ -1,308 +0,0 @@
|
||||||
// SPDX-License-Identifier: GPL-2.0
|
|
||||||
/*
|
|
||||||
* NEON-accelerated implementation of Speck128-XTS and Speck64-XTS
|
|
||||||
* (64-bit version; based on the 32-bit version)
|
|
||||||
*
|
|
||||||
* Copyright (c) 2018 Google, Inc
|
|
||||||
*/
|
|
||||||
|
|
||||||
#include <asm/hwcap.h>
|
|
||||||
#include <asm/neon.h>
|
|
||||||
#include <asm/simd.h>
|
|
||||||
#include <crypto/algapi.h>
|
|
||||||
#include <crypto/gf128mul.h>
|
|
||||||
#include <crypto/speck.h>
|
|
||||||
#include <crypto/xts.h>
|
|
||||||
#include <linux/kernel.h>
|
|
||||||
#include <linux/module.h>
|
|
||||||
|
|
||||||
/* The assembly functions only handle multiples of 128 bytes */
|
|
||||||
#define SPECK_NEON_CHUNK_SIZE 128
|
|
||||||
|
|
||||||
/* Speck128 */
|
|
||||||
|
|
||||||
struct speck128_xts_tfm_ctx {
|
|
||||||
struct speck128_tfm_ctx main_key;
|
|
||||||
struct speck128_tfm_ctx tweak_key;
|
|
||||||
};
|
|
||||||
|
|
||||||
asmlinkage void speck128_xts_encrypt_neon(const u64 *round_keys, int nrounds,
|
|
||||||
void *dst, const void *src,
|
|
||||||
unsigned int nbytes, void *tweak);
|
|
||||||
|
|
||||||
asmlinkage void speck128_xts_decrypt_neon(const u64 *round_keys, int nrounds,
|
|
||||||
void *dst, const void *src,
|
|
||||||
unsigned int nbytes, void *tweak);
|
|
||||||
|
|
||||||
typedef void (*speck128_crypt_one_t)(const struct speck128_tfm_ctx *,
|
|
||||||
u8 *, const u8 *);
|
|
||||||
typedef void (*speck128_xts_crypt_many_t)(const u64 *, int, void *,
|
|
||||||
const void *, unsigned int, void *);
|
|
||||||
|
|
||||||
static __always_inline int
|
|
||||||
__speck128_xts_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
|
|
||||||
struct scatterlist *src, unsigned int nbytes,
|
|
||||||
speck128_crypt_one_t crypt_one,
|
|
||||||
speck128_xts_crypt_many_t crypt_many)
|
|
||||||
{
|
|
||||||
struct crypto_blkcipher *tfm = desc->tfm;
|
|
||||||
const struct speck128_xts_tfm_ctx *ctx = crypto_blkcipher_ctx(tfm);
|
|
||||||
struct blkcipher_walk walk;
|
|
||||||
le128 tweak;
|
|
||||||
int err;
|
|
||||||
|
|
||||||
blkcipher_walk_init(&walk, dst, src, nbytes);
|
|
||||||
err = blkcipher_walk_virt_block(desc, &walk, SPECK_NEON_CHUNK_SIZE);
|
|
||||||
|
|
||||||
crypto_speck128_encrypt(&ctx->tweak_key, (u8 *)&tweak, walk.iv);
|
|
||||||
|
|
||||||
while (walk.nbytes > 0) {
|
|
||||||
unsigned int nbytes = walk.nbytes;
|
|
||||||
u8 *dst = walk.dst.virt.addr;
|
|
||||||
const u8 *src = walk.src.virt.addr;
|
|
||||||
|
|
||||||
if (nbytes >= SPECK_NEON_CHUNK_SIZE && may_use_simd()) {
|
|
||||||
unsigned int count;
|
|
||||||
|
|
||||||
count = round_down(nbytes, SPECK_NEON_CHUNK_SIZE);
|
|
||||||
kernel_neon_begin();
|
|
||||||
(*crypt_many)(ctx->main_key.round_keys,
|
|
||||||
ctx->main_key.nrounds,
|
|
||||||
dst, src, count, &tweak);
|
|
||||||
kernel_neon_end();
|
|
||||||
dst += count;
|
|
||||||
src += count;
|
|
||||||
nbytes -= count;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Handle any remainder with generic code */
|
|
||||||
while (nbytes >= sizeof(tweak)) {
|
|
||||||
le128_xor((le128 *)dst, (const le128 *)src, &tweak);
|
|
||||||
(*crypt_one)(&ctx->main_key, dst, dst);
|
|
||||||
le128_xor((le128 *)dst, (const le128 *)dst, &tweak);
|
|
||||||
gf128mul_x_ble((be128 *)&tweak, (const be128 *)&tweak);
|
|
||||||
|
|
||||||
dst += sizeof(tweak);
|
|
||||||
src += sizeof(tweak);
|
|
||||||
nbytes -= sizeof(tweak);
|
|
||||||
}
|
|
||||||
err = blkcipher_walk_done(desc, &walk, nbytes);
|
|
||||||
}
|
|
||||||
|
|
||||||
return err;
|
|
||||||
}
|
|
||||||
|
|
||||||
static int speck128_xts_encrypt(struct blkcipher_desc *desc,
|
|
||||||
struct scatterlist *dst,
|
|
||||||
struct scatterlist *src,
|
|
||||||
unsigned int nbytes)
|
|
||||||
{
|
|
||||||
return __speck128_xts_crypt(desc, dst, src, nbytes,
|
|
||||||
crypto_speck128_encrypt,
|
|
||||||
speck128_xts_encrypt_neon);
|
|
||||||
}
|
|
||||||
|
|
||||||
static int speck128_xts_decrypt(struct blkcipher_desc *desc,
|
|
||||||
struct scatterlist *dst,
|
|
||||||
struct scatterlist *src,
|
|
||||||
unsigned int nbytes)
|
|
||||||
{
|
|
||||||
return __speck128_xts_crypt(desc, dst, src, nbytes,
|
|
||||||
crypto_speck128_decrypt,
|
|
||||||
speck128_xts_decrypt_neon);
|
|
||||||
}
|
|
||||||
|
|
||||||
static int speck128_xts_setkey(struct crypto_tfm *tfm, const u8 *key,
|
|
||||||
unsigned int keylen)
|
|
||||||
{
|
|
||||||
struct speck128_xts_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
|
|
||||||
int err;
|
|
||||||
|
|
||||||
if (keylen % 2)
|
|
||||||
return -EINVAL;
|
|
||||||
|
|
||||||
keylen /= 2;
|
|
||||||
|
|
||||||
err = crypto_speck128_setkey(&ctx->main_key, key, keylen);
|
|
||||||
if (err)
|
|
||||||
return err;
|
|
||||||
|
|
||||||
return crypto_speck128_setkey(&ctx->tweak_key, key + keylen, keylen);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Speck64 */
|
|
||||||
|
|
||||||
struct speck64_xts_tfm_ctx {
|
|
||||||
struct speck64_tfm_ctx main_key;
|
|
||||||
struct speck64_tfm_ctx tweak_key;
|
|
||||||
};
|
|
||||||
|
|
||||||
asmlinkage void speck64_xts_encrypt_neon(const u32 *round_keys, int nrounds,
|
|
||||||
void *dst, const void *src,
|
|
||||||
unsigned int nbytes, void *tweak);
|
|
||||||
|
|
||||||
asmlinkage void speck64_xts_decrypt_neon(const u32 *round_keys, int nrounds,
|
|
||||||
void *dst, const void *src,
|
|
||||||
unsigned int nbytes, void *tweak);
|
|
||||||
|
|
||||||
typedef void (*speck64_crypt_one_t)(const struct speck64_tfm_ctx *,
|
|
||||||
u8 *, const u8 *);
|
|
||||||
typedef void (*speck64_xts_crypt_many_t)(const u32 *, int, void *,
|
|
||||||
const void *, unsigned int, void *);
|
|
||||||
|
|
||||||
static __always_inline int
|
|
||||||
__speck64_xts_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
|
|
||||||
struct scatterlist *src, unsigned int nbytes,
|
|
||||||
speck64_crypt_one_t crypt_one,
|
|
||||||
speck64_xts_crypt_many_t crypt_many)
|
|
||||||
{
|
|
||||||
struct crypto_blkcipher *tfm = desc->tfm;
|
|
||||||
const struct speck64_xts_tfm_ctx *ctx = crypto_blkcipher_ctx(tfm);
|
|
||||||
struct blkcipher_walk walk;
|
|
||||||
__le64 tweak;
|
|
||||||
int err;
|
|
||||||
|
|
||||||
blkcipher_walk_init(&walk, dst, src, nbytes);
|
|
||||||
err = blkcipher_walk_virt_block(desc, &walk, SPECK_NEON_CHUNK_SIZE);
|
|
||||||
|
|
||||||
crypto_speck64_encrypt(&ctx->tweak_key, (u8 *)&tweak, walk.iv);
|
|
||||||
|
|
||||||
while (walk.nbytes > 0) {
|
|
||||||
unsigned int nbytes = walk.nbytes;
|
|
||||||
u8 *dst = walk.dst.virt.addr;
|
|
||||||
const u8 *src = walk.src.virt.addr;
|
|
||||||
|
|
||||||
if (nbytes >= SPECK_NEON_CHUNK_SIZE && may_use_simd()) {
|
|
||||||
unsigned int count;
|
|
||||||
|
|
||||||
count = round_down(nbytes, SPECK_NEON_CHUNK_SIZE);
|
|
||||||
kernel_neon_begin();
|
|
||||||
(*crypt_many)(ctx->main_key.round_keys,
|
|
||||||
ctx->main_key.nrounds,
|
|
||||||
dst, src, count, &tweak);
|
|
||||||
kernel_neon_end();
|
|
||||||
dst += count;
|
|
||||||
src += count;
|
|
||||||
nbytes -= count;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Handle any remainder with generic code */
|
|
||||||
while (nbytes >= sizeof(tweak)) {
|
|
||||||
*(__le64 *)dst = *(__le64 *)src ^ tweak;
|
|
||||||
(*crypt_one)(&ctx->main_key, dst, dst);
|
|
||||||
*(__le64 *)dst ^= tweak;
|
|
||||||
tweak = cpu_to_le64((le64_to_cpu(tweak) << 1) ^
|
|
||||||
((tweak & cpu_to_le64(1ULL << 63)) ?
|
|
||||||
0x1B : 0));
|
|
||||||
dst += sizeof(tweak);
|
|
||||||
src += sizeof(tweak);
|
|
||||||
nbytes -= sizeof(tweak);
|
|
||||||
}
|
|
||||||
err = blkcipher_walk_done(desc, &walk, nbytes);
|
|
||||||
}
|
|
||||||
|
|
||||||
return err;
|
|
||||||
}
|
|
||||||
|
|
||||||
static int speck64_xts_encrypt(struct blkcipher_desc *desc,
|
|
||||||
struct scatterlist *dst, struct scatterlist *src,
|
|
||||||
unsigned int nbytes)
|
|
||||||
{
|
|
||||||
return __speck64_xts_crypt(desc, dst, src, nbytes,
|
|
||||||
crypto_speck64_encrypt,
|
|
||||||
speck64_xts_encrypt_neon);
|
|
||||||
}
|
|
||||||
|
|
||||||
static int speck64_xts_decrypt(struct blkcipher_desc *desc,
|
|
||||||
struct scatterlist *dst, struct scatterlist *src,
|
|
||||||
unsigned int nbytes)
|
|
||||||
{
|
|
||||||
return __speck64_xts_crypt(desc, dst, src, nbytes,
|
|
||||||
crypto_speck64_decrypt,
|
|
||||||
speck64_xts_decrypt_neon);
|
|
||||||
}
|
|
||||||
|
|
||||||
static int speck64_xts_setkey(struct crypto_tfm *tfm, const u8 *key,
|
|
||||||
unsigned int keylen)
|
|
||||||
{
|
|
||||||
struct speck64_xts_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
|
|
||||||
int err;
|
|
||||||
|
|
||||||
if (keylen % 2)
|
|
||||||
return -EINVAL;
|
|
||||||
|
|
||||||
keylen /= 2;
|
|
||||||
|
|
||||||
err = crypto_speck64_setkey(&ctx->main_key, key, keylen);
|
|
||||||
if (err)
|
|
||||||
return err;
|
|
||||||
|
|
||||||
return crypto_speck64_setkey(&ctx->tweak_key, key + keylen, keylen);
|
|
||||||
}
|
|
||||||
|
|
||||||
static struct crypto_alg speck_algs[] = {
|
|
||||||
{
|
|
||||||
.cra_name = "xts(speck128)",
|
|
||||||
.cra_driver_name = "xts-speck128-neon",
|
|
||||||
.cra_priority = 300,
|
|
||||||
.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
|
|
||||||
.cra_blocksize = SPECK128_BLOCK_SIZE,
|
|
||||||
.cra_type = &crypto_blkcipher_type,
|
|
||||||
.cra_ctxsize = sizeof(struct speck128_xts_tfm_ctx),
|
|
||||||
.cra_alignmask = 7,
|
|
||||||
.cra_module = THIS_MODULE,
|
|
||||||
.cra_u = {
|
|
||||||
.blkcipher = {
|
|
||||||
.min_keysize = 2 * SPECK128_128_KEY_SIZE,
|
|
||||||
.max_keysize = 2 * SPECK128_256_KEY_SIZE,
|
|
||||||
.ivsize = SPECK128_BLOCK_SIZE,
|
|
||||||
.setkey = speck128_xts_setkey,
|
|
||||||
.encrypt = speck128_xts_encrypt,
|
|
||||||
.decrypt = speck128_xts_decrypt,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}, {
|
|
||||||
.cra_name = "xts(speck64)",
|
|
||||||
.cra_driver_name = "xts-speck64-neon",
|
|
||||||
.cra_priority = 300,
|
|
||||||
.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
|
|
||||||
.cra_blocksize = SPECK64_BLOCK_SIZE,
|
|
||||||
.cra_type = &crypto_blkcipher_type,
|
|
||||||
.cra_ctxsize = sizeof(struct speck64_xts_tfm_ctx),
|
|
||||||
.cra_alignmask = 7,
|
|
||||||
.cra_module = THIS_MODULE,
|
|
||||||
.cra_u = {
|
|
||||||
.blkcipher = {
|
|
||||||
.min_keysize = 2 * SPECK64_96_KEY_SIZE,
|
|
||||||
.max_keysize = 2 * SPECK64_128_KEY_SIZE,
|
|
||||||
.ivsize = SPECK64_BLOCK_SIZE,
|
|
||||||
.setkey = speck64_xts_setkey,
|
|
||||||
.encrypt = speck64_xts_encrypt,
|
|
||||||
.decrypt = speck64_xts_decrypt,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
static int __init speck_neon_module_init(void)
|
|
||||||
{
|
|
||||||
if (!(elf_hwcap & HWCAP_ASIMD))
|
|
||||||
return -ENODEV;
|
|
||||||
return crypto_register_algs(speck_algs, ARRAY_SIZE(speck_algs));
|
|
||||||
}
|
|
||||||
|
|
||||||
static void __exit speck_neon_module_exit(void)
|
|
||||||
{
|
|
||||||
crypto_unregister_algs(speck_algs, ARRAY_SIZE(speck_algs));
|
|
||||||
}
|
|
||||||
|
|
||||||
module_init(speck_neon_module_init);
|
|
||||||
module_exit(speck_neon_module_exit);
|
|
||||||
|
|
||||||
MODULE_DESCRIPTION("Speck block cipher (NEON-accelerated)");
|
|
||||||
MODULE_LICENSE("GPL");
|
|
||||||
MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
|
|
||||||
MODULE_ALIAS_CRYPTO("xts(speck128)");
|
|
||||||
MODULE_ALIAS_CRYPTO("xts-speck128-neon");
|
|
||||||
MODULE_ALIAS_CRYPTO("xts(speck64)");
|
|
||||||
MODULE_ALIAS_CRYPTO("xts-speck64-neon");
|
|
||||||
Loading…
Add table
Reference in a new issue