AetherDroid/android_kernel_samsung_on5xelte
1
0
Fork 0
mirror of https://github.com/AetherDroid/android_kernel_samsung_on5xelte.git synced 2025-09-08 01:08:03 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fixed MTP to work with TWRP

Browse source
This commit is contained in:
awab228 2018-06-19 23:16:04 +02:00
commit f6dfaef42e
50820 changed files with 20846062 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

441
drivers/crypto/Kconfig Normal file
View file

@ -0,0 +1,441 @@
menuconfig CRYPTO_HW
bool "Hardware crypto devices"
default y
---help---
Say Y here to get to see options for hardware crypto devices and
processors. This option alone does not add any kernel code.
If you say N, all options in this submenu will be skipped and disabled.
if CRYPTO_HW
config CRYPTO_DEV_PADLOCK
tristate "Support for VIA PadLock ACE"
depends on X86 && !UML
help
Some VIA processors come with an integrated crypto engine
(so called VIA PadLock ACE, Advanced Cryptography Engine)
that provides instructions for very fast cryptographic
operations with supported algorithms.
The instructions are used only when the CPU supports them.
Otherwise software encryption is used.
config CRYPTO_DEV_PADLOCK_AES
tristate "PadLock driver for AES algorithm"
depends on CRYPTO_DEV_PADLOCK
select CRYPTO_BLKCIPHER
select CRYPTO_AES
help
Use VIA PadLock for AES algorithm.
Available in VIA C3 and newer CPUs.
If unsure say M. The compiled module will be
called padlock-aes.
config CRYPTO_DEV_PADLOCK_SHA
tristate "PadLock driver for SHA1 and SHA256 algorithms"
depends on CRYPTO_DEV_PADLOCK
select CRYPTO_HASH
select CRYPTO_SHA1
select CRYPTO_SHA256
help
Use VIA PadLock for SHA1/SHA256 algorithms.
Available in VIA C7 and newer processors.
If unsure say M. The compiled module will be
called padlock-sha.
config CRYPTO_DEV_GEODE
tristate "Support for the Geode LX AES engine"
depends on X86_32 && PCI
select CRYPTO_ALGAPI
select CRYPTO_BLKCIPHER
help
Say 'Y' here to use the AMD Geode LX processor on-board AES
engine for the CryptoAPI AES algorithm.
To compile this driver as a module, choose M here: the module
will be called geode-aes.
config ZCRYPT
tristate "Support for PCI-attached cryptographic adapters"
depends on S390
select HW_RANDOM
help
Select this option if you want to use a PCI-attached cryptographic
adapter like:
+ PCI Cryptographic Accelerator (PCICA)
+ PCI Cryptographic Coprocessor (PCICC)
+ PCI-X Cryptographic Coprocessor (PCIXCC)
+ Crypto Express2 Coprocessor (CEX2C)
+ Crypto Express2 Accelerator (CEX2A)
+ Crypto Express3 Coprocessor (CEX3C)
+ Crypto Express3 Accelerator (CEX3A)
config CRYPTO_SHA1_S390
tristate "SHA1 digest algorithm"
depends on S390
select CRYPTO_HASH
help
This is the s390 hardware accelerated implementation of the
SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2).
It is available as of z990.
config CRYPTO_SHA256_S390
tristate "SHA256 digest algorithm"
depends on S390
select CRYPTO_HASH
help
This is the s390 hardware accelerated implementation of the
SHA256 secure hash standard (DFIPS 180-2).
It is available as of z9.
config CRYPTO_SHA512_S390
tristate "SHA384 and SHA512 digest algorithm"
depends on S390
select CRYPTO_HASH
help
This is the s390 hardware accelerated implementation of the
SHA512 secure hash standard.
It is available as of z10.
config CRYPTO_DES_S390
tristate "DES and Triple DES cipher algorithms"
depends on S390
select CRYPTO_ALGAPI
select CRYPTO_BLKCIPHER
select CRYPTO_DES
help
This is the s390 hardware accelerated implementation of the
DES cipher algorithm (FIPS 46-2), and Triple DES EDE (FIPS 46-3).
As of z990 the ECB and CBC mode are hardware accelerated.
As of z196 the CTR mode is hardware accelerated.
config CRYPTO_AES_S390
tristate "AES cipher algorithms"
depends on S390
select CRYPTO_ALGAPI
select CRYPTO_BLKCIPHER
help
This is the s390 hardware accelerated implementation of the
AES cipher algorithms (FIPS-197).
As of z9 the ECB and CBC modes are hardware accelerated
for 128 bit keys.
As of z10 the ECB and CBC modes are hardware accelerated
for all AES key sizes.
As of z196 the CTR mode is hardware accelerated for all AES
key sizes and XTS mode is hardware accelerated for 256 and
512 bit keys.
config S390_PRNG
tristate "Pseudo random number generator device driver"
depends on S390
default "m"
help
Select this option if you want to use the s390 pseudo random number
generator. The PRNG is part of the cryptographic processor functions
and uses triple-DES to generate secure random numbers like the
ANSI X9.17 standard. User-space programs access the
pseudo-random-number device through the char device /dev/prandom.
It is available as of z9.
config CRYPTO_GHASH_S390
tristate "GHASH digest algorithm"
depends on S390
select CRYPTO_HASH
help
This is the s390 hardware accelerated implementation of the
GHASH message digest algorithm for GCM (Galois/Counter Mode).
It is available as of z196.
config CRYPTO_DEV_MV_CESA
tristate "Marvell's Cryptographic Engine"
depends on PLAT_ORION
select CRYPTO_ALGAPI
select CRYPTO_AES
select CRYPTO_BLKCIPHER2
select CRYPTO_HASH
help
This driver allows you to utilize the Cryptographic Engines and
Security Accelerator (CESA) which can be found on the Marvell Orion
and Kirkwood SoCs, such as QNAP's TS-209.
Currently the driver supports AES in ECB and CBC mode without DMA.
config CRYPTO_DEV_NIAGARA2
tristate "Niagara2 Stream Processing Unit driver"
select CRYPTO_DES
select CRYPTO_ALGAPI
depends on SPARC64
help
Each core of a Niagara2 processor contains a Stream
Processing Unit, which itself contains several cryptographic
sub-units. One set provides the Modular Arithmetic Unit,
used for SSL offload. The other set provides the Cipher
Group, which can perform encryption, decryption, hashing,
checksumming, and raw copies.
config CRYPTO_DEV_HIFN_795X
tristate "Driver HIFN 795x crypto accelerator chips"
select CRYPTO_DES
select CRYPTO_ALGAPI
select CRYPTO_BLKCIPHER
select HW_RANDOM if CRYPTO_DEV_HIFN_795X_RNG
depends on PCI
depends on !ARCH_DMA_ADDR_T_64BIT
help
This option allows you to have support for HIFN 795x crypto adapters.
config CRYPTO_DEV_HIFN_795X_RNG
bool "HIFN 795x random number generator"
depends on CRYPTO_DEV_HIFN_795X
help
Select this option if you want to enable the random number generator
on the HIFN 795x crypto adapters.
source drivers/crypto/caam/Kconfig
config CRYPTO_DEV_TALITOS
tristate "Talitos Freescale Security Engine (SEC)"
select CRYPTO_ALGAPI
select CRYPTO_AUTHENC
select HW_RANDOM
depends on FSL_SOC
help
Say 'Y' here to use the Freescale Security Engine (SEC)
to offload cryptographic algorithm computation.
The Freescale SEC is present on PowerQUICC 'E' processors, such
as the MPC8349E and MPC8548E.
To compile this driver as a module, choose M here: the module
will be called talitos.
config CRYPTO_DEV_IXP4XX
tristate "Driver for IXP4xx crypto hardware acceleration"
depends on ARCH_IXP4XX && IXP4XX_QMGR && IXP4XX_NPE
select CRYPTO_DES
select CRYPTO_ALGAPI
select CRYPTO_AUTHENC
select CRYPTO_BLKCIPHER
help
Driver for the IXP4xx NPE crypto engine.
config CRYPTO_DEV_PPC4XX
tristate "Driver AMCC PPC4xx crypto accelerator"
depends on PPC && 4xx
select CRYPTO_HASH
select CRYPTO_ALGAPI
select CRYPTO_BLKCIPHER
help
This option allows you to have support for AMCC crypto acceleration.
config CRYPTO_DEV_OMAP_SHAM
tristate "Support for OMAP MD5/SHA1/SHA2 hw accelerator"
depends on ARCH_OMAP2PLUS
select CRYPTO_SHA1
select CRYPTO_MD5
select CRYPTO_SHA256
select CRYPTO_SHA512
select CRYPTO_HMAC
help
OMAP processors have MD5/SHA1/SHA2 hw accelerator. Select this if you
want to use the OMAP module for MD5/SHA1/SHA2 algorithms.
config CRYPTO_DEV_OMAP_AES
tristate "Support for OMAP AES hw engine"
depends on ARCH_OMAP2 || ARCH_OMAP3 || ARCH_OMAP2PLUS
select CRYPTO_AES
select CRYPTO_BLKCIPHER2
help
OMAP processors have AES module accelerator. Select this if you
want to use the OMAP module for AES algorithms.
config CRYPTO_DEV_OMAP_DES
tristate "Support for OMAP DES3DES hw engine"
depends on ARCH_OMAP2PLUS
select CRYPTO_DES
select CRYPTO_BLKCIPHER2
help
OMAP processors have DES/3DES module accelerator. Select this if you
want to use the OMAP module for DES and 3DES algorithms. Currently
the ECB and CBC modes of operation supported by the driver. Also
accesses made on unaligned boundaries are also supported.
config CRYPTO_DEV_PICOXCELL
tristate "Support for picoXcell IPSEC and Layer2 crypto engines"
depends on ARCH_PICOXCELL && HAVE_CLK
select CRYPTO_AES
select CRYPTO_AUTHENC
select CRYPTO_ALGAPI
select CRYPTO_DES
select CRYPTO_CBC
select CRYPTO_ECB
select CRYPTO_SEQIV
help
This option enables support for the hardware offload engines in the
Picochip picoXcell SoC devices. Select this for IPSEC ESP offload
and for 3gpp Layer 2 ciphering support.
Saying m here will build a module named pipcoxcell_crypto.
config CRYPTO_DEV_SAHARA
tristate "Support for SAHARA crypto accelerator"
depends on ARCH_MXC && OF
select CRYPTO_BLKCIPHER
select CRYPTO_AES
select CRYPTO_ECB
help
This option enables support for the SAHARA HW crypto accelerator
found in some Freescale i.MX chips.
config CRYPTO_DEV_S5P
tristate "Support for Samsung S5PV210/Exynos crypto accelerator"
depends on ARCH_S5PV210 || ARCH_EXYNOS
select CRYPTO_AES
select CRYPTO_ALGAPI
select CRYPTO_BLKCIPHER
help
This option allows you to have support for S5P crypto acceleration.
Select this to offload Samsung S5PV210 or S5PC110, Exynos from AES
algorithms execution.
config CRYPTO_DEV_NX
bool "Support for IBM Power7+ in-Nest cryptographic acceleration"
depends on PPC64 && IBMVIO && !CPU_LITTLE_ENDIAN
default n
help
Support for Power7+ in-Nest cryptographic acceleration.
if CRYPTO_DEV_NX
source "drivers/crypto/nx/Kconfig"
endif
config CRYPTO_DEV_UX500
tristate "Driver for ST-Ericsson UX500 crypto hardware acceleration"
depends on ARCH_U8500
select CRYPTO_ALGAPI
help
Driver for ST-Ericsson UX500 crypto engine.
if CRYPTO_DEV_UX500
source "drivers/crypto/ux500/Kconfig"
endif # if CRYPTO_DEV_UX500
config CRYPTO_DEV_BFIN_CRC
tristate "Support for Blackfin CRC hardware"
depends on BF60x
help
Newer Blackfin processors have CRC hardware. Select this if you
want to use the Blackfin CRC module.
config CRYPTO_DEV_ATMEL_AES
tristate "Support for Atmel AES hw accelerator"
depends on ARCH_AT91
select CRYPTO_CBC
select CRYPTO_ECB
select CRYPTO_AES
select CRYPTO_ALGAPI
select CRYPTO_BLKCIPHER
select AT_HDMAC
help
Some Atmel processors have AES hw accelerator.
Select this if you want to use the Atmel module for
AES algorithms.
To compile this driver as a module, choose M here: the module
will be called atmel-aes.
config CRYPTO_DEV_ATMEL_TDES
tristate "Support for Atmel DES/TDES hw accelerator"
depends on ARCH_AT91
select CRYPTO_DES
select CRYPTO_CBC
select CRYPTO_ECB
select CRYPTO_ALGAPI
select CRYPTO_BLKCIPHER
help
Some Atmel processors have DES/TDES hw accelerator.
Select this if you want to use the Atmel module for
DES/TDES algorithms.
To compile this driver as a module, choose M here: the module
will be called atmel-tdes.
config CRYPTO_DEV_ATMEL_SHA
tristate "Support for Atmel SHA hw accelerator"
depends on ARCH_AT91
select CRYPTO_SHA1
select CRYPTO_SHA256
select CRYPTO_SHA512
select CRYPTO_ALGAPI
help
Some Atmel processors have SHA1/SHA224/SHA256/SHA384/SHA512
hw accelerator.
Select this if you want to use the Atmel module for
SHA1/SHA224/SHA256/SHA384/SHA512 algorithms.
To compile this driver as a module, choose M here: the module
will be called atmel-sha.
config CRYPTO_DEV_CCP
bool "Support for AMD Cryptographic Coprocessor"
depends on (X86 && PCI) || ARM64
default n
help
The AMD Cryptographic Coprocessor provides hardware support
for encryption, hashing and related operations.
if CRYPTO_DEV_CCP
source "drivers/crypto/ccp/Kconfig"
endif
config CRYPTO_DEV_MXS_DCP
tristate "Support for Freescale MXS DCP"
depends on ARCH_MXS
select CRYPTO_SHA1
select CRYPTO_SHA256
select CRYPTO_CBC
select CRYPTO_ECB
select CRYPTO_AES
select CRYPTO_BLKCIPHER
select CRYPTO_ALGAPI
help
The Freescale i.MX23/i.MX28 has SHA1/SHA256 and AES128 CBC/ECB
co-processor on the die.
To compile this driver as a module, choose M here: the module
will be called mxs-dcp.
source "drivers/crypto/qat/Kconfig"
config CRYPTO_DEV_QCE
tristate "Qualcomm crypto engine accelerator"
depends on (ARCH_QCOM || COMPILE_TEST) && HAS_DMA && HAS_IOMEM
select CRYPTO_AES
select CRYPTO_DES
select CRYPTO_ECB
select CRYPTO_CBC
select CRYPTO_XTS
select CRYPTO_CTR
select CRYPTO_ALGAPI
select CRYPTO_BLKCIPHER
help
This driver supports Qualcomm crypto engine accelerator
hardware. To compile this driver as a module, choose M here. The
module will be called qcrypto.
source "drivers/crypto/fmp/Kconfig"
endif # CRYPTO_HW

28
drivers/crypto/Makefile Normal file
View file

@ -0,0 +1,28 @@
obj-$(CONFIG_CRYPTO_DEV_ATMEL_AES) += atmel-aes.o
obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA) += atmel-sha.o
obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o
obj-$(CONFIG_CRYPTO_DEV_BFIN_CRC) += bfin_crc.o
obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam/
obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o
obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o
obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o
obj-$(CONFIG_CRYPTO_DEV_MV_CESA) += mv_cesa.o
obj-$(CONFIG_CRYPTO_DEV_MXS_DCP) += mxs-dcp.o
obj-$(CONFIG_CRYPTO_DEV_NIAGARA2) += n2_crypto.o
n2_crypto-y := n2_core.o n2_asm.o
obj-$(CONFIG_CRYPTO_DEV_NX) += nx/
obj-$(CONFIG_CRYPTO_DEV_OMAP_AES) += omap-aes.o
obj-$(CONFIG_CRYPTO_DEV_OMAP_DES) += omap-des.o
obj-$(CONFIG_CRYPTO_DEV_OMAP_SHAM) += omap-sham.o
obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o
obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o
obj-$(CONFIG_CRYPTO_DEV_PICOXCELL) += picoxcell_crypto.o
obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += amcc/
obj-$(CONFIG_CRYPTO_DEV_S5P) += s5p-sss.o
obj-$(CONFIG_CRYPTO_DEV_SAHARA) += sahara.o
obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o
obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/
obj-$(CONFIG_CRYPTO_DEV_QAT) += qat/
obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/
obj-y += fmp/

2
drivers/crypto/amcc/Makefile Normal file
View file

@ -0,0 +1,2 @@
obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += crypto4xx.o
crypto4xx-y := crypto4xx_core.o crypto4xx_alg.o crypto4xx_sa.o

295
drivers/crypto/amcc/crypto4xx_alg.c Normal file
View file

@ -0,0 +1,295 @@
/**
* AMCC SoC PPC4xx Crypto Driver
*
* Copyright (c) 2008 Applied Micro Circuits Corporation.
* All rights reserved. James Hsiao <jhsiao@amcc.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* This file implements the Linux crypto algorithms.
*/
#include <linux/kernel.h>
#include <linux/interrupt.h>
#include <linux/spinlock_types.h>
#include <linux/scatterlist.h>
#include <linux/crypto.h>
#include <linux/hash.h>
#include <crypto/internal/hash.h>
#include <linux/dma-mapping.h>
#include <crypto/algapi.h>
#include <crypto/aes.h>
#include <crypto/sha.h>
#include "crypto4xx_reg_def.h"
#include "crypto4xx_sa.h"
#include "crypto4xx_core.h"
static void set_dynamic_sa_command_0(struct dynamic_sa_ctl *sa, u32 save_h,
u32 save_iv, u32 ld_h, u32 ld_iv,
u32 hdr_proc, u32 h, u32 c, u32 pad_type,
u32 op_grp, u32 op, u32 dir)
{
sa->sa_command_0.w = 0;
sa->sa_command_0.bf.save_hash_state = save_h;
sa->sa_command_0.bf.save_iv = save_iv;
sa->sa_command_0.bf.load_hash_state = ld_h;
sa->sa_command_0.bf.load_iv = ld_iv;
sa->sa_command_0.bf.hdr_proc = hdr_proc;
sa->sa_command_0.bf.hash_alg = h;
sa->sa_command_0.bf.cipher_alg = c;
sa->sa_command_0.bf.pad_type = pad_type & 3;
sa->sa_command_0.bf.extend_pad = pad_type >> 2;
sa->sa_command_0.bf.op_group = op_grp;
sa->sa_command_0.bf.opcode = op;
sa->sa_command_0.bf.dir = dir;
}
static void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm,
u32 hmac_mc, u32 cfb, u32 esn,
u32 sn_mask, u32 mute, u32 cp_pad,
u32 cp_pay, u32 cp_hdr)
{
sa->sa_command_1.w = 0;
sa->sa_command_1.bf.crypto_mode31 = (cm & 4) >> 2;
sa->sa_command_1.bf.crypto_mode9_8 = cm & 3;
sa->sa_command_1.bf.feedback_mode = cfb,
sa->sa_command_1.bf.sa_rev = 1;
sa->sa_command_1.bf.extended_seq_num = esn;
sa->sa_command_1.bf.seq_num_mask = sn_mask;
sa->sa_command_1.bf.mutable_bit_proc = mute;
sa->sa_command_1.bf.copy_pad = cp_pad;
sa->sa_command_1.bf.copy_payload = cp_pay;
sa->sa_command_1.bf.copy_hdr = cp_hdr;
}
int crypto4xx_encrypt(struct ablkcipher_request *req)
{
struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
ctx->direction = DIR_OUTBOUND;
ctx->hash_final = 0;
ctx->is_hash = 0;
ctx->pd_ctl = 0x1;
return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
req->nbytes, req->info,
get_dynamic_sa_iv_size(ctx));
}
int crypto4xx_decrypt(struct ablkcipher_request *req)
{
struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
ctx->direction = DIR_INBOUND;
ctx->hash_final = 0;
ctx->is_hash = 0;
ctx->pd_ctl = 1;
return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
req->nbytes, req->info,
get_dynamic_sa_iv_size(ctx));
}
/**
* AES Functions
*/
static int crypto4xx_setkey_aes(struct crypto_ablkcipher *cipher,
const u8 *key,
unsigned int keylen,
unsigned char cm,
u8 fb)
{
struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm);
struct dynamic_sa_ctl *sa;
int rc;
if (keylen != AES_KEYSIZE_256 &&
keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_128) {
crypto_ablkcipher_set_flags(cipher,
CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
/* Create SA */
if (ctx->sa_in_dma_addr || ctx->sa_out_dma_addr)
crypto4xx_free_sa(ctx);
rc = crypto4xx_alloc_sa(ctx, SA_AES128_LEN + (keylen-16) / 4);
if (rc)
return rc;
if (ctx->state_record_dma_addr == 0) {
rc = crypto4xx_alloc_state_record(ctx);
if (rc) {
crypto4xx_free_sa(ctx);
return rc;
}
}
/* Setup SA */
sa = (struct dynamic_sa_ctl *) ctx->sa_in;
ctx->hash_final = 0;
set_dynamic_sa_command_0(sa, SA_NOT_SAVE_HASH, SA_NOT_SAVE_IV,
SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE,
SA_NO_HEADER_PROC, SA_HASH_ALG_NULL,
SA_CIPHER_ALG_AES, SA_PAD_TYPE_ZERO,
SA_OP_GROUP_BASIC, SA_OPCODE_DECRYPT,
DIR_INBOUND);
set_dynamic_sa_command_1(sa, cm, SA_HASH_MODE_HASH,
fb, SA_EXTENDED_SN_OFF,
SA_SEQ_MASK_OFF, SA_MC_ENABLE,
SA_NOT_COPY_PAD, SA_NOT_COPY_PAYLOAD,
SA_NOT_COPY_HDR);
crypto4xx_memcpy_le(ctx->sa_in + get_dynamic_sa_offset_key_field(ctx),
key, keylen);
sa->sa_contents = SA_AES_CONTENTS | (keylen << 2);
sa->sa_command_1.bf.key_len = keylen >> 3;
ctx->is_hash = 0;
ctx->direction = DIR_INBOUND;
memcpy(ctx->sa_in + get_dynamic_sa_offset_state_ptr_field(ctx),
(void *)&ctx->state_record_dma_addr, 4);
ctx->offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(ctx);
memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4);
sa = (struct dynamic_sa_ctl *) ctx->sa_out;
sa->sa_command_0.bf.dir = DIR_OUTBOUND;
return 0;
}
int crypto4xx_setkey_aes_cbc(struct crypto_ablkcipher *cipher,
const u8 *key, unsigned int keylen)
{
return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_CBC,
CRYPTO_FEEDBACK_MODE_NO_FB);
}
/**
* HASH SHA1 Functions
*/
static int crypto4xx_hash_alg_init(struct crypto_tfm *tfm,
unsigned int sa_len,
unsigned char ha,
unsigned char hm)
{
struct crypto_alg *alg = tfm->__crt_alg;
struct crypto4xx_alg *my_alg = crypto_alg_to_crypto4xx_alg(alg);
struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm);
struct dynamic_sa_ctl *sa;
struct dynamic_sa_hash160 *sa_in;
int rc;
ctx->dev = my_alg->dev;
ctx->is_hash = 1;
ctx->hash_final = 0;
/* Create SA */
if (ctx->sa_in_dma_addr || ctx->sa_out_dma_addr)
crypto4xx_free_sa(ctx);
rc = crypto4xx_alloc_sa(ctx, sa_len);
if (rc)
return rc;
if (ctx->state_record_dma_addr == 0) {
crypto4xx_alloc_state_record(ctx);
if (!ctx->state_record_dma_addr) {
crypto4xx_free_sa(ctx);
return -ENOMEM;
}
}
crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
sizeof(struct crypto4xx_ctx));
sa = (struct dynamic_sa_ctl *) ctx->sa_in;
set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV,
SA_NOT_LOAD_HASH, SA_LOAD_IV_FROM_SA,
SA_NO_HEADER_PROC, ha, SA_CIPHER_ALG_NULL,
SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC,
SA_OPCODE_HASH, DIR_INBOUND);
set_dynamic_sa_command_1(sa, 0, SA_HASH_MODE_HASH,
CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF,
SA_SEQ_MASK_OFF, SA_MC_ENABLE,
SA_NOT_COPY_PAD, SA_NOT_COPY_PAYLOAD,
SA_NOT_COPY_HDR);
ctx->direction = DIR_INBOUND;
sa->sa_contents = SA_HASH160_CONTENTS;
sa_in = (struct dynamic_sa_hash160 *) ctx->sa_in;
/* Need to zero hash digest in SA */
memset(sa_in->inner_digest, 0, sizeof(sa_in->inner_digest));
memset(sa_in->outer_digest, 0, sizeof(sa_in->outer_digest));
sa_in->state_ptr = ctx->state_record_dma_addr;
ctx->offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(ctx);
return 0;
}
int crypto4xx_hash_init(struct ahash_request *req)
{
struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
int ds;
struct dynamic_sa_ctl *sa;
sa = (struct dynamic_sa_ctl *) ctx->sa_in;
ds = crypto_ahash_digestsize(
__crypto_ahash_cast(req->base.tfm));
sa->sa_command_0.bf.digest_len = ds >> 2;
sa->sa_command_0.bf.load_hash_state = SA_LOAD_HASH_FROM_SA;
ctx->is_hash = 1;
ctx->direction = DIR_INBOUND;
return 0;
}
int crypto4xx_hash_update(struct ahash_request *req)
{
struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
ctx->is_hash = 1;
ctx->hash_final = 0;
ctx->pd_ctl = 0x11;
ctx->direction = DIR_INBOUND;
return crypto4xx_build_pd(&req->base, ctx, req->src,
(struct scatterlist *) req->result,
req->nbytes, NULL, 0);
}
int crypto4xx_hash_final(struct ahash_request *req)
{
return 0;
}
int crypto4xx_hash_digest(struct ahash_request *req)
{
struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
ctx->hash_final = 1;
ctx->pd_ctl = 0x11;
ctx->direction = DIR_INBOUND;
return crypto4xx_build_pd(&req->base, ctx, req->src,
(struct scatterlist *) req->result,
req->nbytes, NULL, 0);
}
/**
* SHA1 Algorithm
*/
int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm)
{
return crypto4xx_hash_alg_init(tfm, SA_HASH160_LEN, SA_HASH_ALG_SHA1,
SA_HASH_MODE_HASH);
}

1303
drivers/crypto/amcc/crypto4xx_core.c Normal file
View file

File diff suppressed because it is too large Load diff

196
drivers/crypto/amcc/crypto4xx_core.h Normal file
View file

@ -0,0 +1,196 @@
/**
* AMCC SoC PPC4xx Crypto Driver
*
* Copyright (c) 2008 Applied Micro Circuits Corporation.
* All rights reserved. James Hsiao <jhsiao@amcc.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* This is the header file for AMCC Crypto offload Linux device driver for
* use with Linux CryptoAPI.
*/
#ifndef __CRYPTO4XX_CORE_H__
#define __CRYPTO4XX_CORE_H__
#include <crypto/internal/hash.h>
#define PPC460SX_SDR0_SRST 0x201
#define PPC405EX_SDR0_SRST 0x200
#define PPC460EX_SDR0_SRST 0x201
#define PPC460EX_CE_RESET 0x08000000
#define PPC460SX_CE_RESET 0x20000000
#define PPC405EX_CE_RESET 0x00000008
#define CRYPTO4XX_CRYPTO_PRIORITY 300
#define PPC4XX_LAST_PD 63
#define PPC4XX_NUM_PD 64
#define PPC4XX_LAST_GD 1023
#define PPC4XX_NUM_GD 1024
#define PPC4XX_LAST_SD 63
#define PPC4XX_NUM_SD 64
#define PPC4XX_SD_BUFFER_SIZE 2048
#define PD_ENTRY_INUSE 1
#define PD_ENTRY_FREE 0
#define ERING_WAS_FULL 0xffffffff
struct crypto4xx_device;
struct pd_uinfo {
struct crypto4xx_device *dev;
u32 state;
u32 using_sd;
u32 first_gd; /* first gather discriptor
used by this packet */
u32 num_gd; /* number of gather discriptor
used by this packet */
u32 first_sd; /* first scatter discriptor
used by this packet */
u32 num_sd; /* number of scatter discriptors
used by this packet */
void *sa_va; /* shadow sa, when using cp from ctx->sa */
u32 sa_pa;
void *sr_va; /* state record for shadow sa */
u32 sr_pa;
struct scatterlist *dest_va;
struct crypto_async_request *async_req; /* base crypto request
for this packet */
};
struct crypto4xx_device {
struct crypto4xx_core_device *core_dev;
char *name;
u64 ce_phy_address;
void __iomem *ce_base;
void *pdr; /* base address of packet
descriptor ring */
dma_addr_t pdr_pa; /* physical address used to
program ce pdr_base_register */
void *gdr; /* gather descriptor ring */
dma_addr_t gdr_pa; /* physical address used to
program ce gdr_base_register */
void *sdr; /* scatter descriptor ring */
dma_addr_t sdr_pa; /* physical address used to
program ce sdr_base_register */
void *scatter_buffer_va;
dma_addr_t scatter_buffer_pa;
u32 scatter_buffer_size;
void *shadow_sa_pool; /* pool of memory for sa in pd_uinfo */
dma_addr_t shadow_sa_pool_pa;
void *shadow_sr_pool; /* pool of memory for sr in pd_uinfo */
dma_addr_t shadow_sr_pool_pa;
u32 pdr_tail;
u32 pdr_head;
u32 gdr_tail;
u32 gdr_head;
u32 sdr_tail;
u32 sdr_head;
void *pdr_uinfo;
struct list_head alg_list; /* List of algorithm supported
by this device */
};
struct crypto4xx_core_device {
struct device *device;
struct platform_device *ofdev;
struct crypto4xx_device *dev;
u32 int_status;
u32 irq;
struct tasklet_struct tasklet;
spinlock_t lock;
};
struct crypto4xx_ctx {
struct crypto4xx_device *dev;
void *sa_in;
dma_addr_t sa_in_dma_addr;
void *sa_out;
dma_addr_t sa_out_dma_addr;
void *state_record;
dma_addr_t state_record_dma_addr;
u32 sa_len;
u32 offset_to_sr_ptr; /* offset to state ptr, in dynamic sa */
u32 direction;
u32 next_hdr;
u32 save_iv;
u32 pd_ctl_len;
u32 pd_ctl;
u32 bypass;
u32 is_hash;
u32 hash_final;
};
struct crypto4xx_req_ctx {
struct crypto4xx_device *dev; /* Device in which
operation to send to */
void *sa;
u32 sa_dma_addr;
u16 sa_len;
};
struct crypto4xx_alg_common {
u32 type;
union {
struct crypto_alg cipher;
struct ahash_alg hash;
} u;
};
struct crypto4xx_alg {
struct list_head entry;
struct crypto4xx_alg_common alg;
struct crypto4xx_device *dev;
};
static inline struct crypto4xx_alg *crypto_alg_to_crypto4xx_alg(
struct crypto_alg *x)
{
switch (x->cra_flags & CRYPTO_ALG_TYPE_MASK) {
case CRYPTO_ALG_TYPE_AHASH:
return container_of(__crypto_ahash_alg(x),
struct crypto4xx_alg, alg.u.hash);
}
return container_of(x, struct crypto4xx_alg, alg.u.cipher);
}
extern int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size);
extern void crypto4xx_free_sa(struct crypto4xx_ctx *ctx);
extern u32 crypto4xx_alloc_sa_rctx(struct crypto4xx_ctx *ctx,
struct crypto4xx_ctx *rctx);
extern void crypto4xx_free_sa_rctx(struct crypto4xx_ctx *rctx);
extern void crypto4xx_free_ctx(struct crypto4xx_ctx *ctx);
extern u32 crypto4xx_alloc_state_record(struct crypto4xx_ctx *ctx);
extern u32 get_dynamic_sa_offset_state_ptr_field(struct crypto4xx_ctx *ctx);
extern u32 get_dynamic_sa_offset_key_field(struct crypto4xx_ctx *ctx);
extern u32 get_dynamic_sa_iv_size(struct crypto4xx_ctx *ctx);
extern void crypto4xx_memcpy_le(unsigned int *dst,
const unsigned char *buf, int len);
extern u32 crypto4xx_build_pd(struct crypto_async_request *req,
struct crypto4xx_ctx *ctx,
struct scatterlist *src,
struct scatterlist *dst,
unsigned int datalen,
void *iv, u32 iv_len);
extern int crypto4xx_setkey_aes_cbc(struct crypto_ablkcipher *cipher,
const u8 *key, unsigned int keylen);
extern int crypto4xx_encrypt(struct ablkcipher_request *req);
extern int crypto4xx_decrypt(struct ablkcipher_request *req);
extern int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm);
extern int crypto4xx_hash_digest(struct ahash_request *req);
extern int crypto4xx_hash_final(struct ahash_request *req);
extern int crypto4xx_hash_update(struct ahash_request *req);
extern int crypto4xx_hash_init(struct ahash_request *req);
#endif

284
drivers/crypto/amcc/crypto4xx_reg_def.h Normal file
View file

@ -0,0 +1,284 @@
/**
* AMCC SoC PPC4xx Crypto Driver
*
* Copyright (c) 2008 Applied Micro Circuits Corporation.
* All rights reserved. James Hsiao <jhsiao@amcc.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* This filr defines the register set for Security Subsystem
*/
#ifndef __CRYPTO4XX_REG_DEF_H__
#define __CRYPTO4XX_REG_DEF_H__
/* CRYPTO4XX Register offset */
#define CRYPTO4XX_DESCRIPTOR 0x00000000
#define CRYPTO4XX_CTRL_STAT 0x00000000
#define CRYPTO4XX_SOURCE 0x00000004
#define CRYPTO4XX_DEST 0x00000008
#define CRYPTO4XX_SA 0x0000000C
#define CRYPTO4XX_SA_LENGTH 0x00000010
#define CRYPTO4XX_LENGTH 0x00000014
#define CRYPTO4XX_PE_DMA_CFG 0x00000040
#define CRYPTO4XX_PE_DMA_STAT 0x00000044
#define CRYPTO4XX_PDR_BASE 0x00000048
#define CRYPTO4XX_RDR_BASE 0x0000004c
#define CRYPTO4XX_RING_SIZE 0x00000050
#define CRYPTO4XX_RING_CTRL 0x00000054
#define CRYPTO4XX_INT_RING_STAT 0x00000058
#define CRYPTO4XX_EXT_RING_STAT 0x0000005c
#define CRYPTO4XX_IO_THRESHOLD 0x00000060
#define CRYPTO4XX_GATH_RING_BASE 0x00000064
#define CRYPTO4XX_SCAT_RING_BASE 0x00000068
#define CRYPTO4XX_PART_RING_SIZE 0x0000006c
#define CRYPTO4XX_PART_RING_CFG 0x00000070
#define CRYPTO4XX_PDR_BASE_UADDR 0x00000080
#define CRYPTO4XX_RDR_BASE_UADDR 0x00000084
#define CRYPTO4XX_PKT_SRC_UADDR 0x00000088
#define CRYPTO4XX_PKT_DEST_UADDR 0x0000008c
#define CRYPTO4XX_SA_UADDR 0x00000090
#define CRYPTO4XX_GATH_RING_BASE_UADDR 0x000000A0
#define CRYPTO4XX_SCAT_RING_BASE_UADDR 0x000000A4
#define CRYPTO4XX_SEQ_RD 0x00000408
#define CRYPTO4XX_SEQ_MASK_RD 0x0000040C
#define CRYPTO4XX_SA_CMD_0 0x00010600
#define CRYPTO4XX_SA_CMD_1 0x00010604
#define CRYPTO4XX_STATE_PTR 0x000106dc
#define CRYPTO4XX_STATE_IV 0x00010700
#define CRYPTO4XX_STATE_HASH_BYTE_CNT_0 0x00010710
#define CRYPTO4XX_STATE_HASH_BYTE_CNT_1 0x00010714
#define CRYPTO4XX_STATE_IDIGEST_0 0x00010718
#define CRYPTO4XX_STATE_IDIGEST_1 0x0001071c
#define CRYPTO4XX_DATA_IN 0x00018000
#define CRYPTO4XX_DATA_OUT 0x0001c000
#define CRYPTO4XX_INT_UNMASK_STAT 0x000500a0
#define CRYPTO4XX_INT_MASK_STAT 0x000500a4
#define CRYPTO4XX_INT_CLR 0x000500a4
#define CRYPTO4XX_INT_EN 0x000500a8
#define CRYPTO4XX_INT_PKA 0x00000002
#define CRYPTO4XX_INT_PDR_DONE 0x00008000
#define CRYPTO4XX_INT_MA_WR_ERR 0x00020000
#define CRYPTO4XX_INT_MA_RD_ERR 0x00010000
#define CRYPTO4XX_INT_PE_ERR 0x00000200
#define CRYPTO4XX_INT_USER_DMA_ERR 0x00000040
#define CRYPTO4XX_INT_SLAVE_ERR 0x00000010
#define CRYPTO4XX_INT_MASTER_ERR 0x00000008
#define CRYPTO4XX_INT_ERROR 0x00030258
#define CRYPTO4XX_INT_CFG 0x000500ac
#define CRYPTO4XX_INT_DESCR_RD 0x000500b0
#define CRYPTO4XX_INT_DESCR_CNT 0x000500b4
#define CRYPTO4XX_INT_TIMEOUT_CNT 0x000500b8
#define CRYPTO4XX_DEVICE_CTRL 0x00060080
#define CRYPTO4XX_DEVICE_ID 0x00060084
#define CRYPTO4XX_DEVICE_INFO 0x00060088
#define CRYPTO4XX_DMA_USER_SRC 0x00060094
#define CRYPTO4XX_DMA_USER_DEST 0x00060098
#define CRYPTO4XX_DMA_USER_CMD 0x0006009C
#define CRYPTO4XX_DMA_CFG 0x000600d4
#define CRYPTO4XX_BYTE_ORDER_CFG 0x000600d8
#define CRYPTO4XX_ENDIAN_CFG 0x000600d8
#define CRYPTO4XX_PRNG_STAT 0x00070000
#define CRYPTO4XX_PRNG_CTRL 0x00070004
#define CRYPTO4XX_PRNG_SEED_L 0x00070008
#define CRYPTO4XX_PRNG_SEED_H 0x0007000c
#define CRYPTO4XX_PRNG_RES_0 0x00070020
#define CRYPTO4XX_PRNG_RES_1 0x00070024
#define CRYPTO4XX_PRNG_RES_2 0x00070028
#define CRYPTO4XX_PRNG_RES_3 0x0007002C
#define CRYPTO4XX_PRNG_LFSR_L 0x00070030
#define CRYPTO4XX_PRNG_LFSR_H 0x00070034
/**
* Initialize CRYPTO ENGINE registers, and memory bases.
*/
#define PPC4XX_PDR_POLL 0x3ff
#define PPC4XX_OUTPUT_THRESHOLD 2
#define PPC4XX_INPUT_THRESHOLD 2
#define PPC4XX_PD_SIZE 6
#define PPC4XX_CTX_DONE_INT 0x2000
#define PPC4XX_PD_DONE_INT 0x8000
#define PPC4XX_BYTE_ORDER 0x22222
#define PPC4XX_INTERRUPT_CLR 0x3ffff
#define PPC4XX_PRNG_CTRL_AUTO_EN 0x3
#define PPC4XX_DC_3DES_EN 1
#define PPC4XX_INT_DESCR_CNT 4
#define PPC4XX_INT_TIMEOUT_CNT 0
#define PPC4XX_INT_CFG 1
/**
* all follow define are ad hoc
*/
#define PPC4XX_RING_RETRY 100
#define PPC4XX_RING_POLL 100
#define PPC4XX_SDR_SIZE PPC4XX_NUM_SD
#define PPC4XX_GDR_SIZE PPC4XX_NUM_GD
/**
* Generic Security Association (SA) with all possible fields. These will
* never likely used except for reference purpose. These structure format
* can be not changed as the hardware expects them to be layout as defined.
* Field can be removed or reduced but ordering can not be changed.
*/
#define CRYPTO4XX_DMA_CFG_OFFSET 0x40
union ce_pe_dma_cfg {
struct {
u32 rsv:7;
u32 dir_host:1;
u32 rsv1:2;
u32 bo_td_en:1;
u32 dis_pdr_upd:1;
u32 bo_sgpd_en:1;
u32 bo_data_en:1;
u32 bo_sa_en:1;
u32 bo_pd_en:1;
u32 rsv2:4;
u32 dynamic_sa_en:1;
u32 pdr_mode:2;
u32 pe_mode:1;
u32 rsv3:5;
u32 reset_sg:1;
u32 reset_pdr:1;
u32 reset_pe:1;
} bf;
u32 w;
} __attribute__((packed));
#define CRYPTO4XX_PDR_BASE_OFFSET 0x48
#define CRYPTO4XX_RDR_BASE_OFFSET 0x4c
#define CRYPTO4XX_RING_SIZE_OFFSET 0x50
union ce_ring_size {
struct {
u32 ring_offset:16;
u32 rsv:6;
u32 ring_size:10;
} bf;
u32 w;
} __attribute__((packed));
#define CRYPTO4XX_RING_CONTROL_OFFSET 0x54
union ce_ring_contol {
struct {
u32 continuous:1;
u32 rsv:5;
u32 ring_retry_divisor:10;
u32 rsv1:4;
u32 ring_poll_divisor:10;
} bf;
u32 w;
} __attribute__((packed));
#define CRYPTO4XX_IO_THRESHOLD_OFFSET 0x60
union ce_io_threshold {
struct {
u32 rsv:6;
u32 output_threshold:10;
u32 rsv1:6;
u32 input_threshold:10;
} bf;
u32 w;
} __attribute__((packed));
#define CRYPTO4XX_GATHER_RING_BASE_OFFSET 0x64
#define CRYPTO4XX_SCATTER_RING_BASE_OFFSET 0x68
union ce_part_ring_size {
struct {
u32 sdr_size:16;
u32 gdr_size:16;
} bf;
u32 w;
} __attribute__((packed));
#define MAX_BURST_SIZE_32 0
#define MAX_BURST_SIZE_64 1
#define MAX_BURST_SIZE_128 2
#define MAX_BURST_SIZE_256 3
/* gather descriptor control length */
struct gd_ctl_len {
u32 len:16;
u32 rsv:14;
u32 done:1;
u32 ready:1;
} __attribute__((packed));
struct ce_gd {
u32 ptr;
struct gd_ctl_len ctl_len;
} __attribute__((packed));
struct sd_ctl {
u32 ctl:30;
u32 done:1;
u32 rdy:1;
} __attribute__((packed));
struct ce_sd {
u32 ptr;
struct sd_ctl ctl;
} __attribute__((packed));
#define PD_PAD_CTL_32 0x10
#define PD_PAD_CTL_64 0x20
#define PD_PAD_CTL_128 0x40
#define PD_PAD_CTL_256 0x80
union ce_pd_ctl {
struct {
u32 pd_pad_ctl:8;
u32 status:8;
u32 next_hdr:8;
u32 rsv:2;
u32 cached_sa:1;
u32 hash_final:1;
u32 init_arc4:1;
u32 rsv1:1;
u32 pe_done:1;
u32 host_ready:1;
} bf;
u32 w;
} __attribute__((packed));
union ce_pd_ctl_len {
struct {
u32 bypass:8;
u32 pe_done:1;
u32 host_ready:1;
u32 rsv:2;
u32 pkt_len:20;
} bf;
u32 w;
} __attribute__((packed));
struct ce_pd {
union ce_pd_ctl pd_ctl;
u32 src;
u32 dest;
u32 sa; /* get from ctx->sa_dma_addr */
u32 sa_len; /* only if dynamic sa is used */
union ce_pd_ctl_len pd_ctl_len;
} __attribute__((packed));
#endif

108
drivers/crypto/amcc/crypto4xx_sa.c Normal file
View file

@ -0,0 +1,108 @@
/**
* AMCC SoC PPC4xx Crypto Driver
*
* Copyright (c) 2008 Applied Micro Circuits Corporation.
* All rights reserved. James Hsiao <jhsiao@amcc.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* @file crypto4xx_sa.c
*
* This file implements the security context
* associate format.
*/
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/mod_devicetable.h>
#include <linux/interrupt.h>
#include <linux/spinlock_types.h>
#include <linux/highmem.h>
#include <linux/scatterlist.h>
#include <linux/crypto.h>
#include <crypto/algapi.h>
#include <crypto/des.h>
#include "crypto4xx_reg_def.h"
#include "crypto4xx_sa.h"
#include "crypto4xx_core.h"
u32 get_dynamic_sa_offset_iv_field(struct crypto4xx_ctx *ctx)
{
u32 offset;
union dynamic_sa_contents cts;
if (ctx->direction == DIR_INBOUND)
cts.w = ((struct dynamic_sa_ctl *)(ctx->sa_in))->sa_contents;
else
cts.w = ((struct dynamic_sa_ctl *)(ctx->sa_out))->sa_contents;
offset = cts.bf.key_size
+ cts.bf.inner_size
+ cts.bf.outer_size
+ cts.bf.spi
+ cts.bf.seq_num0
+ cts.bf.seq_num1
+ cts.bf.seq_num_mask0
+ cts.bf.seq_num_mask1
+ cts.bf.seq_num_mask2
+ cts.bf.seq_num_mask3;
return sizeof(struct dynamic_sa_ctl) + offset * 4;
}
u32 get_dynamic_sa_offset_state_ptr_field(struct crypto4xx_ctx *ctx)
{
u32 offset;
union dynamic_sa_contents cts;
if (ctx->direction == DIR_INBOUND)
cts.w = ((struct dynamic_sa_ctl *) ctx->sa_in)->sa_contents;
else
cts.w = ((struct dynamic_sa_ctl *) ctx->sa_out)->sa_contents;
offset = cts.bf.key_size
+ cts.bf.inner_size
+ cts.bf.outer_size
+ cts.bf.spi
+ cts.bf.seq_num0
+ cts.bf.seq_num1
+ cts.bf.seq_num_mask0
+ cts.bf.seq_num_mask1
+ cts.bf.seq_num_mask2
+ cts.bf.seq_num_mask3
+ cts.bf.iv0
+ cts.bf.iv1
+ cts.bf.iv2
+ cts.bf.iv3;
return sizeof(struct dynamic_sa_ctl) + offset * 4;
}
u32 get_dynamic_sa_iv_size(struct crypto4xx_ctx *ctx)
{
union dynamic_sa_contents cts;
if (ctx->direction == DIR_INBOUND)
cts.w = ((struct dynamic_sa_ctl *) ctx->sa_in)->sa_contents;
else
cts.w = ((struct dynamic_sa_ctl *) ctx->sa_out)->sa_contents;
return (cts.bf.iv0 + cts.bf.iv1 + cts.bf.iv2 + cts.bf.iv3) * 4;
}
u32 get_dynamic_sa_offset_key_field(struct crypto4xx_ctx *ctx)
{
union dynamic_sa_contents cts;
if (ctx->direction == DIR_INBOUND)
cts.w = ((struct dynamic_sa_ctl *) ctx->sa_in)->sa_contents;
else
cts.w = ((struct dynamic_sa_ctl *) ctx->sa_out)->sa_contents;
return sizeof(struct dynamic_sa_ctl);
}

243
drivers/crypto/amcc/crypto4xx_sa.h Normal file
View file

@ -0,0 +1,243 @@
/**
* AMCC SoC PPC4xx Crypto Driver
*
* Copyright (c) 2008 Applied Micro Circuits Corporation.
* All rights reserved. James Hsiao <jhsiao@amcc.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* This file defines the security context
* associate format.
*/
#ifndef __CRYPTO4XX_SA_H__
#define __CRYPTO4XX_SA_H__
#define AES_IV_SIZE 16
/**
* Contents of Dynamic Security Association (SA) with all possible fields
*/
union dynamic_sa_contents {
struct {
u32 arc4_state_ptr:1;
u32 arc4_ij_ptr:1;
u32 state_ptr:1;
u32 iv3:1;
u32 iv2:1;
u32 iv1:1;
u32 iv0:1;
u32 seq_num_mask3:1;
u32 seq_num_mask2:1;
u32 seq_num_mask1:1;
u32 seq_num_mask0:1;
u32 seq_num1:1;
u32 seq_num0:1;
u32 spi:1;
u32 outer_size:5;
u32 inner_size:5;
u32 key_size:4;
u32 cmd_size:4;
} bf;
u32 w;
} __attribute__((packed));
#define DIR_OUTBOUND 0
#define DIR_INBOUND 1
#define SA_OP_GROUP_BASIC 0
#define SA_OPCODE_ENCRYPT 0
#define SA_OPCODE_DECRYPT 0
#define SA_OPCODE_HASH 3
#define SA_CIPHER_ALG_DES 0
#define SA_CIPHER_ALG_3DES 1
#define SA_CIPHER_ALG_ARC4 2
#define SA_CIPHER_ALG_AES 3
#define SA_CIPHER_ALG_KASUMI 4
#define SA_CIPHER_ALG_NULL 15
#define SA_HASH_ALG_MD5 0
#define SA_HASH_ALG_SHA1 1
#define SA_HASH_ALG_NULL 15
#define SA_HASH_ALG_SHA1_DIGEST_SIZE 20
#define SA_LOAD_HASH_FROM_SA 0
#define SA_LOAD_HASH_FROM_STATE 2
#define SA_NOT_LOAD_HASH 3
#define SA_LOAD_IV_FROM_SA 0
#define SA_LOAD_IV_FROM_INPUT 1
#define SA_LOAD_IV_FROM_STATE 2
#define SA_LOAD_IV_GEN_IV 3
#define SA_PAD_TYPE_CONSTANT 2
#define SA_PAD_TYPE_ZERO 3
#define SA_PAD_TYPE_TLS 5
#define SA_PAD_TYPE_DTLS 5
#define SA_NOT_SAVE_HASH 0
#define SA_SAVE_HASH 1
#define SA_NOT_SAVE_IV 0
#define SA_SAVE_IV 1
#define SA_HEADER_PROC 1
#define SA_NO_HEADER_PROC 0
union sa_command_0 {
struct {
u32 scatter:1;
u32 gather:1;
u32 save_hash_state:1;
u32 save_iv:1;
u32 load_hash_state:2;
u32 load_iv:2;
u32 digest_len:4;
u32 hdr_proc:1;
u32 extend_pad:1;
u32 stream_cipher_pad:1;
u32 rsv:1;
u32 hash_alg:4;
u32 cipher_alg:4;
u32 pad_type:2;
u32 op_group:2;
u32 dir:1;
u32 opcode:3;
} bf;
u32 w;
} __attribute__((packed));
#define CRYPTO_MODE_ECB 0
#define CRYPTO_MODE_CBC 1
#define CRYPTO_FEEDBACK_MODE_NO_FB 0
#define CRYPTO_FEEDBACK_MODE_64BIT_OFB 0
#define CRYPTO_FEEDBACK_MODE_8BIT_CFB 1
#define CRYPTO_FEEDBACK_MODE_1BIT_CFB 2
#define CRYPTO_FEEDBACK_MODE_128BIT_CFB 3
#define SA_AES_KEY_LEN_128 2
#define SA_AES_KEY_LEN_192 3
#define SA_AES_KEY_LEN_256 4
#define SA_REV2 1
/**
* The follow defines bits sa_command_1
* In Basic hash mode this bit define simple hash or hmac.
* In IPsec mode, this bit define muting control.
*/
#define SA_HASH_MODE_HASH 0
#define SA_HASH_MODE_HMAC 1
#define SA_MC_ENABLE 0
#define SA_MC_DISABLE 1
#define SA_NOT_COPY_HDR 0
#define SA_COPY_HDR 1
#define SA_NOT_COPY_PAD 0
#define SA_COPY_PAD 1
#define SA_NOT_COPY_PAYLOAD 0
#define SA_COPY_PAYLOAD 1
#define SA_EXTENDED_SN_OFF 0
#define SA_EXTENDED_SN_ON 1
#define SA_SEQ_MASK_OFF 0
#define SA_SEQ_MASK_ON 1
union sa_command_1 {
struct {
u32 crypto_mode31:1;
u32 save_arc4_state:1;
u32 arc4_stateful:1;
u32 key_len:5;
u32 hash_crypto_offset:8;
u32 sa_rev:2;
u32 byte_offset:1;
u32 hmac_muting:1;
u32 feedback_mode:2;
u32 crypto_mode9_8:2;
u32 extended_seq_num:1;
u32 seq_num_mask:1;
u32 mutable_bit_proc:1;
u32 ip_version:1;
u32 copy_pad:1;
u32 copy_payload:1;
u32 copy_hdr:1;
u32 rsv1:1;
} bf;
u32 w;
} __attribute__((packed));
struct dynamic_sa_ctl {
u32 sa_contents;
union sa_command_0 sa_command_0;
union sa_command_1 sa_command_1;
} __attribute__((packed));
/**
* State Record for Security Association (SA)
*/
struct sa_state_record {
u32 save_iv[4];
u32 save_hash_byte_cnt[2];
u32 save_digest[16];
} __attribute__((packed));
/**
* Security Association (SA) for AES128
*
*/
struct dynamic_sa_aes128 {
struct dynamic_sa_ctl ctrl;
u32 key[4];
u32 iv[4]; /* for CBC, OFC, and CFB mode */
u32 state_ptr;
u32 reserved;
} __attribute__((packed));
#define SA_AES128_LEN (sizeof(struct dynamic_sa_aes128)/4)
#define SA_AES128_CONTENTS 0x3e000042
/*
* Security Association (SA) for AES192
*/
struct dynamic_sa_aes192 {
struct dynamic_sa_ctl ctrl;
u32 key[6];
u32 iv[4]; /* for CBC, OFC, and CFB mode */
u32 state_ptr;
u32 reserved;
} __attribute__((packed));
#define SA_AES192_LEN (sizeof(struct dynamic_sa_aes192)/4)
#define SA_AES192_CONTENTS 0x3e000062
/**
* Security Association (SA) for AES256
*/
struct dynamic_sa_aes256 {
struct dynamic_sa_ctl ctrl;
u32 key[8];
u32 iv[4]; /* for CBC, OFC, and CFB mode */
u32 state_ptr;
u32 reserved;
} __attribute__((packed));
#define SA_AES256_LEN (sizeof(struct dynamic_sa_aes256)/4)
#define SA_AES256_CONTENTS 0x3e000082
#define SA_AES_CONTENTS 0x3e000002
/**
* Security Association (SA) for HASH160: HMAC-SHA1
*/
struct dynamic_sa_hash160 {
struct dynamic_sa_ctl ctrl;
u32 inner_digest[5];
u32 outer_digest[5];
u32 state_ptr;
u32 reserved;
} __attribute__((packed));
#define SA_HASH160_LEN (sizeof(struct dynamic_sa_hash160)/4)
#define SA_HASH160_CONTENTS 0x2000a502
#endif

62
drivers/crypto/atmel-aes-regs.h Normal file
View file

@ -0,0 +1,62 @@
#ifndef __ATMEL_AES_REGS_H__
#define __ATMEL_AES_REGS_H__
#define AES_CR 0x00
#define AES_CR_START (1 << 0)
#define AES_CR_SWRST (1 << 8)
#define AES_CR_LOADSEED (1 << 16)
#define AES_MR 0x04
#define AES_MR_CYPHER_DEC (0 << 0)
#define AES_MR_CYPHER_ENC (1 << 0)
#define AES_MR_DUALBUFF (1 << 3)
#define AES_MR_PROCDLY_MASK (0xF << 4)
#define AES_MR_PROCDLY_OFFSET 4
#define AES_MR_SMOD_MASK (0x3 << 8)
#define AES_MR_SMOD_MANUAL (0x0 << 8)
#define AES_MR_SMOD_AUTO (0x1 << 8)
#define AES_MR_SMOD_IDATAR0 (0x2 << 8)
#define AES_MR_KEYSIZE_MASK (0x3 << 10)
#define AES_MR_KEYSIZE_128 (0x0 << 10)
#define AES_MR_KEYSIZE_192 (0x1 << 10)
#define AES_MR_KEYSIZE_256 (0x2 << 10)
#define AES_MR_OPMOD_MASK (0x7 << 12)
#define AES_MR_OPMOD_ECB (0x0 << 12)
#define AES_MR_OPMOD_CBC (0x1 << 12)
#define AES_MR_OPMOD_OFB (0x2 << 12)
#define AES_MR_OPMOD_CFB (0x3 << 12)
#define AES_MR_OPMOD_CTR (0x4 << 12)
#define AES_MR_LOD (0x1 << 15)
#define AES_MR_CFBS_MASK (0x7 << 16)
#define AES_MR_CFBS_128b (0x0 << 16)
#define AES_MR_CFBS_64b (0x1 << 16)
#define AES_MR_CFBS_32b (0x2 << 16)
#define AES_MR_CFBS_16b (0x3 << 16)
#define AES_MR_CFBS_8b (0x4 << 16)
#define AES_MR_CKEY_MASK (0xF << 20)
#define AES_MR_CKEY_OFFSET 20
#define AES_MR_CMTYP_MASK (0x1F << 24)
#define AES_MR_CMTYP_OFFSET 24
#define AES_IER 0x10
#define AES_IDR 0x14
#define AES_IMR 0x18
#define AES_ISR 0x1C
#define AES_INT_DATARDY (1 << 0)
#define AES_INT_URAD (1 << 8)
#define AES_ISR_URAT_MASK (0xF << 12)
#define AES_ISR_URAT_IDR_WR_PROC (0x0 << 12)
#define AES_ISR_URAT_ODR_RD_PROC (0x1 << 12)
#define AES_ISR_URAT_MR_WR_PROC (0x2 << 12)
#define AES_ISR_URAT_ODR_RD_SUBK (0x3 << 12)
#define AES_ISR_URAT_MR_WR_SUBK (0x4 << 12)
#define AES_ISR_URAT_WOR_RD (0x5 << 12)
#define AES_KEYWR(x) (0x20 + ((x) * 0x04))
#define AES_IDATAR(x) (0x40 + ((x) * 0x04))
#define AES_ODATAR(x) (0x50 + ((x) * 0x04))
#define AES_IVR(x) (0x60 + ((x) * 0x04))
#define AES_HW_VERSION 0xFC
#endif /* __ATMEL_AES_REGS_H__ */

1485
drivers/crypto/atmel-aes.c Normal file
View file

File diff suppressed because it is too large Load diff

51
drivers/crypto/atmel-sha-regs.h Normal file
View file

@ -0,0 +1,51 @@
#ifndef __ATMEL_SHA_REGS_H__
#define __ATMEL_SHA_REGS_H__
#define SHA_REG_DIGEST(x) (0x80 + ((x) * 0x04))
#define SHA_REG_DIN(x) (0x40 + ((x) * 0x04))
#define SHA_CR 0x00
#define SHA_CR_START (1 << 0)
#define SHA_CR_FIRST (1 << 4)
#define SHA_CR_SWRST (1 << 8)
#define SHA_MR 0x04
#define SHA_MR_MODE_MASK (0x3 << 0)
#define SHA_MR_MODE_MANUAL 0x0
#define SHA_MR_MODE_AUTO 0x1
#define SHA_MR_MODE_PDC 0x2
#define SHA_MR_PROCDLY (1 << 4)
#define SHA_MR_ALGO_SHA1 (0 << 8)
#define SHA_MR_ALGO_SHA256 (1 << 8)
#define SHA_MR_ALGO_SHA384 (2 << 8)
#define SHA_MR_ALGO_SHA512 (3 << 8)
#define SHA_MR_ALGO_SHA224 (4 << 8)
#define SHA_MR_DUALBUFF (1 << 16)
#define SHA_IER 0x10
#define SHA_IDR 0x14
#define SHA_IMR 0x18
#define SHA_ISR 0x1C
#define SHA_INT_DATARDY (1 << 0)
#define SHA_INT_ENDTX (1 << 1)
#define SHA_INT_TXBUFE (1 << 2)
#define SHA_INT_URAD (1 << 8)
#define SHA_ISR_URAT_MASK (0x7 << 12)
#define SHA_ISR_URAT_IDR (0x0 << 12)
#define SHA_ISR_URAT_ODR (0x1 << 12)
#define SHA_ISR_URAT_MR (0x2 << 12)
#define SHA_ISR_URAT_WO (0x5 << 12)
#define SHA_HW_VERSION 0xFC
#define SHA_TPR 0x108
#define SHA_TCR 0x10C
#define SHA_TNPR 0x118
#define SHA_TNCR 0x11C
#define SHA_PTCR 0x120
#define SHA_PTCR_TXTEN (1 << 8)
#define SHA_PTCR_TXTDIS (1 << 9)
#define SHA_PTSR 0x124
#define SHA_PTSR_TXTEN (1 << 8)
#endif /* __ATMEL_SHA_REGS_H__ */

1541
drivers/crypto/atmel-sha.c Normal file
View file

File diff suppressed because it is too large Load diff

91
drivers/crypto/atmel-tdes-regs.h Normal file
View file

@ -0,0 +1,91 @@
#ifndef __ATMEL_TDES_REGS_H__
#define __ATMEL_TDES_REGS_H__
#define TDES_CR 0x00
#define TDES_CR_START (1 << 0)
#define TDES_CR_SWRST (1 << 8)
#define TDES_CR_LOADSEED (1 << 16)
#define TDES_MR 0x04
#define TDES_MR_CYPHER_DEC (0 << 0)
#define TDES_MR_CYPHER_ENC (1 << 0)
#define TDES_MR_TDESMOD_MASK (0x3 << 1)
#define TDES_MR_TDESMOD_DES (0x0 << 1)
#define TDES_MR_TDESMOD_TDES (0x1 << 1)
#define TDES_MR_TDESMOD_XTEA (0x2 << 1)
#define TDES_MR_KEYMOD_3KEY (0 << 4)
#define TDES_MR_KEYMOD_2KEY (1 << 4)
#define TDES_MR_SMOD_MASK (0x3 << 8)
#define TDES_MR_SMOD_MANUAL (0x0 << 8)
#define TDES_MR_SMOD_AUTO (0x1 << 8)
#define TDES_MR_SMOD_PDC (0x2 << 8)
#define TDES_MR_OPMOD_MASK (0x3 << 12)
#define TDES_MR_OPMOD_ECB (0x0 << 12)
#define TDES_MR_OPMOD_CBC (0x1 << 12)
#define TDES_MR_OPMOD_OFB (0x2 << 12)
#define TDES_MR_OPMOD_CFB (0x3 << 12)
#define TDES_MR_LOD (0x1 << 15)
#define TDES_MR_CFBS_MASK (0x3 << 16)
#define TDES_MR_CFBS_64b (0x0 << 16)
#define TDES_MR_CFBS_32b (0x1 << 16)
#define TDES_MR_CFBS_16b (0x2 << 16)
#define TDES_MR_CFBS_8b (0x3 << 16)
#define TDES_MR_CKEY_MASK (0xF << 20)
#define TDES_MR_CKEY_OFFSET 20
#define TDES_MR_CTYPE_MASK (0x3F << 24)
#define TDES_MR_CTYPE_OFFSET 24
#define TDES_IER 0x10
#define TDES_IDR 0x14
#define TDES_IMR 0x18
#define TDES_ISR 0x1C
#define TDES_INT_DATARDY (1 << 0)
#define TDES_INT_ENDRX (1 << 1)
#define TDES_INT_ENDTX (1 << 2)
#define TDES_INT_RXBUFF (1 << 3)
#define TDES_INT_TXBUFE (1 << 4)
#define TDES_INT_URAD (1 << 8)
#define TDES_ISR_URAT_MASK (0x3 << 12)
#define TDES_ISR_URAT_IDR (0x0 << 12)
#define TDES_ISR_URAT_ODR (0x1 << 12)
#define TDES_ISR_URAT_MR (0x2 << 12)
#define TDES_ISR_URAT_WO (0x3 << 12)
#define TDES_KEY1W1R 0x20
#define TDES_KEY1W2R 0x24
#define TDES_KEY2W1R 0x28
#define TDES_KEY2W2R 0x2C
#define TDES_KEY3W1R 0x30
#define TDES_KEY3W2R 0x34
#define TDES_IDATA1R 0x40
#define TDES_IDATA2R 0x44
#define TDES_ODATA1R 0x50
#define TDES_ODATA2R 0x54
#define TDES_IV1R 0x60
#define TDES_IV2R 0x64
#define TDES_XTEARNDR 0x70
#define TDES_XTEARNDR_XTEA_RNDS_MASK (0x3F << 0)
#define TDES_XTEARNDR_XTEA_RNDS_OFFSET 0
#define TDES_HW_VERSION 0xFC
#define TDES_RPR 0x100
#define TDES_RCR 0x104
#define TDES_TPR 0x108
#define TDES_TCR 0x10C
#define TDES_RNPR 0x118
#define TDES_RNCR 0x11C
#define TDES_TNPR 0x118
#define TDES_TNCR 0x11C
#define TDES_PTCR 0x120
#define TDES_PTCR_RXTEN (1 << 0)
#define TDES_PTCR_RXTDIS (1 << 1)
#define TDES_PTCR_TXTEN (1 << 8)
#define TDES_PTCR_TXTDIS (1 << 9)
#define TDES_PTSR 0x124
#define TDES_PTSR_RXTEN (1 << 0)
#define TDES_PTSR_TXTEN (1 << 8)
#endif /* __ATMEL_TDES_REGS_H__ */

1536
drivers/crypto/atmel-tdes.c Normal file
View file

File diff suppressed because it is too large Load diff

768
drivers/crypto/bfin_crc.c Normal file
View file

@ -0,0 +1,768 @@
/*
* Cryptographic API.
*
* Support Blackfin CRC HW acceleration.
*
* Copyright 2012 Analog Devices Inc.
*
* Licensed under the GPL-2.
*/
#include <linux/err.h>
#include <linux/device.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/irq.h>
#include <linux/io.h>
#include <linux/platform_device.h>
#include <linux/scatterlist.h>
#include <linux/dma-mapping.h>
#include <linux/delay.h>
#include <linux/unaligned/access_ok.h>
#include <linux/crypto.h>
#include <linux/cryptohash.h>
#include <crypto/scatterwalk.h>
#include <crypto/algapi.h>
#include <crypto/hash.h>
#include <crypto/internal/hash.h>
#include <asm/dma.h>
#include <asm/portmux.h>
#include <asm/io.h>
#include "bfin_crc.h"
#define CRC_CCRYPTO_QUEUE_LENGTH 5
#define DRIVER_NAME "bfin-hmac-crc"
#define CHKSUM_DIGEST_SIZE 4
#define CHKSUM_BLOCK_SIZE 1
#define CRC_MAX_DMA_DESC 100
#define CRC_CRYPTO_STATE_UPDATE 1
#define CRC_CRYPTO_STATE_FINALUPDATE 2
#define CRC_CRYPTO_STATE_FINISH 3
struct bfin_crypto_crc {
struct list_head list;
struct device *dev;
spinlock_t lock;
int irq;
int dma_ch;
u32 poly;
struct crc_register *regs;
struct ahash_request *req; /* current request in operation */
struct dma_desc_array *sg_cpu; /* virt addr of sg dma descriptors */
dma_addr_t sg_dma; /* phy addr of sg dma descriptors */
u8 *sg_mid_buf;
dma_addr_t sg_mid_dma; /* phy addr of sg mid buffer */
struct tasklet_struct done_task;
struct crypto_queue queue; /* waiting requests */
u8 busy:1; /* crc device in operation flag */
};
static struct bfin_crypto_crc_list {
struct list_head dev_list;
spinlock_t lock;
} crc_list;
struct bfin_crypto_crc_reqctx {
struct bfin_crypto_crc *crc;
unsigned int total; /* total request bytes */
size_t sg_buflen; /* bytes for this update */
unsigned int sg_nents;
struct scatterlist *sg; /* sg list head for this update*/
struct scatterlist bufsl[2]; /* chained sg list */
size_t bufnext_len;
size_t buflast_len;
u8 bufnext[CHKSUM_DIGEST_SIZE]; /* extra bytes for next udpate */
u8 buflast[CHKSUM_DIGEST_SIZE]; /* extra bytes from last udpate */
u8 flag;
};
struct bfin_crypto_crc_ctx {
struct bfin_crypto_crc *crc;
u32 key;
};
/*
* derive number of elements in scatterlist
*/
static int sg_count(struct scatterlist *sg_list)
{
struct scatterlist *sg = sg_list;
int sg_nents = 1;
if (sg_list == NULL)
return 0;
while (!sg_is_last(sg)) {
sg_nents++;
sg = scatterwalk_sg_next(sg);
}
return sg_nents;
}
/*
* get element in scatter list by given index
*/
static struct scatterlist *sg_get(struct scatterlist *sg_list, unsigned int nents,
unsigned int index)
{
struct scatterlist *sg = NULL;
int i;
for_each_sg(sg_list, sg, nents, i)
if (i == index)
break;
return sg;
}
static int bfin_crypto_crc_init_hw(struct bfin_crypto_crc *crc, u32 key)
{
writel(0, &crc->regs->datacntrld);
writel(MODE_CALC_CRC << OPMODE_OFFSET, &crc->regs->control);
writel(key, &crc->regs->curresult);
/* setup CRC interrupts */
writel(CMPERRI | DCNTEXPI, &crc->regs->status);
writel(CMPERRI | DCNTEXPI, &crc->regs->intrenset);
return 0;
}
static int bfin_crypto_crc_init(struct ahash_request *req)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
struct bfin_crypto_crc_ctx *crc_ctx = crypto_ahash_ctx(tfm);
struct bfin_crypto_crc_reqctx *ctx = ahash_request_ctx(req);
struct bfin_crypto_crc *crc;
dev_dbg(ctx->crc->dev, "crc_init\n");
spin_lock_bh(&crc_list.lock);
list_for_each_entry(crc, &crc_list.dev_list, list) {
crc_ctx->crc = crc;
break;
}
spin_unlock_bh(&crc_list.lock);
if (sg_count(req->src) > CRC_MAX_DMA_DESC) {
dev_dbg(ctx->crc->dev, "init: requested sg list is too big > %d\n",
CRC_MAX_DMA_DESC);
return -EINVAL;
}
ctx->crc = crc;
ctx->bufnext_len = 0;
ctx->buflast_len = 0;
ctx->sg_buflen = 0;
ctx->total = 0;
ctx->flag = 0;
/* init crc results */
put_unaligned_le32(crc_ctx->key, req->result);
dev_dbg(ctx->crc->dev, "init: digest size: %d\n",
crypto_ahash_digestsize(tfm));
return bfin_crypto_crc_init_hw(crc, crc_ctx->key);
}
static void bfin_crypto_crc_config_dma(struct bfin_crypto_crc *crc)
{
struct scatterlist *sg;
struct bfin_crypto_crc_reqctx *ctx = ahash_request_ctx(crc->req);
int i = 0, j = 0;
unsigned long dma_config;
unsigned int dma_count;
unsigned int dma_addr;
unsigned int mid_dma_count = 0;
int dma_mod;
dma_map_sg(crc->dev, ctx->sg, ctx->sg_nents, DMA_TO_DEVICE);
for_each_sg(ctx->sg, sg, ctx->sg_nents, j) {
dma_addr = sg_dma_address(sg);
/* deduce extra bytes in last sg */
if (sg_is_last(sg))
dma_count = sg_dma_len(sg) - ctx->bufnext_len;
else
dma_count = sg_dma_len(sg);
if (mid_dma_count) {
/* Append last middle dma buffer to 4 bytes with first
bytes in current sg buffer. Move addr of current
sg and deduce the length of current sg.
*/
memcpy(crc->sg_mid_buf +(i << 2) + mid_dma_count,
sg_virt(sg),
CHKSUM_DIGEST_SIZE - mid_dma_count);
dma_addr += CHKSUM_DIGEST_SIZE - mid_dma_count;
dma_count -= CHKSUM_DIGEST_SIZE - mid_dma_count;
dma_config = DMAFLOW_ARRAY | RESTART | NDSIZE_3 |
DMAEN | PSIZE_32 | WDSIZE_32;
/* setup new dma descriptor for next middle dma */
crc->sg_cpu[i].start_addr = crc->sg_mid_dma + (i << 2);
crc->sg_cpu[i].cfg = dma_config;
crc->sg_cpu[i].x_count = 1;
crc->sg_cpu[i].x_modify = CHKSUM_DIGEST_SIZE;
dev_dbg(crc->dev, "%d: crc_dma: start_addr:0x%lx, "
"cfg:0x%lx, x_count:0x%lx, x_modify:0x%lx\n",
i, crc->sg_cpu[i].start_addr,
crc->sg_cpu[i].cfg, crc->sg_cpu[i].x_count,
crc->sg_cpu[i].x_modify);
i++;
}
dma_config = DMAFLOW_ARRAY | RESTART | NDSIZE_3 | DMAEN | PSIZE_32;
/* chop current sg dma len to multiple of 32 bits */
mid_dma_count = dma_count % 4;
dma_count &= ~0x3;
if (dma_addr % 4 == 0) {
dma_config |= WDSIZE_32;
dma_count >>= 2;
dma_mod = 4;
} else if (dma_addr % 2 == 0) {
dma_config |= WDSIZE_16;
dma_count >>= 1;
dma_mod = 2;
} else {
dma_config |= WDSIZE_8;
dma_mod = 1;
}
crc->sg_cpu[i].start_addr = dma_addr;
crc->sg_cpu[i].cfg = dma_config;
crc->sg_cpu[i].x_count = dma_count;
crc->sg_cpu[i].x_modify = dma_mod;
dev_dbg(crc->dev, "%d: crc_dma: start_addr:0x%lx, "
"cfg:0x%lx, x_count:0x%lx, x_modify:0x%lx\n",
i, crc->sg_cpu[i].start_addr,
crc->sg_cpu[i].cfg, crc->sg_cpu[i].x_count,
crc->sg_cpu[i].x_modify);
i++;
if (mid_dma_count) {
/* copy extra bytes to next middle dma buffer */
memcpy(crc->sg_mid_buf + (i << 2),
(u8*)sg_virt(sg) + (dma_count << 2),
mid_dma_count);
}
}
dma_config = DMAFLOW_ARRAY | RESTART | NDSIZE_3 | DMAEN | PSIZE_32 | WDSIZE_32;
/* For final update req, append the buffer for next update as well*/
if (ctx->bufnext_len && (ctx->flag == CRC_CRYPTO_STATE_FINALUPDATE ||
ctx->flag == CRC_CRYPTO_STATE_FINISH)) {
crc->sg_cpu[i].start_addr = dma_map_single(crc->dev, ctx->bufnext,
CHKSUM_DIGEST_SIZE, DMA_TO_DEVICE);
crc->sg_cpu[i].cfg = dma_config;
crc->sg_cpu[i].x_count = 1;
crc->sg_cpu[i].x_modify = CHKSUM_DIGEST_SIZE;
dev_dbg(crc->dev, "%d: crc_dma: start_addr:0x%lx, "
"cfg:0x%lx, x_count:0x%lx, x_modify:0x%lx\n",
i, crc->sg_cpu[i].start_addr,
crc->sg_cpu[i].cfg, crc->sg_cpu[i].x_count,
crc->sg_cpu[i].x_modify);
i++;
}
if (i == 0)
return;
/* Set the last descriptor to stop mode */
crc->sg_cpu[i - 1].cfg &= ~(DMAFLOW | NDSIZE);
crc->sg_cpu[i - 1].cfg |= DI_EN;
set_dma_curr_desc_addr(crc->dma_ch, (unsigned long *)crc->sg_dma);
set_dma_x_count(crc->dma_ch, 0);
set_dma_x_modify(crc->dma_ch, 0);
set_dma_config(crc->dma_ch, dma_config);
}
static int bfin_crypto_crc_handle_queue(struct bfin_crypto_crc *crc,
struct ahash_request *req)
{
struct crypto_async_request *async_req, *backlog;
struct bfin_crypto_crc_reqctx *ctx;
struct scatterlist *sg;
int ret = 0;
int nsg, i, j;
unsigned int nextlen;
unsigned long flags;
u32 reg;
spin_lock_irqsave(&crc->lock, flags);
if (req)
ret = ahash_enqueue_request(&crc->queue, req);
if (crc->busy) {
spin_unlock_irqrestore(&crc->lock, flags);
return ret;
}
backlog = crypto_get_backlog(&crc->queue);
async_req = crypto_dequeue_request(&crc->queue);
if (async_req)
crc->busy = 1;
spin_unlock_irqrestore(&crc->lock, flags);
if (!async_req)
return ret;
if (backlog)
backlog->complete(backlog, -EINPROGRESS);
req = ahash_request_cast(async_req);
crc->req = req;
ctx = ahash_request_ctx(req);
ctx->sg = NULL;
ctx->sg_buflen = 0;
ctx->sg_nents = 0;
dev_dbg(crc->dev, "handling new req, flag=%u, nbytes: %d\n",
ctx->flag, req->nbytes);
if (ctx->flag == CRC_CRYPTO_STATE_FINISH) {
if (ctx->bufnext_len == 0) {
crc->busy = 0;
return 0;
}
/* Pack last crc update buffer to 32bit */
memset(ctx->bufnext + ctx->bufnext_len, 0,
CHKSUM_DIGEST_SIZE - ctx->bufnext_len);
} else {
/* Pack small data which is less than 32bit to buffer for next update. */
if (ctx->bufnext_len + req->nbytes < CHKSUM_DIGEST_SIZE) {
memcpy(ctx->bufnext + ctx->bufnext_len,
sg_virt(req->src), req->nbytes);
ctx->bufnext_len += req->nbytes;
if (ctx->flag == CRC_CRYPTO_STATE_FINALUPDATE &&
ctx->bufnext_len) {
goto finish_update;
} else {
crc->busy = 0;
return 0;
}
}
if (ctx->bufnext_len) {
/* Chain in extra bytes of last update */
ctx->buflast_len = ctx->bufnext_len;
memcpy(ctx->buflast, ctx->bufnext, ctx->buflast_len);
nsg = ctx->sg_buflen ? 2 : 1;
sg_init_table(ctx->bufsl, nsg);
sg_set_buf(ctx->bufsl, ctx->buflast, ctx->buflast_len);
if (nsg > 1)
scatterwalk_sg_chain(ctx->bufsl, nsg,
req->src);
ctx->sg = ctx->bufsl;
} else
ctx->sg = req->src;
/* Chop crc buffer size to multiple of 32 bit */
nsg = ctx->sg_nents = sg_count(ctx->sg);
ctx->sg_buflen = ctx->buflast_len + req->nbytes;
ctx->bufnext_len = ctx->sg_buflen % 4;
ctx->sg_buflen &= ~0x3;
if (ctx->bufnext_len) {
/* copy extra bytes to buffer for next update */
memset(ctx->bufnext, 0, CHKSUM_DIGEST_SIZE);
nextlen = ctx->bufnext_len;
for (i = nsg - 1; i >= 0; i--) {
sg = sg_get(ctx->sg, nsg, i);
j = min(nextlen, sg_dma_len(sg));
memcpy(ctx->bufnext + nextlen - j,
sg_virt(sg) + sg_dma_len(sg) - j, j);
if (j == sg_dma_len(sg))
ctx->sg_nents--;
nextlen -= j;
if (nextlen == 0)
break;
}
}
}
finish_update:
if (ctx->bufnext_len && (ctx->flag == CRC_CRYPTO_STATE_FINALUPDATE ||
ctx->flag == CRC_CRYPTO_STATE_FINISH))
ctx->sg_buflen += CHKSUM_DIGEST_SIZE;
/* set CRC data count before start DMA */
writel(ctx->sg_buflen >> 2, &crc->regs->datacnt);
/* setup and enable CRC DMA */
bfin_crypto_crc_config_dma(crc);
/* finally kick off CRC operation */
reg = readl(&crc->regs->control);
writel(reg | BLKEN, &crc->regs->control);
return -EINPROGRESS;
}
static int bfin_crypto_crc_update(struct ahash_request *req)
{
struct bfin_crypto_crc_reqctx *ctx = ahash_request_ctx(req);
if (!req->nbytes)
return 0;
dev_dbg(ctx->crc->dev, "crc_update\n");
ctx->total += req->nbytes;
ctx->flag = CRC_CRYPTO_STATE_UPDATE;
return bfin_crypto_crc_handle_queue(ctx->crc, req);
}
static int bfin_crypto_crc_final(struct ahash_request *req)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
struct bfin_crypto_crc_ctx *crc_ctx = crypto_ahash_ctx(tfm);
struct bfin_crypto_crc_reqctx *ctx = ahash_request_ctx(req);
dev_dbg(ctx->crc->dev, "crc_final\n");
ctx->flag = CRC_CRYPTO_STATE_FINISH;
crc_ctx->key = 0;
return bfin_crypto_crc_handle_queue(ctx->crc, req);
}
static int bfin_crypto_crc_finup(struct ahash_request *req)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
struct bfin_crypto_crc_ctx *crc_ctx = crypto_ahash_ctx(tfm);
struct bfin_crypto_crc_reqctx *ctx = ahash_request_ctx(req);
dev_dbg(ctx->crc->dev, "crc_finishupdate\n");
ctx->total += req->nbytes;
ctx->flag = CRC_CRYPTO_STATE_FINALUPDATE;
crc_ctx->key = 0;
return bfin_crypto_crc_handle_queue(ctx->crc, req);
}
static int bfin_crypto_crc_digest(struct ahash_request *req)
{
int ret;
ret = bfin_crypto_crc_init(req);
if (ret)
return ret;
return bfin_crypto_crc_finup(req);
}
static int bfin_crypto_crc_setkey(struct crypto_ahash *tfm, const u8 *key,
unsigned int keylen)
{
struct bfin_crypto_crc_ctx *crc_ctx = crypto_ahash_ctx(tfm);
dev_dbg(crc_ctx->crc->dev, "crc_setkey\n");
if (keylen != CHKSUM_DIGEST_SIZE) {
crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
crc_ctx->key = get_unaligned_le32(key);
return 0;
}
static int bfin_crypto_crc_cra_init(struct crypto_tfm *tfm)
{
struct bfin_crypto_crc_ctx *crc_ctx = crypto_tfm_ctx(tfm);
crc_ctx->key = 0;
crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
sizeof(struct bfin_crypto_crc_reqctx));
return 0;
}
static void bfin_crypto_crc_cra_exit(struct crypto_tfm *tfm)
{
}
static struct ahash_alg algs = {
.init = bfin_crypto_crc_init,
.update = bfin_crypto_crc_update,
.final = bfin_crypto_crc_final,
.finup = bfin_crypto_crc_finup,
.digest = bfin_crypto_crc_digest,
.setkey = bfin_crypto_crc_setkey,
.halg.digestsize = CHKSUM_DIGEST_SIZE,
.halg.base = {
.cra_name = "hmac(crc32)",
.cra_driver_name = DRIVER_NAME,
.cra_priority = 100,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC,
.cra_blocksize = CHKSUM_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct bfin_crypto_crc_ctx),
.cra_alignmask = 3,
.cra_module = THIS_MODULE,
.cra_init = bfin_crypto_crc_cra_init,
.cra_exit = bfin_crypto_crc_cra_exit,
}
};
static void bfin_crypto_crc_done_task(unsigned long data)
{
struct bfin_crypto_crc *crc = (struct bfin_crypto_crc *)data;
bfin_crypto_crc_handle_queue(crc, NULL);
}
static irqreturn_t bfin_crypto_crc_handler(int irq, void *dev_id)
{
struct bfin_crypto_crc *crc = dev_id;
u32 reg;
if (readl(&crc->regs->status) & DCNTEXP) {
writel(DCNTEXP, &crc->regs->status);
/* prepare results */
put_unaligned_le32(readl(&crc->regs->result),
crc->req->result);
reg = readl(&crc->regs->control);
writel(reg & ~BLKEN, &crc->regs->control);
crc->busy = 0;
if (crc->req->base.complete)
crc->req->base.complete(&crc->req->base, 0);
tasklet_schedule(&crc->done_task);
return IRQ_HANDLED;
} else
return IRQ_NONE;
}
#ifdef CONFIG_PM
/**
* bfin_crypto_crc_suspend - suspend crc device
* @pdev: device being suspended
* @state: requested suspend state
*/
static int bfin_crypto_crc_suspend(struct platform_device *pdev, pm_message_t state)
{
struct bfin_crypto_crc *crc = platform_get_drvdata(pdev);
int i = 100000;
while ((readl(&crc->regs->control) & BLKEN) && --i)
cpu_relax();
if (i == 0)
return -EBUSY;
return 0;
}
#else
# define bfin_crypto_crc_suspend NULL
#endif
#define bfin_crypto_crc_resume NULL
/**
* bfin_crypto_crc_probe - Initialize module
*
*/
static int bfin_crypto_crc_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct resource *res;
struct bfin_crypto_crc *crc;
unsigned int timeout = 100000;
int ret;
crc = devm_kzalloc(dev, sizeof(*crc), GFP_KERNEL);
if (!crc) {
dev_err(&pdev->dev, "fail to malloc bfin_crypto_crc\n");
return -ENOMEM;
}
crc->dev = dev;
INIT_LIST_HEAD(&crc->list);
spin_lock_init(&crc->lock);
tasklet_init(&crc->done_task, bfin_crypto_crc_done_task, (unsigned long)crc);
crypto_init_queue(&crc->queue, CRC_CCRYPTO_QUEUE_LENGTH);
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
if (res == NULL) {
dev_err(&pdev->dev, "Cannot get IORESOURCE_MEM\n");
return -ENOENT;
}
crc->regs = devm_ioremap_resource(dev, res);
if (IS_ERR((void *)crc->regs)) {
dev_err(&pdev->dev, "Cannot map CRC IO\n");
return PTR_ERR((void *)crc->regs);
}
crc->irq = platform_get_irq(pdev, 0);
if (crc->irq < 0) {
dev_err(&pdev->dev, "No CRC DCNTEXP IRQ specified\n");
return -ENOENT;
}
ret = devm_request_irq(dev, crc->irq, bfin_crypto_crc_handler,
IRQF_SHARED, dev_name(dev), crc);
if (ret) {
dev_err(&pdev->dev, "Unable to request blackfin crc irq\n");
return ret;
}
res = platform_get_resource(pdev, IORESOURCE_DMA, 0);
if (res == NULL) {
dev_err(&pdev->dev, "No CRC DMA channel specified\n");
return -ENOENT;
}
crc->dma_ch = res->start;
ret = request_dma(crc->dma_ch, dev_name(dev));
if (ret) {
dev_err(&pdev->dev, "Unable to attach Blackfin CRC DMA channel\n");
return ret;
}
crc->sg_cpu = dma_alloc_coherent(&pdev->dev, PAGE_SIZE, &crc->sg_dma, GFP_KERNEL);
if (crc->sg_cpu == NULL) {
ret = -ENOMEM;
goto out_error_dma;
}
/*
* need at most CRC_MAX_DMA_DESC sg + CRC_MAX_DMA_DESC middle +
* 1 last + 1 next dma descriptors
*/
crc->sg_mid_buf = (u8 *)(crc->sg_cpu + ((CRC_MAX_DMA_DESC + 1) << 1));
crc->sg_mid_dma = crc->sg_dma + sizeof(struct dma_desc_array)
* ((CRC_MAX_DMA_DESC + 1) << 1);
writel(0, &crc->regs->control);
crc->poly = (u32)pdev->dev.platform_data;
writel(crc->poly, &crc->regs->poly);
while (!(readl(&crc->regs->status) & LUTDONE) && (--timeout) > 0)
cpu_relax();
if (timeout == 0)
dev_info(&pdev->dev, "init crc poly timeout\n");
platform_set_drvdata(pdev, crc);
spin_lock(&crc_list.lock);
list_add(&crc->list, &crc_list.dev_list);
spin_unlock(&crc_list.lock);
if (list_is_singular(&crc_list.dev_list)) {
ret = crypto_register_ahash(&algs);
if (ret) {
dev_err(&pdev->dev,
"Can't register crypto ahash device\n");
goto out_error_dma;
}
}
dev_info(&pdev->dev, "initialized\n");
return 0;
out_error_dma:
if (crc->sg_cpu)
dma_free_coherent(&pdev->dev, PAGE_SIZE, crc->sg_cpu, crc->sg_dma);
free_dma(crc->dma_ch);
return ret;
}
/**
* bfin_crypto_crc_remove - Initialize module
*
*/
static int bfin_crypto_crc_remove(struct platform_device *pdev)
{
struct bfin_crypto_crc *crc = platform_get_drvdata(pdev);
if (!crc)
return -ENODEV;
spin_lock(&crc_list.lock);
list_del(&crc->list);
spin_unlock(&crc_list.lock);
crypto_unregister_ahash(&algs);
tasklet_kill(&crc->done_task);
free_dma(crc->dma_ch);
return 0;
}
static struct platform_driver bfin_crypto_crc_driver = {
.probe = bfin_crypto_crc_probe,
.remove = bfin_crypto_crc_remove,
.suspend = bfin_crypto_crc_suspend,
.resume = bfin_crypto_crc_resume,
.driver = {
.name = DRIVER_NAME,
.owner = THIS_MODULE,
},
};
/**
* bfin_crypto_crc_mod_init - Initialize module
*
* Checks the module params and registers the platform driver.
* Real work is in the platform probe function.
*/
static int __init bfin_crypto_crc_mod_init(void)
{
int ret;
pr_info("Blackfin hardware CRC crypto driver\n");
INIT_LIST_HEAD(&crc_list.dev_list);
spin_lock_init(&crc_list.lock);
ret = platform_driver_register(&bfin_crypto_crc_driver);
if (ret) {
pr_info(KERN_ERR "unable to register driver\n");
return ret;
}
return 0;
}
/**
* bfin_crypto_crc_mod_exit - Deinitialize module
*/
static void __exit bfin_crypto_crc_mod_exit(void)
{
platform_driver_unregister(&bfin_crypto_crc_driver);
}
module_init(bfin_crypto_crc_mod_init);
module_exit(bfin_crypto_crc_mod_exit);
MODULE_AUTHOR("Sonic Zhang <sonic.zhang@analog.com>");
MODULE_DESCRIPTION("Blackfin CRC hardware crypto driver");
MODULE_LICENSE("GPL");

125
drivers/crypto/bfin_crc.h Normal file
View file

@ -0,0 +1,125 @@
/*
* bfin_crc.h - interface to Blackfin CRC controllers
*
* Copyright 2012 Analog Devices Inc.
*
* Licensed under the GPL-2 or later.
*/
#ifndef __BFIN_CRC_H__
#define __BFIN_CRC_H__
/* Function driver which use hardware crc must initialize the structure */
struct crc_info {
/* Input data address */
unsigned char *in_addr;
/* Output data address */
unsigned char *out_addr;
/* Input or output bytes */
unsigned long datasize;
union {
/* CRC to compare with that of input buffer */
unsigned long crc_compare;
/* Value to compare with input data */
unsigned long val_verify;
/* Value to fill */
unsigned long val_fill;
};
/* Value to program the 32b CRC Polynomial */
unsigned long crc_poly;
union {
/* CRC calculated from the input data */
unsigned long crc_result;
/* First failed position to verify input data */
unsigned long pos_verify;
};
/* CRC mirror flags */
unsigned int bitmirr:1;
unsigned int bytmirr:1;
unsigned int w16swp:1;
unsigned int fdsel:1;
unsigned int rsltmirr:1;
unsigned int polymirr:1;
unsigned int cmpmirr:1;
};
/* Userspace interface */
#define CRC_IOC_MAGIC 'C'
#define CRC_IOC_CALC_CRC _IOWR('C', 0x01, unsigned int)
#define CRC_IOC_MEMCPY_CRC _IOWR('C', 0x02, unsigned int)
#define CRC_IOC_VERIFY_VAL _IOWR('C', 0x03, unsigned int)
#define CRC_IOC_FILL_VAL _IOWR('C', 0x04, unsigned int)
#ifdef __KERNEL__
#include <linux/types.h>
#include <linux/spinlock.h>
#include <linux/miscdevice.h>
struct crc_register {
u32 control;
u32 datacnt;
u32 datacntrld;
u32 __pad_1[2];
u32 compare;
u32 fillval;
u32 datafifo;
u32 intren;
u32 intrenset;
u32 intrenclr;
u32 poly;
u32 __pad_2[4];
u32 status;
u32 datacntcap;
u32 __pad_3;
u32 result;
u32 curresult;
u32 __pad_4[3];
u32 revid;
};
/* CRC_STATUS Masks */
#define CMPERR 0x00000002 /* Compare error */
#define DCNTEXP 0x00000010 /* datacnt register expired */
#define IBR 0x00010000 /* Input buffer ready */
#define OBR 0x00020000 /* Output buffer ready */
#define IRR 0x00040000 /* Immediate result readt */
#define LUTDONE 0x00080000 /* Look-up table generation done */
#define FSTAT 0x00700000 /* FIFO status */
#define MAX_FIFO 4 /* Max fifo size */
/* CRC_CONTROL Masks */
#define BLKEN 0x00000001 /* Block enable */
#define OPMODE 0x000000F0 /* Operation mode */
#define OPMODE_OFFSET 4 /* Operation mode mask offset*/
#define MODE_DMACPY_CRC 1 /* MTM CRC compute and compare */
#define MODE_DATA_FILL 2 /* MTM data fill */
#define MODE_CALC_CRC 3 /* MSM CRC compute and compare */
#define MODE_DATA_VERIFY 4 /* MSM data verify */
#define AUTOCLRZ 0x00000100 /* Auto clear to zero */
#define AUTOCLRF 0x00000200 /* Auto clear to one */
#define OBRSTALL 0x00001000 /* Stall on output buffer ready */
#define IRRSTALL 0x00002000 /* Stall on immediate result ready */
#define BITMIRR 0x00010000 /* Mirror bits within each byte of 32-bit input data */
#define BITMIRR_OFFSET 16 /* Mirror bits offset */
#define BYTMIRR 0x00020000 /* Mirror bytes of 32-bit input data */
#define BYTMIRR_OFFSET 17 /* Mirror bytes offset */
#define W16SWP 0x00040000 /* Mirror uppper and lower 16-bit word of 32-bit input data */
#define W16SWP_OFFSET 18 /* Mirror 16-bit word offset */
#define FDSEL 0x00080000 /* FIFO is written after input data is mirrored */
#define FDSEL_OFFSET 19 /* Mirror FIFO offset */
#define RSLTMIRR 0x00100000 /* CRC result registers are mirrored. */
#define RSLTMIRR_OFFSET 20 /* Mirror CRC result offset. */
#define POLYMIRR 0x00200000 /* CRC poly register is mirrored. */
#define POLYMIRR_OFFSET 21 /* Mirror CRC poly offset. */
#define CMPMIRR 0x00400000 /* CRC compare register is mirrored. */
#define CMPMIRR_OFFSET 22 /* Mirror CRC compare offset. */
/* CRC_INTREN Masks */
#define CMPERRI 0x02 /* CRC_ERROR_INTR */
#define DCNTEXPI 0x10 /* CRC_STATUS_INTR */
#endif
#endif

121
drivers/crypto/caam/Kconfig Normal file
View file

@ -0,0 +1,121 @@
config CRYPTO_DEV_FSL_CAAM
tristate "Freescale CAAM-Multicore driver backend"
depends on FSL_SOC
help
Enables the driver module for Freescale's Cryptographic Accelerator
and Assurance Module (CAAM), also known as the SEC version 4 (SEC4).
This module creates job ring devices, and configures h/w
to operate as a DPAA component automatically, depending
on h/w feature availability.
To compile this driver as a module, choose M here: the module
will be called caam.
config CRYPTO_DEV_FSL_CAAM_JR
tristate "Freescale CAAM Job Ring driver backend"
depends on CRYPTO_DEV_FSL_CAAM
default y
help
Enables the driver module for Job Rings which are part of
Freescale's Cryptographic Accelerator
and Assurance Module (CAAM). This module adds a job ring operation
interface.
To compile this driver as a module, choose M here: the module
will be called caam_jr.
config CRYPTO_DEV_FSL_CAAM_RINGSIZE
int "Job Ring size"
depends on CRYPTO_DEV_FSL_CAAM_JR
range 2 9
default "9"
help
Select size of Job Rings as a power of 2, within the
range 2-9 (ring size 4-512).
Examples:
2 => 4
3 => 8
4 => 16
5 => 32
6 => 64
7 => 128
8 => 256
9 => 512
config CRYPTO_DEV_FSL_CAAM_INTC
bool "Job Ring interrupt coalescing"
depends on CRYPTO_DEV_FSL_CAAM_JR
default n
help
Enable the Job Ring's interrupt coalescing feature.
Note: the driver already provides adequate
interrupt coalescing in software.
config CRYPTO_DEV_FSL_CAAM_INTC_COUNT_THLD
int "Job Ring interrupt coalescing count threshold"
depends on CRYPTO_DEV_FSL_CAAM_INTC
range 1 255
default 255
help
Select number of descriptor completions to queue before
raising an interrupt, in the range 1-255. Note that a selection
of 1 functionally defeats the coalescing feature, and a selection
equal or greater than the job ring size will force timeouts.
config CRYPTO_DEV_FSL_CAAM_INTC_TIME_THLD
int "Job Ring interrupt coalescing timer threshold"
depends on CRYPTO_DEV_FSL_CAAM_INTC
range 1 65535
default 2048
help
Select number of bus clocks/64 to timeout in the case that one or
more descriptor completions are queued without reaching the count
threshold. Range is 1-65535.
config CRYPTO_DEV_FSL_CAAM_CRYPTO_API
tristate "Register algorithm implementations with the Crypto API"
depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR
default y
select CRYPTO_ALGAPI
select CRYPTO_AUTHENC
help
Selecting this will offload crypto for users of the
scatterlist crypto API (such as the linux native IPSec
stack) to the SEC4 via job ring.
To compile this as a module, choose M here: the module
will be called caamalg.
config CRYPTO_DEV_FSL_CAAM_AHASH_API
tristate "Register hash algorithm implementations with Crypto API"
depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR
default y
select CRYPTO_HASH
help
Selecting this will offload ahash for users of the
scatterlist crypto API to the SEC4 via job ring.
To compile this as a module, choose M here: the module
will be called caamhash.
config CRYPTO_DEV_FSL_CAAM_RNG_API
tristate "Register caam device for hwrng API"
depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR
default y
select CRYPTO_RNG
select HW_RANDOM
help
Selecting this will register the SEC4 hardware rng to
the hw_random API for suppying the kernel entropy pool.
To compile this as a module, choose M here: the module
will be called caamrng.
config CRYPTO_DEV_FSL_CAAM_DEBUG
bool "Enable debug output in CAAM driver"
depends on CRYPTO_DEV_FSL_CAAM
default n
help
Selecting this will enable printing of various debug
information in the CAAM driver.

15
drivers/crypto/caam/Makefile Normal file
View file

@ -0,0 +1,15 @@
#
# Makefile for the CAAM backend and dependent components
#
ifeq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_DEBUG), y)
EXTRA_CFLAGS := -DDEBUG
endif
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_JR) += caam_jr.o
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o
caam-objs := ctrl.o
caam_jr-objs := jr.o key_gen.o error.o

2544
drivers/crypto/caam/caamalg.c Normal file
View file

File diff suppressed because it is too large Load diff

1960
drivers/crypto/caam/caamhash.c Normal file
View file

File diff suppressed because it is too large Load diff

362
drivers/crypto/caam/caamrng.c Normal file
View file

@ -0,0 +1,362 @@
/*
* caam - Freescale FSL CAAM support for hw_random
*
* Copyright 2011 Freescale Semiconductor, Inc.
*
* Based on caamalg.c crypto API driver.
*
* relationship between job descriptors to shared descriptors:
*
* --------------- --------------
* | JobDesc #0 |-------------------->| ShareDesc |
* | *(buffer 0) | |------------->| (generate) |
* --------------- | | (move) |
* | | (store) |
* --------------- | --------------
* | JobDesc #1 |------|
* | *(buffer 1) |
* ---------------
*
* A job desc looks like this:
*
* ---------------------
* | Header |
* | ShareDesc Pointer |
* | SEQ_OUT_PTR |
* | (output buffer) |
* ---------------------
*
* The SharedDesc never changes, and each job descriptor points to one of two
* buffers for each device, from which the data will be copied into the
* requested destination
*/
#include <linux/hw_random.h>
#include <linux/completion.h>
#include <linux/atomic.h>
#include "compat.h"
#include "regs.h"
#include "intern.h"
#include "desc_constr.h"
#include "jr.h"
#include "error.h"
/*
* Maximum buffer size: maximum number of random, cache-aligned bytes that
* will be generated and moved to seq out ptr (extlen not allowed)
*/
#define RN_BUF_SIZE (0xffff / L1_CACHE_BYTES * \
L1_CACHE_BYTES)
/* length of descriptors */
#define DESC_JOB_O_LEN (CAAM_CMD_SZ * 2 + CAAM_PTR_SZ * 2)
#define DESC_RNG_LEN (10 * CAAM_CMD_SZ)
/* Buffer, its dma address and lock */
struct buf_data {
u8 buf[RN_BUF_SIZE];
dma_addr_t addr;
struct completion filled;
u32 hw_desc[DESC_JOB_O_LEN];
#define BUF_NOT_EMPTY 0
#define BUF_EMPTY 1
#define BUF_PENDING 2 /* Empty, but with job pending --don't submit another */
atomic_t empty;
};
/* rng per-device context */
struct caam_rng_ctx {
struct device *jrdev;
dma_addr_t sh_desc_dma;
u32 sh_desc[DESC_RNG_LEN];
unsigned int cur_buf_idx;
int current_buf;
struct buf_data bufs[2];
};
static struct caam_rng_ctx *rng_ctx;
static inline void rng_unmap_buf(struct device *jrdev, struct buf_data *bd)
{
if (bd->addr)
dma_unmap_single(jrdev, bd->addr, RN_BUF_SIZE,
DMA_FROM_DEVICE);
}
static inline void rng_unmap_ctx(struct caam_rng_ctx *ctx)
{
struct device *jrdev = ctx->jrdev;
if (ctx->sh_desc_dma)
dma_unmap_single(jrdev, ctx->sh_desc_dma, DESC_RNG_LEN,
DMA_TO_DEVICE);
rng_unmap_buf(jrdev, &ctx->bufs[0]);
rng_unmap_buf(jrdev, &ctx->bufs[1]);
}
static void rng_done(struct device *jrdev, u32 *desc, u32 err, void *context)
{
struct buf_data *bd;
bd = (struct buf_data *)((char *)desc -
offsetof(struct buf_data, hw_desc));
if (err)
caam_jr_strstatus(jrdev, err);
atomic_set(&bd->empty, BUF_NOT_EMPTY);
complete(&bd->filled);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "rng refreshed buf@: ",
DUMP_PREFIX_ADDRESS, 16, 4, bd->buf, RN_BUF_SIZE, 1);
#endif
}
static inline int submit_job(struct caam_rng_ctx *ctx, int to_current)
{
struct buf_data *bd = &ctx->bufs[!(to_current ^ ctx->current_buf)];
struct device *jrdev = ctx->jrdev;
u32 *desc = bd->hw_desc;
int err;
dev_dbg(jrdev, "submitting job %d\n", !(to_current ^ ctx->current_buf));
init_completion(&bd->filled);
err = caam_jr_enqueue(jrdev, desc, rng_done, ctx);
if (err)
complete(&bd->filled); /* don't wait on failed job*/
else
atomic_inc(&bd->empty); /* note if pending */
return err;
}
static int caam_read(struct hwrng *rng, void *data, size_t max, bool wait)
{
struct caam_rng_ctx *ctx = rng_ctx;
struct buf_data *bd = &ctx->bufs[ctx->current_buf];
int next_buf_idx, copied_idx;
int err;
if (atomic_read(&bd->empty)) {
/* try to submit job if there wasn't one */
if (atomic_read(&bd->empty) == BUF_EMPTY) {
err = submit_job(ctx, 1);
/* if can't submit job, can't even wait */
if (err)
return 0;
}
/* no immediate data, so exit if not waiting */
if (!wait)
return 0;
/* waiting for pending job */
if (atomic_read(&bd->empty))
wait_for_completion(&bd->filled);
}
next_buf_idx = ctx->cur_buf_idx + max;
dev_dbg(ctx->jrdev, "%s: start reading at buffer %d, idx %d\n",
__func__, ctx->current_buf, ctx->cur_buf_idx);
/* if enough data in current buffer */
if (next_buf_idx < RN_BUF_SIZE) {
memcpy(data, bd->buf + ctx->cur_buf_idx, max);
ctx->cur_buf_idx = next_buf_idx;
return max;
}
/* else, copy what's left... */
copied_idx = RN_BUF_SIZE - ctx->cur_buf_idx;
memcpy(data, bd->buf + ctx->cur_buf_idx, copied_idx);
ctx->cur_buf_idx = 0;
atomic_set(&bd->empty, BUF_EMPTY);
/* ...refill... */
submit_job(ctx, 1);
/* and use next buffer */
ctx->current_buf = !ctx->current_buf;
dev_dbg(ctx->jrdev, "switched to buffer %d\n", ctx->current_buf);
/* since there already is some data read, don't wait */
return copied_idx + caam_read(rng, data + copied_idx,
max - copied_idx, false);
}
static inline int rng_create_sh_desc(struct caam_rng_ctx *ctx)
{
struct device *jrdev = ctx->jrdev;
u32 *desc = ctx->sh_desc;
init_sh_desc(desc, HDR_SHARE_SERIAL);
/* Propagate errors from shared to job descriptor */
append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD);
/* Generate random bytes */
append_operation(desc, OP_ALG_ALGSEL_RNG | OP_TYPE_CLASS1_ALG);
/* Store bytes */
append_seq_fifo_store(desc, RN_BUF_SIZE, FIFOST_TYPE_RNGSTORE);
ctx->sh_desc_dma = dma_map_single(jrdev, desc, desc_bytes(desc),
DMA_TO_DEVICE);
if (dma_mapping_error(jrdev, ctx->sh_desc_dma)) {
dev_err(jrdev, "unable to map shared descriptor\n");
return -ENOMEM;
}
#ifdef DEBUG
print_hex_dump(KERN_ERR, "rng shdesc@: ", DUMP_PREFIX_ADDRESS, 16, 4,
desc, desc_bytes(desc), 1);
#endif
return 0;
}
static inline int rng_create_job_desc(struct caam_rng_ctx *ctx, int buf_id)
{
struct device *jrdev = ctx->jrdev;
struct buf_data *bd = &ctx->bufs[buf_id];
u32 *desc = bd->hw_desc;
int sh_len = desc_len(ctx->sh_desc);
init_job_desc_shared(desc, ctx->sh_desc_dma, sh_len, HDR_SHARE_DEFER |
HDR_REVERSE);
bd->addr = dma_map_single(jrdev, bd->buf, RN_BUF_SIZE, DMA_FROM_DEVICE);
if (dma_mapping_error(jrdev, bd->addr)) {
dev_err(jrdev, "unable to map dst\n");
return -ENOMEM;
}
append_seq_out_ptr_intlen(desc, bd->addr, RN_BUF_SIZE, 0);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "rng job desc@: ", DUMP_PREFIX_ADDRESS, 16, 4,
desc, desc_bytes(desc), 1);
#endif
return 0;
}
static void caam_cleanup(struct hwrng *rng)
{
int i;
struct buf_data *bd;
for (i = 0; i < 2; i++) {
bd = &rng_ctx->bufs[i];
if (atomic_read(&bd->empty) == BUF_PENDING)
wait_for_completion(&bd->filled);
}
rng_unmap_ctx(rng_ctx);
}
static int caam_init_buf(struct caam_rng_ctx *ctx, int buf_id)
{
struct buf_data *bd = &ctx->bufs[buf_id];
int err;
err = rng_create_job_desc(ctx, buf_id);
if (err)
return err;
atomic_set(&bd->empty, BUF_EMPTY);
submit_job(ctx, buf_id == ctx->current_buf);
wait_for_completion(&bd->filled);
return 0;
}
static int caam_init_rng(struct caam_rng_ctx *ctx, struct device *jrdev)
{
int err;
ctx->jrdev = jrdev;
err = rng_create_sh_desc(ctx);
if (err)
return err;
ctx->current_buf = 0;
ctx->cur_buf_idx = 0;
err = caam_init_buf(ctx, 0);
if (err)
return err;
err = caam_init_buf(ctx, 1);
if (err)
return err;
return 0;
}
static struct hwrng caam_rng = {
.name = "rng-caam",
.cleanup = caam_cleanup,
.read = caam_read,
};
static void __exit caam_rng_exit(void)
{
caam_jr_free(rng_ctx->jrdev);
hwrng_unregister(&caam_rng);
kfree(rng_ctx);
}
static int __init caam_rng_init(void)
{
struct device *dev;
struct device_node *dev_node;
struct platform_device *pdev;
struct device *ctrldev;
void *priv;
int err;
dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
if (!dev_node) {
dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
if (!dev_node)
return -ENODEV;
}
pdev = of_find_device_by_node(dev_node);
if (!pdev) {
of_node_put(dev_node);
return -ENODEV;
}
ctrldev = &pdev->dev;
priv = dev_get_drvdata(ctrldev);
of_node_put(dev_node);
/*
* If priv is NULL, it's probably because the caam driver wasn't
* properly initialized (e.g. RNG4 init failed). Thus, bail out here.
*/
if (!priv)
return -ENODEV;
dev = caam_jr_alloc();
if (IS_ERR(dev)) {
pr_err("Job Ring Device allocation for transform failed\n");
return PTR_ERR(dev);
}
rng_ctx = kmalloc(sizeof(struct caam_rng_ctx), GFP_DMA);
if (!rng_ctx)
return -ENOMEM;
err = caam_init_rng(rng_ctx, dev);
if (err)
return err;
dev_info(dev, "registering rng-caam\n");
return hwrng_register(&caam_rng);
}
module_init(caam_rng_init);
module_exit(caam_rng_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("FSL CAAM support for hw_random API");
MODULE_AUTHOR("Freescale Semiconductor - NMG");

40
drivers/crypto/caam/compat.h Normal file
View file

@ -0,0 +1,40 @@
/*
* Copyright 2008-2011 Freescale Semiconductor, Inc.
*/
#ifndef CAAM_COMPAT_H
#define CAAM_COMPAT_H
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/mod_devicetable.h>
#include <linux/device.h>
#include <linux/interrupt.h>
#include <linux/crypto.h>
#include <linux/hash.h>
#include <linux/hw_random.h>
#include <linux/of_platform.h>
#include <linux/dma-mapping.h>
#include <linux/io.h>
#include <linux/spinlock.h>
#include <linux/rtnetlink.h>
#include <linux/in.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/debugfs.h>
#include <linux/circ_buf.h>
#include <net/xfrm.h>
#include <crypto/algapi.h>
#include <crypto/null.h>
#include <crypto/aes.h>
#include <crypto/des.h>
#include <crypto/sha.h>
#include <crypto/md5.h>
#include <crypto/aead.h>
#include <crypto/authenc.h>
#include <crypto/scatterwalk.h>
#include <crypto/internal/skcipher.h>
#include <crypto/internal/hash.h>
#endif /* !defined(CAAM_COMPAT_H) */

734
drivers/crypto/caam/ctrl.c Normal file
View file

@ -0,0 +1,734 @@
/* * CAAM control-plane driver backend
* Controller-level driver, kernel property detection, initialization
*
* Copyright 2008-2012 Freescale Semiconductor, Inc.
*/
#include <linux/device.h>
#include <linux/of_address.h>
#include <linux/of_irq.h>
#include "compat.h"
#include "regs.h"
#include "intern.h"
#include "jr.h"
#include "desc_constr.h"
#include "error.h"
/*
* Descriptor to instantiate RNG State Handle 0 in normal mode and
* load the JDKEK, TDKEK and TDSK registers
*/
static void build_instantiation_desc(u32 *desc, int handle, int do_sk)
{
u32 *jump_cmd, op_flags;
init_job_desc(desc, 0);
op_flags = OP_TYPE_CLASS1_ALG | OP_ALG_ALGSEL_RNG |
(handle << OP_ALG_AAI_SHIFT) | OP_ALG_AS_INIT;
/* INIT RNG in non-test mode */
append_operation(desc, op_flags);
if (!handle && do_sk) {
/*
* For SH0, Secure Keys must be generated as well
*/
/* wait for done */
jump_cmd = append_jump(desc, JUMP_CLASS_CLASS1);
set_jump_tgt_here(desc, jump_cmd);
/*
* load 1 to clear written reg:
* resets the done interrrupt and returns the RNG to idle.
*/
append_load_imm_u32(desc, 1, LDST_SRCDST_WORD_CLRW);
/* Initialize State Handle */
append_operation(desc, OP_TYPE_CLASS1_ALG | OP_ALG_ALGSEL_RNG |
OP_ALG_AAI_RNG4_SK);
}
append_jump(desc, JUMP_CLASS_CLASS1 | JUMP_TYPE_HALT);
}
/* Descriptor for deinstantiation of State Handle 0 of the RNG block. */
static void build_deinstantiation_desc(u32 *desc, int handle)
{
init_job_desc(desc, 0);
/* Uninstantiate State Handle 0 */
append_operation(desc, OP_TYPE_CLASS1_ALG | OP_ALG_ALGSEL_RNG |
(handle << OP_ALG_AAI_SHIFT) | OP_ALG_AS_INITFINAL);
append_jump(desc, JUMP_CLASS_CLASS1 | JUMP_TYPE_HALT);
}
/*
* run_descriptor_deco0 - runs a descriptor on DECO0, under direct control of
* the software (no JR/QI used).
* @ctrldev - pointer to device
* @status - descriptor status, after being run
*
* Return: - 0 if no error occurred
* - -ENODEV if the DECO couldn't be acquired
* - -EAGAIN if an error occurred while executing the descriptor
*/
static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc,
u32 *status)
{
struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev);
struct caam_ctrl __iomem *ctrl = ctrlpriv->ctrl;
struct caam_deco __iomem *deco = ctrlpriv->deco;
unsigned int timeout = 100000;
u32 deco_dbg_reg, flags;
int i;
if (ctrlpriv->virt_en == 1) {
setbits32(&ctrl->deco_rsr, DECORSR_JR0);
while (!(rd_reg32(&ctrl->deco_rsr) & DECORSR_VALID) &&
--timeout)
cpu_relax();
timeout = 100000;
}
setbits32(&ctrl->deco_rq, DECORR_RQD0ENABLE);
while (!(rd_reg32(&ctrl->deco_rq) & DECORR_DEN0) &&
--timeout)
cpu_relax();
if (!timeout) {
dev_err(ctrldev, "failed to acquire DECO 0\n");
clrbits32(&ctrl->deco_rq, DECORR_RQD0ENABLE);
return -ENODEV;
}
for (i = 0; i < desc_len(desc); i++)
wr_reg32(&deco->descbuf[i], *(desc + i));
flags = DECO_JQCR_WHL;
/*
* If the descriptor length is longer than 4 words, then the
* FOUR bit in JRCTRL register must be set.
*/
if (desc_len(desc) >= 4)
flags |= DECO_JQCR_FOUR;
/* Instruct the DECO to execute it */
wr_reg32(&deco->jr_ctl_hi, flags);
timeout = 10000000;
do {
deco_dbg_reg = rd_reg32(&deco->desc_dbg);
/*
* If an error occured in the descriptor, then
* the DECO status field will be set to 0x0D
*/
if ((deco_dbg_reg & DESC_DBG_DECO_STAT_MASK) ==
DESC_DBG_DECO_STAT_HOST_ERR)
break;
cpu_relax();
} while ((deco_dbg_reg & DESC_DBG_DECO_STAT_VALID) && --timeout);
*status = rd_reg32(&deco->op_status_hi) &
DECO_OP_STATUS_HI_ERR_MASK;
if (ctrlpriv->virt_en == 1)
clrbits32(&ctrl->deco_rsr, DECORSR_JR0);
/* Mark the DECO as free */
clrbits32(&ctrl->deco_rq, DECORR_RQD0ENABLE);
if (!timeout)
return -EAGAIN;
return 0;
}
/*
* instantiate_rng - builds and executes a descriptor on DECO0,
* which initializes the RNG block.
* @ctrldev - pointer to device
* @state_handle_mask - bitmask containing the instantiation status
* for the RNG4 state handles which exist in
* the RNG4 block: 1 if it's been instantiated
* by an external entry, 0 otherwise.
* @gen_sk - generate data to be loaded into the JDKEK, TDKEK and TDSK;
* Caution: this can be done only once; if the keys need to be
* regenerated, a POR is required
*
* Return: - 0 if no error occurred
* - -ENOMEM if there isn't enough memory to allocate the descriptor
* - -ENODEV if DECO0 couldn't be acquired
* - -EAGAIN if an error occurred when executing the descriptor
* f.i. there was a RNG hardware error due to not "good enough"
* entropy being aquired.
*/
static int instantiate_rng(struct device *ctrldev, int state_handle_mask,
int gen_sk)
{
struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev);
struct caam_ctrl __iomem *ctrl;
struct rng4tst __iomem *r4tst;
u32 *desc, status, rdsta_val;
int ret = 0, sh_idx;
ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl;
r4tst = &ctrl->r4tst[0];
desc = kmalloc(CAAM_CMD_SZ * 7, GFP_KERNEL);
if (!desc)
return -ENOMEM;
for (sh_idx = 0; sh_idx < RNG4_MAX_HANDLES; sh_idx++) {
/*
* If the corresponding bit is set, this state handle
* was initialized by somebody else, so it's left alone.
*/
if ((1 << sh_idx) & state_handle_mask)
continue;
/* Create the descriptor for instantiating RNG State Handle */
build_instantiation_desc(desc, sh_idx, gen_sk);
/* Try to run it through DECO0 */
ret = run_descriptor_deco0(ctrldev, desc, &status);
/*
* If ret is not 0, or descriptor status is not 0, then
* something went wrong. No need to try the next state
* handle (if available), bail out here.
* Also, if for some reason, the State Handle didn't get
* instantiated although the descriptor has finished
* without any error (HW optimizations for later
* CAAM eras), then try again.
*/
rdsta_val =
rd_reg32(&ctrl->r4tst[0].rdsta) & RDSTA_IFMASK;
if (status || !(rdsta_val & (1 << sh_idx)))
ret = -EAGAIN;
if (ret)
break;
dev_info(ctrldev, "Instantiated RNG4 SH%d\n", sh_idx);
/* Clear the contents before recreating the descriptor */
memset(desc, 0x00, CAAM_CMD_SZ * 7);
}
kfree(desc);
return ret;
}
/*
* deinstantiate_rng - builds and executes a descriptor on DECO0,
* which deinitializes the RNG block.
* @ctrldev - pointer to device
* @state_handle_mask - bitmask containing the instantiation status
* for the RNG4 state handles which exist in
* the RNG4 block: 1 if it's been instantiated
*
* Return: - 0 if no error occurred
* - -ENOMEM if there isn't enough memory to allocate the descriptor
* - -ENODEV if DECO0 couldn't be acquired
* - -EAGAIN if an error occurred when executing the descriptor
*/
static int deinstantiate_rng(struct device *ctrldev, int state_handle_mask)
{
u32 *desc, status;
int sh_idx, ret = 0;
desc = kmalloc(CAAM_CMD_SZ * 3, GFP_KERNEL);
if (!desc)
return -ENOMEM;
for (sh_idx = 0; sh_idx < RNG4_MAX_HANDLES; sh_idx++) {
/*
* If the corresponding bit is set, then it means the state
* handle was initialized by us, and thus it needs to be
* deintialized as well
*/
if ((1 << sh_idx) & state_handle_mask) {
/*
* Create the descriptor for deinstantating this state
* handle
*/
build_deinstantiation_desc(desc, sh_idx);
/* Try to run it through DECO0 */
ret = run_descriptor_deco0(ctrldev, desc, &status);
if (ret || status) {
dev_err(ctrldev,
"Failed to deinstantiate RNG4 SH%d\n",
sh_idx);
break;
}
dev_info(ctrldev, "Deinstantiated RNG4 SH%d\n", sh_idx);
}
}
kfree(desc);
return ret;
}
static int caam_remove(struct platform_device *pdev)
{
struct device *ctrldev;
struct caam_drv_private *ctrlpriv;
struct caam_ctrl __iomem *ctrl;
int ring, ret = 0;
ctrldev = &pdev->dev;
ctrlpriv = dev_get_drvdata(ctrldev);
ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl;
/* Remove platform devices for JobRs */
for (ring = 0; ring < ctrlpriv->total_jobrs; ring++) {
if (ctrlpriv->jrpdev[ring])
of_device_unregister(ctrlpriv->jrpdev[ring]);
}
/* De-initialize RNG state handles initialized by this driver. */
if (ctrlpriv->rng4_sh_init)
deinstantiate_rng(ctrldev, ctrlpriv->rng4_sh_init);
/* Shut down debug views */
#ifdef CONFIG_DEBUG_FS
debugfs_remove_recursive(ctrlpriv->dfs_root);
#endif
/* Unmap controller region */
iounmap(&ctrl);
return ret;
}
/*
* kick_trng - sets the various parameters for enabling the initialization
* of the RNG4 block in CAAM
* @pdev - pointer to the platform device
* @ent_delay - Defines the length (in system clocks) of each entropy sample.
*/
static void kick_trng(struct platform_device *pdev, int ent_delay)
{
struct device *ctrldev = &pdev->dev;
struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev);
struct caam_ctrl __iomem *ctrl;
struct rng4tst __iomem *r4tst;
u32 val;
ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl;
r4tst = &ctrl->r4tst[0];
/* put RNG4 into program mode */
setbits32(&r4tst->rtmctl, RTMCTL_PRGM);
/*
* Performance-wise, it does not make sense to
* set the delay to a value that is lower
* than the last one that worked (i.e. the state handles
* were instantiated properly. Thus, instead of wasting
* time trying to set the values controlling the sample
* frequency, the function simply returns.
*/
val = (rd_reg32(&r4tst->rtsdctl) & RTSDCTL_ENT_DLY_MASK)
>> RTSDCTL_ENT_DLY_SHIFT;
if (ent_delay <= val) {
/* put RNG4 into run mode */
clrbits32(&r4tst->rtmctl, RTMCTL_PRGM);
return;
}
val = rd_reg32(&r4tst->rtsdctl);
val = (val & ~RTSDCTL_ENT_DLY_MASK) |
(ent_delay << RTSDCTL_ENT_DLY_SHIFT);
wr_reg32(&r4tst->rtsdctl, val);
/* min. freq. count, equal to 1/4 of the entropy sample length */
wr_reg32(&r4tst->rtfrqmin, ent_delay >> 2);
/* disable maximum frequency count */
wr_reg32(&r4tst->rtfrqmax, RTFRQMAX_DISABLE);
/* read the control register */
val = rd_reg32(&r4tst->rtmctl);
/*
* select raw sampling in both entropy shifter
* and statistical checker
*/
setbits32(&val, RTMCTL_SAMP_MODE_RAW_ES_SC);
/* put RNG4 into run mode */
clrbits32(&val, RTMCTL_PRGM);
/* write back the control register */
wr_reg32(&r4tst->rtmctl, val);
}
/**
* caam_get_era() - Return the ERA of the SEC on SoC, based
* on "sec-era" propery in the DTS. This property is updated by u-boot.
**/
int caam_get_era(void)
{
struct device_node *caam_node;
for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") {
const uint32_t *prop = (uint32_t *)of_get_property(caam_node,
"fsl,sec-era",
NULL);
return prop ? *prop : -ENOTSUPP;
}
return -ENOTSUPP;
}
EXPORT_SYMBOL(caam_get_era);
/* Probe routine for CAAM top (controller) level */
static int caam_probe(struct platform_device *pdev)
{
int ret, ring, rspec, gen_sk, ent_delay = RTSDCTL_ENT_DLY_MIN;
u64 caam_id;
struct device *dev;
struct device_node *nprop, *np;
struct caam_ctrl __iomem *ctrl;
struct caam_drv_private *ctrlpriv;
#ifdef CONFIG_DEBUG_FS
struct caam_perfmon *perfmon;
#endif
u32 scfgr, comp_params;
u32 cha_vid_ls;
int pg_size;
int BLOCK_OFFSET = 0;
ctrlpriv = devm_kzalloc(&pdev->dev, sizeof(struct caam_drv_private),
GFP_KERNEL);
if (!ctrlpriv)
return -ENOMEM;
dev = &pdev->dev;
dev_set_drvdata(dev, ctrlpriv);
ctrlpriv->pdev = pdev;
nprop = pdev->dev.of_node;
/* Get configuration properties from device tree */
/* First, get register page */
ctrl = of_iomap(nprop, 0);
if (ctrl == NULL) {
dev_err(dev, "caam: of_iomap() failed\n");
return -ENOMEM;
}
/* Finding the page size for using the CTPR_MS register */
comp_params = rd_reg32(&ctrl->perfmon.comp_parms_ms);
pg_size = (comp_params & CTPR_MS_PG_SZ_MASK) >> CTPR_MS_PG_SZ_SHIFT;
/* Allocating the BLOCK_OFFSET based on the supported page size on
* the platform
*/
if (pg_size == 0)
BLOCK_OFFSET = PG_SIZE_4K;
else
BLOCK_OFFSET = PG_SIZE_64K;
ctrlpriv->ctrl = (struct caam_ctrl __force *)ctrl;
ctrlpriv->assure = (struct caam_assurance __force *)
((uint8_t *)ctrl +
BLOCK_OFFSET * ASSURE_BLOCK_NUMBER
);
ctrlpriv->deco = (struct caam_deco __force *)
((uint8_t *)ctrl +
BLOCK_OFFSET * DECO_BLOCK_NUMBER
);
/* Get the IRQ of the controller (for security violations only) */
ctrlpriv->secvio_irq = irq_of_parse_and_map(nprop, 0);
/*
* Enable DECO watchdogs and, if this is a PHYS_ADDR_T_64BIT kernel,
* long pointers in master configuration register
*/
setbits32(&ctrl->mcr, MCFGR_WDENABLE |
(sizeof(dma_addr_t) == sizeof(u64) ? MCFGR_LONG_PTR : 0));
/*
* Read the Compile Time paramters and SCFGR to determine
* if Virtualization is enabled for this platform
*/
scfgr = rd_reg32(&ctrl->scfgr);
ctrlpriv->virt_en = 0;
if (comp_params & CTPR_MS_VIRT_EN_INCL) {
/* VIRT_EN_INCL = 1 & VIRT_EN_POR = 1 or
* VIRT_EN_INCL = 1 & VIRT_EN_POR = 0 & SCFGR_VIRT_EN = 1
*/
if ((comp_params & CTPR_MS_VIRT_EN_POR) ||
(!(comp_params & CTPR_MS_VIRT_EN_POR) &&
(scfgr & SCFGR_VIRT_EN)))
ctrlpriv->virt_en = 1;
} else {
/* VIRT_EN_INCL = 0 && VIRT_EN_POR_VALUE = 1 */
if (comp_params & CTPR_MS_VIRT_EN_POR)
ctrlpriv->virt_en = 1;
}
if (ctrlpriv->virt_en == 1)
setbits32(&ctrl->jrstart, JRSTART_JR0_START |
JRSTART_JR1_START | JRSTART_JR2_START |
JRSTART_JR3_START);
if (sizeof(dma_addr_t) == sizeof(u64))
if (of_device_is_compatible(nprop, "fsl,sec-v5.0"))
dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40));
else
dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36));
else
dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
/*
* Detect and enable JobRs
* First, find out how many ring spec'ed, allocate references
* for all, then go probe each one.
*/
rspec = 0;
for_each_available_child_of_node(nprop, np)
if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") ||
of_device_is_compatible(np, "fsl,sec4.0-job-ring"))
rspec++;
ctrlpriv->jrpdev = devm_kzalloc(&pdev->dev,
sizeof(struct platform_device *) * rspec,
GFP_KERNEL);
if (ctrlpriv->jrpdev == NULL) {
iounmap(&ctrl);
return -ENOMEM;
}
ring = 0;
ctrlpriv->total_jobrs = 0;
for_each_available_child_of_node(nprop, np)
if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") ||
of_device_is_compatible(np, "fsl,sec4.0-job-ring")) {
ctrlpriv->jrpdev[ring] =
of_platform_device_create(np, NULL, dev);
if (!ctrlpriv->jrpdev[ring]) {
pr_warn("JR%d Platform device creation error\n",
ring);
continue;
}
ctrlpriv->jr[ring] = (struct caam_job_ring __force *)
((uint8_t *)ctrl +
(ring + JR_BLOCK_NUMBER) *
BLOCK_OFFSET
);
ctrlpriv->total_jobrs++;
ring++;
}
/* Check to see if QI present. If so, enable */
ctrlpriv->qi_present =
!!(rd_reg32(&ctrl->perfmon.comp_parms_ms) &
CTPR_MS_QI_MASK);
if (ctrlpriv->qi_present) {
ctrlpriv->qi = (struct caam_queue_if __force *)
((uint8_t *)ctrl +
BLOCK_OFFSET * QI_BLOCK_NUMBER
);
/* This is all that's required to physically enable QI */
wr_reg32(&ctrlpriv->qi->qi_control_lo, QICTL_DQEN);
}
/* If no QI and no rings specified, quit and go home */
if ((!ctrlpriv->qi_present) && (!ctrlpriv->total_jobrs)) {
dev_err(dev, "no queues configured, terminating\n");
caam_remove(pdev);
return -ENOMEM;
}
cha_vid_ls = rd_reg32(&ctrl->perfmon.cha_id_ls);
/*
* If SEC has RNG version >= 4 and RNG state handle has not been
* already instantiated, do RNG instantiation
*/
if ((cha_vid_ls & CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT >= 4) {
ctrlpriv->rng4_sh_init =
rd_reg32(&ctrl->r4tst[0].rdsta);
/*
* If the secure keys (TDKEK, JDKEK, TDSK), were already
* generated, signal this to the function that is instantiating
* the state handles. An error would occur if RNG4 attempts
* to regenerate these keys before the next POR.
*/
gen_sk = ctrlpriv->rng4_sh_init & RDSTA_SKVN ? 0 : 1;
ctrlpriv->rng4_sh_init &= RDSTA_IFMASK;
do {
int inst_handles =
rd_reg32(&ctrl->r4tst[0].rdsta) &
RDSTA_IFMASK;
/*
* If either SH were instantiated by somebody else
* (e.g. u-boot) then it is assumed that the entropy
* parameters are properly set and thus the function
* setting these (kick_trng(...)) is skipped.
* Also, if a handle was instantiated, do not change
* the TRNG parameters.
*/
if (!(ctrlpriv->rng4_sh_init || inst_handles)) {
dev_info(dev,
"Entropy delay = %u\n",
ent_delay);
kick_trng(pdev, ent_delay);
ent_delay += 400;
}
/*
* if instantiate_rng(...) fails, the loop will rerun
* and the kick_trng(...) function will modfiy the
* upper and lower limits of the entropy sampling
* interval, leading to a sucessful initialization of
* the RNG.
*/
ret = instantiate_rng(dev, inst_handles,
gen_sk);
if (ret == -EAGAIN)
/*
* if here, the loop will rerun,
* so don't hog the CPU
*/
cpu_relax();
} while ((ret == -EAGAIN) && (ent_delay < RTSDCTL_ENT_DLY_MAX));
if (ret) {
dev_err(dev, "failed to instantiate RNG");
caam_remove(pdev);
return ret;
}
/*
* Set handles init'ed by this module as the complement of the
* already initialized ones
*/
ctrlpriv->rng4_sh_init = ~ctrlpriv->rng4_sh_init & RDSTA_IFMASK;
/* Enable RDB bit so that RNG works faster */
setbits32(&ctrl->scfgr, SCFGR_RDBENABLE);
}
/* NOTE: RTIC detection ought to go here, around Si time */
caam_id = (u64)rd_reg32(&ctrl->perfmon.caam_id_ms) << 32 |
(u64)rd_reg32(&ctrl->perfmon.caam_id_ls);
/* Report "alive" for developer to see */
dev_info(dev, "device ID = 0x%016llx (Era %d)\n", caam_id,
caam_get_era());
dev_info(dev, "job rings = %d, qi = %d\n",
ctrlpriv->total_jobrs, ctrlpriv->qi_present);
#ifdef CONFIG_DEBUG_FS
/*
* FIXME: needs better naming distinction, as some amalgamation of
* "caam" and nprop->full_name. The OF name isn't distinctive,
* but does separate instances
*/
perfmon = (struct caam_perfmon __force *)&ctrl->perfmon;
ctrlpriv->dfs_root = debugfs_create_dir(dev_name(dev), NULL);
ctrlpriv->ctl = debugfs_create_dir("ctl", ctrlpriv->dfs_root);
/* Controller-level - performance monitor counters */
ctrlpriv->ctl_rq_dequeued =
debugfs_create_u64("rq_dequeued",
S_IRUSR | S_IRGRP | S_IROTH,
ctrlpriv->ctl, &perfmon->req_dequeued);
ctrlpriv->ctl_ob_enc_req =
debugfs_create_u64("ob_rq_encrypted",
S_IRUSR | S_IRGRP | S_IROTH,
ctrlpriv->ctl, &perfmon->ob_enc_req);
ctrlpriv->ctl_ib_dec_req =
debugfs_create_u64("ib_rq_decrypted",
S_IRUSR | S_IRGRP | S_IROTH,
ctrlpriv->ctl, &perfmon->ib_dec_req);
ctrlpriv->ctl_ob_enc_bytes =
debugfs_create_u64("ob_bytes_encrypted",
S_IRUSR | S_IRGRP | S_IROTH,
ctrlpriv->ctl, &perfmon->ob_enc_bytes);
ctrlpriv->ctl_ob_prot_bytes =
debugfs_create_u64("ob_bytes_protected",
S_IRUSR | S_IRGRP | S_IROTH,
ctrlpriv->ctl, &perfmon->ob_prot_bytes);
ctrlpriv->ctl_ib_dec_bytes =
debugfs_create_u64("ib_bytes_decrypted",
S_IRUSR | S_IRGRP | S_IROTH,
ctrlpriv->ctl, &perfmon->ib_dec_bytes);
ctrlpriv->ctl_ib_valid_bytes =
debugfs_create_u64("ib_bytes_validated",
S_IRUSR | S_IRGRP | S_IROTH,
ctrlpriv->ctl, &perfmon->ib_valid_bytes);
/* Controller level - global status values */
ctrlpriv->ctl_faultaddr =
debugfs_create_u64("fault_addr",
S_IRUSR | S_IRGRP | S_IROTH,
ctrlpriv->ctl, &perfmon->faultaddr);
ctrlpriv->ctl_faultdetail =
debugfs_create_u32("fault_detail",
S_IRUSR | S_IRGRP | S_IROTH,
ctrlpriv->ctl, &perfmon->faultdetail);
ctrlpriv->ctl_faultstatus =
debugfs_create_u32("fault_status",
S_IRUSR | S_IRGRP | S_IROTH,
ctrlpriv->ctl, &perfmon->status);
/* Internal covering keys (useful in non-secure mode only) */
ctrlpriv->ctl_kek_wrap.data = &ctrlpriv->ctrl->kek[0];
ctrlpriv->ctl_kek_wrap.size = KEK_KEY_SIZE * sizeof(u32);
ctrlpriv->ctl_kek = debugfs_create_blob("kek",
S_IRUSR |
S_IRGRP | S_IROTH,
ctrlpriv->ctl,
&ctrlpriv->ctl_kek_wrap);
ctrlpriv->ctl_tkek_wrap.data = &ctrlpriv->ctrl->tkek[0];
ctrlpriv->ctl_tkek_wrap.size = KEK_KEY_SIZE * sizeof(u32);
ctrlpriv->ctl_tkek = debugfs_create_blob("tkek",
S_IRUSR |
S_IRGRP | S_IROTH,
ctrlpriv->ctl,
&ctrlpriv->ctl_tkek_wrap);
ctrlpriv->ctl_tdsk_wrap.data = &ctrlpriv->ctrl->tdsk[0];
ctrlpriv->ctl_tdsk_wrap.size = KEK_KEY_SIZE * sizeof(u32);
ctrlpriv->ctl_tdsk = debugfs_create_blob("tdsk",
S_IRUSR |
S_IRGRP | S_IROTH,
ctrlpriv->ctl,
&ctrlpriv->ctl_tdsk_wrap);
#endif
return 0;
}
static struct of_device_id caam_match[] = {
{
.compatible = "fsl,sec-v4.0",
},
{
.compatible = "fsl,sec4.0",
},
{},
};
MODULE_DEVICE_TABLE(of, caam_match);
static struct platform_driver caam_driver = {
.driver = {
.name = "caam",
.owner = THIS_MODULE,
.of_match_table = caam_match,
},
.probe = caam_probe,
.remove = caam_remove,
};
module_platform_driver(caam_driver);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("FSL CAAM request backend");
MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");

13
drivers/crypto/caam/ctrl.h Normal file
View file

@ -0,0 +1,13 @@
/*
* CAAM control-plane driver backend public-level include definitions
*
* Copyright 2012 Freescale Semiconductor, Inc.
*/
#ifndef CTRL_H
#define CTRL_H
/* Prototypes for backend-level services exposed to APIs */
int caam_get_era(void);
#endif /* CTRL_H */

1621
drivers/crypto/caam/desc.h Normal file
View file

File diff suppressed because it is too large Load diff

388
drivers/crypto/caam/desc_constr.h Normal file
View file

@ -0,0 +1,388 @@
/*
* caam descriptor construction helper functions
*
* Copyright 2008-2012 Freescale Semiconductor, Inc.
*/
#include "desc.h"
#define IMMEDIATE (1 << 23)
#define CAAM_CMD_SZ sizeof(u32)
#define CAAM_PTR_SZ sizeof(dma_addr_t)
#define CAAM_DESC_BYTES_MAX (CAAM_CMD_SZ * MAX_CAAM_DESCSIZE)
#define DESC_JOB_IO_LEN (CAAM_CMD_SZ * 5 + CAAM_PTR_SZ * 3)
#ifdef DEBUG
#define PRINT_POS do { printk(KERN_DEBUG "%02d: %s\n", desc_len(desc),\
&__func__[sizeof("append")]); } while (0)
#else
#define PRINT_POS
#endif
#define SET_OK_NO_PROP_ERRORS (IMMEDIATE | LDST_CLASS_DECO | \
LDST_SRCDST_WORD_DECOCTRL | \
(LDOFF_CHG_SHARE_OK_NO_PROP << \
LDST_OFFSET_SHIFT))
#define DISABLE_AUTO_INFO_FIFO (IMMEDIATE | LDST_CLASS_DECO | \
LDST_SRCDST_WORD_DECOCTRL | \
(LDOFF_DISABLE_AUTO_NFIFO << LDST_OFFSET_SHIFT))
#define ENABLE_AUTO_INFO_FIFO (IMMEDIATE | LDST_CLASS_DECO | \
LDST_SRCDST_WORD_DECOCTRL | \
(LDOFF_ENABLE_AUTO_NFIFO << LDST_OFFSET_SHIFT))
static inline int desc_len(u32 *desc)
{
return *desc & HDR_DESCLEN_MASK;
}
static inline int desc_bytes(void *desc)
{
return desc_len(desc) * CAAM_CMD_SZ;
}
static inline u32 *desc_end(u32 *desc)
{
return desc + desc_len(desc);
}
static inline void *sh_desc_pdb(u32 *desc)
{
return desc + 1;
}
static inline void init_desc(u32 *desc, u32 options)
{
*desc = (options | HDR_ONE) + 1;
}
static inline void init_sh_desc(u32 *desc, u32 options)
{
PRINT_POS;
init_desc(desc, CMD_SHARED_DESC_HDR | options);
}
static inline void init_sh_desc_pdb(u32 *desc, u32 options, size_t pdb_bytes)
{
u32 pdb_len = (pdb_bytes + CAAM_CMD_SZ - 1) / CAAM_CMD_SZ;
init_sh_desc(desc, (((pdb_len + 1) << HDR_START_IDX_SHIFT) + pdb_len) |
options);
}
static inline void init_job_desc(u32 *desc, u32 options)
{
init_desc(desc, CMD_DESC_HDR | options);
}
static inline void append_ptr(u32 *desc, dma_addr_t ptr)
{
dma_addr_t *offset = (dma_addr_t *)desc_end(desc);
*offset = ptr;
(*desc) += CAAM_PTR_SZ / CAAM_CMD_SZ;
}
static inline void init_job_desc_shared(u32 *desc, dma_addr_t ptr, int len,
u32 options)
{
PRINT_POS;
init_job_desc(desc, HDR_SHARED | options |
(len << HDR_START_IDX_SHIFT));
append_ptr(desc, ptr);
}
static inline void append_data(u32 *desc, void *data, int len)
{
u32 *offset = desc_end(desc);
if (len) /* avoid sparse warning: memcpy with byte count of 0 */
memcpy(offset, data, len);
(*desc) += (len + CAAM_CMD_SZ - 1) / CAAM_CMD_SZ;
}
static inline void append_cmd(u32 *desc, u32 command)
{
u32 *cmd = desc_end(desc);
*cmd = command;
(*desc)++;
}
#define append_u32 append_cmd
static inline void append_u64(u32 *desc, u64 data)
{
u32 *offset = desc_end(desc);
*offset = upper_32_bits(data);
*(++offset) = lower_32_bits(data);
(*desc) += 2;
}
/* Write command without affecting header, and return pointer to next word */
static inline u32 *write_cmd(u32 *desc, u32 command)
{
*desc = command;
return desc + 1;
}
static inline void append_cmd_ptr(u32 *desc, dma_addr_t ptr, int len,
u32 command)
{
append_cmd(desc, command | len);
append_ptr(desc, ptr);
}
/* Write length after pointer, rather than inside command */
static inline void append_cmd_ptr_extlen(u32 *desc, dma_addr_t ptr,
unsigned int len, u32 command)
{
append_cmd(desc, command);
if (!(command & (SQIN_RTO | SQIN_PRE)))
append_ptr(desc, ptr);
append_cmd(desc, len);
}
static inline void append_cmd_data(u32 *desc, void *data, int len,
u32 command)
{
append_cmd(desc, command | IMMEDIATE | len);
append_data(desc, data, len);
}
#define APPEND_CMD_RET(cmd, op) \
static inline u32 *append_##cmd(u32 *desc, u32 options) \
{ \
u32 *cmd = desc_end(desc); \
PRINT_POS; \
append_cmd(desc, CMD_##op | options); \
return cmd; \
}
APPEND_CMD_RET(jump, JUMP)
APPEND_CMD_RET(move, MOVE)
static inline void set_jump_tgt_here(u32 *desc, u32 *jump_cmd)
{
*jump_cmd = *jump_cmd | (desc_len(desc) - (jump_cmd - desc));
}
static inline void set_move_tgt_here(u32 *desc, u32 *move_cmd)
{
*move_cmd &= ~MOVE_OFFSET_MASK;
*move_cmd = *move_cmd | ((desc_len(desc) << (MOVE_OFFSET_SHIFT + 2)) &
MOVE_OFFSET_MASK);
}
#define APPEND_CMD(cmd, op) \
static inline void append_##cmd(u32 *desc, u32 options) \
{ \
PRINT_POS; \
append_cmd(desc, CMD_##op | options); \
}
APPEND_CMD(operation, OPERATION)
#define APPEND_CMD_LEN(cmd, op) \
static inline void append_##cmd(u32 *desc, unsigned int len, u32 options) \
{ \
PRINT_POS; \
append_cmd(desc, CMD_##op | len | options); \
}
APPEND_CMD_LEN(seq_store, SEQ_STORE)
APPEND_CMD_LEN(seq_fifo_load, SEQ_FIFO_LOAD)
APPEND_CMD_LEN(seq_fifo_store, SEQ_FIFO_STORE)
#define APPEND_CMD_PTR(cmd, op) \
static inline void append_##cmd(u32 *desc, dma_addr_t ptr, unsigned int len, \
u32 options) \
{ \
PRINT_POS; \
append_cmd_ptr(desc, ptr, len, CMD_##op | options); \
}
APPEND_CMD_PTR(key, KEY)
APPEND_CMD_PTR(load, LOAD)
APPEND_CMD_PTR(fifo_load, FIFO_LOAD)
APPEND_CMD_PTR(fifo_store, FIFO_STORE)
static inline void append_store(u32 *desc, dma_addr_t ptr, unsigned int len,
u32 options)
{
u32 cmd_src;
cmd_src = options & LDST_SRCDST_MASK;
append_cmd(desc, CMD_STORE | options | len);
/* The following options do not require pointer */
if (!(cmd_src == LDST_SRCDST_WORD_DESCBUF_SHARED ||
cmd_src == LDST_SRCDST_WORD_DESCBUF_JOB ||
cmd_src == LDST_SRCDST_WORD_DESCBUF_JOB_WE ||
cmd_src == LDST_SRCDST_WORD_DESCBUF_SHARED_WE))
append_ptr(desc, ptr);
}
#define APPEND_SEQ_PTR_INTLEN(cmd, op) \
static inline void append_seq_##cmd##_ptr_intlen(u32 *desc, dma_addr_t ptr, \
unsigned int len, \
u32 options) \
{ \
PRINT_POS; \
if (options & (SQIN_RTO | SQIN_PRE)) \
append_cmd(desc, CMD_SEQ_##op##_PTR | len | options); \
else \
append_cmd_ptr(desc, ptr, len, CMD_SEQ_##op##_PTR | options); \
}
APPEND_SEQ_PTR_INTLEN(in, IN)
APPEND_SEQ_PTR_INTLEN(out, OUT)
#define APPEND_CMD_PTR_TO_IMM(cmd, op) \
static inline void append_##cmd##_as_imm(u32 *desc, void *data, \
unsigned int len, u32 options) \
{ \
PRINT_POS; \
append_cmd_data(desc, data, len, CMD_##op | options); \
}
APPEND_CMD_PTR_TO_IMM(load, LOAD);
APPEND_CMD_PTR_TO_IMM(fifo_load, FIFO_LOAD);
#define APPEND_CMD_PTR_EXTLEN(cmd, op) \
static inline void append_##cmd##_extlen(u32 *desc, dma_addr_t ptr, \
unsigned int len, u32 options) \
{ \
PRINT_POS; \
append_cmd_ptr_extlen(desc, ptr, len, CMD_##op | SQIN_EXT | options); \
}
APPEND_CMD_PTR_EXTLEN(seq_in_ptr, SEQ_IN_PTR)
APPEND_CMD_PTR_EXTLEN(seq_out_ptr, SEQ_OUT_PTR)
/*
* Determine whether to store length internally or externally depending on
* the size of its type
*/
#define APPEND_CMD_PTR_LEN(cmd, op, type) \
static inline void append_##cmd(u32 *desc, dma_addr_t ptr, \
type len, u32 options) \
{ \
PRINT_POS; \
if (sizeof(type) > sizeof(u16)) \
append_##cmd##_extlen(desc, ptr, len, options); \
else \
append_##cmd##_intlen(desc, ptr, len, options); \
}
APPEND_CMD_PTR_LEN(seq_in_ptr, SEQ_IN_PTR, u32)
APPEND_CMD_PTR_LEN(seq_out_ptr, SEQ_OUT_PTR, u32)
/*
* 2nd variant for commands whose specified immediate length differs
* from length of immediate data provided, e.g., split keys
*/
#define APPEND_CMD_PTR_TO_IMM2(cmd, op) \
static inline void append_##cmd##_as_imm(u32 *desc, void *data, \
unsigned int data_len, \
unsigned int len, u32 options) \
{ \
PRINT_POS; \
append_cmd(desc, CMD_##op | IMMEDIATE | len | options); \
append_data(desc, data, data_len); \
}
APPEND_CMD_PTR_TO_IMM2(key, KEY);
#define APPEND_CMD_RAW_IMM(cmd, op, type) \
static inline void append_##cmd##_imm_##type(u32 *desc, type immediate, \
u32 options) \
{ \
PRINT_POS; \
append_cmd(desc, CMD_##op | IMMEDIATE | options | sizeof(type)); \
append_cmd(desc, immediate); \
}
APPEND_CMD_RAW_IMM(load, LOAD, u32);
/*
* Append math command. Only the last part of destination and source need to
* be specified
*/
#define APPEND_MATH(op, desc, dest, src_0, src_1, len) \
append_cmd(desc, CMD_MATH | MATH_FUN_##op | MATH_DEST_##dest | \
MATH_SRC0_##src_0 | MATH_SRC1_##src_1 | (u32)len);
#define append_math_add(desc, dest, src0, src1, len) \
APPEND_MATH(ADD, desc, dest, src0, src1, len)
#define append_math_sub(desc, dest, src0, src1, len) \
APPEND_MATH(SUB, desc, dest, src0, src1, len)
#define append_math_add_c(desc, dest, src0, src1, len) \
APPEND_MATH(ADDC, desc, dest, src0, src1, len)
#define append_math_sub_b(desc, dest, src0, src1, len) \
APPEND_MATH(SUBB, desc, dest, src0, src1, len)
#define append_math_and(desc, dest, src0, src1, len) \
APPEND_MATH(AND, desc, dest, src0, src1, len)
#define append_math_or(desc, dest, src0, src1, len) \
APPEND_MATH(OR, desc, dest, src0, src1, len)
#define append_math_xor(desc, dest, src0, src1, len) \
APPEND_MATH(XOR, desc, dest, src0, src1, len)
#define append_math_lshift(desc, dest, src0, src1, len) \
APPEND_MATH(LSHIFT, desc, dest, src0, src1, len)
#define append_math_rshift(desc, dest, src0, src1, len) \
APPEND_MATH(RSHIFT, desc, dest, src0, src1, len)
#define append_math_ldshift(desc, dest, src0, src1, len) \
APPEND_MATH(SHLD, desc, dest, src0, src1, len)
/* Exactly one source is IMM. Data is passed in as u32 value */
#define APPEND_MATH_IMM_u32(op, desc, dest, src_0, src_1, data) \
do { \
APPEND_MATH(op, desc, dest, src_0, src_1, CAAM_CMD_SZ); \
append_cmd(desc, data); \
} while (0)
#define append_math_add_imm_u32(desc, dest, src0, src1, data) \
APPEND_MATH_IMM_u32(ADD, desc, dest, src0, src1, data)
#define append_math_sub_imm_u32(desc, dest, src0, src1, data) \
APPEND_MATH_IMM_u32(SUB, desc, dest, src0, src1, data)
#define append_math_add_c_imm_u32(desc, dest, src0, src1, data) \
APPEND_MATH_IMM_u32(ADDC, desc, dest, src0, src1, data)
#define append_math_sub_b_imm_u32(desc, dest, src0, src1, data) \
APPEND_MATH_IMM_u32(SUBB, desc, dest, src0, src1, data)
#define append_math_and_imm_u32(desc, dest, src0, src1, data) \
APPEND_MATH_IMM_u32(AND, desc, dest, src0, src1, data)
#define append_math_or_imm_u32(desc, dest, src0, src1, data) \
APPEND_MATH_IMM_u32(OR, desc, dest, src0, src1, data)
#define append_math_xor_imm_u32(desc, dest, src0, src1, data) \
APPEND_MATH_IMM_u32(XOR, desc, dest, src0, src1, data)
#define append_math_lshift_imm_u32(desc, dest, src0, src1, data) \
APPEND_MATH_IMM_u32(LSHIFT, desc, dest, src0, src1, data)
#define append_math_rshift_imm_u32(desc, dest, src0, src1, data) \
APPEND_MATH_IMM_u32(RSHIFT, desc, dest, src0, src1, data)
/* Exactly one source is IMM. Data is passed in as u64 value */
#define APPEND_MATH_IMM_u64(op, desc, dest, src_0, src_1, data) \
do { \
u32 upper = (data >> 16) >> 16; \
APPEND_MATH(op, desc, dest, src_0, src_1, CAAM_CMD_SZ * 2 | \
(upper ? 0 : MATH_IFB)); \
if (upper) \
append_u64(desc, data); \
else \
append_u32(desc, data); \
} while (0)
#define append_math_add_imm_u64(desc, dest, src0, src1, data) \
APPEND_MATH_IMM_u64(ADD, desc, dest, src0, src1, data)
#define append_math_sub_imm_u64(desc, dest, src0, src1, data) \
APPEND_MATH_IMM_u64(SUB, desc, dest, src0, src1, data)
#define append_math_add_c_imm_u64(desc, dest, src0, src1, data) \
APPEND_MATH_IMM_u64(ADDC, desc, dest, src0, src1, data)
#define append_math_sub_b_imm_u64(desc, dest, src0, src1, data) \
APPEND_MATH_IMM_u64(SUBB, desc, dest, src0, src1, data)
#define append_math_and_imm_u64(desc, dest, src0, src1, data) \
APPEND_MATH_IMM_u64(AND, desc, dest, src0, src1, data)
#define append_math_or_imm_u64(desc, dest, src0, src1, data) \
APPEND_MATH_IMM_u64(OR, desc, dest, src0, src1, data)
#define append_math_xor_imm_u64(desc, dest, src0, src1, data) \
APPEND_MATH_IMM_u64(XOR, desc, dest, src0, src1, data)
#define append_math_lshift_imm_u64(desc, dest, src0, src1, data) \
APPEND_MATH_IMM_u64(LSHIFT, desc, dest, src0, src1, data)
#define append_math_rshift_imm_u64(desc, dest, src0, src1, data) \
APPEND_MATH_IMM_u64(RSHIFT, desc, dest, src0, src1, data)

239
drivers/crypto/caam/error.c Normal file
View file

@ -0,0 +1,239 @@
/*
* CAAM Error Reporting
*
* Copyright 2009-2011 Freescale Semiconductor, Inc.
*/
#include "compat.h"
#include "regs.h"
#include "intern.h"
#include "desc.h"
#include "jr.h"
#include "error.h"
static const struct {
u8 value;
const char *error_text;
} desc_error_list[] = {
{ 0x00, "No error." },
{ 0x01, "SGT Length Error. The descriptor is trying to read more data than is contained in the SGT table." },
{ 0x02, "SGT Null Entry Error." },
{ 0x03, "Job Ring Control Error. There is a bad value in the Job Ring Control register." },
{ 0x04, "Invalid Descriptor Command. The Descriptor Command field is invalid." },
{ 0x05, "Reserved." },
{ 0x06, "Invalid KEY Command" },
{ 0x07, "Invalid LOAD Command" },
{ 0x08, "Invalid STORE Command" },
{ 0x09, "Invalid OPERATION Command" },
{ 0x0A, "Invalid FIFO LOAD Command" },
{ 0x0B, "Invalid FIFO STORE Command" },
{ 0x0C, "Invalid MOVE/MOVE_LEN Command" },
{ 0x0D, "Invalid JUMP Command. A nonlocal JUMP Command is invalid because the target is not a Job Header Command, or the jump is from a Trusted Descriptor to a Job Descriptor, or because the target Descriptor contains a Shared Descriptor." },
{ 0x0E, "Invalid MATH Command" },
{ 0x0F, "Invalid SIGNATURE Command" },
{ 0x10, "Invalid Sequence Command. A SEQ IN PTR OR SEQ OUT PTR Command is invalid or a SEQ KEY, SEQ LOAD, SEQ FIFO LOAD, or SEQ FIFO STORE decremented the input or output sequence length below 0. This error may result if a built-in PROTOCOL Command has encountered a malformed PDU." },
{ 0x11, "Skip data type invalid. The type must be 0xE or 0xF."},
{ 0x12, "Shared Descriptor Header Error" },
{ 0x13, "Header Error. Invalid length or parity, or certain other problems." },
{ 0x14, "Burster Error. Burster has gotten to an illegal state" },
{ 0x15, "Context Register Length Error. The descriptor is trying to read or write past the end of the Context Register. A SEQ LOAD or SEQ STORE with the VLF bit set was executed with too large a length in the variable length register (VSOL for SEQ STORE or VSIL for SEQ LOAD)." },
{ 0x16, "DMA Error" },
{ 0x17, "Reserved." },
{ 0x1A, "Job failed due to JR reset" },
{ 0x1B, "Job failed due to Fail Mode" },
{ 0x1C, "DECO Watchdog timer timeout error" },
{ 0x1D, "DECO tried to copy a key from another DECO but the other DECO's Key Registers were locked" },
{ 0x1E, "DECO attempted to copy data from a DECO that had an unmasked Descriptor error" },
{ 0x1F, "LIODN error. DECO was trying to share from itself or from another DECO but the two Non-SEQ LIODN values didn't match or the 'shared from' DECO's Descriptor required that the SEQ LIODNs be the same and they aren't." },
{ 0x20, "DECO has completed a reset initiated via the DRR register" },
{ 0x21, "Nonce error. When using EKT (CCM) key encryption option in the FIFO STORE Command, the Nonce counter reached its maximum value and this encryption mode can no longer be used." },
{ 0x22, "Meta data is too large (> 511 bytes) for TLS decap (input frame; block ciphers) and IPsec decap (output frame, when doing the next header byte update) and DCRC (output frame)." },
{ 0x23, "Read Input Frame error" },
{ 0x24, "JDKEK, TDKEK or TDSK not loaded error" },
{ 0x80, "DNR (do not run) error" },
{ 0x81, "undefined protocol command" },
{ 0x82, "invalid setting in PDB" },
{ 0x83, "Anti-replay LATE error" },
{ 0x84, "Anti-replay REPLAY error" },
{ 0x85, "Sequence number overflow" },
{ 0x86, "Sigver invalid signature" },
{ 0x87, "DSA Sign Illegal test descriptor" },
{ 0x88, "Protocol Format Error - A protocol has seen an error in the format of data received. When running RSA, this means that formatting with random padding was used, and did not follow the form: 0x00, 0x02, 8-to-N bytes of non-zero pad, 0x00, F data." },
{ 0x89, "Protocol Size Error - A protocol has seen an error in size. When running RSA, pdb size N < (size of F) when no formatting is used; or pdb size N < (F + 11) when formatting is used." },
{ 0xC1, "Blob Command error: Undefined mode" },
{ 0xC2, "Blob Command error: Secure Memory Blob mode error" },
{ 0xC4, "Blob Command error: Black Blob key or input size error" },
{ 0xC5, "Blob Command error: Invalid key destination" },
{ 0xC8, "Blob Command error: Trusted/Secure mode error" },
{ 0xF0, "IPsec TTL or hop limit field either came in as 0, or was decremented to 0" },
{ 0xF1, "3GPP HFN matches or exceeds the Threshold" },
};
static const char * const cha_id_list[] = {
"",
"AES",
"DES",
"ARC4",
"MDHA",
"RNG",
"SNOW f8",
"Kasumi f8/9",
"PKHA",
"CRCA",
"SNOW f9",
"ZUCE",
"ZUCA",
};
static const char * const err_id_list[] = {
"No error.",
"Mode error.",
"Data size error.",
"Key size error.",
"PKHA A memory size error.",
"PKHA B memory size error.",
"Data arrived out of sequence error.",
"PKHA divide-by-zero error.",
"PKHA modulus even error.",
"DES key parity error.",
"ICV check failed.",
"Hardware error.",
"Unsupported CCM AAD size.",
"Class 1 CHA is not reset",
"Invalid CHA combination was selected",
"Invalid CHA selected.",
};
static const char * const rng_err_id_list[] = {
"",
"",
"",
"Instantiate",
"Not instantiated",
"Test instantiate",
"Prediction resistance",
"Prediction resistance and test request",
"Uninstantiate",
"Secure key generation",
};
static void report_ccb_status(struct device *jrdev, const u32 status,
const char *error)
{
u8 cha_id = (status & JRSTA_CCBERR_CHAID_MASK) >>
JRSTA_CCBERR_CHAID_SHIFT;
u8 err_id = status & JRSTA_CCBERR_ERRID_MASK;
u8 idx = (status & JRSTA_DECOERR_INDEX_MASK) >>
JRSTA_DECOERR_INDEX_SHIFT;
char *idx_str;
const char *cha_str = "unidentified cha_id value 0x";
char cha_err_code[3] = { 0 };
const char *err_str = "unidentified err_id value 0x";
char err_err_code[3] = { 0 };
if (status & JRSTA_DECOERR_JUMP)
idx_str = "jump tgt desc idx";
else
idx_str = "desc idx";
if (cha_id < ARRAY_SIZE(cha_id_list))
cha_str = cha_id_list[cha_id];
else
snprintf(cha_err_code, sizeof(cha_err_code), "%02x", cha_id);
if ((cha_id << JRSTA_CCBERR_CHAID_SHIFT) == JRSTA_CCBERR_CHAID_RNG &&
err_id < ARRAY_SIZE(rng_err_id_list) &&
strlen(rng_err_id_list[err_id])) {
/* RNG-only error */
err_str = rng_err_id_list[err_id];
} else if (err_id < ARRAY_SIZE(err_id_list))
err_str = err_id_list[err_id];
else
snprintf(err_err_code, sizeof(err_err_code), "%02x", err_id);
dev_err(jrdev, "%08x: %s: %s %d: %s%s: %s%s\n",
status, error, idx_str, idx,
cha_str, cha_err_code,
err_str, err_err_code);
}
static void report_jump_status(struct device *jrdev, const u32 status,
const char *error)
{
dev_err(jrdev, "%08x: %s: %s() not implemented\n",
status, error, __func__);
}
static void report_deco_status(struct device *jrdev, const u32 status,
const char *error)
{
u8 err_id = status & JRSTA_DECOERR_ERROR_MASK;
u8 idx = (status & JRSTA_DECOERR_INDEX_MASK) >>
JRSTA_DECOERR_INDEX_SHIFT;
char *idx_str;
const char *err_str = "unidentified error value 0x";
char err_err_code[3] = { 0 };
int i;
if (status & JRSTA_DECOERR_JUMP)
idx_str = "jump tgt desc idx";
else
idx_str = "desc idx";
for (i = 0; i < ARRAY_SIZE(desc_error_list); i++)
if (desc_error_list[i].value == err_id)
break;
if (i != ARRAY_SIZE(desc_error_list) && desc_error_list[i].error_text)
err_str = desc_error_list[i].error_text;
else
snprintf(err_err_code, sizeof(err_err_code), "%02x", err_id);
dev_err(jrdev, "%08x: %s: %s %d: %s%s\n",
status, error, idx_str, idx, err_str, err_err_code);
}
static void report_jr_status(struct device *jrdev, const u32 status,
const char *error)
{
dev_err(jrdev, "%08x: %s: %s() not implemented\n",
status, error, __func__);
}
static void report_cond_code_status(struct device *jrdev, const u32 status,
const char *error)
{
dev_err(jrdev, "%08x: %s: %s() not implemented\n",
status, error, __func__);
}
void caam_jr_strstatus(struct device *jrdev, u32 status)
{
static const struct stat_src {
void (*report_ssed)(struct device *jrdev, const u32 status,
const char *error);
const char *error;
} status_src[] = {
{ NULL, "No error" },
{ NULL, NULL },
{ report_ccb_status, "CCB" },
{ report_jump_status, "Jump" },
{ report_deco_status, "DECO" },
{ NULL, NULL },
{ report_jr_status, "Job Ring" },
{ report_cond_code_status, "Condition Code" },
};
u32 ssrc = status >> JRSTA_SSRC_SHIFT;
const char *error = status_src[ssrc].error;
/*
* If there is no further error handling function, just
* print the error code, error string and exit. Otherwise
* call the handler function.
*/
if (!status_src[ssrc].report_ssed)
dev_err(jrdev, "%08x: %s: \n", status, status_src[ssrc].error);
else
status_src[ssrc].report_ssed(jrdev, status, error);
}
EXPORT_SYMBOL(caam_jr_strstatus);

11
drivers/crypto/caam/error.h Normal file
View file

@ -0,0 +1,11 @@
/*
* CAAM Error Reporting code header
*
* Copyright 2009-2011 Freescale Semiconductor, Inc.
*/
#ifndef CAAM_ERROR_H
#define CAAM_ERROR_H
#define CAAM_ERROR_STR_MAX 302
void caam_jr_strstatus(struct device *jrdev, u32 status);
#endif /* CAAM_ERROR_H */

113
drivers/crypto/caam/intern.h Normal file
View file

@ -0,0 +1,113 @@
/*
* CAAM/SEC 4.x driver backend
* Private/internal definitions between modules
*
* Copyright 2008-2011 Freescale Semiconductor, Inc.
*
*/
#ifndef INTERN_H
#define INTERN_H
/* Currently comes from Kconfig param as a ^2 (driver-required) */
#define JOBR_DEPTH (1 << CONFIG_CRYPTO_DEV_FSL_CAAM_RINGSIZE)
/* Kconfig params for interrupt coalescing if selected (else zero) */
#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_INTC
#define JOBR_INTC JRCFG_ICEN
#define JOBR_INTC_TIME_THLD CONFIG_CRYPTO_DEV_FSL_CAAM_INTC_TIME_THLD
#define JOBR_INTC_COUNT_THLD CONFIG_CRYPTO_DEV_FSL_CAAM_INTC_COUNT_THLD
#else
#define JOBR_INTC 0
#define JOBR_INTC_TIME_THLD 0
#define JOBR_INTC_COUNT_THLD 0
#endif
/*
* Storage for tracking each in-process entry moving across a ring
* Each entry on an output ring needs one of these
*/
struct caam_jrentry_info {
void (*callbk)(struct device *dev, u32 *desc, u32 status, void *arg);
void *cbkarg; /* Argument per ring entry */
u32 *desc_addr_virt; /* Stored virt addr for postprocessing */
dma_addr_t desc_addr_dma; /* Stored bus addr for done matching */
u32 desc_size; /* Stored size for postprocessing, header derived */
};
/* Private sub-storage for a single JobR */
struct caam_drv_private_jr {
struct list_head list_node; /* Job Ring device list */
struct device *dev;
int ridx;
struct caam_job_ring __iomem *rregs; /* JobR's register space */
struct tasklet_struct irqtask;
int irq; /* One per queue */
/* Number of scatterlist crypt transforms active on the JobR */
atomic_t tfm_count ____cacheline_aligned;
/* Job ring info */
int ringsize; /* Size of rings (assume input = output) */
struct caam_jrentry_info *entinfo; /* Alloc'ed 1 per ring entry */
spinlock_t inplock ____cacheline_aligned; /* Input ring index lock */
int inp_ring_write_index; /* Input index "tail" */
int head; /* entinfo (s/w ring) head index */
dma_addr_t *inpring; /* Base of input ring, alloc DMA-safe */
spinlock_t outlock ____cacheline_aligned; /* Output ring index lock */
int out_ring_read_index; /* Output index "tail" */
int tail; /* entinfo (s/w ring) tail index */
struct jr_outentry *outring; /* Base of output ring, DMA-safe */
};
/*
* Driver-private storage for a single CAAM block instance
*/
struct caam_drv_private {
struct device *dev;
struct platform_device **jrpdev; /* Alloc'ed array per sub-device */
struct platform_device *pdev;
/* Physical-presence section */
struct caam_ctrl __iomem *ctrl; /* controller region */
struct caam_deco __iomem *deco; /* DECO/CCB views */
struct caam_assurance __iomem *assure;
struct caam_queue_if __iomem *qi; /* QI control region */
struct caam_job_ring __iomem *jr[4]; /* JobR's register space */
/*
* Detected geometry block. Filled in from device tree if powerpc,
* or from register-based version detection code
*/
u8 total_jobrs; /* Total Job Rings in device */
u8 qi_present; /* Nonzero if QI present in device */
int secvio_irq; /* Security violation interrupt number */
int virt_en; /* Virtualization enabled in CAAM */
#define RNG4_MAX_HANDLES 2
/* RNG4 block */
u32 rng4_sh_init; /* This bitmap shows which of the State
Handles of the RNG4 block are initialized
by this driver */
/*
* debugfs entries for developer view into driver/device
* variables at runtime.
*/
#ifdef CONFIG_DEBUG_FS
struct dentry *dfs_root;
struct dentry *ctl; /* controller dir */
struct dentry *ctl_rq_dequeued, *ctl_ob_enc_req, *ctl_ib_dec_req;
struct dentry *ctl_ob_enc_bytes, *ctl_ob_prot_bytes;
struct dentry *ctl_ib_dec_bytes, *ctl_ib_valid_bytes;
struct dentry *ctl_faultaddr, *ctl_faultdetail, *ctl_faultstatus;
struct debugfs_blob_wrapper ctl_kek_wrap, ctl_tkek_wrap, ctl_tdsk_wrap;
struct dentry *ctl_kek, *ctl_tkek, *ctl_tdsk;
#endif
};
void caam_jr_algapi_init(struct device *dev);
void caam_jr_algapi_remove(struct device *dev);
#endif /* INTERN_H */

541
drivers/crypto/caam/jr.c Normal file
View file

@ -0,0 +1,541 @@
/*
* CAAM/SEC 4.x transport/backend driver
* JobR backend functionality
*
* Copyright 2008-2012 Freescale Semiconductor, Inc.
*/
#include <linux/of_irq.h>
#include <linux/of_address.h>
#include "compat.h"
#include "regs.h"
#include "jr.h"
#include "desc.h"
#include "intern.h"
struct jr_driver_data {
/* List of Physical JobR's with the Driver */
struct list_head jr_list;
spinlock_t jr_alloc_lock; /* jr_list lock */
} ____cacheline_aligned;
static struct jr_driver_data driver_data;
static int caam_reset_hw_jr(struct device *dev)
{
struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
unsigned int timeout = 100000;
/*
* mask interrupts since we are going to poll
* for reset completion status
*/
setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
/* initiate flush (required prior to reset) */
wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
while (((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) ==
JRINT_ERR_HALT_INPROGRESS) && --timeout)
cpu_relax();
if ((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) !=
JRINT_ERR_HALT_COMPLETE || timeout == 0) {
dev_err(dev, "failed to flush job ring %d\n", jrp->ridx);
return -EIO;
}
/* initiate reset */
timeout = 100000;
wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
while ((rd_reg32(&jrp->rregs->jrcommand) & JRCR_RESET) && --timeout)
cpu_relax();
if (timeout == 0) {
dev_err(dev, "failed to reset job ring %d\n", jrp->ridx);
return -EIO;
}
/* unmask interrupts */
clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
return 0;
}
/*
* Shutdown JobR independent of platform property code
*/
int caam_jr_shutdown(struct device *dev)
{
struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
dma_addr_t inpbusaddr, outbusaddr;
int ret;
ret = caam_reset_hw_jr(dev);
tasklet_kill(&jrp->irqtask);
/* Release interrupt */
free_irq(jrp->irq, dev);
/* Free rings */
inpbusaddr = rd_reg64(&jrp->rregs->inpring_base);
outbusaddr = rd_reg64(&jrp->rregs->outring_base);
dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
jrp->inpring, inpbusaddr);
dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH,
jrp->outring, outbusaddr);
kfree(jrp->entinfo);
return ret;
}
static int caam_jr_remove(struct platform_device *pdev)
{
int ret;
struct device *jrdev;
struct caam_drv_private_jr *jrpriv;
jrdev = &pdev->dev;
jrpriv = dev_get_drvdata(jrdev);
/*
* Return EBUSY if job ring already allocated.
*/
if (atomic_read(&jrpriv->tfm_count)) {
dev_err(jrdev, "Device is busy\n");
return -EBUSY;
}
/* Remove the node from Physical JobR list maintained by driver */
spin_lock(&driver_data.jr_alloc_lock);
list_del(&jrpriv->list_node);
spin_unlock(&driver_data.jr_alloc_lock);
/* Release ring */
ret = caam_jr_shutdown(jrdev);
if (ret)
dev_err(jrdev, "Failed to shut down job ring\n");
irq_dispose_mapping(jrpriv->irq);
return ret;
}
/* Main per-ring interrupt handler */
static irqreturn_t caam_jr_interrupt(int irq, void *st_dev)
{
struct device *dev = st_dev;
struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
u32 irqstate;
/*
* Check the output ring for ready responses, kick
* tasklet if jobs done.
*/
irqstate = rd_reg32(&jrp->rregs->jrintstatus);
if (!irqstate)
return IRQ_NONE;
/*
* If JobR error, we got more development work to do
* Flag a bug now, but we really need to shut down and
* restart the queue (and fix code).
*/
if (irqstate & JRINT_JR_ERROR) {
dev_err(dev, "job ring error: irqstate: %08x\n", irqstate);
BUG();
}
/* mask valid interrupts */
setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
/* Have valid interrupt at this point, just ACK and trigger */
wr_reg32(&jrp->rregs->jrintstatus, irqstate);
preempt_disable();
tasklet_schedule(&jrp->irqtask);
preempt_enable();
return IRQ_HANDLED;
}
/* Deferred service handler, run as interrupt-fired tasklet */
static void caam_jr_dequeue(unsigned long devarg)
{
int hw_idx, sw_idx, i, head, tail;
struct device *dev = (struct device *)devarg;
struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg);
u32 *userdesc, userstatus;
void *userarg;
while (rd_reg32(&jrp->rregs->outring_used)) {
head = ACCESS_ONCE(jrp->head);
spin_lock(&jrp->outlock);
sw_idx = tail = jrp->tail;
hw_idx = jrp->out_ring_read_index;
for (i = 0; CIRC_CNT(head, tail + i, JOBR_DEPTH) >= 1; i++) {
sw_idx = (tail + i) & (JOBR_DEPTH - 1);
smp_read_barrier_depends();
if (jrp->outring[hw_idx].desc ==
jrp->entinfo[sw_idx].desc_addr_dma)
break; /* found */
}
/* we should never fail to find a matching descriptor */
BUG_ON(CIRC_CNT(head, tail + i, JOBR_DEPTH) <= 0);
/* Unmap just-run descriptor so we can post-process */
dma_unmap_single(dev, jrp->outring[hw_idx].desc,
jrp->entinfo[sw_idx].desc_size,
DMA_TO_DEVICE);
/* mark completed, avoid matching on a recycled desc addr */
jrp->entinfo[sw_idx].desc_addr_dma = 0;
/* Stash callback params for use outside of lock */
usercall = jrp->entinfo[sw_idx].callbk;
userarg = jrp->entinfo[sw_idx].cbkarg;
userdesc = jrp->entinfo[sw_idx].desc_addr_virt;
userstatus = jrp->outring[hw_idx].jrstatus;
/* set done */
wr_reg32(&jrp->rregs->outring_rmvd, 1);
jrp->out_ring_read_index = (jrp->out_ring_read_index + 1) &
(JOBR_DEPTH - 1);
/*
* if this job completed out-of-order, do not increment
* the tail. Otherwise, increment tail by 1 plus the
* number of subsequent jobs already completed out-of-order
*/
if (sw_idx == tail) {
do {
tail = (tail + 1) & (JOBR_DEPTH - 1);
smp_read_barrier_depends();
} while (CIRC_CNT(head, tail, JOBR_DEPTH) >= 1 &&
jrp->entinfo[tail].desc_addr_dma == 0);
jrp->tail = tail;
}
spin_unlock(&jrp->outlock);
/* Finally, execute user's callback */
usercall(dev, userdesc, userstatus, userarg);
}
/* reenable / unmask IRQs */
clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
}
/**
* caam_jr_alloc() - Alloc a job ring for someone to use as needed.
*
* returns : pointer to the newly allocated physical
* JobR dev can be written to if successful.
**/
struct device *caam_jr_alloc(void)
{
struct caam_drv_private_jr *jrpriv, *min_jrpriv = NULL;
struct device *dev = NULL;
int min_tfm_cnt = INT_MAX;
int tfm_cnt;
spin_lock(&driver_data.jr_alloc_lock);
if (list_empty(&driver_data.jr_list)) {
spin_unlock(&driver_data.jr_alloc_lock);
return ERR_PTR(-ENODEV);
}
list_for_each_entry(jrpriv, &driver_data.jr_list, list_node) {
tfm_cnt = atomic_read(&jrpriv->tfm_count);
if (tfm_cnt < min_tfm_cnt) {
min_tfm_cnt = tfm_cnt;
min_jrpriv = jrpriv;
}
if (!min_tfm_cnt)
break;
}
if (min_jrpriv) {
atomic_inc(&min_jrpriv->tfm_count);
dev = min_jrpriv->dev;
}
spin_unlock(&driver_data.jr_alloc_lock);
return dev;
}
EXPORT_SYMBOL(caam_jr_alloc);
/**
* caam_jr_free() - Free the Job Ring
* @rdev - points to the dev that identifies the Job ring to
* be released.
**/
void caam_jr_free(struct device *rdev)
{
struct caam_drv_private_jr *jrpriv = dev_get_drvdata(rdev);
atomic_dec(&jrpriv->tfm_count);
}
EXPORT_SYMBOL(caam_jr_free);
/**
* caam_jr_enqueue() - Enqueue a job descriptor head. Returns 0 if OK,
* -EBUSY if the queue is full, -EIO if it cannot map the caller's
* descriptor.
* @dev: device of the job ring to be used. This device should have
* been assigned prior by caam_jr_register().
* @desc: points to a job descriptor that execute our request. All
* descriptors (and all referenced data) must be in a DMAable
* region, and all data references must be physical addresses
* accessible to CAAM (i.e. within a PAMU window granted
* to it).
* @cbk: pointer to a callback function to be invoked upon completion
* of this request. This has the form:
* callback(struct device *dev, u32 *desc, u32 stat, void *arg)
* where:
* @dev: contains the job ring device that processed this
* response.
* @desc: descriptor that initiated the request, same as
* "desc" being argued to caam_jr_enqueue().
* @status: untranslated status received from CAAM. See the
* reference manual for a detailed description of
* error meaning, or see the JRSTA definitions in the
* register header file
* @areq: optional pointer to an argument passed with the
* original request
* @areq: optional pointer to a user argument for use at callback
* time.
**/
int caam_jr_enqueue(struct device *dev, u32 *desc,
void (*cbk)(struct device *dev, u32 *desc,
u32 status, void *areq),
void *areq)
{
struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
struct caam_jrentry_info *head_entry;
int head, tail, desc_size;
dma_addr_t desc_dma;
desc_size = (*desc & HDR_JD_LENGTH_MASK) * sizeof(u32);
desc_dma = dma_map_single(dev, desc, desc_size, DMA_TO_DEVICE);
if (dma_mapping_error(dev, desc_dma)) {
dev_err(dev, "caam_jr_enqueue(): can't map jobdesc\n");
return -EIO;
}
spin_lock_bh(&jrp->inplock);
head = jrp->head;
tail = ACCESS_ONCE(jrp->tail);
if (!rd_reg32(&jrp->rregs->inpring_avail) ||
CIRC_SPACE(head, tail, JOBR_DEPTH) <= 0) {
spin_unlock_bh(&jrp->inplock);
dma_unmap_single(dev, desc_dma, desc_size, DMA_TO_DEVICE);
return -EBUSY;
}
head_entry = &jrp->entinfo[head];
head_entry->desc_addr_virt = desc;
head_entry->desc_size = desc_size;
head_entry->callbk = (void *)cbk;
head_entry->cbkarg = areq;
head_entry->desc_addr_dma = desc_dma;
jrp->inpring[jrp->inp_ring_write_index] = desc_dma;
smp_wmb();
jrp->inp_ring_write_index = (jrp->inp_ring_write_index + 1) &
(JOBR_DEPTH - 1);
jrp->head = (head + 1) & (JOBR_DEPTH - 1);
wr_reg32(&jrp->rregs->inpring_jobadd, 1);
spin_unlock_bh(&jrp->inplock);
return 0;
}
EXPORT_SYMBOL(caam_jr_enqueue);
/*
* Init JobR independent of platform property detection
*/
static int caam_jr_init(struct device *dev)
{
struct caam_drv_private_jr *jrp;
dma_addr_t inpbusaddr, outbusaddr;
int i, error;
jrp = dev_get_drvdata(dev);
tasklet_init(&jrp->irqtask, caam_jr_dequeue, (unsigned long)dev);
/* Connect job ring interrupt handler. */
error = request_irq(jrp->irq, caam_jr_interrupt, IRQF_SHARED,
dev_name(dev), dev);
if (error) {
dev_err(dev, "can't connect JobR %d interrupt (%d)\n",
jrp->ridx, jrp->irq);
irq_dispose_mapping(jrp->irq);
jrp->irq = 0;
return -EINVAL;
}
error = caam_reset_hw_jr(dev);
if (error)
return error;
jrp->inpring = dma_alloc_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
&inpbusaddr, GFP_KERNEL);
jrp->outring = dma_alloc_coherent(dev, sizeof(struct jr_outentry) *
JOBR_DEPTH, &outbusaddr, GFP_KERNEL);
jrp->entinfo = kzalloc(sizeof(struct caam_jrentry_info) * JOBR_DEPTH,
GFP_KERNEL);
if ((jrp->inpring == NULL) || (jrp->outring == NULL) ||
(jrp->entinfo == NULL)) {
dev_err(dev, "can't allocate job rings for %d\n",
jrp->ridx);
return -ENOMEM;
}
for (i = 0; i < JOBR_DEPTH; i++)
jrp->entinfo[i].desc_addr_dma = !0;
/* Setup rings */
jrp->inp_ring_write_index = 0;
jrp->out_ring_read_index = 0;
jrp->head = 0;
jrp->tail = 0;
wr_reg64(&jrp->rregs->inpring_base, inpbusaddr);
wr_reg64(&jrp->rregs->outring_base, outbusaddr);
wr_reg32(&jrp->rregs->inpring_size, JOBR_DEPTH);
wr_reg32(&jrp->rregs->outring_size, JOBR_DEPTH);
jrp->ringsize = JOBR_DEPTH;
spin_lock_init(&jrp->inplock);
spin_lock_init(&jrp->outlock);
/* Select interrupt coalescing parameters */
setbits32(&jrp->rregs->rconfig_lo, JOBR_INTC |
(JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) |
(JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT));
return 0;
}
/*
* Probe routine for each detected JobR subsystem.
*/
static int caam_jr_probe(struct platform_device *pdev)
{
struct device *jrdev;
struct device_node *nprop;
struct caam_job_ring __iomem *ctrl;
struct caam_drv_private_jr *jrpriv;
static int total_jobrs;
int error;
jrdev = &pdev->dev;
jrpriv = devm_kmalloc(jrdev, sizeof(struct caam_drv_private_jr),
GFP_KERNEL);
if (!jrpriv)
return -ENOMEM;
dev_set_drvdata(jrdev, jrpriv);
/* save ring identity relative to detection */
jrpriv->ridx = total_jobrs++;
nprop = pdev->dev.of_node;
/* Get configuration properties from device tree */
/* First, get register page */
ctrl = of_iomap(nprop, 0);
if (!ctrl) {
dev_err(jrdev, "of_iomap() failed\n");
return -ENOMEM;
}
jrpriv->rregs = (struct caam_job_ring __force *)ctrl;
if (sizeof(dma_addr_t) == sizeof(u64))
if (of_device_is_compatible(nprop, "fsl,sec-v5.0-job-ring"))
dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(40));
else
dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(36));
else
dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(32));
/* Identify the interrupt */
jrpriv->irq = irq_of_parse_and_map(nprop, 0);
/* Now do the platform independent part */
error = caam_jr_init(jrdev); /* now turn on hardware */
if (error)
return error;
jrpriv->dev = jrdev;
spin_lock(&driver_data.jr_alloc_lock);
list_add_tail(&jrpriv->list_node, &driver_data.jr_list);
spin_unlock(&driver_data.jr_alloc_lock);
atomic_set(&jrpriv->tfm_count, 0);
return 0;
}
static struct of_device_id caam_jr_match[] = {
{
.compatible = "fsl,sec-v4.0-job-ring",
},
{
.compatible = "fsl,sec4.0-job-ring",
},
{},
};
MODULE_DEVICE_TABLE(of, caam_jr_match);
static struct platform_driver caam_jr_driver = {
.driver = {
.name = "caam_jr",
.owner = THIS_MODULE,
.of_match_table = caam_jr_match,
},
.probe = caam_jr_probe,
.remove = caam_jr_remove,
};
static int __init jr_driver_init(void)
{
spin_lock_init(&driver_data.jr_alloc_lock);
INIT_LIST_HEAD(&driver_data.jr_list);
return platform_driver_register(&caam_jr_driver);
}
static void __exit jr_driver_exit(void)
{
platform_driver_unregister(&caam_jr_driver);
}
module_init(jr_driver_init);
module_exit(jr_driver_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("FSL CAAM JR request backend");
MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");

18
drivers/crypto/caam/jr.h Normal file
View file

@ -0,0 +1,18 @@
/*
* CAAM public-level include definitions for the JobR backend
*
* Copyright 2008-2011 Freescale Semiconductor, Inc.
*/
#ifndef JR_H
#define JR_H
/* Prototypes for backend-level services exposed to APIs */
struct device *caam_jr_alloc(void);
void caam_jr_free(struct device *rdev);
int caam_jr_enqueue(struct device *dev, u32 *desc,
void (*cbk)(struct device *dev, u32 *desc, u32 status,
void *areq),
void *areq);
#endif /* JR_H */

123
drivers/crypto/caam/key_gen.c Normal file
View file

@ -0,0 +1,123 @@
/*
* CAAM/SEC 4.x functions for handling key-generation jobs
*
* Copyright 2008-2011 Freescale Semiconductor, Inc.
*
*/
#include "compat.h"
#include "jr.h"
#include "error.h"
#include "desc_constr.h"
#include "key_gen.h"
void split_key_done(struct device *dev, u32 *desc, u32 err,
void *context)
{
struct split_key_result *res = context;
#ifdef DEBUG
dev_err(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
#endif
if (err)
caam_jr_strstatus(dev, err);
res->err = err;
complete(&res->completion);
}
EXPORT_SYMBOL(split_key_done);
/*
get a split ipad/opad key
Split key generation-----------------------------------------------
[00] 0xb0810008 jobdesc: stidx=1 share=never len=8
[01] 0x04000014 key: class2->keyreg len=20
@0xffe01000
[03] 0x84410014 operation: cls2-op sha1 hmac init dec
[04] 0x24940000 fifold: class2 msgdata-last2 len=0 imm
[05] 0xa4000001 jump: class2 local all ->1 [06]
[06] 0x64260028 fifostr: class2 mdsplit-jdk len=40
@0xffe04000
*/
int gen_split_key(struct device *jrdev, u8 *key_out, int split_key_len,
int split_key_pad_len, const u8 *key_in, u32 keylen,
u32 alg_op)
{
u32 *desc;
struct split_key_result result;
dma_addr_t dma_addr_in, dma_addr_out;
int ret = -ENOMEM;
desc = kmalloc(CAAM_CMD_SZ * 6 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
if (!desc) {
dev_err(jrdev, "unable to allocate key input memory\n");
return ret;
}
dma_addr_in = dma_map_single(jrdev, (void *)key_in, keylen,
DMA_TO_DEVICE);
if (dma_mapping_error(jrdev, dma_addr_in)) {
dev_err(jrdev, "unable to map key input memory\n");
goto out_free;
}
dma_addr_out = dma_map_single(jrdev, key_out, split_key_pad_len,
DMA_FROM_DEVICE);
if (dma_mapping_error(jrdev, dma_addr_out)) {
dev_err(jrdev, "unable to map key output memory\n");
goto out_unmap_in;
}
init_job_desc(desc, 0);
append_key(desc, dma_addr_in, keylen, CLASS_2 | KEY_DEST_CLASS_REG);
/* Sets MDHA up into an HMAC-INIT */
append_operation(desc, alg_op | OP_ALG_DECRYPT | OP_ALG_AS_INIT);
/*
* do a FIFO_LOAD of zero, this will trigger the internal key expansion
* into both pads inside MDHA
*/
append_fifo_load_as_imm(desc, NULL, 0, LDST_CLASS_2_CCB |
FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST2);
/*
* FIFO_STORE with the explicit split-key content store
* (0x26 output type)
*/
append_fifo_store(desc, dma_addr_out, split_key_len,
LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1);
print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
result.err = 0;
init_completion(&result.completion);
ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result);
if (!ret) {
/* in progress */
wait_for_completion_interruptible(&result.completion);
ret = result.err;
#ifdef DEBUG
print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key_out,
split_key_pad_len, 1);
#endif
}
dma_unmap_single(jrdev, dma_addr_out, split_key_pad_len,
DMA_FROM_DEVICE);
out_unmap_in:
dma_unmap_single(jrdev, dma_addr_in, keylen, DMA_TO_DEVICE);
out_free:
kfree(desc);
return ret;
}
EXPORT_SYMBOL(gen_split_key);

17
drivers/crypto/caam/key_gen.h Normal file
View file

@ -0,0 +1,17 @@
/*
* CAAM/SEC 4.x definitions for handling key-generation jobs
*
* Copyright 2008-2011 Freescale Semiconductor, Inc.
*
*/
struct split_key_result {
struct completion completion;
int err;
};
void split_key_done(struct device *dev, u32 *desc, u32 err, void *context);
int gen_split_key(struct device *jrdev, u8 *key_out, int split_key_len,
int split_key_pad_len, const u8 *key_in, u32 keylen,
u32 alg_op);

402
drivers/crypto/caam/pdb.h Normal file
View file

@ -0,0 +1,402 @@
/*
* CAAM Protocol Data Block (PDB) definition header file
*
* Copyright 2008-2012 Freescale Semiconductor, Inc.
*
*/
#ifndef CAAM_PDB_H
#define CAAM_PDB_H
/*
* PDB- IPSec ESP Header Modification Options
*/
#define PDBHMO_ESP_DECAP_SHIFT 12
#define PDBHMO_ESP_ENCAP_SHIFT 4
/*
* Encap and Decap - Decrement TTL (Hop Limit) - Based on the value of the
* Options Byte IP version (IPvsn) field:
* if IPv4, decrement the inner IP header TTL field (byte 8);
* if IPv6 decrement the inner IP header Hop Limit field (byte 7).
*/
#define PDBHMO_ESP_DECAP_DEC_TTL (0x02 << PDBHMO_ESP_DECAP_SHIFT)
#define PDBHMO_ESP_ENCAP_DEC_TTL (0x02 << PDBHMO_ESP_ENCAP_SHIFT)
/*
* Decap - DiffServ Copy - Copy the IPv4 TOS or IPv6 Traffic Class byte
* from the outer IP header to the inner IP header.
*/
#define PDBHMO_ESP_DIFFSERV (0x01 << PDBHMO_ESP_DECAP_SHIFT)
/*
* Encap- Copy DF bit -if an IPv4 tunnel mode outer IP header is coming from
* the PDB, copy the DF bit from the inner IP header to the outer IP header.
*/
#define PDBHMO_ESP_DFBIT (0x04 << PDBHMO_ESP_ENCAP_SHIFT)
/*
* PDB - IPSec ESP Encap/Decap Options
*/
#define PDBOPTS_ESP_ARSNONE 0x00 /* no antireplay window */
#define PDBOPTS_ESP_ARS32 0x40 /* 32-entry antireplay window */
#define PDBOPTS_ESP_ARS64 0xc0 /* 64-entry antireplay window */
#define PDBOPTS_ESP_IVSRC 0x20 /* IV comes from internal random gen */
#define PDBOPTS_ESP_ESN 0x10 /* extended sequence included */
#define PDBOPTS_ESP_OUTFMT 0x08 /* output only decapsulation (decap) */
#define PDBOPTS_ESP_IPHDRSRC 0x08 /* IP header comes from PDB (encap) */
#define PDBOPTS_ESP_INCIPHDR 0x04 /* Prepend IP header to output frame */
#define PDBOPTS_ESP_IPVSN 0x02 /* process IPv6 header */
#define PDBOPTS_ESP_AOFL 0x04 /* adjust out frame len (decap, SEC>=5.3)*/
#define PDBOPTS_ESP_TUNNEL 0x01 /* tunnel mode next-header byte */
#define PDBOPTS_ESP_IPV6 0x02 /* ip header version is V6 */
#define PDBOPTS_ESP_DIFFSERV 0x40 /* copy TOS/TC from inner iphdr */
#define PDBOPTS_ESP_UPDATE_CSUM 0x80 /* encap-update ip header checksum */
#define PDBOPTS_ESP_VERIFY_CSUM 0x20 /* decap-validate ip header checksum */
/*
* General IPSec encap/decap PDB definitions
*/
struct ipsec_encap_cbc {
u32 iv[4];
};
struct ipsec_encap_ctr {
u32 ctr_nonce;
u32 ctr_initial;
u32 iv[2];
};
struct ipsec_encap_ccm {
u32 salt; /* lower 24 bits */
u8 b0_flags;
u8 ctr_flags;
u16 ctr_initial;
u32 iv[2];
};
struct ipsec_encap_gcm {
u32 salt; /* lower 24 bits */
u32 rsvd1;
u32 iv[2];
};
struct ipsec_encap_pdb {
u8 hmo_rsvd;
u8 ip_nh;
u8 ip_nh_offset;
u8 options;
u32 seq_num_ext_hi;
u32 seq_num;
union {
struct ipsec_encap_cbc cbc;
struct ipsec_encap_ctr ctr;
struct ipsec_encap_ccm ccm;
struct ipsec_encap_gcm gcm;
};
u32 spi;
u16 rsvd1;
u16 ip_hdr_len;
u32 ip_hdr[0]; /* optional IP Header content */
};
struct ipsec_decap_cbc {
u32 rsvd[2];
};
struct ipsec_decap_ctr {
u32 salt;
u32 ctr_initial;
};
struct ipsec_decap_ccm {
u32 salt;
u8 iv_flags;
u8 ctr_flags;
u16 ctr_initial;
};
struct ipsec_decap_gcm {
u32 salt;
u32 resvd;
};
struct ipsec_decap_pdb {
u16 hmo_ip_hdr_len;
u8 ip_nh_offset;
u8 options;
union {
struct ipsec_decap_cbc cbc;
struct ipsec_decap_ctr ctr;
struct ipsec_decap_ccm ccm;
struct ipsec_decap_gcm gcm;
};
u32 seq_num_ext_hi;
u32 seq_num;
u32 anti_replay[2];
u32 end_index[0];
};
/*
* IPSec ESP Datapath Protocol Override Register (DPOVRD)
*/
struct ipsec_deco_dpovrd {
#define IPSEC_ENCAP_DECO_DPOVRD_USE 0x80
u8 ovrd_ecn;
u8 ip_hdr_len;
u8 nh_offset;
u8 next_header; /* reserved if decap */
};
/*
* IEEE 802.11i WiFi Protocol Data Block
*/
#define WIFI_PDBOPTS_FCS 0x01
#define WIFI_PDBOPTS_AR 0x40
struct wifi_encap_pdb {
u16 mac_hdr_len;
u8 rsvd;
u8 options;
u8 iv_flags;
u8 pri;
u16 pn1;
u32 pn2;
u16 frm_ctrl_mask;
u16 seq_ctrl_mask;
u8 rsvd1[2];
u8 cnst;
u8 key_id;
u8 ctr_flags;
u8 rsvd2;
u16 ctr_init;
};
struct wifi_decap_pdb {
u16 mac_hdr_len;
u8 rsvd;
u8 options;
u8 iv_flags;
u8 pri;
u16 pn1;
u32 pn2;
u16 frm_ctrl_mask;
u16 seq_ctrl_mask;
u8 rsvd1[4];
u8 ctr_flags;
u8 rsvd2;
u16 ctr_init;
};
/*
* IEEE 802.16 WiMAX Protocol Data Block
*/
#define WIMAX_PDBOPTS_FCS 0x01
#define WIMAX_PDBOPTS_AR 0x40 /* decap only */
struct wimax_encap_pdb {
u8 rsvd[3];
u8 options;
u32 nonce;
u8 b0_flags;
u8 ctr_flags;
u16 ctr_init;
/* begin DECO writeback region */
u32 pn;
/* end DECO writeback region */
};
struct wimax_decap_pdb {
u8 rsvd[3];
u8 options;
u32 nonce;
u8 iv_flags;
u8 ctr_flags;
u16 ctr_init;
/* begin DECO writeback region */
u32 pn;
u8 rsvd1[2];
u16 antireplay_len;
u64 antireplay_scorecard;
/* end DECO writeback region */
};
/*
* IEEE 801.AE MacSEC Protocol Data Block
*/
#define MACSEC_PDBOPTS_FCS 0x01
#define MACSEC_PDBOPTS_AR 0x40 /* used in decap only */
struct macsec_encap_pdb {
u16 aad_len;
u8 rsvd;
u8 options;
u64 sci;
u16 ethertype;
u8 tci_an;
u8 rsvd1;
/* begin DECO writeback region */
u32 pn;
/* end DECO writeback region */
};
struct macsec_decap_pdb {
u16 aad_len;
u8 rsvd;
u8 options;
u64 sci;
u8 rsvd1[3];
/* begin DECO writeback region */
u8 antireplay_len;
u32 pn;
u64 antireplay_scorecard;
/* end DECO writeback region */
};
/*
* SSL/TLS/DTLS Protocol Data Blocks
*/
#define TLS_PDBOPTS_ARS32 0x40
#define TLS_PDBOPTS_ARS64 0xc0
#define TLS_PDBOPTS_OUTFMT 0x08
#define TLS_PDBOPTS_IV_WRTBK 0x02 /* 1.1/1.2/DTLS only */
#define TLS_PDBOPTS_EXP_RND_IV 0x01 /* 1.1/1.2/DTLS only */
struct tls_block_encap_pdb {
u8 type;
u8 version[2];
u8 options;
u64 seq_num;
u32 iv[4];
};
struct tls_stream_encap_pdb {
u8 type;
u8 version[2];
u8 options;
u64 seq_num;
u8 i;
u8 j;
u8 rsvd1[2];
};
struct dtls_block_encap_pdb {
u8 type;
u8 version[2];
u8 options;
u16 epoch;
u16 seq_num[3];
u32 iv[4];
};
struct tls_block_decap_pdb {
u8 rsvd[3];
u8 options;
u64 seq_num;
u32 iv[4];
};
struct tls_stream_decap_pdb {
u8 rsvd[3];
u8 options;
u64 seq_num;
u8 i;
u8 j;
u8 rsvd1[2];
};
struct dtls_block_decap_pdb {
u8 rsvd[3];
u8 options;
u16 epoch;
u16 seq_num[3];
u32 iv[4];
u64 antireplay_scorecard;
};
/*
* SRTP Protocol Data Blocks
*/
#define SRTP_PDBOPTS_MKI 0x08
#define SRTP_PDBOPTS_AR 0x40
struct srtp_encap_pdb {
u8 x_len;
u8 mki_len;
u8 n_tag;
u8 options;
u32 cnst0;
u8 rsvd[2];
u16 cnst1;
u16 salt[7];
u16 cnst2;
u32 rsvd1;
u32 roc;
u32 opt_mki;
};
struct srtp_decap_pdb {
u8 x_len;
u8 mki_len;
u8 n_tag;
u8 options;
u32 cnst0;
u8 rsvd[2];
u16 cnst1;
u16 salt[7];
u16 cnst2;
u16 rsvd1;
u16 seq_num;
u32 roc;
u64 antireplay_scorecard;
};
/*
* DSA/ECDSA Protocol Data Blocks
* Two of these exist: DSA-SIGN, and DSA-VERIFY. They are similar
* except for the treatment of "w" for verify, "s" for sign,
* and the placement of "a,b".
*/
#define DSA_PDB_SGF_SHIFT 24
#define DSA_PDB_SGF_MASK (0xff << DSA_PDB_SGF_SHIFT)
#define DSA_PDB_SGF_Q (0x80 << DSA_PDB_SGF_SHIFT)
#define DSA_PDB_SGF_R (0x40 << DSA_PDB_SGF_SHIFT)
#define DSA_PDB_SGF_G (0x20 << DSA_PDB_SGF_SHIFT)
#define DSA_PDB_SGF_W (0x10 << DSA_PDB_SGF_SHIFT)
#define DSA_PDB_SGF_S (0x10 << DSA_PDB_SGF_SHIFT)
#define DSA_PDB_SGF_F (0x08 << DSA_PDB_SGF_SHIFT)
#define DSA_PDB_SGF_C (0x04 << DSA_PDB_SGF_SHIFT)
#define DSA_PDB_SGF_D (0x02 << DSA_PDB_SGF_SHIFT)
#define DSA_PDB_SGF_AB_SIGN (0x02 << DSA_PDB_SGF_SHIFT)
#define DSA_PDB_SGF_AB_VERIFY (0x01 << DSA_PDB_SGF_SHIFT)
#define DSA_PDB_L_SHIFT 7
#define DSA_PDB_L_MASK (0x3ff << DSA_PDB_L_SHIFT)
#define DSA_PDB_N_MASK 0x7f
struct dsa_sign_pdb {
u32 sgf_ln; /* Use DSA_PDB_ defintions per above */
u8 *q;
u8 *r;
u8 *g; /* or Gx,y */
u8 *s;
u8 *f;
u8 *c;
u8 *d;
u8 *ab; /* ECC only */
u8 *u;
};
struct dsa_verify_pdb {
u32 sgf_ln;
u8 *q;
u8 *r;
u8 *g; /* or Gx,y */
u8 *w; /* or Wx,y */
u8 *f;
u8 *c;
u8 *d;
u8 *tmp; /* temporary data block */
u8 *ab; /* only used if ECC processing */
};
#endif

780
drivers/crypto/caam/regs.h Normal file
View file

@ -0,0 +1,780 @@
/*
* CAAM hardware register-level view
*
* Copyright 2008-2011 Freescale Semiconductor, Inc.
*/
#ifndef REGS_H
#define REGS_H
#include <linux/types.h>
#include <linux/io.h>
/*
* Architecture-specific register access methods
*
* CAAM's bus-addressable registers are 64 bits internally.
* They have been wired to be safely accessible on 32-bit
* architectures, however. Registers were organized such
* that (a) they can be contained in 32 bits, (b) if not, then they
* can be treated as two 32-bit entities, or finally (c) if they
* must be treated as a single 64-bit value, then this can safely
* be done with two 32-bit cycles.
*
* For 32-bit operations on 64-bit values, CAAM follows the same
* 64-bit register access conventions as it's predecessors, in that
* writes are "triggered" by a write to the register at the numerically
* higher address, thus, a full 64-bit write cycle requires a write
* to the lower address, followed by a write to the higher address,
* which will latch/execute the write cycle.
*
* For example, let's assume a SW reset of CAAM through the master
* configuration register.
* - SWRST is in bit 31 of MCFG.
* - MCFG begins at base+0x0000.
* - Bits 63-32 are a 32-bit word at base+0x0000 (numerically-lower)
* - Bits 31-0 are a 32-bit word at base+0x0004 (numerically-higher)
*
* (and on Power, the convention is 0-31, 32-63, I know...)
*
* Assuming a 64-bit write to this MCFG to perform a software reset
* would then require a write of 0 to base+0x0000, followed by a
* write of 0x80000000 to base+0x0004, which would "execute" the
* reset.
*
* Of course, since MCFG 63-32 is all zero, we could cheat and simply
* write 0x8000000 to base+0x0004, and the reset would work fine.
* However, since CAAM does contain some write-and-read-intended
* 64-bit registers, this code defines 64-bit access methods for
* the sake of internal consistency and simplicity, and so that a
* clean transition to 64-bit is possible when it becomes necessary.
*
* There are limitations to this that the developer must recognize.
* 32-bit architectures cannot enforce an atomic-64 operation,
* Therefore:
*
* - On writes, since the HW is assumed to latch the cycle on the
* write of the higher-numeric-address word, then ordered
* writes work OK.
*
* - For reads, where a register contains a relevant value of more
* that 32 bits, the hardware employs logic to latch the other
* "half" of the data until read, ensuring an accurate value.
* This is of particular relevance when dealing with CAAM's
* performance counters.
*
*/
#ifdef __BIG_ENDIAN
#define wr_reg32(reg, data) out_be32(reg, data)
#define rd_reg32(reg) in_be32(reg)
#ifdef CONFIG_64BIT
#define wr_reg64(reg, data) out_be64(reg, data)
#define rd_reg64(reg) in_be64(reg)
#endif
#else
#ifdef __LITTLE_ENDIAN
#define wr_reg32(reg, data) __raw_writel(data, reg)
#define rd_reg32(reg) __raw_readl(reg)
#ifdef CONFIG_64BIT
#define wr_reg64(reg, data) __raw_writeq(data, reg)
#define rd_reg64(reg) __raw_readq(reg)
#endif
#endif
#endif
#ifndef CONFIG_64BIT
#ifdef __BIG_ENDIAN
static inline void wr_reg64(u64 __iomem *reg, u64 data)
{
wr_reg32((u32 __iomem *)reg, (data & 0xffffffff00000000ull) >> 32);
wr_reg32((u32 __iomem *)reg + 1, data & 0x00000000ffffffffull);
}
static inline u64 rd_reg64(u64 __iomem *reg)
{
return (((u64)rd_reg32((u32 __iomem *)reg)) << 32) |
((u64)rd_reg32((u32 __iomem *)reg + 1));
}
#else
#ifdef __LITTLE_ENDIAN
static inline void wr_reg64(u64 __iomem *reg, u64 data)
{
wr_reg32((u32 __iomem *)reg + 1, (data & 0xffffffff00000000ull) >> 32);
wr_reg32((u32 __iomem *)reg, data & 0x00000000ffffffffull);
}
static inline u64 rd_reg64(u64 __iomem *reg)
{
return (((u64)rd_reg32((u32 __iomem *)reg + 1)) << 32) |
((u64)rd_reg32((u32 __iomem *)reg));
}
#endif
#endif
#endif
/*
* jr_outentry
* Represents each entry in a JobR output ring
*/
struct jr_outentry {
dma_addr_t desc;/* Pointer to completed descriptor */
u32 jrstatus; /* Status for completed descriptor */
} __packed;
/*
* caam_perfmon - Performance Monitor/Secure Memory Status/
* CAAM Global Status/Component Version IDs
*
* Spans f00-fff wherever instantiated
*/
/* Number of DECOs */
#define CHA_NUM_MS_DECONUM_SHIFT 24
#define CHA_NUM_MS_DECONUM_MASK (0xfull << CHA_NUM_MS_DECONUM_SHIFT)
/* CHA Version IDs */
#define CHA_ID_LS_AES_SHIFT 0
#define CHA_ID_LS_AES_MASK (0xfull << CHA_ID_LS_AES_SHIFT)
#define CHA_ID_LS_DES_SHIFT 4
#define CHA_ID_LS_DES_MASK (0xfull << CHA_ID_LS_DES_SHIFT)
#define CHA_ID_LS_ARC4_SHIFT 8
#define CHA_ID_LS_ARC4_MASK (0xfull << CHA_ID_LS_ARC4_SHIFT)
#define CHA_ID_LS_MD_SHIFT 12
#define CHA_ID_LS_MD_MASK (0xfull << CHA_ID_LS_MD_SHIFT)
#define CHA_ID_LS_RNG_SHIFT 16
#define CHA_ID_LS_RNG_MASK (0xfull << CHA_ID_LS_RNG_SHIFT)
#define CHA_ID_LS_SNW8_SHIFT 20
#define CHA_ID_LS_SNW8_MASK (0xfull << CHA_ID_LS_SNW8_SHIFT)
#define CHA_ID_LS_KAS_SHIFT 24
#define CHA_ID_LS_KAS_MASK (0xfull << CHA_ID_LS_KAS_SHIFT)
#define CHA_ID_LS_PK_SHIFT 28
#define CHA_ID_LS_PK_MASK (0xfull << CHA_ID_LS_PK_SHIFT)
#define CHA_ID_MS_CRC_SHIFT 0
#define CHA_ID_MS_CRC_MASK (0xfull << CHA_ID_MS_CRC_SHIFT)
#define CHA_ID_MS_SNW9_SHIFT 4
#define CHA_ID_MS_SNW9_MASK (0xfull << CHA_ID_MS_SNW9_SHIFT)
#define CHA_ID_MS_DECO_SHIFT 24
#define CHA_ID_MS_DECO_MASK (0xfull << CHA_ID_MS_DECO_SHIFT)
#define CHA_ID_MS_JR_SHIFT 28
#define CHA_ID_MS_JR_MASK (0xfull << CHA_ID_MS_JR_SHIFT)
struct sec_vid {
u16 ip_id;
u8 maj_rev;
u8 min_rev;
};
struct caam_perfmon {
/* Performance Monitor Registers f00-f9f */
u64 req_dequeued; /* PC_REQ_DEQ - Dequeued Requests */
u64 ob_enc_req; /* PC_OB_ENC_REQ - Outbound Encrypt Requests */
u64 ib_dec_req; /* PC_IB_DEC_REQ - Inbound Decrypt Requests */
u64 ob_enc_bytes; /* PC_OB_ENCRYPT - Outbound Bytes Encrypted */
u64 ob_prot_bytes; /* PC_OB_PROTECT - Outbound Bytes Protected */
u64 ib_dec_bytes; /* PC_IB_DECRYPT - Inbound Bytes Decrypted */
u64 ib_valid_bytes; /* PC_IB_VALIDATED Inbound Bytes Validated */
u64 rsvd[13];
/* CAAM Hardware Instantiation Parameters fa0-fbf */
u32 cha_rev_ms; /* CRNR - CHA Rev No. Most significant half*/
u32 cha_rev_ls; /* CRNR - CHA Rev No. Least significant half*/
#define CTPR_MS_QI_SHIFT 25
#define CTPR_MS_QI_MASK (0x1ull << CTPR_MS_QI_SHIFT)
#define CTPR_MS_VIRT_EN_INCL 0x00000001
#define CTPR_MS_VIRT_EN_POR 0x00000002
#define CTPR_MS_PG_SZ_MASK 0x10
#define CTPR_MS_PG_SZ_SHIFT 4
u32 comp_parms_ms; /* CTPR - Compile Parameters Register */
u32 comp_parms_ls; /* CTPR - Compile Parameters Register */
u64 rsvd1[2];
/* CAAM Global Status fc0-fdf */
u64 faultaddr; /* FAR - Fault Address */
u32 faultliodn; /* FALR - Fault Address LIODN */
u32 faultdetail; /* FADR - Fault Addr Detail */
u32 rsvd2;
u32 status; /* CSTA - CAAM Status */
u64 rsvd3;
/* Component Instantiation Parameters fe0-fff */
u32 rtic_id; /* RVID - RTIC Version ID */
u32 ccb_id; /* CCBVID - CCB Version ID */
u32 cha_id_ms; /* CHAVID - CHA Version ID Most Significant*/
u32 cha_id_ls; /* CHAVID - CHA Version ID Least Significant*/
u32 cha_num_ms; /* CHANUM - CHA Number Most Significant */
u32 cha_num_ls; /* CHANUM - CHA Number Least Significant*/
u32 caam_id_ms; /* CAAMVID - CAAM Version ID MS */
u32 caam_id_ls; /* CAAMVID - CAAM Version ID LS */
};
/* LIODN programming for DMA configuration */
#define MSTRID_LOCK_LIODN 0x80000000
#define MSTRID_LOCK_MAKETRUSTED 0x00010000 /* only for JR masterid */
#define MSTRID_LIODN_MASK 0x0fff
struct masterid {
u32 liodn_ms; /* lock and make-trusted control bits */
u32 liodn_ls; /* LIODN for non-sequence and seq access */
};
/* Partition ID for DMA configuration */
struct partid {
u32 rsvd1;
u32 pidr; /* partition ID, DECO */
};
/* RNGB test mode (replicated twice in some configurations) */
/* Padded out to 0x100 */
struct rngtst {
u32 mode; /* RTSTMODEx - Test mode */
u32 rsvd1[3];
u32 reset; /* RTSTRESETx - Test reset control */
u32 rsvd2[3];
u32 status; /* RTSTSSTATUSx - Test status */
u32 rsvd3;
u32 errstat; /* RTSTERRSTATx - Test error status */
u32 rsvd4;
u32 errctl; /* RTSTERRCTLx - Test error control */
u32 rsvd5;
u32 entropy; /* RTSTENTROPYx - Test entropy */
u32 rsvd6[15];
u32 verifctl; /* RTSTVERIFCTLx - Test verification control */
u32 rsvd7;
u32 verifstat; /* RTSTVERIFSTATx - Test verification status */
u32 rsvd8;
u32 verifdata; /* RTSTVERIFDx - Test verification data */
u32 rsvd9;
u32 xkey; /* RTSTXKEYx - Test XKEY */
u32 rsvd10;
u32 oscctctl; /* RTSTOSCCTCTLx - Test osc. counter control */
u32 rsvd11;
u32 oscct; /* RTSTOSCCTx - Test oscillator counter */
u32 rsvd12;
u32 oscctstat; /* RTSTODCCTSTATx - Test osc counter status */
u32 rsvd13[2];
u32 ofifo[4]; /* RTSTOFIFOx - Test output FIFO */
u32 rsvd14[15];
};
/* RNG4 TRNG test registers */
struct rng4tst {
#define RTMCTL_PRGM 0x00010000 /* 1 -> program mode, 0 -> run mode */
#define RTMCTL_SAMP_MODE_VON_NEUMANN_ES_SC 0 /* use von Neumann data in
both entropy shifter and
statistical checker */
#define RTMCTL_SAMP_MODE_RAW_ES_SC 1 /* use raw data in both
entropy shifter and
statistical checker */
#define RTMCTL_SAMP_MODE_VON_NEUMANN_ES_RAW_SC 2 /* use von Neumann data in
entropy shifter, raw data
in statistical checker */
#define RTMCTL_SAMP_MODE_INVALID 3 /* invalid combination */
u32 rtmctl; /* misc. control register */
u32 rtscmisc; /* statistical check misc. register */
u32 rtpkrrng; /* poker range register */
union {
u32 rtpkrmax; /* PRGM=1: poker max. limit register */
u32 rtpkrsq; /* PRGM=0: poker square calc. result register */
};
#define RTSDCTL_ENT_DLY_SHIFT 16
#define RTSDCTL_ENT_DLY_MASK (0xffff << RTSDCTL_ENT_DLY_SHIFT)
#define RTSDCTL_ENT_DLY_MIN 3200
#define RTSDCTL_ENT_DLY_MAX 12800
u32 rtsdctl; /* seed control register */
union {
u32 rtsblim; /* PRGM=1: sparse bit limit register */
u32 rttotsam; /* PRGM=0: total samples register */
};
u32 rtfrqmin; /* frequency count min. limit register */
#define RTFRQMAX_DISABLE (1 << 20)
union {
u32 rtfrqmax; /* PRGM=1: freq. count max. limit register */
u32 rtfrqcnt; /* PRGM=0: freq. count register */
};
u32 rsvd1[40];
#define RDSTA_SKVT 0x80000000
#define RDSTA_SKVN 0x40000000
#define RDSTA_IF0 0x00000001
#define RDSTA_IF1 0x00000002
#define RDSTA_IFMASK (RDSTA_IF1 | RDSTA_IF0)
u32 rdsta;
u32 rsvd2[15];
};
/*
* caam_ctrl - basic core configuration
* starts base + 0x0000 padded out to 0x1000
*/
#define KEK_KEY_SIZE 8
#define TKEK_KEY_SIZE 8
#define TDSK_KEY_SIZE 8
#define DECO_RESET 1 /* Use with DECO reset/availability regs */
#define DECO_RESET_0 (DECO_RESET << 0)
#define DECO_RESET_1 (DECO_RESET << 1)
#define DECO_RESET_2 (DECO_RESET << 2)
#define DECO_RESET_3 (DECO_RESET << 3)
#define DECO_RESET_4 (DECO_RESET << 4)
struct caam_ctrl {
/* Basic Configuration Section 000-01f */
/* Read/Writable */
u32 rsvd1;
u32 mcr; /* MCFG Master Config Register */
u32 rsvd2;
u32 scfgr; /* SCFGR, Security Config Register */
/* Bus Access Configuration Section 010-11f */
/* Read/Writable */
struct masterid jr_mid[4]; /* JRxLIODNR - JobR LIODN setup */
u32 rsvd3[11];
u32 jrstart; /* JRSTART - Job Ring Start Register */
struct masterid rtic_mid[4]; /* RTICxLIODNR - RTIC LIODN setup */
u32 rsvd4[5];
u32 deco_rsr; /* DECORSR - Deco Request Source */
u32 rsvd11;
u32 deco_rq; /* DECORR - DECO Request */
struct partid deco_mid[5]; /* DECOxLIODNR - 1 per DECO */
u32 rsvd5[22];
/* DECO Availability/Reset Section 120-3ff */
u32 deco_avail; /* DAR - DECO availability */
u32 deco_reset; /* DRR - DECO reset */
u32 rsvd6[182];
/* Key Encryption/Decryption Configuration 400-5ff */
/* Read/Writable only while in Non-secure mode */
u32 kek[KEK_KEY_SIZE]; /* JDKEKR - Key Encryption Key */
u32 tkek[TKEK_KEY_SIZE]; /* TDKEKR - Trusted Desc KEK */
u32 tdsk[TDSK_KEY_SIZE]; /* TDSKR - Trusted Desc Signing Key */
u32 rsvd7[32];
u64 sknonce; /* SKNR - Secure Key Nonce */
u32 rsvd8[70];
/* RNG Test/Verification/Debug Access 600-7ff */
/* (Useful in Test/Debug modes only...) */
union {
struct rngtst rtst[2];
struct rng4tst r4tst[2];
};
u32 rsvd9[448];
/* Performance Monitor f00-fff */
struct caam_perfmon perfmon;
};
/*
* Controller master config register defs
*/
#define MCFGR_SWRESET 0x80000000 /* software reset */
#define MCFGR_WDENABLE 0x40000000 /* DECO watchdog enable */
#define MCFGR_WDFAIL 0x20000000 /* DECO watchdog force-fail */
#define MCFGR_DMA_RESET 0x10000000
#define MCFGR_LONG_PTR 0x00010000 /* Use >32-bit desc addressing */
#define SCFGR_RDBENABLE 0x00000400
#define SCFGR_VIRT_EN 0x00008000
#define DECORR_RQD0ENABLE 0x00000001 /* Enable DECO0 for direct access */
#define DECORSR_JR0 0x00000001 /* JR to supply TZ, SDID, ICID */
#define DECORSR_VALID 0x80000000
#define DECORR_DEN0 0x00010000 /* DECO0 available for access*/
/* AXI read cache control */
#define MCFGR_ARCACHE_SHIFT 12
#define MCFGR_ARCACHE_MASK (0xf << MCFGR_ARCACHE_SHIFT)
/* AXI write cache control */
#define MCFGR_AWCACHE_SHIFT 8
#define MCFGR_AWCACHE_MASK (0xf << MCFGR_AWCACHE_SHIFT)
/* AXI pipeline depth */
#define MCFGR_AXIPIPE_SHIFT 4
#define MCFGR_AXIPIPE_MASK (0xf << MCFGR_AXIPIPE_SHIFT)
#define MCFGR_AXIPRI 0x00000008 /* Assert AXI priority sideband */
#define MCFGR_BURST_64 0x00000001 /* Max burst size */
/* JRSTART register offsets */
#define JRSTART_JR0_START 0x00000001 /* Start Job ring 0 */
#define JRSTART_JR1_START 0x00000002 /* Start Job ring 1 */
#define JRSTART_JR2_START 0x00000004 /* Start Job ring 2 */
#define JRSTART_JR3_START 0x00000008 /* Start Job ring 3 */
/*
* caam_job_ring - direct job ring setup
* 1-4 possible per instantiation, base + 1000/2000/3000/4000
* Padded out to 0x1000
*/
struct caam_job_ring {
/* Input ring */
u64 inpring_base; /* IRBAx - Input desc ring baseaddr */
u32 rsvd1;
u32 inpring_size; /* IRSx - Input ring size */
u32 rsvd2;
u32 inpring_avail; /* IRSAx - Input ring room remaining */
u32 rsvd3;
u32 inpring_jobadd; /* IRJAx - Input ring jobs added */
/* Output Ring */
u64 outring_base; /* ORBAx - Output status ring base addr */
u32 rsvd4;
u32 outring_size; /* ORSx - Output ring size */
u32 rsvd5;
u32 outring_rmvd; /* ORJRx - Output ring jobs removed */
u32 rsvd6;
u32 outring_used; /* ORSFx - Output ring slots full */
/* Status/Configuration */
u32 rsvd7;
u32 jroutstatus; /* JRSTAx - JobR output status */
u32 rsvd8;
u32 jrintstatus; /* JRINTx - JobR interrupt status */
u32 rconfig_hi; /* JRxCFG - Ring configuration */
u32 rconfig_lo;
/* Indices. CAAM maintains as "heads" of each queue */
u32 rsvd9;
u32 inp_rdidx; /* IRRIx - Input ring read index */
u32 rsvd10;
u32 out_wtidx; /* ORWIx - Output ring write index */
/* Command/control */
u32 rsvd11;
u32 jrcommand; /* JRCRx - JobR command */
u32 rsvd12[932];
/* Performance Monitor f00-fff */
struct caam_perfmon perfmon;
};
#define JR_RINGSIZE_MASK 0x03ff
/*
* jrstatus - Job Ring Output Status
* All values in lo word
* Also note, same values written out as status through QI
* in the command/status field of a frame descriptor
*/
#define JRSTA_SSRC_SHIFT 28
#define JRSTA_SSRC_MASK 0xf0000000
#define JRSTA_SSRC_NONE 0x00000000
#define JRSTA_SSRC_CCB_ERROR 0x20000000
#define JRSTA_SSRC_JUMP_HALT_USER 0x30000000
#define JRSTA_SSRC_DECO 0x40000000
#define JRSTA_SSRC_JRERROR 0x60000000
#define JRSTA_SSRC_JUMP_HALT_CC 0x70000000
#define JRSTA_DECOERR_JUMP 0x08000000
#define JRSTA_DECOERR_INDEX_SHIFT 8
#define JRSTA_DECOERR_INDEX_MASK 0xff00
#define JRSTA_DECOERR_ERROR_MASK 0x00ff
#define JRSTA_DECOERR_NONE 0x00
#define JRSTA_DECOERR_LINKLEN 0x01
#define JRSTA_DECOERR_LINKPTR 0x02
#define JRSTA_DECOERR_JRCTRL 0x03
#define JRSTA_DECOERR_DESCCMD 0x04
#define JRSTA_DECOERR_ORDER 0x05
#define JRSTA_DECOERR_KEYCMD 0x06
#define JRSTA_DECOERR_LOADCMD 0x07
#define JRSTA_DECOERR_STORECMD 0x08
#define JRSTA_DECOERR_OPCMD 0x09
#define JRSTA_DECOERR_FIFOLDCMD 0x0a
#define JRSTA_DECOERR_FIFOSTCMD 0x0b
#define JRSTA_DECOERR_MOVECMD 0x0c
#define JRSTA_DECOERR_JUMPCMD 0x0d
#define JRSTA_DECOERR_MATHCMD 0x0e
#define JRSTA_DECOERR_SHASHCMD 0x0f
#define JRSTA_DECOERR_SEQCMD 0x10
#define JRSTA_DECOERR_DECOINTERNAL 0x11
#define JRSTA_DECOERR_SHDESCHDR 0x12
#define JRSTA_DECOERR_HDRLEN 0x13
#define JRSTA_DECOERR_BURSTER 0x14
#define JRSTA_DECOERR_DESCSIGNATURE 0x15
#define JRSTA_DECOERR_DMA 0x16
#define JRSTA_DECOERR_BURSTFIFO 0x17
#define JRSTA_DECOERR_JRRESET 0x1a
#define JRSTA_DECOERR_JOBFAIL 0x1b
#define JRSTA_DECOERR_DNRERR 0x80
#define JRSTA_DECOERR_UNDEFPCL 0x81
#define JRSTA_DECOERR_PDBERR 0x82
#define JRSTA_DECOERR_ANRPLY_LATE 0x83
#define JRSTA_DECOERR_ANRPLY_REPLAY 0x84
#define JRSTA_DECOERR_SEQOVF 0x85
#define JRSTA_DECOERR_INVSIGN 0x86
#define JRSTA_DECOERR_DSASIGN 0x87
#define JRSTA_CCBERR_JUMP 0x08000000
#define JRSTA_CCBERR_INDEX_MASK 0xff00
#define JRSTA_CCBERR_INDEX_SHIFT 8
#define JRSTA_CCBERR_CHAID_MASK 0x00f0
#define JRSTA_CCBERR_CHAID_SHIFT 4
#define JRSTA_CCBERR_ERRID_MASK 0x000f
#define JRSTA_CCBERR_CHAID_AES (0x01 << JRSTA_CCBERR_CHAID_SHIFT)
#define JRSTA_CCBERR_CHAID_DES (0x02 << JRSTA_CCBERR_CHAID_SHIFT)
#define JRSTA_CCBERR_CHAID_ARC4 (0x03 << JRSTA_CCBERR_CHAID_SHIFT)
#define JRSTA_CCBERR_CHAID_MD (0x04 << JRSTA_CCBERR_CHAID_SHIFT)
#define JRSTA_CCBERR_CHAID_RNG (0x05 << JRSTA_CCBERR_CHAID_SHIFT)
#define JRSTA_CCBERR_CHAID_SNOW (0x06 << JRSTA_CCBERR_CHAID_SHIFT)
#define JRSTA_CCBERR_CHAID_KASUMI (0x07 << JRSTA_CCBERR_CHAID_SHIFT)
#define JRSTA_CCBERR_CHAID_PK (0x08 << JRSTA_CCBERR_CHAID_SHIFT)
#define JRSTA_CCBERR_CHAID_CRC (0x09 << JRSTA_CCBERR_CHAID_SHIFT)
#define JRSTA_CCBERR_ERRID_NONE 0x00
#define JRSTA_CCBERR_ERRID_MODE 0x01
#define JRSTA_CCBERR_ERRID_DATASIZ 0x02
#define JRSTA_CCBERR_ERRID_KEYSIZ 0x03
#define JRSTA_CCBERR_ERRID_PKAMEMSZ 0x04
#define JRSTA_CCBERR_ERRID_PKBMEMSZ 0x05
#define JRSTA_CCBERR_ERRID_SEQUENCE 0x06
#define JRSTA_CCBERR_ERRID_PKDIVZRO 0x07
#define JRSTA_CCBERR_ERRID_PKMODEVN 0x08
#define JRSTA_CCBERR_ERRID_KEYPARIT 0x09
#define JRSTA_CCBERR_ERRID_ICVCHK 0x0a
#define JRSTA_CCBERR_ERRID_HARDWARE 0x0b
#define JRSTA_CCBERR_ERRID_CCMAAD 0x0c
#define JRSTA_CCBERR_ERRID_INVCHA 0x0f
#define JRINT_ERR_INDEX_MASK 0x3fff0000
#define JRINT_ERR_INDEX_SHIFT 16
#define JRINT_ERR_TYPE_MASK 0xf00
#define JRINT_ERR_TYPE_SHIFT 8
#define JRINT_ERR_HALT_MASK 0xc
#define JRINT_ERR_HALT_SHIFT 2
#define JRINT_ERR_HALT_INPROGRESS 0x4
#define JRINT_ERR_HALT_COMPLETE 0x8
#define JRINT_JR_ERROR 0x02
#define JRINT_JR_INT 0x01
#define JRINT_ERR_TYPE_WRITE 1
#define JRINT_ERR_TYPE_BAD_INPADDR 3
#define JRINT_ERR_TYPE_BAD_OUTADDR 4
#define JRINT_ERR_TYPE_INV_INPWRT 5
#define JRINT_ERR_TYPE_INV_OUTWRT 6
#define JRINT_ERR_TYPE_RESET 7
#define JRINT_ERR_TYPE_REMOVE_OFL 8
#define JRINT_ERR_TYPE_ADD_OFL 9
#define JRCFG_SOE 0x04
#define JRCFG_ICEN 0x02
#define JRCFG_IMSK 0x01
#define JRCFG_ICDCT_SHIFT 8
#define JRCFG_ICTT_SHIFT 16
#define JRCR_RESET 0x01
/*
* caam_assurance - Assurance Controller View
* base + 0x6000 padded out to 0x1000
*/
struct rtic_element {
u64 address;
u32 rsvd;
u32 length;
};
struct rtic_block {
struct rtic_element element[2];
};
struct rtic_memhash {
u32 memhash_be[32];
u32 memhash_le[32];
};
struct caam_assurance {
/* Status/Command/Watchdog */
u32 rsvd1;
u32 status; /* RSTA - Status */
u32 rsvd2;
u32 cmd; /* RCMD - Command */
u32 rsvd3;
u32 ctrl; /* RCTL - Control */
u32 rsvd4;
u32 throttle; /* RTHR - Throttle */
u32 rsvd5[2];
u64 watchdog; /* RWDOG - Watchdog Timer */
u32 rsvd6;
u32 rend; /* REND - Endian corrections */
u32 rsvd7[50];
/* Block access/configuration @ 100/110/120/130 */
struct rtic_block memblk[4]; /* Memory Blocks A-D */
u32 rsvd8[32];
/* Block hashes @ 200/300/400/500 */
struct rtic_memhash hash[4]; /* Block hash values A-D */
u32 rsvd_3[640];
};
/*
* caam_queue_if - QI configuration and control
* starts base + 0x7000, padded out to 0x1000 long
*/
struct caam_queue_if {
u32 qi_control_hi; /* QICTL - QI Control */
u32 qi_control_lo;
u32 rsvd1;
u32 qi_status; /* QISTA - QI Status */
u32 qi_deq_cfg_hi; /* QIDQC - QI Dequeue Configuration */
u32 qi_deq_cfg_lo;
u32 qi_enq_cfg_hi; /* QISEQC - QI Enqueue Command */
u32 qi_enq_cfg_lo;
u32 rsvd2[1016];
};
/* QI control bits - low word */
#define QICTL_DQEN 0x01 /* Enable frame pop */
#define QICTL_STOP 0x02 /* Stop dequeue/enqueue */
#define QICTL_SOE 0x04 /* Stop on error */
/* QI control bits - high word */
#define QICTL_MBSI 0x01
#define QICTL_MHWSI 0x02
#define QICTL_MWSI 0x04
#define QICTL_MDWSI 0x08
#define QICTL_CBSI 0x10 /* CtrlDataByteSwapInput */
#define QICTL_CHWSI 0x20 /* CtrlDataHalfSwapInput */
#define QICTL_CWSI 0x40 /* CtrlDataWordSwapInput */
#define QICTL_CDWSI 0x80 /* CtrlDataDWordSwapInput */
#define QICTL_MBSO 0x0100
#define QICTL_MHWSO 0x0200
#define QICTL_MWSO 0x0400
#define QICTL_MDWSO 0x0800
#define QICTL_CBSO 0x1000 /* CtrlDataByteSwapOutput */
#define QICTL_CHWSO 0x2000 /* CtrlDataHalfSwapOutput */
#define QICTL_CWSO 0x4000 /* CtrlDataWordSwapOutput */
#define QICTL_CDWSO 0x8000 /* CtrlDataDWordSwapOutput */
#define QICTL_DMBS 0x010000
#define QICTL_EPO 0x020000
/* QI status bits */
#define QISTA_PHRDERR 0x01 /* PreHeader Read Error */
#define QISTA_CFRDERR 0x02 /* Compound Frame Read Error */
#define QISTA_OFWRERR 0x04 /* Output Frame Read Error */
#define QISTA_BPDERR 0x08 /* Buffer Pool Depleted */
#define QISTA_BTSERR 0x10 /* Buffer Undersize */
#define QISTA_CFWRERR 0x20 /* Compound Frame Write Err */
#define QISTA_STOPD 0x80000000 /* QI Stopped (see QICTL) */
/* deco_sg_table - DECO view of scatter/gather table */
struct deco_sg_table {
u64 addr; /* Segment Address */
u32 elen; /* E, F bits + 30-bit length */
u32 bpid_offset; /* Buffer Pool ID + 16-bit length */
};
/*
* caam_deco - descriptor controller - CHA cluster block
*
* Only accessible when direct DECO access is turned on
* (done in DECORR, via MID programmed in DECOxMID
*
* 5 typical, base + 0x8000/9000/a000/b000
* Padded out to 0x1000 long
*/
struct caam_deco {
u32 rsvd1;
u32 cls1_mode; /* CxC1MR - Class 1 Mode */
u32 rsvd2;
u32 cls1_keysize; /* CxC1KSR - Class 1 Key Size */
u32 cls1_datasize_hi; /* CxC1DSR - Class 1 Data Size */
u32 cls1_datasize_lo;
u32 rsvd3;
u32 cls1_icvsize; /* CxC1ICVSR - Class 1 ICV size */
u32 rsvd4[5];
u32 cha_ctrl; /* CCTLR - CHA control */
u32 rsvd5;
u32 irq_crtl; /* CxCIRQ - CCB interrupt done/error/clear */
u32 rsvd6;
u32 clr_written; /* CxCWR - Clear-Written */
u32 ccb_status_hi; /* CxCSTA - CCB Status/Error */
u32 ccb_status_lo;
u32 rsvd7[3];
u32 aad_size; /* CxAADSZR - Current AAD Size */
u32 rsvd8;
u32 cls1_iv_size; /* CxC1IVSZR - Current Class 1 IV Size */
u32 rsvd9[7];
u32 pkha_a_size; /* PKASZRx - Size of PKHA A */
u32 rsvd10;
u32 pkha_b_size; /* PKBSZRx - Size of PKHA B */
u32 rsvd11;
u32 pkha_n_size; /* PKNSZRx - Size of PKHA N */
u32 rsvd12;
u32 pkha_e_size; /* PKESZRx - Size of PKHA E */
u32 rsvd13[24];
u32 cls1_ctx[16]; /* CxC1CTXR - Class 1 Context @100 */
u32 rsvd14[48];
u32 cls1_key[8]; /* CxC1KEYR - Class 1 Key @200 */
u32 rsvd15[121];
u32 cls2_mode; /* CxC2MR - Class 2 Mode */
u32 rsvd16;
u32 cls2_keysize; /* CxX2KSR - Class 2 Key Size */
u32 cls2_datasize_hi; /* CxC2DSR - Class 2 Data Size */
u32 cls2_datasize_lo;
u32 rsvd17;
u32 cls2_icvsize; /* CxC2ICVSZR - Class 2 ICV Size */
u32 rsvd18[56];
u32 cls2_ctx[18]; /* CxC2CTXR - Class 2 Context @500 */
u32 rsvd19[46];
u32 cls2_key[32]; /* CxC2KEYR - Class2 Key @600 */
u32 rsvd20[84];
u32 inp_infofifo_hi; /* CxIFIFO - Input Info FIFO @7d0 */
u32 inp_infofifo_lo;
u32 rsvd21[2];
u64 inp_datafifo; /* CxDFIFO - Input Data FIFO */
u32 rsvd22[2];
u64 out_datafifo; /* CxOFIFO - Output Data FIFO */
u32 rsvd23[2];
u32 jr_ctl_hi; /* CxJRR - JobR Control Register @800 */
u32 jr_ctl_lo;
u64 jr_descaddr; /* CxDADR - JobR Descriptor Address */
#define DECO_OP_STATUS_HI_ERR_MASK 0xF00000FF
u32 op_status_hi; /* DxOPSTA - DECO Operation Status */
u32 op_status_lo;
u32 rsvd24[2];
u32 liodn; /* DxLSR - DECO LIODN Status - non-seq */
u32 td_liodn; /* DxLSR - DECO LIODN Status - trustdesc */
u32 rsvd26[6];
u64 math[4]; /* DxMTH - Math register */
u32 rsvd27[8];
struct deco_sg_table gthr_tbl[4]; /* DxGTR - Gather Tables */
u32 rsvd28[16];
struct deco_sg_table sctr_tbl[4]; /* DxSTR - Scatter Tables */
u32 rsvd29[48];
u32 descbuf[64]; /* DxDESB - Descriptor buffer */
u32 rscvd30[193];
#define DESC_DBG_DECO_STAT_HOST_ERR 0x00D00000
#define DESC_DBG_DECO_STAT_VALID 0x80000000
#define DESC_DBG_DECO_STAT_MASK 0x00F00000
u32 desc_dbg; /* DxDDR - DECO Debug Register */
u32 rsvd31[126];
};
#define DECO_JQCR_WHL 0x20000000
#define DECO_JQCR_FOUR 0x10000000
#define JR_BLOCK_NUMBER 1
#define ASSURE_BLOCK_NUMBER 6
#define QI_BLOCK_NUMBER 7
#define DECO_BLOCK_NUMBER 8
#define PG_SIZE_4K 0x1000
#define PG_SIZE_64K 0x10000
#endif /* REGS_H */

118
drivers/crypto/caam/sg_sw_sec4.h Normal file
View file

@ -0,0 +1,118 @@
/*
* CAAM/SEC 4.x functions for using scatterlists in caam driver
*
* Copyright 2008-2011 Freescale Semiconductor, Inc.
*
*/
struct sec4_sg_entry;
/*
* convert single dma address to h/w link table format
*/
static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr,
dma_addr_t dma, u32 len, u32 offset)
{
sec4_sg_ptr->ptr = dma;
sec4_sg_ptr->len = len;
sec4_sg_ptr->reserved = 0;
sec4_sg_ptr->buf_pool_id = 0;
sec4_sg_ptr->offset = offset;
#ifdef DEBUG
print_hex_dump(KERN_ERR, "sec4_sg_ptr@: ",
DUMP_PREFIX_ADDRESS, 16, 4, sec4_sg_ptr,
sizeof(struct sec4_sg_entry), 1);
#endif
}
/*
* convert scatterlist to h/w link table format
* but does not have final bit; instead, returns last entry
*/
static inline struct sec4_sg_entry *
sg_to_sec4_sg(struct scatterlist *sg, int sg_count,
struct sec4_sg_entry *sec4_sg_ptr, u32 offset)
{
while (sg_count) {
dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg),
sg_dma_len(sg), offset);
sec4_sg_ptr++;
sg = scatterwalk_sg_next(sg);
sg_count--;
}
return sec4_sg_ptr - 1;
}
/*
* convert scatterlist to h/w link table format
* scatterlist must have been previously dma mapped
*/
static inline void sg_to_sec4_sg_last(struct scatterlist *sg, int sg_count,
struct sec4_sg_entry *sec4_sg_ptr,
u32 offset)
{
sec4_sg_ptr = sg_to_sec4_sg(sg, sg_count, sec4_sg_ptr, offset);
sec4_sg_ptr->len |= SEC4_SG_LEN_FIN;
}
/* count number of elements in scatterlist */
static inline int __sg_count(struct scatterlist *sg_list, int nbytes,
bool *chained)
{
struct scatterlist *sg = sg_list;
int sg_nents = 0;
while (nbytes > 0) {
sg_nents++;
nbytes -= sg->length;
if (!sg_is_last(sg) && (sg + 1)->length == 0)
*chained = true;
sg = scatterwalk_sg_next(sg);
}
return sg_nents;
}
/* derive number of elements in scatterlist, but return 0 for 1 */
static inline int sg_count(struct scatterlist *sg_list, int nbytes,
bool *chained)
{
int sg_nents = __sg_count(sg_list, nbytes, chained);
if (likely(sg_nents == 1))
return 0;
return sg_nents;
}
static int dma_map_sg_chained(struct device *dev, struct scatterlist *sg,
unsigned int nents, enum dma_data_direction dir,
bool chained)
{
if (unlikely(chained)) {
int i;
for (i = 0; i < nents; i++) {
dma_map_sg(dev, sg, 1, dir);
sg = scatterwalk_sg_next(sg);
}
} else {
dma_map_sg(dev, sg, nents, dir);
}
return nents;
}
static int dma_unmap_sg_chained(struct device *dev, struct scatterlist *sg,
unsigned int nents, enum dma_data_direction dir,
bool chained)
{
if (unlikely(chained)) {
int i;
for (i = 0; i < nents; i++) {
dma_unmap_sg(dev, sg, 1, dir);
sg = scatterwalk_sg_next(sg);
}
} else {
dma_unmap_sg(dev, sg, nents, dir);
}
return nents;
}

24
drivers/crypto/ccp/Kconfig Normal file
View file

@ -0,0 +1,24 @@
config CRYPTO_DEV_CCP_DD
tristate "Cryptographic Coprocessor device driver"
depends on CRYPTO_DEV_CCP
default m
select HW_RANDOM
help
Provides the interface to use the AMD Cryptographic Coprocessor
which can be used to accelerate or offload encryption operations
such as SHA, AES and more. If you choose 'M' here, this module
will be called ccp.
config CRYPTO_DEV_CCP_CRYPTO
tristate "Encryption and hashing acceleration support"
depends on CRYPTO_DEV_CCP_DD
default m
select CRYPTO_ALGAPI
select CRYPTO_HASH
select CRYPTO_BLKCIPHER
select CRYPTO_AUTHENC
help
Support for using the cryptographic API with the AMD Cryptographic
Coprocessor. This module supports acceleration and offload of SHA
and AES algorithms. If you choose 'M' here, this module will be
called ccp_crypto.

15
drivers/crypto/ccp/Makefile Normal file
View file

@ -0,0 +1,15 @@
obj-$(CONFIG_CRYPTO_DEV_CCP_DD) += ccp.o
ccp-objs := ccp-dev.o ccp-ops.o
ifdef CONFIG_X86
ccp-objs += ccp-pci.o
endif
ifdef CONFIG_ARM64
ccp-objs += ccp-platform.o
endif
obj-$(CONFIG_CRYPTO_DEV_CCP_CRYPTO) += ccp-crypto.o
ccp-crypto-objs := ccp-crypto-main.o \
ccp-crypto-aes.o \
ccp-crypto-aes-cmac.o \
ccp-crypto-aes-xts.o \
ccp-crypto-sha.o

365
drivers/crypto/ccp/ccp-crypto-aes-cmac.c Normal file
View file

@ -0,0 +1,365 @@
/*
* AMD Cryptographic Coprocessor (CCP) AES CMAC crypto API support
*
* Copyright (C) 2013 Advanced Micro Devices, Inc.
*
* Author: Tom Lendacky <thomas.lendacky@amd.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/delay.h>
#include <linux/scatterlist.h>
#include <linux/crypto.h>
#include <crypto/algapi.h>
#include <crypto/aes.h>
#include <crypto/hash.h>
#include <crypto/internal/hash.h>
#include <crypto/scatterwalk.h>
#include "ccp-crypto.h"
static int ccp_aes_cmac_complete(struct crypto_async_request *async_req,
int ret)
{
struct ahash_request *req = ahash_request_cast(async_req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx(req);
unsigned int digest_size = crypto_ahash_digestsize(tfm);
if (ret)
goto e_free;
if (rctx->hash_rem) {
/* Save remaining data to buffer */
unsigned int offset = rctx->nbytes - rctx->hash_rem;
scatterwalk_map_and_copy(rctx->buf, rctx->src,
offset, rctx->hash_rem, 0);
rctx->buf_count = rctx->hash_rem;
} else
rctx->buf_count = 0;
/* Update result area if supplied */
if (req->result)
memcpy(req->result, rctx->iv, digest_size);
e_free:
sg_free_table(&rctx->data_sg);
return ret;
}
static int ccp_do_cmac_update(struct ahash_request *req, unsigned int nbytes,
unsigned int final)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
struct ccp_ctx *ctx = crypto_ahash_ctx(tfm);
struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx(req);
struct scatterlist *sg, *cmac_key_sg = NULL;
unsigned int block_size =
crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
unsigned int need_pad, sg_count;
gfp_t gfp;
u64 len;
int ret;
if (!ctx->u.aes.key_len)
return -EINVAL;
if (nbytes)
rctx->null_msg = 0;
len = (u64)rctx->buf_count + (u64)nbytes;
if (!final && (len <= block_size)) {
scatterwalk_map_and_copy(rctx->buf + rctx->buf_count, req->src,
0, nbytes, 0);
rctx->buf_count += nbytes;
return 0;
}
rctx->src = req->src;
rctx->nbytes = nbytes;
rctx->final = final;
rctx->hash_rem = final ? 0 : len & (block_size - 1);
rctx->hash_cnt = len - rctx->hash_rem;
if (!final && !rctx->hash_rem) {
/* CCP can't do zero length final, so keep some data around */
rctx->hash_cnt -= block_size;
rctx->hash_rem = block_size;
}
if (final && (rctx->null_msg || (len & (block_size - 1))))
need_pad = 1;
else
need_pad = 0;
sg_init_one(&rctx->iv_sg, rctx->iv, sizeof(rctx->iv));
/* Build the data scatterlist table - allocate enough entries for all
* possible data pieces (buffer, input data, padding)
*/
sg_count = (nbytes) ? sg_nents(req->src) + 2 : 2;
gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
GFP_KERNEL : GFP_ATOMIC;
ret = sg_alloc_table(&rctx->data_sg, sg_count, gfp);
if (ret)
return ret;
sg = NULL;
if (rctx->buf_count) {
sg_init_one(&rctx->buf_sg, rctx->buf, rctx->buf_count);
sg = ccp_crypto_sg_table_add(&rctx->data_sg, &rctx->buf_sg);
}
if (nbytes)
sg = ccp_crypto_sg_table_add(&rctx->data_sg, req->src);
if (need_pad) {
int pad_length = block_size - (len & (block_size - 1));
rctx->hash_cnt += pad_length;
memset(rctx->pad, 0, sizeof(rctx->pad));
rctx->pad[0] = 0x80;
sg_init_one(&rctx->pad_sg, rctx->pad, pad_length);
sg = ccp_crypto_sg_table_add(&rctx->data_sg, &rctx->pad_sg);
}
if (sg) {
sg_mark_end(sg);
sg = rctx->data_sg.sgl;
}
/* Initialize the K1/K2 scatterlist */
if (final)
cmac_key_sg = (need_pad) ? &ctx->u.aes.k2_sg
: &ctx->u.aes.k1_sg;
memset(&rctx->cmd, 0, sizeof(rctx->cmd));
INIT_LIST_HEAD(&rctx->cmd.entry);
rctx->cmd.engine = CCP_ENGINE_AES;
rctx->cmd.u.aes.type = ctx->u.aes.type;
rctx->cmd.u.aes.mode = ctx->u.aes.mode;
rctx->cmd.u.aes.action = CCP_AES_ACTION_ENCRYPT;
rctx->cmd.u.aes.key = &ctx->u.aes.key_sg;
rctx->cmd.u.aes.key_len = ctx->u.aes.key_len;
rctx->cmd.u.aes.iv = &rctx->iv_sg;
rctx->cmd.u.aes.iv_len = AES_BLOCK_SIZE;
rctx->cmd.u.aes.src = sg;
rctx->cmd.u.aes.src_len = rctx->hash_cnt;
rctx->cmd.u.aes.dst = NULL;
rctx->cmd.u.aes.cmac_key = cmac_key_sg;
rctx->cmd.u.aes.cmac_key_len = ctx->u.aes.kn_len;
rctx->cmd.u.aes.cmac_final = final;
ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd);
return ret;
}
static int ccp_aes_cmac_init(struct ahash_request *req)
{
struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx(req);
memset(rctx, 0, sizeof(*rctx));
rctx->null_msg = 1;
return 0;
}
static int ccp_aes_cmac_update(struct ahash_request *req)
{
return ccp_do_cmac_update(req, req->nbytes, 0);
}
static int ccp_aes_cmac_final(struct ahash_request *req)
{
return ccp_do_cmac_update(req, 0, 1);
}
static int ccp_aes_cmac_finup(struct ahash_request *req)
{
return ccp_do_cmac_update(req, req->nbytes, 1);
}
static int ccp_aes_cmac_digest(struct ahash_request *req)
{
int ret;
ret = ccp_aes_cmac_init(req);
if (ret)
return ret;
return ccp_aes_cmac_finup(req);
}
static int ccp_aes_cmac_setkey(struct crypto_ahash *tfm, const u8 *key,
unsigned int key_len)
{
struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
struct ccp_crypto_ahash_alg *alg =
ccp_crypto_ahash_alg(crypto_ahash_tfm(tfm));
u64 k0_hi, k0_lo, k1_hi, k1_lo, k2_hi, k2_lo;
u64 rb_hi = 0x00, rb_lo = 0x87;
__be64 *gk;
int ret;
switch (key_len) {
case AES_KEYSIZE_128:
ctx->u.aes.type = CCP_AES_TYPE_128;
break;
case AES_KEYSIZE_192:
ctx->u.aes.type = CCP_AES_TYPE_192;
break;
case AES_KEYSIZE_256:
ctx->u.aes.type = CCP_AES_TYPE_256;
break;
default:
crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
ctx->u.aes.mode = alg->mode;
/* Set to zero until complete */
ctx->u.aes.key_len = 0;
/* Set the key for the AES cipher used to generate the keys */
ret = crypto_cipher_setkey(ctx->u.aes.tfm_cipher, key, key_len);
if (ret)
return ret;
/* Encrypt a block of zeroes - use key area in context */
memset(ctx->u.aes.key, 0, sizeof(ctx->u.aes.key));
crypto_cipher_encrypt_one(ctx->u.aes.tfm_cipher, ctx->u.aes.key,
ctx->u.aes.key);
/* Generate K1 and K2 */
k0_hi = be64_to_cpu(*((__be64 *)ctx->u.aes.key));
k0_lo = be64_to_cpu(*((__be64 *)ctx->u.aes.key + 1));
k1_hi = (k0_hi << 1) | (k0_lo >> 63);
k1_lo = k0_lo << 1;
if (ctx->u.aes.key[0] & 0x80) {
k1_hi ^= rb_hi;
k1_lo ^= rb_lo;
}
gk = (__be64 *)ctx->u.aes.k1;
*gk = cpu_to_be64(k1_hi);
gk++;
*gk = cpu_to_be64(k1_lo);
k2_hi = (k1_hi << 1) | (k1_lo >> 63);
k2_lo = k1_lo << 1;
if (ctx->u.aes.k1[0] & 0x80) {
k2_hi ^= rb_hi;
k2_lo ^= rb_lo;
}
gk = (__be64 *)ctx->u.aes.k2;
*gk = cpu_to_be64(k2_hi);
gk++;
*gk = cpu_to_be64(k2_lo);
ctx->u.aes.kn_len = sizeof(ctx->u.aes.k1);
sg_init_one(&ctx->u.aes.k1_sg, ctx->u.aes.k1, sizeof(ctx->u.aes.k1));
sg_init_one(&ctx->u.aes.k2_sg, ctx->u.aes.k2, sizeof(ctx->u.aes.k2));
/* Save the supplied key */
memset(ctx->u.aes.key, 0, sizeof(ctx->u.aes.key));
memcpy(ctx->u.aes.key, key, key_len);
ctx->u.aes.key_len = key_len;
sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len);
return ret;
}
static int ccp_aes_cmac_cra_init(struct crypto_tfm *tfm)
{
struct ccp_ctx *ctx = crypto_tfm_ctx(tfm);
struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
struct crypto_cipher *cipher_tfm;
ctx->complete = ccp_aes_cmac_complete;
ctx->u.aes.key_len = 0;
crypto_ahash_set_reqsize(ahash, sizeof(struct ccp_aes_cmac_req_ctx));
cipher_tfm = crypto_alloc_cipher("aes", 0,
CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
if (IS_ERR(cipher_tfm)) {
pr_warn("could not load aes cipher driver\n");
return PTR_ERR(cipher_tfm);
}
ctx->u.aes.tfm_cipher = cipher_tfm;
return 0;
}
static void ccp_aes_cmac_cra_exit(struct crypto_tfm *tfm)
{
struct ccp_ctx *ctx = crypto_tfm_ctx(tfm);
if (ctx->u.aes.tfm_cipher)
crypto_free_cipher(ctx->u.aes.tfm_cipher);
ctx->u.aes.tfm_cipher = NULL;
}
int ccp_register_aes_cmac_algs(struct list_head *head)
{
struct ccp_crypto_ahash_alg *ccp_alg;
struct ahash_alg *alg;
struct hash_alg_common *halg;
struct crypto_alg *base;
int ret;
ccp_alg = kzalloc(sizeof(*ccp_alg), GFP_KERNEL);
if (!ccp_alg)
return -ENOMEM;
INIT_LIST_HEAD(&ccp_alg->entry);
ccp_alg->mode = CCP_AES_MODE_CMAC;
alg = &ccp_alg->alg;
alg->init = ccp_aes_cmac_init;
alg->update = ccp_aes_cmac_update;
alg->final = ccp_aes_cmac_final;
alg->finup = ccp_aes_cmac_finup;
alg->digest = ccp_aes_cmac_digest;
alg->setkey = ccp_aes_cmac_setkey;
halg = &alg->halg;
halg->digestsize = AES_BLOCK_SIZE;
base = &halg->base;
snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "cmac(aes)");
snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "cmac-aes-ccp");
base->cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_NEED_FALLBACK;
base->cra_blocksize = AES_BLOCK_SIZE;
base->cra_ctxsize = sizeof(struct ccp_ctx);
base->cra_priority = CCP_CRA_PRIORITY;
base->cra_type = &crypto_ahash_type;
base->cra_init = ccp_aes_cmac_cra_init;
base->cra_exit = ccp_aes_cmac_cra_exit;
base->cra_module = THIS_MODULE;
ret = crypto_register_ahash(alg);
if (ret) {
pr_err("%s ahash algorithm registration error (%d)\n",
base->cra_name, ret);
kfree(ccp_alg);
return ret;
}
list_add(&ccp_alg->entry, head);
return 0;
}

279
drivers/crypto/ccp/ccp-crypto-aes-xts.c Normal file
View file

@ -0,0 +1,279 @@
/*
* AMD Cryptographic Coprocessor (CCP) AES XTS crypto API support
*
* Copyright (C) 2013 Advanced Micro Devices, Inc.
*
* Author: Tom Lendacky <thomas.lendacky@amd.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/delay.h>
#include <linux/scatterlist.h>
#include <linux/crypto.h>
#include <crypto/algapi.h>
#include <crypto/aes.h>
#include <crypto/scatterwalk.h>
#include "ccp-crypto.h"
struct ccp_aes_xts_def {
const char *name;
const char *drv_name;
};
static struct ccp_aes_xts_def aes_xts_algs[] = {
{
.name = "xts(aes)",
.drv_name = "xts-aes-ccp",
},
};
struct ccp_unit_size_map {
unsigned int size;
u32 value;
};
static struct ccp_unit_size_map unit_size_map[] = {
{
.size = 4096,
.value = CCP_XTS_AES_UNIT_SIZE_4096,
},
{
.size = 2048,
.value = CCP_XTS_AES_UNIT_SIZE_2048,
},
{
.size = 1024,
.value = CCP_XTS_AES_UNIT_SIZE_1024,
},
{
.size = 512,
.value = CCP_XTS_AES_UNIT_SIZE_512,
},
{
.size = 256,
.value = CCP_XTS_AES_UNIT_SIZE__LAST,
},
{
.size = 128,
.value = CCP_XTS_AES_UNIT_SIZE__LAST,
},
{
.size = 64,
.value = CCP_XTS_AES_UNIT_SIZE__LAST,
},
{
.size = 32,
.value = CCP_XTS_AES_UNIT_SIZE__LAST,
},
{
.size = 16,
.value = CCP_XTS_AES_UNIT_SIZE_16,
},
{
.size = 1,
.value = CCP_XTS_AES_UNIT_SIZE__LAST,
},
};
static int ccp_aes_xts_complete(struct crypto_async_request *async_req, int ret)
{
struct ablkcipher_request *req = ablkcipher_request_cast(async_req);
struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req);
if (ret)
return ret;
memcpy(req->info, rctx->iv, AES_BLOCK_SIZE);
return 0;
}
static int ccp_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
unsigned int key_len)
{
struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ablkcipher_tfm(tfm));
/* Only support 128-bit AES key with a 128-bit Tweak key,
* otherwise use the fallback
*/
switch (key_len) {
case AES_KEYSIZE_128 * 2:
memcpy(ctx->u.aes.key, key, key_len);
break;
}
ctx->u.aes.key_len = key_len / 2;
sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len);
return crypto_ablkcipher_setkey(ctx->u.aes.tfm_ablkcipher, key,
key_len);
}
static int ccp_aes_xts_crypt(struct ablkcipher_request *req,
unsigned int encrypt)
{
struct crypto_tfm *tfm =
crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req);
unsigned int unit;
int ret;
if (!ctx->u.aes.key_len)
return -EINVAL;
if (req->nbytes & (AES_BLOCK_SIZE - 1))
return -EINVAL;
if (!req->info)
return -EINVAL;
for (unit = 0; unit < ARRAY_SIZE(unit_size_map); unit++)
if (!(req->nbytes & (unit_size_map[unit].size - 1)))
break;
if ((unit_size_map[unit].value == CCP_XTS_AES_UNIT_SIZE__LAST) ||
(ctx->u.aes.key_len != AES_KEYSIZE_128)) {
/* Use the fallback to process the request for any
* unsupported unit sizes or key sizes
*/
ablkcipher_request_set_tfm(req, ctx->u.aes.tfm_ablkcipher);
ret = (encrypt) ? crypto_ablkcipher_encrypt(req) :
crypto_ablkcipher_decrypt(req);
ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
return ret;
}
memcpy(rctx->iv, req->info, AES_BLOCK_SIZE);
sg_init_one(&rctx->iv_sg, rctx->iv, AES_BLOCK_SIZE);
memset(&rctx->cmd, 0, sizeof(rctx->cmd));
INIT_LIST_HEAD(&rctx->cmd.entry);
rctx->cmd.engine = CCP_ENGINE_XTS_AES_128;
rctx->cmd.u.xts.action = (encrypt) ? CCP_AES_ACTION_ENCRYPT
: CCP_AES_ACTION_DECRYPT;
rctx->cmd.u.xts.unit_size = unit_size_map[unit].value;
rctx->cmd.u.xts.key = &ctx->u.aes.key_sg;
rctx->cmd.u.xts.key_len = ctx->u.aes.key_len;
rctx->cmd.u.xts.iv = &rctx->iv_sg;
rctx->cmd.u.xts.iv_len = AES_BLOCK_SIZE;
rctx->cmd.u.xts.src = req->src;
rctx->cmd.u.xts.src_len = req->nbytes;
rctx->cmd.u.xts.dst = req->dst;
ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd);
return ret;
}
static int ccp_aes_xts_encrypt(struct ablkcipher_request *req)
{
return ccp_aes_xts_crypt(req, 1);
}
static int ccp_aes_xts_decrypt(struct ablkcipher_request *req)
{
return ccp_aes_xts_crypt(req, 0);
}
static int ccp_aes_xts_cra_init(struct crypto_tfm *tfm)
{
struct ccp_ctx *ctx = crypto_tfm_ctx(tfm);
struct crypto_ablkcipher *fallback_tfm;
ctx->complete = ccp_aes_xts_complete;
ctx->u.aes.key_len = 0;
fallback_tfm = crypto_alloc_ablkcipher(crypto_tfm_alg_name(tfm), 0,
CRYPTO_ALG_ASYNC |
CRYPTO_ALG_NEED_FALLBACK);
if (IS_ERR(fallback_tfm)) {
pr_warn("could not load fallback driver %s\n",
crypto_tfm_alg_name(tfm));
return PTR_ERR(fallback_tfm);
}
ctx->u.aes.tfm_ablkcipher = fallback_tfm;
tfm->crt_ablkcipher.reqsize = sizeof(struct ccp_aes_req_ctx) +
fallback_tfm->base.crt_ablkcipher.reqsize;
return 0;
}
static void ccp_aes_xts_cra_exit(struct crypto_tfm *tfm)
{
struct ccp_ctx *ctx = crypto_tfm_ctx(tfm);
if (ctx->u.aes.tfm_ablkcipher)
crypto_free_ablkcipher(ctx->u.aes.tfm_ablkcipher);
ctx->u.aes.tfm_ablkcipher = NULL;
}
static int ccp_register_aes_xts_alg(struct list_head *head,
const struct ccp_aes_xts_def *def)
{
struct ccp_crypto_ablkcipher_alg *ccp_alg;
struct crypto_alg *alg;
int ret;
ccp_alg = kzalloc(sizeof(*ccp_alg), GFP_KERNEL);
if (!ccp_alg)
return -ENOMEM;
INIT_LIST_HEAD(&ccp_alg->entry);
alg = &ccp_alg->alg;
snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name);
snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
def->drv_name);
alg->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_NEED_FALLBACK;
alg->cra_blocksize = AES_BLOCK_SIZE;
alg->cra_ctxsize = sizeof(struct ccp_ctx);
alg->cra_priority = CCP_CRA_PRIORITY;
alg->cra_type = &crypto_ablkcipher_type;
alg->cra_ablkcipher.setkey = ccp_aes_xts_setkey;
alg->cra_ablkcipher.encrypt = ccp_aes_xts_encrypt;
alg->cra_ablkcipher.decrypt = ccp_aes_xts_decrypt;
alg->cra_ablkcipher.min_keysize = AES_MIN_KEY_SIZE * 2;
alg->cra_ablkcipher.max_keysize = AES_MAX_KEY_SIZE * 2;
alg->cra_ablkcipher.ivsize = AES_BLOCK_SIZE;
alg->cra_init = ccp_aes_xts_cra_init;
alg->cra_exit = ccp_aes_xts_cra_exit;
alg->cra_module = THIS_MODULE;
ret = crypto_register_alg(alg);
if (ret) {
pr_err("%s ablkcipher algorithm registration error (%d)\n",
alg->cra_name, ret);
kfree(ccp_alg);
return ret;
}
list_add(&ccp_alg->entry, head);
return 0;
}
int ccp_register_aes_xts_algs(struct list_head *head)
{
int i, ret;
for (i = 0; i < ARRAY_SIZE(aes_xts_algs); i++) {
ret = ccp_register_aes_xts_alg(head, &aes_xts_algs[i]);
if (ret)
return ret;
}
return 0;
}

369
drivers/crypto/ccp/ccp-crypto-aes.c Normal file
View file

@ -0,0 +1,369 @@
/*
* AMD Cryptographic Coprocessor (CCP) AES crypto API support
*
* Copyright (C) 2013 Advanced Micro Devices, Inc.
*
* Author: Tom Lendacky <thomas.lendacky@amd.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/delay.h>
#include <linux/scatterlist.h>
#include <linux/crypto.h>
#include <crypto/algapi.h>
#include <crypto/aes.h>
#include <crypto/ctr.h>
#include <crypto/scatterwalk.h>
#include "ccp-crypto.h"
static int ccp_aes_complete(struct crypto_async_request *async_req, int ret)
{
struct ablkcipher_request *req = ablkcipher_request_cast(async_req);
struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req);
if (ret)
return ret;
if (ctx->u.aes.mode != CCP_AES_MODE_ECB)
memcpy(req->info, rctx->iv, AES_BLOCK_SIZE);
return 0;
}
static int ccp_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
unsigned int key_len)
{
struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ablkcipher_tfm(tfm));
struct ccp_crypto_ablkcipher_alg *alg =
ccp_crypto_ablkcipher_alg(crypto_ablkcipher_tfm(tfm));
switch (key_len) {
case AES_KEYSIZE_128:
ctx->u.aes.type = CCP_AES_TYPE_128;
break;
case AES_KEYSIZE_192:
ctx->u.aes.type = CCP_AES_TYPE_192;
break;
case AES_KEYSIZE_256:
ctx->u.aes.type = CCP_AES_TYPE_256;
break;
default:
crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
ctx->u.aes.mode = alg->mode;
ctx->u.aes.key_len = key_len;
memcpy(ctx->u.aes.key, key, key_len);
sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len);
return 0;
}
static int ccp_aes_crypt(struct ablkcipher_request *req, bool encrypt)
{
struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req);
struct scatterlist *iv_sg = NULL;
unsigned int iv_len = 0;
int ret;
if (!ctx->u.aes.key_len)
return -EINVAL;
if (((ctx->u.aes.mode == CCP_AES_MODE_ECB) ||
(ctx->u.aes.mode == CCP_AES_MODE_CBC) ||
(ctx->u.aes.mode == CCP_AES_MODE_CFB)) &&
(req->nbytes & (AES_BLOCK_SIZE - 1)))
return -EINVAL;
if (ctx->u.aes.mode != CCP_AES_MODE_ECB) {
if (!req->info)
return -EINVAL;
memcpy(rctx->iv, req->info, AES_BLOCK_SIZE);
iv_sg = &rctx->iv_sg;
iv_len = AES_BLOCK_SIZE;
sg_init_one(iv_sg, rctx->iv, iv_len);
}
memset(&rctx->cmd, 0, sizeof(rctx->cmd));
INIT_LIST_HEAD(&rctx->cmd.entry);
rctx->cmd.engine = CCP_ENGINE_AES;
rctx->cmd.u.aes.type = ctx->u.aes.type;
rctx->cmd.u.aes.mode = ctx->u.aes.mode;
rctx->cmd.u.aes.action =
(encrypt) ? CCP_AES_ACTION_ENCRYPT : CCP_AES_ACTION_DECRYPT;
rctx->cmd.u.aes.key = &ctx->u.aes.key_sg;
rctx->cmd.u.aes.key_len = ctx->u.aes.key_len;
rctx->cmd.u.aes.iv = iv_sg;
rctx->cmd.u.aes.iv_len = iv_len;
rctx->cmd.u.aes.src = req->src;
rctx->cmd.u.aes.src_len = req->nbytes;
rctx->cmd.u.aes.dst = req->dst;
ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd);
return ret;
}
static int ccp_aes_encrypt(struct ablkcipher_request *req)
{
return ccp_aes_crypt(req, true);
}
static int ccp_aes_decrypt(struct ablkcipher_request *req)
{
return ccp_aes_crypt(req, false);
}
static int ccp_aes_cra_init(struct crypto_tfm *tfm)
{
struct ccp_ctx *ctx = crypto_tfm_ctx(tfm);
ctx->complete = ccp_aes_complete;
ctx->u.aes.key_len = 0;
tfm->crt_ablkcipher.reqsize = sizeof(struct ccp_aes_req_ctx);
return 0;
}
static void ccp_aes_cra_exit(struct crypto_tfm *tfm)
{
}
static int ccp_aes_rfc3686_complete(struct crypto_async_request *async_req,
int ret)
{
struct ablkcipher_request *req = ablkcipher_request_cast(async_req);
struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req);
/* Restore the original pointer */
req->info = rctx->rfc3686_info;
return ccp_aes_complete(async_req, ret);
}
static int ccp_aes_rfc3686_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
unsigned int key_len)
{
struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ablkcipher_tfm(tfm));
if (key_len < CTR_RFC3686_NONCE_SIZE)
return -EINVAL;
key_len -= CTR_RFC3686_NONCE_SIZE;
memcpy(ctx->u.aes.nonce, key + key_len, CTR_RFC3686_NONCE_SIZE);
return ccp_aes_setkey(tfm, key, key_len);
}
static int ccp_aes_rfc3686_crypt(struct ablkcipher_request *req, bool encrypt)
{
struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req);
u8 *iv;
/* Initialize the CTR block */
iv = rctx->rfc3686_iv;
memcpy(iv, ctx->u.aes.nonce, CTR_RFC3686_NONCE_SIZE);
iv += CTR_RFC3686_NONCE_SIZE;
memcpy(iv, req->info, CTR_RFC3686_IV_SIZE);
iv += CTR_RFC3686_IV_SIZE;
*(__be32 *)iv = cpu_to_be32(1);
/* Point to the new IV */
rctx->rfc3686_info = req->info;
req->info = rctx->rfc3686_iv;
return ccp_aes_crypt(req, encrypt);
}
static int ccp_aes_rfc3686_encrypt(struct ablkcipher_request *req)
{
return ccp_aes_rfc3686_crypt(req, true);
}
static int ccp_aes_rfc3686_decrypt(struct ablkcipher_request *req)
{
return ccp_aes_rfc3686_crypt(req, false);
}
static int ccp_aes_rfc3686_cra_init(struct crypto_tfm *tfm)
{
struct ccp_ctx *ctx = crypto_tfm_ctx(tfm);
ctx->complete = ccp_aes_rfc3686_complete;
ctx->u.aes.key_len = 0;
tfm->crt_ablkcipher.reqsize = sizeof(struct ccp_aes_req_ctx);
return 0;
}
static void ccp_aes_rfc3686_cra_exit(struct crypto_tfm *tfm)
{
}
static struct crypto_alg ccp_aes_defaults = {
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct ccp_ctx),
.cra_priority = CCP_CRA_PRIORITY,
.cra_type = &crypto_ablkcipher_type,
.cra_init = ccp_aes_cra_init,
.cra_exit = ccp_aes_cra_exit,
.cra_module = THIS_MODULE,
.cra_ablkcipher = {
.setkey = ccp_aes_setkey,
.encrypt = ccp_aes_encrypt,
.decrypt = ccp_aes_decrypt,
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
},
};
static struct crypto_alg ccp_aes_rfc3686_defaults = {
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = CTR_RFC3686_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct ccp_ctx),
.cra_priority = CCP_CRA_PRIORITY,
.cra_type = &crypto_ablkcipher_type,
.cra_init = ccp_aes_rfc3686_cra_init,
.cra_exit = ccp_aes_rfc3686_cra_exit,
.cra_module = THIS_MODULE,
.cra_ablkcipher = {
.setkey = ccp_aes_rfc3686_setkey,
.encrypt = ccp_aes_rfc3686_encrypt,
.decrypt = ccp_aes_rfc3686_decrypt,
.min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
.max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
},
};
struct ccp_aes_def {
enum ccp_aes_mode mode;
const char *name;
const char *driver_name;
unsigned int blocksize;
unsigned int ivsize;
struct crypto_alg *alg_defaults;
};
static struct ccp_aes_def aes_algs[] = {
{
.mode = CCP_AES_MODE_ECB,
.name = "ecb(aes)",
.driver_name = "ecb-aes-ccp",
.blocksize = AES_BLOCK_SIZE,
.ivsize = 0,
.alg_defaults = &ccp_aes_defaults,
},
{
.mode = CCP_AES_MODE_CBC,
.name = "cbc(aes)",
.driver_name = "cbc-aes-ccp",
.blocksize = AES_BLOCK_SIZE,
.ivsize = AES_BLOCK_SIZE,
.alg_defaults = &ccp_aes_defaults,
},
{
.mode = CCP_AES_MODE_CFB,
.name = "cfb(aes)",
.driver_name = "cfb-aes-ccp",
.blocksize = AES_BLOCK_SIZE,
.ivsize = AES_BLOCK_SIZE,
.alg_defaults = &ccp_aes_defaults,
},
{
.mode = CCP_AES_MODE_OFB,
.name = "ofb(aes)",
.driver_name = "ofb-aes-ccp",
.blocksize = 1,
.ivsize = AES_BLOCK_SIZE,
.alg_defaults = &ccp_aes_defaults,
},
{
.mode = CCP_AES_MODE_CTR,
.name = "ctr(aes)",
.driver_name = "ctr-aes-ccp",
.blocksize = 1,
.ivsize = AES_BLOCK_SIZE,
.alg_defaults = &ccp_aes_defaults,
},
{
.mode = CCP_AES_MODE_CTR,
.name = "rfc3686(ctr(aes))",
.driver_name = "rfc3686-ctr-aes-ccp",
.blocksize = 1,
.ivsize = CTR_RFC3686_IV_SIZE,
.alg_defaults = &ccp_aes_rfc3686_defaults,
},
};
static int ccp_register_aes_alg(struct list_head *head,
const struct ccp_aes_def *def)
{
struct ccp_crypto_ablkcipher_alg *ccp_alg;
struct crypto_alg *alg;
int ret;
ccp_alg = kzalloc(sizeof(*ccp_alg), GFP_KERNEL);
if (!ccp_alg)
return -ENOMEM;
INIT_LIST_HEAD(&ccp_alg->entry);
ccp_alg->mode = def->mode;
/* Copy the defaults and override as necessary */
alg = &ccp_alg->alg;
*alg = *def->alg_defaults;
snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name);
snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
def->driver_name);
alg->cra_blocksize = def->blocksize;
alg->cra_ablkcipher.ivsize = def->ivsize;
ret = crypto_register_alg(alg);
if (ret) {
pr_err("%s ablkcipher algorithm registration error (%d)\n",
alg->cra_name, ret);
kfree(ccp_alg);
return ret;
}
list_add(&ccp_alg->entry, head);
return 0;
}
int ccp_register_aes_algs(struct list_head *head)
{
int i, ret;
for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
ret = ccp_register_aes_alg(head, &aes_algs[i]);
if (ret)
return ret;
}
return 0;
}

392
drivers/crypto/ccp/ccp-crypto-main.c Normal file
View file

@ -0,0 +1,392 @@
/*
* AMD Cryptographic Coprocessor (CCP) crypto API support
*
* Copyright (C) 2013 Advanced Micro Devices, Inc.
*
* Author: Tom Lendacky <thomas.lendacky@amd.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/kernel.h>
#include <linux/list.h>
#include <linux/ccp.h>
#include <linux/scatterlist.h>
#include <crypto/internal/hash.h>
#include "ccp-crypto.h"
MODULE_AUTHOR("Tom Lendacky <thomas.lendacky@amd.com>");
MODULE_LICENSE("GPL");
MODULE_VERSION("1.0.0");
MODULE_DESCRIPTION("AMD Cryptographic Coprocessor crypto API support");
static unsigned int aes_disable;
module_param(aes_disable, uint, 0444);
MODULE_PARM_DESC(aes_disable, "Disable use of AES - any non-zero value");
static unsigned int sha_disable;
module_param(sha_disable, uint, 0444);
MODULE_PARM_DESC(sha_disable, "Disable use of SHA - any non-zero value");
/* List heads for the supported algorithms */
static LIST_HEAD(hash_algs);
static LIST_HEAD(cipher_algs);
/* For any tfm, requests for that tfm must be returned on the order
* received. With multiple queues available, the CCP can process more
* than one cmd at a time. Therefore we must maintain a cmd list to insure
* the proper ordering of requests on a given tfm.
*/
struct ccp_crypto_queue {
struct list_head cmds;
struct list_head *backlog;
unsigned int cmd_count;
};
#define CCP_CRYPTO_MAX_QLEN 100
static struct ccp_crypto_queue req_queue;
static spinlock_t req_queue_lock;
struct ccp_crypto_cmd {
struct list_head entry;
struct ccp_cmd *cmd;
/* Save the crypto_tfm and crypto_async_request addresses
* separately to avoid any reference to a possibly invalid
* crypto_async_request structure after invoking the request
* callback
*/
struct crypto_async_request *req;
struct crypto_tfm *tfm;
/* Used for held command processing to determine state */
int ret;
};
struct ccp_crypto_cpu {
struct work_struct work;
struct completion completion;
struct ccp_crypto_cmd *crypto_cmd;
int err;
};
static inline bool ccp_crypto_success(int err)
{
if (err && (err != -EINPROGRESS) && (err != -EBUSY))
return false;
return true;
}
static struct ccp_crypto_cmd *ccp_crypto_cmd_complete(
struct ccp_crypto_cmd *crypto_cmd, struct ccp_crypto_cmd **backlog)
{
struct ccp_crypto_cmd *held = NULL, *tmp;
unsigned long flags;
*backlog = NULL;
spin_lock_irqsave(&req_queue_lock, flags);
/* Held cmds will be after the current cmd in the queue so start
* searching for a cmd with a matching tfm for submission.
*/
tmp = crypto_cmd;
list_for_each_entry_continue(tmp, &req_queue.cmds, entry) {
if (crypto_cmd->tfm != tmp->tfm)
continue;
held = tmp;
break;
}
/* Process the backlog:
* Because cmds can be executed from any point in the cmd list
* special precautions have to be taken when handling the backlog.
*/
if (req_queue.backlog != &req_queue.cmds) {
/* Skip over this cmd if it is the next backlog cmd */
if (req_queue.backlog == &crypto_cmd->entry)
req_queue.backlog = crypto_cmd->entry.next;
*backlog = container_of(req_queue.backlog,
struct ccp_crypto_cmd, entry);
req_queue.backlog = req_queue.backlog->next;
/* Skip over this cmd if it is now the next backlog cmd */
if (req_queue.backlog == &crypto_cmd->entry)
req_queue.backlog = crypto_cmd->entry.next;
}
/* Remove the cmd entry from the list of cmds */
req_queue.cmd_count--;
list_del(&crypto_cmd->entry);
spin_unlock_irqrestore(&req_queue_lock, flags);
return held;
}
static void ccp_crypto_complete(void *data, int err)
{
struct ccp_crypto_cmd *crypto_cmd = data;
struct ccp_crypto_cmd *held, *next, *backlog;
struct crypto_async_request *req = crypto_cmd->req;
struct ccp_ctx *ctx = crypto_tfm_ctx(req->tfm);
int ret;
if (err == -EINPROGRESS) {
/* Only propogate the -EINPROGRESS if necessary */
if (crypto_cmd->ret == -EBUSY) {
crypto_cmd->ret = -EINPROGRESS;
req->complete(req, -EINPROGRESS);
}
return;
}
/* Operation has completed - update the queue before invoking
* the completion callbacks and retrieve the next cmd (cmd with
* a matching tfm) that can be submitted to the CCP.
*/
held = ccp_crypto_cmd_complete(crypto_cmd, &backlog);
if (backlog) {
backlog->ret = -EINPROGRESS;
backlog->req->complete(backlog->req, -EINPROGRESS);
}
/* Transition the state from -EBUSY to -EINPROGRESS first */
if (crypto_cmd->ret == -EBUSY)
req->complete(req, -EINPROGRESS);
/* Completion callbacks */
ret = err;
if (ctx->complete)
ret = ctx->complete(req, ret);
req->complete(req, ret);
/* Submit the next cmd */
while (held) {
/* Since we have already queued the cmd, we must indicate that
* we can backlog so as not to "lose" this request.
*/
held->cmd->flags |= CCP_CMD_MAY_BACKLOG;
ret = ccp_enqueue_cmd(held->cmd);
if (ccp_crypto_success(ret))
break;
/* Error occurred, report it and get the next entry */
ctx = crypto_tfm_ctx(held->req->tfm);
if (ctx->complete)
ret = ctx->complete(held->req, ret);
held->req->complete(held->req, ret);
next = ccp_crypto_cmd_complete(held, &backlog);
if (backlog) {
backlog->ret = -EINPROGRESS;
backlog->req->complete(backlog->req, -EINPROGRESS);
}
kfree(held);
held = next;
}
kfree(crypto_cmd);
}
static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd)
{
struct ccp_crypto_cmd *active = NULL, *tmp;
unsigned long flags;
bool free_cmd = true;
int ret;
spin_lock_irqsave(&req_queue_lock, flags);
/* Check if the cmd can/should be queued */
if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) {
ret = -EBUSY;
if (!(crypto_cmd->cmd->flags & CCP_CMD_MAY_BACKLOG))
goto e_lock;
}
/* Look for an entry with the same tfm. If there is a cmd
* with the same tfm in the list then the current cmd cannot
* be submitted to the CCP yet.
*/
list_for_each_entry(tmp, &req_queue.cmds, entry) {
if (crypto_cmd->tfm != tmp->tfm)
continue;
active = tmp;
break;
}
ret = -EINPROGRESS;
if (!active) {
ret = ccp_enqueue_cmd(crypto_cmd->cmd);
if (!ccp_crypto_success(ret))
goto e_lock; /* Error, don't queue it */
if ((ret == -EBUSY) &&
!(crypto_cmd->cmd->flags & CCP_CMD_MAY_BACKLOG))
goto e_lock; /* Not backlogging, don't queue it */
}
if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) {
ret = -EBUSY;
if (req_queue.backlog == &req_queue.cmds)
req_queue.backlog = &crypto_cmd->entry;
}
crypto_cmd->ret = ret;
req_queue.cmd_count++;
list_add_tail(&crypto_cmd->entry, &req_queue.cmds);
free_cmd = false;
e_lock:
spin_unlock_irqrestore(&req_queue_lock, flags);
if (free_cmd)
kfree(crypto_cmd);
return ret;
}
/**
* ccp_crypto_enqueue_request - queue an crypto async request for processing
* by the CCP
*
* @req: crypto_async_request struct to be processed
* @cmd: ccp_cmd struct to be sent to the CCP
*/
int ccp_crypto_enqueue_request(struct crypto_async_request *req,
struct ccp_cmd *cmd)
{
struct ccp_crypto_cmd *crypto_cmd;
gfp_t gfp;
gfp = req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC;
crypto_cmd = kzalloc(sizeof(*crypto_cmd), gfp);
if (!crypto_cmd)
return -ENOMEM;
/* The tfm pointer must be saved and not referenced from the
* crypto_async_request (req) pointer because it is used after
* completion callback for the request and the req pointer
* might not be valid anymore.
*/
crypto_cmd->cmd = cmd;
crypto_cmd->req = req;
crypto_cmd->tfm = req->tfm;
cmd->callback = ccp_crypto_complete;
cmd->data = crypto_cmd;
if (req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)
cmd->flags |= CCP_CMD_MAY_BACKLOG;
else
cmd->flags &= ~CCP_CMD_MAY_BACKLOG;
return ccp_crypto_enqueue_cmd(crypto_cmd);
}
struct scatterlist *ccp_crypto_sg_table_add(struct sg_table *table,
struct scatterlist *sg_add)
{
struct scatterlist *sg, *sg_last = NULL;
for (sg = table->sgl; sg; sg = sg_next(sg))
if (!sg_page(sg))
break;
BUG_ON(!sg);
for (; sg && sg_add; sg = sg_next(sg), sg_add = sg_next(sg_add)) {
sg_set_page(sg, sg_page(sg_add), sg_add->length,
sg_add->offset);
sg_last = sg;
}
BUG_ON(sg_add);
return sg_last;
}
static int ccp_register_algs(void)
{
int ret;
if (!aes_disable) {
ret = ccp_register_aes_algs(&cipher_algs);
if (ret)
return ret;
ret = ccp_register_aes_cmac_algs(&hash_algs);
if (ret)
return ret;
ret = ccp_register_aes_xts_algs(&cipher_algs);
if (ret)
return ret;
}
if (!sha_disable) {
ret = ccp_register_sha_algs(&hash_algs);
if (ret)
return ret;
}
return 0;
}
static void ccp_unregister_algs(void)
{
struct ccp_crypto_ahash_alg *ahash_alg, *ahash_tmp;
struct ccp_crypto_ablkcipher_alg *ablk_alg, *ablk_tmp;
list_for_each_entry_safe(ahash_alg, ahash_tmp, &hash_algs, entry) {
crypto_unregister_ahash(&ahash_alg->alg);
list_del(&ahash_alg->entry);
kfree(ahash_alg);
}
list_for_each_entry_safe(ablk_alg, ablk_tmp, &cipher_algs, entry) {
crypto_unregister_alg(&ablk_alg->alg);
list_del(&ablk_alg->entry);
kfree(ablk_alg);
}
}
static int ccp_crypto_init(void)
{
int ret;
ret = ccp_present();
if (ret)
return ret;
spin_lock_init(&req_queue_lock);
INIT_LIST_HEAD(&req_queue.cmds);
req_queue.backlog = &req_queue.cmds;
req_queue.cmd_count = 0;
ret = ccp_register_algs();
if (ret)
ccp_unregister_algs();
return ret;
}
static void ccp_crypto_exit(void)
{
ccp_unregister_algs();
}
module_init(ccp_crypto_init);
module_exit(ccp_crypto_exit);

436
drivers/crypto/ccp/ccp-crypto-sha.c Normal file
View file

@ -0,0 +1,436 @@
/*
* AMD Cryptographic Coprocessor (CCP) SHA crypto API support
*
* Copyright (C) 2013 Advanced Micro Devices, Inc.
*
* Author: Tom Lendacky <thomas.lendacky@amd.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/delay.h>
#include <linux/scatterlist.h>
#include <linux/crypto.h>
#include <crypto/algapi.h>
#include <crypto/hash.h>
#include <crypto/internal/hash.h>
#include <crypto/sha.h>
#include <crypto/scatterwalk.h>
#include "ccp-crypto.h"
static int ccp_sha_complete(struct crypto_async_request *async_req, int ret)
{
struct ahash_request *req = ahash_request_cast(async_req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
struct ccp_sha_req_ctx *rctx = ahash_request_ctx(req);
unsigned int digest_size = crypto_ahash_digestsize(tfm);
if (ret)
goto e_free;
if (rctx->hash_rem) {
/* Save remaining data to buffer */
unsigned int offset = rctx->nbytes - rctx->hash_rem;
scatterwalk_map_and_copy(rctx->buf, rctx->src,
offset, rctx->hash_rem, 0);
rctx->buf_count = rctx->hash_rem;
} else
rctx->buf_count = 0;
/* Update result area if supplied */
if (req->result)
memcpy(req->result, rctx->ctx, digest_size);
e_free:
sg_free_table(&rctx->data_sg);
return ret;
}
static int ccp_do_sha_update(struct ahash_request *req, unsigned int nbytes,
unsigned int final)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
struct ccp_ctx *ctx = crypto_ahash_ctx(tfm);
struct ccp_sha_req_ctx *rctx = ahash_request_ctx(req);
struct scatterlist *sg;
unsigned int block_size =
crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
unsigned int sg_count;
gfp_t gfp;
u64 len;
int ret;
len = (u64)rctx->buf_count + (u64)nbytes;
if (!final && (len <= block_size)) {
scatterwalk_map_and_copy(rctx->buf + rctx->buf_count, req->src,
0, nbytes, 0);
rctx->buf_count += nbytes;
return 0;
}
rctx->src = req->src;
rctx->nbytes = nbytes;
rctx->final = final;
rctx->hash_rem = final ? 0 : len & (block_size - 1);
rctx->hash_cnt = len - rctx->hash_rem;
if (!final && !rctx->hash_rem) {
/* CCP can't do zero length final, so keep some data around */
rctx->hash_cnt -= block_size;
rctx->hash_rem = block_size;
}
/* Initialize the context scatterlist */
sg_init_one(&rctx->ctx_sg, rctx->ctx, sizeof(rctx->ctx));
sg = NULL;
if (rctx->buf_count && nbytes) {
/* Build the data scatterlist table - allocate enough entries
* for both data pieces (buffer and input data)
*/
gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
GFP_KERNEL : GFP_ATOMIC;
sg_count = sg_nents(req->src) + 1;
ret = sg_alloc_table(&rctx->data_sg, sg_count, gfp);
if (ret)
return ret;
sg_init_one(&rctx->buf_sg, rctx->buf, rctx->buf_count);
sg = ccp_crypto_sg_table_add(&rctx->data_sg, &rctx->buf_sg);
sg = ccp_crypto_sg_table_add(&rctx->data_sg, req->src);
sg_mark_end(sg);
sg = rctx->data_sg.sgl;
} else if (rctx->buf_count) {
sg_init_one(&rctx->buf_sg, rctx->buf, rctx->buf_count);
sg = &rctx->buf_sg;
} else if (nbytes) {
sg = req->src;
}
rctx->msg_bits += (rctx->hash_cnt << 3); /* Total in bits */
memset(&rctx->cmd, 0, sizeof(rctx->cmd));
INIT_LIST_HEAD(&rctx->cmd.entry);
rctx->cmd.engine = CCP_ENGINE_SHA;
rctx->cmd.u.sha.type = rctx->type;
rctx->cmd.u.sha.ctx = &rctx->ctx_sg;
rctx->cmd.u.sha.ctx_len = sizeof(rctx->ctx);
rctx->cmd.u.sha.src = sg;
rctx->cmd.u.sha.src_len = rctx->hash_cnt;
rctx->cmd.u.sha.opad = ctx->u.sha.key_len ?
&ctx->u.sha.opad_sg : NULL;
rctx->cmd.u.sha.opad_len = ctx->u.sha.key_len ?
ctx->u.sha.opad_count : 0;
rctx->cmd.u.sha.first = rctx->first;
rctx->cmd.u.sha.final = rctx->final;
rctx->cmd.u.sha.msg_bits = rctx->msg_bits;
rctx->first = 0;
ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd);
return ret;
}
static int ccp_sha_init(struct ahash_request *req)
{
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
struct ccp_ctx *ctx = crypto_ahash_ctx(tfm);
struct ccp_sha_req_ctx *rctx = ahash_request_ctx(req);
struct ccp_crypto_ahash_alg *alg =
ccp_crypto_ahash_alg(crypto_ahash_tfm(tfm));
unsigned int block_size =
crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
memset(rctx, 0, sizeof(*rctx));
rctx->type = alg->type;
rctx->first = 1;
if (ctx->u.sha.key_len) {
/* Buffer the HMAC key for first update */
memcpy(rctx->buf, ctx->u.sha.ipad, block_size);
rctx->buf_count = block_size;
}
return 0;
}
static int ccp_sha_update(struct ahash_request *req)
{
return ccp_do_sha_update(req, req->nbytes, 0);
}
static int ccp_sha_final(struct ahash_request *req)
{
return ccp_do_sha_update(req, 0, 1);
}
static int ccp_sha_finup(struct ahash_request *req)
{
return ccp_do_sha_update(req, req->nbytes, 1);
}
static int ccp_sha_digest(struct ahash_request *req)
{
int ret;
ret = ccp_sha_init(req);
if (ret)
return ret;
return ccp_sha_finup(req);
}
static int ccp_sha_setkey(struct crypto_ahash *tfm, const u8 *key,
unsigned int key_len)
{
struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
struct crypto_shash *shash = ctx->u.sha.hmac_tfm;
SHASH_DESC_ON_STACK(sdesc, shash);
unsigned int block_size = crypto_shash_blocksize(shash);
unsigned int digest_size = crypto_shash_digestsize(shash);
int i, ret;
/* Set to zero until complete */
ctx->u.sha.key_len = 0;
/* Clear key area to provide zero padding for keys smaller
* than the block size
*/
memset(ctx->u.sha.key, 0, sizeof(ctx->u.sha.key));
if (key_len > block_size) {
/* Must hash the input key */
sdesc->tfm = shash;
sdesc->flags = crypto_ahash_get_flags(tfm) &
CRYPTO_TFM_REQ_MAY_SLEEP;
ret = crypto_shash_digest(sdesc, key, key_len,
ctx->u.sha.key);
if (ret) {
crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
key_len = digest_size;
} else
memcpy(ctx->u.sha.key, key, key_len);
for (i = 0; i < block_size; i++) {
ctx->u.sha.ipad[i] = ctx->u.sha.key[i] ^ 0x36;
ctx->u.sha.opad[i] = ctx->u.sha.key[i] ^ 0x5c;
}
sg_init_one(&ctx->u.sha.opad_sg, ctx->u.sha.opad, block_size);
ctx->u.sha.opad_count = block_size;
ctx->u.sha.key_len = key_len;
return 0;
}
static int ccp_sha_cra_init(struct crypto_tfm *tfm)
{
struct ccp_ctx *ctx = crypto_tfm_ctx(tfm);
struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
ctx->complete = ccp_sha_complete;
ctx->u.sha.key_len = 0;
crypto_ahash_set_reqsize(ahash, sizeof(struct ccp_sha_req_ctx));
return 0;
}
static void ccp_sha_cra_exit(struct crypto_tfm *tfm)
{
}
static int ccp_hmac_sha_cra_init(struct crypto_tfm *tfm)
{
struct ccp_ctx *ctx = crypto_tfm_ctx(tfm);
struct ccp_crypto_ahash_alg *alg = ccp_crypto_ahash_alg(tfm);
struct crypto_shash *hmac_tfm;
hmac_tfm = crypto_alloc_shash(alg->child_alg, 0, 0);
if (IS_ERR(hmac_tfm)) {
pr_warn("could not load driver %s need for HMAC support\n",
alg->child_alg);
return PTR_ERR(hmac_tfm);
}
ctx->u.sha.hmac_tfm = hmac_tfm;
return ccp_sha_cra_init(tfm);
}
static void ccp_hmac_sha_cra_exit(struct crypto_tfm *tfm)
{
struct ccp_ctx *ctx = crypto_tfm_ctx(tfm);
if (ctx->u.sha.hmac_tfm)
crypto_free_shash(ctx->u.sha.hmac_tfm);
ccp_sha_cra_exit(tfm);
}
struct ccp_sha_def {
const char *name;
const char *drv_name;
enum ccp_sha_type type;
u32 digest_size;
u32 block_size;
};
static struct ccp_sha_def sha_algs[] = {
{
.name = "sha1",
.drv_name = "sha1-ccp",
.type = CCP_SHA_TYPE_1,
.digest_size = SHA1_DIGEST_SIZE,
.block_size = SHA1_BLOCK_SIZE,
},
{
.name = "sha224",
.drv_name = "sha224-ccp",
.type = CCP_SHA_TYPE_224,
.digest_size = SHA224_DIGEST_SIZE,
.block_size = SHA224_BLOCK_SIZE,
},
{
.name = "sha256",
.drv_name = "sha256-ccp",
.type = CCP_SHA_TYPE_256,
.digest_size = SHA256_DIGEST_SIZE,
.block_size = SHA256_BLOCK_SIZE,
},
};
static int ccp_register_hmac_alg(struct list_head *head,
const struct ccp_sha_def *def,
const struct ccp_crypto_ahash_alg *base_alg)
{
struct ccp_crypto_ahash_alg *ccp_alg;
struct ahash_alg *alg;
struct hash_alg_common *halg;
struct crypto_alg *base;
int ret;
ccp_alg = kzalloc(sizeof(*ccp_alg), GFP_KERNEL);
if (!ccp_alg)
return -ENOMEM;
/* Copy the base algorithm and only change what's necessary */
*ccp_alg = *base_alg;
INIT_LIST_HEAD(&ccp_alg->entry);
strncpy(ccp_alg->child_alg, def->name, CRYPTO_MAX_ALG_NAME);
alg = &ccp_alg->alg;
alg->setkey = ccp_sha_setkey;
halg = &alg->halg;
base = &halg->base;
snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", def->name);
snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s",
def->drv_name);
base->cra_init = ccp_hmac_sha_cra_init;
base->cra_exit = ccp_hmac_sha_cra_exit;
ret = crypto_register_ahash(alg);
if (ret) {
pr_err("%s ahash algorithm registration error (%d)\n",
base->cra_name, ret);
kfree(ccp_alg);
return ret;
}
list_add(&ccp_alg->entry, head);
return ret;
}
static int ccp_register_sha_alg(struct list_head *head,
const struct ccp_sha_def *def)
{
struct ccp_crypto_ahash_alg *ccp_alg;
struct ahash_alg *alg;
struct hash_alg_common *halg;
struct crypto_alg *base;
int ret;
ccp_alg = kzalloc(sizeof(*ccp_alg), GFP_KERNEL);
if (!ccp_alg)
return -ENOMEM;
INIT_LIST_HEAD(&ccp_alg->entry);
ccp_alg->type = def->type;
alg = &ccp_alg->alg;
alg->init = ccp_sha_init;
alg->update = ccp_sha_update;
alg->final = ccp_sha_final;
alg->finup = ccp_sha_finup;
alg->digest = ccp_sha_digest;
halg = &alg->halg;
halg->digestsize = def->digest_size;
base = &halg->base;
snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name);
snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
def->drv_name);
base->cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_NEED_FALLBACK;
base->cra_blocksize = def->block_size;
base->cra_ctxsize = sizeof(struct ccp_ctx);
base->cra_priority = CCP_CRA_PRIORITY;
base->cra_type = &crypto_ahash_type;
base->cra_init = ccp_sha_cra_init;
base->cra_exit = ccp_sha_cra_exit;
base->cra_module = THIS_MODULE;
ret = crypto_register_ahash(alg);
if (ret) {
pr_err("%s ahash algorithm registration error (%d)\n",
base->cra_name, ret);
kfree(ccp_alg);
return ret;
}
list_add(&ccp_alg->entry, head);
ret = ccp_register_hmac_alg(head, def, ccp_alg);
return ret;
}
int ccp_register_sha_algs(struct list_head *head)
{
int i, ret;
for (i = 0; i < ARRAY_SIZE(sha_algs); i++) {
ret = ccp_register_sha_alg(head, &sha_algs[i]);
if (ret)
return ret;
}
return 0;
}

197
drivers/crypto/ccp/ccp-crypto.h Normal file
View file

@ -0,0 +1,197 @@
/*
* AMD Cryptographic Coprocessor (CCP) crypto API support
*
* Copyright (C) 2013 Advanced Micro Devices, Inc.
*
* Author: Tom Lendacky <thomas.lendacky@amd.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#ifndef __CCP_CRYPTO_H__
#define __CCP_CRYPTO_H__
#include <linux/list.h>
#include <linux/wait.h>
#include <linux/pci.h>
#include <linux/ccp.h>
#include <linux/crypto.h>
#include <crypto/algapi.h>
#include <crypto/aes.h>
#include <crypto/ctr.h>
#include <crypto/hash.h>
#include <crypto/sha.h>
#define CCP_CRA_PRIORITY 300
struct ccp_crypto_ablkcipher_alg {
struct list_head entry;
u32 mode;
struct crypto_alg alg;
};
struct ccp_crypto_ahash_alg {
struct list_head entry;
const __be32 *init;
u32 type;
u32 mode;
/* Child algorithm used for HMAC, CMAC, etc */
char child_alg[CRYPTO_MAX_ALG_NAME];
struct ahash_alg alg;
};
static inline struct ccp_crypto_ablkcipher_alg *
ccp_crypto_ablkcipher_alg(struct crypto_tfm *tfm)
{
struct crypto_alg *alg = tfm->__crt_alg;
return container_of(alg, struct ccp_crypto_ablkcipher_alg, alg);
}
static inline struct ccp_crypto_ahash_alg *
ccp_crypto_ahash_alg(struct crypto_tfm *tfm)
{
struct crypto_alg *alg = tfm->__crt_alg;
struct ahash_alg *ahash_alg;
ahash_alg = container_of(alg, struct ahash_alg, halg.base);
return container_of(ahash_alg, struct ccp_crypto_ahash_alg, alg);
}
/***** AES related defines *****/
struct ccp_aes_ctx {
/* Fallback cipher for XTS with unsupported unit sizes */
struct crypto_ablkcipher *tfm_ablkcipher;
/* Cipher used to generate CMAC K1/K2 keys */
struct crypto_cipher *tfm_cipher;
enum ccp_engine engine;
enum ccp_aes_type type;
enum ccp_aes_mode mode;
struct scatterlist key_sg;
unsigned int key_len;
u8 key[AES_MAX_KEY_SIZE];
u8 nonce[CTR_RFC3686_NONCE_SIZE];
/* CMAC key structures */
struct scatterlist k1_sg;
struct scatterlist k2_sg;
unsigned int kn_len;
u8 k1[AES_BLOCK_SIZE];
u8 k2[AES_BLOCK_SIZE];
};
struct ccp_aes_req_ctx {
struct scatterlist iv_sg;
u8 iv[AES_BLOCK_SIZE];
/* Fields used for RFC3686 requests */
u8 *rfc3686_info;
u8 rfc3686_iv[AES_BLOCK_SIZE];
struct ccp_cmd cmd;
};
struct ccp_aes_cmac_req_ctx {
unsigned int null_msg;
unsigned int final;
struct scatterlist *src;
unsigned int nbytes;
u64 hash_cnt;
unsigned int hash_rem;
struct sg_table data_sg;
struct scatterlist iv_sg;
u8 iv[AES_BLOCK_SIZE];
struct scatterlist buf_sg;
unsigned int buf_count;
u8 buf[AES_BLOCK_SIZE];
struct scatterlist pad_sg;
unsigned int pad_count;
u8 pad[AES_BLOCK_SIZE];
struct ccp_cmd cmd;
};
/***** SHA related defines *****/
#define MAX_SHA_CONTEXT_SIZE SHA256_DIGEST_SIZE
#define MAX_SHA_BLOCK_SIZE SHA256_BLOCK_SIZE
struct ccp_sha_ctx {
struct scatterlist opad_sg;
unsigned int opad_count;
unsigned int key_len;
u8 key[MAX_SHA_BLOCK_SIZE];
u8 ipad[MAX_SHA_BLOCK_SIZE];
u8 opad[MAX_SHA_BLOCK_SIZE];
struct crypto_shash *hmac_tfm;
};
struct ccp_sha_req_ctx {
enum ccp_sha_type type;
u64 msg_bits;
unsigned int first;
unsigned int final;
struct scatterlist *src;
unsigned int nbytes;
u64 hash_cnt;
unsigned int hash_rem;
struct sg_table data_sg;
struct scatterlist ctx_sg;
u8 ctx[MAX_SHA_CONTEXT_SIZE];
struct scatterlist buf_sg;
unsigned int buf_count;
u8 buf[MAX_SHA_BLOCK_SIZE];
/* CCP driver command */
struct ccp_cmd cmd;
};
/***** Common Context Structure *****/
struct ccp_ctx {
int (*complete)(struct crypto_async_request *req, int ret);
union {
struct ccp_aes_ctx aes;
struct ccp_sha_ctx sha;
} u;
};
int ccp_crypto_enqueue_request(struct crypto_async_request *req,
struct ccp_cmd *cmd);
struct scatterlist *ccp_crypto_sg_table_add(struct sg_table *table,
struct scatterlist *sg_add);
int ccp_register_aes_algs(struct list_head *head);
int ccp_register_aes_cmac_algs(struct list_head *head);
int ccp_register_aes_xts_algs(struct list_head *head);
int ccp_register_sha_algs(struct list_head *head);
#endif

656
drivers/crypto/ccp/ccp-dev.c Normal file
View file

@ -0,0 +1,656 @@
/*
* AMD Cryptographic Coprocessor (CCP) driver
*
* Copyright (C) 2013 Advanced Micro Devices, Inc.
*
* Author: Tom Lendacky <thomas.lendacky@amd.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/kthread.h>
#include <linux/sched.h>
#include <linux/interrupt.h>
#include <linux/spinlock.h>
#include <linux/mutex.h>
#include <linux/delay.h>
#include <linux/hw_random.h>
#include <linux/cpu.h>
#ifdef CONFIG_X86
#include <asm/cpu_device_id.h>
#endif
#include <linux/ccp.h>
#include "ccp-dev.h"
MODULE_AUTHOR("Tom Lendacky <thomas.lendacky@amd.com>");
MODULE_LICENSE("GPL");
MODULE_VERSION("1.0.0");
MODULE_DESCRIPTION("AMD Cryptographic Coprocessor driver");
struct ccp_tasklet_data {
struct completion completion;
struct ccp_cmd *cmd;
};
static struct ccp_device *ccp_dev;
static inline struct ccp_device *ccp_get_device(void)
{
return ccp_dev;
}
static inline void ccp_add_device(struct ccp_device *ccp)
{
ccp_dev = ccp;
}
static inline void ccp_del_device(struct ccp_device *ccp)
{
ccp_dev = NULL;
}
/**
* ccp_present - check if a CCP device is present
*
* Returns zero if a CCP device is present, -ENODEV otherwise.
*/
int ccp_present(void)
{
if (ccp_get_device())
return 0;
return -ENODEV;
}
EXPORT_SYMBOL_GPL(ccp_present);
/**
* ccp_enqueue_cmd - queue an operation for processing by the CCP
*
* @cmd: ccp_cmd struct to be processed
*
* Queue a cmd to be processed by the CCP. If queueing the cmd
* would exceed the defined length of the cmd queue the cmd will
* only be queued if the CCP_CMD_MAY_BACKLOG flag is set and will
* result in a return code of -EBUSY.
*
* The callback routine specified in the ccp_cmd struct will be
* called to notify the caller of completion (if the cmd was not
* backlogged) or advancement out of the backlog. If the cmd has
* advanced out of the backlog the "err" value of the callback
* will be -EINPROGRESS. Any other "err" value during callback is
* the result of the operation.
*
* The cmd has been successfully queued if:
* the return code is -EINPROGRESS or
* the return code is -EBUSY and CCP_CMD_MAY_BACKLOG flag is set
*/
int ccp_enqueue_cmd(struct ccp_cmd *cmd)
{
struct ccp_device *ccp = ccp_get_device();
unsigned long flags;
unsigned int i;
int ret;
if (!ccp)
return -ENODEV;
/* Caller must supply a callback routine */
if (!cmd->callback)
return -EINVAL;
cmd->ccp = ccp;
spin_lock_irqsave(&ccp->cmd_lock, flags);
i = ccp->cmd_q_count;
if (ccp->cmd_count >= MAX_CMD_QLEN) {
ret = -EBUSY;
if (cmd->flags & CCP_CMD_MAY_BACKLOG)
list_add_tail(&cmd->entry, &ccp->backlog);
} else {
ret = -EINPROGRESS;
ccp->cmd_count++;
list_add_tail(&cmd->entry, &ccp->cmd);
/* Find an idle queue */
if (!ccp->suspending) {
for (i = 0; i < ccp->cmd_q_count; i++) {
if (ccp->cmd_q[i].active)
continue;
break;
}
}
}
spin_unlock_irqrestore(&ccp->cmd_lock, flags);
/* If we found an idle queue, wake it up */
if (i < ccp->cmd_q_count)
wake_up_process(ccp->cmd_q[i].kthread);
return ret;
}
EXPORT_SYMBOL_GPL(ccp_enqueue_cmd);
static void ccp_do_cmd_backlog(struct work_struct *work)
{
struct ccp_cmd *cmd = container_of(work, struct ccp_cmd, work);
struct ccp_device *ccp = cmd->ccp;
unsigned long flags;
unsigned int i;
cmd->callback(cmd->data, -EINPROGRESS);
spin_lock_irqsave(&ccp->cmd_lock, flags);
ccp->cmd_count++;
list_add_tail(&cmd->entry, &ccp->cmd);
/* Find an idle queue */
for (i = 0; i < ccp->cmd_q_count; i++) {
if (ccp->cmd_q[i].active)
continue;
break;
}
spin_unlock_irqrestore(&ccp->cmd_lock, flags);
/* If we found an idle queue, wake it up */
if (i < ccp->cmd_q_count)
wake_up_process(ccp->cmd_q[i].kthread);
}
static struct ccp_cmd *ccp_dequeue_cmd(struct ccp_cmd_queue *cmd_q)
{
struct ccp_device *ccp = cmd_q->ccp;
struct ccp_cmd *cmd = NULL;
struct ccp_cmd *backlog = NULL;
unsigned long flags;
spin_lock_irqsave(&ccp->cmd_lock, flags);
cmd_q->active = 0;
if (ccp->suspending) {
cmd_q->suspended = 1;
spin_unlock_irqrestore(&ccp->cmd_lock, flags);
wake_up_interruptible(&ccp->suspend_queue);
return NULL;
}
if (ccp->cmd_count) {
cmd_q->active = 1;
cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
list_del(&cmd->entry);
ccp->cmd_count--;
}
if (!list_empty(&ccp->backlog)) {
backlog = list_first_entry(&ccp->backlog, struct ccp_cmd,
entry);
list_del(&backlog->entry);
}
spin_unlock_irqrestore(&ccp->cmd_lock, flags);
if (backlog) {
INIT_WORK(&backlog->work, ccp_do_cmd_backlog);
schedule_work(&backlog->work);
}
return cmd;
}
static void ccp_do_cmd_complete(unsigned long data)
{
struct ccp_tasklet_data *tdata = (struct ccp_tasklet_data *)data;
struct ccp_cmd *cmd = tdata->cmd;
cmd->callback(cmd->data, cmd->ret);
complete(&tdata->completion);
}
static int ccp_cmd_queue_thread(void *data)
{
struct ccp_cmd_queue *cmd_q = (struct ccp_cmd_queue *)data;
struct ccp_cmd *cmd;
struct ccp_tasklet_data tdata;
struct tasklet_struct tasklet;
tasklet_init(&tasklet, ccp_do_cmd_complete, (unsigned long)&tdata);
set_current_state(TASK_INTERRUPTIBLE);
while (!kthread_should_stop()) {
schedule();
set_current_state(TASK_INTERRUPTIBLE);
cmd = ccp_dequeue_cmd(cmd_q);
if (!cmd)
continue;
__set_current_state(TASK_RUNNING);
/* Execute the command */
cmd->ret = ccp_run_cmd(cmd_q, cmd);
/* Schedule the completion callback */
tdata.cmd = cmd;
init_completion(&tdata.completion);
tasklet_schedule(&tasklet);
wait_for_completion(&tdata.completion);
}
__set_current_state(TASK_RUNNING);
return 0;
}
static int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait)
{
struct ccp_device *ccp = container_of(rng, struct ccp_device, hwrng);
u32 trng_value;
int len = min_t(int, sizeof(trng_value), max);
/*
* Locking is provided by the caller so we can update device
* hwrng-related fields safely
*/
trng_value = ioread32(ccp->io_regs + TRNG_OUT_REG);
if (!trng_value) {
/* Zero is returned if not data is available or if a
* bad-entropy error is present. Assume an error if
* we exceed TRNG_RETRIES reads of zero.
*/
if (ccp->hwrng_retries++ > TRNG_RETRIES)
return -EIO;
return 0;
}
/* Reset the counter and save the rng value */
ccp->hwrng_retries = 0;
memcpy(data, &trng_value, len);
return len;
}
/**
* ccp_alloc_struct - allocate and initialize the ccp_device struct
*
* @dev: device struct of the CCP
*/
struct ccp_device *ccp_alloc_struct(struct device *dev)
{
struct ccp_device *ccp;
ccp = kzalloc(sizeof(*ccp), GFP_KERNEL);
if (ccp == NULL) {
dev_err(dev, "unable to allocate device struct\n");
return NULL;
}
ccp->dev = dev;
INIT_LIST_HEAD(&ccp->cmd);
INIT_LIST_HEAD(&ccp->backlog);
spin_lock_init(&ccp->cmd_lock);
mutex_init(&ccp->req_mutex);
mutex_init(&ccp->ksb_mutex);
ccp->ksb_count = KSB_COUNT;
ccp->ksb_start = 0;
return ccp;
}
/**
* ccp_init - initialize the CCP device
*
* @ccp: ccp_device struct
*/
int ccp_init(struct ccp_device *ccp)
{
struct device *dev = ccp->dev;
struct ccp_cmd_queue *cmd_q;
struct dma_pool *dma_pool;
char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
unsigned int qmr, qim, i;
int ret;
/* Find available queues */
qim = 0;
qmr = ioread32(ccp->io_regs + Q_MASK_REG);
for (i = 0; i < MAX_HW_QUEUES; i++) {
if (!(qmr & (1 << i)))
continue;
/* Allocate a dma pool for this queue */
snprintf(dma_pool_name, sizeof(dma_pool_name), "ccp_q%d", i);
dma_pool = dma_pool_create(dma_pool_name, dev,
CCP_DMAPOOL_MAX_SIZE,
CCP_DMAPOOL_ALIGN, 0);
if (!dma_pool) {
dev_err(dev, "unable to allocate dma pool\n");
ret = -ENOMEM;
goto e_pool;
}
cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
ccp->cmd_q_count++;
cmd_q->ccp = ccp;
cmd_q->id = i;
cmd_q->dma_pool = dma_pool;
/* Reserve 2 KSB regions for the queue */
cmd_q->ksb_key = KSB_START + ccp->ksb_start++;
cmd_q->ksb_ctx = KSB_START + ccp->ksb_start++;
ccp->ksb_count -= 2;
/* Preset some register values and masks that are queue
* number dependent
*/
cmd_q->reg_status = ccp->io_regs + CMD_Q_STATUS_BASE +
(CMD_Q_STATUS_INCR * i);
cmd_q->reg_int_status = ccp->io_regs + CMD_Q_INT_STATUS_BASE +
(CMD_Q_STATUS_INCR * i);
cmd_q->int_ok = 1 << (i * 2);
cmd_q->int_err = 1 << ((i * 2) + 1);
cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status));
init_waitqueue_head(&cmd_q->int_queue);
/* Build queue interrupt mask (two interrupts per queue) */
qim |= cmd_q->int_ok | cmd_q->int_err;
#ifdef CONFIG_ARM64
/* For arm64 set the recommended queue cache settings */
iowrite32(ccp->axcache, ccp->io_regs + CMD_Q_CACHE_BASE +
(CMD_Q_CACHE_INC * i));
#endif
dev_dbg(dev, "queue #%u available\n", i);
}
if (ccp->cmd_q_count == 0) {
dev_notice(dev, "no command queues available\n");
ret = -EIO;
goto e_pool;
}
dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count);
/* Disable and clear interrupts until ready */
iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG);
for (i = 0; i < ccp->cmd_q_count; i++) {
cmd_q = &ccp->cmd_q[i];
ioread32(cmd_q->reg_int_status);
ioread32(cmd_q->reg_status);
}
iowrite32(qim, ccp->io_regs + IRQ_STATUS_REG);
/* Request an irq */
ret = ccp->get_irq(ccp);
if (ret) {
dev_err(dev, "unable to allocate an IRQ\n");
goto e_pool;
}
/* Initialize the queues used to wait for KSB space and suspend */
init_waitqueue_head(&ccp->ksb_queue);
init_waitqueue_head(&ccp->suspend_queue);
/* Create a kthread for each queue */
for (i = 0; i < ccp->cmd_q_count; i++) {
struct task_struct *kthread;
cmd_q = &ccp->cmd_q[i];
kthread = kthread_create(ccp_cmd_queue_thread, cmd_q,
"ccp-q%u", cmd_q->id);
if (IS_ERR(kthread)) {
dev_err(dev, "error creating queue thread (%ld)\n",
PTR_ERR(kthread));
ret = PTR_ERR(kthread);
goto e_kthread;
}
cmd_q->kthread = kthread;
wake_up_process(kthread);
}
/* Register the RNG */
ccp->hwrng.name = "ccp-rng";
ccp->hwrng.read = ccp_trng_read;
ret = hwrng_register(&ccp->hwrng);
if (ret) {
dev_err(dev, "error registering hwrng (%d)\n", ret);
goto e_kthread;
}
/* Make the device struct available before enabling interrupts */
ccp_add_device(ccp);
/* Enable interrupts */
iowrite32(qim, ccp->io_regs + IRQ_MASK_REG);
return 0;
e_kthread:
for (i = 0; i < ccp->cmd_q_count; i++)
if (ccp->cmd_q[i].kthread)
kthread_stop(ccp->cmd_q[i].kthread);
ccp->free_irq(ccp);
e_pool:
for (i = 0; i < ccp->cmd_q_count; i++)
dma_pool_destroy(ccp->cmd_q[i].dma_pool);
return ret;
}
/**
* ccp_destroy - tear down the CCP device
*
* @ccp: ccp_device struct
*/
void ccp_destroy(struct ccp_device *ccp)
{
struct ccp_cmd_queue *cmd_q;
struct ccp_cmd *cmd;
unsigned int qim, i;
/* Remove general access to the device struct */
ccp_del_device(ccp);
/* Unregister the RNG */
hwrng_unregister(&ccp->hwrng);
/* Stop the queue kthreads */
for (i = 0; i < ccp->cmd_q_count; i++)
if (ccp->cmd_q[i].kthread)
kthread_stop(ccp->cmd_q[i].kthread);
/* Build queue interrupt mask (two interrupt masks per queue) */
qim = 0;
for (i = 0; i < ccp->cmd_q_count; i++) {
cmd_q = &ccp->cmd_q[i];
qim |= cmd_q->int_ok | cmd_q->int_err;
}
/* Disable and clear interrupts */
iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG);
for (i = 0; i < ccp->cmd_q_count; i++) {
cmd_q = &ccp->cmd_q[i];
ioread32(cmd_q->reg_int_status);
ioread32(cmd_q->reg_status);
}
iowrite32(qim, ccp->io_regs + IRQ_STATUS_REG);
ccp->free_irq(ccp);
for (i = 0; i < ccp->cmd_q_count; i++)
dma_pool_destroy(ccp->cmd_q[i].dma_pool);
/* Flush the cmd and backlog queue */
while (!list_empty(&ccp->cmd)) {
/* Invoke the callback directly with an error code */
cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
list_del(&cmd->entry);
cmd->callback(cmd->data, -ENODEV);
}
while (!list_empty(&ccp->backlog)) {
/* Invoke the callback directly with an error code */
cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);
list_del(&cmd->entry);
cmd->callback(cmd->data, -ENODEV);
}
}
/**
* ccp_irq_handler - handle interrupts generated by the CCP device
*
* @irq: the irq associated with the interrupt
* @data: the data value supplied when the irq was created
*/
irqreturn_t ccp_irq_handler(int irq, void *data)
{
struct device *dev = data;
struct ccp_device *ccp = dev_get_drvdata(dev);
struct ccp_cmd_queue *cmd_q;
u32 q_int, status;
unsigned int i;
status = ioread32(ccp->io_regs + IRQ_STATUS_REG);
for (i = 0; i < ccp->cmd_q_count; i++) {
cmd_q = &ccp->cmd_q[i];
q_int = status & (cmd_q->int_ok | cmd_q->int_err);
if (q_int) {
cmd_q->int_status = status;
cmd_q->q_status = ioread32(cmd_q->reg_status);
cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);
/* On error, only save the first error value */
if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error)
cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
cmd_q->int_rcvd = 1;
/* Acknowledge the interrupt and wake the kthread */
iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG);
wake_up_interruptible(&cmd_q->int_queue);
}
}
return IRQ_HANDLED;
}
#ifdef CONFIG_PM
bool ccp_queues_suspended(struct ccp_device *ccp)
{
unsigned int suspended = 0;
unsigned long flags;
unsigned int i;
spin_lock_irqsave(&ccp->cmd_lock, flags);
for (i = 0; i < ccp->cmd_q_count; i++)
if (ccp->cmd_q[i].suspended)
suspended++;
spin_unlock_irqrestore(&ccp->cmd_lock, flags);
return ccp->cmd_q_count == suspended;
}
#endif
#ifdef CONFIG_X86
static const struct x86_cpu_id ccp_support[] = {
{ X86_VENDOR_AMD, 22, },
};
#endif
static int __init ccp_mod_init(void)
{
#ifdef CONFIG_X86
struct cpuinfo_x86 *cpuinfo = &boot_cpu_data;
int ret;
if (!x86_match_cpu(ccp_support))
return -ENODEV;
switch (cpuinfo->x86) {
case 22:
if ((cpuinfo->x86_model < 48) || (cpuinfo->x86_model > 63))
return -ENODEV;
ret = ccp_pci_init();
if (ret)
return ret;
/* Don't leave the driver loaded if init failed */
if (!ccp_get_device()) {
ccp_pci_exit();
return -ENODEV;
}
return 0;
break;
}
#endif
#ifdef CONFIG_ARM64
int ret;
ret = ccp_platform_init();
if (ret)
return ret;
/* Don't leave the driver loaded if init failed */
if (!ccp_get_device()) {
ccp_platform_exit();
return -ENODEV;
}
return 0;
#endif
return -ENODEV;
}
static void __exit ccp_mod_exit(void)
{
#ifdef CONFIG_X86
struct cpuinfo_x86 *cpuinfo = &boot_cpu_data;
switch (cpuinfo->x86) {
case 22:
ccp_pci_exit();
break;
}
#endif
#ifdef CONFIG_ARM64
ccp_platform_exit();
#endif
}
module_init(ccp_mod_init);
module_exit(ccp_mod_exit);

280
drivers/crypto/ccp/ccp-dev.h Normal file
View file

@ -0,0 +1,280 @@
/*
* AMD Cryptographic Coprocessor (CCP) driver
*
* Copyright (C) 2013 Advanced Micro Devices, Inc.
*
* Author: Tom Lendacky <thomas.lendacky@amd.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#ifndef __CCP_DEV_H__
#define __CCP_DEV_H__
#include <linux/device.h>
#include <linux/pci.h>
#include <linux/spinlock.h>
#include <linux/mutex.h>
#include <linux/list.h>
#include <linux/wait.h>
#include <linux/dmapool.h>
#include <linux/hw_random.h>
#define MAX_DMAPOOL_NAME_LEN 32
#define MAX_HW_QUEUES 5
#define MAX_CMD_QLEN 100
#define TRNG_RETRIES 10
#define CACHE_NONE 0x00
#define CACHE_WB_NO_ALLOC 0xb7
/****** Register Mappings ******/
#define Q_MASK_REG 0x000
#define TRNG_OUT_REG 0x00c
#define IRQ_MASK_REG 0x040
#define IRQ_STATUS_REG 0x200
#define DEL_CMD_Q_JOB 0x124
#define DEL_Q_ACTIVE 0x00000200
#define DEL_Q_ID_SHIFT 6
#define CMD_REQ0 0x180
#define CMD_REQ_INCR 0x04
#define CMD_Q_STATUS_BASE 0x210
#define CMD_Q_INT_STATUS_BASE 0x214
#define CMD_Q_STATUS_INCR 0x20
#define CMD_Q_CACHE_BASE 0x228
#define CMD_Q_CACHE_INC 0x20
#define CMD_Q_ERROR(__qs) ((__qs) & 0x0000003f);
#define CMD_Q_DEPTH(__qs) (((__qs) >> 12) & 0x0000000f);
/****** REQ0 Related Values ******/
#define REQ0_WAIT_FOR_WRITE 0x00000004
#define REQ0_INT_ON_COMPLETE 0x00000002
#define REQ0_STOP_ON_COMPLETE 0x00000001
#define REQ0_CMD_Q_SHIFT 9
#define REQ0_JOBID_SHIFT 3
/****** REQ1 Related Values ******/
#define REQ1_PROTECT_SHIFT 27
#define REQ1_ENGINE_SHIFT 23
#define REQ1_KEY_KSB_SHIFT 2
#define REQ1_EOM 0x00000002
#define REQ1_INIT 0x00000001
/* AES Related Values */
#define REQ1_AES_TYPE_SHIFT 21
#define REQ1_AES_MODE_SHIFT 18
#define REQ1_AES_ACTION_SHIFT 17
#define REQ1_AES_CFB_SIZE_SHIFT 10
/* XTS-AES Related Values */
#define REQ1_XTS_AES_SIZE_SHIFT 10
/* SHA Related Values */
#define REQ1_SHA_TYPE_SHIFT 21
/* RSA Related Values */
#define REQ1_RSA_MOD_SIZE_SHIFT 10
/* Pass-Through Related Values */
#define REQ1_PT_BW_SHIFT 12
#define REQ1_PT_BS_SHIFT 10
/* ECC Related Values */
#define REQ1_ECC_AFFINE_CONVERT 0x00200000
#define REQ1_ECC_FUNCTION_SHIFT 18
/****** REQ4 Related Values ******/
#define REQ4_KSB_SHIFT 18
#define REQ4_MEMTYPE_SHIFT 16
/****** REQ6 Related Values ******/
#define REQ6_MEMTYPE_SHIFT 16
/****** Key Storage Block ******/
#define KSB_START 77
#define KSB_END 127
#define KSB_COUNT (KSB_END - KSB_START + 1)
#define CCP_KSB_BITS 256
#define CCP_KSB_BYTES 32
#define CCP_JOBID_MASK 0x0000003f
#define CCP_DMAPOOL_MAX_SIZE 64
#define CCP_DMAPOOL_ALIGN (1 << 5)
#define CCP_REVERSE_BUF_SIZE 64
#define CCP_AES_KEY_KSB_COUNT 1
#define CCP_AES_CTX_KSB_COUNT 1
#define CCP_XTS_AES_KEY_KSB_COUNT 1
#define CCP_XTS_AES_CTX_KSB_COUNT 1
#define CCP_SHA_KSB_COUNT 1
#define CCP_RSA_MAX_WIDTH 4096
#define CCP_PASSTHRU_BLOCKSIZE 256
#define CCP_PASSTHRU_MASKSIZE 32
#define CCP_PASSTHRU_KSB_COUNT 1
#define CCP_ECC_MODULUS_BYTES 48 /* 384-bits */
#define CCP_ECC_MAX_OPERANDS 6
#define CCP_ECC_MAX_OUTPUTS 3
#define CCP_ECC_SRC_BUF_SIZE 448
#define CCP_ECC_DST_BUF_SIZE 192
#define CCP_ECC_OPERAND_SIZE 64
#define CCP_ECC_OUTPUT_SIZE 64
#define CCP_ECC_RESULT_OFFSET 60
#define CCP_ECC_RESULT_SUCCESS 0x0001
struct ccp_device;
struct ccp_cmd;
struct ccp_cmd_queue {
struct ccp_device *ccp;
/* Queue identifier */
u32 id;
/* Queue dma pool */
struct dma_pool *dma_pool;
/* Queue reserved KSB regions */
u32 ksb_key;
u32 ksb_ctx;
/* Queue processing thread */
struct task_struct *kthread;
unsigned int active;
unsigned int suspended;
/* Number of free command slots available */
unsigned int free_slots;
/* Interrupt masks */
u32 int_ok;
u32 int_err;
/* Register addresses for queue */
void __iomem *reg_status;
void __iomem *reg_int_status;
/* Status values from job */
u32 int_status;
u32 q_status;
u32 q_int_status;
u32 cmd_error;
/* Interrupt wait queue */
wait_queue_head_t int_queue;
unsigned int int_rcvd;
} ____cacheline_aligned;
struct ccp_device {
struct device *dev;
/*
* Bus specific device information
*/
void *dev_specific;
int (*get_irq)(struct ccp_device *ccp);
void (*free_irq)(struct ccp_device *ccp);
unsigned int irq;
/*
* I/O area used for device communication. The register mapping
* starts at an offset into the mapped bar.
* The CMD_REQx registers and the Delete_Cmd_Queue_Job register
* need to be protected while a command queue thread is accessing
* them.
*/
struct mutex req_mutex ____cacheline_aligned;
void __iomem *io_map;
void __iomem *io_regs;
/*
* Master lists that all cmds are queued on. Because there can be
* more than one CCP command queue that can process a cmd a separate
* backlog list is neeeded so that the backlog completion call
* completes before the cmd is available for execution.
*/
spinlock_t cmd_lock ____cacheline_aligned;
unsigned int cmd_count;
struct list_head cmd;
struct list_head backlog;
/*
* The command queues. These represent the queues available on the
* CCP that are available for processing cmds
*/
struct ccp_cmd_queue cmd_q[MAX_HW_QUEUES];
unsigned int cmd_q_count;
/*
* Support for the CCP True RNG
*/
struct hwrng hwrng;
unsigned int hwrng_retries;
/*
* A counter used to generate job-ids for cmds submitted to the CCP
*/
atomic_t current_id ____cacheline_aligned;
/*
* The CCP uses key storage blocks (KSB) to maintain context for certain
* operations. To prevent multiple cmds from using the same KSB range
* a command queue reserves a KSB range for the duration of the cmd.
* Each queue, will however, reserve 2 KSB blocks for operations that
* only require single KSB entries (eg. AES context/iv and key) in order
* to avoid allocation contention. This will reserve at most 10 KSB
* entries, leaving 40 KSB entries available for dynamic allocation.
*/
struct mutex ksb_mutex ____cacheline_aligned;
DECLARE_BITMAP(ksb, KSB_COUNT);
wait_queue_head_t ksb_queue;
unsigned int ksb_avail;
unsigned int ksb_count;
u32 ksb_start;
/* Suspend support */
unsigned int suspending;
wait_queue_head_t suspend_queue;
/* DMA caching attribute support */
unsigned int axcache;
};
int ccp_pci_init(void);
void ccp_pci_exit(void);
int ccp_platform_init(void);
void ccp_platform_exit(void);
struct ccp_device *ccp_alloc_struct(struct device *dev);
int ccp_init(struct ccp_device *ccp);
void ccp_destroy(struct ccp_device *ccp);
bool ccp_queues_suspended(struct ccp_device *ccp);
irqreturn_t ccp_irq_handler(int irq, void *data);
int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd);
#endif

2126
drivers/crypto/ccp/ccp-ops.c Normal file
View file

File diff suppressed because it is too large Load diff

349
drivers/crypto/ccp/ccp-pci.c Normal file
View file

@ -0,0 +1,349 @@
/*
* AMD Cryptographic Coprocessor (CCP) driver
*
* Copyright (C) 2013 Advanced Micro Devices, Inc.
*
* Author: Tom Lendacky <thomas.lendacky@amd.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/device.h>
#include <linux/pci.h>
#include <linux/pci_ids.h>
#include <linux/dma-mapping.h>
#include <linux/kthread.h>
#include <linux/sched.h>
#include <linux/interrupt.h>
#include <linux/spinlock.h>
#include <linux/delay.h>
#include <linux/ccp.h>
#include "ccp-dev.h"
#define IO_BAR 2
#define IO_OFFSET 0x20000
#define MSIX_VECTORS 2
struct ccp_msix {
u32 vector;
char name[16];
};
struct ccp_pci {
int msix_count;
struct ccp_msix msix[MSIX_VECTORS];
};
static int ccp_get_msix_irqs(struct ccp_device *ccp)
{
struct ccp_pci *ccp_pci = ccp->dev_specific;
struct device *dev = ccp->dev;
struct pci_dev *pdev = container_of(dev, struct pci_dev, dev);
struct msix_entry msix_entry[MSIX_VECTORS];
unsigned int name_len = sizeof(ccp_pci->msix[0].name) - 1;
int v, ret;
for (v = 0; v < ARRAY_SIZE(msix_entry); v++)
msix_entry[v].entry = v;
ret = pci_enable_msix_range(pdev, msix_entry, 1, v);
if (ret < 0)
return ret;
ccp_pci->msix_count = ret;
for (v = 0; v < ccp_pci->msix_count; v++) {
/* Set the interrupt names and request the irqs */
snprintf(ccp_pci->msix[v].name, name_len, "ccp-%u", v);
ccp_pci->msix[v].vector = msix_entry[v].vector;
ret = request_irq(ccp_pci->msix[v].vector, ccp_irq_handler,
0, ccp_pci->msix[v].name, dev);
if (ret) {
dev_notice(dev, "unable to allocate MSI-X IRQ (%d)\n",
ret);
goto e_irq;
}
}
return 0;
e_irq:
while (v--)
free_irq(ccp_pci->msix[v].vector, dev);
pci_disable_msix(pdev);
ccp_pci->msix_count = 0;
return ret;
}
static int ccp_get_msi_irq(struct ccp_device *ccp)
{
struct device *dev = ccp->dev;
struct pci_dev *pdev = container_of(dev, struct pci_dev, dev);
int ret;
ret = pci_enable_msi(pdev);
if (ret)
return ret;
ccp->irq = pdev->irq;
ret = request_irq(ccp->irq, ccp_irq_handler, 0, "ccp", dev);
if (ret) {
dev_notice(dev, "unable to allocate MSI IRQ (%d)\n", ret);
goto e_msi;
}
return 0;
e_msi:
pci_disable_msi(pdev);
return ret;
}
static int ccp_get_irqs(struct ccp_device *ccp)
{
struct device *dev = ccp->dev;
int ret;
ret = ccp_get_msix_irqs(ccp);
if (!ret)
return 0;
/* Couldn't get MSI-X vectors, try MSI */
dev_notice(dev, "could not enable MSI-X (%d), trying MSI\n", ret);
ret = ccp_get_msi_irq(ccp);
if (!ret)
return 0;
/* Couldn't get MSI interrupt */
dev_notice(dev, "could not enable MSI (%d)\n", ret);
return ret;
}
static void ccp_free_irqs(struct ccp_device *ccp)
{
struct ccp_pci *ccp_pci = ccp->dev_specific;
struct device *dev = ccp->dev;
struct pci_dev *pdev = container_of(dev, struct pci_dev, dev);
if (ccp_pci->msix_count) {
while (ccp_pci->msix_count--)
free_irq(ccp_pci->msix[ccp_pci->msix_count].vector,
dev);
pci_disable_msix(pdev);
} else {
free_irq(ccp->irq, dev);
pci_disable_msi(pdev);
}
}
static int ccp_find_mmio_area(struct ccp_device *ccp)
{
struct device *dev = ccp->dev;
struct pci_dev *pdev = container_of(dev, struct pci_dev, dev);
resource_size_t io_len;
unsigned long io_flags;
io_flags = pci_resource_flags(pdev, IO_BAR);
io_len = pci_resource_len(pdev, IO_BAR);
if ((io_flags & IORESOURCE_MEM) && (io_len >= (IO_OFFSET + 0x800)))
return IO_BAR;
return -EIO;
}
static int ccp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
struct ccp_device *ccp;
struct ccp_pci *ccp_pci;
struct device *dev = &pdev->dev;
unsigned int bar;
int ret;
ret = -ENOMEM;
ccp = ccp_alloc_struct(dev);
if (!ccp)
goto e_err;
ccp_pci = kzalloc(sizeof(*ccp_pci), GFP_KERNEL);
if (!ccp_pci) {
ret = -ENOMEM;
goto e_free1;
}
ccp->dev_specific = ccp_pci;
ccp->get_irq = ccp_get_irqs;
ccp->free_irq = ccp_free_irqs;
ret = pci_request_regions(pdev, "ccp");
if (ret) {
dev_err(dev, "pci_request_regions failed (%d)\n", ret);
goto e_free2;
}
ret = pci_enable_device(pdev);
if (ret) {
dev_err(dev, "pci_enable_device failed (%d)\n", ret);
goto e_regions;
}
pci_set_master(pdev);
ret = ccp_find_mmio_area(ccp);
if (ret < 0)
goto e_device;
bar = ret;
ret = -EIO;
ccp->io_map = pci_iomap(pdev, bar, 0);
if (ccp->io_map == NULL) {
dev_err(dev, "pci_iomap failed\n");
goto e_device;
}
ccp->io_regs = ccp->io_map + IO_OFFSET;
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
if (ret) {
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
if (ret) {
dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n",
ret);
goto e_iomap;
}
}
dev_set_drvdata(dev, ccp);
ret = ccp_init(ccp);
if (ret)
goto e_iomap;
dev_notice(dev, "enabled\n");
return 0;
e_iomap:
pci_iounmap(pdev, ccp->io_map);
e_device:
pci_disable_device(pdev);
e_regions:
pci_release_regions(pdev);
e_free2:
kfree(ccp_pci);
e_free1:
kfree(ccp);
e_err:
dev_notice(dev, "initialization failed\n");
return ret;
}
static void ccp_pci_remove(struct pci_dev *pdev)
{
struct device *dev = &pdev->dev;
struct ccp_device *ccp = dev_get_drvdata(dev);
if (!ccp)
return;
ccp_destroy(ccp);
pci_iounmap(pdev, ccp->io_map);
pci_disable_device(pdev);
pci_release_regions(pdev);
kfree(ccp);
dev_notice(dev, "disabled\n");
}
#ifdef CONFIG_PM
static int ccp_pci_suspend(struct pci_dev *pdev, pm_message_t state)
{
struct device *dev = &pdev->dev;
struct ccp_device *ccp = dev_get_drvdata(dev);
unsigned long flags;
unsigned int i;
spin_lock_irqsave(&ccp->cmd_lock, flags);
ccp->suspending = 1;
/* Wake all the queue kthreads to prepare for suspend */
for (i = 0; i < ccp->cmd_q_count; i++)
wake_up_process(ccp->cmd_q[i].kthread);
spin_unlock_irqrestore(&ccp->cmd_lock, flags);
/* Wait for all queue kthreads to say they're done */
while (!ccp_queues_suspended(ccp))
wait_event_interruptible(ccp->suspend_queue,
ccp_queues_suspended(ccp));
return 0;
}
static int ccp_pci_resume(struct pci_dev *pdev)
{
struct device *dev = &pdev->dev;
struct ccp_device *ccp = dev_get_drvdata(dev);
unsigned long flags;
unsigned int i;
spin_lock_irqsave(&ccp->cmd_lock, flags);
ccp->suspending = 0;
/* Wake up all the kthreads */
for (i = 0; i < ccp->cmd_q_count; i++) {
ccp->cmd_q[i].suspended = 0;
wake_up_process(ccp->cmd_q[i].kthread);
}
spin_unlock_irqrestore(&ccp->cmd_lock, flags);
return 0;
}
#endif
static const struct pci_device_id ccp_pci_table[] = {
{ PCI_VDEVICE(AMD, 0x1537), },
/* Last entry must be zero */
{ 0, }
};
MODULE_DEVICE_TABLE(pci, ccp_pci_table);
static struct pci_driver ccp_pci_driver = {
.name = "AMD Cryptographic Coprocessor",
.id_table = ccp_pci_table,
.probe = ccp_pci_probe,
.remove = ccp_pci_remove,
#ifdef CONFIG_PM
.suspend = ccp_pci_suspend,
.resume = ccp_pci_resume,
#endif
};
int ccp_pci_init(void)
{
return pci_register_driver(&ccp_pci_driver);
}
void ccp_pci_exit(void)
{
pci_unregister_driver(&ccp_pci_driver);
}

230
drivers/crypto/ccp/ccp-platform.c Normal file
View file

@ -0,0 +1,230 @@
/*
* AMD Cryptographic Coprocessor (CCP) driver
*
* Copyright (C) 2014 Advanced Micro Devices, Inc.
*
* Author: Tom Lendacky <thomas.lendacky@amd.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/device.h>
#include <linux/platform_device.h>
#include <linux/ioport.h>
#include <linux/dma-mapping.h>
#include <linux/kthread.h>
#include <linux/sched.h>
#include <linux/interrupt.h>
#include <linux/spinlock.h>
#include <linux/delay.h>
#include <linux/ccp.h>
#include <linux/of.h>
#include "ccp-dev.h"
static int ccp_get_irq(struct ccp_device *ccp)
{
struct device *dev = ccp->dev;
struct platform_device *pdev = container_of(dev,
struct platform_device, dev);
int ret;
ret = platform_get_irq(pdev, 0);
if (ret < 0)
return ret;
ccp->irq = ret;
ret = request_irq(ccp->irq, ccp_irq_handler, 0, "ccp", dev);
if (ret) {
dev_notice(dev, "unable to allocate IRQ (%d)\n", ret);
return ret;
}
return 0;
}
static int ccp_get_irqs(struct ccp_device *ccp)
{
struct device *dev = ccp->dev;
int ret;
ret = ccp_get_irq(ccp);
if (!ret)
return 0;
/* Couldn't get an interrupt */
dev_notice(dev, "could not enable interrupts (%d)\n", ret);
return ret;
}
static void ccp_free_irqs(struct ccp_device *ccp)
{
struct device *dev = ccp->dev;
free_irq(ccp->irq, dev);
}
static struct resource *ccp_find_mmio_area(struct ccp_device *ccp)
{
struct device *dev = ccp->dev;
struct platform_device *pdev = container_of(dev,
struct platform_device, dev);
struct resource *ior;
ior = platform_get_resource(pdev, IORESOURCE_MEM, 0);
if (ior && (resource_size(ior) >= 0x800))
return ior;
return NULL;
}
static int ccp_platform_probe(struct platform_device *pdev)
{
struct ccp_device *ccp;
struct device *dev = &pdev->dev;
struct resource *ior;
int ret;
ret = -ENOMEM;
ccp = ccp_alloc_struct(dev);
if (!ccp)
goto e_err;
ccp->dev_specific = NULL;
ccp->get_irq = ccp_get_irqs;
ccp->free_irq = ccp_free_irqs;
ior = ccp_find_mmio_area(ccp);
ccp->io_map = devm_ioremap_resource(dev, ior);
if (IS_ERR(ccp->io_map)) {
ret = PTR_ERR(ccp->io_map);
goto e_free;
}
ccp->io_regs = ccp->io_map;
if (!dev->dma_mask)
dev->dma_mask = &dev->coherent_dma_mask;
*(dev->dma_mask) = DMA_BIT_MASK(48);
dev->coherent_dma_mask = DMA_BIT_MASK(48);
if (of_property_read_bool(dev->of_node, "dma-coherent"))
ccp->axcache = CACHE_WB_NO_ALLOC;
else
ccp->axcache = CACHE_NONE;
dev_set_drvdata(dev, ccp);
ret = ccp_init(ccp);
if (ret)
goto e_free;
dev_notice(dev, "enabled\n");
return 0;
e_free:
kfree(ccp);
e_err:
dev_notice(dev, "initialization failed\n");
return ret;
}
static int ccp_platform_remove(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct ccp_device *ccp = dev_get_drvdata(dev);
ccp_destroy(ccp);
kfree(ccp);
dev_notice(dev, "disabled\n");
return 0;
}
#ifdef CONFIG_PM
static int ccp_platform_suspend(struct platform_device *pdev,
pm_message_t state)
{
struct device *dev = &pdev->dev;
struct ccp_device *ccp = dev_get_drvdata(dev);
unsigned long flags;
unsigned int i;
spin_lock_irqsave(&ccp->cmd_lock, flags);
ccp->suspending = 1;
/* Wake all the queue kthreads to prepare for suspend */
for (i = 0; i < ccp->cmd_q_count; i++)
wake_up_process(ccp->cmd_q[i].kthread);
spin_unlock_irqrestore(&ccp->cmd_lock, flags);
/* Wait for all queue kthreads to say they're done */
while (!ccp_queues_suspended(ccp))
wait_event_interruptible(ccp->suspend_queue,
ccp_queues_suspended(ccp));
return 0;
}
static int ccp_platform_resume(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct ccp_device *ccp = dev_get_drvdata(dev);
unsigned long flags;
unsigned int i;
spin_lock_irqsave(&ccp->cmd_lock, flags);
ccp->suspending = 0;
/* Wake up all the kthreads */
for (i = 0; i < ccp->cmd_q_count; i++) {
ccp->cmd_q[i].suspended = 0;
wake_up_process(ccp->cmd_q[i].kthread);
}
spin_unlock_irqrestore(&ccp->cmd_lock, flags);
return 0;
}
#endif
static const struct of_device_id ccp_platform_ids[] = {
{ .compatible = "amd,ccp-seattle-v1a" },
{ },
};
static struct platform_driver ccp_platform_driver = {
.driver = {
.name = "AMD Cryptographic Coprocessor",
.owner = THIS_MODULE,
.of_match_table = ccp_platform_ids,
},
.probe = ccp_platform_probe,
.remove = ccp_platform_remove,
#ifdef CONFIG_PM
.suspend = ccp_platform_suspend,
.resume = ccp_platform_resume,
#endif
};
int ccp_platform_init(void)
{
return platform_driver_register(&ccp_platform_driver);
}
void ccp_platform_exit(void)
{
platform_driver_unregister(&ccp_platform_driver);
}

48
drivers/crypto/fmp/Kconfig Normal file
View file

@ -0,0 +1,48 @@
config FMP_UFS
tristate "Samsung EXYNOS FMP Driver for UFS"
depends on SCSI_UFSHCD && SCSI_UFSHCD_PLATFORM
---help---
This selects EXYNOS FMP UFS driver.
If you have a controller with this interface, say Y here.
If unsure, say N.
config FMP_MMC
tristate "Samsung EXYNOS FMP Driver for MMC"
depends on MMC_DW
---help---
This selects EXYNOS FMP MMC driver.
If you have a controller with this interface, say Y here.
If unsure, say N.
config FIPS_FMP
tristate "Samsung EXYNOS FMP Validation Driver for FIPS"
depends on FMP_UFS
---help---
This selects Samsung EXYNOS FMP Validation driver for FIPS.
If you have a controller with this interface, say Y here.
If unsure, say N.
choice
prompt "Option for self-test failure"
depends on FIPS_FMP
default NODE_FOR_SELFTEST_FAIL
config NODE_FOR_SELFTEST_FAIL
bool "Set fips fmp node when self-test fails"
depends on FIPS_FMP
help
This select that fips fmp node was set to zero when FMP self-test fails.
config PANIC_FOR_SELFTEST_FAIL
bool "Panic when self-test fails"
depends on FIPS_FMP
help
This select that kernel panic occurs when FMP self-test fails.
endchoice

14
drivers/crypto/fmp/Makefile Normal file
View file

@ -0,0 +1,14 @@
# Exynos FMP makefile
obj-$(CONFIG_FIPS_FMP) += first_file.o
obj-$(CONFIG_FMP_UFS) += fmp_ufs.o
obj-$(CONFIG_FIPS_FMP) += fmpdev.o fmp_integrity.o fmplib.o
obj-$(CONFIG_FIPS_FMP) += hmac_fmp.o sha256_fmp.o
obj-$(CONFIG_FIPS_FMP) += fmp_ufs_fips.o
obj-$(CONFIG_UFS_FMP_ECRYPT_FS) += fmp_derive_iv.o
obj-$(CONFIG_MMC_DW_FMP_ECRYPT_FS) += fmp_derive_iv.o
obj-$(CONFIG_FIPS_FMP) += last_file.o
obj-$(CONFIG_FMP_MMC) += fmp_mmc.o
ccflags-$(CONFIG_FMP_UFS) := -Idrivers/scsi/ufs
ccflags-$(CONFIG_FMP_MMC) := -Idrivers/mmc/host

41
drivers/crypto/fmp/first_file.c Executable file
View file

@ -0,0 +1,41 @@
/*
* First file for Exynos FMP FIPS integrity check
*
* Copyright (C) 2015 Samsung Electronics Co., Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*/
#include <linux/init.h>
/* Keep this on top */
static const char
builtime_fmp_hmac[] = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f};
const int first_fmp_rodata = 10;
int first_fmp_data = 20;
void first_fmp_text(void) __attribute__((unused));
void first_fmp_text(void)
{
}
const char *get_builtime_fmp_hmac(void)
{
return builtime_fmp_hmac;
}
void __init first_fmp_init(void) __attribute__((unused));
void __init first_fmp_init(void)
{
}
void __exit first_fmp_exit(void) __attribute__((unused));
void __exit first_fmp_exit(void)
{
}

148
drivers/crypto/fmp/fmp_derive_iv.c Normal file
View file

@ -0,0 +1,148 @@
/*
* Exynos FMP derive iv for eCryptfs
*
* Copyright (C) 2014 Samsung Electronics Co., Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*/
#include <linux/crypto.h>
#include <linux/scatterlist.h>
#include <linux/errno.h>
#include <linux/err.h>
#include <linux/fs.h>
#define FMP_MAX_IV_BYTES 16
#define FMP_MAX_OFFSET_BYTES 16
#define DEFAULT_HASH "md5"
#define SHA256_HASH "sha256-fmp"
static DEFINE_SPINLOCK(fmp_tfm_lock);
int calculate_sha256(struct crypto_hash *hash_tfm, char *dst, char *src, int len)
{
int ret = -1;
unsigned long flag;
struct scatterlist sg;
struct hash_desc desc = {
.tfm = hash_tfm,
.flags = 0
};
spin_lock_irqsave(&fmp_tfm_lock, flag);
sg_init_one(&sg, (u8 *)src, len);
if (!desc.tfm) {
desc.tfm = crypto_alloc_hash(SHA256_HASH, 0,
CRYPTO_ALG_ASYNC);
if (IS_ERR(desc.tfm)) {
printk(KERN_ERR "%s: Error attemping to allocate crypto context\n", __func__);
goto out;
}
hash_tfm = desc.tfm;
}
ret = crypto_hash_init(&desc);
if (ret) {
printk(KERN_ERR
"%s: Error initializing crypto hash; ret = [%d]\n",
__func__, ret);
goto out;
}
ret = crypto_hash_update(&desc, &sg, len);
if (ret) {
printk(KERN_ERR
"%s: Error updating crypto hash; ret = [%d]\n",
__func__, ret);
goto out;
}
ret = crypto_hash_final(&desc, dst);
if (ret) {
printk(KERN_ERR
"%s: Error finalizing crypto hash; ret = [%d]\n",
__func__, ret);
goto out;
}
out:
spin_unlock_irqrestore(&fmp_tfm_lock, flag);
return ret;
}
int calculate_md5(struct crypto_hash *hash_tfm, char *dst, char *src, int len)
{
int ret = -1;
unsigned long flag;
struct scatterlist sg;
struct hash_desc desc = {
.tfm = hash_tfm,
.flags = 0
};
spin_lock_irqsave(&fmp_tfm_lock, flag);
sg_init_one(&sg, (u8 *)src, len);
if (!desc.tfm) {
desc.tfm = crypto_alloc_hash(DEFAULT_HASH, 0,
CRYPTO_ALG_ASYNC);
if (IS_ERR(desc.tfm)) {
printk(KERN_ERR "%s: Error attemping to allocate crypto context\n", __func__);
goto out;
}
hash_tfm = desc.tfm;
}
ret = crypto_hash_init(&desc);
if (ret) {
printk(KERN_ERR
"%s: Error initializing crypto hash; ret = [%d]\n",
__func__, ret);
goto out;
}
ret = crypto_hash_update(&desc, &sg, len);
if (ret) {
printk(KERN_ERR
"%s: Error updating crypto hash; ret = [%d]\n",
__func__, ret);
goto out;
}
ret = crypto_hash_final(&desc, dst);
if (ret) {
printk(KERN_ERR
"%s: Error finalizing crypto hash; ret = [%d]\n",
__func__, ret);
goto out;
}
out:
spin_unlock_irqrestore(&fmp_tfm_lock, flag);
return ret;
}
int file_enc_derive_iv(struct address_space *mapping, loff_t offset, char *extent_iv)
{
char src[FMP_MAX_IV_BYTES + FMP_MAX_OFFSET_BYTES];
int ret;
memcpy(src, mapping->iv, FMP_MAX_IV_BYTES);
memset(src + FMP_MAX_IV_BYTES, 0, FMP_MAX_OFFSET_BYTES);
snprintf(src + FMP_MAX_IV_BYTES, FMP_MAX_OFFSET_BYTES, "%lld", offset);
#ifdef CONFIG_CRYPTO_FIPS
if (mapping->cc_enable)
ret = calculate_sha256(mapping->hash_tfm, extent_iv, src, FMP_MAX_IV_BYTES + FMP_MAX_OFFSET_BYTES);
else
#endif
ret = calculate_md5(mapping->hash_tfm, extent_iv, src, FMP_MAX_IV_BYTES + FMP_MAX_OFFSET_BYTES);
if (ret) {
printk(KERN_ERR "%s: Error attempting to compute generating IV for a page; ret = [%d]\n", __func__, ret);
goto out;
}
out:
return ret;
}

22
drivers/crypto/fmp/fmp_derive_iv.h Normal file
View file

@ -0,0 +1,22 @@
/*
* Exynos FMP derive iv for eCryptfs
*
* Copyright (C) 2015 Samsung Electronics Co., Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*/
#ifndef _FMP_DERIVE_IV_H_
#define _FMP_DERIVE_IV_H_
#define SHA256_HASH_SIZE 32
#define MD5_DIGEST_SIZE 16
int calculate_sha256(struct crypto_hash *hash_tfm, char *dst, char *src, int len);
int calculate_md5(struct crypto_hash *hash_tfm, char *dst, char *src, int len);
int file_enc_derive_iv(struct address_space *mapping, loff_t offset, char *extent_iv);
#endif

283
drivers/crypto/fmp/fmp_integrity.c Normal file
View file

@ -0,0 +1,283 @@
/*
* Perform FIPS Integrity test on Kernel Crypto API
*
* At build time, hmac(sha256) of crypto code, avaiable in different ELF sections
* of vmlinux file, is generated. vmlinux file is updated with built-time hmac
* in a read-only data variable, so that it is available at run-time
*
* At run time, hmac(sha256) is again calculated using crypto bytes of a running
* At run time, hmac-fmp(sha256-fmp) is again calculated using crypto bytes of a running
* kernel.
* Run time hmac is compared to built time hmac to verify the integrity.
*
*
* Author : Rohit Kothari (r.kothari@samsung.com)
* Date : 11 Feb 2014
*
* Copyright (c) 2014 Samsung Electronics
*
*/
#include <linux/crypto.h>
#include <linux/kallsyms.h>
#include <linux/err.h>
#include <linux/scatterlist.h>
#include <linux/smc.h>
#include "fmpdev_int.h" //for FIPS_FMP_FUNC_TEST macro
//#define FIPS_DEBUG
static const char *symtab[][3] =
{{".text", "first_fmp_text", "last_fmp_text" },
{".rodata", "first_fmp_rodata", "last_fmp_rodata"},
{".init.text", "first_fmp_init", "last_fmp_init" }};
extern const char *get_builtime_fmp_hmac(void);
#ifdef FIPS_DEBUG
static int
dump_bytes(const char *section_name, const char *first_symbol, const char *last_symbol)
{
u8 *start_addr = (u8 *)kallsyms_lookup_name(first_symbol);
u8 *end_addr = (u8 *)kallsyms_lookup_name(last_symbol);
if (!start_addr || !end_addr || start_addr >= end_addr) {
printk(KERN_ERR "FIPS(%s): Error Invalid Addresses in Section : %s, Start_Addr : %p , End_Addr : %p",
__FUNCTION__, section_name, start_addr, end_addr);
return -1;
}
printk(KERN_INFO "FIPS CRYPTO RUNTIME : Section - %s, %s : %p, %s : %p \n", section_name, first_symbol, start_addr, last_symbol, end_addr);
print_hex_dump_bytes("FIPS CRYPTO RUNTIME : ", DUMP_PREFIX_NONE, start_addr, end_addr - start_addr);
return 0;
}
#endif
static int query_symbol_addresses(const char *first_symbol, const char *last_symbol,
unsigned long *start_addr,unsigned long *end_addr)
{
unsigned long start = kallsyms_lookup_name(first_symbol);
unsigned long end = kallsyms_lookup_name(last_symbol);
#ifdef FIPS_DEBUG
printk(KERN_INFO "FIPS CRYPTO RUNTIME : %s : %p, %s : %p\n", first_symbol, (u8*)start, last_symbol, (u8*)end);
#endif
if (!start || !end || start >= end) {
printk(KERN_ERR "FIPS(%s): Error Invalid Addresses.", __FUNCTION__);
return -1;
}
*start_addr = start;
*end_addr = end;
return 0;
}
static int init_hash(struct hash_desc *desc)
{
struct crypto_hash *tfm = NULL;
int ret = -1;
/* Same as build time */
const unsigned char *key = "The quick brown fox jumps over the lazy dog";
tfm = crypto_alloc_hash("hmac-fmp(sha256-fmp)", 0, 0);
if (IS_ERR(tfm)) {
printk(KERN_ERR "FIPS(%s): integrity failed to allocate tfm %ld", __FUNCTION__, PTR_ERR(tfm));
return -1;
}
ret = crypto_hash_setkey (tfm, key, strlen(key));
if (ret) {
printk(KERN_ERR "FIPS(%s): fail at crypto_hash_setkey", __FUNCTION__);
return -1;
}
desc->tfm = tfm;
desc->flags = 0;
ret = crypto_hash_init(desc);
if (ret) {
printk(KERN_ERR "FIPS(%s): fail at crypto_hash_init", __FUNCTION__);
return -1;
}
return 0;
}
static int finalize_hash(struct hash_desc *desc, unsigned char *out, unsigned int out_size)
{
int ret = -1;
if (!desc || !desc->tfm || !out || !out_size) {
printk(KERN_ERR "FIPS(%s): Invalid args", __FUNCTION__);
return ret;
}
if (crypto_hash_digestsize(desc->tfm) > out_size) {
printk(KERN_ERR "FIPS(%s): Not enough space for digest", __FUNCTION__);
return ret;
}
ret = crypto_hash_final(desc, out);
if (ret) {
printk(KERN_ERR "FIPS(%s): crypto_hash_final failed", __FUNCTION__);
return -1;
}
return 0;
}
static int update_hash(struct hash_desc *desc, unsigned char *start_addr, unsigned int size)
{
struct scatterlist sg;
unsigned char *buf = NULL;
unsigned char *cur = NULL;
unsigned int bytes_remaining;
unsigned int bytes;
int ret = -1;
#if FIPS_FMP_FUNC_TEST == 5
static int total = 0;
#endif
buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
if (!buf) {
printk(KERN_ERR "FIPS(%s): kmalloc failed", __FUNCTION__);
return ret;
}
bytes_remaining = size;
cur = start_addr;
while (bytes_remaining > 0) {
if (bytes_remaining >= PAGE_SIZE)
bytes = PAGE_SIZE;
else
bytes = bytes_remaining;
memcpy(buf, cur, bytes);
sg_init_one(&sg, buf, bytes);
#if FIPS_FMP_FUNC_TEST == 5
if (total == 0) {
printk(KERN_INFO "FIPS : Failing Integrity Test");
buf[bytes / 2] += 1;
}
#endif
ret = crypto_hash_update(desc, &sg, bytes);
if (ret) {
printk(KERN_ERR "FIPS(%s): crypto_hash_update failed", __FUNCTION__);
kfree(buf);
buf = 0;
return -1;
}
cur += bytes;
bytes_remaining -= bytes;
#if FIPS_FMP_FUNC_TEST == 5
total += bytes;
#endif
}
if (buf) {
kfree(buf);
buf = 0;
}
return 0;
}
int do_fmp_fw_integrity_check(void)
{
int err = 0;
err = exynos_smc(SMC_CMD_FMP, FMP_FW_INTEGRITY, 0, 0);
if (err) {
printk(KERN_ERR "Fail to check integrity for FMP F/W. err = 0x%x\n", err);
return -1;
}
return 0;
}
int do_fips_fmp_integrity_check(void)
{
int i, rows, err;
unsigned long start_addr = 0;
unsigned long end_addr = 0;
unsigned char runtime_hmac[32];
struct hash_desc desc;
const char *builtime_hmac = 0;
unsigned int size = 0;
err = do_fmp_fw_integrity_check();
if (err) {
printk(KERN_ERR "FIPS(%s): FMP FW Integrity Check Failed", __FUNCTION__);
return -1;
}
err = init_hash(&desc);
if (err) {
printk(KERN_ERR "FIPS(%s): init_hash failed", __FUNCTION__);
return -1;
}
rows = (unsigned int)sizeof(symtab) / sizeof(symtab[0]);
for (i = 0; i < rows; i++) {
err = query_symbol_addresses(symtab[i][1], symtab[i][2], &start_addr, &end_addr);
if (err) {
printk (KERN_ERR "FIPS(%s): Error to get start / end addresses", __FUNCTION__);
crypto_free_hash(desc.tfm);
return -1;
}
#ifdef FIPS_DEBUG
dump_bytes(symtab[i][0], symtab[i][1], symtab[i][2]);
#endif
size = end_addr - start_addr;
err = update_hash(&desc, (unsigned char *)start_addr, size);
if (err) {
printk(KERN_ERR "FIPS(%s): Error to update hash", __FUNCTION__);
crypto_free_hash(desc.tfm);
return -1;
}
}
err = finalize_hash(&desc, runtime_hmac, sizeof(runtime_hmac));
if (err) {
printk(KERN_ERR "FIPS(%s): Error in finalize", __FUNCTION__);
crypto_free_hash(desc.tfm);
return -1;
}
crypto_free_hash(desc.tfm);
builtime_hmac = get_builtime_fmp_hmac();
if (!builtime_hmac) {
printk(KERN_ERR "FIPS(%s): Unable to retrieve builtime_hmac", __FUNCTION__);
return -1;
}
#ifdef FIPS_DEBUG
print_hex_dump_bytes("FIPS CRYPTO RUNTIME : runtime hmac = ", DUMP_PREFIX_NONE, runtime_hmac, sizeof(runtime_hmac));
print_hex_dump_bytes("FIPS CRYPTO RUNTIME : builtime_hmac = ", DUMP_PREFIX_NONE, builtime_hmac , sizeof(runtime_hmac));
#endif
if (!memcmp(builtime_hmac, runtime_hmac, sizeof(runtime_hmac))) {
printk(KERN_INFO "FIPS: Integrity Check Passed");
return 0;
} else {
printk(KERN_ERR "FIPS(%s): Integrity Check Failed", __FUNCTION__);
return -1;
}
return -1;
}
EXPORT_SYMBOL_GPL(do_fips_fmp_integrity_check);

17
drivers/crypto/fmp/fmp_integrity.h Normal file
View file

@ -0,0 +1,17 @@
/*
* Exynos FMP integrity header
*
* Copyright (C) 2015 Samsung Electronics Co., Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*/
#ifndef __FMP_INTEGRITY_H__
#define __FMP_INTEGRITY_H__
int do_fips_fmp_integrity_check(void);
#endif

127
drivers/crypto/fmp/fmp_mmc.c Normal file
View file

@ -0,0 +1,127 @@
/*
* Exynos FMP MMC driver
*
* Copyright (C) 2015 Samsung Electronics Co., Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*/
#include <linux/pagemap.h>
#include <linux/mmc/dw_mmc.h>
#include <linux/smc.h>
#include "dw_mmc-exynos.h"
#if defined(CONFIG_MMC_DW_FMP_ECRYPT_FS)
#include "fmp_derive_iv.h"
#endif
extern volatile unsigned int disk_key_flag;
extern spinlock_t disk_key_lock;
#define byte2word(b0, b1, b2, b3) \
((unsigned int)(b0) << 24) | ((unsigned int)(b1) << 16) | ((unsigned int)(b2) << 8) | (b3)
#define word_in(x, c) byte2word(((unsigned char *)(x) + 4 * (c))[0], ((unsigned char *)(x) + 4 * (c))[1], \
((unsigned char *)(x) + 4 * (c))[2], ((unsigned char *)(x) + 4 * (c))[3])
int fmp_map_sg(struct dw_mci *host, struct idmac_desc_64addr *desc, int idx,
uint32_t sector_key, uint32_t sector, struct mmc_data *data)
{
#if defined(CONFIG_MMC_DW_FMP_DM_CRYPT)
if ((sector_key & DW_MMC_ENCRYPTION_SECTOR_BEGIN) &&
(host->pdata->quirks & DW_MCI_QUIRK_USE_SMU)) { /* disk encryption */
int ret;
/* disk algorithm selector */
IDMAC_SET_DAS(desc, AES_XTS);
desc->des2 |= IDMAC_DES2_DKL;
/* Disk IV */
desc->des28 = 0;
desc->des29 = 0;
desc->des30 = 0;
desc->des31 = htonl(sector);
/* Disk Enc Key, Tweak Key */
if (disk_key_flag) {
/* Disk Enc Key, Tweak Key */
ret = exynos_smc(SMC_CMD_FMP, FMP_KEY_SET, EMMC0_FMP, 0);
if (ret) {
printk(KERN_ERR "Failed to smc call for FMP key setting: %x\n", ret);
return ret;
}
spin_lock(&disk_key_lock);
disk_key_flag = 0;
spin_unlock(&disk_key_lock);
}
}
#endif
#if defined(CONFIG_MMC_DW_FMP_ECRYPT_FS)
if ((sector_key & DW_MMC_FILE_ENCRYPTION_SECTOR_BEGIN) &&
(host->pdata->quirks & DW_MCI_QUIRK_USE_SMU)) { /* file encryption */
int ret;
unsigned int aes_alg = 0;
unsigned int j;
unsigned int last_index = 0;
unsigned long last_inode = 0;
#ifdef CONFIG_CRYPTO_FIPS
char extent_iv[SHA256_HASH_SIZE];
#else
char extent_iv[MD5_DIGEST_SIZE];
#endif
loff_t index;
/* File algorithm selector*/
if (!strncmp(sg_page(&data->sg[idx])->mapping->alg, "aes", sizeof("aes")))
aes_alg = AES_CBC;
else if (!strncmp(sg_page(&data->sg[idx])->mapping->alg, "aesxts", sizeof("aesxts")))
aes_alg = AES_XTS;
else {
printk(KERN_ERR "Invalid file algorithm: %s\n", sg_page(&data->sg[idx])->mapping->alg);
return -1;
}
IDMAC_SET_FAS(desc, aes_alg);
/* File enc key size */
switch (sg_page(&data->sg[idx])->mapping->key_length) {
case 16:
desc->des2 &= ~IDMAC_DES2_FKL;
break;
case 32:
case 64:
desc->des2 |= IDMAC_DES2_FKL;
break;
default:
printk(KERN_ERR "Invalid file key length: %lx\n", sg_page(&data->sg[idx])->mapping->key_length);
return -1;
}
index = sg_page(&data->sg[idx])->index;
if ((last_index != index) || (last_inode != sg_page(&data->sg[idx])->mapping->host->i_ino)) {
index = index - sg_page(&data->sg[idx])->mapping->sensitive_data_index;
ret = file_enc_derive_iv(sg_page(&data->sg[idx])->mapping, index, extent_iv);
if (ret) {
printk(KERN_ERR "Error attemping to derive IV\n");
return ret;
}
}
last_index = sg_page(&data->sg[idx])->index;
last_inode = sg_page(&data->sg[idx])->mapping->host->i_ino;
/* File IV */
desc->des8 = word_in(extent_iv, 3);
desc->des9 = word_in(extent_iv, 2);
desc->des10 = word_in(extent_iv, 1);
desc->des11 = word_in(extent_iv, 0);
/* File Enc key */
for (j = 0; j < sg_page(&data->sg[idx])->mapping->key_length >> 2; j++)
*(&(desc->des12) + j) =
word_in(sg_page(&data->sg[idx])->mapping->key, (sg_page(&data->sg[idx])->mapping->key_length >> 2) - (j + 1));
}
#endif
return 0;
}

289
drivers/crypto/fmp/fmp_ufs.c Normal file
View file

@ -0,0 +1,289 @@
/*
* Exynos FMP UFS driver for FIPS
*
* Copyright (C) 2014 Samsung Electronics Co., Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*/
#include <linux/platform_device.h>
#include <linux/of.h>
#include <linux/of_device.h>
#include <linux/syscalls.h>
#include <linux/smc.h>
#include <ufshcd.h>
#include <ufs-exynos.h>
#if defined(CONFIG_UFS_FMP_ECRYPT_FS)
#include <linux/pagemap.h>
#include "fmp_derive_iv.h"
#endif
#if defined(CONFIG_FIPS_FMP)
#include "fmpdev_info.h"
#endif
#include "fmpdev_int.h" //For FIPS_FMP_FUNC_TEST macro
extern bool in_fmp_fips_err(void);
extern volatile unsigned int disk_key_flag;
extern spinlock_t disk_key_lock;
extern void exynos_ufs_ctrl_hci_core_clk(struct exynos_ufs *ufs, bool en);
#define byte2word(b0, b1, b2, b3) \
((unsigned int)(b0) << 24) | ((unsigned int)(b1) << 16) | ((unsigned int)(b2) << 8) | (b3)
#define word_in(x, c) byte2word(((unsigned char *)(x) + 4 * (c))[0], ((unsigned char *)(x) + 4 * (c))[1], \
((unsigned char *)(x) + 4 * (c))[2], ((unsigned char *)(x) + 4 * (c))[3])
int fmp_map_sg(struct ufshcd_sg_entry *prd_table, struct scatterlist *sg,
uint32_t sector_key, uint32_t idx,
uint32_t sector)
{
#if defined(CONFIG_FIPS_FMP)
if (unlikely(in_fmp_fips_err())) {
printk(KERN_ERR "Fail to work fmp due to fips in error\n");
return -EPERM;
}
#endif
#if defined(CONFIG_UFS_FMP_DM_CRYPT)
/* Disk Encryption */
if (sector_key & UFS_ENCRYPTION_SECTOR_BEGIN) {
/* algorithm */
#if defined(CONFIG_FIPS_FMP)
if (prd_table[idx].size > 0x1000) {
printk(KERN_ERR "Fail to FMP XTS due to invalid size \
for Disk Encrytion. size = 0x%x\n", prd_table[idx].size);
return -EINVAL;
}
#endif
SET_DAS(&prd_table[idx], AES_XTS);
prd_table[idx].size |= DKL;
/* disk IV */
prd_table[idx].disk_iv0 = 0;
prd_table[idx].disk_iv1 = 0;
prd_table[idx].disk_iv2 = 0;
prd_table[idx].disk_iv3 = htonl(sector);
if (disk_key_flag) {
int ret;
if (disk_key_flag == 1)
printk(KERN_INFO "FMP disk encryption key is set\n");
else if (disk_key_flag == 2)
printk(KERN_INFO "FMP disk encryption key is set after clear\n");
ret = exynos_smc(SMC_CMD_FMP, FMP_KEY_SET, UFS_FMP, 0);
if (ret < 0)
panic("Fail to load FMP loadable firmware\n");
else if (ret) {
printk(KERN_ERR "Fail to smc call for FMP key setting. ret = 0x%x\n", ret);
return ret;
}
spin_lock(&disk_key_lock);
disk_key_flag = 0;
spin_unlock(&disk_key_lock);
}
}
#endif
#if defined(CONFIG_UFS_FMP_ECRYPT_FS)
/* File Encryption */
if (sector_key & UFS_FILE_ENCRYPTION_SECTOR_BEGIN) {
int ret;
unsigned int aes_alg, j;
loff_t index;
#ifdef CONFIG_CRYPTO_FIPS
char extent_iv[SHA256_HASH_SIZE];
#else
char extent_iv[MD5_DIGEST_SIZE];
#endif
/* algorithm */
if (!strncmp(sg_page(sg)->mapping->alg, "aes", sizeof("aes")))
aes_alg = AES_CBC;
else if (!strncmp(sg_page(sg)->mapping->alg, "aesxts", sizeof("aesxts"))) {
aes_alg = AES_XTS;
#if defined(CONFIG_FIPS_FMP)
if (prd_table[idx].size > 0x1000) {
printk(KERN_ERR "Fail to FMP XTS due to invalid size \
for File Encrytion. size = 0x%x\n", prd_table[idx].size);
return -EINVAL;
}
#endif
} else {
printk(KERN_ERR "Invalid file encryption algorithm %s \n", sg_page(sg)->mapping->alg);
return -EINVAL;
}
SET_FAS(&prd_table[idx], aes_alg);
/* file encryption key size */
switch (sg_page(sg)->mapping->key_length) {
case 16:
prd_table[idx].size &= ~FKL;
break;
case 32:
case 64:
prd_table[idx].size |= FKL;
break;
default:
printk(KERN_ERR "Invalid file key length %x \n",
(unsigned int)sg_page(sg)->mapping->key_length);
return -EINVAL;
}
index = sg_page(sg)->index;
index = index - sg_page(sg)->mapping->sensitive_data_index;
ret = file_enc_derive_iv(sg_page(sg)->mapping, index, extent_iv);
if (ret) {
printk(KERN_ERR "Error attemping to derive IV. ret = %c\n", ret);
return -EINVAL;
}
/* File IV */
prd_table[idx].file_iv0 = word_in(extent_iv, 3);
prd_table[idx].file_iv1 = word_in(extent_iv, 2);
prd_table[idx].file_iv2 = word_in(extent_iv, 1);
prd_table[idx].file_iv3 = word_in(extent_iv, 0);
/* File Enc key*/
for (j = 0; j < sg_page(sg)->mapping->key_length >> 2; j++)
*(&prd_table[idx].file_enckey0 + j) =
word_in(sg_page(sg)->mapping->key, (sg_page(sg)->mapping->key_length >> 2) - (j + 1));
}
#endif
return 0;
}
EXPORT_SYMBOL_GPL(fmp_map_sg);
#if defined(CONFIG_FIPS_FMP)
int fmp_map_sg_st(struct ufs_hba *hba, struct ufshcd_sg_entry *prd_table,
struct scatterlist *sg, uint32_t sector_key,
uint32_t idx, uint32_t sector)
{
struct ufshcd_sg_entry *prd_table_st = hba->ucd_prdt_ptr_st;
if (sector_key == UFS_BYPASS_SECTOR_BEGIN) {
SET_FAS(&prd_table[idx], CLEAR);
SET_DAS(&prd_table[idx], CLEAR);
return 0;
}
/* algorithm */
if (hba->self_test_mode == XTS_MODE)
SET_FAS(&prd_table[idx], AES_XTS);
else if (hba->self_test_mode == CBC_MODE)
SET_FAS(&prd_table[idx], AES_CBC);
else {
printk(KERN_ERR "Invalid algorithm for FMP FIPS validation. mode = %d\n", hba->self_test_mode);
return -EINVAL;
}
if (prd_table_st->size == 32)
prd_table[idx].size |= FKL;
/* File IV */
prd_table[idx].file_iv0 = prd_table_st->file_iv0;
prd_table[idx].file_iv1 = prd_table_st->file_iv1;
prd_table[idx].file_iv2 = prd_table_st->file_iv2;
prd_table[idx].file_iv3 = prd_table_st->file_iv3;
/* enc key */
prd_table[idx].file_enckey0 = prd_table_st->file_enckey0;
prd_table[idx].file_enckey1 = prd_table_st->file_enckey1;
prd_table[idx].file_enckey2 = prd_table_st->file_enckey2;
prd_table[idx].file_enckey3 = prd_table_st->file_enckey3;
prd_table[idx].file_enckey4 = prd_table_st->file_enckey4;
prd_table[idx].file_enckey5 = prd_table_st->file_enckey5;
prd_table[idx].file_enckey6 = prd_table_st->file_enckey6;
prd_table[idx].file_enckey7 = prd_table_st->file_enckey7;
/* tweak key */
prd_table[idx].file_twkey0 = prd_table_st->file_twkey0;
prd_table[idx].file_twkey1 = prd_table_st->file_twkey1;
prd_table[idx].file_twkey2 = prd_table_st->file_twkey2;
prd_table[idx].file_twkey3 = prd_table_st->file_twkey3;
prd_table[idx].file_twkey4 = prd_table_st->file_twkey4;
prd_table[idx].file_twkey5 = prd_table_st->file_twkey5;
prd_table[idx].file_twkey6 = prd_table_st->file_twkey6;
prd_table[idx].file_twkey7 = prd_table_st->file_twkey7;
return 0;
}
EXPORT_SYMBOL_GPL(fmp_map_sg_st);
#endif
#if defined(CONFIG_UFS_FMP_ECRYPT_FS)
void fmp_clear_sg(struct ufshcd_lrb *lrbp)
{
struct scatterlist *sg;
int sg_segments;
struct ufshcd_sg_entry *prd_table;
struct scsi_cmnd *cmd = lrbp->cmd;
int i;
sg_segments = scsi_sg_count(cmd);
if (sg_segments) {
prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
scsi_for_each_sg(cmd, sg, sg_segments, i) {
if (!((unsigned long)(sg_page(sg)->mapping) & 0x1) && \
sg_page(sg)->mapping && sg_page(sg)->mapping->key \
&& ((unsigned int)(sg_page(sg)->index) >= 2)) {
#if FIPS_FMP_FUNC_TEST == 6 //Key Zeroization
print_hex_dump (KERN_ERR, "FIPS FMP descriptor before zeroize:", DUMP_PREFIX_NONE, 16, 1, &prd_table[i].file_iv0, sizeof(__le32)*20, false );
#endif
memset(&prd_table[i].file_iv0, 0x0, sizeof(__le32)*20);
#if FIPS_FMP_FUNC_TEST == 6 //Key Zeroization
print_hex_dump (KERN_ERR, "FIPS FMP descriptor after zeroize:", DUMP_PREFIX_NONE, 16, 1, &prd_table[i].file_iv0, sizeof(__le32)*20, false );
#endif
}
}
}
return;
}
#endif
#if defined(CONFIG_UFS_FMP_DM_CRYPT)
int fmp_clear_disk_key(void)
{
struct device_node *dev_node;
struct platform_device *pdev;
struct device *dev;
struct ufs_hba *hba;
dev_node = of_find_compatible_node(NULL, NULL, "samsung,exynos-ufs");
if (!dev_node) {
printk(KERN_ERR "Fail to find exynos ufs device node\n");
return -ENODEV;
}
pdev = of_find_device_by_node(dev_node);
if (!pdev) {
printk(KERN_ERR "Fail to find exynos ufs pdev\n");
return -ENODEV;
}
dev = &pdev->dev;
hba = dev_get_drvdata(dev);
if (!hba) {
printk(KERN_ERR "Fail to find hba from dev\n");
return -ENODEV;
}
spin_lock(&disk_key_lock);
disk_key_flag = 2;
spin_unlock(&disk_key_lock);
printk(KERN_INFO "FMP disk encryption key is clear\n");
exynos_ufs_ctrl_hci_core_clk(dev_get_platdata(hba->dev), false);
return exynos_smc(SMC_CMD_FMP, FMP_KEY_CLEAR, 0, 0);
}
#endif

444
drivers/crypto/fmp/fmp_ufs_fips.c Normal file
View file

@ -0,0 +1,444 @@
/*
* Exynos FMP UFS driver for FIPS
*
* Copyright (C) 2014 Samsung Electronics Co., Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*/
#include <linux/platform_device.h>
#include <linux/of.h>
#include <linux/of_device.h>
#include <linux/syscalls.h>
#include <linux/file.h>
#include <linux/buffer_head.h>
#include <linux/mtd/mtd.h>
#include <linux/kdev_t.h>
#include <linux/smc.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_host.h>
#include <scsi/scsi_cmnd.h>
#include <ufshcd.h>
#include "fmpdev_int.h"
#include "../scsi_priv.h"
#if defined(CONFIG_UFS_FMP_ECRYPT_FS)
#include "fmp_derive_iv.h"
#endif
#if defined(CONFIG_FIPS_FMP)
#include "fmpdev_info.h"
#endif
#define byte2word(b0, b1, b2, b3) \
((unsigned int)(b0) << 24) | ((unsigned int)(b1) << 16) | ((unsigned int)(b2) << 8) | (b3)
#define get_word(x, c) byte2word(((unsigned char *)(x) + 4 * (c))[0], ((unsigned char *)(x) + 4 * (c))[1], \
((unsigned char *)(x) + 4 * (c))[2], ((unsigned char *)(x) + 4 * (c))[3])
#define SF_BLK_OFFSET (5)
#define MAX_SCAN_PART (50)
struct ufs_fmp_work {
struct Scsi_Host *host;
struct scsi_device *sdev;
struct block_device *bdev;
sector_t sector;
dev_t devt;
};
struct ufshcd_sg_entry *prd_table;
struct ufshcd_sg_entry *ucd_prdt_ptr_st;
static int ufs_fmp_init(struct device *dev, uint32_t mode)
{
struct ufs_hba *hba;
struct ufs_fmp_work *work;
struct Scsi_Host *host;
work = dev_get_drvdata(dev);
if (!work) {
dev_err(dev, "Fail to get work from platform device\n");
return -ENODEV;
}
host = work->host;
hba = shost_priv(host);
ucd_prdt_ptr_st = kmalloc(sizeof(struct ufshcd_sg_entry), GFP_KERNEL);
if (!ucd_prdt_ptr_st) {
dev_err(dev, "Fail to alloc prdt ptr for self test\n");
return -ENOMEM;
}
hba->ucd_prdt_ptr_st = ucd_prdt_ptr_st;
return 0;
}
static int ufs_fmp_set_key(struct device *dev, uint32_t mode, uint8_t *key, uint32_t key_len)
{
struct ufs_hba *hba;
struct ufs_fmp_work *work;
struct Scsi_Host *host;
struct ufshcd_sg_entry *prd_table;
work = dev_get_drvdata(dev);
if (!work) {
dev_err(dev, "Fail to get work from platform device\n");
return -ENODEV;
}
host = work->host;
hba = shost_priv(host);
if (!hba->ucd_prdt_ptr_st) {
dev_err(dev, "prdt ptr for fips is not allocated\n");
return -ENOMEM;
}
prd_table = hba->ucd_prdt_ptr_st;
if (mode == CBC_MODE) {
SET_FAS(prd_table, CBC_MODE);
switch (key_len) {
case 32:
prd_table->size = 32;
/* encrypt key */
prd_table->file_enckey0 = get_word(key, 7);
prd_table->file_enckey1 = get_word(key, 6);
prd_table->file_enckey2 = get_word(key, 5);
prd_table->file_enckey3 = get_word(key, 4);
prd_table->file_enckey4 = get_word(key, 3);
prd_table->file_enckey5 = get_word(key, 2);
prd_table->file_enckey6 = get_word(key, 1);
prd_table->file_enckey7 = get_word(key, 0);
break;
case 16:
prd_table->size = 16;
/* encrypt key */
prd_table->file_enckey0 = get_word(key, 3);
prd_table->file_enckey1 = get_word(key, 2);
prd_table->file_enckey2 = get_word(key, 1);
prd_table->file_enckey3 = get_word(key, 0);
prd_table->file_enckey4 = 0;
prd_table->file_enckey5 = 0;
prd_table->file_enckey6 = 0;
prd_table->file_enckey7 = 0;
break;
default:
dev_err(dev, "Invalid key length : %d\n", key_len);
return -EINVAL;
}
} else if (mode == XTS_MODE) {
SET_FAS(prd_table, XTS_MODE);
switch (key_len) {
case 64:
prd_table->size = 32;
/* encrypt key */
prd_table->file_enckey0 = get_word(key, 7);
prd_table->file_enckey1 = get_word(key, 6);
prd_table->file_enckey2 = get_word(key, 5);
prd_table->file_enckey3 = get_word(key, 4);
prd_table->file_enckey4 = get_word(key, 3);
prd_table->file_enckey5 = get_word(key, 2);
prd_table->file_enckey6 = get_word(key, 1);
prd_table->file_enckey7 = get_word(key, 0);
/* tweak key */
prd_table->file_twkey0 = get_word(key, 15);
prd_table->file_twkey1 = get_word(key, 14);
prd_table->file_twkey2 = get_word(key, 13);
prd_table->file_twkey3 = get_word(key, 12);
prd_table->file_twkey4 = get_word(key, 11);
prd_table->file_twkey5 = get_word(key, 10);
prd_table->file_twkey6 = get_word(key, 9);
prd_table->file_twkey7 = get_word(key, 8);
break;
case 32:
prd_table->size = 16;
/* encrypt key */
prd_table->file_enckey0 = get_word(key, 3);
prd_table->file_enckey1 = get_word(key, 2);
prd_table->file_enckey2 = get_word(key, 1);
prd_table->file_enckey3 = get_word(key, 0);
prd_table->file_enckey4 = 0;
prd_table->file_enckey5 = 0;
prd_table->file_enckey6 = 0;
prd_table->file_enckey7 = 0;
/* tweak key */
prd_table->file_twkey0 = get_word(key, 7);
prd_table->file_twkey1 = get_word(key, 6);
prd_table->file_twkey2 = get_word(key, 5);
prd_table->file_twkey3 = get_word(key, 4);
prd_table->file_twkey4 = 0;
prd_table->file_twkey5 = 0;
prd_table->file_twkey6 = 0;
prd_table->file_twkey7 = 0;
break;
default:
dev_err(dev, "Invalid key length : %d\n", key_len);
return -EINVAL;
}
} else if (mode == BYPASS_MODE) {
SET_FAS(prd_table, BYPASS_MODE);
/* enc key */
prd_table->file_enckey0 = 0;
prd_table->file_enckey1 = 0;
prd_table->file_enckey2 = 0;
prd_table->file_enckey3 = 0;
prd_table->file_enckey4 = 0;
prd_table->file_enckey5 = 0;
prd_table->file_enckey6 = 0;
prd_table->file_enckey7 = 0;
/* tweak key */
prd_table->file_twkey0 = 0;
prd_table->file_twkey1 = 0;
prd_table->file_twkey2 = 0;
prd_table->file_twkey3 = 0;
prd_table->file_twkey4 = 0;
prd_table->file_twkey5 = 0;
prd_table->file_twkey6 = 0;
prd_table->file_twkey7 = 0;
} else {
dev_err(dev, "Invalid mode : %d\n", mode);
return -EINVAL;
}
dev_info(dev, "%s: ufs fmp key set is done.\n", __func__);
return 0;
}
static int ufs_fmp_set_iv(struct device *dev, uint32_t mode, uint8_t *iv, uint32_t iv_len)
{
struct ufs_hba *hba;
struct ufs_fmp_work *work;
struct Scsi_Host *host;
struct ufshcd_sg_entry *prd_table;
work = dev_get_drvdata(dev);
if (!work) {
dev_err(dev, "Fail to get work from platform device\n");
return -ENODEV;
}
host = work->host;
hba = shost_priv(host);
if (!hba->ucd_prdt_ptr_st) {
dev_err(dev, "prdt ptr for fips is not allocated\n");
return -ENOMEM;
}
prd_table = hba->ucd_prdt_ptr_st;
if (mode == CBC_MODE || mode == XTS_MODE) {
prd_table->file_iv0 = get_word(iv, 3);
prd_table->file_iv1 = get_word(iv, 2);
prd_table->file_iv2 = get_word(iv, 1);
prd_table->file_iv3 = get_word(iv, 0);
} else if (mode == BYPASS_MODE) {
prd_table->file_iv0 = 0;
prd_table->file_iv1 = 0;
prd_table->file_iv2 = 0;
prd_table->file_iv3 = 0;
} else {
dev_err(dev, "Invalid mode : %d\n", mode);
return -EINVAL;
}
return 0;
}
static dev_t find_devt_for_selftest(void)
{
int i, idx = 0;
uint64_t size;
uint64_t size_list[MAX_SCAN_PART];
dev_t devt_list[MAX_SCAN_PART];
dev_t devt_scan, devt;
struct block_device *bdev;
fmode_t fmode = FMODE_WRITE | FMODE_READ;
for (i = 0; i < MAX_SCAN_PART; i++) {
devt_scan = blk_lookup_devt("sda", i);
bdev = blkdev_get_by_dev(devt_scan, fmode, NULL);
if (IS_ERR(bdev))
continue;
else {
size_list[idx++] = (uint64_t)i_size_read(bdev->bd_inode);
devt_list[idx++] = devt_scan;
}
}
if (!idx)
goto err;
for (i = 0; i < idx; i++) {
if (i == 0) {
size = size_list[i];
devt = devt_list[i];
} else {
if (size < size_list[i])
devt = devt_list[i];
}
}
return devt;
err:
return (dev_t)0;
}
static int ufs_fmp_run(struct device *dev, uint32_t mode, uint8_t *data,
uint32_t len, uint32_t write)
{
int ret = 0;
struct ufs_hba *hba;
struct ufs_fmp_work *work;
struct Scsi_Host *host;
static struct buffer_head *bh;
work = dev_get_drvdata(dev);
if (!work) {
dev_err(dev, "Fail to get work from platform device\n");
return -ENODEV;
}
host = work->host;
hba = shost_priv(host);
hba->self_test_mode = mode;
bh = __getblk(work->bdev, work->sector, FMP_BLK_SIZE);
if (!bh) {
dev_err(dev, "Fail to get block from bdev\n");
return -ENODEV;
}
hba->self_test_bh = bh;
get_bh(bh);
if (write == WRITE_MODE) {
memcpy(bh->b_data, data, len);
set_buffer_dirty(bh);
sync_dirty_buffer(bh);
if (buffer_req(bh) && !buffer_uptodate(bh)) {
dev_err(dev, "IO error syncing for FMP fips write\n");
ret = -EIO;
goto out;
}
memset(bh->b_data, 0, FMP_BLK_SIZE);
} else {
lock_buffer(bh);
bh->b_end_io = end_buffer_read_sync;
submit_bh(READ_SYNC, bh);
wait_on_buffer(bh);
if (unlikely(!buffer_uptodate(bh))) {
ret = -EIO;
goto out;
}
memcpy(data, bh->b_data, len);
}
out:
hba->self_test_mode = 0;
hba->self_test_bh = NULL;
put_bh(bh);
return ret;
}
static int ufs_fmp_exit(void)
{
if (ucd_prdt_ptr_st)
kfree(ucd_prdt_ptr_st);
return 0;
}
struct fips_fmp_ops fips_fmp_fops = {
.init = ufs_fmp_init,
.set_key = ufs_fmp_set_key,
.set_iv = ufs_fmp_set_iv,
.run = ufs_fmp_run,
.exit = ufs_fmp_exit,
};
int fips_fmp_init(struct device *dev)
{
struct ufs_fmp_work *work;
struct device_node *dev_node;
struct platform_device *pdev_ufs;
struct device *dev_ufs;
struct ufs_hba *hba;
struct Scsi_Host *host;
struct inode *inode;
struct scsi_device *sdev;
struct super_block *sb;
unsigned long blocksize;
unsigned char blocksize_bits;
sector_t self_test_block;
fmode_t fmode = FMODE_WRITE | FMODE_READ;
work = kmalloc(sizeof(*work), GFP_KERNEL);
if (!work) {
dev_err(dev, "Fail to alloc fmp work buffer\n");
return -ENOMEM;
}
dev_node = of_find_compatible_node(NULL, NULL, "samsung,exynos-ufs");
if (!dev_node) {
dev_err(dev, "Fail to find exynos ufs device node\n");
goto out;
}
pdev_ufs = of_find_device_by_node(dev_node);
if (!pdev_ufs) {
dev_err(dev, "Fail to find exynos ufs pdev\n");
goto out;
}
dev_ufs = &pdev_ufs->dev;
hba = dev_get_drvdata(dev_ufs);
if (!hba) {
dev_err(dev, "Fail to find hba from dev\n");
goto out;
}
host = hba->host;
sdev = to_scsi_device(dev_ufs);
work->host = host;
work->sdev = sdev;
work->devt = find_devt_for_selftest();
if (!work->devt) {
dev_err(dev, "Fail to find devt for self test\n");
return -ENODEV;
}
work->bdev = blkdev_get_by_dev(work->devt, fmode, NULL);
if (IS_ERR(work->bdev)) {
dev_err(dev, "Fail to open block device\n");
return -ENODEV;
}
inode = work->bdev->bd_inode;
sb = inode->i_sb;
blocksize = sb->s_blocksize;
blocksize_bits = sb->s_blocksize_bits;
self_test_block = (i_size_read(inode) - (blocksize * SF_BLK_OFFSET)) >> blocksize_bits;
work->sector = self_test_block;
dev_set_drvdata(dev, work);
return 0;
out:
if (work)
kfree(work);
return -ENODEV;
}
EXPORT_SYMBOL_GPL(fips_fmp_init);

2403
drivers/crypto/fmp/fmpdev.c Normal file
View file

File diff suppressed because it is too large Load diff

22
drivers/crypto/fmp/fmpdev.h Normal file
View file

@ -0,0 +1,22 @@
/*
* Exynos FMP test driver for FIPS
*
* Copyright (C) 2015 Samsung Electronics Co., Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*/
#ifndef _FIPS_FMP_H_
#define _FIPS_FMP_H_
#define FMP_DRV_VERSION "1.1"
int fips_fmp_cipher_init(struct device *dev, uint8_t *enckey, uint8_t *twkey, uint32_t key_len, uint32_t mode);
int fips_fmp_cipher_set_iv(struct device *dev, uint8_t *iv, uint32_t mode);
int fips_fmp_cipher_run(struct device *dev, uint8_t *src, uint8_t *dst, uint32_t len, uint32_t mode, uint32_t enc);
int fips_fmp_cipher_exit(struct device *dev);
#endif

133
drivers/crypto/fmp/fmpdev_info.h Normal file
View file

@ -0,0 +1,133 @@
/*
* Exynos FMP device information for FIPS
*
* Copyright (C) 2015 Samsung Electronics Co., Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*/
#ifndef _FIPS_FMP_INFO_H_
#define _FIPS_FMP_INFO_H_
#define FMP_BLK_SIZE (4096)
#define BYPASS_MODE 0
#define CBC_MODE 1
#define XTS_MODE 2
#define ENCRYPT 1
#define DECRYPT 2
#define WRITE_MODE 1
#define READ_MODE 2
/* API extensions for linux */
#define FMP_HMAC_MAX_KEY_LEN 512
#define FMP_CIPHER_MAX_KEY_LEN 64
enum fmpdev_crypto_op_t {
FMP_AES_CBC = 1,
FMP_AES_XTS = 2,
FMP_SHA2_256_HMAC = 3,
FMP_SHA2_256 = 4,
FMPFW_SHA2_256_HMAC = 5,
FMPFW_SHA2_256 = 6,
FMP_ALGORITHM_ALL, /* Keep updated - see below */
};
#define FMP_ALGORITHM_MAX (FMP_ALGORITHM_ALL - 1)
/* the maximum of the above */
#define EALG_MAX_BLOCK_LEN 16
/* Values for hashes/MAC */
#define AALG_MAX_RESULT_LEN 64
/* maximum length of verbose alg names (depends on CRYPTO_MAX_ALG_NAME) */
#define FMPDEV_MAX_ALG_NAME 64
#define DEFAULT_PREALLOC_PAGES 32
/* input of FMPGSESSION */
struct session_op {
__u32 cipher; /* cryptodev_crypto_op_t */
__u32 mac; /* cryptodev_crypto_op_t */
__u32 keylen;
__u8 __user *key;
__u32 mackeylen;
__u8 __user *mackey;
__u32 ses; /* session identifier */
};
struct session_info_op {
__u32 ses; /* session identifier */
/* verbose names for the requested ciphers */
struct alg_info {
char cra_name[FMPDEV_MAX_ALG_NAME];
char cra_driver_name[FMPDEV_MAX_ALG_NAME];
} cipher_info, hash_info;
__u16 alignmask; /* alignment constraints */
__u32 flags; /* SIOP_FLAGS_* */
};
/* If this flag is set then this algorithm uses
* a driver only available in kernel (software drivers,
* or drivers based on instruction sets do not set this flag).
*
* If multiple algorithms are involved (as in AEAD case), then
* if one of them is kernel-driver-only this flag will be set.
*/
#define SIOP_FLAG_KERNEL_DRIVER_ONLY 1
#define COP_ENCRYPT 0
#define COP_DECRYPT 1
/* input of FMPCRYPT */
struct crypt_op {
__u32 ses; /* session identifier */
__u16 op; /* COP_ENCRYPT or COP_DECRYPT */
__u16 flags; /* see COP_FLAG_* */
__u32 len; /* length of source data */
__u8 __user *src; /* source data */
__u8 __user *dst; /* pointer to output data */
/* pointer to output data for hash/MAC operations */
__u8 __user *mac;
/* initialization vector for encryption operations */
__u8 __user *iv;
__u32 data_unit_len;
__u32 data_unit_seqnumber;
__u8 __user *secondLastEncodedData;
__u8 __user *thirdLastEncodedData;
};
#define COP_FLAG_NONE (0 << 0) /* totally no flag */
#define COP_FLAG_UPDATE (1 << 0) /* multi-update hash mode */
#define COP_FLAG_FINAL (1 << 1) /* multi-update final hash mode */
#define COP_FLAG_WRITE_IV (1 << 2) /* update the IV during operation */
#define COP_FLAG_NO_ZC (1 << 3) /* do not zero-copy */
#define COP_FLAG_AEAD_TLS_TYPE (1 << 4) /* authenticate and encrypt using the
* TLS protocol rules */
#define COP_FLAG_AEAD_SRTP_TYPE (1 << 5) /* authenticate and encrypt using the
* SRTP protocol rules */
#define COP_FLAG_RESET (1 << 6) /* multi-update reset the state.
* should be used in combination
* with COP_FLAG_UPDATE */
#define COP_FLAG_AES_CBC (1 << 7)
#define COP_FLAG_AES_XTS (1 << 8)
#define COP_FLAG_AES_CBC_MCT (1 << 9)
#define FMPGSESSION _IOWR('c', 200, struct session_op)
#define FMPFSESSION _IOWR('c', 201, __u32)
#define FMPGSESSIONINFO _IOWR('c', 202, struct session_info_op)
#define FMPCRYPT _IOWR('c', 203, struct crypt_op)
#define FMP_AES_CBC_MCT _IOWR('c', 204, struct crypt_op)
#endif

247
drivers/crypto/fmp/fmpdev_int.h Normal file
View file

@ -0,0 +1,247 @@
/*
* Exynos FMP device header for FIPS
*
* Copyright (C) 2015 Samsung Electronics Co., Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*/
#ifndef __FMPDEV_INT_H__
#define __FMPDEV_INT_H__
#include <linux/init.h>
#include <linux/sched.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/fdtable.h>
#include <linux/miscdevice.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/scatterlist.h>
#include <crypto/aead.h>
#include "fmpdev_info.h"
/* Used during FIPS Functional test with CMT Lab
* FIPS_FMP_FUNC_TEST 0 - Normal operation
* FIPS_FMP_FUNC_TEST 1 - Fail FMP H/W AES XTS Self test
* FIPS_FMP_FUNC_TEST 2 - Fail FMP H/W AES CBC Self test
* FIPS_FMP_FUNC_TEST 3 - Fail FMP DRV SHA256 Self test
* FIPS_FMP_FUNC_TEST 4 - Fail FMP DRV HMAC Self test
* FIPS_FMP_FUNC_TEST 5 - Fail FMP DRV Integrity test
* FIPS_FMP_FUNC_TEST 6 - Enable FMP Key zeroization logs
*/
#define FIPS_FMP_FUNC_TEST 0
struct fcrypt {
struct list_head list;
struct mutex sem;
};
/* kernel-internal extension to struct crypt_op */
struct kernel_crypt_op {
struct crypt_op cop;
int ivlen;
__u8 iv[EALG_MAX_BLOCK_LEN];
int digestsize;
uint8_t hash_output[AALG_MAX_RESULT_LEN];
struct task_struct *task;
struct mm_struct *mm;
};
struct todo_list_item {
struct list_head __hook;
struct kernel_crypt_op kcop;
int result;
};
struct locked_list {
struct list_head list;
struct mutex lock;
};
struct fmp_info {
int init_status;
struct device *dev;
struct fcrypt fcrypt;
struct locked_list free, todo, done;
int itemcount;
struct work_struct fmptask;
wait_queue_head_t user_waiter;
};
/* compatibility stuff */
#ifdef CONFIG_COMPAT
#include <linux/compat.h>
/* input of FMPGSESSION */
struct compat_session_op {
/* Specify either cipher or mac
*/
uint32_t cipher; /* cryptodev_crypto_op_t */
uint32_t mac; /* cryptodev_crypto_op_t */
uint32_t keylen;
compat_uptr_t key; /* pointer to key data */
uint32_t mackeylen;
compat_uptr_t mackey; /* pointer to mac key data */
uint32_t ses; /* session identifier */
};
/* input of FMPCRYPT */
struct compat_crypt_op {
uint32_t ses; /* session identifier */
uint16_t op; /* COP_ENCRYPT or COP_DECRYPT */
uint16_t flags; /* see COP_FLAG_* */
uint32_t len; /* length of source data */
compat_uptr_t src; /* source data */
compat_uptr_t dst; /* pointer to output data */
compat_uptr_t mac;/* pointer to output data for hash/MAC operations */
compat_uptr_t iv;/* initialization vector for encryption operations */
compat_uptr_t secondLastEncodedData;
compat_uptr_t thirdLastEncodedData;
};
#define COMPAT_FMPGSESSION _IOWR('c', 100, struct compat_session_op)
#define COMPAT_FMPCRYPT _IOWR('c', 101, struct compat_crypt_op)
#define COMPAT_FMP_AES_CBC_MCT _IOWR('c', 102, struct compat_crypt_op)
#endif
/* the maximum of the above */
#define EALG_MAX_BLOCK_LEN 16
struct cipher_data {
int init; /* 0 uninitialized */
int blocksize;
int aead;
int stream;
int ivsize;
int alignmask;
struct {
/* block ciphers */
struct crypto_ablkcipher *s;
struct ablkcipher_request *request;
/* AEAD ciphers */
struct crypto_aead *as;
struct aead_request *arequest;
struct fmpdev_result *result;
uint8_t iv[EALG_MAX_BLOCK_LEN];
} async;
int mode;
};
/* Hash */
struct sha256_fmpfw_info {
uint32_t input;
size_t input_len;
uint32_t output;
uint32_t step;
};
struct hmac_sha256_fmpfw_info {
struct sha256_fmpfw_info s;
uint32_t key;
size_t key_len;
uint32_t hmac_mode;
uint32_t dummy;
};
struct hash_data {
int init; /* 0 uninitialized */
int digestsize;
int alignmask;
struct {
struct crypto_ahash *s;
struct fmpdev_result *result;
struct ahash_request *request;
} async;
struct hmac_sha256_fmpfw_info *fmpfw_info;
};
struct fmpdev_result {
struct completion completion;
int err;
};
/* other internal structs */
struct csession {
struct list_head entry;
struct mutex sem;
struct cipher_data cdata;
struct hash_data hdata;
uint32_t sid;
uint32_t alignmask;
unsigned int array_size;
unsigned int used_pages; /* the number of pages that are used */
/* the number of pages marked as writable (first are the readable) */
unsigned int readable_pages;
struct page **pages;
struct scatterlist *sg;
};
struct csession *fmp_get_session_by_sid(struct fcrypt *fcr, uint32_t sid);
inline static void fmp_put_session(struct csession *ses_ptr)
{
mutex_unlock(&ses_ptr->sem);
}
int adjust_sg_array(struct csession *ses, int pagecount);
#define MAX_TAP 8
#define XBUFSIZE 8
struct cipher_testvec {
char *key;
char *iv;
char *input;
char *result;
unsigned short tap[MAX_TAP];
int np;
unsigned char also_non_np;
unsigned char klen;
unsigned short ilen;
unsigned short rlen;
};
struct hash_testvec {
/* only used with keyed hash algorithms */
char *key;
char *plaintext;
char *digest;
unsigned char tap[MAX_TAP];
unsigned short psize;
unsigned char np;
unsigned char ksize;
};
struct cipher_test_suite {
struct {
struct cipher_testvec *vecs;
unsigned int count;
} enc;
};
struct hash_test_suite {
struct hash_testvec *vecs;
unsigned int count;
};
struct fips_fmp_ops {
int (*init)(struct device *dev, uint32_t mode);
int (*set_key)(struct device *dev, uint32_t mode, uint8_t *key, uint32_t key_len);
int (*set_iv)(struct device *dev, uint32_t mode, uint8_t *iv, uint32_t iv_len);
int (*run)(struct device *dev, uint32_t mode, uint8_t *data, uint32_t len, uint32_t write);
int (*exit)(void);
};
#endif

782
drivers/crypto/fmp/fmplib.c Normal file
View file

@ -0,0 +1,782 @@
/*
* Exynos FMP libary for FIPS
*
* Copyright (C) 2015 Samsung Electronics Co., Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*/
#include <linux/platform_device.h>
#include <linux/of.h>
#include <linux/of_address.h>
#include <linux/of_platform.h>
#include <linux/dma-mapping.h>
#include <linux/smc.h>
#include <crypto/hash.h>
#include <crypto/sha.h>
#include <asm/cacheflush.h>
#include "fmpdev_int.h"
#include "fmpdev.h"
#define BYPASS_MODE 0
#define CBC_MODE 1
#define XTS_MODE 2
#define INIT 0
#define UPDATE 1
#define FINAL 2
static void fmpdev_complete(struct crypto_async_request *req, int err)
{
struct fmpdev_result *res = req->data;
if (err == -EINPROGRESS)
return;
res->err = err;
complete(&res->completion);
}
static inline int waitfor(struct fmp_info *info, struct fmpdev_result *cr,
ssize_t ret)
{
struct device *dev = info->dev;
switch (ret) {
case 0:
break;
case -EINPROGRESS:
case -EBUSY:
wait_for_completion(&cr->completion);
/* At this point we known for sure the request has finished,
* because wait_for_completion above was not interruptible.
* This is important because otherwise hardware or driver
* might try to access memory which will be freed or reused for
* another request. */
if (unlikely(cr->err)) {
dev_err(dev, "error from async request: %d\n", cr->err);
return cr->err;
}
break;
default:
return ret;
}
return 0;
}
/*
* FMP library to call FMP driver
*
* This file is part of linux fmpdev
*/
int fmpdev_cipher_init(struct fmp_info *info, struct cipher_data *out,
const char *alg_name,
uint8_t *enckey, uint8_t *twkey, size_t keylen)
{
int ret;
struct device *dev = info->dev;
memset(out, 0, sizeof(*out));
if (!strcmp(alg_name, "cbc(aes-fmp)"))
out->mode = CBC_MODE;
else if (!strcmp(alg_name, "xts(aes-fmp)"))
out->mode = XTS_MODE;
else {
dev_err(dev, "Invalid mode\n");
return -1;
}
out->blocksize = 16;
out->ivsize = 16;
ret = fips_fmp_cipher_init(dev, enckey, twkey, keylen, out->mode);
if (ret) {
dev_err(dev, "Fail to initialize fmp cipher\n");
return -1;
}
out->init = 1;
return 0;
}
void fmpdev_cipher_deinit(struct cipher_data *cdata)
{
if (cdata->init)
cdata->init = 0;
}
int fmpdev_cipher_set_iv(struct fmp_info *info, struct cipher_data *cdata,
uint8_t *iv, size_t iv_size)
{
int ret;
struct device *dev = info->dev;
ret = fips_fmp_cipher_set_iv(dev, iv, cdata->mode);
if (ret) {
dev_err(dev, "Fail to set fmp iv\n");
return -1;
}
return 0;
}
int fmpdev_cipher_exit(struct fmp_info *info)
{
int ret;
struct device *dev = info->dev;
ret = fips_fmp_cipher_exit(dev);
if (ret) {
dev_err(dev, "Fail to exit fmp\n");
return -1;
}
return 0;
}
static int fmpdev_cipher_encrypt(struct fmp_info *info,
struct cipher_data *cdata,
struct scatterlist *src,
struct scatterlist *dst, size_t len)
{
int ret;
struct device *dev = info->dev;
ret = fips_fmp_cipher_run(dev, sg_virt(src), sg_virt(dst),
len, cdata->mode, ENCRYPT);
if (ret) {
dev_err(dev, "Fail to encrypt using fmp\n");
return -1;
}
return 0;
}
static int fmpdev_cipher_decrypt(struct fmp_info *info,
struct cipher_data *cdata,
struct scatterlist *src,
struct scatterlist *dst, size_t len)
{
int ret;
struct device *dev = info->dev;
ret = fips_fmp_cipher_run(dev, sg_virt(src), sg_virt(dst),
len, cdata->mode, DECRYPT);
if (ret) {
dev_err(dev, "Fail to encrypt using fmp\n");
return -1;
}
return 0;
}
/* Hash functions */
int fmpfw_hash_init(struct fmp_info *info, struct hash_data *hdata,
const char *alg_name, int hmac_mode,
void *mackey, size_t mackeylen)
{
int ret;
unsigned long addr;
struct device *dev = info->dev;
struct hmac_sha256_fmpfw_info *fmpfw_info;
fmpfw_info = (struct hmac_sha256_fmpfw_info *)kzalloc(sizeof(*fmpfw_info), GFP_KERNEL);
if (unlikely(!fmpfw_info)) {
dev_err(dev, "Fail to alloc fmpfw info\n");
ret = ENOMEM;
goto error_alloc_info;
}
if (hmac_mode != 0) {
fmpfw_info->s.step = INIT;
fmpfw_info->hmac_mode = 1;
fmpfw_info->key = (uint32_t)virt_to_phys(mackey);
__flush_dcache_area(mackey, mackeylen);
fmpfw_info->key_len = mackeylen;
__flush_dcache_area(fmpfw_info, sizeof(*fmpfw_info));
addr = virt_to_phys(fmpfw_info);
ret = exynos_smc(SMC_CMD_FMP, FMP_FW_HMAC_SHA2_TEST, (uint32_t)addr, 0);
if (unlikely(ret)) {
dev_err(dev, "Fail to smc call for FMPFW HMAC SHA256 init. ret = 0x%x\n", ret);
ret = -EFAULT;
goto error_hmac_smc;
}
} else {
fmpfw_info->s.step = INIT;
fmpfw_info->hmac_mode = 0;
fmpfw_info->key = 0;
fmpfw_info->key_len = 0;
__flush_dcache_area(fmpfw_info, sizeof(*fmpfw_info));
addr = virt_to_phys(fmpfw_info);
ret = exynos_smc(SMC_CMD_FMP, FMP_FW_SHA2_TEST, (uint32_t)addr, 0);
if (unlikely(ret)) {
dev_err(dev, "Fail to smc call for FMPFW SHA256 init. ret = 0x%x\n", ret);
ret = -EFAULT;
goto error_sha_smc;
}
}
hdata->digestsize = SHA256_DIGEST_SIZE;
hdata->alignmask = 0x0;
hdata->fmpfw_info = fmpfw_info;
hdata->async.result = kzalloc(sizeof(*hdata->async.result), GFP_KERNEL);
if (unlikely(!hdata->async.result)) {
ret = -ENOMEM;
goto error;
}
init_completion(&hdata->async.result->completion);
hdata->init = 1;
return 0;
error_hmac_smc:
error_sha_smc:
kfree(fmpfw_info);
error_alloc_info:
hdata->fmpfw_info = NULL;
error:
kfree(hdata->async.result);
return ret;
}
void fmpfw_hash_deinit(struct hash_data *hdata)
{
if (hdata->init) {
kfree(hdata->async.result);
hdata->init = 0;
}
}
ssize_t fmpfw_hash_update(struct fmp_info *info, struct hash_data *hdata,
struct scatterlist *sg, size_t len)
{
int ret = 0;
unsigned long addr;
struct device *dev = info->dev;
struct hmac_sha256_fmpfw_info *fmpfw_info = hdata->fmpfw_info;
fmpfw_info->s.step = UPDATE;
fmpfw_info->s.input = (uint32_t)sg_phys(sg);
__flush_dcache_area(sg_virt(sg), len);
fmpfw_info->s.input_len = len;
__flush_dcache_area(fmpfw_info, sizeof(*fmpfw_info));
addr = virt_to_phys(fmpfw_info);
reinit_completion(&hdata->async.result->completion);
if (fmpfw_info->hmac_mode) {
ret = exynos_smc(SMC_CMD_FMP, FMP_FW_HMAC_SHA2_TEST, addr, 0);
if (unlikely(ret)) {
dev_err(dev, "Fail to smc call for FMPFW HMAC SHA256 update. ret = 0x%x\n", ret);
ret = -EFAULT;
}
} else {
ret = exynos_smc(SMC_CMD_FMP, FMP_FW_SHA2_TEST, addr, 0);
if (unlikely(ret)) {
dev_err(dev, "Fail to smc call for FMPFW SHA256 update. ret = 0x%x\n", ret);
ret = -EFAULT;
}
}
return waitfor(info, hdata->async.result, ret);
}
int fmpfw_hash_final(struct fmp_info *info, struct hash_data *hdata, void *output)
{
int ret = 0;
unsigned long addr;
struct device *dev = info->dev;
struct hmac_sha256_fmpfw_info *fmpfw_info = hdata->fmpfw_info;
memset(output, 0, hdata->digestsize);
fmpfw_info->s.step = FINAL;
fmpfw_info->s.output = (uint32_t)virt_to_phys(output);
__flush_dcache_area(fmpfw_info, sizeof(*fmpfw_info));
addr = virt_to_phys(fmpfw_info);
reinit_completion(&hdata->async.result->completion);
__dma_unmap_area((void *)output, hdata->digestsize, DMA_FROM_DEVICE);
if (fmpfw_info->hmac_mode) {
ret = exynos_smc(SMC_CMD_FMP, FMP_FW_HMAC_SHA2_TEST, addr, 0);
if (unlikely(ret)) {
dev_err(dev, "Fail to smc call for FMPFW HMAC SHA256 final. ret = 0x%x\n", ret);
ret = -EFAULT;
}
} else {
ret = exynos_smc(SMC_CMD_FMP, FMP_FW_SHA2_TEST, addr, 0);
if (unlikely(ret)) {
dev_err(dev, "Fail to smc call for FMPFW SHA256 final. ret = 0x%x\n", ret);
ret = -EFAULT;
}
}
__dma_unmap_area((void *)output, hdata->digestsize, DMA_FROM_DEVICE);
if (fmpfw_info->hmac_mode)
dev_info(dev, "fmp fw hmac sha256 F/W final is done\n");
else
dev_info(dev, "fmp fw sha256 F/W final is done\n");
kfree(fmpfw_info);
return waitfor(info, hdata->async.result, ret);
}
int fmpdev_hash_init(struct fmp_info *info, struct hash_data *hdata,
const char *alg_name,
int hmac_mode, void *mackey, size_t mackeylen)
{
int ret;
struct device *dev = info->dev;
hdata->fmpfw_info = NULL;
hdata->async.s = crypto_alloc_ahash(alg_name, 0, 0);
if (unlikely(IS_ERR(hdata->async.s))) {
dev_err(dev, "Failed to load transform for %s\n", alg_name);
return -EINVAL;
}
/* Copy the key from user and set to TFM. */
if (hmac_mode != 0) {
ret = crypto_ahash_setkey(hdata->async.s, mackey, mackeylen);
if (unlikely(ret)) {
dev_err(dev, "Setting hmac key failed for %s-%zu.\n",
alg_name, mackeylen*8);
ret = -EINVAL;
goto error;
}
}
hdata->digestsize = crypto_ahash_digestsize(hdata->async.s);
hdata->alignmask = crypto_ahash_alignmask(hdata->async.s);
hdata->async.result = kzalloc(sizeof(*hdata->async.result), GFP_KERNEL);
if (unlikely(!hdata->async.result)) {
ret = -ENOMEM;
goto error;
}
init_completion(&hdata->async.result->completion);
hdata->async.request = ahash_request_alloc(hdata->async.s, GFP_KERNEL);
if (unlikely(!hdata->async.request)) {
dev_err(dev, "error allocating async crypto request\n");
ret = -ENOMEM;
goto error;
}
ahash_request_set_callback(hdata->async.request,
CRYPTO_TFM_REQ_MAY_BACKLOG,
fmpdev_complete, hdata->async.result);
ret = crypto_ahash_init(hdata->async.request);
if (unlikely(ret)) {
dev_err(dev, "error in fmpdev_hash_init()\n");
goto error_request;
}
hdata->init = 1;
return 0;
error_request:
ahash_request_free(hdata->async.request);
error:
kfree(hdata->async.result);
crypto_free_ahash(hdata->async.s);
return ret;
}
void fmpdev_hash_deinit(struct hash_data *hdata)
{
if (hdata->init) {
if (hdata->async.request)
ahash_request_free(hdata->async.request);
kfree(hdata->async.result);
if (hdata->async.s)
crypto_free_ahash(hdata->async.s);
hdata->init = 0;
}
}
int fmpdev_hash_reset(struct fmp_info *info, struct hash_data *hdata)
{
int ret;
struct device *dev = info->dev;
ret = crypto_ahash_init(hdata->async.request);
if (unlikely(ret)) {
dev_err(dev, "error in crypto_hash_init()\n");
return ret;
}
return 0;
}
ssize_t fmpdev_hash_update(struct fmp_info *info, struct hash_data *hdata,
struct scatterlist *sg, size_t len)
{
int ret;
reinit_completion(&hdata->async.result->completion);
ahash_request_set_crypt(hdata->async.request, sg, NULL, len);
ret = crypto_ahash_update(hdata->async.request);
return waitfor(info, hdata->async.result, ret);
}
int fmpdev_hash_final(struct fmp_info *info, struct hash_data *hdata, void *output)
{
int ret;
reinit_completion(&hdata->async.result->completion);
ahash_request_set_crypt(hdata->async.request, NULL, output, 0);
ret = crypto_ahash_final(hdata->async.request);
return waitfor(info, hdata->async.result, ret);
}
static int fmp_n_crypt(struct fmp_info *info, struct csession *ses_ptr,
struct crypt_op *cop,
struct scatterlist *src_sg, struct scatterlist *dst_sg,
uint32_t len)
{
int ret;
struct device *dev = info->dev;
if (cop->op == COP_ENCRYPT) {
if (ses_ptr->hdata.init != 0) {
if (!ses_ptr->hdata.fmpfw_info)
ret = fmpdev_hash_update(info, &ses_ptr->hdata,
src_sg, len);
else
ret = fmpfw_hash_update(info, &ses_ptr->hdata,
src_sg, len);
if (unlikely(ret))
goto out_err;
}
if (ses_ptr->cdata.init != 0) {
ret = fmpdev_cipher_encrypt(info, &ses_ptr->cdata,
src_sg, dst_sg, len);
if (unlikely(ret))
goto out_err;
}
} else {
if (ses_ptr->cdata.init != 0) {
ret = fmpdev_cipher_decrypt(info, &ses_ptr->cdata,
src_sg, dst_sg, len);
if (unlikely(ret))
goto out_err;
}
if (ses_ptr->hdata.init != 0) {
if (!ses_ptr->hdata.fmpfw_info)
ret = fmpdev_hash_update(info, &ses_ptr->hdata,
dst_sg, len);
else
ret = fmpfw_hash_update(info, &ses_ptr->hdata,
dst_sg, len);
if (unlikely(ret))
goto out_err;
}
}
return 0;
out_err:
dev_err(dev, "FMP crypt failure: %d\n", ret);
return ret;
}
static int __fmp_run_std(struct fmp_info *info,
struct csession *ses_ptr, struct crypt_op *cop)
{
char *data;
struct device *dev = info->dev;
char __user *src, *dst;
size_t nbytes, bufsize;
struct scatterlist sg;
int ret = 0;
nbytes = cop->len;
data = (char *)__get_free_page(GFP_KERNEL);
if (unlikely(!data)) {
dev_err(dev, "Error getting free page.\n");
return -ENOMEM;
}
bufsize = PAGE_SIZE < nbytes ? PAGE_SIZE : nbytes;
src = cop->src;
dst = cop->dst;
while (nbytes > 0) {
size_t current_len = nbytes > bufsize ? bufsize : nbytes;
if (unlikely(copy_from_user(data, src, current_len))) {
dev_err(dev, "Error copying %d bytes from user address %p\n",
(int)current_len, src);
ret = -EFAULT;
break;
}
sg_init_one(&sg, data, current_len);
ret = fmp_n_crypt(info, ses_ptr, cop, &sg, &sg, current_len);
if (unlikely(ret)) {
dev_err(dev, "fmp_n_crypt failed\n");
break;
}
if (ses_ptr->cdata.init != 0) {
if (unlikely(copy_to_user(dst, data, current_len))) {
dev_err(dev, "could not copy to user\n");
ret = -EFAULT;
break;
}
}
dst += current_len;
nbytes -= current_len;
src += current_len;
}
free_page((unsigned long)data);
return ret;
}
int fmp_run(struct fmp_info *info, struct fcrypt *fcr, struct kernel_crypt_op *kcop)
{
struct device *dev = info->dev;
struct csession *ses_ptr;
struct crypt_op *cop = &kcop->cop;
int ret = -EINVAL;
if (unlikely(cop->op != COP_ENCRYPT && cop->op != COP_DECRYPT)) {
dev_err(dev, "invalid operation op=%u\n", cop->op);
return -EINVAL;
}
/* this also enters ses_ptr->sem */
ses_ptr = fmp_get_session_by_sid(fcr, cop->ses);
if (unlikely(!ses_ptr)) {
dev_err(dev, "invalid session ID=0x%08X\n", cop->ses);
return -EINVAL;
}
if ((ses_ptr->cdata.init != 0) && (cop->len > PAGE_SIZE)) {
dev_err(dev, "Invalid input length. len = %d\n", cop->len);
return -EINVAL;
}
if (ses_ptr->hdata.init != 0 && (cop->flags == 0 || cop->flags & COP_FLAG_RESET) && \
!ses_ptr->hdata.fmpfw_info) {
ret = fmpdev_hash_reset(info, &ses_ptr->hdata);
if (unlikely(ret)) {
dev_err(dev, "error in fmpdev_hash_reset()");
goto out_unlock;
}
}
if (ses_ptr->cdata.init != 0) {
int blocksize = ses_ptr->cdata.blocksize;
if (unlikely(cop->len % blocksize)) {
dev_err(dev,
"data size (%u) isn't a multiple "
"of block size (%u)\n",
cop->len, blocksize);
ret = -EINVAL;
goto out_unlock;
}
if (cop->flags == COP_FLAG_AES_CBC)
fmpdev_cipher_set_iv(info, &ses_ptr->cdata, kcop->iv, 16);
else if (cop->flags == COP_FLAG_AES_XTS)
fmpdev_cipher_set_iv(info, &ses_ptr->cdata, (uint8_t *)&cop->data_unit_seqnumber, 16);
else {
ret = -EINVAL;
goto out_unlock;
}
}
if (likely(cop->len)) {
ret = __fmp_run_std(info, ses_ptr, &kcop->cop);
if (unlikely(ret))
goto out_unlock;
}
if (ses_ptr->hdata.init != 0 &&
((cop->flags & COP_FLAG_FINAL) ||
(!(cop->flags & COP_FLAG_UPDATE) || cop->len == 0))) {
if (!ses_ptr->hdata.fmpfw_info)
ret = fmpdev_hash_final(info, &ses_ptr->hdata, kcop->hash_output);
else
ret = fmpfw_hash_final(info, &ses_ptr->hdata, kcop->hash_output);
if (unlikely(ret)) {
dev_err(dev, "CryptoAPI failure: %d\n", ret);
goto out_unlock;
}
kcop->digestsize = ses_ptr->hdata.digestsize;
}
out_unlock:
fmp_put_session(ses_ptr);
return ret;
}
int fmp_run_AES_CBC_MCT(struct fmp_info *info, struct fcrypt *fcr,
struct kernel_crypt_op *kcop)
{
struct device *dev = info->dev;
struct csession *ses_ptr;
struct crypt_op *cop = &kcop->cop;
char **Ct = 0;
char **Pt = 0;
int ret = 0, k = 0;
if (unlikely(cop->op != COP_ENCRYPT && cop->op != COP_DECRYPT)) {
dev_err(dev, "invalid operation op=%u\n", cop->op);
return -EINVAL;
}
/* this also enters ses_ptr->sem */
ses_ptr = fmp_get_session_by_sid(fcr, cop->ses);
if (unlikely(!ses_ptr)) {
dev_err(dev, "invalid session ID=0x%08X\n", cop->ses);
return -EINVAL;
}
if (cop->len > PAGE_SIZE) {
dev_err(dev, "Invalid input length. len = %d\n", cop->len);
return -EINVAL;
}
if (ses_ptr->cdata.init != 0) {
int blocksize = ses_ptr->cdata.blocksize;
if (unlikely(cop->len % blocksize)) {
dev_err(dev,
"data size (%u) isn't a multiple "
"of block size (%u)\n",
cop->len, blocksize);
ret = -EINVAL;
goto out_unlock;
}
fmpdev_cipher_set_iv(info, &ses_ptr->cdata, kcop->iv, 16);
}
if (likely(cop->len)) {
if (cop->flags & COP_FLAG_AES_CBC_MCT) {
// do MCT here
char *data;
char __user *src, *dst, *secondLast;
struct scatterlist sg;
size_t nbytes, bufsize;
int ret = 0;
int y = 0;
nbytes = cop->len;
data = (char *)__get_free_page(GFP_KERNEL);
if (unlikely(!data)) {
dev_err(dev, "Error getting free page.\n");
return -ENOMEM;
}
Pt = (char**)kmalloc(1000 * sizeof(char*), GFP_KERNEL);
for (k=0; k<1000; k++)
Pt[k]= (char*)kmalloc(nbytes, GFP_KERNEL);
Ct = (char**)kmalloc(1000 * sizeof(char*), GFP_KERNEL);
for (k=0; k<1000; k++)
Ct[k]= (char*)kmalloc(nbytes, GFP_KERNEL);
bufsize = PAGE_SIZE < nbytes ? PAGE_SIZE : nbytes;
src = cop->src;
dst = cop->dst;
secondLast = cop->secondLastEncodedData;
if (unlikely(copy_from_user(data, src, nbytes))) {
printk(KERN_ERR "Error copying %d bytes from user address %p.\n", (int)nbytes, src);
ret = -EFAULT;
goto out_err;
}
sg_init_one(&sg, data, nbytes);
for (y = 0; y < 1000; y++) {
memcpy(Pt[y], data, nbytes);
ret = fmp_n_crypt(info, ses_ptr, cop, &sg, &sg, nbytes);
memcpy(Ct[y], data, nbytes);
if (y == 998) {
if (unlikely(copy_to_user(secondLast, data, nbytes)))
printk(KERN_ERR "unable to copy second last data for AES_CBC_MCT\n");
else
printk(KERN_ERR "KAMAL copied secondlast data\n");
}
if( y == 0) {
memcpy(data, kcop->iv, kcop->ivlen);
} else {
if(y != 999)
memcpy(data, Ct[y-1], nbytes);
}
if (unlikely(ret)) {
printk(KERN_ERR "fmp_n_crypt failed.\n");
goto out_err;
}
if (cop->op == COP_ENCRYPT)
fmpdev_cipher_set_iv(info, &ses_ptr->cdata, Ct[y], 16);
else if (cop->op == COP_DECRYPT)
fmpdev_cipher_set_iv(info, &ses_ptr->cdata, Pt[y], 16);
} // for loop
if (ses_ptr->cdata.init != 0) {
if (unlikely(copy_to_user(dst, data, nbytes))) {
printk(KERN_ERR "could not copy to user.\n");
ret = -EFAULT;
goto out_err;
}
}
for (k=0; k<1000; k++) {
kfree(Ct[k]);
kfree(Pt[k]);
}
kfree(Ct);
kfree(Pt);
free_page((unsigned long)data);
} else
goto out_unlock;
}
out_unlock:
fmp_put_session(ses_ptr);
return ret;
out_err:
fmp_put_session(ses_ptr);
dev_info(dev, "CryptoAPI failure: %d\n", ret);
return ret;
}

36
drivers/crypto/fmp/fmplib.h Normal file
View file

@ -0,0 +1,36 @@
/*
* Exynos FMP library header
*
* Copyright (C) 2015 Samsung Electronics Co., Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*/
#ifndef __FMPLIB_H__
#define __FMPLIB_H__
int fmpdev_cipher_init(struct fmp_info *info, struct cipher_data *out,
const char *alg_name,
uint8_t *enckey, uint8_t *twkey, size_t keylen);
void fmpdev_cipher_deinit(struct cipher_data *cdata);
int fmpfw_hash_init(struct fmp_info *info, struct hash_data *hdata,
const char *alg_name, int hmac_mode,
void *mackey, size_t mackeylen);
void fmpfw_hash_deinit(struct hash_data *hdata);
int fmpdev_hash_init(struct fmp_info *info, struct hash_data *hdata,
const char *alg_name,
int hmac_mode, void *mackey, size_t mackeylen);
void fmpdev_hash_deinit(struct hash_data *hdata);
int fmpdev_hash_reset(struct fmp_info *info, struct hash_data *hdata);
ssize_t fmpdev_hash_update(struct hash_data *hdata,
struct scatterlist *sg, size_t len);
int fmpdev_hash_final(struct hash_data *hdata, void *output);
int fmp_run(struct fmp_info *info, struct fcrypt *fcr, struct kernel_crypt_op *kcop);
int fmpdev_cipher_exit(struct fmp_info *info);
int fmp_run_AES_CBC_MCT(struct fmp_info *info, struct fcrypt *fcr,
struct kernel_crypt_op *kcop);
#endif

287
drivers/crypto/fmp/hmac_fmp.c Normal file
View file

@ -0,0 +1,287 @@
/*
* Cryptographic API.
*
* HMAC: Keyed-Hashing for Message Authentication (RFC2104).
*
* Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
* Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
*
* The HMAC implementation is derived from USAGI.
* Copyright (c) 2002 Kazunori Miyazawa <miyazawa@linux-ipv6.org> / USAGI
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation; either version 2 of the License, or (at your option)
* any later version.
*
*/
#include <crypto/internal/hash.h>
#include <crypto/scatterwalk.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/scatterlist.h>
#include <linux/string.h>
#if defined(CONFIG_FIPS_FMP)
extern bool in_fmp_fips_err(void);
#endif
struct hmac_ctx {
struct crypto_shash *hash;
};
static inline void *align_ptr(void *p, unsigned int align)
{
return (void *)ALIGN((unsigned long)p, align);
}
static inline struct hmac_ctx *hmac_ctx(struct crypto_shash *tfm)
{
return align_ptr(crypto_shash_ctx_aligned(tfm) +
crypto_shash_statesize(tfm) * 2,
crypto_tfm_ctx_alignment());
}
static int hmac_setkey(struct crypto_shash *parent,
const u8 *inkey, unsigned int keylen)
{
int bs = crypto_shash_blocksize(parent);
int ds = crypto_shash_digestsize(parent);
int ss = crypto_shash_statesize(parent);
char *ipad = crypto_shash_ctx_aligned(parent);
char *opad = ipad + ss;
struct hmac_ctx *ctx = align_ptr(opad + ss,
crypto_tfm_ctx_alignment());
struct crypto_shash *hash = ctx->hash;
struct {
struct shash_desc shash;
char ctx[crypto_shash_descsize(hash)];
} desc;
unsigned int i;
desc.shash.tfm = hash;
desc.shash.flags = crypto_shash_get_flags(parent) &
CRYPTO_TFM_REQ_MAY_SLEEP;
if (keylen > bs) {
int err;
err = crypto_shash_digest(&desc.shash, inkey, keylen, ipad);
if (err)
return err;
keylen = ds;
} else
memcpy(ipad, inkey, keylen);
memset(ipad + keylen, 0, bs - keylen);
memcpy(opad, ipad, bs);
for (i = 0; i < bs; i++) {
ipad[i] ^= 0x36;
opad[i] ^= 0x5c;
}
return crypto_shash_init(&desc.shash) ?:
crypto_shash_update(&desc.shash, ipad, bs) ?:
crypto_shash_export(&desc.shash, ipad) ?:
crypto_shash_init(&desc.shash) ?:
crypto_shash_update(&desc.shash, opad, bs) ?:
crypto_shash_export(&desc.shash, opad);
}
static int hmac_export(struct shash_desc *pdesc, void *out)
{
struct shash_desc *desc = shash_desc_ctx(pdesc);
desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
return crypto_shash_export(desc, out);
}
static int hmac_import(struct shash_desc *pdesc, const void *in)
{
struct shash_desc *desc = shash_desc_ctx(pdesc);
struct hmac_ctx *ctx = hmac_ctx(pdesc->tfm);
desc->tfm = ctx->hash;
desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
return crypto_shash_import(desc, in);
}
static int hmac_init(struct shash_desc *pdesc)
{
return hmac_import(pdesc, crypto_shash_ctx_aligned(pdesc->tfm));
}
static int hmac_update(struct shash_desc *pdesc,
const u8 *data, unsigned int nbytes)
{
struct shash_desc *desc = shash_desc_ctx(pdesc);
desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
return crypto_shash_update(desc, data, nbytes);
}
static int hmac_final(struct shash_desc *pdesc, u8 *out)
{
struct crypto_shash *parent = pdesc->tfm;
int ds = crypto_shash_digestsize(parent);
int ss = crypto_shash_statesize(parent);
char *opad = crypto_shash_ctx_aligned(parent) + ss;
struct shash_desc *desc = shash_desc_ctx(pdesc);
desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
printk(KERN_INFO "%s: fmp hmac sha256 S/W final is done.\n", __func__);
return crypto_shash_final(desc, out) ?:
crypto_shash_import(desc, opad) ?:
crypto_shash_finup(desc, out, ds, out);
}
static int hmac_finup(struct shash_desc *pdesc, const u8 *data,
unsigned int nbytes, u8 *out)
{
struct crypto_shash *parent = pdesc->tfm;
int ds = crypto_shash_digestsize(parent);
int ss = crypto_shash_statesize(parent);
char *opad = crypto_shash_ctx_aligned(parent) + ss;
struct shash_desc *desc = shash_desc_ctx(pdesc);
desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
return crypto_shash_finup(desc, data, nbytes, out) ?:
crypto_shash_import(desc, opad) ?:
crypto_shash_finup(desc, out, ds, out);
}
static int hmac_init_tfm(struct crypto_tfm *tfm)
{
struct crypto_shash *parent = __crypto_shash_cast(tfm);
struct crypto_shash *hash;
struct crypto_instance *inst = (void *)tfm->__crt_alg;
struct crypto_shash_spawn *spawn = crypto_instance_ctx(inst);
struct hmac_ctx *ctx = hmac_ctx(parent);
hash = crypto_spawn_shash(spawn);
if (IS_ERR(hash))
return PTR_ERR(hash);
parent->descsize = sizeof(struct shash_desc) +
crypto_shash_descsize(hash);
ctx->hash = hash;
return 0;
}
static void hmac_exit_tfm(struct crypto_tfm *tfm)
{
struct hmac_ctx *ctx = hmac_ctx(__crypto_shash_cast(tfm));
crypto_free_shash(ctx->hash);
}
static int hmac_create(struct crypto_template *tmpl, struct rtattr **tb)
{
struct shash_instance *inst;
struct crypto_alg *alg;
struct shash_alg *salg;
int err;
int ds;
int ss;
#if defined(CONFIG_FIPS_FMP)
if (unlikely(in_fmp_fips_err())) {
printk(KERN_ERR "FMP HMAC create failed due to fips error\n");
return -EPERM;
}
#endif
err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH);
if (err)
return err;
salg = shash_attr_alg(tb[1], 0, 0);
if (IS_ERR(salg))
return PTR_ERR(salg);
err = -EINVAL;
ds = salg->digestsize;
ss = salg->statesize;
alg = &salg->base;
if (ds > alg->cra_blocksize ||
ss < alg->cra_blocksize)
goto out_put_alg;
inst = shash_alloc_instance("hmac-fmp", alg);
err = PTR_ERR(inst);
if (IS_ERR(inst))
goto out_put_alg;
err = crypto_init_shash_spawn(shash_instance_ctx(inst), salg,
shash_crypto_instance(inst));
if (err)
goto out_free_inst;
inst->alg.base.cra_priority = alg->cra_priority;
inst->alg.base.cra_blocksize = alg->cra_blocksize;
inst->alg.base.cra_alignmask = alg->cra_alignmask;
ss = ALIGN(ss, alg->cra_alignmask + 1);
inst->alg.digestsize = ds;
inst->alg.statesize = ss;
inst->alg.base.cra_ctxsize = sizeof(struct hmac_ctx) +
ALIGN(ss * 2, crypto_tfm_ctx_alignment());
inst->alg.base.cra_init = hmac_init_tfm;
inst->alg.base.cra_exit = hmac_exit_tfm;
inst->alg.init = hmac_init;
inst->alg.update = hmac_update;
inst->alg.final = hmac_final;
inst->alg.finup = hmac_finup;
inst->alg.export = hmac_export;
inst->alg.import = hmac_import;
inst->alg.setkey = hmac_setkey;
err = shash_register_instance(tmpl, inst);
if (err) {
out_free_inst:
shash_free_instance(shash_crypto_instance(inst));
}
out_put_alg:
crypto_mod_put(alg);
return err;
}
static struct crypto_template hmac_tmpl = {
.name = "hmac-fmp",
.create = hmac_create,
.free = shash_free_instance,
.module = THIS_MODULE,
};
static int __init hmac_module_init(void)
{
return crypto_register_template(&hmac_tmpl);
}
static void __exit hmac_module_exit(void)
{
crypto_unregister_template(&hmac_tmpl);
}
module_init(hmac_module_init);
module_exit(hmac_module_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("HMAC hash algorithm");

30
drivers/crypto/fmp/last_file.c Executable file
View file

@ -0,0 +1,30 @@
/*
* Last file for Exynos FMP FIPS integrity check
*
* Copyright (C) 2015 Samsung Electronics Co., Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*/
#include <linux/init.h>
const int last_fmp_rodata = 1000;
int last_fmp_data = 2000;
void last_fmp_text(void) __attribute__((unused));
void last_fmp_text (void)
{
}
void __init last_fmp_init(void) __attribute__((unused));
void __init last_fmp_init(void)
{
}
void __exit last_fmp_exit(void) __attribute__((unused));
void __exit last_fmp_exit(void)
{
}

357
drivers/crypto/fmp/sha256_fmp.c Normal file
View file

@ -0,0 +1,357 @@
/*
* Cryptographic API.
*
* SHA-256, as specified in
* http://csrc.nist.gov/groups/STM/cavp/documents/shs/sha256-384-512.pdf
*
* SHA-256 code by Jean-Luc Cooke <jlcooke@certainkey.com>.
*
* Copyright (c) Jean-Luc Cooke <jlcooke@certainkey.com>
* Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
* Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation; either version 2 of the License, or (at your option)
* any later version.
*
*/
#include <crypto/internal/hash.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/types.h>
#include <crypto/sha.h>
#include <asm/byteorder.h>
#if defined(CONFIG_FIPS_FMP)
extern bool in_fmp_fips_err(void);
#endif
static inline u32 Ch(u32 x, u32 y, u32 z)
{
return z ^ (x & (y ^ z));
}
static inline u32 Maj(u32 x, u32 y, u32 z)
{
return (x & y) | (z & (x | y));
}
#define e0(x) (ror32(x, 2) ^ ror32(x,13) ^ ror32(x,22))
#define e1(x) (ror32(x, 6) ^ ror32(x,11) ^ ror32(x,25))
#define s0(x) (ror32(x, 7) ^ ror32(x,18) ^ (x >> 3))
#define s1(x) (ror32(x,17) ^ ror32(x,19) ^ (x >> 10))
static inline void LOAD_OP(int I, u32 *W, const u8 *input)
{
W[I] = __be32_to_cpu( ((__be32*)(input))[I] );
}
static inline void BLEND_OP(int I, u32 *W)
{
W[I] = s1(W[I-2]) + W[I-7] + s0(W[I-15]) + W[I-16];
}
static void sha256_transform(u32 *state, const u8 *input)
{
u32 a, b, c, d, e, f, g, h, t1, t2;
u32 W[64];
int i;
/* load the input */
for (i = 0; i < 16; i++)
LOAD_OP(i, W, input);
/* now blend */
for (i = 16; i < 64; i++)
BLEND_OP(i, W);
/* load the state into our registers */
a=state[0]; b=state[1]; c=state[2]; d=state[3];
e=state[4]; f=state[5]; g=state[6]; h=state[7];
/* now iterate */
t1 = h + e1(e) + Ch(e,f,g) + 0x428a2f98 + W[ 0];
t2 = e0(a) + Maj(a,b,c); d+=t1; h=t1+t2;
t1 = g + e1(d) + Ch(d,e,f) + 0x71374491 + W[ 1];
t2 = e0(h) + Maj(h,a,b); c+=t1; g=t1+t2;
t1 = f + e1(c) + Ch(c,d,e) + 0xb5c0fbcf + W[ 2];
t2 = e0(g) + Maj(g,h,a); b+=t1; f=t1+t2;
t1 = e + e1(b) + Ch(b,c,d) + 0xe9b5dba5 + W[ 3];
t2 = e0(f) + Maj(f,g,h); a+=t1; e=t1+t2;
t1 = d + e1(a) + Ch(a,b,c) + 0x3956c25b + W[ 4];
t2 = e0(e) + Maj(e,f,g); h+=t1; d=t1+t2;
t1 = c + e1(h) + Ch(h,a,b) + 0x59f111f1 + W[ 5];
t2 = e0(d) + Maj(d,e,f); g+=t1; c=t1+t2;
t1 = b + e1(g) + Ch(g,h,a) + 0x923f82a4 + W[ 6];
t2 = e0(c) + Maj(c,d,e); f+=t1; b=t1+t2;
t1 = a + e1(f) + Ch(f,g,h) + 0xab1c5ed5 + W[ 7];
t2 = e0(b) + Maj(b,c,d); e+=t1; a=t1+t2;
t1 = h + e1(e) + Ch(e,f,g) + 0xd807aa98 + W[ 8];
t2 = e0(a) + Maj(a,b,c); d+=t1; h=t1+t2;
t1 = g + e1(d) + Ch(d,e,f) + 0x12835b01 + W[ 9];
t2 = e0(h) + Maj(h,a,b); c+=t1; g=t1+t2;
t1 = f + e1(c) + Ch(c,d,e) + 0x243185be + W[10];
t2 = e0(g) + Maj(g,h,a); b+=t1; f=t1+t2;
t1 = e + e1(b) + Ch(b,c,d) + 0x550c7dc3 + W[11];
t2 = e0(f) + Maj(f,g,h); a+=t1; e=t1+t2;
t1 = d + e1(a) + Ch(a,b,c) + 0x72be5d74 + W[12];
t2 = e0(e) + Maj(e,f,g); h+=t1; d=t1+t2;
t1 = c + e1(h) + Ch(h,a,b) + 0x80deb1fe + W[13];
t2 = e0(d) + Maj(d,e,f); g+=t1; c=t1+t2;
t1 = b + e1(g) + Ch(g,h,a) + 0x9bdc06a7 + W[14];
t2 = e0(c) + Maj(c,d,e); f+=t1; b=t1+t2;
t1 = a + e1(f) + Ch(f,g,h) + 0xc19bf174 + W[15];
t2 = e0(b) + Maj(b,c,d); e+=t1; a=t1+t2;
t1 = h + e1(e) + Ch(e,f,g) + 0xe49b69c1 + W[16];
t2 = e0(a) + Maj(a,b,c); d+=t1; h=t1+t2;
t1 = g + e1(d) + Ch(d,e,f) + 0xefbe4786 + W[17];
t2 = e0(h) + Maj(h,a,b); c+=t1; g=t1+t2;
t1 = f + e1(c) + Ch(c,d,e) + 0x0fc19dc6 + W[18];
t2 = e0(g) + Maj(g,h,a); b+=t1; f=t1+t2;
t1 = e + e1(b) + Ch(b,c,d) + 0x240ca1cc + W[19];
t2 = e0(f) + Maj(f,g,h); a+=t1; e=t1+t2;
t1 = d + e1(a) + Ch(a,b,c) + 0x2de92c6f + W[20];
t2 = e0(e) + Maj(e,f,g); h+=t1; d=t1+t2;
t1 = c + e1(h) + Ch(h,a,b) + 0x4a7484aa + W[21];
t2 = e0(d) + Maj(d,e,f); g+=t1; c=t1+t2;
t1 = b + e1(g) + Ch(g,h,a) + 0x5cb0a9dc + W[22];
t2 = e0(c) + Maj(c,d,e); f+=t1; b=t1+t2;
t1 = a + e1(f) + Ch(f,g,h) + 0x76f988da + W[23];
t2 = e0(b) + Maj(b,c,d); e+=t1; a=t1+t2;
t1 = h + e1(e) + Ch(e,f,g) + 0x983e5152 + W[24];
t2 = e0(a) + Maj(a,b,c); d+=t1; h=t1+t2;
t1 = g + e1(d) + Ch(d,e,f) + 0xa831c66d + W[25];
t2 = e0(h) + Maj(h,a,b); c+=t1; g=t1+t2;
t1 = f + e1(c) + Ch(c,d,e) + 0xb00327c8 + W[26];
t2 = e0(g) + Maj(g,h,a); b+=t1; f=t1+t2;
t1 = e + e1(b) + Ch(b,c,d) + 0xbf597fc7 + W[27];
t2 = e0(f) + Maj(f,g,h); a+=t1; e=t1+t2;
t1 = d + e1(a) + Ch(a,b,c) + 0xc6e00bf3 + W[28];
t2 = e0(e) + Maj(e,f,g); h+=t1; d=t1+t2;
t1 = c + e1(h) + Ch(h,a,b) + 0xd5a79147 + W[29];
t2 = e0(d) + Maj(d,e,f); g+=t1; c=t1+t2;
t1 = b + e1(g) + Ch(g,h,a) + 0x06ca6351 + W[30];
t2 = e0(c) + Maj(c,d,e); f+=t1; b=t1+t2;
t1 = a + e1(f) + Ch(f,g,h) + 0x14292967 + W[31];
t2 = e0(b) + Maj(b,c,d); e+=t1; a=t1+t2;
t1 = h + e1(e) + Ch(e,f,g) + 0x27b70a85 + W[32];
t2 = e0(a) + Maj(a,b,c); d+=t1; h=t1+t2;
t1 = g + e1(d) + Ch(d,e,f) + 0x2e1b2138 + W[33];
t2 = e0(h) + Maj(h,a,b); c+=t1; g=t1+t2;
t1 = f + e1(c) + Ch(c,d,e) + 0x4d2c6dfc + W[34];
t2 = e0(g) + Maj(g,h,a); b+=t1; f=t1+t2;
t1 = e + e1(b) + Ch(b,c,d) + 0x53380d13 + W[35];
t2 = e0(f) + Maj(f,g,h); a+=t1; e=t1+t2;
t1 = d + e1(a) + Ch(a,b,c) + 0x650a7354 + W[36];
t2 = e0(e) + Maj(e,f,g); h+=t1; d=t1+t2;
t1 = c + e1(h) + Ch(h,a,b) + 0x766a0abb + W[37];
t2 = e0(d) + Maj(d,e,f); g+=t1; c=t1+t2;
t1 = b + e1(g) + Ch(g,h,a) + 0x81c2c92e + W[38];
t2 = e0(c) + Maj(c,d,e); f+=t1; b=t1+t2;
t1 = a + e1(f) + Ch(f,g,h) + 0x92722c85 + W[39];
t2 = e0(b) + Maj(b,c,d); e+=t1; a=t1+t2;
t1 = h + e1(e) + Ch(e,f,g) + 0xa2bfe8a1 + W[40];
t2 = e0(a) + Maj(a,b,c); d+=t1; h=t1+t2;
t1 = g + e1(d) + Ch(d,e,f) + 0xa81a664b + W[41];
t2 = e0(h) + Maj(h,a,b); c+=t1; g=t1+t2;
t1 = f + e1(c) + Ch(c,d,e) + 0xc24b8b70 + W[42];
t2 = e0(g) + Maj(g,h,a); b+=t1; f=t1+t2;
t1 = e + e1(b) + Ch(b,c,d) + 0xc76c51a3 + W[43];
t2 = e0(f) + Maj(f,g,h); a+=t1; e=t1+t2;
t1 = d + e1(a) + Ch(a,b,c) + 0xd192e819 + W[44];
t2 = e0(e) + Maj(e,f,g); h+=t1; d=t1+t2;
t1 = c + e1(h) + Ch(h,a,b) + 0xd6990624 + W[45];
t2 = e0(d) + Maj(d,e,f); g+=t1; c=t1+t2;
t1 = b + e1(g) + Ch(g,h,a) + 0xf40e3585 + W[46];
t2 = e0(c) + Maj(c,d,e); f+=t1; b=t1+t2;
t1 = a + e1(f) + Ch(f,g,h) + 0x106aa070 + W[47];
t2 = e0(b) + Maj(b,c,d); e+=t1; a=t1+t2;
t1 = h + e1(e) + Ch(e,f,g) + 0x19a4c116 + W[48];
t2 = e0(a) + Maj(a,b,c); d+=t1; h=t1+t2;
t1 = g + e1(d) + Ch(d,e,f) + 0x1e376c08 + W[49];
t2 = e0(h) + Maj(h,a,b); c+=t1; g=t1+t2;
t1 = f + e1(c) + Ch(c,d,e) + 0x2748774c + W[50];
t2 = e0(g) + Maj(g,h,a); b+=t1; f=t1+t2;
t1 = e + e1(b) + Ch(b,c,d) + 0x34b0bcb5 + W[51];
t2 = e0(f) + Maj(f,g,h); a+=t1; e=t1+t2;
t1 = d + e1(a) + Ch(a,b,c) + 0x391c0cb3 + W[52];
t2 = e0(e) + Maj(e,f,g); h+=t1; d=t1+t2;
t1 = c + e1(h) + Ch(h,a,b) + 0x4ed8aa4a + W[53];
t2 = e0(d) + Maj(d,e,f); g+=t1; c=t1+t2;
t1 = b + e1(g) + Ch(g,h,a) + 0x5b9cca4f + W[54];
t2 = e0(c) + Maj(c,d,e); f+=t1; b=t1+t2;
t1 = a + e1(f) + Ch(f,g,h) + 0x682e6ff3 + W[55];
t2 = e0(b) + Maj(b,c,d); e+=t1; a=t1+t2;
t1 = h + e1(e) + Ch(e,f,g) + 0x748f82ee + W[56];
t2 = e0(a) + Maj(a,b,c); d+=t1; h=t1+t2;
t1 = g + e1(d) + Ch(d,e,f) + 0x78a5636f + W[57];
t2 = e0(h) + Maj(h,a,b); c+=t1; g=t1+t2;
t1 = f + e1(c) + Ch(c,d,e) + 0x84c87814 + W[58];
t2 = e0(g) + Maj(g,h,a); b+=t1; f=t1+t2;
t1 = e + e1(b) + Ch(b,c,d) + 0x8cc70208 + W[59];
t2 = e0(f) + Maj(f,g,h); a+=t1; e=t1+t2;
t1 = d + e1(a) + Ch(a,b,c) + 0x90befffa + W[60];
t2 = e0(e) + Maj(e,f,g); h+=t1; d=t1+t2;
t1 = c + e1(h) + Ch(h,a,b) + 0xa4506ceb + W[61];
t2 = e0(d) + Maj(d,e,f); g+=t1; c=t1+t2;
t1 = b + e1(g) + Ch(g,h,a) + 0xbef9a3f7 + W[62];
t2 = e0(c) + Maj(c,d,e); f+=t1; b=t1+t2;
t1 = a + e1(f) + Ch(f,g,h) + 0xc67178f2 + W[63];
t2 = e0(b) + Maj(b,c,d); e+=t1; a=t1+t2;
state[0] += a; state[1] += b; state[2] += c; state[3] += d;
state[4] += e; state[5] += f; state[6] += g; state[7] += h;
/* clear any sensitive info... */
a = b = c = d = e = f = g = h = t1 = t2 = 0;
memset(W, 0, 64 * sizeof(u32));
}
static int sha256_init(struct shash_desc *desc)
{
struct sha256_state *sctx = shash_desc_ctx(desc);
#if defined(CONFIG_FIPS_FMP)
if (unlikely(in_fmp_fips_err())) {
printk(KERN_ERR "FMP sha256_init failed due to fips error\n");
return -EPERM;
}
#endif
sctx->state[0] = SHA256_H0;
sctx->state[1] = SHA256_H1;
sctx->state[2] = SHA256_H2;
sctx->state[3] = SHA256_H3;
sctx->state[4] = SHA256_H4;
sctx->state[5] = SHA256_H5;
sctx->state[6] = SHA256_H6;
sctx->state[7] = SHA256_H7;
sctx->count = 0;
return 0;
}
static int sha256_update(struct shash_desc *desc, const u8 *data,
unsigned int len)
{
struct sha256_state *sctx = shash_desc_ctx(desc);
unsigned int partial, done;
const u8 *src;
partial = sctx->count & 0x3f;
sctx->count += len;
done = 0;
src = data;
if ((partial + len) > 63) {
if (partial) {
done = -partial;
memcpy(sctx->buf + partial, data, done + 64);
src = sctx->buf;
}
do {
sha256_transform(sctx->state, src);
done += 64;
src = data + done;
} while (done + 63 < len);
partial = 0;
}
memcpy(sctx->buf + partial, src, len - done);
return 0;
}
static int sha256_final(struct shash_desc *desc, u8 *out)
{
struct sha256_state *sctx = shash_desc_ctx(desc);
__be32 *dst = (__be32 *)out;
__be64 bits;
unsigned int index, pad_len;
int i;
static const u8 padding[64] = { 0x80, };
/* Save number of bits */
bits = cpu_to_be64(sctx->count << 3);
/* Pad out to 56 mod 64. */
index = sctx->count & 0x3f;
pad_len = (index < 56) ? (56 - index) : ((64+56) - index);
sha256_update(desc, padding, pad_len);
/* Append length (before padding) */
sha256_update(desc, (const u8 *)&bits, sizeof(bits));
/* Store state in digest */
for (i = 0; i < 8; i++)
dst[i] = cpu_to_be32(sctx->state[i]);
/* Zeroize sensitive information. */
memset(sctx, 0, sizeof(*sctx));
printk(KERN_INFO "%s: fmp sha256 S/W final is done.\n", __func__);
return 0;
}
static int sha256_export(struct shash_desc *desc, void *out)
{
struct sha256_state *sctx = shash_desc_ctx(desc);
memcpy(out, sctx, sizeof(*sctx));
return 0;
}
static int sha256_import(struct shash_desc *desc, const void *in)
{
struct sha256_state *sctx = shash_desc_ctx(desc);
memcpy(sctx, in, sizeof(*sctx));
return 0;
}
static struct shash_alg sha256_algs[1] = { {
.digestsize = SHA256_DIGEST_SIZE,
.init = sha256_init,
.update = sha256_update,
.final = sha256_final,
.export = sha256_export,
.import = sha256_import,
.descsize = sizeof(struct sha256_state),
.statesize = sizeof(struct sha256_state),
.base = {
.cra_name = "sha256-fmp",
.cra_driver_name= "sha256-fmp",
.cra_flags = CRYPTO_ALG_TYPE_SHASH,
.cra_blocksize = SHA256_BLOCK_SIZE,
.cra_module = THIS_MODULE,
}
}
};
static int __init sha256_generic_mod_init(void)
{
return crypto_register_shashes(sha256_algs, ARRAY_SIZE(sha256_algs));
}
static void __exit sha256_generic_mod_fini(void)
{
crypto_unregister_shashes(sha256_algs, ARRAY_SIZE(sha256_algs));
}
module_init(sha256_generic_mod_init);
module_exit(sha256_generic_mod_fini);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA-256 Secure Hash Algorithm");
MODULE_ALIAS("sha256-fmp");

590
drivers/crypto/geode-aes.c Normal file
View file

@ -0,0 +1,590 @@
/* Copyright (C) 2004-2006, Advanced Micro Devices, Inc.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*/
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/pci_ids.h>
#include <linux/crypto.h>
#include <linux/spinlock.h>
#include <crypto/algapi.h>
#include <crypto/aes.h>
#include <linux/io.h>
#include <linux/delay.h>
#include "geode-aes.h"
/* Static structures */
static void __iomem *_iobase;
static spinlock_t lock;
/* Write a 128 bit field (either a writable key or IV) */
static inline void
_writefield(u32 offset, void *value)
{
int i;
for (i = 0; i < 4; i++)
iowrite32(((u32 *) value)[i], _iobase + offset + (i * 4));
}
/* Read a 128 bit field (either a writable key or IV) */
static inline void
_readfield(u32 offset, void *value)
{
int i;
for (i = 0; i < 4; i++)
((u32 *) value)[i] = ioread32(_iobase + offset + (i * 4));
}
static int
do_crypt(void *src, void *dst, int len, u32 flags)
{
u32 status;
u32 counter = AES_OP_TIMEOUT;
iowrite32(virt_to_phys(src), _iobase + AES_SOURCEA_REG);
iowrite32(virt_to_phys(dst), _iobase + AES_DSTA_REG);
iowrite32(len, _iobase + AES_LENA_REG);
/* Start the operation */
iowrite32(AES_CTRL_START | flags, _iobase + AES_CTRLA_REG);
do {
status = ioread32(_iobase + AES_INTR_REG);
cpu_relax();
} while (!(status & AES_INTRA_PENDING) && --counter);
/* Clear the event */
iowrite32((status & 0xFF) | AES_INTRA_PENDING, _iobase + AES_INTR_REG);
return counter ? 0 : 1;
}
static unsigned int
geode_aes_crypt(struct geode_aes_op *op)
{
u32 flags = 0;
unsigned long iflags;
int ret;
if (op->len == 0)
return 0;
/* If the source and destination is the same, then
* we need to turn on the coherent flags, otherwise
* we don't need to worry
*/
flags |= (AES_CTRL_DCA | AES_CTRL_SCA);
if (op->dir == AES_DIR_ENCRYPT)
flags |= AES_CTRL_ENCRYPT;
/* Start the critical section */
spin_lock_irqsave(&lock, iflags);
if (op->mode == AES_MODE_CBC) {
flags |= AES_CTRL_CBC;
_writefield(AES_WRITEIV0_REG, op->iv);
}
if (!(op->flags & AES_FLAGS_HIDDENKEY)) {
flags |= AES_CTRL_WRKEY;
_writefield(AES_WRITEKEY0_REG, op->key);
}
ret = do_crypt(op->src, op->dst, op->len, flags);
BUG_ON(ret);
if (op->mode == AES_MODE_CBC)
_readfield(AES_WRITEIV0_REG, op->iv);
spin_unlock_irqrestore(&lock, iflags);
return op->len;
}
/* CRYPTO-API Functions */
static int geode_setkey_cip(struct crypto_tfm *tfm, const u8 *key,
unsigned int len)
{
struct geode_aes_op *op = crypto_tfm_ctx(tfm);
unsigned int ret;
op->keylen = len;
if (len == AES_KEYSIZE_128) {
memcpy(op->key, key, len);
return 0;
}
if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256) {
/* not supported at all */
tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
return -EINVAL;
}
/*
* The requested key size is not supported by HW, do a fallback
*/
op->fallback.cip->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
op->fallback.cip->base.crt_flags |= (tfm->crt_flags & CRYPTO_TFM_REQ_MASK);
ret = crypto_cipher_setkey(op->fallback.cip, key, len);
if (ret) {
tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
tfm->crt_flags |= (op->fallback.cip->base.crt_flags & CRYPTO_TFM_RES_MASK);
}
return ret;
}
static int geode_setkey_blk(struct crypto_tfm *tfm, const u8 *key,
unsigned int len)
{
struct geode_aes_op *op = crypto_tfm_ctx(tfm);
unsigned int ret;
op->keylen = len;
if (len == AES_KEYSIZE_128) {
memcpy(op->key, key, len);
return 0;
}
if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256) {
/* not supported at all */
tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
return -EINVAL;
}
/*
* The requested key size is not supported by HW, do a fallback
*/
op->fallback.blk->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
op->fallback.blk->base.crt_flags |= (tfm->crt_flags & CRYPTO_TFM_REQ_MASK);
ret = crypto_blkcipher_setkey(op->fallback.blk, key, len);
if (ret) {
tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
tfm->crt_flags |= (op->fallback.blk->base.crt_flags & CRYPTO_TFM_RES_MASK);
}
return ret;
}
static int fallback_blk_dec(struct blkcipher_desc *desc,
struct scatterlist *dst, struct scatterlist *src,
unsigned int nbytes)
{
unsigned int ret;
struct crypto_blkcipher *tfm;
struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
tfm = desc->tfm;
desc->tfm = op->fallback.blk;
ret = crypto_blkcipher_decrypt_iv(desc, dst, src, nbytes);
desc->tfm = tfm;
return ret;
}
static int fallback_blk_enc(struct blkcipher_desc *desc,
struct scatterlist *dst, struct scatterlist *src,
unsigned int nbytes)
{
unsigned int ret;
struct crypto_blkcipher *tfm;
struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
tfm = desc->tfm;
desc->tfm = op->fallback.blk;
ret = crypto_blkcipher_encrypt_iv(desc, dst, src, nbytes);
desc->tfm = tfm;
return ret;
}
static void
geode_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
struct geode_aes_op *op = crypto_tfm_ctx(tfm);
if (unlikely(op->keylen != AES_KEYSIZE_128)) {
crypto_cipher_encrypt_one(op->fallback.cip, out, in);
return;
}
op->src = (void *) in;
op->dst = (void *) out;
op->mode = AES_MODE_ECB;
op->flags = 0;
op->len = AES_BLOCK_SIZE;
op->dir = AES_DIR_ENCRYPT;
geode_aes_crypt(op);
}
static void
geode_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
struct geode_aes_op *op = crypto_tfm_ctx(tfm);
if (unlikely(op->keylen != AES_KEYSIZE_128)) {
crypto_cipher_decrypt_one(op->fallback.cip, out, in);
return;
}
op->src = (void *) in;
op->dst = (void *) out;
op->mode = AES_MODE_ECB;
op->flags = 0;
op->len = AES_BLOCK_SIZE;
op->dir = AES_DIR_DECRYPT;
geode_aes_crypt(op);
}
static int fallback_init_cip(struct crypto_tfm *tfm)
{
const char *name = crypto_tfm_alg_name(tfm);
struct geode_aes_op *op = crypto_tfm_ctx(tfm);
op->fallback.cip = crypto_alloc_cipher(name, 0,
CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
if (IS_ERR(op->fallback.cip)) {
printk(KERN_ERR "Error allocating fallback algo %s\n", name);
return PTR_ERR(op->fallback.cip);
}
return 0;
}
static void fallback_exit_cip(struct crypto_tfm *tfm)
{
struct geode_aes_op *op = crypto_tfm_ctx(tfm);
crypto_free_cipher(op->fallback.cip);
op->fallback.cip = NULL;
}
static struct crypto_alg geode_alg = {
.cra_name = "aes",
.cra_driver_name = "geode-aes",
.cra_priority = 300,
.cra_alignmask = 15,
.cra_flags = CRYPTO_ALG_TYPE_CIPHER |
CRYPTO_ALG_NEED_FALLBACK,
.cra_init = fallback_init_cip,
.cra_exit = fallback_exit_cip,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct geode_aes_op),
.cra_module = THIS_MODULE,
.cra_u = {
.cipher = {
.cia_min_keysize = AES_MIN_KEY_SIZE,
.cia_max_keysize = AES_MAX_KEY_SIZE,
.cia_setkey = geode_setkey_cip,
.cia_encrypt = geode_encrypt,
.cia_decrypt = geode_decrypt
}
}
};
static int
geode_cbc_decrypt(struct blkcipher_desc *desc,
struct scatterlist *dst, struct scatterlist *src,
unsigned int nbytes)
{
struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk walk;
int err, ret;
if (unlikely(op->keylen != AES_KEYSIZE_128))
return fallback_blk_dec(desc, dst, src, nbytes);
blkcipher_walk_init(&walk, dst, src, nbytes);
err = blkcipher_walk_virt(desc, &walk);
op->iv = walk.iv;
while ((nbytes = walk.nbytes)) {
op->src = walk.src.virt.addr,
op->dst = walk.dst.virt.addr;
op->mode = AES_MODE_CBC;
op->len = nbytes - (nbytes % AES_BLOCK_SIZE);
op->dir = AES_DIR_DECRYPT;
ret = geode_aes_crypt(op);
nbytes -= ret;
err = blkcipher_walk_done(desc, &walk, nbytes);
}
return err;
}
static int
geode_cbc_encrypt(struct blkcipher_desc *desc,
struct scatterlist *dst, struct scatterlist *src,
unsigned int nbytes)
{
struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk walk;
int err, ret;
if (unlikely(op->keylen != AES_KEYSIZE_128))
return fallback_blk_enc(desc, dst, src, nbytes);
blkcipher_walk_init(&walk, dst, src, nbytes);
err = blkcipher_walk_virt(desc, &walk);
op->iv = walk.iv;
while ((nbytes = walk.nbytes)) {
op->src = walk.src.virt.addr,
op->dst = walk.dst.virt.addr;
op->mode = AES_MODE_CBC;
op->len = nbytes - (nbytes % AES_BLOCK_SIZE);
op->dir = AES_DIR_ENCRYPT;
ret = geode_aes_crypt(op);
nbytes -= ret;
err = blkcipher_walk_done(desc, &walk, nbytes);
}
return err;
}
static int fallback_init_blk(struct crypto_tfm *tfm)
{
const char *name = crypto_tfm_alg_name(tfm);
struct geode_aes_op *op = crypto_tfm_ctx(tfm);
op->fallback.blk = crypto_alloc_blkcipher(name, 0,
CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
if (IS_ERR(op->fallback.blk)) {
printk(KERN_ERR "Error allocating fallback algo %s\n", name);
return PTR_ERR(op->fallback.blk);
}
return 0;
}
static void fallback_exit_blk(struct crypto_tfm *tfm)
{
struct geode_aes_op *op = crypto_tfm_ctx(tfm);
crypto_free_blkcipher(op->fallback.blk);
op->fallback.blk = NULL;
}
static struct crypto_alg geode_cbc_alg = {
.cra_name = "cbc(aes)",
.cra_driver_name = "cbc-aes-geode",
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_NEED_FALLBACK,
.cra_init = fallback_init_blk,
.cra_exit = fallback_exit_blk,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct geode_aes_op),
.cra_alignmask = 15,
.cra_type = &crypto_blkcipher_type,
.cra_module = THIS_MODULE,
.cra_u = {
.blkcipher = {
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.setkey = geode_setkey_blk,
.encrypt = geode_cbc_encrypt,
.decrypt = geode_cbc_decrypt,
.ivsize = AES_BLOCK_SIZE,
}
}
};
static int
geode_ecb_decrypt(struct blkcipher_desc *desc,
struct scatterlist *dst, struct scatterlist *src,
unsigned int nbytes)
{
struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk walk;
int err, ret;
if (unlikely(op->keylen != AES_KEYSIZE_128))
return fallback_blk_dec(desc, dst, src, nbytes);
blkcipher_walk_init(&walk, dst, src, nbytes);
err = blkcipher_walk_virt(desc, &walk);
while ((nbytes = walk.nbytes)) {
op->src = walk.src.virt.addr,
op->dst = walk.dst.virt.addr;
op->mode = AES_MODE_ECB;
op->len = nbytes - (nbytes % AES_BLOCK_SIZE);
op->dir = AES_DIR_DECRYPT;
ret = geode_aes_crypt(op);
nbytes -= ret;
err = blkcipher_walk_done(desc, &walk, nbytes);
}
return err;
}
static int
geode_ecb_encrypt(struct blkcipher_desc *desc,
struct scatterlist *dst, struct scatterlist *src,
unsigned int nbytes)
{
struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk walk;
int err, ret;
if (unlikely(op->keylen != AES_KEYSIZE_128))
return fallback_blk_enc(desc, dst, src, nbytes);
blkcipher_walk_init(&walk, dst, src, nbytes);
err = blkcipher_walk_virt(desc, &walk);
while ((nbytes = walk.nbytes)) {
op->src = walk.src.virt.addr,
op->dst = walk.dst.virt.addr;
op->mode = AES_MODE_ECB;
op->len = nbytes - (nbytes % AES_BLOCK_SIZE);
op->dir = AES_DIR_ENCRYPT;
ret = geode_aes_crypt(op);
nbytes -= ret;
ret = blkcipher_walk_done(desc, &walk, nbytes);
}
return err;
}
static struct crypto_alg geode_ecb_alg = {
.cra_name = "ecb(aes)",
.cra_driver_name = "ecb-aes-geode",
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_NEED_FALLBACK,
.cra_init = fallback_init_blk,
.cra_exit = fallback_exit_blk,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct geode_aes_op),
.cra_alignmask = 15,
.cra_type = &crypto_blkcipher_type,
.cra_module = THIS_MODULE,
.cra_u = {
.blkcipher = {
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.setkey = geode_setkey_blk,
.encrypt = geode_ecb_encrypt,
.decrypt = geode_ecb_decrypt,
}
}
};
static void geode_aes_remove(struct pci_dev *dev)
{
crypto_unregister_alg(&geode_alg);
crypto_unregister_alg(&geode_ecb_alg);
crypto_unregister_alg(&geode_cbc_alg);
pci_iounmap(dev, _iobase);
_iobase = NULL;
pci_release_regions(dev);
pci_disable_device(dev);
}
static int geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id)
{
int ret;
ret = pci_enable_device(dev);
if (ret)
return ret;
ret = pci_request_regions(dev, "geode-aes");
if (ret)
goto eenable;
_iobase = pci_iomap(dev, 0, 0);
if (_iobase == NULL) {
ret = -ENOMEM;
goto erequest;
}
spin_lock_init(&lock);
/* Clear any pending activity */
iowrite32(AES_INTR_PENDING | AES_INTR_MASK, _iobase + AES_INTR_REG);
ret = crypto_register_alg(&geode_alg);
if (ret)
goto eiomap;
ret = crypto_register_alg(&geode_ecb_alg);
if (ret)
goto ealg;
ret = crypto_register_alg(&geode_cbc_alg);
if (ret)
goto eecb;
dev_notice(&dev->dev, "GEODE AES engine enabled.\n");
return 0;
eecb:
crypto_unregister_alg(&geode_ecb_alg);
ealg:
crypto_unregister_alg(&geode_alg);
eiomap:
pci_iounmap(dev, _iobase);
erequest:
pci_release_regions(dev);
eenable:
pci_disable_device(dev);
dev_err(&dev->dev, "GEODE AES initialization failed.\n");
return ret;
}
static struct pci_device_id geode_aes_tbl[] = {
{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_LX_AES), } ,
{ 0, }
};
MODULE_DEVICE_TABLE(pci, geode_aes_tbl);
static struct pci_driver geode_aes_driver = {
.name = "Geode LX AES",
.id_table = geode_aes_tbl,
.probe = geode_aes_probe,
.remove = geode_aes_remove,
};
module_pci_driver(geode_aes_driver);
MODULE_AUTHOR("Advanced Micro Devices, Inc.");
MODULE_DESCRIPTION("Geode LX Hardware AES driver");
MODULE_LICENSE("GPL");

73
drivers/crypto/geode-aes.h Normal file
View file

@ -0,0 +1,73 @@
/* Copyright (C) 2003-2006, Advanced Micro Devices, Inc.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*/
#ifndef _GEODE_AES_H_
#define _GEODE_AES_H_
/* driver logic flags */
#define AES_MODE_ECB 0
#define AES_MODE_CBC 1
#define AES_DIR_DECRYPT 0
#define AES_DIR_ENCRYPT 1
#define AES_FLAGS_HIDDENKEY (1 << 0)
/* Register definitions */
#define AES_CTRLA_REG 0x0000
#define AES_CTRL_START 0x01
#define AES_CTRL_DECRYPT 0x00
#define AES_CTRL_ENCRYPT 0x02
#define AES_CTRL_WRKEY 0x04
#define AES_CTRL_DCA 0x08
#define AES_CTRL_SCA 0x10
#define AES_CTRL_CBC 0x20
#define AES_INTR_REG 0x0008
#define AES_INTRA_PENDING (1 << 16)
#define AES_INTRB_PENDING (1 << 17)
#define AES_INTR_PENDING (AES_INTRA_PENDING | AES_INTRB_PENDING)
#define AES_INTR_MASK 0x07
#define AES_SOURCEA_REG 0x0010
#define AES_DSTA_REG 0x0014
#define AES_LENA_REG 0x0018
#define AES_WRITEKEY0_REG 0x0030
#define AES_WRITEIV0_REG 0x0040
/* A very large counter that is used to gracefully bail out of an
* operation in case of trouble
*/
#define AES_OP_TIMEOUT 0x50000
struct geode_aes_op {
void *src;
void *dst;
u32 mode;
u32 dir;
u32 flags;
int len;
u8 key[AES_KEYSIZE_128];
u8 *iv;
union {
struct crypto_blkcipher *blk;
struct crypto_cipher *cip;
} fallback;
u32 keylen;
};
#endif

2801
drivers/crypto/hifn_795x.c Normal file
View file

File diff suppressed because it is too large Load diff

1498
drivers/crypto/ixp4xx_crypto.c Normal file
View file

File diff suppressed because it is too large Load diff

1194
drivers/crypto/mv_cesa.c Normal file
View file

File diff suppressed because it is too large Load diff

150
drivers/crypto/mv_cesa.h Normal file
View file

@ -0,0 +1,150 @@
#ifndef __MV_CRYPTO_H__
#define __MV_CRYPTO_H__
#define DIGEST_INITIAL_VAL_A 0xdd00
#define DIGEST_INITIAL_VAL_B 0xdd04
#define DIGEST_INITIAL_VAL_C 0xdd08
#define DIGEST_INITIAL_VAL_D 0xdd0c
#define DIGEST_INITIAL_VAL_E 0xdd10
#define DES_CMD_REG 0xdd58
#define SEC_ACCEL_CMD 0xde00
#define SEC_CMD_EN_SEC_ACCL0 (1 << 0)
#define SEC_CMD_EN_SEC_ACCL1 (1 << 1)
#define SEC_CMD_DISABLE_SEC (1 << 2)
#define SEC_ACCEL_DESC_P0 0xde04
#define SEC_DESC_P0_PTR(x) (x)
#define SEC_ACCEL_DESC_P1 0xde14
#define SEC_DESC_P1_PTR(x) (x)
#define SEC_ACCEL_CFG 0xde08
#define SEC_CFG_STOP_DIG_ERR (1 << 0)
#define SEC_CFG_CH0_W_IDMA (1 << 7)
#define SEC_CFG_CH1_W_IDMA (1 << 8)
#define SEC_CFG_ACT_CH0_IDMA (1 << 9)
#define SEC_CFG_ACT_CH1_IDMA (1 << 10)
#define SEC_ACCEL_STATUS 0xde0c
#define SEC_ST_ACT_0 (1 << 0)
#define SEC_ST_ACT_1 (1 << 1)
/*
* FPGA_INT_STATUS looks like a FPGA leftover and is documented only in Errata
* 4.12. It looks like that it was part of an IRQ-controller in FPGA and
* someone forgot to remove it while switching to the core and moving to
* SEC_ACCEL_INT_STATUS.
*/
#define FPGA_INT_STATUS 0xdd68
#define SEC_ACCEL_INT_STATUS 0xde20
#define SEC_INT_AUTH_DONE (1 << 0)
#define SEC_INT_DES_E_DONE (1 << 1)
#define SEC_INT_AES_E_DONE (1 << 2)
#define SEC_INT_AES_D_DONE (1 << 3)
#define SEC_INT_ENC_DONE (1 << 4)
#define SEC_INT_ACCEL0_DONE (1 << 5)
#define SEC_INT_ACCEL1_DONE (1 << 6)
#define SEC_INT_ACC0_IDMA_DONE (1 << 7)
#define SEC_INT_ACC1_IDMA_DONE (1 << 8)
#define SEC_ACCEL_INT_MASK 0xde24
#define AES_KEY_LEN (8 * 4)
struct sec_accel_config {
u32 config;
#define CFG_OP_MAC_ONLY 0
#define CFG_OP_CRYPT_ONLY 1
#define CFG_OP_MAC_CRYPT 2
#define CFG_OP_CRYPT_MAC 3
#define CFG_MACM_MD5 (4 << 4)
#define CFG_MACM_SHA1 (5 << 4)
#define CFG_MACM_HMAC_MD5 (6 << 4)
#define CFG_MACM_HMAC_SHA1 (7 << 4)
#define CFG_ENCM_DES (1 << 8)
#define CFG_ENCM_3DES (2 << 8)
#define CFG_ENCM_AES (3 << 8)
#define CFG_DIR_ENC (0 << 12)
#define CFG_DIR_DEC (1 << 12)
#define CFG_ENC_MODE_ECB (0 << 16)
#define CFG_ENC_MODE_CBC (1 << 16)
#define CFG_3DES_EEE (0 << 20)
#define CFG_3DES_EDE (1 << 20)
#define CFG_AES_LEN_128 (0 << 24)
#define CFG_AES_LEN_192 (1 << 24)
#define CFG_AES_LEN_256 (2 << 24)
#define CFG_NOT_FRAG (0 << 30)
#define CFG_FIRST_FRAG (1 << 30)
#define CFG_LAST_FRAG (2 << 30)
#define CFG_MID_FRAG (3 << 30)
u32 enc_p;
#define ENC_P_SRC(x) (x)
#define ENC_P_DST(x) ((x) << 16)
u32 enc_len;
#define ENC_LEN(x) (x)
u32 enc_key_p;
#define ENC_KEY_P(x) (x)
u32 enc_iv;
#define ENC_IV_POINT(x) ((x) << 0)
#define ENC_IV_BUF_POINT(x) ((x) << 16)
u32 mac_src_p;
#define MAC_SRC_DATA_P(x) (x)
#define MAC_SRC_TOTAL_LEN(x) ((x) << 16)
u32 mac_digest;
#define MAC_DIGEST_P(x) (x)
#define MAC_FRAG_LEN(x) ((x) << 16)
u32 mac_iv;
#define MAC_INNER_IV_P(x) (x)
#define MAC_OUTER_IV_P(x) ((x) << 16)
}__attribute__ ((packed));
/*
* /-----------\ 0
* | ACCEL CFG | 4 * 8
* |-----------| 0x20
* | CRYPT KEY | 8 * 4
* |-----------| 0x40
* | IV IN | 4 * 4
* |-----------| 0x40 (inplace)
* | IV BUF | 4 * 4
* |-----------| 0x80
* | DATA IN | 16 * x (max ->max_req_size)
* |-----------| 0x80 (inplace operation)
* | DATA OUT | 16 * x (max ->max_req_size)
* \-----------/ SRAM size
*/
/* Hashing memory map:
* /-----------\ 0
* | ACCEL CFG | 4 * 8
* |-----------| 0x20
* | Inner IV | 5 * 4
* |-----------| 0x34
* | Outer IV | 5 * 4
* |-----------| 0x48
* | Output BUF| 5 * 4
* |-----------| 0x80
* | DATA IN | 64 * x (max ->max_req_size)
* \-----------/ SRAM size
*/
#define SRAM_CONFIG 0x00
#define SRAM_DATA_KEY_P 0x20
#define SRAM_DATA_IV 0x40
#define SRAM_DATA_IV_BUF 0x40
#define SRAM_DATA_IN_START 0x80
#define SRAM_DATA_OUT_START 0x80
#define SRAM_HMAC_IV_IN 0x20
#define SRAM_HMAC_IV_OUT 0x34
#define SRAM_DIGEST_BUF 0x48
#define SRAM_CFG_SPACE 0x80
#endif

1103
drivers/crypto/mxs-dcp.c Normal file
View file

File diff suppressed because it is too large Load diff

95
drivers/crypto/n2_asm.S Normal file
View file

@ -0,0 +1,95 @@
/* n2_asm.S: Hypervisor calls for NCS support.
*
* Copyright (C) 2009 David S. Miller <davem@davemloft.net>
*/
#include <linux/linkage.h>
#include <asm/hypervisor.h>
#include "n2_core.h"
/* o0: queue type
* o1: RA of queue
* o2: num entries in queue
* o3: address of queue handle return
*/
ENTRY(sun4v_ncs_qconf)
mov HV_FAST_NCS_QCONF, %o5
ta HV_FAST_TRAP
stx %o1, [%o3]
retl
nop
ENDPROC(sun4v_ncs_qconf)
/* %o0: queue handle
* %o1: address of queue type return
* %o2: address of queue base address return
* %o3: address of queue num entries return
*/
ENTRY(sun4v_ncs_qinfo)
mov %o1, %g1
mov %o2, %g2
mov %o3, %g3
mov HV_FAST_NCS_QINFO, %o5
ta HV_FAST_TRAP
stx %o1, [%g1]
stx %o2, [%g2]
stx %o3, [%g3]
retl
nop
ENDPROC(sun4v_ncs_qinfo)
/* %o0: queue handle
* %o1: address of head offset return
*/
ENTRY(sun4v_ncs_gethead)
mov %o1, %o2
mov HV_FAST_NCS_GETHEAD, %o5
ta HV_FAST_TRAP
stx %o1, [%o2]
retl
nop
ENDPROC(sun4v_ncs_gethead)
/* %o0: queue handle
* %o1: address of tail offset return
*/
ENTRY(sun4v_ncs_gettail)
mov %o1, %o2
mov HV_FAST_NCS_GETTAIL, %o5
ta HV_FAST_TRAP
stx %o1, [%o2]
retl
nop
ENDPROC(sun4v_ncs_gettail)
/* %o0: queue handle
* %o1: new tail offset
*/
ENTRY(sun4v_ncs_settail)
mov HV_FAST_NCS_SETTAIL, %o5
ta HV_FAST_TRAP
retl
nop
ENDPROC(sun4v_ncs_settail)
/* %o0: queue handle
* %o1: address of devino return
*/
ENTRY(sun4v_ncs_qhandle_to_devino)
mov %o1, %o2
mov HV_FAST_NCS_QHANDLE_TO_DEVINO, %o5
ta HV_FAST_TRAP
stx %o1, [%o2]
retl
nop
ENDPROC(sun4v_ncs_qhandle_to_devino)
/* %o0: queue handle
* %o1: new head offset
*/
ENTRY(sun4v_ncs_sethead_marker)
mov HV_FAST_NCS_SETHEAD_MARKER, %o5
ta HV_FAST_TRAP
retl
nop
ENDPROC(sun4v_ncs_sethead_marker)

2267
drivers/crypto/n2_core.c Normal file
View file

File diff suppressed because it is too large Load diff

231
drivers/crypto/n2_core.h Normal file
View file

@ -0,0 +1,231 @@
#ifndef _N2_CORE_H
#define _N2_CORE_H
#ifndef __ASSEMBLY__
struct ino_blob {
u64 intr;
u64 ino;
};
struct spu_mdesc_info {
u64 cfg_handle;
struct ino_blob *ino_table;
int num_intrs;
};
struct n2_crypto {
struct spu_mdesc_info cwq_info;
struct list_head cwq_list;
};
struct n2_mau {
struct spu_mdesc_info mau_info;
struct list_head mau_list;
};
#define CWQ_ENTRY_SIZE 64
#define CWQ_NUM_ENTRIES 64
#define MAU_ENTRY_SIZE 64
#define MAU_NUM_ENTRIES 64
struct cwq_initial_entry {
u64 control;
u64 src_addr;
u64 auth_key_addr;
u64 auth_iv_addr;
u64 final_auth_state_addr;
u64 enc_key_addr;
u64 enc_iv_addr;
u64 dest_addr;
};
struct cwq_ext_entry {
u64 len;
u64 src_addr;
u64 resv1;
u64 resv2;
u64 resv3;
u64 resv4;
u64 resv5;
u64 resv6;
};
struct cwq_final_entry {
u64 control;
u64 src_addr;
u64 resv1;
u64 resv2;
u64 resv3;
u64 resv4;
u64 resv5;
u64 resv6;
};
#define CONTROL_LEN 0x000000000000ffffULL
#define CONTROL_LEN_SHIFT 0
#define CONTROL_HMAC_KEY_LEN 0x0000000000ff0000ULL
#define CONTROL_HMAC_KEY_LEN_SHIFT 16
#define CONTROL_ENC_TYPE 0x00000000ff000000ULL
#define CONTROL_ENC_TYPE_SHIFT 24
#define ENC_TYPE_ALG_RC4_STREAM 0x00ULL
#define ENC_TYPE_ALG_RC4_NOSTREAM 0x04ULL
#define ENC_TYPE_ALG_DES 0x08ULL
#define ENC_TYPE_ALG_3DES 0x0cULL
#define ENC_TYPE_ALG_AES128 0x10ULL
#define ENC_TYPE_ALG_AES192 0x14ULL
#define ENC_TYPE_ALG_AES256 0x18ULL
#define ENC_TYPE_ALG_RESERVED 0x1cULL
#define ENC_TYPE_ALG_MASK 0x1cULL
#define ENC_TYPE_CHAINING_ECB 0x00ULL
#define ENC_TYPE_CHAINING_CBC 0x01ULL
#define ENC_TYPE_CHAINING_CFB 0x02ULL
#define ENC_TYPE_CHAINING_COUNTER 0x03ULL
#define ENC_TYPE_CHAINING_MASK 0x03ULL
#define CONTROL_AUTH_TYPE 0x0000001f00000000ULL
#define CONTROL_AUTH_TYPE_SHIFT 32
#define AUTH_TYPE_RESERVED 0x00ULL
#define AUTH_TYPE_MD5 0x01ULL
#define AUTH_TYPE_SHA1 0x02ULL
#define AUTH_TYPE_SHA256 0x03ULL
#define AUTH_TYPE_CRC32 0x04ULL
#define AUTH_TYPE_HMAC_MD5 0x05ULL
#define AUTH_TYPE_HMAC_SHA1 0x06ULL
#define AUTH_TYPE_HMAC_SHA256 0x07ULL
#define AUTH_TYPE_TCP_CHECKSUM 0x08ULL
#define AUTH_TYPE_SSL_HMAC_MD5 0x09ULL
#define AUTH_TYPE_SSL_HMAC_SHA1 0x0aULL
#define AUTH_TYPE_SSL_HMAC_SHA256 0x0bULL
#define CONTROL_STRAND 0x000000e000000000ULL
#define CONTROL_STRAND_SHIFT 37
#define CONTROL_HASH_LEN 0x0000ff0000000000ULL
#define CONTROL_HASH_LEN_SHIFT 40
#define CONTROL_INTERRUPT 0x0001000000000000ULL
#define CONTROL_STORE_FINAL_AUTH_STATE 0x0002000000000000ULL
#define CONTROL_RESERVED 0x001c000000000000ULL
#define CONTROL_HV_DONE 0x0004000000000000ULL
#define CONTROL_HV_PROTOCOL_ERROR 0x0008000000000000ULL
#define CONTROL_HV_HARDWARE_ERROR 0x0010000000000000ULL
#define CONTROL_END_OF_BLOCK 0x0020000000000000ULL
#define CONTROL_START_OF_BLOCK 0x0040000000000000ULL
#define CONTROL_ENCRYPT 0x0080000000000000ULL
#define CONTROL_OPCODE 0xff00000000000000ULL
#define CONTROL_OPCODE_SHIFT 56
#define OPCODE_INPLACE_BIT 0x80ULL
#define OPCODE_SSL_KEYBLOCK 0x10ULL
#define OPCODE_COPY 0x20ULL
#define OPCODE_ENCRYPT 0x40ULL
#define OPCODE_AUTH_MAC 0x41ULL
#endif /* !(__ASSEMBLY__) */
/* NCS v2.0 hypervisor interfaces */
#define HV_NCS_QTYPE_MAU 0x01
#define HV_NCS_QTYPE_CWQ 0x02
/* ncs_qconf()
* TRAP: HV_FAST_TRAP
* FUNCTION: HV_FAST_NCS_QCONF
* ARG0: Queue type (HV_NCS_QTYPE_{MAU,CWQ})
* ARG1: Real address of queue, or handle for unconfigure
* ARG2: Number of entries in queue, zero for unconfigure
* RET0: status
* RET1: queue handle
*
* Configure a queue in the stream processing unit.
*
* The real address given as the base must be 64-byte
* aligned.
*
* The queue size can range from a minimum of 2 to a maximum
* of 64. The queue size must be a power of two.
*
* To unconfigure a queue, specify a length of zero and place
* the queue handle into ARG1.
*
* On configure success the hypervisor will set the FIRST, HEAD,
* and TAIL registers to the address of the first entry in the
* queue. The LAST register will be set to point to the last
* entry in the queue.
*/
#define HV_FAST_NCS_QCONF 0x111
/* ncs_qinfo()
* TRAP: HV_FAST_TRAP
* FUNCTION: HV_FAST_NCS_QINFO
* ARG0: Queue handle
* RET0: status
* RET1: Queue type (HV_NCS_QTYPE_{MAU,CWQ})
* RET2: Queue base address
* RET3: Number of entries
*/
#define HV_FAST_NCS_QINFO 0x112
/* ncs_gethead()
* TRAP: HV_FAST_TRAP
* FUNCTION: HV_FAST_NCS_GETHEAD
* ARG0: Queue handle
* RET0: status
* RET1: queue head offset
*/
#define HV_FAST_NCS_GETHEAD 0x113
/* ncs_gettail()
* TRAP: HV_FAST_TRAP
* FUNCTION: HV_FAST_NCS_GETTAIL
* ARG0: Queue handle
* RET0: status
* RET1: queue tail offset
*/
#define HV_FAST_NCS_GETTAIL 0x114
/* ncs_settail()
* TRAP: HV_FAST_TRAP
* FUNCTION: HV_FAST_NCS_SETTAIL
* ARG0: Queue handle
* ARG1: New tail offset
* RET0: status
*/
#define HV_FAST_NCS_SETTAIL 0x115
/* ncs_qhandle_to_devino()
* TRAP: HV_FAST_TRAP
* FUNCTION: HV_FAST_NCS_QHANDLE_TO_DEVINO
* ARG0: Queue handle
* RET0: status
* RET1: devino
*/
#define HV_FAST_NCS_QHANDLE_TO_DEVINO 0x116
/* ncs_sethead_marker()
* TRAP: HV_FAST_TRAP
* FUNCTION: HV_FAST_NCS_SETHEAD_MARKER
* ARG0: Queue handle
* ARG1: New head offset
* RET0: status
*/
#define HV_FAST_NCS_SETHEAD_MARKER 0x117
#ifndef __ASSEMBLY__
extern unsigned long sun4v_ncs_qconf(unsigned long queue_type,
unsigned long queue_ra,
unsigned long num_entries,
unsigned long *qhandle);
extern unsigned long sun4v_ncs_qinfo(unsigned long qhandle,
unsigned long *queue_type,
unsigned long *queue_ra,
unsigned long *num_entries);
extern unsigned long sun4v_ncs_gethead(unsigned long qhandle,
unsigned long *head);
extern unsigned long sun4v_ncs_gettail(unsigned long qhandle,
unsigned long *tail);
extern unsigned long sun4v_ncs_settail(unsigned long qhandle,
unsigned long tail);
extern unsigned long sun4v_ncs_qhandle_to_devino(unsigned long qhandle,
unsigned long *devino);
extern unsigned long sun4v_ncs_sethead_marker(unsigned long qhandle,
unsigned long head);
#endif /* !(__ASSEMBLY__) */
#endif /* _N2_CORE_H */

26
drivers/crypto/nx/Kconfig Normal file
View file

@ -0,0 +1,26 @@
config CRYPTO_DEV_NX_ENCRYPT
tristate "Encryption acceleration support"
depends on PPC64 && IBMVIO
default y
select CRYPTO_AES
select CRYPTO_CBC
select CRYPTO_ECB
select CRYPTO_CCM
select CRYPTO_GCM
select CRYPTO_AUTHENC
select CRYPTO_XCBC
select CRYPTO_SHA256
select CRYPTO_SHA512
help
Support for Power7+ in-Nest encryption acceleration. This
module supports acceleration for AES and SHA2 algorithms. If you
choose 'M' here, this module will be called nx_crypto.
config CRYPTO_DEV_NX_COMPRESS
tristate "Compression acceleration support"
depends on PPC64 && IBMVIO
default y
help
Support for Power7+ in-Nest compression acceleration. This
module supports acceleration for AES and SHA2 algorithms. If you
choose 'M' here, this module will be called nx_compress.

14
drivers/crypto/nx/Makefile Normal file
View file

@ -0,0 +1,14 @@
obj-$(CONFIG_CRYPTO_DEV_NX_ENCRYPT) += nx-crypto.o
nx-crypto-objs := nx.o \
nx_debugfs.o \
nx-aes-cbc.o \
nx-aes-ecb.o \
nx-aes-gcm.o \
nx-aes-ccm.o \
nx-aes-ctr.o \
nx-aes-xcbc.o \
nx-sha256.o \
nx-sha512.o
obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS) += nx-compress.o
nx-compress-objs := nx-842.o

1603
drivers/crypto/nx/nx-842.c Normal file
View file

File diff suppressed because it is too large Load diff

158
drivers/crypto/nx/nx-aes-cbc.c Normal file
View file

@ -0,0 +1,158 @@
/**
* AES CBC routines supporting the Power 7+ Nest Accelerators driver
*
* Copyright (C) 2011-2012 International Business Machines Inc.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; version 2 only.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*
* Author: Kent Yoder <yoder1@us.ibm.com>
*/
#include <crypto/aes.h>
#include <crypto/algapi.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <asm/vio.h>
#include "nx_csbcpb.h"
#include "nx.h"
static int cbc_aes_nx_set_key(struct crypto_tfm *tfm,
const u8 *in_key,
unsigned int key_len)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
nx_ctx_init(nx_ctx, HCOP_FC_AES);
switch (key_len) {
case AES_KEYSIZE_128:
NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128);
nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128];
break;
case AES_KEYSIZE_192:
NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_192);
nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_192];
break;
case AES_KEYSIZE_256:
NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_256);
nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_256];
break;
default:
return -EINVAL;
}
csbcpb->cpb.hdr.mode = NX_MODE_AES_CBC;
memcpy(csbcpb->cpb.aes_cbc.key, in_key, key_len);
return 0;
}
static int cbc_aes_nx_crypt(struct blkcipher_desc *desc,
struct scatterlist *dst,
struct scatterlist *src,
unsigned int nbytes,
int enc)
{
struct nx_crypto_ctx *nx_ctx = crypto_blkcipher_ctx(desc->tfm);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
unsigned long irq_flags;
unsigned int processed = 0, to_process;
u32 max_sg_len;
int rc;
spin_lock_irqsave(&nx_ctx->lock, irq_flags);
max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
nx_ctx->ap->sglen);
if (enc)
NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
else
NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
do {
to_process = min_t(u64, nbytes - processed,
nx_ctx->ap->databytelen);
to_process = min_t(u64, to_process,
NX_PAGE_SIZE * (max_sg_len - 1));
to_process = to_process & ~(AES_BLOCK_SIZE - 1);
rc = nx_build_sg_lists(nx_ctx, desc, dst, src, to_process,
processed, csbcpb->cpb.aes_cbc.iv);
if (rc)
goto out;
if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
rc = -EINVAL;
goto out;
}
rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
if (rc)
goto out;
memcpy(desc->info, csbcpb->cpb.aes_cbc.cv, AES_BLOCK_SIZE);
atomic_inc(&(nx_ctx->stats->aes_ops));
atomic64_add(csbcpb->csb.processed_byte_count,
&(nx_ctx->stats->aes_bytes));
processed += to_process;
} while (processed < nbytes);
out:
spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
static int cbc_aes_nx_encrypt(struct blkcipher_desc *desc,
struct scatterlist *dst,
struct scatterlist *src,
unsigned int nbytes)
{
return cbc_aes_nx_crypt(desc, dst, src, nbytes, 1);
}
static int cbc_aes_nx_decrypt(struct blkcipher_desc *desc,
struct scatterlist *dst,
struct scatterlist *src,
unsigned int nbytes)
{
return cbc_aes_nx_crypt(desc, dst, src, nbytes, 0);
}
struct crypto_alg nx_cbc_aes_alg = {
.cra_name = "cbc(aes)",
.cra_driver_name = "cbc-aes-nx",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct nx_crypto_ctx),
.cra_type = &crypto_blkcipher_type,
.cra_alignmask = 0xf,
.cra_module = THIS_MODULE,
.cra_init = nx_crypto_ctx_aes_cbc_init,
.cra_exit = nx_crypto_ctx_exit,
.cra_blkcipher = {
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
.setkey = cbc_aes_nx_set_key,
.encrypt = cbc_aes_nx_encrypt,
.decrypt = cbc_aes_nx_decrypt,
}
};

611
drivers/crypto/nx/nx-aes-ccm.c Normal file
View file

@ -0,0 +1,611 @@
/**
* AES CCM routines supporting the Power 7+ Nest Accelerators driver
*
* Copyright (C) 2012 International Business Machines Inc.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; version 2 only.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*
* Author: Kent Yoder <yoder1@us.ibm.com>
*/
#include <crypto/internal/aead.h>
#include <crypto/aes.h>
#include <crypto/algapi.h>
#include <crypto/scatterwalk.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <asm/vio.h>
#include "nx_csbcpb.h"
#include "nx.h"
static int ccm_aes_nx_set_key(struct crypto_aead *tfm,
const u8 *in_key,
unsigned int key_len)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead;
nx_ctx_init(nx_ctx, HCOP_FC_AES);
switch (key_len) {
case AES_KEYSIZE_128:
NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128);
NX_CPB_SET_KEY_SIZE(csbcpb_aead, NX_KS_AES_128);
nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128];
break;
default:
return -EINVAL;
}
csbcpb->cpb.hdr.mode = NX_MODE_AES_CCM;
memcpy(csbcpb->cpb.aes_ccm.key, in_key, key_len);
csbcpb_aead->cpb.hdr.mode = NX_MODE_AES_CCA;
memcpy(csbcpb_aead->cpb.aes_cca.key, in_key, key_len);
return 0;
}
static int ccm4309_aes_nx_set_key(struct crypto_aead *tfm,
const u8 *in_key,
unsigned int key_len)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base);
if (key_len < 3)
return -EINVAL;
key_len -= 3;
memcpy(nx_ctx->priv.ccm.nonce, in_key + key_len, 3);
return ccm_aes_nx_set_key(tfm, in_key, key_len);
}
static int ccm_aes_nx_setauthsize(struct crypto_aead *tfm,
unsigned int authsize)
{
switch (authsize) {
case 4:
case 6:
case 8:
case 10:
case 12:
case 14:
case 16:
break;
default:
return -EINVAL;
}
crypto_aead_crt(tfm)->authsize = authsize;
return 0;
}
static int ccm4309_aes_nx_setauthsize(struct crypto_aead *tfm,
unsigned int authsize)
{
switch (authsize) {
case 8:
case 12:
case 16:
break;
default:
return -EINVAL;
}
crypto_aead_crt(tfm)->authsize = authsize;
return 0;
}
/* taken from crypto/ccm.c */
static int set_msg_len(u8 *block, unsigned int msglen, int csize)
{
__be32 data;
memset(block, 0, csize);
block += csize;
if (csize >= 4)
csize = 4;
else if (msglen > (unsigned int)(1 << (8 * csize)))
return -EOVERFLOW;
data = cpu_to_be32(msglen);
memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
return 0;
}
/* taken from crypto/ccm.c */
static inline int crypto_ccm_check_iv(const u8 *iv)
{
/* 2 <= L <= 8, so 1 <= L' <= 7. */
if (1 > iv[0] || iv[0] > 7)
return -EINVAL;
return 0;
}
/* based on code from crypto/ccm.c */
static int generate_b0(u8 *iv, unsigned int assoclen, unsigned int authsize,
unsigned int cryptlen, u8 *b0)
{
unsigned int l, lp, m = authsize;
int rc;
memcpy(b0, iv, 16);
lp = b0[0];
l = lp + 1;
/* set m, bits 3-5 */
*b0 |= (8 * ((m - 2) / 2));
/* set adata, bit 6, if associated data is used */
if (assoclen)
*b0 |= 64;
rc = set_msg_len(b0 + 16 - l, cryptlen, l);
return rc;
}
static int generate_pat(u8 *iv,
struct aead_request *req,
struct nx_crypto_ctx *nx_ctx,
unsigned int authsize,
unsigned int nbytes,
u8 *out)
{
struct nx_sg *nx_insg = nx_ctx->in_sg;
struct nx_sg *nx_outsg = nx_ctx->out_sg;
unsigned int iauth_len = 0;
u8 tmp[16], *b1 = NULL, *b0 = NULL, *result = NULL;
int rc;
/* zero the ctr value */
memset(iv + 15 - iv[0], 0, iv[0] + 1);
/* page 78 of nx_wb.pdf has,
* Note: RFC3610 allows the AAD data to be up to 2^64 -1 bytes
* in length. If a full message is used, the AES CCA implementation
* restricts the maximum AAD length to 2^32 -1 bytes.
* If partial messages are used, the implementation supports
* 2^64 -1 bytes maximum AAD length.
*
* However, in the cryptoapi's aead_request structure,
* assoclen is an unsigned int, thus it cannot hold a length
* value greater than 2^32 - 1.
* Thus the AAD is further constrained by this and is never
* greater than 2^32.
*/
if (!req->assoclen) {
b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0;
} else if (req->assoclen <= 14) {
/* if associated data is 14 bytes or less, we do 1 GCM
* operation on 2 AES blocks, B0 (stored in the csbcpb) and B1,
* which is fed in through the source buffers here */
b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0;
b1 = nx_ctx->priv.ccm.iauth_tag;
iauth_len = req->assoclen;
} else if (req->assoclen <= 65280) {
/* if associated data is less than (2^16 - 2^8), we construct
* B1 differently and feed in the associated data to a CCA
* operation */
b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0;
b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1;
iauth_len = 14;
} else {
b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0;
b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1;
iauth_len = 10;
}
/* generate B0 */
rc = generate_b0(iv, req->assoclen, authsize, nbytes, b0);
if (rc)
return rc;
/* generate B1:
* add control info for associated data
* RFC 3610 and NIST Special Publication 800-38C
*/
if (b1) {
memset(b1, 0, 16);
if (req->assoclen <= 65280) {
*(u16 *)b1 = (u16)req->assoclen;
scatterwalk_map_and_copy(b1 + 2, req->assoc, 0,
iauth_len, SCATTERWALK_FROM_SG);
} else {
*(u16 *)b1 = (u16)(0xfffe);
*(u32 *)&b1[2] = (u32)req->assoclen;
scatterwalk_map_and_copy(b1 + 6, req->assoc, 0,
iauth_len, SCATTERWALK_FROM_SG);
}
}
/* now copy any remaining AAD to scatterlist and call nx... */
if (!req->assoclen) {
return rc;
} else if (req->assoclen <= 14) {
nx_insg = nx_build_sg_list(nx_insg, b1, 16, nx_ctx->ap->sglen);
nx_outsg = nx_build_sg_list(nx_outsg, tmp, 16,
nx_ctx->ap->sglen);
/* inlen should be negative, indicating to phyp that its a
* pointer to an sg list */
nx_ctx->op.inlen = (nx_ctx->in_sg - nx_insg) *
sizeof(struct nx_sg);
nx_ctx->op.outlen = (nx_ctx->out_sg - nx_outsg) *
sizeof(struct nx_sg);
NX_CPB_FDM(nx_ctx->csbcpb) |= NX_FDM_ENDE_ENCRYPT;
NX_CPB_FDM(nx_ctx->csbcpb) |= NX_FDM_INTERMEDIATE;
result = nx_ctx->csbcpb->cpb.aes_ccm.out_pat_or_mac;
rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
if (rc)
return rc;
atomic_inc(&(nx_ctx->stats->aes_ops));
atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes));
} else {
u32 max_sg_len;
unsigned int processed = 0, to_process;
/* page_limit: number of sg entries that fit on one page */
max_sg_len = min_t(u32,
nx_driver.of.max_sg_len/sizeof(struct nx_sg),
nx_ctx->ap->sglen);
processed += iauth_len;
do {
to_process = min_t(u32, req->assoclen - processed,
nx_ctx->ap->databytelen);
to_process = min_t(u64, to_process,
NX_PAGE_SIZE * (max_sg_len - 1));
if ((to_process + processed) < req->assoclen) {
NX_CPB_FDM(nx_ctx->csbcpb_aead) |=
NX_FDM_INTERMEDIATE;
} else {
NX_CPB_FDM(nx_ctx->csbcpb_aead) &=
~NX_FDM_INTERMEDIATE;
}
nx_insg = nx_walk_and_build(nx_ctx->in_sg,
nx_ctx->ap->sglen,
req->assoc, processed,
to_process);
nx_ctx->op_aead.inlen = (nx_ctx->in_sg - nx_insg) *
sizeof(struct nx_sg);
result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0;
rc = nx_hcall_sync(nx_ctx, &nx_ctx->op_aead,
req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
if (rc)
return rc;
memcpy(nx_ctx->csbcpb_aead->cpb.aes_cca.b0,
nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0,
AES_BLOCK_SIZE);
NX_CPB_FDM(nx_ctx->csbcpb_aead) |= NX_FDM_CONTINUATION;
atomic_inc(&(nx_ctx->stats->aes_ops));
atomic64_add(req->assoclen,
&(nx_ctx->stats->aes_bytes));
processed += to_process;
} while (processed < req->assoclen);
result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0;
}
memcpy(out, result, AES_BLOCK_SIZE);
return rc;
}
static int ccm_nx_decrypt(struct aead_request *req,
struct blkcipher_desc *desc)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
unsigned int nbytes = req->cryptlen;
unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
struct nx_ccm_priv *priv = &nx_ctx->priv.ccm;
unsigned long irq_flags;
unsigned int processed = 0, to_process;
u32 max_sg_len;
int rc = -1;
spin_lock_irqsave(&nx_ctx->lock, irq_flags);
nbytes -= authsize;
/* copy out the auth tag to compare with later */
scatterwalk_map_and_copy(priv->oauth_tag,
req->src, nbytes, authsize,
SCATTERWALK_FROM_SG);
rc = generate_pat(desc->info, req, nx_ctx, authsize, nbytes,
csbcpb->cpb.aes_ccm.in_pat_or_b0);
if (rc)
goto out;
/* page_limit: number of sg entries that fit on one page */
max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
nx_ctx->ap->sglen);
do {
/* to_process: the AES_BLOCK_SIZE data chunk to process in this
* update. This value is bound by sg list limits.
*/
to_process = min_t(u64, nbytes - processed,
nx_ctx->ap->databytelen);
to_process = min_t(u64, to_process,
NX_PAGE_SIZE * (max_sg_len - 1));
if ((to_process + processed) < nbytes)
NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
else
NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
NX_CPB_FDM(nx_ctx->csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
rc = nx_build_sg_lists(nx_ctx, desc, req->dst, req->src,
to_process, processed,
csbcpb->cpb.aes_ccm.iv_or_ctr);
if (rc)
goto out;
rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
if (rc)
goto out;
/* for partial completion, copy following for next
* entry into loop...
*/
memcpy(desc->info, csbcpb->cpb.aes_ccm.out_ctr, AES_BLOCK_SIZE);
memcpy(csbcpb->cpb.aes_ccm.in_pat_or_b0,
csbcpb->cpb.aes_ccm.out_pat_or_mac, AES_BLOCK_SIZE);
memcpy(csbcpb->cpb.aes_ccm.in_s0,
csbcpb->cpb.aes_ccm.out_s0, AES_BLOCK_SIZE);
NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
/* update stats */
atomic_inc(&(nx_ctx->stats->aes_ops));
atomic64_add(csbcpb->csb.processed_byte_count,
&(nx_ctx->stats->aes_bytes));
processed += to_process;
} while (processed < nbytes);
rc = memcmp(csbcpb->cpb.aes_ccm.out_pat_or_mac, priv->oauth_tag,
authsize) ? -EBADMSG : 0;
out:
spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
static int ccm_nx_encrypt(struct aead_request *req,
struct blkcipher_desc *desc)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
unsigned int nbytes = req->cryptlen;
unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
unsigned long irq_flags;
unsigned int processed = 0, to_process;
u32 max_sg_len;
int rc = -1;
spin_lock_irqsave(&nx_ctx->lock, irq_flags);
rc = generate_pat(desc->info, req, nx_ctx, authsize, nbytes,
csbcpb->cpb.aes_ccm.in_pat_or_b0);
if (rc)
goto out;
/* page_limit: number of sg entries that fit on one page */
max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
nx_ctx->ap->sglen);
do {
/* to process: the AES_BLOCK_SIZE data chunk to process in this
* update. This value is bound by sg list limits.
*/
to_process = min_t(u64, nbytes - processed,
nx_ctx->ap->databytelen);
to_process = min_t(u64, to_process,
NX_PAGE_SIZE * (max_sg_len - 1));
if ((to_process + processed) < nbytes)
NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
else
NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
rc = nx_build_sg_lists(nx_ctx, desc, req->dst, req->src,
to_process, processed,
csbcpb->cpb.aes_ccm.iv_or_ctr);
if (rc)
goto out;
rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
if (rc)
goto out;
/* for partial completion, copy following for next
* entry into loop...
*/
memcpy(desc->info, csbcpb->cpb.aes_ccm.out_ctr, AES_BLOCK_SIZE);
memcpy(csbcpb->cpb.aes_ccm.in_pat_or_b0,
csbcpb->cpb.aes_ccm.out_pat_or_mac, AES_BLOCK_SIZE);
memcpy(csbcpb->cpb.aes_ccm.in_s0,
csbcpb->cpb.aes_ccm.out_s0, AES_BLOCK_SIZE);
NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
/* update stats */
atomic_inc(&(nx_ctx->stats->aes_ops));
atomic64_add(csbcpb->csb.processed_byte_count,
&(nx_ctx->stats->aes_bytes));
processed += to_process;
} while (processed < nbytes);
/* copy out the auth tag */
scatterwalk_map_and_copy(csbcpb->cpb.aes_ccm.out_pat_or_mac,
req->dst, nbytes, authsize,
SCATTERWALK_TO_SG);
out:
spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
static int ccm4309_aes_nx_encrypt(struct aead_request *req)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
struct blkcipher_desc desc;
u8 *iv = nx_ctx->priv.ccm.iv;
iv[0] = 3;
memcpy(iv + 1, nx_ctx->priv.ccm.nonce, 3);
memcpy(iv + 4, req->iv, 8);
desc.info = iv;
desc.tfm = (struct crypto_blkcipher *)req->base.tfm;
return ccm_nx_encrypt(req, &desc);
}
static int ccm_aes_nx_encrypt(struct aead_request *req)
{
struct blkcipher_desc desc;
int rc;
desc.info = req->iv;
desc.tfm = (struct crypto_blkcipher *)req->base.tfm;
rc = crypto_ccm_check_iv(desc.info);
if (rc)
return rc;
return ccm_nx_encrypt(req, &desc);
}
static int ccm4309_aes_nx_decrypt(struct aead_request *req)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
struct blkcipher_desc desc;
u8 *iv = nx_ctx->priv.ccm.iv;
iv[0] = 3;
memcpy(iv + 1, nx_ctx->priv.ccm.nonce, 3);
memcpy(iv + 4, req->iv, 8);
desc.info = iv;
desc.tfm = (struct crypto_blkcipher *)req->base.tfm;
return ccm_nx_decrypt(req, &desc);
}
static int ccm_aes_nx_decrypt(struct aead_request *req)
{
struct blkcipher_desc desc;
int rc;
desc.info = req->iv;
desc.tfm = (struct crypto_blkcipher *)req->base.tfm;
rc = crypto_ccm_check_iv(desc.info);
if (rc)
return rc;
return ccm_nx_decrypt(req, &desc);
}
/* tell the block cipher walk routines that this is a stream cipher by
* setting cra_blocksize to 1. Even using blkcipher_walk_virt_block
* during encrypt/decrypt doesn't solve this problem, because it calls
* blkcipher_walk_done under the covers, which doesn't use walk->blocksize,
* but instead uses this tfm->blocksize. */
struct crypto_alg nx_ccm_aes_alg = {
.cra_name = "ccm(aes)",
.cra_driver_name = "ccm-aes-nx",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_AEAD |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct nx_crypto_ctx),
.cra_type = &crypto_aead_type,
.cra_module = THIS_MODULE,
.cra_init = nx_crypto_ctx_aes_ccm_init,
.cra_exit = nx_crypto_ctx_exit,
.cra_aead = {
.ivsize = AES_BLOCK_SIZE,
.maxauthsize = AES_BLOCK_SIZE,
.setkey = ccm_aes_nx_set_key,
.setauthsize = ccm_aes_nx_setauthsize,
.encrypt = ccm_aes_nx_encrypt,
.decrypt = ccm_aes_nx_decrypt,
}
};
struct crypto_alg nx_ccm4309_aes_alg = {
.cra_name = "rfc4309(ccm(aes))",
.cra_driver_name = "rfc4309-ccm-aes-nx",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_AEAD |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct nx_crypto_ctx),
.cra_type = &crypto_nivaead_type,
.cra_module = THIS_MODULE,
.cra_init = nx_crypto_ctx_aes_ccm_init,
.cra_exit = nx_crypto_ctx_exit,
.cra_aead = {
.ivsize = 8,
.maxauthsize = AES_BLOCK_SIZE,
.setkey = ccm4309_aes_nx_set_key,
.setauthsize = ccm4309_aes_nx_setauthsize,
.encrypt = ccm4309_aes_nx_encrypt,
.decrypt = ccm4309_aes_nx_decrypt,
.geniv = "seqiv",
}
};

194
drivers/crypto/nx/nx-aes-ctr.c Normal file
View file

@ -0,0 +1,194 @@
/**
* AES CTR routines supporting the Power 7+ Nest Accelerators driver
*
* Copyright (C) 2011-2012 International Business Machines Inc.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; version 2 only.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*
* Author: Kent Yoder <yoder1@us.ibm.com>
*/
#include <crypto/aes.h>
#include <crypto/ctr.h>
#include <crypto/algapi.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <asm/vio.h>
#include "nx_csbcpb.h"
#include "nx.h"
static int ctr_aes_nx_set_key(struct crypto_tfm *tfm,
const u8 *in_key,
unsigned int key_len)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
nx_ctx_init(nx_ctx, HCOP_FC_AES);
switch (key_len) {
case AES_KEYSIZE_128:
NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128);
nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128];
break;
case AES_KEYSIZE_192:
NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_192);
nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_192];
break;
case AES_KEYSIZE_256:
NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_256);
nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_256];
break;
default:
return -EINVAL;
}
csbcpb->cpb.hdr.mode = NX_MODE_AES_CTR;
memcpy(csbcpb->cpb.aes_ctr.key, in_key, key_len);
return 0;
}
static int ctr3686_aes_nx_set_key(struct crypto_tfm *tfm,
const u8 *in_key,
unsigned int key_len)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm);
if (key_len < CTR_RFC3686_NONCE_SIZE)
return -EINVAL;
memcpy(nx_ctx->priv.ctr.iv,
in_key + key_len - CTR_RFC3686_NONCE_SIZE,
CTR_RFC3686_NONCE_SIZE);
key_len -= CTR_RFC3686_NONCE_SIZE;
return ctr_aes_nx_set_key(tfm, in_key, key_len);
}
static int ctr_aes_nx_crypt(struct blkcipher_desc *desc,
struct scatterlist *dst,
struct scatterlist *src,
unsigned int nbytes)
{
struct nx_crypto_ctx *nx_ctx = crypto_blkcipher_ctx(desc->tfm);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
unsigned long irq_flags;
unsigned int processed = 0, to_process;
u32 max_sg_len;
int rc;
spin_lock_irqsave(&nx_ctx->lock, irq_flags);
max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
nx_ctx->ap->sglen);
do {
to_process = min_t(u64, nbytes - processed,
nx_ctx->ap->databytelen);
to_process = min_t(u64, to_process,
NX_PAGE_SIZE * (max_sg_len - 1));
to_process = to_process & ~(AES_BLOCK_SIZE - 1);
rc = nx_build_sg_lists(nx_ctx, desc, dst, src, to_process,
processed, csbcpb->cpb.aes_ctr.iv);
if (rc)
goto out;
if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
rc = -EINVAL;
goto out;
}
rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
if (rc)
goto out;
memcpy(desc->info, csbcpb->cpb.aes_cbc.cv, AES_BLOCK_SIZE);
atomic_inc(&(nx_ctx->stats->aes_ops));
atomic64_add(csbcpb->csb.processed_byte_count,
&(nx_ctx->stats->aes_bytes));
processed += to_process;
} while (processed < nbytes);
out:
spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
static int ctr3686_aes_nx_crypt(struct blkcipher_desc *desc,
struct scatterlist *dst,
struct scatterlist *src,
unsigned int nbytes)
{
struct nx_crypto_ctx *nx_ctx = crypto_blkcipher_ctx(desc->tfm);
u8 *iv = nx_ctx->priv.ctr.iv;
memcpy(iv + CTR_RFC3686_NONCE_SIZE,
desc->info, CTR_RFC3686_IV_SIZE);
iv[15] = 1;
desc->info = nx_ctx->priv.ctr.iv;
return ctr_aes_nx_crypt(desc, dst, src, nbytes);
}
struct crypto_alg nx_ctr_aes_alg = {
.cra_name = "ctr(aes)",
.cra_driver_name = "ctr-aes-nx",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct nx_crypto_ctx),
.cra_type = &crypto_blkcipher_type,
.cra_module = THIS_MODULE,
.cra_init = nx_crypto_ctx_aes_ctr_init,
.cra_exit = nx_crypto_ctx_exit,
.cra_blkcipher = {
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
.setkey = ctr_aes_nx_set_key,
.encrypt = ctr_aes_nx_crypt,
.decrypt = ctr_aes_nx_crypt,
}
};
struct crypto_alg nx_ctr3686_aes_alg = {
.cra_name = "rfc3686(ctr(aes))",
.cra_driver_name = "rfc3686-ctr-aes-nx",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct nx_crypto_ctx),
.cra_type = &crypto_blkcipher_type,
.cra_module = THIS_MODULE,
.cra_init = nx_crypto_ctx_aes_ctr_init,
.cra_exit = nx_crypto_ctx_exit,
.cra_blkcipher = {
.min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
.max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
.ivsize = CTR_RFC3686_IV_SIZE,
.geniv = "seqiv",
.setkey = ctr3686_aes_nx_set_key,
.encrypt = ctr3686_aes_nx_crypt,
.decrypt = ctr3686_aes_nx_crypt,
}
};

157
drivers/crypto/nx/nx-aes-ecb.c Normal file
View file

@ -0,0 +1,157 @@
/**
* AES ECB routines supporting the Power 7+ Nest Accelerators driver
*
* Copyright (C) 2011-2012 International Business Machines Inc.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; version 2 only.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*
* Author: Kent Yoder <yoder1@us.ibm.com>
*/
#include <crypto/aes.h>
#include <crypto/algapi.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <asm/vio.h>
#include "nx_csbcpb.h"
#include "nx.h"
static int ecb_aes_nx_set_key(struct crypto_tfm *tfm,
const u8 *in_key,
unsigned int key_len)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
nx_ctx_init(nx_ctx, HCOP_FC_AES);
switch (key_len) {
case AES_KEYSIZE_128:
NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128);
nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128];
break;
case AES_KEYSIZE_192:
NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_192);
nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_192];
break;
case AES_KEYSIZE_256:
NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_256);
nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_256];
break;
default:
return -EINVAL;
}
csbcpb->cpb.hdr.mode = NX_MODE_AES_ECB;
memcpy(csbcpb->cpb.aes_ecb.key, in_key, key_len);
return 0;
}
static int ecb_aes_nx_crypt(struct blkcipher_desc *desc,
struct scatterlist *dst,
struct scatterlist *src,
unsigned int nbytes,
int enc)
{
struct nx_crypto_ctx *nx_ctx = crypto_blkcipher_ctx(desc->tfm);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
unsigned long irq_flags;
unsigned int processed = 0, to_process;
u32 max_sg_len;
int rc;
spin_lock_irqsave(&nx_ctx->lock, irq_flags);
max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
nx_ctx->ap->sglen);
if (enc)
NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
else
NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
do {
to_process = min_t(u64, nbytes - processed,
nx_ctx->ap->databytelen);
to_process = min_t(u64, to_process,
NX_PAGE_SIZE * (max_sg_len - 1));
to_process = to_process & ~(AES_BLOCK_SIZE - 1);
rc = nx_build_sg_lists(nx_ctx, desc, dst, src, to_process,
processed, NULL);
if (rc)
goto out;
if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
rc = -EINVAL;
goto out;
}
rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
if (rc)
goto out;
atomic_inc(&(nx_ctx->stats->aes_ops));
atomic64_add(csbcpb->csb.processed_byte_count,
&(nx_ctx->stats->aes_bytes));
processed += to_process;
} while (processed < nbytes);
out:
spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
static int ecb_aes_nx_encrypt(struct blkcipher_desc *desc,
struct scatterlist *dst,
struct scatterlist *src,
unsigned int nbytes)
{
return ecb_aes_nx_crypt(desc, dst, src, nbytes, 1);
}
static int ecb_aes_nx_decrypt(struct blkcipher_desc *desc,
struct scatterlist *dst,
struct scatterlist *src,
unsigned int nbytes)
{
return ecb_aes_nx_crypt(desc, dst, src, nbytes, 0);
}
struct crypto_alg nx_ecb_aes_alg = {
.cra_name = "ecb(aes)",
.cra_driver_name = "ecb-aes-nx",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_alignmask = 0xf,
.cra_ctxsize = sizeof(struct nx_crypto_ctx),
.cra_type = &crypto_blkcipher_type,
.cra_module = THIS_MODULE,
.cra_init = nx_crypto_ctx_aes_ecb_init,
.cra_exit = nx_crypto_ctx_exit,
.cra_blkcipher = {
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.setkey = ecb_aes_nx_set_key,
.encrypt = ecb_aes_nx_encrypt,
.decrypt = ecb_aes_nx_decrypt,
}
};

517
drivers/crypto/nx/nx-aes-gcm.c Normal file
View file

@ -0,0 +1,517 @@
/**
* AES GCM routines supporting the Power 7+ Nest Accelerators driver
*
* Copyright (C) 2012 International Business Machines Inc.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; version 2 only.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*
* Author: Kent Yoder <yoder1@us.ibm.com>
*/
#include <crypto/internal/aead.h>
#include <crypto/aes.h>
#include <crypto/algapi.h>
#include <crypto/scatterwalk.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <asm/vio.h>
#include "nx_csbcpb.h"
#include "nx.h"
static int gcm_aes_nx_set_key(struct crypto_aead *tfm,
const u8 *in_key,
unsigned int key_len)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead;
nx_ctx_init(nx_ctx, HCOP_FC_AES);
switch (key_len) {
case AES_KEYSIZE_128:
NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128);
NX_CPB_SET_KEY_SIZE(csbcpb_aead, NX_KS_AES_128);
nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128];
break;
case AES_KEYSIZE_192:
NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_192);
NX_CPB_SET_KEY_SIZE(csbcpb_aead, NX_KS_AES_192);
nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_192];
break;
case AES_KEYSIZE_256:
NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_256);
NX_CPB_SET_KEY_SIZE(csbcpb_aead, NX_KS_AES_256);
nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_256];
break;
default:
return -EINVAL;
}
csbcpb->cpb.hdr.mode = NX_MODE_AES_GCM;
memcpy(csbcpb->cpb.aes_gcm.key, in_key, key_len);
csbcpb_aead->cpb.hdr.mode = NX_MODE_AES_GCA;
memcpy(csbcpb_aead->cpb.aes_gca.key, in_key, key_len);
return 0;
}
static int gcm4106_aes_nx_set_key(struct crypto_aead *tfm,
const u8 *in_key,
unsigned int key_len)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base);
char *nonce = nx_ctx->priv.gcm.nonce;
int rc;
if (key_len < 4)
return -EINVAL;
key_len -= 4;
rc = gcm_aes_nx_set_key(tfm, in_key, key_len);
if (rc)
goto out;
memcpy(nonce, in_key + key_len, 4);
out:
return rc;
}
static int gcm_aes_nx_setauthsize(struct crypto_aead *tfm,
unsigned int authsize)
{
if (authsize > crypto_aead_alg(tfm)->maxauthsize)
return -EINVAL;
crypto_aead_crt(tfm)->authsize = authsize;
return 0;
}
static int gcm4106_aes_nx_setauthsize(struct crypto_aead *tfm,
unsigned int authsize)
{
switch (authsize) {
case 8:
case 12:
case 16:
break;
default:
return -EINVAL;
}
crypto_aead_crt(tfm)->authsize = authsize;
return 0;
}
static int nx_gca(struct nx_crypto_ctx *nx_ctx,
struct aead_request *req,
u8 *out)
{
int rc;
struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead;
struct scatter_walk walk;
struct nx_sg *nx_sg = nx_ctx->in_sg;
unsigned int nbytes = req->assoclen;
unsigned int processed = 0, to_process;
u32 max_sg_len;
if (nbytes <= AES_BLOCK_SIZE) {
scatterwalk_start(&walk, req->assoc);
scatterwalk_copychunks(out, &walk, nbytes, SCATTERWALK_FROM_SG);
scatterwalk_done(&walk, SCATTERWALK_FROM_SG, 0);
return 0;
}
NX_CPB_FDM(csbcpb_aead) &= ~NX_FDM_CONTINUATION;
/* page_limit: number of sg entries that fit on one page */
max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
nx_ctx->ap->sglen);
do {
/*
* to_process: the data chunk to process in this update.
* This value is bound by sg list limits.
*/
to_process = min_t(u64, nbytes - processed,
nx_ctx->ap->databytelen);
to_process = min_t(u64, to_process,
NX_PAGE_SIZE * (max_sg_len - 1));
if ((to_process + processed) < nbytes)
NX_CPB_FDM(csbcpb_aead) |= NX_FDM_INTERMEDIATE;
else
NX_CPB_FDM(csbcpb_aead) &= ~NX_FDM_INTERMEDIATE;
nx_sg = nx_walk_and_build(nx_ctx->in_sg, nx_ctx->ap->sglen,
req->assoc, processed, to_process);
nx_ctx->op_aead.inlen = (nx_ctx->in_sg - nx_sg)
* sizeof(struct nx_sg);
rc = nx_hcall_sync(nx_ctx, &nx_ctx->op_aead,
req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
if (rc)
return rc;
memcpy(csbcpb_aead->cpb.aes_gca.in_pat,
csbcpb_aead->cpb.aes_gca.out_pat,
AES_BLOCK_SIZE);
NX_CPB_FDM(csbcpb_aead) |= NX_FDM_CONTINUATION;
atomic_inc(&(nx_ctx->stats->aes_ops));
atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes));
processed += to_process;
} while (processed < nbytes);
memcpy(out, csbcpb_aead->cpb.aes_gca.out_pat, AES_BLOCK_SIZE);
return rc;
}
static int gmac(struct aead_request *req, struct blkcipher_desc *desc)
{
int rc;
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
struct nx_sg *nx_sg;
unsigned int nbytes = req->assoclen;
unsigned int processed = 0, to_process;
u32 max_sg_len;
/* Set GMAC mode */
csbcpb->cpb.hdr.mode = NX_MODE_AES_GMAC;
NX_CPB_FDM(csbcpb) &= ~NX_FDM_CONTINUATION;
/* page_limit: number of sg entries that fit on one page */
max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
nx_ctx->ap->sglen);
/* Copy IV */
memcpy(csbcpb->cpb.aes_gcm.iv_or_cnt, desc->info, AES_BLOCK_SIZE);
do {
/*
* to_process: the data chunk to process in this update.
* This value is bound by sg list limits.
*/
to_process = min_t(u64, nbytes - processed,
nx_ctx->ap->databytelen);
to_process = min_t(u64, to_process,
NX_PAGE_SIZE * (max_sg_len - 1));
if ((to_process + processed) < nbytes)
NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
else
NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
nx_sg = nx_walk_and_build(nx_ctx->in_sg, nx_ctx->ap->sglen,
req->assoc, processed, to_process);
nx_ctx->op.inlen = (nx_ctx->in_sg - nx_sg)
* sizeof(struct nx_sg);
csbcpb->cpb.aes_gcm.bit_length_data = 0;
csbcpb->cpb.aes_gcm.bit_length_aad = 8 * nbytes;
rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
if (rc)
goto out;
memcpy(csbcpb->cpb.aes_gcm.in_pat_or_aad,
csbcpb->cpb.aes_gcm.out_pat_or_mac, AES_BLOCK_SIZE);
memcpy(csbcpb->cpb.aes_gcm.in_s0,
csbcpb->cpb.aes_gcm.out_s0, AES_BLOCK_SIZE);
NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
atomic_inc(&(nx_ctx->stats->aes_ops));
atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes));
processed += to_process;
} while (processed < nbytes);
out:
/* Restore GCM mode */
csbcpb->cpb.hdr.mode = NX_MODE_AES_GCM;
return rc;
}
static int gcm_empty(struct aead_request *req, struct blkcipher_desc *desc,
int enc)
{
int rc;
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
char out[AES_BLOCK_SIZE];
struct nx_sg *in_sg, *out_sg;
/* For scenarios where the input message is zero length, AES CTR mode
* may be used. Set the source data to be a single block (16B) of all
* zeros, and set the input IV value to be the same as the GMAC IV
* value. - nx_wb 4.8.1.3 */
/* Change to ECB mode */
csbcpb->cpb.hdr.mode = NX_MODE_AES_ECB;
memcpy(csbcpb->cpb.aes_ecb.key, csbcpb->cpb.aes_gcm.key,
sizeof(csbcpb->cpb.aes_ecb.key));
if (enc)
NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
else
NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
/* Encrypt the counter/IV */
in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *) desc->info,
AES_BLOCK_SIZE, nx_ctx->ap->sglen);
out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *) out, sizeof(out),
nx_ctx->ap->sglen);
nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
if (rc)
goto out;
atomic_inc(&(nx_ctx->stats->aes_ops));
/* Copy out the auth tag */
memcpy(csbcpb->cpb.aes_gcm.out_pat_or_mac, out,
crypto_aead_authsize(crypto_aead_reqtfm(req)));
out:
/* Restore XCBC mode */
csbcpb->cpb.hdr.mode = NX_MODE_AES_GCM;
/*
* ECB key uses the same region that GCM AAD and counter, so it's safe
* to just fill it with zeroes.
*/
memset(csbcpb->cpb.aes_ecb.key, 0, sizeof(csbcpb->cpb.aes_ecb.key));
return rc;
}
static int gcm_aes_nx_crypt(struct aead_request *req, int enc)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
struct blkcipher_desc desc;
unsigned int nbytes = req->cryptlen;
unsigned int processed = 0, to_process;
unsigned long irq_flags;
u32 max_sg_len;
int rc = -EINVAL;
spin_lock_irqsave(&nx_ctx->lock, irq_flags);
desc.info = nx_ctx->priv.gcm.iv;
/* initialize the counter */
*(u32 *)(desc.info + NX_GCM_CTR_OFFSET) = 1;
if (nbytes == 0) {
if (req->assoclen == 0)
rc = gcm_empty(req, &desc, enc);
else
rc = gmac(req, &desc);
if (rc)
goto out;
else
goto mac;
}
/* Process associated data */
csbcpb->cpb.aes_gcm.bit_length_aad = req->assoclen * 8;
if (req->assoclen) {
rc = nx_gca(nx_ctx, req, csbcpb->cpb.aes_gcm.in_pat_or_aad);
if (rc)
goto out;
}
/* Set flags for encryption */
NX_CPB_FDM(csbcpb) &= ~NX_FDM_CONTINUATION;
if (enc) {
NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
} else {
NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
nbytes -= crypto_aead_authsize(crypto_aead_reqtfm(req));
}
/* page_limit: number of sg entries that fit on one page */
max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
nx_ctx->ap->sglen);
do {
/*
* to_process: the data chunk to process in this update.
* This value is bound by sg list limits.
*/
to_process = min_t(u64, nbytes - processed,
nx_ctx->ap->databytelen);
to_process = min_t(u64, to_process,
NX_PAGE_SIZE * (max_sg_len - 1));
if ((to_process + processed) < nbytes)
NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
else
NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
csbcpb->cpb.aes_gcm.bit_length_data = nbytes * 8;
desc.tfm = (struct crypto_blkcipher *) req->base.tfm;
rc = nx_build_sg_lists(nx_ctx, &desc, req->dst,
req->src, to_process, processed,
csbcpb->cpb.aes_gcm.iv_or_cnt);
if (rc)
goto out;
rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
if (rc)
goto out;
memcpy(desc.info, csbcpb->cpb.aes_gcm.out_cnt, AES_BLOCK_SIZE);
memcpy(csbcpb->cpb.aes_gcm.in_pat_or_aad,
csbcpb->cpb.aes_gcm.out_pat_or_mac, AES_BLOCK_SIZE);
memcpy(csbcpb->cpb.aes_gcm.in_s0,
csbcpb->cpb.aes_gcm.out_s0, AES_BLOCK_SIZE);
NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
atomic_inc(&(nx_ctx->stats->aes_ops));
atomic64_add(csbcpb->csb.processed_byte_count,
&(nx_ctx->stats->aes_bytes));
processed += to_process;
} while (processed < nbytes);
mac:
if (enc) {
/* copy out the auth tag */
scatterwalk_map_and_copy(csbcpb->cpb.aes_gcm.out_pat_or_mac,
req->dst, nbytes,
crypto_aead_authsize(crypto_aead_reqtfm(req)),
SCATTERWALK_TO_SG);
} else {
u8 *itag = nx_ctx->priv.gcm.iauth_tag;
u8 *otag = csbcpb->cpb.aes_gcm.out_pat_or_mac;
scatterwalk_map_and_copy(itag, req->src, nbytes,
crypto_aead_authsize(crypto_aead_reqtfm(req)),
SCATTERWALK_FROM_SG);
rc = memcmp(itag, otag,
crypto_aead_authsize(crypto_aead_reqtfm(req))) ?
-EBADMSG : 0;
}
out:
spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
static int gcm_aes_nx_encrypt(struct aead_request *req)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
char *iv = nx_ctx->priv.gcm.iv;
memcpy(iv, req->iv, 12);
return gcm_aes_nx_crypt(req, 1);
}
static int gcm_aes_nx_decrypt(struct aead_request *req)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
char *iv = nx_ctx->priv.gcm.iv;
memcpy(iv, req->iv, 12);
return gcm_aes_nx_crypt(req, 0);
}
static int gcm4106_aes_nx_encrypt(struct aead_request *req)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
char *iv = nx_ctx->priv.gcm.iv;
char *nonce = nx_ctx->priv.gcm.nonce;
memcpy(iv, nonce, NX_GCM4106_NONCE_LEN);
memcpy(iv + NX_GCM4106_NONCE_LEN, req->iv, 8);
return gcm_aes_nx_crypt(req, 1);
}
static int gcm4106_aes_nx_decrypt(struct aead_request *req)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
char *iv = nx_ctx->priv.gcm.iv;
char *nonce = nx_ctx->priv.gcm.nonce;
memcpy(iv, nonce, NX_GCM4106_NONCE_LEN);
memcpy(iv + NX_GCM4106_NONCE_LEN, req->iv, 8);
return gcm_aes_nx_crypt(req, 0);
}
/* tell the block cipher walk routines that this is a stream cipher by
* setting cra_blocksize to 1. Even using blkcipher_walk_virt_block
* during encrypt/decrypt doesn't solve this problem, because it calls
* blkcipher_walk_done under the covers, which doesn't use walk->blocksize,
* but instead uses this tfm->blocksize. */
struct crypto_alg nx_gcm_aes_alg = {
.cra_name = "gcm(aes)",
.cra_driver_name = "gcm-aes-nx",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_AEAD,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct nx_crypto_ctx),
.cra_type = &crypto_aead_type,
.cra_module = THIS_MODULE,
.cra_init = nx_crypto_ctx_aes_gcm_init,
.cra_exit = nx_crypto_ctx_exit,
.cra_aead = {
.ivsize = AES_BLOCK_SIZE,
.maxauthsize = AES_BLOCK_SIZE,
.setkey = gcm_aes_nx_set_key,
.setauthsize = gcm_aes_nx_setauthsize,
.encrypt = gcm_aes_nx_encrypt,
.decrypt = gcm_aes_nx_decrypt,
}
};
struct crypto_alg nx_gcm4106_aes_alg = {
.cra_name = "rfc4106(gcm(aes))",
.cra_driver_name = "rfc4106-gcm-aes-nx",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_AEAD,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct nx_crypto_ctx),
.cra_type = &crypto_nivaead_type,
.cra_module = THIS_MODULE,
.cra_init = nx_crypto_ctx_aes_gcm_init,
.cra_exit = nx_crypto_ctx_exit,
.cra_aead = {
.ivsize = 8,
.maxauthsize = AES_BLOCK_SIZE,
.geniv = "seqiv",
.setkey = gcm4106_aes_nx_set_key,
.setauthsize = gcm4106_aes_nx_setauthsize,
.encrypt = gcm4106_aes_nx_encrypt,
.decrypt = gcm4106_aes_nx_decrypt,
}
};

333
drivers/crypto/nx/nx-aes-xcbc.c Normal file
View file

@ -0,0 +1,333 @@
/**
* AES XCBC routines supporting the Power 7+ Nest Accelerators driver
*
* Copyright (C) 2011-2012 International Business Machines Inc.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; version 2 only.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*
* Author: Kent Yoder <yoder1@us.ibm.com>
*/
#include <crypto/internal/hash.h>
#include <crypto/aes.h>
#include <crypto/algapi.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <asm/vio.h>
#include "nx_csbcpb.h"
#include "nx.h"
struct xcbc_state {
u8 state[AES_BLOCK_SIZE];
unsigned int count;
u8 buffer[AES_BLOCK_SIZE];
};
static int nx_xcbc_set_key(struct crypto_shash *desc,
const u8 *in_key,
unsigned int key_len)
{
struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc);
switch (key_len) {
case AES_KEYSIZE_128:
nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128];
break;
default:
return -EINVAL;
}
memcpy(nx_ctx->priv.xcbc.key, in_key, key_len);
return 0;
}
/*
* Based on RFC 3566, for a zero-length message:
*
* n = 1
* K1 = E(K, 0x01010101010101010101010101010101)
* K3 = E(K, 0x03030303030303030303030303030303)
* E[0] = 0x00000000000000000000000000000000
* M[1] = 0x80000000000000000000000000000000 (0 length message with padding)
* E[1] = (K1, M[1] ^ E[0] ^ K3)
* Tag = M[1]
*/
static int nx_xcbc_empty(struct shash_desc *desc, u8 *out)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
struct nx_sg *in_sg, *out_sg;
u8 keys[2][AES_BLOCK_SIZE];
u8 key[32];
int rc = 0;
/* Change to ECB mode */
csbcpb->cpb.hdr.mode = NX_MODE_AES_ECB;
memcpy(key, csbcpb->cpb.aes_xcbc.key, AES_BLOCK_SIZE);
memcpy(csbcpb->cpb.aes_ecb.key, key, AES_BLOCK_SIZE);
NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
/* K1 and K3 base patterns */
memset(keys[0], 0x01, sizeof(keys[0]));
memset(keys[1], 0x03, sizeof(keys[1]));
/* Generate K1 and K3 encrypting the patterns */
in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *) keys, sizeof(keys),
nx_ctx->ap->sglen);
out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *) keys, sizeof(keys),
nx_ctx->ap->sglen);
nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
if (rc)
goto out;
atomic_inc(&(nx_ctx->stats->aes_ops));
/* XOr K3 with the padding for a 0 length message */
keys[1][0] ^= 0x80;
/* Encrypt the final result */
memcpy(csbcpb->cpb.aes_ecb.key, keys[0], AES_BLOCK_SIZE);
in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *) keys[1], sizeof(keys[1]),
nx_ctx->ap->sglen);
out_sg = nx_build_sg_list(nx_ctx->out_sg, out, AES_BLOCK_SIZE,
nx_ctx->ap->sglen);
nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
if (rc)
goto out;
atomic_inc(&(nx_ctx->stats->aes_ops));
out:
/* Restore XCBC mode */
csbcpb->cpb.hdr.mode = NX_MODE_AES_XCBC_MAC;
memcpy(csbcpb->cpb.aes_xcbc.key, key, AES_BLOCK_SIZE);
NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
return rc;
}
static int nx_xcbc_init(struct shash_desc *desc)
{
struct xcbc_state *sctx = shash_desc_ctx(desc);
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
struct nx_sg *out_sg;
nx_ctx_init(nx_ctx, HCOP_FC_AES);
memset(sctx, 0, sizeof *sctx);
NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128);
csbcpb->cpb.hdr.mode = NX_MODE_AES_XCBC_MAC;
memcpy(csbcpb->cpb.aes_xcbc.key, nx_ctx->priv.xcbc.key, AES_BLOCK_SIZE);
memset(nx_ctx->priv.xcbc.key, 0, sizeof *nx_ctx->priv.xcbc.key);
out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *)sctx->state,
AES_BLOCK_SIZE, nx_ctx->ap->sglen);
nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
return 0;
}
static int nx_xcbc_update(struct shash_desc *desc,
const u8 *data,
unsigned int len)
{
struct xcbc_state *sctx = shash_desc_ctx(desc);
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
struct nx_sg *in_sg;
u32 to_process, leftover, total;
u32 max_sg_len;
unsigned long irq_flags;
int rc = 0;
spin_lock_irqsave(&nx_ctx->lock, irq_flags);
total = sctx->count + len;
/* 2 cases for total data len:
* 1: <= AES_BLOCK_SIZE: copy into state, return 0
* 2: > AES_BLOCK_SIZE: process X blocks, copy in leftover
*/
if (total <= AES_BLOCK_SIZE) {
memcpy(sctx->buffer + sctx->count, data, len);
sctx->count += len;
goto out;
}
in_sg = nx_ctx->in_sg;
max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
nx_ctx->ap->sglen);
do {
/* to_process: the AES_BLOCK_SIZE data chunk to process in this
* update */
to_process = min_t(u64, total, nx_ctx->ap->databytelen);
to_process = min_t(u64, to_process,
NX_PAGE_SIZE * (max_sg_len - 1));
to_process = to_process & ~(AES_BLOCK_SIZE - 1);
leftover = total - to_process;
/* the hardware will not accept a 0 byte operation for this
* algorithm and the operation MUST be finalized to be correct.
* So if we happen to get an update that falls on a block sized
* boundary, we must save off the last block to finalize with
* later. */
if (!leftover) {
to_process -= AES_BLOCK_SIZE;
leftover = AES_BLOCK_SIZE;
}
if (sctx->count) {
in_sg = nx_build_sg_list(nx_ctx->in_sg,
(u8 *) sctx->buffer,
sctx->count,
max_sg_len);
}
in_sg = nx_build_sg_list(in_sg,
(u8 *) data,
to_process - sctx->count,
max_sg_len);
nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) *
sizeof(struct nx_sg);
/* we've hit the nx chip previously and we're updating again,
* so copy over the partial digest */
if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
memcpy(csbcpb->cpb.aes_xcbc.cv,
csbcpb->cpb.aes_xcbc.out_cv_mac,
AES_BLOCK_SIZE);
}
NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
rc = -EINVAL;
goto out;
}
rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
if (rc)
goto out;
atomic_inc(&(nx_ctx->stats->aes_ops));
/* everything after the first update is continuation */
NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
total -= to_process;
data += to_process - sctx->count;
sctx->count = 0;
in_sg = nx_ctx->in_sg;
} while (leftover > AES_BLOCK_SIZE);
/* copy the leftover back into the state struct */
memcpy(sctx->buffer, data, leftover);
sctx->count = leftover;
out:
spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
static int nx_xcbc_final(struct shash_desc *desc, u8 *out)
{
struct xcbc_state *sctx = shash_desc_ctx(desc);
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
struct nx_sg *in_sg, *out_sg;
unsigned long irq_flags;
int rc = 0;
spin_lock_irqsave(&nx_ctx->lock, irq_flags);
if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
/* we've hit the nx chip previously, now we're finalizing,
* so copy over the partial digest */
memcpy(csbcpb->cpb.aes_xcbc.cv,
csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE);
} else if (sctx->count == 0) {
/*
* we've never seen an update, so this is a 0 byte op. The
* hardware cannot handle a 0 byte op, so just ECB to
* generate the hash.
*/
rc = nx_xcbc_empty(desc, out);
goto out;
}
/* final is represented by continuing the operation and indicating that
* this is not an intermediate operation */
NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)sctx->buffer,
sctx->count, nx_ctx->ap->sglen);
out_sg = nx_build_sg_list(nx_ctx->out_sg, out, AES_BLOCK_SIZE,
nx_ctx->ap->sglen);
nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
if (!nx_ctx->op.outlen) {
rc = -EINVAL;
goto out;
}
rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
if (rc)
goto out;
atomic_inc(&(nx_ctx->stats->aes_ops));
memcpy(out, csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE);
out:
spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
struct shash_alg nx_shash_aes_xcbc_alg = {
.digestsize = AES_BLOCK_SIZE,
.init = nx_xcbc_init,
.update = nx_xcbc_update,
.final = nx_xcbc_final,
.setkey = nx_xcbc_set_key,
.descsize = sizeof(struct xcbc_state),
.statesize = sizeof(struct xcbc_state),
.base = {
.cra_name = "xcbc(aes)",
.cra_driver_name = "xcbc-aes-nx",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_SHASH,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_module = THIS_MODULE,
.cra_ctxsize = sizeof(struct nx_crypto_ctx),
.cra_init = nx_crypto_ctx_aes_xcbc_init,
.cra_exit = nx_crypto_ctx_exit,
}
};

282
drivers/crypto/nx/nx-sha256.c Normal file
View file

@ -0,0 +1,282 @@
/**
* SHA-256 routines supporting the Power 7+ Nest Accelerators driver
*
* Copyright (C) 2011-2012 International Business Machines Inc.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; version 2 only.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*
* Author: Kent Yoder <yoder1@us.ibm.com>
*/
#include <crypto/internal/hash.h>
#include <crypto/sha.h>
#include <linux/module.h>
#include <asm/vio.h>
#include "nx_csbcpb.h"
#include "nx.h"
static int nx_sha256_init(struct shash_desc *desc)
{
struct sha256_state *sctx = shash_desc_ctx(desc);
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_sg *out_sg;
nx_ctx_init(nx_ctx, HCOP_FC_SHA);
memset(sctx, 0, sizeof *sctx);
nx_ctx->ap = &nx_ctx->props[NX_PROPS_SHA256];
NX_CPB_SET_DIGEST_SIZE(nx_ctx->csbcpb, NX_DS_SHA256);
out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *)sctx->state,
SHA256_DIGEST_SIZE, nx_ctx->ap->sglen);
nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
return 0;
}
static int nx_sha256_update(struct shash_desc *desc, const u8 *data,
unsigned int len)
{
struct sha256_state *sctx = shash_desc_ctx(desc);
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
struct nx_sg *in_sg;
u64 to_process, leftover, total;
u32 max_sg_len;
unsigned long irq_flags;
int rc = 0;
spin_lock_irqsave(&nx_ctx->lock, irq_flags);
/* 2 cases for total data len:
* 1: < SHA256_BLOCK_SIZE: copy into state, return 0
* 2: >= SHA256_BLOCK_SIZE: process X blocks, copy in leftover
*/
total = sctx->count + len;
if (total < SHA256_BLOCK_SIZE) {
memcpy(sctx->buf + sctx->count, data, len);
sctx->count += len;
goto out;
}
in_sg = nx_ctx->in_sg;
max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
nx_ctx->ap->sglen);
do {
/*
* to_process: the SHA256_BLOCK_SIZE data chunk to process in
* this update. This value is also restricted by the sg list
* limits.
*/
to_process = min_t(u64, total, nx_ctx->ap->databytelen);
to_process = min_t(u64, to_process,
NX_PAGE_SIZE * (max_sg_len - 1));
to_process = to_process & ~(SHA256_BLOCK_SIZE - 1);
leftover = total - to_process;
if (sctx->count) {
in_sg = nx_build_sg_list(nx_ctx->in_sg,
(u8 *) sctx->buf,
sctx->count, max_sg_len);
}
in_sg = nx_build_sg_list(in_sg, (u8 *) data,
to_process - sctx->count,
max_sg_len);
nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) *
sizeof(struct nx_sg);
if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
/*
* we've hit the nx chip previously and we're updating
* again, so copy over the partial digest.
*/
memcpy(csbcpb->cpb.sha256.input_partial_digest,
csbcpb->cpb.sha256.message_digest,
SHA256_DIGEST_SIZE);
}
NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
rc = -EINVAL;
goto out;
}
rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
if (rc)
goto out;
atomic_inc(&(nx_ctx->stats->sha256_ops));
csbcpb->cpb.sha256.message_bit_length += (u64)
(csbcpb->cpb.sha256.spbc * 8);
/* everything after the first update is continuation */
NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
total -= to_process;
data += to_process - sctx->count;
sctx->count = 0;
in_sg = nx_ctx->in_sg;
} while (leftover >= SHA256_BLOCK_SIZE);
/* copy the leftover back into the state struct */
if (leftover)
memcpy(sctx->buf, data, leftover);
sctx->count = leftover;
out:
spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
static int nx_sha256_final(struct shash_desc *desc, u8 *out)
{
struct sha256_state *sctx = shash_desc_ctx(desc);
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
struct nx_sg *in_sg, *out_sg;
u32 max_sg_len;
unsigned long irq_flags;
int rc;
spin_lock_irqsave(&nx_ctx->lock, irq_flags);
max_sg_len = min_t(u32, nx_driver.of.max_sg_len, nx_ctx->ap->sglen);
if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
/* we've hit the nx chip previously, now we're finalizing,
* so copy over the partial digest */
memcpy(csbcpb->cpb.sha256.input_partial_digest,
csbcpb->cpb.sha256.message_digest, SHA256_DIGEST_SIZE);
}
/* final is represented by continuing the operation and indicating that
* this is not an intermediate operation */
NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
csbcpb->cpb.sha256.message_bit_length += (u64)(sctx->count * 8);
in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)sctx->buf,
sctx->count, max_sg_len);
out_sg = nx_build_sg_list(nx_ctx->out_sg, out, SHA256_DIGEST_SIZE,
max_sg_len);
nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
if (!nx_ctx->op.outlen) {
rc = -EINVAL;
goto out;
}
rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
if (rc)
goto out;
atomic_inc(&(nx_ctx->stats->sha256_ops));
atomic64_add(csbcpb->cpb.sha256.message_bit_length / 8,
&(nx_ctx->stats->sha256_bytes));
memcpy(out, csbcpb->cpb.sha256.message_digest, SHA256_DIGEST_SIZE);
out:
spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
static int nx_sha256_export(struct shash_desc *desc, void *out)
{
struct sha256_state *sctx = shash_desc_ctx(desc);
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
struct sha256_state *octx = out;
unsigned long irq_flags;
spin_lock_irqsave(&nx_ctx->lock, irq_flags);
octx->count = sctx->count +
(csbcpb->cpb.sha256.message_bit_length / 8);
memcpy(octx->buf, sctx->buf, sizeof(octx->buf));
/* if no data has been processed yet, we need to export SHA256's
* initial data, in case this context gets imported into a software
* context */
if (csbcpb->cpb.sha256.message_bit_length)
memcpy(octx->state, csbcpb->cpb.sha256.message_digest,
SHA256_DIGEST_SIZE);
else {
octx->state[0] = SHA256_H0;
octx->state[1] = SHA256_H1;
octx->state[2] = SHA256_H2;
octx->state[3] = SHA256_H3;
octx->state[4] = SHA256_H4;
octx->state[5] = SHA256_H5;
octx->state[6] = SHA256_H6;
octx->state[7] = SHA256_H7;
}
spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return 0;
}
static int nx_sha256_import(struct shash_desc *desc, const void *in)
{
struct sha256_state *sctx = shash_desc_ctx(desc);
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
const struct sha256_state *ictx = in;
unsigned long irq_flags;
spin_lock_irqsave(&nx_ctx->lock, irq_flags);
memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf));
sctx->count = ictx->count & 0x3f;
csbcpb->cpb.sha256.message_bit_length = (ictx->count & ~0x3f) * 8;
if (csbcpb->cpb.sha256.message_bit_length) {
memcpy(csbcpb->cpb.sha256.message_digest, ictx->state,
SHA256_DIGEST_SIZE);
NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
}
spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return 0;
}
struct shash_alg nx_shash_sha256_alg = {
.digestsize = SHA256_DIGEST_SIZE,
.init = nx_sha256_init,
.update = nx_sha256_update,
.final = nx_sha256_final,
.export = nx_sha256_export,
.import = nx_sha256_import,
.descsize = sizeof(struct sha256_state),
.statesize = sizeof(struct sha256_state),
.base = {
.cra_name = "sha256",
.cra_driver_name = "sha256-nx",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_SHASH,
.cra_blocksize = SHA256_BLOCK_SIZE,
.cra_module = THIS_MODULE,
.cra_ctxsize = sizeof(struct nx_crypto_ctx),
.cra_init = nx_crypto_ctx_sha_init,
.cra_exit = nx_crypto_ctx_exit,
}
};

301
drivers/crypto/nx/nx-sha512.c Normal file
View file

@ -0,0 +1,301 @@
/**
* SHA-512 routines supporting the Power 7+ Nest Accelerators driver
*
* Copyright (C) 2011-2012 International Business Machines Inc.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; version 2 only.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*
* Author: Kent Yoder <yoder1@us.ibm.com>
*/
#include <crypto/internal/hash.h>
#include <crypto/sha.h>
#include <linux/module.h>
#include <asm/vio.h>
#include "nx_csbcpb.h"
#include "nx.h"
static int nx_sha512_init(struct shash_desc *desc)
{
struct sha512_state *sctx = shash_desc_ctx(desc);
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_sg *out_sg;
nx_ctx_init(nx_ctx, HCOP_FC_SHA);
memset(sctx, 0, sizeof *sctx);
nx_ctx->ap = &nx_ctx->props[NX_PROPS_SHA512];
NX_CPB_SET_DIGEST_SIZE(nx_ctx->csbcpb, NX_DS_SHA512);
out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *)sctx->state,
SHA512_DIGEST_SIZE, nx_ctx->ap->sglen);
nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
return 0;
}
static int nx_sha512_update(struct shash_desc *desc, const u8 *data,
unsigned int len)
{
struct sha512_state *sctx = shash_desc_ctx(desc);
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
struct nx_sg *in_sg;
u64 to_process, leftover, total, spbc_bits;
u32 max_sg_len;
unsigned long irq_flags;
int rc = 0;
spin_lock_irqsave(&nx_ctx->lock, irq_flags);
/* 2 cases for total data len:
* 1: < SHA512_BLOCK_SIZE: copy into state, return 0
* 2: >= SHA512_BLOCK_SIZE: process X blocks, copy in leftover
*/
total = sctx->count[0] + len;
if (total < SHA512_BLOCK_SIZE) {
memcpy(sctx->buf + sctx->count[0], data, len);
sctx->count[0] += len;
goto out;
}
in_sg = nx_ctx->in_sg;
max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
nx_ctx->ap->sglen);
do {
/*
* to_process: the SHA512_BLOCK_SIZE data chunk to process in
* this update. This value is also restricted by the sg list
* limits.
*/
to_process = min_t(u64, total, nx_ctx->ap->databytelen);
to_process = min_t(u64, to_process,
NX_PAGE_SIZE * (max_sg_len - 1));
to_process = to_process & ~(SHA512_BLOCK_SIZE - 1);
leftover = total - to_process;
if (sctx->count[0]) {
in_sg = nx_build_sg_list(nx_ctx->in_sg,
(u8 *) sctx->buf,
sctx->count[0], max_sg_len);
}
in_sg = nx_build_sg_list(in_sg, (u8 *) data,
to_process - sctx->count[0],
max_sg_len);
nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) *
sizeof(struct nx_sg);
if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
/*
* we've hit the nx chip previously and we're updating
* again, so copy over the partial digest.
*/
memcpy(csbcpb->cpb.sha512.input_partial_digest,
csbcpb->cpb.sha512.message_digest,
SHA512_DIGEST_SIZE);
}
NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
rc = -EINVAL;
goto out;
}
rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
if (rc)
goto out;
atomic_inc(&(nx_ctx->stats->sha512_ops));
spbc_bits = csbcpb->cpb.sha512.spbc * 8;
csbcpb->cpb.sha512.message_bit_length_lo += spbc_bits;
if (csbcpb->cpb.sha512.message_bit_length_lo < spbc_bits)
csbcpb->cpb.sha512.message_bit_length_hi++;
/* everything after the first update is continuation */
NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
total -= to_process;
data += to_process - sctx->count[0];
sctx->count[0] = 0;
in_sg = nx_ctx->in_sg;
} while (leftover >= SHA512_BLOCK_SIZE);
/* copy the leftover back into the state struct */
if (leftover)
memcpy(sctx->buf, data, leftover);
sctx->count[0] = leftover;
out:
spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
static int nx_sha512_final(struct shash_desc *desc, u8 *out)
{
struct sha512_state *sctx = shash_desc_ctx(desc);
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
struct nx_sg *in_sg, *out_sg;
u32 max_sg_len;
u64 count0;
unsigned long irq_flags;
int rc;
spin_lock_irqsave(&nx_ctx->lock, irq_flags);
max_sg_len = min_t(u32, nx_driver.of.max_sg_len, nx_ctx->ap->sglen);
if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
/* we've hit the nx chip previously, now we're finalizing,
* so copy over the partial digest */
memcpy(csbcpb->cpb.sha512.input_partial_digest,
csbcpb->cpb.sha512.message_digest, SHA512_DIGEST_SIZE);
}
/* final is represented by continuing the operation and indicating that
* this is not an intermediate operation */
NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
count0 = sctx->count[0] * 8;
csbcpb->cpb.sha512.message_bit_length_lo += count0;
if (csbcpb->cpb.sha512.message_bit_length_lo < count0)
csbcpb->cpb.sha512.message_bit_length_hi++;
in_sg = nx_build_sg_list(nx_ctx->in_sg, sctx->buf, sctx->count[0],
max_sg_len);
out_sg = nx_build_sg_list(nx_ctx->out_sg, out, SHA512_DIGEST_SIZE,
max_sg_len);
nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
if (!nx_ctx->op.outlen) {
rc = -EINVAL;
goto out;
}
rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
if (rc)
goto out;
atomic_inc(&(nx_ctx->stats->sha512_ops));
atomic64_add(csbcpb->cpb.sha512.message_bit_length_lo / 8,
&(nx_ctx->stats->sha512_bytes));
memcpy(out, csbcpb->cpb.sha512.message_digest, SHA512_DIGEST_SIZE);
out:
spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
static int nx_sha512_export(struct shash_desc *desc, void *out)
{
struct sha512_state *sctx = shash_desc_ctx(desc);
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
struct sha512_state *octx = out;
unsigned long irq_flags;
spin_lock_irqsave(&nx_ctx->lock, irq_flags);
/* move message_bit_length (128 bits) into count and convert its value
* to bytes */
octx->count[0] = csbcpb->cpb.sha512.message_bit_length_lo >> 3 |
((csbcpb->cpb.sha512.message_bit_length_hi & 7) << 61);
octx->count[1] = csbcpb->cpb.sha512.message_bit_length_hi >> 3;
octx->count[0] += sctx->count[0];
if (octx->count[0] < sctx->count[0])
octx->count[1]++;
memcpy(octx->buf, sctx->buf, sizeof(octx->buf));
/* if no data has been processed yet, we need to export SHA512's
* initial data, in case this context gets imported into a software
* context */
if (csbcpb->cpb.sha512.message_bit_length_hi ||
csbcpb->cpb.sha512.message_bit_length_lo)
memcpy(octx->state, csbcpb->cpb.sha512.message_digest,
SHA512_DIGEST_SIZE);
else {
octx->state[0] = SHA512_H0;
octx->state[1] = SHA512_H1;
octx->state[2] = SHA512_H2;
octx->state[3] = SHA512_H3;
octx->state[4] = SHA512_H4;
octx->state[5] = SHA512_H5;
octx->state[6] = SHA512_H6;
octx->state[7] = SHA512_H7;
}
spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return 0;
}
static int nx_sha512_import(struct shash_desc *desc, const void *in)
{
struct sha512_state *sctx = shash_desc_ctx(desc);
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
const struct sha512_state *ictx = in;
unsigned long irq_flags;
spin_lock_irqsave(&nx_ctx->lock, irq_flags);
memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf));
sctx->count[0] = ictx->count[0] & 0x3f;
csbcpb->cpb.sha512.message_bit_length_lo = (ictx->count[0] & ~0x3f)
<< 3;
csbcpb->cpb.sha512.message_bit_length_hi = ictx->count[1] << 3 |
ictx->count[0] >> 61;
if (csbcpb->cpb.sha512.message_bit_length_hi ||
csbcpb->cpb.sha512.message_bit_length_lo) {
memcpy(csbcpb->cpb.sha512.message_digest, ictx->state,
SHA512_DIGEST_SIZE);
NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
}
spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return 0;
}
struct shash_alg nx_shash_sha512_alg = {
.digestsize = SHA512_DIGEST_SIZE,
.init = nx_sha512_init,
.update = nx_sha512_update,
.final = nx_sha512_final,
.export = nx_sha512_export,
.import = nx_sha512_import,
.descsize = sizeof(struct sha512_state),
.statesize = sizeof(struct sha512_state),
.base = {
.cra_name = "sha512",
.cra_driver_name = "sha512-nx",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_SHASH,
.cra_blocksize = SHA512_BLOCK_SIZE,
.cra_module = THIS_MODULE,
.cra_ctxsize = sizeof(struct nx_crypto_ctx),
.cra_init = nx_crypto_ctx_sha_init,
.cra_exit = nx_crypto_ctx_exit,
}
};

710
drivers/crypto/nx/nx.c Normal file
View file

@ -0,0 +1,710 @@
/**
* Routines supporting the Power 7+ Nest Accelerators driver
*
* Copyright (C) 2011-2012 International Business Machines Inc.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; version 2 only.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*
* Author: Kent Yoder <yoder1@us.ibm.com>
*/
#include <crypto/internal/hash.h>
#include <crypto/hash.h>
#include <crypto/aes.h>
#include <crypto/sha.h>
#include <crypto/algapi.h>
#include <crypto/scatterwalk.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/types.h>
#include <linux/mm.h>
#include <linux/crypto.h>
#include <linux/scatterlist.h>
#include <linux/device.h>
#include <linux/of.h>
#include <asm/hvcall.h>
#include <asm/vio.h>
#include "nx_csbcpb.h"
#include "nx.h"
/**
* nx_hcall_sync - make an H_COP_OP hcall for the passed in op structure
*
* @nx_ctx: the crypto context handle
* @op: PFO operation struct to pass in
* @may_sleep: flag indicating the request can sleep
*
* Make the hcall, retrying while the hardware is busy. If we cannot yield
* the thread, limit the number of retries to 10 here.
*/
int nx_hcall_sync(struct nx_crypto_ctx *nx_ctx,
struct vio_pfo_op *op,
u32 may_sleep)
{
int rc, retries = 10;
struct vio_dev *viodev = nx_driver.viodev;
atomic_inc(&(nx_ctx->stats->sync_ops));
do {
rc = vio_h_cop_sync(viodev, op);
} while (rc == -EBUSY && !may_sleep && retries--);
if (rc) {
dev_dbg(&viodev->dev, "vio_h_cop_sync failed: rc: %d "
"hcall rc: %ld\n", rc, op->hcall_err);
atomic_inc(&(nx_ctx->stats->errors));
atomic_set(&(nx_ctx->stats->last_error), op->hcall_err);
atomic_set(&(nx_ctx->stats->last_error_pid), current->pid);
}
return rc;
}
/**
* nx_build_sg_list - build an NX scatter list describing a single buffer
*
* @sg_head: pointer to the first scatter list element to build
* @start_addr: pointer to the linear buffer
* @len: length of the data at @start_addr
* @sgmax: the largest number of scatter list elements we're allowed to create
*
* This function will start writing nx_sg elements at @sg_head and keep
* writing them until all of the data from @start_addr is described or
* until sgmax elements have been written. Scatter list elements will be
* created such that none of the elements describes a buffer that crosses a 4K
* boundary.
*/
struct nx_sg *nx_build_sg_list(struct nx_sg *sg_head,
u8 *start_addr,
unsigned int len,
u32 sgmax)
{
unsigned int sg_len = 0;
struct nx_sg *sg;
u64 sg_addr = (u64)start_addr;
u64 end_addr;
/* determine the start and end for this address range - slightly
* different if this is in VMALLOC_REGION */
if (is_vmalloc_addr(start_addr))
sg_addr = page_to_phys(vmalloc_to_page(start_addr))
+ offset_in_page(sg_addr);
else
sg_addr = __pa(sg_addr);
end_addr = sg_addr + len;
/* each iteration will write one struct nx_sg element and add the
* length of data described by that element to sg_len. Once @len bytes
* have been described (or @sgmax elements have been written), the
* loop ends. min_t is used to ensure @end_addr falls on the same page
* as sg_addr, if not, we need to create another nx_sg element for the
* data on the next page.
*
* Also when using vmalloc'ed data, every time that a system page
* boundary is crossed the physical address needs to be re-calculated.
*/
for (sg = sg_head; sg_len < len; sg++) {
u64 next_page;
sg->addr = sg_addr;
sg_addr = min_t(u64, NX_PAGE_NUM(sg_addr + NX_PAGE_SIZE),
end_addr);
next_page = (sg->addr & PAGE_MASK) + PAGE_SIZE;
sg->len = min_t(u64, sg_addr, next_page) - sg->addr;
sg_len += sg->len;
if (sg_addr >= next_page &&
is_vmalloc_addr(start_addr + sg_len)) {
sg_addr = page_to_phys(vmalloc_to_page(
start_addr + sg_len));
end_addr = sg_addr + len - sg_len;
}
if ((sg - sg_head) == sgmax) {
pr_err("nx: scatter/gather list overflow, pid: %d\n",
current->pid);
return NULL;
}
}
/* return the moved sg_head pointer */
return sg;
}
/**
* nx_walk_and_build - walk a linux scatterlist and build an nx scatterlist
*
* @nx_dst: pointer to the first nx_sg element to write
* @sglen: max number of nx_sg entries we're allowed to write
* @sg_src: pointer to the source linux scatterlist to walk
* @start: number of bytes to fast-forward past at the beginning of @sg_src
* @src_len: number of bytes to walk in @sg_src
*/
struct nx_sg *nx_walk_and_build(struct nx_sg *nx_dst,
unsigned int sglen,
struct scatterlist *sg_src,
unsigned int start,
unsigned int src_len)
{
struct scatter_walk walk;
struct nx_sg *nx_sg = nx_dst;
unsigned int n, offset = 0, len = src_len;
char *dst;
/* we need to fast forward through @start bytes first */
for (;;) {
scatterwalk_start(&walk, sg_src);
if (start < offset + sg_src->length)
break;
offset += sg_src->length;
sg_src = scatterwalk_sg_next(sg_src);
}
/* start - offset is the number of bytes to advance in the scatterlist
* element we're currently looking at */
scatterwalk_advance(&walk, start - offset);
while (len && nx_sg) {
n = scatterwalk_clamp(&walk, len);
if (!n) {
scatterwalk_start(&walk, sg_next(walk.sg));
n = scatterwalk_clamp(&walk, len);
}
dst = scatterwalk_map(&walk);
nx_sg = nx_build_sg_list(nx_sg, dst, n, sglen);
len -= n;
scatterwalk_unmap(dst);
scatterwalk_advance(&walk, n);
scatterwalk_done(&walk, SCATTERWALK_FROM_SG, len);
}
/* return the moved destination pointer */
return nx_sg;
}
/**
* nx_build_sg_lists - walk the input scatterlists and build arrays of NX
* scatterlists based on them.
*
* @nx_ctx: NX crypto context for the lists we're building
* @desc: the block cipher descriptor for the operation
* @dst: destination scatterlist
* @src: source scatterlist
* @nbytes: length of data described in the scatterlists
* @offset: number of bytes to fast-forward past at the beginning of
* scatterlists.
* @iv: destination for the iv data, if the algorithm requires it
*
* This is common code shared by all the AES algorithms. It uses the block
* cipher walk routines to traverse input and output scatterlists, building
* corresponding NX scatterlists
*/
int nx_build_sg_lists(struct nx_crypto_ctx *nx_ctx,
struct blkcipher_desc *desc,
struct scatterlist *dst,
struct scatterlist *src,
unsigned int nbytes,
unsigned int offset,
u8 *iv)
{
struct nx_sg *nx_insg = nx_ctx->in_sg;
struct nx_sg *nx_outsg = nx_ctx->out_sg;
if (iv)
memcpy(iv, desc->info, AES_BLOCK_SIZE);
nx_insg = nx_walk_and_build(nx_insg, nx_ctx->ap->sglen, src,
offset, nbytes);
nx_outsg = nx_walk_and_build(nx_outsg, nx_ctx->ap->sglen, dst,
offset, nbytes);
/* these lengths should be negative, which will indicate to phyp that
* the input and output parameters are scatterlists, not linear
* buffers */
nx_ctx->op.inlen = (nx_ctx->in_sg - nx_insg) * sizeof(struct nx_sg);
nx_ctx->op.outlen = (nx_ctx->out_sg - nx_outsg) * sizeof(struct nx_sg);
return 0;
}
/**
* nx_ctx_init - initialize an nx_ctx's vio_pfo_op struct
*
* @nx_ctx: the nx context to initialize
* @function: the function code for the op
*/
void nx_ctx_init(struct nx_crypto_ctx *nx_ctx, unsigned int function)
{
spin_lock_init(&nx_ctx->lock);
memset(nx_ctx->kmem, 0, nx_ctx->kmem_len);
nx_ctx->csbcpb->csb.valid |= NX_CSB_VALID_BIT;
nx_ctx->op.flags = function;
nx_ctx->op.csbcpb = __pa(nx_ctx->csbcpb);
nx_ctx->op.in = __pa(nx_ctx->in_sg);
nx_ctx->op.out = __pa(nx_ctx->out_sg);
if (nx_ctx->csbcpb_aead) {
nx_ctx->csbcpb_aead->csb.valid |= NX_CSB_VALID_BIT;
nx_ctx->op_aead.flags = function;
nx_ctx->op_aead.csbcpb = __pa(nx_ctx->csbcpb_aead);
nx_ctx->op_aead.in = __pa(nx_ctx->in_sg);
nx_ctx->op_aead.out = __pa(nx_ctx->out_sg);
}
}
static void nx_of_update_status(struct device *dev,
struct property *p,
struct nx_of *props)
{
if (!strncmp(p->value, "okay", p->length)) {
props->status = NX_WAITING;
props->flags |= NX_OF_FLAG_STATUS_SET;
} else {
dev_info(dev, "%s: status '%s' is not 'okay'\n", __func__,
(char *)p->value);
}
}
static void nx_of_update_sglen(struct device *dev,
struct property *p,
struct nx_of *props)
{
if (p->length != sizeof(props->max_sg_len)) {
dev_err(dev, "%s: unexpected format for "
"ibm,max-sg-len property\n", __func__);
dev_dbg(dev, "%s: ibm,max-sg-len is %d bytes "
"long, expected %zd bytes\n", __func__,
p->length, sizeof(props->max_sg_len));
return;
}
props->max_sg_len = *(u32 *)p->value;
props->flags |= NX_OF_FLAG_MAXSGLEN_SET;
}
static void nx_of_update_msc(struct device *dev,
struct property *p,
struct nx_of *props)
{
struct msc_triplet *trip;
struct max_sync_cop *msc;
unsigned int bytes_so_far, i, lenp;
msc = (struct max_sync_cop *)p->value;
lenp = p->length;
/* You can't tell if the data read in for this property is sane by its
* size alone. This is because there are sizes embedded in the data
* structure. The best we can do is check lengths as we parse and bail
* as soon as a length error is detected. */
bytes_so_far = 0;
while ((bytes_so_far + sizeof(struct max_sync_cop)) <= lenp) {
bytes_so_far += sizeof(struct max_sync_cop);
trip = msc->trip;
for (i = 0;
((bytes_so_far + sizeof(struct msc_triplet)) <= lenp) &&
i < msc->triplets;
i++) {
if (msc->fc > NX_MAX_FC || msc->mode > NX_MAX_MODE) {
dev_err(dev, "unknown function code/mode "
"combo: %d/%d (ignored)\n", msc->fc,
msc->mode);
goto next_loop;
}
switch (trip->keybitlen) {
case 128:
case 160:
props->ap[msc->fc][msc->mode][0].databytelen =
trip->databytelen;
props->ap[msc->fc][msc->mode][0].sglen =
trip->sglen;
break;
case 192:
props->ap[msc->fc][msc->mode][1].databytelen =
trip->databytelen;
props->ap[msc->fc][msc->mode][1].sglen =
trip->sglen;
break;
case 256:
if (msc->fc == NX_FC_AES) {
props->ap[msc->fc][msc->mode][2].
databytelen = trip->databytelen;
props->ap[msc->fc][msc->mode][2].sglen =
trip->sglen;
} else if (msc->fc == NX_FC_AES_HMAC ||
msc->fc == NX_FC_SHA) {
props->ap[msc->fc][msc->mode][1].
databytelen = trip->databytelen;
props->ap[msc->fc][msc->mode][1].sglen =
trip->sglen;
} else {
dev_warn(dev, "unknown function "
"code/key bit len combo"
": (%u/256)\n", msc->fc);
}
break;
case 512:
props->ap[msc->fc][msc->mode][2].databytelen =
trip->databytelen;
props->ap[msc->fc][msc->mode][2].sglen =
trip->sglen;
break;
default:
dev_warn(dev, "unknown function code/key bit "
"len combo: (%u/%u)\n", msc->fc,
trip->keybitlen);
break;
}
next_loop:
bytes_so_far += sizeof(struct msc_triplet);
trip++;
}
msc = (struct max_sync_cop *)trip;
}
props->flags |= NX_OF_FLAG_MAXSYNCCOP_SET;
}
/**
* nx_of_init - read openFirmware values from the device tree
*
* @dev: device handle
* @props: pointer to struct to hold the properties values
*
* Called once at driver probe time, this function will read out the
* openFirmware properties we use at runtime. If all the OF properties are
* acceptable, when we exit this function props->flags will indicate that
* we're ready to register our crypto algorithms.
*/
static void nx_of_init(struct device *dev, struct nx_of *props)
{
struct device_node *base_node = dev->of_node;
struct property *p;
p = of_find_property(base_node, "status", NULL);
if (!p)
dev_info(dev, "%s: property 'status' not found\n", __func__);
else
nx_of_update_status(dev, p, props);
p = of_find_property(base_node, "ibm,max-sg-len", NULL);
if (!p)
dev_info(dev, "%s: property 'ibm,max-sg-len' not found\n",
__func__);
else
nx_of_update_sglen(dev, p, props);
p = of_find_property(base_node, "ibm,max-sync-cop", NULL);
if (!p)
dev_info(dev, "%s: property 'ibm,max-sync-cop' not found\n",
__func__);
else
nx_of_update_msc(dev, p, props);
}
/**
* nx_register_algs - register algorithms with the crypto API
*
* Called from nx_probe()
*
* If all OF properties are in an acceptable state, the driver flags will
* indicate that we're ready and we'll create our debugfs files and register
* out crypto algorithms.
*/
static int nx_register_algs(void)
{
int rc = -1;
if (nx_driver.of.flags != NX_OF_FLAG_MASK_READY)
goto out;
memset(&nx_driver.stats, 0, sizeof(struct nx_stats));
rc = NX_DEBUGFS_INIT(&nx_driver);
if (rc)
goto out;
nx_driver.of.status = NX_OKAY;
rc = crypto_register_alg(&nx_ecb_aes_alg);
if (rc)
goto out;
rc = crypto_register_alg(&nx_cbc_aes_alg);
if (rc)
goto out_unreg_ecb;
rc = crypto_register_alg(&nx_ctr_aes_alg);
if (rc)
goto out_unreg_cbc;
rc = crypto_register_alg(&nx_ctr3686_aes_alg);
if (rc)
goto out_unreg_ctr;
rc = crypto_register_alg(&nx_gcm_aes_alg);
if (rc)
goto out_unreg_ctr3686;
rc = crypto_register_alg(&nx_gcm4106_aes_alg);
if (rc)
goto out_unreg_gcm;
rc = crypto_register_alg(&nx_ccm_aes_alg);
if (rc)
goto out_unreg_gcm4106;
rc = crypto_register_alg(&nx_ccm4309_aes_alg);
if (rc)
goto out_unreg_ccm;
rc = crypto_register_shash(&nx_shash_sha256_alg);
if (rc)
goto out_unreg_ccm4309;
rc = crypto_register_shash(&nx_shash_sha512_alg);
if (rc)
goto out_unreg_s256;
rc = crypto_register_shash(&nx_shash_aes_xcbc_alg);
if (rc)
goto out_unreg_s512;
goto out;
out_unreg_s512:
crypto_unregister_shash(&nx_shash_sha512_alg);
out_unreg_s256:
crypto_unregister_shash(&nx_shash_sha256_alg);
out_unreg_ccm4309:
crypto_unregister_alg(&nx_ccm4309_aes_alg);
out_unreg_ccm:
crypto_unregister_alg(&nx_ccm_aes_alg);
out_unreg_gcm4106:
crypto_unregister_alg(&nx_gcm4106_aes_alg);
out_unreg_gcm:
crypto_unregister_alg(&nx_gcm_aes_alg);
out_unreg_ctr3686:
crypto_unregister_alg(&nx_ctr3686_aes_alg);
out_unreg_ctr:
crypto_unregister_alg(&nx_ctr_aes_alg);
out_unreg_cbc:
crypto_unregister_alg(&nx_cbc_aes_alg);
out_unreg_ecb:
crypto_unregister_alg(&nx_ecb_aes_alg);
out:
return rc;
}
/**
* nx_crypto_ctx_init - create and initialize a crypto api context
*
* @nx_ctx: the crypto api context
* @fc: function code for the context
* @mode: the function code specific mode for this context
*/
static int nx_crypto_ctx_init(struct nx_crypto_ctx *nx_ctx, u32 fc, u32 mode)
{
if (nx_driver.of.status != NX_OKAY) {
pr_err("Attempt to initialize NX crypto context while device "
"is not available!\n");
return -ENODEV;
}
/* we need an extra page for csbcpb_aead for these modes */
if (mode == NX_MODE_AES_GCM || mode == NX_MODE_AES_CCM)
nx_ctx->kmem_len = (4 * NX_PAGE_SIZE) +
sizeof(struct nx_csbcpb);
else
nx_ctx->kmem_len = (3 * NX_PAGE_SIZE) +
sizeof(struct nx_csbcpb);
nx_ctx->kmem = kmalloc(nx_ctx->kmem_len, GFP_KERNEL);
if (!nx_ctx->kmem)
return -ENOMEM;
/* the csbcpb and scatterlists must be 4K aligned pages */
nx_ctx->csbcpb = (struct nx_csbcpb *)(round_up((u64)nx_ctx->kmem,
(u64)NX_PAGE_SIZE));
nx_ctx->in_sg = (struct nx_sg *)((u8 *)nx_ctx->csbcpb + NX_PAGE_SIZE);
nx_ctx->out_sg = (struct nx_sg *)((u8 *)nx_ctx->in_sg + NX_PAGE_SIZE);
if (mode == NX_MODE_AES_GCM || mode == NX_MODE_AES_CCM)
nx_ctx->csbcpb_aead =
(struct nx_csbcpb *)((u8 *)nx_ctx->out_sg +
NX_PAGE_SIZE);
/* give each context a pointer to global stats and their OF
* properties */
nx_ctx->stats = &nx_driver.stats;
memcpy(nx_ctx->props, nx_driver.of.ap[fc][mode],
sizeof(struct alg_props) * 3);
return 0;
}
/* entry points from the crypto tfm initializers */
int nx_crypto_ctx_aes_ccm_init(struct crypto_tfm *tfm)
{
return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_AES,
NX_MODE_AES_CCM);
}
int nx_crypto_ctx_aes_gcm_init(struct crypto_tfm *tfm)
{
return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_AES,
NX_MODE_AES_GCM);
}
int nx_crypto_ctx_aes_ctr_init(struct crypto_tfm *tfm)
{
return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_AES,
NX_MODE_AES_CTR);
}
int nx_crypto_ctx_aes_cbc_init(struct crypto_tfm *tfm)
{
return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_AES,
NX_MODE_AES_CBC);
}
int nx_crypto_ctx_aes_ecb_init(struct crypto_tfm *tfm)
{
return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_AES,
NX_MODE_AES_ECB);
}
int nx_crypto_ctx_sha_init(struct crypto_tfm *tfm)
{
return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_SHA, NX_MODE_SHA);
}
int nx_crypto_ctx_aes_xcbc_init(struct crypto_tfm *tfm)
{
return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_AES,
NX_MODE_AES_XCBC_MAC);
}
/**
* nx_crypto_ctx_exit - destroy a crypto api context
*
* @tfm: the crypto transform pointer for the context
*
* As crypto API contexts are destroyed, this exit hook is called to free the
* memory associated with it.
*/
void nx_crypto_ctx_exit(struct crypto_tfm *tfm)
{
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm);
kzfree(nx_ctx->kmem);
nx_ctx->csbcpb = NULL;
nx_ctx->csbcpb_aead = NULL;
nx_ctx->in_sg = NULL;
nx_ctx->out_sg = NULL;
}
static int nx_probe(struct vio_dev *viodev, const struct vio_device_id *id)
{
dev_dbg(&viodev->dev, "driver probed: %s resource id: 0x%x\n",
viodev->name, viodev->resource_id);
if (nx_driver.viodev) {
dev_err(&viodev->dev, "%s: Attempt to register more than one "
"instance of the hardware\n", __func__);
return -EINVAL;
}
nx_driver.viodev = viodev;
nx_of_init(&viodev->dev, &nx_driver.of);
return nx_register_algs();
}
static int nx_remove(struct vio_dev *viodev)
{
dev_dbg(&viodev->dev, "entering nx_remove for UA 0x%x\n",
viodev->unit_address);
if (nx_driver.of.status == NX_OKAY) {
NX_DEBUGFS_FINI(&nx_driver);
crypto_unregister_alg(&nx_ccm_aes_alg);
crypto_unregister_alg(&nx_ccm4309_aes_alg);
crypto_unregister_alg(&nx_gcm_aes_alg);
crypto_unregister_alg(&nx_gcm4106_aes_alg);
crypto_unregister_alg(&nx_ctr_aes_alg);
crypto_unregister_alg(&nx_ctr3686_aes_alg);
crypto_unregister_alg(&nx_cbc_aes_alg);
crypto_unregister_alg(&nx_ecb_aes_alg);
crypto_unregister_shash(&nx_shash_sha256_alg);
crypto_unregister_shash(&nx_shash_sha512_alg);
crypto_unregister_shash(&nx_shash_aes_xcbc_alg);
}
return 0;
}
/* module wide initialization/cleanup */
static int __init nx_init(void)
{
return vio_register_driver(&nx_driver.viodriver);
}
static void __exit nx_fini(void)
{
vio_unregister_driver(&nx_driver.viodriver);
}
static struct vio_device_id nx_crypto_driver_ids[] = {
{ "ibm,sym-encryption-v1", "ibm,sym-encryption" },
{ "", "" }
};
MODULE_DEVICE_TABLE(vio, nx_crypto_driver_ids);
/* driver state structure */
struct nx_crypto_driver nx_driver = {
.viodriver = {
.id_table = nx_crypto_driver_ids,
.probe = nx_probe,
.remove = nx_remove,
.name = NX_NAME,
},
};
module_init(nx_init);
module_exit(nx_fini);
MODULE_AUTHOR("Kent Yoder <yoder1@us.ibm.com>");
MODULE_DESCRIPTION(NX_STRING);
MODULE_LICENSE("GPL");
MODULE_VERSION(NX_VERSION);

194
drivers/crypto/nx/nx.h Normal file
View file

@ -0,0 +1,194 @@
#ifndef __NX_H__
#define __NX_H__
#define NX_NAME "nx-crypto"
#define NX_STRING "IBM Power7+ Nest Accelerator Crypto Driver"
#define NX_VERSION "1.0"
static const char nx_driver_string[] = NX_STRING;
static const char nx_driver_version[] = NX_VERSION;
/* a scatterlist in the format PHYP is expecting */
struct nx_sg {
u64 addr;
u32 rsvd;
u32 len;
} __attribute((packed));
#define NX_PAGE_SIZE (4096)
#define NX_MAX_SG_ENTRIES (NX_PAGE_SIZE/(sizeof(struct nx_sg)))
enum nx_status {
NX_DISABLED,
NX_WAITING,
NX_OKAY
};
/* msc_triplet and max_sync_cop are used only to assist in parsing the
* openFirmware property */
struct msc_triplet {
u32 keybitlen;
u32 databytelen;
u32 sglen;
} __packed;
struct max_sync_cop {
u32 fc;
u32 mode;
u32 triplets;
struct msc_triplet trip[0];
} __packed;
struct alg_props {
u32 databytelen;
u32 sglen;
};
#define NX_OF_FLAG_MAXSGLEN_SET (1)
#define NX_OF_FLAG_STATUS_SET (2)
#define NX_OF_FLAG_MAXSYNCCOP_SET (4)
#define NX_OF_FLAG_MASK_READY (NX_OF_FLAG_MAXSGLEN_SET | \
NX_OF_FLAG_STATUS_SET | \
NX_OF_FLAG_MAXSYNCCOP_SET)
struct nx_of {
u32 flags;
u32 max_sg_len;
enum nx_status status;
struct alg_props ap[NX_MAX_FC][NX_MAX_MODE][3];
};
struct nx_stats {
atomic_t aes_ops;
atomic64_t aes_bytes;
atomic_t sha256_ops;
atomic64_t sha256_bytes;
atomic_t sha512_ops;
atomic64_t sha512_bytes;
atomic_t sync_ops;
atomic_t errors;
atomic_t last_error;
atomic_t last_error_pid;
};
struct nx_debugfs {
struct dentry *dfs_root;
struct dentry *dfs_aes_ops, *dfs_aes_bytes;
struct dentry *dfs_sha256_ops, *dfs_sha256_bytes;
struct dentry *dfs_sha512_ops, *dfs_sha512_bytes;
struct dentry *dfs_errors, *dfs_last_error, *dfs_last_error_pid;
};
struct nx_crypto_driver {
struct nx_stats stats;
struct nx_of of;
struct vio_dev *viodev;
struct vio_driver viodriver;
struct nx_debugfs dfs;
};
#define NX_GCM4106_NONCE_LEN (4)
#define NX_GCM_CTR_OFFSET (12)
struct nx_gcm_priv {
u8 iv[16];
u8 iauth_tag[16];
u8 nonce[NX_GCM4106_NONCE_LEN];
};
#define NX_CCM_AES_KEY_LEN (16)
#define NX_CCM4309_AES_KEY_LEN (19)
#define NX_CCM4309_NONCE_LEN (3)
struct nx_ccm_priv {
u8 iv[16];
u8 b0[16];
u8 iauth_tag[16];
u8 oauth_tag[16];
u8 nonce[NX_CCM4309_NONCE_LEN];
};
struct nx_xcbc_priv {
u8 key[16];
};
struct nx_ctr_priv {
u8 iv[16];
};
struct nx_crypto_ctx {
spinlock_t lock; /* synchronize access to the context */
void *kmem; /* unaligned, kmalloc'd buffer */
size_t kmem_len; /* length of kmem */
struct nx_csbcpb *csbcpb; /* aligned page given to phyp @ hcall time */
struct vio_pfo_op op; /* operation struct with hcall parameters */
struct nx_csbcpb *csbcpb_aead; /* secondary csbcpb used by AEAD algs */
struct vio_pfo_op op_aead;/* operation struct for csbcpb_aead */
struct nx_sg *in_sg; /* aligned pointer into kmem to an sg list */
struct nx_sg *out_sg; /* aligned pointer into kmem to an sg list */
struct alg_props *ap; /* pointer into props based on our key size */
struct alg_props props[3];/* openFirmware properties for requests */
struct nx_stats *stats; /* pointer into an nx_crypto_driver for stats
reporting */
union {
struct nx_gcm_priv gcm;
struct nx_ccm_priv ccm;
struct nx_xcbc_priv xcbc;
struct nx_ctr_priv ctr;
} priv;
};
/* prototypes */
int nx_crypto_ctx_aes_ccm_init(struct crypto_tfm *tfm);
int nx_crypto_ctx_aes_gcm_init(struct crypto_tfm *tfm);
int nx_crypto_ctx_aes_xcbc_init(struct crypto_tfm *tfm);
int nx_crypto_ctx_aes_ctr_init(struct crypto_tfm *tfm);
int nx_crypto_ctx_aes_cbc_init(struct crypto_tfm *tfm);
int nx_crypto_ctx_aes_ecb_init(struct crypto_tfm *tfm);
int nx_crypto_ctx_sha_init(struct crypto_tfm *tfm);
void nx_crypto_ctx_exit(struct crypto_tfm *tfm);
void nx_ctx_init(struct nx_crypto_ctx *nx_ctx, unsigned int function);
int nx_hcall_sync(struct nx_crypto_ctx *ctx, struct vio_pfo_op *op,
u32 may_sleep);
struct nx_sg *nx_build_sg_list(struct nx_sg *, u8 *, unsigned int, u32);
int nx_build_sg_lists(struct nx_crypto_ctx *, struct blkcipher_desc *,
struct scatterlist *, struct scatterlist *, unsigned int,
unsigned int, u8 *);
struct nx_sg *nx_walk_and_build(struct nx_sg *, unsigned int,
struct scatterlist *, unsigned int,
unsigned int);
#ifdef CONFIG_DEBUG_FS
#define NX_DEBUGFS_INIT(drv) nx_debugfs_init(drv)
#define NX_DEBUGFS_FINI(drv) nx_debugfs_fini(drv)
int nx_debugfs_init(struct nx_crypto_driver *);
void nx_debugfs_fini(struct nx_crypto_driver *);
#else
#define NX_DEBUGFS_INIT(drv) (0)
#define NX_DEBUGFS_FINI(drv) (0)
#endif
#define NX_PAGE_NUM(x) ((u64)(x) & 0xfffffffffffff000ULL)
extern struct crypto_alg nx_cbc_aes_alg;
extern struct crypto_alg nx_ecb_aes_alg;
extern struct crypto_alg nx_gcm_aes_alg;
extern struct crypto_alg nx_gcm4106_aes_alg;
extern struct crypto_alg nx_ctr_aes_alg;
extern struct crypto_alg nx_ctr3686_aes_alg;
extern struct crypto_alg nx_ccm_aes_alg;
extern struct crypto_alg nx_ccm4309_aes_alg;
extern struct shash_alg nx_shash_aes_xcbc_alg;
extern struct shash_alg nx_shash_sha512_alg;
extern struct shash_alg nx_shash_sha256_alg;
extern struct nx_crypto_driver nx_driver;
#define SCATTERWALK_TO_SG 1
#define SCATTERWALK_FROM_SG 0
#endif

205
drivers/crypto/nx/nx_csbcpb.h Normal file
View file

@ -0,0 +1,205 @@
#ifndef __NX_CSBCPB_H__
#define __NX_CSBCPB_H__
struct cop_symcpb_aes_ecb {
u8 key[32];
u8 __rsvd[80];
} __packed;
struct cop_symcpb_aes_cbc {
u8 iv[16];
u8 key[32];
u8 cv[16];
u32 spbc;
u8 __rsvd[44];
} __packed;
struct cop_symcpb_aes_gca {
u8 in_pat[16];
u8 key[32];
u8 out_pat[16];
u32 spbc;
u8 __rsvd[44];
} __packed;
struct cop_symcpb_aes_gcm {
u8 in_pat_or_aad[16];
u8 iv_or_cnt[16];
u64 bit_length_aad;
u64 bit_length_data;
u8 in_s0[16];
u8 key[32];
u8 __rsvd1[16];
u8 out_pat_or_mac[16];
u8 out_s0[16];
u8 out_cnt[16];
u32 spbc;
u8 __rsvd2[12];
} __packed;
struct cop_symcpb_aes_ctr {
u8 iv[16];
u8 key[32];
u8 cv[16];
u32 spbc;
u8 __rsvd2[44];
} __packed;
struct cop_symcpb_aes_cca {
u8 b0[16];
u8 b1[16];
u8 key[16];
u8 out_pat_or_b0[16];
u32 spbc;
u8 __rsvd[44];
} __packed;
struct cop_symcpb_aes_ccm {
u8 in_pat_or_b0[16];
u8 iv_or_ctr[16];
u8 in_s0[16];
u8 key[16];
u8 __rsvd1[48];
u8 out_pat_or_mac[16];
u8 out_s0[16];
u8 out_ctr[16];
u32 spbc;
u8 __rsvd2[12];
} __packed;
struct cop_symcpb_aes_xcbc {
u8 cv[16];
u8 key[16];
u8 __rsvd1[16];
u8 out_cv_mac[16];
u32 spbc;
u8 __rsvd2[44];
} __packed;
struct cop_symcpb_sha256 {
u64 message_bit_length;
u64 __rsvd1;
u8 input_partial_digest[32];
u8 message_digest[32];
u32 spbc;
u8 __rsvd2[44];
} __packed;
struct cop_symcpb_sha512 {
u64 message_bit_length_hi;
u64 message_bit_length_lo;
u8 input_partial_digest[64];
u8 __rsvd1[32];
u8 message_digest[64];
u32 spbc;
u8 __rsvd2[76];
} __packed;
#define NX_FDM_INTERMEDIATE 0x01
#define NX_FDM_CONTINUATION 0x02
#define NX_FDM_ENDE_ENCRYPT 0x80
#define NX_CPB_FDM(c) ((c)->cpb.hdr.fdm)
#define NX_CPB_KS_DS(c) ((c)->cpb.hdr.ks_ds)
#define NX_CPB_KEY_SIZE(c) (NX_CPB_KS_DS(c) >> 4)
#define NX_CPB_SET_KEY_SIZE(c, x) NX_CPB_KS_DS(c) |= ((x) << 4)
#define NX_CPB_SET_DIGEST_SIZE(c, x) NX_CPB_KS_DS(c) |= (x)
struct cop_symcpb_header {
u8 mode;
u8 fdm;
u8 ks_ds;
u8 pad_byte;
u8 __rsvd[12];
} __packed;
struct cop_parameter_block {
struct cop_symcpb_header hdr;
union {
struct cop_symcpb_aes_ecb aes_ecb;
struct cop_symcpb_aes_cbc aes_cbc;
struct cop_symcpb_aes_gca aes_gca;
struct cop_symcpb_aes_gcm aes_gcm;
struct cop_symcpb_aes_cca aes_cca;
struct cop_symcpb_aes_ccm aes_ccm;
struct cop_symcpb_aes_ctr aes_ctr;
struct cop_symcpb_aes_xcbc aes_xcbc;
struct cop_symcpb_sha256 sha256;
struct cop_symcpb_sha512 sha512;
};
} __packed;
#define NX_CSB_VALID_BIT 0x80
/* co-processor status block */
struct cop_status_block {
u8 valid;
u8 crb_seq_number;
u8 completion_code;
u8 completion_extension;
u32 processed_byte_count;
u64 address;
} __packed;
/* Nest accelerator workbook section 4.4 */
struct nx_csbcpb {
unsigned char __rsvd[112];
struct cop_status_block csb;
struct cop_parameter_block cpb;
} __packed;
/* nx_csbcpb related definitions */
#define NX_MODE_AES_ECB 0
#define NX_MODE_AES_CBC 1
#define NX_MODE_AES_GMAC 2
#define NX_MODE_AES_GCA 3
#define NX_MODE_AES_GCM 4
#define NX_MODE_AES_CCA 5
#define NX_MODE_AES_CCM 6
#define NX_MODE_AES_CTR 7
#define NX_MODE_AES_XCBC_MAC 20
#define NX_MODE_SHA 0
#define NX_MODE_SHA_HMAC 1
#define NX_MODE_AES_CBC_HMAC_ETA 8
#define NX_MODE_AES_CBC_HMAC_ATE 9
#define NX_MODE_AES_CBC_HMAC_EAA 10
#define NX_MODE_AES_CTR_HMAC_ETA 12
#define NX_MODE_AES_CTR_HMAC_ATE 13
#define NX_MODE_AES_CTR_HMAC_EAA 14
#define NX_FDM_CI_FULL 0
#define NX_FDM_CI_FIRST 1
#define NX_FDM_CI_LAST 2
#define NX_FDM_CI_MIDDLE 3
#define NX_FDM_PR_NONE 0
#define NX_FDM_PR_PAD 1
#define NX_KS_AES_128 1
#define NX_KS_AES_192 2
#define NX_KS_AES_256 3
#define NX_DS_SHA256 2
#define NX_DS_SHA512 3
#define NX_FC_AES 0
#define NX_FC_SHA 2
#define NX_FC_AES_HMAC 6
#define NX_MAX_FC (NX_FC_AES_HMAC + 1)
#define NX_MAX_MODE (NX_MODE_AES_XCBC_MAC + 1)
#define HCOP_FC_AES NX_FC_AES
#define HCOP_FC_SHA NX_FC_SHA
#define HCOP_FC_AES_HMAC NX_FC_AES_HMAC
/* indices into the array of algorithm properties */
#define NX_PROPS_AES_128 0
#define NX_PROPS_AES_192 1
#define NX_PROPS_AES_256 2
#define NX_PROPS_SHA256 1
#define NX_PROPS_SHA512 2
#endif

103
drivers/crypto/nx/nx_debugfs.c Normal file
View file

@ -0,0 +1,103 @@
/**
* debugfs routines supporting the Power 7+ Nest Accelerators driver
*
* Copyright (C) 2011-2012 International Business Machines Inc.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; version 2 only.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*
* Author: Kent Yoder <yoder1@us.ibm.com>
*/
#include <linux/device.h>
#include <linux/kobject.h>
#include <linux/string.h>
#include <linux/debugfs.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/crypto.h>
#include <crypto/hash.h>
#include <asm/vio.h>
#include "nx_csbcpb.h"
#include "nx.h"
#ifdef CONFIG_DEBUG_FS
/*
* debugfs
*
* For documentation on these attributes, please see:
*
* Documentation/ABI/testing/debugfs-pfo-nx-crypto
*/
int nx_debugfs_init(struct nx_crypto_driver *drv)
{
struct nx_debugfs *dfs = &drv->dfs;
dfs->dfs_root = debugfs_create_dir(NX_NAME, NULL);
dfs->dfs_aes_ops =
debugfs_create_u32("aes_ops",
S_IRUSR | S_IRGRP | S_IROTH,
dfs->dfs_root, (u32 *)&drv->stats.aes_ops);
dfs->dfs_sha256_ops =
debugfs_create_u32("sha256_ops",
S_IRUSR | S_IRGRP | S_IROTH,
dfs->dfs_root,
(u32 *)&drv->stats.sha256_ops);
dfs->dfs_sha512_ops =
debugfs_create_u32("sha512_ops",
S_IRUSR | S_IRGRP | S_IROTH,
dfs->dfs_root,
(u32 *)&drv->stats.sha512_ops);
dfs->dfs_aes_bytes =
debugfs_create_u64("aes_bytes",
S_IRUSR | S_IRGRP | S_IROTH,
dfs->dfs_root,
(u64 *)&drv->stats.aes_bytes);
dfs->dfs_sha256_bytes =
debugfs_create_u64("sha256_bytes",
S_IRUSR | S_IRGRP | S_IROTH,
dfs->dfs_root,
(u64 *)&drv->stats.sha256_bytes);
dfs->dfs_sha512_bytes =
debugfs_create_u64("sha512_bytes",
S_IRUSR | S_IRGRP | S_IROTH,
dfs->dfs_root,
(u64 *)&drv->stats.sha512_bytes);
dfs->dfs_errors =
debugfs_create_u32("errors",
S_IRUSR | S_IRGRP | S_IROTH,
dfs->dfs_root, (u32 *)&drv->stats.errors);
dfs->dfs_last_error =
debugfs_create_u32("last_error",
S_IRUSR | S_IRGRP | S_IROTH,
dfs->dfs_root,
(u32 *)&drv->stats.last_error);
dfs->dfs_last_error_pid =
debugfs_create_u32("last_error_pid",
S_IRUSR | S_IRGRP | S_IROTH,
dfs->dfs_root,
(u32 *)&drv->stats.last_error_pid);
return 0;
}
void
nx_debugfs_fini(struct nx_crypto_driver *drv)
{
debugfs_remove_recursive(drv->dfs.dfs_root);
}
#endif

1336
drivers/crypto/omap-aes.c Normal file
View file

File diff suppressed because it is too large Load diff

1235
drivers/crypto/omap-des.c Normal file
View file

File diff suppressed because it is too large Load diff

2043
drivers/crypto/omap-sham.c Normal file
View file

File diff suppressed because it is too large Load diff

566
drivers/crypto/padlock-aes.c Normal file
View file

@ -0,0 +1,566 @@
/*
* Cryptographic API.
*
* Support for VIA PadLock hardware crypto engine.
*
* Copyright (c) 2004 Michal Ludvig <michal@logix.cz>
*
*/
#include <crypto/algapi.h>
#include <crypto/aes.h>
#include <crypto/padlock.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/percpu.h>
#include <linux/smp.h>
#include <linux/slab.h>
#include <asm/cpu_device_id.h>
#include <asm/byteorder.h>
#include <asm/processor.h>
#include <asm/i387.h>
/*
* Number of data blocks actually fetched for each xcrypt insn.
* Processors with prefetch errata will fetch extra blocks.
*/
static unsigned int ecb_fetch_blocks = 2;
#define MAX_ECB_FETCH_BLOCKS (8)
#define ecb_fetch_bytes (ecb_fetch_blocks * AES_BLOCK_SIZE)
static unsigned int cbc_fetch_blocks = 1;
#define MAX_CBC_FETCH_BLOCKS (4)
#define cbc_fetch_bytes (cbc_fetch_blocks * AES_BLOCK_SIZE)
/* Control word. */
struct cword {
unsigned int __attribute__ ((__packed__))
rounds:4,
algo:3,
keygen:1,
interm:1,
encdec:1,
ksize:2;
} __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
/* Whenever making any changes to the following
* structure *make sure* you keep E, d_data
* and cword aligned on 16 Bytes boundaries and
* the Hardware can access 16 * 16 bytes of E and d_data
* (only the first 15 * 16 bytes matter but the HW reads
* more).
*/
struct aes_ctx {
u32 E[AES_MAX_KEYLENGTH_U32]
__attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
u32 d_data[AES_MAX_KEYLENGTH_U32]
__attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
struct {
struct cword encrypt;
struct cword decrypt;
} cword;
u32 *D;
};
static DEFINE_PER_CPU(struct cword *, paes_last_cword);
/* Tells whether the ACE is capable to generate
the extended key for a given key_len. */
static inline int
aes_hw_extkey_available(uint8_t key_len)
{
/* TODO: We should check the actual CPU model/stepping
as it's possible that the capability will be
added in the next CPU revisions. */
if (key_len == 16)
return 1;
return 0;
}
static inline struct aes_ctx *aes_ctx_common(void *ctx)
{
unsigned long addr = (unsigned long)ctx;
unsigned long align = PADLOCK_ALIGNMENT;
if (align <= crypto_tfm_ctx_alignment())
align = 1;
return (struct aes_ctx *)ALIGN(addr, align);
}
static inline struct aes_ctx *aes_ctx(struct crypto_tfm *tfm)
{
return aes_ctx_common(crypto_tfm_ctx(tfm));
}
static inline struct aes_ctx *blk_aes_ctx(struct crypto_blkcipher *tfm)
{
return aes_ctx_common(crypto_blkcipher_ctx(tfm));
}
static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
unsigned int key_len)
{
struct aes_ctx *ctx = aes_ctx(tfm);
const __le32 *key = (const __le32 *)in_key;
u32 *flags = &tfm->crt_flags;
struct crypto_aes_ctx gen_aes;
int cpu;
if (key_len % 8) {
*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
return -EINVAL;
}
/*
* If the hardware is capable of generating the extended key
* itself we must supply the plain key for both encryption
* and decryption.
*/
ctx->D = ctx->E;
ctx->E[0] = le32_to_cpu(key[0]);
ctx->E[1] = le32_to_cpu(key[1]);
ctx->E[2] = le32_to_cpu(key[2]);
ctx->E[3] = le32_to_cpu(key[3]);
/* Prepare control words. */
memset(&ctx->cword, 0, sizeof(ctx->cword));
ctx->cword.decrypt.encdec = 1;
ctx->cword.encrypt.rounds = 10 + (key_len - 16) / 4;
ctx->cword.decrypt.rounds = ctx->cword.encrypt.rounds;
ctx->cword.encrypt.ksize = (key_len - 16) / 8;
ctx->cword.decrypt.ksize = ctx->cword.encrypt.ksize;
/* Don't generate extended keys if the hardware can do it. */
if (aes_hw_extkey_available(key_len))
goto ok;
ctx->D = ctx->d_data;
ctx->cword.encrypt.keygen = 1;
ctx->cword.decrypt.keygen = 1;
if (crypto_aes_expand_key(&gen_aes, in_key, key_len)) {
*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
return -EINVAL;
}
memcpy(ctx->E, gen_aes.key_enc, AES_MAX_KEYLENGTH);
memcpy(ctx->D, gen_aes.key_dec, AES_MAX_KEYLENGTH);
ok:
for_each_online_cpu(cpu)
if (&ctx->cword.encrypt == per_cpu(paes_last_cword, cpu) ||
&ctx->cword.decrypt == per_cpu(paes_last_cword, cpu))
per_cpu(paes_last_cword, cpu) = NULL;
return 0;
}
/* ====== Encryption/decryption routines ====== */
/* These are the real call to PadLock. */
static inline void padlock_reset_key(struct cword *cword)
{
int cpu = raw_smp_processor_id();
if (cword != per_cpu(paes_last_cword, cpu))
#ifndef CONFIG_X86_64
asm volatile ("pushfl; popfl");
#else
asm volatile ("pushfq; popfq");
#endif
}
static inline void padlock_store_cword(struct cword *cword)
{
per_cpu(paes_last_cword, raw_smp_processor_id()) = cword;
}
/*
* While the padlock instructions don't use FP/SSE registers, they
* generate a spurious DNA fault when cr0.ts is '1'. These instructions
* should be used only inside the irq_ts_save/restore() context
*/
static inline void rep_xcrypt_ecb(const u8 *input, u8 *output, void *key,
struct cword *control_word, int count)
{
asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
: "+S"(input), "+D"(output)
: "d"(control_word), "b"(key), "c"(count));
}
static inline u8 *rep_xcrypt_cbc(const u8 *input, u8 *output, void *key,
u8 *iv, struct cword *control_word, int count)
{
asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
: "+S" (input), "+D" (output), "+a" (iv)
: "d" (control_word), "b" (key), "c" (count));
return iv;
}
static void ecb_crypt_copy(const u8 *in, u8 *out, u32 *key,
struct cword *cword, int count)
{
/*
* Padlock prefetches extra data so we must provide mapped input buffers.
* Assume there are at least 16 bytes of stack already in use.
*/
u8 buf[AES_BLOCK_SIZE * (MAX_ECB_FETCH_BLOCKS - 1) + PADLOCK_ALIGNMENT - 1];
u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
memcpy(tmp, in, count * AES_BLOCK_SIZE);
rep_xcrypt_ecb(tmp, out, key, cword, count);
}
static u8 *cbc_crypt_copy(const u8 *in, u8 *out, u32 *key,
u8 *iv, struct cword *cword, int count)
{
/*
* Padlock prefetches extra data so we must provide mapped input buffers.
* Assume there are at least 16 bytes of stack already in use.
*/
u8 buf[AES_BLOCK_SIZE * (MAX_CBC_FETCH_BLOCKS - 1) + PADLOCK_ALIGNMENT - 1];
u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
memcpy(tmp, in, count * AES_BLOCK_SIZE);
return rep_xcrypt_cbc(tmp, out, key, iv, cword, count);
}
static inline void ecb_crypt(const u8 *in, u8 *out, u32 *key,
struct cword *cword, int count)
{
/* Padlock in ECB mode fetches at least ecb_fetch_bytes of data.
* We could avoid some copying here but it's probably not worth it.
*/
if (unlikely(((unsigned long)in & ~PAGE_MASK) + ecb_fetch_bytes > PAGE_SIZE)) {
ecb_crypt_copy(in, out, key, cword, count);
return;
}
rep_xcrypt_ecb(in, out, key, cword, count);
}
static inline u8 *cbc_crypt(const u8 *in, u8 *out, u32 *key,
u8 *iv, struct cword *cword, int count)
{
/* Padlock in CBC mode fetches at least cbc_fetch_bytes of data. */
if (unlikely(((unsigned long)in & ~PAGE_MASK) + cbc_fetch_bytes > PAGE_SIZE))
return cbc_crypt_copy(in, out, key, iv, cword, count);
return rep_xcrypt_cbc(in, out, key, iv, cword, count);
}
static inline void padlock_xcrypt_ecb(const u8 *input, u8 *output, void *key,
void *control_word, u32 count)
{
u32 initial = count & (ecb_fetch_blocks - 1);
if (count < ecb_fetch_blocks) {
ecb_crypt(input, output, key, control_word, count);
return;
}
if (initial)
asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
: "+S"(input), "+D"(output)
: "d"(control_word), "b"(key), "c"(initial));
asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
: "+S"(input), "+D"(output)
: "d"(control_word), "b"(key), "c"(count - initial));
}
static inline u8 *padlock_xcrypt_cbc(const u8 *input, u8 *output, void *key,
u8 *iv, void *control_word, u32 count)
{
u32 initial = count & (cbc_fetch_blocks - 1);
if (count < cbc_fetch_blocks)
return cbc_crypt(input, output, key, iv, control_word, count);
if (initial)
asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
: "+S" (input), "+D" (output), "+a" (iv)
: "d" (control_word), "b" (key), "c" (initial));
asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
: "+S" (input), "+D" (output), "+a" (iv)
: "d" (control_word), "b" (key), "c" (count-initial));
return iv;
}
static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
struct aes_ctx *ctx = aes_ctx(tfm);
int ts_state;
padlock_reset_key(&ctx->cword.encrypt);
ts_state = irq_ts_save();
ecb_crypt(in, out, ctx->E, &ctx->cword.encrypt, 1);
irq_ts_restore(ts_state);
padlock_store_cword(&ctx->cword.encrypt);
}
static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
struct aes_ctx *ctx = aes_ctx(tfm);
int ts_state;
padlock_reset_key(&ctx->cword.encrypt);
ts_state = irq_ts_save();
ecb_crypt(in, out, ctx->D, &ctx->cword.decrypt, 1);
irq_ts_restore(ts_state);
padlock_store_cword(&ctx->cword.encrypt);
}
static struct crypto_alg aes_alg = {
.cra_name = "aes",
.cra_driver_name = "aes-padlock",
.cra_priority = PADLOCK_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct aes_ctx),
.cra_alignmask = PADLOCK_ALIGNMENT - 1,
.cra_module = THIS_MODULE,
.cra_u = {
.cipher = {
.cia_min_keysize = AES_MIN_KEY_SIZE,
.cia_max_keysize = AES_MAX_KEY_SIZE,
.cia_setkey = aes_set_key,
.cia_encrypt = aes_encrypt,
.cia_decrypt = aes_decrypt,
}
}
};
static int ecb_aes_encrypt(struct blkcipher_desc *desc,
struct scatterlist *dst, struct scatterlist *src,
unsigned int nbytes)
{
struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
struct blkcipher_walk walk;
int err;
int ts_state;
padlock_reset_key(&ctx->cword.encrypt);
blkcipher_walk_init(&walk, dst, src, nbytes);
err = blkcipher_walk_virt(desc, &walk);
ts_state = irq_ts_save();
while ((nbytes = walk.nbytes)) {
padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
ctx->E, &ctx->cword.encrypt,
nbytes / AES_BLOCK_SIZE);
nbytes &= AES_BLOCK_SIZE - 1;
err = blkcipher_walk_done(desc, &walk, nbytes);
}
irq_ts_restore(ts_state);
padlock_store_cword(&ctx->cword.encrypt);
return err;
}
static int ecb_aes_decrypt(struct blkcipher_desc *desc,
struct scatterlist *dst, struct scatterlist *src,
unsigned int nbytes)
{
struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
struct blkcipher_walk walk;
int err;
int ts_state;
padlock_reset_key(&ctx->cword.decrypt);
blkcipher_walk_init(&walk, dst, src, nbytes);
err = blkcipher_walk_virt(desc, &walk);
ts_state = irq_ts_save();
while ((nbytes = walk.nbytes)) {
padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
ctx->D, &ctx->cword.decrypt,
nbytes / AES_BLOCK_SIZE);
nbytes &= AES_BLOCK_SIZE - 1;
err = blkcipher_walk_done(desc, &walk, nbytes);
}
irq_ts_restore(ts_state);
padlock_store_cword(&ctx->cword.encrypt);
return err;
}
static struct crypto_alg ecb_aes_alg = {
.cra_name = "ecb(aes)",
.cra_driver_name = "ecb-aes-padlock",
.cra_priority = PADLOCK_COMPOSITE_PRIORITY,
.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct aes_ctx),
.cra_alignmask = PADLOCK_ALIGNMENT - 1,
.cra_type = &crypto_blkcipher_type,
.cra_module = THIS_MODULE,
.cra_u = {
.blkcipher = {
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.setkey = aes_set_key,
.encrypt = ecb_aes_encrypt,
.decrypt = ecb_aes_decrypt,
}
}
};
static int cbc_aes_encrypt(struct blkcipher_desc *desc,
struct scatterlist *dst, struct scatterlist *src,
unsigned int nbytes)
{
struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
struct blkcipher_walk walk;
int err;
int ts_state;
padlock_reset_key(&ctx->cword.encrypt);
blkcipher_walk_init(&walk, dst, src, nbytes);
err = blkcipher_walk_virt(desc, &walk);
ts_state = irq_ts_save();
while ((nbytes = walk.nbytes)) {
u8 *iv = padlock_xcrypt_cbc(walk.src.virt.addr,
walk.dst.virt.addr, ctx->E,
walk.iv, &ctx->cword.encrypt,
nbytes / AES_BLOCK_SIZE);
memcpy(walk.iv, iv, AES_BLOCK_SIZE);
nbytes &= AES_BLOCK_SIZE - 1;
err = blkcipher_walk_done(desc, &walk, nbytes);
}
irq_ts_restore(ts_state);
padlock_store_cword(&ctx->cword.decrypt);
return err;
}
static int cbc_aes_decrypt(struct blkcipher_desc *desc,
struct scatterlist *dst, struct scatterlist *src,
unsigned int nbytes)
{
struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
struct blkcipher_walk walk;
int err;
int ts_state;
padlock_reset_key(&ctx->cword.encrypt);
blkcipher_walk_init(&walk, dst, src, nbytes);
err = blkcipher_walk_virt(desc, &walk);
ts_state = irq_ts_save();
while ((nbytes = walk.nbytes)) {
padlock_xcrypt_cbc(walk.src.virt.addr, walk.dst.virt.addr,
ctx->D, walk.iv, &ctx->cword.decrypt,
nbytes / AES_BLOCK_SIZE);
nbytes &= AES_BLOCK_SIZE - 1;
err = blkcipher_walk_done(desc, &walk, nbytes);
}
irq_ts_restore(ts_state);
padlock_store_cword(&ctx->cword.encrypt);
return err;
}
static struct crypto_alg cbc_aes_alg = {
.cra_name = "cbc(aes)",
.cra_driver_name = "cbc-aes-padlock",
.cra_priority = PADLOCK_COMPOSITE_PRIORITY,
.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct aes_ctx),
.cra_alignmask = PADLOCK_ALIGNMENT - 1,
.cra_type = &crypto_blkcipher_type,
.cra_module = THIS_MODULE,
.cra_u = {
.blkcipher = {
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
.setkey = aes_set_key,
.encrypt = cbc_aes_encrypt,
.decrypt = cbc_aes_decrypt,
}
}
};
static struct x86_cpu_id padlock_cpu_id[] = {
X86_FEATURE_MATCH(X86_FEATURE_XCRYPT),
{}
};
MODULE_DEVICE_TABLE(x86cpu, padlock_cpu_id);
static int __init padlock_init(void)
{
int ret;
struct cpuinfo_x86 *c = &cpu_data(0);
if (!x86_match_cpu(padlock_cpu_id))
return -ENODEV;
if (!cpu_has_xcrypt_enabled) {
printk(KERN_NOTICE PFX "VIA PadLock detected, but not enabled. Hmm, strange...\n");
return -ENODEV;
}
if ((ret = crypto_register_alg(&aes_alg)))
goto aes_err;
if ((ret = crypto_register_alg(&ecb_aes_alg)))
goto ecb_aes_err;
if ((ret = crypto_register_alg(&cbc_aes_alg)))
goto cbc_aes_err;
printk(KERN_NOTICE PFX "Using VIA PadLock ACE for AES algorithm.\n");
if (c->x86 == 6 && c->x86_model == 15 && c->x86_mask == 2) {
ecb_fetch_blocks = MAX_ECB_FETCH_BLOCKS;
cbc_fetch_blocks = MAX_CBC_FETCH_BLOCKS;
printk(KERN_NOTICE PFX "VIA Nano stepping 2 detected: enabling workaround.\n");
}
out:
return ret;
cbc_aes_err:
crypto_unregister_alg(&ecb_aes_alg);
ecb_aes_err:
crypto_unregister_alg(&aes_alg);
aes_err:
printk(KERN_ERR PFX "VIA PadLock AES initialization failed.\n");
goto out;
}
static void __exit padlock_fini(void)
{
crypto_unregister_alg(&cbc_aes_alg);
crypto_unregister_alg(&ecb_aes_alg);
crypto_unregister_alg(&aes_alg);
}
module_init(padlock_init);
module_exit(padlock_fini);
MODULE_DESCRIPTION("VIA PadLock AES algorithm support");
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Michal Ludvig");
MODULE_ALIAS_CRYPTO("aes");

599
drivers/crypto/padlock-sha.c Normal file
View file

@ -0,0 +1,599 @@
/*
* Cryptographic API.
*
* Support for VIA PadLock hardware crypto engine.
*
* Copyright (c) 2006 Michal Ludvig <michal@logix.cz>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
*/
#include <crypto/internal/hash.h>
#include <crypto/padlock.h>
#include <crypto/sha.h>
#include <linux/err.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/scatterlist.h>
#include <asm/cpu_device_id.h>
#include <asm/i387.h>
struct padlock_sha_desc {
struct shash_desc fallback;
};
struct padlock_sha_ctx {
struct crypto_shash *fallback;
};
static int padlock_sha_init(struct shash_desc *desc)
{
struct padlock_sha_desc *dctx = shash_desc_ctx(desc);
struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm);
dctx->fallback.tfm = ctx->fallback;
dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
return crypto_shash_init(&dctx->fallback);
}
static int padlock_sha_update(struct shash_desc *desc,
const u8 *data, unsigned int length)
{
struct padlock_sha_desc *dctx = shash_desc_ctx(desc);
dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
return crypto_shash_update(&dctx->fallback, data, length);
}
static int padlock_sha_export(struct shash_desc *desc, void *out)
{
struct padlock_sha_desc *dctx = shash_desc_ctx(desc);
return crypto_shash_export(&dctx->fallback, out);
}
static int padlock_sha_import(struct shash_desc *desc, const void *in)
{
struct padlock_sha_desc *dctx = shash_desc_ctx(desc);
struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm);
dctx->fallback.tfm = ctx->fallback;
dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
return crypto_shash_import(&dctx->fallback, in);
}
static inline void padlock_output_block(uint32_t *src,
uint32_t *dst, size_t count)
{
while (count--)
*dst++ = swab32(*src++);
}
static int padlock_sha1_finup(struct shash_desc *desc, const u8 *in,
unsigned int count, u8 *out)
{
/* We can't store directly to *out as it may be unaligned. */
/* BTW Don't reduce the buffer size below 128 Bytes!
* PadLock microcode needs it that big. */
char buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__
((aligned(STACK_ALIGN)));
char *result = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
struct padlock_sha_desc *dctx = shash_desc_ctx(desc);
struct sha1_state state;
unsigned int space;
unsigned int leftover;
int ts_state;
int err;
dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
err = crypto_shash_export(&dctx->fallback, &state);
if (err)
goto out;
if (state.count + count > ULONG_MAX)
return crypto_shash_finup(&dctx->fallback, in, count, out);
leftover = ((state.count - 1) & (SHA1_BLOCK_SIZE - 1)) + 1;
space = SHA1_BLOCK_SIZE - leftover;
if (space) {
if (count > space) {
err = crypto_shash_update(&dctx->fallback, in, space) ?:
crypto_shash_export(&dctx->fallback, &state);
if (err)
goto out;
count -= space;
in += space;
} else {
memcpy(state.buffer + leftover, in, count);
in = state.buffer;
count += leftover;
state.count &= ~(SHA1_BLOCK_SIZE - 1);
}
}
memcpy(result, &state.state, SHA1_DIGEST_SIZE);
/* prevent taking the spurious DNA fault with padlock. */
ts_state = irq_ts_save();
asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" /* rep xsha1 */
: \
: "c"((unsigned long)state.count + count), \
"a"((unsigned long)state.count), \
"S"(in), "D"(result));
irq_ts_restore(ts_state);
padlock_output_block((uint32_t *)result, (uint32_t *)out, 5);
out:
return err;
}
static int padlock_sha1_final(struct shash_desc *desc, u8 *out)
{
u8 buf[4];
return padlock_sha1_finup(desc, buf, 0, out);
}
static int padlock_sha256_finup(struct shash_desc *desc, const u8 *in,
unsigned int count, u8 *out)
{
/* We can't store directly to *out as it may be unaligned. */
/* BTW Don't reduce the buffer size below 128 Bytes!
* PadLock microcode needs it that big. */
char buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__
((aligned(STACK_ALIGN)));
char *result = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
struct padlock_sha_desc *dctx = shash_desc_ctx(desc);
struct sha256_state state;
unsigned int space;
unsigned int leftover;
int ts_state;
int err;
dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
err = crypto_shash_export(&dctx->fallback, &state);
if (err)
goto out;
if (state.count + count > ULONG_MAX)
return crypto_shash_finup(&dctx->fallback, in, count, out);
leftover = ((state.count - 1) & (SHA256_BLOCK_SIZE - 1)) + 1;
space = SHA256_BLOCK_SIZE - leftover;
if (space) {
if (count > space) {
err = crypto_shash_update(&dctx->fallback, in, space) ?:
crypto_shash_export(&dctx->fallback, &state);
if (err)
goto out;
count -= space;
in += space;
} else {
memcpy(state.buf + leftover, in, count);
in = state.buf;
count += leftover;
state.count &= ~(SHA1_BLOCK_SIZE - 1);
}
}
memcpy(result, &state.state, SHA256_DIGEST_SIZE);
/* prevent taking the spurious DNA fault with padlock. */
ts_state = irq_ts_save();
asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" /* rep xsha256 */
: \
: "c"((unsigned long)state.count + count), \
"a"((unsigned long)state.count), \
"S"(in), "D"(result));
irq_ts_restore(ts_state);
padlock_output_block((uint32_t *)result, (uint32_t *)out, 8);
out:
return err;
}
static int padlock_sha256_final(struct shash_desc *desc, u8 *out)
{
u8 buf[4];
return padlock_sha256_finup(desc, buf, 0, out);
}
static int padlock_cra_init(struct crypto_tfm *tfm)
{
struct crypto_shash *hash = __crypto_shash_cast(tfm);
const char *fallback_driver_name = crypto_tfm_alg_name(tfm);
struct padlock_sha_ctx *ctx = crypto_tfm_ctx(tfm);
struct crypto_shash *fallback_tfm;
int err = -ENOMEM;
/* Allocate a fallback and abort if it failed. */
fallback_tfm = crypto_alloc_shash(fallback_driver_name, 0,
CRYPTO_ALG_NEED_FALLBACK);
if (IS_ERR(fallback_tfm)) {
printk(KERN_WARNING PFX "Fallback driver '%s' could not be loaded!\n",
fallback_driver_name);
err = PTR_ERR(fallback_tfm);
goto out;
}
ctx->fallback = fallback_tfm;
hash->descsize += crypto_shash_descsize(fallback_tfm);
return 0;
out:
return err;
}
static void padlock_cra_exit(struct crypto_tfm *tfm)
{
struct padlock_sha_ctx *ctx = crypto_tfm_ctx(tfm);
crypto_free_shash(ctx->fallback);
}
static struct shash_alg sha1_alg = {
.digestsize = SHA1_DIGEST_SIZE,
.init = padlock_sha_init,
.update = padlock_sha_update,
.finup = padlock_sha1_finup,
.final = padlock_sha1_final,
.export = padlock_sha_export,
.import = padlock_sha_import,
.descsize = sizeof(struct padlock_sha_desc),
.statesize = sizeof(struct sha1_state),
.base = {
.cra_name = "sha1",
.cra_driver_name = "sha1-padlock",
.cra_priority = PADLOCK_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_TYPE_SHASH |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = SHA1_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct padlock_sha_ctx),
.cra_module = THIS_MODULE,
.cra_init = padlock_cra_init,
.cra_exit = padlock_cra_exit,
}
};
static struct shash_alg sha256_alg = {
.digestsize = SHA256_DIGEST_SIZE,
.init = padlock_sha_init,
.update = padlock_sha_update,
.finup = padlock_sha256_finup,
.final = padlock_sha256_final,
.export = padlock_sha_export,
.import = padlock_sha_import,
.descsize = sizeof(struct padlock_sha_desc),
.statesize = sizeof(struct sha256_state),
.base = {
.cra_name = "sha256",
.cra_driver_name = "sha256-padlock",
.cra_priority = PADLOCK_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_TYPE_SHASH |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = SHA256_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct padlock_sha_ctx),
.cra_module = THIS_MODULE,
.cra_init = padlock_cra_init,
.cra_exit = padlock_cra_exit,
}
};
/* Add two shash_alg instance for hardware-implemented *
* multiple-parts hash supported by VIA Nano Processor.*/
static int padlock_sha1_init_nano(struct shash_desc *desc)
{
struct sha1_state *sctx = shash_desc_ctx(desc);
*sctx = (struct sha1_state){
.state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 },
};
return 0;
}
static int padlock_sha1_update_nano(struct shash_desc *desc,
const u8 *data, unsigned int len)
{
struct sha1_state *sctx = shash_desc_ctx(desc);
unsigned int partial, done;
const u8 *src;
/*The PHE require the out buffer must 128 bytes and 16-bytes aligned*/
u8 buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__
((aligned(STACK_ALIGN)));
u8 *dst = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
int ts_state;
partial = sctx->count & 0x3f;
sctx->count += len;
done = 0;
src = data;
memcpy(dst, (u8 *)(sctx->state), SHA1_DIGEST_SIZE);
if ((partial + len) >= SHA1_BLOCK_SIZE) {
/* Append the bytes in state's buffer to a block to handle */
if (partial) {
done = -partial;
memcpy(sctx->buffer + partial, data,
done + SHA1_BLOCK_SIZE);
src = sctx->buffer;
ts_state = irq_ts_save();
asm volatile (".byte 0xf3,0x0f,0xa6,0xc8"
: "+S"(src), "+D"(dst) \
: "a"((long)-1), "c"((unsigned long)1));
irq_ts_restore(ts_state);
done += SHA1_BLOCK_SIZE;
src = data + done;
}
/* Process the left bytes from the input data */
if (len - done >= SHA1_BLOCK_SIZE) {
ts_state = irq_ts_save();
asm volatile (".byte 0xf3,0x0f,0xa6,0xc8"
: "+S"(src), "+D"(dst)
: "a"((long)-1),
"c"((unsigned long)((len - done) / SHA1_BLOCK_SIZE)));
irq_ts_restore(ts_state);
done += ((len - done) - (len - done) % SHA1_BLOCK_SIZE);
src = data + done;
}
partial = 0;
}
memcpy((u8 *)(sctx->state), dst, SHA1_DIGEST_SIZE);
memcpy(sctx->buffer + partial, src, len - done);
return 0;
}
static int padlock_sha1_final_nano(struct shash_desc *desc, u8 *out)
{
struct sha1_state *state = (struct sha1_state *)shash_desc_ctx(desc);
unsigned int partial, padlen;
__be64 bits;
static const u8 padding[64] = { 0x80, };
bits = cpu_to_be64(state->count << 3);
/* Pad out to 56 mod 64 */
partial = state->count & 0x3f;
padlen = (partial < 56) ? (56 - partial) : ((64+56) - partial);
padlock_sha1_update_nano(desc, padding, padlen);
/* Append length field bytes */
padlock_sha1_update_nano(desc, (const u8 *)&bits, sizeof(bits));
/* Swap to output */
padlock_output_block((uint32_t *)(state->state), (uint32_t *)out, 5);
return 0;
}
static int padlock_sha256_init_nano(struct shash_desc *desc)
{
struct sha256_state *sctx = shash_desc_ctx(desc);
*sctx = (struct sha256_state){
.state = { SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, \
SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7},
};
return 0;
}
static int padlock_sha256_update_nano(struct shash_desc *desc, const u8 *data,
unsigned int len)
{
struct sha256_state *sctx = shash_desc_ctx(desc);
unsigned int partial, done;
const u8 *src;
/*The PHE require the out buffer must 128 bytes and 16-bytes aligned*/
u8 buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__
((aligned(STACK_ALIGN)));
u8 *dst = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
int ts_state;
partial = sctx->count & 0x3f;
sctx->count += len;
done = 0;
src = data;
memcpy(dst, (u8 *)(sctx->state), SHA256_DIGEST_SIZE);
if ((partial + len) >= SHA256_BLOCK_SIZE) {
/* Append the bytes in state's buffer to a block to handle */
if (partial) {
done = -partial;
memcpy(sctx->buf + partial, data,
done + SHA256_BLOCK_SIZE);
src = sctx->buf;
ts_state = irq_ts_save();
asm volatile (".byte 0xf3,0x0f,0xa6,0xd0"
: "+S"(src), "+D"(dst)
: "a"((long)-1), "c"((unsigned long)1));
irq_ts_restore(ts_state);
done += SHA256_BLOCK_SIZE;
src = data + done;
}
/* Process the left bytes from input data*/
if (len - done >= SHA256_BLOCK_SIZE) {
ts_state = irq_ts_save();
asm volatile (".byte 0xf3,0x0f,0xa6,0xd0"
: "+S"(src), "+D"(dst)
: "a"((long)-1),
"c"((unsigned long)((len - done) / 64)));
irq_ts_restore(ts_state);
done += ((len - done) - (len - done) % 64);
src = data + done;
}
partial = 0;
}
memcpy((u8 *)(sctx->state), dst, SHA256_DIGEST_SIZE);
memcpy(sctx->buf + partial, src, len - done);
return 0;
}
static int padlock_sha256_final_nano(struct shash_desc *desc, u8 *out)
{
struct sha256_state *state =
(struct sha256_state *)shash_desc_ctx(desc);
unsigned int partial, padlen;
__be64 bits;
static const u8 padding[64] = { 0x80, };
bits = cpu_to_be64(state->count << 3);
/* Pad out to 56 mod 64 */
partial = state->count & 0x3f;
padlen = (partial < 56) ? (56 - partial) : ((64+56) - partial);
padlock_sha256_update_nano(desc, padding, padlen);
/* Append length field bytes */
padlock_sha256_update_nano(desc, (const u8 *)&bits, sizeof(bits));
/* Swap to output */
padlock_output_block((uint32_t *)(state->state), (uint32_t *)out, 8);
return 0;
}
static int padlock_sha_export_nano(struct shash_desc *desc,
void *out)
{
int statesize = crypto_shash_statesize(desc->tfm);
void *sctx = shash_desc_ctx(desc);
memcpy(out, sctx, statesize);
return 0;
}
static int padlock_sha_import_nano(struct shash_desc *desc,
const void *in)
{
int statesize = crypto_shash_statesize(desc->tfm);
void *sctx = shash_desc_ctx(desc);
memcpy(sctx, in, statesize);
return 0;
}
static struct shash_alg sha1_alg_nano = {
.digestsize = SHA1_DIGEST_SIZE,
.init = padlock_sha1_init_nano,
.update = padlock_sha1_update_nano,
.final = padlock_sha1_final_nano,
.export = padlock_sha_export_nano,
.import = padlock_sha_import_nano,
.descsize = sizeof(struct sha1_state),
.statesize = sizeof(struct sha1_state),
.base = {
.cra_name = "sha1",
.cra_driver_name = "sha1-padlock-nano",
.cra_priority = PADLOCK_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_TYPE_SHASH,
.cra_blocksize = SHA1_BLOCK_SIZE,
.cra_module = THIS_MODULE,
}
};
static struct shash_alg sha256_alg_nano = {
.digestsize = SHA256_DIGEST_SIZE,
.init = padlock_sha256_init_nano,
.update = padlock_sha256_update_nano,
.final = padlock_sha256_final_nano,
.export = padlock_sha_export_nano,
.import = padlock_sha_import_nano,
.descsize = sizeof(struct sha256_state),
.statesize = sizeof(struct sha256_state),
.base = {
.cra_name = "sha256",
.cra_driver_name = "sha256-padlock-nano",
.cra_priority = PADLOCK_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_TYPE_SHASH,
.cra_blocksize = SHA256_BLOCK_SIZE,
.cra_module = THIS_MODULE,
}
};
static struct x86_cpu_id padlock_sha_ids[] = {
X86_FEATURE_MATCH(X86_FEATURE_PHE),
{}
};
MODULE_DEVICE_TABLE(x86cpu, padlock_sha_ids);
static int __init padlock_init(void)
{
int rc = -ENODEV;
struct cpuinfo_x86 *c = &cpu_data(0);
struct shash_alg *sha1;
struct shash_alg *sha256;
if (!x86_match_cpu(padlock_sha_ids) || !cpu_has_phe_enabled)
return -ENODEV;
/* Register the newly added algorithm module if on *
* VIA Nano processor, or else just do as before */
if (c->x86_model < 0x0f) {
sha1 = &sha1_alg;
sha256 = &sha256_alg;
} else {
sha1 = &sha1_alg_nano;
sha256 = &sha256_alg_nano;
}
rc = crypto_register_shash(sha1);
if (rc)
goto out;
rc = crypto_register_shash(sha256);
if (rc)
goto out_unreg1;
printk(KERN_NOTICE PFX "Using VIA PadLock ACE for SHA1/SHA256 algorithms.\n");
return 0;
out_unreg1:
crypto_unregister_shash(sha1);
out:
printk(KERN_ERR PFX "VIA PadLock SHA1/SHA256 initialization failed.\n");
return rc;
}
static void __exit padlock_fini(void)
{
struct cpuinfo_x86 *c = &cpu_data(0);
if (c->x86_model >= 0x0f) {
crypto_unregister_shash(&sha1_alg_nano);
crypto_unregister_shash(&sha256_alg_nano);
} else {
crypto_unregister_shash(&sha1_alg);
crypto_unregister_shash(&sha256_alg);
}
}
module_init(padlock_init);
module_exit(padlock_fini);
MODULE_DESCRIPTION("VIA PadLock SHA1/SHA256 algorithms support.");
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Michal Ludvig");
MODULE_ALIAS_CRYPTO("sha1-all");
MODULE_ALIAS_CRYPTO("sha256-all");
MODULE_ALIAS_CRYPTO("sha1-padlock");
MODULE_ALIAS_CRYPTO("sha256-padlock");

1861
drivers/crypto/picoxcell_crypto.c Normal file
View file

File diff suppressed because it is too large Load diff

Some files were not shown because too many files have changed in this diff Show more