kvj
/
linux-zen-server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
linux-zen-server/drivers/crypto/keembay/ocs-aes.c

1490 lines
42 KiB
C
Raw Permalink Blame History

// SPDX-License-Identifier: GPL-2.0-only
/*
* Intel Keem Bay OCS AES Crypto Driver.
*
* Copyright (C) 2018-2020 Intel Corporation
*/
#include <linux/dma-mapping.h>
#include <linux/interrupt.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/swab.h>
#include <asm/byteorder.h>
#include <asm/errno.h>
#include <crypto/aes.h>
#include <crypto/gcm.h>
#include "ocs-aes.h"
#define AES_COMMAND_OFFSET 0x0000
#define AES_KEY_0_OFFSET 0x0004
#define AES_KEY_1_OFFSET 0x0008
#define AES_KEY_2_OFFSET 0x000C
#define AES_KEY_3_OFFSET 0x0010
#define AES_KEY_4_OFFSET 0x0014
#define AES_KEY_5_OFFSET 0x0018
#define AES_KEY_6_OFFSET 0x001C
#define AES_KEY_7_OFFSET 0x0020
#define AES_IV_0_OFFSET 0x0024
#define AES_IV_1_OFFSET 0x0028
#define AES_IV_2_OFFSET 0x002C
#define AES_IV_3_OFFSET 0x0030
#define AES_ACTIVE_OFFSET 0x0034
#define AES_STATUS_OFFSET 0x0038
#define AES_KEY_SIZE_OFFSET 0x0044
#define AES_IER_OFFSET 0x0048
#define AES_ISR_OFFSET 0x005C
#define AES_MULTIPURPOSE1_0_OFFSET 0x0200
#define AES_MULTIPURPOSE1_1_OFFSET 0x0204
#define AES_MULTIPURPOSE1_2_OFFSET 0x0208
#define AES_MULTIPURPOSE1_3_OFFSET 0x020C
#define AES_MULTIPURPOSE2_0_OFFSET 0x0220
#define AES_MULTIPURPOSE2_1_OFFSET 0x0224
#define AES_MULTIPURPOSE2_2_OFFSET 0x0228
#define AES_MULTIPURPOSE2_3_OFFSET 0x022C
#define AES_BYTE_ORDER_CFG_OFFSET 0x02C0
#define AES_TLEN_OFFSET 0x0300
#define AES_T_MAC_0_OFFSET 0x0304
#define AES_T_MAC_1_OFFSET 0x0308
#define AES_T_MAC_2_OFFSET 0x030C
#define AES_T_MAC_3_OFFSET 0x0310
#define AES_PLEN_OFFSET 0x0314
#define AES_A_DMA_SRC_ADDR_OFFSET 0x0400
#define AES_A_DMA_DST_ADDR_OFFSET 0x0404
#define AES_A_DMA_SRC_SIZE_OFFSET 0x0408
#define AES_A_DMA_DST_SIZE_OFFSET 0x040C
#define AES_A_DMA_DMA_MODE_OFFSET 0x0410
#define AES_A_DMA_NEXT_SRC_DESCR_OFFSET 0x0418
#define AES_A_DMA_NEXT_DST_DESCR_OFFSET 0x041C
#define AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET 0x0420
#define AES_A_DMA_LOG_OFFSET 0x0424
#define AES_A_DMA_STATUS_OFFSET 0x0428
#define AES_A_DMA_PERF_CNTR_OFFSET 0x042C
#define AES_A_DMA_MSI_ISR_OFFSET 0x0480
#define AES_A_DMA_MSI_IER_OFFSET 0x0484
#define AES_A_DMA_MSI_MASK_OFFSET 0x0488
#define AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET 0x0600
#define AES_A_DMA_OUTBUFFER_READ_FIFO_OFFSET 0x0700
/*
* AES_A_DMA_DMA_MODE register.
* Default: 0x00000000.
* bit[31] ACTIVE
* This bit activates the DMA. When the DMA finishes, it resets
* this bit to zero.
* bit[30:26] Unused by this driver.
* bit[25] SRC_LINK_LIST_EN
* Source link list enable bit. When the linked list is terminated
* this bit is reset by the DMA.
* bit[24] DST_LINK_LIST_EN
* Destination link list enable bit. When the linked list is
* terminated this bit is reset by the DMA.
* bit[23:0] Unused by this driver.
*/
#define AES_A_DMA_DMA_MODE_ACTIVE BIT(31)
#define AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN BIT(25)
#define AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN BIT(24)
/*
* AES_ACTIVE register
* default 0x00000000
* bit[31:10] Reserved
* bit[9] LAST_ADATA
* bit[8] LAST_GCX
* bit[7:2] Reserved
* bit[1] TERMINATION
* bit[0] TRIGGER
*/
#define AES_ACTIVE_LAST_ADATA BIT(9)
#define AES_ACTIVE_LAST_CCM_GCM BIT(8)
#define AES_ACTIVE_TERMINATION BIT(1)
#define AES_ACTIVE_TRIGGER BIT(0)
#define AES_DISABLE_INT 0x00000000
#define AES_DMA_CPD_ERR_INT BIT(8)
#define AES_DMA_OUTBUF_RD_ERR_INT BIT(7)
#define AES_DMA_OUTBUF_WR_ERR_INT BIT(6)
#define AES_DMA_INBUF_RD_ERR_INT BIT(5)
#define AES_DMA_INBUF_WR_ERR_INT BIT(4)
#define AES_DMA_BAD_COMP_INT BIT(3)
#define AES_DMA_SAI_INT BIT(2)
#define AES_DMA_SRC_DONE_INT BIT(0)
#define AES_COMPLETE_INT BIT(1)
#define AES_DMA_MSI_MASK_CLEAR BIT(0)
#define AES_128_BIT_KEY 0x00000000
#define AES_256_BIT_KEY BIT(0)
#define AES_DEACTIVATE_PERF_CNTR 0x00000000
#define AES_ACTIVATE_PERF_CNTR BIT(0)
#define AES_MAX_TAG_SIZE_U32 4
#define OCS_LL_DMA_FLAG_TERMINATE BIT(31)
/*
* There is an inconsistency in the documentation. This is documented as a
* 11-bit value, but it is actually 10-bits.
*/
#define AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK 0x3FF
/*
* During CCM decrypt, the OCS block needs to finish processing the ciphertext
* before the tag is written. For 128-bit mode this required delay is 28 OCS
* clock cycles. For 256-bit mode it is 36 OCS clock cycles.
*/
#define CCM_DECRYPT_DELAY_TAG_CLK_COUNT 36UL
/*
* During CCM decrypt there must be a delay of at least 42 OCS clock cycles
* between setting the TRIGGER bit in AES_ACTIVE and setting the LAST_CCM_GCM
* bit in the same register (as stated in the OCS databook)
*/
#define CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT 42UL
/* See RFC3610 section 2.2 */
#define L_PRIME_MIN (1)
#define L_PRIME_MAX (7)
/*
* CCM IV format from RFC 3610 section 2.3
*
* Octet Number Contents
* ------------ ---------
* 0 Flags
* 1 ... 15-L Nonce N
* 16-L ... 15 Counter i
*
* Flags = L' = L - 1
*/
#define L_PRIME_IDX 0
#define COUNTER_START(lprime) (16 - ((lprime) + 1))
#define COUNTER_LEN(lprime) ((lprime) + 1)
enum aes_counter_mode {
AES_CTR_M_NO_INC = 0,
AES_CTR_M_32_INC = 1,
AES_CTR_M_64_INC = 2,
AES_CTR_M_128_INC = 3,
};
/**
* struct ocs_dma_linked_list - OCS DMA linked list entry.
* @src_addr: Source address of the data.
* @src_len: Length of data to be fetched.
* @next: Next dma_list to fetch.
* @ll_flags: Flags (Freeze @ terminate) for the DMA engine.
*/
struct ocs_dma_linked_list {
u32 src_addr;
u32 src_len;
u32 next;
u32 ll_flags;
} __packed;
/*
* Set endianness of inputs and outputs
* AES_BYTE_ORDER_CFG
* default 0x00000000
* bit [10] - KEY_HI_LO_SWAP
* bit [9] - KEY_HI_SWAP_DWORDS_IN_OCTWORD
* bit [8] - KEY_HI_SWAP_BYTES_IN_DWORD
* bit [7] - KEY_LO_SWAP_DWORDS_IN_OCTWORD
* bit [6] - KEY_LO_SWAP_BYTES_IN_DWORD
* bit [5] - IV_SWAP_DWORDS_IN_OCTWORD
* bit [4] - IV_SWAP_BYTES_IN_DWORD
* bit [3] - DOUT_SWAP_DWORDS_IN_OCTWORD
* bit [2] - DOUT_SWAP_BYTES_IN_DWORD
* bit [1] - DOUT_SWAP_DWORDS_IN_OCTWORD
* bit [0] - DOUT_SWAP_BYTES_IN_DWORD
*/
static inline void aes_a_set_endianness(const struct ocs_aes_dev *aes_dev)
{
iowrite32(0x7FF, aes_dev->base_reg + AES_BYTE_ORDER_CFG_OFFSET);
}
/* Trigger AES process start. */
static inline void aes_a_op_trigger(const struct ocs_aes_dev *aes_dev)
{
iowrite32(AES_ACTIVE_TRIGGER, aes_dev->base_reg + AES_ACTIVE_OFFSET);
}
/* Indicate last bulk of data. */
static inline void aes_a_op_termination(const struct ocs_aes_dev *aes_dev)
{
iowrite32(AES_ACTIVE_TERMINATION,
aes_dev->base_reg + AES_ACTIVE_OFFSET);
}
/*
* Set LAST_CCM_GCM in AES_ACTIVE register and clear all other bits.
*
* Called when DMA is programmed to fetch the last batch of data.
* - For AES-CCM it is called for the last batch of Payload data and Ciphertext
* data.
* - For AES-GCM, it is called for the last batch of Plaintext data and
* Ciphertext data.
*/
static inline void aes_a_set_last_gcx(const struct ocs_aes_dev *aes_dev)
{
iowrite32(AES_ACTIVE_LAST_CCM_GCM,
aes_dev->base_reg + AES_ACTIVE_OFFSET);
}
/* Wait for LAST_CCM_GCM bit to be unset. */
static inline void aes_a_wait_last_gcx(const struct ocs_aes_dev *aes_dev)
{
u32 aes_active_reg;
do {
aes_active_reg = ioread32(aes_dev->base_reg +
AES_ACTIVE_OFFSET);
} while (aes_active_reg & AES_ACTIVE_LAST_CCM_GCM);
}
/* Wait for 10 bits of input occupancy. */
static void aes_a_dma_wait_input_buffer_occupancy(const struct ocs_aes_dev *aes_dev)
{
u32 reg;
do {
reg = ioread32(aes_dev->base_reg + AES_A_DMA_STATUS_OFFSET);
} while (reg & AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK);
}
/*
* Set LAST_CCM_GCM and LAST_ADATA bits in AES_ACTIVE register (and clear all
* other bits).
*
* Called when DMA is programmed to fetch the last batch of Associated Data
* (CCM case) or Additional Authenticated Data (GCM case).
*/
static inline void aes_a_set_last_gcx_and_adata(const struct ocs_aes_dev *aes_dev)
{
iowrite32(AES_ACTIVE_LAST_ADATA | AES_ACTIVE_LAST_CCM_GCM,
aes_dev->base_reg + AES_ACTIVE_OFFSET);
}
/* Set DMA src and dst transfer size to 0 */
static inline void aes_a_dma_set_xfer_size_zero(const struct ocs_aes_dev *aes_dev)
{
iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET);
iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET);
}
/* Activate DMA for zero-byte transfer case. */
static inline void aes_a_dma_active(const struct ocs_aes_dev *aes_dev)
{
iowrite32(AES_A_DMA_DMA_MODE_ACTIVE,
aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
}
/* Activate DMA and enable src linked list */
static inline void aes_a_dma_active_src_ll_en(const struct ocs_aes_dev *aes_dev)
{
iowrite32(AES_A_DMA_DMA_MODE_ACTIVE |
AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN,
aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
}
/* Activate DMA and enable dst linked list */
static inline void aes_a_dma_active_dst_ll_en(const struct ocs_aes_dev *aes_dev)
{
iowrite32(AES_A_DMA_DMA_MODE_ACTIVE |
AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN,
aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
}
/* Activate DMA and enable src and dst linked lists */
static inline void aes_a_dma_active_src_dst_ll_en(const struct ocs_aes_dev *aes_dev)
{
iowrite32(AES_A_DMA_DMA_MODE_ACTIVE |
AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN |
AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN,
aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
}
/* Reset PERF_CNTR to 0 and activate it */
static inline void aes_a_dma_reset_and_activate_perf_cntr(const struct ocs_aes_dev *aes_dev)
{
iowrite32(0x00000000, aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET);
iowrite32(AES_ACTIVATE_PERF_CNTR,
aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET);
}
/* Wait until PERF_CNTR is > delay, then deactivate it */
static inline void aes_a_dma_wait_and_deactivate_perf_cntr(const struct ocs_aes_dev *aes_dev,
int delay)
{
while (ioread32(aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET) < delay)
;
iowrite32(AES_DEACTIVATE_PERF_CNTR,
aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET);
}
/* Disable AES and DMA IRQ. */
static void aes_irq_disable(struct ocs_aes_dev *aes_dev)
{
u32 isr_val = 0;
/* Disable interrupts */
iowrite32(AES_DISABLE_INT,
aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET);
iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET);
/* Clear any pending interrupt */
isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET);
if (isr_val)
iowrite32(isr_val,
aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET);
isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET);
if (isr_val)
iowrite32(isr_val,
aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET);
isr_val = ioread32(aes_dev->base_reg + AES_ISR_OFFSET);
if (isr_val)
iowrite32(isr_val, aes_dev->base_reg + AES_ISR_OFFSET);
}
/* Enable AES or DMA IRQ. IRQ is disabled once fired. */
static void aes_irq_enable(struct ocs_aes_dev *aes_dev, u8 irq)
{
if (irq == AES_COMPLETE_INT) {
/* Ensure DMA error interrupts are enabled */
iowrite32(AES_DMA_CPD_ERR_INT |
AES_DMA_OUTBUF_RD_ERR_INT |
AES_DMA_OUTBUF_WR_ERR_INT |
AES_DMA_INBUF_RD_ERR_INT |
AES_DMA_INBUF_WR_ERR_INT |
AES_DMA_BAD_COMP_INT |
AES_DMA_SAI_INT,
aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET);
/*
* AES_IER
* default 0x00000000
* bits [31:3] - reserved
* bit [2] - EN_SKS_ERR
* bit [1] - EN_AES_COMPLETE
* bit [0] - reserved
*/
iowrite32(AES_COMPLETE_INT, aes_dev->base_reg + AES_IER_OFFSET);
return;
}
if (irq == AES_DMA_SRC_DONE_INT) {
/* Ensure AES interrupts are disabled */
iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET);
/*
* DMA_MSI_IER
* default 0x00000000
* bits [31:9] - reserved
* bit [8] - CPD_ERR_INT_EN
* bit [7] - OUTBUF_RD_ERR_INT_EN
* bit [6] - OUTBUF_WR_ERR_INT_EN
* bit [5] - INBUF_RD_ERR_INT_EN
* bit [4] - INBUF_WR_ERR_INT_EN
* bit [3] - BAD_COMP_INT_EN
* bit [2] - SAI_INT_EN
* bit [1] - DST_DONE_INT_EN
* bit [0] - SRC_DONE_INT_EN
*/
iowrite32(AES_DMA_CPD_ERR_INT |
AES_DMA_OUTBUF_RD_ERR_INT |
AES_DMA_OUTBUF_WR_ERR_INT |
AES_DMA_INBUF_RD_ERR_INT |
AES_DMA_INBUF_WR_ERR_INT |
AES_DMA_BAD_COMP_INT |
AES_DMA_SAI_INT |
AES_DMA_SRC_DONE_INT,
aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET);
}
}
/* Enable and wait for IRQ (either from OCS AES engine or DMA) */
static int ocs_aes_irq_enable_and_wait(struct ocs_aes_dev *aes_dev, u8 irq)
{
int rc;
reinit_completion(&aes_dev->irq_completion);
aes_irq_enable(aes_dev, irq);
rc = wait_for_completion_interruptible(&aes_dev->irq_completion);
if (rc)
return rc;
return aes_dev->dma_err_mask ? -EIO : 0;
}
/* Configure DMA to OCS, linked list mode */
static inline void dma_to_ocs_aes_ll(struct ocs_aes_dev *aes_dev,
dma_addr_t dma_list)
{
iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET);
iowrite32(dma_list,
aes_dev->base_reg + AES_A_DMA_NEXT_SRC_DESCR_OFFSET);
}
/* Configure DMA from OCS, linked list mode */
static inline void dma_from_ocs_aes_ll(struct ocs_aes_dev *aes_dev,
dma_addr_t dma_list)
{
iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET);
iowrite32(dma_list,
aes_dev->base_reg + AES_A_DMA_NEXT_DST_DESCR_OFFSET);
}
irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id)
{
struct ocs_aes_dev *aes_dev = dev_id;
u32 aes_dma_isr;
/* Read DMA ISR status. */
aes_dma_isr = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET);
/* Disable and clear interrupts. */
aes_irq_disable(aes_dev);
/* Save DMA error status. */
aes_dev->dma_err_mask = aes_dma_isr &
(AES_DMA_CPD_ERR_INT |
AES_DMA_OUTBUF_RD_ERR_INT |
AES_DMA_OUTBUF_WR_ERR_INT |
AES_DMA_INBUF_RD_ERR_INT |
AES_DMA_INBUF_WR_ERR_INT |
AES_DMA_BAD_COMP_INT |
AES_DMA_SAI_INT);
/* Signal IRQ completion. */
complete(&aes_dev->irq_completion);
return IRQ_HANDLED;
}
/**
* ocs_aes_set_key() - Write key into OCS AES hardware.
* @aes_dev: The OCS AES device to write the key to.
* @key_size: The size of the key (in bytes).
* @key: The key to write.
* @cipher: The cipher the key is for.
*
* For AES @key_size must be either 16 or 32. For SM4 @key_size must be 16.
*
* Return: 0 on success, negative error code otherwise.
*/
int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, u32 key_size, const u8 *key,
enum ocs_cipher cipher)
{
const u32 *key_u32;
u32 val;
int i;
/* OCS AES supports 128-bit and 256-bit keys only. */
if (cipher == OCS_AES && !(key_size == 32 || key_size == 16)) {
dev_err(aes_dev->dev,
"%d-bit keys not supported by AES cipher\n",
key_size * 8);
return -EINVAL;
}
/* OCS SM4 supports 128-bit keys only. */
if (cipher == OCS_SM4 && key_size != 16) {
dev_err(aes_dev->dev,
"%d-bit keys not supported for SM4 cipher\n",
key_size * 8);
return -EINVAL;
}
if (!key)
return -EINVAL;
key_u32 = (const u32 *)key;
/* Write key to AES_KEY[0-7] registers */
for (i = 0; i < (key_size / sizeof(u32)); i++) {
iowrite32(key_u32[i],
aes_dev->base_reg + AES_KEY_0_OFFSET +
(i * sizeof(u32)));
}
/*
* Write key size
* bits [31:1] - reserved
* bit [0] - AES_KEY_SIZE
* 0 - 128 bit key
* 1 - 256 bit key
*/
val = (key_size == 16) ? AES_128_BIT_KEY : AES_256_BIT_KEY;
iowrite32(val, aes_dev->base_reg + AES_KEY_SIZE_OFFSET);
return 0;
}
/* Write AES_COMMAND */
static inline void set_ocs_aes_command(struct ocs_aes_dev *aes_dev,
enum ocs_cipher cipher,
enum ocs_mode mode,
enum ocs_instruction instruction)
{
u32 val;
/* AES_COMMAND
* default 0x000000CC
* bit [14] - CIPHER_SELECT
* 0 - AES
* 1 - SM4
* bits [11:8] - OCS_AES_MODE
* 0000 - ECB
* 0001 - CBC
* 0010 - CTR
* 0110 - CCM
* 0111 - GCM
* 1001 - CTS
* bits [7:6] - AES_INSTRUCTION
* 00 - ENCRYPT
* 01 - DECRYPT
* 10 - EXPAND
* 11 - BYPASS
* bits [3:2] - CTR_M_BITS
* 00 - No increment
* 01 - Least significant 32 bits are incremented
* 10 - Least significant 64 bits are incremented
* 11 - Full 128 bits are incremented
*/
val = (cipher << 14) | (mode << 8) | (instruction << 6) |
(AES_CTR_M_128_INC << 2);
iowrite32(val, aes_dev->base_reg + AES_COMMAND_OFFSET);
}
static void ocs_aes_init(struct ocs_aes_dev *aes_dev,
enum ocs_mode mode,
enum ocs_cipher cipher,
enum ocs_instruction instruction)
{
/* Ensure interrupts are disabled and pending interrupts cleared. */
aes_irq_disable(aes_dev);
/* Set endianness recommended by data-sheet. */
aes_a_set_endianness(aes_dev);
/* Set AES_COMMAND register. */
set_ocs_aes_command(aes_dev, cipher, mode, instruction);
}
/*
* Write the byte length of the last AES/SM4 block of Payload data (without
* zero padding and without the length of the MAC) in register AES_PLEN.
*/
static inline void ocs_aes_write_last_data_blk_len(struct ocs_aes_dev *aes_dev,
u32 size)
{
u32 val;
if (size == 0) {
val = 0;
goto exit;
}
val = size % AES_BLOCK_SIZE;
if (val == 0)
val = AES_BLOCK_SIZE;
exit:
iowrite32(val, aes_dev->base_reg + AES_PLEN_OFFSET);
}
/*
* Validate inputs according to mode.
* If OK return 0; else return -EINVAL.
*/
static int ocs_aes_validate_inputs(dma_addr_t src_dma_list, u32 src_size,
const u8 *iv, u32 iv_size,
dma_addr_t aad_dma_list, u32 aad_size,
const u8 *tag, u32 tag_size,
enum ocs_cipher cipher, enum ocs_mode mode,
enum ocs_instruction instruction,
dma_addr_t dst_dma_list)
{
/* Ensure cipher, mode and instruction are valid. */
if (!(cipher == OCS_AES || cipher == OCS_SM4))
return -EINVAL;
if (mode != OCS_MODE_ECB && mode != OCS_MODE_CBC &&
mode != OCS_MODE_CTR && mode != OCS_MODE_CCM &&
mode != OCS_MODE_GCM && mode != OCS_MODE_CTS)
return -EINVAL;
if (instruction != OCS_ENCRYPT && instruction != OCS_DECRYPT &&
instruction != OCS_EXPAND && instruction != OCS_BYPASS)
return -EINVAL;
/*
* When instruction is OCS_BYPASS, OCS simply copies data from source
* to destination using DMA.
*
* AES mode is irrelevant, but both source and destination DMA
* linked-list must be defined.
*/
if (instruction == OCS_BYPASS) {
if (src_dma_list == DMA_MAPPING_ERROR ||
dst_dma_list == DMA_MAPPING_ERROR)
return -EINVAL;
return 0;
}
/*
* For performance reasons switch based on mode to limit unnecessary
* conditionals for each mode
*/
switch (mode) {
case OCS_MODE_ECB:
/* Ensure input length is multiple of block size */
if (src_size % AES_BLOCK_SIZE != 0)
return -EINVAL;
/* Ensure source and destination linked lists are created */
if (src_dma_list == DMA_MAPPING_ERROR ||
dst_dma_list == DMA_MAPPING_ERROR)
return -EINVAL;
return 0;
case OCS_MODE_CBC:
/* Ensure input length is multiple of block size */
if (src_size % AES_BLOCK_SIZE != 0)
return -EINVAL;
/* Ensure source and destination linked lists are created */
if (src_dma_list == DMA_MAPPING_ERROR ||
dst_dma_list == DMA_MAPPING_ERROR)
return -EINVAL;
/* Ensure IV is present and block size in length */
if (!iv || iv_size != AES_BLOCK_SIZE)
return -EINVAL;
return 0;
case OCS_MODE_CTR:
/* Ensure input length of 1 byte or greater */
if (src_size == 0)
return -EINVAL;
/* Ensure source and destination linked lists are created */
if (src_dma_list == DMA_MAPPING_ERROR ||
dst_dma_list == DMA_MAPPING_ERROR)
return -EINVAL;
/* Ensure IV is present and block size in length */
if (!iv || iv_size != AES_BLOCK_SIZE)
return -EINVAL;
return 0;
case OCS_MODE_CTS:
/* Ensure input length >= block size */
if (src_size < AES_BLOCK_SIZE)
return -EINVAL;
/* Ensure source and destination linked lists are created */
if (src_dma_list == DMA_MAPPING_ERROR ||
dst_dma_list == DMA_MAPPING_ERROR)
return -EINVAL;
/* Ensure IV is present and block size in length */
if (!iv || iv_size != AES_BLOCK_SIZE)
return -EINVAL;
return 0;
case OCS_MODE_GCM:
/* Ensure IV is present and GCM_AES_IV_SIZE in length */
if (!iv || iv_size != GCM_AES_IV_SIZE)
return -EINVAL;
/*
* If input data present ensure source and destination linked
* lists are created
*/
if (src_size && (src_dma_list == DMA_MAPPING_ERROR ||
dst_dma_list == DMA_MAPPING_ERROR))
return -EINVAL;
/* If aad present ensure aad linked list is created */
if (aad_size && aad_dma_list == DMA_MAPPING_ERROR)
return -EINVAL;
/* Ensure tag destination is set */
if (!tag)
return -EINVAL;
/* Just ensure that tag_size doesn't cause overflows. */
if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32)))
return -EINVAL;
return 0;
case OCS_MODE_CCM:
/* Ensure IV is present and block size in length */
if (!iv || iv_size != AES_BLOCK_SIZE)
return -EINVAL;
/* 2 <= L <= 8, so 1 <= L' <= 7 */
if (iv[L_PRIME_IDX] < L_PRIME_MIN ||
iv[L_PRIME_IDX] > L_PRIME_MAX)
return -EINVAL;
/* If aad present ensure aad linked list is created */
if (aad_size && aad_dma_list == DMA_MAPPING_ERROR)
return -EINVAL;
/* Just ensure that tag_size doesn't cause overflows. */
if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32)))
return -EINVAL;
if (instruction == OCS_DECRYPT) {
/*
* If input data present ensure source and destination
* linked lists are created
*/
if (src_size && (src_dma_list == DMA_MAPPING_ERROR ||
dst_dma_list == DMA_MAPPING_ERROR))
return -EINVAL;
/* Ensure input tag is present */
if (!tag)
return -EINVAL;
return 0;
}
/* Instruction == OCS_ENCRYPT */
/*
* Destination linked list always required (for tag even if no
* input data)
*/
if (dst_dma_list == DMA_MAPPING_ERROR)
return -EINVAL;
/* If input data present ensure src linked list is created */
if (src_size && src_dma_list == DMA_MAPPING_ERROR)
return -EINVAL;
return 0;
default:
return -EINVAL;
}
}
/**
* ocs_aes_op() - Perform AES/SM4 operation.
* @aes_dev: The OCS AES device to use.
* @mode: The mode to use (ECB, CBC, CTR, or CTS).
* @cipher: The cipher to use (AES or SM4).
* @instruction: The instruction to perform (encrypt or decrypt).
* @dst_dma_list: The OCS DMA list mapping output memory.
* @src_dma_list: The OCS DMA list mapping input payload data.
* @src_size: The amount of data mapped by @src_dma_list.
* @iv: The IV vector.
* @iv_size: The size (in bytes) of @iv.
*
* Return: 0 on success, negative error code otherwise.
*/
int ocs_aes_op(struct ocs_aes_dev *aes_dev,
enum ocs_mode mode,
enum ocs_cipher cipher,
enum ocs_instruction instruction,
dma_addr_t dst_dma_list,
dma_addr_t src_dma_list,
u32 src_size,
u8 *iv,
u32 iv_size)
{
u32 *iv32;
int rc;
rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, iv_size, 0, 0,
NULL, 0, cipher, mode, instruction,
dst_dma_list);
if (rc)
return rc;
/*
* ocs_aes_validate_inputs() is a generic check, now ensure mode is not
* GCM or CCM.
*/
if (mode == OCS_MODE_GCM || mode == OCS_MODE_CCM)
return -EINVAL;
/* Cast IV to u32 array. */
iv32 = (u32 *)iv;
ocs_aes_init(aes_dev, mode, cipher, instruction);
if (mode == OCS_MODE_CTS) {
/* Write the byte length of the last data block to engine. */
ocs_aes_write_last_data_blk_len(aes_dev, src_size);
}
/* ECB is the only mode that doesn't use IV. */
if (mode != OCS_MODE_ECB) {
iowrite32(iv32[0], aes_dev->base_reg + AES_IV_0_OFFSET);
iowrite32(iv32[1], aes_dev->base_reg + AES_IV_1_OFFSET);
iowrite32(iv32[2], aes_dev->base_reg + AES_IV_2_OFFSET);
iowrite32(iv32[3], aes_dev->base_reg + AES_IV_3_OFFSET);
}
/* Set AES_ACTIVE.TRIGGER to start the operation. */
aes_a_op_trigger(aes_dev);
/* Configure and activate input / output DMA. */
dma_to_ocs_aes_ll(aes_dev, src_dma_list);
dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
aes_a_dma_active_src_dst_ll_en(aes_dev);
if (mode == OCS_MODE_CTS) {
/*
* For CTS mode, instruct engine to activate ciphertext
* stealing if last block of data is incomplete.
*/
aes_a_set_last_gcx(aes_dev);
} else {
/* For all other modes, just write the 'termination' bit. */
aes_a_op_termination(aes_dev);
}
/* Wait for engine to complete processing. */
rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
if (rc)
return rc;
if (mode == OCS_MODE_CTR) {
/* Read back IV for streaming mode */
iv32[0] = ioread32(aes_dev->base_reg + AES_IV_0_OFFSET);
iv32[1] = ioread32(aes_dev->base_reg + AES_IV_1_OFFSET);
iv32[2] = ioread32(aes_dev->base_reg + AES_IV_2_OFFSET);
iv32[3] = ioread32(aes_dev->base_reg + AES_IV_3_OFFSET);
}
return 0;
}
/* Compute and write J0 to engine registers. */
static void ocs_aes_gcm_write_j0(const struct ocs_aes_dev *aes_dev,
const u8 *iv)
{
const u32 *j0 = (u32 *)iv;
/*
* IV must be 12 bytes; Other sizes not supported as Linux crypto API
* does only expects/allows 12 byte IV for GCM
*/
iowrite32(0x00000001, aes_dev->base_reg + AES_IV_0_OFFSET);
iowrite32(__swab32(j0[2]), aes_dev->base_reg + AES_IV_1_OFFSET);
iowrite32(__swab32(j0[1]), aes_dev->base_reg + AES_IV_2_OFFSET);
iowrite32(__swab32(j0[0]), aes_dev->base_reg + AES_IV_3_OFFSET);
}
/* Read GCM tag from engine registers. */
static inline void ocs_aes_gcm_read_tag(struct ocs_aes_dev *aes_dev,
u8 *tag, u32 tag_size)
{
u32 tag_u32[AES_MAX_TAG_SIZE_U32];
/*
* The Authentication Tag T is stored in Little Endian order in the
* registers with the most significant bytes stored from AES_T_MAC[3]
* downward.
*/
tag_u32[0] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_3_OFFSET));
tag_u32[1] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_2_OFFSET));
tag_u32[2] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_1_OFFSET));
tag_u32[3] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_0_OFFSET));
memcpy(tag, tag_u32, tag_size);
}
/**
* ocs_aes_gcm_op() - Perform GCM operation.
* @aes_dev: The OCS AES device to use.
* @cipher: The Cipher to use (AES or SM4).
* @instruction: The instruction to perform (encrypt or decrypt).
* @dst_dma_list: The OCS DMA list mapping output memory.
* @src_dma_list: The OCS DMA list mapping input payload data.
* @src_size: The amount of data mapped by @src_dma_list.
* @iv: The input IV vector.
* @aad_dma_list: The OCS DMA list mapping input AAD data.
* @aad_size: The amount of data mapped by @aad_dma_list.
* @out_tag: Where to store computed tag.
* @tag_size: The size (in bytes) of @out_tag.
*
* Return: 0 on success, negative error code otherwise.
*/
int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev,
enum ocs_cipher cipher,
enum ocs_instruction instruction,
dma_addr_t dst_dma_list,
dma_addr_t src_dma_list,
u32 src_size,
const u8 *iv,
dma_addr_t aad_dma_list,
u32 aad_size,
u8 *out_tag,
u32 tag_size)
{
u64 bit_len;
u32 val;
int rc;
rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv,
GCM_AES_IV_SIZE, aad_dma_list,
aad_size, out_tag, tag_size, cipher,
OCS_MODE_GCM, instruction,
dst_dma_list);
if (rc)
return rc;
ocs_aes_init(aes_dev, OCS_MODE_GCM, cipher, instruction);
/* Compute and write J0 to OCS HW. */
ocs_aes_gcm_write_j0(aes_dev, iv);
/* Write out_tag byte length */
iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET);
/* Write the byte length of the last plaintext / ciphertext block. */
ocs_aes_write_last_data_blk_len(aes_dev, src_size);
/* Write ciphertext bit length */
bit_len = (u64)src_size * 8;
val = bit_len & 0xFFFFFFFF;
iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_0_OFFSET);
val = bit_len >> 32;
iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_1_OFFSET);
/* Write aad bit length */
bit_len = (u64)aad_size * 8;
val = bit_len & 0xFFFFFFFF;
iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_2_OFFSET);
val = bit_len >> 32;
iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_3_OFFSET);
/* Set AES_ACTIVE.TRIGGER to start the operation. */
aes_a_op_trigger(aes_dev);
/* Process AAD. */
if (aad_size) {
/* If aad present, configure DMA to feed it to the engine. */
dma_to_ocs_aes_ll(aes_dev, aad_dma_list);
aes_a_dma_active_src_ll_en(aes_dev);
/* Instructs engine to pad last block of aad, if needed. */
aes_a_set_last_gcx_and_adata(aes_dev);
/* Wait for DMA transfer to complete. */
rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT);
if (rc)
return rc;
} else {
aes_a_set_last_gcx_and_adata(aes_dev);
}
/* Wait until adata (if present) has been processed. */
aes_a_wait_last_gcx(aes_dev);
aes_a_dma_wait_input_buffer_occupancy(aes_dev);
/* Now process payload. */
if (src_size) {
/* Configure and activate DMA for both input and output data. */
dma_to_ocs_aes_ll(aes_dev, src_dma_list);
dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
aes_a_dma_active_src_dst_ll_en(aes_dev);
} else {
aes_a_dma_set_xfer_size_zero(aes_dev);
aes_a_dma_active(aes_dev);
}
/* Instruct AES/SMA4 engine payload processing is over. */
aes_a_set_last_gcx(aes_dev);
/* Wait for OCS AES engine to complete processing. */
rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
if (rc)
return rc;
ocs_aes_gcm_read_tag(aes_dev, out_tag, tag_size);
return 0;
}
/* Write encrypted tag to AES/SM4 engine. */
static void ocs_aes_ccm_write_encrypted_tag(struct ocs_aes_dev *aes_dev,
const u8 *in_tag, u32 tag_size)
{
int i;
/* Ensure DMA input buffer is empty */
aes_a_dma_wait_input_buffer_occupancy(aes_dev);
/*
* During CCM decrypt, the OCS block needs to finish processing the
* ciphertext before the tag is written. So delay needed after DMA has
* completed writing the ciphertext
*/
aes_a_dma_reset_and_activate_perf_cntr(aes_dev);
aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev,
CCM_DECRYPT_DELAY_TAG_CLK_COUNT);
/* Write encrypted tag to AES/SM4 engine. */
for (i = 0; i < tag_size; i++) {
iowrite8(in_tag[i], aes_dev->base_reg +
AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET);
}
}
/*
* Write B0 CCM block to OCS AES HW.
*
* Note: B0 format is documented in NIST Special Publication 800-38C
* https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf
* (see Section A.2.1)
*/
static int ocs_aes_ccm_write_b0(const struct ocs_aes_dev *aes_dev,
const u8 *iv, u32 adata_size, u32 tag_size,
u32 cryptlen)
{
u8 b0[16]; /* CCM B0 block is 16 bytes long. */
int i, q;
/* Initialize B0 to 0. */
memset(b0, 0, sizeof(b0));
/*
* B0[0] is the 'Flags Octet' and has the following structure:
* bit 7: Reserved
* bit 6: Adata flag
* bit 5-3: t value encoded as (t-2)/2
* bit 2-0: q value encoded as q - 1
*/
/* If there is AAD data, set the Adata flag. */
if (adata_size)
b0[0] |= BIT(6);
/*
* t denotes the octet length of T.
* t can only be an element of { 4, 6, 8, 10, 12, 14, 16} and is
* encoded as (t - 2) / 2
*/
b0[0] |= (((tag_size - 2) / 2) & 0x7) << 3;
/*
* q is the octet length of Q.
* q can only be an element of {2, 3, 4, 5, 6, 7, 8} and is encoded as
* q - 1 == iv[0] & 0x7;
*/
b0[0] |= iv[0] & 0x7;
/*
* Copy the Nonce N from IV to B0; N is located in iv[1]..iv[15 - q]
* and must be copied to b0[1]..b0[15-q].
* q == (iv[0] & 0x7) + 1
*/
q = (iv[0] & 0x7) + 1;
for (i = 1; i <= 15 - q; i++)
b0[i] = iv[i];
/*
* The rest of B0 must contain Q, i.e., the message length.
* Q is encoded in q octets, in big-endian order, so to write it, we
* start from the end of B0 and we move backward.
*/
i = sizeof(b0) - 1;
while (q) {
b0[i] = cryptlen & 0xff;
cryptlen >>= 8;
i--;
q--;
}
/*
* If cryptlen is not zero at this point, it means that its original
* value was too big.
*/
if (cryptlen)
return -EOVERFLOW;
/* Now write B0 to OCS AES input buffer. */
for (i = 0; i < sizeof(b0); i++)
iowrite8(b0[i], aes_dev->base_reg +
AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET);
return 0;
}
/*
* Write adata length to OCS AES HW.
*
* Note: adata len encoding is documented in NIST Special Publication 800-38C
* https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf
* (see Section A.2.2)
*/
static void ocs_aes_ccm_write_adata_len(const struct ocs_aes_dev *aes_dev,
u64 adata_len)
{
u8 enc_a[10]; /* Maximum encoded size: 10 octets. */
int i, len;
/*
* adata_len ('a') is encoded as follows:
* If 0 < a < 2^16 - 2^8 ==> 'a' encoded as [a]16, i.e., two octets
* (big endian).
* If 2^16 - 2^8 ≤ a < 2^32 ==> 'a' encoded as 0xff || 0xfe || [a]32,
* i.e., six octets (big endian).
* If 2^32 ≤ a < 2^64 ==> 'a' encoded as 0xff || 0xff || [a]64,
* i.e., ten octets (big endian).
*/
if (adata_len < 65280) {
len = 2;
*(__be16 *)enc_a = cpu_to_be16(adata_len);
} else if (adata_len <= 0xFFFFFFFF) {
len = 6;
*(__be16 *)enc_a = cpu_to_be16(0xfffe);
*(__be32 *)&enc_a[2] = cpu_to_be32(adata_len);
} else { /* adata_len >= 2^32 */
len = 10;
*(__be16 *)enc_a = cpu_to_be16(0xffff);
*(__be64 *)&enc_a[2] = cpu_to_be64(adata_len);
}
for (i = 0; i < len; i++)
iowrite8(enc_a[i],
aes_dev->base_reg +
AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET);
}
static int ocs_aes_ccm_do_adata(struct ocs_aes_dev *aes_dev,
dma_addr_t adata_dma_list, u32 adata_size)
{
int rc;
if (!adata_size) {
/* Since no aad the LAST_GCX bit can be set now */
aes_a_set_last_gcx_and_adata(aes_dev);
goto exit;
}
/* Adata case. */
/*
* Form the encoding of the Associated data length and write it
* to the AES/SM4 input buffer.
*/
ocs_aes_ccm_write_adata_len(aes_dev, adata_size);
/* Configure the AES/SM4 DMA to fetch the Associated Data */
dma_to_ocs_aes_ll(aes_dev, adata_dma_list);
/* Activate DMA to fetch Associated data. */
aes_a_dma_active_src_ll_en(aes_dev);
/* Set LAST_GCX and LAST_ADATA in AES ACTIVE register. */
aes_a_set_last_gcx_and_adata(aes_dev);
/* Wait for DMA transfer to complete. */
rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT);
if (rc)
return rc;
exit:
/* Wait until adata (if present) has been processed. */
aes_a_wait_last_gcx(aes_dev);
aes_a_dma_wait_input_buffer_occupancy(aes_dev);
return 0;
}
static int ocs_aes_ccm_encrypt_do_payload(struct ocs_aes_dev *aes_dev,
dma_addr_t dst_dma_list,
dma_addr_t src_dma_list,
u32 src_size)
{
if (src_size) {
/*
* Configure and activate DMA for both input and output
* data.
*/
dma_to_ocs_aes_ll(aes_dev, src_dma_list);
dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
aes_a_dma_active_src_dst_ll_en(aes_dev);
} else {
/* Configure and activate DMA for output data only. */
dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
aes_a_dma_active_dst_ll_en(aes_dev);
}
/*
* Set the LAST GCX bit in AES_ACTIVE Register to instruct
* AES/SM4 engine to pad the last block of data.
*/
aes_a_set_last_gcx(aes_dev);
/* We are done, wait for IRQ and return. */
return ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
}
static int ocs_aes_ccm_decrypt_do_payload(struct ocs_aes_dev *aes_dev,
dma_addr_t dst_dma_list,
dma_addr_t src_dma_list,
u32 src_size)
{
if (!src_size) {
/* Let engine process 0-length input. */
aes_a_dma_set_xfer_size_zero(aes_dev);
aes_a_dma_active(aes_dev);
aes_a_set_last_gcx(aes_dev);
return 0;
}
/*
* Configure and activate DMA for both input and output
* data.
*/
dma_to_ocs_aes_ll(aes_dev, src_dma_list);
dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
aes_a_dma_active_src_dst_ll_en(aes_dev);
/*
* Set the LAST GCX bit in AES_ACTIVE Register; this allows the
* AES/SM4 engine to differentiate between encrypted data and
* encrypted MAC.
*/
aes_a_set_last_gcx(aes_dev);
/*
* Enable DMA DONE interrupt; once DMA transfer is over,
* interrupt handler will process the MAC/tag.
*/
return ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT);
}
/*
* Compare Tag to Yr.
*
* Only used at the end of CCM decrypt. If tag == yr, message authentication
* has succeeded.
*/
static inline int ccm_compare_tag_to_yr(struct ocs_aes_dev *aes_dev,
u8 tag_size_bytes)
{
u32 tag[AES_MAX_TAG_SIZE_U32];
u32 yr[AES_MAX_TAG_SIZE_U32];
u8 i;
/* Read Tag and Yr from AES registers. */
for (i = 0; i < AES_MAX_TAG_SIZE_U32; i++) {
tag[i] = ioread32(aes_dev->base_reg +
AES_T_MAC_0_OFFSET + (i * sizeof(u32)));
yr[i] = ioread32(aes_dev->base_reg +
AES_MULTIPURPOSE2_0_OFFSET +
(i * sizeof(u32)));
}
return memcmp(tag, yr, tag_size_bytes) ? -EBADMSG : 0;
}
/**
* ocs_aes_ccm_op() - Perform CCM operation.
* @aes_dev: The OCS AES device to use.
* @cipher: The Cipher to use (AES or SM4).
* @instruction: The instruction to perform (encrypt or decrypt).
* @dst_dma_list: The OCS DMA list mapping output memory.
* @src_dma_list: The OCS DMA list mapping input payload data.
* @src_size: The amount of data mapped by @src_dma_list.
* @iv: The input IV vector.
* @adata_dma_list: The OCS DMA list mapping input A-data.
* @adata_size: The amount of data mapped by @adata_dma_list.
* @in_tag: Input tag.
* @tag_size: The size (in bytes) of @in_tag.
*
* Note: for encrypt the tag is appended to the ciphertext (in the memory
* mapped by @dst_dma_list).
*
* Return: 0 on success, negative error code otherwise.
*/
int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev,
enum ocs_cipher cipher,
enum ocs_instruction instruction,
dma_addr_t dst_dma_list,
dma_addr_t src_dma_list,
u32 src_size,
u8 *iv,
dma_addr_t adata_dma_list,
u32 adata_size,
u8 *in_tag,
u32 tag_size)
{
u32 *iv_32;
u8 lprime;
int rc;
rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv,
AES_BLOCK_SIZE, adata_dma_list, adata_size,
in_tag, tag_size, cipher, OCS_MODE_CCM,
instruction, dst_dma_list);
if (rc)
return rc;
ocs_aes_init(aes_dev, OCS_MODE_CCM, cipher, instruction);
/*
* Note: rfc 3610 and NIST 800-38C require counter of zero to encrypt
* auth tag so ensure this is the case
*/
lprime = iv[L_PRIME_IDX];
memset(&iv[COUNTER_START(lprime)], 0, COUNTER_LEN(lprime));
/*
* Nonce is already converted to ctr0 before being passed into this
* function as iv.
*/
iv_32 = (u32 *)iv;
iowrite32(__swab32(iv_32[0]),
aes_dev->base_reg + AES_MULTIPURPOSE1_3_OFFSET);
iowrite32(__swab32(iv_32[1]),
aes_dev->base_reg + AES_MULTIPURPOSE1_2_OFFSET);
iowrite32(__swab32(iv_32[2]),
aes_dev->base_reg + AES_MULTIPURPOSE1_1_OFFSET);
iowrite32(__swab32(iv_32[3]),
aes_dev->base_reg + AES_MULTIPURPOSE1_0_OFFSET);
/* Write MAC/tag length in register AES_TLEN */
iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET);
/*
* Write the byte length of the last AES/SM4 block of Payload data
* (without zero padding and without the length of the MAC) in register
* AES_PLEN.
*/
ocs_aes_write_last_data_blk_len(aes_dev, src_size);
/* Set AES_ACTIVE.TRIGGER to start the operation. */
aes_a_op_trigger(aes_dev);
aes_a_dma_reset_and_activate_perf_cntr(aes_dev);
/* Form block B0 and write it to the AES/SM4 input buffer. */
rc = ocs_aes_ccm_write_b0(aes_dev, iv, adata_size, tag_size, src_size);
if (rc)
return rc;
/*
* Ensure there has been at least CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT
* clock cycles since TRIGGER bit was set
*/
aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev,
CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT);
/* Process Adata. */
ocs_aes_ccm_do_adata(aes_dev, adata_dma_list, adata_size);
/* For Encrypt case we just process the payload and return. */
if (instruction == OCS_ENCRYPT) {
return ocs_aes_ccm_encrypt_do_payload(aes_dev, dst_dma_list,
src_dma_list, src_size);
}
/* For Decypt we need to process the payload and then the tag. */
rc = ocs_aes_ccm_decrypt_do_payload(aes_dev, dst_dma_list,
src_dma_list, src_size);
if (rc)
return rc;
/* Process MAC/tag directly: feed tag to engine and wait for IRQ. */
ocs_aes_ccm_write_encrypted_tag(aes_dev, in_tag, tag_size);
rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
if (rc)
return rc;
return ccm_compare_tag_to_yr(aes_dev, tag_size);
}
/**
* ocs_create_linked_list_from_sg() - Create OCS DMA linked list from SG list.
* @aes_dev: The OCS AES device the list will be created for.
* @sg: The SG list OCS DMA linked list will be created from. When
* passed to this function, @sg must have been already mapped
* with dma_map_sg().
* @sg_dma_count: The number of DMA-mapped entries in @sg. This must be the
* value returned by dma_map_sg() when @sg was mapped.
* @dll_desc: The OCS DMA dma_list to use to store information about the
* created linked list.
* @data_size: The size of the data (from the SG list) to be mapped into the
* OCS DMA linked list.
* @data_offset: The offset (within the SG list) of the data to be mapped.
*
* Return: 0 on success, negative error code otherwise.
*/
int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev,
struct scatterlist *sg,
int sg_dma_count,
struct ocs_dll_desc *dll_desc,
size_t data_size, size_t data_offset)
{
struct ocs_dma_linked_list *ll = NULL;
struct scatterlist *sg_tmp;
unsigned int tmp;
int dma_nents;
int i;
if (!dll_desc || !sg || !aes_dev)
return -EINVAL;
/* Default values for when no ddl_desc is created. */
dll_desc->vaddr = NULL;
dll_desc->dma_addr = DMA_MAPPING_ERROR;
dll_desc->size = 0;
if (data_size == 0)
return 0;
/* Loop over sg_list until we reach entry at specified offset. */
while (data_offset >= sg_dma_len(sg)) {
data_offset -= sg_dma_len(sg);
sg_dma_count--;
sg = sg_next(sg);
/* If we reach the end of the list, offset was invalid. */
if (!sg || sg_dma_count == 0)
return -EINVAL;
}
/* Compute number of DMA-mapped SG entries to add into OCS DMA list. */
dma_nents = 0;
tmp = 0;
sg_tmp = sg;
while (tmp < data_offset + data_size) {
/* If we reach the end of the list, data_size was invalid. */
if (!sg_tmp)
return -EINVAL;
tmp += sg_dma_len(sg_tmp);
dma_nents++;
sg_tmp = sg_next(sg_tmp);
}
if (dma_nents > sg_dma_count)
return -EINVAL;
/* Allocate the DMA list, one entry for each SG entry. */
dll_desc->size = sizeof(struct ocs_dma_linked_list) * dma_nents;
dll_desc->vaddr = dma_alloc_coherent(aes_dev->dev, dll_desc->size,
&dll_desc->dma_addr, GFP_KERNEL);
if (!dll_desc->vaddr)
return -ENOMEM;
/* Populate DMA linked list entries. */
ll = dll_desc->vaddr;
for (i = 0; i < dma_nents; i++, sg = sg_next(sg)) {
ll[i].src_addr = sg_dma_address(sg) + data_offset;
ll[i].src_len = (sg_dma_len(sg) - data_offset) < data_size ?
(sg_dma_len(sg) - data_offset) : data_size;
data_offset = 0;
data_size -= ll[i].src_len;
/* Current element points to the DMA address of the next one. */
ll[i].next = dll_desc->dma_addr + (sizeof(*ll) * (i + 1));
ll[i].ll_flags = 0;
}
/* Terminate last element. */
ll[i - 1].next = 0;
ll[i - 1].ll_flags = OCS_LL_DMA_FLAG_TERMINATE;
return 0;
}
Reference in New Issue View Git Blame Copy Permalink