CoolPi-Armbian-Rockchip-RK3588-Upstream-Linux-6.1-LTS-Kernel/drivers/crypto/rockchip/rk_crypto_v3_skcipher.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Crypto acceleration support for Rockchip Crypto V2
 *
 * Copyright (c) 2022, Rockchip Electronics Co., Ltd.
 *
 * Author: Lin Jinhan <troy.lin@rock-chips.com>
 *
 */

#include <crypto/scatterwalk.h>
#include <linux/iopoll.h>
#include <linux/module.h>
#include <linux/platform_device.h>

#include "rk_crypto_core.h"
#include "rk_crypto_utils.h"
#include "rk_crypto_skcipher_utils.h"
#include "rk_crypto_v3.h"
#include "rk_crypto_v3_reg.h"

#define RK_POLL_PERIOD_US	100
#define RK_POLL_TIMEOUT_US	50000

static const u32 cipher_algo2bc[] = {
	[CIPHER_ALGO_DES]      = CRYPTO_BC_DES,
	[CIPHER_ALGO_DES3_EDE] = CRYPTO_BC_TDES,
	[CIPHER_ALGO_AES]      = CRYPTO_BC_AES,
	[CIPHER_ALGO_SM4]      = CRYPTO_BC_SM4,
};

static const u32 cipher_mode2bc[] = {
	[CIPHER_MODE_ECB] = CRYPTO_BC_ECB,
	[CIPHER_MODE_CBC] = CRYPTO_BC_CBC,
	[CIPHER_MODE_CFB] = CRYPTO_BC_CFB,
	[CIPHER_MODE_OFB] = CRYPTO_BC_OFB,
	[CIPHER_MODE_CTR] = CRYPTO_BC_CTR,
	[CIPHER_MODE_XTS] = CRYPTO_BC_XTS,
	[CIPHER_MODE_GCM] = CRYPTO_BC_GCM,
};

static inline bool is_check_tag_valid(struct rk_crypto_dev *dev)
{
	/* crypto < v4 need to check tag valid */
	return CRYPTO_MAJOR_VER(CRYPTO_READ(dev, CRYPTO_CRYPTO_VERSION)) < CRYPTO_MAJOR_VER_4;
}

static int rk_crypto_irq_handle(int irq, void *dev_id)
{
	struct rk_crypto_dev *rk_dev = platform_get_drvdata(dev_id);
	u32 interrupt_status;
	struct rk_hw_crypto_v3_info *hw_info =
			(struct rk_hw_crypto_v3_info *)rk_dev->hw_info;
	struct rk_alg_ctx *alg_ctx = rk_cipher_alg_ctx(rk_dev);

	interrupt_status = CRYPTO_READ(rk_dev, CRYPTO_DMA_INT_ST);
	CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_ST, interrupt_status);

	interrupt_status &= CRYPTO_LOCKSTEP_MASK;

	if (interrupt_status != CRYPTO_DST_ITEM_DONE_INT_ST) {
		dev_err(rk_dev->dev, "DMA desc = %p\n", hw_info->hw_desc.lli_head);
		dev_err(rk_dev->dev, "DMA addr_in = %08x\n",
			(u32)alg_ctx->addr_in);
		dev_err(rk_dev->dev, "DMA addr_out = %08x\n",
			(u32)alg_ctx->addr_out);
		dev_err(rk_dev->dev, "DMA count = %08x\n", alg_ctx->count);
		dev_err(rk_dev->dev, "DMA desc_dma = %08x\n",
			(u32)hw_info->hw_desc.lli_head_dma);
		dev_err(rk_dev->dev, "DMA Error status = %08x\n",
			interrupt_status);
		dev_err(rk_dev->dev, "DMA CRYPTO_DMA_LLI_ADDR status = %08x\n",
			CRYPTO_READ(rk_dev, CRYPTO_DMA_LLI_ADDR));
		dev_err(rk_dev->dev, "DMA CRYPTO_DMA_ST status = %08x\n",
			CRYPTO_READ(rk_dev, CRYPTO_DMA_ST));
		dev_err(rk_dev->dev, "DMA CRYPTO_DMA_STATE status = %08x\n",
			CRYPTO_READ(rk_dev, CRYPTO_DMA_STATE));
		dev_err(rk_dev->dev, "DMA CRYPTO_DMA_LLI_RADDR status = %08x\n",
			CRYPTO_READ(rk_dev, CRYPTO_DMA_LLI_RADDR));
		dev_err(rk_dev->dev, "DMA CRYPTO_DMA_SRC_RADDR status = %08x\n",
			CRYPTO_READ(rk_dev, CRYPTO_DMA_SRC_RADDR));
		dev_err(rk_dev->dev, "DMA CRYPTO_DMA_DST_RADDR status = %08x\n",
			CRYPTO_READ(rk_dev, CRYPTO_DMA_DST_RADDR));
		rk_dev->err = -EFAULT;
	}

	return 0;
}

static inline void set_pc_len_reg(struct rk_crypto_dev *rk_dev, u64 pc_len)
{
	u32 chn_base = CRYPTO_CH0_PC_LEN_0;

	CRYPTO_TRACE("PC length = %lu\n", (unsigned long)pc_len);

	CRYPTO_WRITE(rk_dev, chn_base, pc_len & 0xffffffff);
	CRYPTO_WRITE(rk_dev, chn_base + 4, pc_len >> 32);
}

static inline void set_aad_len_reg(struct rk_crypto_dev *rk_dev, u64 aad_len)
{
	u32 chn_base = CRYPTO_CH0_AAD_LEN_0;

	CRYPTO_TRACE("AAD length = %lu\n", (unsigned long)aad_len);

	CRYPTO_WRITE(rk_dev, chn_base, aad_len & 0xffffffff);
	CRYPTO_WRITE(rk_dev, chn_base + 4, aad_len >> 32);
}

static void set_iv_reg(struct rk_crypto_dev *rk_dev, const u8 *iv, u32 iv_len)
{
	if (!iv || iv_len == 0)
		return;

	CRYPTO_DUMPHEX("set iv", iv, iv_len);

	rk_crypto_write_regs(rk_dev, CRYPTO_CH0_IV_0, iv, iv_len);

	CRYPTO_WRITE(rk_dev, CRYPTO_CH0_IV_LEN_0, iv_len);
}

static void write_key_reg(struct rk_crypto_dev *rk_dev, const u8 *key,
			  u32 key_len)
{
	rk_crypto_write_regs(rk_dev, CRYPTO_CH0_KEY_0, key, key_len);
}

static void write_tkey_reg(struct rk_crypto_dev *rk_dev, const u8 *key,
			   u32 key_len)
{
	rk_crypto_write_regs(rk_dev, CRYPTO_CH4_KEY_0, key, key_len);
}

static int get_tag_reg(struct rk_crypto_dev *rk_dev, u8 *tag, u32 tag_len)
{
	int ret = 0;
	u32 reg_ctrl = 0;

	CRYPTO_TRACE("tag_len = %u", tag_len);

	if (tag_len > RK_MAX_TAG_SIZE)
		return -EINVAL;

	if (is_check_tag_valid(rk_dev)) {
		ret = read_poll_timeout_atomic(CRYPTO_READ,
					       reg_ctrl,
					       reg_ctrl & CRYPTO_CH0_TAG_VALID,
					       0,
					       RK_POLL_TIMEOUT_US,
					       false,
					       rk_dev, CRYPTO_TAG_VALID);
		if (ret)
			goto exit;
	}

	rk_crypto_read_regs(rk_dev, CRYPTO_CH0_TAG_0, tag, tag_len);
exit:
	return ret;
}

static bool is_force_fallback(struct rk_crypto_algt *algt, uint32_t key_len)
{
	if (algt->algo != CIPHER_ALGO_AES)
		return false;

	/* crypto v2 not support xts with AES-192 */
	if (algt->mode == CIPHER_MODE_XTS && key_len == AES_KEYSIZE_192 * 2)
		return true;

	if (algt->use_soft_aes192 && key_len == AES_KEYSIZE_192)
		return true;

	return false;
}

static bool is_calc_need_round_up(struct skcipher_request *req)
{
	struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
	struct rk_crypto_algt *algt = rk_cipher_get_algt(cipher);

	return (algt->mode == CIPHER_MODE_CFB ||
			algt->mode == CIPHER_MODE_OFB ||
			algt->mode == CIPHER_MODE_CTR) ? true : false;
}

static void rk_cipher_reset(struct rk_crypto_dev *rk_dev)
{
	int ret;
	u32 tmp = 0, tmp_mask = 0;
	unsigned int  pool_timeout_us = 1000;

	CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0x00);

	tmp = CRYPTO_SW_CC_RESET;
	tmp_mask = tmp << CRYPTO_WRITE_MASK_SHIFT;

	CRYPTO_WRITE(rk_dev, CRYPTO_RST_CTL, tmp | tmp_mask);

	/* This is usually done in 20 clock cycles */
	ret = read_poll_timeout_atomic(CRYPTO_READ, tmp, !tmp, 0,
				       pool_timeout_us, false, rk_dev, CRYPTO_RST_CTL);
	if (ret)
		dev_err(rk_dev->dev, "cipher reset pool timeout %ums.",
			pool_timeout_us);

	CRYPTO_WRITE(rk_dev, CRYPTO_BC_CTL, 0xffff0000);

	/* clear dma int status */
	tmp = CRYPTO_READ(rk_dev, CRYPTO_DMA_INT_ST);
	CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_ST, tmp);
}

static void rk_crypto_complete(struct crypto_async_request *base, int err)
{
	struct rk_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
	struct rk_alg_ctx *alg_ctx = &ctx->algs_ctx;
	struct rk_hw_crypto_v3_info *hw_info = ctx->rk_dev->hw_info;
	struct crypto_lli_desc *lli_desc = hw_info->hw_desc.lli_head;

	CRYPTO_WRITE(ctx->rk_dev, CRYPTO_BC_CTL, 0xffff0000);
	if (err) {
		rk_cipher_reset(ctx->rk_dev);
		pr_err("aligned = %u, align_size = %u\n",
		       alg_ctx->aligned, alg_ctx->align_size);
		pr_err("total = %u, left = %u, count = %u\n",
		       alg_ctx->total, alg_ctx->left_bytes, alg_ctx->count);
		pr_err("lli->src     = %08x\n", lli_desc->src_addr);
		pr_err("lli->src_len = %08x\n", lli_desc->src_len);
		pr_err("lli->dst     = %08x\n", lli_desc->dst_addr);
		pr_err("lli->dst_len = %08x\n", lli_desc->dst_len);
		pr_err("lli->dma_ctl = %08x\n", lli_desc->dma_ctrl);
		pr_err("lli->usr_def = %08x\n", lli_desc->user_define);
		pr_err("lli->next    = %08x\n\n\n", lli_desc->next_addr);
	}

	if (base->complete)
		base->complete(base, err);
}

static int rk_cipher_crypt(struct skcipher_request *req, bool encrypt)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm);
	struct rk_crypto_algt *algt = rk_cipher_get_algt(tfm);

	CRYPTO_TRACE("%s total = %u",
		     encrypt ? "encrypt" : "decrypt", req->cryptlen);

	if (!req->cryptlen) {
		if (algt->mode == CIPHER_MODE_ECB ||
		    algt->mode == CIPHER_MODE_CBC ||
		    algt->mode == CIPHER_MODE_CTR ||
		    algt->mode == CIPHER_MODE_CFB ||
		    algt->mode == CIPHER_MODE_OFB)
			return 0;
		else
			return -EINVAL;
	}

	/* XTS data should >= chunksize */
	if (algt->mode == CIPHER_MODE_XTS) {
		if (req->cryptlen < crypto_skcipher_chunksize(tfm))
			return -EINVAL;

		/* force use unalign branch */
		ctx->algs_ctx.align_size = ctx->rk_dev->vir_max;

		/*  XTS can't pause when use hardware crypto */
		if (req->cryptlen > ctx->rk_dev->vir_max)
			return rk_cipher_fallback(req, ctx, encrypt);
	}

	if (is_force_fallback(algt, ctx->keylen))
		return rk_cipher_fallback(req, ctx, encrypt);

	ctx->mode = cipher_algo2bc[algt->algo] |
		    cipher_mode2bc[algt->mode];
	if (!encrypt)
		ctx->mode |= CRYPTO_BC_DECRYPT;

	if (algt->algo == CIPHER_ALGO_AES) {
		uint32_t key_factor;

		/* The key length of XTS is twice the normal length */
		key_factor = algt->mode == CIPHER_MODE_XTS ? 2 : 1;

		if (ctx->keylen == AES_KEYSIZE_128 * key_factor)
			ctx->mode |= CRYPTO_BC_128_bit_key;
		else if (ctx->keylen == AES_KEYSIZE_192 * key_factor)
			ctx->mode |= CRYPTO_BC_192_bit_key;
		else if (ctx->keylen == AES_KEYSIZE_256 * key_factor)
			ctx->mode |= CRYPTO_BC_256_bit_key;
	}

	ctx->iv_len = crypto_skcipher_ivsize(tfm);

	memset(ctx->iv, 0x00, sizeof(ctx->iv));
	memcpy(ctx->iv, req->iv, ctx->iv_len);

	ctx->is_enc = encrypt;

	CRYPTO_MSG("ctx->mode = %x\n", ctx->mode);
	return rk_skcipher_handle_req(ctx->rk_dev, req);
}

static int rk_cipher_encrypt(struct skcipher_request *req)
{
	return rk_cipher_crypt(req, true);
}

static int rk_cipher_decrypt(struct skcipher_request *req)
{
	return rk_cipher_crypt(req, false);
}

static int rk_ablk_hw_init(struct rk_crypto_dev *rk_dev, u32 algo, u32 mode)
{
	struct rk_cipher_ctx *ctx = rk_cipher_ctx_cast(rk_dev);

	rk_cipher_reset(rk_dev);

	CRYPTO_WRITE(rk_dev, CRYPTO_BC_CTL, 0x00010000);

	if (mode == CIPHER_MODE_XTS) {
		uint32_t tmp_len = ctx->keylen / 2;

		write_key_reg(ctx->rk_dev, ctx->key, tmp_len);
		write_tkey_reg(ctx->rk_dev, ctx->key + tmp_len, tmp_len);
	} else {
		write_key_reg(ctx->rk_dev, ctx->key, ctx->keylen);
	}

	if (mode != CIPHER_MODE_ECB)
		set_iv_reg(rk_dev, ctx->iv, ctx->iv_len);

	ctx->mode |= CRYPTO_BC_ENABLE;

	CRYPTO_WRITE(rk_dev, CRYPTO_FIFO_CTL, 0x00030003);

	CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0x7f);

	CRYPTO_WRITE(rk_dev, CRYPTO_BC_CTL, ctx->mode | CRYPTO_WRITE_MASK_ALL);

	return 0;
}

static int crypto_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag)
{
	struct rk_hw_crypto_v3_info *hw_info =
			(struct rk_hw_crypto_v3_info *)rk_dev->hw_info;
	struct skcipher_request *req =
		skcipher_request_cast(rk_dev->async_req);
	struct rk_alg_ctx *alg_ctx = rk_cipher_alg_ctx(rk_dev);
	struct crypto_lli_desc *lli_head, *lli_tail, *lli_aad;
	u32 calc_len = alg_ctx->count;
	u32 start_flag = CRYPTO_DMA_START;
	int ret;

	if (alg_ctx->aligned)
		ret = rk_crypto_hw_desc_init(&hw_info->hw_desc,
					     alg_ctx->sg_src, alg_ctx->sg_dst, alg_ctx->count);
	else
		ret = rk_crypto_hw_desc_init(&hw_info->hw_desc,
					     &alg_ctx->sg_tmp, &alg_ctx->sg_tmp, alg_ctx->count);
	if (ret)
		return ret;

	lli_head = hw_info->hw_desc.lli_head;
	lli_tail = hw_info->hw_desc.lli_tail;
	lli_aad  = hw_info->hw_desc.lli_aad;

	/*
	 *	the data length is not aligned will use addr_vir to calculate,
	 *	so crypto v2 could round up data length to chunk_size
	 */
	if (!alg_ctx->is_aead && is_calc_need_round_up(req))
		calc_len = round_up(calc_len, alg_ctx->chunk_size);

	CRYPTO_TRACE("calc_len = %u, cryptlen = %u, assoclen= %u, is_aead = %d",
		     calc_len, alg_ctx->total, alg_ctx->assoclen, alg_ctx->is_aead);

	lli_head->user_define = LLI_USER_STRING_START | LLI_USER_CIPHER_START;

	lli_tail->dma_ctrl     = LLI_DMA_CTRL_DST_DONE | LLI_DMA_CTRL_LAST;
	lli_tail->user_define |= LLI_USER_STRING_LAST;
	lli_tail->src_len     += (calc_len - alg_ctx->count);
	lli_tail->dst_len     += (calc_len - alg_ctx->count);

	if (alg_ctx->is_aead) {
		lli_aad->src_addr    = alg_ctx->addr_aad_in;
		lli_aad->src_len     = alg_ctx->assoclen;
		lli_aad->user_define = LLI_USER_CIPHER_START |
				       LLI_USER_STRING_START |
				       LLI_USER_STRING_LAST |
				       LLI_USER_STRING_AAD;
		lli_aad->next_addr   = hw_info->hw_desc.lli_head_dma;

		/* clear cipher start */
		lli_head->user_define &= (~((u32)LLI_USER_CIPHER_START));

		set_pc_len_reg(rk_dev, alg_ctx->total);
		set_aad_len_reg(rk_dev, alg_ctx->assoclen);
	}

	rk_crypto_dump_hw_desc(&hw_info->hw_desc);

	dma_wmb();

	if (alg_ctx->is_aead)
		CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->hw_desc.lli_aad_dma);
	else
		CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->hw_desc.lli_head_dma);

	CRYPTO_WRITE(rk_dev, CRYPTO_DMA_CTL, start_flag | (start_flag << WRITE_MASK));

	return 0;
}

static int rk_ablk_init_tfm(struct crypto_skcipher *tfm)
{
	struct rk_crypto_algt *algt = rk_cipher_get_algt(tfm);
	struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm);
	const char *alg_name = crypto_tfm_alg_name(crypto_skcipher_tfm(tfm));
	struct rk_crypto_dev *rk_dev = algt->rk_dev;
	struct rk_alg_ctx *alg_ctx = &ctx->algs_ctx;

	CRYPTO_TRACE();

	memset(ctx, 0x00, sizeof(*ctx));

	if (!rk_dev->request_crypto)
		return -EFAULT;

	rk_dev->request_crypto(rk_dev, alg_name);

	/* always not aligned for crypto v2 cipher */
	alg_ctx->align_size     = 64;
	alg_ctx->chunk_size     = crypto_skcipher_chunksize(tfm);

	alg_ctx->ops.start      = rk_ablk_start;
	alg_ctx->ops.update     = rk_ablk_rx;
	alg_ctx->ops.complete   = rk_crypto_complete;
	alg_ctx->ops.irq_handle = rk_crypto_irq_handle;

	alg_ctx->ops.hw_init      = rk_ablk_hw_init;
	alg_ctx->ops.hw_dma_start = crypto_dma_start;
	alg_ctx->ops.hw_write_iv  = set_iv_reg;

	ctx->rk_dev = rk_dev;

	if (algt->alg.crypto.base.cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
		CRYPTO_MSG("alloc fallback tfm, name = %s", alg_name);
		ctx->fallback_tfm = crypto_alloc_skcipher(alg_name, 0,
							  CRYPTO_ALG_ASYNC |
							  CRYPTO_ALG_NEED_FALLBACK);
		if (IS_ERR(ctx->fallback_tfm)) {
			CRYPTO_MSG("Could not load fallback driver %s : %ld.\n",
				   alg_name, PTR_ERR(ctx->fallback_tfm));
			ctx->fallback_tfm = NULL;
		}
	}

	return 0;
}

static void rk_ablk_exit_tfm(struct crypto_skcipher *tfm)
{
	struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm);
	const char *alg_name = crypto_tfm_alg_name(crypto_skcipher_tfm(tfm));

	CRYPTO_TRACE();

	if (ctx->fallback_tfm) {
		CRYPTO_MSG("free fallback tfm");
		crypto_free_skcipher(ctx->fallback_tfm);
	}

	ctx->rk_dev->release_crypto(ctx->rk_dev, alg_name);
}

static int rk_aead_init_tfm(struct crypto_aead *tfm)
{
	struct aead_alg *alg = crypto_aead_alg(tfm);
	struct rk_crypto_algt *algt =
		container_of(alg, struct rk_crypto_algt, alg.aead);
	struct rk_cipher_ctx *ctx = crypto_tfm_ctx(&tfm->base);
	const char *alg_name = crypto_tfm_alg_name(&tfm->base);
	struct rk_crypto_dev *rk_dev = algt->rk_dev;
	struct rk_alg_ctx *alg_ctx = &ctx->algs_ctx;

	CRYPTO_TRACE();

	if (!rk_dev->request_crypto)
		return -EFAULT;

	rk_dev->request_crypto(rk_dev, alg_name);

	alg_ctx->align_size     = 64;
	alg_ctx->chunk_size     = crypto_aead_chunksize(tfm);

	alg_ctx->ops.start      = rk_aead_start;
	alg_ctx->ops.update     = rk_ablk_rx;
	alg_ctx->ops.complete   = rk_crypto_complete;
	alg_ctx->ops.irq_handle = rk_crypto_irq_handle;

	alg_ctx->ops.hw_init       = rk_ablk_hw_init;
	alg_ctx->ops.hw_dma_start  = crypto_dma_start;
	alg_ctx->ops.hw_write_iv   = set_iv_reg;
	alg_ctx->ops.hw_get_result = get_tag_reg;

	ctx->rk_dev      = rk_dev;
	alg_ctx->is_aead = 1;

	if (algt->alg.crypto.base.cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
		CRYPTO_MSG("alloc fallback tfm, name = %s", alg_name);
		ctx->fallback_aead =
			crypto_alloc_aead(alg_name, 0,
					  CRYPTO_ALG_ASYNC |
					  CRYPTO_ALG_NEED_FALLBACK);
		if (IS_ERR(ctx->fallback_aead)) {
			dev_err(rk_dev->dev,
				"Load fallback driver %s err: %ld.\n",
				alg_name, PTR_ERR(ctx->fallback_aead));
			ctx->fallback_aead = NULL;
			crypto_aead_set_reqsize(tfm, sizeof(struct aead_request));
		} else {
			crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) +
						crypto_aead_reqsize(ctx->fallback_aead));
		}
	}

	return 0;
}

static void rk_aead_exit_tfm(struct crypto_aead *tfm)
{
	struct rk_cipher_ctx *ctx = crypto_tfm_ctx(&tfm->base);

	CRYPTO_TRACE();

	if (ctx->fallback_aead) {
		CRYPTO_MSG("free fallback tfm");
		crypto_free_aead(ctx->fallback_aead);
	}

	ctx->rk_dev->release_crypto(ctx->rk_dev, crypto_tfm_alg_name(&tfm->base));
}

static int rk_aead_crypt(struct aead_request *req, bool encrypt)
{
	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
	struct rk_cipher_ctx *ctx = crypto_aead_ctx(tfm);
	struct rk_crypto_algt *algt = rk_aead_get_algt(tfm);
	struct scatterlist *sg_src, *sg_dst;
	struct scatterlist src[2], dst[2];
	u64 data_len;
	bool aligned;
	int ret = -EINVAL;

	CRYPTO_TRACE("%s cryptlen = %u, assoclen = %u",
		     encrypt ? "encrypt" : "decrypt",
		     req->cryptlen, req->assoclen);

	data_len = encrypt ? req->cryptlen : (req->cryptlen - crypto_aead_authsize(tfm));

	if (req->assoclen == 0 ||
	    req->cryptlen == 0 ||
	    data_len == 0 ||
	    is_force_fallback(algt, ctx->keylen))
		return rk_aead_fallback(req, ctx, encrypt);

	/* point sg_src and sg_dst skip assoc data */
	sg_src = scatterwalk_ffwd(src, req->src, req->assoclen);
	sg_dst = (req->src == req->dst) ? sg_src : scatterwalk_ffwd(dst, req->dst, req->assoclen);

	aligned = rk_crypto_check_align(sg_src, sg_nents_for_len(sg_src, data_len),
					sg_dst, sg_nents_for_len(sg_dst, data_len),
					64);

	if (sg_nents_for_len(sg_src, data_len) > RK_DEFAULT_LLI_CNT ||
	    sg_nents_for_len(sg_dst, data_len) > RK_DEFAULT_LLI_CNT)
		return rk_aead_fallback(req, ctx, encrypt);

	if (!aligned) {
		if (req->assoclen > ctx->rk_dev->aad_max ||
		    data_len > ctx->rk_dev->vir_max)
			return rk_aead_fallback(req, ctx, encrypt);
	}

	ctx->mode = cipher_algo2bc[algt->algo] |
		    cipher_mode2bc[algt->mode];
	if (!encrypt)
		ctx->mode |= CRYPTO_BC_DECRYPT;

	if (algt->algo == CIPHER_ALGO_AES) {
		if (ctx->keylen == AES_KEYSIZE_128)
			ctx->mode |= CRYPTO_BC_128_bit_key;
		else if (ctx->keylen == AES_KEYSIZE_192)
			ctx->mode |= CRYPTO_BC_192_bit_key;
		else if (ctx->keylen == AES_KEYSIZE_256)
			ctx->mode |= CRYPTO_BC_256_bit_key;
	}

	ctx->iv_len = crypto_aead_ivsize(tfm);

	memset(ctx->iv, 0x00, sizeof(ctx->iv));
	memcpy(ctx->iv, req->iv, ctx->iv_len);

	ctx->is_enc = encrypt;

	CRYPTO_MSG("ctx->mode = %x\n", ctx->mode);
	ret = rk_aead_handle_req(ctx->rk_dev, req);

	return ret;
}

static int rk_aead_encrypt(struct aead_request *req)
{
	return rk_aead_crypt(req, true);
}

static int rk_aead_decrypt(struct aead_request *req)
{
	return rk_aead_crypt(req, false);
}

struct rk_crypto_algt rk_v3_ecb_sm4_alg =
	RK_CIPHER_ALGO_INIT(SM4, ECB, ecb(sm4), ecb-sm4-rk);

struct rk_crypto_algt rk_v3_cbc_sm4_alg =
	RK_CIPHER_ALGO_INIT(SM4, CBC, cbc(sm4), cbc-sm4-rk);

struct rk_crypto_algt rk_v3_xts_sm4_alg =
	RK_CIPHER_ALGO_XTS_INIT(SM4, xts(sm4), xts-sm4-rk);

struct rk_crypto_algt rk_v3_cfb_sm4_alg =
	RK_CIPHER_ALGO_INIT(SM4, CFB, cfb(sm4), cfb-sm4-rk);

struct rk_crypto_algt rk_v3_ofb_sm4_alg =
	RK_CIPHER_ALGO_INIT(SM4, OFB, ofb(sm4), ofb-sm4-rk);

struct rk_crypto_algt rk_v3_ctr_sm4_alg =
	RK_CIPHER_ALGO_INIT(SM4, CTR, ctr(sm4), ctr-sm4-rk);

struct rk_crypto_algt rk_v3_gcm_sm4_alg =
	RK_AEAD_ALGO_INIT(SM4, GCM, gcm(sm4), gcm-sm4-rk);

struct rk_crypto_algt rk_v3_ecb_aes_alg =
	RK_CIPHER_ALGO_INIT(AES, ECB, ecb(aes), ecb-aes-rk);

struct rk_crypto_algt rk_v3_cbc_aes_alg =
	RK_CIPHER_ALGO_INIT(AES, CBC, cbc(aes), cbc-aes-rk);

struct rk_crypto_algt rk_v3_xts_aes_alg =
	RK_CIPHER_ALGO_XTS_INIT(AES, xts(aes), xts-aes-rk);

struct rk_crypto_algt rk_v3_cfb_aes_alg =
	RK_CIPHER_ALGO_INIT(AES, CFB, cfb(aes), cfb-aes-rk);

struct rk_crypto_algt rk_v3_ofb_aes_alg =
	RK_CIPHER_ALGO_INIT(AES, OFB, ofb(aes), ofb-aes-rk);

struct rk_crypto_algt rk_v3_ctr_aes_alg =
	RK_CIPHER_ALGO_INIT(AES, CTR, ctr(aes), ctr-aes-rk);

struct rk_crypto_algt rk_v3_gcm_aes_alg =
	RK_AEAD_ALGO_INIT(AES, GCM, gcm(aes), gcm-aes-rk);

struct rk_crypto_algt rk_v3_ecb_des_alg =
	RK_CIPHER_ALGO_INIT(DES, ECB, ecb(des), ecb-des-rk);

struct rk_crypto_algt rk_v3_cbc_des_alg =
	RK_CIPHER_ALGO_INIT(DES, CBC, cbc(des), cbc-des-rk);

struct rk_crypto_algt rk_v3_cfb_des_alg =
	RK_CIPHER_ALGO_INIT(DES, CFB, cfb(des), cfb-des-rk);

struct rk_crypto_algt rk_v3_ofb_des_alg =
	RK_CIPHER_ALGO_INIT(DES, OFB, ofb(des), ofb-des-rk);

struct rk_crypto_algt rk_v3_ecb_des3_ede_alg =
	RK_CIPHER_ALGO_INIT(DES3_EDE, ECB, ecb(des3_ede), ecb-des3_ede-rk);

struct rk_crypto_algt rk_v3_cbc_des3_ede_alg =
	RK_CIPHER_ALGO_INIT(DES3_EDE, CBC, cbc(des3_ede), cbc-des3_ede-rk);

struct rk_crypto_algt rk_v3_cfb_des3_ede_alg =
	RK_CIPHER_ALGO_INIT(DES3_EDE, CFB, cfb(des3_ede), cfb-des3_ede-rk);

struct rk_crypto_algt rk_v3_ofb_des3_ede_alg =
	RK_CIPHER_ALGO_INIT(DES3_EDE, OFB, ofb(des3_ede), ofb-des3_ede-rk);