CoolPi-Armbian-Rockchip-RK3588-Upstream-Linux-6.1-LTS-Kernel/drivers/video/rockchip/rve/rve_job.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) Rockchip Electronics Co., Ltd.
 *
 * Author: Huang Lee <Putin.li@rock-chips.com>
 */

#define pr_fmt(fmt) "rve_job: " fmt

#include "rve_job.h"
#include "rve_fence.h"
#include "rve_reg.h"

struct rve_job *
rve_scheduler_get_pending_job_list(struct rve_scheduler_t *scheduler)
{
	unsigned long flags;
	struct rve_job *job;

	spin_lock_irqsave(&scheduler->irq_lock, flags);

	job = list_first_entry_or_null(&scheduler->todo_list,
		struct rve_job, head);

	spin_unlock_irqrestore(&scheduler->irq_lock, flags);

	return job;
}

struct rve_job *
rve_scheduler_get_running_job(struct rve_scheduler_t *scheduler)
{
	unsigned long flags;
	struct rve_job *job;

	spin_lock_irqsave(&scheduler->irq_lock, flags);

	job = scheduler->running_job;

	spin_unlock_irqrestore(&scheduler->irq_lock, flags);

	return job;
}

static void rve_scheduler_set_pid_info(struct rve_job *job, ktime_t now)
{
	struct rve_scheduler_t *scheduler;
	bool pid_match_flag = false;
	ktime_t tmp = 0;
	int pid_mark = 0, i;

	scheduler = rve_job_get_scheduler(job);

	for (i = 0; i < RVE_MAX_PID_INFO; i++) {
		if (scheduler->session.pid_info[i].pid == 0)
			scheduler->session.pid_info[i].pid = job->pid;

		if (scheduler->session.pid_info[i].pid == job->pid) {
			pid_match_flag = true;
			scheduler->session.pid_info[i].hw_time_total +=
				(job->hw_running_time - now);
			break;
		}
	}

	if (!pid_match_flag) {
		for (i = 0; i < RVE_MAX_PID_INFO; i++) {
			if (i == 0) {
				tmp = scheduler->session.pid_info[i].hw_time_total;
				continue;
			}

			if (tmp > scheduler->session.pid_info[i].hw_time_total)
				pid_mark = i;
		}

		scheduler->session.pid_info[pid_mark].pid = job->pid;
		scheduler->session.pid_info[pid_mark].hw_time_total +=
					ktime_us_delta(now, job->hw_running_time);
	}
}

struct rve_scheduler_t *rve_job_get_scheduler(struct rve_job *job)
{
	return job->scheduler;
}

struct rve_internal_ctx_t *rve_job_get_internal_ctx(struct rve_job *job)
{
	return job->ctx;
}

static void rve_job_free(struct rve_job *job)
{
#ifdef CONFIG_SYNC_FILE
	if (job->out_fence)
		dma_fence_put(job->out_fence);
#endif

	free_page((unsigned long)job);
}

static int rve_job_cleanup(struct rve_job *job)
{
	ktime_t now = ktime_get();

	if (DEBUGGER_EN(TIME)) {
		pr_info("(pid:%d) job clean use time = %lld\n", job->pid,
			ktime_us_delta(now, job->timestamp));
	}
	rve_job_free(job);

	return 0;
}

void rve_job_session_destroy(struct rve_session *session)
{
	struct rve_scheduler_t *scheduler = NULL;
	struct rve_job *job_pos, *job_q;
	int i;

	unsigned long flags;

	for (i = 0; i < rve_drvdata->num_of_scheduler; i++) {
		scheduler = rve_drvdata->scheduler[i];

		spin_lock_irqsave(&scheduler->irq_lock, flags);

		list_for_each_entry_safe(job_pos, job_q, &scheduler->todo_list, head) {
			if (session == job_pos->session) {
				list_del(&job_pos->head);

				spin_unlock_irqrestore(&scheduler->irq_lock, flags);

				rve_job_free(job_pos);

				spin_lock_irqsave(&scheduler->irq_lock, flags);
			}
		}

		spin_unlock_irqrestore(&scheduler->irq_lock, flags);
	}
}

static struct rve_job *rve_job_alloc(struct rve_internal_ctx_t *ctx)
{
	struct rve_job *job = NULL;

	job = (struct rve_job *)get_zeroed_page(GFP_KERNEL | GFP_DMA32);
	if (!job)
		return NULL;

#ifdef CONFIG_SYNC_FILE
	spin_lock_init(&job->fence_lock);
#endif
	INIT_LIST_HEAD(&job->head);

	job->timestamp = ktime_get();
	job->pid = current->pid;
	job->regcmd_data = &ctx->regcmd_data[ctx->running_job_count];

	job->scheduler = rve_drvdata->scheduler[0];
	job->core = rve_drvdata->scheduler[0]->core;
	job->ctx = ctx;
	ctx->scheduler = job->scheduler;
	job->session = ctx->session;

	if (ctx->priority > 0) {
		if (ctx->priority > RVE_SCHED_PRIORITY_MAX)
			job->priority = RVE_SCHED_PRIORITY_MAX;
		else
			job->priority = ctx->priority;
	}

	return job;
}

static void rve_job_dump_info(struct rve_job *job)
{
	pr_info("job: priority = %d, core = %d\n",
		job->priority, job->core);
}

static int rve_job_run(struct rve_job *job)
{
	struct rve_scheduler_t *scheduler;
	int ret = 0;

	scheduler = rve_job_get_scheduler(job);

#ifndef RVE_PD_AWAYS_ON
	/* enable power */
	ret = rve_power_enable(scheduler);
	if (ret < 0) {
		pr_err("power enable failed");
		return ret;
	}
#endif

	ret = scheduler->ops->init_reg(job);
	if (ret < 0) {
		pr_err("init reg failed");
		goto failed;
	}

	ret = scheduler->ops->set_reg(job, scheduler);
	if (ret < 0) {
		pr_err("set reg failed");
		goto failed;
	}

	/* for debug */
	if (DEBUGGER_EN(MSG))
		rve_job_dump_info(job);

	return ret;

failed:
#ifndef RVE_PD_AWAYS_ON
	rve_power_disable(scheduler);
#endif

	return ret;
}

static void rve_job_next(struct rve_scheduler_t *scheduler)
{
	struct rve_job *job = NULL;
	unsigned long flags;

next_job:
	spin_lock_irqsave(&scheduler->irq_lock, flags);

	if (scheduler->running_job ||
		list_empty(&scheduler->todo_list)) {
		spin_unlock_irqrestore(&scheduler->irq_lock, flags);
		return;
	}

	job = list_first_entry(&scheduler->todo_list, struct rve_job, head);

	list_del_init(&job->head);

	scheduler->job_count--;

	scheduler->running_job = job;

	spin_unlock_irqrestore(&scheduler->irq_lock, flags);

	job->ret = rve_job_run(job);

	/* If some error before hw run */
	if (job->ret < 0) {
		pr_err("some error on rve_job_run before hw start, %s(%d)\n",
			__func__, __LINE__);

		spin_lock_irqsave(&scheduler->irq_lock, flags);

		scheduler->running_job = NULL;

		spin_unlock_irqrestore(&scheduler->irq_lock, flags);

		rve_internal_ctx_signal(job);

		goto next_job;
	}
}

static void rve_job_finish_and_next(struct rve_job *job, int ret)
{
	ktime_t now = ktime_get();
	struct rve_scheduler_t *scheduler;

	job->ret = ret;

	scheduler = rve_job_get_scheduler(job);

	if (DEBUGGER_EN(TIME)) {
		pr_info("hw use time = %lld\n", ktime_us_delta(now, job->hw_running_time));
		pr_info("(pid:%d) job done use time = %lld\n", job->pid,
			ktime_us_delta(now, job->timestamp));
	}

	rve_internal_ctx_signal(job);

	rve_job_next(scheduler);

#ifndef RVE_PD_AWAYS_ON
	rve_power_disable(scheduler);
#endif
}

void rve_job_done(struct rve_scheduler_t *scheduler, int ret)
{
	struct rve_job *job;
	unsigned long flags;
	u32 error_flag;
	uint32_t *cmd_reg;
	int i;

	ktime_t now = ktime_get();

	spin_lock_irqsave(&scheduler->irq_lock, flags);

	job = scheduler->running_job;
	scheduler->running_job = NULL;

	scheduler->timer.busy_time += ktime_us_delta(now, job->hw_recoder_time);

	rve_scheduler_set_pid_info(job, now);

	spin_unlock_irqrestore(&scheduler->irq_lock, flags);

	spin_lock_irqsave(&job->ctx->lock, flags);

	job->ctx->debug_info.max_cost_time_per_sec =
		max(job->ctx->debug_info.last_job_hw_use_time,
			job->ctx->debug_info.max_cost_time_per_sec);
	job->ctx->debug_info.last_job_hw_use_time = ktime_us_delta(now, job->hw_running_time);
	job->ctx->debug_info.hw_time_total += job->ctx->debug_info.last_job_hw_use_time;
	job->ctx->debug_info.last_job_use_time = ktime_us_delta(now, job->timestamp);

	spin_unlock_irqrestore(&job->ctx->lock, flags);

	/* record CFG REG copy to user */
	cmd_reg = job->regcmd_data->cmd_reg;
	for (i = 0; i < 40; i++)
		cmd_reg[18 + i] = rve_read(RVE_CFG_REG + i * 4, scheduler);

	error_flag = rve_read(RVE_SWREG6_IVE_WORK_STA, scheduler);

	rve_get_monitor_info(job);

	if (DEBUGGER_EN(MSG))
		pr_info("irq thread work_status[%.8x]\n", error_flag);

	/* disable llp enable, TODO: support pause mode */
	rve_write(0, RVE_SWLTB3_ENABLE, scheduler);

	rve_job_finish_and_next(job, ret);
}

static void rve_job_timeout_clean(struct rve_scheduler_t *scheduler)
{
	unsigned long flags;
	struct rve_job *job = NULL;
	ktime_t now = ktime_get();

	spin_lock_irqsave(&scheduler->irq_lock, flags);

	job = scheduler->running_job;
	if (job && (job->flags & RVE_ASYNC) &&
	   (ktime_to_ms(ktime_sub(now, job->hw_running_time)) >= RVE_ASYNC_TIMEOUT_DELAY)) {
		scheduler->running_job = NULL;

		spin_unlock_irqrestore(&scheduler->irq_lock, flags);

		scheduler->ops->soft_reset(scheduler);

		rve_internal_ctx_signal(job);

#ifndef RVE_PD_AWAYS_ON
		rve_power_disable(scheduler);
#endif
	} else {
		spin_unlock_irqrestore(&scheduler->irq_lock, flags);
	}
}

static struct rve_scheduler_t *rve_job_schedule(struct rve_job *job)
{
	unsigned long flags;
	struct rve_scheduler_t *scheduler = NULL;
	struct rve_job *job_pos;
	bool first_match = 0;

	scheduler = rve_job_get_scheduler(job);
	if (scheduler == NULL) {
		pr_err("failed to get scheduler, %s(%d)\n", __func__, __LINE__);
		return NULL;
	}

	/* Only async will timeout clean */
	rve_job_timeout_clean(scheduler);

	spin_lock_irqsave(&scheduler->irq_lock, flags);

	/* priority policy set by userspace */
	if (list_empty(&scheduler->todo_list)
		|| (job->priority == RVE_SCHED_PRIORITY_DEFAULT)) {
		list_add_tail(&job->head, &scheduler->todo_list);
	} else {
		list_for_each_entry(job_pos, &scheduler->todo_list, head) {
			if (job->priority > job_pos->priority &&
					(!first_match)) {
				list_add(&job->head, &job_pos->head);
				first_match = true;
			}

			/*
			 * Increase the priority of subsequent tasks
			 * after inserting into the list
			 */
			if (first_match)
				job_pos->priority++;
		}

		if (!first_match)
			list_add_tail(&job->head, &scheduler->todo_list);
	}

	scheduler->job_count++;

	spin_unlock_irqrestore(&scheduler->irq_lock, flags);

	rve_job_next(scheduler);

	return scheduler;
}

static void rve_job_abort_running(struct rve_job *job)
{
	unsigned long flags;
	struct rve_scheduler_t *scheduler;

	scheduler = rve_job_get_scheduler(job);

	spin_lock_irqsave(&scheduler->irq_lock, flags);

	/* invalid job */
	if (job == scheduler->running_job)
		scheduler->running_job = NULL;

	spin_unlock_irqrestore(&scheduler->irq_lock, flags);

	rve_job_cleanup(job);
}

static void rve_job_abort_invalid(struct rve_job *job)
{
	rve_job_cleanup(job);
}

static inline int rve_job_wait(struct rve_job *job)
{
	struct rve_scheduler_t *scheduler;

	int left_time;
	ktime_t now;
	int ret;

	scheduler = rve_job_get_scheduler(job);

	left_time = wait_event_timeout(scheduler->job_done_wq,
		job->ctx->finished_job_count == job->ctx->cmd_num,
		RVE_SYNC_TIMEOUT_DELAY * job->ctx->cmd_num);

	switch (left_time) {
	case 0:
		pr_err("%s timeout", __func__);
		scheduler->ops->soft_reset(scheduler);
		ret = -EBUSY;
		break;
	case -ERESTARTSYS:
		ret = -ERESTARTSYS;
		break;
	default:
		ret = 0;
		break;
	}

	now = ktime_get();

	if (DEBUGGER_EN(TIME))
		pr_info("%s use time = %lld\n", __func__,
			 ktime_to_us(ktime_sub(now, job->hw_running_time)));

	return ret;
}

#ifdef CONFIG_SYNC_FILE
static void rve_job_input_fence_signaled(struct dma_fence *fence,
					 struct dma_fence_cb *_waiter)
{
	struct rve_fence_waiter *waiter = (struct rve_fence_waiter *)_waiter;
	struct rve_scheduler_t *scheduler = NULL;

	ktime_t now;

	now = ktime_get();

	if (DEBUGGER_EN(TIME))
		pr_err("rve job wait in_fence signal use time = %lld\n",
			ktime_to_us(ktime_sub(now, waiter->job->timestamp)));

	scheduler = rve_job_schedule(waiter->job);

	if (scheduler == NULL)
		pr_err("failed to get scheduler, %s(%d)\n", __func__, __LINE__);

	kfree(waiter);
}
#endif

int rve_job_config_by_user_ctx(struct rve_user_ctx_t *user_ctx)
{
	struct rve_pending_ctx_manager *ctx_manager;
	struct rve_internal_ctx_t *ctx;
	int ret = 0;
	unsigned long flags;

	ctx_manager = rve_drvdata->pend_ctx_manager;

	ctx = rve_internal_ctx_lookup(ctx_manager, user_ctx->id);
	if (IS_ERR_OR_NULL(ctx)) {
		pr_err("can not find internal ctx from id[%d]", user_ctx->id);
		return -EINVAL;
	}

	spin_lock_irqsave(&ctx->lock, flags);

	if (ctx->is_running) {
		pr_err("can not re-config when ctx is running");
		spin_unlock_irqrestore(&ctx->lock, flags);
		return -EFAULT;
	}

	spin_unlock_irqrestore(&ctx->lock, flags);

	/* TODO: user cmd_num */
	user_ctx->cmd_num = 1;

	if (ctx->regcmd_data == NULL) {
		ctx->regcmd_data = kmalloc_array(user_ctx->cmd_num,
			sizeof(struct rve_cmd_reg_array_t), GFP_KERNEL);
		if (ctx->regcmd_data == NULL) {
			pr_err("regcmd_data alloc error!\n");
			return -ENOMEM;
		}
	}

	if (unlikely(copy_from_user(ctx->regcmd_data,
					u64_to_user_ptr(user_ctx->regcmd_data),
				    sizeof(struct rve_cmd_reg_array_t) * user_ctx->cmd_num))) {
		pr_err("regcmd_data copy_from_user failed\n");
		ret = -EFAULT;

		goto err_free_regcmd_data;
	}

	ctx->sync_mode = user_ctx->sync_mode;
	ctx->cmd_num = user_ctx->cmd_num;
	ctx->priority = user_ctx->priority;
	ctx->in_fence_fd = user_ctx->in_fence_fd;

	/* TODO: cmd addr */

	return ret;

err_free_regcmd_data:
	kfree(ctx->regcmd_data);
	return ret;
}

int rve_job_commit_by_user_ctx(struct rve_user_ctx_t *user_ctx)
{
	struct rve_pending_ctx_manager *ctx_manager;
	struct rve_internal_ctx_t *ctx;
	int ret = 0;
	unsigned long flags;
	int i;

	ctx_manager = rve_drvdata->pend_ctx_manager;

	ctx = rve_internal_ctx_lookup(ctx_manager, user_ctx->id);
	if (IS_ERR_OR_NULL(ctx)) {
		pr_err("can not find internal ctx from id[%d]", user_ctx->id);
		return -EINVAL;
	}

	spin_lock_irqsave(&ctx->lock, flags);

	if (ctx->is_running) {
		pr_err("can not re-config when ctx is running");
		spin_unlock_irqrestore(&ctx->lock, flags);
		return -EFAULT;
	}

	/* Reset */
	ctx->finished_job_count = 0;
	ctx->running_job_count = 0;
	ctx->is_running = true;
	ctx->disable_auto_cancel = user_ctx->disable_auto_cancel;

	ctx->sync_mode = user_ctx->sync_mode;
	if (ctx->sync_mode == 0)
		ctx->sync_mode = RVE_SYNC;

	spin_unlock_irqrestore(&ctx->lock, flags);

	for (i = 0; i < ctx->cmd_num; i++) {
		ret = rve_job_commit(ctx);
		if (ret < 0) {
			pr_err("rve_job_commit failed, i = %d\n", i);
			return -EFAULT;
		}

		ctx->running_job_count++;
	}

	user_ctx->out_fence_fd = ctx->out_fence_fd;

	if (unlikely(copy_to_user(u64_to_user_ptr(user_ctx->regcmd_data),
				  ctx->regcmd_data,
				  sizeof(struct rve_cmd_reg_array_t) * ctx->cmd_num))) {
		pr_err("ctx->regcmd_data copy_to_user failed\n");
		return -EFAULT;
	}

	if (!ctx->disable_auto_cancel && ctx->sync_mode == RVE_SYNC)
		kref_put(&ctx->refcount, rve_internal_ctx_kref_release);

	return ret;
}

int rve_job_cancel_by_user_ctx(uint32_t ctx_id)
{
	struct rve_pending_ctx_manager *ctx_manager;
	struct rve_internal_ctx_t *ctx;
	int ret = 0;

	ctx_manager = rve_drvdata->pend_ctx_manager;

	ctx = rve_internal_ctx_lookup(ctx_manager, ctx_id);
	if (IS_ERR_OR_NULL(ctx)) {
		pr_err("can not find internal ctx from id[%d]", ctx_id);
		return -EINVAL;
	}

	kref_put(&ctx->refcount, rve_internal_ctx_kref_release);

	return ret;
}

int rve_job_commit(struct rve_internal_ctx_t *ctx)
{
	struct rve_job *job = NULL;
	struct rve_scheduler_t *scheduler = NULL;
#ifdef CONFIG_SYNC_FILE
	struct dma_fence *in_fence;
#endif
	int ret = 0;

	job = rve_job_alloc(ctx);
	if (!job) {
		pr_err("failed to alloc rve job!\n");
		return -ENOMEM;
	}

	if (ctx->sync_mode == RVE_ASYNC) {
#ifdef CONFIG_SYNC_FILE
		job->flags |= RVE_ASYNC;

		if (!ctx->out_fence) {
			ret = rve_out_fence_alloc(job);
			if (ret) {
				rve_job_free(job);
				return ret;
			}
		}

		ctx->out_fence = job->out_fence;

		ctx->out_fence_fd = rve_out_fence_get_fd(job);

		if (ctx->out_fence_fd < 0)
			pr_err("out fence get fd failed");

		if (DEBUGGER_EN(MSG))
			pr_info("in_fence_fd = %d", ctx->in_fence_fd);

		/* if input fence is valiable */
		if (ctx->in_fence_fd > 0) {
			in_fence = rve_get_input_fence(
				ctx->in_fence_fd);
			if (!in_fence) {
				pr_err("%s: failed to get input dma_fence\n",
					 __func__);
				rve_job_free(job);
				return ret;
			}

			/* close input fence fd */
			close_fd(ctx->in_fence_fd);

			ret = dma_fence_get_status(in_fence);
			/* ret = 1: fence has been signaled */
			if (ret == 1) {
				scheduler = rve_job_schedule(job);

				if (scheduler == NULL) {
					pr_err("failed to get scheduler, %s(%d)\n",
						 __func__, __LINE__);
					goto invalid_job;
				}
				/* if input fence is valid */
			} else if (ret == 0) {
				ret = rve_add_dma_fence_callback(job,
					in_fence, rve_job_input_fence_signaled);
				if (ret < 0) {
					pr_err("%s: failed to add fence callback\n",
						 __func__);
					rve_job_free(job);
					return ret;
				}
			} else {
				pr_err("%s: fence status error\n", __func__);
				rve_job_free(job);
				return ret;
			}
		} else {
			scheduler = rve_job_schedule(job);

			if (scheduler == NULL) {
				pr_err("failed to get scheduler, %s(%d)\n",
					 __func__, __LINE__);
				goto invalid_job;
			}
		}

		return ret;
#else
		pr_err("can not support ASYNC mode, please enable CONFIG_SYNC_FILE");
		return -EFAULT;
#endif

	/* RVE_SYNC: wait until job finish */
	} else if (ctx->sync_mode == RVE_SYNC) {
		scheduler = rve_job_schedule(job);

		if (scheduler == NULL) {
			pr_err("failed to get scheduler, %s(%d)\n", __func__,
				 __LINE__);
			goto invalid_job;
		}

		ret = job->ret;
		if (ret < 0) {
			pr_err("some error on job, %s(%d)\n", __func__,
				 __LINE__);
			goto running_job_abort;
		}

		ret = rve_job_wait(job);
		if (ret < 0)
			goto running_job_abort;

		rve_job_cleanup(job);
	}
	return ret;

invalid_job:
	rve_job_abort_invalid(job);
	return ret;

/* only used by SYNC mode */
running_job_abort:
	rve_job_abort_running(job);
	return ret;
}

struct rve_internal_ctx_t *
rve_internal_ctx_lookup(struct rve_pending_ctx_manager *ctx_manager, uint32_t id)
{
	struct rve_internal_ctx_t *ctx = NULL;
	unsigned long flags;

	spin_lock_irqsave(&ctx_manager->lock, flags);

	ctx = idr_find(&ctx_manager->ctx_id_idr, id);

	spin_unlock_irqrestore(&ctx_manager->lock, flags);

	if (ctx == NULL)
		pr_err("can not find internal ctx from id[%d]", id);

	return ctx;
}

/*
 * Called at driver close to release the internal ctx's id references.
 */
static int rve_internal_ctx_free_remove_idr_cb(int id, void *ptr, void *data)
{
	struct rve_internal_ctx_t *ctx = ptr;

	idr_remove(&rve_drvdata->pend_ctx_manager->ctx_id_idr, ctx->id);
	kfree(ctx);

	return 0;
}

static int rve_internal_ctx_free_remove_idr(struct rve_internal_ctx_t *ctx)
{
	struct rve_pending_ctx_manager *ctx_manager;
	unsigned long flags;

	ctx_manager = rve_drvdata->pend_ctx_manager;

	spin_lock_irqsave(&ctx_manager->lock, flags);

	ctx_manager->ctx_count--;
	idr_remove(&ctx_manager->ctx_id_idr, ctx->id);

	spin_unlock_irqrestore(&ctx_manager->lock, flags);

	kfree(ctx);

	return 0;
}

int rve_internal_ctx_signal(struct rve_job *job)
{
	struct rve_internal_ctx_t *ctx;
	struct rve_scheduler_t *scheduler;
	int finished_job_count;
	unsigned long flags;

	scheduler = rve_job_get_scheduler(job);
	if (scheduler == NULL) {
		pr_err("failed to get scheduler, %s(%d)\n", __func__, __LINE__);
		return -EFAULT;
	}

	ctx = rve_job_get_internal_ctx(job);
	if (IS_ERR_OR_NULL(ctx)) {
		pr_err("can not find internal ctx");
		return -EINVAL;
	}

	ctx->regcmd_data = job->regcmd_data;

	spin_lock_irqsave(&ctx->lock, flags);

	finished_job_count = ++ctx->finished_job_count;

	spin_unlock_irqrestore(&ctx->lock, flags);

	if (finished_job_count >= ctx->cmd_num) {
#ifdef CONFIG_SYNC_FILE
		if (ctx->out_fence)
			dma_fence_signal(ctx->out_fence);
#endif

		job->flags |= RVE_JOB_DONE;

		wake_up(&scheduler->job_done_wq);

		spin_lock_irqsave(&ctx->lock, flags);

		ctx->is_running = false;
		ctx->out_fence = NULL;

		spin_unlock_irqrestore(&ctx->lock, flags);

		if (job->flags & RVE_ASYNC) {
			rve_job_cleanup(job);
			if (!ctx->disable_auto_cancel)
				kref_put(&ctx->refcount, rve_internal_ctx_kref_release);
		}
	}

	return 0;
}

int rve_internal_ctx_alloc_to_get_idr_id(struct rve_session *session)
{
	struct rve_pending_ctx_manager *ctx_manager;
	struct rve_internal_ctx_t *ctx;
	unsigned long flags;

	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
	if (ctx == NULL) {
		pr_err("can not kzalloc for rve_pending_ctx_manager\n");
		return -ENOMEM;
	}

	ctx_manager = rve_drvdata->pend_ctx_manager;
	if (ctx_manager == NULL) {
		pr_err("rve_pending_ctx_manager is null!\n");
		goto failed;
	}

	spin_lock_init(&ctx->lock);

	/*
	 * Get the user-visible handle using idr. Preload and perform
	 * allocation under our spinlock.
	 */

	idr_preload(GFP_KERNEL);

	spin_lock_irqsave(&ctx_manager->lock, flags);

	ctx->id = idr_alloc(&ctx_manager->ctx_id_idr, ctx, 1, 0, GFP_ATOMIC);
	if (ctx->id < 0) {
		pr_err("idr_alloc failed");
		spin_unlock_irqrestore(&ctx_manager->lock, flags);
		goto failed;
	}

	ctx_manager->ctx_count++;

	ctx->debug_info.pid = current->pid;
	ctx->debug_info.timestamp = ktime_get();
	ctx->session = session;

	spin_unlock_irqrestore(&ctx_manager->lock, flags);

	idr_preload_end();

	ctx->regcmd_data = NULL;

	kref_init(&ctx->refcount);

	return ctx->id;

failed:
	kfree(ctx);
	return -EFAULT;
}

void rve_internal_ctx_kref_release(struct kref *ref)
{
	struct rve_internal_ctx_t *ctx;
	struct rve_scheduler_t *scheduler = NULL;
	struct rve_job *job_pos, *job_q, *job;
	int i;
	bool need_reset = false;
	unsigned long flags;
	ktime_t now = ktime_get();

	ctx = container_of(ref, struct rve_internal_ctx_t, refcount);

	spin_lock_irqsave(&ctx->lock, flags);
	if (!ctx->is_running || ctx->finished_job_count >= ctx->cmd_num) {
		spin_unlock_irqrestore(&ctx->lock, flags);
		goto free_ctx;
	}
	spin_unlock_irqrestore(&ctx->lock, flags);

	for (i = 0; i < rve_drvdata->num_of_scheduler; i++) {
		scheduler = rve_drvdata->scheduler[i];

		spin_lock_irqsave(&scheduler->irq_lock, flags);

		list_for_each_entry_safe(job_pos, job_q, &scheduler->todo_list, head) {
			if (ctx->id == job_pos->ctx->id) {
				job = job_pos;
				list_del_init(&job_pos->head);

				scheduler->job_count--;
			}
		}

		/* for load */
		if (scheduler->running_job) {
			job = scheduler->running_job;

			if (job->ctx->id == ctx->id) {
				scheduler->running_job = NULL;
				scheduler->timer.busy_time += ktime_us_delta(now, job->hw_recoder_time);
				need_reset = true;
			}
		}

		spin_unlock_irqrestore(&scheduler->irq_lock, flags);

		if (need_reset) {
			pr_err("reset core[%d] by user cancel", scheduler->core);
			scheduler->ops->soft_reset(scheduler);

			rve_job_finish_and_next(job, 0);
		}
	}

free_ctx:
	kfree(ctx->regcmd_data);
	rve_internal_ctx_free_remove_idr(ctx);
}

int rve_ctx_manager_init(struct rve_pending_ctx_manager **ctx_manager_session)
{
	struct rve_pending_ctx_manager *ctx_manager = NULL;

	*ctx_manager_session = kzalloc(sizeof(struct rve_pending_ctx_manager), GFP_KERNEL);
	if (*ctx_manager_session == NULL) {
		pr_err("can not kzalloc for rve_pending_ctx_manager\n");
		return -ENOMEM;
	}

	ctx_manager = *ctx_manager_session;

	spin_lock_init(&ctx_manager->lock);

	idr_init_base(&ctx_manager->ctx_id_idr, 1);

	return 0;
}

int rve_ctx_manager_remove(struct rve_pending_ctx_manager **ctx_manager_session)
{
	struct rve_pending_ctx_manager *ctx_manager = *ctx_manager_session;
	unsigned long flags;

	spin_lock_irqsave(&ctx_manager->lock, flags);

	idr_for_each(&ctx_manager->ctx_id_idr, &rve_internal_ctx_free_remove_idr_cb, ctx_manager);
	idr_destroy(&ctx_manager->ctx_id_idr);

	spin_unlock_irqrestore(&ctx_manager->lock, flags);

	kfree(*ctx_manager_session);

	*ctx_manager_session = NULL;

	return 0;
}