// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) Rockchip Electronics Co., Ltd. * Author: Felix Zeng */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifndef FPGA_PLATFORM #include #endif #include "rknpu_ioctl.h" #include "rknpu_reset.h" #include "rknpu_fence.h" #include "rknpu_drv.h" #include "rknpu_gem.h" #include "rknpu_devfreq.h" #include "rknpu_iommu.h" #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM #include #include #include #include #include "rknpu_gem.h" #endif #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP #include #include "rknpu_mem.h" #endif #define POWER_DOWN_FREQ 200000000 #define NPU_MMU_DISABLED_POLL_PERIOD_US 1000 #define NPU_MMU_DISABLED_POLL_TIMEOUT_US 20000 static int bypass_irq_handler; module_param(bypass_irq_handler, int, 0644); MODULE_PARM_DESC(bypass_irq_handler, "bypass RKNPU irq handler if set it to 1, disabled by default"); static int bypass_soft_reset; module_param(bypass_soft_reset, int, 0644); MODULE_PARM_DESC(bypass_soft_reset, "bypass RKNPU soft reset if set it to 1, disabled by default"); static const struct rknpu_irqs_data rknpu_irqs[] = { { "npu_irq", rknpu_core0_irq_handler } }; static const struct rknpu_irqs_data rk3576_npu_irqs[] = { { "npu0_irq", rknpu_core0_irq_handler }, { "npu1_irq", rknpu_core1_irq_handler } }; static const struct rknpu_irqs_data rk3588_npu_irqs[] = { { "npu0_irq", rknpu_core0_irq_handler }, { "npu1_irq", rknpu_core1_irq_handler }, { "npu2_irq", rknpu_core2_irq_handler } }; static const struct rknpu_amount_data rknpu_old_top_amount = { .offset_clr_all = 0x8010, .offset_dt_wr = 0x8034, .offset_dt_rd = 0x8038, .offset_wt_rd = 0x803c, }; static const struct rknpu_amount_data rknpu_top_amount = { .offset_clr_all = 0x2210, .offset_dt_wr = 0x2234, .offset_dt_rd = 0x2238, .offset_wt_rd = 0x223c }; static const struct rknpu_amount_data rknpu_core_amount = { .offset_clr_all = 0x2410, .offset_dt_wr = 0x2434, .offset_dt_rd = 0x2438, .offset_wt_rd = 0x243c, }; static void rk3576_state_init(struct rknpu_device *rknpu_dev) { void __iomem *rknpu_core_base = rknpu_dev->base[0]; writel(0x1, rknpu_core_base + 0x10); writel(0, rknpu_core_base + 0x1004); writel(0x80000000, rknpu_core_base + 0x1024); writel(1, rknpu_core_base + 0x1004); writel(0x80000000, rknpu_core_base + 0x1024); writel(0x1e, rknpu_core_base + 0x1004); } static int rk3576_cache_sgt_init(struct rknpu_device *rknpu_dev) { struct sg_table *sgt = NULL; struct scatterlist *sgl = NULL; uint64_t block_size_kb[4] = { 448, 64, 448, 64 }; uint64_t block_offset_kb[4] = { 0, 896, 448, 960 }; int core_num = rknpu_dev->config->num_irqs; int ret = 0, i = 0, j = 0; for (i = 0; i < core_num; i++) { sgt = kzalloc(sizeof(struct sg_table), GFP_KERNEL); if (!sgt) goto out_free_table; ret = sg_alloc_table(sgt, core_num, GFP_KERNEL); if (ret) { kfree(sgt); goto out_free_table; } rknpu_dev->cache_sgt[i] = sgt; for_each_sgtable_sg(sgt, sgl, j) { sg_set_page(sgl, NULL, block_size_kb[i * core_num + j] * 1024, block_offset_kb[i * core_num + j] * 1024); } } return 0; out_free_table: for (i = 0; i < core_num; i++) { if (rknpu_dev->cache_sgt[i]) { sg_free_table(rknpu_dev->cache_sgt[i]); kfree(rknpu_dev->cache_sgt[i]); rknpu_dev->cache_sgt[i] = NULL; } } return ret; } static const struct rknpu_config rk356x_rknpu_config = { .bw_priority_addr = 0xfe180008, .bw_priority_length = 0x10, .dma_mask = DMA_BIT_MASK(32), .pc_data_amount_scale = 1, .pc_task_number_bits = 12, .pc_task_number_mask = 0xfff, .pc_task_status_offset = 0x3c, .pc_dma_ctrl = 0, .irqs = rknpu_irqs, .num_irqs = ARRAY_SIZE(rknpu_irqs), .nbuf_phyaddr = 0, .nbuf_size = 0, .max_submit_number = (1 << 12) - 1, .core_mask = 0x1, .amount_top = &rknpu_old_top_amount, .amount_core = NULL, .state_init = NULL, .cache_sgt_init = NULL, }; static const struct rknpu_config rk3588_rknpu_config = { .bw_priority_addr = 0x0, .bw_priority_length = 0x0, .dma_mask = DMA_BIT_MASK(40), .pc_data_amount_scale = 2, .pc_task_number_bits = 12, .pc_task_number_mask = 0xfff, .pc_task_status_offset = 0x3c, .pc_dma_ctrl = 0, .irqs = rk3588_npu_irqs, .num_irqs = ARRAY_SIZE(rk3588_npu_irqs), .nbuf_phyaddr = 0, .nbuf_size = 0, .max_submit_number = (1 << 12) - 1, .core_mask = 0x7, .amount_top = NULL, .amount_core = NULL, .state_init = NULL, .cache_sgt_init = NULL, }; static const struct rknpu_config rk3583_rknpu_config = { .bw_priority_addr = 0x0, .bw_priority_length = 0x0, .dma_mask = DMA_BIT_MASK(40), .pc_data_amount_scale = 2, .pc_task_number_bits = 12, .pc_task_number_mask = 0xfff, .pc_task_status_offset = 0x3c, .pc_dma_ctrl = 0, .irqs = rk3588_npu_irqs, .num_irqs = 2, .nbuf_phyaddr = 0, .nbuf_size = 0, .max_submit_number = (1 << 12) - 1, .core_mask = 0x3, .amount_top = NULL, .amount_core = NULL, .state_init = NULL, .cache_sgt_init = NULL, }; static const struct rknpu_config rv1106_rknpu_config = { .bw_priority_addr = 0x0, .bw_priority_length = 0x0, .dma_mask = DMA_BIT_MASK(32), .pc_data_amount_scale = 2, .pc_task_number_bits = 16, .pc_task_number_mask = 0xffff, .pc_task_status_offset = 0x3c, .pc_dma_ctrl = 0, .irqs = rknpu_irqs, .num_irqs = ARRAY_SIZE(rknpu_irqs), .nbuf_phyaddr = 0, .nbuf_size = 0, .max_submit_number = (1 << 16) - 1, .core_mask = 0x1, .amount_top = &rknpu_old_top_amount, .amount_core = NULL, .state_init = NULL, .cache_sgt_init = NULL, }; static const struct rknpu_config rk3562_rknpu_config = { .bw_priority_addr = 0x0, .bw_priority_length = 0x0, .dma_mask = DMA_BIT_MASK(40), .pc_data_amount_scale = 2, .pc_task_number_bits = 16, .pc_task_number_mask = 0xffff, .pc_task_status_offset = 0x48, .pc_dma_ctrl = 1, .irqs = rknpu_irqs, .num_irqs = ARRAY_SIZE(rknpu_irqs), .nbuf_phyaddr = 0xfe400000, .nbuf_size = 256 * 1024, .max_submit_number = (1 << 16) - 1, .core_mask = 0x1, .amount_top = &rknpu_old_top_amount, .amount_core = NULL, .state_init = NULL, .cache_sgt_init = NULL, }; static const struct rknpu_config rk3576_rknpu_config = { .bw_priority_addr = 0x0, .bw_priority_length = 0x0, .dma_mask = DMA_BIT_MASK(40), .pc_data_amount_scale = 2, .pc_task_number_bits = 16, .pc_task_number_mask = 0xffff, .pc_task_status_offset = 0x48, .pc_dma_ctrl = 1, .irqs = rk3576_npu_irqs, .num_irqs = ARRAY_SIZE(rk3576_npu_irqs), .nbuf_phyaddr = 0x3fe80000, .nbuf_size = 1024 * 1024, .max_submit_number = (1 << 16) - 1, .core_mask = 0x3, .amount_top = &rknpu_top_amount, .amount_core = &rknpu_core_amount, .state_init = rk3576_state_init, .cache_sgt_init = rk3576_cache_sgt_init, }; /* driver probe and init */ static const struct of_device_id rknpu_of_match[] = { { .compatible = "rockchip,rknpu", .data = &rk356x_rknpu_config, }, { .compatible = "rockchip,rk3568-rknpu", .data = &rk356x_rknpu_config, }, { .compatible = "rockchip,rk3588-rknpu", .data = &rk3588_rknpu_config, }, { .compatible = "rockchip,rv1106-rknpu", .data = &rv1106_rknpu_config, }, { .compatible = "rockchip,rk3562-rknpu", .data = &rk3562_rknpu_config, }, { .compatible = "rockchip,rk3576-rknpu", .data = &rk3576_rknpu_config, }, {}, }; static int rknpu_get_drv_version(uint32_t *version) { *version = RKNPU_GET_DRV_VERSION_CODE(DRIVER_MAJOR, DRIVER_MINOR, DRIVER_PATCHLEVEL); return 0; } static int rknpu_power_on(struct rknpu_device *rknpu_dev); static int rknpu_power_off(struct rknpu_device *rknpu_dev); static void rknpu_power_off_delay_work(struct work_struct *power_off_work) { int ret = 0; struct rknpu_device *rknpu_dev = container_of(to_delayed_work(power_off_work), struct rknpu_device, power_off_work); mutex_lock(&rknpu_dev->power_lock); if (atomic_dec_if_positive(&rknpu_dev->power_refcount) == 0) { ret = rknpu_power_off(rknpu_dev); if (ret) atomic_inc(&rknpu_dev->power_refcount); } mutex_unlock(&rknpu_dev->power_lock); if (ret) rknpu_power_put_delay(rknpu_dev); } int rknpu_power_get(struct rknpu_device *rknpu_dev) { int ret = 0; mutex_lock(&rknpu_dev->power_lock); if (atomic_inc_return(&rknpu_dev->power_refcount) == 1) ret = rknpu_power_on(rknpu_dev); mutex_unlock(&rknpu_dev->power_lock); return ret; } int rknpu_power_put(struct rknpu_device *rknpu_dev) { int ret = 0; mutex_lock(&rknpu_dev->power_lock); if (atomic_dec_if_positive(&rknpu_dev->power_refcount) == 0) { ret = rknpu_power_off(rknpu_dev); if (ret) atomic_inc(&rknpu_dev->power_refcount); } mutex_unlock(&rknpu_dev->power_lock); if (ret) rknpu_power_put_delay(rknpu_dev); return ret; } int rknpu_power_put_delay(struct rknpu_device *rknpu_dev) { if (rknpu_dev->power_put_delay == 0) return rknpu_power_put(rknpu_dev); mutex_lock(&rknpu_dev->power_lock); if (atomic_read(&rknpu_dev->power_refcount) == 1) queue_delayed_work( rknpu_dev->power_off_wq, &rknpu_dev->power_off_work, msecs_to_jiffies(rknpu_dev->power_put_delay)); else atomic_dec_if_positive(&rknpu_dev->power_refcount); mutex_unlock(&rknpu_dev->power_lock); return 0; } static int rknpu_action(struct rknpu_device *rknpu_dev, struct rknpu_action *args) { int ret = -EINVAL; switch (args->flags) { case RKNPU_GET_HW_VERSION: ret = rknpu_get_hw_version(rknpu_dev, &args->value); break; case RKNPU_GET_DRV_VERSION: ret = rknpu_get_drv_version(&args->value); break; case RKNPU_GET_FREQ: #ifndef FPGA_PLATFORM args->value = clk_get_rate(rknpu_dev->clks[0].clk); #endif ret = 0; break; case RKNPU_SET_FREQ: break; case RKNPU_GET_VOLT: #ifndef FPGA_PLATFORM args->value = regulator_get_voltage(rknpu_dev->vdd); #endif ret = 0; break; case RKNPU_SET_VOLT: break; case RKNPU_ACT_RESET: ret = rknpu_soft_reset(rknpu_dev); break; case RKNPU_GET_BW_PRIORITY: ret = rknpu_get_bw_priority(rknpu_dev, &args->value, NULL, NULL); break; case RKNPU_SET_BW_PRIORITY: ret = rknpu_set_bw_priority(rknpu_dev, args->value, 0, 0); break; case RKNPU_GET_BW_EXPECT: ret = rknpu_get_bw_priority(rknpu_dev, NULL, &args->value, NULL); break; case RKNPU_SET_BW_EXPECT: ret = rknpu_set_bw_priority(rknpu_dev, 0, args->value, 0); break; case RKNPU_GET_BW_TW: ret = rknpu_get_bw_priority(rknpu_dev, NULL, NULL, &args->value); break; case RKNPU_SET_BW_TW: ret = rknpu_set_bw_priority(rknpu_dev, 0, 0, args->value); break; case RKNPU_ACT_CLR_TOTAL_RW_AMOUNT: ret = rknpu_clear_rw_amount(rknpu_dev); break; case RKNPU_GET_DT_WR_AMOUNT: ret = rknpu_get_rw_amount(rknpu_dev, &args->value, NULL, NULL); break; case RKNPU_GET_DT_RD_AMOUNT: ret = rknpu_get_rw_amount(rknpu_dev, NULL, &args->value, NULL); break; case RKNPU_GET_WT_RD_AMOUNT: ret = rknpu_get_rw_amount(rknpu_dev, NULL, NULL, &args->value); break; case RKNPU_GET_TOTAL_RW_AMOUNT: ret = rknpu_get_total_rw_amount(rknpu_dev, &args->value); break; case RKNPU_GET_IOMMU_EN: args->value = rknpu_dev->iommu_en; ret = 0; break; case RKNPU_SET_PROC_NICE: set_user_nice(current, *(int32_t *)&args->value); ret = 0; break; case RKNPU_GET_TOTAL_SRAM_SIZE: if (rknpu_dev->sram_mm) args->value = rknpu_dev->sram_mm->total_chunks * rknpu_dev->sram_mm->chunk_size; else args->value = 0; ret = 0; break; case RKNPU_GET_FREE_SRAM_SIZE: if (rknpu_dev->sram_mm) args->value = rknpu_dev->sram_mm->free_chunks * rknpu_dev->sram_mm->chunk_size; else args->value = 0; ret = 0; break; case RKNPU_GET_IOMMU_DOMAIN_ID: args->value = rknpu_dev->iommu_domain_id; ret = 0; break; case RKNPU_SET_IOMMU_DOMAIN_ID: { ret = rknpu_iommu_domain_get_and_switch( rknpu_dev, *(int32_t *)&args->value); if (ret) break; rknpu_iommu_domain_put(rknpu_dev); break; } default: ret = -EINVAL; break; } return ret; } #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP static int rknpu_open(struct inode *inode, struct file *file) { struct rknpu_device *rknpu_dev = container_of(file->private_data, struct rknpu_device, miscdev); struct rknpu_session *session = NULL; session = kzalloc(sizeof(*session), GFP_KERNEL); if (!session) { LOG_ERROR("rknpu session alloc failed\n"); return -ENOMEM; } session->rknpu_dev = rknpu_dev; INIT_LIST_HEAD(&session->list); file->private_data = (void *)session; return nonseekable_open(inode, file); } static int rknpu_release(struct inode *inode, struct file *file) { struct rknpu_mem_object *entry; struct rknpu_session *session = file->private_data; struct rknpu_device *rknpu_dev = session->rknpu_dev; LIST_HEAD(local_list); spin_lock(&rknpu_dev->lock); list_replace_init(&session->list, &local_list); file->private_data = NULL; spin_unlock(&rknpu_dev->lock); while (!list_empty(&local_list)) { entry = list_first_entry(&local_list, struct rknpu_mem_object, head); LOG_DEBUG( "Fd close free rknpu_obj: %#llx, rknpu_obj->dma_addr: %#llx\n", (__u64)(uintptr_t)entry, (__u64)entry->dma_addr); vunmap(entry->kv_addr); entry->kv_addr = NULL; if (!entry->owner) dma_buf_put(entry->dmabuf); list_del(&entry->head); kfree(entry); } kfree(session); return 0; } static int rknpu_action_ioctl(struct rknpu_device *rknpu_dev, unsigned long data) { struct rknpu_action args; int ret = -EINVAL; if (unlikely(copy_from_user(&args, (struct rknpu_action *)data, sizeof(struct rknpu_action)))) { LOG_ERROR("%s: copy_from_user failed\n", __func__); ret = -EFAULT; return ret; } ret = rknpu_action(rknpu_dev, &args); if (unlikely(copy_to_user((struct rknpu_action *)data, &args, sizeof(struct rknpu_action)))) { LOG_ERROR("%s: copy_to_user failed\n", __func__); ret = -EFAULT; return ret; } return ret; } static long rknpu_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { long ret = -EINVAL; struct rknpu_device *rknpu_dev = NULL; if (!file->private_data) return -EINVAL; rknpu_dev = ((struct rknpu_session *)file->private_data)->rknpu_dev; rknpu_power_get(rknpu_dev); switch (_IOC_NR(cmd)) { case RKNPU_ACTION: ret = rknpu_action_ioctl(rknpu_dev, arg); break; case RKNPU_SUBMIT: ret = rknpu_submit_ioctl(rknpu_dev, arg); break; case RKNPU_MEM_CREATE: ret = rknpu_mem_create_ioctl(rknpu_dev, file, cmd, arg); break; case RKNPU_MEM_MAP: break; case RKNPU_MEM_DESTROY: ret = rknpu_mem_destroy_ioctl(rknpu_dev, file, arg); break; case RKNPU_MEM_SYNC: ret = rknpu_mem_sync_ioctl(rknpu_dev, arg); break; default: break; } rknpu_power_put_delay(rknpu_dev); return ret; } const struct file_operations rknpu_fops = { .owner = THIS_MODULE, .open = rknpu_open, .release = rknpu_release, .unlocked_ioctl = rknpu_ioctl, #ifdef CONFIG_COMPAT .compat_ioctl = rknpu_ioctl, #endif }; #endif #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM #if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE static const struct vm_operations_struct rknpu_gem_vm_ops = { .fault = rknpu_gem_fault, .open = drm_gem_vm_open, .close = drm_gem_vm_close, }; #endif static int rknpu_action_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { struct rknpu_device *rknpu_dev = dev_get_drvdata(dev->dev); return rknpu_action(rknpu_dev, (struct rknpu_action *)data); } #define RKNPU_IOCTL(func) \ static int __##func(struct drm_device *dev, void *data, \ struct drm_file *file_priv) \ { \ struct rknpu_device *rknpu_dev = dev_get_drvdata(dev->dev); \ int ret = -EINVAL; \ rknpu_power_get(rknpu_dev); \ ret = func(dev, data, file_priv); \ rknpu_power_put_delay(rknpu_dev); \ return ret; \ } RKNPU_IOCTL(rknpu_action_ioctl); RKNPU_IOCTL(rknpu_submit_ioctl); RKNPU_IOCTL(rknpu_gem_create_ioctl); RKNPU_IOCTL(rknpu_gem_map_ioctl); RKNPU_IOCTL(rknpu_gem_destroy_ioctl); RKNPU_IOCTL(rknpu_gem_sync_ioctl); static const struct drm_ioctl_desc rknpu_ioctls[] = { DRM_IOCTL_DEF_DRV(RKNPU_ACTION, __rknpu_action_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(RKNPU_SUBMIT, __rknpu_submit_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(RKNPU_MEM_CREATE, __rknpu_gem_create_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(RKNPU_MEM_MAP, __rknpu_gem_map_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(RKNPU_MEM_DESTROY, __rknpu_gem_destroy_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(RKNPU_MEM_SYNC, __rknpu_gem_sync_ioctl, DRM_RENDER_ALLOW), }; #if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE DEFINE_DRM_GEM_FOPS(rknpu_drm_driver_fops); #else static const struct file_operations rknpu_drm_driver_fops = { .owner = THIS_MODULE, .open = drm_open, .mmap = rknpu_gem_mmap, .poll = drm_poll, .read = drm_read, .unlocked_ioctl = drm_ioctl, #ifdef CONFIG_COMPAT .compat_ioctl = drm_compat_ioctl, #endif .release = drm_release, .llseek = noop_llseek, }; #endif static struct drm_driver rknpu_drm_driver = { #if KERNEL_VERSION(5, 4, 0) <= LINUX_VERSION_CODE .driver_features = DRIVER_GEM | DRIVER_RENDER, #else .driver_features = DRIVER_GEM | DRIVER_PRIME | DRIVER_RENDER, #endif #if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE .gem_free_object_unlocked = rknpu_gem_free_object, .gem_vm_ops = &rknpu_gem_vm_ops, .dumb_destroy = drm_gem_dumb_destroy, .gem_prime_export = drm_gem_prime_export, .gem_prime_get_sg_table = rknpu_gem_prime_get_sg_table, .gem_prime_vmap = rknpu_gem_prime_vmap, .gem_prime_vunmap = rknpu_gem_prime_vunmap, #endif .dumb_create = rknpu_gem_dumb_create, #if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE .dumb_map_offset = rknpu_gem_dumb_map_offset, #else .dumb_map_offset = drm_gem_dumb_map_offset, #endif .prime_handle_to_fd = drm_gem_prime_handle_to_fd, .prime_fd_to_handle = drm_gem_prime_fd_to_handle, #if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE .gem_prime_import = rknpu_gem_prime_import, #else .gem_prime_import = drm_gem_prime_import, #endif .gem_prime_import_sg_table = rknpu_gem_prime_import_sg_table, #if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE .gem_prime_mmap = drm_gem_prime_mmap, #else .gem_prime_mmap = rknpu_gem_prime_mmap, #endif .ioctls = rknpu_ioctls, .num_ioctls = ARRAY_SIZE(rknpu_ioctls), .fops = &rknpu_drm_driver_fops, .name = DRIVER_NAME, .desc = DRIVER_DESC, .date = DRIVER_DATE, .major = DRIVER_MAJOR, .minor = DRIVER_MINOR, .patchlevel = DRIVER_PATCHLEVEL, }; #endif static enum hrtimer_restart hrtimer_handler(struct hrtimer *timer) { struct rknpu_device *rknpu_dev = container_of(timer, struct rknpu_device, timer); struct rknpu_subcore_data *subcore_data = NULL; struct rknpu_job *job = NULL; ktime_t now; unsigned long flags; int i; for (i = 0; i < rknpu_dev->config->num_irqs; i++) { subcore_data = &rknpu_dev->subcore_datas[i]; spin_lock_irqsave(&rknpu_dev->irq_lock, flags); job = subcore_data->job; if (job) { now = ktime_get(); subcore_data->timer.busy_time += ktime_sub(now, job->hw_recoder_time); job->hw_recoder_time = now; } subcore_data->timer.total_busy_time = subcore_data->timer.busy_time; subcore_data->timer.busy_time = 0; spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); } hrtimer_forward_now(timer, rknpu_dev->kt); return HRTIMER_RESTART; } static void rknpu_init_timer(struct rknpu_device *rknpu_dev) { rknpu_dev->kt = ktime_set(0, RKNPU_LOAD_INTERVAL); hrtimer_init(&rknpu_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); rknpu_dev->timer.function = hrtimer_handler; hrtimer_start(&rknpu_dev->timer, rknpu_dev->kt, HRTIMER_MODE_REL); } static void rknpu_cancel_timer(struct rknpu_device *rknpu_dev) { hrtimer_cancel(&rknpu_dev->timer); } static bool rknpu_is_iommu_enable(struct device *dev) { struct device_node *iommu = NULL; iommu = of_parse_phandle(dev->of_node, "iommus", 0); if (!iommu) { LOG_DEV_INFO( dev, "rknpu iommu device-tree entry not found!, using non-iommu mode\n"); return false; } if (!of_device_is_available(iommu)) { LOG_DEV_INFO(dev, "rknpu iommu is disabled, using non-iommu mode\n"); of_node_put(iommu); return false; } of_node_put(iommu); LOG_DEV_INFO(dev, "rknpu iommu is enabled, using iommu mode\n"); return true; } #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM static int drm_fake_dev_register(struct rknpu_device *rknpu_dev) { const struct platform_device_info rknpu_dev_info = { .name = "rknpu_dev", .id = PLATFORM_DEVID_AUTO, .dma_mask = rknpu_dev->config->dma_mask, }; struct platform_device *pdev = NULL; int ret = -EINVAL; pdev = platform_device_register_full(&rknpu_dev_info); if (pdev) { ret = of_dma_configure(&pdev->dev, NULL, true); if (ret) { platform_device_unregister(pdev); pdev = NULL; } } rknpu_dev->fake_dev = pdev ? &pdev->dev : NULL; return ret; } static void drm_fake_dev_unregister(struct rknpu_device *rknpu_dev) { struct platform_device *pdev = NULL; if (!rknpu_dev->fake_dev) return; pdev = to_platform_device(rknpu_dev->fake_dev); platform_device_unregister(pdev); } static int rknpu_drm_probe(struct rknpu_device *rknpu_dev) { struct device *dev = rknpu_dev->dev; struct drm_device *drm_dev = NULL; int ret = -EINVAL; drm_dev = drm_dev_alloc(&rknpu_drm_driver, dev); if (IS_ERR(drm_dev)) return PTR_ERR(drm_dev); /* register the DRM device */ ret = drm_dev_register(drm_dev, 0); if (ret < 0) goto err_free_drm; drm_dev->dev_private = rknpu_dev; rknpu_dev->drm_dev = drm_dev; drm_fake_dev_register(rknpu_dev); return 0; err_free_drm: #if KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE drm_dev_put(drm_dev); #else drm_dev_unref(drm_dev); #endif return ret; } static void rknpu_drm_remove(struct rknpu_device *rknpu_dev) { struct drm_device *drm_dev = rknpu_dev->drm_dev; drm_fake_dev_unregister(rknpu_dev); drm_dev_unregister(drm_dev); #if KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE drm_dev_put(drm_dev); #else drm_dev_unref(drm_dev); #endif } #endif static int rknpu_power_on(struct rknpu_device *rknpu_dev) { struct device *dev = rknpu_dev->dev; int ret = -EINVAL; #ifndef FPGA_PLATFORM if (rknpu_dev->vdd) { ret = regulator_enable(rknpu_dev->vdd); if (ret) { LOG_DEV_ERROR( dev, "failed to enable vdd reg for rknpu, ret: %d\n", ret); return ret; } } if (rknpu_dev->mem) { ret = regulator_enable(rknpu_dev->mem); if (ret) { LOG_DEV_ERROR( dev, "failed to enable mem reg for rknpu, ret: %d\n", ret); return ret; } } #endif ret = clk_bulk_prepare_enable(rknpu_dev->num_clks, rknpu_dev->clks); if (ret) { LOG_DEV_ERROR(dev, "failed to enable clk for rknpu, ret: %d\n", ret); return ret; } #ifndef FPGA_PLATFORM rknpu_devfreq_lock(rknpu_dev); #endif if (rknpu_dev->multiple_domains) { if (rknpu_dev->genpd_dev_npu0) { #if KERNEL_VERSION(5, 5, 0) < LINUX_VERSION_CODE ret = pm_runtime_resume_and_get( rknpu_dev->genpd_dev_npu0); #else ret = pm_runtime_get_sync(rknpu_dev->genpd_dev_npu0); #endif if (ret < 0) { LOG_DEV_ERROR( dev, "failed to get pm runtime for npu0, ret: %d\n", ret); goto out; } } if (rknpu_dev->genpd_dev_npu1) { #if KERNEL_VERSION(5, 5, 0) < LINUX_VERSION_CODE ret = pm_runtime_resume_and_get( rknpu_dev->genpd_dev_npu1); #else ret = pm_runtime_get_sync(rknpu_dev->genpd_dev_npu1); #endif if (ret < 0) { LOG_DEV_ERROR( dev, "failed to get pm runtime for npu1, ret: %d\n", ret); goto out; } } if (rknpu_dev->genpd_dev_npu2) { #if KERNEL_VERSION(5, 5, 0) < LINUX_VERSION_CODE ret = pm_runtime_resume_and_get( rknpu_dev->genpd_dev_npu2); #else ret = pm_runtime_get_sync(rknpu_dev->genpd_dev_npu2); #endif if (ret < 0) { LOG_DEV_ERROR( dev, "failed to get pm runtime for npu2, ret: %d\n", ret); goto out; } } } ret = pm_runtime_get_sync(dev); if (ret < 0) { LOG_DEV_ERROR(dev, "failed to get pm runtime for rknpu, ret: %d\n", ret); } if (rknpu_dev->config->state_init != NULL) rknpu_dev->config->state_init(rknpu_dev); out: #ifndef FPGA_PLATFORM rknpu_devfreq_unlock(rknpu_dev); #endif return ret; } static int rknpu_power_off(struct rknpu_device *rknpu_dev) { struct device *dev = rknpu_dev->dev; #ifndef FPGA_PLATFORM int ret; bool val; rknpu_devfreq_lock(rknpu_dev); #endif pm_runtime_put_sync(dev); if (rknpu_dev->multiple_domains) { #ifndef FPGA_PLATFORM /* * Because IOMMU's runtime suspend callback is asynchronous, * So it may be executed after the NPU is turned off after PD/CLK/VD, * and the runtime suspend callback has a register access. * If the PD/VD/CLK is closed, the register access will crash. * As a workaround, it's safe to close pd stuff until iommu disabled. * If pm runtime framework can handle this issue in the future, remove * this. */ ret = readx_poll_timeout(rockchip_iommu_is_enabled, dev, val, !val, NPU_MMU_DISABLED_POLL_PERIOD_US, NPU_MMU_DISABLED_POLL_TIMEOUT_US); if (ret) { LOG_DEV_ERROR(dev, "iommu still enabled\n"); pm_runtime_get_sync(dev); rknpu_devfreq_unlock(rknpu_dev); return ret; } #else if (rknpu_dev->iommu_en) msleep(20); #endif if (rknpu_dev->genpd_dev_npu2) pm_runtime_put_sync(rknpu_dev->genpd_dev_npu2); if (rknpu_dev->genpd_dev_npu1) pm_runtime_put_sync(rknpu_dev->genpd_dev_npu1); if (rknpu_dev->genpd_dev_npu0) pm_runtime_put_sync(rknpu_dev->genpd_dev_npu0); } #ifndef FPGA_PLATFORM rknpu_devfreq_unlock(rknpu_dev); #endif clk_bulk_disable_unprepare(rknpu_dev->num_clks, rknpu_dev->clks); #ifndef FPGA_PLATFORM if (rknpu_dev->vdd) regulator_disable(rknpu_dev->vdd); if (rknpu_dev->mem) regulator_disable(rknpu_dev->mem); #endif return 0; } static int rknpu_register_irq(struct platform_device *pdev, struct rknpu_device *rknpu_dev) { const struct rknpu_config *config = rknpu_dev->config; struct device *dev = &pdev->dev; #if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE struct resource *res; #endif int i, ret, irq; #if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE res = platform_get_resource_byname(pdev, IORESOURCE_IRQ, config->irqs[0].name); if (res) { /* there are irq names in dts */ for (i = 0; i < config->num_irqs; i++) { irq = platform_get_irq_byname(pdev, config->irqs[i].name); if (irq < 0) { LOG_DEV_ERROR(dev, "no npu %s in dts\n", config->irqs[i].name); return irq; } ret = devm_request_irq(dev, irq, config->irqs[i].irq_hdl, IRQF_SHARED, dev_name(dev), rknpu_dev); if (ret < 0) { LOG_DEV_ERROR(dev, "request %s failed: %d\n", config->irqs[i].name, ret); return ret; } } } else { /* no irq names in dts */ irq = platform_get_irq(pdev, 0); if (irq < 0) { LOG_DEV_ERROR(dev, "no npu irq in dts\n"); return irq; } ret = devm_request_irq(dev, irq, rknpu_core0_irq_handler, IRQF_SHARED, dev_name(dev), rknpu_dev); if (ret < 0) { LOG_DEV_ERROR(dev, "request irq failed: %d\n", ret); return ret; } } #else /* there are irq names in dts */ for (i = 0; i < config->num_irqs; i++) { irq = platform_get_irq_byname(pdev, config->irqs[i].name); if (irq < 0) { irq = platform_get_irq(pdev, i); if (irq < 0) { LOG_DEV_ERROR(dev, "no npu %s in dts\n", config->irqs[i].name); return irq; } } ret = devm_request_irq(dev, irq, config->irqs[i].irq_hdl, IRQF_SHARED, dev_name(dev), rknpu_dev); if (ret < 0) { LOG_DEV_ERROR(dev, "request %s failed: %d\n", config->irqs[i].name, ret); return ret; } } #endif return 0; } static int rknpu_find_sram_resource(struct rknpu_device *rknpu_dev) { struct device *dev = rknpu_dev->dev; struct device_node *sram_node = NULL; struct resource sram_res; uint32_t sram_size = 0; int ret = -EINVAL; /* get sram device node */ sram_node = of_parse_phandle(dev->of_node, "rockchip,sram", 0); rknpu_dev->sram_size = 0; if (!sram_node) return -EINVAL; /* get sram start and size */ ret = of_address_to_resource(sram_node, 0, &sram_res); of_node_put(sram_node); if (ret) return ret; /* check sram start and size is PAGE_SIZE align */ rknpu_dev->sram_start = round_up(sram_res.start, PAGE_SIZE); rknpu_dev->sram_end = round_down( sram_res.start + resource_size(&sram_res), PAGE_SIZE); if (rknpu_dev->sram_end <= rknpu_dev->sram_start) { LOG_DEV_WARN( dev, "invalid sram resource, sram start %pa, sram end %pa\n", &rknpu_dev->sram_start, &rknpu_dev->sram_end); return -EINVAL; } sram_size = rknpu_dev->sram_end - rknpu_dev->sram_start; rknpu_dev->sram_base_io = devm_ioremap(dev, rknpu_dev->sram_start, sram_size); if (IS_ERR(rknpu_dev->sram_base_io)) { LOG_DEV_ERROR(dev, "failed to remap sram base io!\n"); rknpu_dev->sram_base_io = NULL; } rknpu_dev->sram_size = sram_size; LOG_DEV_INFO(dev, "sram region: [%pa, %pa), sram size: %#x\n", &rknpu_dev->sram_start, &rknpu_dev->sram_end, rknpu_dev->sram_size); return 0; } static int rknpu_find_nbuf_resource(struct rknpu_device *rknpu_dev) { struct device *dev = rknpu_dev->dev; if (rknpu_dev->config->nbuf_size == 0) return -EINVAL; rknpu_dev->nbuf_start = rknpu_dev->config->nbuf_phyaddr; rknpu_dev->nbuf_size = rknpu_dev->config->nbuf_size; rknpu_dev->nbuf_base_io = devm_ioremap(dev, rknpu_dev->nbuf_start, rknpu_dev->nbuf_size); if (IS_ERR(rknpu_dev->nbuf_base_io)) { LOG_DEV_ERROR(dev, "failed to remap nbuf base io!\n"); rknpu_dev->nbuf_base_io = NULL; } rknpu_dev->nbuf_end = rknpu_dev->nbuf_start + rknpu_dev->nbuf_size; LOG_DEV_INFO(dev, "nbuf region: [%pa, %pa), nbuf size: %#x\n", &rknpu_dev->nbuf_start, &rknpu_dev->nbuf_end, rknpu_dev->nbuf_size); return 0; } static int rknpu_get_invalid_core_mask(struct device *dev) { int ret = 0; u8 invalid_core_mask = 0; if (of_property_match_string(dev->of_node, "nvmem-cell-names", "cores") >= 0) { ret = rockchip_nvmem_cell_read_u8(dev->of_node, "cores", &invalid_core_mask); /* The default valid npu cores for RK3583 are core0 and core1 */ invalid_core_mask |= RKNPU_CORE2_MASK; if (ret) { LOG_DEV_ERROR( dev, "failed to get specification_serial_number\n"); return invalid_core_mask; } } return (int)invalid_core_mask; } static int rknpu_probe(struct platform_device *pdev) { struct resource *res = NULL; struct rknpu_device *rknpu_dev = NULL; struct device *dev = &pdev->dev; struct device *virt_dev = NULL; const struct of_device_id *match = NULL; const struct rknpu_config *config = NULL; int ret = -EINVAL, i = 0; if (!pdev->dev.of_node) { LOG_DEV_ERROR(dev, "rknpu device-tree data is missing!\n"); return -ENODEV; } match = of_match_device(rknpu_of_match, dev); if (!match) { LOG_DEV_ERROR(dev, "rknpu device-tree entry is missing!\n"); return -ENODEV; } rknpu_dev = devm_kzalloc(dev, sizeof(*rknpu_dev), GFP_KERNEL); if (!rknpu_dev) { LOG_DEV_ERROR(dev, "failed to allocate rknpu device!\n"); return -ENOMEM; } config = of_device_get_match_data(dev); if (!config) return -EINVAL; if (match->data == (void *)&rk3588_rknpu_config) { int invalid_core_mask = rknpu_get_invalid_core_mask(dev); /* The default valid npu cores for RK3583 are core0 and core1 */ if (invalid_core_mask & RKNPU_CORE2_MASK) { if ((invalid_core_mask & RKNPU_CORE0_MASK) || (invalid_core_mask & RKNPU_CORE1_MASK)) { LOG_DEV_ERROR( dev, "rknpu core invalid, invalid core mask: %#x\n", invalid_core_mask); return -ENODEV; } config = &rk3583_rknpu_config; } } rknpu_dev->config = config; rknpu_dev->dev = dev; dev_set_drvdata(dev, rknpu_dev); rknpu_dev->iommu_en = rknpu_is_iommu_enable(dev); if (rknpu_dev->iommu_en) { rknpu_dev->iommu_group = iommu_group_get(dev); if (!rknpu_dev->iommu_group) return -EINVAL; } else { /* Initialize reserved memory resources */ ret = of_reserved_mem_device_init(dev); if (!ret) { LOG_DEV_INFO( dev, "initialize reserved memory for rknpu device!\n"); } } rknpu_dev->bypass_irq_handler = bypass_irq_handler; rknpu_dev->bypass_soft_reset = bypass_soft_reset; rknpu_reset_get(rknpu_dev); rknpu_dev->num_clks = devm_clk_bulk_get_all(dev, &rknpu_dev->clks); if (rknpu_dev->num_clks < 1) { LOG_DEV_ERROR(dev, "failed to get clk source for rknpu\n"); #ifndef FPGA_PLATFORM return -ENODEV; #endif } #ifndef FPGA_PLATFORM rknpu_dev->vdd = devm_regulator_get_optional(dev, "rknpu"); if (IS_ERR(rknpu_dev->vdd)) { if (PTR_ERR(rknpu_dev->vdd) != -ENODEV) { ret = PTR_ERR(rknpu_dev->vdd); LOG_DEV_ERROR( dev, "failed to get vdd regulator for rknpu: %d\n", ret); return ret; } rknpu_dev->vdd = NULL; } rknpu_dev->mem = devm_regulator_get_optional(dev, "mem"); if (IS_ERR(rknpu_dev->mem)) { if (PTR_ERR(rknpu_dev->mem) != -ENODEV) { ret = PTR_ERR(rknpu_dev->mem); LOG_DEV_ERROR( dev, "failed to get mem regulator for rknpu: %d\n", ret); return ret; } rknpu_dev->mem = NULL; } #endif spin_lock_init(&rknpu_dev->lock); spin_lock_init(&rknpu_dev->irq_lock); mutex_init(&rknpu_dev->power_lock); mutex_init(&rknpu_dev->reset_lock); mutex_init(&rknpu_dev->domain_lock); for (i = 0; i < config->num_irqs; i++) { INIT_LIST_HEAD(&rknpu_dev->subcore_datas[i].todo_list); init_waitqueue_head(&rknpu_dev->subcore_datas[i].job_done_wq); rknpu_dev->subcore_datas[i].task_num = 0; res = platform_get_resource(pdev, IORESOURCE_MEM, i); if (!res) { LOG_DEV_ERROR( dev, "failed to get memory resource for rknpu\n"); return -ENXIO; } rknpu_dev->base[i] = devm_ioremap_resource(dev, res); if (PTR_ERR(rknpu_dev->base[i]) == -EBUSY) { rknpu_dev->base[i] = devm_ioremap(dev, res->start, resource_size(res)); } if (IS_ERR(rknpu_dev->base[i])) { LOG_DEV_ERROR(dev, "failed to remap register for rknpu\n"); return PTR_ERR(rknpu_dev->base[i]); } } if (config->bw_priority_length > 0) { rknpu_dev->bw_priority_base = devm_ioremap(dev, config->bw_priority_addr, config->bw_priority_length); if (IS_ERR(rknpu_dev->bw_priority_base)) { LOG_DEV_ERROR( rknpu_dev->dev, "failed to remap bw priority register for rknpu\n"); rknpu_dev->bw_priority_base = NULL; } } if (!rknpu_dev->bypass_irq_handler) { ret = rknpu_register_irq(pdev, rknpu_dev); if (ret) return ret; } else { LOG_DEV_WARN(dev, "bypass irq handler!\n"); } #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM ret = rknpu_drm_probe(rknpu_dev); if (ret) { LOG_DEV_ERROR(dev, "failed to probe device for rknpu\n"); return ret; } #endif #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP rknpu_dev->miscdev.minor = MISC_DYNAMIC_MINOR; rknpu_dev->miscdev.name = "rknpu"; rknpu_dev->miscdev.fops = &rknpu_fops; ret = misc_register(&rknpu_dev->miscdev); if (ret) { LOG_DEV_ERROR(dev, "cannot register miscdev (%d)\n", ret); return ret; } rknpu_dev->heap = rk_dma_heap_find("rk-dma-heap-cma"); if (!rknpu_dev->heap) { LOG_DEV_ERROR(dev, "failed to find cma heap\n"); return -ENOMEM; } rk_dma_heap_set_dev(dev); LOG_DEV_INFO(dev, "Initialized %s: v%d.%d.%d for %s\n", DRIVER_DESC, DRIVER_MAJOR, DRIVER_MINOR, DRIVER_PATCHLEVEL, DRIVER_DATE); #endif #ifdef CONFIG_ROCKCHIP_RKNPU_FENCE ret = rknpu_fence_context_alloc(rknpu_dev); if (ret) { LOG_DEV_ERROR(dev, "failed to allocate fence context for rknpu\n"); goto err_remove_drv; } #endif platform_set_drvdata(pdev, rknpu_dev); pm_runtime_enable(dev); if (of_count_phandle_with_args(dev->of_node, "power-domains", "#power-domain-cells") > 1) { virt_dev = dev_pm_domain_attach_by_name(dev, "npu0"); if (!IS_ERR(virt_dev)) rknpu_dev->genpd_dev_npu0 = virt_dev; virt_dev = dev_pm_domain_attach_by_name(dev, "npu1"); if (!IS_ERR(virt_dev)) rknpu_dev->genpd_dev_npu1 = virt_dev; if (config->num_irqs > 2) { virt_dev = dev_pm_domain_attach_by_name(dev, "npu2"); if (!IS_ERR(virt_dev)) rknpu_dev->genpd_dev_npu2 = virt_dev; } rknpu_dev->multiple_domains = true; } ret = rknpu_power_on(rknpu_dev); if (ret) goto err_remove_drv; #ifndef FPGA_PLATFORM rknpu_devfreq_init(rknpu_dev); #endif // set default power put delay to 3s rknpu_dev->power_put_delay = 3000; rknpu_dev->power_off_wq = create_freezable_workqueue("rknpu_power_off_wq"); if (!rknpu_dev->power_off_wq) { LOG_DEV_ERROR(dev, "rknpu couldn't create power_off workqueue"); ret = -ENOMEM; goto err_devfreq_remove; } INIT_DEFERRABLE_WORK(&rknpu_dev->power_off_work, rknpu_power_off_delay_work); if (IS_ENABLED(CONFIG_NO_GKI) && IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_dev->iommu_en) { if (!rknpu_find_sram_resource(rknpu_dev)) { ret = rknpu_mm_create(rknpu_dev->sram_size, PAGE_SIZE, &rknpu_dev->sram_mm); if (ret != 0) goto err_remove_wq; } else { LOG_DEV_WARN(dev, "could not find sram resource!\n"); } } if (IS_ENABLED(CONFIG_NO_GKI) && rknpu_dev->iommu_en && rknpu_dev->config->nbuf_size > 0) { rknpu_find_nbuf_resource(rknpu_dev); if (rknpu_dev->config->cache_sgt_init != NULL) rknpu_dev->config->cache_sgt_init(rknpu_dev); } if (rknpu_dev->iommu_en) rknpu_iommu_init_domain(rknpu_dev); rknpu_power_off(rknpu_dev); atomic_set(&rknpu_dev->power_refcount, 0); atomic_set(&rknpu_dev->cmdline_power_refcount, 0); atomic_set(&rknpu_dev->iommu_domain_refcount, 0); rknpu_debugger_init(rknpu_dev); rknpu_init_timer(rknpu_dev); return 0; err_remove_wq: destroy_workqueue(rknpu_dev->power_off_wq); err_devfreq_remove: #ifndef FPGA_PLATFORM rknpu_devfreq_remove(rknpu_dev); #endif err_remove_drv: #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM rknpu_drm_remove(rknpu_dev); #endif #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP misc_deregister(&(rknpu_dev->miscdev)); #endif return ret; } static int rknpu_remove(struct platform_device *pdev) { struct rknpu_device *rknpu_dev = platform_get_drvdata(pdev); int i = 0; cancel_delayed_work_sync(&rknpu_dev->power_off_work); destroy_workqueue(rknpu_dev->power_off_wq); rknpu_debugger_remove(rknpu_dev); rknpu_cancel_timer(rknpu_dev); if (rknpu_dev->config->cache_sgt_init != NULL) { for (i = 0; i < RKNPU_CACHE_SG_TABLE_NUM; i++) { if (rknpu_dev->cache_sgt[i]) { sg_free_table(rknpu_dev->cache_sgt[i]); kfree(rknpu_dev->cache_sgt[i]); rknpu_dev->cache_sgt[i] = NULL; } } } for (i = 0; i < rknpu_dev->config->num_irqs; i++) { WARN_ON(rknpu_dev->subcore_datas[i].job); WARN_ON(!list_empty(&rknpu_dev->subcore_datas[i].todo_list)); } if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_dev->sram_mm) rknpu_mm_destroy(rknpu_dev->sram_mm); if (rknpu_dev->iommu_en) { rknpu_iommu_free_domains(rknpu_dev); iommu_group_put(rknpu_dev->iommu_group); } #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM rknpu_drm_remove(rknpu_dev); #endif #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP misc_deregister(&(rknpu_dev->miscdev)); #endif #ifndef FPGA_PLATFORM rknpu_devfreq_remove(rknpu_dev); #endif mutex_lock(&rknpu_dev->power_lock); if (atomic_read(&rknpu_dev->power_refcount) > 0) rknpu_power_off(rknpu_dev); mutex_unlock(&rknpu_dev->power_lock); if (rknpu_dev->multiple_domains) { if (rknpu_dev->genpd_dev_npu0) dev_pm_domain_detach(rknpu_dev->genpd_dev_npu0, true); if (rknpu_dev->genpd_dev_npu1) dev_pm_domain_detach(rknpu_dev->genpd_dev_npu1, true); if (rknpu_dev->genpd_dev_npu2) dev_pm_domain_detach(rknpu_dev->genpd_dev_npu2, true); } pm_runtime_disable(&pdev->dev); return 0; } #ifndef FPGA_PLATFORM #ifdef CONFIG_PM_SLEEP static int rknpu_suspend(struct device *dev) { struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); rknpu_power_get(rknpu_dev); return pm_runtime_force_suspend(dev); } static int rknpu_resume(struct device *dev) { struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); rknpu_power_put_delay(rknpu_dev); return pm_runtime_force_resume(dev); } #endif static int rknpu_runtime_suspend(struct device *dev) { return rknpu_devfreq_runtime_suspend(dev); } static int rknpu_runtime_resume(struct device *dev) { return rknpu_devfreq_runtime_resume(dev); } static const struct dev_pm_ops rknpu_pm_ops = { SET_SYSTEM_SLEEP_PM_OPS(rknpu_suspend, rknpu_resume) SET_RUNTIME_PM_OPS( rknpu_runtime_suspend, rknpu_runtime_resume, NULL) }; #endif static struct platform_driver rknpu_driver = { .probe = rknpu_probe, .remove = rknpu_remove, .driver = { .owner = THIS_MODULE, .name = "RKNPU", #ifndef FPGA_PLATFORM .pm = &rknpu_pm_ops, #endif .of_match_table = of_match_ptr(rknpu_of_match), }, }; static int rknpu_init(void) { return platform_driver_register(&rknpu_driver); } static void rknpu_exit(void) { platform_driver_unregister(&rknpu_driver); } late_initcall(rknpu_init); module_exit(rknpu_exit); MODULE_DESCRIPTION("RKNPU driver"); MODULE_AUTHOR("Felix Zeng "); MODULE_ALIAS("rockchip-rknpu"); MODULE_LICENSE("GPL v2"); MODULE_VERSION(RKNPU_GET_DRV_VERSION_STRING(DRIVER_MAJOR, DRIVER_MINOR, DRIVER_PATCHLEVEL)); #if KERNEL_VERSION(5, 16, 0) < LINUX_VERSION_CODE MODULE_IMPORT_NS(DMA_BUF); #endif