mirrors
/
linux-stable
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
1
0
Fork 0
Code Issues Releases Wiki Activity

drm-misc-next for 6.8: 

UAPI Changes:
   - drm: Introduce CLOSE_FB ioctl
   - drm/dp-mst: Documentation for the PATH property
   - fdinfo: Do not align to a MB if the size is larger than 1MiB
   - virtio-gpu: add explicit virtgpu context debug name
 
 Cross-subsystem Changes:
   - dma-buf: Add dma_fence_timestamp helper
 
 Core Changes:
   - client: Do not acquire module reference
   - edid: split out drm_eld, add SAD helpers
   - format-helper: Cache format conversion buffers
   - sched: Move from a kthread to a workqueue, rename some internal
     functions to make it clearer, implement dynamic job-flow control
   - gpuvm: Provide more features to handle GEM objects
   - tests: Remove slow kunit tests
 
 Driver Changes:
   - ivpu: Update FW API, new debugfs file, a new NOP job submission test
     mode, improve suspend/resume, PM improvements, MMU PT optimizations,
     firmware profiling frequency support, support for uncached buffers,
     switch to gem shmem helpers, replace kthread with threaded
     interrupts
   - panfrost: PM improvements
   - qaic: Allow to run with a single MSI, support host/device time
     synchronization, misc improvements
   - simplefb: Support memory-regions, support power-domains
   - ssd130x: Unitialized variable fixes
   - omapdrm: dma-fence lockdep annotation fix
   - tidss: dma-fence lockdep annotation fix
   - v3d: Support BCM2712 (RaspberryPi5), Support fdinfo and gputop
   - panel:
     - edp: Support AUO B116XTN02, BOE NT116WHM-N21,836X2, NV116WHM-N49
       V8.0, plus a whole bunch of panels used on Mediatek chromebooks.
 -----BEGIN PGP SIGNATURE-----
 
 iHUEABYKAB0WIQRcEzekXsqa64kGDp7j7w1vZxhRxQUCZVc02QAKCRDj7w1vZxhR
 xRXbAPsHQkuUz2n1XWx/8uExPCn+PmS5Lg6IcbtvqpfuFrGt1QEA+DaZf2yfekFj
 FgY66UExyddH/2LDHzpKa0WJk6l6vQw=
 =1HSK
 -----END PGP SIGNATURE-----

Merge tag 'drm-misc-next-2023-11-17' of git://anongit.freedesktop.org/drm/drm-misc into drm-next

drm-misc-next for 6.8:

UAPI Changes:
  - drm: Introduce CLOSE_FB ioctl
  - drm/dp-mst: Documentation for the PATH property
  - fdinfo: Do not align to a MB if the size is larger than 1MiB
  - virtio-gpu: add explicit virtgpu context debug name

Cross-subsystem Changes:
  - dma-buf: Add dma_fence_timestamp helper

Core Changes:
  - client: Do not acquire module reference
  - edid: split out drm_eld, add SAD helpers
  - format-helper: Cache format conversion buffers
  - sched: Move from a kthread to a workqueue, rename some internal
    functions to make it clearer, implement dynamic job-flow control
  - gpuvm: Provide more features to handle GEM objects
  - tests: Remove slow kunit tests

Driver Changes:
  - ivpu: Update FW API, new debugfs file, a new NOP job submission test
    mode, improve suspend/resume, PM improvements, MMU PT optimizations,
    firmware profiling frequency support, support for uncached buffers,
    switch to gem shmem helpers, replace kthread with threaded
    interrupts
  - panfrost: PM improvements
  - qaic: Allow to run with a single MSI, support host/device time
    synchronization, misc improvements
  - simplefb: Support memory-regions, support power-domains
  - ssd130x: Unitialized variable fixes
  - omapdrm: dma-fence lockdep annotation fix
  - tidss: dma-fence lockdep annotation fix
  - v3d: Support BCM2712 (RaspberryPi5), Support fdinfo and gputop
  - panel:
    - edp: Support AUO B116XTN02, BOE NT116WHM-N21,836X2, NV116WHM-N49
      V8.0, plus a whole bunch of panels used on Mediatek chromebooks.

Note that the one missing s-o-b for 0da611a870 ("dma-buf: add
dma_fence_timestamp helper") has been supplied here, and rebasing the
entire tree with upsetting committers didn't seem worth the trouble:
https://lore.kernel.org/dri-devel/ce94020e-a7d4-4799-b87d-fbea7b14a268@gmail.com/

Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
From: Maxime Ripard <mripard@redhat.com>
Link: https://patchwork.freedesktop.org/patch/msgid/y4awn5vcfy2lr2hpauo7rc4nfpnc6kksr7btmnwaz7zk63pwoi@gwwef5iqpzva
This commit is contained in:
Daniel Vetter 2023-11-20 09:50:08 +01:00
parent 98b1cc82c4 3b434a3445
commit c79b972eb8
144 changed files with 5775 additions and 4377 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
Documentation/accel/qaic/aic100.rst
View File

@ -36,8 +36,9 @@ AIC100 DID (0xa100).
AIC100 does not implement FLR (function level reset).
AIC100 implements MSI but does not implement MSI-X. AIC100 requires 17 MSIs to
operate (1 for MHI, 16 for the DMA Bridge).
AIC100 implements MSI but does not implement MSI-X. AIC100 prefers 17 MSIs to
operate (1 for MHI, 16 for the DMA Bridge). Falling back to 1 MSI is possible in
scenarios where reserving 32 MSIs isn't feasible.
As a PCIe device, AIC100 utilizes BARs to provide host interfaces to the device
hardware. AIC100 provides 3, 64-bit BARs.
@ -220,10 +221,14 @@ of the defined channels, and their uses.
+----------------+---------+----------+----------------------------------------+
| QAIC_DEBUG | 18 & 19 | AMSS | Not used. |
+----------------+---------+----------+----------------------------------------+
| QAIC_TIMESYNC | 20 & 21 | SBL/AMSS | Used to synchronize timestamps in the |
| QAIC_TIMESYNC | 20 & 21 | SBL | Used to synchronize timestamps in the |
| | | | device side logs with the host time |
| | | | source. |
+----------------+---------+----------+----------------------------------------+
| QAIC_TIMESYNC | 22 & 23 | AMSS | Used to periodically synchronize |
| _PERIODIC | | | timestamps in the device side logs with|
| | | | the host time source. |
+----------------+---------+----------+----------------------------------------+
DMA Bridge
==========

28
Documentation/accel/qaic/qaic.rst
View File

@ -10,6 +10,9 @@ accelerator products.
Interrupts
==========
IRQ Storm Mitigation
--------------------
While the AIC100 DMA Bridge hardware implements an IRQ storm mitigation
mechanism, it is still possible for an IRQ storm to occur. A storm can happen
if the workload is particularly quick, and the host is responsive. If the host
@ -35,6 +38,26 @@ generates 100k IRQs per second (per /proc/interrupts) is reduced to roughly 64
IRQs over 5 minutes while keeping the host system stable, and having the same
workload throughput performance (within run to run noise variation).
Single MSI Mode
---------------
MultiMSI is not well supported on all systems; virtualized ones even less so
(circa 2023). Between hypervisors masking the PCIe MSI capability structure to
large memory requirements for vIOMMUs (required for supporting MultiMSI), it is
useful to be able to fall back to a single MSI when needed.
To support this fallback, we allow the case where only one MSI is able to be
allocated, and share that one MSI between MHI and the DBCs. The device detects
when only one MSI has been configured and directs the interrupts for the DBCs
to the interrupt normally used for MHI. Unfortunately this means that the
interrupt handlers for every DBC and MHI wake up for every interrupt that
arrives; however, the DBC threaded irq handlers only are started when work to be
done is detected (MHI will always start its threaded handler).
If the DBC is configured to force MSI interrupts, this can circumvent the
software IRQ storm mitigation mentioned above. Since the MSI is shared it is
never disabled, allowing each new entry to the FIFO to trigger a new interrupt.
Neural Network Control (NNC) Protocol
=====================================
@ -178,3 +201,8 @@ overrides this for that call. Default is 5000 (5 seconds).
Sets the polling interval in microseconds (us) when datapath polling is active.
Takes effect at the next polling interval. Default is 100 (100 us).
**timesync_delay_ms (unsigned int)**
Sets the time interval in milliseconds (ms) between two consecutive timesync
operations. Default is 1000 (1000 ms).

1
Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.yaml
View File

@ -17,6 +17,7 @@ properties:
compatible:
enum:
- brcm,2711-v3d
- brcm,2712-v3d
- brcm,7268-v3d
- brcm,7278-v3d

6
Documentation/gpu/drm-kms-helpers.rst
View File

@ -363,6 +363,12 @@ EDID Helper Functions Reference
.. kernel-doc:: drivers/gpu/drm/drm_edid.c
:export:
.. kernel-doc:: include/drm/drm_eld.h
:internal:
.. kernel-doc:: drivers/gpu/drm/drm_eld.c
:export:
SCDC Helper Functions Reference
===============================

6
Documentation/gpu/drm-mm.rst
View File

@ -552,6 +552,12 @@ Overview
.. kernel-doc:: drivers/gpu/drm/scheduler/sched_main.c
:doc: Overview
Flow Control
------------
.. kernel-doc:: drivers/gpu/drm/scheduler/sched_main.c
:doc: Flow Control
Scheduler Function References
-----------------------------

17
Documentation/gpu/todo.rst
View File

@ -621,6 +621,23 @@ Contact: Javier Martinez Canillas <javierm@redhat.com>
Level: Intermediate
Clean up and document former selftests suites
---------------------------------------------
Some KUnit test suites (drm_buddy, drm_cmdline_parser, drm_damage_helper,
drm_format, drm_framebuffer, drm_dp_mst_helper, drm_mm, drm_plane_helper and
drm_rect) are former selftests suites that have been converted over when KUnit
was first introduced.
These suites were fairly undocumented, and with different goals than what unit
tests can be. Trying to identify what each test in these suites actually test
for, whether that makes sense for a unit test, and either remove it if it
doesn't or document it if it does would be of great help.
Contact: Maxime Ripard <mripard@kernel.org>
Level: Intermediate
Enable trinity for DRM
----------------------

9
MAINTAINERS
View File

@ -6503,8 +6503,7 @@ T: git git://anongit.freedesktop.org/drm/drm-misc
F: drivers/gpu/drm/sun4i/sun8i*
DRM DRIVER FOR ARM PL111 CLCD
M: Emma Anholt <emma@anholt.net>
S: Supported
S: Orphan
T: git git://anongit.freedesktop.org/drm/drm-misc
F: drivers/gpu/drm/pl111/
@ -6619,8 +6618,7 @@ F: Documentation/devicetree/bindings/display/panel/himax,hx8394.yaml
F: drivers/gpu/drm/panel/panel-himax-hx8394.c
DRM DRIVER FOR HX8357D PANELS
M: Emma Anholt <emma@anholt.net>
S: Maintained
S: Orphan
T: git git://anongit.freedesktop.org/drm/drm-misc
F: Documentation/devicetree/bindings/display/himax,hx8357d.txt
F: drivers/gpu/drm/tiny/hx8357d.c
@ -7213,8 +7211,8 @@ F: Documentation/devicetree/bindings/display/ti/
F: drivers/gpu/drm/omapdrm/
DRM DRIVERS FOR V3D
M: Emma Anholt <emma@anholt.net>
M: Melissa Wen <mwen@igalia.com>
M: Maíra Canal <mcanal@igalia.com>
S: Supported
T: git git://anongit.freedesktop.org/drm/drm-misc
F: Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.yaml
@ -7222,7 +7220,6 @@ F: drivers/gpu/drm/v3d/
F: include/uapi/drm/v3d_drm.h
DRM DRIVERS FOR VC4
M: Emma Anholt <emma@anholt.net>
M: Maxime Ripard <mripard@kernel.org>
S: Supported
T: git git://github.com/anholt/linux

11
drivers/accel/ivpu/Kconfig
View File

@ -1,16 +1,17 @@
# SPDX-License-Identifier: GPL-2.0-only
config DRM_ACCEL_IVPU
tristate "Intel VPU for Meteor Lake and newer"
tristate "Intel NPU (Neural Processing Unit)"
depends on DRM_ACCEL
depends on X86_64 && !UML
depends on PCI && PCI_MSI
select FW_LOADER
select SHMEM
select DRM_GEM_SHMEM_HELPER
select GENERIC_ALLOCATOR
help
Choose this option if you have a system that has an 14th generation Intel CPU
or newer. VPU stands for Versatile Processing Unit and it's a CPU-integrated
inference accelerator for Computer Vision and Deep Learning applications.
Choose this option if you have a system with an 14th generation
Intel CPU (Meteor Lake) or newer. Intel NPU (formerly called Intel VPU)
is a CPU-integrated inference accelerator for Computer Vision
and Deep Learning applications.
If "M" is selected, the module will be called intel_vpu.

57
drivers/accel/ivpu/ivpu_debugfs.c
View File

@ -14,6 +14,7 @@
#include "ivpu_fw.h"
#include "ivpu_fw_log.h"
#include "ivpu_gem.h"
#include "ivpu_hw.h"
#include "ivpu_jsm_msg.h"
#include "ivpu_pm.h"
@ -115,6 +116,31 @@ static const struct drm_debugfs_info vdev_debugfs_list[] = {
{"reset_pending", reset_pending_show, 0},
};
static ssize_t
dvfs_mode_fops_write(struct file *file, const char __user *user_buf, size_t size, loff_t *pos)
{
struct ivpu_device *vdev = file->private_data;
struct ivpu_fw_info *fw = vdev->fw;
u32 dvfs_mode;
int ret;
ret = kstrtou32_from_user(user_buf, size, 0, &dvfs_mode);
if (ret < 0)
return ret;
fw->dvfs_mode = dvfs_mode;
ivpu_pm_schedule_recovery(vdev);
return size;
}
static const struct file_operations dvfs_mode_fops = {
.owner = THIS_MODULE,
.open = simple_open,
.write = dvfs_mode_fops_write,
};
static int fw_log_show(struct seq_file *s, void *v)
{
struct ivpu_device *vdev = s->private;
@ -151,6 +177,30 @@ static const struct file_operations fw_log_fops = {
.release = single_release,
};
static ssize_t
fw_profiling_freq_fops_write(struct file *file, const char __user *user_buf,
size_t size, loff_t *pos)
{
struct ivpu_device *vdev = file->private_data;
bool enable;
int ret;
ret = kstrtobool_from_user(user_buf, size, &enable);
if (ret < 0)
return ret;
ivpu_hw_profiling_freq_drive(vdev, enable);
ivpu_pm_schedule_recovery(vdev);
return size;
}
static const struct file_operations fw_profiling_freq_fops = {
.owner = THIS_MODULE,
.open = simple_open,
.write = fw_profiling_freq_fops_write,
};
static ssize_t
fw_trace_destination_mask_fops_write(struct file *file, const char __user *user_buf,
size_t size, loff_t *pos)
@ -280,6 +330,9 @@ void ivpu_debugfs_init(struct ivpu_device *vdev)
debugfs_create_file("force_recovery", 0200, debugfs_root, vdev,
&ivpu_force_recovery_fops);
debugfs_create_file("dvfs_mode", 0200, debugfs_root, vdev,
&dvfs_mode_fops);
debugfs_create_file("fw_log", 0644, debugfs_root, vdev,
&fw_log_fops);
debugfs_create_file("fw_trace_destination_mask", 0200, debugfs_root, vdev,
@ -291,4 +344,8 @@ void ivpu_debugfs_init(struct ivpu_device *vdev)
debugfs_create_file("reset_engine", 0200, debugfs_root, vdev,
&ivpu_reset_engine_fops);
if (ivpu_hw_gen(vdev) >= IVPU_HW_40XX)
debugfs_create_file("fw_profiling_freq_drive", 0200,
debugfs_root, vdev, &fw_profiling_freq_fops);
}

49
drivers/accel/ivpu/ivpu_drv.c
View File

@ -31,8 +31,6 @@
__stringify(DRM_IVPU_DRIVER_MINOR) "."
#endif
static const struct drm_driver driver;
static struct lock_class_key submitted_jobs_xa_lock_class_key;
int ivpu_dbg_mask;
@ -41,7 +39,7 @@ MODULE_PARM_DESC(dbg_mask, "Driver debug mask. See IVPU_DBG_* macros.");
int ivpu_test_mode;
module_param_named_unsafe(test_mode, ivpu_test_mode, int, 0644);
MODULE_PARM_DESC(test_mode, "Test mode: 0 - normal operation, 1 - fw unit test, 2 - null hw");
MODULE_PARM_DESC(test_mode, "Test mode mask. See IVPU_TEST_MODE_* macros.");
u8 ivpu_pll_min_ratio;
module_param_named(pll_min_ratio, ivpu_pll_min_ratio, byte, 0644);
@ -93,8 +91,8 @@ static void file_priv_release(struct kref *ref)
ivpu_dbg(vdev, FILE, "file_priv release: ctx %u\n", file_priv->ctx.id);
ivpu_cmdq_release_all(file_priv);
ivpu_bo_remove_all_bos_from_context(&file_priv->ctx);
ivpu_jsm_context_release(vdev, file_priv->ctx.id);
ivpu_bo_remove_all_bos_from_context(vdev, &file_priv->ctx);
ivpu_mmu_user_context_fini(vdev, &file_priv->ctx);
drm_WARN_ON(&vdev->drm, xa_erase_irq(&vdev->context_xa, file_priv->ctx.id) != file_priv);
mutex_destroy(&file_priv->lock);
@ -317,16 +315,14 @@ static int ivpu_wait_for_ready(struct ivpu_device *vdev)
unsigned long timeout;
int ret;
if (ivpu_test_mode == IVPU_TEST_MODE_FW_TEST)
if (ivpu_test_mode & IVPU_TEST_MODE_FW_TEST)
return 0;
ivpu_ipc_consumer_add(vdev, &cons, IVPU_IPC_CHAN_BOOT_MSG);
ivpu_ipc_consumer_add(vdev, &cons, IVPU_IPC_CHAN_BOOT_MSG, NULL);
timeout = jiffies + msecs_to_jiffies(vdev->timeout.boot);
while (1) {
ret = ivpu_ipc_irq_handler(vdev);
if (ret)
break;
ivpu_ipc_irq_handler(vdev, NULL);
ret = ivpu_ipc_receive(vdev, &cons, &ipc_hdr, NULL, 0);
if (ret != -ETIMEDOUT || time_after_eq(jiffies, timeout))
break;
@ -362,7 +358,7 @@ int ivpu_boot(struct ivpu_device *vdev)
int ret;
/* Update boot params located at first 4KB of FW memory */
ivpu_fw_boot_params_setup(vdev, vdev->fw->mem->kvaddr);
ivpu_fw_boot_params_setup(vdev, ivpu_bo_vaddr(vdev->fw->mem));
ret = ivpu_hw_boot_fw(vdev);
if (ret) {
@ -414,7 +410,9 @@ static const struct drm_driver driver = {
.open = ivpu_open,
.postclose = ivpu_postclose,
.gem_prime_import = ivpu_gem_prime_import,
.gem_create_object = ivpu_gem_create_object,
.gem_prime_import_sg_table = drm_gem_shmem_prime_import_sg_table,
.ioctls = ivpu_drm_ioctls,
.num_ioctls = ARRAY_SIZE(ivpu_drm_ioctls),
@ -427,6 +425,13 @@ static const struct drm_driver driver = {
.minor = DRM_IVPU_DRIVER_MINOR,
};
static irqreturn_t ivpu_irq_thread_handler(int irq, void *arg)
{
struct ivpu_device *vdev = arg;
return ivpu_ipc_irq_thread_handler(vdev);
}
static int ivpu_irq_init(struct ivpu_device *vdev)
{
struct pci_dev *pdev = to_pci_dev(vdev->drm.dev);
@ -440,8 +445,8 @@ static int ivpu_irq_init(struct ivpu_device *vdev)
vdev->irq = pci_irq_vector(pdev, 0);
ret = devm_request_irq(vdev->drm.dev, vdev->irq, vdev->hw->ops->irq_handler,
IRQF_NO_AUTOEN, DRIVER_NAME, vdev);
ret = devm_request_threaded_irq(vdev->drm.dev, vdev->irq, vdev->hw->ops->irq_handler,
ivpu_irq_thread_handler, IRQF_NO_AUTOEN, DRIVER_NAME, vdev);
if (ret)
ivpu_err(vdev, "Failed to request an IRQ %d\n", ret);
@ -533,6 +538,11 @@ static int ivpu_dev_init(struct ivpu_device *vdev)
xa_init_flags(&vdev->context_xa, XA_FLAGS_ALLOC);
xa_init_flags(&vdev->submitted_jobs_xa, XA_FLAGS_ALLOC1);
lockdep_set_class(&vdev->submitted_jobs_xa.xa_lock, &submitted_jobs_xa_lock_class_key);
INIT_LIST_HEAD(&vdev->bo_list);
ret = drmm_mutex_init(&vdev->drm, &vdev->bo_list_lock);
if (ret)
goto err_xa_destroy;
ret = ivpu_pci_init(vdev);
if (ret)
@ -550,7 +560,7 @@ static int ivpu_dev_init(struct ivpu_device *vdev)
/* Power up early so the rest of init code can access VPU registers */
ret = ivpu_hw_power_up(vdev);
if (ret)
goto err_xa_destroy;
goto err_power_down;
ret = ivpu_mmu_global_context_init(vdev);
if (ret)
@ -574,20 +584,15 @@ static int ivpu_dev_init(struct ivpu_device *vdev)
ivpu_pm_init(vdev);
ret = ivpu_job_done_thread_init(vdev);
ret = ivpu_boot(vdev);
if (ret)
goto err_ipc_fini;
ret = ivpu_boot(vdev);
if (ret)
goto err_job_done_thread_fini;
ivpu_job_done_consumer_init(vdev);
ivpu_pm_enable(vdev);
return 0;
err_job_done_thread_fini:
ivpu_job_done_thread_fini(vdev);
err_ipc_fini:
ivpu_ipc_fini(vdev);
err_fw_fini:
@ -612,7 +617,7 @@ static void ivpu_dev_fini(struct ivpu_device *vdev)
ivpu_shutdown(vdev);
if (IVPU_WA(d3hot_after_power_off))
pci_set_power_state(to_pci_dev(vdev->drm.dev), PCI_D3hot);
ivpu_job_done_thread_fini(vdev);
ivpu_job_done_consumer_fini(vdev);
ivpu_pm_cancel_recovery(vdev);
ivpu_ipc_fini(vdev);

18
drivers/accel/ivpu/ivpu_drv.h
View File

@ -17,9 +17,10 @@
#include <uapi/drm/ivpu_accel.h>
#include "ivpu_mmu_context.h"
#include "ivpu_ipc.h"
#define DRIVER_NAME "intel_vpu"
#define DRIVER_DESC "Driver for Intel Versatile Processing Unit (VPU)"
#define DRIVER_DESC "Driver for Intel NPU (Neural Processing Unit)"
#define DRIVER_DATE "20230117"
#define PCI_DEVICE_ID_MTL 0x7d1d
@ -88,6 +89,7 @@ struct ivpu_wa_table {
bool d3hot_after_power_off;
bool interrupt_clear_with_0;
bool disable_clock_relinquish;
bool disable_d0i3_msg;
};
struct ivpu_hw_info;
@ -115,8 +117,11 @@ struct ivpu_device {
struct xarray context_xa;
struct xa_limit context_xa_limit;
struct mutex bo_list_lock; /* Protects bo_list */
struct list_head bo_list;
struct xarray submitted_jobs_xa;
struct task_struct *job_done_thread;
struct ivpu_ipc_consumer job_done_consumer;
atomic64_t unique_id_counter;
@ -126,6 +131,7 @@ struct ivpu_device {
int tdr;
int reschedule_suspend;
int autosuspend;
int d0i3_entry_msg;
} timeout;
};
@ -148,9 +154,11 @@ extern u8 ivpu_pll_min_ratio;
extern u8 ivpu_pll_max_ratio;
extern bool ivpu_disable_mmu_cont_pages;
#define IVPU_TEST_MODE_DISABLED 0
#define IVPU_TEST_MODE_FW_TEST 1
#define IVPU_TEST_MODE_NULL_HW 2
#define IVPU_TEST_MODE_FW_TEST BIT(0)
#define IVPU_TEST_MODE_NULL_HW BIT(1)
#define IVPU_TEST_MODE_NULL_SUBMISSION BIT(2)
#define IVPU_TEST_MODE_D0I3_MSG_DISABLE BIT(4)
#define IVPU_TEST_MODE_D0I3_MSG_ENABLE BIT(5)
extern int ivpu_test_mode;
struct ivpu_file_priv *ivpu_file_priv_get(struct ivpu_file_priv *file_priv);

79
drivers/accel/ivpu/ivpu_fw.c
View File

@ -33,12 +33,17 @@
#define ADDR_TO_L2_CACHE_CFG(addr) ((addr) >> 31)
#define IVPU_FW_CHECK_API(vdev, fw_hdr, name, min_major) \
/* Check if FW API is compatible with the driver */
#define IVPU_FW_CHECK_API_COMPAT(vdev, fw_hdr, name, min_major) \
ivpu_fw_check_api(vdev, fw_hdr, #name, \
VPU_##name##_API_VER_INDEX, \
VPU_##name##_API_VER_MAJOR, \
VPU_##name##_API_VER_MINOR, min_major)
/* Check if API version is lower that the given version */
#define IVPU_FW_CHECK_API_VER_LT(vdev, fw_hdr, name, major, minor) \
ivpu_fw_check_api_ver_lt(vdev, fw_hdr, #name, VPU_##name##_API_VER_INDEX, major, minor)
static char *ivpu_firmware;
module_param_named_unsafe(firmware, ivpu_firmware, charp, 0644);
MODULE_PARM_DESC(firmware, "VPU firmware binary in /lib/firmware/..");
@ -105,6 +110,19 @@ ivpu_fw_check_api(struct ivpu_device *vdev, const struct vpu_firmware_header *fw
return 0;
}
static bool
ivpu_fw_check_api_ver_lt(struct ivpu_device *vdev, const struct vpu_firmware_header *fw_hdr,
const char *str, int index, u16 major, u16 minor)
{
u16 fw_major = (u16)(fw_hdr->api_version[index] >> 16);
u16 fw_minor = (u16)(fw_hdr->api_version[index]);
if (fw_major < major || (fw_major == major && fw_minor < minor))
return true;
return false;
}
static int ivpu_fw_parse(struct ivpu_device *vdev)
{
struct ivpu_fw_info *fw = vdev->fw;
@ -164,9 +182,9 @@ static int ivpu_fw_parse(struct ivpu_device *vdev)
ivpu_info(vdev, "Firmware: %s, version: %s", fw->name,
(const char *)fw_hdr + VPU_FW_HEADER_SIZE);
if (IVPU_FW_CHECK_API(vdev, fw_hdr, BOOT, 3))
if (IVPU_FW_CHECK_API_COMPAT(vdev, fw_hdr, BOOT, 3))
return -EINVAL;
if (IVPU_FW_CHECK_API(vdev, fw_hdr, JSM, 3))
if (IVPU_FW_CHECK_API_COMPAT(vdev, fw_hdr, JSM, 3))
return -EINVAL;
fw->runtime_addr = runtime_addr;
@ -182,6 +200,8 @@ static int ivpu_fw_parse(struct ivpu_device *vdev)
fw->trace_destination_mask = VPU_TRACE_DESTINATION_VERBOSE_TRACING;
fw->trace_hw_component_mask = -1;
fw->dvfs_mode = 0;
ivpu_dbg(vdev, FW_BOOT, "Size: file %lu image %u runtime %u shavenn %u\n",
fw->file->size, fw->image_size, fw->runtime_size, fw->shave_nn_size);
ivpu_dbg(vdev, FW_BOOT, "Address: runtime 0x%llx, load 0x%llx, entry point 0x%llx\n",
@ -195,6 +215,24 @@ static void ivpu_fw_release(struct ivpu_device *vdev)
release_firmware(vdev->fw->file);
}
/* Initialize workarounds that depend on FW version */
static void
ivpu_fw_init_wa(struct ivpu_device *vdev)
{
const struct vpu_firmware_header *fw_hdr = (const void *)vdev->fw->file->data;
if (IVPU_FW_CHECK_API_VER_LT(vdev, fw_hdr, BOOT, 3, 17) ||
(ivpu_hw_gen(vdev) > IVPU_HW_37XX) ||
(ivpu_test_mode & IVPU_TEST_MODE_D0I3_MSG_DISABLE))
vdev->wa.disable_d0i3_msg = true;
/* Force enable the feature for testing purposes */
if (ivpu_test_mode & IVPU_TEST_MODE_D0I3_MSG_ENABLE)
vdev->wa.disable_d0i3_msg = false;
IVPU_PRINT_WA(disable_d0i3_msg);
}
static int ivpu_fw_update_global_range(struct ivpu_device *vdev)
{
struct ivpu_fw_info *fw = vdev->fw;
@ -248,7 +286,7 @@ static int ivpu_fw_mem_init(struct ivpu_device *vdev)
if (fw->shave_nn_size) {
fw->mem_shave_nn = ivpu_bo_alloc_internal(vdev, vdev->hw->ranges.shave.start,
fw->shave_nn_size, DRM_IVPU_BO_UNCACHED);
fw->shave_nn_size, DRM_IVPU_BO_WC);
if (!fw->mem_shave_nn) {
ivpu_err(vdev, "Failed to allocate shavenn buffer\n");
ret = -ENOMEM;
@ -297,6 +335,8 @@ int ivpu_fw_init(struct ivpu_device *vdev)
if (ret)
goto err_fw_release;
ivpu_fw_init_wa(vdev);
ret = ivpu_fw_mem_init(vdev);
if (ret)
goto err_fw_release;
@ -422,14 +462,31 @@ static void ivpu_fw_boot_params_print(struct ivpu_device *vdev, struct vpu_boot_
boot_params->punit_telemetry_sram_size);
ivpu_dbg(vdev, FW_BOOT, "boot_params.vpu_telemetry_enable = 0x%x\n",
boot_params->vpu_telemetry_enable);
ivpu_dbg(vdev, FW_BOOT, "boot_params.dvfs_mode = %u\n",
boot_params->dvfs_mode);
ivpu_dbg(vdev, FW_BOOT, "boot_params.d0i3_delayed_entry = %d\n",
boot_params->d0i3_delayed_entry);
ivpu_dbg(vdev, FW_BOOT, "boot_params.d0i3_residency_time_us = %lld\n",
boot_params->d0i3_residency_time_us);
ivpu_dbg(vdev, FW_BOOT, "boot_params.d0i3_entry_vpu_ts = %llu\n",
boot_params->d0i3_entry_vpu_ts);
}
void ivpu_fw_boot_params_setup(struct ivpu_device *vdev, struct vpu_boot_params *boot_params)
{
struct ivpu_bo *ipc_mem_rx = vdev->ipc->mem_rx;
/* In case of warm boot we only have to reset the entrypoint addr */
/* In case of warm boot only update variable params */
if (!ivpu_fw_is_cold_boot(vdev)) {
boot_params->d0i3_residency_time_us =
ktime_us_delta(ktime_get_boottime(), vdev->hw->d0i3_entry_host_ts);
boot_params->d0i3_entry_vpu_ts = vdev->hw->d0i3_entry_vpu_ts;
ivpu_dbg(vdev, FW_BOOT, "boot_params.d0i3_residency_time_us = %lld\n",
boot_params->d0i3_residency_time_us);
ivpu_dbg(vdev, FW_BOOT, "boot_params.d0i3_entry_vpu_ts = %llu\n",
boot_params->d0i3_entry_vpu_ts);
boot_params->save_restore_ret_address = 0;
vdev->pm->is_warmboot = true;
wmb(); /* Flush WC buffers after writing save_restore_ret_address */
@ -442,6 +499,13 @@ void ivpu_fw_boot_params_setup(struct ivpu_device *vdev, struct vpu_boot_params
boot_params->vpu_id = to_pci_dev(vdev->drm.dev)->bus->number;
boot_params->frequency = ivpu_hw_reg_pll_freq_get(vdev);
/*
* This param is a debug firmware feature. It switches default clock
* to higher resolution one for fine-grained and more accurate firmware
* task profiling.
*/
boot_params->perf_clk_frequency = ivpu_hw_profiling_freq_get(vdev);
/*
* Uncached region of VPU address space, covers IPC buffers, job queues
* and log buffers, programmable to L2$ Uncached by VPU MTRR
@ -493,6 +557,11 @@ void ivpu_fw_boot_params_setup(struct ivpu_device *vdev, struct vpu_boot_params
boot_params->punit_telemetry_sram_base = ivpu_hw_reg_telemetry_offset_get(vdev);
boot_params->punit_telemetry_sram_size = ivpu_hw_reg_telemetry_size_get(vdev);
boot_params->vpu_telemetry_enable = ivpu_hw_reg_telemetry_enable_get(vdev);
boot_params->dvfs_mode = vdev->fw->dvfs_mode;
if (!IVPU_WA(disable_d0i3_msg))
boot_params->d0i3_delayed_entry = 1;
boot_params->d0i3_residency_time_us = 0;
boot_params->d0i3_entry_vpu_ts = 0;
wmb(); /* Flush WC buffers after writing bootparams */

1
drivers/accel/ivpu/ivpu_fw.h
View File

@ -27,6 +27,7 @@ struct ivpu_fw_info {
u32 trace_level;
u32 trace_destination_mask;
u64 trace_hw_component_mask;
u32 dvfs_mode;
};
int ivpu_fw_init(struct ivpu_device *vdev);

692
drivers/accel/ivpu/ivpu_gem.c
View File

@ -20,215 +20,18 @@
#include "ivpu_mmu.h"
#include "ivpu_mmu_context.h"
MODULE_IMPORT_NS(DMA_BUF);
static const struct drm_gem_object_funcs ivpu_gem_funcs;
static struct lock_class_key prime_bo_lock_class_key;
static int __must_check prime_alloc_pages_locked(struct ivpu_bo *bo)
static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action)
{
/* Pages are managed by the underlying dma-buf */
return 0;
}
static void prime_free_pages_locked(struct ivpu_bo *bo)
{
/* Pages are managed by the underlying dma-buf */
}
static int prime_map_pages_locked(struct ivpu_bo *bo)
{
struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
struct sg_table *sgt;
sgt = dma_buf_map_attachment_unlocked(bo->base.import_attach, DMA_BIDIRECTIONAL);
if (IS_ERR(sgt)) {
ivpu_err(vdev, "Failed to map attachment: %ld\n", PTR_ERR(sgt));
return PTR_ERR(sgt);
}
bo->sgt = sgt;
return 0;
}
static void prime_unmap_pages_locked(struct ivpu_bo *bo)
{
dma_buf_unmap_attachment_unlocked(bo->base.import_attach, bo->sgt, DMA_BIDIRECTIONAL);
bo->sgt = NULL;
}
static const struct ivpu_bo_ops prime_ops = {
.type = IVPU_BO_TYPE_PRIME,
.name = "prime",
.alloc_pages = prime_alloc_pages_locked,
.free_pages = prime_free_pages_locked,
.map_pages = prime_map_pages_locked,
.unmap_pages = prime_unmap_pages_locked,
};
static int __must_check shmem_alloc_pages_locked(struct ivpu_bo *bo)
{
int npages = ivpu_bo_size(bo) >> PAGE_SHIFT;
struct page **pages;
pages = drm_gem_get_pages(&bo->base);
if (IS_ERR(pages))
return PTR_ERR(pages);
if (bo->flags & DRM_IVPU_BO_WC)
set_pages_array_wc(pages, npages);
else if (bo->flags & DRM_IVPU_BO_UNCACHED)
set_pages_array_uc(pages, npages);
bo->pages = pages;
return 0;
}
static void shmem_free_pages_locked(struct ivpu_bo *bo)
{
if (ivpu_bo_cache_mode(bo) != DRM_IVPU_BO_CACHED)
set_pages_array_wb(bo->pages, ivpu_bo_size(bo) >> PAGE_SHIFT);
drm_gem_put_pages(&bo->base, bo->pages, true, false);
bo->pages = NULL;
}
static int ivpu_bo_map_pages_locked(struct ivpu_bo *bo)
{
int npages = ivpu_bo_size(bo) >> PAGE_SHIFT;
struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
struct sg_table *sgt;
int ret;
sgt = drm_prime_pages_to_sg(&vdev->drm, bo->pages, npages);
if (IS_ERR(sgt)) {
ivpu_err(vdev, "Failed to allocate sgtable\n");
return PTR_ERR(sgt);
}
ret = dma_map_sgtable(vdev->drm.dev, sgt, DMA_BIDIRECTIONAL, 0);
if (ret) {
ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret);
goto err_free_sgt;
}
bo->sgt = sgt;
return 0;
err_free_sgt:
kfree(sgt);
return ret;
}
static void ivpu_bo_unmap_pages_locked(struct ivpu_bo *bo)
{
struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
dma_unmap_sgtable(vdev->drm.dev, bo->sgt, DMA_BIDIRECTIONAL, 0);
sg_free_table(bo->sgt);
kfree(bo->sgt);
bo->sgt = NULL;
}
static const struct ivpu_bo_ops shmem_ops = {
.type = IVPU_BO_TYPE_SHMEM,
.name = "shmem",
.alloc_pages = shmem_alloc_pages_locked,
.free_pages = shmem_free_pages_locked,
.map_pages = ivpu_bo_map_pages_locked,
.unmap_pages = ivpu_bo_unmap_pages_locked,
};
static int __must_check internal_alloc_pages_locked(struct ivpu_bo *bo)
{
unsigned int i, npages = ivpu_bo_size(bo) >> PAGE_SHIFT;
struct page **pages;
int ret;
pages = kvmalloc_array(npages, sizeof(*bo->pages), GFP_KERNEL);
if (!pages)
return -ENOMEM;
for (i = 0; i < npages; i++) {
pages[i] = alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
if (!pages[i]) {
ret = -ENOMEM;
goto err_free_pages;
}
cond_resched();
}
bo->pages = pages;
return 0;
err_free_pages:
while (i--)
put_page(pages[i]);
kvfree(pages);
return ret;
}
static void internal_free_pages_locked(struct ivpu_bo *bo)
{
unsigned int i, npages = ivpu_bo_size(bo) >> PAGE_SHIFT;
if (ivpu_bo_cache_mode(bo) != DRM_IVPU_BO_CACHED)
set_pages_array_wb(bo->pages, ivpu_bo_size(bo) >> PAGE_SHIFT);
for (i = 0; i < npages; i++)
put_page(bo->pages[i]);
kvfree(bo->pages);
bo->pages = NULL;
}
static const struct ivpu_bo_ops internal_ops = {
.type = IVPU_BO_TYPE_INTERNAL,
.name = "internal",
.alloc_pages = internal_alloc_pages_locked,
.free_pages = internal_free_pages_locked,
.map_pages = ivpu_bo_map_pages_locked,
.unmap_pages = ivpu_bo_unmap_pages_locked,
};
static int __must_check ivpu_bo_alloc_and_map_pages_locked(struct ivpu_bo *bo)
{
struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
int ret;
lockdep_assert_held(&bo->lock);
drm_WARN_ON(&vdev->drm, bo->sgt);
ret = bo->ops->alloc_pages(bo);
if (ret) {
ivpu_err(vdev, "Failed to allocate pages for BO: %d", ret);
return ret;
}
ret = bo->ops->map_pages(bo);
if (ret) {
ivpu_err(vdev, "Failed to map pages for BO: %d", ret);
goto err_free_pages;
}
return ret;
err_free_pages:
bo->ops->free_pages(bo);
return ret;
}
static void ivpu_bo_unmap_and_free_pages(struct ivpu_bo *bo)
{
mutex_lock(&bo->lock);
WARN_ON(!bo->sgt);
bo->ops->unmap_pages(bo);
WARN_ON(bo->sgt);
bo->ops->free_pages(bo);
WARN_ON(bo->pages);
mutex_unlock(&bo->lock);
if (bo->ctx)
ivpu_dbg(vdev, BO, "%6s: size %zu has_pages %d dma_mapped %d handle %u ctx %d vpu_addr 0x%llx mmu_mapped %d\n",
action, ivpu_bo_size(bo), (bool)bo->base.pages, (bool)bo->base.sgt,
bo->handle, bo->ctx->id, bo->vpu_addr, bo->mmu_mapped);
else
ivpu_dbg(vdev, BO, "%6s: size %zu has_pages %d dma_mapped %d handle %u (not added to context)\n",
action, ivpu_bo_size(bo), (bool)bo->base.pages, (bool)bo->base.sgt,
bo->handle);
}
/*
@ -245,21 +48,24 @@ int __must_check ivpu_bo_pin(struct ivpu_bo *bo)
mutex_lock(&bo->lock);
if (!bo->vpu_addr) {
ivpu_err(vdev, "vpu_addr not set for BO ctx_id: %d handle: %d\n",
bo->ctx->id, bo->handle);
ivpu_dbg_bo(vdev, bo, "pin");
if (!bo->ctx) {
ivpu_err(vdev, "vpu_addr not allocated for BO %d\n", bo->handle);
ret = -EINVAL;
goto unlock;
}
if (!bo->sgt) {
ret = ivpu_bo_alloc_and_map_pages_locked(bo);
if (ret)
goto unlock;
}
if (!bo->mmu_mapped) {
ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, bo->sgt,
struct sg_table *sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
if (IS_ERR(sgt)) {
ret = PTR_ERR(sgt);
ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret);
goto unlock;
}
ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt,
ivpu_bo_is_snooped(bo));
if (ret) {
ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret);
@ -281,248 +87,213 @@ ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx,
struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
int ret;
if (!range) {
if (bo->flags & DRM_IVPU_BO_SHAVE_MEM)
range = &vdev->hw->ranges.shave;
else if (bo->flags & DRM_IVPU_BO_DMA_MEM)
range = &vdev->hw->ranges.dma;
else
range = &vdev->hw->ranges.user;
}
mutex_lock(&bo->lock);
mutex_lock(&ctx->lock);
ret = ivpu_mmu_context_insert_node_locked(ctx, range, ivpu_bo_size(bo), &bo->mm_node);
ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node);
if (!ret) {
bo->ctx = ctx;
bo->vpu_addr = bo->mm_node.start;
list_add_tail(&bo->ctx_node, &ctx->bo_list);
} else {
ivpu_err(vdev, "Failed to add BO to context %u: %d\n", ctx->id, ret);
}
mutex_unlock(&ctx->lock);
ivpu_dbg_bo(vdev, bo, "alloc");
mutex_unlock(&bo->lock);
return ret;
}
static void ivpu_bo_free_vpu_addr(struct ivpu_bo *bo)
static void ivpu_bo_unbind_locked(struct ivpu_bo *bo)
{
struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
struct ivpu_mmu_context *ctx = bo->ctx;
ivpu_dbg(vdev, BO, "remove from ctx: ctx %d vpu_addr 0x%llx allocated %d mmu_mapped %d\n",
ctx->id, bo->vpu_addr, (bool)bo->sgt, bo->mmu_mapped);
lockdep_assert_held(&bo->lock);
mutex_lock(&bo->lock);
ivpu_dbg_bo(vdev, bo, "unbind");
/* TODO: dma_unmap */
if (bo->mmu_mapped) {
drm_WARN_ON(&vdev->drm, !bo->sgt);
ivpu_mmu_context_unmap_sgt(vdev, ctx, bo->vpu_addr, bo->sgt);
drm_WARN_ON(&vdev->drm, !bo->ctx);
drm_WARN_ON(&vdev->drm, !bo->vpu_addr);
drm_WARN_ON(&vdev->drm, !bo->base.sgt);
ivpu_mmu_context_unmap_sgt(vdev, bo->ctx, bo->vpu_addr, bo->base.sgt);
bo->mmu_mapped = false;
}
mutex_lock(&ctx->lock);
list_del(&bo->ctx_node);
bo->vpu_addr = 0;
bo->ctx = NULL;
ivpu_mmu_context_remove_node_locked(ctx, &bo->mm_node);
mutex_unlock(&ctx->lock);
if (bo->ctx) {
ivpu_mmu_context_remove_node(bo->ctx, &bo->mm_node);
bo->vpu_addr = 0;
bo->ctx = NULL;
}
}
static void ivpu_bo_unbind(struct ivpu_bo *bo)
{
mutex_lock(&bo->lock);
ivpu_bo_unbind_locked(bo);
mutex_unlock(&bo->lock);
}
void ivpu_bo_remove_all_bos_from_context(struct ivpu_mmu_context *ctx)
void ivpu_bo_remove_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
{
struct ivpu_bo *bo, *tmp;
struct ivpu_bo *bo;
list_for_each_entry_safe(bo, tmp, &ctx->bo_list, ctx_node)
ivpu_bo_free_vpu_addr(bo);
if (drm_WARN_ON(&vdev->drm, !ctx))
return;
mutex_lock(&vdev->bo_list_lock);
list_for_each_entry(bo, &vdev->bo_list, bo_list_node) {
mutex_lock(&bo->lock);
if (bo->ctx == ctx)
ivpu_bo_unbind_locked(bo);
mutex_unlock(&bo->lock);
}
mutex_unlock(&vdev->bo_list_lock);
}
struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size)
{
struct ivpu_bo *bo;
if (size == 0 || !PAGE_ALIGNED(size))
return ERR_PTR(-EINVAL);
bo = kzalloc(sizeof(*bo), GFP_KERNEL);
if (!bo)
return ERR_PTR(-ENOMEM);
bo->base.base.funcs = &ivpu_gem_funcs;
bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */
INIT_LIST_HEAD(&bo->bo_list_node);
mutex_init(&bo->lock);
return &bo->base.base;
}
static struct ivpu_bo *
ivpu_bo_alloc(struct ivpu_device *vdev, struct ivpu_mmu_context *mmu_context,
u64 size, u32 flags, const struct ivpu_bo_ops *ops,
const struct ivpu_addr_range *range, u64 user_ptr)
ivpu_bo_create(struct ivpu_device *vdev, u64 size, u32 flags)
{
struct drm_gem_shmem_object *shmem;
struct ivpu_bo *bo;
int ret = 0;
if (drm_WARN_ON(&vdev->drm, size == 0 || !PAGE_ALIGNED(size)))
return ERR_PTR(-EINVAL);
switch (flags & DRM_IVPU_BO_CACHE_MASK) {
case DRM_IVPU_BO_CACHED:
case DRM_IVPU_BO_UNCACHED:
case DRM_IVPU_BO_WC:
break;
default:
return ERR_PTR(-EINVAL);
}
bo = kzalloc(sizeof(*bo), GFP_KERNEL);
if (!bo)
return ERR_PTR(-ENOMEM);
shmem = drm_gem_shmem_create(&vdev->drm, size);
if (IS_ERR(shmem))
return ERR_CAST(shmem);
mutex_init(&bo->lock);
bo->base.funcs = &ivpu_gem_funcs;
bo = to_ivpu_bo(&shmem->base);
bo->base.map_wc = flags & DRM_IVPU_BO_WC;
bo->flags = flags;
bo->ops = ops;
bo->user_ptr = user_ptr;
if (ops->type == IVPU_BO_TYPE_SHMEM)
ret = drm_gem_object_init(&vdev->drm, &bo->base, size);
else
drm_gem_private_object_init(&vdev->drm, &bo->base, size);
mutex_lock(&vdev->bo_list_lock);
list_add_tail(&bo->bo_list_node, &vdev->bo_list);
mutex_unlock(&vdev->bo_list_lock);
if (ret) {
ivpu_err(vdev, "Failed to initialize drm object\n");
goto err_free;
}
if (flags & DRM_IVPU_BO_MAPPABLE) {
ret = drm_gem_create_mmap_offset(&bo->base);
if (ret) {
ivpu_err(vdev, "Failed to allocate mmap offset\n");
goto err_release;
}
}
if (mmu_context) {
ret = ivpu_bo_alloc_vpu_addr(bo, mmu_context, range);
if (ret) {
ivpu_err(vdev, "Failed to add BO to context: %d\n", ret);
goto err_release;
}
}
ivpu_dbg(vdev, BO, "create: vpu_addr 0x%llx size %zu flags 0x%x\n",
bo->vpu_addr, bo->base.base.size, flags);
return bo;
}
err_release:
drm_gem_object_release(&bo->base);
err_free:
kfree(bo);
return ERR_PTR(ret);
static int ivpu_bo_open(struct drm_gem_object *obj, struct drm_file *file)
{
struct ivpu_file_priv *file_priv = file->driver_priv;
struct ivpu_device *vdev = file_priv->vdev;
struct ivpu_bo *bo = to_ivpu_bo(obj);
struct ivpu_addr_range *range;
if (bo->flags & DRM_IVPU_BO_SHAVE_MEM)
range = &vdev->hw->ranges.shave;
else if (bo->flags & DRM_IVPU_BO_DMA_MEM)
range = &vdev->hw->ranges.dma;
else
range = &vdev->hw->ranges.user;
return ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, range);
}
static void ivpu_bo_free(struct drm_gem_object *obj)
{
struct ivpu_device *vdev = to_ivpu_device(obj->dev);
struct ivpu_bo *bo = to_ivpu_bo(obj);
struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
if (bo->ctx)
ivpu_dbg(vdev, BO, "free: ctx %d vpu_addr 0x%llx allocated %d mmu_mapped %d\n",
bo->ctx->id, bo->vpu_addr, (bool)bo->sgt, bo->mmu_mapped);
else
ivpu_dbg(vdev, BO, "free: ctx (released) allocated %d mmu_mapped %d\n",
(bool)bo->sgt, bo->mmu_mapped);
mutex_lock(&vdev->bo_list_lock);
list_del(&bo->bo_list_node);
mutex_unlock(&vdev->bo_list_lock);
drm_WARN_ON(&vdev->drm, !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ));
vunmap(bo->kvaddr);
if (bo->ctx)
ivpu_bo_free_vpu_addr(bo);
if (bo->sgt)
ivpu_bo_unmap_and_free_pages(bo);
if (bo->base.import_attach)
drm_prime_gem_destroy(&bo->base, bo->sgt);
drm_gem_object_release(&bo->base);
ivpu_dbg_bo(vdev, bo, "free");
ivpu_bo_unbind(bo);
mutex_destroy(&bo->lock);
kfree(bo);
drm_WARN_ON(obj->dev, bo->base.pages_use_count > 1);
drm_gem_shmem_free(&bo->base);
}
static int ivpu_bo_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
{
struct ivpu_bo *bo = to_ivpu_bo(obj);
struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
ivpu_dbg(vdev, BO, "mmap: ctx %u handle %u vpu_addr 0x%llx size %zu type %s",
bo->ctx->id, bo->handle, bo->vpu_addr, ivpu_bo_size(bo), bo->ops->name);
if (obj->import_attach) {
/* Drop the reference drm_gem_mmap_obj() acquired.*/
drm_gem_object_put(obj);
vma->vm_private_data = NULL;
return dma_buf_mmap(obj->dma_buf, vma, 0);
}
vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND);
vma->vm_page_prot = ivpu_bo_pgprot(bo, vm_get_page_prot(vma->vm_flags));
return 0;
}
static struct sg_table *ivpu_bo_get_sg_table(struct drm_gem_object *obj)
{
struct ivpu_bo *bo = to_ivpu_bo(obj);
loff_t npages = obj->size >> PAGE_SHIFT;
int ret = 0;
mutex_lock(&bo->lock);
if (!bo->sgt)
ret = ivpu_bo_alloc_and_map_pages_locked(bo);
mutex_unlock(&bo->lock);
if (ret)
return ERR_PTR(ret);
return drm_prime_pages_to_sg(obj->dev, bo->pages, npages);
}
static vm_fault_t ivpu_vm_fault(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
struct drm_gem_object *obj = vma->vm_private_data;
struct ivpu_bo *bo = to_ivpu_bo(obj);
loff_t npages = obj->size >> PAGE_SHIFT;
pgoff_t page_offset;
struct page *page;
vm_fault_t ret;
int err;
mutex_lock(&bo->lock);
if (!bo->sgt) {
err = ivpu_bo_alloc_and_map_pages_locked(bo);
if (err) {
ret = vmf_error(err);
goto unlock;
}
}
/* We don't use vmf->pgoff since that has the fake offset */
page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
if (page_offset >= npages) {
ret = VM_FAULT_SIGBUS;
} else {
page = bo->pages[page_offset];
ret = vmf_insert_pfn(vma, vmf->address, page_to_pfn(page));
}
unlock:
mutex_unlock(&bo->lock);
return ret;
}
static const struct vm_operations_struct ivpu_vm_ops = {
.fault = ivpu_vm_fault,
.open = drm_gem_vm_open,
.close = drm_gem_vm_close,
static const struct dma_buf_ops ivpu_bo_dmabuf_ops = {
.cache_sgt_mapping = true,
.attach = drm_gem_map_attach,
.detach = drm_gem_map_detach,
.map_dma_buf = drm_gem_map_dma_buf,
.unmap_dma_buf = drm_gem_unmap_dma_buf,
.release = drm_gem_dmabuf_release,
.mmap = drm_gem_dmabuf_mmap,
.vmap = drm_gem_dmabuf_vmap,
.vunmap = drm_gem_dmabuf_vunmap,
};
static struct dma_buf *ivpu_bo_export(struct drm_gem_object *obj, int flags)
{
struct drm_device *dev = obj->dev;
struct dma_buf_export_info exp_info = {
.exp_name = KBUILD_MODNAME,
.owner = dev->driver->fops->owner,
.ops = &ivpu_bo_dmabuf_ops,
.size = obj->size,
.flags = flags,
.priv = obj,
.resv = obj->resv,
};
void *sgt;
/*
* Make sure that pages are allocated and dma-mapped before exporting the bo.
* DMA-mapping is required if the bo will be imported to the same device.
*/
sgt = drm_gem_shmem_get_pages_sgt(to_drm_gem_shmem_obj(obj));
if (IS_ERR(sgt))
return sgt;
return drm_gem_dmabuf_export(dev, &exp_info);
}
static const struct drm_gem_object_funcs ivpu_gem_funcs = {
.free = ivpu_bo_free,
.mmap = ivpu_bo_mmap,
.vm_ops = &ivpu_vm_ops,
.get_sg_table = ivpu_bo_get_sg_table,
.open = ivpu_bo_open,
.export = ivpu_bo_export,
.print_info = drm_gem_shmem_object_print_info,
.pin = drm_gem_shmem_object_pin,
.unpin = drm_gem_shmem_object_unpin,
.get_sg_table = drm_gem_shmem_object_get_sg_table,
.vmap = drm_gem_shmem_object_vmap,
.vunmap = drm_gem_shmem_object_vunmap,
.mmap = drm_gem_shmem_object_mmap,
.vm_ops = &drm_gem_shmem_vm_ops,
};
int
ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
{
struct ivpu_file_priv *file_priv = file->driver_priv;
struct ivpu_device *vdev = file_priv->vdev;
@ -537,23 +308,20 @@ ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
if (size == 0)
return -EINVAL;
bo = ivpu_bo_alloc(vdev, &file_priv->ctx, size, args->flags, &shmem_ops, NULL, 0);
bo = ivpu_bo_create(vdev, size, args->flags);
if (IS_ERR(bo)) {
ivpu_err(vdev, "Failed to create BO: %pe (ctx %u size %llu flags 0x%x)",
bo, file_priv->ctx.id, args->size, args->flags);
return PTR_ERR(bo);
}
ret = drm_gem_handle_create(file, &bo->base, &bo->handle);
ret = drm_gem_handle_create(file, &bo->base.base, &bo->handle);
if (!ret) {
args->vpu_addr = bo->vpu_addr;
args->handle = bo->handle;
}
drm_gem_object_put(&bo->base);
ivpu_dbg(vdev, BO, "alloc shmem: ctx %u vpu_addr 0x%llx size %zu flags 0x%x\n",
file_priv->ctx.id, bo->vpu_addr, ivpu_bo_size(bo), bo->flags);
drm_gem_object_put(&bo->base.base);
return ret;
}
@ -563,8 +331,8 @@ ivpu_bo_alloc_internal(struct ivpu_device *vdev, u64 vpu_addr, u64 size, u32 fla
{
const struct ivpu_addr_range *range;
struct ivpu_addr_range fixed_range;
struct iosys_map map;
struct ivpu_bo *bo;
pgprot_t prot;
int ret;
drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(vpu_addr));
@ -578,81 +346,42 @@ ivpu_bo_alloc_internal(struct ivpu_device *vdev, u64 vpu_addr, u64 size, u32 fla
range = &vdev->hw->ranges.global;
}
bo = ivpu_bo_alloc(vdev, &vdev->gctx, size, flags, &internal_ops, range, 0);
bo = ivpu_bo_create(vdev, size, flags);
if (IS_ERR(bo)) {
ivpu_err(vdev, "Failed to create BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)",
bo, vpu_addr, size, flags);
return NULL;
}
ret = ivpu_bo_alloc_vpu_addr(bo, &vdev->gctx, range);
if (ret)
goto err_put;
ret = ivpu_bo_pin(bo);
if (ret)
goto err_put;
if (ivpu_bo_cache_mode(bo) != DRM_IVPU_BO_CACHED)
drm_clflush_pages(bo->pages, ivpu_bo_size(bo) >> PAGE_SHIFT);
if (bo->flags & DRM_IVPU_BO_WC)
set_pages_array_wc(bo->pages, ivpu_bo_size(bo) >> PAGE_SHIFT);
else if (bo->flags & DRM_IVPU_BO_UNCACHED)
set_pages_array_uc(bo->pages, ivpu_bo_size(bo) >> PAGE_SHIFT);
prot = ivpu_bo_pgprot(bo, PAGE_KERNEL);
bo->kvaddr = vmap(bo->pages, ivpu_bo_size(bo) >> PAGE_SHIFT, VM_MAP, prot);
if (!bo->kvaddr) {
ivpu_err(vdev, "Failed to map BO into kernel virtual memory\n");
ret = drm_gem_shmem_vmap(&bo->base, &map);
if (ret)
goto err_put;
}
ivpu_dbg(vdev, BO, "alloc internal: ctx 0 vpu_addr 0x%llx size %zu flags 0x%x\n",
bo->vpu_addr, ivpu_bo_size(bo), flags);
return bo;
err_put:
drm_gem_object_put(&bo->base);
drm_gem_object_put(&bo->base.base);
return NULL;
}
void ivpu_bo_free_internal(struct ivpu_bo *bo)
{
drm_gem_object_put(&bo->base);
}
struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr);
struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev, struct dma_buf *buf)
{
struct ivpu_device *vdev = to_ivpu_device(dev);
struct dma_buf_attachment *attach;
struct ivpu_bo *bo;
attach = dma_buf_attach(buf, dev->dev);
if (IS_ERR(attach))
return ERR_CAST(attach);
get_dma_buf(buf);
bo = ivpu_bo_alloc(vdev, NULL, buf->size, DRM_IVPU_BO_MAPPABLE, &prime_ops, NULL, 0);
if (IS_ERR(bo)) {
ivpu_err(vdev, "Failed to import BO: %pe (size %lu)", bo, buf->size);
goto err_detach;
}
lockdep_set_class(&bo->lock, &prime_bo_lock_class_key);
bo->base.import_attach = attach;
return &bo->base;
err_detach:
dma_buf_detach(buf, attach);
dma_buf_put(buf);
return ERR_CAST(bo);
drm_gem_shmem_vunmap(&bo->base, &map);
drm_gem_object_put(&bo->base.base);
}
int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
{
struct ivpu_file_priv *file_priv = file->driver_priv;
struct ivpu_device *vdev = to_ivpu_device(dev);
struct drm_ivpu_bo_info *args = data;
struct drm_gem_object *obj;
struct ivpu_bo *bo;
@ -665,21 +394,12 @@ int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file
bo = to_ivpu_bo(obj);
mutex_lock(&bo->lock);
if (!bo->ctx) {
ret = ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, NULL);
if (ret) {
ivpu_err(vdev, "Failed to allocate vpu_addr: %d\n", ret);
goto unlock;
}
}
args->flags = bo->flags;
args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node);
args->vpu_addr = bo->vpu_addr;
args->size = obj->size;
unlock:
mutex_unlock(&bo->lock);
drm_gem_object_put(obj);
return ret;
}
@ -714,41 +434,41 @@ static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p)
{
unsigned long dma_refcount = 0;
if (bo->base.dma_buf && bo->base.dma_buf->file)
dma_refcount = atomic_long_read(&bo->base.dma_buf->file->f_count);
mutex_lock(&bo->lock);
drm_printf(p, "%5u %6d %16llx %10lu %10u %12lu %14s\n",
bo->ctx->id, bo->handle, bo->vpu_addr, ivpu_bo_size(bo),
kref_read(&bo->base.refcount), dma_refcount, bo->ops->name);
if (bo->base.base.dma_buf && bo->base.base.dma_buf->file)
dma_refcount = atomic_long_read(&bo->base.base.dma_buf->file->f_count);
drm_printf(p, "%-3u %-6d 0x%-12llx %-10lu 0x%-8x %-4u %-8lu",
bo->ctx->id, bo->handle, bo->vpu_addr, bo->base.base.size,
bo->flags, kref_read(&bo->base.base.refcount), dma_refcount);
if (bo->base.base.import_attach)
drm_printf(p, " imported");
if (bo->base.pages)
drm_printf(p, " has_pages");
if (bo->mmu_mapped)
drm_printf(p, " mmu_mapped");
drm_printf(p, "\n");
mutex_unlock(&bo->lock);
}
void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p)
{
struct ivpu_device *vdev = to_ivpu_device(dev);
struct ivpu_file_priv *file_priv;
unsigned long ctx_id;
struct ivpu_bo *bo;
drm_printf(p, "%5s %6s %16s %10s %10s %12s %14s\n",
"ctx", "handle", "vpu_addr", "size", "refcount", "dma_refcount", "type");
drm_printf(p, "%-3s %-6s %-14s %-10s %-10s %-4s %-8s %s\n",
"ctx", "handle", "vpu_addr", "size", "flags", "refs", "dma_refs", "attribs");
mutex_lock(&vdev->gctx.lock);
list_for_each_entry(bo, &vdev->gctx.bo_list, ctx_node)
mutex_lock(&vdev->bo_list_lock);
list_for_each_entry(bo, &vdev->bo_list, bo_list_node)
ivpu_bo_print_info(bo, p);
mutex_unlock(&vdev->gctx.lock);
xa_for_each(&vdev->context_xa, ctx_id, file_priv) {
file_priv = ivpu_file_priv_get_by_ctx_id(vdev, ctx_id);
if (!file_priv)
continue;
mutex_lock(&file_priv->ctx.lock);
list_for_each_entry(bo, &file_priv->ctx.bo_list, ctx_node)
ivpu_bo_print_info(bo, p);
mutex_unlock(&file_priv->ctx.lock);
ivpu_file_priv_put(&file_priv);
}
mutex_unlock(&vdev->bo_list_lock);
}
void ivpu_bo_list_print(struct drm_device *dev)

75
drivers/accel/ivpu/ivpu_gem.h
View File

@ -6,84 +6,52 @@
#define __IVPU_GEM_H__
#include <drm/drm_gem.h>
#include <drm/drm_gem_shmem_helper.h>
#include <drm/drm_mm.h>
struct dma_buf;
struct ivpu_bo_ops;
struct ivpu_file_priv;
struct ivpu_bo {
struct drm_gem_object base;
const struct ivpu_bo_ops *ops;
struct drm_gem_shmem_object base;
struct ivpu_mmu_context *ctx;
struct list_head ctx_node;
struct list_head bo_list_node;
struct drm_mm_node mm_node;
struct mutex lock; /* Protects: pages, sgt, mmu_mapped */
struct sg_table *sgt;
struct page **pages;
bool mmu_mapped;
void *kvaddr;
struct mutex lock; /* Protects: ctx, mmu_mapped, vpu_addr */
u64 vpu_addr;
u32 handle;
u32 flags;
uintptr_t user_ptr;
u32 job_status;
};
enum ivpu_bo_type {
IVPU_BO_TYPE_SHMEM = 1,
IVPU_BO_TYPE_INTERNAL,
IVPU_BO_TYPE_PRIME,
};
struct ivpu_bo_ops {
enum ivpu_bo_type type;
const char *name;
int (*alloc_pages)(struct ivpu_bo *bo);
void (*free_pages)(struct ivpu_bo *bo);
int (*map_pages)(struct ivpu_bo *bo);
void (*unmap_pages)(struct ivpu_bo *bo);
u32 job_status; /* Valid only for command buffer */
bool mmu_mapped;
};
int ivpu_bo_pin(struct ivpu_bo *bo);
void ivpu_bo_remove_all_bos_from_context(struct ivpu_mmu_context *ctx);
void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p);
void ivpu_bo_list_print(struct drm_device *dev);
void ivpu_bo_remove_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx);
struct ivpu_bo *
ivpu_bo_alloc_internal(struct ivpu_device *vdev, u64 vpu_addr, u64 size, u32 flags);
struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size);
struct ivpu_bo *ivpu_bo_alloc_internal(struct ivpu_device *vdev, u64 vpu_addr, u64 size, u32 flags);
void ivpu_bo_free_internal(struct ivpu_bo *bo);
struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf);
void ivpu_bo_unmap_sgt_and_remove_from_context(struct ivpu_bo *bo);
int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file);
int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file);
int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file);
void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p);
void ivpu_bo_list_print(struct drm_device *dev);
static inline struct ivpu_bo *to_ivpu_bo(struct drm_gem_object *obj)
{
return container_of(obj, struct ivpu_bo, base);
return container_of(obj, struct ivpu_bo, base.base);
}
static inline void *ivpu_bo_vaddr(struct ivpu_bo *bo)
{
return bo->kvaddr;
return bo->base.vaddr;
}
static inline size_t ivpu_bo_size(struct ivpu_bo *bo)
{
return bo->base.size;
}
static inline struct page *ivpu_bo_get_page(struct ivpu_bo *bo, u64 offset)
{
if (offset > ivpu_bo_size(bo) || !bo->pages)
return NULL;
return bo->pages[offset / PAGE_SIZE];
return bo->base.base.size;
}
static inline u32 ivpu_bo_cache_mode(struct ivpu_bo *bo)
@ -96,20 +64,9 @@ static inline bool ivpu_bo_is_snooped(struct ivpu_bo *bo)
return ivpu_bo_cache_mode(bo) == DRM_IVPU_BO_CACHED;
}
static inline pgprot_t ivpu_bo_pgprot(struct ivpu_bo *bo, pgprot_t prot)
{
if (bo->flags & DRM_IVPU_BO_WC)
return pgprot_writecombine(prot);
if (bo->flags & DRM_IVPU_BO_UNCACHED)
return pgprot_noncached(prot);
return prot;
}
static inline struct ivpu_device *ivpu_bo_to_vdev(struct ivpu_bo *bo)
{
return to_ivpu_device(bo->base.dev);
return to_ivpu_device(bo->base.base.dev);
}
static inline void *ivpu_to_cpu_addr(struct ivpu_bo *bo, u32 vpu_addr)

20
drivers/accel/ivpu/ivpu_hw.h
View File

@ -15,8 +15,11 @@ struct ivpu_hw_ops {
int (*power_down)(struct ivpu_device *vdev);
int (*reset)(struct ivpu_device *vdev);
bool (*is_idle)(struct ivpu_device *vdev);
int (*wait_for_idle)(struct ivpu_device *vdev);
void (*wdt_disable)(struct ivpu_device *vdev);
void (*diagnose_failure)(struct ivpu_device *vdev);
u32 (*profiling_freq_get)(struct ivpu_device *vdev);
void (*profiling_freq_drive)(struct ivpu_device *vdev, bool enable);
u32 (*reg_pll_freq_get)(struct ivpu_device *vdev);
u32 (*reg_telemetry_offset_get)(struct ivpu_device *vdev);
u32 (*reg_telemetry_size_get)(struct ivpu_device *vdev);
@ -58,6 +61,8 @@ struct ivpu_hw_info {
u32 sku;
u16 config;
int dma_bits;
ktime_t d0i3_entry_host_ts;
u64 d0i3_entry_vpu_ts;
};
extern const struct ivpu_hw_ops ivpu_hw_37xx_ops;
@ -85,6 +90,11 @@ static inline bool ivpu_hw_is_idle(struct ivpu_device *vdev)
return vdev->hw->ops->is_idle(vdev);
};
static inline int ivpu_hw_wait_for_idle(struct ivpu_device *vdev)
{
return vdev->hw->ops->wait_for_idle(vdev);
};
static inline int ivpu_hw_power_down(struct ivpu_device *vdev)
{
ivpu_dbg(vdev, PM, "HW power down\n");
@ -104,6 +114,16 @@ static inline void ivpu_hw_wdt_disable(struct ivpu_device *vdev)
vdev->hw->ops->wdt_disable(vdev);
};
static inline u32 ivpu_hw_profiling_freq_get(struct ivpu_device *vdev)
{
return vdev->hw->ops->profiling_freq_get(vdev);
};
static inline void ivpu_hw_profiling_freq_drive(struct ivpu_device *vdev, bool enable)
{
return vdev->hw->ops->profiling_freq_drive(vdev, enable);
};
/* Register indirect accesses */
static inline u32 ivpu_hw_reg_pll_freq_get(struct ivpu_device *vdev)
{

70
drivers/accel/ivpu/ivpu_hw_37xx.c
View File

@ -29,6 +29,7 @@
#define PLL_REF_CLK_FREQ (50 * 1000000)
#define PLL_SIMULATION_FREQ (10 * 1000000)
#define PLL_PROF_CLK_FREQ (38400 * 1000)
#define PLL_DEFAULT_EPP_VALUE 0x80
#define TIM_SAFE_ENABLE 0xf1d0dead
@ -37,7 +38,7 @@
#define TIMEOUT_US (150 * USEC_PER_MSEC)
#define PWR_ISLAND_STATUS_TIMEOUT_US (5 * USEC_PER_MSEC)
#define PLL_TIMEOUT_US (1500 * USEC_PER_MSEC)
#define IDLE_TIMEOUT_US (500 * USEC_PER_MSEC)
#define IDLE_TIMEOUT_US (5 * USEC_PER_MSEC)
#define ICB_0_IRQ_MASK ((REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT)) | \
(REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT)) | \
@ -90,6 +91,7 @@ static void ivpu_hw_timeouts_init(struct ivpu_device *vdev)
vdev->timeout.tdr = 2000;
vdev->timeout.reschedule_suspend = 10;
vdev->timeout.autosuspend = 10;
vdev->timeout.d0i3_entry_msg = 5;
}
static int ivpu_pll_wait_for_cmd_send(struct ivpu_device *vdev)
@ -651,10 +653,6 @@ static int ivpu_hw_37xx_power_up(struct ivpu_device *vdev)
{
int ret;
ret = ivpu_hw_37xx_reset(vdev);
if (ret)
ivpu_warn(vdev, "Failed to reset HW: %d\n", ret);
ret = ivpu_hw_37xx_d0i3_disable(vdev);
if (ret)
ivpu_warn(vdev, "Failed to disable D0I3: %d\n", ret);
@ -718,12 +716,28 @@ static bool ivpu_hw_37xx_is_idle(struct ivpu_device *vdev)
REG_TEST_FLD(VPU_37XX_BUTTRESS_VPU_STATUS, IDLE, val);
}
static int ivpu_hw_37xx_wait_for_idle(struct ivpu_device *vdev)
{
return REGB_POLL_FLD(VPU_37XX_BUTTRESS_VPU_STATUS, IDLE, 0x1, IDLE_TIMEOUT_US);
}
static void ivpu_hw_37xx_save_d0i3_entry_timestamp(struct ivpu_device *vdev)
{
vdev->hw->d0i3_entry_host_ts = ktime_get_boottime();
vdev->hw->d0i3_entry_vpu_ts = REGV_RD64(VPU_37XX_CPU_SS_TIM_PERF_FREE_CNT);
}
static int ivpu_hw_37xx_power_down(struct ivpu_device *vdev)
{
int ret = 0;
if (!ivpu_hw_37xx_is_idle(vdev) && ivpu_hw_37xx_reset(vdev))
ivpu_err(vdev, "Failed to reset the VPU\n");
ivpu_hw_37xx_save_d0i3_entry_timestamp(vdev);
if (!ivpu_hw_37xx_is_idle(vdev)) {
ivpu_warn(vdev, "VPU not idle during power down\n");
if (ivpu_hw_37xx_reset(vdev))
ivpu_warn(vdev, "Failed to reset the VPU\n");
}
if (ivpu_pll_disable(vdev)) {
ivpu_err(vdev, "Failed to disable PLL\n");
@ -756,6 +770,16 @@ static void ivpu_hw_37xx_wdt_disable(struct ivpu_device *vdev)
REGV_WR32(VPU_37XX_CPU_SS_TIM_GEN_CONFIG, val);
}
static u32 ivpu_hw_37xx_profiling_freq_get(struct ivpu_device *vdev)
{
return PLL_PROF_CLK_FREQ;
}
static void ivpu_hw_37xx_profiling_freq_drive(struct ivpu_device *vdev, bool enable)
{
/* Profiling freq - is a debug feature. Unavailable on VPU 37XX. */
}
static u32 ivpu_hw_37xx_pll_to_freq(u32 ratio, u32 config)
{
u32 pll_clock = PLL_REF_CLK_FREQ * ratio;
@ -867,17 +891,20 @@ static void ivpu_hw_37xx_irq_noc_firewall_handler(struct ivpu_device *vdev)
}
/* Handler for IRQs from VPU core (irqV) */
static u32 ivpu_hw_37xx_irqv_handler(struct ivpu_device *vdev, int irq)
static bool ivpu_hw_37xx_irqv_handler(struct ivpu_device *vdev, int irq, bool *wake_thread)
{
u32 status = REGV_RD32(VPU_37XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK;
if (!status)
return false;
REGV_WR32(VPU_37XX_HOST_SS_ICB_CLEAR_0, status);
if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT, status))
ivpu_mmu_irq_evtq_handler(vdev);
if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT, status))
ivpu_ipc_irq_handler(vdev);
ivpu_ipc_irq_handler(vdev, wake_thread);
if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT, status))
ivpu_dbg(vdev, IRQ, "MMU sync complete\n");
@ -894,17 +921,17 @@ static u32 ivpu_hw_37xx_irqv_handler(struct ivpu_device *vdev, int irq)
if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, status))
ivpu_hw_37xx_irq_noc_firewall_handler(vdev);
return status;
return true;
}
/* Handler for IRQs from Buttress core (irqB) */
static u32 ivpu_hw_37xx_irqb_handler(struct ivpu_device *vdev, int irq)
static bool ivpu_hw_37xx_irqb_handler(struct ivpu_device *vdev, int irq)
{
u32 status = REGB_RD32(VPU_37XX_BUTTRESS_INTERRUPT_STAT) & BUTTRESS_IRQ_MASK;
bool schedule_recovery = false;
if (status == 0)
return 0;
if (!status)
return false;
if (REG_TEST_FLD(VPU_37XX_BUTTRESS_INTERRUPT_STAT, FREQ_CHANGE, status))
ivpu_dbg(vdev, IRQ, "FREQ_CHANGE irq: %08x",
@ -940,23 +967,27 @@ static u32 ivpu_hw_37xx_irqb_handler(struct ivpu_device *vdev, int irq)
if (schedule_recovery)
ivpu_pm_schedule_recovery(vdev);
return status;
return true;
}
static irqreturn_t ivpu_hw_37xx_irq_handler(int irq, void *ptr)
{
struct ivpu_device *vdev = ptr;
u32 ret_irqv, ret_irqb;
bool irqv_handled, irqb_handled, wake_thread = false;
REGB_WR32(VPU_37XX_BUTTRESS_GLOBAL_INT_MASK, 0x1);
ret_irqv = ivpu_hw_37xx_irqv_handler(vdev, irq);
ret_irqb = ivpu_hw_37xx_irqb_handler(vdev, irq);
irqv_handled = ivpu_hw_37xx_irqv_handler(vdev, irq, &wake_thread);
irqb_handled = ivpu_hw_37xx_irqb_handler(vdev, irq);
/* Re-enable global interrupts to re-trigger MSI for pending interrupts */
REGB_WR32(VPU_37XX_BUTTRESS_GLOBAL_INT_MASK, 0x0);
return IRQ_RETVAL(ret_irqb | ret_irqv);
if (wake_thread)
return IRQ_WAKE_THREAD;
if (irqv_handled || irqb_handled)
return IRQ_HANDLED;
return IRQ_NONE;
}
static void ivpu_hw_37xx_diagnose_failure(struct ivpu_device *vdev)
@ -993,11 +1024,14 @@ const struct ivpu_hw_ops ivpu_hw_37xx_ops = {
.info_init = ivpu_hw_37xx_info_init,
.power_up = ivpu_hw_37xx_power_up,
.is_idle = ivpu_hw_37xx_is_idle,
.wait_for_idle = ivpu_hw_37xx_wait_for_idle,
.power_down = ivpu_hw_37xx_power_down,
.reset = ivpu_hw_37xx_reset,
.boot_fw = ivpu_hw_37xx_boot_fw,
.wdt_disable = ivpu_hw_37xx_wdt_disable,
.diagnose_failure = ivpu_hw_37xx_diagnose_failure,
.profiling_freq_get = ivpu_hw_37xx_profiling_freq_get,
.profiling_freq_drive = ivpu_hw_37xx_profiling_freq_drive,
.reg_pll_freq_get = ivpu_hw_37xx_reg_pll_freq_get,
.reg_telemetry_offset_get = ivpu_hw_37xx_reg_telemetry_offset_get,
.reg_telemetry_size_get = ivpu_hw_37xx_reg_telemetry_size_get,

2
drivers/accel/ivpu/ivpu_hw_37xx_reg.h
View File

@ -240,6 +240,8 @@
#define VPU_37XX_CPU_SS_TIM_GEN_CONFIG 0x06021008u
#define VPU_37XX_CPU_SS_TIM_GEN_CONFIG_WDOG_TO_INT_CLR_MASK BIT_MASK(9)
#define VPU_37XX_CPU_SS_TIM_PERF_FREE_CNT 0x06029000u
#define VPU_37XX_CPU_SS_DOORBELL_0 0x06300000u
#define VPU_37XX_CPU_SS_DOORBELL_0_SET_MASK BIT_MASK(0)

69
drivers/accel/ivpu/ivpu_hw_40xx.c
View File

@ -39,6 +39,7 @@
#define TIMEOUT_US (150 * USEC_PER_MSEC)
#define PWR_ISLAND_STATUS_TIMEOUT_US (5 * USEC_PER_MSEC)
#define PLL_TIMEOUT_US (1500 * USEC_PER_MSEC)
#define IDLE_TIMEOUT_US (5 * USEC_PER_MSEC)
#define WEIGHTS_DEFAULT 0xf711f711u
#define WEIGHTS_ATS_DEFAULT 0x0000f711u
@ -139,18 +140,21 @@ static void ivpu_hw_timeouts_init(struct ivpu_device *vdev)
vdev->timeout.tdr = 2000000;
vdev->timeout.reschedule_suspend = 1000;
vdev->timeout.autosuspend = -1;
vdev->timeout.d0i3_entry_msg = 500;
} else if (ivpu_is_simics(vdev)) {
vdev->timeout.boot = 50;
vdev->timeout.jsm = 500;
vdev->timeout.tdr = 10000;
vdev->timeout.reschedule_suspend = 10;
vdev->timeout.autosuspend = -1;
vdev->timeout.d0i3_entry_msg = 100;
} else {
vdev->timeout.boot = 1000;
vdev->timeout.jsm = 500;
vdev->timeout.tdr = 2000;
vdev->timeout.reschedule_suspend = 10;
vdev->timeout.autosuspend = 10;
vdev->timeout.d0i3_entry_msg = 5;
}
}
@ -824,12 +828,6 @@ static int ivpu_hw_40xx_power_up(struct ivpu_device *vdev)
{
int ret;
ret = ivpu_hw_40xx_reset(vdev);
if (ret) {
ivpu_err(vdev, "Failed to reset HW: %d\n", ret);
return ret;
}
ret = ivpu_hw_40xx_d0i3_disable(vdev);
if (ret)
ivpu_warn(vdev, "Failed to disable D0I3: %d\n", ret);
@ -898,10 +896,23 @@ static bool ivpu_hw_40xx_is_idle(struct ivpu_device *vdev)
REG_TEST_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, IDLE, val);
}
static int ivpu_hw_40xx_wait_for_idle(struct ivpu_device *vdev)
{
return REGB_POLL_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, IDLE, 0x1, IDLE_TIMEOUT_US);
}
static void ivpu_hw_40xx_save_d0i3_entry_timestamp(struct ivpu_device *vdev)
{
vdev->hw->d0i3_entry_host_ts = ktime_get_boottime();
vdev->hw->d0i3_entry_vpu_ts = REGV_RD64(VPU_40XX_CPU_SS_TIM_PERF_EXT_FREE_CNT);
}
static int ivpu_hw_40xx_power_down(struct ivpu_device *vdev)
{
int ret = 0;
ivpu_hw_40xx_save_d0i3_entry_timestamp(vdev);
if (!ivpu_hw_40xx_is_idle(vdev) && ivpu_hw_40xx_reset(vdev))
ivpu_warn(vdev, "Failed to reset the VPU\n");
@ -933,6 +944,19 @@ static void ivpu_hw_40xx_wdt_disable(struct ivpu_device *vdev)
REGV_WR32(VPU_40XX_CPU_SS_TIM_GEN_CONFIG, val);
}
static u32 ivpu_hw_40xx_profiling_freq_get(struct ivpu_device *vdev)
{
return vdev->hw->pll.profiling_freq;
}
static void ivpu_hw_40xx_profiling_freq_drive(struct ivpu_device *vdev, bool enable)
{
if (enable)
vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_HIGH;
else
vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT;
}
/* Register indirect accesses */
static u32 ivpu_hw_40xx_reg_pll_freq_get(struct ivpu_device *vdev)
{
@ -1023,13 +1047,12 @@ static void ivpu_hw_40xx_irq_noc_firewall_handler(struct ivpu_device *vdev)
}
/* Handler for IRQs from VPU core (irqV) */
static irqreturn_t ivpu_hw_40xx_irqv_handler(struct ivpu_device *vdev, int irq)
static bool ivpu_hw_40xx_irqv_handler(struct ivpu_device *vdev, int irq, bool *wake_thread)
{
u32 status = REGV_RD32(VPU_40XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK;
irqreturn_t ret = IRQ_NONE;
if (!status)
return IRQ_NONE;
return false;
REGV_WR32(VPU_40XX_HOST_SS_ICB_CLEAR_0, status);
@ -1037,7 +1060,7 @@ static irqreturn_t ivpu_hw_40xx_irqv_handler(struct ivpu_device *vdev, int irq)
ivpu_mmu_irq_evtq_handler(vdev);
if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT, status))
ret |= ivpu_ipc_irq_handler(vdev);
ivpu_ipc_irq_handler(vdev, wake_thread);
if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT, status))
ivpu_dbg(vdev, IRQ, "MMU sync complete\n");
@ -1054,17 +1077,17 @@ static irqreturn_t ivpu_hw_40xx_irqv_handler(struct ivpu_device *vdev, int irq)
if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, status))
ivpu_hw_40xx_irq_noc_firewall_handler(vdev);
return ret;
return true;
}
/* Handler for IRQs from Buttress core (irqB) */
static irqreturn_t ivpu_hw_40xx_irqb_handler(struct ivpu_device *vdev, int irq)
static bool ivpu_hw_40xx_irqb_handler(struct ivpu_device *vdev, int irq)
{
bool schedule_recovery = false;
u32 status = REGB_RD32(VPU_40XX_BUTTRESS_INTERRUPT_STAT) & BUTTRESS_IRQ_MASK;
if (status == 0)
return IRQ_NONE;
if (!status)
return false;
if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, FREQ_CHANGE, status))
ivpu_dbg(vdev, IRQ, "FREQ_CHANGE");
@ -1116,26 +1139,27 @@ static irqreturn_t ivpu_hw_40xx_irqb_handler(struct ivpu_device *vdev, int irq)
if (schedule_recovery)
ivpu_pm_schedule_recovery(vdev);
return IRQ_HANDLED;
return true;
}
static irqreturn_t ivpu_hw_40xx_irq_handler(int irq, void *ptr)
{
bool irqv_handled, irqb_handled, wake_thread = false;
struct ivpu_device *vdev = ptr;
irqreturn_t ret = IRQ_NONE;
REGB_WR32(VPU_40XX_BUTTRESS_GLOBAL_INT_MASK, 0x1);
ret |= ivpu_hw_40xx_irqv_handler(vdev, irq);
ret |= ivpu_hw_40xx_irqb_handler(vdev, irq);
irqv_handled = ivpu_hw_40xx_irqv_handler(vdev, irq, &wake_thread);
irqb_handled = ivpu_hw_40xx_irqb_handler(vdev, irq);
/* Re-enable global interrupts to re-trigger MSI for pending interrupts */
REGB_WR32(VPU_40XX_BUTTRESS_GLOBAL_INT_MASK, 0x0);
if (ret & IRQ_WAKE_THREAD)
if (wake_thread)
return IRQ_WAKE_THREAD;
return ret;
if (irqv_handled || irqb_handled)
return IRQ_HANDLED;
return IRQ_NONE;
}
static void ivpu_hw_40xx_diagnose_failure(struct ivpu_device *vdev)
@ -1185,11 +1209,14 @@ const struct ivpu_hw_ops ivpu_hw_40xx_ops = {
.info_init = ivpu_hw_40xx_info_init,
.power_up = ivpu_hw_40xx_power_up,
.is_idle = ivpu_hw_40xx_is_idle,
.wait_for_idle = ivpu_hw_40xx_wait_for_idle,
.power_down = ivpu_hw_40xx_power_down,
.reset = ivpu_hw_40xx_reset,
.boot_fw = ivpu_hw_40xx_boot_fw,
.wdt_disable = ivpu_hw_40xx_wdt_disable,
.diagnose_failure = ivpu_hw_40xx_diagnose_failure,
.profiling_freq_get = ivpu_hw_40xx_profiling_freq_get,
.profiling_freq_drive = ivpu_hw_40xx_profiling_freq_drive,
.reg_pll_freq_get = ivpu_hw_40xx_reg_pll_freq_get,
.reg_telemetry_offset_get = ivpu_hw_40xx_reg_telemetry_offset_get,
.reg_telemetry_size_get = ivpu_hw_40xx_reg_telemetry_size_get,

251
drivers/accel/ivpu/ivpu_ipc.c
View File

@ -5,7 +5,7 @@
#include <linux/genalloc.h>
#include <linux/highmem.h>
#include <linux/kthread.h>
#include <linux/pm_runtime.h>
#include <linux/wait.h>
#include "ivpu_drv.h"
@ -17,19 +17,12 @@
#include "ivpu_pm.h"
#define IPC_MAX_RX_MSG 128
#define IS_KTHREAD() (get_current()->flags & PF_KTHREAD)
struct ivpu_ipc_tx_buf {
struct ivpu_ipc_hdr ipc;
struct vpu_jsm_msg jsm;
};
struct ivpu_ipc_rx_msg {
struct list_head link;
struct ivpu_ipc_hdr *ipc_hdr;
struct vpu_jsm_msg *jsm_msg;
};
static void ivpu_ipc_msg_dump(struct ivpu_device *vdev, char *c,
struct ivpu_ipc_hdr *ipc_hdr, u32 vpu_addr)
{
@ -139,8 +132,49 @@ static void ivpu_ipc_tx(struct ivpu_device *vdev, u32 vpu_addr)
ivpu_hw_reg_ipc_tx_set(vdev, vpu_addr);
}
void
ivpu_ipc_consumer_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, u32 channel)
static void
ivpu_ipc_rx_msg_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
struct ivpu_ipc_hdr *ipc_hdr, struct vpu_jsm_msg *jsm_msg)
{
struct ivpu_ipc_info *ipc = vdev->ipc;
struct ivpu_ipc_rx_msg *rx_msg;
lockdep_assert_held(&ipc->cons_lock);
lockdep_assert_irqs_disabled();
rx_msg = kzalloc(sizeof(*rx_msg), GFP_ATOMIC);
if (!rx_msg) {
ivpu_ipc_rx_mark_free(vdev, ipc_hdr, jsm_msg);
return;
}
atomic_inc(&ipc->rx_msg_count);
rx_msg->ipc_hdr = ipc_hdr;
rx_msg->jsm_msg = jsm_msg;
rx_msg->callback = cons->rx_callback;
if (rx_msg->callback) {
list_add_tail(&rx_msg->link, &ipc->cb_msg_list);
} else {
spin_lock(&cons->rx_lock);
list_add_tail(&rx_msg->link, &cons->rx_msg_list);
spin_unlock(&cons->rx_lock);
wake_up(&cons->rx_msg_wq);
}
}
static void
ivpu_ipc_rx_msg_del(struct ivpu_device *vdev, struct ivpu_ipc_rx_msg *rx_msg)
{
list_del(&rx_msg->link);
ivpu_ipc_rx_mark_free(vdev, rx_msg->ipc_hdr, rx_msg->jsm_msg);
atomic_dec(&vdev->ipc->rx_msg_count);
kfree(rx_msg);
}
void ivpu_ipc_consumer_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
u32 channel, ivpu_ipc_rx_callback_t rx_callback)
{
struct ivpu_ipc_info *ipc = vdev->ipc;
@ -148,13 +182,15 @@ ivpu_ipc_consumer_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
cons->channel = channel;
cons->tx_vpu_addr = 0;
cons->request_id = 0;
spin_lock_init(&cons->rx_msg_lock);
cons->aborted = false;
cons->rx_callback = rx_callback;
spin_lock_init(&cons->rx_lock);
INIT_LIST_HEAD(&cons->rx_msg_list);
init_waitqueue_head(&cons->rx_msg_wq);
spin_lock_irq(&ipc->cons_list_lock);
spin_lock_irq(&ipc->cons_lock);
list_add_tail(&cons->link, &ipc->cons_list);
spin_unlock_irq(&ipc->cons_list_lock);
spin_unlock_irq(&ipc->cons_lock);
}
void ivpu_ipc_consumer_del(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons)
@ -162,18 +198,14 @@ void ivpu_ipc_consumer_del(struct ivpu_device *vdev, struct ivpu_ipc_consumer *c
struct ivpu_ipc_info *ipc = vdev->ipc;
struct ivpu_ipc_rx_msg *rx_msg, *r;
spin_lock_irq(&ipc->cons_list_lock);
spin_lock_irq(&ipc->cons_lock);
list_del(&cons->link);
spin_unlock_irq(&ipc->cons_list_lock);
spin_unlock_irq(&ipc->cons_lock);
spin_lock_irq(&cons->rx_msg_lock);
list_for_each_entry_safe(rx_msg, r, &cons->rx_msg_list, link) {
list_del(&rx_msg->link);
ivpu_ipc_rx_mark_free(vdev, rx_msg->ipc_hdr, rx_msg->jsm_msg);
atomic_dec(&ipc->rx_msg_count);
kfree(rx_msg);
}
spin_unlock_irq(&cons->rx_msg_lock);
spin_lock_irq(&cons->rx_lock);
list_for_each_entry_safe(rx_msg, r, &cons->rx_msg_list, link)
ivpu_ipc_rx_msg_del(vdev, rx_msg);
spin_unlock_irq(&cons->rx_lock);
ivpu_ipc_tx_release(vdev, cons->tx_vpu_addr);
}
@ -202,52 +234,61 @@ unlock:
return ret;
}
static bool ivpu_ipc_rx_need_wakeup(struct ivpu_ipc_consumer *cons)
{
bool ret;
spin_lock_irq(&cons->rx_lock);
ret = !list_empty(&cons->rx_msg_list) || cons->aborted;
spin_unlock_irq(&cons->rx_lock);
return ret;
}
int ivpu_ipc_receive(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
struct ivpu_ipc_hdr *ipc_buf,
struct vpu_jsm_msg *ipc_payload, unsigned long timeout_ms)
struct vpu_jsm_msg *jsm_msg, unsigned long timeout_ms)
{
struct ivpu_ipc_info *ipc = vdev->ipc;
struct ivpu_ipc_rx_msg *rx_msg;
int wait_ret, ret = 0;
wait_ret = wait_event_timeout(cons->rx_msg_wq,
(IS_KTHREAD() && kthread_should_stop()) ||
!list_empty(&cons->rx_msg_list),
msecs_to_jiffies(timeout_ms));
if (drm_WARN_ONCE(&vdev->drm, cons->rx_callback, "Consumer works only in async mode\n"))
return -EINVAL;
if (IS_KTHREAD() && kthread_should_stop())
return -EINTR;
wait_ret = wait_event_timeout(cons->rx_msg_wq,
ivpu_ipc_rx_need_wakeup(cons),
msecs_to_jiffies(timeout_ms));
if (wait_ret == 0)
return -ETIMEDOUT;
spin_lock_irq(&cons->rx_msg_lock);
spin_lock_irq(&cons->rx_lock);
if (cons->aborted) {
spin_unlock_irq(&cons->rx_lock);
return -ECANCELED;
}
rx_msg = list_first_entry_or_null(&cons->rx_msg_list, struct ivpu_ipc_rx_msg, link);
if (!rx_msg) {
spin_unlock_irq(&cons->rx_msg_lock);
spin_unlock_irq(&cons->rx_lock);
return -EAGAIN;
}
list_del(&rx_msg->link);
spin_unlock_irq(&cons->rx_msg_lock);
if (ipc_buf)
memcpy(ipc_buf, rx_msg->ipc_hdr, sizeof(*ipc_buf));
if (rx_msg->jsm_msg) {
u32 size = min_t(int, rx_msg->ipc_hdr->data_size, sizeof(*ipc_payload));
u32 size = min_t(int, rx_msg->ipc_hdr->data_size, sizeof(*jsm_msg));
if (rx_msg->jsm_msg->result != VPU_JSM_STATUS_SUCCESS) {
ivpu_dbg(vdev, IPC, "IPC resp result error: %d\n", rx_msg->jsm_msg->result);
ret = -EBADMSG;
}
if (ipc_payload)
memcpy(ipc_payload, rx_msg->jsm_msg, size);
if (jsm_msg)
memcpy(jsm_msg, rx_msg->jsm_msg, size);
}
ivpu_ipc_rx_mark_free(vdev, rx_msg->ipc_hdr, rx_msg->jsm_msg);
atomic_dec(&ipc->rx_msg_count);
kfree(rx_msg);
ivpu_ipc_rx_msg_del(vdev, rx_msg);
spin_unlock_irq(&cons->rx_lock);
return ret;
}
@ -260,7 +301,7 @@ ivpu_ipc_send_receive_internal(struct ivpu_device *vdev, struct vpu_jsm_msg *req
struct ivpu_ipc_consumer cons;
int ret;
ivpu_ipc_consumer_add(vdev, &cons, channel);
ivpu_ipc_consumer_add(vdev, &cons, channel, NULL);
ret = ivpu_ipc_send(vdev, &cons, req);
if (ret) {
@ -285,23 +326,19 @@ consumer_del:
return ret;
}
int ivpu_ipc_send_receive(struct ivpu_device *vdev, struct vpu_jsm_msg *req,
enum vpu_ipc_msg_type expected_resp_type,
struct vpu_jsm_msg *resp, u32 channel,
unsigned long timeout_ms)
int ivpu_ipc_send_receive_active(struct ivpu_device *vdev, struct vpu_jsm_msg *req,
enum vpu_ipc_msg_type expected_resp, struct vpu_jsm_msg *resp,
u32 channel, unsigned long timeout_ms)
{
struct vpu_jsm_msg hb_req = { .type = VPU_JSM_MSG_QUERY_ENGINE_HB };
struct vpu_jsm_msg hb_resp;
int ret, hb_ret;
ret = ivpu_rpm_get(vdev);
if (ret < 0)
return ret;
drm_WARN_ON(&vdev->drm, pm_runtime_status_suspended(vdev->drm.dev));
ret = ivpu_ipc_send_receive_internal(vdev, req, expected_resp_type, resp,
channel, timeout_ms);
ret = ivpu_ipc_send_receive_internal(vdev, req, expected_resp, resp, channel, timeout_ms);
if (ret != -ETIMEDOUT)
goto rpm_put;
return ret;
hb_ret = ivpu_ipc_send_receive_internal(vdev, &hb_req, VPU_JSM_MSG_QUERY_ENGINE_HB_DONE,
&hb_resp, VPU_IPC_CHAN_ASYNC_CMD,
@ -311,7 +348,21 @@ int ivpu_ipc_send_receive(struct ivpu_device *vdev, struct vpu_jsm_msg *req,
ivpu_pm_schedule_recovery(vdev);
}
rpm_put:
return ret;
}
int ivpu_ipc_send_receive(struct ivpu_device *vdev, struct vpu_jsm_msg *req,
enum vpu_ipc_msg_type expected_resp, struct vpu_jsm_msg *resp,
u32 channel, unsigned long timeout_ms)
{
int ret;
ret = ivpu_rpm_get(vdev);
if (ret < 0)
return ret;
ret = ivpu_ipc_send_receive_active(vdev, req, expected_resp, resp, channel, timeout_ms);
ivpu_rpm_put(vdev);
return ret;
}
@ -329,35 +380,7 @@ ivpu_ipc_match_consumer(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons
return false;
}
static void
ivpu_ipc_dispatch(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
struct ivpu_ipc_hdr *ipc_hdr, struct vpu_jsm_msg *jsm_msg)
{
struct ivpu_ipc_info *ipc = vdev->ipc;
struct ivpu_ipc_rx_msg *rx_msg;
unsigned long flags;
lockdep_assert_held(&ipc->cons_list_lock);
rx_msg = kzalloc(sizeof(*rx_msg), GFP_ATOMIC);
if (!rx_msg) {
ivpu_ipc_rx_mark_free(vdev, ipc_hdr, jsm_msg);
return;
}
atomic_inc(&ipc->rx_msg_count);
rx_msg->ipc_hdr = ipc_hdr;
rx_msg->jsm_msg = jsm_msg;
spin_lock_irqsave(&cons->rx_msg_lock, flags);
list_add_tail(&rx_msg->link, &cons->rx_msg_list);
spin_unlock_irqrestore(&cons->rx_msg_lock, flags);
wake_up(&cons->rx_msg_wq);
}
int ivpu_ipc_irq_handler(struct ivpu_device *vdev)
void ivpu_ipc_irq_handler(struct ivpu_device *vdev, bool *wake_thread)
{
struct ivpu_ipc_info *ipc = vdev->ipc;
struct ivpu_ipc_consumer *cons;
@ -375,7 +398,7 @@ int ivpu_ipc_irq_handler(struct ivpu_device *vdev)
vpu_addr = ivpu_hw_reg_ipc_rx_addr_get(vdev);
if (vpu_addr == REG_IO_ERROR) {
ivpu_err_ratelimited(vdev, "Failed to read IPC rx addr register\n");
return -EIO;
return;
}
ipc_hdr = ivpu_to_cpu_addr(ipc->mem_rx, vpu_addr);
@ -405,15 +428,15 @@ int ivpu_ipc_irq_handler(struct ivpu_device *vdev)
}
dispatched = false;
spin_lock_irqsave(&ipc->cons_list_lock, flags);
spin_lock_irqsave(&ipc->cons_lock, flags);
list_for_each_entry(cons, &ipc->cons_list, link) {
if (ivpu_ipc_match_consumer(vdev, cons, ipc_hdr, jsm_msg)) {
ivpu_ipc_dispatch(vdev, cons, ipc_hdr, jsm_msg);
ivpu_ipc_rx_msg_add(vdev, cons, ipc_hdr, jsm_msg);
dispatched = true;
break;
}
}
spin_unlock_irqrestore(&ipc->cons_list_lock, flags);
spin_unlock_irqrestore(&ipc->cons_lock, flags);
if (!dispatched) {
ivpu_dbg(vdev, IPC, "IPC RX msg 0x%x dropped (no consumer)\n", vpu_addr);
@ -421,7 +444,28 @@ int ivpu_ipc_irq_handler(struct ivpu_device *vdev)
}
}
return 0;
if (wake_thread)
*wake_thread = !list_empty(&ipc->cb_msg_list);
}
irqreturn_t ivpu_ipc_irq_thread_handler(struct ivpu_device *vdev)
{
struct ivpu_ipc_info *ipc = vdev->ipc;
struct ivpu_ipc_rx_msg *rx_msg, *r;
struct list_head cb_msg_list;
INIT_LIST_HEAD(&cb_msg_list);
spin_lock_irq(&ipc->cons_lock);
list_splice_tail_init(&ipc->cb_msg_list, &cb_msg_list);
spin_unlock_irq(&ipc->cons_lock);
list_for_each_entry_safe(rx_msg, r, &cb_msg_list, link) {
rx_msg->callback(vdev, rx_msg->ipc_hdr, rx_msg->jsm_msg);
ivpu_ipc_rx_msg_del(vdev, rx_msg);
}
return IRQ_HANDLED;
}
int ivpu_ipc_init(struct ivpu_device *vdev)
@ -456,10 +500,10 @@ int ivpu_ipc_init(struct ivpu_device *vdev)
goto err_free_rx;
}
spin_lock_init(&ipc->cons_lock);
INIT_LIST_HEAD(&ipc->cons_list);
spin_lock_init(&ipc->cons_list_lock);
INIT_LIST_HEAD(&ipc->cb_msg_list);
drmm_mutex_init(&vdev->drm, &ipc->lock);
ivpu_ipc_reset(vdev);
return 0;
@ -472,6 +516,13 @@ err_free_tx:
void ivpu_ipc_fini(struct ivpu_device *vdev)
{
struct ivpu_ipc_info *ipc = vdev->ipc;
drm_WARN_ON(&vdev->drm, ipc->on);
drm_WARN_ON(&vdev->drm, !list_empty(&ipc->cons_list));
drm_WARN_ON(&vdev->drm, !list_empty(&ipc->cb_msg_list));
drm_WARN_ON(&vdev->drm, atomic_read(&ipc->rx_msg_count) > 0);
ivpu_ipc_mem_fini(vdev);
}
@ -488,16 +539,27 @@ void ivpu_ipc_disable(struct ivpu_device *vdev)
{
struct ivpu_ipc_info *ipc = vdev->ipc;
struct ivpu_ipc_consumer *cons, *c;
unsigned long flags;
struct ivpu_ipc_rx_msg *rx_msg, *r;
drm_WARN_ON(&vdev->drm, !list_empty(&ipc->cb_msg_list));
mutex_lock(&ipc->lock);
ipc->on = false;
mutex_unlock(&ipc->lock);
spin_lock_irqsave(&ipc->cons_list_lock, flags);
list_for_each_entry_safe(cons, c, &ipc->cons_list, link)
spin_lock_irq(&ipc->cons_lock);
list_for_each_entry_safe(cons, c, &ipc->cons_list, link) {
spin_lock(&cons->rx_lock);
if (!cons->rx_callback)
cons->aborted = true;
list_for_each_entry_safe(rx_msg, r, &cons->rx_msg_list, link)
ivpu_ipc_rx_msg_del(vdev, rx_msg);
spin_unlock(&cons->rx_lock);
wake_up(&cons->rx_msg_wq);
spin_unlock_irqrestore(&ipc->cons_list_lock, flags);
}
spin_unlock_irq(&ipc->cons_lock);
drm_WARN_ON(&vdev->drm, atomic_read(&ipc->rx_msg_count) > 0);
}
void ivpu_ipc_reset(struct ivpu_device *vdev)
@ -505,6 +567,7 @@ void ivpu_ipc_reset(struct ivpu_device *vdev)
struct ivpu_ipc_info *ipc = vdev->ipc;
mutex_lock(&ipc->lock);
drm_WARN_ON(&vdev->drm, ipc->on);
memset(ivpu_bo_vaddr(ipc->mem_tx), 0, ivpu_bo_size(ipc->mem_tx));
memset(ivpu_bo_vaddr(ipc->mem_rx), 0, ivpu_bo_size(ipc->mem_rx));

33
drivers/accel/ivpu/ivpu_ipc.h
View File

@ -42,13 +42,26 @@ struct ivpu_ipc_hdr {
u8 status;
} __packed __aligned(IVPU_IPC_ALIGNMENT);
typedef void (*ivpu_ipc_rx_callback_t)(struct ivpu_device *vdev,
struct ivpu_ipc_hdr *ipc_hdr,
struct vpu_jsm_msg *jsm_msg);
struct ivpu_ipc_rx_msg {
struct list_head link;
struct ivpu_ipc_hdr *ipc_hdr;
struct vpu_jsm_msg *jsm_msg;
ivpu_ipc_rx_callback_t callback;
};
struct ivpu_ipc_consumer {
struct list_head link;
u32 channel;
u32 tx_vpu_addr;
u32 request_id;
bool aborted;
ivpu_ipc_rx_callback_t rx_callback;
spinlock_t rx_msg_lock; /* Protects rx_msg_list */
spinlock_t rx_lock; /* Protects rx_msg_list and aborted */
struct list_head rx_msg_list;
wait_queue_head_t rx_msg_wq;
};
@ -60,8 +73,9 @@ struct ivpu_ipc_info {
atomic_t rx_msg_count;
spinlock_t cons_list_lock; /* Protects cons_list */
spinlock_t cons_lock; /* Protects cons_list and cb_msg_list */
struct list_head cons_list;
struct list_head cb_msg_list;
atomic_t request_id;
struct mutex lock; /* Lock on status */
@ -75,19 +89,22 @@ void ivpu_ipc_enable(struct ivpu_device *vdev);
void ivpu_ipc_disable(struct ivpu_device *vdev);
void ivpu_ipc_reset(struct ivpu_device *vdev);
int ivpu_ipc_irq_handler(struct ivpu_device *vdev);
void ivpu_ipc_irq_handler(struct ivpu_device *vdev, bool *wake_thread);
irqreturn_t ivpu_ipc_irq_thread_handler(struct ivpu_device *vdev);
void ivpu_ipc_consumer_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
u32 channel);
u32 channel, ivpu_ipc_rx_callback_t callback);
void ivpu_ipc_consumer_del(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons);
int ivpu_ipc_receive(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
struct ivpu_ipc_hdr *ipc_buf, struct vpu_jsm_msg *ipc_payload,
struct ivpu_ipc_hdr *ipc_buf, struct vpu_jsm_msg *jsm_msg,
unsigned long timeout_ms);
int ivpu_ipc_send_receive_active(struct ivpu_device *vdev, struct vpu_jsm_msg *req,
enum vpu_ipc_msg_type expected_resp, struct vpu_jsm_msg *resp,
u32 channel, unsigned long timeout_ms);
int ivpu_ipc_send_receive(struct ivpu_device *vdev, struct vpu_jsm_msg *req,
enum vpu_ipc_msg_type expected_resp_type,
struct vpu_jsm_msg *resp, u32 channel,
unsigned long timeout_ms);
enum vpu_ipc_msg_type expected_resp, struct vpu_jsm_msg *resp,
u32 channel, unsigned long timeout_ms);
#endif /* __IVPU_IPC_H__ */

103
drivers/accel/ivpu/ivpu_job.c
View File

@ -7,7 +7,6 @@
#include <linux/bitfield.h>
#include <linux/highmem.h>
#include <linux/kthread.h>
#include <linux/pci.h>
#include <linux/module.h>
#include <uapi/drm/ivpu_accel.h>
@ -24,10 +23,6 @@
#define JOB_ID_CONTEXT_MASK GENMASK(31, 8)
#define JOB_MAX_BUFFER_COUNT 65535
static unsigned int ivpu_tdr_timeout_ms;
module_param_named(tdr_timeout_ms, ivpu_tdr_timeout_ms, uint, 0644);
MODULE_PARM_DESC(tdr_timeout_ms, "Timeout for device hang detection, in milliseconds, 0 - default");
static void ivpu_cmdq_ring_db(struct ivpu_device *vdev, struct ivpu_cmdq *cmdq)
{
ivpu_hw_reg_db_set(vdev, cmdq->db_id);
@ -196,6 +191,8 @@ static int ivpu_cmdq_push_job(struct ivpu_cmdq *cmdq, struct ivpu_job *job)
entry->batch_buf_addr = job->cmd_buf_vpu_addr;
entry->job_id = job->job_id;
entry->flags = 0;
if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_SUBMISSION))
entry->flags = VPU_JOB_FLAGS_NULL_SUBMISSION_MASK;
wmb(); /* Ensure that tail is updated after filling entry */
header->tail = next_entry;
wmb(); /* Flush WC buffer for jobq header */
@ -264,7 +261,7 @@ static void job_release(struct kref *ref)
for (i = 0; i < job->bo_count; i++)
if (job->bos[i])
drm_gem_object_put(&job->bos[i]->base);
drm_gem_object_put(&job->bos[i]->base.base);
dma_fence_put(job->done_fence);
ivpu_file_priv_put(&job->file_priv);
@ -340,23 +337,12 @@ static int ivpu_job_done(struct ivpu_device *vdev, u32 job_id, u32 job_status)
ivpu_dbg(vdev, JOB, "Job complete: id %3u ctx %2d engine %d status 0x%x\n",
job->job_id, job->file_priv->ctx.id, job->engine_idx, job_status);
ivpu_stop_job_timeout_detection(vdev);
job_put(job);
return 0;
}
static void ivpu_job_done_message(struct ivpu_device *vdev, void *msg)
{
struct vpu_ipc_msg_payload_job_done *payload;
struct vpu_jsm_msg *job_ret_msg = msg;
int ret;
payload = (struct vpu_ipc_msg_payload_job_done *)&job_ret_msg->payload;
ret = ivpu_job_done(vdev, payload->job_id, payload->job_status);
if (ret)
ivpu_err(vdev, "Failed to finish job %d: %d\n", payload->job_id, ret);
}
void ivpu_jobs_abort_all(struct ivpu_device *vdev)
{
struct ivpu_job *job;
@ -398,11 +384,13 @@ static int ivpu_direct_job_submission(struct ivpu_job *job)
if (ret)
goto err_xa_erase;
ivpu_start_job_timeout_detection(vdev);
ivpu_dbg(vdev, JOB, "Job submitted: id %3u addr 0x%llx ctx %2d engine %d next %d\n",
job->job_id, job->cmd_buf_vpu_addr, file_priv->ctx.id,
job->engine_idx, cmdq->jobq->header.tail);
if (ivpu_test_mode == IVPU_TEST_MODE_NULL_HW) {
if (ivpu_test_mode & IVPU_TEST_MODE_NULL_HW) {
ivpu_job_done(vdev, job->job_id, VPU_JSM_STATUS_SUCCESS);
cmdq->jobq->header.head = cmdq->jobq->header.tail;
wmb(); /* Flush WC buffer for jobq header */
@ -448,7 +436,7 @@ ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32
}
bo = job->bos[CMD_BUF_IDX];
if (!dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_READ)) {
if (!dma_resv_test_signaled(bo->base.base.resv, DMA_RESV_USAGE_READ)) {
ivpu_warn(vdev, "Buffer is already in use\n");
return -EBUSY;
}
@ -468,7 +456,7 @@ ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32
}
for (i = 0; i < buf_count; i++) {
ret = dma_resv_reserve_fences(job->bos[i]->base.resv, 1);
ret = dma_resv_reserve_fences(job->bos[i]->base.base.resv, 1);
if (ret) {
ivpu_warn(vdev, "Failed to reserve fences: %d\n", ret);
goto unlock_reservations;
@ -477,7 +465,7 @@ ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32
for (i = 0; i < buf_count; i++) {
usage = (i == CMD_BUF_IDX) ? DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_BOOKKEEP;
dma_resv_add_fence(job->bos[i]->base.resv, job->done_fence, usage);
dma_resv_add_fence(job->bos[i]->base.base.resv, job->done_fence, usage);
}
unlock_reservations:
@ -562,61 +550,36 @@ free_handles:
return ret;
}
static int ivpu_job_done_thread(void *arg)
static void
ivpu_job_done_callback(struct ivpu_device *vdev, struct ivpu_ipc_hdr *ipc_hdr,
struct vpu_jsm_msg *jsm_msg)
{
struct ivpu_device *vdev = (struct ivpu_device *)arg;
struct ivpu_ipc_consumer cons;
struct vpu_jsm_msg jsm_msg;
bool jobs_submitted;
unsigned int timeout;
struct vpu_ipc_msg_payload_job_done *payload;
int ret;
ivpu_dbg(vdev, JOB, "Started %s\n", __func__);
ivpu_ipc_consumer_add(vdev, &cons, VPU_IPC_CHAN_JOB_RET);
while (!kthread_should_stop()) {
timeout = ivpu_tdr_timeout_ms ? ivpu_tdr_timeout_ms : vdev->timeout.tdr;
jobs_submitted = !xa_empty(&vdev->submitted_jobs_xa);
ret = ivpu_ipc_receive(vdev, &cons, NULL, &jsm_msg, timeout);
if (!ret) {
ivpu_job_done_message(vdev, &jsm_msg);
} else if (ret == -ETIMEDOUT) {
if (jobs_submitted && !xa_empty(&vdev->submitted_jobs_xa)) {
ivpu_err(vdev, "TDR detected, timeout %d ms", timeout);
ivpu_hw_diagnose_failure(vdev);
ivpu_pm_schedule_recovery(vdev);
}
}
if (!jsm_msg) {
ivpu_err(vdev, "IPC message has no JSM payload\n");
return;
}
ivpu_ipc_consumer_del(vdev, &cons);
ivpu_jobs_abort_all(vdev);
ivpu_dbg(vdev, JOB, "Stopped %s\n", __func__);
return 0;
}
int ivpu_job_done_thread_init(struct ivpu_device *vdev)
{
struct task_struct *thread;
thread = kthread_run(&ivpu_job_done_thread, (void *)vdev, "ivpu_job_done_thread");
if (IS_ERR(thread)) {
ivpu_err(vdev, "Failed to start job completion thread\n");
return -EIO;
if (jsm_msg->result != VPU_JSM_STATUS_SUCCESS) {
ivpu_err(vdev, "Invalid JSM message result: %d\n", jsm_msg->result);
return;
}
get_task_struct(thread);
wake_up_process(thread);
vdev->job_done_thread = thread;
return 0;
payload = (struct vpu_ipc_msg_payload_job_done *)&jsm_msg->payload;
ret = ivpu_job_done(vdev, payload->job_id, payload->job_status);
if (!ret && !xa_empty(&vdev->submitted_jobs_xa))
ivpu_start_job_timeout_detection(vdev);
}
void ivpu_job_done_thread_fini(struct ivpu_device *vdev)
void ivpu_job_done_consumer_init(struct ivpu_device *vdev)
{
kthread_stop_put(vdev->job_done_thread);
ivpu_ipc_consumer_add(vdev, &vdev->job_done_consumer,
VPU_IPC_CHAN_JOB_RET, ivpu_job_done_callback);
}
void ivpu_job_done_consumer_fini(struct ivpu_device *vdev)
{
ivpu_ipc_consumer_del(vdev, &vdev->job_done_consumer);
}

4
drivers/accel/ivpu/ivpu_job.h
View File

@ -59,8 +59,8 @@ int ivpu_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
void ivpu_cmdq_release_all(struct ivpu_file_priv *file_priv);
void ivpu_cmdq_reset_all_contexts(struct ivpu_device *vdev);
int ivpu_job_done_thread_init(struct ivpu_device *vdev);
void ivpu_job_done_thread_fini(struct ivpu_device *vdev);
void ivpu_job_done_consumer_init(struct ivpu_device *vdev);
void ivpu_job_done_consumer_fini(struct ivpu_device *vdev);
void ivpu_jobs_abort_all(struct ivpu_device *vdev);

38
drivers/accel/ivpu/ivpu_jsm_msg.c
View File

@ -4,6 +4,7 @@
*/
#include "ivpu_drv.h"
#include "ivpu_hw.h"
#include "ivpu_ipc.h"
#include "ivpu_jsm_msg.h"
@ -36,6 +37,17 @@ const char *ivpu_jsm_msg_type_to_str(enum vpu_ipc_msg_type type)
IVPU_CASE_TO_STR(VPU_JSM_MSG_DESTROY_CMD_QUEUE);
IVPU_CASE_TO_STR(VPU_JSM_MSG_SET_CONTEXT_SCHED_PROPERTIES);
IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_REGISTER_DB);
IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_RESUME_CMDQ);
IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_SUSPEND_CMDQ);
IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_RESUME_CMDQ_RSP);
IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_SUSPEND_CMDQ_DONE);
IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG);
IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG_RSP);
IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_SCHEDULING_LOG_NOTIFICATION);
IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_ENGINE_RESUME);
IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_RESUME_ENGINE_DONE);
IVPU_CASE_TO_STR(VPU_JSM_MSG_STATE_DUMP);
IVPU_CASE_TO_STR(VPU_JSM_MSG_STATE_DUMP_RSP);
IVPU_CASE_TO_STR(VPU_JSM_MSG_BLOB_DEINIT);
IVPU_CASE_TO_STR(VPU_JSM_MSG_DYNDBG_CONTROL);
IVPU_CASE_TO_STR(VPU_JSM_MSG_JOB_DONE);
@ -65,6 +77,12 @@ const char *ivpu_jsm_msg_type_to_str(enum vpu_ipc_msg_type type)
IVPU_CASE_TO_STR(VPU_JSM_MSG_SET_CONTEXT_SCHED_PROPERTIES_RSP);
IVPU_CASE_TO_STR(VPU_JSM_MSG_BLOB_DEINIT_DONE);
IVPU_CASE_TO_STR(VPU_JSM_MSG_DYNDBG_CONTROL_RSP);
IVPU_CASE_TO_STR(VPU_JSM_MSG_PWR_D0I3_ENTER);
IVPU_CASE_TO_STR(VPU_JSM_MSG_PWR_D0I3_ENTER_DONE);
IVPU_CASE_TO_STR(VPU_JSM_MSG_DCT_ENABLE);
IVPU_CASE_TO_STR(VPU_JSM_MSG_DCT_ENABLE_DONE);
IVPU_CASE_TO_STR(VPU_JSM_MSG_DCT_DISABLE);
IVPU_CASE_TO_STR(VPU_JSM_MSG_DCT_DISABLE_DONE);
}
#undef IVPU_CASE_TO_STR
@ -243,3 +261,23 @@ int ivpu_jsm_context_release(struct ivpu_device *vdev, u32 host_ssid)
return ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_SSID_RELEASE_DONE, &resp,
VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm);
}
int ivpu_jsm_pwr_d0i3_enter(struct ivpu_device *vdev)
{
struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_PWR_D0I3_ENTER };
struct vpu_jsm_msg resp;
int ret;
if (IVPU_WA(disable_d0i3_msg))
return 0;
req.payload.pwr_d0i3_enter.send_response = 1;
ret = ivpu_ipc_send_receive_active(vdev, &req, VPU_JSM_MSG_PWR_D0I3_ENTER_DONE,
&resp, VPU_IPC_CHAN_GEN_CMD,
vdev->timeout.d0i3_entry_msg);
if (ret)
return ret;
return ivpu_hw_wait_for_idle(vdev);
}

1
drivers/accel/ivpu/ivpu_jsm_msg.h
View File

@ -22,4 +22,5 @@ int ivpu_jsm_trace_get_capability(struct ivpu_device *vdev, u32 *trace_destinati
int ivpu_jsm_trace_set_config(struct ivpu_device *vdev, u32 trace_level, u32 trace_destination_mask,
u64 trace_hw_component_mask);
int ivpu_jsm_context_release(struct ivpu_device *vdev, u32 host_ssid);
int ivpu_jsm_pwr_d0i3_enter(struct ivpu_device *vdev);
#endif

44
drivers/accel/ivpu/ivpu_mmu.c
View File

@ -230,7 +230,12 @@
(REG_FLD(IVPU_MMU_REG_GERROR, MSI_PRIQ_ABT)) | \
(REG_FLD(IVPU_MMU_REG_GERROR, MSI_ABT)))
static char *ivpu_mmu_event_to_str(u32 cmd)
#define IVPU_MMU_CERROR_NONE 0x0
#define IVPU_MMU_CERROR_ILL 0x1
#define IVPU_MMU_CERROR_ABT 0x2
#define IVPU_MMU_CERROR_ATC_INV_SYNC 0x3
static const char *ivpu_mmu_event_to_str(u32 cmd)
{
switch (cmd) {
case IVPU_MMU_EVT_F_UUT:
@ -276,6 +281,22 @@ static char *ivpu_mmu_event_to_str(u32 cmd)
}
}
static const char *ivpu_mmu_cmdq_err_to_str(u32 err)
{
switch (err) {
case IVPU_MMU_CERROR_NONE:
return "No CMDQ Error";
case IVPU_MMU_CERROR_ILL:
return "Illegal command";
case IVPU_MMU_CERROR_ABT:
return "External abort on CMDQ read";
case IVPU_MMU_CERROR_ATC_INV_SYNC:
return "Sync failed to complete ATS invalidation";
default:
return "Unknown CMDQ Error";
}
}
static void ivpu_mmu_config_check(struct ivpu_device *vdev)
{
u32 val_ref;
@ -479,10 +500,7 @@ static int ivpu_mmu_cmdq_sync(struct ivpu_device *vdev)
u64 val;
int ret;
val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_SYNC) |
FIELD_PREP(IVPU_MMU_CMD_SYNC_0_CS, 0x2) |
FIELD_PREP(IVPU_MMU_CMD_SYNC_0_MSH, 0x3) |
FIELD_PREP(IVPU_MMU_CMD_SYNC_0_MSI_ATTR, 0xf);
val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_SYNC);
ret = ivpu_mmu_cmdq_cmd_write(vdev, "SYNC", val, 0);
if (ret)
@ -492,8 +510,15 @@ static int ivpu_mmu_cmdq_sync(struct ivpu_device *vdev)
REGV_WR32(IVPU_MMU_REG_CMDQ_PROD, q->prod);
ret = ivpu_mmu_cmdq_wait_for_cons(vdev);
if (ret)
ivpu_err(vdev, "Timed out waiting for consumer: %d\n", ret);
if (ret) {
u32 err;
val = REGV_RD32(IVPU_MMU_REG_CMDQ_CONS);
err = REG_GET_FLD(IVPU_MMU_REG_CMDQ_CONS, ERR, val);
ivpu_err(vdev, "Timed out waiting for MMU consumer: %d, error: %s\n", ret,
ivpu_mmu_cmdq_err_to_str(err));
}
return ret;
}
@ -750,9 +775,12 @@ int ivpu_mmu_init(struct ivpu_device *vdev)
ivpu_dbg(vdev, MMU, "Init..\n");
drmm_mutex_init(&vdev->drm, &mmu->lock);
ivpu_mmu_config_check(vdev);
ret = drmm_mutex_init(&vdev->drm, &mmu->lock);
if (ret)
return ret;
ret = ivpu_mmu_structs_alloc(vdev);
if (ret)
return ret;

153
drivers/accel/ivpu/ivpu_mmu_context.c
View File

@ -5,6 +5,9 @@
#include <linux/bitfield.h>
#include <linux/highmem.h>
#include <linux/set_memory.h>
#include <drm/drm_cache.h>
#include "ivpu_drv.h"
#include "ivpu_hw.h"
@ -39,12 +42,57 @@
#define IVPU_MMU_ENTRY_MAPPED (IVPU_MMU_ENTRY_FLAG_AF | IVPU_MMU_ENTRY_FLAG_USER | \
IVPU_MMU_ENTRY_FLAG_NG | IVPU_MMU_ENTRY_VALID)
static void *ivpu_pgtable_alloc_page(struct ivpu_device *vdev, dma_addr_t *dma)
{
dma_addr_t dma_addr;
struct page *page;
void *cpu;
page = alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
if (!page)
return NULL;
set_pages_array_wc(&page, 1);
dma_addr = dma_map_page(vdev->drm.dev, page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
if (dma_mapping_error(vdev->drm.dev, dma_addr))
goto err_free_page;
cpu = vmap(&page, 1, VM_MAP, pgprot_writecombine(PAGE_KERNEL));
if (!cpu)
goto err_dma_unmap_page;
*dma = dma_addr;
return cpu;
err_dma_unmap_page:
dma_unmap_page(vdev->drm.dev, dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
err_free_page:
put_page(page);
return NULL;
}
static void ivpu_pgtable_free_page(struct ivpu_device *vdev, u64 *cpu_addr, dma_addr_t dma_addr)
{
struct page *page;
if (cpu_addr) {
page = vmalloc_to_page(cpu_addr);
vunmap(cpu_addr);
dma_unmap_page(vdev->drm.dev, dma_addr & ~IVPU_MMU_ENTRY_FLAGS_MASK, PAGE_SIZE,
DMA_BIDIRECTIONAL);
set_pages_array_wb(&page, 1);
put_page(page);
}
}
static int ivpu_mmu_pgtable_init(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)
{
dma_addr_t pgd_dma;
pgtable->pgd_dma_ptr = dma_alloc_coherent(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pgd_dma,
GFP_KERNEL);
pgtable->pgd_dma_ptr = ivpu_pgtable_alloc_page(vdev, &pgd_dma);
if (!pgtable->pgd_dma_ptr)
return -ENOMEM;
@ -53,13 +101,6 @@ static int ivpu_mmu_pgtable_init(struct ivpu_device *vdev, struct ivpu_mmu_pgtab
return 0;
}
static void ivpu_mmu_pgtable_free(struct ivpu_device *vdev, u64 *cpu_addr, dma_addr_t dma_addr)
{
if (cpu_addr)
dma_free_coherent(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, cpu_addr,
dma_addr & ~IVPU_MMU_ENTRY_FLAGS_MASK);
}
static void ivpu_mmu_pgtables_free(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)
{
int pgd_idx, pud_idx, pmd_idx;
@ -84,19 +125,19 @@ static void ivpu_mmu_pgtables_free(struct ivpu_device *vdev, struct ivpu_mmu_pgt
pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx];
pte_dma = pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx];
ivpu_mmu_pgtable_free(vdev, pte_dma_ptr, pte_dma);
ivpu_pgtable_free_page(vdev, pte_dma_ptr, pte_dma);
}
kfree(pgtable->pte_ptrs[pgd_idx][pud_idx]);
ivpu_mmu_pgtable_free(vdev, pmd_dma_ptr, pmd_dma);
ivpu_pgtable_free_page(vdev, pmd_dma_ptr, pmd_dma);
}
kfree(pgtable->pmd_ptrs[pgd_idx]);
kfree(pgtable->pte_ptrs[pgd_idx]);
ivpu_mmu_pgtable_free(vdev, pud_dma_ptr, pud_dma);
ivpu_pgtable_free_page(vdev, pud_dma_ptr, pud_dma);
}
ivpu_mmu_pgtable_free(vdev, pgtable->pgd_dma_ptr, pgtable->pgd_dma);
ivpu_pgtable_free_page(vdev, pgtable->pgd_dma_ptr, pgtable->pgd_dma);
}
static u64*
@ -108,7 +149,7 @@ ivpu_mmu_ensure_pud(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable,
if (pud_dma_ptr)
return pud_dma_ptr;
pud_dma_ptr = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pud_dma, GFP_KERNEL);
pud_dma_ptr = ivpu_pgtable_alloc_page(vdev, &pud_dma);
if (!pud_dma_ptr)
return NULL;
@ -131,7 +172,7 @@ err_free_pmd_ptrs:
kfree(pgtable->pmd_ptrs[pgd_idx]);
err_free_pud_dma_ptr:
ivpu_mmu_pgtable_free(vdev, pud_dma_ptr, pud_dma);
ivpu_pgtable_free_page(vdev, pud_dma_ptr, pud_dma);
return NULL;
}
@ -145,7 +186,7 @@ ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable,
if (pmd_dma_ptr)
return pmd_dma_ptr;
pmd_dma_ptr = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pmd_dma, GFP_KERNEL);
pmd_dma_ptr = ivpu_pgtable_alloc_page(vdev, &pmd_dma);
if (!pmd_dma_ptr)
return NULL;
@ -160,7 +201,7 @@ ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable,
return pmd_dma_ptr;
err_free_pmd_dma_ptr:
ivpu_mmu_pgtable_free(vdev, pmd_dma_ptr, pmd_dma);
ivpu_pgtable_free_page(vdev, pmd_dma_ptr, pmd_dma);
return NULL;
}
@ -174,7 +215,7 @@ ivpu_mmu_ensure_pte(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable,
if (pte_dma_ptr)
return pte_dma_ptr;
pte_dma_ptr = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pte_dma, GFP_KERNEL);
pte_dma_ptr = ivpu_pgtable_alloc_page(vdev, &pte_dma);
if (!pte_dma_ptr)
return NULL;
@ -249,38 +290,6 @@ static void ivpu_mmu_context_unmap_page(struct ivpu_mmu_context *ctx, u64 vpu_ad
ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] = IVPU_MMU_ENTRY_INVALID;
}
static void
ivpu_mmu_context_flush_page_tables(struct ivpu_mmu_context *ctx, u64 vpu_addr, size_t size)
{
struct ivpu_mmu_pgtable *pgtable = &ctx->pgtable;
u64 end_addr = vpu_addr + size;
/* Align to PMD entry (2 MB) */
vpu_addr &= ~(IVPU_MMU_PTE_MAP_SIZE - 1);
while (vpu_addr < end_addr) {
int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
u64 pud_end = (pgd_idx + 1) * (u64)IVPU_MMU_PUD_MAP_SIZE;
while (vpu_addr < end_addr && vpu_addr < pud_end) {
int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);
u64 pmd_end = (pud_idx + 1) * (u64)IVPU_MMU_PMD_MAP_SIZE;
while (vpu_addr < end_addr && vpu_addr < pmd_end) {
int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
clflush_cache_range(pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx],
IVPU_MMU_PGTABLE_SIZE);
vpu_addr += IVPU_MMU_PTE_MAP_SIZE;
}
clflush_cache_range(pgtable->pmd_ptrs[pgd_idx][pud_idx],
IVPU_MMU_PGTABLE_SIZE);
}
clflush_cache_range(pgtable->pud_ptrs[pgd_idx], IVPU_MMU_PGTABLE_SIZE);
}
clflush_cache_range(pgtable->pgd_dma_ptr, IVPU_MMU_PGTABLE_SIZE);
}
static int
ivpu_mmu_context_map_pages(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
u64 vpu_addr, dma_addr_t dma_addr, size_t size, u64 prot)
@ -327,6 +336,9 @@ ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
u64 prot;
u64 i;
if (drm_WARN_ON(&vdev->drm, !ctx))
return -EINVAL;
if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE))
return -EINVAL;
@ -349,10 +361,11 @@ ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
mutex_unlock(&ctx->lock);
return ret;
}
ivpu_mmu_context_flush_page_tables(ctx, vpu_addr, size);
vpu_addr += size;
}
/* Ensure page table modifications are flushed from wc buffers to memory */
wmb();
mutex_unlock(&ctx->lock);
ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id);
@ -369,8 +382,8 @@ ivpu_mmu_context_unmap_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ct
int ret;
u64 i;
if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE))
ivpu_warn(vdev, "Unaligned vpu_addr: 0x%llx\n", vpu_addr);
if (drm_WARN_ON(&vdev->drm, !ctx))
return;
mutex_lock(&ctx->lock);
@ -378,10 +391,11 @@ ivpu_mmu_context_unmap_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ct
size_t size = sg_dma_len(sg) + sg->offset;
ivpu_mmu_context_unmap_pages(ctx, vpu_addr, size);
ivpu_mmu_context_flush_page_tables(ctx, vpu_addr, size);
vpu_addr += size;
}
/* Ensure page table modifications are flushed from wc buffers to memory */
wmb();
mutex_unlock(&ctx->lock);
ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id);
@ -390,28 +404,34 @@ ivpu_mmu_context_unmap_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ct
}
int
ivpu_mmu_context_insert_node_locked(struct ivpu_mmu_context *ctx,
const struct ivpu_addr_range *range,
u64 size, struct drm_mm_node *node)
ivpu_mmu_context_insert_node(struct ivpu_mmu_context *ctx, const struct ivpu_addr_range *range,
u64 size, struct drm_mm_node *node)
{
lockdep_assert_held(&ctx->lock);
int ret;
WARN_ON(!range);
mutex_lock(&ctx->lock);
if (!ivpu_disable_mmu_cont_pages && size >= IVPU_MMU_CONT_PAGES_SIZE) {
if (!drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_CONT_PAGES_SIZE, 0,
range->start, range->end, DRM_MM_INSERT_BEST))
return 0;
ret = drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_CONT_PAGES_SIZE, 0,
range->start, range->end, DRM_MM_INSERT_BEST);
if (!ret)
goto unlock;
}
return drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_PAGE_SIZE, 0,
range->start, range->end, DRM_MM_INSERT_BEST);
ret = drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_PAGE_SIZE, 0,
range->start, range->end, DRM_MM_INSERT_BEST);
unlock:
mutex_unlock(&ctx->lock);
return ret;
}
void
ivpu_mmu_context_remove_node_locked(struct ivpu_mmu_context *ctx, struct drm_mm_node *node)
ivpu_mmu_context_remove_node(struct ivpu_mmu_context *ctx, struct drm_mm_node *node)
{
lockdep_assert_held(&ctx->lock);
mutex_lock(&ctx->lock);
drm_mm_remove_node(node);
mutex_unlock(&ctx->lock);
}
static int
@ -421,7 +441,6 @@ ivpu_mmu_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u3
int ret;
mutex_init(&ctx->lock);
INIT_LIST_HEAD(&ctx->bo_list);
ret = ivpu_mmu_pgtable_init(vdev, &ctx->pgtable);
if (ret) {

11
drivers/accel/ivpu/ivpu_mmu_context.h
View File

@ -23,10 +23,9 @@ struct ivpu_mmu_pgtable {
};
struct ivpu_mmu_context {
struct mutex lock; /* protects: mm, pgtable, bo_list */
struct mutex lock; /* Protects: mm, pgtable */
struct drm_mm mm;
struct ivpu_mmu_pgtable pgtable;
struct list_head bo_list;
u32 id;
};
@ -39,11 +38,9 @@ int ivpu_mmu_user_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context
void ivpu_mmu_user_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx);
void ivpu_mmu_user_context_mark_invalid(struct ivpu_device *vdev, u32 ssid);
int ivpu_mmu_context_insert_node_locked(struct ivpu_mmu_context *ctx,
const struct ivpu_addr_range *range,
u64 size, struct drm_mm_node *node);
void ivpu_mmu_context_remove_node_locked(struct ivpu_mmu_context *ctx,
struct drm_mm_node *node);
int ivpu_mmu_context_insert_node(struct ivpu_mmu_context *ctx, const struct ivpu_addr_range *range,
u64 size, struct drm_mm_node *node);
void ivpu_mmu_context_remove_node(struct ivpu_mmu_context *ctx, struct drm_mm_node *node);
int ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
u64 vpu_addr, struct sg_table *sgt, bool llc_coherent);

72
drivers/accel/ivpu/ivpu_pm.c
View File

@ -15,6 +15,7 @@
#include "ivpu_fw.h"
#include "ivpu_ipc.h"
#include "ivpu_job.h"
#include "ivpu_jsm_msg.h"
#include "ivpu_mmu.h"
#include "ivpu_pm.h"
@ -22,6 +23,10 @@ static bool ivpu_disable_recovery;
module_param_named_unsafe(disable_recovery, ivpu_disable_recovery, bool, 0644);
MODULE_PARM_DESC(disable_recovery, "Disables recovery when VPU hang is detected");
static unsigned long ivpu_tdr_timeout_ms;
module_param_named(tdr_timeout_ms, ivpu_tdr_timeout_ms, ulong, 0644);
MODULE_PARM_DESC(tdr_timeout_ms, "Timeout for device hang detection, in milliseconds, 0 - default");
#define PM_RESCHEDULE_LIMIT 5
static void ivpu_pm_prepare_cold_boot(struct ivpu_device *vdev)
@ -69,27 +74,31 @@ retry:
ret = ivpu_hw_power_up(vdev);
if (ret) {
ivpu_err(vdev, "Failed to power up HW: %d\n", ret);
return ret;
goto err_power_down;
}
ret = ivpu_mmu_enable(vdev);
if (ret) {
ivpu_err(vdev, "Failed to resume MMU: %d\n", ret);
ivpu_hw_power_down(vdev);
return ret;
goto err_power_down;
}
ret = ivpu_boot(vdev);
if (ret) {
ivpu_mmu_disable(vdev);
ivpu_hw_power_down(vdev);
if (!ivpu_fw_is_cold_boot(vdev)) {
ivpu_warn(vdev, "Failed to resume the FW: %d. Retrying cold boot..\n", ret);
ivpu_pm_prepare_cold_boot(vdev);
goto retry;
} else {
ivpu_err(vdev, "Failed to resume the FW: %d\n", ret);
}
if (ret)
goto err_mmu_disable;
return 0;
err_mmu_disable:
ivpu_mmu_disable(vdev);
err_power_down:
ivpu_hw_power_down(vdev);
if (!ivpu_fw_is_cold_boot(vdev)) {
ivpu_pm_prepare_cold_boot(vdev);
goto retry;
} else {
ivpu_err(vdev, "Failed to resume the FW: %d\n", ret);
}
return ret;
@ -136,6 +145,31 @@ void ivpu_pm_schedule_recovery(struct ivpu_device *vdev)
}
}
static void ivpu_job_timeout_work(struct work_struct *work)
{
struct ivpu_pm_info *pm = container_of(work, struct ivpu_pm_info, job_timeout_work.work);
struct ivpu_device *vdev = pm->vdev;
unsigned long timeout_ms = ivpu_tdr_timeout_ms ? ivpu_tdr_timeout_ms : vdev->timeout.tdr;
ivpu_err(vdev, "TDR detected, timeout %lu ms", timeout_ms);
ivpu_hw_diagnose_failure(vdev);
ivpu_pm_schedule_recovery(vdev);
}
void ivpu_start_job_timeout_detection(struct ivpu_device *vdev)
{
unsigned long timeout_ms = ivpu_tdr_timeout_ms ? ivpu_tdr_timeout_ms : vdev->timeout.tdr;
/* No-op if already queued */
queue_delayed_work(system_wq, &vdev->pm->job_timeout_work, msecs_to_jiffies(timeout_ms));
}
void ivpu_stop_job_timeout_detection(struct ivpu_device *vdev)
{
cancel_delayed_work_sync(&vdev->pm->job_timeout_work);
}
int ivpu_pm_suspend_cb(struct device *dev)
{
struct drm_device *drm = dev_get_drvdata(dev);
@ -153,6 +187,8 @@ int ivpu_pm_suspend_cb(struct device *dev)
}
}
ivpu_jsm_pwr_d0i3_enter(vdev);
ivpu_suspend(vdev);
ivpu_pm_prepare_warm_boot(vdev);
@ -188,6 +224,7 @@ int ivpu_pm_runtime_suspend_cb(struct device *dev)
{
struct drm_device *drm = dev_get_drvdata(dev);
struct ivpu_device *vdev = to_ivpu_device(drm);
bool hw_is_idle = true;
int ret;
ivpu_dbg(vdev, PM, "Runtime suspend..\n");
@ -200,11 +237,16 @@ int ivpu_pm_runtime_suspend_cb(struct device *dev)
return -EAGAIN;
}
if (!vdev->pm->suspend_reschedule_counter)
hw_is_idle = false;
else if (ivpu_jsm_pwr_d0i3_enter(vdev))
hw_is_idle = false;
ret = ivpu_suspend(vdev);
if (ret)
ivpu_err(vdev, "Failed to set suspend VPU: %d\n", ret);
if (!vdev->pm->suspend_reschedule_counter) {
if (!hw_is_idle) {
ivpu_warn(vdev, "VPU failed to enter idle, force suspended.\n");
ivpu_pm_prepare_cold_boot(vdev);
} else {
@ -304,6 +346,7 @@ void ivpu_pm_init(struct ivpu_device *vdev)
atomic_set(&pm->in_reset, 0);
INIT_WORK(&pm->recovery_work, ivpu_pm_recovery_work);
INIT_DELAYED_WORK(&pm->job_timeout_work, ivpu_job_timeout_work);
if (ivpu_disable_recovery)
delay = -1;
@ -318,6 +361,7 @@ void ivpu_pm_init(struct ivpu_device *vdev)
void ivpu_pm_cancel_recovery(struct ivpu_device *vdev)
{
drm_WARN_ON(&vdev->drm, delayed_work_pending(&vdev->pm->job_timeout_work));
cancel_work_sync(&vdev->pm->recovery_work);
}

3
drivers/accel/ivpu/ivpu_pm.h
View File

@ -12,6 +12,7 @@ struct ivpu_device;
struct ivpu_pm_info {
struct ivpu_device *vdev;
struct delayed_work job_timeout_work;
struct work_struct recovery_work;
atomic_t in_reset;
atomic_t reset_counter;
@ -37,5 +38,7 @@ int __must_check ivpu_rpm_get_if_active(struct ivpu_device *vdev);
void ivpu_rpm_put(struct ivpu_device *vdev);
void ivpu_pm_schedule_recovery(struct ivpu_device *vdev);
void ivpu_start_job_timeout_detection(struct ivpu_device *vdev);
void ivpu_stop_job_timeout_detection(struct ivpu_device *vdev);
#endif /* __IVPU_PM_H__ */

90
drivers/accel/ivpu/vpu_boot_api.h
View File

@ -11,7 +11,10 @@
* The bellow values will be used to construct the version info this way:
* fw_bin_header->api_version[VPU_BOOT_API_VER_ID] = (VPU_BOOT_API_VER_MAJOR << 16) |
* VPU_BOOT_API_VER_MINOR;
* VPU_BOOT_API_VER_PATCH will be ignored. KMD and compatibility is not affected if this changes.
* VPU_BOOT_API_VER_PATCH will be ignored. KMD and compatibility is not affected if this changes
* This information is collected by using vpuip_2/application/vpuFirmware/make_std_fw_image.py
* If a header is missing this info we ignore the header, if a header is missing or contains
* partial info a build error will be generated.
*/
/*
@ -24,12 +27,12 @@
* Minor version changes when API backward compatibility is preserved.
* Resets to 0 if Major version is incremented.
*/
#define VPU_BOOT_API_VER_MINOR 12
#define VPU_BOOT_API_VER_MINOR 20
/*
* API header changed (field names, documentation, formatting) but API itself has not been changed
*/
#define VPU_BOOT_API_VER_PATCH 2
#define VPU_BOOT_API_VER_PATCH 4
/*
* Index in the API version table
@ -63,6 +66,12 @@ struct vpu_firmware_header {
/* Size of memory require for firmware execution */
u32 runtime_size;
u32 shave_nn_fw_size;
/* Size of primary preemption buffer. */
u32 preemption_buffer_1_size;
/* Size of secondary preemption buffer. */
u32 preemption_buffer_2_size;
/* Space reserved for future preemption-related fields. */
u32 preemption_reserved[6];
};
/*
@ -89,6 +98,14 @@ enum VPU_BOOT_L2_CACHE_CFG_TYPE {
VPU_BOOT_L2_CACHE_CFG_NUM = 2
};
/** VPU MCA ECC signalling mode. By default, no signalling is used */
enum VPU_BOOT_MCA_ECC_SIGNAL_TYPE {
VPU_BOOT_MCA_ECC_NONE = 0,
VPU_BOOT_MCA_ECC_CORR = 1,
VPU_BOOT_MCA_ECC_FATAL = 2,
VPU_BOOT_MCA_ECC_BOTH = 3
};
/**
* Logging destinations.
*
@ -131,9 +148,11 @@ enum vpu_trace_destination {
#define VPU_TRACE_PROC_BIT_ACT_SHV_3 22
#define VPU_TRACE_PROC_NO_OF_HW_DEVS 23
/* KMB HW component IDs are sequential, so define first and last IDs. */
#define VPU_TRACE_PROC_BIT_KMB_FIRST VPU_TRACE_PROC_BIT_LRT
#define VPU_TRACE_PROC_BIT_KMB_LAST VPU_TRACE_PROC_BIT_SHV_15
/* VPU 30xx HW component IDs are sequential, so define first and last IDs. */
#define VPU_TRACE_PROC_BIT_30XX_FIRST VPU_TRACE_PROC_BIT_LRT
#define VPU_TRACE_PROC_BIT_30XX_LAST VPU_TRACE_PROC_BIT_SHV_15
#define VPU_TRACE_PROC_BIT_KMB_FIRST VPU_TRACE_PROC_BIT_30XX_FIRST
#define VPU_TRACE_PROC_BIT_KMB_LAST VPU_TRACE_PROC_BIT_30XX_LAST
struct vpu_boot_l2_cache_config {
u8 use;
@ -148,6 +167,25 @@ struct vpu_warm_boot_section {
u32 is_clear_op;
};
/*
* When HW scheduling mode is enabled, a present period is defined.
* It will be used by VPU to swap between normal and focus priorities
* to prevent starving of normal priority band (when implemented).
* Host must provide a valid value at boot time in
* `vpu_focus_present_timer_ms`. If the value provided by the host is not within the
* defined range a default value will be used. Here we define the min. and max.
* allowed values and the and default value of the present period. Units are milliseconds.
*/
#define VPU_PRESENT_CALL_PERIOD_MS_DEFAULT 50
#define VPU_PRESENT_CALL_PERIOD_MS_MIN 16
#define VPU_PRESENT_CALL_PERIOD_MS_MAX 10000
/**
* Macros to enable various operation modes within the VPU.
* To be defined as part of 32 bit mask.
*/
#define VPU_OP_MODE_SURVIVABILITY 0x1
struct vpu_boot_params {
u32 magic;
u32 vpu_id;
@ -218,6 +256,7 @@ struct vpu_boot_params {
* the threshold will not be logged); applies to every enabled logging
* destination and loggable HW component. See 'mvLog_t' enum for acceptable
* values.
* TODO: EISW-33556: Move log level definition (mvLog_t) to this file.
*/
u32 default_trace_level;
u32 boot_type;
@ -249,7 +288,36 @@ struct vpu_boot_params {
u32 temp_sensor_period_ms;
/** PLL ratio for efficient clock frequency */
u32 pn_freq_pll_ratio;
u32 pad4[28];
/** DVFS Mode: Default: 0, Max Performance: 1, On Demand: 2, Power Save: 3 */
u32 dvfs_mode;
/**
* Depending on DVFS Mode:
* On-demand: Default if 0.
* Bit 0-7 - uint8_t: Highest residency percent
* Bit 8-15 - uint8_t: High residency percent
* Bit 16-23 - uint8_t: Low residency percent
* Bit 24-31 - uint8_t: Lowest residency percent
* Bit 32-35 - unsigned 4b: PLL Ratio increase amount on highest residency
* Bit 36-39 - unsigned 4b: PLL Ratio increase amount on high residency
* Bit 40-43 - unsigned 4b: PLL Ratio decrease amount on low residency
* Bit 44-47 - unsigned 4b: PLL Ratio decrease amount on lowest frequency
* Bit 48-55 - uint8_t: Period (ms) for residency decisions
* Bit 56-63 - uint8_t: Averaging windows (as multiples of period. Max: 30 decimal)
* Power Save/Max Performance: Unused
*/
u64 dvfs_param;
/**
* D0i3 delayed entry
* Bit0: Disable CPU state save on D0i2 entry flow.
* 0: Every D0i2 entry saves state. Save state IPC message ignored.
* 1: IPC message required to save state on D0i3 entry flow.
*/
u32 d0i3_delayed_entry;
/* Time spent by VPU in D0i3 state */
u64 d0i3_residency_time_us;
/* Value of VPU perf counter at the time of entering D0i3 state . */
u64 d0i3_entry_vpu_ts;
u32 pad4[20];
/* Warm boot information: 0x400 - 0x43F */
u32 warm_boot_sections_count;
u32 warm_boot_start_address_reference;
@ -274,8 +342,12 @@ struct vpu_boot_params {
u32 vpu_scheduling_mode;
/* Present call period in milliseconds. */
u32 vpu_focus_present_timer_ms;
/* Unused/reserved: 0x478 - 0xFFF */
u32 pad6[738];
/* VPU ECC Signaling */
u32 vpu_uses_ecc_mca_signal;
/* Values defined by VPU_OP_MODE* macros */
u32 vpu_operation_mode;
/* Unused/reserved: 0x480 - 0xFFF */
u32 pad6[736];
};
/*

309
drivers/accel/ivpu/vpu_jsm_api.h
View File

@ -22,12 +22,12 @@
/*
* Minor version changes when API backward compatibility is preserved.
*/
#define VPU_JSM_API_VER_MINOR 0
#define VPU_JSM_API_VER_MINOR 15
/*
* API header changed (field names, documentation, formatting) but API itself has not been changed
*/
#define VPU_JSM_API_VER_PATCH 1
#define VPU_JSM_API_VER_PATCH 0
/*
* Index in the API version table
@ -84,11 +84,13 @@
* Job flags bit masks.
*/
#define VPU_JOB_FLAGS_NULL_SUBMISSION_MASK 0x00000001
#define VPU_JOB_FLAGS_PRIVATE_DATA_MASK 0xFF000000
/*
* Sizes of the reserved areas in jobs, in bytes.
*/
#define VPU_JOB_RESERVED_BYTES 16
#define VPU_JOB_RESERVED_BYTES 8
/*
* Sizes of the reserved areas in job queues, in bytes.
*/
@ -108,6 +110,20 @@
*/
#define VPU_DYNDBG_CMD_MAX_LEN 96
/*
* For HWS command queue scheduling, we can prioritise command queues inside the
* same process with a relative in-process priority. Valid values for relative
* priority are given below - max and min.
*/
#define VPU_HWS_COMMAND_QUEUE_MAX_IN_PROCESS_PRIORITY 7
#define VPU_HWS_COMMAND_QUEUE_MIN_IN_PROCESS_PRIORITY -7
/*
* For HWS priority scheduling, we can have multiple realtime priority bands.
* They are numbered 0 to a MAX.
*/
#define VPU_HWS_MAX_REALTIME_PRIORITY_LEVEL 31U
/*
* Job format.
*/
@ -117,8 +133,14 @@ struct vpu_job_queue_entry {
u32 flags; /**< Flags bit field, see VPU_JOB_FLAGS_* above */
u64 root_page_table_addr; /**< Address of root page table to use for this job */
u64 root_page_table_update_counter; /**< Page tables update events counter */
u64 preemption_buffer_address; /**< Address of the preemption buffer to use for this job */
u64 preemption_buffer_size; /**< Size of the preemption buffer to use for this job */
u64 primary_preempt_buf_addr;
/**< Address of the primary preemption buffer to use for this job */
u32 primary_preempt_buf_size;
/**< Size of the primary preemption buffer to use for this job */
u32 secondary_preempt_buf_size;
/**< Size of secondary preemption buffer to use for this job */
u64 secondary_preempt_buf_addr;
/**< Address of secondary preemption buffer to use for this job */
u8 reserved_0[VPU_JOB_RESERVED_BYTES];
};
@ -152,6 +174,46 @@ enum vpu_trace_entity_type {
VPU_TRACE_ENTITY_TYPE_HW_COMPONENT = 2,
};
/*
* HWS specific log buffer header details.
* Total size is 32 bytes.
*/
struct vpu_hws_log_buffer_header {
/* Written by VPU after adding a log entry. Initialised by host to 0. */
u32 first_free_entry_index;
/* Incremented by VPU every time the VPU overwrites the 0th entry;
* initialised by host to 0.
*/
u32 wraparound_count;
/*
* This is the number of buffers that can be stored in the log buffer provided by the host.
* It is written by host before passing buffer to VPU. VPU should consider it read-only.
*/
u64 num_of_entries;
u64 reserved[2];
};
/*
* HWS specific log buffer entry details.
* Total size is 32 bytes.
*/
struct vpu_hws_log_buffer_entry {
/* VPU timestamp must be an invariant timer tick (not impacted by DVFS) */
u64 vpu_timestamp;
/*
* Operation type:
* 0 - context state change
* 1 - queue new work
* 2 - queue unwait sync object
* 3 - queue no more work
* 4 - queue wait sync object
*/
u32 operation_type;
u32 reserved;
/* Operation data depends on operation type */
u64 operation_data[2];
};
/*
* Host <-> VPU IPC messages types.
*/
@ -228,6 +290,23 @@ enum vpu_ipc_msg_type {
* deallocated or reassigned to another context.
*/
VPU_JSM_MSG_HWS_REGISTER_DB = 0x1117,
/** Control command: Log buffer setting */
VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG = 0x1118,
/* Control command: Suspend command queue. */
VPU_JSM_MSG_HWS_SUSPEND_CMDQ = 0x1119,
/* Control command: Resume command queue */
VPU_JSM_MSG_HWS_RESUME_CMDQ = 0x111a,
/* Control command: Resume engine after reset */
VPU_JSM_MSG_HWS_ENGINE_RESUME = 0x111b,
/* Control command: Enable survivability/DCT mode */
VPU_JSM_MSG_DCT_ENABLE = 0x111c,
/* Control command: Disable survivability/DCT mode */
VPU_JSM_MSG_DCT_DISABLE = 0x111d,
/**
* Dump VPU state. To be used for debug purposes only.
* NOTE: Please introduce new ASYNC commands before this one. *
*/
VPU_JSM_MSG_STATE_DUMP = 0x11FF,
/* IPC Host -> Device, General commands */
VPU_JSM_MSG_GENERAL_CMD = 0x1200,
VPU_JSM_MSG_BLOB_DEINIT = VPU_JSM_MSG_GENERAL_CMD,
@ -236,6 +315,10 @@ enum vpu_ipc_msg_type {
* Linux command: `echo '<dyndbg_cmd>' > <debugfs>/dynamic_debug/control`.
*/
VPU_JSM_MSG_DYNDBG_CONTROL = 0x1201,
/**
* Perform the save procedure for the D0i3 entry
*/
VPU_JSM_MSG_PWR_D0I3_ENTER = 0x1202,
/* IPC Device -> Host, Job completion */
VPU_JSM_MSG_JOB_DONE = 0x2100,
/* IPC Device -> Host, Async command completion */
@ -304,11 +387,35 @@ enum vpu_ipc_msg_type {
VPU_JSM_MSG_DESTROY_CMD_QUEUE_RSP = 0x2216,
/** Response to control command: Set context scheduling properties */
VPU_JSM_MSG_SET_CONTEXT_SCHED_PROPERTIES_RSP = 0x2217,
/** Response to control command: Log buffer setting */
VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG_RSP = 0x2218,
/* IPC Device -> Host, HWS notify index entry of log buffer written */
VPU_JSM_MSG_HWS_SCHEDULING_LOG_NOTIFICATION = 0x2219,
/* IPC Device -> Host, HWS completion of a context suspend request */
VPU_JSM_MSG_HWS_SUSPEND_CMDQ_DONE = 0x221a,
/* Response to control command: Resume command queue */
VPU_JSM_MSG_HWS_RESUME_CMDQ_RSP = 0x221b,
/* Response to control command: Resume engine command response */
VPU_JSM_MSG_HWS_RESUME_ENGINE_DONE = 0x221c,
/* Response to control command: Enable survivability/DCT mode */
VPU_JSM_MSG_DCT_ENABLE_DONE = 0x221d,
/* Response to control command: Disable survivability/DCT mode */
VPU_JSM_MSG_DCT_DISABLE_DONE = 0x221e,
/**
* Response to state dump control command.
* NOTE: Please introduce new ASYNC responses before this one. *
*/
VPU_JSM_MSG_STATE_DUMP_RSP = 0x22FF,
/* IPC Device -> Host, General command completion */
VPU_JSM_MSG_GENERAL_CMD_DONE = 0x2300,
VPU_JSM_MSG_BLOB_DEINIT_DONE = VPU_JSM_MSG_GENERAL_CMD_DONE,
/** Response to VPU_JSM_MSG_DYNDBG_CONTROL. */
VPU_JSM_MSG_DYNDBG_CONTROL_RSP = 0x2301,
/**
* Acknowledgment of completion of the save procedure initiated by
* VPU_JSM_MSG_PWR_D0I3_ENTER
*/
VPU_JSM_MSG_PWR_D0I3_ENTER_DONE = 0x2302,
};
enum vpu_ipc_msg_status { VPU_JSM_MSG_FREE, VPU_JSM_MSG_ALLOCATED };
@ -593,12 +700,12 @@ struct vpu_ipc_msg_payload_hws_priority_band_setup {
* Default quantum in 100ns units for scheduling across processes
* within a priority band
*/
u64 process_quantum[VPU_HWS_NUM_PRIORITY_BANDS];
u32 process_quantum[VPU_HWS_NUM_PRIORITY_BANDS];
/*
* Default grace period in 100ns units for processes that preempt each
* other within a priority band
*/
u64 process_grace_period[VPU_HWS_NUM_PRIORITY_BANDS];
u32 process_grace_period[VPU_HWS_NUM_PRIORITY_BANDS];
/*
* For normal priority band, specifies the target VPU percentage
* in situations when it's starved by the focus band.
@ -608,32 +715,51 @@ struct vpu_ipc_msg_payload_hws_priority_band_setup {
u32 reserved_0;
};
/* HWS create command queue request */
/*
* @brief HWS create command queue request.
* Host will create a command queue via this command.
* Note: Cmdq group is a handle of an object which
* may contain one or more command queues.
* @see VPU_JSM_MSG_CREATE_CMD_QUEUE
* @see VPU_JSM_MSG_CREATE_CMD_QUEUE_RSP
*/
struct vpu_ipc_msg_payload_hws_create_cmdq {
/* Process id */
u64 process_id;
/* Host SSID */
u32 host_ssid;
/* Zero Padding */
u32 reserved;
/* Engine for which queue is being created */
u32 engine_idx;
/*
* Cmdq group may be set to 0 or equal to
* cmdq_id while each priority band contains
* only single engine instances.
*/
u64 cmdq_group;
/* Command queue id */
u64 cmdq_id;
/* Command queue base */
u64 cmdq_base;
/* Command queue size */
u32 cmdq_size;
/* Reserved */
/* Zero padding */
u32 reserved_0;
};
/* HWS create command queue response */
/*
* @brief HWS create command queue response.
* @see VPU_JSM_MSG_CREATE_CMD_QUEUE
* @see VPU_JSM_MSG_CREATE_CMD_QUEUE_RSP
*/
struct vpu_ipc_msg_payload_hws_create_cmdq_rsp {
/* Process id */
u64 process_id;
/* Host SSID */
u32 host_ssid;
/* Zero Padding */
u32 reserved;
/* Engine for which queue is being created */
u32 engine_idx;
/* Command queue group */
u64 cmdq_group;
/* Command queue id */
u64 cmdq_id;
};
@ -661,7 +787,7 @@ struct vpu_ipc_msg_payload_hws_set_context_sched_properties {
/* Inside realtime band assigns a further priority */
u32 realtime_priority_level;
/* Priority relative to other contexts in the same process */
u32 in_process_priority;
s32 in_process_priority;
/* Zero padding / Reserved */
u32 reserved_1;
/* Context quantum relative to other contexts of same priority in the same process */
@ -694,6 +820,123 @@ struct vpu_jsm_hws_register_db {
u64 cmdq_size;
};
/*
* @brief Structure to set another buffer to be used for scheduling-related logging.
* The size of the logging buffer and the number of entries is defined as part of the
* buffer itself as described next.
* The log buffer received from the host is made up of;
* - header: 32 bytes in size, as shown in 'struct vpu_hws_log_buffer_header'.
* The header contains the number of log entries in the buffer.
* - log entry: 0 to n-1, each log entry is 32 bytes in size, as shown in
* 'struct vpu_hws_log_buffer_entry'.
* The entry contains the VPU timestamp, operation type and data.
* The host should provide the notify index value of log buffer to VPU. This is a
* value defined within the log buffer and when written to will generate the
* scheduling log notification.
* The host should set engine_idx and vpu_log_buffer_va to 0 to disable logging
* for a particular engine.
* VPU will handle one log buffer for each of supported engines.
* VPU should allow the logging to consume one host_ssid.
* @see VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG
* @see VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG_RSP
* @see VPU_JSM_MSG_HWS_SCHEDULING_LOG_NOTIFICATION
*/
struct vpu_ipc_msg_payload_hws_set_scheduling_log {
/* Engine ordinal */
u32 engine_idx;
/* Host SSID */
u32 host_ssid;
/*
* VPU log buffer virtual address.
* Set to 0 to disable logging for this engine.
*/
u64 vpu_log_buffer_va;
/*
* Notify index of log buffer. VPU_JSM_MSG_HWS_SCHEDULING_LOG_NOTIFICATION
* is generated when an event log is written to this index.
*/
u64 notify_index;
};
/*
* @brief The scheduling log notification is generated by VPU when it writes
* an event into the log buffer at the notify_index. VPU notifies host with
* VPU_JSM_MSG_HWS_SCHEDULING_LOG_NOTIFICATION. This is an asynchronous
* message from VPU to host.
* @see VPU_JSM_MSG_HWS_SCHEDULING_LOG_NOTIFICATION
* @see VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG
*/
struct vpu_ipc_msg_payload_hws_scheduling_log_notification {
/* Engine ordinal */
u32 engine_idx;
/* Zero Padding */
u32 reserved_0;
};
/*
* @brief HWS suspend command queue request and done structure.
* Host will request the suspend of contexts and VPU will;
* - Suspend all work on this context
* - Preempt any running work
* - Asynchronously perform the above and return success immediately once
* all items above are started successfully
* - Notify the host of completion of these operations via
* VPU_JSM_MSG_HWS_SUSPEND_CMDQ_DONE
* - Reject any other context operations on a context with an in-flight
* suspend request running
* Same structure used when VPU notifies host of completion of a context suspend
* request. The ids and suspend fence value reported in this command will match
* the one in the request from the host to suspend the context. Once suspend is
* complete, VPU will not access any data relating to this command queue until
* it is resumed.
* @see VPU_JSM_MSG_HWS_SUSPEND_CMDQ
* @see VPU_JSM_MSG_HWS_SUSPEND_CMDQ_DONE
*/
struct vpu_ipc_msg_payload_hws_suspend_cmdq {
/* Host SSID */
u32 host_ssid;
/* Zero Padding */
u32 reserved_0;
/* Command queue id */
u64 cmdq_id;
/*
* Suspend fence value - reported by the VPU suspend context
* completed once suspend is complete.
*/
u64 suspend_fence_value;
};
/*
* @brief HWS Resume command queue request / response structure.
* Host will request the resume of a context;
* - VPU will resume all work on this context
* - Scheduler will allow this context to be scheduled
* @see VPU_JSM_MSG_HWS_RESUME_CMDQ
* @see VPU_JSM_MSG_HWS_RESUME_CMDQ_RSP
*/
struct vpu_ipc_msg_payload_hws_resume_cmdq {
/* Host SSID */
u32 host_ssid;
/* Zero Padding */
u32 reserved_0;
/* Command queue id */
u64 cmdq_id;
};
/*
* @brief HWS Resume engine request / response structure.
* After a HWS engine reset, all scheduling is stopped on VPU until a engine resume.
* Host shall send this command to resume scheduling of any valid queue.
* @see VPU_JSM_MSG_HWS_RESUME_ENGINE
* @see VPU_JSM_MSG_HWS_RESUME_ENGINE_DONE
*/
struct vpu_ipc_msg_payload_hws_resume_engine {
/* Engine to be resumed */
u32 engine_idx;
/* Reserved */
u32 reserved_0;
};
/**
* Payload for VPU_JSM_MSG_TRACE_SET_CONFIG[_RSP] and
* VPU_JSM_MSG_TRACE_GET_CONFIG_RSP messages.
@ -938,6 +1181,35 @@ struct vpu_ipc_msg_payload_dyndbg_control {
char dyndbg_cmd[VPU_DYNDBG_CMD_MAX_LEN];
};
/**
* Payload for VPU_JSM_MSG_PWR_D0I3_ENTER
*
* This is a bi-directional payload.
*/
struct vpu_ipc_msg_payload_pwr_d0i3_enter {
/**
* 0: VPU_JSM_MSG_PWR_D0I3_ENTER_DONE is not sent to the host driver
* The driver will poll for D0i2 Idle state transitions.
* 1: VPU_JSM_MSG_PWR_D0I3_ENTER_DONE is sent after VPU state save is complete
*/
u32 send_response;
u32 reserved_0;
};
/**
* Payload for VPU_JSM_MSG_DCT_ENABLE message.
*
* Default values for DCT active/inactive times are 5.3ms and 30ms respectively,
* corresponding to a 85% duty cycle. This payload allows the host to tune these
* values according to application requirements.
*/
struct vpu_ipc_msg_payload_pwr_dct_control {
/** Duty cycle active time in microseconds */
u32 dct_active_us;
/** Duty cycle inactive time in microseconds */
u32 dct_inactive_us;
};
/*
* Payloads union, used to define complete message format.
*/
@ -974,6 +1246,13 @@ union vpu_ipc_msg_payload {
struct vpu_ipc_msg_payload_hws_destroy_cmdq hws_destroy_cmdq;
struct vpu_ipc_msg_payload_hws_set_context_sched_properties
hws_set_context_sched_properties;
struct vpu_ipc_msg_payload_hws_set_scheduling_log hws_set_scheduling_log;
struct vpu_ipc_msg_payload_hws_scheduling_log_notification hws_scheduling_log_notification;
struct vpu_ipc_msg_payload_hws_suspend_cmdq hws_suspend_cmdq;
struct vpu_ipc_msg_payload_hws_resume_cmdq hws_resume_cmdq;
struct vpu_ipc_msg_payload_hws_resume_engine hws_resume_engine;
struct vpu_ipc_msg_payload_pwr_d0i3_enter pwr_d0i3_enter;
struct vpu_ipc_msg_payload_pwr_dct_control pwr_dct_control;
};
/*

3
drivers/accel/qaic/Makefile
View File

@ -9,4 +9,5 @@ qaic-y := \
mhi_controller.o \
qaic_control.o \
qaic_data.o \
qaic_drv.o
qaic_drv.o \
qaic_timesync.o

42
drivers/accel/qaic/mhi_controller.c
View File

@ -348,7 +348,7 @@ static struct mhi_channel_config aic100_channels[] = {
.local_elements = 0,
.event_ring = 0,
.dir = DMA_TO_DEVICE,
.ee_mask = MHI_CH_EE_SBL | MHI_CH_EE_AMSS,
.ee_mask = MHI_CH_EE_SBL,
.pollcfg = 0,
.doorbell = MHI_DB_BRST_DISABLE,
.lpm_notify = false,
@ -364,7 +364,39 @@ static struct mhi_channel_config aic100_channels[] = {
.local_elements = 0,
.event_ring = 0,
.dir = DMA_FROM_DEVICE,
.ee_mask = MHI_CH_EE_SBL | MHI_CH_EE_AMSS,
.ee_mask = MHI_CH_EE_SBL,
.pollcfg = 0,
.doorbell = MHI_DB_BRST_DISABLE,
.lpm_notify = false,
.offload_channel = false,
.doorbell_mode_switch = false,
.auto_queue = false,
.wake_capable = false,
},
{
.name = "QAIC_TIMESYNC_PERIODIC",
.num = 22,
.num_elements = 32,
.local_elements = 0,
.event_ring = 0,
.dir = DMA_TO_DEVICE,
.ee_mask = MHI_CH_EE_AMSS,
.pollcfg = 0,
.doorbell = MHI_DB_BRST_DISABLE,
.lpm_notify = false,
.offload_channel = false,
.doorbell_mode_switch = false,
.auto_queue = false,
.wake_capable = false,
},
{
.num = 23,
.name = "QAIC_TIMESYNC_PERIODIC",
.num_elements = 32,
.local_elements = 0,
.event_ring = 0,
.dir = DMA_FROM_DEVICE,
.ee_mask = MHI_CH_EE_AMSS,
.pollcfg = 0,
.doorbell = MHI_DB_BRST_DISABLE,
.lpm_notify = false,
@ -468,7 +500,7 @@ static int mhi_reset_and_async_power_up(struct mhi_controller *mhi_cntrl)
}
struct mhi_controller *qaic_mhi_register_controller(struct pci_dev *pci_dev, void __iomem *mhi_bar,
int mhi_irq)
int mhi_irq, bool shared_msi)
{
struct mhi_controller *mhi_cntrl;
int ret;
@ -500,6 +532,10 @@ struct mhi_controller *qaic_mhi_register_controller(struct pci_dev *pci_dev, voi
return ERR_PTR(-ENOMEM);
mhi_cntrl->irq[0] = mhi_irq;
if (shared_msi) /* MSI shared with data path, no IRQF_NO_SUSPEND */
mhi_cntrl->irq_flags = IRQF_SHARED;
mhi_cntrl->fw_image = "qcom/aic100/sbl.bin";
/* use latest configured timeout */

2
drivers/accel/qaic/mhi_controller.h
View File

@ -8,7 +8,7 @@
#define MHICONTROLLERQAIC_H_
struct mhi_controller *qaic_mhi_register_controller(struct pci_dev *pci_dev, void __iomem *mhi_bar,
int mhi_irq);
int mhi_irq, bool shared_msi);
void qaic_mhi_free_controller(struct mhi_controller *mhi_cntrl, bool link_up);
void qaic_mhi_start_reset(struct mhi_controller *mhi_cntrl);
void qaic_mhi_reset_done(struct mhi_controller *mhi_cntrl);

6
drivers/accel/qaic/qaic.h
View File

@ -123,6 +123,8 @@ struct qaic_device {
struct srcu_struct dev_lock;
/* true: Device under reset; false: Device not under reset */
bool in_reset;
/* true: single MSI is used to operate device */
bool single_msi;
/*
* true: A tx MHI transaction has failed and a rx buffer is still queued
* in control device. Such a buffer is considered lost rx buffer
@ -137,6 +139,10 @@ struct qaic_device {
u32 (*gen_crc)(void *msg);
/* Validate the CRC of a control message */
bool (*valid_crc)(void *msg);
/* MHI "QAIC_TIMESYNC" channel device */
struct mhi_device *qts_ch;
/* Work queue for tasks related to MHI "QAIC_TIMESYNC" channel */
struct workqueue_struct *qts_wq;
};
struct qaic_drm_device {

2
drivers/accel/qaic/qaic_control.c
View File

@ -1138,7 +1138,7 @@ static int abort_dma_cont(struct qaic_device *qdev, struct wrapper_list *wrapper
if (!list_is_first(&wrapper->list, &wrappers->list))
kref_put(&wrapper->ref_count, free_wrapper);
wrapper = add_wrapper(wrappers, offsetof(struct wrapper_msg, trans) + sizeof(*out_trans));
wrapper = add_wrapper(wrappers, sizeof(*wrapper));
if (!wrapper)
return -ENOMEM;

129
drivers/accel/qaic/qaic_data.c
View File

@ -51,6 +51,7 @@
})
#define NUM_EVENTS 128
#define NUM_DELAYS 10
#define fifo_at(base, offset) ((base) + (offset) * get_dbc_req_elem_size())
static unsigned int wait_exec_default_timeout_ms = 5000; /* 5 sec default */
module_param(wait_exec_default_timeout_ms, uint, 0600);
@ -1058,6 +1059,16 @@ unlock_usr_srcu:
return ret;
}
static inline u32 fifo_space_avail(u32 head, u32 tail, u32 q_size)
{
u32 avail = head - tail - 1;
if (head <= tail)
avail += q_size;
return avail;
}
static inline int copy_exec_reqs(struct qaic_device *qdev, struct bo_slice *slice, u32 dbc_id,
u32 head, u32 *ptail)
{
@ -1066,27 +1077,20 @@ static inline int copy_exec_reqs(struct qaic_device *qdev, struct bo_slice *slic
u32 tail = *ptail;
u32 avail;
avail = head - tail;
if (head <= tail)
avail += dbc->nelem;
--avail;
avail = fifo_space_avail(head, tail, dbc->nelem);
if (avail < slice->nents)
return -EAGAIN;
if (tail + slice->nents > dbc->nelem) {
avail = dbc->nelem - tail;
avail = min_t(u32, avail, slice->nents);
memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs,
sizeof(*reqs) * avail);
memcpy(fifo_at(dbc->req_q_base, tail), reqs, sizeof(*reqs) * avail);
reqs += avail;
avail = slice->nents - avail;
if (avail)
memcpy(dbc->req_q_base, reqs, sizeof(*reqs) * avail);
} else {
memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs,
sizeof(*reqs) * slice->nents);
memcpy(fifo_at(dbc->req_q_base, tail), reqs, sizeof(*reqs) * slice->nents);
}
*ptail = (tail + slice->nents) % dbc->nelem;
@ -1094,46 +1098,31 @@ static inline int copy_exec_reqs(struct qaic_device *qdev, struct bo_slice *slic
return 0;
}
/*
* Based on the value of resize we may only need to transmit first_n
* entries and the last entry, with last_bytes to send from the last entry.
* Note that first_n could be 0.
*/
static inline int copy_partial_exec_reqs(struct qaic_device *qdev, struct bo_slice *slice,
u64 resize, u32 dbc_id, u32 head, u32 *ptail)
u64 resize, struct dma_bridge_chan *dbc, u32 head,
u32 *ptail)
{
struct dma_bridge_chan *dbc = &qdev->dbc[dbc_id];
struct dbc_req *reqs = slice->reqs;
struct dbc_req *last_req;
u32 tail = *ptail;
u64 total_bytes;
u64 last_bytes;
u32 first_n;
u32 avail;
int ret;
int i;
avail = head - tail;
if (head <= tail)
avail += dbc->nelem;
avail = fifo_space_avail(head, tail, dbc->nelem);
--avail;
total_bytes = 0;
for (i = 0; i < slice->nents; i++) {
total_bytes += le32_to_cpu(reqs[i].len);
if (total_bytes >= resize)
/*
* After this for loop is complete, first_n represents the index
* of the last DMA request of this slice that needs to be
* transferred after resizing and last_bytes represents DMA size
* of that request.
*/
last_bytes = resize;
for (first_n = 0; first_n < slice->nents; first_n++)
if (last_bytes > le32_to_cpu(reqs[first_n].len))
last_bytes -= le32_to_cpu(reqs[first_n].len);
else
break;
}
if (total_bytes < resize) {
/* User space should have used the full buffer path. */
ret = -EINVAL;
return ret;
}
first_n = i;
last_bytes = i ? resize + le32_to_cpu(reqs[i].len) - total_bytes : resize;
if (avail < (first_n + 1))
return -EAGAIN;
@ -1142,22 +1131,21 @@ static inline int copy_partial_exec_reqs(struct qaic_device *qdev, struct bo_sli
if (tail + first_n > dbc->nelem) {
avail = dbc->nelem - tail;
avail = min_t(u32, avail, first_n);
memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs,
sizeof(*reqs) * avail);
memcpy(fifo_at(dbc->req_q_base, tail), reqs, sizeof(*reqs) * avail);
last_req = reqs + avail;
avail = first_n - avail;
if (avail)
memcpy(dbc->req_q_base, last_req, sizeof(*reqs) * avail);
} else {
memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs,
sizeof(*reqs) * first_n);
memcpy(fifo_at(dbc->req_q_base, tail), reqs, sizeof(*reqs) * first_n);
}
}
/* Copy over the last entry. Here we need to adjust len to the left over
/*
* Copy over the last entry. Here we need to adjust len to the left over
* size, and set src and dst to the entry it is copied to.
*/
last_req = dbc->req_q_base + (tail + first_n) % dbc->nelem * get_dbc_req_elem_size();
last_req = fifo_at(dbc->req_q_base, (tail + first_n) % dbc->nelem);
memcpy(last_req, reqs + slice->nents - 1, sizeof(*reqs));
/*
@ -1168,6 +1156,9 @@ static inline int copy_partial_exec_reqs(struct qaic_device *qdev, struct bo_sli
last_req->len = cpu_to_le32((u32)last_bytes);
last_req->src_addr = reqs[first_n].src_addr;
last_req->dest_addr = reqs[first_n].dest_addr;
if (!last_bytes)
/* Disable DMA transfer */
last_req->cmd = GENMASK(7, 2) & reqs[first_n].cmd;
*ptail = (tail + first_n + 1) % dbc->nelem;
@ -1227,26 +1218,17 @@ static int send_bo_list_to_device(struct qaic_device *qdev, struct drm_file *fil
bo->req_id = dbc->next_req_id++;
list_for_each_entry(slice, &bo->slices, slice) {
/*
* If this slice does not fall under the given
* resize then skip this slice and continue the loop
*/
if (is_partial && pexec[i].resize && pexec[i].resize <= slice->offset)
continue;
for (j = 0; j < slice->nents; j++)
slice->reqs[j].req_id = cpu_to_le16(bo->req_id);
/*
* If it is a partial execute ioctl call then check if
* resize has cut this slice short then do a partial copy
* else do complete copy
*/
if (is_partial && pexec[i].resize &&
pexec[i].resize < slice->offset + slice->size)
if (is_partial && (!pexec[i].resize || pexec[i].resize <= slice->offset))
/* Configure the slice for no DMA transfer */
ret = copy_partial_exec_reqs(qdev, slice, 0, dbc, head, tail);
else if (is_partial && pexec[i].resize < slice->offset + slice->size)
/* Configure the slice to be partially DMA transferred */
ret = copy_partial_exec_reqs(qdev, slice,
pexec[i].resize - slice->offset,
dbc->id, head, tail);
pexec[i].resize - slice->offset, dbc,
head, tail);
else
ret = copy_exec_reqs(qdev, slice, dbc->id, head, tail);
if (ret) {
@ -1466,6 +1448,16 @@ irqreturn_t dbc_irq_handler(int irq, void *data)
rcu_id = srcu_read_lock(&dbc->ch_lock);
if (datapath_polling) {
srcu_read_unlock(&dbc->ch_lock, rcu_id);
/*
* Normally datapath_polling will not have irqs enabled, but
* when running with only one MSI the interrupt is shared with
* MHI so it cannot be disabled. Return ASAP instead.
*/
return IRQ_HANDLED;
}
if (!dbc->usr) {
srcu_read_unlock(&dbc->ch_lock, rcu_id);
return IRQ_HANDLED;
@ -1488,7 +1480,8 @@ irqreturn_t dbc_irq_handler(int irq, void *data)
return IRQ_NONE;
}
disable_irq_nosync(irq);
if (!dbc->qdev->single_msi)
disable_irq_nosync(irq);
srcu_read_unlock(&dbc->ch_lock, rcu_id);
return IRQ_WAKE_THREAD;
}
@ -1559,12 +1552,12 @@ irqreturn_t dbc_irq_threaded_fn(int irq, void *data)
u32 tail;
rcu_id = srcu_read_lock(&dbc->ch_lock);
qdev = dbc->qdev;
head = readl(dbc->dbc_base + RSPHP_OFF);
if (head == U32_MAX) /* PCI link error */
goto error_out;
qdev = dbc->qdev;
read_fifo:
if (!event_count) {
@ -1645,14 +1638,14 @@ read_fifo:
goto read_fifo;
normal_out:
if (likely(!datapath_polling))
if (!qdev->single_msi && likely(!datapath_polling))
enable_irq(irq);
else
else if (unlikely(datapath_polling))
schedule_work(&dbc->poll_work);
/* checking the fifo and enabling irqs is a race, missed event check */
tail = readl(dbc->dbc_base + RSPTP_OFF);
if (tail != U32_MAX && head != tail) {
if (likely(!datapath_polling))
if (!qdev->single_msi && likely(!datapath_polling))
disable_irq_nosync(irq);
goto read_fifo;
}
@ -1661,9 +1654,9 @@ normal_out:
error_out:
srcu_read_unlock(&dbc->ch_lock, rcu_id);
if (likely(!datapath_polling))
if (!qdev->single_msi && likely(!datapath_polling))
enable_irq(irq);
else
else if (unlikely(datapath_polling))
schedule_work(&dbc->poll_work);
return IRQ_HANDLED;

48
drivers/accel/qaic/qaic_drv.c
View File

@ -27,6 +27,7 @@
#include "mhi_controller.h"
#include "qaic.h"
#include "qaic_timesync.h"
MODULE_IMPORT_NS(DMA_BUF);
@ -324,6 +325,7 @@ static void cleanup_qdev(struct qaic_device *qdev)
cleanup_srcu_struct(&qdev->dev_lock);
pci_set_drvdata(qdev->pdev, NULL);
destroy_workqueue(qdev->cntl_wq);
destroy_workqueue(qdev->qts_wq);
}
static struct qaic_device *create_qdev(struct pci_dev *pdev, const struct pci_device_id *id)
@ -347,6 +349,12 @@ static struct qaic_device *create_qdev(struct pci_dev *pdev, const struct pci_de
if (!qdev->cntl_wq)
return NULL;
qdev->qts_wq = alloc_workqueue("qaic_ts", WQ_UNBOUND, 0);
if (!qdev->qts_wq) {
destroy_workqueue(qdev->cntl_wq);
return NULL;
}
pci_set_drvdata(pdev, qdev);
qdev->pdev = pdev;
@ -424,14 +432,24 @@ static int init_msi(struct qaic_device *qdev, struct pci_dev *pdev)
int i;
/* Managed release since we use pcim_enable_device */
ret = pci_alloc_irq_vectors(pdev, 1, 32, PCI_IRQ_MSI);
if (ret < 0)
return ret;
ret = pci_alloc_irq_vectors(pdev, 32, 32, PCI_IRQ_MSI);
if (ret == -ENOSPC) {
ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
if (ret < 0)
return ret;
if (ret < 32) {
pci_err(pdev, "%s: Requested 32 MSIs. Obtained %d MSIs which is less than the 32 required.\n",
__func__, ret);
return -ENODEV;
/*
* Operate in one MSI mode. All interrupts will be directed to
* MSI0; every interrupt will wake up all the interrupt handlers
* (MHI and DBC[0-15]). Since the interrupt is now shared, it is
* not disabled during DBC threaded handler, but only one thread
* will be allowed to run per DBC, so while it can be
* interrupted, it shouldn't race with itself.
*/
qdev->single_msi = true;
pci_info(pdev, "Allocating 32 MSIs failed, operating in 1 MSI mode. Performance may be impacted.\n");
} else if (ret < 0) {
return ret;
}
mhi_irq = pci_irq_vector(pdev, 0);
@ -439,15 +457,17 @@ static int init_msi(struct qaic_device *qdev, struct pci_dev *pdev)
return mhi_irq;
for (i = 0; i < qdev->num_dbc; ++i) {
ret = devm_request_threaded_irq(&pdev->dev, pci_irq_vector(pdev, i + 1),
ret = devm_request_threaded_irq(&pdev->dev,
pci_irq_vector(pdev, qdev->single_msi ? 0 : i + 1),
dbc_irq_handler, dbc_irq_threaded_fn, IRQF_SHARED,
"qaic_dbc", &qdev->dbc[i]);
if (ret)
return ret;
if (datapath_polling) {
qdev->dbc[i].irq = pci_irq_vector(pdev, i + 1);
disable_irq_nosync(qdev->dbc[i].irq);
qdev->dbc[i].irq = pci_irq_vector(pdev, qdev->single_msi ? 0 : i + 1);
if (!qdev->single_msi)
disable_irq_nosync(qdev->dbc[i].irq);
INIT_WORK(&qdev->dbc[i].poll_work, irq_polling_work);
}
}
@ -479,7 +499,8 @@ static int qaic_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
goto cleanup_qdev;
}
qdev->mhi_cntrl = qaic_mhi_register_controller(pdev, qdev->bar_0, mhi_irq);
qdev->mhi_cntrl = qaic_mhi_register_controller(pdev, qdev->bar_0, mhi_irq,
qdev->single_msi);
if (IS_ERR(qdev->mhi_cntrl)) {
ret = PTR_ERR(qdev->mhi_cntrl);
goto cleanup_qdev;
@ -586,6 +607,10 @@ static int __init qaic_init(void)
goto free_pci;
}
ret = qaic_timesync_init();
if (ret)
pr_debug("qaic: qaic_timesync_init failed %d\n", ret);
return 0;
free_pci:
@ -611,6 +636,7 @@ static void __exit qaic_exit(void)
* reinitializing the link_up state after the cleanup is done.
*/
link_up = true;
qaic_timesync_deinit();
mhi_driver_unregister(&qaic_mhi_driver);
pci_unregister_driver(&qaic_pci_driver);
}

395
drivers/accel/qaic/qaic_timesync.c Normal file
View File

@ -0,0 +1,395 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved. */
#include <linux/io.h>
#include <linux/kernel.h>
#include <linux/math64.h>
#include <linux/mhi.h>
#include <linux/mod_devicetable.h>
#include <linux/module.h>
#include <linux/time64.h>
#include <linux/timer.h>
#include "qaic.h"
#include "qaic_timesync.h"
#define QTIMER_REG_OFFSET 0xa28
#define QAIC_TIMESYNC_SIGNATURE 0x55aa
#define QAIC_CONV_QTIMER_TO_US(qtimer) (mul_u64_u32_div(qtimer, 10, 192))
static unsigned int timesync_delay_ms = 1000; /* 1 sec default */
module_param(timesync_delay_ms, uint, 0600);
MODULE_PARM_DESC(timesync_delay_ms, "Delay in ms between two consecutive timesync operations");
enum qts_msg_type {
QAIC_TS_CMD_TO_HOST,
QAIC_TS_SYNC_REQ,
QAIC_TS_ACK_TO_HOST,
QAIC_TS_MSG_TYPE_MAX
};
/**
* struct qts_hdr - Timesync message header structure.
* @signature: Unique signature to identify the timesync message.
* @reserved_1: Reserved for future use.
* @reserved_2: Reserved for future use.
* @msg_type: sub-type of the timesync message.
* @reserved_3: Reserved for future use.
*/
struct qts_hdr {
__le16 signature;
__le16 reserved_1;
u8 reserved_2;
u8 msg_type;
__le16 reserved_3;
} __packed;
/**
* struct qts_timeval - Structure to carry time information.
* @tv_sec: Seconds part of the time.
* @tv_usec: uS (microseconds) part of the time.
*/
struct qts_timeval {
__le64 tv_sec;
__le64 tv_usec;
} __packed;
/**
* struct qts_host_time_sync_msg_data - Structure to denote the timesync message.
* @header: Header of the timesync message.
* @data: Time information.
*/
struct qts_host_time_sync_msg_data {
struct qts_hdr header;
struct qts_timeval data;
} __packed;
/**
* struct mqts_dev - MHI QAIC Timesync Control device.
* @qdev: Pointer to the root device struct driven by QAIC driver.
* @mhi_dev: Pointer to associated MHI device.
* @timer: Timer handle used for timesync.
* @qtimer_addr: Device QTimer register pointer.
* @buff_in_use: atomic variable to track if the sync_msg buffer is in use.
* @dev: Device pointer to qdev->pdev->dev stored for easy access.
* @sync_msg: Buffer used to send timesync message over MHI.
*/
struct mqts_dev {
struct qaic_device *qdev;
struct mhi_device *mhi_dev;
struct timer_list timer;
void __iomem *qtimer_addr;
atomic_t buff_in_use;
struct device *dev;
struct qts_host_time_sync_msg_data *sync_msg;
};
struct qts_resp_msg {
struct qts_hdr hdr;
} __packed;
struct qts_resp {
struct qts_resp_msg data;
struct work_struct work;
struct qaic_device *qdev;
};
#ifdef readq
static u64 read_qtimer(const volatile void __iomem *addr)
{
return readq(addr);
}
#else
static u64 read_qtimer(const volatile void __iomem *addr)
{
u64 low, high;
low = readl(addr);
high = readl(addr + sizeof(u32));
return low | (high << 32);
}
#endif
static void qaic_timesync_ul_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result)
{
struct mqts_dev *mqtsdev = dev_get_drvdata(&mhi_dev->dev);
dev_dbg(mqtsdev->dev, "%s status: %d xfer_len: %zu\n", __func__,
mhi_result->transaction_status, mhi_result->bytes_xferd);
atomic_set(&mqtsdev->buff_in_use, 0);
}
static void qaic_timesync_dl_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result)
{
struct mqts_dev *mqtsdev = dev_get_drvdata(&mhi_dev->dev);
dev_err(mqtsdev->dev, "%s no data expected on dl channel\n", __func__);
}
static void qaic_timesync_timer(struct timer_list *t)
{
struct mqts_dev *mqtsdev = from_timer(mqtsdev, t, timer);
struct qts_host_time_sync_msg_data *sync_msg;
u64 device_qtimer_us;
u64 device_qtimer;
u64 host_time_us;
u64 offset_us;
u64 host_sec;
int ret;
if (atomic_read(&mqtsdev->buff_in_use)) {
dev_dbg(mqtsdev->dev, "%s buffer not free, schedule next cycle\n", __func__);
goto mod_timer;
}
atomic_set(&mqtsdev->buff_in_use, 1);
sync_msg = mqtsdev->sync_msg;
sync_msg->header.signature = cpu_to_le16(QAIC_TIMESYNC_SIGNATURE);
sync_msg->header.msg_type = QAIC_TS_SYNC_REQ;
/* Read host UTC time and convert to uS*/
host_time_us = div_u64(ktime_get_real_ns(), NSEC_PER_USEC);
device_qtimer = read_qtimer(mqtsdev->qtimer_addr);
device_qtimer_us = QAIC_CONV_QTIMER_TO_US(device_qtimer);
/* Offset between host UTC and device time */
offset_us = host_time_us - device_qtimer_us;
host_sec = div_u64(offset_us, USEC_PER_SEC);
sync_msg->data.tv_usec = cpu_to_le64(offset_us - host_sec * USEC_PER_SEC);
sync_msg->data.tv_sec = cpu_to_le64(host_sec);
ret = mhi_queue_buf(mqtsdev->mhi_dev, DMA_TO_DEVICE, sync_msg, sizeof(*sync_msg), MHI_EOT);
if (ret && (ret != -EAGAIN)) {
dev_err(mqtsdev->dev, "%s unable to queue to mhi:%d\n", __func__, ret);
return;
} else if (ret == -EAGAIN) {
atomic_set(&mqtsdev->buff_in_use, 0);
}
mod_timer:
ret = mod_timer(t, jiffies + msecs_to_jiffies(timesync_delay_ms));
if (ret)
dev_err(mqtsdev->dev, "%s mod_timer error:%d\n", __func__, ret);
}
static int qaic_timesync_probe(struct mhi_device *mhi_dev, const struct mhi_device_id *id)
{
struct qaic_device *qdev = pci_get_drvdata(to_pci_dev(mhi_dev->mhi_cntrl->cntrl_dev));
struct mqts_dev *mqtsdev;
struct timer_list *timer;
int ret;
mqtsdev = kzalloc(sizeof(*mqtsdev), GFP_KERNEL);
if (!mqtsdev) {
ret = -ENOMEM;
goto out;
}
timer = &mqtsdev->timer;
mqtsdev->mhi_dev = mhi_dev;
mqtsdev->qdev = qdev;
mqtsdev->dev = &qdev->pdev->dev;
mqtsdev->sync_msg = kzalloc(sizeof(*mqtsdev->sync_msg), GFP_KERNEL);
if (!mqtsdev->sync_msg) {
ret = -ENOMEM;
goto free_mqts_dev;
}
atomic_set(&mqtsdev->buff_in_use, 0);
ret = mhi_prepare_for_transfer(mhi_dev);
if (ret)
goto free_sync_msg;
/* Qtimer register pointer */
mqtsdev->qtimer_addr = qdev->bar_0 + QTIMER_REG_OFFSET;
timer_setup(timer, qaic_timesync_timer, 0);
timer->expires = jiffies + msecs_to_jiffies(timesync_delay_ms);
add_timer(timer);
dev_set_drvdata(&mhi_dev->dev, mqtsdev);
return 0;
free_sync_msg:
kfree(mqtsdev->sync_msg);
free_mqts_dev:
kfree(mqtsdev);
out:
return ret;
};
static void qaic_timesync_remove(struct mhi_device *mhi_dev)
{
struct mqts_dev *mqtsdev = dev_get_drvdata(&mhi_dev->dev);
del_timer_sync(&mqtsdev->timer);
mhi_unprepare_from_transfer(mqtsdev->mhi_dev);
kfree(mqtsdev->sync_msg);
kfree(mqtsdev);
}
static const struct mhi_device_id qaic_timesync_match_table[] = {
{ .chan = "QAIC_TIMESYNC_PERIODIC"},
{},
};
MODULE_DEVICE_TABLE(mhi, qaic_timesync_match_table);
static struct mhi_driver qaic_timesync_driver = {
.id_table = qaic_timesync_match_table,
.remove = qaic_timesync_remove,
.probe = qaic_timesync_probe,
.ul_xfer_cb = qaic_timesync_ul_xfer_cb,
.dl_xfer_cb = qaic_timesync_dl_xfer_cb,
.driver = {
.name = "qaic_timesync_periodic",
},
};
static void qaic_boot_timesync_worker(struct work_struct *work)
{
struct qts_resp *resp = container_of(work, struct qts_resp, work);
struct qts_host_time_sync_msg_data *req;
struct qts_resp_msg data = resp->data;
struct qaic_device *qdev = resp->qdev;
struct mhi_device *mhi_dev;
struct timespec64 ts;
int ret;
mhi_dev = qdev->qts_ch;
/* Queue the response message beforehand to avoid race conditions */
ret = mhi_queue_buf(mhi_dev, DMA_FROM_DEVICE, &resp->data, sizeof(resp->data), MHI_EOT);
if (ret) {
kfree(resp);
dev_warn(&mhi_dev->dev, "Failed to re-queue response buffer %d\n", ret);
return;
}
switch (data.hdr.msg_type) {
case QAIC_TS_CMD_TO_HOST:
req = kzalloc(sizeof(*req), GFP_KERNEL);
if (!req)
break;
req->header = data.hdr;
req->header.msg_type = QAIC_TS_SYNC_REQ;
ktime_get_real_ts64(&ts);
req->data.tv_sec = cpu_to_le64(ts.tv_sec);
req->data.tv_usec = cpu_to_le64(div_u64(ts.tv_nsec, NSEC_PER_USEC));
ret = mhi_queue_buf(mhi_dev, DMA_TO_DEVICE, req, sizeof(*req), MHI_EOT);
if (ret) {
kfree(req);
dev_dbg(&mhi_dev->dev, "Failed to send request message. Error %d\n", ret);
}
break;
case QAIC_TS_ACK_TO_HOST:
dev_dbg(&mhi_dev->dev, "ACK received from device\n");
break;
default:
dev_err(&mhi_dev->dev, "Invalid message type %u.\n", data.hdr.msg_type);
}
}
static int qaic_boot_timesync_queue_resp(struct mhi_device *mhi_dev, struct qaic_device *qdev)
{
struct qts_resp *resp;
int ret;
resp = kzalloc(sizeof(*resp), GFP_KERNEL);
if (!resp)
return -ENOMEM;
resp->qdev = qdev;
INIT_WORK(&resp->work, qaic_boot_timesync_worker);
ret = mhi_queue_buf(mhi_dev, DMA_FROM_DEVICE, &resp->data, sizeof(resp->data), MHI_EOT);
if (ret) {
kfree(resp);
dev_warn(&mhi_dev->dev, "Failed to queue response buffer %d\n", ret);
return ret;
}
return 0;
}
static void qaic_boot_timesync_remove(struct mhi_device *mhi_dev)
{
struct qaic_device *qdev;
qdev = dev_get_drvdata(&mhi_dev->dev);
mhi_unprepare_from_transfer(qdev->qts_ch);
qdev->qts_ch = NULL;
}
static int qaic_boot_timesync_probe(struct mhi_device *mhi_dev, const struct mhi_device_id *id)
{
struct qaic_device *qdev = pci_get_drvdata(to_pci_dev(mhi_dev->mhi_cntrl->cntrl_dev));
int ret;
ret = mhi_prepare_for_transfer(mhi_dev);
if (ret)
return ret;
qdev->qts_ch = mhi_dev;
dev_set_drvdata(&mhi_dev->dev, qdev);
ret = qaic_boot_timesync_queue_resp(mhi_dev, qdev);
if (ret) {
dev_set_drvdata(&mhi_dev->dev, NULL);
qdev->qts_ch = NULL;
mhi_unprepare_from_transfer(mhi_dev);
}
return ret;
}
static void qaic_boot_timesync_ul_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result)
{
kfree(mhi_result->buf_addr);
}
static void qaic_boot_timesync_dl_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result)
{
struct qts_resp *resp = container_of(mhi_result->buf_addr, struct qts_resp, data);
if (mhi_result->transaction_status || mhi_result->bytes_xferd != sizeof(resp->data)) {
kfree(resp);
return;
}
queue_work(resp->qdev->qts_wq, &resp->work);
}
static const struct mhi_device_id qaic_boot_timesync_match_table[] = {
{ .chan = "QAIC_TIMESYNC"},
{},
};
static struct mhi_driver qaic_boot_timesync_driver = {
.id_table = qaic_boot_timesync_match_table,
.remove = qaic_boot_timesync_remove,
.probe = qaic_boot_timesync_probe,
.ul_xfer_cb = qaic_boot_timesync_ul_xfer_cb,
.dl_xfer_cb = qaic_boot_timesync_dl_xfer_cb,
.driver = {
.name = "qaic_timesync",
},
};
int qaic_timesync_init(void)
{
int ret;
ret = mhi_driver_register(&qaic_timesync_driver);
if (ret)
return ret;
ret = mhi_driver_register(&qaic_boot_timesync_driver);
return ret;
}
void qaic_timesync_deinit(void)
{
mhi_driver_unregister(&qaic_boot_timesync_driver);
mhi_driver_unregister(&qaic_timesync_driver);
}

11
drivers/accel/qaic/qaic_timesync.h Normal file
View File

@ -0,0 +1,11 @@
/* SPDX-License-Identifier: GPL-2.0-only
*
* Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
*/
#ifndef __QAIC_TIMESYNC_H__
#define __QAIC_TIMESYNC_H__
int qaic_timesync_init(void);
void qaic_timesync_deinit(void);
#endif /* __QAIC_TIMESYNC_H__ */

12
drivers/gpu/drm/Kconfig
View File

@ -75,15 +75,15 @@ config DRM_KUNIT_TEST_HELPERS
config DRM_KUNIT_TEST
tristate "KUnit tests for DRM" if !KUNIT_ALL_TESTS
depends on DRM && KUNIT
select PRIME_NUMBERS
select DRM_BUDDY
select DRM_DISPLAY_DP_HELPER
select DRM_DISPLAY_HELPER
select DRM_LIB_RANDOM
select DRM_KMS_HELPER
select DRM_BUDDY
select DRM_EXPORT_FOR_TESTS if m
select DRM_KUNIT_TEST_HELPERS
select DRM_EXEC
select DRM_EXPORT_FOR_TESTS if m
select DRM_KMS_HELPER
select DRM_KUNIT_TEST_HELPERS
select DRM_LIB_RANDOM
select PRIME_NUMBERS
default KUNIT_ALL_TESTS
help
This builds unit tests for DRM. This option is not useful for

1
drivers/gpu/drm/Makefile
View File

@ -22,6 +22,7 @@ drm-y := \
drm_drv.o \
drm_dumb_buffers.o \
drm_edid.o \
drm_eld.o \
drm_encoder.o \
drm_file.o \
drm_fourcc.o \

2
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c
View File

@ -290,7 +290,7 @@ static int suspend_resume_compute_scheduler(struct amdgpu_device *adev, bool sus
for (i = 0; i < adev->gfx.num_compute_rings; i++) {
struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
if (!(ring && ring->sched.thread))
if (!(ring && drm_sched_wqueue_ready(&ring->sched)))
continue;
/* stop secheduler and drain ring. */

15
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
View File

@ -1665,9 +1665,9 @@ static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)
for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
struct amdgpu_ring *ring = adev->rings[i];
if (!ring || !ring->sched.thread)
if (!ring || !drm_sched_wqueue_ready(&ring->sched))
continue;
kthread_park(ring->sched.thread);
drm_sched_wqueue_stop(&ring->sched);
}
seq_puts(m, "run ib test:\n");
@ -1681,9 +1681,9 @@ static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)
for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
struct amdgpu_ring *ring = adev->rings[i];
if (!ring || !ring->sched.thread)
if (!ring || !drm_sched_wqueue_ready(&ring->sched))
continue;
kthread_unpark(ring->sched.thread);
drm_sched_wqueue_start(&ring->sched);
}
up_write(&adev->reset_domain->sem);
@ -1903,7 +1903,8 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
ring = adev->rings[val];
if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
if (!ring || !ring->funcs->preempt_ib ||
!drm_sched_wqueue_ready(&ring->sched))
return -EINVAL;
/* the last preemption failed */
@ -1921,7 +1922,7 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
goto pro_end;
/* stop the scheduler */
kthread_park(ring->sched.thread);
drm_sched_wqueue_stop(&ring->sched);
/* preempt the IB */
r = amdgpu_ring_preempt_ib(ring);
@ -1955,7 +1956,7 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
failure:
/* restart the scheduler */
kthread_unpark(ring->sched.thread);
drm_sched_wqueue_start(&ring->sched);
up_read(&adev->reset_domain->sem);

14
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
View File

@ -2573,7 +2573,7 @@ static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
break;
}
r = drm_sched_init(&ring->sched, &amdgpu_sched_ops,
r = drm_sched_init(&ring->sched, &amdgpu_sched_ops, NULL,
DRM_SCHED_PRIORITY_COUNT,
ring->num_hw_submission, 0,
timeout, adev->reset_domain->wq,
@ -4964,7 +4964,7 @@ bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
struct amdgpu_ring *ring = adev->rings[i];
if (!ring || !ring->sched.thread)
if (!ring || !drm_sched_wqueue_ready(&ring->sched))
continue;
spin_lock(&ring->sched.job_list_lock);
@ -5103,7 +5103,7 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
struct amdgpu_ring *ring = adev->rings[i];
if (!ring || !ring->sched.thread)
if (!ring || !drm_sched_wqueue_ready(&ring->sched))
continue;
/* Clear job fence from fence drv to avoid force_completion
@ -5592,7 +5592,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
struct amdgpu_ring *ring = tmp_adev->rings[i];
if (!ring || !ring->sched.thread)
if (!ring || !drm_sched_wqueue_ready(&ring->sched))
continue;
drm_sched_stop(&ring->sched, job ? &job->base : NULL);
@ -5668,7 +5668,7 @@ skip_hw_reset:
for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
struct amdgpu_ring *ring = tmp_adev->rings[i];
if (!ring || !ring->sched.thread)
if (!ring || !drm_sched_wqueue_ready(&ring->sched))
continue;
drm_sched_start(&ring->sched, true);
@ -5991,7 +5991,7 @@ pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_sta
for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
struct amdgpu_ring *ring = adev->rings[i];
if (!ring || !ring->sched.thread)
if (!ring || !drm_sched_wqueue_ready(&ring->sched))
continue;
drm_sched_stop(&ring->sched, NULL);
@ -6119,7 +6119,7 @@ void amdgpu_pci_resume(struct pci_dev *pdev)
for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
struct amdgpu_ring *ring = adev->rings[i];
if (!ring || !ring->sched.thread)
if (!ring || !drm_sched_wqueue_ready(&ring->sched))
continue;
drm_sched_start(&ring->sched, true);

2
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
View File

@ -115,7 +115,7 @@ int amdgpu_job_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
if (!entity)
return 0;
return drm_sched_job_init(&(*job)->base, entity, owner);
return drm_sched_job_init(&(*job)->base, entity, 1, owner);
}
int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev,

1
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
View File

@ -87,6 +87,7 @@
#include <drm/drm_blend.h>
#include <drm/drm_fourcc.h>
#include <drm/drm_edid.h>
#include <drm/drm_eld.h>
#include <drm/drm_vblank.h>
#include <drm/drm_audio_component.h>
#include <drm/drm_gem_atomic_helper.h>

4
drivers/gpu/drm/drm_atomic_helper.c
View File

@ -2382,10 +2382,10 @@ int drm_atomic_helper_setup_commit(struct drm_atomic_state *state,
EXPORT_SYMBOL(drm_atomic_helper_setup_commit);
/**
* drm_atomic_helper_wait_for_dependencies - wait for required preceeding commits
* drm_atomic_helper_wait_for_dependencies - wait for required preceding commits
* @old_state: atomic state object with old state structures
*
* This function waits for all preceeding commits that touch the same CRTC as
* This function waits for all preceding commits that touch the same CRTC as
* @old_state to both be committed to the hardware (as signalled by
* drm_atomic_helper_commit_hw_done()) and executed by the hardware (as signalled
* by calling drm_crtc_send_vblank_event() on the &drm_crtc_state.event).

12
drivers/gpu/drm/drm_client.c
View File

@ -5,7 +5,6 @@
#include <linux/iosys-map.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/seq_file.h>
#include <linux/slab.h>
@ -84,16 +83,13 @@ int drm_client_init(struct drm_device *dev, struct drm_client_dev *client,
if (!drm_core_check_feature(dev, DRIVER_MODESET) || !dev->driver->dumb_create)
return -EOPNOTSUPP;
if (funcs && !try_module_get(funcs->owner))
return -ENODEV;
client->dev = dev;
client->name = name;
client->funcs = funcs;
ret = drm_client_modeset_create(client);
if (ret)
goto err_put_module;
return ret;
ret = drm_client_open(client);
if (ret)
@ -105,10 +101,6 @@ int drm_client_init(struct drm_device *dev, struct drm_client_dev *client,
err_free:
drm_client_modeset_free(client);
err_put_module:
if (funcs)
module_put(funcs->owner);
return ret;
}
EXPORT_SYMBOL(drm_client_init);
@ -177,8 +169,6 @@ void drm_client_release(struct drm_client_dev *client)
drm_client_modeset_free(client);
drm_client_close(client);
drm_dev_put(dev);
if (client->funcs)
module_put(client->funcs->owner);
}
EXPORT_SYMBOL(drm_client_release);

6
drivers/gpu/drm/drm_connector.c
View File

@ -1198,6 +1198,12 @@ static const u32 dp_colorspaces =
* drm_connector_set_path_property(), in the case of DP MST with the
* path property the MST manager created. Userspace cannot change this
* property.
*
* In the case of DP MST, the property has the format
* ``mst:<parent>-<ports>`` where ``<parent>`` is the KMS object ID of the
* parent connector and ``<ports>`` is a hyphen-separated list of DP MST
* port numbers. Note, KMS object IDs are not guaranteed to be stable
* across reboots.
* TILE:
* Connector tile group property to indicate how a set of DRM connector
* compose together into one logical screen. This is used by both high-res

2
drivers/gpu/drm/drm_crtc_internal.h
View File

@ -222,6 +222,8 @@ int drm_mode_addfb2_ioctl(struct drm_device *dev,
void *data, struct drm_file *file_priv);
int drm_mode_rmfb_ioctl(struct drm_device *dev,
void *data, struct drm_file *file_priv);
int drm_mode_closefb_ioctl(struct drm_device *dev,
void *data, struct drm_file *file_priv);
int drm_mode_getfb(struct drm_device *dev,
void *data, struct drm_file *file_priv);
int drm_mode_getfb2_ioctl(struct drm_device *dev,

43
drivers/gpu/drm/drm_edid.c
View File

@ -41,10 +41,12 @@
#include <drm/drm_displayid.h>
#include <drm/drm_drv.h>
#include <drm/drm_edid.h>
#include <drm/drm_eld.h>
#include <drm/drm_encoder.h>
#include <drm/drm_print.h>
#include "drm_crtc_internal.h"
#include "drm_internal.h"
static int oui(u8 first, u8 second, u8 third)
{
@ -5509,6 +5511,27 @@ static void clear_eld(struct drm_connector *connector)
connector->audio_latency[1] = 0;
}
/*
* Get 3-byte SAD buffer from struct cea_sad.
*/
void drm_edid_cta_sad_get(const struct cea_sad *cta_sad, u8 *sad)
{
sad[0] = cta_sad->format << 3 | cta_sad->channels;
sad[1] = cta_sad->freq;
sad[2] = cta_sad->byte2;
}
/*
* Set struct cea_sad from 3-byte SAD buffer.
*/
void drm_edid_cta_sad_set(struct cea_sad *cta_sad, const u8 *sad)
{
cta_sad->format = (sad[0] & 0x78) >> 3;
cta_sad->channels = sad[0] & 0x07;
cta_sad->freq = sad[1] & 0x7f;
cta_sad->byte2 = sad[2];
}
/*
* drm_edid_to_eld - build ELD from EDID
* @connector: connector corresponding to the HDMI/DP sink
@ -5593,7 +5616,7 @@ static void drm_edid_to_eld(struct drm_connector *connector,
}
static int _drm_edid_to_sad(const struct drm_edid *drm_edid,
struct cea_sad **sads)
struct cea_sad **psads)
{
const struct cea_db *db;
struct cea_db_iter iter;
@ -5602,20 +5625,16 @@ static int _drm_edid_to_sad(const struct drm_edid *drm_edid,
cea_db_iter_edid_begin(drm_edid, &iter);
cea_db_iter_for_each(db, &iter) {
if (cea_db_tag(db) == CTA_DB_AUDIO) {
int j;
struct cea_sad *sads;
int i;
count = cea_db_payload_len(db) / 3; /* SAD is 3B */
*sads = kcalloc(count, sizeof(**sads), GFP_KERNEL);
if (!*sads)
sads = kcalloc(count, sizeof(*sads), GFP_KERNEL);
*psads = sads;
if (!sads)
return -ENOMEM;
for (j = 0; j < count; j++) {
const u8 *sad = &db->data[j * 3];
(*sads)[j].format = (sad[0] & 0x78) >> 3;
(*sads)[j].channels = sad[0] & 0x7;
(*sads)[j].freq = sad[1] & 0x7F;
(*sads)[j].byte2 = sad[2];
}
for (i = 0; i < count; i++)
drm_edid_cta_sad_set(&sads[i], &db->data[i * 3]);
break;
}
}

55
drivers/gpu/drm/drm_eld.c Normal file
View File

@ -0,0 +1,55 @@
// SPDX-License-Identifier: MIT
/*
* Copyright © 2023 Intel Corporation
*/
#include <drm/drm_edid.h>
#include <drm/drm_eld.h>
#include "drm_internal.h"
/**
* drm_eld_sad_get - get SAD from ELD to struct cea_sad
* @eld: ELD buffer
* @sad_index: SAD index
* @cta_sad: destination struct cea_sad
*
* @return: 0 on success, or negative on errors
*/
int drm_eld_sad_get(const u8 *eld, int sad_index, struct cea_sad *cta_sad)
{
const u8 *sad;
if (sad_index >= drm_eld_sad_count(eld))
return -EINVAL;
sad = eld + DRM_ELD_CEA_SAD(drm_eld_mnl(eld), sad_index);
drm_edid_cta_sad_set(cta_sad, sad);
return 0;
}
EXPORT_SYMBOL(drm_eld_sad_get);
/**
* drm_eld_sad_set - set SAD to ELD from struct cea_sad
* @eld: ELD buffer
* @sad_index: SAD index
* @cta_sad: source struct cea_sad
*
* @return: 0 on success, or negative on errors
*/
int drm_eld_sad_set(u8 *eld, int sad_index, const struct cea_sad *cta_sad)
{
u8 *sad;
if (sad_index >= drm_eld_sad_count(eld))
return -EINVAL;
sad = eld + DRM_ELD_CEA_SAD(drm_eld_mnl(eld), sad_index);
drm_edid_cta_sad_get(cta_sad, sad);
return 0;
}
EXPORT_SYMBOL(drm_eld_sad_set);

2
drivers/gpu/drm/drm_file.c
View File

@ -913,7 +913,7 @@ static void print_size(struct drm_printer *p, const char *stat,
unsigned u;
for (u = 0; u < ARRAY_SIZE(units) - 1; u++) {
if (sz < SZ_1K)
if (sz == 0 || !IS_ALIGNED(sz, SZ_1K))
break;
sz = div_u64(sz, SZ_1K);
}

27
drivers/gpu/drm/drm_flip_work.c
View File

@ -27,14 +27,12 @@
#include <drm/drm_print.h>
#include <drm/drm_util.h>
/**
* drm_flip_work_allocate_task - allocate a flip-work task
* @data: data associated to the task
* @flags: allocator flags
*
* Allocate a drm_flip_task object and attach private data to it.
*/
struct drm_flip_task *drm_flip_work_allocate_task(void *data, gfp_t flags)
struct drm_flip_task {
struct list_head node;
void *data;
};
static struct drm_flip_task *drm_flip_work_allocate_task(void *data, gfp_t flags)
{
struct drm_flip_task *task;
@ -44,18 +42,8 @@ struct drm_flip_task *drm_flip_work_allocate_task(void *data, gfp_t flags)
return task;
}
EXPORT_SYMBOL(drm_flip_work_allocate_task);
/**
* drm_flip_work_queue_task - queue a specific task
* @work: the flip-work
* @task: the task to handle
*
* Queues task, that will later be run (passed back to drm_flip_func_t
* func) on a work queue after drm_flip_work_commit() is called.
*/
void drm_flip_work_queue_task(struct drm_flip_work *work,
struct drm_flip_task *task)
static void drm_flip_work_queue_task(struct drm_flip_work *work, struct drm_flip_task *task)
{
unsigned long flags;
@ -63,7 +51,6 @@ void drm_flip_work_queue_task(struct drm_flip_work *work,
list_add_tail(&task->node, &work->queued);
spin_unlock_irqrestore(&work->lock, flags);
}
EXPORT_SYMBOL(drm_flip_work_queue_task);
/**
* drm_flip_work_queue - queue work

215
drivers/gpu/drm/drm_format_helper.c
View File

@ -20,6 +20,97 @@
#include <drm/drm_print.h>
#include <drm/drm_rect.h>
/**
* drm_format_conv_state_init - Initialize format-conversion state
* @state: The state to initialize
*
* Clears all fields in struct drm_format_conv_state. The state will
* be empty with no preallocated resources.
*/
void drm_format_conv_state_init(struct drm_format_conv_state *state)
{
state->tmp.mem = NULL;
state->tmp.size = 0;
state->tmp.preallocated = false;
}
EXPORT_SYMBOL(drm_format_conv_state_init);
/**
* drm_format_conv_state_copy - Copy format-conversion state
* @state: Destination state
* @old_state: Source state
*
* Copies format-conversion state from @old_state to @state; except for
* temporary storage.
*/
void drm_format_conv_state_copy(struct drm_format_conv_state *state,
const struct drm_format_conv_state *old_state)
{
/*
* So far, there's only temporary storage here, which we don't
* duplicate. Just clear the fields.
*/
state->tmp.mem = NULL;
state->tmp.size = 0;
state->tmp.preallocated = false;
}
EXPORT_SYMBOL(drm_format_conv_state_copy);
/**
* drm_format_conv_state_reserve - Allocates storage for format conversion
* @state: The format-conversion state
* @new_size: The minimum allocation size
* @flags: Flags for kmalloc()
*
* Allocates at least @new_size bytes and returns a pointer to the memory
* range. After calling this function, previously returned memory blocks
* are invalid. It's best to collect all memory requirements of a format
* conversion and call this function once to allocate the range.
*
* Returns:
* A pointer to the allocated memory range, or NULL otherwise.
*/
void *drm_format_conv_state_reserve(struct drm_format_conv_state *state,
size_t new_size, gfp_t flags)
{
void *mem;
if (new_size <= state->tmp.size)
goto out;
else if (state->tmp.preallocated)
return NULL;
mem = krealloc(state->tmp.mem, new_size, flags);
if (!mem)
return NULL;
state->tmp.mem = mem;
state->tmp.size = new_size;
out:
return state->tmp.mem;
}
EXPORT_SYMBOL(drm_format_conv_state_reserve);
/**
* drm_format_conv_state_release - Releases an format-conversion storage
* @state: The format-conversion state
*
* Releases the memory range references by the format-conversion state.
* After this call, all pointers to the memory are invalid. Prefer
* drm_format_conv_state_init() for cleaning up and unloading a driver.
*/
void drm_format_conv_state_release(struct drm_format_conv_state *state)
{
if (state->tmp.preallocated)
return;
kfree(state->tmp.mem);
state->tmp.mem = NULL;
state->tmp.size = 0;
}
EXPORT_SYMBOL(drm_format_conv_state_release);
static unsigned int clip_offset(const struct drm_rect *clip, unsigned int pitch, unsigned int cpp)
{
return clip->y1 * pitch + clip->x1 * cpp;
@ -45,6 +136,7 @@ EXPORT_SYMBOL(drm_fb_clip_offset);
static int __drm_fb_xfrm(void *dst, unsigned long dst_pitch, unsigned long dst_pixsize,
const void *vaddr, const struct drm_framebuffer *fb,
const struct drm_rect *clip, bool vaddr_cached_hint,
struct drm_format_conv_state *state,
void (*xfrm_line)(void *dbuf, const void *sbuf, unsigned int npixels))
{
unsigned long linepixels = drm_rect_width(clip);
@ -60,7 +152,7 @@ static int __drm_fb_xfrm(void *dst, unsigned long dst_pitch, unsigned long dst_p
* one line at a time.
*/
if (!vaddr_cached_hint) {
stmp = kmalloc(sbuf_len, GFP_KERNEL);
stmp = drm_format_conv_state_reserve(state, sbuf_len, GFP_KERNEL);
if (!stmp)
return -ENOMEM;
}
@ -79,8 +171,6 @@ static int __drm_fb_xfrm(void *dst, unsigned long dst_pitch, unsigned long dst_p
dst += dst_pitch;
}
kfree(stmp);
return 0;
}
@ -88,6 +178,7 @@ static int __drm_fb_xfrm(void *dst, unsigned long dst_pitch, unsigned long dst_p
static int __drm_fb_xfrm_toio(void __iomem *dst, unsigned long dst_pitch, unsigned long dst_pixsize,
const void *vaddr, const struct drm_framebuffer *fb,
const struct drm_rect *clip, bool vaddr_cached_hint,
struct drm_format_conv_state *state,
void (*xfrm_line)(void *dbuf, const void *sbuf, unsigned int npixels))
{
unsigned long linepixels = drm_rect_width(clip);
@ -101,9 +192,9 @@ static int __drm_fb_xfrm_toio(void __iomem *dst, unsigned long dst_pitch, unsign
void *dbuf;
if (vaddr_cached_hint) {
dbuf = kmalloc(dbuf_len, GFP_KERNEL);
dbuf = drm_format_conv_state_reserve(state, dbuf_len, GFP_KERNEL);
} else {
dbuf = kmalloc(stmp_off + sbuf_len, GFP_KERNEL);
dbuf = drm_format_conv_state_reserve(state, stmp_off + sbuf_len, GFP_KERNEL);
stmp = dbuf + stmp_off;
}
if (!dbuf)
@ -124,8 +215,6 @@ static int __drm_fb_xfrm_toio(void __iomem *dst, unsigned long dst_pitch, unsign
dst += dst_pitch;
}
kfree(dbuf);
return 0;
}
@ -134,6 +223,7 @@ static int drm_fb_xfrm(struct iosys_map *dst,
const unsigned int *dst_pitch, const u8 *dst_pixsize,
const struct iosys_map *src, const struct drm_framebuffer *fb,
const struct drm_rect *clip, bool vaddr_cached_hint,
struct drm_format_conv_state *state,
void (*xfrm_line)(void *dbuf, const void *sbuf, unsigned int npixels))
{
static const unsigned int default_dst_pitch[DRM_FORMAT_MAX_PLANES] = {
@ -146,10 +236,12 @@ static int drm_fb_xfrm(struct iosys_map *dst,
/* TODO: handle src in I/O memory here */
if (dst[0].is_iomem)
return __drm_fb_xfrm_toio(dst[0].vaddr_iomem, dst_pitch[0], dst_pixsize[0],
src[0].vaddr, fb, clip, vaddr_cached_hint, xfrm_line);
src[0].vaddr, fb, clip, vaddr_cached_hint, state,
xfrm_line);
else
return __drm_fb_xfrm(dst[0].vaddr, dst_pitch[0], dst_pixsize[0],
src[0].vaddr, fb, clip, vaddr_cached_hint, xfrm_line);
src[0].vaddr, fb, clip, vaddr_cached_hint, state,
xfrm_line);
}
/**
@ -235,6 +327,7 @@ static void drm_fb_swab32_line(void *dbuf, const void *sbuf, unsigned int pixels
* @fb: DRM framebuffer
* @clip: Clip rectangle area to copy
* @cached: Source buffer is mapped cached (eg. not write-combined)
* @state: Transform and conversion state
*
* This function copies parts of a framebuffer to display memory and swaps per-pixel
* bytes during the process. Destination and framebuffer formats must match. The
@ -249,7 +342,8 @@ static void drm_fb_swab32_line(void *dbuf, const void *sbuf, unsigned int pixels
*/
void drm_fb_swab(struct iosys_map *dst, const unsigned int *dst_pitch,
const struct iosys_map *src, const struct drm_framebuffer *fb,
const struct drm_rect *clip, bool cached)
const struct drm_rect *clip, bool cached,
struct drm_format_conv_state *state)
{
const struct drm_format_info *format = fb->format;
u8 cpp = DIV_ROUND_UP(drm_format_info_bpp(format, 0), 8);
@ -268,7 +362,7 @@ void drm_fb_swab(struct iosys_map *dst, const unsigned int *dst_pitch,
return;
}
drm_fb_xfrm(dst, dst_pitch, &cpp, src, fb, clip, cached, swab_line);
drm_fb_xfrm(dst, dst_pitch, &cpp, src, fb, clip, cached, state, swab_line);
}
EXPORT_SYMBOL(drm_fb_swab);
@ -295,6 +389,7 @@ static void drm_fb_xrgb8888_to_rgb332_line(void *dbuf, const void *sbuf, unsigne
* @src: Array of XRGB8888 source buffers
* @fb: DRM framebuffer
* @clip: Clip rectangle area to copy
* @state: Transform and conversion state
*
* This function copies parts of a framebuffer to display memory and converts the
* color format during the process. Destination and framebuffer formats must match. The
@ -309,13 +404,13 @@ static void drm_fb_xrgb8888_to_rgb332_line(void *dbuf, const void *sbuf, unsigne
*/
void drm_fb_xrgb8888_to_rgb332(struct iosys_map *dst, const unsigned int *dst_pitch,
const struct iosys_map *src, const struct drm_framebuffer *fb,
const struct drm_rect *clip)
const struct drm_rect *clip, struct drm_format_conv_state *state)
{
static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
1,
};
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state,
drm_fb_xrgb8888_to_rgb332_line);
}
EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb332);
@ -364,6 +459,7 @@ static void drm_fb_xrgb8888_to_rgb565_swab_line(void *dbuf, const void *sbuf,
* @src: Array of XRGB8888 source buffer
* @fb: DRM framebuffer
* @clip: Clip rectangle area to copy
* @state: Transform and conversion state
* @swab: Swap bytes
*
* This function copies parts of a framebuffer to display memory and converts the
@ -379,7 +475,8 @@ static void drm_fb_xrgb8888_to_rgb565_swab_line(void *dbuf, const void *sbuf,
*/
void drm_fb_xrgb8888_to_rgb565(struct iosys_map *dst, const unsigned int *dst_pitch,
const struct iosys_map *src, const struct drm_framebuffer *fb,
const struct drm_rect *clip, bool swab)
const struct drm_rect *clip, struct drm_format_conv_state *state,
bool swab)
{
static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
2,
@ -392,7 +489,7 @@ void drm_fb_xrgb8888_to_rgb565(struct iosys_map *dst, const unsigned int *dst_pi
else
xfrm_line = drm_fb_xrgb8888_to_rgb565_line;
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, xfrm_line);
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state, xfrm_line);
}
EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb565);
@ -421,6 +518,7 @@ static void drm_fb_xrgb8888_to_xrgb1555_line(void *dbuf, const void *sbuf, unsig
* @src: Array of XRGB8888 source buffer
* @fb: DRM framebuffer
* @clip: Clip rectangle area to copy
* @state: Transform and conversion state
*
* This function copies parts of a framebuffer to display memory and converts
* the color format during the process. The parameters @dst, @dst_pitch and
@ -436,13 +534,13 @@ static void drm_fb_xrgb8888_to_xrgb1555_line(void *dbuf, const void *sbuf, unsig
*/
void drm_fb_xrgb8888_to_xrgb1555(struct iosys_map *dst, const unsigned int *dst_pitch,
const struct iosys_map *src, const struct drm_framebuffer *fb,
const struct drm_rect *clip)
const struct drm_rect *clip, struct drm_format_conv_state *state)
{
static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
2,
};
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state,
drm_fb_xrgb8888_to_xrgb1555_line);
}
EXPORT_SYMBOL(drm_fb_xrgb8888_to_xrgb1555);
@ -473,6 +571,7 @@ static void drm_fb_xrgb8888_to_argb1555_line(void *dbuf, const void *sbuf, unsig
* @src: Array of XRGB8888 source buffer
* @fb: DRM framebuffer
* @clip: Clip rectangle area to copy
* @state: Transform and conversion state
*
* This function copies parts of a framebuffer to display memory and converts
* the color format during the process. The parameters @dst, @dst_pitch and
@ -488,13 +587,13 @@ static void drm_fb_xrgb8888_to_argb1555_line(void *dbuf, const void *sbuf, unsig
*/
void drm_fb_xrgb8888_to_argb1555(struct iosys_map *dst, const unsigned int *dst_pitch,
const struct iosys_map *src, const struct drm_framebuffer *fb,
const struct drm_rect *clip)
const struct drm_rect *clip, struct drm_format_conv_state *state)
{
static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
2,
};
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state,
drm_fb_xrgb8888_to_argb1555_line);
}
EXPORT_SYMBOL(drm_fb_xrgb8888_to_argb1555);
@ -525,6 +624,7 @@ static void drm_fb_xrgb8888_to_rgba5551_line(void *dbuf, const void *sbuf, unsig
* @src: Array of XRGB8888 source buffer
* @fb: DRM framebuffer
* @clip: Clip rectangle area to copy
* @state: Transform and conversion state
*
* This function copies parts of a framebuffer to display memory and converts
* the color format during the process. The parameters @dst, @dst_pitch and
@ -540,13 +640,13 @@ static void drm_fb_xrgb8888_to_rgba5551_line(void *dbuf, const void *sbuf, unsig
*/
void drm_fb_xrgb8888_to_rgba5551(struct iosys_map *dst, const unsigned int *dst_pitch,
const struct iosys_map *src, const struct drm_framebuffer *fb,
const struct drm_rect *clip)
const struct drm_rect *clip, struct drm_format_conv_state *state)
{
static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
2,
};
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state,
drm_fb_xrgb8888_to_rgba5551_line);
}
EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgba5551);
@ -575,6 +675,7 @@ static void drm_fb_xrgb8888_to_rgb888_line(void *dbuf, const void *sbuf, unsigne
* @src: Array of XRGB8888 source buffers
* @fb: DRM framebuffer
* @clip: Clip rectangle area to copy
* @state: Transform and conversion state
*
* This function copies parts of a framebuffer to display memory and converts the
* color format during the process. Destination and framebuffer formats must match. The
@ -590,13 +691,13 @@ static void drm_fb_xrgb8888_to_rgb888_line(void *dbuf, const void *sbuf, unsigne
*/
void drm_fb_xrgb8888_to_rgb888(struct iosys_map *dst, const unsigned int *dst_pitch,
const struct iosys_map *src, const struct drm_framebuffer *fb,
const struct drm_rect *clip)
const struct drm_rect *clip, struct drm_format_conv_state *state)
{
static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
3,
};
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state,
drm_fb_xrgb8888_to_rgb888_line);
}
EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb888);
@ -623,6 +724,7 @@ static void drm_fb_xrgb8888_to_argb8888_line(void *dbuf, const void *sbuf, unsig
* @src: Array of XRGB8888 source buffer
* @fb: DRM framebuffer
* @clip: Clip rectangle area to copy
* @state: Transform and conversion state
*
* This function copies parts of a framebuffer to display memory and converts the
* color format during the process. The parameters @dst, @dst_pitch and @src refer
@ -638,13 +740,13 @@ static void drm_fb_xrgb8888_to_argb8888_line(void *dbuf, const void *sbuf, unsig
*/
void drm_fb_xrgb8888_to_argb8888(struct iosys_map *dst, const unsigned int *dst_pitch,
const struct iosys_map *src, const struct drm_framebuffer *fb,
const struct drm_rect *clip)
const struct drm_rect *clip, struct drm_format_conv_state *state)
{
static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
4,
};
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state,
drm_fb_xrgb8888_to_argb8888_line);
}
EXPORT_SYMBOL(drm_fb_xrgb8888_to_argb8888);
@ -669,13 +771,14 @@ static void drm_fb_xrgb8888_to_abgr8888_line(void *dbuf, const void *sbuf, unsig
static void drm_fb_xrgb8888_to_abgr8888(struct iosys_map *dst, const unsigned int *dst_pitch,
const struct iosys_map *src,
const struct drm_framebuffer *fb,
const struct drm_rect *clip)
const struct drm_rect *clip,
struct drm_format_conv_state *state)
{
static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
4,
};
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state,
drm_fb_xrgb8888_to_abgr8888_line);
}
@ -699,13 +802,14 @@ static void drm_fb_xrgb8888_to_xbgr8888_line(void *dbuf, const void *sbuf, unsig
static void drm_fb_xrgb8888_to_xbgr8888(struct iosys_map *dst, const unsigned int *dst_pitch,
const struct iosys_map *src,
const struct drm_framebuffer *fb,
const struct drm_rect *clip)
const struct drm_rect *clip,
struct drm_format_conv_state *state)
{
static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
4,
};
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state,
drm_fb_xrgb8888_to_xbgr8888_line);
}
@ -735,6 +839,7 @@ static void drm_fb_xrgb8888_to_xrgb2101010_line(void *dbuf, const void *sbuf, un
* @src: Array of XRGB8888 source buffers
* @fb: DRM framebuffer
* @clip: Clip rectangle area to copy
* @state: Transform and conversion state
*
* This function copies parts of a framebuffer to display memory and converts the
* color format during the process. Destination and framebuffer formats must match. The
@ -750,13 +855,14 @@ static void drm_fb_xrgb8888_to_xrgb2101010_line(void *dbuf, const void *sbuf, un
*/
void drm_fb_xrgb8888_to_xrgb2101010(struct iosys_map *dst, const unsigned int *dst_pitch,
const struct iosys_map *src, const struct drm_framebuffer *fb,
const struct drm_rect *clip)
const struct drm_rect *clip,
struct drm_format_conv_state *state)
{
static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
4,
};
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state,
drm_fb_xrgb8888_to_xrgb2101010_line);
}
EXPORT_SYMBOL(drm_fb_xrgb8888_to_xrgb2101010);
@ -788,6 +894,7 @@ static void drm_fb_xrgb8888_to_argb2101010_line(void *dbuf, const void *sbuf, un
* @src: Array of XRGB8888 source buffers
* @fb: DRM framebuffer
* @clip: Clip rectangle area to copy
* @state: Transform and conversion state
*
* This function copies parts of a framebuffer to display memory and converts
* the color format during the process. The parameters @dst, @dst_pitch and
@ -803,13 +910,14 @@ static void drm_fb_xrgb8888_to_argb2101010_line(void *dbuf, const void *sbuf, un
*/
void drm_fb_xrgb8888_to_argb2101010(struct iosys_map *dst, const unsigned int *dst_pitch,
const struct iosys_map *src, const struct drm_framebuffer *fb,
const struct drm_rect *clip)
const struct drm_rect *clip,
struct drm_format_conv_state *state)
{
static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
4,
};
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state,
drm_fb_xrgb8888_to_argb2101010_line);
}
EXPORT_SYMBOL(drm_fb_xrgb8888_to_argb2101010);
@ -839,6 +947,7 @@ static void drm_fb_xrgb8888_to_gray8_line(void *dbuf, const void *sbuf, unsigned
* @src: Array of XRGB8888 source buffers
* @fb: DRM framebuffer
* @clip: Clip rectangle area to copy
* @state: Transform and conversion state
*
* This function copies parts of a framebuffer to display memory and converts the
* color format during the process. Destination and framebuffer formats must match. The
@ -858,13 +967,13 @@ static void drm_fb_xrgb8888_to_gray8_line(void *dbuf, const void *sbuf, unsigned
*/
void drm_fb_xrgb8888_to_gray8(struct iosys_map *dst, const unsigned int *dst_pitch,
const struct iosys_map *src, const struct drm_framebuffer *fb,
const struct drm_rect *clip)
const struct drm_rect *clip, struct drm_format_conv_state *state)
{
static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
1,
};
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state,
drm_fb_xrgb8888_to_gray8_line);
}
EXPORT_SYMBOL(drm_fb_xrgb8888_to_gray8);
@ -878,6 +987,7 @@ EXPORT_SYMBOL(drm_fb_xrgb8888_to_gray8);
* @src: The framebuffer memory to copy from
* @fb: The framebuffer to copy from
* @clip: Clip rectangle area to copy
* @state: Transform and conversion state
*
* This function copies parts of a framebuffer to display memory. If the
* formats of the display and the framebuffer mismatch, the blit function
@ -896,7 +1006,7 @@ EXPORT_SYMBOL(drm_fb_xrgb8888_to_gray8);
*/
int drm_fb_blit(struct iosys_map *dst, const unsigned int *dst_pitch, uint32_t dst_format,
const struct iosys_map *src, const struct drm_framebuffer *fb,
const struct drm_rect *clip)
const struct drm_rect *clip, struct drm_format_conv_state *state)
{
uint32_t fb_format = fb->format->format;
@ -904,44 +1014,44 @@ int drm_fb_blit(struct iosys_map *dst, const unsigned int *dst_pitch, uint32_t d
drm_fb_memcpy(dst, dst_pitch, src, fb, clip);
return 0;
} else if (fb_format == (dst_format | DRM_FORMAT_BIG_ENDIAN)) {
drm_fb_swab(dst, dst_pitch, src, fb, clip, false);
drm_fb_swab(dst, dst_pitch, src, fb, clip, false, state);
return 0;
} else if (fb_format == (dst_format & ~DRM_FORMAT_BIG_ENDIAN)) {
drm_fb_swab(dst, dst_pitch, src, fb, clip, false);
drm_fb_swab(dst, dst_pitch, src, fb, clip, false, state);
return 0;
} else if (fb_format == DRM_FORMAT_XRGB8888) {
if (dst_format == DRM_FORMAT_RGB565) {
drm_fb_xrgb8888_to_rgb565(dst, dst_pitch, src, fb, clip, false);
drm_fb_xrgb8888_to_rgb565(dst, dst_pitch, src, fb, clip, state, false);
return 0;
} else if (dst_format == DRM_FORMAT_XRGB1555) {
drm_fb_xrgb8888_to_xrgb1555(dst, dst_pitch, src, fb, clip);
drm_fb_xrgb8888_to_xrgb1555(dst, dst_pitch, src, fb, clip, state);
return 0;
} else if (dst_format == DRM_FORMAT_ARGB1555) {
drm_fb_xrgb8888_to_argb1555(dst, dst_pitch, src, fb, clip);
drm_fb_xrgb8888_to_argb1555(dst, dst_pitch, src, fb, clip, state);
return 0;
} else if (dst_format == DRM_FORMAT_RGBA5551) {
drm_fb_xrgb8888_to_rgba5551(dst, dst_pitch, src, fb, clip);
drm_fb_xrgb8888_to_rgba5551(dst, dst_pitch, src, fb, clip, state);
return 0;
} else if (dst_format == DRM_FORMAT_RGB888) {
drm_fb_xrgb8888_to_rgb888(dst, dst_pitch, src, fb, clip);
drm_fb_xrgb8888_to_rgb888(dst, dst_pitch, src, fb, clip, state);
return 0;
} else if (dst_format == DRM_FORMAT_ARGB8888) {
drm_fb_xrgb8888_to_argb8888(dst, dst_pitch, src, fb, clip);
drm_fb_xrgb8888_to_argb8888(dst, dst_pitch, src, fb, clip, state);
return 0;
} else if (dst_format == DRM_FORMAT_XBGR8888) {
drm_fb_xrgb8888_to_xbgr8888(dst, dst_pitch, src, fb, clip);
drm_fb_xrgb8888_to_xbgr8888(dst, dst_pitch, src, fb, clip, state);
return 0;
} else if (dst_format == DRM_FORMAT_ABGR8888) {
drm_fb_xrgb8888_to_abgr8888(dst, dst_pitch, src, fb, clip);
drm_fb_xrgb8888_to_abgr8888(dst, dst_pitch, src, fb, clip, state);
return 0;
} else if (dst_format == DRM_FORMAT_XRGB2101010) {
drm_fb_xrgb8888_to_xrgb2101010(dst, dst_pitch, src, fb, clip);
drm_fb_xrgb8888_to_xrgb2101010(dst, dst_pitch, src, fb, clip, state);
return 0;
} else if (dst_format == DRM_FORMAT_ARGB2101010) {
drm_fb_xrgb8888_to_argb2101010(dst, dst_pitch, src, fb, clip);
drm_fb_xrgb8888_to_argb2101010(dst, dst_pitch, src, fb, clip, state);
return 0;
} else if (dst_format == DRM_FORMAT_BGRX8888) {
drm_fb_swab(dst, dst_pitch, src, fb, clip, false);
drm_fb_swab(dst, dst_pitch, src, fb, clip, false, state);
return 0;
}
}
@ -978,6 +1088,7 @@ static void drm_fb_gray8_to_mono_line(void *dbuf, const void *sbuf, unsigned int
* @src: Array of XRGB8888 source buffers
* @fb: DRM framebuffer
* @clip: Clip rectangle area to copy
* @state: Transform and conversion state
*
* This function copies parts of a framebuffer to display memory and converts the
* color format during the process. Destination and framebuffer formats must match. The
@ -1002,7 +1113,7 @@ static void drm_fb_gray8_to_mono_line(void *dbuf, const void *sbuf, unsigned int
*/
void drm_fb_xrgb8888_to_mono(struct iosys_map *dst, const unsigned int *dst_pitch,
const struct iosys_map *src, const struct drm_framebuffer *fb,
const struct drm_rect *clip)
const struct drm_rect *clip, struct drm_format_conv_state *state)
{
static const unsigned int default_dst_pitch[DRM_FORMAT_MAX_PLANES] = {
0, 0, 0, 0
@ -1042,7 +1153,7 @@ void drm_fb_xrgb8888_to_mono(struct iosys_map *dst, const unsigned int *dst_pitc
* Allocate a buffer to be used for both copying from the cma
* memory and to store the intermediate grayscale line pixels.
*/
src32 = kmalloc(len_src32 + linepixels, GFP_KERNEL);
src32 = drm_format_conv_state_reserve(state, len_src32 + linepixels, GFP_KERNEL);
if (!src32)
return;
@ -1056,8 +1167,6 @@ void drm_fb_xrgb8888_to_mono(struct iosys_map *dst, const unsigned int *dst_pitc
vaddr += fb->pitches[0];
mono += dst_pitch_0;
}
kfree(src32);
}
EXPORT_SYMBOL(drm_fb_xrgb8888_to_mono);

75
drivers/gpu/drm/drm_framebuffer.c
View File

@ -394,6 +394,31 @@ static void drm_mode_rmfb_work_fn(struct work_struct *w)
}
}
static int drm_mode_closefb(struct drm_framebuffer *fb,
struct drm_file *file_priv)
{
struct drm_framebuffer *fbl;
bool found = false;
mutex_lock(&file_priv->fbs_lock);
list_for_each_entry(fbl, &file_priv->fbs, filp_head)
if (fb == fbl)
found = true;
if (!found) {
mutex_unlock(&file_priv->fbs_lock);
return -ENOENT;
}
list_del_init(&fb->filp_head);
mutex_unlock(&file_priv->fbs_lock);
/* Drop the reference that was stored in the fbs list */
drm_framebuffer_put(fb);
return 0;
}
/**
* drm_mode_rmfb - remove an FB from the configuration
* @dev: drm device
@ -410,9 +435,8 @@ static void drm_mode_rmfb_work_fn(struct work_struct *w)
int drm_mode_rmfb(struct drm_device *dev, u32 fb_id,
struct drm_file *file_priv)
{
struct drm_framebuffer *fb = NULL;
struct drm_framebuffer *fbl = NULL;
int found = 0;
struct drm_framebuffer *fb;
int ret;
if (!drm_core_check_feature(dev, DRIVER_MODESET))
return -EOPNOTSUPP;
@ -421,24 +445,13 @@ int drm_mode_rmfb(struct drm_device *dev, u32 fb_id,
if (!fb)
return -ENOENT;
mutex_lock(&file_priv->fbs_lock);
list_for_each_entry(fbl, &file_priv->fbs, filp_head)
if (fb == fbl)
found = 1;
if (!found) {
mutex_unlock(&file_priv->fbs_lock);
goto fail_unref;
ret = drm_mode_closefb(fb, file_priv);
if (ret != 0) {
drm_framebuffer_put(fb);
return ret;
}
list_del_init(&fb->filp_head);
mutex_unlock(&file_priv->fbs_lock);
/* drop the reference we picked up in framebuffer lookup */
drm_framebuffer_put(fb);
/*
* we now own the reference that was stored in the fbs list
*
* drm_framebuffer_remove may fail with -EINTR on pending signals,
* so run this in a separate stack as there's no way to correctly
* handle this after the fb is already removed from the lookup table.
@ -457,10 +470,6 @@ int drm_mode_rmfb(struct drm_device *dev, u32 fb_id,
drm_framebuffer_put(fb);
return 0;
fail_unref:
drm_framebuffer_put(fb);
return -ENOENT;
}
int drm_mode_rmfb_ioctl(struct drm_device *dev,
@ -471,6 +480,28 @@ int drm_mode_rmfb_ioctl(struct drm_device *dev,
return drm_mode_rmfb(dev, *fb_id, file_priv);
}
int drm_mode_closefb_ioctl(struct drm_device *dev,
void *data, struct drm_file *file_priv)
{
struct drm_mode_closefb *r = data;
struct drm_framebuffer *fb;
int ret;
if (!drm_core_check_feature(dev, DRIVER_MODESET))
return -EOPNOTSUPP;
if (r->pad)
return -EINVAL;
fb = drm_framebuffer_lookup(dev, file_priv, r->fb_id);
if (!fb)
return -ENOENT;
ret = drm_mode_closefb(fb, file_priv);
drm_framebuffer_put(fb);
return ret;
}
/**
* drm_mode_getfb - get FB info
* @dev: drm device for the ioctl

9
drivers/gpu/drm/drm_gem_atomic_helper.c
View File

@ -218,7 +218,14 @@ void
__drm_gem_duplicate_shadow_plane_state(struct drm_plane *plane,
struct drm_shadow_plane_state *new_shadow_plane_state)
{
struct drm_plane_state *plane_state = plane->state;
struct drm_shadow_plane_state *shadow_plane_state =
to_drm_shadow_plane_state(plane_state);
__drm_atomic_helper_plane_duplicate_state(plane, &new_shadow_plane_state->base);
drm_format_conv_state_copy(&shadow_plane_state->fmtcnv_state,
&new_shadow_plane_state->fmtcnv_state);
}
EXPORT_SYMBOL(__drm_gem_duplicate_shadow_plane_state);
@ -266,6 +273,7 @@ EXPORT_SYMBOL(drm_gem_duplicate_shadow_plane_state);
*/
void __drm_gem_destroy_shadow_plane_state(struct drm_shadow_plane_state *shadow_plane_state)
{
drm_format_conv_state_release(&shadow_plane_state->fmtcnv_state);
__drm_atomic_helper_plane_destroy_state(&shadow_plane_state->base);
}
EXPORT_SYMBOL(__drm_gem_destroy_shadow_plane_state);
@ -302,6 +310,7 @@ void __drm_gem_reset_shadow_plane(struct drm_plane *plane,
struct drm_shadow_plane_state *shadow_plane_state)
{
__drm_atomic_helper_plane_reset(plane, &shadow_plane_state->base);
drm_format_conv_state_init(&shadow_plane_state->fmtcnv_state);
}
EXPORT_SYMBOL(__drm_gem_reset_shadow_plane);

1131
drivers/gpu/drm/drm_gpuvm.c
View File

File diff suppressed because it is too large Load Diff

6
drivers/gpu/drm/drm_internal.h
View File

@ -22,6 +22,7 @@
*/
#include <linux/kthread.h>
#include <linux/types.h>
#include <drm/drm_ioctl.h>
#include <drm/drm_vblank.h>
@ -31,6 +32,7 @@
#define DRM_IF_VERSION(maj, min) (maj << 16 | min)
struct cea_sad;
struct dentry;
struct dma_buf;
struct iosys_map;
@ -267,3 +269,7 @@ int drm_syncobj_query_ioctl(struct drm_device *dev, void *data,
void drm_framebuffer_print_info(struct drm_printer *p, unsigned int indent,
const struct drm_framebuffer *fb);
void drm_framebuffer_debugfs_init(struct drm_device *dev);
/* drm_edid.c */
void drm_edid_cta_sad_get(const struct cea_sad *cta_sad, u8 *sad);
void drm_edid_cta_sad_set(struct cea_sad *cta_sad, const u8 *sad);

1
drivers/gpu/drm/drm_ioctl.c
View File

@ -675,6 +675,7 @@ static const struct drm_ioctl_desc drm_ioctls[] = {
DRM_IOCTL_DEF(DRM_IOCTL_MODE_ADDFB, drm_mode_addfb_ioctl, 0),
DRM_IOCTL_DEF(DRM_IOCTL_MODE_ADDFB2, drm_mode_addfb2_ioctl, 0),
DRM_IOCTL_DEF(DRM_IOCTL_MODE_RMFB, drm_mode_rmfb_ioctl, 0),
DRM_IOCTL_DEF(DRM_IOCTL_MODE_CLOSEFB, drm_mode_closefb_ioctl, 0),
DRM_IOCTL_DEF(DRM_IOCTL_MODE_PAGE_FLIP, drm_mode_page_flip_ioctl, DRM_MASTER),
DRM_IOCTL_DEF(DRM_IOCTL_MODE_DIRTYFB, drm_mode_dirtyfb_ioctl, DRM_MASTER),
DRM_IOCTL_DEF(DRM_IOCTL_MODE_CREATE_DUMB, drm_mode_create_dumb_ioctl, 0),

19
drivers/gpu/drm/drm_mipi_dbi.c
View File

@ -197,12 +197,14 @@ EXPORT_SYMBOL(mipi_dbi_command_stackbuf);
* @fb: The source framebuffer
* @clip: Clipping rectangle of the area to be copied
* @swap: When true, swap MSB/LSB of 16-bit values
* @fmtcnv_state: Format-conversion state
*
* Returns:
* Zero on success, negative error code on failure.
*/
int mipi_dbi_buf_copy(void *dst, struct iosys_map *src, struct drm_framebuffer *fb,
struct drm_rect *clip, bool swap)
struct drm_rect *clip, bool swap,
struct drm_format_conv_state *fmtcnv_state)
{
struct drm_gem_object *gem = drm_gem_fb_get_obj(fb, 0);
struct iosys_map dst_map = IOSYS_MAP_INIT_VADDR(dst);
@ -215,12 +217,13 @@ int mipi_dbi_buf_copy(void *dst, struct iosys_map *src, struct drm_framebuffer *
switch (fb->format->format) {
case DRM_FORMAT_RGB565:
if (swap)
drm_fb_swab(&dst_map, NULL, src, fb, clip, !gem->import_attach);
drm_fb_swab(&dst_map, NULL, src, fb, clip, !gem->import_attach,
fmtcnv_state);
else
drm_fb_memcpy(&dst_map, NULL, src, fb, clip);
break;
case DRM_FORMAT_XRGB8888:
drm_fb_xrgb8888_to_rgb565(&dst_map, NULL, src, fb, clip, swap);
drm_fb_xrgb8888_to_rgb565(&dst_map, NULL, src, fb, clip, fmtcnv_state, swap);
break;
default:
drm_err_once(fb->dev, "Format is not supported: %p4cc\n",
@ -252,7 +255,7 @@ static void mipi_dbi_set_window_address(struct mipi_dbi_dev *dbidev,
}
static void mipi_dbi_fb_dirty(struct iosys_map *src, struct drm_framebuffer *fb,
struct drm_rect *rect)
struct drm_rect *rect, struct drm_format_conv_state *fmtcnv_state)
{
struct mipi_dbi_dev *dbidev = drm_to_mipi_dbi_dev(fb->dev);
unsigned int height = rect->y2 - rect->y1;
@ -270,7 +273,7 @@ static void mipi_dbi_fb_dirty(struct iosys_map *src, struct drm_framebuffer *fb,
if (!dbi->dc || !full || swap ||
fb->format->format == DRM_FORMAT_XRGB8888) {
tr = dbidev->tx_buf;
ret = mipi_dbi_buf_copy(tr, src, fb, rect, swap);
ret = mipi_dbi_buf_copy(tr, src, fb, rect, swap, fmtcnv_state);
if (ret)
goto err_msg;
} else {
@ -332,7 +335,8 @@ void mipi_dbi_pipe_update(struct drm_simple_display_pipe *pipe,
return;
if (drm_atomic_helper_damage_merged(old_state, state, &rect))
mipi_dbi_fb_dirty(&shadow_plane_state->data[0], fb, &rect);
mipi_dbi_fb_dirty(&shadow_plane_state->data[0], fb, &rect,
&shadow_plane_state->fmtcnv_state);
drm_dev_exit(idx);
}
@ -368,7 +372,8 @@ void mipi_dbi_enable_flush(struct mipi_dbi_dev *dbidev,
if (!drm_dev_enter(&dbidev->drm, &idx))
return;
mipi_dbi_fb_dirty(&shadow_plane_state->data[0], fb, &rect);
mipi_dbi_fb_dirty(&shadow_plane_state->data[0], fb, &rect,
&shadow_plane_state->fmtcnv_state);
backlight_enable(dbidev->backlight);
drm_dev_exit(idx);

2
drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
View File

@ -535,7 +535,7 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data,
ret = drm_sched_job_init(&submit->sched_job,
&ctx->sched_entity[args->pipe],
submit->ctx);
1, submit->ctx);
if (ret)
goto err_submit_put;

2
drivers/gpu/drm/etnaviv/etnaviv_gpu.c
View File

@ -1917,7 +1917,7 @@ static int etnaviv_gpu_rpm_suspend(struct device *dev)
u32 idle, mask;
/* If there are any jobs in the HW queue, we're not idle */
if (atomic_read(&gpu->sched.hw_rq_count))
if (atomic_read(&gpu->sched.credit_count))
return -EBUSY;
/* Check whether the hardware (except FE and MC) is idle */

2
drivers/gpu/drm/etnaviv/etnaviv_sched.c
View File

@ -134,7 +134,7 @@ int etnaviv_sched_init(struct etnaviv_gpu *gpu)
{
int ret;
ret = drm_sched_init(&gpu->sched, &etnaviv_sched_ops,
ret = drm_sched_init(&gpu->sched, &etnaviv_sched_ops, NULL,
DRM_SCHED_PRIORITY_COUNT,
etnaviv_hw_jobs_limit, etnaviv_job_hang_limit,
msecs_to_jiffies(500), NULL, NULL,

30
drivers/gpu/drm/gud/gud_pipe.c
View File

@ -51,7 +51,8 @@ static bool gud_is_big_endian(void)
static size_t gud_xrgb8888_to_r124(u8 *dst, const struct drm_format_info *format,
void *src, struct drm_framebuffer *fb,
struct drm_rect *rect)
struct drm_rect *rect,
struct drm_format_conv_state *fmtcnv_state)
{
unsigned int block_width = drm_format_info_block_width(format, 0);
unsigned int bits_per_pixel = 8 / block_width;
@ -75,7 +76,7 @@ static size_t gud_xrgb8888_to_r124(u8 *dst, const struct drm_format_info *format
iosys_map_set_vaddr(&dst_map, buf);
iosys_map_set_vaddr(&vmap, src);
drm_fb_xrgb8888_to_gray8(&dst_map, NULL, &vmap, fb, rect);
drm_fb_xrgb8888_to_gray8(&dst_map, NULL, &vmap, fb, rect, fmtcnv_state);
pix8 = buf;
for (y = 0; y < height; y++) {
@ -152,7 +153,8 @@ static size_t gud_xrgb8888_to_color(u8 *dst, const struct drm_format_info *forma
static int gud_prep_flush(struct gud_device *gdrm, struct drm_framebuffer *fb,
const struct iosys_map *src, bool cached_reads,
const struct drm_format_info *format, struct drm_rect *rect,
struct gud_set_buffer_req *req)
struct gud_set_buffer_req *req,
struct drm_format_conv_state *fmtcnv_state)
{
u8 compression = gdrm->compression;
struct iosys_map dst;
@ -178,23 +180,23 @@ retry:
*/
if (format != fb->format) {
if (format->format == GUD_DRM_FORMAT_R1) {
len = gud_xrgb8888_to_r124(buf, format, vaddr, fb, rect);
len = gud_xrgb8888_to_r124(buf, format, vaddr, fb, rect, fmtcnv_state);
if (!len)
return -ENOMEM;
} else if (format->format == DRM_FORMAT_R8) {
drm_fb_xrgb8888_to_gray8(&dst, NULL, src, fb, rect);
drm_fb_xrgb8888_to_gray8(&dst, NULL, src, fb, rect, fmtcnv_state);
} else if (format->format == DRM_FORMAT_RGB332) {
drm_fb_xrgb8888_to_rgb332(&dst, NULL, src, fb, rect);
drm_fb_xrgb8888_to_rgb332(&dst, NULL, src, fb, rect, fmtcnv_state);
} else if (format->format == DRM_FORMAT_RGB565) {
drm_fb_xrgb8888_to_rgb565(&dst, NULL, src, fb, rect,
drm_fb_xrgb8888_to_rgb565(&dst, NULL, src, fb, rect, fmtcnv_state,
gud_is_big_endian());
} else if (format->format == DRM_FORMAT_RGB888) {
drm_fb_xrgb8888_to_rgb888(&dst, NULL, src, fb, rect);
drm_fb_xrgb8888_to_rgb888(&dst, NULL, src, fb, rect, fmtcnv_state);
} else {
len = gud_xrgb8888_to_color(buf, format, vaddr, fb, rect);
}
} else if (gud_is_big_endian() && format->cpp[0] > 1) {
drm_fb_swab(&dst, NULL, src, fb, rect, cached_reads);
drm_fb_swab(&dst, NULL, src, fb, rect, cached_reads, fmtcnv_state);
} else if (compression && cached_reads && pitch == fb->pitches[0]) {
/* can compress directly from the framebuffer */
buf = vaddr + rect->y1 * pitch;
@ -266,7 +268,8 @@ static int gud_usb_bulk(struct gud_device *gdrm, size_t len)
static int gud_flush_rect(struct gud_device *gdrm, struct drm_framebuffer *fb,
const struct iosys_map *src, bool cached_reads,
const struct drm_format_info *format, struct drm_rect *rect)
const struct drm_format_info *format, struct drm_rect *rect,
struct drm_format_conv_state *fmtcnv_state)
{
struct gud_set_buffer_req req;
size_t len, trlen;
@ -274,7 +277,7 @@ static int gud_flush_rect(struct gud_device *gdrm, struct drm_framebuffer *fb,
drm_dbg(&gdrm->drm, "Flushing [FB:%d] " DRM_RECT_FMT "\n", fb->base.id, DRM_RECT_ARG(rect));
ret = gud_prep_flush(gdrm, fb, src, cached_reads, format, rect, &req);
ret = gud_prep_flush(gdrm, fb, src, cached_reads, format, rect, &req, fmtcnv_state);
if (ret)
return ret;
@ -318,6 +321,7 @@ static void gud_flush_damage(struct gud_device *gdrm, struct drm_framebuffer *fb
const struct iosys_map *src, bool cached_reads,
struct drm_rect *damage)
{
struct drm_format_conv_state fmtcnv_state = DRM_FORMAT_CONV_STATE_INIT;
const struct drm_format_info *format;
unsigned int i, lines;
size_t pitch;
@ -340,7 +344,7 @@ static void gud_flush_damage(struct gud_device *gdrm, struct drm_framebuffer *fb
rect.y1 += i * lines;
rect.y2 = min_t(u32, rect.y1 + lines, damage->y2);
ret = gud_flush_rect(gdrm, fb, src, cached_reads, format, &rect);
ret = gud_flush_rect(gdrm, fb, src, cached_reads, format, &rect, &fmtcnv_state);
if (ret) {
if (ret != -ENODEV && ret != -ECONNRESET &&
ret != -ESHUTDOWN && ret != -EPROTO)
@ -350,6 +354,8 @@ static void gud_flush_damage(struct gud_device *gdrm, struct drm_framebuffer *fb
break;
}
}
drm_format_conv_state_release(&fmtcnv_state);
}
void gud_flush_work(struct work_struct *work)

1
drivers/gpu/drm/i915/display/intel_audio.c
View File

@ -25,6 +25,7 @@
#include <linux/kernel.h>
#include <drm/drm_edid.h>
#include <drm/drm_eld.h>
#include <drm/i915_component.h>
#include "i915_drv.h"

1
drivers/gpu/drm/i915/display/intel_crtc_state_dump.c
View File

@ -4,6 +4,7 @@
*/
#include <drm/drm_edid.h>
#include <drm/drm_eld.h>
#include "i915_drv.h"
#include "intel_crtc_state_dump.h"

1
drivers/gpu/drm/i915/display/intel_sdvo.c
View File

@ -35,6 +35,7 @@
#include <drm/drm_atomic_helper.h>
#include <drm/drm_crtc.h>
#include <drm/drm_edid.h>
#include <drm/drm_eld.h>
#include "i915_drv.h"
#include "i915_reg.h"

2
drivers/gpu/drm/lima/lima_device.c
View File

@ -514,7 +514,7 @@ int lima_device_suspend(struct device *dev)
/* check any task running */
for (i = 0; i < lima_pipe_num; i++) {
if (atomic_read(&ldev->pipe[i].base.hw_rq_count))
if (atomic_read(&ldev->pipe[i].base.credit_count))
return -EBUSY;
}

4
drivers/gpu/drm/lima/lima_sched.c
View File

@ -123,7 +123,7 @@ int lima_sched_task_init(struct lima_sched_task *task,
for (i = 0; i < num_bos; i++)
drm_gem_object_get(&bos[i]->base.base);
err = drm_sched_job_init(&task->base, &context->base, vm);
err = drm_sched_job_init(&task->base, &context->base, 1, vm);
if (err) {
kfree(task->bos);
return err;
@ -488,7 +488,7 @@ int lima_sched_pipe_init(struct lima_sched_pipe *pipe, const char *name)
INIT_WORK(&pipe->recover_work, lima_sched_recover_work);
return drm_sched_init(&pipe->base, &lima_sched_ops,
return drm_sched_init(&pipe->base, &lima_sched_ops, NULL,
DRM_SCHED_PRIORITY_COUNT,
1,
lima_job_hang_limit,

6
drivers/gpu/drm/msm/adreno/adreno_device.c
View File

@ -841,7 +841,8 @@ static void suspend_scheduler(struct msm_gpu *gpu)
*/
for (i = 0; i < gpu->nr_rings; i++) {
struct drm_gpu_scheduler *sched = &gpu->rb[i]->sched;
kthread_park(sched->thread);
drm_sched_wqueue_stop(sched);
}
}
@ -851,7 +852,8 @@ static void resume_scheduler(struct msm_gpu *gpu)
for (i = 0; i < gpu->nr_rings; i++) {
struct drm_gpu_scheduler *sched = &gpu->rb[i]->sched;
kthread_unpark(sched->thread);
drm_sched_wqueue_start(sched);
}
}

2
drivers/gpu/drm/msm/msm_gem_submit.c
View File

@ -48,7 +48,7 @@ static struct msm_gem_submit *submit_create(struct drm_device *dev,
return ERR_PTR(ret);
}
ret = drm_sched_job_init(&submit->base, queue->entity, queue);
ret = drm_sched_job_init(&submit->base, queue->entity, 1, queue);
if (ret) {
kfree(submit->hw_fence);
kfree(submit);

2
drivers/gpu/drm/msm/msm_ringbuffer.c
View File

@ -94,7 +94,7 @@ struct msm_ringbuffer *msm_ringbuffer_new(struct msm_gpu *gpu, int id,
/* currently managing hangcheck ourselves: */
sched_timeout = MAX_SCHEDULE_TIMEOUT;
ret = drm_sched_init(&ring->sched, &msm_sched_ops,
ret = drm_sched_init(&ring->sched, &msm_sched_ops, NULL,
DRM_SCHED_PRIORITY_COUNT,
num_hw_submissions, 0, sched_timeout,
NULL, NULL, to_msm_bo(ring->bo)->name, gpu->dev->dev);

1
drivers/gpu/drm/nouveau/dispnv50/disp.c
View File

@ -38,6 +38,7 @@
#include <drm/drm_atomic.h>
#include <drm/drm_atomic_helper.h>
#include <drm/drm_edid.h>
#include <drm/drm_eld.h>
#include <drm/drm_fb_helper.h>
#include <drm/drm_probe_helper.h>
#include <drm/drm_vblank.h>

11
drivers/gpu/drm/nouveau/nouveau_bo.c
View File

@ -148,10 +148,17 @@ nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
* If nouveau_bo_new() allocated this buffer, the GEM object was never
* initialized, so don't attempt to release it.
*/
if (bo->base.dev)
if (bo->base.dev) {
/* Gem objects not being shared with other VMs get their
* dma_resv from a root GEM object.
*/
if (nvbo->no_share)
drm_gem_object_put(nvbo->r_obj);
drm_gem_object_release(&bo->base);
else
} else {
dma_resv_fini(&bo->base._resv);
}
kfree(nvbo);
}

5
drivers/gpu/drm/nouveau/nouveau_bo.h
View File

@ -26,6 +26,11 @@ struct nouveau_bo {
struct list_head entry;
int pbbo_index;
bool validate_mapped;
/* Root GEM object we derive the dma_resv of in case this BO is not
* shared between VMs.
*/
struct drm_gem_object *r_obj;
bool no_share;
/* GPU address space is independent of CPU word size */

5
drivers/gpu/drm/nouveau/nouveau_drm.c
View File

@ -190,6 +190,8 @@ nouveau_cli_work_queue(struct nouveau_cli *cli, struct dma_fence *fence,
static void
nouveau_cli_fini(struct nouveau_cli *cli)
{
struct nouveau_uvmm *uvmm = nouveau_cli_uvmm_locked(cli);
/* All our channels are dead now, which means all the fences they
* own are signalled, and all callback functions have been called.
*
@ -199,7 +201,8 @@ nouveau_cli_fini(struct nouveau_cli *cli)
WARN_ON(!list_empty(&cli->worker));
usif_client_fini(cli);
nouveau_uvmm_fini(&cli->uvmm);
if (uvmm)
nouveau_uvmm_fini(uvmm);
nouveau_sched_entity_fini(&cli->sched_entity);
nouveau_vmm_fini(&cli->svm);
nouveau_vmm_fini(&cli->vmm);

10
drivers/gpu/drm/nouveau/nouveau_drv.h
View File

@ -93,7 +93,10 @@ struct nouveau_cli {
struct nvif_mmu mmu;
struct nouveau_vmm vmm;
struct nouveau_vmm svm;
struct nouveau_uvmm uvmm;
struct {
struct nouveau_uvmm *ptr;
bool disabled;
} uvmm;
struct nouveau_sched_entity sched_entity;
@ -121,10 +124,7 @@ struct nouveau_cli_work {
static inline struct nouveau_uvmm *
nouveau_cli_uvmm(struct nouveau_cli *cli)
{
if (!cli || !cli->uvmm.vmm.cli)
return NULL;
return &cli->uvmm;
return cli ? cli->uvmm.ptr : NULL;
}
static inline struct nouveau_uvmm *

10
drivers/gpu/drm/nouveau/nouveau_gem.c
View File

@ -111,7 +111,8 @@ nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv)
if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50)
return 0;
if (nvbo->no_share && uvmm && &uvmm->resv != nvbo->bo.base.resv)
if (nvbo->no_share && uvmm &&
drm_gpuvm_resv(&uvmm->base) != nvbo->bo.base.resv)
return -EPERM;
ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
@ -245,7 +246,7 @@ nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain,
if (unlikely(!uvmm))
return -EINVAL;
resv = &uvmm->resv;
resv = drm_gpuvm_resv(&uvmm->base);
}
if (!(domain & (NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART)))
@ -288,6 +289,11 @@ nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain,
if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA)
nvbo->valid_domains &= domain;
if (nvbo->no_share) {
nvbo->r_obj = drm_gpuvm_resv_obj(&uvmm->base);
drm_gem_object_get(nvbo->r_obj);
}
*pnvbo = nvbo;
return 0;
}

4
drivers/gpu/drm/nouveau/nouveau_sched.c
View File

@ -89,7 +89,7 @@ nouveau_job_init(struct nouveau_job *job,
}
ret = drm_sched_job_init(&job->base, &entity->base, NULL);
ret = drm_sched_job_init(&job->base, &entity->base, 1, NULL);
if (ret)
goto err_free_chains;
@ -435,7 +435,7 @@ int nouveau_sched_init(struct nouveau_drm *drm)
if (!drm->sched_wq)
return -ENOMEM;
return drm_sched_init(sched, &nouveau_sched_ops,
return drm_sched_init(sched, &nouveau_sched_ops, NULL,
DRM_SCHED_PRIORITY_COUNT,
NOUVEAU_SCHED_HW_SUBMISSIONS, 0, job_hang_limit,
NULL, NULL, "nouveau_sched", drm->dev->dev);

174
drivers/gpu/drm/nouveau/nouveau_uvmm.c
View File

@ -62,6 +62,8 @@ struct bind_job_op {
enum vm_bind_op op;
u32 flags;
struct drm_gpuvm_bo *vm_bo;
struct {
u64 addr;
u64 range;
@ -929,25 +931,13 @@ nouveau_uvmm_sm_unmap_cleanup(struct nouveau_uvmm *uvmm,
static int
nouveau_uvmm_validate_range(struct nouveau_uvmm *uvmm, u64 addr, u64 range)
{
u64 end = addr + range;
u64 kernel_managed_end = uvmm->kernel_managed_addr +
uvmm->kernel_managed_size;
if (addr & ~PAGE_MASK)
return -EINVAL;
if (range & ~PAGE_MASK)
return -EINVAL;
if (end <= addr)
return -EINVAL;
if (addr < NOUVEAU_VA_SPACE_START ||
end > NOUVEAU_VA_SPACE_END)
return -EINVAL;
if (addr < kernel_managed_end &&
end > uvmm->kernel_managed_addr)
if (!drm_gpuvm_range_valid(&uvmm->base, addr, range))
return -EINVAL;
return 0;
@ -1113,22 +1103,28 @@ bind_validate_region(struct nouveau_job *job)
}
static void
bind_link_gpuvas(struct drm_gpuva_ops *ops, struct nouveau_uvma_prealloc *new)
bind_link_gpuvas(struct bind_job_op *bop)
{
struct nouveau_uvma_prealloc *new = &bop->new;
struct drm_gpuvm_bo *vm_bo = bop->vm_bo;
struct drm_gpuva_ops *ops = bop->ops;
struct drm_gpuva_op *op;
drm_gpuva_for_each_op(op, ops) {
switch (op->op) {
case DRM_GPUVA_OP_MAP:
drm_gpuva_link(&new->map->va);
drm_gpuva_link(&new->map->va, vm_bo);
break;
case DRM_GPUVA_OP_REMAP:
case DRM_GPUVA_OP_REMAP: {
struct drm_gpuva *va = op->remap.unmap->va;
if (op->remap.prev)
drm_gpuva_link(&new->prev->va);
drm_gpuva_link(&new->prev->va, va->vm_bo);
if (op->remap.next)
drm_gpuva_link(&new->next->va);
drm_gpuva_unlink(op->remap.unmap->va);
drm_gpuva_link(&new->next->va, va->vm_bo);
drm_gpuva_unlink(va);
break;
}
case DRM_GPUVA_OP_UNMAP:
drm_gpuva_unlink(op->unmap.va);
break;
@ -1150,10 +1146,17 @@ nouveau_uvmm_bind_job_submit(struct nouveau_job *job)
list_for_each_op(op, &bind_job->ops) {
if (op->op == OP_MAP) {
op->gem.obj = drm_gem_object_lookup(job->file_priv,
op->gem.handle);
if (!op->gem.obj)
struct drm_gem_object *obj = op->gem.obj =
drm_gem_object_lookup(job->file_priv,
op->gem.handle);
if (!obj)
return -ENOENT;
dma_resv_lock(obj->resv, NULL);
op->vm_bo = drm_gpuvm_bo_obtain(&uvmm->base, obj);
dma_resv_unlock(obj->resv);
if (IS_ERR(op->vm_bo))
return PTR_ERR(op->vm_bo);
}
ret = bind_validate_op(job, op);
@ -1364,7 +1367,7 @@ nouveau_uvmm_bind_job_submit(struct nouveau_job *job)
case OP_UNMAP_SPARSE:
case OP_MAP:
case OP_UNMAP:
bind_link_gpuvas(op->ops, &op->new);
bind_link_gpuvas(op);
break;
default:
break;
@ -1511,6 +1514,12 @@ nouveau_uvmm_bind_job_free_work_fn(struct work_struct *work)
if (!IS_ERR_OR_NULL(op->ops))
drm_gpuva_ops_free(&uvmm->base, op->ops);
if (!IS_ERR_OR_NULL(op->vm_bo)) {
dma_resv_lock(obj->resv, NULL);
drm_gpuvm_bo_put(op->vm_bo);
dma_resv_unlock(obj->resv);
}
if (obj)
drm_gem_object_put(obj);
}
@ -1648,18 +1657,6 @@ err_free:
return ret;
}
int
nouveau_uvmm_ioctl_vm_init(struct drm_device *dev,
void *data,
struct drm_file *file_priv)
{
struct nouveau_cli *cli = nouveau_cli(file_priv);
struct drm_nouveau_vm_init *init = data;
return nouveau_uvmm_init(&cli->uvmm, cli, init->kernel_managed_addr,
init->kernel_managed_size);
}
static int
nouveau_uvmm_vm_bind(struct nouveau_uvmm_bind_job_args *args)
{
@ -1776,15 +1773,18 @@ void
nouveau_uvmm_bo_map_all(struct nouveau_bo *nvbo, struct nouveau_mem *mem)
{
struct drm_gem_object *obj = &nvbo->bo.base;
struct drm_gpuvm_bo *vm_bo;
struct drm_gpuva *va;
dma_resv_assert_held(obj->resv);
drm_gem_for_each_gpuva(va, obj) {
struct nouveau_uvma *uvma = uvma_from_va(va);
drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
drm_gpuvm_bo_for_each_va(va, vm_bo) {
struct nouveau_uvma *uvma = uvma_from_va(va);
nouveau_uvma_map(uvma, mem);
drm_gpuva_invalidate(va, false);
nouveau_uvma_map(uvma, mem);
drm_gpuva_invalidate(va, false);
}
}
}
@ -1792,29 +1792,53 @@ void
nouveau_uvmm_bo_unmap_all(struct nouveau_bo *nvbo)
{
struct drm_gem_object *obj = &nvbo->bo.base;
struct drm_gpuvm_bo *vm_bo;
struct drm_gpuva *va;
dma_resv_assert_held(obj->resv);
drm_gem_for_each_gpuva(va, obj) {
struct nouveau_uvma *uvma = uvma_from_va(va);
drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
drm_gpuvm_bo_for_each_va(va, vm_bo) {
struct nouveau_uvma *uvma = uvma_from_va(va);
nouveau_uvma_unmap(uvma);
drm_gpuva_invalidate(va, true);
nouveau_uvma_unmap(uvma);
drm_gpuva_invalidate(va, true);
}
}
}
int
nouveau_uvmm_init(struct nouveau_uvmm *uvmm, struct nouveau_cli *cli,
u64 kernel_managed_addr, u64 kernel_managed_size)
static void
nouveau_uvmm_free(struct drm_gpuvm *gpuvm)
{
int ret;
u64 kernel_managed_end = kernel_managed_addr + kernel_managed_size;
struct nouveau_uvmm *uvmm = uvmm_from_gpuvm(gpuvm);
mutex_init(&uvmm->mutex);
dma_resv_init(&uvmm->resv);
mt_init_flags(&uvmm->region_mt, MT_FLAGS_LOCK_EXTERN);
mt_set_external_lock(&uvmm->region_mt, &uvmm->mutex);
kfree(uvmm);
}
static const struct drm_gpuvm_ops gpuvm_ops = {
.vm_free = nouveau_uvmm_free,
};
int
nouveau_uvmm_ioctl_vm_init(struct drm_device *dev,
void *data,
struct drm_file *file_priv)
{
struct nouveau_uvmm *uvmm;
struct nouveau_cli *cli = nouveau_cli(file_priv);
struct drm_device *drm = cli->drm->dev;
struct drm_gem_object *r_obj;
struct drm_nouveau_vm_init *init = data;
u64 kernel_managed_end;
int ret;
if (check_add_overflow(init->kernel_managed_addr,
init->kernel_managed_size,
&kernel_managed_end))
return -EINVAL;
if (kernel_managed_end > NOUVEAU_VA_SPACE_END)
return -EINVAL;
mutex_lock(&cli->mutex);
@ -1823,39 +1847,48 @@ nouveau_uvmm_init(struct nouveau_uvmm *uvmm, struct nouveau_cli *cli,
goto out_unlock;
}
if (kernel_managed_end <= kernel_managed_addr) {
ret = -EINVAL;
uvmm = kzalloc(sizeof(*uvmm), GFP_KERNEL);
if (!uvmm) {
ret = -ENOMEM;
goto out_unlock;
}
if (kernel_managed_end > NOUVEAU_VA_SPACE_END) {
ret = -EINVAL;
r_obj = drm_gpuvm_resv_object_alloc(drm);
if (!r_obj) {
kfree(uvmm);
ret = -ENOMEM;
goto out_unlock;
}
uvmm->kernel_managed_addr = kernel_managed_addr;
uvmm->kernel_managed_size = kernel_managed_size;
mutex_init(&uvmm->mutex);
mt_init_flags(&uvmm->region_mt, MT_FLAGS_LOCK_EXTERN);
mt_set_external_lock(&uvmm->region_mt, &uvmm->mutex);
drm_gpuvm_init(&uvmm->base, cli->name,
drm_gpuvm_init(&uvmm->base, cli->name, 0, drm, r_obj,
NOUVEAU_VA_SPACE_START,
NOUVEAU_VA_SPACE_END,
kernel_managed_addr, kernel_managed_size,
NULL);
init->kernel_managed_addr,
init->kernel_managed_size,
&gpuvm_ops);
/* GPUVM takes care from here on. */
drm_gem_object_put(r_obj);
ret = nvif_vmm_ctor(&cli->mmu, "uvmm",
cli->vmm.vmm.object.oclass, RAW,
kernel_managed_addr, kernel_managed_size,
NULL, 0, &cli->uvmm.vmm.vmm);
init->kernel_managed_addr,
init->kernel_managed_size,
NULL, 0, &uvmm->vmm.vmm);
if (ret)
goto out_free_gpuva_mgr;
goto out_gpuvm_fini;
cli->uvmm.vmm.cli = cli;
uvmm->vmm.cli = cli;
cli->uvmm.ptr = uvmm;
mutex_unlock(&cli->mutex);
return 0;
out_free_gpuva_mgr:
drm_gpuvm_destroy(&uvmm->base);
out_gpuvm_fini:
drm_gpuvm_put(&uvmm->base);
out_unlock:
mutex_unlock(&cli->mutex);
return ret;
@ -1870,9 +1903,6 @@ nouveau_uvmm_fini(struct nouveau_uvmm *uvmm)
struct nouveau_sched_entity *entity = &cli->sched_entity;
struct drm_gpuva *va, *next;
if (!cli)
return;
rmb(); /* for list_empty to work without lock */
wait_event(entity->job.wq, list_empty(&entity->job.list.head));
@ -1910,8 +1940,6 @@ nouveau_uvmm_fini(struct nouveau_uvmm *uvmm)
mutex_lock(&cli->mutex);
nouveau_vmm_fini(&uvmm->vmm);
drm_gpuvm_destroy(&uvmm->base);
drm_gpuvm_put(&uvmm->base);
mutex_unlock(&cli->mutex);
dma_resv_fini(&uvmm->resv);
}

8
drivers/gpu/drm/nouveau/nouveau_uvmm.h
View File

@ -12,12 +12,6 @@ struct nouveau_uvmm {
struct nouveau_vmm vmm;
struct maple_tree region_mt;
struct mutex mutex;
struct dma_resv resv;
u64 kernel_managed_addr;
u64 kernel_managed_size;
bool disabled;
};
struct nouveau_uvma_region {
@ -82,8 +76,6 @@ struct nouveau_uvmm_bind_job_args {
#define to_uvmm_bind_job(job) container_of((job), struct nouveau_uvmm_bind_job, base)
int nouveau_uvmm_init(struct nouveau_uvmm *uvmm, struct nouveau_cli *cli,
u64 kernel_managed_addr, u64 kernel_managed_size);
void nouveau_uvmm_fini(struct nouveau_uvmm *uvmm);
void nouveau_uvmm_bo_map_all(struct nouveau_bo *nvbov, struct nouveau_mem *mem);

9
drivers/gpu/drm/omapdrm/omap_drv.c
View File

@ -69,7 +69,6 @@ static void omap_atomic_commit_tail(struct drm_atomic_state *old_state)
{
struct drm_device *dev = old_state->dev;
struct omap_drm_private *priv = dev->dev_private;
bool fence_cookie = dma_fence_begin_signalling();
dispc_runtime_get(priv->dispc);
@ -92,6 +91,8 @@ static void omap_atomic_commit_tail(struct drm_atomic_state *old_state)
omap_atomic_wait_for_completion(dev, old_state);
drm_atomic_helper_commit_planes(dev, old_state, 0);
drm_atomic_helper_commit_hw_done(old_state);
} else {
/*
* OMAP3 DSS seems to have issues with the work-around above,
@ -101,12 +102,10 @@ static void omap_atomic_commit_tail(struct drm_atomic_state *old_state)
drm_atomic_helper_commit_planes(dev, old_state, 0);
drm_atomic_helper_commit_modeset_enables(dev, old_state);
drm_atomic_helper_commit_hw_done(old_state);
}
drm_atomic_helper_commit_hw_done(old_state);
dma_fence_end_signalling(fence_cookie);
/*
* Wait for completion of the page flips to ensure that old buffers
* can't be touched by the hardware anymore before cleaning up planes.

58
drivers/gpu/drm/panel/panel-edp.c
View File

@ -973,6 +973,8 @@ static const struct panel_desc auo_b116xak01 = {
},
.delay = {
.hpd_absent = 200,
.unprepare = 500,
.enable = 50,
},
};
@ -1801,6 +1803,12 @@ static const struct panel_delay delay_200_500_e50 = {
.enable = 50,
};
static const struct panel_delay delay_200_500_e80 = {
.hpd_absent = 200,
.unprepare = 500,
.enable = 80,
};
static const struct panel_delay delay_200_500_e80_d50 = {
.hpd_absent = 200,
.unprepare = 500,
@ -1820,6 +1828,19 @@ static const struct panel_delay delay_200_500_e200 = {
.enable = 200,
};
static const struct panel_delay delay_200_500_e200_d10 = {
.hpd_absent = 200,
.unprepare = 500,
.enable = 200,
.disable = 10,
};
static const struct panel_delay delay_200_150_e200 = {
.hpd_absent = 200,
.unprepare = 150,
.enable = 200,
};
#define EDP_PANEL_ENTRY(vend_chr_0, vend_chr_1, vend_chr_2, product_id, _delay, _name) \
{ \
.name = _name, \
@ -1840,34 +1861,69 @@ static const struct edp_panel_entry edp_panels[] = {
EDP_PANEL_ENTRY('A', 'U', 'O', 0x145c, &delay_200_500_e50, "B116XAB01.4"),
EDP_PANEL_ENTRY('A', 'U', 'O', 0x1e9b, &delay_200_500_e50, "B133UAN02.1"),
EDP_PANEL_ENTRY('A', 'U', 'O', 0x1ea5, &delay_200_500_e50, "B116XAK01.6"),
EDP_PANEL_ENTRY('A', 'U', 'O', 0x405c, &auo_b116xak01.delay, "B116XAK01"),
EDP_PANEL_ENTRY('A', 'U', 'O', 0x208d, &delay_200_500_e50, "B140HTN02.1"),
EDP_PANEL_ENTRY('A', 'U', 'O', 0x235c, &delay_200_500_e50, "B116XTN02.3"),
EDP_PANEL_ENTRY('A', 'U', 'O', 0x239b, &delay_200_500_e50, "B116XAN06.1"),
EDP_PANEL_ENTRY('A', 'U', 'O', 0x255c, &delay_200_500_e50, "B116XTN02.5"),
EDP_PANEL_ENTRY('A', 'U', 'O', 0x403d, &delay_200_500_e50, "B140HAN04.0"),
EDP_PANEL_ENTRY('A', 'U', 'O', 0x405c, &auo_b116xak01.delay, "B116XAK01.0"),
EDP_PANEL_ENTRY('A', 'U', 'O', 0x582d, &delay_200_500_e50, "B133UAN01.0"),
EDP_PANEL_ENTRY('A', 'U', 'O', 0x615c, &delay_200_500_e50, "B116XAN06.1"),
EDP_PANEL_ENTRY('A', 'U', 'O', 0x635c, &delay_200_500_e50, "B116XAN06.3"),
EDP_PANEL_ENTRY('A', 'U', 'O', 0x639c, &delay_200_500_e50, "B140HAK02.7"),
EDP_PANEL_ENTRY('A', 'U', 'O', 0x8594, &delay_200_500_e50, "B133UAN01.0"),
EDP_PANEL_ENTRY('A', 'U', 'O', 0xf390, &delay_200_500_e50, "B140XTN07.7"),
EDP_PANEL_ENTRY('B', 'O', 'E', 0x0715, &delay_200_150_e200, "NT116WHM-N21"),
EDP_PANEL_ENTRY('B', 'O', 'E', 0x0731, &delay_200_500_e80, "NT116WHM-N42"),
EDP_PANEL_ENTRY('B', 'O', 'E', 0x0741, &delay_200_500_e200, "NT116WHM-N44"),
EDP_PANEL_ENTRY('B', 'O', 'E', 0x0786, &delay_200_500_p2e80, "NV116WHM-T01"),
EDP_PANEL_ENTRY('B', 'O', 'E', 0x07d1, &boe_nv133fhm_n61.delay, "NV133FHM-N61"),
EDP_PANEL_ENTRY('B', 'O', 'E', 0x07f6, &delay_200_500_e200, "NT140FHM-N44"),
EDP_PANEL_ENTRY('B', 'O', 'E', 0x082d, &boe_nv133fhm_n61.delay, "NV133FHM-N62"),
EDP_PANEL_ENTRY('B', 'O', 'E', 0x08b2, &delay_200_500_e200, "NT140WHM-N49"),
EDP_PANEL_ENTRY('B', 'O', 'E', 0x09c3, &delay_200_500_e50, "NT116WHM-N21,836X2"),
EDP_PANEL_ENTRY('B', 'O', 'E', 0x094b, &delay_200_500_e50, "NT116WHM-N21"),
EDP_PANEL_ENTRY('B', 'O', 'E', 0x0951, &delay_200_500_e80, "NV116WHM-N47"),
EDP_PANEL_ENTRY('B', 'O', 'E', 0x095f, &delay_200_500_e50, "NE135FBM-N41 v8.1"),
EDP_PANEL_ENTRY('B', 'O', 'E', 0x0979, &delay_200_500_e50, "NV116WHM-N49 V8.0"),
EDP_PANEL_ENTRY('B', 'O', 'E', 0x098d, &boe_nv110wtm_n61.delay, "NV110WTM-N61"),
EDP_PANEL_ENTRY('B', 'O', 'E', 0x09ae, &delay_200_500_e200, "NT140FHM-N45"),
EDP_PANEL_ENTRY('B', 'O', 'E', 0x09dd, &delay_200_500_e50, "NT116WHM-N21"),
EDP_PANEL_ENTRY('B', 'O', 'E', 0x0a5d, &delay_200_500_e50, "NV116WHM-N45"),
EDP_PANEL_ENTRY('B', 'O', 'E', 0x0ac5, &delay_200_500_e50, "NV116WHM-N4C"),
EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b43, &delay_200_500_e200, "NV140FHM-T09"),
EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b56, &delay_200_500_e80, "NT140FHM-N47"),
EDP_PANEL_ENTRY('B', 'O', 'E', 0x0c20, &delay_200_500_e80, "NT140FHM-N47"),
EDP_PANEL_ENTRY('C', 'M', 'N', 0x1132, &delay_200_500_e80_d50, "N116BGE-EA2"),
EDP_PANEL_ENTRY('C', 'M', 'N', 0x1138, &innolux_n116bca_ea1.delay, "N116BCA-EA1-RC4"),
EDP_PANEL_ENTRY('C', 'M', 'N', 0x1139, &delay_200_500_e80_d50, "N116BGE-EA2"),
EDP_PANEL_ENTRY('C', 'M', 'N', 0x1145, &delay_200_500_e80_d50, "N116BCN-EB1"),
EDP_PANEL_ENTRY('C', 'M', 'N', 0x114c, &innolux_n116bca_ea1.delay, "N116BCA-EA1"),
EDP_PANEL_ENTRY('C', 'M', 'N', 0x1152, &delay_200_500_e80_d50, "N116BCN-EA1"),
EDP_PANEL_ENTRY('C', 'M', 'N', 0x1153, &delay_200_500_e80_d50, "N116BGE-EA2"),
EDP_PANEL_ENTRY('C', 'M', 'N', 0x1154, &delay_200_500_e80_d50, "N116BCA-EA2"),
EDP_PANEL_ENTRY('C', 'M', 'N', 0x1157, &delay_200_500_e80_d50, "N116BGE-EA2"),
EDP_PANEL_ENTRY('C', 'M', 'N', 0x115b, &delay_200_500_e80_d50, "N116BCN-EB1"),
EDP_PANEL_ENTRY('C', 'M', 'N', 0x1247, &delay_200_500_e80_d50, "N120ACA-EA1"),
EDP_PANEL_ENTRY('C', 'M', 'N', 0x142b, &delay_200_500_e80_d50, "N140HCA-EAC"),
EDP_PANEL_ENTRY('C', 'M', 'N', 0x144f, &delay_200_500_e80_d50, "N140HGA-EA1"),
EDP_PANEL_ENTRY('C', 'M', 'N', 0x1468, &delay_200_500_e80, "N140HGA-EA1"),
EDP_PANEL_ENTRY('C', 'M', 'N', 0x14e5, &delay_200_500_e80_d50, "N140HGA-EA1"),
EDP_PANEL_ENTRY('C', 'M', 'N', 0x14d4, &delay_200_500_e80_d50, "N140HCA-EAC"),
EDP_PANEL_ENTRY('C', 'M', 'N', 0x14d6, &delay_200_500_e80_d50, "N140BGA-EA4"),
EDP_PANEL_ENTRY('H', 'K', 'C', 0x2d5c, &delay_200_500_e200, "MB116AN01-2"),
EDP_PANEL_ENTRY('I', 'V', 'O', 0x048e, &delay_200_500_e200_d10, "M116NWR6 R5"),
EDP_PANEL_ENTRY('I', 'V', 'O', 0x057d, &delay_200_500_e200, "R140NWF5 RH"),
EDP_PANEL_ENTRY('I', 'V', 'O', 0x854a, &delay_200_500_p2e100, "M133NW4J"),
EDP_PANEL_ENTRY('I', 'V', 'O', 0x854b, &delay_200_500_p2e100, "R133NW4K-R0"),
EDP_PANEL_ENTRY('I', 'V', 'O', 0x8c4d, &delay_200_150_e200, "R140NWFM R1"),
EDP_PANEL_ENTRY('K', 'D', 'B', 0x0624, &kingdisplay_kd116n21_30nv_a010.delay, "116N21-30NV-A010"),
EDP_PANEL_ENTRY('K', 'D', 'C', 0x0809, &delay_200_500_e50, "KD116N2930A15"),
EDP_PANEL_ENTRY('K', 'D', 'B', 0x1120, &delay_200_500_e80_d50, "116N29-30NK-C007"),
EDP_PANEL_ENTRY('S', 'H', 'P', 0x1511, &delay_200_500_e50, "LQ140M1JW48"),

2
drivers/gpu/drm/panel/panel-novatek-nt35510.c
View File

@ -1023,7 +1023,7 @@ static const struct nt35510_config nt35510_hydis_hva40wv1 = {
.hdisplay = 480,
.hsync_start = 480 + 2, /* HFP = 2 */
.hsync_end = 480 + 2 + 0, /* HSync = 0 */
.htotal = 480 + 2 + 0 + 5, /* HFP = 5 */
.htotal = 480 + 2 + 0 + 5, /* HBP = 5 */
.vdisplay = 800,
.vsync_start = 800 + 2, /* VFP = 2 */
.vsync_end = 800 + 2 + 0, /* VSync = 0 */

78
drivers/gpu/drm/panfrost/panfrost_device.c
View File

@ -403,7 +403,7 @@ void panfrost_device_reset(struct panfrost_device *pfdev)
panfrost_job_enable_interrupts(pfdev);
}
static int panfrost_device_resume(struct device *dev)
static int panfrost_device_runtime_resume(struct device *dev)
{
struct panfrost_device *pfdev = dev_get_drvdata(dev);
@ -413,7 +413,7 @@ static int panfrost_device_resume(struct device *dev)
return 0;
}
static int panfrost_device_suspend(struct device *dev)
static int panfrost_device_runtime_suspend(struct device *dev)
{
struct panfrost_device *pfdev = dev_get_drvdata(dev);
@ -426,5 +426,75 @@ static int panfrost_device_suspend(struct device *dev)
return 0;
}
EXPORT_GPL_RUNTIME_DEV_PM_OPS(panfrost_pm_ops, panfrost_device_suspend,
panfrost_device_resume, NULL);
static int panfrost_device_resume(struct device *dev)
{
struct panfrost_device *pfdev = dev_get_drvdata(dev);
int ret;
if (pfdev->comp->pm_features & BIT(GPU_PM_VREG_OFF)) {
unsigned long freq = pfdev->pfdevfreq.fast_rate;
struct dev_pm_opp *opp;
opp = dev_pm_opp_find_freq_ceil(dev, &freq);
if (IS_ERR(opp))
return PTR_ERR(opp);
dev_pm_opp_set_opp(dev, opp);
dev_pm_opp_put(opp);
}
if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS)) {
ret = clk_enable(pfdev->clock);
if (ret)
goto err_clk;
if (pfdev->bus_clock) {
ret = clk_enable(pfdev->bus_clock);
if (ret)
goto err_bus_clk;
}
}
ret = pm_runtime_force_resume(dev);
if (ret)
goto err_resume;
return 0;
err_resume:
if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS) && pfdev->bus_clock)
clk_disable(pfdev->bus_clock);
err_bus_clk:
if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS))
clk_disable(pfdev->clock);
err_clk:
if (pfdev->comp->pm_features & BIT(GPU_PM_VREG_OFF))
dev_pm_opp_set_opp(dev, NULL);
return ret;
}
static int panfrost_device_suspend(struct device *dev)
{
struct panfrost_device *pfdev = dev_get_drvdata(dev);
int ret;
ret = pm_runtime_force_suspend(dev);
if (ret)
return ret;
if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS)) {
if (pfdev->bus_clock)
clk_disable(pfdev->bus_clock);
clk_disable(pfdev->clock);
}
if (pfdev->comp->pm_features & BIT(GPU_PM_VREG_OFF))
dev_pm_opp_set_opp(dev, NULL);
return 0;
}
EXPORT_GPL_DEV_PM_OPS(panfrost_pm_ops) = {
RUNTIME_PM_OPS(panfrost_device_runtime_suspend, panfrost_device_runtime_resume, NULL)
SYSTEM_SLEEP_PM_OPS(panfrost_device_suspend, panfrost_device_resume)
};

13
drivers/gpu/drm/panfrost/panfrost_device.h
View File

@ -25,6 +25,16 @@ struct panfrost_perfcnt;
#define NUM_JOB_SLOTS 3
#define MAX_PM_DOMAINS 5
/**
* enum panfrost_gpu_pm - Supported kernel power management features
* @GPU_PM_CLK_DIS: Allow disabling clocks during system suspend
* @GPU_PM_VREG_OFF: Allow turning off regulators during system suspend
*/
enum panfrost_gpu_pm {
GPU_PM_CLK_DIS,
GPU_PM_VREG_OFF,
};
struct panfrost_features {
u16 id;
u16 revision;
@ -75,6 +85,9 @@ struct panfrost_compatible {
/* Vendor implementation quirks callback */
void (*vendor_quirk)(struct panfrost_device *pfdev);
/* Allowed PM features */
u8 pm_features;
};
struct panfrost_device {

5
drivers/gpu/drm/panfrost/panfrost_drv.c
View File

@ -274,7 +274,7 @@ static int panfrost_ioctl_submit(struct drm_device *dev, void *data,
ret = drm_sched_job_init(&job->base,
&file_priv->sched_entity[slot],
NULL);
1, NULL);
if (ret)
goto out_put_job;
@ -734,6 +734,7 @@ static const struct panfrost_compatible mediatek_mt8183_b_data = {
.supply_names = mediatek_mt8183_b_supplies,
.num_pm_domains = ARRAY_SIZE(mediatek_mt8183_pm_domains),
.pm_domain_names = mediatek_mt8183_pm_domains,
.pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF),
};
static const char * const mediatek_mt8186_pm_domains[] = { "core0", "core1" };
@ -742,6 +743,7 @@ static const struct panfrost_compatible mediatek_mt8186_data = {
.supply_names = mediatek_mt8183_b_supplies,
.num_pm_domains = ARRAY_SIZE(mediatek_mt8186_pm_domains),
.pm_domain_names = mediatek_mt8186_pm_domains,
.pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF),
};
static const char * const mediatek_mt8192_supplies[] = { "mali", NULL };
@ -752,6 +754,7 @@ static const struct panfrost_compatible mediatek_mt8192_data = {
.supply_names = mediatek_mt8192_supplies,
.num_pm_domains = ARRAY_SIZE(mediatek_mt8192_pm_domains),
.pm_domain_names = mediatek_mt8192_pm_domains,
.pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF),
};
static const struct of_device_id dt_match[] = {

12
drivers/gpu/drm/panfrost/panfrost_dump.c
View File

@ -220,16 +220,8 @@ void panfrost_core_dump(struct panfrost_job *job)
iter.hdr->bomap.data[0] = bomap - bomap_start;
for_each_sgtable_page(bo->base.sgt, &page_iter, 0) {
struct page *page = sg_page_iter_page(&page_iter);
if (!IS_ERR(page)) {
*bomap++ = page_to_phys(page);
} else {
dev_err(pfdev->dev, "Panfrost Dump: wrong page\n");
*bomap++ = 0;
}
}
for_each_sgtable_page(bo->base.sgt, &page_iter, 0)
*bomap++ = page_to_phys(sg_page_iter_page(&page_iter));
iter.hdr->bomap.iova = mapping->mmnode.start << PAGE_SHIFT;

85
drivers/gpu/drm/panfrost/panfrost_gpu.c
View File

@ -60,14 +60,21 @@ int panfrost_gpu_soft_reset(struct panfrost_device *pfdev)
gpu_write(pfdev, GPU_INT_MASK, 0);
gpu_write(pfdev, GPU_INT_CLEAR, GPU_IRQ_RESET_COMPLETED);
gpu_write(pfdev, GPU_CMD, GPU_CMD_SOFT_RESET);
gpu_write(pfdev, GPU_CMD, GPU_CMD_SOFT_RESET);
ret = readl_relaxed_poll_timeout(pfdev->iomem + GPU_INT_RAWSTAT,
val, val & GPU_IRQ_RESET_COMPLETED, 100, 10000);
val, val & GPU_IRQ_RESET_COMPLETED, 10, 10000);
if (ret) {
dev_err(pfdev->dev, "gpu soft reset timed out\n");
return ret;
dev_err(pfdev->dev, "gpu soft reset timed out, attempting hard reset\n");
gpu_write(pfdev, GPU_CMD, GPU_CMD_HARD_RESET);
ret = readl_relaxed_poll_timeout(pfdev->iomem + GPU_INT_RAWSTAT, val,
val & GPU_IRQ_RESET_COMPLETED, 100, 10000);
if (ret) {
dev_err(pfdev->dev, "gpu hard reset timed out\n");
return ret;
}
}
gpu_write(pfdev, GPU_INT_CLEAR, GPU_IRQ_MASK_ALL);
@ -362,32 +369,42 @@ unsigned long long panfrost_cycle_counter_read(struct panfrost_device *pfdev)
return ((u64)hi << 32) | lo;
}
static u64 panfrost_get_core_mask(struct panfrost_device *pfdev)
{
u64 core_mask;
if (pfdev->features.l2_present == 1)
return U64_MAX;
/*
* Only support one core group now.
* ~(l2_present - 1) unsets all bits in l2_present except
* the bottom bit. (l2_present - 2) has all the bits in
* the first core group set. AND them together to generate
* a mask of cores in the first core group.
*/
core_mask = ~(pfdev->features.l2_present - 1) &
(pfdev->features.l2_present - 2);
dev_info_once(pfdev->dev, "using only 1st core group (%lu cores from %lu)\n",
hweight64(core_mask),
hweight64(pfdev->features.shader_present));
return core_mask;
}
void panfrost_gpu_power_on(struct panfrost_device *pfdev)
{
int ret;
u32 val;
u64 core_mask = U64_MAX;
u64 core_mask;
panfrost_gpu_init_quirks(pfdev);
core_mask = panfrost_get_core_mask(pfdev);
if (pfdev->features.l2_present != 1) {
/*
* Only support one core group now.
* ~(l2_present - 1) unsets all bits in l2_present except
* the bottom bit. (l2_present - 2) has all the bits in
* the first core group set. AND them together to generate
* a mask of cores in the first core group.
*/
core_mask = ~(pfdev->features.l2_present - 1) &
(pfdev->features.l2_present - 2);
dev_info_once(pfdev->dev, "using only 1st core group (%lu cores from %lu)\n",
hweight64(core_mask),
hweight64(pfdev->features.shader_present));
}
gpu_write(pfdev, L2_PWRON_LO, pfdev->features.l2_present & core_mask);
ret = readl_relaxed_poll_timeout(pfdev->iomem + L2_READY_LO,
val, val == (pfdev->features.l2_present & core_mask),
100, 20000);
10, 20000);
if (ret)
dev_err(pfdev->dev, "error powering up gpu L2");
@ -395,22 +412,40 @@ void panfrost_gpu_power_on(struct panfrost_device *pfdev)
pfdev->features.shader_present & core_mask);
ret = readl_relaxed_poll_timeout(pfdev->iomem + SHADER_READY_LO,
val, val == (pfdev->features.shader_present & core_mask),
100, 20000);
10, 20000);
if (ret)
dev_err(pfdev->dev, "error powering up gpu shader");
gpu_write(pfdev, TILER_PWRON_LO, pfdev->features.tiler_present);
ret = readl_relaxed_poll_timeout(pfdev->iomem + TILER_READY_LO,
val, val == pfdev->features.tiler_present, 100, 1000);
val, val == pfdev->features.tiler_present, 10, 1000);
if (ret)
dev_err(pfdev->dev, "error powering up gpu tiler");
}
void panfrost_gpu_power_off(struct panfrost_device *pfdev)
{
gpu_write(pfdev, TILER_PWROFF_LO, 0);
gpu_write(pfdev, SHADER_PWROFF_LO, 0);
gpu_write(pfdev, L2_PWROFF_LO, 0);
u64 core_mask = panfrost_get_core_mask(pfdev);
int ret;
u32 val;
gpu_write(pfdev, SHADER_PWROFF_LO, pfdev->features.shader_present & core_mask);
ret = readl_relaxed_poll_timeout(pfdev->iomem + SHADER_PWRTRANS_LO,
val, !val, 1, 1000);
if (ret)
dev_err(pfdev->dev, "shader power transition timeout");
gpu_write(pfdev, TILER_PWROFF_LO, pfdev->features.tiler_present);
ret = readl_relaxed_poll_timeout(pfdev->iomem + TILER_PWRTRANS_LO,
val, !val, 1, 1000);
if (ret)
dev_err(pfdev->dev, "tiler power transition timeout");
gpu_write(pfdev, L2_PWROFF_LO, pfdev->features.l2_present & core_mask);
ret = readl_poll_timeout(pfdev->iomem + L2_PWRTRANS_LO,
val, !val, 0, 1000);
if (ret)
dev_err(pfdev->dev, "l2 power transition timeout");
}
int panfrost_gpu_init(struct panfrost_device *pfdev)

4
drivers/gpu/drm/panfrost/panfrost_job.c
View File

@ -852,7 +852,7 @@ int panfrost_job_init(struct panfrost_device *pfdev)
js->queue[j].fence_context = dma_fence_context_alloc(1);
ret = drm_sched_init(&js->queue[j].sched,
&panfrost_sched_ops,
&panfrost_sched_ops, NULL,
DRM_SCHED_PRIORITY_COUNT,
nentries, 0,
msecs_to_jiffies(JOB_TIMEOUT_MS),
@ -963,7 +963,7 @@ int panfrost_job_is_idle(struct panfrost_device *pfdev)
for (i = 0; i < NUM_JOB_SLOTS; i++) {
/* If there are any jobs in the HW queue, we're not idle */
if (atomic_read(&js->queue[i].sched.hw_rq_count))
if (atomic_read(&js->queue[i].sched.credit_count))
return false;
}

1
drivers/gpu/drm/panfrost/panfrost_regs.h
View File

@ -44,6 +44,7 @@
GPU_IRQ_MULTIPLE_FAULT)
#define GPU_CMD 0x30
#define GPU_CMD_SOFT_RESET 0x01
#define GPU_CMD_HARD_RESET 0x02
#define GPU_CMD_PERFCNT_CLEAR 0x03
#define GPU_CMD_PERFCNT_SAMPLE 0x04
#define GPU_CMD_CYCLE_COUNT_START 0x05

1
drivers/gpu/drm/radeon/radeon_audio.c
View File

@ -26,6 +26,7 @@
#include <linux/component.h>
#include <drm/drm_crtc.h>
#include <drm/drm_eld.h>
#include "dce6_afmt.h"
#include "evergreen_hdmi.h"
#include "radeon.h"

2
drivers/gpu/drm/scheduler/gpu_scheduler_trace.h
View File

@ -51,7 +51,7 @@ DECLARE_EVENT_CLASS(drm_sched_job,
__assign_str(name, sched_job->sched->name);
__entry->job_count = spsc_queue_count(&entity->job_queue);
__entry->hw_job_count = atomic_read(
&sched_job->sched->hw_rq_count);
&sched_job->sched->credit_count);
),
TP_printk("entity=%p, id=%llu, fence=%p, ring=%s, job count:%u, hw job count:%d",
__entry->entity, __entry->id,

4
drivers/gpu/drm/scheduler/sched_entity.c
View File

@ -370,7 +370,7 @@ static void drm_sched_entity_wakeup(struct dma_fence *f,
container_of(cb, struct drm_sched_entity, cb);
drm_sched_entity_clear_dep(f, cb);
drm_sched_wakeup_if_can_queue(entity->rq->sched);
drm_sched_wakeup(entity->rq->sched, entity);
}
/**
@ -602,7 +602,7 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job)
if (drm_sched_policy == DRM_SCHED_POLICY_FIFO)
drm_sched_rq_update_fifo(entity, submit_ts);
drm_sched_wakeup_if_can_queue(entity->rq->sched);
drm_sched_wakeup(entity->rq->sched, entity);
}
}
EXPORT_SYMBOL(drm_sched_entity_push_job);

486
drivers/gpu/drm/scheduler/sched_main.c
View File

@ -48,7 +48,30 @@
* through the jobs entity pointer.
*/
#include <linux/kthread.h>
/**
* DOC: Flow Control
*
* The DRM GPU scheduler provides a flow control mechanism to regulate the rate
* in which the jobs fetched from scheduler entities are executed.
*
* In this context the &drm_gpu_scheduler keeps track of a driver specified
* credit limit representing the capacity of this scheduler and a credit count;
* every &drm_sched_job carries a driver specified number of credits.
*
* Once a job is executed (but not yet finished), the job's credits contribute
* to the scheduler's credit count until the job is finished. If by executing
* one more job the scheduler's credit count would exceed the scheduler's
* credit limit, the job won't be executed. Instead, the scheduler will wait
* until the credit count has decreased enough to not overflow its credit limit.
* This implies waiting for previously executed jobs.
*
* Optionally, drivers may register a callback (update_job_credits) provided by
* struct drm_sched_backend_ops to update the job's credits dynamically. The
* scheduler executes this callback every time the scheduler considers a job for
* execution and subsequently checks whether the job fits the scheduler's credit
* limit.
*/
#include <linux/wait.h>
#include <linux/sched.h>
#include <linux/completion.h>
@ -76,6 +99,51 @@ int drm_sched_policy = DRM_SCHED_POLICY_FIFO;
MODULE_PARM_DESC(sched_policy, "Specify the scheduling policy for entities on a run-queue, " __stringify(DRM_SCHED_POLICY_RR) " = Round Robin, " __stringify(DRM_SCHED_POLICY_FIFO) " = FIFO (default).");
module_param_named(sched_policy, drm_sched_policy, int, 0444);
static u32 drm_sched_available_credits(struct drm_gpu_scheduler *sched)
{
u32 credits;
drm_WARN_ON(sched, check_sub_overflow(sched->credit_limit,
atomic_read(&sched->credit_count),
&credits));
return credits;
}
/**
* drm_sched_can_queue -- Can we queue more to the hardware?
* @sched: scheduler instance
* @entity: the scheduler entity
*
* Return true if we can push at least one more job from @entity, false
* otherwise.
*/
static bool drm_sched_can_queue(struct drm_gpu_scheduler *sched,
struct drm_sched_entity *entity)
{
struct drm_sched_job *s_job;
s_job = to_drm_sched_job(spsc_queue_peek(&entity->job_queue));
if (!s_job)
return false;
if (sched->ops->update_job_credits) {
s_job->credits = sched->ops->update_job_credits(s_job);
drm_WARN(sched, !s_job->credits,
"Jobs with zero credits bypass job-flow control.\n");
}
/* If a job exceeds the credit limit, truncate it to the credit limit
* itself to guarantee forward progress.
*/
if (drm_WARN(sched, s_job->credits > sched->credit_limit,
"Jobs may not exceed the credit limit, truncate.\n"))
s_job->credits = sched->credit_limit;
return drm_sched_available_credits(sched) >= s_job->credits;
}
static __always_inline bool drm_sched_entity_compare_before(struct rb_node *a,
const struct rb_node *b)
{
@ -187,12 +255,18 @@ void drm_sched_rq_remove_entity(struct drm_sched_rq *rq,
/**
* drm_sched_rq_select_entity_rr - Select an entity which could provide a job to run
*
* @sched: the gpu scheduler
* @rq: scheduler run queue to check.
*
* Try to find a ready entity, returns NULL if none found.
* Try to find the next ready entity.
*
* Return an entity if one is found; return an error-pointer (!NULL) if an
* entity was ready, but the scheduler had insufficient credits to accommodate
* its job; return NULL, if no ready entity was found.
*/
static struct drm_sched_entity *
drm_sched_rq_select_entity_rr(struct drm_sched_rq *rq)
drm_sched_rq_select_entity_rr(struct drm_gpu_scheduler *sched,
struct drm_sched_rq *rq)
{
struct drm_sched_entity *entity;
@ -202,6 +276,14 @@ drm_sched_rq_select_entity_rr(struct drm_sched_rq *rq)
if (entity) {
list_for_each_entry_continue(entity, &rq->entities, list) {
if (drm_sched_entity_is_ready(entity)) {
/* If we can't queue yet, preserve the current
* entity in terms of fairness.
*/
if (!drm_sched_can_queue(sched, entity)) {
spin_unlock(&rq->lock);
return ERR_PTR(-ENOSPC);
}
rq->current_entity = entity;
reinit_completion(&entity->entity_idle);
spin_unlock(&rq->lock);
@ -211,8 +293,15 @@ drm_sched_rq_select_entity_rr(struct drm_sched_rq *rq)
}
list_for_each_entry(entity, &rq->entities, list) {
if (drm_sched_entity_is_ready(entity)) {
/* If we can't queue yet, preserve the current entity in
* terms of fairness.
*/
if (!drm_sched_can_queue(sched, entity)) {
spin_unlock(&rq->lock);
return ERR_PTR(-ENOSPC);
}
rq->current_entity = entity;
reinit_completion(&entity->entity_idle);
spin_unlock(&rq->lock);
@ -231,12 +320,18 @@ drm_sched_rq_select_entity_rr(struct drm_sched_rq *rq)
/**
* drm_sched_rq_select_entity_fifo - Select an entity which provides a job to run
*
* @sched: the gpu scheduler
* @rq: scheduler run queue to check.
*
* Find oldest waiting ready entity, returns NULL if none found.
* Find oldest waiting ready entity.
*
* Return an entity if one is found; return an error-pointer (!NULL) if an
* entity was ready, but the scheduler had insufficient credits to accommodate
* its job; return NULL, if no ready entity was found.
*/
static struct drm_sched_entity *
drm_sched_rq_select_entity_fifo(struct drm_sched_rq *rq)
drm_sched_rq_select_entity_fifo(struct drm_gpu_scheduler *sched,
struct drm_sched_rq *rq)
{
struct rb_node *rb;
@ -246,6 +341,14 @@ drm_sched_rq_select_entity_fifo(struct drm_sched_rq *rq)
entity = rb_entry(rb, struct drm_sched_entity, rb_tree_node);
if (drm_sched_entity_is_ready(entity)) {
/* If we can't queue yet, preserve the current entity in
* terms of fairness.
*/
if (!drm_sched_can_queue(sched, entity)) {
spin_unlock(&rq->lock);
return ERR_PTR(-ENOSPC);
}
rq->current_entity = entity;
reinit_completion(&entity->entity_idle);
break;
@ -256,6 +359,42 @@ drm_sched_rq_select_entity_fifo(struct drm_sched_rq *rq)
return rb ? rb_entry(rb, struct drm_sched_entity, rb_tree_node) : NULL;
}
/**
* drm_sched_run_job_queue - enqueue run-job work
* @sched: scheduler instance
*/
static void drm_sched_run_job_queue(struct drm_gpu_scheduler *sched)
{
if (!READ_ONCE(sched->pause_submit))
queue_work(sched->submit_wq, &sched->work_run_job);
}
/**
* __drm_sched_run_free_queue - enqueue free-job work
* @sched: scheduler instance
*/
static void __drm_sched_run_free_queue(struct drm_gpu_scheduler *sched)
{
if (!READ_ONCE(sched->pause_submit))
queue_work(sched->submit_wq, &sched->work_free_job);
}
/**
* drm_sched_run_free_queue - enqueue free-job work if ready
* @sched: scheduler instance
*/
static void drm_sched_run_free_queue(struct drm_gpu_scheduler *sched)
{
struct drm_sched_job *job;
spin_lock(&sched->job_list_lock);
job = list_first_entry_or_null(&sched->pending_list,
struct drm_sched_job, list);
if (job && dma_fence_is_signaled(&job->s_fence->finished))
__drm_sched_run_free_queue(sched);
spin_unlock(&sched->job_list_lock);
}
/**
* drm_sched_job_done - complete a job
* @s_job: pointer to the job which is done
@ -267,7 +406,7 @@ static void drm_sched_job_done(struct drm_sched_job *s_job, int result)
struct drm_sched_fence *s_fence = s_job->s_fence;
struct drm_gpu_scheduler *sched = s_fence->sched;
atomic_dec(&sched->hw_rq_count);
atomic_sub(s_job->credits, &sched->credit_count);
atomic_dec(sched->score);
trace_drm_sched_process_job(s_fence);
@ -275,7 +414,7 @@ static void drm_sched_job_done(struct drm_sched_job *s_job, int result)
dma_fence_get(&s_fence->finished);
drm_sched_fence_finished(s_fence, result);
dma_fence_put(&s_fence->finished);
wake_up_interruptible(&sched->wake_up_worker);
__drm_sched_run_free_queue(sched);
}
/**
@ -299,11 +438,36 @@ static void drm_sched_job_done_cb(struct dma_fence *f, struct dma_fence_cb *cb)
*/
static void drm_sched_start_timeout(struct drm_gpu_scheduler *sched)
{
lockdep_assert_held(&sched->job_list_lock);
if (sched->timeout != MAX_SCHEDULE_TIMEOUT &&
!list_empty(&sched->pending_list))
queue_delayed_work(sched->timeout_wq, &sched->work_tdr, sched->timeout);
mod_delayed_work(sched->timeout_wq, &sched->work_tdr, sched->timeout);
}
static void drm_sched_start_timeout_unlocked(struct drm_gpu_scheduler *sched)
{
spin_lock(&sched->job_list_lock);
drm_sched_start_timeout(sched);
spin_unlock(&sched->job_list_lock);
}
/**
* drm_sched_tdr_queue_imm: - immediately start job timeout handler
*
* @sched: scheduler for which the timeout handling should be started.
*
* Start timeout handling immediately for the named scheduler.
*/
void drm_sched_tdr_queue_imm(struct drm_gpu_scheduler *sched)
{
spin_lock(&sched->job_list_lock);
sched->timeout = 0;
drm_sched_start_timeout(sched);
spin_unlock(&sched->job_list_lock);
}
EXPORT_SYMBOL(drm_sched_tdr_queue_imm);
/**
* drm_sched_fault - immediately start timeout handler
*
@ -388,7 +552,7 @@ static void drm_sched_job_timedout(struct work_struct *work)
sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work);
/* Protects against concurrent deletion in drm_sched_get_cleanup_job */
/* Protects against concurrent deletion in drm_sched_get_finished_job */
spin_lock(&sched->job_list_lock);
job = list_first_entry_or_null(&sched->pending_list,
struct drm_sched_job, list);
@ -416,11 +580,8 @@ static void drm_sched_job_timedout(struct work_struct *work)
spin_unlock(&sched->job_list_lock);
}
if (status != DRM_GPU_SCHED_STAT_ENODEV) {
spin_lock(&sched->job_list_lock);
drm_sched_start_timeout(sched);
spin_unlock(&sched->job_list_lock);
}
if (status != DRM_GPU_SCHED_STAT_ENODEV)
drm_sched_start_timeout_unlocked(sched);
}
/**
@ -439,13 +600,13 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
{
struct drm_sched_job *s_job, *tmp;
kthread_park(sched->thread);
drm_sched_wqueue_stop(sched);
/*
* Reinsert back the bad job here - now it's safe as
* drm_sched_get_cleanup_job cannot race against us and release the
* drm_sched_get_finished_job cannot race against us and release the
* bad job at this point - we parked (waited for) any in progress
* (earlier) cleanups and drm_sched_get_cleanup_job will not be called
* (earlier) cleanups and drm_sched_get_finished_job will not be called
* now until the scheduler thread is unparked.
*/
if (bad && bad->sched == sched)
@ -468,7 +629,7 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
&s_job->cb)) {
dma_fence_put(s_job->s_fence->parent);
s_job->s_fence->parent = NULL;
atomic_dec(&sched->hw_rq_count);
atomic_sub(s_job->credits, &sched->credit_count);
} else {
/*
* remove job from pending_list.
@ -529,7 +690,7 @@ void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery)
list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {
struct dma_fence *fence = s_job->s_fence->parent;
atomic_inc(&sched->hw_rq_count);
atomic_add(s_job->credits, &sched->credit_count);
if (!full_recovery)
continue;
@ -546,13 +707,10 @@ void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery)
drm_sched_job_done(s_job, -ECANCELED);
}
if (full_recovery) {
spin_lock(&sched->job_list_lock);
drm_sched_start_timeout(sched);
spin_unlock(&sched->job_list_lock);
}
if (full_recovery)
drm_sched_start_timeout_unlocked(sched);
kthread_unpark(sched->thread);
drm_sched_wqueue_start(sched);
}
EXPORT_SYMBOL(drm_sched_start);
@ -613,6 +771,8 @@ EXPORT_SYMBOL(drm_sched_resubmit_jobs);
* drm_sched_job_init - init a scheduler job
* @job: scheduler job to init
* @entity: scheduler entity to use
* @credits: the number of credits this job contributes to the schedulers
* credit limit
* @owner: job owner for debugging
*
* Refer to drm_sched_entity_push_job() documentation
@ -630,7 +790,7 @@ EXPORT_SYMBOL(drm_sched_resubmit_jobs);
*/
int drm_sched_job_init(struct drm_sched_job *job,
struct drm_sched_entity *entity,
void *owner)
u32 credits, void *owner)
{
if (!entity->rq) {
/* This will most likely be followed by missing frames
@ -641,7 +801,13 @@ int drm_sched_job_init(struct drm_sched_job *job,
return -ENOENT;
}
if (unlikely(!credits)) {
pr_err("*ERROR* %s: credits cannot be 0!\n", __func__);
return -EINVAL;
}
job->entity = entity;
job->credits = credits;
job->s_fence = drm_sched_fence_alloc(entity, owner);
if (!job->s_fence)
return -ENOMEM;
@ -854,27 +1020,18 @@ void drm_sched_job_cleanup(struct drm_sched_job *job)
EXPORT_SYMBOL(drm_sched_job_cleanup);
/**
* drm_sched_can_queue -- Can we queue more to the hardware?
* @sched: scheduler instance
*
* Return true if we can push more jobs to the hw, otherwise false.
*/
static bool drm_sched_can_queue(struct drm_gpu_scheduler *sched)
{
return atomic_read(&sched->hw_rq_count) <
sched->hw_submission_limit;
}
/**
* drm_sched_wakeup_if_can_queue - Wake up the scheduler
* drm_sched_wakeup - Wake up the scheduler if it is ready to queue
* @sched: scheduler instance
* @entity: the scheduler entity
*
* Wake up the scheduler if we can queue jobs.
*/
void drm_sched_wakeup_if_can_queue(struct drm_gpu_scheduler *sched)
void drm_sched_wakeup(struct drm_gpu_scheduler *sched,
struct drm_sched_entity *entity)
{
if (drm_sched_can_queue(sched))
wake_up_interruptible(&sched->wake_up_worker);
if (drm_sched_entity_is_ready(entity))
if (drm_sched_can_queue(sched, entity))
drm_sched_run_job_queue(sched);
}
/**
@ -882,7 +1039,11 @@ void drm_sched_wakeup_if_can_queue(struct drm_gpu_scheduler *sched)
*
* @sched: scheduler instance
*
* Returns the entity to process or NULL if none are found.
* Return an entity to process or NULL if none are found.
*
* Note, that we break out of the for-loop when "entity" is non-null, which can
* also be an error-pointer--this assures we don't process lower priority
* run-queues. See comments in the respectively called functions.
*/
static struct drm_sched_entity *
drm_sched_select_entity(struct drm_gpu_scheduler *sched)
@ -890,23 +1051,20 @@ drm_sched_select_entity(struct drm_gpu_scheduler *sched)
struct drm_sched_entity *entity;
int i;
if (!drm_sched_can_queue(sched))
return NULL;
/* Kernel run queue has higher priority than normal run queue*/
for (i = sched->num_rqs - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) {
entity = drm_sched_policy == DRM_SCHED_POLICY_FIFO ?
drm_sched_rq_select_entity_fifo(sched->sched_rq[i]) :
drm_sched_rq_select_entity_rr(sched->sched_rq[i]);
drm_sched_rq_select_entity_fifo(sched, sched->sched_rq[i]) :
drm_sched_rq_select_entity_rr(sched, sched->sched_rq[i]);
if (entity)
break;
}
return entity;
return IS_ERR(entity) ? NULL : entity;
}
/**
* drm_sched_get_cleanup_job - fetch the next finished job to be destroyed
* drm_sched_get_finished_job - fetch the next finished job to be destroyed
*
* @sched: scheduler instance
*
@ -914,7 +1072,7 @@ drm_sched_select_entity(struct drm_gpu_scheduler *sched)
* ready for it to be destroyed.
*/
static struct drm_sched_job *
drm_sched_get_cleanup_job(struct drm_gpu_scheduler *sched)
drm_sched_get_finished_job(struct drm_gpu_scheduler *sched)
{
struct drm_sched_job *job, *next;
@ -934,8 +1092,10 @@ drm_sched_get_cleanup_job(struct drm_gpu_scheduler *sched)
typeof(*next), list);
if (next) {
next->s_fence->scheduled.timestamp =
dma_fence_timestamp(&job->s_fence->finished);
if (test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT,
&next->s_fence->scheduled.flags))
next->s_fence->scheduled.timestamp =
dma_fence_timestamp(&job->s_fence->finished);
/* start TO timer for next job */
drm_sched_start_timeout(sched);
}
@ -985,91 +1145,82 @@ drm_sched_pick_best(struct drm_gpu_scheduler **sched_list,
EXPORT_SYMBOL(drm_sched_pick_best);
/**
* drm_sched_blocked - check if the scheduler is blocked
* drm_sched_free_job_work - worker to call free_job
*
* @sched: scheduler instance
*
* Returns true if blocked, otherwise false.
* @w: free job work
*/
static bool drm_sched_blocked(struct drm_gpu_scheduler *sched)
static void drm_sched_free_job_work(struct work_struct *w)
{
if (kthread_should_park()) {
kthread_parkme();
return true;
}
struct drm_gpu_scheduler *sched =
container_of(w, struct drm_gpu_scheduler, work_free_job);
struct drm_sched_job *job;
return false;
if (READ_ONCE(sched->pause_submit))
return;
job = drm_sched_get_finished_job(sched);
if (job)
sched->ops->free_job(job);
drm_sched_run_free_queue(sched);
drm_sched_run_job_queue(sched);
}
/**
* drm_sched_main - main scheduler thread
* drm_sched_run_job_work - worker to call run_job
*
* @param: scheduler instance
*
* Returns 0.
* @w: run job work
*/
static int drm_sched_main(void *param)
static void drm_sched_run_job_work(struct work_struct *w)
{
struct drm_gpu_scheduler *sched = (struct drm_gpu_scheduler *)param;
struct drm_gpu_scheduler *sched =
container_of(w, struct drm_gpu_scheduler, work_run_job);
struct drm_sched_entity *entity;
struct dma_fence *fence;
struct drm_sched_fence *s_fence;
struct drm_sched_job *sched_job;
int r;
sched_set_fifo_low(current);
if (READ_ONCE(sched->pause_submit))
return;
while (!kthread_should_stop()) {
struct drm_sched_entity *entity = NULL;
struct drm_sched_fence *s_fence;
struct drm_sched_job *sched_job;
struct dma_fence *fence;
struct drm_sched_job *cleanup_job = NULL;
entity = drm_sched_select_entity(sched);
if (!entity)
return;
wait_event_interruptible(sched->wake_up_worker,
(cleanup_job = drm_sched_get_cleanup_job(sched)) ||
(!drm_sched_blocked(sched) &&
(entity = drm_sched_select_entity(sched))) ||
kthread_should_stop());
if (cleanup_job)
sched->ops->free_job(cleanup_job);
if (!entity)
continue;
sched_job = drm_sched_entity_pop_job(entity);
if (!sched_job) {
complete_all(&entity->entity_idle);
continue;
}
s_fence = sched_job->s_fence;
atomic_inc(&sched->hw_rq_count);
drm_sched_job_begin(sched_job);
trace_drm_run_job(sched_job, entity);
fence = sched->ops->run_job(sched_job);
sched_job = drm_sched_entity_pop_job(entity);
if (!sched_job) {
complete_all(&entity->entity_idle);
drm_sched_fence_scheduled(s_fence, fence);
if (!IS_ERR_OR_NULL(fence)) {
/* Drop for original kref_init of the fence */
dma_fence_put(fence);
r = dma_fence_add_callback(fence, &sched_job->cb,
drm_sched_job_done_cb);
if (r == -ENOENT)
drm_sched_job_done(sched_job, fence->error);
else if (r)
DRM_DEV_ERROR(sched->dev, "fence add callback failed (%d)\n",
r);
} else {
drm_sched_job_done(sched_job, IS_ERR(fence) ?
PTR_ERR(fence) : 0);
}
wake_up(&sched->job_scheduled);
return; /* No more work */
}
return 0;
s_fence = sched_job->s_fence;
atomic_add(sched_job->credits, &sched->credit_count);
drm_sched_job_begin(sched_job);
trace_drm_run_job(sched_job, entity);
fence = sched->ops->run_job(sched_job);
complete_all(&entity->entity_idle);
drm_sched_fence_scheduled(s_fence, fence);
if (!IS_ERR_OR_NULL(fence)) {
/* Drop for original kref_init of the fence */
dma_fence_put(fence);
r = dma_fence_add_callback(fence, &sched_job->cb,
drm_sched_job_done_cb);
if (r == -ENOENT)
drm_sched_job_done(sched_job, fence->error);
else if (r)
DRM_DEV_ERROR(sched->dev, "fence add callback failed (%d)\n", r);
} else {
drm_sched_job_done(sched_job, IS_ERR(fence) ?
PTR_ERR(fence) : 0);
}
wake_up(&sched->job_scheduled);
drm_sched_run_job_queue(sched);
}
/**
@ -1077,8 +1228,10 @@ static int drm_sched_main(void *param)
*
* @sched: scheduler instance
* @ops: backend operations for this scheduler
* @submit_wq: workqueue to use for submission. If NULL, an ordered wq is
* allocated and used
* @num_rqs: number of runqueues, one for each priority, up to DRM_SCHED_PRIORITY_COUNT
* @hw_submission: number of hw submissions that can be in flight
* @credit_limit: the number of credits this scheduler can hold from all jobs
* @hang_limit: number of times to allow a job to hang before dropping it
* @timeout: timeout value in jiffies for the scheduler
* @timeout_wq: workqueue to use for timeout work. If NULL, the system_wq is
@ -1091,14 +1244,15 @@ static int drm_sched_main(void *param)
*/
int drm_sched_init(struct drm_gpu_scheduler *sched,
const struct drm_sched_backend_ops *ops,
u32 num_rqs, uint32_t hw_submission, unsigned int hang_limit,
struct workqueue_struct *submit_wq,
u32 num_rqs, u32 credit_limit, unsigned int hang_limit,
long timeout, struct workqueue_struct *timeout_wq,
atomic_t *score, const char *name, struct device *dev)
{
int i, ret;
sched->ops = ops;
sched->hw_submission_limit = hw_submission;
sched->credit_limit = credit_limit;
sched->name = name;
sched->timeout = timeout;
sched->timeout_wq = timeout_wq ? : system_wq;
@ -1121,14 +1275,22 @@ int drm_sched_init(struct drm_gpu_scheduler *sched,
return 0;
}
if (submit_wq) {
sched->submit_wq = submit_wq;
sched->own_submit_wq = false;
} else {
sched->submit_wq = alloc_ordered_workqueue(name, 0);
if (!sched->submit_wq)
return -ENOMEM;
sched->own_submit_wq = true;
}
ret = -ENOMEM;
sched->sched_rq = kmalloc_array(num_rqs, sizeof(*sched->sched_rq),
GFP_KERNEL | __GFP_ZERO);
if (!sched->sched_rq) {
drm_err(sched, "%s: out of memory for sched_rq\n", __func__);
return -ENOMEM;
}
if (!sched->sched_rq)
goto Out_free;
sched->num_rqs = num_rqs;
ret = -ENOMEM;
for (i = DRM_SCHED_PRIORITY_MIN; i < sched->num_rqs; i++) {
sched->sched_rq[i] = kzalloc(sizeof(*sched->sched_rq[i]), GFP_KERNEL);
if (!sched->sched_rq[i])
@ -1136,31 +1298,27 @@ int drm_sched_init(struct drm_gpu_scheduler *sched,
drm_sched_rq_init(sched, sched->sched_rq[i]);
}
init_waitqueue_head(&sched->wake_up_worker);
init_waitqueue_head(&sched->job_scheduled);
INIT_LIST_HEAD(&sched->pending_list);
spin_lock_init(&sched->job_list_lock);
atomic_set(&sched->hw_rq_count, 0);
atomic_set(&sched->credit_count, 0);
INIT_DELAYED_WORK(&sched->work_tdr, drm_sched_job_timedout);
INIT_WORK(&sched->work_run_job, drm_sched_run_job_work);
INIT_WORK(&sched->work_free_job, drm_sched_free_job_work);
atomic_set(&sched->_score, 0);
atomic64_set(&sched->job_id_count, 0);
/* Each scheduler will run on a seperate kernel thread */
sched->thread = kthread_run(drm_sched_main, sched, sched->name);
if (IS_ERR(sched->thread)) {
ret = PTR_ERR(sched->thread);
sched->thread = NULL;
DRM_DEV_ERROR(sched->dev, "Failed to create scheduler for %s.\n", name);
goto Out_unroll;
}
sched->pause_submit = false;
sched->ready = true;
return 0;
Out_unroll:
for (--i ; i >= DRM_SCHED_PRIORITY_MIN; i--)
kfree(sched->sched_rq[i]);
Out_free:
kfree(sched->sched_rq);
sched->sched_rq = NULL;
if (sched->own_submit_wq)
destroy_workqueue(sched->submit_wq);
drm_err(sched, "%s: Failed to setup GPU scheduler--out of memory\n", __func__);
return ret;
}
@ -1178,8 +1336,7 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched)
struct drm_sched_entity *s_entity;
int i;
if (sched->thread)
kthread_stop(sched->thread);
drm_sched_wqueue_stop(sched);
for (i = sched->num_rqs - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) {
struct drm_sched_rq *rq = sched->sched_rq[i];
@ -1202,6 +1359,8 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched)
/* Confirm no work left behind accessing device structures */
cancel_delayed_work_sync(&sched->work_tdr);
if (sched->own_submit_wq)
destroy_workqueue(sched->submit_wq);
sched->ready = false;
kfree(sched->sched_rq);
sched->sched_rq = NULL;
@ -1252,3 +1411,42 @@ void drm_sched_increase_karma(struct drm_sched_job *bad)
}
}
EXPORT_SYMBOL(drm_sched_increase_karma);
/**
* drm_sched_wqueue_ready - Is the scheduler ready for submission
*
* @sched: scheduler instance
*
* Returns true if submission is ready
*/
bool drm_sched_wqueue_ready(struct drm_gpu_scheduler *sched)
{
return sched->ready;
}
EXPORT_SYMBOL(drm_sched_wqueue_ready);
/**
* drm_sched_wqueue_stop - stop scheduler submission
*
* @sched: scheduler instance
*/
void drm_sched_wqueue_stop(struct drm_gpu_scheduler *sched)
{
WRITE_ONCE(sched->pause_submit, true);
cancel_work_sync(&sched->work_run_job);
cancel_work_sync(&sched->work_free_job);
}
EXPORT_SYMBOL(drm_sched_wqueue_stop);
/**
* drm_sched_wqueue_start - start scheduler submission
*
* @sched: scheduler instance
*/
void drm_sched_wqueue_start(struct drm_gpu_scheduler *sched)
{
WRITE_ONCE(sched->pause_submit, false);
queue_work(sched->submit_wq, &sched->work_run_job);
queue_work(sched->submit_wq, &sched->work_free_job);
}
EXPORT_SYMBOL(drm_sched_wqueue_start);

38
drivers/gpu/drm/solomon/ssd130x.c
View File

@ -808,7 +808,8 @@ static void ssd132x_clear_screen(struct ssd130x_device *ssd130x, u8 *data_array)
static int ssd130x_fb_blit_rect(struct drm_framebuffer *fb,
const struct iosys_map *vmap,
struct drm_rect *rect,
u8 *buf, u8 *data_array)
u8 *buf, u8 *data_array,
struct drm_format_conv_state *fmtcnv_state)
{
struct ssd130x_device *ssd130x = drm_to_ssd130x(fb->dev);
struct iosys_map dst;
@ -826,7 +827,7 @@ static int ssd130x_fb_blit_rect(struct drm_framebuffer *fb,
return ret;
iosys_map_set_vaddr(&dst, buf);
drm_fb_xrgb8888_to_mono(&dst, &dst_pitch, vmap, fb, rect);
drm_fb_xrgb8888_to_mono(&dst, &dst_pitch, vmap, fb, rect, fmtcnv_state);
drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE);
@ -838,7 +839,8 @@ static int ssd130x_fb_blit_rect(struct drm_framebuffer *fb,
static int ssd132x_fb_blit_rect(struct drm_framebuffer *fb,
const struct iosys_map *vmap,
struct drm_rect *rect, u8 *buf,
u8 *data_array)
u8 *data_array,
struct drm_format_conv_state *fmtcnv_state)
{
struct ssd130x_device *ssd130x = drm_to_ssd130x(fb->dev);
unsigned int dst_pitch = drm_rect_width(rect);
@ -855,7 +857,7 @@ static int ssd132x_fb_blit_rect(struct drm_framebuffer *fb,
return ret;
iosys_map_set_vaddr(&dst, buf);
drm_fb_xrgb8888_to_gray8(&dst, &dst_pitch, vmap, fb, rect);
drm_fb_xrgb8888_to_gray8(&dst, &dst_pitch, vmap, fb, rect, fmtcnv_state);
drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE);
@ -871,6 +873,7 @@ static int ssd130x_primary_plane_atomic_check(struct drm_plane *plane,
struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
struct ssd130x_plane_state *ssd130x_state = to_ssd130x_plane_state(plane_state);
struct drm_shadow_plane_state *shadow_plane_state = &ssd130x_state->base;
struct drm_crtc *crtc = plane_state->crtc;
struct drm_crtc_state *crtc_state = NULL;
const struct drm_format_info *fi;
@ -895,6 +898,16 @@ static int ssd130x_primary_plane_atomic_check(struct drm_plane *plane,
pitch = drm_format_info_min_pitch(fi, 0, ssd130x->width);
if (plane_state->fb->format != fi) {
void *buf;
/* format conversion necessary; reserve buffer */
buf = drm_format_conv_state_reserve(&shadow_plane_state->fmtcnv_state,
pitch, GFP_KERNEL);
if (!buf)
return -ENOMEM;
}
ssd130x_state->buffer = kcalloc(pitch, ssd130x->height, GFP_KERNEL);
if (!ssd130x_state->buffer)
return -ENOMEM;
@ -909,6 +922,7 @@ static int ssd132x_primary_plane_atomic_check(struct drm_plane *plane,
struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
struct ssd130x_plane_state *ssd130x_state = to_ssd130x_plane_state(plane_state);
struct drm_shadow_plane_state *shadow_plane_state = &ssd130x_state->base;
struct drm_crtc *crtc = plane_state->crtc;
struct drm_crtc_state *crtc_state = NULL;
const struct drm_format_info *fi;
@ -933,6 +947,16 @@ static int ssd132x_primary_plane_atomic_check(struct drm_plane *plane,
pitch = drm_format_info_min_pitch(fi, 0, ssd130x->width);
if (plane_state->fb->format != fi) {
void *buf;
/* format conversion necessary; reserve buffer */
buf = drm_format_conv_state_reserve(&shadow_plane_state->fmtcnv_state,
pitch, GFP_KERNEL);
if (!buf)
return -ENOMEM;
}
ssd130x_state->buffer = kcalloc(pitch, ssd130x->height, GFP_KERNEL);
if (!ssd130x_state->buffer)
return -ENOMEM;
@ -968,7 +992,8 @@ static void ssd130x_primary_plane_atomic_update(struct drm_plane *plane,
ssd130x_fb_blit_rect(fb, &shadow_plane_state->data[0], &dst_clip,
ssd130x_plane_state->buffer,
ssd130x_crtc_state->data_array);
ssd130x_crtc_state->data_array,
&shadow_plane_state->fmtcnv_state);
}
drm_dev_exit(idx);
@ -1002,7 +1027,8 @@ static void ssd132x_primary_plane_atomic_update(struct drm_plane *plane,
ssd132x_fb_blit_rect(fb, &shadow_plane_state->data[0], &dst_clip,
ssd130x_plane_state->buffer,
ssd130x_crtc_state->data_array);
ssd130x_crtc_state->data_array,
&shadow_plane_state->fmtcnv_state);
}
drm_dev_exit(idx);

1
drivers/gpu/drm/tegra/hdmi.c
View File

@ -24,6 +24,7 @@
#include <drm/drm_atomic_helper.h>
#include <drm/drm_crtc.h>
#include <drm/drm_debugfs.h>
#include <drm/drm_eld.h>
#include <drm/drm_file.h>
#include <drm/drm_fourcc.h>
#include <drm/drm_probe_helper.h>

Some files were not shown because too many files have changed in this diff Show More