mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-27 03:10:12 +00:00
![Yong Zhao](/assets/img/avatar_default.png)
Up to this point, those functions are all the same for all ASICs, so no need to call them by functions pointers. Removing the function pointers will greatly increase the code readablity. If there is ever need for those function pointers, we can add it back then. Signed-off-by: Yong Zhao <Yong.Zhao@amd.com> Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
845 lines
24 KiB
C
845 lines
24 KiB
C
/*
|
|
* Copyright 2014 Advanced Micro Devices, Inc.
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
* copy of this software and associated documentation files (the "Software"),
|
|
* to deal in the Software without restriction, including without limitation
|
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
* and/or sell copies of the Software, and to permit persons to whom the
|
|
* Software is furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice shall be included in
|
|
* all copies or substantial portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
|
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
|
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
|
* OTHER DEALINGS IN THE SOFTWARE.
|
|
*
|
|
*/
|
|
|
|
#include <linux/types.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/log2.h>
|
|
#include <linux/sched.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/mutex.h>
|
|
#include <linux/device.h>
|
|
|
|
#include "kfd_pm4_headers.h"
|
|
#include "kfd_pm4_headers_diq.h"
|
|
#include "kfd_kernel_queue.h"
|
|
#include "kfd_priv.h"
|
|
#include "kfd_pm4_opcodes.h"
|
|
#include "cik_regs.h"
|
|
#include "kfd_dbgmgr.h"
|
|
#include "kfd_dbgdev.h"
|
|
#include "kfd_device_queue_manager.h"
|
|
|
|
static void dbgdev_address_watch_disable_nodiq(struct kfd_dev *dev)
|
|
{
|
|
dev->kfd2kgd->address_watch_disable(dev->kgd);
|
|
}
|
|
|
|
static int dbgdev_diq_submit_ib(struct kfd_dbgdev *dbgdev,
|
|
unsigned int pasid, uint64_t vmid0_address,
|
|
uint32_t *packet_buff, size_t size_in_bytes)
|
|
{
|
|
struct pm4__release_mem *rm_packet;
|
|
struct pm4__indirect_buffer_pasid *ib_packet;
|
|
struct kfd_mem_obj *mem_obj;
|
|
size_t pq_packets_size_in_bytes;
|
|
union ULARGE_INTEGER *largep;
|
|
union ULARGE_INTEGER addr;
|
|
struct kernel_queue *kq;
|
|
uint64_t *rm_state;
|
|
unsigned int *ib_packet_buff;
|
|
int status;
|
|
|
|
if (WARN_ON(!size_in_bytes))
|
|
return -EINVAL;
|
|
|
|
kq = dbgdev->kq;
|
|
|
|
pq_packets_size_in_bytes = sizeof(struct pm4__release_mem) +
|
|
sizeof(struct pm4__indirect_buffer_pasid);
|
|
|
|
/*
|
|
* We acquire a buffer from DIQ
|
|
* The receive packet buff will be sitting on the Indirect Buffer
|
|
* and in the PQ we put the IB packet + sync packet(s).
|
|
*/
|
|
status = kq_acquire_packet_buffer(kq,
|
|
pq_packets_size_in_bytes / sizeof(uint32_t),
|
|
&ib_packet_buff);
|
|
if (status) {
|
|
pr_err("kq_acquire_packet_buffer failed\n");
|
|
return status;
|
|
}
|
|
|
|
memset(ib_packet_buff, 0, pq_packets_size_in_bytes);
|
|
|
|
ib_packet = (struct pm4__indirect_buffer_pasid *) (ib_packet_buff);
|
|
|
|
ib_packet->header.count = 3;
|
|
ib_packet->header.opcode = IT_INDIRECT_BUFFER_PASID;
|
|
ib_packet->header.type = PM4_TYPE_3;
|
|
|
|
largep = (union ULARGE_INTEGER *) &vmid0_address;
|
|
|
|
ib_packet->bitfields2.ib_base_lo = largep->u.low_part >> 2;
|
|
ib_packet->bitfields3.ib_base_hi = largep->u.high_part;
|
|
|
|
ib_packet->control = (1 << 23) | (1 << 31) |
|
|
((size_in_bytes / 4) & 0xfffff);
|
|
|
|
ib_packet->bitfields5.pasid = pasid;
|
|
|
|
/*
|
|
* for now we use release mem for GPU-CPU synchronization
|
|
* Consider WaitRegMem + WriteData as a better alternative
|
|
* we get a GART allocations ( gpu/cpu mapping),
|
|
* for the sync variable, and wait until:
|
|
* (a) Sync with HW
|
|
* (b) Sync var is written by CP to mem.
|
|
*/
|
|
rm_packet = (struct pm4__release_mem *) (ib_packet_buff +
|
|
(sizeof(struct pm4__indirect_buffer_pasid) /
|
|
sizeof(unsigned int)));
|
|
|
|
status = kfd_gtt_sa_allocate(dbgdev->dev, sizeof(uint64_t),
|
|
&mem_obj);
|
|
|
|
if (status) {
|
|
pr_err("Failed to allocate GART memory\n");
|
|
kq_rollback_packet(kq);
|
|
return status;
|
|
}
|
|
|
|
rm_state = (uint64_t *) mem_obj->cpu_ptr;
|
|
|
|
*rm_state = QUEUESTATE__ACTIVE_COMPLETION_PENDING;
|
|
|
|
rm_packet->header.opcode = IT_RELEASE_MEM;
|
|
rm_packet->header.type = PM4_TYPE_3;
|
|
rm_packet->header.count = sizeof(struct pm4__release_mem) / 4 - 2;
|
|
|
|
rm_packet->bitfields2.event_type = CACHE_FLUSH_AND_INV_TS_EVENT;
|
|
rm_packet->bitfields2.event_index =
|
|
event_index___release_mem__end_of_pipe;
|
|
|
|
rm_packet->bitfields2.cache_policy = cache_policy___release_mem__lru;
|
|
rm_packet->bitfields2.atc = 0;
|
|
rm_packet->bitfields2.tc_wb_action_ena = 1;
|
|
|
|
addr.quad_part = mem_obj->gpu_addr;
|
|
|
|
rm_packet->bitfields4.address_lo_32b = addr.u.low_part >> 2;
|
|
rm_packet->address_hi = addr.u.high_part;
|
|
|
|
rm_packet->bitfields3.data_sel =
|
|
data_sel___release_mem__send_64_bit_data;
|
|
|
|
rm_packet->bitfields3.int_sel =
|
|
int_sel___release_mem__send_data_after_write_confirm;
|
|
|
|
rm_packet->bitfields3.dst_sel =
|
|
dst_sel___release_mem__memory_controller;
|
|
|
|
rm_packet->data_lo = QUEUESTATE__ACTIVE;
|
|
|
|
kq_submit_packet(kq);
|
|
|
|
/* Wait till CP writes sync code: */
|
|
status = amdkfd_fence_wait_timeout(
|
|
(unsigned int *) rm_state,
|
|
QUEUESTATE__ACTIVE, 1500);
|
|
|
|
kfd_gtt_sa_free(dbgdev->dev, mem_obj);
|
|
|
|
return status;
|
|
}
|
|
|
|
static int dbgdev_register_nodiq(struct kfd_dbgdev *dbgdev)
|
|
{
|
|
/*
|
|
* no action is needed in this case,
|
|
* just make sure diq will not be used
|
|
*/
|
|
|
|
dbgdev->kq = NULL;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int dbgdev_register_diq(struct kfd_dbgdev *dbgdev)
|
|
{
|
|
struct queue_properties properties;
|
|
unsigned int qid;
|
|
struct kernel_queue *kq = NULL;
|
|
int status;
|
|
|
|
properties.type = KFD_QUEUE_TYPE_DIQ;
|
|
|
|
status = pqm_create_queue(dbgdev->pqm, dbgdev->dev, NULL,
|
|
&properties, &qid, NULL);
|
|
|
|
if (status) {
|
|
pr_err("Failed to create DIQ\n");
|
|
return status;
|
|
}
|
|
|
|
pr_debug("DIQ Created with queue id: %d\n", qid);
|
|
|
|
kq = pqm_get_kernel_queue(dbgdev->pqm, qid);
|
|
|
|
if (!kq) {
|
|
pr_err("Error getting DIQ\n");
|
|
pqm_destroy_queue(dbgdev->pqm, qid);
|
|
return -EFAULT;
|
|
}
|
|
|
|
dbgdev->kq = kq;
|
|
|
|
return status;
|
|
}
|
|
|
|
static int dbgdev_unregister_nodiq(struct kfd_dbgdev *dbgdev)
|
|
{
|
|
/* disable watch address */
|
|
dbgdev_address_watch_disable_nodiq(dbgdev->dev);
|
|
return 0;
|
|
}
|
|
|
|
static int dbgdev_unregister_diq(struct kfd_dbgdev *dbgdev)
|
|
{
|
|
/* todo - disable address watch */
|
|
int status;
|
|
|
|
status = pqm_destroy_queue(dbgdev->pqm,
|
|
dbgdev->kq->queue->properties.queue_id);
|
|
dbgdev->kq = NULL;
|
|
|
|
return status;
|
|
}
|
|
|
|
static void dbgdev_address_watch_set_registers(
|
|
const struct dbg_address_watch_info *adw_info,
|
|
union TCP_WATCH_ADDR_H_BITS *addrHi,
|
|
union TCP_WATCH_ADDR_L_BITS *addrLo,
|
|
union TCP_WATCH_CNTL_BITS *cntl,
|
|
unsigned int index, unsigned int vmid)
|
|
{
|
|
union ULARGE_INTEGER addr;
|
|
|
|
addr.quad_part = 0;
|
|
addrHi->u32All = 0;
|
|
addrLo->u32All = 0;
|
|
cntl->u32All = 0;
|
|
|
|
if (adw_info->watch_mask)
|
|
cntl->bitfields.mask =
|
|
(uint32_t) (adw_info->watch_mask[index] &
|
|
ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK);
|
|
else
|
|
cntl->bitfields.mask = ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK;
|
|
|
|
addr.quad_part = (unsigned long long) adw_info->watch_address[index];
|
|
|
|
addrHi->bitfields.addr = addr.u.high_part &
|
|
ADDRESS_WATCH_REG_ADDHIGH_MASK;
|
|
addrLo->bitfields.addr =
|
|
(addr.u.low_part >> ADDRESS_WATCH_REG_ADDLOW_SHIFT);
|
|
|
|
cntl->bitfields.mode = adw_info->watch_mode[index];
|
|
cntl->bitfields.vmid = (uint32_t) vmid;
|
|
/* for now assume it is an ATC address */
|
|
cntl->u32All |= ADDRESS_WATCH_REG_CNTL_ATC_BIT;
|
|
|
|
pr_debug("\t\t%20s %08x\n", "set reg mask :", cntl->bitfields.mask);
|
|
pr_debug("\t\t%20s %08x\n", "set reg add high :",
|
|
addrHi->bitfields.addr);
|
|
pr_debug("\t\t%20s %08x\n", "set reg add low :",
|
|
addrLo->bitfields.addr);
|
|
}
|
|
|
|
static int dbgdev_address_watch_nodiq(struct kfd_dbgdev *dbgdev,
|
|
struct dbg_address_watch_info *adw_info)
|
|
{
|
|
union TCP_WATCH_ADDR_H_BITS addrHi;
|
|
union TCP_WATCH_ADDR_L_BITS addrLo;
|
|
union TCP_WATCH_CNTL_BITS cntl;
|
|
struct kfd_process_device *pdd;
|
|
unsigned int i;
|
|
|
|
/* taking the vmid for that process on the safe way using pdd */
|
|
pdd = kfd_get_process_device_data(dbgdev->dev,
|
|
adw_info->process);
|
|
if (!pdd) {
|
|
pr_err("Failed to get pdd for wave control no DIQ\n");
|
|
return -EFAULT;
|
|
}
|
|
|
|
addrHi.u32All = 0;
|
|
addrLo.u32All = 0;
|
|
cntl.u32All = 0;
|
|
|
|
if ((adw_info->num_watch_points > MAX_WATCH_ADDRESSES) ||
|
|
(adw_info->num_watch_points == 0)) {
|
|
pr_err("num_watch_points is invalid\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (!adw_info->watch_mode || !adw_info->watch_address) {
|
|
pr_err("adw_info fields are not valid\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
for (i = 0; i < adw_info->num_watch_points; i++) {
|
|
dbgdev_address_watch_set_registers(adw_info, &addrHi, &addrLo,
|
|
&cntl, i, pdd->qpd.vmid);
|
|
|
|
pr_debug("\t\t%30s\n", "* * * * * * * * * * * * * * * * * *");
|
|
pr_debug("\t\t%20s %08x\n", "register index :", i);
|
|
pr_debug("\t\t%20s %08x\n", "vmid is :", pdd->qpd.vmid);
|
|
pr_debug("\t\t%20s %08x\n", "Address Low is :",
|
|
addrLo.bitfields.addr);
|
|
pr_debug("\t\t%20s %08x\n", "Address high is :",
|
|
addrHi.bitfields.addr);
|
|
pr_debug("\t\t%20s %08x\n", "Address high is :",
|
|
addrHi.bitfields.addr);
|
|
pr_debug("\t\t%20s %08x\n", "Control Mask is :",
|
|
cntl.bitfields.mask);
|
|
pr_debug("\t\t%20s %08x\n", "Control Mode is :",
|
|
cntl.bitfields.mode);
|
|
pr_debug("\t\t%20s %08x\n", "Control Vmid is :",
|
|
cntl.bitfields.vmid);
|
|
pr_debug("\t\t%20s %08x\n", "Control atc is :",
|
|
cntl.bitfields.atc);
|
|
pr_debug("\t\t%30s\n", "* * * * * * * * * * * * * * * * * *");
|
|
|
|
pdd->dev->kfd2kgd->address_watch_execute(
|
|
dbgdev->dev->kgd,
|
|
i,
|
|
cntl.u32All,
|
|
addrHi.u32All,
|
|
addrLo.u32All);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int dbgdev_address_watch_diq(struct kfd_dbgdev *dbgdev,
|
|
struct dbg_address_watch_info *adw_info)
|
|
{
|
|
struct pm4__set_config_reg *packets_vec;
|
|
union TCP_WATCH_ADDR_H_BITS addrHi;
|
|
union TCP_WATCH_ADDR_L_BITS addrLo;
|
|
union TCP_WATCH_CNTL_BITS cntl;
|
|
struct kfd_mem_obj *mem_obj;
|
|
unsigned int aw_reg_add_dword;
|
|
uint32_t *packet_buff_uint;
|
|
unsigned int i;
|
|
int status;
|
|
size_t ib_size = sizeof(struct pm4__set_config_reg) * 4;
|
|
/* we do not control the vmid in DIQ mode, just a place holder */
|
|
unsigned int vmid = 0;
|
|
|
|
addrHi.u32All = 0;
|
|
addrLo.u32All = 0;
|
|
cntl.u32All = 0;
|
|
|
|
if ((adw_info->num_watch_points > MAX_WATCH_ADDRESSES) ||
|
|
(adw_info->num_watch_points == 0)) {
|
|
pr_err("num_watch_points is invalid\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (!adw_info->watch_mode || !adw_info->watch_address) {
|
|
pr_err("adw_info fields are not valid\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
status = kfd_gtt_sa_allocate(dbgdev->dev, ib_size, &mem_obj);
|
|
|
|
if (status) {
|
|
pr_err("Failed to allocate GART memory\n");
|
|
return status;
|
|
}
|
|
|
|
packet_buff_uint = mem_obj->cpu_ptr;
|
|
|
|
memset(packet_buff_uint, 0, ib_size);
|
|
|
|
packets_vec = (struct pm4__set_config_reg *) (packet_buff_uint);
|
|
|
|
packets_vec[0].header.count = 1;
|
|
packets_vec[0].header.opcode = IT_SET_CONFIG_REG;
|
|
packets_vec[0].header.type = PM4_TYPE_3;
|
|
packets_vec[0].bitfields2.vmid_shift = ADDRESS_WATCH_CNTL_OFFSET;
|
|
packets_vec[0].bitfields2.insert_vmid = 1;
|
|
packets_vec[1].ordinal1 = packets_vec[0].ordinal1;
|
|
packets_vec[1].bitfields2.insert_vmid = 0;
|
|
packets_vec[2].ordinal1 = packets_vec[0].ordinal1;
|
|
packets_vec[2].bitfields2.insert_vmid = 0;
|
|
packets_vec[3].ordinal1 = packets_vec[0].ordinal1;
|
|
packets_vec[3].bitfields2.vmid_shift = ADDRESS_WATCH_CNTL_OFFSET;
|
|
packets_vec[3].bitfields2.insert_vmid = 1;
|
|
|
|
for (i = 0; i < adw_info->num_watch_points; i++) {
|
|
dbgdev_address_watch_set_registers(adw_info,
|
|
&addrHi,
|
|
&addrLo,
|
|
&cntl,
|
|
i,
|
|
vmid);
|
|
|
|
pr_debug("\t\t%30s\n", "* * * * * * * * * * * * * * * * * *");
|
|
pr_debug("\t\t%20s %08x\n", "register index :", i);
|
|
pr_debug("\t\t%20s %08x\n", "vmid is :", vmid);
|
|
pr_debug("\t\t%20s %p\n", "Add ptr is :",
|
|
adw_info->watch_address);
|
|
pr_debug("\t\t%20s %08llx\n", "Add is :",
|
|
adw_info->watch_address[i]);
|
|
pr_debug("\t\t%20s %08x\n", "Address Low is :",
|
|
addrLo.bitfields.addr);
|
|
pr_debug("\t\t%20s %08x\n", "Address high is :",
|
|
addrHi.bitfields.addr);
|
|
pr_debug("\t\t%20s %08x\n", "Control Mask is :",
|
|
cntl.bitfields.mask);
|
|
pr_debug("\t\t%20s %08x\n", "Control Mode is :",
|
|
cntl.bitfields.mode);
|
|
pr_debug("\t\t%20s %08x\n", "Control Vmid is :",
|
|
cntl.bitfields.vmid);
|
|
pr_debug("\t\t%20s %08x\n", "Control atc is :",
|
|
cntl.bitfields.atc);
|
|
pr_debug("\t\t%30s\n", "* * * * * * * * * * * * * * * * * *");
|
|
|
|
aw_reg_add_dword =
|
|
dbgdev->dev->kfd2kgd->address_watch_get_offset(
|
|
dbgdev->dev->kgd,
|
|
i,
|
|
ADDRESS_WATCH_REG_CNTL);
|
|
|
|
packets_vec[0].bitfields2.reg_offset =
|
|
aw_reg_add_dword - AMD_CONFIG_REG_BASE;
|
|
|
|
packets_vec[0].reg_data[0] = cntl.u32All;
|
|
|
|
aw_reg_add_dword =
|
|
dbgdev->dev->kfd2kgd->address_watch_get_offset(
|
|
dbgdev->dev->kgd,
|
|
i,
|
|
ADDRESS_WATCH_REG_ADDR_HI);
|
|
|
|
packets_vec[1].bitfields2.reg_offset =
|
|
aw_reg_add_dword - AMD_CONFIG_REG_BASE;
|
|
packets_vec[1].reg_data[0] = addrHi.u32All;
|
|
|
|
aw_reg_add_dword =
|
|
dbgdev->dev->kfd2kgd->address_watch_get_offset(
|
|
dbgdev->dev->kgd,
|
|
i,
|
|
ADDRESS_WATCH_REG_ADDR_LO);
|
|
|
|
packets_vec[2].bitfields2.reg_offset =
|
|
aw_reg_add_dword - AMD_CONFIG_REG_BASE;
|
|
packets_vec[2].reg_data[0] = addrLo.u32All;
|
|
|
|
/* enable watch flag if address is not zero*/
|
|
if (adw_info->watch_address[i] > 0)
|
|
cntl.bitfields.valid = 1;
|
|
else
|
|
cntl.bitfields.valid = 0;
|
|
|
|
aw_reg_add_dword =
|
|
dbgdev->dev->kfd2kgd->address_watch_get_offset(
|
|
dbgdev->dev->kgd,
|
|
i,
|
|
ADDRESS_WATCH_REG_CNTL);
|
|
|
|
packets_vec[3].bitfields2.reg_offset =
|
|
aw_reg_add_dword - AMD_CONFIG_REG_BASE;
|
|
packets_vec[3].reg_data[0] = cntl.u32All;
|
|
|
|
status = dbgdev_diq_submit_ib(
|
|
dbgdev,
|
|
adw_info->process->pasid,
|
|
mem_obj->gpu_addr,
|
|
packet_buff_uint,
|
|
ib_size);
|
|
|
|
if (status) {
|
|
pr_err("Failed to submit IB to DIQ\n");
|
|
break;
|
|
}
|
|
}
|
|
|
|
kfd_gtt_sa_free(dbgdev->dev, mem_obj);
|
|
return status;
|
|
}
|
|
|
|
static int dbgdev_wave_control_set_registers(
|
|
struct dbg_wave_control_info *wac_info,
|
|
union SQ_CMD_BITS *in_reg_sq_cmd,
|
|
union GRBM_GFX_INDEX_BITS *in_reg_gfx_index)
|
|
{
|
|
int status = 0;
|
|
union SQ_CMD_BITS reg_sq_cmd;
|
|
union GRBM_GFX_INDEX_BITS reg_gfx_index;
|
|
struct HsaDbgWaveMsgAMDGen2 *pMsg;
|
|
|
|
reg_sq_cmd.u32All = 0;
|
|
reg_gfx_index.u32All = 0;
|
|
pMsg = &wac_info->dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2;
|
|
|
|
switch (wac_info->mode) {
|
|
/* Send command to single wave */
|
|
case HSA_DBG_WAVEMODE_SINGLE:
|
|
/*
|
|
* Limit access to the process waves only,
|
|
* by setting vmid check
|
|
*/
|
|
reg_sq_cmd.bits.check_vmid = 1;
|
|
reg_sq_cmd.bits.simd_id = pMsg->ui32.SIMD;
|
|
reg_sq_cmd.bits.wave_id = pMsg->ui32.WaveId;
|
|
reg_sq_cmd.bits.mode = SQ_IND_CMD_MODE_SINGLE;
|
|
|
|
reg_gfx_index.bits.sh_index = pMsg->ui32.ShaderArray;
|
|
reg_gfx_index.bits.se_index = pMsg->ui32.ShaderEngine;
|
|
reg_gfx_index.bits.instance_index = pMsg->ui32.HSACU;
|
|
|
|
break;
|
|
|
|
/* Send command to all waves with matching VMID */
|
|
case HSA_DBG_WAVEMODE_BROADCAST_PROCESS:
|
|
|
|
reg_gfx_index.bits.sh_broadcast_writes = 1;
|
|
reg_gfx_index.bits.se_broadcast_writes = 1;
|
|
reg_gfx_index.bits.instance_broadcast_writes = 1;
|
|
|
|
reg_sq_cmd.bits.mode = SQ_IND_CMD_MODE_BROADCAST;
|
|
|
|
break;
|
|
|
|
/* Send command to all CU waves with matching VMID */
|
|
case HSA_DBG_WAVEMODE_BROADCAST_PROCESS_CU:
|
|
|
|
reg_sq_cmd.bits.check_vmid = 1;
|
|
reg_sq_cmd.bits.mode = SQ_IND_CMD_MODE_BROADCAST;
|
|
|
|
reg_gfx_index.bits.sh_index = pMsg->ui32.ShaderArray;
|
|
reg_gfx_index.bits.se_index = pMsg->ui32.ShaderEngine;
|
|
reg_gfx_index.bits.instance_index = pMsg->ui32.HSACU;
|
|
|
|
break;
|
|
|
|
default:
|
|
return -EINVAL;
|
|
}
|
|
|
|
switch (wac_info->operand) {
|
|
case HSA_DBG_WAVEOP_HALT:
|
|
reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_HALT;
|
|
break;
|
|
|
|
case HSA_DBG_WAVEOP_RESUME:
|
|
reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_RESUME;
|
|
break;
|
|
|
|
case HSA_DBG_WAVEOP_KILL:
|
|
reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_KILL;
|
|
break;
|
|
|
|
case HSA_DBG_WAVEOP_DEBUG:
|
|
reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_DEBUG;
|
|
break;
|
|
|
|
case HSA_DBG_WAVEOP_TRAP:
|
|
if (wac_info->trapId < MAX_TRAPID) {
|
|
reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_TRAP;
|
|
reg_sq_cmd.bits.trap_id = wac_info->trapId;
|
|
} else {
|
|
status = -EINVAL;
|
|
}
|
|
break;
|
|
|
|
default:
|
|
status = -EINVAL;
|
|
break;
|
|
}
|
|
|
|
if (status == 0) {
|
|
*in_reg_sq_cmd = reg_sq_cmd;
|
|
*in_reg_gfx_index = reg_gfx_index;
|
|
}
|
|
|
|
return status;
|
|
}
|
|
|
|
static int dbgdev_wave_control_diq(struct kfd_dbgdev *dbgdev,
|
|
struct dbg_wave_control_info *wac_info)
|
|
{
|
|
|
|
int status;
|
|
union SQ_CMD_BITS reg_sq_cmd;
|
|
union GRBM_GFX_INDEX_BITS reg_gfx_index;
|
|
struct kfd_mem_obj *mem_obj;
|
|
uint32_t *packet_buff_uint;
|
|
struct pm4__set_config_reg *packets_vec;
|
|
size_t ib_size = sizeof(struct pm4__set_config_reg) * 3;
|
|
|
|
reg_sq_cmd.u32All = 0;
|
|
|
|
status = dbgdev_wave_control_set_registers(wac_info, ®_sq_cmd,
|
|
®_gfx_index);
|
|
if (status) {
|
|
pr_err("Failed to set wave control registers\n");
|
|
return status;
|
|
}
|
|
|
|
/* we do not control the VMID in DIQ, so reset it to a known value */
|
|
reg_sq_cmd.bits.vm_id = 0;
|
|
|
|
pr_debug("\t\t %30s\n", "* * * * * * * * * * * * * * * * * *");
|
|
|
|
pr_debug("\t\t mode is: %u\n", wac_info->mode);
|
|
pr_debug("\t\t operand is: %u\n", wac_info->operand);
|
|
pr_debug("\t\t trap id is: %u\n", wac_info->trapId);
|
|
pr_debug("\t\t msg value is: %u\n",
|
|
wac_info->dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value);
|
|
pr_debug("\t\t vmid is: N/A\n");
|
|
|
|
pr_debug("\t\t chk_vmid is : %u\n", reg_sq_cmd.bitfields.check_vmid);
|
|
pr_debug("\t\t command is : %u\n", reg_sq_cmd.bitfields.cmd);
|
|
pr_debug("\t\t queue id is : %u\n", reg_sq_cmd.bitfields.queue_id);
|
|
pr_debug("\t\t simd id is : %u\n", reg_sq_cmd.bitfields.simd_id);
|
|
pr_debug("\t\t mode is : %u\n", reg_sq_cmd.bitfields.mode);
|
|
pr_debug("\t\t vm_id is : %u\n", reg_sq_cmd.bitfields.vm_id);
|
|
pr_debug("\t\t wave_id is : %u\n", reg_sq_cmd.bitfields.wave_id);
|
|
|
|
pr_debug("\t\t ibw is : %u\n",
|
|
reg_gfx_index.bitfields.instance_broadcast_writes);
|
|
pr_debug("\t\t ii is : %u\n",
|
|
reg_gfx_index.bitfields.instance_index);
|
|
pr_debug("\t\t sebw is : %u\n",
|
|
reg_gfx_index.bitfields.se_broadcast_writes);
|
|
pr_debug("\t\t se_ind is : %u\n", reg_gfx_index.bitfields.se_index);
|
|
pr_debug("\t\t sh_ind is : %u\n", reg_gfx_index.bitfields.sh_index);
|
|
pr_debug("\t\t sbw is : %u\n",
|
|
reg_gfx_index.bitfields.sh_broadcast_writes);
|
|
|
|
pr_debug("\t\t %30s\n", "* * * * * * * * * * * * * * * * * *");
|
|
|
|
status = kfd_gtt_sa_allocate(dbgdev->dev, ib_size, &mem_obj);
|
|
|
|
if (status != 0) {
|
|
pr_err("Failed to allocate GART memory\n");
|
|
return status;
|
|
}
|
|
|
|
packet_buff_uint = mem_obj->cpu_ptr;
|
|
|
|
memset(packet_buff_uint, 0, ib_size);
|
|
|
|
packets_vec = (struct pm4__set_config_reg *) packet_buff_uint;
|
|
packets_vec[0].header.count = 1;
|
|
packets_vec[0].header.opcode = IT_SET_UCONFIG_REG;
|
|
packets_vec[0].header.type = PM4_TYPE_3;
|
|
packets_vec[0].bitfields2.reg_offset =
|
|
GRBM_GFX_INDEX / 4 - USERCONFIG_REG_BASE;
|
|
|
|
packets_vec[0].bitfields2.insert_vmid = 0;
|
|
packets_vec[0].reg_data[0] = reg_gfx_index.u32All;
|
|
|
|
packets_vec[1].header.count = 1;
|
|
packets_vec[1].header.opcode = IT_SET_CONFIG_REG;
|
|
packets_vec[1].header.type = PM4_TYPE_3;
|
|
packets_vec[1].bitfields2.reg_offset = SQ_CMD / 4 - AMD_CONFIG_REG_BASE;
|
|
|
|
packets_vec[1].bitfields2.vmid_shift = SQ_CMD_VMID_OFFSET;
|
|
packets_vec[1].bitfields2.insert_vmid = 1;
|
|
packets_vec[1].reg_data[0] = reg_sq_cmd.u32All;
|
|
|
|
/* Restore the GRBM_GFX_INDEX register */
|
|
|
|
reg_gfx_index.u32All = 0;
|
|
reg_gfx_index.bits.sh_broadcast_writes = 1;
|
|
reg_gfx_index.bits.instance_broadcast_writes = 1;
|
|
reg_gfx_index.bits.se_broadcast_writes = 1;
|
|
|
|
|
|
packets_vec[2].ordinal1 = packets_vec[0].ordinal1;
|
|
packets_vec[2].bitfields2.reg_offset =
|
|
GRBM_GFX_INDEX / 4 - USERCONFIG_REG_BASE;
|
|
|
|
packets_vec[2].bitfields2.insert_vmid = 0;
|
|
packets_vec[2].reg_data[0] = reg_gfx_index.u32All;
|
|
|
|
status = dbgdev_diq_submit_ib(
|
|
dbgdev,
|
|
wac_info->process->pasid,
|
|
mem_obj->gpu_addr,
|
|
packet_buff_uint,
|
|
ib_size);
|
|
|
|
if (status)
|
|
pr_err("Failed to submit IB to DIQ\n");
|
|
|
|
kfd_gtt_sa_free(dbgdev->dev, mem_obj);
|
|
|
|
return status;
|
|
}
|
|
|
|
static int dbgdev_wave_control_nodiq(struct kfd_dbgdev *dbgdev,
|
|
struct dbg_wave_control_info *wac_info)
|
|
{
|
|
int status;
|
|
union SQ_CMD_BITS reg_sq_cmd;
|
|
union GRBM_GFX_INDEX_BITS reg_gfx_index;
|
|
struct kfd_process_device *pdd;
|
|
|
|
reg_sq_cmd.u32All = 0;
|
|
|
|
/* taking the VMID for that process on the safe way using PDD */
|
|
pdd = kfd_get_process_device_data(dbgdev->dev, wac_info->process);
|
|
|
|
if (!pdd) {
|
|
pr_err("Failed to get pdd for wave control no DIQ\n");
|
|
return -EFAULT;
|
|
}
|
|
status = dbgdev_wave_control_set_registers(wac_info, ®_sq_cmd,
|
|
®_gfx_index);
|
|
if (status) {
|
|
pr_err("Failed to set wave control registers\n");
|
|
return status;
|
|
}
|
|
|
|
/* for non DIQ we need to patch the VMID: */
|
|
|
|
reg_sq_cmd.bits.vm_id = pdd->qpd.vmid;
|
|
|
|
pr_debug("\t\t %30s\n", "* * * * * * * * * * * * * * * * * *");
|
|
|
|
pr_debug("\t\t mode is: %u\n", wac_info->mode);
|
|
pr_debug("\t\t operand is: %u\n", wac_info->operand);
|
|
pr_debug("\t\t trap id is: %u\n", wac_info->trapId);
|
|
pr_debug("\t\t msg value is: %u\n",
|
|
wac_info->dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value);
|
|
pr_debug("\t\t vmid is: %u\n", pdd->qpd.vmid);
|
|
|
|
pr_debug("\t\t chk_vmid is : %u\n", reg_sq_cmd.bitfields.check_vmid);
|
|
pr_debug("\t\t command is : %u\n", reg_sq_cmd.bitfields.cmd);
|
|
pr_debug("\t\t queue id is : %u\n", reg_sq_cmd.bitfields.queue_id);
|
|
pr_debug("\t\t simd id is : %u\n", reg_sq_cmd.bitfields.simd_id);
|
|
pr_debug("\t\t mode is : %u\n", reg_sq_cmd.bitfields.mode);
|
|
pr_debug("\t\t vm_id is : %u\n", reg_sq_cmd.bitfields.vm_id);
|
|
pr_debug("\t\t wave_id is : %u\n", reg_sq_cmd.bitfields.wave_id);
|
|
|
|
pr_debug("\t\t ibw is : %u\n",
|
|
reg_gfx_index.bitfields.instance_broadcast_writes);
|
|
pr_debug("\t\t ii is : %u\n",
|
|
reg_gfx_index.bitfields.instance_index);
|
|
pr_debug("\t\t sebw is : %u\n",
|
|
reg_gfx_index.bitfields.se_broadcast_writes);
|
|
pr_debug("\t\t se_ind is : %u\n", reg_gfx_index.bitfields.se_index);
|
|
pr_debug("\t\t sh_ind is : %u\n", reg_gfx_index.bitfields.sh_index);
|
|
pr_debug("\t\t sbw is : %u\n",
|
|
reg_gfx_index.bitfields.sh_broadcast_writes);
|
|
|
|
pr_debug("\t\t %30s\n", "* * * * * * * * * * * * * * * * * *");
|
|
|
|
return dbgdev->dev->kfd2kgd->wave_control_execute(dbgdev->dev->kgd,
|
|
reg_gfx_index.u32All,
|
|
reg_sq_cmd.u32All);
|
|
}
|
|
|
|
int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p)
|
|
{
|
|
int status = 0;
|
|
unsigned int vmid;
|
|
uint16_t queried_pasid;
|
|
union SQ_CMD_BITS reg_sq_cmd;
|
|
union GRBM_GFX_INDEX_BITS reg_gfx_index;
|
|
struct kfd_process_device *pdd;
|
|
struct dbg_wave_control_info wac_info;
|
|
int first_vmid_to_scan = dev->vm_info.first_vmid_kfd;
|
|
int last_vmid_to_scan = dev->vm_info.last_vmid_kfd;
|
|
|
|
reg_sq_cmd.u32All = 0;
|
|
status = 0;
|
|
|
|
wac_info.mode = HSA_DBG_WAVEMODE_BROADCAST_PROCESS;
|
|
wac_info.operand = HSA_DBG_WAVEOP_KILL;
|
|
|
|
pr_debug("Killing all process wavefronts\n");
|
|
|
|
/* Scan all registers in the range ATC_VMID8_PASID_MAPPING ..
|
|
* ATC_VMID15_PASID_MAPPING
|
|
* to check which VMID the current process is mapped to.
|
|
*/
|
|
|
|
for (vmid = first_vmid_to_scan; vmid <= last_vmid_to_scan; vmid++) {
|
|
status = dev->kfd2kgd->get_atc_vmid_pasid_mapping_info
|
|
(dev->kgd, vmid, &queried_pasid);
|
|
|
|
if (status && queried_pasid == p->pasid) {
|
|
pr_debug("Killing wave fronts of vmid %d and pasid 0x%x\n",
|
|
vmid, p->pasid);
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (vmid > last_vmid_to_scan) {
|
|
pr_err("Didn't find vmid for pasid 0x%x\n", p->pasid);
|
|
return -EFAULT;
|
|
}
|
|
|
|
/* taking the VMID for that process on the safe way using PDD */
|
|
pdd = kfd_get_process_device_data(dev, p);
|
|
if (!pdd)
|
|
return -EFAULT;
|
|
|
|
status = dbgdev_wave_control_set_registers(&wac_info, ®_sq_cmd,
|
|
®_gfx_index);
|
|
if (status != 0)
|
|
return -EINVAL;
|
|
|
|
/* for non DIQ we need to patch the VMID: */
|
|
reg_sq_cmd.bits.vm_id = vmid;
|
|
|
|
dev->kfd2kgd->wave_control_execute(dev->kgd,
|
|
reg_gfx_index.u32All,
|
|
reg_sq_cmd.u32All);
|
|
|
|
return 0;
|
|
}
|
|
|
|
void kfd_dbgdev_init(struct kfd_dbgdev *pdbgdev, struct kfd_dev *pdev,
|
|
enum DBGDEV_TYPE type)
|
|
{
|
|
pdbgdev->dev = pdev;
|
|
pdbgdev->kq = NULL;
|
|
pdbgdev->type = type;
|
|
pdbgdev->pqm = NULL;
|
|
|
|
switch (type) {
|
|
case DBGDEV_TYPE_NODIQ:
|
|
pdbgdev->dbgdev_register = dbgdev_register_nodiq;
|
|
pdbgdev->dbgdev_unregister = dbgdev_unregister_nodiq;
|
|
pdbgdev->dbgdev_wave_control = dbgdev_wave_control_nodiq;
|
|
pdbgdev->dbgdev_address_watch = dbgdev_address_watch_nodiq;
|
|
break;
|
|
case DBGDEV_TYPE_DIQ:
|
|
default:
|
|
pdbgdev->dbgdev_register = dbgdev_register_diq;
|
|
pdbgdev->dbgdev_unregister = dbgdev_unregister_diq;
|
|
pdbgdev->dbgdev_wave_control = dbgdev_wave_control_diq;
|
|
pdbgdev->dbgdev_address_watch = dbgdev_address_watch_diq;
|
|
break;
|
|
}
|
|
|
|
}
|