linux-stable/drivers/gpu/drm/xe/xe_gt_mcr.c

553 lines
17 KiB
C

// SPDX-License-Identifier: MIT
/*
* Copyright © 2022 Intel Corporation
*/
#include "xe_gt.h"
#include "xe_gt_mcr.h"
#include "xe_gt_topology.h"
#include "xe_gt_types.h"
#include "xe_mmio.h"
#include "gt/intel_gt_regs.h"
/**
* DOC: GT Multicast/Replicated (MCR) Register Support
*
* Some GT registers are designed as "multicast" or "replicated" registers:
* multiple instances of the same register share a single MMIO offset. MCR
* registers are generally used when the hardware needs to potentially track
* independent values of a register per hardware unit (e.g., per-subslice,
* per-L3bank, etc.). The specific types of replication that exist vary
* per-platform.
*
* MMIO accesses to MCR registers are controlled according to the settings
* programmed in the platform's MCR_SELECTOR register(s). MMIO writes to MCR
* registers can be done in either a (i.e., a single write updates all
* instances of the register to the same value) or unicast (a write updates only
* one specific instance). Reads of MCR registers always operate in a unicast
* manner regardless of how the multicast/unicast bit is set in MCR_SELECTOR.
* Selection of a specific MCR instance for unicast operations is referred to
* as "steering."
*
* If MCR register operations are steered toward a hardware unit that is
* fused off or currently powered down due to power gating, the MMIO operation
* is "terminated" by the hardware. Terminated read operations will return a
* value of zero and terminated unicast write operations will be silently
* ignored.
*/
enum {
MCR_OP_READ,
MCR_OP_WRITE
};
static const struct xe_mmio_range xelp_l3bank_steering_table[] = {
{ 0x00B100, 0x00B3FF },
{},
};
/*
* Although the bspec lists more "MSLICE" ranges than shown here, some of those
* are of a "GAM" subclass that has special rules and doesn't need to be
* included here.
*/
static const struct xe_mmio_range xehp_mslice_steering_table[] = {
{ 0x00DD00, 0x00DDFF },
{ 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */
{},
};
static const struct xe_mmio_range xehp_lncf_steering_table[] = {
{ 0x00B000, 0x00B0FF },
{ 0x00D880, 0x00D8FF },
{},
};
/*
* We have several types of MCR registers where steering to (0,0) will always
* provide us with a non-terminated value. We'll stick them all in the same
* table for simplicity.
*/
static const struct xe_mmio_range xehpc_instance0_steering_table[] = {
{ 0x004000, 0x004AFF }, /* HALF-BSLICE */
{ 0x008800, 0x00887F }, /* CC */
{ 0x008A80, 0x008AFF }, /* TILEPSMI */
{ 0x00B000, 0x00B0FF }, /* HALF-BSLICE */
{ 0x00B100, 0x00B3FF }, /* L3BANK */
{ 0x00C800, 0x00CFFF }, /* HALF-BSLICE */
{ 0x00D800, 0x00D8FF }, /* HALF-BSLICE */
{ 0x00DD00, 0x00DDFF }, /* BSLICE */
{ 0x00E900, 0x00E9FF }, /* HALF-BSLICE */
{ 0x00EC00, 0x00EEFF }, /* HALF-BSLICE */
{ 0x00F000, 0x00FFFF }, /* HALF-BSLICE */
{ 0x024180, 0x0241FF }, /* HALF-BSLICE */
{},
};
static const struct xe_mmio_range xelpg_instance0_steering_table[] = {
{ 0x000B00, 0x000BFF }, /* SQIDI */
{ 0x001000, 0x001FFF }, /* SQIDI */
{ 0x004000, 0x0048FF }, /* GAM */
{ 0x008700, 0x0087FF }, /* SQIDI */
{ 0x00B000, 0x00B0FF }, /* NODE */
{ 0x00C800, 0x00CFFF }, /* GAM */
{ 0x00D880, 0x00D8FF }, /* NODE */
{ 0x00DD00, 0x00DDFF }, /* OAAL2 */
{},
};
static const struct xe_mmio_range xelpg_l3bank_steering_table[] = {
{ 0x00B100, 0x00B3FF },
{},
};
static const struct xe_mmio_range xelp_dss_steering_table[] = {
{ 0x008150, 0x00815F },
{ 0x009520, 0x00955F },
{ 0x00DE80, 0x00E8FF },
{ 0x024A00, 0x024A7F },
{},
};
/* DSS steering is used for GSLICE ranges as well */
static const struct xe_mmio_range xehp_dss_steering_table[] = {
{ 0x005200, 0x0052FF }, /* GSLICE */
{ 0x005400, 0x007FFF }, /* GSLICE */
{ 0x008140, 0x00815F }, /* GSLICE (0x8140-0x814F), DSS (0x8150-0x815F) */
{ 0x008D00, 0x008DFF }, /* DSS */
{ 0x0094D0, 0x00955F }, /* GSLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */
{ 0x009680, 0x0096FF }, /* DSS */
{ 0x00D800, 0x00D87F }, /* GSLICE */
{ 0x00DC00, 0x00DCFF }, /* GSLICE */
{ 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved ) */
{ 0x017000, 0x017FFF }, /* GSLICE */
{ 0x024A00, 0x024A7F }, /* DSS */
{},
};
/* DSS steering is used for COMPUTE ranges as well */
static const struct xe_mmio_range xehpc_dss_steering_table[] = {
{ 0x008140, 0x00817F }, /* COMPUTE (0x8140-0x814F & 0x8160-0x817F), DSS (0x8150-0x815F) */
{ 0x0094D0, 0x00955F }, /* COMPUTE (0x94D0-0x951F), DSS (0x9520-0x955F) */
{ 0x009680, 0x0096FF }, /* DSS */
{ 0x00DC00, 0x00DCFF }, /* COMPUTE */
{ 0x00DE80, 0x00E7FF }, /* DSS (0xDF00-0xE1FF reserved ) */
{},
};
/* DSS steering is used for SLICE ranges as well */
static const struct xe_mmio_range xelpg_dss_steering_table[] = {
{ 0x005200, 0x0052FF }, /* SLICE */
{ 0x005500, 0x007FFF }, /* SLICE */
{ 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */
{ 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */
{ 0x009680, 0x0096FF }, /* DSS */
{ 0x00D800, 0x00D87F }, /* SLICE */
{ 0x00DC00, 0x00DCFF }, /* SLICE */
{ 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */
{},
};
static const struct xe_mmio_range xelpmp_oaddrm_steering_table[] = {
{ 0x393200, 0x39323F },
{ 0x393400, 0x3934FF },
{},
};
/*
* DG2 GAM registers are a special case; this table is checked directly in
* xe_gt_mcr_get_nonterminated_steering and is not hooked up via
* gt->steering[].
*/
static const struct xe_mmio_range dg2_gam_ranges[] = {
{ 0x004000, 0x004AFF },
{ 0x00C800, 0x00CFFF },
{ 0x00F000, 0x00FFFF },
{},
};
static void init_steering_l3bank(struct xe_gt *gt)
{
if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) {
u32 mslice_mask = REG_FIELD_GET(GEN12_MEML3_EN_MASK,
xe_mmio_read32(gt, GEN10_MIRROR_FUSE3.reg));
u32 bank_mask = REG_FIELD_GET(GT_L3_EXC_MASK,
xe_mmio_read32(gt, XEHP_FUSE4.reg));
/*
* Group selects mslice, instance selects bank within mslice.
* Bank 0 is always valid _except_ when the bank mask is 010b.
*/
gt->steering[L3BANK].group_target = __ffs(mslice_mask);
gt->steering[L3BANK].instance_target =
bank_mask & BIT(0) ? 0 : 2;
} else {
u32 fuse = REG_FIELD_GET(GEN10_L3BANK_MASK,
~xe_mmio_read32(gt, GEN10_MIRROR_FUSE3.reg));
gt->steering[L3BANK].group_target = 0; /* unused */
gt->steering[L3BANK].instance_target = __ffs(fuse);
}
}
static void init_steering_mslice(struct xe_gt *gt)
{
u32 mask = REG_FIELD_GET(GEN12_MEML3_EN_MASK,
xe_mmio_read32(gt, GEN10_MIRROR_FUSE3.reg));
/*
* mslice registers are valid (not terminated) if either the meml3
* associated with the mslice is present, or at least one DSS associated
* with the mslice is present. There will always be at least one meml3
* so we can just use that to find a non-terminated mslice and ignore
* the DSS fusing.
*/
gt->steering[MSLICE].group_target = __ffs(mask);
gt->steering[MSLICE].instance_target = 0; /* unused */
/*
* LNCF termination is also based on mslice presence, so we'll set
* it up here. Either LNCF within a non-terminated mslice will work,
* so we just always pick LNCF 0 here.
*/
gt->steering[LNCF].group_target = __ffs(mask) << 1;
gt->steering[LNCF].instance_target = 0; /* unused */
}
static void init_steering_dss(struct xe_gt *gt)
{
unsigned int dss = min(xe_dss_mask_group_ffs(gt->fuse_topo.g_dss_mask, 0, 0),
xe_dss_mask_group_ffs(gt->fuse_topo.c_dss_mask, 0, 0));
unsigned int dss_per_grp = gt_to_xe(gt)->info.platform == XE_PVC ? 8 : 4;
gt->steering[DSS].group_target = dss / dss_per_grp;
gt->steering[DSS].instance_target = dss % dss_per_grp;
}
static void init_steering_oaddrm(struct xe_gt *gt)
{
/*
* First instance is only terminated if the entire first media slice
* is absent (i.e., no VCS0 or VECS0).
*/
if (gt->info.engine_mask & (XE_HW_ENGINE_VCS0 | XE_HW_ENGINE_VECS0))
gt->steering[OADDRM].group_target = 0;
else
gt->steering[OADDRM].group_target = 1;
gt->steering[DSS].instance_target = 0; /* unused */
}
static void init_steering_inst0(struct xe_gt *gt)
{
gt->steering[DSS].group_target = 0; /* unused */
gt->steering[DSS].instance_target = 0; /* unused */
}
static const struct {
const char *name;
void (*init)(struct xe_gt *);
} xe_steering_types[] = {
{ "L3BANK", init_steering_l3bank },
{ "MSLICE", init_steering_mslice },
{ "LNCF", NULL }, /* initialized by mslice init */
{ "DSS", init_steering_dss },
{ "OADDRM", init_steering_oaddrm },
{ "INSTANCE 0", init_steering_inst0 },
};
void xe_gt_mcr_init(struct xe_gt *gt)
{
struct xe_device *xe = gt_to_xe(gt);
BUILD_BUG_ON(ARRAY_SIZE(xe_steering_types) != NUM_STEERING_TYPES);
spin_lock_init(&gt->mcr_lock);
if (gt->info.type == XE_GT_TYPE_MEDIA) {
drm_WARN_ON(&xe->drm, MEDIA_VER(xe) < 13);
gt->steering[OADDRM].ranges = xelpmp_oaddrm_steering_table;
} else if (GRAPHICS_VERx100(xe) >= 1270) {
gt->steering[INSTANCE0].ranges = xelpg_instance0_steering_table;
gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table;
gt->steering[DSS].ranges = xelpg_dss_steering_table;
} else if (xe->info.platform == XE_PVC) {
gt->steering[INSTANCE0].ranges = xehpc_instance0_steering_table;
gt->steering[DSS].ranges = xehpc_dss_steering_table;
} else if (xe->info.platform == XE_DG2) {
gt->steering[MSLICE].ranges = xehp_mslice_steering_table;
gt->steering[LNCF].ranges = xehp_lncf_steering_table;
gt->steering[DSS].ranges = xehp_dss_steering_table;
} else {
gt->steering[L3BANK].ranges = xelp_l3bank_steering_table;
gt->steering[DSS].ranges = xelp_dss_steering_table;
}
/* Select non-terminated steering target for each type */
for (int i = 0; i < NUM_STEERING_TYPES; i++)
if (gt->steering[i].ranges && xe_steering_types[i].init)
xe_steering_types[i].init(gt);
}
/*
* xe_gt_mcr_get_nonterminated_steering - find group/instance values that
* will steer a register to a non-terminated instance
* @gt: GT structure
* @reg: register for which the steering is required
* @group: return variable for group steering
* @instance: return variable for instance steering
*
* This function returns a group/instance pair that is guaranteed to work for
* read steering of the given register. Note that a value will be returned even
* if the register is not replicated and therefore does not actually require
* steering.
*
* Returns true if the caller should steer to the @group/@instance values
* returned. Returns false if the caller need not perform any steering (i.e.,
* the DG2 GAM range special case).
*/
static bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt,
i915_mcr_reg_t reg,
u8 *group, u8 *instance)
{
for (int type = 0; type < NUM_STEERING_TYPES; type++) {
if (!gt->steering[type].ranges)
continue;
for (int i = 0; gt->steering[type].ranges[i].end > 0; i++) {
if (xe_mmio_in_range(&gt->steering[type].ranges[i], reg.reg)) {
*group = gt->steering[type].group_target;
*instance = gt->steering[type].instance_target;
return true;
}
}
}
/*
* All MCR registers should usually be part of one of the steering
* ranges we're tracking. However there's one special case: DG2
* GAM registers are technically multicast registers, but are special
* in a number of ways:
* - they have their own dedicated steering control register (they
* don't share 0xFDC with other MCR classes)
* - all reads should be directed to instance 1 (unicast reads against
* other instances are not allowed), and instance 1 is already the
* the hardware's default steering target, which we never change
*
* Ultimately this means that we can just treat them as if they were
* unicast registers and all operations will work properly.
*/
for (int i = 0; dg2_gam_ranges[i].end > 0; i++)
if (xe_mmio_in_range(&dg2_gam_ranges[i], reg.reg))
return false;
/*
* Not found in a steering table and not a DG2 GAM register? We'll
* just steer to 0/0 as a guess and raise a warning.
*/
drm_WARN(&gt_to_xe(gt)->drm, true,
"Did not find MCR register %#x in any MCR steering table\n",
reg.reg);
*group = 0;
*instance = 0;
return true;
}
#define STEER_SEMAPHORE 0xFD0
/*
* Obtain exclusive access to MCR steering. On MTL and beyond we also need
* to synchronize with external clients (e.g., firmware), so a semaphore
* register will also need to be taken.
*/
static void mcr_lock(struct xe_gt *gt)
{
struct xe_device *xe = gt_to_xe(gt);
int ret;
spin_lock(&gt->mcr_lock);
/*
* Starting with MTL we also need to grab a semaphore register
* to synchronize with external agents (e.g., firmware) that now
* shares the same steering control register.
*/
if (GRAPHICS_VERx100(xe) >= 1270)
ret = wait_for_us(xe_mmio_read32(gt, STEER_SEMAPHORE) == 0x1, 10);
drm_WARN_ON_ONCE(&xe->drm, ret == -ETIMEDOUT);
}
static void mcr_unlock(struct xe_gt *gt) {
/* Release hardware semaphore */
if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270)
xe_mmio_write32(gt, STEER_SEMAPHORE, 0x1);
spin_unlock(&gt->mcr_lock);
}
/*
* Access a register with specific MCR steering
*
* Caller needs to make sure the relevant forcewake wells are up.
*/
static u32 rw_with_mcr_steering(struct xe_gt *gt, i915_mcr_reg_t reg, u8 rw_flag,
int group, int instance, u32 value)
{
u32 steer_reg, steer_val, val = 0;
lockdep_assert_held(&gt->mcr_lock);
if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) {
steer_reg = MTL_MCR_SELECTOR.reg;
steer_val = REG_FIELD_PREP(MTL_MCR_GROUPID, group) |
REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance);
} else {
steer_reg = GEN8_MCR_SELECTOR.reg;
steer_val = REG_FIELD_PREP(GEN11_MCR_SLICE_MASK, group) |
REG_FIELD_PREP(GEN11_MCR_SUBSLICE_MASK, instance);
}
/*
* Always leave the hardware in multicast mode when doing reads
* (see comment about Wa_22013088509 below) and only change it
* to unicast mode when doing writes of a specific instance.
*
* No need to save old steering reg value.
*/
if (rw_flag == MCR_OP_READ)
steer_val |= GEN11_MCR_MULTICAST;
xe_mmio_write32(gt, steer_reg, steer_val);
if (rw_flag == MCR_OP_READ)
val = xe_mmio_read32(gt, reg.reg);
else
xe_mmio_write32(gt, reg.reg, value);
/*
* If we turned off the multicast bit (during a write) we're required
* to turn it back on before finishing. The group and instance values
* don't matter since they'll be re-programmed on the next MCR
* operation.
*/
if (rw_flag == MCR_OP_WRITE)
xe_mmio_write32(gt, steer_reg, GEN11_MCR_MULTICAST);
return val;
}
/**
* xe_gt_mcr_unicast_read_any - reads a non-terminated instance of an MCR register
* @gt: GT structure
* @reg: register to read
*
* Reads a GT MCR register. The read will be steered to a non-terminated
* instance (i.e., one that isn't fused off or powered down by power gating).
* This function assumes the caller is already holding any necessary forcewake
* domains.
*
* Returns the value from a non-terminated instance of @reg.
*/
u32 xe_gt_mcr_unicast_read_any(struct xe_gt *gt, i915_mcr_reg_t reg)
{
u8 group, instance;
u32 val;
bool steer;
steer = xe_gt_mcr_get_nonterminated_steering(gt, reg, &group, &instance);
if (steer) {
mcr_lock(gt);
val = rw_with_mcr_steering(gt, reg, MCR_OP_READ,
group, instance, 0);
mcr_unlock(gt);
} else {
/* DG2 GAM special case rules; treat as if unicast */
val = xe_mmio_read32(gt, reg.reg);
}
return val;
}
/**
* xe_gt_mcr_unicast_read - read a specific instance of an MCR register
* @gt: GT structure
* @reg: the MCR register to read
* @group: the MCR group
* @instance: the MCR instance
*
* Returns the value read from an MCR register after steering toward a specific
* group/instance.
*/
u32 xe_gt_mcr_unicast_read(struct xe_gt *gt,
i915_mcr_reg_t reg,
int group, int instance)
{
u32 val;
mcr_lock(gt);
val = rw_with_mcr_steering(gt, reg, MCR_OP_READ, group, instance, 0);
mcr_unlock(gt);
return val;
}
/**
* xe_gt_mcr_unicast_write - write a specific instance of an MCR register
* @gt: GT structure
* @reg: the MCR register to write
* @value: value to write
* @group: the MCR group
* @instance: the MCR instance
*
* Write an MCR register in unicast mode after steering toward a specific
* group/instance.
*/
void xe_gt_mcr_unicast_write(struct xe_gt *gt, i915_mcr_reg_t reg, u32 value,
int group, int instance)
{
mcr_lock(gt);
rw_with_mcr_steering(gt, reg, MCR_OP_WRITE, group, instance, value);
mcr_unlock(gt);
}
/**
* xe_gt_mcr_multicast_write - write a value to all instances of an MCR register
* @gt: GT structure
* @reg: the MCR register to write
* @value: value to write
*
* Write an MCR register in multicast mode to update all instances.
*/
void xe_gt_mcr_multicast_write(struct xe_gt *gt, i915_mcr_reg_t reg, u32 value)
{
/*
* Synchronize with any unicast operations. Once we have exclusive
* access, the MULTICAST bit should already be set, so there's no need
* to touch the steering register.
*/
mcr_lock(gt);
xe_mmio_write32(gt, reg.reg, value);
mcr_unlock(gt);
}
void xe_gt_mcr_steering_dump(struct xe_gt *gt, struct drm_printer *p)
{
for (int i = 0; i < NUM_STEERING_TYPES; i++) {
if (gt->steering[i].ranges) {
drm_printf(p, "%s steering: group=%#x, instance=%#x\n",
xe_steering_types[i].name,
gt->steering[i].group_target,
gt->steering[i].instance_target);
for (int j = 0; gt->steering[i].ranges[j].end; j++)
drm_printf(p, "\t0x%06x - 0x%06x\n",
gt->steering[i].ranges[j].start,
gt->steering[i].ranges[j].end);
}
}
}