drm/nouveau/imem/nv50: embed nvkm_instobj directly into nv04_instobj

This is not as simple as it was for earlier GPUs, due to the need to swap
accessor functions depending on whether BAR2 is usable or not.

We were previously protected by nvkm_instobj's accessor functions keeping
an object mapped permanently, with some unclear magic that managed to hit
the slow-path where needed even if an object was marked as mapped.

That's been replaced here by reference counting maps (some objects, like
page tables can be accessed concurrently), and swapping the functions as
necessary.

Signed-off-by: Ben Skeggs <bskeggs@redhat.com>
This commit is contained in:
Ben Skeggs 2017-11-01 03:56:19 +10:00
parent af515ec8d3
commit be55287aa5
1 changed files with 101 additions and 31 deletions

View File

@ -37,13 +37,14 @@ struct nv50_instmem {
/******************************************************************************
* instmem object implementation
*****************************************************************************/
#define nv50_instobj(p) container_of((p), struct nv50_instobj, memory)
#define nv50_instobj(p) container_of((p), struct nv50_instobj, base.memory)
struct nv50_instobj {
struct nvkm_memory memory;
struct nvkm_instobj base;
struct nv50_instmem *imem;
struct nvkm_mem *mem;
struct nvkm_vma bar;
refcount_t maps;
void *map;
};
@ -93,31 +94,59 @@ nv50_instobj_slow = {
.wr32 = nv50_instobj_wr32_slow,
};
static void
nv50_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data)
{
iowrite32_native(data, nv50_instobj(memory)->map + offset);
}
static u32
nv50_instobj_rd32(struct nvkm_memory *memory, u64 offset)
{
return ioread32_native(nv50_instobj(memory)->map + offset);
}
static const struct nvkm_memory_ptrs
nv50_instobj_fast = {
.rd32 = nv50_instobj_rd32,
.wr32 = nv50_instobj_wr32,
};
static void
nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm)
{
struct nvkm_memory *memory = &iobj->memory;
struct nvkm_subdev *subdev = &iobj->imem->base.subdev;
struct nv50_instmem *imem = iobj->imem;
struct nvkm_memory *memory = &iobj->base.memory;
struct nvkm_subdev *subdev = &imem->base.subdev;
struct nvkm_device *device = subdev->device;
struct nvkm_vma bar = {};
u64 size = nvkm_memory_size(memory);
void __iomem *map;
int ret;
iobj->map = ERR_PTR(-ENOMEM);
/* Attempt to allocate BAR2 address-space and map the object
* into it. The lock has to be dropped while doing this due
* to the possibility of recursion for page table allocation.
*/
mutex_unlock(&subdev->mutex);
ret = nvkm_vm_get(vmm, size, 12, NV_MEM_ACCESS_RW, &bar);
if (ret == 0)
nvkm_memory_map(memory, &bar, 0);
mutex_lock(&subdev->mutex);
if (ret || iobj->bar.node) {
/* We either failed, or another thread beat us. */
mutex_unlock(&subdev->mutex);
nvkm_vm_put(&bar);
mutex_lock(&subdev->mutex);
return;
}
ret = nvkm_vm_get(vmm, size, 12, NV_MEM_ACCESS_RW, &iobj->bar);
if (ret == 0) {
map = ioremap(device->func->resource_addr(device, 3) +
(u32)iobj->bar.offset, size);
if (map) {
nvkm_memory_map(memory, &iobj->bar, 0);
iobj->map = map;
} else {
nvkm_warn(subdev, "PRAMIN ioremap failed\n");
nvkm_vm_put(&iobj->bar);
}
} else {
nvkm_warn(subdev, "PRAMIN exhausted\n");
/* Make the mapping visible to the host. */
iobj->bar = bar;
iobj->map = ioremap(device->func->resource_addr(device, 3) +
(u32)iobj->bar.offset, size);
if (!iobj->map) {
nvkm_warn(subdev, "PRAMIN ioremap failed\n");
nvkm_vm_put(&iobj->bar);
}
}
@ -131,28 +160,66 @@ nv50_instobj_map(struct nvkm_memory *memory, struct nvkm_vma *vma, u64 offset)
static void
nv50_instobj_release(struct nvkm_memory *memory)
{
struct nv50_instobj *iobj = nv50_instobj(memory);
struct nv50_instmem *imem = iobj->imem;
struct nvkm_subdev *subdev = &imem->base.subdev;
nvkm_bar_flush(subdev->device->bar);
if (refcount_dec_and_mutex_lock(&iobj->maps, &subdev->mutex)) {
/* Switch back to NULL accessors when last map is gone. */
iobj->base.memory.ptrs = &nv50_instobj_slow;
mutex_unlock(&subdev->mutex);
}
}
static void __iomem *
nv50_instobj_acquire(struct nvkm_memory *memory)
{
struct nv50_instobj *iobj = nv50_instobj(memory);
struct nv50_instmem *imem = iobj->imem;
struct nvkm_vm *vm;
struct nvkm_instmem *imem = &iobj->imem->base;
struct nvkm_vmm *vmm;
void __iomem *map = NULL;
if (!iobj->map && (vm = nvkm_bar_bar2_vmm(imem->base.subdev.device)))
nv50_instobj_kmap(iobj, vm);
if (!IS_ERR_OR_NULL(iobj->map))
/* Already mapped? */
if (refcount_inc_not_zero(&iobj->maps))
return iobj->map;
return NULL;
/* Take the lock, and re-check that another thread hasn't
* already mapped the object in the meantime.
*/
mutex_lock(&imem->subdev.mutex);
if (refcount_inc_not_zero(&iobj->maps)) {
mutex_unlock(&imem->subdev.mutex);
return iobj->map;
}
/* Attempt to get a direct CPU mapping of the object. */
if (!iobj->map && (vmm = nvkm_bar_bar2_vmm(imem->subdev.device)))
nv50_instobj_kmap(iobj, vmm);
map = iobj->map;
if (!refcount_inc_not_zero(&iobj->maps)) {
if (map)
iobj->base.memory.ptrs = &nv50_instobj_fast;
else
iobj->base.memory.ptrs = &nv50_instobj_slow;
refcount_inc(&iobj->maps);
}
mutex_unlock(&imem->subdev.mutex);
return map;
}
static void
nv50_instobj_boot(struct nvkm_memory *memory, struct nvkm_vmm *vmm)
{
struct nv50_instobj *iobj = nv50_instobj(memory);
struct nvkm_instmem *imem = &iobj->imem->base;
mutex_lock(&imem->subdev.mutex);
nv50_instobj_kmap(iobj, vmm);
mutex_unlock(&imem->subdev.mutex);
}
static u64
@ -177,12 +244,14 @@ static void *
nv50_instobj_dtor(struct nvkm_memory *memory)
{
struct nv50_instobj *iobj = nv50_instobj(memory);
struct nvkm_ram *ram = iobj->imem->base.subdev.device->fb->ram;
if (!IS_ERR_OR_NULL(iobj->map)) {
struct nvkm_instmem *imem = &iobj->imem->base;
struct nvkm_ram *ram = imem->subdev.device->fb->ram;
if (iobj->map) {
iounmap(iobj->map);
nvkm_vm_put(&iobj->bar);
}
ram->func->put(ram, &iobj->mem);
nvkm_instobj_dtor(imem, &iobj->base);
return iobj;
}
@ -209,11 +278,12 @@ nv50_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero,
if (!(iobj = kzalloc(sizeof(*iobj), GFP_KERNEL)))
return -ENOMEM;
*pmemory = &iobj->memory;
*pmemory = &iobj->base.memory;
nvkm_memory_ctor(&nv50_instobj_func, &iobj->memory);
iobj->memory.ptrs = &nv50_instobj_slow;
nvkm_instobj_ctor(&nv50_instobj_func, &imem->base, &iobj->base);
iobj->base.memory.ptrs = &nv50_instobj_slow;
iobj->imem = imem;
refcount_set(&iobj->maps, 0);
size = max((size + 4095) & ~4095, (u32)4096);
align = max((align + 4095) & ~4095, (u32)4096);
@ -240,7 +310,7 @@ static const struct nvkm_instmem_func
nv50_instmem = {
.fini = nv50_instmem_fini,
.memory_new = nv50_instobj_new,
.persistent = false,
.persistent = true,
.zero = false,
};