diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/client.h b/drivers/gpu/drm/nouveau/include/nvkm/core/client.h
index 473d66b22aeb..f79f9025d1c5 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/core/client.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/core/client.h
@@ -5,7 +5,7 @@
 struct nvkm_client {
 	struct nvkm_namedb namedb;
 	struct nvkm_handle *root;
-	struct nvkm_device *device;
+	u64 device;
 	char name[32];
 	u32 debug;
 	struct nvkm_vm *vm;
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c
index 0c293fe410bf..86f567d4e5cd 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.c
@@ -507,7 +507,7 @@ nouveau_drm_device_remove(struct drm_device *dev)
 
 	dev->irq_enabled = false;
 	client = nvxx_client(&drm->client.base);
-	device = client->device;
+	device = nvkm_device_find(client->device);
 	drm_put_dev(dev);
 
 	nvkm_device_del(&device);
diff --git a/drivers/gpu/drm/nouveau/nvkm/core/client.c b/drivers/gpu/drm/nouveau/nvkm/core/client.c
index 5a1ddb3433de..7615cdd75294 100644
--- a/drivers/gpu/drm/nouveau/nvkm/core/client.c
+++ b/drivers/gpu/drm/nouveau/nvkm/core/client.c
@@ -219,7 +219,6 @@ nvkm_client_del(struct nvkm_client **pclient)
 		nvkm_client_fini(client, false);
 		for (i = 0; i < ARRAY_SIZE(client->notify); i++)
 			nvkm_client_notify_del(client, i);
-		nvkm_object_ref(NULL, (struct nvkm_object **)&client->device);
 		nvkm_handle_destroy(client->root);
 		nvkm_namedb_destroy(&client->namedb);
 		*pclient = NULL;
@@ -233,17 +232,12 @@ nvkm_client_sclass[] = {
 };
 
 int
-nvkm_client_new(const char *name, u64 devname, const char *cfg,
+nvkm_client_new(const char *name, u64 device, const char *cfg,
 		const char *dbg, struct nvkm_client **pclient)
 {
-	struct nvkm_device *device;
 	struct nvkm_client *client;
 	int ret;
 
-	device = nvkm_device_find(devname);
-	if (!device)
-		return -ENODEV;
-
 	ret = nvkm_namedb_create(NULL, NULL, &nvkm_client_oclass,
 				 NV_CLIENT_CLASS, nvkm_client_sclass,
 				 0, &client);
@@ -259,8 +253,7 @@ nvkm_client_new(const char *name, u64 devname, const char *cfg,
 	/* prevent init/fini being called, os in in charge of this */
 	atomic_set(&nv_object(client)->usecount, 2);
 
-	nvkm_object_ref(&device->engine.subdev.object,
-			(struct nvkm_object **)&client->device);
+	client->device = device;
 	snprintf(client->name, sizeof(client->name), "%s", name);
 	client->debug = nvkm_dbgopt(dbg, "CLIENT");
 	return 0;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c
index 4867bbd0788f..c16bde41c279 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c
@@ -258,12 +258,12 @@ nvkm_udevice_ctor(struct nvkm_object *parent, struct nvkm_object *engine,
 		oclass = &nvkm_udevice_oclass_super;
 
 	/* find the device subdev that matches what the client requested */
-	device = client->device;
-	if (args->v0.device != ~0) {
+	if (args->v0.device != ~0)
 		device = nvkm_device_find(args->v0.device);
-		if (!device)
-			return -ENODEV;
-	}
+	else
+		device = nvkm_device_find(client->device);
+	if (!device)
+		return -ENODEV;
 
 	ret = nvkm_parent_create(parent, NULL, oclass, 0, nvkm_control_oclass,
 				 (1ULL << NVDEV_ENGINE_DMAOBJ) |