mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-26 19:00:25 +00:00
![Dan Williams](/assets/img/avatar_default.png)
'struct cxl_dev_state' makes too many assumptions about the capabilities of a CXL device. In particular it assumes a CXL device has a mailbox and all of the infrastructure and state that comes along with that. In preparation for supporting accelerator / Type-2 devices that may not have a mailbox and in general maintain a minimal core context structure, make mailbox functionality a super-set of 'struct cxl_dev_state' with 'struct cxl_memdev_state'. With this reorganization it allows for CXL devices that support HDM decoder mapping, but not other general-expander / Type-3 capabilities, to only enable that subset without the rest of the mailbox infrastructure coming along for the ride. Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com> Reviewed-by: Dave Jiang <dave.jiang@intel.com> Link: https://lore.kernel.org/r/168679260240.3436160.15520641540463704524.stgit@dwillia2-xfh.jf.intel.com Signed-off-by: Dan Williams <dan.j.williams@intel.com>
464 lines
11 KiB
C
464 lines
11 KiB
C
// SPDX-License-Identifier: GPL-2.0-only
|
|
/* Copyright(c) 2021 Intel Corporation. All rights reserved. */
|
|
#include <linux/libnvdimm.h>
|
|
#include <asm/unaligned.h>
|
|
#include <linux/device.h>
|
|
#include <linux/module.h>
|
|
#include <linux/ndctl.h>
|
|
#include <linux/async.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/nd.h>
|
|
#include "cxlmem.h"
|
|
#include "cxl.h"
|
|
|
|
extern const struct nvdimm_security_ops *cxl_security_ops;
|
|
|
|
static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
|
|
|
|
static void clear_exclusive(void *mds)
|
|
{
|
|
clear_exclusive_cxl_commands(mds, exclusive_cmds);
|
|
}
|
|
|
|
static void unregister_nvdimm(void *nvdimm)
|
|
{
|
|
nvdimm_delete(nvdimm);
|
|
}
|
|
|
|
static ssize_t provider_show(struct device *dev, struct device_attribute *attr, char *buf)
|
|
{
|
|
struct nvdimm *nvdimm = to_nvdimm(dev);
|
|
struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
|
|
|
|
return sysfs_emit(buf, "%s\n", dev_name(&cxl_nvd->dev));
|
|
}
|
|
static DEVICE_ATTR_RO(provider);
|
|
|
|
static ssize_t id_show(struct device *dev, struct device_attribute *attr, char *buf)
|
|
{
|
|
struct nvdimm *nvdimm = to_nvdimm(dev);
|
|
struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
|
|
struct cxl_dev_state *cxlds = cxl_nvd->cxlmd->cxlds;
|
|
|
|
return sysfs_emit(buf, "%lld\n", cxlds->serial);
|
|
}
|
|
static DEVICE_ATTR_RO(id);
|
|
|
|
static struct attribute *cxl_dimm_attributes[] = {
|
|
&dev_attr_id.attr,
|
|
&dev_attr_provider.attr,
|
|
NULL
|
|
};
|
|
|
|
static const struct attribute_group cxl_dimm_attribute_group = {
|
|
.name = "cxl",
|
|
.attrs = cxl_dimm_attributes,
|
|
};
|
|
|
|
static const struct attribute_group *cxl_dimm_attribute_groups[] = {
|
|
&cxl_dimm_attribute_group,
|
|
NULL
|
|
};
|
|
|
|
static int cxl_nvdimm_probe(struct device *dev)
|
|
{
|
|
struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev);
|
|
struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
|
|
struct cxl_nvdimm_bridge *cxl_nvb = cxlmd->cxl_nvb;
|
|
struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
|
|
unsigned long flags = 0, cmd_mask = 0;
|
|
struct nvdimm *nvdimm;
|
|
int rc;
|
|
|
|
set_exclusive_cxl_commands(mds, exclusive_cmds);
|
|
rc = devm_add_action_or_reset(dev, clear_exclusive, mds);
|
|
if (rc)
|
|
return rc;
|
|
|
|
set_bit(NDD_LABELING, &flags);
|
|
set_bit(NDD_REGISTER_SYNC, &flags);
|
|
set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask);
|
|
set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask);
|
|
set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask);
|
|
nvdimm = __nvdimm_create(cxl_nvb->nvdimm_bus, cxl_nvd,
|
|
cxl_dimm_attribute_groups, flags,
|
|
cmd_mask, 0, NULL, cxl_nvd->dev_id,
|
|
cxl_security_ops, NULL);
|
|
if (!nvdimm)
|
|
return -ENOMEM;
|
|
|
|
dev_set_drvdata(dev, nvdimm);
|
|
return devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm);
|
|
}
|
|
|
|
static struct cxl_driver cxl_nvdimm_driver = {
|
|
.name = "cxl_nvdimm",
|
|
.probe = cxl_nvdimm_probe,
|
|
.id = CXL_DEVICE_NVDIMM,
|
|
.drv = {
|
|
.suppress_bind_attrs = true,
|
|
},
|
|
};
|
|
|
|
static int cxl_pmem_get_config_size(struct cxl_memdev_state *mds,
|
|
struct nd_cmd_get_config_size *cmd,
|
|
unsigned int buf_len)
|
|
{
|
|
if (sizeof(*cmd) > buf_len)
|
|
return -EINVAL;
|
|
|
|
*cmd = (struct nd_cmd_get_config_size){
|
|
.config_size = mds->lsa_size,
|
|
.max_xfer =
|
|
mds->payload_size - sizeof(struct cxl_mbox_set_lsa),
|
|
};
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int cxl_pmem_get_config_data(struct cxl_memdev_state *mds,
|
|
struct nd_cmd_get_config_data_hdr *cmd,
|
|
unsigned int buf_len)
|
|
{
|
|
struct cxl_mbox_get_lsa get_lsa;
|
|
struct cxl_mbox_cmd mbox_cmd;
|
|
int rc;
|
|
|
|
if (sizeof(*cmd) > buf_len)
|
|
return -EINVAL;
|
|
if (struct_size(cmd, out_buf, cmd->in_length) > buf_len)
|
|
return -EINVAL;
|
|
|
|
get_lsa = (struct cxl_mbox_get_lsa) {
|
|
.offset = cpu_to_le32(cmd->in_offset),
|
|
.length = cpu_to_le32(cmd->in_length),
|
|
};
|
|
mbox_cmd = (struct cxl_mbox_cmd) {
|
|
.opcode = CXL_MBOX_OP_GET_LSA,
|
|
.payload_in = &get_lsa,
|
|
.size_in = sizeof(get_lsa),
|
|
.size_out = cmd->in_length,
|
|
.payload_out = cmd->out_buf,
|
|
};
|
|
|
|
rc = cxl_internal_send_cmd(mds, &mbox_cmd);
|
|
cmd->status = 0;
|
|
|
|
return rc;
|
|
}
|
|
|
|
static int cxl_pmem_set_config_data(struct cxl_memdev_state *mds,
|
|
struct nd_cmd_set_config_hdr *cmd,
|
|
unsigned int buf_len)
|
|
{
|
|
struct cxl_mbox_set_lsa *set_lsa;
|
|
struct cxl_mbox_cmd mbox_cmd;
|
|
int rc;
|
|
|
|
if (sizeof(*cmd) > buf_len)
|
|
return -EINVAL;
|
|
|
|
/* 4-byte status follows the input data in the payload */
|
|
if (size_add(struct_size(cmd, in_buf, cmd->in_length), 4) > buf_len)
|
|
return -EINVAL;
|
|
|
|
set_lsa =
|
|
kvzalloc(struct_size(set_lsa, data, cmd->in_length), GFP_KERNEL);
|
|
if (!set_lsa)
|
|
return -ENOMEM;
|
|
|
|
*set_lsa = (struct cxl_mbox_set_lsa) {
|
|
.offset = cpu_to_le32(cmd->in_offset),
|
|
};
|
|
memcpy(set_lsa->data, cmd->in_buf, cmd->in_length);
|
|
mbox_cmd = (struct cxl_mbox_cmd) {
|
|
.opcode = CXL_MBOX_OP_SET_LSA,
|
|
.payload_in = set_lsa,
|
|
.size_in = struct_size(set_lsa, data, cmd->in_length),
|
|
};
|
|
|
|
rc = cxl_internal_send_cmd(mds, &mbox_cmd);
|
|
|
|
/*
|
|
* Set "firmware" status (4-packed bytes at the end of the input
|
|
* payload.
|
|
*/
|
|
put_unaligned(0, (u32 *) &cmd->in_buf[cmd->in_length]);
|
|
kvfree(set_lsa);
|
|
|
|
return rc;
|
|
}
|
|
|
|
static int cxl_pmem_nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd,
|
|
void *buf, unsigned int buf_len)
|
|
{
|
|
struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
|
|
unsigned long cmd_mask = nvdimm_cmd_mask(nvdimm);
|
|
struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
|
|
struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
|
|
|
|
if (!test_bit(cmd, &cmd_mask))
|
|
return -ENOTTY;
|
|
|
|
switch (cmd) {
|
|
case ND_CMD_GET_CONFIG_SIZE:
|
|
return cxl_pmem_get_config_size(mds, buf, buf_len);
|
|
case ND_CMD_GET_CONFIG_DATA:
|
|
return cxl_pmem_get_config_data(mds, buf, buf_len);
|
|
case ND_CMD_SET_CONFIG_DATA:
|
|
return cxl_pmem_set_config_data(mds, buf, buf_len);
|
|
default:
|
|
return -ENOTTY;
|
|
}
|
|
}
|
|
|
|
static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc,
|
|
struct nvdimm *nvdimm, unsigned int cmd, void *buf,
|
|
unsigned int buf_len, int *cmd_rc)
|
|
{
|
|
/*
|
|
* No firmware response to translate, let the transport error
|
|
* code take precedence.
|
|
*/
|
|
*cmd_rc = 0;
|
|
|
|
if (!nvdimm)
|
|
return -ENOTTY;
|
|
return cxl_pmem_nvdimm_ctl(nvdimm, cmd, buf, buf_len);
|
|
}
|
|
|
|
static int detach_nvdimm(struct device *dev, void *data)
|
|
{
|
|
struct cxl_nvdimm *cxl_nvd;
|
|
bool release = false;
|
|
|
|
if (!is_cxl_nvdimm(dev))
|
|
return 0;
|
|
|
|
device_lock(dev);
|
|
if (!dev->driver)
|
|
goto out;
|
|
|
|
cxl_nvd = to_cxl_nvdimm(dev);
|
|
if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data)
|
|
release = true;
|
|
out:
|
|
device_unlock(dev);
|
|
if (release)
|
|
device_release_driver(dev);
|
|
return 0;
|
|
}
|
|
|
|
static void unregister_nvdimm_bus(void *_cxl_nvb)
|
|
{
|
|
struct cxl_nvdimm_bridge *cxl_nvb = _cxl_nvb;
|
|
struct nvdimm_bus *nvdimm_bus = cxl_nvb->nvdimm_bus;
|
|
|
|
bus_for_each_dev(&cxl_bus_type, NULL, cxl_nvb, detach_nvdimm);
|
|
|
|
cxl_nvb->nvdimm_bus = NULL;
|
|
nvdimm_bus_unregister(nvdimm_bus);
|
|
}
|
|
|
|
static int cxl_nvdimm_bridge_probe(struct device *dev)
|
|
{
|
|
struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev);
|
|
|
|
cxl_nvb->nd_desc = (struct nvdimm_bus_descriptor) {
|
|
.provider_name = "CXL",
|
|
.module = THIS_MODULE,
|
|
.ndctl = cxl_pmem_ctl,
|
|
};
|
|
|
|
cxl_nvb->nvdimm_bus =
|
|
nvdimm_bus_register(&cxl_nvb->dev, &cxl_nvb->nd_desc);
|
|
|
|
if (!cxl_nvb->nvdimm_bus)
|
|
return -ENOMEM;
|
|
|
|
return devm_add_action_or_reset(dev, unregister_nvdimm_bus, cxl_nvb);
|
|
}
|
|
|
|
static struct cxl_driver cxl_nvdimm_bridge_driver = {
|
|
.name = "cxl_nvdimm_bridge",
|
|
.probe = cxl_nvdimm_bridge_probe,
|
|
.id = CXL_DEVICE_NVDIMM_BRIDGE,
|
|
.drv = {
|
|
.suppress_bind_attrs = true,
|
|
},
|
|
};
|
|
|
|
static void unregister_nvdimm_region(void *nd_region)
|
|
{
|
|
nvdimm_region_delete(nd_region);
|
|
}
|
|
|
|
static void cxlr_pmem_remove_resource(void *res)
|
|
{
|
|
remove_resource(res);
|
|
}
|
|
|
|
struct cxl_pmem_region_info {
|
|
u64 offset;
|
|
u64 serial;
|
|
};
|
|
|
|
static int cxl_pmem_region_probe(struct device *dev)
|
|
{
|
|
struct nd_mapping_desc mappings[CXL_DECODER_MAX_INTERLEAVE];
|
|
struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
|
|
struct cxl_region *cxlr = cxlr_pmem->cxlr;
|
|
struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
|
|
struct cxl_pmem_region_info *info = NULL;
|
|
struct nd_interleave_set *nd_set;
|
|
struct nd_region_desc ndr_desc;
|
|
struct cxl_nvdimm *cxl_nvd;
|
|
struct nvdimm *nvdimm;
|
|
struct resource *res;
|
|
int rc, i = 0;
|
|
|
|
memset(&mappings, 0, sizeof(mappings));
|
|
memset(&ndr_desc, 0, sizeof(ndr_desc));
|
|
|
|
res = devm_kzalloc(dev, sizeof(*res), GFP_KERNEL);
|
|
if (!res)
|
|
return -ENOMEM;
|
|
|
|
res->name = "Persistent Memory";
|
|
res->start = cxlr_pmem->hpa_range.start;
|
|
res->end = cxlr_pmem->hpa_range.end;
|
|
res->flags = IORESOURCE_MEM;
|
|
res->desc = IORES_DESC_PERSISTENT_MEMORY;
|
|
|
|
rc = insert_resource(&iomem_resource, res);
|
|
if (rc)
|
|
return rc;
|
|
|
|
rc = devm_add_action_or_reset(dev, cxlr_pmem_remove_resource, res);
|
|
if (rc)
|
|
return rc;
|
|
|
|
ndr_desc.res = res;
|
|
ndr_desc.provider_data = cxlr_pmem;
|
|
|
|
ndr_desc.numa_node = memory_add_physaddr_to_nid(res->start);
|
|
ndr_desc.target_node = phys_to_target_node(res->start);
|
|
if (ndr_desc.target_node == NUMA_NO_NODE) {
|
|
ndr_desc.target_node = ndr_desc.numa_node;
|
|
dev_dbg(&cxlr->dev, "changing target node from %d to %d",
|
|
NUMA_NO_NODE, ndr_desc.target_node);
|
|
}
|
|
|
|
nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
|
|
if (!nd_set)
|
|
return -ENOMEM;
|
|
|
|
ndr_desc.memregion = cxlr->id;
|
|
set_bit(ND_REGION_CXL, &ndr_desc.flags);
|
|
set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc.flags);
|
|
|
|
info = kmalloc_array(cxlr_pmem->nr_mappings, sizeof(*info), GFP_KERNEL);
|
|
if (!info)
|
|
return -ENOMEM;
|
|
|
|
for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
|
|
struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
|
|
struct cxl_memdev *cxlmd = m->cxlmd;
|
|
struct cxl_dev_state *cxlds = cxlmd->cxlds;
|
|
|
|
cxl_nvd = cxlmd->cxl_nvd;
|
|
nvdimm = dev_get_drvdata(&cxl_nvd->dev);
|
|
if (!nvdimm) {
|
|
dev_dbg(dev, "[%d]: %s: no nvdimm found\n", i,
|
|
dev_name(&cxlmd->dev));
|
|
rc = -ENODEV;
|
|
goto out_nvd;
|
|
}
|
|
|
|
m->cxl_nvd = cxl_nvd;
|
|
mappings[i] = (struct nd_mapping_desc) {
|
|
.nvdimm = nvdimm,
|
|
.start = m->start,
|
|
.size = m->size,
|
|
.position = i,
|
|
};
|
|
info[i].offset = m->start;
|
|
info[i].serial = cxlds->serial;
|
|
}
|
|
ndr_desc.num_mappings = cxlr_pmem->nr_mappings;
|
|
ndr_desc.mapping = mappings;
|
|
|
|
/*
|
|
* TODO enable CXL labels which skip the need for 'interleave-set cookie'
|
|
*/
|
|
nd_set->cookie1 =
|
|
nd_fletcher64(info, sizeof(*info) * cxlr_pmem->nr_mappings, 0);
|
|
nd_set->cookie2 = nd_set->cookie1;
|
|
ndr_desc.nd_set = nd_set;
|
|
|
|
cxlr_pmem->nd_region =
|
|
nvdimm_pmem_region_create(cxl_nvb->nvdimm_bus, &ndr_desc);
|
|
if (!cxlr_pmem->nd_region) {
|
|
rc = -ENOMEM;
|
|
goto out_nvd;
|
|
}
|
|
|
|
rc = devm_add_action_or_reset(dev, unregister_nvdimm_region,
|
|
cxlr_pmem->nd_region);
|
|
out_nvd:
|
|
kfree(info);
|
|
|
|
return rc;
|
|
}
|
|
|
|
static struct cxl_driver cxl_pmem_region_driver = {
|
|
.name = "cxl_pmem_region",
|
|
.probe = cxl_pmem_region_probe,
|
|
.id = CXL_DEVICE_PMEM_REGION,
|
|
.drv = {
|
|
.suppress_bind_attrs = true,
|
|
},
|
|
};
|
|
|
|
static __init int cxl_pmem_init(void)
|
|
{
|
|
int rc;
|
|
|
|
set_bit(CXL_MEM_COMMAND_ID_SET_SHUTDOWN_STATE, exclusive_cmds);
|
|
set_bit(CXL_MEM_COMMAND_ID_SET_LSA, exclusive_cmds);
|
|
|
|
rc = cxl_driver_register(&cxl_nvdimm_bridge_driver);
|
|
if (rc)
|
|
return rc;
|
|
|
|
rc = cxl_driver_register(&cxl_nvdimm_driver);
|
|
if (rc)
|
|
goto err_nvdimm;
|
|
|
|
rc = cxl_driver_register(&cxl_pmem_region_driver);
|
|
if (rc)
|
|
goto err_region;
|
|
|
|
return 0;
|
|
|
|
err_region:
|
|
cxl_driver_unregister(&cxl_nvdimm_driver);
|
|
err_nvdimm:
|
|
cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
|
|
return rc;
|
|
}
|
|
|
|
static __exit void cxl_pmem_exit(void)
|
|
{
|
|
cxl_driver_unregister(&cxl_pmem_region_driver);
|
|
cxl_driver_unregister(&cxl_nvdimm_driver);
|
|
cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
|
|
}
|
|
|
|
MODULE_LICENSE("GPL v2");
|
|
module_init(cxl_pmem_init);
|
|
module_exit(cxl_pmem_exit);
|
|
MODULE_IMPORT_NS(CXL);
|
|
MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE);
|
|
MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM);
|
|
MODULE_ALIAS_CXL(CXL_DEVICE_PMEM_REGION);
|