mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-26 10:49:33 +00:00
![Hannes Reinecke](/assets/img/avatar_default.png)
Replace the check for DRIVER_SENSE with a check for scsi_status_is_check_condition(). Audit all callsites to ensure the SAM status is set correctly. For backwards compability move the DRIVER_SENSE definition to sg.h, and update sg, bsg, and scsi_ioctl to set the DRIVER_SENSE driver_status whenever SAM_STAT_CHECK_CONDITION is present. [mkp: fix zeroday srp warning] Link: https://lore.kernel.org/r/20210427083046.31620-10-hare@suse.de Signed-off-by: Hannes Reinecke <hare@suse.de> Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com> fix
2218 lines
61 KiB
C
2218 lines
61 KiB
C
// SPDX-License-Identifier: GPL-2.0-or-later
|
|
/*
|
|
* CXL Flash Device Driver
|
|
*
|
|
* Written by: Manoj N. Kumar <manoj@linux.vnet.ibm.com>, IBM Corporation
|
|
* Matthew R. Ochs <mrochs@linux.vnet.ibm.com>, IBM Corporation
|
|
*
|
|
* Copyright (C) 2015 IBM Corporation
|
|
*/
|
|
|
|
#include <linux/delay.h>
|
|
#include <linux/file.h>
|
|
#include <linux/interrupt.h>
|
|
#include <linux/pci.h>
|
|
#include <linux/syscalls.h>
|
|
#include <asm/unaligned.h>
|
|
|
|
#include <scsi/scsi.h>
|
|
#include <scsi/scsi_host.h>
|
|
#include <scsi/scsi_cmnd.h>
|
|
#include <scsi/scsi_eh.h>
|
|
#include <uapi/scsi/cxlflash_ioctl.h>
|
|
|
|
#include "sislite.h"
|
|
#include "common.h"
|
|
#include "vlun.h"
|
|
#include "superpipe.h"
|
|
|
|
struct cxlflash_global global;
|
|
|
|
/**
|
|
* marshal_rele_to_resize() - translate release to resize structure
|
|
* @release: Source structure from which to translate/copy.
|
|
* @resize: Destination structure for the translate/copy.
|
|
*/
|
|
static void marshal_rele_to_resize(struct dk_cxlflash_release *release,
|
|
struct dk_cxlflash_resize *resize)
|
|
{
|
|
resize->hdr = release->hdr;
|
|
resize->context_id = release->context_id;
|
|
resize->rsrc_handle = release->rsrc_handle;
|
|
}
|
|
|
|
/**
|
|
* marshal_det_to_rele() - translate detach to release structure
|
|
* @detach: Destination structure for the translate/copy.
|
|
* @release: Source structure from which to translate/copy.
|
|
*/
|
|
static void marshal_det_to_rele(struct dk_cxlflash_detach *detach,
|
|
struct dk_cxlflash_release *release)
|
|
{
|
|
release->hdr = detach->hdr;
|
|
release->context_id = detach->context_id;
|
|
}
|
|
|
|
/**
|
|
* marshal_udir_to_rele() - translate udirect to release structure
|
|
* @udirect: Source structure from which to translate/copy.
|
|
* @release: Destination structure for the translate/copy.
|
|
*/
|
|
static void marshal_udir_to_rele(struct dk_cxlflash_udirect *udirect,
|
|
struct dk_cxlflash_release *release)
|
|
{
|
|
release->hdr = udirect->hdr;
|
|
release->context_id = udirect->context_id;
|
|
release->rsrc_handle = udirect->rsrc_handle;
|
|
}
|
|
|
|
/**
|
|
* cxlflash_free_errpage() - frees resources associated with global error page
|
|
*/
|
|
void cxlflash_free_errpage(void)
|
|
{
|
|
|
|
mutex_lock(&global.mutex);
|
|
if (global.err_page) {
|
|
__free_page(global.err_page);
|
|
global.err_page = NULL;
|
|
}
|
|
mutex_unlock(&global.mutex);
|
|
}
|
|
|
|
/**
|
|
* cxlflash_stop_term_user_contexts() - stops/terminates known user contexts
|
|
* @cfg: Internal structure associated with the host.
|
|
*
|
|
* When the host needs to go down, all users must be quiesced and their
|
|
* memory freed. This is accomplished by putting the contexts in error
|
|
* state which will notify the user and let them 'drive' the tear down.
|
|
* Meanwhile, this routine camps until all user contexts have been removed.
|
|
*
|
|
* Note that the main loop in this routine will always execute at least once
|
|
* to flush the reset_waitq.
|
|
*/
|
|
void cxlflash_stop_term_user_contexts(struct cxlflash_cfg *cfg)
|
|
{
|
|
struct device *dev = &cfg->dev->dev;
|
|
int i, found = true;
|
|
|
|
cxlflash_mark_contexts_error(cfg);
|
|
|
|
while (true) {
|
|
for (i = 0; i < MAX_CONTEXT; i++)
|
|
if (cfg->ctx_tbl[i]) {
|
|
found = true;
|
|
break;
|
|
}
|
|
|
|
if (!found && list_empty(&cfg->ctx_err_recovery))
|
|
return;
|
|
|
|
dev_dbg(dev, "%s: Wait for user contexts to quiesce...\n",
|
|
__func__);
|
|
wake_up_all(&cfg->reset_waitq);
|
|
ssleep(1);
|
|
found = false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* find_error_context() - locates a context by cookie on the error recovery list
|
|
* @cfg: Internal structure associated with the host.
|
|
* @rctxid: Desired context by id.
|
|
* @file: Desired context by file.
|
|
*
|
|
* Return: Found context on success, NULL on failure
|
|
*/
|
|
static struct ctx_info *find_error_context(struct cxlflash_cfg *cfg, u64 rctxid,
|
|
struct file *file)
|
|
{
|
|
struct ctx_info *ctxi;
|
|
|
|
list_for_each_entry(ctxi, &cfg->ctx_err_recovery, list)
|
|
if ((ctxi->ctxid == rctxid) || (ctxi->file == file))
|
|
return ctxi;
|
|
|
|
return NULL;
|
|
}
|
|
|
|
/**
|
|
* get_context() - obtains a validated and locked context reference
|
|
* @cfg: Internal structure associated with the host.
|
|
* @rctxid: Desired context (raw, un-decoded format).
|
|
* @arg: LUN information or file associated with request.
|
|
* @ctx_ctrl: Control information to 'steer' desired lookup.
|
|
*
|
|
* NOTE: despite the name pid, in linux, current->pid actually refers
|
|
* to the lightweight process id (tid) and can change if the process is
|
|
* multi threaded. The tgid remains constant for the process and only changes
|
|
* when the process of fork. For all intents and purposes, think of tgid
|
|
* as a pid in the traditional sense.
|
|
*
|
|
* Return: Validated context on success, NULL on failure
|
|
*/
|
|
struct ctx_info *get_context(struct cxlflash_cfg *cfg, u64 rctxid,
|
|
void *arg, enum ctx_ctrl ctx_ctrl)
|
|
{
|
|
struct device *dev = &cfg->dev->dev;
|
|
struct ctx_info *ctxi = NULL;
|
|
struct lun_access *lun_access = NULL;
|
|
struct file *file = NULL;
|
|
struct llun_info *lli = arg;
|
|
u64 ctxid = DECODE_CTXID(rctxid);
|
|
int rc;
|
|
pid_t pid = task_tgid_nr(current), ctxpid = 0;
|
|
|
|
if (ctx_ctrl & CTX_CTRL_FILE) {
|
|
lli = NULL;
|
|
file = (struct file *)arg;
|
|
}
|
|
|
|
if (ctx_ctrl & CTX_CTRL_CLONE)
|
|
pid = task_ppid_nr(current);
|
|
|
|
if (likely(ctxid < MAX_CONTEXT)) {
|
|
while (true) {
|
|
mutex_lock(&cfg->ctx_tbl_list_mutex);
|
|
ctxi = cfg->ctx_tbl[ctxid];
|
|
if (ctxi)
|
|
if ((file && (ctxi->file != file)) ||
|
|
(!file && (ctxi->ctxid != rctxid)))
|
|
ctxi = NULL;
|
|
|
|
if ((ctx_ctrl & CTX_CTRL_ERR) ||
|
|
(!ctxi && (ctx_ctrl & CTX_CTRL_ERR_FALLBACK)))
|
|
ctxi = find_error_context(cfg, rctxid, file);
|
|
if (!ctxi) {
|
|
mutex_unlock(&cfg->ctx_tbl_list_mutex);
|
|
goto out;
|
|
}
|
|
|
|
/*
|
|
* Need to acquire ownership of the context while still
|
|
* under the table/list lock to serialize with a remove
|
|
* thread. Use the 'try' to avoid stalling the
|
|
* table/list lock for a single context.
|
|
*
|
|
* Note that the lock order is:
|
|
*
|
|
* cfg->ctx_tbl_list_mutex -> ctxi->mutex
|
|
*
|
|
* Therefore release ctx_tbl_list_mutex before retrying.
|
|
*/
|
|
rc = mutex_trylock(&ctxi->mutex);
|
|
mutex_unlock(&cfg->ctx_tbl_list_mutex);
|
|
if (rc)
|
|
break; /* got the context's lock! */
|
|
}
|
|
|
|
if (ctxi->unavail)
|
|
goto denied;
|
|
|
|
ctxpid = ctxi->pid;
|
|
if (likely(!(ctx_ctrl & CTX_CTRL_NOPID)))
|
|
if (pid != ctxpid)
|
|
goto denied;
|
|
|
|
if (lli) {
|
|
list_for_each_entry(lun_access, &ctxi->luns, list)
|
|
if (lun_access->lli == lli)
|
|
goto out;
|
|
goto denied;
|
|
}
|
|
}
|
|
|
|
out:
|
|
dev_dbg(dev, "%s: rctxid=%016llx ctxinfo=%p ctxpid=%u pid=%u "
|
|
"ctx_ctrl=%u\n", __func__, rctxid, ctxi, ctxpid, pid,
|
|
ctx_ctrl);
|
|
|
|
return ctxi;
|
|
|
|
denied:
|
|
mutex_unlock(&ctxi->mutex);
|
|
ctxi = NULL;
|
|
goto out;
|
|
}
|
|
|
|
/**
|
|
* put_context() - release a context that was retrieved from get_context()
|
|
* @ctxi: Context to release.
|
|
*
|
|
* For now, releasing the context equates to unlocking it's mutex.
|
|
*/
|
|
void put_context(struct ctx_info *ctxi)
|
|
{
|
|
mutex_unlock(&ctxi->mutex);
|
|
}
|
|
|
|
/**
|
|
* afu_attach() - attach a context to the AFU
|
|
* @cfg: Internal structure associated with the host.
|
|
* @ctxi: Context to attach.
|
|
*
|
|
* Upon setting the context capabilities, they must be confirmed with
|
|
* a read back operation as the context might have been closed since
|
|
* the mailbox was unlocked. When this occurs, registration is failed.
|
|
*
|
|
* Return: 0 on success, -errno on failure
|
|
*/
|
|
static int afu_attach(struct cxlflash_cfg *cfg, struct ctx_info *ctxi)
|
|
{
|
|
struct device *dev = &cfg->dev->dev;
|
|
struct afu *afu = cfg->afu;
|
|
struct sisl_ctrl_map __iomem *ctrl_map = ctxi->ctrl_map;
|
|
int rc = 0;
|
|
struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ);
|
|
u64 val;
|
|
int i;
|
|
|
|
/* Unlock cap and restrict user to read/write cmds in translated mode */
|
|
readq_be(&ctrl_map->mbox_r);
|
|
val = (SISL_CTX_CAP_READ_CMD | SISL_CTX_CAP_WRITE_CMD);
|
|
writeq_be(val, &ctrl_map->ctx_cap);
|
|
val = readq_be(&ctrl_map->ctx_cap);
|
|
if (val != (SISL_CTX_CAP_READ_CMD | SISL_CTX_CAP_WRITE_CMD)) {
|
|
dev_err(dev, "%s: ctx may be closed val=%016llx\n",
|
|
__func__, val);
|
|
rc = -EAGAIN;
|
|
goto out;
|
|
}
|
|
|
|
if (afu_is_ocxl_lisn(afu)) {
|
|
/* Set up the LISN effective address for each interrupt */
|
|
for (i = 0; i < ctxi->irqs; i++) {
|
|
val = cfg->ops->get_irq_objhndl(ctxi->ctx, i);
|
|
writeq_be(val, &ctrl_map->lisn_ea[i]);
|
|
}
|
|
|
|
/* Use primary HWQ PASID as identifier for all interrupts */
|
|
val = hwq->ctx_hndl;
|
|
writeq_be(SISL_LISN_PASID(val, val), &ctrl_map->lisn_pasid[0]);
|
|
writeq_be(SISL_LISN_PASID(0UL, val), &ctrl_map->lisn_pasid[1]);
|
|
}
|
|
|
|
/* Set up MMIO registers pointing to the RHT */
|
|
writeq_be((u64)ctxi->rht_start, &ctrl_map->rht_start);
|
|
val = SISL_RHT_CNT_ID((u64)MAX_RHT_PER_CONTEXT, (u64)(hwq->ctx_hndl));
|
|
writeq_be(val, &ctrl_map->rht_cnt_id);
|
|
out:
|
|
dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);
|
|
return rc;
|
|
}
|
|
|
|
/**
|
|
* read_cap16() - issues a SCSI READ_CAP16 command
|
|
* @sdev: SCSI device associated with LUN.
|
|
* @lli: LUN destined for capacity request.
|
|
*
|
|
* The READ_CAP16 can take quite a while to complete. Should an EEH occur while
|
|
* in scsi_execute(), the EEH handler will attempt to recover. As part of the
|
|
* recovery, the handler drains all currently running ioctls, waiting until they
|
|
* have completed before proceeding with a reset. As this routine is used on the
|
|
* ioctl path, this can create a condition where the EEH handler becomes stuck,
|
|
* infinitely waiting for this ioctl thread. To avoid this behavior, temporarily
|
|
* unmark this thread as an ioctl thread by releasing the ioctl read semaphore.
|
|
* This will allow the EEH handler to proceed with a recovery while this thread
|
|
* is still running. Once the scsi_execute() returns, reacquire the ioctl read
|
|
* semaphore and check the adapter state in case it changed while inside of
|
|
* scsi_execute(). The state check will wait if the adapter is still being
|
|
* recovered or return a failure if the recovery failed. In the event that the
|
|
* adapter reset failed, simply return the failure as the ioctl would be unable
|
|
* to continue.
|
|
*
|
|
* Note that the above puts a requirement on this routine to only be called on
|
|
* an ioctl thread.
|
|
*
|
|
* Return: 0 on success, -errno on failure
|
|
*/
|
|
static int read_cap16(struct scsi_device *sdev, struct llun_info *lli)
|
|
{
|
|
struct cxlflash_cfg *cfg = shost_priv(sdev->host);
|
|
struct device *dev = &cfg->dev->dev;
|
|
struct glun_info *gli = lli->parent;
|
|
struct scsi_sense_hdr sshdr;
|
|
u8 *cmd_buf = NULL;
|
|
u8 *scsi_cmd = NULL;
|
|
int rc = 0;
|
|
int result = 0;
|
|
int retry_cnt = 0;
|
|
u32 to = CMD_TIMEOUT * HZ;
|
|
|
|
retry:
|
|
cmd_buf = kzalloc(CMD_BUFSIZE, GFP_KERNEL);
|
|
scsi_cmd = kzalloc(MAX_COMMAND_SIZE, GFP_KERNEL);
|
|
if (unlikely(!cmd_buf || !scsi_cmd)) {
|
|
rc = -ENOMEM;
|
|
goto out;
|
|
}
|
|
|
|
scsi_cmd[0] = SERVICE_ACTION_IN_16; /* read cap(16) */
|
|
scsi_cmd[1] = SAI_READ_CAPACITY_16; /* service action */
|
|
put_unaligned_be32(CMD_BUFSIZE, &scsi_cmd[10]);
|
|
|
|
dev_dbg(dev, "%s: %ssending cmd(%02x)\n", __func__,
|
|
retry_cnt ? "re" : "", scsi_cmd[0]);
|
|
|
|
/* Drop the ioctl read semahpore across lengthy call */
|
|
up_read(&cfg->ioctl_rwsem);
|
|
result = scsi_execute(sdev, scsi_cmd, DMA_FROM_DEVICE, cmd_buf,
|
|
CMD_BUFSIZE, NULL, &sshdr, to, CMD_RETRIES,
|
|
0, 0, NULL);
|
|
down_read(&cfg->ioctl_rwsem);
|
|
rc = check_state(cfg);
|
|
if (rc) {
|
|
dev_err(dev, "%s: Failed state result=%08x\n",
|
|
__func__, result);
|
|
rc = -ENODEV;
|
|
goto out;
|
|
}
|
|
|
|
if (result > 0 && scsi_sense_valid(&sshdr)) {
|
|
if (result & SAM_STAT_CHECK_CONDITION) {
|
|
switch (sshdr.sense_key) {
|
|
case NO_SENSE:
|
|
case RECOVERED_ERROR:
|
|
case NOT_READY:
|
|
result &= ~SAM_STAT_CHECK_CONDITION;
|
|
break;
|
|
case UNIT_ATTENTION:
|
|
switch (sshdr.asc) {
|
|
case 0x29: /* Power on Reset or Device Reset */
|
|
fallthrough;
|
|
case 0x2A: /* Device capacity changed */
|
|
case 0x3F: /* Report LUNs changed */
|
|
/* Retry the command once more */
|
|
if (retry_cnt++ < 1) {
|
|
kfree(cmd_buf);
|
|
kfree(scsi_cmd);
|
|
goto retry;
|
|
}
|
|
}
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (result) {
|
|
dev_err(dev, "%s: command failed, result=%08x\n",
|
|
__func__, result);
|
|
rc = -EIO;
|
|
goto out;
|
|
}
|
|
|
|
/*
|
|
* Read cap was successful, grab values from the buffer;
|
|
* note that we don't need to worry about unaligned access
|
|
* as the buffer is allocated on an aligned boundary.
|
|
*/
|
|
mutex_lock(&gli->mutex);
|
|
gli->max_lba = be64_to_cpu(*((__be64 *)&cmd_buf[0]));
|
|
gli->blk_len = be32_to_cpu(*((__be32 *)&cmd_buf[8]));
|
|
mutex_unlock(&gli->mutex);
|
|
|
|
out:
|
|
kfree(cmd_buf);
|
|
kfree(scsi_cmd);
|
|
|
|
dev_dbg(dev, "%s: maxlba=%lld blklen=%d rc=%d\n",
|
|
__func__, gli->max_lba, gli->blk_len, rc);
|
|
return rc;
|
|
}
|
|
|
|
/**
|
|
* get_rhte() - obtains validated resource handle table entry reference
|
|
* @ctxi: Context owning the resource handle.
|
|
* @rhndl: Resource handle associated with entry.
|
|
* @lli: LUN associated with request.
|
|
*
|
|
* Return: Validated RHTE on success, NULL on failure
|
|
*/
|
|
struct sisl_rht_entry *get_rhte(struct ctx_info *ctxi, res_hndl_t rhndl,
|
|
struct llun_info *lli)
|
|
{
|
|
struct cxlflash_cfg *cfg = ctxi->cfg;
|
|
struct device *dev = &cfg->dev->dev;
|
|
struct sisl_rht_entry *rhte = NULL;
|
|
|
|
if (unlikely(!ctxi->rht_start)) {
|
|
dev_dbg(dev, "%s: Context does not have allocated RHT\n",
|
|
__func__);
|
|
goto out;
|
|
}
|
|
|
|
if (unlikely(rhndl >= MAX_RHT_PER_CONTEXT)) {
|
|
dev_dbg(dev, "%s: Bad resource handle rhndl=%d\n",
|
|
__func__, rhndl);
|
|
goto out;
|
|
}
|
|
|
|
if (unlikely(ctxi->rht_lun[rhndl] != lli)) {
|
|
dev_dbg(dev, "%s: Bad resource handle LUN rhndl=%d\n",
|
|
__func__, rhndl);
|
|
goto out;
|
|
}
|
|
|
|
rhte = &ctxi->rht_start[rhndl];
|
|
if (unlikely(rhte->nmask == 0)) {
|
|
dev_dbg(dev, "%s: Unopened resource handle rhndl=%d\n",
|
|
__func__, rhndl);
|
|
rhte = NULL;
|
|
goto out;
|
|
}
|
|
|
|
out:
|
|
return rhte;
|
|
}
|
|
|
|
/**
|
|
* rhte_checkout() - obtains free/empty resource handle table entry
|
|
* @ctxi: Context owning the resource handle.
|
|
* @lli: LUN associated with request.
|
|
*
|
|
* Return: Free RHTE on success, NULL on failure
|
|
*/
|
|
struct sisl_rht_entry *rhte_checkout(struct ctx_info *ctxi,
|
|
struct llun_info *lli)
|
|
{
|
|
struct cxlflash_cfg *cfg = ctxi->cfg;
|
|
struct device *dev = &cfg->dev->dev;
|
|
struct sisl_rht_entry *rhte = NULL;
|
|
int i;
|
|
|
|
/* Find a free RHT entry */
|
|
for (i = 0; i < MAX_RHT_PER_CONTEXT; i++)
|
|
if (ctxi->rht_start[i].nmask == 0) {
|
|
rhte = &ctxi->rht_start[i];
|
|
ctxi->rht_out++;
|
|
break;
|
|
}
|
|
|
|
if (likely(rhte))
|
|
ctxi->rht_lun[i] = lli;
|
|
|
|
dev_dbg(dev, "%s: returning rhte=%p index=%d\n", __func__, rhte, i);
|
|
return rhte;
|
|
}
|
|
|
|
/**
|
|
* rhte_checkin() - releases a resource handle table entry
|
|
* @ctxi: Context owning the resource handle.
|
|
* @rhte: RHTE to release.
|
|
*/
|
|
void rhte_checkin(struct ctx_info *ctxi,
|
|
struct sisl_rht_entry *rhte)
|
|
{
|
|
u32 rsrc_handle = rhte - ctxi->rht_start;
|
|
|
|
rhte->nmask = 0;
|
|
rhte->fp = 0;
|
|
ctxi->rht_out--;
|
|
ctxi->rht_lun[rsrc_handle] = NULL;
|
|
ctxi->rht_needs_ws[rsrc_handle] = false;
|
|
}
|
|
|
|
/**
|
|
* rht_format1() - populates a RHTE for format 1
|
|
* @rhte: RHTE to populate.
|
|
* @lun_id: LUN ID of LUN associated with RHTE.
|
|
* @perm: Desired permissions for RHTE.
|
|
* @port_sel: Port selection mask
|
|
*/
|
|
static void rht_format1(struct sisl_rht_entry *rhte, u64 lun_id, u32 perm,
|
|
u32 port_sel)
|
|
{
|
|
/*
|
|
* Populate the Format 1 RHT entry for direct access (physical
|
|
* LUN) using the synchronization sequence defined in the
|
|
* SISLite specification.
|
|
*/
|
|
struct sisl_rht_entry_f1 dummy = { 0 };
|
|
struct sisl_rht_entry_f1 *rhte_f1 = (struct sisl_rht_entry_f1 *)rhte;
|
|
|
|
memset(rhte_f1, 0, sizeof(*rhte_f1));
|
|
rhte_f1->fp = SISL_RHT_FP(1U, 0);
|
|
dma_wmb(); /* Make setting of format bit visible */
|
|
|
|
rhte_f1->lun_id = lun_id;
|
|
dma_wmb(); /* Make setting of LUN id visible */
|
|
|
|
/*
|
|
* Use a dummy RHT Format 1 entry to build the second dword
|
|
* of the entry that must be populated in a single write when
|
|
* enabled (valid bit set to TRUE).
|
|
*/
|
|
dummy.valid = 0x80;
|
|
dummy.fp = SISL_RHT_FP(1U, perm);
|
|
dummy.port_sel = port_sel;
|
|
rhte_f1->dw = dummy.dw;
|
|
|
|
dma_wmb(); /* Make remaining RHT entry fields visible */
|
|
}
|
|
|
|
/**
|
|
* cxlflash_lun_attach() - attaches a user to a LUN and manages the LUN's mode
|
|
* @gli: LUN to attach.
|
|
* @mode: Desired mode of the LUN.
|
|
* @locked: Mutex status on current thread.
|
|
*
|
|
* Return: 0 on success, -errno on failure
|
|
*/
|
|
int cxlflash_lun_attach(struct glun_info *gli, enum lun_mode mode, bool locked)
|
|
{
|
|
int rc = 0;
|
|
|
|
if (!locked)
|
|
mutex_lock(&gli->mutex);
|
|
|
|
if (gli->mode == MODE_NONE)
|
|
gli->mode = mode;
|
|
else if (gli->mode != mode) {
|
|
pr_debug("%s: gli_mode=%d requested_mode=%d\n",
|
|
__func__, gli->mode, mode);
|
|
rc = -EINVAL;
|
|
goto out;
|
|
}
|
|
|
|
gli->users++;
|
|
WARN_ON(gli->users <= 0);
|
|
out:
|
|
pr_debug("%s: Returning rc=%d gli->mode=%u gli->users=%u\n",
|
|
__func__, rc, gli->mode, gli->users);
|
|
if (!locked)
|
|
mutex_unlock(&gli->mutex);
|
|
return rc;
|
|
}
|
|
|
|
/**
|
|
* cxlflash_lun_detach() - detaches a user from a LUN and resets the LUN's mode
|
|
* @gli: LUN to detach.
|
|
*
|
|
* When resetting the mode, terminate block allocation resources as they
|
|
* are no longer required (service is safe to call even when block allocation
|
|
* resources were not present - such as when transitioning from physical mode).
|
|
* These resources will be reallocated when needed (subsequent transition to
|
|
* virtual mode).
|
|
*/
|
|
void cxlflash_lun_detach(struct glun_info *gli)
|
|
{
|
|
mutex_lock(&gli->mutex);
|
|
WARN_ON(gli->mode == MODE_NONE);
|
|
if (--gli->users == 0) {
|
|
gli->mode = MODE_NONE;
|
|
cxlflash_ba_terminate(&gli->blka.ba_lun);
|
|
}
|
|
pr_debug("%s: gli->users=%u\n", __func__, gli->users);
|
|
WARN_ON(gli->users < 0);
|
|
mutex_unlock(&gli->mutex);
|
|
}
|
|
|
|
/**
|
|
* _cxlflash_disk_release() - releases the specified resource entry
|
|
* @sdev: SCSI device associated with LUN.
|
|
* @ctxi: Context owning resources.
|
|
* @release: Release ioctl data structure.
|
|
*
|
|
* For LUNs in virtual mode, the virtual LUN associated with the specified
|
|
* resource handle is resized to 0 prior to releasing the RHTE. Note that the
|
|
* AFU sync should _not_ be performed when the context is sitting on the error
|
|
* recovery list. A context on the error recovery list is not known to the AFU
|
|
* due to reset. When the context is recovered, it will be reattached and made
|
|
* known again to the AFU.
|
|
*
|
|
* Return: 0 on success, -errno on failure
|
|
*/
|
|
int _cxlflash_disk_release(struct scsi_device *sdev,
|
|
struct ctx_info *ctxi,
|
|
struct dk_cxlflash_release *release)
|
|
{
|
|
struct cxlflash_cfg *cfg = shost_priv(sdev->host);
|
|
struct device *dev = &cfg->dev->dev;
|
|
struct llun_info *lli = sdev->hostdata;
|
|
struct glun_info *gli = lli->parent;
|
|
struct afu *afu = cfg->afu;
|
|
bool put_ctx = false;
|
|
|
|
struct dk_cxlflash_resize size;
|
|
res_hndl_t rhndl = release->rsrc_handle;
|
|
|
|
int rc = 0;
|
|
int rcr = 0;
|
|
u64 ctxid = DECODE_CTXID(release->context_id),
|
|
rctxid = release->context_id;
|
|
|
|
struct sisl_rht_entry *rhte;
|
|
struct sisl_rht_entry_f1 *rhte_f1;
|
|
|
|
dev_dbg(dev, "%s: ctxid=%llu rhndl=%llu gli->mode=%u gli->users=%u\n",
|
|
__func__, ctxid, release->rsrc_handle, gli->mode, gli->users);
|
|
|
|
if (!ctxi) {
|
|
ctxi = get_context(cfg, rctxid, lli, CTX_CTRL_ERR_FALLBACK);
|
|
if (unlikely(!ctxi)) {
|
|
dev_dbg(dev, "%s: Bad context ctxid=%llu\n",
|
|
__func__, ctxid);
|
|
rc = -EINVAL;
|
|
goto out;
|
|
}
|
|
|
|
put_ctx = true;
|
|
}
|
|
|
|
rhte = get_rhte(ctxi, rhndl, lli);
|
|
if (unlikely(!rhte)) {
|
|
dev_dbg(dev, "%s: Bad resource handle rhndl=%d\n",
|
|
__func__, rhndl);
|
|
rc = -EINVAL;
|
|
goto out;
|
|
}
|
|
|
|
/*
|
|
* Resize to 0 for virtual LUNS by setting the size
|
|
* to 0. This will clear LXT_START and LXT_CNT fields
|
|
* in the RHT entry and properly sync with the AFU.
|
|
*
|
|
* Afterwards we clear the remaining fields.
|
|
*/
|
|
switch (gli->mode) {
|
|
case MODE_VIRTUAL:
|
|
marshal_rele_to_resize(release, &size);
|
|
size.req_size = 0;
|
|
rc = _cxlflash_vlun_resize(sdev, ctxi, &size);
|
|
if (rc) {
|
|
dev_dbg(dev, "%s: resize failed rc %d\n", __func__, rc);
|
|
goto out;
|
|
}
|
|
|
|
break;
|
|
case MODE_PHYSICAL:
|
|
/*
|
|
* Clear the Format 1 RHT entry for direct access
|
|
* (physical LUN) using the synchronization sequence
|
|
* defined in the SISLite specification.
|
|
*/
|
|
rhte_f1 = (struct sisl_rht_entry_f1 *)rhte;
|
|
|
|
rhte_f1->valid = 0;
|
|
dma_wmb(); /* Make revocation of RHT entry visible */
|
|
|
|
rhte_f1->lun_id = 0;
|
|
dma_wmb(); /* Make clearing of LUN id visible */
|
|
|
|
rhte_f1->dw = 0;
|
|
dma_wmb(); /* Make RHT entry bottom-half clearing visible */
|
|
|
|
if (!ctxi->err_recovery_active) {
|
|
rcr = cxlflash_afu_sync(afu, ctxid, rhndl, AFU_HW_SYNC);
|
|
if (unlikely(rcr))
|
|
dev_dbg(dev, "%s: AFU sync failed rc=%d\n",
|
|
__func__, rcr);
|
|
}
|
|
break;
|
|
default:
|
|
WARN(1, "Unsupported LUN mode!");
|
|
goto out;
|
|
}
|
|
|
|
rhte_checkin(ctxi, rhte);
|
|
cxlflash_lun_detach(gli);
|
|
|
|
out:
|
|
if (put_ctx)
|
|
put_context(ctxi);
|
|
dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);
|
|
return rc;
|
|
}
|
|
|
|
int cxlflash_disk_release(struct scsi_device *sdev,
|
|
struct dk_cxlflash_release *release)
|
|
{
|
|
return _cxlflash_disk_release(sdev, NULL, release);
|
|
}
|
|
|
|
/**
|
|
* destroy_context() - releases a context
|
|
* @cfg: Internal structure associated with the host.
|
|
* @ctxi: Context to release.
|
|
*
|
|
* This routine is safe to be called with a a non-initialized context.
|
|
* Also note that the routine conditionally checks for the existence
|
|
* of the context control map before clearing the RHT registers and
|
|
* context capabilities because it is possible to destroy a context
|
|
* while the context is in the error state (previous mapping was
|
|
* removed [so there is no need to worry about clearing] and context
|
|
* is waiting for a new mapping).
|
|
*/
|
|
static void destroy_context(struct cxlflash_cfg *cfg,
|
|
struct ctx_info *ctxi)
|
|
{
|
|
struct afu *afu = cfg->afu;
|
|
|
|
if (ctxi->initialized) {
|
|
WARN_ON(!list_empty(&ctxi->luns));
|
|
|
|
/* Clear RHT registers and drop all capabilities for context */
|
|
if (afu->afu_map && ctxi->ctrl_map) {
|
|
writeq_be(0, &ctxi->ctrl_map->rht_start);
|
|
writeq_be(0, &ctxi->ctrl_map->rht_cnt_id);
|
|
writeq_be(0, &ctxi->ctrl_map->ctx_cap);
|
|
}
|
|
}
|
|
|
|
/* Free memory associated with context */
|
|
free_page((ulong)ctxi->rht_start);
|
|
kfree(ctxi->rht_needs_ws);
|
|
kfree(ctxi->rht_lun);
|
|
kfree(ctxi);
|
|
}
|
|
|
|
/**
|
|
* create_context() - allocates and initializes a context
|
|
* @cfg: Internal structure associated with the host.
|
|
*
|
|
* Return: Allocated context on success, NULL on failure
|
|
*/
|
|
static struct ctx_info *create_context(struct cxlflash_cfg *cfg)
|
|
{
|
|
struct device *dev = &cfg->dev->dev;
|
|
struct ctx_info *ctxi = NULL;
|
|
struct llun_info **lli = NULL;
|
|
u8 *ws = NULL;
|
|
struct sisl_rht_entry *rhte;
|
|
|
|
ctxi = kzalloc(sizeof(*ctxi), GFP_KERNEL);
|
|
lli = kzalloc((MAX_RHT_PER_CONTEXT * sizeof(*lli)), GFP_KERNEL);
|
|
ws = kzalloc((MAX_RHT_PER_CONTEXT * sizeof(*ws)), GFP_KERNEL);
|
|
if (unlikely(!ctxi || !lli || !ws)) {
|
|
dev_err(dev, "%s: Unable to allocate context\n", __func__);
|
|
goto err;
|
|
}
|
|
|
|
rhte = (struct sisl_rht_entry *)get_zeroed_page(GFP_KERNEL);
|
|
if (unlikely(!rhte)) {
|
|
dev_err(dev, "%s: Unable to allocate RHT\n", __func__);
|
|
goto err;
|
|
}
|
|
|
|
ctxi->rht_lun = lli;
|
|
ctxi->rht_needs_ws = ws;
|
|
ctxi->rht_start = rhte;
|
|
out:
|
|
return ctxi;
|
|
|
|
err:
|
|
kfree(ws);
|
|
kfree(lli);
|
|
kfree(ctxi);
|
|
ctxi = NULL;
|
|
goto out;
|
|
}
|
|
|
|
/**
|
|
* init_context() - initializes a previously allocated context
|
|
* @ctxi: Previously allocated context
|
|
* @cfg: Internal structure associated with the host.
|
|
* @ctx: Previously obtained context cookie.
|
|
* @ctxid: Previously obtained process element associated with CXL context.
|
|
* @file: Previously obtained file associated with CXL context.
|
|
* @perms: User-specified permissions.
|
|
* @irqs: User-specified number of interrupts.
|
|
*/
|
|
static void init_context(struct ctx_info *ctxi, struct cxlflash_cfg *cfg,
|
|
void *ctx, int ctxid, struct file *file, u32 perms,
|
|
u64 irqs)
|
|
{
|
|
struct afu *afu = cfg->afu;
|
|
|
|
ctxi->rht_perms = perms;
|
|
ctxi->ctrl_map = &afu->afu_map->ctrls[ctxid].ctrl;
|
|
ctxi->ctxid = ENCODE_CTXID(ctxi, ctxid);
|
|
ctxi->irqs = irqs;
|
|
ctxi->pid = task_tgid_nr(current); /* tgid = pid */
|
|
ctxi->ctx = ctx;
|
|
ctxi->cfg = cfg;
|
|
ctxi->file = file;
|
|
ctxi->initialized = true;
|
|
mutex_init(&ctxi->mutex);
|
|
kref_init(&ctxi->kref);
|
|
INIT_LIST_HEAD(&ctxi->luns);
|
|
INIT_LIST_HEAD(&ctxi->list); /* initialize for list_empty() */
|
|
}
|
|
|
|
/**
|
|
* remove_context() - context kref release handler
|
|
* @kref: Kernel reference associated with context to be removed.
|
|
*
|
|
* When a context no longer has any references it can safely be removed
|
|
* from global access and destroyed. Note that it is assumed the thread
|
|
* relinquishing access to the context holds its mutex.
|
|
*/
|
|
static void remove_context(struct kref *kref)
|
|
{
|
|
struct ctx_info *ctxi = container_of(kref, struct ctx_info, kref);
|
|
struct cxlflash_cfg *cfg = ctxi->cfg;
|
|
u64 ctxid = DECODE_CTXID(ctxi->ctxid);
|
|
|
|
/* Remove context from table/error list */
|
|
WARN_ON(!mutex_is_locked(&ctxi->mutex));
|
|
ctxi->unavail = true;
|
|
mutex_unlock(&ctxi->mutex);
|
|
mutex_lock(&cfg->ctx_tbl_list_mutex);
|
|
mutex_lock(&ctxi->mutex);
|
|
|
|
if (!list_empty(&ctxi->list))
|
|
list_del(&ctxi->list);
|
|
cfg->ctx_tbl[ctxid] = NULL;
|
|
mutex_unlock(&cfg->ctx_tbl_list_mutex);
|
|
mutex_unlock(&ctxi->mutex);
|
|
|
|
/* Context now completely uncoupled/unreachable */
|
|
destroy_context(cfg, ctxi);
|
|
}
|
|
|
|
/**
|
|
* _cxlflash_disk_detach() - detaches a LUN from a context
|
|
* @sdev: SCSI device associated with LUN.
|
|
* @ctxi: Context owning resources.
|
|
* @detach: Detach ioctl data structure.
|
|
*
|
|
* As part of the detach, all per-context resources associated with the LUN
|
|
* are cleaned up. When detaching the last LUN for a context, the context
|
|
* itself is cleaned up and released.
|
|
*
|
|
* Return: 0 on success, -errno on failure
|
|
*/
|
|
static int _cxlflash_disk_detach(struct scsi_device *sdev,
|
|
struct ctx_info *ctxi,
|
|
struct dk_cxlflash_detach *detach)
|
|
{
|
|
struct cxlflash_cfg *cfg = shost_priv(sdev->host);
|
|
struct device *dev = &cfg->dev->dev;
|
|
struct llun_info *lli = sdev->hostdata;
|
|
struct lun_access *lun_access, *t;
|
|
struct dk_cxlflash_release rel;
|
|
bool put_ctx = false;
|
|
|
|
int i;
|
|
int rc = 0;
|
|
u64 ctxid = DECODE_CTXID(detach->context_id),
|
|
rctxid = detach->context_id;
|
|
|
|
dev_dbg(dev, "%s: ctxid=%llu\n", __func__, ctxid);
|
|
|
|
if (!ctxi) {
|
|
ctxi = get_context(cfg, rctxid, lli, CTX_CTRL_ERR_FALLBACK);
|
|
if (unlikely(!ctxi)) {
|
|
dev_dbg(dev, "%s: Bad context ctxid=%llu\n",
|
|
__func__, ctxid);
|
|
rc = -EINVAL;
|
|
goto out;
|
|
}
|
|
|
|
put_ctx = true;
|
|
}
|
|
|
|
/* Cleanup outstanding resources tied to this LUN */
|
|
if (ctxi->rht_out) {
|
|
marshal_det_to_rele(detach, &rel);
|
|
for (i = 0; i < MAX_RHT_PER_CONTEXT; i++) {
|
|
if (ctxi->rht_lun[i] == lli) {
|
|
rel.rsrc_handle = i;
|
|
_cxlflash_disk_release(sdev, ctxi, &rel);
|
|
}
|
|
|
|
/* No need to loop further if we're done */
|
|
if (ctxi->rht_out == 0)
|
|
break;
|
|
}
|
|
}
|
|
|
|
/* Take our LUN out of context, free the node */
|
|
list_for_each_entry_safe(lun_access, t, &ctxi->luns, list)
|
|
if (lun_access->lli == lli) {
|
|
list_del(&lun_access->list);
|
|
kfree(lun_access);
|
|
lun_access = NULL;
|
|
break;
|
|
}
|
|
|
|
/*
|
|
* Release the context reference and the sdev reference that
|
|
* bound this LUN to the context.
|
|
*/
|
|
if (kref_put(&ctxi->kref, remove_context))
|
|
put_ctx = false;
|
|
scsi_device_put(sdev);
|
|
out:
|
|
if (put_ctx)
|
|
put_context(ctxi);
|
|
dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);
|
|
return rc;
|
|
}
|
|
|
|
static int cxlflash_disk_detach(struct scsi_device *sdev,
|
|
struct dk_cxlflash_detach *detach)
|
|
{
|
|
return _cxlflash_disk_detach(sdev, NULL, detach);
|
|
}
|
|
|
|
/**
|
|
* cxlflash_cxl_release() - release handler for adapter file descriptor
|
|
* @inode: File-system inode associated with fd.
|
|
* @file: File installed with adapter file descriptor.
|
|
*
|
|
* This routine is the release handler for the fops registered with
|
|
* the CXL services on an initial attach for a context. It is called
|
|
* when a close (explicity by the user or as part of a process tear
|
|
* down) is performed on the adapter file descriptor returned to the
|
|
* user. The user should be aware that explicitly performing a close
|
|
* considered catastrophic and subsequent usage of the superpipe API
|
|
* with previously saved off tokens will fail.
|
|
*
|
|
* This routine derives the context reference and calls detach for
|
|
* each LUN associated with the context.The final detach operation
|
|
* causes the context itself to be freed. With exception to when the
|
|
* CXL process element (context id) lookup fails (a case that should
|
|
* theoretically never occur), every call into this routine results
|
|
* in a complete freeing of a context.
|
|
*
|
|
* Detaching the LUN is typically an ioctl() operation and the underlying
|
|
* code assumes that ioctl_rwsem has been acquired as a reader. To support
|
|
* that design point, the semaphore is acquired and released around detach.
|
|
*
|
|
* Return: 0 on success
|
|
*/
|
|
static int cxlflash_cxl_release(struct inode *inode, struct file *file)
|
|
{
|
|
struct cxlflash_cfg *cfg = container_of(file->f_op, struct cxlflash_cfg,
|
|
cxl_fops);
|
|
void *ctx = cfg->ops->fops_get_context(file);
|
|
struct device *dev = &cfg->dev->dev;
|
|
struct ctx_info *ctxi = NULL;
|
|
struct dk_cxlflash_detach detach = { { 0 }, 0 };
|
|
struct lun_access *lun_access, *t;
|
|
enum ctx_ctrl ctrl = CTX_CTRL_ERR_FALLBACK | CTX_CTRL_FILE;
|
|
int ctxid;
|
|
|
|
ctxid = cfg->ops->process_element(ctx);
|
|
if (unlikely(ctxid < 0)) {
|
|
dev_err(dev, "%s: Context %p was closed ctxid=%d\n",
|
|
__func__, ctx, ctxid);
|
|
goto out;
|
|
}
|
|
|
|
ctxi = get_context(cfg, ctxid, file, ctrl);
|
|
if (unlikely(!ctxi)) {
|
|
ctxi = get_context(cfg, ctxid, file, ctrl | CTX_CTRL_CLONE);
|
|
if (!ctxi) {
|
|
dev_dbg(dev, "%s: ctxid=%d already free\n",
|
|
__func__, ctxid);
|
|
goto out_release;
|
|
}
|
|
|
|
dev_dbg(dev, "%s: Another process owns ctxid=%d\n",
|
|
__func__, ctxid);
|
|
put_context(ctxi);
|
|
goto out;
|
|
}
|
|
|
|
dev_dbg(dev, "%s: close for ctxid=%d\n", __func__, ctxid);
|
|
|
|
down_read(&cfg->ioctl_rwsem);
|
|
detach.context_id = ctxi->ctxid;
|
|
list_for_each_entry_safe(lun_access, t, &ctxi->luns, list)
|
|
_cxlflash_disk_detach(lun_access->sdev, ctxi, &detach);
|
|
up_read(&cfg->ioctl_rwsem);
|
|
out_release:
|
|
cfg->ops->fd_release(inode, file);
|
|
out:
|
|
dev_dbg(dev, "%s: returning\n", __func__);
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* unmap_context() - clears a previously established mapping
|
|
* @ctxi: Context owning the mapping.
|
|
*
|
|
* This routine is used to switch between the error notification page
|
|
* (dummy page of all 1's) and the real mapping (established by the CXL
|
|
* fault handler).
|
|
*/
|
|
static void unmap_context(struct ctx_info *ctxi)
|
|
{
|
|
unmap_mapping_range(ctxi->file->f_mapping, 0, 0, 1);
|
|
}
|
|
|
|
/**
|
|
* get_err_page() - obtains and allocates the error notification page
|
|
* @cfg: Internal structure associated with the host.
|
|
*
|
|
* Return: error notification page on success, NULL on failure
|
|
*/
|
|
static struct page *get_err_page(struct cxlflash_cfg *cfg)
|
|
{
|
|
struct page *err_page = global.err_page;
|
|
struct device *dev = &cfg->dev->dev;
|
|
|
|
if (unlikely(!err_page)) {
|
|
err_page = alloc_page(GFP_KERNEL);
|
|
if (unlikely(!err_page)) {
|
|
dev_err(dev, "%s: Unable to allocate err_page\n",
|
|
__func__);
|
|
goto out;
|
|
}
|
|
|
|
memset(page_address(err_page), -1, PAGE_SIZE);
|
|
|
|
/* Serialize update w/ other threads to avoid a leak */
|
|
mutex_lock(&global.mutex);
|
|
if (likely(!global.err_page))
|
|
global.err_page = err_page;
|
|
else {
|
|
__free_page(err_page);
|
|
err_page = global.err_page;
|
|
}
|
|
mutex_unlock(&global.mutex);
|
|
}
|
|
|
|
out:
|
|
dev_dbg(dev, "%s: returning err_page=%p\n", __func__, err_page);
|
|
return err_page;
|
|
}
|
|
|
|
/**
|
|
* cxlflash_mmap_fault() - mmap fault handler for adapter file descriptor
|
|
* @vmf: VM fault associated with current fault.
|
|
*
|
|
* To support error notification via MMIO, faults are 'caught' by this routine
|
|
* that was inserted before passing back the adapter file descriptor on attach.
|
|
* When a fault occurs, this routine evaluates if error recovery is active and
|
|
* if so, installs the error page to 'notify' the user about the error state.
|
|
* During normal operation, the fault is simply handled by the original fault
|
|
* handler that was installed by CXL services as part of initializing the
|
|
* adapter file descriptor. The VMA's page protection bits are toggled to
|
|
* indicate cached/not-cached depending on the memory backing the fault.
|
|
*
|
|
* Return: 0 on success, VM_FAULT_SIGBUS on failure
|
|
*/
|
|
static vm_fault_t cxlflash_mmap_fault(struct vm_fault *vmf)
|
|
{
|
|
struct vm_area_struct *vma = vmf->vma;
|
|
struct file *file = vma->vm_file;
|
|
struct cxlflash_cfg *cfg = container_of(file->f_op, struct cxlflash_cfg,
|
|
cxl_fops);
|
|
void *ctx = cfg->ops->fops_get_context(file);
|
|
struct device *dev = &cfg->dev->dev;
|
|
struct ctx_info *ctxi = NULL;
|
|
struct page *err_page = NULL;
|
|
enum ctx_ctrl ctrl = CTX_CTRL_ERR_FALLBACK | CTX_CTRL_FILE;
|
|
vm_fault_t rc = 0;
|
|
int ctxid;
|
|
|
|
ctxid = cfg->ops->process_element(ctx);
|
|
if (unlikely(ctxid < 0)) {
|
|
dev_err(dev, "%s: Context %p was closed ctxid=%d\n",
|
|
__func__, ctx, ctxid);
|
|
goto err;
|
|
}
|
|
|
|
ctxi = get_context(cfg, ctxid, file, ctrl);
|
|
if (unlikely(!ctxi)) {
|
|
dev_dbg(dev, "%s: Bad context ctxid=%d\n", __func__, ctxid);
|
|
goto err;
|
|
}
|
|
|
|
dev_dbg(dev, "%s: fault for context %d\n", __func__, ctxid);
|
|
|
|
if (likely(!ctxi->err_recovery_active)) {
|
|
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
|
|
rc = ctxi->cxl_mmap_vmops->fault(vmf);
|
|
} else {
|
|
dev_dbg(dev, "%s: err recovery active, use err_page\n",
|
|
__func__);
|
|
|
|
err_page = get_err_page(cfg);
|
|
if (unlikely(!err_page)) {
|
|
dev_err(dev, "%s: Could not get err_page\n", __func__);
|
|
rc = VM_FAULT_RETRY;
|
|
goto out;
|
|
}
|
|
|
|
get_page(err_page);
|
|
vmf->page = err_page;
|
|
vma->vm_page_prot = pgprot_cached(vma->vm_page_prot);
|
|
}
|
|
|
|
out:
|
|
if (likely(ctxi))
|
|
put_context(ctxi);
|
|
dev_dbg(dev, "%s: returning rc=%x\n", __func__, rc);
|
|
return rc;
|
|
|
|
err:
|
|
rc = VM_FAULT_SIGBUS;
|
|
goto out;
|
|
}
|
|
|
|
/*
|
|
* Local MMAP vmops to 'catch' faults
|
|
*/
|
|
static const struct vm_operations_struct cxlflash_mmap_vmops = {
|
|
.fault = cxlflash_mmap_fault,
|
|
};
|
|
|
|
/**
|
|
* cxlflash_cxl_mmap() - mmap handler for adapter file descriptor
|
|
* @file: File installed with adapter file descriptor.
|
|
* @vma: VM area associated with mapping.
|
|
*
|
|
* Installs local mmap vmops to 'catch' faults for error notification support.
|
|
*
|
|
* Return: 0 on success, -errno on failure
|
|
*/
|
|
static int cxlflash_cxl_mmap(struct file *file, struct vm_area_struct *vma)
|
|
{
|
|
struct cxlflash_cfg *cfg = container_of(file->f_op, struct cxlflash_cfg,
|
|
cxl_fops);
|
|
void *ctx = cfg->ops->fops_get_context(file);
|
|
struct device *dev = &cfg->dev->dev;
|
|
struct ctx_info *ctxi = NULL;
|
|
enum ctx_ctrl ctrl = CTX_CTRL_ERR_FALLBACK | CTX_CTRL_FILE;
|
|
int ctxid;
|
|
int rc = 0;
|
|
|
|
ctxid = cfg->ops->process_element(ctx);
|
|
if (unlikely(ctxid < 0)) {
|
|
dev_err(dev, "%s: Context %p was closed ctxid=%d\n",
|
|
__func__, ctx, ctxid);
|
|
rc = -EIO;
|
|
goto out;
|
|
}
|
|
|
|
ctxi = get_context(cfg, ctxid, file, ctrl);
|
|
if (unlikely(!ctxi)) {
|
|
dev_dbg(dev, "%s: Bad context ctxid=%d\n", __func__, ctxid);
|
|
rc = -EIO;
|
|
goto out;
|
|
}
|
|
|
|
dev_dbg(dev, "%s: mmap for context %d\n", __func__, ctxid);
|
|
|
|
rc = cfg->ops->fd_mmap(file, vma);
|
|
if (likely(!rc)) {
|
|
/* Insert ourself in the mmap fault handler path */
|
|
ctxi->cxl_mmap_vmops = vma->vm_ops;
|
|
vma->vm_ops = &cxlflash_mmap_vmops;
|
|
}
|
|
|
|
out:
|
|
if (likely(ctxi))
|
|
put_context(ctxi);
|
|
return rc;
|
|
}
|
|
|
|
const struct file_operations cxlflash_cxl_fops = {
|
|
.owner = THIS_MODULE,
|
|
.mmap = cxlflash_cxl_mmap,
|
|
.release = cxlflash_cxl_release,
|
|
};
|
|
|
|
/**
|
|
* cxlflash_mark_contexts_error() - move contexts to error state and list
|
|
* @cfg: Internal structure associated with the host.
|
|
*
|
|
* A context is only moved over to the error list when there are no outstanding
|
|
* references to it. This ensures that a running operation has completed.
|
|
*
|
|
* Return: 0 on success, -errno on failure
|
|
*/
|
|
int cxlflash_mark_contexts_error(struct cxlflash_cfg *cfg)
|
|
{
|
|
int i, rc = 0;
|
|
struct ctx_info *ctxi = NULL;
|
|
|
|
mutex_lock(&cfg->ctx_tbl_list_mutex);
|
|
|
|
for (i = 0; i < MAX_CONTEXT; i++) {
|
|
ctxi = cfg->ctx_tbl[i];
|
|
if (ctxi) {
|
|
mutex_lock(&ctxi->mutex);
|
|
cfg->ctx_tbl[i] = NULL;
|
|
list_add(&ctxi->list, &cfg->ctx_err_recovery);
|
|
ctxi->err_recovery_active = true;
|
|
ctxi->ctrl_map = NULL;
|
|
unmap_context(ctxi);
|
|
mutex_unlock(&ctxi->mutex);
|
|
}
|
|
}
|
|
|
|
mutex_unlock(&cfg->ctx_tbl_list_mutex);
|
|
return rc;
|
|
}
|
|
|
|
/*
|
|
* Dummy NULL fops
|
|
*/
|
|
static const struct file_operations null_fops = {
|
|
.owner = THIS_MODULE,
|
|
};
|
|
|
|
/**
|
|
* check_state() - checks and responds to the current adapter state
|
|
* @cfg: Internal structure associated with the host.
|
|
*
|
|
* This routine can block and should only be used on process context.
|
|
* It assumes that the caller is an ioctl thread and holding the ioctl
|
|
* read semaphore. This is temporarily let up across the wait to allow
|
|
* for draining actively running ioctls. Also note that when waking up
|
|
* from waiting in reset, the state is unknown and must be checked again
|
|
* before proceeding.
|
|
*
|
|
* Return: 0 on success, -errno on failure
|
|
*/
|
|
int check_state(struct cxlflash_cfg *cfg)
|
|
{
|
|
struct device *dev = &cfg->dev->dev;
|
|
int rc = 0;
|
|
|
|
retry:
|
|
switch (cfg->state) {
|
|
case STATE_RESET:
|
|
dev_dbg(dev, "%s: Reset state, going to wait...\n", __func__);
|
|
up_read(&cfg->ioctl_rwsem);
|
|
rc = wait_event_interruptible(cfg->reset_waitq,
|
|
cfg->state != STATE_RESET);
|
|
down_read(&cfg->ioctl_rwsem);
|
|
if (unlikely(rc))
|
|
break;
|
|
goto retry;
|
|
case STATE_FAILTERM:
|
|
dev_dbg(dev, "%s: Failed/Terminating\n", __func__);
|
|
rc = -ENODEV;
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
/**
|
|
* cxlflash_disk_attach() - attach a LUN to a context
|
|
* @sdev: SCSI device associated with LUN.
|
|
* @attach: Attach ioctl data structure.
|
|
*
|
|
* Creates a context and attaches LUN to it. A LUN can only be attached
|
|
* one time to a context (subsequent attaches for the same context/LUN pair
|
|
* are not supported). Additional LUNs can be attached to a context by
|
|
* specifying the 'reuse' flag defined in the cxlflash_ioctl.h header.
|
|
*
|
|
* Return: 0 on success, -errno on failure
|
|
*/
|
|
static int cxlflash_disk_attach(struct scsi_device *sdev,
|
|
struct dk_cxlflash_attach *attach)
|
|
{
|
|
struct cxlflash_cfg *cfg = shost_priv(sdev->host);
|
|
struct device *dev = &cfg->dev->dev;
|
|
struct afu *afu = cfg->afu;
|
|
struct llun_info *lli = sdev->hostdata;
|
|
struct glun_info *gli = lli->parent;
|
|
struct ctx_info *ctxi = NULL;
|
|
struct lun_access *lun_access = NULL;
|
|
int rc = 0;
|
|
u32 perms;
|
|
int ctxid = -1;
|
|
u64 irqs = attach->num_interrupts;
|
|
u64 flags = 0UL;
|
|
u64 rctxid = 0UL;
|
|
struct file *file = NULL;
|
|
|
|
void *ctx = NULL;
|
|
|
|
int fd = -1;
|
|
|
|
if (irqs > 4) {
|
|
dev_dbg(dev, "%s: Cannot support this many interrupts %llu\n",
|
|
__func__, irqs);
|
|
rc = -EINVAL;
|
|
goto out;
|
|
}
|
|
|
|
if (gli->max_lba == 0) {
|
|
dev_dbg(dev, "%s: No capacity info for LUN=%016llx\n",
|
|
__func__, lli->lun_id[sdev->channel]);
|
|
rc = read_cap16(sdev, lli);
|
|
if (rc) {
|
|
dev_err(dev, "%s: Invalid device rc=%d\n",
|
|
__func__, rc);
|
|
rc = -ENODEV;
|
|
goto out;
|
|
}
|
|
dev_dbg(dev, "%s: LBA = %016llx\n", __func__, gli->max_lba);
|
|
dev_dbg(dev, "%s: BLK_LEN = %08x\n", __func__, gli->blk_len);
|
|
}
|
|
|
|
if (attach->hdr.flags & DK_CXLFLASH_ATTACH_REUSE_CONTEXT) {
|
|
rctxid = attach->context_id;
|
|
ctxi = get_context(cfg, rctxid, NULL, 0);
|
|
if (!ctxi) {
|
|
dev_dbg(dev, "%s: Bad context rctxid=%016llx\n",
|
|
__func__, rctxid);
|
|
rc = -EINVAL;
|
|
goto out;
|
|
}
|
|
|
|
list_for_each_entry(lun_access, &ctxi->luns, list)
|
|
if (lun_access->lli == lli) {
|
|
dev_dbg(dev, "%s: Already attached\n",
|
|
__func__);
|
|
rc = -EINVAL;
|
|
goto out;
|
|
}
|
|
}
|
|
|
|
rc = scsi_device_get(sdev);
|
|
if (unlikely(rc)) {
|
|
dev_err(dev, "%s: Unable to get sdev reference\n", __func__);
|
|
goto out;
|
|
}
|
|
|
|
lun_access = kzalloc(sizeof(*lun_access), GFP_KERNEL);
|
|
if (unlikely(!lun_access)) {
|
|
dev_err(dev, "%s: Unable to allocate lun_access\n", __func__);
|
|
rc = -ENOMEM;
|
|
goto err;
|
|
}
|
|
|
|
lun_access->lli = lli;
|
|
lun_access->sdev = sdev;
|
|
|
|
/* Non-NULL context indicates reuse (another context reference) */
|
|
if (ctxi) {
|
|
dev_dbg(dev, "%s: Reusing context for LUN rctxid=%016llx\n",
|
|
__func__, rctxid);
|
|
kref_get(&ctxi->kref);
|
|
list_add(&lun_access->list, &ctxi->luns);
|
|
goto out_attach;
|
|
}
|
|
|
|
ctxi = create_context(cfg);
|
|
if (unlikely(!ctxi)) {
|
|
dev_err(dev, "%s: Failed to create context ctxid=%d\n",
|
|
__func__, ctxid);
|
|
rc = -ENOMEM;
|
|
goto err;
|
|
}
|
|
|
|
ctx = cfg->ops->dev_context_init(cfg->dev, cfg->afu_cookie);
|
|
if (IS_ERR_OR_NULL(ctx)) {
|
|
dev_err(dev, "%s: Could not initialize context %p\n",
|
|
__func__, ctx);
|
|
rc = -ENODEV;
|
|
goto err;
|
|
}
|
|
|
|
rc = cfg->ops->start_work(ctx, irqs);
|
|
if (unlikely(rc)) {
|
|
dev_dbg(dev, "%s: Could not start context rc=%d\n",
|
|
__func__, rc);
|
|
goto err;
|
|
}
|
|
|
|
ctxid = cfg->ops->process_element(ctx);
|
|
if (unlikely((ctxid >= MAX_CONTEXT) || (ctxid < 0))) {
|
|
dev_err(dev, "%s: ctxid=%d invalid\n", __func__, ctxid);
|
|
rc = -EPERM;
|
|
goto err;
|
|
}
|
|
|
|
file = cfg->ops->get_fd(ctx, &cfg->cxl_fops, &fd);
|
|
if (unlikely(fd < 0)) {
|
|
rc = -ENODEV;
|
|
dev_err(dev, "%s: Could not get file descriptor\n", __func__);
|
|
goto err;
|
|
}
|
|
|
|
/* Translate read/write O_* flags from fcntl.h to AFU permission bits */
|
|
perms = SISL_RHT_PERM(attach->hdr.flags + 1);
|
|
|
|
/* Context mutex is locked upon return */
|
|
init_context(ctxi, cfg, ctx, ctxid, file, perms, irqs);
|
|
|
|
rc = afu_attach(cfg, ctxi);
|
|
if (unlikely(rc)) {
|
|
dev_err(dev, "%s: Could not attach AFU rc %d\n", __func__, rc);
|
|
goto err;
|
|
}
|
|
|
|
/*
|
|
* No error paths after this point. Once the fd is installed it's
|
|
* visible to user space and can't be undone safely on this thread.
|
|
* There is no need to worry about a deadlock here because no one
|
|
* knows about us yet; we can be the only one holding our mutex.
|
|
*/
|
|
list_add(&lun_access->list, &ctxi->luns);
|
|
mutex_lock(&cfg->ctx_tbl_list_mutex);
|
|
mutex_lock(&ctxi->mutex);
|
|
cfg->ctx_tbl[ctxid] = ctxi;
|
|
mutex_unlock(&cfg->ctx_tbl_list_mutex);
|
|
fd_install(fd, file);
|
|
|
|
out_attach:
|
|
if (fd != -1)
|
|
flags |= DK_CXLFLASH_APP_CLOSE_ADAP_FD;
|
|
if (afu_is_sq_cmd_mode(afu))
|
|
flags |= DK_CXLFLASH_CONTEXT_SQ_CMD_MODE;
|
|
|
|
attach->hdr.return_flags = flags;
|
|
attach->context_id = ctxi->ctxid;
|
|
attach->block_size = gli->blk_len;
|
|
attach->mmio_size = sizeof(afu->afu_map->hosts[0].harea);
|
|
attach->last_lba = gli->max_lba;
|
|
attach->max_xfer = sdev->host->max_sectors * MAX_SECTOR_UNIT;
|
|
attach->max_xfer /= gli->blk_len;
|
|
|
|
out:
|
|
attach->adap_fd = fd;
|
|
|
|
if (ctxi)
|
|
put_context(ctxi);
|
|
|
|
dev_dbg(dev, "%s: returning ctxid=%d fd=%d bs=%lld rc=%d llba=%lld\n",
|
|
__func__, ctxid, fd, attach->block_size, rc, attach->last_lba);
|
|
return rc;
|
|
|
|
err:
|
|
/* Cleanup CXL context; okay to 'stop' even if it was not started */
|
|
if (!IS_ERR_OR_NULL(ctx)) {
|
|
cfg->ops->stop_context(ctx);
|
|
cfg->ops->release_context(ctx);
|
|
ctx = NULL;
|
|
}
|
|
|
|
/*
|
|
* Here, we're overriding the fops with a dummy all-NULL fops because
|
|
* fput() calls the release fop, which will cause us to mistakenly
|
|
* call into the CXL code. Rather than try to add yet more complexity
|
|
* to that routine (cxlflash_cxl_release) we should try to fix the
|
|
* issue here.
|
|
*/
|
|
if (fd > 0) {
|
|
file->f_op = &null_fops;
|
|
fput(file);
|
|
put_unused_fd(fd);
|
|
fd = -1;
|
|
file = NULL;
|
|
}
|
|
|
|
/* Cleanup our context */
|
|
if (ctxi) {
|
|
destroy_context(cfg, ctxi);
|
|
ctxi = NULL;
|
|
}
|
|
|
|
kfree(lun_access);
|
|
scsi_device_put(sdev);
|
|
goto out;
|
|
}
|
|
|
|
/**
|
|
* recover_context() - recovers a context in error
|
|
* @cfg: Internal structure associated with the host.
|
|
* @ctxi: Context to release.
|
|
* @adap_fd: Adapter file descriptor associated with new/recovered context.
|
|
*
|
|
* Restablishes the state for a context-in-error.
|
|
*
|
|
* Return: 0 on success, -errno on failure
|
|
*/
|
|
static int recover_context(struct cxlflash_cfg *cfg,
|
|
struct ctx_info *ctxi,
|
|
int *adap_fd)
|
|
{
|
|
struct device *dev = &cfg->dev->dev;
|
|
int rc = 0;
|
|
int fd = -1;
|
|
int ctxid = -1;
|
|
struct file *file;
|
|
void *ctx;
|
|
struct afu *afu = cfg->afu;
|
|
|
|
ctx = cfg->ops->dev_context_init(cfg->dev, cfg->afu_cookie);
|
|
if (IS_ERR_OR_NULL(ctx)) {
|
|
dev_err(dev, "%s: Could not initialize context %p\n",
|
|
__func__, ctx);
|
|
rc = -ENODEV;
|
|
goto out;
|
|
}
|
|
|
|
rc = cfg->ops->start_work(ctx, ctxi->irqs);
|
|
if (unlikely(rc)) {
|
|
dev_dbg(dev, "%s: Could not start context rc=%d\n",
|
|
__func__, rc);
|
|
goto err1;
|
|
}
|
|
|
|
ctxid = cfg->ops->process_element(ctx);
|
|
if (unlikely((ctxid >= MAX_CONTEXT) || (ctxid < 0))) {
|
|
dev_err(dev, "%s: ctxid=%d invalid\n", __func__, ctxid);
|
|
rc = -EPERM;
|
|
goto err2;
|
|
}
|
|
|
|
file = cfg->ops->get_fd(ctx, &cfg->cxl_fops, &fd);
|
|
if (unlikely(fd < 0)) {
|
|
rc = -ENODEV;
|
|
dev_err(dev, "%s: Could not get file descriptor\n", __func__);
|
|
goto err2;
|
|
}
|
|
|
|
/* Update with new MMIO area based on updated context id */
|
|
ctxi->ctrl_map = &afu->afu_map->ctrls[ctxid].ctrl;
|
|
|
|
rc = afu_attach(cfg, ctxi);
|
|
if (rc) {
|
|
dev_err(dev, "%s: Could not attach AFU rc %d\n", __func__, rc);
|
|
goto err3;
|
|
}
|
|
|
|
/*
|
|
* No error paths after this point. Once the fd is installed it's
|
|
* visible to user space and can't be undone safely on this thread.
|
|
*/
|
|
ctxi->ctxid = ENCODE_CTXID(ctxi, ctxid);
|
|
ctxi->ctx = ctx;
|
|
ctxi->file = file;
|
|
|
|
/*
|
|
* Put context back in table (note the reinit of the context list);
|
|
* we must first drop the context's mutex and then acquire it in
|
|
* order with the table/list mutex to avoid a deadlock - safe to do
|
|
* here because no one can find us at this moment in time.
|
|
*/
|
|
mutex_unlock(&ctxi->mutex);
|
|
mutex_lock(&cfg->ctx_tbl_list_mutex);
|
|
mutex_lock(&ctxi->mutex);
|
|
list_del_init(&ctxi->list);
|
|
cfg->ctx_tbl[ctxid] = ctxi;
|
|
mutex_unlock(&cfg->ctx_tbl_list_mutex);
|
|
fd_install(fd, file);
|
|
*adap_fd = fd;
|
|
out:
|
|
dev_dbg(dev, "%s: returning ctxid=%d fd=%d rc=%d\n",
|
|
__func__, ctxid, fd, rc);
|
|
return rc;
|
|
|
|
err3:
|
|
fput(file);
|
|
put_unused_fd(fd);
|
|
err2:
|
|
cfg->ops->stop_context(ctx);
|
|
err1:
|
|
cfg->ops->release_context(ctx);
|
|
goto out;
|
|
}
|
|
|
|
/**
|
|
* cxlflash_afu_recover() - initiates AFU recovery
|
|
* @sdev: SCSI device associated with LUN.
|
|
* @recover: Recover ioctl data structure.
|
|
*
|
|
* Only a single recovery is allowed at a time to avoid exhausting CXL
|
|
* resources (leading to recovery failure) in the event that we're up
|
|
* against the maximum number of contexts limit. For similar reasons,
|
|
* a context recovery is retried if there are multiple recoveries taking
|
|
* place at the same time and the failure was due to CXL services being
|
|
* unable to keep up.
|
|
*
|
|
* As this routine is called on ioctl context, it holds the ioctl r/w
|
|
* semaphore that is used to drain ioctls in recovery scenarios. The
|
|
* implementation to achieve the pacing described above (a local mutex)
|
|
* requires that the ioctl r/w semaphore be dropped and reacquired to
|
|
* avoid a 3-way deadlock when multiple process recoveries operate in
|
|
* parallel.
|
|
*
|
|
* Because a user can detect an error condition before the kernel, it is
|
|
* quite possible for this routine to act as the kernel's EEH detection
|
|
* source (MMIO read of mbox_r). Because of this, there is a window of
|
|
* time where an EEH might have been detected but not yet 'serviced'
|
|
* (callback invoked, causing the device to enter reset state). To avoid
|
|
* looping in this routine during that window, a 1 second sleep is in place
|
|
* between the time the MMIO failure is detected and the time a wait on the
|
|
* reset wait queue is attempted via check_state().
|
|
*
|
|
* Return: 0 on success, -errno on failure
|
|
*/
|
|
static int cxlflash_afu_recover(struct scsi_device *sdev,
|
|
struct dk_cxlflash_recover_afu *recover)
|
|
{
|
|
struct cxlflash_cfg *cfg = shost_priv(sdev->host);
|
|
struct device *dev = &cfg->dev->dev;
|
|
struct llun_info *lli = sdev->hostdata;
|
|
struct afu *afu = cfg->afu;
|
|
struct ctx_info *ctxi = NULL;
|
|
struct mutex *mutex = &cfg->ctx_recovery_mutex;
|
|
struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ);
|
|
u64 flags;
|
|
u64 ctxid = DECODE_CTXID(recover->context_id),
|
|
rctxid = recover->context_id;
|
|
long reg;
|
|
bool locked = true;
|
|
int lretry = 20; /* up to 2 seconds */
|
|
int new_adap_fd = -1;
|
|
int rc = 0;
|
|
|
|
atomic_inc(&cfg->recovery_threads);
|
|
up_read(&cfg->ioctl_rwsem);
|
|
rc = mutex_lock_interruptible(mutex);
|
|
down_read(&cfg->ioctl_rwsem);
|
|
if (rc) {
|
|
locked = false;
|
|
goto out;
|
|
}
|
|
|
|
rc = check_state(cfg);
|
|
if (rc) {
|
|
dev_err(dev, "%s: Failed state rc=%d\n", __func__, rc);
|
|
rc = -ENODEV;
|
|
goto out;
|
|
}
|
|
|
|
dev_dbg(dev, "%s: reason=%016llx rctxid=%016llx\n",
|
|
__func__, recover->reason, rctxid);
|
|
|
|
retry:
|
|
/* Ensure that this process is attached to the context */
|
|
ctxi = get_context(cfg, rctxid, lli, CTX_CTRL_ERR_FALLBACK);
|
|
if (unlikely(!ctxi)) {
|
|
dev_dbg(dev, "%s: Bad context ctxid=%llu\n", __func__, ctxid);
|
|
rc = -EINVAL;
|
|
goto out;
|
|
}
|
|
|
|
if (ctxi->err_recovery_active) {
|
|
retry_recover:
|
|
rc = recover_context(cfg, ctxi, &new_adap_fd);
|
|
if (unlikely(rc)) {
|
|
dev_err(dev, "%s: Recovery failed ctxid=%llu rc=%d\n",
|
|
__func__, ctxid, rc);
|
|
if ((rc == -ENODEV) &&
|
|
((atomic_read(&cfg->recovery_threads) > 1) ||
|
|
(lretry--))) {
|
|
dev_dbg(dev, "%s: Going to try again\n",
|
|
__func__);
|
|
mutex_unlock(mutex);
|
|
msleep(100);
|
|
rc = mutex_lock_interruptible(mutex);
|
|
if (rc) {
|
|
locked = false;
|
|
goto out;
|
|
}
|
|
goto retry_recover;
|
|
}
|
|
|
|
goto out;
|
|
}
|
|
|
|
ctxi->err_recovery_active = false;
|
|
|
|
flags = DK_CXLFLASH_APP_CLOSE_ADAP_FD |
|
|
DK_CXLFLASH_RECOVER_AFU_CONTEXT_RESET;
|
|
if (afu_is_sq_cmd_mode(afu))
|
|
flags |= DK_CXLFLASH_CONTEXT_SQ_CMD_MODE;
|
|
|
|
recover->hdr.return_flags = flags;
|
|
recover->context_id = ctxi->ctxid;
|
|
recover->adap_fd = new_adap_fd;
|
|
recover->mmio_size = sizeof(afu->afu_map->hosts[0].harea);
|
|
goto out;
|
|
}
|
|
|
|
/* Test if in error state */
|
|
reg = readq_be(&hwq->ctrl_map->mbox_r);
|
|
if (reg == -1) {
|
|
dev_dbg(dev, "%s: MMIO fail, wait for recovery.\n", __func__);
|
|
|
|
/*
|
|
* Before checking the state, put back the context obtained with
|
|
* get_context() as it is no longer needed and sleep for a short
|
|
* period of time (see prolog notes).
|
|
*/
|
|
put_context(ctxi);
|
|
ctxi = NULL;
|
|
ssleep(1);
|
|
rc = check_state(cfg);
|
|
if (unlikely(rc))
|
|
goto out;
|
|
goto retry;
|
|
}
|
|
|
|
dev_dbg(dev, "%s: MMIO working, no recovery required\n", __func__);
|
|
out:
|
|
if (likely(ctxi))
|
|
put_context(ctxi);
|
|
if (locked)
|
|
mutex_unlock(mutex);
|
|
atomic_dec_if_positive(&cfg->recovery_threads);
|
|
return rc;
|
|
}
|
|
|
|
/**
|
|
* process_sense() - evaluates and processes sense data
|
|
* @sdev: SCSI device associated with LUN.
|
|
* @verify: Verify ioctl data structure.
|
|
*
|
|
* Return: 0 on success, -errno on failure
|
|
*/
|
|
static int process_sense(struct scsi_device *sdev,
|
|
struct dk_cxlflash_verify *verify)
|
|
{
|
|
struct cxlflash_cfg *cfg = shost_priv(sdev->host);
|
|
struct device *dev = &cfg->dev->dev;
|
|
struct llun_info *lli = sdev->hostdata;
|
|
struct glun_info *gli = lli->parent;
|
|
u64 prev_lba = gli->max_lba;
|
|
struct scsi_sense_hdr sshdr = { 0 };
|
|
int rc = 0;
|
|
|
|
rc = scsi_normalize_sense((const u8 *)&verify->sense_data,
|
|
DK_CXLFLASH_VERIFY_SENSE_LEN, &sshdr);
|
|
if (!rc) {
|
|
dev_err(dev, "%s: Failed to normalize sense data\n", __func__);
|
|
rc = -EINVAL;
|
|
goto out;
|
|
}
|
|
|
|
switch (sshdr.sense_key) {
|
|
case NO_SENSE:
|
|
case RECOVERED_ERROR:
|
|
case NOT_READY:
|
|
break;
|
|
case UNIT_ATTENTION:
|
|
switch (sshdr.asc) {
|
|
case 0x29: /* Power on Reset or Device Reset */
|
|
fallthrough;
|
|
case 0x2A: /* Device settings/capacity changed */
|
|
rc = read_cap16(sdev, lli);
|
|
if (rc) {
|
|
rc = -ENODEV;
|
|
break;
|
|
}
|
|
if (prev_lba != gli->max_lba)
|
|
dev_dbg(dev, "%s: Capacity changed old=%lld "
|
|
"new=%lld\n", __func__, prev_lba,
|
|
gli->max_lba);
|
|
break;
|
|
case 0x3F: /* Report LUNs changed, Rescan. */
|
|
scsi_scan_host(cfg->host);
|
|
break;
|
|
default:
|
|
rc = -EIO;
|
|
break;
|
|
}
|
|
break;
|
|
default:
|
|
rc = -EIO;
|
|
break;
|
|
}
|
|
out:
|
|
dev_dbg(dev, "%s: sense_key %x asc %x ascq %x rc %d\n", __func__,
|
|
sshdr.sense_key, sshdr.asc, sshdr.ascq, rc);
|
|
return rc;
|
|
}
|
|
|
|
/**
|
|
* cxlflash_disk_verify() - verifies a LUN is the same and handle size changes
|
|
* @sdev: SCSI device associated with LUN.
|
|
* @verify: Verify ioctl data structure.
|
|
*
|
|
* Return: 0 on success, -errno on failure
|
|
*/
|
|
static int cxlflash_disk_verify(struct scsi_device *sdev,
|
|
struct dk_cxlflash_verify *verify)
|
|
{
|
|
int rc = 0;
|
|
struct ctx_info *ctxi = NULL;
|
|
struct cxlflash_cfg *cfg = shost_priv(sdev->host);
|
|
struct device *dev = &cfg->dev->dev;
|
|
struct llun_info *lli = sdev->hostdata;
|
|
struct glun_info *gli = lli->parent;
|
|
struct sisl_rht_entry *rhte = NULL;
|
|
res_hndl_t rhndl = verify->rsrc_handle;
|
|
u64 ctxid = DECODE_CTXID(verify->context_id),
|
|
rctxid = verify->context_id;
|
|
u64 last_lba = 0;
|
|
|
|
dev_dbg(dev, "%s: ctxid=%llu rhndl=%016llx, hint=%016llx, "
|
|
"flags=%016llx\n", __func__, ctxid, verify->rsrc_handle,
|
|
verify->hint, verify->hdr.flags);
|
|
|
|
ctxi = get_context(cfg, rctxid, lli, 0);
|
|
if (unlikely(!ctxi)) {
|
|
dev_dbg(dev, "%s: Bad context ctxid=%llu\n", __func__, ctxid);
|
|
rc = -EINVAL;
|
|
goto out;
|
|
}
|
|
|
|
rhte = get_rhte(ctxi, rhndl, lli);
|
|
if (unlikely(!rhte)) {
|
|
dev_dbg(dev, "%s: Bad resource handle rhndl=%d\n",
|
|
__func__, rhndl);
|
|
rc = -EINVAL;
|
|
goto out;
|
|
}
|
|
|
|
/*
|
|
* Look at the hint/sense to see if it requires us to redrive
|
|
* inquiry (i.e. the Unit attention is due to the WWN changing).
|
|
*/
|
|
if (verify->hint & DK_CXLFLASH_VERIFY_HINT_SENSE) {
|
|
/* Can't hold mutex across process_sense/read_cap16,
|
|
* since we could have an intervening EEH event.
|
|
*/
|
|
ctxi->unavail = true;
|
|
mutex_unlock(&ctxi->mutex);
|
|
rc = process_sense(sdev, verify);
|
|
if (unlikely(rc)) {
|
|
dev_err(dev, "%s: Failed to validate sense data (%d)\n",
|
|
__func__, rc);
|
|
mutex_lock(&ctxi->mutex);
|
|
ctxi->unavail = false;
|
|
goto out;
|
|
}
|
|
mutex_lock(&ctxi->mutex);
|
|
ctxi->unavail = false;
|
|
}
|
|
|
|
switch (gli->mode) {
|
|
case MODE_PHYSICAL:
|
|
last_lba = gli->max_lba;
|
|
break;
|
|
case MODE_VIRTUAL:
|
|
/* Cast lxt_cnt to u64 for multiply to be treated as 64bit op */
|
|
last_lba = ((u64)rhte->lxt_cnt * MC_CHUNK_SIZE * gli->blk_len);
|
|
last_lba /= CXLFLASH_BLOCK_SIZE;
|
|
last_lba--;
|
|
break;
|
|
default:
|
|
WARN(1, "Unsupported LUN mode!");
|
|
}
|
|
|
|
verify->last_lba = last_lba;
|
|
|
|
out:
|
|
if (likely(ctxi))
|
|
put_context(ctxi);
|
|
dev_dbg(dev, "%s: returning rc=%d llba=%llx\n",
|
|
__func__, rc, verify->last_lba);
|
|
return rc;
|
|
}
|
|
|
|
/**
|
|
* decode_ioctl() - translates an encoded ioctl to an easily identifiable string
|
|
* @cmd: The ioctl command to decode.
|
|
*
|
|
* Return: A string identifying the decoded ioctl.
|
|
*/
|
|
static char *decode_ioctl(unsigned int cmd)
|
|
{
|
|
switch (cmd) {
|
|
case DK_CXLFLASH_ATTACH:
|
|
return __stringify_1(DK_CXLFLASH_ATTACH);
|
|
case DK_CXLFLASH_USER_DIRECT:
|
|
return __stringify_1(DK_CXLFLASH_USER_DIRECT);
|
|
case DK_CXLFLASH_USER_VIRTUAL:
|
|
return __stringify_1(DK_CXLFLASH_USER_VIRTUAL);
|
|
case DK_CXLFLASH_VLUN_RESIZE:
|
|
return __stringify_1(DK_CXLFLASH_VLUN_RESIZE);
|
|
case DK_CXLFLASH_RELEASE:
|
|
return __stringify_1(DK_CXLFLASH_RELEASE);
|
|
case DK_CXLFLASH_DETACH:
|
|
return __stringify_1(DK_CXLFLASH_DETACH);
|
|
case DK_CXLFLASH_VERIFY:
|
|
return __stringify_1(DK_CXLFLASH_VERIFY);
|
|
case DK_CXLFLASH_VLUN_CLONE:
|
|
return __stringify_1(DK_CXLFLASH_VLUN_CLONE);
|
|
case DK_CXLFLASH_RECOVER_AFU:
|
|
return __stringify_1(DK_CXLFLASH_RECOVER_AFU);
|
|
case DK_CXLFLASH_MANAGE_LUN:
|
|
return __stringify_1(DK_CXLFLASH_MANAGE_LUN);
|
|
}
|
|
|
|
return "UNKNOWN";
|
|
}
|
|
|
|
/**
|
|
* cxlflash_disk_direct_open() - opens a direct (physical) disk
|
|
* @sdev: SCSI device associated with LUN.
|
|
* @arg: UDirect ioctl data structure.
|
|
*
|
|
* On successful return, the user is informed of the resource handle
|
|
* to be used to identify the direct lun and the size (in blocks) of
|
|
* the direct lun in last LBA format.
|
|
*
|
|
* Return: 0 on success, -errno on failure
|
|
*/
|
|
static int cxlflash_disk_direct_open(struct scsi_device *sdev, void *arg)
|
|
{
|
|
struct cxlflash_cfg *cfg = shost_priv(sdev->host);
|
|
struct device *dev = &cfg->dev->dev;
|
|
struct afu *afu = cfg->afu;
|
|
struct llun_info *lli = sdev->hostdata;
|
|
struct glun_info *gli = lli->parent;
|
|
struct dk_cxlflash_release rel = { { 0 }, 0 };
|
|
|
|
struct dk_cxlflash_udirect *pphys = (struct dk_cxlflash_udirect *)arg;
|
|
|
|
u64 ctxid = DECODE_CTXID(pphys->context_id),
|
|
rctxid = pphys->context_id;
|
|
u64 lun_size = 0;
|
|
u64 last_lba = 0;
|
|
u64 rsrc_handle = -1;
|
|
u32 port = CHAN2PORTMASK(sdev->channel);
|
|
|
|
int rc = 0;
|
|
|
|
struct ctx_info *ctxi = NULL;
|
|
struct sisl_rht_entry *rhte = NULL;
|
|
|
|
dev_dbg(dev, "%s: ctxid=%llu ls=%llu\n", __func__, ctxid, lun_size);
|
|
|
|
rc = cxlflash_lun_attach(gli, MODE_PHYSICAL, false);
|
|
if (unlikely(rc)) {
|
|
dev_dbg(dev, "%s: Failed attach to LUN (PHYSICAL)\n", __func__);
|
|
goto out;
|
|
}
|
|
|
|
ctxi = get_context(cfg, rctxid, lli, 0);
|
|
if (unlikely(!ctxi)) {
|
|
dev_dbg(dev, "%s: Bad context ctxid=%llu\n", __func__, ctxid);
|
|
rc = -EINVAL;
|
|
goto err1;
|
|
}
|
|
|
|
rhte = rhte_checkout(ctxi, lli);
|
|
if (unlikely(!rhte)) {
|
|
dev_dbg(dev, "%s: Too many opens ctxid=%lld\n",
|
|
__func__, ctxid);
|
|
rc = -EMFILE; /* too many opens */
|
|
goto err1;
|
|
}
|
|
|
|
rsrc_handle = (rhte - ctxi->rht_start);
|
|
|
|
rht_format1(rhte, lli->lun_id[sdev->channel], ctxi->rht_perms, port);
|
|
|
|
last_lba = gli->max_lba;
|
|
pphys->hdr.return_flags = 0;
|
|
pphys->last_lba = last_lba;
|
|
pphys->rsrc_handle = rsrc_handle;
|
|
|
|
rc = cxlflash_afu_sync(afu, ctxid, rsrc_handle, AFU_LW_SYNC);
|
|
if (unlikely(rc)) {
|
|
dev_dbg(dev, "%s: AFU sync failed rc=%d\n", __func__, rc);
|
|
goto err2;
|
|
}
|
|
|
|
out:
|
|
if (likely(ctxi))
|
|
put_context(ctxi);
|
|
dev_dbg(dev, "%s: returning handle=%llu rc=%d llba=%llu\n",
|
|
__func__, rsrc_handle, rc, last_lba);
|
|
return rc;
|
|
|
|
err2:
|
|
marshal_udir_to_rele(pphys, &rel);
|
|
_cxlflash_disk_release(sdev, ctxi, &rel);
|
|
goto out;
|
|
err1:
|
|
cxlflash_lun_detach(gli);
|
|
goto out;
|
|
}
|
|
|
|
/**
|
|
* ioctl_common() - common IOCTL handler for driver
|
|
* @sdev: SCSI device associated with LUN.
|
|
* @cmd: IOCTL command.
|
|
*
|
|
* Handles common fencing operations that are valid for multiple ioctls. Always
|
|
* allow through ioctls that are cleanup oriented in nature, even when operating
|
|
* in a failed/terminating state.
|
|
*
|
|
* Return: 0 on success, -errno on failure
|
|
*/
|
|
static int ioctl_common(struct scsi_device *sdev, unsigned int cmd)
|
|
{
|
|
struct cxlflash_cfg *cfg = shost_priv(sdev->host);
|
|
struct device *dev = &cfg->dev->dev;
|
|
struct llun_info *lli = sdev->hostdata;
|
|
int rc = 0;
|
|
|
|
if (unlikely(!lli)) {
|
|
dev_dbg(dev, "%s: Unknown LUN\n", __func__);
|
|
rc = -EINVAL;
|
|
goto out;
|
|
}
|
|
|
|
rc = check_state(cfg);
|
|
if (unlikely(rc) && (cfg->state == STATE_FAILTERM)) {
|
|
switch (cmd) {
|
|
case DK_CXLFLASH_VLUN_RESIZE:
|
|
case DK_CXLFLASH_RELEASE:
|
|
case DK_CXLFLASH_DETACH:
|
|
dev_dbg(dev, "%s: Command override rc=%d\n",
|
|
__func__, rc);
|
|
rc = 0;
|
|
break;
|
|
}
|
|
}
|
|
out:
|
|
return rc;
|
|
}
|
|
|
|
/**
|
|
* cxlflash_ioctl() - IOCTL handler for driver
|
|
* @sdev: SCSI device associated with LUN.
|
|
* @cmd: IOCTL command.
|
|
* @arg: Userspace ioctl data structure.
|
|
*
|
|
* A read/write semaphore is used to implement a 'drain' of currently
|
|
* running ioctls. The read semaphore is taken at the beginning of each
|
|
* ioctl thread and released upon concluding execution. Additionally the
|
|
* semaphore should be released and then reacquired in any ioctl execution
|
|
* path which will wait for an event to occur that is outside the scope of
|
|
* the ioctl (i.e. an adapter reset). To drain the ioctls currently running,
|
|
* a thread simply needs to acquire the write semaphore.
|
|
*
|
|
* Return: 0 on success, -errno on failure
|
|
*/
|
|
int cxlflash_ioctl(struct scsi_device *sdev, unsigned int cmd, void __user *arg)
|
|
{
|
|
typedef int (*sioctl) (struct scsi_device *, void *);
|
|
|
|
struct cxlflash_cfg *cfg = shost_priv(sdev->host);
|
|
struct device *dev = &cfg->dev->dev;
|
|
struct afu *afu = cfg->afu;
|
|
struct dk_cxlflash_hdr *hdr;
|
|
char buf[sizeof(union cxlflash_ioctls)];
|
|
size_t size = 0;
|
|
bool known_ioctl = false;
|
|
int idx;
|
|
int rc = 0;
|
|
struct Scsi_Host *shost = sdev->host;
|
|
sioctl do_ioctl = NULL;
|
|
|
|
static const struct {
|
|
size_t size;
|
|
sioctl ioctl;
|
|
} ioctl_tbl[] = { /* NOTE: order matters here */
|
|
{sizeof(struct dk_cxlflash_attach), (sioctl)cxlflash_disk_attach},
|
|
{sizeof(struct dk_cxlflash_udirect), cxlflash_disk_direct_open},
|
|
{sizeof(struct dk_cxlflash_release), (sioctl)cxlflash_disk_release},
|
|
{sizeof(struct dk_cxlflash_detach), (sioctl)cxlflash_disk_detach},
|
|
{sizeof(struct dk_cxlflash_verify), (sioctl)cxlflash_disk_verify},
|
|
{sizeof(struct dk_cxlflash_recover_afu), (sioctl)cxlflash_afu_recover},
|
|
{sizeof(struct dk_cxlflash_manage_lun), (sioctl)cxlflash_manage_lun},
|
|
{sizeof(struct dk_cxlflash_uvirtual), cxlflash_disk_virtual_open},
|
|
{sizeof(struct dk_cxlflash_resize), (sioctl)cxlflash_vlun_resize},
|
|
{sizeof(struct dk_cxlflash_clone), (sioctl)cxlflash_disk_clone},
|
|
};
|
|
|
|
/* Hold read semaphore so we can drain if needed */
|
|
down_read(&cfg->ioctl_rwsem);
|
|
|
|
/* Restrict command set to physical support only for internal LUN */
|
|
if (afu->internal_lun)
|
|
switch (cmd) {
|
|
case DK_CXLFLASH_RELEASE:
|
|
case DK_CXLFLASH_USER_VIRTUAL:
|
|
case DK_CXLFLASH_VLUN_RESIZE:
|
|
case DK_CXLFLASH_VLUN_CLONE:
|
|
dev_dbg(dev, "%s: %s not supported for lun_mode=%d\n",
|
|
__func__, decode_ioctl(cmd), afu->internal_lun);
|
|
rc = -EINVAL;
|
|
goto cxlflash_ioctl_exit;
|
|
}
|
|
|
|
switch (cmd) {
|
|
case DK_CXLFLASH_ATTACH:
|
|
case DK_CXLFLASH_USER_DIRECT:
|
|
case DK_CXLFLASH_RELEASE:
|
|
case DK_CXLFLASH_DETACH:
|
|
case DK_CXLFLASH_VERIFY:
|
|
case DK_CXLFLASH_RECOVER_AFU:
|
|
case DK_CXLFLASH_USER_VIRTUAL:
|
|
case DK_CXLFLASH_VLUN_RESIZE:
|
|
case DK_CXLFLASH_VLUN_CLONE:
|
|
dev_dbg(dev, "%s: %s (%08X) on dev(%d/%d/%d/%llu)\n",
|
|
__func__, decode_ioctl(cmd), cmd, shost->host_no,
|
|
sdev->channel, sdev->id, sdev->lun);
|
|
rc = ioctl_common(sdev, cmd);
|
|
if (unlikely(rc))
|
|
goto cxlflash_ioctl_exit;
|
|
|
|
fallthrough;
|
|
|
|
case DK_CXLFLASH_MANAGE_LUN:
|
|
known_ioctl = true;
|
|
idx = _IOC_NR(cmd) - _IOC_NR(DK_CXLFLASH_ATTACH);
|
|
size = ioctl_tbl[idx].size;
|
|
do_ioctl = ioctl_tbl[idx].ioctl;
|
|
|
|
if (likely(do_ioctl))
|
|
break;
|
|
|
|
fallthrough;
|
|
default:
|
|
rc = -EINVAL;
|
|
goto cxlflash_ioctl_exit;
|
|
}
|
|
|
|
if (unlikely(copy_from_user(&buf, arg, size))) {
|
|
dev_err(dev, "%s: copy_from_user() fail size=%lu cmd=%u (%s) arg=%p\n",
|
|
__func__, size, cmd, decode_ioctl(cmd), arg);
|
|
rc = -EFAULT;
|
|
goto cxlflash_ioctl_exit;
|
|
}
|
|
|
|
hdr = (struct dk_cxlflash_hdr *)&buf;
|
|
if (hdr->version != DK_CXLFLASH_VERSION_0) {
|
|
dev_dbg(dev, "%s: Version %u not supported for %s\n",
|
|
__func__, hdr->version, decode_ioctl(cmd));
|
|
rc = -EINVAL;
|
|
goto cxlflash_ioctl_exit;
|
|
}
|
|
|
|
if (hdr->rsvd[0] || hdr->rsvd[1] || hdr->rsvd[2] || hdr->return_flags) {
|
|
dev_dbg(dev, "%s: Reserved/rflags populated\n", __func__);
|
|
rc = -EINVAL;
|
|
goto cxlflash_ioctl_exit;
|
|
}
|
|
|
|
rc = do_ioctl(sdev, (void *)&buf);
|
|
if (likely(!rc))
|
|
if (unlikely(copy_to_user(arg, &buf, size))) {
|
|
dev_err(dev, "%s: copy_to_user() fail size=%lu cmd=%u (%s) arg=%p\n",
|
|
__func__, size, cmd, decode_ioctl(cmd), arg);
|
|
rc = -EFAULT;
|
|
}
|
|
|
|
/* fall through to exit */
|
|
|
|
cxlflash_ioctl_exit:
|
|
up_read(&cfg->ioctl_rwsem);
|
|
if (unlikely(rc && known_ioctl))
|
|
dev_err(dev, "%s: ioctl %s (%08X) on dev(%d/%d/%d/%llu) "
|
|
"returned rc %d\n", __func__,
|
|
decode_ioctl(cmd), cmd, shost->host_no,
|
|
sdev->channel, sdev->id, sdev->lun, rc);
|
|
else
|
|
dev_dbg(dev, "%s: ioctl %s (%08X) on dev(%d/%d/%d/%llu) "
|
|
"returned rc %d\n", __func__, decode_ioctl(cmd),
|
|
cmd, shost->host_no, sdev->channel, sdev->id,
|
|
sdev->lun, rc);
|
|
return rc;
|
|
}
|