mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-10-02 15:18:19 +00:00
virtio: refactor find_vqs
This refactors find_vqs, making it more readable and robust, and fixing two regressions from 2.6.30: - double free_irq causing BUG_ON on device removal - probe failure when vq can't be assigned to msi-x vector (reported on old host kernels) Tested-by: Amit Shah <amit.shah@redhat.com> Signed-off-by: Michael S. Tsirkin <mst@redhat.com> Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
This commit is contained in:
parent
f6c8250703
commit
e969fed542
1 changed files with 119 additions and 93 deletions
|
@ -52,8 +52,10 @@ struct virtio_pci_device
|
||||||
char (*msix_names)[256];
|
char (*msix_names)[256];
|
||||||
/* Number of available vectors */
|
/* Number of available vectors */
|
||||||
unsigned msix_vectors;
|
unsigned msix_vectors;
|
||||||
/* Vectors allocated */
|
/* Vectors allocated, excluding per-vq vectors if any */
|
||||||
unsigned msix_used_vectors;
|
unsigned msix_used_vectors;
|
||||||
|
/* Whether we have vector per vq */
|
||||||
|
bool per_vq_vectors;
|
||||||
};
|
};
|
||||||
|
|
||||||
/* Constants for MSI-X */
|
/* Constants for MSI-X */
|
||||||
|
@ -278,27 +280,24 @@ static void vp_free_vectors(struct virtio_device *vdev)
|
||||||
vp_dev->msix_entries = NULL;
|
vp_dev->msix_entries = NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
static int vp_enable_msix(struct pci_dev *dev, struct msix_entry *entries,
|
static int vp_request_vectors(struct virtio_device *vdev, int nvectors,
|
||||||
int *options, int noptions)
|
bool per_vq_vectors)
|
||||||
{
|
|
||||||
int i;
|
|
||||||
for (i = 0; i < noptions; ++i)
|
|
||||||
if (!pci_enable_msix(dev, entries, options[i]))
|
|
||||||
return options[i];
|
|
||||||
return -EBUSY;
|
|
||||||
}
|
|
||||||
|
|
||||||
static int vp_request_vectors(struct virtio_device *vdev, unsigned max_vqs)
|
|
||||||
{
|
{
|
||||||
struct virtio_pci_device *vp_dev = to_vp_device(vdev);
|
struct virtio_pci_device *vp_dev = to_vp_device(vdev);
|
||||||
const char *name = dev_name(&vp_dev->vdev.dev);
|
const char *name = dev_name(&vp_dev->vdev.dev);
|
||||||
unsigned i, v;
|
unsigned i, v;
|
||||||
int err = -ENOMEM;
|
int err = -ENOMEM;
|
||||||
/* We want at most one vector per queue and one for config changes.
|
|
||||||
* Fallback to separate vectors for config and a shared for queues.
|
if (!nvectors) {
|
||||||
* Finally fall back to regular interrupts. */
|
/* Can't allocate MSI-X vectors, use regular interrupt */
|
||||||
int options[] = { max_vqs + 1, 2 };
|
vp_dev->msix_vectors = 0;
|
||||||
int nvectors = max(options[0], options[1]);
|
err = request_irq(vp_dev->pci_dev->irq, vp_interrupt,
|
||||||
|
IRQF_SHARED, name, vp_dev);
|
||||||
|
if (err)
|
||||||
|
return err;
|
||||||
|
vp_dev->intx_enabled = 1;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
vp_dev->msix_entries = kmalloc(nvectors * sizeof *vp_dev->msix_entries,
|
vp_dev->msix_entries = kmalloc(nvectors * sizeof *vp_dev->msix_entries,
|
||||||
GFP_KERNEL);
|
GFP_KERNEL);
|
||||||
|
@ -312,41 +311,34 @@ static int vp_request_vectors(struct virtio_device *vdev, unsigned max_vqs)
|
||||||
for (i = 0; i < nvectors; ++i)
|
for (i = 0; i < nvectors; ++i)
|
||||||
vp_dev->msix_entries[i].entry = i;
|
vp_dev->msix_entries[i].entry = i;
|
||||||
|
|
||||||
err = vp_enable_msix(vp_dev->pci_dev, vp_dev->msix_entries,
|
err = pci_enable_msix(vp_dev->pci_dev, vp_dev->msix_entries, nvectors);
|
||||||
options, ARRAY_SIZE(options));
|
if (err > 0)
|
||||||
if (err < 0) {
|
err = -ENOSPC;
|
||||||
/* Can't allocate enough MSI-X vectors, use regular interrupt */
|
if (err)
|
||||||
vp_dev->msix_vectors = 0;
|
goto error;
|
||||||
err = request_irq(vp_dev->pci_dev->irq, vp_interrupt,
|
vp_dev->msix_vectors = nvectors;
|
||||||
IRQF_SHARED, name, vp_dev);
|
vp_dev->msix_enabled = 1;
|
||||||
if (err)
|
|
||||||
goto error;
|
|
||||||
vp_dev->intx_enabled = 1;
|
|
||||||
} else {
|
|
||||||
vp_dev->msix_vectors = err;
|
|
||||||
vp_dev->msix_enabled = 1;
|
|
||||||
|
|
||||||
/* Set the vector used for configuration */
|
/* Set the vector used for configuration */
|
||||||
v = vp_dev->msix_used_vectors;
|
v = vp_dev->msix_used_vectors;
|
||||||
snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
|
snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
|
||||||
"%s-config", name);
|
"%s-config", name);
|
||||||
err = request_irq(vp_dev->msix_entries[v].vector,
|
err = request_irq(vp_dev->msix_entries[v].vector,
|
||||||
vp_config_changed, 0, vp_dev->msix_names[v],
|
vp_config_changed, 0, vp_dev->msix_names[v],
|
||||||
vp_dev);
|
vp_dev);
|
||||||
if (err)
|
if (err)
|
||||||
goto error;
|
goto error;
|
||||||
++vp_dev->msix_used_vectors;
|
++vp_dev->msix_used_vectors;
|
||||||
|
|
||||||
iowrite16(v, vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
|
iowrite16(v, vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
|
||||||
/* Verify we had enough resources to assign the vector */
|
/* Verify we had enough resources to assign the vector */
|
||||||
v = ioread16(vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
|
v = ioread16(vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
|
||||||
if (v == VIRTIO_MSI_NO_VECTOR) {
|
if (v == VIRTIO_MSI_NO_VECTOR) {
|
||||||
err = -EBUSY;
|
err = -EBUSY;
|
||||||
goto error;
|
goto error;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (vp_dev->msix_vectors && vp_dev->msix_vectors != max_vqs + 1) {
|
if (!per_vq_vectors) {
|
||||||
/* Shared vector for all VQs */
|
/* Shared vector for all VQs */
|
||||||
v = vp_dev->msix_used_vectors;
|
v = vp_dev->msix_used_vectors;
|
||||||
snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
|
snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
|
||||||
|
@ -366,13 +358,14 @@ static int vp_request_vectors(struct virtio_device *vdev, unsigned max_vqs)
|
||||||
|
|
||||||
static struct virtqueue *vp_find_vq(struct virtio_device *vdev, unsigned index,
|
static struct virtqueue *vp_find_vq(struct virtio_device *vdev, unsigned index,
|
||||||
void (*callback)(struct virtqueue *vq),
|
void (*callback)(struct virtqueue *vq),
|
||||||
const char *name)
|
const char *name,
|
||||||
|
u16 vector)
|
||||||
{
|
{
|
||||||
struct virtio_pci_device *vp_dev = to_vp_device(vdev);
|
struct virtio_pci_device *vp_dev = to_vp_device(vdev);
|
||||||
struct virtio_pci_vq_info *info;
|
struct virtio_pci_vq_info *info;
|
||||||
struct virtqueue *vq;
|
struct virtqueue *vq;
|
||||||
unsigned long flags, size;
|
unsigned long flags, size;
|
||||||
u16 num, vector;
|
u16 num;
|
||||||
int err;
|
int err;
|
||||||
|
|
||||||
/* Select the queue we're interested in */
|
/* Select the queue we're interested in */
|
||||||
|
@ -391,7 +384,7 @@ static struct virtqueue *vp_find_vq(struct virtio_device *vdev, unsigned index,
|
||||||
|
|
||||||
info->queue_index = index;
|
info->queue_index = index;
|
||||||
info->num = num;
|
info->num = num;
|
||||||
info->vector = VIRTIO_MSI_NO_VECTOR;
|
info->vector = vector;
|
||||||
|
|
||||||
size = PAGE_ALIGN(vring_size(num, VIRTIO_PCI_VRING_ALIGN));
|
size = PAGE_ALIGN(vring_size(num, VIRTIO_PCI_VRING_ALIGN));
|
||||||
info->queue = alloc_pages_exact(size, GFP_KERNEL|__GFP_ZERO);
|
info->queue = alloc_pages_exact(size, GFP_KERNEL|__GFP_ZERO);
|
||||||
|
@ -415,22 +408,7 @@ static struct virtqueue *vp_find_vq(struct virtio_device *vdev, unsigned index,
|
||||||
vq->priv = info;
|
vq->priv = info;
|
||||||
info->vq = vq;
|
info->vq = vq;
|
||||||
|
|
||||||
/* allocate per-vq vector if available and necessary */
|
if (vector != VIRTIO_MSI_NO_VECTOR) {
|
||||||
if (callback && vp_dev->msix_used_vectors < vp_dev->msix_vectors) {
|
|
||||||
vector = vp_dev->msix_used_vectors;
|
|
||||||
snprintf(vp_dev->msix_names[vector], sizeof *vp_dev->msix_names,
|
|
||||||
"%s-%s", dev_name(&vp_dev->vdev.dev), name);
|
|
||||||
err = request_irq(vp_dev->msix_entries[vector].vector,
|
|
||||||
vring_interrupt, 0,
|
|
||||||
vp_dev->msix_names[vector], vq);
|
|
||||||
if (err)
|
|
||||||
goto out_request_irq;
|
|
||||||
info->vector = vector;
|
|
||||||
++vp_dev->msix_used_vectors;
|
|
||||||
} else
|
|
||||||
vector = VP_MSIX_VQ_VECTOR;
|
|
||||||
|
|
||||||
if (callback && vp_dev->msix_enabled) {
|
|
||||||
iowrite16(vector, vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
|
iowrite16(vector, vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
|
||||||
vector = ioread16(vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
|
vector = ioread16(vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
|
||||||
if (vector == VIRTIO_MSI_NO_VECTOR) {
|
if (vector == VIRTIO_MSI_NO_VECTOR) {
|
||||||
|
@ -446,11 +424,6 @@ static struct virtqueue *vp_find_vq(struct virtio_device *vdev, unsigned index,
|
||||||
return vq;
|
return vq;
|
||||||
|
|
||||||
out_assign:
|
out_assign:
|
||||||
if (info->vector != VIRTIO_MSI_NO_VECTOR) {
|
|
||||||
free_irq(vp_dev->msix_entries[info->vector].vector, vq);
|
|
||||||
--vp_dev->msix_used_vectors;
|
|
||||||
}
|
|
||||||
out_request_irq:
|
|
||||||
vring_del_virtqueue(vq);
|
vring_del_virtqueue(vq);
|
||||||
out_activate_queue:
|
out_activate_queue:
|
||||||
iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
|
iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
|
||||||
|
@ -472,9 +445,6 @@ static void vp_del_vq(struct virtqueue *vq)
|
||||||
|
|
||||||
iowrite16(info->queue_index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
|
iowrite16(info->queue_index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
|
||||||
|
|
||||||
if (info->vector != VIRTIO_MSI_NO_VECTOR)
|
|
||||||
free_irq(vp_dev->msix_entries[info->vector].vector, vq);
|
|
||||||
|
|
||||||
if (vp_dev->msix_enabled) {
|
if (vp_dev->msix_enabled) {
|
||||||
iowrite16(VIRTIO_MSI_NO_VECTOR,
|
iowrite16(VIRTIO_MSI_NO_VECTOR,
|
||||||
vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
|
vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
|
||||||
|
@ -495,14 +465,72 @@ static void vp_del_vq(struct virtqueue *vq)
|
||||||
/* the config->del_vqs() implementation */
|
/* the config->del_vqs() implementation */
|
||||||
static void vp_del_vqs(struct virtio_device *vdev)
|
static void vp_del_vqs(struct virtio_device *vdev)
|
||||||
{
|
{
|
||||||
|
struct virtio_pci_device *vp_dev = to_vp_device(vdev);
|
||||||
struct virtqueue *vq, *n;
|
struct virtqueue *vq, *n;
|
||||||
|
struct virtio_pci_vq_info *info;
|
||||||
|
|
||||||
list_for_each_entry_safe(vq, n, &vdev->vqs, list)
|
list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
|
||||||
|
info = vq->priv;
|
||||||
|
if (vp_dev->per_vq_vectors)
|
||||||
|
free_irq(vp_dev->msix_entries[info->vector].vector, vq);
|
||||||
vp_del_vq(vq);
|
vp_del_vq(vq);
|
||||||
|
}
|
||||||
|
vp_dev->per_vq_vectors = false;
|
||||||
|
|
||||||
vp_free_vectors(vdev);
|
vp_free_vectors(vdev);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int vp_try_to_find_vqs(struct virtio_device *vdev, unsigned nvqs,
|
||||||
|
struct virtqueue *vqs[],
|
||||||
|
vq_callback_t *callbacks[],
|
||||||
|
const char *names[],
|
||||||
|
int nvectors,
|
||||||
|
bool per_vq_vectors)
|
||||||
|
{
|
||||||
|
struct virtio_pci_device *vp_dev = to_vp_device(vdev);
|
||||||
|
u16 vector;
|
||||||
|
int i, err, allocated_vectors;
|
||||||
|
|
||||||
|
err = vp_request_vectors(vdev, nvectors, per_vq_vectors);
|
||||||
|
if (err)
|
||||||
|
goto error_request;
|
||||||
|
|
||||||
|
vp_dev->per_vq_vectors = per_vq_vectors;
|
||||||
|
allocated_vectors = vp_dev->msix_used_vectors;
|
||||||
|
for (i = 0; i < nvqs; ++i) {
|
||||||
|
if (!callbacks[i] || !vp_dev->msix_enabled)
|
||||||
|
vector = VIRTIO_MSI_NO_VECTOR;
|
||||||
|
else if (vp_dev->per_vq_vectors)
|
||||||
|
vector = allocated_vectors++;
|
||||||
|
else
|
||||||
|
vector = VP_MSIX_VQ_VECTOR;
|
||||||
|
vqs[i] = vp_find_vq(vdev, i, callbacks[i], names[i], vector);
|
||||||
|
if (IS_ERR(vqs[i])) {
|
||||||
|
err = PTR_ERR(vqs[i]);
|
||||||
|
goto error_find;
|
||||||
|
}
|
||||||
|
/* allocate per-vq irq if available and necessary */
|
||||||
|
if (vp_dev->per_vq_vectors && vector != VIRTIO_MSI_NO_VECTOR) {
|
||||||
|
snprintf(vp_dev->msix_names[vector], sizeof *vp_dev->msix_names,
|
||||||
|
"%s-%s", dev_name(&vp_dev->vdev.dev), names[i]);
|
||||||
|
err = request_irq(vp_dev->msix_entries[vector].vector,
|
||||||
|
vring_interrupt, 0,
|
||||||
|
vp_dev->msix_names[vector], vqs[i]);
|
||||||
|
if (err) {
|
||||||
|
vp_del_vq(vqs[i]);
|
||||||
|
goto error_find;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
error_find:
|
||||||
|
vp_del_vqs(vdev);
|
||||||
|
|
||||||
|
error_request:
|
||||||
|
return err;
|
||||||
|
}
|
||||||
|
|
||||||
/* the config->find_vqs() implementation */
|
/* the config->find_vqs() implementation */
|
||||||
static int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
|
static int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
|
||||||
struct virtqueue *vqs[],
|
struct virtqueue *vqs[],
|
||||||
|
@ -510,29 +538,27 @@ static int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
|
||||||
const char *names[])
|
const char *names[])
|
||||||
{
|
{
|
||||||
int vectors = 0;
|
int vectors = 0;
|
||||||
int i, err;
|
int i, uninitialized_var(err);
|
||||||
|
|
||||||
/* How many vectors would we like? */
|
/* How many vectors would we like? */
|
||||||
for (i = 0; i < nvqs; ++i)
|
for (i = 0; i < nvqs; ++i)
|
||||||
if (callbacks[i])
|
if (callbacks[i])
|
||||||
++vectors;
|
++vectors;
|
||||||
|
|
||||||
err = vp_request_vectors(vdev, vectors);
|
/* We want at most one vector per queue and one for config changes. */
|
||||||
if (err)
|
err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
|
||||||
goto error_request;
|
vectors + 1, true);
|
||||||
|
if (!err)
|
||||||
for (i = 0; i < nvqs; ++i) {
|
return 0;
|
||||||
vqs[i] = vp_find_vq(vdev, i, callbacks[i], names[i]);
|
/* Fallback to separate vectors for config and a shared for queues. */
|
||||||
if (IS_ERR(vqs[i]))
|
err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
|
||||||
goto error_find;
|
2, false);
|
||||||
}
|
if (!err)
|
||||||
return 0;
|
return 0;
|
||||||
|
/* Finally fall back to regular interrupts. */
|
||||||
error_find:
|
err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
|
||||||
vp_del_vqs(vdev);
|
0, false);
|
||||||
|
return err;
|
||||||
error_request:
|
|
||||||
return PTR_ERR(vqs[i]);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static struct virtio_config_ops virtio_pci_config_ops = {
|
static struct virtio_config_ops virtio_pci_config_ops = {
|
||||||
|
|
Loading…
Reference in a new issue