From 96ef18a24b87bef7c3d2dad72cf4c3013a9f7f35 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Fri, 24 Jun 2022 10:55:45 +0800 Subject: virtio_ring: remove the arg vq of vring_alloc_desc_extra() The parameter vq of vring_alloc_desc_extra() is useless. This patch removes this parameter. Subsequent patches will call this function to avoid passing useless arguments. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220624025621.128843-6-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 643ca779fcc6..a5ec724c01d8 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -1637,8 +1637,7 @@ static void *virtqueue_detach_unused_buf_packed(struct virtqueue *_vq) return NULL; } -static struct vring_desc_extra *vring_alloc_desc_extra(struct vring_virtqueue *vq, - unsigned int num) +static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num) { struct vring_desc_extra *desc_extra; unsigned int i; @@ -1759,7 +1758,7 @@ static struct virtqueue *vring_create_virtqueue_packed( /* Put everything in free lists. */ vq->free_head = 0; - vq->packed.desc_extra = vring_alloc_desc_extra(vq, num); + vq->packed.desc_extra = vring_alloc_desc_extra(num); if (!vq->packed.desc_extra) goto err_desc_extra; @@ -2248,7 +2247,7 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index, if (!vq->split.desc_state) goto err_state; - vq->split.desc_extra = vring_alloc_desc_extra(vq, vring.num); + vq->split.desc_extra = vring_alloc_desc_extra(vring.num); if (!vq->split.desc_extra) goto err_extra; -- cgit v1.2.3 From ebe797f25f68f28581f46a9cb9c1997ac15c39a0 Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Thu, 30 Jun 2022 15:10:57 -0400 Subject: virtio: VIRTIO_HARDEN_NOTIFICATION is broken This option doesn't really work and breaks too many drivers. Not yet sure what's the right thing to do, for now let's make sure randconfig isn't broken by this. Fixes: c346dae4f3fb ("virtio: disable notification hardening by default") Cc: "Jason Wang" Signed-off-by: Michael S. Tsirkin Acked-by: Jason Wang --- drivers/virtio/Kconfig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig index e1556d2a355a..afb9051e0125 100644 --- a/drivers/virtio/Kconfig +++ b/drivers/virtio/Kconfig @@ -31,11 +31,12 @@ if VIRTIO_MENU config VIRTIO_HARDEN_NOTIFICATION bool "Harden virtio notification" + depends on BROKEN help Enable this to harden the device notifications and suppress those that happen at a time where notifications are illegal. - Experimental: Note that several drivers still have bugs that + Experimental: Note that several drivers still have issues that may cause crashes or hangs when correct handling of notifications is enforced; depending on the subset of drivers and devices you use, this may or may not work. -- cgit v1.2.3 From 02213273f72a43bad3a8e9c18595a2c74ee487e8 Mon Sep 17 00:00:00 2001 From: Minghao Xue Date: Fri, 10 Jun 2022 16:58:27 +0800 Subject: virtio_mmio: add support to set IRQ of a virtio device as wakeup source According to virtio_mmio wakeup flag in device trees, set its IRQ as wakeup source in virtqueue initialization. Signed-off-by: Minghao Xue Message-Id: <1654851507-13891-3-git-send-email-quic_mingxue@quicinc.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_mmio.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index 083ff1eb743d..945cb8fb60b6 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -487,6 +487,9 @@ static int vm_find_vqs(struct virtio_device *vdev, unsigned int nvqs, if (err) return err; + if (of_property_read_bool(vm_dev->pdev->dev.of_node, "wakeup-source")) + enable_irq_wake(irq); + for (i = 0; i < nvqs; ++i) { if (!names[i]) { vqs[i] = NULL; -- cgit v1.2.3 From 0b6fd46ec5f5720b76cbb01300ed9f7b7c6365c4 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Fri, 10 Jun 2022 11:47:37 +0200 Subject: drivers/virtio: Clarify CONFIG_VIRTIO_MEM for unsupported architectures Let's make it clearer that simply unlocking CONFIG_VIRTIO_MEM on an architecture is most probably not sufficient to have it working as expected. Cc: "Michael S. Tsirkin" Cc: Jason Wang Cc: Gavin Shan Signed-off-by: David Hildenbrand Message-Id: <20220610094737.65254-1-david@redhat.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/Kconfig | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig index afb9051e0125..02bbccd602c0 100644 --- a/drivers/virtio/Kconfig +++ b/drivers/virtio/Kconfig @@ -123,9 +123,11 @@ config VIRTIO_MEM This driver provides access to virtio-mem paravirtualized memory devices, allowing to hotplug and hotunplug memory. - This driver was only tested under x86-64 and arm64, but should - theoretically work on all architectures that support memory hotplug - and hotremove. + This driver currently only supports x86-64 and arm64. Although it + should compile on other architectures that implement memory + hot(un)plug, architecture-specific and/or common + code changes may be required for virtio-mem, kdump and kexec to work as + expected. If unsure, say M. -- cgit v1.2.3 From da802961832f9852886304290135457519815497 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:21 +0800 Subject: virtio: record the maximum queue num supported by the device. virtio-net can display the maximum (supported by hardware) ring size in ethtool -g eth0. When the subsequent patch implements vring reset, it can judge whether the ring size passed by the driver is legal based on this. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-2-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- arch/um/drivers/virtio_uml.c | 1 + drivers/platform/mellanox/mlxbf-tmfifo.c | 2 ++ drivers/remoteproc/remoteproc_virtio.c | 2 ++ drivers/s390/virtio/virtio_ccw.c | 3 +++ drivers/virtio/virtio_mmio.c | 2 ++ drivers/virtio/virtio_pci_legacy.c | 2 ++ drivers/virtio/virtio_pci_modern.c | 2 ++ drivers/virtio/virtio_vdpa.c | 2 ++ include/linux/virtio.h | 2 ++ 9 files changed, 18 insertions(+) (limited to 'drivers/virtio') diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c index 82ff3785bf69..e719af8bdf56 100644 --- a/arch/um/drivers/virtio_uml.c +++ b/arch/um/drivers/virtio_uml.c @@ -958,6 +958,7 @@ static struct virtqueue *vu_setup_vq(struct virtio_device *vdev, goto error_create; } vq->priv = info; + vq->num_max = num; num = virtqueue_get_vring_size(vq); if (vu_dev->protocol_features & diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c b/drivers/platform/mellanox/mlxbf-tmfifo.c index 38800e86ed8a..1ae3c56b66b0 100644 --- a/drivers/platform/mellanox/mlxbf-tmfifo.c +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c @@ -959,6 +959,8 @@ static int mlxbf_tmfifo_virtio_find_vqs(struct virtio_device *vdev, goto error; } + vq->num_max = vring->num; + vqs[i] = vq; vring->vq = vq; vq->priv = vring; diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c index d43d74733f0a..0f7706e23eb9 100644 --- a/drivers/remoteproc/remoteproc_virtio.c +++ b/drivers/remoteproc/remoteproc_virtio.c @@ -125,6 +125,8 @@ static struct virtqueue *rp_find_vq(struct virtio_device *vdev, return ERR_PTR(-ENOMEM); } + vq->num_max = num; + rvring->vq = vq; vq->priv = rvring; diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c index 161d3b141f0d..6b86d0280d6b 100644 --- a/drivers/s390/virtio/virtio_ccw.c +++ b/drivers/s390/virtio/virtio_ccw.c @@ -530,6 +530,9 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev, err = -ENOMEM; goto out_err; } + + vq->num_max = info->num; + /* it may have been reduced */ info->num = virtqueue_get_vring_size(vq); diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index 945cb8fb60b6..3ff746e3f24a 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -403,6 +403,8 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in goto error_new_virtqueue; } + vq->num_max = num; + /* Activate the queue */ writel(virtqueue_get_vring_size(vq), vm_dev->base + VIRTIO_MMIO_QUEUE_NUM); if (vm_dev->version == 1) { diff --git a/drivers/virtio/virtio_pci_legacy.c b/drivers/virtio/virtio_pci_legacy.c index a5e5721145c7..2257f1b3d8ae 100644 --- a/drivers/virtio/virtio_pci_legacy.c +++ b/drivers/virtio/virtio_pci_legacy.c @@ -135,6 +135,8 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, if (!vq) return ERR_PTR(-ENOMEM); + vq->num_max = num; + q_pfn = virtqueue_get_desc_addr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT; if (q_pfn >> 32) { dev_err(&vp_dev->pci_dev->dev, diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index 623906b4996c..e7e0b8c850f6 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -218,6 +218,8 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, if (!vq) return ERR_PTR(-ENOMEM); + vq->num_max = num; + /* activate the queue */ vp_modern_set_queue_size(mdev, index, virtqueue_get_vring_size(vq)); vp_modern_queue_address(mdev, index, virtqueue_get_desc_addr(vq), diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index c40f7deb6b5a..9670cc79371d 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -183,6 +183,8 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, goto error_new_virtqueue; } + vq->num_max = max_num; + /* Setup virtqueue callback */ cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL; cb.private = info; diff --git a/include/linux/virtio.h b/include/linux/virtio.h index d8fdf170637c..129bde7521e3 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -19,6 +19,7 @@ * @priv: a pointer for the virtqueue implementation to use. * @index: the zero-based ordinal number for this queue. * @num_free: number of elements we expect to be able to fit. + * @num_max: the maximum number of elements supported by the device. * * A note on @num_free: with indirect buffers, each buffer needs one * element in the queue, otherwise a buffer will need one element per @@ -31,6 +32,7 @@ struct virtqueue { struct virtio_device *vdev; unsigned int index; unsigned int num_free; + unsigned int num_max; void *priv; }; -- cgit v1.2.3 From a62eecb3a9c086fa7b47c5cc74ab8ca4e655a682 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:23 +0800 Subject: virtio_ring: update the document of the virtqueue_detach_unused_buf for queue reset Added documentation for virtqueue_detach_unused_buf, allowing it to be called on queue reset. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-4-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index a5ec724c01d8..17024389b62c 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -2130,8 +2130,8 @@ EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed); * @_vq: the struct virtqueue we're talking about. * * Returns NULL or the "data" token handed to virtqueue_add_*(). - * This is not valid on an active queue; it is useful only for device - * shutdown. + * This is not valid on an active queue; it is useful for device + * shutdown or the reset queue. */ void *virtqueue_detach_unused_buf(struct virtqueue *_vq) { -- cgit v1.2.3 From 3ea19e3265d8d140bdd2905e7c40216368cac589 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:24 +0800 Subject: virtio_ring: extract the logic of freeing vring Introduce vring_free() to free the vring of vq. Subsequent patches will use vring_free() alone. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-5-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 17024389b62c..a3d76fd87983 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -2316,14 +2316,10 @@ struct virtqueue *vring_new_virtqueue(unsigned int index, } EXPORT_SYMBOL_GPL(vring_new_virtqueue); -void vring_del_virtqueue(struct virtqueue *_vq) +static void vring_free(struct virtqueue *_vq) { struct vring_virtqueue *vq = to_vvq(_vq); - spin_lock(&vq->vq.vdev->vqs_list_lock); - list_del(&_vq->list); - spin_unlock(&vq->vq.vdev->vqs_list_lock); - if (vq->we_own_ring) { if (vq->packed_ring) { vring_free_queue(vq->vq.vdev, @@ -2354,6 +2350,18 @@ void vring_del_virtqueue(struct virtqueue *_vq) kfree(vq->split.desc_state); kfree(vq->split.desc_extra); } +} + +void vring_del_virtqueue(struct virtqueue *_vq) +{ + struct vring_virtqueue *vq = to_vvq(_vq); + + spin_lock(&vq->vq.vdev->vqs_list_lock); + list_del(&_vq->list); + spin_unlock(&vq->vq.vdev->vqs_list_lock); + + vring_free(_vq); + kfree(vq); } EXPORT_SYMBOL_GPL(vring_del_virtqueue); -- cgit v1.2.3 From d76136e434973691d8928cec1b3f93d38e30fe6e Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:25 +0800 Subject: virtio_ring: split vring_virtqueue Separate the two inline structures(split and packed) from the structure vring_virtqueue. In this way, we can use these two structures later to pass parameters and retain temporary variables. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-6-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 116 ++++++++++++++++++++++--------------------- 1 file changed, 60 insertions(+), 56 deletions(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index a3d76fd87983..1bc5794e9739 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -85,6 +85,64 @@ struct vring_desc_extra { u16 next; /* The next desc state in a list. */ }; +struct vring_virtqueue_split { + /* Actual memory layout for this queue. */ + struct vring vring; + + /* Last written value to avail->flags */ + u16 avail_flags_shadow; + + /* + * Last written value to avail->idx in + * guest byte order. + */ + u16 avail_idx_shadow; + + /* Per-descriptor state. */ + struct vring_desc_state_split *desc_state; + struct vring_desc_extra *desc_extra; + + /* DMA address and size information */ + dma_addr_t queue_dma_addr; + size_t queue_size_in_bytes; +}; + +struct vring_virtqueue_packed { + /* Actual memory layout for this queue. */ + struct { + unsigned int num; + struct vring_packed_desc *desc; + struct vring_packed_desc_event *driver; + struct vring_packed_desc_event *device; + } vring; + + /* Driver ring wrap counter. */ + bool avail_wrap_counter; + + /* Avail used flags. */ + u16 avail_used_flags; + + /* Index of the next avail descriptor. */ + u16 next_avail_idx; + + /* + * Last written value to driver->flags in + * guest byte order. + */ + u16 event_flags_shadow; + + /* Per-descriptor state. */ + struct vring_desc_state_packed *desc_state; + struct vring_desc_extra *desc_extra; + + /* DMA address and size information */ + dma_addr_t ring_dma_addr; + dma_addr_t driver_event_dma_addr; + dma_addr_t device_event_dma_addr; + size_t ring_size_in_bytes; + size_t event_size_in_bytes; +}; + struct vring_virtqueue { struct virtqueue vq; @@ -124,64 +182,10 @@ struct vring_virtqueue { union { /* Available for split ring */ - struct { - /* Actual memory layout for this queue. */ - struct vring vring; - - /* Last written value to avail->flags */ - u16 avail_flags_shadow; - - /* - * Last written value to avail->idx in - * guest byte order. - */ - u16 avail_idx_shadow; - - /* Per-descriptor state. */ - struct vring_desc_state_split *desc_state; - struct vring_desc_extra *desc_extra; - - /* DMA address and size information */ - dma_addr_t queue_dma_addr; - size_t queue_size_in_bytes; - } split; + struct vring_virtqueue_split split; /* Available for packed ring */ - struct { - /* Actual memory layout for this queue. */ - struct { - unsigned int num; - struct vring_packed_desc *desc; - struct vring_packed_desc_event *driver; - struct vring_packed_desc_event *device; - } vring; - - /* Driver ring wrap counter. */ - bool avail_wrap_counter; - - /* Avail used flags. */ - u16 avail_used_flags; - - /* Index of the next avail descriptor. */ - u16 next_avail_idx; - - /* - * Last written value to driver->flags in - * guest byte order. - */ - u16 event_flags_shadow; - - /* Per-descriptor state. */ - struct vring_desc_state_packed *desc_state; - struct vring_desc_extra *desc_extra; - - /* DMA address and size information */ - dma_addr_t ring_dma_addr; - dma_addr_t driver_event_dma_addr; - dma_addr_t device_event_dma_addr; - size_t ring_size_in_bytes; - size_t event_size_in_bytes; - } packed; + struct vring_virtqueue_packed packed; }; /* How to notify other side. FIXME: commonalize hcalls! */ -- cgit v1.2.3 From 3a897128d3193459a87551e54a7f0a71edb65153 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:26 +0800 Subject: virtio_ring: introduce virtqueue_init() Separate the logic of virtqueue initialization. These variables should be reset during reset. This logic can be called independently when implementing resize/reset later. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-7-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 1bc5794e9739..a63ef2d99955 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -368,6 +368,24 @@ static int vring_mapping_error(const struct vring_virtqueue *vq, return dma_mapping_error(vring_dma_dev(vq), addr); } +static void virtqueue_init(struct vring_virtqueue *vq, u32 num) +{ + vq->vq.num_free = num; + + if (vq->packed_ring) + vq->last_used_idx = 0 | (1 << VRING_PACKED_EVENT_F_WRAP_CTR); + else + vq->last_used_idx = 0; + + vq->event_triggered = false; + vq->num_added = 0; + +#ifdef DEBUG + vq->in_use = false; + vq->last_add_time_valid = false; +#endif +} + /* * Split ring specific functions - *_split(). @@ -1706,7 +1724,6 @@ static struct virtqueue *vring_create_virtqueue_packed( vq->vq.callback = callback; vq->vq.vdev = vdev; vq->vq.name = name; - vq->vq.num_free = num; vq->vq.index = index; vq->we_own_ring = true; vq->notify = notify; @@ -1716,15 +1733,8 @@ static struct virtqueue *vring_create_virtqueue_packed( #else vq->broken = false; #endif - vq->last_used_idx = 0 | (1 << VRING_PACKED_EVENT_F_WRAP_CTR); - vq->event_triggered = false; - vq->num_added = 0; vq->packed_ring = true; vq->use_dma_api = vring_use_dma_api(vdev); -#ifdef DEBUG - vq->in_use = false; - vq->last_add_time_valid = false; -#endif vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) && !context; @@ -1773,6 +1783,8 @@ static struct virtqueue *vring_create_virtqueue_packed( cpu_to_le16(vq->packed.event_flags_shadow); } + virtqueue_init(vq, num); + spin_lock(&vdev->vqs_list_lock); list_add_tail(&vq->vq.list, &vdev->vqs); spin_unlock(&vdev->vqs_list_lock); @@ -2205,7 +2217,6 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index, vq->vq.callback = callback; vq->vq.vdev = vdev; vq->vq.name = name; - vq->vq.num_free = vring.num; vq->vq.index = index; vq->we_own_ring = false; vq->notify = notify; @@ -2215,14 +2226,7 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index, #else vq->broken = false; #endif - vq->last_used_idx = 0; - vq->event_triggered = false; - vq->num_added = 0; vq->use_dma_api = vring_use_dma_api(vdev); -#ifdef DEBUG - vq->in_use = false; - vq->last_add_time_valid = false; -#endif vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) && !context; @@ -2260,6 +2264,8 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index, memset(vq->split.desc_state, 0, vring.num * sizeof(struct vring_desc_state_split)); + virtqueue_init(vq, vring.num); + spin_lock(&vdev->vqs_list_lock); list_add_tail(&vq->vq.list, &vdev->vqs); spin_unlock(&vdev->vqs_list_lock); -- cgit v1.2.3 From 07d9629d49584b6f79faa6158cd7aef7e6919703 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:27 +0800 Subject: virtio_ring: split: stop __vring_new_virtqueue as export symbol There is currently only one place to reference __vring_new_virtqueue() directly from the outside of virtio core. And here vring_new_virtqueue() can be used instead. Subsequent patches will modify __vring_new_virtqueue, so stop it as an export symbol for now. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-8-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 25 ++++++++++++++++--------- include/linux/virtio_ring.h | 10 ---------- tools/virtio/virtio_test.c | 4 ++-- 3 files changed, 18 insertions(+), 21 deletions(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index a63ef2d99955..8ce6cc73d814 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -204,6 +204,14 @@ struct vring_virtqueue { #endif }; +static struct virtqueue *__vring_new_virtqueue(unsigned int index, + struct vring vring, + struct virtio_device *vdev, + bool weak_barriers, + bool context, + bool (*notify)(struct virtqueue *), + void (*callback)(struct virtqueue *), + const char *name); /* * Helpers. @@ -2195,14 +2203,14 @@ irqreturn_t vring_interrupt(int irq, void *_vq) EXPORT_SYMBOL_GPL(vring_interrupt); /* Only available for split ring */ -struct virtqueue *__vring_new_virtqueue(unsigned int index, - struct vring vring, - struct virtio_device *vdev, - bool weak_barriers, - bool context, - bool (*notify)(struct virtqueue *), - void (*callback)(struct virtqueue *), - const char *name) +static struct virtqueue *__vring_new_virtqueue(unsigned int index, + struct vring vring, + struct virtio_device *vdev, + bool weak_barriers, + bool context, + bool (*notify)(struct virtqueue *), + void (*callback)(struct virtqueue *), + const char *name) { struct vring_virtqueue *vq; @@ -2277,7 +2285,6 @@ err_state: kfree(vq); return NULL; } -EXPORT_SYMBOL_GPL(__vring_new_virtqueue); struct virtqueue *vring_create_virtqueue( unsigned int index, diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h index b485b13fa50b..8b8af1a38991 100644 --- a/include/linux/virtio_ring.h +++ b/include/linux/virtio_ring.h @@ -76,16 +76,6 @@ struct virtqueue *vring_create_virtqueue(unsigned int index, void (*callback)(struct virtqueue *vq), const char *name); -/* Creates a virtqueue with a custom layout. */ -struct virtqueue *__vring_new_virtqueue(unsigned int index, - struct vring vring, - struct virtio_device *vdev, - bool weak_barriers, - bool ctx, - bool (*notify)(struct virtqueue *), - void (*callback)(struct virtqueue *), - const char *name); - /* * Creates a virtqueue with a standard layout but a caller-allocated * ring. diff --git a/tools/virtio/virtio_test.c b/tools/virtio/virtio_test.c index 23f142af544a..86a410ddcedd 100644 --- a/tools/virtio/virtio_test.c +++ b/tools/virtio/virtio_test.c @@ -102,8 +102,8 @@ static void vq_reset(struct vq_info *info, int num, struct virtio_device *vdev) memset(info->ring, 0, vring_size(num, 4096)); vring_init(&info->vring, num, info->ring, 4096); - info->vq = __vring_new_virtqueue(info->idx, info->vring, vdev, true, - false, vq_notify, vq_callback, "test"); + info->vq = vring_new_virtqueue(info->idx, num, 4096, vdev, true, false, + info->ring, vq_notify, vq_callback, "test"); assert(info->vq); info->vq->priv = info; } -- cgit v1.2.3 From cd4c812acb8390a3cfa0a178b777979f9cdd3eeb Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:28 +0800 Subject: virtio_ring: split: __vring_new_virtqueue() accept struct vring_virtqueue_split __vring_new_virtqueue() instead accepts struct vring_virtqueue_split. The purpose of this is to pass more information into __vring_new_virtqueue() to make the code simpler and the structure cleaner. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-9-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 8ce6cc73d814..1d51ed46b295 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -205,7 +205,7 @@ struct vring_virtqueue { }; static struct virtqueue *__vring_new_virtqueue(unsigned int index, - struct vring vring, + struct vring_virtqueue_split *vring_split, struct virtio_device *vdev, bool weak_barriers, bool context, @@ -949,6 +949,7 @@ static struct virtqueue *vring_create_virtqueue_split( void (*callback)(struct virtqueue *), const char *name) { + struct vring_virtqueue_split vring_split = {}; struct virtqueue *vq; void *queue = NULL; dma_addr_t dma_addr; @@ -984,10 +985,10 @@ static struct virtqueue *vring_create_virtqueue_split( return NULL; queue_size_in_bytes = vring_size(num, vring_align); - vring_init(&vring, num, queue, vring_align); + vring_init(&vring_split.vring, num, queue, vring_align); - vq = __vring_new_virtqueue(index, vring, vdev, weak_barriers, context, - notify, callback, name); + vq = __vring_new_virtqueue(index, &vring_split, vdev, weak_barriers, + context, notify, callback, name); if (!vq) { vring_free_queue(vdev, queue_size_in_bytes, queue, dma_addr); @@ -2204,7 +2205,7 @@ EXPORT_SYMBOL_GPL(vring_interrupt); /* Only available for split ring */ static struct virtqueue *__vring_new_virtqueue(unsigned int index, - struct vring vring, + struct vring_virtqueue_split *vring_split, struct virtio_device *vdev, bool weak_barriers, bool context, @@ -2246,7 +2247,7 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, vq->split.queue_dma_addr = 0; vq->split.queue_size_in_bytes = 0; - vq->split.vring = vring; + vq->split.vring = vring_split->vring; vq->split.avail_flags_shadow = 0; vq->split.avail_idx_shadow = 0; @@ -2258,21 +2259,21 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, vq->split.avail_flags_shadow); } - vq->split.desc_state = kmalloc_array(vring.num, + vq->split.desc_state = kmalloc_array(vring_split->vring.num, sizeof(struct vring_desc_state_split), GFP_KERNEL); if (!vq->split.desc_state) goto err_state; - vq->split.desc_extra = vring_alloc_desc_extra(vring.num); + vq->split.desc_extra = vring_alloc_desc_extra(vring_split->vring.num); if (!vq->split.desc_extra) goto err_extra; /* Put everything in free lists. */ vq->free_head = 0; - memset(vq->split.desc_state, 0, vring.num * + memset(vq->split.desc_state, 0, vring_split->vring.num * sizeof(struct vring_desc_state_split)); - virtqueue_init(vq, vring.num); + virtqueue_init(vq, vring_split->vring.num); spin_lock(&vdev->vqs_list_lock); list_add_tail(&vq->vq.list, &vdev->vqs); @@ -2322,14 +2323,14 @@ struct virtqueue *vring_new_virtqueue(unsigned int index, void (*callback)(struct virtqueue *vq), const char *name) { - struct vring vring; + struct vring_virtqueue_split vring_split = {}; if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) return NULL; - vring_init(&vring, num, pages, vring_align); - return __vring_new_virtqueue(index, vring, vdev, weak_barriers, context, - notify, callback, name); + vring_init(&vring_split.vring, num, pages, vring_align); + return __vring_new_virtqueue(index, &vring_split, vdev, weak_barriers, + context, notify, callback, name); } EXPORT_SYMBOL_GPL(vring_new_virtqueue); -- cgit v1.2.3 From 89f05d94a33abec2d38f644065ca5cff5f50c4d7 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:29 +0800 Subject: virtio_ring: split: introduce vring_free_split() Free the structure struct vring_vritqueue_split. Subsequent patches require it. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-10-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 1d51ed46b295..71cc7762ec51 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -937,6 +937,17 @@ static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq) return NULL; } +static void vring_free_split(struct vring_virtqueue_split *vring_split, + struct virtio_device *vdev) +{ + vring_free_queue(vdev, vring_split->queue_size_in_bytes, + vring_split->vring.desc, + vring_split->queue_dma_addr); + + kfree(vring_split->desc_state); + kfree(vring_split->desc_extra); +} + static struct virtqueue *vring_create_virtqueue_split( unsigned int index, unsigned int num, -- cgit v1.2.3 From c2d87fe68d13f45f42861e25dcda21b132dbf554 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:30 +0800 Subject: virtio_ring: split: extract the logic of alloc queue Separate the logic of split to create vring queue. This feature is required for subsequent virtuqueue reset vring. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-11-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 65 +++++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 25 deletions(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 71cc7762ec51..5597a9b9e518 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -948,29 +948,19 @@ static void vring_free_split(struct vring_virtqueue_split *vring_split, kfree(vring_split->desc_extra); } -static struct virtqueue *vring_create_virtqueue_split( - unsigned int index, - unsigned int num, - unsigned int vring_align, - struct virtio_device *vdev, - bool weak_barriers, - bool may_reduce_num, - bool context, - bool (*notify)(struct virtqueue *), - void (*callback)(struct virtqueue *), - const char *name) +static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split, + struct virtio_device *vdev, + u32 num, + unsigned int vring_align, + bool may_reduce_num) { - struct vring_virtqueue_split vring_split = {}; - struct virtqueue *vq; void *queue = NULL; dma_addr_t dma_addr; - size_t queue_size_in_bytes; - struct vring vring; /* We assume num is a power of 2. */ if (num & (num - 1)) { dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num); - return NULL; + return -EINVAL; } /* TODO: allocate each queue chunk individually */ @@ -981,11 +971,11 @@ static struct virtqueue *vring_create_virtqueue_split( if (queue) break; if (!may_reduce_num) - return NULL; + return -ENOMEM; } if (!num) - return NULL; + return -ENOMEM; if (!queue) { /* Try to get a single page. You are my only hope! */ @@ -993,21 +983,46 @@ static struct virtqueue *vring_create_virtqueue_split( &dma_addr, GFP_KERNEL|__GFP_ZERO); } if (!queue) - return NULL; + return -ENOMEM; + + vring_init(&vring_split->vring, num, queue, vring_align); - queue_size_in_bytes = vring_size(num, vring_align); - vring_init(&vring_split.vring, num, queue, vring_align); + vring_split->queue_dma_addr = dma_addr; + vring_split->queue_size_in_bytes = vring_size(num, vring_align); + + return 0; +} + +static struct virtqueue *vring_create_virtqueue_split( + unsigned int index, + unsigned int num, + unsigned int vring_align, + struct virtio_device *vdev, + bool weak_barriers, + bool may_reduce_num, + bool context, + bool (*notify)(struct virtqueue *), + void (*callback)(struct virtqueue *), + const char *name) +{ + struct vring_virtqueue_split vring_split = {}; + struct virtqueue *vq; + int err; + + err = vring_alloc_queue_split(&vring_split, vdev, num, vring_align, + may_reduce_num); + if (err) + return NULL; vq = __vring_new_virtqueue(index, &vring_split, vdev, weak_barriers, context, notify, callback, name); if (!vq) { - vring_free_queue(vdev, queue_size_in_bytes, queue, - dma_addr); + vring_free_split(&vring_split, vdev); return NULL; } - to_vvq(vq)->split.queue_dma_addr = dma_addr; - to_vvq(vq)->split.queue_size_in_bytes = queue_size_in_bytes; + to_vvq(vq)->split.queue_dma_addr = vring_split.queue_dma_addr; + to_vvq(vq)->split.queue_size_in_bytes = vring_split.queue_size_in_bytes; to_vvq(vq)->we_own_ring = true; return vq; -- cgit v1.2.3 From a2b36c8d7ddbaff801bcaa4592522cd3b7b08112 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:31 +0800 Subject: virtio_ring: split: extract the logic of alloc state and extra Separate the logic of creating desc_state, desc_extra, and subsequent patches will call it independently. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-12-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 52 ++++++++++++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 16 deletions(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 5597a9b9e518..03fc656b1f40 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -212,6 +212,7 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, bool (*notify)(struct virtqueue *), void (*callback)(struct virtqueue *), const char *name); +static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num); /* * Helpers. @@ -937,6 +938,32 @@ static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq) return NULL; } +static int vring_alloc_state_extra_split(struct vring_virtqueue_split *vring_split) +{ + struct vring_desc_state_split *state; + struct vring_desc_extra *extra; + u32 num = vring_split->vring.num; + + state = kmalloc_array(num, sizeof(struct vring_desc_state_split), GFP_KERNEL); + if (!state) + goto err_state; + + extra = vring_alloc_desc_extra(num); + if (!extra) + goto err_extra; + + memset(state, 0, num * sizeof(struct vring_desc_state_split)); + + vring_split->desc_state = state; + vring_split->desc_extra = extra; + return 0; + +err_extra: + kfree(state); +err_state: + return -ENOMEM; +} + static void vring_free_split(struct vring_virtqueue_split *vring_split, struct virtio_device *vdev) { @@ -2240,6 +2267,7 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, const char *name) { struct vring_virtqueue *vq; + int err; if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) return NULL; @@ -2285,19 +2313,17 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, vq->split.avail_flags_shadow); } - vq->split.desc_state = kmalloc_array(vring_split->vring.num, - sizeof(struct vring_desc_state_split), GFP_KERNEL); - if (!vq->split.desc_state) - goto err_state; - - vq->split.desc_extra = vring_alloc_desc_extra(vring_split->vring.num); - if (!vq->split.desc_extra) - goto err_extra; + err = vring_alloc_state_extra_split(vring_split); + if (err) { + kfree(vq); + return NULL; + } /* Put everything in free lists. */ vq->free_head = 0; - memset(vq->split.desc_state, 0, vring_split->vring.num * - sizeof(struct vring_desc_state_split)); + + vq->split.desc_state = vring_split->desc_state; + vq->split.desc_extra = vring_split->desc_extra; virtqueue_init(vq, vring_split->vring.num); @@ -2305,12 +2331,6 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, list_add_tail(&vq->vq.list, &vdev->vqs); spin_unlock(&vdev->vqs_list_lock); return &vq->vq; - -err_extra: - kfree(vq->split.desc_state); -err_state: - kfree(vq); - return NULL; } struct virtqueue *vring_create_virtqueue( -- cgit v1.2.3 From 198fa7be96e52bc89c9e8a7e1c3b9e059ff203a0 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:32 +0800 Subject: virtio_ring: split: extract the logic of vring init Separate the logic of initializing vring, and subsequent patches will call it separately. This function completes the variable initialization of split vring. It together with the logic of atatch constitutes the initialization of vring. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-13-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 03fc656b1f40..e456cc16ea2a 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -938,6 +938,25 @@ static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq) return NULL; } +static void virtqueue_vring_init_split(struct vring_virtqueue_split *vring_split, + struct vring_virtqueue *vq) +{ + struct virtio_device *vdev; + + vdev = vq->vq.vdev; + + vring_split->avail_flags_shadow = 0; + vring_split->avail_idx_shadow = 0; + + /* No callback? Tell other side not to bother us. */ + if (!vq->vq.callback) { + vring_split->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; + if (!vq->event) + vring_split->vring.avail->flags = cpu_to_virtio16(vdev, + vring_split->avail_flags_shadow); + } +} + static int vring_alloc_state_extra_split(struct vring_virtqueue_split *vring_split) { struct vring_desc_state_split *state; @@ -2302,16 +2321,6 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, vq->split.queue_size_in_bytes = 0; vq->split.vring = vring_split->vring; - vq->split.avail_flags_shadow = 0; - vq->split.avail_idx_shadow = 0; - - /* No callback? Tell other side not to bother us. */ - if (!callback) { - vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; - if (!vq->event) - vq->split.vring.avail->flags = cpu_to_virtio16(vdev, - vq->split.avail_flags_shadow); - } err = vring_alloc_state_extra_split(vring_split); if (err) { @@ -2325,6 +2334,8 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, vq->split.desc_state = vring_split->desc_state; vq->split.desc_extra = vring_split->desc_extra; + virtqueue_vring_init_split(vring_split, vq); + virtqueue_init(vq, vring_split->vring.num); spin_lock(&vdev->vqs_list_lock); -- cgit v1.2.3 From e1d6a423ea1867a3a84f12a99981f036acb8f354 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:33 +0800 Subject: virtio_ring: split: extract the logic of attach vring Separate the logic of attach vring, subsequent patches will call it separately. virtqueue_vring_init_split() completes the initialization of other variables of vring split. We can directly use vq->split = *vring_split to complete attach. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-14-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index e456cc16ea2a..d0df887c3ada 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -957,6 +957,15 @@ static void virtqueue_vring_init_split(struct vring_virtqueue_split *vring_split } } +static void virtqueue_vring_attach_split(struct vring_virtqueue *vq, + struct vring_virtqueue_split *vring_split) +{ + vq->split = *vring_split; + + /* Put everything in free lists. */ + vq->free_head = 0; +} + static int vring_alloc_state_extra_split(struct vring_virtqueue_split *vring_split) { struct vring_desc_state_split *state; @@ -1067,8 +1076,6 @@ static struct virtqueue *vring_create_virtqueue_split( return NULL; } - to_vvq(vq)->split.queue_dma_addr = vring_split.queue_dma_addr; - to_vvq(vq)->split.queue_size_in_bytes = vring_split.queue_size_in_bytes; to_vvq(vq)->we_own_ring = true; return vq; @@ -2317,26 +2324,16 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM)) vq->weak_barriers = false; - vq->split.queue_dma_addr = 0; - vq->split.queue_size_in_bytes = 0; - - vq->split.vring = vring_split->vring; - err = vring_alloc_state_extra_split(vring_split); if (err) { kfree(vq); return NULL; } - /* Put everything in free lists. */ - vq->free_head = 0; - - vq->split.desc_state = vring_split->desc_state; - vq->split.desc_extra = vring_split->desc_extra; - virtqueue_vring_init_split(vring_split, vq); virtqueue_init(vq, vring_split->vring.num); + virtqueue_vring_attach_split(vq, vring_split); spin_lock(&vdev->vqs_list_lock); list_add_tail(&vq->vq.list, &vdev->vqs); -- cgit v1.2.3 From e5175b419a1394e77ff418dd9bfaf15555dfe594 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:34 +0800 Subject: virtio_ring: split: introduce virtqueue_reinit_split() Introduce a function to initialize vq without allocating new ring, desc_state, desc_extra. Subsequent patches will call this function after reset vq to reinitialize vq. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-15-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index d0df887c3ada..948f8da7b780 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -957,6 +957,29 @@ static void virtqueue_vring_init_split(struct vring_virtqueue_split *vring_split } } +static void virtqueue_reinit_split(struct vring_virtqueue *vq) +{ + int num; + + num = vq->split.vring.num; + + vq->split.vring.avail->flags = 0; + vq->split.vring.avail->idx = 0; + + /* reset avail event */ + vq->split.vring.avail->ring[num] = 0; + + vq->split.vring.used->flags = 0; + vq->split.vring.used->idx = 0; + + /* reset used event */ + *(__virtio16 *)&(vq->split.vring.used->ring[num]) = 0; + + virtqueue_init(vq, num); + + virtqueue_vring_init_split(&vq->split, vq); +} + static void virtqueue_vring_attach_split(struct vring_virtqueue *vq, struct vring_virtqueue_split *vring_split) { -- cgit v1.2.3 From af36b16f6c1e51975a3815eb21c21c47f3114393 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:35 +0800 Subject: virtio_ring: split: reserve vring_align, may_reduce_num In vring_alloc_queue_split() save vring_align, may_reduce_num to structure vring_virtqueue_split. Used to create a new vring when implementing resize. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-16-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 948f8da7b780..8cbb19cdb1ce 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -105,6 +105,13 @@ struct vring_virtqueue_split { /* DMA address and size information */ dma_addr_t queue_dma_addr; size_t queue_size_in_bytes; + + /* + * The parameters for creating vrings are reserved for creating new + * vring. + */ + u32 vring_align; + bool may_reduce_num; }; struct vring_virtqueue_packed { @@ -1068,6 +1075,9 @@ static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split, vring_split->queue_dma_addr = dma_addr; vring_split->queue_size_in_bytes = vring_size(num, vring_align); + vring_split->vring_align = vring_align; + vring_split->may_reduce_num = may_reduce_num; + return 0; } -- cgit v1.2.3 From 6fea20e5611fc5c8264f04a878a98c06b6209eed Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:36 +0800 Subject: virtio_ring: split: introduce virtqueue_resize_split() virtio ring split supports resize. Only after the new vring is successfully allocated based on the new num, we will release the old vring. In any case, an error is returned, indicating that the vring still points to the old vring. In the case of an error, re-initialize(virtqueue_reinit_split()) the virtqueue to ensure that the vring can be used. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-17-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 8cbb19cdb1ce..1852912d711e 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -220,6 +220,7 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, void (*callback)(struct virtqueue *), const char *name); static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num); +static void vring_free(struct virtqueue *_vq); /* * Helpers. @@ -1114,6 +1115,39 @@ static struct virtqueue *vring_create_virtqueue_split( return vq; } +static int virtqueue_resize_split(struct virtqueue *_vq, u32 num) +{ + struct vring_virtqueue_split vring_split = {}; + struct vring_virtqueue *vq = to_vvq(_vq); + struct virtio_device *vdev = _vq->vdev; + int err; + + err = vring_alloc_queue_split(&vring_split, vdev, num, + vq->split.vring_align, + vq->split.may_reduce_num); + if (err) + goto err; + + err = vring_alloc_state_extra_split(&vring_split); + if (err) + goto err_state_extra; + + vring_free(&vq->vq); + + virtqueue_vring_init_split(&vring_split, vq); + + virtqueue_init(vq, vring_split.vring.num); + virtqueue_vring_attach_split(vq, &vring_split); + + return 0; + +err_state_extra: + vring_free_split(&vring_split, vdev); +err: + virtqueue_reinit_split(vq); + return -ENOMEM; +} + /* * Packed ring specific functions - *_packed(). -- cgit v1.2.3 From 6356f8bb01ddf4a6d9f2da16f7c0e3c093c7f548 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:37 +0800 Subject: virtio_ring: packed: introduce vring_free_packed Free the structure struct vring_vritqueue_packed. Subsequent patches require it. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-18-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 1852912d711e..2993d817871a 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -1832,6 +1832,28 @@ static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num) return desc_extra; } +static void vring_free_packed(struct vring_virtqueue_packed *vring_packed, + struct virtio_device *vdev) +{ + if (vring_packed->vring.desc) + vring_free_queue(vdev, vring_packed->ring_size_in_bytes, + vring_packed->vring.desc, + vring_packed->ring_dma_addr); + + if (vring_packed->vring.driver) + vring_free_queue(vdev, vring_packed->event_size_in_bytes, + vring_packed->vring.driver, + vring_packed->driver_event_dma_addr); + + if (vring_packed->vring.device) + vring_free_queue(vdev, vring_packed->event_size_in_bytes, + vring_packed->vring.device, + vring_packed->device_event_dma_addr); + + kfree(vring_packed->desc_state); + kfree(vring_packed->desc_extra); +} + static struct virtqueue *vring_create_virtqueue_packed( unsigned int index, unsigned int num, -- cgit v1.2.3 From 6b60b9c008e56e43cdda6ae618f7f58679a8901a Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:38 +0800 Subject: virtio_ring: packed: extract the logic of alloc queue Separate the logic of packed to create vring queue. This feature is required for subsequent virtuqueue reset vring. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-19-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 80 ++++++++++++++++++++++++++++---------------- 1 file changed, 51 insertions(+), 29 deletions(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 2993d817871a..8209923ea7d7 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -1854,19 +1854,10 @@ static void vring_free_packed(struct vring_virtqueue_packed *vring_packed, kfree(vring_packed->desc_extra); } -static struct virtqueue *vring_create_virtqueue_packed( - unsigned int index, - unsigned int num, - unsigned int vring_align, - struct virtio_device *vdev, - bool weak_barriers, - bool may_reduce_num, - bool context, - bool (*notify)(struct virtqueue *), - void (*callback)(struct virtqueue *), - const char *name) +static int vring_alloc_queue_packed(struct vring_virtqueue_packed *vring_packed, + struct virtio_device *vdev, + u32 num) { - struct vring_virtqueue *vq; struct vring_packed_desc *ring; struct vring_packed_desc_event *driver, *device; dma_addr_t ring_dma_addr, driver_event_dma_addr, device_event_dma_addr; @@ -1878,7 +1869,11 @@ static struct virtqueue *vring_create_virtqueue_packed( &ring_dma_addr, GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO); if (!ring) - goto err_ring; + goto err; + + vring_packed->vring.desc = ring; + vring_packed->ring_dma_addr = ring_dma_addr; + vring_packed->ring_size_in_bytes = ring_size_in_bytes; event_size_in_bytes = sizeof(struct vring_packed_desc_event); @@ -1886,13 +1881,47 @@ static struct virtqueue *vring_create_virtqueue_packed( &driver_event_dma_addr, GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO); if (!driver) - goto err_driver; + goto err; + + vring_packed->vring.driver = driver; + vring_packed->event_size_in_bytes = event_size_in_bytes; + vring_packed->driver_event_dma_addr = driver_event_dma_addr; device = vring_alloc_queue(vdev, event_size_in_bytes, &device_event_dma_addr, GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO); if (!device) - goto err_device; + goto err; + + vring_packed->vring.device = device; + vring_packed->device_event_dma_addr = device_event_dma_addr; + + vring_packed->vring.num = num; + + return 0; + +err: + vring_free_packed(vring_packed, vdev); + return -ENOMEM; +} + +static struct virtqueue *vring_create_virtqueue_packed( + unsigned int index, + unsigned int num, + unsigned int vring_align, + struct virtio_device *vdev, + bool weak_barriers, + bool may_reduce_num, + bool context, + bool (*notify)(struct virtqueue *), + void (*callback)(struct virtqueue *), + const char *name) +{ + struct vring_virtqueue_packed vring_packed = {}; + struct vring_virtqueue *vq; + + if (vring_alloc_queue_packed(&vring_packed, vdev, num)) + goto err_ring; vq = kmalloc(sizeof(*vq), GFP_KERNEL); if (!vq) @@ -1920,17 +1949,14 @@ static struct virtqueue *vring_create_virtqueue_packed( if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM)) vq->weak_barriers = false; - vq->packed.ring_dma_addr = ring_dma_addr; - vq->packed.driver_event_dma_addr = driver_event_dma_addr; - vq->packed.device_event_dma_addr = device_event_dma_addr; + vq->packed.ring_dma_addr = vring_packed.ring_dma_addr; + vq->packed.driver_event_dma_addr = vring_packed.driver_event_dma_addr; + vq->packed.device_event_dma_addr = vring_packed.device_event_dma_addr; - vq->packed.ring_size_in_bytes = ring_size_in_bytes; - vq->packed.event_size_in_bytes = event_size_in_bytes; + vq->packed.ring_size_in_bytes = vring_packed.ring_size_in_bytes; + vq->packed.event_size_in_bytes = vring_packed.event_size_in_bytes; - vq->packed.vring.num = num; - vq->packed.vring.desc = ring; - vq->packed.vring.driver = driver; - vq->packed.vring.device = device; + vq->packed.vring = vring_packed.vring; vq->packed.next_avail_idx = 0; vq->packed.avail_wrap_counter = 1; @@ -1972,11 +1998,7 @@ err_desc_extra: err_desc_state: kfree(vq); err_vq: - vring_free_queue(vdev, event_size_in_bytes, device, device_event_dma_addr); -err_device: - vring_free_queue(vdev, event_size_in_bytes, driver, driver_event_dma_addr); -err_driver: - vring_free_queue(vdev, ring_size_in_bytes, ring, ring_dma_addr); + vring_free_packed(&vring_packed, vdev); err_ring: return NULL; } -- cgit v1.2.3 From ef3167cfd5f28bc86d68e82fc52d07ed03dff2f7 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:39 +0800 Subject: virtio_ring: packed: extract the logic of alloc state and extra Separate the logic for alloc desc_state and desc_extra, which will be called separately by subsequent patches. Use struct vring_packed to pass desc_state, desc_extra. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-20-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 48 +++++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 14 deletions(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 8209923ea7d7..533af061d009 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -1905,6 +1905,33 @@ err: return -ENOMEM; } +static int vring_alloc_state_extra_packed(struct vring_virtqueue_packed *vring_packed) +{ + struct vring_desc_state_packed *state; + struct vring_desc_extra *extra; + u32 num = vring_packed->vring.num; + + state = kmalloc_array(num, sizeof(struct vring_desc_state_packed), GFP_KERNEL); + if (!state) + goto err_desc_state; + + memset(state, 0, num * sizeof(struct vring_desc_state_packed)); + + extra = vring_alloc_desc_extra(num); + if (!extra) + goto err_desc_extra; + + vring_packed->desc_state = state; + vring_packed->desc_extra = extra; + + return 0; + +err_desc_extra: + kfree(state); +err_desc_state: + return -ENOMEM; +} + static struct virtqueue *vring_create_virtqueue_packed( unsigned int index, unsigned int num, @@ -1919,6 +1946,7 @@ static struct virtqueue *vring_create_virtqueue_packed( { struct vring_virtqueue_packed vring_packed = {}; struct vring_virtqueue *vq; + int err; if (vring_alloc_queue_packed(&vring_packed, vdev, num)) goto err_ring; @@ -1963,21 +1991,15 @@ static struct virtqueue *vring_create_virtqueue_packed( vq->packed.event_flags_shadow = 0; vq->packed.avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL; - vq->packed.desc_state = kmalloc_array(num, - sizeof(struct vring_desc_state_packed), - GFP_KERNEL); - if (!vq->packed.desc_state) - goto err_desc_state; - - memset(vq->packed.desc_state, 0, - num * sizeof(struct vring_desc_state_packed)); + err = vring_alloc_state_extra_packed(&vring_packed); + if (err) + goto err_state_extra; /* Put everything in free lists. */ vq->free_head = 0; - vq->packed.desc_extra = vring_alloc_desc_extra(num); - if (!vq->packed.desc_extra) - goto err_desc_extra; + vq->packed.desc_state = vring_packed.desc_state; + vq->packed.desc_extra = vring_packed.desc_extra; /* No callback? Tell other side not to bother us. */ if (!callback) { @@ -1993,9 +2015,7 @@ static struct virtqueue *vring_create_virtqueue_packed( spin_unlock(&vdev->vqs_list_lock); return &vq->vq; -err_desc_extra: - kfree(vq->packed.desc_state); -err_desc_state: +err_state_extra: kfree(vq); err_vq: vring_free_packed(&vring_packed, vdev); -- cgit v1.2.3 From 1a107c87ebcf04ede1b86e5de192df34b6e50bd7 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:40 +0800 Subject: virtio_ring: packed: extract the logic of vring init Separate the logic of initializing vring, and subsequent patches will call it separately. This function completes the variable initialization of packed vring. It together with the logic of atatch constitutes the initialization of vring. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-21-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 533af061d009..a6f90e6e621b 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -1932,6 +1932,22 @@ err_desc_state: return -ENOMEM; } +static void virtqueue_vring_init_packed(struct vring_virtqueue_packed *vring_packed, + bool callback) +{ + vring_packed->next_avail_idx = 0; + vring_packed->avail_wrap_counter = 1; + vring_packed->event_flags_shadow = 0; + vring_packed->avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL; + + /* No callback? Tell other side not to bother us. */ + if (!callback) { + vring_packed->event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE; + vring_packed->vring.driver->flags = + cpu_to_le16(vring_packed->event_flags_shadow); + } +} + static struct virtqueue *vring_create_virtqueue_packed( unsigned int index, unsigned int num, @@ -1986,11 +2002,6 @@ static struct virtqueue *vring_create_virtqueue_packed( vq->packed.vring = vring_packed.vring; - vq->packed.next_avail_idx = 0; - vq->packed.avail_wrap_counter = 1; - vq->packed.event_flags_shadow = 0; - vq->packed.avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL; - err = vring_alloc_state_extra_packed(&vring_packed); if (err) goto err_state_extra; @@ -2001,12 +2012,7 @@ static struct virtqueue *vring_create_virtqueue_packed( vq->packed.desc_state = vring_packed.desc_state; vq->packed.desc_extra = vring_packed.desc_extra; - /* No callback? Tell other side not to bother us. */ - if (!callback) { - vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE; - vq->packed.vring.driver->flags = - cpu_to_le16(vq->packed.event_flags_shadow); - } + virtqueue_vring_init_packed(&vring_packed, !!callback); virtqueue_init(vq, num); -- cgit v1.2.3 From 51d649f14aae0986c62cf798b262f99b49836e88 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:41 +0800 Subject: virtio_ring: packed: extract the logic of attach vring Separate the logic of attach vring, the subsequent patch will call it separately. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-22-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index a6f90e6e621b..93d2a950f536 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -1948,6 +1948,15 @@ static void virtqueue_vring_init_packed(struct vring_virtqueue_packed *vring_pac } } +static void virtqueue_vring_attach_packed(struct vring_virtqueue *vq, + struct vring_virtqueue_packed *vring_packed) +{ + vq->packed = *vring_packed; + + /* Put everything in free lists. */ + vq->free_head = 0; +} + static struct virtqueue *vring_create_virtqueue_packed( unsigned int index, unsigned int num, @@ -1993,28 +2002,14 @@ static struct virtqueue *vring_create_virtqueue_packed( if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM)) vq->weak_barriers = false; - vq->packed.ring_dma_addr = vring_packed.ring_dma_addr; - vq->packed.driver_event_dma_addr = vring_packed.driver_event_dma_addr; - vq->packed.device_event_dma_addr = vring_packed.device_event_dma_addr; - - vq->packed.ring_size_in_bytes = vring_packed.ring_size_in_bytes; - vq->packed.event_size_in_bytes = vring_packed.event_size_in_bytes; - - vq->packed.vring = vring_packed.vring; - err = vring_alloc_state_extra_packed(&vring_packed); if (err) goto err_state_extra; - /* Put everything in free lists. */ - vq->free_head = 0; - - vq->packed.desc_state = vring_packed.desc_state; - vq->packed.desc_extra = vring_packed.desc_extra; - virtqueue_vring_init_packed(&vring_packed, !!callback); virtqueue_init(vq, num); + virtqueue_vring_attach_packed(vq, &vring_packed); spin_lock(&vdev->vqs_list_lock); list_add_tail(&vq->vq.list, &vdev->vqs); -- cgit v1.2.3 From 56775e141b18790f70e05537f3a1417565e766ac Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:42 +0800 Subject: virtio_ring: packed: introduce virtqueue_reinit_packed() Introduce a function to initialize vq without allocating new ring, desc_state, desc_extra. Subsequent patches will call this function after reset vq to reinitialize vq. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-23-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 93d2a950f536..db06fb0ddfd6 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -1957,6 +1957,18 @@ static void virtqueue_vring_attach_packed(struct vring_virtqueue *vq, vq->free_head = 0; } +static void virtqueue_reinit_packed(struct vring_virtqueue *vq) +{ + memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes); + memset(vq->packed.vring.driver, 0, vq->packed.event_size_in_bytes); + + /* we need to reset the desc.flags. For more, see is_used_desc_packed() */ + memset(vq->packed.vring.desc, 0, vq->packed.ring_size_in_bytes); + + virtqueue_init(vq, vq->packed.vring.num); + virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback); +} + static struct virtqueue *vring_create_virtqueue_packed( unsigned int index, unsigned int num, -- cgit v1.2.3 From 947f9fcf674f8f3f09e01dfb5d8e564650875878 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:43 +0800 Subject: virtio_ring: packed: introduce virtqueue_resize_packed() virtio ring packed supports resize. Only after the new vring is successfully allocated based on the new num, we will release the old vring. In any case, an error is returned, indicating that the vring still points to the old vring. In the case of an error, re-initialize(by virtqueue_reinit_packed()) the virtqueue to ensure that the vring can be used. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-24-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index db06fb0ddfd6..bea5a3448217 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -2036,6 +2036,36 @@ err_ring: return NULL; } +static int virtqueue_resize_packed(struct virtqueue *_vq, u32 num) +{ + struct vring_virtqueue_packed vring_packed = {}; + struct vring_virtqueue *vq = to_vvq(_vq); + struct virtio_device *vdev = _vq->vdev; + int err; + + if (vring_alloc_queue_packed(&vring_packed, vdev, num)) + goto err_ring; + + err = vring_alloc_state_extra_packed(&vring_packed); + if (err) + goto err_state_extra; + + vring_free(&vq->vq); + + virtqueue_vring_init_packed(&vring_packed, !!vq->vq.callback); + + virtqueue_init(vq, vring_packed.vring.num); + virtqueue_vring_attach_packed(vq, &vring_packed); + + return 0; + +err_state_extra: + vring_free_packed(&vring_packed, vdev); +err_ring: + virtqueue_reinit_packed(vq); + return -ENOMEM; +} + /* * Generic functions and exported symbols. -- cgit v1.2.3 From c790e8e1817f1a17c05e64f1c4f16f231b8529d5 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:44 +0800 Subject: virtio_ring: introduce virtqueue_resize() Introduce virtqueue_resize() to implement the resize of vring. Based on these, the driver can dynamically adjust the size of the vring. For example: ethtool -G. virtqueue_resize() implements resize based on the vq reset function. In case of failure to allocate a new vring, it will give up resize and use the original vring. During this process, if the re-enable reset vq fails, the vq can no longer be used. Although the probability of this situation is not high. The parameter recycle is used to recycle the buffer that is no longer used. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-25-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 69 ++++++++++++++++++++++++++++++++++++++++++++ include/linux/virtio.h | 3 ++ 2 files changed, 72 insertions(+) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index bea5a3448217..6447a09e2e38 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -2539,6 +2539,75 @@ struct virtqueue *vring_create_virtqueue( } EXPORT_SYMBOL_GPL(vring_create_virtqueue); +/** + * virtqueue_resize - resize the vring of vq + * @_vq: the struct virtqueue we're talking about. + * @num: new ring num + * @recycle: callback for recycle the useless buffer + * + * When it is really necessary to create a new vring, it will set the current vq + * into the reset state. Then call the passed callback to recycle the buffer + * that is no longer used. Only after the new vring is successfully created, the + * old vring will be released. + * + * Caller must ensure we don't call this with other virtqueue operations + * at the same time (except where noted). + * + * Returns zero or a negative error. + * 0: success. + * -ENOMEM: Failed to allocate a new ring, fall back to the original ring size. + * vq can still work normally + * -EBUSY: Failed to sync with device, vq may not work properly + * -ENOENT: Transport or device not supported + * -E2BIG/-EINVAL: num error + * -EPERM: Operation not permitted + * + */ +int virtqueue_resize(struct virtqueue *_vq, u32 num, + void (*recycle)(struct virtqueue *vq, void *buf)) +{ + struct vring_virtqueue *vq = to_vvq(_vq); + struct virtio_device *vdev = vq->vq.vdev; + void *buf; + int err; + + if (!vq->we_own_ring) + return -EPERM; + + if (num > vq->vq.num_max) + return -E2BIG; + + if (!num) + return -EINVAL; + + if ((vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num) == num) + return 0; + + if (!vdev->config->disable_vq_and_reset) + return -ENOENT; + + if (!vdev->config->enable_vq_after_reset) + return -ENOENT; + + err = vdev->config->disable_vq_and_reset(_vq); + if (err) + return err; + + while ((buf = virtqueue_detach_unused_buf(_vq)) != NULL) + recycle(_vq, buf); + + if (vq->packed_ring) + err = virtqueue_resize_packed(_vq, num); + else + err = virtqueue_resize_split(_vq, num); + + if (vdev->config->enable_vq_after_reset(_vq)) + return -EBUSY; + + return err; +} +EXPORT_SYMBOL_GPL(virtqueue_resize); + /* Only available for split ring */ struct virtqueue *vring_new_virtqueue(unsigned int index, unsigned int num, diff --git a/include/linux/virtio.h b/include/linux/virtio.h index 129bde7521e3..62e31bca5602 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -91,6 +91,9 @@ dma_addr_t virtqueue_get_desc_addr(struct virtqueue *vq); dma_addr_t virtqueue_get_avail_addr(struct virtqueue *vq); dma_addr_t virtqueue_get_used_addr(struct virtqueue *vq); +int virtqueue_resize(struct virtqueue *vq, u32 num, + void (*recycle)(struct virtqueue *vq, void *buf)); + /** * virtio_device - representation of a device using virtio * @index: unique position on the virtio bus -- cgit v1.2.3 From 3251063155032729b8793ac3957136ae25c0bafa Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:46 +0800 Subject: virtio: allow to unbreak/break virtqueue individually This patch allows the new introduced __virtqueue_break()/__virtqueue_unbreak() to break/unbreak the virtqueue. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-27-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 24 ++++++++++++++++++++++++ include/linux/virtio.h | 3 +++ 2 files changed, 27 insertions(+) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 6447a09e2e38..accb3ae6cc95 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -2724,6 +2724,30 @@ unsigned int virtqueue_get_vring_size(struct virtqueue *_vq) } EXPORT_SYMBOL_GPL(virtqueue_get_vring_size); +/* + * This function should only be called by the core, not directly by the driver. + */ +void __virtqueue_break(struct virtqueue *_vq) +{ + struct vring_virtqueue *vq = to_vvq(_vq); + + /* Pairs with READ_ONCE() in virtqueue_is_broken(). */ + WRITE_ONCE(vq->broken, true); +} +EXPORT_SYMBOL_GPL(__virtqueue_break); + +/* + * This function should only be called by the core, not directly by the driver. + */ +void __virtqueue_unbreak(struct virtqueue *_vq) +{ + struct vring_virtqueue *vq = to_vvq(_vq); + + /* Pairs with READ_ONCE() in virtqueue_is_broken(). */ + WRITE_ONCE(vq->broken, false); +} +EXPORT_SYMBOL_GPL(__virtqueue_unbreak); + bool virtqueue_is_broken(struct virtqueue *_vq) { struct vring_virtqueue *vq = to_vvq(_vq); diff --git a/include/linux/virtio.h b/include/linux/virtio.h index 62e31bca5602..d45ee82a4470 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -138,6 +138,9 @@ bool is_virtio_device(struct device *dev); void virtio_break_device(struct virtio_device *dev); void __virtio_unbreak_device(struct virtio_device *dev); +void __virtqueue_break(struct virtqueue *_vq); +void __virtqueue_unbreak(struct virtqueue *_vq); + void virtio_config_changed(struct virtio_device *dev); #ifdef CONFIG_PM_SLEEP int virtio_device_freeze(struct virtio_device *dev); -- cgit v1.2.3 From 4913e85441b40386c4bb093f188b955d8165f1b7 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:48 +0800 Subject: virtio_ring: struct virtqueue introduce reset Introduce a new member reset to the structure virtqueue to determine whether the current vq is in the reset state. Subsequent patches will use it. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-29-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 2 ++ include/linux/virtio.h | 2 ++ 2 files changed, 4 insertions(+) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index accb3ae6cc95..d66c8e6d0ef3 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -1996,6 +1996,7 @@ static struct virtqueue *vring_create_virtqueue_packed( vq->vq.vdev = vdev; vq->vq.name = name; vq->vq.index = index; + vq->vq.reset = false; vq->we_own_ring = true; vq->notify = notify; vq->weak_barriers = weak_barriers; @@ -2481,6 +2482,7 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, vq->vq.vdev = vdev; vq->vq.name = name; vq->vq.index = index; + vq->vq.reset = false; vq->we_own_ring = false; vq->notify = notify; vq->weak_barriers = weak_barriers; diff --git a/include/linux/virtio.h b/include/linux/virtio.h index d45ee82a4470..a3f73bb6733e 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -20,6 +20,7 @@ * @index: the zero-based ordinal number for this queue. * @num_free: number of elements we expect to be able to fit. * @num_max: the maximum number of elements supported by the device. + * @reset: vq is in reset state or not. * * A note on @num_free: with indirect buffers, each buffer needs one * element in the queue, otherwise a buffer will need one element per @@ -34,6 +35,7 @@ struct virtqueue { unsigned int num_free; unsigned int num_max; void *priv; + bool reset; }; int virtqueue_add_outbuf(struct virtqueue *vq, -- cgit v1.2.3 From 0b50cece0b7857732d2055f2c77f8730c10f9196 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:50 +0800 Subject: virtio_pci: introduce helper to get/set queue reset Introduce new helpers to implement queue reset and get queue reset status. https://github.com/oasis-tcs/virtio-spec/issues/124 https://github.com/oasis-tcs/virtio-spec/issues/139 Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-31-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_pci_modern_dev.c | 39 ++++++++++++++++++++++++++++++++++ include/linux/virtio_pci_modern.h | 2 ++ 2 files changed, 41 insertions(+) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_pci_modern_dev.c b/drivers/virtio/virtio_pci_modern_dev.c index fa2a9445bb18..869cb46bef96 100644 --- a/drivers/virtio/virtio_pci_modern_dev.c +++ b/drivers/virtio/virtio_pci_modern_dev.c @@ -3,6 +3,7 @@ #include #include #include +#include /* * vp_modern_map_capability - map a part of virtio pci capability @@ -474,6 +475,44 @@ void vp_modern_set_status(struct virtio_pci_modern_device *mdev, } EXPORT_SYMBOL_GPL(vp_modern_set_status); +/* + * vp_modern_get_queue_reset - get the queue reset status + * @mdev: the modern virtio-pci device + * @index: queue index + */ +int vp_modern_get_queue_reset(struct virtio_pci_modern_device *mdev, u16 index) +{ + struct virtio_pci_modern_common_cfg __iomem *cfg; + + cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common; + + vp_iowrite16(index, &cfg->cfg.queue_select); + return vp_ioread16(&cfg->queue_reset); +} +EXPORT_SYMBOL_GPL(vp_modern_get_queue_reset); + +/* + * vp_modern_set_queue_reset - reset the queue + * @mdev: the modern virtio-pci device + * @index: queue index + */ +void vp_modern_set_queue_reset(struct virtio_pci_modern_device *mdev, u16 index) +{ + struct virtio_pci_modern_common_cfg __iomem *cfg; + + cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common; + + vp_iowrite16(index, &cfg->cfg.queue_select); + vp_iowrite16(1, &cfg->queue_reset); + + while (vp_ioread16(&cfg->queue_reset)) + msleep(1); + + while (vp_ioread16(&cfg->cfg.queue_enable)) + msleep(1); +} +EXPORT_SYMBOL_GPL(vp_modern_set_queue_reset); + /* * vp_modern_queue_vector - set the MSIX vector for a specific virtqueue * @mdev: the modern virtio-pci device diff --git a/include/linux/virtio_pci_modern.h b/include/linux/virtio_pci_modern.h index 05123b9a606f..c4eeb79b0139 100644 --- a/include/linux/virtio_pci_modern.h +++ b/include/linux/virtio_pci_modern.h @@ -113,4 +113,6 @@ void __iomem * vp_modern_map_vq_notify(struct virtio_pci_modern_device *mdev, u16 index, resource_size_t *pa); int vp_modern_probe(struct virtio_pci_modern_device *mdev); void vp_modern_remove(struct virtio_pci_modern_device *mdev); +int vp_modern_get_queue_reset(struct virtio_pci_modern_device *mdev, u16 index); +void vp_modern_set_queue_reset(struct virtio_pci_modern_device *mdev, u16 index); #endif -- cgit v1.2.3 From 56bdc06139404b3b06ed75ec99b93445d7e0b8c3 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:51 +0800 Subject: virtio_pci: extract the logic of active vq for modern pci Introduce vp_active_vq() to configure vring to backend after vq attach vring. And configure vq vector if necessary. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-32-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_pci_modern.c | 46 +++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 18 deletions(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index e7e0b8c850f6..9041d9a41b7d 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -176,6 +176,29 @@ static void vp_reset(struct virtio_device *vdev) vp_synchronize_vectors(vdev); } +static int vp_active_vq(struct virtqueue *vq, u16 msix_vec) +{ + struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); + struct virtio_pci_modern_device *mdev = &vp_dev->mdev; + unsigned long index; + + index = vq->index; + + /* activate the queue */ + vp_modern_set_queue_size(mdev, index, virtqueue_get_vring_size(vq)); + vp_modern_queue_address(mdev, index, virtqueue_get_desc_addr(vq), + virtqueue_get_avail_addr(vq), + virtqueue_get_used_addr(vq)); + + if (msix_vec != VIRTIO_MSI_NO_VECTOR) { + msix_vec = vp_modern_queue_vector(mdev, index, msix_vec); + if (msix_vec == VIRTIO_MSI_NO_VECTOR) + return -EBUSY; + } + + return 0; +} + static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector) { return vp_modern_config_vector(&vp_dev->mdev, vector); @@ -220,32 +243,19 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, vq->num_max = num; - /* activate the queue */ - vp_modern_set_queue_size(mdev, index, virtqueue_get_vring_size(vq)); - vp_modern_queue_address(mdev, index, virtqueue_get_desc_addr(vq), - virtqueue_get_avail_addr(vq), - virtqueue_get_used_addr(vq)); + err = vp_active_vq(vq, msix_vec); + if (err) + goto err; vq->priv = (void __force *)vp_modern_map_vq_notify(mdev, index, NULL); if (!vq->priv) { err = -ENOMEM; - goto err_map_notify; - } - - if (msix_vec != VIRTIO_MSI_NO_VECTOR) { - msix_vec = vp_modern_queue_vector(mdev, index, msix_vec); - if (msix_vec == VIRTIO_MSI_NO_VECTOR) { - err = -EBUSY; - goto err_assign_vector; - } + goto err; } return vq; -err_assign_vector: - if (!mdev->notify_base) - pci_iounmap(mdev->pci_dev, (void __iomem __force *)vq->priv); -err_map_notify: +err: vring_del_virtqueue(vq); return ERR_PTR(err); } -- cgit v1.2.3 From 04ca0b0b16f11faf74fa92468dab51b8372586cd Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:52 +0800 Subject: virtio_pci: support VIRTIO_F_RING_RESET This patch implements virtio pci support for QUEUE RESET. Performing reset on a queue is divided into these steps: 1. notify the device to reset the queue 2. recycle the buffer submitted 3. reset the vring (may re-alloc) 4. mmap vring to device, and enable the queue This patch implements virtio_reset_vq(), virtio_enable_resetq() in the pci scenario. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-33-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_pci_common.c | 12 ++++-- drivers/virtio/virtio_pci_modern.c | 88 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 3 deletions(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c index ca51fcc9daab..ad258a9d3b9f 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -214,9 +214,15 @@ static void vp_del_vq(struct virtqueue *vq) struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index]; unsigned long flags; - spin_lock_irqsave(&vp_dev->lock, flags); - list_del(&info->node); - spin_unlock_irqrestore(&vp_dev->lock, flags); + /* + * If it fails during re-enable reset vq. This way we won't rejoin + * info->node to the queue. Prevent unexpected irqs. + */ + if (!vq->reset) { + spin_lock_irqsave(&vp_dev->lock, flags); + list_del(&info->node); + spin_unlock_irqrestore(&vp_dev->lock, flags); + } vp_dev->del_vq(info); kfree(info); diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index 9041d9a41b7d..c3b9f2761849 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -34,6 +34,9 @@ static void vp_transport_features(struct virtio_device *vdev, u64 features) if ((features & BIT_ULL(VIRTIO_F_SR_IOV)) && pci_find_ext_capability(pci_dev, PCI_EXT_CAP_ID_SRIOV)) __virtio_set_bit(vdev, VIRTIO_F_SR_IOV); + + if (features & BIT_ULL(VIRTIO_F_RING_RESET)) + __virtio_set_bit(vdev, VIRTIO_F_RING_RESET); } /* virtio config->finalize_features() implementation */ @@ -199,6 +202,87 @@ static int vp_active_vq(struct virtqueue *vq, u16 msix_vec) return 0; } +static int vp_modern_disable_vq_and_reset(struct virtqueue *vq) +{ + struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); + struct virtio_pci_modern_device *mdev = &vp_dev->mdev; + struct virtio_pci_vq_info *info; + unsigned long flags; + + if (!virtio_has_feature(vq->vdev, VIRTIO_F_RING_RESET)) + return -ENOENT; + + vp_modern_set_queue_reset(mdev, vq->index); + + info = vp_dev->vqs[vq->index]; + + /* delete vq from irq handler */ + spin_lock_irqsave(&vp_dev->lock, flags); + list_del(&info->node); + spin_unlock_irqrestore(&vp_dev->lock, flags); + + INIT_LIST_HEAD(&info->node); + +#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION + __virtqueue_break(vq); +#endif + + /* For the case where vq has an exclusive irq, call synchronize_irq() to + * wait for completion. + * + * note: We can't use disable_irq() since it conflicts with the affinity + * managed IRQ that is used by some drivers. + */ + if (vp_dev->per_vq_vectors && info->msix_vector != VIRTIO_MSI_NO_VECTOR) + synchronize_irq(pci_irq_vector(vp_dev->pci_dev, info->msix_vector)); + + vq->reset = true; + + return 0; +} + +static int vp_modern_enable_vq_after_reset(struct virtqueue *vq) +{ + struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); + struct virtio_pci_modern_device *mdev = &vp_dev->mdev; + struct virtio_pci_vq_info *info; + unsigned long flags, index; + int err; + + if (!vq->reset) + return -EBUSY; + + index = vq->index; + info = vp_dev->vqs[index]; + + if (vp_modern_get_queue_reset(mdev, index)) + return -EBUSY; + + if (vp_modern_get_queue_enable(mdev, index)) + return -EBUSY; + + err = vp_active_vq(vq, info->msix_vector); + if (err) + return err; + + if (vq->callback) { + spin_lock_irqsave(&vp_dev->lock, flags); + list_add(&info->node, &vp_dev->virtqueues); + spin_unlock_irqrestore(&vp_dev->lock, flags); + } else { + INIT_LIST_HEAD(&info->node); + } + +#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION + __virtqueue_unbreak(vq); +#endif + + vp_modern_set_queue_enable(&vp_dev->mdev, index, true); + vq->reset = false; + + return 0; +} + static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector) { return vp_modern_config_vector(&vp_dev->mdev, vector); @@ -413,6 +497,8 @@ static const struct virtio_config_ops virtio_pci_config_nodev_ops = { .set_vq_affinity = vp_set_vq_affinity, .get_vq_affinity = vp_get_vq_affinity, .get_shm_region = vp_get_shm_region, + .disable_vq_and_reset = vp_modern_disable_vq_and_reset, + .enable_vq_after_reset = vp_modern_enable_vq_after_reset, }; static const struct virtio_config_ops virtio_pci_config_ops = { @@ -431,6 +517,8 @@ static const struct virtio_config_ops virtio_pci_config_ops = { .set_vq_affinity = vp_set_vq_affinity, .get_vq_affinity = vp_get_vq_affinity, .get_shm_region = vp_get_shm_region, + .disable_vq_and_reset = vp_modern_disable_vq_and_reset, + .enable_vq_after_reset = vp_modern_enable_vq_after_reset, }; /* the PCI probing function */ -- cgit v1.2.3 From a10fba0377145fccefea4dc4dd5915b7ed87e546 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:53 +0800 Subject: virtio: find_vqs() add arg sizes find_vqs() adds a new parameter sizes to specify the size of each vq vring. NULL as sizes means that all queues in find_vqs() use the maximum size. A value in the array is 0, which means that the corresponding queue uses the maximum size. In the split scenario, the meaning of size is the largest size, because it may be limited by memory, the virtio core will try a smaller size. And the size is power of 2. Signed-off-by: Xuan Zhuo Acked-by: Hans de Goede Reviewed-by: Mathieu Poirier Acked-by: Jason Wang Message-Id: <20220801063902.129329-34-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- arch/um/drivers/virtio_uml.c | 2 +- drivers/platform/mellanox/mlxbf-tmfifo.c | 1 + drivers/remoteproc/remoteproc_virtio.c | 1 + drivers/s390/virtio/virtio_ccw.c | 1 + drivers/virtio/virtio_mmio.c | 1 + drivers/virtio/virtio_pci_common.c | 2 +- drivers/virtio/virtio_pci_common.h | 2 +- drivers/virtio/virtio_pci_modern.c | 7 +++++-- drivers/virtio/virtio_vdpa.c | 1 + include/linux/virtio_config.h | 14 +++++++++----- 10 files changed, 22 insertions(+), 10 deletions(-) (limited to 'drivers/virtio') diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c index e719af8bdf56..79e38afd4b91 100644 --- a/arch/um/drivers/virtio_uml.c +++ b/arch/um/drivers/virtio_uml.c @@ -1011,7 +1011,7 @@ error_kzalloc: static int vu_find_vqs(struct virtio_device *vdev, unsigned nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], const bool *ctx, + const char * const names[], u32 sizes[], const bool *ctx, struct irq_affinity *desc) { struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev); diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c b/drivers/platform/mellanox/mlxbf-tmfifo.c index 1ae3c56b66b0..8be13d416f48 100644 --- a/drivers/platform/mellanox/mlxbf-tmfifo.c +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c @@ -928,6 +928,7 @@ static int mlxbf_tmfifo_virtio_find_vqs(struct virtio_device *vdev, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], + u32 sizes[], const bool *ctx, struct irq_affinity *desc) { diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c index 0f7706e23eb9..81c4f5776109 100644 --- a/drivers/remoteproc/remoteproc_virtio.c +++ b/drivers/remoteproc/remoteproc_virtio.c @@ -158,6 +158,7 @@ static int rproc_virtio_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], + u32 sizes[], const bool * ctx, struct irq_affinity *desc) { diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c index 6b86d0280d6b..72500cd2dbf5 100644 --- a/drivers/s390/virtio/virtio_ccw.c +++ b/drivers/s390/virtio/virtio_ccw.c @@ -635,6 +635,7 @@ static int virtio_ccw_find_vqs(struct virtio_device *vdev, unsigned nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], + u32 sizes[], const bool *ctx, struct irq_affinity *desc) { diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index 3ff746e3f24a..dfcecfd7aba1 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -474,6 +474,7 @@ static int vm_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], + u32 sizes[], const bool *ctx, struct irq_affinity *desc) { diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c index ad258a9d3b9f..7ad734584823 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -396,7 +396,7 @@ out_del_vqs: /* the config->find_vqs() implementation */ int vp_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], const bool *ctx, + const char * const names[], u32 sizes[], const bool *ctx, struct irq_affinity *desc) { int err; diff --git a/drivers/virtio/virtio_pci_common.h b/drivers/virtio/virtio_pci_common.h index 23112d84218f..a5ff838b85a5 100644 --- a/drivers/virtio/virtio_pci_common.h +++ b/drivers/virtio/virtio_pci_common.h @@ -110,7 +110,7 @@ void vp_del_vqs(struct virtio_device *vdev); /* the config->find_vqs() implementation */ int vp_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], const bool *ctx, + const char * const names[], u32 sizes[], const bool *ctx, struct irq_affinity *desc); const char *vp_bus_name(struct virtio_device *vdev); diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index c3b9f2761849..be51ec849252 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -347,12 +347,15 @@ err: static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], const bool *ctx, + const char * const names[], + u32 sizes[], + const bool *ctx, struct irq_affinity *desc) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); struct virtqueue *vq; - int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, ctx, desc); + int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, sizes, ctx, + desc); if (rc) return rc; diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index 9670cc79371d..832d2c5b1b19 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -269,6 +269,7 @@ static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], + u32 sizes[], const bool *ctx, struct irq_affinity *desc) { diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h index 36ec7be1f480..888f7e96f0c7 100644 --- a/include/linux/virtio_config.h +++ b/include/linux/virtio_config.h @@ -55,6 +55,7 @@ struct virtio_shm_region { * include a NULL entry for vqs that do not need a callback * names: array of virtqueue names (mainly for debugging) * include a NULL entry for vqs unused by driver + * sizes: array of virtqueue sizes * Returns 0 on success or error status * @del_vqs: free virtqueues found by find_vqs(). * @synchronize_cbs: synchronize with the virtqueue callbacks (optional) @@ -103,7 +104,9 @@ struct virtio_config_ops { void (*reset)(struct virtio_device *vdev); int (*find_vqs)(struct virtio_device *, unsigned nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], const bool *ctx, + const char * const names[], + u32 sizes[], + const bool *ctx, struct irq_affinity *desc); void (*del_vqs)(struct virtio_device *); void (*synchronize_cbs)(struct virtio_device *); @@ -212,7 +215,7 @@ struct virtqueue *virtio_find_single_vq(struct virtio_device *vdev, const char *names[] = { n }; struct virtqueue *vq; int err = vdev->config->find_vqs(vdev, 1, &vq, callbacks, names, NULL, - NULL); + NULL, NULL); if (err < 0) return ERR_PTR(err); return vq; @@ -224,7 +227,8 @@ int virtio_find_vqs(struct virtio_device *vdev, unsigned nvqs, const char * const names[], struct irq_affinity *desc) { - return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, NULL, desc); + return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, NULL, + NULL, desc); } static inline @@ -233,8 +237,8 @@ int virtio_find_vqs_ctx(struct virtio_device *vdev, unsigned nvqs, const char * const names[], const bool *ctx, struct irq_affinity *desc) { - return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, ctx, - desc); + return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, NULL, + ctx, desc); } /** -- cgit v1.2.3 From cdb44806fca2d0ad29ca644cbf1505433902ee0c Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:54 +0800 Subject: virtio_pci: support the arg sizes of find_vqs() Virtio PCI supports new parameter sizes of find_vqs(). Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-35-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_pci_common.c | 18 ++++++++++-------- drivers/virtio/virtio_pci_common.h | 1 + drivers/virtio/virtio_pci_legacy.c | 6 +++++- drivers/virtio/virtio_pci_modern.c | 10 +++++++--- 4 files changed, 23 insertions(+), 12 deletions(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c index 7ad734584823..00ad476a815d 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -174,6 +174,7 @@ error: static struct virtqueue *vp_setup_vq(struct virtio_device *vdev, unsigned int index, void (*callback)(struct virtqueue *vq), const char *name, + u32 size, bool ctx, u16 msix_vec) { @@ -186,7 +187,7 @@ static struct virtqueue *vp_setup_vq(struct virtio_device *vdev, unsigned int in if (!info) return ERR_PTR(-ENOMEM); - vq = vp_dev->setup_vq(vp_dev, info, index, callback, name, ctx, + vq = vp_dev->setup_vq(vp_dev, info, index, callback, name, size, ctx, msix_vec); if (IS_ERR(vq)) goto out_info; @@ -283,7 +284,7 @@ void vp_del_vqs(struct virtio_device *vdev) static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], bool per_vq_vectors, + const char * const names[], u32 sizes[], bool per_vq_vectors, const bool *ctx, struct irq_affinity *desc) { @@ -326,8 +327,8 @@ static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned int nvqs, else msix_vec = VP_MSIX_VQ_VECTOR; vqs[i] = vp_setup_vq(vdev, queue_idx++, callbacks[i], names[i], - ctx ? ctx[i] : false, - msix_vec); + sizes ? sizes[i] : 0, + ctx ? ctx[i] : false, msix_vec); if (IS_ERR(vqs[i])) { err = PTR_ERR(vqs[i]); goto error_find; @@ -357,7 +358,7 @@ error_find: static int vp_find_vqs_intx(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], const bool *ctx) + const char * const names[], u32 sizes[], const bool *ctx) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); int i, err, queue_idx = 0; @@ -379,6 +380,7 @@ static int vp_find_vqs_intx(struct virtio_device *vdev, unsigned int nvqs, continue; } vqs[i] = vp_setup_vq(vdev, queue_idx++, callbacks[i], names[i], + sizes ? sizes[i] : 0, ctx ? ctx[i] : false, VIRTIO_MSI_NO_VECTOR); if (IS_ERR(vqs[i])) { @@ -402,15 +404,15 @@ int vp_find_vqs(struct virtio_device *vdev, unsigned int nvqs, int err; /* Try MSI-X with one vector per queue. */ - err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, true, ctx, desc); + err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, sizes, true, ctx, desc); if (!err) return 0; /* Fallback: MSI-X with one vector for config, one shared for queues. */ - err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, false, ctx, desc); + err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, sizes, false, ctx, desc); if (!err) return 0; /* Finally fall back to regular interrupts. */ - return vp_find_vqs_intx(vdev, nvqs, vqs, callbacks, names, ctx); + return vp_find_vqs_intx(vdev, nvqs, vqs, callbacks, names, sizes, ctx); } const char *vp_bus_name(struct virtio_device *vdev) diff --git a/drivers/virtio/virtio_pci_common.h b/drivers/virtio/virtio_pci_common.h index a5ff838b85a5..c0448378b698 100644 --- a/drivers/virtio/virtio_pci_common.h +++ b/drivers/virtio/virtio_pci_common.h @@ -80,6 +80,7 @@ struct virtio_pci_device { unsigned int idx, void (*callback)(struct virtqueue *vq), const char *name, + u32 size, bool ctx, u16 msix_vec); void (*del_vq)(struct virtio_pci_vq_info *info); diff --git a/drivers/virtio/virtio_pci_legacy.c b/drivers/virtio/virtio_pci_legacy.c index 2257f1b3d8ae..d75e5c4e637f 100644 --- a/drivers/virtio/virtio_pci_legacy.c +++ b/drivers/virtio/virtio_pci_legacy.c @@ -112,6 +112,7 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, unsigned int index, void (*callback)(struct virtqueue *vq), const char *name, + u32 size, bool ctx, u16 msix_vec) { @@ -125,10 +126,13 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, if (!num || vp_legacy_get_queue_enable(&vp_dev->ldev, index)) return ERR_PTR(-ENOENT); + if (!size || size > num) + size = num; + info->msix_vector = msix_vec; /* create the vring */ - vq = vring_create_virtqueue(index, num, + vq = vring_create_virtqueue(index, size, VIRTIO_PCI_VRING_ALIGN, &vp_dev->vdev, true, false, ctx, vp_notify, callback, name); diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index be51ec849252..f7965c5dd36b 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -293,6 +293,7 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, unsigned int index, void (*callback)(struct virtqueue *vq), const char *name, + u32 size, bool ctx, u16 msix_vec) { @@ -310,15 +311,18 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, if (!num || vp_modern_get_queue_enable(mdev, index)) return ERR_PTR(-ENOENT); - if (num & (num - 1)) { - dev_warn(&vp_dev->pci_dev->dev, "bad queue size %u", num); + if (!size || size > num) + size = num; + + if (size & (size - 1)) { + dev_warn(&vp_dev->pci_dev->dev, "bad queue size %u", size); return ERR_PTR(-EINVAL); } info->msix_vector = msix_vec; /* create the vring */ - vq = vring_create_virtqueue(index, num, + vq = vring_create_virtqueue(index, size, SMP_CACHE_BYTES, &vp_dev->vdev, true, true, ctx, vp_notify, callback, name); -- cgit v1.2.3 From fbed86abba6e0472d98079790e58060e4332608a Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:55 +0800 Subject: virtio_mmio: support the arg sizes of find_vqs() Virtio MMIO support the new parameter sizes of find_vqs(). Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-36-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_mmio.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index dfcecfd7aba1..c492a57531c6 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -360,7 +360,7 @@ static void vm_synchronize_cbs(struct virtio_device *vdev) static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int index, void (*callback)(struct virtqueue *vq), - const char *name, bool ctx) + const char *name, u32 size, bool ctx) { struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); struct virtio_mmio_vq_info *info; @@ -395,8 +395,11 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in goto error_new_virtqueue; } + if (!size || size > num) + size = num; + /* Create the vring */ - vq = vring_create_virtqueue(index, num, VIRTIO_MMIO_VRING_ALIGN, vdev, + vq = vring_create_virtqueue(index, size, VIRTIO_MMIO_VRING_ALIGN, vdev, true, true, ctx, vm_notify, callback, name); if (!vq) { err = -ENOMEM; @@ -500,6 +503,7 @@ static int vm_find_vqs(struct virtio_device *vdev, unsigned int nvqs, } vqs[i] = vm_setup_vq(vdev, queue_idx++, callbacks[i], names[i], + sizes ? sizes[i] : 0, ctx ? ctx[i] : false); if (IS_ERR(vqs[i])) { vm_del_vqs(vdev); -- cgit v1.2.3 From 95bf9798779af166b1d6d184d794834214a1a261 Mon Sep 17 00:00:00 2001 From: Bo Liu Date: Wed, 6 Jul 2022 23:17:51 -0400 Subject: virtio: Check dev_set_name() return value It's possible that dev_set_name() returns -ENOMEM, catch and handle this. Signed-off-by: Bo Liu Message-Id: <20220707031751.4802-1-liubo03@inspur.com> Signed-off-by: Michael S. Tsirkin Acked-by: Jason Wang --- drivers/virtio/virtio.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index 7deeed30d1f3..ddd4466da83b 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c @@ -428,7 +428,9 @@ int register_virtio_device(struct virtio_device *dev) goto out; dev->index = err; - dev_set_name(&dev->dev, "virtio%u", dev->index); + err = dev_set_name(&dev->dev, "virtio%u", dev->index); + if (err) + goto out_ida_remove; err = virtio_device_of_init(dev); if (err) -- cgit v1.2.3 From 99e8927d8a4da8eb8a8a5904dc13a3156be8e7c0 Mon Sep 17 00:00:00 2001 From: Bo Liu Date: Wed, 10 Aug 2022 04:51:51 -0400 Subject: virtio_vdpa: support the arg sizes of find_vqs() Virtio vdpa support the new parameter sizes of find_vqs(). Signed-off-by: Bo Liu Message-Id: <20220810085151.7251-1-liubo03@inspur.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_vdpa.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'drivers/virtio') diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index 832d2c5b1b19..9bc4d110b800 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -131,7 +131,7 @@ static irqreturn_t virtio_vdpa_virtqueue_cb(void *private) static struct virtqueue * virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, void (*callback)(struct virtqueue *vq), - const char *name, bool ctx) + const char *name, u32 size, bool ctx) { struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); struct vdpa_device *vdpa = vd_get_vdpa(vdev); @@ -168,14 +168,17 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, goto error_new_virtqueue; } + if (!size || size > max_num) + size = max_num; + if (ops->get_vq_num_min) min_num = ops->get_vq_num_min(vdpa); - may_reduce_num = (max_num == min_num) ? false : true; + may_reduce_num = (size == min_num) ? false : true; /* Create the vring */ align = ops->get_vq_align(vdpa); - vq = vring_create_virtqueue(index, max_num, align, vdev, + vq = vring_create_virtqueue(index, size, align, vdev, true, may_reduce_num, ctx, virtio_vdpa_notify, callback, name); if (!vq) { @@ -285,9 +288,9 @@ static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs, continue; } - vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++, - callbacks[i], names[i], ctx ? - ctx[i] : false); + vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++, callbacks[i], + names[i], sizes ? sizes[i] : 0, + ctx ? ctx[i] : false); if (IS_ERR(vqs[i])) { err = PTR_ERR(vqs[i]); goto err_setup_vq; -- cgit v1.2.3