summaryrefslogtreecommitdiff
path: root/virt
diff options
context:
space:
mode:
Diffstat (limited to 'virt')
-rw-r--r--virt/kvm/dirty_ring.c3
-rw-r--r--virt/kvm/eventfd.c12
-rw-r--r--virt/kvm/guest_memfd.c38
-rw-r--r--virt/kvm/kvm_main.c9
-rw-r--r--virt/kvm/kvm_mm.h7
-rw-r--r--virt/kvm/vfio.c98
6 files changed, 72 insertions, 95 deletions
diff --git a/virt/kvm/dirty_ring.c b/virt/kvm/dirty_ring.c
index 02bc6b00d76c..572b854edf74 100644
--- a/virt/kvm/dirty_ring.c
+++ b/virt/kvm/dirty_ring.c
@@ -63,7 +63,8 @@ static void kvm_reset_dirty_gfn(struct kvm *kvm, u32 slot, u64 offset, u64 mask)
memslot = id_to_memslot(__kvm_memslots(kvm, as_id), id);
- if (!memslot || (offset + __fls(mask)) >= memslot->npages)
+ if (!memslot || offset >= memslot->npages ||
+ offset + __fls(mask) >= memslot->npages)
return;
KVM_MMU_LOCK(kvm);
diff --git a/virt/kvm/eventfd.c b/virt/kvm/eventfd.c
index 0e8b8a2c5b79..93ad2ebc963f 100644
--- a/virt/kvm/eventfd.c
+++ b/virt/kvm/eventfd.c
@@ -24,6 +24,7 @@
#include <linux/slab.h>
#include <linux/seqlock.h>
#include <linux/irqbypass.h>
+#include <linux/unaligned.h>
#include <trace/events/kvm.h>
#include <kvm/iodev.h>
@@ -779,21 +780,18 @@ ioeventfd_in_range(struct _ioeventfd *p, gpa_t addr, int len, const void *val)
return true;
/* otherwise, we have to actually compare the data */
-
- BUG_ON(!IS_ALIGNED((unsigned long)val, len));
-
switch (len) {
case 1:
- _val = *(u8 *)val;
+ _val = get_unaligned((u8 *)val);
break;
case 2:
- _val = *(u16 *)val;
+ _val = get_unaligned((u16 *)val);
break;
case 4:
- _val = *(u32 *)val;
+ _val = get_unaligned((u32 *)val);
break;
case 8:
- _val = *(u64 *)val;
+ _val = get_unaligned((u64 *)val);
break;
default:
return false;
diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c
index 69c9d6d546b2..db57c5766ab6 100644
--- a/virt/kvm/guest_memfd.c
+++ b/virt/kvm/guest_memfd.c
@@ -157,7 +157,7 @@ static enum kvm_gfn_range_filter kvm_gmem_get_invalidate_filter(struct inode *in
return KVM_FILTER_PRIVATE;
}
-static void __kvm_gmem_invalidate_begin(struct gmem_file *f, pgoff_t start,
+static void __kvm_gmem_invalidate_start(struct gmem_file *f, pgoff_t start,
pgoff_t end,
enum kvm_gfn_range_filter attr_filter)
{
@@ -181,7 +181,7 @@ static void __kvm_gmem_invalidate_begin(struct gmem_file *f, pgoff_t start,
found_memslot = true;
KVM_MMU_LOCK(kvm);
- kvm_mmu_invalidate_begin(kvm);
+ kvm_mmu_invalidate_start(kvm);
}
flush |= kvm_mmu_unmap_gfn_range(kvm, &gfn_range);
@@ -194,7 +194,7 @@ static void __kvm_gmem_invalidate_begin(struct gmem_file *f, pgoff_t start,
KVM_MMU_UNLOCK(kvm);
}
-static void kvm_gmem_invalidate_begin(struct inode *inode, pgoff_t start,
+static void kvm_gmem_invalidate_start(struct inode *inode, pgoff_t start,
pgoff_t end)
{
enum kvm_gfn_range_filter attr_filter;
@@ -203,7 +203,7 @@ static void kvm_gmem_invalidate_begin(struct inode *inode, pgoff_t start,
attr_filter = kvm_gmem_get_invalidate_filter(inode);
kvm_gmem_for_each_file(f, inode)
- __kvm_gmem_invalidate_begin(f, start, end, attr_filter);
+ __kvm_gmem_invalidate_start(f, start, end, attr_filter);
}
static void __kvm_gmem_invalidate_end(struct gmem_file *f, pgoff_t start,
@@ -238,7 +238,7 @@ static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len)
*/
filemap_invalidate_lock(inode->i_mapping);
- kvm_gmem_invalidate_begin(inode, start, end);
+ kvm_gmem_invalidate_start(inode, start, end);
truncate_inode_pages_range(inode->i_mapping, offset, offset + len - 1);
@@ -352,7 +352,7 @@ static int kvm_gmem_release(struct inode *inode, struct file *file)
* Zap all SPTEs pointed at by this file. Do not free the backing
* memory, as its lifetime is associated with the inode, not the file.
*/
- __kvm_gmem_invalidate_begin(f, 0, -1ul,
+ __kvm_gmem_invalidate_start(f, 0, -1ul,
kvm_gmem_get_invalidate_filter(inode));
__kvm_gmem_invalidate_end(f, 0, -1ul);
@@ -438,11 +438,12 @@ static int kvm_gmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpo
}
static struct mempolicy *kvm_gmem_get_policy(struct vm_area_struct *vma,
- unsigned long addr, pgoff_t *pgoff)
+ unsigned long addr, pgoff_t *ilx)
{
+ pgoff_t pgoff = vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT);
struct inode *inode = file_inode(vma->vm_file);
- *pgoff = vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT);
+ *ilx = inode->i_ino;
/*
* Return the memory policy for this index, or NULL if none is set.
@@ -453,7 +454,7 @@ static struct mempolicy *kvm_gmem_get_policy(struct vm_area_struct *vma,
* can then replace NULL with the default memory policy instead of the
* current task's memory policy.
*/
- return mpol_shared_policy_lookup(&GMEM_I(inode)->policy, *pgoff);
+ return mpol_shared_policy_lookup(&GMEM_I(inode)->policy, pgoff);
}
#endif /* CONFIG_NUMA */
@@ -504,7 +505,7 @@ static int kvm_gmem_error_folio(struct address_space *mapping, struct folio *fol
start = folio->index;
end = start + folio_nr_pages(folio);
- kvm_gmem_invalidate_begin(mapping->host, start, end);
+ kvm_gmem_invalidate_start(mapping->host, start, end);
/*
* Do not truncate the range, what action is taken in response to the
@@ -640,15 +641,16 @@ int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args)
}
int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot,
- unsigned int fd, loff_t offset)
+ unsigned int fd, uoff_t offset)
{
- loff_t size = slot->npages << PAGE_SHIFT;
+ uoff_t size = slot->npages << PAGE_SHIFT;
unsigned long start, end;
struct gmem_file *f;
struct inode *inode;
struct file *file;
int r = -EINVAL;
+ BUILD_BUG_ON(sizeof(gpa_t) != sizeof(offset));
BUILD_BUG_ON(sizeof(gfn_t) != sizeof(slot->gmem.pgoff));
file = fget(fd);
@@ -664,8 +666,7 @@ int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot,
inode = file_inode(file);
- if (offset < 0 || !PAGE_ALIGNED(offset) ||
- offset + size > i_size_read(inode))
+ if (!PAGE_ALIGNED(offset) || offset + size > i_size_read(inode))
goto err;
filemap_invalidate_lock(inode->i_mapping);
@@ -675,6 +676,7 @@ int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot,
if (!xa_empty(&f->bindings) &&
xa_find(&f->bindings, &start, end - 1, XA_PRESENT)) {
+ r = -EEXIST;
filemap_invalidate_unlock(inode->i_mapping);
goto err;
}
@@ -858,7 +860,8 @@ out_unlock:
return ret;
}
-long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long npages,
+long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src,
+ long npages, bool may_writeback_src,
kvm_gmem_populate_cb post_populate, void *opaque)
{
struct kvm_memory_slot *slot;
@@ -892,8 +895,9 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long
if (src) {
unsigned long uaddr = (unsigned long)src + i * PAGE_SIZE;
+ unsigned int flags = may_writeback_src ? FOLL_WRITE : 0;
- ret = get_user_pages_fast(uaddr, 1, 0, &src_page);
+ ret = get_user_pages_fast(uaddr, 1, flags, &src_page);
if (ret < 0)
break;
if (ret != 1) {
@@ -973,8 +977,6 @@ static int kvm_gmem_init_fs_context(struct fs_context *fc)
if (!init_pseudo(fc, GUEST_MEMFD_MAGIC))
return -ENOMEM;
- fc->s_iflags |= SB_I_NOEXEC;
- fc->s_iflags |= SB_I_NODEV;
ctx = fc->fs_private;
ctx->ops = &kvm_gmem_super_operations;
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 89489996fbc1..e44c20c04961 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -670,7 +670,7 @@ static __always_inline bool kvm_age_hva_range_no_flush(struct mmu_notifier *mn,
return kvm_age_hva_range(mn, start, end, handler, false);
}
-void kvm_mmu_invalidate_begin(struct kvm *kvm)
+void kvm_mmu_invalidate_start(struct kvm *kvm)
{
lockdep_assert_held_write(&kvm->mmu_lock);
/*
@@ -726,7 +726,7 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
.start = range->start,
.end = range->end,
.handler = kvm_mmu_unmap_gfn_range,
- .on_lock = kvm_mmu_invalidate_begin,
+ .on_lock = kvm_mmu_invalidate_start,
.flush_on_ret = true,
.may_block = mmu_notifier_range_blockable(range),
};
@@ -2542,7 +2542,7 @@ static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
.end = end,
.arg.attributes = attributes,
.handler = kvm_pre_set_memory_attributes,
- .on_lock = kvm_mmu_invalidate_begin,
+ .on_lock = kvm_mmu_invalidate_start,
.flush_on_ret = true,
.may_block = true,
};
@@ -3520,7 +3520,8 @@ void mark_page_dirty_in_slot(struct kvm *kvm,
if (WARN_ON_ONCE(vcpu && vcpu->kvm != kvm))
return;
- WARN_ON_ONCE(!vcpu && !kvm_arch_allow_write_without_running_vcpu(kvm));
+ WARN_ON_ONCE(!vcpu && refcount_read(&kvm->users_count) &&
+ !kvm_arch_allow_write_without_running_vcpu(kvm));
#endif
if (memslot && kvm_slot_dirty_track_enabled(memslot)) {
diff --git a/virt/kvm/kvm_mm.h b/virt/kvm/kvm_mm.h
index 9fcc5d5b7f8d..7510ca915dd1 100644
--- a/virt/kvm/kvm_mm.h
+++ b/virt/kvm/kvm_mm.h
@@ -3,6 +3,9 @@
#ifndef __KVM_MM_H__
#define __KVM_MM_H__ 1
+#include <linux/kvm.h>
+#include <linux/kvm_types.h>
+
/*
* Architectures can choose whether to use an rwlock or spinlock
* for the mmu_lock. These macros, for use in common code
@@ -72,7 +75,7 @@ int kvm_gmem_init(struct module *module);
void kvm_gmem_exit(void);
int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args);
int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot,
- unsigned int fd, loff_t offset);
+ unsigned int fd, uoff_t offset);
void kvm_gmem_unbind(struct kvm_memory_slot *slot);
#else
static inline int kvm_gmem_init(struct module *module)
@@ -82,7 +85,7 @@ static inline int kvm_gmem_init(struct module *module)
static inline void kvm_gmem_exit(void) {};
static inline int kvm_gmem_bind(struct kvm *kvm,
struct kvm_memory_slot *slot,
- unsigned int fd, loff_t offset)
+ unsigned int fd, uoff_t offset)
{
WARN_ON_ONCE(1);
return -EIO;
diff --git a/virt/kvm/vfio.c b/virt/kvm/vfio.c
index 9f9acb66cc1e..6cdc4e9a333a 100644
--- a/virt/kvm/vfio.c
+++ b/virt/kvm/vfio.c
@@ -144,33 +144,26 @@ static int kvm_vfio_file_add(struct kvm_device *dev, unsigned int fd)
{
struct kvm_vfio *kv = dev->private;
struct kvm_vfio_file *kvf;
- struct file *filp;
- int ret = 0;
+ struct file *filp __free(fput) = NULL;
filp = fget(fd);
if (!filp)
return -EBADF;
/* Ensure the FD is a vfio FD. */
- if (!kvm_vfio_file_is_valid(filp)) {
- ret = -EINVAL;
- goto out_fput;
- }
+ if (!kvm_vfio_file_is_valid(filp))
+ return -EINVAL;
- mutex_lock(&kv->lock);
+ guard(mutex)(&kv->lock);
list_for_each_entry(kvf, &kv->file_list, node) {
- if (kvf->file == filp) {
- ret = -EEXIST;
- goto out_unlock;
- }
+ if (kvf->file == filp)
+ return -EEXIST;
}
kvf = kzalloc_obj(*kvf, GFP_KERNEL_ACCOUNT);
- if (!kvf) {
- ret = -ENOMEM;
- goto out_unlock;
- }
+ if (!kvf)
+ return -ENOMEM;
kvf->file = get_file(filp);
list_add_tail(&kvf->node, &kv->file_list);
@@ -178,11 +171,18 @@ static int kvm_vfio_file_add(struct kvm_device *dev, unsigned int fd)
kvm_vfio_file_set_kvm(kvf->file, dev->kvm);
kvm_vfio_update_coherency(dev);
-out_unlock:
- mutex_unlock(&kv->lock);
-out_fput:
- fput(filp);
- return ret;
+ return 0;
+}
+
+static void kvm_vfio_file_free(struct kvm_device *dev, struct kvm_vfio_file *kvf)
+{
+#ifdef CONFIG_SPAPR_TCE_IOMMU
+ kvm_spapr_tce_release_vfio_group(dev->kvm, kvf);
+#endif
+ kvm_vfio_file_set_kvm(kvf->file, NULL);
+ fput(kvf->file);
+ list_del(&kvf->node);
+ kfree(kvf);
}
static int kvm_vfio_file_del(struct kvm_device *dev, unsigned int fd)
@@ -190,34 +190,21 @@ static int kvm_vfio_file_del(struct kvm_device *dev, unsigned int fd)
struct kvm_vfio *kv = dev->private;
struct kvm_vfio_file *kvf;
CLASS(fd, f)(fd);
- int ret;
if (fd_empty(f))
return -EBADF;
- ret = -ENOENT;
-
- mutex_lock(&kv->lock);
+ guard(mutex)(&kv->lock);
list_for_each_entry(kvf, &kv->file_list, node) {
- if (kvf->file != fd_file(f))
- continue;
-
- list_del(&kvf->node);
-#ifdef CONFIG_SPAPR_TCE_IOMMU
- kvm_spapr_tce_release_vfio_group(dev->kvm, kvf);
-#endif
- kvm_vfio_file_set_kvm(kvf->file, NULL);
- fput(kvf->file);
- kfree(kvf);
- ret = 0;
- break;
+ if (kvf->file == fd_file(f)) {
+ kvm_vfio_file_free(dev, kvf);
+ kvm_vfio_update_coherency(dev);
+ return 0;
+ }
}
- kvm_vfio_update_coherency(dev);
-
- mutex_unlock(&kv->lock);
- return ret;
+ return -ENOENT;
}
#ifdef CONFIG_SPAPR_TCE_IOMMU
@@ -227,7 +214,6 @@ static int kvm_vfio_file_set_spapr_tce(struct kvm_device *dev,
struct kvm_vfio_spapr_tce param;
struct kvm_vfio *kv = dev->private;
struct kvm_vfio_file *kvf;
- int ret;
if (copy_from_user(&param, arg, sizeof(struct kvm_vfio_spapr_tce)))
return -EFAULT;
@@ -236,9 +222,7 @@ static int kvm_vfio_file_set_spapr_tce(struct kvm_device *dev,
if (fd_empty(f))
return -EBADF;
- ret = -ENOENT;
-
- mutex_lock(&kv->lock);
+ guard(mutex)(&kv->lock);
list_for_each_entry(kvf, &kv->file_list, node) {
if (kvf->file != fd_file(f))
@@ -246,20 +230,15 @@ static int kvm_vfio_file_set_spapr_tce(struct kvm_device *dev,
if (!kvf->iommu_group) {
kvf->iommu_group = kvm_vfio_file_iommu_group(kvf->file);
- if (WARN_ON_ONCE(!kvf->iommu_group)) {
- ret = -EIO;
- goto err_fdput;
- }
+ if (WARN_ON_ONCE(!kvf->iommu_group))
+ return -EIO;
}
- ret = kvm_spapr_tce_attach_iommu_group(dev->kvm, param.tablefd,
- kvf->iommu_group);
- break;
+ return kvm_spapr_tce_attach_iommu_group(dev->kvm, param.tablefd,
+ kvf->iommu_group);
}
-err_fdput:
- mutex_unlock(&kv->lock);
- return ret;
+ return -ENOENT;
}
#endif
@@ -326,15 +305,8 @@ static void kvm_vfio_release(struct kvm_device *dev)
struct kvm_vfio *kv = dev->private;
struct kvm_vfio_file *kvf, *tmp;
- list_for_each_entry_safe(kvf, tmp, &kv->file_list, node) {
-#ifdef CONFIG_SPAPR_TCE_IOMMU
- kvm_spapr_tce_release_vfio_group(dev->kvm, kvf);
-#endif
- kvm_vfio_file_set_kvm(kvf->file, NULL);
- fput(kvf->file);
- list_del(&kvf->node);
- kfree(kvf);
- }
+ list_for_each_entry_safe(kvf, tmp, &kv->file_list, node)
+ kvm_vfio_file_free(dev, kvf);
kvm_vfio_update_coherency(dev);