diff options
author | Andrea Arcangeli <aarcange@redhat.com> | 2015-09-04 15:46:48 -0700 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2015-09-04 16:54:41 -0700 |
commit | 3004ec9cabf49f43fae2b2bd1855a4720f1def7a (patch) | |
tree | d323151df9559ac13d28b84817a2c695fbeea17d /fs/userfaultfd.c | |
parent | 15b726ef048b31a24b3fefb6863083a25fe34800 (diff) | |
download | lwn-3004ec9cabf49f43fae2b2bd1855a4720f1def7a.tar.gz lwn-3004ec9cabf49f43fae2b2bd1855a4720f1def7a.zip |
userfaultfd: allocate the userfaultfd_ctx cacheline aligned
Use proper slab to guarantee alignment.
Signed-off-by: Andrea Arcangeli <aarcange@redhat.com>
Acked-by: Pavel Emelyanov <xemul@parallels.com>
Cc: Sanidhya Kashyap <sanidhya.gatech@gmail.com>
Cc: zhang.zhanghailiang@huawei.com
Cc: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Andres Lagar-Cavilla <andreslc@google.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Hugh Dickins <hughd@google.com>
Cc: Peter Feiner <pfeiner@google.com>
Cc: "Dr. David Alan Gilbert" <dgilbert@redhat.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: "Huangpeng (Peter)" <peter.huangpeng@huawei.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat (limited to 'fs/userfaultfd.c')
-rw-r--r-- | fs/userfaultfd.c | 39 |
1 files changed, 31 insertions, 8 deletions
diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 232cbf37c59f..8977a4e8a7f8 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -27,20 +27,26 @@ #include <linux/ioctl.h> #include <linux/security.h> +static struct kmem_cache *userfaultfd_ctx_cachep __read_mostly; + enum userfaultfd_state { UFFD_STATE_WAIT_API, UFFD_STATE_RUNNING, }; +/* + * Start with fault_pending_wqh and fault_wqh so they're more likely + * to be in the same cacheline. + */ struct userfaultfd_ctx { - /* pseudo fd refcounting */ - atomic_t refcount; /* waitqueue head for the pending (i.e. not read) userfaults */ wait_queue_head_t fault_pending_wqh; /* waitqueue head for the userfaults */ wait_queue_head_t fault_wqh; /* waitqueue head for the pseudo fd to wakeup poll/read */ wait_queue_head_t fd_wqh; + /* pseudo fd refcounting */ + atomic_t refcount; /* userfaultfd syscall flags */ unsigned int flags; /* state machine */ @@ -130,7 +136,7 @@ static void userfaultfd_ctx_put(struct userfaultfd_ctx *ctx) VM_BUG_ON(spin_is_locked(&ctx->fd_wqh.lock)); VM_BUG_ON(waitqueue_active(&ctx->fd_wqh)); mmput(ctx->mm); - kfree(ctx); + kmem_cache_free(userfaultfd_ctx_cachep, ctx); } } @@ -1028,6 +1034,15 @@ static const struct file_operations userfaultfd_fops = { .llseek = noop_llseek, }; +static void init_once_userfaultfd_ctx(void *mem) +{ + struct userfaultfd_ctx *ctx = (struct userfaultfd_ctx *) mem; + + init_waitqueue_head(&ctx->fault_pending_wqh); + init_waitqueue_head(&ctx->fault_wqh); + init_waitqueue_head(&ctx->fd_wqh); +} + /** * userfaultfd_file_create - Creates an userfaultfd file pointer. * @flags: Flags for the userfaultfd file. @@ -1058,14 +1073,11 @@ static struct file *userfaultfd_file_create(int flags) goto out; file = ERR_PTR(-ENOMEM); - ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); + ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL); if (!ctx) goto out; atomic_set(&ctx->refcount, 1); - init_waitqueue_head(&ctx->fault_pending_wqh); - init_waitqueue_head(&ctx->fault_wqh); - init_waitqueue_head(&ctx->fd_wqh); ctx->flags = flags; ctx->state = UFFD_STATE_WAIT_API; ctx->released = false; @@ -1076,7 +1088,7 @@ static struct file *userfaultfd_file_create(int flags) file = anon_inode_getfile("[userfaultfd]", &userfaultfd_fops, ctx, O_RDWR | (flags & UFFD_SHARED_FCNTL_FLAGS)); if (IS_ERR(file)) - kfree(ctx); + kmem_cache_free(userfaultfd_ctx_cachep, ctx); out: return file; } @@ -1105,3 +1117,14 @@ err_put_unused_fd: return error; } + +static int __init userfaultfd_init(void) +{ + userfaultfd_ctx_cachep = kmem_cache_create("userfaultfd_ctx_cache", + sizeof(struct userfaultfd_ctx), + 0, + SLAB_HWCACHE_ALIGN|SLAB_PANIC, + init_once_userfaultfd_ctx); + return 0; +} +__initcall(userfaultfd_init); |