diff options
author | Nick Piggin <npiggin@suse.de> | 2010-01-29 15:38:31 -0800 |
---|---|---|
committer | Thomas Gleixner <tglx@linutronix.de> | 2010-04-27 17:32:49 +0200 |
commit | 6f22d55b46fbf80b018009ece79f15b8582843e5 (patch) | |
tree | 4623783cdf5b9c59482617f0a3437a852dc52e24 | |
parent | 20d3da9e29645abfba13536a82cf04a4076f9ce6 (diff) | |
download | lwn-6f22d55b46fbf80b018009ece79f15b8582843e5.tar.gz lwn-6f22d55b46fbf80b018009ece79f15b8582843e5.zip |
fs-inode_lock-scale-10
Impelemnt lazy inode lru similarly to dcache. This should reduce inode list
lock acquisition (todo: measure).
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: John Stultz <johnstul@us.ibm.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
-rw-r--r-- | fs/fs-writeback.c | 21 | ||||
-rw-r--r-- | fs/inode.c | 61 | ||||
-rw-r--r-- | include/linux/fs.h | 7 | ||||
-rw-r--r-- | include/linux/writeback.h | 1 |
4 files changed, 42 insertions, 48 deletions
diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index 07d70704078e..df99a99f99a2 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -544,7 +544,7 @@ select_queue: /* * The inode is clean, inuse */ - list_move(&inode->i_list, &inode_in_use); + list_del_init(&inode->i_list); } else { /* * The inode is clean, unused @@ -1151,8 +1151,6 @@ static void wait_sb_inodes(struct super_block *sb) */ WARN_ON(!rwsem_is_locked(&sb->s_umount)); - spin_lock(&sb_inode_list_lock); - /* * Data integrity sync. Must wait for all pages under writeback, * because there may have been pages dirtied before our sync @@ -1160,7 +1158,8 @@ static void wait_sb_inodes(struct super_block *sb) * In which case, the inode may not be on the dirty list, but * we still have to wait for that writeout. */ - list_for_each_entry(inode, &sb->s_inodes, i_sb_list) { + rcu_read_lock(); + list_for_each_entry_rcu(inode, &sb->s_inodes, i_sb_list) { struct address_space *mapping; mapping = inode->i_mapping; @@ -1174,13 +1173,13 @@ static void wait_sb_inodes(struct super_block *sb) } __iget(inode); spin_unlock(&inode->i_lock); - spin_unlock(&sb_inode_list_lock); + rcu_read_unlock(); /* * We hold a reference to 'inode' so it couldn't have been - * removed from s_inodes list while we dropped the - * sb_inode_list_lock. We cannot iput the inode now as we can - * be holding the last reference and we cannot iput it under - * spinlock. So we keep the reference and iput it later. + * removed from s_inodes list while we dropped the i_lock. We + * cannot iput the inode now as we can be holding the last + * reference and we cannot iput it under spinlock. So we keep + * the reference and iput it later. */ iput(old_inode); old_inode = inode; @@ -1189,9 +1188,9 @@ static void wait_sb_inodes(struct super_block *sb) cond_resched(); - spin_lock(&sb_inode_list_lock); + rcu_read_lock(); } - spin_unlock(&sb_inode_list_lock); + rcu_read_unlock(); iput(old_inode); } diff --git a/fs/inode.c b/fs/inode.c index 1d7ca6ed7dbd..5039ddf4f0e0 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -74,7 +74,6 @@ static unsigned int i_hash_shift __read_mostly; * allowing for low-overhead inode sync() operations. */ -LIST_HEAD(inode_in_use); LIST_HEAD(inode_unused); struct inode_hash_bucket { @@ -266,6 +265,7 @@ void inode_init_once(struct inode *inode) INIT_HLIST_NODE(&inode->i_hash); INIT_LIST_HEAD(&inode->i_dentry); INIT_LIST_HEAD(&inode->i_devices); + INIT_LIST_HEAD(&inode->i_list); INIT_RADIX_TREE(&inode->i_data.page_tree, GFP_ATOMIC); spin_lock_init(&inode->i_data.tree_lock); spin_lock_init(&inode->i_data.i_mmap_lock); @@ -291,24 +291,6 @@ static void init_once(void *foo) inode_init_once(inode); } -/* - * inode_lock must be held - */ -void __iget(struct inode *inode) -{ - assert_spin_locked(&inode->i_lock); - inode->i_count++; - if (inode->i_count > 1) - return; - - if (!(inode->i_state & (I_DIRTY|I_SYNC))) { - spin_lock(&wb_inode_list_lock); - list_move(&inode->i_list, &inode_in_use); - spin_unlock(&wb_inode_list_lock); - } - atomic_dec(&inodes_stat.nr_unused); -} - /** * clear_inode - clear an inode * @inode: inode to clear @@ -352,7 +334,7 @@ static void dispose_list(struct list_head *head) struct inode *inode; inode = list_first_entry(head, struct inode, i_list); - list_del(&inode->i_list); + list_del_init(&inode->i_list); if (inode->i_data.nrpages) truncate_inode_pages(&inode->i_data, 0); @@ -405,11 +387,12 @@ static int invalidate_list(struct list_head *head, struct list_head *dispose) invalidate_inode_buffers(inode); if (!inode->i_count) { spin_lock(&wb_inode_list_lock); - list_move(&inode->i_list, dispose); + list_del(&inode->i_list); spin_unlock(&wb_inode_list_lock); WARN_ON(inode->i_state & I_NEW); inode->i_state |= I_FREEING; spin_unlock(&inode->i_lock); + list_add(&inode->i_list, dispose); count++; continue; } @@ -496,7 +479,13 @@ again: spin_unlock(&wb_inode_list_lock); goto again; } - if (inode->i_state || inode->i_count) { + if (inode->i_count) { + list_del_init(&inode->i_list); + spin_unlock(&inode->i_lock); + atomic_dec(&inodes_stat.nr_unused); + continue; + } + if (inode->i_state) { list_move(&inode->i_list, &inode_unused); spin_unlock(&inode->i_lock); continue; @@ -512,6 +501,7 @@ again: again2: spin_lock(&wb_inode_list_lock); + /* XXX: may no longer work well */ if (inode != list_entry(inode_unused.next, struct inode, i_list)) continue; /* wrong inode or list_empty */ @@ -660,9 +650,6 @@ __inode_add_to_lists(struct super_block *sb, struct inode_hash_bucket *b, atomic_inc(&inodes_stat.nr_inodes); list_add(&inode->i_sb_list, &sb->s_inodes); spin_unlock(&sb_inode_list_lock); - spin_lock(&wb_inode_list_lock); - list_add(&inode->i_list, &inode_in_use); - spin_unlock(&wb_inode_list_lock); if (b) { spin_lock(&b->lock); hlist_add_head(&inode->i_hash, &b->head); @@ -1311,9 +1298,11 @@ void generic_delete_inode(struct inode *inode) { const struct super_operations *op = inode->i_sb->s_op; - spin_lock(&wb_inode_list_lock); - list_del_init(&inode->i_list); - spin_unlock(&wb_inode_list_lock); + if (!list_empty(&inode->i_list)) { + spin_lock(&wb_inode_list_lock); + list_del_init(&inode->i_list); + spin_unlock(&wb_inode_list_lock); + } list_del_init(&inode->i_sb_list); spin_unlock(&sb_inode_list_lock); WARN_ON(inode->i_state & I_NEW); @@ -1365,12 +1354,12 @@ int generic_detach_inode(struct inode *inode) struct super_block *sb = inode->i_sb; if (!hlist_unhashed(&inode->i_hash)) { - if (!(inode->i_state & (I_DIRTY|I_SYNC))) { + if (list_empty(&inode->i_list)) { spin_lock(&wb_inode_list_lock); - list_move(&inode->i_list, &inode_unused); + list_add(&inode->i_list, &inode_unused); spin_unlock(&wb_inode_list_lock); + atomic_inc(&inodes_stat.nr_unused); } - atomic_inc(&inodes_stat.nr_unused); if (sb->s_flags & MS_ACTIVE) { spin_unlock(&inode->i_lock); spin_unlock(&sb_inode_list_lock); @@ -1386,11 +1375,13 @@ int generic_detach_inode(struct inode *inode) WARN_ON(inode->i_state & I_NEW); inode->i_state &= ~I_WILL_FREE; __remove_inode_hash(inode); + } + if (!list_empty(&inode->i_list)) { + spin_lock(&wb_inode_list_lock); + list_del_init(&inode->i_list); + spin_unlock(&wb_inode_list_lock); atomic_dec(&inodes_stat.nr_unused); } - spin_lock(&wb_inode_list_lock); - list_del_init(&inode->i_list); - spin_unlock(&wb_inode_list_lock); list_del_init(&inode->i_sb_list); spin_unlock(&sb_inode_list_lock); WARN_ON(inode->i_state & I_NEW); @@ -1726,7 +1717,7 @@ void __init inode_init(void) inode_hashtable = alloc_large_system_hash("Inode-cache", - sizeof(struct hlist_head), + sizeof(struct inode_hash_bucket), ihash_entries, 14, 0, diff --git a/include/linux/fs.h b/include/linux/fs.h index 3551528a25f6..b86542c8d68c 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2174,7 +2174,6 @@ extern int insert_inode_locked4(struct inode *, unsigned long, int (*test)(struc extern int insert_inode_locked(struct inode *); extern void unlock_new_inode(struct inode *); -extern void __iget(struct inode * inode); extern void iget_failed(struct inode *); extern void clear_inode(struct inode *); extern void destroy_inode(struct inode *); @@ -2393,6 +2392,12 @@ extern int generic_show_options(struct seq_file *m, struct vfsmount *mnt); extern void save_mount_options(struct super_block *sb, char *options); extern void replace_mount_options(struct super_block *sb, char *options); +static inline void __iget(struct inode *inode) +{ + assert_spin_locked(&inode->i_lock); + inode->i_count++; +} + static inline ino_t parent_ino(struct dentry *dentry) { ino_t res; diff --git a/include/linux/writeback.h b/include/linux/writeback.h index 05479e576a77..15e8bcd90cd1 100644 --- a/include/linux/writeback.h +++ b/include/linux/writeback.h @@ -11,7 +11,6 @@ struct backing_dev_info; extern spinlock_t sb_inode_list_lock; extern spinlock_t wb_inode_list_lock; -extern struct list_head inode_in_use; extern struct list_head inode_unused; /* |